LCOV - code coverage report
Current view: top level - gcc/config/i386 - i386-expand.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 85.4 % 16838 14384
Test Date: 2026-11-14 16:39:50 Functions: 93.9 % 297 279
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Copyright (C) 1988-2026 Free Software Foundation, Inc.
       2              : 
       3              : This file is part of GCC.
       4              : 
       5              : GCC is free software; you can redistribute it and/or modify
       6              : it under the terms of the GNU General Public License as published by
       7              : the Free Software Foundation; either version 3, or (at your option)
       8              : any later version.
       9              : 
      10              : GCC is distributed in the hope that it will be useful,
      11              : but WITHOUT ANY WARRANTY; without even the implied warranty of
      12              : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      13              : GNU General Public License for more details.
      14              : 
      15              : You should have received a copy of the GNU General Public License
      16              : along with GCC; see the file COPYING3.  If not see
      17              : <http://www.gnu.org/licenses/>.  */
      18              : 
      19              : #define IN_TARGET_CODE 1
      20              : 
      21              : #include "config.h"
      22              : #include "system.h"
      23              : #include "coretypes.h"
      24              : #include "backend.h"
      25              : #include "rtl.h"
      26              : #include "tree.h"
      27              : #include "memmodel.h"
      28              : #include "gimple.h"
      29              : #include "cfghooks.h"
      30              : #include "cfgloop.h"
      31              : #include "df.h"
      32              : #include "tm_p.h"
      33              : #include "stringpool.h"
      34              : #include "expmed.h"
      35              : #include "optabs.h"
      36              : #include "regs.h"
      37              : #include "emit-rtl.h"
      38              : #include "recog.h"
      39              : #include "cgraph.h"
      40              : #include "diagnostic.h"
      41              : #include "cfgbuild.h"
      42              : #include "alias.h"
      43              : #include "fold-const.h"
      44              : #include "attribs.h"
      45              : #include "calls.h"
      46              : #include "stor-layout.h"
      47              : #include "varasm.h"
      48              : #include "output.h"
      49              : #include "insn-attr.h"
      50              : #include "flags.h"
      51              : #include "except.h"
      52              : #include "explow.h"
      53              : #include "expr.h"
      54              : #include "cfgrtl.h"
      55              : #include "common/common-target.h"
      56              : #include "langhooks.h"
      57              : #include "reload.h"
      58              : #include "gimplify.h"
      59              : #include "dwarf2.h"
      60              : #include "tm-constrs.h"
      61              : #include "cselib.h"
      62              : #include "sched-int.h"
      63              : #include "opts.h"
      64              : #include "tree-pass.h"
      65              : #include "context.h"
      66              : #include "pass_manager.h"
      67              : #include "target-globals.h"
      68              : #include "gimple-iterator.h"
      69              : #include "shrink-wrap.h"
      70              : #include "builtins.h"
      71              : #include "rtl-iter.h"
      72              : #include "tree-iterator.h"
      73              : #include "dbgcnt.h"
      74              : #include "case-cfn-macros.h"
      75              : #include "dojump.h"
      76              : #include "fold-const-call.h"
      77              : #include "tree-vrp.h"
      78              : #include "tree-ssanames.h"
      79              : #include "selftest.h"
      80              : #include "selftest-rtl.h"
      81              : #include "print-rtl.h"
      82              : #include "intl.h"
      83              : #include "ifcvt.h"
      84              : #include "symbol-summary.h"
      85              : #include "sreal.h"
      86              : #include "ipa-cp.h"
      87              : #include "ipa-prop.h"
      88              : #include "ipa-fnsummary.h"
      89              : #include "wide-int-bitmask.h"
      90              : #include "tree-vector-builder.h"
      91              : #include "debug.h"
      92              : #include "dwarf2out.h"
      93              : #include "i386-options.h"
      94              : #include "i386-builtins.h"
      95              : #include "i386-expand.h"
      96              : #include "asan.h"
      97              : #include "function-abi.h"
      98              : 
      99              : /* Split one or more double-mode RTL references into pairs of half-mode
     100              :    references.  The RTL can be REG, offsettable MEM, integer constant, or
     101              :    CONST_DOUBLE.  "operands" is a pointer to an array of double-mode RTLs to
     102              :    split and "num" is its length.  lo_half and hi_half are output arrays
     103              :    that parallel "operands".  */
     104              : 
     105              : void
     106      4306083 : split_double_mode (machine_mode mode, rtx operands[],
     107              :                    int num, rtx lo_half[], rtx hi_half[])
     108              : {
     109      4306083 :   machine_mode half_mode;
     110      4306083 :   unsigned int byte;
     111      4306083 :   rtx mem_op = NULL_RTX;
     112      4306083 :   int mem_num = 0;
     113              : 
     114      4306083 :   switch (mode)
     115              :     {
     116              :     case E_TImode:
     117              :       half_mode = DImode;
     118              :       break;
     119       660318 :     case E_DImode:
     120       660318 :       half_mode = SImode;
     121       660318 :       break;
     122            6 :     case E_P2HImode:
     123            6 :       half_mode = HImode;
     124            6 :       break;
     125           30 :     case E_P2QImode:
     126           30 :       half_mode = QImode;
     127           30 :       break;
     128            0 :     default:
     129            0 :       gcc_unreachable ();
     130              :     }
     131              : 
     132      4306083 :   byte = GET_MODE_SIZE (half_mode);
     133              : 
     134      8858203 :   while (num--)
     135              :     {
     136      4552120 :       rtx op = operands[num];
     137              : 
     138              :       /* simplify_subreg refuse to split volatile memory addresses,
     139              :          but we still have to handle it.  */
     140      4552120 :       if (MEM_P (op))
     141              :         {
     142      1775944 :           if (mem_op && rtx_equal_p (op, mem_op))
     143              :             {
     144         2307 :               lo_half[num] = lo_half[mem_num];
     145         2307 :               hi_half[num] = hi_half[mem_num];
     146              :             }
     147              :           else
     148              :             {
     149      1773637 :               mem_op = op;
     150      1773637 :               mem_num = num;
     151      1773637 :               lo_half[num] = adjust_address (op, half_mode, 0);
     152      1773637 :               hi_half[num] = adjust_address (op, half_mode, byte);
     153              :             }
     154              :         }
     155              :       else
     156              :         {
     157      2776176 :           lo_half[num] = simplify_gen_subreg (half_mode, op,
     158      2776176 :                                               GET_MODE (op) == VOIDmode
     159              :                                               ? mode : GET_MODE (op), 0);
     160              : 
     161      2776176 :           rtx tmp = simplify_gen_subreg (half_mode, op,
     162      2776176 :                                          GET_MODE (op) == VOIDmode
     163      2776176 :                                          ? mode : GET_MODE (op), byte);
     164              :           /* simplify_gen_subreg will return NULL RTX for the
     165              :              high half of the paradoxical subreg.  */
     166      2776176 :           if (tmp)
     167      2776175 :             hi_half[num] = tmp;
     168            1 :           else if (can_create_pseudo_p ())
     169            0 :             hi_half[num] = gen_reg_rtx (half_mode);
     170              :           else
     171            1 :             hi_half[num] = CONST0_RTX (half_mode);
     172              :         }
     173              :     }
     174      4306083 : }
     175              : 
     176              : /* Emit the double word assignment DST = { LO, HI }.  */
     177              : 
     178              : void
     179       106091 : split_double_concat (machine_mode mode, rtx dst, rtx lo, rtx hi)
     180              : {
     181       106091 :   rtx dlo, dhi;
     182       106091 :   int deleted_move_count = 0;
     183       106091 :   split_double_mode (mode, &dst, 1, &dlo, &dhi);
     184              :   /* Constraints ensure that if both lo and hi are MEMs, then
     185              :      dst has early-clobber and thus addresses of MEMs don't use
     186              :      dlo/dhi registers.  Otherwise if at least one of li and hi are MEMs,
     187              :      dlo/dhi are registers.  */
     188       106091 :   if (MEM_P (lo)
     189         5707 :       && rtx_equal_p (dlo, hi)
     190       107066 :       && reg_overlap_mentioned_p (dhi, lo))
     191              :     {
     192              :       /* If dlo is same as hi and lo's address uses dhi register,
     193              :          code below would first emit_move_insn (dhi, hi)
     194              :          and then emit_move_insn (dlo, lo).  But the former
     195              :          would invalidate lo's address.  Load into dhi first,
     196              :          then swap.  */
     197          187 :       emit_move_insn (dhi, lo);
     198          187 :       lo = dhi;
     199              :     }
     200       105904 :   else if (MEM_P (hi)
     201         9746 :            && !MEM_P (lo)
     202         6904 :            && !rtx_equal_p (dlo, lo)
     203       107462 :            && reg_overlap_mentioned_p (dlo, hi))
     204              :     {
     205              :       /* In this case, code below would first emit_move_insn (dlo, lo)
     206              :          and then emit_move_insn (dhi, hi).  But the former would
     207              :          invalidate hi's address.  */
     208           14 :       if (rtx_equal_p (dhi, lo))
     209              :         {
     210              :           /* We can't load into dhi first, so load into dlo
     211              :              first and we'll swap.  */
     212            5 :           emit_move_insn (dlo, hi);
     213            5 :           hi = dlo;
     214              :         }
     215              :       else
     216              :         {
     217              :           /* Load into dhi first.  */
     218            9 :           emit_move_insn (dhi, hi);
     219            9 :           hi = dhi;
     220              :         }
     221              :     }
     222       106091 :   if (!rtx_equal_p (dlo, hi))
     223              :     {
     224        90552 :       if (!rtx_equal_p (dlo, lo))
     225        37973 :         emit_move_insn (dlo, lo);
     226              :       else
     227              :         deleted_move_count++;
     228        90552 :       if (!rtx_equal_p (dhi, hi))
     229        83665 :         emit_move_insn (dhi, hi);
     230              :       else
     231         6887 :         deleted_move_count++;
     232              :     }
     233        15539 :   else if (!rtx_equal_p (lo, dhi))
     234              :     {
     235         8175 :       if (!rtx_equal_p (dhi, hi))
     236         8175 :         emit_move_insn (dhi, hi);
     237              :       else
     238              :         deleted_move_count++;
     239         8175 :       if (!rtx_equal_p (dlo, lo))
     240         8069 :         emit_move_insn (dlo, lo);
     241              :       else
     242          106 :         deleted_move_count++;
     243              :     }
     244         7364 :   else if (mode == TImode)
     245         7343 :     emit_insn (gen_swapdi (dlo, dhi));
     246              :   else
     247           21 :     emit_insn (gen_swapsi (dlo, dhi));
     248              : 
     249       106091 :   if (deleted_move_count == 2)
     250         3170 :     emit_note (NOTE_INSN_DELETED);
     251       106091 : }
     252              : 
     253              : 
     254              : /* Generate either "mov $0, reg" or "xor reg, reg", as appropriate
     255              :    for the target.  */
     256              : 
     257              : void
     258       129534 : ix86_expand_clear (rtx dest)
     259              : {
     260       129534 :   rtx tmp;
     261              : 
     262              :   /* We play register width games, which are only valid after reload.  */
     263       129534 :   gcc_assert (reload_completed);
     264              : 
     265              :   /* Avoid HImode and its attendant prefix byte.  */
     266       259068 :   if (GET_MODE_SIZE (GET_MODE (dest)) < 4)
     267         1028 :     dest = gen_rtx_REG (SImode, REGNO (dest));
     268       129534 :   tmp = gen_rtx_SET (dest, const0_rtx);
     269              : 
     270       129534 :   if (!TARGET_USE_MOV0 || optimize_insn_for_size_p ())
     271              :     {
     272       129534 :       rtx clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
     273       129534 :       tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, clob));
     274              :     }
     275              : 
     276       129534 :   emit_insn (tmp);
     277       129534 : }
     278              : 
     279              : /* Return true if V can be broadcasted from an integer of WIDTH bits
     280              :    which is returned in VAL_BROADCAST.  Otherwise, return false.  */
     281              : 
     282              : static bool
     283         4875 : ix86_broadcast (HOST_WIDE_INT v, unsigned int width,
     284              :                 HOST_WIDE_INT &val_broadcast)
     285              : {
     286         4875 :   wide_int val = wi::uhwi (v, HOST_BITS_PER_WIDE_INT);
     287         4875 :   val_broadcast = wi::extract_uhwi (val, 0, width);
     288         6578 :   for (unsigned int i = width; i < HOST_BITS_PER_WIDE_INT; i += width)
     289              :     {
     290         5117 :       HOST_WIDE_INT each = wi::extract_uhwi (val, i, width);
     291         5117 :       if (val_broadcast != each)
     292              :         return false;
     293              :     }
     294         1461 :   val_broadcast = sext_hwi (val_broadcast, width);
     295         1461 :   return true;
     296         4875 : }
     297              : 
     298              : /* Convert the CONST_WIDE_INT operand OP to broadcast in MODE.  */
     299              : 
     300              : rtx
     301        36643 : ix86_convert_const_wide_int_to_broadcast (machine_mode mode, rtx op)
     302              : {
     303              :   /* Don't use integer vector broadcast if we can't move from GPR to SSE
     304              :      register directly.  */
     305        36643 :   if (!TARGET_INTER_UNIT_MOVES_TO_VEC)
     306              :     return nullptr;
     307              : 
     308        36643 :   unsigned int msize = GET_MODE_SIZE (mode);
     309              : 
     310              :   /* Only optimized for vpbroadcast[bwsd]/vbroadcastss with xmm/ymm/zmm.  */
     311        36643 :   if (msize != 16 && msize != 32 && msize != 64)
     312              :     return nullptr;
     313              : 
     314              :   /* Convert CONST_WIDE_INT to a non-standard SSE constant integer
     315              :      broadcast only if vector broadcast is available.  */
     316        36643 :   if (!TARGET_AVX
     317         1674 :       || !CONST_WIDE_INT_P (op)
     318         1611 :       || standard_sse_constant_p (op, mode)
     319        38254 :       || (CONST_WIDE_INT_NUNITS (op) * HOST_BITS_PER_WIDE_INT
     320         1611 :           != GET_MODE_BITSIZE (mode)))
     321              :     return nullptr;
     322              : 
     323         1603 :   HOST_WIDE_INT val = CONST_WIDE_INT_ELT (op, 0);
     324         1603 :   HOST_WIDE_INT val_broadcast;
     325         1603 :   scalar_int_mode broadcast_mode;
     326              :   /* vpbroadcastb zmm requires TARGET_AVX512BW.  */
     327          716 :   if ((msize == 64 ? TARGET_AVX512BW : TARGET_AVX2)
     328         2100 :       && ix86_broadcast (val, GET_MODE_BITSIZE (QImode),
     329              :                          val_broadcast))
     330              :     broadcast_mode = QImode;
     331          658 :   else if ((msize == 64 ? TARGET_AVX512BW : TARGET_AVX2)
     332         1979 :            && ix86_broadcast (val, GET_MODE_BITSIZE (HImode),
     333              :                               val_broadcast))
     334              :     broadcast_mode = HImode;
     335              :   /* vbroadcasts[sd] only support memory operand w/o AVX2.
     336              :      When msize == 16, pshufs is used for vec_duplicate.
     337              :      when msize == 64, vpbroadcastd is used, and TARGET_AVX512F must be existed.  */
     338          416 :   else if ((msize != 32 || TARGET_AVX2)
     339         1779 :            && ix86_broadcast (val, GET_MODE_BITSIZE (SImode),
     340              :                            val_broadcast))
     341              :     broadcast_mode = SImode;
     342         1399 :   else if (TARGET_64BIT && (msize != 32 || TARGET_AVX2)
     343         2656 :            && ix86_broadcast (val, GET_MODE_BITSIZE (DImode),
     344              :                               val_broadcast))
     345              :     broadcast_mode = DImode;
     346              :   else
     347              :     return nullptr;
     348              : 
     349              :   /* Check if OP can be broadcasted from VAL.  */
     350         1783 :   for (int i = 1; i < CONST_WIDE_INT_NUNITS (op); i++)
     351         1568 :     if (val != CONST_WIDE_INT_ELT (op, i))
     352              :       return nullptr;
     353              : 
     354          215 :   unsigned int nunits = (GET_MODE_SIZE (mode)
     355          215 :                          / GET_MODE_SIZE (broadcast_mode));
     356          215 :   machine_mode vector_mode;
     357          215 :   if (!mode_for_vector (broadcast_mode, nunits).exists (&vector_mode))
     358            0 :     gcc_unreachable ();
     359          215 :   rtx target = gen_reg_rtx (vector_mode);
     360          215 :   bool ok = ix86_expand_vector_init_duplicate (false, vector_mode,
     361              :                                                target,
     362              :                                                GEN_INT (val_broadcast));
     363          215 :   if (!ok)
     364              :     return nullptr;
     365          215 :   target = lowpart_subreg (mode, target, vector_mode);
     366          215 :   return target;
     367              : }
     368              : 
     369              : void
     370     75627176 : ix86_expand_move (machine_mode mode, rtx operands[])
     371              : {
     372     75627176 :   rtx op0, op1;
     373     75627176 :   rtx tmp, addend = NULL_RTX;
     374     75627176 :   enum tls_model model;
     375              : 
     376     75627176 :   op0 = operands[0];
     377     75627176 :   op1 = operands[1];
     378              : 
     379              :   /* Avoid complex sets of likely spilled hard registers before reload.  */
     380     75627176 :   if (!ix86_hardreg_mov_ok (op0, op1))
     381              :     {
     382       141258 :       tmp = gen_reg_rtx (mode);
     383       141258 :       operands[0] = tmp;
     384       141258 :       ix86_expand_move (mode, operands);
     385       141258 :       operands[0] = op0;
     386       141258 :       operands[1] = tmp;
     387       141258 :       op1 = tmp;
     388              :     }
     389              : 
     390     75627176 :   switch (GET_CODE (op1))
     391              :     {
     392       355288 :     case CONST:
     393       355288 :       tmp = XEXP (op1, 0);
     394              : 
     395       355288 :       if (GET_CODE (tmp) != PLUS
     396       343555 :           || !SYMBOL_REF_P (XEXP (tmp, 0)))
     397              :         break;
     398              : 
     399       340881 :       op1 = XEXP (tmp, 0);
     400       340881 :       addend = XEXP (tmp, 1);
     401              :       /* FALLTHRU */
     402              : 
     403      5161844 :     case SYMBOL_REF:
     404      5161844 :       model = SYMBOL_REF_TLS_MODEL (op1);
     405              : 
     406      5161844 :       if (model)
     407        10200 :         op1 = legitimize_tls_address (op1, model, true);
     408      5151644 :       else if (ix86_force_load_from_GOT_p (op1))
     409              :         {
     410              :           /* Load the external function address via GOT slot to avoid PLT.  */
     411           24 :           op1 = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, op1),
     412              :                                 (TARGET_64BIT
     413              :                                  ? UNSPEC_GOTPCREL
     414              :                                  : UNSPEC_GOT));
     415           24 :           op1 = gen_rtx_CONST (Pmode, op1);
     416           24 :           op1 = gen_const_mem (Pmode, op1);
     417           20 :           set_mem_alias_set (op1, GOT_ALIAS_SET);
     418              :         }
     419              :       else
     420              :         {
     421              : #if TARGET_PECOFF
     422              :           tmp = legitimize_pe_coff_symbol (op1, addend != NULL_RTX);
     423              : 
     424              :           if (tmp)
     425              :             {
     426              :               op1 = tmp;
     427              :               if (!addend)
     428              :                 break;
     429              :             }
     430              :           else
     431              : #endif
     432      5151624 :             {
     433      5151624 :               op1 = operands[1];
     434      5151624 :               break;
     435              :             }
     436              :         }
     437              : 
     438        10220 :       if (addend)
     439              :         {
     440         2787 :           op1 = force_operand (op1, NULL_RTX);
     441         2796 :           op1 = expand_simple_binop (Pmode, PLUS, op1, addend,
     442              :                                      op0, 1, OPTAB_DIRECT);
     443              :         }
     444              :       else
     445         7433 :         op1 = force_operand (op1, op0);
     446              : 
     447        10220 :       if (op1 == op0)
     448              :         return;
     449              : 
     450         1152 :       op1 = convert_to_mode (mode, op1, 1);
     451              : 
     452              :     default:
     453              :       break;
     454              : 
     455      1609270 :     case SUBREG:
     456              :       /* Transform TImode paradoxical SUBREG into zero_extendditi2.  */
     457      1609270 :       if (TARGET_64BIT
     458      1367124 :           && mode == TImode
     459              :           && SUBREG_P (op1)
     460        74382 :           && GET_MODE (SUBREG_REG (op1)) == DImode
     461      1656088 :           && SUBREG_BYTE (op1) == 0)
     462        46818 :         op1 = gen_rtx_ZERO_EXTEND (TImode, SUBREG_REG (op1));
     463              :       /* As not all values in XFmode are representable in real_value,
     464              :          we might be called with unfoldable SUBREGs of constants.  */
     465      1609270 :       if (mode == XFmode
     466         3186 :           && CONSTANT_P (SUBREG_REG (op1))
     467            0 :           && can_create_pseudo_p ())
     468              :         {
     469            0 :           machine_mode imode = GET_MODE (SUBREG_REG (op1));
     470            0 :           rtx r = force_const_mem (imode, SUBREG_REG (op1));
     471            0 :           if (r)
     472            0 :             r = validize_mem (r);
     473              :           else
     474            0 :             r = force_reg (imode, SUBREG_REG (op1));
     475            0 :           op1 = simplify_gen_subreg (mode, r, imode, SUBREG_BYTE (op1));
     476              :         }
     477              :       break;
     478              :     }
     479              : 
     480     75618108 :   if ((flag_pic || MACHOPIC_INDIRECT)
     481     75618108 :       && symbolic_operand (op1, mode))
     482              :     {
     483              : #if TARGET_MACHO
     484              :       if (TARGET_MACHO && !TARGET_64BIT)
     485              :         {
     486              :           /* dynamic-no-pic */
     487              :           if (MACHOPIC_INDIRECT)
     488              :             {
     489              :               tmp = (op0 && REG_P (op0) && mode == Pmode)
     490              :                     ? op0 : gen_reg_rtx (Pmode);
     491              :               op1 = machopic_indirect_data_reference (op1, tmp);
     492              :               if (MACHOPIC_PURE)
     493              :                 op1 = machopic_legitimize_pic_address (op1, mode,
     494              :                                                        tmp == op1 ? 0 : tmp);
     495              :             }
     496              :           if (op0 != op1 && !MEM_P (op0))
     497              :             {
     498              :               rtx insn = gen_rtx_SET (op0, op1);
     499              :               emit_insn (insn);
     500              :               return;
     501              :             }
     502              :         }
     503              : #endif
     504              : 
     505       716747 :       if (MEM_P (op0))
     506       101825 :         op1 = force_reg (mode, op1);
     507       614922 :       else if (!(TARGET_64BIT && x86_64_movabs_operand (op1, DImode)))
     508              :         {
     509       614865 :           rtx reg = can_create_pseudo_p () ? NULL_RTX : op0;
     510       614865 :           op1 = legitimize_pic_address (op1, reg);
     511       614865 :           if (op0 == op1)
     512              :             return;
     513       614865 :           op1 = convert_to_mode (mode, op1, 1);
     514              :         }
     515              :     }
     516              :   else
     517              :     {
     518     74901361 :       if (MEM_P (op0)
     519    102256028 :           && (PUSH_ROUNDING (GET_MODE_SIZE (mode)) != GET_MODE_SIZE (mode)
     520     10978671 :               || !push_operand (op0, mode))
     521     87569873 :           && MEM_P (op1))
     522      2158821 :         op1 = force_reg (mode, op1);
     523              : 
     524     74901361 :       if (push_operand (op0, mode)
     525     74901361 :           && ! general_no_elim_operand (op1, mode))
     526         1007 :         op1 = copy_to_mode_reg (mode, op1);
     527              : 
     528              :       /* Force large constants in 64bit compilation into register
     529              :          to get them CSEed.  */
     530     74901361 :       if (can_create_pseudo_p ()
     531     69099971 :           && (mode == DImode) && TARGET_64BIT
     532     36186759 :           && immediate_operand (op1, mode)
     533      7900934 :           && !x86_64_zext_immediate_operand (op1, VOIDmode)
     534       736754 :           && !register_operand (op0, mode)
     535     75080348 :           && optimize)
     536       126541 :         op1 = copy_to_mode_reg (mode, op1);
     537              : 
     538     74901361 :       if (can_create_pseudo_p ())
     539              :         {
     540     69099971 :           if (CONST_DOUBLE_P (op1))
     541              :             {
     542              :               /* If we are loading a floating point constant to a
     543              :                  register, force the value to memory now, since we'll
     544              :                  get better code out the back end.  */
     545              : 
     546       911466 :               op1 = validize_mem (force_const_mem (mode, op1));
     547       911466 :               if (!register_operand (op0, mode))
     548              :                 {
     549       133061 :                   tmp = gen_reg_rtx (mode);
     550       133061 :                   emit_insn (gen_rtx_SET (tmp, op1));
     551       133061 :                   emit_move_insn (op0, tmp);
     552       133061 :                   return;
     553              :                 }
     554              :             }
     555              :         }
     556              :     }
     557              : 
     558              :   /* Special case inserting 64-bit values into a TImode register.  */
     559     75485047 :   if (TARGET_64BIT
     560              :       /* Disable for -O0 (see PR110587) unless naked (PR110533).  */
     561     65599762 :       && (optimize || ix86_function_naked (current_function_decl))
     562     45067629 :       && (mode == DImode || mode == DFmode)
     563     30842773 :       && SUBREG_P (op0)
     564       508987 :       && GET_MODE (SUBREG_REG (op0)) == TImode
     565       417192 :       && REG_P (SUBREG_REG (op0))
     566     75902239 :       && REG_P (op1))
     567              :     {
     568              :       /* Use *insvti_lowpart_1 to set lowpart.  */
     569       187774 :       if (SUBREG_BYTE (op0) == 0)
     570              :         {
     571        57244 :           wide_int mask = wi::mask (64, true, 128);
     572        57244 :           tmp = immed_wide_int_const (mask, TImode);
     573        57244 :           op0 = SUBREG_REG (op0);
     574        57244 :           tmp = gen_rtx_AND (TImode, copy_rtx (op0), tmp);
     575        57244 :           if (mode == DFmode)
     576          372 :             op1 = gen_lowpart (DImode, op1);
     577        57244 :           op1 = gen_rtx_ZERO_EXTEND (TImode, op1);
     578        57244 :           op1 = gen_rtx_IOR (TImode, tmp, op1);
     579        57244 :         }
     580              :       /* Use *insvti_highpart_1 to set highpart.  */
     581       130530 :       else if (SUBREG_BYTE (op0) == 8)
     582              :         {
     583       130530 :           wide_int mask = wi::mask (64, false, 128);
     584       130530 :           tmp = immed_wide_int_const (mask, TImode);
     585       130530 :           op0 = SUBREG_REG (op0);
     586       130530 :           tmp = gen_rtx_AND (TImode, copy_rtx (op0), tmp);
     587       130530 :           if (mode == DFmode)
     588          226 :             op1 = gen_lowpart (DImode, op1);
     589       130530 :           op1 = gen_rtx_ZERO_EXTEND (TImode, op1);
     590       130530 :           op1 = gen_rtx_ASHIFT (TImode, op1, GEN_INT (64));
     591       130530 :           op1 = gen_rtx_IOR (TImode, tmp, op1);
     592       130530 :         }
     593              :     }
     594              : 
     595     75485047 :   emit_insn (gen_rtx_SET (op0, op1));
     596              : }
     597              : 
     598              : /* OP is a memref of CONST_VECTOR, return scalar constant mem
     599              :    if CONST_VECTOR is a vec_duplicate, else return NULL.  */
     600              : rtx
     601      2654036 : ix86_broadcast_from_constant (machine_mode mode, rtx op)
     602              : {
     603      2654036 :   int nunits = GET_MODE_NUNITS (mode);
     604      2654036 :   if (nunits < 2)
     605              :     return nullptr;
     606              : 
     607              :   /* Don't use integer vector broadcast if we can't move from GPR to SSE
     608              :      register directly.  */
     609      2519976 :   if (!TARGET_INTER_UNIT_MOVES_TO_VEC
     610         8030 :       && INTEGRAL_MODE_P (mode))
     611              :     return nullptr;
     612              : 
     613              :   /* Convert CONST_VECTOR to a non-standard SSE constant integer
     614              :      broadcast only if vector broadcast is available.  */
     615      2514635 :   if (standard_sse_constant_p (op, mode))
     616              :     return nullptr;
     617              : 
     618      5029270 :   if (GET_MODE_INNER (mode) == TImode)
     619              :     return nullptr;
     620              : 
     621      2514525 :   rtx constant = get_pool_constant (XEXP (op, 0));
     622      2514525 :   if (!CONST_VECTOR_P (constant))
     623              :     return nullptr;
     624              : 
     625              :   /* There could be some rtx like
     626              :      (mem/u/c:V16QI (symbol_ref/u:DI ("*.LC1")))
     627              :      but with "*.LC1" refer to V2DI constant vector.  */
     628      2514525 :   if (GET_MODE (constant) != mode)
     629              :     {
     630          776 :       constant = simplify_subreg (mode, constant, GET_MODE (constant),
     631              :                                   0);
     632          776 :       if (constant == nullptr || !CONST_VECTOR_P (constant))
     633              :         return nullptr;
     634              :     }
     635              : 
     636      2514525 :   rtx first = XVECEXP (constant, 0, 0);
     637              : 
     638      8679868 :   for (int i = 1; i < nunits; ++i)
     639              :     {
     640      7967304 :       rtx tmp = XVECEXP (constant, 0, i);
     641              :       /* Vector duplicate value.  */
     642      7967304 :       if (!rtx_equal_p (tmp, first))
     643              :         return nullptr;
     644              :     }
     645              : 
     646              :   return first;
     647              : }
     648              : 
     649              : void
     650      4742844 : ix86_expand_vector_move (machine_mode mode, rtx operands[])
     651              : {
     652      4742844 :   rtx op0 = operands[0], op1 = operands[1];
     653              :   /* Use GET_MODE_BITSIZE instead of GET_MODE_ALIGNMENT for IA MCU
     654              :      psABI since the biggest alignment is 4 byte for IA MCU psABI.  */
     655      4742844 :   unsigned int align = (TARGET_IAMCU
     656      4742844 :                         ? GET_MODE_BITSIZE (mode)
     657      4742844 :                         : GET_MODE_ALIGNMENT (mode));
     658              : 
     659      4742844 :   if (push_operand (op0, VOIDmode))
     660         2920 :     op0 = emit_move_resolve_push (mode, op0);
     661              : 
     662              :   /* Force constants other than zero into memory.  We do not know how
     663              :      the instructions used to build constants modify the upper 64 bits
     664              :      of the register, once we have that information we may be able
     665              :      to handle some of them more efficiently.  */
     666      4742844 :   if (can_create_pseudo_p ()
     667      4545418 :       && (CONSTANT_P (op1)
     668      4212316 :           || (SUBREG_P (op1)
     669       329385 :               && CONSTANT_P (SUBREG_REG (op1))))
     670      5075960 :       && ((register_operand (op0, mode)
     671       275336 :            && !standard_sse_constant_p (op1, mode))
     672              :           /* ix86_expand_vector_move_misalign() does not like constants.  */
     673              :           || (SSE_REG_MODE_P (mode)
     674       275313 :               && MEM_P (op0)
     675        42262 :               && MEM_ALIGN (op0) < align)))
     676              :     {
     677         4964 :       if (SUBREG_P (op1))
     678              :         {
     679           14 :           machine_mode imode = GET_MODE (SUBREG_REG (op1));
     680           14 :           rtx r = force_const_mem (imode, SUBREG_REG (op1));
     681           14 :           if (r)
     682           14 :             r = validize_mem (r);
     683              :           else
     684            0 :             r = force_reg (imode, SUBREG_REG (op1));
     685           14 :           op1 = simplify_gen_subreg (mode, r, imode, SUBREG_BYTE (op1));
     686              :         }
     687              :       else
     688              :         {
     689         4950 :           machine_mode mode = GET_MODE (op0);
     690         4950 :           rtx tmp = ix86_convert_const_wide_int_to_broadcast
     691         4950 :             (mode, op1);
     692         4950 :           if (tmp == nullptr)
     693         4929 :             op1 = validize_mem (force_const_mem (mode, op1));
     694              :           else
     695              :             op1 = tmp;
     696              :         }
     697              :     }
     698              : 
     699      4742844 :   if (can_create_pseudo_p ()
     700      4545418 :       && GET_MODE_SIZE (mode) >= 16
     701      3824971 :       && VECTOR_MODE_P (mode)
     702      8354277 :       && (MEM_P (op1)
     703       761965 :           && SYMBOL_REF_P (XEXP (op1, 0))
     704       523657 :           && CONSTANT_POOL_ADDRESS_P (XEXP (op1, 0))))
     705              :     {
     706       506181 :       rtx first = ix86_broadcast_from_constant (mode, op1);
     707              :       /* Only broadcast floating-point constants when a broadcast instruction
     708              :          is available (AVX, or AVX2 for 16-bit elements), avoiding sequences
     709              :          such as movss + shufps.  For integer constants, guard broadcasting
     710              :          with TARGET_PREFER_BCST_FROM_INTEGER.  */
     711       506181 :       if (first != nullptr
     712       644442 :           && (FLOAT_MODE_P (mode)
     713        21008 :               ? (GET_MODE_UNIT_SIZE (mode) < 4 ? TARGET_AVX2 : TARGET_AVX)
     714       117253 :               : TARGET_PREFER_BCST_FROM_INTEGER))
     715              :         {
     716              :           /* Broadcast to XMM/YMM/ZMM register from an integer
     717              :              constant or scalar mem.  */
     718       122248 :           rtx tmp = gen_reg_rtx (mode);
     719       122248 :           if (FLOAT_MODE_P (mode))
     720         9994 :             first = force_const_mem (GET_MODE_INNER (mode), first);
     721       122248 :           bool ok = ix86_expand_vector_init_duplicate (false, mode,
     722              :                                                        tmp, first);
     723       122248 :           if (!ok && !TARGET_64BIT && GET_MODE_INNER (mode) == DImode)
     724              :             {
     725            0 :               first = force_const_mem (GET_MODE_INNER (mode), first);
     726            0 :               ok = ix86_expand_vector_init_duplicate (false, mode,
     727              :                                                       tmp, first);
     728              :             }
     729       122248 :           if (ok)
     730              :             {
     731       122248 :               emit_move_insn (op0, tmp);
     732       122248 :               return;
     733              :             }
     734              :         }
     735              :     }
     736              : 
     737              :   /* We need to check memory alignment for SSE mode since attribute
     738              :      can make operands unaligned.  */
     739      4620596 :   if (can_create_pseudo_p ()
     740              :       && SSE_REG_MODE_P (mode)
     741      9328466 :       && ((MEM_P (op0) && (MEM_ALIGN (op0) < align))
     742      4180615 :           || (MEM_P (op1) && (MEM_ALIGN (op1) < align))))
     743              :     {
     744       332314 :       rtx tmp[2];
     745              : 
     746              :       /* ix86_expand_vector_move_misalign() does not like both
     747              :          arguments in memory.  */
     748       332314 :       if (!register_operand (op0, mode)
     749       332314 :           && !register_operand (op1, mode))
     750              :         {
     751        96028 :           rtx scratch = gen_reg_rtx (mode);
     752        96028 :           emit_move_insn (scratch, op1);
     753        96028 :           op1 = scratch;
     754              :         }
     755              : 
     756       332314 :       tmp[0] = op0; tmp[1] = op1;
     757       332314 :       ix86_expand_vector_move_misalign (mode, tmp);
     758       332314 :       return;
     759              :     }
     760              : 
     761              :   /* Special case TImode to 128-bit vector conversions via V2DI.  */
     762      1175466 :   if (VECTOR_MODE_P (mode)
     763      4238897 :       && GET_MODE_SIZE (mode) == 16
     764      3018445 :       && SUBREG_P (op1)
     765       259367 :       && GET_MODE (SUBREG_REG (op1)) == TImode
     766         3204 :       && TARGET_64BIT && TARGET_SSE
     767      4290866 :       && ix86_pre_reload_split ())
     768              :     {
     769         2151 :       rtx tmp = gen_reg_rtx (V2DImode);
     770         2151 :       rtx lo = gen_reg_rtx (DImode);
     771         2151 :       rtx hi = gen_reg_rtx (DImode);
     772         2151 :       emit_move_insn (lo, gen_lowpart (DImode, SUBREG_REG (op1)));
     773         2151 :       emit_move_insn (hi, gen_highpart (DImode, SUBREG_REG (op1)));
     774         2151 :       emit_insn (gen_vec_concatv2di (tmp, lo, hi));
     775         2151 :       emit_move_insn (op0, gen_lowpart (mode, tmp));
     776         2151 :       return;
     777              :     }
     778              : 
     779              :   /* If operand0 is a hard register, make operand1 a pseudo.  */
     780      4286131 :   if (can_create_pseudo_p ()
     781      8374836 :       && !ix86_hardreg_mov_ok (op0, op1))
     782              :     {
     783          159 :       rtx tmp = gen_reg_rtx (GET_MODE (op0));
     784          159 :       emit_move_insn (tmp, op1);
     785          159 :       emit_move_insn (op0, tmp);
     786          159 :       return;
     787              :     }
     788              : 
     789              :   /* Make operand1 a register if it isn't already.  */
     790      4285972 :   if (can_create_pseudo_p ()
     791      4088546 :       && !register_operand (op0, mode)
     792      5437392 :       && !register_operand (op1, mode))
     793              :     {
     794       225358 :       rtx tmp = gen_reg_rtx (GET_MODE (op0));
     795       225358 :       emit_move_insn (tmp, op1);
     796       225358 :       emit_move_insn (op0, tmp);
     797       225358 :       return;
     798              :     }
     799              : 
     800      4060614 :   emit_insn (gen_rtx_SET (op0, op1));
     801              : }
     802              : 
     803              : /* Split 32-byte AVX unaligned load and store if needed.  */
     804              : 
     805              : static void
     806        13541 : ix86_avx256_split_vector_move_misalign (rtx op0, rtx op1)
     807              : {
     808        13541 :   rtx m;
     809        13541 :   rtx (*extract) (rtx, rtx, rtx);
     810        13541 :   machine_mode mode;
     811              : 
     812        13541 :   if ((MEM_P (op1) && !TARGET_AVX256_SPLIT_UNALIGNED_LOAD)
     813         5324 :       || (MEM_P (op0) && !TARGET_AVX256_SPLIT_UNALIGNED_STORE))
     814              :     {
     815        13515 :       emit_insn (gen_rtx_SET (op0, op1));
     816        13515 :       return;
     817              :     }
     818              : 
     819           26 :   rtx orig_op0 = NULL_RTX;
     820           26 :   mode = GET_MODE (op0);
     821           26 :   switch (GET_MODE_CLASS (mode))
     822              :     {
     823            9 :     case MODE_VECTOR_INT:
     824            9 :     case MODE_INT:
     825            9 :       if (mode != V32QImode)
     826              :         {
     827            7 :           if (!MEM_P (op0))
     828              :             {
     829            3 :               orig_op0 = op0;
     830            3 :               op0 = gen_reg_rtx (V32QImode);
     831              :             }
     832              :           else
     833            4 :             op0 = gen_lowpart (V32QImode, op0);
     834            7 :           op1 = gen_lowpart (V32QImode, op1);
     835            7 :           mode = V32QImode;
     836              :         }
     837              :       break;
     838              :     case MODE_VECTOR_FLOAT:
     839              :       break;
     840            0 :     default:
     841            0 :       gcc_unreachable ();
     842              :     }
     843              : 
     844           26 :   switch (mode)
     845              :     {
     846            0 :     default:
     847            0 :       gcc_unreachable ();
     848              :     case E_V32QImode:
     849              :       extract = gen_avx_vextractf128v32qi;
     850              :       mode = V16QImode;
     851              :       break;
     852            1 :     case E_V16BFmode:
     853            1 :       extract = gen_avx_vextractf128v16bf;
     854            1 :       mode = V8BFmode;
     855            1 :       break;
     856            0 :     case E_V16HFmode:
     857            0 :       extract = gen_avx_vextractf128v16hf;
     858            0 :       mode = V8HFmode;
     859            0 :       break;
     860            8 :     case E_V8SFmode:
     861            8 :       extract = gen_avx_vextractf128v8sf;
     862            8 :       mode = V4SFmode;
     863            8 :       break;
     864            8 :     case E_V4DFmode:
     865            8 :       extract = gen_avx_vextractf128v4df;
     866            8 :       mode = V2DFmode;
     867            8 :       break;
     868              :     }
     869              : 
     870           26 :   if (MEM_P (op1))
     871              :     {
     872            9 :       rtx r = gen_reg_rtx (mode);
     873            9 :       m = adjust_address (op1, mode, 0);
     874            9 :       emit_move_insn (r, m);
     875            9 :       m = adjust_address (op1, mode, 16);
     876            9 :       r = gen_rtx_VEC_CONCAT (GET_MODE (op0), r, m);
     877            9 :       emit_move_insn (op0, r);
     878              :     }
     879           17 :   else if (MEM_P (op0))
     880              :     {
     881           17 :       m = adjust_address (op0, mode, 0);
     882           17 :       emit_insn (extract (m, op1, const0_rtx));
     883           17 :       m = adjust_address (op0, mode, 16);
     884           17 :       emit_insn (extract (m, copy_rtx (op1), const1_rtx));
     885              :     }
     886              :   else
     887            0 :     gcc_unreachable ();
     888              : 
     889           26 :   if (orig_op0)
     890            3 :     emit_move_insn (orig_op0, gen_lowpart (GET_MODE (orig_op0), op0));
     891              : }
     892              : 
     893              : /* Implement the movmisalign patterns for SSE.  Non-SSE modes go
     894              :    straight to ix86_expand_vector_move.  */
     895              : /* Code generation for scalar reg-reg moves of single and double precision data:
     896              :      if (x86_sse_partial_reg_dependency == true | x86_sse_split_regs == true)
     897              :        movaps reg, reg
     898              :      else
     899              :        movss reg, reg
     900              :      if (x86_sse_partial_reg_dependency == true)
     901              :        movapd reg, reg
     902              :      else
     903              :        movsd reg, reg
     904              : 
     905              :    Code generation for scalar loads of double precision data:
     906              :      if (x86_sse_split_regs == true)
     907              :        movlpd mem, reg      (gas syntax)
     908              :      else
     909              :        movsd mem, reg
     910              : 
     911              :    Code generation for unaligned packed loads of single precision data
     912              :    (x86_sse_unaligned_move_optimal overrides x86_sse_partial_reg_dependency):
     913              :      if (x86_sse_unaligned_move_optimal)
     914              :        movups mem, reg
     915              : 
     916              :      if (x86_sse_partial_reg_dependency == true)
     917              :        {
     918              :          xorps  reg, reg
     919              :          movlps mem, reg
     920              :          movhps mem+8, reg
     921              :        }
     922              :      else
     923              :        {
     924              :          movlps mem, reg
     925              :          movhps mem+8, reg
     926              :        }
     927              : 
     928              :    Code generation for unaligned packed loads of double precision data
     929              :    (x86_sse_unaligned_move_optimal overrides x86_sse_split_regs):
     930              :      if (x86_sse_unaligned_move_optimal)
     931              :        movupd mem, reg
     932              : 
     933              :      if (x86_sse_split_regs == true)
     934              :        {
     935              :          movlpd mem, reg
     936              :          movhpd mem+8, reg
     937              :        }
     938              :      else
     939              :        {
     940              :          movsd  mem, reg
     941              :          movhpd mem+8, reg
     942              :        }
     943              :  */
     944              : 
     945              : void
     946       660610 : ix86_expand_vector_move_misalign (machine_mode mode, rtx operands[])
     947              : {
     948       660610 :   rtx op0, op1, m;
     949              : 
     950       660610 :   op0 = operands[0];
     951       660610 :   op1 = operands[1];
     952              : 
     953              :   /* Use unaligned load/store for AVX512 or when optimizing for size.  */
     954      1321220 :   if (GET_MODE_SIZE (mode) == 64 || optimize_insn_for_size_p ())
     955              :     {
     956        26668 :       emit_insn (gen_rtx_SET (op0, op1));
     957        26668 :       return;
     958              :     }
     959              : 
     960       633942 :   if (TARGET_AVX)
     961              :     {
     962        64462 :       if (GET_MODE_SIZE (mode) == 32)
     963        13541 :         ix86_avx256_split_vector_move_misalign (op0, op1);
     964              :       else
     965              :         /* Always use 128-bit mov<mode>_internal pattern for AVX.  */
     966        18690 :         emit_insn (gen_rtx_SET (op0, op1));
     967              :       return;
     968              :     }
     969              : 
     970       601711 :   if (TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
     971           95 :       || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL)
     972              :     {
     973       601616 :       emit_insn (gen_rtx_SET (op0, op1));
     974       601616 :       return;
     975              :     }
     976              : 
     977              :   /* ??? If we have typed data, then it would appear that using
     978              :      movdqu is the only way to get unaligned data loaded with
     979              :      integer type.  */
     980           95 :   if (TARGET_SSE2 && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
     981              :     {
     982           81 :       emit_insn (gen_rtx_SET (op0, op1));
     983           81 :       return;
     984              :     }
     985              : 
     986           14 :   if (MEM_P (op1))
     987              :     {
     988            6 :       if (TARGET_SSE2 && mode == V2DFmode)
     989              :         {
     990            2 :           rtx zero;
     991              : 
     992              :           /* When SSE registers are split into halves, we can avoid
     993              :              writing to the top half twice.  */
     994            2 :           if (TARGET_SSE_SPLIT_REGS)
     995              :             {
     996            2 :               emit_clobber (op0);
     997            2 :               zero = op0;
     998              :             }
     999              :           else
    1000              :             {
    1001              :               /* ??? Not sure about the best option for the Intel chips.
    1002              :                  The following would seem to satisfy; the register is
    1003              :                  entirely cleared, breaking the dependency chain.  We
    1004              :                  then store to the upper half, with a dependency depth
    1005              :                  of one.  A rumor has it that Intel recommends two movsd
    1006              :                  followed by an unpacklpd, but this is unconfirmed.  And
    1007              :                  given that the dependency depth of the unpacklpd would
    1008              :                  still be one, I'm not sure why this would be better.  */
    1009            0 :               zero = CONST0_RTX (V2DFmode);
    1010              :             }
    1011              : 
    1012            2 :           m = adjust_address (op1, DFmode, 0);
    1013            2 :           emit_insn (gen_sse2_loadlpd (op0, zero, m));
    1014            2 :           m = adjust_address (op1, DFmode, 8);
    1015            2 :           emit_insn (gen_sse2_loadhpd (op0, op0, m));
    1016            2 :         }
    1017              :       else
    1018              :         {
    1019            4 :           rtx t;
    1020              : 
    1021            4 :           if (mode != V4SFmode)
    1022            0 :             t = gen_reg_rtx (V4SFmode);
    1023              :           else
    1024              :             t = op0;
    1025              : 
    1026            4 :           if (TARGET_SSE_PARTIAL_REG_DEPENDENCY)
    1027            2 :             emit_move_insn (t, CONST0_RTX (V4SFmode));
    1028              :           else
    1029            2 :             emit_clobber (t);
    1030              : 
    1031            4 :           m = adjust_address (op1, V2SFmode, 0);
    1032            4 :           emit_insn (gen_sse_loadlps (t, t, m));
    1033            4 :           m = adjust_address (op1, V2SFmode, 8);
    1034            4 :           emit_insn (gen_sse_loadhps (t, t, m));
    1035            4 :           if (mode != V4SFmode)
    1036            0 :             emit_move_insn (op0, gen_lowpart (mode, t));
    1037              :         }
    1038              :     }
    1039            8 :   else if (MEM_P (op0))
    1040              :     {
    1041            8 :       if (TARGET_SSE2 && mode == V2DFmode)
    1042              :         {
    1043            2 :           m = adjust_address (op0, DFmode, 0);
    1044            2 :           emit_insn (gen_sse2_storelpd (m, op1));
    1045            2 :           m = adjust_address (op0, DFmode, 8);
    1046            2 :           emit_insn (gen_sse2_storehpd (m, op1));
    1047              :         }
    1048              :       else
    1049              :         {
    1050            6 :           if (mode != V4SFmode)
    1051            0 :             op1 = gen_lowpart (V4SFmode, op1);
    1052              : 
    1053            6 :           m = adjust_address (op0, V2SFmode, 0);
    1054            6 :           emit_insn (gen_sse_storelps (m, op1));
    1055            6 :           m = adjust_address (op0, V2SFmode, 8);
    1056            6 :           emit_insn (gen_sse_storehps (m, copy_rtx (op1)));
    1057              :         }
    1058              :     }
    1059              :   else
    1060            0 :     gcc_unreachable ();
    1061              : }
    1062              : 
    1063              : /* Move bits 64:95 to bits 32:63.  */
    1064              : 
    1065              : void
    1066          972 : ix86_move_vector_high_sse_to_mmx (rtx op)
    1067              : {
    1068          972 :   rtx mask = gen_rtx_PARALLEL (VOIDmode,
    1069              :                                gen_rtvec (4, GEN_INT (0), GEN_INT (2),
    1070              :                                           GEN_INT (0), GEN_INT (0)));
    1071          972 :   rtx dest = lowpart_subreg (V4SImode, op, GET_MODE (op));
    1072          972 :   op = gen_rtx_VEC_SELECT (V4SImode, dest, mask);
    1073          972 :   rtx insn = gen_rtx_SET (dest, op);
    1074          972 :   emit_insn (insn);
    1075          972 : }
    1076              : 
    1077              : /* Split MMX pack with signed/unsigned saturation with SSE/SSE2.  */
    1078              : 
    1079              : void
    1080          882 : ix86_split_mmx_pack (rtx operands[], enum rtx_code code)
    1081              : {
    1082          882 :   rtx op0 = operands[0];
    1083          882 :   rtx op1 = operands[1];
    1084          882 :   rtx op2 = operands[2];
    1085          882 :   rtx src;
    1086              : 
    1087          882 :   machine_mode dmode = GET_MODE (op0);
    1088          882 :   machine_mode smode = GET_MODE (op1);
    1089          882 :   machine_mode inner_dmode = GET_MODE_INNER (dmode);
    1090          882 :   machine_mode inner_smode = GET_MODE_INNER (smode);
    1091              : 
    1092              :   /* Get the corresponding SSE mode for destination.  */
    1093          882 :   int nunits = 16 / GET_MODE_SIZE (inner_dmode);
    1094         1764 :   machine_mode sse_dmode = mode_for_vector (GET_MODE_INNER (dmode),
    1095         1764 :                                             nunits).require ();
    1096          882 :   machine_mode sse_half_dmode = mode_for_vector (GET_MODE_INNER (dmode),
    1097         1764 :                                                  nunits / 2).require ();
    1098              : 
    1099              :   /* Get the corresponding SSE mode for source.  */
    1100          882 :   nunits = 16 / GET_MODE_SIZE (inner_smode);
    1101         1764 :   machine_mode sse_smode = mode_for_vector (GET_MODE_INNER (smode),
    1102         1764 :                                             nunits).require ();
    1103              : 
    1104              :   /* Generate SSE pack with signed/unsigned saturation.  */
    1105          882 :   rtx dest = lowpart_subreg (sse_dmode, op0, GET_MODE (op0));
    1106          882 :   op1 = lowpart_subreg (sse_smode, op1, GET_MODE (op1));
    1107          882 :   op2 = lowpart_subreg (sse_smode, op2, GET_MODE (op2));
    1108              : 
    1109              :   /* paskusdw/packuswb does unsigned saturation of a signed source
    1110              :      which is different from generic us_truncate RTX.  */
    1111          882 :   if (code == US_TRUNCATE)
    1112          780 :     src = gen_rtx_UNSPEC (sse_dmode,
    1113              :                           gen_rtvec (2, op1, op2),
    1114              :                           UNSPEC_US_TRUNCATE);
    1115              :   else
    1116              :     {
    1117          102 :       op1 = gen_rtx_fmt_e (code, sse_half_dmode, op1);
    1118          102 :       op2 = gen_rtx_fmt_e (code, sse_half_dmode, op2);
    1119          102 :       src = gen_rtx_VEC_CONCAT (sse_dmode, op1, op2);
    1120              :     }
    1121              : 
    1122          882 :   emit_move_insn (dest, src);
    1123              : 
    1124          882 :   ix86_move_vector_high_sse_to_mmx (op0);
    1125          882 : }
    1126              : 
    1127              : /* Split MMX punpcklXX/punpckhXX with SSE punpcklXX.  This is also used
    1128              :    for a full unpack of OPERANDS[1] and OPERANDS[2] into a wider
    1129              :    OPERANDS[0].  */
    1130              : 
    1131              : void
    1132         6722 : ix86_split_mmx_punpck (rtx operands[], bool high_p)
    1133              : {
    1134         6722 :   rtx op0 = operands[0];
    1135         6722 :   rtx op1 = operands[1];
    1136         6722 :   rtx op2 = operands[2];
    1137         6722 :   machine_mode mode = GET_MODE (op1);
    1138         6722 :   rtx mask;
    1139              :   /* The corresponding SSE mode.  */
    1140         6722 :   machine_mode sse_mode, double_sse_mode;
    1141              : 
    1142         6722 :   switch (mode)
    1143              :     {
    1144         1785 :     case E_V8QImode:
    1145         1785 :     case E_V4QImode:
    1146         1785 :     case E_V2QImode:
    1147         1785 :       sse_mode = V16QImode;
    1148         1785 :       double_sse_mode = V32QImode;
    1149         1785 :       mask = gen_rtx_PARALLEL (VOIDmode,
    1150              :                                gen_rtvec (16,
    1151              :                                           GEN_INT (0), GEN_INT (16),
    1152              :                                           GEN_INT (1), GEN_INT (17),
    1153              :                                           GEN_INT (2), GEN_INT (18),
    1154              :                                           GEN_INT (3), GEN_INT (19),
    1155              :                                           GEN_INT (4), GEN_INT (20),
    1156              :                                           GEN_INT (5), GEN_INT (21),
    1157              :                                           GEN_INT (6), GEN_INT (22),
    1158              :                                           GEN_INT (7), GEN_INT (23)));
    1159         1785 :       break;
    1160              : 
    1161         3600 :     case E_V4HImode:
    1162         3600 :     case E_V2HImode:
    1163         3600 :       sse_mode = V8HImode;
    1164         3600 :       double_sse_mode = V16HImode;
    1165         3600 :       mask = gen_rtx_PARALLEL (VOIDmode,
    1166              :                                gen_rtvec (8,
    1167              :                                           GEN_INT (0), GEN_INT (8),
    1168              :                                           GEN_INT (1), GEN_INT (9),
    1169              :                                           GEN_INT (2), GEN_INT (10),
    1170              :                                           GEN_INT (3), GEN_INT (11)));
    1171         3600 :       break;
    1172              : 
    1173          937 :     case E_V2SImode:
    1174          937 :       sse_mode = V4SImode;
    1175          937 :       double_sse_mode = V8SImode;
    1176          937 :       mask = gen_rtx_PARALLEL (VOIDmode,
    1177              :                                gen_rtvec (4,
    1178              :                                           GEN_INT (0), GEN_INT (4),
    1179              :                                           GEN_INT (1), GEN_INT (5)));
    1180          937 :       break;
    1181              : 
    1182          400 :     case E_V2SFmode:
    1183          400 :       sse_mode = V4SFmode;
    1184          400 :       double_sse_mode = V8SFmode;
    1185          400 :       mask = gen_rtx_PARALLEL (VOIDmode,
    1186              :                                gen_rtvec (4,
    1187              :                                           GEN_INT (0), GEN_INT (4),
    1188              :                                           GEN_INT (1), GEN_INT (5)));
    1189          400 :       break;
    1190              : 
    1191            0 :     default:
    1192            0 :       gcc_unreachable ();
    1193              :     }
    1194              : 
    1195              :   /* Generate SSE punpcklXX.  */
    1196         6722 :   rtx dest = lowpart_subreg (sse_mode, op0, GET_MODE (op0));
    1197         6722 :   op1 = lowpart_subreg (sse_mode, op1, GET_MODE (op1));
    1198         6722 :   op2 = lowpart_subreg (sse_mode, op2, GET_MODE (op2));
    1199              : 
    1200         6722 :   op1 = gen_rtx_VEC_CONCAT (double_sse_mode, op1, op2);
    1201         6722 :   op2 = gen_rtx_VEC_SELECT (sse_mode, op1, mask);
    1202         6722 :   rtx insn = gen_rtx_SET (dest, op2);
    1203         6722 :   emit_insn (insn);
    1204              : 
    1205              :   /* Move high bits to low bits.  */
    1206         6722 :   if (high_p)
    1207              :     {
    1208         2525 :       if (sse_mode == V4SFmode)
    1209              :         {
    1210          112 :           mask = gen_rtx_PARALLEL (VOIDmode,
    1211              :                                    gen_rtvec (4, GEN_INT (2), GEN_INT (3),
    1212              :                                               GEN_INT (4), GEN_INT (5)));
    1213          112 :           op2 = gen_rtx_VEC_CONCAT (V8SFmode, dest, dest);
    1214          112 :           op1 = gen_rtx_VEC_SELECT (V4SFmode, op2, mask);
    1215              :         }
    1216              :       else
    1217              :         {
    1218         2413 :           int sz = GET_MODE_SIZE (mode);
    1219              : 
    1220         2413 :           if (sz == 4)
    1221          243 :             mask = gen_rtx_PARALLEL (VOIDmode,
    1222              :                                      gen_rtvec (4, GEN_INT (1), GEN_INT (0),
    1223              :                                                 GEN_INT (0), GEN_INT (1)));
    1224         2170 :           else if (sz == 8)
    1225         2170 :             mask = gen_rtx_PARALLEL (VOIDmode,
    1226              :                                      gen_rtvec (4, GEN_INT (2), GEN_INT (3),
    1227              :                                                 GEN_INT (0), GEN_INT (1)));
    1228              :           else
    1229            0 :             gcc_unreachable ();
    1230              : 
    1231         2413 :           dest = lowpart_subreg (V4SImode, dest, GET_MODE (dest));
    1232         2413 :           op1 = gen_rtx_VEC_SELECT (V4SImode, dest, mask);
    1233              :         }
    1234              : 
    1235         2525 :       insn = gen_rtx_SET (dest, op1);
    1236         2525 :       emit_insn (insn);
    1237              :     }
    1238         6722 : }
    1239              : 
    1240              : /* Helper function of ix86_fixup_binary_operands to canonicalize
    1241              :    operand order.  Returns true if the operands should be swapped.  */
    1242              : 
    1243              : static bool
    1244    179256939 : ix86_swap_binary_operands_p (enum rtx_code code, machine_mode mode,
    1245              :                              rtx operands[])
    1246              : {
    1247    179256939 :   rtx dst = operands[0];
    1248    179256939 :   rtx src1 = operands[1];
    1249    179256939 :   rtx src2 = operands[2];
    1250              : 
    1251              :   /* If the operation is not commutative, we can't do anything.  */
    1252    179256939 :   if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
    1253     28309863 :       && GET_RTX_CLASS (code) != RTX_COMM_COMPARE)
    1254              :     return false;
    1255              : 
    1256              :   /* Highest priority is that src1 should match dst.  */
    1257    150959435 :   if (rtx_equal_p (dst, src1))
    1258              :     return false;
    1259    109894510 :   if (rtx_equal_p (dst, src2))
    1260              :     return true;
    1261              : 
    1262              :   /* Next highest priority is that immediate constants come second.  */
    1263    109799205 :   if (immediate_operand (src2, mode))
    1264              :     return false;
    1265     26812628 :   if (immediate_operand (src1, mode))
    1266              :     return true;
    1267              : 
    1268              :   /* Lowest priority is that memory references should come second.  */
    1269     26812628 :   if (MEM_P (src2))
    1270              :     return false;
    1271     25284382 :   if (MEM_P (src1))
    1272       579775 :     return true;
    1273              : 
    1274              :   return false;
    1275              : }
    1276              : 
    1277              : /* Fix up OPERANDS to satisfy ix86_binary_operator_ok.  Return the
    1278              :    destination to use for the operation.  If different from the true
    1279              :    destination in operands[0], a copy operation will be required except
    1280              :    under TARGET_APX_NDD.  */
    1281              : 
    1282              : rtx
    1283     13864672 : ix86_fixup_binary_operands (enum rtx_code code, machine_mode mode,
    1284              :                             rtx operands[], bool use_ndd)
    1285              : {
    1286     13864672 :   rtx dst = operands[0];
    1287     13864672 :   rtx src1 = operands[1];
    1288     13864672 :   rtx src2 = operands[2];
    1289              : 
    1290              :   /* Canonicalize operand order.  */
    1291     13864672 :   if (ix86_swap_binary_operands_p (code, mode, operands))
    1292              :     {
    1293              :       /* It is invalid to swap operands of different modes.  */
    1294        91223 :       gcc_assert (GET_MODE (src1) == GET_MODE (src2));
    1295              : 
    1296              :       std::swap (src1, src2);
    1297              :     }
    1298              : 
    1299              :   /* Both source operands cannot be in memory.  */
    1300     13864672 :   if (MEM_P (src1) && MEM_P (src2))
    1301              :     {
    1302              :       /* Optimization: Only read from memory once.  */
    1303       111820 :       if (rtx_equal_p (src1, src2))
    1304              :         {
    1305           19 :           src2 = force_reg (mode, src2);
    1306           19 :           src1 = src2;
    1307              :         }
    1308       111801 :       else if (rtx_equal_p (dst, src1))
    1309         3545 :         src2 = force_reg (mode, src2);
    1310              :       else
    1311       108256 :         src1 = force_reg (mode, src1);
    1312              :     }
    1313              : 
    1314              :   /* If the destination is memory, and we do not have matching source
    1315              :      operands, do things in registers.  */
    1316     13864672 :   if (MEM_P (dst) && !rtx_equal_p (dst, src1))
    1317       497626 :     dst = gen_reg_rtx (mode);
    1318              : 
    1319              :   /* Source 1 cannot be a constant.  */
    1320     13864672 :   if (CONSTANT_P (src1))
    1321          713 :     src1 = force_reg (mode, src1);
    1322              : 
    1323              :   /* Source 1 cannot be a non-matching memory.  */
    1324     13864672 :   if (!use_ndd && MEM_P (src1) && !rtx_equal_p (dst, src1))
    1325       455848 :     src1 = force_reg (mode, src1);
    1326              : 
    1327              :   /* Improve address combine.  */
    1328     13864672 :   if (code == PLUS
    1329     10264197 :       && GET_MODE_CLASS (mode) == MODE_INT
    1330     10153411 :       && MEM_P (src2))
    1331       182810 :     src2 = force_reg (mode, src2);
    1332              : 
    1333     13864672 :   operands[1] = src1;
    1334     13864672 :   operands[2] = src2;
    1335     13864672 :   return dst;
    1336              : }
    1337              : 
    1338              : /* Similarly, but assume that the destination has already been
    1339              :    set up properly.  */
    1340              : 
    1341              : void
    1342       301105 : ix86_fixup_binary_operands_no_copy (enum rtx_code code,
    1343              :                                     machine_mode mode, rtx operands[],
    1344              :                                     bool use_ndd)
    1345              : {
    1346       301105 :   rtx dst = ix86_fixup_binary_operands (code, mode, operands, use_ndd);
    1347       301105 :   gcc_assert (dst == operands[0]);
    1348       301105 : }
    1349              : 
    1350              : /* Attempt to expand a binary operator.  Make the expansion closer to the
    1351              :    actual machine, then just general_operand, which will allow 3 separate
    1352              :    memory references (one output, two input) in a single insn.  */
    1353              : 
    1354              : void
    1355     13563438 : ix86_expand_binary_operator (enum rtx_code code, machine_mode mode,
    1356              :                              rtx operands[], bool use_ndd)
    1357              : {
    1358     13563438 :   rtx src1, src2, dst, op, clob;
    1359              : 
    1360     13563438 :   dst = ix86_fixup_binary_operands (code, mode, operands, use_ndd);
    1361     13563438 :   src1 = operands[1];
    1362     13563438 :   src2 = operands[2];
    1363              : 
    1364              :  /* Emit the instruction.  */
    1365              : 
    1366     13563438 :   op = gen_rtx_SET (dst, gen_rtx_fmt_ee (code, mode, src1, src2));
    1367              : 
    1368     13563438 :   if (reload_completed
    1369        92688 :       && code == PLUS
    1370          927 :       && !rtx_equal_p (dst, src1)
    1371     13563438 :       && !use_ndd)
    1372              :     {
    1373              :       /* This is going to be an LEA; avoid splitting it later.  */
    1374            0 :       emit_insn (op);
    1375              :     }
    1376              :   else
    1377              :     {
    1378     13563438 :       clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
    1379     13563438 :       emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
    1380              :     }
    1381              : 
    1382              :   /* Fix up the destination if needed.  */
    1383     13563438 :   if (dst != operands[0])
    1384       497617 :     emit_move_insn (operands[0], dst);
    1385     13563438 : }
    1386              : 
    1387              : /* Expand vector logical operation CODE (AND, IOR, XOR) in MODE with
    1388              :    the given OPERANDS.  */
    1389              : 
    1390              : void
    1391        87056 : ix86_expand_vector_logical_operator (enum rtx_code code, machine_mode mode,
    1392              :                                      rtx operands[])
    1393              : {
    1394        87056 :   rtx op1 = NULL_RTX, op2 = NULL_RTX;
    1395        87056 :   if (SUBREG_P (operands[1]))
    1396              :     {
    1397          372 :       op1 = operands[1];
    1398          372 :       op2 = operands[2];
    1399              :     }
    1400        86684 :   else if (SUBREG_P (operands[2]))
    1401              :     {
    1402              :       op1 = operands[2];
    1403              :       op2 = operands[1];
    1404              :     }
    1405              :   /* Optimize (__m128i) d | (__m128i) e and similar code
    1406              :      when d and e are float vectors into float vector logical
    1407              :      insn.  In C/C++ without using intrinsics there is no other way
    1408              :      to express vector logical operation on float vectors than
    1409              :      to cast them temporarily to integer vectors.  */
    1410         3218 :   if (op1
    1411         3218 :       && !TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
    1412         3218 :       && (SUBREG_P (op2) || CONST_VECTOR_P (op2))
    1413          358 :       && GET_MODE_CLASS (GET_MODE (SUBREG_REG (op1))) == MODE_VECTOR_FLOAT
    1414          453 :       && GET_MODE_SIZE (GET_MODE (SUBREG_REG (op1))) == GET_MODE_SIZE (mode)
    1415          151 :       && SUBREG_BYTE (op1) == 0
    1416          151 :       && (CONST_VECTOR_P (op2)
    1417            1 :           || (GET_MODE (SUBREG_REG (op1)) == GET_MODE (SUBREG_REG (op2))
    1418            1 :               && SUBREG_BYTE (op2) == 0))
    1419          151 :       && can_create_pseudo_p ())
    1420              :     {
    1421          151 :       rtx dst;
    1422          151 :       switch (GET_MODE (SUBREG_REG (op1)))
    1423              :         {
    1424           67 :         case E_V4SFmode:
    1425           67 :         case E_V8SFmode:
    1426           67 :         case E_V16SFmode:
    1427           67 :         case E_V2DFmode:
    1428           67 :         case E_V4DFmode:
    1429           67 :         case E_V8DFmode:
    1430           67 :           dst = gen_reg_rtx (GET_MODE (SUBREG_REG (op1)));
    1431           67 :           if (CONST_VECTOR_P (op2))
    1432              :             {
    1433           66 :               op2 = gen_lowpart (GET_MODE (dst), op2);
    1434           66 :               op2 = force_reg (GET_MODE (dst), op2);
    1435              :             }
    1436              :           else
    1437              :             {
    1438            1 :               op1 = operands[1];
    1439            1 :               op2 = SUBREG_REG (operands[2]);
    1440            1 :               if (!vector_operand (op2, GET_MODE (dst)))
    1441            0 :                 op2 = force_reg (GET_MODE (dst), op2);
    1442              :             }
    1443           67 :           op1 = SUBREG_REG (op1);
    1444           67 :           if (!vector_operand (op1, GET_MODE (dst)))
    1445            0 :             op1 = force_reg (GET_MODE (dst), op1);
    1446           67 :           emit_insn (gen_rtx_SET (dst,
    1447              :                                   gen_rtx_fmt_ee (code, GET_MODE (dst),
    1448              :                                                   op1, op2)));
    1449           67 :           emit_move_insn (operands[0], gen_lowpart (mode, dst));
    1450           67 :           return;
    1451              :         default:
    1452              :           break;
    1453              :         }
    1454              :     }
    1455        86989 :   if (!vector_operand (operands[1], mode))
    1456            1 :     operands[1] = force_reg (mode, operands[1]);
    1457        86989 :   if (!vector_operand (operands[2], mode))
    1458        13894 :     operands[2] = force_reg (mode, operands[2]);
    1459        86989 :   ix86_fixup_binary_operands_no_copy (code, mode, operands);
    1460        86989 :   emit_insn (gen_rtx_SET (operands[0],
    1461              :                           gen_rtx_fmt_ee (code, mode, operands[1],
    1462              :                                           operands[2])));
    1463              : }
    1464              : 
    1465              : /* Return TRUE or FALSE depending on whether the binary operator meets the
    1466              :    appropriate constraints.  */
    1467              : 
    1468              : bool
    1469    166525219 : ix86_binary_operator_ok (enum rtx_code code, machine_mode mode,
    1470              :                          rtx operands[3], bool use_ndd)
    1471              : {
    1472    166525219 :   rtx dst = operands[0];
    1473    166525219 :   rtx src1 = operands[1];
    1474    166525219 :   rtx src2 = operands[2];
    1475              : 
    1476              :   /* Both source operands cannot be in memory.  */
    1477    158757454 :   if ((MEM_P (src1) || bcst_mem_operand (src1, mode))
    1478    166525596 :       && (MEM_P (src2) || bcst_mem_operand (src2, mode)))
    1479              :     return false;
    1480              : 
    1481              :   /* Canonicalize operand order for commutative operators.  */
    1482    165392267 :   if (ix86_swap_binary_operands_p (code, mode, operands))
    1483       583857 :     std::swap (src1, src2);
    1484              : 
    1485              :   /* If the destination is memory, we must have a matching source operand.  */
    1486    165392267 :   if (MEM_P (dst) && !rtx_equal_p (dst, src1))
    1487              :     return false;
    1488              : 
    1489              :   /* Source 1 cannot be a constant.  */
    1490    160177116 :   if (CONSTANT_P (src1))
    1491              :     return false;
    1492              : 
    1493              :   /* Source 1 cannot be a non-matching memory.  */
    1494    160174034 :   if (!use_ndd && MEM_P (src1) && !rtx_equal_p (dst, src1))
    1495              :     /* Support "andhi/andsi/anddi" as a zero-extending move.  */
    1496      4745031 :     return (code == AND
    1497       586510 :             && (mode == HImode
    1498       586510 :                 || mode == SImode
    1499       361512 :                 || (TARGET_64BIT && mode == DImode))
    1500      5078245 :             && satisfies_constraint_L (src2));
    1501              : 
    1502              :   return true;
    1503              : }
    1504              : 
    1505              : /* Attempt to expand a unary operator.  Make the expansion closer to the
    1506              :    actual machine, then just general_operand, which will allow 2 separate
    1507              :    memory references (one output, one input) in a single insn.  */
    1508              : 
    1509              : void
    1510       134917 : ix86_expand_unary_operator (enum rtx_code code, machine_mode mode,
    1511              :                             rtx operands[], bool use_ndd)
    1512              : {
    1513       134917 :   bool matching_memory = false;
    1514       134917 :   rtx src, dst, op, clob;
    1515              : 
    1516       134917 :   dst = operands[0];
    1517       134917 :   src = operands[1];
    1518              : 
    1519              :   /* If the destination is memory, and we do not have matching source
    1520              :      operands, do things in registers.  */
    1521       134917 :   if (MEM_P (dst))
    1522              :     {
    1523        12201 :       if (rtx_equal_p (dst, src))
    1524              :         matching_memory = true;
    1525              :       else
    1526        11891 :         dst = gen_reg_rtx (mode);
    1527              :     }
    1528              : 
    1529              :   /* When source operand is memory, destination must match.  */
    1530       134917 :   if (!use_ndd && MEM_P (src) && !matching_memory)
    1531         4638 :     src = force_reg (mode, src);
    1532              : 
    1533              :   /* Emit the instruction.  */
    1534              : 
    1535       134917 :   op = gen_rtx_SET (dst, gen_rtx_fmt_e (code, mode, src));
    1536              : 
    1537       134917 :   if (code == NOT)
    1538        71954 :     emit_insn (op);
    1539              :   else
    1540              :     {
    1541        62963 :       clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
    1542        62963 :       emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
    1543              :     }
    1544              : 
    1545              :   /* Fix up the destination if needed.  */
    1546       134917 :   if (dst != operands[0])
    1547        11891 :     emit_move_insn (operands[0], dst);
    1548       134917 : }
    1549              : 
    1550              : /* Return TRUE or FALSE depending on whether the unary operator meets the
    1551              :    appropriate constraints.  */
    1552              : 
    1553              : bool
    1554      2092578 : ix86_unary_operator_ok (enum rtx_code,
    1555              :                         machine_mode,
    1556              :                         rtx operands[2],
    1557              :                         bool use_ndd)
    1558              : {
    1559              :   /* If one of operands is memory, source and destination must match.  */
    1560      2092578 :   if ((MEM_P (operands[0])
    1561      1949512 :        || (!use_ndd && MEM_P (operands[1])))
    1562      2122927 :       && ! rtx_equal_p (operands[0], operands[1]))
    1563              :     return false;
    1564              :   return true;
    1565              : }
    1566              : 
    1567              : /* Predict just emitted jump instruction to be taken with probability PROB.  */
    1568              : 
    1569              : static void
    1570        12881 : predict_jump (int prob)
    1571              : {
    1572        12881 :   rtx_insn *insn = get_last_insn ();
    1573        12881 :   gcc_assert (JUMP_P (insn));
    1574        12881 :   add_reg_br_prob_note (insn, profile_probability::from_reg_br_prob_base (prob));
    1575        12881 : }
    1576              : 
    1577              : /* Split 32bit/64bit divmod with 8bit unsigned divmod if dividend and
    1578              :    divisor are within the range [0-255].  */
    1579              : 
    1580              : void
    1581           27 : ix86_split_idivmod (machine_mode mode, rtx operands[],
    1582              :                     bool unsigned_p)
    1583              : {
    1584           27 :   rtx_code_label *end_label, *qimode_label;
    1585           27 :   rtx div, mod;
    1586           27 :   rtx_insn *insn;
    1587           27 :   rtx scratch, tmp0, tmp1, tmp2;
    1588           27 :   rtx (*gen_divmod4_1) (rtx, rtx, rtx, rtx);
    1589              : 
    1590           27 :   operands[2] = force_reg (mode, operands[2]);
    1591           27 :   operands[3] = force_reg (mode, operands[3]);
    1592              : 
    1593           27 :   switch (mode)
    1594              :     {
    1595           20 :     case E_SImode:
    1596           20 :       if (GET_MODE (operands[0]) == SImode)
    1597              :         {
    1598           16 :           if (GET_MODE (operands[1]) == SImode)
    1599           14 :             gen_divmod4_1 = unsigned_p ? gen_udivmodsi4_1 : gen_divmodsi4_1;
    1600              :           else
    1601            2 :             gen_divmod4_1
    1602            2 :               = unsigned_p ? gen_udivmodsi4_zext_2 : gen_divmodsi4_zext_2;
    1603              :         }
    1604              :       else
    1605            4 :         gen_divmod4_1
    1606            4 :           = unsigned_p ? gen_udivmodsi4_zext_1 : gen_divmodsi4_zext_1;
    1607              :       break;
    1608              : 
    1609            7 :     case E_DImode:
    1610            7 :       gen_divmod4_1 = unsigned_p ? gen_udivmoddi4_1 : gen_divmoddi4_1;
    1611              :       break;
    1612              : 
    1613            0 :     default:
    1614            0 :       gcc_unreachable ();
    1615              :     }
    1616              : 
    1617           27 :   end_label = gen_label_rtx ();
    1618           27 :   qimode_label = gen_label_rtx ();
    1619              : 
    1620           27 :   scratch = gen_reg_rtx (mode);
    1621              : 
    1622              :   /* Use 8bit unsigned divimod if dividend and divisor are within
    1623              :      the range [0-255].  */
    1624           27 :   emit_move_insn (scratch, operands[2]);
    1625           27 :   scratch = expand_simple_binop (mode, IOR, scratch, operands[3],
    1626              :                                  scratch, 1, OPTAB_DIRECT);
    1627           27 :   emit_insn (gen_test_ccno_1 (mode, scratch, GEN_INT (-0x100)));
    1628           27 :   tmp0 = gen_rtx_REG (CCNOmode, FLAGS_REG);
    1629           27 :   tmp0 = gen_rtx_EQ (VOIDmode, tmp0, const0_rtx);
    1630           27 :   tmp0 = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp0,
    1631              :                                gen_rtx_LABEL_REF (VOIDmode, qimode_label),
    1632              :                                pc_rtx);
    1633           27 :   insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp0));
    1634           27 :   predict_jump (REG_BR_PROB_BASE * 50 / 100);
    1635           27 :   JUMP_LABEL (insn) = qimode_label;
    1636              : 
    1637              :   /* Generate original signed/unsigned divimod.  */
    1638           27 :   emit_insn (gen_divmod4_1 (operands[0], operands[1],
    1639              :                             operands[2], operands[3]));
    1640              : 
    1641              :   /* Branch to the end.  */
    1642           27 :   emit_jump_insn (gen_jump (end_label));
    1643           27 :   emit_barrier ();
    1644              : 
    1645              :   /* Generate 8bit unsigned divide.  */
    1646           27 :   emit_label (qimode_label);
    1647              :   /* Don't use operands[0] for result of 8bit divide since not all
    1648              :      registers support QImode ZERO_EXTRACT.  */
    1649           27 :   tmp0 = lowpart_subreg (HImode, scratch, mode);
    1650           27 :   tmp1 = lowpart_subreg (HImode, operands[2], mode);
    1651           27 :   tmp2 = lowpart_subreg (QImode, operands[3], mode);
    1652           27 :   emit_insn (gen_udivmodhiqi3 (tmp0, tmp1, tmp2));
    1653              : 
    1654           27 :   if (unsigned_p)
    1655              :     {
    1656           12 :       div = gen_rtx_UDIV (mode, operands[2], operands[3]);
    1657           12 :       mod = gen_rtx_UMOD (mode, operands[2], operands[3]);
    1658              :     }
    1659              :   else
    1660              :     {
    1661           15 :       div = gen_rtx_DIV (mode, operands[2], operands[3]);
    1662           15 :       mod = gen_rtx_MOD (mode, operands[2], operands[3]);
    1663              :     }
    1664           27 :   if (mode == SImode)
    1665              :     {
    1666           20 :       if (GET_MODE (operands[0]) != SImode)
    1667            4 :         div = gen_rtx_ZERO_EXTEND (DImode, div);
    1668           20 :       if (GET_MODE (operands[1]) != SImode)
    1669            2 :         mod = gen_rtx_ZERO_EXTEND (DImode, mod);
    1670              :     }
    1671              : 
    1672              :   /* Extract remainder from AH.  */
    1673           27 :   scratch = gen_lowpart (GET_MODE (operands[1]), scratch);
    1674           27 :   tmp1 = gen_rtx_ZERO_EXTRACT (GET_MODE (operands[1]), scratch,
    1675              :                                GEN_INT (8), GEN_INT (8));
    1676           27 :   insn = emit_move_insn (operands[1], tmp1);
    1677           27 :   set_unique_reg_note (insn, REG_EQUAL, mod);
    1678              : 
    1679              :   /* Zero extend quotient from AL.  */
    1680           27 :   tmp1 = gen_lowpart (QImode, tmp0);
    1681           27 :   insn = emit_insn (gen_extend_insn
    1682           27 :                     (operands[0], tmp1,
    1683           27 :                      GET_MODE (operands[0]), QImode, 1));
    1684           27 :   set_unique_reg_note (insn, REG_EQUAL, div);
    1685              : 
    1686           27 :   emit_label (end_label);
    1687           27 : }
    1688              : 
    1689              : /* Emit x86 binary operand CODE in mode MODE, where the first operand
    1690              :    matches destination.  RTX includes clobber of FLAGS_REG.  */
    1691              : 
    1692              : void
    1693         9077 : ix86_emit_binop (enum rtx_code code, machine_mode mode,
    1694              :                  rtx dst, rtx src)
    1695              : {
    1696         9077 :   rtx op, clob;
    1697              : 
    1698         9077 :   op = gen_rtx_SET (dst, gen_rtx_fmt_ee (code, mode, dst, src));
    1699         9077 :   clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
    1700              : 
    1701         9077 :   emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
    1702         9077 : }
    1703              : 
    1704              : /* Return true if regno1 def is nearest to the insn.  */
    1705              : 
    1706              : static bool
    1707           17 : find_nearest_reg_def (rtx_insn *insn, int regno1, int regno2)
    1708              : {
    1709           17 :   rtx_insn *prev = insn;
    1710           17 :   rtx_insn *start = BB_HEAD (BLOCK_FOR_INSN (insn));
    1711              : 
    1712           17 :   if (insn == start)
    1713              :     return false;
    1714           44 :   while (prev && prev != start)
    1715              :     {
    1716           34 :       if (!INSN_P (prev) || !NONDEBUG_INSN_P (prev))
    1717              :         {
    1718           10 :           prev = PREV_INSN (prev);
    1719           10 :           continue;
    1720              :         }
    1721           24 :       if (insn_defines_reg (regno1, INVALID_REGNUM, prev))
    1722              :         return true;
    1723           17 :       else if (insn_defines_reg (regno2, INVALID_REGNUM, prev))
    1724              :         return false;
    1725           17 :       prev = PREV_INSN (prev);
    1726              :     }
    1727              : 
    1728              :   /* None of the regs is defined in the bb.  */
    1729              :   return false;
    1730              : }
    1731              : 
    1732              : /* INSN_UID of the last insn emitted by zero store peephole2s.  */
    1733              : int ix86_last_zero_store_uid;
    1734              : 
    1735              : /* Split lea instructions into a sequence of instructions
    1736              :    which are executed on ALU to avoid AGU stalls.
    1737              :    It is assumed that it is allowed to clobber flags register
    1738              :    at lea position.  */
    1739              : 
    1740              : void
    1741         7115 : ix86_split_lea_for_addr (rtx_insn *insn, rtx operands[], machine_mode mode)
    1742              : {
    1743         7115 :   unsigned int regno0, regno1, regno2;
    1744         7115 :   struct ix86_address parts;
    1745         7115 :   rtx target, tmp;
    1746         7115 :   int ok, adds;
    1747              : 
    1748         7115 :   ok = ix86_decompose_address (operands[1], &parts);
    1749         7115 :   gcc_assert (ok);
    1750              : 
    1751         7115 :   target = gen_lowpart (mode, operands[0]);
    1752              : 
    1753         7115 :   regno0 = true_regnum (target);
    1754         7115 :   regno1 = INVALID_REGNUM;
    1755         7115 :   regno2 = INVALID_REGNUM;
    1756              : 
    1757         7115 :   if (parts.base)
    1758              :     {
    1759         7108 :       parts.base = gen_lowpart (mode, parts.base);
    1760         7108 :       regno1 = true_regnum (parts.base);
    1761              :     }
    1762              : 
    1763         7115 :   if (parts.index)
    1764              :     {
    1765         7111 :       parts.index = gen_lowpart (mode, parts.index);
    1766         7111 :       regno2 = true_regnum (parts.index);
    1767              :     }
    1768              : 
    1769         7115 :   if (parts.disp)
    1770          195 :     parts.disp = gen_lowpart (mode, parts.disp);
    1771              : 
    1772         7115 :   if (parts.scale > 1)
    1773              :     {
    1774              :       /* Case r1 = r1 + ...  */
    1775           11 :       if (regno1 == regno0)
    1776              :         {
    1777              :           /* If we have a case r1 = r1 + C * r2 then we
    1778              :              should use multiplication which is very
    1779              :              expensive.  Assume cost model is wrong if we
    1780              :              have such case here.  */
    1781            0 :           gcc_assert (regno2 != regno0);
    1782              : 
    1783            0 :           for (adds = parts.scale; adds > 0; adds--)
    1784            0 :             ix86_emit_binop (PLUS, mode, target, parts.index);
    1785              :         }
    1786              :       else
    1787              :         {
    1788              :           /* r1 = r2 + r3 * C case.  Need to move r3 into r1.  */
    1789           11 :           if (regno0 != regno2)
    1790            8 :             emit_insn (gen_rtx_SET (target, parts.index));
    1791              : 
    1792              :           /* Use shift for scaling, but emit it as MULT instead
    1793              :              to avoid it being immediately peephole2 optimized back
    1794              :              into lea.  */
    1795           11 :           ix86_emit_binop (MULT, mode, target, GEN_INT (parts.scale));
    1796              : 
    1797           11 :           if (parts.base)
    1798            4 :             ix86_emit_binop (PLUS, mode, target, parts.base);
    1799              : 
    1800           11 :           if (parts.disp && parts.disp != const0_rtx)
    1801            2 :             ix86_emit_binop (PLUS, mode, target, parts.disp);
    1802              :         }
    1803              :     }
    1804         7104 :   else if (!parts.base && !parts.index)
    1805              :     {
    1806            0 :       gcc_assert(parts.disp);
    1807            0 :       emit_insn (gen_rtx_SET (target, parts.disp));
    1808              :     }
    1809              :   else
    1810              :     {
    1811         7104 :       if (!parts.base)
    1812              :         {
    1813            0 :           if (regno0 != regno2)
    1814            0 :             emit_insn (gen_rtx_SET (target, parts.index));
    1815              :         }
    1816         7104 :       else if (!parts.index)
    1817              :         {
    1818            4 :           if (regno0 != regno1)
    1819            2 :             emit_insn (gen_rtx_SET (target, parts.base));
    1820              :         }
    1821              :       else
    1822              :         {
    1823         7100 :           if (regno0 == regno1)
    1824              :             tmp = parts.index;
    1825         3684 :           else if (regno0 == regno2)
    1826              :             tmp = parts.base;
    1827              :           else
    1828              :             {
    1829           17 :               rtx tmp1;
    1830              : 
    1831              :               /* Find better operand for SET instruction, depending
    1832              :                  on which definition is farther from the insn.  */
    1833           17 :               if (find_nearest_reg_def (insn, regno1, regno2))
    1834            7 :                 tmp = parts.index, tmp1 = parts.base;
    1835              :               else
    1836           10 :                 tmp = parts.base, tmp1 = parts.index;
    1837              : 
    1838           17 :               emit_insn (gen_rtx_SET (target, tmp));
    1839              : 
    1840           17 :               if (parts.disp && parts.disp != const0_rtx)
    1841            0 :                 ix86_emit_binop (PLUS, mode, target, parts.disp);
    1842              : 
    1843           17 :               ix86_emit_binop (PLUS, mode, target, tmp1);
    1844           17 :               return;
    1845              :             }
    1846              : 
    1847         7083 :           ix86_emit_binop (PLUS, mode, target, tmp);
    1848              :         }
    1849              : 
    1850         7087 :       if (parts.disp && parts.disp != const0_rtx)
    1851            5 :         ix86_emit_binop (PLUS, mode, target, parts.disp);
    1852              :     }
    1853              : }
    1854              : 
    1855              : /* Post-reload splitter for converting an SF or DFmode value in an
    1856              :    SSE register into an unsigned SImode.  */
    1857              : 
    1858              : void
    1859            0 : ix86_split_convert_uns_si_sse (rtx operands[])
    1860              : {
    1861            0 :   machine_mode vecmode;
    1862            0 :   rtx value, large, zero_or_two31, input, two31, x;
    1863              : 
    1864            0 :   large = operands[1];
    1865            0 :   zero_or_two31 = operands[2];
    1866            0 :   input = operands[3];
    1867            0 :   two31 = operands[4];
    1868            0 :   vecmode = GET_MODE (large);
    1869            0 :   value = gen_rtx_REG (vecmode, REGNO (operands[0]));
    1870              : 
    1871              :   /* Load up the value into the low element.  We must ensure that the other
    1872              :      elements are valid floats -- zero is the easiest such value.  */
    1873            0 :   if (MEM_P (input))
    1874              :     {
    1875            0 :       if (vecmode == V4SFmode)
    1876            0 :         emit_insn (gen_vec_setv4sf_0 (value, CONST0_RTX (V4SFmode), input));
    1877              :       else
    1878            0 :         emit_insn (gen_sse2_loadlpd (value, CONST0_RTX (V2DFmode), input));
    1879              :     }
    1880              :   else
    1881              :     {
    1882            0 :       input = gen_rtx_REG (vecmode, REGNO (input));
    1883            0 :       emit_move_insn (value, CONST0_RTX (vecmode));
    1884            0 :       if (vecmode == V4SFmode)
    1885            0 :         emit_insn (gen_sse_movss_v4sf (value, value, input));
    1886              :       else
    1887            0 :         emit_insn (gen_sse2_movsd_v2df (value, value, input));
    1888              :     }
    1889              : 
    1890            0 :   emit_move_insn (large, two31);
    1891            0 :   emit_move_insn (zero_or_two31, MEM_P (two31) ? large : two31);
    1892              : 
    1893            0 :   x = gen_rtx_fmt_ee (LE, vecmode, large, value);
    1894            0 :   emit_insn (gen_rtx_SET (large, x));
    1895              : 
    1896            0 :   x = gen_rtx_AND (vecmode, zero_or_two31, large);
    1897            0 :   emit_insn (gen_rtx_SET (zero_or_two31, x));
    1898              : 
    1899            0 :   x = gen_rtx_MINUS (vecmode, value, zero_or_two31);
    1900            0 :   emit_insn (gen_rtx_SET (value, x));
    1901              : 
    1902            0 :   large = gen_rtx_REG (V4SImode, REGNO (large));
    1903            0 :   emit_insn (gen_ashlv4si3 (large, large, GEN_INT (31)));
    1904              : 
    1905            0 :   x = gen_rtx_REG (V4SImode, REGNO (value));
    1906            0 :   if (vecmode == V4SFmode)
    1907            0 :     emit_insn (gen_fix_truncv4sfv4si2 (x, value));
    1908              :   else
    1909            0 :     emit_insn (gen_sse2_cvttpd2dq (x, value));
    1910            0 :   value = x;
    1911              : 
    1912            0 :   emit_insn (gen_xorv4si3 (value, value, large));
    1913            0 : }
    1914              : 
    1915              : /* Convert an unsigned DImode value into a DFmode, using only SSE.
    1916              :    Expects the 64-bit DImode to be supplied in a pair of integral
    1917              :    registers.  Requires SSE2; will use SSE3 if available.  For x86_32,
    1918              :    -mfpmath=sse, !optimize_size only.  */
    1919              : 
    1920              : void
    1921            0 : ix86_expand_convert_uns_didf_sse (rtx target, rtx input)
    1922              : {
    1923            0 :   REAL_VALUE_TYPE bias_lo_rvt, bias_hi_rvt;
    1924            0 :   rtx int_xmm, fp_xmm;
    1925            0 :   rtx biases, exponents;
    1926            0 :   rtx x;
    1927              : 
    1928            0 :   int_xmm = gen_reg_rtx (V4SImode);
    1929            0 :   if (TARGET_INTER_UNIT_MOVES_TO_VEC)
    1930            0 :     emit_insn (gen_movdi_to_sse (int_xmm, input));
    1931            0 :   else if (TARGET_SSE_SPLIT_REGS)
    1932              :     {
    1933            0 :       emit_clobber (int_xmm);
    1934            0 :       emit_move_insn (gen_lowpart (DImode, int_xmm), input);
    1935              :     }
    1936              :   else
    1937              :     {
    1938            0 :       x = gen_reg_rtx (V2DImode);
    1939            0 :       ix86_expand_vector_init_one_nonzero (false, V2DImode, x, input, 0);
    1940            0 :       emit_move_insn (int_xmm, gen_lowpart (V4SImode, x));
    1941              :     }
    1942              : 
    1943            0 :   x = gen_rtx_CONST_VECTOR (V4SImode,
    1944              :                             gen_rtvec (4, GEN_INT (0x43300000UL),
    1945              :                                        GEN_INT (0x45300000UL),
    1946              :                                        const0_rtx, const0_rtx));
    1947            0 :   exponents = validize_mem (force_const_mem (V4SImode, x));
    1948              : 
    1949              :   /* int_xmm = {0x45300000UL, fp_xmm/hi, 0x43300000, fp_xmm/lo } */
    1950            0 :   emit_insn (gen_vec_interleave_lowv4si (int_xmm, int_xmm, exponents));
    1951              : 
    1952              :   /* Concatenating (juxtaposing) (0x43300000UL ## fp_value_low_xmm)
    1953              :      yields a valid DF value equal to (0x1.0p52 + double(fp_value_lo_xmm)).
    1954              :      Similarly (0x45300000UL ## fp_value_hi_xmm) yields
    1955              :      (0x1.0p84 + double(fp_value_hi_xmm)).
    1956              :      Note these exponents differ by 32.  */
    1957              : 
    1958            0 :   fp_xmm = copy_to_mode_reg (V2DFmode, gen_lowpart (V2DFmode, int_xmm));
    1959              : 
    1960              :   /* Subtract off those 0x1.0p52 and 0x1.0p84 biases, to produce values
    1961              :      in [0,2**32-1] and [0]+[2**32,2**64-1] respectively.  */
    1962            0 :   real_ldexp (&bias_lo_rvt, &dconst1, 52);
    1963            0 :   real_ldexp (&bias_hi_rvt, &dconst1, 84);
    1964            0 :   biases = const_double_from_real_value (bias_lo_rvt, DFmode);
    1965            0 :   x = const_double_from_real_value (bias_hi_rvt, DFmode);
    1966            0 :   biases = gen_rtx_CONST_VECTOR (V2DFmode, gen_rtvec (2, biases, x));
    1967            0 :   biases = validize_mem (force_const_mem (V2DFmode, biases));
    1968            0 :   emit_insn (gen_subv2df3 (fp_xmm, fp_xmm, biases));
    1969              : 
    1970              :   /* Add the upper and lower DFmode values together.  */
    1971            0 :   if (TARGET_SSE3)
    1972            0 :     emit_insn (gen_sse3_haddv2df3 (fp_xmm, fp_xmm, fp_xmm));
    1973              :   else
    1974              :     {
    1975            0 :       x = copy_to_mode_reg (V2DFmode, fp_xmm);
    1976            0 :       emit_insn (gen_vec_interleave_highv2df (fp_xmm, fp_xmm, fp_xmm));
    1977            0 :       emit_insn (gen_addv2df3 (fp_xmm, fp_xmm, x));
    1978              :     }
    1979              : 
    1980            0 :   ix86_expand_vector_extract (false, target, fp_xmm, 0);
    1981            0 : }
    1982              : 
    1983              : /* Not used, but eases macroization of patterns.  */
    1984              : void
    1985            0 : ix86_expand_convert_uns_sixf_sse (rtx, rtx)
    1986              : {
    1987            0 :   gcc_unreachable ();
    1988              : }
    1989              : 
    1990              : static rtx ix86_expand_sse_fabs (rtx op0, rtx *smask);
    1991              : 
    1992              : /* Convert an unsigned SImode value into a DFmode.  Only currently used
    1993              :    for SSE, but applicable anywhere.  */
    1994              : 
    1995              : void
    1996            0 : ix86_expand_convert_uns_sidf_sse (rtx target, rtx input)
    1997              : {
    1998            0 :   REAL_VALUE_TYPE TWO31r;
    1999            0 :   rtx x, fp;
    2000              : 
    2001            0 :   x = expand_simple_binop (SImode, PLUS, input, GEN_INT (-2147483647 - 1),
    2002              :                            NULL, 1, OPTAB_DIRECT);
    2003              : 
    2004            0 :   fp = gen_reg_rtx (DFmode);
    2005            0 :   emit_insn (gen_floatsidf2 (fp, x));
    2006              : 
    2007            0 :   real_ldexp (&TWO31r, &dconst1, 31);
    2008            0 :   x = const_double_from_real_value (TWO31r, DFmode);
    2009              : 
    2010            0 :   x = expand_simple_binop (DFmode, PLUS, fp, x, target, 0, OPTAB_DIRECT);
    2011              : 
    2012              :   /* Remove the sign with FE_DOWNWARD, where x - x = -0.0.  */
    2013            0 :   if (HONOR_SIGNED_ZEROS (DFmode) && flag_rounding_math)
    2014            0 :     x = ix86_expand_sse_fabs (x, NULL);
    2015              : 
    2016            0 :   if (x != target)
    2017            0 :     emit_move_insn (target, x);
    2018            0 : }
    2019              : 
    2020              : /* Convert a signed DImode value into a DFmode.  Only used for SSE in
    2021              :    32-bit mode; otherwise we have a direct convert instruction.  */
    2022              : 
    2023              : void
    2024            0 : ix86_expand_convert_sign_didf_sse (rtx target, rtx input)
    2025              : {
    2026            0 :   REAL_VALUE_TYPE TWO32r;
    2027            0 :   rtx fp_lo, fp_hi, x;
    2028              : 
    2029            0 :   fp_lo = gen_reg_rtx (DFmode);
    2030            0 :   fp_hi = gen_reg_rtx (DFmode);
    2031              : 
    2032            0 :   emit_insn (gen_floatsidf2 (fp_hi, gen_highpart (SImode, input)));
    2033              : 
    2034            0 :   real_ldexp (&TWO32r, &dconst1, 32);
    2035            0 :   x = const_double_from_real_value (TWO32r, DFmode);
    2036            0 :   fp_hi = expand_simple_binop (DFmode, MULT, fp_hi, x, fp_hi, 0, OPTAB_DIRECT);
    2037              : 
    2038            0 :   ix86_expand_convert_uns_sidf_sse (fp_lo, gen_lowpart (SImode, input));
    2039              : 
    2040            0 :   x = expand_simple_binop (DFmode, PLUS, fp_hi, fp_lo, target,
    2041              :                            0, OPTAB_DIRECT);
    2042            0 :   if (x != target)
    2043            0 :     emit_move_insn (target, x);
    2044            0 : }
    2045              : 
    2046              : /* Convert an unsigned SImode value into a SFmode, using only SSE.
    2047              :    For x86_32, -mfpmath=sse, !optimize_size only.  */
    2048              : void
    2049            0 : ix86_expand_convert_uns_sisf_sse (rtx target, rtx input)
    2050              : {
    2051            0 :   REAL_VALUE_TYPE ONE16r;
    2052            0 :   rtx fp_hi, fp_lo, int_hi, int_lo, x;
    2053              : 
    2054            0 :   real_ldexp (&ONE16r, &dconst1, 16);
    2055            0 :   x = const_double_from_real_value (ONE16r, SFmode);
    2056            0 :   int_lo = expand_simple_binop (SImode, AND, input, GEN_INT(0xffff),
    2057              :                                       NULL, 0, OPTAB_DIRECT);
    2058            0 :   int_hi = expand_simple_binop (SImode, LSHIFTRT, input, GEN_INT(16),
    2059              :                                       NULL, 0, OPTAB_DIRECT);
    2060            0 :   fp_hi = gen_reg_rtx (SFmode);
    2061            0 :   fp_lo = gen_reg_rtx (SFmode);
    2062            0 :   emit_insn (gen_floatsisf2 (fp_hi, int_hi));
    2063            0 :   emit_insn (gen_floatsisf2 (fp_lo, int_lo));
    2064            0 :   if (TARGET_FMA)
    2065              :     {
    2066            0 :       x = validize_mem (force_const_mem (SFmode, x));
    2067            0 :       fp_hi = gen_rtx_FMA (SFmode, fp_hi, x, fp_lo);
    2068            0 :       emit_move_insn (target, fp_hi);
    2069              :     }
    2070              :   else
    2071              :     {
    2072            0 :       fp_hi = expand_simple_binop (SFmode, MULT, fp_hi, x, fp_hi,
    2073              :                                    0, OPTAB_DIRECT);
    2074            0 :       fp_hi = expand_simple_binop (SFmode, PLUS, fp_hi, fp_lo, target,
    2075              :                                    0, OPTAB_DIRECT);
    2076            0 :       if (!rtx_equal_p (target, fp_hi))
    2077            0 :         emit_move_insn (target, fp_hi);
    2078              :     }
    2079            0 : }
    2080              : 
    2081              : /* floatunsv{4,8}siv{4,8}sf2 expander.  Expand code to convert
    2082              :    a vector of unsigned ints VAL to vector of floats TARGET.  */
    2083              : 
    2084              : void
    2085           48 : ix86_expand_vector_convert_uns_vsivsf (rtx target, rtx val)
    2086              : {
    2087           48 :   rtx tmp[8];
    2088           48 :   REAL_VALUE_TYPE TWO16r;
    2089           48 :   machine_mode intmode = GET_MODE (val);
    2090           48 :   machine_mode fltmode = GET_MODE (target);
    2091           48 :   rtx (*cvt) (rtx, rtx);
    2092              : 
    2093           48 :   if (intmode == V4SImode)
    2094              :     cvt = gen_floatv4siv4sf2;
    2095              :   else
    2096            2 :     cvt = gen_floatv8siv8sf2;
    2097           48 :   tmp[0] = ix86_build_const_vector (intmode, 1, GEN_INT (0xffff));
    2098           48 :   tmp[0] = force_reg (intmode, tmp[0]);
    2099           48 :   tmp[1] = expand_simple_binop (intmode, AND, val, tmp[0], NULL_RTX, 1,
    2100              :                                 OPTAB_DIRECT);
    2101           48 :   tmp[2] = expand_simple_binop (intmode, LSHIFTRT, val, GEN_INT (16),
    2102              :                                 NULL_RTX, 1, OPTAB_DIRECT);
    2103           48 :   tmp[3] = gen_reg_rtx (fltmode);
    2104           48 :   emit_insn (cvt (tmp[3], tmp[1]));
    2105           48 :   tmp[4] = gen_reg_rtx (fltmode);
    2106           48 :   emit_insn (cvt (tmp[4], tmp[2]));
    2107           48 :   real_ldexp (&TWO16r, &dconst1, 16);
    2108           48 :   tmp[5] = const_double_from_real_value (TWO16r, SFmode);
    2109           48 :   tmp[5] = force_reg (fltmode, ix86_build_const_vector (fltmode, 1, tmp[5]));
    2110           48 :   if (TARGET_FMA)
    2111              :     {
    2112            1 :       tmp[6] = gen_rtx_FMA (fltmode, tmp[4], tmp[5], tmp[3]);
    2113            1 :       emit_move_insn (target, tmp[6]);
    2114              :     }
    2115              :   else
    2116              :     {
    2117           47 :       tmp[6] = expand_simple_binop (fltmode, MULT, tmp[4], tmp[5],
    2118              :                                     NULL_RTX, 1, OPTAB_DIRECT);
    2119           47 :       tmp[7] = expand_simple_binop (fltmode, PLUS, tmp[3], tmp[6],
    2120              :                                     target, 1, OPTAB_DIRECT);
    2121           47 :       if (tmp[7] != target)
    2122            0 :         emit_move_insn (target, tmp[7]);
    2123              :     }
    2124           48 : }
    2125              : 
    2126              : /* Adjust a V*SFmode/V*DFmode value VAL so that *sfix_trunc* resp. fix_trunc*
    2127              :    pattern can be used on it instead of fixuns_trunc*.
    2128              :    This is done by doing just signed conversion if < 0x1p31, and otherwise by
    2129              :    subtracting 0x1p31 first and xoring in 0x80000000 from *XORP afterwards.  */
    2130              : 
    2131              : rtx
    2132          326 : ix86_expand_adjust_ufix_to_sfix_si (rtx val, rtx *xorp)
    2133              : {
    2134          326 :   REAL_VALUE_TYPE TWO31r;
    2135          326 :   rtx two31r, tmp[4];
    2136          326 :   machine_mode mode = GET_MODE (val);
    2137          326 :   machine_mode scalarmode = GET_MODE_INNER (mode);
    2138          652 :   machine_mode intmode = GET_MODE_SIZE (mode) == 32 ? V8SImode : V4SImode;
    2139          326 :   rtx (*cmp) (rtx, rtx, rtx, rtx);
    2140          326 :   int i;
    2141              : 
    2142         1304 :   for (i = 0; i < 3; i++)
    2143          978 :     tmp[i] = gen_reg_rtx (mode);
    2144          326 :   real_ldexp (&TWO31r, &dconst1, 31);
    2145          326 :   two31r = const_double_from_real_value (TWO31r, scalarmode);
    2146          326 :   two31r = ix86_build_const_vector (mode, 1, two31r);
    2147          326 :   two31r = force_reg (mode, two31r);
    2148          326 :   switch (mode)
    2149              :     {
    2150              :     case E_V8SFmode: cmp = gen_avx_maskcmpv8sf3; break;
    2151           10 :     case E_V4SFmode: cmp = gen_sse_maskcmpv4sf3; break;
    2152           16 :     case E_V4DFmode: cmp = gen_avx_maskcmpv4df3; break;
    2153          300 :     case E_V2DFmode: cmp = gen_sse2_maskcmpv2df3; break;
    2154            0 :     default: gcc_unreachable ();
    2155              :     }
    2156          326 :   tmp[3] = gen_rtx_LE (mode, two31r, val);
    2157          326 :   emit_insn (cmp (tmp[0], two31r, val, tmp[3]));
    2158          326 :   tmp[1] = expand_simple_binop (mode, AND, tmp[0], two31r, tmp[1],
    2159              :                                 0, OPTAB_DIRECT);
    2160          326 :   if (intmode == V4SImode || TARGET_AVX2)
    2161          652 :     *xorp = expand_simple_binop (intmode, ASHIFT,
    2162          326 :                                  gen_lowpart (intmode, tmp[0]),
    2163              :                                  GEN_INT (31), NULL_RTX, 0,
    2164              :                                  OPTAB_DIRECT);
    2165              :   else
    2166              :     {
    2167            0 :       rtx two31 = gen_int_mode (HOST_WIDE_INT_1U << 31, SImode);
    2168            0 :       two31 = ix86_build_const_vector (intmode, 1, two31);
    2169            0 :       *xorp = expand_simple_binop (intmode, AND,
    2170            0 :                                    gen_lowpart (intmode, tmp[0]),
    2171              :                                    two31, NULL_RTX, 0,
    2172              :                                    OPTAB_DIRECT);
    2173              :     }
    2174          326 :   return expand_simple_binop (mode, MINUS, val, tmp[1], tmp[2],
    2175          326 :                               0, OPTAB_DIRECT);
    2176              : }
    2177              : 
    2178              : /* Generate code for floating point ABS or NEG.  */
    2179              : 
    2180              : void
    2181        35272 : ix86_expand_fp_absneg_operator (enum rtx_code code, machine_mode mode,
    2182              :                                 rtx operands[])
    2183              : {
    2184        35272 :   rtx set, dst, src;
    2185        35272 :   bool use_sse = false;
    2186        35272 :   bool vector_mode = VECTOR_MODE_P (mode);
    2187        35272 :   machine_mode vmode = mode;
    2188        35272 :   rtvec par;
    2189              : 
    2190        35272 :   switch (mode)
    2191              :   {
    2192              :   case E_HFmode:
    2193              :     use_sse = true;
    2194              :     vmode = V8HFmode;
    2195              :     break;
    2196            0 :   case E_BFmode:
    2197            0 :     use_sse = true;
    2198            0 :     vmode = V8BFmode;
    2199            0 :     break;
    2200         9505 :   case E_SFmode:
    2201         9505 :     use_sse = TARGET_SSE_MATH && TARGET_SSE;
    2202              :     vmode = V4SFmode;
    2203              :     break;
    2204        16223 :   case E_DFmode:
    2205        16223 :     use_sse = TARGET_SSE_MATH && TARGET_SSE2;
    2206              :     vmode = V2DFmode;
    2207              :     break;
    2208         9343 :   default:
    2209         9343 :     use_sse = vector_mode || mode == TFmode;
    2210         9343 :     break;
    2211              :   }
    2212              : 
    2213        35272 :   dst = operands[0];
    2214        35272 :   src = operands[1];
    2215              : 
    2216        35272 :   set = gen_rtx_fmt_e (code, mode, src);
    2217        35272 :   set = gen_rtx_SET (dst, set);
    2218              : 
    2219        35272 :   if (use_sse)
    2220              :     {
    2221        29522 :       rtx mask, use, clob;
    2222              : 
    2223              :       /* NEG and ABS performed with SSE use bitwise mask operations.
    2224              :          Create the appropriate mask now.  */
    2225        29522 :       mask = ix86_build_signbit_mask (vmode, vector_mode, code == ABS);
    2226        29522 :       use = gen_rtx_USE (VOIDmode, mask);
    2227        29522 :       if (vector_mode || mode == TFmode)
    2228         4968 :         par = gen_rtvec (2, set, use);
    2229              :       else
    2230              :         {
    2231        24554 :           clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
    2232        24554 :           par = gen_rtvec (3, set, use, clob);
    2233              :         }
    2234              :     }
    2235              :   else
    2236              :     {
    2237         5750 :       rtx clob;
    2238              : 
    2239              :       /* Changing of sign for FP values is doable using integer unit too.  */
    2240         5750 :       clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
    2241         5750 :       par = gen_rtvec (2, set, clob);
    2242              :     }
    2243              : 
    2244        35272 :   emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
    2245        35272 : }
    2246              : 
    2247              : /* Deconstruct a floating point ABS or NEG operation
    2248              :    with integer registers into integer operations.  */
    2249              : 
    2250              : void
    2251           24 : ix86_split_fp_absneg_operator (enum rtx_code code, machine_mode mode,
    2252              :                                rtx operands[])
    2253              : {
    2254           24 :   enum rtx_code absneg_op;
    2255           24 :   rtx dst, set;
    2256              : 
    2257           24 :   gcc_assert (operands_match_p (operands[0], operands[1]));
    2258              : 
    2259           24 :   switch (mode)
    2260              :     {
    2261            0 :     case E_SFmode:
    2262            0 :       dst = gen_lowpart (SImode, operands[0]);
    2263              : 
    2264            0 :       if (code == ABS)
    2265              :         {
    2266            0 :           set = gen_int_mode (0x7fffffff, SImode);
    2267            0 :           absneg_op = AND;
    2268              :         }
    2269              :       else
    2270              :         {
    2271            0 :           set = gen_int_mode (0x80000000, SImode);
    2272            0 :           absneg_op = XOR;
    2273              :         }
    2274            0 :       set = gen_rtx_fmt_ee (absneg_op, SImode, dst, set);
    2275            0 :       break;
    2276              : 
    2277            1 :     case E_DFmode:
    2278            1 :       if (TARGET_64BIT)
    2279              :         {
    2280            1 :           dst = gen_lowpart (DImode, operands[0]);
    2281            1 :           dst = gen_rtx_ZERO_EXTRACT (DImode, dst, const1_rtx, GEN_INT (63));
    2282              : 
    2283            1 :           if (code == ABS)
    2284            0 :             set = const0_rtx;
    2285              :           else
    2286            1 :             set = gen_rtx_NOT (DImode, dst);
    2287              :         }
    2288              :       else
    2289              :         {
    2290            0 :           dst = gen_highpart (SImode, operands[0]);
    2291              : 
    2292            0 :           if (code == ABS)
    2293              :             {
    2294            0 :               set = gen_int_mode (0x7fffffff, SImode);
    2295            0 :               absneg_op = AND;
    2296              :             }
    2297              :           else
    2298              :             {
    2299            0 :               set = gen_int_mode (0x80000000, SImode);
    2300            0 :               absneg_op = XOR;
    2301              :             }
    2302            0 :           set = gen_rtx_fmt_ee (absneg_op, SImode, dst, set);
    2303              :         }
    2304              :       break;
    2305              : 
    2306           23 :     case E_XFmode:
    2307           23 :       dst = gen_rtx_REG (SImode,
    2308           23 :                          REGNO (operands[0]) + (TARGET_64BIT ? 1 : 2));
    2309           23 :       if (code == ABS)
    2310              :         {
    2311            1 :           set = GEN_INT (0x7fff);
    2312            1 :           absneg_op = AND;
    2313              :         }
    2314              :       else
    2315              :         {
    2316           22 :           set = GEN_INT (0x8000);
    2317           22 :           absneg_op = XOR;
    2318              :         }
    2319           23 :       set = gen_rtx_fmt_ee (absneg_op, SImode, dst, set);
    2320           23 :       break;
    2321              : 
    2322            0 :     default:
    2323            0 :       gcc_unreachable ();
    2324              :     }
    2325              : 
    2326           24 :   set = gen_rtx_SET (dst, set);
    2327              : 
    2328           24 :   rtx clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
    2329           24 :   rtvec par = gen_rtvec (2, set, clob);
    2330              : 
    2331           24 :   emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
    2332           24 : }
    2333              : 
    2334              : /* Expand a copysign operation.  Special case operand 0 being a constant.  */
    2335              : 
    2336              : void
    2337        23278 : ix86_expand_copysign (rtx operands[])
    2338              : {
    2339        23278 :   machine_mode mode, vmode;
    2340        23278 :   rtx dest, vdest, op0, op1, mask, op2, op3;
    2341              : 
    2342        23278 :   mode = GET_MODE (operands[0]);
    2343              : 
    2344        23278 :   switch (mode)
    2345              :   {
    2346              :   case E_HFmode:
    2347              :     vmode = V8HFmode;
    2348              :     break;
    2349            0 :   case E_BFmode:
    2350            0 :     vmode = V8BFmode;
    2351            0 :     break;
    2352        11571 :   case E_SFmode:
    2353        11571 :     vmode = V4SFmode;
    2354        11571 :     break;
    2355        11567 :   case E_DFmode:
    2356        11567 :     vmode = V2DFmode;
    2357        11567 :     break;
    2358          127 :   case E_TFmode:
    2359          127 :     vmode = mode;
    2360          127 :     break;
    2361            0 :   default:
    2362            0 :     gcc_unreachable();
    2363              :   }
    2364              : 
    2365        23278 :   if (rtx_equal_p (operands[1], operands[2]))
    2366              :     {
    2367            0 :       emit_move_insn (operands[0], operands[1]);
    2368            0 :       return;
    2369              :     }
    2370              : 
    2371        23278 :   dest = operands[0];
    2372        23278 :   vdest = lowpart_subreg (vmode, dest, mode);
    2373        23278 :   if (vdest == NULL_RTX)
    2374            0 :     vdest = gen_reg_rtx (vmode);
    2375              :   else
    2376              :     dest = NULL_RTX;
    2377        23278 :   op1 = lowpart_subreg (vmode, force_reg (mode, operands[1]), mode);
    2378        46542 :   mask = ix86_build_signbit_mask (vmode, TARGET_AVX512F && mode != HFmode, 0);
    2379              : 
    2380        23278 :   if (CONST_DOUBLE_P (operands[2]))
    2381              :     {
    2382           88 :       if (real_isneg (CONST_DOUBLE_REAL_VALUE (operands[2])))
    2383              :         /* Simplify b = copysign (a, negative) to b = mask | a.  */
    2384           85 :         op1 = gen_rtx_IOR (vmode, mask, op1);
    2385              :       else
    2386              :         {
    2387              :           /* Simplify b = copysign (a, positive) to b = invert_mask & a.  */
    2388            3 :           rtx invert_mask
    2389            3 :             = ix86_build_signbit_mask (vmode,
    2390            3 :                                        TARGET_AVX512F && mode != HFmode,
    2391              :                                        true);
    2392            3 :           op1 = gen_rtx_AND (vmode, invert_mask, op1);
    2393              :         }
    2394           88 :       emit_move_insn (vdest, op1);
    2395           88 :       if (dest)
    2396            0 :         emit_move_insn (dest, lowpart_subreg (mode, vdest, vmode));
    2397              :       return;
    2398              :     }
    2399              :   else
    2400        23190 :     op0 = lowpart_subreg (vmode, force_reg (mode, operands[2]), mode);
    2401              : 
    2402        23190 :   op2 = gen_reg_rtx (vmode);
    2403        23190 :   op3 = gen_reg_rtx (vmode);
    2404        23190 :   rtx invert_mask;
    2405              :   /* NB: Generate vmovdqa, vpandn, vpand, vpor for AVX and generate pand,
    2406              :      pand, por for SSE.  */
    2407        23190 :   if (TARGET_AVX)
    2408           32 :     invert_mask = gen_rtx_NOT (vmode, mask);
    2409              :   else
    2410        46316 :     invert_mask = ix86_build_signbit_mask (vmode,
    2411              :                                            TARGET_AVX512F && mode != HFmode,
    2412              :                                            true);
    2413        23190 :   emit_move_insn (op2, gen_rtx_AND (vmode, invert_mask, op1));
    2414        23190 :   emit_move_insn (op3, gen_rtx_AND (vmode, mask, op0));
    2415        23190 :   emit_move_insn (vdest, gen_rtx_IOR (vmode, op2, op3));
    2416        23190 :   if (dest)
    2417            0 :     emit_move_insn (dest, lowpart_subreg (mode, vdest, vmode));
    2418              : }
    2419              : 
    2420              : /* Expand an xorsign operation.  */
    2421              : 
    2422              : void
    2423           20 : ix86_expand_xorsign (rtx operands[])
    2424              : {
    2425           20 :   machine_mode mode, vmode;
    2426           20 :   rtx dest, vdest, op0, op1, mask, x, temp;
    2427              : 
    2428           20 :   dest = operands[0];
    2429           20 :   op0 = operands[1];
    2430           20 :   op1 = operands[2];
    2431              : 
    2432           20 :   mode = GET_MODE (dest);
    2433              : 
    2434           20 :   switch (mode)
    2435              :   {
    2436              :   case E_HFmode:
    2437              :     vmode = V8HFmode;
    2438              :     break;
    2439              :   case E_BFmode:
    2440              :     vmode = V8BFmode;
    2441              :     break;
    2442              :   case E_SFmode:
    2443              :     vmode = V4SFmode;
    2444              :     break;
    2445              :   case E_DFmode:
    2446              :     vmode = V2DFmode;
    2447              :     break;
    2448            0 :   default:
    2449            0 :     gcc_unreachable ();
    2450           20 :     break;
    2451              :   }
    2452              : 
    2453           20 :   temp = gen_reg_rtx (vmode);
    2454           20 :   mask = ix86_build_signbit_mask (vmode, 0, 0);
    2455              : 
    2456           20 :   op1 = lowpart_subreg (vmode, force_reg (mode, op1), mode);
    2457           20 :   x = gen_rtx_AND (vmode, op1, mask);
    2458           20 :   emit_insn (gen_rtx_SET (temp, x));
    2459              : 
    2460           20 :   op0 = lowpart_subreg (vmode, force_reg (mode, op0), mode);
    2461           20 :   x = gen_rtx_XOR (vmode, temp, op0);
    2462              : 
    2463           20 :   vdest = lowpart_subreg (vmode, dest, mode);
    2464           20 :   if (vdest == NULL_RTX)
    2465            0 :     vdest = gen_reg_rtx (vmode);
    2466              :   else
    2467              :     dest = NULL_RTX;
    2468           20 :   emit_insn (gen_rtx_SET (vdest, x));
    2469              : 
    2470           20 :   if (dest)
    2471            0 :     emit_move_insn (dest, lowpart_subreg (mode, vdest, vmode));
    2472           20 : }
    2473              : 
    2474              : static rtx ix86_expand_compare (enum rtx_code code, rtx op0, rtx op1);
    2475              : 
    2476              : void
    2477      6767784 : ix86_expand_branch (enum rtx_code code, rtx op0, rtx op1, rtx label)
    2478              : {
    2479      6767784 :   machine_mode mode = GET_MODE (op0);
    2480      6767784 :   rtx tmp;
    2481              : 
    2482              :   /* Handle special case - vector comparison with boolean result, transform
    2483              :      it using ptest instruction or vpcmpeq + kortest.  */
    2484      6767784 :   if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
    2485      6748362 :       || (mode == TImode && !TARGET_64BIT)
    2486      6748362 :       || mode == OImode
    2487     13516146 :       || GET_MODE_SIZE (mode) == 64)
    2488              :     {
    2489        19422 :       unsigned msize = GET_MODE_SIZE (mode);
    2490        19422 :       machine_mode p_mode
    2491        19422 :         = msize == 64 ? V16SImode : msize == 32 ? V4DImode : V2DImode;
    2492              :       /* kortest set CF when result is 0xFFFF (op0 == op1).  */
    2493        19422 :       rtx flag = gen_rtx_REG (msize == 64 ? CCCmode : CCZmode, FLAGS_REG);
    2494              : 
    2495        19422 :       gcc_assert (code == EQ || code == NE);
    2496              : 
    2497              :       /* Using vpcmpeq zmm zmm k + kortest for 512-bit vectors.  */
    2498        19422 :       if (msize == 64)
    2499              :         {
    2500         2451 :           if (mode != V16SImode)
    2501              :             {
    2502         2451 :               op0 = lowpart_subreg (p_mode, force_reg (mode, op0), mode);
    2503         2451 :               op1 = lowpart_subreg (p_mode, force_reg (mode, op1), mode);
    2504              :             }
    2505              : 
    2506         2451 :           tmp = gen_reg_rtx (HImode);
    2507         2451 :           emit_insn (gen_avx512f_cmpv16si3 (tmp, op0, op1, GEN_INT (0)));
    2508         2451 :           emit_insn (gen_kortesthi_ccc (tmp, tmp));
    2509              :         }
    2510              :       /* Using ptest for 128/256-bit vectors.  */
    2511              :       else
    2512              :         {
    2513        16971 :           if (GET_MODE_CLASS (mode) != MODE_VECTOR_INT)
    2514              :             {
    2515            0 :               op0 = lowpart_subreg (p_mode, force_reg (mode, op0), mode);
    2516            0 :               op1 = lowpart_subreg (p_mode, force_reg (mode, op1), mode);
    2517            0 :               mode = p_mode;
    2518              :             }
    2519              : 
    2520              :           /* Generate XOR since we can't check that one operand is zero
    2521              :              vector.  */
    2522        16971 :           tmp = gen_reg_rtx (mode);
    2523        16971 :           rtx ops[3] = { tmp, op0, op1 };
    2524        16971 :           ix86_expand_vector_logical_operator (XOR, mode, ops);
    2525        16971 :           tmp = gen_lowpart (p_mode, tmp);
    2526        16971 :           emit_insn (gen_rtx_SET (gen_rtx_REG (CCZmode, FLAGS_REG),
    2527              :                                   gen_rtx_UNSPEC (CCZmode,
    2528              :                                                   gen_rtvec (2, tmp, tmp),
    2529              :                                                   UNSPEC_PTEST)));
    2530              :         }
    2531        19422 :       tmp = gen_rtx_fmt_ee (code, VOIDmode, flag, const0_rtx);
    2532        19422 :       tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
    2533              :                                   gen_rtx_LABEL_REF (VOIDmode, label),
    2534              :                                   pc_rtx);
    2535        19422 :       emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
    2536        19422 :       return;
    2537              :     }
    2538              : 
    2539      6748362 :   switch (mode)
    2540              :     {
    2541      6713689 :     case E_HFmode:
    2542      6713689 :     case E_SFmode:
    2543      6713689 :     case E_DFmode:
    2544      6713689 :     case E_XFmode:
    2545      6713689 :     case E_QImode:
    2546      6713689 :     case E_HImode:
    2547      6713689 :     case E_SImode:
    2548      6713689 :       simple:
    2549      6713689 :       tmp = ix86_expand_compare (code, op0, op1);
    2550      6713689 :       tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
    2551              :                                   gen_rtx_LABEL_REF (VOIDmode, label),
    2552              :                                   pc_rtx);
    2553      6713689 :       emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
    2554      6713689 :       return;
    2555              : 
    2556            7 :     case E_BFmode:
    2557            7 :       gcc_assert (TARGET_AVX10_2 && !flag_trapping_math);
    2558            7 :       goto simple;
    2559              : 
    2560      2718432 :     case E_DImode:
    2561      2718432 :       if (TARGET_64BIT)
    2562      2687754 :         goto simple;
    2563              :       /* FALLTHRU */
    2564        93911 :     case E_TImode:
    2565              :       /* DI and TI mode equality/inequality comparisons may be performed
    2566              :          on SSE registers.  Avoid splitting them, except when optimizing
    2567              :          for size.  */
    2568        93911 :       if ((code == EQ || code == NE)
    2569        93911 :           && !optimize_insn_for_size_p ())
    2570        59238 :         goto simple;
    2571              : 
    2572              :       /* Expand DImode branch into multiple compare+branch.  */
    2573        34673 :       {
    2574        34673 :         rtx lo[2], hi[2];
    2575        34673 :         rtx_code_label *label2;
    2576        34673 :         enum rtx_code code1, code2, code3;
    2577        34673 :         machine_mode submode;
    2578              : 
    2579        34673 :         if (CONSTANT_P (op0) && !CONSTANT_P (op1))
    2580              :           {
    2581            0 :             std::swap (op0, op1);
    2582            0 :             code = swap_condition (code);
    2583              :           }
    2584              : 
    2585        34673 :         split_double_mode (mode, &op0, 1, lo+0, hi+0);
    2586        34673 :         split_double_mode (mode, &op1, 1, lo+1, hi+1);
    2587              : 
    2588        34673 :         submode = mode == DImode ? SImode : DImode;
    2589              : 
    2590              :         /* If we are doing less-than or greater-or-equal-than,
    2591              :            op1 is a constant and the low word is zero, then we can just
    2592              :            examine the high word.  Similarly for low word -1 and
    2593              :            less-or-equal-than or greater-than.  */
    2594              : 
    2595        34673 :         if (CONST_INT_P (hi[1]))
    2596        21070 :           switch (code)
    2597              :             {
    2598        11426 :             case LT: case LTU: case GE: case GEU:
    2599        11426 :               if (lo[1] == const0_rtx)
    2600              :                 {
    2601        10993 :                   ix86_expand_branch (code, hi[0], hi[1], label);
    2602        10993 :                   return;
    2603              :                 }
    2604              :               break;
    2605         8098 :             case LE: case LEU: case GT: case GTU:
    2606         8098 :               if (lo[1] == constm1_rtx)
    2607              :                 {
    2608          530 :                   ix86_expand_branch (code, hi[0], hi[1], label);
    2609          530 :                   return;
    2610              :                 }
    2611              :               break;
    2612              :             default:
    2613              :               break;
    2614              :             }
    2615              : 
    2616              :         /* Emulate comparisons that do not depend on Zero flag with
    2617              :            double-word subtraction.  Note that only Overflow, Sign
    2618              :            and Carry flags are valid, so swap arguments and condition
    2619              :            of comparisons that would otherwise test Zero flag.  */
    2620              : 
    2621        23150 :         switch (code)
    2622              :           {
    2623        12922 :           case LE: case LEU: case GT: case GTU:
    2624        12922 :             std::swap (lo[0], lo[1]);
    2625        12922 :             std::swap (hi[0], hi[1]);
    2626        12922 :             code = swap_condition (code);
    2627              :             /* FALLTHRU */
    2628              : 
    2629        19661 :           case LT: case LTU: case GE: case GEU:
    2630        19661 :             {
    2631        19661 :               bool uns = (code == LTU || code == GEU);
    2632         4113 :               rtx (*sbb_insn) (machine_mode, rtx, rtx, rtx)
    2633        19661 :                 = uns ? gen_sub3_carry_ccc : gen_sub3_carry_ccgz;
    2634              : 
    2635        19661 :               if (!nonimmediate_operand (lo[0], submode))
    2636         7568 :                 lo[0] = force_reg (submode, lo[0]);
    2637        19661 :               if (!x86_64_general_operand (lo[1], submode))
    2638            0 :                 lo[1] = force_reg (submode, lo[1]);
    2639              : 
    2640        19661 :               if (!register_operand (hi[0], submode))
    2641         8353 :                 hi[0] = force_reg (submode, hi[0]);
    2642        15548 :               if ((uns && !nonimmediate_operand (hi[1], submode))
    2643        19661 :                   || (!uns && !x86_64_general_operand (hi[1], submode)))
    2644          315 :                 hi[1] = force_reg (submode, hi[1]);
    2645              : 
    2646        19661 :               emit_insn (gen_cmp_1 (submode, lo[0], lo[1]));
    2647              : 
    2648        19661 :               tmp = gen_rtx_SCRATCH (submode);
    2649        19661 :               emit_insn (sbb_insn (submode, tmp, hi[0], hi[1]));
    2650              : 
    2651        23774 :               tmp = gen_rtx_REG (uns ? CCCmode : CCGZmode, FLAGS_REG);
    2652        19661 :               ix86_expand_branch (code, tmp, const0_rtx, label);
    2653        19661 :               return;
    2654              :             }
    2655              : 
    2656         3489 :           default:
    2657         3489 :             break;
    2658              :           }
    2659              : 
    2660              :         /* Otherwise, we need two or three jumps.  */
    2661              : 
    2662         3489 :         label2 = gen_label_rtx ();
    2663              : 
    2664         3489 :         code1 = code;
    2665         3489 :         code2 = swap_condition (code);
    2666         3489 :         code3 = unsigned_condition (code);
    2667              : 
    2668         3489 :         switch (code)
    2669              :           {
    2670              :           case LT: case GT: case LTU: case GTU:
    2671              :             break;
    2672              : 
    2673              :           case LE:   code1 = LT;  code2 = GT;  break;
    2674              :           case GE:   code1 = GT;  code2 = LT;  break;
    2675              :           case LEU:  code1 = LTU; code2 = GTU; break;
    2676              :           case GEU:  code1 = GTU; code2 = LTU; break;
    2677              : 
    2678              :           case EQ:   code1 = UNKNOWN; code2 = NE;  break;
    2679              :           case NE:   code2 = UNKNOWN; break;
    2680              : 
    2681            0 :           default:
    2682            0 :             gcc_unreachable ();
    2683              :           }
    2684              : 
    2685              :         /*
    2686              :          * a < b =>
    2687              :          *    if (hi(a) < hi(b)) goto true;
    2688              :          *    if (hi(a) > hi(b)) goto false;
    2689              :          *    if (lo(a) < lo(b)) goto true;
    2690              :          *  false:
    2691              :          */
    2692              : 
    2693              :         if (code1 != UNKNOWN)
    2694         2593 :           ix86_expand_branch (code1, hi[0], hi[1], label);
    2695         3489 :         if (code2 != UNKNOWN)
    2696          896 :           ix86_expand_branch (code2, hi[0], hi[1], label2);
    2697              : 
    2698         3489 :         ix86_expand_branch (code3, lo[0], lo[1], label);
    2699              : 
    2700         3489 :         if (code2 != UNKNOWN)
    2701          896 :           emit_label (label2);
    2702              :         return;
    2703              :       }
    2704              : 
    2705        20161 :     default:
    2706        20161 :       gcc_assert (GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC);
    2707        20161 :       goto simple;
    2708              :     }
    2709              : }
    2710              : 
    2711              : /* Figure out whether to use unordered fp comparisons.  */
    2712              : 
    2713              : static bool
    2714      1176230 : ix86_unordered_fp_compare (enum rtx_code code)
    2715              : {
    2716      1176230 :   if (!TARGET_IEEE_FP)
    2717              :     return false;
    2718              : 
    2719      1171646 :   switch (code)
    2720              :     {
    2721              :     case LT:
    2722              :     case LE:
    2723              :     case GT:
    2724              :     case GE:
    2725              :     case LTGT:
    2726              :       return false;
    2727              : 
    2728              :     case EQ:
    2729              :     case NE:
    2730              : 
    2731              :     case UNORDERED:
    2732              :     case ORDERED:
    2733              :     case UNLT:
    2734              :     case UNLE:
    2735              :     case UNGT:
    2736              :     case UNGE:
    2737              :     case UNEQ:
    2738              :       return true;
    2739              : 
    2740            0 :     default:
    2741            0 :       gcc_unreachable ();
    2742              :     }
    2743              : }
    2744              : 
    2745              : /* Return a comparison we can do and that it is equivalent to
    2746              :    swap_condition (code) apart possibly from orderedness.
    2747              :    But, never change orderedness if TARGET_IEEE_FP, returning
    2748              :    UNKNOWN in that case if necessary.  */
    2749              : 
    2750              : static enum rtx_code
    2751        39957 : ix86_fp_swap_condition (enum rtx_code code)
    2752              : {
    2753        39957 :   switch (code)
    2754              :     {
    2755         2865 :     case GT:                   /* GTU - CF=0 & ZF=0 */
    2756         2865 :       return TARGET_IEEE_FP ? UNKNOWN : UNLT;
    2757          544 :     case GE:                   /* GEU - CF=0 */
    2758          544 :       return TARGET_IEEE_FP ? UNKNOWN : UNLE;
    2759          524 :     case UNLT:                 /* LTU - CF=1 */
    2760          524 :       return TARGET_IEEE_FP ? UNKNOWN : GT;
    2761         6365 :     case UNLE:                 /* LEU - CF=1 | ZF=1 */
    2762         6365 :       return TARGET_IEEE_FP ? UNKNOWN : GE;
    2763        29659 :     default:
    2764        29659 :       return swap_condition (code);
    2765              :     }
    2766              : }
    2767              : 
    2768              : /* Return cost of comparison CODE using the best strategy for performance.
    2769              :    All following functions do use number of instructions as a cost metrics.
    2770              :    In future this should be tweaked to compute bytes for optimize_size and
    2771              :    take into account performance of various instructions on various CPUs.  */
    2772              : 
    2773              : static int
    2774      1175064 : ix86_fp_comparison_cost (enum rtx_code code)
    2775              : {
    2776      1175064 :   int arith_cost;
    2777              : 
    2778              :   /* The cost of code using bit-twiddling on %ah.  */
    2779      1175064 :   switch (code)
    2780              :     {
    2781              :     case UNLE:
    2782              :     case UNLT:
    2783              :     case LTGT:
    2784              :     case GT:
    2785              :     case GE:
    2786              :     case UNORDERED:
    2787              :     case ORDERED:
    2788              :     case UNEQ:
    2789              :       arith_cost = 4;
    2790              :       break;
    2791        90314 :     case LT:
    2792        90314 :     case NE:
    2793        90314 :     case EQ:
    2794        90314 :     case UNGE:
    2795        90314 :       arith_cost = TARGET_IEEE_FP ? 5 : 4;
    2796              :       break;
    2797        27439 :     case LE:
    2798        27439 :     case UNGT:
    2799      1085584 :       arith_cost = TARGET_IEEE_FP ? 6 : 4;
    2800              :       break;
    2801            0 :     default:
    2802            0 :       gcc_unreachable ();
    2803              :     }
    2804              : 
    2805      1175064 :   switch (ix86_fp_comparison_strategy (code))
    2806              :     {
    2807      1175064 :     case IX86_FPCMP_COMI:
    2808      1175064 :       return arith_cost > 4 ? 3 : 2;
    2809            0 :     case IX86_FPCMP_SAHF:
    2810            0 :       return arith_cost > 4 ? 4 : 3;
    2811              :     default:
    2812              :       return arith_cost;
    2813              :     }
    2814              : }
    2815              : 
    2816              : /* Swap, force into registers, or otherwise massage the two operands
    2817              :    to a fp comparison.  The operands are updated in place; the new
    2818              :    comparison code is returned.  */
    2819              : 
    2820              : static enum rtx_code
    2821       587532 : ix86_prepare_fp_compare_args (enum rtx_code code, rtx *pop0, rtx *pop1)
    2822              : {
    2823       587603 :   bool unordered_compare = ix86_unordered_fp_compare (code);
    2824       587603 :   rtx op0 = *pop0, op1 = *pop1;
    2825       587603 :   machine_mode op_mode = GET_MODE (op0);
    2826       587603 :   bool is_sse = SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (op_mode);
    2827              : 
    2828       585199 :   if (op_mode == BFmode && (!TARGET_AVX10_2 || flag_trapping_math))
    2829              :     {
    2830           71 :       rtx op = gen_lowpart (HImode, op0);
    2831           71 :       if (CONST_INT_P (op))
    2832            0 :         op = simplify_const_unary_operation (FLOAT_EXTEND, SFmode,
    2833              :                                              op0, BFmode);
    2834              :       else
    2835              :         {
    2836           71 :           rtx t1 = gen_reg_rtx (SImode);
    2837           71 :           emit_insn (gen_zero_extendhisi2 (t1, op));
    2838           71 :           emit_insn (gen_ashlsi3 (t1, t1, GEN_INT (16)));
    2839           71 :           op = gen_lowpart (SFmode, t1);
    2840              :         }
    2841           71 :       *pop0 = op;
    2842           71 :       op = gen_lowpart (HImode, op1);
    2843           71 :       if (CONST_INT_P (op))
    2844            6 :         op = simplify_const_unary_operation (FLOAT_EXTEND, SFmode,
    2845              :                                              op1, BFmode);
    2846              :       else
    2847              :         {
    2848           65 :           rtx t1 = gen_reg_rtx (SImode);
    2849           65 :           emit_insn (gen_zero_extendhisi2 (t1, op));
    2850           65 :           emit_insn (gen_ashlsi3 (t1, t1, GEN_INT (16)));
    2851           65 :           op = gen_lowpart (SFmode, t1);
    2852              :         }
    2853           71 :       *pop1 = op;
    2854           71 :       return ix86_prepare_fp_compare_args (code, pop0, pop1);
    2855              :     }
    2856              : 
    2857              :   /* All of the unordered compare instructions only work on registers.
    2858              :      The same is true of the fcomi compare instructions.  The XFmode
    2859              :      compare instructions require registers except when comparing
    2860              :      against zero or when converting operand 1 from fixed point to
    2861              :      floating point.  */
    2862              : 
    2863       587532 :   if (!is_sse
    2864       587532 :       && (unordered_compare
    2865         8306 :           || (op_mode == XFmode
    2866        10507 :               && ! (standard_80387_constant_p (op0) == 1
    2867         5250 :                     || standard_80387_constant_p (op1) == 1)
    2868         4806 :               && GET_CODE (op1) != FLOAT)
    2869         3500 :           || ix86_fp_comparison_strategy (code) == IX86_FPCMP_COMI))
    2870              :     {
    2871       148637 :       op0 = force_reg (op_mode, op0);
    2872       148637 :       op1 = force_reg (op_mode, op1);
    2873              :     }
    2874              :   else
    2875              :     {
    2876              :       /* %%% We only allow op1 in memory; op0 must be st(0).  So swap
    2877              :          things around if they appear profitable, otherwise force op0
    2878              :          into a register.  */
    2879              : 
    2880       438895 :       if (standard_80387_constant_p (op0) == 0
    2881       438895 :           || (MEM_P (op0)
    2882        59835 :               && ! (standard_80387_constant_p (op1) == 0
    2883        44076 :                     || MEM_P (op1))))
    2884              :         {
    2885        39957 :           enum rtx_code new_code = ix86_fp_swap_condition (code);
    2886        39957 :           if (new_code != UNKNOWN)
    2887              :             {
    2888              :               std::swap (op0, op1);
    2889       438895 :               code = new_code;
    2890              :             }
    2891              :         }
    2892              : 
    2893       438895 :       if (!REG_P (op0))
    2894        55502 :         op0 = force_reg (op_mode, op0);
    2895              : 
    2896       438895 :       if (CONSTANT_P (op1))
    2897              :         {
    2898       195658 :           int tmp = standard_80387_constant_p (op1);
    2899       195658 :           if (tmp == 0)
    2900        75517 :             op1 = validize_mem (force_const_mem (op_mode, op1));
    2901       120141 :           else if (tmp == 1)
    2902              :             {
    2903        65965 :               if (TARGET_CMOVE)
    2904        65965 :                 op1 = force_reg (op_mode, op1);
    2905              :             }
    2906              :           else
    2907        54176 :             op1 = force_reg (op_mode, op1);
    2908              :         }
    2909              :     }
    2910              : 
    2911              :   /* Try to rearrange the comparison to make it cheaper.  */
    2912       587532 :   if (ix86_fp_comparison_cost (code)
    2913       587532 :       > ix86_fp_comparison_cost (swap_condition (code))
    2914       587532 :       && (REG_P (op1) || can_create_pseudo_p ()))
    2915              :     {
    2916            0 :       std::swap (op0, op1);
    2917            0 :       code = swap_condition (code);
    2918            0 :       if (!REG_P (op0))
    2919            0 :         op0 = force_reg (op_mode, op0);
    2920              :     }
    2921              : 
    2922       587532 :   *pop0 = op0;
    2923       587532 :   *pop1 = op1;
    2924       587532 :   return code;
    2925              : }
    2926              : 
    2927              : /* Generate insn patterns to do a floating point compare of OPERANDS.  */
    2928              : 
    2929              : static rtx
    2930       587532 : ix86_expand_fp_compare (enum rtx_code code, rtx op0, rtx op1)
    2931              : {
    2932       587532 :   bool unordered_compare = ix86_unordered_fp_compare (code);
    2933       587532 :   machine_mode cmp_mode;
    2934       587532 :   rtx tmp, scratch;
    2935              : 
    2936       587532 :   code = ix86_prepare_fp_compare_args (code, &op0, &op1);
    2937       587532 :   machine_mode op_mode = GET_MODE (op0);
    2938              : 
    2939       587532 :   tmp = gen_rtx_COMPARE (CCFPmode, op0, op1);
    2940       587532 :   if (unordered_compare)
    2941       507031 :     tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_NOTRAP);
    2942              : 
    2943              :   /* Do fcomi/sahf based test when profitable.  */
    2944       587532 :   switch (ix86_fp_comparison_strategy (code))
    2945              :     {
    2946       587532 :     case IX86_FPCMP_COMI:
    2947       587532 :       tmp = gen_rtx_COMPARE (CCFPmode, op0, op1);
    2948              :       /* VCOMX/VUCOMX only have DF/SF/HF mode instructions.  */
    2949       587532 :       if (TARGET_AVX10_2
    2950         1792 :           && (code == EQ || code == NE)
    2951          982 :           && (op_mode == HFmode || op_mode == SFmode || op_mode == DFmode))
    2952          972 :         tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_OPTCOMX);
    2953              :       /* We only have vcomisbf16, No vcomubf16 nor vcomxbf16 */
    2954       587532 :       if (op_mode != BFmode && unordered_compare)
    2955       507023 :         tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_NOTRAP);
    2956       587532 :       cmp_mode = CCFPmode;
    2957       587532 :       emit_insn (gen_rtx_SET (gen_rtx_REG (CCFPmode, FLAGS_REG), tmp));
    2958       587532 :       break;
    2959              : 
    2960            0 :     case IX86_FPCMP_SAHF:
    2961            0 :       cmp_mode = CCFPmode;
    2962            0 :       tmp = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW);
    2963            0 :       scratch = gen_reg_rtx (HImode);
    2964            0 :       emit_insn (gen_rtx_SET (scratch, tmp));
    2965            0 :       emit_insn (gen_x86_sahf_1 (scratch));
    2966            0 :       break;
    2967              : 
    2968            0 :     case IX86_FPCMP_ARITH:
    2969            0 :       cmp_mode = CCNOmode;
    2970            0 :       tmp = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW);
    2971            0 :       scratch = gen_reg_rtx (HImode);
    2972            0 :       emit_insn (gen_rtx_SET (scratch, tmp));
    2973              : 
    2974              :       /* In the unordered case, we have to check C2 for NaN's, which
    2975              :          doesn't happen to work out to anything nice combination-wise.
    2976              :          So do some bit twiddling on the value we've got in AH to come
    2977              :          up with an appropriate set of condition codes.  */
    2978              : 
    2979            0 :       switch (code)
    2980              :         {
    2981            0 :         case GT:
    2982            0 :         case UNGT:
    2983            0 :           if (code == GT || !TARGET_IEEE_FP)
    2984              :             {
    2985            0 :               emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x45)));
    2986            0 :               code = EQ;
    2987              :             }
    2988              :           else
    2989              :             {
    2990            0 :               emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
    2991            0 :               emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
    2992            0 :               emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x44)));
    2993            0 :               cmp_mode = CCmode;
    2994            0 :               code = GEU;
    2995              :             }
    2996              :           break;
    2997            0 :         case LT:
    2998            0 :         case UNLT:
    2999            0 :           if (code == LT && TARGET_IEEE_FP)
    3000              :             {
    3001            0 :               emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
    3002            0 :               emit_insn (gen_cmpqi_ext_3 (scratch, const1_rtx));
    3003            0 :               cmp_mode = CCmode;
    3004            0 :               code = EQ;
    3005              :             }
    3006              :           else
    3007              :             {
    3008            0 :               emit_insn (gen_testqi_ext_1_ccno (scratch, const1_rtx));
    3009            0 :               code = NE;
    3010              :             }
    3011              :           break;
    3012            0 :         case GE:
    3013            0 :         case UNGE:
    3014            0 :           if (code == GE || !TARGET_IEEE_FP)
    3015              :             {
    3016            0 :               emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x05)));
    3017            0 :               code = EQ;
    3018              :             }
    3019              :           else
    3020              :             {
    3021            0 :               emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
    3022            0 :               emit_insn (gen_xorqi_ext_1_cc (scratch, scratch, const1_rtx));
    3023            0 :               code = NE;
    3024              :             }
    3025              :           break;
    3026            0 :         case LE:
    3027            0 :         case UNLE:
    3028            0 :           if (code == LE && TARGET_IEEE_FP)
    3029              :             {
    3030            0 :               emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
    3031            0 :               emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
    3032            0 :               emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
    3033            0 :               cmp_mode = CCmode;
    3034            0 :               code = LTU;
    3035              :             }
    3036              :           else
    3037              :             {
    3038            0 :               emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x45)));
    3039            0 :               code = NE;
    3040              :             }
    3041              :           break;
    3042            0 :         case EQ:
    3043            0 :         case UNEQ:
    3044            0 :           if (code == EQ && TARGET_IEEE_FP)
    3045              :             {
    3046            0 :               emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
    3047            0 :               emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
    3048            0 :               cmp_mode = CCmode;
    3049            0 :               code = EQ;
    3050              :             }
    3051              :           else
    3052              :             {
    3053            0 :               emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x40)));
    3054            0 :               code = NE;
    3055              :             }
    3056              :           break;
    3057            0 :         case NE:
    3058            0 :         case LTGT:
    3059            0 :           if (code == NE && TARGET_IEEE_FP)
    3060              :             {
    3061            0 :               emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
    3062            0 :               emit_insn (gen_xorqi_ext_1_cc (scratch, scratch,
    3063              :                                              GEN_INT (0x40)));
    3064            0 :               code = NE;
    3065              :             }
    3066              :           else
    3067              :             {
    3068            0 :               emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x40)));
    3069            0 :               code = EQ;
    3070              :             }
    3071              :           break;
    3072              : 
    3073            0 :         case UNORDERED:
    3074            0 :           emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x04)));
    3075            0 :           code = NE;
    3076            0 :           break;
    3077            0 :         case ORDERED:
    3078            0 :           emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x04)));
    3079            0 :           code = EQ;
    3080            0 :           break;
    3081              : 
    3082            0 :         default:
    3083            0 :           gcc_unreachable ();
    3084              :         }
    3085              :         break;
    3086              : 
    3087            0 :     default:
    3088            0 :       gcc_unreachable();
    3089              :     }
    3090              : 
    3091              :   /* Return the test that should be put into the flags user, i.e.
    3092              :      the bcc, scc, or cmov instruction.  */
    3093       587532 :   return gen_rtx_fmt_ee (code, VOIDmode,
    3094              :                          gen_rtx_REG (cmp_mode, FLAGS_REG),
    3095              :                          const0_rtx);
    3096              : }
    3097              : 
    3098              : /* Generate insn patterns to do an integer compare of OPERANDS.  */
    3099              : 
    3100              : static rtx
    3101      7170282 : ix86_expand_int_compare (enum rtx_code code, rtx op0, rtx op1)
    3102              : {
    3103      7170282 :   machine_mode cmpmode;
    3104      7170282 :   rtx tmp, flags;
    3105              : 
    3106              :   /* Swap operands to emit carry flag comparison.  */
    3107      7170282 :   if ((code == GTU || code == LEU)
    3108      7170282 :       && nonimmediate_operand (op1, VOIDmode))
    3109              :     {
    3110       145211 :       std::swap (op0, op1);
    3111       145211 :       code = swap_condition (code);
    3112              :     }
    3113              : 
    3114      7170282 :   cmpmode = SELECT_CC_MODE (code, op0, op1);
    3115      7170282 :   flags = gen_rtx_REG (cmpmode, FLAGS_REG);
    3116              : 
    3117              :   /* Attempt to use PTEST, if available, when testing vector modes for
    3118              :      equality/inequality against zero.  */
    3119      7170282 :   if (op1 == const0_rtx
    3120      2974141 :       && SUBREG_P (op0)
    3121        24503 :       && cmpmode == CCZmode
    3122        11450 :       && SUBREG_BYTE (op0) == 0
    3123         9775 :       && REG_P (SUBREG_REG (op0))
    3124         9775 :       && VECTOR_MODE_P (GET_MODE (SUBREG_REG (op0)))
    3125            8 :       && TARGET_SSE4_1
    3126            2 :       && GET_MODE (op0) == TImode
    3127      7170286 :       && GET_MODE_SIZE (GET_MODE (SUBREG_REG (op0))) == 16)
    3128              :     {
    3129            2 :       tmp = SUBREG_REG (op0);
    3130            2 :       if (GET_MODE (tmp) == V8HFmode || GET_MODE (tmp) == V8BFmode)
    3131            1 :         tmp = gen_lowpart (V8HImode, tmp);
    3132            2 :       tmp = gen_rtx_UNSPEC (CCZmode, gen_rtvec (2, tmp, tmp), UNSPEC_PTEST);
    3133              :     }
    3134              :   else
    3135      7170280 :     tmp = gen_rtx_COMPARE (cmpmode, op0, op1);
    3136              : 
    3137              :   /* This is very simple, but making the interface the same as in the
    3138              :      FP case makes the rest of the code easier.  */
    3139      7170282 :   emit_insn (gen_rtx_SET (flags, tmp));
    3140              : 
    3141              :   /* Return the test that should be put into the flags user, i.e.
    3142              :      the bcc, scc, or cmov instruction.  */
    3143      7170282 :   return gen_rtx_fmt_ee (code, VOIDmode, flags, const0_rtx);
    3144              : }
    3145              : 
    3146              : static rtx
    3147      7909763 : ix86_expand_compare (enum rtx_code code, rtx op0, rtx op1)
    3148              : {
    3149      7909763 :   rtx ret;
    3150              : 
    3151      7909763 :   if (GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC)
    3152       153885 :     ret = gen_rtx_fmt_ee (code, VOIDmode, op0, op1);
    3153              : 
    3154      7755878 :   else if (SCALAR_FLOAT_MODE_P (GET_MODE (op0)))
    3155              :     {
    3156       585596 :       gcc_assert (!DECIMAL_FLOAT_MODE_P (GET_MODE (op0)));
    3157       585596 :       ret = ix86_expand_fp_compare (code, op0, op1);
    3158              :     }
    3159              :   else
    3160      7170282 :     ret = ix86_expand_int_compare (code, op0, op1);
    3161              : 
    3162      7909763 :   return ret;
    3163              : }
    3164              : 
    3165              : void
    3166       629906 : ix86_expand_setcc (rtx dest, enum rtx_code code, rtx op0, rtx op1)
    3167              : {
    3168       629906 :   rtx ret;
    3169              : 
    3170       629906 :   gcc_assert (GET_MODE (dest) == QImode);
    3171              : 
    3172       629906 :   ret = ix86_expand_compare (code, op0, op1);
    3173       629906 :   PUT_MODE (ret, QImode);
    3174       629906 :   emit_insn (gen_rtx_SET (dest, ret));
    3175       629906 : }
    3176              : 
    3177              : /* Expand floating point op0 <=> op1, i.e.
    3178              :    dest = op0 == op1 ? 0 : op0 < op1 ? -1 : op0 > op1 ? 1 : -128.  */
    3179              : 
    3180              : void
    3181          174 : ix86_expand_fp_spaceship (rtx dest, rtx op0, rtx op1, rtx op2)
    3182              : {
    3183          174 :   gcc_checking_assert (ix86_fp_comparison_strategy (GT) != IX86_FPCMP_ARITH);
    3184          174 :   rtx zero = NULL_RTX;
    3185          174 :   if (op2 != const0_rtx
    3186           52 :       && (TARGET_IEEE_FP || TARGET_ZERO_EXTEND_WITH_AND)
    3187           34 :       && GET_MODE (dest) == SImode)
    3188           34 :     zero = force_reg (SImode, const0_rtx);
    3189          174 :   rtx gt = ix86_expand_fp_compare (GT, op0, op1);
    3190          174 :   rtx_code_label *l0 = op2 == const0_rtx ? gen_label_rtx () : nullptr;
    3191          174 :   rtx_code_label *l1 = op2 == const0_rtx ? gen_label_rtx () : nullptr;
    3192          174 :   rtx_code_label *l2 = TARGET_IEEE_FP ? gen_label_rtx () : nullptr;
    3193          174 :   rtx_code_label *lend = gen_label_rtx ();
    3194          174 :   rtx tmp;
    3195          174 :   rtx_insn *jmp;
    3196          174 :   if (l2)
    3197              :     {
    3198          137 :       rtx un = gen_rtx_fmt_ee (UNORDERED, VOIDmode,
    3199              :                                gen_rtx_REG (CCFPmode, FLAGS_REG), const0_rtx);
    3200          137 :       tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, un,
    3201              :                                   gen_rtx_LABEL_REF (VOIDmode, l2), pc_rtx);
    3202          137 :       jmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
    3203          137 :       add_reg_br_prob_note (jmp, profile_probability:: very_unlikely ());
    3204              :     }
    3205          174 :   if (op2 == const0_rtx)
    3206              :     {
    3207          122 :       rtx eq = gen_rtx_fmt_ee (UNEQ, VOIDmode,
    3208              :                                gen_rtx_REG (CCFPmode, FLAGS_REG), const0_rtx);
    3209          122 :       tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, eq,
    3210              :                                   gen_rtx_LABEL_REF (VOIDmode, l0), pc_rtx);
    3211          122 :       jmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
    3212          122 :       add_reg_br_prob_note (jmp, profile_probability::unlikely ());
    3213          122 :       tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, gt,
    3214              :                                   gen_rtx_LABEL_REF (VOIDmode, l1), pc_rtx);
    3215          122 :       jmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
    3216          122 :       add_reg_br_prob_note (jmp, profile_probability::even ());
    3217          122 :       emit_move_insn (dest, constm1_rtx);
    3218          122 :       emit_jump (lend);
    3219          122 :       emit_label (l0);
    3220          122 :       emit_move_insn (dest, const0_rtx);
    3221          122 :       emit_jump (lend);
    3222          122 :       emit_label (l1);
    3223          122 :       emit_move_insn (dest, const1_rtx);
    3224              :     }
    3225              :   else
    3226              :     {
    3227           52 :       rtx lt_tmp = NULL_RTX;
    3228           52 :       if (GET_MODE (dest) != SImode || !TARGET_ZERO_EXTEND_WITH_AND)
    3229              :         {
    3230           52 :           lt_tmp = gen_reg_rtx (QImode);
    3231           52 :           ix86_expand_setcc (lt_tmp, UNLT, gen_rtx_REG (CCFPmode, FLAGS_REG),
    3232              :                              const0_rtx);
    3233           52 :           if (GET_MODE (dest) != QImode)
    3234              :             {
    3235           52 :               tmp = gen_reg_rtx (GET_MODE (dest));
    3236           52 :               emit_insn (gen_rtx_SET (tmp,
    3237              :                                       gen_rtx_ZERO_EXTEND (GET_MODE (dest),
    3238              :                                                            lt_tmp)));
    3239           52 :               lt_tmp = tmp;
    3240              :             }
    3241              :         }
    3242           52 :       rtx gt_tmp;
    3243           52 :       if (zero)
    3244              :         {
    3245              :           /* If TARGET_IEEE_FP and dest has SImode, emit SImode clear
    3246              :              before the floating point comparison and use setcc_si_slp
    3247              :              pattern to hide it from the combiner, so that it doesn't
    3248              :              undo it.  Similarly for TARGET_ZERO_EXTEND_WITH_AND, where
    3249              :              the ZERO_EXTEND normally emitted would need to be AND
    3250              :              with flags clobber.  */
    3251           34 :           tmp = ix86_expand_compare (GT, XEXP (gt, 0), const0_rtx);
    3252           34 :           PUT_MODE (tmp, QImode);
    3253           34 :           emit_insn (gen_setcc_si_slp (zero, tmp, zero));
    3254           34 :           gt_tmp = zero;
    3255              :         }
    3256              :       else
    3257              :         {
    3258           18 :           gt_tmp = gen_reg_rtx (QImode);
    3259           18 :           ix86_expand_setcc (gt_tmp, GT, XEXP (gt, 0), const0_rtx);
    3260           18 :           if (GET_MODE (dest) != QImode)
    3261              :             {
    3262           18 :               tmp = gen_reg_rtx (GET_MODE (dest));
    3263           18 :               emit_insn (gen_rtx_SET (tmp,
    3264              :                                       gen_rtx_ZERO_EXTEND (GET_MODE (dest),
    3265              :                                                            gt_tmp)));
    3266           18 :               gt_tmp = tmp;
    3267              :             }
    3268              :         }
    3269           52 :       if (lt_tmp)
    3270              :         {
    3271           52 :           tmp = expand_simple_binop (GET_MODE (dest), MINUS, gt_tmp, lt_tmp,
    3272              :                                      dest, 0, OPTAB_DIRECT);
    3273           52 :           if (!rtx_equal_p (tmp, dest))
    3274            0 :             emit_move_insn (dest, tmp);
    3275              :         }
    3276              :       else
    3277              :         {
    3278              :           /* For TARGET_ZERO_EXTEND_WITH_AND emit sbb directly, as we can't
    3279              :              do ZERO_EXTEND without clobbering flags.  */
    3280            0 :           tmp = ix86_expand_compare (UNLT, XEXP (gt, 0), const0_rtx);
    3281            0 :           PUT_MODE (tmp, SImode);
    3282            0 :           emit_insn (gen_subsi3_carry (dest, gt_tmp,
    3283            0 :                                        force_reg (GET_MODE (dest), const0_rtx),
    3284              :                                        XEXP (gt, 0), tmp));
    3285              :         }
    3286              :     }
    3287          174 :   emit_jump (lend);
    3288          174 :   if (l2)
    3289              :     {
    3290          137 :       emit_label (l2);
    3291          137 :       emit_move_insn (dest, op2 == const0_rtx ? GEN_INT (-128) : op2);
    3292              :     }
    3293          174 :   emit_label (lend);
    3294          174 : }
    3295              : 
    3296              : /* Expand integral op0 <=> op1, i.e.
    3297              :    dest = op0 == op1 ? 0 : op0 < op1 ? -1 : 1.  */
    3298              : 
    3299              : void
    3300          159 : ix86_expand_int_spaceship (rtx dest, rtx op0, rtx op1, rtx op2)
    3301              : {
    3302          159 :   gcc_assert (INTVAL (op2));
    3303          159 :   rtx zero1 = NULL_RTX, zero2 = NULL_RTX;
    3304          159 :   if (TARGET_ZERO_EXTEND_WITH_AND && GET_MODE (dest) == SImode)
    3305              :     {
    3306            0 :       zero1 = force_reg (SImode, const0_rtx);
    3307            0 :       if (INTVAL (op2) != 1)
    3308            0 :         zero2 = force_reg (SImode, const0_rtx);
    3309              :     }
    3310              : 
    3311              :   /* Not using ix86_expand_int_compare here, so that it doesn't swap
    3312              :      operands nor optimize CC mode - we need a mode usable for both
    3313              :      LT and GT resp. LTU and GTU comparisons with the same unswapped
    3314              :      operands.  */
    3315          195 :   rtx flags = gen_rtx_REG (INTVAL (op2) != 1 ? CCGCmode : CCmode, FLAGS_REG);
    3316          159 :   rtx tmp = gen_rtx_COMPARE (GET_MODE (flags), op0, op1);
    3317          159 :   emit_insn (gen_rtx_SET (flags, tmp));
    3318          159 :   rtx lt_tmp = NULL_RTX;
    3319          159 :   if (zero2)
    3320              :     {
    3321              :       /* For TARGET_ZERO_EXTEND_WITH_AND, emit setcc_si_slp to avoid
    3322              :          ZERO_EXTEND.  */
    3323            0 :       tmp = ix86_expand_compare (LT, flags, const0_rtx);
    3324            0 :       PUT_MODE (tmp, QImode);
    3325            0 :       emit_insn (gen_setcc_si_slp (zero2, tmp, zero2));
    3326            0 :       lt_tmp = zero2;
    3327              :     }
    3328          159 :   else if (!zero1)
    3329              :     {
    3330          159 :       lt_tmp = gen_reg_rtx (QImode);
    3331          195 :       ix86_expand_setcc (lt_tmp, INTVAL (op2) != 1 ? LT : LTU, flags,
    3332              :                          const0_rtx);
    3333          159 :       if (GET_MODE (dest) != QImode)
    3334              :         {
    3335          159 :           tmp = gen_reg_rtx (GET_MODE (dest));
    3336          159 :           emit_insn (gen_rtx_SET (tmp, gen_rtx_ZERO_EXTEND (GET_MODE (dest),
    3337              :                                                             lt_tmp)));
    3338          159 :           lt_tmp = tmp;
    3339              :         }
    3340              :     }
    3341          159 :   rtx gt_tmp;
    3342          159 :   if (zero1)
    3343              :     {
    3344              :       /* For TARGET_ZERO_EXTEND_WITH_AND, emit setcc_si_slp to avoid
    3345              :          ZERO_EXTEND.  */
    3346            0 :       tmp = ix86_expand_compare (INTVAL (op2) != 1 ? GT : GTU, flags,
    3347              :                                  const0_rtx);
    3348            0 :       PUT_MODE (tmp, QImode);
    3349            0 :       emit_insn (gen_setcc_si_slp (zero1, tmp, zero1));
    3350            0 :       gt_tmp = zero1;
    3351              :     }
    3352              :   else
    3353              :     {
    3354          159 :       gt_tmp = gen_reg_rtx (QImode);
    3355          195 :       ix86_expand_setcc (gt_tmp, INTVAL (op2) != 1 ? GT : GTU, flags,
    3356              :                          const0_rtx);
    3357          159 :       if (GET_MODE (dest) != QImode)
    3358              :         {
    3359          159 :           tmp = gen_reg_rtx (GET_MODE (dest));
    3360          159 :           emit_insn (gen_rtx_SET (tmp, gen_rtx_ZERO_EXTEND (GET_MODE (dest),
    3361              :                                                             gt_tmp)));
    3362          159 :           gt_tmp = tmp;
    3363              :         }
    3364              :     }
    3365          159 :   if (lt_tmp)
    3366              :     {
    3367          159 :       tmp = expand_simple_binop (GET_MODE (dest), MINUS, gt_tmp, lt_tmp, dest,
    3368              :                                  0, OPTAB_DIRECT);
    3369          159 :       if (!rtx_equal_p (tmp, dest))
    3370            0 :         emit_move_insn (dest, tmp);
    3371              :     }
    3372              :   else
    3373              :     {
    3374              :       /* For TARGET_ZERO_EXTEND_WITH_AND emit sbb directly, as we can't
    3375              :          do ZERO_EXTEND without clobbering flags.  */
    3376            0 :       tmp = ix86_expand_compare (LTU, flags, const0_rtx);
    3377            0 :       PUT_MODE (tmp, SImode);
    3378            0 :       emit_insn (gen_subsi3_carry (dest, gt_tmp,
    3379            0 :                                    force_reg (GET_MODE (dest), const0_rtx),
    3380              :                                    flags, tmp));
    3381              :     }
    3382          159 : }
    3383              : 
    3384              : /* Expand comparison setting or clearing carry flag.  Return true when
    3385              :    successful and set pop for the operation.  */
    3386              : static bool
    3387        23870 : ix86_expand_carry_flag_compare (enum rtx_code code, rtx op0, rtx op1, rtx *pop)
    3388              : {
    3389        47740 :   machine_mode mode
    3390        23870 :     = GET_MODE (op0) != VOIDmode ? GET_MODE (op0) : GET_MODE (op1);
    3391              : 
    3392              :   /* Do not handle double-mode compares that go through special path.  */
    3393        26022 :   if (mode == (TARGET_64BIT ? TImode : DImode))
    3394              :     return false;
    3395              : 
    3396        23862 :   if (SCALAR_FLOAT_MODE_P (mode))
    3397              :     {
    3398         1764 :       rtx compare_op;
    3399         1764 :       rtx_insn *compare_seq;
    3400              : 
    3401         1764 :       gcc_assert (!DECIMAL_FLOAT_MODE_P (mode));
    3402              : 
    3403              :       /* Shortcut:  following common codes never translate
    3404              :          into carry flag compares.  */
    3405         1764 :       if (code == EQ || code == NE || code == UNEQ || code == LTGT
    3406              :           || code == ORDERED || code == UNORDERED)
    3407              :         return false;
    3408              : 
    3409              :       /* These comparisons require zero flag; swap operands so they won't.  */
    3410              :       if ((code == GT || code == UNLE || code == LE || code == UNGT)
    3411         1696 :           && !TARGET_IEEE_FP)
    3412              :         {
    3413            2 :           std::swap (op0, op1);
    3414            2 :           code = swap_condition (code);
    3415              :         }
    3416              : 
    3417              :       /* Try to expand the comparison and verify that we end up with
    3418              :          carry flag based comparison.  This fails to be true only when
    3419              :          we decide to expand comparison using arithmetic that is not
    3420              :          too common scenario.  */
    3421         1762 :       start_sequence ();
    3422         1762 :       compare_op = ix86_expand_fp_compare (code, op0, op1);
    3423         1762 :       compare_seq = end_sequence ();
    3424              : 
    3425         1762 :       if (GET_MODE (XEXP (compare_op, 0)) == CCFPmode)
    3426         1762 :         code = ix86_fp_compare_code_to_integer (GET_CODE (compare_op));
    3427              :       else
    3428            0 :         code = GET_CODE (compare_op);
    3429              : 
    3430         1762 :       if (code != LTU && code != GEU)
    3431              :         return false;
    3432              : 
    3433           66 :       emit_insn (compare_seq);
    3434           66 :       *pop = compare_op;
    3435           66 :       return true;
    3436              :     }
    3437              : 
    3438        22098 :   if (!INTEGRAL_MODE_P (mode))
    3439              :     return false;
    3440              : 
    3441        22026 :   switch (code)
    3442              :     {
    3443              :     case LTU:
    3444              :     case GEU:
    3445              :       break;
    3446              : 
    3447              :     /* Convert a==0 into (unsigned)a<1.  */
    3448        18001 :     case EQ:
    3449        18001 :     case NE:
    3450        18001 :       if (op1 != const0_rtx)
    3451              :         return false;
    3452         9481 :       op1 = const1_rtx;
    3453         9481 :       code = (code == EQ ? LTU : GEU);
    3454              :       break;
    3455              : 
    3456              :     /* Convert a>b into b<a or a>=b-1.  */
    3457          920 :     case GTU:
    3458          920 :     case LEU:
    3459          920 :       if (CONST_INT_P (op1))
    3460              :         {
    3461          882 :           op1 = gen_int_mode (INTVAL (op1) + 1, GET_MODE (op0));
    3462              :           /* Bail out on overflow.  We still can swap operands but that
    3463              :              would force loading of the constant into register.  */
    3464          882 :           if (op1 == const0_rtx
    3465          882 :               || !x86_64_immediate_operand (op1, GET_MODE (op1)))
    3466              :             return false;
    3467          882 :           code = (code == GTU ? GEU : LTU);
    3468              :         }
    3469              :       else
    3470              :         {
    3471           38 :           std::swap (op0, op1);
    3472           38 :           code = (code == GTU ? LTU : GEU);
    3473              :         }
    3474              :       break;
    3475              : 
    3476              :     /* Convert a>=0 into (unsigned)a<0x80000000.  */
    3477         1445 :     case LT:
    3478         1445 :     case GE:
    3479         1445 :       if (mode == DImode || op1 != const0_rtx)
    3480              :         return false;
    3481          100 :       op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
    3482           50 :       code = (code == LT ? GEU : LTU);
    3483              :       break;
    3484          965 :     case LE:
    3485          965 :     case GT:
    3486          965 :       if (mode == DImode || op1 != constm1_rtx)
    3487              :         return false;
    3488            0 :       op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
    3489            0 :       code = (code == LE ? GEU : LTU);
    3490              :       break;
    3491              : 
    3492              :     default:
    3493              :       return false;
    3494              :     }
    3495              :   /* Swapping operands may cause constant to appear as first operand.  */
    3496        11146 :   if (!nonimmediate_operand (op0, VOIDmode))
    3497              :     {
    3498            0 :       if (!can_create_pseudo_p ())
    3499              :         return false;
    3500            0 :       op0 = force_reg (mode, op0);
    3501              :     }
    3502        11146 :   *pop = ix86_expand_compare (code, op0, op1);
    3503        11146 :   gcc_assert (GET_CODE (*pop) == LTU || GET_CODE (*pop) == GEU);
    3504              :   return true;
    3505              : }
    3506              : 
    3507              : /* Expand conditional increment or decrement using adb/sbb instructions.
    3508              :    The default case using setcc followed by the conditional move can be
    3509              :    done by generic code.  */
    3510              : bool
    3511         2758 : ix86_expand_int_addcc (rtx operands[])
    3512              : {
    3513         2758 :   enum rtx_code code = GET_CODE (operands[1]);
    3514         2758 :   rtx flags;
    3515         2758 :   rtx (*insn) (machine_mode, rtx, rtx, rtx, rtx, rtx);
    3516         2758 :   rtx compare_op;
    3517         2758 :   rtx val = const0_rtx;
    3518         2758 :   bool fpcmp = false;
    3519         2758 :   machine_mode mode;
    3520         2758 :   rtx op0 = XEXP (operands[1], 0);
    3521         2758 :   rtx op1 = XEXP (operands[1], 1);
    3522              : 
    3523         2758 :   if (operands[3] != const1_rtx
    3524         2338 :       && operands[3] != constm1_rtx)
    3525              :     return false;
    3526         1174 :   if (!ix86_expand_carry_flag_compare (code, op0, op1, &compare_op))
    3527              :      return false;
    3528          342 :   code = GET_CODE (compare_op);
    3529              : 
    3530          342 :   flags = XEXP (compare_op, 0);
    3531              : 
    3532          342 :   if (GET_MODE (flags) == CCFPmode)
    3533              :     {
    3534            1 :       fpcmp = true;
    3535            1 :       code = ix86_fp_compare_code_to_integer (code);
    3536              :     }
    3537              : 
    3538          342 :   if (code != LTU)
    3539              :     {
    3540          175 :       val = constm1_rtx;
    3541          175 :       if (fpcmp)
    3542            1 :         PUT_CODE (compare_op,
    3543              :                   reverse_condition_maybe_unordered
    3544              :                     (GET_CODE (compare_op)));
    3545              :       else
    3546          174 :         PUT_CODE (compare_op, reverse_condition (GET_CODE (compare_op)));
    3547              :     }
    3548              : 
    3549          342 :   mode = GET_MODE (operands[0]);
    3550              : 
    3551              :   /* Construct either adc or sbb insn.  */
    3552          342 :   if ((code == LTU) == (operands[3] == constm1_rtx))
    3553              :     insn = gen_sub3_carry;
    3554              :   else
    3555          125 :     insn = gen_add3_carry;
    3556              : 
    3557          342 :   emit_insn (insn (mode, operands[0], operands[2], val, flags, compare_op));
    3558              : 
    3559          342 :   return true;
    3560              : }
    3561              : 
    3562              : bool
    3563       524407 : ix86_expand_int_movcc (rtx operands[])
    3564              : {
    3565       524407 :   enum rtx_code code = GET_CODE (operands[1]), compare_code;
    3566       524407 :   rtx_insn *compare_seq;
    3567       524407 :   rtx compare_op;
    3568       524407 :   machine_mode mode = GET_MODE (operands[0]);
    3569       524407 :   bool sign_bit_compare_p = false;
    3570       524407 :   bool negate_cc_compare_p = false;
    3571       524407 :   rtx op0 = XEXP (operands[1], 0);
    3572       524407 :   rtx op1 = XEXP (operands[1], 1);
    3573       524407 :   rtx op2 = operands[2];
    3574       524407 :   rtx op3 = operands[3];
    3575              : 
    3576       524407 :   if (GET_MODE (op0) == TImode
    3577       508803 :       || (GET_MODE (op0) == DImode
    3578       128488 :           && !TARGET_64BIT))
    3579              :     return false;
    3580              : 
    3581       507625 :   if (GET_MODE (op0) == BFmode
    3582       507625 :       && !ix86_fp_comparison_operator (operands[1], VOIDmode))
    3583              :     return false;
    3584              : 
    3585       507625 :   start_sequence ();
    3586       507625 :   compare_op = ix86_expand_compare (code, op0, op1);
    3587       507625 :   compare_seq = end_sequence ();
    3588              : 
    3589       507625 :   compare_code = GET_CODE (compare_op);
    3590              : 
    3591       507625 :   if ((op1 == const0_rtx && (code == GE || code == LT))
    3592       463330 :       || (op1 == constm1_rtx && (code == GT || code == LE)))
    3593              :     sign_bit_compare_p = true;
    3594              : 
    3595              :   /* op0 == op1 ? op0 : op3 is equivalent to op0 == op1 ? op1 : op3,
    3596              :      but if op1 is a constant, the latter form allows more optimizations,
    3597              :      either through the last 2 ops being constant handling, or the one
    3598              :      constant and one variable cases.  On the other side, for cmov the
    3599              :      former might be better as we don't need to load the constant into
    3600              :      another register.  */
    3601       463330 :   if (code == EQ && CONST_INT_P (op1) && rtx_equal_p (op0, op2))
    3602              :     op2 = op1;
    3603              :   /* Similarly for op0 != op1 ? op2 : op0 and op0 != op1 ? op2 : op1.  */
    3604       507104 :   else if (code == NE && CONST_INT_P (op1) && rtx_equal_p (op0, op3))
    3605              :     op3 = op1;
    3606              : 
    3607              :   /* Don't attempt mode expansion here -- if we had to expand 5 or 6
    3608              :      HImode insns, we'd be swallowed in word prefix ops.  */
    3609              : 
    3610         5054 :   if ((mode != HImode || TARGET_FAST_PREFIX)
    3611       540865 :       && (mode != (TARGET_64BIT ? TImode : DImode))
    3612       507625 :       && CONST_INT_P (op2)
    3613       538103 :       && CONST_INT_P (op3))
    3614              :     {
    3615        23704 :       rtx out = operands[0];
    3616        23704 :       HOST_WIDE_INT ct = INTVAL (op2);
    3617        23704 :       HOST_WIDE_INT cf = INTVAL (op3);
    3618        23704 :       HOST_WIDE_INT diff;
    3619              : 
    3620        23704 :       if ((mode == SImode
    3621         8723 :            || (TARGET_64BIT && mode == DImode))
    3622        20073 :           && (GET_MODE (op0) == SImode
    3623        14798 :               || (TARGET_64BIT && GET_MODE (op0) == DImode)))
    3624              :         {
    3625              :           /* Special case x != 0 ? -1 : y.  */
    3626        15329 :           if (code == NE && op1 == const0_rtx && ct == -1)
    3627              :             {
    3628              :               negate_cc_compare_p = true;
    3629              :               std::swap (ct, cf);
    3630              :               code = EQ;
    3631              :             }
    3632        15205 :           else if (code == EQ && op1 == const0_rtx && cf == -1)
    3633        23704 :             negate_cc_compare_p = true;
    3634              :         }
    3635              : 
    3636        23704 :       diff = (unsigned HOST_WIDE_INT) ct - cf;
    3637              :       /* Make sure we can represent the difference between the two values.  */
    3638        23704 :       if ((diff > 0) != ((cf < 0) != (ct < 0) ? cf < 0 : cf < ct))
    3639       524407 :         return false;
    3640              : 
    3641              :       /*  Sign bit compares are better done using shifts than we do by using
    3642              :           sbb.  */
    3643        23626 :       if (sign_bit_compare_p
    3644        23626 :           || negate_cc_compare_p
    3645        23626 :           || ix86_expand_carry_flag_compare (code, op0, op1, &compare_op))
    3646              :         {
    3647              :           /* Detect overlap between destination and compare sources.  */
    3648        11800 :           rtx tmp = out;
    3649              : 
    3650        11800 :           if (negate_cc_compare_p)
    3651              :             {
    3652          291 :               if (GET_MODE (op0) == DImode)
    3653          117 :                 emit_insn (gen_x86_negdi_ccc (gen_reg_rtx (DImode), op0));
    3654              :               else
    3655          174 :                 emit_insn (gen_x86_negsi_ccc (gen_reg_rtx (SImode),
    3656          174 :                                               gen_lowpart (SImode, op0)));
    3657              : 
    3658          291 :               tmp = gen_reg_rtx (mode);
    3659          291 :               if (mode == DImode)
    3660          137 :                 emit_insn (gen_x86_movdicc_0_m1_neg (tmp));
    3661              :               else
    3662          154 :                 emit_insn (gen_x86_movsicc_0_m1_neg (gen_lowpart (SImode,
    3663              :                                                                   tmp)));
    3664              :             }
    3665        11509 :           else if (!sign_bit_compare_p)
    3666              :             {
    3667        10870 :               rtx flags;
    3668        10870 :               bool fpcmp = false;
    3669              : 
    3670        10870 :               compare_code = GET_CODE (compare_op);
    3671              : 
    3672        10870 :               flags = XEXP (compare_op, 0);
    3673              : 
    3674        10870 :               if (GET_MODE (flags) == CCFPmode)
    3675              :                 {
    3676           65 :                   fpcmp = true;
    3677           65 :                   compare_code
    3678           65 :                     = ix86_fp_compare_code_to_integer (compare_code);
    3679              :                 }
    3680              : 
    3681              :               /* To simplify rest of code, restrict to the GEU case.  */
    3682        10870 :               if (compare_code == LTU)
    3683              :                 {
    3684         6519 :                   std::swap (ct, cf);
    3685         6519 :                   compare_code = reverse_condition (compare_code);
    3686         6519 :                   code = reverse_condition (code);
    3687              :                 }
    3688              :               else
    3689              :                 {
    3690         4351 :                   if (fpcmp)
    3691           65 :                     PUT_CODE (compare_op,
    3692              :                               reverse_condition_maybe_unordered
    3693              :                                 (GET_CODE (compare_op)));
    3694              :                   else
    3695         4286 :                     PUT_CODE (compare_op,
    3696              :                               reverse_condition (GET_CODE (compare_op)));
    3697              :                 }
    3698              : 
    3699        10870 :               diff = (unsigned HOST_WIDE_INT) ct - cf;
    3700              :               /* Make sure we can represent the difference
    3701              :                  between the two values.  */
    3702        10870 :               if ((diff > 0) != ((cf < 0) != (ct < 0) ? cf < 0 : cf < ct))
    3703              :                 return false;
    3704              : 
    3705        10869 :               if (reg_overlap_mentioned_p (out, compare_op))
    3706            0 :                 tmp = gen_reg_rtx (mode);
    3707              : 
    3708        10869 :               if (mode == DImode)
    3709         1926 :                 emit_insn (gen_x86_movdicc_0_m1 (tmp, flags, compare_op));
    3710              :               else
    3711         8943 :                 emit_insn (gen_x86_movsicc_0_m1 (gen_lowpart (SImode, tmp),
    3712              :                                                  flags, compare_op));
    3713              :             }
    3714              :           else
    3715              :             {
    3716          639 :               if (code == GT || code == GE)
    3717          383 :                 code = reverse_condition (code);
    3718              :               else
    3719              :                 {
    3720          256 :                   std::swap (ct, cf);
    3721              : 
    3722          256 :                   diff = (unsigned HOST_WIDE_INT) ct - cf;
    3723              :                   /* Make sure we can represent the difference
    3724              :                      between the two values.  */
    3725          256 :                   if ((diff > 0) != ((cf < 0) != (ct < 0) ? cf < 0 : cf < ct))
    3726              :                     return false;
    3727              :                 }
    3728          636 :               tmp = emit_store_flag (tmp, code, op0, op1, VOIDmode, 0, -1);
    3729              :             }
    3730              : 
    3731        11796 :           if (diff == 1)
    3732              :             {
    3733              :               /*
    3734              :                * cmpl op0,op1
    3735              :                * sbbl dest,dest
    3736              :                * [addl dest, ct]
    3737              :                *
    3738              :                * Size 5 - 8.
    3739              :                */
    3740          717 :               if (ct)
    3741          555 :                 tmp = expand_simple_binop (mode, PLUS,
    3742              :                                            tmp, GEN_INT (ct),
    3743              :                                            copy_rtx (tmp), 1, OPTAB_DIRECT);
    3744              :             }
    3745        11079 :           else if (cf == -1)
    3746              :             {
    3747              :               /*
    3748              :                * cmpl op0,op1
    3749              :                * sbbl dest,dest
    3750              :                * orl $ct, dest
    3751              :                *
    3752              :                * Size 8.
    3753              :                */
    3754          689 :               tmp = expand_simple_binop (mode, IOR,
    3755              :                                          tmp, GEN_INT (ct),
    3756              :                                          copy_rtx (tmp), 1, OPTAB_DIRECT);
    3757              :             }
    3758        10390 :           else if (diff == -1 && ct)
    3759              :             {
    3760              :               /*
    3761              :                * cmpl op0,op1
    3762              :                * sbbl dest,dest
    3763              :                * notl dest
    3764              :                * [addl dest, cf]
    3765              :                *
    3766              :                * Size 8 - 11.
    3767              :                */
    3768          468 :               tmp = expand_simple_unop (mode, NOT, tmp, copy_rtx (tmp), 1);
    3769          468 :               if (cf)
    3770          449 :                 tmp = expand_simple_binop (mode, PLUS,
    3771              :                                            copy_rtx (tmp), GEN_INT (cf),
    3772              :                                            copy_rtx (tmp), 1, OPTAB_DIRECT);
    3773              :             }
    3774              :           else
    3775              :             {
    3776              :               /*
    3777              :                * cmpl op0,op1
    3778              :                * sbbl dest,dest
    3779              :                * [notl dest]
    3780              :                * andl cf - ct, dest
    3781              :                * [addl dest, ct]
    3782              :                *
    3783              :                * Size 8 - 11.
    3784              :                */
    3785              : 
    3786         9922 :               if (cf == 0)
    3787              :                 {
    3788         1305 :                   cf = ct;
    3789         1305 :                   ct = 0;
    3790         1305 :                   tmp = expand_simple_unop (mode, NOT, tmp, copy_rtx (tmp), 1);
    3791              :                 }
    3792              : 
    3793         9922 :               HOST_WIDE_INT ival = (unsigned HOST_WIDE_INT) cf - ct;
    3794              :               /* Make sure we can represent the difference
    3795              :                  between the two values.  */
    3796         9922 :               if ((ival > 0) != ((ct < 0) != (cf < 0) ? ct < 0 : ct < cf))
    3797       524407 :                 return false;
    3798              : 
    3799         9922 :               tmp = expand_simple_binop (mode, AND,
    3800              :                                          copy_rtx (tmp),
    3801         9922 :                                          gen_int_mode (ival, mode),
    3802              :                                          copy_rtx (tmp), 1, OPTAB_DIRECT);
    3803         9922 :               if (ct)
    3804         7152 :                 tmp = expand_simple_binop (mode, PLUS,
    3805              :                                            copy_rtx (tmp), GEN_INT (ct),
    3806              :                                            copy_rtx (tmp), 1, OPTAB_DIRECT);
    3807              :             }
    3808              : 
    3809        11796 :           if (!rtx_equal_p (tmp, out))
    3810          476 :             emit_move_insn (copy_rtx (out), copy_rtx (tmp));
    3811              : 
    3812              :           return true;
    3813              :         }
    3814              : 
    3815        11826 :       if (diff < 0)
    3816              :         {
    3817         3630 :           machine_mode cmp_mode = GET_MODE (op0);
    3818         3630 :           enum rtx_code new_code;
    3819              : 
    3820         3630 :           if (SCALAR_FLOAT_MODE_P (cmp_mode))
    3821              :             {
    3822           77 :               gcc_assert (!DECIMAL_FLOAT_MODE_P (cmp_mode));
    3823              : 
    3824              :               /* We may be reversing a non-trapping
    3825              :                  comparison to a trapping comparison.  */
    3826          150 :                   if (HONOR_NANS (cmp_mode) && flag_trapping_math
    3827           70 :                       && code != EQ && code != NE
    3828          147 :                       && code != ORDERED && code != UNORDERED)
    3829              :                     new_code = UNKNOWN;
    3830              :                   else
    3831            7 :                     new_code = reverse_condition_maybe_unordered (code);
    3832              :             }
    3833              :           else
    3834         3553 :             new_code = ix86_reverse_condition (code, cmp_mode);
    3835         3560 :           if (new_code != UNKNOWN)
    3836              :             {
    3837         3560 :               std::swap (ct, cf);
    3838              : 
    3839         3560 :               diff = (unsigned HOST_WIDE_INT) ct - cf;
    3840              :               /* Make sure we can represent the difference
    3841              :                  between the two values.  */
    3842         3560 :               if ((diff > 0) != ((cf < 0) != (ct < 0) ? cf < 0 : cf < ct))
    3843              :                 return false;
    3844              : 
    3845              :               code = new_code;
    3846              :             }
    3847              :         }
    3848              : 
    3849        11826 :       compare_code = UNKNOWN;
    3850        11826 :       if (GET_MODE_CLASS (GET_MODE (op0)) == MODE_INT
    3851        10098 :           && CONST_INT_P (op1))
    3852              :         {
    3853         3768 :           if (op1 == const0_rtx
    3854          221 :               && (code == LT || code == GE))
    3855              :             compare_code = code;
    3856         3768 :           else if (op1 == constm1_rtx)
    3857              :             {
    3858          110 :               if (code == LE)
    3859              :                 compare_code = LT;
    3860          110 :               else if (code == GT)
    3861              :                 compare_code = GE;
    3862              :             }
    3863              :         }
    3864              : 
    3865              :       /* Optimize dest = (op0 < 0) ? -1 : cf.  */
    3866              :       if (compare_code != UNKNOWN
    3867            0 :           && GET_MODE (op0) == GET_MODE (out)
    3868            0 :           && (cf == -1 || ct == -1))
    3869              :         {
    3870              :           /* If lea code below could be used, only optimize
    3871              :              if it results in a 2 insn sequence.  */
    3872              : 
    3873            0 :           if (! (diff == 1 || diff == 2 || diff == 4 || diff == 8
    3874            0 :                  || diff == 3 || diff == 5 || diff == 9)
    3875            0 :               || (compare_code == LT && ct == -1)
    3876            0 :               || (compare_code == GE && cf == -1))
    3877              :             {
    3878              :               /*
    3879              :                * notl op1       (if necessary)
    3880              :                * sarl $31, op1
    3881              :                * orl cf, op1
    3882              :                */
    3883            0 :               if (ct != -1)
    3884              :                 {
    3885            0 :                   cf = ct;
    3886            0 :                   ct = -1;
    3887            0 :                   code = reverse_condition (code);
    3888              :                 }
    3889              : 
    3890            0 :               out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, -1);
    3891              : 
    3892            0 :               out = expand_simple_binop (mode, IOR,
    3893              :                                          out, GEN_INT (cf),
    3894              :                                          out, 1, OPTAB_DIRECT);
    3895            0 :               if (out != operands[0])
    3896            0 :                 emit_move_insn (operands[0], out);
    3897              : 
    3898              :               return true;
    3899              :             }
    3900              :         }
    3901              : 
    3902              : 
    3903        15907 :       if ((diff == 1 || diff == 2 || diff == 4 || diff == 8
    3904         4081 :            || diff == 3 || diff == 5 || diff == 9)
    3905         8304 :           && ((mode != QImode && mode != HImode) || !TARGET_PARTIAL_REG_STALL)
    3906        20130 :           && (mode != DImode
    3907         2101 :               || x86_64_immediate_operand (GEN_INT (cf), VOIDmode)))
    3908              :         {
    3909              :           /*
    3910              :            * xorl dest,dest
    3911              :            * cmpl op1,op2
    3912              :            * setcc dest
    3913              :            * lea cf(dest*(ct-cf)),dest
    3914              :            *
    3915              :            * Size 14.
    3916              :            *
    3917              :            * This also catches the degenerate setcc-only case.
    3918              :            */
    3919              : 
    3920         8304 :           rtx tmp;
    3921         8304 :           int nops;
    3922              : 
    3923         8304 :           out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, 1);
    3924              : 
    3925         8304 :           nops = 0;
    3926              :           /* On x86_64 the lea instruction operates on Pmode, so we need
    3927              :              to get arithmetics done in proper mode to match.  */
    3928         8304 :           if (diff == 1)
    3929         6894 :             tmp = copy_rtx (out);
    3930              :           else
    3931              :             {
    3932         1410 :               rtx out1;
    3933         1410 :               out1 = copy_rtx (out);
    3934         1410 :               tmp = gen_rtx_MULT (mode, out1, GEN_INT (diff & ~1));
    3935         1410 :               nops++;
    3936         1410 :               if (diff & 1)
    3937              :                 {
    3938          440 :                   tmp = gen_rtx_PLUS (mode, tmp, out1);
    3939          440 :                   nops++;
    3940              :                 }
    3941              :             }
    3942         8304 :           if (cf != 0)
    3943              :             {
    3944         7130 :               tmp = plus_constant (mode, tmp, cf);
    3945         7130 :               nops++;
    3946              :             }
    3947         8304 :           if (!rtx_equal_p (tmp, out))
    3948              :             {
    3949         7592 :               if (nops == 1)
    3950         6338 :                 out = force_operand (tmp, copy_rtx (out));
    3951              :               else
    3952         1254 :                 emit_insn (gen_rtx_SET (copy_rtx (out), copy_rtx (tmp)));
    3953              :             }
    3954         8304 :           if (!rtx_equal_p (out, operands[0]))
    3955          620 :             emit_move_insn (operands[0], copy_rtx (out));
    3956              : 
    3957              :           return true;
    3958              :         }
    3959              : 
    3960              :       /*
    3961              :        * General case:                  Jumpful:
    3962              :        *   xorl dest,dest               cmpl op1, op2
    3963              :        *   cmpl op1, op2                movl ct, dest
    3964              :        *   setcc dest                   jcc 1f
    3965              :        *   decl dest                    movl cf, dest
    3966              :        *   andl (cf-ct),dest            1:
    3967              :        *   addl ct,dest
    3968              :        *
    3969              :        * Size 20.                       Size 14.
    3970              :        *
    3971              :        * This is reasonably steep, but branch mispredict costs are
    3972              :        * high on modern cpus, so consider failing only if optimizing
    3973              :        * for space.
    3974              :        */
    3975              : 
    3976         3522 :       if ((!TARGET_CMOVE || (mode == QImode && TARGET_PARTIAL_REG_STALL))
    3977         3522 :           && BRANCH_COST (optimize_insn_for_speed_p (),
    3978              :                           false) >= 2)
    3979              :         {
    3980            0 :           if (cf == 0)
    3981              :             {
    3982            0 :               machine_mode cmp_mode = GET_MODE (op0);
    3983            0 :               enum rtx_code new_code;
    3984              : 
    3985            0 :               if (SCALAR_FLOAT_MODE_P (cmp_mode))
    3986              :                 {
    3987            0 :                   gcc_assert (!DECIMAL_FLOAT_MODE_P (cmp_mode));
    3988              : 
    3989              :                   /* We may be reversing a non-trapping
    3990              :                      comparison to a trapping comparison.  */
    3991            0 :                   if (HONOR_NANS (cmp_mode) && flag_trapping_math
    3992            0 :                       && code != EQ && code != NE
    3993            0 :                       && code != ORDERED && code != UNORDERED)
    3994              :                     new_code = UNKNOWN;
    3995              :                   else
    3996            0 :                     new_code = reverse_condition_maybe_unordered (code);
    3997              : 
    3998              :                 }
    3999              :               else
    4000              :                 {
    4001            0 :                   new_code = ix86_reverse_condition (code, cmp_mode);
    4002            0 :                   if (compare_code != UNKNOWN && new_code != UNKNOWN)
    4003            0 :                     compare_code = reverse_condition (compare_code);
    4004              :                 }
    4005              : 
    4006            0 :               if (new_code != UNKNOWN)
    4007              :                 {
    4008            0 :                   cf = ct;
    4009            0 :                   ct = 0;
    4010            0 :                   code = new_code;
    4011              :                 }
    4012              :             }
    4013              : 
    4014            0 :           if (compare_code != UNKNOWN)
    4015              :             {
    4016              :               /* notl op1       (if needed)
    4017              :                  sarl $31, op1
    4018              :                  andl (cf-ct), op1
    4019              :                  addl ct, op1
    4020              : 
    4021              :                  For x < 0 (resp. x <= -1) there will be no notl,
    4022              :                  so if possible swap the constants to get rid of the
    4023              :                  complement.
    4024              :                  True/false will be -1/0 while code below (store flag
    4025              :                  followed by decrement) is 0/-1, so the constants need
    4026              :                  to be exchanged once more.  */
    4027              : 
    4028            0 :               if (compare_code == GE || !cf)
    4029              :                 {
    4030            0 :                   code = reverse_condition (code);
    4031            0 :                   compare_code = LT;
    4032              :                 }
    4033              :               else
    4034              :                 std::swap (ct, cf);
    4035              : 
    4036            0 :               out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, -1);
    4037              :             }
    4038              :           else
    4039              :             {
    4040            0 :               out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, 1);
    4041              : 
    4042            0 :               out = expand_simple_binop (mode, PLUS, copy_rtx (out),
    4043              :                                          constm1_rtx,
    4044              :                                          copy_rtx (out), 1, OPTAB_DIRECT);
    4045              :             }
    4046              : 
    4047            0 :           HOST_WIDE_INT ival = (unsigned HOST_WIDE_INT) cf - ct;
    4048              :           /* Make sure we can represent the difference
    4049              :              between the two values.  */
    4050            0 :           if ((ival > 0) != ((ct < 0) != (cf < 0) ? ct < 0 : ct < cf))
    4051              :             return false;
    4052              : 
    4053            0 :           out = expand_simple_binop (mode, AND, copy_rtx (out),
    4054            0 :                                      gen_int_mode (ival, mode),
    4055              :                                      copy_rtx (out), 1, OPTAB_DIRECT);
    4056            0 :           if (ct)
    4057            0 :             out = expand_simple_binop (mode, PLUS, copy_rtx (out), GEN_INT (ct),
    4058              :                                        copy_rtx (out), 1, OPTAB_DIRECT);
    4059            0 :           if (!rtx_equal_p (out, operands[0]))
    4060            0 :             emit_move_insn (operands[0], copy_rtx (out));
    4061              : 
    4062              :           return true;
    4063              :         }
    4064              :     }
    4065              : 
    4066       487443 :   if (!TARGET_CMOVE || (mode == QImode && TARGET_PARTIAL_REG_STALL))
    4067              :     {
    4068              :       /* Try a few things more with specific constants and a variable.  */
    4069              : 
    4070            0 :       optab op;
    4071            0 :       rtx var, orig_out, out, tmp;
    4072              : 
    4073            0 :       if (BRANCH_COST (optimize_insn_for_speed_p (), false) <= 2)
    4074              :         return false;
    4075              : 
    4076            0 :       operands[2] = op2;
    4077            0 :       operands[3] = op3;
    4078              : 
    4079              :       /* If one of the two operands is an interesting constant, load a
    4080              :          constant with the above and mask it in with a logical operation.  */
    4081              : 
    4082            0 :       if (CONST_INT_P (operands[2]))
    4083              :         {
    4084            0 :           var = operands[3];
    4085            0 :           if (INTVAL (operands[2]) == 0 && operands[3] != constm1_rtx)
    4086            0 :             operands[3] = constm1_rtx, op = and_optab;
    4087            0 :           else if (INTVAL (operands[2]) == -1 && operands[3] != const0_rtx)
    4088            0 :             operands[3] = const0_rtx, op = ior_optab;
    4089              :           else
    4090              :             return false;
    4091              :         }
    4092            0 :       else if (CONST_INT_P (operands[3]))
    4093              :         {
    4094            0 :           var = operands[2];
    4095            0 :           if (INTVAL (operands[3]) == 0 && operands[2] != constm1_rtx)
    4096              :             {
    4097              :               /* For smin (x, 0), expand as "x < 0 ? x : 0" instead of
    4098              :                  "x <= 0 ? x : 0" to enable sign_bit_compare_p.  */
    4099            0 :               if (code == LE && op1 == const0_rtx && rtx_equal_p (op0, var))
    4100            0 :                 operands[1] = simplify_gen_relational (LT, VOIDmode,
    4101            0 :                                                        GET_MODE (op0),
    4102              :                                                        op0, const0_rtx);
    4103              : 
    4104            0 :               operands[2] = constm1_rtx;
    4105            0 :               op = and_optab;
    4106              :             }
    4107            0 :           else if (INTVAL (operands[3]) == -1 && operands[3] != const0_rtx)
    4108            0 :             operands[2] = const0_rtx, op = ior_optab;
    4109              :           else
    4110              :             return false;
    4111              :         }
    4112              :       else
    4113              :         return false;
    4114              : 
    4115            0 :       orig_out = operands[0];
    4116            0 :       tmp = gen_reg_rtx (mode);
    4117            0 :       operands[0] = tmp;
    4118              : 
    4119              :       /* Recurse to get the constant loaded.  */
    4120            0 :       if (!ix86_expand_int_movcc (operands))
    4121              :         return false;
    4122              : 
    4123              :       /* Mask in the interesting variable.  */
    4124            0 :       out = expand_binop (mode, op, var, tmp, orig_out, 0,
    4125              :                           OPTAB_WIDEN);
    4126            0 :       if (!rtx_equal_p (out, orig_out))
    4127            0 :         emit_move_insn (copy_rtx (orig_out), copy_rtx (out));
    4128              : 
    4129              :       return true;
    4130              :     }
    4131              : 
    4132              :   /*
    4133              :    * For comparison with above,
    4134              :    *
    4135              :    * movl cf,dest
    4136              :    * movl ct,tmp
    4137              :    * cmpl op1,op2
    4138              :    * cmovcc tmp,dest
    4139              :    *
    4140              :    * Size 15.
    4141              :    */
    4142              : 
    4143       487443 :   if (! nonimmediate_operand (operands[2], mode))
    4144        17577 :     operands[2] = force_reg (mode, operands[2]);
    4145       487443 :   if (! nonimmediate_operand (operands[3], mode))
    4146       171361 :     operands[3] = force_reg (mode, operands[3]);
    4147              : 
    4148       487443 :   if (! register_operand (operands[2], VOIDmode)
    4149       487443 :       && (mode == QImode
    4150         1093 :           || ! register_operand (operands[3], VOIDmode)))
    4151         1564 :     operands[2] = force_reg (mode, operands[2]);
    4152              : 
    4153       487443 :   if (mode == QImode
    4154       487443 :       && ! register_operand (operands[3], VOIDmode))
    4155          592 :     operands[3] = force_reg (mode, operands[3]);
    4156              : 
    4157       487443 :   emit_insn (compare_seq);
    4158       487443 :   emit_insn (gen_rtx_SET (operands[0],
    4159              :                           gen_rtx_IF_THEN_ELSE (mode,
    4160              :                                                 compare_op, operands[2],
    4161              :                                                 operands[3])));
    4162       487443 :   return true;
    4163              : }
    4164              : 
    4165              : /* Detect conditional moves that exactly match min/max operational
    4166              :    semantics.  Note that this is IEEE safe, as long as we don't
    4167              :    interchange the operands.
    4168              : 
    4169              :    Returns FALSE if this conditional move doesn't match a MIN/MAX,
    4170              :    and TRUE if the operation is successful and instructions are emitted.  */
    4171              : 
    4172              : static bool
    4173        12368 : ix86_expand_sse_fp_minmax (rtx dest, enum rtx_code code, rtx cmp_op0,
    4174              :                            rtx cmp_op1, rtx if_true, rtx if_false)
    4175              : {
    4176        12368 :   machine_mode mode = GET_MODE (dest);
    4177        12368 :   bool is_min;
    4178        12368 :   rtx tmp;
    4179              : 
    4180        12368 :   if (code == LT)
    4181              :     ;
    4182         3304 :   else if (code == LE && !HONOR_NANS (mode))
    4183              :     {
    4184              :       /* We can swap LE to GE and then invert to LT.  */
    4185              :       std::swap (cmp_op0, cmp_op1);
    4186              :       std::swap (if_true, if_false);
    4187              :     }
    4188         3263 :   else if (code == UNGE)
    4189              :     std::swap (if_true, if_false);
    4190              :   else
    4191              :     return false;
    4192              : 
    4193        11222 :   if (rtx_equal_p (cmp_op0, if_true) && rtx_equal_p (cmp_op1, if_false))
    4194              :     is_min = true;
    4195         7220 :   else if (rtx_equal_p (cmp_op1, if_true) && rtx_equal_p (cmp_op0, if_false))
    4196              :     is_min = false;
    4197              :   else
    4198              :     return false;
    4199              : 
    4200         7596 :   if (immediate_operand (if_false, mode))
    4201            8 :     if_false = force_reg (mode, if_false);
    4202         7596 :   if (immediate_operand (if_true, mode))
    4203            0 :     if_true = force_reg (mode, if_true);
    4204              : 
    4205              :   /* We want to check HONOR_NANS and HONOR_SIGNED_ZEROS here,
    4206              :      but MODE may be a vector mode and thus not appropriate.  */
    4207         7596 :   if (!flag_finite_math_only || flag_signed_zeros)
    4208              :     {
    4209         7596 :       int u = is_min ? UNSPEC_IEEE_MIN : UNSPEC_IEEE_MAX;
    4210         7596 :       rtvec v;
    4211              : 
    4212         7596 :       if_true = force_reg (mode, if_true);
    4213         7596 :       v = gen_rtvec (2, if_true, if_false);
    4214         7596 :       tmp = gen_rtx_UNSPEC (mode, v, u);
    4215         7596 :     }
    4216              :   else
    4217              :     {
    4218            0 :       code = is_min ? SMIN : SMAX;
    4219            0 :       if (MEM_P (if_true) && MEM_P (if_false))
    4220            0 :         if_true = force_reg (mode, if_true);
    4221            0 :       tmp = gen_rtx_fmt_ee (code, mode, if_true, if_false);
    4222              :     }
    4223              : 
    4224         7596 :   emit_insn (gen_rtx_SET (dest, tmp));
    4225         7596 :   return true;
    4226              : }
    4227              : 
    4228              : /* Return true if MODE is valid for vector compare to mask register,
    4229              :    Same result for conditionl vector move with mask register.  */
    4230              : static bool
    4231        14928 : ix86_valid_mask_cmp_mode (machine_mode mode)
    4232              : {
    4233              :   /* XOP has its own vector conditional movement.  */
    4234        14928 :   if (TARGET_XOP && !TARGET_AVX512F)
    4235              :     return false;
    4236              : 
    4237              :   /* HFmode only supports vcmpsh whose dest is mask register.  */
    4238        14922 :   if (TARGET_AVX512FP16 && mode == HFmode)
    4239              :     return true;
    4240              : 
    4241              :   /* AVX512F is needed for mask operation.  */
    4242        14830 :   if (!(TARGET_AVX512F && VECTOR_MODE_P (mode)))
    4243              :     return false;
    4244              : 
    4245              :   /* AVX512BW is needed for vector QI/HImode,
    4246              :      AVX512VL is needed for 128/256-bit vector.  */
    4247          182 :   machine_mode inner_mode = GET_MODE_INNER (mode);
    4248          182 :   int vector_size = GET_MODE_SIZE (mode);
    4249          182 :   if ((inner_mode == QImode || inner_mode == HImode) && !TARGET_AVX512BW)
    4250              :     return false;
    4251              : 
    4252          162 :   return vector_size == 64 || TARGET_AVX512VL;
    4253              : }
    4254              : 
    4255              : /* Return true if integer mask comparison should be used.  */
    4256              : static bool
    4257        59593 : ix86_use_mask_cmp_p (machine_mode mode, machine_mode cmp_mode,
    4258              :                      rtx op_true, rtx op_false)
    4259              : {
    4260        59593 :   int vector_size = GET_MODE_SIZE (mode);
    4261              : 
    4262        59593 :   if (cmp_mode == HFmode)
    4263              :     return true;
    4264        59501 :   else if (vector_size < 16)
    4265              :     return false;
    4266        49758 :   else if (vector_size == 64)
    4267              :     return true;
    4268        99400 :   else if (GET_MODE_INNER (cmp_mode) == HFmode)
    4269              :     return true;
    4270        99400 :   else if (GET_MODE_INNER (cmp_mode) == BFmode)
    4271              :     return true;
    4272              : 
    4273              :   /* When op_true is NULL, op_false must be NULL, or vice versa.  */
    4274        49700 :   gcc_assert (!op_true == !op_false);
    4275              : 
    4276              :   /* When op_true/op_false is NULL or cmp_mode is not valid mask cmp mode,
    4277              :      vector dest is required.  */
    4278        49700 :   if (!op_true || !ix86_valid_mask_cmp_mode (cmp_mode))
    4279              :     return false;
    4280              : 
    4281              :   /* Exclude those that could be optimized in ix86_expand_sse_movcc.  */
    4282           48 :   if (op_false == CONST0_RTX (mode)
    4283           48 :       || op_true == CONST0_RTX (mode)
    4284           48 :       || (INTEGRAL_MODE_P (mode)
    4285           40 :           && (op_true == CONSTM1_RTX (mode)
    4286           40 :               || op_false == CONSTM1_RTX (mode))))
    4287            0 :     return false;
    4288              : 
    4289              :   return true;
    4290              : }
    4291              : 
    4292              : /* Expand an SSE comparison.  Return the register with the result.  */
    4293              : 
    4294              : static rtx
    4295        41018 : ix86_expand_sse_cmp (rtx dest, enum rtx_code code, rtx cmp_op0, rtx cmp_op1,
    4296              :                      rtx op_true, rtx op_false)
    4297              : {
    4298        41018 :   machine_mode mode = GET_MODE (dest);
    4299        41018 :   machine_mode cmp_ops_mode = GET_MODE (cmp_op0);
    4300              : 
    4301              :   /* In general case result of comparison can differ from operands' type.  */
    4302        41018 :   machine_mode cmp_mode;
    4303              : 
    4304              :   /* In AVX512F the result of comparison is an integer mask.  */
    4305        41018 :   bool maskcmp = false;
    4306        41018 :   rtx x;
    4307              : 
    4308        41018 :   if (ix86_use_mask_cmp_p (mode, cmp_ops_mode, op_true, op_false))
    4309              :     {
    4310          145 :       unsigned int nbits = GET_MODE_NUNITS (cmp_ops_mode);
    4311          145 :       maskcmp = true;
    4312          145 :       cmp_mode = nbits > 8 ? int_mode_for_size (nbits, 0).require () : E_QImode;
    4313              :     }
    4314              :   else
    4315              :     cmp_mode = cmp_ops_mode;
    4316              : 
    4317        41018 :   cmp_op0 = force_reg (cmp_ops_mode, cmp_op0);
    4318              : 
    4319        82036 :   bool (*op1_predicate)(rtx, machine_mode)
    4320        41018 :     = VECTOR_MODE_P (cmp_ops_mode) ? vector_operand : nonimmediate_operand;
    4321              : 
    4322        41018 :   if (!op1_predicate (cmp_op1, cmp_ops_mode))
    4323            0 :     cmp_op1 = force_reg (cmp_ops_mode, cmp_op1);
    4324              : 
    4325        41018 :   if (optimize
    4326          503 :       || (maskcmp && cmp_mode != mode)
    4327          503 :       || (op_true && reg_overlap_mentioned_p (dest, op_true))
    4328        41521 :       || (op_false && reg_overlap_mentioned_p (dest, op_false)))
    4329        80885 :     dest = gen_reg_rtx (maskcmp ? cmp_mode : mode);
    4330              : 
    4331        41018 :   if (maskcmp)
    4332              :     {
    4333          145 :       bool ok = ix86_expand_mask_vec_cmp (dest, code, cmp_op0, cmp_op1);
    4334          145 :       gcc_assert (ok);
    4335              :       return dest;
    4336              :     }
    4337              : 
    4338        40873 :   x = gen_rtx_fmt_ee (code, cmp_mode, cmp_op0, cmp_op1);
    4339              : 
    4340        40873 :   if (cmp_mode != mode)
    4341              :     {
    4342         8090 :       x = force_reg (cmp_ops_mode, x);
    4343         8090 :       convert_move (dest, x, false);
    4344              :     }
    4345              :   else
    4346        32783 :     emit_insn (gen_rtx_SET (dest, x));
    4347              : 
    4348              :   return dest;
    4349              : }
    4350              : 
    4351              : /* Emit x86 binary operand CODE in mode MODE for SSE vector
    4352              :    instructions that can be performed using GP registers.  */
    4353              : 
    4354              : static void
    4355         9673 : ix86_emit_vec_binop (enum rtx_code code, machine_mode mode,
    4356              :                      rtx dst, rtx src1, rtx src2)
    4357              : {
    4358         9673 :   rtx tmp;
    4359              : 
    4360         9673 :   tmp = gen_rtx_SET (dst, gen_rtx_fmt_ee (code, mode, src1, src2));
    4361              : 
    4362         9673 :   if (GET_MODE_SIZE (mode) <= GET_MODE_SIZE (SImode)
    4363         9673 :       && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
    4364              :     {
    4365           94 :       rtx clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
    4366           94 :       tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, clob));
    4367              :     }
    4368              : 
    4369         9673 :   emit_insn (tmp);
    4370         9673 : }
    4371              : 
    4372              : /* Expand DEST = CMP ? OP_TRUE : OP_FALSE into a sequence of logical
    4373              :    operations.  This is used for both scalar and vector conditional moves.  */
    4374              : 
    4375              : void
    4376        13282 : ix86_expand_sse_movcc (rtx dest, rtx cmp, rtx op_true, rtx op_false)
    4377              : {
    4378        13282 :   machine_mode mode = GET_MODE (dest);
    4379        13282 :   machine_mode cmpmode = GET_MODE (cmp);
    4380        13282 :   rtx x;
    4381              : 
    4382              :   /* Simplify trivial VEC_COND_EXPR to avoid ICE in pr97506.  */
    4383        13282 :   if (rtx_equal_p (op_true, op_false))
    4384              :     {
    4385            0 :       emit_move_insn (dest, op_true);
    4386            0 :       return;
    4387              :     }
    4388              : 
    4389              :   /* If we have an integer mask and FP value then we need
    4390              :      to cast mask to FP mode.  */
    4391        13282 :   if (mode != cmpmode && VECTOR_MODE_P (cmpmode))
    4392              :     {
    4393         1652 :       cmp = force_reg (cmpmode, cmp);
    4394         1652 :       cmp = gen_rtx_SUBREG (mode, cmp, 0);
    4395              :     }
    4396              : 
    4397              :   /* In AVX512F the result of comparison is an integer mask.  */
    4398        13282 :   if (mode != cmpmode
    4399         1797 :       && GET_MODE_CLASS (cmpmode) == MODE_INT)
    4400              :     {
    4401          145 :       gcc_assert (ix86_valid_mask_cmp_mode (mode));
    4402              :       /* Using scalar/vector move with mask register.  */
    4403          145 :       cmp = force_reg (cmpmode, cmp);
    4404              :       /* Optimize for mask zero.  */
    4405          290 :       op_true = (op_true != CONST0_RTX (mode)
    4406          145 :                  ? force_reg (mode, op_true) : op_true);
    4407          290 :       op_false = (op_false != CONST0_RTX (mode)
    4408          145 :                   ? force_reg (mode, op_false) : op_false);
    4409          145 :       if (op_true == CONST0_RTX (mode))
    4410              :         {
    4411            0 :           if (cmpmode == E_DImode && !TARGET_64BIT)
    4412              :             {
    4413            0 :               x = gen_reg_rtx (cmpmode);
    4414            0 :               emit_insn (gen_knotdi (x, cmp));
    4415              :             }
    4416              :           else
    4417            0 :             x = expand_simple_unop (cmpmode, NOT, cmp, NULL, 1);
    4418              :           cmp = x;
    4419              :           /* Reverse op_true op_false.  */
    4420              :           std::swap (op_true, op_false);
    4421              :         }
    4422              : 
    4423          145 :       if (mode == HFmode)
    4424           92 :         emit_insn (gen_movhf_mask (dest, op_true, op_false, cmp));
    4425              :       else
    4426           53 :         emit_insn (gen_rtx_SET (dest,
    4427              :                                 gen_rtx_VEC_MERGE (mode,
    4428              :                                                    op_true, op_false, cmp)));
    4429              :       return;
    4430              :     }
    4431              : 
    4432        13137 :   if (vector_all_ones_operand (op_true, mode)
    4433        13137 :       && op_false == CONST0_RTX (mode))
    4434              :     {
    4435            3 :       emit_move_insn (dest, cmp);
    4436            3 :       return;
    4437              :     }
    4438        13134 :   else if (op_false == CONST0_RTX (mode))
    4439              :     {
    4440          967 :       x = expand_simple_binop (mode, AND, cmp, op_true,
    4441              :                                dest, 1, OPTAB_DIRECT);
    4442          967 :       if (x != dest)
    4443            0 :         emit_move_insn (dest, x);
    4444              :       return;
    4445              :     }
    4446        12167 :   else if (op_true == CONST0_RTX (mode))
    4447              :     {
    4448           90 :       op_false = force_reg (mode, op_false);
    4449           90 :       x = gen_rtx_NOT (mode, cmp);
    4450           90 :       ix86_emit_vec_binop (AND, mode, dest, x, op_false);
    4451           90 :       return;
    4452              :     }
    4453        12077 :   else if (vector_all_ones_operand (op_true, mode))
    4454              :     {
    4455            3 :       x = expand_simple_binop (mode, IOR, cmp, op_false,
    4456              :                                dest, 1, OPTAB_DIRECT);
    4457            3 :       if (x != dest)
    4458            0 :         emit_move_insn (dest, x);
    4459              :       return;
    4460              :     }
    4461              : 
    4462        12074 :   if (TARGET_XOP)
    4463              :     {
    4464           65 :       op_true = force_reg (mode, op_true);
    4465              : 
    4466           65 :       if (GET_MODE_SIZE (mode) < 16
    4467           65 :           || !nonimmediate_operand (op_false, mode))
    4468           50 :         op_false = force_reg (mode, op_false);
    4469              : 
    4470           65 :       emit_insn (gen_rtx_SET (dest,
    4471              :                               gen_rtx_IF_THEN_ELSE (mode, cmp,
    4472              :                                                     op_true, op_false)));
    4473           65 :       return;
    4474              :     }
    4475              : 
    4476        12009 :   rtx (*gen) (rtx, rtx, rtx, rtx) = NULL;
    4477        12009 :   machine_mode blend_mode = mode;
    4478              : 
    4479        12009 :   switch (mode)
    4480              :     {
    4481           43 :     case E_V2SFmode:
    4482           43 :       if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
    4483              :         gen = gen_mmx_blendvps;
    4484              :       break;
    4485          373 :     case E_V4SFmode:
    4486          373 :       if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
    4487              :         gen = gen_sse4_1_blendvps;
    4488              :       break;
    4489          236 :     case E_V2DFmode:
    4490          236 :       if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
    4491              :         gen = gen_sse4_1_blendvpd;
    4492              :       break;
    4493         3440 :     case E_SFmode:
    4494         3440 :       if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
    4495              :         gen = gen_sse4_1_blendvss;
    4496              :       break;
    4497         1102 :     case E_DFmode:
    4498         1102 :       if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
    4499              :         gen = gen_sse4_1_blendvsd;
    4500              :       break;
    4501          263 :     case E_V8QImode:
    4502          263 :     case E_V4HImode:
    4503          263 :     case E_V4HFmode:
    4504          263 :     case E_V4BFmode:
    4505          263 :     case E_V2SImode:
    4506          263 :       if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
    4507              :         {
    4508              :           gen = gen_mmx_pblendvb_v8qi;
    4509              :           blend_mode = V8QImode;
    4510              :         }
    4511              :       break;
    4512           95 :     case E_V4QImode:
    4513           95 :     case E_V2HImode:
    4514           95 :     case E_V2HFmode:
    4515           95 :     case E_V2BFmode:
    4516           95 :       if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
    4517              :         {
    4518              :           gen = gen_mmx_pblendvb_v4qi;
    4519              :           blend_mode = V4QImode;
    4520              :         }
    4521              :       break;
    4522           36 :     case E_V2QImode:
    4523           36 :       if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
    4524              :         gen = gen_mmx_pblendvb_v2qi;
    4525              :       break;
    4526         5768 :     case E_V16QImode:
    4527         5768 :     case E_V8HImode:
    4528         5768 :     case E_V8HFmode:
    4529         5768 :     case E_V8BFmode:
    4530         5768 :     case E_V4SImode:
    4531         5768 :     case E_V2DImode:
    4532         5768 :     case E_V1TImode:
    4533         5768 :       if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
    4534              :         {
    4535              :           gen = gen_sse4_1_pblendvb;
    4536              :           blend_mode = V16QImode;
    4537              :         }
    4538              :       break;
    4539           83 :     case E_V8SFmode:
    4540           83 :       if (TARGET_AVX && TARGET_SSE_MOVCC_USE_BLENDV)
    4541              :         gen = gen_avx_blendvps256;
    4542              :       break;
    4543          192 :     case E_V4DFmode:
    4544          192 :       if (TARGET_AVX && TARGET_SSE_MOVCC_USE_BLENDV)
    4545              :         gen = gen_avx_blendvpd256;
    4546              :       break;
    4547          378 :     case E_V32QImode:
    4548          378 :     case E_V16HImode:
    4549          378 :     case E_V16HFmode:
    4550          378 :     case E_V16BFmode:
    4551          378 :     case E_V8SImode:
    4552          378 :     case E_V4DImode:
    4553          378 :       if (TARGET_AVX2 && TARGET_SSE_MOVCC_USE_BLENDV)
    4554              :         {
    4555              :           gen = gen_avx2_pblendvb;
    4556              :           blend_mode = V32QImode;
    4557              :         }
    4558              :       break;
    4559              : 
    4560            0 :     case E_V64QImode:
    4561            0 :       gen = gen_avx512bw_blendmv64qi;
    4562            0 :       break;
    4563            0 :     case E_V32HImode:
    4564            0 :       gen = gen_avx512bw_blendmv32hi;
    4565            0 :       break;
    4566            0 :     case E_V32HFmode:
    4567            0 :       gen = gen_avx512bw_blendmv32hf;
    4568            0 :       break;
    4569            0 :     case E_V32BFmode:
    4570            0 :       gen = gen_avx512bw_blendmv32bf;
    4571            0 :       break;
    4572            0 :     case E_V16SImode:
    4573            0 :       gen = gen_avx512f_blendmv16si;
    4574            0 :       break;
    4575            0 :     case E_V8DImode:
    4576            0 :       gen = gen_avx512f_blendmv8di;
    4577            0 :       break;
    4578            0 :     case E_V8DFmode:
    4579            0 :       gen = gen_avx512f_blendmv8df;
    4580            0 :       break;
    4581              :     case E_V16SFmode:
    4582              :       gen = gen_avx512f_blendmv16sf;
    4583              :       break;
    4584              : 
    4585              :     default:
    4586              :       break;
    4587              :     }
    4588              : 
    4589            0 :   if (gen != NULL)
    4590              :     {
    4591         2083 :       if (GET_MODE_SIZE (mode) < 16
    4592         2083 :           || !vector_operand (op_true, mode))
    4593          559 :         op_true = force_reg (mode, op_true);
    4594         2083 :       op_false = force_reg (mode, op_false);
    4595              : 
    4596         2083 :       if (blend_mode == mode)
    4597              :         x = dest;
    4598              :       else
    4599              :         {
    4600         1015 :           x = gen_reg_rtx (blend_mode);
    4601         1015 :           op_false = gen_lowpart (blend_mode, op_false);
    4602         1015 :           op_true = gen_lowpart (blend_mode, op_true);
    4603         1015 :           cmp = gen_lowpart (blend_mode, cmp);
    4604              :         }
    4605              : 
    4606         2083 :       emit_insn (gen (x, op_false, op_true, cmp));
    4607              : 
    4608         2083 :       if (x != dest)
    4609         1015 :         emit_move_insn (dest, gen_lowpart (mode, x));
    4610              :     }
    4611         9926 :   else if (CONST_VECTOR_P (op_true) && CONST_VECTOR_P (op_false))
    4612              :     {
    4613          444 :       rtx tmp = simplify_const_binary_operation (XOR, mode, op_true, op_false);
    4614          444 :       tmp = expand_simple_binop (mode, AND, cmp, tmp,
    4615              :                                  NULL, 1, OPTAB_DIRECT);
    4616          444 :       tmp = expand_simple_binop (mode, XOR, tmp, op_false,
    4617              :                                  dest, 1, OPTAB_DIRECT);
    4618          444 :       if (tmp != dest)
    4619            0 :         emit_move_insn (dest, tmp);
    4620              :     }
    4621              :   else
    4622              :     {
    4623         9482 :       rtx t2 = expand_simple_binop (mode, AND, cmp, op_true,
    4624              :                                     NULL, 1, OPTAB_DIRECT);
    4625              : 
    4626         9482 :       rtx t3 = gen_reg_rtx (mode);
    4627         9482 :       x = gen_rtx_NOT (mode, cmp);
    4628         9482 :       op_false = force_reg (mode, op_false);
    4629         9482 :       ix86_emit_vec_binop (AND, mode, t3, x, op_false);
    4630              : 
    4631         9482 :       x = expand_simple_binop (mode, IOR, t3, t2,
    4632              :                                dest, 1, OPTAB_DIRECT);
    4633         9482 :       if (x != dest)
    4634            0 :         emit_move_insn (dest, x);
    4635              :     }
    4636              : }
    4637              : 
    4638              : /* Swap, force into registers, or otherwise massage the two operands
    4639              :    to an sse comparison with a mask result.  Thus we differ a bit from
    4640              :    ix86_prepare_fp_compare_args which expects to produce a flags result.
    4641              : 
    4642              :    The DEST operand exists to help determine whether to commute commutative
    4643              :    operators.  The POP0/POP1 operands are updated in place.  The new
    4644              :    comparison code is returned, or UNKNOWN if not implementable.  */
    4645              : 
    4646              : static enum rtx_code
    4647        20442 : ix86_prepare_sse_fp_compare_args (rtx dest, enum rtx_code code,
    4648              :                                   rtx *pop0, rtx *pop1)
    4649              : {
    4650        20442 :   switch (code)
    4651              :     {
    4652           54 :     case LTGT:
    4653           54 :     case UNEQ:
    4654              :       /* AVX supports all the needed comparisons.  */
    4655           54 :       if (TARGET_AVX)
    4656              :         break;
    4657              :       /* We have no LTGT as an operator.  We could implement it with
    4658              :          NE & ORDERED, but this requires an extra temporary.  It's
    4659              :          not clear that it's worth it.  */
    4660              :       return UNKNOWN;
    4661              : 
    4662              :     case LT:
    4663              :     case LE:
    4664              :     case UNGT:
    4665              :     case UNGE:
    4666              :       /* These are supported directly.  */
    4667              :       break;
    4668              : 
    4669         6011 :     case EQ:
    4670         6011 :     case NE:
    4671         6011 :     case UNORDERED:
    4672         6011 :     case ORDERED:
    4673              :       /* AVX has 3 operand comparisons, no need to swap anything.  */
    4674         6011 :       if (TARGET_AVX)
    4675              :         break;
    4676              :       /* For commutative operators, try to canonicalize the destination
    4677              :          operand to be first in the comparison - this helps reload to
    4678              :          avoid extra moves.  */
    4679         1412 :       if (!dest || !rtx_equal_p (dest, *pop1))
    4680              :         break;
    4681              :       /* FALLTHRU */
    4682              : 
    4683        13276 :     case GE:
    4684        13276 :     case GT:
    4685        13276 :     case UNLE:
    4686        13276 :     case UNLT:
    4687              :       /* These are not supported directly before AVX, and furthermore
    4688              :          ix86_expand_sse_fp_minmax only optimizes LT/UNGE.  Swap the
    4689              :          comparison operands to transform into something that is
    4690              :          supported.  */
    4691        13276 :       std::swap (*pop0, *pop1);
    4692        13276 :       code = swap_condition (code);
    4693        13276 :       break;
    4694              : 
    4695            0 :     default:
    4696            0 :       gcc_unreachable ();
    4697              :     }
    4698              : 
    4699              :   return code;
    4700              : }
    4701              : 
    4702              : /* Expand a floating-point conditional move.  Return true if successful.  */
    4703              : 
    4704              : bool
    4705        81796 : ix86_expand_fp_movcc (rtx operands[])
    4706              : {
    4707        81796 :   machine_mode mode = GET_MODE (operands[0]);
    4708        81796 :   enum rtx_code code = GET_CODE (operands[1]);
    4709        81796 :   rtx tmp, compare_op;
    4710        81796 :   rtx op0 = XEXP (operands[1], 0);
    4711        81796 :   rtx op1 = XEXP (operands[1], 1);
    4712              : 
    4713        81796 :   if (GET_MODE (op0) == BFmode
    4714        81796 :       && !ix86_fp_comparison_operator (operands[1], VOIDmode))
    4715              :     return false;
    4716              : 
    4717        81796 :   if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
    4718              :     {
    4719        67318 :       machine_mode cmode;
    4720              : 
    4721              :       /* Since we've no cmove for sse registers, don't force bad register
    4722              :          allocation just to gain access to it.  Deny movcc when the
    4723              :          comparison mode doesn't match the move mode.  */
    4724        67318 :       cmode = GET_MODE (op0);
    4725        67318 :       if (cmode == VOIDmode)
    4726            0 :         cmode = GET_MODE (op1);
    4727        67318 :       if (cmode != mode)
    4728              :         return false;
    4729              : 
    4730        12378 :       code = ix86_prepare_sse_fp_compare_args (operands[0], code, &op0, &op1);
    4731        12378 :       if (code == UNKNOWN)
    4732              :         return false;
    4733              : 
    4734        12368 :       if (ix86_expand_sse_fp_minmax (operands[0], code, op0, op1,
    4735              :                                      operands[2], operands[3]))
    4736              :         return true;
    4737              : 
    4738         4772 :       tmp = ix86_expand_sse_cmp (operands[0], code, op0, op1,
    4739              :                                  operands[2], operands[3]);
    4740         4772 :       ix86_expand_sse_movcc (operands[0], tmp, operands[2], operands[3]);
    4741         4772 :       return true;
    4742              :     }
    4743              : 
    4744        14478 :   if (GET_MODE (op0) == TImode
    4745        14478 :       || (GET_MODE (op0) == DImode
    4746           72 :           && !TARGET_64BIT))
    4747              :     return false;
    4748              : 
    4749              :   /* The floating point conditional move instructions don't directly
    4750              :      support conditions resulting from a signed integer comparison.  */
    4751              : 
    4752        14406 :   compare_op = ix86_expand_compare (code, op0, op1);
    4753        14406 :   if (!fcmov_comparison_operator (compare_op, VOIDmode))
    4754              :     {
    4755          146 :       tmp = gen_reg_rtx (QImode);
    4756          146 :       ix86_expand_setcc (tmp, code, op0, op1);
    4757              : 
    4758          146 :       compare_op = ix86_expand_compare (NE, tmp, const0_rtx);
    4759              :     }
    4760              : 
    4761        14406 :   operands[2] = force_reg (mode, operands[2]);
    4762        14406 :   operands[3] = force_reg (mode, operands[3]);
    4763        14406 :   emit_insn (gen_rtx_SET (operands[0],
    4764              :                           gen_rtx_IF_THEN_ELSE (mode, compare_op,
    4765              :                                                 operands[2], operands[3])));
    4766              : 
    4767        14406 :   return true;
    4768              : }
    4769              : 
    4770              : /* Expand a conditional fma: operands[0] = operands[1]
    4771              :    ? fma (operands[2], operands[3], operands[4]) : operands[5].  */
    4772              : 
    4773              : void
    4774          322 : ix86_expand_cond_fma (rtx operands[], bool neg_mult, bool neg_add)
    4775              : {
    4776          322 :   machine_mode mode = GET_MODE (operands[0]);
    4777          322 :   rtx mask = operands[1];
    4778          322 :   rtx op1 = operands[2], op2 = operands[3], op3 = operands[4];
    4779          322 :   rtx op4 = operands[5];
    4780          322 :   rtx tmp, fma;
    4781              : 
    4782              :   /* The multiplication is commutative, so an else value equal to the
    4783              :      second operand for mult can be merged just like first operand
    4784              :      after swapping.  */
    4785          322 :   if (rtx_equal_p (op4, op2))
    4786           54 :     std::swap (op1, op2);
    4787              : 
    4788          472 :   fma = gen_rtx_FMA (mode, neg_mult ? gen_rtx_NEG (mode, op1) : op1,
    4789              :                      op2, neg_add ? gen_rtx_NEG (mode, op3) : op3);
    4790              : 
    4791              :   /* Do not emit an extra move.  */
    4792          322 :   if (rtx_equal_p (op4, op1)
    4793          246 :       || rtx_equal_p (op4, op3)
    4794          486 :       || op4 == CONST0_RTX (mode))
    4795              :     {
    4796          250 :       emit_insn (gen_rtx_SET (operands[0],
    4797              :                               gen_rtx_VEC_MERGE (mode, fma, op4, mask)));
    4798          250 :       return;
    4799              :     }
    4800              : 
    4801           72 :   tmp = gen_reg_rtx (mode);
    4802           72 :   emit_insn (gen_rtx_SET (tmp, gen_rtx_VEC_MERGE (mode, fma,
    4803              :                                                   CONST0_RTX (mode), mask)));
    4804           72 :   emit_move_insn (operands[0], gen_rtx_VEC_MERGE (mode, tmp, op4, mask));
    4805              : }
    4806              : 
    4807              : /* Helper for ix86_cmp_code_to_pcmp_immediate for int modes.  */
    4808              : 
    4809              : static int
    4810         4985 : ix86_int_cmp_code_to_pcmp_immediate (enum rtx_code code)
    4811              : {
    4812         4985 :   switch (code)
    4813              :     {
    4814              :     case EQ:
    4815              :       return 0;
    4816          380 :     case LT:
    4817          380 :     case LTU:
    4818          380 :       return 1;
    4819          220 :     case LE:
    4820          220 :     case LEU:
    4821          220 :       return 2;
    4822         3165 :     case NE:
    4823         3165 :       return 4;
    4824          299 :     case GE:
    4825          299 :     case GEU:
    4826          299 :       return 5;
    4827          497 :     case GT:
    4828          497 :     case GTU:
    4829          497 :       return 6;
    4830            0 :     default:
    4831            0 :       gcc_unreachable ();
    4832              :     }
    4833              : }
    4834              : 
    4835              : /* Helper for ix86_cmp_code_to_pcmp_immediate for fp modes.  */
    4836              : 
    4837              : static int
    4838         1822 : ix86_fp_cmp_code_to_pcmp_immediate (enum rtx_code code)
    4839              : {
    4840         1822 :   switch (code)
    4841              :     {
    4842              :     case EQ:
    4843              :       return 0x00;
    4844          374 :     case NE:
    4845          374 :       return 0x04;
    4846          510 :     case GT:
    4847          510 :       return 0x0e;
    4848           88 :     case LE:
    4849           88 :       return 0x02;
    4850           53 :     case GE:
    4851           53 :       return 0x0d;
    4852          638 :     case LT:
    4853          638 :       return 0x01;
    4854            3 :     case UNLE:
    4855            3 :       return 0x1a;
    4856            3 :     case UNLT:
    4857            3 :       return 0x19;
    4858           12 :     case UNGE:
    4859           12 :       return 0x15;
    4860           45 :     case UNGT:
    4861           45 :       return 0x16;
    4862            3 :     case UNEQ:
    4863            3 :       return 0x08;
    4864            2 :     case LTGT:
    4865            2 :       return 0x1c;
    4866            2 :     case ORDERED:
    4867            2 :       return 0x07;
    4868            2 :     case UNORDERED:
    4869            2 :       return 0x03;
    4870            0 :     default:
    4871            0 :       gcc_unreachable ();
    4872              :     }
    4873              : }
    4874              : 
    4875              : /* Return immediate value to be used in UNSPEC_PCMP
    4876              :    for comparison CODE in MODE.  */
    4877              : 
    4878              : static int
    4879         6807 : ix86_cmp_code_to_pcmp_immediate (enum rtx_code code, machine_mode mode)
    4880              : {
    4881         6807 :   if (FLOAT_MODE_P (mode))
    4882         1822 :     return ix86_fp_cmp_code_to_pcmp_immediate (code);
    4883         4985 :   return ix86_int_cmp_code_to_pcmp_immediate (code);
    4884              : }
    4885              : 
    4886              : /* Expand AVX-512 vector comparison.  */
    4887              : 
    4888              : bool
    4889         6807 : ix86_expand_mask_vec_cmp (rtx dest, enum rtx_code code, rtx cmp_op0, rtx cmp_op1)
    4890              : {
    4891         6807 :   machine_mode mask_mode = GET_MODE (dest);
    4892         6807 :   machine_mode cmp_mode = GET_MODE (cmp_op0);
    4893         6807 :   rtx imm = GEN_INT (ix86_cmp_code_to_pcmp_immediate (code, cmp_mode));
    4894         6807 :   int unspec_code;
    4895         6807 :   rtx unspec;
    4896              : 
    4897         6807 :   switch (code)
    4898              :     {
    4899              :     case LEU:
    4900              :     case GTU:
    4901              :     case GEU:
    4902              :     case LTU:
    4903              :       unspec_code = UNSPEC_UNSIGNED_PCMP;
    4904              :       break;
    4905              : 
    4906         6398 :     default:
    4907         6398 :       unspec_code = UNSPEC_PCMP;
    4908              :     }
    4909              : 
    4910         6807 :   unspec = gen_rtx_UNSPEC (mask_mode, gen_rtvec (3, cmp_op0, cmp_op1, imm),
    4911              :                            unspec_code);
    4912         6807 :   emit_insn (gen_rtx_SET (dest, unspec));
    4913              : 
    4914         6807 :   return true;
    4915              : }
    4916              : 
    4917              : /* Expand fp vector comparison.  */
    4918              : 
    4919              : bool
    4920         8064 : ix86_expand_fp_vec_cmp (rtx operands[])
    4921              : {
    4922         8064 :   enum rtx_code code = GET_CODE (operands[1]);
    4923         8064 :   rtx cmp;
    4924              : 
    4925         8064 :   code = ix86_prepare_sse_fp_compare_args (operands[0], code,
    4926              :                                            &operands[2], &operands[3]);
    4927         8064 :   if (code == UNKNOWN)
    4928              :     {
    4929           26 :       rtx temp;
    4930           26 :       switch (GET_CODE (operands[1]))
    4931              :         {
    4932            4 :         case LTGT:
    4933            4 :           temp = ix86_expand_sse_cmp (operands[0], LT, operands[2],
    4934              :                                       operands[3], NULL, NULL);
    4935            4 :           cmp = ix86_expand_sse_cmp (operands[0], LT, operands[3],
    4936              :                                      operands[2], NULL, NULL);
    4937            4 :           code = IOR;
    4938            4 :           break;
    4939           22 :         case UNEQ:
    4940           22 :           temp = ix86_expand_sse_cmp (operands[0], UNORDERED, operands[2],
    4941              :                                       operands[3], NULL, NULL);
    4942           22 :           cmp = ix86_expand_sse_cmp (operands[0], EQ, operands[2],
    4943              :                                      operands[3], NULL, NULL);
    4944           22 :           code = IOR;
    4945           22 :           break;
    4946            0 :         default:
    4947            0 :           gcc_unreachable ();
    4948              :         }
    4949           26 :       cmp = expand_simple_binop (GET_MODE (cmp), code, temp, cmp, cmp, 1,
    4950              :                                  OPTAB_DIRECT);
    4951              :     }
    4952              :   else
    4953         8038 :     cmp = ix86_expand_sse_cmp (operands[0], code, operands[2], operands[3],
    4954              :                                NULL, NULL);
    4955              : 
    4956         8064 :   if (operands[0] != cmp)
    4957         7984 :     emit_move_insn (operands[0], cmp);
    4958              : 
    4959         8064 :   return true;
    4960              : }
    4961              : 
    4962              : static rtx
    4963        18719 : ix86_expand_int_sse_cmp (rtx dest, enum rtx_code code, rtx cop0, rtx cop1,
    4964              :                          rtx op_true, rtx op_false, bool *negate)
    4965              : {
    4966        18719 :   machine_mode data_mode = GET_MODE (dest);
    4967        18719 :   machine_mode mode = GET_MODE (cop0);
    4968        18719 :   rtx x;
    4969              : 
    4970        18719 :   *negate = false;
    4971              : 
    4972              :   /* XOP supports all of the comparisons on all 128-bit vector int types.  */
    4973        18719 :   if (TARGET_XOP
    4974          147 :       && GET_MODE_CLASS (mode) == MODE_VECTOR_INT
    4975        18866 :       && GET_MODE_SIZE (mode) <= 16)
    4976              :     ;
    4977              :   /* AVX512F supports all of the comparisons
    4978              :      on all 128/256/512-bit vector int types.  */
    4979        18575 :   else if (ix86_use_mask_cmp_p (data_mode, mode, op_true, op_false))
    4980              :     ;
    4981              :   else
    4982              :     {
    4983              :       /* Canonicalize the comparison to EQ, GT, GTU.  */
    4984        18522 :       switch (code)
    4985              :         {
    4986              :         case EQ:
    4987              :         case GT:
    4988              :         case GTU:
    4989              :           break;
    4990              : 
    4991          962 :         case LE:
    4992          962 :         case LEU:
    4993              :           /* x <= cst can be handled as x < cst + 1 unless there is
    4994              :              wrap around in cst + 1.  */
    4995          962 :           if (CONST_VECTOR_P (cop1)
    4996         1645 :               && GET_MODE_INNER (mode) != TImode)
    4997              :             {
    4998          683 :               unsigned int n_elts = GET_MODE_NUNITS (mode), i;
    4999          683 :               machine_mode eltmode = GET_MODE_INNER (mode);
    5000         4404 :               for (i = 0; i < n_elts; ++i)
    5001              :                 {
    5002         3722 :                   rtx elt = CONST_VECTOR_ELT (cop1, i);
    5003         3722 :                   if (!CONST_INT_P (elt))
    5004              :                     break;
    5005         3722 :                   if (code == LE)
    5006              :                     {
    5007              :                       /* For LE punt if some element is signed maximum.  */
    5008         2546 :                       if ((INTVAL (elt) & (GET_MODE_MASK (eltmode) >> 1))
    5009              :                           == (GET_MODE_MASK (eltmode) >> 1))
    5010              :                         break;
    5011              :                     }
    5012              :                   /* For LEU punt if some element is unsigned maximum.  */
    5013         1176 :                   else if (elt == constm1_rtx)
    5014              :                     break;
    5015              :                 }
    5016          683 :               if (i == n_elts)
    5017              :                 {
    5018          682 :                   rtvec v = rtvec_alloc (n_elts);
    5019         5084 :                   for (i = 0; i < n_elts; ++i)
    5020         3720 :                     RTVEC_ELT (v, i)
    5021         3720 :                       = gen_int_mode (INTVAL (CONST_VECTOR_ELT (cop1, i)) + 1,
    5022              :                                       eltmode);
    5023          682 :                   cop1 = gen_rtx_CONST_VECTOR (mode, v);
    5024          682 :                   std::swap (cop0, cop1);
    5025          682 :                   code = code == LE ? GT : GTU;
    5026              :                   break;
    5027              :                 }
    5028              :             }
    5029              :           /* FALLTHRU */
    5030         3394 :         case NE:
    5031         3394 :           code = reverse_condition (code);
    5032         3394 :           *negate = true;
    5033         3394 :           break;
    5034              : 
    5035          547 :         case GE:
    5036          547 :         case GEU:
    5037              :           /* x >= cst can be handled as x > cst - 1 unless there is
    5038              :              wrap around in cst - 1.  */
    5039          547 :           if (CONST_VECTOR_P (cop1)
    5040          842 :               && GET_MODE_INNER (mode) != TImode)
    5041              :             {
    5042          295 :               unsigned int n_elts = GET_MODE_NUNITS (mode), i;
    5043          295 :               machine_mode eltmode = GET_MODE_INNER (mode);
    5044         2211 :               for (i = 0; i < n_elts; ++i)
    5045              :                 {
    5046         1964 :                   rtx elt = CONST_VECTOR_ELT (cop1, i);
    5047         1964 :                   if (!CONST_INT_P (elt))
    5048              :                     break;
    5049         1964 :                   if (code == GE)
    5050              :                     {
    5051              :                       /* For GE punt if some element is signed minimum.  */
    5052         1916 :                       if (INTVAL (elt) < 0
    5053          136 :                           && ((INTVAL (elt) & (GET_MODE_MASK (eltmode) >> 1))
    5054              :                               == 0))
    5055              :                         break;
    5056              :                     }
    5057              :                   /* For GEU punt if some element is zero.  */
    5058           48 :                   else if (elt == const0_rtx)
    5059              :                     break;
    5060              :                 }
    5061          295 :               if (i == n_elts)
    5062              :                 {
    5063          247 :                   rtvec v = rtvec_alloc (n_elts);
    5064         2410 :                   for (i = 0; i < n_elts; ++i)
    5065         1916 :                     RTVEC_ELT (v, i)
    5066         1916 :                       = gen_int_mode (INTVAL (CONST_VECTOR_ELT (cop1, i)) - 1,
    5067              :                                       eltmode);
    5068          247 :                   cop1 = gen_rtx_CONST_VECTOR (mode, v);
    5069          247 :                   code = code == GE ? GT : GTU;
    5070              :                   break;
    5071              :                 }
    5072              :             }
    5073          300 :           code = reverse_condition (code);
    5074          300 :           *negate = true;
    5075              :           /* FALLTHRU */
    5076              : 
    5077         1666 :         case LT:
    5078         1666 :         case LTU:
    5079         1666 :           std::swap (cop0, cop1);
    5080         1666 :           code = swap_condition (code);
    5081         1666 :           break;
    5082              : 
    5083            0 :         default:
    5084            0 :           gcc_unreachable ();
    5085              :         }
    5086              : 
    5087              :       /* Only SSE4.1/SSE4.2 supports V2DImode.  */
    5088        18522 :       if (mode == V2DImode)
    5089              :         {
    5090          843 :           switch (code)
    5091              :             {
    5092          642 :             case EQ:
    5093              :               /* SSE4.1 supports EQ.  */
    5094          642 :               if (!TARGET_SSE4_1)
    5095        18719 :                 return NULL;
    5096              :               break;
    5097              : 
    5098          201 :             case GT:
    5099          201 :             case GTU:
    5100              :               /* SSE4.2 supports GT/GTU.  */
    5101          201 :               if (!TARGET_SSE4_2)
    5102              :                 return NULL;
    5103              :               break;
    5104              : 
    5105            0 :             default:
    5106            0 :               gcc_unreachable ();
    5107              :             }
    5108              :         }
    5109              : 
    5110        18522 :       if (CONST_VECTOR_P (cop0))
    5111         1349 :         cop0 = force_reg (mode, cop0);
    5112        17173 :       else if (CONST_VECTOR_P (cop1))
    5113         7970 :         cop1 = force_reg (mode, cop1);
    5114              : 
    5115        18522 :       rtx optrue = op_true ? op_true : CONSTM1_RTX (data_mode);
    5116        18522 :       rtx opfalse = op_false ? op_false : CONST0_RTX (data_mode);
    5117        18522 :       if (*negate)
    5118         3694 :         std::swap (optrue, opfalse);
    5119              : 
    5120              :       /* Transform x > y ? 0 : -1 (i.e. x <= y ? -1 : 0 or x <= y) when
    5121              :          not using integer masks into min (x, y) == x ? -1 : 0 (i.e.
    5122              :          min (x, y) == x).  While we add one instruction (the minimum),
    5123              :          we remove the need for two instructions in the negation, as the
    5124              :          result is done this way.
    5125              :          When using masks, do it for SI/DImode element types, as it is shorter
    5126              :          than the two subtractions.  */
    5127        18522 :       if ((code != EQ
    5128         7657 :            && GET_MODE_SIZE (mode) != 64
    5129         7657 :            && vector_all_ones_operand (opfalse, data_mode)
    5130          580 :            && optrue == CONST0_RTX (data_mode))
    5131        25599 :           || (code == GTU
    5132         2164 :               && GET_MODE_SIZE (GET_MODE_INNER (mode)) >= 4
    5133              :               /* Don't do it if not using integer masks and we'd end up with
    5134              :                  the right values in the registers though.  */
    5135          805 :               && (GET_MODE_SIZE (mode) == 64
    5136          805 :                   || !vector_all_ones_operand (optrue, data_mode)
    5137          676 :                   || opfalse != CONST0_RTX (data_mode))))
    5138              :         {
    5139          709 :           rtx (*gen) (rtx, rtx, rtx) = NULL;
    5140              : 
    5141          709 :           switch (mode)
    5142              :             {
    5143            0 :             case E_V16SImode:
    5144            0 :               gen = (code == GTU) ? gen_uminv16si3 : gen_sminv16si3;
    5145              :               break;
    5146            0 :             case E_V8DImode:
    5147            0 :               gen = (code == GTU) ? gen_uminv8di3 : gen_sminv8di3;
    5148            0 :               cop0 = force_reg (mode, cop0);
    5149            0 :               cop1 = force_reg (mode, cop1);
    5150            0 :               break;
    5151           24 :             case E_V32QImode:
    5152           24 :               if (TARGET_AVX2)
    5153           24 :                 gen = (code == GTU) ? gen_uminv32qi3 : gen_sminv32qi3;
    5154              :               break;
    5155           24 :             case E_V16HImode:
    5156           24 :               if (TARGET_AVX2)
    5157           24 :                 gen = (code == GTU) ? gen_uminv16hi3 : gen_sminv16hi3;
    5158              :               break;
    5159           25 :             case E_V8SImode:
    5160           25 :               if (TARGET_AVX2)
    5161           25 :                 gen = (code == GTU) ? gen_uminv8si3 : gen_sminv8si3;
    5162              :               break;
    5163           20 :             case E_V4DImode:
    5164           20 :               if (TARGET_AVX512VL)
    5165              :                 {
    5166            0 :                   gen = (code == GTU) ? gen_uminv4di3 : gen_sminv4di3;
    5167            0 :                   cop0 = force_reg (mode, cop0);
    5168            0 :                   cop1 = force_reg (mode, cop1);
    5169              :                 }
    5170              :               break;
    5171           64 :             case E_V16QImode:
    5172           64 :               if (code == GTU && TARGET_SSE2)
    5173              :                 gen = gen_uminv16qi3;
    5174           28 :               else if (code == GT && TARGET_SSE4_1)
    5175              :                 gen = gen_sminv16qi3;
    5176              :               break;
    5177           44 :             case E_V8QImode:
    5178           44 :               if (code == GTU && TARGET_SSE2)
    5179              :                 gen = gen_uminv8qi3;
    5180           42 :               else if (code == GT && TARGET_SSE4_1)
    5181              :                 gen = gen_sminv8qi3;
    5182              :               break;
    5183           13 :             case E_V4QImode:
    5184           13 :               if (code == GTU && TARGET_SSE2)
    5185              :                 gen = gen_uminv4qi3;
    5186            2 :               else if (code == GT && TARGET_SSE4_1)
    5187              :                 gen = gen_sminv4qi3;
    5188              :               break;
    5189            8 :             case E_V2QImode:
    5190            8 :               if (code == GTU && TARGET_SSE2)
    5191              :                 gen = gen_uminv2qi3;
    5192            6 :               else if (code == GT && TARGET_SSE4_1)
    5193              :                 gen = gen_sminv2qi3;
    5194              :               break;
    5195           73 :             case E_V8HImode:
    5196           73 :               if (code == GTU && TARGET_SSE4_1)
    5197              :                 gen = gen_uminv8hi3;
    5198           63 :               else if (code == GT && TARGET_SSE2)
    5199              :                 gen = gen_sminv8hi3;
    5200              :               break;
    5201            4 :             case E_V4HImode:
    5202            4 :               if (code == GTU && TARGET_SSE4_1)
    5203              :                 gen = gen_uminv4hi3;
    5204            4 :               else if (code == GT && TARGET_SSE2)
    5205              :                 gen = gen_sminv4hi3;
    5206              :               break;
    5207           16 :             case E_V2HImode:
    5208           16 :               if (code == GTU && TARGET_SSE4_1)
    5209              :                 gen = gen_uminv2hi3;
    5210           16 :               else if (code == GT && TARGET_SSE2)
    5211              :                 gen = gen_sminv2hi3;
    5212              :               break;
    5213          254 :             case E_V4SImode:
    5214          254 :               if (TARGET_SSE4_1)
    5215           52 :                 gen = (code == GTU) ? gen_uminv4si3 : gen_sminv4si3;
    5216              :               break;
    5217          116 :             case E_V2SImode:
    5218          116 :               if (TARGET_SSE4_1)
    5219            0 :                 gen = (code == GTU) ? gen_uminv2si3 : gen_sminv2si3;
    5220              :               break;
    5221           24 :             case E_V2DImode:
    5222           24 :               if (TARGET_AVX512VL)
    5223              :                 {
    5224            0 :                   gen = (code == GTU) ? gen_uminv2di3 : gen_sminv2di3;
    5225            0 :                   cop0 = force_reg (mode, cop0);
    5226            0 :                   cop1 = force_reg (mode, cop1);
    5227              :                 }
    5228              :               break;
    5229              :             default:
    5230              :               break;
    5231              :             }
    5232              : 
    5233            0 :           if (gen)
    5234              :             {
    5235          280 :               rtx tem = gen_reg_rtx (mode);
    5236          280 :               if (!vector_operand (cop0, mode))
    5237            0 :                 cop0 = force_reg (mode, cop0);
    5238          280 :               if (!vector_operand (cop1, mode))
    5239            0 :                 cop1 = force_reg (mode, cop1);
    5240          280 :               *negate = !*negate;
    5241          280 :               emit_insn (gen (tem, cop0, cop1));
    5242          280 :               cop1 = tem;
    5243          280 :               code = EQ;
    5244              :             }
    5245              :         }
    5246              : 
    5247              :       /* Unsigned parallel compare is not supported by the hardware.
    5248              :          Play some tricks to turn this into a signed comparison
    5249              :          against 0.  */
    5250        18522 :       if (code == GTU)
    5251              :         {
    5252         1219 :           cop0 = force_reg (mode, cop0);
    5253              : 
    5254         1219 :           switch (mode)
    5255              :             {
    5256          911 :             case E_V16SImode:
    5257          911 :             case E_V8DImode:
    5258          911 :             case E_V8SImode:
    5259          911 :             case E_V4DImode:
    5260          911 :             case E_V4SImode:
    5261          911 :             case E_V2SImode:
    5262          911 :             case E_V2DImode:
    5263          911 :                 {
    5264          911 :                   rtx t1, t2, mask;
    5265              : 
    5266              :                   /* Subtract (-(INT MAX) - 1) from both operands to make
    5267              :                      them signed.  */
    5268          911 :                   mask = ix86_build_signbit_mask (mode, true, false);
    5269          911 :                   t1 = gen_reg_rtx (mode);
    5270          911 :                   emit_insn (gen_sub3_insn (t1, cop0, mask));
    5271              : 
    5272          911 :                   t2 = gen_reg_rtx (mode);
    5273          911 :                   emit_insn (gen_sub3_insn (t2, cop1, mask));
    5274              : 
    5275          911 :                   cop0 = t1;
    5276          911 :                   cop1 = t2;
    5277          911 :                   code = GT;
    5278              :                 }
    5279          911 :               break;
    5280              : 
    5281          308 :             case E_V64QImode:
    5282          308 :             case E_V32HImode:
    5283          308 :             case E_V32QImode:
    5284          308 :             case E_V16HImode:
    5285          308 :             case E_V16QImode:
    5286          308 :             case E_V8QImode:
    5287          308 :             case E_V4QImode:
    5288          308 :             case E_V2QImode:
    5289          308 :             case E_V8HImode:
    5290          308 :             case E_V4HImode:
    5291          308 :             case E_V2HImode:
    5292              :               /* Perform a parallel unsigned saturating subtraction.  */
    5293          308 :               x = gen_reg_rtx (mode);
    5294          308 :               emit_insn (gen_rtx_SET
    5295              :                          (x, gen_rtx_US_MINUS (mode, cop0, cop1)));
    5296          308 :               cop0 = x;
    5297          308 :               cop1 = CONST0_RTX (mode);
    5298          308 :               code = EQ;
    5299          308 :               *negate = !*negate;
    5300          308 :               break;
    5301              : 
    5302            0 :             default:
    5303            0 :               gcc_unreachable ();
    5304              :             }
    5305              :         }
    5306              :     }
    5307              : 
    5308        18719 :   if (*negate)
    5309         3660 :     std::swap (op_true, op_false);
    5310              : 
    5311        18719 :   if (CONST_VECTOR_P (cop1))
    5312          323 :     cop1 = force_reg (mode, cop1);
    5313              : 
    5314              :   /* Allow the comparison to be done in one mode, but the movcc to
    5315              :      happen in another mode.  */
    5316        18719 :   if (data_mode == mode)
    5317        18677 :     x = ix86_expand_sse_cmp (dest, code, cop0, cop1, op_true, op_false);
    5318              :   else
    5319              :     {
    5320          126 :       gcc_assert (GET_MODE_SIZE (data_mode) == GET_MODE_SIZE (mode));
    5321           42 :       x = ix86_expand_sse_cmp (gen_reg_rtx (mode), code, cop0, cop1,
    5322              :                                op_true, op_false);
    5323           42 :       if (GET_MODE (x) == mode)
    5324           24 :         x = gen_lowpart (data_mode, x);
    5325              :     }
    5326              : 
    5327              :   return x;
    5328              : }
    5329              : 
    5330              : /* Expand integer vector comparison.  */
    5331              : 
    5332              : bool
    5333        11754 : ix86_expand_int_vec_cmp (rtx operands[])
    5334              : {
    5335        11754 :   rtx_code code = GET_CODE (operands[1]);
    5336        11754 :   bool negate = false;
    5337        11754 :   rtx cmp = ix86_expand_int_sse_cmp (operands[0], code, operands[2],
    5338              :                                      operands[3], NULL, NULL, &negate);
    5339              : 
    5340        11754 :   if (!cmp)
    5341              :     return false;
    5342              : 
    5343        11754 :   if (negate)
    5344              :     {
    5345         3752 :       if (TARGET_AVX512F && GET_MODE_SIZE (GET_MODE (cmp)) >= 16)
    5346           93 :         cmp = gen_rtx_XOR (GET_MODE (cmp), cmp, CONSTM1_RTX (GET_MODE (cmp)));
    5347              :       else
    5348              :         {
    5349         7062 :           cmp = ix86_expand_int_sse_cmp (operands[0], EQ, cmp,
    5350         3531 :                                          CONST0_RTX (GET_MODE (cmp)),
    5351              :                                          NULL, NULL, &negate);
    5352         3531 :           gcc_assert (!negate);
    5353              :         }
    5354              :     }
    5355              : 
    5356        11754 :   if (operands[0] != cmp)
    5357        11459 :     emit_move_insn (operands[0], cmp);
    5358              : 
    5359              :   return true;
    5360              : }
    5361              : 
    5362              : /* Expand a floating-point vector conditional move; a vcond operation
    5363              :    rather than a movcc operation.  */
    5364              : 
    5365              : bool
    5366            0 : ix86_expand_fp_vcond (rtx operands[])
    5367              : {
    5368            0 :   enum rtx_code code = GET_CODE (operands[3]);
    5369            0 :   rtx cmp;
    5370              : 
    5371            0 :   code = ix86_prepare_sse_fp_compare_args (operands[0], code,
    5372              :                                            &operands[4], &operands[5]);
    5373            0 :   if (code == UNKNOWN)
    5374              :     {
    5375            0 :       rtx temp;
    5376            0 :       switch (GET_CODE (operands[3]))
    5377              :         {
    5378            0 :         case LTGT:
    5379            0 :           temp = ix86_expand_sse_cmp (operands[0], ORDERED, operands[4],
    5380              :                                       operands[5], operands[0], operands[0]);
    5381            0 :           cmp = ix86_expand_sse_cmp (operands[0], NE, operands[4],
    5382              :                                      operands[5], operands[1], operands[2]);
    5383            0 :           code = AND;
    5384            0 :           break;
    5385            0 :         case UNEQ:
    5386            0 :           temp = ix86_expand_sse_cmp (operands[0], UNORDERED, operands[4],
    5387              :                                       operands[5], operands[0], operands[0]);
    5388            0 :           cmp = ix86_expand_sse_cmp (operands[0], EQ, operands[4],
    5389              :                                      operands[5], operands[1], operands[2]);
    5390            0 :           code = IOR;
    5391            0 :           break;
    5392            0 :         default:
    5393            0 :           gcc_unreachable ();
    5394              :         }
    5395            0 :       cmp = expand_simple_binop (GET_MODE (cmp), code, temp, cmp, cmp, 1,
    5396              :                                  OPTAB_DIRECT);
    5397            0 :       ix86_expand_sse_movcc (operands[0], cmp, operands[1], operands[2]);
    5398            0 :       return true;
    5399              :     }
    5400              : 
    5401            0 :   if (ix86_expand_sse_fp_minmax (operands[0], code, operands[4],
    5402              :                                  operands[5], operands[1], operands[2]))
    5403              :     return true;
    5404              : 
    5405            0 :   cmp = ix86_expand_sse_cmp (operands[0], code, operands[4], operands[5],
    5406              :                              operands[1], operands[2]);
    5407            0 :   ix86_expand_sse_movcc (operands[0], cmp, operands[1], operands[2]);
    5408            0 :   return true;
    5409              : }
    5410              : 
    5411              : /* Expand a signed/unsigned integral vector conditional move.  */
    5412              : 
    5413              : bool
    5414         3434 : ix86_expand_int_vcond (rtx operands[])
    5415              : {
    5416         3434 :   machine_mode data_mode = GET_MODE (operands[0]);
    5417         3434 :   machine_mode mode = GET_MODE (operands[4]);
    5418         3434 :   enum rtx_code code = GET_CODE (operands[3]);
    5419         3434 :   bool negate = false;
    5420         3434 :   rtx x, cop0, cop1;
    5421              : 
    5422         3434 :   cop0 = operands[4];
    5423         3434 :   cop1 = operands[5];
    5424              : 
    5425              :   /* Try to optimize x < 0 ? -1 : 0 into (signed) x >> 31
    5426              :      and x < 0 ? 1 : 0 into (unsigned) x >> 31.  */
    5427         3434 :   if ((code == LT || code == GE)
    5428            0 :       && data_mode == mode
    5429            0 :       && cop1 == CONST0_RTX (mode)
    5430            0 :       && operands[1 + (code == LT)] == CONST0_RTX (data_mode)
    5431            0 :       && GET_MODE_UNIT_SIZE (data_mode) > 1
    5432            0 :       && GET_MODE_UNIT_SIZE (data_mode) <= 8
    5433         3434 :       && (GET_MODE_SIZE (data_mode) == 16
    5434            0 :           || (TARGET_AVX2 && GET_MODE_SIZE (data_mode) == 32)))
    5435              :     {
    5436            0 :       rtx negop = operands[2 - (code == LT)];
    5437            0 :       int shift = GET_MODE_UNIT_BITSIZE (data_mode) - 1;
    5438            0 :       if (negop == CONST1_RTX (data_mode))
    5439              :         {
    5440            0 :           rtx res = expand_simple_binop (mode, LSHIFTRT, cop0, GEN_INT (shift),
    5441              :                                          operands[0], 1, OPTAB_DIRECT);
    5442            0 :           if (res != operands[0])
    5443            0 :             emit_move_insn (operands[0], res);
    5444              :           return true;
    5445              :         }
    5446            0 :       else if (GET_MODE_INNER (data_mode) != DImode
    5447            0 :                && vector_all_ones_operand (negop, data_mode))
    5448              :         {
    5449            0 :           rtx res = expand_simple_binop (mode, ASHIFTRT, cop0, GEN_INT (shift),
    5450              :                                          operands[0], 0, OPTAB_DIRECT);
    5451            0 :           if (res != operands[0])
    5452            0 :             emit_move_insn (operands[0], res);
    5453              :           return true;
    5454              :         }
    5455              :     }
    5456              : 
    5457         3434 :   if (!nonimmediate_operand (cop1, mode))
    5458          126 :     cop1 = force_reg (mode, cop1);
    5459         3434 :   if (!general_operand (operands[1], data_mode))
    5460            0 :     operands[1] = force_reg (data_mode, operands[1]);
    5461         3434 :   if (!general_operand (operands[2], data_mode))
    5462            0 :     operands[2] = force_reg (data_mode, operands[2]);
    5463              : 
    5464         3434 :   x = ix86_expand_int_sse_cmp (operands[0], code, cop0, cop1,
    5465              :                                operands[1], operands[2], &negate);
    5466              : 
    5467         3434 :   if (!x)
    5468              :     return false;
    5469              : 
    5470         3434 :   ix86_expand_sse_movcc (operands[0], x, operands[1+negate],
    5471         3434 :                          operands[2-negate]);
    5472         3434 :   return true;
    5473              : }
    5474              : 
    5475              : static bool
    5476       128101 : ix86_expand_vec_perm_vpermt2 (rtx target, rtx mask, rtx op0, rtx op1,
    5477              :                               struct expand_vec_perm_d *d)
    5478              : {
    5479              :   /* ix86_expand_vec_perm_vpermt2 is called from both const and non-const
    5480              :      expander, so args are either in d, or in op0, op1 etc.  */
    5481       128101 :   machine_mode mode = GET_MODE (d ? d->op0 : op0);
    5482       128101 :   machine_mode maskmode = mode;
    5483       128101 :   rtx (*gen) (rtx, rtx, rtx, rtx) = NULL;
    5484              : 
    5485       128101 :   switch (mode)
    5486              :     {
    5487        24232 :     case E_V16QImode:
    5488        24232 :       if (TARGET_AVX512VL && TARGET_AVX512VBMI)
    5489              :         gen = gen_avx512vl_vpermt2varv16qi3;
    5490              :       break;
    5491          514 :     case E_V32QImode:
    5492          514 :       if (TARGET_AVX512VL && TARGET_AVX512VBMI)
    5493              :         gen = gen_avx512vl_vpermt2varv32qi3;
    5494              :       break;
    5495          191 :     case E_V64QImode:
    5496          191 :       if (TARGET_AVX512VBMI)
    5497              :         gen = gen_avx512bw_vpermt2varv64qi3;
    5498              :       break;
    5499        13865 :     case E_V8HImode:
    5500        13865 :       if (TARGET_AVX512VL && TARGET_AVX512BW)
    5501              :         gen = gen_avx512vl_vpermt2varv8hi3;
    5502              :       break;
    5503          775 :     case E_V16HImode:
    5504          775 :       if (TARGET_AVX512VL && TARGET_AVX512BW)
    5505              :         gen = gen_avx512vl_vpermt2varv16hi3;
    5506              :       break;
    5507          342 :     case E_V32HImode:
    5508          342 :       if (TARGET_AVX512BW)
    5509              :         gen = gen_avx512bw_vpermt2varv32hi3;
    5510              :       break;
    5511        34621 :     case E_V4SImode:
    5512        34621 :       if (TARGET_AVX512VL)
    5513              :         gen = gen_avx512vl_vpermt2varv4si3;
    5514              :       break;
    5515         1137 :     case E_V8SImode:
    5516         1137 :       if (TARGET_AVX512VL)
    5517              :         gen = gen_avx512vl_vpermt2varv8si3;
    5518              :       break;
    5519          129 :     case E_V16SImode:
    5520          129 :       if (TARGET_AVX512F)
    5521              :         gen = gen_avx512f_vpermt2varv16si3;
    5522              :       break;
    5523        10776 :     case E_V4SFmode:
    5524        10776 :       if (TARGET_AVX512VL)
    5525              :         {
    5526              :           gen = gen_avx512vl_vpermt2varv4sf3;
    5527              :           maskmode = V4SImode;
    5528              :         }
    5529              :       break;
    5530         6031 :     case E_V8SFmode:
    5531         6031 :       if (TARGET_AVX512VL)
    5532              :         {
    5533              :           gen = gen_avx512vl_vpermt2varv8sf3;
    5534              :           maskmode = V8SImode;
    5535              :         }
    5536              :       break;
    5537          207 :     case E_V16SFmode:
    5538          207 :       if (TARGET_AVX512F)
    5539              :         {
    5540              :           gen = gen_avx512f_vpermt2varv16sf3;
    5541              :           maskmode = V16SImode;
    5542              :         }
    5543              :       break;
    5544            2 :     case E_V2DImode:
    5545            2 :       if (TARGET_AVX512VL)
    5546              :         gen = gen_avx512vl_vpermt2varv2di3;
    5547              :       break;
    5548          347 :     case E_V4DImode:
    5549          347 :       if (TARGET_AVX512VL)
    5550              :         gen = gen_avx512vl_vpermt2varv4di3;
    5551              :       break;
    5552           10 :     case E_V8DImode:
    5553           10 :       if (TARGET_AVX512F)
    5554              :         gen = gen_avx512f_vpermt2varv8di3;
    5555              :       break;
    5556            2 :     case E_V2DFmode:
    5557            2 :       if (TARGET_AVX512VL)
    5558              :         {
    5559              :           gen = gen_avx512vl_vpermt2varv2df3;
    5560              :           maskmode = V2DImode;
    5561              :         }
    5562              :       break;
    5563         1868 :     case E_V4DFmode:
    5564         1868 :       if (TARGET_AVX512VL)
    5565              :         {
    5566              :           gen = gen_avx512vl_vpermt2varv4df3;
    5567              :           maskmode = V4DImode;
    5568              :         }
    5569              :       break;
    5570          198 :     case E_V8DFmode:
    5571          198 :       if (TARGET_AVX512F)
    5572              :         {
    5573              :           gen = gen_avx512f_vpermt2varv8df3;
    5574              :           maskmode = V8DImode;
    5575              :         }
    5576              :       break;
    5577              :     default:
    5578              :       break;
    5579              :     }
    5580              : 
    5581              :   if (gen == NULL)
    5582              :     return false;
    5583              : 
    5584          903 :   if (d && d->testing_p)
    5585              :     return true;
    5586              : 
    5587              :   /* ix86_expand_vec_perm_vpermt2 is called from both const and non-const
    5588              :      expander, so args are either in d, or in op0, op1 etc.  */
    5589          893 :   if (d)
    5590              :     {
    5591          893 :       rtx vec[64];
    5592          893 :       target = d->target;
    5593          893 :       op0 = d->op0;
    5594          893 :       op1 = d->op1;
    5595        15617 :       for (int i = 0; i < d->nelt; ++i)
    5596        14724 :         vec[i] = GEN_INT (d->perm[i]);
    5597          893 :       mask = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (d->nelt, vec));
    5598              :     }
    5599              : 
    5600          901 :   emit_insn (gen (target, force_reg (maskmode, mask), op0, op1));
    5601          901 :   return true;
    5602              : }
    5603              : 
    5604              : /* Expand a variable vector permutation.  */
    5605              : 
    5606              : void
    5607           18 : ix86_expand_vec_perm (rtx operands[])
    5608              : {
    5609           18 :   rtx target = operands[0];
    5610           18 :   rtx op0 = operands[1];
    5611           18 :   rtx op1 = operands[2];
    5612           18 :   rtx mask = operands[3];
    5613           18 :   rtx t1, t2, t3, t4, t5, t6, t7, t8, vt, vt2, vec[32];
    5614           18 :   machine_mode mode = GET_MODE (op0);
    5615           18 :   machine_mode maskmode = GET_MODE (mask);
    5616           18 :   int w, e, i;
    5617           18 :   bool one_operand_shuffle = rtx_equal_p (op0, op1);
    5618              : 
    5619              :   /* Number of elements in the vector.  */
    5620           18 :   w = GET_MODE_NUNITS (mode);
    5621           18 :   e = GET_MODE_UNIT_SIZE (mode);
    5622           18 :   gcc_assert (w <= 64);
    5623              : 
    5624              :   /* For HF mode vector, convert it to HI using subreg.  */
    5625           36 :   if (GET_MODE_INNER (mode) == HFmode)
    5626              :     {
    5627            6 :       machine_mode orig_mode = mode;
    5628            6 :       mode = mode_for_vector (HImode, w).require ();
    5629            6 :       target = lowpart_subreg (mode, target, orig_mode);
    5630            6 :       op0 = lowpart_subreg (mode, op0, orig_mode);
    5631            6 :       op1 = lowpart_subreg (mode, op1, orig_mode);
    5632              :     }
    5633              : 
    5634           18 :   if (TARGET_AVX512F && one_operand_shuffle)
    5635              :     {
    5636            5 :       rtx (*gen) (rtx, rtx, rtx) = NULL;
    5637            5 :       switch (mode)
    5638              :         {
    5639              :         case E_V16SImode:
    5640              :           gen = gen_avx512f_permvarv16si;
    5641              :           break;
    5642            0 :         case E_V16SFmode:
    5643            0 :           gen = gen_avx512f_permvarv16sf;
    5644            0 :           break;
    5645            0 :         case E_V8DImode:
    5646            0 :           gen = gen_avx512f_permvarv8di;
    5647            0 :           break;
    5648            0 :         case E_V8DFmode:
    5649            0 :           gen = gen_avx512f_permvarv8df;
    5650            0 :           break;
    5651              :         default:
    5652              :           break;
    5653              :         }
    5654            0 :       if (gen != NULL)
    5655              :         {
    5656            0 :           emit_insn (gen (target, op0, mask));
    5657           16 :           return;
    5658              :         }
    5659              :     }
    5660              : 
    5661           18 :   if (ix86_expand_vec_perm_vpermt2 (target, mask, op0, op1, NULL))
    5662              :     return;
    5663              : 
    5664           10 :   if (TARGET_AVX2)
    5665              :     {
    5666            5 :       if (mode == V4DImode || mode == V4DFmode || mode == V16HImode)
    5667              :         {
    5668              :           /* Unfortunately, the VPERMQ and VPERMPD instructions only support
    5669              :              an constant shuffle operand.  With a tiny bit of effort we can
    5670              :              use VPERMD instead.  A re-interpretation stall for V4DFmode is
    5671              :              unfortunate but there's no avoiding it.
    5672              :              Similarly for V16HImode we don't have instructions for variable
    5673              :              shuffling, while for V32QImode we can use after preparing suitable
    5674              :              masks vpshufb; vpshufb; vpermq; vpor.  */
    5675              : 
    5676              :           if (mode == V16HImode)
    5677              :             {
    5678              :               maskmode = mode = V32QImode;
    5679              :               w = 32;
    5680              :               e = 1;
    5681              :             }
    5682              :           else
    5683              :             {
    5684              :               maskmode = mode = V8SImode;
    5685              :               w = 8;
    5686              :               e = 4;
    5687              :             }
    5688            0 :           t1 = gen_reg_rtx (maskmode);
    5689              : 
    5690              :           /* Replicate the low bits of the V4DImode mask into V8SImode:
    5691              :                mask = { A B C D }
    5692              :                t1 = { A A B B C C D D }.  */
    5693            0 :           for (i = 0; i < w / 2; ++i)
    5694            0 :             vec[i*2 + 1] = vec[i*2] = GEN_INT (i * 2);
    5695            0 :           vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
    5696            0 :           vt = force_reg (maskmode, vt);
    5697            0 :           mask = gen_lowpart (maskmode, mask);
    5698            0 :           if (maskmode == V8SImode)
    5699            0 :             emit_insn (gen_avx2_permvarv8si (t1, mask, vt));
    5700              :           else
    5701            0 :             emit_insn (gen_avx2_pshufbv32qi3 (t1, mask, vt));
    5702              : 
    5703              :           /* Multiply the shuffle indices by two.  */
    5704            0 :           t1 = expand_simple_binop (maskmode, PLUS, t1, t1, t1, 1,
    5705              :                                     OPTAB_DIRECT);
    5706              : 
    5707              :           /* Add one to the odd shuffle indices:
    5708              :                 t1 = { A*2, A*2+1, B*2, B*2+1, ... }.  */
    5709            0 :           for (i = 0; i < w / 2; ++i)
    5710              :             {
    5711            0 :               vec[i * 2] = const0_rtx;
    5712            0 :               vec[i * 2 + 1] = const1_rtx;
    5713              :             }
    5714            0 :           vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
    5715            0 :           vt = validize_mem (force_const_mem (maskmode, vt));
    5716            0 :           t1 = expand_simple_binop (maskmode, PLUS, t1, vt, t1, 1,
    5717              :                                     OPTAB_DIRECT);
    5718              : 
    5719              :           /* Continue as if V8SImode (resp. V32QImode) was used initially.  */
    5720            0 :           operands[3] = mask = t1;
    5721            0 :           target = gen_reg_rtx (mode);
    5722            0 :           op0 = gen_lowpart (mode, op0);
    5723            0 :           op1 = gen_lowpart (mode, op1);
    5724              :         }
    5725              : 
    5726            5 :       switch (mode)
    5727              :         {
    5728            1 :         case E_V8SImode:
    5729              :           /* The VPERMD and VPERMPS instructions already properly ignore
    5730              :              the high bits of the shuffle elements.  No need for us to
    5731              :              perform an AND ourselves.  */
    5732            1 :           if (one_operand_shuffle)
    5733              :             {
    5734            0 :               emit_insn (gen_avx2_permvarv8si (target, op0, mask));
    5735            0 :               if (target != operands[0])
    5736            0 :                 emit_move_insn (operands[0],
    5737            0 :                                 gen_lowpart (GET_MODE (operands[0]), target));
    5738              :             }
    5739              :           else
    5740              :             {
    5741            1 :               t1 = gen_reg_rtx (V8SImode);
    5742            1 :               t2 = gen_reg_rtx (V8SImode);
    5743            1 :               emit_insn (gen_avx2_permvarv8si (t1, op0, mask));
    5744            1 :               emit_insn (gen_avx2_permvarv8si (t2, op1, mask));
    5745            1 :               goto merge_two;
    5746              :             }
    5747              :           return;
    5748              : 
    5749            0 :         case E_V8SFmode:
    5750            0 :           mask = gen_lowpart (V8SImode, mask);
    5751            0 :           if (one_operand_shuffle)
    5752            0 :             emit_insn (gen_avx2_permvarv8sf (target, op0, mask));
    5753              :           else
    5754              :             {
    5755            0 :               t1 = gen_reg_rtx (V8SFmode);
    5756            0 :               t2 = gen_reg_rtx (V8SFmode);
    5757            0 :               emit_insn (gen_avx2_permvarv8sf (t1, op0, mask));
    5758            0 :               emit_insn (gen_avx2_permvarv8sf (t2, op1, mask));
    5759            0 :               goto merge_two;
    5760              :             }
    5761            0 :           return;
    5762              : 
    5763            1 :         case E_V4SImode:
    5764            1 :           if (one_operand_shuffle)
    5765              :             break; /* Handled below for TARGET_AVX.  */
    5766              :           /* By combining the two 128-bit input vectors into one 256-bit
    5767              :              input vector, we can use VPERMD and VPERMPS for the full
    5768              :              two-operand shuffle.  */
    5769            0 :           t1 = gen_reg_rtx (V8SImode);
    5770            0 :           t2 = gen_reg_rtx (V8SImode);
    5771            0 :           emit_insn (gen_avx_vec_concatv8si (t1, op0, op1));
    5772            0 :           emit_insn (gen_avx_vec_concatv8si (t2, mask, mask));
    5773            0 :           emit_insn (gen_avx2_permvarv8si (t1, t1, t2));
    5774            0 :           emit_insn (gen_avx_vextractf128v8si (target, t1, const0_rtx));
    5775            0 :           return;
    5776              : 
    5777            1 :         case E_V4SFmode:
    5778            1 :           if (one_operand_shuffle)
    5779              :             break; /* Handled below for TARGET_AVX.  */
    5780            0 :           t1 = gen_reg_rtx (V8SFmode);
    5781            0 :           t2 = gen_reg_rtx (V8SImode);
    5782            0 :           mask = gen_lowpart (V4SImode, mask);
    5783            0 :           emit_insn (gen_avx_vec_concatv8sf (t1, op0, op1));
    5784            0 :           emit_insn (gen_avx_vec_concatv8si (t2, mask, mask));
    5785            0 :           emit_insn (gen_avx2_permvarv8sf (t1, t1, t2));
    5786            0 :           emit_insn (gen_avx_vextractf128v8sf (target, t1, const0_rtx));
    5787            0 :           return;
    5788              : 
    5789            0 :         case E_V32QImode:
    5790            0 :           t1 = gen_reg_rtx (V32QImode);
    5791            0 :           t2 = gen_reg_rtx (V32QImode);
    5792            0 :           t3 = gen_reg_rtx (V32QImode);
    5793            0 :           vt2 = GEN_INT (-128);
    5794            0 :           vt = gen_const_vec_duplicate (V32QImode, vt2);
    5795            0 :           vt = force_reg (V32QImode, vt);
    5796            0 :           for (i = 0; i < 32; i++)
    5797            0 :             vec[i] = i < 16 ? vt2 : const0_rtx;
    5798            0 :           vt2 = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, vec));
    5799            0 :           vt2 = force_reg (V32QImode, vt2);
    5800              :           /* From mask create two adjusted masks, which contain the same
    5801              :              bits as mask in the low 7 bits of each vector element.
    5802              :              The first mask will have the most significant bit clear
    5803              :              if it requests element from the same 128-bit lane
    5804              :              and MSB set if it requests element from the other 128-bit lane.
    5805              :              The second mask will have the opposite values of the MSB,
    5806              :              and additionally will have its 128-bit lanes swapped.
    5807              :              E.g. { 07 12 1e 09 ... | 17 19 05 1f ... } mask vector will have
    5808              :              t1   { 07 92 9e 09 ... | 17 19 85 1f ... } and
    5809              :              t3   { 97 99 05 9f ... | 87 12 1e 89 ... } where each ...
    5810              :              stands for other 12 bytes.  */
    5811              :           /* The bit whether element is from the same lane or the other
    5812              :              lane is bit 4, so shift it up by 3 to the MSB position.  */
    5813            0 :           t5 = gen_reg_rtx (V4DImode);
    5814            0 :           emit_insn (gen_ashlv4di3 (t5, gen_lowpart (V4DImode, mask),
    5815              :                                     GEN_INT (3)));
    5816              :           /* Clear MSB bits from the mask just in case it had them set.  */
    5817            0 :           emit_insn (gen_avx2_andnotv32qi3 (t2, vt, mask));
    5818              :           /* After this t1 will have MSB set for elements from other lane.  */
    5819            0 :           emit_insn (gen_xorv32qi3 (t1, gen_lowpart (V32QImode, t5), vt2));
    5820              :           /* Clear bits other than MSB.  */
    5821            0 :           emit_insn (gen_andv32qi3 (t1, t1, vt));
    5822              :           /* Or in the lower bits from mask into t3.  */
    5823            0 :           emit_insn (gen_iorv32qi3 (t3, t1, t2));
    5824              :           /* And invert MSB bits in t1, so MSB is set for elements from the same
    5825              :              lane.  */
    5826            0 :           emit_insn (gen_xorv32qi3 (t1, t1, vt));
    5827              :           /* Swap 128-bit lanes in t3.  */
    5828            0 :           t6 = gen_reg_rtx (V4DImode);
    5829            0 :           emit_insn (gen_avx2_permv4di_1 (t6, gen_lowpart (V4DImode, t3),
    5830              :                                           const2_rtx, GEN_INT (3),
    5831              :                                           const0_rtx, const1_rtx));
    5832              :           /* And or in the lower bits from mask into t1.  */
    5833            0 :           emit_insn (gen_iorv32qi3 (t1, t1, t2));
    5834            0 :           if (one_operand_shuffle)
    5835              :             {
    5836              :               /* Each of these shuffles will put 0s in places where
    5837              :                  element from the other 128-bit lane is needed, otherwise
    5838              :                  will shuffle in the requested value.  */
    5839            0 :               emit_insn (gen_avx2_pshufbv32qi3 (t3, op0,
    5840            0 :                                                 gen_lowpart (V32QImode, t6)));
    5841            0 :               emit_insn (gen_avx2_pshufbv32qi3 (t1, op0, t1));
    5842              :               /* For t3 the 128-bit lanes are swapped again.  */
    5843            0 :               t7 = gen_reg_rtx (V4DImode);
    5844            0 :               emit_insn (gen_avx2_permv4di_1 (t7, gen_lowpart (V4DImode, t3),
    5845              :                                               const2_rtx, GEN_INT (3),
    5846              :                                               const0_rtx, const1_rtx));
    5847              :               /* And oring both together leads to the result.  */
    5848            0 :               emit_insn (gen_iorv32qi3 (target, t1,
    5849            0 :                                         gen_lowpart (V32QImode, t7)));
    5850            0 :               if (target != operands[0])
    5851            0 :                 emit_move_insn (operands[0],
    5852            0 :                                 gen_lowpart (GET_MODE (operands[0]), target));
    5853              :               return;
    5854              :             }
    5855              : 
    5856            0 :           t4 = gen_reg_rtx (V32QImode);
    5857              :           /* Similarly to the above one_operand_shuffle code,
    5858              :              just for repeated twice for each operand.  merge_two:
    5859              :              code will merge the two results together.  */
    5860            0 :           emit_insn (gen_avx2_pshufbv32qi3 (t4, op0,
    5861            0 :                                             gen_lowpart (V32QImode, t6)));
    5862            0 :           emit_insn (gen_avx2_pshufbv32qi3 (t3, op1,
    5863            0 :                                             gen_lowpart (V32QImode, t6)));
    5864            0 :           emit_insn (gen_avx2_pshufbv32qi3 (t2, op0, t1));
    5865            0 :           emit_insn (gen_avx2_pshufbv32qi3 (t1, op1, t1));
    5866            0 :           t7 = gen_reg_rtx (V4DImode);
    5867            0 :           emit_insn (gen_avx2_permv4di_1 (t7, gen_lowpart (V4DImode, t4),
    5868              :                                           const2_rtx, GEN_INT (3),
    5869              :                                           const0_rtx, const1_rtx));
    5870            0 :           t8 = gen_reg_rtx (V4DImode);
    5871            0 :           emit_insn (gen_avx2_permv4di_1 (t8, gen_lowpart (V4DImode, t3),
    5872              :                                           const2_rtx, GEN_INT (3),
    5873              :                                           const0_rtx, const1_rtx));
    5874            0 :           emit_insn (gen_iorv32qi3 (t4, t2, gen_lowpart (V32QImode, t7)));
    5875            0 :           emit_insn (gen_iorv32qi3 (t3, t1, gen_lowpart (V32QImode, t8)));
    5876            0 :           t1 = t4;
    5877            0 :           t2 = t3;
    5878            0 :           goto merge_two;
    5879              : 
    5880            2 :         default:
    5881            4 :           gcc_assert (GET_MODE_SIZE (mode) <= 16);
    5882              :           break;
    5883              :         }
    5884              :     }
    5885              : 
    5886            9 :   if (TARGET_AVX && one_operand_shuffle)
    5887            8 :     switch (mode)
    5888              :       {
    5889            2 :       case V4SImode:
    5890            2 :         op0 = gen_lowpart (V4SFmode, op0);
    5891            2 :         t1 = gen_reg_rtx (V4SFmode);
    5892            2 :         emit_insn (gen_avx_vpermilvarv4sf3 (t1, op0, mask));
    5893            2 :         emit_move_insn (target, gen_lowpart (mode, t1));
    5894            2 :         return;
    5895            2 :       case V4SFmode:
    5896            2 :         emit_insn (gen_avx_vpermilvarv4sf3 (target, op0, mask));
    5897            2 :         return;
    5898            2 :       case V2DImode:
    5899            2 :         op0 = gen_lowpart (V2DFmode, op0);
    5900            2 :         t1 = gen_reg_rtx (V2DImode);
    5901            2 :         t2 = gen_reg_rtx (V2DFmode);
    5902            2 :         emit_insn (gen_addv2di3 (t1, mask, mask));
    5903            2 :         emit_insn (gen_avx_vpermilvarv2df3 (t2, op0, t1));
    5904            2 :         emit_move_insn (target, gen_lowpart (mode, t2));
    5905            2 :         return;
    5906            2 :       case V2DFmode:
    5907            2 :         t1 = gen_reg_rtx (V2DImode);
    5908            2 :         emit_insn (gen_addv2di3 (t1, mask, mask));
    5909            2 :         emit_insn (gen_avx_vpermilvarv2df3 (target, op0, t1));
    5910            2 :         return;
    5911              :       default:
    5912              :         break;
    5913              :       }
    5914              : 
    5915            1 :   if (TARGET_XOP)
    5916              :     {
    5917              :       /* The XOP VPPERM insn supports three inputs.  By ignoring the
    5918              :          one_operand_shuffle special case, we avoid creating another
    5919              :          set of constant vectors in memory.  */
    5920            0 :       one_operand_shuffle = false;
    5921              : 
    5922              :       /* mask = mask & {2*w-1, ...} */
    5923            0 :       vt = GEN_INT (2*w - 1);
    5924              :     }
    5925              :   else
    5926              :     {
    5927              :       /* mask = mask & {w-1, ...} */
    5928            1 :       vt = GEN_INT (w - 1);
    5929              :     }
    5930              : 
    5931            1 :   vt = gen_const_vec_duplicate (maskmode, vt);
    5932            1 :   mask = expand_simple_binop (maskmode, AND, mask, vt,
    5933              :                               NULL_RTX, 0, OPTAB_DIRECT);
    5934              : 
    5935              :   /* For non-QImode operations, convert the word permutation control
    5936              :      into a byte permutation control.  */
    5937            1 :   if (mode != V16QImode)
    5938              :     {
    5939            1 :       mask = expand_simple_binop (maskmode, ASHIFT, mask,
    5940            2 :                                   GEN_INT (exact_log2 (e)),
    5941              :                                   NULL_RTX, 0, OPTAB_DIRECT);
    5942              : 
    5943              :       /* Convert mask to vector of chars.  */
    5944            1 :       mask = force_reg (V16QImode, gen_lowpart (V16QImode, mask));
    5945              : 
    5946              :       /* Replicate each of the input bytes into byte positions:
    5947              :          (v2di) --> {0,0,0,0,0,0,0,0, 8,8,8,8,8,8,8,8}
    5948              :          (v4si) --> {0,0,0,0, 4,4,4,4, 8,8,8,8, 12,12,12,12}
    5949              :          (v8hi) --> {0,0, 2,2, 4,4, 6,6, ...}.  */
    5950           18 :       for (i = 0; i < 16; ++i)
    5951           16 :         vec[i] = GEN_INT (i/e * e);
    5952            1 :       vt = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, vec));
    5953            1 :       vt = validize_mem (force_const_mem (V16QImode, vt));
    5954            1 :       if (TARGET_XOP)
    5955            0 :         emit_insn (gen_xop_pperm (mask, mask, mask, vt));
    5956              :       else
    5957            1 :         emit_insn (gen_ssse3_pshufbv16qi3 (mask, mask, vt));
    5958              : 
    5959              :       /* Convert it into the byte positions by doing
    5960              :          mask = mask + {0,1,..,16/w, 0,1,..,16/w, ...}  */
    5961           17 :       for (i = 0; i < 16; ++i)
    5962           16 :         vec[i] = GEN_INT (i % e);
    5963            1 :       vt = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, vec));
    5964            1 :       vt = validize_mem (force_const_mem (V16QImode, vt));
    5965            1 :       emit_insn (gen_addv16qi3 (mask, mask, vt));
    5966              :     }
    5967              : 
    5968              :   /* The actual shuffle operations all operate on V16QImode.  */
    5969            1 :   op0 = gen_lowpart (V16QImode, op0);
    5970            1 :   op1 = gen_lowpart (V16QImode, op1);
    5971              : 
    5972            1 :   if (TARGET_XOP)
    5973              :     {
    5974            0 :       if (GET_MODE (target) != V16QImode)
    5975            0 :         target = gen_reg_rtx (V16QImode);
    5976            0 :       emit_insn (gen_xop_pperm (target, op0, op1, mask));
    5977            0 :       if (target != operands[0])
    5978            0 :         emit_move_insn (operands[0],
    5979            0 :                         gen_lowpart (GET_MODE (operands[0]), target));
    5980              :     }
    5981            1 :   else if (one_operand_shuffle)
    5982              :     {
    5983            1 :       if (GET_MODE (target) != V16QImode)
    5984            1 :         target = gen_reg_rtx (V16QImode);
    5985            1 :       emit_insn (gen_ssse3_pshufbv16qi3 (target, op0, mask));
    5986            1 :       if (target != operands[0])
    5987            1 :         emit_move_insn (operands[0],
    5988            1 :                         gen_lowpart (GET_MODE (operands[0]), target));
    5989              :     }
    5990              :   else
    5991              :     {
    5992            0 :       rtx xops[6];
    5993            0 :       bool ok;
    5994              : 
    5995              :       /* Shuffle the two input vectors independently.  */
    5996            0 :       t1 = gen_reg_rtx (V16QImode);
    5997            0 :       t2 = gen_reg_rtx (V16QImode);
    5998            0 :       emit_insn (gen_ssse3_pshufbv16qi3 (t1, op0, mask));
    5999            0 :       emit_insn (gen_ssse3_pshufbv16qi3 (t2, op1, mask));
    6000              : 
    6001            1 :  merge_two:
    6002              :       /* Then merge them together.  The key is whether any given control
    6003              :          element contained a bit set that indicates the second word.  */
    6004            1 :       mask = operands[3];
    6005            1 :       vt = GEN_INT (w);
    6006            1 :       if (maskmode == V2DImode && !TARGET_SSE4_1)
    6007              :         {
    6008              :           /* Without SSE4.1, we don't have V2DImode EQ.  Perform one
    6009              :              more shuffle to convert the V2DI input mask into a V4SI
    6010              :              input mask.  At which point the masking that expand_int_vcond
    6011              :              will work as desired.  */
    6012            0 :           rtx t3 = gen_reg_rtx (V4SImode);
    6013            0 :           emit_insn (gen_sse2_pshufd_1 (t3, gen_lowpart (V4SImode, mask),
    6014              :                                         const0_rtx, const0_rtx,
    6015              :                                         const2_rtx, const2_rtx));
    6016            0 :           mask = t3;
    6017            0 :           maskmode = V4SImode;
    6018            0 :           e = w = 4;
    6019              :         }
    6020              : 
    6021            1 :       vt = gen_const_vec_duplicate (maskmode, vt);
    6022            1 :       vt = force_reg (maskmode, vt);
    6023            1 :       mask = expand_simple_binop (maskmode, AND, mask, vt,
    6024              :                                   NULL_RTX, 0, OPTAB_DIRECT);
    6025              : 
    6026            1 :       if (GET_MODE (target) != mode)
    6027            0 :         target = gen_reg_rtx (mode);
    6028            1 :       xops[0] = target;
    6029            1 :       xops[1] = gen_lowpart (mode, t2);
    6030            1 :       xops[2] = gen_lowpart (mode, t1);
    6031            1 :       xops[3] = gen_rtx_EQ (maskmode, mask, vt);
    6032            1 :       xops[4] = mask;
    6033            1 :       xops[5] = vt;
    6034            1 :       ok = ix86_expand_int_vcond (xops);
    6035            1 :       gcc_assert (ok);
    6036            1 :       if (target != operands[0])
    6037            0 :         emit_move_insn (operands[0],
    6038            0 :                         gen_lowpart (GET_MODE (operands[0]), target));
    6039              :     }
    6040              : }
    6041              : 
    6042              : /* Extend SRC into next wider integer vector type.  UNSIGNED_P is
    6043              :    true if we should do zero extension, else sign extension.  */
    6044              : 
    6045              : void
    6046          753 : ix86_expand_sse_extend (rtx dest, rtx src, bool unsigned_p)
    6047              : {
    6048          753 :   machine_mode imode = GET_MODE (src);
    6049          753 :   rtx ops[3];
    6050              : 
    6051          753 :   switch (imode)
    6052              :     {
    6053          753 :     case E_V8QImode:
    6054          753 :     case E_V4QImode:
    6055          753 :     case E_V2QImode:
    6056          753 :     case E_V4HImode:
    6057          753 :     case E_V2HImode:
    6058          753 :     case E_V2SImode:
    6059          753 :       break;
    6060            0 :     default:
    6061            0 :       gcc_unreachable ();
    6062              :     }
    6063              : 
    6064          753 :   ops[0] = dest;
    6065              : 
    6066          753 :   ops[1] = force_reg (imode, src);
    6067              : 
    6068          753 :   if (unsigned_p)
    6069          236 :     ops[2] = force_reg (imode, CONST0_RTX (imode));
    6070              :   else
    6071          517 :     ops[2] = ix86_expand_sse_cmp (gen_reg_rtx (imode), GT, CONST0_RTX (imode),
    6072              :                                   ops[1], pc_rtx, pc_rtx);
    6073              : 
    6074          753 :   ix86_split_mmx_punpck (ops, false);
    6075          753 : }
    6076              : 
    6077              : /* Unpack SRC into the next wider integer vector type.  UNSIGNED_P is
    6078              :    true if we should do zero extension, else sign extension.  HIGH_P is
    6079              :    true if we want the N/2 high elements, else the low elements.  */
    6080              : 
    6081              : void
    6082        18992 : ix86_expand_sse_unpack (rtx dest, rtx src, bool unsigned_p, bool high_p)
    6083              : {
    6084        18992 :   machine_mode imode = GET_MODE (src);
    6085        18992 :   rtx tmp;
    6086              : 
    6087        18992 :   if (TARGET_SSE4_1)
    6088              :     {
    6089         6582 :       rtx (*unpack)(rtx, rtx);
    6090         6582 :       rtx (*extract)(rtx, rtx) = NULL;
    6091         6582 :       machine_mode halfmode = BLKmode;
    6092              : 
    6093         6582 :       switch (imode)
    6094              :         {
    6095          198 :         case E_V64QImode:
    6096          198 :           if (unsigned_p)
    6097              :             unpack = gen_avx512bw_zero_extendv32qiv32hi2;
    6098              :           else
    6099           64 :             unpack = gen_avx512bw_sign_extendv32qiv32hi2;
    6100          198 :           halfmode = V32QImode;
    6101          198 :           extract
    6102          198 :             = high_p ? gen_vec_extract_hi_v64qi : gen_vec_extract_lo_v64qi;
    6103              :           break;
    6104          711 :         case E_V32QImode:
    6105          711 :           if (unsigned_p)
    6106              :             unpack = gen_avx2_zero_extendv16qiv16hi2;
    6107              :           else
    6108          142 :             unpack = gen_avx2_sign_extendv16qiv16hi2;
    6109          711 :           halfmode = V16QImode;
    6110          711 :           extract
    6111          711 :             = high_p ? gen_vec_extract_hi_v32qi : gen_vec_extract_lo_v32qi;
    6112              :           break;
    6113          104 :         case E_V32HImode:
    6114          104 :           if (unsigned_p)
    6115              :             unpack = gen_avx512f_zero_extendv16hiv16si2;
    6116              :           else
    6117           64 :             unpack = gen_avx512f_sign_extendv16hiv16si2;
    6118          104 :           halfmode = V16HImode;
    6119          104 :           extract
    6120          104 :             = high_p ? gen_vec_extract_hi_v32hi : gen_vec_extract_lo_v32hi;
    6121              :           break;
    6122          417 :         case E_V16HImode:
    6123          417 :           if (unsigned_p)
    6124              :             unpack = gen_avx2_zero_extendv8hiv8si2;
    6125              :           else
    6126          302 :             unpack = gen_avx2_sign_extendv8hiv8si2;
    6127          417 :           halfmode = V8HImode;
    6128          417 :           extract
    6129          417 :             = high_p ? gen_vec_extract_hi_v16hi : gen_vec_extract_lo_v16hi;
    6130              :           break;
    6131          106 :         case E_V16SImode:
    6132          106 :           if (unsigned_p)
    6133              :             unpack = gen_avx512f_zero_extendv8siv8di2;
    6134              :           else
    6135           88 :             unpack = gen_avx512f_sign_extendv8siv8di2;
    6136          106 :           halfmode = V8SImode;
    6137          106 :           extract
    6138          106 :             = high_p ? gen_vec_extract_hi_v16si : gen_vec_extract_lo_v16si;
    6139              :           break;
    6140          380 :         case E_V8SImode:
    6141          380 :           if (unsigned_p)
    6142              :             unpack = gen_avx2_zero_extendv4siv4di2;
    6143              :           else
    6144          318 :             unpack = gen_avx2_sign_extendv4siv4di2;
    6145          380 :           halfmode = V4SImode;
    6146          380 :           extract
    6147          380 :             = high_p ? gen_vec_extract_hi_v8si : gen_vec_extract_lo_v8si;
    6148              :           break;
    6149         2661 :         case E_V16QImode:
    6150         2661 :           if (unsigned_p)
    6151              :             unpack = gen_sse4_1_zero_extendv8qiv8hi2;
    6152              :           else
    6153          308 :             unpack = gen_sse4_1_sign_extendv8qiv8hi2;
    6154              :           break;
    6155         1033 :         case E_V8HImode:
    6156         1033 :           if (unsigned_p)
    6157              :             unpack = gen_sse4_1_zero_extendv4hiv4si2;
    6158              :           else
    6159          804 :             unpack = gen_sse4_1_sign_extendv4hiv4si2;
    6160              :           break;
    6161          544 :         case E_V4SImode:
    6162          544 :           if (unsigned_p)
    6163              :             unpack = gen_sse4_1_zero_extendv2siv2di2;
    6164              :           else
    6165          484 :             unpack = gen_sse4_1_sign_extendv2siv2di2;
    6166              :           break;
    6167          127 :         case E_V8QImode:
    6168          127 :           if (unsigned_p)
    6169              :             unpack = gen_sse4_1_zero_extendv4qiv4hi2;
    6170              :           else
    6171           86 :             unpack = gen_sse4_1_sign_extendv4qiv4hi2;
    6172              :           break;
    6173          295 :         case E_V4HImode:
    6174          295 :           if (unsigned_p)
    6175              :             unpack = gen_sse4_1_zero_extendv2hiv2si2;
    6176              :           else
    6177          236 :             unpack = gen_sse4_1_sign_extendv2hiv2si2;
    6178              :           break;
    6179            6 :         case E_V4QImode:
    6180            6 :           if (unsigned_p)
    6181              :             unpack = gen_sse4_1_zero_extendv2qiv2hi2;
    6182              :           else
    6183            0 :             unpack = gen_sse4_1_sign_extendv2qiv2hi2;
    6184              :           break;
    6185            0 :         default:
    6186            0 :           gcc_unreachable ();
    6187              :         }
    6188              : 
    6189        13164 :       if (GET_MODE_SIZE (imode) >= 32)
    6190              :         {
    6191         1916 :           tmp = gen_reg_rtx (halfmode);
    6192         1916 :           emit_insn (extract (tmp, src));
    6193              :         }
    6194         4666 :       else if (high_p)
    6195              :         {
    6196         2476 :           switch (GET_MODE_SIZE (imode))
    6197              :             {
    6198         1021 :             case 16:
    6199              :               /* Shift higher 8 bytes to lower 8 bytes.  */
    6200         1021 :               tmp = gen_reg_rtx (V1TImode);
    6201         1021 :               emit_insn (gen_sse2_lshrv1ti3 (tmp, gen_lowpart (V1TImode, src),
    6202              :                                              GEN_INT (64)));
    6203         1021 :               break;
    6204          214 :             case 8:
    6205              :               /* Shift higher 4 bytes to lower 4 bytes.  */
    6206          214 :               tmp = gen_reg_rtx (V1DImode);
    6207          214 :               emit_insn (gen_mmx_lshrv1di3 (tmp, gen_lowpart (V1DImode, src),
    6208              :                                             GEN_INT (32)));
    6209          214 :               break;
    6210            3 :             case 4:
    6211              :               /* Shift higher 2 bytes to lower 2 bytes.  */
    6212            3 :               tmp = gen_reg_rtx (V1SImode);
    6213            3 :               emit_insn (gen_mmx_lshrv1si3 (tmp, gen_lowpart (V1SImode, src),
    6214              :                                             GEN_INT (16)));
    6215            3 :               break;
    6216            0 :             default:
    6217            0 :               gcc_unreachable ();
    6218              :             }
    6219              : 
    6220         1238 :           tmp = gen_lowpart (imode, tmp);
    6221              :         }
    6222              :       else
    6223              :         tmp = src;
    6224              : 
    6225         6582 :       emit_insn (unpack (dest, tmp));
    6226              :     }
    6227              :   else
    6228              :     {
    6229        12410 :       rtx (*unpack)(rtx, rtx, rtx);
    6230              : 
    6231        12410 :       switch (imode)
    6232              :         {
    6233         3392 :         case E_V16QImode:
    6234         3392 :           if (high_p)
    6235              :             unpack = gen_vec_interleave_highv16qi;
    6236              :           else
    6237         1701 :             unpack = gen_vec_interleave_lowv16qi;
    6238              :           break;
    6239         5126 :         case E_V8HImode:
    6240         5126 :           if (high_p)
    6241              :             unpack = gen_vec_interleave_highv8hi;
    6242              :           else
    6243         2565 :             unpack = gen_vec_interleave_lowv8hi;
    6244              :           break;
    6245         2106 :         case E_V4SImode:
    6246         2106 :           if (high_p)
    6247              :             unpack = gen_vec_interleave_highv4si;
    6248              :           else
    6249         1054 :             unpack = gen_vec_interleave_lowv4si;
    6250              :           break;
    6251          606 :         case E_V8QImode:
    6252          606 :           if (high_p)
    6253              :             unpack = gen_mmx_punpckhbw;
    6254              :           else
    6255          305 :             unpack = gen_mmx_punpcklbw;
    6256              :           break;
    6257         1162 :         case E_V4HImode:
    6258         1162 :           if (high_p)
    6259              :             unpack = gen_mmx_punpckhwd;
    6260              :           else
    6261          583 :             unpack = gen_mmx_punpcklwd;
    6262              :           break;
    6263           18 :         case E_V4QImode:
    6264           18 :           if (high_p)
    6265              :             unpack = gen_mmx_punpckhbw_low;
    6266              :           else
    6267            9 :             unpack = gen_mmx_punpcklbw_low;
    6268              :           break;
    6269            0 :         default:
    6270            0 :           gcc_unreachable ();
    6271              :         }
    6272              : 
    6273        12410 :       if (unsigned_p)
    6274         4962 :         tmp = force_reg (imode, CONST0_RTX (imode));
    6275              :       else
    6276         7448 :         tmp = ix86_expand_sse_cmp (gen_reg_rtx (imode), GT, CONST0_RTX (imode),
    6277              :                                    src, pc_rtx, pc_rtx);
    6278              : 
    6279        12410 :       rtx tmp2 = gen_reg_rtx (imode);
    6280        12410 :       emit_insn (unpack (tmp2, src, tmp));
    6281        12410 :       emit_move_insn (dest, gen_lowpart (GET_MODE (dest), tmp2));
    6282              :     }
    6283        18992 : }
    6284              : 
    6285              : /* Return true if mem is pool constant which contains a const_vector
    6286              :    perm index, assign the index to PERM.  */
    6287              : bool
    6288           35 : ix86_extract_perm_from_pool_constant (int* perm, rtx mem)
    6289              : {
    6290           35 :   machine_mode mode = GET_MODE (mem);
    6291           35 :   int nelt = GET_MODE_NUNITS (mode);
    6292              : 
    6293           35 :   if (!INTEGRAL_MODE_P (mode))
    6294              :     return false;
    6295              : 
    6296              :     /* Needs to be constant pool.  */
    6297           35 :   if (!(MEM_P (mem))
    6298           35 :       || !SYMBOL_REF_P (XEXP (mem, 0))
    6299           70 :       || !CONSTANT_POOL_ADDRESS_P (XEXP (mem, 0)))
    6300              :    return false;
    6301              : 
    6302           35 :   rtx constant = get_pool_constant (XEXP (mem, 0));
    6303              : 
    6304           35 :   if (!CONST_VECTOR_P (constant))
    6305              :     return false;
    6306              : 
    6307              :   /* There could be some rtx like
    6308              :      (mem/u/c:V16QI (symbol_ref/u:DI ("*.LC1")))
    6309              :      but with "*.LC1" refer to V2DI constant vector.  */
    6310           35 :   if (GET_MODE (constant) != mode)
    6311              :     {
    6312            0 :       constant = simplify_subreg (mode, constant, GET_MODE (constant), 0);
    6313              : 
    6314            0 :       if (constant == nullptr || !CONST_VECTOR_P (constant))
    6315              :         return false;
    6316              :     }
    6317              : 
    6318          771 :   for (int i = 0; i != nelt; i++)
    6319          736 :     perm[i] = UINTVAL (XVECEXP (constant, 0, i));
    6320              : 
    6321              :   return true;
    6322              : }
    6323              : 
    6324              : /* Split operands 0 and 1 into half-mode parts.  Similar to split_double_mode,
    6325              :    but works for floating pointer parameters and nonoffsetable memories.
    6326              :    For pushes, it returns just stack offsets; the values will be saved
    6327              :    in the right order.  Maximally three parts are generated.  */
    6328              : 
    6329              : static int
    6330      4221956 : ix86_split_to_parts (rtx operand, rtx *parts, machine_mode mode)
    6331              : {
    6332      4221956 :   int size;
    6333              : 
    6334      4221956 :   if (!TARGET_64BIT)
    6335      1610096 :     size = mode==XFmode ? 3 : GET_MODE_SIZE (mode) / 4;
    6336              :   else
    6337      6832536 :     size = (GET_MODE_SIZE (mode) + 4) / 8;
    6338              : 
    6339      4221956 :   gcc_assert (!REG_P (operand) || !MMX_REGNO_P (REGNO (operand)));
    6340      4221956 :   gcc_assert (size >= 2 && size <= 4);
    6341              : 
    6342              :   /* Optimize constant pool reference to immediates.  This is used by fp
    6343              :      moves, that force all constants to memory to allow combining.  */
    6344      4221956 :   if (MEM_P (operand) && MEM_READONLY_P (operand))
    6345        38093 :     operand = avoid_constant_pool_reference (operand);
    6346              : 
    6347      4221956 :   if (MEM_P (operand) && !offsettable_memref_p (operand))
    6348              :     {
    6349              :       /* The only non-offsetable memories we handle are pushes.  */
    6350       184169 :       int ok = push_operand (operand, VOIDmode);
    6351              : 
    6352       184169 :       gcc_assert (ok);
    6353              : 
    6354       184169 :       operand = copy_rtx (operand);
    6355       184169 :       PUT_MODE (operand, word_mode);
    6356       184169 :       parts[0] = parts[1] = parts[2] = parts[3] = operand;
    6357       184169 :       return size;
    6358              :     }
    6359              : 
    6360      4037787 :   if (CONST_VECTOR_P (operand))
    6361              :     {
    6362        41742 :       scalar_int_mode imode = int_mode_for_mode (mode).require ();
    6363              :       /* Caution: if we looked through a constant pool memory above,
    6364              :          the operand may actually have a different mode now.  That's
    6365              :          ok, since we want to pun this all the way back to an integer.  */
    6366        41742 :       operand = simplify_subreg (imode, operand, GET_MODE (operand), 0);
    6367        41742 :       gcc_assert (operand != NULL);
    6368        41742 :       mode = imode;
    6369              :     }
    6370              : 
    6371      4037787 :   if (!TARGET_64BIT)
    6372              :     {
    6373       647651 :       if (mode == DImode)
    6374       530194 :         split_double_mode (mode, &operand, 1, &parts[0], &parts[1]);
    6375              :       else
    6376              :         {
    6377       117457 :           int i;
    6378              : 
    6379       117457 :           if (REG_P (operand))
    6380              :             {
    6381        61028 :               gcc_assert (reload_completed);
    6382       183084 :               for (i = 0; i < size; i++)
    6383       122056 :                 parts[i] = gen_rtx_REG (SImode, REGNO (operand) + i);
    6384              :             }
    6385        56429 :           else if (offsettable_memref_p (operand))
    6386              :             {
    6387        55085 :               operand = adjust_address (operand, SImode, 0);
    6388        55085 :               parts[0] = operand;
    6389       110681 :               for (i = 1; i < size; i++)
    6390        55596 :                 parts[i] = adjust_address (operand, SImode, 4 * i);
    6391              :             }
    6392         1344 :           else if (CONST_DOUBLE_P (operand))
    6393              :             {
    6394         1344 :               const REAL_VALUE_TYPE *r;
    6395         1344 :               long l[4];
    6396              : 
    6397         1344 :               r = CONST_DOUBLE_REAL_VALUE (operand);
    6398         1344 :               switch (mode)
    6399              :                 {
    6400            0 :                 case E_TFmode:
    6401            0 :                   real_to_target (l, r, mode);
    6402            0 :                   parts[3] = gen_int_mode (l[3], SImode);
    6403            0 :                   parts[2] = gen_int_mode (l[2], SImode);
    6404            0 :                   break;
    6405          198 :                 case E_XFmode:
    6406              :                   /* We can't use REAL_VALUE_TO_TARGET_LONG_DOUBLE since
    6407              :                      long double may not be 80-bit.  */
    6408          198 :                   real_to_target (l, r, mode);
    6409          198 :                   parts[2] = gen_int_mode (l[2], SImode);
    6410          198 :                   break;
    6411         1146 :                 case E_DFmode:
    6412         1146 :                   REAL_VALUE_TO_TARGET_DOUBLE (*r, l);
    6413         1146 :                   break;
    6414            0 :                 default:
    6415            0 :                   gcc_unreachable ();
    6416              :                 }
    6417         1344 :               parts[1] = gen_int_mode (l[1], SImode);
    6418         1344 :               parts[0] = gen_int_mode (l[0], SImode);
    6419              :             }
    6420              :           else
    6421            0 :             gcc_unreachable ();
    6422              :         }
    6423              :     }
    6424              :   else
    6425              :     {
    6426      3390136 :       if (mode == TImode)
    6427      3369994 :         split_double_mode (mode, &operand, 1, &parts[0], &parts[1]);
    6428      3390136 :       if (mode == XFmode || mode == TFmode)
    6429              :         {
    6430        20142 :           machine_mode upper_mode = mode==XFmode ? SImode : DImode;
    6431        20142 :           if (REG_P (operand))
    6432              :             {
    6433         1472 :               gcc_assert (reload_completed);
    6434         1472 :               parts[0] = gen_rtx_REG (DImode, REGNO (operand) + 0);
    6435         1472 :               parts[1] = gen_rtx_REG (upper_mode, REGNO (operand) + 1);
    6436              :             }
    6437        18670 :           else if (offsettable_memref_p (operand))
    6438              :             {
    6439        14849 :               operand = adjust_address (operand, DImode, 0);
    6440        14849 :               parts[0] = operand;
    6441        14849 :               parts[1] = adjust_address (operand, upper_mode, 8);
    6442              :             }
    6443         3821 :           else if (CONST_DOUBLE_P (operand))
    6444              :             {
    6445         3821 :               long l[4];
    6446              : 
    6447         3821 :               real_to_target (l, CONST_DOUBLE_REAL_VALUE (operand), mode);
    6448              : 
    6449              :               /* real_to_target puts 32-bit pieces in each long.  */
    6450         7642 :               parts[0] = gen_int_mode ((l[0] & HOST_WIDE_INT_C (0xffffffff))
    6451         3821 :                                        | ((l[1] & HOST_WIDE_INT_C (0xffffffff))
    6452         3821 :                                           << 32), DImode);
    6453              : 
    6454         3821 :               if (upper_mode == SImode)
    6455         2793 :                 parts[1] = gen_int_mode (l[2], SImode);
    6456              :               else
    6457         1028 :                 parts[1]
    6458         1028 :                   = gen_int_mode ((l[2] & HOST_WIDE_INT_C (0xffffffff))
    6459         1028 :                                   | ((l[3] & HOST_WIDE_INT_C (0xffffffff))
    6460         1028 :                                      << 32), DImode);
    6461              :             }
    6462              :           else
    6463            0 :             gcc_unreachable ();
    6464              :         }
    6465              :     }
    6466              : 
    6467              :   return size;
    6468              : }
    6469              : 
    6470              : /* Emit insns to perform a move or push of DI, DF, XF, and TF values.
    6471              :    Return false when normal moves are needed; true when all required
    6472              :    insns have been emitted.  Operands 2-4 contain the input values
    6473              :    int the correct order; operands 5-7 contain the output values.  */
    6474              : 
    6475              : void
    6476      2123940 : ix86_split_long_move (rtx operands[])
    6477              : {
    6478      2123940 :   rtx part[2][4];
    6479      2123940 :   int nparts, i, j;
    6480      2123940 :   int push = 0;
    6481      2123940 :   int collisions = 0;
    6482      2123940 :   machine_mode mode = GET_MODE (operands[0]);
    6483      2123940 :   bool collisionparts[4];
    6484              : 
    6485              :   /* The DFmode expanders may ask us to move double.
    6486              :      For 64bit target this is single move.  By hiding the fact
    6487              :      here we simplify i386.md splitters.  */
    6488      3845036 :   if (TARGET_64BIT && GET_MODE_SIZE (GET_MODE (operands[0])) == 8)
    6489              :     {
    6490              :       /* Optimize constant pool reference to immediates.  This is used by
    6491              :          fp moves, that force all constants to memory to allow combining.  */
    6492              : 
    6493        12962 :       if (MEM_P (operands[1])
    6494        12538 :           && SYMBOL_REF_P (XEXP (operands[1], 0))
    6495        13606 :           && CONSTANT_POOL_ADDRESS_P (XEXP (operands[1], 0)))
    6496          155 :         operands[1] = get_pool_constant (XEXP (operands[1], 0));
    6497        12962 :       if (push_operand (operands[0], VOIDmode))
    6498              :         {
    6499        12962 :           operands[0] = copy_rtx (operands[0]);
    6500        12962 :           PUT_MODE (operands[0], word_mode);
    6501              :         }
    6502              :       else
    6503            0 :         operands[0] = gen_lowpart (DImode, operands[0]);
    6504        12962 :       operands[1] = gen_lowpart (DImode, operands[1]);
    6505        12962 :       emit_move_insn (operands[0], operands[1]);
    6506        12962 :       return;
    6507              :     }
    6508              : 
    6509              :   /* The only non-offsettable memory we handle is push.  */
    6510      2110978 :   if (push_operand (operands[0], VOIDmode))
    6511              :     push = 1;
    6512              :   else
    6513      1926809 :     gcc_assert (!MEM_P (operands[0])
    6514              :                 || offsettable_memref_p (operands[0]));
    6515              : 
    6516      2110978 :   nparts = ix86_split_to_parts (operands[1], part[1], GET_MODE (operands[0]));
    6517      2110978 :   ix86_split_to_parts (operands[0], part[0], GET_MODE (operands[0]));
    6518              : 
    6519              :   /* When emitting push, take care for source operands on the stack.  */
    6520       184169 :   if (push && MEM_P (operands[1])
    6521      2208626 :       && reg_overlap_mentioned_p (stack_pointer_rtx, operands[1]))
    6522              :     {
    6523        56120 :       rtx src_base = XEXP (part[1][nparts - 1], 0);
    6524              : 
    6525              :       /* Compensate for the stack decrement by 4.  */
    6526        56120 :       if (!TARGET_64BIT && nparts == 3
    6527        51397 :           && mode == XFmode && TARGET_128BIT_LONG_DOUBLE)
    6528            0 :         src_base = plus_constant (Pmode, src_base, 4);
    6529              : 
    6530              :       /* src_base refers to the stack pointer and is
    6531              :          automatically decreased by emitted push.  */
    6532       168616 :       for (i = 0; i < nparts; i++)
    6533       112496 :         part[1][i] = change_address (part[1][i],
    6534       112496 :                                      GET_MODE (part[1][i]), src_base);
    6535              :     }
    6536              : 
    6537              :   /* We need to do copy in the right order in case an address register
    6538              :      of the source overlaps the destination.  */
    6539      2110978 :   if (REG_P (part[0][0]) && MEM_P (part[1][0]))
    6540              :     {
    6541              :       rtx tmp;
    6542              : 
    6543      2383713 :       for (i = 0; i < nparts; i++)
    6544              :         {
    6545      1589142 :           collisionparts[i]
    6546      1589142 :             = reg_overlap_mentioned_p (part[0][i], XEXP (part[1][0], 0));
    6547      1589142 :           if (collisionparts[i])
    6548        16928 :             collisions++;
    6549              :         }
    6550              : 
    6551              :       /* Collision in the middle part can be handled by reordering.  */
    6552       794571 :       if (collisions == 1 && nparts == 3 && collisionparts [1])
    6553              :         {
    6554            0 :           std::swap (part[0][1], part[0][2]);
    6555            0 :           std::swap (part[1][1], part[1][2]);
    6556              :         }
    6557       794571 :       else if (collisions == 1
    6558       794571 :                && nparts == 4
    6559            0 :                && (collisionparts [1] || collisionparts [2]))
    6560              :         {
    6561            0 :           if (collisionparts [1])
    6562              :             {
    6563            0 :               std::swap (part[0][1], part[0][2]);
    6564            0 :               std::swap (part[1][1], part[1][2]);
    6565              :             }
    6566              :           else
    6567              :             {
    6568            0 :               std::swap (part[0][2], part[0][3]);
    6569            0 :               std::swap (part[1][2], part[1][3]);
    6570              :             }
    6571              :         }
    6572              : 
    6573              :       /* If there are more collisions, we can't handle it by reordering.
    6574              :          Do an lea to the last part and use only one colliding move.  */
    6575       794571 :       else if (collisions > 1)
    6576              :         {
    6577           75 :           rtx base, addr;
    6578              : 
    6579           75 :           collisions = 1;
    6580              : 
    6581           75 :           base = part[0][nparts - 1];
    6582              : 
    6583              :           /* Handle the case when the last part isn't valid for lea.
    6584              :              Happens in 64-bit mode storing the 12-byte XFmode.  */
    6585          106 :           if (GET_MODE (base) != Pmode)
    6586            0 :             base = gen_rtx_REG (Pmode, REGNO (base));
    6587              : 
    6588           75 :           addr = XEXP (part[1][0], 0);
    6589           75 :           if (TARGET_TLS_DIRECT_SEG_REFS)
    6590              :             {
    6591           75 :               struct ix86_address parts;
    6592           75 :               int ok = ix86_decompose_address (addr, &parts);
    6593           75 :               gcc_assert (ok);
    6594              :               /* It is not valid to use %gs: or %fs: in lea.  */
    6595           75 :               gcc_assert (parts.seg == ADDR_SPACE_GENERIC);
    6596              :             }
    6597           75 :           emit_insn (gen_rtx_SET (base, addr));
    6598           75 :           part[1][0] = replace_equiv_address (part[1][0], base);
    6599          150 :           for (i = 1; i < nparts; i++)
    6600              :             {
    6601          137 :               tmp = plus_constant (Pmode, base, UNITS_PER_WORD * i);
    6602           75 :               part[1][i] = replace_equiv_address (part[1][i], tmp);
    6603              :             }
    6604              :         }
    6605              :     }
    6606              : 
    6607      2110978 :   if (push)
    6608              :     {
    6609       184169 :       if (!TARGET_64BIT)
    6610              :         {
    6611       158037 :           if (nparts == 3)
    6612              :             {
    6613          571 :               if (TARGET_128BIT_LONG_DOUBLE && mode == XFmode)
    6614            0 :                 emit_insn (gen_add2_insn (stack_pointer_rtx, GEN_INT (-4)));
    6615          571 :               emit_move_insn (part[0][2], part[1][2]);
    6616              :             }
    6617       157466 :           else if (nparts == 4)
    6618              :             {
    6619            0 :               emit_move_insn (part[0][3], part[1][3]);
    6620            0 :               emit_move_insn (part[0][2], part[1][2]);
    6621              :             }
    6622              :         }
    6623              :       else
    6624              :         {
    6625              :           /* In 64bit mode we don't have 32bit push available.  In case this is
    6626              :              register, it is OK - we will just use larger counterpart.  We also
    6627              :              retype memory - these comes from attempt to avoid REX prefix on
    6628              :              moving of second half of TFmode value.  */
    6629        26132 :           if (GET_MODE (part[1][1]) == SImode)
    6630              :             {
    6631        11888 :               switch (GET_CODE (part[1][1]))
    6632              :                 {
    6633        11459 :                 case MEM:
    6634        11459 :                   part[1][1] = adjust_address (part[1][1], DImode, 0);
    6635        11459 :                   break;
    6636              : 
    6637          429 :                 case REG:
    6638          429 :                   part[1][1] = gen_rtx_REG (DImode, REGNO (part[1][1]));
    6639          429 :                   break;
    6640              : 
    6641            0 :                 default:
    6642            0 :                   gcc_unreachable ();
    6643              :                 }
    6644              : 
    6645        11888 :               if (GET_MODE (part[1][0]) == SImode)
    6646            0 :                 part[1][0] = part[1][1];
    6647              :             }
    6648              :         }
    6649       184169 :       emit_move_insn (part[0][1], part[1][1]);
    6650       184169 :       emit_move_insn (part[0][0], part[1][0]);
    6651       184169 :       return;
    6652              :     }
    6653              : 
    6654              :   /* Choose correct order to not overwrite the source before it is copied.  */
    6655      1926809 :   if ((REG_P (part[0][0])
    6656      1050430 :        && REG_P (part[1][1])
    6657        93799 :        && (REGNO (part[0][0]) == REGNO (part[1][1])
    6658        76184 :            || (nparts == 3
    6659            0 :                && REGNO (part[0][0]) == REGNO (part[1][2]))
    6660        76184 :            || (nparts == 4
    6661            0 :                && REGNO (part[0][0]) == REGNO (part[1][3]))))
    6662      2959624 :       || (collisions > 0
    6663        16853 :           && reg_overlap_mentioned_p (part[0][0], XEXP (part[1][0], 0))))
    6664              :     {
    6665       102651 :       for (i = 0, j = nparts - 1; i < nparts; i++, j--)
    6666              :         {
    6667        68434 :           operands[2 + i] = part[0][j];
    6668        68434 :           operands[6 + i] = part[1][j];
    6669              :         }
    6670              :     }
    6671              :   else
    6672              :     {
    6673      5677845 :       for (i = 0; i < nparts; i++)
    6674              :         {
    6675      3785253 :           operands[2 + i] = part[0][i];
    6676      3785253 :           operands[6 + i] = part[1][i];
    6677              :         }
    6678              :     }
    6679              : 
    6680              :   /* Attempt to locally unCSE nonzero constants.  */
    6681      3853687 :   for (j = 0; j < nparts - 1; j++)
    6682      1926878 :     if (CONST_INT_P (operands[6 + j])
    6683       236822 :         && operands[6 + j] != const0_rtx
    6684        65723 :         && REG_P (operands[2 + j]))
    6685       116128 :       for (i = j; i < nparts - 1; i++)
    6686        58064 :         if (CONST_INT_P (operands[7 + i])
    6687        58064 :             && INTVAL (operands[7 + i]) == INTVAL (operands[6 + j]))
    6688        22643 :           operands[7 + i] = operands[2 + j];
    6689              : 
    6690      5780496 :   for (i = 0; i < nparts; i++)
    6691      3853687 :     emit_move_insn (operands[2 + i], operands[6 + i]);
    6692              : 
    6693              :   return;
    6694              : }
    6695              : 
    6696              : /* Helper function of ix86_split_ashl used to generate an SImode/DImode
    6697              :    left shift by a constant, either using a single shift or
    6698              :    a sequence of add instructions.  */
    6699              : 
    6700              : static void
    6701         4408 : ix86_expand_ashl_const (rtx operand, int count, machine_mode mode)
    6702              : {
    6703         4408 :   if (count == 1
    6704         4408 :       || (count * ix86_cost->add <= ix86_cost->shift_const
    6705            0 :           && !optimize_insn_for_size_p ()))
    6706              :     {
    6707           16 :       while (count-- > 0)
    6708            8 :         emit_insn (gen_add2_insn (operand, operand));
    6709              :     }
    6710              :   else
    6711              :     {
    6712         4400 :       rtx (*insn)(rtx, rtx, rtx);
    6713              : 
    6714         4400 :       insn = mode == DImode ? gen_ashlsi3 : gen_ashldi3;
    6715         4400 :       emit_insn (insn (operand, operand, GEN_INT (count)));
    6716              :     }
    6717         4408 : }
    6718              : 
    6719              : void
    6720        10624 : ix86_split_ashl (rtx *operands, rtx scratch, machine_mode mode)
    6721              : {
    6722        10624 :   rtx (*gen_ashl3)(rtx, rtx, rtx);
    6723        10624 :   rtx (*gen_shld)(rtx, rtx, rtx);
    6724        10624 :   int half_width = GET_MODE_BITSIZE (mode) >> 1;
    6725        10624 :   machine_mode half_mode;
    6726              : 
    6727        10624 :   rtx low[2], high[2];
    6728        10624 :   int count;
    6729              : 
    6730        10624 :   if (CONST_INT_P (operands[2]))
    6731              :     {
    6732         8904 :       split_double_mode (mode, operands, 2, low, high);
    6733         8904 :       count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
    6734              : 
    6735         8904 :       if (count >= half_width)
    6736              :         {
    6737         2709 :           emit_move_insn (high[0], low[1]);
    6738         2709 :           ix86_expand_clear (low[0]);
    6739              : 
    6740         2709 :           if (count > half_width)
    6741          153 :             ix86_expand_ashl_const (high[0], count - half_width, mode);
    6742              :         }
    6743         6195 :       else if (count == 1)
    6744              :         {
    6745         1940 :           if (!rtx_equal_p (operands[0], operands[1]))
    6746            0 :             emit_move_insn (operands[0], operands[1]);
    6747         1940 :           rtx x3 = gen_rtx_REG (CCCmode, FLAGS_REG);
    6748         1940 :           rtx x4 = gen_rtx_LTU (mode, x3, const0_rtx);
    6749         1940 :           half_mode = mode == DImode ? SImode : DImode;
    6750         1940 :           emit_insn (gen_add3_cc_overflow_1 (half_mode, low[0],
    6751              :                                              low[0], low[0]));
    6752         1940 :           emit_insn (gen_add3_carry (half_mode, high[0], high[0], high[0],
    6753              :                                      x3, x4));
    6754              :         }
    6755              :       else
    6756              :         {
    6757         4255 :           gen_shld = mode == DImode ? gen_x86_shld : gen_x86_64_shld;
    6758              : 
    6759         4255 :           if (!rtx_equal_p (operands[0], operands[1]))
    6760            0 :             emit_move_insn (operands[0], operands[1]);
    6761              : 
    6762         4255 :           emit_insn (gen_shld (high[0], low[0], GEN_INT (count)));
    6763         4255 :           ix86_expand_ashl_const (low[0], count, mode);
    6764              :         }
    6765         9165 :       return;
    6766              :     }
    6767              : 
    6768         1720 :   split_double_mode (mode, operands, 1, low, high);
    6769         1720 :   half_mode = mode == DImode ? SImode : DImode;
    6770              : 
    6771         1720 :   gen_ashl3 = mode == DImode ? gen_ashlsi3 : gen_ashldi3;
    6772              : 
    6773         1720 :   if (operands[1] == const1_rtx)
    6774              :     {
    6775              :       /* Assuming we've chosen a QImode capable registers, then 1 << N
    6776              :          can be done with two 32/64-bit shifts, no branches, no cmoves.  */
    6777          261 :       if (ANY_QI_REG_P (low[0]) && ANY_QI_REG_P (high[0]))
    6778              :         {
    6779          151 :           rtx s, d, flags = gen_rtx_REG (CCZmode, FLAGS_REG);
    6780              : 
    6781          151 :           ix86_expand_clear (low[0]);
    6782          151 :           ix86_expand_clear (high[0]);
    6783          151 :           emit_insn (gen_testqi_ccz_1 (operands[2], GEN_INT (half_width)));
    6784              : 
    6785          151 :           d = gen_lowpart (QImode, low[0]);
    6786          151 :           d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
    6787          151 :           s = gen_rtx_EQ (QImode, flags, const0_rtx);
    6788          151 :           emit_insn (gen_rtx_SET (d, s));
    6789              : 
    6790          151 :           d = gen_lowpart (QImode, high[0]);
    6791          151 :           d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
    6792          151 :           s = gen_rtx_NE (QImode, flags, const0_rtx);
    6793          151 :           emit_insn (gen_rtx_SET (d, s));
    6794              :         }
    6795              : 
    6796              :       /* Otherwise, we can get the same results by manually performing
    6797              :          a bit extract operation on bit 5/6, and then performing the two
    6798              :          shifts.  The two methods of getting 0/1 into low/high are exactly
    6799              :          the same size.  Avoiding the shift in the bit extract case helps
    6800              :          pentium4 a bit; no one else seems to care much either way.  */
    6801              :       else
    6802              :         {
    6803          110 :           rtx (*gen_lshr3)(rtx, rtx, rtx);
    6804          110 :           rtx (*gen_and3)(rtx, rtx, rtx);
    6805          110 :           rtx (*gen_xor3)(rtx, rtx, rtx);
    6806          110 :           HOST_WIDE_INT bits;
    6807          110 :           rtx x;
    6808              : 
    6809          110 :           if (mode == DImode)
    6810              :             {
    6811              :               gen_lshr3 = gen_lshrsi3;
    6812              :               gen_and3 = gen_andsi3;
    6813              :               gen_xor3 = gen_xorsi3;
    6814              :               bits = 5;
    6815              :             }
    6816              :           else
    6817              :             {
    6818            0 :               gen_lshr3 = gen_lshrdi3;
    6819            0 :               gen_and3 = gen_anddi3;
    6820            0 :               gen_xor3 = gen_xordi3;
    6821            0 :               bits = 6;
    6822              :             }
    6823              : 
    6824          110 :           if (TARGET_PARTIAL_REG_STALL && !optimize_insn_for_size_p ())
    6825            0 :             x = gen_rtx_ZERO_EXTEND (half_mode, operands[2]);
    6826              :           else
    6827          110 :             x = gen_lowpart (half_mode, operands[2]);
    6828          110 :           emit_insn (gen_rtx_SET (high[0], x));
    6829              : 
    6830          110 :           emit_insn (gen_lshr3 (high[0], high[0], GEN_INT (bits)));
    6831          110 :           emit_insn (gen_and3 (high[0], high[0], const1_rtx));
    6832          110 :           emit_move_insn (low[0], high[0]);
    6833          110 :           emit_insn (gen_xor3 (low[0], low[0], const1_rtx));
    6834              :         }
    6835              : 
    6836          261 :       emit_insn (gen_ashl3 (low[0], low[0], operands[2]));
    6837          261 :       emit_insn (gen_ashl3 (high[0], high[0], operands[2]));
    6838          261 :       return;
    6839              :     }
    6840              : 
    6841         1459 :   if (operands[1] == constm1_rtx)
    6842              :     {
    6843              :       /* For -1 << N, we can avoid the shld instruction, because we
    6844              :          know that we're shifting 0...31/63 ones into a -1.  */
    6845          109 :       emit_move_insn (low[0], constm1_rtx);
    6846          109 :       if (optimize_insn_for_size_p ())
    6847            6 :         emit_move_insn (high[0], low[0]);
    6848              :       else
    6849          103 :         emit_move_insn (high[0], constm1_rtx);
    6850              :     }
    6851              :   else
    6852              :     {
    6853         1350 :       gen_shld = mode == DImode ? gen_x86_shld : gen_x86_64_shld;
    6854              : 
    6855         1350 :       if (!rtx_equal_p (operands[0], operands[1]))
    6856            0 :         emit_move_insn (operands[0], operands[1]);
    6857              : 
    6858         1350 :       split_double_mode (mode, operands, 1, low, high);
    6859         1350 :       emit_insn (gen_shld (high[0], low[0], operands[2]));
    6860              :     }
    6861              : 
    6862         1459 :   emit_insn (gen_ashl3 (low[0], low[0], operands[2]));
    6863              : 
    6864         1459 :   if (TARGET_CMOVE && scratch)
    6865              :     {
    6866          998 :       ix86_expand_clear (scratch);
    6867          998 :       emit_insn (gen_x86_shift_adj_1
    6868              :                  (half_mode, high[0], low[0], operands[2], scratch));
    6869              :     }
    6870              :   else
    6871          461 :     emit_insn (gen_x86_shift_adj_2 (half_mode, high[0], low[0], operands[2]));
    6872              : }
    6873              : 
    6874              : void
    6875         6331 : ix86_split_ashr (rtx *operands, rtx scratch, machine_mode mode)
    6876              : {
    6877         5085 :   rtx (*gen_ashr3)(rtx, rtx, rtx)
    6878         6331 :     = mode == DImode ? gen_ashrsi3 : gen_ashrdi3;
    6879         6331 :   rtx (*gen_shrd)(rtx, rtx, rtx);
    6880         6331 :   int half_width = GET_MODE_BITSIZE (mode) >> 1;
    6881              : 
    6882         6331 :   rtx low[2], high[2];
    6883         6331 :   int count;
    6884              : 
    6885         6331 :   if (CONST_INT_P (operands[2]))
    6886              :     {
    6887         6157 :       split_double_mode (mode, operands, 2, low, high);
    6888         6157 :       count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
    6889              : 
    6890        12314 :       if (count == GET_MODE_BITSIZE (mode) - 1)
    6891              :         {
    6892           93 :           emit_move_insn (high[0], high[1]);
    6893           93 :           emit_insn (gen_ashr3 (high[0], high[0],
    6894           93 :                                 GEN_INT (half_width - 1)));
    6895           93 :           emit_move_insn (low[0], high[0]);
    6896              : 
    6897              :         }
    6898         6064 :       else if (count >= half_width)
    6899              :         {
    6900         1999 :           emit_move_insn (low[0], high[1]);
    6901         1999 :           emit_move_insn (high[0], low[0]);
    6902         1999 :           emit_insn (gen_ashr3 (high[0], high[0],
    6903         1999 :                                 GEN_INT (half_width - 1)));
    6904              : 
    6905         1999 :           if (count > half_width)
    6906           38 :             emit_insn (gen_ashr3 (low[0], low[0],
    6907           38 :                                   GEN_INT (count - half_width)));
    6908              :         }
    6909         4065 :       else if (count == 1
    6910          767 :                && (TARGET_USE_RCR || optimize_size > 1))
    6911              :         {
    6912            1 :           if (!rtx_equal_p (operands[0], operands[1]))
    6913            0 :             emit_move_insn (operands[0], operands[1]);
    6914            1 :           if (mode == DImode)
    6915              :             {
    6916            0 :               emit_insn (gen_ashrsi3_carry (high[0], high[0]));
    6917            0 :               emit_insn (gen_rcrsi2 (low[0], low[0]));
    6918              :             }
    6919              :           else
    6920              :             {
    6921            1 :               emit_insn (gen_ashrdi3_carry (high[0], high[0]));
    6922            1 :               emit_insn (gen_rcrdi2 (low[0], low[0]));
    6923              :             }
    6924              :         }
    6925              :       else
    6926              :         {
    6927         4064 :           gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
    6928              : 
    6929         4064 :           if (!rtx_equal_p (operands[0], operands[1]))
    6930            0 :             emit_move_insn (operands[0], operands[1]);
    6931              : 
    6932         4064 :           emit_insn (gen_shrd (low[0], high[0], GEN_INT (count)));
    6933         4064 :           emit_insn (gen_ashr3 (high[0], high[0], GEN_INT (count)));
    6934              :         }
    6935              :     }
    6936              :   else
    6937              :     {
    6938          174 :       machine_mode half_mode;
    6939              : 
    6940          174 :       gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
    6941              : 
    6942          174 :      if (!rtx_equal_p (operands[0], operands[1]))
    6943            0 :         emit_move_insn (operands[0], operands[1]);
    6944              : 
    6945          174 :       split_double_mode (mode, operands, 1, low, high);
    6946          174 :       half_mode = mode == DImode ? SImode : DImode;
    6947              : 
    6948          174 :       emit_insn (gen_shrd (low[0], high[0], operands[2]));
    6949          174 :       emit_insn (gen_ashr3 (high[0], high[0], operands[2]));
    6950              : 
    6951          174 :       if (TARGET_CMOVE && scratch)
    6952              :         {
    6953          134 :           emit_move_insn (scratch, high[0]);
    6954          134 :           emit_insn (gen_ashr3 (scratch, scratch,
    6955          134 :                                 GEN_INT (half_width - 1)));
    6956          134 :           emit_insn (gen_x86_shift_adj_1
    6957              :                      (half_mode, low[0], high[0], operands[2], scratch));
    6958              :         }
    6959              :       else
    6960           40 :         emit_insn (gen_x86_shift_adj_3
    6961              :                    (half_mode, low[0], high[0], operands[2]));
    6962              :     }
    6963         6331 : }
    6964              : 
    6965              : void
    6966        16830 : ix86_split_lshr (rtx *operands, rtx scratch, machine_mode mode)
    6967              : {
    6968         6795 :   rtx (*gen_lshr3)(rtx, rtx, rtx)
    6969        16830 :     = mode == DImode ? gen_lshrsi3 : gen_lshrdi3;
    6970        16830 :   rtx (*gen_shrd)(rtx, rtx, rtx);
    6971        16830 :   int half_width = GET_MODE_BITSIZE (mode) >> 1;
    6972              : 
    6973        16830 :   rtx low[2], high[2];
    6974        16830 :   int count;
    6975              : 
    6976        16830 :   if (CONST_INT_P (operands[2]))
    6977              :     {
    6978        15395 :       split_double_mode (mode, operands, 2, low, high);
    6979        15395 :       count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
    6980              : 
    6981        15395 :       if (count >= half_width)
    6982              :         {
    6983        11923 :           emit_move_insn (low[0], high[1]);
    6984        11923 :           ix86_expand_clear (high[0]);
    6985              : 
    6986        11923 :           if (count > half_width)
    6987         1483 :             emit_insn (gen_lshr3 (low[0], low[0],
    6988         1483 :                                   GEN_INT (count - half_width)));
    6989              :         }
    6990         3472 :       else if (count == 1
    6991          649 :                && (TARGET_USE_RCR || optimize_size > 1))
    6992              :         {
    6993            1 :           if (!rtx_equal_p (operands[0], operands[1]))
    6994            0 :             emit_move_insn (operands[0], operands[1]);
    6995            1 :           if (mode == DImode)
    6996              :             {
    6997            0 :               emit_insn (gen_lshrsi3_carry (high[0], high[0]));
    6998            0 :               emit_insn (gen_rcrsi2 (low[0], low[0]));
    6999              :             }
    7000              :           else
    7001              :             {
    7002            1 :               emit_insn (gen_lshrdi3_carry (high[0], high[0]));
    7003            1 :               emit_insn (gen_rcrdi2 (low[0], low[0]));
    7004              :             }
    7005              :         }
    7006              :       else
    7007              :         {
    7008         3471 :           gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
    7009              : 
    7010         3471 :           if (!rtx_equal_p (operands[0], operands[1]))
    7011            0 :             emit_move_insn (operands[0], operands[1]);
    7012              : 
    7013         3471 :           emit_insn (gen_shrd (low[0], high[0], GEN_INT (count)));
    7014         3471 :           emit_insn (gen_lshr3 (high[0], high[0], GEN_INT (count)));
    7015              :         }
    7016              :     }
    7017              :   else
    7018              :     {
    7019         1435 :       machine_mode half_mode;
    7020              : 
    7021         1435 :       gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
    7022              : 
    7023         1435 :       if (!rtx_equal_p (operands[0], operands[1]))
    7024            0 :         emit_move_insn (operands[0], operands[1]);
    7025              : 
    7026         1435 :       split_double_mode (mode, operands, 1, low, high);
    7027         1435 :       half_mode = mode == DImode ? SImode : DImode;
    7028              : 
    7029         1435 :       emit_insn (gen_shrd (low[0], high[0], operands[2]));
    7030         1435 :       emit_insn (gen_lshr3 (high[0], high[0], operands[2]));
    7031              : 
    7032         1435 :       if (TARGET_CMOVE && scratch)
    7033              :         {
    7034         1150 :           ix86_expand_clear (scratch);
    7035         1150 :           emit_insn (gen_x86_shift_adj_1
    7036              :                      (half_mode, low[0], high[0], operands[2], scratch));
    7037              :         }
    7038              :       else
    7039          285 :         emit_insn (gen_x86_shift_adj_2
    7040              :                    (half_mode, low[0], high[0], operands[2]));
    7041              :     }
    7042        16830 : }
    7043              : 
    7044              : /* Helper function to split TImode ashl under NDD.  */
    7045              : void
    7046            1 : ix86_split_ashl_ndd (rtx *operands, rtx scratch)
    7047              : {
    7048            1 :   gcc_assert (TARGET_APX_NDD);
    7049            1 :   int half_width = GET_MODE_BITSIZE (TImode) >> 1;
    7050              : 
    7051            1 :   rtx low[2], high[2];
    7052            1 :   int count;
    7053              : 
    7054            1 :   split_double_mode (TImode, operands, 2, low, high);
    7055            1 :   if (CONST_INT_P (operands[2]))
    7056              :     {
    7057            0 :       count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (TImode) - 1);
    7058              : 
    7059            0 :       if (count >= half_width)
    7060              :         {
    7061            0 :           count = count - half_width;
    7062            0 :           if (count == 0)
    7063              :             {
    7064            0 :               if (!rtx_equal_p (high[0], low[1]))
    7065            0 :                 emit_move_insn (high[0], low[1]);
    7066              :             }
    7067            0 :           else if (count == 1)
    7068            0 :             emit_insn (gen_adddi3 (high[0], low[1], low[1]));
    7069              :           else
    7070            0 :             emit_insn (gen_ashldi3 (high[0], low[1], GEN_INT (count)));
    7071              : 
    7072            0 :           ix86_expand_clear (low[0]);
    7073              :         }
    7074            0 :       else if (count == 1)
    7075              :         {
    7076            0 :           rtx x3 = gen_rtx_REG (CCCmode, FLAGS_REG);
    7077            0 :           rtx x4 = gen_rtx_LTU (TImode, x3, const0_rtx);
    7078            0 :           emit_insn (gen_add3_cc_overflow_1 (DImode, low[0],
    7079              :                                              low[1], low[1]));
    7080            0 :           emit_insn (gen_add3_carry (DImode, high[0], high[1], high[1],
    7081              :                                      x3, x4));
    7082              :         }
    7083              :       else
    7084              :         {
    7085            0 :           emit_insn (gen_x86_64_shld_ndd (high[0], high[1], low[1],
    7086              :                                           GEN_INT (count)));
    7087            0 :           emit_insn (gen_ashldi3 (low[0], low[1], GEN_INT (count)));
    7088              :         }
    7089              :     }
    7090              :   else
    7091              :     {
    7092            1 :       emit_insn (gen_x86_64_shld_ndd (high[0], high[1], low[1],
    7093              :                                       operands[2]));
    7094            1 :       emit_insn (gen_ashldi3 (low[0], low[1], operands[2]));
    7095            1 :       if (TARGET_CMOVE && scratch)
    7096              :         {
    7097            1 :           ix86_expand_clear (scratch);
    7098            1 :           emit_insn (gen_x86_shift_adj_1
    7099              :                      (DImode, high[0], low[0], operands[2], scratch));
    7100              :         }
    7101              :       else
    7102            0 :         emit_insn (gen_x86_shift_adj_2 (DImode, high[0], low[0], operands[2]));
    7103              :     }
    7104            1 : }
    7105              : 
    7106              : /* Helper function to split TImode l/ashr under NDD.  */
    7107              : void
    7108            2 : ix86_split_rshift_ndd (enum rtx_code code, rtx *operands, rtx scratch)
    7109              : {
    7110            2 :   gcc_assert (TARGET_APX_NDD);
    7111            2 :   int half_width = GET_MODE_BITSIZE (TImode) >> 1;
    7112            2 :   bool ashr_p = code == ASHIFTRT;
    7113            2 :   rtx (*gen_shr)(rtx, rtx, rtx) = ashr_p ? gen_ashrdi3
    7114              :                                          : gen_lshrdi3;
    7115              : 
    7116            2 :   rtx low[2], high[2];
    7117            2 :   int count;
    7118              : 
    7119            2 :   split_double_mode (TImode, operands, 2, low, high);
    7120            2 :   if (CONST_INT_P (operands[2]))
    7121              :     {
    7122            0 :       count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (TImode) - 1);
    7123              : 
    7124            0 :       if (ashr_p && (count == GET_MODE_BITSIZE (TImode) - 1))
    7125              :         {
    7126            0 :           emit_insn (gen_shr (high[0], high[1],
    7127              :                               GEN_INT (half_width - 1)));
    7128            0 :           emit_move_insn (low[0], high[0]);
    7129              :         }
    7130            0 :       else if (count >= half_width)
    7131              :         {
    7132            0 :           if (ashr_p)
    7133            0 :             emit_insn (gen_shr (high[0], high[1],
    7134              :                                 GEN_INT (half_width - 1)));
    7135              :           else
    7136            0 :             ix86_expand_clear (high[0]);
    7137              : 
    7138            0 :           if (count > half_width)
    7139            0 :             emit_insn (gen_shr (low[0], high[1],
    7140            0 :                                 GEN_INT (count - half_width)));
    7141              :           else
    7142            0 :             emit_move_insn (low[0], high[1]);
    7143              :         }
    7144              :       else
    7145              :         {
    7146            0 :           emit_insn (gen_x86_64_shrd_ndd (low[0], low[1], high[1],
    7147              :                                           GEN_INT (count)));
    7148            0 :           emit_insn (gen_shr (high[0], high[1], GEN_INT (count)));
    7149              :         }
    7150              :     }
    7151              :   else
    7152              :     {
    7153            2 :       emit_insn (gen_x86_64_shrd_ndd (low[0], low[1], high[1],
    7154              :                                       operands[2]));
    7155            2 :       emit_insn (gen_shr (high[0], high[1], operands[2]));
    7156              : 
    7157            2 :       if (TARGET_CMOVE && scratch)
    7158              :         {
    7159            2 :           if (ashr_p)
    7160              :             {
    7161            1 :               emit_move_insn (scratch, high[0]);
    7162            1 :               emit_insn (gen_shr (scratch, scratch,
    7163              :                                   GEN_INT (half_width - 1)));
    7164              :             }
    7165              :           else
    7166            1 :             ix86_expand_clear (scratch);
    7167              : 
    7168            2 :           emit_insn (gen_x86_shift_adj_1
    7169              :                      (DImode, low[0], high[0], operands[2], scratch));
    7170              :         }
    7171            0 :       else if (ashr_p)
    7172            0 :         emit_insn (gen_x86_shift_adj_3
    7173              :                    (DImode, low[0], high[0], operands[2]));
    7174              :       else
    7175            0 :         emit_insn (gen_x86_shift_adj_2
    7176              :                    (DImode, low[0], high[0], operands[2]));
    7177              :     }
    7178            2 : }
    7179              : 
    7180              : /* Expand move of V1TI mode register X to a new TI mode register.  */
    7181              : static rtx
    7182           18 : ix86_expand_v1ti_to_ti (rtx x)
    7183              : {
    7184           18 :   rtx result = gen_reg_rtx (TImode);
    7185           18 :   if (TARGET_SSE2)
    7186              :     {
    7187           18 :       rtx temp = force_reg (V2DImode, gen_lowpart (V2DImode, x));
    7188           18 :       rtx lo = gen_lowpart (DImode, result);
    7189           18 :       emit_insn (gen_vec_extractv2didi (lo, temp, const0_rtx));
    7190           18 :       rtx hi = gen_highpart (DImode, result);
    7191           18 :       emit_insn (gen_vec_extractv2didi (hi, temp, const1_rtx));
    7192              :     }
    7193              :   else
    7194            0 :     emit_move_insn (result, gen_lowpart (TImode, x));
    7195           18 :   return result;
    7196              : }
    7197              : 
    7198              : /* Expand move of TI mode register X to a new V1TI mode register.  */
    7199              : static rtx
    7200           18 : ix86_expand_ti_to_v1ti (rtx x)
    7201              : {
    7202           18 :   if (TARGET_SSE2)
    7203              :     {
    7204           18 :       rtx lo = gen_lowpart (DImode, x);
    7205           18 :       rtx hi = gen_highpart (DImode, x);
    7206           18 :       rtx tmp = gen_reg_rtx (V2DImode);
    7207           18 :       emit_insn (gen_vec_concatv2di (tmp, lo, hi));
    7208           18 :       return force_reg (V1TImode, gen_lowpart (V1TImode, tmp));
    7209              :     }
    7210              : 
    7211            0 :   return force_reg (V1TImode, gen_lowpart (V1TImode, x));
    7212              : }
    7213              : 
    7214              : /* Expand V1TI mode shift (of rtx_code CODE) by constant.  */
    7215              : void
    7216           43 : ix86_expand_v1ti_shift (enum rtx_code code, rtx operands[])
    7217              : {
    7218           43 :   rtx op1 = force_reg (V1TImode, operands[1]);
    7219              : 
    7220           43 :   if (!CONST_INT_P (operands[2]))
    7221              :     {
    7222            7 :       rtx tmp1 = ix86_expand_v1ti_to_ti (op1);
    7223            7 :       rtx tmp2 = gen_reg_rtx (TImode);
    7224            3 :       rtx (*shift) (rtx, rtx, rtx)
    7225            7 :             = (code == ASHIFT) ? gen_ashlti3 : gen_lshrti3;
    7226            7 :       emit_insn (shift (tmp2, tmp1, operands[2]));
    7227            7 :       rtx tmp3 = ix86_expand_ti_to_v1ti (tmp2);
    7228            7 :       emit_move_insn (operands[0], tmp3);
    7229            7 :       return;
    7230              :     }
    7231              : 
    7232           36 :   HOST_WIDE_INT bits = INTVAL (operands[2]) & 127;
    7233              : 
    7234           36 :   if (bits == 0)
    7235              :     {
    7236            0 :       emit_move_insn (operands[0], op1);
    7237            0 :       return;
    7238              :     }
    7239              : 
    7240           36 :   if ((bits & 7) == 0)
    7241              :     {
    7242            0 :       rtx tmp = gen_reg_rtx (V1TImode);
    7243            0 :       if (code == ASHIFT)
    7244            0 :         emit_insn (gen_sse2_ashlv1ti3 (tmp, op1, GEN_INT (bits)));
    7245              :       else
    7246            0 :         emit_insn (gen_sse2_lshrv1ti3 (tmp, op1, GEN_INT (bits)));
    7247            0 :       emit_move_insn (operands[0], tmp);
    7248            0 :       return;
    7249              :     }
    7250              : 
    7251           36 :   rtx tmp1 = gen_reg_rtx (V1TImode);
    7252           36 :   if (code == ASHIFT)
    7253           18 :     emit_insn (gen_sse2_ashlv1ti3 (tmp1, op1, GEN_INT (64)));
    7254              :   else
    7255           18 :     emit_insn (gen_sse2_lshrv1ti3 (tmp1, op1, GEN_INT (64)));
    7256              : 
    7257              :   /* tmp2 is operands[1] shifted by 64, in V2DImode.  */
    7258           36 :   rtx tmp2 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp1));
    7259              : 
    7260              :   /* tmp3 will be the V2DImode result.  */
    7261           36 :   rtx tmp3 = gen_reg_rtx (V2DImode);
    7262              : 
    7263           36 :   if (bits > 64)
    7264              :     {
    7265           18 :       if (code == ASHIFT)
    7266            9 :         emit_insn (gen_ashlv2di3 (tmp3, tmp2, GEN_INT (bits - 64)));
    7267              :       else
    7268            9 :         emit_insn (gen_lshrv2di3 (tmp3, tmp2, GEN_INT (bits - 64)));
    7269              :     }
    7270              :   else
    7271              :     {
    7272              :       /* tmp4 is operands[1], in V2DImode.  */
    7273           18 :       rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
    7274              : 
    7275           18 :       rtx tmp5 = gen_reg_rtx (V2DImode);
    7276           18 :       if (code == ASHIFT)
    7277            9 :         emit_insn (gen_ashlv2di3 (tmp5, tmp4, GEN_INT (bits)));
    7278              :       else
    7279            9 :         emit_insn (gen_lshrv2di3 (tmp5, tmp4, GEN_INT (bits)));
    7280              : 
    7281           18 :       rtx tmp6 = gen_reg_rtx (V2DImode);
    7282           18 :       if (code == ASHIFT)
    7283            9 :         emit_insn (gen_lshrv2di3 (tmp6, tmp2, GEN_INT (64 - bits)));
    7284              :       else
    7285            9 :         emit_insn (gen_ashlv2di3 (tmp6, tmp2, GEN_INT (64 - bits)));
    7286              : 
    7287           18 :       emit_insn (gen_iorv2di3 (tmp3, tmp5, tmp6));
    7288              :     }
    7289              : 
    7290              :   /* Convert the result back to V1TImode and store in operands[0].  */
    7291           36 :   rtx tmp7 = force_reg (V1TImode, gen_lowpart (V1TImode, tmp3));
    7292           36 :   emit_move_insn (operands[0], tmp7);
    7293              : }
    7294              : 
    7295              : /* Expand V1TI mode rotate (of rtx_code CODE) by constant.  */
    7296              : void
    7297           39 : ix86_expand_v1ti_rotate (enum rtx_code code, rtx operands[])
    7298              : {
    7299           39 :   rtx op1 = force_reg (V1TImode, operands[1]);
    7300              : 
    7301           39 :   if (!CONST_INT_P (operands[2]))
    7302              :     {
    7303            8 :       rtx tmp1 = ix86_expand_v1ti_to_ti (op1);
    7304            8 :       rtx tmp2 = gen_reg_rtx (TImode);
    7305            4 :       rtx (*rotate) (rtx, rtx, rtx)
    7306            8 :             = (code == ROTATE) ? gen_rotlti3 : gen_rotrti3;
    7307            8 :       emit_insn (rotate (tmp2, tmp1, operands[2]));
    7308            8 :       rtx tmp3 = ix86_expand_ti_to_v1ti (tmp2);
    7309            8 :       emit_move_insn (operands[0], tmp3);
    7310            8 :       return;
    7311              :     }
    7312              : 
    7313           31 :   HOST_WIDE_INT bits = INTVAL (operands[2]) & 127;
    7314              : 
    7315           31 :   if (bits == 0)
    7316              :     {
    7317            0 :       emit_move_insn (operands[0], op1);
    7318            0 :       return;
    7319              :     }
    7320              : 
    7321           31 :   if (code == ROTATERT)
    7322           16 :     bits = 128 - bits;
    7323              : 
    7324           31 :   if ((bits & 31) == 0)
    7325              :     {
    7326            5 :       rtx tmp2 = gen_reg_rtx (V4SImode);
    7327            5 :       rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
    7328            5 :       if (bits == 32)
    7329            1 :         emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0x93)));
    7330            4 :       else if (bits == 64)
    7331            2 :         emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0x4e)));
    7332              :       else
    7333            2 :         emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0x39)));
    7334            5 :       emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp2));
    7335            5 :       return;
    7336              :     }
    7337              : 
    7338           26 :   if ((bits & 7) == 0)
    7339              :     {
    7340            6 :       rtx tmp1 = gen_reg_rtx (V1TImode);
    7341            6 :       rtx tmp2 = gen_reg_rtx (V1TImode);
    7342            6 :       rtx tmp3 = gen_reg_rtx (V1TImode);
    7343              : 
    7344            6 :       emit_insn (gen_sse2_ashlv1ti3 (tmp1, op1, GEN_INT (bits)));
    7345            6 :       emit_insn (gen_sse2_lshrv1ti3 (tmp2, op1, GEN_INT (128 - bits)));
    7346            6 :       emit_insn (gen_iorv1ti3 (tmp3, tmp1, tmp2));
    7347            6 :       emit_move_insn (operands[0], tmp3);
    7348            6 :       return;
    7349              :     }
    7350              : 
    7351           20 :   rtx op1_v4si = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
    7352              : 
    7353           20 :   rtx lobits;
    7354           20 :   rtx hibits;
    7355              : 
    7356           20 :   switch (bits >> 5)
    7357              :     {
    7358            7 :     case 0:
    7359            7 :       lobits = op1_v4si;
    7360            7 :       hibits = gen_reg_rtx (V4SImode);
    7361            7 :       emit_insn (gen_sse2_pshufd (hibits, op1_v4si, GEN_INT (0x93)));
    7362            7 :       break;
    7363              : 
    7364            2 :     case 1:
    7365            2 :       lobits = gen_reg_rtx (V4SImode);
    7366            2 :       hibits = gen_reg_rtx (V4SImode);
    7367            2 :       emit_insn (gen_sse2_pshufd (lobits, op1_v4si, GEN_INT (0x93)));
    7368            2 :       emit_insn (gen_sse2_pshufd (hibits, op1_v4si, GEN_INT (0x4e)));
    7369            2 :       break;
    7370              : 
    7371            2 :     case 2:
    7372            2 :       lobits = gen_reg_rtx (V4SImode);
    7373            2 :       hibits = gen_reg_rtx (V4SImode);
    7374            2 :       emit_insn (gen_sse2_pshufd (lobits, op1_v4si, GEN_INT (0x4e)));
    7375            2 :       emit_insn (gen_sse2_pshufd (hibits, op1_v4si, GEN_INT (0x39)));
    7376            2 :       break;
    7377              : 
    7378            9 :     default:
    7379            9 :       lobits = gen_reg_rtx (V4SImode);
    7380            9 :       emit_insn (gen_sse2_pshufd (lobits, op1_v4si, GEN_INT (0x39)));
    7381            9 :       hibits = op1_v4si;
    7382            9 :       break;
    7383              :     }
    7384              : 
    7385           20 :   rtx tmp1 = gen_reg_rtx (V4SImode);
    7386           20 :   rtx tmp2 = gen_reg_rtx (V4SImode);
    7387           20 :   rtx tmp3 = gen_reg_rtx (V4SImode);
    7388              : 
    7389           20 :   emit_insn (gen_ashlv4si3 (tmp1, lobits, GEN_INT (bits & 31)));
    7390           20 :   emit_insn (gen_lshrv4si3 (tmp2, hibits, GEN_INT (32 - (bits & 31))));
    7391           20 :   emit_insn (gen_iorv4si3 (tmp3, tmp1, tmp2));
    7392              : 
    7393           20 :   emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp3));
    7394              : }
    7395              : 
    7396              : /* Expand V1TI mode ashiftrt by constant.  */
    7397              : void
    7398          109 : ix86_expand_v1ti_ashiftrt (rtx operands[])
    7399              : {
    7400          109 :   rtx op1 = force_reg (V1TImode, operands[1]);
    7401              : 
    7402          109 :   if (!CONST_INT_P (operands[2]))
    7403              :     {
    7404            3 :       rtx tmp1 = ix86_expand_v1ti_to_ti (op1);
    7405            3 :       rtx tmp2 = gen_reg_rtx (TImode);
    7406            3 :       emit_insn (gen_ashrti3 (tmp2, tmp1, operands[2]));
    7407            3 :       rtx tmp3 = ix86_expand_ti_to_v1ti (tmp2);
    7408            3 :       emit_move_insn (operands[0], tmp3);
    7409            3 :       return;
    7410              :     }
    7411              : 
    7412          106 :   HOST_WIDE_INT bits = INTVAL (operands[2]) & 127;
    7413              : 
    7414          106 :   if (bits == 0)
    7415              :     {
    7416            0 :       emit_move_insn (operands[0], op1);
    7417            0 :       return;
    7418              :     }
    7419              : 
    7420          106 :   if (bits == 127)
    7421              :     {
    7422              :       /* Two operations.  */
    7423            3 :       rtx tmp1 = force_reg(V4SImode, gen_lowpart (V4SImode, op1));
    7424            3 :       rtx tmp2 = gen_reg_rtx (V4SImode);
    7425            3 :       emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
    7426              : 
    7427            3 :       rtx tmp3 = gen_reg_rtx (V4SImode);
    7428            3 :       emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
    7429              : 
    7430            3 :       emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp3));
    7431            3 :       return;
    7432              :     }
    7433              : 
    7434          103 :   if (bits == 64)
    7435              :     {
    7436              :       /* Three operations.  */
    7437            3 :       rtx tmp1 = force_reg(V4SImode, gen_lowpart (V4SImode, op1));
    7438            3 :       rtx tmp2 = gen_reg_rtx (V4SImode);
    7439            3 :       emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
    7440              : 
    7441            3 :       rtx tmp3 = gen_reg_rtx (V4SImode);
    7442            3 :       emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
    7443              : 
    7444            3 :       rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp1));
    7445            3 :       rtx tmp5 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
    7446            3 :       rtx tmp6 = gen_reg_rtx (V2DImode);
    7447            3 :       emit_insn (gen_vec_interleave_highv2di (tmp6, tmp4, tmp5));
    7448              : 
    7449            3 :       emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp6));
    7450            3 :       return;
    7451              :     }
    7452              : 
    7453          100 :   if (bits == 96)
    7454              :     {
    7455              :       /* Three operations.  */
    7456            3 :       rtx tmp1 = force_reg(V4SImode, gen_lowpart (V4SImode, op1));
    7457            3 :       rtx tmp2 = gen_reg_rtx (V4SImode);
    7458            3 :       emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (31)));
    7459              : 
    7460            3 :       rtx tmp3 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp1));
    7461            3 :       rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp2));
    7462            3 :       rtx tmp5 = gen_reg_rtx (V2DImode);
    7463            3 :       emit_insn (gen_vec_interleave_highv2di (tmp5, tmp3, tmp4));
    7464              : 
    7465            3 :       rtx tmp6 = force_reg(V4SImode, gen_lowpart (V4SImode, tmp5));
    7466            3 :       rtx tmp7 = gen_reg_rtx (V4SImode);
    7467            3 :       emit_insn (gen_sse2_pshufd (tmp7, tmp6, GEN_INT (0xfd)));
    7468              : 
    7469            3 :       emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp7));
    7470            3 :       return;
    7471              :     }
    7472              : 
    7473           97 :   if (bits >= 111)
    7474              :     {
    7475              :       /* Three operations.  */
    7476           21 :       rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
    7477           21 :       rtx tmp2 = gen_reg_rtx (V4SImode);
    7478           21 :       emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits - 96)));
    7479              : 
    7480           21 :       rtx tmp3 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp2));
    7481           21 :       rtx tmp4 = gen_reg_rtx (V8HImode);
    7482           21 :       emit_insn (gen_sse2_pshufhw (tmp4, tmp3, GEN_INT (0xfe)));
    7483              : 
    7484           21 :       rtx tmp5 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp4));
    7485           21 :       rtx tmp6 = gen_reg_rtx (V4SImode);
    7486           21 :       emit_insn (gen_sse2_pshufd (tmp6, tmp5, GEN_INT (0xfe)));
    7487              : 
    7488           21 :       emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp6));
    7489           21 :       return;
    7490              :     }
    7491              : 
    7492           76 :   if (TARGET_AVX2 || TARGET_SSE4_1)
    7493              :     {
    7494              :       /* Three operations.  */
    7495           50 :       if (bits == 32)
    7496              :         {
    7497            2 :           rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
    7498            2 :           rtx tmp2 = gen_reg_rtx (V4SImode);
    7499            2 :           emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (31)));
    7500              : 
    7501            2 :           rtx tmp3 = gen_reg_rtx (V1TImode);
    7502            2 :           emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (32)));
    7503              : 
    7504            2 :           if (TARGET_AVX2)
    7505              :             {
    7506            1 :               rtx tmp4 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp3));
    7507            1 :               rtx tmp5 = gen_reg_rtx (V4SImode);
    7508            1 :               emit_insn (gen_avx2_pblenddv4si (tmp5, tmp2, tmp4,
    7509              :                                                GEN_INT (7)));
    7510              : 
    7511            1 :               emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp5));
    7512              :             }
    7513              :           else
    7514              :             {
    7515            1 :               rtx tmp4 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp2));
    7516            1 :               rtx tmp5 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp3));
    7517            1 :               rtx tmp6 = gen_reg_rtx (V8HImode);
    7518            1 :               emit_insn (gen_sse4_1_pblendw (tmp6, tmp4, tmp5,
    7519              :                                              GEN_INT (0x3f)));
    7520              : 
    7521            1 :               emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp6));
    7522              :             }
    7523              :           return;
    7524              :         }
    7525              : 
    7526              :       /* Three operations.  */
    7527              :       if (bits == 8 || bits == 16 || bits == 24)
    7528              :         {
    7529            6 :           rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
    7530            6 :           rtx tmp2 = gen_reg_rtx (V4SImode);
    7531            6 :           emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits)));
    7532              : 
    7533            6 :           rtx tmp3 = gen_reg_rtx (V1TImode);
    7534            6 :           emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (bits)));
    7535              : 
    7536            6 :           if (TARGET_AVX2)
    7537              :             {
    7538            3 :               rtx tmp4 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp3));
    7539            3 :               rtx tmp5 = gen_reg_rtx (V4SImode);
    7540            3 :               emit_insn (gen_avx2_pblenddv4si (tmp5, tmp2, tmp4,
    7541              :                                                GEN_INT (7)));
    7542              : 
    7543            3 :               emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp5));
    7544              :             }
    7545              :           else
    7546              :             {
    7547            3 :               rtx tmp4 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp2));
    7548            3 :               rtx tmp5 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp3));
    7549            3 :               rtx tmp6 = gen_reg_rtx (V8HImode);
    7550            3 :               emit_insn (gen_sse4_1_pblendw (tmp6, tmp4, tmp5,
    7551              :                                              GEN_INT (0x3f)));
    7552              : 
    7553            3 :               emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp6));
    7554              :             }
    7555              :           return;
    7556              :         }
    7557              :     }
    7558              : 
    7559           68 :   if (bits > 96)
    7560              :     {
    7561              :       /* Four operations.  */
    7562            3 :       rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
    7563            3 :       rtx tmp2 = gen_reg_rtx (V4SImode);
    7564            3 :       emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits - 96)));
    7565              : 
    7566            3 :       rtx tmp3 = gen_reg_rtx (V4SImode);
    7567            3 :       emit_insn (gen_ashrv4si3 (tmp3, tmp1, GEN_INT (31)));
    7568              : 
    7569            3 :       rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp2));
    7570            3 :       rtx tmp5 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
    7571            3 :       rtx tmp6 = gen_reg_rtx (V2DImode);
    7572            3 :       emit_insn (gen_vec_interleave_highv2di (tmp6, tmp4, tmp5));
    7573              : 
    7574            3 :       rtx tmp7 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp6));
    7575            3 :       rtx tmp8 = gen_reg_rtx (V4SImode);
    7576            3 :       emit_insn (gen_sse2_pshufd (tmp8, tmp7, GEN_INT (0xfd)));
    7577              : 
    7578            3 :       emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp8));
    7579            3 :       return;
    7580              :     }
    7581              : 
    7582           65 :   if (TARGET_SSE4_1 && (bits == 48 || bits == 80))
    7583              :     {
    7584              :       /* Four operations.  */
    7585            4 :       rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
    7586            4 :       rtx tmp2 = gen_reg_rtx (V4SImode);
    7587            4 :       emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
    7588              : 
    7589            4 :       rtx tmp3 = gen_reg_rtx (V4SImode);
    7590            4 :       emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
    7591              : 
    7592            4 :       rtx tmp4 = gen_reg_rtx (V1TImode);
    7593            4 :       emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (bits)));
    7594              : 
    7595            4 :       rtx tmp5 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp3));
    7596            4 :       rtx tmp6 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp4));
    7597            4 :       rtx tmp7 = gen_reg_rtx (V8HImode);
    7598            6 :       emit_insn (gen_sse4_1_pblendw (tmp7, tmp5, tmp6,
    7599              :                                      GEN_INT (bits == 48 ? 0x1f : 0x07)));
    7600              : 
    7601            4 :       emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp7));
    7602            4 :       return;
    7603              :     }
    7604              : 
    7605           61 :   if ((bits & 7) == 0)
    7606              :     {
    7607              :       /* Five operations.  */
    7608            9 :       rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
    7609            9 :       rtx tmp2 = gen_reg_rtx (V4SImode);
    7610            9 :       emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
    7611              : 
    7612            9 :       rtx tmp3 = gen_reg_rtx (V4SImode);
    7613            9 :       emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
    7614              : 
    7615            9 :       rtx tmp4 = gen_reg_rtx (V1TImode);
    7616            9 :       emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (bits)));
    7617              : 
    7618            9 :       rtx tmp5 = force_reg (V1TImode, gen_lowpart (V1TImode, tmp3));
    7619            9 :       rtx tmp6 = gen_reg_rtx (V1TImode);
    7620            9 :       emit_insn (gen_sse2_ashlv1ti3 (tmp6, tmp5, GEN_INT (128 - bits)));
    7621              : 
    7622            9 :       rtx tmp7 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp4));
    7623            9 :       rtx tmp8 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp6));
    7624            9 :       rtx tmp9 = gen_reg_rtx (V2DImode);
    7625            9 :       emit_insn (gen_iorv2di3 (tmp9, tmp7, tmp8));
    7626              : 
    7627            9 :       emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp9));
    7628            9 :       return;
    7629              :     }
    7630              : 
    7631           52 :   if (TARGET_AVX2 && bits < 32)
    7632              :     {
    7633              :       /* Six operations.  */
    7634            9 :       rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
    7635            9 :       rtx tmp2 = gen_reg_rtx (V4SImode);
    7636            9 :       emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits)));
    7637              : 
    7638            9 :       rtx tmp3 = gen_reg_rtx (V1TImode);
    7639            9 :       emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (64)));
    7640              : 
    7641            9 :       rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
    7642            9 :       rtx tmp5 = gen_reg_rtx (V2DImode);
    7643            9 :       emit_insn (gen_lshrv2di3 (tmp5, tmp4, GEN_INT (bits)));
    7644              : 
    7645            9 :       rtx tmp6 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
    7646            9 :       rtx tmp7 = gen_reg_rtx (V2DImode);
    7647            9 :       emit_insn (gen_ashlv2di3 (tmp7, tmp6, GEN_INT (64 - bits)));
    7648              : 
    7649            9 :       rtx tmp8 = gen_reg_rtx (V2DImode);
    7650            9 :       emit_insn (gen_iorv2di3 (tmp8, tmp5, tmp7));
    7651              : 
    7652            9 :       rtx tmp9 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp8));
    7653            9 :       rtx tmp10 = gen_reg_rtx (V4SImode);
    7654            9 :       emit_insn (gen_avx2_pblenddv4si (tmp10, tmp2, tmp9, GEN_INT (7)));
    7655              : 
    7656            9 :       emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp10));
    7657            9 :       return;
    7658              :     }
    7659              : 
    7660           43 :   if (TARGET_SSE4_1 && bits < 15)
    7661              :     {
    7662              :       /* Six operations.  */
    7663            4 :       rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
    7664            4 :       rtx tmp2 = gen_reg_rtx (V4SImode);
    7665            4 :       emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits)));
    7666              : 
    7667            4 :       rtx tmp3 = gen_reg_rtx (V1TImode);
    7668            4 :       emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (64)));
    7669              : 
    7670            4 :       rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
    7671            4 :       rtx tmp5 = gen_reg_rtx (V2DImode);
    7672            4 :       emit_insn (gen_lshrv2di3 (tmp5, tmp4, GEN_INT (bits)));
    7673              : 
    7674            4 :       rtx tmp6 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
    7675            4 :       rtx tmp7 = gen_reg_rtx (V2DImode);
    7676            4 :       emit_insn (gen_ashlv2di3 (tmp7, tmp6, GEN_INT (64 - bits)));
    7677              : 
    7678            4 :       rtx tmp8 = gen_reg_rtx (V2DImode);
    7679            4 :       emit_insn (gen_iorv2di3 (tmp8, tmp5, tmp7));
    7680              : 
    7681            4 :       rtx tmp9 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp2));
    7682            4 :       rtx tmp10 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp8));
    7683            4 :       rtx tmp11 = gen_reg_rtx (V8HImode);
    7684            4 :       emit_insn (gen_sse4_1_pblendw (tmp11, tmp9, tmp10, GEN_INT (0x3f)));
    7685              : 
    7686            4 :       emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp11));
    7687            4 :       return;
    7688              :     }
    7689              : 
    7690           18 :   if (bits == 1)
    7691              :     {
    7692              :       /* Eight operations.  */
    7693            1 :       rtx tmp1 = gen_reg_rtx (V1TImode);
    7694            1 :       emit_insn (gen_sse2_lshrv1ti3 (tmp1, op1, GEN_INT (64)));
    7695              : 
    7696            1 :       rtx tmp2 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
    7697            1 :       rtx tmp3 = gen_reg_rtx (V2DImode);
    7698            1 :       emit_insn (gen_lshrv2di3 (tmp3, tmp2, GEN_INT (1)));
    7699              : 
    7700            1 :       rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp1));
    7701            1 :       rtx tmp5 = gen_reg_rtx (V2DImode);
    7702            1 :       emit_insn (gen_ashlv2di3 (tmp5, tmp4, GEN_INT (63)));
    7703              : 
    7704            1 :       rtx tmp6 = gen_reg_rtx (V2DImode);
    7705            1 :       emit_insn (gen_iorv2di3 (tmp6, tmp3, tmp5));
    7706              : 
    7707            1 :       rtx tmp7 = gen_reg_rtx (V2DImode);
    7708            1 :       emit_insn (gen_lshrv2di3 (tmp7, tmp2, GEN_INT (63)));
    7709              : 
    7710            1 :       rtx tmp8 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp7));
    7711            1 :       rtx tmp9 = gen_reg_rtx (V4SImode);
    7712            1 :       emit_insn (gen_sse2_pshufd (tmp9, tmp8, GEN_INT (0xbf)));
    7713              : 
    7714            1 :       rtx tmp10 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp9));
    7715            1 :       rtx tmp11 = gen_reg_rtx (V2DImode);
    7716            1 :       emit_insn (gen_ashlv2di3 (tmp11, tmp10, GEN_INT (31)));
    7717              : 
    7718            1 :       rtx tmp12 = gen_reg_rtx (V2DImode);
    7719            1 :       emit_insn (gen_iorv2di3 (tmp12, tmp6, tmp11));
    7720              : 
    7721            1 :       emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp12));
    7722            1 :       return;
    7723              :     }
    7724              : 
    7725           38 :   if (bits > 64)
    7726              :     {
    7727              :       /* Eight operations.  */
    7728           12 :       rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
    7729           12 :       rtx tmp2 = gen_reg_rtx (V4SImode);
    7730           12 :       emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
    7731              : 
    7732           12 :       rtx tmp3 = gen_reg_rtx (V4SImode);
    7733           12 :       emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
    7734              : 
    7735           12 :       rtx tmp4 = gen_reg_rtx (V1TImode);
    7736           12 :       emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (64)));
    7737              : 
    7738           12 :       rtx tmp5 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp4));
    7739           12 :       rtx tmp6 = gen_reg_rtx (V2DImode);
    7740           12 :       emit_insn (gen_lshrv2di3 (tmp6, tmp5, GEN_INT (bits - 64)));
    7741              : 
    7742           12 :       rtx tmp7 = force_reg (V1TImode, gen_lowpart (V1TImode, tmp3));
    7743           12 :       rtx tmp8 = gen_reg_rtx (V1TImode);
    7744           12 :       emit_insn (gen_sse2_ashlv1ti3 (tmp8, tmp7, GEN_INT (64)));
    7745              : 
    7746           12 :       rtx tmp9 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
    7747           12 :       rtx tmp10 = gen_reg_rtx (V2DImode);
    7748           12 :       emit_insn (gen_ashlv2di3 (tmp10, tmp9, GEN_INT (128 - bits)));
    7749              : 
    7750           12 :       rtx tmp11 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp8));
    7751           12 :       rtx tmp12 = gen_reg_rtx (V2DImode);
    7752           12 :       emit_insn (gen_iorv2di3 (tmp12, tmp10, tmp11));
    7753              : 
    7754           12 :       rtx tmp13 = gen_reg_rtx (V2DImode);
    7755           12 :       emit_insn (gen_iorv2di3 (tmp13, tmp6, tmp12));
    7756              : 
    7757           12 :       emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp13));
    7758              :     }
    7759              :   else
    7760              :     {
    7761              :       /* Nine operations.  */
    7762           26 :       rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
    7763           26 :       rtx tmp2 = gen_reg_rtx (V4SImode);
    7764           26 :       emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
    7765              : 
    7766           26 :       rtx tmp3 = gen_reg_rtx (V4SImode);
    7767           26 :       emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
    7768              : 
    7769           26 :       rtx tmp4 = gen_reg_rtx (V1TImode);
    7770           26 :       emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (64)));
    7771              : 
    7772           26 :       rtx tmp5 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
    7773           26 :       rtx tmp6 = gen_reg_rtx (V2DImode);
    7774           26 :       emit_insn (gen_lshrv2di3 (tmp6, tmp5, GEN_INT (bits)));
    7775              : 
    7776           26 :       rtx tmp7 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp4));
    7777           26 :       rtx tmp8 = gen_reg_rtx (V2DImode);
    7778           26 :       emit_insn (gen_ashlv2di3 (tmp8, tmp7, GEN_INT (64 - bits)));
    7779              : 
    7780           26 :       rtx tmp9 = gen_reg_rtx (V2DImode);
    7781           26 :       emit_insn (gen_iorv2di3 (tmp9, tmp6, tmp8));
    7782              : 
    7783           26 :       rtx tmp10 = force_reg (V1TImode, gen_lowpart (V1TImode, tmp3));
    7784           26 :       rtx tmp11 = gen_reg_rtx (V1TImode);
    7785           26 :       emit_insn (gen_sse2_ashlv1ti3 (tmp11, tmp10, GEN_INT (64)));
    7786              : 
    7787           26 :       rtx tmp12 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp11));
    7788           26 :       rtx tmp13 = gen_reg_rtx (V2DImode);
    7789           26 :       emit_insn (gen_ashlv2di3 (tmp13, tmp12, GEN_INT (64 - bits)));
    7790              : 
    7791           26 :       rtx tmp14 = gen_reg_rtx (V2DImode);
    7792           26 :       emit_insn (gen_iorv2di3 (tmp14, tmp9, tmp13));
    7793              : 
    7794           26 :       emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp14));
    7795              :     }
    7796              : }
    7797              : 
    7798              : /* Expand V2DI mode ashiftrt.  */
    7799              : void
    7800          519 : ix86_expand_v2di_ashiftrt (rtx operands[])
    7801              : {
    7802          519 :   if (operands[2] == const0_rtx)
    7803              :     {
    7804            0 :       emit_move_insn (operands[0], operands[1]);
    7805            0 :       return;
    7806              :     }
    7807              : 
    7808          519 :   if (TARGET_SSE4_2
    7809          144 :       && CONST_INT_P (operands[2])
    7810          144 :       && UINTVAL (operands[2]) >= 63
    7811          527 :       && !optimize_insn_for_size_p ())
    7812              :     {
    7813            8 :       rtx zero = force_reg (V2DImode, CONST0_RTX (V2DImode));
    7814            8 :       emit_insn (gen_sse4_2_gtv2di3 (operands[0], zero, operands[1]));
    7815            8 :       return;
    7816              :     }
    7817              : 
    7818          511 :   if (CONST_INT_P (operands[2])
    7819          485 :       && (!TARGET_XOP || UINTVAL (operands[2]) >= 63))
    7820              :     {
    7821          389 :       vec_perm_builder sel (4, 4, 1);
    7822          389 :       sel.quick_grow (4);
    7823          389 :       rtx arg0, arg1;
    7824          389 :       rtx op1 = lowpart_subreg (V4SImode,
    7825              :                                 force_reg (V2DImode, operands[1]),
    7826              :                                 V2DImode);
    7827          389 :       rtx target = gen_reg_rtx (V4SImode);
    7828          389 :       if (UINTVAL (operands[2]) >= 63)
    7829              :         {
    7830          132 :           arg0 = arg1 = gen_reg_rtx (V4SImode);
    7831          132 :           emit_insn (gen_ashrv4si3 (arg0, op1, GEN_INT (31)));
    7832          132 :           sel[0] = 1;
    7833          132 :           sel[1] = 1;
    7834          132 :           sel[2] = 3;
    7835          132 :           sel[3] = 3;
    7836              :         }
    7837          257 :       else if (INTVAL (operands[2]) > 32)
    7838              :         {
    7839           30 :           arg0 = gen_reg_rtx (V4SImode);
    7840           30 :           arg1 = gen_reg_rtx (V4SImode);
    7841           30 :           emit_insn (gen_ashrv4si3 (arg1, op1, GEN_INT (31)));
    7842           30 :           emit_insn (gen_ashrv4si3 (arg0, op1,
    7843           30 :                                     GEN_INT (INTVAL (operands[2]) - 32)));
    7844           30 :           sel[0] = 1;
    7845           30 :           sel[1] = 5;
    7846           30 :           sel[2] = 3;
    7847           30 :           sel[3] = 7;
    7848              :         }
    7849          227 :       else if (INTVAL (operands[2]) == 32)
    7850              :         {
    7851            5 :           arg0 = op1;
    7852            5 :           arg1 = gen_reg_rtx (V4SImode);
    7853            5 :           emit_insn (gen_ashrv4si3 (arg1, op1, GEN_INT (31)));
    7854            5 :           sel[0] = 1;
    7855            5 :           sel[1] = 5;
    7856            5 :           sel[2] = 3;
    7857            5 :           sel[3] = 7;
    7858              :         }
    7859              :       else
    7860              :         {
    7861          222 :           arg0 = gen_reg_rtx (V2DImode);
    7862          222 :           arg1 = gen_reg_rtx (V4SImode);
    7863          222 :           emit_insn (gen_lshrv2di3 (arg0, operands[1], operands[2]));
    7864          222 :           emit_insn (gen_ashrv4si3 (arg1, op1, operands[2]));
    7865          222 :           arg0 = lowpart_subreg (V4SImode, arg0, V2DImode);
    7866          222 :           sel[0] = 0;
    7867          222 :           sel[1] = 5;
    7868          222 :           sel[2] = 2;
    7869          222 :           sel[3] = 7;
    7870              :         }
    7871          521 :       vec_perm_indices indices (sel, arg0 != arg1 ? 2 : 1, 4);
    7872          389 :       rtx op0 = operands[0];
    7873          389 :       bool ok = targetm.vectorize.vec_perm_const (V4SImode, V4SImode,
    7874              :                                                   target, arg0, arg1,
    7875              :                                                   indices);
    7876          389 :       gcc_assert (ok);
    7877          389 :       emit_move_insn (op0, lowpart_subreg (V2DImode, target, V4SImode));
    7878          389 :       return;
    7879          389 :     }
    7880          122 :   if (!TARGET_XOP)
    7881              :     {
    7882           26 :       rtx zero = force_reg (V2DImode, CONST0_RTX (V2DImode));
    7883           26 :       rtx zero_or_all_ones;
    7884           26 :       if (TARGET_SSE4_2)
    7885              :         {
    7886            0 :           zero_or_all_ones = gen_reg_rtx (V2DImode);
    7887            0 :           emit_insn (gen_sse4_2_gtv2di3 (zero_or_all_ones, zero,
    7888              :                                          operands[1]));
    7889              :         }
    7890              :       else
    7891              :         {
    7892           26 :           rtx temp = gen_reg_rtx (V4SImode);
    7893           26 :           emit_insn (gen_ashrv4si3 (temp,
    7894              :                                     lowpart_subreg (V4SImode,
    7895              :                                                     force_reg (V2DImode,
    7896              :                                                                operands[1]),
    7897              :                                                     V2DImode),
    7898              :                                     GEN_INT (31)));
    7899           26 :           zero_or_all_ones = gen_reg_rtx (V4SImode);
    7900           26 :           emit_insn (gen_sse2_pshufd_1 (zero_or_all_ones, temp,
    7901              :                                         const1_rtx, const1_rtx,
    7902              :                                         GEN_INT (3), GEN_INT (3)));
    7903           26 :           zero_or_all_ones = lowpart_subreg (V2DImode, zero_or_all_ones,
    7904              :                                              V4SImode);
    7905              :         }
    7906           26 :       rtx lshr_res = gen_reg_rtx (V2DImode);
    7907           26 :       emit_insn (gen_lshrv2di3 (lshr_res, operands[1], operands[2]));
    7908           26 :       rtx ashl_res = gen_reg_rtx (V2DImode);
    7909           26 :       rtx amount;
    7910           26 :       if (TARGET_64BIT)
    7911              :         {
    7912           26 :           amount = gen_reg_rtx (DImode);
    7913           26 :           emit_insn (gen_subdi3 (amount, force_reg (DImode, GEN_INT (64)),
    7914              :                                  operands[2]));
    7915              :         }
    7916              :       else
    7917              :         {
    7918            0 :           rtx temp = gen_reg_rtx (SImode);
    7919            0 :           emit_insn (gen_subsi3 (temp, force_reg (SImode, GEN_INT (64)),
    7920              :                                  lowpart_subreg (SImode, operands[2],
    7921              :                                                  DImode)));
    7922            0 :           amount = gen_reg_rtx (V4SImode);
    7923            0 :           emit_insn (gen_vec_setv4si_0 (amount, CONST0_RTX (V4SImode),
    7924              :                                         temp));
    7925              :         }
    7926           26 :       amount = lowpart_subreg (DImode, amount, GET_MODE (amount));
    7927           26 :       emit_insn (gen_ashlv2di3 (ashl_res, zero_or_all_ones, amount));
    7928           26 :       emit_insn (gen_iorv2di3 (operands[0], lshr_res, ashl_res));
    7929           26 :       return;
    7930              :     }
    7931              : 
    7932           96 :   rtx reg = gen_reg_rtx (V2DImode);
    7933           96 :   rtx par;
    7934           96 :   bool negate = false;
    7935           96 :   int i;
    7936              : 
    7937           96 :   if (CONST_INT_P (operands[2]))
    7938           96 :     operands[2] = GEN_INT (-INTVAL (operands[2]));
    7939              :   else
    7940              :     negate = true;
    7941              : 
    7942           96 :   par = gen_rtx_PARALLEL (V2DImode, rtvec_alloc (2));
    7943          288 :   for (i = 0; i < 2; i++)
    7944          192 :     XVECEXP (par, 0, i) = operands[2];
    7945              : 
    7946           96 :   emit_insn (gen_vec_initv2didi (reg, par));
    7947              : 
    7948           96 :   if (negate)
    7949            0 :     emit_insn (gen_negv2di2 (reg, reg));
    7950              : 
    7951           96 :   emit_insn (gen_xop_shav2di3 (operands[0], operands[1], reg));
    7952              : }
    7953              : 
    7954              : /* Replace all occurrences of REG FROM with REG TO in X, including
    7955              :    occurrences with different modes.  */
    7956              : 
    7957              : rtx
    7958        43368 : ix86_replace_reg_with_reg (rtx x, rtx from, rtx to)
    7959              : {
    7960        43368 :   gcc_checking_assert (REG_P (from)
    7961              :                        && REG_P (to)
    7962              :                        && GET_MODE (from) == GET_MODE (to));
    7963        43368 :   if (!reg_overlap_mentioned_p (from, x))
    7964              :     return x;
    7965          144 :   rtx ret = copy_rtx (x);
    7966          144 :   subrtx_ptr_iterator::array_type array;
    7967          732 :   FOR_EACH_SUBRTX_PTR (iter, array, &ret, NONCONST)
    7968              :     {
    7969          588 :       rtx *loc = *iter;
    7970          588 :       x = *loc;
    7971          588 :       if (REG_P (x) && REGNO (x) == REGNO (from))
    7972              :         {
    7973          144 :           if (x == from)
    7974          144 :             *loc = to;
    7975              :           else
    7976              :             {
    7977            0 :               gcc_checking_assert (REG_NREGS (x) == 1);
    7978            0 :               *loc = gen_rtx_REG (GET_MODE (x), REGNO (to));
    7979              :             }
    7980              :         }
    7981              :     }
    7982          144 :   return ret;
    7983          144 : }
    7984              : 
    7985              : /* Return mode for the memcpy/memset loop counter.  Prefer SImode over
    7986              :    DImode for constant loop counts.  */
    7987              : 
    7988              : static machine_mode
    7989        45193 : counter_mode (rtx count_exp)
    7990              : {
    7991          709 :   if (GET_MODE (count_exp) != VOIDmode)
    7992        32908 :     return GET_MODE (count_exp);
    7993        12285 :   if (!CONST_INT_P (count_exp))
    7994            0 :     return Pmode;
    7995              :   if (TARGET_64BIT && (INTVAL (count_exp) & ~0xffffffff))
    7996              :     return DImode;
    7997              :   return SImode;
    7998              : }
    7999              : 
    8000              : /* When ISSETMEM is FALSE, output simple loop to move memory pointer to SRCPTR
    8001              :    to DESTPTR via chunks of MODE unrolled UNROLL times, overall size is COUNT
    8002              :    specified in bytes.  When ISSETMEM is TRUE, output the equivalent loop to set
    8003              :    memory by VALUE (supposed to be in MODE).
    8004              : 
    8005              :    The size is rounded down to whole number of chunk size moved at once.
    8006              :    SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info.  */
    8007              : 
    8008              : 
    8009              : static void
    8010         8218 : expand_set_or_cpymem_via_loop (rtx destmem, rtx srcmem,
    8011              :                                rtx destptr, rtx srcptr, rtx value,
    8012              :                                rtx count, machine_mode mode, int unroll,
    8013              :                                int expected_size, bool issetmem)
    8014              : {
    8015         8218 :   rtx_code_label *out_label = nullptr;
    8016         8218 :   rtx_code_label *top_label = nullptr;
    8017         8218 :   rtx iter, tmp;
    8018         8218 :   machine_mode iter_mode = counter_mode (count);
    8019         8218 :   int piece_size_n = GET_MODE_SIZE (mode) * unroll;
    8020         8218 :   rtx piece_size = GEN_INT (piece_size_n);
    8021        16436 :   rtx piece_size_mask = GEN_INT (~((GET_MODE_SIZE (mode) * unroll) - 1));
    8022         8218 :   rtx size;
    8023         8218 :   int i;
    8024         8218 :   int loop_count;
    8025              : 
    8026         8218 :   if (expected_size != -1 && CONST_INT_P (count))
    8027         7234 :     loop_count = INTVAL (count) / GET_MODE_SIZE (mode) / unroll;
    8028              :   else
    8029              :     loop_count = -1;
    8030              : 
    8031              :   /* Don't generate the loop if the loop count is 1.  */
    8032         7234 :   if (loop_count != 1)
    8033              :     {
    8034         8167 :       top_label = gen_label_rtx ();
    8035         8167 :       out_label = gen_label_rtx ();
    8036              :     }
    8037         8218 :   iter = gen_reg_rtx (iter_mode);
    8038              : 
    8039         8218 :   size = expand_simple_binop (iter_mode, AND, count, piece_size_mask,
    8040              :                               NULL, 1, OPTAB_DIRECT);
    8041              :   /* Those two should combine.  */
    8042         8218 :   if (piece_size == const1_rtx)
    8043              :     {
    8044          275 :       emit_cmp_and_jump_insns (size, const0_rtx, EQ, NULL_RTX, iter_mode,
    8045              :                                true, out_label);
    8046          275 :       predict_jump (REG_BR_PROB_BASE * 10 / 100);
    8047              :     }
    8048         8218 :   emit_move_insn (iter, const0_rtx);
    8049              : 
    8050         8218 :   if (loop_count != 1)
    8051         8167 :     emit_label (top_label);
    8052              : 
    8053        10940 :   tmp = convert_modes (Pmode, iter_mode, iter, true);
    8054              : 
    8055              :   /* This assert could be relaxed - in this case we'll need to compute
    8056              :      smallest power of two, containing in PIECE_SIZE_N and pass it to
    8057              :      offset_address.  */
    8058         8218 :   gcc_assert ((piece_size_n & (piece_size_n - 1)) == 0);
    8059         8218 :   destmem = offset_address (destmem, tmp, piece_size_n);
    8060         8218 :   destmem = adjust_address (destmem, mode, 0);
    8061              : 
    8062         8218 :   if (!issetmem)
    8063              :     {
    8064         2886 :       srcmem = offset_address (srcmem, copy_rtx (tmp), piece_size_n);
    8065         2886 :       srcmem = adjust_address (srcmem, mode, 0);
    8066              : 
    8067              :       /* When unrolling for chips that reorder memory reads and writes,
    8068              :          we can save registers by using single temporary.
    8069              :          Also using 4 temporaries is overkill in 32bit mode.  */
    8070         2886 :       if (!TARGET_64BIT && 0)
    8071              :         {
    8072              :           for (i = 0; i < unroll; i++)
    8073              :             {
    8074              :               if (i)
    8075              :                 {
    8076              :                   destmem = adjust_address (copy_rtx (destmem), mode,
    8077              :                                             GET_MODE_SIZE (mode));
    8078              :                   srcmem = adjust_address (copy_rtx (srcmem), mode,
    8079              :                                            GET_MODE_SIZE (mode));
    8080              :                 }
    8081              :               emit_move_insn (destmem, srcmem);
    8082              :             }
    8083              :         }
    8084              :       else
    8085              :         {
    8086         2886 :           rtx tmpreg[4];
    8087         2886 :           gcc_assert (unroll <= 4);
    8088        13628 :           for (i = 0; i < unroll; i++)
    8089              :             {
    8090        10742 :               tmpreg[i] = gen_reg_rtx (mode);
    8091        10742 :               if (i)
    8092        15712 :                 srcmem = adjust_address (copy_rtx (srcmem), mode,
    8093              :                                          GET_MODE_SIZE (mode));
    8094        10742 :               emit_move_insn (tmpreg[i], srcmem);
    8095              :             }
    8096        13628 :           for (i = 0; i < unroll; i++)
    8097              :             {
    8098        10742 :               if (i)
    8099        15712 :                 destmem = adjust_address (copy_rtx (destmem), mode,
    8100              :                                           GET_MODE_SIZE (mode));
    8101        10742 :               emit_move_insn (destmem, tmpreg[i]);
    8102              :             }
    8103              :         }
    8104              :     }
    8105              :   else
    8106        26501 :     for (i = 0; i < unroll; i++)
    8107              :       {
    8108        21169 :         if (i)
    8109        31674 :           destmem = adjust_address (copy_rtx (destmem), mode,
    8110              :                                     GET_MODE_SIZE (mode));
    8111        21169 :         emit_move_insn (destmem, value);
    8112              :       }
    8113              : 
    8114         8218 :   tmp = expand_simple_binop (iter_mode, PLUS, iter, piece_size, iter,
    8115              :                              true, OPTAB_LIB_WIDEN);
    8116         8218 :   if (tmp != iter)
    8117            0 :     emit_move_insn (iter, tmp);
    8118              : 
    8119         8218 :   if (loop_count != 1)
    8120              :     {
    8121         8167 :       emit_cmp_and_jump_insns (iter, size, LT, NULL_RTX, iter_mode,
    8122              :                                true, top_label);
    8123         8167 :       if (expected_size != -1)
    8124              :         {
    8125         7291 :           expected_size /= GET_MODE_SIZE (mode) * unroll;
    8126         7291 :           if (expected_size == 0)
    8127            0 :             predict_jump (0);
    8128         7291 :           else if (expected_size > REG_BR_PROB_BASE)
    8129            2 :             predict_jump (REG_BR_PROB_BASE - 1);
    8130              :           else
    8131         7289 :             predict_jump (REG_BR_PROB_BASE
    8132         7289 :                           - (REG_BR_PROB_BASE + expected_size / 2)
    8133         7289 :                             / expected_size);
    8134              :         }
    8135              :       else
    8136          876 :         predict_jump (REG_BR_PROB_BASE * 80 / 100);
    8137              :     }
    8138         8218 :   iter = ix86_zero_extend_to_Pmode (iter);
    8139        10940 :   tmp = expand_simple_binop (Pmode, PLUS, destptr, iter, destptr,
    8140              :                              true, OPTAB_LIB_WIDEN);
    8141         8218 :   if (tmp != destptr)
    8142            0 :     emit_move_insn (destptr, tmp);
    8143         8218 :   if (!issetmem)
    8144              :     {
    8145         4208 :       tmp = expand_simple_binop (Pmode, PLUS, srcptr, iter, srcptr,
    8146              :                                  true, OPTAB_LIB_WIDEN);
    8147         2886 :       if (tmp != srcptr)
    8148            0 :         emit_move_insn (srcptr, tmp);
    8149              :     }
    8150         8218 :   if (loop_count != 1)
    8151         8167 :     emit_label (out_label);
    8152         8218 : }
    8153              : 
    8154              : /* Divide COUNTREG by SCALE.  */
    8155              : static rtx
    8156        19366 : scale_counter (rtx countreg, int scale)
    8157              : {
    8158        19366 :   rtx sc;
    8159              : 
    8160        19366 :   if (scale == 1)
    8161              :     return countreg;
    8162        13351 :   if (CONST_INT_P (countreg))
    8163        13333 :     return GEN_INT (INTVAL (countreg) / scale);
    8164           18 :   gcc_assert (REG_P (countreg));
    8165              : 
    8166           54 :   sc = expand_simple_binop (GET_MODE (countreg), LSHIFTRT, countreg,
    8167           36 :                             GEN_INT (exact_log2 (scale)),
    8168              :                             NULL, 1, OPTAB_DIRECT);
    8169           18 :   return sc;
    8170              : }
    8171              : 
    8172              : /* Output "rep; mov" or "rep; stos" instruction depending on ISSETMEM argument.
    8173              :    When ISSETMEM is true, arguments SRCMEM and SRCPTR are ignored.
    8174              :    When ISSETMEM is false, arguments VALUE and ORIG_VALUE are ignored.
    8175              :    For setmem case, VALUE is a promoted to a wider size ORIG_VALUE.
    8176              :    ORIG_VALUE is the original value passed to memset to fill the memory with.
    8177              :    Other arguments have same meaning as for previous function.  */
    8178              : 
    8179              : static void
    8180        19366 : expand_set_or_cpymem_via_rep (rtx destmem, rtx srcmem,
    8181              :                            rtx destptr, rtx srcptr, rtx value, rtx orig_value,
    8182              :                            rtx count,
    8183              :                            machine_mode mode, bool issetmem)
    8184              : {
    8185        19366 :   rtx destexp;
    8186        19366 :   rtx srcexp;
    8187        19366 :   rtx countreg;
    8188        19366 :   HOST_WIDE_INT rounded_count;
    8189              : 
    8190              :   /* If possible, it is shorter to use rep movs.
    8191              :      TODO: Maybe it is better to move this logic to decide_alg.  */
    8192        19366 :   if (mode == QImode && CONST_INT_P (count) && !(INTVAL (count) & 3)
    8193          276 :       && !TARGET_PREFER_KNOWN_REP_MOVSB_STOSB
    8194          272 :       && (!issetmem || orig_value == const0_rtx))
    8195        19366 :     mode = SImode;
    8196              : 
    8197        19366 :   if (destptr != XEXP (destmem, 0) || GET_MODE (destmem) != BLKmode)
    8198        19087 :     destmem = adjust_automodify_address_nv (destmem, BLKmode, destptr, 0);
    8199              : 
    8200        38732 :   countreg = ix86_zero_extend_to_Pmode (scale_counter (count,
    8201        19366 :                                                        GET_MODE_SIZE (mode)));
    8202        19366 :   if (mode != QImode)
    8203              :     {
    8204        40340 :       destexp = gen_rtx_ASHIFT (Pmode, countreg,
    8205              :                                 GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
    8206        13638 :       destexp = gen_rtx_PLUS (Pmode, destexp, destptr);
    8207              :     }
    8208              :   else
    8209         6033 :     destexp = gen_rtx_PLUS (Pmode, destptr, countreg);
    8210        19366 :   if ((!issetmem || orig_value == const0_rtx) && CONST_INT_P (count))
    8211              :     {
    8212        14254 :       rounded_count
    8213        14254 :         = ROUND_DOWN (INTVAL (count), (HOST_WIDE_INT) GET_MODE_SIZE (mode));
    8214        14254 :       destmem = shallow_copy_rtx (destmem);
    8215        14254 :       set_mem_size (destmem, rounded_count);
    8216              :     }
    8217         5120 :   else if (MEM_SIZE_KNOWN_P (destmem))
    8218          386 :     clear_mem_size (destmem);
    8219              : 
    8220        19366 :   if (issetmem)
    8221              :     {
    8222         8230 :       value = force_reg (mode, gen_lowpart (mode, value));
    8223         8230 :       emit_insn (gen_rep_stos (destptr, countreg, destmem, value, destexp));
    8224              :     }
    8225              :   else
    8226              :     {
    8227        11136 :       if (srcptr != XEXP (srcmem, 0) || GET_MODE (srcmem) != BLKmode)
    8228        10919 :         srcmem = adjust_automodify_address_nv (srcmem, BLKmode, srcptr, 0);
    8229        11136 :       if (mode != QImode)
    8230              :         {
    8231        20391 :           srcexp = gen_rtx_ASHIFT (Pmode, countreg,
    8232              :                                    GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
    8233         6937 :           srcexp = gen_rtx_PLUS (Pmode, srcexp, srcptr);
    8234              :         }
    8235              :       else
    8236         4423 :         srcexp = gen_rtx_PLUS (Pmode, srcptr, countreg);
    8237        11136 :       if (CONST_INT_P (count))
    8238              :         {
    8239         7230 :           rounded_count
    8240         7230 :             = ROUND_DOWN (INTVAL (count), (HOST_WIDE_INT) GET_MODE_SIZE (mode));
    8241         7230 :           srcmem = shallow_copy_rtx (srcmem);
    8242         7230 :           set_mem_size (srcmem, rounded_count);
    8243              :         }
    8244              :       else
    8245              :         {
    8246         3921 :           if (MEM_SIZE_KNOWN_P (srcmem))
    8247            0 :             clear_mem_size (srcmem);
    8248              :         }
    8249        11136 :       emit_insn (gen_rep_mov (destptr, destmem, srcptr, srcmem, countreg,
    8250              :                               destexp, srcexp));
    8251              :     }
    8252        19366 : }
    8253              : 
    8254              : /* This function emits moves to copy SIZE_TO_MOVE bytes from SRCMEM to
    8255              :    DESTMEM.
    8256              :    SRC is passed by pointer to be updated on return.
    8257              :    Return value is updated DST.  */
    8258              : static rtx
    8259           13 : emit_memmov (rtx destmem, rtx *srcmem, rtx destptr, rtx srcptr,
    8260              :              HOST_WIDE_INT size_to_move)
    8261              : {
    8262           13 :   rtx dst = destmem, src = *srcmem, tempreg;
    8263           13 :   enum insn_code code;
    8264           13 :   machine_mode move_mode;
    8265           13 :   int piece_size, i;
    8266              : 
    8267              :   /* Find the widest mode in which we could perform moves.
    8268              :      Start with the biggest power of 2 less than SIZE_TO_MOVE and half
    8269              :      it until move of such size is supported.  */
    8270           13 :   piece_size = 1 << floor_log2 (size_to_move);
    8271           26 :   while (!int_mode_for_size (piece_size * BITS_PER_UNIT, 0).exists (&move_mode)
    8272           26 :          || (code = optab_handler (mov_optab, move_mode)) == CODE_FOR_nothing)
    8273              :     {
    8274            0 :       gcc_assert (piece_size > 1);
    8275            0 :       piece_size >>= 1;
    8276              :     }
    8277              : 
    8278              :   /* Find the corresponding vector mode with the same size as MOVE_MODE.
    8279              :      MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.).  */
    8280           26 :   if (GET_MODE_SIZE (move_mode) > UNITS_PER_WORD)
    8281              :     {
    8282            0 :       int nunits = GET_MODE_SIZE (move_mode) / UNITS_PER_WORD;
    8283            0 :       if (!mode_for_vector (word_mode, nunits).exists (&move_mode)
    8284            0 :           || (code = optab_handler (mov_optab, move_mode)) == CODE_FOR_nothing)
    8285              :         {
    8286            0 :           move_mode = word_mode;
    8287            0 :           piece_size = GET_MODE_SIZE (move_mode);
    8288            0 :           code = optab_handler (mov_optab, move_mode);
    8289              :         }
    8290              :     }
    8291           13 :   gcc_assert (code != CODE_FOR_nothing);
    8292              : 
    8293           13 :   dst = adjust_automodify_address_nv (dst, move_mode, destptr, 0);
    8294           13 :   src = adjust_automodify_address_nv (src, move_mode, srcptr, 0);
    8295              : 
    8296              :   /* Emit moves.  We'll need SIZE_TO_MOVE/PIECE_SIZES moves.  */
    8297           13 :   gcc_assert (size_to_move % piece_size == 0);
    8298              : 
    8299           26 :   for (i = 0; i < size_to_move; i += piece_size)
    8300              :     {
    8301              :       /* We move from memory to memory, so we'll need to do it via
    8302              :          a temporary register.  */
    8303           13 :       tempreg = gen_reg_rtx (move_mode);
    8304           13 :       emit_insn (GEN_FCN (code) (tempreg, src));
    8305           13 :       emit_insn (GEN_FCN (code) (dst, tempreg));
    8306              : 
    8307           26 :       emit_move_insn (destptr,
    8308           13 :                       plus_constant (Pmode, copy_rtx (destptr), piece_size));
    8309           26 :       emit_move_insn (srcptr,
    8310           13 :                       plus_constant (Pmode, copy_rtx (srcptr), piece_size));
    8311              : 
    8312           13 :       dst = adjust_automodify_address_nv (dst, move_mode, destptr,
    8313              :                                           piece_size);
    8314           13 :       src = adjust_automodify_address_nv (src, move_mode, srcptr,
    8315              :                                           piece_size);
    8316              :     }
    8317              : 
    8318              :   /* Update DST and SRC rtx.  */
    8319           13 :   *srcmem = src;
    8320           13 :   return dst;
    8321              : }
    8322              : 
    8323              : /* Helper function for the string operations below.  Dest VARIABLE whether
    8324              :    it is aligned to VALUE bytes.  If true, jump to the label.  */
    8325              : 
    8326              : static rtx_code_label *
    8327         3254 : ix86_expand_aligntest (rtx variable, int value, bool epilogue)
    8328              : {
    8329         3254 :   rtx_code_label *label = gen_label_rtx ();
    8330         3254 :   rtx tmpcount = gen_reg_rtx (GET_MODE (variable));
    8331         3254 :   if (GET_MODE (variable) == DImode)
    8332          684 :     emit_insn (gen_anddi3 (tmpcount, variable, GEN_INT (value)));
    8333              :   else
    8334         2570 :     emit_insn (gen_andsi3 (tmpcount, variable, GEN_INT (value)));
    8335         3254 :   emit_cmp_and_jump_insns (tmpcount, const0_rtx, EQ, 0, GET_MODE (variable),
    8336              :                            1, label);
    8337         3254 :   if (epilogue)
    8338            0 :     predict_jump (REG_BR_PROB_BASE * 50 / 100);
    8339              :   else
    8340         3254 :     predict_jump (REG_BR_PROB_BASE * 90 / 100);
    8341         3254 :   return label;
    8342              : }
    8343              : 
    8344              : 
    8345              : /* Output code to copy at most count & (max_size - 1) bytes from SRC to DEST.  */
    8346              : 
    8347              : static void
    8348         7289 : expand_cpymem_epilogue (rtx destmem, rtx srcmem,
    8349              :                         rtx destptr, rtx srcptr, rtx count, int max_size)
    8350              : {
    8351         7289 :   rtx src, dest;
    8352         7289 :   if (CONST_INT_P (count))
    8353              :     {
    8354         7224 :       unsigned HOST_WIDE_INT countval = UINTVAL (count);
    8355         7224 :       unsigned HOST_WIDE_INT epilogue_size = countval % max_size;
    8356         7224 :       unsigned int destalign = MEM_ALIGN (destmem);
    8357         7224 :       cfun->machine->by_pieces_in_use = true;
    8358         7224 :       move_by_pieces (destmem, srcmem, epilogue_size, destalign,
    8359              :                       RETURN_BEGIN);
    8360         7224 :       cfun->machine->by_pieces_in_use = false;
    8361         7224 :       return;
    8362              :     }
    8363           65 :   if (max_size > 8)
    8364              :     {
    8365           65 :       count = expand_simple_binop (GET_MODE (count), AND, count, GEN_INT (max_size - 1),
    8366              :                                     count, 1, OPTAB_DIRECT);
    8367           65 :       expand_set_or_cpymem_via_loop (destmem, srcmem, destptr, srcptr, NULL,
    8368              :                                      count, QImode, 1, 4, false);
    8369           65 :       return;
    8370              :     }
    8371              : 
    8372              :   /* When there are stringops, we can cheaply increase dest and src pointers.
    8373              :      Otherwise we save code size by maintaining offset (zero is readily
    8374              :      available from preceding rep operation) and using x86 addressing modes.
    8375              :    */
    8376            0 :   if (TARGET_SINGLE_STRINGOP)
    8377              :     {
    8378            0 :       if (max_size > 4)
    8379              :         {
    8380            0 :           rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
    8381            0 :           src = change_address (srcmem, SImode, srcptr);
    8382            0 :           dest = change_address (destmem, SImode, destptr);
    8383            0 :           emit_insn (gen_strmov (destptr, dest, srcptr, src));
    8384            0 :           emit_label (label);
    8385            0 :           LABEL_NUSES (label) = 1;
    8386              :         }
    8387            0 :       if (max_size > 2)
    8388              :         {
    8389            0 :           rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
    8390            0 :           src = change_address (srcmem, HImode, srcptr);
    8391            0 :           dest = change_address (destmem, HImode, destptr);
    8392            0 :           emit_insn (gen_strmov (destptr, dest, srcptr, src));
    8393            0 :           emit_label (label);
    8394            0 :           LABEL_NUSES (label) = 1;
    8395              :         }
    8396            0 :       if (max_size > 1)
    8397              :         {
    8398            0 :           rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
    8399            0 :           src = change_address (srcmem, QImode, srcptr);
    8400            0 :           dest = change_address (destmem, QImode, destptr);
    8401            0 :           emit_insn (gen_strmov (destptr, dest, srcptr, src));
    8402            0 :           emit_label (label);
    8403            0 :           LABEL_NUSES (label) = 1;
    8404              :         }
    8405              :     }
    8406              :   else
    8407              :     {
    8408            0 :       rtx offset = force_reg (Pmode, const0_rtx);
    8409            0 :       rtx tmp;
    8410              : 
    8411            0 :       if (max_size > 4)
    8412              :         {
    8413            0 :           rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
    8414            0 :           src = change_address (srcmem, SImode, srcptr);
    8415            0 :           dest = change_address (destmem, SImode, destptr);
    8416            0 :           emit_move_insn (dest, src);
    8417            0 :           tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (4), NULL,
    8418              :                                      true, OPTAB_LIB_WIDEN);
    8419            0 :           if (tmp != offset)
    8420            0 :             emit_move_insn (offset, tmp);
    8421            0 :           emit_label (label);
    8422            0 :           LABEL_NUSES (label) = 1;
    8423              :         }
    8424            0 :       if (max_size > 2)
    8425              :         {
    8426            0 :           rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
    8427            0 :           tmp = gen_rtx_PLUS (Pmode, srcptr, offset);
    8428            0 :           src = change_address (srcmem, HImode, tmp);
    8429            0 :           tmp = gen_rtx_PLUS (Pmode, destptr, offset);
    8430            0 :           dest = change_address (destmem, HImode, tmp);
    8431            0 :           emit_move_insn (dest, src);
    8432            0 :           tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (2), tmp,
    8433              :                                      true, OPTAB_LIB_WIDEN);
    8434            0 :           if (tmp != offset)
    8435            0 :             emit_move_insn (offset, tmp);
    8436            0 :           emit_label (label);
    8437            0 :           LABEL_NUSES (label) = 1;
    8438              :         }
    8439            0 :       if (max_size > 1)
    8440              :         {
    8441            0 :           rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
    8442            0 :           tmp = gen_rtx_PLUS (Pmode, srcptr, offset);
    8443            0 :           src = change_address (srcmem, QImode, tmp);
    8444            0 :           tmp = gen_rtx_PLUS (Pmode, destptr, offset);
    8445            0 :           dest = change_address (destmem, QImode, tmp);
    8446            0 :           emit_move_insn (dest, src);
    8447            0 :           emit_label (label);
    8448            0 :           LABEL_NUSES (label) = 1;
    8449              :         }
    8450              :     }
    8451              : }
    8452              : 
    8453              : /* This function emits moves to fill SIZE_TO_MOVE bytes starting from DESTMEM
    8454              :    with value PROMOTED_VAL.
    8455              :    SRC is passed by pointer to be updated on return.
    8456              :    Return value is updated DST.  */
    8457              : static rtx
    8458            6 : emit_memset (rtx destmem, rtx destptr, rtx promoted_val,
    8459              :              HOST_WIDE_INT size_to_move)
    8460              : {
    8461            6 :   rtx dst = destmem;
    8462            6 :   enum insn_code code;
    8463            6 :   machine_mode move_mode;
    8464            6 :   int piece_size, i;
    8465              : 
    8466              :   /* Find the widest mode in which we could perform moves.
    8467              :      Start with the biggest power of 2 less than SIZE_TO_MOVE and half
    8468              :      it until move of such size is supported.  */
    8469            6 :   move_mode = GET_MODE (promoted_val);
    8470            6 :   if (move_mode == VOIDmode)
    8471            0 :     move_mode = QImode;
    8472           12 :   if (size_to_move < GET_MODE_SIZE (move_mode))
    8473              :     {
    8474            5 :       unsigned int move_bits = size_to_move * BITS_PER_UNIT;
    8475            5 :       move_mode = int_mode_for_size (move_bits, 0).require ();
    8476            5 :       promoted_val = gen_lowpart (move_mode, promoted_val);
    8477              :     }
    8478            6 :   piece_size = GET_MODE_SIZE (move_mode);
    8479            6 :   code = optab_handler (mov_optab, move_mode);
    8480            6 :   gcc_assert (code != CODE_FOR_nothing && promoted_val != NULL_RTX);
    8481              : 
    8482            6 :   dst = adjust_automodify_address_nv (dst, move_mode, destptr, 0);
    8483              : 
    8484              :   /* Emit moves.  We'll need SIZE_TO_MOVE/PIECE_SIZES moves.  */
    8485            6 :   gcc_assert (size_to_move % piece_size == 0);
    8486              : 
    8487           12 :   for (i = 0; i < size_to_move; i += piece_size)
    8488              :     {
    8489            6 :       if (piece_size <= UNITS_PER_WORD)
    8490              :         {
    8491            4 :           emit_insn (gen_strset (destptr, dst, promoted_val));
    8492            4 :           dst = adjust_automodify_address_nv (dst, move_mode, destptr,
    8493              :                                               piece_size);
    8494            4 :           continue;
    8495              :         }
    8496              : 
    8497            2 :       emit_insn (GEN_FCN (code) (dst, promoted_val));
    8498              : 
    8499            4 :       emit_move_insn (destptr,
    8500            2 :                       plus_constant (Pmode, copy_rtx (destptr), piece_size));
    8501              : 
    8502            2 :       dst = adjust_automodify_address_nv (dst, move_mode, destptr,
    8503              :                                           piece_size);
    8504              :     }
    8505              : 
    8506              :   /* Update DST rtx.  */
    8507            6 :   return dst;
    8508              : }
    8509              : /* Output code to set at most count & (max_size - 1) bytes starting by DEST.  */
    8510              : static void
    8511           37 : expand_setmem_epilogue_via_loop (rtx destmem, rtx destptr, rtx value,
    8512              :                                  rtx count, int max_size)
    8513              : {
    8514           74 :   count = expand_simple_binop (counter_mode (count), AND, count,
    8515           37 :                                GEN_INT (max_size - 1), count, 1, OPTAB_DIRECT);
    8516           37 :   expand_set_or_cpymem_via_loop (destmem, NULL, destptr, NULL,
    8517           37 :                                  gen_lowpart (QImode, value), count, QImode,
    8518              :                                  1, max_size / 2, true);
    8519           37 : }
    8520              : 
    8521              : /* Callback routine for store_by_pieces.  Return the RTL of a register
    8522              :    containing GET_MODE_SIZE (MODE) bytes in the RTL register op_p which
    8523              :    is an integer or a word vector register.  If PREV_P isn't nullptr,
    8524              :    it has the RTL info from the previous iteration.  */
    8525              : 
    8526              : static rtx
    8527         4968 : setmem_epilogue_gen_val (void *op_p, void *prev_p, HOST_WIDE_INT,
    8528              :                          fixed_size_mode mode)
    8529              : {
    8530         4968 :   rtx target;
    8531         4968 :   by_pieces_prev *prev = (by_pieces_prev *) prev_p;
    8532         4968 :   if (prev)
    8533              :     {
    8534         4968 :       rtx prev_op = prev->data;
    8535         4968 :       if (prev_op)
    8536              :         {
    8537         2880 :           machine_mode prev_mode = GET_MODE (prev_op);
    8538         2880 :           if (prev_mode == mode)
    8539              :             return prev_op;
    8540           54 :           if (VECTOR_MODE_P (prev_mode)
    8541         1077 :               && VECTOR_MODE_P (mode)
    8542         1131 :               && GET_MODE_INNER (prev_mode) == GET_MODE_INNER (mode))
    8543              :             {
    8544            0 :               target = gen_rtx_SUBREG (mode, prev_op, 0);
    8545            0 :               return target;
    8546              :             }
    8547              :         }
    8548              :     }
    8549              : 
    8550         3219 :   rtx op = (rtx) op_p;
    8551         3219 :   machine_mode op_mode = GET_MODE (op);
    8552              : 
    8553         3219 :   if (VECTOR_MODE_P (mode))
    8554              :     {
    8555         3644 :       gcc_assert (GET_MODE_INNER (mode) == QImode);
    8556              : 
    8557         1822 :       unsigned int op_size = GET_MODE_SIZE (op_mode);
    8558         1822 :       unsigned int size = GET_MODE_SIZE (mode);
    8559         1822 :       unsigned int nunits;
    8560         1822 :       machine_mode vec_mode;
    8561         1822 :       if (op_size < size)
    8562              :         {
    8563              :           /* If OP size is smaller than MODE size, duplicate it.  */
    8564            1 :           nunits = size / GET_MODE_SIZE (QImode);
    8565            1 :           vec_mode = mode_for_vector (QImode, nunits).require ();
    8566            1 :           nunits = size / op_size;
    8567            1 :           gcc_assert (SCALAR_INT_MODE_P (op_mode));
    8568            1 :           machine_mode dup_mode
    8569            1 :             = mode_for_vector (as_a <scalar_mode> (op_mode),
    8570            2 :                                nunits).require ();
    8571            1 :           target = gen_reg_rtx (vec_mode);
    8572            1 :           op = gen_vec_duplicate (dup_mode, op);
    8573            1 :           rtx dup_op = gen_reg_rtx (dup_mode);
    8574            1 :           emit_move_insn (dup_op, op);
    8575            1 :           op = gen_rtx_SUBREG (vec_mode, dup_op, 0);
    8576            1 :           emit_move_insn (target, op);
    8577            1 :           return target;
    8578              :         }
    8579         1821 :       nunits = op_size / GET_MODE_SIZE (QImode);
    8580         1821 :       vec_mode = mode_for_vector (QImode, nunits).require ();
    8581         1821 :       target = gen_reg_rtx (vec_mode);
    8582         1821 :       op = gen_rtx_SUBREG (vec_mode, op, 0);
    8583         1821 :       emit_move_insn (target, op);
    8584         1821 :       if (op_size == size)
    8585              :         return target;
    8586              : 
    8587            0 :       rtx tmp = gen_reg_rtx (mode);
    8588            0 :       target = gen_rtx_SUBREG (mode, target, 0);
    8589            0 :       emit_move_insn (tmp, target);
    8590            0 :       return tmp;
    8591              :     }
    8592              : 
    8593         1397 :   if (VECTOR_MODE_P (op_mode))
    8594              :     {
    8595         2784 :       gcc_assert (GET_MODE_INNER (op_mode) == word_mode);
    8596         1392 :       target = gen_reg_rtx (word_mode);
    8597         1392 :       op = gen_rtx_SUBREG (word_mode, op, 0);
    8598         1392 :       emit_move_insn (target, op);
    8599              :     }
    8600              :   else
    8601              :     target = op;
    8602              : 
    8603         1397 :   if (mode == GET_MODE (target))
    8604              :     return target;
    8605              : 
    8606          246 :   rtx tmp = gen_reg_rtx (mode);
    8607          246 :   target = gen_rtx_SUBREG (mode, target, 0);
    8608          246 :   emit_move_insn (tmp, target);
    8609          246 :   return tmp;
    8610              : }
    8611              : 
    8612              : /* Output code to set at most count & (max_size - 1) bytes starting by DEST.  */
    8613              : static void
    8614        10043 : expand_setmem_epilogue (rtx destmem, rtx destptr, rtx value, rtx vec_value,
    8615              :                         rtx count, int max_size)
    8616              : {
    8617        10043 :   rtx dest;
    8618              : 
    8619        10043 :   if (CONST_INT_P (count))
    8620              :     {
    8621        10006 :       unsigned HOST_WIDE_INT countval = UINTVAL (count);
    8622        10006 :       unsigned HOST_WIDE_INT epilogue_size = countval % max_size;
    8623        10006 :       unsigned int destalign = MEM_ALIGN (destmem);
    8624        10006 :       cfun->machine->by_pieces_in_use = true;
    8625        16581 :       store_by_pieces (destmem, epilogue_size, setmem_epilogue_gen_val,
    8626              :                        vec_value ? vec_value : value, destalign, true,
    8627              :                        RETURN_BEGIN);
    8628        10006 :       cfun->machine->by_pieces_in_use = false;
    8629        10006 :       return;
    8630              :     }
    8631           37 :   if (max_size > 32)
    8632              :     {
    8633           37 :       expand_setmem_epilogue_via_loop (destmem, destptr, value, count, max_size);
    8634           37 :       return;
    8635              :     }
    8636            0 :   if (max_size > 16)
    8637              :     {
    8638            0 :       rtx_code_label *label = ix86_expand_aligntest (count, 16, true);
    8639            0 :       if (TARGET_64BIT)
    8640              :         {
    8641            0 :           dest = change_address (destmem, DImode, destptr);
    8642            0 :           emit_insn (gen_strset (destptr, dest, value));
    8643            0 :           dest = adjust_automodify_address_nv (dest, DImode, destptr, 8);
    8644            0 :           emit_insn (gen_strset (destptr, dest, value));
    8645              :         }
    8646              :       else
    8647              :         {
    8648            0 :           dest = change_address (destmem, SImode, destptr);
    8649            0 :           emit_insn (gen_strset (destptr, dest, value));
    8650            0 :           dest = adjust_automodify_address_nv (dest, SImode, destptr, 4);
    8651            0 :           emit_insn (gen_strset (destptr, dest, value));
    8652            0 :           dest = adjust_automodify_address_nv (dest, SImode, destptr, 8);
    8653            0 :           emit_insn (gen_strset (destptr, dest, value));
    8654            0 :           dest = adjust_automodify_address_nv (dest, SImode, destptr, 12);
    8655            0 :           emit_insn (gen_strset (destptr, dest, value));
    8656              :         }
    8657            0 :       emit_label (label);
    8658            0 :       LABEL_NUSES (label) = 1;
    8659              :     }
    8660            0 :   if (max_size > 8)
    8661              :     {
    8662            0 :       rtx_code_label *label = ix86_expand_aligntest (count, 8, true);
    8663            0 :       if (TARGET_64BIT)
    8664              :         {
    8665            0 :           dest = change_address (destmem, DImode, destptr);
    8666            0 :           emit_insn (gen_strset (destptr, dest, value));
    8667              :         }
    8668              :       else
    8669              :         {
    8670            0 :           dest = change_address (destmem, SImode, destptr);
    8671            0 :           emit_insn (gen_strset (destptr, dest, value));
    8672            0 :           dest = adjust_automodify_address_nv (dest, SImode, destptr, 4);
    8673            0 :           emit_insn (gen_strset (destptr, dest, value));
    8674              :         }
    8675            0 :       emit_label (label);
    8676            0 :       LABEL_NUSES (label) = 1;
    8677              :     }
    8678            0 :   if (max_size > 4)
    8679              :     {
    8680            0 :       rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
    8681            0 :       dest = change_address (destmem, SImode, destptr);
    8682            0 :       emit_insn (gen_strset (destptr, dest, gen_lowpart (SImode, value)));
    8683            0 :       emit_label (label);
    8684            0 :       LABEL_NUSES (label) = 1;
    8685              :     }
    8686            0 :   if (max_size > 2)
    8687              :     {
    8688            0 :       rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
    8689            0 :       dest = change_address (destmem, HImode, destptr);
    8690            0 :       emit_insn (gen_strset (destptr, dest, gen_lowpart (HImode, value)));
    8691            0 :       emit_label (label);
    8692            0 :       LABEL_NUSES (label) = 1;
    8693              :     }
    8694            0 :   if (max_size > 1)
    8695              :     {
    8696            0 :       rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
    8697            0 :       dest = change_address (destmem, QImode, destptr);
    8698            0 :       emit_insn (gen_strset (destptr, dest, gen_lowpart (QImode, value)));
    8699            0 :       emit_label (label);
    8700            0 :       LABEL_NUSES (label) = 1;
    8701              :     }
    8702              : }
    8703              : 
    8704              : /* Adjust COUNTER by the VALUE.  */
    8705              : static void
    8706           19 : ix86_adjust_counter (rtx countreg, HOST_WIDE_INT value)
    8707              : {
    8708           19 :   emit_insn (gen_add2_insn (countreg, GEN_INT (-value)));
    8709           19 : }
    8710              : 
    8711              : /* Depending on ISSETMEM, copy enough from SRCMEM to DESTMEM or set enough to
    8712              :    DESTMEM to align it to DESIRED_ALIGNMENT.  Original alignment is ALIGN.
    8713              :    Depending on ISSETMEM, either arguments SRCMEM/SRCPTR or VALUE/VEC_VALUE are
    8714              :    ignored.
    8715              :    Return value is updated DESTMEM.  */
    8716              : 
    8717              : static rtx
    8718            7 : expand_set_or_cpymem_prologue (rtx destmem, rtx srcmem,
    8719              :                                   rtx destptr, rtx srcptr, rtx value,
    8720              :                                   rtx vec_value, rtx count, int align,
    8721              :                                   int desired_alignment, bool issetmem)
    8722              : {
    8723            7 :   int i;
    8724           35 :   for (i = 1; i < desired_alignment; i <<= 1)
    8725              :     {
    8726           28 :       if (align <= i)
    8727              :         {
    8728           19 :           rtx_code_label *label = ix86_expand_aligntest (destptr, i, false);
    8729           19 :           if (issetmem)
    8730              :             {
    8731           12 :               if (vec_value && i > GET_MODE_SIZE (GET_MODE (value)))
    8732            2 :                 destmem = emit_memset (destmem, destptr, vec_value, i);
    8733              :               else
    8734            4 :                 destmem = emit_memset (destmem, destptr, value, i);
    8735              :             }
    8736              :           else
    8737           13 :             destmem = emit_memmov (destmem, &srcmem, destptr, srcptr, i);
    8738           19 :           ix86_adjust_counter (count, i);
    8739           19 :           emit_label (label);
    8740           19 :           LABEL_NUSES (label) = 1;
    8741           19 :           set_mem_align (destmem, i * 2 * BITS_PER_UNIT);
    8742              :         }
    8743              :     }
    8744            7 :   return destmem;
    8745              : }
    8746              : 
    8747              : /* Test if COUNT&SIZE is nonzero and if so, expand movme
    8748              :    or setmem sequence that is valid for SIZE..2*SIZE-1 bytes
    8749              :    and jump to DONE_LABEL.  */
    8750              : static void
    8751         2588 : expand_small_cpymem_or_setmem (rtx destmem, rtx srcmem,
    8752              :                                rtx destptr, rtx srcptr,
    8753              :                                rtx value, rtx vec_value,
    8754              :                                rtx count, int size,
    8755              :                                rtx done_label, bool issetmem)
    8756              : {
    8757         2588 :   rtx_code_label *label = ix86_expand_aligntest (count, size, false);
    8758         2588 :   machine_mode mode = int_mode_for_size (size * BITS_PER_UNIT, 1).else_blk ();
    8759         2588 :   rtx modesize;
    8760         2588 :   rtx scalar_value = value;
    8761         2588 :   int n;
    8762              : 
    8763              :   /* If we do not have vector value to copy, we must reduce size.  */
    8764         2588 :   if (issetmem)
    8765              :     {
    8766          871 :       if (!vec_value)
    8767              :         {
    8768            2 :           if (GET_MODE (value) == VOIDmode && size > 8)
    8769            0 :             mode = Pmode;
    8770            6 :           else if (GET_MODE_SIZE (mode) > GET_MODE_SIZE (GET_MODE (value)))
    8771            0 :             mode = GET_MODE (value);
    8772              :         }
    8773              :       else
    8774          869 :         mode = GET_MODE (vec_value), value = vec_value;
    8775              :     }
    8776              :   else
    8777              :     {
    8778              :       /* Choose appropriate vector mode.  */
    8779         1717 :       if (size >= 32)
    8780          429 :         switch (MOVE_MAX)
    8781              :           {
    8782            0 :           case 64:
    8783            0 :             if (size >= 64)
    8784              :               {
    8785              :                 mode = V64QImode;
    8786              :                 break;
    8787              :               }
    8788              :             /* FALLTHRU */
    8789            0 :           case 32:
    8790            0 :             mode = V32QImode;
    8791            0 :             break;
    8792              :           case 16:
    8793              :             mode = V16QImode;
    8794              :             break;
    8795              :           case 8:
    8796              :             mode = DImode;
    8797              :             break;
    8798            0 :           default:
    8799            0 :             gcc_unreachable ();
    8800              :           }
    8801         1288 :       else if (size >= 16)
    8802          429 :         mode = TARGET_SSE ? V16QImode : DImode;
    8803         1717 :       srcmem = change_address (srcmem, mode, srcptr);
    8804              :     }
    8805         3457 :   if (issetmem && vec_value && GET_MODE_SIZE (mode) > size)
    8806              :     {
    8807              :       /* For memset with vector and the size is smaller than the vector
    8808              :          size, first try the narrower vector, otherwise, use the
    8809              :          original value. */
    8810          439 :       machine_mode inner_mode = GET_MODE_INNER (mode);
    8811          439 :       unsigned int nunits = size / GET_MODE_SIZE (inner_mode);
    8812          439 :       if (nunits > 1)
    8813              :         {
    8814          328 :           mode = mode_for_vector (GET_MODE_INNER (mode),
    8815          328 :                                   nunits).require ();
    8816          164 :           value = gen_rtx_SUBREG (mode, value, 0);
    8817              :         }
    8818              :       else
    8819              :         {
    8820          275 :           scalar_int_mode smode
    8821          275 :             = smallest_int_mode_for_size (size * BITS_PER_UNIT).require ();
    8822          825 :           gcc_assert (GET_MODE_SIZE (GET_MODE (scalar_value))
    8823              :                       >= GET_MODE_SIZE (smode));
    8824          275 :           mode = smode;
    8825          275 :           if (GET_MODE (scalar_value) == mode)
    8826              :             value = scalar_value;
    8827              :           else
    8828           60 :             value = gen_rtx_SUBREG (mode, scalar_value, 0);
    8829              :         }
    8830              :     }
    8831         2588 :   destmem = change_address (destmem, mode, destptr);
    8832         5176 :   modesize = GEN_INT (GET_MODE_SIZE (mode));
    8833         5176 :   gcc_assert (GET_MODE_SIZE (mode) <= size);
    8834        11640 :   for (n = 0; n * GET_MODE_SIZE (mode) < size; n++)
    8835              :     {
    8836         3232 :       if (issetmem)
    8837         1086 :         emit_move_insn (destmem, gen_lowpart (mode, value));
    8838              :       else
    8839              :         {
    8840         2146 :           emit_move_insn (destmem, srcmem);
    8841         4292 :           srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
    8842              :         }
    8843         6464 :       destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
    8844              :     }
    8845              : 
    8846         2588 :   destmem = offset_address (destmem, count, 1);
    8847         5176 :   destmem = offset_address (destmem, GEN_INT (-2 * size),
    8848         2588 :                             GET_MODE_SIZE (mode));
    8849         2588 :   if (!issetmem)
    8850              :     {
    8851         1717 :       srcmem = offset_address (srcmem, count, 1);
    8852         3434 :       srcmem = offset_address (srcmem, GEN_INT (-2 * size),
    8853         1717 :                                GET_MODE_SIZE (mode));
    8854              :     }
    8855        11640 :   for (n = 0; n * GET_MODE_SIZE (mode) < size; n++)
    8856              :     {
    8857         3232 :       if (issetmem)
    8858         1086 :         emit_move_insn (destmem, gen_lowpart (mode, value));
    8859              :       else
    8860              :         {
    8861         2146 :           emit_move_insn (destmem, srcmem);
    8862         4292 :           srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
    8863              :         }
    8864         6464 :       destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
    8865              :     }
    8866         2588 :   emit_jump_insn (gen_jump (done_label));
    8867         2588 :   emit_barrier ();
    8868              : 
    8869         2588 :   emit_label (label);
    8870         2588 :   LABEL_NUSES (label) = 1;
    8871         2588 : }
    8872              : 
    8873              : /* Handle small memcpy (up to SIZE that is supposed to be small power of 2.
    8874              :    and get ready for the main memcpy loop by copying initial DESIRED_ALIGN-ALIGN
    8875              :    bytes and last SIZE bytes adjusitng DESTPTR/SRCPTR/COUNT in a way we can
    8876              :    proceed with an loop copying SIZE bytes at once. Do moves in MODE.
    8877              :    DONE_LABEL is a label after the whole copying sequence. The label is created
    8878              :    on demand if *DONE_LABEL is NULL.
    8879              :    MIN_SIZE is minimal size of block copied.  This value gets adjusted for new
    8880              :    bounds after the initial copies.
    8881              : 
    8882              :    DESTMEM/SRCMEM are memory expressions pointing to the copies block,
    8883              :    DESTPTR/SRCPTR are pointers to the block. DYNAMIC_CHECK indicate whether
    8884              :    we will dispatch to a library call for large blocks.
    8885              : 
    8886              :    In pseudocode we do:
    8887              : 
    8888              :    if (COUNT < SIZE)
    8889              :      {
    8890              :        Assume that SIZE is 4. Bigger sizes are handled analogously
    8891              :        if (COUNT & 4)
    8892              :          {
    8893              :             copy 4 bytes from SRCPTR to DESTPTR
    8894              :             copy 4 bytes from SRCPTR + COUNT - 4 to DESTPTR + COUNT - 4
    8895              :             goto done_label
    8896              :          }
    8897              :        if (!COUNT)
    8898              :          goto done_label;
    8899              :        copy 1 byte from SRCPTR to DESTPTR
    8900              :        if (COUNT & 2)
    8901              :          {
    8902              :             copy 2 bytes from SRCPTR to DESTPTR
    8903              :             copy 2 bytes from SRCPTR + COUNT - 2 to DESTPTR + COUNT - 2
    8904              :          }
    8905              :      }
    8906              :    else
    8907              :      {
    8908              :        copy at least DESIRED_ALIGN-ALIGN bytes from SRCPTR to DESTPTR
    8909              :        copy SIZE bytes from SRCPTR + COUNT - SIZE to DESTPTR + COUNT -SIZE
    8910              : 
    8911              :        OLD_DESPTR = DESTPTR;
    8912              :        Align DESTPTR up to DESIRED_ALIGN
    8913              :        SRCPTR += DESTPTR - OLD_DESTPTR
    8914              :        COUNT -= DEST_PTR - OLD_DESTPTR
    8915              :        if (DYNAMIC_CHECK)
    8916              :          Round COUNT down to multiple of SIZE
    8917              :        << optional caller supplied zero size guard is here >>
    8918              :        << optional caller supplied dynamic check is here >>
    8919              :        << caller supplied main copy loop is here >>
    8920              :      }
    8921              :    done_label:
    8922              :   */
    8923              : static void
    8924         4006 : expand_set_or_cpymem_prologue_epilogue_by_misaligned_moves (rtx destmem, rtx srcmem,
    8925              :                                                             rtx *destptr, rtx *srcptr,
    8926              :                                                             machine_mode mode,
    8927              :                                                             rtx value, rtx vec_value,
    8928              :                                                             rtx *count,
    8929              :                                                             rtx_code_label **done_label,
    8930              :                                                             int size,
    8931              :                                                             int desired_align,
    8932              :                                                             int align,
    8933              :                                                             unsigned HOST_WIDE_INT *min_size,
    8934              :                                                             bool dynamic_check,
    8935              :                                                             bool issetmem)
    8936              : {
    8937         4006 :   rtx_code_label *loop_label = NULL, *label;
    8938         4006 :   int n;
    8939         4006 :   rtx modesize;
    8940         4006 :   int prolog_size = 0;
    8941         4006 :   rtx mode_value;
    8942              : 
    8943              :   /* Chose proper value to copy.  */
    8944         4006 :   if (issetmem && VECTOR_MODE_P (mode))
    8945              :     mode_value = vec_value;
    8946              :   else
    8947         4006 :     mode_value = value;
    8948         8012 :   gcc_assert (GET_MODE_SIZE (mode) <= size);
    8949              : 
    8950              :   /* See if block is big or small, handle small blocks.  */
    8951         4006 :   if (!CONST_INT_P (*count) && *min_size < (unsigned HOST_WIDE_INT)size)
    8952              :     {
    8953          647 :       int size2 = size;
    8954          647 :       loop_label = gen_label_rtx ();
    8955              : 
    8956          647 :       if (!*done_label)
    8957          647 :         *done_label = gen_label_rtx ();
    8958              : 
    8959          647 :       emit_cmp_and_jump_insns (*count, GEN_INT (size2), GE, 0, GET_MODE (*count),
    8960              :                                1, loop_label);
    8961          647 :       size2 >>= 1;
    8962              : 
    8963              :       /* Handle sizes > 3.  */
    8964         3235 :       for (;size2 > 2; size2 >>= 1)
    8965         2588 :         expand_small_cpymem_or_setmem (destmem, srcmem,
    8966              :                                        *destptr, *srcptr,
    8967              :                                        value, vec_value,
    8968              :                                        *count,
    8969              :                                        size2, *done_label, issetmem);
    8970              :       /* Nothing to copy?  Jump to DONE_LABEL if so */
    8971          647 :       emit_cmp_and_jump_insns (*count, const0_rtx, EQ, 0, GET_MODE (*count),
    8972              :                                1, *done_label);
    8973              : 
    8974              :       /* Do a byte copy.  */
    8975          647 :       destmem = change_address (destmem, QImode, *destptr);
    8976          647 :       if (issetmem)
    8977          217 :         emit_move_insn (destmem, gen_lowpart (QImode, value));
    8978              :       else
    8979              :         {
    8980          430 :           srcmem = change_address (srcmem, QImode, *srcptr);
    8981          430 :           emit_move_insn (destmem, srcmem);
    8982              :         }
    8983              : 
    8984              :       /* Handle sizes 2 and 3.  */
    8985          647 :       label = ix86_expand_aligntest (*count, 2, false);
    8986          647 :       destmem = change_address (destmem, HImode, *destptr);
    8987          647 :       destmem = offset_address (destmem, *count, 1);
    8988          647 :       destmem = offset_address (destmem, GEN_INT (-2), 2);
    8989          647 :       if (issetmem)
    8990          217 :         emit_move_insn (destmem, gen_lowpart (HImode, value));
    8991              :       else
    8992              :         {
    8993          430 :           srcmem = change_address (srcmem, HImode, *srcptr);
    8994          430 :           srcmem = offset_address (srcmem, *count, 1);
    8995          430 :           srcmem = offset_address (srcmem, GEN_INT (-2), 2);
    8996          430 :           emit_move_insn (destmem, srcmem);
    8997              :         }
    8998              : 
    8999          647 :       emit_label (label);
    9000          647 :       LABEL_NUSES (label) = 1;
    9001          647 :       emit_jump_insn (gen_jump (*done_label));
    9002          647 :       emit_barrier ();
    9003              :     }
    9004              :   else
    9005         3359 :     gcc_assert (*min_size >= (unsigned HOST_WIDE_INT)size
    9006              :                 || UINTVAL (*count) >= (unsigned HOST_WIDE_INT)size);
    9007              : 
    9008              :   /* Start memcpy for COUNT >= SIZE.  */
    9009          647 :   if (loop_label)
    9010              :     {
    9011          647 :        emit_label (loop_label);
    9012          647 :        LABEL_NUSES (loop_label) = 1;
    9013              :     }
    9014              : 
    9015              :   /* Copy first desired_align bytes.  */
    9016         4006 :   if (!issetmem)
    9017         2094 :     srcmem = change_address (srcmem, mode, *srcptr);
    9018         4006 :   destmem = change_address (destmem, mode, *destptr);
    9019         4006 :   modesize = GEN_INT (GET_MODE_SIZE (mode));
    9020         8029 :   for (n = 0; prolog_size < desired_align - align; n++)
    9021              :     {
    9022           17 :       if (issetmem)
    9023            1 :         emit_move_insn (destmem, mode_value);
    9024              :       else
    9025              :         {
    9026           16 :           emit_move_insn (destmem, srcmem);
    9027           32 :           srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
    9028              :         }
    9029           34 :       destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
    9030           34 :       prolog_size += GET_MODE_SIZE (mode);
    9031              :     }
    9032              : 
    9033              : 
    9034              :   /* Copy last SIZE bytes.  */
    9035         4006 :   destmem = offset_address (destmem, *count, 1);
    9036         4006 :   destmem = offset_address (destmem,
    9037         4006 :                             GEN_INT (-size - prolog_size),
    9038              :                             1);
    9039         4006 :   if (issetmem)
    9040         1912 :     emit_move_insn (destmem, mode_value);
    9041              :   else
    9042              :     {
    9043         2094 :       srcmem = offset_address (srcmem, *count, 1);
    9044         2094 :       srcmem = offset_address (srcmem,
    9045              :                                GEN_INT (-size - prolog_size),
    9046              :                                1);
    9047         2094 :       emit_move_insn (destmem, srcmem);
    9048              :     }
    9049        30368 :   for (n = 1; n * GET_MODE_SIZE (mode) < size; n++)
    9050              :     {
    9051        11178 :       destmem = offset_address (destmem, modesize, 1);
    9052        11178 :       if (issetmem)
    9053         5550 :         emit_move_insn (destmem, mode_value);
    9054              :       else
    9055              :         {
    9056         5628 :           srcmem = offset_address (srcmem, modesize, 1);
    9057         5628 :           emit_move_insn (destmem, srcmem);
    9058              :         }
    9059              :     }
    9060              : 
    9061              :   /* Align destination.  */
    9062         4006 :   if (desired_align > 1 && desired_align > align)
    9063              :     {
    9064           17 :       rtx saveddest = *destptr;
    9065              : 
    9066           17 :       gcc_assert (desired_align <= size);
    9067              :       /* Align destptr up, place it to new register.  */
    9068           17 :       *destptr = expand_simple_binop (GET_MODE (*destptr), PLUS, *destptr,
    9069              :                                       GEN_INT (prolog_size),
    9070              :                                       NULL_RTX, 1, OPTAB_DIRECT);
    9071           17 :       if (REG_P (*destptr) && REG_P (saveddest) && REG_POINTER (saveddest))
    9072           17 :         REG_POINTER (*destptr) = 1;
    9073           17 :       *destptr = expand_simple_binop (GET_MODE (*destptr), AND, *destptr,
    9074           17 :                                       GEN_INT (-desired_align),
    9075              :                                       *destptr, 1, OPTAB_DIRECT);
    9076              :       /* See how many bytes we skipped.  */
    9077           17 :       saveddest = expand_simple_binop (GET_MODE (*destptr), MINUS, saveddest,
    9078              :                                        *destptr,
    9079              :                                        NULL_RTX, 1, OPTAB_DIRECT);
    9080              :       /* Adjust srcptr and count.  */
    9081           17 :       if (!issetmem)
    9082           16 :         *srcptr = expand_simple_binop (GET_MODE (*srcptr), MINUS, *srcptr,
    9083              :                                        saveddest, *srcptr, 1, OPTAB_DIRECT);
    9084           17 :       *count = expand_simple_binop (GET_MODE (*count), PLUS, *count,
    9085              :                                     saveddest, *count, 1, OPTAB_DIRECT);
    9086              :       /* We copied at most size + prolog_size.  */
    9087           17 :       if (*min_size > (unsigned HOST_WIDE_INT)(size + prolog_size))
    9088           16 :         *min_size
    9089           16 :           = ROUND_DOWN (*min_size - size, (unsigned HOST_WIDE_INT)size);
    9090              :       else
    9091              :         *min_size = 0;
    9092              : 
    9093              :       /* Our loops always round down the block size, but for dispatch to
    9094              :          library we need precise value.  */
    9095           17 :       if (dynamic_check)
    9096           17 :         *count = expand_simple_binop (GET_MODE (*count), AND, *count,
    9097              :                                       GEN_INT (-size), *count, 1, OPTAB_DIRECT);
    9098              :     }
    9099              :   else
    9100              :     {
    9101         3989 :       gcc_assert (prolog_size == 0);
    9102              :       /* Decrease count, so we won't end up copying last word twice.  */
    9103         3989 :       if (!CONST_INT_P (*count))
    9104          646 :         *count = expand_simple_binop (GET_MODE (*count), PLUS, *count,
    9105              :                                       constm1_rtx, *count, 1, OPTAB_DIRECT);
    9106              :       else
    9107         3343 :         *count = GEN_INT (ROUND_DOWN (UINTVAL (*count) - 1,
    9108              :                                       (unsigned HOST_WIDE_INT)size));
    9109         3989 :       if (*min_size)
    9110         3777 :         *min_size = ROUND_DOWN (*min_size - 1, (unsigned HOST_WIDE_INT)size);
    9111              :     }
    9112         4006 : }
    9113              : 
    9114              : 
    9115              : /* This function is like the previous one, except here we know how many bytes
    9116              :    need to be copied.  That allows us to update alignment not only of DST, which
    9117              :    is returned, but also of SRC, which is passed as a pointer for that
    9118              :    reason.  */
    9119              : static rtx
    9120            0 : expand_set_or_cpymem_constant_prologue (rtx dst, rtx *srcp, rtx destreg,
    9121              :                                            rtx srcreg, rtx value, rtx vec_value,
    9122              :                                            int desired_align, int align_bytes,
    9123              :                                            bool issetmem)
    9124              : {
    9125            0 :   rtx src = NULL;
    9126            0 :   rtx orig_dst = dst;
    9127            0 :   rtx orig_src = NULL;
    9128            0 :   int piece_size = 1;
    9129            0 :   int copied_bytes = 0;
    9130              : 
    9131            0 :   if (!issetmem)
    9132              :     {
    9133            0 :       gcc_assert (srcp != NULL);
    9134            0 :       src = *srcp;
    9135            0 :       orig_src = src;
    9136              :     }
    9137              : 
    9138            0 :   for (piece_size = 1;
    9139            0 :        piece_size <= desired_align && copied_bytes < align_bytes;
    9140            0 :        piece_size <<= 1)
    9141              :     {
    9142            0 :       if (align_bytes & piece_size)
    9143              :         {
    9144            0 :           if (issetmem)
    9145              :             {
    9146            0 :               if (vec_value && piece_size > GET_MODE_SIZE (GET_MODE (value)))
    9147            0 :                 dst = emit_memset (dst, destreg, vec_value, piece_size);
    9148              :               else
    9149            0 :                 dst = emit_memset (dst, destreg, value, piece_size);
    9150              :             }
    9151              :           else
    9152            0 :             dst = emit_memmov (dst, &src, destreg, srcreg, piece_size);
    9153            0 :           copied_bytes += piece_size;
    9154              :         }
    9155              :     }
    9156            0 :   if (MEM_ALIGN (dst) < (unsigned int) desired_align * BITS_PER_UNIT)
    9157            0 :     set_mem_align (dst, desired_align * BITS_PER_UNIT);
    9158            0 :   if (MEM_SIZE_KNOWN_P (orig_dst))
    9159            0 :     set_mem_size (dst, MEM_SIZE (orig_dst) - align_bytes);
    9160              : 
    9161            0 :   if (!issetmem)
    9162              :     {
    9163            0 :       int src_align_bytes = get_mem_align_offset (src, desired_align
    9164              :                                                        * BITS_PER_UNIT);
    9165            0 :       if (src_align_bytes >= 0)
    9166            0 :         src_align_bytes = desired_align - src_align_bytes;
    9167            0 :       if (src_align_bytes >= 0)
    9168              :         {
    9169              :           unsigned int src_align;
    9170            0 :           for (src_align = desired_align; src_align >= 2; src_align >>= 1)
    9171              :             {
    9172            0 :               if ((src_align_bytes & (src_align - 1))
    9173            0 :                    == (align_bytes & (src_align - 1)))
    9174              :                 break;
    9175              :             }
    9176            0 :           if (src_align > (unsigned int) desired_align)
    9177              :             src_align = desired_align;
    9178            0 :           if (MEM_ALIGN (src) < src_align * BITS_PER_UNIT)
    9179            0 :             set_mem_align (src, src_align * BITS_PER_UNIT);
    9180              :         }
    9181            0 :       if (MEM_SIZE_KNOWN_P (orig_src))
    9182            0 :         set_mem_size (src, MEM_SIZE (orig_src) - align_bytes);
    9183            0 :       *srcp = src;
    9184              :     }
    9185              : 
    9186            0 :   return dst;
    9187              : }
    9188              : 
    9189              : /* Return true if ALG can be used in current context.
    9190              :    Assume we expand memset if MEMSET is true.  */
    9191              : static bool
    9192      1450862 : alg_usable_p (enum stringop_alg alg, bool memset,
    9193              :               addr_space_t dst_as, addr_space_t src_as)
    9194              : {
    9195      1450862 :   if (alg == no_stringop)
    9196              :     return false;
    9197              :   /* It is not possible to use a library call if we have non-default
    9198              :      address space.  We can do better than the generic byte-at-a-time
    9199              :      loop, used as a fallback.  */
    9200      1450862 :   if (alg == libcall &&
    9201       832319 :       !(ADDR_SPACE_GENERIC_P (dst_as) && ADDR_SPACE_GENERIC_P (src_as)))
    9202              :     return false;
    9203      1450848 :   if (alg == vector_loop)
    9204       310190 :     return TARGET_SSE || TARGET_AVX;
    9205              :   /* Algorithms using the rep prefix want at least edi and ecx;
    9206              :      additionally, memset wants eax and memcpy wants esi.  Don't
    9207              :      consider such algorithms if the user has appropriated those
    9208              :      registers for their own purposes, or if we have the destination
    9209              :      in the non-default address space, since string insns cannot
    9210              :      override the destination segment.  */
    9211      1140658 :   if (alg == rep_prefix_1_byte
    9212              :       || alg == rep_prefix_4_byte
    9213      1140658 :       || alg == rep_prefix_8_byte)
    9214              :     {
    9215        39137 :       if (fixed_regs[CX_REG]
    9216        39133 :           || fixed_regs[DI_REG]
    9217        39129 :           || (memset ? fixed_regs[AX_REG] : fixed_regs[SI_REG])
    9218        39125 :           || !ADDR_SPACE_GENERIC_P (dst_as)
    9219        78262 :           || !(ADDR_SPACE_GENERIC_P (src_as) || Pmode == word_mode))
    9220           12 :         return false;
    9221              :     }
    9222              :   return true;
    9223              : }
    9224              : 
    9225              : /* Given COUNT and EXPECTED_SIZE, decide on codegen of string operation.  */
    9226              : static enum stringop_alg
    9227       288635 : decide_alg (HOST_WIDE_INT count, HOST_WIDE_INT expected_size,
    9228              :             unsigned HOST_WIDE_INT min_size, unsigned HOST_WIDE_INT max_size,
    9229              :             bool memset, bool zero_memset, addr_space_t dst_as,
    9230              :             addr_space_t src_as, int *dynamic_check, bool *noalign, bool recur)
    9231              : {
    9232       288635 :   const struct stringop_algs *algs;
    9233       288635 :   bool optimize_for_speed;
    9234       288635 :   int max = 0;
    9235       288635 :   const struct processor_costs *cost;
    9236       288635 :   int i;
    9237       288635 :   bool any_alg_usable_p = false;
    9238              : 
    9239       288635 :   *noalign = false;
    9240       288635 :   *dynamic_check = -1;
    9241              : 
    9242              :   /* Even if the string operation call is cold, we still might spend a lot
    9243              :      of time processing large blocks.  */
    9244       288635 :   if (optimize_function_for_size_p (cfun)
    9245       288635 :       || (optimize_insn_for_size_p ()
    9246        12089 :           && (max_size < 256
    9247         3825 :               || (expected_size != -1 && expected_size < 256))))
    9248              :     optimize_for_speed = false;
    9249              :   else
    9250       269211 :     optimize_for_speed = true;
    9251              : 
    9252       269211 :   cost = optimize_for_speed ? ix86_cost : &ix86_size_cost;
    9253       288635 :   if (memset)
    9254        89349 :     algs = &cost->memset[TARGET_64BIT != 0];
    9255              :   else
    9256       208160 :     algs = &cost->memcpy[TARGET_64BIT != 0];
    9257              : 
    9258              :   /* See maximal size for user defined algorithm.  */
    9259      1443175 :   for (i = 0; i < MAX_STRINGOP_ALGS; i++)
    9260              :     {
    9261      1154540 :       enum stringop_alg candidate = algs->size[i].alg;
    9262      1154540 :       bool usable = alg_usable_p (candidate, memset, dst_as, src_as);
    9263      1154540 :       any_alg_usable_p |= usable;
    9264              : 
    9265      1154540 :       if (candidate != libcall && candidate && usable)
    9266       557733 :         max = algs->size[i].max;
    9267              :     }
    9268              : 
    9269              :   /* If expected size is not known but max size is small enough
    9270              :      so inline version is a win, set expected size into
    9271              :      the range.  */
    9272       288635 :   if (((max > 1 && (unsigned HOST_WIDE_INT) max >= max_size) || max == -1)
    9273        38792 :       && expected_size == -1)
    9274        15598 :     expected_size = min_size / 2 + max_size / 2;
    9275              : 
    9276              :   /* If user specified the algorithm, honor it if possible.  */
    9277       288635 :   if (ix86_stringop_alg != no_stringop
    9278       288635 :       && alg_usable_p (ix86_stringop_alg, memset, dst_as, src_as))
    9279              :     return ix86_stringop_alg;
    9280              :   /* rep; movq or rep; movl is the smallest variant.  */
    9281       288497 :   else if (!optimize_for_speed)
    9282              :     {
    9283        19341 :       *noalign = true;
    9284        19341 :       if (!count || (count & 3) || (memset && !zero_memset))
    9285         6012 :         return alg_usable_p (rep_prefix_1_byte, memset, dst_as, src_as)
    9286         6012 :                ? rep_prefix_1_byte : loop_1_byte;
    9287              :       else
    9288        13329 :         return alg_usable_p (rep_prefix_4_byte, memset, dst_as, src_as)
    9289        13329 :                ? rep_prefix_4_byte : loop;
    9290              :     }
    9291              :   /* Very tiny blocks are best handled via the loop, REP is expensive to
    9292              :      setup.  */
    9293       269156 :   else if (expected_size != -1 && expected_size < 4)
    9294              :     return loop_1_byte;
    9295       267608 :   else if (expected_size != -1)
    9296              :     {
    9297              :       enum stringop_alg alg = libcall;
    9298              :       bool alg_noalign = false;
    9299       345675 :       for (i = 0; i < MAX_STRINGOP_ALGS; i++)
    9300              :         {
    9301              :           /* We get here if the algorithms that were not libcall-based
    9302              :              were rep-prefix based and we are unable to use rep prefixes
    9303              :              based on global register usage.  Break out of the loop and
    9304              :              use the heuristic below.  */
    9305       339938 :           if (algs->size[i].max == 0)
    9306              :             break;
    9307       339938 :           if (algs->size[i].max >= expected_size || algs->size[i].max == -1)
    9308              :             {
    9309       133765 :               enum stringop_alg candidate = algs->size[i].alg;
    9310              : 
    9311       133765 :               if (candidate != libcall
    9312       133765 :                   && alg_usable_p (candidate, memset, dst_as, src_as))
    9313              :                 {
    9314        24917 :                   alg = candidate;
    9315        24917 :                   alg_noalign = algs->size[i].noalign;
    9316              :                 }
    9317              :               /* Honor TARGET_INLINE_ALL_STRINGOPS by picking
    9318              :                  last non-libcall inline algorithm.  */
    9319       133765 :               if (TARGET_INLINE_ALL_STRINGOPS)
    9320              :                 {
    9321              :                   /* When the current size is best to be copied by a libcall,
    9322              :                      but we are still forced to inline, run the heuristic below
    9323              :                      that will pick code for medium sized blocks.  */
    9324        20035 :                   if (alg != libcall)
    9325              :                     {
    9326         8552 :                       *noalign = alg_noalign;
    9327         8552 :                       return alg;
    9328              :                     }
    9329        11483 :                   else if (!any_alg_usable_p)
    9330              :                     break;
    9331              :                 }
    9332       113730 :               else if (alg_usable_p (candidate, memset, dst_as, src_as)
    9333       113730 :                        && !(TARGET_PREFER_KNOWN_REP_MOVSB_STOSB
    9334           34 :                             && candidate == rep_prefix_1_byte
    9335              :                             /* NB: If min_size != max_size, size is
    9336              :                                unknown.  */
    9337           34 :                             && min_size != max_size))
    9338              :                 {
    9339       113705 :                   *noalign = algs->size[i].noalign;
    9340       113705 :                   return candidate;
    9341              :                 }
    9342              :             }
    9343              :         }
    9344              :     }
    9345              :   /* When asked to inline the call anyway, try to pick meaningful choice.
    9346              :      We look for maximal size of block that is faster to copy by hand and
    9347              :      take blocks of at most of that size guessing that average size will
    9348              :      be roughly half of the block.
    9349              : 
    9350              :      If this turns out to be bad, we might simply specify the preferred
    9351              :      choice in ix86_costs.  */
    9352       138173 :   if ((TARGET_INLINE_ALL_STRINGOPS || TARGET_INLINE_STRINGOPS_DYNAMICALLY)
    9353       145357 :       && (algs->unknown_size == libcall
    9354            0 :           || !alg_usable_p (algs->unknown_size, memset, dst_as, src_as)))
    9355              :     {
    9356         7184 :       enum stringop_alg alg;
    9357         7184 :       HOST_WIDE_INT new_expected_size = (max > 0 ? max : 4096) / 2;
    9358              : 
    9359              :       /* If there aren't any usable algorithms or if recursing already,
    9360              :          then recursing on smaller sizes or same size isn't going to
    9361              :          find anything.  Just return the simple byte-at-a-time copy loop.  */
    9362         7184 :       if (!any_alg_usable_p || recur)
    9363              :         {
    9364              :           /* Pick something reasonable.  */
    9365            0 :           if (TARGET_INLINE_STRINGOPS_DYNAMICALLY && !recur)
    9366            0 :             *dynamic_check = 128;
    9367              :           return loop_1_byte;
    9368              :         }
    9369         7184 :       alg = decide_alg (count, new_expected_size, min_size, max_size,
    9370              :                         memset, zero_memset, dst_as, src_as,
    9371              :                         dynamic_check, noalign, true);
    9372         7184 :       gcc_assert (*dynamic_check == -1);
    9373         7184 :       if (TARGET_INLINE_STRINGOPS_DYNAMICALLY)
    9374           10 :         *dynamic_check = max;
    9375              :       else
    9376         7174 :         gcc_assert (alg != libcall);
    9377              :       return alg;
    9378              :     }
    9379              : 
    9380              :   /* Try to use some reasonable fallback algorithm.  Note that for
    9381              :      non-default address spaces we default to a loop instead of
    9382              :      a libcall.  */
    9383              : 
    9384       138167 :   bool have_as = !(ADDR_SPACE_GENERIC_P (dst_as)
    9385              :                    && ADDR_SPACE_GENERIC_P (src_as));
    9386              : 
    9387       138167 :   return (alg_usable_p (algs->unknown_size, memset, dst_as, src_as)
    9388       138167 :           ? algs->unknown_size : have_as ? loop : libcall);
    9389              : }
    9390              : 
    9391              : /* Decide on alignment.  We know that the operand is already aligned to ALIGN
    9392              :    (ALIGN can be based on profile feedback and thus it is not 100% guaranteed).  */
    9393              : static int
    9394        27527 : decide_alignment (int align,
    9395              :                   enum stringop_alg alg,
    9396              :                   int expected_size,
    9397              :                   machine_mode move_mode)
    9398              : {
    9399        27527 :   int desired_align = 0;
    9400              : 
    9401        27527 :   gcc_assert (alg != no_stringop);
    9402              : 
    9403        27527 :   if (alg == libcall)
    9404              :     return 0;
    9405        27527 :   if (move_mode == VOIDmode)
    9406              :     return 0;
    9407              : 
    9408        27527 :   desired_align = GET_MODE_SIZE (move_mode);
    9409              :   /* PentiumPro has special logic triggering for 8 byte aligned blocks.
    9410              :      copying whole cacheline at once.  */
    9411        27527 :   if (TARGET_CPU_P (PENTIUMPRO)
    9412            0 :       && (alg == rep_prefix_4_byte || alg == rep_prefix_1_byte))
    9413        27527 :     desired_align = 8;
    9414              : 
    9415        27527 :   if (optimize_size)
    9416        10586 :     desired_align = 1;
    9417        27527 :   if (desired_align < align)
    9418              :     desired_align = align;
    9419        27527 :   if (expected_size != -1 && expected_size < 4)
    9420            1 :     desired_align = align;
    9421              : 
    9422              :   return desired_align;
    9423              : }
    9424              : 
    9425              : 
    9426              : /* Helper function for memcpy.  For QImode value 0xXY produce
    9427              :    0xXYXYXYXY of wide specified by MODE.  This is essentially
    9428              :    a * 0x10101010, but we can do slightly better than
    9429              :    synth_mult by unwinding the sequence by hand on CPUs with
    9430              :    slow multiply.  */
    9431              : static rtx
    9432        18616 : promote_duplicated_reg (machine_mode mode, rtx val)
    9433              : {
    9434        18616 :   if (val == const0_rtx)
    9435        16101 :     return copy_to_mode_reg (mode, CONST0_RTX (mode));
    9436              : 
    9437         2515 :   machine_mode valmode = GET_MODE (val);
    9438         2515 :   if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
    9439              :     {
    9440              :       /* Duplicate the scalar value for integer vector.  */
    9441         1899 :       gcc_assert ((val == const0_rtx || val == constm1_rtx)
    9442              :                   || GET_MODE_INNER (mode) == valmode);
    9443          954 :       rtx dup = gen_reg_rtx (mode);
    9444          954 :       bool ok = ix86_expand_vector_init_duplicate (false, mode, dup,
    9445              :                                                    val);
    9446          954 :       gcc_assert (ok);
    9447              :       return dup;
    9448              :     }
    9449              : 
    9450         1561 :   rtx tmp;
    9451         1561 :   int nops = mode == DImode ? 3 : 2;
    9452              : 
    9453           41 :   gcc_assert (mode == SImode || mode == DImode);
    9454         1561 :   if (CONST_INT_P (val))
    9455              :     {
    9456         1248 :       HOST_WIDE_INT v = INTVAL (val) & 255;
    9457              : 
    9458         1248 :       v |= v << 8;
    9459         1248 :       v |= v << 16;
    9460         1248 :       if (mode == DImode)
    9461         1219 :         v |= (v << 16) << 16;
    9462         1248 :       return copy_to_mode_reg (mode, gen_int_mode (v, mode));
    9463              :     }
    9464              : 
    9465          313 :   if (valmode == VOIDmode)
    9466              :     valmode = QImode;
    9467          313 :   if (valmode != QImode)
    9468            0 :     val = gen_lowpart (QImode, val);
    9469          313 :   if (mode == QImode)
    9470              :     return val;
    9471          313 :   if (!TARGET_PARTIAL_REG_STALL)
    9472          313 :     nops--;
    9473          313 :   if (ix86_cost->mult_init[mode == DImode ? 3 : 2]
    9474          313 :       + ix86_cost->mult_bit * (mode == DImode ? 8 : 4)
    9475          313 :       <= (ix86_cost->shift_const + ix86_cost->add) * nops
    9476          313 :           + (COSTS_N_INSNS (TARGET_PARTIAL_REG_STALL == 0)))
    9477              :     {
    9478          313 :       rtx reg = convert_modes (mode, QImode, val, true);
    9479          313 :       tmp = promote_duplicated_reg (mode, const1_rtx);
    9480          313 :       return expand_simple_binop (mode, MULT, reg, tmp, NULL, 1,
    9481          313 :                                   OPTAB_DIRECT);
    9482              :     }
    9483              :   else
    9484              :     {
    9485            0 :       rtx reg = convert_modes (mode, QImode, val, true);
    9486              : 
    9487            0 :       if (!TARGET_PARTIAL_REG_STALL)
    9488            0 :         emit_insn (gen_insv_1 (mode, reg, reg));
    9489              :       else
    9490              :         {
    9491            0 :           tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (8),
    9492              :                                      NULL, 1, OPTAB_DIRECT);
    9493            0 :           reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1,
    9494              :                                      OPTAB_DIRECT);
    9495              :         }
    9496            0 :       tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (16),
    9497              :                                  NULL, 1, OPTAB_DIRECT);
    9498            0 :       reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
    9499            0 :       if (mode == SImode)
    9500              :         return reg;
    9501            0 :       tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (32),
    9502              :                                  NULL, 1, OPTAB_DIRECT);
    9503            0 :       reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
    9504            0 :       return reg;
    9505              :     }
    9506              : }
    9507              : 
    9508              : /* Duplicate value VAL using promote_duplicated_reg into maximal size that will
    9509              :    be needed by main loop copying SIZE_NEEDED chunks and prologue getting
    9510              :    alignment from ALIGN to DESIRED_ALIGN.  */
    9511              : static rtx
    9512        13567 : promote_duplicated_reg_to_size (rtx val, int size_needed, int desired_align,
    9513              :                                 int align)
    9514              : {
    9515        13567 :   rtx promoted_val;
    9516              : 
    9517        13567 :   if (TARGET_64BIT
    9518        12087 :       && (size_needed > 4 || (desired_align > align && desired_align > 4)))
    9519         3942 :     promoted_val = promote_duplicated_reg (DImode, val);
    9520         9625 :   else if (size_needed > 2 || (desired_align > align && desired_align > 2))
    9521         8013 :     promoted_val = promote_duplicated_reg (SImode, val);
    9522         1612 :   else if (size_needed > 1 || (desired_align > align && desired_align > 1))
    9523            0 :     promoted_val = promote_duplicated_reg (HImode, val);
    9524              :   else
    9525              :     promoted_val = val;
    9526              : 
    9527        13567 :   return promoted_val;
    9528              : }
    9529              : 
    9530              : /* Copy the address to a Pmode register.  This is used for x32 to
    9531              :    truncate DImode TLS address to a SImode register. */
    9532              : 
    9533              : static rtx
    9534        66435 : ix86_copy_addr_to_reg (rtx addr)
    9535              : {
    9536        66435 :   rtx reg;
    9537        71008 :   if (GET_MODE (addr) == Pmode || GET_MODE (addr) == VOIDmode)
    9538              :     {
    9539        66435 :       reg = copy_addr_to_reg (addr);
    9540        66435 :       REG_POINTER (reg) = 1;
    9541        66435 :       return reg;
    9542              :     }
    9543              :   else
    9544              :     {
    9545            0 :       gcc_assert (GET_MODE (addr) == DImode && Pmode == SImode);
    9546            0 :       reg = copy_to_mode_reg (DImode, addr);
    9547            0 :       REG_POINTER (reg) = 1;
    9548            0 :       return gen_rtx_SUBREG (SImode, reg, 0);
    9549              :     }
    9550              : }
    9551              : 
    9552              : /* Expand string move (memcpy) ot store (memset) operation.  Use i386 string
    9553              :    operations when profitable.  The code depends upon architecture, block size
    9554              :    and alignment, but always has one of the following overall structures:
    9555              : 
    9556              :    Aligned move sequence:
    9557              : 
    9558              :      1) Prologue guard: Conditional that jumps up to epilogues for small
    9559              :         blocks that can be handled by epilogue alone.  This is faster
    9560              :         but also needed for correctness, since prologue assume the block
    9561              :         is larger than the desired alignment.
    9562              : 
    9563              :         Optional dynamic check for size and libcall for large
    9564              :         blocks is emitted here too, with -minline-stringops-dynamically.
    9565              : 
    9566              :      2) Prologue: copy first few bytes in order to get destination
    9567              :         aligned to DESIRED_ALIGN.  It is emitted only when ALIGN is less
    9568              :         than DESIRED_ALIGN and up to DESIRED_ALIGN - ALIGN bytes can be
    9569              :         copied.  We emit either a jump tree on power of two sized
    9570              :         blocks, or a byte loop.
    9571              : 
    9572              :      3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
    9573              :         with specified algorithm.
    9574              : 
    9575              :      4) Epilogue: code copying tail of the block that is too small to be
    9576              :         handled by main body (or up to size guarded by prologue guard).
    9577              : 
    9578              :   Misaligned move sequence
    9579              : 
    9580              :      1) missaligned move prologue/epilogue containing:
    9581              :         a) Prologue handling small memory blocks and jumping to done_label
    9582              :            (skipped if blocks are known to be large enough)
    9583              :         b) Single move copying first DESIRED_ALIGN-ALIGN bytes if alignment is
    9584              :            needed by single possibly misaligned move
    9585              :            (skipped if alignment is not needed)
    9586              :         c) Copy of last SIZE_NEEDED bytes by possibly misaligned moves
    9587              : 
    9588              :      2) Zero size guard dispatching to done_label, if needed
    9589              : 
    9590              :      3) dispatch to library call, if needed,
    9591              : 
    9592              :      3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
    9593              :         with specified algorithm.  */
    9594              : bool
    9595       153513 : ix86_expand_set_or_cpymem (rtx dst, rtx src, rtx count_exp, rtx val_exp,
    9596              :                            rtx align_exp, rtx expected_align_exp,
    9597              :                            rtx expected_size_exp, rtx min_size_exp,
    9598              :                            rtx max_size_exp, rtx probable_max_size_exp,
    9599              :                            bool issetmem)
    9600              : {
    9601       153513 :   if (TARGET_MISALIGNED_MOVE_STRING_PRO_EPILOGUES)
    9602              :     {
    9603              :       /* Expand bounded memset and memcpy as memmove if misaligned moves
    9604              :          are preferred.  Since
    9605              : 
    9606              :          commit b41f96465190751561f6909e858604ceab00595b
    9607              :          Author: H.J. Lu <hjl.tools@gmail.com>
    9608              :          Date:   Mon Oct 20 16:14:34 2025 +0800
    9609              : 
    9610              :          x86-64: Inline memmove with overlapping unaligned loads and stores.
    9611              : 
    9612              :          inlines memmove with overlapping unaligned and stores, which
    9613              :          reduces the numbers of branches and memory moves, comparing
    9614              :          against the regular memset and memcpy inlining.  */
    9615       153495 :       rtx operands[9];
    9616       153495 :       operands[0] = dst;
    9617       153495 :       operands[1] = issetmem ? val_exp : src;
    9618       153495 :       operands[2] = count_exp;
    9619       153495 :       operands[3] = align_exp;
    9620       153495 :       operands[4] = expected_align_exp;
    9621       153495 :       operands[5] = expected_size_exp;
    9622       153495 :       operands[6] = min_size_exp;
    9623       153495 :       operands[7] = max_size_exp;
    9624       153495 :       operands[8] = probable_max_size_exp;
    9625       153495 :       if (ix86_expand_set_or_movmem (operands, !issetmem, issetmem))
    9626         8808 :         return true;
    9627              :     }
    9628              : 
    9629       144705 :   rtx destreg;
    9630       144705 :   rtx srcreg = NULL;
    9631       144705 :   rtx_code_label *label = NULL;
    9632       144705 :   rtx tmp;
    9633       144705 :   rtx_code_label *jump_around_label = NULL;
    9634       144705 :   HOST_WIDE_INT align = 1;
    9635       144705 :   unsigned HOST_WIDE_INT count = 0;
    9636       144705 :   HOST_WIDE_INT expected_size = -1;
    9637       144705 :   int size_needed = 0, epilogue_size_needed;
    9638       144705 :   int desired_align = 0, align_bytes = 0;
    9639       144705 :   enum stringop_alg alg;
    9640       144705 :   rtx promoted_val = NULL;
    9641       144705 :   rtx vec_promoted_val = NULL;
    9642       144705 :   bool force_loopy_epilogue = false;
    9643       144705 :   int dynamic_check;
    9644       144705 :   bool need_zero_guard = false;
    9645       144705 :   bool noalign;
    9646       144705 :   machine_mode move_mode = VOIDmode;
    9647       144705 :   int unroll_factor = 1;
    9648              :   /* TODO: Once value ranges are available, fill in proper data.  */
    9649       144705 :   unsigned HOST_WIDE_INT min_size = HOST_WIDE_INT_0U;
    9650       144705 :   unsigned HOST_WIDE_INT max_size = HOST_WIDE_INT_M1U;
    9651       144705 :   unsigned HOST_WIDE_INT probable_max_size = HOST_WIDE_INT_M1U;
    9652       144705 :   bool misaligned_prologue_used = false;
    9653       144705 :   addr_space_t dst_as, src_as = ADDR_SPACE_GENERIC;
    9654              : 
    9655       144705 :   if (CONST_INT_P (align_exp))
    9656       144705 :     align = INTVAL (align_exp);
    9657              :   /* i386 can do misaligned access on reasonably increased cost.  */
    9658       144705 :   if (CONST_INT_P (expected_align_exp)
    9659       144705 :       && INTVAL (expected_align_exp) > align)
    9660              :     align = INTVAL (expected_align_exp);
    9661              :   /* ALIGN is the minimum of destination and source alignment, but we care here
    9662              :      just about destination alignment.  */
    9663       138231 :   else if (!issetmem
    9664       229116 :            && MEM_ALIGN (dst) > (unsigned HOST_WIDE_INT) align * BITS_PER_UNIT)
    9665         3194 :     align = MEM_ALIGN (dst) / BITS_PER_UNIT;
    9666              : 
    9667       144705 :   if (CONST_INT_P (count_exp))
    9668              :     {
    9669        71802 :       min_size = max_size = probable_max_size = count = expected_size
    9670        71802 :         = INTVAL (count_exp);
    9671              :       /* When COUNT is 0, there is nothing to do.  */
    9672        71802 :       if (!count)
    9673              :         return true;
    9674              :     }
    9675              :   else
    9676              :     {
    9677        72903 :       if (min_size_exp)
    9678        72903 :         min_size = INTVAL (min_size_exp);
    9679        72903 :       if (max_size_exp)
    9680        57795 :         max_size = INTVAL (max_size_exp);
    9681        72903 :       if (probable_max_size_exp)
    9682        59239 :         probable_max_size = INTVAL (probable_max_size_exp);
    9683        72903 :       if (CONST_INT_P (expected_size_exp))
    9684        72903 :         expected_size = INTVAL (expected_size_exp);
    9685              :      }
    9686              : 
    9687              :   /* Make sure we don't need to care about overflow later on.  */
    9688       144705 :   if (count > (HOST_WIDE_INT_1U << 30))
    9689              :     return false;
    9690              : 
    9691       144530 :   dst_as = MEM_ADDR_SPACE (dst);
    9692       144530 :   if (!issetmem)
    9693        97245 :     src_as = MEM_ADDR_SPACE (src);
    9694              : 
    9695              :   /* Step 0: Decide on preferred algorithm, desired alignment and
    9696              :      size of chunks to be copied by main loop.  */
    9697       144530 :   alg = decide_alg (count, expected_size, min_size, probable_max_size,
    9698        47285 :                     issetmem, issetmem && val_exp == const0_rtx,
    9699              :                     dst_as, src_as, &dynamic_check, &noalign, false);
    9700              : 
    9701       144530 :   if (dump_file)
    9702           13 :     fprintf (dump_file, "Selected stringop expansion strategy: %s\n",
    9703           13 :              stringop_alg_names[alg]);
    9704              : 
    9705       144530 :   if (alg == libcall)
    9706              :     return false;
    9707        27527 :   gcc_assert (alg != no_stringop);
    9708              : 
    9709        27527 :   if (!count)
    9710         5602 :     count_exp = copy_to_mode_reg (GET_MODE (count_exp), count_exp);
    9711        27527 :   destreg = ix86_copy_addr_to_reg (XEXP (dst, 0));
    9712        27527 :   if (!issetmem)
    9713        13960 :     srcreg = ix86_copy_addr_to_reg (XEXP (src, 0));
    9714              : 
    9715        27527 :   bool aligned_dstmem = false;
    9716        27527 :   unsigned int nunits = issetmem ? STORE_MAX_PIECES : MOVE_MAX;
    9717        27527 :   bool single_insn_p = count && count <= nunits;
    9718        27527 :   if (single_insn_p)
    9719              :     {
    9720              :       /* If it can be done with a single instruction, use vector
    9721              :          instruction and don't align destination.  */
    9722            7 :       alg = vector_loop;
    9723            7 :       noalign = true;
    9724            7 :       dynamic_check = -1;
    9725              :     }
    9726              : 
    9727        27527 :   unroll_factor = 1;
    9728        27527 :   move_mode = word_mode;
    9729        27527 :   switch (alg)
    9730              :     {
    9731            0 :     case libcall:
    9732            0 :     case no_stringop:
    9733            0 :     case last_alg:
    9734            0 :       gcc_unreachable ();
    9735          173 :     case loop_1_byte:
    9736          173 :       need_zero_guard = true;
    9737          173 :       move_mode = QImode;
    9738          173 :       break;
    9739           40 :     case loop:
    9740           40 :       need_zero_guard = true;
    9741           40 :       break;
    9742           19 :     case unrolled_loop:
    9743           19 :       need_zero_guard = true;
    9744           19 :       unroll_factor = (TARGET_64BIT ? 4 : 2);
    9745              :       break;
    9746         7929 :     case vector_loop:
    9747         7929 :       need_zero_guard = true;
    9748         7929 :       unroll_factor = 4;
    9749              :       /* Get the vector mode to move STORE_MAX_PIECES/MOVE_MAX bytes.  */
    9750         7929 :       nunits /= UNITS_PER_WORD;
    9751         7929 :       if (nunits > 1)
    9752              :         {
    9753         7925 :           move_mode = mode_for_vector (word_mode, nunits).require ();
    9754         7925 :           gcc_assert (optab_handler (mov_optab, move_mode)
    9755              :                       != CODE_FOR_nothing);
    9756              :         }
    9757              :       break;
    9758           25 :     case rep_prefix_8_byte:
    9759           25 :       move_mode = DImode;
    9760           25 :       break;
    9761        13325 :     case rep_prefix_4_byte:
    9762        13325 :       move_mode = SImode;
    9763        13325 :       break;
    9764         6016 :     case rep_prefix_1_byte:
    9765         6016 :       move_mode = QImode;
    9766         6016 :       break;
    9767              :     }
    9768        27527 :   size_needed = GET_MODE_SIZE (move_mode) * unroll_factor;
    9769        27527 :   epilogue_size_needed = size_needed;
    9770              : 
    9771              :   /* If we are going to call any library calls conditionally, make sure any
    9772              :      pending stack adjustment happen before the first conditional branch,
    9773              :      otherwise they will be emitted before the library call only and won't
    9774              :      happen from the other branches.  */
    9775        27527 :   if (dynamic_check != -1)
    9776            2 :     do_pending_stack_adjust ();
    9777              : 
    9778        27527 :   desired_align = decide_alignment (align, alg, expected_size, move_mode);
    9779        27527 :   if (!TARGET_ALIGN_STRINGOPS || noalign)
    9780        27272 :     align = desired_align;
    9781              : 
    9782              :   /* Step 1: Prologue guard.  */
    9783              : 
    9784              :   /* Alignment code needs count to be in register.  */
    9785        27527 :   if (CONST_INT_P (count_exp) && desired_align > align)
    9786              :     {
    9787           22 :       if (INTVAL (count_exp) > desired_align
    9788           22 :           && INTVAL (count_exp) > size_needed)
    9789              :         {
    9790           22 :           align_bytes
    9791           22 :             = get_mem_align_offset (dst, desired_align * BITS_PER_UNIT);
    9792           22 :           if (align_bytes <= 0)
    9793              :             align_bytes = 0;
    9794              :           else
    9795            0 :             align_bytes = desired_align - align_bytes;
    9796              :         }
    9797            0 :       if (align_bytes == 0)
    9798           44 :         count_exp = force_reg (counter_mode (count_exp), count_exp);
    9799              :     }
    9800        27527 :   gcc_assert (desired_align >= 1 && align >= 1);
    9801              : 
    9802        27527 :   if (!single_insn_p)
    9803              :     {
    9804              :       /* Misaligned move sequences handle both prologue and epilogue
    9805              :          at once.  Default code generation results in a smaller code
    9806              :          for large alignments and also avoids redundant job when sizes
    9807              :          are known precisely.  */
    9808        27520 :       misaligned_prologue_used
    9809        55040 :         = (TARGET_MISALIGNED_MOVE_STRING_PRO_EPILOGUES
    9810        27514 :            && MAX (desired_align, epilogue_size_needed) <= 32
    9811        19156 :            && desired_align <= epilogue_size_needed
    9812        32250 :            && ((desired_align > align && !align_bytes)
    9813         4713 :                || (!count && epilogue_size_needed > 1)));
    9814              : 
    9815              :       /* Destination is aligned after the misaligned prologue.  */
    9816        27520 :       aligned_dstmem = misaligned_prologue_used;
    9817              : 
    9818        27520 :       if (noalign
    9819        27265 :           && !misaligned_prologue_used
    9820        27265 :           && (!count
    9821        21834 :               || count > (unsigned HOST_WIDE_INT) epilogue_size_needed))
    9822              :         {
    9823              :           /* Also use misaligned prologue if count > epilogue size,
    9824              :              alignment isn't needed and destination isn't aligned.
    9825              :              Since alignment isn't needed, the destination after
    9826              :              prologue won't be aligned.  */
    9827        27215 :           aligned_dstmem = (GET_MODE_ALIGNMENT (move_mode)
    9828        27215 :                             <= MEM_ALIGN (dst));
    9829        27215 :           if (!aligned_dstmem)
    9830         3989 :             misaligned_prologue_used = true;
    9831              :         }
    9832              :     }
    9833              : 
    9834              :   /* Do the cheap promotion to allow better CSE across the
    9835              :      main loop and epilogue (ie one load of the big constant in the
    9836              :      front of all code.
    9837              :      For now the misaligned move sequences do not have fast path
    9838              :      without broadcasting.  */
    9839        27527 :   if (issetmem
    9840        13567 :       && (alg == vector_loop
    9841         8252 :           || CONST_INT_P (val_exp)
    9842           49 :           || misaligned_prologue_used))
    9843              :     {
    9844         8204 :       if (alg == vector_loop)
    9845              :         {
    9846         5315 :           promoted_val = promote_duplicated_reg_to_size (val_exp,
    9847         5315 :                                                          UNITS_PER_WORD,
    9848              :                                                          desired_align, align);
    9849              :           /* Duplicate the promoted scalar value if not 0 nor -1.  */
    9850         5315 :           vec_promoted_val
    9851         5315 :             = promote_duplicated_reg (move_mode,
    9852         5315 :                                       (val_exp == const0_rtx
    9853          582 :                                        || val_exp == constm1_rtx)
    9854              :                                       ? val_exp : promoted_val);
    9855              :         }
    9856              :       else
    9857              :         {
    9858         8204 :           promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
    9859              :                                                          desired_align, align);
    9860              :         }
    9861              :     }
    9862              :   /* Misaligned move sequences handles both prologues and epilogues at once.
    9863              :      Default code generation results in smaller code for large alignments and
    9864              :      also avoids redundant job when sizes are known precisely.  */
    9865        27479 :   if (misaligned_prologue_used)
    9866              :     {
    9867              :       /* Misaligned move prologue handled small blocks by itself.  */
    9868         4006 :       expand_set_or_cpymem_prologue_epilogue_by_misaligned_moves
    9869         4006 :            (dst, src, &destreg, &srcreg,
    9870              :             move_mode, promoted_val, vec_promoted_val,
    9871              :             &count_exp,
    9872              :             &jump_around_label,
    9873         4006 :             desired_align < align
    9874            0 :             ? MAX (desired_align, epilogue_size_needed) : epilogue_size_needed,
    9875              :             desired_align, align, &min_size, dynamic_check, issetmem);
    9876         4006 :       if (!issetmem)
    9877         2094 :         src = change_address (src, BLKmode, srcreg);
    9878         4006 :       dst = change_address (dst, BLKmode, destreg);
    9879         4006 :       if (aligned_dstmem)
    9880           17 :         set_mem_align (dst, desired_align * BITS_PER_UNIT);
    9881         4006 :       epilogue_size_needed = 0;
    9882         4006 :       if (need_zero_guard
    9883         3727 :           && min_size < (unsigned HOST_WIDE_INT) size_needed)
    9884              :         {
    9885              :           /* It is possible that we copied enough so the main loop will not
    9886              :              execute.  */
    9887          645 :           gcc_assert (size_needed > 1);
    9888          645 :           if (jump_around_label == NULL_RTX)
    9889            0 :             jump_around_label = gen_label_rtx ();
    9890         1290 :           emit_cmp_and_jump_insns (count_exp,
    9891              :                                    GEN_INT (size_needed),
    9892              :                                    LTU, 0, counter_mode (count_exp), 1, jump_around_label);
    9893          645 :           if (expected_size == -1
    9894            0 :               || expected_size < (desired_align - align) / 2 + size_needed)
    9895          645 :             predict_jump (REG_BR_PROB_BASE * 20 / 100);
    9896              :           else
    9897            0 :             predict_jump (REG_BR_PROB_BASE * 60 / 100);
    9898              :         }
    9899              :     }
    9900              :   /* Ensure that alignment prologue won't copy past end of block.  */
    9901        23521 :   else if (size_needed > 1 || (desired_align > 1 && desired_align > align))
    9902              :     {
    9903        17332 :       epilogue_size_needed = MAX (size_needed - 1, desired_align - align);
    9904              :       /* Epilogue always copies COUNT_EXP & EPILOGUE_SIZE_NEEDED bytes.
    9905              :          Make sure it is power of 2.  */
    9906        17332 :       epilogue_size_needed = 1 << (floor_log2 (epilogue_size_needed) + 1);
    9907              : 
    9908              :       /* To improve performance of small blocks, we jump around the VAL
    9909              :          promoting mode.  This mean that if the promoted VAL is not constant,
    9910              :          we might not use it in the epilogue and have to use byte
    9911              :          loop variant.  */
    9912        17332 :       if (issetmem && epilogue_size_needed > 2 && !promoted_val)
    9913        17332 :         force_loopy_epilogue = true;
    9914        17332 :       if ((count && count < (unsigned HOST_WIDE_INT) epilogue_size_needed)
    9915        17322 :           || max_size < (unsigned HOST_WIDE_INT) epilogue_size_needed)
    9916              :         {
    9917              :           /* If main algorithm works on QImode, no epilogue is needed.
    9918              :              For small sizes just don't align anything.  */
    9919           44 :           if (size_needed == 1)
    9920            0 :             desired_align = align;
    9921              :           else
    9922           44 :             goto epilogue;
    9923              :         }
    9924        17288 :       else if (!count
    9925           62 :                && min_size < (unsigned HOST_WIDE_INT) epilogue_size_needed)
    9926              :         {
    9927           62 :           label = gen_label_rtx ();
    9928          124 :           emit_cmp_and_jump_insns (count_exp,
    9929              :                                    GEN_INT (epilogue_size_needed),
    9930              :                                    LTU, 0, counter_mode (count_exp), 1, label);
    9931           62 :           if (expected_size == -1 || expected_size < epilogue_size_needed)
    9932           62 :             predict_jump (REG_BR_PROB_BASE * 60 / 100);
    9933              :           else
    9934            0 :             predict_jump (REG_BR_PROB_BASE * 20 / 100);
    9935              :         }
    9936              :     }
    9937              : 
    9938              :   /* Emit code to decide on runtime whether library call or inline should be
    9939              :      used.  */
    9940        27483 :   if (dynamic_check != -1)
    9941              :     {
    9942            2 :       if (!issetmem && CONST_INT_P (count_exp))
    9943              :         {
    9944            1 :           if (UINTVAL (count_exp) >= (unsigned HOST_WIDE_INT)dynamic_check)
    9945              :             {
    9946            1 :               emit_block_copy_via_libcall (dst, src, count_exp);
    9947            1 :               count_exp = const0_rtx;
    9948            1 :               goto epilogue;
    9949              :             }
    9950              :         }
    9951              :       else
    9952              :         {
    9953            1 :           rtx_code_label *hot_label = gen_label_rtx ();
    9954            1 :           if (jump_around_label == NULL_RTX)
    9955            1 :             jump_around_label = gen_label_rtx ();
    9956            2 :           emit_cmp_and_jump_insns (count_exp, GEN_INT (dynamic_check - 1),
    9957              :                                    LEU, 0, counter_mode (count_exp),
    9958              :                                    1, hot_label);
    9959            1 :           predict_jump (REG_BR_PROB_BASE * 90 / 100);
    9960            1 :           if (issetmem)
    9961            1 :             set_storage_via_libcall (dst, count_exp, val_exp);
    9962              :           else
    9963            0 :             emit_block_copy_via_libcall (dst, src, count_exp);
    9964            1 :           emit_jump (jump_around_label);
    9965            1 :           emit_label (hot_label);
    9966              :         }
    9967              :     }
    9968              : 
    9969              :   /* Step 2: Alignment prologue.  */
    9970              :   /* Do the expensive promotion once we branched off the small blocks.  */
    9971        27482 :   if (issetmem && !promoted_val)
    9972           48 :     promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
    9973              :                                                    desired_align, align);
    9974              : 
    9975        27482 :   if (desired_align > align && !misaligned_prologue_used)
    9976              :     {
    9977            7 :       if (align_bytes == 0)
    9978              :         {
    9979              :           /* Except for the first move in prologue, we no longer know
    9980              :              constant offset in aliasing info.  It don't seems to worth
    9981              :              the pain to maintain it for the first move, so throw away
    9982              :              the info early.  */
    9983            7 :           dst = change_address (dst, BLKmode, destreg);
    9984            7 :           if (!issetmem)
    9985            5 :             src = change_address (src, BLKmode, srcreg);
    9986            7 :           dst = expand_set_or_cpymem_prologue (dst, src, destreg, srcreg,
    9987              :                                             promoted_val, vec_promoted_val,
    9988              :                                             count_exp, align, desired_align,
    9989              :                                             issetmem);
    9990              :           /* At most desired_align - align bytes are copied.  */
    9991            7 :           if (min_size < (unsigned)(desired_align - align))
    9992            0 :             min_size = 0;
    9993              :           else
    9994            7 :             min_size -= desired_align - align;
    9995              :         }
    9996              :       else
    9997              :         {
    9998              :           /* If we know how many bytes need to be stored before dst is
    9999              :              sufficiently aligned, maintain aliasing info accurately.  */
   10000            0 :           dst = expand_set_or_cpymem_constant_prologue (dst, &src, destreg,
   10001              :                                                            srcreg,
   10002              :                                                            promoted_val,
   10003              :                                                            vec_promoted_val,
   10004              :                                                            desired_align,
   10005              :                                                            align_bytes,
   10006              :                                                            issetmem);
   10007              : 
   10008            0 :           count_exp = plus_constant (counter_mode (count_exp),
   10009            0 :                                      count_exp, -align_bytes);
   10010            0 :           count -= align_bytes;
   10011            0 :           min_size -= align_bytes;
   10012            0 :           max_size -= align_bytes;
   10013              :         }
   10014            7 :       if (need_zero_guard
   10015            7 :           && min_size < (unsigned HOST_WIDE_INT) size_needed
   10016            1 :           && (count < (unsigned HOST_WIDE_INT) size_needed
   10017            0 :               || (align_bytes == 0
   10018            0 :                   && count < ((unsigned HOST_WIDE_INT) size_needed
   10019            0 :                               + desired_align - align))))
   10020              :         {
   10021              :           /* It is possible that we copied enough so the main loop will not
   10022              :              execute.  */
   10023            1 :           gcc_assert (size_needed > 1);
   10024            1 :           if (label == NULL_RTX)
   10025            0 :             label = gen_label_rtx ();
   10026            2 :           emit_cmp_and_jump_insns (count_exp,
   10027              :                                    GEN_INT (size_needed),
   10028              :                                    LTU, 0, counter_mode (count_exp), 1, label);
   10029            1 :           if (expected_size == -1
   10030            0 :               || expected_size < (desired_align - align) / 2 + size_needed)
   10031            1 :             predict_jump (REG_BR_PROB_BASE * 20 / 100);
   10032              :           else
   10033            0 :             predict_jump (REG_BR_PROB_BASE * 60 / 100);
   10034              :         }
   10035              :     }
   10036        27482 :   if (label && size_needed == 1)
   10037              :     {
   10038            0 :       emit_label (label);
   10039            0 :       LABEL_NUSES (label) = 1;
   10040            0 :       label = NULL;
   10041            0 :       epilogue_size_needed = 1;
   10042            0 :       if (issetmem)
   10043            0 :         promoted_val = val_exp;
   10044              :     }
   10045        27482 :   else if (label == NULL_RTX && !misaligned_prologue_used)
   10046        23415 :     epilogue_size_needed = size_needed;
   10047              : 
   10048              :   /* Step 3: Main loop.  */
   10049              : 
   10050        27482 :   switch (alg)
   10051              :     {
   10052              :     case libcall:
   10053              :     case no_stringop:
   10054              :     case last_alg:
   10055              :       gcc_unreachable ();
   10056          232 :     case loop_1_byte:
   10057          232 :     case loop:
   10058          232 :     case unrolled_loop:
   10059          232 :       expand_set_or_cpymem_via_loop (dst, src, destreg, srcreg, promoted_val,
   10060              :                                      count_exp, move_mode, unroll_factor,
   10061              :                                      expected_size, issetmem);
   10062          232 :       break;
   10063         7884 :     case vector_loop:
   10064         7884 :       expand_set_or_cpymem_via_loop (dst, src, destreg, srcreg,
   10065              :                                      vec_promoted_val, count_exp, move_mode,
   10066              :                                      unroll_factor, expected_size, issetmem);
   10067         7884 :       break;
   10068        19366 :     case rep_prefix_8_byte:
   10069        19366 :     case rep_prefix_4_byte:
   10070        19366 :     case rep_prefix_1_byte:
   10071        19366 :       expand_set_or_cpymem_via_rep (dst, src, destreg, srcreg, promoted_val,
   10072              :                                        val_exp, count_exp, move_mode, issetmem);
   10073        19366 :       break;
   10074              :     }
   10075              :   /* Adjust properly the offset of src and dest memory for aliasing.  */
   10076        27482 :   if (CONST_INT_P (count_exp))
   10077              :     {
   10078        21892 :       if (!issetmem)
   10079         9395 :         src = adjust_automodify_address_nv (src, BLKmode, srcreg,
   10080              :                                             (count / size_needed) * size_needed);
   10081        21892 :       dst = adjust_automodify_address_nv (dst, BLKmode, destreg,
   10082              :                                           (count / size_needed) * size_needed);
   10083              :     }
   10084              :   else
   10085              :     {
   10086         5590 :       if (!issetmem)
   10087         4562 :         src = change_address (src, BLKmode, srcreg);
   10088         5590 :       dst = change_address (dst, BLKmode, destreg);
   10089              :     }
   10090              : 
   10091              :   /* Step 4: Epilogue to copy the remaining bytes.  */
   10092        27527 :  epilogue:
   10093        27527 :   if (label)
   10094              :     {
   10095              :       /* When the main loop is done, COUNT_EXP might hold original count,
   10096              :          while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
   10097              :          Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
   10098              :          bytes. Compensate if needed.  */
   10099              : 
   10100           62 :       if (size_needed < epilogue_size_needed)
   10101              :         {
   10102            0 :           tmp = expand_simple_binop (counter_mode (count_exp), AND, count_exp,
   10103            0 :                                      GEN_INT (size_needed - 1), count_exp, 1,
   10104              :                                      OPTAB_DIRECT);
   10105            0 :           if (tmp != count_exp)
   10106            0 :             emit_move_insn (count_exp, tmp);
   10107              :         }
   10108           62 :       emit_label (label);
   10109           62 :       LABEL_NUSES (label) = 1;
   10110              :     }
   10111              : 
   10112        27527 :   if (count_exp != const0_rtx && epilogue_size_needed > 1)
   10113              :     {
   10114        17332 :       if (force_loopy_epilogue)
   10115            0 :         expand_setmem_epilogue_via_loop (dst, destreg, val_exp, count_exp,
   10116              :                                          epilogue_size_needed);
   10117              :       else
   10118              :         {
   10119        17332 :           if (issetmem)
   10120        10043 :             expand_setmem_epilogue (dst, destreg, promoted_val,
   10121              :                                     vec_promoted_val, count_exp,
   10122              :                                     epilogue_size_needed);
   10123              :           else
   10124         7289 :             expand_cpymem_epilogue (dst, src, destreg, srcreg, count_exp,
   10125              :                                     epilogue_size_needed);
   10126              :         }
   10127              :     }
   10128        27527 :   if (jump_around_label)
   10129          648 :     emit_label (jump_around_label);
   10130              :   return true;
   10131              : }
   10132              : 
   10133              : /* Fully unroll memmove of known size with up to 8 registers.  */
   10134              : 
   10135              : static bool
   10136         1896 : ix86_expand_unroll_movmem (rtx dst, rtx src, rtx destreg, rtx srcreg,
   10137              :                            unsigned HOST_WIDE_INT count,
   10138              :                            machine_mode mode)
   10139              : {
   10140              :   /* If 8 registers registers can cover all memory, load them into
   10141              :      registers and store them together to avoid possible address
   10142              :      overlap between source and destination.  */
   10143         1896 :   unsigned HOST_WIDE_INT moves = count / GET_MODE_SIZE (mode);
   10144         1896 :   if (moves == 0)
   10145              :     {
   10146            0 :       mode = smallest_int_mode_for_size
   10147            0 :         (count * BITS_PER_UNIT).require ();
   10148            0 :       if (count == GET_MODE_SIZE (mode))
   10149              :         moves = 1;
   10150              :       else
   10151              :         {
   10152              :           /* Reduce the smallest move size by half so that MOVES == 1.  */
   10153            0 :           mode = smallest_int_mode_for_size
   10154            0 :             (GET_MODE_BITSIZE (mode) / 2).require ();
   10155            0 :           moves = count / GET_MODE_SIZE (mode);
   10156            0 :           gcc_assert (moves == 1);
   10157              :         }
   10158              :     }
   10159         1896 :   else if (moves > 8)
   10160              :     return false;
   10161              : 
   10162         1887 :   unsigned int i;
   10163         1887 :   rtx tmp[9];
   10164              : 
   10165         4342 :   for (i = 0; i < moves; i++)
   10166         2455 :     tmp[i] = gen_reg_rtx (mode);
   10167              : 
   10168         1887 :   rtx srcmem = change_address (src, mode, srcreg);
   10169         6229 :   for (i = 0; i < moves; i++)
   10170              :     {
   10171         2455 :       emit_move_insn (tmp[i], srcmem);
   10172         4910 :       srcmem = offset_address (srcmem,
   10173         2455 :                                GEN_INT (GET_MODE_SIZE (mode)),
   10174         2455 :                                GET_MODE_SIZE (mode));
   10175              :     }
   10176              : 
   10177         1887 :   unsigned int epilogue_size = count & (GET_MODE_SIZE (mode) - 1);
   10178         1887 :   machine_mode epilogue_mode = VOIDmode;
   10179         1887 :   if (epilogue_size)
   10180              :     {
   10181              :       /* Handle the remaining bytes with overlapping move.  */
   10182         1714 :       epilogue_mode = smallest_int_mode_for_size
   10183         1714 :         (epilogue_size * BITS_PER_UNIT).require ();
   10184         1714 :       tmp[8] = gen_reg_rtx (epilogue_mode);
   10185         1714 :       srcmem = adjust_address (srcmem, epilogue_mode, 0);
   10186         1714 :       srcmem = offset_address (srcmem, GEN_INT (epilogue_size), 1);
   10187         3428 :       srcmem = offset_address (srcmem,
   10188         1714 :                                GEN_INT (-GET_MODE_SIZE (epilogue_mode)),
   10189         1714 :                                GET_MODE_SIZE (epilogue_mode));
   10190         1714 :       emit_move_insn (tmp[8], srcmem);
   10191              :     }
   10192              : 
   10193         1887 :   rtx destmem = change_address (dst, mode, destreg);
   10194         6229 :   for (i = 0; i < moves; i++)
   10195              :     {
   10196         2455 :       emit_move_insn (destmem, tmp[i]);
   10197         4910 :       destmem = offset_address (destmem,
   10198         2455 :                                 GEN_INT (GET_MODE_SIZE (mode)),
   10199         2455 :                                 GET_MODE_SIZE (mode));
   10200              :     }
   10201              : 
   10202         1887 :   if (epilogue_size)
   10203              :     {
   10204              :       /* Use overlapping move.  */
   10205         1714 :       destmem = adjust_address (destmem, epilogue_mode, 0);
   10206         1714 :       destmem = offset_address (destmem, GEN_INT (epilogue_size), 1);
   10207         3428 :       destmem = offset_address (destmem,
   10208         1714 :                                 GEN_INT (-GET_MODE_SIZE (epilogue_mode)),
   10209         1714 :                                 GET_MODE_SIZE (epilogue_mode));
   10210         1714 :       emit_move_insn (destmem, tmp[8]);
   10211              :     }
   10212              : 
   10213              :   return true;
   10214              : }
   10215              : 
   10216              : /* Value kind in MEMSET_VALS:
   10217              : 
   10218              :    memset_val_byte:   The value rtx in QImode.
   10219              :    memset_val_word:   The value rtx in word_mode.
   10220              :    memset_val_vector: The value rtx in QI vector mode.
   10221              : 
   10222              :  */
   10223              : enum memset_val_kind
   10224              : {
   10225              :   memset_val_byte = 0,
   10226              :   memset_val_word = 1,
   10227              :   memset_val_vector = 2,
   10228              :   memset_val_max = 3
   10229              : };
   10230              : 
   10231              : /* Return a value rtx in MODE for memset from MEMSET_VALS.  */
   10232              : 
   10233              : static rtx
   10234         4318 : ix86_expand_memset_val (rtx *memset_vals, machine_mode mode)
   10235              : {
   10236         4318 :   rtx byte_val = memset_vals[memset_val_byte];
   10237              : 
   10238         4318 :   if (mode == QImode)
   10239              :     return byte_val;
   10240         4318 :   else if (mode == word_mode)
   10241          985 :     return memset_vals[memset_val_word];
   10242              : 
   10243              :   /* All-zero/all-ones is a property of the original byte value, so
   10244              :      detect it once here rather than re-deriving it from each slot.  */
   10245         3333 :   if (byte_val == const0_rtx)
   10246         1254 :     return CONST0_RTX (mode);
   10247         2079 :   if (byte_val == constm1_rtx)
   10248           96 :     return CONSTM1_RTX (mode);
   10249              : 
   10250         1983 :   if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
   10251              :     {
   10252          688 :       if (GET_MODE (memset_vals[memset_val_vector]) == mode)
   10253              :         return memset_vals[memset_val_vector];
   10254            7 :       return gen_rtx_SUBREG (mode, memset_vals[memset_val_vector], 0);
   10255              :     }
   10256              : 
   10257         1295 :   gcc_assert (mode == HImode || mode == SImode);
   10258         1295 :   return gen_rtx_SUBREG (mode, memset_vals[memset_val_word], 0);
   10259              : }
   10260              : 
   10261              : /* Expand memmove of size with MOVES * mode size and MOVES <= 4.  If
   10262              :    FORWARD is true, copy forward.  Otherwise copy backward.  */
   10263              : 
   10264              : static void
   10265         2419 : ix86_expand_n_move_set_or_movmem (rtx destmem, rtx srcmem,
   10266              :                                   rtx *memset_vals, machine_mode mode,
   10267              :                                   unsigned int moves, bool forward)
   10268              : {
   10269         2419 :   gcc_assert (moves <= 4);
   10270              : 
   10271         2419 :   unsigned int i;
   10272         2419 :   rtx tmp[8];
   10273              : 
   10274         2419 :   rtx step;
   10275         2419 :   if (forward)
   10276         2542 :     step = GEN_INT (GET_MODE_SIZE (mode));
   10277              :   else
   10278         2296 :     step = GEN_INT (-GET_MODE_SIZE (mode));
   10279              : 
   10280         2419 :   if (memset_vals)
   10281              :     {
   10282              :       /* Expand memset.  */
   10283           95 :       rtx val = ix86_expand_memset_val (memset_vals, mode);
   10284          570 :       for (i = 0; i < moves; i++)
   10285          380 :         tmp[i] = val;
   10286              :     }
   10287              :   else
   10288              :     {
   10289              :       /* Expand memmove.  */
   10290        11620 :       for (i = 0; i < moves; i++)
   10291         9296 :         tmp[i] = gen_reg_rtx (mode);
   10292              : 
   10293              :       /* Load MOVES.  */
   10294         9296 :       for (i = 0; i < moves - 1; i++)
   10295              :         {
   10296         6972 :           emit_move_insn (tmp[i], srcmem);
   10297        13944 :           srcmem = offset_address (srcmem, step, GET_MODE_SIZE (mode));
   10298              :         }
   10299         2324 :       emit_move_insn (tmp[i], srcmem);
   10300              :     }
   10301              : 
   10302              :   /* Store MOVES.  */
   10303         9676 :   for (i = 0; i < moves - 1; i++)
   10304              :     {
   10305         7257 :       emit_move_insn (destmem, tmp[i]);
   10306        14514 :       destmem = offset_address (destmem, step, GET_MODE_SIZE (mode));
   10307              :     }
   10308         2419 :   emit_move_insn (destmem, tmp[i]);
   10309         2419 : }
   10310              : 
   10311              : /* Load MOVES of mode size into REGS.  If LAST is true, load the
   10312              :    last MOVES.  Otherwise, load the first MOVES.  */
   10313              : 
   10314              : static void
   10315         2324 : ix86_expand_load_movmem (rtx src, rtx srcreg, rtx count_exp,
   10316              :                          machine_mode mode, unsigned int moves,
   10317              :                          rtx regs[], bool last)
   10318              : {
   10319         2324 :   unsigned int i;
   10320              : 
   10321        11620 :   for (i = 0; i < moves; i++)
   10322         9296 :     regs[i] = gen_reg_rtx (mode);
   10323              : 
   10324         2324 :   rtx srcmem = change_address (src, mode, srcreg);
   10325         2324 :   rtx step;
   10326         2324 :   if (last)
   10327              :     {
   10328         1176 :       srcmem = offset_address (srcmem, count_exp, 1);
   10329         2352 :       step = GEN_INT (-GET_MODE_SIZE (mode));
   10330         2352 :       srcmem = offset_address (srcmem, step, GET_MODE_SIZE (mode));
   10331              :     }
   10332              :   else
   10333         2296 :     step = GEN_INT (GET_MODE_SIZE (mode));
   10334              : 
   10335         9296 :   for (i = 0; i < moves - 1; i++)
   10336              :     {
   10337         6972 :       emit_move_insn (regs[i], srcmem);
   10338        13944 :       srcmem = offset_address (srcmem, step, GET_MODE_SIZE (mode));
   10339              :     }
   10340         2324 :   emit_move_insn (regs[i], srcmem);
   10341         2324 : }
   10342              : 
   10343              : /* Store MOVES of mode size into REGS.  If LAST is true, store the
   10344              :    last MOVES.  Otherwise, store the first MOVES.  */
   10345              : 
   10346              : static void
   10347         2419 : ix86_expand_store_movmem (rtx dst, rtx destreg, rtx count_exp,
   10348              :                           machine_mode mode, unsigned int moves,
   10349              :                           rtx regs[], bool last)
   10350              : {
   10351         2419 :   unsigned int i;
   10352              : 
   10353         2419 :   rtx destmem = change_address (dst, mode, destreg);
   10354         2419 :   rtx step;
   10355         2419 :   if (last)
   10356              :     {
   10357         1271 :       destmem = offset_address (destmem, count_exp, 1);
   10358         2542 :       step = GEN_INT (-GET_MODE_SIZE (mode));
   10359         2542 :       destmem = offset_address (destmem, step, GET_MODE_SIZE (mode));
   10360              :     }
   10361              :   else
   10362         2296 :     step = GEN_INT (GET_MODE_SIZE (mode));
   10363              : 
   10364         9676 :   for (i = 0; i < moves - 1; i++)
   10365              :     {
   10366         7257 :       emit_move_insn (destmem, regs[i]);
   10367        14514 :       destmem = offset_address (destmem, step, GET_MODE_SIZE (mode));
   10368              :     }
   10369         2419 :   emit_move_insn (destmem, regs[i]);
   10370         2419 : }
   10371              : 
   10372              : /* Expand memmove of size between (MOVES / 2) * mode size and
   10373              :    MOVES * mode size with overlapping load and store.  MOVES is even.
   10374              :    MOVES >= 2 and MOVES <= 8.  */
   10375              : 
   10376              : static void
   10377        44171 : ix86_expand_n_overlapping_move_set_or_movmem (rtx dst, rtx src,
   10378              :                                               rtx *memset_vals,
   10379              :                                               rtx destreg, rtx srcreg,
   10380              :                                               rtx count_exp,
   10381              :                                               machine_mode mode,
   10382              :                                               unsigned int moves)
   10383              : {
   10384        44171 :   gcc_assert (moves >= 2 && moves <= 8 && (moves & 1) == 0);
   10385              : 
   10386        44171 :   unsigned int half_moves = moves / 2;
   10387        44171 :   unsigned int i, j;
   10388        44171 :   rtx tmp[8];
   10389              : 
   10390        44171 :   if (memset_vals)
   10391              :     {
   10392              :       /* Expand memset.  */
   10393         4128 :       rtx val = ix86_expand_memset_val (memset_vals, mode);
   10394        18500 :       for (i = 0; i < moves; i++)
   10395        10244 :         tmp[i] = val;
   10396              :     }
   10397              :   else
   10398              :     {
   10399              :       /* Expand memmove.  */
   10400       134899 :       for (i = 0; i < moves; i++)
   10401        94856 :         tmp[i] = gen_reg_rtx (mode);
   10402              : 
   10403        40043 :       rtx base_srcmem = change_address (src, mode, srcreg);
   10404              : 
   10405              :       /* Load the first half.  */
   10406        40043 :       rtx srcmem = base_srcmem;
   10407        87471 :       for (i = 0; i < half_moves - 1; i++)
   10408              :         {
   10409         7385 :           emit_move_insn (tmp[i], srcmem);
   10410        14770 :           srcmem = offset_address (srcmem,
   10411         7385 :                                    GEN_INT (GET_MODE_SIZE (mode)),
   10412         7385 :                                    GET_MODE_SIZE (mode));
   10413              :         }
   10414        40043 :       emit_move_insn (tmp[i], srcmem);
   10415              : 
   10416              :       /* Load the second half.  */
   10417        40043 :       srcmem = offset_address (base_srcmem, count_exp, 1);
   10418        40043 :       srcmem = offset_address (srcmem,
   10419        40043 :                                GEN_INT (-GET_MODE_SIZE (mode)),
   10420        40043 :                                GET_MODE_SIZE (mode));
   10421        87471 :       for (j = half_moves, i = 0; i < half_moves - 1; i++, j++)
   10422              :         {
   10423         7385 :           emit_move_insn (tmp[j], srcmem);
   10424        14770 :           srcmem = offset_address (srcmem,
   10425         7385 :                                    GEN_INT (-GET_MODE_SIZE (mode)),
   10426         7385 :                                    GET_MODE_SIZE (mode));
   10427              :         }
   10428        40043 :       emit_move_insn (tmp[j], srcmem);
   10429              :     }
   10430              : 
   10431        44171 :   rtx base_destmem = change_address (dst, mode, destreg);
   10432              : 
   10433              :   /* Store the first half.  */
   10434        44171 :   rtx destmem = base_destmem;
   10435        96721 :   for (i = 0; i < half_moves - 1; i++)
   10436              :     {
   10437         8379 :       emit_move_insn (destmem, tmp[i]);
   10438        16758 :       destmem = offset_address (destmem,
   10439         8379 :                                 GEN_INT (GET_MODE_SIZE (mode)),
   10440         8379 :                                 GET_MODE_SIZE (mode));
   10441              :     }
   10442        44171 :   emit_move_insn (destmem, tmp[i]);
   10443              : 
   10444              :   /* Store the second half.  */
   10445        44171 :   destmem = offset_address (base_destmem, count_exp, 1);
   10446        88342 :   destmem = offset_address (destmem, GEN_INT (-GET_MODE_SIZE (mode)),
   10447        44171 :                             GET_MODE_SIZE (mode));
   10448        96721 :   for (j = half_moves, i = 0; i < half_moves - 1; i++, j++)
   10449              :     {
   10450         8379 :       emit_move_insn (destmem, tmp[j]);
   10451        16758 :       destmem = offset_address (destmem, GEN_INT (-GET_MODE_SIZE (mode)),
   10452         8379 :                                 GET_MODE_SIZE (mode));
   10453              :     }
   10454        44171 :   emit_move_insn (destmem, tmp[j]);
   10455        44171 : }
   10456              : 
   10457              : /* Expand memmove of size < mode size which is <= 64.  */
   10458              : 
   10459              : static void
   10460        10912 : ix86_expand_less_move_set_or_movmem (rtx dst, rtx src, rtx *memset_vals,
   10461              :                                      rtx destreg, rtx srcreg,
   10462              :                                      rtx count_exp,
   10463              :                                      unsigned HOST_WIDE_INT min_size,
   10464              :                                      machine_mode mode,
   10465              :                                      rtx_code_label *done_label)
   10466              : {
   10467        10912 :   bool skip = false;
   10468        10912 :   machine_mode count_mode = counter_mode (count_exp);
   10469              : 
   10470        10912 :   rtx_code_label *between_32_63_label
   10471        10912 :     = GET_MODE_SIZE (mode) > 32 ? gen_label_rtx () : nullptr;
   10472              :   /* Jump to BETWEEN_32_64_LABEL if size >= 32 and size < 64.  */
   10473            7 :   if (between_32_63_label)
   10474              :     {
   10475            7 :       if (min_size && min_size >= 32)
   10476              :         {
   10477            1 :           emit_jump_insn (gen_jump (between_32_63_label));
   10478            1 :           emit_barrier ();
   10479            1 :           skip = true;
   10480              :         }
   10481              :       else
   10482            6 :         emit_cmp_and_jump_insns (count_exp, GEN_INT (32), GEU,
   10483              :                                  nullptr, count_mode, 1,
   10484              :                                  between_32_63_label);
   10485              :     }
   10486              : 
   10487            7 :   rtx_code_label *between_16_31_label
   10488        10911 :     = (!skip && GET_MODE_SIZE (mode) > 16) ? gen_label_rtx () : nullptr;
   10489              :   /* Jump to BETWEEN_16_31_LABEL if size >= 16 and size < 31.  */
   10490           21 :   if (between_16_31_label)
   10491              :     {
   10492           21 :       if (min_size && min_size >= 16)
   10493              :         {
   10494            2 :           emit_jump_insn (gen_jump (between_16_31_label));
   10495            2 :           emit_barrier ();
   10496            2 :           skip = true;
   10497              :         }
   10498              :       else
   10499           19 :         emit_cmp_and_jump_insns (count_exp, GEN_INT (16), GEU,
   10500              :                                  nullptr, count_mode, 1,
   10501              :                                  between_16_31_label);
   10502              :     }
   10503              : 
   10504            2 :   rtx_code_label *between_8_15_label
   10505        21819 :     = (!skip && GET_MODE_SIZE (mode) > 8) ? gen_label_rtx () : nullptr;
   10506              :   /* Jump to BETWEEN_8_15_LABEL if size >= 8 and size < 15.  */
   10507         8665 :   if (between_8_15_label)
   10508              :     {
   10509         8665 :       if (min_size && min_size >= 8)
   10510              :         {
   10511          476 :           emit_jump_insn (gen_jump (between_8_15_label));
   10512          476 :           emit_barrier ();
   10513          476 :           skip = true;
   10514              :         }
   10515              :       else
   10516         8189 :         emit_cmp_and_jump_insns (count_exp, GEN_INT (8), GEU,
   10517              :                                  nullptr, count_mode, 1,
   10518              :                                  between_8_15_label);
   10519              :     }
   10520              : 
   10521          476 :   rtx_code_label *between_4_7_label
   10522        20869 :     = (!skip && GET_MODE_SIZE (mode) > 4) ? gen_label_rtx () : nullptr;
   10523              :   /* Jump to BETWEEN_4_7_LABEL if size >= 4 and size < 7.  */
   10524         9517 :   if (between_4_7_label)
   10525              :     {
   10526         9517 :       if (min_size && min_size >= 4)
   10527              :         {
   10528          385 :           emit_jump_insn (gen_jump (between_4_7_label));
   10529          385 :           emit_barrier ();
   10530          385 :           skip = true;
   10531              :         }
   10532              :       else
   10533         9132 :         emit_cmp_and_jump_insns (count_exp, GEN_INT (4), GEU,
   10534              :                                  nullptr, count_mode, 1,
   10535              :                                  between_4_7_label);
   10536              :     }
   10537              : 
   10538          385 :   rtx_code_label *between_2_3_label
   10539        20575 :     = (!skip && GET_MODE_SIZE (mode) > 2) ? gen_label_rtx () : nullptr;
   10540              :   /* Jump to BETWEEN_2_3_LABEL if size >= 2 and size < 3.  */
   10541         9822 :   if (between_2_3_label)
   10542              :     {
   10543         9822 :       if (min_size && min_size >= 2)
   10544              :         {
   10545         2076 :           emit_jump_insn (gen_jump (between_2_3_label));
   10546         2076 :           emit_barrier ();
   10547         2076 :           skip = true;
   10548              :         }
   10549              :       else
   10550         7746 :         emit_cmp_and_jump_insns (count_exp, GEN_INT (1), GT,
   10551              :                                  nullptr, count_mode, 1,
   10552              :                                  between_2_3_label);
   10553              :     }
   10554              : 
   10555        10912 :   if (!skip)
   10556              :     {
   10557         7972 :       rtx_code_label *zero_label
   10558         7972 :         = min_size == 0 ? gen_label_rtx () : nullptr;
   10559              :       /* Skip if size == 0.  */
   10560         2716 :       if (zero_label)
   10561         2716 :         emit_cmp_and_jump_insns (count_exp, GEN_INT (1), LT,
   10562              :                                  nullptr, count_mode, 1,
   10563              :                                  zero_label,
   10564              :                                  profile_probability::unlikely ());
   10565              : 
   10566              :       /* Move 1 byte.  */
   10567         7972 :       rtx tmp0;
   10568              :       /* Use the value rtx in QImode for memset.  */
   10569         7972 :       if (memset_vals)
   10570          912 :         tmp0 = memset_vals[memset_val_byte];
   10571              :       else
   10572              :         {
   10573         7060 :           tmp0 = gen_reg_rtx (QImode);
   10574         7060 :           rtx srcmem = change_address (src, QImode, srcreg);
   10575         7060 :           emit_move_insn (tmp0, srcmem);
   10576              :         }
   10577         7972 :       rtx destmem = change_address (dst, QImode, destreg);
   10578         7972 :       emit_move_insn (destmem, tmp0);
   10579              : 
   10580         7972 :       if (zero_label)
   10581         2716 :         emit_label (zero_label);
   10582              : 
   10583         7972 :       emit_jump_insn (gen_jump (done_label));
   10584         7972 :       emit_barrier ();
   10585              :     }
   10586              : 
   10587              :   /* For each size band, memset uses a QI-vector mode above a word so it
   10588              :      can broadcast the fill value, while memmove uses the same-size
   10589              :      scalar integer mode; at and below a word both use the scalar
   10590              :      integer mode.  */
   10591        10912 :   struct {
   10592              :     rtx_code_label *label;
   10593              :     machine_mode set_mode;
   10594              :     machine_mode move_mode;
   10595        10912 :   } bands[] = {
   10596              :     { between_32_63_label, V32QImode, OImode },
   10597              :     { between_16_31_label, V16QImode, TImode },
   10598              :     { between_8_15_label,  DImode,    DImode },
   10599              :     { between_4_7_label,   SImode,    SImode },
   10600              :     { between_2_3_label,   HImode,    HImode },
   10601        10912 :   };
   10602              : 
   10603        65472 :   for (auto &band : bands)
   10604        54560 :     if (band.label)
   10605              :       {
   10606        28032 :        emit_label (band.label);
   10607        28032 :        machine_mode bmode = memset_vals ? band.set_mode : band.move_mode;
   10608        28032 :        ix86_expand_n_overlapping_move_set_or_movmem (dst, src,
   10609              :                                                      memset_vals,
   10610              :                                                      destreg, srcreg,
   10611              :                                                      count_exp, bmode,
   10612              :                                                      2);
   10613        28032 :        emit_jump_insn (gen_jump (done_label));
   10614        28032 :        emit_barrier ();
   10615              :       }
   10616        10912 : }
   10617              : 
   10618              : /* Expand movmem with overlapping unaligned loads and stores:
   10619              :    1. Load all sources into registers and store them together to avoid
   10620              :       possible address overlap between source and destination.
   10621              :    2. For known size, first try to fully unroll with 8 registers.
   10622              :    3. For size <= 2 * MOVE_MAX, load all sources into 2 registers first
   10623              :       and then store them together.
   10624              :    4. For size > 2 * MOVE_MAX and size <= 4 * MOVE_MAX, load all sources
   10625              :       into 4 registers first and then store them together.
   10626              :    5. For size > 4 * MOVE_MAX and size <= 8 * MOVE_MAX, load all sources
   10627              :       into 8 registers first and then store them together.
   10628              :    6. For size > 8 * MOVE_MAX,
   10629              :       a. If address of destination > address of source, copy backward
   10630              :          with a 4 * MOVE_MAX loop with unaligned loads and stores.  Load
   10631              :          the first 4 * MOVE_MAX into 4 registers before the loop and
   10632              :          store them after the loop to support overlapping addresses.
   10633              :       b. Otherwise, copy forward with a 4 * MOVE_MAX loop with unaligned
   10634              :          loads and stores.  Load the last 4 * MOVE_MAX into 4 registers
   10635              :          before the loop and store them after the loop to support
   10636              :          overlapping addresses.
   10637              :  */
   10638              : 
   10639              : bool
   10640       171280 : ix86_expand_set_or_movmem (rtx operands[], bool iscpymem, bool issetmem)
   10641              : {
   10642              :   /* Since there are much less registers available in 32-bit mode, don't
   10643              :      inline movmem in 32-bit mode.  */
   10644       171280 :   if (!TARGET_64BIT || optimize_insn_for_size_p ())
   10645              :     return false;
   10646              : 
   10647       137112 :   rtx dst = operands[0];
   10648       137112 :   rtx src, memset_val_exp;
   10649       137112 :   if (issetmem)
   10650              :     {
   10651        35856 :       src = nullptr;
   10652        35856 :       memset_val_exp = operands[1];
   10653              :     }
   10654              :   else
   10655              :     {
   10656       101256 :       src = operands[1];
   10657       101256 :       memset_val_exp = nullptr;
   10658              :     }
   10659       137112 :   rtx count_exp = operands[2];
   10660       137112 :   rtx expected_size_exp = operands[5];
   10661       137112 :   rtx min_size_exp = operands[6];
   10662       137112 :   rtx max_size_exp = operands[7];
   10663       137112 :   rtx probable_max_size_exp = operands[8];
   10664       137112 :   unsigned HOST_WIDE_INT count = HOST_WIDE_INT_0U;
   10665       137112 :   HOST_WIDE_INT expected_size = HOST_WIDE_INT_M1U;
   10666       137112 :   unsigned HOST_WIDE_INT min_size = HOST_WIDE_INT_0U;
   10667       137112 :   unsigned HOST_WIDE_INT max_size = HOST_WIDE_INT_M1U;
   10668       137112 :   unsigned HOST_WIDE_INT probable_max_size = HOST_WIDE_INT_M1U;
   10669              : 
   10670       137112 :   if (CONST_INT_P (count_exp))
   10671              :     {
   10672        54768 :       min_size = max_size = probable_max_size = count = expected_size
   10673        54768 :         = INTVAL (count_exp);
   10674              :       /* When COUNT is 0, there is nothing to do.  */
   10675        54768 :       if (!count)
   10676              :         return true;
   10677              :     }
   10678              :   else
   10679              :     {
   10680        82344 :       if (min_size_exp)
   10681        82344 :         min_size = INTVAL (min_size_exp);
   10682        82344 :       if (max_size_exp)
   10683        62892 :         max_size = INTVAL (max_size_exp);
   10684        82344 :       if (probable_max_size_exp)
   10685        64839 :         probable_max_size = INTVAL (probable_max_size_exp);
   10686        82344 :       if (CONST_INT_P (expected_size_exp))
   10687        82344 :         expected_size = INTVAL (expected_size_exp);
   10688              : 
   10689              :       /* NB: This assert may fail without the fixes for
   10690              :          https://gcc.gnu.org/bugzilla/show_bug.cgi?id=125977
   10691              :        */
   10692        82344 :       gcc_assert (min_size != max_size);
   10693              :     }
   10694              : 
   10695              :   /* Make sure we don't need to care about overflow later on.  */
   10696        54766 :   if (count > (HOST_WIDE_INT_1U << 30))
   10697              :     return false;
   10698              : 
   10699       136921 :   addr_space_t dst_as = MEM_ADDR_SPACE (dst);
   10700       136921 :   addr_space_t src_as = (issetmem
   10701       136921 :                          ? ADDR_SPACE_GENERIC
   10702       101128 :                          : MEM_ADDR_SPACE (src));
   10703       101128 :   int dynamic_check;
   10704       101128 :   bool noalign;
   10705       136921 :   enum stringop_alg alg = decide_alg (count, expected_size, min_size,
   10706              :                                       probable_max_size, issetmem,
   10707              :                                       (issetmem
   10708        35793 :                                        && memset_val_exp == const0_rtx),
   10709              :                                       dst_as, src_as, &dynamic_check,
   10710              :                                       &noalign, false);
   10711       136921 :   if (alg == libcall)
   10712              :     return false;
   10713              : 
   10714              :   /* Expand memcpy and memset like memmove only for bounded size.  */
   10715        18376 :   if (iscpymem || issetmem)
   10716              :     {
   10717        14137 :        unsigned HOST_WIDE_INT unbounded
   10718        14137 :          = GET_MODE_MASK (counter_mode (count_exp));
   10719        14137 :        if (count != 0                           /* Fixed size.  */
   10720        14137 :           || max_size == 0                      /* Unbounded size.  */
   10721         9118 :           || max_size == unbounded)             /* Unbounded size.  */
   10722              :          return false;
   10723              :     }
   10724              : 
   10725        13045 :   rtx destreg = ix86_copy_addr_to_reg (XEXP (dst, 0));
   10726        13045 :   rtx srcreg = (issetmem
   10727        13045 :                 ? nullptr
   10728        11903 :                 : ix86_copy_addr_to_reg (XEXP (src, 0)));
   10729              : 
   10730        13045 :   unsigned int move_max = MOVE_MAX;
   10731        13045 :   machine_mode mode = smallest_int_mode_for_size
   10732        13045 :     (move_max * BITS_PER_UNIT).require ();
   10733        13045 :   if (probable_max_size && probable_max_size < move_max)
   10734              :     {
   10735              :       /* Get a usable MOVE_MAX.  */
   10736         3867 :       mode = smallest_int_mode_for_size
   10737         3867 :         (probable_max_size * BITS_PER_UNIT).require ();
   10738              :       /* Reduce MOVE_MAX by half so that MOVE_MAX can be used.  */
   10739         7734 :       if (GET_MODE_SIZE (mode) > probable_max_size)
   10740         3355 :         mode = smallest_int_mode_for_size
   10741         3355 :           (GET_MODE_BITSIZE (mode) / 2).require ();
   10742         7734 :       move_max = GET_MODE_SIZE (mode);
   10743              :     }
   10744              : 
   10745              :   /* Try to fully unroll memmove of known size first.  */
   10746        13045 :   if (count
   10747        13045 :       && ix86_expand_unroll_movmem (dst, src, destreg, srcreg, count,
   10748              :                                     mode))
   10749              :     return true;
   10750              : 
   10751        11158 :   rtx memset_vals[memset_val_max];
   10752        11158 :   rtx *memset_vals_p;
   10753        11158 :   if (issetmem)
   10754              :     {
   10755              :       /* Use vector mode if MODE size > word size.  */
   10756         1142 :       unsigned int size = GET_MODE_SIZE (mode);
   10757         1142 :       poly_uint64 nunits;
   10758         1142 :       if (size > UNITS_PER_WORD)
   10759              :         {
   10760          639 :           nunits = size / GET_MODE_SIZE (QImode);
   10761          639 :           mode = mode_for_vector (QImode, nunits).require ();
   10762              :         }
   10763              : 
   10764              :       /* Populate MEMSET_VALS to expand memset.  */
   10765         1142 :       rtx val_word;
   10766         1142 :       memset_vals[memset_val_byte] = memset_val_exp;
   10767         1142 :       if (memset_val_exp == const0_rtx || memset_val_exp == constm1_rtx)
   10768              :         val_word = memset_val_exp;
   10769              :       else
   10770          661 :         val_word = promote_duplicated_reg (word_mode, memset_val_exp);
   10771         1142 :       memset_vals[memset_val_word] = val_word;
   10772         2284 :       if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
   10773              :         {
   10774          639 :           if (memset_val_exp == const0_rtx)
   10775          243 :             memset_vals[memset_val_vector] = CONST0_RTX (mode);
   10776          396 :           else if (memset_val_exp == constm1_rtx)
   10777           24 :             memset_vals[memset_val_vector] = CONSTM1_RTX (mode);
   10778              :           else
   10779              :             {
   10780              :               /* Use the vector mode based on WORD_MODE to avoid extra
   10781              :                  GPR moves.  */
   10782          372 :               nunits = size / UNITS_PER_WORD;
   10783          372 :               machine_mode vector_mode
   10784          372 :                 = mode_for_vector (word_mode, nunits).require ();
   10785          372 :               rtx vector = promote_duplicated_reg (vector_mode,
   10786              :                                                    val_word);
   10787          372 :               memset_vals[memset_val_vector]
   10788          372 :                 = convert_to_mode (mode, vector, 1);
   10789              :             }
   10790              :         }
   10791              :       else
   10792          503 :         memset_vals[memset_val_vector] = nullptr;
   10793         1142 :       memset_vals_p = memset_vals;
   10794              :     }
   10795              :   else
   10796              :     memset_vals_p = nullptr;
   10797              : 
   10798        11158 :   rtx_code_label *done_label = gen_label_rtx ();
   10799              : 
   10800        11158 :   rtx_code_label *less_vec_label = nullptr;
   10801        11158 :   if (min_size == 0 || min_size < move_max)
   10802        10912 :     less_vec_label = gen_label_rtx ();
   10803              : 
   10804        11158 :   machine_mode count_mode = counter_mode (count_exp);
   10805              : 
   10806              :   /* Jump to LESS_VEC_LABEL if size < MOVE_MAX.  */
   10807        11158 :   if (less_vec_label)
   10808        10912 :     emit_cmp_and_jump_insns (count_exp, GEN_INT (move_max), LTU,
   10809              :                              nullptr, count_mode, 1,
   10810              :                              less_vec_label);
   10811              : 
   10812        11158 :   rtx_code_label *more_2x_vec_label = nullptr;
   10813        11158 :   if (probable_max_size == 0 || probable_max_size > 2 * move_max)
   10814         3360 :     more_2x_vec_label = gen_label_rtx ();
   10815              : 
   10816              :   /* Jump to MORE_2X_VEC_LABEL if size > 2 * MOVE_MAX.  */
   10817         3360 :   if (more_2x_vec_label)
   10818         3360 :     emit_cmp_and_jump_insns (count_exp, GEN_INT (2 * move_max), GTU,
   10819              :                              nullptr, count_mode, 1,
   10820              :                              more_2x_vec_label);
   10821              : 
   10822        11158 :   if (max_size != 1 && (min_size == 0 || min_size <= 2 * move_max))
   10823              :     {
   10824              :       /* Max size != 1, size >= MOVE_MAX and size <= 2 * MOVE_MAX.  */
   10825        11114 :       ix86_expand_n_overlapping_move_set_or_movmem (dst, src,
   10826              :                                                     memset_vals_p,
   10827              :                                                     destreg, srcreg,
   10828              :                                                     count_exp, mode, 2);
   10829        11114 :       emit_jump_insn (gen_jump (done_label));
   10830        11114 :       emit_barrier ();
   10831              :     }
   10832              : 
   10833        11158 :   if (less_vec_label)
   10834              :     {
   10835              :       /* Size < MOVE_MAX.  */
   10836        10912 :       emit_label (less_vec_label);
   10837        10912 :       ix86_expand_less_move_set_or_movmem (dst, src, memset_vals_p,
   10838              :                                            destreg, srcreg, count_exp,
   10839              :                                            min_size, mode, done_label);
   10840        10912 :       emit_jump_insn (gen_jump (done_label));
   10841        10912 :       emit_barrier ();
   10842              :     }
   10843              : 
   10844        11158 :   if (more_2x_vec_label)
   10845              :     {
   10846              :       /* Size > 2 * MOVE_MAX and destination may overlap with source.  */
   10847         3360 :       emit_label (more_2x_vec_label);
   10848              : 
   10849         3360 :       rtx_code_label *more_8x_vec_label = nullptr;
   10850         3360 :       if (probable_max_size == 0 || probable_max_size > 8 * move_max)
   10851         1271 :         more_8x_vec_label = gen_label_rtx ();
   10852              : 
   10853              :       /* Jump to MORE_8X_VEC_LABEL if size > 8 * MOVE_MAX.  */
   10854         1271 :       if (more_8x_vec_label)
   10855         1271 :         emit_cmp_and_jump_insns (count_exp, GEN_INT (8 * move_max), GTU,
   10856              :                                  nullptr, count_mode, 1,
   10857              :                                  more_8x_vec_label);
   10858              : 
   10859         3360 :       rtx_code_label *last_4x_vec_label = nullptr;
   10860         3360 :       if (min_size == 0 || min_size <= 4 * move_max)
   10861         3348 :         last_4x_vec_label = gen_label_rtx ();
   10862              : 
   10863              :       /* Jump to LAST_4X_VEC_LABEL if size <= 4 * MOVE_MAX.  */
   10864         3348 :       if (last_4x_vec_label)
   10865         3348 :         emit_cmp_and_jump_insns (count_exp, GEN_INT (4 * move_max), LEU,
   10866              :                                  nullptr, count_mode, 1,
   10867              :                                  last_4x_vec_label);
   10868              : 
   10869         3360 :       if (probable_max_size == 0 || probable_max_size > 4 * move_max)
   10870              :         {
   10871              :           /* Size > 4 * MOVE_MAX and size <= 8 * MOVE_MAX.  */
   10872         1677 :           ix86_expand_n_overlapping_move_set_or_movmem (dst, src,
   10873              :                                                         memset_vals_p,
   10874              :                                                         destreg, srcreg,
   10875              :                                                         count_exp, mode,
   10876              :                                                         8);
   10877         1677 :           emit_jump_insn (gen_jump (done_label));
   10878         1677 :           emit_barrier ();
   10879              :         }
   10880              : 
   10881         3360 :       if (last_4x_vec_label)
   10882              :         {
   10883              :           /* Size > 2 * MOVE_MAX and size <= 4 * MOVE_MAX.  */
   10884         3348 :           emit_label (last_4x_vec_label);
   10885         3348 :           ix86_expand_n_overlapping_move_set_or_movmem (dst, src,
   10886              :                                                         memset_vals_p,
   10887              :                                                         destreg, srcreg,
   10888              :                                                         count_exp, mode,
   10889              :                                                         4);
   10890         3348 :           emit_jump_insn (gen_jump (done_label));
   10891         3348 :           emit_barrier ();
   10892              :         }
   10893              : 
   10894         3360 :       if (more_8x_vec_label)
   10895              :         {
   10896              :           /* Size > 8 * MOVE_MAX.  */
   10897         1271 :           emit_label (more_8x_vec_label);
   10898              : 
   10899         1271 :           rtx loop_count = gen_reg_rtx (count_mode);
   10900         1271 :           emit_move_insn (loop_count, count_exp);
   10901              : 
   10902         1271 :           rtx_code_label *more_8x_vec_backward_label;
   10903         1271 :           rtx base_destreg;
   10904         1271 :           rtx srcmem;
   10905         1271 :           rtx regs[4];
   10906         1271 :           if (iscpymem || issetmem)
   10907              :             {
   10908              :               /* Always store forward for memcpy and memset.  */
   10909          123 :               more_8x_vec_backward_label = nullptr;
   10910          123 :               if (iscpymem)
   10911              :                 {
   10912              :                   /* Load the last 4 * MOVE_MAX for memcpy.  */
   10913           28 :                   ix86_expand_load_movmem (src, srcreg, count_exp, mode,
   10914              :                                            ARRAY_SIZE (regs), regs,
   10915              :                                            true);
   10916           28 :                   srcmem = change_address (src, mode, srcreg);
   10917              :                 }
   10918              :               else
   10919              :                 {
   10920              :                   /* Fill REGS with MEMSET_VALS for memset.  */
   10921           95 :                   rtx val = ix86_expand_memset_val (memset_vals, mode);
   10922          570 :                   for (unsigned int i = 0; i < 4; i++)
   10923          380 :                     regs[i] = val;
   10924              :                   srcmem = nullptr;
   10925              :                 }
   10926          123 :               base_destreg = gen_reg_rtx (GET_MODE (destreg));
   10927          123 :               emit_move_insn (base_destreg, destreg);
   10928              :             }
   10929              :           else
   10930              :             {
   10931              :               /* Jump to MORE_8X_VEC_BACKWARD_LABEL if source address is
   10932              :                  lower than destination address.  */
   10933         1148 :               more_8x_vec_backward_label = gen_label_rtx ();
   10934         1148 :               emit_cmp_and_jump_insns (srcreg, destreg, LTU, nullptr,
   10935         1148 :                                        GET_MODE (destreg), 1,
   10936              :                                        more_8x_vec_backward_label);
   10937              : 
   10938              :               /* Skip if source == destination which is less common.  */
   10939         1148 :               emit_cmp_and_jump_insns (srcreg, destreg, EQ, nullptr,
   10940         1148 :                                        GET_MODE (destreg), 1, done_label,
   10941              :                                        profile_probability::unlikely ());
   10942              : 
   10943         1148 :               base_destreg = gen_reg_rtx (GET_MODE (destreg));
   10944         1148 :               emit_move_insn (base_destreg, destreg);
   10945              : 
   10946              :               /* Load the last 4 * MOVE_MAX.  */
   10947         1148 :               ix86_expand_load_movmem (src, srcreg, count_exp, mode,
   10948              :                                        ARRAY_SIZE (regs), regs, true);
   10949              : 
   10950         1148 :               srcmem = change_address (src, mode, srcreg);
   10951              :             }
   10952              : 
   10953         1271 :           rtx destmem = change_address (dst, mode, destreg);
   10954              : 
   10955              :           /* Copy forward with a 4 * MOVE_MAX loop.  */
   10956         1271 :           rtx_code_label *loop_4x_vec_forward_label = gen_label_rtx ();
   10957         1271 :           emit_label (loop_4x_vec_forward_label);
   10958              : 
   10959         1271 :           ix86_expand_n_move_set_or_movmem (destmem, srcmem,
   10960              :                                             memset_vals_p, mode, 4,
   10961              :                                             true);
   10962              : 
   10963         1271 :           rtx tmp;
   10964         1271 :           rtx delta = GEN_INT (4 * MOVE_MAX);
   10965              : 
   10966              :           /* Decrement LOOP_COUNT by 4 * MOVE_MAX.  */
   10967         1271 :           tmp = expand_simple_binop (GET_MODE (loop_count), MINUS,
   10968              :                                      loop_count, delta, nullptr, 1,
   10969              :                                      OPTAB_DIRECT);
   10970         1271 :           if (tmp != loop_count)
   10971         1271 :             emit_move_insn (loop_count, tmp);
   10972              : 
   10973              :           /* Increment DESTREG and SRCREG by 4 * MOVE_MAX.  */
   10974         1271 :           tmp = expand_simple_binop (GET_MODE (destreg), PLUS,
   10975              :                                      destreg, delta, nullptr, 1,
   10976              :                                      OPTAB_DIRECT);
   10977         1271 :           if (tmp != destreg)
   10978         1271 :             emit_move_insn (destreg, tmp);
   10979         1271 :           if (!issetmem)
   10980              :             {
   10981         1176 :               tmp = expand_simple_binop (GET_MODE (srcreg), PLUS,
   10982              :                                          srcreg, delta, nullptr, 1,
   10983              :                                          OPTAB_DIRECT);
   10984         1176 :               if (tmp != srcreg)
   10985         1176 :                 emit_move_insn (srcreg, tmp);
   10986              :             }
   10987              : 
   10988              :           /* Stop if LOOP_EXP <= 4 * MOVE_MAX.  */
   10989         1271 :           emit_cmp_and_jump_insns (loop_count, delta, GTU, nullptr,
   10990         1271 :                                    GET_MODE (loop_count), 1,
   10991              :                                    loop_4x_vec_forward_label);
   10992              : 
   10993              :           /* Store the last 4 * MOVE_MAX.  */
   10994         1271 :           ix86_expand_store_movmem (dst, base_destreg, count_exp, mode,
   10995              :                                     ARRAY_SIZE (regs), regs, true);
   10996              : 
   10997         1271 :           emit_jump_insn (gen_jump (done_label));
   10998         1271 :           emit_barrier ();
   10999              : 
   11000         1271 :           if (more_8x_vec_backward_label)
   11001              :             {
   11002              :               /* Copy backward with a 4 * MOVE_MAX loop.  */
   11003         1148 :               emit_label (more_8x_vec_backward_label);
   11004              : 
   11005         1148 :               base_destreg = gen_reg_rtx (GET_MODE (destreg));
   11006         1148 :               emit_move_insn (base_destreg, destreg);
   11007              : 
   11008              :               /* Load the first 4 * MOVE_MAX.  */
   11009         1148 :               ix86_expand_load_movmem (src, srcreg, count_exp, mode,
   11010              :                                        ARRAY_SIZE (regs), regs, false);
   11011              : 
   11012              :               /* Increment DESTREG and SRCREG by COUNT_EXP.  */
   11013         1148 :               tmp = expand_simple_binop (GET_MODE (destreg), PLUS,
   11014              :                                          destreg, count_exp, nullptr, 1,
   11015              :                                          OPTAB_DIRECT);
   11016         1148 :               if (tmp != destreg)
   11017         1148 :                 emit_move_insn (destreg, tmp);
   11018         1148 :               tmp = expand_simple_binop (GET_MODE (srcreg), PLUS, srcreg,
   11019              :                                          count_exp, nullptr, 1,
   11020              :                                          OPTAB_DIRECT);
   11021         1148 :               if (tmp != srcreg)
   11022         1148 :                 emit_move_insn (srcreg, tmp);
   11023              : 
   11024         1148 :               srcmem = change_address (src, mode, srcreg);
   11025         1148 :               destmem = change_address (dst, mode, destreg);
   11026         2296 :               rtx step = GEN_INT (-GET_MODE_SIZE (mode));
   11027         1148 :               srcmem = offset_address (srcmem, step,
   11028         1148 :                                        GET_MODE_SIZE (mode));
   11029         1148 :               destmem = offset_address (destmem, step,
   11030         1148 :                                         GET_MODE_SIZE (mode));
   11031              : 
   11032         1148 :               rtx_code_label *loop_4x_vec_backward_label
   11033         1148 :                 = gen_label_rtx ();
   11034         1148 :               emit_label (loop_4x_vec_backward_label);
   11035              : 
   11036         1148 :               ix86_expand_n_move_set_or_movmem (destmem, srcmem,
   11037              :                                                 memset_vals_p, mode, 4,
   11038              :                                                 false);
   11039              : 
   11040              :               /* Decrement LOOP_COUNT by 4 * MOVE_MAX.  */
   11041         1148 :               tmp = expand_simple_binop (GET_MODE (loop_count), MINUS,
   11042              :                                          loop_count, delta, nullptr, 1,
   11043              :                                          OPTAB_DIRECT);
   11044         1148 :               if (tmp != loop_count)
   11045         1148 :                 emit_move_insn (loop_count, tmp);
   11046              : 
   11047              :               /* Decrement DESTREG and SRCREG by 4 * MOVE_MAX.  */
   11048         1148 :               tmp = expand_simple_binop (GET_MODE (destreg), MINUS,
   11049              :                                          destreg, delta, nullptr, 1,
   11050              :                                          OPTAB_DIRECT);
   11051         1148 :               if (tmp != destreg)
   11052         1148 :                 emit_move_insn (destreg, tmp);
   11053         1148 :               tmp = expand_simple_binop (GET_MODE (srcreg), MINUS,
   11054              :                                          srcreg, delta, nullptr, 1,
   11055              :                                          OPTAB_DIRECT);
   11056         1148 :               if (tmp != srcreg)
   11057         1148 :                 emit_move_insn (srcreg, tmp);
   11058              : 
   11059              :               /* Stop if LOOP_EXP <= 4 * MOVE_MAX.  */
   11060         1148 :               emit_cmp_and_jump_insns (loop_count, delta, GTU, nullptr,
   11061         1148 :                                        GET_MODE (loop_count), 1,
   11062              :                                        loop_4x_vec_backward_label);
   11063              : 
   11064              :               /* Store the first 4 * MOVE_MAX.  */
   11065         1148 :               ix86_expand_store_movmem (dst, base_destreg, count_exp,
   11066              :                                         mode, ARRAY_SIZE (regs), regs,
   11067              :                                         false);
   11068              : 
   11069         1148 :               emit_jump_insn (gen_jump (done_label));
   11070         1148 :               emit_barrier ();
   11071              :             }
   11072              :         }
   11073              :     }
   11074              : 
   11075        11158 :   emit_label (done_label);
   11076              : 
   11077        11158 :   return true;
   11078              : }
   11079              : 
   11080              : /* Expand cmpstrn or memcmp.  */
   11081              : 
   11082              : bool
   11083       171578 : ix86_expand_cmpstrn_or_cmpmem (rtx result, rtx src1, rtx src2,
   11084              :                                rtx length, rtx align, bool is_cmpstrn)
   11085              : {
   11086              :   /* Expand strncmp and memcmp only with -minline-all-stringops since
   11087              :      "repz cmpsb" can be much slower than strncmp and memcmp functions
   11088              :      implemented with vector instructions, see
   11089              : 
   11090              :      https://gcc.gnu.org/bugzilla/show_bug.cgi?id=43052
   11091              :    */
   11092       171578 :   if (!TARGET_INLINE_ALL_STRINGOPS)
   11093              :     return false;
   11094              : 
   11095              :   /* Can't use this if the user has appropriated ecx, esi or edi.  */
   11096         5802 :   if (fixed_regs[CX_REG] || fixed_regs[SI_REG] || fixed_regs[DI_REG])
   11097              :     return false;
   11098              : 
   11099         5802 :   if (is_cmpstrn)
   11100              :     {
   11101              :       /* For strncmp, length is the maximum length, which can be larger
   11102              :          than actual string lengths.  We can expand the cmpstrn pattern
   11103              :          to "repz cmpsb" only if one of the strings is a constant so
   11104              :          that expand_builtin_strncmp() can write the length argument to
   11105              :          be the minimum of the const string length and the actual length
   11106              :          argument.  Otherwise, "repz cmpsb" may pass the 0 byte.  */
   11107           69 :       tree t1 = MEM_EXPR (src1);
   11108           69 :       tree t2 = MEM_EXPR (src2);
   11109          138 :       if (!((t1 && TREE_CODE (t1) == MEM_REF
   11110           69 :              && TREE_CODE (TREE_OPERAND (t1, 0)) == ADDR_EXPR
   11111            0 :              && (TREE_CODE (TREE_OPERAND (TREE_OPERAND (t1, 0), 0))
   11112              :                  == STRING_CST))
   11113           69 :             || (t2 && TREE_CODE (t2) == MEM_REF
   11114           69 :                 && TREE_CODE (TREE_OPERAND (t2, 0)) == ADDR_EXPR
   11115           69 :                 && (TREE_CODE (TREE_OPERAND (TREE_OPERAND (t2, 0), 0))
   11116              :                     == STRING_CST))))
   11117              :         return false;
   11118              :     }
   11119              : 
   11120         5802 :   rtx addr1 = copy_addr_to_reg (XEXP (src1, 0));
   11121         5802 :   rtx addr2 = copy_addr_to_reg (XEXP (src2, 0));
   11122         5802 :   if (addr1 != XEXP (src1, 0))
   11123         5802 :     src1 = replace_equiv_address_nv (src1, addr1);
   11124         5802 :   if (addr2 != XEXP (src2, 0))
   11125         5802 :     src2 = replace_equiv_address_nv (src2, addr2);
   11126              : 
   11127              :   /* NB: Make a copy of the data length to avoid changing the original
   11128              :      data length by cmpstrnqi patterns.  */
   11129         5802 :   length = ix86_zero_extend_to_Pmode (length);
   11130         8720 :   rtx lengthreg = gen_reg_rtx (Pmode);
   11131         5802 :   emit_move_insn (lengthreg, length);
   11132              : 
   11133              :   /* If we are testing strict equality, we can use known alignment to
   11134              :      good advantage.  This may be possible with combine, particularly
   11135              :      once cc0 is dead.  */
   11136         5802 :   if (CONST_INT_P (length))
   11137              :     {
   11138            0 :       if (length == const0_rtx)
   11139              :         {
   11140            0 :           emit_move_insn (result, const0_rtx);
   11141            0 :           return true;
   11142              :         }
   11143            0 :       emit_insn (gen_cmpstrnqi_nz_1 (addr1, addr2, lengthreg, align,
   11144              :                                      src1, src2));
   11145              :     }
   11146              :   else
   11147              :     {
   11148         8720 :       emit_insn (gen_cmp_1 (Pmode, lengthreg, lengthreg));
   11149         5802 :       emit_insn (gen_cmpstrnqi_1 (addr1, addr2, lengthreg, align,
   11150              :                                   src1, src2));
   11151              :     }
   11152              : 
   11153         5802 :   rtx out = gen_lowpart (QImode, result);
   11154         5802 :   emit_insn (gen_cmpintqi (out));
   11155         5802 :   emit_move_insn (result, gen_rtx_SIGN_EXTEND (SImode, out));
   11156              : 
   11157         5802 :   return true;
   11158              : }
   11159              : 
   11160              : /* Expand the appropriate insns for doing strlen if not just doing
   11161              :    repnz; scasb
   11162              : 
   11163              :    out = result, initialized with the start address
   11164              :    align_rtx = alignment of the address.
   11165              :    scratch = scratch register, initialized with the startaddress when
   11166              :         not aligned, otherwise undefined
   11167              : 
   11168              :    This is just the body. It needs the initializations mentioned above and
   11169              :    some address computing at the end.  These things are done in i386.md.  */
   11170              : 
   11171              : static void
   11172           11 : ix86_expand_strlensi_unroll_1 (rtx out, rtx src, rtx align_rtx)
   11173              : {
   11174           11 :   int align;
   11175           11 :   rtx tmp;
   11176           11 :   rtx_code_label *align_2_label = NULL;
   11177           11 :   rtx_code_label *align_3_label = NULL;
   11178           11 :   rtx_code_label *align_4_label = gen_label_rtx ();
   11179           11 :   rtx_code_label *end_0_label = gen_label_rtx ();
   11180           11 :   rtx mem;
   11181           11 :   rtx tmpreg = gen_reg_rtx (SImode);
   11182           11 :   rtx scratch = gen_reg_rtx (SImode);
   11183           11 :   rtx cmp;
   11184              : 
   11185           11 :   align = 0;
   11186           11 :   if (CONST_INT_P (align_rtx))
   11187           11 :     align = INTVAL (align_rtx);
   11188              : 
   11189              :   /* Loop to check 1..3 bytes for null to get an aligned pointer.  */
   11190              : 
   11191              :   /* Is there a known alignment and is it less than 4?  */
   11192           11 :   if (align < 4)
   11193              :     {
   11194           15 :       rtx scratch1 = gen_reg_rtx (Pmode);
   11195           11 :       emit_move_insn (scratch1, out);
   11196              :       /* Is there a known alignment and is it not 2? */
   11197           11 :       if (align != 2)
   11198              :         {
   11199           11 :           align_3_label = gen_label_rtx (); /* Label when aligned to 3-byte */
   11200           11 :           align_2_label = gen_label_rtx (); /* Label when aligned to 2-byte */
   11201              : 
   11202              :           /* Leave just the 3 lower bits.  */
   11203           15 :           align_rtx = expand_binop (Pmode, and_optab, scratch1, GEN_INT (3),
   11204              :                                     NULL_RTX, 0, OPTAB_WIDEN);
   11205              : 
   11206           15 :           emit_cmp_and_jump_insns (align_rtx, const0_rtx, EQ, NULL,
   11207           11 :                                    Pmode, 1, align_4_label);
   11208           15 :           emit_cmp_and_jump_insns (align_rtx, const2_rtx, EQ, NULL,
   11209           11 :                                    Pmode, 1, align_2_label);
   11210           15 :           emit_cmp_and_jump_insns (align_rtx, const2_rtx, GTU, NULL,
   11211           11 :                                    Pmode, 1, align_3_label);
   11212              :         }
   11213              :       else
   11214              :         {
   11215              :           /* Since the alignment is 2, we have to check 2 or 0 bytes;
   11216              :              check if is aligned to 4 - byte.  */
   11217              : 
   11218            0 :           align_rtx = expand_binop (Pmode, and_optab, scratch1, const2_rtx,
   11219              :                                     NULL_RTX, 0, OPTAB_WIDEN);
   11220              : 
   11221            0 :           emit_cmp_and_jump_insns (align_rtx, const0_rtx, EQ, NULL,
   11222            0 :                                    Pmode, 1, align_4_label);
   11223              :         }
   11224              : 
   11225           11 :       mem = change_address (src, QImode, out);
   11226              : 
   11227              :       /* Now compare the bytes.  */
   11228              : 
   11229              :       /* Compare the first n unaligned byte on a byte per byte basis.  */
   11230           11 :       emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL,
   11231              :                                QImode, 1, end_0_label);
   11232              : 
   11233              :       /* Increment the address.  */
   11234           11 :       emit_insn (gen_add2_insn (out, const1_rtx));
   11235              : 
   11236              :       /* Not needed with an alignment of 2 */
   11237           11 :       if (align != 2)
   11238              :         {
   11239           11 :           emit_label (align_2_label);
   11240              : 
   11241           11 :           emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1,
   11242              :                                    end_0_label);
   11243              : 
   11244           11 :           emit_insn (gen_add2_insn (out, const1_rtx));
   11245              : 
   11246           11 :           emit_label (align_3_label);
   11247              :         }
   11248              : 
   11249           11 :       emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1,
   11250              :                                end_0_label);
   11251              : 
   11252           11 :       emit_insn (gen_add2_insn (out, const1_rtx));
   11253              :     }
   11254              : 
   11255              :   /* Generate loop to check 4 bytes at a time.  It is not a good idea to
   11256              :      align this loop.  It gives only huge programs, but does not help to
   11257              :      speed up.  */
   11258           11 :   emit_label (align_4_label);
   11259              : 
   11260           11 :   mem = change_address (src, SImode, out);
   11261           11 :   emit_move_insn (scratch, mem);
   11262           11 :   emit_insn (gen_add2_insn (out, GEN_INT (4)));
   11263              : 
   11264              :   /* This formula yields a nonzero result iff one of the bytes is zero.
   11265              :      This saves three branches inside loop and many cycles.  */
   11266              : 
   11267           11 :   emit_insn (gen_addsi3 (tmpreg, scratch, GEN_INT (-0x01010101)));
   11268           11 :   emit_insn (gen_one_cmplsi2 (scratch, scratch));
   11269           11 :   emit_insn (gen_andsi3 (tmpreg, tmpreg, scratch));
   11270           11 :   emit_insn (gen_andsi3 (tmpreg, tmpreg,
   11271              :                          gen_int_mode (0x80808080, SImode)));
   11272           11 :   emit_cmp_and_jump_insns (tmpreg, const0_rtx, EQ, 0, SImode, 1,
   11273              :                            align_4_label);
   11274              : 
   11275           11 :   if (TARGET_CMOVE)
   11276              :     {
   11277           11 :        rtx reg = gen_reg_rtx (SImode);
   11278           15 :        rtx reg2 = gen_reg_rtx (Pmode);
   11279           11 :        emit_move_insn (reg, tmpreg);
   11280           11 :        emit_insn (gen_lshrsi3 (reg, reg, GEN_INT (16)));
   11281              : 
   11282              :        /* If zero is not in the first two bytes, move two bytes forward.  */
   11283           11 :        emit_insn (gen_testsi_ccno_1 (tmpreg, GEN_INT (0x8080)));
   11284           11 :        tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
   11285           11 :        tmp = gen_rtx_EQ (VOIDmode, tmp, const0_rtx);
   11286           11 :        emit_insn (gen_rtx_SET (tmpreg,
   11287              :                                gen_rtx_IF_THEN_ELSE (SImode, tmp,
   11288              :                                                      reg,
   11289              :                                                      tmpreg)));
   11290              :        /* Emit lea manually to avoid clobbering of flags.  */
   11291           15 :        emit_insn (gen_rtx_SET (reg2, plus_constant (Pmode, out, 2)));
   11292              : 
   11293           11 :        tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
   11294           11 :        tmp = gen_rtx_EQ (VOIDmode, tmp, const0_rtx);
   11295           15 :        emit_insn (gen_rtx_SET (out,
   11296              :                                gen_rtx_IF_THEN_ELSE (Pmode, tmp,
   11297              :                                                      reg2,
   11298              :                                                      out)));
   11299           11 :     }
   11300              :   else
   11301              :     {
   11302            0 :        rtx_code_label *end_2_label = gen_label_rtx ();
   11303              :        /* Is zero in the first two bytes? */
   11304              : 
   11305            0 :        emit_insn (gen_testsi_ccno_1 (tmpreg, GEN_INT (0x8080)));
   11306            0 :        tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
   11307            0 :        tmp = gen_rtx_NE (VOIDmode, tmp, const0_rtx);
   11308            0 :        tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
   11309              :                             gen_rtx_LABEL_REF (VOIDmode, end_2_label),
   11310              :                             pc_rtx);
   11311            0 :        rtx_insn *jump = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
   11312            0 :        JUMP_LABEL (jump) = end_2_label;
   11313              : 
   11314              :        /* Not in the first two.  Move two bytes forward.  */
   11315            0 :        emit_insn (gen_lshrsi3 (tmpreg, tmpreg, GEN_INT (16)));
   11316            0 :        emit_insn (gen_add2_insn (out, const2_rtx));
   11317              : 
   11318            0 :        emit_label (end_2_label);
   11319              : 
   11320              :     }
   11321              : 
   11322              :   /* Avoid branch in fixing the byte.  */
   11323           11 :   tmpreg = gen_lowpart (QImode, tmpreg);
   11324           11 :   emit_insn (gen_addqi3_cconly_overflow (tmpreg, tmpreg));
   11325           11 :   tmp = gen_rtx_REG (CCmode, FLAGS_REG);
   11326           11 :   cmp = gen_rtx_LTU (VOIDmode, tmp, const0_rtx);
   11327           15 :   emit_insn (gen_sub3_carry (Pmode, out, out, GEN_INT (3), tmp, cmp));
   11328              : 
   11329           11 :   emit_label (end_0_label);
   11330           11 : }
   11331              : 
   11332              : /* Expand strlen.  */
   11333              : 
   11334              : bool
   11335        13648 : ix86_expand_strlen (rtx out, rtx src, rtx eoschar, rtx align)
   11336              : {
   11337        13648 : if (TARGET_UNROLL_STRLEN
   11338        13648 :            && TARGET_INLINE_ALL_STRINGOPS
   11339           11 :            && eoschar == const0_rtx
   11340           11 :            && optimize > 1)
   11341              :     {
   11342              :       /* The generic case of strlen expander is long.  Avoid it's
   11343              :          expanding unless TARGET_INLINE_ALL_STRINGOPS.  */
   11344           15 :       rtx addr = force_reg (Pmode, XEXP (src, 0));
   11345              :       /* Well it seems that some optimizer does not combine a call like
   11346              :          foo(strlen(bar), strlen(bar));
   11347              :          when the move and the subtraction is done here.  It does calculate
   11348              :          the length just once when these instructions are done inside of
   11349              :          output_strlen_unroll().  But I think since &bar[strlen(bar)] is
   11350              :          often used and I use one fewer register for the lifetime of
   11351              :          output_strlen_unroll() this is better.  */
   11352              : 
   11353           11 :       emit_move_insn (out, addr);
   11354              : 
   11355           11 :       ix86_expand_strlensi_unroll_1 (out, src, align);
   11356              : 
   11357              :       /* strlensi_unroll_1 returns the address of the zero at the end of
   11358              :          the string, like memchr(), so compute the length by subtracting
   11359              :          the start address.  */
   11360           11 :       emit_insn (gen_sub2_insn (out, addr));
   11361           11 :       return true;
   11362              :     }
   11363              :   else
   11364              :     return false;
   11365              : }
   11366              : 
   11367              : /* For given symbol (function) construct code to compute address of it's PLT
   11368              :    entry in large x86-64 PIC model.  */
   11369              : 
   11370              : static rtx
   11371           34 : construct_plt_address (rtx symbol)
   11372              : {
   11373           34 :   rtx tmp, unspec;
   11374              : 
   11375           34 :   gcc_assert (SYMBOL_REF_P (symbol));
   11376           34 :   gcc_assert (ix86_cmodel == CM_LARGE_PIC && !TARGET_PECOFF);
   11377           34 :   gcc_assert (Pmode == DImode);
   11378              : 
   11379           34 :   tmp = gen_reg_rtx (Pmode);
   11380           34 :   unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, symbol), UNSPEC_PLTOFF);
   11381              : 
   11382           34 :   emit_move_insn (tmp, gen_rtx_CONST (Pmode, unspec));
   11383           34 :   emit_insn (gen_add2_insn (tmp, pic_offset_table_rtx));
   11384           34 :   return tmp;
   11385              : }
   11386              : 
   11387              : rtx_insn *
   11388      6425601 : ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
   11389              :                   rtx callarg2,
   11390              :                   rtx pop, bool sibcall)
   11391              : {
   11392      6425601 :   rtx vec[3];
   11393      6425601 :   rtx use = NULL, call;
   11394      6425601 :   unsigned int vec_len = 0;
   11395      6425601 :   tree fndecl;
   11396              : 
   11397      6425601 :   if (SYMBOL_REF_P (XEXP (fnaddr, 0)))
   11398              :     {
   11399      6238568 :       fndecl = SYMBOL_REF_DECL (XEXP (fnaddr, 0));
   11400      6238568 :       if (fndecl)
   11401              :         {
   11402      5978590 :           if (lookup_attribute ("interrupt",
   11403      5978590 :                                 TYPE_ATTRIBUTES (TREE_TYPE (fndecl))))
   11404            1 :             error ("interrupt service routine cannot be called directly");
   11405      5978590 :           if (fndecl == current_function_decl
   11406      5978590 :               && decl_binds_to_current_def_p (fndecl))
   11407        11485 :             cfun->machine->recursive_function = true;
   11408              :         }
   11409              :     }
   11410              :   else
   11411              :     fndecl = NULL_TREE;
   11412              : 
   11413      6425601 :   if (pop == const0_rtx)
   11414            0 :     pop = NULL;
   11415      6425601 :   gcc_assert (!TARGET_64BIT || !pop);
   11416              : 
   11417      6425601 :   rtx addr = XEXP (fnaddr, 0);
   11418      6425601 :   if (TARGET_MACHO && !TARGET_64BIT)
   11419              :     {
   11420              : #if TARGET_MACHO
   11421              :       if (flag_pic && SYMBOL_REF_P (XEXP (fnaddr, 0)))
   11422              :         fnaddr = machopic_indirect_call_target (fnaddr);
   11423              : #endif
   11424              :     }
   11425              :   else
   11426              :     {
   11427              :       /* Static functions and indirect calls don't need the pic register.  Also,
   11428              :          check if PLT was explicitly avoided via no-plt or "noplt" attribute, making
   11429              :          it an indirect call.  */
   11430      6425601 :       if (flag_pic
   11431       729096 :           && SYMBOL_REF_P (addr)
   11432      7127016 :           && ix86_call_use_plt_p (addr))
   11433              :         {
   11434       539166 :           if (flag_plt
   11435       539166 :               && (SYMBOL_REF_DECL (addr) == NULL_TREE
   11436       539132 :                   || !lookup_attribute ("noplt",
   11437       539132 :                                         DECL_ATTRIBUTES (SYMBOL_REF_DECL (addr)))))
   11438              :             {
   11439       539131 :               if (!TARGET_64BIT
   11440       360083 :                   || (ix86_cmodel == CM_LARGE_PIC
   11441              :                       && DEFAULT_ABI != MS_ABI))
   11442              :                 {
   11443       537178 :                   use_reg (&use, gen_rtx_REG (Pmode,
   11444              :                                               REAL_PIC_OFFSET_TABLE_REGNUM));
   11445       179082 :                   if (ix86_use_pseudo_pic_reg ())
   11446       358130 :                     emit_move_insn (gen_rtx_REG (Pmode,
   11447       179082 :                                                  REAL_PIC_OFFSET_TABLE_REGNUM),
   11448              :                                     pic_offset_table_rtx);
   11449              :                 }
   11450              :             }
   11451           35 :           else if (!TARGET_PECOFF && !TARGET_MACHO)
   11452              :             {
   11453           35 :               if (TARGET_64BIT
   11454           35 :                   && ix86_cmodel == CM_LARGE_PIC
   11455              :                   && DEFAULT_ABI != MS_ABI)
   11456              :                 {
   11457            1 :                   fnaddr = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr),
   11458              :                                            UNSPEC_GOT);
   11459            1 :                   fnaddr = gen_rtx_CONST (Pmode, fnaddr);
   11460            1 :                   fnaddr = force_reg (Pmode, fnaddr);
   11461            1 :                   fnaddr = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, fnaddr);
   11462              :                 }
   11463           34 :               else if (TARGET_64BIT)
   11464              :                 {
   11465           38 :                   fnaddr = gen_rtx_UNSPEC (Pmode,
   11466              :                                            gen_rtvec (1, addr),
   11467              :                                            UNSPEC_GOTPCREL);
   11468           38 :                   fnaddr = gen_rtx_CONST (Pmode, fnaddr);
   11469              :                 }
   11470              :               else
   11471              :                 {
   11472            0 :                   fnaddr = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr),
   11473              :                                            UNSPEC_GOT);
   11474            0 :                   fnaddr = gen_rtx_CONST (Pmode, fnaddr);
   11475            0 :                   fnaddr = gen_rtx_PLUS (Pmode, pic_offset_table_rtx,
   11476              :                                          fnaddr);
   11477              :                 }
   11478           39 :               fnaddr = gen_const_mem (Pmode, fnaddr);
   11479              :               /* Pmode may not be the same as word_mode for x32, which
   11480              :                  doesn't support indirect branch via 32-bit memory slot.
   11481              :                  Since x32 GOT slot is 64 bit with zero upper 32 bits,
   11482              :                  indirect branch via x32 GOT slot is OK.  */
   11483           35 :               if (GET_MODE (fnaddr) != word_mode)
   11484            4 :                 fnaddr = gen_rtx_ZERO_EXTEND (word_mode, fnaddr);
   11485           35 :               fnaddr = gen_rtx_MEM (QImode, fnaddr);
   11486              :             }
   11487              :         }
   11488              :     }
   11489              : 
   11490              :   /* Skip setting up RAX register for -mskip-rax-setup when there are no
   11491              :      parameters passed in vector registers.  */
   11492      6425601 :   if (TARGET_64BIT
   11493      5584843 :       && (INTVAL (callarg2) > 0
   11494      5523505 :           || (INTVAL (callarg2) == 0
   11495       331967 :               && (TARGET_SSE || !flag_skip_rax_setup))))
   11496              :     {
   11497       393303 :       rtx al = gen_rtx_REG (QImode, AX_REG);
   11498       393303 :       emit_move_insn (al, callarg2);
   11499       393303 :       use_reg (&use, al);
   11500              :     }
   11501              : 
   11502      6425601 :   if (ix86_cmodel == CM_LARGE_PIC
   11503              :       && !TARGET_PECOFF
   11504           45 :       && MEM_P (fnaddr)
   11505           45 :       && SYMBOL_REF_P (XEXP (fnaddr, 0))
   11506      6425638 :       && !local_symbolic_operand (XEXP (fnaddr, 0), VOIDmode))
   11507           34 :     fnaddr = gen_rtx_MEM (QImode, construct_plt_address (XEXP (fnaddr, 0)));
   11508              :   /* Since x32 GOT slot is 64 bit with zero upper 32 bits, indirect
   11509              :      branch via x32 GOT slot is OK.  */
   11510      6425567 :   else if (TARGET_X32
   11511           74 :       && MEM_P (fnaddr)
   11512           74 :       && GET_CODE (XEXP (fnaddr, 0)) == ZERO_EXTEND
   11513            8 :       && GOT_memory_operand (XEXP (XEXP (fnaddr, 0), 0), Pmode)
   11514      6425571 :       && !TARGET_INDIRECT_BRANCH_REGISTER)
   11515              :     ;
   11516      6425567 :   else if (sibcall
   11517      6425567 :            ? !sibcall_insn_operand (XEXP (fnaddr, 0), word_mode)
   11518      6297132 :            : !call_insn_operand (XEXP (fnaddr, 0), word_mode))
   11519              :     {
   11520          532 :       fnaddr = convert_to_mode (word_mode, XEXP (fnaddr, 0), 1);
   11521          532 :       fnaddr = gen_rtx_MEM (QImode, copy_to_mode_reg (word_mode, fnaddr));
   11522              :     }
   11523              : 
   11524              :   /* PR100665: Hwasan may tag code pointer which is not supported by LAM,
   11525              :      mask off code pointers here.
   11526              :      TODO: also need to handle indirect jump.  */
   11527      6426613 :   if (ix86_memtag_can_tag_addresses () && !fndecl
   11528      6425625 :       && sanitize_flags_p (SANITIZE_HWADDRESS))
   11529              :     {
   11530           24 :       rtx untagged_addr = ix86_memtag_untagged_pointer (XEXP (fnaddr, 0),
   11531              :                                                         NULL_RTX);
   11532           24 :       fnaddr = gen_rtx_MEM (QImode, untagged_addr);
   11533              :     }
   11534              : 
   11535      6425601 :   call = gen_rtx_CALL (VOIDmode, fnaddr, callarg1);
   11536              : 
   11537      6425601 :   if (retval)
   11538      2546061 :     call = gen_rtx_SET (retval, call);
   11539      6425601 :   vec[vec_len++] = call;
   11540              : 
   11541      6425601 :   if (pop)
   11542              :     {
   11543       450294 :       pop = gen_rtx_PLUS (Pmode, stack_pointer_rtx, pop);
   11544       225147 :       pop = gen_rtx_SET (stack_pointer_rtx, pop);
   11545       225147 :       vec[vec_len++] = pop;
   11546              :     }
   11547              : 
   11548              :   /* Set here, but it may get cleared later.  */
   11549      5584843 :   if (TARGET_64BIT_MS_ABI
   11550        73446 :       && (!callarg2 || INTVAL (callarg2) != -2)
   11551      6491909 :       && TARGET_CALL_MS2SYSV_XLOGUES)
   11552              :     {
   11553         7046 :       if (!TARGET_SSE)
   11554              :         ;
   11555              : 
   11556              :       /* Don't break hot-patched functions.  */
   11557         7046 :       else if (ix86_function_ms_hook_prologue (current_function_decl))
   11558              :         ;
   11559              : 
   11560              :       /* TODO: Cases not yet examined.  */
   11561         7046 :       else if (flag_split_stack)
   11562            0 :         warn_once_call_ms2sysv_xlogues ("-fsplit-stack");
   11563              : 
   11564              :       else
   11565              :         {
   11566         7046 :           gcc_assert (!reload_completed);
   11567         7046 :           cfun->machine->call_ms2sysv = true;
   11568              :         }
   11569              :     }
   11570              : 
   11571      6425601 :   if (TARGET_MACHO && TARGET_64BIT && !sibcall
   11572              :       && (!fndecl || TREE_PUBLIC (fndecl)
   11573              :           || (SYMBOL_REF_P (addr) && !SYMBOL_REF_LOCAL_P (addr)))
   11574              :       && (!fndecl || !lookup_attribute ("no_caller_saved_registers",
   11575              :                                         TYPE_ATTRIBUTES (TREE_TYPE (fndecl)))))
   11576              :     {
   11577              :       /* We allow public functions defined in a TU to bind locally for PIC
   11578              :          code (the default) on 64bit Mach-O.
   11579              :          If such functions are not inlined, we cannot tell at compile-time if
   11580              :          they will be called via the lazy symbol resolver (this can depend on
   11581              :          options given at link-time).  Therefore, we must assume that the lazy
   11582              :          resolver could be used which clobbers R11 and R10.
   11583              :          User specification of "no caller saved regs" overides.  */
   11584              :       clobber_reg (&use, gen_rtx_REG (DImode, R11_REG));
   11585              :       clobber_reg (&use, gen_rtx_REG (DImode, R10_REG));
   11586              :     }
   11587              : 
   11588      6425601 :   if (vec_len > 1)
   11589       225147 :     call = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (vec_len, vec));
   11590      6425601 :   rtx_insn *call_insn = emit_call_insn (call);
   11591      6425601 :   if (use)
   11592       572384 :     CALL_INSN_FUNCTION_USAGE (call_insn) = use;
   11593              : 
   11594      6425601 :   return call_insn;
   11595              : }
   11596              : 
   11597              : /* Split simple return with popping POPC bytes from stack to indirect
   11598              :    branch with stack adjustment .  */
   11599              : 
   11600              : void
   11601            0 : ix86_split_simple_return_pop_internal (rtx popc)
   11602              : {
   11603            0 :   struct machine_function *m = cfun->machine;
   11604            0 :   rtx ecx = gen_rtx_REG (SImode, CX_REG);
   11605            0 :   rtx_insn *insn;
   11606              : 
   11607              :   /* There is no "pascal" calling convention in any 64bit ABI.  */
   11608            0 :   gcc_assert (!TARGET_64BIT);
   11609              : 
   11610            0 :   insn = emit_insn (gen_pop (ecx));
   11611            0 :   m->fs.cfa_offset -= UNITS_PER_WORD;
   11612            0 :   m->fs.sp_offset -= UNITS_PER_WORD;
   11613              : 
   11614            0 :   rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
   11615            0 :   x = gen_rtx_SET (stack_pointer_rtx, x);
   11616            0 :   add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
   11617            0 :   add_reg_note (insn, REG_CFA_REGISTER, gen_rtx_SET (ecx, pc_rtx));
   11618            0 :   RTX_FRAME_RELATED_P (insn) = 1;
   11619              : 
   11620            0 :   x = gen_rtx_PLUS (Pmode, stack_pointer_rtx, popc);
   11621            0 :   x = gen_rtx_SET (stack_pointer_rtx, x);
   11622            0 :   insn = emit_insn (x);
   11623            0 :   add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
   11624            0 :   RTX_FRAME_RELATED_P (insn) = 1;
   11625              : 
   11626              :   /* Now return address is in ECX.  */
   11627            0 :   emit_jump_insn (gen_simple_return_indirect_internal (ecx));
   11628            0 : }
   11629              : 
   11630              : /* Errors in the source file can cause expand_expr to return const0_rtx
   11631              :    where we expect a vector.  To avoid crashing, use one of the vector
   11632              :    clear instructions.  */
   11633              : 
   11634              : static rtx
   11635       204065 : safe_vector_operand (rtx x, machine_mode mode)
   11636              : {
   11637            0 :   if (x == const0_rtx)
   11638            0 :     x = CONST0_RTX (mode);
   11639           24 :   return x;
   11640              : }
   11641              : 
   11642              : /* Subroutine of ix86_expand_builtin to take care of binop insns.  */
   11643              : 
   11644              : static rtx
   11645         8858 : ix86_expand_binop_builtin (enum insn_code icode, tree exp, rtx target)
   11646              : {
   11647         8858 :   rtx pat;
   11648         8858 :   tree arg0 = CALL_EXPR_ARG (exp, 0);
   11649         8858 :   tree arg1 = CALL_EXPR_ARG (exp, 1);
   11650         8858 :   rtx op0 = expand_normal (arg0);
   11651         8858 :   rtx op1 = expand_normal (arg1);
   11652         8858 :   machine_mode tmode = insn_data[icode].operand[0].mode;
   11653         8858 :   machine_mode mode0 = insn_data[icode].operand[1].mode;
   11654         8858 :   machine_mode mode1 = insn_data[icode].operand[2].mode;
   11655              : 
   11656         8858 :   if (VECTOR_MODE_P (mode0))
   11657         8847 :     op0 = safe_vector_operand (op0, mode0);
   11658         8858 :   if (VECTOR_MODE_P (mode1))
   11659         8711 :     op1 = safe_vector_operand (op1, mode1);
   11660              : 
   11661         2793 :   if (optimize || !target
   11662         2793 :       || GET_MODE (target) != tmode
   11663        11651 :       || !insn_data[icode].operand[0].predicate (target, tmode))
   11664         6118 :     target = gen_reg_rtx (tmode);
   11665              : 
   11666         8858 :   if (GET_MODE (op1) == SImode && mode1 == TImode)
   11667              :     {
   11668            0 :       rtx x = gen_reg_rtx (V4SImode);
   11669            0 :       emit_insn (gen_sse2_loadd (x, op1));
   11670            0 :       op1 = gen_lowpart (TImode, x);
   11671              :     }
   11672              : 
   11673         8858 :   if (!insn_data[icode].operand[1].predicate (op0, mode0))
   11674         1378 :     op0 = copy_to_mode_reg (mode0, op0);
   11675         8858 :   if (!insn_data[icode].operand[2].predicate (op1, mode1))
   11676          805 :     op1 = copy_to_mode_reg (mode1, op1);
   11677              : 
   11678         8858 :   pat = GEN_FCN (icode) (target, op0, op1);
   11679         8858 :   if (! pat)
   11680              :     return 0;
   11681              : 
   11682         8858 :   emit_insn (pat);
   11683              : 
   11684         8858 :   return target;
   11685              : }
   11686              : 
   11687              : /* Subroutine of ix86_expand_builtin to take care of 2-4 argument insns.  */
   11688              : 
   11689              : static rtx
   11690         1816 : ix86_expand_multi_arg_builtin (enum insn_code icode, tree exp, rtx target,
   11691              :                                enum ix86_builtin_func_type m_type,
   11692              :                                enum rtx_code sub_code)
   11693              : {
   11694         1816 :   rtx pat;
   11695         1816 :   unsigned int i, nargs;
   11696         1816 :   bool comparison_p = false;
   11697         1816 :   bool tf_p = false;
   11698         1816 :   bool last_arg_constant = false;
   11699         1816 :   int num_memory = 0;
   11700         1816 :   rtx xops[4];
   11701              : 
   11702         1816 :   machine_mode tmode = insn_data[icode].operand[0].mode;
   11703              : 
   11704         1816 :   switch (m_type)
   11705              :     {
   11706              :     case MULTI_ARG_4_DF2_DI_I:
   11707              :     case MULTI_ARG_4_DF2_DI_I1:
   11708              :     case MULTI_ARG_4_SF2_SI_I:
   11709              :     case MULTI_ARG_4_SF2_SI_I1:
   11710              :       nargs = 4;
   11711              :       last_arg_constant = true;
   11712              :       break;
   11713              : 
   11714          845 :     case MULTI_ARG_3_SF:
   11715          845 :     case MULTI_ARG_3_DF:
   11716          845 :     case MULTI_ARG_3_SF2:
   11717          845 :     case MULTI_ARG_3_DF2:
   11718          845 :     case MULTI_ARG_3_DI:
   11719          845 :     case MULTI_ARG_3_SI:
   11720          845 :     case MULTI_ARG_3_SI_DI:
   11721          845 :     case MULTI_ARG_3_HI:
   11722          845 :     case MULTI_ARG_3_HI_SI:
   11723          845 :     case MULTI_ARG_3_QI:
   11724          845 :     case MULTI_ARG_3_DI2:
   11725          845 :     case MULTI_ARG_3_SI2:
   11726          845 :     case MULTI_ARG_3_HI2:
   11727          845 :     case MULTI_ARG_3_QI2:
   11728          845 :       nargs = 3;
   11729          845 :       break;
   11730              : 
   11731          128 :     case MULTI_ARG_2_SF:
   11732          128 :     case MULTI_ARG_2_DF:
   11733          128 :     case MULTI_ARG_2_DI:
   11734          128 :     case MULTI_ARG_2_SI:
   11735          128 :     case MULTI_ARG_2_HI:
   11736          128 :     case MULTI_ARG_2_QI:
   11737          128 :       nargs = 2;
   11738          128 :       break;
   11739              : 
   11740           64 :     case MULTI_ARG_2_DI_IMM:
   11741           64 :     case MULTI_ARG_2_SI_IMM:
   11742           64 :     case MULTI_ARG_2_HI_IMM:
   11743           64 :     case MULTI_ARG_2_QI_IMM:
   11744           64 :       nargs = 2;
   11745           64 :       last_arg_constant = true;
   11746           64 :       break;
   11747              : 
   11748          187 :     case MULTI_ARG_1_SF:
   11749          187 :     case MULTI_ARG_1_DF:
   11750          187 :     case MULTI_ARG_1_SF2:
   11751          187 :     case MULTI_ARG_1_DF2:
   11752          187 :     case MULTI_ARG_1_DI:
   11753          187 :     case MULTI_ARG_1_SI:
   11754          187 :     case MULTI_ARG_1_HI:
   11755          187 :     case MULTI_ARG_1_QI:
   11756          187 :     case MULTI_ARG_1_SI_DI:
   11757          187 :     case MULTI_ARG_1_HI_DI:
   11758          187 :     case MULTI_ARG_1_HI_SI:
   11759          187 :     case MULTI_ARG_1_QI_DI:
   11760          187 :     case MULTI_ARG_1_QI_SI:
   11761          187 :     case MULTI_ARG_1_QI_HI:
   11762          187 :       nargs = 1;
   11763          187 :       break;
   11764              : 
   11765          384 :     case MULTI_ARG_2_DI_CMP:
   11766          384 :     case MULTI_ARG_2_SI_CMP:
   11767          384 :     case MULTI_ARG_2_HI_CMP:
   11768          384 :     case MULTI_ARG_2_QI_CMP:
   11769          384 :       nargs = 2;
   11770          384 :       comparison_p = true;
   11771          384 :       break;
   11772              : 
   11773          128 :     case MULTI_ARG_2_SF_TF:
   11774          128 :     case MULTI_ARG_2_DF_TF:
   11775          128 :     case MULTI_ARG_2_DI_TF:
   11776          128 :     case MULTI_ARG_2_SI_TF:
   11777          128 :     case MULTI_ARG_2_HI_TF:
   11778          128 :     case MULTI_ARG_2_QI_TF:
   11779          128 :       nargs = 2;
   11780          128 :       tf_p = true;
   11781          128 :       break;
   11782              : 
   11783            0 :     default:
   11784            0 :       gcc_unreachable ();
   11785              :     }
   11786              : 
   11787          628 :   if (optimize || !target
   11788          628 :       || GET_MODE (target) != tmode
   11789         2420 :       || !insn_data[icode].operand[0].predicate (target, tmode))
   11790         1212 :     target = gen_reg_rtx (tmode);
   11791          604 :   else if (memory_operand (target, tmode))
   11792            0 :     num_memory++;
   11793              : 
   11794         1816 :   gcc_assert (nargs <= ARRAY_SIZE (xops));
   11795              : 
   11796         6258 :   for (i = 0; i < nargs; i++)
   11797              :     {
   11798         4450 :       tree arg = CALL_EXPR_ARG (exp, i);
   11799         4450 :       rtx op = expand_normal (arg);
   11800         4450 :       int adjust = (comparison_p) ? 1 : 0;
   11801         4450 :       machine_mode mode = insn_data[icode].operand[i+adjust+1].mode;
   11802              : 
   11803         4450 :       if (last_arg_constant && i == nargs - 1)
   11804              :         {
   11805          144 :           if (!insn_data[icode].operand[i + 1].predicate (op, mode))
   11806              :             {
   11807           30 :               enum insn_code new_icode = icode;
   11808           30 :               switch (icode)
   11809              :                 {
   11810            8 :                 case CODE_FOR_xop_vpermil2v2df3:
   11811            8 :                 case CODE_FOR_xop_vpermil2v4sf3:
   11812            8 :                 case CODE_FOR_xop_vpermil2v4df3:
   11813            8 :                 case CODE_FOR_xop_vpermil2v8sf3:
   11814            8 :                   error ("the last argument must be a 2-bit immediate");
   11815            8 :                   return gen_reg_rtx (tmode);
   11816            5 :                 case CODE_FOR_xop_rotlv2di3:
   11817            5 :                   new_icode = CODE_FOR_rotlv2di3;
   11818            5 :                   goto xop_rotl;
   11819            5 :                 case CODE_FOR_xop_rotlv4si3:
   11820            5 :                   new_icode = CODE_FOR_rotlv4si3;
   11821            5 :                   goto xop_rotl;
   11822            6 :                 case CODE_FOR_xop_rotlv8hi3:
   11823            6 :                   new_icode = CODE_FOR_rotlv8hi3;
   11824            6 :                   goto xop_rotl;
   11825              :                 case CODE_FOR_xop_rotlv16qi3:
   11826              :                   new_icode = CODE_FOR_rotlv16qi3;
   11827           22 :                 xop_rotl:
   11828           22 :                   if (CONST_INT_P (op))
   11829              :                     {
   11830            6 :                       int mask = GET_MODE_UNIT_BITSIZE (tmode) - 1;
   11831            6 :                       op = GEN_INT (INTVAL (op) & mask);
   11832            6 :                       gcc_checking_assert
   11833              :                         (insn_data[icode].operand[i + 1].predicate (op, mode));
   11834              :                     }
   11835              :                   else
   11836              :                     {
   11837           16 :                       gcc_checking_assert
   11838              :                         (nargs == 2
   11839              :                          && insn_data[new_icode].operand[0].mode == tmode
   11840              :                          && insn_data[new_icode].operand[1].mode == tmode
   11841              :                          && insn_data[new_icode].operand[2].mode == mode
   11842              :                          && insn_data[new_icode].operand[0].predicate
   11843              :                             == insn_data[icode].operand[0].predicate
   11844              :                          && insn_data[new_icode].operand[1].predicate
   11845              :                             == insn_data[icode].operand[1].predicate);
   11846           16 :                       icode = new_icode;
   11847           16 :                       goto non_constant;
   11848              :                     }
   11849              :                   break;
   11850            0 :                 default:
   11851            0 :                   gcc_unreachable ();
   11852              :                 }
   11853              :             }
   11854              :         }
   11855              :       else
   11856              :         {
   11857         4306 :         non_constant:
   11858         4322 :           if (VECTOR_MODE_P (mode))
   11859         4306 :             op = safe_vector_operand (op, mode);
   11860              : 
   11861              :           /* If we aren't optimizing, only allow one memory operand to be
   11862              :              generated.  */
   11863         4322 :           if (memory_operand (op, mode))
   11864          826 :             num_memory++;
   11865              : 
   11866         4322 :           gcc_assert (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode);
   11867              : 
   11868         4322 :           if (optimize
   11869         1506 :               || !insn_data[icode].operand[i+adjust+1].predicate (op, mode)
   11870         5750 :               || num_memory > 1)
   11871         3401 :             op = force_reg (mode, op);
   11872              :         }
   11873              : 
   11874         4442 :       xops[i] = op;
   11875              :     }
   11876              : 
   11877         1808 :   switch (nargs)
   11878              :     {
   11879          187 :     case 1:
   11880          187 :       pat = GEN_FCN (icode) (target, xops[0]);
   11881          187 :       break;
   11882              : 
   11883          704 :     case 2:
   11884          704 :       if (tf_p)
   11885          128 :         pat = GEN_FCN (icode) (target, xops[0], xops[1],
   11886          128 :                                GEN_INT ((int)sub_code));
   11887          576 :       else if (! comparison_p)
   11888          192 :         pat = GEN_FCN (icode) (target, xops[0], xops[1]);
   11889              :       else
   11890              :         {
   11891          384 :           rtx cmp_op = gen_rtx_fmt_ee (sub_code, GET_MODE (target),
   11892              :                                        xops[0], xops[1]);
   11893              : 
   11894          384 :           pat = GEN_FCN (icode) (target, cmp_op, xops[0], xops[1]);
   11895              :         }
   11896              :       break;
   11897              : 
   11898          845 :     case 3:
   11899          845 :       pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2]);
   11900          845 :       break;
   11901              : 
   11902           72 :     case 4:
   11903           72 :       pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2], xops[3]);
   11904           72 :       break;
   11905              : 
   11906              :     default:
   11907              :       gcc_unreachable ();
   11908              :     }
   11909              : 
   11910         1808 :   if (! pat)
   11911              :     return 0;
   11912              : 
   11913         1808 :   emit_insn (pat);
   11914         1808 :   return target;
   11915              : }
   11916              : 
   11917              : /* Subroutine of ix86_expand_args_builtin to take care of scalar unop
   11918              :    insns with vec_merge.  */
   11919              : 
   11920              : static rtx
   11921           52 : ix86_expand_unop_vec_merge_builtin (enum insn_code icode, tree exp,
   11922              :                                     rtx target)
   11923              : {
   11924           52 :   rtx pat;
   11925           52 :   tree arg0 = CALL_EXPR_ARG (exp, 0);
   11926           52 :   rtx op1, op0 = expand_normal (arg0);
   11927           52 :   machine_mode tmode = insn_data[icode].operand[0].mode;
   11928           52 :   machine_mode mode0 = insn_data[icode].operand[1].mode;
   11929              : 
   11930           16 :   if (optimize || !target
   11931           16 :       || GET_MODE (target) != tmode
   11932           68 :       || !insn_data[icode].operand[0].predicate (target, tmode))
   11933           36 :     target = gen_reg_rtx (tmode);
   11934              : 
   11935           52 :   if (VECTOR_MODE_P (mode0))
   11936           52 :     op0 = safe_vector_operand (op0, mode0);
   11937              : 
   11938           36 :   if ((optimize && !register_operand (op0, mode0))
   11939           88 :       || !insn_data[icode].operand[1].predicate (op0, mode0))
   11940            0 :     op0 = copy_to_mode_reg (mode0, op0);
   11941              : 
   11942           52 :   op1 = op0;
   11943           52 :   if (!insn_data[icode].operand[2].predicate (op1, mode0))
   11944           16 :     op1 = copy_to_mode_reg (mode0, op1);
   11945              : 
   11946           52 :   pat = GEN_FCN (icode) (target, op0, op1);
   11947           52 :   if (! pat)
   11948              :     return 0;
   11949           52 :   emit_insn (pat);
   11950           52 :   return target;
   11951              : }
   11952              : 
   11953              : /* Subroutine of ix86_expand_builtin to take care of comparison insns.  */
   11954              : 
   11955              : static rtx
   11956          608 : ix86_expand_sse_compare (const struct builtin_description *d,
   11957              :                          tree exp, rtx target, bool swap)
   11958              : {
   11959          608 :   rtx pat;
   11960          608 :   tree arg0 = CALL_EXPR_ARG (exp, 0);
   11961          608 :   tree arg1 = CALL_EXPR_ARG (exp, 1);
   11962          608 :   rtx op0 = expand_normal (arg0);
   11963          608 :   rtx op1 = expand_normal (arg1);
   11964          608 :   rtx op2;
   11965          608 :   machine_mode tmode = insn_data[d->icode].operand[0].mode;
   11966          608 :   machine_mode mode0 = insn_data[d->icode].operand[1].mode;
   11967          608 :   machine_mode mode1 = insn_data[d->icode].operand[2].mode;
   11968          608 :   enum rtx_code comparison = d->comparison;
   11969              : 
   11970          608 :   if (VECTOR_MODE_P (mode0))
   11971          608 :     op0 = safe_vector_operand (op0, mode0);
   11972          608 :   if (VECTOR_MODE_P (mode1))
   11973          608 :     op1 = safe_vector_operand (op1, mode1);
   11974              : 
   11975              :   /* Swap operands if we have a comparison that isn't available in
   11976              :      hardware.  */
   11977          608 :   if (swap)
   11978           80 :     std::swap (op0, op1);
   11979              : 
   11980          202 :   if (optimize || !target
   11981          202 :       || GET_MODE (target) != tmode
   11982          810 :       || !insn_data[d->icode].operand[0].predicate (target, tmode))
   11983          406 :     target = gen_reg_rtx (tmode);
   11984              : 
   11985          406 :   if ((optimize && !register_operand (op0, mode0))
   11986          958 :       || !insn_data[d->icode].operand[1].predicate (op0, mode0))
   11987          258 :     op0 = copy_to_mode_reg (mode0, op0);
   11988          406 :   if ((optimize && !register_operand (op1, mode1))
   11989          958 :       || !insn_data[d->icode].operand[2].predicate (op1, mode1))
   11990           56 :     op1 = copy_to_mode_reg (mode1, op1);
   11991              : 
   11992          608 :   op2 = gen_rtx_fmt_ee (comparison, mode0, op0, op1);
   11993          608 :   pat = GEN_FCN (d->icode) (target, op0, op1, op2);
   11994          608 :   if (! pat)
   11995              :     return 0;
   11996          608 :   emit_insn (pat);
   11997          608 :   return target;
   11998              : }
   11999              : 
   12000              : /* Subroutine of ix86_sse_comi and ix86_sse_comi_round to take care of
   12001              :  * ordered EQ or unordered NE, generate PF jump.  */
   12002              : 
   12003              : static rtx
   12004          646 : ix86_ssecom_setcc (const enum rtx_code comparison,
   12005              :                    bool check_unordered, machine_mode mode,
   12006              :                    rtx set_dst, rtx target)
   12007              : {
   12008              : 
   12009          646 :   rtx_code_label *label = NULL;
   12010              : 
   12011              :   /* NB: For ordered EQ or unordered NE, check ZF alone isn't sufficient
   12012              :      with NAN operands.
   12013              :      Under TARGET_AVX10_2, VCOMX/VUCOMX are generated instead of
   12014              :      COMI/UCOMI.  VCOMX/VUCOMX will not set ZF for NAN operands.  */
   12015          646 :   if (check_unordered)
   12016              :     {
   12017          122 :       gcc_assert (comparison == EQ || comparison == NE);
   12018              : 
   12019          122 :       rtx flag = gen_rtx_REG (CCFPmode, FLAGS_REG);
   12020          122 :       label = gen_label_rtx ();
   12021          122 :       rtx tmp = gen_rtx_fmt_ee (UNORDERED, VOIDmode, flag, const0_rtx);
   12022          122 :       tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
   12023              :                                   gen_rtx_LABEL_REF (VOIDmode, label),
   12024              :                                   pc_rtx);
   12025          122 :       emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
   12026              :     }
   12027              : 
   12028              :   /* NB: Set CCFPmode and check a different CCmode which is in subset
   12029              :      of CCFPmode.  */
   12030          646 :   if (GET_MODE (set_dst) != mode)
   12031              :     {
   12032          200 :       gcc_assert (mode == CCAmode || mode == CCCmode
   12033              :                   || mode == CCOmode || mode == CCPmode
   12034              :                   || mode == CCSmode || mode == CCZmode);
   12035          200 :       set_dst = gen_rtx_REG (mode, FLAGS_REG);
   12036              :     }
   12037              : 
   12038          646 :   emit_insn (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
   12039              :                           gen_rtx_fmt_ee (comparison, QImode,
   12040              :                                           set_dst,
   12041              :                                           const0_rtx)));
   12042              : 
   12043          646 :   if (label)
   12044          122 :     emit_label (label);
   12045              : 
   12046          646 :   return SUBREG_REG (target);
   12047              : }
   12048              : 
   12049              : /* Subroutine of ix86_expand_builtin to take care of comi insns.  */
   12050              : 
   12051              : static rtx
   12052          547 : ix86_expand_sse_comi (const struct builtin_description *d, tree exp,
   12053              :                       rtx target, bool comx_ok)
   12054              : {
   12055          547 :   rtx pat, set_dst;
   12056          547 :   tree arg0 = CALL_EXPR_ARG (exp, 0);
   12057          547 :   tree arg1 = CALL_EXPR_ARG (exp, 1);
   12058          547 :   rtx op0 = expand_normal (arg0);
   12059          547 :   rtx op1 = expand_normal (arg1);
   12060          547 :   enum insn_code icode = d->icode;
   12061          547 :   const struct insn_data_d *insn_p = &insn_data[icode];
   12062          547 :   machine_mode mode0 = insn_p->operand[0].mode;
   12063          547 :   machine_mode mode1 = insn_p->operand[1].mode;
   12064              : 
   12065          547 :   if (VECTOR_MODE_P (mode0))
   12066          547 :     op0 = safe_vector_operand (op0, mode0);
   12067          547 :   if (VECTOR_MODE_P (mode1))
   12068          547 :     op1 = safe_vector_operand (op1, mode1);
   12069              : 
   12070          547 :   enum rtx_code comparison = d->comparison;
   12071          547 :   rtx const_val = const0_rtx;
   12072              : 
   12073          547 :   bool check_unordered = false;
   12074          547 :   machine_mode mode = CCFPmode;
   12075          547 :   switch (comparison)
   12076              :     {
   12077          194 :     case LE:    /* -> GE  */
   12078          194 :     case LT:    /* -> GT  */
   12079          194 :       std::swap (op0, op1);
   12080          194 :       comparison = swap_condition (comparison);
   12081              :       /* FALLTHRU */
   12082              :     case GT:
   12083              :     case GE:
   12084              :       break;
   12085           73 :     case EQ:
   12086           73 :       if (!TARGET_AVX10_2 || !comx_ok)
   12087           45 :         check_unordered = true;
   12088              :       mode = CCZmode;
   12089              :       break;
   12090           96 :     case NE:
   12091           96 :       if (!TARGET_AVX10_2 || !comx_ok)
   12092           68 :         check_unordered = true;
   12093           96 :       mode = CCZmode;
   12094           96 :       const_val = const1_rtx;
   12095           96 :       break;
   12096            0 :     default:
   12097            0 :       gcc_unreachable ();
   12098              :     }
   12099              : 
   12100          547 :   target = gen_reg_rtx (SImode);
   12101          547 :   emit_move_insn (target, const_val);
   12102          547 :   target = gen_rtx_SUBREG (QImode, target, 0);
   12103              : 
   12104          426 :   if ((optimize && !register_operand (op0, mode0))
   12105          925 :       || !insn_p->operand[0].predicate (op0, mode0))
   12106          169 :     op0 = copy_to_mode_reg (mode0, op0);
   12107          426 :   if ((optimize && !register_operand (op1, mode1))
   12108          924 :       || !insn_p->operand[1].predicate (op1, mode1))
   12109           49 :     op1 = copy_to_mode_reg (mode1, op1);
   12110              : 
   12111          547 :   if ((comparison == EQ || comparison == NE)
   12112          169 :       && TARGET_AVX10_2 && comx_ok)
   12113              :     {
   12114           56 :       switch (icode)
   12115              :         {
   12116              :         case CODE_FOR_sse_comi:
   12117              :           icode = CODE_FOR_avx10_2_comxsf;
   12118              :           break;
   12119           14 :         case CODE_FOR_sse_ucomi:
   12120           14 :           icode = CODE_FOR_avx10_2_ucomxsf;
   12121           14 :           break;
   12122           14 :         case CODE_FOR_sse2_comi:
   12123           14 :           icode = CODE_FOR_avx10_2_comxdf;
   12124           14 :           break;
   12125           14 :         case CODE_FOR_sse2_ucomi:
   12126           14 :           icode = CODE_FOR_avx10_2_ucomxdf;
   12127           14 :           break;
   12128              : 
   12129            0 :         default:
   12130            0 :           gcc_unreachable ();
   12131              :         }
   12132              :     }
   12133          547 :   pat = GEN_FCN (icode) (op0, op1);
   12134          547 :   if (! pat)
   12135              :     return 0;
   12136              : 
   12137          547 :   set_dst = SET_DEST (pat);
   12138          547 :   emit_insn (pat);
   12139          547 :   return ix86_ssecom_setcc (comparison, check_unordered, mode,
   12140          547 :                             set_dst, target);
   12141              : }
   12142              : 
   12143              : /* Subroutines of ix86_expand_args_builtin to take care of round insns.  */
   12144              : 
   12145              : static rtx
   12146            0 : ix86_expand_sse_round (const struct builtin_description *d, tree exp,
   12147              :                        rtx target)
   12148              : {
   12149            0 :   rtx pat;
   12150            0 :   tree arg0 = CALL_EXPR_ARG (exp, 0);
   12151            0 :   rtx op1, op0 = expand_normal (arg0);
   12152            0 :   machine_mode tmode = insn_data[d->icode].operand[0].mode;
   12153            0 :   machine_mode mode0 = insn_data[d->icode].operand[1].mode;
   12154              : 
   12155            0 :   if (optimize || target == 0
   12156            0 :       || GET_MODE (target) != tmode
   12157            0 :       || !insn_data[d->icode].operand[0].predicate (target, tmode))
   12158            0 :     target = gen_reg_rtx (tmode);
   12159              : 
   12160            0 :   if (VECTOR_MODE_P (mode0))
   12161            0 :     op0 = safe_vector_operand (op0, mode0);
   12162              : 
   12163            0 :   if ((optimize && !register_operand (op0, mode0))
   12164            0 :       || !insn_data[d->icode].operand[0].predicate (op0, mode0))
   12165            0 :     op0 = copy_to_mode_reg (mode0, op0);
   12166              : 
   12167            0 :   op1 = GEN_INT (d->comparison);
   12168              : 
   12169            0 :   pat = GEN_FCN (d->icode) (target, op0, op1);
   12170            0 :   if (! pat)
   12171              :     return 0;
   12172            0 :   emit_insn (pat);
   12173            0 :   return target;
   12174              : }
   12175              : 
   12176              : static rtx
   12177           12 : ix86_expand_sse_round_vec_pack_sfix (const struct builtin_description *d,
   12178              :                                      tree exp, rtx target)
   12179              : {
   12180           12 :   rtx pat;
   12181           12 :   tree arg0 = CALL_EXPR_ARG (exp, 0);
   12182           12 :   tree arg1 = CALL_EXPR_ARG (exp, 1);
   12183           12 :   rtx op0 = expand_normal (arg0);
   12184           12 :   rtx op1 = expand_normal (arg1);
   12185           12 :   rtx op2;
   12186           12 :   machine_mode tmode = insn_data[d->icode].operand[0].mode;
   12187           12 :   machine_mode mode0 = insn_data[d->icode].operand[1].mode;
   12188           12 :   machine_mode mode1 = insn_data[d->icode].operand[2].mode;
   12189              : 
   12190            0 :   if (optimize || target == 0
   12191            0 :       || GET_MODE (target) != tmode
   12192           12 :       || !insn_data[d->icode].operand[0].predicate (target, tmode))
   12193           12 :     target = gen_reg_rtx (tmode);
   12194              : 
   12195           12 :   op0 = safe_vector_operand (op0, mode0);
   12196           12 :   op1 = safe_vector_operand (op1, mode1);
   12197              : 
   12198           12 :   if ((optimize && !register_operand (op0, mode0))
   12199           12 :       || !insn_data[d->icode].operand[0].predicate (op0, mode0))
   12200           12 :     op0 = copy_to_mode_reg (mode0, op0);
   12201           12 :   if ((optimize && !register_operand (op1, mode1))
   12202           12 :       || !insn_data[d->icode].operand[1].predicate (op1, mode1))
   12203           12 :     op1 = copy_to_mode_reg (mode1, op1);
   12204              : 
   12205           12 :   op2 = GEN_INT (d->comparison);
   12206              : 
   12207           12 :   pat = GEN_FCN (d->icode) (target, op0, op1, op2);
   12208           12 :   if (! pat)
   12209              :     return 0;
   12210           12 :   emit_insn (pat);
   12211           12 :   return target;
   12212              : }
   12213              : 
   12214              : /* Subroutine of ix86_expand_builtin to take care of ptest insns.  */
   12215              : 
   12216              : static rtx
   12217          239 : ix86_expand_sse_ptest (const struct builtin_description *d, tree exp,
   12218              :                        rtx target)
   12219              : {
   12220          239 :   rtx pat;
   12221          239 :   tree arg0 = CALL_EXPR_ARG (exp, 0);
   12222          239 :   tree arg1 = CALL_EXPR_ARG (exp, 1);
   12223          239 :   rtx op0 = expand_normal (arg0);
   12224          239 :   rtx op1 = expand_normal (arg1);
   12225          239 :   machine_mode mode0 = insn_data[d->icode].operand[0].mode;
   12226          239 :   machine_mode mode1 = insn_data[d->icode].operand[1].mode;
   12227          239 :   enum rtx_code comparison = d->comparison;
   12228          239 :   rtx result = NULL_RTX;
   12229              : 
   12230          239 :   if (VECTOR_MODE_P (mode0))
   12231          239 :     op0 = safe_vector_operand (op0, mode0);
   12232          239 :   if (VECTOR_MODE_P (mode1))
   12233          239 :     op1 = safe_vector_operand (op1, mode1);
   12234              : 
   12235          239 :   switch (d->code)
   12236              :     {
   12237           49 :     case IX86_BUILTIN_PTESTZ:
   12238           49 :     case IX86_BUILTIN_PTESTZ256:
   12239              :       // Returns (OP0 & OP1) == 0
   12240           49 :       if (rtx_equal_p (op0, CONST0_RTX (mode0))
   12241           49 :           || rtx_equal_p (op1, CONST0_RTX (mode1)))
   12242            2 :         result = const1_rtx;
   12243           47 :       else if (rtx_equal_p (op0, CONSTM1_RTX (mode0)))
   12244              :         {
   12245            1 :           op1 = force_reg (mode1, op1);
   12246            1 :           op0 = op1;
   12247              :         }
   12248           46 :       else if (rtx_equal_p (op1, CONSTM1_RTX (mode1)))
   12249              :         {
   12250            1 :           op0 = force_reg (mode0, op0);
   12251            1 :           op1 = op0;
   12252              :         }
   12253           45 :       else if (MEM_P (op0) && !MEM_P (op1))
   12254              :         std::swap (op0, op1);
   12255              :       break;
   12256              : 
   12257           31 :     case IX86_BUILTIN_PTESTC:
   12258           31 :     case IX86_BUILTIN_PTESTC256:
   12259              :       // Returns (~OP0 & OP1) == 0
   12260           31 :       if (rtx_equal_p (op0, CONSTM1_RTX (mode0))
   12261           31 :           || rtx_equal_p (op1, CONST0_RTX (mode1))
   12262           62 :           || rtx_equal_p (op0, op1))
   12263            2 :         result = const1_rtx;
   12264              :       break;
   12265              : 
   12266           27 :     case IX86_BUILTIN_PTESTNZC:
   12267           27 :     case IX86_BUILTIN_PTESTNZC256:
   12268              :       // Returns ((OP0 && OP1) != 0) && ((~OP0 && OP1) != 0)
   12269           27 :       if (rtx_equal_p (op0, CONST0_RTX (mode0))
   12270           26 :           || rtx_equal_p (op0, CONSTM1_RTX (mode0))
   12271           26 :           || rtx_equal_p (op1, CONST0_RTX (mode1))
   12272           53 :           || rtx_equal_p (op0, op1))
   12273            1 :         result = const0_rtx;
   12274              :       break;
   12275              : 
   12276              :     default:
   12277              :       break;
   12278              :     }
   12279              : 
   12280          167 :   if ((optimize && !register_operand (op0, mode0))
   12281          210 :       || !insn_data[d->icode].operand[0].predicate (op0, mode0)
   12282          377 :       || result)
   12283          104 :     op0 = copy_to_mode_reg (mode0, op0);
   12284          167 :   if ((optimize && !register_operand (op1, mode1))
   12285          211 :       || !insn_data[d->icode].operand[1].predicate (op1, mode1)
   12286          450 :       || result)
   12287           31 :     op1 = copy_to_mode_reg (mode1, op1);
   12288              : 
   12289          239 :   if (result)
   12290              :     {
   12291            5 :       if (!target)
   12292            0 :         target = gen_reg_rtx (SImode);
   12293            5 :       emit_move_insn (target, result);
   12294            5 :       return target;
   12295              :     }
   12296              : 
   12297          234 :   target = gen_reg_rtx (SImode);
   12298          234 :   emit_move_insn (target, const0_rtx);
   12299          234 :   target = gen_rtx_SUBREG (QImode, target, 0);
   12300              : 
   12301          234 :   pat = GEN_FCN (d->icode) (op0, op1);
   12302          234 :   if (! pat)
   12303              :     return 0;
   12304          234 :   emit_insn (pat);
   12305          234 :   emit_insn (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
   12306              :                           gen_rtx_fmt_ee (comparison, QImode,
   12307              :                                           SET_DEST (pat),
   12308              :                                           const0_rtx)));
   12309              : 
   12310          234 :   return SUBREG_REG (target);
   12311              : }
   12312              : 
   12313              : /* Subroutine of ix86_expand_builtin to take care of pcmpestr[im] insns.  */
   12314              : 
   12315              : static rtx
   12316          216 : ix86_expand_sse_pcmpestr (const struct builtin_description *d,
   12317              :                           tree exp, rtx target)
   12318              : {
   12319          216 :   rtx pat;
   12320          216 :   tree arg0 = CALL_EXPR_ARG (exp, 0);
   12321          216 :   tree arg1 = CALL_EXPR_ARG (exp, 1);
   12322          216 :   tree arg2 = CALL_EXPR_ARG (exp, 2);
   12323          216 :   tree arg3 = CALL_EXPR_ARG (exp, 3);
   12324          216 :   tree arg4 = CALL_EXPR_ARG (exp, 4);
   12325          216 :   rtx scratch0, scratch1;
   12326          216 :   rtx op0 = expand_normal (arg0);
   12327          216 :   rtx op1 = expand_normal (arg1);
   12328          216 :   rtx op2 = expand_normal (arg2);
   12329          216 :   rtx op3 = expand_normal (arg3);
   12330          216 :   rtx op4 = expand_normal (arg4);
   12331          216 :   machine_mode tmode0, tmode1, modev2, modei3, modev4, modei5, modeimm;
   12332              : 
   12333          216 :   tmode0 = insn_data[d->icode].operand[0].mode;
   12334          216 :   tmode1 = insn_data[d->icode].operand[1].mode;
   12335          216 :   modev2 = insn_data[d->icode].operand[2].mode;
   12336          216 :   modei3 = insn_data[d->icode].operand[3].mode;
   12337          216 :   modev4 = insn_data[d->icode].operand[4].mode;
   12338          216 :   modei5 = insn_data[d->icode].operand[5].mode;
   12339          216 :   modeimm = insn_data[d->icode].operand[6].mode;
   12340              : 
   12341          216 :   if (VECTOR_MODE_P (modev2))
   12342          216 :     op0 = safe_vector_operand (op0, modev2);
   12343          216 :   if (VECTOR_MODE_P (modev4))
   12344          216 :     op2 = safe_vector_operand (op2, modev4);
   12345              : 
   12346          216 :   if (!insn_data[d->icode].operand[2].predicate (op0, modev2))
   12347            6 :     op0 = copy_to_mode_reg (modev2, op0);
   12348          216 :   if (!insn_data[d->icode].operand[3].predicate (op1, modei3))
   12349           34 :     op1 = copy_to_mode_reg (modei3, op1);
   12350          160 :   if ((optimize && !register_operand (op2, modev4))
   12351          371 :       || !insn_data[d->icode].operand[4].predicate (op2, modev4))
   12352            5 :     op2 = copy_to_mode_reg (modev4, op2);
   12353          216 :   if (!insn_data[d->icode].operand[5].predicate (op3, modei5))
   12354           34 :     op3 = copy_to_mode_reg (modei5, op3);
   12355              : 
   12356          216 :   if (!insn_data[d->icode].operand[6].predicate (op4, modeimm))
   12357              :     {
   12358           21 :       error ("the fifth argument must be an 8-bit immediate");
   12359           21 :       return const0_rtx;
   12360              :     }
   12361              : 
   12362          195 :   if (d->code == IX86_BUILTIN_PCMPESTRI128)
   12363              :     {
   12364            5 :       if (optimize || !target
   12365            5 :           || GET_MODE (target) != tmode0
   12366           34 :           || !insn_data[d->icode].operand[0].predicate (target, tmode0))
   12367           24 :         target = gen_reg_rtx (tmode0);
   12368              : 
   12369           29 :       scratch1 = gen_reg_rtx (tmode1);
   12370              : 
   12371           29 :       pat = GEN_FCN (d->icode) (target, scratch1, op0, op1, op2, op3, op4);
   12372              :     }
   12373          166 :   else if (d->code == IX86_BUILTIN_PCMPESTRM128)
   12374              :     {
   12375            5 :       if (optimize || !target
   12376            5 :           || GET_MODE (target) != tmode1
   12377           36 :           || !insn_data[d->icode].operand[1].predicate (target, tmode1))
   12378           26 :         target = gen_reg_rtx (tmode1);
   12379              : 
   12380           31 :       scratch0 = gen_reg_rtx (tmode0);
   12381              : 
   12382           31 :       pat = GEN_FCN (d->icode) (scratch0, target, op0, op1, op2, op3, op4);
   12383              :     }
   12384              :   else
   12385              :     {
   12386          135 :       gcc_assert (d->flag);
   12387              : 
   12388          135 :       scratch0 = gen_reg_rtx (tmode0);
   12389          135 :       scratch1 = gen_reg_rtx (tmode1);
   12390              : 
   12391          135 :       pat = GEN_FCN (d->icode) (scratch0, scratch1, op0, op1, op2, op3, op4);
   12392              :     }
   12393              : 
   12394          195 :   if (! pat)
   12395              :     return 0;
   12396              : 
   12397          195 :   emit_insn (pat);
   12398              : 
   12399          195 :   if (d->flag)
   12400              :     {
   12401          135 :       target = gen_reg_rtx (SImode);
   12402          135 :       emit_move_insn (target, const0_rtx);
   12403          135 :       target = gen_rtx_SUBREG (QImode, target, 0);
   12404              : 
   12405          135 :       emit_insn
   12406          135 :         (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
   12407              :                       gen_rtx_fmt_ee (EQ, QImode,
   12408              :                                       gen_rtx_REG ((machine_mode) d->flag,
   12409              :                                                    FLAGS_REG),
   12410              :                                       const0_rtx)));
   12411          135 :       return SUBREG_REG (target);
   12412              :     }
   12413              :   else
   12414              :     return target;
   12415              : }
   12416              : 
   12417              : 
   12418              : /* Subroutine of ix86_expand_builtin to take care of pcmpistr[im] insns.  */
   12419              : 
   12420              : static rtx
   12421          275 : ix86_expand_sse_pcmpistr (const struct builtin_description *d,
   12422              :                           tree exp, rtx target)
   12423              : {
   12424          275 :   rtx pat;
   12425          275 :   tree arg0 = CALL_EXPR_ARG (exp, 0);
   12426          275 :   tree arg1 = CALL_EXPR_ARG (exp, 1);
   12427          275 :   tree arg2 = CALL_EXPR_ARG (exp, 2);
   12428          275 :   rtx scratch0, scratch1;
   12429          275 :   rtx op0 = expand_normal (arg0);
   12430          275 :   rtx op1 = expand_normal (arg1);
   12431          275 :   rtx op2 = expand_normal (arg2);
   12432          275 :   machine_mode tmode0, tmode1, modev2, modev3, modeimm;
   12433              : 
   12434          275 :   tmode0 = insn_data[d->icode].operand[0].mode;
   12435          275 :   tmode1 = insn_data[d->icode].operand[1].mode;
   12436          275 :   modev2 = insn_data[d->icode].operand[2].mode;
   12437          275 :   modev3 = insn_data[d->icode].operand[3].mode;
   12438          275 :   modeimm = insn_data[d->icode].operand[4].mode;
   12439              : 
   12440          275 :   if (VECTOR_MODE_P (modev2))
   12441          275 :     op0 = safe_vector_operand (op0, modev2);
   12442          275 :   if (VECTOR_MODE_P (modev3))
   12443          275 :     op1 = safe_vector_operand (op1, modev3);
   12444              : 
   12445          275 :   if (!insn_data[d->icode].operand[2].predicate (op0, modev2))
   12446            4 :     op0 = copy_to_mode_reg (modev2, op0);
   12447          210 :   if ((optimize && !register_operand (op1, modev3))
   12448          481 :       || !insn_data[d->icode].operand[3].predicate (op1, modev3))
   12449            4 :     op1 = copy_to_mode_reg (modev3, op1);
   12450              : 
   12451          275 :   if (!insn_data[d->icode].operand[4].predicate (op2, modeimm))
   12452              :     {
   12453           21 :       error ("the third argument must be an 8-bit immediate");
   12454           21 :       return const0_rtx;
   12455              :     }
   12456              : 
   12457          254 :   if (d->code == IX86_BUILTIN_PCMPISTRI128)
   12458              :     {
   12459            5 :       if (optimize || !target
   12460            5 :           || GET_MODE (target) != tmode0
   12461           38 :           || !insn_data[d->icode].operand[0].predicate (target, tmode0))
   12462           28 :         target = gen_reg_rtx (tmode0);
   12463              : 
   12464           33 :       scratch1 = gen_reg_rtx (tmode1);
   12465              : 
   12466           33 :       pat = GEN_FCN (d->icode) (target, scratch1, op0, op1, op2);
   12467              :     }
   12468          221 :   else if (d->code == IX86_BUILTIN_PCMPISTRM128)
   12469              :     {
   12470            8 :       if (optimize || !target
   12471            8 :           || GET_MODE (target) != tmode1
   12472           58 :           || !insn_data[d->icode].operand[1].predicate (target, tmode1))
   12473           42 :         target = gen_reg_rtx (tmode1);
   12474              : 
   12475           50 :       scratch0 = gen_reg_rtx (tmode0);
   12476              : 
   12477           50 :       pat = GEN_FCN (d->icode) (scratch0, target, op0, op1, op2);
   12478              :     }
   12479              :   else
   12480              :     {
   12481          171 :       gcc_assert (d->flag);
   12482              : 
   12483          171 :       scratch0 = gen_reg_rtx (tmode0);
   12484          171 :       scratch1 = gen_reg_rtx (tmode1);
   12485              : 
   12486          171 :       pat = GEN_FCN (d->icode) (scratch0, scratch1, op0, op1, op2);
   12487              :     }
   12488              : 
   12489          254 :   if (! pat)
   12490              :     return 0;
   12491              : 
   12492          254 :   emit_insn (pat);
   12493              : 
   12494          254 :   if (d->flag)
   12495              :     {
   12496          171 :       target = gen_reg_rtx (SImode);
   12497          171 :       emit_move_insn (target, const0_rtx);
   12498          171 :       target = gen_rtx_SUBREG (QImode, target, 0);
   12499              : 
   12500          171 :       emit_insn
   12501          171 :         (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
   12502              :                       gen_rtx_fmt_ee (EQ, QImode,
   12503              :                                       gen_rtx_REG ((machine_mode) d->flag,
   12504              :                                                    FLAGS_REG),
   12505              :                                       const0_rtx)));
   12506          171 :       return SUBREG_REG (target);
   12507              :     }
   12508              :   else
   12509              :     return target;
   12510              : }
   12511              : 
   12512              : /* Fixup modeless constants to fit required mode.  */
   12513              : 
   12514              : static rtx
   12515       269052 : fixup_modeless_constant (rtx x, machine_mode mode)
   12516              : {
   12517       269052 :   if (GET_MODE (x) == VOIDmode)
   12518        41970 :     x = convert_to_mode (mode, x, 1);
   12519       269052 :   return x;
   12520              : }
   12521              : 
   12522              : /* Expand the outgoing argument ARG to extract unsigned char and short
   12523              :    integer constants suitable for the predicates and the instruction
   12524              :    templates which expect the unsigned expanded value.  */
   12525              : 
   12526              : static rtx
   12527       290568 : ix86_expand_unsigned_small_int_cst_argument (tree arg)
   12528              : {
   12529              :   /* When passing 0xff as an unsigned char function argument with the
   12530              :      C frontend promotion, expand_normal gets
   12531              : 
   12532              :      <integer_cst 0x7fffe6aa23a8 type <integer_type 0x7fffe98225e8 int> constant 255>
   12533              : 
   12534              :      and returns the rtx value using the sign-extended representation:
   12535              : 
   12536              :      (const_int 255 [0xff])
   12537              : 
   12538              :      Without the C frontend promotion, expand_normal gets
   12539              : 
   12540              :      <integer_cst 0x7fffe9824018 type <integer_type 0x7fffe9822348 unsigned char > constant 255>
   12541              : 
   12542              :      and returns
   12543              : 
   12544              :      (const_int -1 [0xffffffffffffffff])
   12545              : 
   12546              :      which doesn't work with the predicates nor the instruction templates
   12547              :      which expect the unsigned expanded value.  Extract the unsigned char
   12548              :      and short integer constants to return
   12549              : 
   12550              :      (const_int 255 [0xff])
   12551              : 
   12552              :      so that the expanded value is always unsigned, without the C frontend
   12553              :      promotion.  */
   12554              : 
   12555       290568 :   if (TREE_CODE (arg) == INTEGER_CST)
   12556              :     {
   12557        61129 :       tree type = TREE_TYPE (arg);
   12558        61129 :       if (INTEGRAL_TYPE_P (type)
   12559        61129 :           && TYPE_UNSIGNED (type)
   12560        83689 :           && TYPE_PRECISION (type) < TYPE_PRECISION (integer_type_node))
   12561              :         {
   12562        18990 :           HOST_WIDE_INT cst = TREE_INT_CST_LOW (arg);
   12563        18990 :           return GEN_INT (cst);
   12564              :         }
   12565              :     }
   12566              : 
   12567       271578 :   return expand_normal (arg);
   12568              : }
   12569              : 
   12570              : /* Subroutine of ix86_expand_builtin to take care of insns with
   12571              :    variable number of operands.  */
   12572              : 
   12573              : static rtx
   12574        74502 : ix86_expand_args_builtin (const struct builtin_description *d,
   12575              :                           tree exp, rtx target)
   12576              : {
   12577        74502 :   rtx pat, real_target;
   12578        74502 :   unsigned int i, nargs;
   12579        74502 :   unsigned int nargs_constant = 0;
   12580        74502 :   unsigned int mask_pos = 0;
   12581        74502 :   int num_memory = 0;
   12582        74502 :   rtx xops[6];
   12583        74502 :   bool second_arg_count = false;
   12584        74502 :   enum insn_code icode = d->icode;
   12585        74502 :   const struct insn_data_d *insn_p = &insn_data[icode];
   12586        74502 :   machine_mode tmode = insn_p->operand[0].mode;
   12587        74502 :   machine_mode rmode = VOIDmode;
   12588        74502 :   bool swap = false;
   12589        74502 :   enum rtx_code comparison = d->comparison;
   12590              : 
   12591        74502 :   switch ((enum ix86_builtin_func_type) d->flag)
   12592              :     {
   12593            0 :     case V2DF_FTYPE_V2DF_ROUND:
   12594            0 :     case V4DF_FTYPE_V4DF_ROUND:
   12595            0 :     case V8DF_FTYPE_V8DF_ROUND:
   12596            0 :     case V4SF_FTYPE_V4SF_ROUND:
   12597            0 :     case V8SF_FTYPE_V8SF_ROUND:
   12598            0 :     case V16SF_FTYPE_V16SF_ROUND:
   12599            0 :     case V8HF_FTYPE_V8HF_ROUND:
   12600            0 :     case V16HF_FTYPE_V16HF_ROUND:
   12601            0 :     case V32HF_FTYPE_V32HF_ROUND:
   12602            0 :     case V4SI_FTYPE_V4SF_ROUND:
   12603            0 :     case V8SI_FTYPE_V8SF_ROUND:
   12604            0 :     case V16SI_FTYPE_V16SF_ROUND:
   12605            0 :       return ix86_expand_sse_round (d, exp, target);
   12606           12 :     case V4SI_FTYPE_V2DF_V2DF_ROUND:
   12607           12 :     case V8SI_FTYPE_V4DF_V4DF_ROUND:
   12608           12 :     case V16SI_FTYPE_V8DF_V8DF_ROUND:
   12609           12 :       return ix86_expand_sse_round_vec_pack_sfix (d, exp, target);
   12610          239 :     case INT_FTYPE_V8SF_V8SF_PTEST:
   12611          239 :     case INT_FTYPE_V4DI_V4DI_PTEST:
   12612          239 :     case INT_FTYPE_V4DF_V4DF_PTEST:
   12613          239 :     case INT_FTYPE_V4SF_V4SF_PTEST:
   12614          239 :     case INT_FTYPE_V2DI_V2DI_PTEST:
   12615          239 :     case INT_FTYPE_V2DF_V2DF_PTEST:
   12616          239 :       return ix86_expand_sse_ptest (d, exp, target);
   12617              :     case FLOAT128_FTYPE_FLOAT128:
   12618              :     case FLOAT_FTYPE_FLOAT:
   12619              :     case FLOAT_FTYPE_BFLOAT16:
   12620              :     case INT_FTYPE_INT:
   12621              :     case UINT_FTYPE_UINT:
   12622              :     case UINT16_FTYPE_UINT16:
   12623              :     case UINT64_FTYPE_INT:
   12624              :     case UINT64_FTYPE_UINT64:
   12625              :     case INT64_FTYPE_INT64:
   12626              :     case INT64_FTYPE_V4SF:
   12627              :     case INT64_FTYPE_V2DF:
   12628              :     case INT_FTYPE_V16QI:
   12629              :     case INT_FTYPE_V8QI:
   12630              :     case INT_FTYPE_V8SF:
   12631              :     case INT_FTYPE_V4DF:
   12632              :     case INT_FTYPE_V4SF:
   12633              :     case INT_FTYPE_V2DF:
   12634              :     case INT_FTYPE_V32QI:
   12635              :     case V16QI_FTYPE_V16QI:
   12636              :     case V8SI_FTYPE_V8SF:
   12637              :     case V8SI_FTYPE_V4SI:
   12638              :     case V8HI_FTYPE_V8HI:
   12639              :     case V8HI_FTYPE_V16QI:
   12640              :     case V8QI_FTYPE_V8QI:
   12641              :     case V8SF_FTYPE_V8SF:
   12642              :     case V8SF_FTYPE_V8SI:
   12643              :     case V8SF_FTYPE_V4SF:
   12644              :     case V8SF_FTYPE_V8HI:
   12645              :     case V4SI_FTYPE_V4SI:
   12646              :     case V4SI_FTYPE_V16QI:
   12647              :     case V4SI_FTYPE_V4SF:
   12648              :     case V4SI_FTYPE_V8SI:
   12649              :     case V4SI_FTYPE_V8HI:
   12650              :     case V4SI_FTYPE_V4DF:
   12651              :     case V4SI_FTYPE_V2DF:
   12652              :     case V4HI_FTYPE_V4HI:
   12653              :     case V4DF_FTYPE_V4DF:
   12654              :     case V4DF_FTYPE_V4SI:
   12655              :     case V4DF_FTYPE_V4SF:
   12656              :     case V4DF_FTYPE_V2DF:
   12657              :     case V4SF_FTYPE_V4SF:
   12658              :     case V4SF_FTYPE_V4SI:
   12659              :     case V4SF_FTYPE_V8SF:
   12660              :     case V4SF_FTYPE_V4DF:
   12661              :     case V4SF_FTYPE_V8HI:
   12662              :     case V4SF_FTYPE_V2DF:
   12663              :     case V2DI_FTYPE_V2DI:
   12664              :     case V2DI_FTYPE_V16QI:
   12665              :     case V2DI_FTYPE_V8HI:
   12666              :     case V2DI_FTYPE_V4SI:
   12667              :     case V2DF_FTYPE_V2DF:
   12668              :     case V2DF_FTYPE_V4SI:
   12669              :     case V2DF_FTYPE_V4DF:
   12670              :     case V2DF_FTYPE_V4SF:
   12671              :     case V2DF_FTYPE_V2SI:
   12672              :     case V2SI_FTYPE_V2SI:
   12673              :     case V2SI_FTYPE_V4SF:
   12674              :     case V2SI_FTYPE_V2SF:
   12675              :     case V2SI_FTYPE_V2DF:
   12676              :     case V2SF_FTYPE_V2SF:
   12677              :     case V2SF_FTYPE_V2SI:
   12678              :     case V32QI_FTYPE_V32QI:
   12679              :     case V32QI_FTYPE_V16QI:
   12680              :     case V16HI_FTYPE_V16HI:
   12681              :     case V16HI_FTYPE_V8HI:
   12682              :     case V8SI_FTYPE_V8SI:
   12683              :     case V16HI_FTYPE_V16QI:
   12684              :     case V8SI_FTYPE_V16QI:
   12685              :     case V4DI_FTYPE_V16QI:
   12686              :     case V8SI_FTYPE_V8HI:
   12687              :     case V4DI_FTYPE_V8HI:
   12688              :     case V4DI_FTYPE_V4SI:
   12689              :     case V4DI_FTYPE_V2DI:
   12690              :     case UQI_FTYPE_UQI:
   12691              :     case UHI_FTYPE_UHI:
   12692              :     case USI_FTYPE_USI:
   12693              :     case USI_FTYPE_UQI:
   12694              :     case USI_FTYPE_UHI:
   12695              :     case UDI_FTYPE_UDI:
   12696              :     case UHI_FTYPE_V16QI:
   12697              :     case USI_FTYPE_V32QI:
   12698              :     case UDI_FTYPE_V64QI:
   12699              :     case V16QI_FTYPE_UHI:
   12700              :     case V32QI_FTYPE_USI:
   12701              :     case V64QI_FTYPE_UDI:
   12702              :     case V8HI_FTYPE_UQI:
   12703              :     case V16HI_FTYPE_UHI:
   12704              :     case V32HI_FTYPE_USI:
   12705              :     case V4SI_FTYPE_UQI:
   12706              :     case V8SI_FTYPE_UQI:
   12707              :     case V4SI_FTYPE_UHI:
   12708              :     case V8SI_FTYPE_UHI:
   12709              :     case UQI_FTYPE_V8HI:
   12710              :     case UHI_FTYPE_V16HI:
   12711              :     case USI_FTYPE_V32HI:
   12712              :     case UQI_FTYPE_V4SI:
   12713              :     case UQI_FTYPE_V8SI:
   12714              :     case UHI_FTYPE_V16SI:
   12715              :     case UQI_FTYPE_V2DI:
   12716              :     case UQI_FTYPE_V4DI:
   12717              :     case UQI_FTYPE_V8DI:
   12718              :     case V16SI_FTYPE_UHI:
   12719              :     case V2DI_FTYPE_UQI:
   12720              :     case V4DI_FTYPE_UQI:
   12721              :     case V16SI_FTYPE_INT:
   12722              :     case V16SF_FTYPE_V8SF:
   12723              :     case V16SI_FTYPE_V8SI:
   12724              :     case V16SF_FTYPE_V4SF:
   12725              :     case V16SI_FTYPE_V4SI:
   12726              :     case V16SI_FTYPE_V16SF:
   12727              :     case V16SI_FTYPE_V16SI:
   12728              :     case V64QI_FTYPE_V64QI:
   12729              :     case V32HI_FTYPE_V32HI:
   12730              :     case V16SF_FTYPE_V16SF:
   12731              :     case V8DI_FTYPE_UQI:
   12732              :     case V8DI_FTYPE_V8DI:
   12733              :     case V8DF_FTYPE_V4DF:
   12734              :     case V8DF_FTYPE_V2DF:
   12735              :     case V8DF_FTYPE_V8DF:
   12736              :     case V4DI_FTYPE_V4DI:
   12737              :     case V16BF_FTYPE_V16SF:
   12738              :     case V8BF_FTYPE_V8SF:
   12739              :     case V8BF_FTYPE_V4SF:
   12740              :     case V16QI_FTYPE_V32QI:
   12741              :     case V32QI_FTYPE_V64QI:
   12742              :       nargs = 1;
   12743              :       break;
   12744           52 :     case V4SF_FTYPE_V4SF_VEC_MERGE:
   12745           52 :     case V2DF_FTYPE_V2DF_VEC_MERGE:
   12746           52 :       return ix86_expand_unop_vec_merge_builtin (icode, exp, target);
   12747         9386 :     case FLOAT128_FTYPE_FLOAT128_FLOAT128:
   12748         9386 :     case V16QI_FTYPE_V16QI_V16QI:
   12749         9386 :     case V16QI_FTYPE_V8HI_V8HI:
   12750         9386 :     case V16HF_FTYPE_V16HF_V16HF:
   12751         9386 :     case V16SF_FTYPE_V16SF_V16SF:
   12752         9386 :     case V16SI_FTYPE_V16SI_V16SI:
   12753         9386 :     case V8QI_FTYPE_V8QI_V8QI:
   12754         9386 :     case V8QI_FTYPE_V4HI_V4HI:
   12755         9386 :     case V8HI_FTYPE_V8HI_V8HI:
   12756         9386 :     case V8HI_FTYPE_V16QI_V16QI:
   12757         9386 :     case V8HI_FTYPE_V4SI_V4SI:
   12758         9386 :     case V8HF_FTYPE_V8HF_V8HF:
   12759         9386 :     case V8SF_FTYPE_V8SF_V8SF:
   12760         9386 :     case V8SF_FTYPE_V8SF_V8SI:
   12761         9386 :     case V8DF_FTYPE_V8DF_V8DF:
   12762         9386 :     case V4SI_FTYPE_V4SI_V4SI:
   12763         9386 :     case V4SI_FTYPE_V8HI_V8HI:
   12764         9386 :     case V4SI_FTYPE_V2DF_V2DF:
   12765         9386 :     case V4HI_FTYPE_V4HI_V4HI:
   12766         9386 :     case V4HI_FTYPE_V8QI_V8QI:
   12767         9386 :     case V4HI_FTYPE_V2SI_V2SI:
   12768         9386 :     case V4DF_FTYPE_V4DF_V4DF:
   12769         9386 :     case V4DF_FTYPE_V4DF_V4DI:
   12770         9386 :     case V4SF_FTYPE_V4SF_V4SF:
   12771         9386 :     case V4SF_FTYPE_V4SF_V4SI:
   12772         9386 :     case V4SF_FTYPE_V4SF_V2SI:
   12773         9386 :     case V4SF_FTYPE_V4SF_V2DF:
   12774         9386 :     case V4SF_FTYPE_V4SF_UINT:
   12775         9386 :     case V4SF_FTYPE_V4SF_DI:
   12776         9386 :     case V4SF_FTYPE_V4SF_SI:
   12777         9386 :     case V4DI_FTYPE_V4DI_V2DI:
   12778         9386 :     case V2DI_FTYPE_V2DI_V2DI:
   12779         9386 :     case V2DI_FTYPE_V16QI_V16QI:
   12780         9386 :     case V2DI_FTYPE_V4SI_V4SI:
   12781         9386 :     case V2DI_FTYPE_V2DI_V16QI:
   12782         9386 :     case V2SI_FTYPE_V2SI_V2SI:
   12783         9386 :     case V2SI_FTYPE_V4HI_V4HI:
   12784         9386 :     case V2SI_FTYPE_V2SF_V2SF:
   12785         9386 :     case V2DF_FTYPE_V2DF_V2DF:
   12786         9386 :     case V2DF_FTYPE_V2DF_V4SF:
   12787         9386 :     case V2DF_FTYPE_V2DF_V2DI:
   12788         9386 :     case V2DF_FTYPE_V2DF_DI:
   12789         9386 :     case V2DF_FTYPE_V2DF_SI:
   12790         9386 :     case V2DF_FTYPE_V2DF_UINT:
   12791         9386 :     case V2SF_FTYPE_V2SF_V2SF:
   12792         9386 :     case V1DI_FTYPE_V1DI_V1DI:
   12793         9386 :     case V1DI_FTYPE_V8QI_V8QI:
   12794         9386 :     case V1DI_FTYPE_V2SI_V2SI:
   12795         9386 :     case V32QI_FTYPE_V16HI_V16HI:
   12796         9386 :     case V16HI_FTYPE_V8SI_V8SI:
   12797         9386 :     case V64QI_FTYPE_V64QI_V64QI:
   12798         9386 :     case V32QI_FTYPE_V32QI_V32QI:
   12799         9386 :     case V32BF_FTYPE_V32BF_V32BF:
   12800         9386 :     case V16BF_FTYPE_V16BF_V16BF:
   12801         9386 :     case V8BF_FTYPE_V8BF_V8BF:
   12802         9386 :     case V16HI_FTYPE_V32QI_V32QI:
   12803         9386 :     case V16HI_FTYPE_V16HI_V16HI:
   12804         9386 :     case V8SI_FTYPE_V4DF_V4DF:
   12805         9386 :     case V8SI_FTYPE_V8SI_V8SI:
   12806         9386 :     case V8SI_FTYPE_V16HI_V16HI:
   12807         9386 :     case V4DI_FTYPE_V4DI_V4DI:
   12808         9386 :     case V4DI_FTYPE_V8SI_V8SI:
   12809         9386 :     case V4DI_FTYPE_V32QI_V32QI:
   12810         9386 :     case V8DI_FTYPE_V64QI_V64QI:
   12811         9386 :       if (comparison == UNKNOWN)
   12812         8858 :         return ix86_expand_binop_builtin (icode, exp, target);
   12813              :       nargs = 2;
   12814              :       break;
   12815           80 :     case V4SF_FTYPE_V4SF_V4SF_SWAP:
   12816           80 :     case V2DF_FTYPE_V2DF_V2DF_SWAP:
   12817           80 :       gcc_assert (comparison != UNKNOWN);
   12818              :       nargs = 2;
   12819              :       swap = true;
   12820              :       break;
   12821         1481 :     case V16HI_FTYPE_V16HI_V8HI_COUNT:
   12822         1481 :     case V16HI_FTYPE_V16HI_SI_COUNT:
   12823         1481 :     case V8SI_FTYPE_V8SI_V4SI_COUNT:
   12824         1481 :     case V8SI_FTYPE_V8SI_SI_COUNT:
   12825         1481 :     case V4DI_FTYPE_V4DI_V2DI_COUNT:
   12826         1481 :     case V4DI_FTYPE_V4DI_INT_COUNT:
   12827         1481 :     case V8HI_FTYPE_V8HI_V8HI_COUNT:
   12828         1481 :     case V8HI_FTYPE_V8HI_SI_COUNT:
   12829         1481 :     case V4SI_FTYPE_V4SI_V4SI_COUNT:
   12830         1481 :     case V4SI_FTYPE_V4SI_SI_COUNT:
   12831         1481 :     case V4HI_FTYPE_V4HI_V4HI_COUNT:
   12832         1481 :     case V4HI_FTYPE_V4HI_SI_COUNT:
   12833         1481 :     case V2DI_FTYPE_V2DI_V2DI_COUNT:
   12834         1481 :     case V2DI_FTYPE_V2DI_SI_COUNT:
   12835         1481 :     case V2SI_FTYPE_V2SI_V2SI_COUNT:
   12836         1481 :     case V2SI_FTYPE_V2SI_SI_COUNT:
   12837         1481 :     case V1DI_FTYPE_V1DI_V1DI_COUNT:
   12838         1481 :     case V1DI_FTYPE_V1DI_SI_COUNT:
   12839         1481 :       nargs = 2;
   12840         1481 :       second_arg_count = true;
   12841         1481 :       break;
   12842         1408 :     case V16HI_FTYPE_V16HI_INT_V16HI_UHI_COUNT:
   12843         1408 :     case V16HI_FTYPE_V16HI_V8HI_V16HI_UHI_COUNT:
   12844         1408 :     case V16SI_FTYPE_V16SI_INT_V16SI_UHI_COUNT:
   12845         1408 :     case V16SI_FTYPE_V16SI_V4SI_V16SI_UHI_COUNT:
   12846         1408 :     case V2DI_FTYPE_V2DI_INT_V2DI_UQI_COUNT:
   12847         1408 :     case V2DI_FTYPE_V2DI_V2DI_V2DI_UQI_COUNT:
   12848         1408 :     case V32HI_FTYPE_V32HI_INT_V32HI_USI_COUNT:
   12849         1408 :     case V32HI_FTYPE_V32HI_V8HI_V32HI_USI_COUNT:
   12850         1408 :     case V4DI_FTYPE_V4DI_INT_V4DI_UQI_COUNT:
   12851         1408 :     case V4DI_FTYPE_V4DI_V2DI_V4DI_UQI_COUNT:
   12852         1408 :     case V4SI_FTYPE_V4SI_INT_V4SI_UQI_COUNT:
   12853         1408 :     case V4SI_FTYPE_V4SI_V4SI_V4SI_UQI_COUNT:
   12854         1408 :     case V8DI_FTYPE_V8DI_INT_V8DI_UQI_COUNT:
   12855         1408 :     case V8DI_FTYPE_V8DI_V2DI_V8DI_UQI_COUNT:
   12856         1408 :     case V8HI_FTYPE_V8HI_INT_V8HI_UQI_COUNT:
   12857         1408 :     case V8HI_FTYPE_V8HI_V8HI_V8HI_UQI_COUNT:
   12858         1408 :     case V8SI_FTYPE_V8SI_INT_V8SI_UQI_COUNT:
   12859         1408 :     case V8SI_FTYPE_V8SI_V4SI_V8SI_UQI_COUNT:
   12860         1408 :       nargs = 4;
   12861         1408 :       second_arg_count = true;
   12862         1408 :       break;
   12863          977 :     case UINT64_FTYPE_UINT64_UINT64:
   12864          977 :     case UINT_FTYPE_UINT_UINT:
   12865          977 :     case UINT_FTYPE_UINT_USHORT:
   12866          977 :     case UINT_FTYPE_UINT_UCHAR:
   12867          977 :     case UINT16_FTYPE_UINT16_INT:
   12868          977 :     case UINT8_FTYPE_UINT8_INT:
   12869          977 :     case UQI_FTYPE_UQI_UQI:
   12870          977 :     case UHI_FTYPE_UHI_UHI:
   12871          977 :     case USI_FTYPE_USI_USI:
   12872          977 :     case UDI_FTYPE_UDI_UDI:
   12873          977 :     case V16SI_FTYPE_V8DF_V8DF:
   12874          977 :     case V32BF_FTYPE_V16SF_V16SF:
   12875          977 :     case V16BF_FTYPE_V8SF_V8SF:
   12876          977 :     case V8BF_FTYPE_V4SF_V4SF:
   12877          977 :     case V16BF_FTYPE_V16SF_UHI:
   12878          977 :     case V8BF_FTYPE_V8SF_UQI:
   12879          977 :     case V8BF_FTYPE_V4SF_UQI:
   12880          977 :     case V16QI_FTYPE_V16QI_V8HF:
   12881          977 :       nargs = 2;
   12882          977 :       break;
   12883          639 :     case V2DI_FTYPE_V2DI_INT_CONVERT:
   12884          639 :       nargs = 2;
   12885          639 :       rmode = V1TImode;
   12886          639 :       nargs_constant = 1;
   12887          639 :       break;
   12888           42 :     case V4DI_FTYPE_V4DI_INT_CONVERT:
   12889           42 :       nargs = 2;
   12890           42 :       rmode = V2TImode;
   12891           42 :       nargs_constant = 1;
   12892           42 :       break;
   12893           16 :     case V8DI_FTYPE_V8DI_INT_CONVERT:
   12894           16 :       nargs = 2;
   12895           16 :       rmode = V4TImode;
   12896           16 :       nargs_constant = 1;
   12897           16 :       break;
   12898         2424 :     case V8HI_FTYPE_V8HI_INT:
   12899         2424 :     case V8HI_FTYPE_V8SF_INT:
   12900         2424 :     case V16HI_FTYPE_V16SF_INT:
   12901         2424 :     case V8HI_FTYPE_V4SF_INT:
   12902         2424 :     case V8SF_FTYPE_V8SF_INT:
   12903         2424 :     case V4SF_FTYPE_V16SF_INT:
   12904         2424 :     case V16SF_FTYPE_V16SF_INT:
   12905         2424 :     case V4SI_FTYPE_V4SI_INT:
   12906         2424 :     case V4SI_FTYPE_V8SI_INT:
   12907         2424 :     case V4HI_FTYPE_V4HI_INT:
   12908         2424 :     case V4DF_FTYPE_V4DF_INT:
   12909         2424 :     case V4DF_FTYPE_V8DF_INT:
   12910         2424 :     case V4SF_FTYPE_V4SF_INT:
   12911         2424 :     case V4SF_FTYPE_V8SF_INT:
   12912         2424 :     case V2DI_FTYPE_V2DI_INT:
   12913         2424 :     case V2DF_FTYPE_V2DF_INT:
   12914         2424 :     case V2DF_FTYPE_V4DF_INT:
   12915         2424 :     case V16HI_FTYPE_V16HI_INT:
   12916         2424 :     case V8SI_FTYPE_V8SI_INT:
   12917         2424 :     case V16SI_FTYPE_V16SI_INT:
   12918         2424 :     case V4SI_FTYPE_V16SI_INT:
   12919         2424 :     case V4DI_FTYPE_V4DI_INT:
   12920         2424 :     case V2DI_FTYPE_V4DI_INT:
   12921         2424 :     case V4DI_FTYPE_V8DI_INT:
   12922         2424 :     case UQI_FTYPE_UQI_UQI_CONST:
   12923         2424 :     case UHI_FTYPE_UHI_UQI:
   12924         2424 :     case USI_FTYPE_USI_UQI:
   12925         2424 :     case UDI_FTYPE_UDI_UQI:
   12926         2424 :       nargs = 2;
   12927         2424 :       nargs_constant = 1;
   12928         2424 :       break;
   12929        19988 :     case V16QI_FTYPE_V16QI_V16QI_V16QI:
   12930        19988 :     case V8SF_FTYPE_V8SF_V8SF_V8SF:
   12931        19988 :     case V4DF_FTYPE_V4DF_V4DF_V4DF:
   12932        19988 :     case V4SF_FTYPE_V4SF_V4SF_V4SF:
   12933        19988 :     case V2DF_FTYPE_V2DF_V2DF_V2DF:
   12934        19988 :     case V32QI_FTYPE_V32QI_V32QI_V32QI:
   12935        19988 :     case UHI_FTYPE_V16SI_V16SI_UHI:
   12936        19988 :     case UQI_FTYPE_V8DI_V8DI_UQI:
   12937        19988 :     case V16HI_FTYPE_V16SI_V16HI_UHI:
   12938        19988 :     case V16QI_FTYPE_V16SI_V16QI_UHI:
   12939        19988 :     case V16QI_FTYPE_V8DI_V16QI_UQI:
   12940        19988 :     case V32HF_FTYPE_V32HF_V32HF_USI:
   12941        19988 :     case V16SF_FTYPE_V16SF_V16SF_UHI:
   12942        19988 :     case V16SF_FTYPE_V4SF_V16SF_UHI:
   12943        19988 :     case V16SI_FTYPE_SI_V16SI_UHI:
   12944        19988 :     case V16SI_FTYPE_V16HI_V16SI_UHI:
   12945        19988 :     case V16SI_FTYPE_V16QI_V16SI_UHI:
   12946        19988 :     case V8SF_FTYPE_V4SF_V8SF_UQI:
   12947        19988 :     case V4DF_FTYPE_V2DF_V4DF_UQI:
   12948        19988 :     case V8SI_FTYPE_V4SI_V8SI_UQI:
   12949        19988 :     case V8SI_FTYPE_SI_V8SI_UQI:
   12950        19988 :     case V4SI_FTYPE_V4SI_V4SI_UQI:
   12951        19988 :     case V4SI_FTYPE_SI_V4SI_UQI:
   12952        19988 :     case V4DI_FTYPE_V2DI_V4DI_UQI:
   12953        19988 :     case V4DI_FTYPE_DI_V4DI_UQI:
   12954        19988 :     case V2DI_FTYPE_V2DI_V2DI_UQI:
   12955        19988 :     case V2DI_FTYPE_DI_V2DI_UQI:
   12956        19988 :     case V64QI_FTYPE_V64QI_V64QI_UDI:
   12957        19988 :     case V64QI_FTYPE_V16QI_V64QI_UDI:
   12958        19988 :     case V64QI_FTYPE_QI_V64QI_UDI:
   12959        19988 :     case V32QI_FTYPE_V32QI_V32QI_USI:
   12960        19988 :     case V32QI_FTYPE_V16QI_V32QI_USI:
   12961        19988 :     case V32QI_FTYPE_QI_V32QI_USI:
   12962        19988 :     case V16QI_FTYPE_V16QI_V16QI_UHI:
   12963        19988 :     case V16QI_FTYPE_QI_V16QI_UHI:
   12964        19988 :     case V32HI_FTYPE_V8HI_V32HI_USI:
   12965        19988 :     case V32HI_FTYPE_V32BF_V32HI_USI:
   12966        19988 :     case V32HI_FTYPE_HI_V32HI_USI:
   12967        19988 :     case V16HI_FTYPE_V8HI_V16HI_UHI:
   12968        19988 :     case V16HI_FTYPE_V16BF_V16HI_UHI:
   12969        19988 :     case V16HI_FTYPE_HI_V16HI_UHI:
   12970        19988 :     case V8HI_FTYPE_V8HI_V8HI_UQI:
   12971        19988 :     case V8HI_FTYPE_V8BF_V8HI_UQI:
   12972        19988 :     case V8BF_FTYPE_V8BF_V8BF_UQI:
   12973        19988 :     case V8HI_FTYPE_HI_V8HI_UQI:
   12974        19988 :     case V16HF_FTYPE_V16HF_V16HF_UHI:
   12975        19988 :     case V8SF_FTYPE_V8HI_V8SF_UQI:
   12976        19988 :     case V4SF_FTYPE_V8HI_V4SF_UQI:
   12977        19988 :     case V8SI_FTYPE_V8HF_V8SI_UQI:
   12978        19988 :     case V8SF_FTYPE_V8HF_V8SF_UQI:
   12979        19988 :     case V8SI_FTYPE_V8SF_V8SI_UQI:
   12980        19988 :     case V4SI_FTYPE_V4SF_V4SI_UQI:
   12981        19988 :     case V4SI_FTYPE_V8HF_V4SI_UQI:
   12982        19988 :     case V4SF_FTYPE_V8HF_V4SF_UQI:
   12983        19988 :     case V4DI_FTYPE_V8HF_V4DI_UQI:
   12984        19988 :     case V4DI_FTYPE_V4SF_V4DI_UQI:
   12985        19988 :     case V2DI_FTYPE_V8HF_V2DI_UQI:
   12986        19988 :     case V2DI_FTYPE_V4SF_V2DI_UQI:
   12987        19988 :     case V8HF_FTYPE_V8HF_V8HF_UQI:
   12988        19988 :     case V8HF_FTYPE_V8HF_V8HF_V8HF:
   12989        19988 :     case V8HF_FTYPE_V8HI_V8HF_UQI:
   12990        19988 :     case V8HF_FTYPE_V8SI_V8HF_UQI:
   12991        19988 :     case V8HF_FTYPE_V8SF_V8HF_UQI:
   12992        19988 :     case V8HF_FTYPE_V4SI_V8HF_UQI:
   12993        19988 :     case V8HF_FTYPE_V4SF_V8HF_UQI:
   12994        19988 :     case V8HF_FTYPE_V4DI_V8HF_UQI:
   12995        19988 :     case V8HF_FTYPE_V4DF_V8HF_UQI:
   12996        19988 :     case V8HF_FTYPE_V2DI_V8HF_UQI:
   12997        19988 :     case V8HF_FTYPE_V2DF_V8HF_UQI:
   12998        19988 :     case V4SF_FTYPE_V4DI_V4SF_UQI:
   12999        19988 :     case V4SF_FTYPE_V2DI_V4SF_UQI:
   13000        19988 :     case V4DF_FTYPE_V4DI_V4DF_UQI:
   13001        19988 :     case V4DF_FTYPE_V8HF_V4DF_UQI:
   13002        19988 :     case V2DF_FTYPE_V8HF_V2DF_UQI:
   13003        19988 :     case V2DF_FTYPE_V2DI_V2DF_UQI:
   13004        19988 :     case V16QI_FTYPE_V8HI_V16QI_UQI:
   13005        19988 :     case V16QI_FTYPE_V16HI_V16QI_UHI:
   13006        19988 :     case V16QI_FTYPE_V4SI_V16QI_UQI:
   13007        19988 :     case V16QI_FTYPE_V8SI_V16QI_UQI:
   13008        19988 :     case V8HI_FTYPE_V8HF_V8HI_UQI:
   13009        19988 :     case V8HI_FTYPE_V4SI_V8HI_UQI:
   13010        19988 :     case V8HI_FTYPE_V8SI_V8HI_UQI:
   13011        19988 :     case V16QI_FTYPE_V2DI_V16QI_UQI:
   13012        19988 :     case V16QI_FTYPE_V4DI_V16QI_UQI:
   13013        19988 :     case V8HI_FTYPE_V2DI_V8HI_UQI:
   13014        19988 :     case V8HI_FTYPE_V4DI_V8HI_UQI:
   13015        19988 :     case V4SI_FTYPE_V2DI_V4SI_UQI:
   13016        19988 :     case V4SI_FTYPE_V4DI_V4SI_UQI:
   13017        19988 :     case V32QI_FTYPE_V32HI_V32QI_USI:
   13018        19988 :     case UHI_FTYPE_V16QI_V16QI_UHI:
   13019        19988 :     case USI_FTYPE_V32QI_V32QI_USI:
   13020        19988 :     case UDI_FTYPE_V64QI_V64QI_UDI:
   13021        19988 :     case UQI_FTYPE_V8HI_V8HI_UQI:
   13022        19988 :     case UHI_FTYPE_V16HI_V16HI_UHI:
   13023        19988 :     case USI_FTYPE_V32HI_V32HI_USI:
   13024        19988 :     case UQI_FTYPE_V4SI_V4SI_UQI:
   13025        19988 :     case UQI_FTYPE_V8SI_V8SI_UQI:
   13026        19988 :     case UQI_FTYPE_V2DI_V2DI_UQI:
   13027        19988 :     case UQI_FTYPE_V4DI_V4DI_UQI:
   13028        19988 :     case V4SF_FTYPE_V2DF_V4SF_UQI:
   13029        19988 :     case V4SF_FTYPE_V4DF_V4SF_UQI:
   13030        19988 :     case V16SI_FTYPE_V16SI_V16SI_UHI:
   13031        19988 :     case V16SI_FTYPE_V4SI_V16SI_UHI:
   13032        19988 :     case V2DI_FTYPE_V4SI_V2DI_UQI:
   13033        19988 :     case V2DI_FTYPE_V8HI_V2DI_UQI:
   13034        19988 :     case V2DI_FTYPE_V16QI_V2DI_UQI:
   13035        19988 :     case V4DI_FTYPE_V4DI_V4DI_UQI:
   13036        19988 :     case V4DI_FTYPE_V4SI_V4DI_UQI:
   13037        19988 :     case V4DI_FTYPE_V8HI_V4DI_UQI:
   13038        19988 :     case V4DI_FTYPE_V16QI_V4DI_UQI:
   13039        19988 :     case V4DI_FTYPE_V4DF_V4DI_UQI:
   13040        19988 :     case V2DI_FTYPE_V2DF_V2DI_UQI:
   13041        19988 :     case V4SI_FTYPE_V4DF_V4SI_UQI:
   13042        19988 :     case V4SI_FTYPE_V2DF_V4SI_UQI:
   13043        19988 :     case V4SI_FTYPE_V8HI_V4SI_UQI:
   13044        19988 :     case V4SI_FTYPE_V16QI_V4SI_UQI:
   13045        19988 :     case V4DI_FTYPE_V4DI_V4DI_V4DI:
   13046        19988 :     case V8DF_FTYPE_V2DF_V8DF_UQI:
   13047        19988 :     case V8DF_FTYPE_V4DF_V8DF_UQI:
   13048        19988 :     case V8DF_FTYPE_V8DF_V8DF_UQI:
   13049        19988 :     case V8SF_FTYPE_V8SF_V8SF_UQI:
   13050        19988 :     case V8SF_FTYPE_V8SI_V8SF_UQI:
   13051        19988 :     case V4DF_FTYPE_V4DF_V4DF_UQI:
   13052        19988 :     case V4SF_FTYPE_V4SF_V4SF_UQI:
   13053        19988 :     case V2DF_FTYPE_V2DF_V2DF_UQI:
   13054        19988 :     case V2DF_FTYPE_V4SF_V2DF_UQI:
   13055        19988 :     case V2DF_FTYPE_V4SI_V2DF_UQI:
   13056        19988 :     case V4SF_FTYPE_V4SI_V4SF_UQI:
   13057        19988 :     case V4DF_FTYPE_V4SF_V4DF_UQI:
   13058        19988 :     case V4DF_FTYPE_V4SI_V4DF_UQI:
   13059        19988 :     case V8SI_FTYPE_V8SI_V8SI_UQI:
   13060        19988 :     case V8SI_FTYPE_V8HI_V8SI_UQI:
   13061        19988 :     case V8SI_FTYPE_V16QI_V8SI_UQI:
   13062        19988 :     case V8DF_FTYPE_V8SI_V8DF_UQI:
   13063        19988 :     case V8DI_FTYPE_DI_V8DI_UQI:
   13064        19988 :     case V16SF_FTYPE_V8SF_V16SF_UHI:
   13065        19988 :     case V16SI_FTYPE_V8SI_V16SI_UHI:
   13066        19988 :     case V16HF_FTYPE_V16HI_V16HF_UHI:
   13067        19988 :     case V16HF_FTYPE_V16HF_V16HF_V16HF:
   13068        19988 :     case V16HI_FTYPE_V16HF_V16HI_UHI:
   13069        19988 :     case V16HI_FTYPE_V16HI_V16HI_UHI:
   13070        19988 :     case V16BF_FTYPE_V16BF_V16BF_UHI:
   13071        19988 :     case V8HI_FTYPE_V16QI_V8HI_UQI:
   13072        19988 :     case V16HI_FTYPE_V16QI_V16HI_UHI:
   13073        19988 :     case V32HI_FTYPE_V32HI_V32HI_USI:
   13074        19988 :     case V32BF_FTYPE_V32BF_V32BF_USI:
   13075        19988 :     case V32HI_FTYPE_V32QI_V32HI_USI:
   13076        19988 :     case V8DI_FTYPE_V16QI_V8DI_UQI:
   13077        19988 :     case V8DI_FTYPE_V2DI_V8DI_UQI:
   13078        19988 :     case V8DI_FTYPE_V4DI_V8DI_UQI:
   13079        19988 :     case V8DI_FTYPE_V8DI_V8DI_UQI:
   13080        19988 :     case V8DI_FTYPE_V8HI_V8DI_UQI:
   13081        19988 :     case V8DI_FTYPE_V8SI_V8DI_UQI:
   13082        19988 :     case V8HI_FTYPE_V8DI_V8HI_UQI:
   13083        19988 :     case V8SI_FTYPE_V8DI_V8SI_UQI:
   13084        19988 :     case V4SI_FTYPE_V4SI_V4SI_V4SI:
   13085        19988 :     case V4DI_FTYPE_V4DI_V4DI_V2DI:
   13086        19988 :     case V16SI_FTYPE_V16SI_V16SI_V16SI:
   13087        19988 :     case V8DI_FTYPE_V8DI_V8DI_V8DI:
   13088        19988 :     case V32HI_FTYPE_V32HI_V32HI_V32HI:
   13089        19988 :     case V2DI_FTYPE_V2DI_V2DI_V2DI:
   13090        19988 :     case V16HI_FTYPE_V16HI_V16HI_V16HI:
   13091        19988 :     case V8SI_FTYPE_V8SI_V8SI_V8SI:
   13092        19988 :     case V8HI_FTYPE_V8HI_V8HI_V8HI:
   13093        19988 :     case V32BF_FTYPE_V16SF_V16SF_USI:
   13094        19988 :     case V16BF_FTYPE_V8SF_V8SF_UHI:
   13095        19988 :     case V8BF_FTYPE_V4SF_V4SF_UQI:
   13096        19988 :     case V16BF_FTYPE_V16SF_V16BF_UHI:
   13097        19988 :     case V8BF_FTYPE_V8SF_V8BF_UQI:
   13098        19988 :     case V8BF_FTYPE_V4SF_V8BF_UQI:
   13099        19988 :     case V16SF_FTYPE_V16SF_V32BF_V32BF:
   13100        19988 :     case V8SF_FTYPE_V8SF_V16BF_V16BF:
   13101        19988 :     case V4SF_FTYPE_V4SF_V8BF_V8BF:
   13102        19988 :     case V16QI_FTYPE_V16QI_V8HF_V8HF:
   13103        19988 :     case V32QI_FTYPE_V32QI_V16HF_V16HF:
   13104        19988 :     case V64QI_FTYPE_V64QI_V32HF_V32HF:
   13105        19988 :     case V16QI_FTYPE_V8HF_V16QI_UQI:
   13106        19988 :     case V16QI_FTYPE_V16HF_V16QI_UHI:
   13107        19988 :     case V32QI_FTYPE_V32HF_V32QI_USI:
   13108        19988 :     case V8HF_FTYPE_V16QI_V8HF_UQI:
   13109        19988 :     case V16HF_FTYPE_V16QI_V16HF_UHI:
   13110        19988 :     case V32HF_FTYPE_V32QI_V32HF_USI:
   13111        19988 :     case V16SI_FTYPE_V16SF_V16SI_UHI:
   13112        19988 :     case V32HI_FTYPE_V32HF_V32HI_USI:
   13113        19988 :     case V8DI_FTYPE_V8SF_V8DI_UQI:
   13114        19988 :     case V8DI_FTYPE_V8DF_V8DI_UQI:
   13115        19988 :     case V8SI_FTYPE_V8DF_V8SI_UQI:
   13116        19988 :     case V16QI_FTYPE_V4SF_V16QI_UQI:
   13117        19988 :     case V16QI_FTYPE_V8SF_V16QI_UQI:
   13118        19988 :     case V16QI_FTYPE_V16SF_V16QI_UHI:
   13119        19988 :     case V4SF_FTYPE_V16QI_V4SF_UQI:
   13120        19988 :     case V8SF_FTYPE_V16QI_V8SF_UQI:
   13121        19988 :     case V16SF_FTYPE_V16QI_V16SF_UHI:
   13122        19988 :     case V64QI_FTYPE_V32QI_V64QI_UDI:
   13123        19988 :       nargs = 3;
   13124        19988 :       break;
   13125         1484 :     case V32QI_FTYPE_V32QI_V32QI_INT:
   13126         1484 :     case V16HI_FTYPE_V16HI_V16HI_INT:
   13127         1484 :     case V16QI_FTYPE_V16QI_V16QI_INT:
   13128         1484 :     case V4DI_FTYPE_V4DI_V4DI_INT:
   13129         1484 :     case V8HI_FTYPE_V8HI_V8HI_INT:
   13130         1484 :     case V8SI_FTYPE_V8SI_V8SI_INT:
   13131         1484 :     case V8SI_FTYPE_V8SI_V4SI_INT:
   13132         1484 :     case V8SF_FTYPE_V8SF_V8SF_INT:
   13133         1484 :     case V8SF_FTYPE_V8SF_V4SF_INT:
   13134         1484 :     case V4SI_FTYPE_V4SI_V4SI_INT:
   13135         1484 :     case V4DF_FTYPE_V4DF_V4DF_INT:
   13136         1484 :     case V16SF_FTYPE_V16SF_V16SF_INT:
   13137         1484 :     case V16SF_FTYPE_V16SF_V4SF_INT:
   13138         1484 :     case V16SI_FTYPE_V16SI_V4SI_INT:
   13139         1484 :     case V4DF_FTYPE_V4DF_V2DF_INT:
   13140         1484 :     case V4SF_FTYPE_V4SF_V4SF_INT:
   13141         1484 :     case V2DI_FTYPE_V2DI_V2DI_INT:
   13142         1484 :     case V4DI_FTYPE_V4DI_V2DI_INT:
   13143         1484 :     case V2DF_FTYPE_V2DF_V2DF_INT:
   13144         1484 :     case UQI_FTYPE_V8DI_V8UDI_INT:
   13145         1484 :     case UQI_FTYPE_V8DF_V8DF_INT:
   13146         1484 :     case UQI_FTYPE_V2DF_V2DF_INT:
   13147         1484 :     case UQI_FTYPE_V4SF_V4SF_INT:
   13148         1484 :     case UHI_FTYPE_V16SI_V16SI_INT:
   13149         1484 :     case UHI_FTYPE_V16SF_V16SF_INT:
   13150         1484 :     case V64QI_FTYPE_V64QI_V64QI_INT:
   13151         1484 :     case V32HI_FTYPE_V32HI_V32HI_INT:
   13152         1484 :     case V16SI_FTYPE_V16SI_V16SI_INT:
   13153         1484 :     case V8DI_FTYPE_V8DI_V8DI_INT:
   13154         1484 :       nargs = 3;
   13155         1484 :       nargs_constant = 1;
   13156         1484 :       break;
   13157           47 :     case V4DI_FTYPE_V4DI_V4DI_INT_CONVERT:
   13158           47 :       nargs = 3;
   13159           47 :       rmode = V4DImode;
   13160           47 :       nargs_constant = 1;
   13161           47 :       break;
   13162           80 :     case V2DI_FTYPE_V2DI_V2DI_INT_CONVERT:
   13163           80 :       nargs = 3;
   13164           80 :       rmode = V2DImode;
   13165           80 :       nargs_constant = 1;
   13166           80 :       break;
   13167           48 :     case V1DI_FTYPE_V1DI_V1DI_INT_CONVERT:
   13168           48 :       nargs = 3;
   13169           48 :       rmode = DImode;
   13170           48 :       nargs_constant = 1;
   13171           48 :       break;
   13172           20 :     case V2DI_FTYPE_V2DI_UINT_UINT:
   13173           20 :       nargs = 3;
   13174           20 :       nargs_constant = 2;
   13175           20 :       break;
   13176            8 :     case V8DI_FTYPE_V8DI_V8DI_INT_CONVERT:
   13177            8 :       nargs = 3;
   13178            8 :       rmode = V8DImode;
   13179            8 :       nargs_constant = 1;
   13180            8 :       break;
   13181           16 :     case V8DI_FTYPE_V8DI_V8DI_INT_V8DI_UDI_CONVERT:
   13182           16 :       nargs = 5;
   13183           16 :       rmode = V8DImode;
   13184           16 :       mask_pos = 2;
   13185           16 :       nargs_constant = 1;
   13186           16 :       break;
   13187          320 :     case QI_FTYPE_V8DF_INT_UQI:
   13188          320 :     case QI_FTYPE_V4DF_INT_UQI:
   13189          320 :     case QI_FTYPE_V2DF_INT_UQI:
   13190          320 :     case HI_FTYPE_V16SF_INT_UHI:
   13191          320 :     case QI_FTYPE_V8SF_INT_UQI:
   13192          320 :     case QI_FTYPE_V4SF_INT_UQI:
   13193          320 :     case QI_FTYPE_V8HF_INT_UQI:
   13194          320 :     case HI_FTYPE_V16HF_INT_UHI:
   13195          320 :     case SI_FTYPE_V32HF_INT_USI:
   13196          320 :     case QI_FTYPE_V8BF_INT_UQI:
   13197          320 :     case HI_FTYPE_V16BF_INT_UHI:
   13198          320 :     case SI_FTYPE_V32BF_INT_USI:
   13199          320 :     case V4SI_FTYPE_V4SI_V4SI_UHI:
   13200          320 :     case V8SI_FTYPE_V8SI_V8SI_UHI:
   13201          320 :       nargs = 3;
   13202          320 :       mask_pos = 1;
   13203          320 :       nargs_constant = 1;
   13204          320 :       break;
   13205           17 :     case V4DI_FTYPE_V4DI_V4DI_INT_V4DI_USI_CONVERT:
   13206           17 :       nargs = 5;
   13207           17 :       rmode = V4DImode;
   13208           17 :       mask_pos = 2;
   13209           17 :       nargs_constant = 1;
   13210           17 :       break;
   13211           17 :     case V2DI_FTYPE_V2DI_V2DI_INT_V2DI_UHI_CONVERT:
   13212           17 :       nargs = 5;
   13213           17 :       rmode = V2DImode;
   13214           17 :       mask_pos = 2;
   13215           17 :       nargs_constant = 1;
   13216           17 :       break;
   13217        17783 :     case V32QI_FTYPE_V32QI_V32QI_V32QI_USI:
   13218        17783 :     case V32HI_FTYPE_V32HI_V32HI_V32HI_USI:
   13219        17783 :     case V32BF_FTYPE_V32BF_V32BF_V32BF_USI:
   13220        17783 :     case V32HI_FTYPE_V64QI_V64QI_V32HI_USI:
   13221        17783 :     case V16SI_FTYPE_V32HI_V32HI_V16SI_UHI:
   13222        17783 :     case V64QI_FTYPE_V64QI_V64QI_V64QI_UDI:
   13223        17783 :     case V32HI_FTYPE_V32HI_V8HI_V32HI_USI:
   13224        17783 :     case V16HI_FTYPE_V16HI_V8HI_V16HI_UHI:
   13225        17783 :     case V8SI_FTYPE_V8SI_V4SI_V8SI_UQI:
   13226        17783 :     case V4DI_FTYPE_V4DI_V2DI_V4DI_UQI:
   13227        17783 :     case V64QI_FTYPE_V32HI_V32HI_V64QI_UDI:
   13228        17783 :     case V32QI_FTYPE_V16HI_V16HI_V32QI_USI:
   13229        17783 :     case V16QI_FTYPE_V8HI_V8HI_V16QI_UHI:
   13230        17783 :     case V32HI_FTYPE_V16SI_V16SI_V32HI_USI:
   13231        17783 :     case V16HI_FTYPE_V8SI_V8SI_V16HI_UHI:
   13232        17783 :     case V8HI_FTYPE_V4SI_V4SI_V8HI_UQI:
   13233        17783 :     case V4DF_FTYPE_V4DF_V4DI_V4DF_UQI:
   13234        17783 :     case V32HF_FTYPE_V32HF_V32HF_V32HF_USI:
   13235        17783 :     case V8SF_FTYPE_V8SF_V8SI_V8SF_UQI:
   13236        17783 :     case V4SF_FTYPE_V4SF_V4SI_V4SF_UQI:
   13237        17783 :     case V2DF_FTYPE_V2DF_V2DI_V2DF_UQI:
   13238        17783 :     case V2DI_FTYPE_V4SI_V4SI_V2DI_UQI:
   13239        17783 :     case V4DI_FTYPE_V8SI_V8SI_V4DI_UQI:
   13240        17783 :     case V4DF_FTYPE_V4DI_V4DF_V4DF_UQI:
   13241        17783 :     case V8SF_FTYPE_V8SI_V8SF_V8SF_UQI:
   13242        17783 :     case V2DF_FTYPE_V2DI_V2DF_V2DF_UQI:
   13243        17783 :     case V4SF_FTYPE_V4SI_V4SF_V4SF_UQI:
   13244        17783 :     case V16SF_FTYPE_V16SF_V16SF_V16SF_UHI:
   13245        17783 :     case V16SF_FTYPE_V16SF_V16SI_V16SF_UHI:
   13246        17783 :     case V16SF_FTYPE_V16SI_V16SF_V16SF_UHI:
   13247        17783 :     case V16SI_FTYPE_V16SI_V16SI_V16SI_UHI:
   13248        17783 :     case V16SI_FTYPE_V16SI_V4SI_V16SI_UHI:
   13249        17783 :     case V8HI_FTYPE_V8HI_V8HI_V8HI_UQI:
   13250        17783 :     case V8BF_FTYPE_V8BF_V8BF_V8BF_UQI:
   13251        17783 :     case V8SI_FTYPE_V8SI_V8SI_V8SI_UQI:
   13252        17783 :     case V4SI_FTYPE_V4SI_V4SI_V4SI_UQI:
   13253        17783 :     case V16HF_FTYPE_V16HF_V16HF_V16HF_UQI:
   13254        17783 :     case V16HF_FTYPE_V16HF_V16HF_V16HF_UHI:
   13255        17783 :     case V8SF_FTYPE_V8SF_V8SF_V8SF_UQI:
   13256        17783 :     case V16QI_FTYPE_V16QI_V16QI_V16QI_UHI:
   13257        17783 :     case V16HI_FTYPE_V16HI_V16HI_V16HI_UHI:
   13258        17783 :     case V16BF_FTYPE_V16BF_V16BF_V16BF_UHI:
   13259        17783 :     case V2DI_FTYPE_V2DI_V2DI_V2DI_UQI:
   13260        17783 :     case V2DF_FTYPE_V2DF_V2DF_V2DF_UQI:
   13261        17783 :     case V4DI_FTYPE_V4DI_V4DI_V4DI_UQI:
   13262        17783 :     case V4DF_FTYPE_V4DF_V4DF_V4DF_UQI:
   13263        17783 :     case V8HF_FTYPE_V8HF_V8HF_V8HF_UQI:
   13264        17783 :     case V4SF_FTYPE_V4SF_V4SF_V4SF_UQI:
   13265        17783 :     case V8DF_FTYPE_V8DF_V8DF_V8DF_UQI:
   13266        17783 :     case V8DF_FTYPE_V8DF_V8DI_V8DF_UQI:
   13267        17783 :     case V8DF_FTYPE_V8DI_V8DF_V8DF_UQI:
   13268        17783 :     case V8DI_FTYPE_V16SI_V16SI_V8DI_UQI:
   13269        17783 :     case V8DI_FTYPE_V8DI_V2DI_V8DI_UQI:
   13270        17783 :     case V8DI_FTYPE_V8DI_V8DI_V8DI_UQI:
   13271        17783 :     case V8HI_FTYPE_V16QI_V16QI_V8HI_UQI:
   13272        17783 :     case V16HI_FTYPE_V32QI_V32QI_V16HI_UHI:
   13273        17783 :     case V8SI_FTYPE_V16HI_V16HI_V8SI_UQI:
   13274        17783 :     case V4SI_FTYPE_V8HI_V8HI_V4SI_UQI:
   13275        17783 :     case V32BF_FTYPE_V16SF_V16SF_V32BF_USI:
   13276        17783 :     case V16BF_FTYPE_V8SF_V8SF_V16BF_UHI:
   13277        17783 :     case V8BF_FTYPE_V4SF_V4SF_V8BF_UQI:
   13278        17783 :     case V32HF_FTYPE_V16SF_V16SF_V32HF_USI:
   13279        17783 :     case V16HF_FTYPE_V8SF_V8SF_V16HF_UHI:
   13280        17783 :     case V8HF_FTYPE_V4SF_V4SF_V8HF_UQI:
   13281        17783 :     case V16QI_FTYPE_V8HF_V8HF_V16QI_UHI:
   13282        17783 :     case V32QI_FTYPE_V16HF_V16HF_V32QI_USI:
   13283        17783 :     case V64QI_FTYPE_V32HF_V32HF_V64QI_UDI:
   13284        17783 :     case V16QI_FTYPE_V16QI_V8HF_V16QI_UHI:
   13285        17783 :     case V16QI_FTYPE_V32QI_V16HF_V16QI_UHI:
   13286        17783 :     case V32QI_FTYPE_V64QI_V32HF_V32QI_USI:
   13287        17783 :     case V16QI_FTYPE_V4SI_V4SF_V16QI_UQI:
   13288        17783 :     case V16QI_FTYPE_V8SI_V8SF_V16QI_UQI:
   13289        17783 :     case V16QI_FTYPE_V16SI_V16SF_V16QI_UHI:
   13290        17783 :       nargs = 4;
   13291        17783 :       break;
   13292           11 :     case V2DF_FTYPE_V2DF_V2DF_V2DI_INT:
   13293           11 :     case V4DF_FTYPE_V4DF_V4DF_V4DI_INT:
   13294           11 :     case V4SF_FTYPE_V4SF_V4SF_V4SI_INT:
   13295           11 :     case V8SF_FTYPE_V8SF_V8SF_V8SI_INT:
   13296           11 :     case V16SF_FTYPE_V16SF_V16SF_V16SI_INT:
   13297           11 :     case V4SI_FTYPE_V4SI_V4SI_V4SI_INT:
   13298           11 :       nargs = 4;
   13299           11 :       nargs_constant = 1;
   13300           11 :       break;
   13301         3718 :     case UQI_FTYPE_V4DI_V4DI_INT_UQI:
   13302         3718 :     case UQI_FTYPE_V8SI_V8SI_INT_UQI:
   13303         3718 :     case QI_FTYPE_V4DF_V4DF_INT_UQI:
   13304         3718 :     case QI_FTYPE_V8SF_V8SF_INT_UQI:
   13305         3718 :     case UHI_FTYPE_V16HF_V16HF_INT_UHI:
   13306         3718 :     case UQI_FTYPE_V2DI_V2DI_INT_UQI:
   13307         3718 :     case UQI_FTYPE_V4SI_V4SI_INT_UQI:
   13308         3718 :     case UQI_FTYPE_V2DF_V2DF_INT_UQI:
   13309         3718 :     case UQI_FTYPE_V4SF_V4SF_INT_UQI:
   13310         3718 :     case UQI_FTYPE_V8HF_V8HF_INT_UQI:
   13311         3718 :     case UDI_FTYPE_V64QI_V64QI_INT_UDI:
   13312         3718 :     case USI_FTYPE_V32QI_V32QI_INT_USI:
   13313         3718 :     case UHI_FTYPE_V16QI_V16QI_INT_UHI:
   13314         3718 :     case USI_FTYPE_V32HI_V32HI_INT_USI:
   13315         3718 :     case USI_FTYPE_V32BF_V32BF_INT_USI:
   13316         3718 :     case USI_FTYPE_V32HF_V32HF_INT_USI:
   13317         3718 :     case UHI_FTYPE_V16HI_V16HI_INT_UHI:
   13318         3718 :     case UHI_FTYPE_V16BF_V16BF_INT_UHI:
   13319         3718 :     case UQI_FTYPE_V8HI_V8HI_INT_UQI:
   13320         3718 :     case UQI_FTYPE_V8BF_V8BF_INT_UQI:
   13321         3718 :       nargs = 4;
   13322         3718 :       mask_pos = 1;
   13323         3718 :       nargs_constant = 1;
   13324         3718 :       break;
   13325           23 :     case V2DI_FTYPE_V2DI_V2DI_UINT_UINT:
   13326           23 :       nargs = 4;
   13327           23 :       nargs_constant = 2;
   13328           23 :       break;
   13329           69 :     case UCHAR_FTYPE_UCHAR_UINT_UINT_PUNSIGNED:
   13330           69 :     case UCHAR_FTYPE_UCHAR_ULONGLONG_ULONGLONG_PULONGLONG:
   13331           69 :     case V16SF_FTYPE_V16SF_V32BF_V32BF_UHI:
   13332           69 :     case V8SF_FTYPE_V8SF_V16BF_V16BF_UQI:
   13333           69 :     case V4SF_FTYPE_V4SF_V8BF_V8BF_UQI:
   13334           69 :       nargs = 4;
   13335           69 :       break;
   13336          679 :     case UQI_FTYPE_V8DI_V8DI_INT_UQI:
   13337          679 :     case UHI_FTYPE_V16SI_V16SI_INT_UHI:
   13338          679 :       mask_pos = 1;
   13339          679 :       nargs = 4;
   13340          679 :       nargs_constant = 1;
   13341          679 :       break;
   13342         4056 :     case V8SF_FTYPE_V8SF_INT_V8SF_UQI:
   13343         4056 :     case V4SF_FTYPE_V4SF_INT_V4SF_UQI:
   13344         4056 :     case V2DF_FTYPE_V4DF_INT_V2DF_UQI:
   13345         4056 :     case V2DI_FTYPE_V4DI_INT_V2DI_UQI:
   13346         4056 :     case V8SF_FTYPE_V16SF_INT_V8SF_UQI:
   13347         4056 :     case V8SI_FTYPE_V16SI_INT_V8SI_UQI:
   13348         4056 :     case V2DF_FTYPE_V8DF_INT_V2DF_UQI:
   13349         4056 :     case V2DI_FTYPE_V8DI_INT_V2DI_UQI:
   13350         4056 :     case V4SF_FTYPE_V8SF_INT_V4SF_UQI:
   13351         4056 :     case V4SI_FTYPE_V8SI_INT_V4SI_UQI:
   13352         4056 :     case V8HI_FTYPE_V8SF_INT_V8HI_UQI:
   13353         4056 :     case V8HI_FTYPE_V4SF_INT_V8HI_UQI:
   13354         4056 :     case V32HI_FTYPE_V32HI_INT_V32HI_USI:
   13355         4056 :     case V16HI_FTYPE_V16HI_INT_V16HI_UHI:
   13356         4056 :     case V8HI_FTYPE_V8HI_INT_V8HI_UQI:
   13357         4056 :     case V32BF_FTYPE_V32BF_INT_V32BF_USI:
   13358         4056 :     case V16BF_FTYPE_V16BF_INT_V16BF_UHI:
   13359         4056 :     case V8BF_FTYPE_V8BF_INT_V8BF_UQI:
   13360         4056 :     case V4DI_FTYPE_V4DI_INT_V4DI_UQI:
   13361         4056 :     case V2DI_FTYPE_V2DI_INT_V2DI_UQI:
   13362         4056 :     case V8SI_FTYPE_V8SI_INT_V8SI_UQI:
   13363         4056 :     case V4SI_FTYPE_V4SI_INT_V4SI_UQI:
   13364         4056 :     case V4DF_FTYPE_V4DF_INT_V4DF_UQI:
   13365         4056 :     case V2DF_FTYPE_V2DF_INT_V2DF_UQI:
   13366         4056 :     case V8DF_FTYPE_V8DF_INT_V8DF_UQI:
   13367         4056 :     case V16SF_FTYPE_V16SF_INT_V16SF_UHI:
   13368         4056 :     case V16HI_FTYPE_V16SF_INT_V16HI_UHI:
   13369         4056 :     case V16SI_FTYPE_V16SI_INT_V16SI_UHI:
   13370         4056 :     case V16HF_FTYPE_V16HF_INT_V16HF_UHI:
   13371         4056 :     case V8HF_FTYPE_V8HF_INT_V8HF_UQI:
   13372         4056 :     case V4SI_FTYPE_V16SI_INT_V4SI_UQI:
   13373         4056 :     case V4DI_FTYPE_V8DI_INT_V4DI_UQI:
   13374         4056 :     case V4DF_FTYPE_V8DF_INT_V4DF_UQI:
   13375         4056 :     case V4SF_FTYPE_V16SF_INT_V4SF_UQI:
   13376         4056 :     case V8DI_FTYPE_V8DI_INT_V8DI_UQI:
   13377         4056 :     case V16QI_FTYPE_V16QI_INT_V16QI_UHI:
   13378         4056 :     case V32QI_FTYPE_V32QI_INT_V32QI_USI:
   13379         4056 :     case V64QI_FTYPE_V64QI_INT_V64QI_UDI:
   13380         4056 :       nargs = 4;
   13381         4056 :       mask_pos = 2;
   13382         4056 :       nargs_constant = 1;
   13383         4056 :       break;
   13384         1726 :     case V16SF_FTYPE_V16SF_V4SF_INT_V16SF_UHI:
   13385         1726 :     case V16SI_FTYPE_V16SI_V4SI_INT_V16SI_UHI:
   13386         1726 :     case V8DF_FTYPE_V8DF_V8DF_INT_V8DF_UQI:
   13387         1726 :     case V8DI_FTYPE_V8DI_V8DI_INT_V8DI_UQI:
   13388         1726 :     case V16SF_FTYPE_V16SF_V16SF_INT_V16SF_UHI:
   13389         1726 :     case V16SI_FTYPE_V16SI_V16SI_INT_V16SI_UHI:
   13390         1726 :     case V4SF_FTYPE_V4SF_V4SF_INT_V4SF_UQI:
   13391         1726 :     case V2DF_FTYPE_V2DF_V2DF_INT_V2DF_UQI:
   13392         1726 :     case V8DF_FTYPE_V8DF_V4DF_INT_V8DF_UQI:
   13393         1726 :     case V8DI_FTYPE_V8DI_V4DI_INT_V8DI_UQI:
   13394         1726 :     case V4DF_FTYPE_V4DF_V4DF_INT_V4DF_UQI:
   13395         1726 :     case V8SF_FTYPE_V8SF_V8SF_INT_V8SF_UQI:
   13396         1726 :     case V8DF_FTYPE_V8DF_V2DF_INT_V8DF_UQI:
   13397         1726 :     case V8DI_FTYPE_V8DI_V2DI_INT_V8DI_UQI:
   13398         1726 :     case V8SI_FTYPE_V8SI_V8SI_INT_V8SI_UQI:
   13399         1726 :     case V4DI_FTYPE_V4DI_V4DI_INT_V4DI_UQI:
   13400         1726 :     case V4SI_FTYPE_V4SI_V4SI_INT_V4SI_UQI:
   13401         1726 :     case V2DI_FTYPE_V2DI_V2DI_INT_V2DI_UQI:
   13402         1726 :     case V32HI_FTYPE_V64QI_V64QI_INT_V32HI_USI:
   13403         1726 :     case V16HI_FTYPE_V32QI_V32QI_INT_V16HI_UHI:
   13404         1726 :     case V8HI_FTYPE_V16QI_V16QI_INT_V8HI_UQI:
   13405         1726 :     case V16SF_FTYPE_V16SF_V8SF_INT_V16SF_UHI:
   13406         1726 :     case V16SI_FTYPE_V16SI_V8SI_INT_V16SI_UHI:
   13407         1726 :     case V8SF_FTYPE_V8SF_V4SF_INT_V8SF_UQI:
   13408         1726 :     case V8SI_FTYPE_V8SI_V4SI_INT_V8SI_UQI:
   13409         1726 :     case V4DI_FTYPE_V4DI_V2DI_INT_V4DI_UQI:
   13410         1726 :     case V4DF_FTYPE_V4DF_V2DF_INT_V4DF_UQI:
   13411         1726 :       nargs = 5;
   13412         1726 :       mask_pos = 2;
   13413         1726 :       nargs_constant = 1;
   13414         1726 :       break;
   13415          268 :     case V8DI_FTYPE_V8DI_V8DI_V8DI_INT_UQI:
   13416          268 :     case V16SI_FTYPE_V16SI_V16SI_V16SI_INT_UHI:
   13417          268 :     case V2DF_FTYPE_V2DF_V2DF_V2DI_INT_UQI:
   13418          268 :     case V4SF_FTYPE_V4SF_V4SF_V4SI_INT_UQI:
   13419          268 :     case V8SF_FTYPE_V8SF_V8SF_V8SI_INT_UQI:
   13420          268 :     case V8SI_FTYPE_V8SI_V8SI_V8SI_INT_UQI:
   13421          268 :     case V4DF_FTYPE_V4DF_V4DF_V4DI_INT_UQI:
   13422          268 :     case V4DI_FTYPE_V4DI_V4DI_V4DI_INT_UQI:
   13423          268 :     case V4SI_FTYPE_V4SI_V4SI_V4SI_INT_UQI:
   13424          268 :     case V2DI_FTYPE_V2DI_V2DI_V2DI_INT_UQI:
   13425          268 :       nargs = 5;
   13426          268 :       mask_pos = 1;
   13427          268 :       nargs_constant = 1;
   13428          268 :       break;
   13429          732 :     case V64QI_FTYPE_V64QI_V64QI_INT_V64QI_UDI:
   13430          732 :     case V32QI_FTYPE_V32QI_V32QI_INT_V32QI_USI:
   13431          732 :     case V16QI_FTYPE_V16QI_V16QI_INT_V16QI_UHI:
   13432          732 :     case V32HI_FTYPE_V32HI_V32HI_INT_V32HI_INT:
   13433          732 :     case V16SI_FTYPE_V16SI_V16SI_INT_V16SI_INT:
   13434          732 :     case V8DI_FTYPE_V8DI_V8DI_INT_V8DI_INT:
   13435          732 :     case V16HI_FTYPE_V16HI_V16HI_INT_V16HI_INT:
   13436          732 :     case V8SI_FTYPE_V8SI_V8SI_INT_V8SI_INT:
   13437          732 :     case V4DI_FTYPE_V4DI_V4DI_INT_V4DI_INT:
   13438          732 :     case V8HI_FTYPE_V8HI_V8HI_INT_V8HI_INT:
   13439          732 :     case V4SI_FTYPE_V4SI_V4SI_INT_V4SI_INT:
   13440          732 :     case V2DI_FTYPE_V2DI_V2DI_INT_V2DI_INT:
   13441          732 :     case V8BF_FTYPE_V8BF_V8BF_INT_V8BF_UQI:
   13442          732 :     case V16BF_FTYPE_V16BF_V16BF_INT_V16BF_UHI:
   13443          732 :     case V32BF_FTYPE_V32BF_V32BF_INT_V32BF_USI:
   13444          732 :     case V16HF_FTYPE_V16HF_V16HF_INT_V16HF_UHI:
   13445          732 :     case V8HF_FTYPE_V8HF_V8HF_INT_V8HF_UQI:
   13446          732 :       nargs = 5;
   13447          732 :       mask_pos = 1;
   13448          732 :       nargs_constant = 2;
   13449          732 :       break;
   13450              : 
   13451            0 :     default:
   13452            0 :       gcc_unreachable ();
   13453              :     }
   13454              : 
   13455        58097 :   gcc_assert (nargs <= ARRAY_SIZE (xops));
   13456              : 
   13457        65341 :   if (comparison != UNKNOWN)
   13458              :     {
   13459          608 :       gcc_assert (nargs == 2);
   13460          608 :       return ix86_expand_sse_compare (d, exp, target, swap);
   13461              :     }
   13462              : 
   13463        64733 :   if (rmode == VOIDmode || rmode == tmode)
   13464              :     {
   13465        64548 :       if (optimize
   13466        18379 :           || target == 0
   13467        18379 :           || GET_MODE (target) != tmode
   13468        82725 :           || !insn_p->operand[0].predicate (target, tmode))
   13469        46459 :         target = gen_reg_rtx (tmode);
   13470        18089 :       else if (memory_operand (target, tmode))
   13471          578 :         num_memory++;
   13472              :       real_target = target;
   13473              :     }
   13474              :   else
   13475              :     {
   13476          185 :       real_target = gen_reg_rtx (tmode);
   13477          185 :       target = lowpart_subreg (rmode, real_target, tmode);
   13478              :     }
   13479              : 
   13480       272952 :   for (i = 0; i < nargs; i++)
   13481              :     {
   13482       208461 :       tree arg = CALL_EXPR_ARG (exp, i);
   13483       208461 :       rtx op = ix86_expand_unsigned_small_int_cst_argument (arg);
   13484       208461 :       machine_mode mode = insn_p->operand[i + 1].mode;
   13485              :       /* Need to fixup modeless constant before testing predicate.  */
   13486       208461 :       op = fixup_modeless_constant (op, mode);
   13487       208461 :       bool match = insn_p->operand[i + 1].predicate (op, mode);
   13488              : 
   13489       208461 :       if (second_arg_count && i == 1)
   13490              :         {
   13491              :           /* SIMD shift insns take either an 8-bit immediate or
   13492              :              register as count.  But builtin functions take int as
   13493              :              count.  If count doesn't match, we put it in register.
   13494              :              The instructions are using 64-bit count, if op is just
   13495              :              32-bit, zero-extend it, as negative shift counts
   13496              :              are undefined behavior and zero-extension is more
   13497              :              efficient.  */
   13498         2889 :           if (!match)
   13499              :             {
   13500         1750 :               if (SCALAR_INT_MODE_P (GET_MODE (op)))
   13501          489 :                 op = convert_modes (mode, GET_MODE (op), op, 1);
   13502              :               else
   13503         1261 :                 op = lowpart_subreg (mode, op, GET_MODE (op));
   13504         1750 :               if (!insn_p->operand[i + 1].predicate (op, mode))
   13505          190 :                 op = copy_to_reg (op);
   13506              :             }
   13507              :         }
   13508       205572 :       else if ((mask_pos && (nargs - i - mask_pos) == nargs_constant) ||
   13509       157102 :                (!mask_pos && (nargs - i) <= nargs_constant))
   13510              :         {
   13511        16430 :           if (!match)
   13512          233 :             switch (icode)
   13513              :               {
   13514            2 :               case CODE_FOR_avx_vinsertf128v4di:
   13515            2 :               case CODE_FOR_avx_vextractf128v4di:
   13516            2 :                 error ("the last argument must be an 1-bit immediate");
   13517            2 :                 return const0_rtx;
   13518              : 
   13519            8 :               case CODE_FOR_avx512f_cmpv8di3_mask:
   13520            8 :               case CODE_FOR_avx512f_cmpv16si3_mask:
   13521            8 :               case CODE_FOR_avx512f_ucmpv8di3_mask:
   13522            8 :               case CODE_FOR_avx512f_ucmpv16si3_mask:
   13523            8 :               case CODE_FOR_avx512vl_cmpv4di3_mask:
   13524            8 :               case CODE_FOR_avx512vl_cmpv8si3_mask:
   13525            8 :               case CODE_FOR_avx512vl_ucmpv4di3_mask:
   13526            8 :               case CODE_FOR_avx512vl_ucmpv8si3_mask:
   13527            8 :               case CODE_FOR_avx512vl_cmpv2di3_mask:
   13528            8 :               case CODE_FOR_avx512vl_cmpv4si3_mask:
   13529            8 :               case CODE_FOR_avx512vl_ucmpv2di3_mask:
   13530            8 :               case CODE_FOR_avx512vl_ucmpv4si3_mask:
   13531            8 :                 error ("the last argument must be a 3-bit immediate");
   13532            8 :                 return const0_rtx;
   13533              : 
   13534           24 :               case CODE_FOR_sse4_1_roundsd:
   13535           24 :               case CODE_FOR_sse4_1_roundss:
   13536              : 
   13537           24 :               case CODE_FOR_sse4_1_roundpd:
   13538           24 :               case CODE_FOR_sse4_1_roundps:
   13539           24 :               case CODE_FOR_avx_roundpd256:
   13540           24 :               case CODE_FOR_avx_roundps256:
   13541              : 
   13542           24 :               case CODE_FOR_sse4_1_roundpd_vec_pack_sfix:
   13543           24 :               case CODE_FOR_sse4_1_roundps_sfix:
   13544           24 :               case CODE_FOR_avx_roundpd_vec_pack_sfix256:
   13545           24 :               case CODE_FOR_avx_roundps_sfix256:
   13546              : 
   13547           24 :               case CODE_FOR_sse4_1_blendps:
   13548           24 :               case CODE_FOR_avx_blendpd256:
   13549           24 :               case CODE_FOR_avx_vpermilv4df:
   13550           24 :               case CODE_FOR_avx_vpermilv4df_mask:
   13551           24 :               case CODE_FOR_avx512f_getmantv8df_mask:
   13552           24 :               case CODE_FOR_avx512f_getmantv16sf_mask:
   13553           24 :               case CODE_FOR_avx512vl_getmantv16hf_mask:
   13554           24 :               case CODE_FOR_avx512vl_getmantv8sf_mask:
   13555           24 :               case CODE_FOR_avx512vl_getmantv4df_mask:
   13556           24 :               case CODE_FOR_avx512fp16_getmantv8hf_mask:
   13557           24 :               case CODE_FOR_avx512vl_getmantv4sf_mask:
   13558           24 :               case CODE_FOR_avx512vl_getmantv2df_mask:
   13559           24 :               case CODE_FOR_avx512dq_rangepv8df_mask_round:
   13560           24 :               case CODE_FOR_avx512dq_rangepv16sf_mask_round:
   13561           24 :               case CODE_FOR_avx512dq_rangepv4df_mask:
   13562           24 :               case CODE_FOR_avx512dq_rangepv8sf_mask:
   13563           24 :               case CODE_FOR_avx512dq_rangepv2df_mask:
   13564           24 :               case CODE_FOR_avx512dq_rangepv4sf_mask:
   13565           24 :               case CODE_FOR_avx_shufpd256_mask:
   13566           24 :                 error ("the last argument must be a 4-bit immediate");
   13567           24 :                 return const0_rtx;
   13568              : 
   13569           15 :               case CODE_FOR_sha1rnds4:
   13570           15 :               case CODE_FOR_sse4_1_blendpd:
   13571           15 :               case CODE_FOR_avx_vpermilv2df:
   13572           15 :               case CODE_FOR_avx_vpermilv2df_mask:
   13573           15 :               case CODE_FOR_xop_vpermil2v2df3:
   13574           15 :               case CODE_FOR_xop_vpermil2v4sf3:
   13575           15 :               case CODE_FOR_xop_vpermil2v4df3:
   13576           15 :               case CODE_FOR_xop_vpermil2v8sf3:
   13577           15 :               case CODE_FOR_avx512f_vinsertf32x4_mask:
   13578           15 :               case CODE_FOR_avx512f_vinserti32x4_mask:
   13579           15 :               case CODE_FOR_avx512f_vextractf32x4_mask:
   13580           15 :               case CODE_FOR_avx512f_vextracti32x4_mask:
   13581           15 :               case CODE_FOR_sse2_shufpd:
   13582           15 :               case CODE_FOR_sse2_shufpd_mask:
   13583           15 :               case CODE_FOR_avx512dq_shuf_f64x2_mask:
   13584           15 :               case CODE_FOR_avx512dq_shuf_i64x2_mask:
   13585           15 :               case CODE_FOR_avx512vl_shuf_i32x4_mask:
   13586           15 :               case CODE_FOR_avx512vl_shuf_f32x4_mask:
   13587           15 :                 error ("the last argument must be a 2-bit immediate");
   13588           15 :                 return const0_rtx;
   13589              : 
   13590           30 :               case CODE_FOR_avx_vextractf128v4df:
   13591           30 :               case CODE_FOR_avx_vextractf128v8sf:
   13592           30 :               case CODE_FOR_avx_vextractf128v8si:
   13593           30 :               case CODE_FOR_avx_vinsertf128v4df:
   13594           30 :               case CODE_FOR_avx_vinsertf128v8sf:
   13595           30 :               case CODE_FOR_avx_vinsertf128v8si:
   13596           30 :               case CODE_FOR_avx512f_vinsertf64x4_mask:
   13597           30 :               case CODE_FOR_avx512f_vinserti64x4_mask:
   13598           30 :               case CODE_FOR_avx512f_vextractf64x4_mask:
   13599           30 :               case CODE_FOR_avx512f_vextracti64x4_mask:
   13600           30 :               case CODE_FOR_avx512dq_vinsertf32x8_mask:
   13601           30 :               case CODE_FOR_avx512dq_vinserti32x8_mask:
   13602           30 :               case CODE_FOR_avx512vl_vinsertv4df:
   13603           30 :               case CODE_FOR_avx512vl_vinsertv4di:
   13604           30 :               case CODE_FOR_avx512vl_vinsertv8sf:
   13605           30 :               case CODE_FOR_avx512vl_vinsertv8si:
   13606           30 :                 error ("the last argument must be a 1-bit immediate");
   13607           30 :                 return const0_rtx;
   13608              : 
   13609           16 :               case CODE_FOR_avx_vmcmpv2df3:
   13610           16 :               case CODE_FOR_avx_vmcmpv4sf3:
   13611           16 :               case CODE_FOR_avx_cmpv2df3:
   13612           16 :               case CODE_FOR_avx_cmpv4sf3:
   13613           16 :                 if (CONST_INT_P (op) && IN_RANGE (INTVAL (op), 8, 31))
   13614              :                   {
   13615            4 :                     error ("'%s' needs isa option %s", d->name, "-mavx");
   13616            4 :                     return const0_rtx;
   13617              :                   }
   13618              :                 /* FALLTHRU */
   13619           18 :               case CODE_FOR_avx_cmpv4df3:
   13620           18 :               case CODE_FOR_avx_cmpv8sf3:
   13621           18 :               case CODE_FOR_avx512f_cmpv8df3_mask:
   13622           18 :               case CODE_FOR_avx512f_cmpv16sf3_mask:
   13623           18 :               case CODE_FOR_avx512f_vmcmpv2df3_mask:
   13624           18 :               case CODE_FOR_avx512f_vmcmpv4sf3_mask:
   13625           18 :               case CODE_FOR_avx512bw_cmpv32hf3_mask:
   13626           18 :               case CODE_FOR_avx512vl_cmpv16hf3_mask:
   13627           18 :               case CODE_FOR_avx512fp16_cmpv8hf3_mask:
   13628           18 :                 error ("the last argument must be a 5-bit immediate");
   13629           18 :                 return const0_rtx;
   13630              : 
   13631          132 :               default:
   13632          132 :                 switch (nargs_constant)
   13633              :                   {
   13634            8 :                   case 2:
   13635            8 :                     if ((mask_pos && (nargs - i - mask_pos) == nargs_constant) ||
   13636            8 :                         (!mask_pos && (nargs - i) == nargs_constant))
   13637              :                       {
   13638            4 :                         error ("the next to last argument must be an 8-bit immediate");
   13639            4 :                         break;
   13640              :                       }
   13641              :                     /* FALLTHRU */
   13642          128 :                   case 1:
   13643          128 :                     error ("the last argument must be an 8-bit immediate");
   13644          128 :                     break;
   13645            0 :                   default:
   13646            0 :                     gcc_unreachable ();
   13647              :                   }
   13648          132 :                 return const0_rtx;
   13649              :               }
   13650        16197 :           if ((icode == CODE_FOR_vunpackbv16qi_mask
   13651        16197 :                || icode == CODE_FOR_vunpackbv32qi_mask
   13652              :                || icode == CODE_FOR_vunpackbv64qi_mask)
   13653          108 :               && CONST_INT_P (op))
   13654              :             {
   13655          108 :               char val = INTVAL (op);
   13656          108 :               if ((val & 0xc0)
   13657          108 :                   || (!(val & 0x18))
   13658          107 :                   || ((val & 0x02) && ((val & 0x1c) != 0x08))
   13659          102 :                   || ((val & 0x01) && (((val & 0x1c) >> 2) > 0x4)))
   13660              :                 {
   13661            9 :                   error ("the last argument must not use reserved value "
   13662              :                          "immediate");
   13663            9 :                   return const0_rtx;
   13664              :                 }
   13665              :             }
   13666              :         }
   13667              :       else
   13668              :         {
   13669       189142 :           if (VECTOR_MODE_P (mode))
   13670       136656 :             op = safe_vector_operand (op, mode);
   13671              : 
   13672              :           /* If we aren't optimizing, only allow one memory operand to
   13673              :              be generated.  */
   13674       189142 :           if (memory_operand (op, mode))
   13675              :             {
   13676        30648 :               num_memory++;
   13677        30648 :               if (!optimize && num_memory > 1)
   13678        13873 :                 op = copy_to_mode_reg (mode, op);
   13679              :             }
   13680              : 
   13681       189142 :           if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
   13682              :             {
   13683       187013 :               if (!match)
   13684        43549 :                 op = copy_to_mode_reg (mode, op);
   13685              :             }
   13686              :           else
   13687              :             {
   13688         2129 :               op = copy_to_reg (op);
   13689         2129 :               op = lowpart_subreg (mode, op, GET_MODE (op));
   13690              :             }
   13691              :         }
   13692              : 
   13693       208219 :       xops[i] = op;
   13694              :     }
   13695              : 
   13696        64491 :   switch (nargs)
   13697              :     {
   13698         6636 :     case 1:
   13699         6636 :       pat = GEN_FCN (icode) (real_target, xops[0]);
   13700         6636 :       break;
   13701         5526 :     case 2:
   13702         5526 :       pat = GEN_FCN (icode) (real_target, xops[0], xops[1]);
   13703         5526 :       break;
   13704        21905 :     case 3:
   13705        21905 :       pat = GEN_FCN (icode) (real_target, xops[0], xops[1], xops[2]);
   13706        21905 :       break;
   13707        27684 :     case 4:
   13708        27684 :       pat = GEN_FCN (icode) (real_target, xops[0], xops[1],
   13709        27684 :                              xops[2], xops[3]);
   13710        27684 :       break;
   13711         2740 :     case 5:
   13712         2740 :       pat = GEN_FCN (icode) (real_target, xops[0], xops[1],
   13713         2740 :                              xops[2], xops[3], xops[4]);
   13714         2740 :       break;
   13715              :     case 6:
   13716              :       pat = GEN_FCN (icode) (real_target, xops[0], xops[1],
   13717              :                              xops[2], xops[3], xops[4], xops[5]);
   13718              :       break;
   13719              :     default:
   13720              :       gcc_unreachable ();
   13721              :     }
   13722              : 
   13723        64491 :   if (! pat)
   13724              :     return 0;
   13725              : 
   13726        64491 :   emit_insn (pat);
   13727        64491 :   return target;
   13728              : }
   13729              : 
   13730              : /* Transform pattern of following layout:
   13731              :      (set A
   13732              :        (unspec [B C] UNSPEC_EMBEDDED_ROUNDING))
   13733              :      )
   13734              :    into:
   13735              :      (set (A B)) */
   13736              : 
   13737              : static rtx
   13738         4958 : ix86_erase_embedded_rounding (rtx pat)
   13739              : {
   13740         4958 :   if (NONJUMP_INSN_P (pat))
   13741          680 :     pat = PATTERN (pat);
   13742              : 
   13743         4958 :   gcc_assert (GET_CODE (pat) == SET);
   13744         4958 :   rtx src = SET_SRC (pat);
   13745         4958 :   gcc_assert (XVECLEN (src, 0) == 2);
   13746         4958 :   rtx p0 = XVECEXP (src, 0, 0);
   13747         4958 :   gcc_assert (GET_CODE (src) == UNSPEC
   13748              :               && XINT (src, 1) == UNSPEC_EMBEDDED_ROUNDING);
   13749         4958 :   rtx res = gen_rtx_SET (SET_DEST (pat), p0);
   13750         4958 :   return res;
   13751              : }
   13752              : 
   13753              : /* Subroutine of ix86_expand_round_builtin to take care of comi insns
   13754              :    with rounding.  */
   13755              : static rtx
   13756          103 : ix86_expand_sse_comi_round (const struct builtin_description *d,
   13757              :                             tree exp, rtx target, bool comx_ok)
   13758              : {
   13759          103 :   rtx pat, set_dst;
   13760          103 :   tree arg0 = CALL_EXPR_ARG (exp, 0);
   13761          103 :   tree arg1 = CALL_EXPR_ARG (exp, 1);
   13762          103 :   tree arg2 = CALL_EXPR_ARG (exp, 2);
   13763          103 :   tree arg3 = CALL_EXPR_ARG (exp, 3);
   13764          103 :   rtx op0 = expand_normal (arg0);
   13765          103 :   rtx op1 = expand_normal (arg1);
   13766          103 :   rtx op2 = expand_normal (arg2);
   13767          103 :   rtx op3 = expand_normal (arg3);
   13768          103 :   enum insn_code icode = d->icode;
   13769          103 :   const struct insn_data_d *insn_p = &insn_data[icode];
   13770          103 :   machine_mode mode0 = insn_p->operand[0].mode;
   13771          103 :   machine_mode mode1 = insn_p->operand[1].mode;
   13772              : 
   13773              :   /* See avxintrin.h for values.  */
   13774          103 :   static const enum rtx_code comparisons[32] =
   13775              :     {
   13776              :       EQ, LT, LE, UNORDERED, NE, UNGE, UNGT, ORDERED,
   13777              :       UNEQ, UNLT, UNLE, UNORDERED, LTGT, GE, GT, ORDERED,
   13778              :       EQ, LT, LE, UNORDERED, NE, UNGE, UNGT, ORDERED,
   13779              :       UNEQ, UNLT, UNLE, UNORDERED, LTGT, GE, GT, ORDERED
   13780              :     };
   13781          103 :   static const bool ordereds[32] =
   13782              :     {
   13783              :       true,  true,  true,  false, false, false, false, true,
   13784              :       false, false, false, true,  true,  true,  true,  false,
   13785              :       true,  true,  true,  false, false, false, false, true,
   13786              :       false, false, false, true,  true,  true,  true,  false
   13787              :     };
   13788          103 :   static const bool non_signalings[32] =
   13789              :     {
   13790              :       true,  false, false, true,  true,  false, false, true,
   13791              :       true,  false, false, true,  true,  false, false, true,
   13792              :       false, true,  true,  false, false, true,  true,  false,
   13793              :       false, true,  true,  false, false, true,  true,  false
   13794              :     };
   13795              : 
   13796          103 :   if (!CONST_INT_P (op2))
   13797              :     {
   13798            0 :       error ("the third argument must be comparison constant");
   13799            0 :       return const0_rtx;
   13800              :     }
   13801          103 :   if (INTVAL (op2) < 0 || INTVAL (op2) >= 32)
   13802              :     {
   13803            0 :       error ("incorrect comparison mode");
   13804            0 :       return const0_rtx;
   13805              :     }
   13806              : 
   13807          103 :   if (!insn_p->operand[2].predicate (op3, SImode))
   13808              :     {
   13809            4 :       error ("incorrect rounding operand");
   13810            4 :       return const0_rtx;
   13811              :     }
   13812              : 
   13813           99 :   if (VECTOR_MODE_P (mode0))
   13814           99 :     op0 = safe_vector_operand (op0, mode0);
   13815           99 :   if (VECTOR_MODE_P (mode1))
   13816           99 :     op1 = safe_vector_operand (op1, mode1);
   13817              : 
   13818           99 :   enum rtx_code comparison = comparisons[INTVAL (op2)];
   13819           99 :   enum rtx_code orig_comp = comparison;
   13820           99 :   bool ordered = ordereds[INTVAL (op2)];
   13821           99 :   bool non_signaling = non_signalings[INTVAL (op2)];
   13822           99 :   rtx const_val = const0_rtx;
   13823              : 
   13824           99 :   bool check_unordered = false;
   13825           99 :   machine_mode mode = CCFPmode;
   13826           99 :   switch (comparison)
   13827              :     {
   13828            8 :     case ORDERED:
   13829            8 :       if (!ordered)
   13830              :         {
   13831            4 :           if (TARGET_AVX10_2 && comx_ok)
   13832              :             {
   13833              :               /* Unlike VCOMI{SH,SS,SD}, VCOMX{SH,SS,SD} will set SF
   13834              :                  differently. So directly return true here.  */
   13835            0 :               target = gen_reg_rtx (SImode);
   13836            0 :               emit_move_insn (target, const1_rtx);
   13837            0 :               return target;
   13838              :             }
   13839              :           else
   13840              :             {
   13841              :               /* NB: Use CCSmode/NE for _CMP_TRUE_UQ/_CMP_TRUE_US.  */
   13842              :               if (!non_signaling)
   13843           99 :                 ordered = true;
   13844           99 :               mode = CCSmode;
   13845              :             }
   13846              :         }
   13847              :       else
   13848              :         {
   13849              :           /* NB: Use CCPmode/NE for _CMP_ORD_Q/_CMP_ORD_S.  */
   13850              :           if (non_signaling)
   13851           99 :             ordered = false;
   13852              :           mode = CCPmode;
   13853              :         }
   13854              :       comparison = NE;
   13855              :       break;
   13856            8 :     case UNORDERED:
   13857            8 :       if (ordered)
   13858              :         {
   13859            4 :           if (TARGET_AVX10_2 && comx_ok)
   13860              :             {
   13861              :               /* Unlike VCOMI{SH,SS,SD}, VCOMX{SH,SS,SD} will set SF
   13862              :                  differently. So directly return false here.  */
   13863            0 :               target = gen_reg_rtx (SImode);
   13864            0 :               emit_move_insn (target, const0_rtx);
   13865            0 :               return target;
   13866              :             }
   13867              :           else
   13868              :             {
   13869              :               /* NB: Use CCSmode/EQ for _CMP_FALSE_OQ/_CMP_FALSE_OS.  */
   13870              :               if (non_signaling)
   13871              :                 ordered = false;
   13872              :               mode = CCSmode;
   13873              :             }
   13874              :         }
   13875              :       else
   13876              :         {
   13877              :           /* NB: Use CCPmode/NE for _CMP_UNORD_Q/_CMP_UNORD_S.  */
   13878              :           if (!non_signaling)
   13879           99 :             ordered = true;
   13880           99 :           mode = CCPmode;
   13881              :         }
   13882              :       comparison = EQ;
   13883              :       break;
   13884              : 
   13885           40 :     case LE:    /* -> GE  */
   13886           40 :     case LT:    /* -> GT  */
   13887           40 :     case UNGE:  /* -> UNLE  */
   13888           40 :     case UNGT:  /* -> UNLT  */
   13889           40 :       std::swap (op0, op1);
   13890           40 :       comparison = swap_condition (comparison);
   13891              :       /* FALLTHRU */
   13892              :     case GT:
   13893              :     case GE:
   13894              :     case UNEQ:
   13895              :     case UNLT:
   13896              :     case UNLE:
   13897              :     case LTGT:
   13898              :       /* These are supported by CCFPmode.  NB: Use ordered/signaling
   13899              :          COMI or unordered/non-signaling UCOMI.  Both set ZF, PF, CF
   13900              :          with NAN operands.  */
   13901              :       if (ordered == non_signaling)
   13902              :         ordered = !ordered;
   13903              :       break;
   13904              :       /* NB: COMI/UCOMI will set ZF with NAN operands.  Use CCZmode for
   13905              :          _CMP_EQ_OQ/_CMP_EQ_OS.
   13906              :          Under TARGET_AVX10_2, VCOMX/VUCOMX are always generated instead
   13907              :          of COMI/UCOMI, VCOMX/VUCOMX will not set ZF with NAN.  */
   13908            8 :     case EQ:
   13909            8 :       if (!TARGET_AVX10_2 || !comx_ok)
   13910            5 :         check_unordered = true;
   13911              :       mode = CCZmode;
   13912              :       break;
   13913            7 :     case NE:
   13914              :       /* NB: COMI/UCOMI will set ZF with NAN operands.  Use CCZmode for
   13915              :          _CMP_NEQ_UQ/_CMP_NEQ_US.
   13916              :          Under TARGET_AVX10_2, VCOMX/VUCOMX are always generated instead
   13917              :          of COMI/UCOMI, VCOMX/VUCOMX will not set ZF with NAN.  */
   13918            7 :       gcc_assert (!ordered);
   13919            7 :       if (!TARGET_AVX10_2 || !comx_ok)
   13920            4 :         check_unordered = true;
   13921            7 :       mode = CCZmode;
   13922            7 :       const_val = const1_rtx;
   13923            7 :       break;
   13924              :     default:
   13925              :       gcc_unreachable ();
   13926              :     }
   13927              : 
   13928           99 :   target = gen_reg_rtx (SImode);
   13929           99 :   emit_move_insn (target, const_val);
   13930           99 :   target = gen_rtx_SUBREG (QImode, target, 0);
   13931              : 
   13932           93 :   if ((optimize && !register_operand (op0, mode0))
   13933          192 :       || !insn_p->operand[0].predicate (op0, mode0))
   13934            6 :     op0 = copy_to_mode_reg (mode0, op0);
   13935           93 :   if ((optimize && !register_operand (op1, mode1))
   13936          192 :       || !insn_p->operand[1].predicate (op1, mode1))
   13937            6 :     op1 = copy_to_mode_reg (mode1, op1);
   13938              : 
   13939              :     /* Generate comx instead of comi when EQ/NE to avoid NAN checks.
   13940              :        Use orig_comp to exclude ORDERED/UNORDERED cases.  */
   13941           99 :   if ((orig_comp == EQ || orig_comp == NE)
   13942           15 :       && TARGET_AVX10_2 && comx_ok)
   13943              :     {
   13944            6 :       switch (icode)
   13945              :         {
   13946              :         case CODE_FOR_avx512fp16_comi_round:
   13947           99 :           icode = CODE_FOR_avx10_2_comxhf_round;
   13948              :           break;
   13949            4 :         case CODE_FOR_sse_comi_round:
   13950            4 :           icode = CODE_FOR_avx10_2_comxsf_round;
   13951            4 :           break;
   13952            2 :         case CODE_FOR_sse2_comi_round:
   13953            2 :           icode = CODE_FOR_avx10_2_comxdf_round;
   13954            2 :           break;
   13955              : 
   13956              :         default:
   13957              :           break;
   13958              :         }
   13959              :     }
   13960              : 
   13961              :   /* Generate comi instead of comx when UNEQ/LTGT to avoid NAN checks.  */
   13962           99 :   if ((comparison == UNEQ || comparison == LTGT)
   13963            8 :        && TARGET_AVX10_2 && comx_ok)
   13964              :     {
   13965            0 :       switch (icode)
   13966              :         {
   13967              :         case CODE_FOR_avx10_2_comxhf_round:
   13968           99 :           icode = CODE_FOR_avx512fp16_comi_round;
   13969              :           break;
   13970            0 :         case CODE_FOR_avx10_2_comxsf_round:
   13971            0 :           icode = CODE_FOR_sse_comi_round;
   13972            0 :           break;
   13973            0 :         case CODE_FOR_avx10_2_comxdf_round:
   13974            0 :           icode = CODE_FOR_sse2_comi_round;
   13975            0 :           break;
   13976              : 
   13977              :         default:
   13978              :           break;
   13979              :         }
   13980              :     }
   13981              : 
   13982              :   /*
   13983              :      1. COMI/VCOMX: ordered and signaling.
   13984              :      2. UCOMI/VUCOMX: unordered and non-signaling.
   13985              :    */
   13986           99 :   if (non_signaling)
   13987           38 :     switch (icode)
   13988              :       {
   13989              :       case CODE_FOR_sse_comi_round:
   13990              :         icode = CODE_FOR_sse_ucomi_round;
   13991              :         break;
   13992           17 :       case CODE_FOR_sse2_comi_round:
   13993           17 :         icode = CODE_FOR_sse2_ucomi_round;
   13994           17 :         break;
   13995            0 :       case CODE_FOR_avx512fp16_comi_round:
   13996            0 :         icode = CODE_FOR_avx512fp16_ucomi_round;
   13997            0 :         break;
   13998            3 :       case CODE_FOR_avx10_2_comxsf_round:
   13999            3 :         icode = CODE_FOR_avx10_2_ucomxsf_round;
   14000            3 :         break;
   14001            0 :       case CODE_FOR_avx10_2_comxhf_round:
   14002            0 :         icode = CODE_FOR_avx10_2_ucomxhf_round;
   14003            0 :         break;
   14004            1 :       case CODE_FOR_avx10_2_comxdf_round:
   14005            1 :         icode = CODE_FOR_avx10_2_ucomxdf_round;
   14006            1 :         break;
   14007            0 :       default:
   14008            0 :         gcc_unreachable ();
   14009              :       }
   14010              : 
   14011           99 :   pat = GEN_FCN (icode) (op0, op1, op3);
   14012           99 :   if (! pat)
   14013              :     return 0;
   14014              : 
   14015              :   /* Rounding operand can be either NO_ROUND or ROUND_SAE at this point.  */
   14016           99 :   if (INTVAL (op3) == NO_ROUND)
   14017              :     {
   14018            1 :       pat = ix86_erase_embedded_rounding (pat);
   14019            1 :       if (! pat)
   14020              :         return 0;
   14021              : 
   14022            1 :       set_dst = SET_DEST (pat);
   14023              :     }
   14024              :   else
   14025              :     {
   14026           98 :       gcc_assert (GET_CODE (pat) == SET);
   14027           98 :       set_dst = SET_DEST (pat);
   14028              :     }
   14029              : 
   14030           99 :   emit_insn (pat);
   14031              : 
   14032           99 :   return ix86_ssecom_setcc (comparison, check_unordered, mode,
   14033           99 :                             set_dst, target);
   14034              : }
   14035              : 
   14036              : static rtx
   14037        15660 : ix86_expand_round_builtin (const struct builtin_description *d,
   14038              :                            tree exp, rtx target)
   14039              : {
   14040        15660 :   rtx pat;
   14041        15660 :   unsigned int i, nargs;
   14042        15660 :   rtx xops[6];
   14043        15660 :   enum insn_code icode = d->icode;
   14044        15660 :   const struct insn_data_d *insn_p = &insn_data[icode];
   14045        15660 :   machine_mode tmode = insn_p->operand[0].mode;
   14046        15660 :   unsigned int nargs_constant = 0;
   14047        15660 :   unsigned int redundant_embed_rnd = 0;
   14048              : 
   14049        15660 :   switch ((enum ix86_builtin_func_type) d->flag)
   14050              :     {
   14051              :     case UINT64_FTYPE_V2DF_INT:
   14052              :     case UINT64_FTYPE_V4SF_INT:
   14053              :     case UINT64_FTYPE_V8HF_INT:
   14054              :     case UINT_FTYPE_V2DF_INT:
   14055              :     case UINT_FTYPE_V4SF_INT:
   14056              :     case UINT_FTYPE_V8HF_INT:
   14057              :     case INT64_FTYPE_V2DF_INT:
   14058              :     case INT64_FTYPE_V4SF_INT:
   14059              :     case INT64_FTYPE_V8HF_INT:
   14060              :     case INT_FTYPE_V2DF_INT:
   14061              :     case INT_FTYPE_V4SF_INT:
   14062              :     case INT_FTYPE_V8HF_INT:
   14063              :       nargs = 2;
   14064              :       break;
   14065          637 :     case V32HF_FTYPE_V32HF_V32HF_INT:
   14066          637 :     case V8HF_FTYPE_V8HF_V8HF_INT:
   14067          637 :     case V8HF_FTYPE_V8HF_INT_INT:
   14068          637 :     case V8HF_FTYPE_V8HF_UINT_INT:
   14069          637 :     case V8HF_FTYPE_V8HF_INT64_INT:
   14070          637 :     case V8HF_FTYPE_V8HF_UINT64_INT:
   14071          637 :     case V4SF_FTYPE_V4SF_UINT_INT:
   14072          637 :     case V4SF_FTYPE_V4SF_UINT64_INT:
   14073          637 :     case V2DF_FTYPE_V2DF_UINT64_INT:
   14074          637 :     case V4SF_FTYPE_V4SF_INT_INT:
   14075          637 :     case V4SF_FTYPE_V4SF_INT64_INT:
   14076          637 :     case V2DF_FTYPE_V2DF_INT64_INT:
   14077          637 :     case V4SF_FTYPE_V4SF_V4SF_INT:
   14078          637 :     case V2DF_FTYPE_V2DF_V2DF_INT:
   14079          637 :     case V4SF_FTYPE_V4SF_V2DF_INT:
   14080          637 :     case V2DF_FTYPE_V2DF_V4SF_INT:
   14081          637 :       nargs = 3;
   14082          637 :       break;
   14083         4554 :     case V8SF_FTYPE_V8DF_V8SF_QI_INT:
   14084         4554 :     case V8DF_FTYPE_V8DF_V8DF_QI_INT:
   14085         4554 :     case V32HI_FTYPE_V32HF_V32HI_USI_INT:
   14086         4554 :     case V32HI_FTYPE_V32BF_V32HI_USI_INT:
   14087         4554 :     case V8SI_FTYPE_V8DF_V8SI_QI_INT:
   14088         4554 :     case V8DI_FTYPE_V8HF_V8DI_UQI_INT:
   14089         4554 :     case V8DI_FTYPE_V8DF_V8DI_QI_INT:
   14090         4554 :     case V8SF_FTYPE_V8DI_V8SF_QI_INT:
   14091         4554 :     case V8DF_FTYPE_V8DI_V8DF_QI_INT:
   14092         4554 :     case V8DF_FTYPE_V8HF_V8DF_UQI_INT:
   14093         4554 :     case V16SF_FTYPE_V16HF_V16SF_UHI_INT:
   14094         4554 :     case V32HF_FTYPE_V32HI_V32HF_USI_INT:
   14095         4554 :     case V32HF_FTYPE_V32HF_V32HF_USI_INT:
   14096         4554 :     case V32HF_FTYPE_V32HF_V32HF_V32HF_INT:
   14097         4554 :     case V16SF_FTYPE_V16SF_V16SF_HI_INT:
   14098         4554 :     case V8DI_FTYPE_V8SF_V8DI_QI_INT:
   14099         4554 :     case V16SF_FTYPE_V16SI_V16SF_HI_INT:
   14100         4554 :     case V16SI_FTYPE_V16SF_V16SI_HI_INT:
   14101         4554 :     case V16SI_FTYPE_V16SF_V16SI_UHI_INT:
   14102         4554 :     case V16SI_FTYPE_V16HF_V16SI_UHI_INT:
   14103         4554 :     case V16HF_FTYPE_V16SI_V16HF_UHI_INT:
   14104         4554 :     case V8DF_FTYPE_V8SF_V8DF_QI_INT:
   14105         4554 :     case V16SF_FTYPE_V16HI_V16SF_HI_INT:
   14106         4554 :     case V2DF_FTYPE_V2DF_V2DF_V2DF_INT:
   14107         4554 :     case V4SF_FTYPE_V4SF_V4SF_V4SF_INT:
   14108         4554 :     case V8HF_FTYPE_V8DI_V8HF_UQI_INT:
   14109         4554 :     case V8HF_FTYPE_V8DF_V8HF_UQI_INT:
   14110         4554 :     case V16HF_FTYPE_V16SF_V16HF_UHI_INT:
   14111         4554 :     case V16HI_FTYPE_V16BF_V16HI_UHI_INT:
   14112         4554 :     case V8HF_FTYPE_V8HF_V8HF_V8HF_INT:
   14113         4554 :       nargs = 4;
   14114         4554 :       break;
   14115          166 :     case V4SF_FTYPE_V4SF_V4SF_INT_INT:
   14116          166 :     case V2DF_FTYPE_V2DF_V2DF_INT_INT:
   14117          166 :       nargs_constant = 2;
   14118          166 :       nargs = 4;
   14119          166 :       break;
   14120          103 :     case INT_FTYPE_V4SF_V4SF_INT_INT:
   14121          103 :     case INT_FTYPE_V2DF_V2DF_INT_INT:
   14122          103 :       return ix86_expand_sse_comi_round (d, exp, target, true);
   14123         6226 :     case V8DF_FTYPE_V8DF_V8DF_V8DF_UQI_INT:
   14124         6226 :     case V2DF_FTYPE_V2DF_V2DF_V2DF_UQI_INT:
   14125         6226 :     case V4SF_FTYPE_V4SF_V4SF_V4SF_UQI_INT:
   14126         6226 :     case V4SF_FTYPE_V8HF_V4SF_V4SF_UQI_INT:
   14127         6226 :     case V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT:
   14128         6226 :     case V32HF_FTYPE_V32HF_V32HF_V32HF_UHI_INT:
   14129         6226 :     case V32HF_FTYPE_V32HF_V32HF_V32HF_USI_INT:
   14130         6226 :     case V2DF_FTYPE_V8HF_V2DF_V2DF_UQI_INT:
   14131         6226 :     case V2DF_FTYPE_V2DF_V2DF_V2DF_QI_INT:
   14132         6226 :     case V2DF_FTYPE_V2DF_V4SF_V2DF_QI_INT:
   14133         6226 :     case V2DF_FTYPE_V2DF_V4SF_V2DF_UQI_INT:
   14134         6226 :     case V4SF_FTYPE_V4SF_V4SF_V4SF_QI_INT:
   14135         6226 :     case V4SF_FTYPE_V4SF_V2DF_V4SF_QI_INT:
   14136         6226 :     case V4SF_FTYPE_V4SF_V2DF_V4SF_UQI_INT:
   14137         6226 :     case V8HF_FTYPE_V8HF_V8HF_V8HF_UQI_INT:
   14138         6226 :     case V8HF_FTYPE_V2DF_V8HF_V8HF_UQI_INT:
   14139         6226 :     case V8HF_FTYPE_V4SF_V8HF_V8HF_UQI_INT:
   14140         6226 :     case V32HF_FTYPE_V16SF_V16SF_V32HF_USI_INT:
   14141         6226 :       nargs = 5;
   14142         6226 :       break;
   14143          635 :     case V32HF_FTYPE_V32HF_INT_V32HF_USI_INT:
   14144          635 :     case V16SF_FTYPE_V16SF_INT_V16SF_HI_INT:
   14145          635 :     case V8DF_FTYPE_V8DF_INT_V8DF_QI_INT:
   14146          635 :     case V8DF_FTYPE_V8DF_INT_V8DF_UQI_INT:
   14147          635 :     case V16SF_FTYPE_V16SF_INT_V16SF_UHI_INT:
   14148          635 :       nargs_constant = 4;
   14149          635 :       nargs = 5;
   14150          635 :       break;
   14151         1173 :     case UQI_FTYPE_V8DF_V8DF_INT_UQI_INT:
   14152         1173 :     case UQI_FTYPE_V2DF_V2DF_INT_UQI_INT:
   14153         1173 :     case UHI_FTYPE_V16SF_V16SF_INT_UHI_INT:
   14154         1173 :     case UQI_FTYPE_V4SF_V4SF_INT_UQI_INT:
   14155         1173 :     case USI_FTYPE_V32HF_V32HF_INT_USI_INT:
   14156         1173 :     case UQI_FTYPE_V8HF_V8HF_INT_UQI_INT:
   14157         1173 :       nargs_constant = 3;
   14158         1173 :       nargs = 5;
   14159         1173 :       break;
   14160         1071 :     case V16SF_FTYPE_V16SF_V16SF_INT_V16SF_HI_INT:
   14161         1071 :     case V8DF_FTYPE_V8DF_V8DF_INT_V8DF_QI_INT:
   14162         1071 :     case V4SF_FTYPE_V4SF_V4SF_INT_V4SF_QI_INT:
   14163         1071 :     case V2DF_FTYPE_V2DF_V2DF_INT_V2DF_QI_INT:
   14164         1071 :     case V2DF_FTYPE_V2DF_V2DF_INT_V2DF_UQI_INT:
   14165         1071 :     case V4SF_FTYPE_V4SF_V4SF_INT_V4SF_UQI_INT:
   14166         1071 :     case V8HF_FTYPE_V8HF_V8HF_INT_V8HF_UQI_INT:
   14167         1071 :     case V8DF_FTYPE_V8DF_V8DF_INT_V8DF_UQI_INT:
   14168         1071 :     case V32HF_FTYPE_V32HF_V32HF_INT_V32HF_USI_INT:
   14169         1071 :     case V16SF_FTYPE_V16SF_V16SF_INT_V16SF_UHI_INT:
   14170         1071 :       nargs = 6;
   14171         1071 :       nargs_constant = 4;
   14172         1071 :       break;
   14173          252 :     case V8DF_FTYPE_V8DF_V8DF_V8DI_INT_QI_INT:
   14174          252 :     case V16SF_FTYPE_V16SF_V16SF_V16SI_INT_HI_INT:
   14175          252 :     case V2DF_FTYPE_V2DF_V2DF_V2DI_INT_QI_INT:
   14176          252 :     case V4SF_FTYPE_V4SF_V4SF_V4SI_INT_QI_INT:
   14177          252 :       nargs = 6;
   14178          252 :       nargs_constant = 3;
   14179          252 :       break;
   14180            0 :     default:
   14181            0 :       gcc_unreachable ();
   14182              :     }
   14183        14714 :   gcc_assert (nargs <= ARRAY_SIZE (xops));
   14184              : 
   14185        15557 :   if (optimize
   14186         4293 :       || target == 0
   14187         4293 :       || GET_MODE (target) != tmode
   14188        19850 :       || !insn_p->operand[0].predicate (target, tmode))
   14189        11264 :     target = gen_reg_rtx (tmode);
   14190              : 
   14191        85491 :   for (i = 0; i < nargs; i++)
   14192              :     {
   14193        70489 :       tree arg = CALL_EXPR_ARG (exp, i);
   14194        70489 :       rtx op = ix86_expand_unsigned_small_int_cst_argument (arg);
   14195        70489 :       machine_mode mode = insn_p->operand[i + 1].mode;
   14196        70489 :       bool match = insn_p->operand[i + 1].predicate (op, mode);
   14197              : 
   14198        70489 :       if (i == nargs - nargs_constant)
   14199              :         {
   14200         3297 :           if (!match)
   14201              :             {
   14202           40 :               switch (icode)
   14203              :                 {
   14204           12 :                 case CODE_FOR_avx512f_getmantv8df_mask_round:
   14205           12 :                 case CODE_FOR_avx512f_getmantv16sf_mask_round:
   14206           12 :                 case CODE_FOR_avx512bw_getmantv32hf_mask_round:
   14207           12 :                 case CODE_FOR_avx512f_vgetmantv2df_round:
   14208           12 :                 case CODE_FOR_avx512f_vgetmantv2df_mask_round:
   14209           12 :                 case CODE_FOR_avx512f_vgetmantv4sf_round:
   14210           12 :                 case CODE_FOR_avx512f_vgetmantv4sf_mask_round:
   14211           12 :                 case CODE_FOR_avx512f_vgetmantv8hf_mask_round:
   14212           12 :                   error ("the immediate argument must be a 4-bit immediate");
   14213           12 :                   return const0_rtx;
   14214            8 :                 case CODE_FOR_avx512f_cmpv8df3_mask_round:
   14215            8 :                 case CODE_FOR_avx512f_cmpv16sf3_mask_round:
   14216            8 :                 case CODE_FOR_avx512f_vmcmpv2df3_mask_round:
   14217            8 :                 case CODE_FOR_avx512f_vmcmpv4sf3_mask_round:
   14218            8 :                 case CODE_FOR_avx512f_vmcmpv8hf3_mask_round:
   14219            8 :                 case CODE_FOR_avx512bw_cmpv32hf3_mask_round:
   14220            8 :                   error ("the immediate argument must be a 5-bit immediate");
   14221            8 :                   return const0_rtx;
   14222           20 :                 default:
   14223           20 :                   error ("the immediate argument must be an 8-bit immediate");
   14224           20 :                   return const0_rtx;
   14225              :                 }
   14226              :             }
   14227              :         }
   14228        67192 :       else if (i == nargs-1)
   14229              :         {
   14230        15517 :           if (!insn_p->operand[nargs].predicate (op, SImode))
   14231              :             {
   14232          515 :               error ("incorrect rounding operand");
   14233          515 :               return const0_rtx;
   14234              :             }
   14235              : 
   14236              :           /* If there is no rounding use normal version of the pattern.  */
   14237        15002 :           if (INTVAL (op) == NO_ROUND)
   14238              :             {
   14239              :               /* Skip erasing embedded rounding for below expanders who
   14240              :                  generates multiple insns.  In ix86_erase_embedded_rounding
   14241              :                  the pattern will be transformed to a single set, and emit_insn
   14242              :                  appends the set instead of insert it to chain.  So the insns
   14243              :                  emitted inside define_expander would be ignored.  */
   14244         4989 :               switch (icode)
   14245              :                 {
   14246              :                 case CODE_FOR_avx512bw_fmaddc_v32hf_mask1_round:
   14247              :                 case CODE_FOR_avx512bw_fcmaddc_v32hf_mask1_round:
   14248              :                 case CODE_FOR_avx512fp16_fmaddcsh_v8hf_mask1_round:
   14249              :                 case CODE_FOR_avx512fp16_fcmaddcsh_v8hf_mask1_round:
   14250              :                 case CODE_FOR_avx512fp16_fmaddcsh_v8hf_mask3_round:
   14251              :                 case CODE_FOR_avx512fp16_fcmaddcsh_v8hf_mask3_round:
   14252              :                   redundant_embed_rnd = 0;
   14253              :                   break;
   14254         4957 :                 default:
   14255         4957 :                   redundant_embed_rnd = 1;
   14256         4957 :                   break;
   14257              :                 }
   14258              :             }
   14259              :         }
   14260              :       else
   14261              :         {
   14262        51675 :           if (VECTOR_MODE_P (mode))
   14263        37773 :             op = safe_vector_operand (op, mode);
   14264              : 
   14265        51675 :           op = fixup_modeless_constant (op, mode);
   14266              : 
   14267        51675 :           if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
   14268              :             {
   14269        51675 :               if (optimize || !match)
   14270        45347 :                 op = copy_to_mode_reg (mode, op);
   14271              :             }
   14272              :           else
   14273              :             {
   14274            0 :               op = copy_to_reg (op);
   14275            0 :               op = lowpart_subreg (mode, op, GET_MODE (op));
   14276              :             }
   14277              :         }
   14278              : 
   14279        69934 :       xops[i] = op;
   14280              :     }
   14281              : 
   14282        15002 :   switch (nargs)
   14283              :     {
   14284              :     case 1:
   14285              :       pat = GEN_FCN (icode) (target, xops[0]);
   14286              :       break;
   14287          810 :     case 2:
   14288          810 :       pat = GEN_FCN (icode) (target, xops[0], xops[1]);
   14289          810 :       break;
   14290          593 :     case 3:
   14291          593 :       pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2]);
   14292          593 :       break;
   14293         4596 :     case 4:
   14294         4596 :       pat = GEN_FCN (icode) (target, xops[0], xops[1],
   14295         4596 :                              xops[2], xops[3]);
   14296         4596 :       break;
   14297         7730 :     case 5:
   14298         7730 :       pat = GEN_FCN (icode) (target, xops[0], xops[1],
   14299         7730 :                              xops[2], xops[3], xops[4]);
   14300         7730 :       break;
   14301         1273 :     case 6:
   14302         1273 :       pat = GEN_FCN (icode) (target, xops[0], xops[1],
   14303         1273 :                              xops[2], xops[3], xops[4], xops[5]);
   14304         1273 :       break;
   14305              :     default:
   14306              :       gcc_unreachable ();
   14307              :     }
   14308              : 
   14309        15002 :   if (!pat)
   14310              :     return 0;
   14311              : 
   14312        15002 :   if (redundant_embed_rnd)
   14313         4957 :     pat = ix86_erase_embedded_rounding (pat);
   14314              : 
   14315        15002 :   emit_insn (pat);
   14316        15002 :   return target;
   14317              : }
   14318              : 
   14319              : /* Subroutine of ix86_expand_builtin to take care of special insns
   14320              :    with variable number of operands.  */
   14321              : 
   14322              : static rtx
   14323        27160 : ix86_expand_special_args_builtin (const struct builtin_description *d,
   14324              :                                   tree exp, rtx target)
   14325              : {
   14326        27160 :   tree arg;
   14327        27160 :   rtx pat, op;
   14328        27160 :   unsigned int i, nargs, arg_adjust, memory;
   14329        27160 :   unsigned int constant = 100;
   14330        27160 :   bool aligned_mem = false;
   14331        27160 :   rtx xops[4];
   14332        27160 :   enum insn_code icode = d->icode;
   14333        27160 :   const struct insn_data_d *insn_p = &insn_data[icode];
   14334        27160 :   machine_mode tmode = insn_p->operand[0].mode;
   14335        27160 :   enum { load, store } klass;
   14336              : 
   14337        27160 :   switch ((enum ix86_builtin_func_type) d->flag)
   14338              :     {
   14339        15382 :     case VOID_FTYPE_VOID:
   14340        15382 :       emit_insn (GEN_FCN (icode) (target));
   14341        15382 :       return 0;
   14342              :     case VOID_FTYPE_UINT64:
   14343              :     case VOID_FTYPE_UNSIGNED:
   14344              :       nargs = 0;
   14345              :       klass = store;
   14346              :       memory = 0;
   14347              :       break;
   14348              : 
   14349         7510 :     case INT_FTYPE_VOID:
   14350         7510 :     case USHORT_FTYPE_VOID:
   14351         7510 :     case UINT64_FTYPE_VOID:
   14352         7510 :     case UINT_FTYPE_VOID:
   14353         7510 :     case UINT8_FTYPE_VOID:
   14354         7510 :     case UNSIGNED_FTYPE_VOID:
   14355         7510 :       nargs = 0;
   14356         7510 :       klass = load;
   14357         7510 :       memory = 0;
   14358         7510 :       break;
   14359          360 :     case CHAR_FTYPE_PCCHAR:
   14360          360 :     case SHORT_FTYPE_PCSHORT:
   14361          360 :     case INT_FTYPE_PCINT:
   14362          360 :     case INT64_FTYPE_PCINT64:
   14363          360 :     case UINT64_FTYPE_PUNSIGNED:
   14364          360 :     case V2DI_FTYPE_PV2DI:
   14365          360 :     case V4DI_FTYPE_PV4DI:
   14366          360 :     case V32QI_FTYPE_PCCHAR:
   14367          360 :     case V16QI_FTYPE_PCCHAR:
   14368          360 :     case V8SF_FTYPE_PCV4SF:
   14369          360 :     case V8SF_FTYPE_PCFLOAT:
   14370          360 :     case V4SF_FTYPE_PCFLOAT:
   14371          360 :     case V4SF_FTYPE_PCFLOAT16:
   14372          360 :     case V4SF_FTYPE_PCBFLOAT16:
   14373          360 :     case V4SF_FTYPE_PCV8BF:
   14374          360 :     case V4SF_FTYPE_PCV8HF:
   14375          360 :     case V8SF_FTYPE_PCFLOAT16:
   14376          360 :     case V8SF_FTYPE_PCBFLOAT16:
   14377          360 :     case V8SF_FTYPE_PCV16HF:
   14378          360 :     case V8SF_FTYPE_PCV16BF:
   14379          360 :     case V4DF_FTYPE_PCV2DF:
   14380          360 :     case V4DF_FTYPE_PCDOUBLE:
   14381          360 :     case V2DF_FTYPE_PCDOUBLE:
   14382          360 :     case VOID_FTYPE_PVOID:
   14383          360 :     case V8DI_FTYPE_PV8DI:
   14384          360 :       nargs = 1;
   14385          360 :       klass = load;
   14386          360 :       memory = 0;
   14387          360 :       switch (icode)
   14388              :         {
   14389              :         case CODE_FOR_sse4_1_movntdqa:
   14390              :         case CODE_FOR_avx2_movntdqa:
   14391              :         case CODE_FOR_avx512f_movntdqa:
   14392              :           aligned_mem = true;
   14393              :           break;
   14394              :         default:
   14395              :           break;
   14396              :         }
   14397              :       break;
   14398          373 :     case VOID_FTYPE_PV2SF_V4SF:
   14399          373 :     case VOID_FTYPE_PV8DI_V8DI:
   14400          373 :     case VOID_FTYPE_PV4DI_V4DI:
   14401          373 :     case VOID_FTYPE_PV2DI_V2DI:
   14402          373 :     case VOID_FTYPE_PCHAR_V32QI:
   14403          373 :     case VOID_FTYPE_PCHAR_V16QI:
   14404          373 :     case VOID_FTYPE_PFLOAT_V16SF:
   14405          373 :     case VOID_FTYPE_PFLOAT_V8SF:
   14406          373 :     case VOID_FTYPE_PFLOAT_V4SF:
   14407          373 :     case VOID_FTYPE_PDOUBLE_V8DF:
   14408          373 :     case VOID_FTYPE_PDOUBLE_V4DF:
   14409          373 :     case VOID_FTYPE_PDOUBLE_V2DF:
   14410          373 :     case VOID_FTYPE_PLONGLONG_LONGLONG:
   14411          373 :     case VOID_FTYPE_PULONGLONG_ULONGLONG:
   14412          373 :     case VOID_FTYPE_PUNSIGNED_UNSIGNED:
   14413          373 :     case VOID_FTYPE_PINT_INT:
   14414          373 :       nargs = 1;
   14415          373 :       klass = store;
   14416              :       /* Reserve memory operand for target.  */
   14417          373 :       memory = ARRAY_SIZE (xops);
   14418          373 :       switch (icode)
   14419              :         {
   14420              :         /* These builtins and instructions require the memory
   14421              :            to be properly aligned.  */
   14422              :         case CODE_FOR_avx_movntv4di:
   14423              :         case CODE_FOR_sse2_movntv2di:
   14424              :         case CODE_FOR_avx_movntv8sf:
   14425              :         case CODE_FOR_sse_movntv4sf:
   14426              :         case CODE_FOR_sse4a_vmmovntv4sf:
   14427              :         case CODE_FOR_avx_movntv4df:
   14428              :         case CODE_FOR_sse2_movntv2df:
   14429              :         case CODE_FOR_sse4a_vmmovntv2df:
   14430              :         case CODE_FOR_sse2_movntidi:
   14431              :         case CODE_FOR_sse_movntq:
   14432              :         case CODE_FOR_sse2_movntisi:
   14433              :         case CODE_FOR_avx512f_movntv16sf:
   14434              :         case CODE_FOR_avx512f_movntv8df:
   14435              :         case CODE_FOR_avx512f_movntv8di:
   14436              :           aligned_mem = true;
   14437              :           break;
   14438              :         default:
   14439              :           break;
   14440              :         }
   14441              :       break;
   14442            0 :     case VOID_FTYPE_PVOID_PCVOID:
   14443            0 :         nargs = 1;
   14444            0 :         klass = store;
   14445            0 :         memory = 0;
   14446              : 
   14447            0 :         break;
   14448           26 :     case V4SF_FTYPE_V4SF_PCV2SF:
   14449           26 :     case V2DF_FTYPE_V2DF_PCDOUBLE:
   14450           26 :       nargs = 2;
   14451           26 :       klass = load;
   14452           26 :       memory = 1;
   14453           26 :       break;
   14454           93 :     case V8SF_FTYPE_PCV8SF_V8SI:
   14455           93 :     case V4DF_FTYPE_PCV4DF_V4DI:
   14456           93 :     case V4SF_FTYPE_PCV4SF_V4SI:
   14457           93 :     case V2DF_FTYPE_PCV2DF_V2DI:
   14458           93 :     case V8SI_FTYPE_PCV8SI_V8SI:
   14459           93 :     case V4DI_FTYPE_PCV4DI_V4DI:
   14460           93 :     case V4SI_FTYPE_PCV4SI_V4SI:
   14461           93 :     case V2DI_FTYPE_PCV2DI_V2DI:
   14462           93 :     case VOID_FTYPE_INT_INT64:
   14463           93 :       nargs = 2;
   14464           93 :       klass = load;
   14465           93 :       memory = 0;
   14466           93 :       break;
   14467          360 :     case VOID_FTYPE_PV8DF_V8DF_UQI:
   14468          360 :     case VOID_FTYPE_PV4DF_V4DF_UQI:
   14469          360 :     case VOID_FTYPE_PV2DF_V2DF_UQI:
   14470          360 :     case VOID_FTYPE_PV16SF_V16SF_UHI:
   14471          360 :     case VOID_FTYPE_PV8SF_V8SF_UQI:
   14472          360 :     case VOID_FTYPE_PV4SF_V4SF_UQI:
   14473          360 :     case VOID_FTYPE_PV8DI_V8DI_UQI:
   14474          360 :     case VOID_FTYPE_PV4DI_V4DI_UQI:
   14475          360 :     case VOID_FTYPE_PV2DI_V2DI_UQI:
   14476          360 :     case VOID_FTYPE_PV16SI_V16SI_UHI:
   14477          360 :     case VOID_FTYPE_PV8SI_V8SI_UQI:
   14478          360 :     case VOID_FTYPE_PV4SI_V4SI_UQI:
   14479          360 :     case VOID_FTYPE_PV64QI_V64QI_UDI:
   14480          360 :     case VOID_FTYPE_PV32HI_V32HI_USI:
   14481          360 :     case VOID_FTYPE_PV32QI_V32QI_USI:
   14482          360 :     case VOID_FTYPE_PV16QI_V16QI_UHI:
   14483          360 :     case VOID_FTYPE_PV16HI_V16HI_UHI:
   14484          360 :     case VOID_FTYPE_PV8HI_V8HI_UQI:
   14485          360 :       switch (icode)
   14486              :         {
   14487              :         /* These builtins and instructions require the memory
   14488              :            to be properly aligned.  */
   14489          144 :         case CODE_FOR_avx512f_storev16sf_mask:
   14490          144 :         case CODE_FOR_avx512f_storev16si_mask:
   14491          144 :         case CODE_FOR_avx512f_storev8df_mask:
   14492          144 :         case CODE_FOR_avx512f_storev8di_mask:
   14493          144 :         case CODE_FOR_avx512vl_storev8sf_mask:
   14494          144 :         case CODE_FOR_avx512vl_storev8si_mask:
   14495          144 :         case CODE_FOR_avx512vl_storev4df_mask:
   14496          144 :         case CODE_FOR_avx512vl_storev4di_mask:
   14497          144 :         case CODE_FOR_avx512vl_storev4sf_mask:
   14498          144 :         case CODE_FOR_avx512vl_storev4si_mask:
   14499          144 :         case CODE_FOR_avx512vl_storev2df_mask:
   14500          144 :         case CODE_FOR_avx512vl_storev2di_mask:
   14501          144 :           aligned_mem = true;
   14502          144 :           break;
   14503              :         default:
   14504              :           break;
   14505              :         }
   14506              :       /* FALLTHRU */
   14507              :     case VOID_FTYPE_PV8SF_V8SI_V8SF:
   14508              :     case VOID_FTYPE_PV4DF_V4DI_V4DF:
   14509              :     case VOID_FTYPE_PV4SF_V4SI_V4SF:
   14510              :     case VOID_FTYPE_PV2DF_V2DI_V2DF:
   14511              :     case VOID_FTYPE_PV8SI_V8SI_V8SI:
   14512              :     case VOID_FTYPE_PV4DI_V4DI_V4DI:
   14513              :     case VOID_FTYPE_PV4SI_V4SI_V4SI:
   14514              :     case VOID_FTYPE_PV2DI_V2DI_V2DI:
   14515              :     case VOID_FTYPE_PV8SI_V8DI_UQI:
   14516              :     case VOID_FTYPE_PV8HI_V8DI_UQI:
   14517              :     case VOID_FTYPE_PV16HI_V16SI_UHI:
   14518              :     case VOID_FTYPE_PUDI_V8DI_UQI:
   14519              :     case VOID_FTYPE_PV16QI_V16SI_UHI:
   14520              :     case VOID_FTYPE_PV4SI_V4DI_UQI:
   14521              :     case VOID_FTYPE_PUDI_V2DI_UQI:
   14522              :     case VOID_FTYPE_PUDI_V4DI_UQI:
   14523              :     case VOID_FTYPE_PUSI_V2DI_UQI:
   14524              :     case VOID_FTYPE_PV8HI_V8SI_UQI:
   14525              :     case VOID_FTYPE_PUDI_V4SI_UQI:
   14526              :     case VOID_FTYPE_PUSI_V4DI_UQI:
   14527              :     case VOID_FTYPE_PUHI_V2DI_UQI:
   14528              :     case VOID_FTYPE_PUDI_V8SI_UQI:
   14529              :     case VOID_FTYPE_PUSI_V4SI_UQI:
   14530              :     case VOID_FTYPE_PCHAR_V64QI_UDI:
   14531              :     case VOID_FTYPE_PCHAR_V32QI_USI:
   14532              :     case VOID_FTYPE_PCHAR_V16QI_UHI:
   14533              :     case VOID_FTYPE_PSHORT_V32HI_USI:
   14534              :     case VOID_FTYPE_PSHORT_V16HI_UHI:
   14535              :     case VOID_FTYPE_PSHORT_V8HI_UQI:
   14536              :     case VOID_FTYPE_PINT_V16SI_UHI:
   14537              :     case VOID_FTYPE_PINT_V8SI_UQI:
   14538              :     case VOID_FTYPE_PINT_V4SI_UQI:
   14539              :     case VOID_FTYPE_PINT64_V8DI_UQI:
   14540              :     case VOID_FTYPE_PINT64_V4DI_UQI:
   14541              :     case VOID_FTYPE_PINT64_V2DI_UQI:
   14542              :     case VOID_FTYPE_PDOUBLE_V8DF_UQI:
   14543              :     case VOID_FTYPE_PDOUBLE_V4DF_UQI:
   14544              :     case VOID_FTYPE_PDOUBLE_V2DF_UQI:
   14545              :     case VOID_FTYPE_PFLOAT_V16SF_UHI:
   14546              :     case VOID_FTYPE_PFLOAT_V8SF_UQI:
   14547              :     case VOID_FTYPE_PFLOAT_V4SF_UQI:
   14548              :     case VOID_FTYPE_PCFLOAT16_V8HF_UQI:
   14549              :     case VOID_FTYPE_PV32QI_V32HI_USI:
   14550              :     case VOID_FTYPE_PV16QI_V16HI_UHI:
   14551              :     case VOID_FTYPE_PUDI_V8HI_UQI:
   14552              :       nargs = 2;
   14553              :       klass = store;
   14554              :       /* Reserve memory operand for target.  */
   14555              :       memory = ARRAY_SIZE (xops);
   14556              :       break;
   14557         1243 :     case V4SF_FTYPE_PCV4SF_V4SF_UQI:
   14558         1243 :     case V8SF_FTYPE_PCV8SF_V8SF_UQI:
   14559         1243 :     case V16SF_FTYPE_PCV16SF_V16SF_UHI:
   14560         1243 :     case V4SI_FTYPE_PCV4SI_V4SI_UQI:
   14561         1243 :     case V8SI_FTYPE_PCV8SI_V8SI_UQI:
   14562         1243 :     case V16SI_FTYPE_PCV16SI_V16SI_UHI:
   14563         1243 :     case V2DF_FTYPE_PCV2DF_V2DF_UQI:
   14564         1243 :     case V4DF_FTYPE_PCV4DF_V4DF_UQI:
   14565         1243 :     case V8DF_FTYPE_PCV8DF_V8DF_UQI:
   14566         1243 :     case V2DI_FTYPE_PCV2DI_V2DI_UQI:
   14567         1243 :     case V4DI_FTYPE_PCV4DI_V4DI_UQI:
   14568         1243 :     case V8DI_FTYPE_PCV8DI_V8DI_UQI:
   14569         1243 :     case V64QI_FTYPE_PCV64QI_V64QI_UDI:
   14570         1243 :     case V32HI_FTYPE_PCV32HI_V32HI_USI:
   14571         1243 :     case V32QI_FTYPE_PCV32QI_V32QI_USI:
   14572         1243 :     case V16QI_FTYPE_PCV16QI_V16QI_UHI:
   14573         1243 :     case V16HI_FTYPE_PCV16HI_V16HI_UHI:
   14574         1243 :     case V8HI_FTYPE_PCV8HI_V8HI_UQI:
   14575         1243 :       switch (icode)
   14576              :         {
   14577              :         /* These builtins and instructions require the memory
   14578              :            to be properly aligned.  */
   14579          288 :         case CODE_FOR_avx512f_loadv16sf_mask:
   14580          288 :         case CODE_FOR_avx512f_loadv16si_mask:
   14581          288 :         case CODE_FOR_avx512f_loadv8df_mask:
   14582          288 :         case CODE_FOR_avx512f_loadv8di_mask:
   14583          288 :         case CODE_FOR_avx512vl_loadv8sf_mask:
   14584          288 :         case CODE_FOR_avx512vl_loadv8si_mask:
   14585          288 :         case CODE_FOR_avx512vl_loadv4df_mask:
   14586          288 :         case CODE_FOR_avx512vl_loadv4di_mask:
   14587          288 :         case CODE_FOR_avx512vl_loadv4sf_mask:
   14588          288 :         case CODE_FOR_avx512vl_loadv4si_mask:
   14589          288 :         case CODE_FOR_avx512vl_loadv2df_mask:
   14590          288 :         case CODE_FOR_avx512vl_loadv2di_mask:
   14591          288 :         case CODE_FOR_avx512bw_loadv64qi_mask:
   14592          288 :         case CODE_FOR_avx512vl_loadv32qi_mask:
   14593          288 :         case CODE_FOR_avx512vl_loadv16qi_mask:
   14594          288 :         case CODE_FOR_avx512bw_loadv32hi_mask:
   14595          288 :         case CODE_FOR_avx512vl_loadv16hi_mask:
   14596          288 :         case CODE_FOR_avx512vl_loadv8hi_mask:
   14597          288 :           aligned_mem = true;
   14598          288 :           break;
   14599              :         default:
   14600              :           break;
   14601              :         }
   14602              :       /* FALLTHRU */
   14603              :     case V64QI_FTYPE_PCCHAR_V64QI_UDI:
   14604              :     case V32QI_FTYPE_PCCHAR_V32QI_USI:
   14605              :     case V16QI_FTYPE_PCCHAR_V16QI_UHI:
   14606              :     case V32HI_FTYPE_PCSHORT_V32HI_USI:
   14607              :     case V16HI_FTYPE_PCSHORT_V16HI_UHI:
   14608              :     case V8HI_FTYPE_PCSHORT_V8HI_UQI:
   14609              :     case V16SI_FTYPE_PCINT_V16SI_UHI:
   14610              :     case V8SI_FTYPE_PCINT_V8SI_UQI:
   14611              :     case V4SI_FTYPE_PCINT_V4SI_UQI:
   14612              :     case V8DI_FTYPE_PCINT64_V8DI_UQI:
   14613              :     case V4DI_FTYPE_PCINT64_V4DI_UQI:
   14614              :     case V2DI_FTYPE_PCINT64_V2DI_UQI:
   14615              :     case V8DF_FTYPE_PCDOUBLE_V8DF_UQI:
   14616              :     case V4DF_FTYPE_PCDOUBLE_V4DF_UQI:
   14617              :     case V2DF_FTYPE_PCDOUBLE_V2DF_UQI:
   14618              :     case V16SF_FTYPE_PCFLOAT_V16SF_UHI:
   14619              :     case V8SF_FTYPE_PCFLOAT_V8SF_UQI:
   14620              :     case V4SF_FTYPE_PCFLOAT_V4SF_UQI:
   14621              :     case V8HF_FTYPE_PCFLOAT16_V8HF_UQI:
   14622              :       nargs = 3;
   14623              :       klass = load;
   14624              :       memory = 0;
   14625              :       break;
   14626          105 :     case INT_FTYPE_PINT_INT_INT_INT:
   14627          105 :     case LONGLONG_FTYPE_PLONGLONG_LONGLONG_LONGLONG_INT:
   14628          105 :       nargs = 4;
   14629          105 :       klass = load;
   14630          105 :       memory = 0;
   14631          105 :       constant = 3;
   14632          105 :       break;
   14633            0 :     default:
   14634            0 :       gcc_unreachable ();
   14635              :     }
   14636              : 
   14637         8271 :   gcc_assert (nargs <= ARRAY_SIZE (xops));
   14638              : 
   14639        11778 :   if (klass == store)
   14640              :     {
   14641         1913 :       arg = CALL_EXPR_ARG (exp, 0);
   14642         1913 :       op = expand_normal (arg);
   14643         1913 :       gcc_assert (target == 0);
   14644         1913 :       if (memory)
   14645              :         {
   14646         1750 :           op = ix86_zero_extend_to_Pmode (op);
   14647         1750 :           target = gen_rtx_MEM (tmode, op);
   14648              :           /* target at this point has just BITS_PER_UNIT MEM_ALIGN
   14649              :              on it.  Try to improve it using get_pointer_alignment,
   14650              :              and if the special builtin is one that requires strict
   14651              :              mode alignment, also from it's GET_MODE_ALIGNMENT.
   14652              :              Failure to do so could lead to ix86_legitimate_combined_insn
   14653              :              rejecting all changes to such insns.  */
   14654         1750 :           unsigned int align = get_pointer_alignment (arg);
   14655         1750 :           if (aligned_mem && align < GET_MODE_ALIGNMENT (tmode))
   14656          275 :             align = GET_MODE_ALIGNMENT (tmode);
   14657         3500 :           if (MEM_ALIGN (target) < align)
   14658          422 :             set_mem_align (target, align);
   14659              :         }
   14660              :       else
   14661          163 :         target = force_reg (tmode, op);
   14662              :       arg_adjust = 1;
   14663              :     }
   14664              :   else
   14665              :     {
   14666         9865 :       arg_adjust = 0;
   14667         9865 :       if (optimize
   14668         2919 :           || target == 0
   14669         2919 :           || !register_operand (target, tmode)
   14670        12773 :           || GET_MODE (target) != tmode)
   14671         6957 :         target = gen_reg_rtx (tmode);
   14672              :     }
   14673              : 
   14674        21236 :   for (i = 0; i < nargs; i++)
   14675              :     {
   14676         9458 :       machine_mode mode = insn_p->operand[i + 1].mode;
   14677              : 
   14678         9458 :       arg = CALL_EXPR_ARG (exp, i + arg_adjust);
   14679         9458 :       op = ix86_expand_unsigned_small_int_cst_argument (arg);
   14680              : 
   14681         9458 :       if (i == memory)
   14682              :         {
   14683              :           /* This must be the memory operand.  */
   14684         2355 :           op = ix86_zero_extend_to_Pmode (op);
   14685         2355 :           op = gen_rtx_MEM (mode, op);
   14686              :           /* op at this point has just BITS_PER_UNIT MEM_ALIGN
   14687              :              on it.  Try to improve it using get_pointer_alignment,
   14688              :              and if the special builtin is one that requires strict
   14689              :              mode alignment, also from it's GET_MODE_ALIGNMENT.
   14690              :              Failure to do so could lead to ix86_legitimate_combined_insn
   14691              :              rejecting all changes to such insns.  */
   14692         2355 :           unsigned int align = get_pointer_alignment (arg);
   14693         2355 :           if (aligned_mem && align < GET_MODE_ALIGNMENT (mode))
   14694          299 :             align = GET_MODE_ALIGNMENT (mode);
   14695         4710 :           if (MEM_ALIGN (op) < align)
   14696          523 :             set_mem_align (op, align);
   14697              :         }
   14698         7103 :       else if (i == constant)
   14699              :         {
   14700              :           /* This must be the constant.  */
   14701          105 :           if (!insn_p->operand[nargs].predicate(op, SImode))
   14702              :             {
   14703            0 :               error ("the fourth argument must be one of enum %qs", "_CMPCCX_ENUM");
   14704            0 :               return const0_rtx;
   14705              :             }
   14706              :         }
   14707              :       else
   14708              :         {
   14709              :           /* This must be register.  */
   14710         6998 :           if (VECTOR_MODE_P (mode))
   14711         3508 :             op = safe_vector_operand (op, mode);
   14712              : 
   14713         6998 :           op = fixup_modeless_constant (op, mode);
   14714              : 
   14715              :           /* NB: 3-operands load implied it's a mask load or v{p}expand*,
   14716              :              and that mask operand should be at the end.
   14717              :              Keep all-ones mask which would be simplified by the expander.  */
   14718         1771 :           if (nargs == 3 && i == 2 && klass == load
   14719         1771 :               && constm1_operand (op, mode)
   14720         7171 :               && insn_p->operand[i].predicate (op, mode))
   14721              :             ;
   14722         6998 :           else if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
   14723         6998 :             op = copy_to_mode_reg (mode, op);
   14724              :           else
   14725              :             {
   14726            0 :               op = copy_to_reg (op);
   14727            0 :               op = lowpart_subreg (mode, op, GET_MODE (op));
   14728              :             }
   14729              :         }
   14730              : 
   14731         9458 :       xops[i]= op;
   14732              :     }
   14733              : 
   14734        11778 :   switch (nargs)
   14735              :     {
   14736         7673 :     case 0:
   14737         7673 :       pat = GEN_FCN (icode) (target);
   14738         7673 :       break;
   14739          733 :     case 1:
   14740          733 :       pat = GEN_FCN (icode) (target, xops[0]);
   14741          733 :       break;
   14742         1496 :     case 2:
   14743         1496 :       pat = GEN_FCN (icode) (target, xops[0], xops[1]);
   14744         1496 :       break;
   14745         1771 :     case 3:
   14746         1771 :       pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2]);
   14747         1771 :       break;
   14748          105 :     case 4:
   14749          105 :       pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2], xops[3]);
   14750          105 :       break;
   14751              :     default:
   14752              :       gcc_unreachable ();
   14753              :     }
   14754              : 
   14755        11778 :   if (! pat)
   14756              :     return 0;
   14757              : 
   14758        11778 :   emit_insn (pat);
   14759        11778 :   return klass == store ? 0 : target;
   14760              : }
   14761              : 
   14762              : /* Subroutine of ix86_expand_builtin to take care of amx insns
   14763              :    with variable number of operands.  */
   14764              : 
   14765              : static rtx
   14766          192 : ix86_expand_ace_builtin (const struct builtin_description *d, tree exp,
   14767              :                          rtx target)
   14768              : {
   14769          192 :   tree arg;
   14770          192 :   rtx pat, op;
   14771          192 :   unsigned int i, nargs, arg_adjust = 0, constant = 100;
   14772          192 :   bool tmm_src = false;
   14773          192 :   rtx xops[4];
   14774          192 :   enum insn_code icode = d->icode;
   14775          192 :   const struct insn_data_d *insn_p = &insn_data[icode];
   14776              : 
   14777          192 :   switch ((enum ix86_builtin_func_type) d->flag)
   14778              :     {
   14779              :     case VOID_FTYPE_UQI:
   14780              :       nargs = 1;
   14781              :       break;
   14782           60 :     case V16SF_FTYPE_UQI_SI:
   14783           60 :     case V32BF_FTYPE_UQI_SI:
   14784           60 :     case V32HF_FTYPE_UQI_SI:
   14785           60 :     case V16SI_FTYPE_UQI_SI:
   14786           60 :       nargs = 2;
   14787           60 :       tmm_src = true;
   14788           60 :       break;
   14789           70 :     case VOID_FTYPE_UQI_V16SI_SI:
   14790           70 :     case VOID_FTYPE_UQI_V32BF_V32BF:
   14791           70 :     case VOID_FTYPE_UQI_V64QI_V64QI:
   14792           70 :       nargs = 3;
   14793           70 :       break;
   14794           50 :     case VOID_FTYPE_UQI_V64QI_V64QI_SI:
   14795           50 :       nargs = 4;
   14796           50 :       constant = 3;
   14797           50 :       break;
   14798              : 
   14799            0 :     default:
   14800            0 :       gcc_unreachable ();
   14801              :     }
   14802              : 
   14803          180 :   gcc_assert (nargs <= ARRAY_SIZE (xops));
   14804              : 
   14805          192 :   if (tmm_src)
   14806              :     {
   14807           60 :       machine_mode tmode = insn_p->operand[0].mode;
   14808           60 :       arg_adjust = 1;
   14809           60 :       if (optimize
   14810           18 :           || target == 0
   14811           18 :           || !register_operand (target, tmode)
   14812           78 :           || GET_MODE (target) != tmode)
   14813           42 :         target = gen_reg_rtx (tmode);
   14814              :     }
   14815              : 
   14816          732 :   for (i = 0; i < nargs; i++)
   14817              :     {
   14818          542 :       machine_mode mode = insn_p->operand[i + arg_adjust].mode;
   14819              : 
   14820          542 :       arg = CALL_EXPR_ARG (exp, i);
   14821          542 :       op = ix86_expand_unsigned_small_int_cst_argument (arg);
   14822              : 
   14823          542 :       if (i == 0 || i == constant)
   14824              :         {
   14825          242 :           if (!insn_p->operand[i + arg_adjust].predicate (op, SImode))
   14826              :             {
   14827            2 :               if (i == 0)
   14828              :                 /* This must be the tmm reg number constant.  */
   14829            2 :                 error ("the tmm register number argument must be between 0 to 7");
   14830              :               else
   14831              :                 /* This must be the constant.  */
   14832            0 :                 error ("the argument must be constant");
   14833            2 :               return const0_rtx;
   14834              :             }
   14835              :         }
   14836              :       else
   14837              :         {
   14838              :           /* This must be register.  */
   14839          300 :           if (VECTOR_MODE_P (mode))
   14840          220 :             op = safe_vector_operand (op, mode);
   14841              : 
   14842          300 :           op = fixup_modeless_constant (op, mode);
   14843              : 
   14844          300 :           if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
   14845          300 :             op = copy_to_mode_reg (mode, op);
   14846              :           else
   14847              :             {
   14848            0 :               op = copy_to_reg (op);
   14849            0 :               op = lowpart_subreg (mode, op, GET_MODE (op));
   14850              :             }
   14851              :         }
   14852              : 
   14853          540 :       xops[i] = op;
   14854              :     }
   14855              : 
   14856          190 :   if (tmm_src)
   14857              :     {
   14858           60 :       switch (nargs)
   14859              :         {
   14860           60 :         case 2:
   14861           60 :           pat = GEN_FCN (icode) (target, xops[0], xops[1]);
   14862           60 :           break;
   14863            0 :         default:
   14864            0 :           gcc_unreachable ();
   14865              :         }
   14866              :   }
   14867              :   else
   14868              :     {
   14869          130 :       switch (nargs)
   14870              :         {
   14871           10 :         case 1:
   14872           10 :           pat = GEN_FCN (icode) (xops[0]);
   14873           10 :           break;
   14874           70 :         case 3:
   14875           70 :           pat = GEN_FCN (icode) (xops[0], xops[1], xops[2]);
   14876           70 :           break;
   14877           50 :         case 4:
   14878           50 :           pat = GEN_FCN (icode) (xops[0], xops[1], xops[2], xops[3]);
   14879           50 :           break;
   14880            0 :         default:
   14881            0 :           gcc_unreachable ();
   14882              :         }
   14883              :     }
   14884              : 
   14885          190 :   if (!pat)
   14886              :     return 0;
   14887              : 
   14888          190 :   emit_insn (pat);
   14889          190 :   return tmm_src ? target : 0;
   14890              : }
   14891              : 
   14892              : /* Return the integer constant in ARG.  Constrain it to be in the range
   14893              :    of the subparts of VEC_TYPE; issue an error if not.  */
   14894              : 
   14895              : static int
   14896          604 : get_element_number (tree vec_type, tree arg)
   14897              : {
   14898          604 :   unsigned HOST_WIDE_INT elt, max = TYPE_VECTOR_SUBPARTS (vec_type) - 1;
   14899              : 
   14900          604 :   if (!tree_fits_uhwi_p (arg)
   14901          604 :       || (elt = tree_to_uhwi (arg), elt > max))
   14902              :     {
   14903            0 :       error ("selector must be an integer constant in the range "
   14904              :              "[0, %wi]", max);
   14905            0 :       return 0;
   14906              :     }
   14907              : 
   14908          604 :   return elt;
   14909              : }
   14910              : 
   14911              : /* A subroutine of ix86_expand_builtin.  These builtins are a wrapper around
   14912              :    ix86_expand_vector_init.  We DO have language-level syntax for this, in
   14913              :    the form of  (type){ init-list }.  Except that since we can't place emms
   14914              :    instructions from inside the compiler, we can't allow the use of MMX
   14915              :    registers unless the user explicitly asks for it.  So we do *not* define
   14916              :    vec_set/vec_extract/vec_init patterns for MMX modes in mmx.md.  Instead
   14917              :    we have builtins invoked by mmintrin.h that gives us license to emit
   14918              :    these sorts of instructions.  */
   14919              : 
   14920              : static rtx
   14921          229 : ix86_expand_vec_init_builtin (tree type, tree exp, rtx target)
   14922              : {
   14923          229 :   machine_mode tmode = TYPE_MODE (type);
   14924          229 :   machine_mode inner_mode = GET_MODE_INNER (tmode);
   14925          229 :   int i, n_elt = GET_MODE_NUNITS (tmode);
   14926          229 :   rtvec v = rtvec_alloc (n_elt);
   14927              : 
   14928          229 :   gcc_assert (VECTOR_MODE_P (tmode));
   14929          229 :   gcc_assert (call_expr_nargs (exp) == n_elt);
   14930              : 
   14931         1203 :   for (i = 0; i < n_elt; ++i)
   14932              :     {
   14933          974 :       rtx x = expand_normal (CALL_EXPR_ARG (exp, i));
   14934          974 :       RTVEC_ELT (v, i) = gen_lowpart (inner_mode, x);
   14935              :     }
   14936              : 
   14937          229 :   if (!target || !register_operand (target, tmode))
   14938            0 :     target = gen_reg_rtx (tmode);
   14939              : 
   14940          229 :   ix86_expand_vector_init (true, target, gen_rtx_PARALLEL (tmode, v));
   14941          229 :   return target;
   14942              : }
   14943              : 
   14944              : /* A subroutine of ix86_expand_builtin.  These builtins are a wrapper around
   14945              :    ix86_expand_vector_extract.  They would be redundant (for non-MMX) if we
   14946              :    had a language-level syntax for referencing vector elements.  */
   14947              : 
   14948              : static rtx
   14949          400 : ix86_expand_vec_ext_builtin (tree exp, rtx target)
   14950              : {
   14951          400 :   machine_mode tmode, mode0;
   14952          400 :   tree arg0, arg1;
   14953          400 :   int elt;
   14954          400 :   rtx op0;
   14955              : 
   14956          400 :   arg0 = CALL_EXPR_ARG (exp, 0);
   14957          400 :   arg1 = CALL_EXPR_ARG (exp, 1);
   14958              : 
   14959          400 :   op0 = expand_normal (arg0);
   14960          400 :   elt = get_element_number (TREE_TYPE (arg0), arg1);
   14961              : 
   14962          400 :   tmode = TYPE_MODE (TREE_TYPE (TREE_TYPE (arg0)));
   14963          400 :   mode0 = TYPE_MODE (TREE_TYPE (arg0));
   14964          400 :   gcc_assert (VECTOR_MODE_P (mode0));
   14965              : 
   14966          400 :   op0 = force_reg (mode0, op0);
   14967              : 
   14968          400 :   if (optimize || !target || !register_operand (target, tmode))
   14969          321 :     target = gen_reg_rtx (tmode);
   14970              : 
   14971          400 :   ix86_expand_vector_extract (true, target, op0, elt);
   14972              : 
   14973          400 :   return target;
   14974              : }
   14975              : 
   14976              : /* A subroutine of ix86_expand_builtin.  These builtins are a wrapper around
   14977              :    ix86_expand_vector_set.  They would be redundant (for non-MMX) if we had
   14978              :    a language-level syntax for referencing vector elements.  */
   14979              : 
   14980              : static rtx
   14981          204 : ix86_expand_vec_set_builtin (tree exp)
   14982              : {
   14983          204 :   machine_mode tmode, mode1;
   14984          204 :   tree arg0, arg1, arg2;
   14985          204 :   int elt;
   14986          204 :   rtx op0, op1, target;
   14987              : 
   14988          204 :   arg0 = CALL_EXPR_ARG (exp, 0);
   14989          204 :   arg1 = CALL_EXPR_ARG (exp, 1);
   14990          204 :   arg2 = CALL_EXPR_ARG (exp, 2);
   14991              : 
   14992          204 :   tmode = TYPE_MODE (TREE_TYPE (arg0));
   14993          204 :   mode1 = TYPE_MODE (TREE_TYPE (TREE_TYPE (arg0)));
   14994          204 :   gcc_assert (VECTOR_MODE_P (tmode));
   14995              : 
   14996          204 :   op0 = expand_expr (arg0, NULL_RTX, tmode, EXPAND_NORMAL);
   14997          204 :   op1 = expand_expr (arg1, NULL_RTX, mode1, EXPAND_NORMAL);
   14998          204 :   elt = get_element_number (TREE_TYPE (arg0), arg2);
   14999              : 
   15000          204 :   if (GET_MODE (op1) != mode1)
   15001           82 :     op1 = convert_modes (mode1, GET_MODE (op1), op1, true);
   15002              : 
   15003          204 :   op0 = force_reg (tmode, op0);
   15004          204 :   if (op1 != CONST0_RTX (mode1))
   15005          204 :     op1 = force_reg (mode1, op1);
   15006              : 
   15007              :   /* OP0 is the source of these builtin functions and shouldn't be
   15008              :      modified.  Create a copy, use it and return it as target.  */
   15009          204 :   target = gen_reg_rtx (tmode);
   15010          204 :   emit_move_insn (target, op0);
   15011          204 :   ix86_expand_vector_set (true, target, op1, elt);
   15012              : 
   15013          204 :   return target;
   15014              : }
   15015              : 
   15016              : /* Return true if the necessary isa options for this builtin exist,
   15017              :    else false.
   15018              :    fcode = DECL_MD_FUNCTION_CODE (fndecl);  */
   15019              : bool
   15020      1337694 : ix86_check_builtin_isa_match (unsigned int fcode,
   15021              :                               HOST_WIDE_INT* pbisa,
   15022              :                               HOST_WIDE_INT* pbisa2)
   15023              : {
   15024      1337694 :   HOST_WIDE_INT isa = ix86_isa_flags;
   15025      1337694 :   HOST_WIDE_INT isa2 = ix86_isa_flags2;
   15026      1337694 :   HOST_WIDE_INT bisa = ix86_builtins_isa[fcode].isa;
   15027      1337694 :   HOST_WIDE_INT bisa2 = ix86_builtins_isa[fcode].isa2;
   15028      1337694 :   HOST_WIDE_INT tmp_isa = isa, tmp_isa2 = isa2;
   15029              :   /* The general case is we require all the ISAs specified in bisa{,2}
   15030              :      to be enabled.
   15031              :      The exceptions are:
   15032              :      OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A
   15033              :      OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32
   15034              :      OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4
   15035              :      (OPTION_MASK_ISA_AVX512VNNI | OPTION_MASK_ISA_AVX512VL) or
   15036              :        OPTION_MASK_ISA2_AVXVNNI
   15037              :      (OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL) or
   15038              :        OPTION_MASK_ISA2_AVXIFMA
   15039              :      (OPTION_MASK_ISA_AVX512VL | OPTION_MASK_ISA2_AVX512BF16) or
   15040              :        OPTION_MASK_ISA2_AVXNECONVERT
   15041              :      OPTION_MASK_ISA_AES or (OPTION_MASK_ISA_AVX512VL | OPTION_MASK_ISA2_VAES)
   15042              :      OPTION_MASK_ISA2_AVX10V1AUX or OPTION_MASK_ISA2_AVXVNNIINT8
   15043              :      OPTION_MASK_ISA2_AVX10V1AUX or OPTION_MASK_ISA2_AVXVNNIINT16
   15044              :      OPTION_MASK_ISA2_AMX_TILE or OPTION_MASK_ISA2_ACEV1
   15045              :      where for each such pair it is sufficient if either of the ISAs is
   15046              :      enabled, plus if it is ored with other options also those others.
   15047              :      OPTION_MASK_ISA_MMX in bisa is satisfied also if TARGET_MMX_WITH_SSE.  */
   15048              : 
   15049              : #define SHARE_BUILTIN(A1, A2, B1, B2) \
   15050              :   if ((((bisa & (A1)) == (A1) && (bisa2 & (A2)) == (A2)) \
   15051              :        && ((bisa & (B1)) == (B1) && (bisa2 & (B2)) == (B2))) \
   15052              :       && (((isa & (A1)) == (A1) && (isa2 & (A2)) == (A2)) \
   15053              :           || ((isa & (B1)) == (B1) && (isa2 & (B2)) == (B2)))) \
   15054              :     { \
   15055              :       tmp_isa |= (A1) | (B1); \
   15056              :       tmp_isa2 |= (A2) | (B2); \
   15057              :     }
   15058              : 
   15059      1337694 :   SHARE_BUILTIN (OPTION_MASK_ISA_SSE, 0, OPTION_MASK_ISA_3DNOW_A, 0);
   15060      1337694 :   SHARE_BUILTIN (OPTION_MASK_ISA_SSE4_2, 0, OPTION_MASK_ISA_CRC32, 0);
   15061      1337694 :   SHARE_BUILTIN (OPTION_MASK_ISA_FMA, 0, OPTION_MASK_ISA_FMA4, 0);
   15062      1337694 :   SHARE_BUILTIN (OPTION_MASK_ISA_AVX512VNNI | OPTION_MASK_ISA_AVX512VL, 0, 0,
   15063      1337694 :                  OPTION_MASK_ISA2_AVXVNNI);
   15064      1337694 :   SHARE_BUILTIN (OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, 0, 0,
   15065      1337694 :                  OPTION_MASK_ISA2_AVXIFMA);
   15066      1337694 :   SHARE_BUILTIN (OPTION_MASK_ISA_AVX512VL, OPTION_MASK_ISA2_AVX512BF16, 0,
   15067      1337694 :                  OPTION_MASK_ISA2_AVXNECONVERT);
   15068      1337694 :   SHARE_BUILTIN (OPTION_MASK_ISA_AES, 0, OPTION_MASK_ISA_AVX512VL,
   15069      1337694 :                  OPTION_MASK_ISA2_VAES);
   15070      1337694 :   SHARE_BUILTIN (0, OPTION_MASK_ISA2_AVXVNNIINT8, 0,
   15071      1337694 :                  OPTION_MASK_ISA2_AVX10V1AUX);
   15072      1337694 :   SHARE_BUILTIN (0, OPTION_MASK_ISA2_AVXVNNIINT16, 0,
   15073      1337694 :                  OPTION_MASK_ISA2_AVX10V1AUX);
   15074      1337694 :   SHARE_BUILTIN (0, OPTION_MASK_ISA2_AMX_TILE, 0, OPTION_MASK_ISA2_ACEV1);
   15075      1337694 :   isa = tmp_isa;
   15076      1337694 :   isa2 = tmp_isa2;
   15077              : 
   15078      1337694 :   if ((bisa & OPTION_MASK_ISA_MMX) && !TARGET_MMX && TARGET_MMX_WITH_SSE
   15079              :       /* __builtin_ia32_maskmovq requires MMX registers.  */
   15080         4531 :       && fcode != IX86_BUILTIN_MASKMOVQ)
   15081              :     {
   15082         4522 :       bisa &= ~OPTION_MASK_ISA_MMX;
   15083         4522 :       bisa |= OPTION_MASK_ISA_SSE2;
   15084              :     }
   15085              : 
   15086      1337694 :   if (pbisa)
   15087       176874 :     *pbisa = bisa;
   15088      1337694 :   if (pbisa2)
   15089       176874 :     *pbisa2 = bisa2;
   15090              : 
   15091      1337694 :   return (bisa & isa) == bisa && (bisa2 & isa2) == bisa2;
   15092              : }
   15093              : 
   15094              : /* Emit instructions to set the carry flag from ARG.  */
   15095              : 
   15096              : void
   15097        13894 : ix86_expand_carry (rtx arg)
   15098              : {
   15099        13894 :   if (!CONST_INT_P (arg) || arg == const0_rtx)
   15100              :     {
   15101        13887 :       arg = convert_to_mode (QImode, arg, 1);
   15102        13887 :       arg = copy_to_mode_reg (QImode, arg);
   15103        13887 :       emit_insn (gen_addqi3_cconly_overflow (arg, constm1_rtx));
   15104              :     }
   15105              :   else
   15106            7 :     emit_insn (gen_x86_stc ());
   15107        13894 : }
   15108              : 
   15109              : /* Expand an expression EXP that calls a built-in function,
   15110              :    with result going to TARGET if that's convenient
   15111              :    (and in mode MODE if that's convenient).
   15112              :    SUBTARGET may be used as the target for computing one of EXP's operands.
   15113              :    IGNORE is nonzero if the value is to be ignored.  */
   15114              : 
   15115              : rtx
   15116       177680 : ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
   15117              :                      machine_mode mode, int ignore)
   15118              : {
   15119       177680 :   size_t i;
   15120       177680 :   enum insn_code icode, icode2;
   15121       177680 :   tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
   15122       177680 :   tree arg0, arg1, arg2, arg3, arg4;
   15123       177680 :   rtx op0, op1, op2, op3, op4, pat, pat2, insn;
   15124       177680 :   machine_mode mode0, mode1, mode2, mode3, mode4;
   15125       177680 :   unsigned int fcode = DECL_MD_FUNCTION_CODE (fndecl);
   15126       177680 :   HOST_WIDE_INT bisa, bisa2;
   15127              : 
   15128              :   /* For CPU builtins that can be folded, fold first and expand the fold.  */
   15129       177680 :   switch (fcode)
   15130              :     {
   15131          201 :     case IX86_BUILTIN_CPU_INIT:
   15132          201 :       {
   15133              :         /* Make it call __cpu_indicator_init in libgcc.  */
   15134          201 :         tree call_expr, fndecl, type;
   15135          201 :         type = build_function_type_list (integer_type_node, NULL_TREE);
   15136          201 :         fndecl = build_fn_decl ("__cpu_indicator_init", type);
   15137          201 :         call_expr = build_call_expr (fndecl, 0);
   15138          201 :         return expand_expr (call_expr, target, mode, EXPAND_NORMAL);
   15139              :       }
   15140          605 :     case IX86_BUILTIN_CPU_IS:
   15141          605 :     case IX86_BUILTIN_CPU_SUPPORTS:
   15142          605 :       {
   15143          605 :         tree arg0 = CALL_EXPR_ARG (exp, 0);
   15144          605 :         tree fold_expr = fold_builtin_cpu (fndecl, &arg0);
   15145          605 :         gcc_assert (fold_expr != NULL_TREE);
   15146          605 :         return expand_expr (fold_expr, target, mode, EXPAND_NORMAL);
   15147              :       }
   15148              :     }
   15149              : 
   15150       176874 :   if (!ix86_check_builtin_isa_match (fcode, &bisa, &bisa2))
   15151              :     {
   15152           23 :       bool add_abi_p = bisa & OPTION_MASK_ISA_64BIT;
   15153           23 :       if (TARGET_ABI_X32)
   15154            0 :         bisa |= OPTION_MASK_ABI_X32;
   15155              :       else
   15156           23 :         bisa |= OPTION_MASK_ABI_64;
   15157           23 :       char *opts = ix86_target_string (bisa, bisa2, 0, 0, NULL, NULL,
   15158              :                                        (enum fpmath_unit) 0,
   15159              :                                        (enum prefer_vector_width) 0,
   15160              :                                        PVW_NONE, false, add_abi_p);
   15161           23 :       if (!opts)
   15162            0 :         error ("%qE needs unknown isa option", fndecl);
   15163              :       else
   15164              :         {
   15165           23 :           gcc_assert (opts != NULL);
   15166           23 :           error ("%qE needs isa option %s", fndecl, opts);
   15167           23 :           free (opts);
   15168              :         }
   15169           23 :       return expand_call (exp, target, ignore);
   15170              :     }
   15171              : 
   15172       176851 :   switch (fcode)
   15173              :     {
   15174           35 :     case IX86_BUILTIN_MASKMOVQ:
   15175           35 :     case IX86_BUILTIN_MASKMOVDQU:
   15176           34 :       icode = (fcode == IX86_BUILTIN_MASKMOVQ
   15177           35 :                ? CODE_FOR_mmx_maskmovq
   15178              :                : CODE_FOR_sse2_maskmovdqu);
   15179              :       /* Note the arg order is different from the operand order.  */
   15180           35 :       arg1 = CALL_EXPR_ARG (exp, 0);
   15181           35 :       arg2 = CALL_EXPR_ARG (exp, 1);
   15182           35 :       arg0 = CALL_EXPR_ARG (exp, 2);
   15183           35 :       op0 = expand_normal (arg0);
   15184           35 :       op1 = expand_normal (arg1);
   15185           35 :       op2 = expand_normal (arg2);
   15186           35 :       mode0 = insn_data[icode].operand[0].mode;
   15187           35 :       mode1 = insn_data[icode].operand[1].mode;
   15188           35 :       mode2 = insn_data[icode].operand[2].mode;
   15189              : 
   15190           35 :       op0 = ix86_zero_extend_to_Pmode (op0);
   15191           35 :       op0 = gen_rtx_MEM (mode1, op0);
   15192              : 
   15193           35 :       if (!insn_data[icode].operand[0].predicate (op0, mode0))
   15194            0 :         op0 = copy_to_mode_reg (mode0, op0);
   15195           35 :       if (!insn_data[icode].operand[1].predicate (op1, mode1))
   15196            2 :         op1 = copy_to_mode_reg (mode1, op1);
   15197           35 :       if (!insn_data[icode].operand[2].predicate (op2, mode2))
   15198            2 :         op2 = copy_to_mode_reg (mode2, op2);
   15199           35 :       pat = GEN_FCN (icode) (op0, op1, op2);
   15200           35 :       if (! pat)
   15201        56483 :         return 0;
   15202           35 :       emit_insn (pat);
   15203           35 :       return 0;
   15204              : 
   15205        21943 :     case IX86_BUILTIN_LDMXCSR:
   15206        21943 :       op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
   15207        21943 :       target = assign_stack_temp (SImode, GET_MODE_SIZE (SImode));
   15208        21943 :       emit_move_insn (target, op0);
   15209        21943 :       emit_insn (gen_sse_ldmxcsr (target));
   15210        21943 :       return 0;
   15211              : 
   15212        14717 :     case IX86_BUILTIN_STMXCSR:
   15213        14717 :       target = assign_stack_temp (SImode, GET_MODE_SIZE (SImode));
   15214        14717 :       emit_insn (gen_sse_stmxcsr (target));
   15215        14717 :       return copy_to_mode_reg (SImode, target);
   15216              : 
   15217           11 :     case IX86_BUILTIN_CLFLUSH:
   15218           11 :         arg0 = CALL_EXPR_ARG (exp, 0);
   15219           11 :         op0 = expand_normal (arg0);
   15220           11 :         icode = CODE_FOR_sse2_clflush;
   15221           11 :         if (!insn_data[icode].operand[0].predicate (op0, Pmode))
   15222            5 :           op0 = ix86_zero_extend_to_Pmode (op0);
   15223              : 
   15224           11 :         emit_insn (gen_sse2_clflush (op0));
   15225           11 :         return 0;
   15226              : 
   15227           19 :     case IX86_BUILTIN_CLWB:
   15228           19 :         arg0 = CALL_EXPR_ARG (exp, 0);
   15229           19 :         op0 = expand_normal (arg0);
   15230           19 :         icode = CODE_FOR_clwb;
   15231           19 :         if (!insn_data[icode].operand[0].predicate (op0, Pmode))
   15232            9 :           op0 = ix86_zero_extend_to_Pmode (op0);
   15233              : 
   15234           19 :         emit_insn (gen_clwb (op0));
   15235           19 :         return 0;
   15236              : 
   15237           19 :     case IX86_BUILTIN_CLFLUSHOPT:
   15238           19 :         arg0 = CALL_EXPR_ARG (exp, 0);
   15239           19 :         op0 = expand_normal (arg0);
   15240           19 :         icode = CODE_FOR_clflushopt;
   15241           19 :         if (!insn_data[icode].operand[0].predicate (op0, Pmode))
   15242            9 :           op0 = ix86_zero_extend_to_Pmode (op0);
   15243              : 
   15244           19 :         emit_insn (gen_clflushopt (op0));
   15245           19 :         return 0;
   15246              : 
   15247           47 :     case IX86_BUILTIN_MONITOR:
   15248           47 :     case IX86_BUILTIN_MONITORX:
   15249           47 :       arg0 = CALL_EXPR_ARG (exp, 0);
   15250           47 :       arg1 = CALL_EXPR_ARG (exp, 1);
   15251           47 :       arg2 = CALL_EXPR_ARG (exp, 2);
   15252           47 :       op0 = expand_normal (arg0);
   15253           47 :       op1 = expand_normal (arg1);
   15254           47 :       op2 = expand_normal (arg2);
   15255           47 :       if (!REG_P (op0))
   15256           19 :         op0 = ix86_zero_extend_to_Pmode (op0);
   15257           47 :       if (!REG_P (op1))
   15258           22 :         op1 = copy_to_mode_reg (SImode, op1);
   15259           47 :       if (!REG_P (op2))
   15260           25 :         op2 = copy_to_mode_reg (SImode, op2);
   15261              : 
   15262           47 :       emit_insn (fcode == IX86_BUILTIN_MONITOR
   15263           26 :                  ? gen_sse3_monitor (Pmode, op0, op1, op2)
   15264           21 :                  : gen_monitorx (Pmode, op0, op1, op2));
   15265           47 :       return 0;
   15266              : 
   15267           25 :     case IX86_BUILTIN_MWAIT:
   15268           25 :       arg0 = CALL_EXPR_ARG (exp, 0);
   15269           25 :       arg1 = CALL_EXPR_ARG (exp, 1);
   15270           25 :       op0 = expand_normal (arg0);
   15271           25 :       op1 = expand_normal (arg1);
   15272           25 :       if (!REG_P (op0))
   15273           13 :         op0 = copy_to_mode_reg (SImode, op0);
   15274           25 :       if (!REG_P (op1))
   15275           11 :         op1 = copy_to_mode_reg (SImode, op1);
   15276           25 :       emit_insn (gen_sse3_mwait (op0, op1));
   15277           25 :       return 0;
   15278              : 
   15279           21 :     case IX86_BUILTIN_MWAITX:
   15280           21 :       arg0 = CALL_EXPR_ARG (exp, 0);
   15281           21 :       arg1 = CALL_EXPR_ARG (exp, 1);
   15282           21 :       arg2 = CALL_EXPR_ARG (exp, 2);
   15283           21 :       op0 = expand_normal (arg0);
   15284           21 :       op1 = expand_normal (arg1);
   15285           21 :       op2 = expand_normal (arg2);
   15286           21 :       if (!REG_P (op0))
   15287           11 :         op0 = copy_to_mode_reg (SImode, op0);
   15288           21 :       if (!REG_P (op1))
   15289           10 :         op1 = copy_to_mode_reg (SImode, op1);
   15290           21 :       if (!REG_P (op2))
   15291           11 :         op2 = copy_to_mode_reg (SImode, op2);
   15292           21 :       emit_insn (gen_mwaitx (op0, op1, op2));
   15293           21 :       return 0;
   15294              : 
   15295           21 :     case IX86_BUILTIN_UMONITOR:
   15296           21 :       arg0 = CALL_EXPR_ARG (exp, 0);
   15297           21 :       op0 = expand_normal (arg0);
   15298              : 
   15299           21 :       op0 = ix86_zero_extend_to_Pmode (op0);
   15300           21 :       emit_insn (gen_umonitor (Pmode, op0));
   15301           21 :       return 0;
   15302              : 
   15303           42 :     case IX86_BUILTIN_UMWAIT:
   15304           42 :     case IX86_BUILTIN_TPAUSE:
   15305           42 :       arg0 = CALL_EXPR_ARG (exp, 0);
   15306           42 :       arg1 = CALL_EXPR_ARG (exp, 1);
   15307           42 :       op0 = expand_normal (arg0);
   15308           42 :       op1 = expand_normal (arg1);
   15309              : 
   15310           42 :       if (!REG_P (op0))
   15311           20 :         op0 = copy_to_mode_reg (SImode, op0);
   15312              : 
   15313           42 :       op1 = force_reg (DImode, op1);
   15314              : 
   15315           42 :       if (TARGET_64BIT)
   15316              :         {
   15317           42 :           op2 = expand_simple_binop (DImode, LSHIFTRT, op1, GEN_INT (32),
   15318              :                                      NULL, 1, OPTAB_DIRECT);
   15319           42 :           switch (fcode)
   15320              :             {
   15321              :             case IX86_BUILTIN_UMWAIT:
   15322              :               icode = CODE_FOR_umwait_rex64;
   15323              :               break;
   15324           21 :             case IX86_BUILTIN_TPAUSE:
   15325           21 :               icode = CODE_FOR_tpause_rex64;
   15326           21 :               break;
   15327            0 :             default:
   15328            0 :               gcc_unreachable ();
   15329              :             }
   15330              : 
   15331           42 :           op2 = gen_lowpart (SImode, op2);
   15332           42 :           op1 = gen_lowpart (SImode, op1);
   15333           42 :           pat = GEN_FCN (icode) (op0, op1, op2);
   15334              :         }
   15335              :       else
   15336              :         {
   15337            0 :           switch (fcode)
   15338              :             {
   15339              :             case IX86_BUILTIN_UMWAIT:
   15340              :               icode = CODE_FOR_umwait;
   15341              :               break;
   15342            0 :             case IX86_BUILTIN_TPAUSE:
   15343            0 :               icode = CODE_FOR_tpause;
   15344            0 :               break;
   15345            0 :             default:
   15346            0 :               gcc_unreachable ();
   15347              :             }
   15348            0 :           pat = GEN_FCN (icode) (op0, op1);
   15349              :         }
   15350              : 
   15351           42 :       if (!pat)
   15352              :         return 0;
   15353              : 
   15354           42 :       emit_insn (pat);
   15355              : 
   15356           42 :       if (target == 0
   15357           42 :           || !register_operand (target, QImode))
   15358            0 :         target = gen_reg_rtx (QImode);
   15359              : 
   15360           42 :       pat = gen_rtx_EQ (QImode, gen_rtx_REG (CCCmode, FLAGS_REG),
   15361              :                         const0_rtx);
   15362           42 :       emit_insn (gen_rtx_SET (target, pat));
   15363              : 
   15364           42 :       return target;
   15365              : 
   15366           20 :     case IX86_BUILTIN_TESTUI:
   15367           20 :       emit_insn (gen_testui ());
   15368              : 
   15369           20 :       if (target == 0
   15370           20 :           || !register_operand (target, QImode))
   15371            0 :         target = gen_reg_rtx (QImode);
   15372              : 
   15373           20 :       pat = gen_rtx_LTU (QImode, gen_rtx_REG (CCCmode, FLAGS_REG),
   15374              :                          const0_rtx);
   15375           20 :       emit_insn (gen_rtx_SET (target, pat));
   15376              : 
   15377           20 :       return target;
   15378              : 
   15379           19 :     case IX86_BUILTIN_CLZERO:
   15380           19 :       arg0 = CALL_EXPR_ARG (exp, 0);
   15381           19 :       op0 = expand_normal (arg0);
   15382           19 :       if (!REG_P (op0))
   15383            9 :         op0 = ix86_zero_extend_to_Pmode (op0);
   15384           19 :       emit_insn (gen_clzero (Pmode, op0));
   15385           19 :       return 0;
   15386              : 
   15387           19 :     case IX86_BUILTIN_CLDEMOTE:
   15388           19 :       arg0 = CALL_EXPR_ARG (exp, 0);
   15389           19 :       op0 = expand_normal (arg0);
   15390           19 :       icode = CODE_FOR_cldemote;
   15391           19 :       if (!insn_data[icode].operand[0].predicate (op0, Pmode))
   15392            9 :         op0 = ix86_zero_extend_to_Pmode (op0);
   15393              : 
   15394           19 :       emit_insn (gen_cldemote (op0));
   15395           19 :       return 0;
   15396              : 
   15397           11 :     case IX86_BUILTIN_LOADIWKEY:
   15398           11 :       {
   15399           11 :         arg0 = CALL_EXPR_ARG (exp, 0);
   15400           11 :         arg1 = CALL_EXPR_ARG (exp, 1);
   15401           11 :         arg2 = CALL_EXPR_ARG (exp, 2);
   15402           11 :         arg3 = CALL_EXPR_ARG (exp, 3);
   15403              : 
   15404           11 :         op0 = expand_normal (arg0);
   15405           11 :         op1 = expand_normal (arg1);
   15406           11 :         op2 = expand_normal (arg2);
   15407           11 :         op3 = expand_normal (arg3);
   15408              : 
   15409           11 :         if (!REG_P (op0))
   15410            5 :           op0 = copy_to_mode_reg (V2DImode, op0);
   15411           11 :         if (!REG_P (op1))
   15412            5 :           op1 = copy_to_mode_reg (V2DImode, op1);
   15413           11 :         if (!REG_P (op2))
   15414            5 :           op2 = copy_to_mode_reg (V2DImode, op2);
   15415           11 :         if (!REG_P (op3))
   15416            5 :           op3 = copy_to_mode_reg (SImode, op3);
   15417              : 
   15418           11 :         emit_insn (gen_loadiwkey (op0, op1, op2, op3));
   15419              : 
   15420           11 :         return 0;
   15421              :       }
   15422              : 
   15423           12 :     case IX86_BUILTIN_AESDEC128KLU8:
   15424           12 :       icode = CODE_FOR_aesdec128klu8;
   15425           12 :       goto aesdecenc_expand;
   15426              : 
   15427           12 :     case IX86_BUILTIN_AESDEC256KLU8:
   15428           12 :       icode = CODE_FOR_aesdec256klu8;
   15429           12 :       goto aesdecenc_expand;
   15430              : 
   15431           12 :     case IX86_BUILTIN_AESENC128KLU8:
   15432           12 :       icode = CODE_FOR_aesenc128klu8;
   15433           12 :       goto aesdecenc_expand;
   15434              : 
   15435              :     case IX86_BUILTIN_AESENC256KLU8:
   15436              :       icode = CODE_FOR_aesenc256klu8;
   15437              : 
   15438           48 :     aesdecenc_expand:
   15439              : 
   15440           48 :       arg0 = CALL_EXPR_ARG (exp, 0); // __m128i *odata
   15441           48 :       arg1 = CALL_EXPR_ARG (exp, 1); // __m128i idata
   15442           48 :       arg2 = CALL_EXPR_ARG (exp, 2); // const void *p
   15443              : 
   15444           48 :       op0 = expand_normal (arg0);
   15445           48 :       op1 = expand_normal (arg1);
   15446           48 :       op2 = expand_normal (arg2);
   15447              : 
   15448           48 :       if (!address_operand (op0, V2DImode))
   15449              :         {
   15450           16 :           op0 = convert_memory_address (Pmode, op0);
   15451           16 :           op0 = copy_addr_to_reg (op0);
   15452              :         }
   15453           48 :       op0 = gen_rtx_MEM (V2DImode, op0);
   15454              : 
   15455           48 :       if (!REG_P (op1))
   15456           20 :         op1 = copy_to_mode_reg (V2DImode, op1);
   15457              : 
   15458           48 :       if (!address_operand (op2, VOIDmode))
   15459              :         {
   15460           16 :           op2 = convert_memory_address (Pmode, op2);
   15461           16 :           op2 = copy_addr_to_reg (op2);
   15462              :         }
   15463           48 :       op2 = gen_rtx_MEM (BLKmode, op2);
   15464              : 
   15465           48 :       emit_insn (GEN_FCN (icode) (op1, op1, op2));
   15466              : 
   15467           48 :       if (target == 0)
   15468            4 :         target = gen_reg_rtx (QImode);
   15469              : 
   15470              :       /* NB: For aesenc/aesdec keylocker insn, ZF will be set when runtime
   15471              :          error occurs. Then the output should be cleared for safety. */
   15472           48 :       rtx_code_label *ok_label;
   15473           48 :       rtx tmp;
   15474              : 
   15475           48 :       tmp = gen_rtx_REG (CCZmode, FLAGS_REG);
   15476           48 :       pat = gen_rtx_EQ (QImode, tmp, const0_rtx);
   15477           48 :       ok_label = gen_label_rtx ();
   15478           48 :       emit_cmp_and_jump_insns (tmp, const0_rtx, NE, 0, GET_MODE (tmp),
   15479              :                                true, ok_label);
   15480              :       /* Usually the runtime error seldom occur, so predict OK path as
   15481              :          hotspot to optimize it as fallthrough block. */
   15482           48 :       predict_jump (REG_BR_PROB_BASE * 90 / 100);
   15483              : 
   15484           48 :       emit_insn (gen_rtx_SET (op1, const0_rtx));
   15485              : 
   15486           48 :       emit_label (ok_label);
   15487           48 :       emit_insn (gen_rtx_SET (target, pat));
   15488           48 :       emit_insn (gen_rtx_SET (op0, op1));
   15489              : 
   15490           48 :       return target;
   15491              : 
   15492           11 :     case IX86_BUILTIN_AESDECWIDE128KLU8:
   15493           11 :       icode = CODE_FOR_aesdecwide128klu8;
   15494           11 :       goto wideaesdecenc_expand;
   15495              : 
   15496           11 :     case IX86_BUILTIN_AESDECWIDE256KLU8:
   15497           11 :       icode = CODE_FOR_aesdecwide256klu8;
   15498           11 :       goto wideaesdecenc_expand;
   15499              : 
   15500           11 :     case IX86_BUILTIN_AESENCWIDE128KLU8:
   15501           11 :       icode = CODE_FOR_aesencwide128klu8;
   15502           11 :       goto wideaesdecenc_expand;
   15503              : 
   15504              :     case IX86_BUILTIN_AESENCWIDE256KLU8:
   15505              :       icode = CODE_FOR_aesencwide256klu8;
   15506              : 
   15507           44 :     wideaesdecenc_expand:
   15508              : 
   15509           44 :       rtx xmm_regs[8];
   15510           44 :       rtx op;
   15511              : 
   15512           44 :       arg0 = CALL_EXPR_ARG (exp, 0); // __m128i * odata
   15513           44 :       arg1 = CALL_EXPR_ARG (exp, 1); // const __m128i * idata
   15514           44 :       arg2 = CALL_EXPR_ARG (exp, 2); // const void *p
   15515              : 
   15516           44 :       op0 = expand_normal (arg0);
   15517           44 :       op1 = expand_normal (arg1);
   15518           44 :       op2 = expand_normal (arg2);
   15519              : 
   15520           44 :       if (GET_MODE (op1) != Pmode)
   15521            0 :         op1 = convert_to_mode (Pmode, op1, 1);
   15522              : 
   15523           44 :       if (!address_operand (op2, VOIDmode))
   15524              :         {
   15525           16 :           op2 = convert_memory_address (Pmode, op2);
   15526           16 :           op2 = copy_addr_to_reg (op2);
   15527              :         }
   15528           44 :       op2 = gen_rtx_MEM (BLKmode, op2);
   15529              : 
   15530          440 :       for (i = 0; i < 8; i++)
   15531              :         {
   15532          352 :           xmm_regs[i] = gen_rtx_REG (V2DImode, GET_SSE_REGNO (i));
   15533              : 
   15534          352 :           op = gen_rtx_MEM (V2DImode,
   15535          352 :                             plus_constant (Pmode, op1, (i * 16)));
   15536              : 
   15537          352 :           emit_move_insn (xmm_regs[i], op);
   15538              :         }
   15539              : 
   15540           44 :       emit_insn (GEN_FCN (icode) (op2));
   15541              : 
   15542           44 :       if (target == 0)
   15543            0 :         target = gen_reg_rtx (QImode);
   15544              : 
   15545           44 :       tmp = gen_rtx_REG (CCZmode, FLAGS_REG);
   15546           44 :       pat = gen_rtx_EQ (QImode, tmp, const0_rtx);
   15547           44 :       ok_label = gen_label_rtx ();
   15548           44 :       emit_cmp_and_jump_insns (tmp, const0_rtx, NE, 0, GET_MODE (tmp),
   15549              :                                true, ok_label);
   15550           44 :       predict_jump (REG_BR_PROB_BASE * 90 / 100);
   15551              : 
   15552          440 :       for (i = 0; i < 8; i++)
   15553          352 :         emit_insn (gen_rtx_SET (xmm_regs[i], const0_rtx));
   15554              : 
   15555           44 :       emit_label (ok_label);
   15556           44 :       emit_insn (gen_rtx_SET (target, pat));
   15557              : 
   15558           44 :       if (GET_MODE (op0) != Pmode)
   15559            0 :         op0 = convert_to_mode (Pmode, op0, 1);
   15560              : 
   15561          396 :       for (i = 0; i < 8; i++)
   15562              :         {
   15563          352 :           op = gen_rtx_MEM (V2DImode,
   15564          352 :                             plus_constant (Pmode, op0, (i * 16)));
   15565          352 :           emit_move_insn (op, xmm_regs[i]);
   15566              :         }
   15567              : 
   15568              :       return target;
   15569              : 
   15570           13 :     case IX86_BUILTIN_ENCODEKEY128U32:
   15571           13 :       {
   15572           13 :         rtx op, xmm_regs[7];
   15573              : 
   15574           13 :         arg0 = CALL_EXPR_ARG (exp, 0); // unsigned int htype
   15575           13 :         arg1 = CALL_EXPR_ARG (exp, 1); // __m128i key
   15576           13 :         arg2 = CALL_EXPR_ARG (exp, 2); // void *h
   15577              : 
   15578           13 :         op0 = expand_normal (arg0);
   15579           13 :         op1 = expand_normal (arg1);
   15580           13 :         op2 = expand_normal (arg2);
   15581              : 
   15582           13 :         if (!REG_P (op0))
   15583            7 :           op0 = copy_to_mode_reg (SImode, op0);
   15584              : 
   15585           13 :         if (GET_MODE (op2) != Pmode)
   15586            1 :           op2 = convert_to_mode (Pmode, op2, 1);
   15587              : 
   15588           13 :         op = gen_rtx_REG (V2DImode, GET_SSE_REGNO (0));
   15589           13 :         emit_move_insn (op, op1);
   15590              : 
   15591           65 :         for (i = 0; i < 3; i++)
   15592           39 :           xmm_regs[i] = gen_rtx_REG (V2DImode, GET_SSE_REGNO (i));
   15593              : 
   15594           13 :         if (target == 0 || !register_operand (target, SImode))
   15595            2 :           target = gen_reg_rtx (SImode);
   15596              : 
   15597           13 :         emit_insn (gen_encodekey128u32 (target, op0));
   15598              : 
   15599           65 :         for (i = 0; i < 3; i++)
   15600              :           {
   15601           39 :             op = gen_rtx_MEM (V2DImode,
   15602           39 :                               plus_constant (Pmode, op2, (i * 16)));
   15603           39 :             emit_move_insn (op, xmm_regs[i]);
   15604              :           }
   15605              : 
   15606           13 :         return target;
   15607              :       }
   15608           13 :     case IX86_BUILTIN_ENCODEKEY256U32:
   15609           13 :       {
   15610           13 :         rtx op, xmm_regs[7];
   15611              : 
   15612           13 :         arg0 = CALL_EXPR_ARG (exp, 0); // unsigned int htype
   15613           13 :         arg1 = CALL_EXPR_ARG (exp, 1); // __m128i keylow
   15614           13 :         arg2 = CALL_EXPR_ARG (exp, 2); // __m128i keyhi
   15615           13 :         arg3 = CALL_EXPR_ARG (exp, 3); // void *h
   15616              : 
   15617           13 :         op0 = expand_normal (arg0);
   15618           13 :         op1 = expand_normal (arg1);
   15619           13 :         op2 = expand_normal (arg2);
   15620           13 :         op3 = expand_normal (arg3);
   15621              : 
   15622           13 :         if (!REG_P (op0))
   15623            7 :           op0 = copy_to_mode_reg (SImode, op0);
   15624              : 
   15625           13 :         if (GET_MODE (op3) != Pmode)
   15626            1 :           op3 = convert_to_mode (Pmode, op3, 1);
   15627              : 
   15628              :         /* Force to use xmm0, xmm1 for keylow, keyhi*/
   15629           13 :         op = gen_rtx_REG (V2DImode, GET_SSE_REGNO (0));
   15630           13 :         emit_move_insn (op, op1);
   15631           13 :         op = gen_rtx_REG (V2DImode, GET_SSE_REGNO (1));
   15632           13 :         emit_move_insn (op, op2);
   15633              : 
   15634           78 :         for (i = 0; i < 4; i++)
   15635           52 :           xmm_regs[i] = gen_rtx_REG (V2DImode, GET_SSE_REGNO (i));
   15636              : 
   15637           13 :         if (target == 0 || !register_operand (target, SImode))
   15638            2 :           target = gen_reg_rtx (SImode);
   15639              : 
   15640           13 :         emit_insn (gen_encodekey256u32 (target, op0));
   15641              : 
   15642           78 :         for (i = 0; i < 4; i++)
   15643              :           {
   15644           52 :             op = gen_rtx_MEM (V2DImode,
   15645           52 :                               plus_constant (Pmode, op3, (i * 16)));
   15646           52 :             emit_move_insn (op, xmm_regs[i]);
   15647              :           }
   15648              : 
   15649           13 :         return target;
   15650              :       }
   15651              : 
   15652           48 :     case IX86_BUILTIN_PREFETCH:
   15653           48 :       {
   15654           48 :         arg0 = CALL_EXPR_ARG (exp, 0); // const void *
   15655           48 :         arg1 = CALL_EXPR_ARG (exp, 1); // const int
   15656           48 :         arg2 = CALL_EXPR_ARG (exp, 2); // const int
   15657           48 :         arg3 = CALL_EXPR_ARG (exp, 3); // const int
   15658              : 
   15659           48 :         op0 = expand_normal (arg0);
   15660           48 :         op1 = expand_normal (arg1);
   15661           48 :         op2 = expand_normal (arg2);
   15662           48 :         op3 = expand_normal (arg3);
   15663              : 
   15664           48 :         if (!CONST_INT_P (op1) || !CONST_INT_P (op2) || !CONST_INT_P (op3))
   15665              :           {
   15666            0 :             error ("second, third and fourth argument must be a const");
   15667            0 :             return const0_rtx;
   15668              :           }
   15669              : 
   15670           48 :         if (!IN_RANGE (INTVAL (op1), 0, 2))
   15671              :           {
   15672            1 :             warning (0, "invalid second argument to"
   15673              :                      " %<__builtin_ia32_prefetch%>; using zero");
   15674            1 :             op1 = const0_rtx;
   15675              :           }
   15676              : 
   15677           48 :         if (INTVAL (op3) == 1)
   15678              :           {
   15679            4 :             if (!IN_RANGE (INTVAL (op2), 2, 3))
   15680              :               {
   15681            1 :                 error ("invalid third argument");
   15682            1 :                 return const0_rtx;
   15683              :               }
   15684              : 
   15685            3 :             if (TARGET_64BIT && TARGET_PREFETCHI
   15686            6 :                 && local_func_symbolic_operand (op0, GET_MODE (op0)))
   15687            2 :               emit_insn (gen_prefetchi (op0, op2));
   15688              :             else
   15689              :               {
   15690            1 :                 warning (0, "instruction prefetch applies when in 64-bit mode"
   15691              :                             " with RIP-relative addressing and"
   15692              :                             " option %<-mprefetchi%>;"
   15693              :                             " they stay NOPs otherwise");
   15694            1 :                 emit_insn (gen_nop ());
   15695              :               }
   15696              :           }
   15697              :         else
   15698              :           {
   15699           44 :             if (INTVAL (op3) != 0)
   15700            1 :               warning (0, "invalid fourth argument to"
   15701              :                           " %<__builtin_ia32_prefetch%>; using zero");
   15702              : 
   15703           44 :             if (!address_operand (op0, VOIDmode))
   15704              :               {
   15705           10 :                 op0 = convert_memory_address (Pmode, op0);
   15706           10 :                 op0 = copy_addr_to_reg (op0);
   15707              :               }
   15708              : 
   15709           44 :             if (!IN_RANGE (INTVAL (op2), 0, 3))
   15710              :               {
   15711            1 :                 warning (0, "invalid third argument to %<__builtin_ia32_prefetch%>; using zero");
   15712            1 :                 op2 = const0_rtx;
   15713              :               }
   15714              : 
   15715           44 :             if (TARGET_3DNOW
   15716           26 :                 || TARGET_PREFETCH_SSE
   15717            0 :                 || TARGET_PRFCHW
   15718            0 :                 || TARGET_MOVRS)
   15719           44 :               emit_insn (gen_prefetch (op0, op1, op2));
   15720            0 :             else if (!MEM_P (op0) && side_effects_p (op0))
   15721              :               /* Don't do anything with direct references to volatile memory,
   15722              :                  but generate code to handle other side effects.  */
   15723            0 :               emit_insn (op0);
   15724              :           }
   15725              : 
   15726              :         return 0;
   15727              :       }
   15728              : 
   15729           21 :     case IX86_BUILTIN_PREFETCHI:
   15730           21 :       {
   15731           21 :         arg0 = CALL_EXPR_ARG (exp, 0); // const void *
   15732           21 :         arg1 = CALL_EXPR_ARG (exp, 1); // const int
   15733              : 
   15734           21 :         op0 = expand_normal (arg0);
   15735           21 :         op1 = expand_normal (arg1);
   15736              : 
   15737           21 :         if (!CONST_INT_P (op1))
   15738              :           {
   15739            0 :             error ("second argument must be a const");
   15740            0 :             return const0_rtx;
   15741              :           }
   15742              : 
   15743              :         /* GOT/PLT_PIC should not be available for instruction prefetch.
   15744              :            It must be real instruction address.  */
   15745           21 :         if (TARGET_64BIT
   15746           21 :             && local_func_symbolic_operand (op0, GET_MODE (op0)))
   15747            4 :           emit_insn (gen_prefetchi (op0, op1));
   15748              :         else
   15749              :           {
   15750              :             /* Ignore the hint.  */
   15751           17 :             warning (0, "instruction prefetch applies when in 64-bit mode"
   15752              :                         " with RIP-relative addressing and"
   15753              :                         " option %<-mprefetchi%>;"
   15754              :                         " they stay NOPs otherwise");
   15755           17 :             emit_insn (gen_nop ());
   15756              :           }
   15757              : 
   15758              :         return 0;
   15759              :       }
   15760              : 
   15761           53 :     case IX86_BUILTIN_URDMSR:
   15762           53 :     case IX86_BUILTIN_UWRMSR:
   15763           53 :       {
   15764           53 :         arg0 = CALL_EXPR_ARG (exp, 0);
   15765           53 :         op0 = expand_normal (arg0);
   15766              : 
   15767           53 :         if (CONST_INT_P (op0))
   15768              :           {
   15769           12 :             unsigned HOST_WIDE_INT val = UINTVAL (op0);
   15770           12 :             if (val > 0xffffffff)
   15771            2 :               op0 = force_reg (DImode, op0);
   15772              :           }
   15773              :         else
   15774           41 :           op0 = force_reg (DImode, op0);
   15775              : 
   15776           53 :         if (fcode == IX86_BUILTIN_UWRMSR)
   15777              :           {
   15778           26 :             arg1 = CALL_EXPR_ARG (exp, 1);
   15779           26 :             op1 = expand_normal (arg1);
   15780           26 :             op1 = force_reg (DImode, op1);
   15781           26 :             icode = CODE_FOR_uwrmsr;
   15782           26 :             target = 0;
   15783              :           }
   15784              :         else
   15785              :           {
   15786           27 :             if (target == 0 || !register_operand (target, DImode))
   15787            1 :               target = gen_reg_rtx (DImode);
   15788              :             icode = CODE_FOR_urdmsr;
   15789              :             op1 = op0;
   15790              :             op0 = target;
   15791              :           }
   15792           53 :         emit_insn (GEN_FCN (icode) (op0, op1));
   15793           53 :         return target;
   15794              :       }
   15795              : 
   15796           13 :     case IX86_BUILTIN_BSR0INIT:
   15797           13 :       {
   15798           13 :         target = gen_rtx_REG (V32SImode, BSR0_REG);
   15799           13 :         emit_insn (gen_bsrinit (target));
   15800           13 :         return 0;
   15801              :       }
   15802              : 
   15803           11 :     case IX86_BUILTIN_BSR0MOVF:
   15804           11 :       {
   15805           11 :         arg0 = CALL_EXPR_ARG (exp, 0);
   15806           11 :         arg1 = CALL_EXPR_ARG (exp, 1);
   15807           11 :         op0 = expand_normal (arg0);
   15808           11 :         op1 = expand_normal (arg1);
   15809              : 
   15810           11 :         target = gen_rtx_REG (V32SImode, BSR0_REG);
   15811           11 :         if (CONST_VECTOR_P (op0) || MEM_P (op0))
   15812            1 :           op0 = force_reg (V16SImode, op0);
   15813           11 :         if (CONST_VECTOR_P (op1))
   15814            0 :           op1 = force_reg (V16SImode, op1);
   15815           11 :         emit_insn (gen_bsrmovf (target, op0, op1));
   15816           11 :         return 0;
   15817              :       }
   15818              : 
   15819           22 :     case IX86_BUILTIN_BSR0MOVHINSERT:
   15820           22 :     case IX86_BUILTIN_BSR0MOVLINSERT:
   15821           22 :       {
   15822           22 :         arg0 = CALL_EXPR_ARG (exp, 0);
   15823           22 :         op0 = expand_normal (arg0);
   15824              : 
   15825           22 :         if (fcode == IX86_BUILTIN_BSR0MOVHINSERT)
   15826              :           icode = CODE_FOR_bsrmovh_load;
   15827              :         else
   15828           11 :           icode = CODE_FOR_bsrmovl_load;
   15829           22 :         target = gen_rtx_REG (V32SImode, BSR0_REG);
   15830           22 :         if (CONST_VECTOR_P (op0))
   15831            0 :           op0 = force_reg (V16SImode, op0);
   15832           22 :         emit_insn (GEN_FCN (icode) (target, op0));
   15833           22 :         return 0;
   15834              :       }
   15835              : 
   15836           22 :     case IX86_BUILTIN_BSR0MOVHEXTRACT:
   15837           22 :     case IX86_BUILTIN_BSR0MOVLEXTRACT:
   15838           22 :       {
   15839           22 :         op0 = gen_rtx_REG (V32SImode, BSR0_REG);
   15840           22 :         if (fcode == IX86_BUILTIN_BSR0MOVHEXTRACT)
   15841              :           icode = CODE_FOR_bsrmovh_store;
   15842              :         else
   15843           11 :           icode = CODE_FOR_bsrmovl_store;
   15844           22 :         if (target == 0 || !register_operand (target, V16SImode))
   15845            0 :           target = gen_reg_rtx (V16SImode);
   15846           22 :         emit_insn (GEN_FCN (icode) (target, op0));
   15847           22 :         return target;
   15848              :       }
   15849              : 
   15850          229 :     case IX86_BUILTIN_VEC_INIT_V2SI:
   15851          229 :     case IX86_BUILTIN_VEC_INIT_V4HI:
   15852          229 :     case IX86_BUILTIN_VEC_INIT_V8QI:
   15853          229 :       return ix86_expand_vec_init_builtin (TREE_TYPE (exp), exp, target);
   15854              : 
   15855          400 :     case IX86_BUILTIN_VEC_EXT_V2DF:
   15856          400 :     case IX86_BUILTIN_VEC_EXT_V2DI:
   15857          400 :     case IX86_BUILTIN_VEC_EXT_V4SF:
   15858          400 :     case IX86_BUILTIN_VEC_EXT_V4SI:
   15859          400 :     case IX86_BUILTIN_VEC_EXT_V8HI:
   15860          400 :     case IX86_BUILTIN_VEC_EXT_V2SI:
   15861          400 :     case IX86_BUILTIN_VEC_EXT_V4HI:
   15862          400 :     case IX86_BUILTIN_VEC_EXT_V16QI:
   15863          400 :       return ix86_expand_vec_ext_builtin (exp, target);
   15864              : 
   15865          204 :     case IX86_BUILTIN_VEC_SET_V2DI:
   15866          204 :     case IX86_BUILTIN_VEC_SET_V4SF:
   15867          204 :     case IX86_BUILTIN_VEC_SET_V4SI:
   15868          204 :     case IX86_BUILTIN_VEC_SET_V8HI:
   15869          204 :     case IX86_BUILTIN_VEC_SET_V4HI:
   15870          204 :     case IX86_BUILTIN_VEC_SET_V16QI:
   15871          204 :       return ix86_expand_vec_set_builtin (exp);
   15872              : 
   15873            0 :     case IX86_BUILTIN_NANQ:
   15874            0 :     case IX86_BUILTIN_NANSQ:
   15875            0 :       return expand_call (exp, target, ignore);
   15876              : 
   15877           18 :     case IX86_BUILTIN_RDPID:
   15878              : 
   15879           18 :       op0 = gen_reg_rtx (word_mode);
   15880              : 
   15881           18 :       if (TARGET_64BIT)
   15882              :         {
   15883           18 :           insn = gen_rdpid_rex64 (op0);
   15884           18 :           op0 = convert_to_mode (SImode, op0, 1);
   15885              :         }
   15886              :       else
   15887            0 :         insn = gen_rdpid (op0);
   15888              : 
   15889           18 :       emit_insn (insn);
   15890              : 
   15891           18 :       if (target == 0
   15892           18 :           || !register_operand (target, SImode))
   15893            0 :         target = gen_reg_rtx (SImode);
   15894              : 
   15895           18 :       emit_move_insn (target, op0);
   15896           18 :       return target;
   15897              : 
   15898           75 :     case IX86_BUILTIN_2INTERSECTD512:
   15899           75 :     case IX86_BUILTIN_2INTERSECTQ512:
   15900           75 :     case IX86_BUILTIN_2INTERSECTD256:
   15901           75 :     case IX86_BUILTIN_2INTERSECTQ256:
   15902           75 :     case IX86_BUILTIN_2INTERSECTD128:
   15903           75 :     case IX86_BUILTIN_2INTERSECTQ128:
   15904           75 :       arg0 = CALL_EXPR_ARG (exp, 0);
   15905           75 :       arg1 = CALL_EXPR_ARG (exp, 1);
   15906           75 :       arg2 = CALL_EXPR_ARG (exp, 2);
   15907           75 :       arg3 = CALL_EXPR_ARG (exp, 3);
   15908           75 :       op0 = expand_normal (arg0);
   15909           75 :       op1 = expand_normal (arg1);
   15910           75 :       op2 = expand_normal (arg2);
   15911           75 :       op3 = expand_normal (arg3);
   15912              : 
   15913           75 :       if (!address_operand (op0, VOIDmode))
   15914              :         {
   15915           25 :           op0 = convert_memory_address (Pmode, op0);
   15916           25 :           op0 = copy_addr_to_reg (op0);
   15917              :         }
   15918           75 :       if (!address_operand (op1, VOIDmode))
   15919              :         {
   15920           25 :           op1 = convert_memory_address (Pmode, op1);
   15921           25 :           op1 = copy_addr_to_reg (op1);
   15922              :         }
   15923              : 
   15924           75 :       switch (fcode)
   15925              :         {
   15926              :         case IX86_BUILTIN_2INTERSECTD512:
   15927              :           mode4 = P2HImode;
   15928              :           icode = CODE_FOR_avx512vp2intersect_2intersectv16si;
   15929              :           break;
   15930              :         case IX86_BUILTIN_2INTERSECTQ512:
   15931              :           mode4 = P2QImode;
   15932              :           icode = CODE_FOR_avx512vp2intersect_2intersectv8di;
   15933              :           break;
   15934              :         case IX86_BUILTIN_2INTERSECTD256:
   15935              :           mode4 = P2QImode;
   15936              :           icode = CODE_FOR_avx512vp2intersect_2intersectv8si;
   15937              :           break;
   15938              :         case IX86_BUILTIN_2INTERSECTQ256:
   15939              :           mode4 = P2QImode;
   15940              :           icode = CODE_FOR_avx512vp2intersect_2intersectv4di;
   15941              :           break;
   15942              :         case IX86_BUILTIN_2INTERSECTD128:
   15943              :           mode4 = P2QImode;
   15944              :           icode = CODE_FOR_avx512vp2intersect_2intersectv4si;
   15945              :           break;
   15946              :         case IX86_BUILTIN_2INTERSECTQ128:
   15947              :           mode4 = P2QImode;
   15948              :           icode = CODE_FOR_avx512vp2intersect_2intersectv2di;
   15949              :           break;
   15950            0 :         default:
   15951            0 :           gcc_unreachable ();
   15952              :         }
   15953              : 
   15954           75 :       mode2 = insn_data[icode].operand[1].mode;
   15955           75 :       mode3 = insn_data[icode].operand[2].mode;
   15956           75 :       if (!insn_data[icode].operand[1].predicate (op2, mode2))
   15957           25 :         op2 = copy_to_mode_reg (mode2, op2);
   15958           75 :       if (!insn_data[icode].operand[2].predicate (op3, mode3))
   15959            6 :         op3 = copy_to_mode_reg (mode3, op3);
   15960              : 
   15961           75 :       op4 = gen_reg_rtx (mode4);
   15962           75 :       emit_insn (GEN_FCN (icode) (op4, op2, op3));
   15963           75 :       mode0 = mode4 == P2HImode ? HImode : QImode;
   15964           75 :       emit_move_insn (gen_rtx_MEM (mode0, op0),
   15965           75 :                       gen_lowpart (mode0, op4));
   15966           75 :       emit_move_insn (gen_rtx_MEM (mode0, op1),
   15967              :                       gen_highpart (mode0, op4));
   15968              : 
   15969           75 :       return 0;
   15970              : 
   15971          102 :     case IX86_BUILTIN_RDPMC:
   15972          102 :     case IX86_BUILTIN_RDTSC:
   15973          102 :     case IX86_BUILTIN_RDTSCP:
   15974          102 :     case IX86_BUILTIN_XGETBV:
   15975              : 
   15976          102 :       op0 = gen_reg_rtx (DImode);
   15977          102 :       op1 = gen_reg_rtx (DImode);
   15978              : 
   15979          102 :       if (fcode == IX86_BUILTIN_RDPMC)
   15980              :         {
   15981           22 :           arg0 = CALL_EXPR_ARG (exp, 0);
   15982           22 :           op2 = expand_normal (arg0);
   15983           22 :           if (!register_operand (op2, SImode))
   15984           11 :             op2 = copy_to_mode_reg (SImode, op2);
   15985              : 
   15986           22 :           insn = (TARGET_64BIT
   15987           22 :                   ? gen_rdpmc_rex64 (op0, op1, op2)
   15988            0 :                   : gen_rdpmc (op0, op2));
   15989           22 :           emit_insn (insn);
   15990              :         }
   15991           80 :       else if (fcode == IX86_BUILTIN_XGETBV)
   15992              :         {
   15993           22 :           arg0 = CALL_EXPR_ARG (exp, 0);
   15994           22 :           op2 = expand_normal (arg0);
   15995           22 :           if (!register_operand (op2, SImode))
   15996            1 :             op2 = copy_to_mode_reg (SImode, op2);
   15997              : 
   15998           22 :           insn = (TARGET_64BIT
   15999           22 :                   ? gen_xgetbv_rex64 (op0, op1, op2)
   16000            0 :                   : gen_xgetbv (op0, op2));
   16001           22 :           emit_insn (insn);
   16002              :         }
   16003           58 :       else if (fcode == IX86_BUILTIN_RDTSC)
   16004              :         {
   16005           36 :           insn = (TARGET_64BIT
   16006           36 :                   ? gen_rdtsc_rex64 (op0, op1)
   16007            2 :                   : gen_rdtsc (op0));
   16008           36 :           emit_insn (insn);
   16009              :         }
   16010              :       else
   16011              :         {
   16012           22 :           op2 = gen_reg_rtx (SImode);
   16013              : 
   16014           22 :           insn = (TARGET_64BIT
   16015           22 :                   ? gen_rdtscp_rex64 (op0, op1, op2)
   16016            0 :                   : gen_rdtscp (op0, op2));
   16017           22 :           emit_insn (insn);
   16018              : 
   16019           22 :           arg0 = CALL_EXPR_ARG (exp, 0);
   16020           22 :           op4 = expand_normal (arg0);
   16021           22 :           if (!address_operand (op4, VOIDmode))
   16022              :             {
   16023           10 :               op4 = convert_memory_address (Pmode, op4);
   16024           10 :               op4 = copy_addr_to_reg (op4);
   16025              :             }
   16026           22 :           emit_move_insn (gen_rtx_MEM (SImode, op4), op2);
   16027              :         }
   16028              : 
   16029          102 :       if (target == 0
   16030          102 :           || !register_operand (target, DImode))
   16031           10 :         target = gen_reg_rtx (DImode);
   16032              : 
   16033          102 :       if (TARGET_64BIT)
   16034              :         {
   16035          100 :           op1 = expand_simple_binop (DImode, ASHIFT, op1, GEN_INT (32),
   16036              :                                      op1, 1, OPTAB_DIRECT);
   16037          100 :           op0 = expand_simple_binop (DImode, IOR, op0, op1,
   16038              :                                      op0, 1, OPTAB_DIRECT);
   16039              :         }
   16040              : 
   16041          102 :       emit_move_insn (target, op0);
   16042          102 :       return target;
   16043              : 
   16044           62 :     case IX86_BUILTIN_ENQCMD:
   16045           62 :     case IX86_BUILTIN_ENQCMDS:
   16046           62 :     case IX86_BUILTIN_MOVDIR64B:
   16047              : 
   16048           62 :       arg0 = CALL_EXPR_ARG (exp, 0);
   16049           62 :       arg1 = CALL_EXPR_ARG (exp, 1);
   16050           62 :       op0 = expand_normal (arg0);
   16051           62 :       op1 = expand_normal (arg1);
   16052              : 
   16053           62 :       op0 = ix86_zero_extend_to_Pmode (op0);
   16054           62 :       if (!address_operand (op1, VOIDmode))
   16055              :       {
   16056           29 :         op1 = convert_memory_address (Pmode, op1);
   16057           29 :         op1 = copy_addr_to_reg (op1);
   16058              :       }
   16059           62 :       op1 = gen_rtx_MEM (XImode, op1);
   16060              : 
   16061           62 :       if (fcode == IX86_BUILTIN_MOVDIR64B)
   16062              :         {
   16063           25 :           emit_insn (gen_movdir64b (Pmode, op0, op1));
   16064           24 :           return 0;
   16065              :         }
   16066              :       else
   16067              :         {
   16068           38 :           if (target == 0
   16069           38 :               || !register_operand (target, SImode))
   16070            0 :             target = gen_reg_rtx (SImode);
   16071              : 
   16072           38 :           emit_move_insn (target, const0_rtx);
   16073           38 :           target = gen_rtx_SUBREG (QImode, target, 0);
   16074              : 
   16075           19 :           int unspecv = (fcode == IX86_BUILTIN_ENQCMD
   16076           38 :                          ? UNSPECV_ENQCMD
   16077              :                          : UNSPECV_ENQCMDS);
   16078           38 :           icode = code_for_enqcmd (unspecv, Pmode);
   16079           38 :           emit_insn (GEN_FCN (icode) (op0, op1));
   16080              : 
   16081           38 :           emit_insn
   16082           38 :             (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
   16083              :                           gen_rtx_fmt_ee (EQ, QImode,
   16084              :                                           gen_rtx_REG (CCZmode, FLAGS_REG),
   16085              :                                           const0_rtx)));
   16086           38 :           return SUBREG_REG (target);
   16087              :         }
   16088              : 
   16089        14707 :     case IX86_BUILTIN_FXSAVE:
   16090        14707 :     case IX86_BUILTIN_FXRSTOR:
   16091        14707 :     case IX86_BUILTIN_FXSAVE64:
   16092        14707 :     case IX86_BUILTIN_FXRSTOR64:
   16093        14707 :     case IX86_BUILTIN_FNSTENV:
   16094        14707 :     case IX86_BUILTIN_FLDENV:
   16095        14707 :       mode0 = BLKmode;
   16096        14707 :       switch (fcode)
   16097              :         {
   16098              :         case IX86_BUILTIN_FXSAVE:
   16099              :           icode = CODE_FOR_fxsave;
   16100              :           break;
   16101           19 :         case IX86_BUILTIN_FXRSTOR:
   16102           19 :           icode = CODE_FOR_fxrstor;
   16103           19 :           break;
   16104           23 :         case IX86_BUILTIN_FXSAVE64:
   16105           23 :           icode = CODE_FOR_fxsave64;
   16106           23 :           break;
   16107           21 :         case IX86_BUILTIN_FXRSTOR64:
   16108           21 :           icode = CODE_FOR_fxrstor64;
   16109           21 :           break;
   16110         7260 :         case IX86_BUILTIN_FNSTENV:
   16111         7260 :           icode = CODE_FOR_fnstenv;
   16112         7260 :           break;
   16113         7364 :         case IX86_BUILTIN_FLDENV:
   16114         7364 :           icode = CODE_FOR_fldenv;
   16115         7364 :           break;
   16116            0 :         default:
   16117            0 :           gcc_unreachable ();
   16118              :         }
   16119              : 
   16120        14707 :       arg0 = CALL_EXPR_ARG (exp, 0);
   16121        14707 :       op0 = expand_normal (arg0);
   16122              : 
   16123        14707 :       if (!address_operand (op0, VOIDmode))
   16124              :         {
   16125           36 :           op0 = convert_memory_address (Pmode, op0);
   16126           36 :           op0 = copy_addr_to_reg (op0);
   16127              :         }
   16128        14707 :       op0 = gen_rtx_MEM (mode0, op0);
   16129              : 
   16130        14707 :       pat = GEN_FCN (icode) (op0);
   16131        14707 :       if (pat)
   16132        14707 :         emit_insn (pat);
   16133              :       return 0;
   16134              : 
   16135           21 :     case IX86_BUILTIN_XSETBV:
   16136           21 :       arg0 = CALL_EXPR_ARG (exp, 0);
   16137           21 :       arg1 = CALL_EXPR_ARG (exp, 1);
   16138           21 :       op0 = expand_normal (arg0);
   16139           21 :       op1 = expand_normal (arg1);
   16140              : 
   16141           21 :       if (!REG_P (op0))
   16142            1 :         op0 = copy_to_mode_reg (SImode, op0);
   16143              : 
   16144           21 :       op1 = force_reg (DImode, op1);
   16145              : 
   16146           21 :       if (TARGET_64BIT)
   16147              :         {
   16148           21 :           op2 = expand_simple_binop (DImode, LSHIFTRT, op1, GEN_INT (32),
   16149              :                                      NULL, 1, OPTAB_DIRECT);
   16150              : 
   16151           21 :           icode = CODE_FOR_xsetbv_rex64;
   16152              : 
   16153           21 :           op2 = gen_lowpart (SImode, op2);
   16154           21 :           op1 = gen_lowpart (SImode, op1);
   16155           21 :           pat = GEN_FCN (icode) (op0, op1, op2);
   16156              :         }
   16157              :       else
   16158              :         {
   16159            0 :           icode = CODE_FOR_xsetbv;
   16160              : 
   16161            0 :           pat = GEN_FCN (icode) (op0, op1);
   16162              :         }
   16163           21 :       if (pat)
   16164           21 :         emit_insn (pat);
   16165              :       return 0;
   16166              : 
   16167          232 :     case IX86_BUILTIN_XSAVE:
   16168          232 :     case IX86_BUILTIN_XRSTOR:
   16169          232 :     case IX86_BUILTIN_XSAVE64:
   16170          232 :     case IX86_BUILTIN_XRSTOR64:
   16171          232 :     case IX86_BUILTIN_XSAVEOPT:
   16172          232 :     case IX86_BUILTIN_XSAVEOPT64:
   16173          232 :     case IX86_BUILTIN_XSAVES:
   16174          232 :     case IX86_BUILTIN_XRSTORS:
   16175          232 :     case IX86_BUILTIN_XSAVES64:
   16176          232 :     case IX86_BUILTIN_XRSTORS64:
   16177          232 :     case IX86_BUILTIN_XSAVEC:
   16178          232 :     case IX86_BUILTIN_XSAVEC64:
   16179          232 :       arg0 = CALL_EXPR_ARG (exp, 0);
   16180          232 :       arg1 = CALL_EXPR_ARG (exp, 1);
   16181          232 :       op0 = expand_normal (arg0);
   16182          232 :       op1 = expand_normal (arg1);
   16183              : 
   16184          232 :       if (!address_operand (op0, VOIDmode))
   16185              :         {
   16186          108 :           op0 = convert_memory_address (Pmode, op0);
   16187          108 :           op0 = copy_addr_to_reg (op0);
   16188              :         }
   16189          232 :       op0 = gen_rtx_MEM (BLKmode, op0);
   16190              : 
   16191          232 :       op1 = force_reg (DImode, op1);
   16192              : 
   16193          232 :       if (TARGET_64BIT)
   16194              :         {
   16195          232 :           op2 = expand_simple_binop (DImode, LSHIFTRT, op1, GEN_INT (32),
   16196              :                                      NULL, 1, OPTAB_DIRECT);
   16197          232 :           switch (fcode)
   16198              :             {
   16199              :             case IX86_BUILTIN_XSAVE:
   16200              :               icode = CODE_FOR_xsave_rex64;
   16201              :               break;
   16202           19 :             case IX86_BUILTIN_XRSTOR:
   16203           19 :               icode = CODE_FOR_xrstor_rex64;
   16204           19 :               break;
   16205           21 :             case IX86_BUILTIN_XSAVE64:
   16206           21 :               icode = CODE_FOR_xsave64;
   16207           21 :               break;
   16208           21 :             case IX86_BUILTIN_XRSTOR64:
   16209           21 :               icode = CODE_FOR_xrstor64;
   16210           21 :               break;
   16211           19 :             case IX86_BUILTIN_XSAVEOPT:
   16212           19 :               icode = CODE_FOR_xsaveopt_rex64;
   16213           19 :               break;
   16214           19 :             case IX86_BUILTIN_XSAVEOPT64:
   16215           19 :               icode = CODE_FOR_xsaveopt64;
   16216           19 :               break;
   16217           19 :             case IX86_BUILTIN_XSAVES:
   16218           19 :               icode = CODE_FOR_xsaves_rex64;
   16219           19 :               break;
   16220           19 :             case IX86_BUILTIN_XRSTORS:
   16221           19 :               icode = CODE_FOR_xrstors_rex64;
   16222           19 :               break;
   16223           19 :             case IX86_BUILTIN_XSAVES64:
   16224           19 :               icode = CODE_FOR_xsaves64;
   16225           19 :               break;
   16226           19 :             case IX86_BUILTIN_XRSTORS64:
   16227           19 :               icode = CODE_FOR_xrstors64;
   16228           19 :               break;
   16229           19 :             case IX86_BUILTIN_XSAVEC:
   16230           19 :               icode = CODE_FOR_xsavec_rex64;
   16231           19 :               break;
   16232           19 :             case IX86_BUILTIN_XSAVEC64:
   16233           19 :               icode = CODE_FOR_xsavec64;
   16234           19 :               break;
   16235            0 :             default:
   16236            0 :               gcc_unreachable ();
   16237              :             }
   16238              : 
   16239          232 :           op2 = gen_lowpart (SImode, op2);
   16240          232 :           op1 = gen_lowpart (SImode, op1);
   16241          232 :           pat = GEN_FCN (icode) (op0, op1, op2);
   16242              :         }
   16243              :       else
   16244              :         {
   16245            0 :           switch (fcode)
   16246              :             {
   16247              :             case IX86_BUILTIN_XSAVE:
   16248              :               icode = CODE_FOR_xsave;
   16249              :               break;
   16250              :             case IX86_BUILTIN_XRSTOR:
   16251              :               icode = CODE_FOR_xrstor;
   16252              :               break;
   16253              :             case IX86_BUILTIN_XSAVEOPT:
   16254              :               icode = CODE_FOR_xsaveopt;
   16255              :               break;
   16256              :             case IX86_BUILTIN_XSAVES:
   16257              :               icode = CODE_FOR_xsaves;
   16258              :               break;
   16259              :             case IX86_BUILTIN_XRSTORS:
   16260              :               icode = CODE_FOR_xrstors;
   16261              :               break;
   16262              :             case IX86_BUILTIN_XSAVEC:
   16263              :               icode = CODE_FOR_xsavec;
   16264              :               break;
   16265            0 :             default:
   16266            0 :               gcc_unreachable ();
   16267              :             }
   16268            0 :           pat = GEN_FCN (icode) (op0, op1);
   16269              :         }
   16270              : 
   16271          232 :       if (pat)
   16272          232 :         emit_insn (pat);
   16273              :       return 0;
   16274              : 
   16275          139 :     case IX86_BUILTIN_LDTILECFG:
   16276          139 :     case IX86_BUILTIN_STTILECFG:
   16277          139 :       arg0 = CALL_EXPR_ARG (exp, 0);
   16278          139 :       op0 = expand_normal (arg0);
   16279              : 
   16280          139 :       if (!address_operand (op0, VOIDmode))
   16281              :         {
   16282            8 :           op0 = convert_memory_address (Pmode, op0);
   16283            8 :           op0 = copy_addr_to_reg (op0);
   16284              :         }
   16285          139 :       op0 = gen_rtx_MEM (BLKmode, op0);
   16286          139 :       if (fcode == IX86_BUILTIN_LDTILECFG)
   16287              :         icode = CODE_FOR_ldtilecfg;
   16288              :       else
   16289           89 :         icode = CODE_FOR_sttilecfg;
   16290          139 :       pat = GEN_FCN (icode) (op0);
   16291          139 :       emit_insn (pat);
   16292          139 :       return 0;
   16293              : 
   16294           18 :     case IX86_BUILTIN_LLWPCB:
   16295           18 :       arg0 = CALL_EXPR_ARG (exp, 0);
   16296           18 :       op0 = expand_normal (arg0);
   16297              : 
   16298           18 :       if (!register_operand (op0, Pmode))
   16299            9 :         op0 = ix86_zero_extend_to_Pmode (op0);
   16300           18 :       emit_insn (gen_lwp_llwpcb (Pmode, op0));
   16301           18 :       return 0;
   16302              : 
   16303           18 :     case IX86_BUILTIN_SLWPCB:
   16304           18 :       if (!target
   16305           18 :           || !register_operand (target, Pmode))
   16306            0 :         target = gen_reg_rtx (Pmode);
   16307           18 :       emit_insn (gen_lwp_slwpcb (Pmode, target));
   16308           18 :       return target;
   16309              : 
   16310           51 :     case IX86_BUILTIN_LWPVAL32:
   16311           51 :     case IX86_BUILTIN_LWPVAL64:
   16312           51 :     case IX86_BUILTIN_LWPINS32:
   16313           51 :     case IX86_BUILTIN_LWPINS64:
   16314           51 :       mode = ((fcode == IX86_BUILTIN_LWPVAL32
   16315           51 :                || fcode == IX86_BUILTIN_LWPINS32)
   16316           51 :               ? SImode : DImode);
   16317              : 
   16318           51 :       if (fcode == IX86_BUILTIN_LWPVAL32
   16319           51 :           || fcode == IX86_BUILTIN_LWPVAL64)
   16320           26 :         icode = code_for_lwp_lwpval (mode);
   16321              :       else
   16322           25 :         icode = code_for_lwp_lwpins (mode);
   16323              : 
   16324           51 :       arg0 = CALL_EXPR_ARG (exp, 0);
   16325           51 :       arg1 = CALL_EXPR_ARG (exp, 1);
   16326           51 :       arg2 = CALL_EXPR_ARG (exp, 2);
   16327           51 :       op0 = expand_normal (arg0);
   16328           51 :       op1 = expand_normal (arg1);
   16329           51 :       op2 = expand_normal (arg2);
   16330           51 :       mode0 = insn_data[icode].operand[0].mode;
   16331              : 
   16332           51 :       if (!insn_data[icode].operand[0].predicate (op0, mode0))
   16333           13 :         op0 = copy_to_mode_reg (mode0, op0);
   16334           51 :       if (!insn_data[icode].operand[1].predicate (op1, SImode))
   16335            0 :         op1 = copy_to_mode_reg (SImode, op1);
   16336              : 
   16337           51 :       if (!CONST_INT_P (op2))
   16338              :         {
   16339            0 :           error ("the last argument must be a 32-bit immediate");
   16340            0 :           return const0_rtx;
   16341              :         }
   16342              : 
   16343           51 :       emit_insn (GEN_FCN (icode) (op0, op1, op2));
   16344              : 
   16345           51 :       if (fcode == IX86_BUILTIN_LWPINS32
   16346           51 :           || fcode == IX86_BUILTIN_LWPINS64)
   16347              :         {
   16348           25 :           if (target == 0
   16349           25 :               || !nonimmediate_operand (target, QImode))
   16350            0 :             target = gen_reg_rtx (QImode);
   16351              : 
   16352           25 :           pat = gen_rtx_EQ (QImode, gen_rtx_REG (CCCmode, FLAGS_REG),
   16353              :                             const0_rtx);
   16354           25 :           emit_insn (gen_rtx_SET (target, pat));
   16355              : 
   16356           25 :           return target;
   16357              :         }
   16358              :       else
   16359              :         return 0;
   16360              : 
   16361           18 :     case IX86_BUILTIN_BEXTRI32:
   16362           18 :     case IX86_BUILTIN_BEXTRI64:
   16363           18 :       mode = (fcode == IX86_BUILTIN_BEXTRI32 ? SImode : DImode);
   16364              : 
   16365           18 :       arg0 = CALL_EXPR_ARG (exp, 0);
   16366           18 :       arg1 = CALL_EXPR_ARG (exp, 1);
   16367           18 :       op0 = expand_normal (arg0);
   16368           18 :       op1 = expand_normal (arg1);
   16369              : 
   16370           18 :       if (!CONST_INT_P (op1))
   16371              :         {
   16372            0 :           error ("last argument must be an immediate");
   16373            0 :           return const0_rtx;
   16374              :         }
   16375              :       else
   16376              :         {
   16377           18 :           unsigned char lsb_index = UINTVAL (op1);
   16378           18 :           unsigned char length = UINTVAL (op1) >> 8;
   16379              : 
   16380           18 :           unsigned char bitsize = GET_MODE_BITSIZE (mode);
   16381              : 
   16382           18 :           icode = code_for_tbm_bextri (mode);
   16383              : 
   16384           18 :           mode1 = insn_data[icode].operand[1].mode;
   16385           18 :           if (!insn_data[icode].operand[1].predicate (op0, mode1))
   16386           12 :             op0 = copy_to_mode_reg (mode1, op0);
   16387              : 
   16388           18 :           mode0 = insn_data[icode].operand[0].mode;
   16389           18 :           if (target == 0
   16390           18 :               || !register_operand (target, mode0))
   16391            0 :             target = gen_reg_rtx (mode0);
   16392              : 
   16393           18 :           if (length == 0 || lsb_index >= bitsize)
   16394              :             {
   16395            8 :               emit_move_insn (target, const0_rtx);
   16396            8 :               return target;
   16397              :             }
   16398              : 
   16399           10 :           if (length + lsb_index > bitsize)
   16400            5 :             length = bitsize - lsb_index;
   16401              : 
   16402           10 :           op1 = GEN_INT (length);
   16403           10 :           op2 = GEN_INT (lsb_index);
   16404              : 
   16405           10 :           emit_insn (GEN_FCN (icode) (target, op0, op1, op2));
   16406           10 :           return target;
   16407              :         }
   16408              : 
   16409           21 :     case IX86_BUILTIN_RDRAND16_STEP:
   16410           21 :       mode = HImode;
   16411           21 :       goto rdrand_step;
   16412              : 
   16413           42 :     case IX86_BUILTIN_RDRAND32_STEP:
   16414           42 :       mode = SImode;
   16415           42 :       goto rdrand_step;
   16416              : 
   16417              :     case IX86_BUILTIN_RDRAND64_STEP:
   16418              :       mode = DImode;
   16419              : 
   16420           84 : rdrand_step:
   16421           84 :       arg0 = CALL_EXPR_ARG (exp, 0);
   16422           84 :       op1 = expand_normal (arg0);
   16423           84 :       if (!address_operand (op1, VOIDmode))
   16424              :         {
   16425           29 :           op1 = convert_memory_address (Pmode, op1);
   16426           29 :           op1 = copy_addr_to_reg (op1);
   16427              :         }
   16428              : 
   16429           84 :       op0 = gen_reg_rtx (mode);
   16430           84 :       emit_insn (gen_rdrand (mode, op0));
   16431              : 
   16432           84 :       emit_move_insn (gen_rtx_MEM (mode, op1), op0);
   16433              : 
   16434           84 :       op1 = force_reg (SImode, const1_rtx);
   16435              : 
   16436              :       /* Emit SImode conditional move.  */
   16437           84 :       if (mode == HImode)
   16438              :         {
   16439           21 :           if (TARGET_ZERO_EXTEND_WITH_AND
   16440           21 :               && optimize_function_for_speed_p (cfun))
   16441              :             {
   16442            0 :               op2 = force_reg (SImode, const0_rtx);
   16443              : 
   16444            0 :               emit_insn (gen_movstricthi
   16445            0 :                          (gen_lowpart (HImode, op2), op0));
   16446              :             }
   16447              :           else
   16448              :             {
   16449           21 :               op2 = gen_reg_rtx (SImode);
   16450              : 
   16451           21 :               emit_insn (gen_zero_extendhisi2 (op2, op0));
   16452              :             }
   16453              :         }
   16454           63 :       else if (mode == SImode)
   16455              :         op2 = op0;
   16456              :       else
   16457           21 :         op2 = gen_rtx_SUBREG (SImode, op0, 0);
   16458              : 
   16459           84 :       if (target == 0
   16460           84 :           || !register_operand (target, SImode))
   16461            7 :         target = gen_reg_rtx (SImode);
   16462              : 
   16463           84 :       pat = gen_rtx_GEU (VOIDmode, gen_rtx_REG (CCCmode, FLAGS_REG),
   16464              :                          const0_rtx);
   16465           84 :       emit_insn (gen_rtx_SET (target,
   16466              :                               gen_rtx_IF_THEN_ELSE (SImode, pat, op2, op1)));
   16467           84 :       return target;
   16468              : 
   16469           19 :     case IX86_BUILTIN_RDSEED16_STEP:
   16470           19 :       mode = HImode;
   16471           19 :       goto rdseed_step;
   16472              : 
   16473           28 :     case IX86_BUILTIN_RDSEED32_STEP:
   16474           28 :       mode = SImode;
   16475           28 :       goto rdseed_step;
   16476              : 
   16477              :     case IX86_BUILTIN_RDSEED64_STEP:
   16478              :       mode = DImode;
   16479              : 
   16480           67 : rdseed_step:
   16481           67 :       arg0 = CALL_EXPR_ARG (exp, 0);
   16482           67 :       op1 = expand_normal (arg0);
   16483           67 :       if (!address_operand (op1, VOIDmode))
   16484              :         {
   16485           28 :           op1 = convert_memory_address (Pmode, op1);
   16486           28 :           op1 = copy_addr_to_reg (op1);
   16487              :         }
   16488              : 
   16489           67 :       op0 = gen_reg_rtx (mode);
   16490           67 :       emit_insn (gen_rdseed (mode, op0));
   16491              : 
   16492           67 :       emit_move_insn (gen_rtx_MEM (mode, op1), op0);
   16493              : 
   16494           67 :       op2 = gen_reg_rtx (QImode);
   16495              : 
   16496           67 :       pat = gen_rtx_LTU (QImode, gen_rtx_REG (CCCmode, FLAGS_REG),
   16497              :                          const0_rtx);
   16498           67 :       emit_insn (gen_rtx_SET (op2, pat));
   16499              : 
   16500           67 :       if (target == 0
   16501           67 :           || !register_operand (target, SImode))
   16502            1 :         target = gen_reg_rtx (SImode);
   16503              : 
   16504           67 :       emit_insn (gen_zero_extendqisi2 (target, op2));
   16505           67 :       return target;
   16506              : 
   16507           38 :     case IX86_BUILTIN_SBB32:
   16508           38 :       icode = CODE_FOR_subborrowsi;
   16509           38 :       icode2 = CODE_FOR_subborrowsi_0;
   16510           38 :       mode0 = SImode;
   16511           38 :       mode1 = DImode;
   16512           38 :       mode2 = CCmode;
   16513           38 :       goto handlecarry;
   16514              : 
   16515           45 :     case IX86_BUILTIN_SBB64:
   16516           45 :       icode = CODE_FOR_subborrowdi;
   16517           45 :       icode2 = CODE_FOR_subborrowdi_0;
   16518           45 :       mode0 = DImode;
   16519           45 :       mode1 = TImode;
   16520           45 :       mode2 = CCmode;
   16521           45 :       goto handlecarry;
   16522              : 
   16523           68 :     case IX86_BUILTIN_ADDCARRYX32:
   16524           68 :       icode = CODE_FOR_addcarrysi;
   16525           68 :       icode2 = CODE_FOR_addcarrysi_0;
   16526           68 :       mode0 = SImode;
   16527           68 :       mode1 = DImode;
   16528           68 :       mode2 = CCCmode;
   16529           68 :       goto handlecarry;
   16530              : 
   16531              :     case IX86_BUILTIN_ADDCARRYX64:
   16532              :       icode = CODE_FOR_addcarrydi;
   16533              :       icode2 = CODE_FOR_addcarrydi_0;
   16534              :       mode0 = DImode;
   16535              :       mode1 = TImode;
   16536              :       mode2 = CCCmode;
   16537              : 
   16538          215 :     handlecarry:
   16539          215 :       arg0 = CALL_EXPR_ARG (exp, 0); /* unsigned char c_in.  */
   16540          215 :       arg1 = CALL_EXPR_ARG (exp, 1); /* unsigned int src1.  */
   16541          215 :       arg2 = CALL_EXPR_ARG (exp, 2); /* unsigned int src2.  */
   16542          215 :       arg3 = CALL_EXPR_ARG (exp, 3); /* unsigned int *sum_out.  */
   16543              : 
   16544          215 :       op1 = expand_normal (arg0);
   16545              : 
   16546          215 :       op2 = expand_normal (arg1);
   16547          215 :       if (!register_operand (op2, mode0))
   16548          120 :         op2 = copy_to_mode_reg (mode0, op2);
   16549              : 
   16550          215 :       op3 = expand_normal (arg2);
   16551          215 :       if (!register_operand (op3, mode0))
   16552          123 :         op3 = copy_to_mode_reg (mode0, op3);
   16553              : 
   16554          215 :       op4 = expand_normal (arg3);
   16555          215 :       if (!address_operand (op4, VOIDmode))
   16556              :         {
   16557           68 :           op4 = convert_memory_address (Pmode, op4);
   16558           68 :           op4 = copy_addr_to_reg (op4);
   16559              :         }
   16560              : 
   16561          215 :       op0 = gen_reg_rtx (mode0);
   16562          215 :       if (op1 == const0_rtx)
   16563              :         {
   16564              :           /* If arg0 is 0, optimize right away into add or sub
   16565              :              instruction that sets CCCmode flags.  */
   16566           21 :           op1 = gen_rtx_REG (mode2, FLAGS_REG);
   16567           21 :           emit_insn (GEN_FCN (icode2) (op0, op2, op3));
   16568              :         }
   16569              :       else
   16570              :         {
   16571              :           /* Generate CF from input operand.  */
   16572          194 :           ix86_expand_carry (op1);
   16573              : 
   16574              :           /* Generate instruction that consumes CF.  */
   16575          194 :           op1 = gen_rtx_REG (CCCmode, FLAGS_REG);
   16576          194 :           pat = gen_rtx_LTU (mode1, op1, const0_rtx);
   16577          194 :           pat2 = gen_rtx_LTU (mode0, op1, const0_rtx);
   16578          194 :           emit_insn (GEN_FCN (icode) (op0, op2, op3, op1, pat, pat2));
   16579              :         }
   16580              : 
   16581              :       /* Return current CF value.  */
   16582          215 :       if (target == 0)
   16583           14 :         target = gen_reg_rtx (QImode);
   16584              : 
   16585          215 :       pat = gen_rtx_LTU (QImode, op1, const0_rtx);
   16586          215 :       emit_insn (gen_rtx_SET (target, pat));
   16587              : 
   16588              :       /* Store the result.  */
   16589          215 :       emit_move_insn (gen_rtx_MEM (mode0, op4), op0);
   16590              : 
   16591          215 :       return target;
   16592              : 
   16593           24 :     case IX86_BUILTIN_READ_FLAGS:
   16594           24 :       if (ignore)
   16595            3 :         return const0_rtx;
   16596              : 
   16597           21 :       emit_insn (gen_pushfl ());
   16598              : 
   16599           21 :       if (optimize
   16600           11 :           || target == NULL_RTX
   16601           11 :           || !nonimmediate_operand (target, word_mode)
   16602           32 :           || GET_MODE (target) != word_mode)
   16603           10 :         target = gen_reg_rtx (word_mode);
   16604              : 
   16605           21 :       emit_insn (gen_pop (target));
   16606           21 :       return target;
   16607              : 
   16608           21 :     case IX86_BUILTIN_WRITE_FLAGS:
   16609              : 
   16610           21 :       arg0 = CALL_EXPR_ARG (exp, 0);
   16611           21 :       op0 = expand_normal (arg0);
   16612           21 :       if (!general_no_elim_operand (op0, word_mode))
   16613            0 :         op0 = copy_to_mode_reg (word_mode, op0);
   16614              : 
   16615           21 :       emit_insn (gen_push (op0));
   16616           21 :       emit_insn (gen_popfl ());
   16617           21 :       return 0;
   16618              : 
   16619           22 :     case IX86_BUILTIN_KTESTC8:
   16620           22 :       icode = CODE_FOR_ktestqi;
   16621           22 :       mode3 = CCCmode;
   16622           22 :       goto kortest;
   16623              : 
   16624           22 :     case IX86_BUILTIN_KTESTZ8:
   16625           22 :       icode = CODE_FOR_ktestqi;
   16626           22 :       mode3 = CCZmode;
   16627           22 :       goto kortest;
   16628              : 
   16629           22 :     case IX86_BUILTIN_KTESTC16:
   16630           22 :       icode = CODE_FOR_ktesthi;
   16631           22 :       mode3 = CCCmode;
   16632           22 :       goto kortest;
   16633              : 
   16634           22 :     case IX86_BUILTIN_KTESTZ16:
   16635           22 :       icode = CODE_FOR_ktesthi;
   16636           22 :       mode3 = CCZmode;
   16637           22 :       goto kortest;
   16638              : 
   16639           22 :     case IX86_BUILTIN_KTESTC32:
   16640           22 :       icode = CODE_FOR_ktestsi;
   16641           22 :       mode3 = CCCmode;
   16642           22 :       goto kortest;
   16643              : 
   16644           22 :     case IX86_BUILTIN_KTESTZ32:
   16645           22 :       icode = CODE_FOR_ktestsi;
   16646           22 :       mode3 = CCZmode;
   16647           22 :       goto kortest;
   16648              : 
   16649           22 :     case IX86_BUILTIN_KTESTC64:
   16650           22 :       icode = CODE_FOR_ktestdi;
   16651           22 :       mode3 = CCCmode;
   16652           22 :       goto kortest;
   16653              : 
   16654           22 :     case IX86_BUILTIN_KTESTZ64:
   16655           22 :       icode = CODE_FOR_ktestdi;
   16656           22 :       mode3 = CCZmode;
   16657           22 :       goto kortest;
   16658              : 
   16659           22 :     case IX86_BUILTIN_KORTESTC8:
   16660           22 :       icode = CODE_FOR_kortestqi;
   16661           22 :       mode3 = CCCmode;
   16662           22 :       goto kortest;
   16663              : 
   16664           76 :     case IX86_BUILTIN_KORTESTZ8:
   16665           76 :       icode = CODE_FOR_kortestqi;
   16666           76 :       mode3 = CCZmode;
   16667           76 :       goto kortest;
   16668              : 
   16669           38 :     case IX86_BUILTIN_KORTESTC16:
   16670           38 :       icode = CODE_FOR_kortesthi;
   16671           38 :       mode3 = CCCmode;
   16672           38 :       goto kortest;
   16673              : 
   16674           91 :     case IX86_BUILTIN_KORTESTZ16:
   16675           91 :       icode = CODE_FOR_kortesthi;
   16676           91 :       mode3 = CCZmode;
   16677           91 :       goto kortest;
   16678              : 
   16679           22 :     case IX86_BUILTIN_KORTESTC32:
   16680           22 :       icode = CODE_FOR_kortestsi;
   16681           22 :       mode3 = CCCmode;
   16682           22 :       goto kortest;
   16683              : 
   16684           79 :     case IX86_BUILTIN_KORTESTZ32:
   16685           79 :       icode = CODE_FOR_kortestsi;
   16686           79 :       mode3 = CCZmode;
   16687           79 :       goto kortest;
   16688              : 
   16689           22 :     case IX86_BUILTIN_KORTESTC64:
   16690           22 :       icode = CODE_FOR_kortestdi;
   16691           22 :       mode3 = CCCmode;
   16692           22 :       goto kortest;
   16693              : 
   16694              :     case IX86_BUILTIN_KORTESTZ64:
   16695              :       icode = CODE_FOR_kortestdi;
   16696              :       mode3 = CCZmode;
   16697              : 
   16698          610 :     kortest:
   16699          610 :       arg0 = CALL_EXPR_ARG (exp, 0); /* Mask reg src1.  */
   16700          610 :       arg1 = CALL_EXPR_ARG (exp, 1); /* Mask reg src2.  */
   16701          610 :       op0 = expand_normal (arg0);
   16702          610 :       op1 = expand_normal (arg1);
   16703              : 
   16704          610 :       mode0 = insn_data[icode].operand[0].mode;
   16705          610 :       mode1 = insn_data[icode].operand[1].mode;
   16706              : 
   16707          610 :       if (GET_MODE (op0) != VOIDmode)
   16708          610 :         op0 = force_reg (GET_MODE (op0), op0);
   16709              : 
   16710          610 :       op0 = gen_lowpart (mode0, op0);
   16711              : 
   16712          610 :       if (!insn_data[icode].operand[0].predicate (op0, mode0))
   16713            0 :         op0 = copy_to_mode_reg (mode0, op0);
   16714              : 
   16715          610 :       if (GET_MODE (op1) != VOIDmode)
   16716          609 :         op1 = force_reg (GET_MODE (op1), op1);
   16717              : 
   16718          610 :       op1 = gen_lowpart (mode1, op1);
   16719              : 
   16720          610 :       if (!insn_data[icode].operand[1].predicate (op1, mode1))
   16721            1 :         op1 = copy_to_mode_reg (mode1, op1);
   16722              : 
   16723          610 :       target = gen_reg_rtx (QImode);
   16724              : 
   16725              :       /* Emit kortest.  */
   16726          610 :       emit_insn (GEN_FCN (icode) (op0, op1));
   16727              :       /* And use setcc to return result from flags.  */
   16728          610 :       ix86_expand_setcc (target, EQ,
   16729              :                          gen_rtx_REG (mode3, FLAGS_REG), const0_rtx);
   16730          610 :       return target;
   16731              : 
   16732           20 :     case IX86_BUILTIN_GATHERSIV2DF:
   16733           20 :       icode = CODE_FOR_avx2_gathersiv2df;
   16734           20 :       goto gather_gen;
   16735           18 :     case IX86_BUILTIN_GATHERSIV4DF:
   16736           18 :       icode = CODE_FOR_avx2_gathersiv4df;
   16737           18 :       goto gather_gen;
   16738           21 :     case IX86_BUILTIN_GATHERDIV2DF:
   16739           21 :       icode = CODE_FOR_avx2_gatherdiv2df;
   16740           21 :       goto gather_gen;
   16741           32 :     case IX86_BUILTIN_GATHERDIV4DF:
   16742           32 :       icode = CODE_FOR_avx2_gatherdiv4df;
   16743           32 :       goto gather_gen;
   16744           28 :     case IX86_BUILTIN_GATHERSIV4SF:
   16745           28 :       icode = CODE_FOR_avx2_gathersiv4sf;
   16746           28 :       goto gather_gen;
   16747           35 :     case IX86_BUILTIN_GATHERSIV8SF:
   16748           35 :       icode = CODE_FOR_avx2_gathersiv8sf;
   16749           35 :       goto gather_gen;
   16750           24 :     case IX86_BUILTIN_GATHERDIV4SF:
   16751           24 :       icode = CODE_FOR_avx2_gatherdiv4sf;
   16752           24 :       goto gather_gen;
   16753           18 :     case IX86_BUILTIN_GATHERDIV8SF:
   16754           18 :       icode = CODE_FOR_avx2_gatherdiv8sf;
   16755           18 :       goto gather_gen;
   16756           18 :     case IX86_BUILTIN_GATHERSIV2DI:
   16757           18 :       icode = CODE_FOR_avx2_gathersiv2di;
   16758           18 :       goto gather_gen;
   16759           18 :     case IX86_BUILTIN_GATHERSIV4DI:
   16760           18 :       icode = CODE_FOR_avx2_gathersiv4di;
   16761           18 :       goto gather_gen;
   16762           27 :     case IX86_BUILTIN_GATHERDIV2DI:
   16763           27 :       icode = CODE_FOR_avx2_gatherdiv2di;
   16764           27 :       goto gather_gen;
   16765           29 :     case IX86_BUILTIN_GATHERDIV4DI:
   16766           29 :       icode = CODE_FOR_avx2_gatherdiv4di;
   16767           29 :       goto gather_gen;
   16768           20 :     case IX86_BUILTIN_GATHERSIV4SI:
   16769           20 :       icode = CODE_FOR_avx2_gathersiv4si;
   16770           20 :       goto gather_gen;
   16771           22 :     case IX86_BUILTIN_GATHERSIV8SI:
   16772           22 :       icode = CODE_FOR_avx2_gathersiv8si;
   16773           22 :       goto gather_gen;
   16774           28 :     case IX86_BUILTIN_GATHERDIV4SI:
   16775           28 :       icode = CODE_FOR_avx2_gatherdiv4si;
   16776           28 :       goto gather_gen;
   16777           18 :     case IX86_BUILTIN_GATHERDIV8SI:
   16778           18 :       icode = CODE_FOR_avx2_gatherdiv8si;
   16779           18 :       goto gather_gen;
   16780           16 :     case IX86_BUILTIN_GATHERALTSIV4DF:
   16781           16 :       icode = CODE_FOR_avx2_gathersiv4df;
   16782           16 :       goto gather_gen;
   16783           16 :     case IX86_BUILTIN_GATHERALTDIV8SF:
   16784           16 :       icode = CODE_FOR_avx2_gatherdiv8sf;
   16785           16 :       goto gather_gen;
   16786            4 :     case IX86_BUILTIN_GATHERALTSIV4DI:
   16787            4 :       icode = CODE_FOR_avx2_gathersiv4di;
   16788            4 :       goto gather_gen;
   16789           12 :     case IX86_BUILTIN_GATHERALTDIV8SI:
   16790           12 :       icode = CODE_FOR_avx2_gatherdiv8si;
   16791           12 :       goto gather_gen;
   16792           36 :     case IX86_BUILTIN_GATHER3SIV16SF:
   16793           36 :       icode = CODE_FOR_avx512f_gathersiv16sf;
   16794           36 :       goto gather_gen;
   16795           24 :     case IX86_BUILTIN_GATHER3SIV8DF:
   16796           24 :       icode = CODE_FOR_avx512f_gathersiv8df;
   16797           24 :       goto gather_gen;
   16798           24 :     case IX86_BUILTIN_GATHER3DIV16SF:
   16799           24 :       icode = CODE_FOR_avx512f_gatherdiv16sf;
   16800           24 :       goto gather_gen;
   16801           37 :     case IX86_BUILTIN_GATHER3DIV8DF:
   16802           37 :       icode = CODE_FOR_avx512f_gatherdiv8df;
   16803           37 :       goto gather_gen;
   16804           30 :     case IX86_BUILTIN_GATHER3SIV16SI:
   16805           30 :       icode = CODE_FOR_avx512f_gathersiv16si;
   16806           30 :       goto gather_gen;
   16807           24 :     case IX86_BUILTIN_GATHER3SIV8DI:
   16808           24 :       icode = CODE_FOR_avx512f_gathersiv8di;
   16809           24 :       goto gather_gen;
   16810           24 :     case IX86_BUILTIN_GATHER3DIV16SI:
   16811           24 :       icode = CODE_FOR_avx512f_gatherdiv16si;
   16812           24 :       goto gather_gen;
   16813           38 :     case IX86_BUILTIN_GATHER3DIV8DI:
   16814           38 :       icode = CODE_FOR_avx512f_gatherdiv8di;
   16815           38 :       goto gather_gen;
   16816           16 :     case IX86_BUILTIN_GATHER3ALTSIV8DF:
   16817           16 :       icode = CODE_FOR_avx512f_gathersiv8df;
   16818           16 :       goto gather_gen;
   16819           22 :     case IX86_BUILTIN_GATHER3ALTDIV16SF:
   16820           22 :       icode = CODE_FOR_avx512f_gatherdiv16sf;
   16821           22 :       goto gather_gen;
   16822           14 :     case IX86_BUILTIN_GATHER3ALTSIV8DI:
   16823           14 :       icode = CODE_FOR_avx512f_gathersiv8di;
   16824           14 :       goto gather_gen;
   16825           18 :     case IX86_BUILTIN_GATHER3ALTDIV16SI:
   16826           18 :       icode = CODE_FOR_avx512f_gatherdiv16si;
   16827           18 :       goto gather_gen;
   16828           18 :     case IX86_BUILTIN_GATHER3SIV2DF:
   16829           18 :       icode = CODE_FOR_avx512vl_gathersiv2df;
   16830           18 :       goto gather_gen;
   16831           10 :     case IX86_BUILTIN_GATHER3SIV4DF:
   16832           10 :       icode = CODE_FOR_avx512vl_gathersiv4df;
   16833           10 :       goto gather_gen;
   16834           15 :     case IX86_BUILTIN_GATHER3DIV2DF:
   16835           15 :       icode = CODE_FOR_avx512vl_gatherdiv2df;
   16836           15 :       goto gather_gen;
   16837           16 :     case IX86_BUILTIN_GATHER3DIV4DF:
   16838           16 :       icode = CODE_FOR_avx512vl_gatherdiv4df;
   16839           16 :       goto gather_gen;
   16840           14 :     case IX86_BUILTIN_GATHER3SIV4SF:
   16841           14 :       icode = CODE_FOR_avx512vl_gathersiv4sf;
   16842           14 :       goto gather_gen;
   16843           12 :     case IX86_BUILTIN_GATHER3SIV8SF:
   16844           12 :       icode = CODE_FOR_avx512vl_gathersiv8sf;
   16845           12 :       goto gather_gen;
   16846           22 :     case IX86_BUILTIN_GATHER3DIV4SF:
   16847           22 :       icode = CODE_FOR_avx512vl_gatherdiv4sf;
   16848           22 :       goto gather_gen;
   16849           10 :     case IX86_BUILTIN_GATHER3DIV8SF:
   16850           10 :       icode = CODE_FOR_avx512vl_gatherdiv8sf;
   16851           10 :       goto gather_gen;
   16852           20 :     case IX86_BUILTIN_GATHER3SIV2DI:
   16853           20 :       icode = CODE_FOR_avx512vl_gathersiv2di;
   16854           20 :       goto gather_gen;
   16855           10 :     case IX86_BUILTIN_GATHER3SIV4DI:
   16856           10 :       icode = CODE_FOR_avx512vl_gathersiv4di;
   16857           10 :       goto gather_gen;
   16858           15 :     case IX86_BUILTIN_GATHER3DIV2DI:
   16859           15 :       icode = CODE_FOR_avx512vl_gatherdiv2di;
   16860           15 :       goto gather_gen;
   16861           14 :     case IX86_BUILTIN_GATHER3DIV4DI:
   16862           14 :       icode = CODE_FOR_avx512vl_gatherdiv4di;
   16863           14 :       goto gather_gen;
   16864           14 :     case IX86_BUILTIN_GATHER3SIV4SI:
   16865           14 :       icode = CODE_FOR_avx512vl_gathersiv4si;
   16866           14 :       goto gather_gen;
   16867           12 :     case IX86_BUILTIN_GATHER3SIV8SI:
   16868           12 :       icode = CODE_FOR_avx512vl_gathersiv8si;
   16869           12 :       goto gather_gen;
   16870           24 :     case IX86_BUILTIN_GATHER3DIV4SI:
   16871           24 :       icode = CODE_FOR_avx512vl_gatherdiv4si;
   16872           24 :       goto gather_gen;
   16873           10 :     case IX86_BUILTIN_GATHER3DIV8SI:
   16874           10 :       icode = CODE_FOR_avx512vl_gatherdiv8si;
   16875           10 :       goto gather_gen;
   16876            4 :     case IX86_BUILTIN_GATHER3ALTSIV4DF:
   16877            4 :       icode = CODE_FOR_avx512vl_gathersiv4df;
   16878            4 :       goto gather_gen;
   16879            8 :     case IX86_BUILTIN_GATHER3ALTDIV8SF:
   16880            8 :       icode = CODE_FOR_avx512vl_gatherdiv8sf;
   16881            8 :       goto gather_gen;
   16882            6 :     case IX86_BUILTIN_GATHER3ALTSIV4DI:
   16883            6 :       icode = CODE_FOR_avx512vl_gathersiv4di;
   16884            6 :       goto gather_gen;
   16885           10 :     case IX86_BUILTIN_GATHER3ALTDIV8SI:
   16886           10 :       icode = CODE_FOR_avx512vl_gatherdiv8si;
   16887           10 :       goto gather_gen;
   16888           40 :     case IX86_BUILTIN_SCATTERSIV16SF:
   16889           40 :       icode = CODE_FOR_avx512f_scattersiv16sf;
   16890           40 :       goto scatter_gen;
   16891           27 :     case IX86_BUILTIN_SCATTERSIV8DF:
   16892           27 :       icode = CODE_FOR_avx512f_scattersiv8df;
   16893           27 :       goto scatter_gen;
   16894           24 :     case IX86_BUILTIN_SCATTERDIV16SF:
   16895           24 :       icode = CODE_FOR_avx512f_scatterdiv16sf;
   16896           24 :       goto scatter_gen;
   16897           33 :     case IX86_BUILTIN_SCATTERDIV8DF:
   16898           33 :       icode = CODE_FOR_avx512f_scatterdiv8df;
   16899           33 :       goto scatter_gen;
   16900           30 :     case IX86_BUILTIN_SCATTERSIV16SI:
   16901           30 :       icode = CODE_FOR_avx512f_scattersiv16si;
   16902           30 :       goto scatter_gen;
   16903           24 :     case IX86_BUILTIN_SCATTERSIV8DI:
   16904           24 :       icode = CODE_FOR_avx512f_scattersiv8di;
   16905           24 :       goto scatter_gen;
   16906           24 :     case IX86_BUILTIN_SCATTERDIV16SI:
   16907           24 :       icode = CODE_FOR_avx512f_scatterdiv16si;
   16908           24 :       goto scatter_gen;
   16909           29 :     case IX86_BUILTIN_SCATTERDIV8DI:
   16910           29 :       icode = CODE_FOR_avx512f_scatterdiv8di;
   16911           29 :       goto scatter_gen;
   16912           18 :     case IX86_BUILTIN_SCATTERSIV8SF:
   16913           18 :       icode = CODE_FOR_avx512vl_scattersiv8sf;
   16914           18 :       goto scatter_gen;
   16915           20 :     case IX86_BUILTIN_SCATTERSIV4SF:
   16916           20 :       icode = CODE_FOR_avx512vl_scattersiv4sf;
   16917           20 :       goto scatter_gen;
   16918           16 :     case IX86_BUILTIN_SCATTERSIV4DF:
   16919           16 :       icode = CODE_FOR_avx512vl_scattersiv4df;
   16920           16 :       goto scatter_gen;
   16921           16 :     case IX86_BUILTIN_SCATTERSIV2DF:
   16922           16 :       icode = CODE_FOR_avx512vl_scattersiv2df;
   16923           16 :       goto scatter_gen;
   16924           16 :     case IX86_BUILTIN_SCATTERDIV8SF:
   16925           16 :       icode = CODE_FOR_avx512vl_scatterdiv8sf;
   16926           16 :       goto scatter_gen;
   16927           16 :     case IX86_BUILTIN_SCATTERDIV4SF:
   16928           16 :       icode = CODE_FOR_avx512vl_scatterdiv4sf;
   16929           16 :       goto scatter_gen;
   16930           18 :     case IX86_BUILTIN_SCATTERDIV4DF:
   16931           18 :       icode = CODE_FOR_avx512vl_scatterdiv4df;
   16932           18 :       goto scatter_gen;
   16933           18 :     case IX86_BUILTIN_SCATTERDIV2DF:
   16934           18 :       icode = CODE_FOR_avx512vl_scatterdiv2df;
   16935           18 :       goto scatter_gen;
   16936           22 :     case IX86_BUILTIN_SCATTERSIV8SI:
   16937           22 :       icode = CODE_FOR_avx512vl_scattersiv8si;
   16938           22 :       goto scatter_gen;
   16939           24 :     case IX86_BUILTIN_SCATTERSIV4SI:
   16940           24 :       icode = CODE_FOR_avx512vl_scattersiv4si;
   16941           24 :       goto scatter_gen;
   16942           16 :     case IX86_BUILTIN_SCATTERSIV4DI:
   16943           16 :       icode = CODE_FOR_avx512vl_scattersiv4di;
   16944           16 :       goto scatter_gen;
   16945           16 :     case IX86_BUILTIN_SCATTERSIV2DI:
   16946           16 :       icode = CODE_FOR_avx512vl_scattersiv2di;
   16947           16 :       goto scatter_gen;
   16948           16 :     case IX86_BUILTIN_SCATTERDIV8SI:
   16949           16 :       icode = CODE_FOR_avx512vl_scatterdiv8si;
   16950           16 :       goto scatter_gen;
   16951           16 :     case IX86_BUILTIN_SCATTERDIV4SI:
   16952           16 :       icode = CODE_FOR_avx512vl_scatterdiv4si;
   16953           16 :       goto scatter_gen;
   16954           18 :     case IX86_BUILTIN_SCATTERDIV4DI:
   16955           18 :       icode = CODE_FOR_avx512vl_scatterdiv4di;
   16956           18 :       goto scatter_gen;
   16957           18 :     case IX86_BUILTIN_SCATTERDIV2DI:
   16958           18 :       icode = CODE_FOR_avx512vl_scatterdiv2di;
   16959           18 :       goto scatter_gen;
   16960           16 :     case IX86_BUILTIN_SCATTERALTSIV8DF:
   16961           16 :       icode = CODE_FOR_avx512f_scattersiv8df;
   16962           16 :       goto scatter_gen;
   16963           12 :     case IX86_BUILTIN_SCATTERALTDIV16SF:
   16964           12 :       icode = CODE_FOR_avx512f_scatterdiv16sf;
   16965           12 :       goto scatter_gen;
   16966            8 :     case IX86_BUILTIN_SCATTERALTSIV8DI:
   16967            8 :       icode = CODE_FOR_avx512f_scattersiv8di;
   16968            8 :       goto scatter_gen;
   16969           24 :     case IX86_BUILTIN_SCATTERALTDIV16SI:
   16970           24 :       icode = CODE_FOR_avx512f_scatterdiv16si;
   16971           24 :       goto scatter_gen;
   16972            4 :     case IX86_BUILTIN_SCATTERALTSIV4DF:
   16973            4 :       icode = CODE_FOR_avx512vl_scattersiv4df;
   16974            4 :       goto scatter_gen;
   16975            4 :     case IX86_BUILTIN_SCATTERALTDIV8SF:
   16976            4 :       icode = CODE_FOR_avx512vl_scatterdiv8sf;
   16977            4 :       goto scatter_gen;
   16978            4 :     case IX86_BUILTIN_SCATTERALTSIV4DI:
   16979            4 :       icode = CODE_FOR_avx512vl_scattersiv4di;
   16980            4 :       goto scatter_gen;
   16981            4 :     case IX86_BUILTIN_SCATTERALTDIV8SI:
   16982            4 :       icode = CODE_FOR_avx512vl_scatterdiv8si;
   16983            4 :       goto scatter_gen;
   16984            8 :     case IX86_BUILTIN_SCATTERALTSIV2DF:
   16985            8 :       icode = CODE_FOR_avx512vl_scattersiv2df;
   16986            8 :       goto scatter_gen;
   16987            8 :     case IX86_BUILTIN_SCATTERALTDIV4SF:
   16988            8 :       icode = CODE_FOR_avx512vl_scatterdiv4sf;
   16989            8 :       goto scatter_gen;
   16990            8 :     case IX86_BUILTIN_SCATTERALTSIV2DI:
   16991            8 :       icode = CODE_FOR_avx512vl_scattersiv2di;
   16992            8 :       goto scatter_gen;
   16993            8 :     case IX86_BUILTIN_SCATTERALTDIV4SI:
   16994            8 :       icode = CODE_FOR_avx512vl_scatterdiv4si;
   16995            8 :       goto scatter_gen;
   16996              : 
   16997          995 :     gather_gen:
   16998          995 :       rtx half;
   16999          995 :       rtx (*gen) (rtx, rtx);
   17000              : 
   17001          995 :       arg0 = CALL_EXPR_ARG (exp, 0);
   17002          995 :       arg1 = CALL_EXPR_ARG (exp, 1);
   17003          995 :       arg2 = CALL_EXPR_ARG (exp, 2);
   17004          995 :       arg3 = CALL_EXPR_ARG (exp, 3);
   17005          995 :       arg4 = CALL_EXPR_ARG (exp, 4);
   17006          995 :       op0 = expand_normal (arg0);
   17007          995 :       op1 = expand_normal (arg1);
   17008          995 :       op2 = expand_normal (arg2);
   17009          995 :       op3 = ix86_expand_unsigned_small_int_cst_argument (arg3);
   17010          995 :       op4 = expand_normal (arg4);
   17011              :       /* Note the arg order is different from the operand order.  */
   17012          995 :       mode0 = insn_data[icode].operand[1].mode;
   17013          995 :       mode2 = insn_data[icode].operand[3].mode;
   17014          995 :       mode3 = insn_data[icode].operand[4].mode;
   17015          995 :       mode4 = insn_data[icode].operand[5].mode;
   17016              : 
   17017          995 :       if (target == NULL_RTX
   17018          995 :           || GET_MODE (target) != insn_data[icode].operand[0].mode
   17019         1886 :           || !insn_data[icode].operand[0].predicate (target,
   17020              :                                                      GET_MODE (target)))
   17021          105 :         subtarget = gen_reg_rtx (insn_data[icode].operand[0].mode);
   17022              :       else
   17023              :         subtarget = target;
   17024              : 
   17025          995 :       switch (fcode)
   17026              :         {
   17027           30 :         case IX86_BUILTIN_GATHER3ALTSIV8DF:
   17028           30 :         case IX86_BUILTIN_GATHER3ALTSIV8DI:
   17029           30 :           half = gen_reg_rtx (V8SImode);
   17030           30 :           if (!nonimmediate_operand (op2, V16SImode))
   17031            0 :             op2 = copy_to_mode_reg (V16SImode, op2);
   17032           30 :           emit_insn (gen_vec_extract_lo_v16si (half, op2));
   17033           30 :           op2 = half;
   17034           30 :           break;
   17035           30 :         case IX86_BUILTIN_GATHER3ALTSIV4DF:
   17036           30 :         case IX86_BUILTIN_GATHER3ALTSIV4DI:
   17037           30 :         case IX86_BUILTIN_GATHERALTSIV4DF:
   17038           30 :         case IX86_BUILTIN_GATHERALTSIV4DI:
   17039           30 :           half = gen_reg_rtx (V4SImode);
   17040           30 :           if (!nonimmediate_operand (op2, V8SImode))
   17041            0 :             op2 = copy_to_mode_reg (V8SImode, op2);
   17042           30 :           emit_insn (gen_vec_extract_lo_v8si (half, op2));
   17043           30 :           op2 = half;
   17044           30 :           break;
   17045           40 :         case IX86_BUILTIN_GATHER3ALTDIV16SF:
   17046           40 :         case IX86_BUILTIN_GATHER3ALTDIV16SI:
   17047           40 :           half = gen_reg_rtx (mode0);
   17048           40 :           if (mode0 == V8SFmode)
   17049              :             gen = gen_vec_extract_lo_v16sf;
   17050              :           else
   17051           18 :             gen = gen_vec_extract_lo_v16si;
   17052           40 :           if (!nonimmediate_operand (op0, GET_MODE (op0)))
   17053           40 :             op0 = copy_to_mode_reg (GET_MODE (op0), op0);
   17054           40 :           emit_insn (gen (half, op0));
   17055           40 :           op0 = half;
   17056           40 :           op3 = lowpart_subreg (QImode, op3, HImode);
   17057           40 :           break;
   17058           46 :         case IX86_BUILTIN_GATHER3ALTDIV8SF:
   17059           46 :         case IX86_BUILTIN_GATHER3ALTDIV8SI:
   17060           46 :         case IX86_BUILTIN_GATHERALTDIV8SF:
   17061           46 :         case IX86_BUILTIN_GATHERALTDIV8SI:
   17062           46 :           half = gen_reg_rtx (mode0);
   17063           46 :           if (mode0 == V4SFmode)
   17064              :             gen = gen_vec_extract_lo_v8sf;
   17065              :           else
   17066           22 :             gen = gen_vec_extract_lo_v8si;
   17067           46 :           if (!nonimmediate_operand (op0, GET_MODE (op0)))
   17068           46 :             op0 = copy_to_mode_reg (GET_MODE (op0), op0);
   17069           46 :           emit_insn (gen (half, op0));
   17070           46 :           op0 = half;
   17071           46 :           if (VECTOR_MODE_P (GET_MODE (op3)))
   17072              :             {
   17073           28 :               half = gen_reg_rtx (mode0);
   17074           28 :               if (!nonimmediate_operand (op3, GET_MODE (op3)))
   17075           12 :                 op3 = copy_to_mode_reg (GET_MODE (op3), op3);
   17076           28 :               emit_insn (gen (half, op3));
   17077           28 :               op3 = half;
   17078              :             }
   17079              :           break;
   17080              :         default:
   17081              :           break;
   17082              :         }
   17083              : 
   17084              :       /* Force memory operand only with base register here.  But we
   17085              :          don't want to do it on memory operand for other builtin
   17086              :          functions.  */
   17087          995 :       op1 = ix86_zero_extend_to_Pmode (op1);
   17088              : 
   17089          995 :       if (!insn_data[icode].operand[1].predicate (op0, mode0))
   17090          394 :         op0 = copy_to_mode_reg (mode0, op0);
   17091          998 :       if (!insn_data[icode].operand[2].predicate (op1, Pmode))
   17092            0 :         op1 = copy_to_mode_reg (Pmode, op1);
   17093          995 :       if (!insn_data[icode].operand[3].predicate (op2, mode2))
   17094          224 :         op2 = copy_to_mode_reg (mode2, op2);
   17095              : 
   17096          995 :       op3 = fixup_modeless_constant (op3, mode3);
   17097              : 
   17098          995 :       if (GET_MODE (op3) == mode3 || GET_MODE (op3) == VOIDmode)
   17099              :         {
   17100          995 :           if (!insn_data[icode].operand[4].predicate (op3, mode3))
   17101          352 :             op3 = copy_to_mode_reg (mode3, op3);
   17102              :         }
   17103              :       else
   17104              :         {
   17105            0 :           op3 = copy_to_reg (op3);
   17106            0 :           op3 = lowpart_subreg (mode3, op3, GET_MODE (op3));
   17107              :         }
   17108          995 :       if (!insn_data[icode].operand[5].predicate (op4, mode4))
   17109              :         {
   17110            0 :           error ("the last argument must be scale 1, 2, 4, 8");
   17111            0 :           return const0_rtx;
   17112              :         }
   17113              : 
   17114              :       /* Optimize.  If mask is known to have all high bits set,
   17115              :          replace op0 with pc_rtx to signal that the instruction
   17116              :          overwrites the whole destination and doesn't use its
   17117              :          previous contents.  */
   17118          995 :       if (optimize)
   17119              :         {
   17120          905 :           if (TREE_CODE (arg3) == INTEGER_CST)
   17121              :             {
   17122          209 :               if (integer_all_onesp (arg3))
   17123          201 :                 op0 = pc_rtx;
   17124              :             }
   17125          696 :           else if (TREE_CODE (arg3) == VECTOR_CST)
   17126              :             {
   17127              :               unsigned int negative = 0;
   17128          727 :               for (i = 0; i < VECTOR_CST_NELTS (arg3); ++i)
   17129              :                 {
   17130          596 :                   tree cst = VECTOR_CST_ELT (arg3, i);
   17131          596 :                   if (TREE_CODE (cst) == INTEGER_CST
   17132          596 :                       && tree_int_cst_sign_bit (cst))
   17133          286 :                     negative++;
   17134          310 :                   else if (TREE_CODE (cst) == REAL_CST
   17135          310 :                            && REAL_VALUE_NEGATIVE (TREE_REAL_CST (cst)))
   17136          282 :                     negative++;
   17137              :                 }
   17138          131 :               if (negative == TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg3)))
   17139          117 :                 op0 = pc_rtx;
   17140              :             }
   17141          565 :           else if (TREE_CODE (arg3) == SSA_NAME
   17142          565 :                    && VECTOR_TYPE_P (TREE_TYPE (arg3)))
   17143              :             {
   17144              :               /* Recognize also when mask is like:
   17145              :                  __v2df src = _mm_setzero_pd ();
   17146              :                  __v2df mask = _mm_cmpeq_pd (src, src);
   17147              :                  or
   17148              :                  __v8sf src = _mm256_setzero_ps ();
   17149              :                  __v8sf mask = _mm256_cmp_ps (src, src, _CMP_EQ_OQ);
   17150              :                  as that is a cheaper way to load all ones into
   17151              :                  a register than having to load a constant from
   17152              :                  memory.  */
   17153          251 :               gimple *def_stmt = SSA_NAME_DEF_STMT (arg3);
   17154          251 :               if (is_gimple_call (def_stmt))
   17155              :                 {
   17156           76 :                   tree fndecl = gimple_call_fndecl (def_stmt);
   17157           76 :                   if (fndecl
   17158           76 :                       && fndecl_built_in_p (fndecl, BUILT_IN_MD))
   17159           67 :                     switch (DECL_MD_FUNCTION_CODE (fndecl))
   17160              :                       {
   17161           24 :                       case IX86_BUILTIN_CMPPD:
   17162           24 :                       case IX86_BUILTIN_CMPPS:
   17163           24 :                       case IX86_BUILTIN_CMPPD256:
   17164           24 :                       case IX86_BUILTIN_CMPPS256:
   17165           24 :                         if (!integer_zerop (gimple_call_arg (def_stmt, 2)))
   17166              :                           break;
   17167              :                         /* FALLTHRU */
   17168           49 :                       case IX86_BUILTIN_CMPEQPD:
   17169           49 :                       case IX86_BUILTIN_CMPEQPS:
   17170           49 :                         if (initializer_zerop (gimple_call_arg (def_stmt, 0))
   17171           49 :                             && initializer_zerop (gimple_call_arg (def_stmt,
   17172              :                                                                    1)))
   17173           49 :                           op0 = pc_rtx;
   17174              :                         break;
   17175              :                       default:
   17176              :                         break;
   17177              :                       }
   17178              :                 }
   17179              :             }
   17180              :         }
   17181              : 
   17182          995 :       pat = GEN_FCN (icode) (subtarget, op0, op1, op2, op3, op4);
   17183          995 :       if (! pat)
   17184            0 :         return const0_rtx;
   17185          995 :       emit_insn (pat);
   17186              : 
   17187          995 :       switch (fcode)
   17188              :         {
   17189           24 :         case IX86_BUILTIN_GATHER3DIV16SF:
   17190           24 :           if (target == NULL_RTX)
   17191            0 :             target = gen_reg_rtx (V8SFmode);
   17192           24 :           emit_insn (gen_vec_extract_lo_v16sf (target, subtarget));
   17193           24 :           break;
   17194           24 :         case IX86_BUILTIN_GATHER3DIV16SI:
   17195           24 :           if (target == NULL_RTX)
   17196            0 :             target = gen_reg_rtx (V8SImode);
   17197           24 :           emit_insn (gen_vec_extract_lo_v16si (target, subtarget));
   17198           24 :           break;
   17199           28 :         case IX86_BUILTIN_GATHER3DIV8SF:
   17200           28 :         case IX86_BUILTIN_GATHERDIV8SF:
   17201           28 :           if (target == NULL_RTX)
   17202            0 :             target = gen_reg_rtx (V4SFmode);
   17203           28 :           emit_insn (gen_vec_extract_lo_v8sf (target, subtarget));
   17204           28 :           break;
   17205           28 :         case IX86_BUILTIN_GATHER3DIV8SI:
   17206           28 :         case IX86_BUILTIN_GATHERDIV8SI:
   17207           28 :           if (target == NULL_RTX)
   17208            0 :             target = gen_reg_rtx (V4SImode);
   17209           28 :           emit_insn (gen_vec_extract_lo_v8si (target, subtarget));
   17210           28 :           break;
   17211              :         default:
   17212              :           target = subtarget;
   17213              :           break;
   17214              :         }
   17215              :       return target;
   17216              : 
   17217          623 :     scatter_gen:
   17218          623 :       arg0 = CALL_EXPR_ARG (exp, 0);
   17219          623 :       arg1 = CALL_EXPR_ARG (exp, 1);
   17220          623 :       arg2 = CALL_EXPR_ARG (exp, 2);
   17221          623 :       arg3 = CALL_EXPR_ARG (exp, 3);
   17222          623 :       arg4 = CALL_EXPR_ARG (exp, 4);
   17223          623 :       op0 = expand_normal (arg0);
   17224          623 :       op1 = ix86_expand_unsigned_small_int_cst_argument (arg1);
   17225          623 :       op2 = expand_normal (arg2);
   17226          623 :       op3 = expand_normal (arg3);
   17227          623 :       op4 = expand_normal (arg4);
   17228          623 :       mode1 = insn_data[icode].operand[1].mode;
   17229          623 :       mode2 = insn_data[icode].operand[2].mode;
   17230          623 :       mode3 = insn_data[icode].operand[3].mode;
   17231          623 :       mode4 = insn_data[icode].operand[4].mode;
   17232              : 
   17233              :       /* Scatter instruction stores operand op3 to memory with
   17234              :          indices from op2 and scale from op4 under writemask op1.
   17235              :          If index operand op2 has more elements then source operand
   17236              :          op3 one need to use only its low half. And vice versa.  */
   17237          623 :       switch (fcode)
   17238              :         {
   17239           24 :         case IX86_BUILTIN_SCATTERALTSIV8DF:
   17240           24 :         case IX86_BUILTIN_SCATTERALTSIV8DI:
   17241           24 :           half = gen_reg_rtx (V8SImode);
   17242           24 :           if (!nonimmediate_operand (op2, V16SImode))
   17243            0 :             op2 = copy_to_mode_reg (V16SImode, op2);
   17244           24 :           emit_insn (gen_vec_extract_lo_v16si (half, op2));
   17245           24 :           op2 = half;
   17246           24 :           break;
   17247           36 :         case IX86_BUILTIN_SCATTERALTDIV16SF:
   17248           36 :         case IX86_BUILTIN_SCATTERALTDIV16SI:
   17249           36 :           half = gen_reg_rtx (mode3);
   17250           36 :           if (mode3 == V8SFmode)
   17251              :             gen = gen_vec_extract_lo_v16sf;
   17252              :           else
   17253           24 :             gen = gen_vec_extract_lo_v16si;
   17254           36 :           if (!nonimmediate_operand (op3, GET_MODE (op3)))
   17255            0 :             op3 = copy_to_mode_reg (GET_MODE (op3), op3);
   17256           36 :           emit_insn (gen (half, op3));
   17257           36 :           op3 = half;
   17258           36 :           break;
   17259            8 :         case IX86_BUILTIN_SCATTERALTSIV4DF:
   17260            8 :         case IX86_BUILTIN_SCATTERALTSIV4DI:
   17261            8 :           half = gen_reg_rtx (V4SImode);
   17262            8 :           if (!nonimmediate_operand (op2, V8SImode))
   17263            0 :             op2 = copy_to_mode_reg (V8SImode, op2);
   17264            8 :           emit_insn (gen_vec_extract_lo_v8si (half, op2));
   17265            8 :           op2 = half;
   17266            8 :           break;
   17267            8 :         case IX86_BUILTIN_SCATTERALTDIV8SF:
   17268            8 :         case IX86_BUILTIN_SCATTERALTDIV8SI:
   17269            8 :           half = gen_reg_rtx (mode3);
   17270            8 :           if (mode3 == V4SFmode)
   17271              :             gen = gen_vec_extract_lo_v8sf;
   17272              :           else
   17273            4 :             gen = gen_vec_extract_lo_v8si;
   17274            8 :           if (!nonimmediate_operand (op3, GET_MODE (op3)))
   17275            0 :             op3 = copy_to_mode_reg (GET_MODE (op3), op3);
   17276            8 :           emit_insn (gen (half, op3));
   17277            8 :           op3 = half;
   17278            8 :           break;
   17279           16 :         case IX86_BUILTIN_SCATTERALTSIV2DF:
   17280           16 :         case IX86_BUILTIN_SCATTERALTSIV2DI:
   17281           16 :           if (!nonimmediate_operand (op2, V4SImode))
   17282            0 :             op2 = copy_to_mode_reg (V4SImode, op2);
   17283              :           break;
   17284           16 :         case IX86_BUILTIN_SCATTERALTDIV4SF:
   17285           16 :         case IX86_BUILTIN_SCATTERALTDIV4SI:
   17286           16 :           if (!nonimmediate_operand (op3, GET_MODE (op3)))
   17287            0 :             op3 = copy_to_mode_reg (GET_MODE (op3), op3);
   17288              :           break;
   17289              :         default:
   17290              :           break;
   17291              :         }
   17292              : 
   17293              :       /* Force memory operand only with base register here.  But we
   17294              :          don't want to do it on memory operand for other builtin
   17295              :          functions.  */
   17296          633 :       op0 = force_reg (Pmode, convert_to_mode (Pmode, op0, 1));
   17297              : 
   17298          628 :       if (!insn_data[icode].operand[0].predicate (op0, Pmode))
   17299            0 :         op0 = copy_to_mode_reg (Pmode, op0);
   17300              : 
   17301          623 :       op1 = fixup_modeless_constant (op1, mode1);
   17302              : 
   17303          623 :       if (GET_MODE (op1) == mode1 || GET_MODE (op1) == VOIDmode)
   17304              :         {
   17305          607 :           if (!insn_data[icode].operand[1].predicate (op1, mode1))
   17306          273 :             op1 = copy_to_mode_reg (mode1, op1);
   17307              :         }
   17308              :       else
   17309              :         {
   17310           16 :           op1 = copy_to_reg (op1);
   17311           16 :           op1 = lowpart_subreg (mode1, op1, GET_MODE (op1));
   17312              :         }
   17313              : 
   17314          623 :       if (!insn_data[icode].operand[2].predicate (op2, mode2))
   17315           57 :         op2 = copy_to_mode_reg (mode2, op2);
   17316              : 
   17317          623 :       if (!insn_data[icode].operand[3].predicate (op3, mode3))
   17318           82 :         op3 = copy_to_mode_reg (mode3, op3);
   17319              : 
   17320          623 :       if (!insn_data[icode].operand[4].predicate (op4, mode4))
   17321              :         {
   17322            0 :           error ("the last argument must be scale 1, 2, 4, 8");
   17323            0 :           return const0_rtx;
   17324              :         }
   17325              : 
   17326          623 :       pat = GEN_FCN (icode) (op0, op1, op2, op3, op4);
   17327          623 :       if (! pat)
   17328            0 :         return const0_rtx;
   17329              : 
   17330          623 :       emit_insn (pat);
   17331          623 :       return 0;
   17332              : 
   17333           23 :     case IX86_BUILTIN_XABORT:
   17334           23 :       icode = CODE_FOR_xabort;
   17335           23 :       arg0 = CALL_EXPR_ARG (exp, 0);
   17336           23 :       op0 = expand_normal (arg0);
   17337           23 :       mode0 = insn_data[icode].operand[0].mode;
   17338           23 :       if (!insn_data[icode].operand[0].predicate (op0, mode0))
   17339              :         {
   17340            0 :           error ("the argument to %<xabort%> intrinsic must "
   17341              :                  "be an 8-bit immediate");
   17342            0 :           return const0_rtx;
   17343              :         }
   17344           23 :       emit_insn (gen_xabort (op0));
   17345           23 :       return 0;
   17346              : 
   17347           55 :     case IX86_BUILTIN_RDSSPD:
   17348           55 :     case IX86_BUILTIN_RDSSPQ:
   17349           55 :       mode = (fcode == IX86_BUILTIN_RDSSPD ? SImode : DImode);
   17350              : 
   17351           55 :       if (target == 0
   17352           55 :           || !register_operand (target, mode))
   17353            0 :         target = gen_reg_rtx (mode);
   17354              : 
   17355           55 :       op0 = force_reg (mode, const0_rtx);
   17356              : 
   17357           55 :       emit_insn (gen_rdssp (mode, target, op0));
   17358           55 :       return target;
   17359              : 
   17360           55 :     case IX86_BUILTIN_INCSSPD:
   17361           55 :     case IX86_BUILTIN_INCSSPQ:
   17362           55 :       mode = (fcode == IX86_BUILTIN_INCSSPD ? SImode : DImode);
   17363              : 
   17364           55 :       arg0 = CALL_EXPR_ARG (exp, 0);
   17365           55 :       op0 = expand_normal (arg0);
   17366              : 
   17367           55 :       op0 = force_reg (mode, op0);
   17368              : 
   17369           55 :       emit_insn (gen_incssp (mode, op0));
   17370           55 :       return 0;
   17371              : 
   17372           20 :     case IX86_BUILTIN_HRESET:
   17373           20 :       icode = CODE_FOR_hreset;
   17374           20 :       arg0 = CALL_EXPR_ARG (exp, 0);
   17375           20 :       op0 = expand_normal (arg0);
   17376           20 :       op0 = force_reg (SImode, op0);
   17377           20 :       emit_insn (gen_hreset (op0));
   17378           20 :       return 0;
   17379              : 
   17380           39 :     case IX86_BUILTIN_RSTORSSP:
   17381           39 :     case IX86_BUILTIN_CLRSSBSY:
   17382           39 :       arg0 = CALL_EXPR_ARG (exp, 0);
   17383           39 :       op0 = expand_normal (arg0);
   17384           20 :       icode = (fcode == IX86_BUILTIN_RSTORSSP
   17385           39 :                ? CODE_FOR_rstorssp
   17386              :                : CODE_FOR_clrssbsy);
   17387              : 
   17388           39 :       if (!address_operand (op0, VOIDmode))
   17389              :         {
   17390           18 :           op0 = convert_memory_address (Pmode, op0);
   17391           18 :           op0 = copy_addr_to_reg (op0);
   17392              :         }
   17393           39 :       emit_insn (GEN_FCN (icode) (gen_rtx_MEM (DImode, op0)));
   17394           39 :       return 0;
   17395              : 
   17396           81 :     case IX86_BUILTIN_WRSSD:
   17397           81 :     case IX86_BUILTIN_WRSSQ:
   17398           81 :     case IX86_BUILTIN_WRUSSD:
   17399           81 :     case IX86_BUILTIN_WRUSSQ:
   17400           81 :       mode = ((fcode == IX86_BUILTIN_WRSSD
   17401           81 :                || fcode == IX86_BUILTIN_WRUSSD)
   17402           81 :               ? SImode : DImode);
   17403              : 
   17404           81 :       arg0 = CALL_EXPR_ARG (exp, 0);
   17405           81 :       op0 = expand_normal (arg0);
   17406           81 :       arg1 = CALL_EXPR_ARG (exp, 1);
   17407           81 :       op1 = expand_normal (arg1);
   17408              : 
   17409           81 :       op0 = force_reg (mode, op0);
   17410              : 
   17411           81 :       if (!address_operand (op1, VOIDmode))
   17412              :         {
   17413           36 :           op1 = convert_memory_address (Pmode, op1);
   17414           36 :           op1 = copy_addr_to_reg (op1);
   17415              :         }
   17416           81 :       op1 = gen_rtx_MEM (mode, op1);
   17417              : 
   17418           81 :       icode = ((fcode == IX86_BUILTIN_WRSSD
   17419           81 :                 || fcode == IX86_BUILTIN_WRSSQ)
   17420           81 :                ? code_for_wrss (mode)
   17421           40 :                : code_for_wruss (mode));
   17422           81 :       emit_insn (GEN_FCN (icode) (op0, op1));
   17423              : 
   17424           81 :       return 0;
   17425              : 
   17426       120368 :     default:
   17427       120368 :       break;
   17428              :     }
   17429              : 
   17430       120368 :   if (fcode >= IX86_BUILTIN__BDESC_SPECIAL_ARGS_FIRST
   17431       120368 :       && fcode <= IX86_BUILTIN__BDESC_SPECIAL_ARGS_LAST)
   17432              :     {
   17433        27029 :       i = fcode - IX86_BUILTIN__BDESC_SPECIAL_ARGS_FIRST;
   17434        27029 :       return ix86_expand_special_args_builtin (bdesc_special_args + i, exp,
   17435        27029 :                                                target);
   17436              :     }
   17437              : 
   17438        93339 :   if (fcode >= IX86_BUILTIN__BDESC_PURE_ARGS_FIRST
   17439        93339 :       && fcode <= IX86_BUILTIN__BDESC_PURE_ARGS_LAST)
   17440              :     {
   17441           93 :       i = fcode - IX86_BUILTIN__BDESC_PURE_ARGS_FIRST;
   17442           93 :       return ix86_expand_special_args_builtin (bdesc_pure_args + i, exp,
   17443           93 :                                                target);
   17444              :     }
   17445              : 
   17446        93246 :   if (fcode >= IX86_BUILTIN__BDESC_ARGS_FIRST
   17447        93246 :       && fcode <= IX86_BUILTIN__BDESC_ARGS_LAST)
   17448              :     {
   17449        74576 :       i = fcode - IX86_BUILTIN__BDESC_ARGS_FIRST;
   17450              : 
   17451        74576 :       switch (fcode)
   17452              :         {
   17453            0 :           case IX86_BUILTIN_RDPID:
   17454            0 :             return ix86_expand_special_args_builtin (bdesc_args + i, exp,
   17455            0 :                                                      target);
   17456           74 :           case IX86_BUILTIN_VCOMISBF16EQ:
   17457           74 :           case IX86_BUILTIN_VCOMISBF16NE:
   17458           74 :           case IX86_BUILTIN_VCOMISBF16GT:
   17459           74 :           case IX86_BUILTIN_VCOMISBF16GE:
   17460           74 :           case IX86_BUILTIN_VCOMISBF16LT:
   17461           74 :           case IX86_BUILTIN_VCOMISBF16LE:
   17462           74 :             return ix86_expand_sse_comi (bdesc_args + i, exp, target, false);
   17463           15 :           case IX86_BUILTIN_FABSQ:
   17464           15 :           case IX86_BUILTIN_COPYSIGNQ:
   17465           15 :             if (!TARGET_SSE)
   17466              :               /* Emit a normal call if SSE isn't available.  */
   17467            0 :               return expand_call (exp, target, ignore);
   17468              :             /* FALLTHRU */
   17469        74502 :           default:
   17470        74502 :             return ix86_expand_args_builtin (bdesc_args + i, exp, target);
   17471              :           }
   17472              :     }
   17473              : 
   17474        18670 :   if (fcode >= IX86_BUILTIN__BDESC_COMI_FIRST
   17475        18670 :       && fcode <= IX86_BUILTIN__BDESC_COMI_LAST)
   17476              :     {
   17477          473 :       i = fcode - IX86_BUILTIN__BDESC_COMI_FIRST;
   17478          473 :       return ix86_expand_sse_comi (bdesc_comi + i, exp, target, true);
   17479              :     }
   17480              : 
   17481        18197 :   if (fcode >= IX86_BUILTIN__BDESC_ROUND_ARGS_FIRST
   17482        18197 :       && fcode <= IX86_BUILTIN__BDESC_ROUND_ARGS_LAST)
   17483              :     {
   17484        15660 :       i = fcode - IX86_BUILTIN__BDESC_ROUND_ARGS_FIRST;
   17485        15660 :       return ix86_expand_round_builtin (bdesc_round_args + i, exp, target);
   17486              :     }
   17487              : 
   17488         2537 :   if (fcode >= IX86_BUILTIN__BDESC_PCMPESTR_FIRST
   17489         2537 :       && fcode <= IX86_BUILTIN__BDESC_PCMPESTR_LAST)
   17490              :     {
   17491          216 :       i = fcode - IX86_BUILTIN__BDESC_PCMPESTR_FIRST;
   17492          216 :       return ix86_expand_sse_pcmpestr (bdesc_pcmpestr + i, exp, target);
   17493              :     }
   17494              : 
   17495         2321 :   if (fcode >= IX86_BUILTIN__BDESC_PCMPISTR_FIRST
   17496         2321 :       && fcode <= IX86_BUILTIN__BDESC_PCMPISTR_LAST)
   17497              :     {
   17498          275 :       i = fcode - IX86_BUILTIN__BDESC_PCMPISTR_FIRST;
   17499          275 :       return ix86_expand_sse_pcmpistr (bdesc_pcmpistr + i, exp, target);
   17500              :     }
   17501              : 
   17502         2046 :   if (fcode >= IX86_BUILTIN__BDESC_MULTI_ARG_FIRST
   17503         2046 :       && fcode <= IX86_BUILTIN__BDESC_MULTI_ARG_LAST)
   17504              :     {
   17505         1816 :       i = fcode - IX86_BUILTIN__BDESC_MULTI_ARG_FIRST;
   17506         1816 :       const struct builtin_description *d = bdesc_multi_arg + i;
   17507         1816 :       return ix86_expand_multi_arg_builtin (d->icode, exp, target,
   17508              :                                             (enum ix86_builtin_func_type)
   17509         1816 :                                             d->flag, d->comparison);
   17510              :     }
   17511              : 
   17512          230 :   if (fcode >= IX86_BUILTIN__BDESC_CET_FIRST
   17513          230 :       && fcode <= IX86_BUILTIN__BDESC_CET_LAST)
   17514              :     {
   17515           38 :       i = fcode - IX86_BUILTIN__BDESC_CET_FIRST;
   17516           38 :       return ix86_expand_special_args_builtin (bdesc_cet + i, exp,
   17517           38 :                                                target);
   17518              :     }
   17519              : 
   17520          192 :   if (fcode >= IX86_BUILTIN__BDESC_ACE_FIRST
   17521          192 :       && fcode <= IX86_BUILTIN__BDESC_ACE_LAST)
   17522              :     {
   17523          192 :       i = fcode - IX86_BUILTIN__BDESC_ACE_FIRST;
   17524          192 :       return ix86_expand_ace_builtin (bdesc_ace + i, exp, target);
   17525              :     }
   17526              : 
   17527            0 :   gcc_unreachable ();
   17528              : }
   17529              : 
   17530              : /* See below where shifts are handled for explanation of this enum.  */
   17531              : enum ix86_vec_bcast_alg
   17532              : {
   17533              :   VEC_BCAST_PXOR,
   17534              :   VEC_BCAST_PCMPEQ,
   17535              :   VEC_BCAST_PABSB,
   17536              :   VEC_BCAST_PADDB,
   17537              :   VEC_BCAST_PSRLW,
   17538              :   VEC_BCAST_PSRLD,
   17539              :   VEC_BCAST_PSLLW,
   17540              :   VEC_BCAST_PSLLD
   17541              : };
   17542              : 
   17543              : struct ix86_vec_bcast_map_simode_t
   17544              : {
   17545              :   unsigned int key;
   17546              :   enum ix86_vec_bcast_alg alg;
   17547              :   unsigned int arg;
   17548              : };
   17549              : 
   17550              : /* This table must be kept sorted as values are looked-up using bsearch.  */
   17551              : static const ix86_vec_bcast_map_simode_t ix86_vec_bcast_map_simode[] = {
   17552              :   { 0x00000000, VEC_BCAST_PXOR,    0 },
   17553              :   { 0x00000001, VEC_BCAST_PSRLD,  31 },
   17554              :   { 0x00000003, VEC_BCAST_PSRLD,  30 },
   17555              :   { 0x00000007, VEC_BCAST_PSRLD,  29 },
   17556              :   { 0x0000000f, VEC_BCAST_PSRLD,  28 },
   17557              :   { 0x0000001f, VEC_BCAST_PSRLD,  27 },
   17558              :   { 0x0000003f, VEC_BCAST_PSRLD,  26 },
   17559              :   { 0x0000007f, VEC_BCAST_PSRLD,  25 },
   17560              :   { 0x000000ff, VEC_BCAST_PSRLD,  24 },
   17561              :   { 0x000001ff, VEC_BCAST_PSRLD,  23 },
   17562              :   { 0x000003ff, VEC_BCAST_PSRLD,  22 },
   17563              :   { 0x000007ff, VEC_BCAST_PSRLD,  21 },
   17564              :   { 0x00000fff, VEC_BCAST_PSRLD,  20 },
   17565              :   { 0x00001fff, VEC_BCAST_PSRLD,  19 },
   17566              :   { 0x00003fff, VEC_BCAST_PSRLD,  18 },
   17567              :   { 0x00007fff, VEC_BCAST_PSRLD,  17 },
   17568              :   { 0x0000ffff, VEC_BCAST_PSRLD,  16 },
   17569              :   { 0x00010001, VEC_BCAST_PSRLW,  15 },
   17570              :   { 0x0001ffff, VEC_BCAST_PSRLD,  15 },
   17571              :   { 0x00030003, VEC_BCAST_PSRLW,  14 },
   17572              :   { 0x0003ffff, VEC_BCAST_PSRLD,  14 },
   17573              :   { 0x00070007, VEC_BCAST_PSRLW,  13 },
   17574              :   { 0x0007ffff, VEC_BCAST_PSRLD,  13 },
   17575              :   { 0x000f000f, VEC_BCAST_PSRLW,  12 },
   17576              :   { 0x000fffff, VEC_BCAST_PSRLD,  12 },
   17577              :   { 0x001f001f, VEC_BCAST_PSRLW,  11 },
   17578              :   { 0x001fffff, VEC_BCAST_PSRLD,  11 },
   17579              :   { 0x003f003f, VEC_BCAST_PSRLW,  10 },
   17580              :   { 0x003fffff, VEC_BCAST_PSRLD,  10 },
   17581              :   { 0x007f007f, VEC_BCAST_PSRLW,   9 },
   17582              :   { 0x007fffff, VEC_BCAST_PSRLD,   9 },
   17583              :   { 0x00ff00ff, VEC_BCAST_PSRLW,   8 },
   17584              :   { 0x00ffffff, VEC_BCAST_PSRLD,   8 },
   17585              :   { 0x01010101, VEC_BCAST_PABSB,   0 },
   17586              :   { 0x01ff01ff, VEC_BCAST_PSRLW,   7 },
   17587              :   { 0x01ffffff, VEC_BCAST_PSRLD,   7 },
   17588              :   { 0x03ff03ff, VEC_BCAST_PSRLW,   6 },
   17589              :   { 0x03ffffff, VEC_BCAST_PSRLD,   6 },
   17590              :   { 0x07ff07ff, VEC_BCAST_PSRLW,   5 },
   17591              :   { 0x07ffffff, VEC_BCAST_PSRLD,   5 },
   17592              :   { 0x0fff0fff, VEC_BCAST_PSRLW,   4 },
   17593              :   { 0x0fffffff, VEC_BCAST_PSRLD,   4 },
   17594              :   { 0x1fff1fff, VEC_BCAST_PSRLW,   3 },
   17595              :   { 0x1fffffff, VEC_BCAST_PSRLD,   3 },
   17596              :   { 0x3fff3fff, VEC_BCAST_PSRLW,   2 },
   17597              :   { 0x3fffffff, VEC_BCAST_PSRLD,   2 },
   17598              :   { 0x7fff7fff, VEC_BCAST_PSRLW,   1 },
   17599              :   { 0x7fffffff, VEC_BCAST_PSRLD,   1 },
   17600              :   { 0x80000000, VEC_BCAST_PSLLD,  31 },
   17601              :   { 0x80008000, VEC_BCAST_PSLLW,  15 },
   17602              :   { 0xc0000000, VEC_BCAST_PSLLD,  30 },
   17603              :   { 0xc000c000, VEC_BCAST_PSLLW,  14 },
   17604              :   { 0xe0000000, VEC_BCAST_PSLLD,  29 },
   17605              :   { 0xe000e000, VEC_BCAST_PSLLW,  13 },
   17606              :   { 0xf0000000, VEC_BCAST_PSLLD,  28 },
   17607              :   { 0xf000f000, VEC_BCAST_PSLLW,  12 },
   17608              :   { 0xf8000000, VEC_BCAST_PSLLD,  27 },
   17609              :   { 0xf800f800, VEC_BCAST_PSLLW,  11 },
   17610              :   { 0xfc000000, VEC_BCAST_PSLLD,  26 },
   17611              :   { 0xfc00fc00, VEC_BCAST_PSLLW,  10 },
   17612              :   { 0xfe000000, VEC_BCAST_PSLLD,  25 },
   17613              :   { 0xfe00fe00, VEC_BCAST_PSLLW,   9 },
   17614              :   { 0xfefefefe, VEC_BCAST_PADDB,   0 },
   17615              :   { 0xff000000, VEC_BCAST_PSLLD,  24 },
   17616              :   { 0xff00ff00, VEC_BCAST_PSLLW,   8 },
   17617              :   { 0xff800000, VEC_BCAST_PSLLD,  23 },
   17618              :   { 0xff80ff80, VEC_BCAST_PSLLW,   7 },
   17619              :   { 0xffc00000, VEC_BCAST_PSLLD,  22 },
   17620              :   { 0xffc0ffc0, VEC_BCAST_PSLLW,   6 },
   17621              :   { 0xffe00000, VEC_BCAST_PSLLD,  21 },
   17622              :   { 0xffe0ffe0, VEC_BCAST_PSLLW,   5 },
   17623              :   { 0xfff00000, VEC_BCAST_PSLLD,  20 },
   17624              :   { 0xfff0fff0, VEC_BCAST_PSLLW,   4 },
   17625              :   { 0xfff80000, VEC_BCAST_PSLLD,  19 },
   17626              :   { 0xfff8fff8, VEC_BCAST_PSLLW,   3 },
   17627              :   { 0xfffc0000, VEC_BCAST_PSLLD,  18 },
   17628              :   { 0xfffcfffc, VEC_BCAST_PSLLW,   2 },
   17629              :   { 0xfffe0000, VEC_BCAST_PSLLD,  17 },
   17630              :   { 0xfffefffe, VEC_BCAST_PSLLW,   1 },
   17631              :   { 0xffff0000, VEC_BCAST_PSLLD,  16 },
   17632              :   { 0xffff8000, VEC_BCAST_PSLLD,  15 },
   17633              :   { 0xffffc000, VEC_BCAST_PSLLD,  14 },
   17634              :   { 0xffffe000, VEC_BCAST_PSLLD,  13 },
   17635              :   { 0xfffff000, VEC_BCAST_PSLLD,  12 },
   17636              :   { 0xfffff800, VEC_BCAST_PSLLD,  11 },
   17637              :   { 0xfffffc00, VEC_BCAST_PSLLD,  10 },
   17638              :   { 0xfffffe00, VEC_BCAST_PSLLD,   9 },
   17639              :   { 0xffffff00, VEC_BCAST_PSLLD,   8 },
   17640              :   { 0xffffff80, VEC_BCAST_PSLLD,   7 },
   17641              :   { 0xffffffc0, VEC_BCAST_PSLLD,   6 },
   17642              :   { 0xffffffe0, VEC_BCAST_PSLLD,   5 },
   17643              :   { 0xfffffff0, VEC_BCAST_PSLLD,   4 },
   17644              :   { 0xfffffff8, VEC_BCAST_PSLLD,   3 },
   17645              :   { 0xfffffffc, VEC_BCAST_PSLLD,   2 },
   17646              :   { 0xfffffffe, VEC_BCAST_PSLLD,   1 },
   17647              :   { 0xffffffff, VEC_BCAST_PCMPEQ,  0 }
   17648              : };
   17649              : 
   17650              : /* Comparator for bsearch on ix86_vec_bcast_map.  */
   17651              : static int
   17652       355692 : ix86_vec_bcast_map_simode_cmp (const void *key, const void *entry)
   17653              : {
   17654       355692 :   return (*(const unsigned int*)key)
   17655       355692 :          - ((const ix86_vec_bcast_map_simode_t*)entry)->key;
   17656              : }
   17657              : 
   17658              : /* A subroutine of ix86_vector_duplicate_value.  Tries to efficiently
   17659              :    materialize V4SImode, V8SImode and V16SImode vectors from SImode
   17660              :    integer constants.  */
   17661              : static bool
   17662        53907 : ix86_vector_duplicate_simode_const (machine_mode mode, rtx target,
   17663              :                                     unsigned int val)
   17664              : {
   17665        53907 :   const ix86_vec_bcast_map_simode_t *entry;
   17666        53907 :   rtx tmp1, tmp2;
   17667              : 
   17668        53907 :   entry = (const ix86_vec_bcast_map_simode_t*)
   17669        53907 :           bsearch(&val, ix86_vec_bcast_map_simode,
   17670              :                   ARRAY_SIZE (ix86_vec_bcast_map_simode),
   17671              :                   sizeof (ix86_vec_bcast_map_simode_t),
   17672              :                   ix86_vec_bcast_map_simode_cmp);
   17673        53907 :   if (!entry)
   17674              :     return false;
   17675              : 
   17676        19800 :   switch (entry->alg)
   17677              :     {
   17678         2576 :     case VEC_BCAST_PXOR:
   17679         2576 :       if ((mode == V8SImode && !TARGET_AVX2)
   17680         2576 :           || (mode == V16SImode && !TARGET_AVX512F))
   17681              :         return false;
   17682         2576 :       emit_move_insn (target, CONST0_RTX (mode));
   17683         2576 :       return true;
   17684              : 
   17685          223 :     case VEC_BCAST_PCMPEQ:
   17686          223 :       if ((mode == V4SImode && !TARGET_SSE2)
   17687          222 :           || (mode == V8SImode && !TARGET_AVX2)
   17688          195 :           || (mode == V16SImode && !TARGET_AVX512F))
   17689              :         return false;
   17690          195 :       emit_move_insn (target, CONSTM1_RTX (mode));
   17691          195 :       return true;
   17692              : 
   17693          653 :     case VEC_BCAST_PABSB:
   17694          653 :       if (mode == V4SImode && TARGET_SSE2)
   17695              :         {
   17696          526 :           tmp1 = gen_reg_rtx (V16QImode);
   17697          526 :           emit_move_insn (tmp1, CONSTM1_RTX (V16QImode));
   17698          526 :           tmp2 = gen_reg_rtx (V16QImode);
   17699          526 :           emit_insn (gen_absv16qi2 (tmp2, tmp1));
   17700              :         }
   17701          127 :       else if (mode == V8SImode && TARGET_AVX2)
   17702              :         {
   17703           69 :           tmp1 = gen_reg_rtx (V32QImode);
   17704           69 :           emit_move_insn (tmp1, CONSTM1_RTX (V32QImode));
   17705           69 :           tmp2 = gen_reg_rtx (V32QImode);
   17706           69 :           emit_insn (gen_absv32qi2 (tmp2, tmp1));
   17707              :         }
   17708           58 :       else if (mode == V16SImode && TARGET_AVX512BW)
   17709              :         {
   17710           50 :           tmp1 = gen_reg_rtx (V64QImode);
   17711           50 :           emit_move_insn (tmp1, CONSTM1_RTX (V64QImode));
   17712           50 :           tmp2 = gen_reg_rtx (V64QImode);
   17713           50 :           emit_insn (gen_absv64qi2 (tmp2, tmp1));
   17714              :         }
   17715              :       else
   17716              :         return false;
   17717              :       break;
   17718              : 
   17719           98 :     case VEC_BCAST_PADDB:
   17720           98 :       if (mode == V4SImode && TARGET_SSE2)
   17721              :         {
   17722           94 :           tmp1 = gen_reg_rtx (V16QImode);
   17723           94 :           emit_move_insn (tmp1, CONSTM1_RTX (V16QImode));
   17724           94 :           tmp2 = gen_reg_rtx (V16QImode);
   17725           94 :           emit_insn (gen_addv16qi3 (tmp2, tmp1, tmp1));
   17726              :         }
   17727            4 :       else if (mode == V8SImode && TARGET_AVX2)
   17728              :         {
   17729            1 :           tmp1 = gen_reg_rtx (V32QImode);
   17730            1 :           emit_move_insn (tmp1, CONSTM1_RTX (V32QImode));
   17731            1 :           tmp2 = gen_reg_rtx (V32QImode);
   17732            1 :           emit_insn (gen_addv32qi3 (tmp2, tmp1, tmp1));
   17733              :         }
   17734            3 :       else if (mode == V16SImode && TARGET_AVX512BW)
   17735              :         {
   17736            3 :           tmp1 = gen_reg_rtx (V64QImode);
   17737            3 :           emit_move_insn (tmp1, CONSTM1_RTX (V64QImode));
   17738            3 :           tmp2 = gen_reg_rtx (V64QImode);
   17739            3 :           emit_insn (gen_addv64qi3 (tmp2, tmp1, tmp1));
   17740              :         }
   17741              :       else
   17742              :         return false;
   17743              :       break;
   17744              : 
   17745         3772 :     case VEC_BCAST_PSRLW:
   17746         3772 :       if (mode == V4SImode && TARGET_SSE2)
   17747              :         {
   17748         3549 :           tmp1 = gen_reg_rtx (V8HImode);
   17749         3549 :           emit_move_insn (tmp1, CONSTM1_RTX (V8HImode));
   17750         3549 :           tmp2 = gen_reg_rtx (V8HImode);
   17751         3549 :           emit_insn (gen_lshrv8hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
   17752              :         }
   17753          223 :       else if (mode == V8SImode && TARGET_AVX2)
   17754              :         {
   17755          129 :           tmp1 = gen_reg_rtx (V16HImode);
   17756          129 :           emit_move_insn (tmp1, CONSTM1_RTX (V16HImode));
   17757          129 :           tmp2 = gen_reg_rtx (V16HImode);
   17758          129 :           emit_insn (gen_lshrv16hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
   17759              :         }
   17760           94 :       else if (mode == V16SImode && TARGET_AVX512BW)
   17761              :         {
   17762           91 :           tmp1 = gen_reg_rtx (V32HImode);
   17763           91 :           emit_move_insn (tmp1, CONSTM1_RTX (V32HImode));
   17764           91 :           tmp2 = gen_reg_rtx (V32HImode);
   17765           91 :           emit_insn (gen_lshrv32hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
   17766              :         }
   17767              :       else
   17768              :         return false;
   17769              :       break;
   17770              : 
   17771        10507 :     case VEC_BCAST_PSRLD:
   17772        10507 :       if (mode == V4SImode && TARGET_SSE2)
   17773              :         {
   17774         7628 :           tmp1 = gen_reg_rtx (V4SImode);
   17775         7628 :           emit_move_insn (tmp1, CONSTM1_RTX (V4SImode));
   17776         7628 :           emit_insn (gen_lshrv4si3 (target, tmp1, GEN_INT (entry->arg)));
   17777         7628 :           return true;
   17778              :         }
   17779         2879 :       else if (mode == V8SImode && TARGET_AVX2)
   17780              :         {
   17781         1064 :           tmp1 = gen_reg_rtx (V8SImode);
   17782         1064 :           emit_move_insn (tmp1, CONSTM1_RTX (V8SImode));
   17783         1064 :           emit_insn (gen_lshrv8si3 (target, tmp1, GEN_INT (entry->arg)));
   17784         1064 :           return true;
   17785              :         }
   17786         1815 :       else if (mode == V16SImode && TARGET_AVX512F)
   17787              :         {
   17788          963 :           tmp1 = gen_reg_rtx (V16SImode);
   17789          963 :           emit_move_insn (tmp1, CONSTM1_RTX (V16SImode));
   17790          963 :           emit_insn (gen_lshrv16si3 (target, tmp1, GEN_INT (entry->arg)));
   17791          963 :           return true;
   17792              :         }
   17793              :       else
   17794              :         return false;
   17795          137 :       break;
   17796              : 
   17797          137 :     case VEC_BCAST_PSLLW:
   17798          137 :       if (mode == V4SImode && TARGET_SSE2)
   17799              :         {
   17800          107 :           tmp1 = gen_reg_rtx (V8HImode);
   17801          107 :           emit_move_insn (tmp1, CONSTM1_RTX (V8HImode));
   17802          107 :           tmp2 = gen_reg_rtx (V8HImode);
   17803          107 :           emit_insn (gen_ashlv8hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
   17804              :         }
   17805           30 :       else if (mode == V8SImode && TARGET_AVX2)
   17806              :         {
   17807           21 :           tmp1 = gen_reg_rtx (V16HImode);
   17808           21 :           emit_move_insn (tmp1, CONSTM1_RTX (V16HImode));
   17809           21 :           tmp2 = gen_reg_rtx (V16HImode);
   17810           21 :           emit_insn (gen_ashlv16hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
   17811              :         }
   17812            9 :       else if (mode == V16SImode && TARGET_AVX512BW)
   17813              :         {
   17814            9 :           tmp1 = gen_reg_rtx (V32HImode);
   17815            9 :           emit_move_insn (tmp1, CONSTM1_RTX (V32HImode));
   17816            9 :           tmp2 = gen_reg_rtx (V32HImode);
   17817            9 :           emit_insn (gen_ashlv32hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
   17818              :         }
   17819              :       else
   17820              :         return false;
   17821              :       break;
   17822              : 
   17823         1834 :     case VEC_BCAST_PSLLD:
   17824         1834 :       if (mode == V4SImode && TARGET_SSE2)
   17825              :         {
   17826         1786 :           tmp1 = gen_reg_rtx (V4SImode);
   17827         1786 :           emit_move_insn (tmp1, CONSTM1_RTX (V4SImode));
   17828         1786 :           emit_insn (gen_ashlv4si3 (target, tmp1, GEN_INT (entry->arg)));
   17829         1786 :           return true;
   17830              :         }
   17831           48 :       else if (mode == V8SImode && TARGET_AVX2)
   17832              :         {
   17833           30 :           tmp1 = gen_reg_rtx (V8SImode);
   17834           30 :           emit_move_insn (tmp1, CONSTM1_RTX (V8SImode));
   17835           30 :           emit_insn (gen_ashlv8si3 (target, tmp1, GEN_INT (entry->arg)));
   17836           30 :           return true;
   17837              :         }
   17838           18 :       else if (mode == V16SImode && TARGET_AVX512F)
   17839              :         {
   17840           18 :           tmp1 = gen_reg_rtx (V16SImode);
   17841           18 :           emit_move_insn (tmp1, CONSTM1_RTX (V16SImode));
   17842           18 :           emit_insn (gen_ashlv16si3 (target, tmp1, GEN_INT (entry->arg)));
   17843           18 :           return true;
   17844              :         }
   17845              :       else
   17846              :         return false;
   17847              : 
   17848              :     default:
   17849              :       return false;
   17850              :     }
   17851              : 
   17852         4649 :   emit_move_insn (target, gen_lowpart (mode, tmp2));
   17853         4649 :   return true;
   17854              : }
   17855              : 
   17856              : /* A subroutine of ix86_expand_vector_init_duplicate.  Tries to
   17857              :    fill target with val via vec_duplicate.  */
   17858              : 
   17859              : static bool
   17860       146388 : ix86_vector_duplicate_value (machine_mode mode, rtx target, rtx val)
   17861              : {
   17862       146388 :   bool ok;
   17863       146388 :   rtx_insn *insn;
   17864       146388 :   rtx dup;
   17865              : 
   17866       146388 :   if ((mode == V4SImode || mode == V8SImode || mode == V16SImode)
   17867        61897 :       && CONST_INT_P (val)
   17868        53907 :       && ix86_vector_duplicate_simode_const (mode, target, INTVAL (val)))
   17869              :     return true;
   17870              : 
   17871              :   /* Save/restore recog_data in case this is called from splitters
   17872              :      or other routines where recog_data needs to stay valid across
   17873              :      force_reg.  See PR106577.  */
   17874       127479 :   recog_state_saver recog_save;
   17875              : 
   17876              :   /* First attempt to recognize VAL as-is.  */
   17877       127479 :   dup = gen_vec_duplicate (mode, val);
   17878       127479 :   insn = emit_insn (gen_rtx_SET (target, dup));
   17879       127479 :   if (recog_memoized (insn) < 0)
   17880              :     {
   17881        98707 :       rtx_insn *seq;
   17882        98707 :       machine_mode innermode = GET_MODE_INNER (mode);
   17883        98707 :       rtx reg;
   17884              : 
   17885              :       /* If that fails, force VAL into a register or mem.  */
   17886              : 
   17887        98707 :       start_sequence ();
   17888              : 
   17889            0 :       if (!TARGET_PREFER_BCST_FROM_INTEGER && CONST_INT_P (val)
   17890            0 :           && GET_MODE_BITSIZE (innermode) <= HOST_BITS_PER_WIDE_INT
   17891        98707 :           && GET_MODE_BITSIZE(mode) >= 128)
   17892            0 :         reg = validize_mem (force_const_mem (innermode, val));
   17893              :       else
   17894              :         {
   17895        98707 :           reg = force_reg (innermode, val);
   17896        98707 :           if (GET_MODE (reg) != innermode)
   17897            0 :             reg = gen_lowpart (innermode, reg);
   17898              :         }
   17899              : 
   17900        98707 :       SET_SRC (PATTERN (insn)) = gen_vec_duplicate (mode, reg);
   17901        98707 :       seq = end_sequence ();
   17902        98707 :       if (seq)
   17903        98707 :         emit_insn_before (seq, insn);
   17904              : 
   17905        98707 :       ok = recog_memoized (insn) >= 0;
   17906        98707 :       gcc_assert (ok);
   17907              :     }
   17908       127479 :   return true;
   17909       127479 : }
   17910              : 
   17911              : /* Get a vector mode of the same size as the original but with elements
   17912              :    twice as wide.  This is only guaranteed to apply to integral vectors.  */
   17913              : 
   17914              : static machine_mode
   17915        27594 : get_mode_wider_vector (machine_mode o)
   17916              : {
   17917              :   /* ??? Rely on the ordering that genmodes.cc gives to vectors.  */
   17918        27594 :   machine_mode n = GET_MODE_NEXT_MODE (o).require ();
   17919        82782 :   gcc_assert (GET_MODE_NUNITS (o) == GET_MODE_NUNITS (n) * 2);
   17920        82782 :   gcc_assert (GET_MODE_SIZE (o) == GET_MODE_SIZE (n));
   17921        27594 :   return n;
   17922              : }
   17923              : 
   17924              : static bool expand_vec_perm_broadcast_1 (struct expand_vec_perm_d *d);
   17925              : static bool expand_vec_perm_1 (struct expand_vec_perm_d *d);
   17926              : 
   17927              : /* A subroutine of ix86_expand_vector_init.  Store into TARGET a vector
   17928              :    with all elements equal to VAR.  Return true if successful.  */
   17929              : 
   17930              : bool
   17931       174936 : ix86_expand_vector_init_duplicate (bool mmx_ok, machine_mode mode,
   17932              :                                    rtx target, rtx val)
   17933              : {
   17934       174936 :   bool ok;
   17935              : 
   17936       174936 :   switch (mode)
   17937              :     {
   17938        71993 :     case E_V2DImode:
   17939        71993 :       if (CONST_INT_P (val))
   17940              :         {
   17941        62778 :           int tmp = (int)INTVAL (val);
   17942        62778 :           if (tmp == (int)(INTVAL (val) >> 32))
   17943              :             {
   17944           78 :               rtx reg = gen_reg_rtx (V4SImode);
   17945           78 :               ok = ix86_vector_duplicate_value (V4SImode, reg,
   17946              :                                                 GEN_INT (tmp));
   17947           78 :               if (ok)
   17948              :                 {
   17949           78 :                   emit_move_insn (target, gen_lowpart (V2DImode, reg));
   17950           78 :                   return true;
   17951              :                 }
   17952              :             }
   17953              :         }
   17954        71915 :       return ix86_vector_duplicate_value (mode, target, val);
   17955              : 
   17956         1088 :     case E_V4DImode:
   17957         1088 :       if (CONST_INT_P (val))
   17958              :         {
   17959          778 :           int tmp = (int)INTVAL (val);
   17960          778 :           if (tmp == (int)(INTVAL (val) >> 32))
   17961              :             {
   17962           54 :               rtx reg = gen_reg_rtx (V8SImode);
   17963           54 :               ok = ix86_vector_duplicate_value (V8SImode, reg,
   17964              :                                                 GEN_INT (tmp));
   17965           54 :               if (ok)
   17966              :                 {
   17967           54 :                   emit_move_insn (target, gen_lowpart (V4DImode, reg));
   17968           54 :                   return true;
   17969              :                 }
   17970              :             }
   17971              :         }
   17972         1034 :       return ix86_vector_duplicate_value (mode, target, val);
   17973              : 
   17974          465 :     case E_V8DImode:
   17975          465 :       if (CONST_INT_P (val))
   17976              :         {
   17977          264 :           int tmp = (int)INTVAL (val);
   17978          264 :           if (tmp == (int)(INTVAL (val) >> 32))
   17979              :             {
   17980           24 :               rtx reg = gen_reg_rtx (V16SImode);
   17981           24 :               ok = ix86_vector_duplicate_value (V16SImode, reg,
   17982              :                                                 GEN_INT (tmp));
   17983           24 :               if (ok)
   17984              :                 {
   17985           24 :                   emit_move_insn (target, gen_lowpart (V8DImode, reg));
   17986           24 :                   return true;
   17987              :                 }
   17988              :             }
   17989              :         }
   17990          441 :       return ix86_vector_duplicate_value (mode, target, val);
   17991              : 
   17992         2535 :     case E_V2SImode:
   17993         2535 :     case E_V2SFmode:
   17994         2535 :       if (!mmx_ok)
   17995              :         return false;
   17996              :       /* FALLTHRU */
   17997              : 
   17998        71847 :     case E_V4DFmode:
   17999        71847 :     case E_V8SFmode:
   18000        71847 :     case E_V8SImode:
   18001        71847 :     case E_V2DFmode:
   18002        71847 :     case E_V4SFmode:
   18003        71847 :     case E_V4SImode:
   18004        71847 :     case E_V16SImode:
   18005        71847 :     case E_V16SFmode:
   18006        71847 :     case E_V8DFmode:
   18007        71847 :       return ix86_vector_duplicate_value (mode, target, val);
   18008              : 
   18009          388 :     case E_V4HImode:
   18010          388 :       if (!mmx_ok)
   18011              :         return false;
   18012          385 :       if (TARGET_SSE || TARGET_3DNOW_A)
   18013              :         {
   18014          385 :           rtx x;
   18015              : 
   18016          385 :           val = gen_lowpart (SImode, val);
   18017          385 :           if (CONST_INT_P (val))
   18018              :             return false;
   18019          383 :           x = gen_rtx_TRUNCATE (HImode, val);
   18020          383 :           x = gen_rtx_VEC_DUPLICATE (mode, x);
   18021          383 :           emit_insn (gen_rtx_SET (target, x));
   18022          383 :           return true;
   18023              :         }
   18024            0 :       goto widen;
   18025              : 
   18026            5 :     case E_V4HFmode:
   18027            5 :     case E_V4BFmode:
   18028            5 :       if (TARGET_MMX_WITH_SSE)
   18029              :         {
   18030           10 :           val = force_reg (GET_MODE_INNER (mode), val);
   18031            5 :           rtx x = gen_rtx_VEC_DUPLICATE (mode, val);
   18032            5 :           emit_insn (gen_rtx_SET (target, x));
   18033            5 :           return true;
   18034              :         }
   18035              :       return false;
   18036              : 
   18037          127 :     case E_V2HImode:
   18038          127 :       if (TARGET_SSE2)
   18039              :         {
   18040          127 :           rtx x;
   18041              : 
   18042          127 :           val = gen_lowpart (SImode, val);
   18043          127 :           if (CONST_INT_P (val))
   18044              :             return false;
   18045          127 :           x = gen_rtx_TRUNCATE (HImode, val);
   18046          127 :           x = gen_rtx_VEC_DUPLICATE (mode, x);
   18047          127 :           emit_insn (gen_rtx_SET (target, x));
   18048          127 :           return true;
   18049              :         }
   18050              :       return false;
   18051              : 
   18052            5 :     case E_V2HFmode:
   18053            5 :     case E_V2BFmode:
   18054            5 :       if (TARGET_SSE2)
   18055              :         {
   18056           10 :           val = force_reg (GET_MODE_INNER (mode), val);
   18057            5 :           rtx x = gen_rtx_VEC_DUPLICATE (mode, val);
   18058            5 :           emit_insn (gen_rtx_SET (target, x));
   18059            5 :           return true;
   18060              :         }
   18061              :       return false;
   18062              : 
   18063          291 :     case E_V8QImode:
   18064          291 :     case E_V4QImode:
   18065          291 :       if (!mmx_ok)
   18066              :         return false;
   18067          290 :       goto widen;
   18068              : 
   18069        14403 :     case E_V8HImode:
   18070        14403 :       if (CONST_INT_P (val))
   18071        13855 :         goto widen;
   18072              :       /* FALLTHRU */
   18073              : 
   18074          862 :     case E_V8HFmode:
   18075          862 :     case E_V8BFmode:
   18076          862 :       if (TARGET_AVX2)
   18077          396 :         return ix86_vector_duplicate_value (mode, target, val);
   18078              : 
   18079          466 :       if (TARGET_SSE2)
   18080              :         {
   18081         1201 :           struct expand_vec_perm_d dperm;
   18082         1201 :           rtx tmp1, tmp2;
   18083              : 
   18084          466 :         permute:
   18085         1201 :           memset (&dperm, 0, sizeof (dperm));
   18086         1201 :           dperm.target = target;
   18087         1201 :           dperm.vmode = mode;
   18088         1201 :           dperm.nelt = GET_MODE_NUNITS (mode);
   18089         1201 :           dperm.op0 = dperm.op1 = gen_reg_rtx (mode);
   18090         1201 :           dperm.one_operand_p = true;
   18091              : 
   18092         1201 :           if (mode == V8HFmode || mode == V8BFmode)
   18093              :             {
   18094            6 :               tmp1 = force_reg (GET_MODE_INNER (mode), val);
   18095            3 :               tmp2 = gen_reg_rtx (mode);
   18096            3 :               emit_insn (gen_vec_set_0 (mode, tmp2, CONST0_RTX (mode), tmp1));
   18097            3 :               tmp1 = gen_lowpart (mode, tmp2);
   18098              :             }
   18099              :           else
   18100              :             {
   18101              :               /* Extend to SImode using a paradoxical SUBREG.  */
   18102         1198 :               tmp1 = gen_reg_rtx (SImode);
   18103         1198 :               emit_move_insn (tmp1, gen_lowpart (SImode, val));
   18104              : 
   18105              :               /* Insert the SImode value as
   18106              :                  low element of a V4SImode vector.  */
   18107         1198 :               tmp2 = gen_reg_rtx (V4SImode);
   18108         1198 :               emit_insn (gen_vec_setv4si_0 (tmp2, CONST0_RTX (V4SImode), tmp1));
   18109         1198 :               tmp1 = gen_lowpart (mode, tmp2);
   18110              :             }
   18111              : 
   18112         1201 :           emit_move_insn (dperm.op0, tmp1);
   18113         1201 :           ok = (expand_vec_perm_1 (&dperm)
   18114         1201 :                 || expand_vec_perm_broadcast_1 (&dperm));
   18115            0 :           gcc_assert (ok);
   18116         1201 :           return ok;
   18117              :         }
   18118            0 :       goto widen;
   18119              : 
   18120         9921 :     case E_V16QImode:
   18121         9921 :       if (CONST_INT_P (val))
   18122         9125 :         goto widen;
   18123          796 :       if (TARGET_AVX2)
   18124           61 :         return ix86_vector_duplicate_value (mode, target, val);
   18125              : 
   18126          735 :       if (TARGET_SSE2)
   18127          735 :         goto permute;
   18128            0 :       goto widen;
   18129              : 
   18130        25942 :     widen:
   18131              :       /* Replicate the value once into the next wider mode and recurse.  */
   18132        25942 :       {
   18133        25942 :         machine_mode smode, wsmode, wvmode;
   18134        25942 :         rtx x;
   18135              : 
   18136        25942 :         smode = GET_MODE_INNER (mode);
   18137        25942 :         wvmode = get_mode_wider_vector (mode);
   18138        25942 :         wsmode = GET_MODE_INNER (wvmode);
   18139              : 
   18140        25942 :         val = convert_modes (wsmode, smode, val, true);
   18141              : 
   18142        25942 :         if (CONST_INT_P (val))
   18143              :           {
   18144        51306 :             x = simplify_binary_operation (ASHIFT, wsmode, val,
   18145        25653 :                                            GEN_INT (GET_MODE_BITSIZE (smode)));
   18146        25653 :             val = simplify_binary_operation (IOR, wsmode, val, x);
   18147              :           }
   18148          289 :         else if (smode == QImode && !TARGET_PARTIAL_REG_STALL)
   18149          289 :           emit_insn (gen_insv_1 (wsmode, val, val));
   18150              :         else
   18151              :           {
   18152            0 :             x = expand_simple_binop (wsmode, ASHIFT, val,
   18153            0 :                                      GEN_INT (GET_MODE_BITSIZE (smode)),
   18154              :                                      NULL_RTX, 1, OPTAB_LIB_WIDEN);
   18155            0 :             val = expand_simple_binop (wsmode, IOR, val, x, x, 1,
   18156              :                                        OPTAB_LIB_WIDEN);
   18157              :           }
   18158              : 
   18159        25942 :         x = gen_reg_rtx (wvmode);
   18160        25942 :         ok = ix86_expand_vector_init_duplicate (mmx_ok, wvmode, x, val);
   18161        25942 :         if (!ok)
   18162              :           return false;
   18163        25941 :         emit_move_insn (target, gen_lowpart (GET_MODE (target), x));
   18164        25941 :         return true;
   18165              :       }
   18166              : 
   18167         1729 :     case E_V16HImode:
   18168         1729 :     case E_V32QImode:
   18169         1729 :       if (CONST_INT_P (val))
   18170         1438 :         goto widen;
   18171              :       /* FALLTHRU */
   18172              : 
   18173          374 :     case E_V16HFmode:
   18174          374 :     case E_V16BFmode:
   18175          374 :       if (TARGET_AVX2)
   18176          348 :         return ix86_vector_duplicate_value (mode, target, val);
   18177              :       else
   18178              :         {
   18179           26 :           machine_mode hvmode;
   18180           26 :           switch (mode)
   18181              :             {
   18182              :             case V16HImode:
   18183              :               hvmode = V8HImode;
   18184              :               break;
   18185            0 :             case V16HFmode:
   18186            0 :               hvmode = V8HFmode;
   18187            0 :               break;
   18188            1 :             case V16BFmode:
   18189            1 :               hvmode = V8BFmode;
   18190            1 :               break;
   18191           12 :             case V32QImode:
   18192           12 :               hvmode = V16QImode;
   18193           12 :               break;
   18194              :             default:
   18195              :               gcc_unreachable ();
   18196              :             }
   18197           26 :           rtx x = gen_reg_rtx (hvmode);
   18198              : 
   18199           26 :           ok = ix86_expand_vector_init_duplicate (false, hvmode, x, val);
   18200           26 :           if (!ok)
   18201              :             return false;
   18202              : 
   18203           26 :           x = gen_rtx_VEC_CONCAT (mode, x, x);
   18204           26 :           emit_insn (gen_rtx_SET (target, x));
   18205              :         }
   18206           26 :       return true;
   18207              : 
   18208         1363 :     case E_V32HImode:
   18209         1363 :     case E_V64QImode:
   18210         1363 :       if (CONST_INT_P (val))
   18211         1234 :         goto widen;
   18212              :       /* FALLTHRU */
   18213              : 
   18214          210 :     case E_V32HFmode:
   18215          210 :     case E_V32BFmode:
   18216          210 :       if (TARGET_AVX512BW)
   18217          190 :         return ix86_vector_duplicate_value (mode, target, val);
   18218              :       else
   18219              :         {
   18220           20 :           machine_mode hvmode;
   18221           20 :           switch (mode)
   18222              :             {
   18223              :             case V32HImode:
   18224              :               hvmode = V16HImode;
   18225              :               break;
   18226            0 :             case V32HFmode:
   18227            0 :               hvmode = V16HFmode;
   18228            0 :               break;
   18229            1 :             case V32BFmode:
   18230            1 :               hvmode = V16BFmode;
   18231            1 :               break;
   18232           10 :             case V64QImode:
   18233           10 :               hvmode = V32QImode;
   18234           10 :               break;
   18235              :             default:
   18236              :               gcc_unreachable ();
   18237              :             }
   18238           20 :           rtx x = gen_reg_rtx (hvmode);
   18239              : 
   18240           20 :           ok = ix86_expand_vector_init_duplicate (false, hvmode, x, val);
   18241           20 :           if (!ok)
   18242              :             return false;
   18243              : 
   18244           20 :           x = gen_rtx_VEC_CONCAT (mode, x, x);
   18245           20 :           emit_insn (gen_rtx_SET (target, x));
   18246              :         }
   18247           20 :       return true;
   18248              : 
   18249              :     default:
   18250              :       return false;
   18251              :     }
   18252              : }
   18253              : 
   18254              : /* A subroutine of ix86_expand_vector_init.  Store into TARGET a vector
   18255              :    whose ONE_VAR element is VAR, and other elements are zero.  Return true
   18256              :    if successful.  */
   18257              : 
   18258              : bool
   18259        13349 : ix86_expand_vector_init_one_nonzero (bool mmx_ok, machine_mode mode,
   18260              :                                      rtx target, rtx var, int one_var)
   18261              : {
   18262        13349 :   rtx x, tmp;
   18263        13349 :   bool use_vector_set = false;
   18264        13349 :   rtx (*gen_vec_set_0) (rtx, rtx, rtx) = NULL;
   18265              : 
   18266        13349 :   switch (mode)
   18267              :     {
   18268         8509 :     case E_V2DImode:
   18269         8509 :       if (TARGET_64BIT || MEM_P (var))
   18270              :         {
   18271         8500 :           if (!REG_P (var) && !MEM_P (var))
   18272         5612 :             var = force_reg (DImode, var);
   18273         8500 :           x = gen_rtx_VEC_CONCAT (V2DImode, var, CONST0_RTX (DImode));
   18274         8500 :           if (!one_var)
   18275         2350 :             emit_insn (gen_rtx_SET (target, x));
   18276         6150 :           else if (TARGET_SSE2)
   18277              :             {
   18278         6150 :               tmp = gen_reg_rtx (V2DImode);
   18279         6150 :               emit_insn (gen_rtx_SET (tmp, x));
   18280         6150 :               emit_insn (gen_vec_shl_v2di (target, tmp, GEN_INT (64)));
   18281              :             }
   18282              :           else
   18283              :             {
   18284            0 :               rtx tmp1 = gen_reg_rtx (V2DImode);
   18285            0 :               emit_insn (gen_rtx_SET (tmp1, x));
   18286            0 :               rtx tmp2 = gen_reg_rtx (V4SImode);
   18287            0 :               emit_move_insn (tmp2, gen_lowpart (V4SImode, tmp1));
   18288            0 :               emit_insn (gen_sse_shufps_v4si (tmp2, tmp2, tmp2,
   18289              :                                               GEN_INT (2), GEN_INT (3),
   18290              :                                               GEN_INT (4), GEN_INT (5)));
   18291            0 :               emit_move_insn (target, gen_lowpart (V2DImode, tmp2));
   18292              :             }
   18293              :         }
   18294              :       else
   18295              :         {
   18296            9 :           rtx lo = force_reg (SImode, gen_lowpart (SImode, var));
   18297            9 :           rtx hi = force_reg (SImode, gen_highpart (SImode, var));
   18298            9 :           tmp = gen_reg_rtx (V4SImode);
   18299            9 :           if (TARGET_SSE4_1)
   18300              :             {
   18301            0 :               rtx tmp1 = gen_reg_rtx (V4SImode);
   18302            0 :               emit_insn (gen_vec_setv4si_0 (tmp1, CONST0_RTX (V4SImode), lo));
   18303            0 :               emit_insn (gen_sse4_1_pinsrd (tmp, tmp1, hi, GEN_INT (2)));
   18304              :             }
   18305              :           else
   18306              :             {
   18307            9 :               rtx ltmp = gen_reg_rtx (V4SImode);
   18308            9 :               rtx htmp = gen_reg_rtx (V4SImode);
   18309            9 :               emit_insn (gen_vec_setv4si_0 (ltmp, CONST0_RTX (V4SImode), lo));
   18310            9 :               emit_insn (gen_vec_setv4si_0 (htmp, CONST0_RTX (V4SImode), hi));
   18311            9 :               emit_insn (gen_vec_interleave_lowv4si (tmp, ltmp, htmp));
   18312              :             }
   18313            9 :           if (!one_var)
   18314            3 :             emit_move_insn (target, gen_lowpart (V2DImode, tmp));
   18315            6 :           else if (TARGET_SSE2)
   18316              :             {
   18317            6 :               rtx tmp2 = gen_reg_rtx (V2DImode);
   18318            6 :               emit_move_insn (tmp2, gen_lowpart (V2DImode, tmp));
   18319            6 :               emit_insn (gen_vec_shl_v2di (target, tmp2, GEN_INT (64)));
   18320              :             }
   18321              :           else
   18322              :             {
   18323            0 :               rtx tmp2 = gen_reg_rtx (V2DImode);
   18324            0 :               emit_insn (gen_sse_shufps_v4si (tmp2, tmp, tmp,
   18325              :                                               GEN_INT (2), GEN_INT (3),
   18326              :                                               GEN_INT (4), GEN_INT (5)));
   18327            0 :               emit_move_insn (target, gen_lowpart (V2DImode, tmp2));
   18328              :             }
   18329              :         }
   18330              :       return true;
   18331          106 :     case E_V2DFmode:
   18332          106 :       if (!REG_P (var) && !MEM_P (var))
   18333            0 :         var = force_reg (DFmode, var);
   18334          106 :       x = gen_rtx_VEC_CONCAT (V2DFmode, var, CONST0_RTX (DFmode));
   18335          106 :       if (one_var)
   18336              :         {
   18337           27 :           tmp = gen_reg_rtx (V2DFmode);
   18338           27 :           emit_insn (gen_rtx_SET (tmp, x));
   18339           27 :           emit_insn (gen_vec_shl_v2df (target, tmp, GEN_INT (64)));
   18340              :         }
   18341              :       else
   18342           79 :         emit_insn (gen_rtx_SET (target, x));
   18343              :       return true;
   18344          358 :     case E_V4SImode:
   18345          358 :       var = force_reg (SImode, var);
   18346          358 :       x = gen_rtx_VEC_DUPLICATE (V4SImode, var);
   18347          358 :       x = gen_rtx_VEC_MERGE (V4SImode, x, CONST0_RTX (V4SImode), const1_rtx);
   18348          358 :       if (!one_var)
   18349          308 :         emit_insn (gen_rtx_SET (target, x));
   18350           50 :       else if (TARGET_SSE2)
   18351              :         {
   18352           47 :           rtx tmp = gen_reg_rtx (V4SImode);
   18353           47 :           emit_insn (gen_rtx_SET (tmp, x));
   18354           47 :           emit_insn (gen_vec_shl_v4si (target, tmp, GEN_INT (one_var * 32)));
   18355              :         }
   18356              :       else
   18357              :         {
   18358            3 :           rtx tmp = gen_reg_rtx (V4SImode);
   18359            3 :           emit_insn (gen_rtx_SET (tmp, x));
   18360           10 :           emit_insn (gen_sse_shufps_v4si (target, tmp, tmp,
   18361              :                                           const1_rtx,
   18362            3 :                                           GEN_INT (one_var == 1 ? 0 : 1),
   18363              :                                           GEN_INT (one_var == 2 ? 0+4 : 1+4),
   18364              :                                           GEN_INT (one_var == 3 ? 0+4 : 1+4)));
   18365              :         }
   18366              :       return true;
   18367         3174 :     case E_V4SFmode:
   18368         3174 :       if (TARGET_SSE4_1)
   18369              :         {
   18370          522 :           if (!REG_P (var) && !MEM_P (var))
   18371            0 :             var = force_reg (SFmode, var);
   18372          522 :           if (one_var)
   18373           12 :             emit_insn (gen_sse4_1_insertps_v4sf_init (target, var,
   18374              :                                                       CONST0_RTX (V4SFmode),
   18375           12 :                                                       GEN_INT (1 << one_var)));
   18376              :           else
   18377          510 :             emit_insn (gen_vec_setv4sf_0 (target, CONST0_RTX (V4SFmode), var));
   18378              :           return true;
   18379              :         }
   18380         2652 :       if (TARGET_SSE2 && REG_P (var))
   18381              :         {
   18382           56 :           if (one_var == 3)
   18383            1 :             emit_insn (gen_sse2_insertps_v4sf_3 (target, var,
   18384              :                                                  CONST0_RTX (V4SFmode)));
   18385              :           else
   18386              :             {
   18387           55 :               rtx tmp = gen_reg_rtx (V4SFmode);
   18388           55 :               emit_insn (gen_sse2_insertps_v4sf_3 (tmp, var,
   18389              :                                                    CONST0_RTX (V4SFmode)));
   18390           55 :               emit_insn (gen_vec_shr_v4sf (target, tmp,
   18391           55 :                                            GEN_INT ((3 - one_var) * 32)));
   18392              :             }
   18393              :           return true;
   18394              :         }
   18395         2596 :       if (!one_var)
   18396         2580 :         emit_insn (gen_vec_setv4sf_0 (target, CONST0_RTX (V4SFmode), var));
   18397           16 :       else if (TARGET_SSE2)
   18398              :         {
   18399           13 :           rtx tmp = gen_reg_rtx (V4SFmode);
   18400           13 :           emit_insn (gen_vec_setv4sf_0 (tmp, CONST0_RTX (V4SFmode), var));
   18401           13 :           emit_insn (gen_vec_shl_v4sf (target, tmp, GEN_INT (one_var * 32)));
   18402              :         }
   18403              :       else
   18404              :         {
   18405            3 :           rtx tmp = gen_reg_rtx (V4SFmode);
   18406            3 :           emit_insn (gen_vec_setv4sf_0 (tmp, CONST0_RTX (V4SFmode), var));
   18407           10 :           emit_insn (gen_sse_shufps_v4sf (target, tmp, tmp,
   18408              :                                           const1_rtx,
   18409            3 :                                           GEN_INT (one_var == 1 ? 0 : 1),
   18410              :                                           GEN_INT (one_var == 2 ? 0+4 : 1+4),
   18411              :                                           GEN_INT (one_var == 3 ? 0+4 : 1+4)));
   18412              :         }
   18413              :       return true;
   18414            7 :     case E_V4DImode:
   18415            7 :       if (TARGET_AVX2 && (TARGET_64BIT || MEM_P (var)))
   18416              :         {
   18417            6 :           if (!REG_P (var) && !MEM_P (var))
   18418            0 :             var = force_reg (DImode, var);
   18419            6 :           x = gen_rtx_VEC_DUPLICATE (V4DImode, var);
   18420            6 :           x = gen_rtx_VEC_MERGE (V4DImode, x, CONST0_RTX (V4DImode),
   18421              :                                  const1_rtx);
   18422            6 :           if (one_var)
   18423              :             {
   18424            0 :               tmp = gen_reg_rtx (V4DImode);
   18425            0 :               emit_insn (gen_rtx_SET (tmp, x));
   18426            0 :               emit_insn (gen_avx2_permv4di_1 (target, tmp,
   18427              :                                               const1_rtx,
   18428            0 :                                               GEN_INT (one_var == 1 ? 0 : 1),
   18429            0 :                                               GEN_INT (one_var == 2 ? 0 : 1),
   18430            0 :                                               GEN_INT (one_var == 3 ? 0 : 1)));
   18431              :             }
   18432              :           else
   18433            6 :             emit_insn (gen_rtx_SET (target, x));
   18434              :         }
   18435              :       else
   18436              :         {
   18437            1 :           tmp = gen_reg_rtx (V2DImode);
   18438            1 :           if (!ix86_expand_vector_init_one_nonzero (mmx_ok, V2DImode, tmp,
   18439              :                                                     var, one_var & 1))
   18440            0 :             gcc_unreachable ();
   18441            1 :           rtx zero = CONST0_RTX (V2DImode);
   18442            1 :           if (one_var >= 2)
   18443              :             {
   18444            0 :               zero = force_reg (V2DImode, zero);
   18445            0 :               emit_insn (gen_avx_vec_concatv4di (target, zero, tmp));
   18446              :             }
   18447              :           else
   18448            1 :             emit_insn (gen_avx_vec_concatv4di (target, tmp, zero));
   18449              :         }
   18450              :       return true;
   18451           13 :     case E_V4DFmode:
   18452           13 :       if (TARGET_AVX2)
   18453              :         {
   18454           13 :           if (!REG_P (var) && !MEM_P (var))
   18455            0 :             var = force_reg (DFmode, var);
   18456           13 :           x = gen_rtx_VEC_DUPLICATE (V4DFmode, var);
   18457           13 :           x = gen_rtx_VEC_MERGE (V4DFmode, x, CONST0_RTX (V4DFmode),
   18458              :                                  const1_rtx);
   18459           13 :           if (one_var)
   18460              :             {
   18461            0 :               tmp = gen_reg_rtx (V4DFmode);
   18462            0 :               emit_insn (gen_rtx_SET (tmp, x));
   18463            0 :               emit_insn (gen_avx2_permv4df_1 (target, tmp,
   18464              :                                               const1_rtx,
   18465            0 :                                               GEN_INT (one_var == 1 ? 0 : 1),
   18466            0 :                                               GEN_INT (one_var == 2 ? 0 : 1),
   18467            0 :                                               GEN_INT (one_var == 3 ? 0 : 1)));
   18468              :             }
   18469              :           else
   18470           13 :             emit_insn (gen_rtx_SET (target, x));
   18471              :         }
   18472              :       else
   18473              :         {
   18474            0 :           tmp = gen_reg_rtx (V2DFmode);
   18475            0 :           if (!ix86_expand_vector_init_one_nonzero (mmx_ok, V2DFmode, tmp,
   18476              :                                                     var, one_var & 1))
   18477            0 :             gcc_unreachable ();
   18478            0 :           rtx zero = CONST0_RTX (V2DFmode);
   18479            0 :           if (one_var >= 2)
   18480              :             {
   18481            0 :               zero = force_reg (V2DFmode, zero);
   18482            0 :               emit_insn (gen_avx_vec_concatv4df (target, zero, tmp));
   18483              :             }
   18484              :           else
   18485            0 :             emit_insn (gen_avx_vec_concatv4df (target, tmp, zero));
   18486              :         }
   18487              :       return true;
   18488           86 :     case E_V16QImode:
   18489           86 :       use_vector_set = TARGET_SSE4_1;
   18490           86 :       break;
   18491          102 :     case E_V8HImode:
   18492          102 :       use_vector_set = TARGET_SSE2;
   18493          102 :       gen_vec_set_0 = TARGET_AVX512FP16 && one_var == 0
   18494          102 :         ? gen_vec_setv8hi_0 : NULL;
   18495              :       break;
   18496           15 :     case E_V8QImode:
   18497           15 :       use_vector_set = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
   18498              :       break;
   18499           14 :     case E_V4HImode:
   18500           14 :     case E_V4HFmode:
   18501           14 :     case E_V4BFmode:
   18502           14 :       use_vector_set = TARGET_SSE || TARGET_3DNOW_A;
   18503              :       break;
   18504           32 :     case E_V4QImode:
   18505           32 :       use_vector_set = TARGET_SSE4_1;
   18506           32 :       break;
   18507            0 :     case E_V32QImode:
   18508            0 :       use_vector_set = TARGET_AVX;
   18509            0 :       break;
   18510            5 :     case E_V16HImode:
   18511            5 :       use_vector_set = TARGET_AVX;
   18512            5 :       gen_vec_set_0 = TARGET_AVX512FP16 && one_var == 0
   18513            5 :         ? gen_vec_setv16hi_0 : NULL;
   18514              :       break;
   18515            5 :     case E_V8SImode:
   18516            5 :       use_vector_set = TARGET_AVX;
   18517            5 :       gen_vec_set_0 = gen_vec_setv8si_0;
   18518            5 :       break;
   18519           22 :     case E_V8SFmode:
   18520           22 :       use_vector_set = TARGET_AVX;
   18521           22 :       gen_vec_set_0 = gen_vec_setv8sf_0;
   18522           22 :       break;
   18523           17 :     case E_V16SImode:
   18524           17 :       use_vector_set = TARGET_AVX512F && one_var == 0;
   18525              :       gen_vec_set_0 = gen_vec_setv16si_0;
   18526              :       break;
   18527           22 :     case E_V16SFmode:
   18528           22 :       use_vector_set = TARGET_AVX512F && one_var == 0;
   18529              :       gen_vec_set_0 = gen_vec_setv16sf_0;
   18530              :       break;
   18531            0 :     case E_V8DFmode:
   18532            0 :       use_vector_set = TARGET_AVX512F && one_var == 0;
   18533              :       gen_vec_set_0 = gen_vec_setv8df_0;
   18534              :       break;
   18535            2 :     case E_V8DImode:
   18536              :       /* Use ix86_expand_vector_set in 64bit mode only.  */
   18537            2 :       use_vector_set = TARGET_AVX512F && TARGET_64BIT && one_var == 0;
   18538              :       gen_vec_set_0 = gen_vec_setv8di_0;
   18539              :       break;
   18540           39 :     case E_V8HFmode:
   18541           39 :       use_vector_set = TARGET_AVX512FP16 && one_var == 0;
   18542              :       gen_vec_set_0 = gen_vec_setv8hf_0;
   18543              :       break;
   18544            9 :     case E_V16HFmode:
   18545            9 :       use_vector_set = TARGET_AVX512FP16 && one_var == 0;
   18546              :       gen_vec_set_0 = gen_vec_setv16hf_0;
   18547              :       break;
   18548            6 :     case E_V32HFmode:
   18549            6 :       use_vector_set = TARGET_AVX512FP16 && one_var == 0;
   18550              :       gen_vec_set_0 = gen_vec_setv32hf_0;
   18551              :       break;
   18552            2 :     case E_V8BFmode:
   18553            2 :       use_vector_set = TARGET_AVX512FP16 && one_var == 0;
   18554              :       gen_vec_set_0 = gen_vec_setv8bf_0;
   18555              :       break;
   18556            0 :     case E_V16BFmode:
   18557            0 :       use_vector_set = TARGET_AVX512FP16 && one_var == 0;
   18558              :       gen_vec_set_0 = gen_vec_setv16bf_0;
   18559              :       break;
   18560            0 :     case E_V32BFmode:
   18561            0 :       use_vector_set = TARGET_AVX512FP16 && one_var == 0;
   18562              :       gen_vec_set_0 = gen_vec_setv32bf_0;
   18563              :       break;
   18564            4 :     case E_V32HImode:
   18565            4 :       use_vector_set = TARGET_AVX512FP16 && one_var == 0;
   18566              :       gen_vec_set_0 = gen_vec_setv32hi_0;
   18567              :     default:
   18568              :       break;
   18569              :     }
   18570              : 
   18571          252 :   if (use_vector_set)
   18572              :     {
   18573          301 :       if (gen_vec_set_0 && one_var == 0)
   18574              :         {
   18575          316 :           var = force_reg (GET_MODE_INNER (mode), var);
   18576          158 :           emit_insn (gen_vec_set_0 (target, CONST0_RTX (mode), var));
   18577          158 :           return true;
   18578              :         }
   18579          143 :       emit_insn (gen_rtx_SET (target, CONST0_RTX (mode)));
   18580          286 :       var = force_reg (GET_MODE_INNER (mode), var);
   18581          143 :       ix86_expand_vector_set (mmx_ok, target, var, one_var);
   18582          143 :       return true;
   18583              :     }
   18584              : 
   18585          881 :   switch (mode)
   18586              :     {
   18587          780 :     case E_V2SFmode:
   18588          780 :     case E_V2SImode:
   18589          780 :       if (!mmx_ok || one_var != 0)
   18590              :         return false;
   18591          378 :       var = force_reg (GET_MODE_INNER (mode), var);
   18592          378 :       x = gen_rtx_VEC_CONCAT (mode, var, CONST0_RTX (GET_MODE_INNER (mode)));
   18593          189 :       emit_insn (gen_rtx_SET (target, x));
   18594          189 :       return true;
   18595              : 
   18596            0 :     case E_V2DFmode:
   18597            0 :     case E_V2DImode:
   18598            0 :       if (one_var != 0)
   18599              :         return false;
   18600            0 :       var = force_reg (GET_MODE_INNER (mode), var);
   18601            0 :       x = gen_rtx_VEC_CONCAT (mode, var, CONST0_RTX (GET_MODE_INNER (mode)));
   18602            0 :       emit_insn (gen_rtx_SET (target, x));
   18603            0 :       return true;
   18604              : 
   18605            0 :     case E_V8HImode:
   18606            0 :       if (one_var > 1 && !TARGET_SSE2)
   18607              :         return false;
   18608              :       /* Zero extend the variable element to SImode and recurse.  */
   18609            0 :       var = convert_modes (SImode, HImode, var, true);
   18610            0 :       var = force_reg (SImode, var);
   18611            0 :       if (one_var == 1)
   18612              :         {
   18613            0 :           var = simplify_gen_binary (ASHIFT, SImode, var, GEN_INT (16));
   18614            0 :           var = force_reg (SImode, var);
   18615            0 :           one_var = 0;
   18616              :         }
   18617              :       else
   18618            0 :         one_var *= 16;
   18619            0 :       x = gen_reg_rtx (V4SImode);
   18620            0 :       if (!ix86_expand_vector_init_one_nonzero (mmx_ok, V4SImode, x, var, 0))
   18621            0 :         gcc_unreachable ();
   18622            0 :       if (one_var)
   18623              :         {
   18624            0 :           tmp = gen_reg_rtx (V4SImode);
   18625            0 :           emit_insn (gen_vec_shl_v4si (tmp, x, GEN_INT (one_var)));
   18626            0 :           x = tmp;
   18627              :         }
   18628            0 :       emit_move_insn (target, gen_lowpart (mode, x));
   18629            0 :       return true;
   18630              : 
   18631           30 :     case E_V16QImode:
   18632           30 :       if (one_var > 3 && !TARGET_SSE2)
   18633              :         return false;
   18634              :       /* Zero extend the variable element to SImode and recurse.  */
   18635           30 :       var = convert_modes (SImode, QImode, var, true);
   18636           30 :       var = force_reg (SImode, var);
   18637           30 :       if (one_var < 4)
   18638              :         {
   18639           14 :           var = simplify_gen_binary (ASHIFT, SImode, var,
   18640           14 :                                      GEN_INT (one_var * 8));
   18641           14 :           var = force_reg (SImode, var);
   18642           14 :           one_var = 0;
   18643              :         }
   18644              :       else
   18645           16 :         one_var *= 8;
   18646           30 :       x = gen_reg_rtx (V4SImode);
   18647           30 :       if (!ix86_expand_vector_init_one_nonzero (mmx_ok, V4SImode, x, var, 0))
   18648            0 :         gcc_unreachable ();
   18649           30 :       if (one_var)
   18650              :         {
   18651           16 :           tmp = gen_reg_rtx (V4SImode);
   18652           16 :           emit_insn (gen_vec_shl_v4si (tmp, x, GEN_INT (one_var)));
   18653           16 :           x = tmp;
   18654              :         }
   18655           30 :       emit_move_insn (target, gen_lowpart (V16QImode, x));
   18656           30 :       return true;
   18657              : 
   18658            0 :     case E_V4HImode:
   18659            0 :       if (!mmx_ok || one_var > 1)
   18660              :         return false;
   18661              :       /* Zero extend the variable element to SImode and recurse.  */
   18662            0 :       var = convert_modes (SImode, HImode, var, true);
   18663            0 :       var = force_reg (SImode, var);
   18664            0 :       if (one_var == 1)
   18665              :         {
   18666            0 :           var = simplify_gen_binary (ASHIFT, SImode, var, GEN_INT (16));
   18667            0 :           var = force_reg (SImode, var);
   18668              :         }
   18669            0 :       x = gen_reg_rtx (V2SImode);
   18670            0 :       if (!ix86_expand_vector_init_one_nonzero (true, V2SImode, x, var, 0))
   18671            0 :         gcc_unreachable ();
   18672            0 :       emit_move_insn (target, gen_lowpart (V4HImode, x));
   18673            0 :       return true;
   18674              : 
   18675           14 :     case E_V8QImode:
   18676           14 :       if (!mmx_ok || one_var > 3)
   18677              :         return false;
   18678              :       /* Zero extend the variable element to SImode and recurse.  */
   18679           14 :       var = convert_modes (SImode, QImode, var, true);
   18680           14 :       var = force_reg (SImode, var);
   18681           14 :       if (one_var)
   18682              :         {
   18683            0 :           var = simplify_gen_binary (ASHIFT, SImode, var,
   18684            0 :                                      GEN_INT (one_var * 8));
   18685            0 :           var = force_reg (SImode, var);
   18686              :         }
   18687           14 :       x = gen_reg_rtx (V2SImode);
   18688           14 :       if (!ix86_expand_vector_init_one_nonzero (true, V2SImode, x, var, 0))
   18689            0 :         gcc_unreachable ();
   18690           14 :       emit_move_insn (target, gen_lowpart (V8QImode, x));
   18691           14 :       return true;
   18692              : 
   18693              :     default:
   18694              :       return false;
   18695              :     }
   18696              : }
   18697              : 
   18698              : /* A subroutine of ix86_expand_vector_init.  Store into TARGET a vector
   18699              :    consisting of the values in VALS.  It is known that all elements
   18700              :    except ONE_VAR are constants.  Return true if successful.  */
   18701              : 
   18702              : static bool
   18703         1683 : ix86_expand_vector_init_one_var (bool mmx_ok, machine_mode mode,
   18704              :                                  rtx target, rtx vals, int one_var)
   18705              : {
   18706         1683 :   rtx var = XVECEXP (vals, 0, one_var);
   18707         1683 :   machine_mode wmode;
   18708         1683 :   rtx const_vec, x;
   18709              : 
   18710         1683 :   const_vec = copy_rtx (vals);
   18711         1683 :   XVECEXP (const_vec, 0, one_var) = CONST0_RTX (GET_MODE_INNER (mode));
   18712         1683 :   const_vec = gen_rtx_CONST_VECTOR (mode, XVEC (const_vec, 0));
   18713              : 
   18714         1683 :   switch (mode)
   18715              :     {
   18716              :     case E_V2DFmode:
   18717              :     case E_V2DImode:
   18718              :     case E_V2SFmode:
   18719              :     case E_V2SImode:
   18720              :       /* For the two element vectors, it's just as easy to use
   18721              :          the general case.  */
   18722              :       return false;
   18723              : 
   18724            3 :     case E_V4DImode:
   18725              :       /* Use ix86_expand_vector_set in 64bit mode only.  */
   18726            3 :       if (!TARGET_64BIT)
   18727              :         return false;
   18728              :       /* FALLTHRU */
   18729              :     case E_V8HFmode:
   18730              :     case E_V16HFmode:
   18731              :     case E_V8BFmode:
   18732              :     case E_V16BFmode:
   18733              :     case E_V4DFmode:
   18734              :     case E_V8SFmode:
   18735              :     case E_V8SImode:
   18736              :     case E_V16HImode:
   18737              :     case E_V32QImode:
   18738              :     case E_V4SFmode:
   18739              :     case E_V4SImode:
   18740              :     case E_V8HImode:
   18741              :     case E_V4HImode:
   18742              :     case E_V4HFmode:
   18743              :     case E_V4BFmode:
   18744              :       break;
   18745              : 
   18746            1 :     case E_V16QImode:
   18747            1 :       if (TARGET_SSE4_1)
   18748              :         break;
   18749            1 :       wmode = V8HImode;
   18750            1 :       goto widen;
   18751            0 :     case E_V8QImode:
   18752            0 :       if (TARGET_MMX_WITH_SSE && TARGET_SSE4_1)
   18753              :         break;
   18754            0 :       wmode = V4HImode;
   18755            0 :       goto widen;
   18756           38 :     case E_V4QImode:
   18757           38 :       if (TARGET_SSE4_1)
   18758              :         break;
   18759              :       wmode = V2HImode;
   18760           39 :     widen:
   18761              :       /* There's no way to set one QImode entry easily.  Combine
   18762              :          the variable value with its adjacent constant value, and
   18763              :          promote to an HImode set.  */
   18764           39 :       x = XVECEXP (vals, 0, one_var ^ 1);
   18765           39 :       if (one_var & 1)
   18766              :         {
   18767            8 :           var = convert_modes (HImode, QImode, var, true);
   18768            8 :           var = expand_simple_binop (HImode, ASHIFT, var, GEN_INT (8),
   18769              :                                      NULL_RTX, 1, OPTAB_LIB_WIDEN);
   18770            8 :           x = GEN_INT (INTVAL (x) & 0xff);
   18771              :         }
   18772              :       else
   18773              :         {
   18774           31 :           var = convert_modes (HImode, QImode, var, true);
   18775           31 :           x = gen_int_mode (UINTVAL (x) << 8, HImode);
   18776              :         }
   18777           39 :       if (x != const0_rtx)
   18778            7 :         var = expand_simple_binop (HImode, IOR, var, x, var,
   18779              :                                    1, OPTAB_LIB_WIDEN);
   18780              : 
   18781           39 :       x = gen_reg_rtx (wmode);
   18782           39 :       emit_move_insn (x, gen_lowpart (wmode, const_vec));
   18783           39 :       ix86_expand_vector_set (mmx_ok, x, var, one_var >> 1);
   18784              : 
   18785           39 :       emit_move_insn (target, gen_lowpart (mode, x));
   18786           39 :       return true;
   18787              : 
   18788              :     default:
   18789              :       return false;
   18790              :     }
   18791              : 
   18792          191 :   emit_move_insn (target, const_vec);
   18793          191 :   ix86_expand_vector_set (mmx_ok, target, var, one_var);
   18794          191 :   return true;
   18795              : }
   18796              : 
   18797              : /* A subroutine of ix86_expand_vector_init_general.  Use vector
   18798              :    concatenate to handle the most general case: all values variable,
   18799              :    and none identical.  */
   18800              : 
   18801              : static void
   18802        24797 : ix86_expand_vector_init_concat (machine_mode mode,
   18803              :                                 rtx target, rtx *ops, int n)
   18804              : {
   18805        24797 :   machine_mode half_mode = VOIDmode;
   18806        24797 :   rtx half[2];
   18807        24797 :   rtvec v;
   18808        24797 :   int i, j;
   18809              : 
   18810        24797 :   switch (n)
   18811              :     {
   18812        23758 :     case 2:
   18813        23758 :       switch (mode)
   18814              :         {
   18815              :         case E_V32HFmode:
   18816              :           half_mode = V16HFmode;
   18817              :           break;
   18818            0 :         case E_V32BFmode:
   18819            0 :           half_mode = V16BFmode;
   18820            0 :           break;
   18821           59 :         case E_V16SImode:
   18822           59 :           half_mode = V8SImode;
   18823           59 :           break;
   18824           34 :         case E_V16SFmode:
   18825           34 :           half_mode = V8SFmode;
   18826           34 :           break;
   18827           84 :         case E_V8DImode:
   18828           84 :           half_mode = V4DImode;
   18829           84 :           break;
   18830           62 :         case E_V8DFmode:
   18831           62 :           half_mode = V4DFmode;
   18832           62 :           break;
   18833            0 :         case E_V16HFmode:
   18834            0 :           half_mode = V8HFmode;
   18835            0 :           break;
   18836            0 :         case E_V16BFmode:
   18837            0 :           half_mode = V8BFmode;
   18838            0 :           break;
   18839           39 :         case E_V8SImode:
   18840           39 :           half_mode = V4SImode;
   18841           39 :           break;
   18842            7 :         case E_V8SFmode:
   18843            7 :           half_mode = V4SFmode;
   18844            7 :           break;
   18845          125 :         case E_V4DImode:
   18846          125 :           half_mode = V2DImode;
   18847          125 :           break;
   18848           57 :         case E_V4DFmode:
   18849           57 :           half_mode = V2DFmode;
   18850           57 :           break;
   18851          997 :         case E_V4SImode:
   18852          997 :           half_mode = V2SImode;
   18853          997 :           break;
   18854         1147 :         case E_V4SFmode:
   18855         1147 :           half_mode = V2SFmode;
   18856         1147 :           break;
   18857            0 :         case E_V2DImode:
   18858            0 :           half_mode = DImode;
   18859            0 :           break;
   18860        17387 :         case E_V2SImode:
   18861        17387 :           half_mode = SImode;
   18862        17387 :           break;
   18863            0 :         case E_V2DFmode:
   18864            0 :           half_mode = DFmode;
   18865            0 :           break;
   18866         3760 :         case E_V2SFmode:
   18867         3760 :           half_mode = SFmode;
   18868         3760 :           break;
   18869            0 :         default:
   18870            0 :           gcc_unreachable ();
   18871              :         }
   18872              : 
   18873        23758 :       if (!register_operand (ops[1], half_mode))
   18874        10088 :         ops[1] = force_reg (half_mode, ops[1]);
   18875        23758 :       if (!register_operand (ops[0], half_mode))
   18876         5982 :         ops[0] = force_reg (half_mode, ops[0]);
   18877        23758 :       emit_insn (gen_rtx_SET (target, gen_rtx_VEC_CONCAT (mode, ops[0],
   18878              :                                                           ops[1])));
   18879        23758 :       break;
   18880              : 
   18881          863 :     case 4:
   18882          863 :       switch (mode)
   18883              :         {
   18884              :         case E_V4DImode:
   18885              :           half_mode = V2DImode;
   18886              :           break;
   18887            0 :         case E_V4DFmode:
   18888            0 :           half_mode = V2DFmode;
   18889            0 :           break;
   18890            0 :         case E_V4SImode:
   18891            0 :           half_mode = V2SImode;
   18892            0 :           break;
   18893          863 :         case E_V4SFmode:
   18894          863 :           half_mode = V2SFmode;
   18895          863 :           break;
   18896            0 :         default:
   18897            0 :           gcc_unreachable ();
   18898              :         }
   18899          863 :       goto half;
   18900              : 
   18901          105 :     case 8:
   18902          105 :       switch (mode)
   18903              :         {
   18904              :         case E_V8DImode:
   18905              :           half_mode = V4DImode;
   18906              :           break;
   18907           62 :         case E_V8DFmode:
   18908           62 :           half_mode = V4DFmode;
   18909           62 :           break;
   18910            0 :         case E_V8SImode:
   18911            0 :           half_mode = V4SImode;
   18912            0 :           break;
   18913            0 :         case E_V8SFmode:
   18914            0 :           half_mode = V4SFmode;
   18915            0 :           break;
   18916            0 :         default:
   18917            0 :           gcc_unreachable ();
   18918              :         }
   18919          105 :       goto half;
   18920              : 
   18921           71 :     case 16:
   18922           71 :       switch (mode)
   18923              :         {
   18924              :         case E_V16SImode:
   18925              :           half_mode = V8SImode;
   18926              :           break;
   18927           33 :         case E_V16SFmode:
   18928           33 :           half_mode = V8SFmode;
   18929           33 :           break;
   18930            0 :         default:
   18931            0 :           gcc_unreachable ();
   18932              :         }
   18933           71 :       goto half;
   18934              : 
   18935         1039 : half:
   18936              :       /* FIXME: We process inputs backward to help RA.  PR 36222.  */
   18937         1039 :       i = n - 1;
   18938         3117 :       for (j = 1; j != -1; j--)
   18939              :         {
   18940         2078 :           half[j] = gen_reg_rtx (half_mode);
   18941         2078 :           switch (n >> 1)
   18942              :             {
   18943         1726 :             case 2:
   18944         1726 :               v = gen_rtvec (2, ops[i-1], ops[i]);
   18945         1726 :               i -= 2;
   18946         1726 :               break;
   18947          210 :             case 4:
   18948          210 :               v = gen_rtvec (4, ops[i-3], ops[i-2], ops[i-1], ops[i]);
   18949          210 :               i -= 4;
   18950          210 :               break;
   18951          142 :             case 8:
   18952          284 :               v = gen_rtvec (8, ops[i-7], ops[i-6], ops[i-5], ops[i-4],
   18953          142 :                              ops[i-3], ops[i-2], ops[i-1], ops[i]);
   18954          142 :               i -= 8;
   18955          142 :               break;
   18956              :             default:
   18957              :               gcc_unreachable ();
   18958              :             }
   18959         2078 :           ix86_expand_vector_init (false, half[j],
   18960              :                                    gen_rtx_PARALLEL (half_mode, v));
   18961              :         }
   18962              : 
   18963         1039 :       ix86_expand_vector_init_concat (mode, target, half, 2);
   18964         1039 :       break;
   18965              : 
   18966            0 :     default:
   18967            0 :       gcc_unreachable ();
   18968              :     }
   18969        24797 : }
   18970              : 
   18971              : /* A subroutine of ix86_expand_vector_init_general.  Use vector
   18972              :    interleave to handle the most general case: all values variable,
   18973              :    and none identical.  */
   18974              : 
   18975              : static void
   18976         3489 : ix86_expand_vector_init_interleave (machine_mode mode,
   18977              :                                     rtx target, rtx *ops, int n)
   18978              : {
   18979         3489 :   machine_mode first_imode, second_imode, third_imode, inner_mode;
   18980         3489 :   int i, j;
   18981         3489 :   rtx op, op0, op1;
   18982         3489 :   rtx (*gen_load_even) (rtx, rtx, rtx);
   18983         3489 :   rtx (*gen_interleave_first_low) (rtx, rtx, rtx);
   18984         3489 :   rtx (*gen_interleave_second_low) (rtx, rtx, rtx);
   18985              : 
   18986         3489 :   switch (mode)
   18987              :     {
   18988              :     case E_V8HFmode:
   18989              :       gen_load_even = gen_vec_interleave_lowv8hf;
   18990              :       gen_interleave_first_low = gen_vec_interleave_lowv4si;
   18991              :       gen_interleave_second_low = gen_vec_interleave_lowv2di;
   18992              :       inner_mode = HFmode;
   18993              :       first_imode = V4SImode;
   18994              :       second_imode = V2DImode;
   18995              :       third_imode = VOIDmode;
   18996              :       break;
   18997          487 :     case E_V8BFmode:
   18998          487 :       gen_load_even = gen_vec_interleave_lowv8bf;
   18999          487 :       gen_interleave_first_low = gen_vec_interleave_lowv4si;
   19000          487 :       gen_interleave_second_low = gen_vec_interleave_lowv2di;
   19001          487 :       inner_mode = BFmode;
   19002          487 :       first_imode = V4SImode;
   19003          487 :       second_imode = V2DImode;
   19004          487 :       third_imode = VOIDmode;
   19005          487 :       break;
   19006          412 :     case E_V8HImode:
   19007          412 :       gen_load_even = gen_vec_setv8hi;
   19008          412 :       gen_interleave_first_low = gen_vec_interleave_lowv4si;
   19009          412 :       gen_interleave_second_low = gen_vec_interleave_lowv2di;
   19010          412 :       inner_mode = HImode;
   19011          412 :       first_imode = V4SImode;
   19012          412 :       second_imode = V2DImode;
   19013          412 :       third_imode = VOIDmode;
   19014          412 :       break;
   19015          362 :     case E_V16QImode:
   19016          362 :       gen_load_even = gen_vec_setv16qi;
   19017          362 :       gen_interleave_first_low = gen_vec_interleave_lowv8hi;
   19018          362 :       gen_interleave_second_low = gen_vec_interleave_lowv4si;
   19019          362 :       inner_mode = QImode;
   19020          362 :       first_imode = V8HImode;
   19021          362 :       second_imode = V4SImode;
   19022          362 :       third_imode = V2DImode;
   19023          362 :       break;
   19024            0 :     default:
   19025            0 :       gcc_unreachable ();
   19026              :     }
   19027              : 
   19028        18893 :   for (i = 0; i < n; i++)
   19029              :     {
   19030        15404 :       op = ops [i + i];
   19031        15404 :       if (inner_mode == HFmode || inner_mode == BFmode)
   19032              :         {
   19033        10860 :           rtx even, odd;
   19034              :           /* Use vpuncklwd to pack 2 HFmode or BFmode.  */
   19035         1948 :           machine_mode vec_mode =
   19036        10860 :             (inner_mode == HFmode) ? V8HFmode : V8BFmode;
   19037        10860 :           op0 = gen_reg_rtx (vec_mode);
   19038        10860 :           even = lowpart_subreg (vec_mode,
   19039              :                                  force_reg (inner_mode, op), inner_mode);
   19040        10860 :           odd = lowpart_subreg (vec_mode,
   19041        10860 :                                 force_reg (inner_mode, ops[i + i + 1]),
   19042              :                                 inner_mode);
   19043        10860 :           emit_insn (gen_load_even (op0, even, odd));
   19044              :         }
   19045              :       else
   19046              :         {
   19047              :           /* Extend the odd element to SImode using a paradoxical SUBREG.  */
   19048         4544 :           op0 = gen_reg_rtx (SImode);
   19049         4544 :           emit_move_insn (op0, gen_lowpart (SImode, op));
   19050              : 
   19051              :           /* Insert the SImode value as low element of V4SImode vector.  */
   19052         4544 :           op1 = gen_reg_rtx (V4SImode);
   19053         4544 :           op0 = gen_rtx_VEC_MERGE (V4SImode,
   19054              :                                    gen_rtx_VEC_DUPLICATE (V4SImode,
   19055              :                                                           op0),
   19056              :                                    CONST0_RTX (V4SImode),
   19057              :                                    const1_rtx);
   19058         4544 :           emit_insn (gen_rtx_SET (op1, op0));
   19059              : 
   19060              :           /* Cast the V4SImode vector back to a vector in original mode.  */
   19061         4544 :           op0 = gen_reg_rtx (mode);
   19062         4544 :           emit_move_insn (op0, gen_lowpart (mode, op1));
   19063              : 
   19064              :           /* Load even elements into the second position.  */
   19065         4544 :           emit_insn (gen_load_even (op0,
   19066              :                                     force_reg (inner_mode,
   19067         4544 :                                                ops[i + i + 1]),
   19068              :                                     const1_rtx));
   19069              :         }
   19070              : 
   19071              :       /* Cast vector to FIRST_IMODE vector.  */
   19072        15404 :       ops[i] = gen_reg_rtx (first_imode);
   19073        15404 :       emit_move_insn (ops[i], gen_lowpart (first_imode, op0));
   19074              :     }
   19075              : 
   19076              :   /* Interleave low FIRST_IMODE vectors.  */
   19077        11191 :   for (i = j = 0; i < n; i += 2, j++)
   19078              :     {
   19079         7702 :       op0 = gen_reg_rtx (first_imode);
   19080         7702 :       emit_insn (gen_interleave_first_low (op0, ops[i], ops[i + 1]));
   19081              : 
   19082              :       /* Cast FIRST_IMODE vector to SECOND_IMODE vector.  */
   19083         7702 :       ops[j] = gen_reg_rtx (second_imode);
   19084         7702 :       emit_move_insn (ops[j], gen_lowpart (second_imode, op0));
   19085              :     }
   19086              : 
   19087              :   /* Interleave low SECOND_IMODE vectors.  */
   19088         3489 :   switch (second_imode)
   19089              :     {
   19090              :     case E_V4SImode:
   19091         1086 :       for (i = j = 0; i < n / 2; i += 2, j++)
   19092              :         {
   19093          724 :           op0 = gen_reg_rtx (second_imode);
   19094          724 :           emit_insn (gen_interleave_second_low (op0, ops[i],
   19095          724 :                                                 ops[i + 1]));
   19096              : 
   19097              :           /* Cast the SECOND_IMODE vector to the THIRD_IMODE
   19098              :              vector.  */
   19099          724 :           ops[j] = gen_reg_rtx (third_imode);
   19100          724 :           emit_move_insn (ops[j], gen_lowpart (third_imode, op0));
   19101              :         }
   19102              :       second_imode = V2DImode;
   19103              :       gen_interleave_second_low = gen_vec_interleave_lowv2di;
   19104              :       /* FALLTHRU */
   19105              : 
   19106         3489 :     case E_V2DImode:
   19107         3489 :       op0 = gen_reg_rtx (second_imode);
   19108         3489 :       emit_insn (gen_interleave_second_low (op0, ops[0],
   19109              :                                             ops[1]));
   19110              : 
   19111              :       /* Cast the SECOND_IMODE vector back to a vector on original
   19112              :          mode.  */
   19113         3489 :       emit_insn (gen_rtx_SET (target, gen_lowpart (mode, op0)));
   19114         3489 :       break;
   19115              : 
   19116              :     default:
   19117              :       gcc_unreachable ();
   19118              :     }
   19119         3489 : }
   19120              : 
   19121              : /* Count number of non-zero integer constants in OPS array of length N.  */
   19122              : 
   19123              : static int
   19124         5866 : nonzero_int_const_count (rtx *ops, int n)
   19125              : {
   19126         5866 :   int result = 0;
   19127         5866 :   int i;
   19128        33070 :   for (i = 0; i < n; i++)
   19129        27204 :     if (CONST_INT_P (ops[i]) && ops[i] != const0_rtx)
   19130          201 :       result++;
   19131         5866 :   return result;
   19132              : }
   19133              : 
   19134              : /* Count number of non-zero float constants in OPS array of length N.  */
   19135              : 
   19136              : static int
   19137         1881 : nonzero_float_const_count (rtx *ops, int n)
   19138              : {
   19139         1881 :   int result = 0;
   19140         1881 :   int i;
   19141        10313 :   for (i = 0; i < n; i++)
   19142         8432 :     if (CONST_DOUBLE_P (ops[i]) && ops[i] != CONST0_RTX (SFmode))
   19143           12 :       result++;
   19144         1881 :   return result;
   19145              : }
   19146              : 
   19147              : /* Count number of non-zero double constants in OPS array of length N.  */
   19148              : 
   19149              : static int
   19150          495 : nonzero_double_const_count (rtx *ops, int n)
   19151              : {
   19152          495 :   int result = 0;
   19153          495 :   int i;
   19154         2475 :   for (i = 0; i < n; i++)
   19155         1980 :     if (CONST_DOUBLE_P (ops[i]) && ops[i] != CONST0_RTX (DFmode))
   19156            2 :       result++;
   19157          495 :   return result;
   19158              : }
   19159              : 
   19160              : 
   19161              : /* A subroutine of ix86_expand_vector_init_general.  Handle the most
   19162              :    general case: all values variable and none identical.  */
   19163              : 
   19164              : static void
   19165          977 : ix86_expand_vector_init_insert (machine_mode mode, rtx target,
   19166              :                                 rtx *ops, int n)
   19167              : {
   19168          977 :   machine_mode inner_mode = GET_MODE_INNER (mode);
   19169          977 :   rtx (*pinsr)(rtx, rtx, rtx, rtx);
   19170          977 :   rtx tmp = gen_reg_rtx (mode);
   19171          977 :   rtx var, x;
   19172          977 :   int i;
   19173              : 
   19174          977 :   var = ops[0];
   19175              : 
   19176          977 :   switch (mode)
   19177              :     {
   19178           98 :     case E_V16QImode:
   19179           98 :       if (var != const0_rtx)
   19180              :         {
   19181           98 :           if (REG_P (var)
   19182           52 :               && ops[1] != const0_rtx
   19183           43 :               && ops[2] != const0_rtx
   19184           40 :               && ops[3] != const0_rtx)
   19185           40 :             var = gen_lowpart (SImode, var);
   19186              :           else
   19187           58 :             var = convert_modes (SImode, QImode, var, true);
   19188           98 :           var = force_reg (SImode, var);
   19189           98 :           x = gen_reg_rtx (V4SImode);
   19190           98 :           emit_insn (gen_vec_setv4si_0 (x, CONST0_RTX (V4SImode), var));
   19191           98 :           emit_move_insn (tmp, gen_lowpart (V16QImode, x));
   19192              :         }
   19193              :       else
   19194            0 :         emit_move_insn (tmp, CONST0_RTX (mode));
   19195              :       pinsr = gen_sse4_1_pinsrb;
   19196              :       break;
   19197          557 :     case E_V8HImode:
   19198          557 :       if (var != const0_rtx)
   19199              :         {
   19200          553 :           if (REG_P (var)
   19201          368 :               && ops[1] != const0_rtx)
   19202          368 :             var = gen_lowpart (SImode, var);
   19203              :           else
   19204          185 :             var = convert_modes (SImode, HImode, var, true);
   19205          553 :           var = force_reg (SImode, var);
   19206          553 :           x = gen_reg_rtx (V4SImode);
   19207          553 :           emit_insn (gen_vec_setv4si_0 (x, CONST0_RTX (V4SImode), var));
   19208          553 :           emit_move_insn (tmp, gen_lowpart (V8HImode, x));
   19209              :         }
   19210              :       else
   19211            4 :         emit_move_insn (tmp, CONST0_RTX (mode));
   19212              :       pinsr = gen_sse2_pinsrw;
   19213              :       break;
   19214          322 :     case E_V4SImode:
   19215          322 :       if (var != const0_rtx)
   19216              :         {
   19217          319 :           if (!REG_P (var) && !MEM_P (var))
   19218            4 :             var = force_reg (SImode, var);
   19219          319 :           emit_insn (gen_vec_setv4si_0 (tmp, CONST0_RTX (mode), var));
   19220              :         }
   19221              :       else
   19222            3 :         emit_move_insn (tmp, CONST0_RTX (mode));
   19223              :       pinsr = gen_sse4_1_pinsrd;
   19224              :       break;
   19225              : 
   19226            0 :     default:
   19227            0 :       gcc_unreachable ();
   19228              :     }
   19229              : 
   19230         7312 :   for (i = 1; i < n; i++)
   19231         6335 :     if (ops[i] != CONST0_RTX (inner_mode))
   19232              :       {
   19233         6007 :         rtx val = ops[i];
   19234         6007 :         if (!REG_P (val) && !MEM_P (val))
   19235          245 :           val = force_reg (inner_mode, val);
   19236         6007 :         emit_insn (pinsr (tmp, tmp, val, GEN_INT (1 << i)));
   19237              :       }
   19238          977 :   emit_move_insn (target, tmp);
   19239          977 : }
   19240              : 
   19241              : /* Helper function.  Determine if the given OPS array of size N
   19242              :    contains only zeros and one other value (possible repeated).
   19243              :    If TRUE, *VAR returns the value, PERM[i] contains 0 for
   19244              :    this value and 1 for a CONST0_RTX.  */
   19245              : 
   19246              : static bool
   19247        19611 : onevar_perm_p (const rtx *ops, int n, int *perm, rtx *var)
   19248              : {
   19249        19611 :   bool found = false;
   19250        19611 :   int i;
   19251              : 
   19252        40782 :   for (i = 0; i < n; i++)
   19253        40542 :     if (ops[i] == const0_rtx
   19254        39654 :         || ops[i] == CONST0_RTX (SFmode)
   19255        39464 :         || ops[i] == CONST0_RTX (DFmode))
   19256         1078 :       perm[i] = 1;
   19257        39464 :     else if (!found)
   19258              :       {
   19259        19611 :         *var = ops[i];
   19260        19611 :         found = true;
   19261        19611 :         perm[i] = 0;
   19262              :       }
   19263        19853 :     else if (rtx_equal_p (*var, ops[i]))
   19264          482 :       perm[i] = 0;
   19265              :     else
   19266              :       return false;
   19267              : 
   19268              :   return found;
   19269              : }
   19270              : 
   19271              : /* Helper function.  Determine if the given OPS array of size N
   19272              :    contains only zeros and two other values (possible repeated).
   19273              :    If TRUE, VARS returns the values, PERM[i] contains 0 for
   19274              :    the first value, 1 for the second value and 2 for CONST0_RTX.  */
   19275              : 
   19276              : static bool
   19277         7371 : twovar_perm_p (const rtx *ops, int n, int *perm, rtx *vars)
   19278              : {
   19279         7371 :   int count = 0;
   19280         7371 :   int i;
   19281              : 
   19282        24252 :   for (i = 0; i < n; i++)
   19283        23519 :     if (ops[i] == const0_rtx
   19284        23030 :         || ops[i] == CONST0_RTX (SFmode)
   19285        22892 :         || ops[i] == CONST0_RTX (DFmode))
   19286          627 :       perm[i] = 2;
   19287        22892 :     else if (count == 0)
   19288              :       {
   19289         7371 :         vars[0] = ops[i];
   19290         7371 :         perm[i] = 0;
   19291         7371 :         count = 1;
   19292              :       }
   19293        15521 :     else if (rtx_equal_p (vars[0], ops[i]))
   19294         1126 :       perm[i] = 0;
   19295        14395 :     else if (count == 1)
   19296              :       {
   19297         7371 :         vars[1] = ops[i];
   19298         7371 :         perm[i] = 1;
   19299         7371 :         count = 2;
   19300              :       }
   19301         7024 :     else if (rtx_equal_p (vars[1], ops[i]))
   19302          386 :       perm[i] = 1;
   19303              :     else
   19304              :       return false;
   19305              : 
   19306          733 :   return count == 2;
   19307              : }
   19308              : 
   19309              : /* A subroutine of ix86_expand_vector_init for V2DImode.  */
   19310              : 
   19311              : static void
   19312        60401 : ix86_expand_vector_init_v2di (rtx target, rtx *ops)
   19313              : {
   19314        60401 :   if (ops[0] == const0_rtx && ops[1] == const0_rtx)
   19315            2 :     emit_move_insn (target, CONST0_RTX (V2DImode));
   19316        60399 :   else if (CONST_INT_P (ops[0]) && CONST_INT_P (ops[1]))
   19317              :     {
   19318            0 :       rtx vec = gen_rtx_CONST_VECTOR (V2DImode, gen_rtvec_v (2, ops));
   19319            0 :       emit_move_insn (target, vec);
   19320            0 :     }
   19321        60399 :   else if (rtx_equal_p (ops[0], ops[1]))
   19322              :     {
   19323           60 :       rtx val = ops[0];
   19324           60 :       if (!REG_P (val) && !MEM_P (val))
   19325            0 :         val = force_reg (DImode, val);
   19326              :       /* TARGET_SSE has *vec_dupv2di.  */
   19327           60 :       emit_move_insn (target, gen_rtx_VEC_DUPLICATE (V2DImode, val));
   19328              :     }
   19329              :   else
   19330              :     {
   19331        60339 :       rtx op0 = force_reg (DImode, ops[0]);
   19332        60339 :       rtx op1 = force_reg (DImode, ops[1]);
   19333        60339 :       emit_insn (gen_vec_concatv2di (target, op0, op1));
   19334              :     }
   19335        60401 : }
   19336              : 
   19337              : /* A subroutine of ix86_expand_vector_init for V2DFmode.  */
   19338              : 
   19339              : static void
   19340         4053 : ix86_expand_vector_init_v2df (rtx target, rtx *ops)
   19341              : {
   19342         4053 :   if (ops[0] == CONST0_RTX (DFmode)
   19343            0 :       && ops[1] == CONST0_RTX (DFmode))
   19344            0 :     emit_move_insn (target, CONST0_RTX (V2DFmode));
   19345         4053 :   else if (CONST_DOUBLE_P (ops[0])
   19346           32 :            && CONST_DOUBLE_P (ops[1]))
   19347              :     {
   19348            0 :       rtx vec = gen_rtx_CONST_VECTOR (V2DFmode, gen_rtvec_v (2, ops));
   19349            0 :       emit_move_insn (target, vec);
   19350            0 :     }
   19351         4053 :   else if (TARGET_SSE2
   19352         4053 :            && rtx_equal_p (ops[0], ops[1]))
   19353              :     {
   19354          102 :       rtx val = ops[0];
   19355          102 :       if (!REG_P (val) && !MEM_P (val))
   19356            0 :         val = force_reg (DFmode, val);
   19357          102 :       emit_move_insn (target, gen_rtx_VEC_DUPLICATE (V2DFmode, val));
   19358              :     }
   19359              :   else
   19360              :     {
   19361         3951 :       rtx op0 = force_reg (DFmode, ops[0]);
   19362         3951 :       rtx op1 = force_reg (DFmode, ops[1]);
   19363         3951 :       emit_insn (gen_vec_concatv2df (target, op0, op1));
   19364              :     }
   19365         4053 : }
   19366              : 
   19367              : /* A subroutine of ix86_expand_vector_init for V4SImode.  */
   19368              : 
   19369              : static void
   19370        34786 : ix86_expand_vector_init_v4si (rtx target, rtx *ops)
   19371              : {
   19372        34786 :   rtx vars[4];
   19373        34786 :   int perm[4];
   19374              : 
   19375        34786 :   if (ops[0] == const0_rtx
   19376          335 :       && ops[1] == const0_rtx
   19377          172 :       && ops[2] == const0_rtx
   19378            7 :       && ops[3] == const0_rtx)
   19379            6 :     emit_move_insn (target, CONST0_RTX (V4SImode));
   19380        34780 :   else if (ops[1] == const0_rtx
   19381        19725 :            && ops[2] == const0_rtx
   19382        19500 :            && ops[3] == const0_rtx)
   19383              :     {
   19384        19477 :       rtx val = ops[0];
   19385        19477 :       if (!REG_P (val) && !MEM_P (val))
   19386         2915 :         val = force_reg (SImode, val);
   19387        19477 :       emit_insn (gen_vec_setv4si_0 (target, CONST0_RTX (V4SImode), val));
   19388        19477 :     }
   19389        15303 :   else if (rtx_equal_p (ops[0], ops[1])
   19390          325 :            && rtx_equal_p (ops[0], ops[2])
   19391        15308 :            && rtx_equal_p (ops[0], ops[3]))
   19392              :     {
   19393            2 :       rtx val = ops[0];
   19394            2 :       if (!REG_P (val) && !MEM_P (val))
   19395            0 :         val = force_reg (SImode, val);
   19396            2 :       emit_move_insn (target, gen_rtx_VEC_DUPLICATE (V4SImode, val));
   19397              :     }
   19398        15301 :   else if (CONST_INT_P (ops[0])
   19399          442 :            && CONST_INT_P (ops[1])
   19400          181 :            && CONST_INT_P (ops[2])
   19401            1 :            && CONST_INT_P (ops[3]))
   19402              :     {
   19403            1 :       rtx vec = gen_rtx_CONST_VECTOR (V4SImode, gen_rtvec_v (4, ops));
   19404            1 :       emit_move_insn (target, vec);
   19405            1 :     }
   19406        15300 :   else if (onevar_perm_p (ops, 4, perm, vars))
   19407              :     {
   19408          186 :       rtx tmp = gen_reg_rtx (V4SImode);
   19409          186 :       vars[1] = const0_rtx;
   19410          186 :       vars[2] = const0_rtx;
   19411          186 :       vars[3] = const0_rtx;
   19412          186 :       ix86_expand_vector_init_v4si (tmp, vars);
   19413          186 :       emit_insn (gen_sse_shufps_v4si (target, tmp, tmp,
   19414          186 :                                       GEN_INT (perm[0]),
   19415          186 :                                       GEN_INT (perm[1]),
   19416          186 :                                       GEN_INT (perm[2] + 4),
   19417          186 :                                       GEN_INT (perm[3] + 4)));
   19418              :     }
   19419        15114 :   else if (ops[2] == const0_rtx && ops[3] == const0_rtx)
   19420              :     {
   19421         9644 :       rtx tmp1 = gen_reg_rtx (V4SImode);
   19422         9644 :       vars[0] = ops[0];
   19423         9644 :       vars[1] = const0_rtx;
   19424         9644 :       vars[2] = const0_rtx;
   19425         9644 :       vars[3] = const0_rtx;
   19426         9644 :       ix86_expand_vector_init_v4si (tmp1, vars);
   19427              : 
   19428         9644 :       if (TARGET_SSE4_1)
   19429              :         {
   19430           72 :           rtx val = ops[1];
   19431           72 :           if (!REG_P (val) && !MEM_P (val))
   19432            2 :             val = force_reg (SImode, val);
   19433           72 :           emit_insn (gen_sse4_1_pinsrd (target, tmp1, val, GEN_INT (2)));
   19434              :         }
   19435              :       else
   19436              :         {
   19437         9572 :           rtx tmp2 = gen_reg_rtx (V4SImode);
   19438         9572 :           vars[0] = ops[1];
   19439         9572 :           ix86_expand_vector_init_v4si (tmp2, vars);
   19440         9572 :           emit_insn (gen_vec_interleave_lowv4si (target, tmp1, tmp2));
   19441              :         }
   19442              :     }
   19443         5470 :   else if (TARGET_SSE4_1
   19444          360 :            && ops[1] == const0_rtx
   19445            4 :            && (ops[2] == const0_rtx || ops[3] == const0_rtx))
   19446              :     /* { a, 0, b, 0 } and { a, 0, 0, b } become mov; pinsrd.  */
   19447            4 :     ix86_expand_vector_init_insert (V4SImode, target, ops, 4);
   19448         5466 :   else if (twovar_perm_p (ops, 4, perm, vars))
   19449              :     {
   19450          482 :       rtx tmp = gen_reg_rtx (V4SImode);
   19451          482 :       vars[2] = const0_rtx;
   19452          482 :       vars[3] = const0_rtx;
   19453          482 :       ix86_expand_vector_init_v4si (tmp, vars);
   19454          482 :       emit_insn (gen_sse_shufps_v4si (target, tmp, tmp,
   19455          482 :                                       GEN_INT (perm[0]),
   19456          482 :                                       GEN_INT (perm[1]),
   19457          482 :                                       GEN_INT (perm[2] + 4),
   19458          482 :                                       GEN_INT (perm[3] + 4)));
   19459              :     }
   19460         4984 :   else if (nonzero_int_const_count (ops, 4) >= 2)
   19461              :     {
   19462              :       rtx csts[4];
   19463              :       int i;
   19464          445 :       for (i = 0; i < 4; i++)
   19465          356 :         if (CONST_INT_P (ops[i]))
   19466              :           {
   19467          178 :             csts[i] = ops[i];
   19468          178 :             vars[i] = const0_rtx;
   19469              :           }
   19470              :         else
   19471              :           {
   19472          178 :             csts[i] = const0_rtx;
   19473          178 :             vars[i] = ops[i];
   19474              :           }
   19475           89 :       rtx tmp1 = gen_reg_rtx (V4SImode);
   19476           89 :       ix86_expand_vector_init_v4si (tmp1, vars);
   19477           89 :       rtx tmp2 = gen_reg_rtx (V4SImode);
   19478           89 :       rtx vec = gen_rtx_CONST_VECTOR (V4SImode, gen_rtvec_v (4, csts));
   19479           89 :       emit_move_insn (tmp2, vec);
   19480           89 :       emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V4SImode, tmp1, tmp2)));
   19481              :     }
   19482         4895 :   else if (TARGET_SSE4_1)
   19483          318 :     ix86_expand_vector_init_insert (V4SImode, target, ops, 4);
   19484              :   else
   19485              :     {
   19486         4577 :       rtx tmp1 = gen_reg_rtx (V4SImode);
   19487         4577 :       rtx tmp2 = gen_reg_rtx (V4SImode);
   19488         4577 :       vars[0] = ops[0];
   19489         4577 :       vars[1] = ops[1];
   19490         4577 :       vars[2] = const0_rtx;
   19491         4577 :       vars[3] = const0_rtx;
   19492         4577 :       ix86_expand_vector_init_v4si (tmp1, vars);
   19493         4577 :       vars[0] = ops[2];
   19494         4577 :       vars[1] = ops[3];
   19495         4577 :       ix86_expand_vector_init_v4si (tmp2, vars);
   19496         4577 :       emit_insn (gen_sse_shufps_v4si (target, tmp1, tmp2,
   19497              :                                       const0_rtx, const1_rtx,
   19498              :                                       GEN_INT (4), GEN_INT (5)));
   19499              :     }
   19500        34786 : }
   19501              : 
   19502              : /* A subroutine of ix86_expand_vector_init for V4SFmode.  */
   19503              : 
   19504              : static void
   19505         6440 : ix86_expand_vector_init_v4sf (rtx target, rtx *ops)
   19506              : {
   19507         6440 :   rtx vars[4];
   19508         6440 :   int perm[4];
   19509              : 
   19510         6440 :   if (ops[0] == CONST0_RTX (SFmode) 
   19511           21 :       && ops[1] == CONST0_RTX (SFmode)
   19512            1 :       && ops[2] == CONST0_RTX (SFmode)
   19513            0 :       && ops[3] == CONST0_RTX (SFmode))
   19514            0 :     emit_move_insn (target, CONST0_RTX (V4SFmode));
   19515         6440 :   else if (ops[1] == CONST0_RTX (SFmode)
   19516         2696 :            && ops[2] == CONST0_RTX (SFmode)
   19517         2586 :            && ops[3] == CONST0_RTX (SFmode))
   19518         2586 :     ix86_expand_vector_init_one_nonzero (false, V4SFmode, target, ops[0], 0);
   19519         3854 :   else if (rtx_equal_p (ops[0], ops[1])
   19520          180 :            && rtx_equal_p (ops[0], ops[2])
   19521         3917 :            && rtx_equal_p (ops[0], ops[3]))
   19522              :     {
   19523           56 :       rtx val = ops[0];
   19524           56 :       if (!REG_P (val) && !MEM_P (val))
   19525            0 :         val = force_reg (SFmode, val);
   19526           56 :       emit_move_insn (target, gen_rtx_VEC_DUPLICATE (V4SFmode, val));
   19527              :     }
   19528         3798 :   else if (CONST_DOUBLE_P (ops[0])
   19529           29 :            && CONST_DOUBLE_P (ops[1])
   19530            2 :            && CONST_DOUBLE_P (ops[2])
   19531            0 :            && CONST_DOUBLE_P (ops[3]))
   19532              :     {
   19533            0 :       rtx vec = gen_rtx_CONST_VECTOR (V4SFmode, gen_rtvec_v (4, ops));
   19534            0 :       emit_move_insn (target, vec);
   19535            0 :     }
   19536         3798 :   else if (ops[0] == CONST0_RTX (SFmode)
   19537           21 :            && ops[2] == CONST0_RTX (SFmode)
   19538           16 :            && ops[3] == CONST0_RTX (SFmode))
   19539            0 :     ix86_expand_vector_init_one_nonzero (false, V4SFmode, target, ops[1], 1);
   19540         3798 :   else if (ops[0] == CONST0_RTX (SFmode)
   19541           21 :            && ops[1] == CONST0_RTX (SFmode)
   19542            1 :            && ops[3] == CONST0_RTX (SFmode))
   19543            0 :     ix86_expand_vector_init_one_nonzero (false, V4SFmode, target, ops[2], 2);
   19544         3798 :   else if (ops[0] == CONST0_RTX (SFmode)
   19545           21 :            && ops[1] == CONST0_RTX (SFmode)
   19546            1 :            && ops[2] == CONST0_RTX (SFmode))
   19547            0 :     ix86_expand_vector_init_one_nonzero (false, V4SFmode, target, ops[3], 3);
   19548         3798 :   else if (onevar_perm_p (ops, 4, perm, vars))
   19549              :     {
   19550           54 :       rtx tmp = gen_reg_rtx (V4SFmode);
   19551           54 :       if (TARGET_SSE2 && !TARGET_SSE4_1 && REG_P (ops[0]))
   19552              :         {
   19553           25 :           emit_insn (gen_sse2_insertps_v4sf_3 (tmp, ops[0],
   19554              :                                                CONST0_RTX (V4SFmode)));
   19555          100 :           emit_insn (gen_sse_shufps_v4sf (target, tmp, tmp,
   19556           25 :                                           GEN_INT (perm[0] ? 0 : 3),
   19557           25 :                                           GEN_INT (perm[1] ? 0 : 3),
   19558           25 :                                           GEN_INT (perm[2] ? 4 : 7),
   19559           25 :                                           GEN_INT (perm[3] ? 4 : 7)));
   19560              :         }
   19561              :       else
   19562              :         {
   19563           29 :           vars[1] = CONST0_RTX (SFmode);
   19564           29 :           vars[2] = CONST0_RTX (SFmode);
   19565           29 :           vars[3] = CONST0_RTX (SFmode);
   19566           29 :           ix86_expand_vector_init_v4sf (tmp, vars);
   19567           29 :           emit_insn (gen_sse_shufps_v4sf (target, tmp, tmp,
   19568           29 :                                           GEN_INT (perm[0]),
   19569           29 :                                           GEN_INT (perm[1]),
   19570           29 :                                           GEN_INT (perm[2] + 4),
   19571           29 :                                           GEN_INT (perm[3] + 4)));
   19572              :         }
   19573              :     }
   19574         3744 :   else if (ops[2] == CONST0_RTX (SFmode)
   19575         1841 :            && ops[3] == CONST0_RTX (SFmode))
   19576              :     {
   19577         1839 :       if (TARGET_SSE4_1)
   19578              :         {
   19579          442 :           rtx tmp = gen_reg_rtx (V4SFmode);
   19580          442 :           rtx val = ops[0];
   19581          442 :           if (!REG_P (val) && !MEM_P (val))
   19582            0 :             val = force_reg (SFmode, val);
   19583          442 :           emit_insn (gen_vec_setv4sf_0 (tmp, CONST0_RTX (V4SFmode), val));
   19584          442 :           val = ops[1];
   19585          442 :           if (!REG_P (val) && !MEM_P (val))
   19586            0 :             val = force_reg (SFmode, val);
   19587          442 :           emit_insn (gen_vec_setv4sf_sse4_1 (target, tmp, val, GEN_INT (2)));
   19588              :         }
   19589         1397 :       else if (MEM_P (ops[0]) && MEM_P (ops[1]))
   19590              :         {
   19591         1278 :           rtx tmp1 = gen_reg_rtx (V4SFmode);
   19592         1278 :           vars[0] = ops[0];
   19593         1278 :           vars[1] = CONST0_RTX (SFmode);
   19594         1278 :           vars[2] = CONST0_RTX (SFmode);
   19595         1278 :           vars[3] = CONST0_RTX (SFmode);
   19596         1278 :           ix86_expand_vector_init_v4sf (tmp1, vars);
   19597              : 
   19598         1278 :           rtx tmp2 = gen_reg_rtx (V4SFmode);
   19599         1278 :           vars[0] = ops[1];
   19600         1278 :           ix86_expand_vector_init_v4sf (tmp2, vars);
   19601         1278 :           emit_insn (gen_vec_interleave_lowv4sf (target, tmp1, tmp2));
   19602         1278 :         }
   19603          119 :       else if (TARGET_SSE2)
   19604              :         {
   19605          115 :           rtx tmp = gen_reg_rtx (V2SFmode);
   19606          115 :           ix86_expand_vector_init_concat (V2SFmode, tmp, ops, 2);
   19607          115 :           tmp = gen_rtx_VEC_CONCAT (V4SFmode, tmp, CONST0_RTX (V2SFmode));
   19608          115 :           emit_insn (gen_rtx_SET (target, tmp));
   19609              :         }
   19610              :       else
   19611              :         {
   19612            4 :           rtx tmp1 = gen_reg_rtx (V2SFmode);
   19613            4 :           ix86_expand_vector_init_concat (V2SFmode, tmp1, ops, 2);
   19614            4 :           rtx tmp2 = gen_reg_rtx (V2SFmode);
   19615              :           /* Without both MMX and TARGET_MMX_WITH_SSE there's no movv2sf,
   19616              :              go through emit_move_insn so the zero can be materialized in
   19617              :              the corresponding integer mode.  */
   19618            4 :           emit_move_insn (tmp2, CONST0_RTX (V2SFmode));
   19619            4 :           rtx tmp = gen_rtx_VEC_CONCAT (V4SFmode, tmp1, tmp2);
   19620            4 :           emit_insn (gen_rtx_SET (target, tmp));
   19621              :         }
   19622              :     }
   19623         1905 :   else if (MEM_P (ops[0])
   19624          791 :            && ops[1] == CONST0_RTX (SFmode)
   19625            0 :            && MEM_P (ops[2])
   19626            0 :            && ops[3] == CONST0_RTX (SFmode))
   19627              :     {
   19628            0 :       rtx tmp1 = gen_reg_rtx (V4SFmode);
   19629            0 :       ix86_expand_vector_init_one_nonzero (false, V4SFmode, tmp1, ops[0], 0);
   19630            0 :       rtx tmp2 = gen_reg_rtx (V4SFmode);
   19631            0 :       ix86_expand_vector_init_one_nonzero (false, V4SFmode, tmp2, ops[2], 0);
   19632            0 :       emit_insn (gen_sse_movlhps (target, tmp1, tmp2));
   19633            0 :     }
   19634         1905 :   else if (twovar_perm_p (ops, 4, perm, vars))
   19635              :     {
   19636          251 :       rtx tmp = gen_reg_rtx (V4SFmode);
   19637          251 :       vars[2] = CONST0_RTX (SFmode);
   19638          251 :       vars[3] = CONST0_RTX (SFmode);
   19639          251 :       ix86_expand_vector_init_v4sf (tmp, vars);
   19640          251 :       emit_insn (gen_sse_shufps_v4sf (target, tmp, tmp,
   19641          251 :                                       GEN_INT (perm[0]),
   19642          251 :                                       GEN_INT (perm[1]),
   19643          251 :                                       GEN_INT (perm[2] + 4),
   19644          251 :                                       GEN_INT (perm[3] + 4)));
   19645              :     }
   19646         1654 :   else if (nonzero_float_const_count (ops, 4) >= 2)
   19647              :     {
   19648              :       rtx csts[4];
   19649              :       int i;
   19650           10 :       for (i = 0; i < 4; i++)
   19651            8 :         if (CONST_DOUBLE_P (ops[i]))
   19652              :           {
   19653            4 :             csts[i] = ops[i];
   19654            4 :             vars[i] = CONST0_RTX (SFmode);
   19655              :           }
   19656              :         else
   19657              :           {
   19658            4 :             csts[i] = CONST0_RTX (SFmode);
   19659            4 :             vars[i] = ops[i];
   19660              :           }
   19661            2 :       rtx tmp1 = gen_reg_rtx (V4SFmode);
   19662            2 :       ix86_expand_vector_init_v4sf (tmp1, vars);
   19663            2 :       rtx tmp2 = gen_reg_rtx (V4SFmode);
   19664            2 :       rtx vec = gen_rtx_CONST_VECTOR (V4SFmode, gen_rtvec_v (4, csts));
   19665            2 :       emit_move_insn (tmp2, vec);
   19666            2 :       emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V4SFmode, tmp1, tmp2)));
   19667              :     }
   19668         1652 :   else if (TARGET_SSE4_1 && optimize_insn_for_size_p ())
   19669              :     {
   19670            1 :       int i;
   19671            1 :       rtx tmp = gen_reg_rtx (V4SFmode);
   19672            1 :       bool first_p = true;
   19673            6 :       for (i = 0; i < 4; i++)
   19674            4 :         if (ops[i] != CONST0_RTX (SFmode))
   19675              :           {
   19676            4 :             rtx idx = GEN_INT (1 << i);
   19677            4 :             rtx val = ops[i];
   19678            4 :             if (!REG_P (val) && !MEM_P (val))
   19679            0 :               val = force_reg (SFmode, val);
   19680            4 :             rtx pat;
   19681            4 :             if (!first_p)
   19682            3 :               pat = gen_vec_setv4sf_sse4_1 (tmp, tmp, val, idx);
   19683            1 :             else if (i == 0)
   19684            1 :               pat = gen_vec_setv4sf_0 (tmp, CONST0_RTX (V4SFmode), val);
   19685              :             else
   19686            0 :               pat = gen_sse4_1_insertps_v4sf_init (tmp, val,
   19687              :                                                    CONST0_RTX (V4SFmode), idx);
   19688            4 :             emit_insn (pat);
   19689            4 :             first_p = false;
   19690              :           }
   19691            1 :       emit_move_insn (target, tmp);
   19692              :     }
   19693         1651 :   else if (MEM_P (ops[0])
   19694          790 :            && MEM_P (ops[1])
   19695          789 :            && MEM_P (ops[2])
   19696          788 :            && MEM_P (ops[3]))
   19697              :     {
   19698          788 :       rtx tmp1 = gen_reg_rtx (V4SFmode);
   19699          788 :       rtx tmp2 = gen_reg_rtx (V4SFmode);
   19700          788 :       vars[0] = ops[0];
   19701          788 :       vars[1] = ops[1];
   19702          788 :       vars[2] = CONST0_RTX (SFmode);
   19703          788 :       vars[3] = CONST0_RTX (SFmode);
   19704          788 :       ix86_expand_vector_init_v4sf (tmp1, vars);
   19705          788 :       vars[0] = ops[2];
   19706          788 :       vars[1] = ops[3];
   19707          788 :       ix86_expand_vector_init_v4sf (tmp2, vars);
   19708          788 :       emit_insn (gen_sse_shufps_v4sf (target, tmp1, tmp2,
   19709              :                                       const0_rtx, const1_rtx,
   19710              :                                       GEN_INT (4), GEN_INT (5)));
   19711          788 :     }
   19712              :   else
   19713          863 :     ix86_expand_vector_init_concat (V4SFmode, target, ops, 4);
   19714         6440 : }
   19715              : 
   19716              : /* A subroutine of ix86_expand_vector_init for V8HImode.  */
   19717              : 
   19718              : static bool
   19719          557 : ix86_expand_vector_init_v8hi (rtx target, rtx *ops)
   19720              : {
   19721          557 :   rtx vars[8];
   19722          557 :   int i;
   19723              : 
   19724          557 :   bool all_zero_p = true;
   19725          561 :   for (i = 0; i < 8; i++)
   19726          561 :     if (ops[i] != const0_rtx)
   19727              :       {
   19728              :         all_zero_p = false;
   19729              :         break;
   19730              :       }
   19731          557 :   if (all_zero_p)
   19732              :     {
   19733            0 :       emit_move_insn (target, CONST0_RTX (V8HImode));
   19734            0 :       return true;
   19735              :     }
   19736              : 
   19737          566 :   bool all_const_p = true;
   19738          566 :   for (i = 0; i < 8; i++)
   19739          566 :     if (!CONST_INT_P (ops[i]))
   19740              :       {
   19741              :         all_const_p = false;
   19742              :         break;
   19743              :       }
   19744          557 :   if (all_const_p)
   19745              :     {
   19746            0 :       rtx vec = gen_rtx_CONST_VECTOR (V8HImode, gen_rtvec_v (8, ops));
   19747            0 :       emit_move_insn (target, vec);
   19748            0 :       return true;
   19749              :     }
   19750              : 
   19751          557 :   if (TARGET_SSE2
   19752          557 :       && nonzero_int_const_count (ops, 8) >= 2)
   19753              :     {
   19754              :       rtx csts[8];
   19755            0 :       for (i = 0; i < 8; i++)
   19756            0 :         if (CONST_INT_P (ops[i]))
   19757              :           {
   19758            0 :             csts[i] = ops[i];
   19759            0 :             vars[i] = const0_rtx;
   19760              :           }
   19761              :         else
   19762              :           {
   19763            0 :             csts[i] = const0_rtx;
   19764            0 :             vars[i] = ops[i];
   19765              :           }
   19766            0 :       rtx tmp1 = gen_reg_rtx (V8HImode);
   19767            0 :       if (!ix86_expand_vector_init_v8hi (tmp1, vars))
   19768            0 :         gcc_unreachable ();
   19769            0 :       rtx tmp2 = gen_reg_rtx (V8HImode);
   19770            0 :       rtx vec = gen_rtx_CONST_VECTOR (V8HImode, gen_rtvec_v (8, csts));
   19771            0 :       emit_move_insn (tmp2, vec);
   19772            0 :       emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V8HImode, tmp1, tmp2)));
   19773            0 :       return true;
   19774              :     }
   19775              : 
   19776          557 :   if (TARGET_SSE2)
   19777              :     {
   19778          557 :       ix86_expand_vector_init_insert (V8HImode, target, ops, 8);
   19779          557 :       return true;
   19780              :     }
   19781              :   return false;
   19782              : }
   19783              : 
   19784              : /* A subroutine of ix86_expand_vector_init_v16qi.
   19785              :    Place OPS[0..3] in an SImode REG or CONST_INT.  */
   19786              : 
   19787              : static rtx
   19788            0 : ix86_expand_vector_init_qi4 (rtx *ops)
   19789              : {
   19790            0 :   rtx vars[4];
   19791            0 :   rtx word;
   19792            0 :   int i;
   19793              : 
   19794            0 :   if (CONST_INT_P (ops[0])
   19795            0 :       && CONST_INT_P (ops[1])
   19796            0 :       && CONST_INT_P (ops[2])
   19797            0 :       && CONST_INT_P (ops[3]))
   19798              :     {
   19799            0 :       HOST_WIDE_INT val = (UINTVAL (ops[0]) & 0xff)
   19800            0 :                           + ((UINTVAL (ops[1]) & 0xff) << 8)
   19801            0 :                           + ((UINTVAL (ops[2]) & 0xff) << 16)
   19802            0 :                           + ((UINTVAL (ops[3]) & 0xff) << 24);
   19803            0 :       return gen_int_mode (val, SImode);
   19804              :     }
   19805              : 
   19806            0 :   if (ops[1] == const0_rtx
   19807            0 :       && ops[2] == const0_rtx
   19808            0 :       && ops[3] == const0_rtx)
   19809            0 :     return convert_modes (SImode, QImode, ops[0], true);
   19810              : 
   19811              : 
   19812            0 :   if (rtx_equal_p (ops[0], ops[2])
   19813            0 :       && rtx_equal_p (ops[1], ops[3]))
   19814              :     {
   19815            0 :       vars[0] = ops[0];
   19816            0 :       vars[1] = ops[1];
   19817            0 :       vars[2] = const0_rtx;
   19818            0 :       vars[3] = const0_rtx;
   19819            0 :       rtx tmp1 = ix86_expand_vector_init_qi4 (vars);
   19820            0 :       rtx tmp2 = expand_simple_binop (SImode, ASHIFT, tmp1, GEN_INT (16),
   19821              :                                       NULL_RTX, 1, OPTAB_LIB_WIDEN);
   19822            0 :       return expand_simple_binop (SImode, PLUS, tmp2, tmp1,
   19823            0 :                                   NULL_RTX, 1, OPTAB_LIB_WIDEN);
   19824              :     }
   19825              : 
   19826              :   HOST_WIDE_INT val = 0;
   19827            0 :   for (i = 0; i < 4; i++)
   19828            0 :     if (CONST_INT_P (ops[i]) && ops[i] != const0_rtx)
   19829              :       {
   19830            0 :         val += (UINTVAL (ops[i]) & 0xff) << (i * 8);
   19831            0 :         vars[i] = const0_rtx;
   19832              :       }
   19833              :     else
   19834            0 :       vars[i] = ops[i];
   19835              : 
   19836            0 :   if (val != 0)
   19837              :     {
   19838            0 :       rtx tmp = ix86_expand_vector_init_qi4 (vars);
   19839            0 :       return expand_simple_binop (SImode, IOR, tmp,
   19840            0 :                                   gen_int_mode (val, SImode),
   19841              :                                   NULL_RTX, 1, OPTAB_LIB_WIDEN);
   19842              :     }
   19843              : 
   19844            0 :   if (ops[0] == const0_rtx && ops[1] == const0_rtx)
   19845              :     {
   19846            0 :       if (ops[2] == const0_rtx)
   19847              :         {
   19848            0 :           word = convert_modes (SImode, QImode, ops[3], true);
   19849            0 :           return expand_simple_binop (SImode, ASHIFT, word, GEN_INT (24),
   19850            0 :                                       NULL_RTX, 1, OPTAB_LIB_WIDEN);
   19851              :         }
   19852            0 :       vars[0] = ops[2];
   19853            0 :       vars[1] = ops[3];
   19854            0 :       vars[2] = const0_rtx;
   19855            0 :       vars[3] = const0_rtx;
   19856            0 :       word = ix86_expand_vector_init_qi4 (vars);
   19857            0 :       return expand_simple_binop (SImode, ASHIFT, word, GEN_INT (16),
   19858            0 :                                   NULL_RTX, 1, OPTAB_LIB_WIDEN);
   19859              :     }
   19860              : 
   19861            0 :   vars[0] = ops[1];
   19862            0 :   vars[1] = ops[2];
   19863            0 :   vars[2] = ops[3];
   19864            0 :   vars[3] = const0_rtx;
   19865            0 :   word = ix86_expand_vector_init_qi4 (vars);
   19866            0 :   word = expand_simple_binop (SImode, ASHIFT, word, GEN_INT (8),
   19867              :                               NULL_RTX, 1, OPTAB_LIB_WIDEN);
   19868            0 :   if (ops[0] != const0_rtx)
   19869              :     {
   19870            0 :       rtx elt = convert_modes (SImode, QImode, ops[0], true);
   19871            0 :       word = expand_simple_binop (SImode, PLUS, word, elt,
   19872              :                                   NULL_RTX, 1, OPTAB_LIB_WIDEN);
   19873              :     }
   19874              :   return word;
   19875              : }
   19876              : 
   19877              : /* A subroutine of ix86_expand_vector_init_v16qi.
   19878              :    Place OPS[0..7] in an DImode REG or CONST_INT.  */
   19879              : 
   19880              : static rtx
   19881         4513 : ix86_expand_vector_init_qi8 (rtx *ops)
   19882              : {
   19883         4513 :   rtx vars[8];
   19884         4513 :   rtx word;
   19885         4513 :   int i;
   19886              : 
   19887         4513 :   if (CONST_INT_P (ops[0])
   19888           34 :       && CONST_INT_P (ops[1])
   19889           12 :       && CONST_INT_P (ops[2])
   19890           12 :       && CONST_INT_P (ops[3])
   19891           12 :       && CONST_INT_P (ops[4])
   19892           12 :       && CONST_INT_P (ops[5])
   19893           12 :       && CONST_INT_P (ops[6])
   19894           12 :       && CONST_INT_P (ops[7]))
   19895              :     {
   19896           12 :       HOST_WIDE_INT val = (UINTVAL (ops[0]) & 0xff)
   19897           12 :                           + ((UINTVAL (ops[1]) & 0xff) << 8)
   19898           12 :                           + ((UINTVAL (ops[2]) & 0xff) << 16)
   19899           12 :                           + ((UINTVAL (ops[3]) & 0xff) << 24)
   19900           12 :                           + ((UINTVAL (ops[4]) & 0xff) << 32)
   19901           12 :                           + ((UINTVAL (ops[5]) & 0xff) << 40)
   19902           12 :                           + ((UINTVAL (ops[6]) & 0xff) << 48)
   19903           12 :                           + ((UINTVAL (ops[7]) & 0xff) << 56);
   19904           12 :       return gen_int_mode (val, DImode);
   19905              :     }
   19906              : 
   19907         4501 :   if (ops[1] == const0_rtx
   19908          640 :       && ops[2] == const0_rtx
   19909          627 :       && ops[3] == const0_rtx
   19910          627 :       && ops[4] == const0_rtx
   19911          626 :       && ops[5] == const0_rtx
   19912          626 :       && ops[6] == const0_rtx
   19913          626 :       && ops[7] == const0_rtx)
   19914          626 :     return convert_modes (DImode, QImode, ops[0], true);
   19915              : 
   19916         3875 :   if (rtx_equal_p (ops[0], ops[4])
   19917           62 :       && rtx_equal_p (ops[1], ops[5])
   19918           42 :       && rtx_equal_p (ops[2], ops[6])
   19919         3911 :       && rtx_equal_p (ops[3], ops[7]))
   19920              :     {
   19921           30 :       vars[0] = ops[0];
   19922           30 :       vars[1] = ops[1];
   19923           30 :       vars[2] = ops[2];
   19924           30 :       vars[3] = ops[3];
   19925           30 :       vars[4] = const0_rtx;
   19926           30 :       vars[5] = const0_rtx;
   19927           30 :       vars[6] = const0_rtx;
   19928           30 :       vars[7] = const0_rtx;
   19929           30 :       rtx tmp1 = ix86_expand_vector_init_qi8 (vars);
   19930           30 :       rtx tmp2 = expand_simple_binop (DImode, ASHIFT, tmp1, GEN_INT (32),
   19931              :                                       NULL_RTX, 1, OPTAB_LIB_WIDEN);
   19932           30 :       return expand_simple_binop (DImode, PLUS, tmp2, tmp1,
   19933           30 :                                   NULL_RTX, 1, OPTAB_LIB_WIDEN);
   19934              :     }
   19935              : 
   19936              :   HOST_WIDE_INT val = 0;
   19937        34605 :   for (i = 0; i < 8; i++)
   19938        30760 :     if (CONST_INT_P (ops[i]) && ops[i] != const0_rtx)
   19939              :       {
   19940          380 :         val += (UINTVAL (ops[i]) & 0xff) << (i * 8);
   19941          380 :         vars[i] = const0_rtx;
   19942              :       }
   19943              :     else
   19944        30380 :       vars[i] = ops[i];
   19945              : 
   19946         3845 :   if (val != 0)
   19947              :     {
   19948           58 :       rtx tmp = ix86_expand_vector_init_qi8 (vars);
   19949           58 :       return expand_simple_binop (DImode, IOR, tmp,
   19950           58 :                                   gen_int_mode (val, DImode),
   19951              :                                   NULL_RTX, 1, OPTAB_LIB_WIDEN);
   19952              :     }
   19953              : 
   19954         3787 :   if (rtx_equal_p (ops[0], ops[2])
   19955           76 :       && rtx_equal_p (ops[1], ops[3])
   19956           69 :       && ops[4] == const0_rtx
   19957           46 :       && ops[5] == const0_rtx
   19958           46 :       && ops[6] == const0_rtx
   19959         3833 :       && ops[7] == const0_rtx)
   19960              :     {
   19961           46 :       vars[0] = ops[0];
   19962           46 :       vars[1] = ops[1];
   19963           46 :       vars[2] = const0_rtx;
   19964           46 :       vars[3] = const0_rtx;
   19965           46 :       vars[4] = const0_rtx;
   19966           46 :       vars[5] = const0_rtx;
   19967           46 :       vars[6] = const0_rtx;
   19968           46 :       vars[7] = const0_rtx;
   19969           46 :       rtx tmp1 = ix86_expand_vector_init_qi8 (vars);
   19970           46 :       rtx tmp2 = expand_simple_binop (DImode, ASHIFT, tmp1, GEN_INT (16),
   19971              :                                       NULL_RTX, 1, OPTAB_LIB_WIDEN);
   19972           46 :       return expand_simple_binop (DImode, PLUS, tmp2, tmp1,
   19973           46 :                                   NULL_RTX, 1, OPTAB_LIB_WIDEN);
   19974              :     }
   19975              : 
   19976         3741 :   if (ops[0] == const0_rtx && ops[1] == const0_rtx)
   19977              :     {
   19978              :       i = 2;
   19979            0 :       while (ops[i] == const0_rtx)
   19980            0 :         i++;
   19981            0 :       vars[0] = ops[i];
   19982            0 :       vars[1] = i + 1 < 8 ? ops[i + 1] : const0_rtx;
   19983            0 :       vars[2] = i + 2 < 8 ? ops[i + 2] : const0_rtx;
   19984            0 :       vars[3] = i + 3 < 8 ? ops[i + 3] : const0_rtx;
   19985            0 :       vars[4] = i + 4 < 8 ? ops[i + 4] : const0_rtx;
   19986            0 :       vars[5] = i + 5 < 8 ? ops[i + 5] : const0_rtx;
   19987            0 :       vars[6] = const0_rtx;
   19988            0 :       vars[7] = const0_rtx;
   19989            0 :       word = ix86_expand_vector_init_qi8 (vars);
   19990            0 :       return expand_simple_binop (DImode, ASHIFT, word, GEN_INT (i * 8),
   19991            0 :                                   NULL_RTX, 1, OPTAB_LIB_WIDEN);
   19992              :     }
   19993              : 
   19994         3741 :   vars[0] = ops[1];
   19995         3741 :   vars[1] = ops[2];
   19996         3741 :   vars[2] = ops[3];
   19997         3741 :   vars[3] = ops[4];
   19998         3741 :   vars[4] = ops[5];
   19999         3741 :   vars[5] = ops[6];
   20000         3741 :   vars[6] = ops[7];
   20001         3741 :   vars[7] = const0_rtx;
   20002         3741 :   word = ix86_expand_vector_init_qi8 (vars);
   20003         3741 :   word = expand_simple_binop (DImode, ASHIFT, word, GEN_INT (8),
   20004              :                               NULL_RTX, 1, OPTAB_LIB_WIDEN);
   20005         3741 :   if (ops[0] != const0_rtx)
   20006              :     {
   20007         3721 :       rtx elt = convert_modes (DImode, QImode, ops[0], true);
   20008         3721 :       word = expand_simple_binop (DImode, PLUS, word, elt,
   20009              :                                   NULL_RTX, 1, OPTAB_LIB_WIDEN);
   20010              :     }
   20011              :   return word;
   20012              : }
   20013              : 
   20014              : /* A subroutine of ix86_expand_vector_init for V16QImode.  */
   20015              : 
   20016              : static void
   20017          445 : ix86_expand_vector_init_v16qi (rtx target, rtx *ops)
   20018              : {
   20019          445 :   rtx vars[16];
   20020          445 :   bool ok;
   20021          445 :   int i;
   20022              : 
   20023          445 :   bool all_zero_p = true;
   20024          450 :   for (i = 0; i < 16; i++)
   20025          450 :     if (ops[i] != const0_rtx)
   20026              :       {
   20027              :         all_zero_p = false;
   20028              :         break;
   20029              :       }
   20030          445 :   if (all_zero_p)
   20031              :     {
   20032            0 :       emit_move_insn (target, CONST0_RTX (V16QImode));
   20033            1 :       return;
   20034              :     }
   20035              : 
   20036          578 :   bool all_same_p = true;
   20037          578 :   for (i = 1; i < 16; i++)
   20038          578 :     if (!rtx_equal_p (ops[i], ops[0]))
   20039              :       {
   20040              :         all_same_p = false;
   20041              :         break;
   20042              :       }
   20043          445 :   if (all_same_p)
   20044              :     {
   20045            0 :       ok = ix86_expand_vector_init_duplicate (false, V16QImode,
   20046              :                                               target, ops[0]);
   20047            0 :       gcc_assert (ok);
   20048              :       return;
   20049              :     }
   20050              : 
   20051              :   /* abababababababab */
   20052          445 :   if (TARGET_AVX2
   20053           72 :       && rtx_equal_p (ops[0], ops[2])
   20054            4 :       && rtx_equal_p (ops[1], ops[3])
   20055            4 :       && rtx_equal_p (ops[0], ops[4])
   20056            2 :       && rtx_equal_p (ops[1], ops[5])
   20057            2 :       && rtx_equal_p (ops[0], ops[6])
   20058            2 :       && rtx_equal_p (ops[1], ops[7])
   20059            2 :       && rtx_equal_p (ops[0], ops[8])
   20060            1 :       && rtx_equal_p (ops[1], ops[9])
   20061            1 :       && rtx_equal_p (ops[0], ops[10])
   20062            1 :       && rtx_equal_p (ops[1], ops[11])
   20063            1 :       && rtx_equal_p (ops[0], ops[12])
   20064            1 :       && rtx_equal_p (ops[1], ops[13])
   20065            1 :       && rtx_equal_p (ops[0], ops[14])
   20066          446 :       && rtx_equal_p (ops[1], ops[15]))
   20067              :     {
   20068            1 :       rtx tmp1 = gen_reg_rtx (V16QImode);
   20069            1 :       if (REG_P (ops[0]))
   20070              :         {
   20071            1 :           rtx op0 = force_reg (SImode, gen_lowpart (SImode, ops[0]));
   20072            1 :           rtx tmp = gen_reg_rtx (V4SImode);
   20073            1 :           emit_insn (gen_vec_setv4si_0 (tmp, CONST0_RTX (V4SImode), op0));
   20074            1 :           emit_move_insn (tmp1, gen_lowpart (V16QImode, tmp));
   20075            1 :           rtx op1 = ops[1];
   20076            1 :           if (!REG_P (op1) && !MEM_P (op1))
   20077            0 :             op1 = force_reg (QImode, op1);
   20078            1 :           emit_insn (gen_sse4_1_pinsrb (tmp1, tmp1, op1, GEN_INT (2)));
   20079              :         }
   20080              :       else
   20081              :         {
   20082            0 :           vars[0] = ops[0];
   20083            0 :           vars[1] = ops[1];
   20084            0 :           for (i = 2; i < 16; i++)
   20085            0 :             vars[i] = const0_rtx;
   20086            0 :           ix86_expand_vector_init_v16qi (tmp1, vars);
   20087              :         }
   20088            1 :       tmp1 = gen_lowpart (V8HImode, tmp1);
   20089            1 :       rtx tmp2 = gen_reg_rtx (V8HImode);
   20090            1 :       emit_insn (gen_avx2_pbroadcastv8hi (tmp2, tmp1));
   20091            1 :       emit_move_insn (target, gen_lowpart (V16QImode, tmp2));
   20092            1 :       return;
   20093              :     }
   20094              : 
   20095          451 :   bool all_const_p = true;
   20096          451 :   for (i = 0; i < 16; i++)
   20097          451 :     if (!CONST_INT_P (ops[i]))
   20098              :       {
   20099              :         all_const_p = false;
   20100              :         break;
   20101              :       }
   20102          444 :   if (all_const_p)
   20103              :     {
   20104              :       int last_nonzero = 15;
   20105            0 :       while (ops[last_nonzero] == const0_rtx)
   20106            0 :         last_nonzero--;
   20107            0 :       if (last_nonzero < 4)
   20108              :         {
   20109            0 :           HOST_WIDE_INT val = (UINTVAL (ops[0]) & 0xff)
   20110            0 :                               + ((UINTVAL (ops[1]) & 0xff) << 8)
   20111            0 :                               + ((UINTVAL (ops[2]) & 0xff) << 16)
   20112            0 :                               + ((UINTVAL (ops[3]) & 0xff) << 24);
   20113            0 :           rtx tmp1 = force_reg (SImode, gen_int_mode (val, SImode));
   20114            0 :           rtx tmp2 = gen_reg_rtx (V4SImode);
   20115            0 :           emit_insn (gen_vec_setv4si_0 (tmp2, CONST0_RTX (V4SImode), tmp1));
   20116            0 :           emit_move_insn (target, gen_lowpart (V16QImode, tmp2));
   20117            0 :           return;
   20118              :         }
   20119            0 :       rtx vec = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, ops));
   20120            0 :       emit_move_insn (target, vec);
   20121            0 :       return;
   20122              :     }
   20123              : 
   20124          444 :   if (TARGET_SSE4_1
   20125          444 :       && nonzero_int_const_count (ops, 16) >= 2)
   20126              :     {
   20127              :       rtx csts[16];
   20128              :       int count = 0;
   20129            0 :       for (i = 0; i < 16; i++)
   20130            0 :         if (CONST_INT_P (ops[i]))
   20131              :           {
   20132            0 :             csts[i] = ops[i];
   20133            0 :             vars[i] = const0_rtx;
   20134              :           }
   20135              :         else
   20136              :           {
   20137            0 :             csts[i] = const0_rtx;
   20138            0 :             vars[i] = ops[i];
   20139            0 :             count++;
   20140              :           }
   20141            0 :       rtx tmp1 = gen_reg_rtx (V16QImode);
   20142            0 :       if (count <= 3)
   20143              :         {
   20144            0 :           ix86_expand_vector_init_v16qi (tmp1, csts);
   20145            0 :           for (i = 0; i < 16; i++)
   20146            0 :             if (vars[i] != const0_rtx)
   20147              :               {
   20148            0 :                 rtx elt = vars[i];
   20149            0 :                 if (!REG_P (elt) && !MEM_P (elt))
   20150            0 :                   elt = force_reg (QImode, elt);
   20151            0 :                 emit_insn (gen_sse4_1_pinsrb (tmp1, tmp1, elt,
   20152            0 :                                               GEN_INT (1 << i)));
   20153              :               }
   20154            0 :           emit_move_insn (target, tmp1);
   20155            0 :           return;
   20156              :         }
   20157            0 :       ix86_expand_vector_init_v16qi (tmp1, vars);
   20158            0 :       rtx tmp2 = gen_reg_rtx (V16QImode);
   20159            0 :       rtx vec = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, csts));
   20160            0 :       emit_move_insn (tmp2, vec);
   20161            0 :       emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V16QImode, tmp1, tmp2)));
   20162              :     }
   20163          444 :   else if (TARGET_SSE4_1)
   20164           98 :     ix86_expand_vector_init_insert (V16QImode, target, ops, 16);
   20165          346 :   else if (TARGET_64BIT)
   20166              :     {
   20167          346 :       rtx tmp = gen_reg_rtx (V2DImode);
   20168          346 :       vars[0] = ix86_expand_vector_init_qi8 (ops);
   20169          346 :       if (rtx_equal_p (ops[0], ops[8])
   20170           57 :           && rtx_equal_p (ops[1], ops[9])
   20171           54 :           && rtx_equal_p (ops[2], ops[10])
   20172           54 :           && rtx_equal_p (ops[3], ops[11])
   20173           54 :           && rtx_equal_p (ops[4], ops[12])
   20174           54 :           && rtx_equal_p (ops[5], ops[13])
   20175           54 :           && rtx_equal_p (ops[6], ops[14])
   20176          400 :           && rtx_equal_p (ops[7], ops[15]))
   20177           54 :         vars[1] = vars[0];
   20178              :       else
   20179          292 :         vars[1] = ix86_expand_vector_init_qi8 (ops + 8);
   20180          346 :       ix86_expand_vector_init_v2di (tmp, vars);
   20181          346 :       emit_move_insn (target, gen_lowpart (V16QImode, tmp));
   20182              :     }
   20183              :   else
   20184              :     {
   20185            0 :       rtx tmp = gen_reg_rtx (V4SImode);
   20186            0 :       vars[0] = ix86_expand_vector_init_qi4 (ops);
   20187            0 :       if (rtx_equal_p (ops[0], ops[4])
   20188            0 :           && rtx_equal_p (ops[1], ops[5])
   20189            0 :           && rtx_equal_p (ops[2], ops[6])
   20190            0 :           && rtx_equal_p (ops[3], ops[7]))
   20191            0 :         vars[1] = vars[0];
   20192              :       else
   20193            0 :         vars[1] = ix86_expand_vector_init_qi4 (ops + 4);
   20194            0 :       if (rtx_equal_p (ops[0], ops[8])
   20195            0 :           && rtx_equal_p (ops[1], ops[9])
   20196            0 :           && rtx_equal_p (ops[2], ops[10])
   20197            0 :           && rtx_equal_p (ops[3], ops[11]))
   20198            0 :         vars[2] = vars[0];
   20199            0 :       else if (rtx_equal_p (ops[4], ops[8])
   20200            0 :                && rtx_equal_p (ops[5], ops[9])
   20201            0 :                && rtx_equal_p (ops[6], ops[10])
   20202            0 :                && rtx_equal_p (ops[7], ops[11]))
   20203            0 :         vars[2] = vars[1];
   20204              :       else
   20205            0 :         vars[2] = ix86_expand_vector_init_qi4 (ops + 8);
   20206            0 :       if (rtx_equal_p (ops[0], ops[12])
   20207            0 :           && rtx_equal_p (ops[1], ops[13])
   20208            0 :           && rtx_equal_p (ops[2], ops[14])
   20209            0 :           && rtx_equal_p (ops[3], ops[15]))
   20210            0 :         vars[3] = vars[0];
   20211            0 :       else if (rtx_equal_p (ops[4], ops[12])
   20212            0 :                && rtx_equal_p (ops[5], ops[13])
   20213            0 :                && rtx_equal_p (ops[6], ops[14])
   20214            0 :                && rtx_equal_p (ops[7], ops[15]))
   20215            0 :         vars[3] = vars[1];
   20216            0 :       else if (rtx_equal_p (ops[8], ops[12])
   20217            0 :                && rtx_equal_p (ops[9], ops[13])
   20218            0 :                && rtx_equal_p (ops[10], ops[14])
   20219            0 :                && rtx_equal_p (ops[11], ops[15]))
   20220            0 :         vars[3] = vars[2];
   20221              :       else
   20222            0 :         vars[3] = ix86_expand_vector_init_qi4 (ops + 12);
   20223            0 :       ix86_expand_vector_init_v4si (tmp, vars);
   20224            0 :       emit_move_insn (target, gen_lowpart (V16QImode, tmp));
   20225              :     }
   20226              : }
   20227              : 
   20228              : /* A subroutine of ix86_expand_vector_init for V4DImode.  */
   20229              : 
   20230              : static void
   20231          145 : ix86_expand_vector_init_v4di (rtx target, rtx *ops)
   20232              : {
   20233          145 :   rtx vars[4];
   20234          145 :   int perm[4];
   20235              : 
   20236          145 :   if (ops[0] == const0_rtx
   20237            0 :       && ops[1] == const0_rtx
   20238            0 :       && ops[2] == const0_rtx
   20239            0 :       && ops[3] == const0_rtx)
   20240            0 :     emit_move_insn (target, CONST0_RTX (V4DImode));
   20241          145 :   else if (ops[1] == const0_rtx
   20242            8 :            && ops[2] == const0_rtx
   20243            0 :            && ops[3] == const0_rtx)
   20244              :     {
   20245            0 :       rtx val = ops[0];
   20246            0 :       if (!REG_P (val) && !MEM_P (val))
   20247            0 :         val = force_reg (DImode, val);
   20248            0 :       emit_insn (gen_vec_setv4di_0 (target, CONST0_RTX (V4DImode), val));
   20249            0 :     }
   20250            0 :   else if ((TARGET_64BIT || MEM_P (ops[0]))
   20251          145 :            && rtx_equal_p (ops[0], ops[1])
   20252            0 :            && rtx_equal_p (ops[0], ops[2])
   20253          145 :            && rtx_equal_p (ops[0], ops[3]))
   20254              :     {
   20255            0 :       rtx val = ops[0];
   20256            0 :       if (!REG_P (val) && !MEM_P (val))
   20257            0 :         val = force_reg (DImode, val);
   20258            0 :       emit_insn (gen_vec_dupv4di (target, val));
   20259              :     }
   20260          145 :   else if (CONST_INT_P (ops[0])
   20261            2 :            && CONST_INT_P (ops[1])
   20262            0 :            && CONST_INT_P (ops[2])
   20263            0 :            && CONST_INT_P (ops[3]))
   20264              :     {
   20265            0 :       rtx vec = gen_rtx_CONST_VECTOR (V4DImode, gen_rtvec_v (4, ops));
   20266            0 :       emit_move_insn (target, vec);
   20267            0 :     }
   20268          145 :   else if (TARGET_AVX2 
   20269          145 :            && onevar_perm_p (ops, 4, perm, vars))
   20270              :     {
   20271            0 :       rtx tmp = gen_reg_rtx (V4DImode);
   20272            0 :       vars[1] = const0_rtx;
   20273            0 :       vars[2] = const0_rtx;
   20274            0 :       vars[3] = const0_rtx;
   20275            0 :       ix86_expand_vector_init_v4di (tmp, vars);
   20276            0 :       emit_insn (gen_avx2_permv4di_1 (target, tmp,
   20277            0 :                                       GEN_INT (perm[0]),
   20278            0 :                                       GEN_INT (perm[1]),
   20279            0 :                                       GEN_INT (perm[2]),
   20280            0 :                                       GEN_INT (perm[3])));
   20281              :     }
   20282          145 :   else if (rtx_equal_p (ops[0], ops[2])
   20283          145 :            && rtx_equal_p (ops[1], ops[3]))
   20284              :     {
   20285            2 :       rtx tmp = gen_reg_rtx (V2DImode);
   20286            2 :       ix86_expand_vector_init_v2di (tmp, ops);
   20287            2 :       emit_insn (gen_avx_vec_concatv4di (target, tmp, tmp));
   20288              :     }
   20289          143 :   else if (nonzero_int_const_count (ops, 4) >= 2)
   20290              :     {
   20291              :       rtx csts[4];
   20292              :       int i;
   20293            0 :       for (i = 0; i < 4; i++)
   20294            0 :         if (CONST_INT_P (ops[i]))
   20295              :           {
   20296            0 :             csts[i] = ops[i];
   20297            0 :             vars[i] = const0_rtx;
   20298              :           }
   20299              :         else
   20300              :           {
   20301            0 :             csts[i] = const0_rtx;
   20302            0 :             vars[i] = ops[i];
   20303              :           }
   20304            0 :       rtx tmp1 = gen_reg_rtx (V4DImode);
   20305            0 :       ix86_expand_vector_init_v4di (tmp1, vars);
   20306            0 :       rtx tmp2 = gen_reg_rtx (V4DImode);
   20307            0 :       rtx vec = gen_rtx_CONST_VECTOR (V4DImode, gen_rtvec_v (4, csts));
   20308            0 :       emit_move_insn (tmp2, vec);
   20309            0 :       emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V4DImode, tmp1, tmp2)));
   20310              :     }
   20311              :   else
   20312              :     {
   20313          143 :       rtx tmp1 = gen_reg_rtx (V2DImode);
   20314          143 :       ix86_expand_vector_init_v2di (tmp1, ops);
   20315          143 :       rtx tmp2 = gen_reg_rtx (V2DImode);
   20316          143 :       ix86_expand_vector_init_v2di (tmp2, ops + 2);
   20317          143 :       emit_insn (gen_avx_vec_concatv4di (target, tmp1, tmp2));
   20318              :     }
   20319          145 : }
   20320              : 
   20321              : /* A subroutine of ix86_expand_vector_init for V4DFmode.  */
   20322              : 
   20323              : static void
   20324          511 : ix86_expand_vector_init_v4df (rtx target, rtx *ops)
   20325              : {
   20326          511 :   rtx vars[4];
   20327          511 :   int perm[4];
   20328              : 
   20329          511 :   if (ops[0] == CONST0_RTX (DFmode)
   20330            0 :       && ops[1] == CONST0_RTX (DFmode)
   20331            0 :       && ops[2] == CONST0_RTX (DFmode)
   20332            0 :       && ops[3] == CONST0_RTX (DFmode))
   20333            0 :     emit_move_insn (target, CONST0_RTX (V4DFmode));
   20334          511 :   else if (ops[1] == CONST0_RTX (DFmode)
   20335            0 :            && ops[2] == CONST0_RTX (DFmode)
   20336            0 :            && ops[3] == CONST0_RTX (DFmode))
   20337              :     {
   20338            0 :       rtx val = ops[0];
   20339            0 :       if (!REG_P (val) && !MEM_P (val))
   20340            0 :         val = force_reg (DFmode, val);
   20341            0 :       emit_insn (gen_vec_setv4df_0 (target, CONST0_RTX (V4DFmode), val));
   20342            0 :     }
   20343          511 :   else if (rtx_equal_p (ops[0], ops[1])
   20344           53 :            && rtx_equal_p (ops[0], ops[2])
   20345          515 :            && rtx_equal_p (ops[0], ops[3]))
   20346              :     {
   20347            0 :       rtx val = ops[0];
   20348            0 :       if (!REG_P (val) && !MEM_P (val))
   20349            0 :         val = force_reg (DFmode, val);
   20350            0 :       emit_insn (gen_vec_dupv4df (target, val));
   20351              :     }
   20352          511 :   else if (CONST_DOUBLE_P (ops[0])
   20353            4 :            && CONST_DOUBLE_P (ops[1])
   20354            0 :            && CONST_DOUBLE_P (ops[2])
   20355            0 :            && CONST_DOUBLE_P (ops[3]))
   20356              :     {
   20357            0 :       rtx vec = gen_rtx_CONST_VECTOR (V4DFmode, gen_rtvec_v (4, ops));
   20358            0 :       emit_move_insn (target, vec);
   20359            0 :     }
   20360          511 :   else if (TARGET_AVX2 
   20361          511 :            && onevar_perm_p (ops, 4, perm, vars))
   20362              :     {
   20363            0 :       rtx tmp = gen_reg_rtx (V4DFmode);
   20364            0 :       vars[1] = CONST0_RTX (DFmode);
   20365            0 :       vars[2] = CONST0_RTX (DFmode);
   20366            0 :       vars[3] = CONST0_RTX (DFmode);
   20367            0 :       ix86_expand_vector_init_v4df (tmp, vars);
   20368            0 :       emit_insn (gen_avx2_permv4df_1 (target, tmp,
   20369            0 :                                       GEN_INT (perm[0]),
   20370            0 :                                       GEN_INT (perm[1]),
   20371            0 :                                       GEN_INT (perm[2]),
   20372            0 :                                       GEN_INT (perm[3])));
   20373              :     }
   20374          511 :   else if (rtx_equal_p (ops[0], ops[2])
   20375          511 :            && rtx_equal_p (ops[1], ops[3]))
   20376              :     {
   20377           16 :       rtx tmp = gen_reg_rtx (V2DFmode);
   20378           16 :       ix86_expand_vector_init_v2df (tmp, ops);
   20379           16 :       emit_insn (gen_avx_vec_concatv4df (target, tmp, tmp));
   20380              :     }
   20381          495 :   else if (nonzero_double_const_count (ops, 4) >= 2)
   20382              :     {
   20383              :       rtx csts[4];
   20384              :       int i;
   20385            0 :       for (i = 0; i < 4; i++)
   20386            0 :         if (CONST_DOUBLE_P (ops[i]))
   20387              :           {
   20388            0 :             csts[i] = ops[i];
   20389            0 :             vars[i] = CONST0_RTX (DFmode);
   20390              :           }
   20391              :         else
   20392              :           {
   20393            0 :             csts[i] = CONST0_RTX (DFmode);
   20394            0 :             vars[i] = ops[i];
   20395              :           }
   20396            0 :       rtx tmp1 = gen_reg_rtx (V4DFmode);
   20397            0 :       ix86_expand_vector_init_v4df (tmp1, vars);
   20398            0 :       rtx tmp2 = gen_reg_rtx (V4DFmode);
   20399            0 :       rtx vec = gen_rtx_CONST_VECTOR (V4DFmode, gen_rtvec_v (4, csts));
   20400            0 :       emit_move_insn (tmp2, vec);
   20401            0 :       emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V4DFmode, tmp1, tmp2)));
   20402              :     }
   20403              :   else
   20404              :     {
   20405          495 :       rtx tmp1 = gen_reg_rtx (V2DFmode);
   20406          495 :       ix86_expand_vector_init_v2df (tmp1, ops);
   20407          495 :       rtx tmp2 = gen_reg_rtx (V2DFmode);
   20408          495 :       ix86_expand_vector_init_v2df (tmp2, ops + 2);
   20409          495 :       emit_insn (gen_avx_vec_concatv4df (target, tmp1, tmp2));
   20410              :     }
   20411          511 : }
   20412              : 
   20413              : /* A subroutine of ix86_expand_vector_init for V8SImode.  */
   20414              : 
   20415              : static void
   20416          135 : ix86_expand_vector_init_v8si (rtx target, rtx *ops)
   20417              : {
   20418          135 :   rtx vars[8];
   20419              : 
   20420          135 :   if (ops[0] == const0_rtx
   20421            7 :       && ops[1] == const0_rtx
   20422            7 :       && ops[2] == const0_rtx
   20423            7 :       && ops[3] == const0_rtx
   20424            6 :       && ops[4] == const0_rtx
   20425            3 :       && ops[5] == const0_rtx
   20426            0 :       && ops[6] == const0_rtx
   20427            0 :       && ops[7] == const0_rtx)
   20428            0 :     emit_move_insn (target, CONST0_RTX (V8SImode));
   20429          135 :   else if (ops[1] == const0_rtx
   20430            7 :            && ops[2] == const0_rtx
   20431            7 :            && ops[3] == const0_rtx
   20432            6 :            && ops[4] == const0_rtx
   20433            3 :            && ops[5] == const0_rtx
   20434            0 :            && ops[6] == const0_rtx
   20435            0 :            && ops[7] == const0_rtx)
   20436              :     {
   20437            0 :       rtx val = ops[0];
   20438            0 :       if (!REG_P (val) && !MEM_P (val))
   20439            0 :         val = force_reg (SImode, val);
   20440            0 :       emit_insn (gen_vec_setv8si_0 (target, CONST0_RTX (V8SImode), val));
   20441            0 :     }
   20442          135 :   else if (rtx_equal_p (ops[0], ops[1])
   20443           10 :            && rtx_equal_p (ops[0], ops[2])
   20444            8 :            && rtx_equal_p (ops[0], ops[3])
   20445            7 :            && rtx_equal_p (ops[0], ops[4])
   20446            3 :            && rtx_equal_p (ops[0], ops[5])
   20447            0 :            && rtx_equal_p (ops[0], ops[6])
   20448          135 :            && rtx_equal_p (ops[0], ops[7]))
   20449              :     {
   20450            0 :       rtx val = ops[0];
   20451            0 :       if (!REG_P (val) && !MEM_P (val))
   20452            0 :         val = force_reg (SImode, val);
   20453            0 :       emit_insn (gen_vec_dupv8si (target, val));
   20454              :     }
   20455          135 :   else if (TARGET_AVX2
   20456          122 :            && rtx_equal_p (ops[0], ops[2])
   20457           32 :            && rtx_equal_p (ops[0], ops[4])
   20458           28 :            && rtx_equal_p (ops[0], ops[6])
   20459           24 :            && rtx_equal_p (ops[1], ops[3])
   20460           24 :            && rtx_equal_p (ops[1], ops[5])
   20461          159 :            && rtx_equal_p (ops[1], ops[7]))
   20462              :     {
   20463           24 :       rtx tmp_ops[4] = { ops[0], ops[1], const0_rtx, const0_rtx };
   20464           24 :       rtx tmp1 = gen_reg_rtx (V4SImode);
   20465           24 :       ix86_expand_vector_init_v4si (tmp1, tmp_ops);
   20466           24 :       tmp1 = gen_lowpart (V2DImode, tmp1);
   20467           24 :       rtx tmp2 = gen_reg_rtx (V4DImode);
   20468           24 :       emit_insn (gen_avx2_pbroadcastv4di (tmp2, tmp1));
   20469           24 :       emit_move_insn (target, gen_lowpart (V8SImode, tmp2));
   20470              :     }
   20471          111 :   else if (ops[4] == const0_rtx
   20472            9 :            && ops[5] == const0_rtx
   20473            5 :            && ops[6] == const0_rtx
   20474            5 :            && ops[7] == const0_rtx)
   20475              :     {
   20476            5 :       rtx tmp = gen_reg_rtx (V4SImode);
   20477            5 :       ix86_expand_vector_init_v4si (tmp, ops);
   20478            5 :       emit_insn (gen_avx_vec_concatv8si (target, tmp, CONST0_RTX (V4SImode)));
   20479            5 :     }
   20480          106 :   else if (CONST_INT_P (ops[0])
   20481            9 :            && CONST_INT_P (ops[1])
   20482            7 :            && CONST_INT_P (ops[2])
   20483            7 :            && CONST_INT_P (ops[3])
   20484            7 :            && CONST_INT_P (ops[4])
   20485            4 :            && CONST_INT_P (ops[5])
   20486            0 :            && CONST_INT_P (ops[6])
   20487            0 :            && CONST_INT_P (ops[7]))
   20488              :     {
   20489            0 :       rtx vec = gen_rtx_CONST_VECTOR (V8SImode, gen_rtvec_v (8, ops));
   20490            0 :       emit_move_insn (target, vec);
   20491            0 :     }
   20492          106 :   else if (rtx_equal_p (ops[0], ops[4])
   20493           26 :            && rtx_equal_p (ops[1], ops[5])
   20494           22 :            && rtx_equal_p (ops[2], ops[6])
   20495          128 :            && rtx_equal_p (ops[3], ops[7]))
   20496              :     {
   20497           22 :       rtx tmp = gen_reg_rtx (V4SImode);
   20498           22 :       ix86_expand_vector_init_v4si (tmp, ops);
   20499           22 :       emit_insn (gen_avx_vec_concatv8si (target, tmp, tmp));
   20500              :     }
   20501           84 :   else if (nonzero_int_const_count (ops, 8) >= 2)
   20502              :     {
   20503              :       rtx csts[8];
   20504              :       int i;
   20505            9 :       for (i = 0; i < 8; i++)
   20506            8 :         if (CONST_INT_P (ops[i]))
   20507              :           {
   20508            6 :             csts[i] = ops[i];
   20509            6 :             vars[i] = const0_rtx;
   20510              :           }
   20511              :         else
   20512              :           {
   20513            2 :             csts[i] = const0_rtx;
   20514            2 :             vars[i] = ops[i];
   20515              :           }
   20516            1 :       rtx tmp1 = gen_reg_rtx (V8SImode);
   20517            1 :       ix86_expand_vector_init_v8si (tmp1, vars);
   20518            1 :       rtx tmp2 = gen_reg_rtx (V8SImode);
   20519            1 :       rtx vec = gen_rtx_CONST_VECTOR (V8SImode, gen_rtvec_v (8, csts));
   20520            1 :       emit_move_insn (tmp2, vec);
   20521            1 :       emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V8SImode, tmp1, tmp2)));
   20522              :     }
   20523              :   else
   20524              :     {
   20525           83 :       rtx tmp1 = gen_reg_rtx (V4SImode);
   20526           83 :       ix86_expand_vector_init_v4si (tmp1, ops);
   20527           83 :       rtx tmp2 = gen_reg_rtx (V4SImode);
   20528           83 :       ix86_expand_vector_init_v4si (tmp2, ops + 4);
   20529           83 :       emit_insn (gen_avx_vec_concatv8si (target, tmp1, tmp2));
   20530              :     }
   20531          135 : }
   20532              : 
   20533              : /* A subroutine of ix86_expand_vector_init for V8SFmode.  */
   20534              : 
   20535              : static void
   20536          249 : ix86_expand_vector_init_v8sf (rtx target, rtx *ops)
   20537              : {
   20538          249 :   rtx vars[8];
   20539              : 
   20540          249 :   if (ops[0] == CONST0_RTX (SFmode)
   20541            0 :       && ops[1] == CONST0_RTX (SFmode)
   20542            0 :       && ops[2] == CONST0_RTX (SFmode)
   20543            0 :       && ops[3] == CONST0_RTX (SFmode)
   20544            0 :       && ops[4] == CONST0_RTX (SFmode)
   20545            0 :       && ops[5] == CONST0_RTX (SFmode)
   20546            0 :       && ops[6] == CONST0_RTX (SFmode)
   20547            0 :       && ops[7] == CONST0_RTX (SFmode))
   20548            0 :     emit_move_insn (target, CONST0_RTX (V8SFmode));
   20549          249 :   else if (ops[1] == CONST0_RTX (SFmode)
   20550           24 :            && ops[2] == CONST0_RTX (SFmode)
   20551            0 :            && ops[3] == CONST0_RTX (SFmode)
   20552            0 :            && ops[4] == CONST0_RTX (SFmode)
   20553            0 :            && ops[5] == CONST0_RTX (SFmode)
   20554            0 :            && ops[6] == CONST0_RTX (SFmode)
   20555            0 :            && ops[7] == CONST0_RTX (SFmode))
   20556              :     {
   20557            0 :       rtx val = ops[0];
   20558            0 :       if (!REG_P (val) && !MEM_P (val))
   20559            0 :         val = force_reg (SFmode, val);
   20560            0 :       emit_insn (gen_vec_setv8sf_0 (target, CONST0_RTX (V8SFmode), val));
   20561            0 :     }
   20562          249 :   else if (TARGET_AVX2
   20563          175 :            && rtx_equal_p (ops[0], ops[1])
   20564           32 :            && rtx_equal_p (ops[0], ops[2])
   20565            0 :            && rtx_equal_p (ops[0], ops[3])
   20566            0 :            && rtx_equal_p (ops[0], ops[4])
   20567            0 :            && rtx_equal_p (ops[0], ops[5])
   20568            0 :            && rtx_equal_p (ops[0], ops[6])
   20569          249 :            && rtx_equal_p (ops[0], ops[7]))
   20570              :     {
   20571            0 :       rtx val = ops[0];
   20572            0 :       if (!REG_P (val) && !MEM_P (val))
   20573            0 :         val = force_reg (SFmode, val);
   20574            0 :       emit_insn (gen_avx2_vec_dupv8sf_1 (target, val));
   20575              :     }
   20576          249 :   else if (TARGET_AVX2
   20577          175 :            && rtx_equal_p (ops[0], ops[2])
   20578           32 :            && rtx_equal_p (ops[0], ops[4])
   20579           32 :            && rtx_equal_p (ops[0], ops[6])
   20580           32 :            && rtx_equal_p (ops[1], ops[3])
   20581            0 :            && rtx_equal_p (ops[1], ops[5])
   20582          249 :            && rtx_equal_p (ops[1], ops[7]))
   20583              :     {
   20584            0 :       rtx tmp_ops[4] = { ops[0], ops[1], CONST0_RTX (SFmode),
   20585            0 :                                          CONST0_RTX (SFmode) };
   20586            0 :       rtx tmp1 = gen_reg_rtx (V4SFmode);
   20587            0 :       ix86_expand_vector_init_v4sf (tmp1, tmp_ops);
   20588            0 :       tmp1 = gen_lowpart (V2DImode, tmp1);
   20589            0 :       rtx tmp2 = gen_reg_rtx (V4DImode);
   20590            0 :       emit_insn (gen_avx2_pbroadcastv4di (tmp2, tmp1));
   20591            0 :       emit_move_insn (target, gen_lowpart (V8SFmode, tmp2));
   20592              :     }
   20593          249 :   else if (ops[4] == CONST0_RTX (SFmode)
   20594            0 :            && ops[5] == CONST0_RTX (SFmode)
   20595            0 :            && ops[6] == CONST0_RTX (SFmode)
   20596            0 :            && ops[7] == CONST0_RTX (SFmode))
   20597              :     {
   20598            0 :       rtx tmp = gen_reg_rtx (V4SFmode);
   20599            0 :       ix86_expand_vector_init_v4sf (tmp, ops);
   20600            0 :       emit_insn (gen_avx_vec_concatv8sf (target, tmp, CONST0_RTX (V4SFmode)));
   20601            0 :     }
   20602          249 :   else if (CONST_DOUBLE_P (ops[0])
   20603            2 :            && CONST_DOUBLE_P (ops[1])
   20604            0 :            && CONST_DOUBLE_P (ops[2])
   20605            0 :            && CONST_DOUBLE_P (ops[3])
   20606            0 :            && CONST_DOUBLE_P (ops[4])
   20607            0 :            && CONST_DOUBLE_P (ops[5])
   20608            0 :            && CONST_DOUBLE_P (ops[6])
   20609            0 :            && CONST_DOUBLE_P (ops[7]))
   20610              :     {
   20611            0 :       rtx vec = gen_rtx_CONST_VECTOR (V8SFmode, gen_rtvec_v (8, ops));
   20612            0 :       emit_move_insn (target, vec);
   20613            0 :     }
   20614          249 :   else if (rtx_equal_p (ops[0], ops[4])
   20615           78 :            && rtx_equal_p (ops[1], ops[5])
   20616           22 :            && rtx_equal_p (ops[2], ops[6])
   20617          271 :            && rtx_equal_p (ops[3], ops[7]))
   20618              :     {
   20619           22 :       rtx tmp = gen_reg_rtx (V4SFmode);
   20620           22 :       ix86_expand_vector_init_v4sf (tmp, ops);
   20621           22 :       emit_insn (gen_avx_vec_concatv8sf (target, tmp, tmp));
   20622              :     }
   20623          227 :   else if (nonzero_float_const_count (ops, 8) >= 2)
   20624              :     {
   20625              :       rtx csts[8];
   20626              :       int i;
   20627            0 :       for (i = 0; i < 8; i++)
   20628            0 :         if (CONST_DOUBLE_P (ops[i]))
   20629              :           {
   20630            0 :             csts[i] = ops[i];
   20631            0 :             vars[i] = CONST0_RTX (SFmode);
   20632              :           }
   20633              :         else
   20634              :           {
   20635            0 :             csts[i] = CONST0_RTX (SFmode);
   20636            0 :             vars[i] = ops[i];
   20637              :           }
   20638            0 :       rtx tmp1 = gen_reg_rtx (V8SFmode);
   20639            0 :       ix86_expand_vector_init_v8sf (tmp1, vars);
   20640            0 :       rtx tmp2 = gen_reg_rtx (V8SFmode);
   20641            0 :       rtx vec = gen_rtx_CONST_VECTOR (V8SFmode, gen_rtvec_v (8, csts));
   20642            0 :       emit_move_insn (tmp2, vec);
   20643            0 :       emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V8SFmode, tmp1, tmp2)));
   20644              :     }
   20645              :   else
   20646              :     {
   20647          227 :       rtx tmp1 = gen_reg_rtx (V4SFmode);
   20648          227 :       ix86_expand_vector_init_v4sf (tmp1, ops);
   20649          227 :       rtx tmp2 = gen_reg_rtx (V4SFmode);
   20650          227 :       ix86_expand_vector_init_v4sf (tmp2, ops + 4);
   20651          227 :       emit_insn (gen_avx_vec_concatv8sf (target, tmp1, tmp2));
   20652              :     }
   20653          249 : }
   20654              : 
   20655              : 
   20656              : /* A subroutine of ix86_expand_vector_init.  Handle the most general case:
   20657              :    all values variable, and none identical.  */
   20658              : 
   20659              : static void
   20660       103040 : ix86_expand_vector_init_general (bool mmx_ok, machine_mode mode,
   20661              :                                  rtx target, rtx vals)
   20662              : {
   20663       103040 :   rtx ops[64], op0, op1, op2, op3, op4, op5;
   20664       103040 :   machine_mode half_mode = VOIDmode;
   20665       103040 :   machine_mode quarter_mode = VOIDmode;
   20666       103040 :   machine_mode int_inner_mode = VOIDmode;
   20667       103040 :   int n, i;
   20668              : 
   20669       103040 :   switch (mode)
   20670              :     {
   20671        59767 :     case E_V2DImode:
   20672        59767 :       ops[0] = XVECEXP (vals, 0, 0);
   20673        59767 :       ops[1] = XVECEXP (vals, 0, 1);
   20674        59767 :       ix86_expand_vector_init_v2di (target, ops);
   20675       154530 :       return;
   20676              : 
   20677         3047 :     case E_V2DFmode:
   20678         3047 :       ops[0] = XVECEXP (vals, 0, 0);
   20679         3047 :       ops[1] = XVECEXP (vals, 0, 1);
   20680         3047 :       ix86_expand_vector_init_v2df (target, ops);
   20681         3047 :       return;
   20682              : 
   20683              :     case E_V4SImode:
   20684        27210 :       for (i = 0; i < 4; i++)
   20685        21768 :         ops[i] = XVECEXP (vals, 0, i);
   20686         5442 :       ix86_expand_vector_init_v4si (target, ops);
   20687         5442 :       return;
   20688              : 
   20689              :     case E_V4SFmode:
   20690         7750 :       for (i = 0; i < 4; i++)
   20691         6200 :         ops[i] = XVECEXP (vals, 0, i);
   20692         1550 :       ix86_expand_vector_init_v4sf (target, ops);
   20693         1550 :       return;
   20694              : 
   20695              :     case E_V8HImode:
   20696         5013 :       for (i = 0; i < 8; i++)
   20697         4456 :         ops[i] = XVECEXP (vals, 0, i);
   20698          557 :       if (ix86_expand_vector_init_v8hi (target, ops))
   20699              :         return;
   20700              :       break;
   20701              : 
   20702              :     case E_V16QImode:
   20703         7565 :       for (i = 0; i < 16; i++)
   20704         7120 :         ops[i] = XVECEXP (vals, 0, i);
   20705          445 :       ix86_expand_vector_init_v16qi (target, ops);
   20706          445 :       return;
   20707              : 
   20708              :     case E_V4DImode:
   20709          725 :       for (i = 0; i < 4; i++)
   20710          580 :         ops[i] = XVECEXP (vals, 0, i);
   20711          145 :       ix86_expand_vector_init_v4di (target,ops);
   20712          145 :       return;
   20713              : 
   20714              :     case E_V4DFmode:
   20715         2555 :       for (i = 0; i < 4; i++)
   20716         2044 :         ops[i] = XVECEXP (vals, 0, i);
   20717          511 :       ix86_expand_vector_init_v4df (target,ops);
   20718          511 :       return;
   20719              : 
   20720              :     case E_V8SImode:
   20721         1206 :       for (i = 0; i < 8; i++)
   20722         1072 :         ops[i] = XVECEXP (vals, 0, i);
   20723          134 :       ix86_expand_vector_init_v8si (target,ops);
   20724          134 :       return;
   20725              : 
   20726              :     case E_V8SFmode:
   20727         2241 :       for (i = 0; i < 8; i++)
   20728         1992 :         ops[i] = XVECEXP (vals, 0, i);
   20729          249 :       ix86_expand_vector_init_v8sf (target,ops);
   20730          249 :       return;
   20731              : 
   20732        21028 :     case E_V2SFmode:
   20733        21028 :     case E_V2SImode:
   20734        21028 :       if (!mmx_ok && !TARGET_SSE)
   20735              :         break;
   20736              :       /* FALLTHRU */
   20737              : 
   20738        21204 :     case E_V16SImode:
   20739        21204 :     case E_V16SFmode:
   20740        21204 :     case E_V8DFmode:
   20741        21204 :     case E_V8DImode:
   20742        21204 :       n = GET_MODE_NUNITS (mode);
   20743        65236 :       for (i = 0; i < n; i++)
   20744        44032 :         ops[i] = XVECEXP (vals, 0, i);
   20745        21204 :       ix86_expand_vector_init_concat (mode, target, ops, n);
   20746        21204 :       return;
   20747              : 
   20748              :     case E_V2TImode:
   20749          132 :       for (i = 0; i < 2; i++)
   20750           88 :         ops[i] = gen_lowpart (V2DImode, XVECEXP (vals, 0, i));
   20751           44 :       op0 = gen_reg_rtx (V4DImode);
   20752           44 :       ix86_expand_vector_init_concat (V4DImode, op0, ops, 2);
   20753           44 :       emit_move_insn (target, gen_lowpart (GET_MODE (target), op0));
   20754           44 :       return;
   20755              : 
   20756              :     case E_V4TImode:
   20757          195 :       for (i = 0; i < 4; i++)
   20758          156 :         ops[i] = gen_lowpart (V2DImode, XVECEXP (vals, 0, i));
   20759           39 :       ops[4] = gen_reg_rtx (V4DImode);
   20760           39 :       ix86_expand_vector_init_concat (V4DImode, ops[4], ops, 2);
   20761           39 :       ops[5] = gen_reg_rtx (V4DImode);
   20762           39 :       ix86_expand_vector_init_concat (V4DImode, ops[5], ops + 2, 2);
   20763           39 :       op0 = gen_reg_rtx (V8DImode);
   20764           39 :       ix86_expand_vector_init_concat (V8DImode, op0, ops + 4, 2);
   20765           39 :       emit_move_insn (target, gen_lowpart (GET_MODE (target), op0));
   20766           39 :       return;
   20767              : 
   20768           69 :     case E_V32QImode:
   20769           69 :       half_mode = V16QImode;
   20770           69 :       goto half;
   20771              : 
   20772           64 :     case E_V16HImode:
   20773           64 :       half_mode = V8HImode;
   20774           64 :       goto half;
   20775              : 
   20776          237 :     case E_V16HFmode:
   20777          237 :       half_mode = V8HFmode;
   20778          237 :       goto half;
   20779              : 
   20780           95 :     case E_V16BFmode:
   20781           95 :       half_mode = V8BFmode;
   20782           95 :       goto half;
   20783              : 
   20784          465 : half:
   20785          465 :       n = GET_MODE_NUNITS (mode);
   20786         9009 :       for (i = 0; i < n; i++)
   20787         8544 :         ops[i] = XVECEXP (vals, 0, i);
   20788          465 :       op0 = gen_reg_rtx (half_mode);
   20789          465 :       op1 = gen_reg_rtx (half_mode);
   20790          465 :       ix86_expand_vector_init_interleave (half_mode, op0, ops,
   20791              :                                           n >> 2);
   20792          465 :       ix86_expand_vector_init_interleave (half_mode, op1,
   20793          465 :                                           &ops [n >> 1], n >> 2);
   20794          465 :       emit_insn (gen_rtx_SET (target, gen_rtx_VEC_CONCAT (mode, op0, op1)));
   20795          465 :       return;
   20796              : 
   20797           56 :     case E_V64QImode:
   20798           56 :       quarter_mode = V16QImode;
   20799           56 :       half_mode = V32QImode;
   20800           56 :       goto quarter;
   20801              : 
   20802           71 :     case E_V32HImode:
   20803           71 :       quarter_mode = V8HImode;
   20804           71 :       half_mode = V16HImode;
   20805           71 :       goto quarter;
   20806              : 
   20807          287 :     case E_V32HFmode:
   20808          287 :       quarter_mode = V8HFmode;
   20809          287 :       half_mode = V16HFmode;
   20810          287 :       goto quarter;
   20811              : 
   20812           51 :     case E_V32BFmode:
   20813           51 :       quarter_mode = V8BFmode;
   20814           51 :       half_mode = V16BFmode;
   20815           51 :       goto quarter;
   20816              : 
   20817          465 : quarter:
   20818          465 :       n = GET_MODE_NUNITS (mode);
   20819        17137 :       for (i = 0; i < n; i++)
   20820        16672 :         ops[i] = XVECEXP (vals, 0, i);
   20821          465 :       op0 = gen_reg_rtx (quarter_mode);
   20822          465 :       op1 = gen_reg_rtx (quarter_mode);
   20823          465 :       op2 = gen_reg_rtx (quarter_mode);
   20824          465 :       op3 = gen_reg_rtx (quarter_mode);
   20825          465 :       op4 = gen_reg_rtx (half_mode);
   20826          465 :       op5 = gen_reg_rtx (half_mode);
   20827          465 :       ix86_expand_vector_init_interleave (quarter_mode, op0, ops,
   20828              :                                           n >> 3);
   20829          465 :       ix86_expand_vector_init_interleave (quarter_mode, op1,
   20830          465 :                                           &ops [n >> 2], n >> 3);
   20831          465 :       ix86_expand_vector_init_interleave (quarter_mode, op2,
   20832          465 :                                           &ops [n >> 1], n >> 3);
   20833          465 :       ix86_expand_vector_init_interleave (quarter_mode, op3,
   20834          465 :                                           &ops [(n >> 1) | (n >> 2)], n >> 3);
   20835          465 :       emit_insn (gen_rtx_SET (op4, gen_rtx_VEC_CONCAT (half_mode, op0, op1)));
   20836          465 :       emit_insn (gen_rtx_SET (op5, gen_rtx_VEC_CONCAT (half_mode, op2, op3)));
   20837          465 :       emit_insn (gen_rtx_SET (target, gen_rtx_VEC_CONCAT (mode, op4, op5)));
   20838          465 :       return;
   20839              : 
   20840          699 :     case E_V8HFmode:
   20841          699 :     case E_V8BFmode:
   20842          699 :       n = GET_MODE_NUNITS (mode);
   20843         6291 :       for (i = 0; i < n; i++)
   20844         5592 :         ops[i] = XVECEXP (vals, 0, i);
   20845          699 :       ix86_expand_vector_init_interleave (mode, target, ops, n >> 1);
   20846          699 :       return;
   20847              : 
   20848              :     case E_V4HFmode:
   20849              :     case E_V4BFmode:
   20850              :     case E_V2HFmode:
   20851              :     case E_V2BFmode:
   20852         8277 :       int_inner_mode = HImode;
   20853              :       break;
   20854              : 
   20855              :     case E_V4HImode:
   20856              :     case E_V8QImode:
   20857              : 
   20858              :     case E_V2HImode:
   20859              :     case E_V4QImode:
   20860              :       break;
   20861              : 
   20862            0 :     default:
   20863            0 :       gcc_unreachable ();
   20864              :     }
   20865              : 
   20866         8277 :     {
   20867         8277 :       int i, j, n_elts, n_words, n_elt_per_word;
   20868         8277 :       machine_mode tmp_mode, inner_mode;
   20869         8277 :       rtx words[4], shift;
   20870              : 
   20871        16625 :       tmp_mode = (GET_MODE_SIZE (mode) < UNITS_PER_WORD) ? SImode : word_mode;
   20872              : 
   20873         8277 :       inner_mode = GET_MODE_INNER (mode);
   20874         8277 :       n_elts = GET_MODE_NUNITS (mode);
   20875        16554 :       n_words = GET_MODE_SIZE (mode) / GET_MODE_SIZE (tmp_mode);
   20876         8277 :       n_elt_per_word = n_elts / n_words;
   20877         8277 :       shift = GEN_INT (GET_MODE_BITSIZE (inner_mode));
   20878              : 
   20879        24839 :       for (i = 0; i < n_words; ++i)
   20880              :         {
   20881              :           rtx word = NULL_RTX;
   20882              : 
   20883        41287 :           for (j = 0; j < n_elt_per_word; ++j)
   20884              :             {
   20885        33002 :               rtx elt = XVECEXP (vals, 0, (i+1)*n_elt_per_word - j - 1);
   20886        33002 :               if (int_inner_mode != E_VOIDmode)
   20887              :                 {
   20888          328 :                   gcc_assert (TARGET_SSE2 && int_inner_mode == HImode);
   20889          328 :                   rtx tmp = gen_reg_rtx (int_inner_mode);
   20890          328 :                   elt = lowpart_subreg (int_inner_mode,
   20891              :                                         force_reg (inner_mode, elt),
   20892              :                                         inner_mode);
   20893          328 :                   emit_move_insn (tmp, elt);
   20894          328 :                   elt = tmp;
   20895              :                 }
   20896        33002 :               elt = convert_modes (tmp_mode, inner_mode, elt, true);
   20897              : 
   20898        33002 :               if (j == 0)
   20899              :                 word = elt;
   20900              :               else
   20901              :                 {
   20902        24717 :                   word = expand_simple_binop (tmp_mode, ASHIFT, word, shift,
   20903              :                                               NULL_RTX, 1, OPTAB_LIB_WIDEN);
   20904        24717 :                   word = expand_simple_binop (tmp_mode, IOR, word, elt,
   20905              :                                               NULL_RTX, 1, OPTAB_LIB_WIDEN);
   20906              :                 }
   20907              :             }
   20908              : 
   20909         8285 :           words[i] = word;
   20910              :         }
   20911              : 
   20912         8277 :       if (n_words == 1)
   20913         8269 :         emit_move_insn (target, gen_lowpart (mode, words[0]));
   20914            8 :       else if (n_words == 2)
   20915              :         {
   20916            8 :           gcc_assert (tmp_mode == DImode || tmp_mode == SImode);
   20917            8 :           machine_mode concat_mode = tmp_mode == DImode ? V2DImode : V2SImode;
   20918            8 :           rtx tmp = gen_reg_rtx (concat_mode);
   20919            8 :           vals = gen_rtx_PARALLEL (concat_mode, gen_rtvec_v (2, words));
   20920            8 :           ix86_expand_vector_init_general (mmx_ok, concat_mode, tmp, vals);
   20921            8 :           emit_move_insn (target, gen_lowpart (mode, tmp));
   20922              :         }
   20923            0 :       else if (n_words == 4)
   20924              :         {
   20925            0 :           rtx tmp = gen_reg_rtx (V4SImode);
   20926            0 :           gcc_assert (tmp_mode == SImode);
   20927            0 :           vals = gen_rtx_PARALLEL (V4SImode, gen_rtvec_v (4, words));
   20928            0 :           ix86_expand_vector_init_general (false, V4SImode, tmp, vals);
   20929            0 :           emit_move_insn (target, gen_lowpart (mode, tmp));
   20930              :         }
   20931              :       else
   20932            0 :         gcc_unreachable ();
   20933              :     }
   20934              : }
   20935              : 
   20936              : /* Initialize vector TARGET via VALS.  Suppress the use of MMX
   20937              :    instructions unless MMX_OK is true.  */
   20938              : 
   20939              : void
   20940       121237 : ix86_expand_vector_init (bool mmx_ok, rtx target, rtx vals)
   20941              : {
   20942       121237 :   machine_mode mode = GET_MODE (target);
   20943       121237 :   machine_mode inner_mode = GET_MODE_INNER (mode);
   20944       121237 :   int n_elts = GET_MODE_NUNITS (mode);
   20945       121237 :   int n_var = 0, one_var = -1;
   20946       121237 :   bool all_same = true, all_const_zero = true;
   20947       121237 :   int i;
   20948       121237 :   rtx x;
   20949              : 
   20950              :   /* Handle first initialization from vector elts.  */
   20951       121237 :   if (n_elts != XVECLEN (vals, 0))
   20952              :     {
   20953         1411 :       rtx subtarget = target;
   20954         1411 :       x = XVECEXP (vals, 0, 0);
   20955         2822 :       gcc_assert (GET_MODE_INNER (GET_MODE (x)) == inner_mode);
   20956         2822 :       if (GET_MODE_NUNITS (GET_MODE (x)) * 2 == n_elts)
   20957              :         {
   20958         1411 :           rtx ops[2] = { XVECEXP (vals, 0, 0), XVECEXP (vals, 0, 1) };
   20959         1411 :           if (inner_mode == QImode
   20960         1411 :               || inner_mode == HImode
   20961         1411 :               || inner_mode == TImode
   20962              :               || inner_mode == HFmode
   20963              :               || inner_mode == BFmode)
   20964              :             {
   20965          197 :               unsigned int n_bits = n_elts * GET_MODE_SIZE (inner_mode);
   20966          197 :               scalar_mode elt_mode = inner_mode == TImode ? DImode : SImode;
   20967          197 :               n_bits /= GET_MODE_SIZE (elt_mode);
   20968          197 :               mode = mode_for_vector (elt_mode, n_bits).require ();
   20969          197 :               inner_mode = mode_for_vector (elt_mode, n_bits / 2).require ();
   20970          197 :               ops[0] = gen_lowpart (inner_mode, ops[0]);
   20971          197 :               ops[1] = gen_lowpart (inner_mode, ops[1]);
   20972          197 :               subtarget = gen_reg_rtx (mode);
   20973              :             }
   20974         1411 :           ix86_expand_vector_init_concat (mode, subtarget, ops, 2);
   20975         1411 :           if (subtarget != target)
   20976          197 :             emit_move_insn (target, gen_lowpart (GET_MODE (target), subtarget));
   20977         1411 :           return;
   20978              :         }
   20979            0 :       gcc_unreachable ();
   20980              :     }
   20981              : 
   20982       450124 :   for (i = 0; i < n_elts; ++i)
   20983              :     {
   20984       330298 :       x = XVECEXP (vals, 0, i);
   20985       639005 :       if (!(CONST_SCALAR_INT_P (x)
   20986       312638 :             || CONST_DOUBLE_P (x)
   20987              :             || CONST_FIXED_P (x)))
   20988       308707 :         n_var++, one_var = i;
   20989        21591 :       else if (x != CONST0_RTX (inner_mode))
   20990         3040 :         all_const_zero = false;
   20991       330298 :       if (i > 0 && !rtx_equal_p (x, XVECEXP (vals, 0, 0)))
   20992              :         all_same = false;
   20993              :     }
   20994              : 
   20995              :   /* Handle the zero vector as special case.  */
   20996       119826 :   if (n_var == 0 && all_const_zero)
   20997              :     {
   20998            3 :       emit_move_insn (target, CONST0_RTX (mode));
   20999            3 :       return;
   21000              :     }
   21001              : 
   21002              :   /* If all values are identical, broadcast the value.  */
   21003       119823 :   if (all_same
   21004       127139 :       && ix86_expand_vector_init_duplicate (mmx_ok, mode, target,
   21005         7316 :                                             XVECEXP (vals, 0, 0)))
   21006              :     return;
   21007              : 
   21008              :   /* Constants are best loaded from the constant pool.  */
   21009       113346 :   if (n_var == 0)
   21010              :     {
   21011           14 :       emit_move_insn (target, gen_rtx_CONST_VECTOR (mode, XVEC (vals, 0)));
   21012           14 :       return;
   21013              :     }
   21014              : 
   21015              :   /* Values where only one field is non-constant are best loaded from
   21016              :      the pool and overwritten via move later.  */
   21017       113332 :   if (n_var == 1)
   21018              :     {
   21019        11753 :       if (all_const_zero
   21020        22471 :           && ix86_expand_vector_init_one_nonzero (mmx_ok, mode, target,
   21021        10718 :                                                   XVECEXP (vals, 0, one_var),
   21022              :                                                   one_var))
   21023              :         return;
   21024              : 
   21025         1683 :       if (ix86_expand_vector_init_one_var (mmx_ok, mode, target, vals, one_var))
   21026              :         return;
   21027              :     }
   21028              : 
   21029       103032 :   ix86_expand_vector_init_general (mmx_ok, mode, target, vals);
   21030              : }
   21031              : 
   21032              : /* Implemented as
   21033              :    V setg (V v, int idx, T val)
   21034              :    {
   21035              :      V idxv = (V){idx, idx, idx, idx, idx, idx, idx, idx};
   21036              :      V valv = (V){val, val, val, val, val, val, val, val};
   21037              :      V mask = ((V){0, 1, 2, 3, 4, 5, 6, 7} == idxv);
   21038              :      v = (v & ~mask) | (valv & mask);
   21039              :      return v;
   21040              :    }.  */
   21041              : void
   21042          129 : ix86_expand_vector_set_var (rtx target, rtx val, rtx idx)
   21043              : {
   21044          129 :   rtx vec[64];
   21045          129 :   machine_mode mode = GET_MODE (target);
   21046          129 :   machine_mode cmp_mode = mode;
   21047          129 :   int n_elts = GET_MODE_NUNITS (mode);
   21048          129 :   rtx valv,idxv,constv,idx_tmp;
   21049          129 :   bool ok = false;
   21050              : 
   21051          258 :   val = force_reg (GET_MODE_INNER (mode), val);
   21052              : 
   21053              :   /* 512-bits vector byte/word broadcast and comparison only available
   21054              :      under TARGET_AVX512BW, break 512-bits vector into two 256-bits vector
   21055              :      when without TARGET_AVX512BW.  */
   21056          129 :   if ((mode == V32HImode || mode == V32HFmode || mode == V32BFmode
   21057          123 :        || mode == V64QImode)
   21058           10 :       && !TARGET_AVX512BW)
   21059              :     {
   21060            3 :       gcc_assert (TARGET_AVX512F);
   21061            3 :       rtx vhi, vlo, idx_hi;
   21062            3 :       machine_mode half_mode;
   21063            3 :       rtx (*extract_hi)(rtx, rtx);
   21064            3 :       rtx (*extract_lo)(rtx, rtx);
   21065              : 
   21066            3 :       if (mode == V32HImode)
   21067              :         {
   21068              :           half_mode = V16HImode;
   21069              :           extract_hi = gen_vec_extract_hi_v32hi;
   21070              :           extract_lo = gen_vec_extract_lo_v32hi;
   21071              :         }
   21072              :       else if (mode == V32HFmode)
   21073              :         {
   21074              :           half_mode = V16HFmode;
   21075              :           extract_hi = gen_vec_extract_hi_v32hf;
   21076              :           extract_lo = gen_vec_extract_lo_v32hf;
   21077              :         }
   21078              :       else if (mode == V32BFmode)
   21079              :         {
   21080              :           half_mode = V16BFmode;
   21081              :           extract_hi = gen_vec_extract_hi_v32bf;
   21082              :           extract_lo = gen_vec_extract_lo_v32bf;
   21083              :         }
   21084              :       else
   21085              :         {
   21086            3 :           half_mode = V32QImode;
   21087            3 :           extract_hi = gen_vec_extract_hi_v64qi;
   21088            3 :           extract_lo = gen_vec_extract_lo_v64qi;
   21089              :         }
   21090              : 
   21091            3 :       vhi = gen_reg_rtx (half_mode);
   21092            3 :       vlo = gen_reg_rtx (half_mode);
   21093            3 :       idx_hi = gen_reg_rtx (GET_MODE (idx));
   21094            3 :       emit_insn (extract_hi (vhi, target));
   21095            3 :       emit_insn (extract_lo (vlo, target));
   21096            3 :       vec[0] = idx_hi;
   21097            3 :       vec[1] = idx;
   21098            3 :       vec[2] = GEN_INT (n_elts/2);
   21099            3 :       ix86_expand_binary_operator (MINUS, GET_MODE (idx), vec);
   21100            3 :       ix86_expand_vector_set_var (vhi, val, idx_hi);
   21101            3 :       ix86_expand_vector_set_var (vlo, val, idx);
   21102            3 :       emit_insn (gen_rtx_SET (target, gen_rtx_VEC_CONCAT (mode, vlo, vhi)));
   21103            3 :       return;
   21104              :     }
   21105              : 
   21106          504 :   if (FLOAT_MODE_P (GET_MODE_INNER (mode)))
   21107              :     {
   21108           42 :       switch (mode)
   21109              :         {
   21110              :         case E_V2DFmode:
   21111              :           cmp_mode = V2DImode;
   21112              :           break;
   21113            6 :         case E_V4DFmode:
   21114            6 :           cmp_mode = V4DImode;
   21115            6 :           break;
   21116            4 :         case E_V8DFmode:
   21117            4 :           cmp_mode = V8DImode;
   21118            4 :           break;
   21119            2 :         case E_V2SFmode:
   21120            2 :           cmp_mode = V2SImode;
   21121            2 :           break;
   21122            6 :         case E_V4SFmode:
   21123            6 :           cmp_mode = V4SImode;
   21124            6 :           break;
   21125            6 :         case E_V8SFmode:
   21126            6 :           cmp_mode = V8SImode;
   21127            6 :           break;
   21128            5 :         case E_V16SFmode:
   21129            5 :           cmp_mode = V16SImode;
   21130            5 :           break;
   21131            1 :         case E_V2HFmode:
   21132            1 :         case E_V2BFmode:
   21133            1 :           cmp_mode = V2HImode;
   21134            1 :           break;
   21135            1 :         case E_V4HFmode:
   21136            1 :         case E_V4BFmode:
   21137            1 :           cmp_mode = V4HImode;
   21138            1 :           break;
   21139              :         case E_V8HFmode:
   21140            2 :           cmp_mode = V8HImode;
   21141              :           break;
   21142              :         case E_V16HFmode:
   21143            2 :           cmp_mode = V16HImode;
   21144              :           break;
   21145              :         case E_V32HFmode:
   21146            1 :           cmp_mode = V32HImode;
   21147              :           break;
   21148              :         case E_V8BFmode:
   21149            2 :           cmp_mode = V8HImode;
   21150              :           break;
   21151              :         case E_V16BFmode:
   21152            2 :           cmp_mode = V16HImode;
   21153              :           break;
   21154              :         case E_V32BFmode:
   21155            1 :           cmp_mode = V32HImode;
   21156              :           break;
   21157            0 :         default:
   21158            0 :           gcc_unreachable ();
   21159              :         }
   21160              :     }
   21161              : 
   21162         1604 :   for (int i = 0; i != n_elts; i++)
   21163         1478 :     vec[i] = GEN_INT (i);
   21164          126 :   constv = gen_rtx_CONST_VECTOR (cmp_mode, gen_rtvec_v (n_elts, vec));
   21165          126 :   valv = gen_reg_rtx (mode);
   21166          126 :   idxv = gen_reg_rtx (cmp_mode);
   21167          252 :   idx_tmp = convert_to_mode (GET_MODE_INNER (cmp_mode), idx, 1);
   21168              : 
   21169          126 :   ok = ix86_expand_vector_init_duplicate (TARGET_MMX_WITH_SSE,
   21170              :                                           mode, valv, val);
   21171          126 :   gcc_assert (ok);
   21172          126 :   ok = ix86_expand_vector_init_duplicate (TARGET_MMX_WITH_SSE,
   21173              :                                           cmp_mode, idxv, idx_tmp);
   21174          126 :   gcc_assert (ok);
   21175          126 :   vec[0] = target;
   21176          126 :   vec[1] = valv;
   21177          126 :   vec[2] = target;
   21178          126 :   vec[3] = gen_rtx_EQ (mode, idxv, constv);
   21179          126 :   vec[4] = idxv;
   21180          126 :   vec[5] = constv;
   21181          126 :   ok = ix86_expand_int_vcond (vec);
   21182          126 :   gcc_assert (ok);
   21183              : }
   21184              : 
   21185              : void
   21186         6245 : ix86_expand_vector_set (bool mmx_ok, rtx target, rtx val, int elt)
   21187              : {
   21188         6245 :   machine_mode mode = GET_MODE (target);
   21189         6245 :   machine_mode inner_mode = GET_MODE_INNER (mode);
   21190         6245 :   machine_mode half_mode;
   21191         6245 :   bool use_vec_merge = false;
   21192         6245 :   bool blendm_const = false;
   21193         6245 :   rtx tmp;
   21194         6245 :   static rtx (*gen_extract[8][2]) (rtx, rtx)
   21195              :     = {
   21196              :         { gen_vec_extract_lo_v32qi, gen_vec_extract_hi_v32qi },
   21197              :         { gen_vec_extract_lo_v16hi, gen_vec_extract_hi_v16hi },
   21198              :         { gen_vec_extract_lo_v8si, gen_vec_extract_hi_v8si },
   21199              :         { gen_vec_extract_lo_v4di, gen_vec_extract_hi_v4di },
   21200              :         { gen_vec_extract_lo_v8sf, gen_vec_extract_hi_v8sf },
   21201              :         { gen_vec_extract_lo_v4df, gen_vec_extract_hi_v4df },
   21202              :         { gen_vec_extract_lo_v16hf, gen_vec_extract_hi_v16hf },
   21203              :         { gen_vec_extract_lo_v16bf, gen_vec_extract_hi_v16bf }
   21204              :       };
   21205         6245 :   static rtx (*gen_insert[8][2]) (rtx, rtx, rtx)
   21206              :     = {
   21207              :         { gen_vec_set_lo_v32qi, gen_vec_set_hi_v32qi },
   21208              :         { gen_vec_set_lo_v16hi, gen_vec_set_hi_v16hi },
   21209              :         { gen_vec_set_lo_v8si, gen_vec_set_hi_v8si },
   21210              :         { gen_vec_set_lo_v4di, gen_vec_set_hi_v4di },
   21211              :         { gen_vec_set_lo_v8sf, gen_vec_set_hi_v8sf },
   21212              :         { gen_vec_set_lo_v4df, gen_vec_set_hi_v4df },
   21213              :         { gen_vec_set_lo_v16hf, gen_vec_set_hi_v16hf },
   21214              :         { gen_vec_set_lo_v16bf, gen_vec_set_hi_v16bf },
   21215              :       };
   21216         6245 :   int i, j, n;
   21217         6245 :   machine_mode mmode = VOIDmode;
   21218         6245 :   rtx (*gen_blendm) (rtx, rtx, rtx, rtx);
   21219              : 
   21220        12490 :   if (!IN_RANGE (elt, 0, GET_MODE_NUNITS (mode)))
   21221              :     {
   21222            0 :       emit_move_insn (target, target);
   21223            0 :       return;
   21224              :     }
   21225         6245 :   if (TARGET_SSE4_1 && mode == V4SImode && val == const0_rtx)
   21226              :     {
   21227           27 :       emit_insn (gen_sse4_1_insertps_v4si_zero (target, target,
   21228              :                                                 CONST0_RTX (V4SImode),
   21229           27 :                                                 GEN_INT ((1 << elt) ^ 15)));
   21230           27 :       return;
   21231              :     }
   21232         6218 :   if (TARGET_SSE4_1 && mode == V4SFmode && val == CONST0_RTX (SFmode))
   21233              :     {
   21234           25 :       emit_insn (gen_sse4_1_insertps_v4sf_zero (target, target,
   21235              :                                                 CONST0_RTX (V4SFmode),
   21236           25 :                                                 GEN_INT ((1 << elt) ^ 15)));
   21237           25 :       return;
   21238              :     }
   21239              : 
   21240        12386 :   val = force_reg (GET_MODE_INNER (mode), val);
   21241              : 
   21242         6193 :   switch (mode)
   21243              :     {
   21244          189 :     case E_V2SImode:
   21245          189 :       use_vec_merge = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
   21246              :       if (use_vec_merge)
   21247              :         break;
   21248              :       /* FALLTHRU */
   21249              : 
   21250          174 :     case E_V2SFmode:
   21251          174 :       if (mmx_ok)
   21252              :         {
   21253          344 :           tmp = gen_reg_rtx (GET_MODE_INNER (mode));
   21254          172 :           ix86_expand_vector_extract (true, tmp, target, 1 - elt);
   21255          172 :           if (elt == 0)
   21256            0 :             tmp = gen_rtx_VEC_CONCAT (mode, val, tmp);
   21257              :           else
   21258          172 :             tmp = gen_rtx_VEC_CONCAT (mode, tmp, val);
   21259          172 :           emit_insn (gen_rtx_SET (target, tmp));
   21260          172 :           return;
   21261              :         }
   21262              :       break;
   21263              : 
   21264          160 :     case E_V2DImode:
   21265          160 :       use_vec_merge = TARGET_SSE4_1 && TARGET_64BIT;
   21266           82 :       if (use_vec_merge)
   21267              :         break;
   21268              : 
   21269          164 :       tmp = gen_reg_rtx (GET_MODE_INNER (mode));
   21270           82 :       ix86_expand_vector_extract (false, tmp, target, 1 - elt);
   21271           82 :       if (elt == 0)
   21272           65 :         tmp = gen_rtx_VEC_CONCAT (mode, val, tmp);
   21273              :       else
   21274           17 :         tmp = gen_rtx_VEC_CONCAT (mode, tmp, val);
   21275           82 :       emit_insn (gen_rtx_SET (target, tmp));
   21276           82 :       return;
   21277              : 
   21278          127 :     case E_V2DFmode:
   21279              :       /* NB: For ELT == 0, use standard scalar operation patterns which
   21280              :          preserve the rest of the vector for combiner:
   21281              : 
   21282              :          (vec_merge:V2DF
   21283              :            (vec_duplicate:V2DF (reg:DF))
   21284              :            (reg:V2DF)
   21285              :            (const_int 1))
   21286              :        */
   21287          127 :       if (elt == 0)
   21288           64 :         goto do_vec_merge;
   21289              : 
   21290           63 :       {
   21291           63 :         rtx op0, op1;
   21292              : 
   21293              :         /* For the two element vectors, we implement a VEC_CONCAT with
   21294              :            the extraction of the other element.  */
   21295              : 
   21296           63 :         tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (1 - elt)));
   21297           63 :         tmp = gen_rtx_VEC_SELECT (inner_mode, target, tmp);
   21298              : 
   21299           63 :         if (elt == 0)
   21300              :           op0 = val, op1 = tmp;
   21301              :         else
   21302           63 :           op0 = tmp, op1 = val;
   21303              : 
   21304           63 :         tmp = gen_rtx_VEC_CONCAT (mode, op0, op1);
   21305           63 :         emit_insn (gen_rtx_SET (target, tmp));
   21306              :       }
   21307           63 :       return;
   21308              : 
   21309           80 :     case E_V4SFmode:
   21310           80 :       use_vec_merge = TARGET_SSE4_1;
   21311           80 :       if (use_vec_merge)
   21312              :         break;
   21313              : 
   21314           66 :       switch (elt)
   21315              :         {
   21316              :         case 0:
   21317              :           use_vec_merge = true;
   21318              :           break;
   21319              : 
   21320            2 :         case 1:
   21321              :           /* tmp = target = A B C D */
   21322            2 :           tmp = copy_to_reg (target);
   21323              :           /* target = A A B B */
   21324            2 :           emit_insn (gen_vec_interleave_lowv4sf (target, target, target));
   21325              :           /* target = X A B B */
   21326            2 :           ix86_expand_vector_set (false, target, val, 0);
   21327              :           /* target = A X C D  */
   21328            2 :           emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
   21329              :                                           const1_rtx, const0_rtx,
   21330              :                                           GEN_INT (2+4), GEN_INT (3+4)));
   21331            2 :           return;
   21332              : 
   21333            1 :         case 2:
   21334              :           /* tmp = target = A B C D */
   21335            1 :           tmp = copy_to_reg (target);
   21336              :           /* tmp = X B C D */
   21337            1 :           ix86_expand_vector_set (false, tmp, val, 0);
   21338              :           /* target = A B X D */
   21339            1 :           emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
   21340              :                                           const0_rtx, const1_rtx,
   21341              :                                           GEN_INT (0+4), GEN_INT (3+4)));
   21342            1 :           return;
   21343              : 
   21344            4 :         case 3:
   21345              :           /* tmp = target = A B C D */
   21346            4 :           tmp = copy_to_reg (target);
   21347              :           /* tmp = X B C D */
   21348            4 :           ix86_expand_vector_set (false, tmp, val, 0);
   21349              :           /* target = A B X D */
   21350            4 :           emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
   21351              :                                           const0_rtx, const1_rtx,
   21352              :                                           GEN_INT (2+4), GEN_INT (0+4)));
   21353            4 :           return;
   21354              : 
   21355            0 :         default:
   21356            0 :           gcc_unreachable ();
   21357              :         }
   21358              :       break;
   21359              : 
   21360          402 :     case E_V4SImode:
   21361          402 :       use_vec_merge = TARGET_SSE4_1;
   21362          402 :       if (use_vec_merge)
   21363              :         break;
   21364              : 
   21365              :       /* Element 0 handled by vec_merge below.  */
   21366          283 :       if (elt == 0)
   21367              :         {
   21368              :           use_vec_merge = true;
   21369              :           break;
   21370              :         }
   21371              : 
   21372           92 :       if (TARGET_SSE2)
   21373              :         {
   21374              :           /* With SSE2, use integer shuffles to swap element 0 and ELT,
   21375              :              store into element 0, then shuffle them back.  */
   21376              : 
   21377           92 :           rtx order[4];
   21378              : 
   21379           92 :           order[0] = GEN_INT (elt);
   21380           92 :           order[1] = const1_rtx;
   21381           92 :           order[2] = const2_rtx;
   21382           92 :           order[3] = GEN_INT (3);
   21383           92 :           order[elt] = const0_rtx;
   21384              : 
   21385           92 :           emit_insn (gen_sse2_pshufd_1 (target, target, order[0],
   21386              :                                         order[1], order[2], order[3]));
   21387              : 
   21388           92 :           ix86_expand_vector_set (false, target, val, 0);
   21389              : 
   21390           92 :           emit_insn (gen_sse2_pshufd_1 (target, target, order[0],
   21391              :                                         order[1], order[2], order[3]));
   21392              :         }
   21393              :       else
   21394              :         {
   21395              :           /* For SSE1, we have to reuse the V4SF code.  */
   21396            0 :           rtx t = gen_reg_rtx (V4SFmode);
   21397            0 :           emit_move_insn (t, gen_lowpart (V4SFmode, target));
   21398            0 :           ix86_expand_vector_set (false, t, gen_lowpart (SFmode, val), elt);
   21399            0 :           emit_move_insn (target, gen_lowpart (mode, t));
   21400              :         }
   21401              :       return;
   21402              : 
   21403         2018 :     case E_V8HImode:
   21404         2018 :     case E_V8HFmode:
   21405         2018 :     case E_V8BFmode:
   21406         2018 :     case E_V2HImode:
   21407         2018 :     case E_V2HFmode:
   21408         2018 :     case E_V2BFmode:
   21409         2018 :       use_vec_merge = TARGET_SSE2;
   21410         2018 :       break;
   21411           51 :     case E_V4HImode:
   21412           51 :     case E_V4HFmode:
   21413           51 :     case E_V4BFmode:
   21414           51 :       use_vec_merge = mmx_ok && (TARGET_SSE || TARGET_3DNOW_A);
   21415              :       break;
   21416              : 
   21417         3019 :     case E_V16QImode:
   21418         3019 :     case E_V4QImode:
   21419         3019 :       use_vec_merge = TARGET_SSE4_1;
   21420         3019 :       break;
   21421              : 
   21422            1 :     case E_V8QImode:
   21423            1 :       use_vec_merge = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
   21424              :       break;
   21425              : 
   21426            3 :     case E_V32QImode:
   21427            3 :       half_mode = V16QImode;
   21428            3 :       j = 0;
   21429            3 :       n = 16;
   21430            3 :       goto half;
   21431              : 
   21432           17 :     case E_V16HFmode:
   21433           17 :     case E_V16BFmode:
   21434              :       /* For ELT == 0, vec_setv8hf_0 can save 1 vpbroadcastw.  */
   21435           17 :       if (TARGET_AVX2 && elt != 0)
   21436              :         {
   21437           12 :           mmode = SImode;
   21438           12 :           gen_blendm = ((mode == E_V16HFmode) ? gen_avx2_pblendph_1
   21439              :                                                 : gen_avx2_pblendbf_1);
   21440              :           blendm_const = true;
   21441              :           break;
   21442              :         }
   21443              :       else
   21444              :         {
   21445            5 :           half_mode = ((mode == E_V16HFmode) ? V8HFmode : V8BFmode);
   21446            3 :           j = ((mode == E_V16HFmode) ? 6 : 7);
   21447            5 :           n = 8;
   21448            5 :           goto half;
   21449              :         }
   21450              : 
   21451            5 :     case E_V16HImode:
   21452            5 :       half_mode = V8HImode;
   21453            5 :       j = 1;
   21454            5 :       n = 8;
   21455            5 :       goto half;
   21456              : 
   21457           15 :     case E_V8SImode:
   21458           15 :       half_mode = V4SImode;
   21459           15 :       j = 2;
   21460           15 :       n = 4;
   21461           15 :       goto half;
   21462              : 
   21463           15 :     case E_V4DImode:
   21464           15 :       half_mode = V2DImode;
   21465           15 :       j = 3;
   21466           15 :       n = 2;
   21467           15 :       goto half;
   21468              : 
   21469            4 :     case E_V8SFmode:
   21470            4 :       half_mode = V4SFmode;
   21471            4 :       j = 4;
   21472            4 :       n = 4;
   21473            4 :       goto half;
   21474              : 
   21475            6 :     case E_V4DFmode:
   21476            6 :       half_mode = V2DFmode;
   21477            6 :       j = 5;
   21478            6 :       n = 2;
   21479            6 :       goto half;
   21480              : 
   21481           53 : half:
   21482              :       /* Compute offset.  */
   21483           53 :       i = elt / n;
   21484           53 :       elt %= n;
   21485              : 
   21486           53 :       gcc_assert (i <= 1);
   21487              : 
   21488              :       /* Extract the half.  */
   21489           53 :       tmp = gen_reg_rtx (half_mode);
   21490           53 :       emit_insn (gen_extract[j][i] (tmp, target));
   21491              : 
   21492              :       /* Put val in tmp at elt.  */
   21493           53 :       ix86_expand_vector_set (false, tmp, val, elt);
   21494              : 
   21495              :       /* Put it back.  */
   21496           53 :       emit_insn (gen_insert[j][i] (target, target, tmp));
   21497           53 :       return;
   21498              : 
   21499            8 :     case E_V8DFmode:
   21500            8 :       if (TARGET_AVX512F)
   21501              :         {
   21502              :           mmode = QImode;
   21503              :           gen_blendm = gen_avx512f_blendmv8df;
   21504              :         }
   21505              :       break;
   21506              : 
   21507            7 :     case E_V8DImode:
   21508            7 :       if (TARGET_AVX512F)
   21509              :         {
   21510              :           mmode = QImode;
   21511              :           gen_blendm = gen_avx512f_blendmv8di;
   21512              :         }
   21513              :       break;
   21514              : 
   21515            0 :     case E_V16SFmode:
   21516            0 :       if (TARGET_AVX512F)
   21517              :         {
   21518              :           mmode = HImode;
   21519              :           gen_blendm = gen_avx512f_blendmv16sf;
   21520              :         }
   21521              :       break;
   21522              : 
   21523            0 :     case E_V16SImode:
   21524            0 :       if (TARGET_AVX512F)
   21525              :         {
   21526              :           mmode = HImode;
   21527              :           gen_blendm = gen_avx512f_blendmv16si;
   21528              :         }
   21529              :       break;
   21530              : 
   21531           12 :     case E_V32HFmode:
   21532           12 :       if (TARGET_AVX512BW)
   21533              :         {
   21534              :           mmode = SImode;
   21535              :           gen_blendm = gen_avx512bw_blendmv32hf;
   21536              :         }
   21537              :       break;
   21538           12 :     case E_V32BFmode:
   21539           12 :       if (TARGET_AVX512BW)
   21540              :         {
   21541              :           mmode = SImode;
   21542              :           gen_blendm = gen_avx512bw_blendmv32bf;
   21543              :         }
   21544              :       break;
   21545           11 :     case E_V32HImode:
   21546           11 :       if (TARGET_AVX512BW)
   21547              :         {
   21548              :           mmode = SImode;
   21549              :           gen_blendm = gen_avx512bw_blendmv32hi;
   21550              :         }
   21551            7 :       else if (TARGET_AVX512F)
   21552              :         {
   21553            7 :           half_mode = E_V8HImode;
   21554            7 :           n = 8;
   21555            7 :           goto quarter;
   21556              :         }
   21557              :       break;
   21558              : 
   21559           12 :     case E_V64QImode:
   21560           12 :       if (TARGET_AVX512BW)
   21561              :         {
   21562              :           mmode = DImode;
   21563              :           gen_blendm = gen_avx512bw_blendmv64qi;
   21564              :         }
   21565            6 :       else if (TARGET_AVX512F)
   21566              :         {
   21567            6 :           half_mode = E_V16QImode;
   21568            6 :           n = 16;
   21569            6 :           goto quarter;
   21570              :         }
   21571              :       break;
   21572              : 
   21573           13 : quarter:
   21574              :       /* Compute offset.  */
   21575           13 :       i = elt / n;
   21576           13 :       elt %= n;
   21577              : 
   21578           13 :       gcc_assert (i <= 3);
   21579              : 
   21580           13 :       {
   21581              :         /* Extract the quarter.  */
   21582           13 :         tmp = gen_reg_rtx (V4SImode);
   21583           13 :         rtx tmp2 = gen_lowpart (V16SImode, target);
   21584           13 :         rtx mask = gen_reg_rtx (QImode);
   21585              : 
   21586           13 :         emit_move_insn (mask, constm1_rtx);
   21587           13 :         emit_insn (gen_avx512f_vextracti32x4_mask (tmp, tmp2, GEN_INT (i),
   21588              :                                                    tmp, mask));
   21589              : 
   21590           13 :         tmp2 = gen_reg_rtx (half_mode);
   21591           13 :         emit_move_insn (tmp2, gen_lowpart (half_mode, tmp));
   21592           13 :         tmp = tmp2;
   21593              : 
   21594              :         /* Put val in tmp at elt.  */
   21595           13 :         ix86_expand_vector_set (false, tmp, val, elt);
   21596              : 
   21597              :         /* Put it back.  */
   21598           13 :         tmp2 = gen_reg_rtx (V16SImode);
   21599           13 :         rtx tmp3 = gen_lowpart (V16SImode, target);
   21600           13 :         mask = gen_reg_rtx (HImode);
   21601           13 :         emit_move_insn (mask, constm1_rtx);
   21602           13 :         tmp = gen_lowpart (V4SImode, tmp);
   21603           13 :         emit_insn (gen_avx512f_vinserti32x4_mask (tmp2, tmp3, tmp, GEN_INT (i),
   21604              :                                                   tmp3, mask));
   21605           13 :         emit_move_insn (target, gen_lowpart (mode, tmp2));
   21606              :       }
   21607           13 :       return;
   21608              : 
   21609              :     default:
   21610              :       break;
   21611              :     }
   21612              : 
   21613         5043 :   if (mmode != VOIDmode)
   21614              :     {
   21615           55 :       tmp = gen_reg_rtx (mode);
   21616           55 :       emit_insn (gen_rtx_SET (tmp, gen_rtx_VEC_DUPLICATE (mode, val)));
   21617           55 :       rtx merge_mask = gen_int_mode (HOST_WIDE_INT_1U << elt, mmode);
   21618              :       /* The avx512*_blendm<mode> expanders have different operand order
   21619              :          from VEC_MERGE.  In VEC_MERGE, the first input operand is used for
   21620              :          elements where the mask is set and second input operand otherwise,
   21621              :          in {sse,avx}*_*blend* the first input operand is used for elements
   21622              :          where the mask is clear and second input operand otherwise.  */
   21623           55 :       if (!blendm_const)
   21624           43 :         merge_mask = force_reg (mmode, merge_mask);
   21625           55 :       emit_insn (gen_blendm (target, target, tmp, merge_mask));
   21626              :     }
   21627         5592 :   else if (use_vec_merge)
   21628              :     {
   21629         5580 : do_vec_merge:
   21630         5644 :       if (!nonimmediate_operand (val, inner_mode))
   21631            0 :         val = force_reg (inner_mode, val);
   21632         5644 :       tmp = gen_rtx_VEC_DUPLICATE (mode, val);
   21633         5644 :       tmp = gen_rtx_VEC_MERGE (mode, tmp, target,
   21634              :                                GEN_INT (HOST_WIDE_INT_1U << elt));
   21635         5644 :       emit_insn (gen_rtx_SET (target, tmp));
   21636              :     }
   21637              :   else
   21638              :     {
   21639           24 :       rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode));
   21640              : 
   21641           12 :       emit_move_insn (mem, target);
   21642              : 
   21643           24 :       tmp = adjust_address (mem, inner_mode, elt * GET_MODE_SIZE (inner_mode));
   21644           12 :       emit_move_insn (tmp, val);
   21645              : 
   21646           12 :       emit_move_insn (target, mem);
   21647              :     }
   21648              : }
   21649              : 
   21650              : void
   21651        83846 : ix86_expand_vector_extract (bool mmx_ok, rtx target, rtx vec, int elt)
   21652              : {
   21653        89321 :   machine_mode mode = GET_MODE (vec);
   21654        89321 :   machine_mode inner_mode = GET_MODE_INNER (mode);
   21655        89321 :   bool use_vec_extr = false;
   21656        89321 :   rtx tmp;
   21657              : 
   21658        89321 :   switch (mode)
   21659              :     {
   21660         5530 :     case E_V2SImode:
   21661         5530 :       use_vec_extr = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
   21662              :       if (use_vec_extr)
   21663              :         break;
   21664              :       /* FALLTHRU */
   21665              : 
   21666         8335 :     case E_V2SFmode:
   21667         8335 :       if (!mmx_ok)
   21668              :         break;
   21669              :       /* FALLTHRU */
   21670              : 
   21671              :     case E_V2DFmode:
   21672              :     case E_V2DImode:
   21673              :     case E_V2TImode:
   21674              :     case E_V4TImode:
   21675              :       use_vec_extr = true;
   21676              :       break;
   21677              : 
   21678         7693 :     case E_V4SFmode:
   21679         7693 :       use_vec_extr = TARGET_SSE4_1;
   21680         7693 :       if (use_vec_extr)
   21681              :         break;
   21682              : 
   21683         3968 :       switch (elt)
   21684              :         {
   21685              :         case 0:
   21686              :           tmp = vec;
   21687              :           break;
   21688              : 
   21689         1730 :         case 1:
   21690         1730 :         case 3:
   21691         1730 :           tmp = gen_reg_rtx (mode);
   21692         1730 :           emit_insn (gen_sse_shufps_v4sf (tmp, vec, vec,
   21693              :                                        GEN_INT (elt), GEN_INT (elt),
   21694         1730 :                                        GEN_INT (elt+4), GEN_INT (elt+4)));
   21695         1730 :           break;
   21696              : 
   21697          856 :         case 2:
   21698          856 :           tmp = gen_reg_rtx (mode);
   21699          856 :           emit_insn (gen_vec_interleave_highv4sf (tmp, vec, vec));
   21700          856 :           break;
   21701              : 
   21702            0 :         default:
   21703            0 :           gcc_unreachable ();
   21704              :         }
   21705              :       vec = tmp;
   21706              :       use_vec_extr = true;
   21707              :       elt = 0;
   21708              :       break;
   21709              : 
   21710        25612 :     case E_V4SImode:
   21711        25612 :       use_vec_extr = TARGET_SSE4_1;
   21712        25612 :       if (use_vec_extr)
   21713              :         break;
   21714              : 
   21715        19693 :       if (TARGET_SSE2)
   21716              :         {
   21717        19689 :           switch (elt)
   21718              :             {
   21719              :             case 0:
   21720              :               tmp = vec;
   21721              :               break;
   21722              : 
   21723         6610 :             case 1:
   21724         6610 :             case 3:
   21725         6610 :               tmp = gen_reg_rtx (mode);
   21726         6610 :               emit_insn (gen_sse2_pshufd_1 (tmp, vec,
   21727              :                                             GEN_INT (elt), GEN_INT (elt),
   21728              :                                             GEN_INT (elt), GEN_INT (elt)));
   21729         6610 :               break;
   21730              : 
   21731         3239 :             case 2:
   21732         3239 :               tmp = gen_reg_rtx (mode);
   21733         3239 :               emit_insn (gen_vec_interleave_highv4si (tmp, vec, vec));
   21734         3239 :               break;
   21735              : 
   21736            0 :             default:
   21737            0 :               gcc_unreachable ();
   21738              :             }
   21739              :           vec = tmp;
   21740              :           use_vec_extr = true;
   21741              :           elt = 0;
   21742              :         }
   21743              :       else
   21744              :         {
   21745              :           /* For SSE1, we have to reuse the V4SF code.  */
   21746            4 :           ix86_expand_vector_extract (false, gen_lowpart (SFmode, target),
   21747            4 :                                       gen_lowpart (V4SFmode, vec), elt);
   21748            4 :           return;
   21749              :         }
   21750              :       break;
   21751              : 
   21752         6993 :     case E_V8HImode:
   21753         6993 :     case E_V8HFmode:
   21754         6993 :     case E_V8BFmode:
   21755         6993 :     case E_V2HImode:
   21756         6993 :     case E_V2HFmode:
   21757         6993 :     case E_V2BFmode:
   21758         6993 :       use_vec_extr = TARGET_SSE2;
   21759         6993 :       break;
   21760         1138 :     case E_V4HImode:
   21761         1138 :     case E_V4HFmode:
   21762         1138 :     case E_V4BFmode:
   21763         1138 :       use_vec_extr = mmx_ok && (TARGET_SSE || TARGET_3DNOW_A);
   21764              :       break;
   21765              : 
   21766         8439 :     case E_V16QImode:
   21767         8439 :       use_vec_extr = TARGET_SSE4_1;
   21768         8439 :       if (!use_vec_extr
   21769         6826 :           && TARGET_SSE2
   21770         6826 :           && elt == 0
   21771        12367 :           && (optimize_insn_for_size_p () || TARGET_INTER_UNIT_MOVES_FROM_VEC))
   21772              :         {
   21773         3927 :           tmp = gen_reg_rtx (SImode);
   21774         3927 :           ix86_expand_vector_extract (false, tmp, gen_lowpart (V4SImode, vec),
   21775              :                                       0);
   21776         3927 :           emit_insn (gen_rtx_SET (target, gen_lowpart (QImode, tmp)));
   21777         3927 :           return;
   21778              :         }
   21779              :       break;
   21780           79 :     case E_V4QImode:
   21781           79 :       use_vec_extr = TARGET_SSE4_1;
   21782           79 :       break;
   21783              : 
   21784          221 :     case E_V8SFmode:
   21785          221 :       if (TARGET_AVX)
   21786              :         {
   21787          221 :           tmp = gen_reg_rtx (V4SFmode);
   21788          221 :           if (elt < 4)
   21789          106 :             emit_insn (gen_vec_extract_lo_v8sf (tmp, vec));
   21790              :           else
   21791          115 :             emit_insn (gen_vec_extract_hi_v8sf (tmp, vec));
   21792          221 :           ix86_expand_vector_extract (false, target, tmp, elt & 3);
   21793          221 :           return;
   21794              :         }
   21795              :       break;
   21796              : 
   21797          363 :     case E_V4DFmode:
   21798          363 :       if (TARGET_AVX)
   21799              :         {
   21800          363 :           tmp = gen_reg_rtx (V2DFmode);
   21801          363 :           if (elt < 2)
   21802          194 :             emit_insn (gen_vec_extract_lo_v4df (tmp, vec));
   21803              :           else
   21804          169 :             emit_insn (gen_vec_extract_hi_v4df (tmp, vec));
   21805          363 :           ix86_expand_vector_extract (false, target, tmp, elt & 1);
   21806          363 :           return;
   21807              :         }
   21808              :       break;
   21809              : 
   21810          253 :     case E_V32QImode:
   21811          253 :       if (TARGET_AVX)
   21812              :         {
   21813          253 :           tmp = gen_reg_rtx (V16QImode);
   21814          253 :           if (elt < 16)
   21815          130 :             emit_insn (gen_vec_extract_lo_v32qi (tmp, vec));
   21816              :           else
   21817          123 :             emit_insn (gen_vec_extract_hi_v32qi (tmp, vec));
   21818          253 :           ix86_expand_vector_extract (false, target, tmp, elt & 15);
   21819          253 :           return;
   21820              :         }
   21821              :       break;
   21822              : 
   21823          616 :     case E_V16HImode:
   21824          616 :       if (TARGET_AVX)
   21825              :         {
   21826          616 :           tmp = gen_reg_rtx (V8HImode);
   21827          616 :           if (elt < 8)
   21828          304 :             emit_insn (gen_vec_extract_lo_v16hi (tmp, vec));
   21829              :           else
   21830          312 :             emit_insn (gen_vec_extract_hi_v16hi (tmp, vec));
   21831          616 :           ix86_expand_vector_extract (false, target, tmp, elt & 7);
   21832          616 :           return;
   21833              :         }
   21834              :       break;
   21835              : 
   21836         1123 :     case E_V8SImode:
   21837         1123 :       if (TARGET_AVX)
   21838              :         {
   21839         1123 :           tmp = gen_reg_rtx (V4SImode);
   21840         1123 :           if (elt < 4)
   21841          540 :             emit_insn (gen_vec_extract_lo_v8si (tmp, vec));
   21842              :           else
   21843          583 :             emit_insn (gen_vec_extract_hi_v8si (tmp, vec));
   21844         1123 :           ix86_expand_vector_extract (false, target, tmp, elt & 3);
   21845         1123 :           return;
   21846              :         }
   21847              :       break;
   21848              : 
   21849         1349 :     case E_V4DImode:
   21850         1349 :       if (TARGET_AVX)
   21851              :         {
   21852         1349 :           tmp = gen_reg_rtx (V2DImode);
   21853         1349 :           if (elt < 2)
   21854          711 :             emit_insn (gen_vec_extract_lo_v4di (tmp, vec));
   21855              :           else
   21856          638 :             emit_insn (gen_vec_extract_hi_v4di (tmp, vec));
   21857         1349 :           ix86_expand_vector_extract (false, target, tmp, elt & 1);
   21858         1349 :           return;
   21859              :         }
   21860              :       break;
   21861              : 
   21862            8 :     case E_V32HImode:
   21863            8 :       if (TARGET_AVX512BW)
   21864              :         {
   21865            8 :           tmp = gen_reg_rtx (V16HImode);
   21866            8 :           if (elt < 16)
   21867            3 :             emit_insn (gen_vec_extract_lo_v32hi (tmp, vec));
   21868              :           else
   21869            5 :             emit_insn (gen_vec_extract_hi_v32hi (tmp, vec));
   21870            8 :           ix86_expand_vector_extract (false, target, tmp, elt & 15);
   21871            8 :           return;
   21872              :         }
   21873              :       break;
   21874              : 
   21875           10 :     case E_V64QImode:
   21876           10 :       if (TARGET_AVX512BW)
   21877              :         {
   21878           10 :           tmp = gen_reg_rtx (V32QImode);
   21879           10 :           if (elt < 32)
   21880            5 :             emit_insn (gen_vec_extract_lo_v64qi (tmp, vec));
   21881              :           else
   21882            5 :             emit_insn (gen_vec_extract_hi_v64qi (tmp, vec));
   21883           10 :           ix86_expand_vector_extract (false, target, tmp, elt & 31);
   21884           10 :           return;
   21885              :         }
   21886              :       break;
   21887              : 
   21888           25 :     case E_V16SFmode:
   21889           25 :       tmp = gen_reg_rtx (V8SFmode);
   21890           25 :       if (elt < 8)
   21891           15 :         emit_insn (gen_vec_extract_lo_v16sf (tmp, vec));
   21892              :       else
   21893           10 :         emit_insn (gen_vec_extract_hi_v16sf (tmp, vec));
   21894           25 :       ix86_expand_vector_extract (false, target, tmp, elt & 7);
   21895           25 :       return;
   21896              : 
   21897          106 :     case E_V8DFmode:
   21898          106 :       tmp = gen_reg_rtx (V4DFmode);
   21899          106 :       if (elt < 4)
   21900           66 :         emit_insn (gen_vec_extract_lo_v8df (tmp, vec));
   21901              :       else
   21902           40 :         emit_insn (gen_vec_extract_hi_v8df (tmp, vec));
   21903          106 :       ix86_expand_vector_extract (false, target, tmp, elt & 3);
   21904          106 :       return;
   21905              : 
   21906          239 :     case E_V16SImode:
   21907          239 :       tmp = gen_reg_rtx (V8SImode);
   21908          239 :       if (elt < 8)
   21909          128 :         emit_insn (gen_vec_extract_lo_v16si (tmp, vec));
   21910              :       else
   21911          111 :         emit_insn (gen_vec_extract_hi_v16si (tmp, vec));
   21912          239 :       ix86_expand_vector_extract (false, target, tmp, elt & 7);
   21913          239 :       return;
   21914              : 
   21915          671 :     case E_V8DImode:
   21916          671 :       tmp = gen_reg_rtx (V4DImode);
   21917          671 :       if (elt < 4)
   21918          384 :         emit_insn (gen_vec_extract_lo_v8di (tmp, vec));
   21919              :       else
   21920          287 :         emit_insn (gen_vec_extract_hi_v8di (tmp, vec));
   21921          671 :       ix86_expand_vector_extract (false, target, tmp, elt & 3);
   21922          671 :       return;
   21923              : 
   21924           45 :     case E_V32HFmode:
   21925           45 :     case E_V32BFmode:
   21926           45 :       if (TARGET_AVX512BW)
   21927              :         {
   21928           45 :           tmp = (mode == E_V32HFmode
   21929           45 :                  ? gen_reg_rtx (V16HFmode)
   21930            7 :                  : gen_reg_rtx (V16BFmode));
   21931           45 :           if (elt < 16)
   21932           31 :             emit_insn (gen_vec_extract_lo (mode, tmp, vec));
   21933              :           else
   21934           14 :             emit_insn (gen_vec_extract_hi (mode, tmp, vec));
   21935           45 :           ix86_expand_vector_extract (false, target, tmp, elt & 15);
   21936           45 :           return;
   21937              :         }
   21938              :       break;
   21939              : 
   21940          442 :     case E_V16HFmode:
   21941          442 :     case E_V16BFmode:
   21942          442 :       if (TARGET_AVX)
   21943              :         {
   21944          442 :           tmp = (mode == E_V16HFmode
   21945          442 :                  ? gen_reg_rtx (V8HFmode)
   21946          339 :                  : gen_reg_rtx (V8BFmode));
   21947          442 :           if (elt < 8)
   21948          233 :             emit_insn (gen_vec_extract_lo (mode, tmp, vec));
   21949              :           else
   21950          209 :             emit_insn (gen_vec_extract_hi (mode, tmp, vec));
   21951          442 :           ix86_expand_vector_extract (false, target, tmp, elt & 7);
   21952          442 :           return;
   21953              :         }
   21954              :       break;
   21955              : 
   21956         1347 :     case E_V8QImode:
   21957         1347 :       use_vec_extr = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
   21958              :       /* ??? Could extract the appropriate HImode element and shift.  */
   21959              :       break;
   21960              : 
   21961              :     default:
   21962              :       break;
   21963              :     }
   21964              : 
   21965        35939 :   if (use_vec_extr)
   21966              :     {
   21967        75039 :       tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (elt)));
   21968        75039 :       tmp = gen_rtx_VEC_SELECT (inner_mode, vec, tmp);
   21969              : 
   21970              :       /* Let the rtl optimizers know about the zero extension performed.  */
   21971        75039 :       if (inner_mode == QImode || inner_mode == HImode)
   21972              :         {
   21973         9947 :           rtx reg = gen_reg_rtx (SImode);
   21974         9947 :           tmp = gen_rtx_ZERO_EXTEND (SImode, tmp);
   21975         9947 :           emit_move_insn (reg, tmp);
   21976         9947 :           tmp = gen_lowpart (inner_mode, reg);
   21977         9947 :           SUBREG_PROMOTED_VAR_P (tmp) = 1;
   21978         9947 :           SUBREG_PROMOTED_SET (tmp, 1);
   21979              :         }
   21980              : 
   21981        75039 :       emit_move_insn (target, tmp);
   21982              :     }
   21983              :   else
   21984              :     {
   21985         9760 :       rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode));
   21986              : 
   21987         4880 :       emit_move_insn (mem, vec);
   21988              : 
   21989         9760 :       tmp = adjust_address (mem, inner_mode, elt*GET_MODE_SIZE (inner_mode));
   21990         4880 :       emit_move_insn (target, tmp);
   21991              :     }
   21992              : }
   21993              : 
   21994              : /* Generate code to copy vector bits i / 2 ... i - 1 from vector SRC
   21995              :    to bits 0 ... i / 2 - 1 of vector DEST, which has the same mode.
   21996              :    The upper bits of DEST are undefined, though they shouldn't cause
   21997              :    exceptions (some bits from src or all zeros are ok).  */
   21998              : 
   21999              : static void
   22000        39216 : emit_reduc_half (rtx dest, rtx src, int i)
   22001              : {
   22002        39216 :   rtx tem, d = dest;
   22003        39216 :   switch (GET_MODE (src))
   22004              :     {
   22005         6214 :     case E_V4SFmode:
   22006         6214 :       if (i == 128)
   22007         3107 :         tem = gen_sse_movhlps (dest, src, src);
   22008              :       else
   22009         3107 :         tem = gen_sse_shufps_v4sf (dest, src, src, const1_rtx, const1_rtx,
   22010              :                                    GEN_INT (1 + 4), GEN_INT (1 + 4));
   22011              :       break;
   22012         3771 :     case E_V2DFmode:
   22013         3771 :       tem = gen_vec_interleave_highv2df (dest, src, src);
   22014         3771 :       break;
   22015           78 :     case E_V4QImode:
   22016           78 :       d = gen_reg_rtx (V1SImode);
   22017           78 :       tem = gen_mmx_lshrv1si3 (d, gen_lowpart (V1SImode, src),
   22018           78 :                                GEN_INT (i / 2));
   22019           78 :       break;
   22020          602 :     case E_V8QImode:
   22021          602 :     case E_V4HImode:
   22022          602 :       d = gen_reg_rtx (V1DImode);
   22023          602 :       tem = gen_mmx_lshrv1di3 (d, gen_lowpart (V1DImode, src),
   22024          602 :                                GEN_INT (i / 2));
   22025          602 :       break;
   22026        28551 :     case E_V16QImode:
   22027        28551 :     case E_V8HImode:
   22028        28551 :     case E_V8HFmode:
   22029        28551 :     case E_V4SImode:
   22030        28551 :     case E_V2DImode:
   22031        28551 :       if (TARGET_SSE_REDUCTION_PREFER_PSHUF)
   22032              :         {
   22033           11 :           if (i == 128)
   22034              :             {
   22035            5 :               d = gen_reg_rtx (V4SImode);
   22036           10 :               tem = gen_sse2_pshufd_1 (
   22037            5 :                   d, force_reg (V4SImode, gen_lowpart (V4SImode, src)),
   22038              :                   GEN_INT (2), GEN_INT (3), GEN_INT (2), GEN_INT (3));
   22039            5 :               break;
   22040              :             }
   22041            6 :           else if (i == 64)
   22042              :             {
   22043            5 :               d = gen_reg_rtx (V4SImode);
   22044           10 :               tem = gen_sse2_pshufd_1 (
   22045            5 :                   d, force_reg (V4SImode, gen_lowpart (V4SImode, src)),
   22046              :                   GEN_INT (1), GEN_INT (1), GEN_INT (1), GEN_INT (1));
   22047            5 :               break;
   22048              :             }
   22049            1 :           else if (i == 32)
   22050              :             {
   22051            1 :               d = gen_reg_rtx (V8HImode);
   22052            2 :               tem = gen_sse2_pshuflw_1 (
   22053            1 :                   d, force_reg (V8HImode, gen_lowpart (V8HImode, src)),
   22054              :                   GEN_INT (1), GEN_INT (1), GEN_INT (1), GEN_INT (1));
   22055            1 :               break;
   22056              :             }
   22057              :         }
   22058        28540 :       d = gen_reg_rtx (V1TImode);
   22059        28540 :       tem = gen_sse2_lshrv1ti3 (d, gen_lowpart (V1TImode, src),
   22060        28540 :                                 GEN_INT (i / 2));
   22061        28540 :       break;
   22062            0 :     case E_V8SFmode:
   22063            0 :       if (i == 256)
   22064            0 :         tem = gen_avx_vperm2f128v8sf3 (dest, src, src, const1_rtx);
   22065              :       else
   22066            0 :         tem = gen_avx_shufps256 (dest, src, src,
   22067              :                                  GEN_INT (i == 128 ? 2 + (3 << 2) : 1));
   22068              :       break;
   22069            0 :     case E_V4DFmode:
   22070            0 :       if (i == 256)
   22071            0 :         tem = gen_avx_vperm2f128v4df3 (dest, src, src, const1_rtx);
   22072              :       else
   22073            0 :         tem = gen_avx_shufpd256 (dest, src, src, const1_rtx);
   22074              :       break;
   22075            0 :     case E_V32QImode:
   22076            0 :     case E_V16HImode:
   22077            0 :     case E_V16HFmode:
   22078            0 :     case E_V8SImode:
   22079            0 :     case E_V4DImode:
   22080            0 :       if (i == 256)
   22081              :         {
   22082            0 :           if (GET_MODE (dest) != V4DImode)
   22083            0 :             d = gen_reg_rtx (V4DImode);
   22084            0 :           tem = gen_avx2_permv2ti (d, gen_lowpart (V4DImode, src),
   22085            0 :                                    gen_lowpart (V4DImode, src),
   22086              :                                    const1_rtx);
   22087              :         }
   22088              :       else
   22089              :         {
   22090            0 :           d = gen_reg_rtx (V2TImode);
   22091            0 :           tem = gen_avx2_lshrv2ti3 (d, gen_lowpart (V2TImode, src),
   22092            0 :                                     GEN_INT (i / 2));
   22093              :         }
   22094              :       break;
   22095            0 :     case E_V64QImode:
   22096            0 :     case E_V32HImode:
   22097            0 :     case E_V32HFmode:
   22098            0 :       if (i < 64)
   22099              :         {
   22100            0 :           d = gen_reg_rtx (V4TImode);
   22101            0 :           tem = gen_avx512bw_lshrv4ti3 (d, gen_lowpart (V4TImode, src),
   22102            0 :                                         GEN_INT (i / 2));
   22103            0 :           break;
   22104              :         }
   22105              :       /* FALLTHRU */
   22106            0 :     case E_V16SImode:
   22107            0 :     case E_V16SFmode:
   22108            0 :     case E_V8DImode:
   22109            0 :     case E_V8DFmode:
   22110            0 :       if (i > 128)
   22111            0 :         tem = gen_avx512f_shuf_i32x4_1 (gen_lowpart (V16SImode, dest),
   22112            0 :                                         gen_lowpart (V16SImode, src),
   22113            0 :                                         gen_lowpart (V16SImode, src),
   22114              :                                         GEN_INT (0x4 + (i == 512 ? 4 : 0)),
   22115              :                                         GEN_INT (0x5 + (i == 512 ? 4 : 0)),
   22116              :                                         GEN_INT (0x6 + (i == 512 ? 4 : 0)),
   22117              :                                         GEN_INT (0x7 + (i == 512 ? 4 : 0)),
   22118              :                                         GEN_INT (0xC), GEN_INT (0xD),
   22119              :                                         GEN_INT (0xE), GEN_INT (0xF),
   22120              :                                         GEN_INT (0x10), GEN_INT (0x11),
   22121              :                                         GEN_INT (0x12), GEN_INT (0x13),
   22122              :                                         GEN_INT (0x14), GEN_INT (0x15),
   22123              :                                         GEN_INT (0x16), GEN_INT (0x17));
   22124              :       else
   22125            0 :         tem = gen_avx512f_pshufd_1 (gen_lowpart (V16SImode, dest),
   22126            0 :                                     gen_lowpart (V16SImode, src),
   22127              :                                     GEN_INT (i == 128 ? 0x2 : 0x1),
   22128              :                                     GEN_INT (0x3),
   22129              :                                     GEN_INT (0x3),
   22130              :                                     GEN_INT (0x3),
   22131              :                                     GEN_INT (i == 128 ? 0x6 : 0x5),
   22132              :                                     GEN_INT (0x7),
   22133              :                                     GEN_INT (0x7),
   22134              :                                     GEN_INT (0x7),
   22135              :                                     GEN_INT (i == 128 ? 0xA : 0x9),
   22136              :                                     GEN_INT (0xB),
   22137              :                                     GEN_INT (0xB),
   22138              :                                     GEN_INT (0xB),
   22139              :                                     GEN_INT (i == 128 ? 0xE : 0xD),
   22140              :                                     GEN_INT (0xF),
   22141              :                                     GEN_INT (0xF),
   22142              :                                     GEN_INT (0xF));
   22143              :       break;
   22144            0 :     default:
   22145            0 :       gcc_unreachable ();
   22146              :     }
   22147        39216 :   emit_insn (tem);
   22148        39216 :   if (d != dest)
   22149        29231 :     emit_move_insn (dest, gen_lowpart (GET_MODE (dest), d));
   22150        39216 : }
   22151              : 
   22152              : /* Expand a vector reduction.  FN is the binary pattern to reduce;
   22153              :    DEST is the destination; IN is the input vector.  */
   22154              : 
   22155              : void
   22156        17941 : ix86_expand_reduc (rtx (*fn) (rtx, rtx, rtx), rtx dest, rtx in)
   22157              : {
   22158        17941 :   rtx half, dst, vec = in;
   22159        17941 :   machine_mode mode = GET_MODE (in);
   22160        17941 :   int i;
   22161              : 
   22162              :   /* SSE4 has a special instruction for V8HImode UMIN reduction.  */
   22163        17941 :   if (TARGET_SSE4_1
   22164        10106 :       && mode == V8HImode
   22165          780 :       && fn == gen_uminv8hi3)
   22166              :     {
   22167            4 :       emit_insn (gen_sse4_1_phminposuw (dest, in));
   22168            4 :       return;
   22169              :     }
   22170              : 
   22171              :   /* SSE3 has haddpd, some targets prefer that over movhlpd plus add.  */
   22172        17937 :   if (TARGET_SSE3
   22173        10154 :       && TARGET_V2DF_REDUCTION_PREFER_HADDPD
   22174           10 :       && mode == V2DFmode
   22175           10 :       && fn == gen_addv2df3)
   22176              :     {
   22177           10 :       emit_insn (gen_sse3_haddv2df3 (dest, in, in));
   22178           10 :       return;
   22179              :     }
   22180              : 
   22181        35854 :   for (i = GET_MODE_BITSIZE (mode);
   22182       114286 :        i > GET_MODE_UNIT_BITSIZE (mode);
   22183        39216 :        i >>= 1)
   22184              :     {
   22185        39216 :       half = gen_reg_rtx (mode);
   22186        39216 :       emit_reduc_half (half, vec, i);
   22187        78432 :       if (i == GET_MODE_UNIT_BITSIZE (mode) * 2)
   22188              :         dst = dest;
   22189              :       else
   22190        21289 :         dst = gen_reg_rtx (mode);
   22191        39216 :       emit_insn (fn (dst, half, vec));
   22192        39216 :       vec = dst;
   22193              :     }
   22194              : }
   22195              : 
   22196              : /* Output code to perform a conditional jump to LABEL, if C2 flag in
   22197              :    FP status register is set.  */
   22198              : 
   22199              : void
   22200          285 : ix86_emit_fp_unordered_jump (rtx label)
   22201              : {
   22202          285 :   rtx reg = gen_reg_rtx (HImode);
   22203          285 :   rtx_insn *insn;
   22204          285 :   rtx temp;
   22205              : 
   22206          285 :   emit_insn (gen_x86_fnstsw_1 (reg));
   22207              : 
   22208          285 :   if (TARGET_SAHF && (TARGET_USE_SAHF || optimize_insn_for_size_p ()))
   22209              :     {
   22210           37 :       emit_insn (gen_x86_sahf_1 (reg));
   22211              : 
   22212           37 :       temp = gen_rtx_REG (CCmode, FLAGS_REG);
   22213           37 :       temp = gen_rtx_UNORDERED (VOIDmode, temp, const0_rtx);
   22214              :     }
   22215              :   else
   22216              :     {
   22217          248 :       emit_insn (gen_testqi_ext_1_ccno (reg, GEN_INT (0x04)));
   22218              : 
   22219          248 :       temp = gen_rtx_REG (CCNOmode, FLAGS_REG);
   22220          248 :       temp = gen_rtx_NE (VOIDmode, temp, const0_rtx);
   22221              :     }
   22222              : 
   22223          285 :   temp = gen_rtx_IF_THEN_ELSE (VOIDmode, temp,
   22224              :                               gen_rtx_LABEL_REF (VOIDmode, label),
   22225              :                               pc_rtx);
   22226          285 :   insn = emit_jump_insn (gen_rtx_SET (pc_rtx, temp));
   22227          285 :   predict_jump (REG_BR_PROB_BASE * 10 / 100);
   22228          285 :   JUMP_LABEL (insn) = label;
   22229          285 : }
   22230              : 
   22231              : /* Output code to perform an sinh XFmode calculation.  */
   22232              : 
   22233              : void
   22234            2 : ix86_emit_i387_sinh (rtx op0, rtx op1)
   22235              : {
   22236            2 :   rtx e1 = gen_reg_rtx (XFmode);
   22237            2 :   rtx e2 = gen_reg_rtx (XFmode);
   22238            2 :   rtx scratch = gen_reg_rtx (HImode);
   22239            2 :   rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
   22240            2 :   rtx half = const_double_from_real_value (dconsthalf, XFmode);
   22241            2 :   rtx cst1, tmp;
   22242            2 :   rtx_code_label *jump_label = gen_label_rtx ();
   22243            2 :   rtx_insn *insn;
   22244              : 
   22245              :   /* scratch = fxam (op1) */
   22246            2 :   emit_insn (gen_fxamxf2_i387 (scratch, op1));
   22247              : 
   22248              :   /* e1 = expm1 (|op1|) */
   22249            2 :   emit_insn (gen_absxf2 (e2, op1));
   22250            2 :   emit_insn (gen_expm1xf2 (e1, e2));
   22251              : 
   22252              :   /* e2 = e1 / (e1 + 1.0) + e1 */
   22253            2 :   cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
   22254            2 :   emit_insn (gen_addxf3 (e2, e1, cst1));
   22255            2 :   emit_insn (gen_divxf3 (e2, e1, e2));
   22256            2 :   emit_insn (gen_addxf3 (e2, e2, e1));
   22257              : 
   22258              :   /* flags = signbit (op1) */
   22259            2 :   emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
   22260              : 
   22261              :   /* if (flags) then e2 = -e2 */
   22262            2 :   tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
   22263              :                               gen_rtx_EQ (VOIDmode, flags, const0_rtx),
   22264              :                               gen_rtx_LABEL_REF (VOIDmode, jump_label),
   22265              :                               pc_rtx);
   22266            2 :   insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
   22267            2 :   predict_jump (REG_BR_PROB_BASE * 50 / 100);
   22268            2 :   JUMP_LABEL (insn) = jump_label;
   22269              : 
   22270            2 :   emit_insn (gen_negxf2 (e2, e2));
   22271              : 
   22272            2 :   emit_label (jump_label);
   22273            2 :   LABEL_NUSES (jump_label) = 1;
   22274              : 
   22275              :   /* op0 = 0.5 * e2 */
   22276            2 :   half = force_reg (XFmode, half);
   22277            2 :   emit_insn (gen_mulxf3 (op0, e2, half));
   22278            2 : }
   22279              : 
   22280              : /* Output code to perform an cosh XFmode calculation.  */
   22281              : 
   22282              : void
   22283            3 : ix86_emit_i387_cosh (rtx op0, rtx op1)
   22284              : {
   22285            3 :   rtx e1 = gen_reg_rtx (XFmode);
   22286            3 :   rtx e2 = gen_reg_rtx (XFmode);
   22287            3 :   rtx half = const_double_from_real_value (dconsthalf, XFmode);
   22288            3 :   rtx cst1;
   22289              : 
   22290              :   /* e1 = exp (op1) */
   22291            3 :   emit_insn (gen_expxf2 (e1, op1));
   22292              : 
   22293              :   /* e2 = e1 + 1.0 / e1 */
   22294            3 :   cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
   22295            3 :   emit_insn (gen_divxf3 (e2, cst1, e1));
   22296            3 :   emit_insn (gen_addxf3 (e2, e1, e2));
   22297              : 
   22298              :   /* op0 = 0.5 * e2 */
   22299            3 :   half = force_reg (XFmode, half);
   22300            3 :   emit_insn (gen_mulxf3 (op0, e2, half));
   22301            3 : }
   22302              : 
   22303              : /* Output code to perform an tanh XFmode calculation.  */
   22304              : 
   22305              : void
   22306            1 : ix86_emit_i387_tanh (rtx op0, rtx op1)
   22307              : {
   22308            1 :   rtx e1 = gen_reg_rtx (XFmode);
   22309            1 :   rtx e2 = gen_reg_rtx (XFmode);
   22310            1 :   rtx scratch = gen_reg_rtx (HImode);
   22311            1 :   rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
   22312            1 :   rtx cst2, tmp;
   22313            1 :   rtx_code_label *jump_label = gen_label_rtx ();
   22314            1 :   rtx_insn *insn;
   22315              : 
   22316              :   /* scratch = fxam (op1) */
   22317            1 :   emit_insn (gen_fxamxf2_i387 (scratch, op1));
   22318              : 
   22319              :   /* e1 = expm1 (-|2 * op1|) */
   22320            1 :   emit_insn (gen_addxf3 (e2, op1, op1));
   22321            1 :   emit_insn (gen_absxf2 (e2, e2));
   22322            1 :   emit_insn (gen_negxf2 (e2, e2));
   22323            1 :   emit_insn (gen_expm1xf2 (e1, e2));
   22324              : 
   22325              :   /* e2 = e1 / (e1 + 2.0) */
   22326            1 :   cst2 = force_reg (XFmode, CONST2_RTX (XFmode));
   22327            1 :   emit_insn (gen_addxf3 (e2, e1, cst2));
   22328            1 :   emit_insn (gen_divxf3 (e2, e1, e2));
   22329              : 
   22330              :   /* flags = signbit (op1) */
   22331            1 :   emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
   22332              : 
   22333              :   /* if (!flags) then e2 = -e2 */
   22334            1 :   tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
   22335              :                               gen_rtx_NE (VOIDmode, flags, const0_rtx),
   22336              :                               gen_rtx_LABEL_REF (VOIDmode, jump_label),
   22337              :                               pc_rtx);
   22338            1 :   insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
   22339            1 :   predict_jump (REG_BR_PROB_BASE * 50 / 100);
   22340            1 :   JUMP_LABEL (insn) = jump_label;
   22341              : 
   22342            1 :   emit_insn (gen_negxf2 (e2, e2));
   22343              : 
   22344            1 :   emit_label (jump_label);
   22345            1 :   LABEL_NUSES (jump_label) = 1;
   22346              : 
   22347            1 :   emit_move_insn (op0, e2);
   22348            1 : }
   22349              : 
   22350              : /* Output code to perform an asinh XFmode calculation.  */
   22351              : 
   22352              : void
   22353            0 : ix86_emit_i387_asinh (rtx op0, rtx op1)
   22354              : {
   22355            0 :   rtx e1 = gen_reg_rtx (XFmode);
   22356            0 :   rtx e2 = gen_reg_rtx (XFmode);
   22357            0 :   rtx scratch = gen_reg_rtx (HImode);
   22358            0 :   rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
   22359            0 :   rtx cst1, tmp;
   22360            0 :   rtx_code_label *jump_label = gen_label_rtx ();
   22361            0 :   rtx_insn *insn;
   22362              : 
   22363              :   /* e2 = sqrt (op1^2 + 1.0) + 1.0 */
   22364            0 :   emit_insn (gen_mulxf3 (e1, op1, op1));
   22365            0 :   cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
   22366            0 :   emit_insn (gen_addxf3 (e2, e1, cst1));
   22367            0 :   emit_insn (gen_sqrtxf2 (e2, e2));
   22368            0 :   emit_insn (gen_addxf3 (e2, e2, cst1));
   22369              : 
   22370              :   /* e1 = e1 / e2 */
   22371            0 :   emit_insn (gen_divxf3 (e1, e1, e2));
   22372              : 
   22373              :   /* scratch = fxam (op1) */
   22374            0 :   emit_insn (gen_fxamxf2_i387 (scratch, op1));
   22375              : 
   22376              :   /* e1 = e1 + |op1| */
   22377            0 :   emit_insn (gen_absxf2 (e2, op1));
   22378            0 :   emit_insn (gen_addxf3 (e1, e1, e2));
   22379              : 
   22380              :   /* e2 = log1p (e1) */
   22381            0 :   ix86_emit_i387_log1p (e2, e1);
   22382              : 
   22383              :   /* flags = signbit (op1) */
   22384            0 :   emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
   22385              : 
   22386              :   /* if (flags) then e2 = -e2 */
   22387            0 :   tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
   22388              :                               gen_rtx_EQ (VOIDmode, flags, const0_rtx),
   22389              :                               gen_rtx_LABEL_REF (VOIDmode, jump_label),
   22390              :                               pc_rtx);
   22391            0 :   insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
   22392            0 :   predict_jump (REG_BR_PROB_BASE * 50 / 100);
   22393            0 :   JUMP_LABEL (insn) = jump_label;
   22394              : 
   22395            0 :   emit_insn (gen_negxf2 (e2, e2));
   22396              : 
   22397            0 :   emit_label (jump_label);
   22398            0 :   LABEL_NUSES (jump_label) = 1;
   22399              : 
   22400            0 :   emit_move_insn (op0, e2);
   22401            0 : }
   22402              : 
   22403              : /* Output code to perform an acosh XFmode calculation.  */
   22404              : 
   22405              : void
   22406            0 : ix86_emit_i387_acosh (rtx op0, rtx op1)
   22407              : {
   22408            0 :   rtx e1 = gen_reg_rtx (XFmode);
   22409            0 :   rtx e2 = gen_reg_rtx (XFmode);
   22410            0 :   rtx cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
   22411              : 
   22412              :   /* e2 = sqrt (op1 + 1.0) */
   22413            0 :   emit_insn (gen_addxf3 (e2, op1, cst1));
   22414            0 :   emit_insn (gen_sqrtxf2 (e2, e2));
   22415              : 
   22416              :   /* e1 = sqrt (op1 - 1.0) */
   22417            0 :   emit_insn (gen_subxf3 (e1, op1, cst1));
   22418            0 :   emit_insn (gen_sqrtxf2 (e1, e1));
   22419              : 
   22420              :   /* e1 = e1 * e2 */
   22421            0 :   emit_insn (gen_mulxf3 (e1, e1, e2));
   22422              : 
   22423              :   /* e1 = e1 + op1 */
   22424            0 :   emit_insn (gen_addxf3 (e1, e1, op1));
   22425              : 
   22426              :   /* op0 = log (e1) */
   22427            0 :   emit_insn (gen_logxf2 (op0, e1));
   22428            0 : }
   22429              : 
   22430              : /* Output code to perform an atanh XFmode calculation.  */
   22431              : 
   22432              : void
   22433            4 : ix86_emit_i387_atanh (rtx op0, rtx op1)
   22434              : {
   22435            4 :   rtx e1 = gen_reg_rtx (XFmode);
   22436            4 :   rtx e2 = gen_reg_rtx (XFmode);
   22437            4 :   rtx scratch = gen_reg_rtx (HImode);
   22438            4 :   rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
   22439            4 :   rtx half = const_double_from_real_value (dconsthalf, XFmode);
   22440            4 :   rtx cst1, tmp;
   22441            4 :   rtx_code_label *jump_label = gen_label_rtx ();
   22442            4 :   rtx_insn *insn;
   22443              : 
   22444              :   /* scratch = fxam (op1) */
   22445            4 :   emit_insn (gen_fxamxf2_i387 (scratch, op1));
   22446              : 
   22447              :   /* e2 = |op1| */
   22448            4 :   emit_insn (gen_absxf2 (e2, op1));
   22449              : 
   22450              :   /* e1 = -(e2 + e2) / (e2 + 1.0) */
   22451            4 :   cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
   22452            4 :   emit_insn (gen_addxf3 (e1, e2, cst1));
   22453            4 :   emit_insn (gen_addxf3 (e2, e2, e2));
   22454            4 :   emit_insn (gen_negxf2 (e2, e2));
   22455            4 :   emit_insn (gen_divxf3 (e1, e2, e1));
   22456              : 
   22457              :   /* e2 = log1p (e1) */
   22458            4 :   ix86_emit_i387_log1p (e2, e1);
   22459              : 
   22460              :   /* flags = signbit (op1) */
   22461            4 :   emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
   22462              : 
   22463              :   /* if (!flags) then e2 = -e2 */
   22464            4 :   tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
   22465              :                               gen_rtx_NE (VOIDmode, flags, const0_rtx),
   22466              :                               gen_rtx_LABEL_REF (VOIDmode, jump_label),
   22467              :                               pc_rtx);
   22468            4 :   insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
   22469            4 :   predict_jump (REG_BR_PROB_BASE * 50 / 100);
   22470            4 :   JUMP_LABEL (insn) = jump_label;
   22471              : 
   22472            4 :   emit_insn (gen_negxf2 (e2, e2));
   22473              : 
   22474            4 :   emit_label (jump_label);
   22475            4 :   LABEL_NUSES (jump_label) = 1;
   22476              : 
   22477              :   /* op0 = 0.5 * e2 */
   22478            4 :   half = force_reg (XFmode, half);
   22479            4 :   emit_insn (gen_mulxf3 (op0, e2, half));
   22480            4 : }
   22481              : 
   22482              : /* Output code to perform a log1p XFmode calculation.  */
   22483              : 
   22484              : void
   22485            5 : ix86_emit_i387_log1p (rtx op0, rtx op1)
   22486              : {
   22487            5 :   rtx_code_label *label1 = gen_label_rtx ();
   22488            5 :   rtx_code_label *label2 = gen_label_rtx ();
   22489              : 
   22490            5 :   rtx tmp = gen_reg_rtx (XFmode);
   22491            5 :   rtx res = gen_reg_rtx (XFmode);
   22492            5 :   rtx cst, cstln2, cst1;
   22493            5 :   rtx_insn *insn;
   22494              : 
   22495              :   /* The emit_jump call emits pending stack adjust, make sure it is emitted
   22496              :      before the conditional jump, otherwise the stack adjustment will be
   22497              :      only conditional.  */
   22498            5 :   do_pending_stack_adjust ();
   22499              : 
   22500            5 :   cst = const_double_from_real_value
   22501            5 :     (REAL_VALUE_ATOF ("0.29289321881345247561810596348408353", XFmode), XFmode);
   22502            5 :   cstln2 = force_reg (XFmode, standard_80387_constant_rtx (4)); /* fldln2 */
   22503              : 
   22504            5 :   emit_insn (gen_absxf2 (tmp, op1));
   22505              : 
   22506            5 :   cst = force_reg (XFmode, cst);
   22507            5 :   ix86_expand_branch (GE, tmp, cst, label1);
   22508            5 :   predict_jump (REG_BR_PROB_BASE * 10 / 100);
   22509            5 :   insn = get_last_insn ();
   22510            5 :   JUMP_LABEL (insn) = label1;
   22511              : 
   22512            5 :   emit_insn (gen_fyl2xp1xf3_i387 (res, op1, cstln2));
   22513            5 :   emit_jump (label2);
   22514              : 
   22515            5 :   emit_label (label1);
   22516            5 :   LABEL_NUSES (label1) = 1;
   22517              : 
   22518            5 :   cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
   22519            5 :   emit_insn (gen_rtx_SET (tmp, gen_rtx_PLUS (XFmode, op1, cst1)));
   22520            5 :   emit_insn (gen_fyl2xxf3_i387 (res, tmp, cstln2));
   22521              : 
   22522            5 :   emit_label (label2);
   22523            5 :   LABEL_NUSES (label2) = 1;
   22524              : 
   22525            5 :   emit_move_insn (op0, res);
   22526            5 : }
   22527              : 
   22528              : /* Emit code for round calculation.  */
   22529              : void
   22530           60 : ix86_emit_i387_round (rtx op0, rtx op1)
   22531              : {
   22532           60 :   machine_mode inmode = GET_MODE (op1);
   22533           60 :   machine_mode outmode = GET_MODE (op0);
   22534           60 :   rtx e1 = gen_reg_rtx (XFmode);
   22535           60 :   rtx e2 = gen_reg_rtx (XFmode);
   22536           60 :   rtx scratch = gen_reg_rtx (HImode);
   22537           60 :   rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
   22538           60 :   rtx half = const_double_from_real_value (dconsthalf, XFmode);
   22539           60 :   rtx res = gen_reg_rtx (outmode);
   22540           60 :   rtx_code_label *jump_label = gen_label_rtx ();
   22541           60 :   rtx (*floor_insn) (rtx, rtx);
   22542           60 :   rtx (*neg_insn) (rtx, rtx);
   22543           60 :   rtx_insn *insn;
   22544           60 :   rtx tmp;
   22545              : 
   22546           60 :   switch (inmode)
   22547              :     {
   22548           29 :     case E_SFmode:
   22549           29 :     case E_DFmode:
   22550           29 :       tmp = gen_reg_rtx (XFmode);
   22551              : 
   22552           29 :       emit_insn (gen_rtx_SET (tmp, gen_rtx_FLOAT_EXTEND (XFmode, op1)));
   22553           29 :       op1 = tmp;
   22554           29 :       break;
   22555              :     case E_XFmode:
   22556              :       break;
   22557            0 :     default:
   22558            0 :       gcc_unreachable ();
   22559              :     }
   22560              : 
   22561           60 :   switch (outmode)
   22562              :     {
   22563              :     case E_SFmode:
   22564              :       floor_insn = gen_frndintxf2_floor;
   22565              :       neg_insn = gen_negsf2;
   22566              :       break;
   22567            6 :     case E_DFmode:
   22568            6 :       floor_insn = gen_frndintxf2_floor;
   22569            6 :       neg_insn = gen_negdf2;
   22570            6 :       break;
   22571           10 :     case E_XFmode:
   22572           10 :       floor_insn = gen_frndintxf2_floor;
   22573           10 :       neg_insn = gen_negxf2;
   22574           10 :       break;
   22575            0 :     case E_HImode:
   22576            0 :       floor_insn = gen_lfloorxfhi2;
   22577            0 :       neg_insn = gen_neghi2;
   22578            0 :       break;
   22579            6 :     case E_SImode:
   22580            6 :       floor_insn = gen_lfloorxfsi2;
   22581            6 :       neg_insn = gen_negsi2;
   22582            6 :       break;
   22583           36 :     case E_DImode:
   22584           36 :       floor_insn = gen_lfloorxfdi2;
   22585           36 :       neg_insn = gen_negdi2;
   22586           36 :       break;
   22587            0 :     default:
   22588            0 :       gcc_unreachable ();
   22589              :     }
   22590              : 
   22591              :   /* round(a) = sgn(a) * floor(fabs(a) + 0.5) */
   22592              : 
   22593              :   /* scratch = fxam(op1) */
   22594           60 :   emit_insn (gen_fxamxf2_i387 (scratch, op1));
   22595              : 
   22596              :   /* e1 = fabs(op1) */
   22597           60 :   emit_insn (gen_absxf2 (e1, op1));
   22598              : 
   22599              :   /* e2 = e1 + 0.5 */
   22600           60 :   half = force_reg (XFmode, half);
   22601           60 :   emit_insn (gen_rtx_SET (e2, gen_rtx_PLUS (XFmode, e1, half)));
   22602              : 
   22603              :   /* res = floor(e2) */
   22604           60 :   switch (outmode)
   22605              :     {
   22606            8 :     case E_SFmode:
   22607            8 :     case E_DFmode:
   22608            8 :       {
   22609            8 :         tmp = gen_reg_rtx (XFmode);
   22610              : 
   22611            8 :         emit_insn (floor_insn (tmp, e2));
   22612            8 :         emit_insn (gen_rtx_SET (res,
   22613              :                                 gen_rtx_UNSPEC (outmode, gen_rtvec (1, tmp),
   22614              :                                                 UNSPEC_TRUNC_NOOP)));
   22615              :       }
   22616            8 :       break;
   22617           52 :     default:
   22618           52 :       emit_insn (floor_insn (res, e2));
   22619              :     }
   22620              : 
   22621              :   /* flags = signbit(a) */
   22622           60 :   emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
   22623              : 
   22624              :   /* if (flags) then res = -res */
   22625           60 :   tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
   22626              :                               gen_rtx_EQ (VOIDmode, flags, const0_rtx),
   22627              :                               gen_rtx_LABEL_REF (VOIDmode, jump_label),
   22628              :                               pc_rtx);
   22629           60 :   insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
   22630           60 :   predict_jump (REG_BR_PROB_BASE * 50 / 100);
   22631           60 :   JUMP_LABEL (insn) = jump_label;
   22632              : 
   22633           60 :   emit_insn (neg_insn (res, res));
   22634              : 
   22635           60 :   emit_label (jump_label);
   22636           60 :   LABEL_NUSES (jump_label) = 1;
   22637              : 
   22638           60 :   emit_move_insn (op0, res);
   22639           60 : }
   22640              : 
   22641              : /* Output code to perform a Newton-Rhapson approximation of a single precision
   22642              :    floating point divide [http://en.wikipedia.org/wiki/N-th_root_algorithm].  */
   22643              : 
   22644              : void
   22645           56 : ix86_emit_swdivsf (rtx res, rtx a, rtx b, machine_mode mode)
   22646              : {
   22647           56 :   rtx x0, x1, e0, e1;
   22648              : 
   22649           56 :   x0 = gen_reg_rtx (mode);
   22650           56 :   e0 = gen_reg_rtx (mode);
   22651           56 :   e1 = gen_reg_rtx (mode);
   22652           56 :   x1 = gen_reg_rtx (mode);
   22653              : 
   22654           56 :   b = force_reg (mode, b);
   22655              : 
   22656              :   /* x0 = rcp(b) estimate */
   22657           56 :   if (mode == V16SFmode || mode == V8DFmode)
   22658              :     {
   22659            0 :       emit_insn (gen_rtx_SET (x0, gen_rtx_UNSPEC (mode, gen_rtvec (1, b),
   22660              :                                                   UNSPEC_RCP14)));
   22661              :     }
   22662              :   else
   22663           56 :     emit_insn (gen_rtx_SET (x0, gen_rtx_UNSPEC (mode, gen_rtvec (1, b),
   22664              :                                                 UNSPEC_RCP)));
   22665              : 
   22666           56 :   unsigned vector_size = GET_MODE_SIZE (mode);
   22667              : 
   22668              :   /* (a - (rcp(b) * a * b)) * rcp(b) + rcp(b) * a
   22669              :      N-R step with 2 fma implementation.  */
   22670           56 :   if (TARGET_FMA
   22671           55 :       || (TARGET_AVX512F && vector_size == 64)
   22672           55 :       || (TARGET_AVX512VL && (vector_size == 32 || vector_size == 16)))
   22673              :     {
   22674              :       /* e0 = x0 * a  */
   22675            1 :       emit_insn (gen_rtx_SET (e0, gen_rtx_MULT (mode, x0, a)));
   22676              :       /* e1 = e0 * b - a  */
   22677            1 :       emit_insn (gen_rtx_SET (e1, gen_rtx_FMA (mode, e0, b,
   22678              :                                                gen_rtx_NEG (mode, a))));
   22679              :       /* res = - e1 * x0 + e0  */
   22680            1 :       emit_insn (gen_rtx_SET (res, gen_rtx_FMA (mode,
   22681              :                                                gen_rtx_NEG (mode, e1),
   22682              :                                                x0, e0)));
   22683              :     }
   22684              :   else
   22685              :     /* a / b = a * ((rcp(b) + rcp(b)) - (b * rcp(b) * rcp (b))) */
   22686              :     {
   22687              :       /* e0 = x0 * b */
   22688           55 :       emit_insn (gen_rtx_SET (e0, gen_rtx_MULT (mode, x0, b)));
   22689              : 
   22690              :       /* e1 = x0 + x0 */
   22691           55 :       emit_insn (gen_rtx_SET (e1, gen_rtx_PLUS (mode, x0, x0)));
   22692              : 
   22693              :       /* e0 = x0 * e0 */
   22694           55 :       emit_insn (gen_rtx_SET (e0, gen_rtx_MULT (mode, x0, e0)));
   22695              : 
   22696              :       /* x1 = e1 - e0 */
   22697           55 :       emit_insn (gen_rtx_SET (x1, gen_rtx_MINUS (mode, e1, e0)));
   22698              : 
   22699              :       /* res = a * x1 */
   22700           55 :       emit_insn (gen_rtx_SET (res, gen_rtx_MULT (mode, a, x1)));
   22701              :     }
   22702           56 : }
   22703              : 
   22704              : /* Output code to perform a Newton-Rhapson approximation of a
   22705              :    single precision floating point [reciprocal] square root.  */
   22706              : 
   22707              : void
   22708           86 : ix86_emit_swsqrtsf (rtx res, rtx a, machine_mode mode, bool recip)
   22709              : {
   22710           86 :   rtx x0, e0, e1, e2, e3, mthree, mhalf;
   22711           86 :   REAL_VALUE_TYPE r;
   22712           86 :   int unspec;
   22713              : 
   22714           86 :   x0 = gen_reg_rtx (mode);
   22715           86 :   e0 = gen_reg_rtx (mode);
   22716           86 :   e1 = gen_reg_rtx (mode);
   22717           86 :   e2 = gen_reg_rtx (mode);
   22718           86 :   e3 = gen_reg_rtx (mode);
   22719              : 
   22720           86 :   real_from_integer (&r, VOIDmode, -3, SIGNED);
   22721           86 :   mthree = const_double_from_real_value (r, SFmode);
   22722              : 
   22723           86 :   real_arithmetic (&r, NEGATE_EXPR, &dconsthalf, NULL);
   22724           86 :   mhalf = const_double_from_real_value (r, SFmode);
   22725           86 :   unspec = UNSPEC_RSQRT;
   22726              : 
   22727           86 :   if (VECTOR_MODE_P (mode))
   22728              :     {
   22729           66 :       mthree = ix86_build_const_vector (mode, true, mthree);
   22730           66 :       mhalf = ix86_build_const_vector (mode, true, mhalf);
   22731              :       /* There is no 512-bit rsqrt.  There is however rsqrt14.  */
   22732          132 :       if (GET_MODE_SIZE (mode) == 64)
   22733            0 :         unspec = UNSPEC_RSQRT14;
   22734              :     }
   22735              : 
   22736              :   /* sqrt(a)  = -0.5 * a * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0)
   22737              :      rsqrt(a) = -0.5     * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0) */
   22738              : 
   22739           86 :   a = force_reg (mode, a);
   22740              : 
   22741              :   /* x0 = rsqrt(a) estimate */
   22742           86 :   emit_insn (gen_rtx_SET (x0, gen_rtx_UNSPEC (mode, gen_rtvec (1, a),
   22743              :                                               unspec)));
   22744              : 
   22745              :   /* If (a == 0.0) Filter out infinity to prevent NaN for sqrt(0.0).  */
   22746           86 :   if (!recip)
   22747              :     {
   22748           57 :       rtx zero = force_reg (mode, CONST0_RTX(mode));
   22749           57 :       rtx mask;
   22750              : 
   22751              :       /* Handle masked compare.  */
   22752          110 :       if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 64)
   22753              :         {
   22754            0 :           mask = gen_reg_rtx (HImode);
   22755              :           /* Imm value 0x4 corresponds to not-equal comparison.  */
   22756            0 :           emit_insn (gen_avx512f_cmpv16sf3 (mask, zero, a, GEN_INT (0x4)));
   22757            0 :           emit_insn (gen_avx512f_blendmv16sf (x0, zero, x0, mask));
   22758              :         }
   22759              :       else
   22760              :         {
   22761           57 :           mask = gen_reg_rtx (mode);
   22762           57 :           emit_insn (gen_rtx_SET (mask, gen_rtx_NE (mode, zero, a)));
   22763           57 :           emit_insn (gen_rtx_SET (x0, gen_rtx_AND (mode, x0, mask)));
   22764              :         }
   22765              :     }
   22766              : 
   22767           86 :   mthree = force_reg (mode, mthree);
   22768              : 
   22769              :   /* e0 = x0 * a */
   22770           86 :   emit_insn (gen_rtx_SET (e0, gen_rtx_MULT (mode, x0, a)));
   22771              : 
   22772           86 :   unsigned vector_size = GET_MODE_SIZE (mode);
   22773           86 :   if (TARGET_FMA
   22774           78 :       || (TARGET_AVX512F && vector_size == 64)
   22775           78 :       || (TARGET_AVX512VL && (vector_size == 32 || vector_size == 16)))
   22776           16 :     emit_insn (gen_rtx_SET (e2,
   22777              :                             gen_rtx_FMA (mode, e0, x0, mthree)));
   22778              :   else
   22779              :     {
   22780              :       /* e1 = e0 * x0 */
   22781           70 :       emit_insn (gen_rtx_SET (e1, gen_rtx_MULT (mode, e0, x0)));
   22782              : 
   22783              :       /* e2 = e1 - 3. */
   22784           70 :       emit_insn (gen_rtx_SET (e2, gen_rtx_PLUS (mode, e1, mthree)));
   22785              :     }
   22786              : 
   22787           86 :   mhalf = force_reg (mode, mhalf);
   22788           86 :   if (recip)
   22789              :     /* e3 = -.5 * x0 */
   22790           29 :     emit_insn (gen_rtx_SET (e3, gen_rtx_MULT (mode, x0, mhalf)));
   22791              :   else
   22792              :     /* e3 = -.5 * e0 */
   22793           57 :     emit_insn (gen_rtx_SET (e3, gen_rtx_MULT (mode, e0, mhalf)));
   22794              :   /* ret = e2 * e3 */
   22795           86 :   emit_insn (gen_rtx_SET (res, gen_rtx_MULT (mode, e2, e3)));
   22796           86 : }
   22797              : 
   22798              : /* Expand fabs (OP0) and return a new rtx that holds the result.  The
   22799              :    mask for masking out the sign-bit is stored in *SMASK, if that is
   22800              :    non-null.  */
   22801              : 
   22802              : static rtx
   22803         1091 : ix86_expand_sse_fabs (rtx op0, rtx *smask)
   22804              : {
   22805         1091 :   machine_mode vmode, mode = GET_MODE (op0);
   22806         1091 :   rtx xa, mask;
   22807              : 
   22808         1091 :   xa = gen_reg_rtx (mode);
   22809         1091 :   if (mode == SFmode)
   22810              :     vmode = V4SFmode;
   22811          495 :   else if (mode == DFmode)
   22812              :     vmode = V2DFmode;
   22813              :   else
   22814            0 :     vmode = mode;
   22815         1091 :   mask = ix86_build_signbit_mask (vmode, VECTOR_MODE_P (mode), true);
   22816         1091 :   if (!VECTOR_MODE_P (mode))
   22817              :     {
   22818              :       /* We need to generate a scalar mode mask in this case.  */
   22819         1091 :       rtx tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
   22820         1091 :       tmp = gen_rtx_VEC_SELECT (mode, mask, tmp);
   22821         1091 :       mask = gen_reg_rtx (mode);
   22822         1091 :       emit_insn (gen_rtx_SET (mask, tmp));
   22823              :     }
   22824         1091 :   emit_insn (gen_rtx_SET (xa, gen_rtx_AND (mode, op0, mask)));
   22825              : 
   22826         1091 :   if (smask)
   22827         1026 :     *smask = mask;
   22828              : 
   22829         1091 :   return xa;
   22830              : }
   22831              : 
   22832              : /* Expands a comparison of OP0 with OP1 using comparison code CODE,
   22833              :    swapping the operands if SWAP_OPERANDS is true.  The expanded
   22834              :    code is a forward jump to a newly created label in case the
   22835              :    comparison is true.  The generated label rtx is returned.  */
   22836              : static rtx_code_label *
   22837         1095 : ix86_expand_sse_compare_and_jump (enum rtx_code code, rtx op0, rtx op1,
   22838              :                                   bool swap_operands)
   22839              : {
   22840         1095 :   bool unordered_compare = ix86_unordered_fp_compare (code);
   22841         1095 :   rtx_code_label *label;
   22842         1095 :   rtx tmp, reg;
   22843              : 
   22844         1095 :   if (swap_operands)
   22845           35 :     std::swap (op0, op1);
   22846              : 
   22847         1095 :   label = gen_label_rtx ();
   22848         1095 :   tmp = gen_rtx_COMPARE (CCFPmode, op0, op1);
   22849         1095 :   if (unordered_compare)
   22850          939 :     tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_NOTRAP);
   22851         1095 :   reg = gen_rtx_REG (CCFPmode, FLAGS_REG);
   22852         1095 :   emit_insn (gen_rtx_SET (reg, tmp));
   22853         1095 :   tmp = gen_rtx_fmt_ee (code, VOIDmode, reg, const0_rtx);
   22854         1095 :   tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
   22855              :                               gen_rtx_LABEL_REF (VOIDmode, label), pc_rtx);
   22856         1095 :   rtx_insn *jump = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
   22857         1095 :   JUMP_LABEL (jump) = label;
   22858              : 
   22859         1095 :   return label;
   22860              : }
   22861              : 
   22862              : /* Expand a mask generating SSE comparison instruction comparing OP0 with OP1
   22863              :    using comparison code CODE.  Operands are swapped for the comparison if
   22864              :    SWAP_OPERANDS is true.  Returns a rtx for the generated mask.  */
   22865              : static rtx
   22866          554 : ix86_expand_sse_compare_mask (enum rtx_code code, rtx op0, rtx op1,
   22867              :                               bool swap_operands)
   22868              : {
   22869          554 :   rtx (*insn)(rtx, rtx, rtx, rtx);
   22870          554 :   machine_mode mode = GET_MODE (op0);
   22871          554 :   rtx mask = gen_reg_rtx (mode);
   22872              : 
   22873          554 :   if (swap_operands)
   22874          365 :     std::swap (op0, op1);
   22875              : 
   22876          554 :   insn = mode == DFmode ? gen_setcc_df_sse : gen_setcc_sf_sse;
   22877              : 
   22878          554 :   emit_insn (insn (mask, op0, op1,
   22879              :                    gen_rtx_fmt_ee (code, mode, op0, op1)));
   22880          554 :   return mask;
   22881              : }
   22882              : 
   22883              : /* Expand copysign from SIGN to the positive value ABS_VALUE
   22884              :    storing in RESULT.  If MASK is non-null, it shall be a mask to mask out
   22885              :    the sign-bit.  */
   22886              : 
   22887              : static void
   22888         1058 : ix86_sse_copysign_to_positive (rtx result, rtx abs_value, rtx sign, rtx mask)
   22889              : {
   22890         1058 :   machine_mode mode = GET_MODE (sign);
   22891         1058 :   rtx sgn = gen_reg_rtx (mode);
   22892         1058 :   if (mask == NULL_RTX)
   22893              :     {
   22894           28 :       machine_mode vmode;
   22895              : 
   22896           28 :       if (mode == SFmode)
   22897              :         vmode = V4SFmode;
   22898              :       else if (mode == DFmode)
   22899              :         vmode = V2DFmode;
   22900              :       else if (mode == HFmode)
   22901              :         vmode = V8HFmode;
   22902              :       else
   22903           28 :         vmode = mode;
   22904              : 
   22905           28 :       mask = ix86_build_signbit_mask (vmode, VECTOR_MODE_P (mode), false);
   22906           28 :       if (!VECTOR_MODE_P (mode))
   22907              :         {
   22908              :           /* We need to generate a scalar mode mask in this case.  */
   22909           28 :           rtx tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
   22910           28 :           tmp = gen_rtx_VEC_SELECT (mode, mask, tmp);
   22911           28 :           mask = gen_reg_rtx (mode);
   22912           28 :           emit_insn (gen_rtx_SET (mask, tmp));
   22913              :         }
   22914              :     }
   22915              :   else
   22916         1030 :     mask = gen_rtx_NOT (mode, mask);
   22917         1058 :   emit_insn (gen_rtx_SET (sgn, gen_rtx_AND (mode, mask, sign)));
   22918         1058 :   emit_insn (gen_rtx_SET (result, gen_rtx_IOR (mode, abs_value, sgn)));
   22919         1058 : }
   22920              : 
   22921              : /* Expand SSE sequence for computing lround from OP1 storing
   22922              :    into OP0.  */
   22923              : 
   22924              : void
   22925           28 : ix86_expand_lround (rtx op0, rtx op1)
   22926              : {
   22927              :   /* C code for the stuff we're doing below:
   22928              :         tmp = op1 + copysign (nextafter (0.5, 0.0), op1)
   22929              :         return (long)tmp;
   22930              :    */
   22931           28 :   machine_mode mode = GET_MODE (op1);
   22932           28 :   const struct real_format *fmt;
   22933           28 :   REAL_VALUE_TYPE pred_half, half_minus_pred_half;
   22934           28 :   rtx adj;
   22935              : 
   22936              :   /* load nextafter (0.5, 0.0) */
   22937           28 :   fmt = REAL_MODE_FORMAT (mode);
   22938           28 :   real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
   22939           28 :   real_arithmetic (&pred_half, MINUS_EXPR, &dconsthalf, &half_minus_pred_half);
   22940              : 
   22941              :   /* adj = copysign (0.5, op1) */
   22942           28 :   adj = force_reg (mode, const_double_from_real_value (pred_half, mode));
   22943           28 :   ix86_sse_copysign_to_positive (adj, adj, force_reg (mode, op1), NULL_RTX);
   22944              : 
   22945              :   /* adj = op1 + adj */
   22946           28 :   adj = expand_simple_binop (mode, PLUS, adj, op1, NULL_RTX, 0, OPTAB_DIRECT);
   22947              : 
   22948              :   /* op0 = (imode)adj */
   22949           28 :   expand_fix (op0, adj, 0);
   22950           28 : }
   22951              : 
   22952              : /* Expand SSE2 sequence for computing lround from OPERAND1 storing
   22953              :    into OPERAND0.  */
   22954              : 
   22955              : void
   22956           69 : ix86_expand_lfloorceil (rtx op0, rtx op1, bool do_floor)
   22957              : {
   22958              :   /* C code for the stuff we're doing below (for do_floor):
   22959              :         xi = (long)op1;
   22960              :         xi -= (double)xi > op1 ? 1 : 0;
   22961              :         return xi;
   22962              :    */
   22963           69 :   machine_mode fmode = GET_MODE (op1);
   22964           69 :   machine_mode imode = GET_MODE (op0);
   22965           69 :   rtx ireg, freg, tmp;
   22966           69 :   rtx_code_label *label;
   22967              : 
   22968              :   /* reg = (long)op1 */
   22969           69 :   ireg = gen_reg_rtx (imode);
   22970           69 :   expand_fix (ireg, op1, 0);
   22971              : 
   22972              :   /* freg = (double)reg */
   22973           69 :   freg = gen_reg_rtx (fmode);
   22974           69 :   expand_float (freg, ireg, 0);
   22975              : 
   22976              :   /* ireg = (freg > op1) ? ireg - 1 : ireg */
   22977           69 :   label = ix86_expand_sse_compare_and_jump (UNLE,
   22978              :                                             freg, op1, !do_floor);
   22979          104 :   tmp = expand_simple_binop (imode, do_floor ? MINUS : PLUS,
   22980              :                              ireg, const1_rtx, NULL_RTX, 0, OPTAB_DIRECT);
   22981           69 :   emit_move_insn (ireg, tmp);
   22982              : 
   22983           69 :   emit_label (label);
   22984           69 :   LABEL_NUSES (label) = 1;
   22985              : 
   22986           69 :   emit_move_insn (op0, ireg);
   22987           69 : }
   22988              : 
   22989              : /* Generate and return a rtx of mode MODE for 2**n where n is the number
   22990              :    of bits of the mantissa of MODE, which must be one of DFmode or SFmode.  */
   22991              : 
   22992              : static rtx
   22993         1026 : ix86_gen_TWO52 (machine_mode mode)
   22994              : {
   22995         1026 :   const struct real_format *fmt;
   22996         1026 :   REAL_VALUE_TYPE TWO52r;
   22997         1026 :   rtx TWO52;
   22998              : 
   22999         1026 :   fmt = REAL_MODE_FORMAT (mode);
   23000         1026 :   real_2expN (&TWO52r, fmt->p - 1, mode);
   23001         1026 :   TWO52 = const_double_from_real_value (TWO52r, mode);
   23002         1026 :   TWO52 = force_reg (mode, TWO52);
   23003              : 
   23004         1026 :   return TWO52;
   23005              : }
   23006              : 
   23007              : /* Expand rint rounding OPERAND1 and storing the result in OPERAND0.  */
   23008              : 
   23009              : void
   23010          133 : ix86_expand_rint (rtx operand0, rtx operand1)
   23011              : {
   23012              :   /* C code for the stuff we're doing below:
   23013              :         xa = fabs (operand1);
   23014              :         if (!isless (xa, 2**52))
   23015              :           return operand1;
   23016              :         two52 = 2**52;
   23017              :         if (flag_rounding_math)
   23018              :           {
   23019              :             two52 = copysign (two52, operand1);
   23020              :             xa = operand1;
   23021              :           }
   23022              :         xa = xa + two52 - two52;
   23023              :         return copysign (xa, operand1);
   23024              :    */
   23025          133 :   machine_mode mode = GET_MODE (operand0);
   23026          133 :   rtx res, xa, TWO52, mask;
   23027          133 :   rtx_code_label *label;
   23028              : 
   23029          133 :   TWO52 = ix86_gen_TWO52 (mode);
   23030              : 
   23031              :   /* Temporary for holding the result, initialized to the input
   23032              :      operand to ease control flow.  */
   23033          133 :   res = copy_to_reg (operand1);
   23034              : 
   23035              :   /* xa = abs (operand1) */
   23036          133 :   xa = ix86_expand_sse_fabs (res, &mask);
   23037              : 
   23038              :   /* if (!isless (xa, TWO52)) goto label; */
   23039          133 :   label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
   23040              : 
   23041          133 :   if (flag_rounding_math)
   23042              :     {
   23043           65 :       ix86_sse_copysign_to_positive (TWO52, TWO52, res, mask);
   23044           65 :       xa = res;
   23045              :     }
   23046              : 
   23047          133 :   xa = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
   23048          133 :   xa = expand_simple_binop (mode, MINUS, xa, TWO52, xa, 0, OPTAB_DIRECT);
   23049              : 
   23050              :   /* Remove the sign with FE_DOWNWARD, where x - x = -0.0.  */
   23051          133 :   if (HONOR_SIGNED_ZEROS (mode) && flag_rounding_math)
   23052           65 :     xa = ix86_expand_sse_fabs (xa, NULL);
   23053              : 
   23054          133 :   ix86_sse_copysign_to_positive (res, xa, res, mask);
   23055              : 
   23056          133 :   emit_label (label);
   23057          133 :   LABEL_NUSES (label) = 1;
   23058              : 
   23059          133 :   emit_move_insn (operand0, res);
   23060          133 : }
   23061              : 
   23062              : /* Expand SSE2 sequence for computing floor or ceil
   23063              :    from OPERAND1 storing into OPERAND0.  */
   23064              : void
   23065          554 : ix86_expand_floorceil (rtx operand0, rtx operand1, bool do_floor)
   23066              : {
   23067              :   /* C code for the stuff we expand below.
   23068              :         double xa = fabs (x), x2;
   23069              :         if (!isless (xa, TWO52))
   23070              :           return x;
   23071              :         x2 = (double)(long)x;
   23072              : 
   23073              :      Compensate.  Floor:
   23074              :         if (x2 > x)
   23075              :           x2 -= 1;
   23076              :      Compensate.  Ceil:
   23077              :         if (x2 < x)
   23078              :           x2 += 1;
   23079              : 
   23080              :         if (HONOR_SIGNED_ZEROS (mode))
   23081              :           return copysign (x2, x);
   23082              :         return x2;
   23083              :    */
   23084          554 :   machine_mode mode = GET_MODE (operand0);
   23085          554 :   rtx xa, xi, TWO52, tmp, one, res, mask;
   23086          554 :   rtx_code_label *label;
   23087              : 
   23088          554 :   TWO52 = ix86_gen_TWO52 (mode);
   23089              : 
   23090              :   /* Temporary for holding the result, initialized to the input
   23091              :      operand to ease control flow.  */
   23092          554 :   res = copy_to_reg (operand1);
   23093              : 
   23094              :   /* xa = abs (operand1) */
   23095          554 :   xa = ix86_expand_sse_fabs (res, &mask);
   23096              : 
   23097              :   /* if (!isless (xa, TWO52)) goto label; */
   23098          554 :   label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
   23099              : 
   23100              :   /* xa = (double)(long)x */
   23101          554 :   xi = gen_reg_rtx (int_mode_for_mode (mode).require ());
   23102          554 :   expand_fix (xi, res, 0);
   23103          554 :   expand_float (xa, xi, 0);
   23104              : 
   23105              :   /* generate 1.0 */
   23106          554 :   one = force_reg (mode, const_double_from_real_value (dconst1, mode));
   23107              : 
   23108              :   /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
   23109          554 :   tmp = ix86_expand_sse_compare_mask (UNGT, xa, res, !do_floor);
   23110          554 :   emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, one, tmp)));
   23111          919 :   tmp = expand_simple_binop (mode, do_floor ? MINUS : PLUS,
   23112              :                              xa, tmp, NULL_RTX, 0, OPTAB_DIRECT);
   23113          554 :   if (HONOR_SIGNED_ZEROS (mode))
   23114              :     {
   23115              :       /* Remove the sign with FE_DOWNWARD, where x - x = -0.0.  */
   23116          507 :       if (do_floor && flag_rounding_math)
   23117            0 :         tmp = ix86_expand_sse_fabs (tmp, NULL);
   23118              : 
   23119          507 :       ix86_sse_copysign_to_positive (tmp, tmp, res, mask);
   23120              :     }
   23121          554 :   emit_move_insn (res, tmp);
   23122              : 
   23123          554 :   emit_label (label);
   23124          554 :   LABEL_NUSES (label) = 1;
   23125              : 
   23126          554 :   emit_move_insn (operand0, res);
   23127          554 : }
   23128              : 
   23129              : /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
   23130              :    into OPERAND0 without relying on DImode truncation via cvttsd2siq
   23131              :    that is only available on 64bit targets.  */
   23132              : void
   23133            0 : ix86_expand_floorceildf_32 (rtx operand0, rtx operand1, bool do_floor)
   23134              : {
   23135              :   /* C code for the stuff we expand below.
   23136              :         double xa = fabs (x), x2;
   23137              :         if (!isless (xa, TWO52))
   23138              :           return x;
   23139              :         xa = xa + TWO52 - TWO52;
   23140              :         x2 = copysign (xa, x);
   23141              : 
   23142              :      Compensate.  Floor:
   23143              :         if (x2 > x)
   23144              :           x2 -= 1;
   23145              :      Compensate.  Ceil:
   23146              :         if (x2 < x)
   23147              :           x2 += 1;
   23148              : 
   23149              :         if (HONOR_SIGNED_ZEROS (mode))
   23150              :           x2 = copysign (x2, x);
   23151              :         return x2;
   23152              :    */
   23153            0 :   machine_mode mode = GET_MODE (operand0);
   23154            0 :   rtx xa, TWO52, tmp, one, res, mask;
   23155            0 :   rtx_code_label *label;
   23156              : 
   23157            0 :   TWO52 = ix86_gen_TWO52 (mode);
   23158              : 
   23159              :   /* Temporary for holding the result, initialized to the input
   23160              :      operand to ease control flow.  */
   23161            0 :   res = copy_to_reg (operand1);
   23162              : 
   23163              :   /* xa = abs (operand1) */
   23164            0 :   xa = ix86_expand_sse_fabs (res, &mask);
   23165              : 
   23166              :   /* if (!isless (xa, TWO52)) goto label; */
   23167            0 :   label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
   23168              : 
   23169              :   /* xa = xa + TWO52 - TWO52; */
   23170            0 :   xa = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
   23171            0 :   xa = expand_simple_binop (mode, MINUS, xa, TWO52, xa, 0, OPTAB_DIRECT);
   23172              : 
   23173              :   /* xa = copysign (xa, operand1) */
   23174            0 :   ix86_sse_copysign_to_positive (xa, xa, res, mask);
   23175              : 
   23176              :   /* generate 1.0 */
   23177            0 :   one = force_reg (mode, const_double_from_real_value (dconst1, mode));
   23178              : 
   23179              :   /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
   23180            0 :   tmp = ix86_expand_sse_compare_mask (UNGT, xa, res, !do_floor);
   23181            0 :   emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, one, tmp)));
   23182            0 :   tmp = expand_simple_binop (mode, do_floor ? MINUS : PLUS,
   23183              :                              xa, tmp, NULL_RTX, 0, OPTAB_DIRECT);
   23184            0 :   if (HONOR_SIGNED_ZEROS (mode))
   23185              :     {
   23186              :       /* Remove the sign with FE_DOWNWARD, where x - x = -0.0.  */
   23187            0 :       if (do_floor && flag_rounding_math)
   23188            0 :         tmp = ix86_expand_sse_fabs (tmp, NULL);
   23189              : 
   23190            0 :       ix86_sse_copysign_to_positive (tmp, tmp, res, mask);
   23191              :     }
   23192            0 :   emit_move_insn (res, tmp);
   23193              : 
   23194            0 :   emit_label (label);
   23195            0 :   LABEL_NUSES (label) = 1;
   23196              : 
   23197            0 :   emit_move_insn (operand0, res);
   23198            0 : }
   23199              : 
   23200              : /* Expand SSE sequence for computing trunc
   23201              :    from OPERAND1 storing into OPERAND0.  */
   23202              : void
   23203          325 : ix86_expand_trunc (rtx operand0, rtx operand1)
   23204              : {
   23205              :   /* C code for SSE variant we expand below.
   23206              :         double xa = fabs (x), x2;
   23207              :         if (!isless (xa, TWO52))
   23208              :           return x;
   23209              :         x2 = (double)(long)x;
   23210              :         if (HONOR_SIGNED_ZEROS (mode))
   23211              :           return copysign (x2, x);
   23212              :         return x2;
   23213              :    */
   23214          325 :   machine_mode mode = GET_MODE (operand0);
   23215          325 :   rtx xa, xi, TWO52, res, mask;
   23216          325 :   rtx_code_label *label;
   23217              : 
   23218          325 :   TWO52 = ix86_gen_TWO52 (mode);
   23219              : 
   23220              :   /* Temporary for holding the result, initialized to the input
   23221              :      operand to ease control flow.  */
   23222          325 :   res = copy_to_reg (operand1);
   23223              : 
   23224              :   /* xa = abs (operand1) */
   23225          325 :   xa = ix86_expand_sse_fabs (res, &mask);
   23226              : 
   23227              :   /* if (!isless (xa, TWO52)) goto label; */
   23228          325 :   label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
   23229              : 
   23230              :   /* xa = (double)(long)x */
   23231          325 :   xi = gen_reg_rtx (int_mode_for_mode (mode).require ());
   23232          325 :   expand_fix (xi, res, 0);
   23233          325 :   expand_float (xa, xi, 0);
   23234              : 
   23235          325 :   if (HONOR_SIGNED_ZEROS (mode))
   23236          311 :     ix86_sse_copysign_to_positive (xa, xa, res, mask);
   23237              : 
   23238          325 :   emit_move_insn (res, xa);
   23239              : 
   23240          325 :   emit_label (label);
   23241          325 :   LABEL_NUSES (label) = 1;
   23242              : 
   23243          325 :   emit_move_insn (operand0, res);
   23244          325 : }
   23245              : 
   23246              : /* Expand SSE sequence for computing trunc from OPERAND1 storing
   23247              :    into OPERAND0 without relying on DImode truncation via cvttsd2siq
   23248              :    that is only available on 64bit targets.  */
   23249              : void
   23250            0 : ix86_expand_truncdf_32 (rtx operand0, rtx operand1)
   23251              : {
   23252            0 :   machine_mode mode = GET_MODE (operand0);
   23253            0 :   rtx xa, xa2, TWO52, tmp, one, res, mask;
   23254            0 :   rtx_code_label *label;
   23255              : 
   23256              :   /* C code for SSE variant we expand below.
   23257              :         double xa = fabs (x), x2;
   23258              :         if (!isless (xa, TWO52))
   23259              :           return x;
   23260              :         xa2 = xa + TWO52 - TWO52;
   23261              :      Compensate:
   23262              :         if (xa2 > xa)
   23263              :           xa2 -= 1.0;
   23264              :         x2 = copysign (xa2, x);
   23265              :         return x2;
   23266              :    */
   23267              : 
   23268            0 :   TWO52 = ix86_gen_TWO52 (mode);
   23269              : 
   23270              :   /* Temporary for holding the result, initialized to the input
   23271              :      operand to ease control flow.  */
   23272            0 :   res =copy_to_reg (operand1);
   23273              : 
   23274              :   /* xa = abs (operand1) */
   23275            0 :   xa = ix86_expand_sse_fabs (res, &mask);
   23276              : 
   23277              :   /* if (!isless (xa, TWO52)) goto label; */
   23278            0 :   label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
   23279              : 
   23280              :   /* xa2 = xa + TWO52 - TWO52; */
   23281            0 :   xa2 = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
   23282            0 :   xa2 = expand_simple_binop (mode, MINUS, xa2, TWO52, xa2, 0, OPTAB_DIRECT);
   23283              : 
   23284              :   /* generate 1.0 */
   23285            0 :   one = force_reg (mode, const_double_from_real_value (dconst1, mode));
   23286              : 
   23287              :   /* Compensate: xa2 = xa2 - (xa2 > xa ? 1 : 0)  */
   23288            0 :   tmp = ix86_expand_sse_compare_mask (UNGT, xa2, xa, false);
   23289            0 :   emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, one, tmp)));
   23290            0 :   tmp = expand_simple_binop (mode, MINUS,
   23291              :                              xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
   23292              :   /* Remove the sign with FE_DOWNWARD, where x - x = -0.0.  */
   23293            0 :   if (HONOR_SIGNED_ZEROS (mode) && flag_rounding_math)
   23294            0 :     tmp = ix86_expand_sse_fabs (tmp, NULL);
   23295              : 
   23296              :   /* res = copysign (xa2, operand1) */
   23297            0 :   ix86_sse_copysign_to_positive (res, tmp, res, mask);
   23298              : 
   23299            0 :   emit_label (label);
   23300            0 :   LABEL_NUSES (label) = 1;
   23301              : 
   23302            0 :   emit_move_insn (operand0, res);
   23303            0 : }
   23304              : 
   23305              : /* Expand truncsfbf2 like vcvtneps2bf16, which doesn't honor SNAN,
   23306              :    turns sNAN into qNAN quietly, it always rounds to nearest even
   23307              :    and flushes denormals to zero.  We can expand the conversion
   23308              :    inline as (fromi + 0x7fff + ((fromi >> 16) & 1)) >> 16, flushing
   23309              :    denormals to zero.  */
   23310              : 
   23311              : void
   23312           74 : ix86_expand_truncsfbf2 (rtx op0, rtx op1)
   23313              : {
   23314           74 :   rtx set;
   23315              : 
   23316           74 :   if (TARGET_AVXNECONVERT
   23317           72 :       || (TARGET_AVX512BF16 && TARGET_AVX512VL))
   23318              :     {
   23319           71 :       set = gen_truncsfbf2_vcvtneps2bf16 (op0, op1);
   23320           71 :       emit_insn (set);
   23321           71 :       return;
   23322              :     }
   23323              : 
   23324              :   /* Convert OP1 to REG1_SI in SImode.  */
   23325            3 :   rtx reg1_si = gen_reg_rtx (SImode);
   23326            3 :   rtx op1_si = gen_lowpart (SImode, op1);
   23327            3 :   set = gen_rtx_SET (reg1_si, op1_si);
   23328            3 :   emit_insn (set);
   23329              : 
   23330              :   /* Set TMP0 to REG1_SI >> 16. */
   23331            3 :   rtx tmp0 = expand_simple_binop (SImode, LSHIFTRT, reg1_si,
   23332              :                                   GEN_INT(16), nullptr, 0,
   23333              :                                   OPTAB_DIRECT);
   23334              : 
   23335            3 :   rtx_code_label *zero_label = gen_label_rtx ();
   23336            3 :   rtx_code_label *cast_label = gen_label_rtx ();
   23337              : 
   23338            3 :   rtx result = gen_reg_rtx (SImode);
   23339              : 
   23340              :   /* OP1 is zero or denormal if (REG1_SI & 0x7f800000) == 0.  */
   23341            3 :   rtx tmp1 = expand_simple_binop (SImode, AND, reg1_si,
   23342              :                                   GEN_INT(0x7f800000), nullptr, 0,
   23343              :                                   OPTAB_DIRECT);
   23344              : 
   23345            3 :   emit_cmp_and_jump_insns (tmp1, const0_rtx, EQ, nullptr, SImode,
   23346              :                            true, zero_label,
   23347              :                            profile_probability::unlikely ());
   23348              : 
   23349              :   /* Set TMP1 to TMP0 & 1. */
   23350            3 :   tmp1 = expand_simple_binop (SImode, AND, tmp0, const1_rtx, nullptr,
   23351              :                               0, OPTAB_DIRECT);
   23352              : 
   23353              :   /* Set TMP2 to REG1_SI + 0x7fff.  */
   23354            3 :   rtx tmp2 = expand_simple_binop (SImode, PLUS, reg1_si,
   23355              :                                   GEN_INT (0x7fff), nullptr, 0,
   23356              :                                   OPTAB_DIRECT);
   23357              :   /* Set TMP1 to TMP2 + TMP1.  */
   23358            3 :   tmp1 = expand_simple_binop (SImode, PLUS, tmp2, tmp1, nullptr, 0,
   23359              :                               OPTAB_DIRECT);
   23360              : 
   23361              :   /* Set TMP1 to TMP1 >> 16. */
   23362            3 :   tmp1 = expand_simple_binop (SImode, LSHIFTRT, tmp1, GEN_INT(16),
   23363              :                               nullptr, 0, OPTAB_DIRECT);
   23364              : 
   23365              :   /* Move TMP1 to RESULT.  */
   23366            3 :   emit_move_insn (result, tmp1);
   23367              : 
   23368            3 :   emit_jump_insn (gen_jump (cast_label));
   23369            3 :   emit_barrier ();
   23370              : 
   23371            3 :   emit_label (zero_label);
   23372              : 
   23373              :   /* Flush TMP0 to zero while keeping the sign bit for zero and
   23374              :      denormal.  */
   23375            3 :   tmp1 = expand_simple_binop (SImode, AND, tmp0, GEN_INT(0x8000),
   23376              :                               nullptr, 0, OPTAB_DIRECT);
   23377              : 
   23378              :   /* Move TMP1 to RESULT.  */
   23379            3 :   emit_move_insn (result, tmp1);
   23380              : 
   23381            3 :   emit_label (cast_label);
   23382              : 
   23383              :   /* Cast RESULT to TMP2 in HImode.  */
   23384            3 :   tmp2 = gen_reg_rtx (HImode);
   23385            3 :   tmp0 = gen_lowpart (HImode, result);
   23386            3 :   set = gen_rtx_SET (tmp2, tmp0);
   23387            3 :   emit_insn (set);
   23388              : 
   23389              :   /* Convert TMP2 to OP0.  */
   23390            3 :   tmp0 = gen_lowpart (BFmode, tmp2);
   23391            3 :   set = gen_rtx_SET (op0, tmp0);
   23392            3 :   emit_insn (set);
   23393              : }
   23394              : 
   23395              : /* Expand SSE sequence for computing round
   23396              :    from OPERAND1 storing into OPERAND0.  */
   23397              : void
   23398           14 : ix86_expand_round (rtx operand0, rtx operand1)
   23399              : {
   23400              :   /* C code for the stuff we're doing below:
   23401              :         double xa = fabs (x);
   23402              :         if (!isless (xa, TWO52))
   23403              :           return x;
   23404              :         xa = (double)(long)(xa + nextafter (0.5, 0.0));
   23405              :         return copysign (xa, x);
   23406              :    */
   23407           14 :   machine_mode mode = GET_MODE (operand0);
   23408           14 :   rtx res, TWO52, xa, xi, half, mask;
   23409           14 :   rtx_code_label *label;
   23410           14 :   const struct real_format *fmt;
   23411           14 :   REAL_VALUE_TYPE pred_half, half_minus_pred_half;
   23412              : 
   23413              :   /* Temporary for holding the result, initialized to the input
   23414              :      operand to ease control flow.  */
   23415           14 :   res = copy_to_reg (operand1);
   23416              : 
   23417           14 :   TWO52 = ix86_gen_TWO52 (mode);
   23418           14 :   xa = ix86_expand_sse_fabs (res, &mask);
   23419           14 :   label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
   23420              : 
   23421              :   /* load nextafter (0.5, 0.0) */
   23422           14 :   fmt = REAL_MODE_FORMAT (mode);
   23423           14 :   real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
   23424           14 :   real_arithmetic (&pred_half, MINUS_EXPR, &dconsthalf, &half_minus_pred_half);
   23425              : 
   23426              :   /* xa = xa + 0.5 */
   23427           14 :   half = force_reg (mode, const_double_from_real_value (pred_half, mode));
   23428           14 :   xa = expand_simple_binop (mode, PLUS, xa, half, NULL_RTX, 0, OPTAB_DIRECT);
   23429              : 
   23430              :   /* xa = (double)(int64_t)xa */
   23431           14 :   xi = gen_reg_rtx (int_mode_for_mode (mode).require ());
   23432           14 :   expand_fix (xi, xa, 0);
   23433           14 :   expand_float (xa, xi, 0);
   23434              : 
   23435              :   /* res = copysign (xa, operand1) */
   23436           14 :   ix86_sse_copysign_to_positive (res, xa, res, mask);
   23437              : 
   23438           14 :   emit_label (label);
   23439           14 :   LABEL_NUSES (label) = 1;
   23440              : 
   23441           14 :   emit_move_insn (operand0, res);
   23442           14 : }
   23443              : 
   23444              : /* Expand SSE sequence for computing round from OPERAND1 storing
   23445              :    into OPERAND0 without relying on DImode truncation via cvttsd2siq
   23446              :    that is only available on 64bit targets.  */
   23447              : void
   23448            0 : ix86_expand_rounddf_32 (rtx operand0, rtx operand1)
   23449              : {
   23450              :   /* C code for the stuff we expand below.
   23451              :         double xa = fabs (x), xa2, x2;
   23452              :         if (!isless (xa, TWO52))
   23453              :           return x;
   23454              :      Using the absolute value and copying back sign makes
   23455              :      -0.0 -> -0.0 correct.
   23456              :         xa2 = xa + TWO52 - TWO52;
   23457              :      Compensate.
   23458              :         dxa = xa2 - xa;
   23459              :         if (dxa <= -0.5)
   23460              :           xa2 += 1;
   23461              :         else if (dxa > 0.5)
   23462              :           xa2 -= 1;
   23463              :         x2 = copysign (xa2, x);
   23464              :         return x2;
   23465              :    */
   23466            0 :   machine_mode mode = GET_MODE (operand0);
   23467            0 :   rtx xa, xa2, dxa, TWO52, tmp, half, mhalf, one, res, mask;
   23468            0 :   rtx_code_label *label;
   23469              : 
   23470            0 :   TWO52 = ix86_gen_TWO52 (mode);
   23471              : 
   23472              :   /* Temporary for holding the result, initialized to the input
   23473              :      operand to ease control flow.  */
   23474            0 :   res = copy_to_reg (operand1);
   23475              : 
   23476              :   /* xa = abs (operand1) */
   23477            0 :   xa = ix86_expand_sse_fabs (res, &mask);
   23478              : 
   23479              :   /* if (!isless (xa, TWO52)) goto label; */
   23480            0 :   label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
   23481              : 
   23482              :   /* xa2 = xa + TWO52 - TWO52; */
   23483            0 :   xa2 = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
   23484            0 :   xa2 = expand_simple_binop (mode, MINUS, xa2, TWO52, xa2, 0, OPTAB_DIRECT);
   23485              : 
   23486              :   /* dxa = xa2 - xa; */
   23487            0 :   dxa = expand_simple_binop (mode, MINUS, xa2, xa, NULL_RTX, 0, OPTAB_DIRECT);
   23488              : 
   23489              :   /* generate 0.5, 1.0 and -0.5 */
   23490            0 :   half = force_reg (mode, const_double_from_real_value (dconsthalf, mode));
   23491            0 :   one = expand_simple_binop (mode, PLUS, half, half, NULL_RTX, 0, OPTAB_DIRECT);
   23492            0 :   mhalf = expand_simple_binop (mode, MINUS, half, one, NULL_RTX,
   23493              :                                0, OPTAB_DIRECT);
   23494              : 
   23495              :   /* Compensate.  */
   23496              :   /* xa2 = xa2 - (dxa > 0.5 ? 1 : 0) */
   23497            0 :   tmp = ix86_expand_sse_compare_mask (UNGT, dxa, half, false);
   23498            0 :   emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, tmp, one)));
   23499            0 :   xa2 = expand_simple_binop (mode, MINUS, xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
   23500              :   /* xa2 = xa2 + (dxa <= -0.5 ? 1 : 0) */
   23501            0 :   tmp = ix86_expand_sse_compare_mask (UNGE, mhalf, dxa, false);
   23502            0 :   emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, tmp, one)));
   23503            0 :   xa2 = expand_simple_binop (mode, PLUS, xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
   23504              : 
   23505              :   /* res = copysign (xa2, operand1) */
   23506            0 :   ix86_sse_copysign_to_positive (res, xa2, res, mask);
   23507              : 
   23508            0 :   emit_label (label);
   23509            0 :   LABEL_NUSES (label) = 1;
   23510              : 
   23511            0 :   emit_move_insn (operand0, res);
   23512            0 : }
   23513              : 
   23514              : /* Expand SSE sequence for computing round
   23515              :    from OP1 storing into OP0 using sse4 round insn.  */
   23516              : void
   23517           10 : ix86_expand_round_sse4 (rtx op0, rtx op1)
   23518              : {
   23519           10 :   machine_mode mode = GET_MODE (op0);
   23520           10 :   rtx e1, e2, res, half;
   23521           10 :   const struct real_format *fmt;
   23522           10 :   REAL_VALUE_TYPE pred_half, half_minus_pred_half;
   23523           10 :   rtx (*gen_copysign) (rtx, rtx, rtx);
   23524           10 :   rtx (*gen_round) (rtx, rtx, rtx);
   23525              : 
   23526           10 :   switch (mode)
   23527              :     {
   23528              :     case E_HFmode:
   23529              :       gen_copysign = gen_copysignhf3;
   23530              :       gen_round = gen_sse4_1_roundhf2;
   23531              :       break;
   23532            4 :     case E_SFmode:
   23533            4 :       gen_copysign = gen_copysignsf3;
   23534            4 :       gen_round = gen_sse4_1_roundsf2;
   23535            4 :       break;
   23536            4 :     case E_DFmode:
   23537            4 :       gen_copysign = gen_copysigndf3;
   23538            4 :       gen_round = gen_sse4_1_rounddf2;
   23539            4 :       break;
   23540            0 :     default:
   23541            0 :       gcc_unreachable ();
   23542              :     }
   23543              : 
   23544              :   /* round (a) = trunc (a + copysign (0.5, a)) */
   23545              : 
   23546              :   /* load nextafter (0.5, 0.0) */
   23547           10 :   fmt = REAL_MODE_FORMAT (mode);
   23548           10 :   real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
   23549           10 :   real_arithmetic (&pred_half, MINUS_EXPR, &dconsthalf, &half_minus_pred_half);
   23550           10 :   half = const_double_from_real_value (pred_half, mode);
   23551              : 
   23552              :   /* e1 = copysign (0.5, op1) */
   23553           10 :   e1 = gen_reg_rtx (mode);
   23554           10 :   emit_insn (gen_copysign (e1, half, op1));
   23555              : 
   23556              :   /* e2 = op1 + e1 */
   23557           10 :   e2 = expand_simple_binop (mode, PLUS, op1, e1, NULL_RTX, 0, OPTAB_DIRECT);
   23558              : 
   23559              :   /* res = trunc (e2) */
   23560           10 :   res = gen_reg_rtx (mode);
   23561           10 :   emit_insn (gen_round (res, e2, GEN_INT (ROUND_TRUNC)));
   23562              : 
   23563           10 :   emit_move_insn (op0, res);
   23564           10 : }
   23565              : 
   23566              : /* A cached (set (nil) (vselect (vconcat (nil) (nil)) (parallel [])))
   23567              :    insn, so that expand_vselect{,_vconcat} doesn't have to create a fresh
   23568              :    insn every time.  */
   23569              : 
   23570              : static GTY(()) rtx_insn *vselect_insn;
   23571              : 
   23572              : /* Initialize vselect_insn.  */
   23573              : 
   23574              : static void
   23575         7910 : init_vselect_insn (void)
   23576              : {
   23577         7910 :   unsigned i;
   23578         7910 :   rtx x;
   23579              : 
   23580         7910 :   x = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (MAX_VECT_LEN));
   23581       514150 :   for (i = 0; i < MAX_VECT_LEN; ++i)
   23582       506240 :     XVECEXP (x, 0, i) = const0_rtx;
   23583         7910 :   x = gen_rtx_VEC_SELECT (V2DFmode, gen_rtx_VEC_CONCAT (V4DFmode, const0_rtx,
   23584              :                                                         const0_rtx), x);
   23585         7910 :   x = gen_rtx_SET (const0_rtx, x);
   23586         7910 :   start_sequence ();
   23587         7910 :   vselect_insn = emit_insn (x);
   23588         7910 :   end_sequence ();
   23589         7910 : }
   23590              : 
   23591              : /* Construct (set target (vec_select op0 (parallel perm))) and
   23592              :    return true if that's a valid instruction in the active ISA.  */
   23593              : 
   23594              : static bool
   23595       556003 : expand_vselect (rtx target, rtx op0, const unsigned char *perm,
   23596              :                 unsigned nelt, bool testing_p)
   23597              : {
   23598       556003 :   unsigned int i;
   23599       556003 :   rtx x, save_vconcat;
   23600       556003 :   int icode;
   23601              : 
   23602       556003 :   if (vselect_insn == NULL_RTX)
   23603         1677 :     init_vselect_insn ();
   23604              : 
   23605       556003 :   x = XEXP (SET_SRC (PATTERN (vselect_insn)), 1);
   23606       556003 :   PUT_NUM_ELEM (XVEC (x, 0), nelt);
   23607      4266571 :   for (i = 0; i < nelt; ++i)
   23608      3710568 :     XVECEXP (x, 0, i) = GEN_INT (perm[i]);
   23609       556003 :   save_vconcat = XEXP (SET_SRC (PATTERN (vselect_insn)), 0);
   23610       556003 :   XEXP (SET_SRC (PATTERN (vselect_insn)), 0) = op0;
   23611       556003 :   PUT_MODE (SET_SRC (PATTERN (vselect_insn)), GET_MODE (target));
   23612       556003 :   SET_DEST (PATTERN (vselect_insn)) = target;
   23613       556003 :   icode = recog_memoized (vselect_insn);
   23614              : 
   23615       556003 :   if (icode >= 0 && !testing_p)
   23616        72927 :     emit_insn (copy_rtx (PATTERN (vselect_insn)));
   23617              : 
   23618       556003 :   SET_DEST (PATTERN (vselect_insn)) = const0_rtx;
   23619       556003 :   XEXP (SET_SRC (PATTERN (vselect_insn)), 0) = save_vconcat;
   23620       556003 :   INSN_CODE (vselect_insn) = -1;
   23621              : 
   23622       556003 :   return icode >= 0;
   23623              : }
   23624              : 
   23625              : /* Similar, but generate a vec_concat from op0 and op1 as well.  */
   23626              : 
   23627              : static bool
   23628       489042 : expand_vselect_vconcat (rtx target, rtx op0, rtx op1,
   23629              :                         const unsigned char *perm, unsigned nelt,
   23630              :                         bool testing_p)
   23631              : {
   23632       489042 :   machine_mode v2mode;
   23633       489042 :   rtx x;
   23634       489042 :   bool ok;
   23635              : 
   23636       489042 :   if (vselect_insn == NULL_RTX)
   23637         6233 :     init_vselect_insn ();
   23638              : 
   23639       489042 :   if (!GET_MODE_2XWIDER_MODE (GET_MODE (op0)).exists (&v2mode))
   23640              :     return false;
   23641       489042 :   x = XEXP (SET_SRC (PATTERN (vselect_insn)), 0);
   23642       489042 :   PUT_MODE (x, v2mode);
   23643       489042 :   XEXP (x, 0) = op0;
   23644       489042 :   XEXP (x, 1) = op1;
   23645       489042 :   ok = expand_vselect (target, x, perm, nelt, testing_p);
   23646       489042 :   XEXP (x, 0) = const0_rtx;
   23647       489042 :   XEXP (x, 1) = const0_rtx;
   23648       489042 :   return ok;
   23649              : }
   23650              : 
   23651              : /* A subroutine of ix86_expand_vec_perm_const_1.  Try to implement D
   23652              :    using movss or movsd.  */
   23653              : static bool
   23654       311637 : expand_vec_perm_movs (struct expand_vec_perm_d *d)
   23655              : {
   23656       311637 :   machine_mode vmode = d->vmode;
   23657       311637 :   unsigned i, nelt = d->nelt;
   23658       311637 :   rtx x;
   23659              : 
   23660       311637 :   if (d->one_operand_p)
   23661              :     return false;
   23662              : 
   23663       285670 :   if (!(TARGET_SSE && (vmode == V4SFmode || vmode == V4SImode))
   23664       132393 :       && !(TARGET_MMX_WITH_SSE && (vmode == V2SFmode || vmode == V2SImode))
   23665        91389 :       && !(TARGET_SSE2 && (vmode == V2DFmode || vmode == V2DImode)))
   23666              :     return false;
   23667              : 
   23668              :   /* Only the first element is changed.  */
   23669       203895 :   if (d->perm[0] != nelt && d->perm[0] != 0)
   23670              :     return false;
   23671       124540 :   for (i = 1; i < nelt; ++i)
   23672       109703 :     if (d->perm[i] != i + nelt - d->perm[0])
   23673              :       return false;
   23674              : 
   23675        14837 :   if (d->testing_p)
   23676              :     return true;
   23677              : 
   23678         7060 :   if (d->perm[0] == nelt)
   23679            0 :     x = gen_rtx_VEC_MERGE (vmode, d->op1, d->op0, GEN_INT (1));
   23680              :   else
   23681         7060 :     x = gen_rtx_VEC_MERGE (vmode, d->op0, d->op1, GEN_INT (1));
   23682              : 
   23683         7060 :   emit_insn (gen_rtx_SET (d->target, x));
   23684              : 
   23685         7060 :   return true;
   23686              : }
   23687              : 
   23688              : /* A subroutine of ix86_expand_vec_perm_const_1.  Try to implement D
   23689              :    using insertps.  */
   23690              : static bool
   23691       296800 : expand_vec_perm_insertps (struct expand_vec_perm_d *d)
   23692              : {
   23693       296800 :   machine_mode vmode = d->vmode;
   23694       296800 :   unsigned i, cnt_s, nelt = d->nelt;
   23695       296800 :   int cnt_d = -1;
   23696       296800 :   rtx src, dst;
   23697              : 
   23698       296800 :   if (d->one_operand_p)
   23699              :     return false;
   23700              : 
   23701       270833 :   if (!(TARGET_SSE4_1
   23702        37318 :         && (vmode == V4SFmode || vmode == V4SImode
   23703        27169 :             || (TARGET_MMX_WITH_SSE
   23704        20821 :                 && (vmode == V2SFmode || vmode == V2SImode)))))
   23705              :     return false;
   23706              : 
   23707        54969 :   for (i = 0; i < nelt; ++i)
   23708              :     {
   23709        51939 :       if (d->perm[i] == i)
   23710        10489 :         continue;
   23711        41450 :       if (cnt_d != -1)
   23712              :         {
   23713              :           cnt_d = -1;
   23714              :           break;
   23715              :         }
   23716        22240 :       cnt_d = i;
   23717              :     }
   23718              : 
   23719        22240 :   if (cnt_d == -1)
   23720              :     {
   23721        43292 :       for (i = 0; i < nelt; ++i)
   23722              :         {
   23723        40672 :           if (d->perm[i] == i + nelt)
   23724         4872 :             continue;
   23725        35800 :           if (cnt_d != -1)
   23726              :             return false;
   23727        19210 :           cnt_d = i;
   23728              :         }
   23729              : 
   23730         2620 :       if (cnt_d == -1)
   23731              :         return false;
   23732              :     }
   23733              : 
   23734         5650 :   if (d->testing_p)
   23735              :     return true;
   23736              : 
   23737          570 :   gcc_assert (cnt_d != -1);
   23738              : 
   23739          570 :   cnt_s = d->perm[cnt_d];
   23740          570 :   if (cnt_s < nelt)
   23741              :     {
   23742          246 :       src = d->op0;
   23743          246 :       dst = d->op1;
   23744              :     }
   23745              :   else
   23746              :     {
   23747          324 :       cnt_s -= nelt;
   23748          324 :       src = d->op1;
   23749          324 :       dst = d->op0;
   23750              :      }
   23751          570 :   gcc_assert (cnt_s < nelt);
   23752              : 
   23753          570 :   rtx x = gen_sse4_1_insertps (vmode, d->target, dst, src,
   23754          570 :                                GEN_INT (cnt_s << 6 | cnt_d << 4));
   23755          570 :   emit_insn (x);
   23756              : 
   23757          570 :   return true;
   23758              : }
   23759              : 
   23760              : /* A subroutine of ix86_expand_vec_perm_const_1.  Try to implement D
   23761              :    in terms of blendp[sd] / pblendw / pblendvb / vpblendd.  */
   23762              : 
   23763              : static bool
   23764       317330 : expand_vec_perm_blend (struct expand_vec_perm_d *d)
   23765              : {
   23766       317330 :   machine_mode mmode, vmode = d->vmode;
   23767       317330 :   unsigned i, nelt = d->nelt;
   23768       317330 :   unsigned HOST_WIDE_INT mask;
   23769       317330 :   rtx target, op0, op1, maskop, x;
   23770       317330 :   rtx rperm[32], vperm;
   23771              : 
   23772       317330 :   if (d->one_operand_p)
   23773              :     return false;
   23774         6874 :   if (TARGET_AVX512F && GET_MODE_SIZE (vmode) == 64
   23775       292364 :       && (TARGET_AVX512BW
   23776          660 :           || GET_MODE_UNIT_SIZE (vmode) >= 4))
   23777              :     ;
   23778       303589 :   else if (TARGET_AVX2 && GET_MODE_SIZE (vmode) == 32)
   23779              :     ;
   23780       284961 :   else if (TARGET_AVX && (vmode == V4DFmode || vmode == V8SFmode))
   23781              :     ;
   23782       279498 :   else if (TARGET_SSE4_1
   23783       315936 :            && (GET_MODE_SIZE (vmode) == 16
   23784        30280 :                || (TARGET_MMX_WITH_SSE && GET_MODE_SIZE (vmode) == 8)
   23785         3699 :                || GET_MODE_SIZE (vmode) == 4))
   23786              :     ;
   23787              :   else
   23788              :     return false;
   23789              : 
   23790              :   /* This is a blend, not a permute.  Elements must stay in their
   23791              :      respective lanes.  */
   23792       100406 :   for (i = 0; i < nelt; ++i)
   23793              :     {
   23794        94713 :       unsigned e = d->perm[i];
   23795        94713 :       if (!(e == i || e == i + nelt))
   23796              :         return false;
   23797              :     }
   23798              : 
   23799         5693 :   if (d->testing_p)
   23800              :     return true;
   23801              : 
   23802              :   /* ??? Without SSE4.1, we could implement this with and/andn/or.  This
   23803              :      decision should be extracted elsewhere, so that we only try that
   23804              :      sequence once all budget==3 options have been tried.  */
   23805         2752 :   target = d->target;
   23806         2752 :   op0 = d->op0;
   23807         2752 :   op1 = d->op1;
   23808         2752 :   mask = 0;
   23809              : 
   23810         2752 :   switch (vmode)
   23811              :     {
   23812              :     case E_V8DFmode:
   23813              :     case E_V16SFmode:
   23814              :     case E_V4DFmode:
   23815              :     case E_V8SFmode:
   23816              :     case E_V2DFmode:
   23817              :     case E_V4SFmode:
   23818              :     case E_V2SFmode:
   23819              :     case E_V2HImode:
   23820              :     case E_V4HImode:
   23821              :     case E_V8HImode:
   23822              :     case E_V8SImode:
   23823              :     case E_V32HImode:
   23824              :     case E_V64QImode:
   23825              :     case E_V16SImode:
   23826              :     case E_V8DImode:
   23827        10595 :       for (i = 0; i < nelt; ++i)
   23828         9142 :         mask |= ((unsigned HOST_WIDE_INT) (d->perm[i] >= nelt)) << i;
   23829              :       break;
   23830              : 
   23831              :     case E_V2DImode:
   23832           18 :       for (i = 0; i < 2; ++i)
   23833           18 :         mask |= (d->perm[i] >= 2 ? 15 : 0) << (i * 4);
   23834            6 :       vmode = V8HImode;
   23835            6 :       goto do_subreg;
   23836              : 
   23837              :     case E_V2SImode:
   23838           24 :       for (i = 0; i < 2; ++i)
   23839           24 :         mask |= (d->perm[i] >= 2 ? 3 : 0) << (i * 2);
   23840            8 :       vmode = V4HImode;
   23841            8 :       goto do_subreg;
   23842              : 
   23843          879 :     case E_V4SImode:
   23844          879 :       if (TARGET_AVX2)
   23845              :         {
   23846              :           /* Use vpblendd instead of vpblendw.  */
   23847          225 :           for (i = 0; i < nelt; ++i)
   23848          180 :             mask |= ((unsigned HOST_WIDE_INT) (d->perm[i] >= nelt)) << i;
   23849              :           break;
   23850              :         }
   23851              :       else
   23852              :         {
   23853         4170 :           for (i = 0; i < 4; ++i)
   23854         5200 :             mask |= (d->perm[i] >= 4 ? 3 : 0) << (i * 2);
   23855          834 :           vmode = V8HImode;
   23856          834 :           goto do_subreg;
   23857              :         }
   23858              : 
   23859              :     case E_V16QImode:
   23860              :       /* See if bytes move in pairs so we can use pblendw with
   23861              :          an immediate argument, rather than pblendvb with a vector
   23862              :          argument.  */
   23863          102 :       for (i = 0; i < 16; i += 2)
   23864          100 :         if (d->perm[i] + 1 != d->perm[i + 1])
   23865              :           {
   23866          290 :           use_pblendvb:
   23867         3502 :             for (i = 0; i < nelt; ++i)
   23868         3212 :               rperm[i] = (d->perm[i] < nelt ? const0_rtx : constm1_rtx);
   23869              : 
   23870          290 :           finish_pblendvb:
   23871          291 :             vperm = gen_rtx_CONST_VECTOR (vmode, gen_rtvec_v (nelt, rperm));
   23872          291 :             vperm = force_reg (vmode, vperm);
   23873              : 
   23874          582 :             if (GET_MODE_SIZE (vmode) == 4)
   23875          135 :               emit_insn (gen_mmx_pblendvb_v4qi (target, op0, op1, vperm));
   23876          312 :             else if (GET_MODE_SIZE (vmode) == 8)
   23877           40 :               emit_insn (gen_mmx_pblendvb_v8qi (target, op0, op1, vperm));
   23878          232 :             else if (GET_MODE_SIZE (vmode) == 16)
   23879           83 :               emit_insn (gen_sse4_1_pblendvb (target, op0, op1, vperm));
   23880              :             else
   23881           33 :               emit_insn (gen_avx2_pblendvb (target, op0, op1, vperm));
   23882          291 :             if (target != d->target)
   23883            1 :               emit_move_insn (d->target, gen_lowpart (d->vmode, target));
   23884              :             return true;
   23885              :           }
   23886              : 
   23887           18 :       for (i = 0; i < 8; ++i)
   23888           16 :         mask |= (d->perm[i * 2] >= 16) << i;
   23889              :       vmode = V8HImode;
   23890              :       /* FALLTHRU */
   23891              : 
   23892          931 :     do_subreg:
   23893          931 :       target = gen_reg_rtx (vmode);
   23894          931 :       op0 = gen_lowpart (vmode, op0);
   23895          931 :       op1 = gen_lowpart (vmode, op1);
   23896          931 :       break;
   23897              : 
   23898              :     case E_V8QImode:
   23899           40 :       for (i = 0; i < 8; i += 2)
   23900           40 :         if (d->perm[i] + 1 != d->perm[i + 1])
   23901           40 :           goto use_pblendvb;
   23902              : 
   23903            0 :       for (i = 0; i < 4; ++i)
   23904            0 :         mask |= (d->perm[i * 2] >= 8) << i;
   23905            0 :       vmode = V4HImode;
   23906            0 :       goto do_subreg;
   23907              : 
   23908              :     case E_V4QImode:
   23909          153 :       for (i = 0; i < 4; i += 2)
   23910          150 :         if (d->perm[i] + 1 != d->perm[i + 1])
   23911          135 :           goto use_pblendvb;
   23912              : 
   23913            9 :       for (i = 0; i < 2; ++i)
   23914            6 :         mask |= (d->perm[i * 2] >= 4) << i;
   23915            3 :       vmode = V2HImode;
   23916            3 :       goto do_subreg;
   23917              : 
   23918              :     case E_V32QImode:
   23919              :       /* See if bytes move in pairs.  If not, vpblendvb must be used.  */
   23920          916 :       for (i = 0; i < 32; i += 2)
   23921          864 :         if (d->perm[i] + 1 != d->perm[i + 1])
   23922           32 :           goto use_pblendvb;
   23923              :       /* See if bytes move in quadruplets.  If yes, vpblendd
   23924              :          with immediate can be used.  */
   23925          468 :       for (i = 0; i < 32; i += 4)
   23926          416 :         if (d->perm[i] + 2 != d->perm[i + 2])
   23927              :           break;
   23928           52 :       if (i < 32)
   23929              :         {
   23930              :           /* See if bytes move the same in both lanes.  If yes,
   23931              :              vpblendw with immediate can be used.  */
   23932            0 :           for (i = 0; i < 16; i += 2)
   23933            0 :             if (d->perm[i] + 16 != d->perm[i + 16])
   23934            0 :               goto use_pblendvb;
   23935              : 
   23936              :           /* Use vpblendw.  */
   23937            0 :           for (i = 0; i < 16; ++i)
   23938            0 :             mask |= (d->perm[i * 2] >= 32) << i;
   23939            0 :           vmode = V16HImode;
   23940            0 :           goto do_subreg;
   23941              :         }
   23942              : 
   23943              :       /* Use vpblendd.  */
   23944          468 :       for (i = 0; i < 8; ++i)
   23945          416 :         mask |= (d->perm[i * 4] >= 32) << i;
   23946           52 :       vmode = V8SImode;
   23947           52 :       goto do_subreg;
   23948              : 
   23949              :     case E_V16HImode:
   23950              :       /* See if words move in pairs.  If yes, vpblendd can be used.  */
   23951          177 :       for (i = 0; i < 16; i += 2)
   23952          161 :         if (d->perm[i] + 1 != d->perm[i + 1])
   23953              :           break;
   23954           49 :       if (i < 16)
   23955              :         {
   23956              :           /* See if words move the same in both lanes.  If not,
   23957              :              vpblendvb must be used.  */
   23958          290 :           for (i = 0; i < 8; i++)
   23959          258 :             if (d->perm[i] + 8 != d->perm[i + 8])
   23960              :               {
   23961              :                 /* Use vpblendvb.  */
   23962           33 :                 for (i = 0; i < 32; ++i)
   23963           32 :                   rperm[i] = (d->perm[i / 2] < 16 ? const0_rtx : constm1_rtx);
   23964              : 
   23965            1 :                 vmode = V32QImode;
   23966            1 :                 nelt = 32;
   23967            1 :                 target = gen_reg_rtx (vmode);
   23968            1 :                 op0 = gen_lowpart (vmode, op0);
   23969            1 :                 op1 = gen_lowpart (vmode, op1);
   23970            1 :                 goto finish_pblendvb;
   23971              :               }
   23972              : 
   23973              :           /* Use vpblendw.  */
   23974          544 :           for (i = 0; i < 16; ++i)
   23975          512 :             mask |= (d->perm[i] >= 16) << i;
   23976              :           break;
   23977              :         }
   23978              : 
   23979              :       /* Use vpblendd.  */
   23980          144 :       for (i = 0; i < 8; ++i)
   23981          128 :         mask |= (d->perm[i * 2] >= 16) << i;
   23982           16 :       vmode = V8SImode;
   23983           16 :       goto do_subreg;
   23984              : 
   23985              :     case E_V4DImode:
   23986              :       /* Use vpblendd.  */
   23987           50 :       for (i = 0; i < 4; ++i)
   23988           59 :         mask |= (d->perm[i] >= 4 ? 3 : 0) << (i * 2);
   23989           10 :       vmode = V8SImode;
   23990           10 :       goto do_subreg;
   23991              : 
   23992            0 :     default:
   23993            0 :       gcc_unreachable ();
   23994              :     }
   23995              : 
   23996         2461 :   switch (vmode)
   23997              :     {
   23998              :     case E_V8DFmode:
   23999              :     case E_V8DImode:
   24000              :       mmode = QImode;
   24001              :       break;
   24002            1 :     case E_V16SFmode:
   24003            1 :     case E_V16SImode:
   24004            1 :       mmode = HImode;
   24005            1 :       break;
   24006            6 :     case E_V32HImode:
   24007            6 :       mmode = SImode;
   24008            6 :       break;
   24009            1 :     case E_V64QImode:
   24010            1 :       mmode = DImode;
   24011            1 :       break;
   24012              :     default:
   24013              :       mmode = VOIDmode;
   24014              :     }
   24015              : 
   24016              :   /* Canonicalize vec_merge.  */
   24017         2461 :   if (swap_commutative_operands_p (op1, op0)
   24018              :       /* Two operands have same precedence, then
   24019              :          first bit of mask select first operand.  */
   24020         2461 :       || (!swap_commutative_operands_p (op0, op1)
   24021         2461 :           && !(mask & 1)))
   24022              :     {
   24023         2454 :       unsigned n_elts = GET_MODE_NUNITS (vmode);
   24024         2454 :       std::swap (op0, op1);
   24025         2454 :       unsigned HOST_WIDE_INT mask_all = HOST_WIDE_INT_1U;
   24026         2454 :       if (n_elts == HOST_BITS_PER_WIDE_INT)
   24027              :         mask_all  = -1;
   24028              :       else
   24029         2453 :         mask_all = (HOST_WIDE_INT_1U << n_elts) - 1;
   24030         2454 :       mask = ~mask & mask_all;
   24031              :     }
   24032              : 
   24033         2461 :   if (mmode != VOIDmode)
   24034           17 :     maskop = force_reg (mmode, gen_int_mode (mask, mmode));
   24035              :   else
   24036         2444 :     maskop = GEN_INT (mask);
   24037              : 
   24038              :   /* This matches five different patterns with the different modes.  */
   24039         2461 :   x = gen_rtx_VEC_MERGE (vmode, op1, op0, maskop);
   24040         2461 :   x = gen_rtx_SET (target, x);
   24041         2461 :   emit_insn (x);
   24042         2461 :   if (target != d->target)
   24043          931 :     emit_move_insn (d->target, gen_lowpart (d->vmode, target));
   24044              : 
   24045              :   return true;
   24046              : }
   24047              : 
   24048              : /* A subroutine of ix86_expand_vec_perm_const_1.  Try to implement D
   24049              :    in terms of the variable form of vpermilps.
   24050              : 
   24051              :    Note that we will have already failed the immediate input vpermilps,
   24052              :    which requires that the high and low part shuffle be identical; the
   24053              :    variable form doesn't require that.  */
   24054              : 
   24055              : static bool
   24056       140739 : expand_vec_perm_vpermil (struct expand_vec_perm_d *d)
   24057              : {
   24058       140739 :   rtx rperm[8], vperm;
   24059       140739 :   unsigned i;
   24060              : 
   24061       140739 :   if (!TARGET_AVX || !d->one_operand_p
   24062        10984 :       || (d->vmode != V8SImode && d->vmode != V8SFmode))
   24063              :     return false;
   24064              : 
   24065              :   /* We can only permute within the 128-bit lane.  */
   24066        15665 :   for (i = 0; i < 8; ++i)
   24067              :     {
   24068        15077 :       unsigned e = d->perm[i];
   24069        15077 :       if (i < 4 ? e >= 4 : e < 4)
   24070              :         return false;
   24071              :     }
   24072              : 
   24073          588 :   if (d->testing_p)
   24074              :     return true;
   24075              : 
   24076          531 :   for (i = 0; i < 8; ++i)
   24077              :     {
   24078          472 :       unsigned e = d->perm[i];
   24079              : 
   24080              :       /* Within each 128-bit lane, the elements of op0 are numbered
   24081              :          from 0 and the elements of op1 are numbered from 4.  */
   24082          472 :       if (e >= 8 + 4)
   24083            0 :         e -= 8;
   24084          472 :       else if (e >= 4)
   24085          236 :         e -= 4;
   24086              : 
   24087          472 :       rperm[i] = GEN_INT (e);
   24088              :     }
   24089              : 
   24090           59 :   vperm = gen_rtx_CONST_VECTOR (V8SImode, gen_rtvec_v (8, rperm));
   24091           59 :   vperm = force_reg (V8SImode, vperm);
   24092           59 :   rtx target = d->target;
   24093           59 :   rtx op0 = d->op0;
   24094           59 :   if (d->vmode == V8SImode)
   24095              :     {
   24096            7 :       target = lowpart_subreg (V8SFmode, target, V8SImode);
   24097            7 :       op0 = lowpart_subreg (V8SFmode, op0, V8SImode);
   24098              :     }
   24099              : 
   24100           59 :   emit_insn (gen_avx_vpermilvarv8sf3 (target, op0, vperm));
   24101              : 
   24102           59 :   return true;
   24103              : }
   24104              : 
   24105              : /* For V*[QHS]Imode permutations, check if the same permutation
   24106              :    can't be performed in a 2x, 4x or 8x wider inner mode.  */
   24107              : 
   24108              : static bool
   24109       165687 : canonicalize_vector_int_perm (const struct expand_vec_perm_d *d,
   24110              :                               struct expand_vec_perm_d *nd)
   24111              : {
   24112       165687 :   int i;
   24113       165687 :   machine_mode mode = VOIDmode;
   24114              : 
   24115       165687 :   switch (d->vmode)
   24116              :     {
   24117              :     case E_V8QImode: mode = V4HImode; break;
   24118        30582 :     case E_V16QImode: mode = V8HImode; break;
   24119          708 :     case E_V32QImode: mode = V16HImode; break;
   24120          268 :     case E_V64QImode: mode = V32HImode; break;
   24121        12653 :     case E_V4HImode: mode = V2SImode; break;
   24122        21502 :     case E_V8HImode: mode = V4SImode; break;
   24123         1006 :     case E_V16HImode: mode = V8SImode; break;
   24124          397 :     case E_V32HImode: mode = V16SImode; break;
   24125        41805 :     case E_V4SImode: mode = V2DImode; break;
   24126         1429 :     case E_V8SImode: mode = V4DImode; break;
   24127           65 :     case E_V16SImode: mode = V8DImode; break;
   24128              :     default: return false;
   24129              :     }
   24130       210359 :   for (i = 0; i < d->nelt; i += 2)
   24131       195624 :     if ((d->perm[i] & 1) || d->perm[i + 1] != d->perm[i] + 1)
   24132              :       return false;
   24133        14735 :   nd->vmode = mode;
   24134        14735 :   nd->nelt = d->nelt / 2;
   24135        96159 :   for (i = 0; i < nd->nelt; i++)
   24136        81424 :     nd->perm[i] = d->perm[2 * i] / 2;
   24137        29470 :   if (GET_MODE_INNER (mode) != DImode)
   24138        12972 :     canonicalize_vector_int_perm (nd, nd);
   24139        14735 :   if (nd != d)
   24140              :     {
   24141         9391 :       nd->one_operand_p = d->one_operand_p;
   24142         9391 :       nd->testing_p = d->testing_p;
   24143         9391 :       if (d->op0 == d->op1)
   24144         2782 :         nd->op0 = nd->op1 = gen_lowpart (nd->vmode, d->op0);
   24145              :       else
   24146              :         {
   24147         6609 :           nd->op0 = gen_lowpart (nd->vmode, d->op0);
   24148         6609 :           nd->op1 = gen_lowpart (nd->vmode, d->op1);
   24149              :         }
   24150         9391 :       if (d->testing_p)
   24151         6089 :         nd->target = gen_raw_REG (nd->vmode, LAST_VIRTUAL_REGISTER + 1);
   24152              :       else
   24153         3302 :         nd->target = gen_reg_rtx (nd->vmode);
   24154              :     }
   24155              :   return true;
   24156              : }
   24157              : 
   24158              : /* Return true if permutation D can be performed as VMODE permutation
   24159              :    instead.  */
   24160              : 
   24161              : static bool
   24162         5930 : valid_perm_using_mode_p (machine_mode vmode, struct expand_vec_perm_d *d)
   24163              : {
   24164         5930 :   unsigned int i, j, chunk;
   24165              : 
   24166         5930 :   if (GET_MODE_CLASS (vmode) != MODE_VECTOR_INT
   24167         5930 :       || GET_MODE_CLASS (d->vmode) != MODE_VECTOR_INT
   24168        14502 :       || GET_MODE_SIZE (vmode) != GET_MODE_SIZE (d->vmode))
   24169              :     return false;
   24170              : 
   24171         8572 :   if (GET_MODE_NUNITS (vmode) >= d->nelt)
   24172              :     return true;
   24173              : 
   24174         4022 :   chunk = d->nelt / GET_MODE_NUNITS (vmode);
   24175         5216 :   for (i = 0; i < d->nelt; i += chunk)
   24176         4969 :     if (d->perm[i] & (chunk - 1))
   24177              :       return false;
   24178              :     else
   24179         7516 :       for (j = 1; j < chunk; ++j)
   24180         6322 :         if (d->perm[i] + j != d->perm[i + j])
   24181              :           return false;
   24182              : 
   24183              :   return true;
   24184              : }
   24185              : 
   24186              : /* A subroutine of ix86_expand_vec_perm_const_1.  Try to implement D
   24187              :    in terms of pshufb, vpperm, vpermq, vpermd, vpermps or vperm2i128.  */
   24188              : 
   24189              : static bool
   24190       140151 : expand_vec_perm_pshufb (struct expand_vec_perm_d *d)
   24191              : {
   24192       140151 :   unsigned i, nelt, eltsz, mask;
   24193       140151 :   unsigned char perm[64];
   24194       140151 :   machine_mode vmode;
   24195       140151 :   struct expand_vec_perm_d nd;
   24196       140151 :   rtx rperm[64], vperm, target, op0, op1;
   24197              : 
   24198       140151 :   nelt = d->nelt;
   24199              : 
   24200       140151 :   if (!d->one_operand_p)
   24201       229748 :     switch (GET_MODE_SIZE (d->vmode))
   24202              :       {
   24203         8471 :       case 4:
   24204         8471 :         if (!TARGET_XOP)
   24205              :           return false;
   24206              :         vmode = V4QImode;
   24207              :         break;
   24208              : 
   24209        21048 :       case 8:
   24210        21048 :         if (!TARGET_XOP)
   24211              :           return false;
   24212              :         vmode = V8QImode;
   24213              :         break;
   24214              : 
   24215        76463 :       case 16:
   24216        76463 :         if (!TARGET_XOP)
   24217              :           return false;
   24218              :         vmode = V16QImode;
   24219              :         break;
   24220              : 
   24221         7996 :       case 32:
   24222         7996 :         if (!TARGET_AVX2)
   24223              :           return false;
   24224              : 
   24225         3998 :         if (valid_perm_using_mode_p (V2TImode, d))
   24226              :           {
   24227           56 :             if (d->testing_p)
   24228              :               return true;
   24229              : 
   24230              :             /* Use vperm2i128 insn.  The pattern uses
   24231              :                V4DImode instead of V2TImode.  */
   24232           52 :             target = d->target;
   24233           52 :             if (d->vmode != V4DImode)
   24234           12 :               target = gen_reg_rtx (V4DImode);
   24235           52 :             op0 = gen_lowpart (V4DImode, d->op0);
   24236           52 :             op1 = gen_lowpart (V4DImode, d->op1);
   24237           52 :             rperm[0]
   24238           52 :               = GEN_INT ((d->perm[0] / (nelt / 2))
   24239              :                          | ((d->perm[nelt / 2] / (nelt / 2)) * 16));
   24240           52 :             emit_insn (gen_avx2_permv2ti (target, op0, op1, rperm[0]));
   24241           52 :             if (target != d->target)
   24242           12 :               emit_move_insn (d->target, gen_lowpart (d->vmode, target));
   24243              :             return true;
   24244              :           }
   24245              :         /* FALLTHRU */
   24246              : 
   24247              :       default:
   24248              :         return false;
   24249              :       }
   24250              :   else
   24251        50554 :     switch (GET_MODE_SIZE (d->vmode))
   24252              :       {
   24253         3716 :       case 4:
   24254         3716 :         if (!TARGET_SSSE3)
   24255              :           return false;
   24256              :         vmode = V4QImode;
   24257              :         break;
   24258              : 
   24259         2277 :       case 8:
   24260         2277 :         if (!TARGET_SSSE3)
   24261              :           return false;
   24262              :         vmode = V8QImode;
   24263              :         break;
   24264              : 
   24265        14381 :       case 16:
   24266        14381 :         if (!TARGET_SSSE3)
   24267              :           return false;
   24268              :         vmode = V16QImode;
   24269              :         break;
   24270              : 
   24271         4501 :       case 32:
   24272         4501 :         if (!TARGET_AVX2)
   24273              :           return false;
   24274              : 
   24275              :         /* V4DImode should be already handled through
   24276              :            expand_vselect by vpermq instruction.  */
   24277         1924 :         gcc_assert (d->vmode != V4DImode);
   24278              : 
   24279         1924 :         vmode = V32QImode;
   24280         1924 :         if (d->vmode == V8SImode
   24281         1559 :             || d->vmode == V16HImode
   24282         1343 :             || d->vmode == V32QImode)
   24283              :           {
   24284              :             /* First see if vpermq can be used for
   24285              :                V8SImode/V16HImode/V32QImode.  */
   24286          879 :             if (valid_perm_using_mode_p (V4DImode, d))
   24287              :               {
   24288          770 :                 for (i = 0; i < 4; i++)
   24289          616 :                   perm[i] = (d->perm[i * nelt / 4] * 4 / nelt) & 3;
   24290          154 :                 if (d->testing_p)
   24291              :                   return true;
   24292           58 :                 target = gen_reg_rtx (V4DImode);
   24293           58 :                 if (expand_vselect (target, gen_lowpart (V4DImode, d->op0),
   24294              :                                     perm, 4, false))
   24295              :                   {
   24296          116 :                     emit_move_insn (d->target,
   24297           58 :                                     gen_lowpart (d->vmode, target));
   24298           58 :                     return true;
   24299              :                   }
   24300              :                 return false;
   24301              :               }
   24302              : 
   24303              :             /* Next see if vpermd can be used.  */
   24304          725 :             if (valid_perm_using_mode_p (V8SImode, d))
   24305              :               vmode = V8SImode;
   24306              :           }
   24307              :         /* Or if vpermps can be used.  */
   24308         1045 :         else if (d->vmode == V8SFmode)
   24309              :           vmode = V8SImode;
   24310              : 
   24311              :         if (vmode == V32QImode)
   24312              :           {
   24313              :             /* vpshufb only works intra lanes, it is not
   24314              :                possible to shuffle bytes in between the lanes.  */
   24315         6473 :             for (i = 0; i < nelt; ++i)
   24316         6291 :               if ((d->perm[i] ^ i) & (nelt / 2))
   24317              :                 return false;
   24318              :           }
   24319              :         break;
   24320              : 
   24321          402 :       case 64:
   24322          402 :         if (!TARGET_AVX512BW)
   24323              :           return false;
   24324              : 
   24325              :         /* If vpermq didn't work, vpshufb won't work either.  */
   24326          204 :         if (d->vmode == V8DFmode || d->vmode == V8DImode)
   24327              :           return false;
   24328              : 
   24329          175 :         vmode = V64QImode;
   24330          175 :         if (d->vmode == V16SImode
   24331          150 :             || d->vmode == V32HImode
   24332           50 :             || d->vmode == V64QImode)
   24333              :           {
   24334              :             /* First see if vpermq can be used for
   24335              :                V16SImode/V32HImode/V64QImode.  */
   24336          164 :             if (valid_perm_using_mode_p (V8DImode, d))
   24337              :               {
   24338            0 :                 for (i = 0; i < 8; i++)
   24339            0 :                   perm[i] = (d->perm[i * nelt / 8] * 8 / nelt) & 7;
   24340            0 :                 if (d->testing_p)
   24341              :                   return true;
   24342            0 :                 target = gen_reg_rtx (V8DImode);
   24343            0 :                 if (expand_vselect (target, gen_lowpart (V8DImode, d->op0),
   24344              :                                     perm, 8, false))
   24345              :                   {
   24346            0 :                     emit_move_insn (d->target,
   24347            0 :                                     gen_lowpart (d->vmode, target));
   24348            0 :                     return true;
   24349              :                   }
   24350              :                 return false;
   24351              :               }
   24352              : 
   24353              :             /* Next see if vpermd can be used.  */
   24354          164 :             if (valid_perm_using_mode_p (V16SImode, d))
   24355              :               vmode = V16SImode;
   24356              :           }
   24357              :         /* Or if vpermps can be used.  */
   24358           11 :         else if (d->vmode == V16SFmode)
   24359              :           vmode = V16SImode;
   24360              : 
   24361              :         if (vmode == V64QImode)
   24362              :           {
   24363              :             /* vpshufb only works intra lanes, it is not
   24364              :                possible to shuffle bytes in between the lanes.  */
   24365          578 :             for (i = 0; i < nelt; ++i)
   24366          578 :               if ((d->perm[i] ^ i) & (3 * nelt / 4))
   24367              :                 return false;
   24368              :           }
   24369              :         break;
   24370              : 
   24371              :       default:
   24372              :         return false;
   24373              :       }
   24374              : 
   24375        11523 :   if (d->testing_p)
   24376              :     return true;
   24377              : 
   24378              :   /* Try to avoid variable permutation instruction.  */
   24379         8800 :   if (canonicalize_vector_int_perm (d, &nd) && expand_vec_perm_1 (&nd))
   24380              :     {
   24381         1839 :       emit_move_insn (d->target, gen_lowpart (d->vmode, nd.target));
   24382         1839 :       return true;
   24383              :     }
   24384              : 
   24385         6961 :   if (vmode == V8SImode)
   24386         8964 :     for (i = 0; i < 8; ++i)
   24387         7968 :       rperm[i] = GEN_INT ((d->perm[i * nelt / 8] * 8 / nelt) & 7);
   24388         5965 :   else if (vmode == V16SImode)
   24389          612 :     for (i = 0; i < 16; ++i)
   24390          576 :       rperm[i] = GEN_INT ((d->perm[i * nelt / 16] * 16 / nelt) & 15);
   24391              :   else
   24392              :     {
   24393         5929 :       eltsz = GET_MODE_UNIT_SIZE (d->vmode);
   24394         5929 :       if (!d->one_operand_p)
   24395         3212 :         mask = 2 * nelt - 1;
   24396         2717 :       else if (vmode == V64QImode)
   24397            0 :         mask = nelt / 4 - 1;
   24398         2717 :       else if (vmode == V32QImode)
   24399          176 :         mask = nelt / 2 - 1;
   24400              :       else
   24401         2541 :         mask = nelt - 1;
   24402              : 
   24403        59309 :       for (i = 0; i < nelt; ++i)
   24404              :         {
   24405        53380 :           unsigned j, e = d->perm[i] & mask;
   24406       148712 :           for (j = 0; j < eltsz; ++j)
   24407        95332 :             rperm[i * eltsz + j] = GEN_INT (e * eltsz + j);
   24408              :         }
   24409              :     }
   24410              : 
   24411         6961 :   machine_mode vpmode = vmode;
   24412              : 
   24413         6961 :   nelt = GET_MODE_SIZE (vmode);
   24414              : 
   24415              :   /* Emulate narrow modes with V16QI instructions.  */
   24416         6961 :   if (nelt < 16)
   24417              :     {
   24418          222 :       rtx m128 = GEN_INT (-128);
   24419              : 
   24420              :       /* Remap elements from the second operand, as we have to
   24421              :          account for inactive top elements from the first operand.  */
   24422          222 :       if (!d->one_operand_p)
   24423              :         {
   24424          243 :           for (i = 0; i < nelt; ++i)
   24425              :             {
   24426          216 :               unsigned ival = UINTVAL (rperm[i]);
   24427          216 :               if (ival >= nelt)
   24428          108 :                 rperm[i] = GEN_INT (ival + 16 - nelt);
   24429              :             }
   24430              :         }
   24431              : 
   24432              :       /* Fill inactive elements in the top positions with zeros.  */
   24433         2570 :       for (i = nelt; i < 16; ++i)
   24434         2348 :         rperm[i] = m128;
   24435              : 
   24436              :       vpmode = V16QImode;
   24437              :     }
   24438              : 
   24439        13922 :   vperm = gen_rtx_CONST_VECTOR (vpmode,
   24440         6961 :                                 gen_rtvec_v (GET_MODE_NUNITS (vpmode), rperm));
   24441         6961 :   vperm = force_reg (vpmode, vperm);
   24442              : 
   24443         6961 :   if (vmode == d->vmode)
   24444         2415 :     target = d->target;
   24445              :   else
   24446         4546 :     target = gen_reg_rtx (vmode);
   24447              : 
   24448         6961 :   op0 = gen_lowpart (vmode, d->op0);
   24449              : 
   24450         6961 :   if (d->one_operand_p)
   24451              :     {
   24452         3749 :       rtx (*gen) (rtx, rtx, rtx);
   24453              : 
   24454         3749 :       if (vmode == V4QImode)
   24455              :         gen = gen_mmx_pshufbv4qi3;
   24456              :       else if (vmode == V8QImode)
   24457              :         gen = gen_mmx_pshufbv8qi3;
   24458              :       else if (vmode == V16QImode)
   24459              :         gen = gen_ssse3_pshufbv16qi3;
   24460              :       else if (vmode == V32QImode)
   24461              :         gen = gen_avx2_pshufbv32qi3;
   24462              :       else if (vmode == V64QImode)
   24463              :         gen = gen_avx512bw_pshufbv64qi3;
   24464              :       else if (vmode == V8SFmode)
   24465              :         gen = gen_avx2_permvarv8sf;
   24466              :       else if (vmode == V8SImode)
   24467              :         gen = gen_avx2_permvarv8si;
   24468              :       else if (vmode == V16SFmode)
   24469              :         gen = gen_avx512f_permvarv16sf;
   24470              :       else if (vmode == V16SImode)
   24471              :         gen = gen_avx512f_permvarv16si;
   24472              :       else
   24473              :         gcc_unreachable ();
   24474              : 
   24475         3749 :       emit_insn (gen (target, op0, vperm));
   24476              :     }
   24477              :   else
   24478              :     {
   24479         3212 :       rtx (*gen) (rtx, rtx, rtx, rtx);
   24480              : 
   24481         3212 :       op1 = gen_lowpart (vmode, d->op1);
   24482              : 
   24483         3212 :       if (vmode == V4QImode)
   24484              :         gen = gen_mmx_ppermv32;
   24485              :       else if (vmode == V8QImode)
   24486              :         gen = gen_mmx_ppermv64;
   24487              :       else if (vmode == V16QImode)
   24488              :         gen = gen_xop_pperm;
   24489              :       else
   24490            0 :         gcc_unreachable ();
   24491              : 
   24492         3212 :       emit_insn (gen (target, op0, op1, vperm));
   24493              :     }
   24494              : 
   24495         6961 :   if (target != d->target)
   24496         4546 :     emit_move_insn (d->target, gen_lowpart (d->vmode, target));
   24497              : 
   24498              :   return true;
   24499              : }
   24500              : 
   24501              : /* Try to expand one-operand permutation with constant mask.  */
   24502              : 
   24503              : static bool
   24504       128294 : ix86_expand_vec_one_operand_perm_avx512 (struct expand_vec_perm_d *d)
   24505              : {
   24506       128294 :   machine_mode mode = GET_MODE (d->op0);
   24507       128294 :   machine_mode maskmode = mode;
   24508       256588 :   unsigned inner_size = GET_MODE_SIZE (GET_MODE_INNER (mode));
   24509       128294 :   rtx (*gen) (rtx, rtx, rtx) = NULL;
   24510       128294 :   rtx target, op0, mask;
   24511       128294 :   rtx vec[64];
   24512              : 
   24513       128294 :   if (!rtx_equal_p (d->op0, d->op1))
   24514              :     return false;
   24515              : 
   24516        17927 :   if (!TARGET_AVX512F)
   24517              :     return false;
   24518              : 
   24519              :   /* Accept VNxHImode and VNxQImode now.  */
   24520          744 :   if (!TARGET_AVX512VL && GET_MODE_SIZE (mode) < 64)
   24521              :     return false;
   24522              : 
   24523              :   /* vpermw.  */
   24524          465 :   if (!TARGET_AVX512BW && inner_size == 2)
   24525              :     return false;
   24526              : 
   24527              :   /* vpermb.  */
   24528          330 :   if (!TARGET_AVX512VBMI && inner_size == 1)
   24529              :     return false;
   24530              : 
   24531          211 :   switch (mode)
   24532              :     {
   24533              :     case E_V16SImode:
   24534              :       gen = gen_avx512f_permvarv16si;
   24535              :       break;
   24536            5 :     case E_V16SFmode:
   24537            5 :       gen = gen_avx512f_permvarv16sf;
   24538            5 :       maskmode = V16SImode;
   24539            5 :       break;
   24540            1 :     case E_V8DImode:
   24541            1 :       gen = gen_avx512f_permvarv8di;
   24542            1 :       break;
   24543           30 :     case E_V8DFmode:
   24544           30 :       gen = gen_avx512f_permvarv8df;
   24545           30 :       maskmode = V8DImode;
   24546           30 :       break;
   24547          106 :     case E_V32HImode:
   24548          106 :       gen = gen_avx512bw_permvarv32hi;
   24549          106 :       break;
   24550           14 :     case E_V16HImode:
   24551           14 :       gen = gen_avx512vl_permvarv16hi;
   24552           14 :       break;
   24553            6 :     case E_V8HImode:
   24554            6 :       gen = gen_avx512vl_permvarv8hi;
   24555            6 :       break;
   24556            4 :     case E_V64QImode:
   24557            4 :       gen = gen_avx512bw_permvarv64qi;
   24558            4 :       break;
   24559            2 :     case E_V32QImode:
   24560            2 :       gen = gen_avx512vl_permvarv32qi;
   24561            2 :       break;
   24562            0 :     case E_V16QImode:
   24563            0 :       gen = gen_avx512vl_permvarv16qi;
   24564            0 :       break;
   24565              : 
   24566              :     default:
   24567              :       return false;
   24568              :     }
   24569              : 
   24570          211 :   if (d->testing_p)
   24571              :     return true;
   24572              : 
   24573          201 :   target = d->target;
   24574          201 :   op0 = d->op0;
   24575         5041 :   for (int i = 0; i < d->nelt; ++i)
   24576         4840 :     vec[i] = GEN_INT (d->perm[i]);
   24577          201 :   mask = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (d->nelt, vec));
   24578          201 :   emit_insn (gen (target, op0, force_reg (maskmode, mask)));
   24579          201 :   return true;
   24580              : }
   24581              : 
   24582              : static bool expand_vec_perm_palignr (struct expand_vec_perm_d *d, bool);
   24583              : 
   24584              : /* A subroutine of ix86_expand_vec_perm_const_1.  Try to instantiate D
   24585              :    in a single instruction.  */
   24586              : 
   24587              : static bool
   24588       352398 : expand_vec_perm_1 (struct expand_vec_perm_d *d)
   24589              : {
   24590       352398 :   unsigned i, nelt = d->nelt;
   24591       352398 :   struct expand_vec_perm_d nd;
   24592              : 
   24593              :   /* Check plain VEC_SELECT first, because AVX has instructions that could
   24594              :      match both SEL and SEL+CONCAT, but the plain SEL will allow a memory
   24595              :      input where SEL+CONCAT may not.  */
   24596       352398 :   if (d->one_operand_p)
   24597              :     {
   24598              :       int mask = nelt - 1;
   24599              :       bool identity_perm = true;
   24600              :       bool broadcast_perm = true;
   24601              : 
   24602       520044 :       for (i = 0; i < nelt; i++)
   24603              :         {
   24604       456528 :           nd.perm[i] = d->perm[i] & mask;
   24605       456528 :           if (nd.perm[i] != i)
   24606       349876 :             identity_perm = false;
   24607       456528 :           if (nd.perm[i])
   24608       376998 :             broadcast_perm = false;
   24609              :         }
   24610              : 
   24611        63516 :       if (identity_perm)
   24612              :         {
   24613           11 :           if (!d->testing_p)
   24614            5 :             emit_move_insn (d->target, d->op0);
   24615              :           return true;
   24616              :         }
   24617        63505 :       else if (broadcast_perm && TARGET_AVX2)
   24618              :         {
   24619              :           /* Use vpbroadcast{b,w,d}.  */
   24620          381 :           rtx (*gen) (rtx, rtx) = NULL;
   24621          381 :           switch (d->vmode)
   24622              :             {
   24623            1 :             case E_V64QImode:
   24624            1 :               if (TARGET_AVX512BW)
   24625              :                 gen = gen_avx512bw_vec_dupv64qi_1;
   24626              :               break;
   24627            4 :             case E_V32QImode:
   24628            4 :               gen = gen_avx2_pbroadcastv32qi_1;
   24629            4 :               break;
   24630            1 :             case E_V32HImode:
   24631            1 :               if (TARGET_AVX512BW)
   24632              :                 gen = gen_avx512bw_vec_dupv32hi_1;
   24633              :               break;
   24634            4 :             case E_V16HImode:
   24635            4 :               gen = gen_avx2_pbroadcastv16hi_1;
   24636            4 :               break;
   24637            1 :             case E_V16SImode:
   24638            1 :               if (TARGET_AVX512F)
   24639              :                 gen = gen_avx512f_vec_dupv16si_1;
   24640              :               break;
   24641            4 :             case E_V8SImode:
   24642            4 :               gen = gen_avx2_pbroadcastv8si_1;
   24643            4 :               break;
   24644            4 :             case E_V16QImode:
   24645            4 :               gen = gen_avx2_pbroadcastv16qi;
   24646            4 :               break;
   24647            5 :             case E_V8HImode:
   24648            5 :               gen = gen_avx2_pbroadcastv8hi;
   24649            5 :               break;
   24650            0 :             case E_V16SFmode:
   24651            0 :               if (TARGET_AVX512F)
   24652              :                 gen = gen_avx512f_vec_dupv16sf_1;
   24653              :               break;
   24654              :             case E_V8SFmode:
   24655              :               gen = gen_avx2_vec_dupv8sf_1;
   24656              :               break;
   24657            0 :             case E_V8DFmode:
   24658            0 :               if (TARGET_AVX512F)
   24659              :                 gen = gen_avx512f_vec_dupv8df_1;
   24660              :               break;
   24661            0 :             case E_V8DImode:
   24662            0 :               if (TARGET_AVX512F)
   24663              :                 gen = gen_avx512f_vec_dupv8di_1;
   24664              :               break;
   24665              :             /* For other modes prefer other shuffles this function creates.  */
   24666              :             default: break;
   24667              :             }
   24668           21 :           if (gen != NULL)
   24669              :             {
   24670           24 :               if (!d->testing_p)
   24671           24 :                 emit_insn (gen (d->target, d->op0));
   24672              :               return true;
   24673              :             }
   24674              :         }
   24675              : 
   24676        63481 :       if (expand_vselect (d->target, d->op0, nd.perm, nelt, d->testing_p))
   24677              :         return true;
   24678              : 
   24679              :       /* There are plenty of patterns in sse.md that are written for
   24680              :          SEL+CONCAT and are not replicated for a single op.  Perhaps
   24681              :          that should be changed, to avoid the nastiness here.  */
   24682              : 
   24683              :       /* Recognize interleave style patterns, which means incrementing
   24684              :          every other permutation operand.  */
   24685       200489 :       for (i = 0; i < nelt; i += 2)
   24686              :         {
   24687       164499 :           nd.perm[i] = d->perm[i] & mask;
   24688       164499 :           nd.perm[i + 1] = (d->perm[i + 1] & mask) + nelt;
   24689              :         }
   24690        35990 :       if (expand_vselect_vconcat (d->target, d->op0, d->op0, nd.perm, nelt,
   24691              :                                   d->testing_p))
   24692              :         return true;
   24693              : 
   24694              :       /* Recognize shufps, which means adding {0, 0, nelt, nelt}.  */
   24695        31768 :       if (nelt >= 4)
   24696              :         {
   24697       108349 :           for (i = 0; i < nelt; i += 4)
   24698              :             {
   24699        76581 :               nd.perm[i + 0] = d->perm[i + 0] & mask;
   24700        76581 :               nd.perm[i + 1] = d->perm[i + 1] & mask;
   24701        76581 :               nd.perm[i + 2] = (d->perm[i + 2] & mask) + nelt;
   24702        76581 :               nd.perm[i + 3] = (d->perm[i + 3] & mask) + nelt;
   24703              :             }
   24704              : 
   24705        31768 :           if (expand_vselect_vconcat (d->target, d->op0, d->op0, nd.perm, nelt,
   24706              :                                       d->testing_p))
   24707              :             return true;
   24708              :         }
   24709              :     }
   24710              : 
   24711              :   /* Try the SSE4.1 blend variable merge instructions.  */
   24712       314849 :   if (expand_vec_perm_blend (d))
   24713              :     return true;
   24714              : 
   24715              :   /* Try movss/movsd instructions.  */
   24716       311637 :   if (expand_vec_perm_movs (d))
   24717              :     return true;
   24718              : 
   24719              :   /* Try the SSE4.1 insertps instruction.  */
   24720       296800 :   if (expand_vec_perm_insertps (d))
   24721              :     return true;
   24722              : 
   24723              :   /* Try the fully general two operand permute.  */
   24724       291150 :   if (expand_vselect_vconcat (d->target, d->op0, d->op1, d->perm, nelt,
   24725              :                               d->testing_p))
   24726              :     return true;
   24727              : 
   24728              :   /* Recognize interleave style patterns with reversed operands.  */
   24729       140894 :   if (!d->one_operand_p)
   24730              :     {
   24731       915531 :       for (i = 0; i < nelt; ++i)
   24732              :         {
   24733       800502 :           unsigned e = d->perm[i];
   24734       800502 :           if (e >= nelt)
   24735       391744 :             e -= nelt;
   24736              :           else
   24737       408758 :             e += nelt;
   24738       800502 :           nd.perm[i] = e;
   24739              :         }
   24740              : 
   24741       115029 :       if (expand_vselect_vconcat (d->target, d->op1, d->op0, nd.perm, nelt,
   24742              :                                   d->testing_p))
   24743              :         return true;
   24744              :     }
   24745              : 
   24746              :   /* Try one of the AVX vpermil variable permutations.  */
   24747       140739 :   if (expand_vec_perm_vpermil (d))
   24748              :     return true;
   24749              : 
   24750              :   /* Try the SSSE3 pshufb or XOP vpperm or AVX2 vperm2i128,
   24751              :      vpshufb, vpermd, vpermps or vpermq variable permutation.  */
   24752       140151 :   if (expand_vec_perm_pshufb (d))
   24753              :     return true;
   24754              : 
   24755              :   /* Try the AVX2 vpalignr instruction.  */
   24756       128418 :   if (expand_vec_perm_palignr (d, true))
   24757              :     return true;
   24758              : 
   24759              :   /* Try the AVX512F vperm{w,b,s,d} instructions  */
   24760       128294 :   if (ix86_expand_vec_one_operand_perm_avx512 (d))
   24761              :     return true;
   24762              : 
   24763              :   /* Try the AVX512F vpermt2/vpermi2 instructions.  */
   24764       128083 :   if (ix86_expand_vec_perm_vpermt2 (NULL_RTX, NULL_RTX, NULL_RTX, NULL_RTX, d))
   24765              :     return true;
   24766              : 
   24767              :   /* See if we can get the same permutation in different vector integer
   24768              :      mode.  */
   24769       127188 :   if (canonicalize_vector_int_perm (d, &nd) && expand_vec_perm_1 (&nd))
   24770              :     {
   24771         6850 :       if (!d->testing_p)
   24772         1242 :         emit_move_insn (d->target, gen_lowpart (d->vmode, nd.target));
   24773              :       return true;
   24774              :     }
   24775              :   return false;
   24776              : }
   24777              : 
   24778              : /* Canonicalize vec_perm index to make the first index
   24779              :    always comes from the first vector.  */
   24780              : static void
   24781         8209 : ix86_vec_perm_index_canon (struct expand_vec_perm_d *d)
   24782              : {
   24783         8209 :   unsigned nelt = d->nelt;
   24784         8209 :   if (d->perm[0] < nelt)
   24785              :     return;
   24786              : 
   24787            5 :   for (unsigned i = 0; i != nelt; i++)
   24788            4 :     d->perm[i] = (d->perm[i] + nelt) % (2 * nelt);
   24789              : 
   24790            1 :   std::swap (d->op0, d->op1);
   24791            1 :   return;
   24792              : }
   24793              : 
   24794              : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
   24795              :    in terms of a pair of shufps+ shufps/pshufd instructions.  */
   24796              : static bool
   24797        88420 : expand_vec_perm_shufps_shufps (struct expand_vec_perm_d *d)
   24798              : {
   24799        88420 :   unsigned char perm1[4];
   24800        88420 :   machine_mode vmode = d->vmode;
   24801        88420 :   bool ok;
   24802        88420 :   unsigned i, j, k, count = 0;
   24803              : 
   24804        88420 :   if (d->one_operand_p
   24805        82775 :       || (vmode != V4SImode && vmode != V4SFmode))
   24806              :     return false;
   24807              : 
   24808        36292 :   if (d->testing_p)
   24809              :     return true;
   24810              : 
   24811         8209 :   ix86_vec_perm_index_canon (d);
   24812        49254 :   for (i = 0; i < 4; ++i)
   24813        51356 :     count += d->perm[i] > 3 ? 1 : 0;
   24814              : 
   24815         8209 :   gcc_assert (count & 3);
   24816              : 
   24817         8209 :   rtx tmp = gen_reg_rtx (vmode);
   24818              :   /* 2 from op0 and 2 from op1.  */
   24819         8209 :   if (count == 2)
   24820              :     {
   24821              :       unsigned char perm2[4];
   24822        18515 :       for (i = 0, j = 0, k = 2; i < 4; ++i)
   24823        14812 :         if (d->perm[i] & 4)
   24824              :           {
   24825         7406 :             perm1[k++] = d->perm[i];
   24826         7406 :             perm2[i] = k - 1;
   24827              :           }
   24828              :         else
   24829              :           {
   24830         7406 :             perm1[j++] = d->perm[i];
   24831         7406 :             perm2[i] = j - 1;
   24832              :           }
   24833              : 
   24834              :       /* shufps.  */
   24835         7406 :       ok = expand_vselect_vconcat (tmp, d->op0, d->op1,
   24836         3703 :                                   perm1, d->nelt, false);
   24837         3703 :       gcc_assert (ok);
   24838         3703 :       if (vmode == V4SImode && TARGET_SSE2)
   24839              :       /* pshufd.  */
   24840         2138 :         ok = expand_vselect (d->target, tmp,
   24841         2138 :                              perm2, d->nelt, false);
   24842              :       else
   24843              :         {
   24844              :           /* shufps.  */
   24845         1565 :           perm2[2] += 4;
   24846         1565 :           perm2[3] += 4;
   24847         1565 :           ok = expand_vselect_vconcat (d->target, tmp, tmp,
   24848         1565 :                                        perm2, d->nelt, false);
   24849              :         }
   24850         3703 :       gcc_assert (ok);
   24851              :     }
   24852              :   /* 3 from one op and 1 from another.  */
   24853              :   else
   24854              :     {
   24855        22530 :       unsigned pair_idx = 8, lone_idx = 8, shift;
   24856              : 
   24857              :       /* Find the lone index.  */
   24858        22530 :       for (i = 0; i < 4; ++i)
   24859        18024 :         if ((d->perm[i] > 3 && count == 1)
   24860        14720 :             || (d->perm[i] < 4 && count == 3))
   24861        18024 :           lone_idx = i;
   24862              : 
   24863              :       /* When lone_idx is not 0, it must from second op(count == 1).  */
   24864         5708 :       gcc_assert (count == (lone_idx ? 1 : 3));
   24865              : 
   24866              :       /* Find the pair index that sits in the same half as the lone index.  */
   24867         4506 :       shift = lone_idx & 2;
   24868         4506 :       pair_idx = 1 - lone_idx + 2 * shift;
   24869              : 
   24870              :       /* First permutate lone index and pair index into the same vector as
   24871              :          [ lone, lone, pair, pair ].  */
   24872         9012 :       perm1[1] = perm1[0]
   24873         4506 :         = (count == 3) ? d->perm[lone_idx] : d->perm[lone_idx] - 4;
   24874         9012 :       perm1[3] = perm1[2]
   24875         4506 :         = (count == 3) ? d->perm[pair_idx] : d->perm[pair_idx] + 4;
   24876              : 
   24877              :       /* Always put the vector contains lone indx at the first.  */
   24878         4506 :       if (count == 1)
   24879         3304 :         std::swap (d->op0, d->op1);
   24880              : 
   24881              :       /* shufps.  */
   24882         9012 :       ok = expand_vselect_vconcat (tmp, d->op0, d->op1,
   24883         4506 :                                    perm1, d->nelt, false);
   24884         4506 :       gcc_assert (ok);
   24885              : 
   24886              :       /* Refine lone and pair index to original order.  */
   24887         4506 :       perm1[shift] = lone_idx << 1;
   24888         4506 :       perm1[shift + 1] = pair_idx << 1;
   24889              : 
   24890              :       /* Select the remaining 2 elements in another vector.  */
   24891        13518 :       for (i = 2 - shift; i < 4 - shift; ++i)
   24892         9012 :         perm1[i] = lone_idx == 1 ? d->perm[i] + 4 : d->perm[i];
   24893              : 
   24894              :       /* Adjust to original selector.  */
   24895         4506 :       if (lone_idx > 1)
   24896         2232 :         std::swap (tmp, d->op1);
   24897              : 
   24898              :       /* shufps.  */
   24899         9012 :       ok = expand_vselect_vconcat (d->target, tmp, d->op1,
   24900         4506 :                                    perm1, d->nelt, false);
   24901              : 
   24902         4506 :       gcc_assert (ok);
   24903              :     }
   24904              : 
   24905              :   return true;
   24906              : }
   24907              : 
   24908              : /* A subroutine of ix86_expand_vec_perm_const_1.  Try to implement D
   24909              :    in terms of a pair of pshuflw + pshufhw instructions.  */
   24910              : 
   24911              : static bool
   24912       105328 : expand_vec_perm_pshuflw_pshufhw (struct expand_vec_perm_d *d)
   24913              : {
   24914       105328 :   unsigned char perm2[MAX_VECT_LEN];
   24915       105328 :   unsigned i;
   24916       105328 :   bool ok;
   24917              : 
   24918       105328 :   if (d->vmode != V8HImode || !d->one_operand_p)
   24919              :     return false;
   24920              : 
   24921              :   /* The two permutations only operate in 64-bit lanes.  */
   24922        12993 :   for (i = 0; i < 4; ++i)
   24923        10488 :     if (d->perm[i] >= 4)
   24924              :       return false;
   24925        12469 :   for (i = 4; i < 8; ++i)
   24926         9978 :     if (d->perm[i] < 4)
   24927              :       return false;
   24928              : 
   24929         2491 :   if (d->testing_p)
   24930              :     return true;
   24931              : 
   24932              :   /* Emit the pshuflw.  */
   24933          136 :   memcpy (perm2, d->perm, 4);
   24934          680 :   for (i = 4; i < 8; ++i)
   24935          544 :     perm2[i] = i;
   24936          136 :   ok = expand_vselect (d->target, d->op0, perm2, 8, d->testing_p);
   24937          136 :   gcc_assert (ok);
   24938              : 
   24939              :   /* Emit the pshufhw.  */
   24940          136 :   memcpy (perm2 + 4, d->perm + 4, 4);
   24941          680 :   for (i = 0; i < 4; ++i)
   24942          544 :     perm2[i] = i;
   24943          136 :   ok = expand_vselect (d->target, d->target, perm2, 8, d->testing_p);
   24944          136 :   gcc_assert (ok);
   24945              : 
   24946              :   return true;
   24947              : }
   24948              : 
   24949              : /* Try to permute 2 64-bit vectors by punpckldq + 128-bit vector shuffle.  */
   24950              : static bool
   24951        52128 : expand_vec_perm_punpckldq_pshuf (struct expand_vec_perm_d *d)
   24952              : {
   24953        52128 :   if (GET_MODE_BITSIZE (d->vmode) != 64
   24954        17524 :       || !TARGET_MMX_WITH_SSE
   24955        69652 :       || d->one_operand_p)
   24956              :     return false;
   24957              : 
   24958        16030 :   machine_mode widen_vmode;
   24959        16030 :   switch (d->vmode)
   24960              :     {
   24961              :     /* pshufd.  */
   24962              :     case E_V2SImode:
   24963              :       widen_vmode = V4SImode;
   24964              :       break;
   24965              : 
   24966              :     /* pshufd.  */
   24967         1501 :     case E_V2SFmode:
   24968         1501 :       widen_vmode = V4SFmode;
   24969         1501 :       break;
   24970              : 
   24971         5293 :     case E_V4HImode:
   24972         5293 :       widen_vmode = V8HImode;
   24973              :       /* pshufb.  */
   24974         5293 :       if (!TARGET_SSSE3)
   24975              :         return false;
   24976              :       break;
   24977              : 
   24978         6292 :     case E_V8QImode:
   24979              :       /* pshufb.  */
   24980         6292 :       widen_vmode = V16QImode;
   24981         6292 :       if (!TARGET_SSSE3)
   24982              :         return false;
   24983              :       break;
   24984              : 
   24985              :     default:
   24986              :       return false;
   24987              :     }
   24988              : 
   24989         6544 :   if (d->testing_p)
   24990              :     return true;
   24991              : 
   24992          403 :   struct expand_vec_perm_d dperm;
   24993          403 :   dperm.target = gen_reg_rtx (widen_vmode);
   24994          403 :   rtx op0 = gen_reg_rtx (widen_vmode);
   24995          403 :   emit_move_insn (op0, gen_rtx_VEC_CONCAT (widen_vmode, d->op0, d->op1));
   24996          403 :   dperm.op0 = op0;
   24997          403 :   dperm.op1 = op0;
   24998          403 :   dperm.vmode = widen_vmode;
   24999          403 :   unsigned nelt = GET_MODE_NUNITS (widen_vmode);
   25000          403 :   dperm.nelt = nelt;
   25001          403 :   dperm.one_operand_p = true;
   25002          403 :   dperm.testing_p = false;
   25003              : 
   25004         2143 :   for (unsigned i = 0; i != nelt / 2; i++)
   25005              :     {
   25006         1740 :       dperm.perm[i] = d->perm[i];
   25007         1740 :       dperm.perm[i + nelt / 2] = d->perm[i];
   25008              :     }
   25009              : 
   25010          403 :   gcc_assert (expand_vec_perm_1 (&dperm));
   25011          403 :   emit_move_insn (d->target, lowpart_subreg (d->vmode,
   25012              :                                              dperm.target,
   25013              :                                              dperm.vmode));
   25014          403 :   return true;
   25015              : }
   25016              : 
   25017              : /* A subroutine of ix86_expand_vec_perm_const_1.  Try to simplify
   25018              :    the permutation using the SSSE3 palignr instruction.  This succeeds
   25019              :    when all of the elements in PERM fit within one vector and we merely
   25020              :    need to shift them down so that a single vector permutation has a
   25021              :    chance to succeed.  If SINGLE_INSN_ONLY_P, succeed if only
   25022              :    the vpalignr instruction itself can perform the requested permutation.  */
   25023              : 
   25024              : static bool
   25025       231255 : expand_vec_perm_palignr (struct expand_vec_perm_d *d, bool single_insn_only_p)
   25026              : {
   25027       231255 :   unsigned i, nelt = d->nelt;
   25028       231255 :   unsigned min, max, minswap, maxswap;
   25029       231255 :   bool in_order, ok, swap = false;
   25030       231255 :   rtx shift, target;
   25031       231255 :   struct expand_vec_perm_d dcopy;
   25032              : 
   25033              :   /* Even with AVX, palignr only operates on 128-bit vectors,
   25034              :      in AVX2 palignr operates on both 128-bit lanes.  */
   25035       117286 :   if ((!TARGET_SSSE3 || GET_MODE_SIZE (d->vmode) != 16)
   25036       273623 :       && (!TARGET_AVX2 || GET_MODE_SIZE (d->vmode) != 32))
   25037              :     return false;
   25038              : 
   25039        34402 :   min = 2 * nelt;
   25040        34402 :   max = 0;
   25041        34402 :   minswap = 2 * nelt;
   25042        34402 :   maxswap = 0;
   25043       238478 :   for (i = 0; i < nelt; ++i)
   25044              :     {
   25045       204076 :       unsigned e = d->perm[i];
   25046       204076 :       unsigned eswap = d->perm[i] ^ nelt;
   25047       408152 :       if (GET_MODE_SIZE (d->vmode) == 32)
   25048              :         {
   25049        69192 :           e = (e & ((nelt / 2) - 1)) | ((e & nelt) >> 1);
   25050        69192 :           eswap = e ^ (nelt / 2);
   25051              :         }
   25052       204076 :       if (e < min)
   25053              :         min = e;
   25054       204076 :       if (e > max)
   25055              :         max = e;
   25056       204076 :       if (eswap < minswap)
   25057              :         minswap = eswap;
   25058       204076 :       if (eswap > maxswap)
   25059              :         maxswap = eswap;
   25060              :     }
   25061        34402 :   if (min == 0
   25062        50666 :       || max - min >= (GET_MODE_SIZE (d->vmode) == 32 ? nelt / 2 : nelt))
   25063              :     {
   25064        31202 :       if (d->one_operand_p
   25065        30933 :           || minswap == 0
   25066        66072 :           || maxswap - minswap >= (GET_MODE_SIZE (d->vmode) == 32
   25067        17435 :                                    ? nelt / 2 : nelt))
   25068              :         return false;
   25069              :       swap = true;
   25070              :       min = minswap;
   25071         6420 :       max = maxswap;
   25072              :     }
   25073              : 
   25074              :   /* Given that we have SSSE3, we know we'll be able to implement the
   25075              :      single operand permutation after the palignr with pshufb for
   25076              :      128-bit vectors.  If SINGLE_INSN_ONLY_P, in_order has to be computed
   25077              :      first.  */
   25078         6474 :   if (d->testing_p && GET_MODE_SIZE (d->vmode) == 16 && !single_insn_only_p)
   25079              :     return true;
   25080              : 
   25081         6420 :   dcopy = *d;
   25082         6420 :   if (swap)
   25083              :     {
   25084         3220 :       dcopy.op0 = d->op1;
   25085         3220 :       dcopy.op1 = d->op0;
   25086        16172 :       for (i = 0; i < nelt; ++i)
   25087        12952 :         dcopy.perm[i] ^= nelt;
   25088              :     }
   25089              : 
   25090         6420 :   in_order = true;
   25091        32668 :   for (i = 0; i < nelt; ++i)
   25092              :     {
   25093        26248 :       unsigned e = dcopy.perm[i];
   25094        26248 :       if (GET_MODE_SIZE (d->vmode) == 32
   25095         1152 :           && e >= nelt
   25096        26510 :           && (e & (nelt / 2 - 1)) < min)
   25097          262 :         e = e - min - (nelt / 2);
   25098              :       else
   25099        25986 :         e = e - min;
   25100        26248 :       if (e != i)
   25101        19394 :         in_order = false;
   25102        26248 :       dcopy.perm[i] = e;
   25103              :     }
   25104         6420 :   dcopy.one_operand_p = true;
   25105              : 
   25106         6420 :   if (single_insn_only_p && !in_order)
   25107              :     return false;
   25108              : 
   25109              :   /* For AVX2, test whether we can permute the result in one instruction.  */
   25110         3271 :   if (d->testing_p)
   25111              :     {
   25112           54 :       if (in_order)
   25113              :         return true;
   25114            0 :       dcopy.op1 = dcopy.op0;
   25115            0 :       return expand_vec_perm_1 (&dcopy);
   25116              :     }
   25117              : 
   25118         6434 :   shift = GEN_INT (min * GET_MODE_UNIT_BITSIZE (d->vmode));
   25119         6434 :   if (GET_MODE_SIZE (d->vmode) == 16)
   25120              :     {
   25121         3145 :       target = gen_reg_rtx (V1TImode);
   25122         3145 :       emit_insn (gen_ssse3_palignrv1ti (target,
   25123         3145 :                                         gen_lowpart (V1TImode, dcopy.op1),
   25124         3145 :                                         gen_lowpart (V1TImode, dcopy.op0),
   25125              :                                         shift));
   25126              :     }
   25127              :   else
   25128              :     {
   25129           72 :       target = gen_reg_rtx (V2TImode);
   25130           72 :       emit_insn (gen_avx2_palignrv2ti (target,
   25131           72 :                                        gen_lowpart (V2TImode, dcopy.op1),
   25132           72 :                                        gen_lowpart (V2TImode, dcopy.op0),
   25133              :                                        shift));
   25134              :     }
   25135              : 
   25136         3217 :   dcopy.op0 = dcopy.op1 = gen_lowpart (d->vmode, target);
   25137              : 
   25138              :   /* Test for the degenerate case where the alignment by itself
   25139              :      produces the desired permutation.  */
   25140         3217 :   if (in_order)
   25141              :     {
   25142           70 :       emit_move_insn (d->target, dcopy.op0);
   25143           70 :       return true;
   25144              :     }
   25145              : 
   25146         3147 :   ok = expand_vec_perm_1 (&dcopy);
   25147         3159 :   gcc_assert (ok || GET_MODE_SIZE (d->vmode) == 32);
   25148              : 
   25149              :   return ok;
   25150              : }
   25151              : 
   25152              : /* A subroutine of ix86_expand_vec_perm_const_1.  Try to simplify
   25153              :    the permutation using the SSE4_1 pblendv instruction.  Potentially
   25154              :    reduces permutation from 2 pshufb and or to 1 pshufb and pblendv.  */
   25155              : 
   25156              : static bool
   25157        93355 : expand_vec_perm_pblendv (struct expand_vec_perm_d *d)
   25158              : {
   25159        93355 :   unsigned i, which, nelt = d->nelt;
   25160        93355 :   struct expand_vec_perm_d dcopy, dcopy1;
   25161        93355 :   machine_mode vmode = d->vmode;
   25162        93355 :   bool ok;
   25163              : 
   25164              :   /* Use the same checks as in expand_vec_perm_blend.  */
   25165        93355 :   if (d->one_operand_p)
   25166              :     return false;
   25167        91549 :   if (TARGET_AVX2 && GET_MODE_SIZE (vmode) == 32)
   25168              :     ;
   25169        86003 :   else if (TARGET_AVX && (vmode == V4DFmode || vmode == V8SFmode))
   25170              :     ;
   25171        82912 :   else if (TARGET_SSE4_1
   25172        93503 :            && (GET_MODE_SIZE (vmode) == 16
   25173         9198 :                || (TARGET_MMX_WITH_SSE && GET_MODE_SIZE (vmode) == 8)
   25174         2777 :                || GET_MODE_SIZE (vmode) == 4))
   25175              :     ;
   25176              :   else
   25177              :     return false;
   25178              : 
   25179              :   /* Figure out where permutation elements stay not in their
   25180              :      respective lanes.  */
   25181       106827 :   for (i = 0, which = 0; i < nelt; ++i)
   25182              :     {
   25183        91774 :       unsigned e = d->perm[i];
   25184        91774 :       if (e != i)
   25185       124373 :         which |= (e < nelt ? 1 : 2);
   25186              :     }
   25187              :   /* We can pblend the part where elements stay not in their
   25188              :      respective lanes only when these elements are all in one
   25189              :      half of a permutation.
   25190              :      {0 1 8 3 4 5 9 7} is ok as 8, 9 are at not at their respective
   25191              :      lanes, but both 8 and 9 >= 8
   25192              :      {0 1 8 3 4 5 2 7} is not ok as 2 and 8 are not at their
   25193              :      respective lanes and 8 >= 8, but 2 not.  */
   25194        15053 :   if (which != 1 && which != 2)
   25195              :     return false;
   25196         4815 :   if (d->testing_p && GET_MODE_SIZE (vmode) == 16)
   25197              :     return true;
   25198              : 
   25199              :   /* First we apply one operand permutation to the part where
   25200              :      elements stay not in their respective lanes.  */
   25201         2778 :   dcopy = *d;
   25202         2778 :   if (which == 2)
   25203         2778 :     dcopy.op0 = dcopy.op1 = d->op1;
   25204              :   else
   25205            0 :     dcopy.op0 = dcopy.op1 = d->op0;
   25206         2778 :   if (!d->testing_p)
   25207          741 :     dcopy.target = gen_reg_rtx (vmode);
   25208         2778 :   dcopy.one_operand_p = true;
   25209              : 
   25210        21494 :   for (i = 0; i < nelt; ++i)
   25211        18716 :     dcopy.perm[i] = d->perm[i] & (nelt - 1);
   25212              : 
   25213         2778 :   ok = expand_vec_perm_1 (&dcopy);
   25214         5556 :   if (GET_MODE_SIZE (vmode) != 16 && !ok)
   25215              :     return false;
   25216              :   else
   25217         2435 :     gcc_assert (ok);
   25218         2435 :   if (d->testing_p)
   25219              :     return true;
   25220              : 
   25221              :   /* Next we put permuted elements into their positions.  */
   25222          679 :   dcopy1 = *d;
   25223          679 :   if (which == 2)
   25224          679 :     dcopy1.op1 = dcopy.target;
   25225              :   else
   25226            0 :     dcopy1.op0 = dcopy.target;
   25227              : 
   25228         5751 :   for (i = 0; i < nelt; ++i)
   25229         5072 :     dcopy1.perm[i] = ((d->perm[i] >= nelt) ? (nelt + i) : i);
   25230              : 
   25231          679 :   ok = expand_vec_perm_blend (&dcopy1);
   25232          679 :   gcc_assert (ok);
   25233              : 
   25234              :   return true;
   25235              : }
   25236              : 
   25237              : static bool expand_vec_perm_interleave3 (struct expand_vec_perm_d *d);
   25238              : 
   25239              : /* A subroutine of ix86_expand_vec_perm_const_1.  Try to simplify
   25240              :    a two vector permutation into a single vector permutation by using
   25241              :    an interleave operation to merge the vectors.  */
   25242              : 
   25243              : static bool
   25244        99702 : expand_vec_perm_interleave2 (struct expand_vec_perm_d *d)
   25245              : {
   25246        99702 :   struct expand_vec_perm_d dremap, dfinal;
   25247        99702 :   unsigned i, nelt = d->nelt, nelt2 = nelt / 2;
   25248        99702 :   unsigned HOST_WIDE_INT contents;
   25249        99702 :   unsigned char remap[2 * MAX_VECT_LEN];
   25250        99702 :   rtx_insn *seq;
   25251        99702 :   bool ok, same_halves = false;
   25252              : 
   25253        99702 :   if (GET_MODE_SIZE (d->vmode) == 4
   25254       178874 :       || GET_MODE_SIZE (d->vmode) == 8
   25255       240222 :       || GET_MODE_SIZE (d->vmode) == 16)
   25256              :     {
   25257        93398 :       if (d->one_operand_p)
   25258              :         return false;
   25259              :     }
   25260        12608 :   else if (GET_MODE_SIZE (d->vmode) == 32)
   25261              :     {
   25262         5987 :       if (!TARGET_AVX)
   25263              :         return false;
   25264              :       /* For 32-byte modes allow even d->one_operand_p.
   25265              :          The lack of cross-lane shuffling in some instructions
   25266              :          might prevent a single insn shuffle.  */
   25267         5987 :       dfinal = *d;
   25268         5987 :       dfinal.testing_p = true;
   25269              :       /* If expand_vec_perm_interleave3 can expand this into
   25270              :          a 3 insn sequence, give up and let it be expanded as
   25271              :          3 insn sequence.  While that is one insn longer,
   25272              :          it doesn't need a memory operand and in the common
   25273              :          case that both interleave low and high permutations
   25274              :          with the same operands are adjacent needs 4 insns
   25275              :          for both after CSE.  */
   25276         5987 :       if (expand_vec_perm_interleave3 (&dfinal))
   25277              :         return false;
   25278              :     }
   25279              :   else
   25280              :     return false;
   25281              : 
   25282              :   /* Examine from whence the elements come.  */
   25283        93594 :   contents = 0;
   25284       695944 :   for (i = 0; i < nelt; ++i)
   25285       602350 :     contents |= HOST_WIDE_INT_1U << d->perm[i];
   25286              : 
   25287        93594 :   memset (remap, 0xff, sizeof (remap));
   25288        93594 :   dremap = *d;
   25289              : 
   25290        93594 :   if (GET_MODE_SIZE (d->vmode) == 4
   25291       178755 :       || GET_MODE_SIZE (d->vmode) == 8)
   25292              :     {
   25293        26112 :       unsigned HOST_WIDE_INT h1, h2, h3, h4;
   25294              : 
   25295              :       /* Split the two input vectors into 4 halves.  */
   25296        26112 :       h1 = (HOST_WIDE_INT_1U << nelt2) - 1;
   25297        26112 :       h2 = h1 << nelt2;
   25298        26112 :       h3 = h2 << nelt2;
   25299        26112 :       h4 = h3 << nelt2;
   25300              : 
   25301              :       /* If the elements from the low halves use interleave low,
   25302              :          and similarly for interleave high.  */
   25303        26112 :       if ((contents & (h1 | h3)) == contents)
   25304              :         {
   25305              :           /* punpckl* */
   25306         2832 :           for (i = 0; i < nelt2; ++i)
   25307              :             {
   25308         1936 :               remap[i] = i * 2;
   25309         1936 :               remap[i + nelt] = i * 2 + 1;
   25310         1936 :               dremap.perm[i * 2] = i;
   25311         1936 :               dremap.perm[i * 2 + 1] = i + nelt;
   25312              :             }
   25313              :         }
   25314        25216 :       else if ((contents & (h2 | h4)) == contents)
   25315              :         {
   25316              :           /* punpckh* */
   25317         2277 :           for (i = 0; i < nelt2; ++i)
   25318              :             {
   25319         1548 :               remap[i + nelt2] = i * 2;
   25320         1548 :               remap[i + nelt + nelt2] = i * 2 + 1;
   25321         1548 :               dremap.perm[i * 2] = i + nelt2;
   25322         1548 :               dremap.perm[i * 2 + 1] = i + nelt + nelt2;
   25323              :             }
   25324              :         }
   25325              :       else
   25326              :         return false;
   25327              :     }
   25328       134964 :   else if (GET_MODE_SIZE (d->vmode) == 16)
   25329              :     {
   25330        61721 :       unsigned HOST_WIDE_INT h1, h2, h3, h4;
   25331              : 
   25332              :       /* Split the two input vectors into 4 halves.  */
   25333        61721 :       h1 = (HOST_WIDE_INT_1U << nelt2) - 1;
   25334        61721 :       h2 = h1 << nelt2;
   25335        61721 :       h3 = h2 << nelt2;
   25336        61721 :       h4 = h3 << nelt2;
   25337              : 
   25338              :       /* If the elements from the low halves use interleave low, and similarly
   25339              :          for interleave high.  If the elements are from mis-matched halves, we
   25340              :          can use shufps for V4SF/V4SI or do a DImode shuffle.  */
   25341        61721 :       if ((contents & (h1 | h3)) == contents)
   25342              :         {
   25343              :           /* punpckl* */
   25344         5449 :           for (i = 0; i < nelt2; ++i)
   25345              :             {
   25346         3976 :               remap[i] = i * 2;
   25347         3976 :               remap[i + nelt] = i * 2 + 1;
   25348         3976 :               dremap.perm[i * 2] = i;
   25349         3976 :               dremap.perm[i * 2 + 1] = i + nelt;
   25350              :             }
   25351         1473 :           if (!TARGET_SSE2 && d->vmode == V4SImode)
   25352            0 :             dremap.vmode = V4SFmode;
   25353              :         }
   25354        60248 :       else if ((contents & (h2 | h4)) == contents)
   25355              :         {
   25356              :           /* punpckh* */
   25357         4446 :           for (i = 0; i < nelt2; ++i)
   25358              :             {
   25359         3200 :               remap[i + nelt2] = i * 2;
   25360         3200 :               remap[i + nelt + nelt2] = i * 2 + 1;
   25361         3200 :               dremap.perm[i * 2] = i + nelt2;
   25362         3200 :               dremap.perm[i * 2 + 1] = i + nelt + nelt2;
   25363              :             }
   25364         1246 :           if (!TARGET_SSE2 && d->vmode == V4SImode)
   25365            0 :             dremap.vmode = V4SFmode;
   25366              :         }
   25367        59002 :       else if ((contents & (h1 | h4)) == contents)
   25368              :         {
   25369              :           /* shufps */
   25370         3188 :           for (i = 0; i < nelt2; ++i)
   25371              :             {
   25372         2364 :               remap[i] = i;
   25373         2364 :               remap[i + nelt + nelt2] = i + nelt2;
   25374         2364 :               dremap.perm[i] = i;
   25375         2364 :               dremap.perm[i + nelt2] = i + nelt + nelt2;
   25376              :             }
   25377          824 :           if (nelt != 4)
   25378              :             {
   25379              :               /* shufpd */
   25380          120 :               dremap.vmode = V2DImode;
   25381          120 :               dremap.nelt = 2;
   25382          120 :               dremap.perm[0] = 0;
   25383          120 :               dremap.perm[1] = 3;
   25384              :             }
   25385              :         }
   25386        58178 :       else if ((contents & (h2 | h3)) == contents)
   25387              :         {
   25388              :           /* shufps */
   25389         4684 :           for (i = 0; i < nelt2; ++i)
   25390              :             {
   25391         3438 :               remap[i + nelt2] = i;
   25392         3438 :               remap[i + nelt] = i + nelt2;
   25393         3438 :               dremap.perm[i] = i + nelt2;
   25394         3438 :               dremap.perm[i + nelt2] = i + nelt;
   25395              :             }
   25396         1246 :           if (nelt != 4)
   25397              :             {
   25398              :               /* shufpd */
   25399          195 :               dremap.vmode = V2DImode;
   25400          195 :               dremap.nelt = 2;
   25401          195 :               dremap.perm[0] = 1;
   25402          195 :               dremap.perm[1] = 2;
   25403              :             }
   25404              :         }
   25405              :       else
   25406              :         return false;
   25407              :     }
   25408              :   else
   25409              :     {
   25410         5761 :       unsigned int nelt4 = nelt / 4, nzcnt = 0;
   25411         5761 :       unsigned HOST_WIDE_INT q[8];
   25412         5761 :       unsigned int nonzero_halves[4];
   25413              : 
   25414              :       /* Split the two input vectors into 8 quarters.  */
   25415         5761 :       q[0] = (HOST_WIDE_INT_1U << nelt4) - 1;
   25416        46088 :       for (i = 1; i < 8; ++i)
   25417        40327 :         q[i] = q[0] << (nelt4 * i);
   25418        28805 :       for (i = 0; i < 4; ++i)
   25419        23044 :         if (((q[2 * i] | q[2 * i + 1]) & contents) != 0)
   25420              :           {
   25421        20526 :             nonzero_halves[nzcnt] = i;
   25422        20526 :             ++nzcnt;
   25423              :           }
   25424              : 
   25425         5761 :       if (nzcnt == 1)
   25426              :         {
   25427          215 :           gcc_assert (d->one_operand_p);
   25428          215 :           nonzero_halves[1] = nonzero_halves[0];
   25429          215 :           same_halves = true;
   25430              :         }
   25431         5546 :       else if (d->one_operand_p)
   25432              :         {
   25433          115 :           gcc_assert (nonzero_halves[0] == 0);
   25434          115 :           gcc_assert (nonzero_halves[1] == 1);
   25435              :         }
   25436              : 
   25437         5761 :       if (nzcnt <= 2)
   25438              :         {
   25439          582 :           if (d->perm[0] / nelt2 == nonzero_halves[1])
   25440              :             {
   25441              :               /* Attempt to increase the likelihood that dfinal
   25442              :                  shuffle will be intra-lane.  */
   25443          315 :               std::swap (nonzero_halves[0], nonzero_halves[1]);
   25444              :             }
   25445              : 
   25446              :           /* vperm2f128 or vperm2i128.  */
   25447         3532 :           for (i = 0; i < nelt2; ++i)
   25448              :             {
   25449         2950 :               remap[i + nonzero_halves[1] * nelt2] = i + nelt2;
   25450         2950 :               remap[i + nonzero_halves[0] * nelt2] = i;
   25451         2950 :               dremap.perm[i + nelt2] = i + nonzero_halves[1] * nelt2;
   25452         2950 :               dremap.perm[i] = i + nonzero_halves[0] * nelt2;
   25453              :             }
   25454              : 
   25455          582 :           if (d->vmode != V8SFmode
   25456              :               && d->vmode != V4DFmode
   25457              :               && d->vmode != V8SImode)
   25458              :             {
   25459          188 :               dremap.vmode = V8SImode;
   25460          188 :               dremap.nelt = 8;
   25461          940 :               for (i = 0; i < 4; ++i)
   25462              :                 {
   25463          752 :                   dremap.perm[i] = i + nonzero_halves[0] * 4;
   25464          752 :                   dremap.perm[i + 4] = i + nonzero_halves[1] * 4;
   25465              :                 }
   25466              :             }
   25467              :         }
   25468         5179 :       else if (d->one_operand_p)
   25469         4714 :         return false;
   25470         5179 :       else if (TARGET_AVX2
   25471         2132 :                && (contents & (q[0] | q[2] | q[4] | q[6])) == contents)
   25472              :         {
   25473              :           /* vpunpckl* */
   25474          491 :           for (i = 0; i < nelt4; ++i)
   25475              :             {
   25476          247 :               remap[i] = i * 2;
   25477          247 :               remap[i + nelt] = i * 2 + 1;
   25478          247 :               remap[i + nelt2] = i * 2 + nelt2;
   25479          247 :               remap[i + nelt + nelt2] = i * 2 + nelt2 + 1;
   25480          247 :               dremap.perm[i * 2] = i;
   25481          247 :               dremap.perm[i * 2 + 1] = i + nelt;
   25482          247 :               dremap.perm[i * 2 + nelt2] = i + nelt2;
   25483          247 :               dremap.perm[i * 2 + nelt2 + 1] = i + nelt + nelt2;
   25484              :             }
   25485              :         }
   25486         4935 :       else if (TARGET_AVX2
   25487         1888 :                && (contents & (q[1] | q[3] | q[5] | q[7])) == contents)
   25488              :         {
   25489              :           /* vpunpckh* */
   25490          445 :           for (i = 0; i < nelt4; ++i)
   25491              :             {
   25492          224 :               remap[i + nelt4] = i * 2;
   25493          224 :               remap[i + nelt + nelt4] = i * 2 + 1;
   25494          224 :               remap[i + nelt2 + nelt4] = i * 2 + nelt2;
   25495          224 :               remap[i + nelt + nelt2 + nelt4] = i * 2 + nelt2 + 1;
   25496          224 :               dremap.perm[i * 2] = i + nelt4;
   25497          224 :               dremap.perm[i * 2 + 1] = i + nelt + nelt4;
   25498          224 :               dremap.perm[i * 2 + nelt2] = i + nelt2 + nelt4;
   25499          224 :               dremap.perm[i * 2 + nelt2 + 1] = i + nelt + nelt2 + nelt4;
   25500              :             }
   25501              :         }
   25502              :       else
   25503              :         return false;
   25504              :     }
   25505              : 
   25506              :   /* Use the remapping array set up above to move the elements from their
   25507              :      swizzled locations into their final destinations.  */
   25508         7461 :   dfinal = *d;
   25509        48169 :   for (i = 0; i < nelt; ++i)
   25510              :     {
   25511        40708 :       unsigned e = remap[d->perm[i]];
   25512        40708 :       gcc_assert (e < nelt);
   25513              :       /* If same_halves is true, both halves of the remapped vector are the
   25514              :          same.  Avoid cross-lane accesses if possible.  */
   25515        40708 :       if (same_halves && i >= nelt2)
   25516              :         {
   25517          792 :           gcc_assert (e < nelt2);
   25518          792 :           dfinal.perm[i] = e + nelt2;
   25519              :         }
   25520              :       else
   25521        39916 :         dfinal.perm[i] = e;
   25522              :     }
   25523         7461 :   if (!d->testing_p)
   25524              :     {
   25525         2689 :       dremap.target = gen_reg_rtx (dremap.vmode);
   25526         2689 :       dfinal.op0 = gen_lowpart (dfinal.vmode, dremap.target);
   25527              :     }
   25528         7461 :   dfinal.op1 = dfinal.op0;
   25529         7461 :   dfinal.one_operand_p = true;
   25530              : 
   25531              :   /* Test if the final remap can be done with a single insn.  For V4SFmode or
   25532              :      V4SImode this *will* succeed.  For V8HImode or V16QImode it may not.  */
   25533         7461 :   start_sequence ();
   25534         7461 :   ok = expand_vec_perm_1 (&dfinal);
   25535         7461 :   seq = end_sequence ();
   25536              : 
   25537         7461 :   if (!ok)
   25538              :     return false;
   25539              : 
   25540         6179 :   if (d->testing_p)
   25541              :     return true;
   25542              : 
   25543         2650 :   if (dremap.vmode != dfinal.vmode)
   25544              :     {
   25545           57 :       dremap.op0 = gen_lowpart (dremap.vmode, dremap.op0);
   25546           57 :       dremap.op1 = gen_lowpart (dremap.vmode, dremap.op1);
   25547              :     }
   25548              : 
   25549         2650 :   ok = expand_vec_perm_1 (&dremap);
   25550         2650 :   gcc_assert (ok);
   25551              : 
   25552         2650 :   emit_insn (seq);
   25553         2650 :   return true;
   25554              : }
   25555              : 
   25556              : /* A subroutine of ix86_expand_vec_perm_const_1.  Try to simplify
   25557              :    a single vector cross-lane permutation into vpermq followed
   25558              :    by any of the single insn permutations.  */
   25559              : 
   25560              : static bool
   25561        93423 : expand_vec_perm_vpermq_perm_1 (struct expand_vec_perm_d *d)
   25562              : {
   25563        93423 :   struct expand_vec_perm_d dremap, dfinal;
   25564        93423 :   unsigned i, j, nelt = d->nelt, nelt2 = nelt / 2, nelt4 = nelt / 4;
   25565        93423 :   unsigned contents[2];
   25566        93423 :   bool ok;
   25567              : 
   25568        93423 :   if (!(TARGET_AVX2
   25569         4069 :         && (d->vmode == V32QImode || d->vmode == V16HImode)
   25570          248 :         && d->one_operand_p))
   25571              :     return false;
   25572              : 
   25573            7 :   contents[0] = 0;
   25574            7 :   contents[1] = 0;
   25575          103 :   for (i = 0; i < nelt2; ++i)
   25576              :     {
   25577           96 :       contents[0] |= 1u << (d->perm[i] / nelt4);
   25578           96 :       contents[1] |= 1u << (d->perm[i + nelt2] / nelt4);
   25579              :     }
   25580              : 
   25581            7 :   for (i = 0; i < 2; ++i)
   25582              :     {
   25583              :       unsigned int cnt = 0;
   25584           21 :       for (j = 0; j < 4; ++j)
   25585           21 :         if ((contents[i] & (1u << j)) != 0 && ++cnt > 2)
   25586              :           return false;
   25587              :     }
   25588              : 
   25589            0 :   if (d->testing_p)
   25590              :     return true;
   25591              : 
   25592            0 :   dremap = *d;
   25593            0 :   dremap.vmode = V4DImode;
   25594            0 :   dremap.nelt = 4;
   25595            0 :   dremap.target = gen_reg_rtx (V4DImode);
   25596            0 :   dremap.op0 = gen_lowpart (V4DImode, d->op0);
   25597            0 :   dremap.op1 = dremap.op0;
   25598            0 :   dremap.one_operand_p = true;
   25599            0 :   for (i = 0; i < 2; ++i)
   25600              :     {
   25601              :       unsigned int cnt = 0;
   25602            0 :       for (j = 0; j < 4; ++j)
   25603            0 :         if ((contents[i] & (1u << j)) != 0)
   25604            0 :           dremap.perm[2 * i + cnt++] = j;
   25605            0 :       for (; cnt < 2; ++cnt)
   25606            0 :         dremap.perm[2 * i + cnt] = 0;
   25607              :     }
   25608              : 
   25609            0 :   dfinal = *d;
   25610            0 :   dfinal.op0 = gen_lowpart (dfinal.vmode, dremap.target);
   25611            0 :   dfinal.op1 = dfinal.op0;
   25612            0 :   dfinal.one_operand_p = true;
   25613            0 :   for (i = 0, j = 0; i < nelt; ++i)
   25614              :     {
   25615            0 :       if (i == nelt2)
   25616            0 :         j = 2;
   25617            0 :       dfinal.perm[i] = (d->perm[i] & (nelt4 - 1)) | (j ? nelt2 : 0);
   25618            0 :       if ((d->perm[i] / nelt4) == dremap.perm[j])
   25619              :         ;
   25620            0 :       else if ((d->perm[i] / nelt4) == dremap.perm[j + 1])
   25621            0 :         dfinal.perm[i] |= nelt4;
   25622              :       else
   25623            0 :         gcc_unreachable ();
   25624              :     }
   25625              : 
   25626            0 :   ok = expand_vec_perm_1 (&dremap);
   25627            0 :   gcc_assert (ok);
   25628              : 
   25629            0 :   ok = expand_vec_perm_1 (&dfinal);
   25630            0 :   gcc_assert (ok);
   25631              : 
   25632              :   return true;
   25633              : }
   25634              : 
   25635              : static bool canonicalize_perm (struct expand_vec_perm_d *d);
   25636              : 
   25637              : /* A subroutine of ix86_expand_vec_perm_const_1.  Try to expand
   25638              :    a vector permutation using two instructions, vperm2f128 resp.
   25639              :    vperm2i128 followed by any single in-lane permutation.  */
   25640              : 
   25641              : static bool
   25642        93423 : expand_vec_perm_vperm2f128 (struct expand_vec_perm_d *d)
   25643              : {
   25644        93423 :   struct expand_vec_perm_d dfirst, dsecond;
   25645        93423 :   unsigned i, j, nelt = d->nelt, nelt2 = nelt / 2, perm;
   25646        93423 :   bool ok;
   25647              : 
   25648        93423 :   if (!TARGET_AVX
   25649        21590 :       || GET_MODE_SIZE (d->vmode) != 32
   25650        98488 :       || (d->vmode != V8SFmode && d->vmode != V4DFmode && !TARGET_AVX2))
   25651              :     return false;
   25652              : 
   25653         4881 :   dsecond = *d;
   25654         4881 :   dsecond.one_operand_p = false;
   25655         4881 :   dsecond.testing_p = true;
   25656              : 
   25657              :   /* ((perm << 2)|perm) & 0x33 is the vperm2[fi]128
   25658              :      immediate.  For perm < 16 the second permutation uses
   25659              :      d->op0 as first operand, for perm >= 16 it uses d->op1
   25660              :      as first operand.  The second operand is the result of
   25661              :      vperm2[fi]128.  */
   25662       159747 :   for (perm = 0; perm < 32; perm++)
   25663              :     {
   25664              :       /* Ignore permutations which do not move anything cross-lane.  */
   25665       154949 :       if (perm < 16)
   25666              :         {
   25667              :           /* The second shuffle for e.g. V4DFmode has
   25668              :              0123 and ABCD operands.
   25669              :              Ignore AB23, as 23 is already in the second lane
   25670              :              of the first operand.  */
   25671        77729 :           if ((perm & 0xc) == (1 << 2)) continue;
   25672              :           /* And 01CD, as 01 is in the first lane of the first
   25673              :              operand.  */
   25674        58281 :           if ((perm & 3) == 0) continue;
   25675              :           /* And 4567, as then the vperm2[fi]128 doesn't change
   25676              :              anything on the original 4567 second operand.  */
   25677        43693 :           if ((perm & 0xf) == ((3 << 2) | 2)) continue;
   25678              :         }
   25679              :       else
   25680              :         {
   25681              :           /* The second shuffle for e.g. V4DFmode has
   25682              :              4567 and ABCD operands.
   25683              :              Ignore AB67, as 67 is already in the second lane
   25684              :              of the first operand.  */
   25685        77220 :           if ((perm & 0xc) == (3 << 2)) continue;
   25686              :           /* And 45CD, as 45 is in the first lane of the first
   25687              :              operand.  */
   25688        58028 :           if ((perm & 3) == 2) continue;
   25689              :           /* And 0123, as then the vperm2[fi]128 doesn't change
   25690              :              anything on the original 0123 first operand.  */
   25691        43544 :           if ((perm & 0xf) == (1 << 2)) continue;
   25692              :         }
   25693              : 
   25694       203615 :       for (i = 0; i < nelt; i++)
   25695              :         {
   25696       202296 :           j = d->perm[i] / nelt2;
   25697       374984 :           if (j == ((perm >> (2 * (i >= nelt2))) & 3))
   25698        48950 :             dsecond.perm[i] = nelt + (i & nelt2) + (d->perm[i] & (nelt2 - 1));
   25699       257136 :           else if (j == (unsigned) (i >= nelt2) + 2 * (perm >= 16))
   25700        77116 :             dsecond.perm[i] = d->perm[i] & (nelt - 1);
   25701              :           else
   25702              :             break;
   25703              :         }
   25704              : 
   25705        77549 :       if (i == nelt)
   25706              :         {
   25707         1319 :           start_sequence ();
   25708         1319 :           ok = expand_vec_perm_1 (&dsecond);
   25709         1319 :           end_sequence ();
   25710              :         }
   25711              :       else
   25712              :         ok = false;
   25713              : 
   25714         1319 :       if (ok)
   25715              :         {
   25716           68 :           if (d->testing_p)
   25717              :             return true;
   25718              : 
   25719              :           /* Found a usable second shuffle.  dfirst will be
   25720              :              vperm2f128 on d->op0 and d->op1.  */
   25721           46 :           dsecond.testing_p = false;
   25722           46 :           dfirst = *d;
   25723           46 :           dfirst.target = gen_reg_rtx (d->vmode);
   25724          270 :           for (i = 0; i < nelt; i++)
   25725          448 :             dfirst.perm[i] = (i & (nelt2 - 1))
   25726          336 :                              + ((perm >> (2 * (i >= nelt2))) & 3) * nelt2;
   25727              : 
   25728           46 :           canonicalize_perm (&dfirst);
   25729           46 :           ok = expand_vec_perm_1 (&dfirst);
   25730           46 :           gcc_assert (ok);
   25731              : 
   25732              :           /* And dsecond is some single insn shuffle, taking
   25733              :              d->op0 and result of vperm2f128 (if perm < 16) or
   25734              :              d->op1 and result of vperm2f128 (otherwise).  */
   25735           46 :           if (perm >= 16)
   25736           46 :             dsecond.op0 = dsecond.op1;
   25737           46 :           dsecond.op1 = dfirst.target;
   25738              : 
   25739           46 :           ok = expand_vec_perm_1 (&dsecond);
   25740           46 :           gcc_assert (ok);
   25741              : 
   25742              :           return true;
   25743              :         }
   25744              : 
   25745              :       /* For one operand, the only useful vperm2f128 permutation is 0x01
   25746              :          aka lanes swap.  */
   25747        77481 :       if (d->one_operand_p)
   25748              :         return false;
   25749              :     }
   25750              : 
   25751              :   return false;
   25752              : }
   25753              : 
   25754              : /* A subroutine of ix86_expand_vec_perm_const_1.  Try to simplify
   25755              :    a two vector permutation using 2 intra-lane interleave insns
   25756              :    and cross-lane shuffle for 32-byte vectors.  */
   25757              : 
   25758              : static bool
   25759        34511 : expand_vec_perm_interleave3 (struct expand_vec_perm_d *d)
   25760              : {
   25761        34511 :   unsigned i, nelt;
   25762        34511 :   rtx (*gen) (rtx, rtx, rtx);
   25763              : 
   25764        34511 :   if (d->one_operand_p)
   25765              :     return false;
   25766        31909 :   if (TARGET_AVX2 && GET_MODE_SIZE (d->vmode) == 32)
   25767              :     ;
   25768        25401 :   else if (TARGET_AVX && (d->vmode == V8SFmode || d->vmode == V4DFmode))
   25769              :     ;
   25770              :   else
   25771              :     return false;
   25772              : 
   25773         7869 :   nelt = d->nelt;
   25774         7869 :   if (d->perm[0] != 0 && d->perm[0] != nelt / 2)
   25775              :     return false;
   25776         7726 :   for (i = 0; i < nelt; i += 2)
   25777         7354 :     if (d->perm[i] != d->perm[0] + i / 2
   25778         6359 :         || d->perm[i + 1] != d->perm[0] + i / 2 + nelt)
   25779              :       return false;
   25780              : 
   25781          372 :   if (d->testing_p)
   25782              :     return true;
   25783              : 
   25784           56 :   switch (d->vmode)
   25785              :     {
   25786           32 :     case E_V32QImode:
   25787           32 :       if (d->perm[0])
   25788              :         gen = gen_vec_interleave_highv32qi;
   25789              :       else
   25790           16 :         gen = gen_vec_interleave_lowv32qi;
   25791              :       break;
   25792           18 :     case E_V16HImode:
   25793           18 :       if (d->perm[0])
   25794              :         gen = gen_vec_interleave_highv16hi;
   25795              :       else
   25796            9 :         gen = gen_vec_interleave_lowv16hi;
   25797              :       break;
   25798            0 :     case E_V8SImode:
   25799            0 :       if (d->perm[0])
   25800              :         gen = gen_vec_interleave_highv8si;
   25801              :       else
   25802            0 :         gen = gen_vec_interleave_lowv8si;
   25803              :       break;
   25804            4 :     case E_V4DImode:
   25805            4 :       if (d->perm[0])
   25806              :         gen = gen_vec_interleave_highv4di;
   25807              :       else
   25808            2 :         gen = gen_vec_interleave_lowv4di;
   25809              :       break;
   25810            2 :     case E_V8SFmode:
   25811            2 :       if (d->perm[0])
   25812              :         gen = gen_vec_interleave_highv8sf;
   25813              :       else
   25814            1 :         gen = gen_vec_interleave_lowv8sf;
   25815              :       break;
   25816            0 :     case E_V4DFmode:
   25817            0 :       if (d->perm[0])
   25818              :         gen = gen_vec_interleave_highv4df;
   25819              :       else
   25820            0 :         gen = gen_vec_interleave_lowv4df;
   25821              :       break;
   25822            0 :     default:
   25823            0 :       gcc_unreachable ();
   25824              :     }
   25825              : 
   25826           56 :   emit_insn (gen (d->target, d->op0, d->op1));
   25827           56 :   return true;
   25828              : }
   25829              : 
   25830              : /* A subroutine of ix86_expand_vec_perm_const_1.  Try to implement
   25831              :    a single vector permutation using a single intra-lane vector
   25832              :    permutation, vperm2f128 swapping the lanes and vblend* insn blending
   25833              :    the non-swapped and swapped vectors together.  */
   25834              : 
   25835              : static bool
   25836        28378 : expand_vec_perm_vperm2f128_vblend (struct expand_vec_perm_d *d)
   25837              : {
   25838        28378 :   struct expand_vec_perm_d dfirst, dsecond;
   25839        28378 :   unsigned i, j, msk, nelt = d->nelt, nelt2 = nelt / 2;
   25840        28378 :   rtx_insn *seq;
   25841        28378 :   bool ok;
   25842        28378 :   rtx (*blend) (rtx, rtx, rtx, rtx) = NULL;
   25843              : 
   25844        28378 :   if (!TARGET_AVX
   25845         2765 :       || TARGET_AVX2
   25846         1686 :       || (d->vmode != V8SFmode && d->vmode != V4DFmode)
   25847         1502 :       || !d->one_operand_p)
   25848              :     return false;
   25849              : 
   25850            0 :   dfirst = *d;
   25851            0 :   for (i = 0; i < nelt; i++)
   25852            0 :     dfirst.perm[i] = 0xff;
   25853            0 :   for (i = 0, msk = 0; i < nelt; i++)
   25854              :     {
   25855            0 :       j = (d->perm[i] & nelt2) ? i | nelt2 : i & ~nelt2;
   25856            0 :       if (dfirst.perm[j] != 0xff && dfirst.perm[j] != d->perm[i])
   25857              :         return false;
   25858            0 :       dfirst.perm[j] = d->perm[i];
   25859            0 :       if (j != i)
   25860            0 :         msk |= (1 << i);
   25861              :     }
   25862            0 :   for (i = 0; i < nelt; i++)
   25863            0 :     if (dfirst.perm[i] == 0xff)
   25864            0 :       dfirst.perm[i] = i;
   25865              : 
   25866            0 :   if (!d->testing_p)
   25867            0 :     dfirst.target = gen_reg_rtx (dfirst.vmode);
   25868              : 
   25869            0 :   start_sequence ();
   25870            0 :   ok = expand_vec_perm_1 (&dfirst);
   25871            0 :   seq = end_sequence ();
   25872              : 
   25873            0 :   if (!ok)
   25874              :     return false;
   25875              : 
   25876            0 :   if (d->testing_p)
   25877              :     return true;
   25878              : 
   25879            0 :   emit_insn (seq);
   25880              : 
   25881            0 :   dsecond = *d;
   25882            0 :   dsecond.op0 = dfirst.target;
   25883            0 :   dsecond.op1 = dfirst.target;
   25884            0 :   dsecond.one_operand_p = true;
   25885            0 :   dsecond.target = gen_reg_rtx (dsecond.vmode);
   25886            0 :   for (i = 0; i < nelt; i++)
   25887            0 :     dsecond.perm[i] = i ^ nelt2;
   25888              : 
   25889            0 :   ok = expand_vec_perm_1 (&dsecond);
   25890            0 :   gcc_assert (ok);
   25891              : 
   25892            0 :   blend = d->vmode == V8SFmode ? gen_avx_blendps256 : gen_avx_blendpd256;
   25893            0 :   emit_insn (blend (d->target, dfirst.target, dsecond.target, GEN_INT (msk)));
   25894            0 :   return true;
   25895              : }
   25896              : 
   25897              : /* A subroutine of ix86_expand_vec_perm_const_1.  Try to implement
   25898              :    a two vector permutation using two single vector permutations and
   25899              :    {,v}{,p}unpckl{ps,pd,bw,wd,dq}.  If two_insn, succeed only if one
   25900              :    of dfirst or dsecond is identity permutation.  */
   25901              : 
   25902              : static bool
   25903       119298 : expand_vec_perm_2perm_interleave (struct expand_vec_perm_d *d, bool two_insn)
   25904              : {
   25905       119298 :   unsigned i, nelt = d->nelt, nelt2 = nelt / 2, lane = nelt;
   25906       119298 :   struct expand_vec_perm_d dfirst, dsecond, dfinal;
   25907       119298 :   bool ident1 = true, ident2 = true;
   25908              : 
   25909       119298 :   if (d->one_operand_p)
   25910              :     return false;
   25911              : 
   25912       216102 :   if (GET_MODE_SIZE (d->vmode) == 16)
   25913              :     {
   25914        65490 :       if (!TARGET_SSE)
   25915              :         return false;
   25916        65490 :       if (d->vmode != V4SFmode && d->vmode != V2DFmode && !TARGET_SSE2)
   25917              :         return false;
   25918              :     }
   25919        85122 :   else if (GET_MODE_SIZE (d->vmode) == 32)
   25920              :     {
   25921         6593 :       if (!TARGET_AVX)
   25922              :         return false;
   25923         6593 :       if (d->vmode != V8SFmode && d->vmode != V4DFmode && !TARGET_AVX2)
   25924              :         return false;
   25925              :       lane = nelt2;
   25926              :     }
   25927              :   else
   25928              :     return false;
   25929              : 
   25930       233792 :   for (i = 1; i < nelt; i++)
   25931       199953 :     if ((d->perm[i] >= nelt) != ((d->perm[0] >= nelt) ^ (i & 1)))
   25932              :       return false;
   25933              : 
   25934        33839 :   dfirst = *d;
   25935        33839 :   dsecond = *d;
   25936        33839 :   dfinal = *d;
   25937        33839 :   dfirst.op1 = dfirst.op0;
   25938        33839 :   dfirst.one_operand_p = true;
   25939        33839 :   dsecond.op0 = dsecond.op1;
   25940        33839 :   dsecond.one_operand_p = true;
   25941              : 
   25942       219787 :   for (i = 0; i < nelt; i++)
   25943       185948 :     if (d->perm[i] >= nelt)
   25944              :       {
   25945        92974 :         dsecond.perm[i / 2 + (i >= lane ? lane / 2 : 0)] = d->perm[i] - nelt;
   25946        92974 :         if (d->perm[i] - nelt != i / 2 + (i >= lane ? lane / 2 : 0))
   25947        85074 :           ident2 = false;
   25948        92974 :         dsecond.perm[i / 2 + (i >= lane ? lane : lane / 2)]
   25949        92974 :           = d->perm[i] - nelt;
   25950              :       }
   25951              :     else
   25952              :       {
   25953        92974 :         dfirst.perm[i / 2 + (i >= lane ? lane / 2 : 0)] = d->perm[i];
   25954        92974 :         if (d->perm[i] != i / 2 + (i >= lane ? lane / 2 : 0))
   25955        76972 :           ident1 = false;
   25956        92974 :         dfirst.perm[i / 2 + (i >= lane ? lane : lane / 2)] = d->perm[i];
   25957              :       }
   25958              : 
   25959        33839 :   if (two_insn && !ident1 && !ident2)
   25960              :     return false;
   25961              : 
   25962         3893 :   if (!d->testing_p)
   25963              :     {
   25964          214 :       if (!ident1)
   25965          146 :         dfinal.op0 = dfirst.target = gen_reg_rtx (d->vmode);
   25966          214 :       if (!ident2)
   25967          146 :         dfinal.op1 = dsecond.target = gen_reg_rtx (d->vmode);
   25968          214 :       if (d->perm[0] >= nelt)
   25969            0 :         std::swap (dfinal.op0, dfinal.op1);
   25970              :     }
   25971              : 
   25972         3893 :   bool ok;
   25973         3893 :   rtx_insn *seq1 = NULL, *seq2 = NULL;
   25974              : 
   25975         3893 :   if (!ident1)
   25976              :     {
   25977         2781 :       start_sequence ();
   25978         2781 :       ok = expand_vec_perm_1 (&dfirst);
   25979         2781 :       seq1 = end_sequence ();
   25980              : 
   25981         2781 :       if (!ok)
   25982              :         return false;
   25983              :     }
   25984              : 
   25985         2212 :   if (!ident2)
   25986              :     {
   25987         1922 :       start_sequence ();
   25988         1922 :       ok = expand_vec_perm_1 (&dsecond);
   25989         1922 :       seq2 = end_sequence ();
   25990              : 
   25991         1922 :       if (!ok)
   25992              :         return false;
   25993              :     }
   25994              : 
   25995          550 :   if (d->testing_p)
   25996              :     return true;
   25997              : 
   25998          680 :   for (i = 0; i < nelt; i++)
   25999              :     {
   26000          544 :       dfinal.perm[i] = i / 2;
   26001          544 :       if (i >= lane)
   26002            4 :         dfinal.perm[i] += lane / 2;
   26003          544 :       if ((i & 1) != 0)
   26004          272 :         dfinal.perm[i] += nelt;
   26005              :     }
   26006          136 :   emit_insn (seq1);
   26007          136 :   emit_insn (seq2);
   26008          136 :   ok = expand_vselect_vconcat (dfinal.target, dfinal.op0, dfinal.op1,
   26009              :                                dfinal.perm, dfinal.nelt, false);
   26010          136 :   gcc_assert (ok);
   26011              :   return true;
   26012              : }
   26013              : 
   26014              : /* A subroutine of ix86_expand_vec_perm_const_1.  Try to simplify
   26015              :    the permutation using two single vector permutations and the SSE4_1 pblendv
   26016              :    instruction.  If two_insn, succeed only if one of dfirst or dsecond is
   26017              :    identity permutation.  */
   26018              : 
   26019              : static bool
   26020       118748 : expand_vec_perm_2perm_pblendv (struct expand_vec_perm_d *d, bool two_insn)
   26021              : {
   26022       118748 :   unsigned i, nelt = d->nelt;
   26023       118748 :   struct expand_vec_perm_d dfirst, dsecond, dfinal;
   26024       118748 :   machine_mode vmode = d->vmode;
   26025       118748 :   bool ident1 = true, ident2 = true;
   26026              : 
   26027              :   /* Use the same checks as in expand_vec_perm_blend.  */
   26028       118748 :   if (d->one_operand_p)
   26029              :     return false;
   26030       111051 :   if (TARGET_AVX2 && GET_MODE_SIZE (vmode) == 32)
   26031              :     ;
   26032       105787 :   else if (TARGET_AVX && (vmode == V4DFmode || vmode == V8SFmode))
   26033              :     ;
   26034       101474 :   else if (TARGET_SSE4_1
   26035       110732 :            && (GET_MODE_SIZE (vmode) == 16
   26036         8092 :                || (TARGET_MMX_WITH_SSE && GET_MODE_SIZE (vmode) == 8)
   26037         2327 :                || GET_MODE_SIZE (vmode) == 4))
   26038              :     ;
   26039              :   else
   26040              :     return false;
   26041              : 
   26042        14613 :   dfirst = *d;
   26043        14613 :   dsecond = *d;
   26044        14613 :   dfinal = *d;
   26045        14613 :   dfirst.op1 = dfirst.op0;
   26046        14613 :   dfirst.one_operand_p = true;
   26047        14613 :   dsecond.op0 = dsecond.op1;
   26048        14613 :   dsecond.one_operand_p = true;
   26049              : 
   26050       109015 :   for (i = 0; i < nelt; ++i)
   26051        94402 :     if (d->perm[i] >= nelt)
   26052              :       {
   26053        47494 :         dfirst.perm[i] = 0xff;
   26054        47494 :         dsecond.perm[i] = d->perm[i] - nelt;
   26055        47494 :         if (d->perm[i] != i + nelt)
   26056        94402 :           ident2 = false;
   26057              :       }
   26058              :     else
   26059              :       {
   26060        46908 :         dsecond.perm[i] = 0xff;
   26061        46908 :         dfirst.perm[i] = d->perm[i];
   26062        46908 :         if (d->perm[i] != i)
   26063        94402 :           ident1 = false;
   26064              :       }
   26065              : 
   26066        14613 :   if (two_insn && !ident1 && !ident2)
   26067              :     return false;
   26068              : 
   26069              :   /* For now.  Ideally treat 0xff as a wildcard.  */
   26070        38146 :   for (i = 0; i < nelt; ++i)
   26071        33596 :     if (dfirst.perm[i] == 0xff)
   26072              :       {
   26073        17971 :         if (GET_MODE_SIZE (vmode) == 32
   26074        17971 :             && dfirst.perm[i ^ (nelt / 2)] != 0xff)
   26075        10707 :           dfirst.perm[i] = dfirst.perm[i ^ (nelt / 2)] ^ (nelt / 2);
   26076              :         else
   26077         7264 :           dfirst.perm[i] = i;
   26078              :       }
   26079              :     else
   26080              :       {
   26081        15625 :         if (GET_MODE_SIZE (vmode) == 32
   26082        15625 :             && dsecond.perm[i ^ (nelt / 2)] != 0xff)
   26083         9058 :           dsecond.perm[i] = dsecond.perm[i ^ (nelt / 2)] ^ (nelt / 2);
   26084              :         else
   26085         6567 :           dsecond.perm[i] = i;
   26086              :       }
   26087              : 
   26088         4550 :   if (!d->testing_p)
   26089              :     {
   26090         2144 :       if (!ident1)
   26091         2020 :         dfinal.op0 = dfirst.target = gen_reg_rtx (d->vmode);
   26092         2144 :       if (!ident2)
   26093          823 :         dfinal.op1 = dsecond.target = gen_reg_rtx (d->vmode);
   26094              :     }
   26095              : 
   26096         4550 :   bool ok;
   26097         4550 :   rtx_insn *seq1 = NULL, *seq2 = NULL;
   26098              : 
   26099         4550 :   if (!ident1)
   26100              :     {
   26101         3865 :       start_sequence ();
   26102         3865 :       ok = expand_vec_perm_1 (&dfirst);
   26103         3865 :       seq1 = end_sequence ();
   26104              : 
   26105         3865 :       if (!ok)
   26106              :         return false;
   26107              :     }
   26108              : 
   26109         3435 :   if (!ident2)
   26110              :     {
   26111         1293 :       start_sequence ();
   26112         1293 :       ok = expand_vec_perm_1 (&dsecond);
   26113         1293 :       seq2 = end_sequence ();
   26114              : 
   26115         1293 :       if (!ok)
   26116              :         return false;
   26117              :     }
   26118              : 
   26119         2702 :   if (d->testing_p)
   26120              :     return true;
   26121              : 
   26122        13794 :   for (i = 0; i < nelt; ++i)
   26123        11992 :     dfinal.perm[i] = (d->perm[i] >= nelt ? i + nelt : i);
   26124              : 
   26125         1802 :   emit_insn (seq1);
   26126         1802 :   emit_insn (seq2);
   26127         1802 :   ok = expand_vec_perm_blend (&dfinal);
   26128         1802 :   gcc_assert (ok);
   26129              :   return true;
   26130              : }
   26131              : 
   26132              : /* A subroutine of ix86_expand_vec_perm_const_1.
   26133              :    Implement a permutation with psrlw, psllw and por.
   26134              :    It handles case:
   26135              :    __builtin_shufflevector (v,v,1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14);
   26136              :    __builtin_shufflevector (v,v,1,0,3,2,5,4,7,6); */
   26137              : 
   26138              : static bool
   26139        27626 : expand_vec_perm_psrlw_psllw_por (struct expand_vec_perm_d *d)
   26140              : {
   26141        27626 :   unsigned i;
   26142        27626 :   rtx (*gen_shr) (rtx, rtx, rtx);
   26143        27626 :   rtx (*gen_shl) (rtx, rtx, rtx);
   26144        27626 :   rtx (*gen_or) (rtx, rtx, rtx);
   26145        27626 :   machine_mode mode = VOIDmode;
   26146              : 
   26147        27626 :   if (!TARGET_SSE2 || !d->one_operand_p)
   26148              :     return false;
   26149              : 
   26150         5602 :   switch (d->vmode)
   26151              :     {
   26152         1494 :     case E_V8QImode:
   26153         1494 :       if (!TARGET_MMX_WITH_SSE)
   26154              :         return false;
   26155              :       mode = V4HImode;
   26156              :       gen_shr = gen_lshrv4hi3;
   26157              :       gen_shl = gen_ashlv4hi3;
   26158              :       gen_or = gen_iorv4hi3;
   26159              :       break;
   26160              :     case E_V16QImode:
   26161              :       mode = V8HImode;
   26162              :       gen_shr = gen_lshrv8hi3;
   26163              :       gen_shl = gen_ashlv8hi3;
   26164              :       gen_or = gen_iorv8hi3;
   26165              :       break;
   26166              :     default: return false;
   26167              :     }
   26168              : 
   26169         3318 :   if (!rtx_equal_p (d->op0, d->op1))
   26170              :     return false;
   26171              : 
   26172        12486 :   for (i = 0; i < d->nelt; i += 2)
   26173        11032 :     if (d->perm[i] != i + 1 || d->perm[i + 1] != i)
   26174              :       return false;
   26175              : 
   26176         1454 :   if (d->testing_p)
   26177              :     return true;
   26178              : 
   26179           30 :   rtx tmp1 = gen_reg_rtx (mode);
   26180           30 :   rtx tmp2 = gen_reg_rtx (mode);
   26181           30 :   rtx op0 = force_reg (d->vmode, d->op0);
   26182              : 
   26183           30 :   emit_move_insn (tmp1, lowpart_subreg (mode, op0, d->vmode));
   26184           30 :   emit_move_insn (tmp2, lowpart_subreg (mode, op0, d->vmode));
   26185           30 :   emit_insn (gen_shr (tmp1, tmp1, GEN_INT (8)));
   26186           30 :   emit_insn (gen_shl (tmp2, tmp2, GEN_INT (8)));
   26187           30 :   emit_insn (gen_or (tmp1, tmp1, tmp2));
   26188           30 :   emit_move_insn (d->target, lowpart_subreg (d->vmode, tmp1, mode));
   26189              : 
   26190           30 :   return true;
   26191              : }
   26192              : 
   26193              : /* A subroutine of ix86_expand_vec_perm_const_1.  Implement a V4DF
   26194              :    permutation using two vperm2f128, followed by a vshufpd insn blending
   26195              :    the two vectors together.  */
   26196              : 
   26197              : static bool
   26198        31402 : expand_vec_perm_2vperm2f128_vshuf (struct expand_vec_perm_d *d)
   26199              : {
   26200        31402 :   struct expand_vec_perm_d dfirst, dsecond, dthird;
   26201        31402 :   bool ok;
   26202              : 
   26203        31402 :   if (!TARGET_AVX || (d->vmode != V4DFmode))
   26204              :     return false;
   26205              : 
   26206         1149 :   if (d->testing_p)
   26207              :     return true;
   26208              : 
   26209          206 :   dfirst = *d;
   26210          206 :   dsecond = *d;
   26211          206 :   dthird = *d;
   26212              : 
   26213          206 :   dfirst.perm[0] = (d->perm[0] & ~1);
   26214          206 :   dfirst.perm[1] = (d->perm[0] & ~1) + 1;
   26215          206 :   dfirst.perm[2] = (d->perm[2] & ~1);
   26216          206 :   dfirst.perm[3] = (d->perm[2] & ~1) + 1;
   26217          206 :   dsecond.perm[0] = (d->perm[1] & ~1);
   26218          206 :   dsecond.perm[1] = (d->perm[1] & ~1) + 1;
   26219          206 :   dsecond.perm[2] = (d->perm[3] & ~1);
   26220          206 :   dsecond.perm[3] = (d->perm[3] & ~1) + 1;
   26221          206 :   dthird.perm[0] = (d->perm[0] % 2);
   26222          206 :   dthird.perm[1] = (d->perm[1] % 2) + 4;
   26223          206 :   dthird.perm[2] = (d->perm[2] % 2) + 2;
   26224          206 :   dthird.perm[3] = (d->perm[3] % 2) + 6;
   26225              : 
   26226          206 :   dfirst.target = gen_reg_rtx (dfirst.vmode);
   26227          206 :   dsecond.target = gen_reg_rtx (dsecond.vmode);
   26228          206 :   dthird.op0 = dfirst.target;
   26229          206 :   dthird.op1 = dsecond.target;
   26230          206 :   dthird.one_operand_p = false;
   26231              : 
   26232          206 :   canonicalize_perm (&dfirst);
   26233          206 :   canonicalize_perm (&dsecond);
   26234              : 
   26235          206 :   ok = expand_vec_perm_1 (&dfirst)
   26236          206 :        && expand_vec_perm_1 (&dsecond)
   26237          412 :        && expand_vec_perm_1 (&dthird);
   26238              : 
   26239            0 :   gcc_assert (ok);
   26240              : 
   26241              :   return true;
   26242              : }
   26243              : 
   26244              : static bool ix86_expand_vec_perm_const_1 (struct expand_vec_perm_d *);
   26245              : 
   26246              : /* A subroutine of ix86_expand_vec_perm_const_1.  Try to implement
   26247              :    a two vector permutation using two intra-lane vector
   26248              :    permutations, vperm2f128 swapping the lanes and vblend* insn blending
   26249              :    the non-swapped and swapped vectors together.  */
   26250              : 
   26251              : static bool
   26252        16727 : expand_vec_perm2_vperm2f128_vblend (struct expand_vec_perm_d *d)
   26253              : {
   26254        16727 :   struct expand_vec_perm_d dfirst, dsecond, dthird;
   26255        16727 :   unsigned i, j, msk, nelt = d->nelt, nelt2 = nelt / 2, which1 = 0, which2 = 0;
   26256        16727 :   rtx_insn *seq1, *seq2;
   26257        16727 :   bool ok;
   26258        16727 :   rtx (*blend) (rtx, rtx, rtx, rtx) = NULL;
   26259              : 
   26260        16727 :   if (!TARGET_AVX
   26261          794 :       || TARGET_AVX2
   26262          530 :       || (d->vmode != V8SFmode && d->vmode != V4DFmode)
   26263          403 :       || d->one_operand_p)
   26264              :     return false;
   26265              : 
   26266          403 :   dfirst = *d;
   26267          403 :   dsecond = *d;
   26268         3627 :   for (i = 0; i < nelt; i++)
   26269              :     {
   26270         3224 :       dfirst.perm[i] = 0xff;
   26271         3224 :       dsecond.perm[i] = 0xff;
   26272              :     }
   26273         3627 :   for (i = 0, msk = 0; i < nelt; i++)
   26274              :     {
   26275         3224 :       j = (d->perm[i] & nelt2) ? i | nelt2 : i & ~nelt2;
   26276         3224 :       if (j == i)
   26277              :         {
   26278         2498 :           dfirst.perm[j] = d->perm[i];
   26279         4130 :           which1 |= (d->perm[i] < nelt ? 1 : 2);
   26280              :         }
   26281              :       else
   26282              :         {
   26283          726 :           dsecond.perm[j] = d->perm[i];
   26284          726 :           which2 |= (d->perm[i] < nelt ? 1 : 2);
   26285          726 :           msk |= (1U << i);
   26286              :         }
   26287              :     }
   26288          403 :   if (msk == 0 || msk == (1U << nelt) - 1)
   26289              :     return false;
   26290              : 
   26291          403 :   if (!d->testing_p)
   26292              :     {
   26293           40 :       dfirst.target = gen_reg_rtx (dfirst.vmode);
   26294           40 :       dsecond.target = gen_reg_rtx (dsecond.vmode);
   26295              :     }
   26296              : 
   26297         3627 :   for (i = 0; i < nelt; i++)
   26298              :     {
   26299         3224 :       if (dfirst.perm[i] == 0xff)
   26300          726 :         dfirst.perm[i] = (which1 == 2 ? i + nelt : i);
   26301         3224 :       if (dsecond.perm[i] == 0xff)
   26302         2498 :         dsecond.perm[i] = (which2 == 2 ? i + nelt : i);
   26303              :     }
   26304          403 :   canonicalize_perm (&dfirst);
   26305          403 :   start_sequence ();
   26306          403 :   ok = ix86_expand_vec_perm_const_1 (&dfirst);
   26307          403 :   seq1 = end_sequence ();
   26308              : 
   26309          403 :   if (!ok)
   26310              :     return false;
   26311              : 
   26312          403 :   canonicalize_perm (&dsecond);
   26313          403 :   start_sequence ();
   26314          403 :   ok = ix86_expand_vec_perm_const_1 (&dsecond);
   26315          403 :   seq2 = end_sequence ();
   26316              : 
   26317          403 :   if (!ok)
   26318              :     return false;
   26319              : 
   26320          403 :   if (d->testing_p)
   26321              :     return true;
   26322              : 
   26323           40 :   emit_insn (seq1);
   26324           40 :   emit_insn (seq2);
   26325              : 
   26326           40 :   dthird = *d;
   26327           40 :   dthird.op0 = dsecond.target;
   26328           40 :   dthird.op1 = dsecond.target;
   26329           40 :   dthird.one_operand_p = true;
   26330           40 :   dthird.target = gen_reg_rtx (dthird.vmode);
   26331          360 :   for (i = 0; i < nelt; i++)
   26332          320 :     dthird.perm[i] = i ^ nelt2;
   26333              : 
   26334           40 :   ok = expand_vec_perm_1 (&dthird);
   26335           40 :   gcc_assert (ok);
   26336              : 
   26337           40 :   blend = d->vmode == V8SFmode ? gen_avx_blendps256 : gen_avx_blendpd256;
   26338           40 :   emit_insn (blend (d->target, dfirst.target, dthird.target, GEN_INT (msk)));
   26339           40 :   return true;
   26340              : }
   26341              : 
   26342              : /* A subroutine of expand_vec_perm_even_odd_1.  Implement the double-word
   26343              :    permutation with two pshufb insns and an ior.  We should have already
   26344              :    failed all two instruction sequences.  */
   26345              : 
   26346              : static bool
   26347        30274 : expand_vec_perm_pshufb2 (struct expand_vec_perm_d *d)
   26348              : {
   26349        30274 :   rtx rperm[2][16], vperm, l, h, op, m128;
   26350        30274 :   unsigned int i, nelt, eltsz;
   26351        30274 :   machine_mode mode;
   26352        30274 :   rtx (*gen) (rtx, rtx, rtx);
   26353              : 
   26354        34622 :   if (!TARGET_SSSE3 || (GET_MODE_SIZE (d->vmode) != 16
   26355         8606 :                         && GET_MODE_SIZE (d->vmode) != 8
   26356         8566 :                         && GET_MODE_SIZE (d->vmode) != 4))
   26357              :     return false;
   26358         1437 :   gcc_assert (!d->one_operand_p);
   26359              : 
   26360         1437 :   if (d->testing_p)
   26361              :     return true;
   26362              : 
   26363          202 :   switch (GET_MODE_SIZE (d->vmode))
   26364              :     {
   26365              :     case 4:
   26366              :       mode = V4QImode;
   26367              :       gen = gen_mmx_pshufbv4qi3;
   26368              :       break;
   26369           20 :     case 8:
   26370           20 :       mode = V8QImode;
   26371           20 :       gen = gen_mmx_pshufbv8qi3;
   26372           20 :       break;
   26373           45 :     case 16:
   26374           45 :       mode = V16QImode;
   26375           45 :       gen = gen_ssse3_pshufbv16qi3;
   26376           45 :       break;
   26377            0 :     default:
   26378            0 :       gcc_unreachable ();
   26379              :     }
   26380              : 
   26381          101 :   nelt = d->nelt;
   26382          101 :   eltsz = GET_MODE_UNIT_SIZE (d->vmode);
   26383              : 
   26384              :   /* Generate two permutation masks.  If the required element is within
   26385              :      the given vector it is shuffled into the proper lane.  If the required
   26386              :      element is in the other vector, force a zero into the lane by setting
   26387              :      bit 7 in the permutation mask.  */
   26388          101 :   m128 = GEN_INT (-128);
   26389         1130 :   for (i = 0; i < nelt; ++i)
   26390              :     {
   26391          928 :       unsigned j, k, e = d->perm[i];
   26392          928 :       unsigned which = (e >= nelt);
   26393          928 :       if (e >= nelt)
   26394          480 :         e -= nelt;
   26395              : 
   26396         1952 :       for (j = 0; j < eltsz; ++j)
   26397              :         {
   26398         1024 :           rperm[which][i*eltsz + j] = GEN_INT (e*eltsz + j);
   26399         1024 :           rperm[1-which][i*eltsz + j] = m128;
   26400              :         }
   26401              : 
   26402         9024 :       for (k = i*eltsz + j; k < 16; ++k)
   26403         8096 :         rperm[0][k] = rperm[1][k] = m128;
   26404              :     }
   26405              : 
   26406          101 :   vperm = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, rperm[0]));
   26407          101 :   vperm = force_reg (V16QImode, vperm);
   26408              : 
   26409          101 :   l = gen_reg_rtx (mode);
   26410          101 :   op = gen_lowpart (mode, d->op0);
   26411          101 :   emit_insn (gen (l, op, vperm));
   26412              : 
   26413          101 :   vperm = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, rperm[1]));
   26414          101 :   vperm = force_reg (V16QImode, vperm);
   26415              : 
   26416          101 :   h = gen_reg_rtx (mode);
   26417          101 :   op = gen_lowpart (mode, d->op1);
   26418          101 :   emit_insn (gen (h, op, vperm));
   26419              : 
   26420          101 :   op = d->target;
   26421          101 :   if (d->vmode != mode)
   26422           22 :     op = gen_reg_rtx (mode);
   26423          101 :   ix86_emit_vec_binop (IOR, mode, op, l, h);
   26424          101 :   if (op != d->target)
   26425           22 :     emit_move_insn (d->target, gen_lowpart (d->vmode, op));
   26426              : 
   26427              :   return true;
   26428              : }
   26429              : 
   26430              : /* Implement arbitrary permutation of one V32QImode and V16QImode operand
   26431              :    with two vpshufb insns, vpermq and vpor.  We should have already failed
   26432              :    all two or three instruction sequences.  */
   26433              : 
   26434              : static bool
   26435        25063 : expand_vec_perm_vpshufb2_vpermq (struct expand_vec_perm_d *d)
   26436              : {
   26437        25063 :   rtx rperm[2][32], vperm, l, h, hp, op, m128;
   26438        25063 :   unsigned int i, nelt, eltsz;
   26439              : 
   26440        25063 :   if (!TARGET_AVX2
   26441          374 :       || !d->one_operand_p
   26442          172 :       || (d->vmode != V32QImode && d->vmode != V16HImode))
   26443              :     return false;
   26444              : 
   26445            7 :   if (d->testing_p)
   26446              :     return true;
   26447              : 
   26448            7 :   nelt = d->nelt;
   26449            7 :   eltsz = GET_MODE_UNIT_SIZE (d->vmode);
   26450              : 
   26451              :   /* Generate two permutation masks.  If the required element is within
   26452              :      the same lane, it is shuffled in.  If the required element from the
   26453              :      other lane, force a zero by setting bit 7 in the permutation mask.
   26454              :      In the other mask the mask has non-negative elements if element
   26455              :      is requested from the other lane, but also moved to the other lane,
   26456              :      so that the result of vpshufb can have the two V2TImode halves
   26457              :      swapped.  */
   26458            7 :   m128 = GEN_INT (-128);
   26459          206 :   for (i = 0; i < nelt; ++i)
   26460              :     {
   26461          192 :       unsigned j, e = d->perm[i] & (nelt / 2 - 1);
   26462          192 :       unsigned which = ((d->perm[i] ^ i) & (nelt / 2)) * eltsz;
   26463              : 
   26464          416 :       for (j = 0; j < eltsz; ++j)
   26465              :         {
   26466          224 :           rperm[!!which][(i * eltsz + j) ^ which] = GEN_INT (e * eltsz + j);
   26467          224 :           rperm[!which][(i * eltsz + j) ^ (which ^ 16)] = m128;
   26468              :         }
   26469              :     }
   26470              : 
   26471            7 :   vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[1]));
   26472            7 :   vperm = force_reg (V32QImode, vperm);
   26473              : 
   26474            7 :   h = gen_reg_rtx (V32QImode);
   26475            7 :   op = gen_lowpart (V32QImode, d->op0);
   26476            7 :   emit_insn (gen_avx2_pshufbv32qi3 (h, op, vperm));
   26477              : 
   26478              :   /* Swap the 128-byte lanes of h into hp.  */
   26479            7 :   hp = gen_reg_rtx (V4DImode);
   26480            7 :   op = gen_lowpart (V4DImode, h);
   26481            7 :   emit_insn (gen_avx2_permv4di_1 (hp, op, const2_rtx, GEN_INT (3), const0_rtx,
   26482              :                                   const1_rtx));
   26483              : 
   26484            7 :   vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[0]));
   26485            7 :   vperm = force_reg (V32QImode, vperm);
   26486              : 
   26487            7 :   l = gen_reg_rtx (V32QImode);
   26488            7 :   op = gen_lowpart (V32QImode, d->op0);
   26489            7 :   emit_insn (gen_avx2_pshufbv32qi3 (l, op, vperm));
   26490              : 
   26491            7 :   op = d->target;
   26492            7 :   if (d->vmode != V32QImode)
   26493            2 :     op = gen_reg_rtx (V32QImode);
   26494            7 :   emit_insn (gen_iorv32qi3 (op, l, gen_lowpart (V32QImode, hp)));
   26495            7 :   if (op != d->target)
   26496            2 :     emit_move_insn (d->target, gen_lowpart (d->vmode, op));
   26497              : 
   26498              :   return true;
   26499              : }
   26500              : 
   26501              : /* A subroutine of expand_vec_perm_even_odd_1.  Implement extract-even
   26502              :    and extract-odd permutations of two V32QImode and V16QImode operand
   26503              :    with two vpshufb insns, vpor and vpermq.  We should have already
   26504              :    failed all two or three instruction sequences.  */
   26505              : 
   26506              : static bool
   26507        25056 : expand_vec_perm_vpshufb2_vpermq_even_odd (struct expand_vec_perm_d *d)
   26508              : {
   26509        25056 :   rtx rperm[2][32], vperm, l, h, ior, op, m128;
   26510        25056 :   unsigned int i, nelt, eltsz;
   26511              : 
   26512        25056 :   if (!TARGET_AVX2
   26513          367 :       || d->one_operand_p
   26514          202 :       || (d->vmode != V32QImode && d->vmode != V16HImode))
   26515              :     return false;
   26516              : 
   26517          112 :   for (i = 0; i < d->nelt; ++i)
   26518          112 :     if ((d->perm[i] ^ (i * 2)) & (3 * d->nelt / 2))
   26519              :       return false;
   26520              : 
   26521            0 :   if (d->testing_p)
   26522              :     return true;
   26523              : 
   26524            0 :   nelt = d->nelt;
   26525            0 :   eltsz = GET_MODE_UNIT_SIZE (d->vmode);
   26526              : 
   26527              :   /* Generate two permutation masks.  In the first permutation mask
   26528              :      the first quarter will contain indexes for the first half
   26529              :      of the op0, the second quarter will contain bit 7 set, third quarter
   26530              :      will contain indexes for the second half of the op0 and the
   26531              :      last quarter bit 7 set.  In the second permutation mask
   26532              :      the first quarter will contain bit 7 set, the second quarter
   26533              :      indexes for the first half of the op1, the third quarter bit 7 set
   26534              :      and last quarter indexes for the second half of the op1.
   26535              :      I.e. the first mask e.g. for V32QImode extract even will be:
   26536              :      0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128
   26537              :      (all values masked with 0xf except for -128) and second mask
   26538              :      for extract even will be
   26539              :      -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe.  */
   26540            0 :   m128 = GEN_INT (-128);
   26541            0 :   for (i = 0; i < nelt; ++i)
   26542              :     {
   26543            0 :       unsigned j, e = d->perm[i] & (nelt / 2 - 1);
   26544            0 :       unsigned which = d->perm[i] >= nelt;
   26545            0 :       unsigned xorv = (i >= nelt / 4 && i < 3 * nelt / 4) ? 24 : 0;
   26546              : 
   26547            0 :       for (j = 0; j < eltsz; ++j)
   26548              :         {
   26549            0 :           rperm[which][(i * eltsz + j) ^ xorv] = GEN_INT (e * eltsz + j);
   26550            0 :           rperm[1 - which][(i * eltsz + j) ^ xorv] = m128;
   26551              :         }
   26552              :     }
   26553              : 
   26554            0 :   vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[0]));
   26555            0 :   vperm = force_reg (V32QImode, vperm);
   26556              : 
   26557            0 :   l = gen_reg_rtx (V32QImode);
   26558            0 :   op = gen_lowpart (V32QImode, d->op0);
   26559            0 :   emit_insn (gen_avx2_pshufbv32qi3 (l, op, vperm));
   26560              : 
   26561            0 :   vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[1]));
   26562            0 :   vperm = force_reg (V32QImode, vperm);
   26563              : 
   26564            0 :   h = gen_reg_rtx (V32QImode);
   26565            0 :   op = gen_lowpart (V32QImode, d->op1);
   26566            0 :   emit_insn (gen_avx2_pshufbv32qi3 (h, op, vperm));
   26567              : 
   26568            0 :   ior = gen_reg_rtx (V32QImode);
   26569            0 :   emit_insn (gen_iorv32qi3 (ior, l, h));
   26570              : 
   26571              :   /* Permute the V4DImode quarters using { 0, 2, 1, 3 } permutation.  */
   26572            0 :   op = gen_reg_rtx (V4DImode);
   26573            0 :   ior = gen_lowpart (V4DImode, ior);
   26574            0 :   emit_insn (gen_avx2_permv4di_1 (op, ior, const0_rtx, const2_rtx,
   26575              :                                   const1_rtx, GEN_INT (3)));
   26576            0 :   emit_move_insn (d->target, gen_lowpart (d->vmode, op));
   26577              : 
   26578            0 :   return true;
   26579              : }
   26580              : 
   26581              : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement a
   26582              :    permutation (which is a bland) with and, andnot and or when pshufb is not available.
   26583              : 
   26584              :    It handles case:
   26585              :    __builtin_shufflevector (v1, v2, 0, 9, 2, 11, 4, 13, 6, 15);
   26586              :    __builtin_shufflevector (v1, v2, 8, 1, 2, 11, 4, 13, 6, 15);
   26587              : 
   26588              :    An element[i] must be chosen between op0[i] and op1[i] to satisfy the
   26589              :    requirement.
   26590              :  */
   26591              : 
   26592              : static bool
   26593        26172 : expand_vec_perm_pand_pandn_por (struct expand_vec_perm_d *d)
   26594              : {
   26595        26172 :   rtx rperm[16], vperm;
   26596        26172 :   unsigned int i, nelt = d->nelt;
   26597              : 
   26598        26172 :   if (!TARGET_SSE2
   26599        26172 :       || d->one_operand_p
   26600        22024 :       || (d->vmode != V16QImode && d->vmode != V8HImode))
   26601              :     return false;
   26602              : 
   26603         8356 :   if (d->perm[0] != 0)
   26604              :     return false;
   26605              : 
   26606              :   /* The dest[i] must select an element between op0[i] and op1[i].  */
   26607        17225 :   for (i = 1; i < nelt; i++)
   26608        16120 :     if ((d->perm[i] % nelt) != i)
   26609              :       return false;
   26610              : 
   26611         1105 :   if (d->testing_p)
   26612              :      return true;
   26613              : 
   26614              :   /* Generates a blend mask for the operators AND and ANDNOT.  */
   26615          112 :   machine_mode inner_mode = GET_MODE_INNER (d->vmode);
   26616         1216 :   for (i = 0; i < nelt; i++)
   26617         1667 :     rperm[i] = (d->perm[i] <  nelt) ? CONSTM1_RTX (inner_mode)
   26618          563 :       : CONST0_RTX (inner_mode);
   26619              : 
   26620          112 :   vperm = gen_rtx_CONST_VECTOR (d->vmode, gen_rtvec_v (nelt, rperm));
   26621          112 :   vperm = force_reg (d->vmode, vperm);
   26622              : 
   26623          112 :   ix86_expand_sse_movcc (d->target, vperm, d->op0, d->op1);
   26624              : 
   26625          112 :   return true;
   26626              : }
   26627              : 
   26628              : /* Implement permutation with pslldq + psrldq + por when pshufb is not
   26629              :    available.  */
   26630              : static bool
   26631        45954 : expand_vec_perm_pslldq_psrldq_por (struct expand_vec_perm_d *d, bool pandn)
   26632              : {
   26633        45954 :   unsigned i, nelt = d->nelt;
   26634        45954 :   unsigned start1, end1 = -1;
   26635        45954 :   machine_mode vmode = d->vmode, imode;
   26636        45954 :   int start2 = -1;
   26637        45954 :   bool clear_op0, clear_op1;
   26638        45954 :   unsigned inner_size;
   26639        45954 :   rtx op0, op1, dop1;
   26640        45954 :   rtx (*gen_vec_shr) (rtx, rtx, rtx);
   26641        45954 :   rtx (*gen_vec_shl) (rtx, rtx, rtx);
   26642              : 
   26643              :   /* pshufd can be used for V4SI/V2DI under TARGET_SSE2.  */
   26644        45954 :   if (!TARGET_SSE2 || (vmode != E_V16QImode && vmode != E_V8HImode))
   26645              :     return false;
   26646              : 
   26647        15147 :   start1 = d->perm[0];
   26648        47340 :   for (i = 1; i < nelt; i++)
   26649              :     {
   26650        46333 :       if (d->perm[i] != d->perm[i-1] + 1
   26651        17654 :           || d->perm[i] == nelt)
   26652              :         {
   26653        29287 :           if (start2 == -1)
   26654              :             {
   26655        15147 :               start2 = d->perm[i];
   26656        15147 :               end1 = d->perm[i-1];
   26657              :             }
   26658              :           else
   26659              :             return false;
   26660              :         }
   26661              :     }
   26662              : 
   26663         1007 :   clear_op0 = end1 != nelt - 1;
   26664         1007 :   clear_op1 = start2 % nelt != 0;
   26665              :   /* pandn/pand is needed to clear upper/lower bits of op0/op1.  */
   26666         1007 :   if (!pandn && (clear_op0 || clear_op1))
   26667              :     return false;
   26668              : 
   26669          649 :   if (d->testing_p)
   26670              :     return true;
   26671              : 
   26672           89 :   gen_vec_shr = vmode == E_V16QImode ? gen_vec_shr_v16qi : gen_vec_shr_v8hi;
   26673           28 :   gen_vec_shl = vmode == E_V16QImode ? gen_vec_shl_v16qi : gen_vec_shl_v8hi;
   26674           89 :   imode = GET_MODE_INNER (vmode);
   26675           89 :   inner_size = GET_MODE_BITSIZE (imode);
   26676           89 :   op0 = gen_reg_rtx (vmode);
   26677           89 :   op1 = gen_reg_rtx (vmode);
   26678              : 
   26679           89 :   if (start1)
   26680           84 :     emit_insn (gen_vec_shr (op0, d->op0, GEN_INT (start1 * inner_size)));
   26681              :   else
   26682            5 :     emit_move_insn (op0, d->op0);
   26683              : 
   26684           89 :   dop1 = d->op1;
   26685           89 :   if (d->one_operand_p)
   26686           64 :     dop1 = d->op0;
   26687              : 
   26688           89 :   int shl_offset = end1 - start1 + 1 - start2 % nelt;
   26689           89 :   if (shl_offset)
   26690           49 :     emit_insn (gen_vec_shl (op1, dop1, GEN_INT (shl_offset * inner_size)));
   26691              :   else
   26692           40 :     emit_move_insn (op1, dop1);
   26693              : 
   26694              :   /* Clear lower/upper bits for op0/op1.  */
   26695           89 :   if (clear_op0 || clear_op1)
   26696              :     {
   26697              :       rtx vec[16];
   26698              :       rtx const_vec;
   26699              :       rtx clear;
   26700          989 :       for (i = 0; i != nelt; i++)
   26701              :         {
   26702          920 :           if (i < (end1 - start1 + 1))
   26703          346 :             vec[i] = gen_int_mode ((HOST_WIDE_INT_1U << inner_size) - 1, imode);
   26704              :           else
   26705          574 :             vec[i] = CONST0_RTX (imode);
   26706              :         }
   26707           69 :       const_vec = gen_rtx_CONST_VECTOR (vmode, gen_rtvec_v (nelt, vec));
   26708           69 :       const_vec = validize_mem (force_const_mem (vmode, const_vec));
   26709           69 :       clear = force_reg (vmode, const_vec);
   26710              : 
   26711           69 :       if (clear_op0)
   26712           61 :         emit_move_insn (op0, gen_rtx_AND (vmode, op0, clear));
   26713           69 :       if (clear_op1)
   26714           56 :         emit_move_insn (op1, gen_rtx_AND (vmode,
   26715              :                                           gen_rtx_NOT (vmode, clear),
   26716              :                                           op1));
   26717              :     }
   26718              : 
   26719           89 :   emit_move_insn (d->target, gen_rtx_IOR (vmode, op0, op1));
   26720           89 :   return true;
   26721              : }
   26722              : 
   26723              : /* A subroutine of expand_vec_perm_even_odd_1.  Implement extract-even
   26724              :    and extract-odd permutations of two V8QI, V8HI, V16QI, V16HI or V32QI
   26725              :    operands with two "and" and "pack" or two "shift" and "pack" insns.
   26726              :    We should have already failed all two instruction sequences.  */
   26727              : 
   26728              : static bool
   26729        47988 : expand_vec_perm_even_odd_pack (struct expand_vec_perm_d *d)
   26730              : {
   26731        47988 :   rtx op, dop0, dop1, t;
   26732        47988 :   unsigned i, odd, c, s, nelt = d->nelt;
   26733        47988 :   int pblendw_i = 0;
   26734        47988 :   bool end_perm = false;
   26735        47988 :   machine_mode half_mode;
   26736        47988 :   rtx (*gen_and) (rtx, rtx, rtx);
   26737        47988 :   rtx (*gen_pack) (rtx, rtx, rtx);
   26738        47988 :   rtx (*gen_shift) (rtx, rtx, rtx);
   26739              : 
   26740        47988 :   if (d->one_operand_p)
   26741              :     return false;
   26742              : 
   26743        42343 :   switch (d->vmode)
   26744              :     {
   26745         4787 :     case E_V4HImode:
   26746              :       /* Required for "pack".  */
   26747         4787 :       if (!TARGET_SSE4_1)
   26748              :         return false;
   26749              :       c = 0xffff;
   26750              :       s = 16;
   26751              :       half_mode = V2SImode;
   26752              :       gen_and = gen_andv2si3;
   26753              :       gen_pack = gen_mmx_packusdw;
   26754              :       gen_shift = gen_lshrv2si3;
   26755              :       pblendw_i = 0x5;
   26756              :       break;
   26757         6215 :     case E_V8HImode:
   26758              :       /* Required for "pack".  */
   26759         6215 :       if (!TARGET_SSE4_1)
   26760              :         return false;
   26761              :       c = 0xffff;
   26762              :       s = 16;
   26763              :       half_mode = V4SImode;
   26764              :       gen_and = gen_andv4si3;
   26765              :       gen_pack = gen_sse4_1_packusdw;
   26766              :       gen_shift = gen_lshrv4si3;
   26767              :       pblendw_i = 0x55;
   26768              :       break;
   26769              :     case E_V8QImode:
   26770              :       /* No check as all instructions are SSE2.  */
   26771              :       c = 0xff;
   26772              :       s = 8;
   26773              :       half_mode = V4HImode;
   26774              :       gen_and = gen_andv4hi3;
   26775              :       gen_pack = gen_mmx_packuswb;
   26776              :       gen_shift = gen_lshrv4hi3;
   26777              :       break;
   26778        14773 :     case E_V16QImode:
   26779              :       /* No check as all instructions are SSE2.  */
   26780        14773 :       c = 0xff;
   26781        14773 :       s = 8;
   26782        14773 :       half_mode = V8HImode;
   26783        14773 :       gen_and = gen_andv8hi3;
   26784        14773 :       gen_pack = gen_sse2_packuswb;
   26785        14773 :       gen_shift = gen_lshrv8hi3;
   26786        14773 :       break;
   26787          440 :     case E_V16HImode:
   26788          440 :       if (!TARGET_AVX2)
   26789              :         return false;
   26790              :       c = 0xffff;
   26791              :       s = 16;
   26792              :       half_mode = V8SImode;
   26793              :       gen_and = gen_andv8si3;
   26794              :       gen_pack = gen_avx2_packusdw;
   26795              :       gen_shift = gen_lshrv8si3;
   26796              :       pblendw_i = 0x5555;
   26797              :       end_perm = true;
   26798              :       break;
   26799          269 :     case E_V32QImode:
   26800          269 :       if (!TARGET_AVX2)
   26801              :         return false;
   26802              :       c = 0xff;
   26803              :       s = 8;
   26804              :       half_mode = V16HImode;
   26805              :       gen_and = gen_andv16hi3;
   26806              :       gen_pack = gen_avx2_packuswb;
   26807              :       gen_shift = gen_lshrv16hi3;
   26808              :       end_perm = true;
   26809              :       break;
   26810              :     default:
   26811              :       /* Only V4HI, V8QI, V8HI, V16QI, V16HI and V32QI modes
   26812              :          are more profitable than general shuffles.  */
   26813              :       return false;
   26814              :     }
   26815              : 
   26816              :   /* Check that permutation is even or odd.  */
   26817        20933 :   odd = d->perm[0];
   26818        20933 :   if (odd > 1)
   26819              :     return false;
   26820              : 
   26821       234419 :   for (i = 1; i < nelt; ++i)
   26822       217893 :     if (d->perm[i] != 2 * i + odd)
   26823              :       return false;
   26824              : 
   26825        16526 :   if (d->testing_p)
   26826              :     return true;
   26827              : 
   26828         5714 :   dop0 = gen_reg_rtx (half_mode);
   26829         5714 :   dop1 = gen_reg_rtx (half_mode);
   26830         5714 :   if (odd == 0)
   26831              :     {
   26832              :       /* Use pblendw since const_vector 0 should be cheaper than
   26833              :          const_vector 0xffff.  */
   26834         4993 :       if (d->vmode == V4HImode
   26835              :           || d->vmode == E_V8HImode
   26836              :           || d->vmode == E_V16HImode)
   26837              :         {
   26838          876 :           rtx dop0_t = gen_reg_rtx (d->vmode);
   26839          876 :           rtx dop1_t = gen_reg_rtx (d->vmode);
   26840          876 :           t = gen_reg_rtx (d->vmode);
   26841          876 :           emit_move_insn (t, CONST0_RTX (d->vmode));
   26842              : 
   26843          876 :           emit_move_insn (dop0_t, gen_rtx_VEC_MERGE (d->vmode, d->op0, t,
   26844              :                                                      GEN_INT (pblendw_i)));
   26845          876 :           emit_move_insn (dop1_t, gen_rtx_VEC_MERGE (d->vmode, d->op1, t,
   26846              :                                                      GEN_INT (pblendw_i)));
   26847              : 
   26848          876 :           emit_move_insn (dop0, gen_lowpart (half_mode, dop0_t));
   26849          876 :           emit_move_insn (dop1, gen_lowpart (half_mode, dop1_t));
   26850          876 :         }
   26851              :       else
   26852              :         {
   26853         4117 :           t = gen_const_vec_duplicate (half_mode, GEN_INT (c));
   26854         4117 :           t = force_reg (half_mode, t);
   26855         4117 :           emit_insn (gen_and (dop0, t, gen_lowpart (half_mode, d->op0)));
   26856         4117 :           emit_insn (gen_and (dop1, t, gen_lowpart (half_mode, d->op1)));
   26857              :         }
   26858              :     }
   26859              :   else
   26860              :     {
   26861         1442 :       emit_insn (gen_shift (dop0,
   26862          721 :                             gen_lowpart (half_mode, d->op0),
   26863              :                             GEN_INT (s)));
   26864         1442 :       emit_insn (gen_shift (dop1,
   26865          721 :                             gen_lowpart (half_mode, d->op1),
   26866              :                             GEN_INT (s)));
   26867              :     }
   26868              :   /* In AVX2 for 256 bit case we need to permute pack result.  */
   26869         5714 :   if (TARGET_AVX2 && end_perm)
   26870              :     {
   26871          412 :       op = gen_reg_rtx (d->vmode);
   26872          412 :       t = gen_reg_rtx (V4DImode);
   26873          412 :       emit_insn (gen_pack (op, dop0, dop1));
   26874          824 :       emit_insn (gen_avx2_permv4di_1 (t,
   26875          412 :                                       gen_lowpart (V4DImode, op),
   26876              :                                       const0_rtx,
   26877              :                                       const2_rtx,
   26878              :                                       const1_rtx,
   26879              :                                       GEN_INT (3)));
   26880          412 :       emit_move_insn (d->target, gen_lowpart (d->vmode, t));
   26881              :     }
   26882              :   else
   26883         5302 :     emit_insn (gen_pack (d->target, dop0, dop1));
   26884              : 
   26885              :   return true;
   26886              : }
   26887              : 
   26888              : /* A subroutine of expand_vec_perm_even_odd_1.  Implement extract-even
   26889              :    and extract-odd permutations of two V64QI operands
   26890              :    with two "shifts", two "truncs" and one "concat" insns for "odd"
   26891              :    and two "truncs" and one concat insn for "even."
   26892              :    Have already failed all two instruction sequences.  */
   26893              : 
   26894              : static bool
   26895        25093 : expand_vec_perm_even_odd_trunc (struct expand_vec_perm_d *d)
   26896              : {
   26897        25093 :   rtx t1, t2, t3, t4;
   26898        25093 :   unsigned i, odd, nelt = d->nelt;
   26899              : 
   26900        25093 :   if (!TARGET_AVX512BW
   26901           67 :       || d->one_operand_p
   26902           31 :       || d->vmode != V64QImode)
   26903              :     return false;
   26904              : 
   26905              :   /* Check that permutation is even or odd.  */
   26906           31 :   odd = d->perm[0];
   26907           31 :   if (odd > 1)
   26908              :     return false;
   26909              : 
   26910         1214 :   for (i = 1; i < nelt; ++i)
   26911         1196 :     if (d->perm[i] != 2 * i + odd)
   26912              :       return false;
   26913              : 
   26914           18 :   if (d->testing_p)
   26915              :     return true;
   26916              : 
   26917              : 
   26918           18 :   if (odd)
   26919              :     {
   26920            1 :       t1 = gen_reg_rtx (V32HImode);
   26921            1 :       t2 = gen_reg_rtx (V32HImode);
   26922            2 :       emit_insn (gen_lshrv32hi3 (t1,
   26923            1 :                                  gen_lowpart (V32HImode, d->op0),
   26924              :                                  GEN_INT (8)));
   26925            2 :       emit_insn (gen_lshrv32hi3 (t2,
   26926            1 :                                  gen_lowpart (V32HImode, d->op1),
   26927              :                                  GEN_INT (8)));
   26928              :     }
   26929              :   else
   26930              :     {
   26931           17 :       t1 = gen_lowpart (V32HImode, d->op0);
   26932           17 :       t2 = gen_lowpart (V32HImode, d->op1);
   26933              :     }
   26934              : 
   26935           18 :   t3 = gen_reg_rtx (V32QImode);
   26936           18 :   t4 = gen_reg_rtx (V32QImode);
   26937           18 :   emit_insn (gen_avx512bw_truncatev32hiv32qi2 (t3, t1));
   26938           18 :   emit_insn (gen_avx512bw_truncatev32hiv32qi2 (t4, t2));
   26939           18 :   emit_insn (gen_avx_vec_concatv64qi (d->target, t3, t4));
   26940              : 
   26941           18 :   return true;
   26942              : }
   26943              : 
   26944              : /* A subroutine of ix86_expand_vec_perm_const_1.  Implement extract-even
   26945              :    and extract-odd permutations.  */
   26946              : 
   26947              : static bool
   26948        13262 : expand_vec_perm_even_odd_1 (struct expand_vec_perm_d *d, unsigned odd)
   26949              : {
   26950        13262 :   rtx t1, t2, t3, t4, t5;
   26951              : 
   26952        13262 :   switch (d->vmode)
   26953              :     {
   26954           19 :     case E_V4DFmode:
   26955           19 :       if (d->testing_p)
   26956              :         break;
   26957            1 :       t1 = gen_reg_rtx (V4DFmode);
   26958            1 :       t2 = gen_reg_rtx (V4DFmode);
   26959              : 
   26960              :       /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }.  */
   26961            1 :       emit_insn (gen_avx_vperm2f128v4df3 (t1, d->op0, d->op1, GEN_INT (0x20)));
   26962            1 :       emit_insn (gen_avx_vperm2f128v4df3 (t2, d->op0, d->op1, GEN_INT (0x31)));
   26963              : 
   26964              :       /* Now an unpck[lh]pd will produce the result required.  */
   26965            1 :       if (odd)
   26966            0 :         t3 = gen_avx_unpckhpd256 (d->target, t1, t2);
   26967              :       else
   26968            1 :         t3 = gen_avx_unpcklpd256 (d->target, t1, t2);
   26969            1 :       emit_insn (t3);
   26970            1 :       break;
   26971              : 
   26972         1086 :     case E_V8SFmode:
   26973         1086 :       {
   26974         1086 :         int mask = odd ? 0xdd : 0x88;
   26975              : 
   26976         1086 :         if (d->testing_p)
   26977              :           break;
   26978          186 :         t1 = gen_reg_rtx (V8SFmode);
   26979          186 :         t2 = gen_reg_rtx (V8SFmode);
   26980          186 :         t3 = gen_reg_rtx (V8SFmode);
   26981              : 
   26982              :         /* Shuffle within the 128-bit lanes to produce:
   26983              :            { 0 2 8 a 4 6 c e } | { 1 3 9 b 5 7 d f }.  */
   26984          186 :         emit_insn (gen_avx_shufps256 (t1, d->op0, d->op1,
   26985              :                                       GEN_INT (mask)));
   26986              : 
   26987              :         /* Shuffle the lanes around to produce:
   26988              :            { 4 6 c e 0 2 8 a } and { 5 7 d f 1 3 9 b }.  */
   26989          186 :         emit_insn (gen_avx_vperm2f128v8sf3 (t2, t1, t1,
   26990              :                                             GEN_INT (0x3)));
   26991              : 
   26992              :         /* Shuffle within the 128-bit lanes to produce:
   26993              :            { 0 2 4 6 4 6 0 2 } | { 1 3 5 7 5 7 1 3 }.  */
   26994          186 :         emit_insn (gen_avx_shufps256 (t3, t1, t2, GEN_INT (0x44)));
   26995              : 
   26996              :         /* Shuffle within the 128-bit lanes to produce:
   26997              :            { 8 a c e c e 8 a } | { 9 b d f d f 9 b }.  */
   26998          186 :         emit_insn (gen_avx_shufps256 (t2, t1, t2, GEN_INT (0xee)));
   26999              : 
   27000              :         /* Shuffle the lanes around to produce:
   27001              :            { 0 2 4 6 8 a c e } | { 1 3 5 7 9 b d f }.  */
   27002          186 :         emit_insn (gen_avx_vperm2f128v8sf3 (d->target, t3, t2,
   27003              :                                             GEN_INT (0x20)));
   27004              :       }
   27005          186 :       break;
   27006              : 
   27007            0 :     case E_V2DFmode:
   27008            0 :     case E_V4SFmode:
   27009            0 :     case E_V2DImode:
   27010            0 :     case E_V2SImode:
   27011            0 :     case E_V4SImode:
   27012            0 :     case E_V2HImode:
   27013              :       /* These are always directly implementable by expand_vec_perm_1.  */
   27014            0 :       gcc_unreachable ();
   27015              : 
   27016            0 :     case E_V2SFmode:
   27017            0 :       gcc_assert (TARGET_MMX_WITH_SSE);
   27018              :       /* We have no suitable instructions.  */
   27019            0 :       if (d->testing_p)
   27020            0 :         return false;
   27021              :       break;
   27022              : 
   27023         1580 :     case E_V4QImode:
   27024         1580 :       if (TARGET_SSSE3 && !TARGET_SLOW_PSHUFB)
   27025            0 :         return expand_vec_perm_pshufb2 (d);
   27026              :       else
   27027              :         {
   27028         1580 :           if (d->testing_p)
   27029              :             break;
   27030              :           /* We need 2*log2(N)-1 operations to achieve odd/even
   27031              :              with interleave. */
   27032          180 :           t1 = gen_reg_rtx (V4QImode);
   27033          180 :           emit_insn (gen_mmx_punpckhbw_low (t1, d->op0, d->op1));
   27034          180 :           emit_insn (gen_mmx_punpcklbw_low (d->target, d->op0, d->op1));
   27035          180 :           if (odd)
   27036           41 :             t2 = gen_mmx_punpckhbw_low (d->target, d->target, t1);
   27037              :           else
   27038          139 :             t2 = gen_mmx_punpcklbw_low (d->target, d->target, t1);
   27039          180 :           emit_insn (t2);
   27040              :         }
   27041          180 :       break;
   27042              : 
   27043         1596 :     case E_V4HImode:
   27044         1596 :       if (TARGET_SSE4_1)
   27045           92 :         return expand_vec_perm_even_odd_pack (d);
   27046         1504 :       else if (TARGET_SSSE3 && !TARGET_SLOW_PSHUFB)
   27047           20 :         return expand_vec_perm_pshufb2 (d);
   27048              :       else
   27049              :         {
   27050         1484 :           if (d->testing_p)
   27051              :             break;
   27052              :           /* We need 2*log2(N)-1 operations to achieve odd/even
   27053              :              with interleave. */
   27054          495 :           t1 = gen_reg_rtx (V4HImode);
   27055          495 :           emit_insn (gen_mmx_punpckhwd (t1, d->op0, d->op1));
   27056          495 :           emit_insn (gen_mmx_punpcklwd (d->target, d->op0, d->op1));
   27057          495 :           if (odd)
   27058            8 :             t2 = gen_mmx_punpckhwd (d->target, d->target, t1);
   27059              :           else
   27060          487 :             t2 = gen_mmx_punpcklwd (d->target, d->target, t1);
   27061          495 :           emit_insn (t2);
   27062              :         }
   27063          495 :       break;
   27064              : 
   27065         6892 :     case E_V8HImode:
   27066         6892 :       if (TARGET_SSE4_1)
   27067          444 :         return expand_vec_perm_even_odd_pack (d);
   27068         6448 :       else if (TARGET_SSSE3 && !TARGET_SLOW_PSHUFB)
   27069            1 :         return expand_vec_perm_pshufb2 (d);
   27070              :       else
   27071              :         {
   27072         6447 :           if (d->testing_p)
   27073              :             break;
   27074              :           /* We need 2*log2(N)-1 operations to achieve odd/even
   27075              :              with interleave. */
   27076         2867 :           t1 = gen_reg_rtx (V8HImode);
   27077         2867 :           t2 = gen_reg_rtx (V8HImode);
   27078         2867 :           emit_insn (gen_vec_interleave_highv8hi (t1, d->op0, d->op1));
   27079         2867 :           emit_insn (gen_vec_interleave_lowv8hi (d->target, d->op0, d->op1));
   27080         2867 :           emit_insn (gen_vec_interleave_highv8hi (t2, d->target, t1));
   27081         2867 :           emit_insn (gen_vec_interleave_lowv8hi (d->target, d->target, t1));
   27082         2867 :           if (odd)
   27083           97 :             t3 = gen_vec_interleave_highv8hi (d->target, d->target, t2);
   27084              :           else
   27085         2770 :             t3 = gen_vec_interleave_lowv8hi (d->target, d->target, t2);
   27086         2867 :           emit_insn (t3);
   27087              :         }
   27088         2867 :       break;
   27089              : 
   27090         1400 :     case E_V8QImode:
   27091         1400 :     case E_V16QImode:
   27092         1400 :       return expand_vec_perm_even_odd_pack (d);
   27093              : 
   27094          468 :     case E_V16HImode:
   27095          468 :     case E_V32QImode:
   27096          468 :       return expand_vec_perm_even_odd_pack (d);
   27097              : 
   27098           26 :     case E_V64QImode:
   27099           26 :       return expand_vec_perm_even_odd_trunc (d);
   27100              : 
   27101           19 :     case E_V4DImode:
   27102           19 :       if (!TARGET_AVX2)
   27103              :         {
   27104           19 :           struct expand_vec_perm_d d_copy = *d;
   27105           19 :           d_copy.vmode = V4DFmode;
   27106           19 :           if (d->testing_p)
   27107           18 :             d_copy.target = gen_raw_REG (V4DFmode, LAST_VIRTUAL_REGISTER + 1);
   27108              :           else
   27109            1 :             d_copy.target = gen_reg_rtx (V4DFmode);
   27110           19 :           d_copy.op0 = gen_lowpart (V4DFmode, d->op0);
   27111           19 :           d_copy.op1 = gen_lowpart (V4DFmode, d->op1);
   27112           19 :           if (expand_vec_perm_even_odd_1 (&d_copy, odd))
   27113              :             {
   27114           19 :               if (!d->testing_p)
   27115            1 :                 emit_move_insn (d->target,
   27116            1 :                                 gen_lowpart (V4DImode, d_copy.target));
   27117              :               return true;
   27118              :             }
   27119              :           return false;
   27120              :         }
   27121              : 
   27122            0 :       if (d->testing_p)
   27123              :         break;
   27124              : 
   27125            0 :       t1 = gen_reg_rtx (V4DImode);
   27126            0 :       t2 = gen_reg_rtx (V4DImode);
   27127              : 
   27128              :       /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }.  */
   27129            0 :       emit_insn (gen_avx2_permv2ti (t1, d->op0, d->op1, GEN_INT (0x20)));
   27130            0 :       emit_insn (gen_avx2_permv2ti (t2, d->op0, d->op1, GEN_INT (0x31)));
   27131              : 
   27132              :       /* Now an vpunpck[lh]qdq will produce the result required.  */
   27133            0 :       if (odd)
   27134            0 :         t3 = gen_avx2_interleave_highv4di (d->target, t1, t2);
   27135              :       else
   27136            0 :         t3 = gen_avx2_interleave_lowv4di (d->target, t1, t2);
   27137            0 :       emit_insn (t3);
   27138            0 :       break;
   27139              : 
   27140          176 :     case E_V8SImode:
   27141          176 :       if (!TARGET_AVX2)
   27142              :         {
   27143           38 :           struct expand_vec_perm_d d_copy = *d;
   27144           38 :           d_copy.vmode = V8SFmode;
   27145           38 :           if (d->testing_p)
   27146           38 :             d_copy.target = gen_raw_REG (V8SFmode, LAST_VIRTUAL_REGISTER + 1);
   27147              :           else
   27148            0 :             d_copy.target = gen_reg_rtx (V8SFmode);
   27149           38 :           d_copy.op0 = gen_lowpart (V8SFmode, d->op0);
   27150           38 :           d_copy.op1 = gen_lowpart (V8SFmode, d->op1);
   27151           38 :           if (expand_vec_perm_even_odd_1 (&d_copy, odd))
   27152              :             {
   27153           38 :               if (!d->testing_p)
   27154            0 :                 emit_move_insn (d->target,
   27155            0 :                                 gen_lowpart (V8SImode, d_copy.target));
   27156              :               return true;
   27157              :             }
   27158              :           return false;
   27159              :         }
   27160              : 
   27161          138 :       if (d->testing_p)
   27162              :         break;
   27163              : 
   27164          138 :       t1 = gen_reg_rtx (V8SImode);
   27165          138 :       t2 = gen_reg_rtx (V8SImode);
   27166          138 :       t3 = gen_reg_rtx (V4DImode);
   27167          138 :       t4 = gen_reg_rtx (V4DImode);
   27168          138 :       t5 = gen_reg_rtx (V4DImode);
   27169              : 
   27170              :       /* Shuffle the lanes around into
   27171              :          { 0 1 2 3 8 9 a b } and { 4 5 6 7 c d e f }.  */
   27172          276 :       emit_insn (gen_avx2_permv2ti (t3, gen_lowpart (V4DImode, d->op0),
   27173          138 :                                     gen_lowpart (V4DImode, d->op1),
   27174              :                                     GEN_INT (0x20)));
   27175          276 :       emit_insn (gen_avx2_permv2ti (t4, gen_lowpart (V4DImode, d->op0),
   27176          138 :                                     gen_lowpart (V4DImode, d->op1),
   27177              :                                     GEN_INT (0x31)));
   27178              : 
   27179              :       /* Swap the 2nd and 3rd position in each lane into
   27180              :          { 0 2 1 3 8 a 9 b } and { 4 6 5 7 c e d f }.  */
   27181          138 :       emit_insn (gen_avx2_pshufdv3 (t1, gen_lowpart (V8SImode, t3),
   27182              :                                     GEN_INT (2 * 4 + 1 * 16 + 3 * 64)));
   27183          138 :       emit_insn (gen_avx2_pshufdv3 (t2, gen_lowpart (V8SImode, t4),
   27184              :                                     GEN_INT (2 * 4 + 1 * 16 + 3 * 64)));
   27185              : 
   27186              :       /* Now an vpunpck[lh]qdq will produce
   27187              :          { 0 2 4 6 8 a c e } resp. { 1 3 5 7 9 b d f }.  */
   27188          138 :       if (odd)
   27189            0 :         t3 = gen_avx2_interleave_highv4di (t5, gen_lowpart (V4DImode, t1),
   27190            0 :                                            gen_lowpart (V4DImode, t2));
   27191              :       else
   27192          138 :         t3 = gen_avx2_interleave_lowv4di (t5, gen_lowpart (V4DImode, t1),
   27193          138 :                                           gen_lowpart (V4DImode, t2));
   27194          138 :       emit_insn (t3);
   27195          138 :       emit_move_insn (d->target, gen_lowpart (V8SImode, t5));
   27196          138 :       break;
   27197              : 
   27198            0 :     default:
   27199            0 :       gcc_unreachable ();
   27200              :     }
   27201              : 
   27202              :   return true;
   27203              : }
   27204              : 
   27205              : /* A subroutine of ix86_expand_vec_perm_const_1.  Pattern match
   27206              :    extract-even and extract-odd permutations.  */
   27207              : 
   27208              : static bool
   27209        25007 : expand_vec_perm_even_odd (struct expand_vec_perm_d *d)
   27210              : {
   27211        25007 :   unsigned i, odd, nelt = d->nelt;
   27212              : 
   27213        25007 :   odd = d->perm[0];
   27214        25007 :   if (odd != 0 && odd != 1)
   27215              :     return false;
   27216              : 
   27217        65727 :   for (i = 1; i < nelt; ++i)
   27218        57753 :     if (d->perm[i] != 2 * i + odd)
   27219              :       return false;
   27220              : 
   27221         7974 :   if (d->vmode == E_V32HImode
   27222           12 :       && d->testing_p
   27223           12 :       && !TARGET_AVX512BW)
   27224              :     return false;
   27225              : 
   27226         7962 :   return expand_vec_perm_even_odd_1 (d, odd);
   27227              : }
   27228              : 
   27229              : /* A subroutine of ix86_expand_vec_perm_const_1.  Implement broadcast
   27230              :    permutations.  We assume that expand_vec_perm_1 has already failed.  */
   27231              : 
   27232              : static bool
   27233         1094 : expand_vec_perm_broadcast_1 (struct expand_vec_perm_d *d)
   27234              : {
   27235         1094 :   unsigned elt = d->perm[0], nelt2 = d->nelt / 2;
   27236         1094 :   machine_mode vmode = d->vmode;
   27237         1094 :   rtx (*gen) (rtx, rtx, rtx);
   27238         1094 :   unsigned char perm2[4];
   27239         1094 :   rtx op0 = d->op0, dest;
   27240         1094 :   bool ok;
   27241              : 
   27242         1094 :   switch (vmode)
   27243              :     {
   27244            0 :     case E_V4DFmode:
   27245            0 :     case E_V8SFmode:
   27246              :       /* These are special-cased in sse.md so that we can optionally
   27247              :          use the vbroadcast instruction.  They expand to two insns
   27248              :          if the input happens to be in a register.  */
   27249            0 :       gcc_unreachable ();
   27250              : 
   27251            0 :     case E_V2DFmode:
   27252            0 :     case E_V2SFmode:
   27253            0 :     case E_V4SFmode:
   27254            0 :     case E_V2DImode:
   27255            0 :     case E_V2SImode:
   27256            0 :     case E_V4SImode:
   27257            0 :     case E_V2HImode:
   27258            0 :     case E_V4HImode:
   27259              :       /* These are always implementable using standard shuffle patterns.  */
   27260            0 :       gcc_unreachable ();
   27261              : 
   27262           16 :     case E_V4QImode:
   27263              :       /* This can be implemented via interleave and pshuflw.  */
   27264           16 :       if (d->testing_p)
   27265              :         return true;
   27266              : 
   27267            8 :       if (elt >= nelt2)
   27268              :         {
   27269            4 :           gen = gen_mmx_punpckhbw_low;
   27270            4 :           elt -= nelt2;
   27271              :         }
   27272              :       else
   27273              :         gen = gen_mmx_punpcklbw_low;
   27274              : 
   27275            8 :       dest = gen_reg_rtx (vmode);
   27276            8 :       emit_insn (gen (dest, op0, op0));
   27277            8 :       vmode = get_mode_wider_vector (vmode);
   27278            8 :       op0 = gen_lowpart (vmode, dest);
   27279              : 
   27280            8 :       memset (perm2, elt, 2);
   27281            8 :       dest = gen_reg_rtx (vmode);
   27282            8 :       ok = expand_vselect (dest, op0, perm2, 2, d->testing_p);
   27283            8 :       gcc_assert (ok);
   27284              : 
   27285            8 :       emit_move_insn (d->target, gen_lowpart (d->vmode, dest));
   27286            8 :       return true;
   27287              : 
   27288            4 :     case E_V8QImode:
   27289              :       /* This can be implemented via interleave.  We save one insn by
   27290              :          stopping once we have promoted to V2SImode and then use pshufd.  */
   27291            4 :       if (d->testing_p)
   27292              :         return true;
   27293            4 :       do
   27294              :         {
   27295            4 :           if (elt >= nelt2)
   27296              :             {
   27297            1 :               gen = vmode == V8QImode ? gen_mmx_punpckhbw
   27298              :                                       : gen_mmx_punpckhwd;
   27299            1 :               elt -= nelt2;
   27300              :             }
   27301              :           else
   27302            3 :             gen = vmode == V8QImode ? gen_mmx_punpcklbw
   27303              :                                     : gen_mmx_punpcklwd;
   27304            4 :           nelt2 /= 2;
   27305              : 
   27306            4 :           dest = gen_reg_rtx (vmode);
   27307            4 :           emit_insn (gen (dest, op0, op0));
   27308            4 :           vmode = get_mode_wider_vector (vmode);
   27309            4 :           op0 = gen_lowpart (vmode, dest);
   27310              :         }
   27311            4 :       while (vmode != V2SImode);
   27312              : 
   27313            2 :       memset (perm2, elt, 2);
   27314            2 :       dest = gen_reg_rtx (vmode);
   27315            2 :       ok = expand_vselect (dest, op0, perm2, 2, d->testing_p);
   27316            2 :       gcc_assert (ok);
   27317              : 
   27318            2 :       emit_move_insn (d->target, gen_lowpart (d->vmode, dest));
   27319            2 :       return true;
   27320              : 
   27321         1065 :     case E_V8HImode:
   27322         1065 :     case E_V16QImode:
   27323              :       /* These can be implemented via interleave.  We save one insn by
   27324              :          stopping once we have promoted to V4SImode and then use pshufd.  */
   27325         1065 :       if (d->testing_p)
   27326              :         return true;
   27327         1640 :       do
   27328              :         {
   27329         1640 :           if (elt >= nelt2)
   27330              :             {
   27331           16 :               gen = vmode == V16QImode ? gen_vec_interleave_highv16qi
   27332              :                                        : gen_vec_interleave_highv8hi;
   27333           16 :               elt -= nelt2;
   27334              :             }
   27335              :           else
   27336         1624 :             gen = vmode == V16QImode ? gen_vec_interleave_lowv16qi
   27337              :                                      : gen_vec_interleave_lowv8hi;
   27338         1640 :           nelt2 /= 2;
   27339              : 
   27340         1640 :           dest = gen_reg_rtx (vmode);
   27341         1640 :           emit_insn (gen (dest, op0, op0));
   27342         1640 :           vmode = get_mode_wider_vector (vmode);
   27343         1640 :           op0 = gen_lowpart (vmode, dest);
   27344              :         }
   27345         1640 :       while (vmode != V4SImode);
   27346              : 
   27347         1001 :       memset (perm2, elt, 4);
   27348         1001 :       dest = gen_reg_rtx (vmode);
   27349         1001 :       ok = expand_vselect (dest, op0, perm2, 4, d->testing_p);
   27350         1001 :       gcc_assert (ok);
   27351              : 
   27352         1001 :       emit_move_insn (d->target, gen_lowpart (d->vmode, dest));
   27353         1001 :       return true;
   27354              : 
   27355            1 :     case E_V8HFmode:
   27356            1 :     case E_V8BFmode:
   27357              :       /* This can be implemented via interleave and pshufd.  */
   27358            1 :       if (d->testing_p)
   27359              :         return true;
   27360              : 
   27361            1 :       rtx (*gen_interleave) (machine_mode, rtx, rtx, rtx);
   27362            1 :       if (elt >= nelt2)
   27363              :         {
   27364            0 :           gen_interleave = gen_vec_interleave_high;
   27365            0 :           elt -= nelt2;
   27366              :         }
   27367              :       else
   27368              :         gen_interleave = gen_vec_interleave_low;
   27369            1 :       nelt2 /= 2;
   27370              : 
   27371            1 :       dest = gen_reg_rtx (vmode);
   27372            1 :       emit_insn (gen_interleave (vmode, dest, op0, op0));
   27373              : 
   27374            1 :       vmode = V4SImode;
   27375            1 :       op0 = gen_lowpart (vmode, dest);
   27376              : 
   27377            1 :       memset (perm2, elt, 4);
   27378            1 :       dest = gen_reg_rtx (vmode);
   27379            1 :       ok = expand_vselect (dest, op0, perm2, 4, d->testing_p);
   27380            1 :       gcc_assert (ok);
   27381              : 
   27382            1 :       emit_move_insn (d->target, gen_lowpart (d->vmode, dest));
   27383            1 :       return true;
   27384              : 
   27385            0 :     case E_V32QImode:
   27386            0 :     case E_V16HImode:
   27387            0 :     case E_V8SImode:
   27388            0 :     case E_V4DImode:
   27389              :       /* For AVX2 broadcasts of the first element vpbroadcast* or
   27390              :          vpermq should be used by expand_vec_perm_1.  */
   27391            0 :       gcc_assert (!TARGET_AVX2 || d->perm[0]);
   27392              :       return false;
   27393              : 
   27394            6 :     case E_V64QImode:
   27395            6 :       gcc_assert (!TARGET_AVX512BW || d->perm[0]);
   27396              :       return false;
   27397              : 
   27398            2 :     case E_V32HImode:
   27399            2 :       gcc_assert (!TARGET_AVX512BW);
   27400              :       return false;
   27401              : 
   27402            0 :     default:
   27403            0 :       gcc_unreachable ();
   27404              :     }
   27405              : }
   27406              : 
   27407              : /* A subroutine of ix86_expand_vec_perm_const_1.  Pattern match
   27408              :    broadcast permutations.  */
   27409              : 
   27410              : static bool
   27411        93523 : expand_vec_perm_broadcast (struct expand_vec_perm_d *d)
   27412              : {
   27413        93523 :   unsigned i, elt, nelt = d->nelt;
   27414              : 
   27415        93523 :   if (!d->one_operand_p)
   27416              :     return false;
   27417              : 
   27418         5749 :   elt = d->perm[0];
   27419         8303 :   for (i = 1; i < nelt; ++i)
   27420         8195 :     if (d->perm[i] != elt)
   27421              :       return false;
   27422              : 
   27423          108 :   return expand_vec_perm_broadcast_1 (d);
   27424              : }
   27425              : 
   27426              : /* Implement arbitrary permutations of two V64QImode operands
   27427              :    with 2 vperm[it]2w, 2 vpshufb and one vpor instruction.  */
   27428              : static bool
   27429        25056 : expand_vec_perm_vpermt2_vpshub2 (struct expand_vec_perm_d *d)
   27430              : {
   27431        25056 :   if (!TARGET_AVX512BW || !(d->vmode == V64QImode))
   27432              :     return false;
   27433              : 
   27434           49 :   if (d->testing_p)
   27435              :     return true;
   27436              : 
   27437           49 :   struct expand_vec_perm_d ds[2];
   27438           49 :   rtx rperm[128], vperm, target0, target1;
   27439           49 :   unsigned int i, nelt;
   27440           49 :   machine_mode vmode;
   27441              : 
   27442           49 :   nelt = d->nelt;
   27443           49 :   vmode = V64QImode;
   27444              : 
   27445          147 :   for (i = 0; i < 2; i++)
   27446              :     {
   27447           98 :       ds[i] = *d;
   27448           98 :       ds[i].vmode = V32HImode;
   27449           98 :       ds[i].nelt = 32;
   27450           98 :       ds[i].target = gen_reg_rtx (V32HImode);
   27451           98 :       ds[i].op0 = gen_lowpart (V32HImode, d->op0);
   27452           98 :       ds[i].op1 = gen_lowpart (V32HImode, d->op1);
   27453              :     }
   27454              : 
   27455              :   /* Prepare permutations such that the first one takes care of
   27456              :      putting the even bytes into the right positions or one higher
   27457              :      positions (ds[0]) and the second one takes care of
   27458              :      putting the odd bytes into the right positions or one below
   27459              :      (ds[1]).  */
   27460              : 
   27461         3185 :   for (i = 0; i < nelt; i++)
   27462              :     {
   27463         3136 :       ds[i & 1].perm[i / 2] = d->perm[i] / 2;
   27464         3136 :       if (i & 1)
   27465              :         {
   27466         1568 :           rperm[i] = constm1_rtx;
   27467         1568 :           rperm[i + 64] = GEN_INT ((i & 14) + (d->perm[i] & 1));
   27468              :         }
   27469              :       else
   27470              :         {
   27471         1568 :           rperm[i] = GEN_INT ((i & 14) + (d->perm[i] & 1));
   27472         1568 :           rperm[i + 64] = constm1_rtx;
   27473              :         }
   27474              :     }
   27475              : 
   27476           49 :   bool ok = expand_vec_perm_1 (&ds[0]);
   27477           49 :   gcc_assert (ok);
   27478           49 :   ds[0].target = gen_lowpart (V64QImode, ds[0].target);
   27479              : 
   27480           49 :   ok = expand_vec_perm_1 (&ds[1]);
   27481           49 :   gcc_assert (ok);
   27482           49 :   ds[1].target = gen_lowpart (V64QImode, ds[1].target);
   27483              : 
   27484           49 :   vperm = gen_rtx_CONST_VECTOR (V64QImode, gen_rtvec_v (64, rperm));
   27485           49 :   vperm = force_reg (vmode, vperm);
   27486           49 :   target0 = gen_reg_rtx (V64QImode);
   27487           49 :   emit_insn (gen_avx512bw_pshufbv64qi3 (target0, ds[0].target, vperm));
   27488              : 
   27489           49 :   vperm = gen_rtx_CONST_VECTOR (V64QImode, gen_rtvec_v (64, rperm + 64));
   27490           49 :   vperm = force_reg (vmode, vperm);
   27491           49 :   target1 = gen_reg_rtx (V64QImode);
   27492           49 :   emit_insn (gen_avx512bw_pshufbv64qi3 (target1, ds[1].target, vperm));
   27493              : 
   27494           49 :   emit_insn (gen_iorv64qi3 (d->target, target0, target1));
   27495           49 :   return true;
   27496              : }
   27497              : 
   27498              : /* Implement arbitrary permutation of two V32QImode and V16QImode operands
   27499              :    with 4 vpshufb insns, 2 vpermq and 3 vpor.  We should have already failed
   27500              :    all the shorter instruction sequences.  */
   27501              : 
   27502              : static bool
   27503        16781 : expand_vec_perm_vpshufb4_vpermq2 (struct expand_vec_perm_d *d)
   27504              : {
   27505        16781 :   rtx rperm[4][32], vperm, l[2], h[2], op, m128;
   27506        16781 :   unsigned int i, nelt, eltsz;
   27507        16781 :   bool used[4];
   27508              : 
   27509        16781 :   if (!TARGET_AVX2
   27510          318 :       || d->one_operand_p
   27511          189 :       || (d->vmode != V32QImode && d->vmode != V16HImode))
   27512              :     return false;
   27513              : 
   27514           54 :   if (d->testing_p)
   27515              :     return true;
   27516              : 
   27517           54 :   nelt = d->nelt;
   27518           54 :   eltsz = GET_MODE_UNIT_SIZE (d->vmode);
   27519              : 
   27520              :   /* Generate 4 permutation masks.  If the required element is within
   27521              :      the same lane, it is shuffled in.  If the required element from the
   27522              :      other lane, force a zero by setting bit 7 in the permutation mask.
   27523              :      In the other mask the mask has non-negative elements if element
   27524              :      is requested from the other lane, but also moved to the other lane,
   27525              :      so that the result of vpshufb can have the two V2TImode halves
   27526              :      swapped.  */
   27527           54 :   m128 = GEN_INT (-128);
   27528         1836 :   for (i = 0; i < 32; ++i)
   27529              :     {
   27530         1728 :       rperm[0][i] = m128;
   27531         1728 :       rperm[1][i] = m128;
   27532         1728 :       rperm[2][i] = m128;
   27533         1728 :       rperm[3][i] = m128;
   27534              :     }
   27535           54 :   used[0] = false;
   27536           54 :   used[1] = false;
   27537           54 :   used[2] = false;
   27538           54 :   used[3] = false;
   27539         1590 :   for (i = 0; i < nelt; ++i)
   27540              :     {
   27541         1536 :       unsigned j, e = d->perm[i] & (nelt / 2 - 1);
   27542         1536 :       unsigned xlane = ((d->perm[i] ^ i) & (nelt / 2)) * eltsz;
   27543         2074 :       unsigned int which = ((d->perm[i] & nelt) ? 2 : 0) + (xlane ? 1 : 0);
   27544              : 
   27545         3264 :       for (j = 0; j < eltsz; ++j)
   27546         1728 :         rperm[which][(i * eltsz + j) ^ xlane] = GEN_INT (e * eltsz + j);
   27547         1536 :       used[which] = true;
   27548              :     }
   27549              : 
   27550          162 :   for (i = 0; i < 2; ++i)
   27551              :     {
   27552          108 :       if (!used[2 * i + 1])
   27553              :         {
   27554           22 :           h[i] = NULL_RTX;
   27555           22 :           continue;
   27556              :         }
   27557           86 :       vperm = gen_rtx_CONST_VECTOR (V32QImode,
   27558           86 :                                     gen_rtvec_v (32, rperm[2 * i + 1]));
   27559           86 :       vperm = force_reg (V32QImode, vperm);
   27560           86 :       h[i] = gen_reg_rtx (V32QImode);
   27561           86 :       op = gen_lowpart (V32QImode, i ? d->op1 : d->op0);
   27562           86 :       emit_insn (gen_avx2_pshufbv32qi3 (h[i], op, vperm));
   27563              :     }
   27564              : 
   27565              :   /* Swap the 128-byte lanes of h[X].  */
   27566          162 :   for (i = 0; i < 2; ++i)
   27567              :    {
   27568          108 :      if (h[i] == NULL_RTX)
   27569           22 :        continue;
   27570           86 :      op = gen_reg_rtx (V4DImode);
   27571           86 :      emit_insn (gen_avx2_permv4di_1 (op, gen_lowpart (V4DImode, h[i]),
   27572              :                                      const2_rtx, GEN_INT (3), const0_rtx,
   27573              :                                      const1_rtx));
   27574           86 :      h[i] = gen_lowpart (V32QImode, op);
   27575              :    }
   27576              : 
   27577          162 :   for (i = 0; i < 2; ++i)
   27578              :     {
   27579          108 :       if (!used[2 * i])
   27580              :         {
   27581            0 :           l[i] = NULL_RTX;
   27582            0 :           continue;
   27583              :         }
   27584          108 :       vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[2 * i]));
   27585          108 :       vperm = force_reg (V32QImode, vperm);
   27586          108 :       l[i] = gen_reg_rtx (V32QImode);
   27587          108 :       op = gen_lowpart (V32QImode, i ? d->op1 : d->op0);
   27588          108 :       emit_insn (gen_avx2_pshufbv32qi3 (l[i], op, vperm));
   27589              :     }
   27590              : 
   27591          162 :   for (i = 0; i < 2; ++i)
   27592              :     {
   27593          108 :       if (h[i] && l[i])
   27594              :         {
   27595           86 :           op = gen_reg_rtx (V32QImode);
   27596           86 :           emit_insn (gen_iorv32qi3 (op, l[i], h[i]));
   27597           86 :           l[i] = op;
   27598              :         }
   27599           22 :       else if (h[i])
   27600            0 :         l[i] = h[i];
   27601              :     }
   27602              : 
   27603           54 :   gcc_assert (l[0] && l[1]);
   27604           54 :   op = d->target;
   27605           54 :   if (d->vmode != V32QImode)
   27606           12 :     op = gen_reg_rtx (V32QImode);
   27607           54 :   emit_insn (gen_iorv32qi3 (op, l[0], l[1]));
   27608           54 :   if (op != d->target)
   27609           12 :     emit_move_insn (d->target, gen_lowpart (d->vmode, op));
   27610              :   return true;
   27611              : }
   27612              : 
   27613              : /* The guts of ix86_vectorize_vec_perm_const.  With all of the interface bits
   27614              :    taken care of, perform the expansion in D and return true on success.  */
   27615              : 
   27616              : static bool
   27617       304851 : ix86_expand_vec_perm_const_1 (struct expand_vec_perm_d *d)
   27618              : {
   27619              :   /* Try a single instruction expansion.  */
   27620       304851 :   if (expand_vec_perm_1 (d))
   27621              :     return true;
   27622              : 
   27623              :   /* Try sequences of two instructions.  */
   27624              : 
   27625       105328 :   if (expand_vec_perm_pshuflw_pshufhw (d))
   27626              :     return true;
   27627              : 
   27628       102837 :   if (expand_vec_perm_palignr (d, false))
   27629              :     return true;
   27630              : 
   27631        99702 :   if (expand_vec_perm_interleave2 (d))
   27632              :     return true;
   27633              : 
   27634        93523 :   if (expand_vec_perm_broadcast (d))
   27635              :     return true;
   27636              : 
   27637        93423 :   if (expand_vec_perm_vpermq_perm_1 (d))
   27638              :     return true;
   27639              : 
   27640        93423 :   if (expand_vec_perm_vperm2f128 (d))
   27641              :     return true;
   27642              : 
   27643        93355 :   if (expand_vec_perm_pblendv (d))
   27644              :     return true;
   27645              : 
   27646        90920 :   if (expand_vec_perm_2perm_interleave (d, true))
   27647              :     return true;
   27648              : 
   27649        90562 :   if (expand_vec_perm_2perm_pblendv (d, true))
   27650              :     return true;
   27651              : 
   27652        88420 :   if (expand_vec_perm_shufps_shufps (d))
   27653              :     return true;
   27654              : 
   27655        52128 :   if (expand_vec_perm_punpckldq_pshuf (d))
   27656              :     return true;
   27657              : 
   27658              :   /* Try sequences of three instructions.  */
   27659              : 
   27660        45584 :   if (expand_vec_perm_even_odd_pack (d))
   27661              :     return true;
   27662              : 
   27663        31402 :   if (expand_vec_perm_2vperm2f128_vshuf (d))
   27664              :     return true;
   27665              : 
   27666        30253 :   if (expand_vec_perm_pshufb2 (d))
   27667              :     return true;
   27668              : 
   27669        28837 :   if (expand_vec_perm_pslldq_psrldq_por (d, false))
   27670              :     return true;
   27671              : 
   27672        28524 :   if (expand_vec_perm_interleave3 (d))
   27673              :     return true;
   27674              : 
   27675        28378 :   if (expand_vec_perm_vperm2f128_vblend (d))
   27676              :     return true;
   27677              : 
   27678        28378 :   if (expand_vec_perm_2perm_interleave (d, false))
   27679              :     return true;
   27680              : 
   27681        28186 :   if (expand_vec_perm_2perm_pblendv (d, false))
   27682              :     return true;
   27683              : 
   27684        27626 :   if (expand_vec_perm_psrlw_psllw_por (d))
   27685              :     return true;
   27686              : 
   27687        26172 :   if (expand_vec_perm_pand_pandn_por (d))
   27688              :     return true;
   27689              : 
   27690              :   /* Try sequences of four instructions.  */
   27691              : 
   27692        25067 :   if (expand_vec_perm_even_odd_trunc (d))
   27693              :     return true;
   27694        25063 :   if (expand_vec_perm_vpshufb2_vpermq (d))
   27695              :     return true;
   27696              : 
   27697        25056 :   if (expand_vec_perm_vpshufb2_vpermq_even_odd (d))
   27698              :     return true;
   27699              : 
   27700        25056 :   if (expand_vec_perm_vpermt2_vpshub2 (d))
   27701              :     return true;
   27702              : 
   27703              :   /* ??? Look for narrow permutations whose element orderings would
   27704              :      allow the promotion to a wider mode.  */
   27705              : 
   27706              :   /* ??? Look for sequences of interleave or a wider permute that place
   27707              :      the data into the correct lanes for a half-vector shuffle like
   27708              :      pshuf[lh]w or vpermilps.  */
   27709              : 
   27710              :   /* ??? Look for sequences of interleave that produce the desired results.
   27711              :      The combinatorics of punpck[lh] get pretty ugly... */
   27712              : 
   27713        25007 :   if (expand_vec_perm_even_odd (d))
   27714              :     return true;
   27715              : 
   27716              :   /* Generate four or five instructions.  */
   27717        17117 :   if (expand_vec_perm_pslldq_psrldq_por (d, true))
   27718              :     return true;
   27719              : 
   27720              :   /* Even longer sequences.  */
   27721        16781 :   if (expand_vec_perm_vpshufb4_vpermq2 (d))
   27722              :     return true;
   27723              : 
   27724              :   /* See if we can get the same permutation in different vector integer
   27725              :      mode.  */
   27726        16727 :   struct expand_vec_perm_d nd;
   27727        16727 :   if (canonicalize_vector_int_perm (d, &nd) && expand_vec_perm_1 (&nd))
   27728              :     {
   27729            0 :       if (!d->testing_p)
   27730            0 :         emit_move_insn (d->target, gen_lowpart (d->vmode, nd.target));
   27731              :       return true;
   27732              :     }
   27733              : 
   27734              :   /* Even longer, including recursion to ix86_expand_vec_perm_const_1.  */
   27735        16727 :   if (expand_vec_perm2_vperm2f128_vblend (d))
   27736              :     return true;
   27737              : 
   27738              :   return false;
   27739              : }
   27740              : 
   27741              : /* If a permutation only uses one operand, make it clear. Returns true
   27742              :    if the permutation references both operands.  */
   27743              : 
   27744              : static bool
   27745        75302 : canonicalize_perm (struct expand_vec_perm_d *d)
   27746              : {
   27747        75302 :   int i, which, nelt = d->nelt;
   27748              : 
   27749       450858 :   for (i = which = 0; i < nelt; ++i)
   27750       511692 :     which |= (d->perm[i] < nelt ? 1 : 2);
   27751              : 
   27752        75302 :   d->one_operand_p = true;
   27753        75302 :   switch (which)
   27754              :     {
   27755            0 :     default:
   27756            0 :       gcc_unreachable();
   27757              : 
   27758        56721 :     case 3:
   27759        56721 :       if (!rtx_equal_p (d->op0, d->op1))
   27760              :         {
   27761        56688 :           d->one_operand_p = false;
   27762        56688 :           break;
   27763              :         }
   27764              :       /* The elements of PERM do not suggest that only the first operand
   27765              :          is used, but both operands are identical.  Allow easier matching
   27766              :          of the permutation by folding the permutation into the single
   27767              :          input vector.  */
   27768              :       /* FALLTHRU */
   27769              : 
   27770          415 :     case 2:
   27771         3517 :       for (i = 0; i < nelt; ++i)
   27772         3102 :         d->perm[i] &= nelt - 1;
   27773          415 :       d->op0 = d->op1;
   27774          415 :       break;
   27775              : 
   27776        18199 :     case 1:
   27777        18199 :       d->op1 = d->op0;
   27778        18199 :       break;
   27779              :     }
   27780              : 
   27781        75302 :   return (which == 3);
   27782              : }
   27783              : 
   27784              : /* Implement TARGET_VECTORIZE_VEC_PERM_CONST.  */
   27785              : 
   27786              : bool
   27787       838087 : ix86_vectorize_vec_perm_const (machine_mode vmode, machine_mode op_mode,
   27788              :                                rtx target, rtx op0, rtx op1,
   27789              :                                const vec_perm_indices &sel)
   27790              : {
   27791       838087 :   if (vmode != op_mode)
   27792              :     return false;
   27793              : 
   27794       830363 :   struct expand_vec_perm_d d;
   27795       830363 :   unsigned char perm[MAX_VECT_LEN];
   27796       830363 :   unsigned int i, nelt, which;
   27797       830363 :   bool two_args;
   27798              : 
   27799              :   /* For HF and BF mode vector, convert it to HI using subreg.  */
   27800      2488634 :   if (GET_MODE_INNER (vmode) == HFmode || GET_MODE_INNER (vmode) == BFmode)
   27801              :     {
   27802         2485 :       machine_mode orig_mode = vmode;
   27803         4970 :       vmode = mode_for_vector (HImode,
   27804         2485 :                                GET_MODE_NUNITS (vmode)).require ();
   27805         2485 :       if (target)
   27806          436 :         target = lowpart_subreg (vmode, target, orig_mode);
   27807         2485 :       if (op0)
   27808          436 :         op0 = lowpart_subreg (vmode, op0, orig_mode);
   27809         2485 :       if (op1)
   27810          436 :         op1 = lowpart_subreg (vmode, op1, orig_mode);
   27811              :     }
   27812              : 
   27813       830363 :   d.target = target;
   27814       830363 :   d.op0 = op0;
   27815       830363 :   d.op1 = op1;
   27816              : 
   27817       830363 :   d.vmode = vmode;
   27818       830363 :   gcc_assert (VECTOR_MODE_P (d.vmode));
   27819       830363 :   d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
   27820       830363 :   d.testing_p = !target;
   27821              : 
   27822       830363 :   gcc_assert (sel.length () == nelt);
   27823       830363 :   gcc_checking_assert (sizeof (d.perm) == sizeof (perm));
   27824              : 
   27825              :   /* Given sufficient ISA support we can just return true here
   27826              :      for selected vector modes.  */
   27827       830363 :   switch (d.vmode)
   27828              :     {
   27829         1612 :     case E_V16SFmode:
   27830         1612 :     case E_V16SImode:
   27831         1612 :     case E_V8DImode:
   27832         1612 :     case E_V8DFmode:
   27833         1612 :       if (!TARGET_AVX512F)
   27834              :         return false;
   27835              :       /* All implementable with a single vperm[it]2 insn.  */
   27836         1612 :       if (d.testing_p)
   27837              :         return true;
   27838              :       break;
   27839          323 :     case E_V32HImode:
   27840          323 :       if (!TARGET_AVX512F)
   27841              :         return false;
   27842          323 :       if (d.testing_p && TARGET_AVX512BW)
   27843              :         /* All implementable with a single vperm[it]2 insn.  */
   27844              :         return true;
   27845              :       break;
   27846          715 :     case E_V64QImode:
   27847          715 :       if (!TARGET_AVX512F)
   27848              :         return false;
   27849          715 :       if (d.testing_p && TARGET_AVX512BW)
   27850              :         /* Implementable with 2 vperm[it]2, 2 vpshufb and 1 or insn.  */
   27851              :         return true;
   27852              :       break;
   27853        12799 :     case E_V8SImode:
   27854        12799 :     case E_V8SFmode:
   27855        12799 :     case E_V4DFmode:
   27856        12799 :     case E_V4DImode:
   27857        12799 :       if (!TARGET_AVX)
   27858              :         return false;
   27859        12799 :       if (d.testing_p && TARGET_AVX512VL)
   27860              :         /* All implementable with a single vperm[it]2 insn.  */
   27861              :         return true;
   27862              :       break;
   27863          613 :     case E_V16HImode:
   27864          613 :       if (!TARGET_SSE2)
   27865              :         return false;
   27866          613 :       if (d.testing_p && TARGET_AVX2)
   27867              :         /* Implementable with 4 vpshufb insns, 2 vpermq and 3 vpor insns.  */
   27868              :         return true;
   27869              :       break;
   27870          685 :     case E_V32QImode:
   27871          685 :       if (!TARGET_SSE2)
   27872              :         return false;
   27873          685 :       if (d.testing_p && TARGET_AVX2)
   27874              :         /* Implementable with 4 vpshufb insns, 2 vpermq and 3 vpor insns.  */
   27875              :         return true;
   27876              :       break;
   27877        40689 :     case E_V8HImode:
   27878        40689 :     case E_V16QImode:
   27879        40689 :       if (!TARGET_SSE2)
   27880              :         return false;
   27881              :       /* Fall through.  */
   27882       249755 :     case E_V4SImode:
   27883       249755 :     case E_V4SFmode:
   27884       249755 :       if (!TARGET_SSE)
   27885              :         return false;
   27886              :       /* All implementable with a single vpperm insn.  */
   27887       249755 :       if (d.testing_p && TARGET_XOP)
   27888              :         return true;
   27889              :       /* All implementable with 2 pshufb + 1 ior.  */
   27890       249649 :       if (d.testing_p && TARGET_SSSE3)
   27891              :         return true;
   27892              :       break;
   27893       152632 :     case E_V2SFmode:
   27894       152632 :     case E_V2SImode:
   27895       152632 :     case E_V4HImode:
   27896       152632 :     case E_V8QImode:
   27897       152632 :       if (!TARGET_MMX_WITH_SSE)
   27898              :         return false;
   27899              :       break;
   27900        21376 :     case E_V2HImode:
   27901        21376 :       if (!TARGET_SSE2)
   27902              :         return false;
   27903              :       /* All implementable with *punpckwd.  */
   27904        21376 :       if (d.testing_p)
   27905              :         return true;
   27906              :       break;
   27907        13018 :     case E_V4QImode:
   27908        13018 :       if (!TARGET_SSE2)
   27909              :         return false;
   27910              :       break;
   27911       374325 :     case E_V2DImode:
   27912       374325 :     case E_V2DFmode:
   27913       374325 :       if (!TARGET_SSE)
   27914              :         return false;
   27915              :       /* All implementable with shufpd or unpck[lh]pd.  */
   27916       374325 :       if (d.testing_p)
   27917              :         return true;
   27918              :       break;
   27919              :     default:
   27920              :       return false;
   27921              :     }
   27922              : 
   27923      2385215 :   for (i = which = 0; i < nelt; ++i)
   27924              :     {
   27925      1952068 :       unsigned char e = sel[i];
   27926      1952068 :       gcc_assert (e < 2 * nelt);
   27927      1952068 :       d.perm[i] = e;
   27928      1952068 :       perm[i] = e;
   27929      2619472 :       which |= (e < nelt ? 1 : 2);
   27930              :     }
   27931              : 
   27932       433147 :   if (d.testing_p)
   27933              :     {
   27934              :       /* For all elements from second vector, fold the elements to first.  */
   27935       359109 :       if (which == 2)
   27936         1357 :         for (i = 0; i < nelt; ++i)
   27937         1204 :           d.perm[i] -= nelt;
   27938              : 
   27939              :       /* Check whether the mask can be applied to the vector type.  */
   27940       359109 :       d.one_operand_p = (which != 3);
   27941              : 
   27942              :       /* Implementable with shufps, pshufd or pshuflw.  */
   27943       359109 :       if (d.one_operand_p
   27944              :           && (d.vmode == V4SFmode || d.vmode == V2SFmode
   27945              :               || d.vmode == V4SImode || d.vmode == V2SImode
   27946              :               || d.vmode == V4HImode || d.vmode == V2HImode))
   27947              :         return true;
   27948              : 
   27949              :       /* Otherwise we have to go through the motions and see if we can
   27950              :          figure out how to generate the requested permutation.  */
   27951       227053 :       d.target = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 1);
   27952       227053 :       d.op1 = d.op0 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 2);
   27953       227053 :       if (!d.one_operand_p)
   27954       211083 :         d.op1 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 3);
   27955              : 
   27956       227053 :       start_sequence ();
   27957       227053 :       bool ret = ix86_expand_vec_perm_const_1 (&d);
   27958       227053 :       end_sequence ();
   27959              : 
   27960       227053 :       return ret;
   27961              :     }
   27962              : 
   27963        74038 :   two_args = canonicalize_perm (&d);
   27964              : 
   27965              :   /* If one of the operands is a zero vector, try to match pmovzx.  */
   27966        74038 :   if (two_args && (d.op0 == CONST0_RTX (vmode) || d.op1 == CONST0_RTX (vmode)))
   27967              :     {
   27968          689 :       struct expand_vec_perm_d dzero = d;
   27969          689 :       if (d.op0 == CONST0_RTX (vmode))
   27970              :         {
   27971          452 :           d.op1 = dzero.op1 = force_reg (vmode, d.op1);
   27972          452 :           std::swap (dzero.op0, dzero.op1);
   27973         7852 :           for (i = 0; i < nelt; ++i)
   27974         7400 :             dzero.perm[i] ^= nelt;
   27975              :         }
   27976              :       else
   27977          237 :         d.op0 = dzero.op0 = force_reg (vmode, d.op0);
   27978              : 
   27979          689 :       if (expand_vselect_vconcat (dzero.target, dzero.op0, dzero.op1,
   27980              :                                   dzero.perm, nelt, dzero.testing_p))
   27981          261 :         return true;
   27982              :     }
   27983              : 
   27984              :   /* Force operands into registers.  */
   27985        73777 :   rtx nop0 = force_reg (vmode, d.op0);
   27986        73777 :   if (d.op0 == d.op1)
   27987        18213 :     d.op1 = nop0;
   27988        73777 :   d.op0 = nop0;
   27989        73777 :   d.op1 = force_reg (vmode, d.op1);
   27990              : 
   27991        73777 :   if (ix86_expand_vec_perm_const_1 (&d))
   27992              :     return true;
   27993              : 
   27994              :   /* If the selector says both arguments are needed, but the operands are the
   27995              :      same, the above tried to expand with one_operand_p and flattened selector.
   27996              :      If that didn't work, retry without one_operand_p; we succeeded with that
   27997              :      during testing.  */
   27998            0 :   if (two_args && d.one_operand_p)
   27999              :     {
   28000            0 :       d.one_operand_p = false;
   28001            0 :       memcpy (d.perm, perm, sizeof (perm));
   28002            0 :       return ix86_expand_vec_perm_const_1 (&d);
   28003              :     }
   28004              : 
   28005              :   return false;
   28006              : }
   28007              : 
   28008              : void
   28009         8488 : ix86_expand_vec_extract_even_odd (rtx targ, rtx op0, rtx op1, unsigned odd)
   28010              : {
   28011         8488 :   struct expand_vec_perm_d d;
   28012         8488 :   unsigned i, nelt;
   28013              : 
   28014         8488 :   d.target = targ;
   28015         8488 :   d.op0 = op0;
   28016         8488 :   d.op1 = op1;
   28017         8488 :   d.vmode = GET_MODE (targ);
   28018         8488 :   d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
   28019         8488 :   d.one_operand_p = false;
   28020         8488 :   d.testing_p = false;
   28021              : 
   28022        80584 :   for (i = 0; i < nelt; ++i)
   28023        72096 :     d.perm[i] = i * 2 + odd;
   28024              : 
   28025              :   /* We'll either be able to implement the permutation directly...  */
   28026         8488 :   if (expand_vec_perm_1 (&d))
   28027              :     return;
   28028              : 
   28029              :   /* ... or we use the special-case patterns.  */
   28030         5243 :   expand_vec_perm_even_odd_1 (&d, odd);
   28031              : }
   28032              : 
   28033              : static void
   28034          944 : ix86_expand_vec_interleave (rtx targ, rtx op0, rtx op1, bool high_p)
   28035              : {
   28036          944 :   struct expand_vec_perm_d d;
   28037          944 :   unsigned i, nelt, base;
   28038          944 :   bool ok;
   28039              : 
   28040          944 :   d.target = targ;
   28041          944 :   d.op0 = op0;
   28042          944 :   d.op1 = op1;
   28043          944 :   d.vmode = GET_MODE (targ);
   28044          944 :   d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
   28045          944 :   d.one_operand_p = false;
   28046          944 :   d.testing_p = false;
   28047              : 
   28048          944 :   base = high_p ? nelt / 2 : 0;
   28049         3744 :   for (i = 0; i < nelt / 2; ++i)
   28050              :     {
   28051         2800 :       d.perm[i * 2] = i + base;
   28052         2800 :       d.perm[i * 2 + 1] = i + base + nelt;
   28053              :     }
   28054              : 
   28055              :   /* Note that for AVX this isn't one instruction.  */
   28056          944 :   ok = ix86_expand_vec_perm_const_1 (&d);
   28057          944 :   gcc_assert (ok);
   28058          944 : }
   28059              : 
   28060              : /* Expand a vector operation shift by constant for a V*QImode in terms of the
   28061              :    same operation on V*HImode. Return true if success. */
   28062              : static bool
   28063          459 : ix86_expand_vec_shift_qihi_constant (enum rtx_code code,
   28064              :                                      rtx dest, rtx op1, rtx op2)
   28065              : {
   28066          459 :   machine_mode qimode, himode;
   28067          459 :   HOST_WIDE_INT and_constant, xor_constant;
   28068          459 :   HOST_WIDE_INT shift_amount;
   28069          459 :   rtx vec_const_and, vec_const_xor;
   28070          459 :   rtx tmp, op1_subreg;
   28071          459 :   rtx (*gen_shift) (rtx, rtx, rtx);
   28072          459 :   rtx (*gen_and) (rtx, rtx, rtx);
   28073          459 :   rtx (*gen_xor) (rtx, rtx, rtx);
   28074          459 :   rtx (*gen_sub) (rtx, rtx, rtx);
   28075              : 
   28076              :   /* Only optimize shift by constant.  */
   28077          459 :   if (!CONST_INT_P (op2))
   28078              :     return false;
   28079              : 
   28080          459 :   qimode = GET_MODE (dest);
   28081          459 :   shift_amount = INTVAL (op2);
   28082              :   /* Do nothing when shift amount greater equal 8.  */
   28083          459 :   if (shift_amount > 7)
   28084              :     return false;
   28085              : 
   28086          459 :   gcc_assert (code == ASHIFT || code == ASHIFTRT || code == LSHIFTRT);
   28087              : 
   28088              : 
   28089          459 :   if (shift_amount == 7
   28090          459 :       && code == ASHIFTRT)
   28091              :     {
   28092           84 :       if (qimode == V16QImode
   28093            9 :           || qimode == V32QImode)
   28094              :         {
   28095           83 :           rtx zero = gen_reg_rtx (qimode);
   28096           83 :           emit_move_insn (zero, CONST0_RTX (qimode));
   28097           83 :           emit_move_insn (dest, gen_rtx_fmt_ee (GT, qimode, zero, op1));
   28098           83 :         }
   28099              :       else
   28100              :         {
   28101            1 :           gcc_assert (qimode == V64QImode);
   28102            1 :           rtx kmask = gen_reg_rtx (DImode);
   28103            1 :           emit_insn (gen_avx512bw_cvtb2maskv64qi (kmask, op1));
   28104            1 :           emit_insn (gen_avx512bw_cvtmask2bv64qi (dest, kmask));
   28105              :         }
   28106              :       return true;
   28107              :     }
   28108              : 
   28109              :   /* Record sign bit.  */
   28110          375 :   xor_constant = 1 << (8 - shift_amount - 1);
   28111              : 
   28112              :   /* Zero upper/lower bits shift from left/right element.  */
   28113          375 :   and_constant
   28114          375 :     = (code == ASHIFT ? 256 - (1 << shift_amount)
   28115          344 :        : (1 << (8 - shift_amount)) - 1);
   28116              : 
   28117          375 :   switch (qimode)
   28118              :     {
   28119          362 :     case V16QImode:
   28120          362 :       himode = V8HImode;
   28121          260 :       gen_shift =
   28122              :         ((code == ASHIFT)
   28123          362 :          ? gen_ashlv8hi3
   28124          340 :          : (code == ASHIFTRT) ? gen_ashrv8hi3 : gen_lshrv8hi3);
   28125              :       gen_and = gen_andv16qi3;
   28126              :       gen_xor = gen_xorv16qi3;
   28127              :       gen_sub = gen_subv16qi3;
   28128              :       break;
   28129            6 :     case V32QImode:
   28130            6 :       himode = V16HImode;
   28131            1 :       gen_shift =
   28132              :         ((code == ASHIFT)
   28133            6 :          ? gen_ashlv16hi3
   28134            2 :          : (code == ASHIFTRT) ? gen_ashrv16hi3 : gen_lshrv16hi3);
   28135              :       gen_and = gen_andv32qi3;
   28136              :       gen_xor = gen_xorv32qi3;
   28137              :       gen_sub = gen_subv32qi3;
   28138              :       break;
   28139            7 :     case V64QImode:
   28140            7 :       himode = V32HImode;
   28141            1 :       gen_shift =
   28142              :         ((code == ASHIFT)
   28143            7 :          ? gen_ashlv32hi3
   28144            2 :          : (code == ASHIFTRT) ? gen_ashrv32hi3 : gen_lshrv32hi3);
   28145              :       gen_and = gen_andv64qi3;
   28146              :       gen_xor = gen_xorv64qi3;
   28147              :       gen_sub = gen_subv64qi3;
   28148              :       break;
   28149            0 :     default:
   28150            0 :       gcc_unreachable ();
   28151              :     }
   28152              : 
   28153          375 :   tmp = gen_reg_rtx (himode);
   28154          375 :   vec_const_and = gen_reg_rtx (qimode);
   28155          375 :   op1_subreg = lowpart_subreg (himode, op1, qimode);
   28156              : 
   28157              :   /* For ASHIFT and LSHIFTRT, perform operation like
   28158              :      vpsllw/vpsrlw $shift_amount, %op1, %dest.
   28159              :      vpand %vec_const_and, %dest.  */
   28160          375 :   emit_insn (gen_shift (tmp, op1_subreg, op2));
   28161          375 :   emit_move_insn (dest, simplify_gen_subreg (qimode, tmp, himode, 0));
   28162          375 :   emit_move_insn (vec_const_and,
   28163              :                   ix86_build_const_vector (qimode, true,
   28164          375 :                                            gen_int_mode (and_constant, QImode)));
   28165          375 :   emit_insn (gen_and (dest, dest, vec_const_and));
   28166              : 
   28167              :   /* For ASHIFTRT, perform extra operation like
   28168              :      vpxor %vec_const_xor, %dest, %dest
   28169              :      vpsubb %vec_const_xor, %dest, %dest  */
   28170          375 :   if (code == ASHIFTRT)
   28171              :     {
   28172           82 :       vec_const_xor = gen_reg_rtx (qimode);
   28173           82 :       emit_move_insn (vec_const_xor,
   28174              :                       ix86_build_const_vector (qimode, true,
   28175           82 :                                                gen_int_mode (xor_constant, QImode)));
   28176           82 :       emit_insn (gen_xor (dest, dest, vec_const_xor));
   28177           82 :       emit_insn (gen_sub (dest, dest, vec_const_xor));
   28178              :     }
   28179              :   return true;
   28180              : }
   28181              : 
   28182              : void
   28183         1451 : ix86_expand_vecop_qihi_partial (enum rtx_code code, rtx dest, rtx op1, rtx op2)
   28184              : {
   28185         1451 :   machine_mode qimode = GET_MODE (dest);
   28186         1451 :   rtx qop1, qop2, hop1, hop2, qdest, hdest;
   28187         1451 :   bool op2vec = GET_MODE_CLASS (GET_MODE (op2)) == MODE_VECTOR_INT;
   28188         1451 :   bool uns_p = code != ASHIFTRT;
   28189              : 
   28190         1451 :   switch (qimode)
   28191              :     {
   28192         1451 :     case E_V4QImode:
   28193         1451 :     case E_V8QImode:
   28194         1451 :       break;
   28195            0 :     default:
   28196            0 :       gcc_unreachable ();
   28197              :     }
   28198              : 
   28199         1451 :   qop1 = lowpart_subreg (V16QImode, force_reg (qimode, op1), qimode);
   28200              : 
   28201         1451 :   if (op2vec)
   28202         1317 :     qop2 = lowpart_subreg (V16QImode, force_reg (qimode, op2), qimode);
   28203              :   else
   28204              :     qop2 = op2;
   28205              : 
   28206         1451 :   qdest = gen_reg_rtx (V16QImode);
   28207              : 
   28208         1451 :   if (CONST_INT_P (op2)
   28209          122 :       && (code == ASHIFT || code == LSHIFTRT || code == ASHIFTRT)
   28210              :       /* With AVX512 it's cheaper to do vpmovsxbw/op/vpmovwb.
   28211              :          Even with SSE4.1 the alternative is better.  */
   28212          122 :       && !TARGET_SSE4_1
   28213         1505 :       && ix86_expand_vec_shift_qihi_constant (code, qdest, qop1, qop2))
   28214              :     {
   28215           54 :       emit_move_insn (dest, gen_lowpart (qimode, qdest));
   28216           54 :       return;
   28217              :     }
   28218              : 
   28219         1397 :   if (CONST_INT_P (op2)
   28220           68 :       && code == ASHIFTRT
   28221           14 :       && INTVAL (op2) == 7)
   28222              :     {
   28223            4 :       rtx zero = gen_reg_rtx (qimode);
   28224            4 :       emit_move_insn (zero, CONST0_RTX (qimode));
   28225            4 :       emit_move_insn (dest, gen_rtx_fmt_ee (GT, qimode, zero, op1));
   28226            4 :       return;
   28227              :     }
   28228              : 
   28229         1393 :   switch (code)
   28230              :     {
   28231         1304 :     case MULT:
   28232         1304 :       gcc_assert (op2vec);
   28233         1304 :       if (!TARGET_SSE4_1)
   28234              :         {
   28235              :           /* Unpack data such that we've got a source byte in each low byte
   28236              :              of each word.  We don't care what goes into the high byte of
   28237              :              each word.  Rather than trying to get zero in there, most
   28238              :              convenient is to let it be a copy of the low byte.  */
   28239          251 :           hop1 = copy_to_reg (qop1);
   28240          251 :           hop2 = copy_to_reg (qop2);
   28241          251 :           emit_insn (gen_vec_interleave_lowv16qi (hop1, hop1, hop1));
   28242          251 :           emit_insn (gen_vec_interleave_lowv16qi (hop2, hop2, hop2));
   28243          251 :           break;
   28244              :         }
   28245              :       /* FALLTHRU */
   28246         1142 :     case ASHIFT:
   28247         1142 :     case ASHIFTRT:
   28248         1142 :     case LSHIFTRT:
   28249         1142 :       hop1 = gen_reg_rtx (V8HImode);
   28250         1142 :       ix86_expand_sse_unpack (hop1, qop1, uns_p, false);
   28251              :       /* mult/vashr/vlshr/vashl  */
   28252         1142 :       if (op2vec)
   28253              :         {
   28254         1066 :           hop2 = gen_reg_rtx (V8HImode);
   28255         1066 :           ix86_expand_sse_unpack (hop2, qop2, uns_p, false);
   28256              :         }
   28257              :       else
   28258              :         hop2 = qop2;
   28259              : 
   28260              :       break;
   28261            0 :     default:
   28262            0 :       gcc_unreachable ();
   28263              :     }
   28264              : 
   28265         1393 :   if (code != MULT && op2vec)
   28266              :     {
   28267              :       /* Expand vashr/vlshr/vashl.  */
   28268           13 :       hdest = gen_reg_rtx (V8HImode);
   28269           13 :       emit_insn (gen_rtx_SET (hdest,
   28270              :                               simplify_gen_binary (code, V8HImode,
   28271              :                                                    hop1, hop2)));
   28272              :     }
   28273              :   else
   28274              :     /* Expand mult/ashr/lshr/ashl.  */
   28275         1380 :     hdest = expand_simple_binop (V8HImode, code, hop1, hop2,
   28276              :                                 NULL_RTX, 1, OPTAB_DIRECT);
   28277              : 
   28278         1393 :   if (TARGET_AVX512BW && TARGET_AVX512VL)
   28279              :     {
   28280           57 :       if (qimode == V8QImode)
   28281              :         qdest = dest;
   28282              :       else
   28283           10 :         qdest = gen_reg_rtx (V8QImode);
   28284              : 
   28285           57 :       emit_insn (gen_truncv8hiv8qi2 (qdest, hdest));
   28286              :     }
   28287              :   else
   28288              :     {
   28289         1336 :       struct expand_vec_perm_d d;
   28290         1336 :       rtx qres = gen_lowpart (V16QImode, hdest);
   28291         1336 :       bool ok;
   28292         1336 :       int i;
   28293              : 
   28294              :       /* Merge the data back into the right place.  */
   28295         1336 :       d.target = qdest;
   28296         1336 :       d.op0 = d.op1 = qres;
   28297         1336 :       d.vmode = V16QImode;
   28298         1336 :       d.nelt = 16;
   28299         1336 :       d.one_operand_p = TARGET_SSSE3;
   28300         1336 :       d.testing_p = false;
   28301              : 
   28302        22712 :       for (i = 0; i < d.nelt; ++i)
   28303        21376 :         d.perm[i] = i * 2;
   28304              : 
   28305         1336 :       ok = ix86_expand_vec_perm_const_1 (&d);
   28306         1336 :       gcc_assert (ok);
   28307              :     }
   28308              : 
   28309         1393 :   if (qdest != dest)
   28310         1346 :     emit_move_insn (dest, gen_lowpart (qimode, qdest));
   28311              : }
   28312              : 
   28313              : /* Emit instruction in 2x wider mode.  For example, optimize
   28314              :    vector MUL generation like
   28315              : 
   28316              :    vpmovzxbw ymm2, xmm0
   28317              :    vpmovzxbw ymm3, xmm1
   28318              :    vpmullw   ymm4, ymm2, ymm3
   28319              :    vpmovwb   xmm0, ymm4
   28320              : 
   28321              :    it would take less instructions than ix86_expand_vecop_qihi.
   28322              :    Return true if success.  */
   28323              : 
   28324              : static bool
   28325         1185 : ix86_expand_vecop_qihi2 (enum rtx_code code, rtx dest, rtx op1, rtx op2)
   28326              : {
   28327         1185 :   machine_mode himode, qimode = GET_MODE (dest);
   28328         1185 :   machine_mode wqimode;
   28329         1185 :   rtx qop1, qop2, hop1, hop2, hdest;
   28330         1185 :   rtx (*gen_truncate)(rtx, rtx) = NULL;
   28331         1185 :   bool op2vec = GET_MODE_CLASS (GET_MODE (op2)) == MODE_VECTOR_INT;
   28332         1185 :   bool uns_p = code != ASHIFTRT;
   28333              : 
   28334              :   /* Without VPMOVWB (provided by AVX512BW ISA), the expansion uses the
   28335              :      generic permutation to merge the data back into the right place.  This
   28336              :      permutation results in VPERMQ, which is slow, so better fall back to
   28337              :      ix86_expand_vecop_qihi.  */
   28338         1185 :   if (!TARGET_AVX512BW
   28339          327 :       || (qimode == V16QImode && !TARGET_AVX512VL)
   28340              :       /* There are no V64HImode instructions.  */
   28341          327 :       || qimode == V64QImode)
   28342              :      return false;
   28343              : 
   28344              :   /* Do not generate ymm/zmm instructions when
   28345              :      target prefers 128/256 bit vector width.  */
   28346          317 :   if ((qimode == V16QImode && TARGET_PREFER_AVX128)
   28347          317 :       || (qimode == V32QImode && TARGET_PREFER_AVX256))
   28348              :     return false;
   28349              : 
   28350          312 :   switch (qimode)
   28351              :     {
   28352              :     case E_V16QImode:
   28353              :       himode = V16HImode;
   28354              :       gen_truncate = gen_truncv16hiv16qi2;
   28355              :       break;
   28356           57 :     case E_V32QImode:
   28357           57 :       himode = V32HImode;
   28358           57 :       gen_truncate = gen_truncv32hiv32qi2;
   28359           57 :       break;
   28360            0 :     default:
   28361            0 :       gcc_unreachable ();
   28362              :     }
   28363              : 
   28364          312 :   wqimode = GET_MODE_2XWIDER_MODE (qimode).require ();
   28365          312 :   qop1 = lowpart_subreg (wqimode, force_reg (qimode, op1), qimode);
   28366              : 
   28367          312 :   if (op2vec)
   28368          312 :     qop2 = lowpart_subreg (wqimode, force_reg (qimode, op2), qimode);
   28369              :   else
   28370              :     qop2 = op2;
   28371              : 
   28372          312 :   hop1 = gen_reg_rtx (himode);
   28373          312 :   ix86_expand_sse_unpack (hop1, qop1, uns_p, false);
   28374              : 
   28375          312 :   if (op2vec)
   28376              :     {
   28377          312 :       hop2 = gen_reg_rtx (himode);
   28378          312 :       ix86_expand_sse_unpack (hop2, qop2, uns_p, false);
   28379              :     }
   28380              :   else
   28381              :     hop2 = qop2;
   28382              : 
   28383          312 :   if (code != MULT && op2vec)
   28384              :     {
   28385              :       /* Expand vashr/vlshr/vashl.  */
   28386           14 :       hdest = gen_reg_rtx (himode);
   28387           14 :       emit_insn (gen_rtx_SET (hdest,
   28388              :                               simplify_gen_binary (code, himode,
   28389              :                                                    hop1, hop2)));
   28390              :     }
   28391              :   else
   28392              :     /* Expand mult/ashr/lshr/ashl.  */
   28393          298 :     hdest = expand_simple_binop (himode, code, hop1, hop2,
   28394              :                                  NULL_RTX, 1, OPTAB_DIRECT);
   28395              : 
   28396          312 :   emit_insn (gen_truncate (dest, hdest));
   28397          312 :   return true;
   28398              : }
   28399              : 
   28400              : /* Expand a vector operation CODE for a V*QImode in terms of the
   28401              :    same operation on V*HImode.  */
   28402              : 
   28403              : void
   28404         1590 : ix86_expand_vecop_qihi (enum rtx_code code, rtx dest, rtx op1, rtx op2)
   28405              : {
   28406         1590 :   machine_mode qimode = GET_MODE (dest);
   28407         1590 :   machine_mode himode;
   28408         1590 :   rtx (*gen_il) (rtx, rtx, rtx);
   28409         1590 :   rtx (*gen_ih) (rtx, rtx, rtx);
   28410         1590 :   rtx op1_l, op1_h, op2_l, op2_h, res_l, res_h;
   28411         1590 :   bool op2vec = GET_MODE_CLASS (GET_MODE (op2)) == MODE_VECTOR_INT;
   28412         1590 :   struct expand_vec_perm_d d;
   28413         1590 :   bool full_interleave = true;
   28414         1590 :   bool uns_p = code != ASHIFTRT;
   28415         1590 :   bool ok;
   28416         1590 :   int i;
   28417              : 
   28418         1590 :   if (CONST_INT_P (op2)
   28419          405 :       && (code == ASHIFT || code == LSHIFTRT || code == ASHIFTRT)
   28420         1995 :       && ix86_expand_vec_shift_qihi_constant (code, dest, op1, op2))
   28421          717 :     return;
   28422              : 
   28423         1185 :   if (ix86_expand_vecop_qihi2 (code, dest, op1, op2))
   28424              :     return;
   28425              : 
   28426          873 :   switch (qimode)
   28427              :     {
   28428              :     case E_V16QImode:
   28429              :       himode = V8HImode;
   28430              :       break;
   28431           44 :     case E_V32QImode:
   28432           44 :       himode = V16HImode;
   28433           44 :       break;
   28434           10 :     case E_V64QImode:
   28435           10 :       himode = V32HImode;
   28436           10 :       break;
   28437            0 :     default:
   28438            0 :       gcc_unreachable ();
   28439              :     }
   28440              : 
   28441          873 :   switch (code)
   28442              :     {
   28443          819 :     case MULT:
   28444          819 :       gcc_assert (op2vec);
   28445              :       /* Unpack data such that we've got a source byte in each low byte of
   28446              :          each word.  We don't care what goes into the high byte of each word.
   28447              :          Rather than trying to get zero in there, most convenient is to let
   28448              :          it be a copy of the low byte.  */
   28449          819 :       switch (qimode)
   28450              :         {
   28451              :         case E_V16QImode:
   28452              :           gen_il = gen_vec_interleave_lowv16qi;
   28453              :           gen_ih = gen_vec_interleave_highv16qi;
   28454              :           break;
   28455           44 :         case E_V32QImode:
   28456           44 :           gen_il = gen_avx2_interleave_lowv32qi;
   28457           44 :           gen_ih = gen_avx2_interleave_highv32qi;
   28458           44 :           full_interleave = false;
   28459           44 :           break;
   28460            8 :         case E_V64QImode:
   28461            8 :           gen_il = gen_avx512bw_interleave_lowv64qi;
   28462            8 :           gen_ih = gen_avx512bw_interleave_highv64qi;
   28463            8 :           full_interleave = false;
   28464            8 :           break;
   28465              :         default:
   28466              :           gcc_unreachable ();
   28467              :         }
   28468              : 
   28469          819 :       op2_l = gen_reg_rtx (qimode);
   28470          819 :       op2_h = gen_reg_rtx (qimode);
   28471          819 :       emit_insn (gen_il (op2_l, op2, op2));
   28472          819 :       emit_insn (gen_ih (op2_h, op2, op2));
   28473              : 
   28474          819 :       op1_l = gen_reg_rtx (qimode);
   28475          819 :       op1_h = gen_reg_rtx (qimode);
   28476          819 :       emit_insn (gen_il (op1_l, op1, op1));
   28477          819 :       emit_insn (gen_ih (op1_h, op1, op1));
   28478          819 :       break;
   28479              : 
   28480           54 :     case ASHIFT:
   28481           54 :     case ASHIFTRT:
   28482           54 :     case LSHIFTRT:
   28483           54 :       op1_l = gen_reg_rtx (himode);
   28484           54 :       op1_h = gen_reg_rtx (himode);
   28485           54 :       ix86_expand_sse_unpack (op1_l, op1, uns_p, false);
   28486           54 :       ix86_expand_sse_unpack (op1_h, op1, uns_p, true);
   28487              :       /* vashr/vlshr/vashl  */
   28488           54 :       if (op2vec)
   28489              :         {
   28490            2 :           rtx tmp = force_reg (qimode, op2);
   28491            2 :           op2_l = gen_reg_rtx (himode);
   28492            2 :           op2_h = gen_reg_rtx (himode);
   28493            2 :           ix86_expand_sse_unpack (op2_l, tmp, uns_p, false);
   28494            2 :           ix86_expand_sse_unpack (op2_h, tmp, uns_p, true);
   28495              :         }
   28496              :       else
   28497              :         op2_l = op2_h = op2;
   28498              : 
   28499              :       break;
   28500            0 :     default:
   28501            0 :       gcc_unreachable ();
   28502              :     }
   28503              : 
   28504          873 :   if (code != MULT && op2vec)
   28505              :     {
   28506              :       /* Expand vashr/vlshr/vashl.  */
   28507            2 :       res_l = gen_reg_rtx (himode);
   28508            2 :       res_h = gen_reg_rtx (himode);
   28509            2 :       emit_insn (gen_rtx_SET (res_l,
   28510              :                               simplify_gen_binary (code, himode,
   28511              :                                                    op1_l, op2_l)));
   28512            2 :       emit_insn (gen_rtx_SET (res_h,
   28513              :                               simplify_gen_binary (code, himode,
   28514              :                                                    op1_h, op2_h)));
   28515              :     }
   28516              :   else
   28517              :     {
   28518              :       /* Expand mult/ashr/lshr/ashl.  */
   28519          871 :       res_l = expand_simple_binop (himode, code, op1_l, op2_l, NULL_RTX,
   28520              :                                    1, OPTAB_DIRECT);
   28521          871 :       res_h = expand_simple_binop (himode, code, op1_h, op2_h, NULL_RTX,
   28522              :                                    1, OPTAB_DIRECT);
   28523              :     }
   28524              : 
   28525          873 :   gcc_assert (res_l && res_h);
   28526              : 
   28527              :   /* Merge the data back into the right place.  */
   28528          873 :   d.target = dest;
   28529          873 :   d.op0 = gen_lowpart (qimode, res_l);
   28530          873 :   d.op1 = gen_lowpart (qimode, res_h);
   28531          873 :   d.vmode = qimode;
   28532          873 :   d.nelt = GET_MODE_NUNITS (qimode);
   28533          873 :   d.one_operand_p = false;
   28534          873 :   d.testing_p = false;
   28535              : 
   28536          873 :   if (full_interleave)
   28537              :     {
   28538              :       /* We used the full interleave, the desired
   28539              :          results are in the even elements.  */
   28540        14053 :       for (i = 0; i < d.nelt; ++i)
   28541        13232 :         d.perm[i] = i * 2;
   28542              :     }
   28543              :   else
   28544              :     {
   28545              :       /* For AVX, the interleave used above was not cross-lane.  So the
   28546              :          extraction is evens but with the second and third quarter swapped.
   28547              :          Happily, that is even one insn shorter than even extraction.
   28548              :          For AVX512BW we have 4 lanes.  We extract evens from within a lane,
   28549              :          always first from the first and then from the second source operand,
   28550              :          the index bits above the low 4 bits remains the same.
   28551              :          Thus, for d.nelt == 32 we want permutation
   28552              :          0,2,4,..14, 32,34,36,..46, 16,18,20,..30, 48,50,52,..62
   28553              :          and for d.nelt == 64 we want permutation
   28554              :          0,2,4,..14, 64,66,68,..78, 16,18,20,..30, 80,82,84,..94,
   28555              :          32,34,36,..46, 96,98,100,..110, 48,50,52,..62, 112,114,116,..126.  */
   28556         1972 :       for (i = 0; i < d.nelt; ++i)
   28557         2880 :         d.perm[i] = ((i * 2) & 14) + ((i & 8) ? d.nelt : 0) + (i & ~15);
   28558              :     }
   28559              : 
   28560          873 :   ok = ix86_expand_vec_perm_const_1 (&d);
   28561          873 :   gcc_assert (ok);
   28562              : }
   28563              : 
   28564              : /* Helper function of ix86_expand_mul_widen_evenodd.  Return true
   28565              :    if op is CONST_VECTOR with all odd elements equal to their
   28566              :    preceding element.  */
   28567              : 
   28568              : static bool
   28569         9132 : const_vector_equal_evenodd_p (rtx op)
   28570              : {
   28571         9132 :   machine_mode mode = GET_MODE (op);
   28572         9132 :   int i, nunits = GET_MODE_NUNITS (mode);
   28573         9132 :   if (!CONST_VECTOR_P (op)
   28574         9132 :       || nunits != CONST_VECTOR_NUNITS (op))
   28575              :     return false;
   28576         3642 :   for (i = 0; i < nunits; i += 2)
   28577         2933 :     if (CONST_VECTOR_ELT (op, i) != CONST_VECTOR_ELT (op, i + 1))
   28578              :       return false;
   28579              :   return true;
   28580              : }
   28581              : 
   28582              : void
   28583         9253 : ix86_expand_mul_widen_evenodd (rtx dest, rtx op1, rtx op2,
   28584              :                                bool uns_p, bool odd_p)
   28585              : {
   28586         9253 :   machine_mode mode = GET_MODE (op1);
   28587         9253 :   machine_mode wmode = GET_MODE (dest);
   28588         9253 :   rtx x;
   28589         9253 :   rtx orig_op1 = op1, orig_op2 = op2;
   28590              : 
   28591         9253 :   if (!nonimmediate_operand (op1, mode))
   28592            0 :     op1 = force_reg (mode, op1);
   28593         9253 :   if (!nonimmediate_operand (op2, mode))
   28594         3414 :     op2 = force_reg (mode, op2);
   28595              : 
   28596              :   /* We only play even/odd games with vectors of SImode.  */
   28597         9253 :   gcc_assert (mode == V4SImode || mode == V8SImode || mode == V16SImode);
   28598              : 
   28599              :   /* If we're looking for the odd results, shift those members down to
   28600              :      the even slots.  For some cpus this is faster than a PSHUFD.  */
   28601         9253 :   if (odd_p)
   28602              :     {
   28603              :       /* For XOP use vpmacsdqh, but only for smult, as it is only
   28604              :          signed.  */
   28605         4584 :       if (TARGET_XOP && mode == V4SImode && !uns_p)
   28606              :         {
   28607           18 :           x = force_reg (wmode, CONST0_RTX (wmode));
   28608           18 :           emit_insn (gen_xop_pmacsdqh (dest, op1, op2, x));
   28609           18 :           return;
   28610              :         }
   28611              : 
   28612         9132 :       x = GEN_INT (GET_MODE_UNIT_BITSIZE (mode));
   28613         4566 :       if (!const_vector_equal_evenodd_p (orig_op1))
   28614         4566 :         op1 = expand_binop (wmode, lshr_optab, gen_lowpart (wmode, op1),
   28615              :                             x, NULL, 1, OPTAB_DIRECT);
   28616         4566 :       if (!const_vector_equal_evenodd_p (orig_op2))
   28617         3857 :         op2 = expand_binop (wmode, lshr_optab, gen_lowpart (wmode, op2),
   28618              :                             x, NULL, 1, OPTAB_DIRECT);
   28619         4566 :       op1 = gen_lowpart (mode, op1);
   28620         4566 :       op2 = gen_lowpart (mode, op2);
   28621              :     }
   28622              : 
   28623         9235 :   if (mode == V16SImode)
   28624              :     {
   28625            2 :       if (uns_p)
   28626            0 :         x = gen_vec_widen_umult_even_v16si (dest, op1, op2);
   28627              :       else
   28628            2 :         x = gen_vec_widen_smult_even_v16si (dest, op1, op2);
   28629              :     }
   28630         9233 :   else if (mode == V8SImode)
   28631              :     {
   28632          139 :       if (uns_p)
   28633           59 :         x = gen_vec_widen_umult_even_v8si (dest, op1, op2);
   28634              :       else
   28635           80 :         x = gen_vec_widen_smult_even_v8si (dest, op1, op2);
   28636              :     }
   28637         9094 :   else if (uns_p)
   28638         7991 :     x = gen_vec_widen_umult_even_v4si (dest, op1, op2);
   28639         1103 :   else if (TARGET_SSE4_1)
   28640          367 :     x = gen_sse4_1_mulv2siv2di3 (dest, op1, op2);
   28641              :   else
   28642              :     {
   28643          736 :       rtx s1, s2, t0, t1, t2;
   28644              : 
   28645              :       /* The easiest way to implement this without PMULDQ is to go through
   28646              :          the motions as if we are performing a full 64-bit multiply.  With
   28647              :          the exception that we need to do less shuffling of the elements.  */
   28648              : 
   28649              :       /* Compute the sign-extension, aka highparts, of the two operands.  */
   28650          736 :       s1 = ix86_expand_sse_cmp (gen_reg_rtx (mode), GT, CONST0_RTX (mode),
   28651              :                                 op1, pc_rtx, pc_rtx);
   28652          736 :       s2 = ix86_expand_sse_cmp (gen_reg_rtx (mode), GT, CONST0_RTX (mode),
   28653              :                                 op2, pc_rtx, pc_rtx);
   28654              : 
   28655              :       /* Multiply LO(A) * HI(B), and vice-versa.  */
   28656          736 :       t1 = gen_reg_rtx (wmode);
   28657          736 :       t2 = gen_reg_rtx (wmode);
   28658          736 :       emit_insn (gen_vec_widen_umult_even_v4si (t1, s1, op2));
   28659          736 :       emit_insn (gen_vec_widen_umult_even_v4si (t2, s2, op1));
   28660              : 
   28661              :       /* Multiply LO(A) * LO(B).  */
   28662          736 :       t0 = gen_reg_rtx (wmode);
   28663          736 :       emit_insn (gen_vec_widen_umult_even_v4si (t0, op1, op2));
   28664              : 
   28665              :       /* Combine and shift the highparts into place.  */
   28666          736 :       t1 = expand_binop (wmode, add_optab, t1, t2, t1, 1, OPTAB_DIRECT);
   28667          736 :       t1 = expand_binop (wmode, ashl_optab, t1, GEN_INT (32), t1,
   28668              :                          1, OPTAB_DIRECT);
   28669              : 
   28670              :       /* Combine high and low parts.  */
   28671          736 :       force_expand_binop (wmode, add_optab, t0, t1, dest, 1, OPTAB_DIRECT);
   28672          736 :       return;
   28673              :     }
   28674         8499 :   emit_insn (x);
   28675              : }
   28676              : 
   28677              : void
   28678         1011 : ix86_expand_mul_widen_hilo (rtx dest, rtx op1, rtx op2,
   28679              :                             bool uns_p, bool high_p)
   28680              : {
   28681         1011 :   machine_mode wmode = GET_MODE (dest);
   28682         1011 :   machine_mode mode = GET_MODE (op1);
   28683         1011 :   rtx t1, t2, t3, t4, mask;
   28684              : 
   28685         1011 :   switch (mode)
   28686              :     {
   28687          299 :     case E_V4SImode:
   28688          299 :       t1 = gen_reg_rtx (mode);
   28689          299 :       t2 = gen_reg_rtx (mode);
   28690          299 :       if (TARGET_XOP && !uns_p)
   28691              :         {
   28692              :           /* With XOP, we have pmacsdqh, aka mul_widen_odd.  In this case,
   28693              :              shuffle the elements once so that all elements are in the right
   28694              :              place for immediate use: { A C B D }.  */
   28695           33 :           emit_insn (gen_sse2_pshufd_1 (t1, op1, const0_rtx, const2_rtx,
   28696              :                                         const1_rtx, GEN_INT (3)));
   28697           33 :           emit_insn (gen_sse2_pshufd_1 (t2, op2, const0_rtx, const2_rtx,
   28698              :                                         const1_rtx, GEN_INT (3)));
   28699              :         }
   28700              :       else
   28701              :         {
   28702              :           /* Put the elements into place for the multiply.  */
   28703          266 :           ix86_expand_vec_interleave (t1, op1, op1, high_p);
   28704          266 :           ix86_expand_vec_interleave (t2, op2, op2, high_p);
   28705          266 :           high_p = false;
   28706              :         }
   28707          299 :       ix86_expand_mul_widen_evenodd (dest, t1, t2, uns_p, high_p);
   28708          299 :       break;
   28709              : 
   28710           70 :     case E_V8SImode:
   28711              :       /* Shuffle the elements between the lanes.  After this we
   28712              :          have { A B E F | C D G H } for each operand.  */
   28713           70 :       t1 = gen_reg_rtx (V4DImode);
   28714           70 :       t2 = gen_reg_rtx (V4DImode);
   28715           70 :       emit_insn (gen_avx2_permv4di_1 (t1, gen_lowpart (V4DImode, op1),
   28716              :                                       const0_rtx, const2_rtx,
   28717              :                                       const1_rtx, GEN_INT (3)));
   28718           70 :       emit_insn (gen_avx2_permv4di_1 (t2, gen_lowpart (V4DImode, op2),
   28719              :                                       const0_rtx, const2_rtx,
   28720              :                                       const1_rtx, GEN_INT (3)));
   28721              : 
   28722              :       /* Shuffle the elements within the lanes.  After this we
   28723              :          have { A A B B | C C D D } or { E E F F | G G H H }.  */
   28724           70 :       t3 = gen_reg_rtx (V8SImode);
   28725           70 :       t4 = gen_reg_rtx (V8SImode);
   28726          105 :       mask = GEN_INT (high_p
   28727              :                       ? 2 + (2 << 2) + (3 << 4) + (3 << 6)
   28728              :                       : 0 + (0 << 2) + (1 << 4) + (1 << 6));
   28729           70 :       emit_insn (gen_avx2_pshufdv3 (t3, gen_lowpart (V8SImode, t1), mask));
   28730           70 :       emit_insn (gen_avx2_pshufdv3 (t4, gen_lowpart (V8SImode, t2), mask));
   28731              : 
   28732           70 :       ix86_expand_mul_widen_evenodd (dest, t3, t4, uns_p, false);
   28733           70 :       break;
   28734              : 
   28735          412 :     case E_V8HImode:
   28736          412 :     case E_V16HImode:
   28737          412 :       t1 = expand_binop (mode, smul_optab, op1, op2, NULL_RTX,
   28738              :                          uns_p, OPTAB_DIRECT);
   28739          662 :       t2 = expand_binop (mode,
   28740              :                          uns_p ? umul_highpart_optab : smul_highpart_optab,
   28741              :                          op1, op2, NULL_RTX, uns_p, OPTAB_DIRECT);
   28742          412 :       gcc_assert (t1 && t2);
   28743              : 
   28744          412 :       t3 = gen_reg_rtx (mode);
   28745          412 :       ix86_expand_vec_interleave (t3, t1, t2, high_p);
   28746          412 :       emit_move_insn (dest, gen_lowpart (wmode, t3));
   28747          412 :       break;
   28748              : 
   28749          230 :     case E_V16QImode:
   28750          230 :     case E_V32QImode:
   28751          230 :     case E_V32HImode:
   28752          230 :     case E_V16SImode:
   28753          230 :     case E_V64QImode:
   28754          230 :       t1 = gen_reg_rtx (wmode);
   28755          230 :       t2 = gen_reg_rtx (wmode);
   28756          230 :       ix86_expand_sse_unpack (t1, op1, uns_p, high_p);
   28757          230 :       ix86_expand_sse_unpack (t2, op2, uns_p, high_p);
   28758              : 
   28759          230 :       emit_insn (gen_rtx_SET (dest, gen_rtx_MULT (wmode, t1, t2)));
   28760          230 :       break;
   28761              : 
   28762            0 :     default:
   28763            0 :       gcc_unreachable ();
   28764              :     }
   28765         1011 : }
   28766              : 
   28767              : void
   28768         3829 : ix86_expand_sse2_mulv4si3 (rtx op0, rtx op1, rtx op2)
   28769              : {
   28770         3829 :   rtx res_1, res_2, res_3, res_4;
   28771              : 
   28772         3829 :   res_1 = gen_reg_rtx (V4SImode);
   28773         3829 :   res_2 = gen_reg_rtx (V4SImode);
   28774         3829 :   res_3 = gen_reg_rtx (V2DImode);
   28775         3829 :   res_4 = gen_reg_rtx (V2DImode);
   28776         3829 :   ix86_expand_mul_widen_evenodd (res_3, op1, op2, true, false);
   28777         3829 :   ix86_expand_mul_widen_evenodd (res_4, op1, op2, true, true);
   28778              : 
   28779              :   /* Move the results in element 2 down to element 1; we don't care
   28780              :      what goes in elements 2 and 3.  Then we can merge the parts
   28781              :      back together with an interleave.
   28782              : 
   28783              :      Note that two other sequences were tried:
   28784              :      (1) Use interleaves at the start instead of psrldq, which allows
   28785              :      us to use a single shufps to merge things back at the end.
   28786              :      (2) Use shufps here to combine the two vectors, then pshufd to
   28787              :      put the elements in the correct order.
   28788              :      In both cases the cost of the reformatting stall was too high
   28789              :      and the overall sequence slower.  */
   28790              : 
   28791         3829 :   emit_insn (gen_sse2_pshufd_1 (res_1, gen_lowpart (V4SImode, res_3),
   28792              :                                 const0_rtx, const2_rtx,
   28793              :                                 const0_rtx, const0_rtx));
   28794         3829 :   emit_insn (gen_sse2_pshufd_1 (res_2, gen_lowpart (V4SImode, res_4),
   28795              :                                 const0_rtx, const2_rtx,
   28796              :                                 const0_rtx, const0_rtx));
   28797         3829 :   rtx_insn *insn = emit_insn (gen_vec_interleave_lowv4si (op0, res_1, res_2));
   28798              : 
   28799         3829 :   set_unique_reg_note (insn, REG_EQUAL, gen_rtx_MULT (V4SImode, op1, op2));
   28800         3829 : }
   28801              : 
   28802              : void
   28803          612 : ix86_expand_sse2_mulvxdi3 (rtx op0, rtx op1, rtx op2)
   28804              : {
   28805          612 :   machine_mode mode = GET_MODE (op0);
   28806          612 :   rtx t1, t2, t3, t4, t5, t6;
   28807              : 
   28808          612 :   if (TARGET_AVX512DQ && mode == V8DImode)
   28809           32 :     emit_insn (gen_avx512dq_mulv8di3 (op0, op1, op2));
   28810          580 :   else if (TARGET_AVX512DQ && TARGET_AVX512VL && mode == V4DImode)
   28811           32 :     emit_insn (gen_avx512dq_mulv4di3 (op0, op1, op2));
   28812          548 :   else if (TARGET_AVX512DQ && TARGET_AVX512VL && mode == V2DImode)
   28813           37 :     emit_insn (gen_avx512dq_mulv2di3 (op0, op1, op2));
   28814          511 :   else if (TARGET_XOP && mode == V2DImode)
   28815              :     {
   28816              :       /* op1: A,B,C,D, op2: E,F,G,H */
   28817            2 :       op1 = gen_lowpart (V4SImode, op1);
   28818            2 :       op2 = gen_lowpart (V4SImode, op2);
   28819              : 
   28820            2 :       t1 = gen_reg_rtx (V4SImode);
   28821            2 :       t2 = gen_reg_rtx (V4SImode);
   28822            2 :       t3 = gen_reg_rtx (V2DImode);
   28823            2 :       t4 = gen_reg_rtx (V2DImode);
   28824              : 
   28825              :       /* t1: B,A,D,C */
   28826            2 :       emit_insn (gen_sse2_pshufd_1 (t1, op1,
   28827              :                                     GEN_INT (1),
   28828              :                                     GEN_INT (0),
   28829              :                                     GEN_INT (3),
   28830              :                                     GEN_INT (2)));
   28831              : 
   28832              :       /* t2: (B*E),(A*F),(D*G),(C*H) */
   28833            2 :       emit_insn (gen_mulv4si3 (t2, t1, op2));
   28834              : 
   28835              :       /* t3: (B*E)+(A*F), (D*G)+(C*H) */
   28836            2 :       emit_insn (gen_xop_phadddq (t3, t2));
   28837              : 
   28838              :       /* t4: ((B*E)+(A*F))<<32, ((D*G)+(C*H))<<32 */
   28839            2 :       emit_insn (gen_ashlv2di3 (t4, t3, GEN_INT (32)));
   28840              : 
   28841              :       /* Multiply lower parts and add all */
   28842            2 :       t5 = gen_reg_rtx (V2DImode);
   28843            2 :       emit_insn (gen_vec_widen_umult_even_v4si (t5,
   28844            2 :                                         gen_lowpart (V4SImode, op1),
   28845            2 :                                         gen_lowpart (V4SImode, op2)));
   28846            2 :       force_expand_binop (mode, add_optab, t5, t4, op0, 1, OPTAB_DIRECT);
   28847              :     }
   28848              :   else
   28849              :     {
   28850          509 :       machine_mode nmode;
   28851          509 :       rtx (*umul) (rtx, rtx, rtx);
   28852              : 
   28853          509 :       if (mode == V2DImode)
   28854              :         {
   28855              :           umul = gen_vec_widen_umult_even_v4si;
   28856              :           nmode = V4SImode;
   28857              :         }
   28858          319 :       else if (mode == V4DImode)
   28859              :         {
   28860              :           umul = gen_vec_widen_umult_even_v8si;
   28861              :           nmode = V8SImode;
   28862              :         }
   28863          116 :       else if (mode == V8DImode)
   28864              :         {
   28865              :           umul = gen_vec_widen_umult_even_v16si;
   28866              :           nmode = V16SImode;
   28867              :         }
   28868              :       else
   28869            0 :         gcc_unreachable ();
   28870              : 
   28871              : 
   28872              :       /* Multiply low parts.  */
   28873          509 :       t1 = gen_reg_rtx (mode);
   28874          509 :       emit_insn (umul (t1, gen_lowpart (nmode, op1), gen_lowpart (nmode, op2)));
   28875              : 
   28876              :       /* Shift input vectors right 32 bits so we can multiply high parts.  */
   28877          509 :       t6 = GEN_INT (32);
   28878          509 :       t2 = expand_binop (mode, lshr_optab, op1, t6, NULL, 1, OPTAB_DIRECT);
   28879          509 :       t3 = expand_binop (mode, lshr_optab, op2, t6, NULL, 1, OPTAB_DIRECT);
   28880              : 
   28881              :       /* Multiply high parts by low parts.  */
   28882          509 :       t4 = gen_reg_rtx (mode);
   28883          509 :       t5 = gen_reg_rtx (mode);
   28884          509 :       emit_insn (umul (t4, gen_lowpart (nmode, t2), gen_lowpart (nmode, op2)));
   28885          509 :       emit_insn (umul (t5, gen_lowpart (nmode, t3), gen_lowpart (nmode, op1)));
   28886              : 
   28887              :       /* Combine and shift the highparts back.  */
   28888          509 :       t4 = expand_binop (mode, add_optab, t4, t5, t4, 1, OPTAB_DIRECT);
   28889          509 :       t4 = expand_binop (mode, ashl_optab, t4, t6, t4, 1, OPTAB_DIRECT);
   28890              : 
   28891              :       /* Combine high and low parts.  */
   28892          509 :       force_expand_binop (mode, add_optab, t1, t4, op0, 1, OPTAB_DIRECT);
   28893              :     }
   28894              : 
   28895          612 :   set_unique_reg_note (get_last_insn (), REG_EQUAL,
   28896              :                        gen_rtx_MULT (mode, op1, op2));
   28897          612 : }
   28898              : 
   28899              : void
   28900           30 : ix86_expand_umulvndi_highpart (rtx op0, rtx op1, rtx op2)
   28901              : {
   28902              :   /* The following sequence can be used with SSE2.  */
   28903           30 :   machine_mode mode = GET_MODE (op0);
   28904           30 :   rtx (*umul) (rtx, rtx, rtx);
   28905           30 :   machine_mode nmode;
   28906           30 :   if (mode == V2DImode)
   28907              :     {
   28908              :       umul = gen_vec_widen_umult_even_v4si;
   28909              :       nmode = V4SImode;
   28910              :     }
   28911              :   else if (mode == V4DImode)
   28912              :     {
   28913              :       umul = gen_vec_widen_umult_even_v8si;
   28914              :       nmode = V8SImode;
   28915              :     }
   28916              :   else if (mode == V8DImode)
   28917              :     {
   28918              :       umul = gen_vec_widen_umult_even_v16si;
   28919              :       nmode = V16SImode;
   28920              :     }
   28921              :   else
   28922            0 :     gcc_unreachable ();
   28923              : 
   28924           30 :   rtx t32 = GEN_INT (32);
   28925              : 
   28926              :   /* Shift input vectors right 32 bits so we can get high parts of operands.  */
   28927           30 :   rtx op1_hi = expand_binop (mode, lshr_optab, op1, t32, NULL, 1, OPTAB_DIRECT);
   28928           30 :   rtx op2_hi = expand_binop (mode, lshr_optab, op2, t32, NULL, 1, OPTAB_DIRECT);
   28929              : 
   28930              :   /* Multiply low parts.  */
   28931           30 :   rtx p0 = gen_reg_rtx (mode);
   28932           30 :   emit_insn (umul (p0, gen_lowpart (nmode, op1), gen_lowpart (nmode, op2)));
   28933              : 
   28934              :   /* Multiply high parts by low parts.  */
   28935           30 :   rtx p1 = gen_reg_rtx (mode);
   28936           30 :   emit_insn (umul (p1, gen_lowpart (nmode, op1_hi), gen_lowpart (nmode, op2)));
   28937           30 :   rtx p2 = gen_reg_rtx (mode);
   28938           30 :   emit_insn (umul (p2, gen_lowpart (nmode, op1), gen_lowpart (nmode, op2_hi)));
   28939              : 
   28940              :   /* Multiply high parts.  */
   28941           30 :   rtx p3 = gen_reg_rtx (mode);
   28942           30 :   emit_insn (umul (p3, gen_lowpart (nmode, op1_hi),
   28943           30 :                    gen_lowpart (nmode, op2_hi)));
   28944              : 
   28945              :   /* Fold the carry in once cross product at a time.
   28946              :      t = (p0 >> 32) + p1  */
   28947           30 :   rtx p0_hi = expand_binop (mode, lshr_optab, p0, t32, NULL, 1, OPTAB_DIRECT);
   28948           30 :   rtx t = expand_binop (mode, add_optab, p0_hi, p1, NULL, 1, OPTAB_DIRECT);
   28949              : 
   28950              :   /* Extract low 32 bits of products by masking with 0xffffffff.  */
   28951           30 :   rtx mask = gen_reg_rtx (mode);
   28952           30 :   emit_insn (gen_rtx_SET (mask, CONSTM1_RTX (mode)));
   28953           30 :   mask = expand_binop (mode, lshr_optab, mask, t32, NULL, 1, OPTAB_DIRECT);
   28954              : 
   28955              :   /* u = (t & MASK) + p2  */
   28956           30 :   rtx u = expand_binop (mode, and_optab, t, mask, NULL, 1, OPTAB_DIRECT);
   28957           30 :   u = expand_binop (mode, add_optab, u, p2, NULL, 1, OPTAB_DIRECT);
   28958              : 
   28959              :   /* high = p3 + (t >> 32) + (u >> 32)  */
   28960           30 :   rtx t_hi = expand_binop (mode, lshr_optab, t, t32, NULL, 1, OPTAB_DIRECT);
   28961           30 :   rtx u_hi = expand_binop (mode, lshr_optab, u, t32, NULL, 1, OPTAB_DIRECT);
   28962           30 :   rtx high = expand_binop (mode, add_optab, p3, t_hi, NULL, 1, OPTAB_DIRECT);
   28963           30 :   force_expand_binop (mode, add_optab, high, u_hi, op0, 1, OPTAB_DIRECT);
   28964           30 : }
   28965              : 
   28966              : /* Return 1 if control transfer instruction INSN
   28967              :    should be encoded with notrack prefix.  */
   28968              : 
   28969              : bool
   28970     15216135 : ix86_notrack_prefixed_insn_p (rtx_insn *insn)
   28971              : {
   28972     15216135 :   if (!insn || !((flag_cf_protection & CF_BRANCH)))
   28973              :     return false;
   28974              : 
   28975      4117626 :   if (CALL_P (insn))
   28976              :     {
   28977      1492206 :       rtx call = get_call_rtx_from (insn);
   28978      1492206 :       gcc_assert (call != NULL_RTX);
   28979      1492206 :       rtx addr = XEXP (call, 0);
   28980              : 
   28981              :       /* Do not emit 'notrack' if it's not an indirect call.  */
   28982      1492206 :       if (MEM_P (addr)
   28983      1492206 :           && SYMBOL_REF_P (XEXP (addr, 0)))
   28984              :         return false;
   28985              :       else
   28986        65987 :         return find_reg_note (insn, REG_CALL_NOCF_CHECK, 0);
   28987              :     }
   28988              : 
   28989      2625420 :   if (JUMP_P (insn) && !flag_cet_switch)
   28990              :     {
   28991      2611766 :       rtx target = JUMP_LABEL (insn);
   28992      2611766 :       if (target == NULL_RTX || ANY_RETURN_P (target))
   28993              :         return false;
   28994              : 
   28995              :       /* Check the jump is a switch table.  */
   28996      2611728 :       rtx_insn *label = as_a<rtx_insn *> (target);
   28997      2611728 :       rtx_insn *table = next_insn (label);
   28998      2611728 :       if (table == NULL_RTX || !JUMP_TABLE_DATA_P (table))
   28999              :         return false;
   29000              :       else
   29001         5474 :         return true;
   29002              :     }
   29003              :   return false;
   29004              : }
   29005              : 
   29006              : /* Calculate integer abs() using only SSE2 instructions.  */
   29007              : 
   29008              : void
   29009          653 : ix86_expand_sse2_abs (rtx target, rtx input)
   29010              : {
   29011          653 :   machine_mode mode = GET_MODE (target);
   29012          653 :   rtx tmp0, tmp1, x;
   29013              : 
   29014          653 :   switch (mode)
   29015              :     {
   29016           50 :     case E_V2DImode:
   29017           50 :     case E_V4DImode:
   29018              :       /* For 64-bit signed integer X, with SSE4.2 use
   29019              :          pxor t0, t0; pcmpgtq X, t0; pxor t0, X; psubq t0, X.
   29020              :          Otherwise handle it similarly to V4SImode, except use 64 as W instead of
   29021              :          32 and use logical instead of arithmetic right shift (which is
   29022              :          unimplemented) and subtract.  */
   29023           50 :       if (TARGET_SSE4_2)
   29024              :         {
   29025            9 :           tmp0 = gen_reg_rtx (mode);
   29026            9 :           tmp1 = gen_reg_rtx (mode);
   29027            9 :           emit_move_insn (tmp1, CONST0_RTX (mode));
   29028            9 :           if (mode == E_V2DImode)
   29029            6 :             emit_insn (gen_sse4_2_gtv2di3 (tmp0, tmp1, input));
   29030              :           else
   29031            3 :             emit_insn (gen_avx2_gtv4di3 (tmp0, tmp1, input));
   29032              :         }
   29033              :       else
   29034              :         {
   29035           82 :           tmp0 = expand_simple_binop (mode, LSHIFTRT, input,
   29036           41 :                                       GEN_INT (GET_MODE_UNIT_BITSIZE (mode)
   29037              :                                                - 1), NULL, 0, OPTAB_DIRECT);
   29038           41 :           tmp0 = expand_simple_unop (mode, NEG, tmp0, NULL, false);
   29039              :         }
   29040              : 
   29041           50 :       tmp1 = expand_simple_binop (mode, XOR, tmp0, input,
   29042              :                                   NULL, 0, OPTAB_DIRECT);
   29043           50 :       x = expand_simple_binop (mode, MINUS, tmp1, tmp0,
   29044              :                                target, 0, OPTAB_DIRECT);
   29045           50 :       break;
   29046              : 
   29047           76 :     case E_V4SImode:
   29048              :       /* For 32-bit signed integer X, the best way to calculate the absolute
   29049              :          value of X is (((signed) X >> (W-1)) ^ X) - ((signed) X >> (W-1)).  */
   29050           76 :       tmp0 = expand_simple_binop (mode, ASHIFTRT, input,
   29051           76 :                                   GEN_INT (GET_MODE_UNIT_BITSIZE (mode) - 1),
   29052              :                                   NULL, 0, OPTAB_DIRECT);
   29053           76 :       tmp1 = expand_simple_binop (mode, XOR, tmp0, input,
   29054              :                                   NULL, 0, OPTAB_DIRECT);
   29055           76 :       x = expand_simple_binop (mode, MINUS, tmp1, tmp0,
   29056              :                                target, 0, OPTAB_DIRECT);
   29057           76 :       break;
   29058              : 
   29059           97 :     case E_V8HImode:
   29060              :       /* For 16-bit signed integer X, the best way to calculate the absolute
   29061              :          value of X is max (X, -X), as SSE2 provides the PMAXSW insn.  */
   29062           97 :       tmp0 = expand_unop (mode, neg_optab, input, NULL_RTX, 0);
   29063              : 
   29064           97 :       x = expand_simple_binop (mode, SMAX, tmp0, input,
   29065              :                                target, 0, OPTAB_DIRECT);
   29066           97 :       break;
   29067              : 
   29068          430 :     case E_V16QImode:
   29069              :       /* For 8-bit signed integer X, the best way to calculate the absolute
   29070              :          value of X is min ((unsigned char) X, (unsigned char) (-X)),
   29071              :          as SSE2 provides the PMINUB insn.  */
   29072          430 :       tmp0 = expand_unop (mode, neg_optab, input, NULL_RTX, 0);
   29073              : 
   29074          430 :       x = expand_simple_binop (V16QImode, UMIN, tmp0, input,
   29075              :                                target, 0, OPTAB_DIRECT);
   29076          430 :       break;
   29077              : 
   29078            0 :     default:
   29079            0 :       gcc_unreachable ();
   29080              :     }
   29081              : 
   29082          653 :   if (x != target)
   29083            0 :     emit_move_insn (target, x);
   29084          653 : }
   29085              : 
   29086              : /* Expand an extract from a vector register through pextr insn.
   29087              :    Return true if successful.  */
   29088              : 
   29089              : bool
   29090       120648 : ix86_expand_pextr (rtx *operands)
   29091              : {
   29092       120648 :   rtx dst = operands[0];
   29093       120648 :   rtx src = operands[1];
   29094              : 
   29095       120648 :   unsigned int size = INTVAL (operands[2]);
   29096       120648 :   unsigned int pos = INTVAL (operands[3]);
   29097              : 
   29098       120648 :   if (SUBREG_P (dst))
   29099              :     {
   29100              :       /* Reject non-lowpart subregs.  */
   29101        77236 :       if (SUBREG_BYTE (dst) > 0)
   29102              :         return false;
   29103        77107 :       dst = SUBREG_REG (dst);
   29104              :     }
   29105              : 
   29106       120519 :   if (SUBREG_P (src))
   29107              :     {
   29108        34489 :       pos += SUBREG_BYTE (src) * BITS_PER_UNIT;
   29109        34489 :       src = SUBREG_REG (src);
   29110              :     }
   29111              : 
   29112       120519 :   switch (GET_MODE (src))
   29113              :     {
   29114            0 :     case E_V16QImode:
   29115            0 :     case E_V8HImode:
   29116            0 :     case E_V4SImode:
   29117            0 :     case E_V2DImode:
   29118            0 :     case E_V1TImode:
   29119            0 :       {
   29120            0 :         machine_mode srcmode, dstmode;
   29121            0 :         rtx d, pat;
   29122              : 
   29123            0 :         if (!int_mode_for_size (size, 0).exists (&dstmode))
   29124            0 :           return false;
   29125              : 
   29126            0 :         switch (dstmode)
   29127              :           {
   29128            0 :           case E_QImode:
   29129            0 :             if (!TARGET_SSE4_1)
   29130              :               return false;
   29131              :             srcmode = V16QImode;
   29132              :             break;
   29133              : 
   29134            0 :           case E_HImode:
   29135            0 :             if (!TARGET_SSE2)
   29136              :               return false;
   29137              :             srcmode = V8HImode;
   29138              :             break;
   29139              : 
   29140            0 :           case E_SImode:
   29141            0 :             if (!TARGET_SSE4_1)
   29142              :               return false;
   29143              :             srcmode = V4SImode;
   29144              :             break;
   29145              : 
   29146            0 :           case E_DImode:
   29147            0 :             gcc_assert (TARGET_64BIT);
   29148            0 :             if (!TARGET_SSE4_1)
   29149              :               return false;
   29150              :             srcmode = V2DImode;
   29151              :             break;
   29152              : 
   29153              :           default:
   29154              :             return false;
   29155              :           }
   29156              : 
   29157              :         /* Reject extractions from misaligned positions.  */
   29158            0 :         if (pos & (size-1))
   29159              :           return false;
   29160              : 
   29161            0 :         if (GET_MODE (dst) == dstmode)
   29162              :           d = dst;
   29163              :         else
   29164            0 :           d = gen_reg_rtx (dstmode);
   29165              : 
   29166              :         /* Construct insn pattern.  */
   29167            0 :         pat = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (pos / size)));
   29168            0 :         pat = gen_rtx_VEC_SELECT (dstmode, gen_lowpart (srcmode, src), pat);
   29169              : 
   29170              :         /* Let the rtl optimizers know about the zero extension performed.  */
   29171            0 :         if (dstmode == QImode || dstmode == HImode)
   29172              :           {
   29173            0 :             pat = gen_rtx_ZERO_EXTEND (SImode, pat);
   29174            0 :             d = gen_lowpart (SImode, d);
   29175              :           }
   29176              : 
   29177            0 :         emit_insn (gen_rtx_SET (d, pat));
   29178              : 
   29179            0 :         if (d != dst)
   29180            0 :           emit_move_insn (dst, gen_lowpart (GET_MODE (dst), d));
   29181              :         return true;
   29182              :       }
   29183              : 
   29184              :     default:
   29185              :       return false;
   29186              :     }
   29187              : }
   29188              : 
   29189              : /* Expand an insert into a vector register through pinsr insn.
   29190              :    Return true if successful.  */
   29191              : 
   29192              : bool
   29193       110159 : ix86_expand_pinsr (rtx *operands)
   29194              : {
   29195       110159 :   rtx dst = operands[0];
   29196       110159 :   rtx src = operands[3];
   29197              : 
   29198       110159 :   unsigned int size = INTVAL (operands[1]);
   29199       110159 :   unsigned int pos = INTVAL (operands[2]);
   29200              : 
   29201       110159 :   if (SUBREG_P (dst))
   29202              :     {
   29203        63134 :       pos += SUBREG_BYTE (dst) * BITS_PER_UNIT;
   29204        63134 :       dst = SUBREG_REG (dst);
   29205              :     }
   29206              : 
   29207       110159 :   switch (GET_MODE (dst))
   29208              :     {
   29209           20 :     case E_V16QImode:
   29210           20 :     case E_V8HImode:
   29211           20 :     case E_V4SImode:
   29212           20 :     case E_V2DImode:
   29213           20 :     case E_V1TImode:
   29214           20 :       {
   29215           20 :         machine_mode srcmode, dstmode;
   29216           20 :         rtx (*pinsr)(rtx, rtx, rtx, rtx);
   29217           20 :         rtx d;
   29218              : 
   29219           20 :         if (!int_mode_for_size (size, 0).exists (&srcmode))
   29220            0 :           return false;
   29221              : 
   29222           20 :         switch (srcmode)
   29223              :           {
   29224            1 :           case E_QImode:
   29225            1 :             if (!TARGET_SSE4_1)
   29226              :               return false;
   29227              :             dstmode = V16QImode;
   29228              :             pinsr = gen_sse4_1_pinsrb;
   29229              :             break;
   29230              : 
   29231            5 :           case E_HImode:
   29232            5 :             if (!TARGET_SSE2)
   29233              :               return false;
   29234              :             dstmode = V8HImode;
   29235              :             pinsr = gen_sse2_pinsrw;
   29236              :             break;
   29237              : 
   29238           14 :           case E_SImode:
   29239           14 :             if (!TARGET_SSE4_1)
   29240              :               return false;
   29241              :             dstmode = V4SImode;
   29242              :             pinsr = gen_sse4_1_pinsrd;
   29243              :             break;
   29244              : 
   29245            0 :           case E_DImode:
   29246            0 :             gcc_assert (TARGET_64BIT);
   29247            0 :             if (!TARGET_SSE4_1)
   29248              :               return false;
   29249              :             dstmode = V2DImode;
   29250              :             pinsr = gen_sse4_1_pinsrq;
   29251              :             break;
   29252              : 
   29253              :           default:
   29254              :             return false;
   29255              :           }
   29256              : 
   29257              :         /* Reject insertions to misaligned positions.  */
   29258            7 :         if (pos & (size-1))
   29259              :           return false;
   29260              : 
   29261            7 :         if (SUBREG_P (src))
   29262              :           {
   29263            7 :             unsigned int srcpos = SUBREG_BYTE (src);
   29264              : 
   29265            7 :             if (srcpos > 0)
   29266              :               {
   29267            0 :                 rtx extr_ops[4];
   29268              : 
   29269            0 :                 extr_ops[0] = gen_reg_rtx (srcmode);
   29270            0 :                 extr_ops[1] = gen_lowpart (srcmode, SUBREG_REG (src));
   29271            0 :                 extr_ops[2] = GEN_INT (size);
   29272            0 :                 extr_ops[3] = GEN_INT (srcpos * BITS_PER_UNIT);
   29273              : 
   29274            0 :                 if (!ix86_expand_pextr (extr_ops))
   29275            0 :                   return false;
   29276              : 
   29277            0 :                 src = extr_ops[0];
   29278              :               }
   29279              :             else
   29280            7 :               src = gen_lowpart (srcmode, SUBREG_REG (src));
   29281              :           }
   29282              : 
   29283            7 :         if (GET_MODE (dst) == dstmode)
   29284              :           d = dst;
   29285              :         else
   29286            7 :           d = gen_reg_rtx (dstmode);
   29287              : 
   29288            7 :         emit_insn (pinsr (d, gen_lowpart (dstmode, dst),
   29289            7 :                           gen_lowpart (srcmode, src),
   29290            7 :                           GEN_INT (1 << (pos / size))));
   29291            7 :         if (d != dst)
   29292            7 :           emit_move_insn (dst, gen_lowpart (GET_MODE (dst), d));
   29293              :         return true;
   29294              :       }
   29295              : 
   29296              :     default:
   29297              :       return false;
   29298              :     }
   29299              : }
   29300              : 
   29301              : /* All CPUs prefer to avoid cross-lane operations so perform reductions
   29302              :    upper against lower halves up to SSE reg size.  */
   29303              : 
   29304              : machine_mode
   29305         1847 : ix86_split_reduction (machine_mode mode)
   29306              : {
   29307              :   /* Reduce lowpart against highpart until we reach SSE reg width to
   29308              :      avoid cross-lane operations.  */
   29309         1847 :   switch (mode)
   29310              :     {
   29311              :     case E_V8DImode:
   29312              :     case E_V4DImode:
   29313              :       return V2DImode;
   29314            9 :     case E_V16SImode:
   29315            9 :     case E_V8SImode:
   29316            9 :       return V4SImode;
   29317            8 :     case E_V32HImode:
   29318            8 :     case E_V16HImode:
   29319            8 :       return V8HImode;
   29320            4 :     case E_V64QImode:
   29321            4 :     case E_V32QImode:
   29322            4 :       return V16QImode;
   29323            5 :     case E_V16SFmode:
   29324            5 :     case E_V8SFmode:
   29325            5 :       return V4SFmode;
   29326           16 :     case E_V8DFmode:
   29327           16 :     case E_V4DFmode:
   29328           16 :       return V2DFmode;
   29329         1800 :     default:
   29330         1800 :       return mode;
   29331              :     }
   29332              : }
   29333              : 
   29334              : /* Generate call to __divmoddi4.  */
   29335              : 
   29336              : void
   29337          892 : ix86_expand_divmod_libfunc (rtx libfunc, machine_mode mode,
   29338              :                             rtx op0, rtx op1,
   29339              :                             rtx *quot_p, rtx *rem_p)
   29340              : {
   29341         1784 :   rtx rem = assign_stack_temp (mode, GET_MODE_SIZE (mode));
   29342              : 
   29343          892 :   rtx quot = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
   29344              :                                       mode, op0, mode, op1, mode,
   29345          892 :                                       XEXP (rem, 0), Pmode);
   29346          892 :   *quot_p = quot;
   29347          892 :   *rem_p = rem;
   29348          892 : }
   29349              : 
   29350              : void
   29351           80 : ix86_expand_atomic_fetch_op_loop (rtx target, rtx mem, rtx val,
   29352              :                                   enum rtx_code code, bool after,
   29353              :                                   bool doubleword)
   29354              : {
   29355           80 :   rtx old_reg, new_reg, old_mem, success;
   29356           80 :   machine_mode mode = GET_MODE (target);
   29357           80 :   rtx_code_label *loop_label = NULL;
   29358              : 
   29359           80 :   old_reg = gen_reg_rtx (mode);
   29360           80 :   new_reg = old_reg;
   29361           80 :   old_mem = copy_to_reg (mem);
   29362           80 :   loop_label = gen_label_rtx ();
   29363           80 :   emit_label (loop_label);
   29364           80 :   emit_move_insn (old_reg, old_mem);
   29365              : 
   29366              :   /* return value for atomic_fetch_op.  */
   29367           80 :   if (!after)
   29368           40 :     emit_move_insn (target, old_reg);
   29369              : 
   29370           80 :   if (code == NOT)
   29371              :     {
   29372           20 :       new_reg = expand_simple_binop (mode, AND, new_reg, val, NULL_RTX,
   29373              :                                      true, OPTAB_LIB_WIDEN);
   29374           20 :       new_reg = expand_simple_unop (mode, code, new_reg, NULL_RTX, true);
   29375              :     }
   29376              :   else
   29377           60 :     new_reg = expand_simple_binop (mode, code, new_reg, val, NULL_RTX,
   29378              :                                    true, OPTAB_LIB_WIDEN);
   29379              : 
   29380              :   /* return value for atomic_op_fetch.  */
   29381           80 :   if (after)
   29382           40 :     emit_move_insn (target, new_reg);
   29383              : 
   29384           80 :   success = NULL_RTX;
   29385              : 
   29386           80 :   ix86_expand_cmpxchg_loop (&success, old_mem, mem, old_reg, new_reg,
   29387              :                             gen_int_mode (MEMMODEL_SYNC_SEQ_CST,
   29388              :                                           SImode),
   29389              :                             doubleword, loop_label);
   29390           80 : }
   29391              : 
   29392              : /* Relax cmpxchg instruction, param loop_label indicates whether
   29393              :    the instruction should be relaxed with a pause loop.  If not,
   29394              :    it will be relaxed to an atomic load + compare, and skip
   29395              :    cmpxchg instruction if mem != exp_input.  */
   29396              : 
   29397              : void
   29398           90 : ix86_expand_cmpxchg_loop (rtx *ptarget_bool, rtx target_val,
   29399              :                           rtx mem, rtx exp_input, rtx new_input,
   29400              :                           rtx mem_model, bool doubleword,
   29401              :                           rtx_code_label *loop_label)
   29402              : {
   29403           90 :   rtx_code_label *cmp_label = NULL;
   29404           90 :   rtx_code_label *done_label = NULL;
   29405           90 :   rtx target_bool = NULL_RTX, new_mem = NULL_RTX;
   29406           90 :   rtx (*gen) (rtx, rtx, rtx, rtx, rtx) = NULL;
   29407           90 :   rtx (*gendw) (rtx, rtx, rtx, rtx, rtx, rtx) = NULL;
   29408           90 :   machine_mode mode = GET_MODE (target_val), hmode = mode;
   29409              : 
   29410           90 :   if (*ptarget_bool == NULL)
   29411           80 :     target_bool = gen_reg_rtx (QImode);
   29412              :   else
   29413              :     target_bool = *ptarget_bool;
   29414              : 
   29415           90 :   cmp_label = gen_label_rtx ();
   29416           90 :   done_label = gen_label_rtx ();
   29417              : 
   29418           90 :   new_mem = gen_reg_rtx (mode);
   29419              :   /* Load memory first.  */
   29420           90 :   expand_atomic_load (new_mem, mem, MEMMODEL_SEQ_CST);
   29421              : 
   29422           90 :   switch (mode)
   29423              :     {
   29424              :     case E_TImode:
   29425              :       gendw = gen_atomic_compare_and_swapti_doubleword;
   29426              :       hmode = DImode;
   29427              :       break;
   29428           18 :     case E_DImode:
   29429           18 :       if (doubleword)
   29430              :         {
   29431              :           gendw = gen_atomic_compare_and_swapdi_doubleword;
   29432              :           hmode = SImode;
   29433              :         }
   29434              :       else
   29435              :         gen = gen_atomic_compare_and_swapdi_1;
   29436              :       break;
   29437           18 :     case E_SImode:
   29438           18 :       gen = gen_atomic_compare_and_swapsi_1;
   29439           18 :       break;
   29440           18 :     case E_HImode:
   29441           18 :       gen = gen_atomic_compare_and_swaphi_1;
   29442           18 :       break;
   29443           18 :     case E_QImode:
   29444           18 :       gen = gen_atomic_compare_and_swapqi_1;
   29445           18 :       break;
   29446            0 :     default:
   29447            0 :       gcc_unreachable ();
   29448              :     }
   29449              : 
   29450              :   /* Compare mem value with expected value.  */
   29451           72 :   if (doubleword)
   29452              :     {
   29453           18 :       rtx low_new_mem = gen_lowpart (hmode, new_mem);
   29454           18 :       rtx low_exp_input = gen_lowpart (hmode, exp_input);
   29455           18 :       rtx high_new_mem = gen_highpart (hmode, new_mem);
   29456           18 :       rtx high_exp_input = gen_highpart (hmode, exp_input);
   29457           18 :       emit_cmp_and_jump_insns (low_new_mem, low_exp_input, NE, NULL_RTX,
   29458              :                                hmode, 1, cmp_label,
   29459              :                                profile_probability::guessed_never ());
   29460           18 :       emit_cmp_and_jump_insns (high_new_mem, high_exp_input, NE, NULL_RTX,
   29461              :                                hmode, 1, cmp_label,
   29462              :                                profile_probability::guessed_never ());
   29463              :     }
   29464              :   else
   29465           72 :     emit_cmp_and_jump_insns (new_mem, exp_input, NE, NULL_RTX,
   29466           72 :                              GET_MODE (exp_input), 1, cmp_label,
   29467              :                              profile_probability::guessed_never ());
   29468              : 
   29469              :   /* Directly emits cmpxchg here.  */
   29470           90 :   if (doubleword)
   29471           36 :     emit_insn (gendw (target_val, mem, exp_input,
   29472           18 :                       gen_lowpart (hmode, new_input),
   29473              :                       gen_highpart (hmode, new_input),
   29474              :                       mem_model));
   29475              :   else
   29476           72 :     emit_insn (gen (target_val, mem, exp_input, new_input, mem_model));
   29477              : 
   29478           90 :   if (!loop_label)
   29479              :   {
   29480           10 :     emit_jump_insn (gen_jump (done_label));
   29481           10 :     emit_barrier ();
   29482           10 :     emit_label (cmp_label);
   29483           10 :     emit_move_insn (target_val, new_mem);
   29484           10 :     emit_label (done_label);
   29485           10 :     ix86_expand_setcc (target_bool, EQ, gen_rtx_REG (CCZmode, FLAGS_REG),
   29486              :                        const0_rtx);
   29487              :   }
   29488              :   else
   29489              :   {
   29490           80 :     ix86_expand_setcc (target_bool, EQ, gen_rtx_REG (CCZmode, FLAGS_REG),
   29491              :                        const0_rtx);
   29492           80 :     emit_cmp_and_jump_insns (target_bool, const0_rtx, EQ, const0_rtx,
   29493           80 :                              GET_MODE (target_bool), 1, loop_label,
   29494              :                              profile_probability::guessed_never ());
   29495           80 :     emit_jump_insn (gen_jump (done_label));
   29496           80 :     emit_barrier ();
   29497              : 
   29498              :     /* If mem is not expected, pause and loop back.  */
   29499           80 :     emit_label (cmp_label);
   29500           80 :     emit_move_insn (target_val, new_mem);
   29501           80 :     emit_insn (gen_pause ());
   29502           80 :     emit_jump_insn (gen_jump (loop_label));
   29503           80 :     emit_barrier ();
   29504           80 :     emit_label (done_label);
   29505              :   }
   29506              : 
   29507           90 :   *ptarget_bool = target_bool;
   29508           90 : }
   29509              : 
   29510              : /* Convert a BFmode VAL to SFmode without signaling sNaNs.
   29511              :    This is done by returning SF SUBREG of ((HI SUBREG) (VAL)) << 16.  */
   29512              : 
   29513              : rtx
   29514         4136 : ix86_expand_fast_convert_bf_to_sf (rtx val)
   29515              : {
   29516         4136 :   rtx op = gen_lowpart (HImode, val), ret;
   29517         4136 :   if (CONST_INT_P (op))
   29518              :     {
   29519          613 :       ret = simplify_const_unary_operation (FLOAT_EXTEND, SFmode,
   29520              :                                             val, BFmode);
   29521          613 :       if (ret)
   29522              :         return ret;
   29523              :       /* FLOAT_EXTEND simplification will fail if VAL is a sNaN.  */
   29524            1 :       ret = gen_reg_rtx (SImode);
   29525            1 :       emit_move_insn (ret, GEN_INT (INTVAL (op) & 0xffff));
   29526            1 :       emit_insn (gen_ashlsi3 (ret, ret, GEN_INT (16)));
   29527            1 :       return gen_lowpart (SFmode, ret);
   29528              :     }
   29529              : 
   29530         3523 :   ret = gen_reg_rtx (SFmode);
   29531         3523 :   emit_insn (gen_extendbfsf2_1 (ret, force_reg (BFmode, val)));
   29532         3523 :   return ret;
   29533              : }
   29534              : 
   29535              : rtx
   29536        65582 : ix86_gen_ccmp_first (rtx_insn **prep_seq, rtx_insn **gen_seq,
   29537              :                         rtx_code code, tree treeop0, tree treeop1)
   29538              : {
   29539        65582 :   if (!TARGET_APX_CCMP)
   29540              :     return NULL_RTX;
   29541              : 
   29542        65582 :   rtx op0, op1, res;
   29543        65582 :   machine_mode op_mode;
   29544              : 
   29545        65582 :   start_sequence ();
   29546        65582 :   expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
   29547              : 
   29548        65582 :   op_mode = GET_MODE (op0);
   29549        65582 :   if (op_mode == VOIDmode)
   29550            0 :     op_mode = GET_MODE (op1);
   29551              : 
   29552              :   /* We only supports following scalar comparisons that use just 1
   29553              :      instruction: DI/SI/HI/QI/XF/DF/SF/HF.
   29554              :      Unordered/Ordered compare cannot be correctly identified by
   29555              :      ccmp so they are not supported.  */
   29556        32771 :   if (!(op_mode == DImode || op_mode == SImode
   29557        65582 :         || op_mode == HImode || op_mode == QImode
   29558        32780 :         || ((op_mode == XFmode || op_mode == DFmode || op_mode == SFmode)
   29559           10 :             && (TARGET_80387
   29560            1 :                 || (SSE_FLOAT_MODE_P (op_mode) && TARGET_SSE_MATH)))
   29561            0 :         || (op_mode == HFmode && TARGET_AVX512FP16))
   29562        32811 :       || code == ORDERED
   29563        32811 :       || code == UNORDERED)
   29564              :     {
   29565        32771 :       end_sequence ();
   29566        32771 :       return NULL_RTX;
   29567              :     }
   29568              : 
   29569              :   /* Canonicalize the operands according to mode.  */
   29570        32811 :   if (SCALAR_INT_MODE_P (op_mode))
   29571              :     {
   29572        32802 :       if (!nonimmediate_operand (op0, op_mode))
   29573            0 :         op0 = force_reg (op_mode, op0);
   29574        32802 :       if (!x86_64_general_operand (op1, op_mode))
   29575            0 :         op1 = force_reg (op_mode, op1);
   29576              :     }
   29577              :   else
   29578              :     {
   29579              :       /* op0/op1 can be canonicallized from expand_fp_compare, so
   29580              :          just adjust the code to make it generate supported fp
   29581              :          condition.  */
   29582            9 :       if (ix86_fp_compare_code_to_integer (code) == UNKNOWN)
   29583              :         {
   29584              :           /* First try to split condition if we don't need to honor
   29585              :              NaNs, as the ORDERED/UNORDERED check always fall
   29586              :              through.  */
   29587            6 :           if (!HONOR_NANS (op_mode))
   29588              :             {
   29589            6 :               rtx_code first_code;
   29590            6 :               split_comparison (code, op_mode, &first_code, &code);
   29591              :             }
   29592              :           /* Otherwise try to swap the operand order and check if
   29593              :              the comparison is supported.  */
   29594              :           else
   29595              :             {
   29596            0 :               code = swap_condition (code);
   29597            0 :               std::swap (op0, op1);
   29598              :             }
   29599              : 
   29600            6 :           if (ix86_fp_compare_code_to_integer (code) == UNKNOWN)
   29601              :             {
   29602            0 :               end_sequence ();
   29603            0 :               return NULL_RTX;
   29604              :             }
   29605              :         }
   29606              :     }
   29607              : 
   29608        32811 :   *prep_seq = end_sequence ();
   29609              : 
   29610        32811 :   start_sequence ();
   29611              : 
   29612        32811 :   res = ix86_expand_compare (code, op0, op1);
   29613              : 
   29614        32811 :   if (!res)
   29615              :     {
   29616              :       end_sequence ();
   29617              :       return NULL_RTX;
   29618              :     }
   29619        32811 :   *gen_seq = end_sequence ();
   29620              : 
   29621        32811 :   return res;
   29622              : }
   29623              : 
   29624              : rtx
   29625        32814 : ix86_gen_ccmp_next (rtx_insn **prep_seq, rtx_insn **gen_seq, rtx prev,
   29626              :                        rtx_code cmp_code, tree treeop0, tree treeop1,
   29627              :                        rtx_code bit_code)
   29628              : {
   29629        32814 :   if (!TARGET_APX_CCMP)
   29630              :     return NULL_RTX;
   29631              : 
   29632        32814 :   rtx op0, op1, target;
   29633        32814 :   machine_mode op_mode, cmp_mode, cc_mode = CCmode;
   29634        32814 :   int unsignedp = TYPE_UNSIGNED (TREE_TYPE (treeop0));
   29635        32814 :   insn_code icode;
   29636        32814 :   rtx_code prev_code;
   29637        32814 :   struct expand_operand ops[5];
   29638        32814 :   int dfv;
   29639              : 
   29640              :   /* Exit early for non integer modes to avoid O(n^2) part of expand_operands. */
   29641        32814 :   cmp_mode = op_mode = TYPE_MODE (TREE_TYPE (treeop0));
   29642              : 
   29643        32814 :   if (!(op_mode == DImode || op_mode == SImode || op_mode == HImode
   29644              :         || op_mode == QImode))
   29645              :     return NULL_RTX;
   29646              : 
   29647           34 :   push_to_sequence (*prep_seq);
   29648           34 :   expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
   29649              : 
   29650           34 :   icode = code_for_ccmp (op_mode);
   29651              : 
   29652           34 :   op0 = prepare_operand (icode, op0, 2, op_mode, cmp_mode, unsignedp);
   29653           34 :   op1 = prepare_operand (icode, op1, 3, op_mode, cmp_mode, unsignedp);
   29654           34 :   if (!op0 || !op1)
   29655              :     {
   29656            0 :       end_sequence ();
   29657            0 :       return NULL_RTX;
   29658              :     }
   29659              : 
   29660           34 :   *prep_seq = end_sequence ();
   29661              : 
   29662           34 :   target = gen_rtx_REG (cc_mode, FLAGS_REG);
   29663           34 :   dfv = ix86_get_flags_cc ((rtx_code) cmp_code);
   29664              : 
   29665           34 :   prev_code = GET_CODE (prev);
   29666              :   /* Fixup FP compare code here.  */
   29667           34 :   if (GET_MODE (XEXP (prev, 0)) == CCFPmode)
   29668            9 :     prev_code = ix86_fp_compare_code_to_integer (prev_code);
   29669              : 
   29670           34 :   if (bit_code != AND)
   29671           17 :     prev_code = reverse_condition (prev_code);
   29672              :   else
   29673           17 :     dfv = (int)(dfv ^ 1);
   29674              : 
   29675           34 :   prev = gen_rtx_fmt_ee (prev_code, VOIDmode, XEXP (prev, 0),
   29676              :                          const0_rtx);
   29677              : 
   29678           34 :   create_fixed_operand (&ops[0], target);
   29679           34 :   create_fixed_operand (&ops[1], prev);
   29680           34 :   create_fixed_operand (&ops[2], op0);
   29681           34 :   create_fixed_operand (&ops[3], op1);
   29682           34 :   create_fixed_operand (&ops[4], GEN_INT (dfv));
   29683              : 
   29684           34 :   push_to_sequence (*gen_seq);
   29685           34 :   if (!maybe_expand_insn (icode, 5, ops))
   29686              :     {
   29687            0 :       end_sequence ();
   29688            0 :       return NULL_RTX;
   29689              :     }
   29690              : 
   29691           34 :   *gen_seq = end_sequence ();
   29692              : 
   29693           34 :   return gen_rtx_fmt_ee ((rtx_code) cmp_code, VOIDmode, target, const0_rtx);
   29694              : }
   29695              : 
   29696              : /* Attempt to convert a CONST_VECTOR into a bcst_mem_operand.
   29697              :    Returns NULL_RTX if X is cannot be expressed as a suitable
   29698              :    VEC_DUPLICATE in mode MODE.  */
   29699              : 
   29700              : static rtx
   29701           48 : ix86_gen_bcst_mem (machine_mode mode, rtx x)
   29702              : {
   29703           48 :   if (!TARGET_AVX512F
   29704           48 :       || !CONST_VECTOR_P (x)
   29705           64 :       || (!TARGET_AVX512VL && GET_MODE_SIZE (mode) != 64)
   29706          147 :       || !VALID_BCST_MODE_P (GET_MODE_INNER (mode))
   29707              :          /* Disallow HFmode broadcast.  */
   29708          126 :       || GET_MODE_SIZE (GET_MODE_INNER (mode)) < 4)
   29709              :     return NULL_RTX;
   29710              : 
   29711           21 :   rtx cst = CONST_VECTOR_ELT (x, 0);
   29712           21 :   if (!CONST_SCALAR_INT_P (cst)
   29713           15 :       && !CONST_DOUBLE_P (cst)
   29714            0 :       && !CONST_FIXED_P (cst))
   29715              :     return NULL_RTX;
   29716              : 
   29717           21 :   int n_elts = GET_MODE_NUNITS (mode);
   29718           42 :   if (CONST_VECTOR_NUNITS (x) != n_elts)
   29719              :     return NULL_RTX;
   29720              : 
   29721          150 :   for (int i = 1; i < n_elts; i++)
   29722          129 :     if (!rtx_equal_p (cst, CONST_VECTOR_ELT (x, i)))
   29723              :       return NULL_RTX;
   29724              : 
   29725           42 :   rtx mem = force_const_mem (GET_MODE_INNER (mode), cst);
   29726           21 :   return gen_rtx_VEC_DUPLICATE (mode, validize_mem (mem));
   29727              : }
   29728              : 
   29729              : /* Determine the ternlog immediate index that implements 3-operand
   29730              :    ternary logic expression OP.  This uses and modifies the 3 element
   29731              :    array ARGS to record and check the leaves, either 3 REGs, or 2 REGs
   29732              :    and MEM.  Returns an index between 0 and 255 for a valid ternlog,
   29733              :    or -1 if the expression isn't suitable.  */
   29734              : 
   29735              : int
   29736      7592958 : ix86_ternlog_idx (rtx op, rtx *args)
   29737              : {
   29738      7592958 :   int idx0, idx1;
   29739              : 
   29740      7592958 :   if (!op)
   29741              :     return -1;
   29742              : 
   29743      7592958 :   switch (GET_CODE (op))
   29744              :     {
   29745       748676 :     case SUBREG:
   29746       748676 :       if (!register_operand (op, GET_MODE (op)))
   29747              :         return -1;
   29748              :       /* FALLTHRU */
   29749              : 
   29750      3746169 :     case REG:
   29751      3746169 :       if (!args[0])
   29752              :         {
   29753      1930205 :           args[0] = op;
   29754      1930205 :           return 0xf0;
   29755              :         }
   29756      1815964 :       if (rtx_equal_p (op, args[0]))
   29757              :         return 0xf0;
   29758      1789437 :       if (!args[1])
   29759              :         {
   29760      1512563 :           args[1] = op;
   29761      1512563 :           return 0xcc;
   29762              :         }
   29763       276874 :       if (rtx_equal_p (op, args[1]))
   29764              :         return 0xcc;
   29765       260376 :       if (!args[2])
   29766              :         {
   29767       237153 :           args[2] = op;
   29768       237153 :           return 0xaa;
   29769              :         }
   29770        23223 :       if (rtx_equal_p (op, args[2]))
   29771         1461 :         return 0xaa;
   29772              :       return -1;
   29773              : 
   29774        19671 :     case VEC_DUPLICATE:
   29775        19671 :       if (!bcst_mem_operand (op, GET_MODE (op)))
   29776              :         return -1;
   29777          308 :       goto do_mem_operand;
   29778              : 
   29779       355062 :     case MEM:
   29780       355062 :       if (!memory_operand (op, GET_MODE (op)))
   29781              :         return -1;
   29782       354919 :       if (MEM_P (op)
   29783       354919 :           && MEM_VOLATILE_P (op)
   29784       355013 :           && !volatile_ok)
   29785              :         return -1;
   29786              :       /* FALLTHRU */
   29787              : 
   29788       470864 :     case CONST_VECTOR:
   29789       470864 : do_mem_operand:
   29790       470864 :       if (!args[2])
   29791              :         {
   29792       423350 :           args[2] = op;
   29793       423350 :           return 0xaa;
   29794              :         }
   29795              :       /* Maximum of one volatile memory reference per expression.  */
   29796        47514 :       if (side_effects_p (op))
   29797              :         return -1;
   29798        47514 :       if (rtx_equal_p (op, args[2]))
   29799              :         return 0xaa;
   29800              :       /* Check if CONST_VECTOR is the ones-complement of args[2].  */
   29801        47463 :       if (CONST_VECTOR_P (op)
   29802         3655 :           && CONST_VECTOR_P (args[2])
   29803        47708 :           && rtx_equal_p (simplify_const_unary_operation (NOT, GET_MODE (op),
   29804          245 :                                                           op, GET_MODE (op)),
   29805              :                           args[2]))
   29806              :         return 0x55;
   29807        47276 :       if (!args[0])
   29808              :         {
   29809        45452 :           args[0] = op;
   29810        45452 :           return 0xf0;
   29811              :         }
   29812         1824 :       if (rtx_equal_p (op, args[0]))
   29813              :         return 0xf0;
   29814              :       /* Check if CONST_VECTOR is the ones-complement of args[0].  */
   29815         1824 :       if (CONST_VECTOR_P (op)
   29816          111 :           && CONST_VECTOR_P (args[0])
   29817         1866 :           && rtx_equal_p (simplify_const_unary_operation (NOT, GET_MODE (op),
   29818           42 :                                                           op, GET_MODE (op)),
   29819              :                           args[0]))
   29820              :         return 0x0f;
   29821         1782 :       if (!args[1])
   29822              :         {
   29823         1740 :           args[1] = op;
   29824         1740 :           return 0xcc;
   29825              :         }
   29826           42 :       if (rtx_equal_p (op, args[1]))
   29827              :         return 0xcc;
   29828              :       /* Check if CONST_VECTOR is the ones-complement of args[1].  */
   29829           42 :       if (CONST_VECTOR_P (op)
   29830            0 :           && CONST_VECTOR_P (args[1])
   29831           42 :           && rtx_equal_p (simplify_const_unary_operation (NOT, GET_MODE (op),
   29832            0 :                                                           op, GET_MODE (op)),
   29833              :                           args[1]))
   29834            0 :         return 0x33;
   29835              :       return -1;
   29836              : 
   29837       181198 :     case NOT:
   29838       181198 :       idx0 = ix86_ternlog_idx (XEXP (op, 0), args);
   29839       181198 :       return (idx0 >= 0) ? idx0 ^ 0xff : -1;
   29840              : 
   29841      1364388 :     case AND:
   29842      1364388 :       idx0 = ix86_ternlog_idx (XEXP (op, 0), args);
   29843      1364388 :       if (idx0 < 0)
   29844              :         return -1;
   29845      1122653 :       idx1 = ix86_ternlog_idx (XEXP (op, 1), args);
   29846      1122653 :       return (idx1 >= 0) ? idx0 & idx1 : -1;
   29847              : 
   29848       955519 :     case IOR:
   29849       955519 :       idx0 = ix86_ternlog_idx (XEXP (op, 0), args);
   29850       955519 :       if (idx0 < 0)
   29851              :         return -1;
   29852       708958 :       idx1 = ix86_ternlog_idx (XEXP (op, 1), args);
   29853       708958 :       return (idx1 >= 0) ? idx0 | idx1 : -1;
   29854              : 
   29855       456814 :     case XOR:
   29856       456814 :       idx0 = ix86_ternlog_idx (XEXP (op, 0), args);
   29857       456814 :       if (idx0 < 0)
   29858              :         return -1;
   29859       428395 :       if (vector_all_ones_operand (XEXP (op, 1), GET_MODE (op)))
   29860         6711 :         return idx0 ^ 0xff;
   29861       421684 :       idx1 = ix86_ternlog_idx (XEXP (op, 1), args);
   29862       421684 :       return (idx1 >= 0) ? idx0 ^ idx1 : -1;
   29863              : 
   29864         7878 :     case UNSPEC:
   29865         7878 :       if (XINT (op, 1) != UNSPEC_VTERNLOG
   29866            0 :           || XVECLEN (op, 0) != 4
   29867            0 :           || !CONST_INT_P (XVECEXP (op, 0, 3)))
   29868              :         return -1;
   29869              : 
   29870              :       /* TODO: Handle permuted operands.  */
   29871            0 :       if (ix86_ternlog_idx (XVECEXP (op, 0, 0), args) != 0xf0
   29872            0 :           || ix86_ternlog_idx (XVECEXP (op, 0, 1), args) != 0xcc
   29873            0 :           || ix86_ternlog_idx (XVECEXP (op, 0, 2), args) != 0xaa)
   29874              :         return -1;
   29875            0 :       return INTVAL (XVECEXP (op, 0, 3));
   29876              : 
   29877              :     default:
   29878              :       return -1;
   29879              :     }
   29880              : }
   29881              : 
   29882              : /* Return TRUE if OP (in mode MODE) is the leaf of a ternary logic
   29883              :    expression, such as a register or a memory reference.  */
   29884              : 
   29885              : bool
   29886      3520270 : ix86_ternlog_leaf_p (rtx op, machine_mode mode)
   29887              : {
   29888              :   /* We can't use memory_operand here, as it may return a different
   29889              :      value before and after reload (for volatile MEMs) which creates
   29890              :      problems splitting instructions.  */
   29891      3520270 :   return register_operand (op, mode)
   29892       740319 :          || MEM_P (op)
   29893       401118 :          || CONST_VECTOR_P (op)
   29894      3813675 :          || bcst_mem_operand (op, mode);
   29895              : }
   29896              : 
   29897              : /* Test whether OP is a 3-operand ternary logic expression suitable
   29898              :    for use in a ternlog instruction.  */
   29899              : 
   29900              : bool
   29901      2345732 : ix86_ternlog_operand_p (rtx op)
   29902              : {
   29903      2345732 :   rtx op0, op1;
   29904      2345732 :   rtx args[3];
   29905              : 
   29906      2345732 :   args[0] = NULL_RTX;
   29907      2345732 :   args[1] = NULL_RTX;
   29908      2345732 :   args[2] = NULL_RTX;
   29909      2345732 :   int idx = ix86_ternlog_idx (op, args);
   29910      2345732 :   if (idx < 0)
   29911              :     return false;
   29912              : 
   29913              :   /* Don't match simple (binary or unary) expressions.  */
   29914      1905922 :   machine_mode mode = GET_MODE (op);
   29915      1905922 :   switch (GET_CODE (op))
   29916              :     {
   29917       883592 :     case AND:
   29918       883592 :       op0 = XEXP (op, 0);
   29919       883592 :       op1 = XEXP (op, 1);
   29920              : 
   29921              :       /* Prefer pand.  */
   29922       883592 :       if (ix86_ternlog_leaf_p (op0, mode)
   29923       883592 :           && ix86_ternlog_leaf_p (op1, mode))
   29924              :         return false;
   29925              :       /* Prefer pandn.  */
   29926       110375 :       if (GET_CODE (op0) == NOT
   29927        74582 :           && register_operand (XEXP (op0, 0), mode)
   29928       181767 :           && ix86_ternlog_leaf_p (op1, mode))
   29929        69905 :         return false;
   29930              :       break;
   29931              : 
   29932       621615 :     case IOR:
   29933              :       /* Prefer por.  */
   29934       621615 :       if (ix86_ternlog_leaf_p (XEXP (op, 0), mode)
   29935       621615 :           && ix86_ternlog_leaf_p (XEXP (op, 1), mode))
   29936       462770 :         return false;
   29937              :       break;
   29938              : 
   29939       365080 :     case XOR:
   29940       365080 :       op1 = XEXP (op, 1);
   29941              :       /* Prefer pxor, or one_cmpl<vmode>2.  */
   29942       365080 :       if (ix86_ternlog_leaf_p (XEXP (op, 0), mode)
   29943       365080 :           && ix86_ternlog_leaf_p (XEXP (op, 1), mode))
   29944       342602 :         return false;
   29945              :       break;
   29946              : 
   29947              :     default:
   29948              :       break;
   29949              :     }
   29950              :   return true;
   29951              : }
   29952              : 
   29953              : /* Helper function for ix86_expand_ternlog.  */
   29954              : static rtx
   29955            0 : ix86_expand_ternlog_binop (enum rtx_code code, machine_mode mode,
   29956              :                            rtx op0, rtx op1, rtx target)
   29957              : {
   29958            0 :   if (GET_MODE (op0) != mode)
   29959            0 :     op0 = gen_lowpart (mode, op0);
   29960            0 :   if (GET_MODE (op1) != mode)
   29961            0 :     op1 = gen_lowpart (mode, op1);
   29962              : 
   29963            0 :   if (CONST_VECTOR_P (op0))
   29964            0 :     op0 = validize_mem (force_const_mem (mode, op0));
   29965            0 :   if (CONST_VECTOR_P (op1))
   29966            0 :     op1 = validize_mem (force_const_mem (mode, op1));
   29967              : 
   29968            0 :   if (!register_operand (op0, mode))
   29969              :     {
   29970            0 :       if (!register_operand (op1, mode))
   29971              :         {
   29972              :           /* We can't use force_reg (op0, mode).  */
   29973            0 :           rtx reg = gen_reg_rtx (mode);
   29974            0 :           emit_move_insn (reg, op0);
   29975            0 :           op0 = reg;
   29976              :         }
   29977              :       else
   29978              :         std::swap (op0, op1);
   29979              :     }
   29980            0 :   rtx ops[3] = { target, op0, op1 };
   29981            0 :   ix86_expand_vector_logical_operator (code, mode, ops);
   29982            0 :   return target;
   29983              : }
   29984              : 
   29985              : 
   29986              : /* Helper function for ix86_expand_ternlog.  */
   29987              : static rtx
   29988            0 : ix86_expand_ternlog_andnot (machine_mode mode, rtx op0, rtx op1, rtx target)
   29989              : {
   29990            0 :   if (GET_MODE (op0) != mode)
   29991            0 :     op0 = gen_lowpart (mode, op0);
   29992            0 :   op0 = gen_rtx_NOT (mode, op0);
   29993            0 :   if (GET_MODE (op1) != mode)
   29994            0 :     op1 = gen_lowpart (mode, op1);
   29995            0 :   if (CONST_VECTOR_P (op1))
   29996            0 :     op1 = validize_mem (force_const_mem (mode, op1));
   29997            0 :   emit_move_insn (target, gen_rtx_AND (mode, op0, op1));
   29998            0 :   return target;
   29999              : }
   30000              : 
   30001              : /* Expand a 3-operand ternary logic expression.  Return TARGET. */
   30002              : rtx
   30003         2430 : ix86_expand_ternlog (machine_mode mode, rtx op0, rtx op1, rtx op2, int idx,
   30004              :                      rtx target)
   30005              : {
   30006         2430 :   rtx tmp0, tmp1, tmp2;
   30007              : 
   30008         2430 :   if (!target)
   30009            3 :     target = gen_reg_rtx (mode);
   30010              : 
   30011              :   /* Canonicalize ternlog index for degenerate (duplicated) operands.  */
   30012         2430 :   if (rtx_equal_p (op0, op1) && rtx_equal_p (op0, op2))
   30013            0 :     switch (idx & 0x81)
   30014              :       {
   30015              :       case 0x00:
   30016              :         idx = 0x00;
   30017              :         break;
   30018              :       case 0x01:
   30019              :         idx = 0x0f;
   30020              :         break;
   30021              :       case 0x80:
   30022              :         idx = 0xf0;
   30023              :         break;
   30024              :       case 0x81:
   30025              :         idx = 0xff;
   30026              :         break;
   30027              :       }
   30028              : 
   30029         2430 :   switch (idx & 0xff)
   30030              :     {
   30031            0 :     case 0x00:
   30032            0 :       if ((!op0 || !side_effects_p (op0))
   30033            0 :           && (!op1 || !side_effects_p (op1))
   30034            0 :           && (!op2 || !side_effects_p (op2)))
   30035              :         {
   30036            0 :           emit_move_insn (target, CONST0_RTX (mode));
   30037            0 :           return target;
   30038              :         }
   30039              :       break;
   30040              : 
   30041            0 :     case 0x0a: /* ~a&c */
   30042            0 :       if ((!op1 || !side_effects_p (op1))
   30043            0 :           && op0 && register_operand (op0, mode)
   30044            0 :           && op2 && ix86_ternlog_leaf_p (op2, mode))
   30045            0 :         return ix86_expand_ternlog_andnot (mode, op0, op2, target);
   30046              :       break;
   30047              : 
   30048            0 :     case 0x0c: /* ~a&b */
   30049            0 :       if ((!op2 || !side_effects_p (op2))
   30050            0 :           && op0 && register_operand (op0, mode)
   30051            0 :           && op1 && ix86_ternlog_leaf_p (op1, mode))
   30052            0 :         return ix86_expand_ternlog_andnot (mode, op0, op1, target);
   30053              :       break;
   30054              : 
   30055           82 :     case 0x0f:  /* ~a */
   30056            0 :       if ((!op1 || !side_effects_p (op1))
   30057           82 :           && (!op2 || !side_effects_p (op2))
   30058          164 :           && op0)
   30059              :         {
   30060           82 :           emit_move_insn (target, gen_rtx_XOR (mode, op0, CONSTM1_RTX (mode)));
   30061           82 :           return target;
   30062              :         }
   30063              :       break;
   30064              : 
   30065            0 :     case 0x22: /* ~b&c */
   30066            0 :       if ((!op0 || !side_effects_p (op0))
   30067            0 :           && op1 && register_operand (op1, mode)
   30068            0 :           && op2 && ix86_ternlog_leaf_p (op2, mode))
   30069            0 :         return ix86_expand_ternlog_andnot (mode, op1, op2, target);
   30070              :       break;
   30071              : 
   30072            0 :     case 0x30: /* ~b&a */
   30073            0 :       if ((!op2 || !side_effects_p (op2))
   30074            0 :           && op0 && ix86_ternlog_leaf_p (op0, mode)
   30075            0 :           && op1 && register_operand (op1, mode))
   30076            0 :         return ix86_expand_ternlog_andnot (mode, op1, op0, target);
   30077              :       break;
   30078              : 
   30079            0 :     case 0x33:  /* ~b */
   30080            0 :       if ((!op0 || !side_effects_p (op0))
   30081            0 :           && (!op2 || !side_effects_p (op2))
   30082            0 :           && op1)
   30083              :         {
   30084            0 :           emit_move_insn (target, gen_rtx_XOR (mode, op1, CONSTM1_RTX (mode)));
   30085            0 :           return target;
   30086              :         }
   30087              :       break;
   30088              : 
   30089            0 :     case 0x3c:  /* a^b */
   30090            0 :       if (op0 && ix86_ternlog_leaf_p (op0, mode)
   30091            0 :           && op1 && ix86_ternlog_leaf_p (op1, mode)
   30092            0 :           && (!op2 || !side_effects_p (op2)))
   30093            0 :         return ix86_expand_ternlog_binop (XOR, mode, op0, op1, target);
   30094              :       break;
   30095              : 
   30096            0 :     case 0x44: /* ~c&b */
   30097            0 :       if ((!op0 || !side_effects_p (op0))
   30098            0 :           && op1 && ix86_ternlog_leaf_p (op1, mode)
   30099            0 :           && op2 && register_operand (op2, mode))
   30100            0 :         return ix86_expand_ternlog_andnot (mode, op2, op1, target);
   30101              :       break;
   30102              : 
   30103            2 :     case 0x50: /* ~c&a */
   30104            0 :       if ((!op1 || !side_effects_p (op1))
   30105            2 :           && op0 && ix86_ternlog_leaf_p (op0, mode)
   30106            4 :           && op2 && register_operand (op2, mode))
   30107            0 :         return ix86_expand_ternlog_andnot (mode, op2, op0, target);
   30108              :       break;
   30109              : 
   30110            4 :     case 0x55:  /* ~c */
   30111            1 :       if ((!op0 || !side_effects_p (op0))
   30112            4 :           && (!op1 || !side_effects_p (op1))
   30113            8 :           && op2)
   30114              :         {
   30115            4 :           emit_move_insn (target, gen_rtx_XOR (mode, op2, CONSTM1_RTX (mode)));
   30116            4 :           return target;
   30117              :         }
   30118              :       break;
   30119              : 
   30120            0 :     case 0x5a:  /* a^c */
   30121            0 :       if (op0 && ix86_ternlog_leaf_p (op0, mode)
   30122            0 :           && op2 && ix86_ternlog_leaf_p (op2, mode)
   30123            0 :           && (!op1 || !side_effects_p (op1)))
   30124            0 :         return ix86_expand_ternlog_binop (XOR, mode, op0, op2, target);
   30125              :       break;
   30126              : 
   30127            0 :     case 0x66:  /* b^c */
   30128            0 :       if ((!op0 || !side_effects_p (op0))
   30129            0 :           && op1 && ix86_ternlog_leaf_p (op1, mode)
   30130            0 :           && op2 && ix86_ternlog_leaf_p (op2, mode))
   30131            0 :         return ix86_expand_ternlog_binop (XOR, mode, op1, op2, target);
   30132              :       break;
   30133              : 
   30134            0 :     case 0x88:  /* b&c */
   30135            0 :       if ((!op0 || !side_effects_p (op0))
   30136            0 :           && op1 && ix86_ternlog_leaf_p (op1, mode)
   30137            0 :           && op2 && ix86_ternlog_leaf_p (op2, mode))
   30138            0 :         return ix86_expand_ternlog_binop (AND, mode, op1, op2, target);
   30139              :       break;
   30140              : 
   30141            0 :     case 0xa0:  /* a&c */
   30142            0 :       if ((!op1 || !side_effects_p (op1))
   30143            0 :           && op0 && ix86_ternlog_leaf_p (op0, mode)
   30144            0 :           && op2 && ix86_ternlog_leaf_p (op2, mode))
   30145            0 :         return ix86_expand_ternlog_binop (AND, mode, op0, op2, target);
   30146              :       break;
   30147              : 
   30148            0 :     case 0xaa:  /* c */
   30149            0 :       if ((!op0 || !side_effects_p (op0))
   30150            0 :           && (!op1 || !side_effects_p (op1))
   30151            0 :           && op2)
   30152              :         {
   30153            0 :           if (GET_MODE (op2) != mode)
   30154            0 :             op2 = gen_lowpart (mode, op2);
   30155            0 :           emit_move_insn (target, op2);
   30156            0 :           return target;
   30157              :         }
   30158              :       break;
   30159              : 
   30160            0 :     case 0xc0:  /* a&b */
   30161            0 :       if (op0 && ix86_ternlog_leaf_p (op0, mode)
   30162            0 :           && op1 && ix86_ternlog_leaf_p (op1, mode)
   30163            0 :           && (!op2 || !side_effects_p (op2)))
   30164            0 :         return ix86_expand_ternlog_binop (AND, mode, op0, op1, target);
   30165              :       break;
   30166              : 
   30167            0 :     case 0xcc:  /* b */
   30168            0 :       if ((!op0 || !side_effects_p (op0))
   30169            0 :           && op1
   30170            0 :           && (!op2 || !side_effects_p (op2)))
   30171              :         {
   30172            0 :           if (GET_MODE (op1) != mode)
   30173            0 :             op1 = gen_lowpart (mode, op1);
   30174            0 :           emit_move_insn (target, op1);
   30175            0 :           return target;
   30176              :         }
   30177              :       break;
   30178              : 
   30179            0 :     case 0xee:  /* b|c */
   30180            0 :       if ((!op0 || !side_effects_p (op0))
   30181            0 :           && op1 && ix86_ternlog_leaf_p (op1, mode)
   30182            0 :           && op2 && ix86_ternlog_leaf_p (op2, mode))
   30183            0 :         return ix86_expand_ternlog_binop (IOR, mode, op1, op2, target);
   30184              :       break;
   30185              : 
   30186            6 :     case 0xf0:  /* a */
   30187            6 :       if (op0
   30188            6 :           && (!op1 || !side_effects_p (op1))
   30189           12 :           && (!op2 || !side_effects_p (op2)))
   30190              :         {
   30191            6 :           if (GET_MODE (op0) != mode)
   30192            0 :             op0 = gen_lowpart (mode, op0);
   30193            6 :           emit_move_insn (target, op0);
   30194            6 :           return target;
   30195              :         }
   30196              :       break;
   30197              : 
   30198            0 :     case 0xfa:  /* a|c */
   30199            0 :       if (op0 && ix86_ternlog_leaf_p (op0, mode)
   30200            0 :           && op2 && ix86_ternlog_leaf_p (op2, mode)
   30201            0 :           && (!op1 || !side_effects_p (op1)))
   30202            0 :         return ix86_expand_ternlog_binop (IOR, mode, op0, op2, target);
   30203              :       break;
   30204              : 
   30205            0 :     case 0xfc:  /* a|b */
   30206            0 :       if (op0 && ix86_ternlog_leaf_p (op0, mode)
   30207            0 :           && op1 && ix86_ternlog_leaf_p (op1, mode)
   30208            0 :           && (!op2 || !side_effects_p (op2)))
   30209            0 :         return ix86_expand_ternlog_binop (IOR, mode, op0, op1, target);
   30210              :       break;
   30211              : 
   30212            0 :     case 0xff:
   30213            0 :       if ((!op0 || !side_effects_p (op0))
   30214            0 :           && (!op1 || !side_effects_p (op1))
   30215            0 :           && (!op2 || !side_effects_p (op2)))
   30216              :         {
   30217            0 :           emit_move_insn (target, CONSTM1_RTX (mode));
   30218            0 :           return target;
   30219              :         }
   30220              :       break;
   30221              :     }
   30222              : 
   30223         2338 :   if (!register_operand (op0, mode))
   30224              :     {
   30225              :       /* We can't use force_reg (mode, op0).  */
   30226           12 :       tmp0 = gen_reg_rtx (GET_MODE (op0));
   30227           12 :       emit_move_insn (tmp0,op0);
   30228              :     }
   30229              :   else
   30230              :     tmp0 = op0;
   30231         2338 :   if (GET_MODE (tmp0) != mode)
   30232            0 :     tmp0 = gen_lowpart (mode, tmp0);
   30233              : 
   30234         2338 :   if (!op1 || rtx_equal_p (op0, op1))
   30235            6 :     tmp1 = copy_rtx (tmp0);
   30236         2332 :   else if (!register_operand (op1, mode))
   30237              :     {
   30238              :       /* We can't use force_reg (mode, op1).  */
   30239           28 :       tmp1 = gen_reg_rtx (GET_MODE (op1));
   30240           28 :       emit_move_insn (tmp1, op1);
   30241              :     }
   30242              :   else
   30243              :     tmp1 = op1;
   30244         2338 :   if (GET_MODE (tmp1) != mode)
   30245            0 :     tmp1 = gen_lowpart (mode, tmp1);
   30246              : 
   30247         2338 :   if (!op2 || rtx_equal_p (op0, op2))
   30248           74 :     tmp2 = copy_rtx (tmp0);
   30249         2264 :   else if (rtx_equal_p (op1, op2))
   30250            0 :     tmp2 = copy_rtx (tmp1);
   30251         2264 :   else if (CONST_VECTOR_P (op2))
   30252              :     {
   30253           43 :       if (GET_MODE (op2) != mode)
   30254            0 :         op2 = gen_lowpart (mode, op2);
   30255           43 :       tmp2 = ix86_gen_bcst_mem (mode, op2);
   30256           43 :       if (!tmp2)
   30257              :         {
   30258           25 :           machine_mode bcst32_mode = mode;
   30259           25 :           machine_mode bcst64_mode = mode;
   30260           25 :           switch (mode)
   30261              :             {
   30262            1 :             case V1TImode:
   30263            1 :             case V4SImode:
   30264            1 :             case V4SFmode:
   30265            1 :             case V8HImode:
   30266            1 :             case V16QImode:
   30267            1 :               bcst32_mode = V4SImode;
   30268            1 :               bcst64_mode = V2DImode;
   30269            1 :               break;
   30270              : 
   30271            0 :             case V2TImode:
   30272            0 :             case V8SImode:
   30273            0 :             case V8SFmode:
   30274            0 :             case V16HImode:
   30275            0 :             case V32QImode:
   30276            0 :               bcst32_mode = V8SImode;
   30277            0 :               bcst64_mode = V4DImode;
   30278            0 :               break;
   30279              : 
   30280            3 :             case V4TImode:
   30281            3 :             case V16SImode:
   30282            3 :             case V16SFmode:
   30283            3 :             case V32HImode:
   30284            3 :             case V64QImode:
   30285            3 :               bcst32_mode = V16SImode;
   30286            3 :               bcst64_mode = V8DImode;
   30287            3 :               break;
   30288              : 
   30289              :             default:
   30290              :               break;
   30291              :             }
   30292              : 
   30293           25 :           if (bcst32_mode != mode)
   30294              :             {
   30295            4 :               tmp2 = gen_lowpart (bcst32_mode, op2);
   30296            4 :               if (ix86_gen_bcst_mem (bcst32_mode, tmp2))
   30297              :                 {
   30298            3 :                   tmp2 = ix86_expand_ternlog (bcst32_mode,
   30299            3 :                                               gen_lowpart (bcst32_mode, tmp0),
   30300            3 :                                               gen_lowpart (bcst32_mode, tmp1),
   30301              :                                               tmp2, idx, NULL_RTX);
   30302            3 :                   emit_move_insn (target, gen_lowpart (mode, tmp2));
   30303            3 :                   return target;
   30304              :                 }
   30305              :             }
   30306              : 
   30307           22 :           if (bcst64_mode != mode)
   30308              :             {
   30309            1 :               tmp2 = gen_lowpart (bcst64_mode, op2);
   30310            1 :               if (ix86_gen_bcst_mem (bcst64_mode, tmp2))
   30311              :                 {
   30312            0 :                   tmp2 = ix86_expand_ternlog (bcst64_mode,
   30313            0 :                                               gen_lowpart (bcst64_mode, tmp0),
   30314            0 :                                               gen_lowpart (bcst64_mode, tmp1),
   30315              :                                               tmp2, idx, NULL_RTX);
   30316            0 :                   emit_move_insn (target, gen_lowpart (mode, tmp2));
   30317            0 :                   return target;
   30318              :                 }
   30319              :             }
   30320              : 
   30321           22 :           tmp2 = force_const_mem (mode, op2);
   30322           22 :           rtx bcast = ix86_broadcast_from_constant (mode, tmp2);
   30323           22 :           tmp2 = validize_mem (tmp2);
   30324           22 :           if (bcast)
   30325              :             {
   30326           12 :               rtx reg2 = gen_reg_rtx (mode);
   30327           12 :               bool ok = ix86_expand_vector_init_duplicate (false, mode,
   30328              :                                                            reg2, bcast);
   30329           12 :               if (ok)
   30330         2335 :                 tmp2 = reg2;
   30331              :             }
   30332              :         }
   30333              :     }
   30334              :   else
   30335              :     tmp2 = op2;
   30336         2335 :   if (GET_MODE (tmp2) != mode)
   30337            0 :     tmp2 = gen_lowpart (mode, tmp2);
   30338              :   /* Some memory_operands are not vector_memory_operands.  */
   30339         2335 :   if (!bcst_vector_operand (tmp2, mode))
   30340            0 :     tmp2 = force_reg (mode, tmp2);
   30341              : 
   30342         2335 :   rtvec vec = gen_rtvec (4, tmp0, tmp1, tmp2, GEN_INT (idx));
   30343         2335 :   emit_move_insn (target, gen_rtx_UNSPEC (mode, vec, UNSPEC_VTERNLOG));
   30344         2335 :   return target;
   30345              : }
   30346              : 
   30347              : /* GF2P8AFFINEQB matrixes to implement shift and rotate.  */
   30348              : 
   30349              : static const uint64_t matrix_ashift[8] =
   30350              : {
   30351              :   0,
   30352              :   0x0001020408102040, /* 1 l */
   30353              :   0x0000010204081020, /* 2 l */
   30354              :   0x0000000102040810, /* 3 l */
   30355              :   0x0000000001020408, /* 4 l */
   30356              :   0x0000000000010204, /* 5 l */
   30357              :   0x0000000000000102, /* 6 l */
   30358              :   0x0000000000000001  /* 7 l */
   30359              : };
   30360              : 
   30361              : static const uint64_t matrix_lshiftrt[8] =
   30362              : {
   30363              :   0,
   30364              :   0x0204081020408000, /* 1 r */
   30365              :   0x0408102040800000, /* 2 r */
   30366              :   0x0810204080000000, /* 3 r */
   30367              :   0x1020408000000000, /* 4 r */
   30368              :   0x2040800000000000, /* 5 r */
   30369              :   0x4080000000000000, /* 6 r */
   30370              :   0x8000000000000000  /* 7 r */
   30371              : };
   30372              : 
   30373              : static const uint64_t matrix_ashiftrt[8] =
   30374              : {
   30375              :   0,
   30376              :   0x0204081020408080, /* 1 r */
   30377              :   0x0408102040808080, /* 2 r */
   30378              :   0x0810204080808080, /* 3 r */
   30379              :   0x1020408080808080, /* 4 r */
   30380              :   0x2040808080808080, /* 5 r */
   30381              :   0x4080808080808080, /* 6 r */
   30382              :   0x8080808080808080  /* 7 r */
   30383              : };
   30384              : 
   30385              : static const uint64_t matrix_rotate[8] =
   30386              : {
   30387              :   0,
   30388              :   0x8001020408102040, /* 1 rol8 */
   30389              :   0x4080010204081020, /* 2 rol8 */
   30390              :   0x2040800102040810, /* 3 rol8 */
   30391              :   0x1020408001020408, /* 4 rol8 */
   30392              :   0x0810204080010204, /* 5 rol8 */
   30393              :   0x0408102040800102, /* 6 rol8 */
   30394              :   0x0204081020408001  /* 7 rol8 */
   30395              : };
   30396              : 
   30397              : static const uint64_t matrix_rotatert[8] =
   30398              : {
   30399              :   0,
   30400              :   0x0204081020408001, /* 1 ror8 */
   30401              :   0x0408102040800102, /* 2 ror8 */
   30402              :   0x0810204080010204, /* 3 ror8 */
   30403              :   0x1020408001020408, /* 4 ror8 */
   30404              :   0x2040800102040810, /* 5 ror8 */
   30405              :   0x4080010204081020, /* 6 ror8 */
   30406              :   0x8001020408102040  /* 7 ror8 */
   30407              : };
   30408              : 
   30409              : /* Return rtx to load a 64bit GF2P8AFFINE GP(2) matrix implementing a shift
   30410              :    for CODE and shift count COUNT into register with vector of size of SRC.  */
   30411              : 
   30412              : rtx
   30413          217 : ix86_vgf2p8affine_shift_matrix (rtx src, rtx count, enum rtx_code code)
   30414              : {
   30415          217 :   machine_mode mode = GET_MODE (src);
   30416          217 :   const uint64_t *matrix;
   30417          217 :   unsigned shift = INTVAL (count) & 7;
   30418          217 :   gcc_assert (shift > 0 && shift < 8);
   30419              : 
   30420          217 :   switch (code)
   30421              :     {
   30422              :     case ASHIFT:
   30423              :       matrix = matrix_ashift;
   30424              :       break;
   30425           30 :     case ASHIFTRT:
   30426           30 :       matrix = matrix_ashiftrt;
   30427           30 :       break;
   30428           34 :     case LSHIFTRT:
   30429           34 :       matrix = matrix_lshiftrt;
   30430           34 :       break;
   30431           34 :     case ROTATE:
   30432           34 :       matrix = matrix_rotate;
   30433           34 :       break;
   30434           35 :     case ROTATERT:
   30435           35 :       matrix = matrix_rotatert;
   30436           35 :       break;
   30437            0 :     default:
   30438            0 :       gcc_unreachable ();
   30439              :     }
   30440              : 
   30441          217 :   int nelts = GET_MODE_NUNITS (mode);
   30442          217 :   rtvec vec = rtvec_alloc (nelts);
   30443          217 :   uint64_t ma = matrix[shift];
   30444         7417 :   for (int i = 0; i < nelts; i++)
   30445         7200 :     RTVEC_ELT (vec, i) = gen_int_mode ((ma >> ((i % 8) * 8)) & 0xff, QImode);
   30446              : 
   30447          217 :   return force_reg (mode, gen_rtx_CONST_VECTOR (mode, vec));
   30448              : }
   30449              : 
   30450              : /* Trunc a vector to a narrow vector, like v4di -> v4si.  */
   30451              : 
   30452              : void
   30453           62 : ix86_expand_trunc_with_avx2_noavx512f (rtx output, rtx input, machine_mode cvt_mode)
   30454              : {
   30455           62 :   machine_mode out_mode = GET_MODE (output);
   30456           62 :   machine_mode in_mode = GET_MODE (input);
   30457           62 :   int len = GET_MODE_SIZE (in_mode);
   30458          248 :   gcc_assert (len == GET_MODE_SIZE (cvt_mode)
   30459              :               && GET_MODE_INNER (out_mode) == GET_MODE_INNER (cvt_mode)
   30460              :               && (REG_P (input) || SUBREG_P (input)));
   30461           62 :   scalar_mode inner_out_mode = GET_MODE_INNER (out_mode);
   30462          124 :   int in_innersize = GET_MODE_SIZE (GET_MODE_INNER (in_mode));
   30463           62 :   int out_innersize = GET_MODE_SIZE (inner_out_mode);
   30464              : 
   30465           62 :   struct expand_vec_perm_d d;
   30466           62 :   d.target = gen_reg_rtx (cvt_mode);
   30467           62 :   d.op0 = lowpart_subreg (cvt_mode, force_reg(in_mode, input), in_mode);
   30468           62 :   d.op1 = d.op0;
   30469           62 :   d.vmode = cvt_mode;
   30470           62 :   d.nelt = GET_MODE_NUNITS (cvt_mode);
   30471           62 :   d.testing_p = false;
   30472           62 :   d.one_operand_p = true;
   30473              : 
   30474              :   /* Init perm. Put the needed bits of input in order and
   30475              :      fill the rest of bits by default.  */
   30476          678 :   for (int i = 0; i < d.nelt; ++i)
   30477              :     {
   30478          616 :       d.perm[i] = i;
   30479         1232 :       if (i < GET_MODE_NUNITS (out_mode))
   30480          242 :         d.perm[i] = i * (in_innersize / out_innersize);
   30481              :     }
   30482              : 
   30483           62 :   bool ok = ix86_expand_vec_perm_const_1(&d);
   30484           62 :   gcc_assert (ok);
   30485           62 :   emit_move_insn (output, gen_lowpart (out_mode, d.target));
   30486           62 : }
   30487              : 
   30488              : /* Implement truncv8sfv8bf2 with vector permutation.  */
   30489              : void
   30490            8 : ix86_expand_vector_sf2bf_with_vec_perm (rtx dest, rtx src)
   30491              : {
   30492            8 :   machine_mode vperm_mode, src_mode = GET_MODE (src);
   30493            8 :   switch (src_mode)
   30494              :     {
   30495              :     case V16SFmode:
   30496              :       vperm_mode = V32BFmode;
   30497              :       break;
   30498            2 :     case V8SFmode:
   30499            2 :       vperm_mode = V16BFmode;
   30500            2 :       break;
   30501            4 :     case V4SFmode:
   30502            4 :       vperm_mode = V8BFmode;
   30503            4 :       break;
   30504            0 :     default:
   30505            0 :       gcc_unreachable ();
   30506              :     }
   30507              : 
   30508            8 :   int nelt = GET_MODE_NUNITS (vperm_mode);
   30509            8 :   vec_perm_builder sel (nelt, nelt, 1);
   30510            8 :   sel.quick_grow (nelt);
   30511          144 :   for (int i = 0; i != nelt; i++)
   30512          128 :     sel[i] = (2 * i + 1) % nelt;
   30513           16 :   vec_perm_indices indices (sel, 1, nelt);
   30514              : 
   30515            8 :   rtx target = gen_reg_rtx (vperm_mode);
   30516            8 :   rtx op0 = lowpart_subreg (vperm_mode,
   30517              :                             force_reg (src_mode, src),
   30518              :                             src_mode);
   30519            8 :   bool ok = targetm.vectorize.vec_perm_const (vperm_mode, vperm_mode,
   30520              :                                               target, op0, op0, indices);
   30521            8 :   gcc_assert (ok);
   30522            8 :   emit_move_insn (dest, lowpart_subreg (GET_MODE (dest), target, vperm_mode));
   30523            8 : }
   30524              : 
   30525              : /* Implement extendv8bf2v8sf2 with vector permutation.  */
   30526              : void
   30527            8 : ix86_expand_vector_bf2sf_with_vec_perm (rtx dest, rtx src)
   30528              : {
   30529            8 :   machine_mode vperm_mode, src_mode = GET_MODE (src);
   30530            8 :   switch (src_mode)
   30531              :     {
   30532              :     case V16BFmode:
   30533              :       vperm_mode = V32BFmode;
   30534              :       break;
   30535            2 :     case V8BFmode:
   30536            2 :       vperm_mode = V16BFmode;
   30537            2 :       break;
   30538            4 :     case V4BFmode:
   30539            4 :       vperm_mode = V8BFmode;
   30540            4 :       break;
   30541            0 :     default:
   30542            0 :       gcc_unreachable ();
   30543              :     }
   30544              : 
   30545            8 :   int nelt = GET_MODE_NUNITS (vperm_mode);
   30546            8 :   vec_perm_builder sel (nelt, nelt, 1);
   30547            8 :   sel.quick_grow (nelt);
   30548          144 :   for (int i = 0, k = 0, j = nelt; i != nelt; i++)
   30549          128 :     sel[i] = i & 1 ? j++ : k++;
   30550              : 
   30551           16 :   vec_perm_indices indices (sel, 2, nelt);
   30552              : 
   30553            8 :   rtx target = gen_reg_rtx (vperm_mode);
   30554            8 :   rtx op1 = lowpart_subreg (vperm_mode,
   30555              :                             force_reg (src_mode, src),
   30556              :                             src_mode);
   30557            8 :   rtx op0 = CONST0_RTX (vperm_mode);
   30558            8 :   bool ok = targetm.vectorize.vec_perm_const (vperm_mode, vperm_mode,
   30559              :                                               target, op0, op1, indices);
   30560            8 :   gcc_assert (ok);
   30561            8 :   emit_move_insn (dest, lowpart_subreg (GET_MODE (dest), target, vperm_mode));
   30562            8 : }
   30563              : 
   30564              : /* Implement bitreverse<mode>2 using vbitrevb or gf2p8affineqb.  */
   30565              : 
   30566              : void
   30567           13 : ix86_expand_bitreverse (rtx dest, rtx src)
   30568              : {
   30569           13 :   machine_mode mode = GET_MODE (dest);
   30570           13 :   rtx temp;
   30571           26 :   if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
   30572              :     {
   30573            1 :       rtx temp1 = gen_reg_rtx (mode == TImode ? V2DImode : V4SImode);
   30574            1 :       rtx temp2 = gen_reg_rtx (mode == TImode ? V2DImode : V4SImode);
   30575            1 :       if (mode == TImode)
   30576              :         {
   30577            1 :           temp = lowpart_subreg (DImode, src, TImode);
   30578            1 :           emit_insn (gen_rtx_SET (temp1, gen_rtx_VEC_CONCAT (V2DImode, temp,
   30579              :                                                              const0_rtx)));
   30580            1 :           temp = gen_highpart (DImode, src);
   30581            1 :           emit_insn (gen_rtx_SET (temp2, gen_rtx_VEC_CONCAT (V2DImode, temp,
   30582              :                                                              const0_rtx)));
   30583              :         }
   30584              :       else
   30585              :         {
   30586            0 :           temp = lowpart_subreg (SImode, src, DImode);
   30587            0 :           emit_insn (gen_vec_setv4si_0 (temp1, CONST0_RTX (V4SImode), temp));
   30588            0 :           temp = gen_highpart (SImode, src);
   30589            0 :           emit_insn (gen_vec_setv4si_0 (temp2, CONST0_RTX (V4SImode), temp));
   30590            0 :           temp1 = lowpart_subreg (V2DImode, temp1, V4SImode);
   30591            0 :           temp2 = lowpart_subreg (V2DImode, temp2, V4SImode);
   30592              :         }
   30593            1 :       temp = gen_reg_rtx (V2DImode);
   30594            1 :       emit_insn (gen_vec_interleave_lowv2di (temp, temp1, temp2));
   30595              :     }
   30596           12 :   else if (mode != DImode)
   30597              :     {
   30598            9 :       if (mode != SImode)
   30599              :         {
   30600            6 :           src = force_reg (mode, src);
   30601            6 :           src = lowpart_subreg (SImode, src, mode);
   30602              :         }
   30603            9 :       temp = gen_reg_rtx (V4SImode);
   30604            9 :       emit_insn (gen_vec_setv4si_0 (temp, CONST0_RTX (V4SImode), src));
   30605              :     }
   30606              :   else
   30607              :     {
   30608            3 :       temp = gen_reg_rtx (V2DImode);
   30609            3 :       emit_insn (gen_rtx_SET (temp, gen_rtx_VEC_CONCAT (V2DImode, src,
   30610              :                                                         const0_rtx)));
   30611              :     }
   30612           13 :   src = temp;
   30613           13 :   temp = gen_reg_rtx (V16QImode);
   30614           13 :   src = lowpart_subreg (V16QImode, src, GET_MODE (src));
   30615           13 :   if (TARGET_AVX512BMM && TARGET_AVX512VL)
   30616            4 :     emit_insn (gen_avx512bmm_vbitrevb_v16qi (temp, src));
   30617            9 :   else if (TARGET_GFNI)
   30618              :     {
   30619            9 :       rtx src2 = gen_rtx_CONST_VECTOR (V16QImode,
   30620              :                                        gen_rtvec (16, GEN_INT (1), GEN_INT (2),
   30621              :                                                   GEN_INT (4), GEN_INT (8),
   30622              :                                                   GEN_INT (16), GEN_INT (32),
   30623              :                                                   GEN_INT (64), GEN_INT (-128),
   30624              :                                                   GEN_INT (1), GEN_INT (2),
   30625              :                                                   GEN_INT (4), GEN_INT (8),
   30626              :                                                   GEN_INT (16), GEN_INT (32),
   30627              :                                                   GEN_INT (64),
   30628              :                                                   GEN_INT (-128)));
   30629            9 :       src2 = validize_mem (force_const_mem (V16QImode, src2));
   30630            9 :       emit_insn (gen_vgf2p8affineqb_v16qi (temp, src, src2, const0_rtx));
   30631              :     }
   30632           13 :   if (mode == QImode)
   30633              :     {
   30634            3 :       rtx temp1 = gen_reg_rtx (SImode);
   30635            3 :       rtx temp2 = lowpart_subreg (V4SImode, temp, V16QImode);
   30636            3 :       rtx temp3 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
   30637            3 :       emit_insn (gen_rtx_SET (temp1,
   30638              :                               gen_rtx_VEC_SELECT (SImode, temp2, temp3)));
   30639            3 :       emit_move_insn (dest, lowpart_subreg (QImode, temp1, SImode));
   30640            3 :       return;
   30641              :     }
   30642           27 :   rtx target = gen_reg_rtx ((GET_MODE_SIZE (mode) < 4 || !TARGET_64BIT)
   30643            7 :                             ? SImode : mode == TImode ? DImode : mode);
   30644           10 :   emit_move_insn (target, lowpart_subreg (GET_MODE (target), temp, V16QImode));
   30645           20 :   if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
   30646              :     {
   30647            1 :       rtx temp1 = gen_reg_rtx (GET_MODE (target));
   30648            1 :       if (mode == TImode || TARGET_SSE4_1)
   30649              :         {
   30650            1 :           rtx temp2 = lowpart_subreg (mode == TImode ? V2DImode : V4SImode,
   30651              :                                       temp, V16QImode);
   30652            1 :           rtx temp3 = gen_rtx_PARALLEL (VOIDmode,
   30653              :                                         gen_rtvec (1, GEN_INT (mode == TImode
   30654              :                                                                ? 1 : 2)));
   30655            1 :           emit_insn (gen_rtx_SET (temp1,
   30656              :                                   gen_rtx_VEC_SELECT (GET_MODE (target), temp2,
   30657              :                                                       temp3)));
   30658            1 :         }
   30659              :       else
   30660              :         {
   30661            0 :           rtx temp2 = gen_reg_rtx (V4SImode);
   30662            0 :           rtx temp3 = lowpart_subreg (V4SImode, temp, V16QImode);
   30663            0 :           emit_insn (gen_sse2_pshufd (temp2, temp3, GEN_INT (0xaa)));
   30664            0 :           emit_move_insn (temp1, lowpart_subreg (GET_MODE (target), temp2,
   30665              :                                                  V4SImode));
   30666              :         }
   30667            1 :       rtx temp4 = gen_reg_rtx (GET_MODE (target));
   30668            1 :       rtx temp5 = gen_reg_rtx (GET_MODE (target));
   30669            0 :       rtx (*gen_bswap) (rtx, rtx)
   30670            1 :         = mode == TImode ? gen_bswapdi2 : gen_bswapsi2;
   30671            1 :       emit_insn (gen_bswap (temp4, target));
   30672            1 :       emit_insn (gen_bswap (temp5, temp1));
   30673            1 :       temp4 = gen_rtx_ZERO_EXTEND (mode, temp4);
   30674            1 :       temp5 = gen_rtx_ZERO_EXTEND (mode, temp5);
   30675            1 :       rtx shift = GEN_INT (GET_MODE_PRECISION (GET_MODE (target)));
   30676            1 :       temp4 = gen_rtx_ASHIFT (mode, temp4, shift);
   30677            1 :       emit_insn (gen_rtx_SET (dest, gen_rtx_IOR (mode, temp4, temp5)));
   30678            1 :       return;
   30679              :     }
   30680            9 :   if (mode == HImode)
   30681              :     {
   30682            3 :       target = lowpart_subreg (mode, target, SImode);
   30683            3 :       emit_insn (gen_bswaphi2 (dest, target));
   30684              :     }
   30685            6 :   else if (mode == SImode)
   30686            3 :     emit_insn (gen_bswapsi2 (dest, target));
   30687              :   else
   30688            3 :     emit_insn (gen_rtx_SET (dest, gen_rtx_BSWAP (mode, target)));
   30689              : }
   30690              : 
   30691              : /* Expand LCP stall or long immediate peephole for INSN.  Use the
   30692              :    previous scratch register if possible.  If USE_XOR is true,
   30693              :    generate "*movsi_xor".  */
   30694              : 
   30695              : void
   30696        31679 : ix86_expand_lcp_stall_peephole (rtx_insn *insn, rtx *operands,
   30697              :                                 bool use_xor)
   30698              : {
   30699        31679 :   rtx imm, scratch;
   30700        31679 :   machine_mode mode = GET_MODE (operands[0]);
   30701              : 
   30702              :   /* Get the immediate operand and the allocated scratch register.  */
   30703        31679 :   if (use_xor)
   30704              :     {
   30705        10207 :       imm = const0_rtx;
   30706        10207 :       scratch = operands[1];
   30707              :     }
   30708              :   else
   30709              :     {
   30710        21472 :       imm = operands[1];
   30711        21472 :       scratch = operands[2];
   30712              :     }
   30713              : 
   30714        31679 :   df_ref def;
   30715        31679 :   rtx_insn *prev, *prev_insn = nullptr;
   30716        31679 :   rtx set, prev_scratch = nullptr;
   30717              : 
   30718              :   /* Scan backward for the previous scratch register def with IMM in
   30719              :      the same basic block.  */
   30720        31679 :   basic_block bb = BLOCK_FOR_INSN (insn);
   30721       949459 :   for (prev = PREV_INSN (insn);
   30722       949459 :        prev != BB_HEAD (bb);
   30723       917780 :        prev = PREV_INSN (prev))
   30724              :     {
   30725       922678 :       if (NONDEBUG_INSN_P (prev))
   30726      2084474 :         FOR_EACH_INSN_DEF (def, prev)
   30727      1758277 :           if (!DF_REF_IS_ARTIFICIAL (def)
   30728      1758277 :               && !DF_REF_FLAGS_IS_SET (def, DF_REF_MAY_CLOBBER)
   30729       174633 :               && !DF_REF_FLAGS_IS_SET (def, DF_REF_MUST_CLOBBER))
   30730              :             {
   30731       153892 :               rtx reg = DF_REF_REG (def);
   30732       153892 :               if (HARD_REGISTER_P (reg) && REGNO (reg) != FLAGS_REG)
   30733              :                 {
   30734       152739 :                   set = single_set (prev);
   30735       152739 :                   if (!set)
   30736          484 :                     continue;
   30737              : 
   30738       152255 :                   rtx dest = SET_DEST (set);
   30739              : 
   30740              :                   /* Reject DEST if a register is not wide enough to
   30741              :                      supply MODE or invalid for QImode.  */
   30742       133058 :                   if (!GENERAL_REG_P (dest)
   30743       230436 :                       || (GET_MODE_SIZE (GET_MODE (dest))
   30744       115218 :                           < GET_MODE_SIZE (mode))
   30745       266311 :                       || (mode == QImode
   30746            0 :                           && !ANY_QI_REGNO_P (REGNO (dest))))
   30747        38199 :                     continue;
   30748              : 
   30749       114056 :                   rtx src = SET_SRC (set);
   30750       114056 :                   if (rtx_equal_p (src, imm))
   30751              :                     {
   30752              :                       /* A previous scratch register is found.  */
   30753              :                       prev_scratch = dest;
   30754              :                       prev_insn = prev;
   30755              :                       break;
   30756              :                     }
   30757              :                 }
   30758              :             }
   30759              : 
   30760       922678 :       if (prev_scratch)
   30761              :         break;
   30762              :     }
   30763              : 
   30764        31679 :   if (prev_scratch)
   30765              :     {
   30766              :       /* The previous scratch register is unusable if it is set between
   30767              :          PREV_INSN and INSN.  */
   30768         4898 :       unsigned int regno = REGNO (prev_scratch);
   30769              : 
   30770              :       /* Scan backward for the previous scratch register def.  */
   30771         4898 :       for (prev = PREV_INSN (insn);
   30772       128881 :            prev != prev_insn;
   30773       123983 :            prev = PREV_INSN (prev))
   30774              :         {
   30775       125490 :           if (NONDEBUG_INSN_P (prev))
   30776       253677 :             FOR_EACH_INSN_DEF (def, prev)
   30777       176504 :               if (HARD_REGISTER_P (DF_REF_REAL_REG (def))
   30778       176504 :                   && DF_REF_REGNO (def) == regno)
   30779              :                 {
   30780              :                   /* Since the previous scratch register is set, it is
   30781              :                      unusable.  */
   30782         1507 :                   if (dump_file)
   30783              :                     {
   30784            0 :                       fprintf (dump_file,
   30785              :                                "\nThe previous scratch register:\n\n");
   30786            0 :                       print_rtl_single (dump_file, prev_scratch);
   30787            0 :                       fprintf (dump_file, "\nset in:\n\n");
   30788            0 :                       print_rtl_single (dump_file, prev_insn);
   30789            0 :                       fprintf (dump_file,
   30790              :                                "\nis unusable by:\n\n");
   30791            0 :                       print_rtl_single (dump_file, insn);
   30792            0 :                       fprintf (dump_file,
   30793              :                                "\nsince it is overridden by:\n\n");
   30794            0 :                       print_rtl_single (dump_file, prev);
   30795            0 :                       fprintf (dump_file, "\n");
   30796              :                     }
   30797              :                   prev_scratch = nullptr;
   30798              :                   break;
   30799              :                 }
   30800              : 
   30801       123983 :           if (!prev_scratch)
   30802              :             break;
   30803              :         }
   30804              : 
   30805         4898 :       if (prev_scratch)
   30806              :         {
   30807              :           /* Ignore the allocated scratch register and use the previous
   30808              :              scratch register.  */
   30809         3391 :           if (dump_file)
   30810              :             {
   30811            0 :               fprintf (dump_file,
   30812              :                        "\nIgnore the allocated scratch register:\n\n");
   30813            0 :               print_rtl_single (dump_file, scratch);
   30814            0 :               fprintf (dump_file,
   30815              :                        "\nand use the previous scratch register:\n\n");
   30816            0 :               print_rtl_single (dump_file, prev_scratch);
   30817            0 :               fprintf (dump_file, "\nset in:\n\n");
   30818            0 :               print_rtl_single (dump_file, prev_insn);
   30819            0 :               fprintf (dump_file, "\nfor:\n\n");
   30820            0 :               print_rtl_single (dump_file, insn);
   30821            0 :               fprintf (dump_file, "\n");
   30822              :             }
   30823         3391 :           scratch = gen_lowpart (mode, prev_scratch);
   30824         3391 :           set = gen_rtx_SET (operands[0], scratch);
   30825         3391 :           emit_insn (set);
   30826         3391 :           return;
   30827              :         }
   30828              :     }
   30829              : 
   30830              :   /* If there is no usable previous scratch register, use the allocated
   30831              :      scratch register.  */
   30832        28288 :   if (use_xor)
   30833              :     {
   30834              :       /* Generate "*movsi_xor".  */
   30835         8686 :       rtx x = gen_lowpart (SImode, scratch);
   30836         8686 :       rtvec p = rtvec_alloc (2);
   30837         8686 :       set = gen_rtx_SET (x, const0_rtx);
   30838         8686 :       RTVEC_ELT (p, 0) = set;
   30839         8686 :       rtx clobber = gen_rtx_REG (CCmode, FLAGS_REG);
   30840         8686 :       RTVEC_ELT (p, 1) = gen_rtx_CLOBBER (VOIDmode, clobber);
   30841         8686 :       set = gen_rtx_PARALLEL (VOIDmode, p);
   30842              :     }
   30843              :   else
   30844        19602 :     set = gen_rtx_SET (scratch, imm);
   30845        28288 :   emit_insn (set);
   30846        28288 :   set = gen_rtx_SET (operands[0], scratch);
   30847        28288 :   emit_insn (set);
   30848              : }
   30849              : 
   30850              : /* Generate assembly to calculate CRC using pclmulqdq instruction.
   30851              :    OPERANDS[1] is input CRC,
   30852              :    OPERANDS[2] is data (message),
   30853              :    OPERANDS[3] is the polynomial without the leading 1.  */
   30854              : 
   30855              : void
   30856           14 : ix86_expand_crc_using_pclmul (scalar_mode crc_mode,
   30857              :                               scalar_mode data_mode,
   30858              :                               rtx *operands)
   30859              : {
   30860              :   /* Check and keep arguments.  */
   30861           14 :   gcc_assert (!CONST_INT_P (operands[0]));
   30862           14 :   gcc_assert (CONST_INT_P (operands[3]));
   30863           14 :   rtx crc = operands[1];
   30864           14 :   rtx data = operands[2];
   30865           14 :   rtx polynomial = operands[3];
   30866              : 
   30867           14 :   unsigned HOST_WIDE_INT crc_size = GET_MODE_BITSIZE (crc_mode);
   30868           14 :   unsigned HOST_WIDE_INT data_size = GET_MODE_BITSIZE (data_mode);
   30869           14 :   gcc_assert (crc_size <= 64);
   30870           14 :   gcc_assert (data_size <= crc_size);
   30871              : 
   30872           14 :   machine_mode op_mode = crc_size == 64 ? DImode : SImode;
   30873              : 
   30874              :   /* Calculate the quotient.  */
   30875           14 :   unsigned HOST_WIDE_INT
   30876           14 :       q = gf2n_poly_long_div_quotient (UINTVAL (polynomial), crc_size);
   30877              : 
   30878           14 :   if (crc_size > data_size)
   30879            4 :     crc = expand_shift (RSHIFT_EXPR, op_mode, crc, crc_size - data_size,
   30880              :                         NULL_RTX, 1);
   30881              : 
   30882              :   /* Keep the quotient and polynomial in V2DImode.  */
   30883           14 :   rtvec q_poly_constants = gen_rtvec (2, gen_int_mode (q, DImode),
   30884           14 :                                       gen_int_mode (UINTVAL (polynomial),
   30885              :                                                     DImode));
   30886           14 :   rtx constants_v2di = gen_rtx_CONST_VECTOR (V2DImode, q_poly_constants);
   30887           14 :   rtx constants_reg = gen_reg_rtx (V2DImode);
   30888           14 :   emit_move_insn (constants_reg, constants_v2di);
   30889              : 
   30890              :   /* crc ^ data and keep in V2DImode.  */
   30891           14 :   rtx cd_xor = expand_binop (op_mode, xor_optab, crc, data,
   30892              :                              NULL_RTX, 1, OPTAB_WIDEN);
   30893           14 :   rtx crc_temp;
   30894           14 :   if (op_mode == DImode)
   30895              :     {
   30896            6 :       crc_temp = gen_reg_rtx (V2DImode);
   30897            6 :       emit_insn (gen_vec_concatv2di (crc_temp, cd_xor, cd_xor));
   30898              :     }
   30899              :   else
   30900              :     {
   30901            8 :       crc_temp = gen_reg_rtx (V4SImode);
   30902            8 :       emit_insn (gen_vec_setv4si_0 (crc_temp, CONST0_RTX (V4SImode), cd_xor));
   30903            8 :       crc_temp = gen_lowpart (V2DImode, crc_temp);
   30904              :     }
   30905              : 
   30906              :   /* Perform carry-less multiplication with quotient.  */
   30907           14 :   rtx res = gen_reg_rtx (V2DImode);
   30908           14 :   emit_insn (gen_pclmulqdq (res, crc_temp, constants_reg, const0_rtx));
   30909              : 
   30910           14 :   if (op_mode == DImode)
   30911              :     {
   30912              :       /* Perform res ^ crc_temp to handle 65th bit of quotient.  */
   30913            6 :       res = expand_binop (V2DImode, xor_optab, res, crc_temp,
   30914              :                           NULL_RTX, 1, OPTAB_DIRECT);
   30915              :     }
   30916              :   else
   30917            8 :     res = expand_shift (RSHIFT_EXPR, V2DImode, res, crc_size, NULL_RTX, 1);
   30918              : 
   30919              :   /* Perform carry-less multiplication with polynomial and get low part.  */
   30920           22 :   emit_insn (gen_pclmulqdq (res, res, constants_reg,
   30921              :                             GEN_INT (op_mode == DImode ? 0x11 : 0x10)));
   30922              : 
   30923           14 :   rtx crc_part = gen_reg_rtx (op_mode);
   30924           14 :   if (op_mode == DImode)
   30925            6 :     emit_insn (gen_vec_extractv2didi (crc_part, res, const0_rtx));
   30926              :   else
   30927              :     {
   30928            8 :       res = gen_lowpart (V4SImode, res);
   30929            8 :       emit_insn (gen_vec_extractv4sisi (crc_part, res, const0_rtx));
   30930              :     }
   30931              : 
   30932           14 :   if (crc_size > data_size)
   30933              :     {
   30934            4 :       rtx shift = expand_shift (LSHIFT_EXPR, op_mode, operands[1], data_size,
   30935              :                                 NULL_RTX, 1);
   30936            4 :       crc_part = expand_binop (op_mode, xor_optab, crc_part, shift,
   30937              :                                NULL_RTX, 1, OPTAB_DIRECT);
   30938              :     }
   30939           14 :   emit_move_insn (operands[0], gen_lowpart (crc_mode, crc_part));
   30940           14 : }
   30941              : 
   30942              : /* Generate assembly to calculate reversed CRC using pclmulqdq instruction.
   30943              :    OPERANDS[1] is input CRC,
   30944              :    OPERANDS[2] is data (message),
   30945              :    OPERANDS[3] is the polynomial without the leading 1.  */
   30946              : 
   30947              : void
   30948           18 : ix86_expand_reversed_crc_using_pclmul (scalar_mode crc_mode,
   30949              :                                        scalar_mode data_mode,
   30950              :                                        rtx *operands)
   30951              : {
   30952              :   /* Check and keep arguments.  */
   30953           18 :   gcc_assert (!CONST_INT_P (operands[0]));
   30954           18 :   gcc_assert (CONST_INT_P (operands[3]));
   30955           18 :   rtx crc = operands[1];
   30956           18 :   rtx data = operands[2];
   30957           18 :   rtx polynomial = operands[3];
   30958              : 
   30959           18 :   unsigned HOST_WIDE_INT crc_size = GET_MODE_BITSIZE (crc_mode);
   30960           18 :   unsigned HOST_WIDE_INT data_size = GET_MODE_BITSIZE (data_mode);
   30961           18 :   gcc_assert (crc_size <= 64);
   30962           18 :   gcc_assert (data_size <= crc_size);
   30963              : 
   30964           18 :   machine_mode op_mode = crc_size == 64 ? DImode : SImode;
   30965              : 
   30966              :   /* Calculate the quotient.  */
   30967           18 :   unsigned HOST_WIDE_INT
   30968           18 :       q = gf2n_poly_long_div_quotient (UINTVAL (polynomial), crc_size);
   30969              :   /* Reflect the calculated quotient.  */
   30970           18 :   q = (reflect_hwi (q, crc_size) << 1) + 1;
   30971              : 
   30972           18 :   unsigned HOST_WIDE_INT reflected_op3 = reflect_hwi (UINTVAL (polynomial),
   30973              :                                                       crc_size);
   30974              :   /* Keep the quotient and reflected polynomial in V2DImode.  */
   30975           18 :   rtvec q_poly_constants = gen_rtvec (2, gen_int_mode (q, DImode),
   30976           18 :                                       gen_int_mode (reflected_op3 << 1,
   30977              :                                                     DImode));
   30978           18 :   rtx constants_v2di = gen_rtx_CONST_VECTOR (V2DImode, q_poly_constants);
   30979           18 :   rtx constants_reg = gen_reg_rtx (V2DImode);
   30980           18 :   emit_move_insn (constants_reg, constants_v2di);
   30981              : 
   30982              :   /* crc ^ data and keep in V2DImode.  */
   30983           18 :   rtx cd_xor = expand_binop (op_mode, xor_optab, crc, data,
   30984              :                              NULL_RTX, 1, OPTAB_WIDEN);
   30985           18 :   rtx crc_temp;
   30986           18 :   if (op_mode == DImode)
   30987              :     {
   30988           14 :       crc_temp = gen_reg_rtx (V2DImode);
   30989           14 :       emit_insn (gen_vec_concatv2di (crc_temp, cd_xor, const0_rtx));
   30990              :     }
   30991              :   else
   30992              :     {
   30993            4 :       crc_temp = gen_reg_rtx (V4SImode);
   30994            4 :       emit_insn (gen_vec_setv4si_0 (crc_temp, CONST0_RTX (V4SImode), cd_xor));
   30995            4 :       crc_temp = gen_lowpart (V2DImode, crc_temp);
   30996              :     }
   30997              : 
   30998              :   /* Perform carry-less multiplication with quotient.  */
   30999           18 :   emit_insn (gen_pclmulqdq (crc_temp, crc_temp, constants_reg, const0_rtx));
   31000           18 :   if (data_size < 64)
   31001           12 :     crc_temp = expand_shift (LSHIFT_EXPR, V2DImode, crc_temp, 64 - data_size,
   31002              :                              NULL_RTX, 0);
   31003              : 
   31004           18 :   rtx res = gen_reg_rtx (V2DImode);
   31005              :   /* Perform carry-less multiplication with polynomial.  */
   31006           18 :   emit_insn (gen_pclmulqdq (res, crc_temp, constants_reg, GEN_INT (0x10)));
   31007           18 :   if (op_mode == DImode && (UINTVAL (polynomial) & 1))
   31008              :     {
   31009              :       /* Perform xor of res[127:64] with crc_temp[63:0].
   31010              :          This is needed when the 65th bit of reflected polynomial is 1.  */
   31011            8 :       rtx tmp = gen_reg_rtx (V2DImode);
   31012            8 :       emit_insn (gen_vec_interleave_lowv2di (tmp, crc_temp, crc_temp));
   31013            8 :       res = expand_binop (V2DImode, xor_optab, res, tmp,
   31014              :                           NULL_RTX, 1, OPTAB_DIRECT);
   31015              :     }
   31016              : 
   31017           18 :   rtx res_high = gen_reg_rtx (op_mode);
   31018           18 :   if (op_mode == DImode)
   31019           14 :     emit_insn (gen_vec_extractv2didi (res_high, res, const1_rtx));
   31020              :   else
   31021              :     {
   31022            4 :       res = gen_lowpart (V4SImode, res);
   31023            4 :       emit_insn (gen_vec_extractv4sisi (res_high, res, const2_rtx));
   31024              :     }
   31025              : 
   31026           18 :   if (crc_size > data_size)
   31027              :     {
   31028            8 :       rtx shift = expand_shift (RSHIFT_EXPR, op_mode, crc, data_size,
   31029              :                                 NULL_RTX, 1);
   31030            8 :       res_high = expand_binop (op_mode, xor_optab, res_high, shift,
   31031              :                                NULL_RTX, 1, OPTAB_DIRECT);
   31032              :     }
   31033           18 :   emit_move_insn (operands[0], gen_lowpart (crc_mode, res_high));
   31034           18 : }
   31035              : 
   31036              : #include "gt-i386-expand.h"
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.