Line data Source code
1 : /* Copyright (C) 1988-2026 Free Software Foundation, Inc.
2 :
3 : This file is part of GCC.
4 :
5 : GCC is free software; you can redistribute it and/or modify
6 : it under the terms of the GNU General Public License as published by
7 : the Free Software Foundation; either version 3, or (at your option)
8 : any later version.
9 :
10 : GCC is distributed in the hope that it will be useful,
11 : but WITHOUT ANY WARRANTY; without even the implied warranty of
12 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 : GNU General Public License for more details.
14 :
15 : You should have received a copy of the GNU General Public License
16 : along with GCC; see the file COPYING3. If not see
17 : <http://www.gnu.org/licenses/>. */
18 :
19 : #define IN_TARGET_CODE 1
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "memmodel.h"
28 : #include "gimple.h"
29 : #include "cfghooks.h"
30 : #include "cfgloop.h"
31 : #include "df.h"
32 : #include "tm_p.h"
33 : #include "stringpool.h"
34 : #include "expmed.h"
35 : #include "optabs.h"
36 : #include "regs.h"
37 : #include "emit-rtl.h"
38 : #include "recog.h"
39 : #include "cgraph.h"
40 : #include "diagnostic.h"
41 : #include "cfgbuild.h"
42 : #include "alias.h"
43 : #include "fold-const.h"
44 : #include "attribs.h"
45 : #include "calls.h"
46 : #include "stor-layout.h"
47 : #include "varasm.h"
48 : #include "output.h"
49 : #include "insn-attr.h"
50 : #include "flags.h"
51 : #include "except.h"
52 : #include "explow.h"
53 : #include "expr.h"
54 : #include "cfgrtl.h"
55 : #include "common/common-target.h"
56 : #include "langhooks.h"
57 : #include "reload.h"
58 : #include "gimplify.h"
59 : #include "dwarf2.h"
60 : #include "tm-constrs.h"
61 : #include "cselib.h"
62 : #include "sched-int.h"
63 : #include "opts.h"
64 : #include "tree-pass.h"
65 : #include "context.h"
66 : #include "pass_manager.h"
67 : #include "target-globals.h"
68 : #include "gimple-iterator.h"
69 : #include "shrink-wrap.h"
70 : #include "builtins.h"
71 : #include "rtl-iter.h"
72 : #include "tree-iterator.h"
73 : #include "dbgcnt.h"
74 : #include "case-cfn-macros.h"
75 : #include "dojump.h"
76 : #include "fold-const-call.h"
77 : #include "tree-vrp.h"
78 : #include "tree-ssanames.h"
79 : #include "selftest.h"
80 : #include "selftest-rtl.h"
81 : #include "print-rtl.h"
82 : #include "intl.h"
83 : #include "ifcvt.h"
84 : #include "symbol-summary.h"
85 : #include "sreal.h"
86 : #include "ipa-cp.h"
87 : #include "ipa-prop.h"
88 : #include "ipa-fnsummary.h"
89 : #include "wide-int-bitmask.h"
90 : #include "tree-vector-builder.h"
91 : #include "debug.h"
92 : #include "dwarf2out.h"
93 : #include "i386-options.h"
94 : #include "i386-builtins.h"
95 : #include "i386-expand.h"
96 : #include "asan.h"
97 : #include "function-abi.h"
98 :
99 : /* Split one or more double-mode RTL references into pairs of half-mode
100 : references. The RTL can be REG, offsettable MEM, integer constant, or
101 : CONST_DOUBLE. "operands" is a pointer to an array of double-mode RTLs to
102 : split and "num" is its length. lo_half and hi_half are output arrays
103 : that parallel "operands". */
104 :
105 : void
106 4261879 : split_double_mode (machine_mode mode, rtx operands[],
107 : int num, rtx lo_half[], rtx hi_half[])
108 : {
109 4261879 : machine_mode half_mode;
110 4261879 : unsigned int byte;
111 4261879 : rtx mem_op = NULL_RTX;
112 4261879 : int mem_num = 0;
113 :
114 4261879 : switch (mode)
115 : {
116 : case E_TImode:
117 : half_mode = DImode;
118 : break;
119 660764 : case E_DImode:
120 660764 : half_mode = SImode;
121 660764 : break;
122 6 : case E_P2HImode:
123 6 : half_mode = HImode;
124 6 : break;
125 30 : case E_P2QImode:
126 30 : half_mode = QImode;
127 30 : break;
128 0 : default:
129 0 : gcc_unreachable ();
130 : }
131 :
132 4261879 : byte = GET_MODE_SIZE (half_mode);
133 :
134 8764216 : while (num--)
135 : {
136 4502337 : rtx op = operands[num];
137 :
138 : /* simplify_subreg refuse to split volatile memory addresses,
139 : but we still have to handle it. */
140 4502337 : if (MEM_P (op))
141 : {
142 1767360 : if (mem_op && rtx_equal_p (op, mem_op))
143 : {
144 2412 : lo_half[num] = lo_half[mem_num];
145 2412 : hi_half[num] = hi_half[mem_num];
146 : }
147 : else
148 : {
149 1764948 : mem_op = op;
150 1764948 : mem_num = num;
151 1764948 : lo_half[num] = adjust_address (op, half_mode, 0);
152 1764948 : hi_half[num] = adjust_address (op, half_mode, byte);
153 : }
154 : }
155 : else
156 : {
157 2734977 : lo_half[num] = simplify_gen_subreg (half_mode, op,
158 2734977 : GET_MODE (op) == VOIDmode
159 : ? mode : GET_MODE (op), 0);
160 :
161 2734977 : rtx tmp = simplify_gen_subreg (half_mode, op,
162 2734977 : GET_MODE (op) == VOIDmode
163 2734977 : ? mode : GET_MODE (op), byte);
164 : /* simplify_gen_subreg will return NULL RTX for the
165 : high half of the paradoxical subreg. */
166 2734977 : hi_half[num] = tmp ? tmp : gen_reg_rtx (half_mode);
167 : }
168 : }
169 4261879 : }
170 :
171 : /* Emit the double word assignment DST = { LO, HI }. */
172 :
173 : void
174 103895 : split_double_concat (machine_mode mode, rtx dst, rtx lo, rtx hi)
175 : {
176 103895 : rtx dlo, dhi;
177 103895 : int deleted_move_count = 0;
178 103895 : split_double_mode (mode, &dst, 1, &dlo, &dhi);
179 : /* Constraints ensure that if both lo and hi are MEMs, then
180 : dst has early-clobber and thus addresses of MEMs don't use
181 : dlo/dhi registers. Otherwise if at least one of li and hi are MEMs,
182 : dlo/dhi are registers. */
183 103895 : if (MEM_P (lo)
184 5595 : && rtx_equal_p (dlo, hi)
185 104862 : && reg_overlap_mentioned_p (dhi, lo))
186 : {
187 : /* If dlo is same as hi and lo's address uses dhi register,
188 : code below would first emit_move_insn (dhi, hi)
189 : and then emit_move_insn (dlo, lo). But the former
190 : would invalidate lo's address. Load into dhi first,
191 : then swap. */
192 193 : emit_move_insn (dhi, lo);
193 193 : lo = dhi;
194 : }
195 103702 : else if (MEM_P (hi)
196 9504 : && !MEM_P (lo)
197 6658 : && !rtx_equal_p (dlo, lo)
198 105087 : && reg_overlap_mentioned_p (dlo, hi))
199 : {
200 : /* In this case, code below would first emit_move_insn (dlo, lo)
201 : and then emit_move_insn (dhi, hi). But the former would
202 : invalidate hi's address. */
203 11 : if (rtx_equal_p (dhi, lo))
204 : {
205 : /* We can't load into dhi first, so load into dlo
206 : first and we'll swap. */
207 5 : emit_move_insn (dlo, hi);
208 5 : hi = dlo;
209 : }
210 : else
211 : {
212 : /* Load into dhi first. */
213 6 : emit_move_insn (dhi, hi);
214 6 : hi = dhi;
215 : }
216 : }
217 103895 : if (!rtx_equal_p (dlo, hi))
218 : {
219 89841 : if (!rtx_equal_p (dlo, lo))
220 39853 : emit_move_insn (dlo, lo);
221 : else
222 : deleted_move_count++;
223 89841 : if (!rtx_equal_p (dhi, hi))
224 83740 : emit_move_insn (dhi, hi);
225 : else
226 6101 : deleted_move_count++;
227 : }
228 14054 : else if (!rtx_equal_p (lo, dhi))
229 : {
230 7112 : if (!rtx_equal_p (dhi, hi))
231 7112 : emit_move_insn (dhi, hi);
232 : else
233 : deleted_move_count++;
234 7112 : if (!rtx_equal_p (dlo, lo))
235 7010 : emit_move_insn (dlo, lo);
236 : else
237 102 : deleted_move_count++;
238 : }
239 6942 : else if (mode == TImode)
240 6921 : emit_insn (gen_swapdi (dlo, dhi));
241 : else
242 21 : emit_insn (gen_swapsi (dlo, dhi));
243 :
244 103895 : if (deleted_move_count == 2)
245 3125 : emit_note (NOTE_INSN_DELETED);
246 103895 : }
247 :
248 :
249 : /* Generate either "mov $0, reg" or "xor reg, reg", as appropriate
250 : for the target. */
251 :
252 : void
253 123380 : ix86_expand_clear (rtx dest)
254 : {
255 123380 : rtx tmp;
256 :
257 : /* We play register width games, which are only valid after reload. */
258 123380 : gcc_assert (reload_completed);
259 :
260 : /* Avoid HImode and its attendant prefix byte. */
261 246760 : if (GET_MODE_SIZE (GET_MODE (dest)) < 4)
262 1040 : dest = gen_rtx_REG (SImode, REGNO (dest));
263 123380 : tmp = gen_rtx_SET (dest, const0_rtx);
264 :
265 123380 : if (!TARGET_USE_MOV0 || optimize_insn_for_size_p ())
266 : {
267 123380 : rtx clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
268 123380 : tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, clob));
269 : }
270 :
271 123380 : emit_insn (tmp);
272 123380 : }
273 :
274 : /* Return true if V can be broadcasted from an integer of WIDTH bits
275 : which is returned in VAL_BROADCAST. Otherwise, return false. */
276 :
277 : static bool
278 4851 : ix86_broadcast (HOST_WIDE_INT v, unsigned int width,
279 : HOST_WIDE_INT &val_broadcast)
280 : {
281 4851 : wide_int val = wi::uhwi (v, HOST_BITS_PER_WIDE_INT);
282 4851 : val_broadcast = wi::extract_uhwi (val, 0, width);
283 6543 : for (unsigned int i = width; i < HOST_BITS_PER_WIDE_INT; i += width)
284 : {
285 5089 : HOST_WIDE_INT each = wi::extract_uhwi (val, i, width);
286 5089 : if (val_broadcast != each)
287 : return false;
288 : }
289 1454 : val_broadcast = sext_hwi (val_broadcast, width);
290 1454 : return true;
291 4851 : }
292 :
293 : /* Convert the CONST_WIDE_INT operand OP to broadcast in MODE. */
294 :
295 : rtx
296 36366 : ix86_convert_const_wide_int_to_broadcast (machine_mode mode, rtx op)
297 : {
298 : /* Don't use integer vector broadcast if we can't move from GPR to SSE
299 : register directly. */
300 36366 : if (!TARGET_INTER_UNIT_MOVES_TO_VEC)
301 : return nullptr;
302 :
303 36366 : unsigned int msize = GET_MODE_SIZE (mode);
304 :
305 : /* Only optimized for vpbroadcast[bwsd]/vbroadcastss with xmm/ymm/zmm. */
306 36366 : if (msize != 16 && msize != 32 && msize != 64)
307 : return nullptr;
308 :
309 : /* Convert CONST_WIDE_INT to a non-standard SSE constant integer
310 : broadcast only if vector broadcast is available. */
311 36366 : if (!TARGET_AVX
312 1666 : || !CONST_WIDE_INT_P (op)
313 1603 : || standard_sse_constant_p (op, mode)
314 37969 : || (CONST_WIDE_INT_NUNITS (op) * HOST_BITS_PER_WIDE_INT
315 1603 : != GET_MODE_BITSIZE (mode)))
316 : return nullptr;
317 :
318 1595 : HOST_WIDE_INT val = CONST_WIDE_INT_ELT (op, 0);
319 1595 : HOST_WIDE_INT val_broadcast;
320 1595 : scalar_int_mode broadcast_mode;
321 : /* vpbroadcastb zmm requires TARGET_AVX512BW. */
322 712 : if ((msize == 64 ? TARGET_AVX512BW : TARGET_AVX2)
323 2089 : && ix86_broadcast (val, GET_MODE_BITSIZE (QImode),
324 : val_broadcast))
325 : broadcast_mode = QImode;
326 654 : else if ((msize == 64 ? TARGET_AVX512BW : TARGET_AVX2)
327 1968 : && ix86_broadcast (val, GET_MODE_BITSIZE (HImode),
328 : val_broadcast))
329 : broadcast_mode = HImode;
330 : /* vbroadcasts[sd] only support memory operand w/o AVX2.
331 : When msize == 16, pshufs is used for vec_duplicate.
332 : when msize == 64, vpbroadcastd is used, and TARGET_AVX512F must be existed. */
333 412 : else if ((msize != 32 || TARGET_AVX2)
334 1768 : && ix86_broadcast (val, GET_MODE_BITSIZE (SImode),
335 : val_broadcast))
336 : broadcast_mode = SImode;
337 1391 : else if (TARGET_64BIT && (msize != 32 || TARGET_AVX2)
338 2641 : && ix86_broadcast (val, GET_MODE_BITSIZE (DImode),
339 : val_broadcast))
340 : broadcast_mode = DImode;
341 : else
342 : return nullptr;
343 :
344 : /* Check if OP can be broadcasted from VAL. */
345 1776 : for (int i = 1; i < CONST_WIDE_INT_NUNITS (op); i++)
346 1561 : if (val != CONST_WIDE_INT_ELT (op, i))
347 : return nullptr;
348 :
349 215 : unsigned int nunits = (GET_MODE_SIZE (mode)
350 215 : / GET_MODE_SIZE (broadcast_mode));
351 215 : machine_mode vector_mode;
352 215 : if (!mode_for_vector (broadcast_mode, nunits).exists (&vector_mode))
353 0 : gcc_unreachable ();
354 215 : rtx target = gen_reg_rtx (vector_mode);
355 215 : bool ok = ix86_expand_vector_init_duplicate (false, vector_mode,
356 : target,
357 : GEN_INT (val_broadcast));
358 215 : if (!ok)
359 : return nullptr;
360 215 : target = lowpart_subreg (mode, target, vector_mode);
361 215 : return target;
362 : }
363 :
364 : void
365 75200438 : ix86_expand_move (machine_mode mode, rtx operands[])
366 : {
367 75200438 : rtx op0, op1;
368 75200438 : rtx tmp, addend = NULL_RTX;
369 75200438 : enum tls_model model;
370 :
371 75200438 : op0 = operands[0];
372 75200438 : op1 = operands[1];
373 :
374 : /* Avoid complex sets of likely spilled hard registers before reload. */
375 75200438 : if (!ix86_hardreg_mov_ok (op0, op1))
376 : {
377 141292 : tmp = gen_reg_rtx (mode);
378 141292 : operands[0] = tmp;
379 141292 : ix86_expand_move (mode, operands);
380 141292 : operands[0] = op0;
381 141292 : operands[1] = tmp;
382 141292 : op1 = tmp;
383 : }
384 :
385 75200438 : switch (GET_CODE (op1))
386 : {
387 351778 : case CONST:
388 351778 : tmp = XEXP (op1, 0);
389 :
390 351778 : if (GET_CODE (tmp) != PLUS
391 340032 : || !SYMBOL_REF_P (XEXP (tmp, 0)))
392 : break;
393 :
394 337383 : op1 = XEXP (tmp, 0);
395 337383 : addend = XEXP (tmp, 1);
396 : /* FALLTHRU */
397 :
398 5080391 : case SYMBOL_REF:
399 5080391 : model = SYMBOL_REF_TLS_MODEL (op1);
400 :
401 5080391 : if (model)
402 10167 : op1 = legitimize_tls_address (op1, model, true);
403 5070224 : else if (ix86_force_load_from_GOT_p (op1))
404 : {
405 : /* Load the external function address via GOT slot to avoid PLT. */
406 24 : op1 = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, op1),
407 : (TARGET_64BIT
408 : ? UNSPEC_GOTPCREL
409 : : UNSPEC_GOT));
410 24 : op1 = gen_rtx_CONST (Pmode, op1);
411 24 : op1 = gen_const_mem (Pmode, op1);
412 20 : set_mem_alias_set (op1, GOT_ALIAS_SET);
413 : }
414 : else
415 : {
416 : #if TARGET_PECOFF
417 : tmp = legitimize_pe_coff_symbol (op1, addend != NULL_RTX);
418 :
419 : if (tmp)
420 : {
421 : op1 = tmp;
422 : if (!addend)
423 : break;
424 : }
425 : else
426 : #endif
427 5070204 : {
428 5070204 : op1 = operands[1];
429 5070204 : break;
430 : }
431 : }
432 :
433 10187 : if (addend)
434 : {
435 2787 : op1 = force_operand (op1, NULL_RTX);
436 2796 : op1 = expand_simple_binop (Pmode, PLUS, op1, addend,
437 : op0, 1, OPTAB_DIRECT);
438 : }
439 : else
440 7400 : op1 = force_operand (op1, op0);
441 :
442 10187 : if (op1 == op0)
443 : return;
444 :
445 1152 : op1 = convert_to_mode (mode, op1, 1);
446 :
447 : default:
448 : break;
449 :
450 1605743 : case SUBREG:
451 : /* Transform TImode paradoxical SUBREG into zero_extendditi2. */
452 1605743 : if (TARGET_64BIT
453 1360534 : && mode == TImode
454 : && SUBREG_P (op1)
455 75113 : && GET_MODE (SUBREG_REG (op1)) == DImode
456 1652272 : && SUBREG_BYTE (op1) == 0)
457 46529 : op1 = gen_rtx_ZERO_EXTEND (TImode, SUBREG_REG (op1));
458 : /* As not all values in XFmode are representable in real_value,
459 : we might be called with unfoldable SUBREGs of constants. */
460 1605743 : if (mode == XFmode
461 3448 : && CONSTANT_P (SUBREG_REG (op1))
462 0 : && can_create_pseudo_p ())
463 : {
464 0 : machine_mode imode = GET_MODE (SUBREG_REG (op1));
465 0 : rtx r = force_const_mem (imode, SUBREG_REG (op1));
466 0 : if (r)
467 0 : r = validize_mem (r);
468 : else
469 0 : r = force_reg (imode, SUBREG_REG (op1));
470 0 : op1 = simplify_gen_subreg (mode, r, imode, SUBREG_BYTE (op1));
471 : }
472 : break;
473 : }
474 :
475 75191403 : if ((flag_pic || MACHOPIC_INDIRECT)
476 75191403 : && symbolic_operand (op1, mode))
477 : {
478 : #if TARGET_MACHO
479 : if (TARGET_MACHO && !TARGET_64BIT)
480 : {
481 : /* dynamic-no-pic */
482 : if (MACHOPIC_INDIRECT)
483 : {
484 : tmp = (op0 && REG_P (op0) && mode == Pmode)
485 : ? op0 : gen_reg_rtx (Pmode);
486 : op1 = machopic_indirect_data_reference (op1, tmp);
487 : if (MACHOPIC_PURE)
488 : op1 = machopic_legitimize_pic_address (op1, mode,
489 : tmp == op1 ? 0 : tmp);
490 : }
491 : if (op0 != op1 && !MEM_P (op0))
492 : {
493 : rtx insn = gen_rtx_SET (op0, op1);
494 : emit_insn (insn);
495 : return;
496 : }
497 : }
498 : #endif
499 :
500 724907 : if (MEM_P (op0))
501 101718 : op1 = force_reg (mode, op1);
502 623189 : else if (!(TARGET_64BIT && x86_64_movabs_operand (op1, DImode)))
503 : {
504 623132 : rtx reg = can_create_pseudo_p () ? NULL_RTX : op0;
505 623132 : op1 = legitimize_pic_address (op1, reg);
506 623132 : if (op0 == op1)
507 : return;
508 623132 : op1 = convert_to_mode (mode, op1, 1);
509 : }
510 : }
511 : else
512 : {
513 74466496 : if (MEM_P (op0)
514 101530318 : && (PUSH_ROUNDING (GET_MODE_SIZE (mode)) != GET_MODE_SIZE (mode)
515 10919593 : || !push_operand (op0, mode))
516 86988451 : && MEM_P (op1))
517 2153440 : op1 = force_reg (mode, op1);
518 :
519 74466496 : if (push_operand (op0, mode)
520 74466496 : && ! general_no_elim_operand (op1, mode))
521 1004 : op1 = copy_to_mode_reg (mode, op1);
522 :
523 : /* Force large constants in 64bit compilation into register
524 : to get them CSEed. */
525 74466496 : if (can_create_pseudo_p ()
526 68660385 : && (mode == DImode) && TARGET_64BIT
527 35954875 : && immediate_operand (op1, mode)
528 7777364 : && !x86_64_zext_immediate_operand (op1, VOIDmode)
529 736352 : && !register_operand (op0, mode)
530 74645781 : && optimize)
531 127165 : op1 = copy_to_mode_reg (mode, op1);
532 :
533 74466496 : if (can_create_pseudo_p ())
534 : {
535 68660385 : if (CONST_DOUBLE_P (op1))
536 : {
537 : /* If we are loading a floating point constant to a
538 : register, force the value to memory now, since we'll
539 : get better code out the back end. */
540 :
541 906654 : op1 = validize_mem (force_const_mem (mode, op1));
542 906654 : if (!register_operand (op0, mode))
543 : {
544 130778 : tmp = gen_reg_rtx (mode);
545 130778 : emit_insn (gen_rtx_SET (tmp, op1));
546 130778 : emit_move_insn (op0, tmp);
547 130778 : return;
548 : }
549 : }
550 : }
551 : }
552 :
553 : /* Special case inserting 64-bit values into a TImode register. */
554 75060625 : if (TARGET_64BIT
555 : /* Disable for -O0 (see PR110587) unless naked (PR110533). */
556 65182296 : && (optimize || ix86_function_naked (current_function_decl))
557 44717364 : && (mode == DImode || mode == DFmode)
558 30657153 : && SUBREG_P (op0)
559 506637 : && GET_MODE (SUBREG_REG (op0)) == TImode
560 412334 : && REG_P (SUBREG_REG (op0))
561 75472959 : && REG_P (op1))
562 : {
563 : /* Use *insvti_lowpart_1 to set lowpart. */
564 183954 : if (SUBREG_BYTE (op0) == 0)
565 : {
566 55815 : wide_int mask = wi::mask (64, true, 128);
567 55815 : tmp = immed_wide_int_const (mask, TImode);
568 55815 : op0 = SUBREG_REG (op0);
569 55815 : tmp = gen_rtx_AND (TImode, copy_rtx (op0), tmp);
570 55815 : if (mode == DFmode)
571 375 : op1 = gen_lowpart (DImode, op1);
572 55815 : op1 = gen_rtx_ZERO_EXTEND (TImode, op1);
573 55815 : op1 = gen_rtx_IOR (TImode, tmp, op1);
574 55815 : }
575 : /* Use *insvti_highpart_1 to set highpart. */
576 128139 : else if (SUBREG_BYTE (op0) == 8)
577 : {
578 128139 : wide_int mask = wi::mask (64, false, 128);
579 128139 : tmp = immed_wide_int_const (mask, TImode);
580 128139 : op0 = SUBREG_REG (op0);
581 128139 : tmp = gen_rtx_AND (TImode, copy_rtx (op0), tmp);
582 128139 : if (mode == DFmode)
583 226 : op1 = gen_lowpart (DImode, op1);
584 128139 : op1 = gen_rtx_ZERO_EXTEND (TImode, op1);
585 128139 : op1 = gen_rtx_ASHIFT (TImode, op1, GEN_INT (64));
586 128139 : op1 = gen_rtx_IOR (TImode, tmp, op1);
587 128139 : }
588 : }
589 :
590 75060625 : emit_insn (gen_rtx_SET (op0, op1));
591 : }
592 :
593 : /* OP is a memref of CONST_VECTOR, return scalar constant mem
594 : if CONST_VECTOR is a vec_duplicate, else return NULL. */
595 : rtx
596 2568256 : ix86_broadcast_from_constant (machine_mode mode, rtx op)
597 : {
598 2568256 : int nunits = GET_MODE_NUNITS (mode);
599 2568256 : if (nunits < 2)
600 : return nullptr;
601 :
602 : /* Don't use integer vector broadcast if we can't move from GPR to SSE
603 : register directly. */
604 2436566 : if (!TARGET_INTER_UNIT_MOVES_TO_VEC
605 8015 : && INTEGRAL_MODE_P (mode))
606 : return nullptr;
607 :
608 : /* Convert CONST_VECTOR to a non-standard SSE constant integer
609 : broadcast only if vector broadcast is available. */
610 2431156 : if (standard_sse_constant_p (op, mode))
611 : return nullptr;
612 :
613 4862312 : if (GET_MODE_INNER (mode) == TImode)
614 : return nullptr;
615 :
616 2431046 : rtx constant = get_pool_constant (XEXP (op, 0));
617 2431046 : if (!CONST_VECTOR_P (constant))
618 : return nullptr;
619 :
620 : /* There could be some rtx like
621 : (mem/u/c:V16QI (symbol_ref/u:DI ("*.LC1")))
622 : but with "*.LC1" refer to V2DI constant vector. */
623 2431046 : if (GET_MODE (constant) != mode)
624 : {
625 776 : constant = simplify_subreg (mode, constant, GET_MODE (constant),
626 : 0);
627 776 : if (constant == nullptr || !CONST_VECTOR_P (constant))
628 : return nullptr;
629 : }
630 :
631 2431046 : rtx first = XVECEXP (constant, 0, 0);
632 :
633 8476105 : for (int i = 1; i < nunits; ++i)
634 : {
635 7811309 : rtx tmp = XVECEXP (constant, 0, i);
636 : /* Vector duplicate value. */
637 7811309 : if (!rtx_equal_p (tmp, first))
638 : return nullptr;
639 : }
640 :
641 : return first;
642 : }
643 :
644 : void
645 4780083 : ix86_expand_vector_move (machine_mode mode, rtx operands[])
646 : {
647 4780083 : rtx op0 = operands[0], op1 = operands[1];
648 : /* Use GET_MODE_BITSIZE instead of GET_MODE_ALIGNMENT for IA MCU
649 : psABI since the biggest alignment is 4 byte for IA MCU psABI. */
650 4780083 : unsigned int align = (TARGET_IAMCU
651 4780083 : ? GET_MODE_BITSIZE (mode)
652 4780083 : : GET_MODE_ALIGNMENT (mode));
653 :
654 4780083 : if (push_operand (op0, VOIDmode))
655 2919 : op0 = emit_move_resolve_push (mode, op0);
656 :
657 : /* Force constants other than zero into memory. We do not know how
658 : the instructions used to build constants modify the upper 64 bits
659 : of the register, once we have that information we may be able
660 : to handle some of them more efficiently. */
661 4780083 : if (can_create_pseudo_p ()
662 4584887 : && (CONSTANT_P (op1)
663 4250804 : || (SUBREG_P (op1)
664 335726 : && CONSTANT_P (SUBREG_REG (op1))))
665 5114180 : && ((register_operand (op0, mode)
666 275549 : && !standard_sse_constant_p (op1, mode))
667 : /* ix86_expand_vector_move_misalign() does not like constants. */
668 : || (SSE_REG_MODE_P (mode)
669 276209 : && MEM_P (op0)
670 42933 : && MEM_ALIGN (op0) < align)))
671 : {
672 4939 : if (SUBREG_P (op1))
673 : {
674 14 : machine_mode imode = GET_MODE (SUBREG_REG (op1));
675 14 : rtx r = force_const_mem (imode, SUBREG_REG (op1));
676 14 : if (r)
677 14 : r = validize_mem (r);
678 : else
679 0 : r = force_reg (imode, SUBREG_REG (op1));
680 14 : op1 = simplify_gen_subreg (mode, r, imode, SUBREG_BYTE (op1));
681 : }
682 : else
683 : {
684 4925 : machine_mode mode = GET_MODE (op0);
685 4925 : rtx tmp = ix86_convert_const_wide_int_to_broadcast
686 4925 : (mode, op1);
687 4925 : if (tmp == nullptr)
688 4904 : op1 = validize_mem (force_const_mem (mode, op1));
689 : else
690 : op1 = tmp;
691 : }
692 : }
693 :
694 4780083 : if (can_create_pseudo_p ()
695 4584887 : && GET_MODE_SIZE (mode) >= 16
696 3856128 : && VECTOR_MODE_P (mode)
697 8421477 : && (MEM_P (op1)
698 751304 : && SYMBOL_REF_P (XEXP (op1, 0))
699 509091 : && CONSTANT_POOL_ADDRESS_P (XEXP (op1, 0))))
700 : {
701 492225 : rtx first = ix86_broadcast_from_constant (mode, op1);
702 492225 : if (first != nullptr)
703 : {
704 : /* Broadcast to XMM/YMM/ZMM register from an integer
705 : constant or scalar mem. */
706 129326 : rtx tmp = gen_reg_rtx (mode);
707 129326 : if (FLOAT_MODE_P (mode))
708 29916 : first = force_const_mem (GET_MODE_INNER (mode), first);
709 129326 : bool ok = ix86_expand_vector_init_duplicate (false, mode,
710 : tmp, first);
711 129326 : if (!ok && !TARGET_64BIT && GET_MODE_INNER (mode) == DImode)
712 : {
713 0 : first = force_const_mem (GET_MODE_INNER (mode), first);
714 0 : ok = ix86_expand_vector_init_duplicate (false, mode,
715 : tmp, first);
716 : }
717 129326 : if (ok)
718 : {
719 129326 : emit_move_insn (op0, tmp);
720 129326 : return;
721 : }
722 : }
723 : }
724 :
725 : /* We need to check memory alignment for SSE mode since attribute
726 : can make operands unaligned. */
727 4650757 : if (can_create_pseudo_p ()
728 : && SSE_REG_MODE_P (mode)
729 9371624 : && ((MEM_P (op0) && (MEM_ALIGN (op0) < align))
730 4190522 : || (MEM_P (op1) && (MEM_ALIGN (op1) < align))))
731 : {
732 327429 : rtx tmp[2];
733 :
734 : /* ix86_expand_vector_move_misalign() does not like both
735 : arguments in memory. */
736 327429 : if (!register_operand (op0, mode)
737 327429 : && !register_operand (op1, mode))
738 : {
739 96167 : rtx scratch = gen_reg_rtx (mode);
740 96167 : emit_move_insn (scratch, op1);
741 96167 : op1 = scratch;
742 : }
743 :
744 327429 : tmp[0] = op0; tmp[1] = op1;
745 327429 : ix86_expand_vector_move_misalign (mode, tmp);
746 327429 : return;
747 : }
748 :
749 : /* Special case TImode to 128-bit vector conversions via V2DI. */
750 1163512 : if (VECTOR_MODE_P (mode)
751 4272814 : && GET_MODE_SIZE (mode) == 16
752 3048278 : && SUBREG_P (op1)
753 265408 : && GET_MODE (SUBREG_REG (op1)) == TImode
754 3330 : && TARGET_64BIT && TARGET_SSE
755 4326027 : && ix86_pre_reload_split ())
756 : {
757 2591 : rtx tmp = gen_reg_rtx (V2DImode);
758 2591 : rtx lo = gen_reg_rtx (DImode);
759 2591 : rtx hi = gen_reg_rtx (DImode);
760 2591 : emit_move_insn (lo, gen_lowpart (DImode, SUBREG_REG (op1)));
761 2591 : emit_move_insn (hi, gen_highpart (DImode, SUBREG_REG (op1)));
762 2591 : emit_insn (gen_vec_concatv2di (tmp, lo, hi));
763 2591 : emit_move_insn (op0, gen_lowpart (mode, tmp));
764 2591 : return;
765 : }
766 :
767 : /* If operand0 is a hard register, make operand1 a pseudo. */
768 4320737 : if (can_create_pseudo_p ()
769 8446278 : && !ix86_hardreg_mov_ok (op0, op1))
770 : {
771 141 : rtx tmp = gen_reg_rtx (GET_MODE (op0));
772 141 : emit_move_insn (tmp, op1);
773 141 : emit_move_insn (op0, tmp);
774 141 : return;
775 : }
776 :
777 : /* Make operand1 a register if it isn't already. */
778 4320596 : if (can_create_pseudo_p ()
779 4125400 : && !register_operand (op0, mode)
780 5469848 : && !register_operand (op1, mode))
781 : {
782 221443 : rtx tmp = gen_reg_rtx (GET_MODE (op0));
783 221443 : emit_move_insn (tmp, op1);
784 221443 : emit_move_insn (op0, tmp);
785 221443 : return;
786 : }
787 :
788 4099153 : emit_insn (gen_rtx_SET (op0, op1));
789 : }
790 :
791 : /* Split 32-byte AVX unaligned load and store if needed. */
792 :
793 : static void
794 13115 : ix86_avx256_split_vector_move_misalign (rtx op0, rtx op1)
795 : {
796 13115 : rtx m;
797 13115 : rtx (*extract) (rtx, rtx, rtx);
798 13115 : machine_mode mode;
799 :
800 13115 : if ((MEM_P (op1) && !TARGET_AVX256_SPLIT_UNALIGNED_LOAD)
801 5066 : || (MEM_P (op0) && !TARGET_AVX256_SPLIT_UNALIGNED_STORE))
802 : {
803 13089 : emit_insn (gen_rtx_SET (op0, op1));
804 13089 : return;
805 : }
806 :
807 26 : rtx orig_op0 = NULL_RTX;
808 26 : mode = GET_MODE (op0);
809 26 : switch (GET_MODE_CLASS (mode))
810 : {
811 9 : case MODE_VECTOR_INT:
812 9 : case MODE_INT:
813 9 : if (mode != V32QImode)
814 : {
815 7 : if (!MEM_P (op0))
816 : {
817 3 : orig_op0 = op0;
818 3 : op0 = gen_reg_rtx (V32QImode);
819 : }
820 : else
821 4 : op0 = gen_lowpart (V32QImode, op0);
822 7 : op1 = gen_lowpart (V32QImode, op1);
823 7 : mode = V32QImode;
824 : }
825 : break;
826 : case MODE_VECTOR_FLOAT:
827 : break;
828 0 : default:
829 0 : gcc_unreachable ();
830 : }
831 :
832 26 : switch (mode)
833 : {
834 0 : default:
835 0 : gcc_unreachable ();
836 : case E_V32QImode:
837 : extract = gen_avx_vextractf128v32qi;
838 : mode = V16QImode;
839 : break;
840 1 : case E_V16BFmode:
841 1 : extract = gen_avx_vextractf128v16bf;
842 1 : mode = V8BFmode;
843 1 : break;
844 0 : case E_V16HFmode:
845 0 : extract = gen_avx_vextractf128v16hf;
846 0 : mode = V8HFmode;
847 0 : break;
848 8 : case E_V8SFmode:
849 8 : extract = gen_avx_vextractf128v8sf;
850 8 : mode = V4SFmode;
851 8 : break;
852 8 : case E_V4DFmode:
853 8 : extract = gen_avx_vextractf128v4df;
854 8 : mode = V2DFmode;
855 8 : break;
856 : }
857 :
858 26 : if (MEM_P (op1))
859 : {
860 9 : rtx r = gen_reg_rtx (mode);
861 9 : m = adjust_address (op1, mode, 0);
862 9 : emit_move_insn (r, m);
863 9 : m = adjust_address (op1, mode, 16);
864 9 : r = gen_rtx_VEC_CONCAT (GET_MODE (op0), r, m);
865 9 : emit_move_insn (op0, r);
866 : }
867 17 : else if (MEM_P (op0))
868 : {
869 17 : m = adjust_address (op0, mode, 0);
870 17 : emit_insn (extract (m, op1, const0_rtx));
871 17 : m = adjust_address (op0, mode, 16);
872 17 : emit_insn (extract (m, copy_rtx (op1), const1_rtx));
873 : }
874 : else
875 0 : gcc_unreachable ();
876 :
877 26 : if (orig_op0)
878 3 : emit_move_insn (orig_op0, gen_lowpart (GET_MODE (orig_op0), op0));
879 : }
880 :
881 : /* Implement the movmisalign patterns for SSE. Non-SSE modes go
882 : straight to ix86_expand_vector_move. */
883 : /* Code generation for scalar reg-reg moves of single and double precision data:
884 : if (x86_sse_partial_reg_dependency == true | x86_sse_split_regs == true)
885 : movaps reg, reg
886 : else
887 : movss reg, reg
888 : if (x86_sse_partial_reg_dependency == true)
889 : movapd reg, reg
890 : else
891 : movsd reg, reg
892 :
893 : Code generation for scalar loads of double precision data:
894 : if (x86_sse_split_regs == true)
895 : movlpd mem, reg (gas syntax)
896 : else
897 : movsd mem, reg
898 :
899 : Code generation for unaligned packed loads of single precision data
900 : (x86_sse_unaligned_move_optimal overrides x86_sse_partial_reg_dependency):
901 : if (x86_sse_unaligned_move_optimal)
902 : movups mem, reg
903 :
904 : if (x86_sse_partial_reg_dependency == true)
905 : {
906 : xorps reg, reg
907 : movlps mem, reg
908 : movhps mem+8, reg
909 : }
910 : else
911 : {
912 : movlps mem, reg
913 : movhps mem+8, reg
914 : }
915 :
916 : Code generation for unaligned packed loads of double precision data
917 : (x86_sse_unaligned_move_optimal overrides x86_sse_split_regs):
918 : if (x86_sse_unaligned_move_optimal)
919 : movupd mem, reg
920 :
921 : if (x86_sse_split_regs == true)
922 : {
923 : movlpd mem, reg
924 : movhpd mem+8, reg
925 : }
926 : else
927 : {
928 : movsd mem, reg
929 : movhpd mem+8, reg
930 : }
931 : */
932 :
933 : void
934 661443 : ix86_expand_vector_move_misalign (machine_mode mode, rtx operands[])
935 : {
936 661443 : rtx op0, op1, m;
937 :
938 661443 : op0 = operands[0];
939 661443 : op1 = operands[1];
940 :
941 : /* Use unaligned load/store for AVX512 or when optimizing for size. */
942 1322886 : if (GET_MODE_SIZE (mode) == 64 || optimize_insn_for_size_p ())
943 : {
944 25371 : emit_insn (gen_rtx_SET (op0, op1));
945 25371 : return;
946 : }
947 :
948 636072 : if (TARGET_AVX)
949 : {
950 63052 : if (GET_MODE_SIZE (mode) == 32)
951 13115 : ix86_avx256_split_vector_move_misalign (op0, op1);
952 : else
953 : /* Always use 128-bit mov<mode>_internal pattern for AVX. */
954 18411 : emit_insn (gen_rtx_SET (op0, op1));
955 : return;
956 : }
957 :
958 604546 : if (TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
959 95 : || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL)
960 : {
961 604451 : emit_insn (gen_rtx_SET (op0, op1));
962 604451 : return;
963 : }
964 :
965 : /* ??? If we have typed data, then it would appear that using
966 : movdqu is the only way to get unaligned data loaded with
967 : integer type. */
968 95 : if (TARGET_SSE2 && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
969 : {
970 81 : emit_insn (gen_rtx_SET (op0, op1));
971 81 : return;
972 : }
973 :
974 14 : if (MEM_P (op1))
975 : {
976 6 : if (TARGET_SSE2 && mode == V2DFmode)
977 : {
978 2 : rtx zero;
979 :
980 : /* When SSE registers are split into halves, we can avoid
981 : writing to the top half twice. */
982 2 : if (TARGET_SSE_SPLIT_REGS)
983 : {
984 2 : emit_clobber (op0);
985 2 : zero = op0;
986 : }
987 : else
988 : {
989 : /* ??? Not sure about the best option for the Intel chips.
990 : The following would seem to satisfy; the register is
991 : entirely cleared, breaking the dependency chain. We
992 : then store to the upper half, with a dependency depth
993 : of one. A rumor has it that Intel recommends two movsd
994 : followed by an unpacklpd, but this is unconfirmed. And
995 : given that the dependency depth of the unpacklpd would
996 : still be one, I'm not sure why this would be better. */
997 0 : zero = CONST0_RTX (V2DFmode);
998 : }
999 :
1000 2 : m = adjust_address (op1, DFmode, 0);
1001 2 : emit_insn (gen_sse2_loadlpd (op0, zero, m));
1002 2 : m = adjust_address (op1, DFmode, 8);
1003 2 : emit_insn (gen_sse2_loadhpd (op0, op0, m));
1004 2 : }
1005 : else
1006 : {
1007 4 : rtx t;
1008 :
1009 4 : if (mode != V4SFmode)
1010 0 : t = gen_reg_rtx (V4SFmode);
1011 : else
1012 : t = op0;
1013 :
1014 4 : if (TARGET_SSE_PARTIAL_REG_DEPENDENCY)
1015 2 : emit_move_insn (t, CONST0_RTX (V4SFmode));
1016 : else
1017 2 : emit_clobber (t);
1018 :
1019 4 : m = adjust_address (op1, V2SFmode, 0);
1020 4 : emit_insn (gen_sse_loadlps (t, t, m));
1021 4 : m = adjust_address (op1, V2SFmode, 8);
1022 4 : emit_insn (gen_sse_loadhps (t, t, m));
1023 4 : if (mode != V4SFmode)
1024 0 : emit_move_insn (op0, gen_lowpart (mode, t));
1025 : }
1026 : }
1027 8 : else if (MEM_P (op0))
1028 : {
1029 8 : if (TARGET_SSE2 && mode == V2DFmode)
1030 : {
1031 2 : m = adjust_address (op0, DFmode, 0);
1032 2 : emit_insn (gen_sse2_storelpd (m, op1));
1033 2 : m = adjust_address (op0, DFmode, 8);
1034 2 : emit_insn (gen_sse2_storehpd (m, op1));
1035 : }
1036 : else
1037 : {
1038 6 : if (mode != V4SFmode)
1039 0 : op1 = gen_lowpart (V4SFmode, op1);
1040 :
1041 6 : m = adjust_address (op0, V2SFmode, 0);
1042 6 : emit_insn (gen_sse_storelps (m, op1));
1043 6 : m = adjust_address (op0, V2SFmode, 8);
1044 6 : emit_insn (gen_sse_storehps (m, copy_rtx (op1)));
1045 : }
1046 : }
1047 : else
1048 0 : gcc_unreachable ();
1049 : }
1050 :
1051 : /* Move bits 64:95 to bits 32:63. */
1052 :
1053 : void
1054 849 : ix86_move_vector_high_sse_to_mmx (rtx op)
1055 : {
1056 849 : rtx mask = gen_rtx_PARALLEL (VOIDmode,
1057 : gen_rtvec (4, GEN_INT (0), GEN_INT (2),
1058 : GEN_INT (0), GEN_INT (0)));
1059 849 : rtx dest = lowpart_subreg (V4SImode, op, GET_MODE (op));
1060 849 : op = gen_rtx_VEC_SELECT (V4SImode, dest, mask);
1061 849 : rtx insn = gen_rtx_SET (dest, op);
1062 849 : emit_insn (insn);
1063 849 : }
1064 :
1065 : /* Split MMX pack with signed/unsigned saturation with SSE/SSE2. */
1066 :
1067 : void
1068 759 : ix86_split_mmx_pack (rtx operands[], enum rtx_code code)
1069 : {
1070 759 : rtx op0 = operands[0];
1071 759 : rtx op1 = operands[1];
1072 759 : rtx op2 = operands[2];
1073 759 : rtx src;
1074 :
1075 759 : machine_mode dmode = GET_MODE (op0);
1076 759 : machine_mode smode = GET_MODE (op1);
1077 759 : machine_mode inner_dmode = GET_MODE_INNER (dmode);
1078 759 : machine_mode inner_smode = GET_MODE_INNER (smode);
1079 :
1080 : /* Get the corresponding SSE mode for destination. */
1081 759 : int nunits = 16 / GET_MODE_SIZE (inner_dmode);
1082 1518 : machine_mode sse_dmode = mode_for_vector (GET_MODE_INNER (dmode),
1083 1518 : nunits).require ();
1084 759 : machine_mode sse_half_dmode = mode_for_vector (GET_MODE_INNER (dmode),
1085 1518 : nunits / 2).require ();
1086 :
1087 : /* Get the corresponding SSE mode for source. */
1088 759 : nunits = 16 / GET_MODE_SIZE (inner_smode);
1089 1518 : machine_mode sse_smode = mode_for_vector (GET_MODE_INNER (smode),
1090 1518 : nunits).require ();
1091 :
1092 : /* Generate SSE pack with signed/unsigned saturation. */
1093 759 : rtx dest = lowpart_subreg (sse_dmode, op0, GET_MODE (op0));
1094 759 : op1 = lowpart_subreg (sse_smode, op1, GET_MODE (op1));
1095 759 : op2 = lowpart_subreg (sse_smode, op2, GET_MODE (op2));
1096 :
1097 : /* paskusdw/packuswb does unsigned saturation of a signed source
1098 : which is different from generic us_truncate RTX. */
1099 759 : if (code == US_TRUNCATE)
1100 657 : src = gen_rtx_UNSPEC (sse_dmode,
1101 : gen_rtvec (2, op1, op2),
1102 : UNSPEC_US_TRUNCATE);
1103 : else
1104 : {
1105 102 : op1 = gen_rtx_fmt_e (code, sse_half_dmode, op1);
1106 102 : op2 = gen_rtx_fmt_e (code, sse_half_dmode, op2);
1107 102 : src = gen_rtx_VEC_CONCAT (sse_dmode, op1, op2);
1108 : }
1109 :
1110 759 : emit_move_insn (dest, src);
1111 :
1112 759 : ix86_move_vector_high_sse_to_mmx (op0);
1113 759 : }
1114 :
1115 : /* Split MMX punpcklXX/punpckhXX with SSE punpcklXX. This is also used
1116 : for a full unpack of OPERANDS[1] and OPERANDS[2] into a wider
1117 : OPERANDS[0]. */
1118 :
1119 : void
1120 6525 : ix86_split_mmx_punpck (rtx operands[], bool high_p)
1121 : {
1122 6525 : rtx op0 = operands[0];
1123 6525 : rtx op1 = operands[1];
1124 6525 : rtx op2 = operands[2];
1125 6525 : machine_mode mode = GET_MODE (op1);
1126 6525 : rtx mask;
1127 : /* The corresponding SSE mode. */
1128 6525 : machine_mode sse_mode, double_sse_mode;
1129 :
1130 6525 : switch (mode)
1131 : {
1132 1776 : case E_V8QImode:
1133 1776 : case E_V4QImode:
1134 1776 : case E_V2QImode:
1135 1776 : sse_mode = V16QImode;
1136 1776 : double_sse_mode = V32QImode;
1137 1776 : mask = gen_rtx_PARALLEL (VOIDmode,
1138 : gen_rtvec (16,
1139 : GEN_INT (0), GEN_INT (16),
1140 : GEN_INT (1), GEN_INT (17),
1141 : GEN_INT (2), GEN_INT (18),
1142 : GEN_INT (3), GEN_INT (19),
1143 : GEN_INT (4), GEN_INT (20),
1144 : GEN_INT (5), GEN_INT (21),
1145 : GEN_INT (6), GEN_INT (22),
1146 : GEN_INT (7), GEN_INT (23)));
1147 1776 : break;
1148 :
1149 3434 : case E_V4HImode:
1150 3434 : case E_V2HImode:
1151 3434 : sse_mode = V8HImode;
1152 3434 : double_sse_mode = V16HImode;
1153 3434 : mask = gen_rtx_PARALLEL (VOIDmode,
1154 : gen_rtvec (8,
1155 : GEN_INT (0), GEN_INT (8),
1156 : GEN_INT (1), GEN_INT (9),
1157 : GEN_INT (2), GEN_INT (10),
1158 : GEN_INT (3), GEN_INT (11)));
1159 3434 : break;
1160 :
1161 919 : case E_V2SImode:
1162 919 : sse_mode = V4SImode;
1163 919 : double_sse_mode = V8SImode;
1164 919 : mask = gen_rtx_PARALLEL (VOIDmode,
1165 : gen_rtvec (4,
1166 : GEN_INT (0), GEN_INT (4),
1167 : GEN_INT (1), GEN_INT (5)));
1168 919 : break;
1169 :
1170 396 : case E_V2SFmode:
1171 396 : sse_mode = V4SFmode;
1172 396 : double_sse_mode = V8SFmode;
1173 396 : mask = gen_rtx_PARALLEL (VOIDmode,
1174 : gen_rtvec (4,
1175 : GEN_INT (0), GEN_INT (4),
1176 : GEN_INT (1), GEN_INT (5)));
1177 396 : break;
1178 :
1179 0 : default:
1180 0 : gcc_unreachable ();
1181 : }
1182 :
1183 : /* Generate SSE punpcklXX. */
1184 6525 : rtx dest = lowpart_subreg (sse_mode, op0, GET_MODE (op0));
1185 6525 : op1 = lowpart_subreg (sse_mode, op1, GET_MODE (op1));
1186 6525 : op2 = lowpart_subreg (sse_mode, op2, GET_MODE (op2));
1187 :
1188 6525 : op1 = gen_rtx_VEC_CONCAT (double_sse_mode, op1, op2);
1189 6525 : op2 = gen_rtx_VEC_SELECT (sse_mode, op1, mask);
1190 6525 : rtx insn = gen_rtx_SET (dest, op2);
1191 6525 : emit_insn (insn);
1192 :
1193 : /* Move high bits to low bits. */
1194 6525 : if (high_p)
1195 : {
1196 2421 : if (sse_mode == V4SFmode)
1197 : {
1198 110 : mask = gen_rtx_PARALLEL (VOIDmode,
1199 : gen_rtvec (4, GEN_INT (2), GEN_INT (3),
1200 : GEN_INT (4), GEN_INT (5)));
1201 110 : op2 = gen_rtx_VEC_CONCAT (V8SFmode, dest, dest);
1202 110 : op1 = gen_rtx_VEC_SELECT (V4SFmode, op2, mask);
1203 : }
1204 : else
1205 : {
1206 2311 : int sz = GET_MODE_SIZE (mode);
1207 :
1208 2311 : if (sz == 4)
1209 239 : mask = gen_rtx_PARALLEL (VOIDmode,
1210 : gen_rtvec (4, GEN_INT (1), GEN_INT (0),
1211 : GEN_INT (0), GEN_INT (1)));
1212 2072 : else if (sz == 8)
1213 2072 : mask = gen_rtx_PARALLEL (VOIDmode,
1214 : gen_rtvec (4, GEN_INT (2), GEN_INT (3),
1215 : GEN_INT (0), GEN_INT (1)));
1216 : else
1217 0 : gcc_unreachable ();
1218 :
1219 2311 : dest = lowpart_subreg (V4SImode, dest, GET_MODE (dest));
1220 2311 : op1 = gen_rtx_VEC_SELECT (V4SImode, dest, mask);
1221 : }
1222 :
1223 2421 : insn = gen_rtx_SET (dest, op1);
1224 2421 : emit_insn (insn);
1225 : }
1226 6525 : }
1227 :
1228 : /* Helper function of ix86_fixup_binary_operands to canonicalize
1229 : operand order. Returns true if the operands should be swapped. */
1230 :
1231 : static bool
1232 179079719 : ix86_swap_binary_operands_p (enum rtx_code code, machine_mode mode,
1233 : rtx operands[])
1234 : {
1235 179079719 : rtx dst = operands[0];
1236 179079719 : rtx src1 = operands[1];
1237 179079719 : rtx src2 = operands[2];
1238 :
1239 : /* If the operation is not commutative, we can't do anything. */
1240 179079719 : if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
1241 28445777 : && GET_RTX_CLASS (code) != RTX_COMM_COMPARE)
1242 : return false;
1243 :
1244 : /* Highest priority is that src1 should match dst. */
1245 150646603 : if (rtx_equal_p (dst, src1))
1246 : return false;
1247 110040103 : if (rtx_equal_p (dst, src2))
1248 : return true;
1249 :
1250 : /* Next highest priority is that immediate constants come second. */
1251 109949257 : if (immediate_operand (src2, mode))
1252 : return false;
1253 26698835 : if (immediate_operand (src1, mode))
1254 : return true;
1255 :
1256 : /* Lowest priority is that memory references should come second. */
1257 26698835 : if (MEM_P (src2))
1258 : return false;
1259 25225441 : if (MEM_P (src1))
1260 567520 : return true;
1261 :
1262 : return false;
1263 : }
1264 :
1265 : /* Fix up OPERANDS to satisfy ix86_binary_operator_ok. Return the
1266 : destination to use for the operation. If different from the true
1267 : destination in operands[0], a copy operation will be required except
1268 : under TARGET_APX_NDD. */
1269 :
1270 : rtx
1271 13732937 : ix86_fixup_binary_operands (enum rtx_code code, machine_mode mode,
1272 : rtx operands[], bool use_ndd)
1273 : {
1274 13732937 : rtx dst = operands[0];
1275 13732937 : rtx src1 = operands[1];
1276 13732937 : rtx src2 = operands[2];
1277 :
1278 : /* Canonicalize operand order. */
1279 13732937 : if (ix86_swap_binary_operands_p (code, mode, operands))
1280 : {
1281 : /* It is invalid to swap operands of different modes. */
1282 89635 : gcc_assert (GET_MODE (src1) == GET_MODE (src2));
1283 :
1284 : std::swap (src1, src2);
1285 : }
1286 :
1287 : /* Both source operands cannot be in memory. */
1288 13732937 : if (MEM_P (src1) && MEM_P (src2))
1289 : {
1290 : /* Optimization: Only read from memory once. */
1291 113779 : if (rtx_equal_p (src1, src2))
1292 : {
1293 19 : src2 = force_reg (mode, src2);
1294 19 : src1 = src2;
1295 : }
1296 113760 : else if (rtx_equal_p (dst, src1))
1297 3470 : src2 = force_reg (mode, src2);
1298 : else
1299 110290 : src1 = force_reg (mode, src1);
1300 : }
1301 :
1302 : /* If the destination is memory, and we do not have matching source
1303 : operands, do things in registers. */
1304 13732937 : if (MEM_P (dst) && !rtx_equal_p (dst, src1))
1305 496240 : dst = gen_reg_rtx (mode);
1306 :
1307 : /* Source 1 cannot be a constant. */
1308 13732937 : if (CONSTANT_P (src1))
1309 711 : src1 = force_reg (mode, src1);
1310 :
1311 : /* Source 1 cannot be a non-matching memory. */
1312 13732937 : if (!use_ndd && MEM_P (src1) && !rtx_equal_p (dst, src1))
1313 461404 : src1 = force_reg (mode, src1);
1314 :
1315 : /* Improve address combine. */
1316 13732937 : if (code == PLUS
1317 10081576 : && GET_MODE_CLASS (mode) == MODE_INT
1318 9967870 : && MEM_P (src2))
1319 180119 : src2 = force_reg (mode, src2);
1320 :
1321 13732937 : operands[1] = src1;
1322 13732937 : operands[2] = src2;
1323 13732937 : return dst;
1324 : }
1325 :
1326 : /* Similarly, but assume that the destination has already been
1327 : set up properly. */
1328 :
1329 : void
1330 301007 : ix86_fixup_binary_operands_no_copy (enum rtx_code code,
1331 : machine_mode mode, rtx operands[],
1332 : bool use_ndd)
1333 : {
1334 301007 : rtx dst = ix86_fixup_binary_operands (code, mode, operands, use_ndd);
1335 301007 : gcc_assert (dst == operands[0]);
1336 301007 : }
1337 :
1338 : /* Attempt to expand a binary operator. Make the expansion closer to the
1339 : actual machine, then just general_operand, which will allow 3 separate
1340 : memory references (one output, two input) in a single insn. */
1341 :
1342 : void
1343 13431801 : ix86_expand_binary_operator (enum rtx_code code, machine_mode mode,
1344 : rtx operands[], bool use_ndd)
1345 : {
1346 13431801 : rtx src1, src2, dst, op, clob;
1347 :
1348 13431801 : dst = ix86_fixup_binary_operands (code, mode, operands, use_ndd);
1349 13431801 : src1 = operands[1];
1350 13431801 : src2 = operands[2];
1351 :
1352 : /* Emit the instruction. */
1353 :
1354 13431801 : op = gen_rtx_SET (dst, gen_rtx_fmt_ee (code, mode, src1, src2));
1355 :
1356 13431801 : if (reload_completed
1357 89263 : && code == PLUS
1358 908 : && !rtx_equal_p (dst, src1)
1359 13431801 : && !use_ndd)
1360 : {
1361 : /* This is going to be an LEA; avoid splitting it later. */
1362 0 : emit_insn (op);
1363 : }
1364 : else
1365 : {
1366 13431801 : clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
1367 13431801 : emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
1368 : }
1369 :
1370 : /* Fix up the destination if needed. */
1371 13431801 : if (dst != operands[0])
1372 496231 : emit_move_insn (operands[0], dst);
1373 13431801 : }
1374 :
1375 : /* Expand vector logical operation CODE (AND, IOR, XOR) in MODE with
1376 : the given OPERANDS. */
1377 :
1378 : void
1379 85863 : ix86_expand_vector_logical_operator (enum rtx_code code, machine_mode mode,
1380 : rtx operands[])
1381 : {
1382 85863 : rtx op1 = NULL_RTX, op2 = NULL_RTX;
1383 85863 : if (SUBREG_P (operands[1]))
1384 : {
1385 372 : op1 = operands[1];
1386 372 : op2 = operands[2];
1387 : }
1388 85491 : else if (SUBREG_P (operands[2]))
1389 : {
1390 : op1 = operands[2];
1391 : op2 = operands[1];
1392 : }
1393 : /* Optimize (__m128i) d | (__m128i) e and similar code
1394 : when d and e are float vectors into float vector logical
1395 : insn. In C/C++ without using intrinsics there is no other way
1396 : to express vector logical operation on float vectors than
1397 : to cast them temporarily to integer vectors. */
1398 3224 : if (op1
1399 3224 : && !TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
1400 3224 : && (SUBREG_P (op2) || CONST_VECTOR_P (op2))
1401 358 : && GET_MODE_CLASS (GET_MODE (SUBREG_REG (op1))) == MODE_VECTOR_FLOAT
1402 453 : && GET_MODE_SIZE (GET_MODE (SUBREG_REG (op1))) == GET_MODE_SIZE (mode)
1403 151 : && SUBREG_BYTE (op1) == 0
1404 151 : && (CONST_VECTOR_P (op2)
1405 1 : || (GET_MODE (SUBREG_REG (op1)) == GET_MODE (SUBREG_REG (op2))
1406 1 : && SUBREG_BYTE (op2) == 0))
1407 151 : && can_create_pseudo_p ())
1408 : {
1409 151 : rtx dst;
1410 151 : switch (GET_MODE (SUBREG_REG (op1)))
1411 : {
1412 67 : case E_V4SFmode:
1413 67 : case E_V8SFmode:
1414 67 : case E_V16SFmode:
1415 67 : case E_V2DFmode:
1416 67 : case E_V4DFmode:
1417 67 : case E_V8DFmode:
1418 67 : dst = gen_reg_rtx (GET_MODE (SUBREG_REG (op1)));
1419 67 : if (CONST_VECTOR_P (op2))
1420 : {
1421 66 : op2 = gen_lowpart (GET_MODE (dst), op2);
1422 66 : op2 = force_reg (GET_MODE (dst), op2);
1423 : }
1424 : else
1425 : {
1426 1 : op1 = operands[1];
1427 1 : op2 = SUBREG_REG (operands[2]);
1428 1 : if (!vector_operand (op2, GET_MODE (dst)))
1429 0 : op2 = force_reg (GET_MODE (dst), op2);
1430 : }
1431 67 : op1 = SUBREG_REG (op1);
1432 67 : if (!vector_operand (op1, GET_MODE (dst)))
1433 0 : op1 = force_reg (GET_MODE (dst), op1);
1434 67 : emit_insn (gen_rtx_SET (dst,
1435 : gen_rtx_fmt_ee (code, GET_MODE (dst),
1436 : op1, op2)));
1437 67 : emit_move_insn (operands[0], gen_lowpart (mode, dst));
1438 67 : return;
1439 : default:
1440 : break;
1441 : }
1442 : }
1443 85796 : if (!vector_operand (operands[1], mode))
1444 1 : operands[1] = force_reg (mode, operands[1]);
1445 85796 : if (!vector_operand (operands[2], mode))
1446 12890 : operands[2] = force_reg (mode, operands[2]);
1447 85796 : ix86_fixup_binary_operands_no_copy (code, mode, operands);
1448 85796 : emit_insn (gen_rtx_SET (operands[0],
1449 : gen_rtx_fmt_ee (code, mode, operands[1],
1450 : operands[2])));
1451 : }
1452 :
1453 : /* Return TRUE or FALSE depending on whether the binary operator meets the
1454 : appropriate constraints. */
1455 :
1456 : bool
1457 166432539 : ix86_binary_operator_ok (enum rtx_code code, machine_mode mode,
1458 : rtx operands[3], bool use_ndd)
1459 : {
1460 166432539 : rtx dst = operands[0];
1461 166432539 : rtx src1 = operands[1];
1462 166432539 : rtx src2 = operands[2];
1463 :
1464 : /* Both source operands cannot be in memory. */
1465 158663609 : if ((MEM_P (src1) || bcst_mem_operand (src1, mode))
1466 166432928 : && (MEM_P (src2) || bcst_mem_operand (src2, mode)))
1467 : return false;
1468 :
1469 : /* Canonicalize operand order for commutative operators. */
1470 165346782 : if (ix86_swap_binary_operands_p (code, mode, operands))
1471 568731 : std::swap (src1, src2);
1472 :
1473 : /* If the destination is memory, we must have a matching source operand. */
1474 165346782 : if (MEM_P (dst) && !rtx_equal_p (dst, src1))
1475 : return false;
1476 :
1477 : /* Source 1 cannot be a constant. */
1478 160159761 : if (CONSTANT_P (src1))
1479 : return false;
1480 :
1481 : /* Source 1 cannot be a non-matching memory. */
1482 160156712 : if (!use_ndd && MEM_P (src1) && !rtx_equal_p (dst, src1))
1483 : /* Support "andhi/andsi/anddi" as a zero-extending move. */
1484 4813426 : return (code == AND
1485 556766 : && (mode == HImode
1486 556766 : || mode == SImode
1487 341519 : || (TARGET_64BIT && mode == DImode))
1488 5125685 : && satisfies_constraint_L (src2));
1489 :
1490 : return true;
1491 : }
1492 :
1493 : /* Attempt to expand a unary operator. Make the expansion closer to the
1494 : actual machine, then just general_operand, which will allow 2 separate
1495 : memory references (one output, one input) in a single insn. */
1496 :
1497 : void
1498 131325 : ix86_expand_unary_operator (enum rtx_code code, machine_mode mode,
1499 : rtx operands[], bool use_ndd)
1500 : {
1501 131325 : bool matching_memory = false;
1502 131325 : rtx src, dst, op, clob;
1503 :
1504 131325 : dst = operands[0];
1505 131325 : src = operands[1];
1506 :
1507 : /* If the destination is memory, and we do not have matching source
1508 : operands, do things in registers. */
1509 131325 : if (MEM_P (dst))
1510 : {
1511 11802 : if (rtx_equal_p (dst, src))
1512 : matching_memory = true;
1513 : else
1514 11477 : dst = gen_reg_rtx (mode);
1515 : }
1516 :
1517 : /* When source operand is memory, destination must match. */
1518 131325 : if (!use_ndd && MEM_P (src) && !matching_memory)
1519 4599 : src = force_reg (mode, src);
1520 :
1521 : /* Emit the instruction. */
1522 :
1523 131325 : op = gen_rtx_SET (dst, gen_rtx_fmt_e (code, mode, src));
1524 :
1525 131325 : if (code == NOT)
1526 71310 : emit_insn (op);
1527 : else
1528 : {
1529 60015 : clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
1530 60015 : emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
1531 : }
1532 :
1533 : /* Fix up the destination if needed. */
1534 131325 : if (dst != operands[0])
1535 11477 : emit_move_insn (operands[0], dst);
1536 131325 : }
1537 :
1538 : /* Return TRUE or FALSE depending on whether the unary operator meets the
1539 : appropriate constraints. */
1540 :
1541 : bool
1542 2085228 : ix86_unary_operator_ok (enum rtx_code,
1543 : machine_mode,
1544 : rtx operands[2],
1545 : bool use_ndd)
1546 : {
1547 : /* If one of operands is memory, source and destination must match. */
1548 2085228 : if ((MEM_P (operands[0])
1549 1946794 : || (!use_ndd && MEM_P (operands[1])))
1550 2114547 : && ! rtx_equal_p (operands[0], operands[1]))
1551 : return false;
1552 : return true;
1553 : }
1554 :
1555 : /* Predict just emitted jump instruction to be taken with probability PROB. */
1556 :
1557 : static void
1558 12977 : predict_jump (int prob)
1559 : {
1560 12977 : rtx_insn *insn = get_last_insn ();
1561 12977 : gcc_assert (JUMP_P (insn));
1562 12977 : add_reg_br_prob_note (insn, profile_probability::from_reg_br_prob_base (prob));
1563 12977 : }
1564 :
1565 : /* Split 32bit/64bit divmod with 8bit unsigned divmod if dividend and
1566 : divisor are within the range [0-255]. */
1567 :
1568 : void
1569 27 : ix86_split_idivmod (machine_mode mode, rtx operands[],
1570 : bool unsigned_p)
1571 : {
1572 27 : rtx_code_label *end_label, *qimode_label;
1573 27 : rtx div, mod;
1574 27 : rtx_insn *insn;
1575 27 : rtx scratch, tmp0, tmp1, tmp2;
1576 27 : rtx (*gen_divmod4_1) (rtx, rtx, rtx, rtx);
1577 :
1578 27 : operands[2] = force_reg (mode, operands[2]);
1579 27 : operands[3] = force_reg (mode, operands[3]);
1580 :
1581 27 : switch (mode)
1582 : {
1583 20 : case E_SImode:
1584 20 : if (GET_MODE (operands[0]) == SImode)
1585 : {
1586 16 : if (GET_MODE (operands[1]) == SImode)
1587 14 : gen_divmod4_1 = unsigned_p ? gen_udivmodsi4_1 : gen_divmodsi4_1;
1588 : else
1589 2 : gen_divmod4_1
1590 2 : = unsigned_p ? gen_udivmodsi4_zext_2 : gen_divmodsi4_zext_2;
1591 : }
1592 : else
1593 4 : gen_divmod4_1
1594 4 : = unsigned_p ? gen_udivmodsi4_zext_1 : gen_divmodsi4_zext_1;
1595 : break;
1596 :
1597 7 : case E_DImode:
1598 7 : gen_divmod4_1 = unsigned_p ? gen_udivmoddi4_1 : gen_divmoddi4_1;
1599 : break;
1600 :
1601 0 : default:
1602 0 : gcc_unreachable ();
1603 : }
1604 :
1605 27 : end_label = gen_label_rtx ();
1606 27 : qimode_label = gen_label_rtx ();
1607 :
1608 27 : scratch = gen_reg_rtx (mode);
1609 :
1610 : /* Use 8bit unsigned divimod if dividend and divisor are within
1611 : the range [0-255]. */
1612 27 : emit_move_insn (scratch, operands[2]);
1613 27 : scratch = expand_simple_binop (mode, IOR, scratch, operands[3],
1614 : scratch, 1, OPTAB_DIRECT);
1615 27 : emit_insn (gen_test_ccno_1 (mode, scratch, GEN_INT (-0x100)));
1616 27 : tmp0 = gen_rtx_REG (CCNOmode, FLAGS_REG);
1617 27 : tmp0 = gen_rtx_EQ (VOIDmode, tmp0, const0_rtx);
1618 27 : tmp0 = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp0,
1619 : gen_rtx_LABEL_REF (VOIDmode, qimode_label),
1620 : pc_rtx);
1621 27 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp0));
1622 27 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
1623 27 : JUMP_LABEL (insn) = qimode_label;
1624 :
1625 : /* Generate original signed/unsigned divimod. */
1626 27 : emit_insn (gen_divmod4_1 (operands[0], operands[1],
1627 : operands[2], operands[3]));
1628 :
1629 : /* Branch to the end. */
1630 27 : emit_jump_insn (gen_jump (end_label));
1631 27 : emit_barrier ();
1632 :
1633 : /* Generate 8bit unsigned divide. */
1634 27 : emit_label (qimode_label);
1635 : /* Don't use operands[0] for result of 8bit divide since not all
1636 : registers support QImode ZERO_EXTRACT. */
1637 27 : tmp0 = lowpart_subreg (HImode, scratch, mode);
1638 27 : tmp1 = lowpart_subreg (HImode, operands[2], mode);
1639 27 : tmp2 = lowpart_subreg (QImode, operands[3], mode);
1640 27 : emit_insn (gen_udivmodhiqi3 (tmp0, tmp1, tmp2));
1641 :
1642 27 : if (unsigned_p)
1643 : {
1644 12 : div = gen_rtx_UDIV (mode, operands[2], operands[3]);
1645 12 : mod = gen_rtx_UMOD (mode, operands[2], operands[3]);
1646 : }
1647 : else
1648 : {
1649 15 : div = gen_rtx_DIV (mode, operands[2], operands[3]);
1650 15 : mod = gen_rtx_MOD (mode, operands[2], operands[3]);
1651 : }
1652 27 : if (mode == SImode)
1653 : {
1654 20 : if (GET_MODE (operands[0]) != SImode)
1655 4 : div = gen_rtx_ZERO_EXTEND (DImode, div);
1656 20 : if (GET_MODE (operands[1]) != SImode)
1657 2 : mod = gen_rtx_ZERO_EXTEND (DImode, mod);
1658 : }
1659 :
1660 : /* Extract remainder from AH. */
1661 27 : scratch = gen_lowpart (GET_MODE (operands[1]), scratch);
1662 27 : tmp1 = gen_rtx_ZERO_EXTRACT (GET_MODE (operands[1]), scratch,
1663 : GEN_INT (8), GEN_INT (8));
1664 27 : insn = emit_move_insn (operands[1], tmp1);
1665 27 : set_unique_reg_note (insn, REG_EQUAL, mod);
1666 :
1667 : /* Zero extend quotient from AL. */
1668 27 : tmp1 = gen_lowpart (QImode, tmp0);
1669 27 : insn = emit_insn (gen_extend_insn
1670 27 : (operands[0], tmp1,
1671 27 : GET_MODE (operands[0]), QImode, 1));
1672 27 : set_unique_reg_note (insn, REG_EQUAL, div);
1673 :
1674 27 : emit_label (end_label);
1675 27 : }
1676 :
1677 : /* Emit x86 binary operand CODE in mode MODE, where the first operand
1678 : matches destination. RTX includes clobber of FLAGS_REG. */
1679 :
1680 : void
1681 8212 : ix86_emit_binop (enum rtx_code code, machine_mode mode,
1682 : rtx dst, rtx src)
1683 : {
1684 8212 : rtx op, clob;
1685 :
1686 8212 : op = gen_rtx_SET (dst, gen_rtx_fmt_ee (code, mode, dst, src));
1687 8212 : clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
1688 :
1689 8212 : emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
1690 8212 : }
1691 :
1692 : /* Return true if regno1 def is nearest to the insn. */
1693 :
1694 : static bool
1695 15 : find_nearest_reg_def (rtx_insn *insn, int regno1, int regno2)
1696 : {
1697 15 : rtx_insn *prev = insn;
1698 15 : rtx_insn *start = BB_HEAD (BLOCK_FOR_INSN (insn));
1699 :
1700 15 : if (insn == start)
1701 : return false;
1702 40 : while (prev && prev != start)
1703 : {
1704 30 : if (!INSN_P (prev) || !NONDEBUG_INSN_P (prev))
1705 : {
1706 10 : prev = PREV_INSN (prev);
1707 10 : continue;
1708 : }
1709 20 : if (insn_defines_reg (regno1, INVALID_REGNUM, prev))
1710 : return true;
1711 15 : else if (insn_defines_reg (regno2, INVALID_REGNUM, prev))
1712 : return false;
1713 15 : prev = PREV_INSN (prev);
1714 : }
1715 :
1716 : /* None of the regs is defined in the bb. */
1717 : return false;
1718 : }
1719 :
1720 : /* INSN_UID of the last insn emitted by zero store peephole2s. */
1721 : int ix86_last_zero_store_uid;
1722 :
1723 : /* Split lea instructions into a sequence of instructions
1724 : which are executed on ALU to avoid AGU stalls.
1725 : It is assumed that it is allowed to clobber flags register
1726 : at lea position. */
1727 :
1728 : void
1729 6259 : ix86_split_lea_for_addr (rtx_insn *insn, rtx operands[], machine_mode mode)
1730 : {
1731 6259 : unsigned int regno0, regno1, regno2;
1732 6259 : struct ix86_address parts;
1733 6259 : rtx target, tmp;
1734 6259 : int ok, adds;
1735 :
1736 6259 : ok = ix86_decompose_address (operands[1], &parts);
1737 6259 : gcc_assert (ok);
1738 :
1739 6259 : target = gen_lowpart (mode, operands[0]);
1740 :
1741 6259 : regno0 = true_regnum (target);
1742 6259 : regno1 = INVALID_REGNUM;
1743 6259 : regno2 = INVALID_REGNUM;
1744 :
1745 6259 : if (parts.base)
1746 : {
1747 6251 : parts.base = gen_lowpart (mode, parts.base);
1748 6251 : regno1 = true_regnum (parts.base);
1749 : }
1750 :
1751 6259 : if (parts.index)
1752 : {
1753 6255 : parts.index = gen_lowpart (mode, parts.index);
1754 6255 : regno2 = true_regnum (parts.index);
1755 : }
1756 :
1757 6259 : if (parts.disp)
1758 226 : parts.disp = gen_lowpart (mode, parts.disp);
1759 :
1760 6259 : if (parts.scale > 1)
1761 : {
1762 : /* Case r1 = r1 + ... */
1763 11 : if (regno1 == regno0)
1764 : {
1765 : /* If we have a case r1 = r1 + C * r2 then we
1766 : should use multiplication which is very
1767 : expensive. Assume cost model is wrong if we
1768 : have such case here. */
1769 0 : gcc_assert (regno2 != regno0);
1770 :
1771 0 : for (adds = parts.scale; adds > 0; adds--)
1772 0 : ix86_emit_binop (PLUS, mode, target, parts.index);
1773 : }
1774 : else
1775 : {
1776 : /* r1 = r2 + r3 * C case. Need to move r3 into r1. */
1777 11 : if (regno0 != regno2)
1778 8 : emit_insn (gen_rtx_SET (target, parts.index));
1779 :
1780 : /* Use shift for scaling, but emit it as MULT instead
1781 : to avoid it being immediately peephole2 optimized back
1782 : into lea. */
1783 11 : ix86_emit_binop (MULT, mode, target, GEN_INT (parts.scale));
1784 :
1785 11 : if (parts.base)
1786 3 : ix86_emit_binop (PLUS, mode, target, parts.base);
1787 :
1788 11 : if (parts.disp && parts.disp != const0_rtx)
1789 2 : ix86_emit_binop (PLUS, mode, target, parts.disp);
1790 : }
1791 : }
1792 6248 : else if (!parts.base && !parts.index)
1793 : {
1794 0 : gcc_assert(parts.disp);
1795 0 : emit_insn (gen_rtx_SET (target, parts.disp));
1796 : }
1797 : else
1798 : {
1799 6248 : if (!parts.base)
1800 : {
1801 0 : if (regno0 != regno2)
1802 0 : emit_insn (gen_rtx_SET (target, parts.index));
1803 : }
1804 6248 : else if (!parts.index)
1805 : {
1806 4 : if (regno0 != regno1)
1807 2 : emit_insn (gen_rtx_SET (target, parts.base));
1808 : }
1809 : else
1810 : {
1811 6244 : if (regno0 == regno1)
1812 : tmp = parts.index;
1813 3201 : else if (regno0 == regno2)
1814 : tmp = parts.base;
1815 : else
1816 : {
1817 15 : rtx tmp1;
1818 :
1819 : /* Find better operand for SET instruction, depending
1820 : on which definition is farther from the insn. */
1821 15 : if (find_nearest_reg_def (insn, regno1, regno2))
1822 5 : tmp = parts.index, tmp1 = parts.base;
1823 : else
1824 10 : tmp = parts.base, tmp1 = parts.index;
1825 :
1826 15 : emit_insn (gen_rtx_SET (target, tmp));
1827 :
1828 15 : if (parts.disp && parts.disp != const0_rtx)
1829 0 : ix86_emit_binop (PLUS, mode, target, parts.disp);
1830 :
1831 15 : ix86_emit_binop (PLUS, mode, target, tmp1);
1832 15 : return;
1833 : }
1834 :
1835 6229 : ix86_emit_binop (PLUS, mode, target, tmp);
1836 : }
1837 :
1838 6233 : if (parts.disp && parts.disp != const0_rtx)
1839 5 : ix86_emit_binop (PLUS, mode, target, parts.disp);
1840 : }
1841 : }
1842 :
1843 : /* Post-reload splitter for converting an SF or DFmode value in an
1844 : SSE register into an unsigned SImode. */
1845 :
1846 : void
1847 0 : ix86_split_convert_uns_si_sse (rtx operands[])
1848 : {
1849 0 : machine_mode vecmode;
1850 0 : rtx value, large, zero_or_two31, input, two31, x;
1851 :
1852 0 : large = operands[1];
1853 0 : zero_or_two31 = operands[2];
1854 0 : input = operands[3];
1855 0 : two31 = operands[4];
1856 0 : vecmode = GET_MODE (large);
1857 0 : value = gen_rtx_REG (vecmode, REGNO (operands[0]));
1858 :
1859 : /* Load up the value into the low element. We must ensure that the other
1860 : elements are valid floats -- zero is the easiest such value. */
1861 0 : if (MEM_P (input))
1862 : {
1863 0 : if (vecmode == V4SFmode)
1864 0 : emit_insn (gen_vec_setv4sf_0 (value, CONST0_RTX (V4SFmode), input));
1865 : else
1866 0 : emit_insn (gen_sse2_loadlpd (value, CONST0_RTX (V2DFmode), input));
1867 : }
1868 : else
1869 : {
1870 0 : input = gen_rtx_REG (vecmode, REGNO (input));
1871 0 : emit_move_insn (value, CONST0_RTX (vecmode));
1872 0 : if (vecmode == V4SFmode)
1873 0 : emit_insn (gen_sse_movss_v4sf (value, value, input));
1874 : else
1875 0 : emit_insn (gen_sse2_movsd_v2df (value, value, input));
1876 : }
1877 :
1878 0 : emit_move_insn (large, two31);
1879 0 : emit_move_insn (zero_or_two31, MEM_P (two31) ? large : two31);
1880 :
1881 0 : x = gen_rtx_fmt_ee (LE, vecmode, large, value);
1882 0 : emit_insn (gen_rtx_SET (large, x));
1883 :
1884 0 : x = gen_rtx_AND (vecmode, zero_or_two31, large);
1885 0 : emit_insn (gen_rtx_SET (zero_or_two31, x));
1886 :
1887 0 : x = gen_rtx_MINUS (vecmode, value, zero_or_two31);
1888 0 : emit_insn (gen_rtx_SET (value, x));
1889 :
1890 0 : large = gen_rtx_REG (V4SImode, REGNO (large));
1891 0 : emit_insn (gen_ashlv4si3 (large, large, GEN_INT (31)));
1892 :
1893 0 : x = gen_rtx_REG (V4SImode, REGNO (value));
1894 0 : if (vecmode == V4SFmode)
1895 0 : emit_insn (gen_fix_truncv4sfv4si2 (x, value));
1896 : else
1897 0 : emit_insn (gen_sse2_cvttpd2dq (x, value));
1898 0 : value = x;
1899 :
1900 0 : emit_insn (gen_xorv4si3 (value, value, large));
1901 0 : }
1902 :
1903 : /* Convert an unsigned DImode value into a DFmode, using only SSE.
1904 : Expects the 64-bit DImode to be supplied in a pair of integral
1905 : registers. Requires SSE2; will use SSE3 if available. For x86_32,
1906 : -mfpmath=sse, !optimize_size only. */
1907 :
1908 : void
1909 0 : ix86_expand_convert_uns_didf_sse (rtx target, rtx input)
1910 : {
1911 0 : REAL_VALUE_TYPE bias_lo_rvt, bias_hi_rvt;
1912 0 : rtx int_xmm, fp_xmm;
1913 0 : rtx biases, exponents;
1914 0 : rtx x;
1915 :
1916 0 : int_xmm = gen_reg_rtx (V4SImode);
1917 0 : if (TARGET_INTER_UNIT_MOVES_TO_VEC)
1918 0 : emit_insn (gen_movdi_to_sse (int_xmm, input));
1919 0 : else if (TARGET_SSE_SPLIT_REGS)
1920 : {
1921 0 : emit_clobber (int_xmm);
1922 0 : emit_move_insn (gen_lowpart (DImode, int_xmm), input);
1923 : }
1924 : else
1925 : {
1926 0 : x = gen_reg_rtx (V2DImode);
1927 0 : ix86_expand_vector_init_one_nonzero (false, V2DImode, x, input, 0);
1928 0 : emit_move_insn (int_xmm, gen_lowpart (V4SImode, x));
1929 : }
1930 :
1931 0 : x = gen_rtx_CONST_VECTOR (V4SImode,
1932 : gen_rtvec (4, GEN_INT (0x43300000UL),
1933 : GEN_INT (0x45300000UL),
1934 : const0_rtx, const0_rtx));
1935 0 : exponents = validize_mem (force_const_mem (V4SImode, x));
1936 :
1937 : /* int_xmm = {0x45300000UL, fp_xmm/hi, 0x43300000, fp_xmm/lo } */
1938 0 : emit_insn (gen_vec_interleave_lowv4si (int_xmm, int_xmm, exponents));
1939 :
1940 : /* Concatenating (juxtaposing) (0x43300000UL ## fp_value_low_xmm)
1941 : yields a valid DF value equal to (0x1.0p52 + double(fp_value_lo_xmm)).
1942 : Similarly (0x45300000UL ## fp_value_hi_xmm) yields
1943 : (0x1.0p84 + double(fp_value_hi_xmm)).
1944 : Note these exponents differ by 32. */
1945 :
1946 0 : fp_xmm = copy_to_mode_reg (V2DFmode, gen_lowpart (V2DFmode, int_xmm));
1947 :
1948 : /* Subtract off those 0x1.0p52 and 0x1.0p84 biases, to produce values
1949 : in [0,2**32-1] and [0]+[2**32,2**64-1] respectively. */
1950 0 : real_ldexp (&bias_lo_rvt, &dconst1, 52);
1951 0 : real_ldexp (&bias_hi_rvt, &dconst1, 84);
1952 0 : biases = const_double_from_real_value (bias_lo_rvt, DFmode);
1953 0 : x = const_double_from_real_value (bias_hi_rvt, DFmode);
1954 0 : biases = gen_rtx_CONST_VECTOR (V2DFmode, gen_rtvec (2, biases, x));
1955 0 : biases = validize_mem (force_const_mem (V2DFmode, biases));
1956 0 : emit_insn (gen_subv2df3 (fp_xmm, fp_xmm, biases));
1957 :
1958 : /* Add the upper and lower DFmode values together. */
1959 0 : if (TARGET_SSE3)
1960 0 : emit_insn (gen_sse3_haddv2df3 (fp_xmm, fp_xmm, fp_xmm));
1961 : else
1962 : {
1963 0 : x = copy_to_mode_reg (V2DFmode, fp_xmm);
1964 0 : emit_insn (gen_vec_interleave_highv2df (fp_xmm, fp_xmm, fp_xmm));
1965 0 : emit_insn (gen_addv2df3 (fp_xmm, fp_xmm, x));
1966 : }
1967 :
1968 0 : ix86_expand_vector_extract (false, target, fp_xmm, 0);
1969 0 : }
1970 :
1971 : /* Not used, but eases macroization of patterns. */
1972 : void
1973 0 : ix86_expand_convert_uns_sixf_sse (rtx, rtx)
1974 : {
1975 0 : gcc_unreachable ();
1976 : }
1977 :
1978 : static rtx ix86_expand_sse_fabs (rtx op0, rtx *smask);
1979 :
1980 : /* Convert an unsigned SImode value into a DFmode. Only currently used
1981 : for SSE, but applicable anywhere. */
1982 :
1983 : void
1984 0 : ix86_expand_convert_uns_sidf_sse (rtx target, rtx input)
1985 : {
1986 0 : REAL_VALUE_TYPE TWO31r;
1987 0 : rtx x, fp;
1988 :
1989 0 : x = expand_simple_binop (SImode, PLUS, input, GEN_INT (-2147483647 - 1),
1990 : NULL, 1, OPTAB_DIRECT);
1991 :
1992 0 : fp = gen_reg_rtx (DFmode);
1993 0 : emit_insn (gen_floatsidf2 (fp, x));
1994 :
1995 0 : real_ldexp (&TWO31r, &dconst1, 31);
1996 0 : x = const_double_from_real_value (TWO31r, DFmode);
1997 :
1998 0 : x = expand_simple_binop (DFmode, PLUS, fp, x, target, 0, OPTAB_DIRECT);
1999 :
2000 : /* Remove the sign with FE_DOWNWARD, where x - x = -0.0. */
2001 0 : if (HONOR_SIGNED_ZEROS (DFmode) && flag_rounding_math)
2002 0 : x = ix86_expand_sse_fabs (x, NULL);
2003 :
2004 0 : if (x != target)
2005 0 : emit_move_insn (target, x);
2006 0 : }
2007 :
2008 : /* Convert a signed DImode value into a DFmode. Only used for SSE in
2009 : 32-bit mode; otherwise we have a direct convert instruction. */
2010 :
2011 : void
2012 0 : ix86_expand_convert_sign_didf_sse (rtx target, rtx input)
2013 : {
2014 0 : REAL_VALUE_TYPE TWO32r;
2015 0 : rtx fp_lo, fp_hi, x;
2016 :
2017 0 : fp_lo = gen_reg_rtx (DFmode);
2018 0 : fp_hi = gen_reg_rtx (DFmode);
2019 :
2020 0 : emit_insn (gen_floatsidf2 (fp_hi, gen_highpart (SImode, input)));
2021 :
2022 0 : real_ldexp (&TWO32r, &dconst1, 32);
2023 0 : x = const_double_from_real_value (TWO32r, DFmode);
2024 0 : fp_hi = expand_simple_binop (DFmode, MULT, fp_hi, x, fp_hi, 0, OPTAB_DIRECT);
2025 :
2026 0 : ix86_expand_convert_uns_sidf_sse (fp_lo, gen_lowpart (SImode, input));
2027 :
2028 0 : x = expand_simple_binop (DFmode, PLUS, fp_hi, fp_lo, target,
2029 : 0, OPTAB_DIRECT);
2030 0 : if (x != target)
2031 0 : emit_move_insn (target, x);
2032 0 : }
2033 :
2034 : /* Convert an unsigned SImode value into a SFmode, using only SSE.
2035 : For x86_32, -mfpmath=sse, !optimize_size only. */
2036 : void
2037 0 : ix86_expand_convert_uns_sisf_sse (rtx target, rtx input)
2038 : {
2039 0 : REAL_VALUE_TYPE ONE16r;
2040 0 : rtx fp_hi, fp_lo, int_hi, int_lo, x;
2041 :
2042 0 : real_ldexp (&ONE16r, &dconst1, 16);
2043 0 : x = const_double_from_real_value (ONE16r, SFmode);
2044 0 : int_lo = expand_simple_binop (SImode, AND, input, GEN_INT(0xffff),
2045 : NULL, 0, OPTAB_DIRECT);
2046 0 : int_hi = expand_simple_binop (SImode, LSHIFTRT, input, GEN_INT(16),
2047 : NULL, 0, OPTAB_DIRECT);
2048 0 : fp_hi = gen_reg_rtx (SFmode);
2049 0 : fp_lo = gen_reg_rtx (SFmode);
2050 0 : emit_insn (gen_floatsisf2 (fp_hi, int_hi));
2051 0 : emit_insn (gen_floatsisf2 (fp_lo, int_lo));
2052 0 : if (TARGET_FMA)
2053 : {
2054 0 : x = validize_mem (force_const_mem (SFmode, x));
2055 0 : fp_hi = gen_rtx_FMA (SFmode, fp_hi, x, fp_lo);
2056 0 : emit_move_insn (target, fp_hi);
2057 : }
2058 : else
2059 : {
2060 0 : fp_hi = expand_simple_binop (SFmode, MULT, fp_hi, x, fp_hi,
2061 : 0, OPTAB_DIRECT);
2062 0 : fp_hi = expand_simple_binop (SFmode, PLUS, fp_hi, fp_lo, target,
2063 : 0, OPTAB_DIRECT);
2064 0 : if (!rtx_equal_p (target, fp_hi))
2065 0 : emit_move_insn (target, fp_hi);
2066 : }
2067 0 : }
2068 :
2069 : /* floatunsv{4,8}siv{4,8}sf2 expander. Expand code to convert
2070 : a vector of unsigned ints VAL to vector of floats TARGET. */
2071 :
2072 : void
2073 54 : ix86_expand_vector_convert_uns_vsivsf (rtx target, rtx val)
2074 : {
2075 54 : rtx tmp[8];
2076 54 : REAL_VALUE_TYPE TWO16r;
2077 54 : machine_mode intmode = GET_MODE (val);
2078 54 : machine_mode fltmode = GET_MODE (target);
2079 54 : rtx (*cvt) (rtx, rtx);
2080 :
2081 54 : if (intmode == V4SImode)
2082 : cvt = gen_floatv4siv4sf2;
2083 : else
2084 2 : cvt = gen_floatv8siv8sf2;
2085 54 : tmp[0] = ix86_build_const_vector (intmode, 1, GEN_INT (0xffff));
2086 54 : tmp[0] = force_reg (intmode, tmp[0]);
2087 54 : tmp[1] = expand_simple_binop (intmode, AND, val, tmp[0], NULL_RTX, 1,
2088 : OPTAB_DIRECT);
2089 54 : tmp[2] = expand_simple_binop (intmode, LSHIFTRT, val, GEN_INT (16),
2090 : NULL_RTX, 1, OPTAB_DIRECT);
2091 54 : tmp[3] = gen_reg_rtx (fltmode);
2092 54 : emit_insn (cvt (tmp[3], tmp[1]));
2093 54 : tmp[4] = gen_reg_rtx (fltmode);
2094 54 : emit_insn (cvt (tmp[4], tmp[2]));
2095 54 : real_ldexp (&TWO16r, &dconst1, 16);
2096 54 : tmp[5] = const_double_from_real_value (TWO16r, SFmode);
2097 54 : tmp[5] = force_reg (fltmode, ix86_build_const_vector (fltmode, 1, tmp[5]));
2098 54 : if (TARGET_FMA)
2099 : {
2100 1 : tmp[6] = gen_rtx_FMA (fltmode, tmp[4], tmp[5], tmp[3]);
2101 1 : emit_move_insn (target, tmp[6]);
2102 : }
2103 : else
2104 : {
2105 53 : tmp[6] = expand_simple_binop (fltmode, MULT, tmp[4], tmp[5],
2106 : NULL_RTX, 1, OPTAB_DIRECT);
2107 53 : tmp[7] = expand_simple_binop (fltmode, PLUS, tmp[3], tmp[6],
2108 : target, 1, OPTAB_DIRECT);
2109 53 : if (tmp[7] != target)
2110 0 : emit_move_insn (target, tmp[7]);
2111 : }
2112 54 : }
2113 :
2114 : /* Adjust a V*SFmode/V*DFmode value VAL so that *sfix_trunc* resp. fix_trunc*
2115 : pattern can be used on it instead of fixuns_trunc*.
2116 : This is done by doing just signed conversion if < 0x1p31, and otherwise by
2117 : subtracting 0x1p31 first and xoring in 0x80000000 from *XORP afterwards. */
2118 :
2119 : rtx
2120 324 : ix86_expand_adjust_ufix_to_sfix_si (rtx val, rtx *xorp)
2121 : {
2122 324 : REAL_VALUE_TYPE TWO31r;
2123 324 : rtx two31r, tmp[4];
2124 324 : machine_mode mode = GET_MODE (val);
2125 324 : machine_mode scalarmode = GET_MODE_INNER (mode);
2126 648 : machine_mode intmode = GET_MODE_SIZE (mode) == 32 ? V8SImode : V4SImode;
2127 324 : rtx (*cmp) (rtx, rtx, rtx, rtx);
2128 324 : int i;
2129 :
2130 1296 : for (i = 0; i < 3; i++)
2131 972 : tmp[i] = gen_reg_rtx (mode);
2132 324 : real_ldexp (&TWO31r, &dconst1, 31);
2133 324 : two31r = const_double_from_real_value (TWO31r, scalarmode);
2134 324 : two31r = ix86_build_const_vector (mode, 1, two31r);
2135 324 : two31r = force_reg (mode, two31r);
2136 324 : switch (mode)
2137 : {
2138 : case E_V8SFmode: cmp = gen_avx_maskcmpv8sf3; break;
2139 10 : case E_V4SFmode: cmp = gen_sse_maskcmpv4sf3; break;
2140 16 : case E_V4DFmode: cmp = gen_avx_maskcmpv4df3; break;
2141 298 : case E_V2DFmode: cmp = gen_sse2_maskcmpv2df3; break;
2142 0 : default: gcc_unreachable ();
2143 : }
2144 324 : tmp[3] = gen_rtx_LE (mode, two31r, val);
2145 324 : emit_insn (cmp (tmp[0], two31r, val, tmp[3]));
2146 324 : tmp[1] = expand_simple_binop (mode, AND, tmp[0], two31r, tmp[1],
2147 : 0, OPTAB_DIRECT);
2148 324 : if (intmode == V4SImode || TARGET_AVX2)
2149 648 : *xorp = expand_simple_binop (intmode, ASHIFT,
2150 324 : gen_lowpart (intmode, tmp[0]),
2151 : GEN_INT (31), NULL_RTX, 0,
2152 : OPTAB_DIRECT);
2153 : else
2154 : {
2155 0 : rtx two31 = gen_int_mode (HOST_WIDE_INT_1U << 31, SImode);
2156 0 : two31 = ix86_build_const_vector (intmode, 1, two31);
2157 0 : *xorp = expand_simple_binop (intmode, AND,
2158 0 : gen_lowpart (intmode, tmp[0]),
2159 : two31, NULL_RTX, 0,
2160 : OPTAB_DIRECT);
2161 : }
2162 324 : return expand_simple_binop (mode, MINUS, val, tmp[1], tmp[2],
2163 324 : 0, OPTAB_DIRECT);
2164 : }
2165 :
2166 : /* Generate code for floating point ABS or NEG. */
2167 :
2168 : void
2169 35004 : ix86_expand_fp_absneg_operator (enum rtx_code code, machine_mode mode,
2170 : rtx operands[])
2171 : {
2172 35004 : rtx set, dst, src;
2173 35004 : bool use_sse = false;
2174 35004 : bool vector_mode = VECTOR_MODE_P (mode);
2175 35004 : machine_mode vmode = mode;
2176 35004 : rtvec par;
2177 :
2178 35004 : switch (mode)
2179 : {
2180 : case E_HFmode:
2181 : use_sse = true;
2182 : vmode = V8HFmode;
2183 : break;
2184 0 : case E_BFmode:
2185 0 : use_sse = true;
2186 0 : vmode = V8BFmode;
2187 0 : break;
2188 9422 : case E_SFmode:
2189 9422 : use_sse = TARGET_SSE_MATH && TARGET_SSE;
2190 : vmode = V4SFmode;
2191 : break;
2192 16092 : case E_DFmode:
2193 16092 : use_sse = TARGET_SSE_MATH && TARGET_SSE2;
2194 : vmode = V2DFmode;
2195 : break;
2196 9289 : default:
2197 9289 : use_sse = vector_mode || mode == TFmode;
2198 9289 : break;
2199 : }
2200 :
2201 35004 : dst = operands[0];
2202 35004 : src = operands[1];
2203 :
2204 35004 : set = gen_rtx_fmt_e (code, mode, src);
2205 35004 : set = gen_rtx_SET (dst, set);
2206 :
2207 35004 : if (use_sse)
2208 : {
2209 29285 : rtx mask, use, clob;
2210 :
2211 : /* NEG and ABS performed with SSE use bitwise mask operations.
2212 : Create the appropriate mask now. */
2213 29285 : mask = ix86_build_signbit_mask (vmode, vector_mode, code == ABS);
2214 29285 : use = gen_rtx_USE (VOIDmode, mask);
2215 29285 : if (vector_mode || mode == TFmode)
2216 4927 : par = gen_rtvec (2, set, use);
2217 : else
2218 : {
2219 24358 : clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
2220 24358 : par = gen_rtvec (3, set, use, clob);
2221 : }
2222 : }
2223 : else
2224 : {
2225 5719 : rtx clob;
2226 :
2227 : /* Changing of sign for FP values is doable using integer unit too. */
2228 5719 : clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
2229 5719 : par = gen_rtvec (2, set, clob);
2230 : }
2231 :
2232 35004 : emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
2233 35004 : }
2234 :
2235 : /* Deconstruct a floating point ABS or NEG operation
2236 : with integer registers into integer operations. */
2237 :
2238 : void
2239 24 : ix86_split_fp_absneg_operator (enum rtx_code code, machine_mode mode,
2240 : rtx operands[])
2241 : {
2242 24 : enum rtx_code absneg_op;
2243 24 : rtx dst, set;
2244 :
2245 24 : gcc_assert (operands_match_p (operands[0], operands[1]));
2246 :
2247 24 : switch (mode)
2248 : {
2249 0 : case E_SFmode:
2250 0 : dst = gen_lowpart (SImode, operands[0]);
2251 :
2252 0 : if (code == ABS)
2253 : {
2254 0 : set = gen_int_mode (0x7fffffff, SImode);
2255 0 : absneg_op = AND;
2256 : }
2257 : else
2258 : {
2259 0 : set = gen_int_mode (0x80000000, SImode);
2260 0 : absneg_op = XOR;
2261 : }
2262 0 : set = gen_rtx_fmt_ee (absneg_op, SImode, dst, set);
2263 0 : break;
2264 :
2265 1 : case E_DFmode:
2266 1 : if (TARGET_64BIT)
2267 : {
2268 1 : dst = gen_lowpart (DImode, operands[0]);
2269 1 : dst = gen_rtx_ZERO_EXTRACT (DImode, dst, const1_rtx, GEN_INT (63));
2270 :
2271 1 : if (code == ABS)
2272 0 : set = const0_rtx;
2273 : else
2274 1 : set = gen_rtx_NOT (DImode, dst);
2275 : }
2276 : else
2277 : {
2278 0 : dst = gen_highpart (SImode, operands[0]);
2279 :
2280 0 : if (code == ABS)
2281 : {
2282 0 : set = gen_int_mode (0x7fffffff, SImode);
2283 0 : absneg_op = AND;
2284 : }
2285 : else
2286 : {
2287 0 : set = gen_int_mode (0x80000000, SImode);
2288 0 : absneg_op = XOR;
2289 : }
2290 0 : set = gen_rtx_fmt_ee (absneg_op, SImode, dst, set);
2291 : }
2292 : break;
2293 :
2294 23 : case E_XFmode:
2295 23 : dst = gen_rtx_REG (SImode,
2296 23 : REGNO (operands[0]) + (TARGET_64BIT ? 1 : 2));
2297 23 : if (code == ABS)
2298 : {
2299 1 : set = GEN_INT (0x7fff);
2300 1 : absneg_op = AND;
2301 : }
2302 : else
2303 : {
2304 22 : set = GEN_INT (0x8000);
2305 22 : absneg_op = XOR;
2306 : }
2307 23 : set = gen_rtx_fmt_ee (absneg_op, SImode, dst, set);
2308 23 : break;
2309 :
2310 0 : default:
2311 0 : gcc_unreachable ();
2312 : }
2313 :
2314 24 : set = gen_rtx_SET (dst, set);
2315 :
2316 24 : rtx clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
2317 24 : rtvec par = gen_rtvec (2, set, clob);
2318 :
2319 24 : emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
2320 24 : }
2321 :
2322 : /* Expand a copysign operation. Special case operand 0 being a constant. */
2323 :
2324 : void
2325 23246 : ix86_expand_copysign (rtx operands[])
2326 : {
2327 23246 : machine_mode mode, vmode;
2328 23246 : rtx dest, vdest, op0, op1, mask, op2, op3;
2329 :
2330 23246 : mode = GET_MODE (operands[0]);
2331 :
2332 23246 : switch (mode)
2333 : {
2334 : case E_HFmode:
2335 : vmode = V8HFmode;
2336 : break;
2337 0 : case E_BFmode:
2338 0 : vmode = V8BFmode;
2339 0 : break;
2340 11566 : case E_SFmode:
2341 11566 : vmode = V4SFmode;
2342 11566 : break;
2343 11541 : case E_DFmode:
2344 11541 : vmode = V2DFmode;
2345 11541 : break;
2346 127 : case E_TFmode:
2347 127 : vmode = mode;
2348 127 : break;
2349 0 : default:
2350 0 : gcc_unreachable();
2351 : }
2352 :
2353 23246 : if (rtx_equal_p (operands[1], operands[2]))
2354 : {
2355 0 : emit_move_insn (operands[0], operands[1]);
2356 0 : return;
2357 : }
2358 :
2359 23246 : dest = operands[0];
2360 23246 : vdest = lowpart_subreg (vmode, dest, mode);
2361 23246 : if (vdest == NULL_RTX)
2362 0 : vdest = gen_reg_rtx (vmode);
2363 : else
2364 : dest = NULL_RTX;
2365 23246 : op1 = lowpart_subreg (vmode, force_reg (mode, operands[1]), mode);
2366 46478 : mask = ix86_build_signbit_mask (vmode, TARGET_AVX512F && mode != HFmode, 0);
2367 :
2368 23246 : if (CONST_DOUBLE_P (operands[2]))
2369 : {
2370 83 : if (real_isneg (CONST_DOUBLE_REAL_VALUE (operands[2])))
2371 : /* Simplify b = copysign (a, negative) to b = mask | a. */
2372 80 : op1 = gen_rtx_IOR (vmode, mask, op1);
2373 : else
2374 : {
2375 : /* Simplify b = copysign (a, positive) to b = invert_mask & a. */
2376 3 : rtx invert_mask
2377 3 : = ix86_build_signbit_mask (vmode,
2378 3 : TARGET_AVX512F && mode != HFmode,
2379 : true);
2380 3 : op1 = gen_rtx_AND (vmode, invert_mask, op1);
2381 : }
2382 83 : emit_move_insn (vdest, op1);
2383 83 : if (dest)
2384 0 : emit_move_insn (dest, lowpart_subreg (mode, vdest, vmode));
2385 : return;
2386 : }
2387 : else
2388 23163 : op0 = lowpart_subreg (vmode, force_reg (mode, operands[2]), mode);
2389 :
2390 23163 : op2 = gen_reg_rtx (vmode);
2391 23163 : op3 = gen_reg_rtx (vmode);
2392 23163 : rtx invert_mask;
2393 : /* NB: Generate vmovdqa, vpandn, vpand, vpor for AVX and generate pand,
2394 : pand, por for SSE. */
2395 23163 : if (TARGET_AVX)
2396 31 : invert_mask = gen_rtx_NOT (vmode, mask);
2397 : else
2398 46264 : invert_mask = ix86_build_signbit_mask (vmode,
2399 : TARGET_AVX512F && mode != HFmode,
2400 : true);
2401 23163 : emit_move_insn (op2, gen_rtx_AND (vmode, invert_mask, op1));
2402 23163 : emit_move_insn (op3, gen_rtx_AND (vmode, mask, op0));
2403 23163 : emit_move_insn (vdest, gen_rtx_IOR (vmode, op2, op3));
2404 23163 : if (dest)
2405 0 : emit_move_insn (dest, lowpart_subreg (mode, vdest, vmode));
2406 : }
2407 :
2408 : /* Expand an xorsign operation. */
2409 :
2410 : void
2411 20 : ix86_expand_xorsign (rtx operands[])
2412 : {
2413 20 : machine_mode mode, vmode;
2414 20 : rtx dest, vdest, op0, op1, mask, x, temp;
2415 :
2416 20 : dest = operands[0];
2417 20 : op0 = operands[1];
2418 20 : op1 = operands[2];
2419 :
2420 20 : mode = GET_MODE (dest);
2421 :
2422 20 : switch (mode)
2423 : {
2424 : case E_HFmode:
2425 : vmode = V8HFmode;
2426 : break;
2427 : case E_BFmode:
2428 : vmode = V8BFmode;
2429 : break;
2430 : case E_SFmode:
2431 : vmode = V4SFmode;
2432 : break;
2433 : case E_DFmode:
2434 : vmode = V2DFmode;
2435 : break;
2436 0 : default:
2437 0 : gcc_unreachable ();
2438 20 : break;
2439 : }
2440 :
2441 20 : temp = gen_reg_rtx (vmode);
2442 20 : mask = ix86_build_signbit_mask (vmode, 0, 0);
2443 :
2444 20 : op1 = lowpart_subreg (vmode, force_reg (mode, op1), mode);
2445 20 : x = gen_rtx_AND (vmode, op1, mask);
2446 20 : emit_insn (gen_rtx_SET (temp, x));
2447 :
2448 20 : op0 = lowpart_subreg (vmode, force_reg (mode, op0), mode);
2449 20 : x = gen_rtx_XOR (vmode, temp, op0);
2450 :
2451 20 : vdest = lowpart_subreg (vmode, dest, mode);
2452 20 : if (vdest == NULL_RTX)
2453 0 : vdest = gen_reg_rtx (vmode);
2454 : else
2455 : dest = NULL_RTX;
2456 20 : emit_insn (gen_rtx_SET (vdest, x));
2457 :
2458 20 : if (dest)
2459 0 : emit_move_insn (dest, lowpart_subreg (mode, vdest, vmode));
2460 20 : }
2461 :
2462 : static rtx ix86_expand_compare (enum rtx_code code, rtx op0, rtx op1);
2463 :
2464 : void
2465 6761280 : ix86_expand_branch (enum rtx_code code, rtx op0, rtx op1, rtx label)
2466 : {
2467 6761280 : machine_mode mode = GET_MODE (op0);
2468 6761280 : rtx tmp;
2469 :
2470 : /* Handle special case - vector comparison with boolean result, transform
2471 : it using ptest instruction or vpcmpeq + kortest. */
2472 6761280 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
2473 6741917 : || (mode == TImode && !TARGET_64BIT)
2474 6741917 : || mode == OImode
2475 13503197 : || GET_MODE_SIZE (mode) == 64)
2476 : {
2477 19363 : unsigned msize = GET_MODE_SIZE (mode);
2478 19363 : machine_mode p_mode
2479 19363 : = msize == 64 ? V16SImode : msize == 32 ? V4DImode : V2DImode;
2480 : /* kortest set CF when result is 0xFFFF (op0 == op1). */
2481 19363 : rtx flag = gen_rtx_REG (msize == 64 ? CCCmode : CCZmode, FLAGS_REG);
2482 :
2483 19363 : gcc_assert (code == EQ || code == NE);
2484 :
2485 : /* Using vpcmpeq zmm zmm k + kortest for 512-bit vectors. */
2486 19363 : if (msize == 64)
2487 : {
2488 2431 : if (mode != V16SImode)
2489 : {
2490 2431 : op0 = lowpart_subreg (p_mode, force_reg (mode, op0), mode);
2491 2431 : op1 = lowpart_subreg (p_mode, force_reg (mode, op1), mode);
2492 : }
2493 :
2494 2431 : tmp = gen_reg_rtx (HImode);
2495 2431 : emit_insn (gen_avx512f_cmpv16si3 (tmp, op0, op1, GEN_INT (0)));
2496 2431 : emit_insn (gen_kortesthi_ccc (tmp, tmp));
2497 : }
2498 : /* Using ptest for 128/256-bit vectors. */
2499 : else
2500 : {
2501 16932 : if (GET_MODE_CLASS (mode) != MODE_VECTOR_INT)
2502 : {
2503 0 : op0 = lowpart_subreg (p_mode, force_reg (mode, op0), mode);
2504 0 : op1 = lowpart_subreg (p_mode, force_reg (mode, op1), mode);
2505 0 : mode = p_mode;
2506 : }
2507 :
2508 : /* Generate XOR since we can't check that one operand is zero
2509 : vector. */
2510 16932 : tmp = gen_reg_rtx (mode);
2511 16932 : rtx ops[3] = { tmp, op0, op1 };
2512 16932 : ix86_expand_vector_logical_operator (XOR, mode, ops);
2513 16932 : tmp = gen_lowpart (p_mode, tmp);
2514 16932 : emit_insn (gen_rtx_SET (gen_rtx_REG (CCZmode, FLAGS_REG),
2515 : gen_rtx_UNSPEC (CCZmode,
2516 : gen_rtvec (2, tmp, tmp),
2517 : UNSPEC_PTEST)));
2518 : }
2519 19363 : tmp = gen_rtx_fmt_ee (code, VOIDmode, flag, const0_rtx);
2520 19363 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
2521 : gen_rtx_LABEL_REF (VOIDmode, label),
2522 : pc_rtx);
2523 19363 : emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
2524 19363 : return;
2525 : }
2526 :
2527 6741917 : switch (mode)
2528 : {
2529 6707814 : case E_HFmode:
2530 6707814 : case E_SFmode:
2531 6707814 : case E_DFmode:
2532 6707814 : case E_XFmode:
2533 6707814 : case E_QImode:
2534 6707814 : case E_HImode:
2535 6707814 : case E_SImode:
2536 6707814 : simple:
2537 6707814 : tmp = ix86_expand_compare (code, op0, op1);
2538 6707814 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
2539 : gen_rtx_LABEL_REF (VOIDmode, label),
2540 : pc_rtx);
2541 6707814 : emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
2542 6707814 : return;
2543 :
2544 7 : case E_BFmode:
2545 7 : gcc_assert (TARGET_AVX10_2 && !flag_trapping_math);
2546 7 : goto simple;
2547 :
2548 2731947 : case E_DImode:
2549 2731947 : if (TARGET_64BIT)
2550 2701280 : goto simple;
2551 : /* FALLTHRU */
2552 92637 : case E_TImode:
2553 : /* DI and TI mode equality/inequality comparisons may be performed
2554 : on SSE registers. Avoid splitting them, except when optimizing
2555 : for size. */
2556 92637 : if ((code == EQ || code == NE)
2557 92637 : && !optimize_insn_for_size_p ())
2558 58534 : goto simple;
2559 :
2560 : /* Expand DImode branch into multiple compare+branch. */
2561 34103 : {
2562 34103 : rtx lo[2], hi[2];
2563 34103 : rtx_code_label *label2;
2564 34103 : enum rtx_code code1, code2, code3;
2565 34103 : machine_mode submode;
2566 :
2567 34103 : if (CONSTANT_P (op0) && !CONSTANT_P (op1))
2568 : {
2569 0 : std::swap (op0, op1);
2570 0 : code = swap_condition (code);
2571 : }
2572 :
2573 34103 : split_double_mode (mode, &op0, 1, lo+0, hi+0);
2574 34103 : split_double_mode (mode, &op1, 1, lo+1, hi+1);
2575 :
2576 34103 : submode = mode == DImode ? SImode : DImode;
2577 :
2578 : /* If we are doing less-than or greater-or-equal-than,
2579 : op1 is a constant and the low word is zero, then we can just
2580 : examine the high word. Similarly for low word -1 and
2581 : less-or-equal-than or greater-than. */
2582 :
2583 34103 : if (CONST_INT_P (hi[1]))
2584 20748 : switch (code)
2585 : {
2586 11348 : case LT: case LTU: case GE: case GEU:
2587 11348 : if (lo[1] == const0_rtx)
2588 : {
2589 10924 : ix86_expand_branch (code, hi[0], hi[1], label);
2590 10924 : return;
2591 : }
2592 : break;
2593 7850 : case LE: case LEU: case GT: case GTU:
2594 7850 : if (lo[1] == constm1_rtx)
2595 : {
2596 530 : ix86_expand_branch (code, hi[0], hi[1], label);
2597 530 : return;
2598 : }
2599 : break;
2600 : default:
2601 : break;
2602 : }
2603 :
2604 : /* Emulate comparisons that do not depend on Zero flag with
2605 : double-word subtraction. Note that only Overflow, Sign
2606 : and Carry flags are valid, so swap arguments and condition
2607 : of comparisons that would otherwise test Zero flag. */
2608 :
2609 22649 : switch (code)
2610 : {
2611 12763 : case LE: case LEU: case GT: case GTU:
2612 12763 : std::swap (lo[0], lo[1]);
2613 12763 : std::swap (hi[0], hi[1]);
2614 12763 : code = swap_condition (code);
2615 : /* FALLTHRU */
2616 :
2617 19616 : case LT: case LTU: case GE: case GEU:
2618 19616 : {
2619 19616 : bool uns = (code == LTU || code == GEU);
2620 4062 : rtx (*sbb_insn) (machine_mode, rtx, rtx, rtx)
2621 19616 : = uns ? gen_sub3_carry_ccc : gen_sub3_carry_ccgz;
2622 :
2623 19616 : if (!nonimmediate_operand (lo[0], submode))
2624 7320 : lo[0] = force_reg (submode, lo[0]);
2625 19616 : if (!x86_64_general_operand (lo[1], submode))
2626 0 : lo[1] = force_reg (submode, lo[1]);
2627 :
2628 19616 : if (!register_operand (hi[0], submode))
2629 8105 : hi[0] = force_reg (submode, hi[0]);
2630 15554 : if ((uns && !nonimmediate_operand (hi[1], submode))
2631 19616 : || (!uns && !x86_64_general_operand (hi[1], submode)))
2632 315 : hi[1] = force_reg (submode, hi[1]);
2633 :
2634 19616 : emit_insn (gen_cmp_1 (submode, lo[0], lo[1]));
2635 :
2636 19616 : tmp = gen_rtx_SCRATCH (submode);
2637 19616 : emit_insn (sbb_insn (submode, tmp, hi[0], hi[1]));
2638 :
2639 23678 : tmp = gen_rtx_REG (uns ? CCCmode : CCGZmode, FLAGS_REG);
2640 19616 : ix86_expand_branch (code, tmp, const0_rtx, label);
2641 19616 : return;
2642 : }
2643 :
2644 3033 : default:
2645 3033 : break;
2646 : }
2647 :
2648 : /* Otherwise, we need two or three jumps. */
2649 :
2650 3033 : label2 = gen_label_rtx ();
2651 :
2652 3033 : code1 = code;
2653 3033 : code2 = swap_condition (code);
2654 3033 : code3 = unsigned_condition (code);
2655 :
2656 3033 : switch (code)
2657 : {
2658 : case LT: case GT: case LTU: case GTU:
2659 : break;
2660 :
2661 : case LE: code1 = LT; code2 = GT; break;
2662 : case GE: code1 = GT; code2 = LT; break;
2663 : case LEU: code1 = LTU; code2 = GTU; break;
2664 : case GEU: code1 = GTU; code2 = LTU; break;
2665 :
2666 : case EQ: code1 = UNKNOWN; code2 = NE; break;
2667 : case NE: code2 = UNKNOWN; break;
2668 :
2669 0 : default:
2670 0 : gcc_unreachable ();
2671 : }
2672 :
2673 : /*
2674 : * a < b =>
2675 : * if (hi(a) < hi(b)) goto true;
2676 : * if (hi(a) > hi(b)) goto false;
2677 : * if (lo(a) < lo(b)) goto true;
2678 : * false:
2679 : */
2680 :
2681 : if (code1 != UNKNOWN)
2682 2347 : ix86_expand_branch (code1, hi[0], hi[1], label);
2683 3033 : if (code2 != UNKNOWN)
2684 686 : ix86_expand_branch (code2, hi[0], hi[1], label2);
2685 :
2686 3033 : ix86_expand_branch (code3, lo[0], lo[1], label);
2687 :
2688 3033 : if (code2 != UNKNOWN)
2689 686 : emit_label (label2);
2690 : return;
2691 : }
2692 :
2693 20125 : default:
2694 20125 : gcc_assert (GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC);
2695 20125 : goto simple;
2696 : }
2697 : }
2698 :
2699 : /* Figure out whether to use unordered fp comparisons. */
2700 :
2701 : static bool
2702 1162672 : ix86_unordered_fp_compare (enum rtx_code code)
2703 : {
2704 1162672 : if (!TARGET_IEEE_FP)
2705 : return false;
2706 :
2707 1158118 : switch (code)
2708 : {
2709 : case LT:
2710 : case LE:
2711 : case GT:
2712 : case GE:
2713 : case LTGT:
2714 : return false;
2715 :
2716 : case EQ:
2717 : case NE:
2718 :
2719 : case UNORDERED:
2720 : case ORDERED:
2721 : case UNLT:
2722 : case UNLE:
2723 : case UNGT:
2724 : case UNGE:
2725 : case UNEQ:
2726 : return true;
2727 :
2728 0 : default:
2729 0 : gcc_unreachable ();
2730 : }
2731 : }
2732 :
2733 : /* Return a comparison we can do and that it is equivalent to
2734 : swap_condition (code) apart possibly from orderedness.
2735 : But, never change orderedness if TARGET_IEEE_FP, returning
2736 : UNKNOWN in that case if necessary. */
2737 :
2738 : static enum rtx_code
2739 38674 : ix86_fp_swap_condition (enum rtx_code code)
2740 : {
2741 38674 : switch (code)
2742 : {
2743 1878 : case GT: /* GTU - CF=0 & ZF=0 */
2744 1878 : return TARGET_IEEE_FP ? UNKNOWN : UNLT;
2745 544 : case GE: /* GEU - CF=0 */
2746 544 : return TARGET_IEEE_FP ? UNKNOWN : UNLE;
2747 524 : case UNLT: /* LTU - CF=1 */
2748 524 : return TARGET_IEEE_FP ? UNKNOWN : GT;
2749 6362 : case UNLE: /* LEU - CF=1 | ZF=1 */
2750 6362 : return TARGET_IEEE_FP ? UNKNOWN : GE;
2751 29366 : default:
2752 29366 : return swap_condition (code);
2753 : }
2754 : }
2755 :
2756 : /* Return cost of comparison CODE using the best strategy for performance.
2757 : All following functions do use number of instructions as a cost metrics.
2758 : In future this should be tweaked to compute bytes for optimize_size and
2759 : take into account performance of various instructions on various CPUs. */
2760 :
2761 : static int
2762 1161534 : ix86_fp_comparison_cost (enum rtx_code code)
2763 : {
2764 1161534 : int arith_cost;
2765 :
2766 : /* The cost of code using bit-twiddling on %ah. */
2767 1161534 : switch (code)
2768 : {
2769 : case UNLE:
2770 : case UNLT:
2771 : case LTGT:
2772 : case GT:
2773 : case GE:
2774 : case UNORDERED:
2775 : case ORDERED:
2776 : case UNEQ:
2777 : arith_cost = 4;
2778 : break;
2779 85371 : case LT:
2780 85371 : case NE:
2781 85371 : case EQ:
2782 85371 : case UNGE:
2783 85371 : arith_cost = TARGET_IEEE_FP ? 5 : 4;
2784 : break;
2785 27058 : case LE:
2786 27058 : case UNGT:
2787 1076991 : arith_cost = TARGET_IEEE_FP ? 6 : 4;
2788 : break;
2789 0 : default:
2790 0 : gcc_unreachable ();
2791 : }
2792 :
2793 1161534 : switch (ix86_fp_comparison_strategy (code))
2794 : {
2795 1161534 : case IX86_FPCMP_COMI:
2796 1161534 : return arith_cost > 4 ? 3 : 2;
2797 0 : case IX86_FPCMP_SAHF:
2798 0 : return arith_cost > 4 ? 4 : 3;
2799 : default:
2800 : return arith_cost;
2801 : }
2802 : }
2803 :
2804 : /* Swap, force into registers, or otherwise massage the two operands
2805 : to a fp comparison. The operands are updated in place; the new
2806 : comparison code is returned. */
2807 :
2808 : static enum rtx_code
2809 580767 : ix86_prepare_fp_compare_args (enum rtx_code code, rtx *pop0, rtx *pop1)
2810 : {
2811 580838 : bool unordered_compare = ix86_unordered_fp_compare (code);
2812 580838 : rtx op0 = *pop0, op1 = *pop1;
2813 580838 : machine_mode op_mode = GET_MODE (op0);
2814 580838 : bool is_sse = SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (op_mode);
2815 :
2816 578434 : if (op_mode == BFmode && (!TARGET_AVX10_2 || flag_trapping_math))
2817 : {
2818 71 : rtx op = gen_lowpart (HImode, op0);
2819 71 : if (CONST_INT_P (op))
2820 0 : op = simplify_const_unary_operation (FLOAT_EXTEND, SFmode,
2821 : op0, BFmode);
2822 : else
2823 : {
2824 71 : rtx t1 = gen_reg_rtx (SImode);
2825 71 : emit_insn (gen_zero_extendhisi2 (t1, op));
2826 71 : emit_insn (gen_ashlsi3 (t1, t1, GEN_INT (16)));
2827 71 : op = gen_lowpart (SFmode, t1);
2828 : }
2829 71 : *pop0 = op;
2830 71 : op = gen_lowpart (HImode, op1);
2831 71 : if (CONST_INT_P (op))
2832 6 : op = simplify_const_unary_operation (FLOAT_EXTEND, SFmode,
2833 : op1, BFmode);
2834 : else
2835 : {
2836 65 : rtx t1 = gen_reg_rtx (SImode);
2837 65 : emit_insn (gen_zero_extendhisi2 (t1, op));
2838 65 : emit_insn (gen_ashlsi3 (t1, t1, GEN_INT (16)));
2839 65 : op = gen_lowpart (SFmode, t1);
2840 : }
2841 71 : *pop1 = op;
2842 71 : return ix86_prepare_fp_compare_args (code, pop0, pop1);
2843 : }
2844 :
2845 : /* All of the unordered compare instructions only work on registers.
2846 : The same is true of the fcomi compare instructions. The XFmode
2847 : compare instructions require registers except when comparing
2848 : against zero or when converting operand 1 from fixed point to
2849 : floating point. */
2850 :
2851 580767 : if (!is_sse
2852 580767 : && (unordered_compare
2853 8187 : || (op_mode == XFmode
2854 10477 : && ! (standard_80387_constant_p (op0) == 1
2855 5235 : || standard_80387_constant_p (op1) == 1)
2856 4792 : && GET_CODE (op1) != FLOAT)
2857 3395 : || ix86_fp_comparison_strategy (code) == IX86_FPCMP_COMI))
2858 : {
2859 148436 : op0 = force_reg (op_mode, op0);
2860 148436 : op1 = force_reg (op_mode, op1);
2861 : }
2862 : else
2863 : {
2864 : /* %%% We only allow op1 in memory; op0 must be st(0). So swap
2865 : things around if they appear profitable, otherwise force op0
2866 : into a register. */
2867 :
2868 432331 : if (standard_80387_constant_p (op0) == 0
2869 432331 : || (MEM_P (op0)
2870 58270 : && ! (standard_80387_constant_p (op1) == 0
2871 42627 : || MEM_P (op1))))
2872 : {
2873 38674 : enum rtx_code new_code = ix86_fp_swap_condition (code);
2874 38674 : if (new_code != UNKNOWN)
2875 : {
2876 : std::swap (op0, op1);
2877 432331 : code = new_code;
2878 : }
2879 : }
2880 :
2881 432331 : if (!REG_P (op0))
2882 54144 : op0 = force_reg (op_mode, op0);
2883 :
2884 432331 : if (CONSTANT_P (op1))
2885 : {
2886 195270 : int tmp = standard_80387_constant_p (op1);
2887 195270 : if (tmp == 0)
2888 75235 : op1 = validize_mem (force_const_mem (op_mode, op1));
2889 120035 : else if (tmp == 1)
2890 : {
2891 65859 : if (TARGET_CMOVE)
2892 65859 : op1 = force_reg (op_mode, op1);
2893 : }
2894 : else
2895 54176 : op1 = force_reg (op_mode, op1);
2896 : }
2897 : }
2898 :
2899 : /* Try to rearrange the comparison to make it cheaper. */
2900 580767 : if (ix86_fp_comparison_cost (code)
2901 580767 : > ix86_fp_comparison_cost (swap_condition (code))
2902 580767 : && (REG_P (op1) || can_create_pseudo_p ()))
2903 : {
2904 0 : std::swap (op0, op1);
2905 0 : code = swap_condition (code);
2906 0 : if (!REG_P (op0))
2907 0 : op0 = force_reg (op_mode, op0);
2908 : }
2909 :
2910 580767 : *pop0 = op0;
2911 580767 : *pop1 = op1;
2912 580767 : return code;
2913 : }
2914 :
2915 : /* Generate insn patterns to do a floating point compare of OPERANDS. */
2916 :
2917 : static rtx
2918 580767 : ix86_expand_fp_compare (enum rtx_code code, rtx op0, rtx op1)
2919 : {
2920 580767 : bool unordered_compare = ix86_unordered_fp_compare (code);
2921 580767 : machine_mode cmp_mode;
2922 580767 : rtx tmp, scratch;
2923 :
2924 580767 : code = ix86_prepare_fp_compare_args (code, &op0, &op1);
2925 580767 : machine_mode op_mode = GET_MODE (op0);
2926 :
2927 580767 : tmp = gen_rtx_COMPARE (CCFPmode, op0, op1);
2928 580767 : if (unordered_compare)
2929 505269 : tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_NOTRAP);
2930 :
2931 : /* Do fcomi/sahf based test when profitable. */
2932 580767 : switch (ix86_fp_comparison_strategy (code))
2933 : {
2934 580767 : case IX86_FPCMP_COMI:
2935 580767 : tmp = gen_rtx_COMPARE (CCFPmode, op0, op1);
2936 : /* VCOMX/VUCOMX only have DF/SF/HF mode instructions. */
2937 580767 : if (TARGET_AVX10_2
2938 1792 : && (code == EQ || code == NE)
2939 982 : && (op_mode == HFmode || op_mode == SFmode || op_mode == DFmode))
2940 972 : tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_OPTCOMX);
2941 : /* We only have vcomisbf16, No vcomubf16 nor vcomxbf16 */
2942 580767 : if (op_mode != BFmode && unordered_compare)
2943 505261 : tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_NOTRAP);
2944 580767 : cmp_mode = CCFPmode;
2945 580767 : emit_insn (gen_rtx_SET (gen_rtx_REG (CCFPmode, FLAGS_REG), tmp));
2946 580767 : break;
2947 :
2948 0 : case IX86_FPCMP_SAHF:
2949 0 : cmp_mode = CCFPmode;
2950 0 : tmp = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW);
2951 0 : scratch = gen_reg_rtx (HImode);
2952 0 : emit_insn (gen_rtx_SET (scratch, tmp));
2953 0 : emit_insn (gen_x86_sahf_1 (scratch));
2954 0 : break;
2955 :
2956 0 : case IX86_FPCMP_ARITH:
2957 0 : cmp_mode = CCNOmode;
2958 0 : tmp = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW);
2959 0 : scratch = gen_reg_rtx (HImode);
2960 0 : emit_insn (gen_rtx_SET (scratch, tmp));
2961 :
2962 : /* In the unordered case, we have to check C2 for NaN's, which
2963 : doesn't happen to work out to anything nice combination-wise.
2964 : So do some bit twiddling on the value we've got in AH to come
2965 : up with an appropriate set of condition codes. */
2966 :
2967 0 : switch (code)
2968 : {
2969 0 : case GT:
2970 0 : case UNGT:
2971 0 : if (code == GT || !TARGET_IEEE_FP)
2972 : {
2973 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x45)));
2974 0 : code = EQ;
2975 : }
2976 : else
2977 : {
2978 0 : emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
2979 0 : emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
2980 0 : emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x44)));
2981 0 : cmp_mode = CCmode;
2982 0 : code = GEU;
2983 : }
2984 : break;
2985 0 : case LT:
2986 0 : case UNLT:
2987 0 : if (code == LT && TARGET_IEEE_FP)
2988 : {
2989 0 : emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
2990 0 : emit_insn (gen_cmpqi_ext_3 (scratch, const1_rtx));
2991 0 : cmp_mode = CCmode;
2992 0 : code = EQ;
2993 : }
2994 : else
2995 : {
2996 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, const1_rtx));
2997 0 : code = NE;
2998 : }
2999 : break;
3000 0 : case GE:
3001 0 : case UNGE:
3002 0 : if (code == GE || !TARGET_IEEE_FP)
3003 : {
3004 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x05)));
3005 0 : code = EQ;
3006 : }
3007 : else
3008 : {
3009 0 : emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
3010 0 : emit_insn (gen_xorqi_ext_1_cc (scratch, scratch, const1_rtx));
3011 0 : code = NE;
3012 : }
3013 : break;
3014 0 : case LE:
3015 0 : case UNLE:
3016 0 : if (code == LE && TARGET_IEEE_FP)
3017 : {
3018 0 : emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
3019 0 : emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
3020 0 : emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
3021 0 : cmp_mode = CCmode;
3022 0 : code = LTU;
3023 : }
3024 : else
3025 : {
3026 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x45)));
3027 0 : code = NE;
3028 : }
3029 : break;
3030 0 : case EQ:
3031 0 : case UNEQ:
3032 0 : if (code == EQ && TARGET_IEEE_FP)
3033 : {
3034 0 : emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
3035 0 : emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
3036 0 : cmp_mode = CCmode;
3037 0 : code = EQ;
3038 : }
3039 : else
3040 : {
3041 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x40)));
3042 0 : code = NE;
3043 : }
3044 : break;
3045 0 : case NE:
3046 0 : case LTGT:
3047 0 : if (code == NE && TARGET_IEEE_FP)
3048 : {
3049 0 : emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
3050 0 : emit_insn (gen_xorqi_ext_1_cc (scratch, scratch,
3051 : GEN_INT (0x40)));
3052 0 : code = NE;
3053 : }
3054 : else
3055 : {
3056 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x40)));
3057 0 : code = EQ;
3058 : }
3059 : break;
3060 :
3061 0 : case UNORDERED:
3062 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x04)));
3063 0 : code = NE;
3064 0 : break;
3065 0 : case ORDERED:
3066 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x04)));
3067 0 : code = EQ;
3068 0 : break;
3069 :
3070 0 : default:
3071 0 : gcc_unreachable ();
3072 : }
3073 : break;
3074 :
3075 0 : default:
3076 0 : gcc_unreachable();
3077 : }
3078 :
3079 : /* Return the test that should be put into the flags user, i.e.
3080 : the bcc, scc, or cmov instruction. */
3081 580767 : return gen_rtx_fmt_ee (code, VOIDmode,
3082 : gen_rtx_REG (cmp_mode, FLAGS_REG),
3083 : const0_rtx);
3084 : }
3085 :
3086 : /* Generate insn patterns to do an integer compare of OPERANDS. */
3087 :
3088 : static rtx
3089 7101898 : ix86_expand_int_compare (enum rtx_code code, rtx op0, rtx op1)
3090 : {
3091 7101898 : machine_mode cmpmode;
3092 7101898 : rtx tmp, flags;
3093 :
3094 : /* Swap operands to emit carry flag comparison. */
3095 7101898 : if ((code == GTU || code == LEU)
3096 7101898 : && nonimmediate_operand (op1, VOIDmode))
3097 : {
3098 150878 : std::swap (op0, op1);
3099 150878 : code = swap_condition (code);
3100 : }
3101 :
3102 7101898 : cmpmode = SELECT_CC_MODE (code, op0, op1);
3103 7101898 : flags = gen_rtx_REG (cmpmode, FLAGS_REG);
3104 :
3105 : /* Attempt to use PTEST, if available, when testing vector modes for
3106 : equality/inequality against zero. */
3107 7101898 : if (op1 == const0_rtx
3108 2950476 : && SUBREG_P (op0)
3109 24290 : && cmpmode == CCZmode
3110 11289 : && SUBREG_BYTE (op0) == 0
3111 9607 : && REG_P (SUBREG_REG (op0))
3112 9607 : && VECTOR_MODE_P (GET_MODE (SUBREG_REG (op0)))
3113 8 : && TARGET_SSE4_1
3114 2 : && GET_MODE (op0) == TImode
3115 7101902 : && GET_MODE_SIZE (GET_MODE (SUBREG_REG (op0))) == 16)
3116 : {
3117 2 : tmp = SUBREG_REG (op0);
3118 2 : if (GET_MODE (tmp) == V8HFmode || GET_MODE (tmp) == V8BFmode)
3119 1 : tmp = gen_lowpart (V8HImode, tmp);
3120 2 : tmp = gen_rtx_UNSPEC (CCZmode, gen_rtvec (2, tmp, tmp), UNSPEC_PTEST);
3121 : }
3122 : else
3123 7101896 : tmp = gen_rtx_COMPARE (cmpmode, op0, op1);
3124 :
3125 : /* This is very simple, but making the interface the same as in the
3126 : FP case makes the rest of the code easier. */
3127 7101898 : emit_insn (gen_rtx_SET (flags, tmp));
3128 :
3129 : /* Return the test that should be put into the flags user, i.e.
3130 : the bcc, scc, or cmov instruction. */
3131 7101898 : return gen_rtx_fmt_ee (code, VOIDmode, flags, const0_rtx);
3132 : }
3133 :
3134 : static rtx
3135 7814227 : ix86_expand_compare (enum rtx_code code, rtx op0, rtx op1)
3136 : {
3137 7814227 : rtx ret;
3138 :
3139 7814227 : if (GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC)
3140 133494 : ret = gen_rtx_fmt_ee (code, VOIDmode, op0, op1);
3141 :
3142 7680733 : else if (SCALAR_FLOAT_MODE_P (GET_MODE (op0)))
3143 : {
3144 578835 : gcc_assert (!DECIMAL_FLOAT_MODE_P (GET_MODE (op0)));
3145 578835 : ret = ix86_expand_fp_compare (code, op0, op1);
3146 : }
3147 : else
3148 7101898 : ret = ix86_expand_int_compare (code, op0, op1);
3149 :
3150 7814227 : return ret;
3151 : }
3152 :
3153 : void
3154 638869 : ix86_expand_setcc (rtx dest, enum rtx_code code, rtx op0, rtx op1)
3155 : {
3156 638869 : rtx ret;
3157 :
3158 638869 : gcc_assert (GET_MODE (dest) == QImode);
3159 :
3160 638869 : ret = ix86_expand_compare (code, op0, op1);
3161 638869 : PUT_MODE (ret, QImode);
3162 638869 : emit_insn (gen_rtx_SET (dest, ret));
3163 638869 : }
3164 :
3165 : /* Expand floating point op0 <=> op1, i.e.
3166 : dest = op0 == op1 ? 0 : op0 < op1 ? -1 : op0 > op1 ? 1 : -128. */
3167 :
3168 : void
3169 176 : ix86_expand_fp_spaceship (rtx dest, rtx op0, rtx op1, rtx op2)
3170 : {
3171 176 : gcc_checking_assert (ix86_fp_comparison_strategy (GT) != IX86_FPCMP_ARITH);
3172 176 : rtx zero = NULL_RTX;
3173 176 : if (op2 != const0_rtx
3174 52 : && (TARGET_IEEE_FP || TARGET_ZERO_EXTEND_WITH_AND)
3175 34 : && GET_MODE (dest) == SImode)
3176 34 : zero = force_reg (SImode, const0_rtx);
3177 176 : rtx gt = ix86_expand_fp_compare (GT, op0, op1);
3178 176 : rtx l0 = op2 == const0_rtx ? gen_label_rtx () : NULL_RTX;
3179 176 : rtx l1 = op2 == const0_rtx ? gen_label_rtx () : NULL_RTX;
3180 176 : rtx l2 = TARGET_IEEE_FP ? gen_label_rtx () : NULL_RTX;
3181 176 : rtx lend = gen_label_rtx ();
3182 176 : rtx tmp;
3183 176 : rtx_insn *jmp;
3184 176 : if (l2)
3185 : {
3186 139 : rtx un = gen_rtx_fmt_ee (UNORDERED, VOIDmode,
3187 : gen_rtx_REG (CCFPmode, FLAGS_REG), const0_rtx);
3188 139 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, un,
3189 : gen_rtx_LABEL_REF (VOIDmode, l2), pc_rtx);
3190 139 : jmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
3191 139 : add_reg_br_prob_note (jmp, profile_probability:: very_unlikely ());
3192 : }
3193 176 : if (op2 == const0_rtx)
3194 : {
3195 124 : rtx eq = gen_rtx_fmt_ee (UNEQ, VOIDmode,
3196 : gen_rtx_REG (CCFPmode, FLAGS_REG), const0_rtx);
3197 124 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, eq,
3198 : gen_rtx_LABEL_REF (VOIDmode, l0), pc_rtx);
3199 124 : jmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
3200 124 : add_reg_br_prob_note (jmp, profile_probability::unlikely ());
3201 124 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, gt,
3202 : gen_rtx_LABEL_REF (VOIDmode, l1), pc_rtx);
3203 124 : jmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
3204 124 : add_reg_br_prob_note (jmp, profile_probability::even ());
3205 124 : emit_move_insn (dest, constm1_rtx);
3206 124 : emit_jump (lend);
3207 124 : emit_label (l0);
3208 124 : emit_move_insn (dest, const0_rtx);
3209 124 : emit_jump (lend);
3210 124 : emit_label (l1);
3211 124 : emit_move_insn (dest, const1_rtx);
3212 : }
3213 : else
3214 : {
3215 52 : rtx lt_tmp = NULL_RTX;
3216 52 : if (GET_MODE (dest) != SImode || !TARGET_ZERO_EXTEND_WITH_AND)
3217 : {
3218 52 : lt_tmp = gen_reg_rtx (QImode);
3219 52 : ix86_expand_setcc (lt_tmp, UNLT, gen_rtx_REG (CCFPmode, FLAGS_REG),
3220 : const0_rtx);
3221 52 : if (GET_MODE (dest) != QImode)
3222 : {
3223 52 : tmp = gen_reg_rtx (GET_MODE (dest));
3224 52 : emit_insn (gen_rtx_SET (tmp,
3225 : gen_rtx_ZERO_EXTEND (GET_MODE (dest),
3226 : lt_tmp)));
3227 52 : lt_tmp = tmp;
3228 : }
3229 : }
3230 52 : rtx gt_tmp;
3231 52 : if (zero)
3232 : {
3233 : /* If TARGET_IEEE_FP and dest has SImode, emit SImode clear
3234 : before the floating point comparison and use setcc_si_slp
3235 : pattern to hide it from the combiner, so that it doesn't
3236 : undo it. Similarly for TARGET_ZERO_EXTEND_WITH_AND, where
3237 : the ZERO_EXTEND normally emitted would need to be AND
3238 : with flags clobber. */
3239 34 : tmp = ix86_expand_compare (GT, XEXP (gt, 0), const0_rtx);
3240 34 : PUT_MODE (tmp, QImode);
3241 34 : emit_insn (gen_setcc_si_slp (zero, tmp, zero));
3242 34 : gt_tmp = zero;
3243 : }
3244 : else
3245 : {
3246 18 : gt_tmp = gen_reg_rtx (QImode);
3247 18 : ix86_expand_setcc (gt_tmp, GT, XEXP (gt, 0), const0_rtx);
3248 18 : if (GET_MODE (dest) != QImode)
3249 : {
3250 18 : tmp = gen_reg_rtx (GET_MODE (dest));
3251 18 : emit_insn (gen_rtx_SET (tmp,
3252 : gen_rtx_ZERO_EXTEND (GET_MODE (dest),
3253 : gt_tmp)));
3254 18 : gt_tmp = tmp;
3255 : }
3256 : }
3257 52 : if (lt_tmp)
3258 : {
3259 52 : tmp = expand_simple_binop (GET_MODE (dest), MINUS, gt_tmp, lt_tmp,
3260 : dest, 0, OPTAB_DIRECT);
3261 52 : if (!rtx_equal_p (tmp, dest))
3262 0 : emit_move_insn (dest, tmp);
3263 : }
3264 : else
3265 : {
3266 : /* For TARGET_ZERO_EXTEND_WITH_AND emit sbb directly, as we can't
3267 : do ZERO_EXTEND without clobbering flags. */
3268 0 : tmp = ix86_expand_compare (UNLT, XEXP (gt, 0), const0_rtx);
3269 0 : PUT_MODE (tmp, SImode);
3270 0 : emit_insn (gen_subsi3_carry (dest, gt_tmp,
3271 0 : force_reg (GET_MODE (dest), const0_rtx),
3272 : XEXP (gt, 0), tmp));
3273 : }
3274 : }
3275 176 : emit_jump (lend);
3276 176 : if (l2)
3277 : {
3278 139 : emit_label (l2);
3279 139 : emit_move_insn (dest, op2 == const0_rtx ? GEN_INT (-128) : op2);
3280 : }
3281 176 : emit_label (lend);
3282 176 : }
3283 :
3284 : /* Expand integral op0 <=> op1, i.e.
3285 : dest = op0 == op1 ? 0 : op0 < op1 ? -1 : 1. */
3286 :
3287 : void
3288 164 : ix86_expand_int_spaceship (rtx dest, rtx op0, rtx op1, rtx op2)
3289 : {
3290 164 : gcc_assert (INTVAL (op2));
3291 164 : rtx zero1 = NULL_RTX, zero2 = NULL_RTX;
3292 164 : if (TARGET_ZERO_EXTEND_WITH_AND && GET_MODE (dest) == SImode)
3293 : {
3294 0 : zero1 = force_reg (SImode, const0_rtx);
3295 0 : if (INTVAL (op2) != 1)
3296 0 : zero2 = force_reg (SImode, const0_rtx);
3297 : }
3298 :
3299 : /* Not using ix86_expand_int_compare here, so that it doesn't swap
3300 : operands nor optimize CC mode - we need a mode usable for both
3301 : LT and GT resp. LTU and GTU comparisons with the same unswapped
3302 : operands. */
3303 204 : rtx flags = gen_rtx_REG (INTVAL (op2) != 1 ? CCGCmode : CCmode, FLAGS_REG);
3304 164 : rtx tmp = gen_rtx_COMPARE (GET_MODE (flags), op0, op1);
3305 164 : emit_insn (gen_rtx_SET (flags, tmp));
3306 164 : rtx lt_tmp = NULL_RTX;
3307 164 : if (zero2)
3308 : {
3309 : /* For TARGET_ZERO_EXTEND_WITH_AND, emit setcc_si_slp to avoid
3310 : ZERO_EXTEND. */
3311 0 : tmp = ix86_expand_compare (LT, flags, const0_rtx);
3312 0 : PUT_MODE (tmp, QImode);
3313 0 : emit_insn (gen_setcc_si_slp (zero2, tmp, zero2));
3314 0 : lt_tmp = zero2;
3315 : }
3316 164 : else if (!zero1)
3317 : {
3318 164 : lt_tmp = gen_reg_rtx (QImode);
3319 204 : ix86_expand_setcc (lt_tmp, INTVAL (op2) != 1 ? LT : LTU, flags,
3320 : const0_rtx);
3321 164 : if (GET_MODE (dest) != QImode)
3322 : {
3323 164 : tmp = gen_reg_rtx (GET_MODE (dest));
3324 164 : emit_insn (gen_rtx_SET (tmp, gen_rtx_ZERO_EXTEND (GET_MODE (dest),
3325 : lt_tmp)));
3326 164 : lt_tmp = tmp;
3327 : }
3328 : }
3329 164 : rtx gt_tmp;
3330 164 : if (zero1)
3331 : {
3332 : /* For TARGET_ZERO_EXTEND_WITH_AND, emit setcc_si_slp to avoid
3333 : ZERO_EXTEND. */
3334 0 : tmp = ix86_expand_compare (INTVAL (op2) != 1 ? GT : GTU, flags,
3335 : const0_rtx);
3336 0 : PUT_MODE (tmp, QImode);
3337 0 : emit_insn (gen_setcc_si_slp (zero1, tmp, zero1));
3338 0 : gt_tmp = zero1;
3339 : }
3340 : else
3341 : {
3342 164 : gt_tmp = gen_reg_rtx (QImode);
3343 204 : ix86_expand_setcc (gt_tmp, INTVAL (op2) != 1 ? GT : GTU, flags,
3344 : const0_rtx);
3345 164 : if (GET_MODE (dest) != QImode)
3346 : {
3347 164 : tmp = gen_reg_rtx (GET_MODE (dest));
3348 164 : emit_insn (gen_rtx_SET (tmp, gen_rtx_ZERO_EXTEND (GET_MODE (dest),
3349 : gt_tmp)));
3350 164 : gt_tmp = tmp;
3351 : }
3352 : }
3353 164 : if (lt_tmp)
3354 : {
3355 164 : tmp = expand_simple_binop (GET_MODE (dest), MINUS, gt_tmp, lt_tmp, dest,
3356 : 0, OPTAB_DIRECT);
3357 164 : if (!rtx_equal_p (tmp, dest))
3358 0 : emit_move_insn (dest, tmp);
3359 : }
3360 : else
3361 : {
3362 : /* For TARGET_ZERO_EXTEND_WITH_AND emit sbb directly, as we can't
3363 : do ZERO_EXTEND without clobbering flags. */
3364 0 : tmp = ix86_expand_compare (LTU, flags, const0_rtx);
3365 0 : PUT_MODE (tmp, SImode);
3366 0 : emit_insn (gen_subsi3_carry (dest, gt_tmp,
3367 0 : force_reg (GET_MODE (dest), const0_rtx),
3368 : flags, tmp));
3369 : }
3370 164 : }
3371 :
3372 : /* Expand comparison setting or clearing carry flag. Return true when
3373 : successful and set pop for the operation. */
3374 : static bool
3375 23898 : ix86_expand_carry_flag_compare (enum rtx_code code, rtx op0, rtx op1, rtx *pop)
3376 : {
3377 47796 : machine_mode mode
3378 23898 : = GET_MODE (op0) != VOIDmode ? GET_MODE (op0) : GET_MODE (op1);
3379 :
3380 : /* Do not handle double-mode compares that go through special path. */
3381 26094 : if (mode == (TARGET_64BIT ? TImode : DImode))
3382 : return false;
3383 :
3384 23890 : if (SCALAR_FLOAT_MODE_P (mode))
3385 : {
3386 1758 : rtx compare_op;
3387 1758 : rtx_insn *compare_seq;
3388 :
3389 1758 : gcc_assert (!DECIMAL_FLOAT_MODE_P (mode));
3390 :
3391 : /* Shortcut: following common codes never translate
3392 : into carry flag compares. */
3393 1758 : if (code == EQ || code == NE || code == UNEQ || code == LTGT
3394 : || code == ORDERED || code == UNORDERED)
3395 : return false;
3396 :
3397 : /* These comparisons require zero flag; swap operands so they won't. */
3398 : if ((code == GT || code == UNLE || code == LE || code == UNGT)
3399 1690 : && !TARGET_IEEE_FP)
3400 : {
3401 2 : std::swap (op0, op1);
3402 2 : code = swap_condition (code);
3403 : }
3404 :
3405 : /* Try to expand the comparison and verify that we end up with
3406 : carry flag based comparison. This fails to be true only when
3407 : we decide to expand comparison using arithmetic that is not
3408 : too common scenario. */
3409 1756 : start_sequence ();
3410 1756 : compare_op = ix86_expand_fp_compare (code, op0, op1);
3411 1756 : compare_seq = end_sequence ();
3412 :
3413 1756 : if (GET_MODE (XEXP (compare_op, 0)) == CCFPmode)
3414 1756 : code = ix86_fp_compare_code_to_integer (GET_CODE (compare_op));
3415 : else
3416 0 : code = GET_CODE (compare_op);
3417 :
3418 1756 : if (code != LTU && code != GEU)
3419 : return false;
3420 :
3421 66 : emit_insn (compare_seq);
3422 66 : *pop = compare_op;
3423 66 : return true;
3424 : }
3425 :
3426 22132 : if (!INTEGRAL_MODE_P (mode))
3427 : return false;
3428 :
3429 22061 : switch (code)
3430 : {
3431 : case LTU:
3432 : case GEU:
3433 : break;
3434 :
3435 : /* Convert a==0 into (unsigned)a<1. */
3436 18104 : case EQ:
3437 18104 : case NE:
3438 18104 : if (op1 != const0_rtx)
3439 : return false;
3440 9635 : op1 = const1_rtx;
3441 9635 : code = (code == EQ ? LTU : GEU);
3442 : break;
3443 :
3444 : /* Convert a>b into b<a or a>=b-1. */
3445 916 : case GTU:
3446 916 : case LEU:
3447 916 : if (CONST_INT_P (op1))
3448 : {
3449 878 : op1 = gen_int_mode (INTVAL (op1) + 1, GET_MODE (op0));
3450 : /* Bail out on overflow. We still can swap operands but that
3451 : would force loading of the constant into register. */
3452 878 : if (op1 == const0_rtx
3453 878 : || !x86_64_immediate_operand (op1, GET_MODE (op1)))
3454 : return false;
3455 878 : code = (code == GTU ? GEU : LTU);
3456 : }
3457 : else
3458 : {
3459 38 : std::swap (op0, op1);
3460 38 : code = (code == GTU ? LTU : GEU);
3461 : }
3462 : break;
3463 :
3464 : /* Convert a>=0 into (unsigned)a<0x80000000. */
3465 1435 : case LT:
3466 1435 : case GE:
3467 1435 : if (mode == DImode || op1 != const0_rtx)
3468 : return false;
3469 100 : op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
3470 50 : code = (code == LT ? GEU : LTU);
3471 : break;
3472 961 : case LE:
3473 961 : case GT:
3474 961 : if (mode == DImode || op1 != constm1_rtx)
3475 : return false;
3476 0 : op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
3477 0 : code = (code == LE ? GEU : LTU);
3478 : break;
3479 :
3480 : default:
3481 : return false;
3482 : }
3483 : /* Swapping operands may cause constant to appear as first operand. */
3484 11246 : if (!nonimmediate_operand (op0, VOIDmode))
3485 : {
3486 0 : if (!can_create_pseudo_p ())
3487 : return false;
3488 0 : op0 = force_reg (mode, op0);
3489 : }
3490 11246 : *pop = ix86_expand_compare (code, op0, op1);
3491 11246 : gcc_assert (GET_CODE (*pop) == LTU || GET_CODE (*pop) == GEU);
3492 : return true;
3493 : }
3494 :
3495 : /* Expand conditional increment or decrement using adb/sbb instructions.
3496 : The default case using setcc followed by the conditional move can be
3497 : done by generic code. */
3498 : bool
3499 2824 : ix86_expand_int_addcc (rtx operands[])
3500 : {
3501 2824 : enum rtx_code code = GET_CODE (operands[1]);
3502 2824 : rtx flags;
3503 2824 : rtx (*insn) (machine_mode, rtx, rtx, rtx, rtx, rtx);
3504 2824 : rtx compare_op;
3505 2824 : rtx val = const0_rtx;
3506 2824 : bool fpcmp = false;
3507 2824 : machine_mode mode;
3508 2824 : rtx op0 = XEXP (operands[1], 0);
3509 2824 : rtx op1 = XEXP (operands[1], 1);
3510 :
3511 2824 : if (operands[3] != const1_rtx
3512 2412 : && operands[3] != constm1_rtx)
3513 : return false;
3514 1153 : if (!ix86_expand_carry_flag_compare (code, op0, op1, &compare_op))
3515 : return false;
3516 328 : code = GET_CODE (compare_op);
3517 :
3518 328 : flags = XEXP (compare_op, 0);
3519 :
3520 328 : if (GET_MODE (flags) == CCFPmode)
3521 : {
3522 1 : fpcmp = true;
3523 1 : code = ix86_fp_compare_code_to_integer (code);
3524 : }
3525 :
3526 328 : if (code != LTU)
3527 : {
3528 172 : val = constm1_rtx;
3529 172 : if (fpcmp)
3530 1 : PUT_CODE (compare_op,
3531 : reverse_condition_maybe_unordered
3532 : (GET_CODE (compare_op)));
3533 : else
3534 171 : PUT_CODE (compare_op, reverse_condition (GET_CODE (compare_op)));
3535 : }
3536 :
3537 328 : mode = GET_MODE (operands[0]);
3538 :
3539 : /* Construct either adc or sbb insn. */
3540 328 : if ((code == LTU) == (operands[3] == constm1_rtx))
3541 : insn = gen_sub3_carry;
3542 : else
3543 131 : insn = gen_add3_carry;
3544 :
3545 328 : emit_insn (insn (mode, operands[0], operands[2], val, flags, compare_op));
3546 :
3547 328 : return true;
3548 : }
3549 :
3550 : bool
3551 425741 : ix86_expand_int_movcc (rtx operands[])
3552 : {
3553 425741 : enum rtx_code code = GET_CODE (operands[1]), compare_code;
3554 425741 : rtx_insn *compare_seq;
3555 425741 : rtx compare_op;
3556 425741 : machine_mode mode = GET_MODE (operands[0]);
3557 425741 : bool sign_bit_compare_p = false;
3558 425741 : bool negate_cc_compare_p = false;
3559 425741 : rtx op0 = XEXP (operands[1], 0);
3560 425741 : rtx op1 = XEXP (operands[1], 1);
3561 425741 : rtx op2 = operands[2];
3562 425741 : rtx op3 = operands[3];
3563 :
3564 425741 : if (GET_MODE (op0) == TImode
3565 410279 : || (GET_MODE (op0) == DImode
3566 104891 : && !TARGET_64BIT))
3567 : return false;
3568 :
3569 409093 : if (GET_MODE (op0) == BFmode
3570 409093 : && !ix86_fp_comparison_operator (operands[1], VOIDmode))
3571 : return false;
3572 :
3573 409093 : start_sequence ();
3574 409093 : compare_op = ix86_expand_compare (code, op0, op1);
3575 409093 : compare_seq = end_sequence ();
3576 :
3577 409093 : compare_code = GET_CODE (compare_op);
3578 :
3579 409093 : if ((op1 == const0_rtx && (code == GE || code == LT))
3580 367958 : || (op1 == constm1_rtx && (code == GT || code == LE)))
3581 : sign_bit_compare_p = true;
3582 :
3583 : /* op0 == op1 ? op0 : op3 is equivalent to op0 == op1 ? op1 : op3,
3584 : but if op1 is a constant, the latter form allows more optimizations,
3585 : either through the last 2 ops being constant handling, or the one
3586 : constant and one variable cases. On the other side, for cmov the
3587 : former might be better as we don't need to load the constant into
3588 : another register. */
3589 367958 : if (code == EQ && CONST_INT_P (op1) && rtx_equal_p (op0, op2))
3590 : op2 = op1;
3591 : /* Similarly for op0 != op1 ? op2 : op0 and op0 != op1 ? op2 : op1. */
3592 408593 : else if (code == NE && CONST_INT_P (op1) && rtx_equal_p (op0, op3))
3593 : op3 = op1;
3594 :
3595 : /* Don't attempt mode expansion here -- if we had to expand 5 or 6
3596 : HImode insns, we'd be swallowed in word prefix ops. */
3597 :
3598 4798 : if ((mode != HImode || TARGET_FAST_PREFIX)
3599 437195 : && (mode != (TARGET_64BIT ? TImode : DImode))
3600 409093 : && CONST_INT_P (op2)
3601 439516 : && CONST_INT_P (op3))
3602 : {
3603 23715 : rtx out = operands[0];
3604 23715 : HOST_WIDE_INT ct = INTVAL (op2);
3605 23715 : HOST_WIDE_INT cf = INTVAL (op3);
3606 23715 : HOST_WIDE_INT diff;
3607 :
3608 23715 : if ((mode == SImode
3609 8822 : || (TARGET_64BIT && mode == DImode))
3610 20160 : && (GET_MODE (op0) == SImode
3611 14928 : || (TARGET_64BIT && GET_MODE (op0) == DImode)))
3612 : {
3613 : /* Special case x != 0 ? -1 : y. */
3614 15254 : if (code == NE && op1 == const0_rtx && ct == -1)
3615 : {
3616 : negate_cc_compare_p = true;
3617 : std::swap (ct, cf);
3618 : code = EQ;
3619 : }
3620 15136 : else if (code == EQ && op1 == const0_rtx && cf == -1)
3621 23715 : negate_cc_compare_p = true;
3622 : }
3623 :
3624 23715 : diff = (unsigned HOST_WIDE_INT) ct - cf;
3625 : /* Make sure we can represent the difference between the two values. */
3626 23715 : if ((diff > 0) != ((cf < 0) != (ct < 0) ? cf < 0 : cf < ct))
3627 425741 : return false;
3628 :
3629 : /* Sign bit compares are better done using shifts than we do by using
3630 : sbb. */
3631 23637 : if (sign_bit_compare_p
3632 23637 : || negate_cc_compare_p
3633 23637 : || ix86_expand_carry_flag_compare (code, op0, op1, &compare_op))
3634 : {
3635 : /* Detect overlap between destination and compare sources. */
3636 11876 : rtx tmp = out;
3637 :
3638 11876 : if (negate_cc_compare_p)
3639 : {
3640 291 : if (GET_MODE (op0) == DImode)
3641 115 : emit_insn (gen_x86_negdi_ccc (gen_reg_rtx (DImode), op0));
3642 : else
3643 176 : emit_insn (gen_x86_negsi_ccc (gen_reg_rtx (SImode),
3644 176 : gen_lowpart (SImode, op0)));
3645 :
3646 291 : tmp = gen_reg_rtx (mode);
3647 291 : if (mode == DImode)
3648 135 : emit_insn (gen_x86_movdicc_0_m1_neg (tmp));
3649 : else
3650 156 : emit_insn (gen_x86_movsicc_0_m1_neg (gen_lowpart (SImode,
3651 : tmp)));
3652 : }
3653 11585 : else if (!sign_bit_compare_p)
3654 : {
3655 10984 : rtx flags;
3656 10984 : bool fpcmp = false;
3657 :
3658 10984 : compare_code = GET_CODE (compare_op);
3659 :
3660 10984 : flags = XEXP (compare_op, 0);
3661 :
3662 10984 : if (GET_MODE (flags) == CCFPmode)
3663 : {
3664 65 : fpcmp = true;
3665 65 : compare_code
3666 65 : = ix86_fp_compare_code_to_integer (compare_code);
3667 : }
3668 :
3669 : /* To simplify rest of code, restrict to the GEU case. */
3670 10984 : if (compare_code == LTU)
3671 : {
3672 6570 : std::swap (ct, cf);
3673 6570 : compare_code = reverse_condition (compare_code);
3674 6570 : code = reverse_condition (code);
3675 : }
3676 : else
3677 : {
3678 4414 : if (fpcmp)
3679 65 : PUT_CODE (compare_op,
3680 : reverse_condition_maybe_unordered
3681 : (GET_CODE (compare_op)));
3682 : else
3683 4349 : PUT_CODE (compare_op,
3684 : reverse_condition (GET_CODE (compare_op)));
3685 : }
3686 :
3687 10984 : diff = (unsigned HOST_WIDE_INT) ct - cf;
3688 : /* Make sure we can represent the difference
3689 : between the two values. */
3690 10984 : if ((diff > 0) != ((cf < 0) != (ct < 0) ? cf < 0 : cf < ct))
3691 : return false;
3692 :
3693 10983 : if (reg_overlap_mentioned_p (out, compare_op))
3694 0 : tmp = gen_reg_rtx (mode);
3695 :
3696 10983 : if (mode == DImode)
3697 2098 : emit_insn (gen_x86_movdicc_0_m1 (tmp, flags, compare_op));
3698 : else
3699 8885 : emit_insn (gen_x86_movsicc_0_m1 (gen_lowpart (SImode, tmp),
3700 : flags, compare_op));
3701 : }
3702 : else
3703 : {
3704 601 : if (code == GT || code == GE)
3705 371 : code = reverse_condition (code);
3706 : else
3707 : {
3708 230 : std::swap (ct, cf);
3709 :
3710 230 : diff = (unsigned HOST_WIDE_INT) ct - cf;
3711 : /* Make sure we can represent the difference
3712 : between the two values. */
3713 230 : if ((diff > 0) != ((cf < 0) != (ct < 0) ? cf < 0 : cf < ct))
3714 : return false;
3715 : }
3716 598 : tmp = emit_store_flag (tmp, code, op0, op1, VOIDmode, 0, -1);
3717 : }
3718 :
3719 11872 : if (diff == 1)
3720 : {
3721 : /*
3722 : * cmpl op0,op1
3723 : * sbbl dest,dest
3724 : * [addl dest, ct]
3725 : *
3726 : * Size 5 - 8.
3727 : */
3728 761 : if (ct)
3729 598 : tmp = expand_simple_binop (mode, PLUS,
3730 : tmp, GEN_INT (ct),
3731 : copy_rtx (tmp), 1, OPTAB_DIRECT);
3732 : }
3733 11111 : else if (cf == -1)
3734 : {
3735 : /*
3736 : * cmpl op0,op1
3737 : * sbbl dest,dest
3738 : * orl $ct, dest
3739 : *
3740 : * Size 8.
3741 : */
3742 678 : tmp = expand_simple_binop (mode, IOR,
3743 : tmp, GEN_INT (ct),
3744 : copy_rtx (tmp), 1, OPTAB_DIRECT);
3745 : }
3746 10433 : else if (diff == -1 && ct)
3747 : {
3748 : /*
3749 : * cmpl op0,op1
3750 : * sbbl dest,dest
3751 : * notl dest
3752 : * [addl dest, cf]
3753 : *
3754 : * Size 8 - 11.
3755 : */
3756 500 : tmp = expand_simple_unop (mode, NOT, tmp, copy_rtx (tmp), 1);
3757 500 : if (cf)
3758 481 : tmp = expand_simple_binop (mode, PLUS,
3759 : copy_rtx (tmp), GEN_INT (cf),
3760 : copy_rtx (tmp), 1, OPTAB_DIRECT);
3761 : }
3762 : else
3763 : {
3764 : /*
3765 : * cmpl op0,op1
3766 : * sbbl dest,dest
3767 : * [notl dest]
3768 : * andl cf - ct, dest
3769 : * [addl dest, ct]
3770 : *
3771 : * Size 8 - 11.
3772 : */
3773 :
3774 9933 : if (cf == 0)
3775 : {
3776 1296 : cf = ct;
3777 1296 : ct = 0;
3778 1296 : tmp = expand_simple_unop (mode, NOT, tmp, copy_rtx (tmp), 1);
3779 : }
3780 :
3781 9933 : HOST_WIDE_INT ival = (unsigned HOST_WIDE_INT) cf - ct;
3782 : /* Make sure we can represent the difference
3783 : between the two values. */
3784 9933 : if ((ival > 0) != ((ct < 0) != (cf < 0) ? ct < 0 : ct < cf))
3785 425741 : return false;
3786 :
3787 9933 : tmp = expand_simple_binop (mode, AND,
3788 : copy_rtx (tmp),
3789 9933 : gen_int_mode (ival, mode),
3790 : copy_rtx (tmp), 1, OPTAB_DIRECT);
3791 9933 : if (ct)
3792 7126 : tmp = expand_simple_binop (mode, PLUS,
3793 : copy_rtx (tmp), GEN_INT (ct),
3794 : copy_rtx (tmp), 1, OPTAB_DIRECT);
3795 : }
3796 :
3797 11872 : if (!rtx_equal_p (tmp, out))
3798 454 : emit_move_insn (copy_rtx (out), copy_rtx (tmp));
3799 :
3800 : return true;
3801 : }
3802 :
3803 11761 : if (diff < 0)
3804 : {
3805 3631 : machine_mode cmp_mode = GET_MODE (op0);
3806 3631 : enum rtx_code new_code;
3807 :
3808 3631 : if (SCALAR_FLOAT_MODE_P (cmp_mode))
3809 : {
3810 77 : gcc_assert (!DECIMAL_FLOAT_MODE_P (cmp_mode));
3811 :
3812 : /* We may be reversing a non-trapping
3813 : comparison to a trapping comparison. */
3814 150 : if (HONOR_NANS (cmp_mode) && flag_trapping_math
3815 70 : && code != EQ && code != NE
3816 147 : && code != ORDERED && code != UNORDERED)
3817 : new_code = UNKNOWN;
3818 : else
3819 7 : new_code = reverse_condition_maybe_unordered (code);
3820 : }
3821 : else
3822 3554 : new_code = ix86_reverse_condition (code, cmp_mode);
3823 3561 : if (new_code != UNKNOWN)
3824 : {
3825 3561 : std::swap (ct, cf);
3826 :
3827 3561 : diff = (unsigned HOST_WIDE_INT) ct - cf;
3828 : /* Make sure we can represent the difference
3829 : between the two values. */
3830 3561 : if ((diff > 0) != ((cf < 0) != (ct < 0) ? cf < 0 : cf < ct))
3831 : return false;
3832 :
3833 : code = new_code;
3834 : }
3835 : }
3836 :
3837 11761 : compare_code = UNKNOWN;
3838 11761 : if (GET_MODE_CLASS (GET_MODE (op0)) == MODE_INT
3839 10040 : && CONST_INT_P (op1))
3840 : {
3841 3823 : if (op1 == const0_rtx
3842 221 : && (code == LT || code == GE))
3843 : compare_code = code;
3844 3823 : else if (op1 == constm1_rtx)
3845 : {
3846 102 : if (code == LE)
3847 : compare_code = LT;
3848 102 : else if (code == GT)
3849 : compare_code = GE;
3850 : }
3851 : }
3852 :
3853 : /* Optimize dest = (op0 < 0) ? -1 : cf. */
3854 : if (compare_code != UNKNOWN
3855 0 : && GET_MODE (op0) == GET_MODE (out)
3856 0 : && (cf == -1 || ct == -1))
3857 : {
3858 : /* If lea code below could be used, only optimize
3859 : if it results in a 2 insn sequence. */
3860 :
3861 0 : if (! (diff == 1 || diff == 2 || diff == 4 || diff == 8
3862 0 : || diff == 3 || diff == 5 || diff == 9)
3863 0 : || (compare_code == LT && ct == -1)
3864 0 : || (compare_code == GE && cf == -1))
3865 : {
3866 : /*
3867 : * notl op1 (if necessary)
3868 : * sarl $31, op1
3869 : * orl cf, op1
3870 : */
3871 0 : if (ct != -1)
3872 : {
3873 0 : cf = ct;
3874 0 : ct = -1;
3875 0 : code = reverse_condition (code);
3876 : }
3877 :
3878 0 : out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, -1);
3879 :
3880 0 : out = expand_simple_binop (mode, IOR,
3881 : out, GEN_INT (cf),
3882 : out, 1, OPTAB_DIRECT);
3883 0 : if (out != operands[0])
3884 0 : emit_move_insn (operands[0], out);
3885 :
3886 : return true;
3887 : }
3888 : }
3889 :
3890 :
3891 15818 : if ((diff == 1 || diff == 2 || diff == 4 || diff == 8
3892 4057 : || diff == 3 || diff == 5 || diff == 9)
3893 8248 : && ((mode != QImode && mode != HImode) || !TARGET_PARTIAL_REG_STALL)
3894 20009 : && (mode != DImode
3895 2108 : || x86_64_immediate_operand (GEN_INT (cf), VOIDmode)))
3896 : {
3897 : /*
3898 : * xorl dest,dest
3899 : * cmpl op1,op2
3900 : * setcc dest
3901 : * lea cf(dest*(ct-cf)),dest
3902 : *
3903 : * Size 14.
3904 : *
3905 : * This also catches the degenerate setcc-only case.
3906 : */
3907 :
3908 8248 : rtx tmp;
3909 8248 : int nops;
3910 :
3911 8248 : out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, 1);
3912 :
3913 8248 : nops = 0;
3914 : /* On x86_64 the lea instruction operates on Pmode, so we need
3915 : to get arithmetics done in proper mode to match. */
3916 8248 : if (diff == 1)
3917 6816 : tmp = copy_rtx (out);
3918 : else
3919 : {
3920 1432 : rtx out1;
3921 1432 : out1 = copy_rtx (out);
3922 1432 : tmp = gen_rtx_MULT (mode, out1, GEN_INT (diff & ~1));
3923 1432 : nops++;
3924 1432 : if (diff & 1)
3925 : {
3926 437 : tmp = gen_rtx_PLUS (mode, tmp, out1);
3927 437 : nops++;
3928 : }
3929 : }
3930 8248 : if (cf != 0)
3931 : {
3932 7035 : tmp = plus_constant (mode, tmp, cf);
3933 7035 : nops++;
3934 : }
3935 8248 : if (!rtx_equal_p (tmp, out))
3936 : {
3937 7525 : if (nops == 1)
3938 6274 : out = force_operand (tmp, copy_rtx (out));
3939 : else
3940 1251 : emit_insn (gen_rtx_SET (copy_rtx (out), copy_rtx (tmp)));
3941 : }
3942 8248 : if (!rtx_equal_p (out, operands[0]))
3943 652 : emit_move_insn (operands[0], copy_rtx (out));
3944 :
3945 : return true;
3946 : }
3947 :
3948 : /*
3949 : * General case: Jumpful:
3950 : * xorl dest,dest cmpl op1, op2
3951 : * cmpl op1, op2 movl ct, dest
3952 : * setcc dest jcc 1f
3953 : * decl dest movl cf, dest
3954 : * andl (cf-ct),dest 1:
3955 : * addl ct,dest
3956 : *
3957 : * Size 20. Size 14.
3958 : *
3959 : * This is reasonably steep, but branch mispredict costs are
3960 : * high on modern cpus, so consider failing only if optimizing
3961 : * for space.
3962 : */
3963 :
3964 3513 : if ((!TARGET_CMOVE || (mode == QImode && TARGET_PARTIAL_REG_STALL))
3965 3513 : && BRANCH_COST (optimize_insn_for_speed_p (),
3966 : false) >= 2)
3967 : {
3968 0 : if (cf == 0)
3969 : {
3970 0 : machine_mode cmp_mode = GET_MODE (op0);
3971 0 : enum rtx_code new_code;
3972 :
3973 0 : if (SCALAR_FLOAT_MODE_P (cmp_mode))
3974 : {
3975 0 : gcc_assert (!DECIMAL_FLOAT_MODE_P (cmp_mode));
3976 :
3977 : /* We may be reversing a non-trapping
3978 : comparison to a trapping comparison. */
3979 0 : if (HONOR_NANS (cmp_mode) && flag_trapping_math
3980 0 : && code != EQ && code != NE
3981 0 : && code != ORDERED && code != UNORDERED)
3982 : new_code = UNKNOWN;
3983 : else
3984 0 : new_code = reverse_condition_maybe_unordered (code);
3985 :
3986 : }
3987 : else
3988 : {
3989 0 : new_code = ix86_reverse_condition (code, cmp_mode);
3990 0 : if (compare_code != UNKNOWN && new_code != UNKNOWN)
3991 0 : compare_code = reverse_condition (compare_code);
3992 : }
3993 :
3994 0 : if (new_code != UNKNOWN)
3995 : {
3996 0 : cf = ct;
3997 0 : ct = 0;
3998 0 : code = new_code;
3999 : }
4000 : }
4001 :
4002 0 : if (compare_code != UNKNOWN)
4003 : {
4004 : /* notl op1 (if needed)
4005 : sarl $31, op1
4006 : andl (cf-ct), op1
4007 : addl ct, op1
4008 :
4009 : For x < 0 (resp. x <= -1) there will be no notl,
4010 : so if possible swap the constants to get rid of the
4011 : complement.
4012 : True/false will be -1/0 while code below (store flag
4013 : followed by decrement) is 0/-1, so the constants need
4014 : to be exchanged once more. */
4015 :
4016 0 : if (compare_code == GE || !cf)
4017 : {
4018 0 : code = reverse_condition (code);
4019 0 : compare_code = LT;
4020 : }
4021 : else
4022 : std::swap (ct, cf);
4023 :
4024 0 : out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, -1);
4025 : }
4026 : else
4027 : {
4028 0 : out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, 1);
4029 :
4030 0 : out = expand_simple_binop (mode, PLUS, copy_rtx (out),
4031 : constm1_rtx,
4032 : copy_rtx (out), 1, OPTAB_DIRECT);
4033 : }
4034 :
4035 0 : HOST_WIDE_INT ival = (unsigned HOST_WIDE_INT) cf - ct;
4036 : /* Make sure we can represent the difference
4037 : between the two values. */
4038 0 : if ((ival > 0) != ((ct < 0) != (cf < 0) ? ct < 0 : ct < cf))
4039 : return false;
4040 :
4041 0 : out = expand_simple_binop (mode, AND, copy_rtx (out),
4042 0 : gen_int_mode (ival, mode),
4043 : copy_rtx (out), 1, OPTAB_DIRECT);
4044 0 : if (ct)
4045 0 : out = expand_simple_binop (mode, PLUS, copy_rtx (out), GEN_INT (ct),
4046 : copy_rtx (out), 1, OPTAB_DIRECT);
4047 0 : if (!rtx_equal_p (out, operands[0]))
4048 0 : emit_move_insn (operands[0], copy_rtx (out));
4049 :
4050 : return true;
4051 : }
4052 : }
4053 :
4054 388891 : if (!TARGET_CMOVE || (mode == QImode && TARGET_PARTIAL_REG_STALL))
4055 : {
4056 : /* Try a few things more with specific constants and a variable. */
4057 :
4058 0 : optab op;
4059 0 : rtx var, orig_out, out, tmp;
4060 :
4061 0 : if (BRANCH_COST (optimize_insn_for_speed_p (), false) <= 2)
4062 : return false;
4063 :
4064 0 : operands[2] = op2;
4065 0 : operands[3] = op3;
4066 :
4067 : /* If one of the two operands is an interesting constant, load a
4068 : constant with the above and mask it in with a logical operation. */
4069 :
4070 0 : if (CONST_INT_P (operands[2]))
4071 : {
4072 0 : var = operands[3];
4073 0 : if (INTVAL (operands[2]) == 0 && operands[3] != constm1_rtx)
4074 0 : operands[3] = constm1_rtx, op = and_optab;
4075 0 : else if (INTVAL (operands[2]) == -1 && operands[3] != const0_rtx)
4076 0 : operands[3] = const0_rtx, op = ior_optab;
4077 : else
4078 : return false;
4079 : }
4080 0 : else if (CONST_INT_P (operands[3]))
4081 : {
4082 0 : var = operands[2];
4083 0 : if (INTVAL (operands[3]) == 0 && operands[2] != constm1_rtx)
4084 : {
4085 : /* For smin (x, 0), expand as "x < 0 ? x : 0" instead of
4086 : "x <= 0 ? x : 0" to enable sign_bit_compare_p. */
4087 0 : if (code == LE && op1 == const0_rtx && rtx_equal_p (op0, var))
4088 0 : operands[1] = simplify_gen_relational (LT, VOIDmode,
4089 0 : GET_MODE (op0),
4090 : op0, const0_rtx);
4091 :
4092 0 : operands[2] = constm1_rtx;
4093 0 : op = and_optab;
4094 : }
4095 0 : else if (INTVAL (operands[3]) == -1 && operands[3] != const0_rtx)
4096 0 : operands[2] = const0_rtx, op = ior_optab;
4097 : else
4098 : return false;
4099 : }
4100 : else
4101 : return false;
4102 :
4103 0 : orig_out = operands[0];
4104 0 : tmp = gen_reg_rtx (mode);
4105 0 : operands[0] = tmp;
4106 :
4107 : /* Recurse to get the constant loaded. */
4108 0 : if (!ix86_expand_int_movcc (operands))
4109 : return false;
4110 :
4111 : /* Mask in the interesting variable. */
4112 0 : out = expand_binop (mode, op, var, tmp, orig_out, 0,
4113 : OPTAB_WIDEN);
4114 0 : if (!rtx_equal_p (out, orig_out))
4115 0 : emit_move_insn (copy_rtx (orig_out), copy_rtx (out));
4116 :
4117 : return true;
4118 : }
4119 :
4120 : /*
4121 : * For comparison with above,
4122 : *
4123 : * movl cf,dest
4124 : * movl ct,tmp
4125 : * cmpl op1,op2
4126 : * cmovcc tmp,dest
4127 : *
4128 : * Size 15.
4129 : */
4130 :
4131 388891 : if (! nonimmediate_operand (operands[2], mode))
4132 17627 : operands[2] = force_reg (mode, operands[2]);
4133 388891 : if (! nonimmediate_operand (operands[3], mode))
4134 168900 : operands[3] = force_reg (mode, operands[3]);
4135 :
4136 388891 : if (! register_operand (operands[2], VOIDmode)
4137 388891 : && (mode == QImode
4138 1093 : || ! register_operand (operands[3], VOIDmode)))
4139 1564 : operands[2] = force_reg (mode, operands[2]);
4140 :
4141 388891 : if (mode == QImode
4142 388891 : && ! register_operand (operands[3], VOIDmode))
4143 592 : operands[3] = force_reg (mode, operands[3]);
4144 :
4145 388891 : emit_insn (compare_seq);
4146 388891 : emit_insn (gen_rtx_SET (operands[0],
4147 : gen_rtx_IF_THEN_ELSE (mode,
4148 : compare_op, operands[2],
4149 : operands[3])));
4150 388891 : return true;
4151 : }
4152 :
4153 : /* Detect conditional moves that exactly match min/max operational
4154 : semantics. Note that this is IEEE safe, as long as we don't
4155 : interchange the operands.
4156 :
4157 : Returns FALSE if this conditional move doesn't match a MIN/MAX,
4158 : and TRUE if the operation is successful and instructions are emitted. */
4159 :
4160 : static bool
4161 9677 : ix86_expand_sse_fp_minmax (rtx dest, enum rtx_code code, rtx cmp_op0,
4162 : rtx cmp_op1, rtx if_true, rtx if_false)
4163 : {
4164 9677 : machine_mode mode = GET_MODE (dest);
4165 9677 : bool is_min;
4166 9677 : rtx tmp;
4167 :
4168 9677 : if (code == LT)
4169 : ;
4170 3214 : else if (code == LE && !HONOR_NANS (mode))
4171 : {
4172 : /* We can swap LE to GE and then invert to LT. */
4173 : std::swap (cmp_op0, cmp_op1);
4174 : std::swap (if_true, if_false);
4175 : }
4176 3173 : else if (code == UNGE)
4177 : std::swap (if_true, if_false);
4178 : else
4179 : return false;
4180 :
4181 8601 : if (rtx_equal_p (cmp_op0, if_true) && rtx_equal_p (cmp_op1, if_false))
4182 : is_min = true;
4183 4616 : else if (rtx_equal_p (cmp_op1, if_true) && rtx_equal_p (cmp_op0, if_false))
4184 : is_min = false;
4185 : else
4186 : return false;
4187 :
4188 7547 : if (immediate_operand (if_false, mode))
4189 8 : if_false = force_reg (mode, if_false);
4190 7547 : if (immediate_operand (if_true, mode))
4191 0 : if_true = force_reg (mode, if_true);
4192 :
4193 : /* We want to check HONOR_NANS and HONOR_SIGNED_ZEROS here,
4194 : but MODE may be a vector mode and thus not appropriate. */
4195 7547 : if (!flag_finite_math_only || flag_signed_zeros)
4196 : {
4197 7547 : int u = is_min ? UNSPEC_IEEE_MIN : UNSPEC_IEEE_MAX;
4198 7547 : rtvec v;
4199 :
4200 7547 : if_true = force_reg (mode, if_true);
4201 7547 : v = gen_rtvec (2, if_true, if_false);
4202 7547 : tmp = gen_rtx_UNSPEC (mode, v, u);
4203 7547 : }
4204 : else
4205 : {
4206 0 : code = is_min ? SMIN : SMAX;
4207 0 : if (MEM_P (if_true) && MEM_P (if_false))
4208 0 : if_true = force_reg (mode, if_true);
4209 0 : tmp = gen_rtx_fmt_ee (code, mode, if_true, if_false);
4210 : }
4211 :
4212 7547 : emit_insn (gen_rtx_SET (dest, tmp));
4213 7547 : return true;
4214 : }
4215 :
4216 : /* Return true if MODE is valid for vector compare to mask register,
4217 : Same result for conditionl vector move with mask register. */
4218 : static bool
4219 15596 : ix86_valid_mask_cmp_mode (machine_mode mode)
4220 : {
4221 : /* XOP has its own vector conditional movement. */
4222 15596 : if (TARGET_XOP && !TARGET_AVX512F)
4223 : return false;
4224 :
4225 : /* HFmode only supports vcmpsh whose dest is mask register. */
4226 15590 : if (TARGET_AVX512FP16 && mode == HFmode)
4227 : return true;
4228 :
4229 : /* AVX512F is needed for mask operation. */
4230 15498 : if (!(TARGET_AVX512F && VECTOR_MODE_P (mode)))
4231 : return false;
4232 :
4233 : /* AVX512BW is needed for vector QI/HImode,
4234 : AVX512VL is needed for 128/256-bit vector. */
4235 182 : machine_mode inner_mode = GET_MODE_INNER (mode);
4236 182 : int vector_size = GET_MODE_SIZE (mode);
4237 182 : if ((inner_mode == QImode || inner_mode == HImode) && !TARGET_AVX512BW)
4238 : return false;
4239 :
4240 162 : return vector_size == 64 || TARGET_AVX512VL;
4241 : }
4242 :
4243 : /* Return true if integer mask comparison should be used. */
4244 : static bool
4245 56943 : ix86_use_mask_cmp_p (machine_mode mode, machine_mode cmp_mode,
4246 : rtx op_true, rtx op_false)
4247 : {
4248 56943 : int vector_size = GET_MODE_SIZE (mode);
4249 :
4250 56943 : if (cmp_mode == HFmode)
4251 : return true;
4252 56851 : else if (vector_size < 16)
4253 : return false;
4254 49883 : else if (vector_size == 64)
4255 : return true;
4256 99650 : else if (GET_MODE_INNER (cmp_mode) == HFmode)
4257 : return true;
4258 99650 : else if (GET_MODE_INNER (cmp_mode) == BFmode)
4259 : return true;
4260 :
4261 : /* When op_true is NULL, op_false must be NULL, or vice versa. */
4262 49825 : gcc_assert (!op_true == !op_false);
4263 :
4264 : /* When op_true/op_false is NULL or cmp_mode is not valid mask cmp mode,
4265 : vector dest is required. */
4266 49825 : if (!op_true || !ix86_valid_mask_cmp_mode (cmp_mode))
4267 : return false;
4268 :
4269 : /* Exclude those that could be optimized in ix86_expand_sse_movcc. */
4270 48 : if (op_false == CONST0_RTX (mode)
4271 48 : || op_true == CONST0_RTX (mode)
4272 48 : || (INTEGRAL_MODE_P (mode)
4273 40 : && (op_true == CONSTM1_RTX (mode)
4274 40 : || op_false == CONSTM1_RTX (mode))))
4275 0 : return false;
4276 :
4277 : return true;
4278 : }
4279 :
4280 : /* Expand an SSE comparison. Return the register with the result. */
4281 :
4282 : static rtx
4283 38602 : ix86_expand_sse_cmp (rtx dest, enum rtx_code code, rtx cmp_op0, rtx cmp_op1,
4284 : rtx op_true, rtx op_false)
4285 : {
4286 38602 : machine_mode mode = GET_MODE (dest);
4287 38602 : machine_mode cmp_ops_mode = GET_MODE (cmp_op0);
4288 :
4289 : /* In general case result of comparison can differ from operands' type. */
4290 38602 : machine_mode cmp_mode;
4291 :
4292 : /* In AVX512F the result of comparison is an integer mask. */
4293 38602 : bool maskcmp = false;
4294 38602 : rtx x;
4295 :
4296 38602 : if (ix86_use_mask_cmp_p (mode, cmp_ops_mode, op_true, op_false))
4297 : {
4298 145 : unsigned int nbits = GET_MODE_NUNITS (cmp_ops_mode);
4299 145 : maskcmp = true;
4300 145 : cmp_mode = nbits > 8 ? int_mode_for_size (nbits, 0).require () : E_QImode;
4301 : }
4302 : else
4303 : cmp_mode = cmp_ops_mode;
4304 :
4305 38602 : cmp_op0 = force_reg (cmp_ops_mode, cmp_op0);
4306 :
4307 77204 : bool (*op1_predicate)(rtx, machine_mode)
4308 38602 : = VECTOR_MODE_P (cmp_ops_mode) ? vector_operand : nonimmediate_operand;
4309 :
4310 38602 : if (!op1_predicate (cmp_op1, cmp_ops_mode))
4311 0 : cmp_op1 = force_reg (cmp_ops_mode, cmp_op1);
4312 :
4313 38602 : if (optimize
4314 503 : || (maskcmp && cmp_mode != mode)
4315 503 : || (op_true && reg_overlap_mentioned_p (dest, op_true))
4316 39105 : || (op_false && reg_overlap_mentioned_p (dest, op_false)))
4317 76053 : dest = gen_reg_rtx (maskcmp ? cmp_mode : mode);
4318 :
4319 38602 : if (maskcmp)
4320 : {
4321 145 : bool ok = ix86_expand_mask_vec_cmp (dest, code, cmp_op0, cmp_op1);
4322 145 : gcc_assert (ok);
4323 : return dest;
4324 : }
4325 :
4326 38457 : x = gen_rtx_fmt_ee (code, cmp_mode, cmp_op0, cmp_op1);
4327 :
4328 38457 : if (cmp_mode != mode)
4329 : {
4330 8037 : x = force_reg (cmp_ops_mode, x);
4331 8037 : convert_move (dest, x, false);
4332 : }
4333 : else
4334 30420 : emit_insn (gen_rtx_SET (dest, x));
4335 :
4336 : return dest;
4337 : }
4338 :
4339 : /* Emit x86 binary operand CODE in mode MODE for SSE vector
4340 : instructions that can be performed using GP registers. */
4341 :
4342 : static void
4343 6954 : ix86_emit_vec_binop (enum rtx_code code, machine_mode mode,
4344 : rtx dst, rtx src1, rtx src2)
4345 : {
4346 6954 : rtx tmp;
4347 :
4348 6954 : tmp = gen_rtx_SET (dst, gen_rtx_fmt_ee (code, mode, src1, src2));
4349 :
4350 6954 : if (GET_MODE_SIZE (mode) <= GET_MODE_SIZE (SImode)
4351 6954 : && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
4352 : {
4353 94 : rtx clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
4354 94 : tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, clob));
4355 : }
4356 :
4357 6954 : emit_insn (tmp);
4358 6954 : }
4359 :
4360 : /* Expand DEST = CMP ? OP_TRUE : OP_FALSE into a sequence of logical
4361 : operations. This is used for both scalar and vector conditional moves. */
4362 :
4363 : void
4364 10625 : ix86_expand_sse_movcc (rtx dest, rtx cmp, rtx op_true, rtx op_false)
4365 : {
4366 10625 : machine_mode mode = GET_MODE (dest);
4367 10625 : machine_mode cmpmode = GET_MODE (cmp);
4368 10625 : rtx x;
4369 :
4370 : /* Simplify trivial VEC_COND_EXPR to avoid ICE in pr97506. */
4371 10625 : if (rtx_equal_p (op_true, op_false))
4372 : {
4373 0 : emit_move_insn (dest, op_true);
4374 0 : return;
4375 : }
4376 :
4377 : /* If we have an integer mask and FP value then we need
4378 : to cast mask to FP mode. */
4379 10625 : if (mode != cmpmode && VECTOR_MODE_P (cmpmode))
4380 : {
4381 1667 : cmp = force_reg (cmpmode, cmp);
4382 1667 : cmp = gen_rtx_SUBREG (mode, cmp, 0);
4383 : }
4384 :
4385 : /* In AVX512F the result of comparison is an integer mask. */
4386 10625 : if (mode != cmpmode
4387 1812 : && GET_MODE_CLASS (cmpmode) == MODE_INT)
4388 : {
4389 145 : gcc_assert (ix86_valid_mask_cmp_mode (mode));
4390 : /* Using scalar/vector move with mask register. */
4391 145 : cmp = force_reg (cmpmode, cmp);
4392 : /* Optimize for mask zero. */
4393 290 : op_true = (op_true != CONST0_RTX (mode)
4394 145 : ? force_reg (mode, op_true) : op_true);
4395 290 : op_false = (op_false != CONST0_RTX (mode)
4396 145 : ? force_reg (mode, op_false) : op_false);
4397 145 : if (op_true == CONST0_RTX (mode))
4398 : {
4399 0 : if (cmpmode == E_DImode && !TARGET_64BIT)
4400 : {
4401 0 : x = gen_reg_rtx (cmpmode);
4402 0 : emit_insn (gen_knotdi (x, cmp));
4403 : }
4404 : else
4405 0 : x = expand_simple_unop (cmpmode, NOT, cmp, NULL, 1);
4406 : cmp = x;
4407 : /* Reverse op_true op_false. */
4408 : std::swap (op_true, op_false);
4409 : }
4410 :
4411 145 : if (mode == HFmode)
4412 92 : emit_insn (gen_movhf_mask (dest, op_true, op_false, cmp));
4413 : else
4414 53 : emit_insn (gen_rtx_SET (dest,
4415 : gen_rtx_VEC_MERGE (mode,
4416 : op_true, op_false, cmp)));
4417 : return;
4418 : }
4419 :
4420 10480 : if (vector_all_ones_operand (op_true, mode)
4421 10480 : && op_false == CONST0_RTX (mode))
4422 : {
4423 3 : emit_move_insn (dest, cmp);
4424 3 : return;
4425 : }
4426 10477 : else if (op_false == CONST0_RTX (mode))
4427 : {
4428 984 : x = expand_simple_binop (mode, AND, cmp, op_true,
4429 : dest, 1, OPTAB_DIRECT);
4430 984 : if (x != dest)
4431 0 : emit_move_insn (dest, x);
4432 : return;
4433 : }
4434 9493 : else if (op_true == CONST0_RTX (mode))
4435 : {
4436 118 : op_false = force_reg (mode, op_false);
4437 118 : x = gen_rtx_NOT (mode, cmp);
4438 118 : ix86_emit_vec_binop (AND, mode, dest, x, op_false);
4439 118 : return;
4440 : }
4441 9375 : else if (vector_all_ones_operand (op_true, mode))
4442 : {
4443 3 : x = expand_simple_binop (mode, IOR, cmp, op_false,
4444 : dest, 1, OPTAB_DIRECT);
4445 3 : if (x != dest)
4446 0 : emit_move_insn (dest, x);
4447 : return;
4448 : }
4449 :
4450 9372 : if (TARGET_XOP)
4451 : {
4452 65 : op_true = force_reg (mode, op_true);
4453 :
4454 65 : if (GET_MODE_SIZE (mode) < 16
4455 65 : || !nonimmediate_operand (op_false, mode))
4456 50 : op_false = force_reg (mode, op_false);
4457 :
4458 65 : emit_insn (gen_rtx_SET (dest,
4459 : gen_rtx_IF_THEN_ELSE (mode, cmp,
4460 : op_true, op_false)));
4461 65 : return;
4462 : }
4463 :
4464 9307 : rtx (*gen) (rtx, rtx, rtx, rtx) = NULL;
4465 9307 : machine_mode blend_mode = mode;
4466 :
4467 9307 : switch (mode)
4468 : {
4469 43 : case E_V2SFmode:
4470 43 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4471 : gen = gen_mmx_blendvps;
4472 : break;
4473 373 : case E_V4SFmode:
4474 373 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4475 : gen = gen_sse4_1_blendvps;
4476 : break;
4477 188 : case E_V2DFmode:
4478 188 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4479 : gen = gen_sse4_1_blendvpd;
4480 : break;
4481 1096 : case E_SFmode:
4482 1096 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4483 : gen = gen_sse4_1_blendvss;
4484 : break;
4485 806 : case E_DFmode:
4486 806 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4487 : gen = gen_sse4_1_blendvsd;
4488 : break;
4489 229 : case E_V8QImode:
4490 229 : case E_V4HImode:
4491 229 : case E_V4HFmode:
4492 229 : case E_V4BFmode:
4493 229 : case E_V2SImode:
4494 229 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4495 : {
4496 : gen = gen_mmx_pblendvb_v8qi;
4497 : blend_mode = V8QImode;
4498 : }
4499 : break;
4500 95 : case E_V4QImode:
4501 95 : case E_V2HImode:
4502 95 : case E_V2HFmode:
4503 95 : case E_V2BFmode:
4504 95 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4505 : {
4506 : gen = gen_mmx_pblendvb_v4qi;
4507 : blend_mode = V4QImode;
4508 : }
4509 : break;
4510 36 : case E_V2QImode:
4511 36 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4512 : gen = gen_mmx_pblendvb_v2qi;
4513 : break;
4514 5788 : case E_V16QImode:
4515 5788 : case E_V8HImode:
4516 5788 : case E_V8HFmode:
4517 5788 : case E_V8BFmode:
4518 5788 : case E_V4SImode:
4519 5788 : case E_V2DImode:
4520 5788 : case E_V1TImode:
4521 5788 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4522 : {
4523 : gen = gen_sse4_1_pblendvb;
4524 : blend_mode = V16QImode;
4525 : }
4526 : break;
4527 83 : case E_V8SFmode:
4528 83 : if (TARGET_AVX && TARGET_SSE_MOVCC_USE_BLENDV)
4529 : gen = gen_avx_blendvps256;
4530 : break;
4531 192 : case E_V4DFmode:
4532 192 : if (TARGET_AVX && TARGET_SSE_MOVCC_USE_BLENDV)
4533 : gen = gen_avx_blendvpd256;
4534 : break;
4535 378 : case E_V32QImode:
4536 378 : case E_V16HImode:
4537 378 : case E_V16HFmode:
4538 378 : case E_V16BFmode:
4539 378 : case E_V8SImode:
4540 378 : case E_V4DImode:
4541 378 : if (TARGET_AVX2 && TARGET_SSE_MOVCC_USE_BLENDV)
4542 : {
4543 : gen = gen_avx2_pblendvb;
4544 : blend_mode = V32QImode;
4545 : }
4546 : break;
4547 :
4548 0 : case E_V64QImode:
4549 0 : gen = gen_avx512bw_blendmv64qi;
4550 0 : break;
4551 0 : case E_V32HImode:
4552 0 : gen = gen_avx512bw_blendmv32hi;
4553 0 : break;
4554 0 : case E_V32HFmode:
4555 0 : gen = gen_avx512bw_blendmv32hf;
4556 0 : break;
4557 0 : case E_V32BFmode:
4558 0 : gen = gen_avx512bw_blendmv32bf;
4559 0 : break;
4560 0 : case E_V16SImode:
4561 0 : gen = gen_avx512f_blendmv16si;
4562 0 : break;
4563 0 : case E_V8DImode:
4564 0 : gen = gen_avx512f_blendmv8di;
4565 0 : break;
4566 0 : case E_V8DFmode:
4567 0 : gen = gen_avx512f_blendmv8df;
4568 0 : break;
4569 : case E_V16SFmode:
4570 : gen = gen_avx512f_blendmv16sf;
4571 : break;
4572 :
4573 : default:
4574 : break;
4575 : }
4576 :
4577 0 : if (gen != NULL)
4578 : {
4579 2100 : if (GET_MODE_SIZE (mode) < 16
4580 2100 : || !vector_operand (op_true, mode))
4581 567 : op_true = force_reg (mode, op_true);
4582 2100 : op_false = force_reg (mode, op_false);
4583 :
4584 2100 : if (blend_mode == mode)
4585 : x = dest;
4586 : else
4587 : {
4588 1032 : x = gen_reg_rtx (blend_mode);
4589 1032 : op_false = gen_lowpart (blend_mode, op_false);
4590 1032 : op_true = gen_lowpart (blend_mode, op_true);
4591 1032 : cmp = gen_lowpart (blend_mode, cmp);
4592 : }
4593 :
4594 2100 : emit_insn (gen (x, op_false, op_true, cmp));
4595 :
4596 2100 : if (x != dest)
4597 1032 : emit_move_insn (dest, gen_lowpart (mode, x));
4598 : }
4599 7207 : else if (CONST_VECTOR_P (op_true) && CONST_VECTOR_P (op_false))
4600 : {
4601 472 : rtx tmp = simplify_const_binary_operation (XOR, mode, op_true, op_false);
4602 472 : tmp = expand_simple_binop (mode, AND, cmp, tmp,
4603 : NULL, 1, OPTAB_DIRECT);
4604 472 : tmp = expand_simple_binop (mode, XOR, tmp, op_false,
4605 : dest, 1, OPTAB_DIRECT);
4606 472 : if (tmp != dest)
4607 0 : emit_move_insn (dest, tmp);
4608 : }
4609 : else
4610 : {
4611 6735 : rtx t2 = expand_simple_binop (mode, AND, cmp, op_true,
4612 : NULL, 1, OPTAB_DIRECT);
4613 :
4614 6735 : rtx t3 = gen_reg_rtx (mode);
4615 6735 : x = gen_rtx_NOT (mode, cmp);
4616 6735 : op_false = force_reg (mode, op_false);
4617 6735 : ix86_emit_vec_binop (AND, mode, t3, x, op_false);
4618 :
4619 6735 : x = expand_simple_binop (mode, IOR, t3, t2,
4620 : dest, 1, OPTAB_DIRECT);
4621 6735 : if (x != dest)
4622 0 : emit_move_insn (dest, x);
4623 : }
4624 : }
4625 :
4626 : /* Swap, force into registers, or otherwise massage the two operands
4627 : to an sse comparison with a mask result. Thus we differ a bit from
4628 : ix86_prepare_fp_compare_args which expects to produce a flags result.
4629 :
4630 : The DEST operand exists to help determine whether to commute commutative
4631 : operators. The POP0/POP1 operands are updated in place. The new
4632 : comparison code is returned, or UNKNOWN if not implementable. */
4633 :
4634 : static enum rtx_code
4635 17704 : ix86_prepare_sse_fp_compare_args (rtx dest, enum rtx_code code,
4636 : rtx *pop0, rtx *pop1)
4637 : {
4638 17704 : switch (code)
4639 : {
4640 57 : case LTGT:
4641 57 : case UNEQ:
4642 : /* AVX supports all the needed comparisons. */
4643 57 : if (TARGET_AVX)
4644 : break;
4645 : /* We have no LTGT as an operator. We could implement it with
4646 : NE & ORDERED, but this requires an extra temporary. It's
4647 : not clear that it's worth it. */
4648 : return UNKNOWN;
4649 :
4650 : case LT:
4651 : case LE:
4652 : case UNGT:
4653 : case UNGE:
4654 : /* These are supported directly. */
4655 : break;
4656 :
4657 6001 : case EQ:
4658 6001 : case NE:
4659 6001 : case UNORDERED:
4660 6001 : case ORDERED:
4661 : /* AVX has 3 operand comparisons, no need to swap anything. */
4662 6001 : if (TARGET_AVX)
4663 : break;
4664 : /* For commutative operators, try to canonicalize the destination
4665 : operand to be first in the comparison - this helps reload to
4666 : avoid extra moves. */
4667 1426 : if (!dest || !rtx_equal_p (dest, *pop1))
4668 : break;
4669 : /* FALLTHRU */
4670 :
4671 10561 : case GE:
4672 10561 : case GT:
4673 10561 : case UNLE:
4674 10561 : case UNLT:
4675 : /* These are not supported directly before AVX, and furthermore
4676 : ix86_expand_sse_fp_minmax only optimizes LT/UNGE. Swap the
4677 : comparison operands to transform into something that is
4678 : supported. */
4679 10561 : std::swap (*pop0, *pop1);
4680 10561 : code = swap_condition (code);
4681 10561 : break;
4682 :
4683 0 : default:
4684 0 : gcc_unreachable ();
4685 : }
4686 :
4687 : return code;
4688 : }
4689 :
4690 : /* Expand a floating-point conditional move. Return true if successful. */
4691 :
4692 : bool
4693 78743 : ix86_expand_fp_movcc (rtx operands[])
4694 : {
4695 78743 : machine_mode mode = GET_MODE (operands[0]);
4696 78743 : enum rtx_code code = GET_CODE (operands[1]);
4697 78743 : rtx tmp, compare_op;
4698 78743 : rtx op0 = XEXP (operands[1], 0);
4699 78743 : rtx op1 = XEXP (operands[1], 1);
4700 :
4701 78743 : if (GET_MODE (op0) == BFmode
4702 78743 : && !ix86_fp_comparison_operator (operands[1], VOIDmode))
4703 : return false;
4704 :
4705 78743 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
4706 : {
4707 64469 : machine_mode cmode;
4708 :
4709 : /* Since we've no cmove for sse registers, don't force bad register
4710 : allocation just to gain access to it. Deny movcc when the
4711 : comparison mode doesn't match the move mode. */
4712 64469 : cmode = GET_MODE (op0);
4713 64469 : if (cmode == VOIDmode)
4714 0 : cmode = GET_MODE (op1);
4715 64469 : if (cmode != mode)
4716 : return false;
4717 :
4718 9687 : code = ix86_prepare_sse_fp_compare_args (operands[0], code, &op0, &op1);
4719 9687 : if (code == UNKNOWN)
4720 : return false;
4721 :
4722 9677 : if (ix86_expand_sse_fp_minmax (operands[0], code, op0, op1,
4723 : operands[2], operands[3]))
4724 : return true;
4725 :
4726 2130 : tmp = ix86_expand_sse_cmp (operands[0], code, op0, op1,
4727 : operands[2], operands[3]);
4728 2130 : ix86_expand_sse_movcc (operands[0], tmp, operands[2], operands[3]);
4729 2130 : return true;
4730 : }
4731 :
4732 14274 : if (GET_MODE (op0) == TImode
4733 14274 : || (GET_MODE (op0) == DImode
4734 72 : && !TARGET_64BIT))
4735 : return false;
4736 :
4737 : /* The floating point conditional move instructions don't directly
4738 : support conditions resulting from a signed integer comparison. */
4739 :
4740 14202 : compare_op = ix86_expand_compare (code, op0, op1);
4741 14202 : if (!fcmov_comparison_operator (compare_op, VOIDmode))
4742 : {
4743 158 : tmp = gen_reg_rtx (QImode);
4744 158 : ix86_expand_setcc (tmp, code, op0, op1);
4745 :
4746 158 : compare_op = ix86_expand_compare (NE, tmp, const0_rtx);
4747 : }
4748 :
4749 14202 : operands[2] = force_reg (mode, operands[2]);
4750 14202 : operands[3] = force_reg (mode, operands[3]);
4751 14202 : emit_insn (gen_rtx_SET (operands[0],
4752 : gen_rtx_IF_THEN_ELSE (mode, compare_op,
4753 : operands[2], operands[3])));
4754 :
4755 14202 : return true;
4756 : }
4757 :
4758 : /* Helper for ix86_cmp_code_to_pcmp_immediate for int modes. */
4759 :
4760 : static int
4761 4940 : ix86_int_cmp_code_to_pcmp_immediate (enum rtx_code code)
4762 : {
4763 4940 : switch (code)
4764 : {
4765 : case EQ:
4766 : return 0;
4767 380 : case LT:
4768 380 : case LTU:
4769 380 : return 1;
4770 220 : case LE:
4771 220 : case LEU:
4772 220 : return 2;
4773 3116 : case NE:
4774 3116 : return 4;
4775 299 : case GE:
4776 299 : case GEU:
4777 299 : return 5;
4778 503 : case GT:
4779 503 : case GTU:
4780 503 : return 6;
4781 0 : default:
4782 0 : gcc_unreachable ();
4783 : }
4784 : }
4785 :
4786 : /* Helper for ix86_cmp_code_to_pcmp_immediate for fp modes. */
4787 :
4788 : static int
4789 1811 : ix86_fp_cmp_code_to_pcmp_immediate (enum rtx_code code)
4790 : {
4791 1811 : switch (code)
4792 : {
4793 : case EQ:
4794 : return 0x00;
4795 378 : case NE:
4796 378 : return 0x04;
4797 514 : case GT:
4798 514 : return 0x0e;
4799 88 : case LE:
4800 88 : return 0x02;
4801 53 : case GE:
4802 53 : return 0x0d;
4803 626 : case LT:
4804 626 : return 0x01;
4805 2 : case UNLE:
4806 2 : return 0x0a;
4807 2 : case UNLT:
4808 2 : return 0x09;
4809 11 : case UNGE:
4810 11 : return 0x05;
4811 44 : case UNGT:
4812 44 : return 0x06;
4813 2 : case UNEQ:
4814 2 : return 0x18;
4815 0 : case LTGT:
4816 0 : return 0x0c;
4817 2 : case ORDERED:
4818 2 : return 0x07;
4819 2 : case UNORDERED:
4820 2 : return 0x03;
4821 0 : default:
4822 0 : gcc_unreachable ();
4823 : }
4824 : }
4825 :
4826 : /* Return immediate value to be used in UNSPEC_PCMP
4827 : for comparison CODE in MODE. */
4828 :
4829 : static int
4830 6751 : ix86_cmp_code_to_pcmp_immediate (enum rtx_code code, machine_mode mode)
4831 : {
4832 6751 : if (FLOAT_MODE_P (mode))
4833 1811 : return ix86_fp_cmp_code_to_pcmp_immediate (code);
4834 4940 : return ix86_int_cmp_code_to_pcmp_immediate (code);
4835 : }
4836 :
4837 : /* Expand AVX-512 vector comparison. */
4838 :
4839 : bool
4840 6751 : ix86_expand_mask_vec_cmp (rtx dest, enum rtx_code code, rtx cmp_op0, rtx cmp_op1)
4841 : {
4842 6751 : machine_mode mask_mode = GET_MODE (dest);
4843 6751 : machine_mode cmp_mode = GET_MODE (cmp_op0);
4844 6751 : rtx imm = GEN_INT (ix86_cmp_code_to_pcmp_immediate (code, cmp_mode));
4845 6751 : int unspec_code;
4846 6751 : rtx unspec;
4847 :
4848 6751 : switch (code)
4849 : {
4850 : case LEU:
4851 : case GTU:
4852 : case GEU:
4853 : case LTU:
4854 : unspec_code = UNSPEC_UNSIGNED_PCMP;
4855 : break;
4856 :
4857 6336 : default:
4858 6336 : unspec_code = UNSPEC_PCMP;
4859 : }
4860 :
4861 6751 : unspec = gen_rtx_UNSPEC (mask_mode, gen_rtvec (3, cmp_op0, cmp_op1, imm),
4862 : unspec_code);
4863 6751 : emit_insn (gen_rtx_SET (dest, unspec));
4864 :
4865 6751 : return true;
4866 : }
4867 :
4868 : /* Expand fp vector comparison. */
4869 :
4870 : bool
4871 8017 : ix86_expand_fp_vec_cmp (rtx operands[])
4872 : {
4873 8017 : enum rtx_code code = GET_CODE (operands[1]);
4874 8017 : rtx cmp;
4875 :
4876 8017 : code = ix86_prepare_sse_fp_compare_args (operands[0], code,
4877 : &operands[2], &operands[3]);
4878 8017 : if (code == UNKNOWN)
4879 : {
4880 20 : rtx temp;
4881 20 : switch (GET_CODE (operands[1]))
4882 : {
4883 2 : case LTGT:
4884 2 : temp = ix86_expand_sse_cmp (operands[0], ORDERED, operands[2],
4885 : operands[3], NULL, NULL);
4886 2 : cmp = ix86_expand_sse_cmp (operands[0], NE, operands[2],
4887 : operands[3], NULL, NULL);
4888 2 : code = AND;
4889 2 : break;
4890 18 : case UNEQ:
4891 18 : temp = ix86_expand_sse_cmp (operands[0], UNORDERED, operands[2],
4892 : operands[3], NULL, NULL);
4893 18 : cmp = ix86_expand_sse_cmp (operands[0], EQ, operands[2],
4894 : operands[3], NULL, NULL);
4895 18 : code = IOR;
4896 18 : break;
4897 0 : default:
4898 0 : gcc_unreachable ();
4899 : }
4900 20 : cmp = expand_simple_binop (GET_MODE (cmp), code, temp, cmp, cmp, 1,
4901 : OPTAB_DIRECT);
4902 : }
4903 : else
4904 7997 : cmp = ix86_expand_sse_cmp (operands[0], code, operands[2], operands[3],
4905 : NULL, NULL);
4906 :
4907 8017 : if (operands[0] != cmp)
4908 7937 : emit_move_insn (operands[0], cmp);
4909 :
4910 8017 : return true;
4911 : }
4912 :
4913 : static rtx
4914 18533 : ix86_expand_int_sse_cmp (rtx dest, enum rtx_code code, rtx cop0, rtx cop1,
4915 : rtx op_true, rtx op_false, bool *negate)
4916 : {
4917 18533 : machine_mode data_mode = GET_MODE (dest);
4918 18533 : machine_mode mode = GET_MODE (cop0);
4919 18533 : rtx x;
4920 :
4921 18533 : *negate = false;
4922 :
4923 : /* XOP supports all of the comparisons on all 128-bit vector int types. */
4924 18533 : if (TARGET_XOP
4925 195 : && GET_MODE_CLASS (mode) == MODE_VECTOR_INT
4926 18728 : && GET_MODE_SIZE (mode) <= 16)
4927 : ;
4928 : /* AVX512F supports all of the comparisons
4929 : on all 128/256/512-bit vector int types. */
4930 18341 : else if (ix86_use_mask_cmp_p (data_mode, mode, op_true, op_false))
4931 : ;
4932 : else
4933 : {
4934 : /* Canonicalize the comparison to EQ, GT, GTU. */
4935 18288 : switch (code)
4936 : {
4937 : case EQ:
4938 : case GT:
4939 : case GTU:
4940 : break;
4941 :
4942 968 : case LE:
4943 968 : case LEU:
4944 : /* x <= cst can be handled as x < cst + 1 unless there is
4945 : wrap around in cst + 1. */
4946 968 : if (CONST_VECTOR_P (cop1)
4947 1655 : && GET_MODE_INNER (mode) != TImode)
4948 : {
4949 687 : unsigned int n_elts = GET_MODE_NUNITS (mode), i;
4950 687 : machine_mode eltmode = GET_MODE_INNER (mode);
4951 4424 : for (i = 0; i < n_elts; ++i)
4952 : {
4953 3738 : rtx elt = CONST_VECTOR_ELT (cop1, i);
4954 3738 : if (!CONST_INT_P (elt))
4955 : break;
4956 3738 : if (code == LE)
4957 : {
4958 : /* For LE punt if some element is signed maximum. */
4959 2562 : if ((INTVAL (elt) & (GET_MODE_MASK (eltmode) >> 1))
4960 : == (GET_MODE_MASK (eltmode) >> 1))
4961 : break;
4962 : }
4963 : /* For LEU punt if some element is unsigned maximum. */
4964 1176 : else if (elt == constm1_rtx)
4965 : break;
4966 : }
4967 687 : if (i == n_elts)
4968 : {
4969 686 : rtvec v = rtvec_alloc (n_elts);
4970 5108 : for (i = 0; i < n_elts; ++i)
4971 3736 : RTVEC_ELT (v, i)
4972 3736 : = gen_int_mode (INTVAL (CONST_VECTOR_ELT (cop1, i)) + 1,
4973 : eltmode);
4974 686 : cop1 = gen_rtx_CONST_VECTOR (mode, v);
4975 686 : std::swap (cop0, cop1);
4976 686 : code = code == LE ? GT : GTU;
4977 : break;
4978 : }
4979 : }
4980 : /* FALLTHRU */
4981 3309 : case NE:
4982 3309 : code = reverse_condition (code);
4983 3309 : *negate = true;
4984 3309 : break;
4985 :
4986 543 : case GE:
4987 543 : case GEU:
4988 : /* x >= cst can be handled as x > cst - 1 unless there is
4989 : wrap around in cst - 1. */
4990 543 : if (CONST_VECTOR_P (cop1)
4991 840 : && GET_MODE_INNER (mode) != TImode)
4992 : {
4993 297 : unsigned int n_elts = GET_MODE_NUNITS (mode), i;
4994 297 : machine_mode eltmode = GET_MODE_INNER (mode);
4995 2221 : for (i = 0; i < n_elts; ++i)
4996 : {
4997 1972 : rtx elt = CONST_VECTOR_ELT (cop1, i);
4998 1972 : if (!CONST_INT_P (elt))
4999 : break;
5000 1972 : if (code == GE)
5001 : {
5002 : /* For GE punt if some element is signed minimum. */
5003 1924 : if (INTVAL (elt) < 0
5004 136 : && ((INTVAL (elt) & (GET_MODE_MASK (eltmode) >> 1))
5005 : == 0))
5006 : break;
5007 : }
5008 : /* For GEU punt if some element is zero. */
5009 48 : else if (elt == const0_rtx)
5010 : break;
5011 : }
5012 297 : if (i == n_elts)
5013 : {
5014 249 : rtvec v = rtvec_alloc (n_elts);
5015 2422 : for (i = 0; i < n_elts; ++i)
5016 1924 : RTVEC_ELT (v, i)
5017 1924 : = gen_int_mode (INTVAL (CONST_VECTOR_ELT (cop1, i)) - 1,
5018 : eltmode);
5019 249 : cop1 = gen_rtx_CONST_VECTOR (mode, v);
5020 249 : code = code == GE ? GT : GTU;
5021 : break;
5022 : }
5023 : }
5024 294 : code = reverse_condition (code);
5025 294 : *negate = true;
5026 : /* FALLTHRU */
5027 :
5028 1657 : case LT:
5029 1657 : case LTU:
5030 1657 : std::swap (cop0, cop1);
5031 1657 : code = swap_condition (code);
5032 1657 : break;
5033 :
5034 0 : default:
5035 0 : gcc_unreachable ();
5036 : }
5037 :
5038 : /* Only SSE4.1/SSE4.2 supports V2DImode. */
5039 18288 : if (mode == V2DImode)
5040 : {
5041 758 : switch (code)
5042 : {
5043 554 : case EQ:
5044 : /* SSE4.1 supports EQ. */
5045 554 : if (!TARGET_SSE4_1)
5046 18533 : return NULL;
5047 : break;
5048 :
5049 204 : case GT:
5050 204 : case GTU:
5051 : /* SSE4.2 supports GT/GTU. */
5052 204 : if (!TARGET_SSE4_2)
5053 : return NULL;
5054 : break;
5055 :
5056 0 : default:
5057 0 : gcc_unreachable ();
5058 : }
5059 : }
5060 :
5061 18288 : if (CONST_VECTOR_P (cop0))
5062 1335 : cop0 = force_reg (mode, cop0);
5063 16953 : else if (CONST_VECTOR_P (cop1))
5064 7655 : cop1 = force_reg (mode, cop1);
5065 :
5066 18288 : rtx optrue = op_true ? op_true : CONSTM1_RTX (data_mode);
5067 18288 : rtx opfalse = op_false ? op_false : CONST0_RTX (data_mode);
5068 18288 : if (*negate)
5069 3603 : std::swap (optrue, opfalse);
5070 :
5071 : /* Transform x > y ? 0 : -1 (i.e. x <= y ? -1 : 0 or x <= y) when
5072 : not using integer masks into min (x, y) == x ? -1 : 0 (i.e.
5073 : min (x, y) == x). While we add one instruction (the minimum),
5074 : we remove the need for two instructions in the negation, as the
5075 : result is done this way.
5076 : When using masks, do it for SI/DImode element types, as it is shorter
5077 : than the two subtractions. */
5078 18288 : if ((code != EQ
5079 7641 : && GET_MODE_SIZE (mode) != 64
5080 7641 : && vector_all_ones_operand (opfalse, data_mode)
5081 576 : && optrue == CONST0_RTX (data_mode))
5082 25353 : || (code == GTU
5083 2082 : && GET_MODE_SIZE (GET_MODE_INNER (mode)) >= 4
5084 : /* Don't do it if not using integer masks and we'd end up with
5085 : the right values in the registers though. */
5086 696 : && (GET_MODE_SIZE (mode) == 64
5087 696 : || !vector_all_ones_operand (optrue, data_mode)
5088 571 : || opfalse != CONST0_RTX (data_mode))))
5089 : {
5090 701 : rtx (*gen) (rtx, rtx, rtx) = NULL;
5091 :
5092 701 : switch (mode)
5093 : {
5094 0 : case E_V16SImode:
5095 0 : gen = (code == GTU) ? gen_uminv16si3 : gen_sminv16si3;
5096 : break;
5097 0 : case E_V8DImode:
5098 0 : gen = (code == GTU) ? gen_uminv8di3 : gen_sminv8di3;
5099 0 : cop0 = force_reg (mode, cop0);
5100 0 : cop1 = force_reg (mode, cop1);
5101 0 : break;
5102 24 : case E_V32QImode:
5103 24 : if (TARGET_AVX2)
5104 24 : gen = (code == GTU) ? gen_uminv32qi3 : gen_sminv32qi3;
5105 : break;
5106 24 : case E_V16HImode:
5107 24 : if (TARGET_AVX2)
5108 24 : gen = (code == GTU) ? gen_uminv16hi3 : gen_sminv16hi3;
5109 : break;
5110 25 : case E_V8SImode:
5111 25 : if (TARGET_AVX2)
5112 25 : gen = (code == GTU) ? gen_uminv8si3 : gen_sminv8si3;
5113 : break;
5114 20 : case E_V4DImode:
5115 20 : if (TARGET_AVX512VL)
5116 : {
5117 0 : gen = (code == GTU) ? gen_uminv4di3 : gen_sminv4di3;
5118 0 : cop0 = force_reg (mode, cop0);
5119 0 : cop1 = force_reg (mode, cop1);
5120 : }
5121 : break;
5122 64 : case E_V16QImode:
5123 64 : if (code == GTU && TARGET_SSE2)
5124 : gen = gen_uminv16qi3;
5125 28 : else if (code == GT && TARGET_SSE4_1)
5126 : gen = gen_sminv16qi3;
5127 : break;
5128 44 : case E_V8QImode:
5129 44 : if (code == GTU && TARGET_SSE2)
5130 : gen = gen_uminv8qi3;
5131 42 : else if (code == GT && TARGET_SSE4_1)
5132 : gen = gen_sminv8qi3;
5133 : break;
5134 13 : case E_V4QImode:
5135 13 : if (code == GTU && TARGET_SSE2)
5136 : gen = gen_uminv4qi3;
5137 2 : else if (code == GT && TARGET_SSE4_1)
5138 : gen = gen_sminv4qi3;
5139 : break;
5140 8 : case E_V2QImode:
5141 8 : if (code == GTU && TARGET_SSE2)
5142 : gen = gen_uminv2qi3;
5143 6 : else if (code == GT && TARGET_SSE4_1)
5144 : gen = gen_sminv2qi3;
5145 : break;
5146 73 : case E_V8HImode:
5147 73 : if (code == GTU && TARGET_SSE4_1)
5148 : gen = gen_uminv8hi3;
5149 63 : else if (code == GT && TARGET_SSE2)
5150 : gen = gen_sminv8hi3;
5151 : break;
5152 4 : case E_V4HImode:
5153 4 : if (code == GTU && TARGET_SSE4_1)
5154 : gen = gen_uminv4hi3;
5155 4 : else if (code == GT && TARGET_SSE2)
5156 : gen = gen_sminv4hi3;
5157 : break;
5158 16 : case E_V2HImode:
5159 16 : if (code == GTU && TARGET_SSE4_1)
5160 : gen = gen_uminv2hi3;
5161 16 : else if (code == GT && TARGET_SSE2)
5162 : gen = gen_sminv2hi3;
5163 : break;
5164 249 : case E_V4SImode:
5165 249 : if (TARGET_SSE4_1)
5166 51 : gen = (code == GTU) ? gen_uminv4si3 : gen_sminv4si3;
5167 : break;
5168 113 : case E_V2SImode:
5169 113 : if (TARGET_SSE4_1)
5170 0 : gen = (code == GTU) ? gen_uminv2si3 : gen_sminv2si3;
5171 : break;
5172 24 : case E_V2DImode:
5173 24 : if (TARGET_AVX512VL)
5174 : {
5175 0 : gen = (code == GTU) ? gen_uminv2di3 : gen_sminv2di3;
5176 0 : cop0 = force_reg (mode, cop0);
5177 0 : cop1 = force_reg (mode, cop1);
5178 : }
5179 : break;
5180 : default:
5181 : break;
5182 : }
5183 :
5184 0 : if (gen)
5185 : {
5186 279 : rtx tem = gen_reg_rtx (mode);
5187 279 : if (!vector_operand (cop0, mode))
5188 0 : cop0 = force_reg (mode, cop0);
5189 279 : if (!vector_operand (cop1, mode))
5190 0 : cop1 = force_reg (mode, cop1);
5191 279 : *negate = !*negate;
5192 279 : emit_insn (gen (tem, cop0, cop1));
5193 279 : cop1 = tem;
5194 279 : code = EQ;
5195 : }
5196 : }
5197 :
5198 : /* Unsigned parallel compare is not supported by the hardware.
5199 : Play some tricks to turn this into a signed comparison
5200 : against 0. */
5201 18288 : if (code == GTU)
5202 : {
5203 1178 : cop0 = force_reg (mode, cop0);
5204 :
5205 1178 : switch (mode)
5206 : {
5207 802 : case E_V16SImode:
5208 802 : case E_V8DImode:
5209 802 : case E_V8SImode:
5210 802 : case E_V4DImode:
5211 802 : case E_V4SImode:
5212 802 : case E_V2SImode:
5213 802 : case E_V2DImode:
5214 802 : {
5215 802 : rtx t1, t2, mask;
5216 :
5217 : /* Subtract (-(INT MAX) - 1) from both operands to make
5218 : them signed. */
5219 802 : mask = ix86_build_signbit_mask (mode, true, false);
5220 802 : t1 = gen_reg_rtx (mode);
5221 802 : emit_insn (gen_sub3_insn (t1, cop0, mask));
5222 :
5223 802 : t2 = gen_reg_rtx (mode);
5224 802 : emit_insn (gen_sub3_insn (t2, cop1, mask));
5225 :
5226 802 : cop0 = t1;
5227 802 : cop1 = t2;
5228 802 : code = GT;
5229 : }
5230 802 : break;
5231 :
5232 376 : case E_V64QImode:
5233 376 : case E_V32HImode:
5234 376 : case E_V32QImode:
5235 376 : case E_V16HImode:
5236 376 : case E_V16QImode:
5237 376 : case E_V8QImode:
5238 376 : case E_V4QImode:
5239 376 : case E_V2QImode:
5240 376 : case E_V8HImode:
5241 376 : case E_V4HImode:
5242 376 : case E_V2HImode:
5243 : /* Perform a parallel unsigned saturating subtraction. */
5244 376 : x = gen_reg_rtx (mode);
5245 376 : emit_insn (gen_rtx_SET
5246 : (x, gen_rtx_US_MINUS (mode, cop0, cop1)));
5247 376 : cop0 = x;
5248 376 : cop1 = CONST0_RTX (mode);
5249 376 : code = EQ;
5250 376 : *negate = !*negate;
5251 376 : break;
5252 :
5253 0 : default:
5254 0 : gcc_unreachable ();
5255 : }
5256 : }
5257 : }
5258 :
5259 18533 : if (*negate)
5260 3638 : std::swap (op_true, op_false);
5261 :
5262 18533 : if (CONST_VECTOR_P (cop1))
5263 439 : cop1 = force_reg (mode, cop1);
5264 :
5265 : /* Allow the comparison to be done in one mode, but the movcc to
5266 : happen in another mode. */
5267 18533 : if (data_mode == mode)
5268 18491 : x = ix86_expand_sse_cmp (dest, code, cop0, cop1, op_true, op_false);
5269 : else
5270 : {
5271 126 : gcc_assert (GET_MODE_SIZE (data_mode) == GET_MODE_SIZE (mode));
5272 42 : x = ix86_expand_sse_cmp (gen_reg_rtx (mode), code, cop0, cop1,
5273 : op_true, op_false);
5274 42 : if (GET_MODE (x) == mode)
5275 24 : x = gen_lowpart (data_mode, x);
5276 : }
5277 :
5278 : return x;
5279 : }
5280 :
5281 : /* Expand integer vector comparison. */
5282 :
5283 : bool
5284 11578 : ix86_expand_int_vec_cmp (rtx operands[])
5285 : {
5286 11578 : rtx_code code = GET_CODE (operands[1]);
5287 11578 : bool negate = false;
5288 11578 : rtx cmp = ix86_expand_int_sse_cmp (operands[0], code, operands[2],
5289 : operands[3], NULL, NULL, &negate);
5290 :
5291 11578 : if (!cmp)
5292 : return false;
5293 :
5294 11578 : if (negate)
5295 : {
5296 3648 : if (TARGET_AVX512F && GET_MODE_SIZE (GET_MODE (cmp)) >= 16)
5297 91 : cmp = gen_rtx_XOR (GET_MODE (cmp), cmp, CONSTM1_RTX (GET_MODE (cmp)));
5298 : else
5299 : {
5300 6862 : cmp = ix86_expand_int_sse_cmp (operands[0], EQ, cmp,
5301 3431 : CONST0_RTX (GET_MODE (cmp)),
5302 : NULL, NULL, &negate);
5303 3431 : gcc_assert (!negate);
5304 : }
5305 : }
5306 :
5307 11578 : if (operands[0] != cmp)
5308 11283 : emit_move_insn (operands[0], cmp);
5309 :
5310 : return true;
5311 : }
5312 :
5313 : /* Expand a floating-point vector conditional move; a vcond operation
5314 : rather than a movcc operation. */
5315 :
5316 : bool
5317 0 : ix86_expand_fp_vcond (rtx operands[])
5318 : {
5319 0 : enum rtx_code code = GET_CODE (operands[3]);
5320 0 : rtx cmp;
5321 :
5322 0 : code = ix86_prepare_sse_fp_compare_args (operands[0], code,
5323 : &operands[4], &operands[5]);
5324 0 : if (code == UNKNOWN)
5325 : {
5326 0 : rtx temp;
5327 0 : switch (GET_CODE (operands[3]))
5328 : {
5329 0 : case LTGT:
5330 0 : temp = ix86_expand_sse_cmp (operands[0], ORDERED, operands[4],
5331 : operands[5], operands[0], operands[0]);
5332 0 : cmp = ix86_expand_sse_cmp (operands[0], NE, operands[4],
5333 : operands[5], operands[1], operands[2]);
5334 0 : code = AND;
5335 0 : break;
5336 0 : case UNEQ:
5337 0 : temp = ix86_expand_sse_cmp (operands[0], UNORDERED, operands[4],
5338 : operands[5], operands[0], operands[0]);
5339 0 : cmp = ix86_expand_sse_cmp (operands[0], EQ, operands[4],
5340 : operands[5], operands[1], operands[2]);
5341 0 : code = IOR;
5342 0 : break;
5343 0 : default:
5344 0 : gcc_unreachable ();
5345 : }
5346 0 : cmp = expand_simple_binop (GET_MODE (cmp), code, temp, cmp, cmp, 1,
5347 : OPTAB_DIRECT);
5348 0 : ix86_expand_sse_movcc (operands[0], cmp, operands[1], operands[2]);
5349 0 : return true;
5350 : }
5351 :
5352 0 : if (ix86_expand_sse_fp_minmax (operands[0], code, operands[4],
5353 : operands[5], operands[1], operands[2]))
5354 : return true;
5355 :
5356 0 : cmp = ix86_expand_sse_cmp (operands[0], code, operands[4], operands[5],
5357 : operands[1], operands[2]);
5358 0 : ix86_expand_sse_movcc (operands[0], cmp, operands[1], operands[2]);
5359 0 : return true;
5360 : }
5361 :
5362 : /* Expand a signed/unsigned integral vector conditional move. */
5363 :
5364 : bool
5365 3524 : ix86_expand_int_vcond (rtx operands[])
5366 : {
5367 3524 : machine_mode data_mode = GET_MODE (operands[0]);
5368 3524 : machine_mode mode = GET_MODE (operands[4]);
5369 3524 : enum rtx_code code = GET_CODE (operands[3]);
5370 3524 : bool negate = false;
5371 3524 : rtx x, cop0, cop1;
5372 :
5373 3524 : cop0 = operands[4];
5374 3524 : cop1 = operands[5];
5375 :
5376 : /* Try to optimize x < 0 ? -1 : 0 into (signed) x >> 31
5377 : and x < 0 ? 1 : 0 into (unsigned) x >> 31. */
5378 3524 : if ((code == LT || code == GE)
5379 0 : && data_mode == mode
5380 0 : && cop1 == CONST0_RTX (mode)
5381 0 : && operands[1 + (code == LT)] == CONST0_RTX (data_mode)
5382 0 : && GET_MODE_UNIT_SIZE (data_mode) > 1
5383 0 : && GET_MODE_UNIT_SIZE (data_mode) <= 8
5384 3524 : && (GET_MODE_SIZE (data_mode) == 16
5385 0 : || (TARGET_AVX2 && GET_MODE_SIZE (data_mode) == 32)))
5386 : {
5387 0 : rtx negop = operands[2 - (code == LT)];
5388 0 : int shift = GET_MODE_UNIT_BITSIZE (data_mode) - 1;
5389 0 : if (negop == CONST1_RTX (data_mode))
5390 : {
5391 0 : rtx res = expand_simple_binop (mode, LSHIFTRT, cop0, GEN_INT (shift),
5392 : operands[0], 1, OPTAB_DIRECT);
5393 0 : if (res != operands[0])
5394 0 : emit_move_insn (operands[0], res);
5395 : return true;
5396 : }
5397 0 : else if (GET_MODE_INNER (data_mode) != DImode
5398 0 : && vector_all_ones_operand (negop, data_mode))
5399 : {
5400 0 : rtx res = expand_simple_binop (mode, ASHIFTRT, cop0, GEN_INT (shift),
5401 : operands[0], 0, OPTAB_DIRECT);
5402 0 : if (res != operands[0])
5403 0 : emit_move_insn (operands[0], res);
5404 : return true;
5405 : }
5406 : }
5407 :
5408 3524 : if (!nonimmediate_operand (cop1, mode))
5409 126 : cop1 = force_reg (mode, cop1);
5410 3524 : if (!general_operand (operands[1], data_mode))
5411 0 : operands[1] = force_reg (data_mode, operands[1]);
5412 3524 : if (!general_operand (operands[2], data_mode))
5413 0 : operands[2] = force_reg (data_mode, operands[2]);
5414 :
5415 3524 : x = ix86_expand_int_sse_cmp (operands[0], code, cop0, cop1,
5416 : operands[1], operands[2], &negate);
5417 :
5418 3524 : if (!x)
5419 : return false;
5420 :
5421 3524 : ix86_expand_sse_movcc (operands[0], x, operands[1+negate],
5422 3524 : operands[2-negate]);
5423 3524 : return true;
5424 : }
5425 :
5426 : static bool
5427 126651 : ix86_expand_vec_perm_vpermt2 (rtx target, rtx mask, rtx op0, rtx op1,
5428 : struct expand_vec_perm_d *d)
5429 : {
5430 : /* ix86_expand_vec_perm_vpermt2 is called from both const and non-const
5431 : expander, so args are either in d, or in op0, op1 etc. */
5432 126651 : machine_mode mode = GET_MODE (d ? d->op0 : op0);
5433 126651 : machine_mode maskmode = mode;
5434 126651 : rtx (*gen) (rtx, rtx, rtx, rtx) = NULL;
5435 :
5436 126651 : switch (mode)
5437 : {
5438 24191 : case E_V16QImode:
5439 24191 : if (TARGET_AVX512VL && TARGET_AVX512VBMI)
5440 : gen = gen_avx512vl_vpermt2varv16qi3;
5441 : break;
5442 513 : case E_V32QImode:
5443 513 : if (TARGET_AVX512VL && TARGET_AVX512VBMI)
5444 : gen = gen_avx512vl_vpermt2varv32qi3;
5445 : break;
5446 190 : case E_V64QImode:
5447 190 : if (TARGET_AVX512VBMI)
5448 : gen = gen_avx512bw_vpermt2varv64qi3;
5449 : break;
5450 13485 : case E_V8HImode:
5451 13485 : if (TARGET_AVX512VL && TARGET_AVX512BW)
5452 : gen = gen_avx512vl_vpermt2varv8hi3;
5453 : break;
5454 773 : case E_V16HImode:
5455 773 : if (TARGET_AVX512VL && TARGET_AVX512BW)
5456 : gen = gen_avx512vl_vpermt2varv16hi3;
5457 : break;
5458 331 : case E_V32HImode:
5459 331 : if (TARGET_AVX512BW)
5460 : gen = gen_avx512bw_vpermt2varv32hi3;
5461 : break;
5462 34256 : case E_V4SImode:
5463 34256 : if (TARGET_AVX512VL)
5464 : gen = gen_avx512vl_vpermt2varv4si3;
5465 : break;
5466 1138 : case E_V8SImode:
5467 1138 : if (TARGET_AVX512VL)
5468 : gen = gen_avx512vl_vpermt2varv8si3;
5469 : break;
5470 126 : case E_V16SImode:
5471 126 : if (TARGET_AVX512F)
5472 : gen = gen_avx512f_vpermt2varv16si3;
5473 : break;
5474 10515 : case E_V4SFmode:
5475 10515 : if (TARGET_AVX512VL)
5476 : {
5477 : gen = gen_avx512vl_vpermt2varv4sf3;
5478 : maskmode = V4SImode;
5479 : }
5480 : break;
5481 6047 : case E_V8SFmode:
5482 6047 : if (TARGET_AVX512VL)
5483 : {
5484 : gen = gen_avx512vl_vpermt2varv8sf3;
5485 : maskmode = V8SImode;
5486 : }
5487 : break;
5488 207 : case E_V16SFmode:
5489 207 : if (TARGET_AVX512F)
5490 : {
5491 : gen = gen_avx512f_vpermt2varv16sf3;
5492 : maskmode = V16SImode;
5493 : }
5494 : break;
5495 2 : case E_V2DImode:
5496 2 : if (TARGET_AVX512VL)
5497 : gen = gen_avx512vl_vpermt2varv2di3;
5498 : break;
5499 347 : case E_V4DImode:
5500 347 : if (TARGET_AVX512VL)
5501 : gen = gen_avx512vl_vpermt2varv4di3;
5502 : break;
5503 10 : case E_V8DImode:
5504 10 : if (TARGET_AVX512F)
5505 : gen = gen_avx512f_vpermt2varv8di3;
5506 : break;
5507 2 : case E_V2DFmode:
5508 2 : if (TARGET_AVX512VL)
5509 : {
5510 : gen = gen_avx512vl_vpermt2varv2df3;
5511 : maskmode = V2DImode;
5512 : }
5513 : break;
5514 1996 : case E_V4DFmode:
5515 1996 : if (TARGET_AVX512VL)
5516 : {
5517 : gen = gen_avx512vl_vpermt2varv4df3;
5518 : maskmode = V4DImode;
5519 : }
5520 : break;
5521 194 : case E_V8DFmode:
5522 194 : if (TARGET_AVX512F)
5523 : {
5524 : gen = gen_avx512f_vpermt2varv8df3;
5525 : maskmode = V8DImode;
5526 : }
5527 : break;
5528 : default:
5529 : break;
5530 : }
5531 :
5532 : if (gen == NULL)
5533 : return false;
5534 :
5535 884 : if (d && d->testing_p)
5536 : return true;
5537 :
5538 : /* ix86_expand_vec_perm_vpermt2 is called from both const and non-const
5539 : expander, so args are either in d, or in op0, op1 etc. */
5540 873 : if (d)
5541 : {
5542 873 : rtx vec[64];
5543 873 : target = d->target;
5544 873 : op0 = d->op0;
5545 873 : op1 = d->op1;
5546 15125 : for (int i = 0; i < d->nelt; ++i)
5547 14252 : vec[i] = GEN_INT (d->perm[i]);
5548 873 : mask = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (d->nelt, vec));
5549 : }
5550 :
5551 881 : emit_insn (gen (target, force_reg (maskmode, mask), op0, op1));
5552 881 : return true;
5553 : }
5554 :
5555 : /* Expand a variable vector permutation. */
5556 :
5557 : void
5558 18 : ix86_expand_vec_perm (rtx operands[])
5559 : {
5560 18 : rtx target = operands[0];
5561 18 : rtx op0 = operands[1];
5562 18 : rtx op1 = operands[2];
5563 18 : rtx mask = operands[3];
5564 18 : rtx t1, t2, t3, t4, t5, t6, t7, t8, vt, vt2, vec[32];
5565 18 : machine_mode mode = GET_MODE (op0);
5566 18 : machine_mode maskmode = GET_MODE (mask);
5567 18 : int w, e, i;
5568 18 : bool one_operand_shuffle = rtx_equal_p (op0, op1);
5569 :
5570 : /* Number of elements in the vector. */
5571 18 : w = GET_MODE_NUNITS (mode);
5572 18 : e = GET_MODE_UNIT_SIZE (mode);
5573 18 : gcc_assert (w <= 64);
5574 :
5575 : /* For HF mode vector, convert it to HI using subreg. */
5576 36 : if (GET_MODE_INNER (mode) == HFmode)
5577 : {
5578 6 : machine_mode orig_mode = mode;
5579 6 : mode = mode_for_vector (HImode, w).require ();
5580 6 : target = lowpart_subreg (mode, target, orig_mode);
5581 6 : op0 = lowpart_subreg (mode, op0, orig_mode);
5582 6 : op1 = lowpart_subreg (mode, op1, orig_mode);
5583 : }
5584 :
5585 18 : if (TARGET_AVX512F && one_operand_shuffle)
5586 : {
5587 5 : rtx (*gen) (rtx, rtx, rtx) = NULL;
5588 5 : switch (mode)
5589 : {
5590 : case E_V16SImode:
5591 : gen = gen_avx512f_permvarv16si;
5592 : break;
5593 0 : case E_V16SFmode:
5594 0 : gen = gen_avx512f_permvarv16sf;
5595 0 : break;
5596 0 : case E_V8DImode:
5597 0 : gen = gen_avx512f_permvarv8di;
5598 0 : break;
5599 0 : case E_V8DFmode:
5600 0 : gen = gen_avx512f_permvarv8df;
5601 0 : break;
5602 : default:
5603 : break;
5604 : }
5605 0 : if (gen != NULL)
5606 : {
5607 0 : emit_insn (gen (target, op0, mask));
5608 16 : return;
5609 : }
5610 : }
5611 :
5612 18 : if (ix86_expand_vec_perm_vpermt2 (target, mask, op0, op1, NULL))
5613 : return;
5614 :
5615 10 : if (TARGET_AVX2)
5616 : {
5617 5 : if (mode == V4DImode || mode == V4DFmode || mode == V16HImode)
5618 : {
5619 : /* Unfortunately, the VPERMQ and VPERMPD instructions only support
5620 : an constant shuffle operand. With a tiny bit of effort we can
5621 : use VPERMD instead. A re-interpretation stall for V4DFmode is
5622 : unfortunate but there's no avoiding it.
5623 : Similarly for V16HImode we don't have instructions for variable
5624 : shuffling, while for V32QImode we can use after preparing suitable
5625 : masks vpshufb; vpshufb; vpermq; vpor. */
5626 :
5627 : if (mode == V16HImode)
5628 : {
5629 : maskmode = mode = V32QImode;
5630 : w = 32;
5631 : e = 1;
5632 : }
5633 : else
5634 : {
5635 : maskmode = mode = V8SImode;
5636 : w = 8;
5637 : e = 4;
5638 : }
5639 0 : t1 = gen_reg_rtx (maskmode);
5640 :
5641 : /* Replicate the low bits of the V4DImode mask into V8SImode:
5642 : mask = { A B C D }
5643 : t1 = { A A B B C C D D }. */
5644 0 : for (i = 0; i < w / 2; ++i)
5645 0 : vec[i*2 + 1] = vec[i*2] = GEN_INT (i * 2);
5646 0 : vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
5647 0 : vt = force_reg (maskmode, vt);
5648 0 : mask = gen_lowpart (maskmode, mask);
5649 0 : if (maskmode == V8SImode)
5650 0 : emit_insn (gen_avx2_permvarv8si (t1, mask, vt));
5651 : else
5652 0 : emit_insn (gen_avx2_pshufbv32qi3 (t1, mask, vt));
5653 :
5654 : /* Multiply the shuffle indices by two. */
5655 0 : t1 = expand_simple_binop (maskmode, PLUS, t1, t1, t1, 1,
5656 : OPTAB_DIRECT);
5657 :
5658 : /* Add one to the odd shuffle indices:
5659 : t1 = { A*2, A*2+1, B*2, B*2+1, ... }. */
5660 0 : for (i = 0; i < w / 2; ++i)
5661 : {
5662 0 : vec[i * 2] = const0_rtx;
5663 0 : vec[i * 2 + 1] = const1_rtx;
5664 : }
5665 0 : vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
5666 0 : vt = validize_mem (force_const_mem (maskmode, vt));
5667 0 : t1 = expand_simple_binop (maskmode, PLUS, t1, vt, t1, 1,
5668 : OPTAB_DIRECT);
5669 :
5670 : /* Continue as if V8SImode (resp. V32QImode) was used initially. */
5671 0 : operands[3] = mask = t1;
5672 0 : target = gen_reg_rtx (mode);
5673 0 : op0 = gen_lowpart (mode, op0);
5674 0 : op1 = gen_lowpart (mode, op1);
5675 : }
5676 :
5677 5 : switch (mode)
5678 : {
5679 1 : case E_V8SImode:
5680 : /* The VPERMD and VPERMPS instructions already properly ignore
5681 : the high bits of the shuffle elements. No need for us to
5682 : perform an AND ourselves. */
5683 1 : if (one_operand_shuffle)
5684 : {
5685 0 : emit_insn (gen_avx2_permvarv8si (target, op0, mask));
5686 0 : if (target != operands[0])
5687 0 : emit_move_insn (operands[0],
5688 0 : gen_lowpart (GET_MODE (operands[0]), target));
5689 : }
5690 : else
5691 : {
5692 1 : t1 = gen_reg_rtx (V8SImode);
5693 1 : t2 = gen_reg_rtx (V8SImode);
5694 1 : emit_insn (gen_avx2_permvarv8si (t1, op0, mask));
5695 1 : emit_insn (gen_avx2_permvarv8si (t2, op1, mask));
5696 1 : goto merge_two;
5697 : }
5698 : return;
5699 :
5700 0 : case E_V8SFmode:
5701 0 : mask = gen_lowpart (V8SImode, mask);
5702 0 : if (one_operand_shuffle)
5703 0 : emit_insn (gen_avx2_permvarv8sf (target, op0, mask));
5704 : else
5705 : {
5706 0 : t1 = gen_reg_rtx (V8SFmode);
5707 0 : t2 = gen_reg_rtx (V8SFmode);
5708 0 : emit_insn (gen_avx2_permvarv8sf (t1, op0, mask));
5709 0 : emit_insn (gen_avx2_permvarv8sf (t2, op1, mask));
5710 0 : goto merge_two;
5711 : }
5712 0 : return;
5713 :
5714 1 : case E_V4SImode:
5715 1 : if (one_operand_shuffle)
5716 : break; /* Handled below for TARGET_AVX. */
5717 : /* By combining the two 128-bit input vectors into one 256-bit
5718 : input vector, we can use VPERMD and VPERMPS for the full
5719 : two-operand shuffle. */
5720 0 : t1 = gen_reg_rtx (V8SImode);
5721 0 : t2 = gen_reg_rtx (V8SImode);
5722 0 : emit_insn (gen_avx_vec_concatv8si (t1, op0, op1));
5723 0 : emit_insn (gen_avx_vec_concatv8si (t2, mask, mask));
5724 0 : emit_insn (gen_avx2_permvarv8si (t1, t1, t2));
5725 0 : emit_insn (gen_avx_vextractf128v8si (target, t1, const0_rtx));
5726 0 : return;
5727 :
5728 1 : case E_V4SFmode:
5729 1 : if (one_operand_shuffle)
5730 : break; /* Handled below for TARGET_AVX. */
5731 0 : t1 = gen_reg_rtx (V8SFmode);
5732 0 : t2 = gen_reg_rtx (V8SImode);
5733 0 : mask = gen_lowpart (V4SImode, mask);
5734 0 : emit_insn (gen_avx_vec_concatv8sf (t1, op0, op1));
5735 0 : emit_insn (gen_avx_vec_concatv8si (t2, mask, mask));
5736 0 : emit_insn (gen_avx2_permvarv8sf (t1, t1, t2));
5737 0 : emit_insn (gen_avx_vextractf128v8sf (target, t1, const0_rtx));
5738 0 : return;
5739 :
5740 0 : case E_V32QImode:
5741 0 : t1 = gen_reg_rtx (V32QImode);
5742 0 : t2 = gen_reg_rtx (V32QImode);
5743 0 : t3 = gen_reg_rtx (V32QImode);
5744 0 : vt2 = GEN_INT (-128);
5745 0 : vt = gen_const_vec_duplicate (V32QImode, vt2);
5746 0 : vt = force_reg (V32QImode, vt);
5747 0 : for (i = 0; i < 32; i++)
5748 0 : vec[i] = i < 16 ? vt2 : const0_rtx;
5749 0 : vt2 = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, vec));
5750 0 : vt2 = force_reg (V32QImode, vt2);
5751 : /* From mask create two adjusted masks, which contain the same
5752 : bits as mask in the low 7 bits of each vector element.
5753 : The first mask will have the most significant bit clear
5754 : if it requests element from the same 128-bit lane
5755 : and MSB set if it requests element from the other 128-bit lane.
5756 : The second mask will have the opposite values of the MSB,
5757 : and additionally will have its 128-bit lanes swapped.
5758 : E.g. { 07 12 1e 09 ... | 17 19 05 1f ... } mask vector will have
5759 : t1 { 07 92 9e 09 ... | 17 19 85 1f ... } and
5760 : t3 { 97 99 05 9f ... | 87 12 1e 89 ... } where each ...
5761 : stands for other 12 bytes. */
5762 : /* The bit whether element is from the same lane or the other
5763 : lane is bit 4, so shift it up by 3 to the MSB position. */
5764 0 : t5 = gen_reg_rtx (V4DImode);
5765 0 : emit_insn (gen_ashlv4di3 (t5, gen_lowpart (V4DImode, mask),
5766 : GEN_INT (3)));
5767 : /* Clear MSB bits from the mask just in case it had them set. */
5768 0 : emit_insn (gen_avx2_andnotv32qi3 (t2, vt, mask));
5769 : /* After this t1 will have MSB set for elements from other lane. */
5770 0 : emit_insn (gen_xorv32qi3 (t1, gen_lowpart (V32QImode, t5), vt2));
5771 : /* Clear bits other than MSB. */
5772 0 : emit_insn (gen_andv32qi3 (t1, t1, vt));
5773 : /* Or in the lower bits from mask into t3. */
5774 0 : emit_insn (gen_iorv32qi3 (t3, t1, t2));
5775 : /* And invert MSB bits in t1, so MSB is set for elements from the same
5776 : lane. */
5777 0 : emit_insn (gen_xorv32qi3 (t1, t1, vt));
5778 : /* Swap 128-bit lanes in t3. */
5779 0 : t6 = gen_reg_rtx (V4DImode);
5780 0 : emit_insn (gen_avx2_permv4di_1 (t6, gen_lowpart (V4DImode, t3),
5781 : const2_rtx, GEN_INT (3),
5782 : const0_rtx, const1_rtx));
5783 : /* And or in the lower bits from mask into t1. */
5784 0 : emit_insn (gen_iorv32qi3 (t1, t1, t2));
5785 0 : if (one_operand_shuffle)
5786 : {
5787 : /* Each of these shuffles will put 0s in places where
5788 : element from the other 128-bit lane is needed, otherwise
5789 : will shuffle in the requested value. */
5790 0 : emit_insn (gen_avx2_pshufbv32qi3 (t3, op0,
5791 0 : gen_lowpart (V32QImode, t6)));
5792 0 : emit_insn (gen_avx2_pshufbv32qi3 (t1, op0, t1));
5793 : /* For t3 the 128-bit lanes are swapped again. */
5794 0 : t7 = gen_reg_rtx (V4DImode);
5795 0 : emit_insn (gen_avx2_permv4di_1 (t7, gen_lowpart (V4DImode, t3),
5796 : const2_rtx, GEN_INT (3),
5797 : const0_rtx, const1_rtx));
5798 : /* And oring both together leads to the result. */
5799 0 : emit_insn (gen_iorv32qi3 (target, t1,
5800 0 : gen_lowpart (V32QImode, t7)));
5801 0 : if (target != operands[0])
5802 0 : emit_move_insn (operands[0],
5803 0 : gen_lowpart (GET_MODE (operands[0]), target));
5804 : return;
5805 : }
5806 :
5807 0 : t4 = gen_reg_rtx (V32QImode);
5808 : /* Similarly to the above one_operand_shuffle code,
5809 : just for repeated twice for each operand. merge_two:
5810 : code will merge the two results together. */
5811 0 : emit_insn (gen_avx2_pshufbv32qi3 (t4, op0,
5812 0 : gen_lowpart (V32QImode, t6)));
5813 0 : emit_insn (gen_avx2_pshufbv32qi3 (t3, op1,
5814 0 : gen_lowpart (V32QImode, t6)));
5815 0 : emit_insn (gen_avx2_pshufbv32qi3 (t2, op0, t1));
5816 0 : emit_insn (gen_avx2_pshufbv32qi3 (t1, op1, t1));
5817 0 : t7 = gen_reg_rtx (V4DImode);
5818 0 : emit_insn (gen_avx2_permv4di_1 (t7, gen_lowpart (V4DImode, t4),
5819 : const2_rtx, GEN_INT (3),
5820 : const0_rtx, const1_rtx));
5821 0 : t8 = gen_reg_rtx (V4DImode);
5822 0 : emit_insn (gen_avx2_permv4di_1 (t8, gen_lowpart (V4DImode, t3),
5823 : const2_rtx, GEN_INT (3),
5824 : const0_rtx, const1_rtx));
5825 0 : emit_insn (gen_iorv32qi3 (t4, t2, gen_lowpart (V32QImode, t7)));
5826 0 : emit_insn (gen_iorv32qi3 (t3, t1, gen_lowpart (V32QImode, t8)));
5827 0 : t1 = t4;
5828 0 : t2 = t3;
5829 0 : goto merge_two;
5830 :
5831 2 : default:
5832 4 : gcc_assert (GET_MODE_SIZE (mode) <= 16);
5833 : break;
5834 : }
5835 : }
5836 :
5837 9 : if (TARGET_AVX && one_operand_shuffle)
5838 8 : switch (mode)
5839 : {
5840 2 : case V4SImode:
5841 2 : op0 = gen_lowpart (V4SFmode, op0);
5842 2 : t1 = gen_reg_rtx (V4SFmode);
5843 2 : emit_insn (gen_avx_vpermilvarv4sf3 (t1, op0, mask));
5844 2 : emit_move_insn (target, gen_lowpart (mode, t1));
5845 2 : return;
5846 2 : case V4SFmode:
5847 2 : emit_insn (gen_avx_vpermilvarv4sf3 (target, op0, mask));
5848 2 : return;
5849 2 : case V2DImode:
5850 2 : op0 = gen_lowpart (V2DFmode, op0);
5851 2 : t1 = gen_reg_rtx (V2DImode);
5852 2 : t2 = gen_reg_rtx (V2DFmode);
5853 2 : emit_insn (gen_addv2di3 (t1, mask, mask));
5854 2 : emit_insn (gen_avx_vpermilvarv2df3 (t2, op0, t1));
5855 2 : emit_move_insn (target, gen_lowpart (mode, t2));
5856 2 : return;
5857 2 : case V2DFmode:
5858 2 : t1 = gen_reg_rtx (V2DImode);
5859 2 : emit_insn (gen_addv2di3 (t1, mask, mask));
5860 2 : emit_insn (gen_avx_vpermilvarv2df3 (target, op0, t1));
5861 2 : return;
5862 : default:
5863 : break;
5864 : }
5865 :
5866 1 : if (TARGET_XOP)
5867 : {
5868 : /* The XOP VPPERM insn supports three inputs. By ignoring the
5869 : one_operand_shuffle special case, we avoid creating another
5870 : set of constant vectors in memory. */
5871 0 : one_operand_shuffle = false;
5872 :
5873 : /* mask = mask & {2*w-1, ...} */
5874 0 : vt = GEN_INT (2*w - 1);
5875 : }
5876 : else
5877 : {
5878 : /* mask = mask & {w-1, ...} */
5879 1 : vt = GEN_INT (w - 1);
5880 : }
5881 :
5882 1 : vt = gen_const_vec_duplicate (maskmode, vt);
5883 1 : mask = expand_simple_binop (maskmode, AND, mask, vt,
5884 : NULL_RTX, 0, OPTAB_DIRECT);
5885 :
5886 : /* For non-QImode operations, convert the word permutation control
5887 : into a byte permutation control. */
5888 1 : if (mode != V16QImode)
5889 : {
5890 1 : mask = expand_simple_binop (maskmode, ASHIFT, mask,
5891 2 : GEN_INT (exact_log2 (e)),
5892 : NULL_RTX, 0, OPTAB_DIRECT);
5893 :
5894 : /* Convert mask to vector of chars. */
5895 1 : mask = force_reg (V16QImode, gen_lowpart (V16QImode, mask));
5896 :
5897 : /* Replicate each of the input bytes into byte positions:
5898 : (v2di) --> {0,0,0,0,0,0,0,0, 8,8,8,8,8,8,8,8}
5899 : (v4si) --> {0,0,0,0, 4,4,4,4, 8,8,8,8, 12,12,12,12}
5900 : (v8hi) --> {0,0, 2,2, 4,4, 6,6, ...}. */
5901 18 : for (i = 0; i < 16; ++i)
5902 16 : vec[i] = GEN_INT (i/e * e);
5903 1 : vt = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, vec));
5904 1 : vt = validize_mem (force_const_mem (V16QImode, vt));
5905 1 : if (TARGET_XOP)
5906 0 : emit_insn (gen_xop_pperm (mask, mask, mask, vt));
5907 : else
5908 1 : emit_insn (gen_ssse3_pshufbv16qi3 (mask, mask, vt));
5909 :
5910 : /* Convert it into the byte positions by doing
5911 : mask = mask + {0,1,..,16/w, 0,1,..,16/w, ...} */
5912 17 : for (i = 0; i < 16; ++i)
5913 16 : vec[i] = GEN_INT (i % e);
5914 1 : vt = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, vec));
5915 1 : vt = validize_mem (force_const_mem (V16QImode, vt));
5916 1 : emit_insn (gen_addv16qi3 (mask, mask, vt));
5917 : }
5918 :
5919 : /* The actual shuffle operations all operate on V16QImode. */
5920 1 : op0 = gen_lowpart (V16QImode, op0);
5921 1 : op1 = gen_lowpart (V16QImode, op1);
5922 :
5923 1 : if (TARGET_XOP)
5924 : {
5925 0 : if (GET_MODE (target) != V16QImode)
5926 0 : target = gen_reg_rtx (V16QImode);
5927 0 : emit_insn (gen_xop_pperm (target, op0, op1, mask));
5928 0 : if (target != operands[0])
5929 0 : emit_move_insn (operands[0],
5930 0 : gen_lowpart (GET_MODE (operands[0]), target));
5931 : }
5932 1 : else if (one_operand_shuffle)
5933 : {
5934 1 : if (GET_MODE (target) != V16QImode)
5935 1 : target = gen_reg_rtx (V16QImode);
5936 1 : emit_insn (gen_ssse3_pshufbv16qi3 (target, op0, mask));
5937 1 : if (target != operands[0])
5938 1 : emit_move_insn (operands[0],
5939 1 : gen_lowpart (GET_MODE (operands[0]), target));
5940 : }
5941 : else
5942 : {
5943 0 : rtx xops[6];
5944 0 : bool ok;
5945 :
5946 : /* Shuffle the two input vectors independently. */
5947 0 : t1 = gen_reg_rtx (V16QImode);
5948 0 : t2 = gen_reg_rtx (V16QImode);
5949 0 : emit_insn (gen_ssse3_pshufbv16qi3 (t1, op0, mask));
5950 0 : emit_insn (gen_ssse3_pshufbv16qi3 (t2, op1, mask));
5951 :
5952 1 : merge_two:
5953 : /* Then merge them together. The key is whether any given control
5954 : element contained a bit set that indicates the second word. */
5955 1 : mask = operands[3];
5956 1 : vt = GEN_INT (w);
5957 1 : if (maskmode == V2DImode && !TARGET_SSE4_1)
5958 : {
5959 : /* Without SSE4.1, we don't have V2DImode EQ. Perform one
5960 : more shuffle to convert the V2DI input mask into a V4SI
5961 : input mask. At which point the masking that expand_int_vcond
5962 : will work as desired. */
5963 0 : rtx t3 = gen_reg_rtx (V4SImode);
5964 0 : emit_insn (gen_sse2_pshufd_1 (t3, gen_lowpart (V4SImode, mask),
5965 : const0_rtx, const0_rtx,
5966 : const2_rtx, const2_rtx));
5967 0 : mask = t3;
5968 0 : maskmode = V4SImode;
5969 0 : e = w = 4;
5970 : }
5971 :
5972 1 : vt = gen_const_vec_duplicate (maskmode, vt);
5973 1 : vt = force_reg (maskmode, vt);
5974 1 : mask = expand_simple_binop (maskmode, AND, mask, vt,
5975 : NULL_RTX, 0, OPTAB_DIRECT);
5976 :
5977 1 : if (GET_MODE (target) != mode)
5978 0 : target = gen_reg_rtx (mode);
5979 1 : xops[0] = target;
5980 1 : xops[1] = gen_lowpart (mode, t2);
5981 1 : xops[2] = gen_lowpart (mode, t1);
5982 1 : xops[3] = gen_rtx_EQ (maskmode, mask, vt);
5983 1 : xops[4] = mask;
5984 1 : xops[5] = vt;
5985 1 : ok = ix86_expand_int_vcond (xops);
5986 1 : gcc_assert (ok);
5987 1 : if (target != operands[0])
5988 0 : emit_move_insn (operands[0],
5989 0 : gen_lowpart (GET_MODE (operands[0]), target));
5990 : }
5991 : }
5992 :
5993 : /* Extend SRC into next wider integer vector type. UNSIGNED_P is
5994 : true if we should do zero extension, else sign extension. */
5995 :
5996 : void
5997 809 : ix86_expand_sse_extend (rtx dest, rtx src, bool unsigned_p)
5998 : {
5999 809 : machine_mode imode = GET_MODE (src);
6000 809 : rtx ops[3];
6001 :
6002 809 : switch (imode)
6003 : {
6004 809 : case E_V8QImode:
6005 809 : case E_V4QImode:
6006 809 : case E_V2QImode:
6007 809 : case E_V4HImode:
6008 809 : case E_V2HImode:
6009 809 : case E_V2SImode:
6010 809 : break;
6011 0 : default:
6012 0 : gcc_unreachable ();
6013 : }
6014 :
6015 809 : ops[0] = dest;
6016 :
6017 809 : ops[1] = force_reg (imode, src);
6018 :
6019 809 : if (unsigned_p)
6020 281 : ops[2] = force_reg (imode, CONST0_RTX (imode));
6021 : else
6022 528 : ops[2] = ix86_expand_sse_cmp (gen_reg_rtx (imode), GT, CONST0_RTX (imode),
6023 : ops[1], pc_rtx, pc_rtx);
6024 :
6025 809 : ix86_split_mmx_punpck (ops, false);
6026 809 : }
6027 :
6028 : /* Unpack SRC into the next wider integer vector type. UNSIGNED_P is
6029 : true if we should do zero extension, else sign extension. HIGH_P is
6030 : true if we want the N/2 high elements, else the low elements. */
6031 :
6032 : void
6033 19396 : ix86_expand_sse_unpack (rtx dest, rtx src, bool unsigned_p, bool high_p)
6034 : {
6035 19396 : machine_mode imode = GET_MODE (src);
6036 19396 : rtx tmp;
6037 :
6038 19396 : if (TARGET_SSE4_1)
6039 : {
6040 6524 : rtx (*unpack)(rtx, rtx);
6041 6524 : rtx (*extract)(rtx, rtx) = NULL;
6042 6524 : machine_mode halfmode = BLKmode;
6043 :
6044 6524 : switch (imode)
6045 : {
6046 198 : case E_V64QImode:
6047 198 : if (unsigned_p)
6048 : unpack = gen_avx512bw_zero_extendv32qiv32hi2;
6049 : else
6050 64 : unpack = gen_avx512bw_sign_extendv32qiv32hi2;
6051 198 : halfmode = V32QImode;
6052 198 : extract
6053 198 : = high_p ? gen_vec_extract_hi_v64qi : gen_vec_extract_lo_v64qi;
6054 : break;
6055 711 : case E_V32QImode:
6056 711 : if (unsigned_p)
6057 : unpack = gen_avx2_zero_extendv16qiv16hi2;
6058 : else
6059 142 : unpack = gen_avx2_sign_extendv16qiv16hi2;
6060 711 : halfmode = V16QImode;
6061 711 : extract
6062 711 : = high_p ? gen_vec_extract_hi_v32qi : gen_vec_extract_lo_v32qi;
6063 : break;
6064 104 : case E_V32HImode:
6065 104 : if (unsigned_p)
6066 : unpack = gen_avx512f_zero_extendv16hiv16si2;
6067 : else
6068 64 : unpack = gen_avx512f_sign_extendv16hiv16si2;
6069 104 : halfmode = V16HImode;
6070 104 : extract
6071 104 : = high_p ? gen_vec_extract_hi_v32hi : gen_vec_extract_lo_v32hi;
6072 : break;
6073 417 : case E_V16HImode:
6074 417 : if (unsigned_p)
6075 : unpack = gen_avx2_zero_extendv8hiv8si2;
6076 : else
6077 302 : unpack = gen_avx2_sign_extendv8hiv8si2;
6078 417 : halfmode = V8HImode;
6079 417 : extract
6080 417 : = high_p ? gen_vec_extract_hi_v16hi : gen_vec_extract_lo_v16hi;
6081 : break;
6082 106 : case E_V16SImode:
6083 106 : if (unsigned_p)
6084 : unpack = gen_avx512f_zero_extendv8siv8di2;
6085 : else
6086 88 : unpack = gen_avx512f_sign_extendv8siv8di2;
6087 106 : halfmode = V8SImode;
6088 106 : extract
6089 106 : = high_p ? gen_vec_extract_hi_v16si : gen_vec_extract_lo_v16si;
6090 : break;
6091 382 : case E_V8SImode:
6092 382 : if (unsigned_p)
6093 : unpack = gen_avx2_zero_extendv4siv4di2;
6094 : else
6095 320 : unpack = gen_avx2_sign_extendv4siv4di2;
6096 382 : halfmode = V4SImode;
6097 382 : extract
6098 382 : = high_p ? gen_vec_extract_hi_v8si : gen_vec_extract_lo_v8si;
6099 : break;
6100 2619 : case E_V16QImode:
6101 2619 : if (unsigned_p)
6102 : unpack = gen_sse4_1_zero_extendv8qiv8hi2;
6103 : else
6104 292 : unpack = gen_sse4_1_sign_extendv8qiv8hi2;
6105 : break;
6106 1013 : case E_V8HImode:
6107 1013 : if (unsigned_p)
6108 : unpack = gen_sse4_1_zero_extendv4hiv4si2;
6109 : else
6110 796 : unpack = gen_sse4_1_sign_extendv4hiv4si2;
6111 : break;
6112 546 : case E_V4SImode:
6113 546 : if (unsigned_p)
6114 : unpack = gen_sse4_1_zero_extendv2siv2di2;
6115 : else
6116 486 : unpack = gen_sse4_1_sign_extendv2siv2di2;
6117 : break;
6118 127 : case E_V8QImode:
6119 127 : if (unsigned_p)
6120 : unpack = gen_sse4_1_zero_extendv4qiv4hi2;
6121 : else
6122 86 : unpack = gen_sse4_1_sign_extendv4qiv4hi2;
6123 : break;
6124 295 : case E_V4HImode:
6125 295 : if (unsigned_p)
6126 : unpack = gen_sse4_1_zero_extendv2hiv2si2;
6127 : else
6128 236 : unpack = gen_sse4_1_sign_extendv2hiv2si2;
6129 : break;
6130 6 : case E_V4QImode:
6131 6 : if (unsigned_p)
6132 : unpack = gen_sse4_1_zero_extendv2qiv2hi2;
6133 : else
6134 0 : unpack = gen_sse4_1_sign_extendv2qiv2hi2;
6135 : break;
6136 0 : default:
6137 0 : gcc_unreachable ();
6138 : }
6139 :
6140 13048 : if (GET_MODE_SIZE (imode) >= 32)
6141 : {
6142 1918 : tmp = gen_reg_rtx (halfmode);
6143 1918 : emit_insn (extract (tmp, src));
6144 : }
6145 4606 : else if (high_p)
6146 : {
6147 2420 : switch (GET_MODE_SIZE (imode))
6148 : {
6149 993 : case 16:
6150 : /* Shift higher 8 bytes to lower 8 bytes. */
6151 993 : tmp = gen_reg_rtx (V1TImode);
6152 993 : emit_insn (gen_sse2_lshrv1ti3 (tmp, gen_lowpart (V1TImode, src),
6153 : GEN_INT (64)));
6154 993 : break;
6155 214 : case 8:
6156 : /* Shift higher 4 bytes to lower 4 bytes. */
6157 214 : tmp = gen_reg_rtx (V1DImode);
6158 214 : emit_insn (gen_mmx_lshrv1di3 (tmp, gen_lowpart (V1DImode, src),
6159 : GEN_INT (32)));
6160 214 : break;
6161 3 : case 4:
6162 : /* Shift higher 2 bytes to lower 2 bytes. */
6163 3 : tmp = gen_reg_rtx (V1SImode);
6164 3 : emit_insn (gen_mmx_lshrv1si3 (tmp, gen_lowpart (V1SImode, src),
6165 : GEN_INT (16)));
6166 3 : break;
6167 0 : default:
6168 0 : gcc_unreachable ();
6169 : }
6170 :
6171 1210 : tmp = gen_lowpart (imode, tmp);
6172 : }
6173 : else
6174 : tmp = src;
6175 :
6176 6524 : emit_insn (unpack (dest, tmp));
6177 : }
6178 : else
6179 : {
6180 12872 : rtx (*unpack)(rtx, rtx, rtx);
6181 :
6182 12872 : switch (imode)
6183 : {
6184 3527 : case E_V16QImode:
6185 3527 : if (high_p)
6186 : unpack = gen_vec_interleave_highv16qi;
6187 : else
6188 1769 : unpack = gen_vec_interleave_lowv16qi;
6189 : break;
6190 5251 : case E_V8HImode:
6191 5251 : if (high_p)
6192 : unpack = gen_vec_interleave_highv8hi;
6193 : else
6194 2628 : unpack = gen_vec_interleave_lowv8hi;
6195 : break;
6196 2398 : case E_V4SImode:
6197 2398 : if (high_p)
6198 : unpack = gen_vec_interleave_highv4si;
6199 : else
6200 1199 : unpack = gen_vec_interleave_lowv4si;
6201 : break;
6202 578 : case E_V8QImode:
6203 578 : if (high_p)
6204 : unpack = gen_mmx_punpckhbw;
6205 : else
6206 290 : unpack = gen_mmx_punpcklbw;
6207 : break;
6208 1104 : case E_V4HImode:
6209 1104 : if (high_p)
6210 : unpack = gen_mmx_punpckhwd;
6211 : else
6212 553 : unpack = gen_mmx_punpcklwd;
6213 : break;
6214 14 : case E_V4QImode:
6215 14 : if (high_p)
6216 : unpack = gen_mmx_punpckhbw_low;
6217 : else
6218 7 : unpack = gen_mmx_punpcklbw_low;
6219 : break;
6220 0 : default:
6221 0 : gcc_unreachable ();
6222 : }
6223 :
6224 12872 : if (unsigned_p)
6225 4990 : tmp = force_reg (imode, CONST0_RTX (imode));
6226 : else
6227 7882 : tmp = ix86_expand_sse_cmp (gen_reg_rtx (imode), GT, CONST0_RTX (imode),
6228 : src, pc_rtx, pc_rtx);
6229 :
6230 12872 : rtx tmp2 = gen_reg_rtx (imode);
6231 12872 : emit_insn (unpack (tmp2, src, tmp));
6232 12872 : emit_move_insn (dest, gen_lowpart (GET_MODE (dest), tmp2));
6233 : }
6234 19396 : }
6235 :
6236 : /* Return true if mem is pool constant which contains a const_vector
6237 : perm index, assign the index to PERM. */
6238 : bool
6239 35 : ix86_extract_perm_from_pool_constant (int* perm, rtx mem)
6240 : {
6241 35 : machine_mode mode = GET_MODE (mem);
6242 35 : int nelt = GET_MODE_NUNITS (mode);
6243 :
6244 35 : if (!INTEGRAL_MODE_P (mode))
6245 : return false;
6246 :
6247 : /* Needs to be constant pool. */
6248 35 : if (!(MEM_P (mem))
6249 35 : || !SYMBOL_REF_P (XEXP (mem, 0))
6250 70 : || !CONSTANT_POOL_ADDRESS_P (XEXP (mem, 0)))
6251 : return false;
6252 :
6253 35 : rtx constant = get_pool_constant (XEXP (mem, 0));
6254 :
6255 35 : if (!CONST_VECTOR_P (constant))
6256 : return false;
6257 :
6258 : /* There could be some rtx like
6259 : (mem/u/c:V16QI (symbol_ref/u:DI ("*.LC1")))
6260 : but with "*.LC1" refer to V2DI constant vector. */
6261 35 : if (GET_MODE (constant) != mode)
6262 : {
6263 0 : constant = simplify_subreg (mode, constant, GET_MODE (constant), 0);
6264 :
6265 0 : if (constant == nullptr || !CONST_VECTOR_P (constant))
6266 : return false;
6267 : }
6268 :
6269 771 : for (int i = 0; i != nelt; i++)
6270 736 : perm[i] = UINTVAL (XVECEXP (constant, 0, i));
6271 :
6272 : return true;
6273 : }
6274 :
6275 : /* Split operands 0 and 1 into half-mode parts. Similar to split_double_mode,
6276 : but works for floating pointer parameters and nonoffsetable memories.
6277 : For pushes, it returns just stack offsets; the values will be saved
6278 : in the right order. Maximally three parts are generated. */
6279 :
6280 : static int
6281 4198796 : ix86_split_to_parts (rtx operand, rtx *parts, machine_mode mode)
6282 : {
6283 4198796 : int size;
6284 :
6285 4198796 : if (!TARGET_64BIT)
6286 1637520 : size = mode==XFmode ? 3 : GET_MODE_SIZE (mode) / 4;
6287 : else
6288 6758744 : size = (GET_MODE_SIZE (mode) + 4) / 8;
6289 :
6290 4198796 : gcc_assert (!REG_P (operand) || !MMX_REGNO_P (REGNO (operand)));
6291 4198796 : gcc_assert (size >= 2 && size <= 4);
6292 :
6293 : /* Optimize constant pool reference to immediates. This is used by fp
6294 : moves, that force all constants to memory to allow combining. */
6295 4198796 : if (MEM_P (operand) && MEM_READONLY_P (operand))
6296 38085 : operand = avoid_constant_pool_reference (operand);
6297 :
6298 4198796 : if (MEM_P (operand) && !offsettable_memref_p (operand))
6299 : {
6300 : /* The only non-offsetable memories we handle are pushes. */
6301 184233 : int ok = push_operand (operand, VOIDmode);
6302 :
6303 184233 : gcc_assert (ok);
6304 :
6305 184233 : operand = copy_rtx (operand);
6306 184233 : PUT_MODE (operand, word_mode);
6307 184233 : parts[0] = parts[1] = parts[2] = parts[3] = operand;
6308 184233 : return size;
6309 : }
6310 :
6311 4014563 : if (CONST_VECTOR_P (operand))
6312 : {
6313 41948 : scalar_int_mode imode = int_mode_for_mode (mode).require ();
6314 : /* Caution: if we looked through a constant pool memory above,
6315 : the operand may actually have a different mode now. That's
6316 : ok, since we want to pun this all the way back to an integer. */
6317 41948 : operand = simplify_subreg (imode, operand, GET_MODE (operand), 0);
6318 41948 : gcc_assert (operand != NULL);
6319 41948 : mode = imode;
6320 : }
6321 :
6322 4014563 : if (!TARGET_64BIT)
6323 : {
6324 661399 : if (mode == DImode)
6325 530577 : split_double_mode (mode, &operand, 1, &parts[0], &parts[1]);
6326 : else
6327 : {
6328 130822 : int i;
6329 :
6330 130822 : if (REG_P (operand))
6331 : {
6332 67778 : gcc_assert (reload_completed);
6333 203334 : for (i = 0; i < size; i++)
6334 135556 : parts[i] = gen_rtx_REG (SImode, REGNO (operand) + i);
6335 : }
6336 63044 : else if (offsettable_memref_p (operand))
6337 : {
6338 61698 : operand = adjust_address (operand, SImode, 0);
6339 61698 : parts[0] = operand;
6340 123931 : for (i = 1; i < size; i++)
6341 62233 : parts[i] = adjust_address (operand, SImode, 4 * i);
6342 : }
6343 1346 : else if (CONST_DOUBLE_P (operand))
6344 : {
6345 1346 : const REAL_VALUE_TYPE *r;
6346 1346 : long l[4];
6347 :
6348 1346 : r = CONST_DOUBLE_REAL_VALUE (operand);
6349 1346 : switch (mode)
6350 : {
6351 0 : case E_TFmode:
6352 0 : real_to_target (l, r, mode);
6353 0 : parts[3] = gen_int_mode (l[3], SImode);
6354 0 : parts[2] = gen_int_mode (l[2], SImode);
6355 0 : break;
6356 198 : case E_XFmode:
6357 : /* We can't use REAL_VALUE_TO_TARGET_LONG_DOUBLE since
6358 : long double may not be 80-bit. */
6359 198 : real_to_target (l, r, mode);
6360 198 : parts[2] = gen_int_mode (l[2], SImode);
6361 198 : break;
6362 1148 : case E_DFmode:
6363 1148 : REAL_VALUE_TO_TARGET_DOUBLE (*r, l);
6364 1148 : break;
6365 0 : default:
6366 0 : gcc_unreachable ();
6367 : }
6368 1346 : parts[1] = gen_int_mode (l[1], SImode);
6369 1346 : parts[0] = gen_int_mode (l[0], SImode);
6370 : }
6371 : else
6372 0 : gcc_unreachable ();
6373 : }
6374 : }
6375 : else
6376 : {
6377 3353164 : if (mode == TImode)
6378 3332248 : split_double_mode (mode, &operand, 1, &parts[0], &parts[1]);
6379 3353164 : if (mode == XFmode || mode == TFmode)
6380 : {
6381 20916 : machine_mode upper_mode = mode==XFmode ? SImode : DImode;
6382 20916 : if (REG_P (operand))
6383 : {
6384 1484 : gcc_assert (reload_completed);
6385 1484 : parts[0] = gen_rtx_REG (DImode, REGNO (operand) + 0);
6386 1484 : parts[1] = gen_rtx_REG (upper_mode, REGNO (operand) + 1);
6387 : }
6388 19432 : else if (offsettable_memref_p (operand))
6389 : {
6390 15300 : operand = adjust_address (operand, DImode, 0);
6391 15300 : parts[0] = operand;
6392 15300 : parts[1] = adjust_address (operand, upper_mode, 8);
6393 : }
6394 4132 : else if (CONST_DOUBLE_P (operand))
6395 : {
6396 4132 : long l[4];
6397 :
6398 4132 : real_to_target (l, CONST_DOUBLE_REAL_VALUE (operand), mode);
6399 :
6400 : /* real_to_target puts 32-bit pieces in each long. */
6401 8264 : parts[0] = gen_int_mode ((l[0] & HOST_WIDE_INT_C (0xffffffff))
6402 4132 : | ((l[1] & HOST_WIDE_INT_C (0xffffffff))
6403 4132 : << 32), DImode);
6404 :
6405 4132 : if (upper_mode == SImode)
6406 2953 : parts[1] = gen_int_mode (l[2], SImode);
6407 : else
6408 1179 : parts[1]
6409 1179 : = gen_int_mode ((l[2] & HOST_WIDE_INT_C (0xffffffff))
6410 1179 : | ((l[3] & HOST_WIDE_INT_C (0xffffffff))
6411 1179 : << 32), DImode);
6412 : }
6413 : else
6414 0 : gcc_unreachable ();
6415 : }
6416 : }
6417 :
6418 : return size;
6419 : }
6420 :
6421 : /* Emit insns to perform a move or push of DI, DF, XF, and TF values.
6422 : Return false when normal moves are needed; true when all required
6423 : insns have been emitted. Operands 2-4 contain the input values
6424 : int the correct order; operands 5-7 contain the output values. */
6425 :
6426 : void
6427 2112358 : ix86_split_long_move (rtx operands[])
6428 : {
6429 2112358 : rtx part[2][4];
6430 2112358 : int nparts, i, j;
6431 2112358 : int push = 0;
6432 2112358 : int collisions = 0;
6433 2112358 : machine_mode mode = GET_MODE (operands[0]);
6434 2112358 : bool collisionparts[4];
6435 :
6436 : /* The DFmode expanders may ask us to move double.
6437 : For 64bit target this is single move. By hiding the fact
6438 : here we simplify i386.md splitters. */
6439 3815004 : if (TARGET_64BIT && GET_MODE_SIZE (GET_MODE (operands[0])) == 8)
6440 : {
6441 : /* Optimize constant pool reference to immediates. This is used by
6442 : fp moves, that force all constants to memory to allow combining. */
6443 :
6444 12960 : if (MEM_P (operands[1])
6445 12537 : && SYMBOL_REF_P (XEXP (operands[1], 0))
6446 13604 : && CONSTANT_POOL_ADDRESS_P (XEXP (operands[1], 0)))
6447 155 : operands[1] = get_pool_constant (XEXP (operands[1], 0));
6448 12960 : if (push_operand (operands[0], VOIDmode))
6449 : {
6450 12960 : operands[0] = copy_rtx (operands[0]);
6451 12960 : PUT_MODE (operands[0], word_mode);
6452 : }
6453 : else
6454 0 : operands[0] = gen_lowpart (DImode, operands[0]);
6455 12960 : operands[1] = gen_lowpart (DImode, operands[1]);
6456 12960 : emit_move_insn (operands[0], operands[1]);
6457 12960 : return;
6458 : }
6459 :
6460 : /* The only non-offsettable memory we handle is push. */
6461 2099398 : if (push_operand (operands[0], VOIDmode))
6462 : push = 1;
6463 : else
6464 1915165 : gcc_assert (!MEM_P (operands[0])
6465 : || offsettable_memref_p (operands[0]));
6466 :
6467 2099398 : nparts = ix86_split_to_parts (operands[1], part[1], GET_MODE (operands[0]));
6468 2099398 : ix86_split_to_parts (operands[0], part[0], GET_MODE (operands[0]));
6469 :
6470 : /* When emitting push, take care for source operands on the stack. */
6471 184233 : if (push && MEM_P (operands[1])
6472 2197043 : && reg_overlap_mentioned_p (stack_pointer_rtx, operands[1]))
6473 : {
6474 55992 : rtx src_base = XEXP (part[1][nparts - 1], 0);
6475 :
6476 : /* Compensate for the stack decrement by 4. */
6477 55992 : if (!TARGET_64BIT && nparts == 3
6478 51304 : && mode == XFmode && TARGET_128BIT_LONG_DOUBLE)
6479 0 : src_base = plus_constant (Pmode, src_base, 4);
6480 :
6481 : /* src_base refers to the stack pointer and is
6482 : automatically decreased by emitted push. */
6483 168256 : for (i = 0; i < nparts; i++)
6484 112264 : part[1][i] = change_address (part[1][i],
6485 112264 : GET_MODE (part[1][i]), src_base);
6486 : }
6487 :
6488 : /* We need to do copy in the right order in case an address register
6489 : of the source overlaps the destination. */
6490 2099398 : if (REG_P (part[0][0]) && MEM_P (part[1][0]))
6491 : {
6492 : rtx tmp;
6493 :
6494 2394126 : for (i = 0; i < nparts; i++)
6495 : {
6496 1596084 : collisionparts[i]
6497 1596084 : = reg_overlap_mentioned_p (part[0][i], XEXP (part[1][0], 0));
6498 1596084 : if (collisionparts[i])
6499 16902 : collisions++;
6500 : }
6501 :
6502 : /* Collision in the middle part can be handled by reordering. */
6503 798042 : if (collisions == 1 && nparts == 3 && collisionparts [1])
6504 : {
6505 0 : std::swap (part[0][1], part[0][2]);
6506 0 : std::swap (part[1][1], part[1][2]);
6507 : }
6508 798042 : else if (collisions == 1
6509 798042 : && nparts == 4
6510 0 : && (collisionparts [1] || collisionparts [2]))
6511 : {
6512 0 : if (collisionparts [1])
6513 : {
6514 0 : std::swap (part[0][1], part[0][2]);
6515 0 : std::swap (part[1][1], part[1][2]);
6516 : }
6517 : else
6518 : {
6519 0 : std::swap (part[0][2], part[0][3]);
6520 0 : std::swap (part[1][2], part[1][3]);
6521 : }
6522 : }
6523 :
6524 : /* If there are more collisions, we can't handle it by reordering.
6525 : Do an lea to the last part and use only one colliding move. */
6526 798042 : else if (collisions > 1)
6527 : {
6528 77 : rtx base, addr;
6529 :
6530 77 : collisions = 1;
6531 :
6532 77 : base = part[0][nparts - 1];
6533 :
6534 : /* Handle the case when the last part isn't valid for lea.
6535 : Happens in 64-bit mode storing the 12-byte XFmode. */
6536 112 : if (GET_MODE (base) != Pmode)
6537 0 : base = gen_rtx_REG (Pmode, REGNO (base));
6538 :
6539 77 : addr = XEXP (part[1][0], 0);
6540 77 : if (TARGET_TLS_DIRECT_SEG_REFS)
6541 : {
6542 77 : struct ix86_address parts;
6543 77 : int ok = ix86_decompose_address (addr, &parts);
6544 77 : gcc_assert (ok);
6545 : /* It is not valid to use %gs: or %fs: in lea. */
6546 77 : gcc_assert (parts.seg == ADDR_SPACE_GENERIC);
6547 : }
6548 77 : emit_insn (gen_rtx_SET (base, addr));
6549 77 : part[1][0] = replace_equiv_address (part[1][0], base);
6550 154 : for (i = 1; i < nparts; i++)
6551 : {
6552 147 : tmp = plus_constant (Pmode, base, UNITS_PER_WORD * i);
6553 77 : part[1][i] = replace_equiv_address (part[1][i], tmp);
6554 : }
6555 : }
6556 : }
6557 :
6558 2099398 : if (push)
6559 : {
6560 184233 : if (!TARGET_64BIT)
6561 : {
6562 158025 : if (nparts == 3)
6563 : {
6564 595 : if (TARGET_128BIT_LONG_DOUBLE && mode == XFmode)
6565 0 : emit_insn (gen_add2_insn (stack_pointer_rtx, GEN_INT (-4)));
6566 595 : emit_move_insn (part[0][2], part[1][2]);
6567 : }
6568 157430 : else if (nparts == 4)
6569 : {
6570 0 : emit_move_insn (part[0][3], part[1][3]);
6571 0 : emit_move_insn (part[0][2], part[1][2]);
6572 : }
6573 : }
6574 : else
6575 : {
6576 : /* In 64bit mode we don't have 32bit push available. In case this is
6577 : register, it is OK - we will just use larger counterpart. We also
6578 : retype memory - these comes from attempt to avoid REX prefix on
6579 : moving of second half of TFmode value. */
6580 26208 : if (GET_MODE (part[1][1]) == SImode)
6581 : {
6582 12013 : switch (GET_CODE (part[1][1]))
6583 : {
6584 11578 : case MEM:
6585 11578 : part[1][1] = adjust_address (part[1][1], DImode, 0);
6586 11578 : break;
6587 :
6588 435 : case REG:
6589 435 : part[1][1] = gen_rtx_REG (DImode, REGNO (part[1][1]));
6590 435 : break;
6591 :
6592 0 : default:
6593 0 : gcc_unreachable ();
6594 : }
6595 :
6596 12013 : if (GET_MODE (part[1][0]) == SImode)
6597 0 : part[1][0] = part[1][1];
6598 : }
6599 : }
6600 184233 : emit_move_insn (part[0][1], part[1][1]);
6601 184233 : emit_move_insn (part[0][0], part[1][0]);
6602 184233 : return;
6603 : }
6604 :
6605 : /* Choose correct order to not overwrite the source before it is copied. */
6606 1915165 : if ((REG_P (part[0][0])
6607 1050214 : && REG_P (part[1][1])
6608 92066 : && (REGNO (part[0][0]) == REGNO (part[1][1])
6609 74801 : || (nparts == 3
6610 0 : && REGNO (part[0][0]) == REGNO (part[1][2]))
6611 74801 : || (nparts == 4
6612 0 : && REGNO (part[0][0]) == REGNO (part[1][3]))))
6613 2948114 : || (collisions > 0
6614 16825 : && reg_overlap_mentioned_p (part[0][0], XEXP (part[1][0], 0))))
6615 : {
6616 101517 : for (i = 0, j = nparts - 1; i < nparts; i++, j--)
6617 : {
6618 67678 : operands[2 + i] = part[0][j];
6619 67678 : operands[6 + i] = part[1][j];
6620 : }
6621 : }
6622 : else
6623 : {
6624 5644047 : for (i = 0; i < nparts; i++)
6625 : {
6626 3762721 : operands[2 + i] = part[0][i];
6627 3762721 : operands[6 + i] = part[1][i];
6628 : }
6629 : }
6630 :
6631 : /* Attempt to locally unCSE nonzero constants. */
6632 3830399 : for (j = 0; j < nparts - 1; j++)
6633 1915234 : if (CONST_INT_P (operands[6 + j])
6634 227594 : && operands[6 + j] != const0_rtx
6635 63481 : && REG_P (operands[2 + j]))
6636 112884 : for (i = j; i < nparts - 1; i++)
6637 56442 : if (CONST_INT_P (operands[7 + i])
6638 56442 : && INTVAL (operands[7 + i]) == INTVAL (operands[6 + j]))
6639 22634 : operands[7 + i] = operands[2 + j];
6640 :
6641 5745564 : for (i = 0; i < nparts; i++)
6642 3830399 : emit_move_insn (operands[2 + i], operands[6 + i]);
6643 :
6644 : return;
6645 : }
6646 :
6647 : /* Helper function of ix86_split_ashl used to generate an SImode/DImode
6648 : left shift by a constant, either using a single shift or
6649 : a sequence of add instructions. */
6650 :
6651 : static void
6652 4512 : ix86_expand_ashl_const (rtx operand, int count, machine_mode mode)
6653 : {
6654 4512 : if (count == 1
6655 4512 : || (count * ix86_cost->add <= ix86_cost->shift_const
6656 0 : && !optimize_insn_for_size_p ()))
6657 : {
6658 16 : while (count-- > 0)
6659 8 : emit_insn (gen_add2_insn (operand, operand));
6660 : }
6661 : else
6662 : {
6663 4504 : rtx (*insn)(rtx, rtx, rtx);
6664 :
6665 4504 : insn = mode == DImode ? gen_ashlsi3 : gen_ashldi3;
6666 4504 : emit_insn (insn (operand, operand, GEN_INT (count)));
6667 : }
6668 4512 : }
6669 :
6670 : void
6671 10700 : ix86_split_ashl (rtx *operands, rtx scratch, machine_mode mode)
6672 : {
6673 10700 : rtx (*gen_ashl3)(rtx, rtx, rtx);
6674 10700 : rtx (*gen_shld)(rtx, rtx, rtx);
6675 10700 : int half_width = GET_MODE_BITSIZE (mode) >> 1;
6676 10700 : machine_mode half_mode;
6677 :
6678 10700 : rtx low[2], high[2];
6679 10700 : int count;
6680 :
6681 10700 : if (CONST_INT_P (operands[2]))
6682 : {
6683 8986 : split_double_mode (mode, operands, 2, low, high);
6684 8986 : count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
6685 :
6686 8986 : if (count >= half_width)
6687 : {
6688 2705 : emit_move_insn (high[0], low[1]);
6689 2705 : ix86_expand_clear (low[0]);
6690 :
6691 2705 : if (count > half_width)
6692 147 : ix86_expand_ashl_const (high[0], count - half_width, mode);
6693 : }
6694 6281 : else if (count == 1)
6695 : {
6696 1916 : if (!rtx_equal_p (operands[0], operands[1]))
6697 0 : emit_move_insn (operands[0], operands[1]);
6698 1916 : rtx x3 = gen_rtx_REG (CCCmode, FLAGS_REG);
6699 1916 : rtx x4 = gen_rtx_LTU (mode, x3, const0_rtx);
6700 1916 : half_mode = mode == DImode ? SImode : DImode;
6701 1916 : emit_insn (gen_add3_cc_overflow_1 (half_mode, low[0],
6702 : low[0], low[0]));
6703 1916 : emit_insn (gen_add3_carry (half_mode, high[0], high[0], high[0],
6704 : x3, x4));
6705 : }
6706 : else
6707 : {
6708 4365 : gen_shld = mode == DImode ? gen_x86_shld : gen_x86_64_shld;
6709 :
6710 4365 : if (!rtx_equal_p (operands[0], operands[1]))
6711 0 : emit_move_insn (operands[0], operands[1]);
6712 :
6713 4365 : emit_insn (gen_shld (high[0], low[0], GEN_INT (count)));
6714 4365 : ix86_expand_ashl_const (low[0], count, mode);
6715 : }
6716 9251 : return;
6717 : }
6718 :
6719 1714 : split_double_mode (mode, operands, 1, low, high);
6720 1714 : half_mode = mode == DImode ? SImode : DImode;
6721 :
6722 1714 : gen_ashl3 = mode == DImode ? gen_ashlsi3 : gen_ashldi3;
6723 :
6724 1714 : if (operands[1] == const1_rtx)
6725 : {
6726 : /* Assuming we've chosen a QImode capable registers, then 1 << N
6727 : can be done with two 32/64-bit shifts, no branches, no cmoves. */
6728 265 : if (ANY_QI_REG_P (low[0]) && ANY_QI_REG_P (high[0]))
6729 : {
6730 154 : rtx s, d, flags = gen_rtx_REG (CCZmode, FLAGS_REG);
6731 :
6732 154 : ix86_expand_clear (low[0]);
6733 154 : ix86_expand_clear (high[0]);
6734 154 : emit_insn (gen_testqi_ccz_1 (operands[2], GEN_INT (half_width)));
6735 :
6736 154 : d = gen_lowpart (QImode, low[0]);
6737 154 : d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
6738 154 : s = gen_rtx_EQ (QImode, flags, const0_rtx);
6739 154 : emit_insn (gen_rtx_SET (d, s));
6740 :
6741 154 : d = gen_lowpart (QImode, high[0]);
6742 154 : d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
6743 154 : s = gen_rtx_NE (QImode, flags, const0_rtx);
6744 154 : emit_insn (gen_rtx_SET (d, s));
6745 : }
6746 :
6747 : /* Otherwise, we can get the same results by manually performing
6748 : a bit extract operation on bit 5/6, and then performing the two
6749 : shifts. The two methods of getting 0/1 into low/high are exactly
6750 : the same size. Avoiding the shift in the bit extract case helps
6751 : pentium4 a bit; no one else seems to care much either way. */
6752 : else
6753 : {
6754 111 : rtx (*gen_lshr3)(rtx, rtx, rtx);
6755 111 : rtx (*gen_and3)(rtx, rtx, rtx);
6756 111 : rtx (*gen_xor3)(rtx, rtx, rtx);
6757 111 : HOST_WIDE_INT bits;
6758 111 : rtx x;
6759 :
6760 111 : if (mode == DImode)
6761 : {
6762 : gen_lshr3 = gen_lshrsi3;
6763 : gen_and3 = gen_andsi3;
6764 : gen_xor3 = gen_xorsi3;
6765 : bits = 5;
6766 : }
6767 : else
6768 : {
6769 0 : gen_lshr3 = gen_lshrdi3;
6770 0 : gen_and3 = gen_anddi3;
6771 0 : gen_xor3 = gen_xordi3;
6772 0 : bits = 6;
6773 : }
6774 :
6775 111 : if (TARGET_PARTIAL_REG_STALL && !optimize_insn_for_size_p ())
6776 0 : x = gen_rtx_ZERO_EXTEND (half_mode, operands[2]);
6777 : else
6778 111 : x = gen_lowpart (half_mode, operands[2]);
6779 111 : emit_insn (gen_rtx_SET (high[0], x));
6780 :
6781 111 : emit_insn (gen_lshr3 (high[0], high[0], GEN_INT (bits)));
6782 111 : emit_insn (gen_and3 (high[0], high[0], const1_rtx));
6783 111 : emit_move_insn (low[0], high[0]);
6784 111 : emit_insn (gen_xor3 (low[0], low[0], const1_rtx));
6785 : }
6786 :
6787 265 : emit_insn (gen_ashl3 (low[0], low[0], operands[2]));
6788 265 : emit_insn (gen_ashl3 (high[0], high[0], operands[2]));
6789 265 : return;
6790 : }
6791 :
6792 1449 : if (operands[1] == constm1_rtx)
6793 : {
6794 : /* For -1 << N, we can avoid the shld instruction, because we
6795 : know that we're shifting 0...31/63 ones into a -1. */
6796 113 : emit_move_insn (low[0], constm1_rtx);
6797 113 : if (optimize_insn_for_size_p ())
6798 6 : emit_move_insn (high[0], low[0]);
6799 : else
6800 107 : emit_move_insn (high[0], constm1_rtx);
6801 : }
6802 : else
6803 : {
6804 1336 : gen_shld = mode == DImode ? gen_x86_shld : gen_x86_64_shld;
6805 :
6806 1336 : if (!rtx_equal_p (operands[0], operands[1]))
6807 0 : emit_move_insn (operands[0], operands[1]);
6808 :
6809 1336 : split_double_mode (mode, operands, 1, low, high);
6810 1336 : emit_insn (gen_shld (high[0], low[0], operands[2]));
6811 : }
6812 :
6813 1449 : emit_insn (gen_ashl3 (low[0], low[0], operands[2]));
6814 :
6815 1449 : if (TARGET_CMOVE && scratch)
6816 : {
6817 981 : ix86_expand_clear (scratch);
6818 981 : emit_insn (gen_x86_shift_adj_1
6819 : (half_mode, high[0], low[0], operands[2], scratch));
6820 : }
6821 : else
6822 468 : emit_insn (gen_x86_shift_adj_2 (half_mode, high[0], low[0], operands[2]));
6823 : }
6824 :
6825 : void
6826 6461 : ix86_split_ashr (rtx *operands, rtx scratch, machine_mode mode)
6827 : {
6828 5213 : rtx (*gen_ashr3)(rtx, rtx, rtx)
6829 6461 : = mode == DImode ? gen_ashrsi3 : gen_ashrdi3;
6830 6461 : rtx (*gen_shrd)(rtx, rtx, rtx);
6831 6461 : int half_width = GET_MODE_BITSIZE (mode) >> 1;
6832 :
6833 6461 : rtx low[2], high[2];
6834 6461 : int count;
6835 :
6836 6461 : if (CONST_INT_P (operands[2]))
6837 : {
6838 6288 : split_double_mode (mode, operands, 2, low, high);
6839 6288 : count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
6840 :
6841 12576 : if (count == GET_MODE_BITSIZE (mode) - 1)
6842 : {
6843 92 : emit_move_insn (high[0], high[1]);
6844 92 : emit_insn (gen_ashr3 (high[0], high[0],
6845 92 : GEN_INT (half_width - 1)));
6846 92 : emit_move_insn (low[0], high[0]);
6847 :
6848 : }
6849 6196 : else if (count >= half_width)
6850 : {
6851 1997 : emit_move_insn (low[0], high[1]);
6852 1997 : emit_move_insn (high[0], low[0]);
6853 1997 : emit_insn (gen_ashr3 (high[0], high[0],
6854 1997 : GEN_INT (half_width - 1)));
6855 :
6856 1997 : if (count > half_width)
6857 38 : emit_insn (gen_ashr3 (low[0], low[0],
6858 38 : GEN_INT (count - half_width)));
6859 : }
6860 4199 : else if (count == 1
6861 765 : && (TARGET_USE_RCR || optimize_size > 1))
6862 : {
6863 1 : if (!rtx_equal_p (operands[0], operands[1]))
6864 0 : emit_move_insn (operands[0], operands[1]);
6865 1 : if (mode == DImode)
6866 : {
6867 0 : emit_insn (gen_ashrsi3_carry (high[0], high[0]));
6868 0 : emit_insn (gen_rcrsi2 (low[0], low[0]));
6869 : }
6870 : else
6871 : {
6872 1 : emit_insn (gen_ashrdi3_carry (high[0], high[0]));
6873 1 : emit_insn (gen_rcrdi2 (low[0], low[0]));
6874 : }
6875 : }
6876 : else
6877 : {
6878 4198 : gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
6879 :
6880 4198 : if (!rtx_equal_p (operands[0], operands[1]))
6881 0 : emit_move_insn (operands[0], operands[1]);
6882 :
6883 4198 : emit_insn (gen_shrd (low[0], high[0], GEN_INT (count)));
6884 4198 : emit_insn (gen_ashr3 (high[0], high[0], GEN_INT (count)));
6885 : }
6886 : }
6887 : else
6888 : {
6889 173 : machine_mode half_mode;
6890 :
6891 173 : gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
6892 :
6893 173 : if (!rtx_equal_p (operands[0], operands[1]))
6894 0 : emit_move_insn (operands[0], operands[1]);
6895 :
6896 173 : split_double_mode (mode, operands, 1, low, high);
6897 173 : half_mode = mode == DImode ? SImode : DImode;
6898 :
6899 173 : emit_insn (gen_shrd (low[0], high[0], operands[2]));
6900 173 : emit_insn (gen_ashr3 (high[0], high[0], operands[2]));
6901 :
6902 173 : if (TARGET_CMOVE && scratch)
6903 : {
6904 133 : emit_move_insn (scratch, high[0]);
6905 133 : emit_insn (gen_ashr3 (scratch, scratch,
6906 133 : GEN_INT (half_width - 1)));
6907 133 : emit_insn (gen_x86_shift_adj_1
6908 : (half_mode, low[0], high[0], operands[2], scratch));
6909 : }
6910 : else
6911 40 : emit_insn (gen_x86_shift_adj_3
6912 : (half_mode, low[0], high[0], operands[2]));
6913 : }
6914 6461 : }
6915 :
6916 : void
6917 16848 : ix86_split_lshr (rtx *operands, rtx scratch, machine_mode mode)
6918 : {
6919 6707 : rtx (*gen_lshr3)(rtx, rtx, rtx)
6920 16848 : = mode == DImode ? gen_lshrsi3 : gen_lshrdi3;
6921 16848 : rtx (*gen_shrd)(rtx, rtx, rtx);
6922 16848 : int half_width = GET_MODE_BITSIZE (mode) >> 1;
6923 :
6924 16848 : rtx low[2], high[2];
6925 16848 : int count;
6926 :
6927 16848 : if (CONST_INT_P (operands[2]))
6928 : {
6929 15411 : split_double_mode (mode, operands, 2, low, high);
6930 15411 : count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
6931 :
6932 15411 : if (count >= half_width)
6933 : {
6934 11995 : emit_move_insn (low[0], high[1]);
6935 11995 : ix86_expand_clear (high[0]);
6936 :
6937 11995 : if (count > half_width)
6938 1510 : emit_insn (gen_lshr3 (low[0], low[0],
6939 1510 : GEN_INT (count - half_width)));
6940 : }
6941 3416 : else if (count == 1
6942 643 : && (TARGET_USE_RCR || optimize_size > 1))
6943 : {
6944 1 : if (!rtx_equal_p (operands[0], operands[1]))
6945 0 : emit_move_insn (operands[0], operands[1]);
6946 1 : if (mode == DImode)
6947 : {
6948 0 : emit_insn (gen_lshrsi3_carry (high[0], high[0]));
6949 0 : emit_insn (gen_rcrsi2 (low[0], low[0]));
6950 : }
6951 : else
6952 : {
6953 1 : emit_insn (gen_lshrdi3_carry (high[0], high[0]));
6954 1 : emit_insn (gen_rcrdi2 (low[0], low[0]));
6955 : }
6956 : }
6957 : else
6958 : {
6959 3415 : gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
6960 :
6961 3415 : if (!rtx_equal_p (operands[0], operands[1]))
6962 0 : emit_move_insn (operands[0], operands[1]);
6963 :
6964 3415 : emit_insn (gen_shrd (low[0], high[0], GEN_INT (count)));
6965 3415 : emit_insn (gen_lshr3 (high[0], high[0], GEN_INT (count)));
6966 : }
6967 : }
6968 : else
6969 : {
6970 1437 : machine_mode half_mode;
6971 :
6972 1437 : gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
6973 :
6974 1437 : if (!rtx_equal_p (operands[0], operands[1]))
6975 0 : emit_move_insn (operands[0], operands[1]);
6976 :
6977 1437 : split_double_mode (mode, operands, 1, low, high);
6978 1437 : half_mode = mode == DImode ? SImode : DImode;
6979 :
6980 1437 : emit_insn (gen_shrd (low[0], high[0], operands[2]));
6981 1437 : emit_insn (gen_lshr3 (high[0], high[0], operands[2]));
6982 :
6983 1437 : if (TARGET_CMOVE && scratch)
6984 : {
6985 1136 : ix86_expand_clear (scratch);
6986 1136 : emit_insn (gen_x86_shift_adj_1
6987 : (half_mode, low[0], high[0], operands[2], scratch));
6988 : }
6989 : else
6990 301 : emit_insn (gen_x86_shift_adj_2
6991 : (half_mode, low[0], high[0], operands[2]));
6992 : }
6993 16848 : }
6994 :
6995 : /* Helper function to split TImode ashl under NDD. */
6996 : void
6997 1 : ix86_split_ashl_ndd (rtx *operands, rtx scratch)
6998 : {
6999 1 : gcc_assert (TARGET_APX_NDD);
7000 1 : int half_width = GET_MODE_BITSIZE (TImode) >> 1;
7001 :
7002 1 : rtx low[2], high[2];
7003 1 : int count;
7004 :
7005 1 : split_double_mode (TImode, operands, 2, low, high);
7006 1 : if (CONST_INT_P (operands[2]))
7007 : {
7008 0 : count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (TImode) - 1);
7009 :
7010 0 : if (count >= half_width)
7011 : {
7012 0 : count = count - half_width;
7013 0 : if (count == 0)
7014 : {
7015 0 : if (!rtx_equal_p (high[0], low[1]))
7016 0 : emit_move_insn (high[0], low[1]);
7017 : }
7018 0 : else if (count == 1)
7019 0 : emit_insn (gen_adddi3 (high[0], low[1], low[1]));
7020 : else
7021 0 : emit_insn (gen_ashldi3 (high[0], low[1], GEN_INT (count)));
7022 :
7023 0 : ix86_expand_clear (low[0]);
7024 : }
7025 0 : else if (count == 1)
7026 : {
7027 0 : rtx x3 = gen_rtx_REG (CCCmode, FLAGS_REG);
7028 0 : rtx x4 = gen_rtx_LTU (TImode, x3, const0_rtx);
7029 0 : emit_insn (gen_add3_cc_overflow_1 (DImode, low[0],
7030 : low[1], low[1]));
7031 0 : emit_insn (gen_add3_carry (DImode, high[0], high[1], high[1],
7032 : x3, x4));
7033 : }
7034 : else
7035 : {
7036 0 : emit_insn (gen_x86_64_shld_ndd (high[0], high[1], low[1],
7037 : GEN_INT (count)));
7038 0 : emit_insn (gen_ashldi3 (low[0], low[1], GEN_INT (count)));
7039 : }
7040 : }
7041 : else
7042 : {
7043 1 : emit_insn (gen_x86_64_shld_ndd (high[0], high[1], low[1],
7044 : operands[2]));
7045 1 : emit_insn (gen_ashldi3 (low[0], low[1], operands[2]));
7046 1 : if (TARGET_CMOVE && scratch)
7047 : {
7048 1 : ix86_expand_clear (scratch);
7049 1 : emit_insn (gen_x86_shift_adj_1
7050 : (DImode, high[0], low[0], operands[2], scratch));
7051 : }
7052 : else
7053 0 : emit_insn (gen_x86_shift_adj_2 (DImode, high[0], low[0], operands[2]));
7054 : }
7055 1 : }
7056 :
7057 : /* Helper function to split TImode l/ashr under NDD. */
7058 : void
7059 2 : ix86_split_rshift_ndd (enum rtx_code code, rtx *operands, rtx scratch)
7060 : {
7061 2 : gcc_assert (TARGET_APX_NDD);
7062 2 : int half_width = GET_MODE_BITSIZE (TImode) >> 1;
7063 2 : bool ashr_p = code == ASHIFTRT;
7064 2 : rtx (*gen_shr)(rtx, rtx, rtx) = ashr_p ? gen_ashrdi3
7065 : : gen_lshrdi3;
7066 :
7067 2 : rtx low[2], high[2];
7068 2 : int count;
7069 :
7070 2 : split_double_mode (TImode, operands, 2, low, high);
7071 2 : if (CONST_INT_P (operands[2]))
7072 : {
7073 0 : count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (TImode) - 1);
7074 :
7075 0 : if (ashr_p && (count == GET_MODE_BITSIZE (TImode) - 1))
7076 : {
7077 0 : emit_insn (gen_shr (high[0], high[1],
7078 : GEN_INT (half_width - 1)));
7079 0 : emit_move_insn (low[0], high[0]);
7080 : }
7081 0 : else if (count >= half_width)
7082 : {
7083 0 : if (ashr_p)
7084 0 : emit_insn (gen_shr (high[0], high[1],
7085 : GEN_INT (half_width - 1)));
7086 : else
7087 0 : ix86_expand_clear (high[0]);
7088 :
7089 0 : if (count > half_width)
7090 0 : emit_insn (gen_shr (low[0], high[1],
7091 0 : GEN_INT (count - half_width)));
7092 : else
7093 0 : emit_move_insn (low[0], high[1]);
7094 : }
7095 : else
7096 : {
7097 0 : emit_insn (gen_x86_64_shrd_ndd (low[0], low[1], high[1],
7098 : GEN_INT (count)));
7099 0 : emit_insn (gen_shr (high[0], high[1], GEN_INT (count)));
7100 : }
7101 : }
7102 : else
7103 : {
7104 2 : emit_insn (gen_x86_64_shrd_ndd (low[0], low[1], high[1],
7105 : operands[2]));
7106 2 : emit_insn (gen_shr (high[0], high[1], operands[2]));
7107 :
7108 2 : if (TARGET_CMOVE && scratch)
7109 : {
7110 2 : if (ashr_p)
7111 : {
7112 1 : emit_move_insn (scratch, high[0]);
7113 1 : emit_insn (gen_shr (scratch, scratch,
7114 : GEN_INT (half_width - 1)));
7115 : }
7116 : else
7117 1 : ix86_expand_clear (scratch);
7118 :
7119 2 : emit_insn (gen_x86_shift_adj_1
7120 : (DImode, low[0], high[0], operands[2], scratch));
7121 : }
7122 0 : else if (ashr_p)
7123 0 : emit_insn (gen_x86_shift_adj_3
7124 : (DImode, low[0], high[0], operands[2]));
7125 : else
7126 0 : emit_insn (gen_x86_shift_adj_2
7127 : (DImode, low[0], high[0], operands[2]));
7128 : }
7129 2 : }
7130 :
7131 : /* Expand move of V1TI mode register X to a new TI mode register. */
7132 : static rtx
7133 17 : ix86_expand_v1ti_to_ti (rtx x)
7134 : {
7135 17 : rtx result = gen_reg_rtx (TImode);
7136 17 : if (TARGET_SSE2)
7137 : {
7138 17 : rtx temp = force_reg (V2DImode, gen_lowpart (V2DImode, x));
7139 17 : rtx lo = gen_lowpart (DImode, result);
7140 17 : emit_insn (gen_vec_extractv2didi (lo, temp, const0_rtx));
7141 17 : rtx hi = gen_highpart (DImode, result);
7142 17 : emit_insn (gen_vec_extractv2didi (hi, temp, const1_rtx));
7143 : }
7144 : else
7145 0 : emit_move_insn (result, gen_lowpart (TImode, x));
7146 17 : return result;
7147 : }
7148 :
7149 : /* Expand move of TI mode register X to a new V1TI mode register. */
7150 : static rtx
7151 17 : ix86_expand_ti_to_v1ti (rtx x)
7152 : {
7153 17 : if (TARGET_SSE2)
7154 : {
7155 17 : rtx lo = gen_lowpart (DImode, x);
7156 17 : rtx hi = gen_highpart (DImode, x);
7157 17 : rtx tmp = gen_reg_rtx (V2DImode);
7158 17 : emit_insn (gen_vec_concatv2di (tmp, lo, hi));
7159 17 : return force_reg (V1TImode, gen_lowpart (V1TImode, tmp));
7160 : }
7161 :
7162 0 : return force_reg (V1TImode, gen_lowpart (V1TImode, x));
7163 : }
7164 :
7165 : /* Expand V1TI mode shift (of rtx_code CODE) by constant. */
7166 : void
7167 42 : ix86_expand_v1ti_shift (enum rtx_code code, rtx operands[])
7168 : {
7169 42 : rtx op1 = force_reg (V1TImode, operands[1]);
7170 :
7171 42 : if (!CONST_INT_P (operands[2]))
7172 : {
7173 6 : rtx tmp1 = ix86_expand_v1ti_to_ti (op1);
7174 6 : rtx tmp2 = gen_reg_rtx (TImode);
7175 3 : rtx (*shift) (rtx, rtx, rtx)
7176 6 : = (code == ASHIFT) ? gen_ashlti3 : gen_lshrti3;
7177 6 : emit_insn (shift (tmp2, tmp1, operands[2]));
7178 6 : rtx tmp3 = ix86_expand_ti_to_v1ti (tmp2);
7179 6 : emit_move_insn (operands[0], tmp3);
7180 6 : return;
7181 : }
7182 :
7183 36 : HOST_WIDE_INT bits = INTVAL (operands[2]) & 127;
7184 :
7185 36 : if (bits == 0)
7186 : {
7187 0 : emit_move_insn (operands[0], op1);
7188 0 : return;
7189 : }
7190 :
7191 36 : if ((bits & 7) == 0)
7192 : {
7193 0 : rtx tmp = gen_reg_rtx (V1TImode);
7194 0 : if (code == ASHIFT)
7195 0 : emit_insn (gen_sse2_ashlv1ti3 (tmp, op1, GEN_INT (bits)));
7196 : else
7197 0 : emit_insn (gen_sse2_lshrv1ti3 (tmp, op1, GEN_INT (bits)));
7198 0 : emit_move_insn (operands[0], tmp);
7199 0 : return;
7200 : }
7201 :
7202 36 : rtx tmp1 = gen_reg_rtx (V1TImode);
7203 36 : if (code == ASHIFT)
7204 18 : emit_insn (gen_sse2_ashlv1ti3 (tmp1, op1, GEN_INT (64)));
7205 : else
7206 18 : emit_insn (gen_sse2_lshrv1ti3 (tmp1, op1, GEN_INT (64)));
7207 :
7208 : /* tmp2 is operands[1] shifted by 64, in V2DImode. */
7209 36 : rtx tmp2 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp1));
7210 :
7211 : /* tmp3 will be the V2DImode result. */
7212 36 : rtx tmp3 = gen_reg_rtx (V2DImode);
7213 :
7214 36 : if (bits > 64)
7215 : {
7216 18 : if (code == ASHIFT)
7217 9 : emit_insn (gen_ashlv2di3 (tmp3, tmp2, GEN_INT (bits - 64)));
7218 : else
7219 9 : emit_insn (gen_lshrv2di3 (tmp3, tmp2, GEN_INT (bits - 64)));
7220 : }
7221 : else
7222 : {
7223 : /* tmp4 is operands[1], in V2DImode. */
7224 18 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
7225 :
7226 18 : rtx tmp5 = gen_reg_rtx (V2DImode);
7227 18 : if (code == ASHIFT)
7228 9 : emit_insn (gen_ashlv2di3 (tmp5, tmp4, GEN_INT (bits)));
7229 : else
7230 9 : emit_insn (gen_lshrv2di3 (tmp5, tmp4, GEN_INT (bits)));
7231 :
7232 18 : rtx tmp6 = gen_reg_rtx (V2DImode);
7233 18 : if (code == ASHIFT)
7234 9 : emit_insn (gen_lshrv2di3 (tmp6, tmp2, GEN_INT (64 - bits)));
7235 : else
7236 9 : emit_insn (gen_ashlv2di3 (tmp6, tmp2, GEN_INT (64 - bits)));
7237 :
7238 18 : emit_insn (gen_iorv2di3 (tmp3, tmp5, tmp6));
7239 : }
7240 :
7241 : /* Convert the result back to V1TImode and store in operands[0]. */
7242 36 : rtx tmp7 = force_reg (V1TImode, gen_lowpart (V1TImode, tmp3));
7243 36 : emit_move_insn (operands[0], tmp7);
7244 : }
7245 :
7246 : /* Expand V1TI mode rotate (of rtx_code CODE) by constant. */
7247 : void
7248 39 : ix86_expand_v1ti_rotate (enum rtx_code code, rtx operands[])
7249 : {
7250 39 : rtx op1 = force_reg (V1TImode, operands[1]);
7251 :
7252 39 : if (!CONST_INT_P (operands[2]))
7253 : {
7254 8 : rtx tmp1 = ix86_expand_v1ti_to_ti (op1);
7255 8 : rtx tmp2 = gen_reg_rtx (TImode);
7256 4 : rtx (*rotate) (rtx, rtx, rtx)
7257 8 : = (code == ROTATE) ? gen_rotlti3 : gen_rotrti3;
7258 8 : emit_insn (rotate (tmp2, tmp1, operands[2]));
7259 8 : rtx tmp3 = ix86_expand_ti_to_v1ti (tmp2);
7260 8 : emit_move_insn (operands[0], tmp3);
7261 8 : return;
7262 : }
7263 :
7264 31 : HOST_WIDE_INT bits = INTVAL (operands[2]) & 127;
7265 :
7266 31 : if (bits == 0)
7267 : {
7268 0 : emit_move_insn (operands[0], op1);
7269 0 : return;
7270 : }
7271 :
7272 31 : if (code == ROTATERT)
7273 16 : bits = 128 - bits;
7274 :
7275 31 : if ((bits & 31) == 0)
7276 : {
7277 5 : rtx tmp2 = gen_reg_rtx (V4SImode);
7278 5 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7279 5 : if (bits == 32)
7280 1 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0x93)));
7281 4 : else if (bits == 64)
7282 2 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0x4e)));
7283 : else
7284 2 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0x39)));
7285 5 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp2));
7286 5 : return;
7287 : }
7288 :
7289 26 : if ((bits & 7) == 0)
7290 : {
7291 6 : rtx tmp1 = gen_reg_rtx (V1TImode);
7292 6 : rtx tmp2 = gen_reg_rtx (V1TImode);
7293 6 : rtx tmp3 = gen_reg_rtx (V1TImode);
7294 :
7295 6 : emit_insn (gen_sse2_ashlv1ti3 (tmp1, op1, GEN_INT (bits)));
7296 6 : emit_insn (gen_sse2_lshrv1ti3 (tmp2, op1, GEN_INT (128 - bits)));
7297 6 : emit_insn (gen_iorv1ti3 (tmp3, tmp1, tmp2));
7298 6 : emit_move_insn (operands[0], tmp3);
7299 6 : return;
7300 : }
7301 :
7302 20 : rtx op1_v4si = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7303 :
7304 20 : rtx lobits;
7305 20 : rtx hibits;
7306 :
7307 20 : switch (bits >> 5)
7308 : {
7309 7 : case 0:
7310 7 : lobits = op1_v4si;
7311 7 : hibits = gen_reg_rtx (V4SImode);
7312 7 : emit_insn (gen_sse2_pshufd (hibits, op1_v4si, GEN_INT (0x93)));
7313 7 : break;
7314 :
7315 2 : case 1:
7316 2 : lobits = gen_reg_rtx (V4SImode);
7317 2 : hibits = gen_reg_rtx (V4SImode);
7318 2 : emit_insn (gen_sse2_pshufd (lobits, op1_v4si, GEN_INT (0x93)));
7319 2 : emit_insn (gen_sse2_pshufd (hibits, op1_v4si, GEN_INT (0x4e)));
7320 2 : break;
7321 :
7322 2 : case 2:
7323 2 : lobits = gen_reg_rtx (V4SImode);
7324 2 : hibits = gen_reg_rtx (V4SImode);
7325 2 : emit_insn (gen_sse2_pshufd (lobits, op1_v4si, GEN_INT (0x4e)));
7326 2 : emit_insn (gen_sse2_pshufd (hibits, op1_v4si, GEN_INT (0x39)));
7327 2 : break;
7328 :
7329 9 : default:
7330 9 : lobits = gen_reg_rtx (V4SImode);
7331 9 : emit_insn (gen_sse2_pshufd (lobits, op1_v4si, GEN_INT (0x39)));
7332 9 : hibits = op1_v4si;
7333 9 : break;
7334 : }
7335 :
7336 20 : rtx tmp1 = gen_reg_rtx (V4SImode);
7337 20 : rtx tmp2 = gen_reg_rtx (V4SImode);
7338 20 : rtx tmp3 = gen_reg_rtx (V4SImode);
7339 :
7340 20 : emit_insn (gen_ashlv4si3 (tmp1, lobits, GEN_INT (bits & 31)));
7341 20 : emit_insn (gen_lshrv4si3 (tmp2, hibits, GEN_INT (32 - (bits & 31))));
7342 20 : emit_insn (gen_iorv4si3 (tmp3, tmp1, tmp2));
7343 :
7344 20 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp3));
7345 : }
7346 :
7347 : /* Expand V1TI mode ashiftrt by constant. */
7348 : void
7349 109 : ix86_expand_v1ti_ashiftrt (rtx operands[])
7350 : {
7351 109 : rtx op1 = force_reg (V1TImode, operands[1]);
7352 :
7353 109 : if (!CONST_INT_P (operands[2]))
7354 : {
7355 3 : rtx tmp1 = ix86_expand_v1ti_to_ti (op1);
7356 3 : rtx tmp2 = gen_reg_rtx (TImode);
7357 3 : emit_insn (gen_ashrti3 (tmp2, tmp1, operands[2]));
7358 3 : rtx tmp3 = ix86_expand_ti_to_v1ti (tmp2);
7359 3 : emit_move_insn (operands[0], tmp3);
7360 3 : return;
7361 : }
7362 :
7363 106 : HOST_WIDE_INT bits = INTVAL (operands[2]) & 127;
7364 :
7365 106 : if (bits == 0)
7366 : {
7367 0 : emit_move_insn (operands[0], op1);
7368 0 : return;
7369 : }
7370 :
7371 106 : if (bits == 127)
7372 : {
7373 : /* Two operations. */
7374 3 : rtx tmp1 = force_reg(V4SImode, gen_lowpart (V4SImode, op1));
7375 3 : rtx tmp2 = gen_reg_rtx (V4SImode);
7376 3 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
7377 :
7378 3 : rtx tmp3 = gen_reg_rtx (V4SImode);
7379 3 : emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
7380 :
7381 3 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp3));
7382 3 : return;
7383 : }
7384 :
7385 103 : if (bits == 64)
7386 : {
7387 : /* Three operations. */
7388 3 : rtx tmp1 = force_reg(V4SImode, gen_lowpart (V4SImode, op1));
7389 3 : rtx tmp2 = gen_reg_rtx (V4SImode);
7390 3 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
7391 :
7392 3 : rtx tmp3 = gen_reg_rtx (V4SImode);
7393 3 : emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
7394 :
7395 3 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp1));
7396 3 : rtx tmp5 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
7397 3 : rtx tmp6 = gen_reg_rtx (V2DImode);
7398 3 : emit_insn (gen_vec_interleave_highv2di (tmp6, tmp4, tmp5));
7399 :
7400 3 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp6));
7401 3 : return;
7402 : }
7403 :
7404 100 : if (bits == 96)
7405 : {
7406 : /* Three operations. */
7407 3 : rtx tmp1 = force_reg(V4SImode, gen_lowpart (V4SImode, op1));
7408 3 : rtx tmp2 = gen_reg_rtx (V4SImode);
7409 3 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (31)));
7410 :
7411 3 : rtx tmp3 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp1));
7412 3 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp2));
7413 3 : rtx tmp5 = gen_reg_rtx (V2DImode);
7414 3 : emit_insn (gen_vec_interleave_highv2di (tmp5, tmp3, tmp4));
7415 :
7416 3 : rtx tmp6 = force_reg(V4SImode, gen_lowpart (V4SImode, tmp5));
7417 3 : rtx tmp7 = gen_reg_rtx (V4SImode);
7418 3 : emit_insn (gen_sse2_pshufd (tmp7, tmp6, GEN_INT (0xfd)));
7419 :
7420 3 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp7));
7421 3 : return;
7422 : }
7423 :
7424 97 : if (bits >= 111)
7425 : {
7426 : /* Three operations. */
7427 21 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7428 21 : rtx tmp2 = gen_reg_rtx (V4SImode);
7429 21 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits - 96)));
7430 :
7431 21 : rtx tmp3 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp2));
7432 21 : rtx tmp4 = gen_reg_rtx (V8HImode);
7433 21 : emit_insn (gen_sse2_pshufhw (tmp4, tmp3, GEN_INT (0xfe)));
7434 :
7435 21 : rtx tmp5 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp4));
7436 21 : rtx tmp6 = gen_reg_rtx (V4SImode);
7437 21 : emit_insn (gen_sse2_pshufd (tmp6, tmp5, GEN_INT (0xfe)));
7438 :
7439 21 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp6));
7440 21 : return;
7441 : }
7442 :
7443 76 : if (TARGET_AVX2 || TARGET_SSE4_1)
7444 : {
7445 : /* Three operations. */
7446 50 : if (bits == 32)
7447 : {
7448 2 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7449 2 : rtx tmp2 = gen_reg_rtx (V4SImode);
7450 2 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (31)));
7451 :
7452 2 : rtx tmp3 = gen_reg_rtx (V1TImode);
7453 2 : emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (32)));
7454 :
7455 2 : if (TARGET_AVX2)
7456 : {
7457 1 : rtx tmp4 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp3));
7458 1 : rtx tmp5 = gen_reg_rtx (V4SImode);
7459 1 : emit_insn (gen_avx2_pblenddv4si (tmp5, tmp2, tmp4,
7460 : GEN_INT (7)));
7461 :
7462 1 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp5));
7463 : }
7464 : else
7465 : {
7466 1 : rtx tmp4 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp2));
7467 1 : rtx tmp5 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp3));
7468 1 : rtx tmp6 = gen_reg_rtx (V8HImode);
7469 1 : emit_insn (gen_sse4_1_pblendw (tmp6, tmp4, tmp5,
7470 : GEN_INT (0x3f)));
7471 :
7472 1 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp6));
7473 : }
7474 : return;
7475 : }
7476 :
7477 : /* Three operations. */
7478 : if (bits == 8 || bits == 16 || bits == 24)
7479 : {
7480 6 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7481 6 : rtx tmp2 = gen_reg_rtx (V4SImode);
7482 6 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits)));
7483 :
7484 6 : rtx tmp3 = gen_reg_rtx (V1TImode);
7485 6 : emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (bits)));
7486 :
7487 6 : if (TARGET_AVX2)
7488 : {
7489 3 : rtx tmp4 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp3));
7490 3 : rtx tmp5 = gen_reg_rtx (V4SImode);
7491 3 : emit_insn (gen_avx2_pblenddv4si (tmp5, tmp2, tmp4,
7492 : GEN_INT (7)));
7493 :
7494 3 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp5));
7495 : }
7496 : else
7497 : {
7498 3 : rtx tmp4 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp2));
7499 3 : rtx tmp5 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp3));
7500 3 : rtx tmp6 = gen_reg_rtx (V8HImode);
7501 3 : emit_insn (gen_sse4_1_pblendw (tmp6, tmp4, tmp5,
7502 : GEN_INT (0x3f)));
7503 :
7504 3 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp6));
7505 : }
7506 : return;
7507 : }
7508 : }
7509 :
7510 68 : if (bits > 96)
7511 : {
7512 : /* Four operations. */
7513 3 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7514 3 : rtx tmp2 = gen_reg_rtx (V4SImode);
7515 3 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits - 96)));
7516 :
7517 3 : rtx tmp3 = gen_reg_rtx (V4SImode);
7518 3 : emit_insn (gen_ashrv4si3 (tmp3, tmp1, GEN_INT (31)));
7519 :
7520 3 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp2));
7521 3 : rtx tmp5 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
7522 3 : rtx tmp6 = gen_reg_rtx (V2DImode);
7523 3 : emit_insn (gen_vec_interleave_highv2di (tmp6, tmp4, tmp5));
7524 :
7525 3 : rtx tmp7 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp6));
7526 3 : rtx tmp8 = gen_reg_rtx (V4SImode);
7527 3 : emit_insn (gen_sse2_pshufd (tmp8, tmp7, GEN_INT (0xfd)));
7528 :
7529 3 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp8));
7530 3 : return;
7531 : }
7532 :
7533 65 : if (TARGET_SSE4_1 && (bits == 48 || bits == 80))
7534 : {
7535 : /* Four operations. */
7536 4 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7537 4 : rtx tmp2 = gen_reg_rtx (V4SImode);
7538 4 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
7539 :
7540 4 : rtx tmp3 = gen_reg_rtx (V4SImode);
7541 4 : emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
7542 :
7543 4 : rtx tmp4 = gen_reg_rtx (V1TImode);
7544 4 : emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (bits)));
7545 :
7546 4 : rtx tmp5 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp3));
7547 4 : rtx tmp6 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp4));
7548 4 : rtx tmp7 = gen_reg_rtx (V8HImode);
7549 6 : emit_insn (gen_sse4_1_pblendw (tmp7, tmp5, tmp6,
7550 : GEN_INT (bits == 48 ? 0x1f : 0x07)));
7551 :
7552 4 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp7));
7553 4 : return;
7554 : }
7555 :
7556 61 : if ((bits & 7) == 0)
7557 : {
7558 : /* Five operations. */
7559 9 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7560 9 : rtx tmp2 = gen_reg_rtx (V4SImode);
7561 9 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
7562 :
7563 9 : rtx tmp3 = gen_reg_rtx (V4SImode);
7564 9 : emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
7565 :
7566 9 : rtx tmp4 = gen_reg_rtx (V1TImode);
7567 9 : emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (bits)));
7568 :
7569 9 : rtx tmp5 = force_reg (V1TImode, gen_lowpart (V1TImode, tmp3));
7570 9 : rtx tmp6 = gen_reg_rtx (V1TImode);
7571 9 : emit_insn (gen_sse2_ashlv1ti3 (tmp6, tmp5, GEN_INT (128 - bits)));
7572 :
7573 9 : rtx tmp7 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp4));
7574 9 : rtx tmp8 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp6));
7575 9 : rtx tmp9 = gen_reg_rtx (V2DImode);
7576 9 : emit_insn (gen_iorv2di3 (tmp9, tmp7, tmp8));
7577 :
7578 9 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp9));
7579 9 : return;
7580 : }
7581 :
7582 52 : if (TARGET_AVX2 && bits < 32)
7583 : {
7584 : /* Six operations. */
7585 9 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7586 9 : rtx tmp2 = gen_reg_rtx (V4SImode);
7587 9 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits)));
7588 :
7589 9 : rtx tmp3 = gen_reg_rtx (V1TImode);
7590 9 : emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (64)));
7591 :
7592 9 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
7593 9 : rtx tmp5 = gen_reg_rtx (V2DImode);
7594 9 : emit_insn (gen_lshrv2di3 (tmp5, tmp4, GEN_INT (bits)));
7595 :
7596 9 : rtx tmp6 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
7597 9 : rtx tmp7 = gen_reg_rtx (V2DImode);
7598 9 : emit_insn (gen_ashlv2di3 (tmp7, tmp6, GEN_INT (64 - bits)));
7599 :
7600 9 : rtx tmp8 = gen_reg_rtx (V2DImode);
7601 9 : emit_insn (gen_iorv2di3 (tmp8, tmp5, tmp7));
7602 :
7603 9 : rtx tmp9 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp8));
7604 9 : rtx tmp10 = gen_reg_rtx (V4SImode);
7605 9 : emit_insn (gen_avx2_pblenddv4si (tmp10, tmp2, tmp9, GEN_INT (7)));
7606 :
7607 9 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp10));
7608 9 : return;
7609 : }
7610 :
7611 43 : if (TARGET_SSE4_1 && bits < 15)
7612 : {
7613 : /* Six operations. */
7614 4 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7615 4 : rtx tmp2 = gen_reg_rtx (V4SImode);
7616 4 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits)));
7617 :
7618 4 : rtx tmp3 = gen_reg_rtx (V1TImode);
7619 4 : emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (64)));
7620 :
7621 4 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
7622 4 : rtx tmp5 = gen_reg_rtx (V2DImode);
7623 4 : emit_insn (gen_lshrv2di3 (tmp5, tmp4, GEN_INT (bits)));
7624 :
7625 4 : rtx tmp6 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
7626 4 : rtx tmp7 = gen_reg_rtx (V2DImode);
7627 4 : emit_insn (gen_ashlv2di3 (tmp7, tmp6, GEN_INT (64 - bits)));
7628 :
7629 4 : rtx tmp8 = gen_reg_rtx (V2DImode);
7630 4 : emit_insn (gen_iorv2di3 (tmp8, tmp5, tmp7));
7631 :
7632 4 : rtx tmp9 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp2));
7633 4 : rtx tmp10 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp8));
7634 4 : rtx tmp11 = gen_reg_rtx (V8HImode);
7635 4 : emit_insn (gen_sse4_1_pblendw (tmp11, tmp9, tmp10, GEN_INT (0x3f)));
7636 :
7637 4 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp11));
7638 4 : return;
7639 : }
7640 :
7641 18 : if (bits == 1)
7642 : {
7643 : /* Eight operations. */
7644 1 : rtx tmp1 = gen_reg_rtx (V1TImode);
7645 1 : emit_insn (gen_sse2_lshrv1ti3 (tmp1, op1, GEN_INT (64)));
7646 :
7647 1 : rtx tmp2 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
7648 1 : rtx tmp3 = gen_reg_rtx (V2DImode);
7649 1 : emit_insn (gen_lshrv2di3 (tmp3, tmp2, GEN_INT (1)));
7650 :
7651 1 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp1));
7652 1 : rtx tmp5 = gen_reg_rtx (V2DImode);
7653 1 : emit_insn (gen_ashlv2di3 (tmp5, tmp4, GEN_INT (63)));
7654 :
7655 1 : rtx tmp6 = gen_reg_rtx (V2DImode);
7656 1 : emit_insn (gen_iorv2di3 (tmp6, tmp3, tmp5));
7657 :
7658 1 : rtx tmp7 = gen_reg_rtx (V2DImode);
7659 1 : emit_insn (gen_lshrv2di3 (tmp7, tmp2, GEN_INT (63)));
7660 :
7661 1 : rtx tmp8 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp7));
7662 1 : rtx tmp9 = gen_reg_rtx (V4SImode);
7663 1 : emit_insn (gen_sse2_pshufd (tmp9, tmp8, GEN_INT (0xbf)));
7664 :
7665 1 : rtx tmp10 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp9));
7666 1 : rtx tmp11 = gen_reg_rtx (V2DImode);
7667 1 : emit_insn (gen_ashlv2di3 (tmp11, tmp10, GEN_INT (31)));
7668 :
7669 1 : rtx tmp12 = gen_reg_rtx (V2DImode);
7670 1 : emit_insn (gen_iorv2di3 (tmp12, tmp6, tmp11));
7671 :
7672 1 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp12));
7673 1 : return;
7674 : }
7675 :
7676 38 : if (bits > 64)
7677 : {
7678 : /* Eight operations. */
7679 12 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7680 12 : rtx tmp2 = gen_reg_rtx (V4SImode);
7681 12 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
7682 :
7683 12 : rtx tmp3 = gen_reg_rtx (V4SImode);
7684 12 : emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
7685 :
7686 12 : rtx tmp4 = gen_reg_rtx (V1TImode);
7687 12 : emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (64)));
7688 :
7689 12 : rtx tmp5 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp4));
7690 12 : rtx tmp6 = gen_reg_rtx (V2DImode);
7691 12 : emit_insn (gen_lshrv2di3 (tmp6, tmp5, GEN_INT (bits - 64)));
7692 :
7693 12 : rtx tmp7 = force_reg (V1TImode, gen_lowpart (V1TImode, tmp3));
7694 12 : rtx tmp8 = gen_reg_rtx (V1TImode);
7695 12 : emit_insn (gen_sse2_ashlv1ti3 (tmp8, tmp7, GEN_INT (64)));
7696 :
7697 12 : rtx tmp9 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
7698 12 : rtx tmp10 = gen_reg_rtx (V2DImode);
7699 12 : emit_insn (gen_ashlv2di3 (tmp10, tmp9, GEN_INT (128 - bits)));
7700 :
7701 12 : rtx tmp11 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp8));
7702 12 : rtx tmp12 = gen_reg_rtx (V2DImode);
7703 12 : emit_insn (gen_iorv2di3 (tmp12, tmp10, tmp11));
7704 :
7705 12 : rtx tmp13 = gen_reg_rtx (V2DImode);
7706 12 : emit_insn (gen_iorv2di3 (tmp13, tmp6, tmp12));
7707 :
7708 12 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp13));
7709 : }
7710 : else
7711 : {
7712 : /* Nine operations. */
7713 26 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7714 26 : rtx tmp2 = gen_reg_rtx (V4SImode);
7715 26 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
7716 :
7717 26 : rtx tmp3 = gen_reg_rtx (V4SImode);
7718 26 : emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
7719 :
7720 26 : rtx tmp4 = gen_reg_rtx (V1TImode);
7721 26 : emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (64)));
7722 :
7723 26 : rtx tmp5 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
7724 26 : rtx tmp6 = gen_reg_rtx (V2DImode);
7725 26 : emit_insn (gen_lshrv2di3 (tmp6, tmp5, GEN_INT (bits)));
7726 :
7727 26 : rtx tmp7 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp4));
7728 26 : rtx tmp8 = gen_reg_rtx (V2DImode);
7729 26 : emit_insn (gen_ashlv2di3 (tmp8, tmp7, GEN_INT (64 - bits)));
7730 :
7731 26 : rtx tmp9 = gen_reg_rtx (V2DImode);
7732 26 : emit_insn (gen_iorv2di3 (tmp9, tmp6, tmp8));
7733 :
7734 26 : rtx tmp10 = force_reg (V1TImode, gen_lowpart (V1TImode, tmp3));
7735 26 : rtx tmp11 = gen_reg_rtx (V1TImode);
7736 26 : emit_insn (gen_sse2_ashlv1ti3 (tmp11, tmp10, GEN_INT (64)));
7737 :
7738 26 : rtx tmp12 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp11));
7739 26 : rtx tmp13 = gen_reg_rtx (V2DImode);
7740 26 : emit_insn (gen_ashlv2di3 (tmp13, tmp12, GEN_INT (64 - bits)));
7741 :
7742 26 : rtx tmp14 = gen_reg_rtx (V2DImode);
7743 26 : emit_insn (gen_iorv2di3 (tmp14, tmp9, tmp13));
7744 :
7745 26 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp14));
7746 : }
7747 : }
7748 :
7749 : /* Expand V2DI mode ashiftrt. */
7750 : void
7751 438 : ix86_expand_v2di_ashiftrt (rtx operands[])
7752 : {
7753 438 : if (operands[2] == const0_rtx)
7754 : {
7755 0 : emit_move_insn (operands[0], operands[1]);
7756 0 : return;
7757 : }
7758 :
7759 438 : if (TARGET_SSE4_2
7760 135 : && CONST_INT_P (operands[2])
7761 135 : && UINTVAL (operands[2]) >= 63
7762 446 : && !optimize_insn_for_size_p ())
7763 : {
7764 8 : rtx zero = force_reg (V2DImode, CONST0_RTX (V2DImode));
7765 8 : emit_insn (gen_sse4_2_gtv2di3 (operands[0], zero, operands[1]));
7766 8 : return;
7767 : }
7768 :
7769 430 : if (CONST_INT_P (operands[2])
7770 404 : && (!TARGET_XOP || UINTVAL (operands[2]) >= 63))
7771 : {
7772 308 : vec_perm_builder sel (4, 4, 1);
7773 308 : sel.quick_grow (4);
7774 308 : rtx arg0, arg1;
7775 308 : rtx op1 = lowpart_subreg (V4SImode,
7776 : force_reg (V2DImode, operands[1]),
7777 : V2DImode);
7778 308 : rtx target = gen_reg_rtx (V4SImode);
7779 308 : if (UINTVAL (operands[2]) >= 63)
7780 : {
7781 116 : arg0 = arg1 = gen_reg_rtx (V4SImode);
7782 116 : emit_insn (gen_ashrv4si3 (arg0, op1, GEN_INT (31)));
7783 116 : sel[0] = 1;
7784 116 : sel[1] = 1;
7785 116 : sel[2] = 3;
7786 116 : sel[3] = 3;
7787 : }
7788 192 : else if (INTVAL (operands[2]) > 32)
7789 : {
7790 21 : arg0 = gen_reg_rtx (V4SImode);
7791 21 : arg1 = gen_reg_rtx (V4SImode);
7792 21 : emit_insn (gen_ashrv4si3 (arg1, op1, GEN_INT (31)));
7793 21 : emit_insn (gen_ashrv4si3 (arg0, op1,
7794 21 : GEN_INT (INTVAL (operands[2]) - 32)));
7795 21 : sel[0] = 1;
7796 21 : sel[1] = 5;
7797 21 : sel[2] = 3;
7798 21 : sel[3] = 7;
7799 : }
7800 171 : else if (INTVAL (operands[2]) == 32)
7801 : {
7802 5 : arg0 = op1;
7803 5 : arg1 = gen_reg_rtx (V4SImode);
7804 5 : emit_insn (gen_ashrv4si3 (arg1, op1, GEN_INT (31)));
7805 5 : sel[0] = 1;
7806 5 : sel[1] = 5;
7807 5 : sel[2] = 3;
7808 5 : sel[3] = 7;
7809 : }
7810 : else
7811 : {
7812 166 : arg0 = gen_reg_rtx (V2DImode);
7813 166 : arg1 = gen_reg_rtx (V4SImode);
7814 166 : emit_insn (gen_lshrv2di3 (arg0, operands[1], operands[2]));
7815 166 : emit_insn (gen_ashrv4si3 (arg1, op1, operands[2]));
7816 166 : arg0 = lowpart_subreg (V4SImode, arg0, V2DImode);
7817 166 : sel[0] = 0;
7818 166 : sel[1] = 5;
7819 166 : sel[2] = 2;
7820 166 : sel[3] = 7;
7821 : }
7822 424 : vec_perm_indices indices (sel, arg0 != arg1 ? 2 : 1, 4);
7823 308 : rtx op0 = operands[0];
7824 308 : bool ok = targetm.vectorize.vec_perm_const (V4SImode, V4SImode,
7825 : target, arg0, arg1,
7826 : indices);
7827 308 : gcc_assert (ok);
7828 308 : emit_move_insn (op0, lowpart_subreg (V2DImode, target, V4SImode));
7829 308 : return;
7830 308 : }
7831 122 : if (!TARGET_XOP)
7832 : {
7833 26 : rtx zero = force_reg (V2DImode, CONST0_RTX (V2DImode));
7834 26 : rtx zero_or_all_ones;
7835 26 : if (TARGET_SSE4_2)
7836 : {
7837 0 : zero_or_all_ones = gen_reg_rtx (V2DImode);
7838 0 : emit_insn (gen_sse4_2_gtv2di3 (zero_or_all_ones, zero,
7839 : operands[1]));
7840 : }
7841 : else
7842 : {
7843 26 : rtx temp = gen_reg_rtx (V4SImode);
7844 26 : emit_insn (gen_ashrv4si3 (temp,
7845 : lowpart_subreg (V4SImode,
7846 : force_reg (V2DImode,
7847 : operands[1]),
7848 : V2DImode),
7849 : GEN_INT (31)));
7850 26 : zero_or_all_ones = gen_reg_rtx (V4SImode);
7851 26 : emit_insn (gen_sse2_pshufd_1 (zero_or_all_ones, temp,
7852 : const1_rtx, const1_rtx,
7853 : GEN_INT (3), GEN_INT (3)));
7854 26 : zero_or_all_ones = lowpart_subreg (V2DImode, zero_or_all_ones,
7855 : V4SImode);
7856 : }
7857 26 : rtx lshr_res = gen_reg_rtx (V2DImode);
7858 26 : emit_insn (gen_lshrv2di3 (lshr_res, operands[1], operands[2]));
7859 26 : rtx ashl_res = gen_reg_rtx (V2DImode);
7860 26 : rtx amount;
7861 26 : if (TARGET_64BIT)
7862 : {
7863 26 : amount = gen_reg_rtx (DImode);
7864 26 : emit_insn (gen_subdi3 (amount, force_reg (DImode, GEN_INT (64)),
7865 : operands[2]));
7866 : }
7867 : else
7868 : {
7869 0 : rtx temp = gen_reg_rtx (SImode);
7870 0 : emit_insn (gen_subsi3 (temp, force_reg (SImode, GEN_INT (64)),
7871 : lowpart_subreg (SImode, operands[2],
7872 : DImode)));
7873 0 : amount = gen_reg_rtx (V4SImode);
7874 0 : emit_insn (gen_vec_setv4si_0 (amount, CONST0_RTX (V4SImode),
7875 : temp));
7876 : }
7877 26 : amount = lowpart_subreg (DImode, amount, GET_MODE (amount));
7878 26 : emit_insn (gen_ashlv2di3 (ashl_res, zero_or_all_ones, amount));
7879 26 : emit_insn (gen_iorv2di3 (operands[0], lshr_res, ashl_res));
7880 26 : return;
7881 : }
7882 :
7883 96 : rtx reg = gen_reg_rtx (V2DImode);
7884 96 : rtx par;
7885 96 : bool negate = false;
7886 96 : int i;
7887 :
7888 96 : if (CONST_INT_P (operands[2]))
7889 96 : operands[2] = GEN_INT (-INTVAL (operands[2]));
7890 : else
7891 : negate = true;
7892 :
7893 96 : par = gen_rtx_PARALLEL (V2DImode, rtvec_alloc (2));
7894 288 : for (i = 0; i < 2; i++)
7895 192 : XVECEXP (par, 0, i) = operands[2];
7896 :
7897 96 : emit_insn (gen_vec_initv2didi (reg, par));
7898 :
7899 96 : if (negate)
7900 0 : emit_insn (gen_negv2di2 (reg, reg));
7901 :
7902 96 : emit_insn (gen_xop_shav2di3 (operands[0], operands[1], reg));
7903 : }
7904 :
7905 : /* Replace all occurrences of REG FROM with REG TO in X, including
7906 : occurrences with different modes. */
7907 :
7908 : rtx
7909 43421 : ix86_replace_reg_with_reg (rtx x, rtx from, rtx to)
7910 : {
7911 43421 : gcc_checking_assert (REG_P (from)
7912 : && REG_P (to)
7913 : && GET_MODE (from) == GET_MODE (to));
7914 43421 : if (!reg_overlap_mentioned_p (from, x))
7915 : return x;
7916 70 : rtx ret = copy_rtx (x);
7917 70 : subrtx_ptr_iterator::array_type array;
7918 362 : FOR_EACH_SUBRTX_PTR (iter, array, &ret, NONCONST)
7919 : {
7920 292 : rtx *loc = *iter;
7921 292 : x = *loc;
7922 292 : if (REG_P (x) && REGNO (x) == REGNO (from))
7923 : {
7924 70 : if (x == from)
7925 70 : *loc = to;
7926 : else
7927 : {
7928 0 : gcc_checking_assert (REG_NREGS (x) == 1);
7929 0 : *loc = gen_rtx_REG (GET_MODE (x), REGNO (to));
7930 : }
7931 : }
7932 : }
7933 70 : return ret;
7934 70 : }
7935 :
7936 : /* Return mode for the memcpy/memset loop counter. Prefer SImode over
7937 : DImode for constant loop counts. */
7938 :
7939 : static machine_mode
7940 44859 : counter_mode (rtx count_exp)
7941 : {
7942 726 : if (GET_MODE (count_exp) != VOIDmode)
7943 32568 : return GET_MODE (count_exp);
7944 12291 : if (!CONST_INT_P (count_exp))
7945 0 : return Pmode;
7946 : if (TARGET_64BIT && (INTVAL (count_exp) & ~0xffffffff))
7947 : return DImode;
7948 : return SImode;
7949 : }
7950 :
7951 : /* When ISSETMEM is FALSE, output simple loop to move memory pointer to SRCPTR
7952 : to DESTPTR via chunks of MODE unrolled UNROLL times, overall size is COUNT
7953 : specified in bytes. When ISSETMEM is TRUE, output the equivalent loop to set
7954 : memory by VALUE (supposed to be in MODE).
7955 :
7956 : The size is rounded down to whole number of chunk size moved at once.
7957 : SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info. */
7958 :
7959 :
7960 : static void
7961 8230 : expand_set_or_cpymem_via_loop (rtx destmem, rtx srcmem,
7962 : rtx destptr, rtx srcptr, rtx value,
7963 : rtx count, machine_mode mode, int unroll,
7964 : int expected_size, bool issetmem)
7965 : {
7966 8230 : rtx_code_label *out_label = nullptr;
7967 8230 : rtx_code_label *top_label = nullptr;
7968 8230 : rtx iter, tmp;
7969 8230 : machine_mode iter_mode = counter_mode (count);
7970 8230 : int piece_size_n = GET_MODE_SIZE (mode) * unroll;
7971 8230 : rtx piece_size = GEN_INT (piece_size_n);
7972 16460 : rtx piece_size_mask = GEN_INT (~((GET_MODE_SIZE (mode) * unroll) - 1));
7973 8230 : rtx size;
7974 8230 : int i;
7975 8230 : int loop_count;
7976 :
7977 8230 : if (expected_size != -1 && CONST_INT_P (count))
7978 7229 : loop_count = INTVAL (count) / GET_MODE_SIZE (mode) / unroll;
7979 : else
7980 : loop_count = -1;
7981 :
7982 : /* Don't generate the loop if the loop count is 1. */
7983 7229 : if (loop_count != 1)
7984 : {
7985 8179 : top_label = gen_label_rtx ();
7986 8179 : out_label = gen_label_rtx ();
7987 : }
7988 8230 : iter = gen_reg_rtx (iter_mode);
7989 :
7990 8230 : size = expand_simple_binop (iter_mode, AND, count, piece_size_mask,
7991 : NULL, 1, OPTAB_DIRECT);
7992 : /* Those two should combine. */
7993 8230 : if (piece_size == const1_rtx)
7994 : {
7995 275 : emit_cmp_and_jump_insns (size, const0_rtx, EQ, NULL_RTX, iter_mode,
7996 : true, out_label);
7997 275 : predict_jump (REG_BR_PROB_BASE * 10 / 100);
7998 : }
7999 8230 : emit_move_insn (iter, const0_rtx);
8000 :
8001 8230 : if (loop_count != 1)
8002 8179 : emit_label (top_label);
8003 :
8004 10953 : tmp = convert_modes (Pmode, iter_mode, iter, true);
8005 :
8006 : /* This assert could be relaxed - in this case we'll need to compute
8007 : smallest power of two, containing in PIECE_SIZE_N and pass it to
8008 : offset_address. */
8009 8230 : gcc_assert ((piece_size_n & (piece_size_n - 1)) == 0);
8010 8230 : destmem = offset_address (destmem, tmp, piece_size_n);
8011 8230 : destmem = adjust_address (destmem, mode, 0);
8012 :
8013 8230 : if (!issetmem)
8014 : {
8015 2897 : srcmem = offset_address (srcmem, copy_rtx (tmp), piece_size_n);
8016 2897 : srcmem = adjust_address (srcmem, mode, 0);
8017 :
8018 : /* When unrolling for chips that reorder memory reads and writes,
8019 : we can save registers by using single temporary.
8020 : Also using 4 temporaries is overkill in 32bit mode. */
8021 2897 : if (!TARGET_64BIT && 0)
8022 : {
8023 : for (i = 0; i < unroll; i++)
8024 : {
8025 : if (i)
8026 : {
8027 : destmem = adjust_address (copy_rtx (destmem), mode,
8028 : GET_MODE_SIZE (mode));
8029 : srcmem = adjust_address (copy_rtx (srcmem), mode,
8030 : GET_MODE_SIZE (mode));
8031 : }
8032 : emit_move_insn (destmem, srcmem);
8033 : }
8034 : }
8035 : else
8036 : {
8037 2897 : rtx tmpreg[4];
8038 2897 : gcc_assert (unroll <= 4);
8039 13671 : for (i = 0; i < unroll; i++)
8040 : {
8041 10774 : tmpreg[i] = gen_reg_rtx (mode);
8042 10774 : if (i)
8043 15754 : srcmem = adjust_address (copy_rtx (srcmem), mode,
8044 : GET_MODE_SIZE (mode));
8045 10774 : emit_move_insn (tmpreg[i], srcmem);
8046 : }
8047 13671 : for (i = 0; i < unroll; i++)
8048 : {
8049 10774 : if (i)
8050 15754 : destmem = adjust_address (copy_rtx (destmem), mode,
8051 : GET_MODE_SIZE (mode));
8052 10774 : emit_move_insn (destmem, tmpreg[i]);
8053 : }
8054 : }
8055 : }
8056 : else
8057 26500 : for (i = 0; i < unroll; i++)
8058 : {
8059 21167 : if (i)
8060 31668 : destmem = adjust_address (copy_rtx (destmem), mode,
8061 : GET_MODE_SIZE (mode));
8062 21167 : emit_move_insn (destmem, value);
8063 : }
8064 :
8065 8230 : tmp = expand_simple_binop (iter_mode, PLUS, iter, piece_size, iter,
8066 : true, OPTAB_LIB_WIDEN);
8067 8230 : if (tmp != iter)
8068 0 : emit_move_insn (iter, tmp);
8069 :
8070 8230 : if (loop_count != 1)
8071 : {
8072 8179 : emit_cmp_and_jump_insns (iter, size, LT, NULL_RTX, iter_mode,
8073 : true, top_label);
8074 8179 : if (expected_size != -1)
8075 : {
8076 7292 : expected_size /= GET_MODE_SIZE (mode) * unroll;
8077 7292 : if (expected_size == 0)
8078 0 : predict_jump (0);
8079 7292 : else if (expected_size > REG_BR_PROB_BASE)
8080 2 : predict_jump (REG_BR_PROB_BASE - 1);
8081 : else
8082 7290 : predict_jump (REG_BR_PROB_BASE
8083 7290 : - (REG_BR_PROB_BASE + expected_size / 2)
8084 7290 : / expected_size);
8085 : }
8086 : else
8087 887 : predict_jump (REG_BR_PROB_BASE * 80 / 100);
8088 : }
8089 8230 : iter = ix86_zero_extend_to_Pmode (iter);
8090 10953 : tmp = expand_simple_binop (Pmode, PLUS, destptr, iter, destptr,
8091 : true, OPTAB_LIB_WIDEN);
8092 8230 : if (tmp != destptr)
8093 0 : emit_move_insn (destptr, tmp);
8094 8230 : if (!issetmem)
8095 : {
8096 4222 : tmp = expand_simple_binop (Pmode, PLUS, srcptr, iter, srcptr,
8097 : true, OPTAB_LIB_WIDEN);
8098 2897 : if (tmp != srcptr)
8099 0 : emit_move_insn (srcptr, tmp);
8100 : }
8101 8230 : if (loop_count != 1)
8102 8179 : emit_label (out_label);
8103 8230 : }
8104 :
8105 : /* Divide COUNTREG by SCALE. */
8106 : static rtx
8107 18876 : scale_counter (rtx countreg, int scale)
8108 : {
8109 18876 : rtx sc;
8110 :
8111 18876 : if (scale == 1)
8112 : return countreg;
8113 13049 : if (CONST_INT_P (countreg))
8114 13031 : return GEN_INT (INTVAL (countreg) / scale);
8115 18 : gcc_assert (REG_P (countreg));
8116 :
8117 54 : sc = expand_simple_binop (GET_MODE (countreg), LSHIFTRT, countreg,
8118 36 : GEN_INT (exact_log2 (scale)),
8119 : NULL, 1, OPTAB_DIRECT);
8120 18 : return sc;
8121 : }
8122 :
8123 : /* Output "rep; mov" or "rep; stos" instruction depending on ISSETMEM argument.
8124 : When ISSETMEM is true, arguments SRCMEM and SRCPTR are ignored.
8125 : When ISSETMEM is false, arguments VALUE and ORIG_VALUE are ignored.
8126 : For setmem case, VALUE is a promoted to a wider size ORIG_VALUE.
8127 : ORIG_VALUE is the original value passed to memset to fill the memory with.
8128 : Other arguments have same meaning as for previous function. */
8129 :
8130 : static void
8131 18876 : expand_set_or_cpymem_via_rep (rtx destmem, rtx srcmem,
8132 : rtx destptr, rtx srcptr, rtx value, rtx orig_value,
8133 : rtx count,
8134 : machine_mode mode, bool issetmem)
8135 : {
8136 18876 : rtx destexp;
8137 18876 : rtx srcexp;
8138 18876 : rtx countreg;
8139 18876 : HOST_WIDE_INT rounded_count;
8140 :
8141 : /* If possible, it is shorter to use rep movs.
8142 : TODO: Maybe it is better to move this logic to decide_alg. */
8143 18876 : if (mode == QImode && CONST_INT_P (count) && !(INTVAL (count) & 3)
8144 268 : && !TARGET_PREFER_KNOWN_REP_MOVSB_STOSB
8145 264 : && (!issetmem || orig_value == const0_rtx))
8146 18876 : mode = SImode;
8147 :
8148 18876 : if (destptr != XEXP (destmem, 0) || GET_MODE (destmem) != BLKmode)
8149 18598 : destmem = adjust_automodify_address_nv (destmem, BLKmode, destptr, 0);
8150 :
8151 37752 : countreg = ix86_zero_extend_to_Pmode (scale_counter (count,
8152 18876 : GET_MODE_SIZE (mode)));
8153 18876 : if (mode != QImode)
8154 : {
8155 39406 : destexp = gen_rtx_ASHIFT (Pmode, countreg,
8156 : GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
8157 13308 : destexp = gen_rtx_PLUS (Pmode, destexp, destptr);
8158 : }
8159 : else
8160 5846 : destexp = gen_rtx_PLUS (Pmode, destptr, countreg);
8161 18876 : if ((!issetmem || orig_value == const0_rtx) && CONST_INT_P (count))
8162 : {
8163 13723 : rounded_count
8164 13723 : = ROUND_DOWN (INTVAL (count), (HOST_WIDE_INT) GET_MODE_SIZE (mode));
8165 13723 : destmem = shallow_copy_rtx (destmem);
8166 13723 : set_mem_size (destmem, rounded_count);
8167 : }
8168 5161 : else if (MEM_SIZE_KNOWN_P (destmem))
8169 363 : clear_mem_size (destmem);
8170 :
8171 18876 : if (issetmem)
8172 : {
8173 7963 : value = force_reg (mode, gen_lowpart (mode, value));
8174 7963 : emit_insn (gen_rep_stos (destptr, countreg, destmem, value, destexp));
8175 : }
8176 : else
8177 : {
8178 10913 : if (srcptr != XEXP (srcmem, 0) || GET_MODE (srcmem) != BLKmode)
8179 10699 : srcmem = adjust_automodify_address_nv (srcmem, BLKmode, srcptr, 0);
8180 10913 : if (mode != QImode)
8181 : {
8182 19489 : srcexp = gen_rtx_ASHIFT (Pmode, countreg,
8183 : GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
8184 6619 : srcexp = gen_rtx_PLUS (Pmode, srcexp, srcptr);
8185 : }
8186 : else
8187 4493 : srcexp = gen_rtx_PLUS (Pmode, srcptr, countreg);
8188 10913 : if (CONST_INT_P (count))
8189 : {
8190 6934 : rounded_count
8191 6934 : = ROUND_DOWN (INTVAL (count), (HOST_WIDE_INT) GET_MODE_SIZE (mode));
8192 6934 : srcmem = shallow_copy_rtx (srcmem);
8193 6934 : set_mem_size (srcmem, rounded_count);
8194 : }
8195 : else
8196 : {
8197 3994 : if (MEM_SIZE_KNOWN_P (srcmem))
8198 0 : clear_mem_size (srcmem);
8199 : }
8200 10913 : emit_insn (gen_rep_mov (destptr, destmem, srcptr, srcmem, countreg,
8201 : destexp, srcexp));
8202 : }
8203 18876 : }
8204 :
8205 : /* This function emits moves to copy SIZE_TO_MOVE bytes from SRCMEM to
8206 : DESTMEM.
8207 : SRC is passed by pointer to be updated on return.
8208 : Return value is updated DST. */
8209 : static rtx
8210 13 : emit_memmov (rtx destmem, rtx *srcmem, rtx destptr, rtx srcptr,
8211 : HOST_WIDE_INT size_to_move)
8212 : {
8213 13 : rtx dst = destmem, src = *srcmem, tempreg;
8214 13 : enum insn_code code;
8215 13 : machine_mode move_mode;
8216 13 : int piece_size, i;
8217 :
8218 : /* Find the widest mode in which we could perform moves.
8219 : Start with the biggest power of 2 less than SIZE_TO_MOVE and half
8220 : it until move of such size is supported. */
8221 13 : piece_size = 1 << floor_log2 (size_to_move);
8222 26 : while (!int_mode_for_size (piece_size * BITS_PER_UNIT, 0).exists (&move_mode)
8223 26 : || (code = optab_handler (mov_optab, move_mode)) == CODE_FOR_nothing)
8224 : {
8225 0 : gcc_assert (piece_size > 1);
8226 0 : piece_size >>= 1;
8227 : }
8228 :
8229 : /* Find the corresponding vector mode with the same size as MOVE_MODE.
8230 : MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.). */
8231 39 : if (GET_MODE_SIZE (move_mode) > GET_MODE_SIZE (word_mode))
8232 : {
8233 0 : int nunits = GET_MODE_SIZE (move_mode) / GET_MODE_SIZE (word_mode);
8234 0 : if (!mode_for_vector (word_mode, nunits).exists (&move_mode)
8235 0 : || (code = optab_handler (mov_optab, move_mode)) == CODE_FOR_nothing)
8236 : {
8237 0 : move_mode = word_mode;
8238 0 : piece_size = GET_MODE_SIZE (move_mode);
8239 0 : code = optab_handler (mov_optab, move_mode);
8240 : }
8241 : }
8242 13 : gcc_assert (code != CODE_FOR_nothing);
8243 :
8244 13 : dst = adjust_automodify_address_nv (dst, move_mode, destptr, 0);
8245 13 : src = adjust_automodify_address_nv (src, move_mode, srcptr, 0);
8246 :
8247 : /* Emit moves. We'll need SIZE_TO_MOVE/PIECE_SIZES moves. */
8248 13 : gcc_assert (size_to_move % piece_size == 0);
8249 :
8250 26 : for (i = 0; i < size_to_move; i += piece_size)
8251 : {
8252 : /* We move from memory to memory, so we'll need to do it via
8253 : a temporary register. */
8254 13 : tempreg = gen_reg_rtx (move_mode);
8255 13 : emit_insn (GEN_FCN (code) (tempreg, src));
8256 13 : emit_insn (GEN_FCN (code) (dst, tempreg));
8257 :
8258 26 : emit_move_insn (destptr,
8259 13 : plus_constant (Pmode, copy_rtx (destptr), piece_size));
8260 26 : emit_move_insn (srcptr,
8261 13 : plus_constant (Pmode, copy_rtx (srcptr), piece_size));
8262 :
8263 13 : dst = adjust_automodify_address_nv (dst, move_mode, destptr,
8264 : piece_size);
8265 13 : src = adjust_automodify_address_nv (src, move_mode, srcptr,
8266 : piece_size);
8267 : }
8268 :
8269 : /* Update DST and SRC rtx. */
8270 13 : *srcmem = src;
8271 13 : return dst;
8272 : }
8273 :
8274 : /* Helper function for the string operations below. Dest VARIABLE whether
8275 : it is aligned to VALUE bytes. If true, jump to the label. */
8276 :
8277 : static rtx_code_label *
8278 3321 : ix86_expand_aligntest (rtx variable, int value, bool epilogue)
8279 : {
8280 3321 : rtx_code_label *label = gen_label_rtx ();
8281 3321 : rtx tmpcount = gen_reg_rtx (GET_MODE (variable));
8282 3321 : if (GET_MODE (variable) == DImode)
8283 746 : emit_insn (gen_anddi3 (tmpcount, variable, GEN_INT (value)));
8284 : else
8285 2575 : emit_insn (gen_andsi3 (tmpcount, variable, GEN_INT (value)));
8286 3321 : emit_cmp_and_jump_insns (tmpcount, const0_rtx, EQ, 0, GET_MODE (variable),
8287 : 1, label);
8288 3321 : if (epilogue)
8289 0 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
8290 : else
8291 3321 : predict_jump (REG_BR_PROB_BASE * 90 / 100);
8292 3321 : return label;
8293 : }
8294 :
8295 :
8296 : /* Output code to copy at most count & (max_size - 1) bytes from SRC to DEST. */
8297 :
8298 : static void
8299 7000 : expand_cpymem_epilogue (rtx destmem, rtx srcmem,
8300 : rtx destptr, rtx srcptr, rtx count, int max_size)
8301 : {
8302 7000 : rtx src, dest;
8303 7000 : if (CONST_INT_P (count))
8304 : {
8305 6935 : unsigned HOST_WIDE_INT countval = UINTVAL (count);
8306 6935 : unsigned HOST_WIDE_INT epilogue_size = countval % max_size;
8307 6935 : unsigned int destalign = MEM_ALIGN (destmem);
8308 6935 : cfun->machine->by_pieces_in_use = true;
8309 6935 : move_by_pieces (destmem, srcmem, epilogue_size, destalign,
8310 : RETURN_BEGIN);
8311 6935 : cfun->machine->by_pieces_in_use = false;
8312 6935 : return;
8313 : }
8314 65 : if (max_size > 8)
8315 : {
8316 65 : count = expand_simple_binop (GET_MODE (count), AND, count, GEN_INT (max_size - 1),
8317 : count, 1, OPTAB_DIRECT);
8318 65 : expand_set_or_cpymem_via_loop (destmem, srcmem, destptr, srcptr, NULL,
8319 : count, QImode, 1, 4, false);
8320 65 : return;
8321 : }
8322 :
8323 : /* When there are stringops, we can cheaply increase dest and src pointers.
8324 : Otherwise we save code size by maintaining offset (zero is readily
8325 : available from preceding rep operation) and using x86 addressing modes.
8326 : */
8327 0 : if (TARGET_SINGLE_STRINGOP)
8328 : {
8329 0 : if (max_size > 4)
8330 : {
8331 0 : rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
8332 0 : src = change_address (srcmem, SImode, srcptr);
8333 0 : dest = change_address (destmem, SImode, destptr);
8334 0 : emit_insn (gen_strmov (destptr, dest, srcptr, src));
8335 0 : emit_label (label);
8336 0 : LABEL_NUSES (label) = 1;
8337 : }
8338 0 : if (max_size > 2)
8339 : {
8340 0 : rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
8341 0 : src = change_address (srcmem, HImode, srcptr);
8342 0 : dest = change_address (destmem, HImode, destptr);
8343 0 : emit_insn (gen_strmov (destptr, dest, srcptr, src));
8344 0 : emit_label (label);
8345 0 : LABEL_NUSES (label) = 1;
8346 : }
8347 0 : if (max_size > 1)
8348 : {
8349 0 : rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
8350 0 : src = change_address (srcmem, QImode, srcptr);
8351 0 : dest = change_address (destmem, QImode, destptr);
8352 0 : emit_insn (gen_strmov (destptr, dest, srcptr, src));
8353 0 : emit_label (label);
8354 0 : LABEL_NUSES (label) = 1;
8355 : }
8356 : }
8357 : else
8358 : {
8359 0 : rtx offset = force_reg (Pmode, const0_rtx);
8360 0 : rtx tmp;
8361 :
8362 0 : if (max_size > 4)
8363 : {
8364 0 : rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
8365 0 : src = change_address (srcmem, SImode, srcptr);
8366 0 : dest = change_address (destmem, SImode, destptr);
8367 0 : emit_move_insn (dest, src);
8368 0 : tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (4), NULL,
8369 : true, OPTAB_LIB_WIDEN);
8370 0 : if (tmp != offset)
8371 0 : emit_move_insn (offset, tmp);
8372 0 : emit_label (label);
8373 0 : LABEL_NUSES (label) = 1;
8374 : }
8375 0 : if (max_size > 2)
8376 : {
8377 0 : rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
8378 0 : tmp = gen_rtx_PLUS (Pmode, srcptr, offset);
8379 0 : src = change_address (srcmem, HImode, tmp);
8380 0 : tmp = gen_rtx_PLUS (Pmode, destptr, offset);
8381 0 : dest = change_address (destmem, HImode, tmp);
8382 0 : emit_move_insn (dest, src);
8383 0 : tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (2), tmp,
8384 : true, OPTAB_LIB_WIDEN);
8385 0 : if (tmp != offset)
8386 0 : emit_move_insn (offset, tmp);
8387 0 : emit_label (label);
8388 0 : LABEL_NUSES (label) = 1;
8389 : }
8390 0 : if (max_size > 1)
8391 : {
8392 0 : rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
8393 0 : tmp = gen_rtx_PLUS (Pmode, srcptr, offset);
8394 0 : src = change_address (srcmem, QImode, tmp);
8395 0 : tmp = gen_rtx_PLUS (Pmode, destptr, offset);
8396 0 : dest = change_address (destmem, QImode, tmp);
8397 0 : emit_move_insn (dest, src);
8398 0 : emit_label (label);
8399 0 : LABEL_NUSES (label) = 1;
8400 : }
8401 : }
8402 : }
8403 :
8404 : /* This function emits moves to fill SIZE_TO_MOVE bytes starting from DESTMEM
8405 : with value PROMOTED_VAL.
8406 : SRC is passed by pointer to be updated on return.
8407 : Return value is updated DST. */
8408 : static rtx
8409 6 : emit_memset (rtx destmem, rtx destptr, rtx promoted_val,
8410 : HOST_WIDE_INT size_to_move)
8411 : {
8412 6 : rtx dst = destmem;
8413 6 : enum insn_code code;
8414 6 : machine_mode move_mode;
8415 6 : int piece_size, i;
8416 :
8417 : /* Find the widest mode in which we could perform moves.
8418 : Start with the biggest power of 2 less than SIZE_TO_MOVE and half
8419 : it until move of such size is supported. */
8420 6 : move_mode = GET_MODE (promoted_val);
8421 6 : if (move_mode == VOIDmode)
8422 0 : move_mode = QImode;
8423 12 : if (size_to_move < GET_MODE_SIZE (move_mode))
8424 : {
8425 5 : unsigned int move_bits = size_to_move * BITS_PER_UNIT;
8426 5 : move_mode = int_mode_for_size (move_bits, 0).require ();
8427 5 : promoted_val = gen_lowpart (move_mode, promoted_val);
8428 : }
8429 6 : piece_size = GET_MODE_SIZE (move_mode);
8430 6 : code = optab_handler (mov_optab, move_mode);
8431 6 : gcc_assert (code != CODE_FOR_nothing && promoted_val != NULL_RTX);
8432 :
8433 6 : dst = adjust_automodify_address_nv (dst, move_mode, destptr, 0);
8434 :
8435 : /* Emit moves. We'll need SIZE_TO_MOVE/PIECE_SIZES moves. */
8436 6 : gcc_assert (size_to_move % piece_size == 0);
8437 :
8438 12 : for (i = 0; i < size_to_move; i += piece_size)
8439 : {
8440 12 : if (piece_size <= GET_MODE_SIZE (word_mode))
8441 : {
8442 4 : emit_insn (gen_strset (destptr, dst, promoted_val));
8443 4 : dst = adjust_automodify_address_nv (dst, move_mode, destptr,
8444 : piece_size);
8445 4 : continue;
8446 : }
8447 :
8448 2 : emit_insn (GEN_FCN (code) (dst, promoted_val));
8449 :
8450 4 : emit_move_insn (destptr,
8451 2 : plus_constant (Pmode, copy_rtx (destptr), piece_size));
8452 :
8453 2 : dst = adjust_automodify_address_nv (dst, move_mode, destptr,
8454 : piece_size);
8455 : }
8456 :
8457 : /* Update DST rtx. */
8458 6 : return dst;
8459 : }
8460 : /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
8461 : static void
8462 37 : expand_setmem_epilogue_via_loop (rtx destmem, rtx destptr, rtx value,
8463 : rtx count, int max_size)
8464 : {
8465 74 : count = expand_simple_binop (counter_mode (count), AND, count,
8466 37 : GEN_INT (max_size - 1), count, 1, OPTAB_DIRECT);
8467 37 : expand_set_or_cpymem_via_loop (destmem, NULL, destptr, NULL,
8468 37 : gen_lowpart (QImode, value), count, QImode,
8469 : 1, max_size / 2, true);
8470 37 : }
8471 :
8472 : /* Callback routine for store_by_pieces. Return the RTL of a register
8473 : containing GET_MODE_SIZE (MODE) bytes in the RTL register op_p which
8474 : is an integer or a word vector register. If PREV_P isn't nullptr,
8475 : it has the RTL info from the previous iteration. */
8476 :
8477 : static rtx
8478 4967 : setmem_epilogue_gen_val (void *op_p, void *prev_p, HOST_WIDE_INT,
8479 : fixed_size_mode mode)
8480 : {
8481 4967 : rtx target;
8482 4967 : by_pieces_prev *prev = (by_pieces_prev *) prev_p;
8483 4967 : if (prev)
8484 : {
8485 4967 : rtx prev_op = prev->data;
8486 4967 : if (prev_op)
8487 : {
8488 2881 : machine_mode prev_mode = GET_MODE (prev_op);
8489 2881 : if (prev_mode == mode)
8490 : return prev_op;
8491 54 : if (VECTOR_MODE_P (prev_mode)
8492 1068 : && VECTOR_MODE_P (mode)
8493 1122 : && GET_MODE_INNER (prev_mode) == GET_MODE_INNER (mode))
8494 : {
8495 0 : target = gen_rtx_SUBREG (mode, prev_op, 0);
8496 0 : return target;
8497 : }
8498 : }
8499 : }
8500 :
8501 3208 : rtx op = (rtx) op_p;
8502 3208 : machine_mode op_mode = GET_MODE (op);
8503 :
8504 3208 : if (VECTOR_MODE_P (mode))
8505 : {
8506 3638 : gcc_assert (GET_MODE_INNER (mode) == QImode);
8507 :
8508 1819 : unsigned int op_size = GET_MODE_SIZE (op_mode);
8509 1819 : unsigned int size = GET_MODE_SIZE (mode);
8510 1819 : unsigned int nunits;
8511 1819 : machine_mode vec_mode;
8512 1819 : if (op_size < size)
8513 : {
8514 : /* If OP size is smaller than MODE size, duplicate it. */
8515 1 : nunits = size / GET_MODE_SIZE (QImode);
8516 1 : vec_mode = mode_for_vector (QImode, nunits).require ();
8517 1 : nunits = size / op_size;
8518 1 : gcc_assert (SCALAR_INT_MODE_P (op_mode));
8519 1 : machine_mode dup_mode
8520 1 : = mode_for_vector (as_a <scalar_mode> (op_mode),
8521 2 : nunits).require ();
8522 1 : target = gen_reg_rtx (vec_mode);
8523 1 : op = gen_vec_duplicate (dup_mode, op);
8524 1 : rtx dup_op = gen_reg_rtx (dup_mode);
8525 1 : emit_move_insn (dup_op, op);
8526 1 : op = gen_rtx_SUBREG (vec_mode, dup_op, 0);
8527 1 : emit_move_insn (target, op);
8528 1 : return target;
8529 : }
8530 1818 : nunits = op_size / GET_MODE_SIZE (QImode);
8531 1818 : vec_mode = mode_for_vector (QImode, nunits).require ();
8532 1818 : target = gen_reg_rtx (vec_mode);
8533 1818 : op = gen_rtx_SUBREG (vec_mode, op, 0);
8534 1818 : emit_move_insn (target, op);
8535 1818 : if (op_size == size)
8536 : return target;
8537 :
8538 0 : rtx tmp = gen_reg_rtx (mode);
8539 0 : target = gen_rtx_SUBREG (mode, target, 0);
8540 0 : emit_move_insn (tmp, target);
8541 0 : return tmp;
8542 : }
8543 :
8544 1389 : if (VECTOR_MODE_P (op_mode))
8545 : {
8546 2768 : gcc_assert (GET_MODE_INNER (op_mode) == word_mode);
8547 1384 : target = gen_reg_rtx (word_mode);
8548 1384 : op = gen_rtx_SUBREG (word_mode, op, 0);
8549 1384 : emit_move_insn (target, op);
8550 : }
8551 : else
8552 : target = op;
8553 :
8554 1389 : if (mode == GET_MODE (target))
8555 : return target;
8556 :
8557 246 : rtx tmp = gen_reg_rtx (mode);
8558 246 : target = gen_rtx_SUBREG (mode, target, 0);
8559 246 : emit_move_insn (tmp, target);
8560 246 : return tmp;
8561 : }
8562 :
8563 : /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
8564 : static void
8565 10019 : expand_setmem_epilogue (rtx destmem, rtx destptr, rtx value, rtx vec_value,
8566 : rtx count, int max_size)
8567 : {
8568 10019 : rtx dest;
8569 :
8570 10019 : if (CONST_INT_P (count))
8571 : {
8572 9982 : unsigned HOST_WIDE_INT countval = UINTVAL (count);
8573 9982 : unsigned HOST_WIDE_INT epilogue_size = countval % max_size;
8574 9982 : unsigned int destalign = MEM_ALIGN (destmem);
8575 9982 : cfun->machine->by_pieces_in_use = true;
8576 16545 : store_by_pieces (destmem, epilogue_size, setmem_epilogue_gen_val,
8577 : vec_value ? vec_value : value, destalign, true,
8578 : RETURN_BEGIN);
8579 9982 : cfun->machine->by_pieces_in_use = false;
8580 9982 : return;
8581 : }
8582 37 : if (max_size > 32)
8583 : {
8584 37 : expand_setmem_epilogue_via_loop (destmem, destptr, value, count, max_size);
8585 37 : return;
8586 : }
8587 0 : if (max_size > 16)
8588 : {
8589 0 : rtx_code_label *label = ix86_expand_aligntest (count, 16, true);
8590 0 : if (TARGET_64BIT)
8591 : {
8592 0 : dest = change_address (destmem, DImode, destptr);
8593 0 : emit_insn (gen_strset (destptr, dest, value));
8594 0 : dest = adjust_automodify_address_nv (dest, DImode, destptr, 8);
8595 0 : emit_insn (gen_strset (destptr, dest, value));
8596 : }
8597 : else
8598 : {
8599 0 : dest = change_address (destmem, SImode, destptr);
8600 0 : emit_insn (gen_strset (destptr, dest, value));
8601 0 : dest = adjust_automodify_address_nv (dest, SImode, destptr, 4);
8602 0 : emit_insn (gen_strset (destptr, dest, value));
8603 0 : dest = adjust_automodify_address_nv (dest, SImode, destptr, 8);
8604 0 : emit_insn (gen_strset (destptr, dest, value));
8605 0 : dest = adjust_automodify_address_nv (dest, SImode, destptr, 12);
8606 0 : emit_insn (gen_strset (destptr, dest, value));
8607 : }
8608 0 : emit_label (label);
8609 0 : LABEL_NUSES (label) = 1;
8610 : }
8611 0 : if (max_size > 8)
8612 : {
8613 0 : rtx_code_label *label = ix86_expand_aligntest (count, 8, true);
8614 0 : if (TARGET_64BIT)
8615 : {
8616 0 : dest = change_address (destmem, DImode, destptr);
8617 0 : emit_insn (gen_strset (destptr, dest, value));
8618 : }
8619 : else
8620 : {
8621 0 : dest = change_address (destmem, SImode, destptr);
8622 0 : emit_insn (gen_strset (destptr, dest, value));
8623 0 : dest = adjust_automodify_address_nv (dest, SImode, destptr, 4);
8624 0 : emit_insn (gen_strset (destptr, dest, value));
8625 : }
8626 0 : emit_label (label);
8627 0 : LABEL_NUSES (label) = 1;
8628 : }
8629 0 : if (max_size > 4)
8630 : {
8631 0 : rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
8632 0 : dest = change_address (destmem, SImode, destptr);
8633 0 : emit_insn (gen_strset (destptr, dest, gen_lowpart (SImode, value)));
8634 0 : emit_label (label);
8635 0 : LABEL_NUSES (label) = 1;
8636 : }
8637 0 : if (max_size > 2)
8638 : {
8639 0 : rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
8640 0 : dest = change_address (destmem, HImode, destptr);
8641 0 : emit_insn (gen_strset (destptr, dest, gen_lowpart (HImode, value)));
8642 0 : emit_label (label);
8643 0 : LABEL_NUSES (label) = 1;
8644 : }
8645 0 : if (max_size > 1)
8646 : {
8647 0 : rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
8648 0 : dest = change_address (destmem, QImode, destptr);
8649 0 : emit_insn (gen_strset (destptr, dest, gen_lowpart (QImode, value)));
8650 0 : emit_label (label);
8651 0 : LABEL_NUSES (label) = 1;
8652 : }
8653 : }
8654 :
8655 : /* Adjust COUNTER by the VALUE. */
8656 : static void
8657 19 : ix86_adjust_counter (rtx countreg, HOST_WIDE_INT value)
8658 : {
8659 19 : emit_insn (gen_add2_insn (countreg, GEN_INT (-value)));
8660 19 : }
8661 :
8662 : /* Depending on ISSETMEM, copy enough from SRCMEM to DESTMEM or set enough to
8663 : DESTMEM to align it to DESIRED_ALIGNMENT. Original alignment is ALIGN.
8664 : Depending on ISSETMEM, either arguments SRCMEM/SRCPTR or VALUE/VEC_VALUE are
8665 : ignored.
8666 : Return value is updated DESTMEM. */
8667 :
8668 : static rtx
8669 7 : expand_set_or_cpymem_prologue (rtx destmem, rtx srcmem,
8670 : rtx destptr, rtx srcptr, rtx value,
8671 : rtx vec_value, rtx count, int align,
8672 : int desired_alignment, bool issetmem)
8673 : {
8674 7 : int i;
8675 35 : for (i = 1; i < desired_alignment; i <<= 1)
8676 : {
8677 28 : if (align <= i)
8678 : {
8679 19 : rtx_code_label *label = ix86_expand_aligntest (destptr, i, false);
8680 19 : if (issetmem)
8681 : {
8682 12 : if (vec_value && i > GET_MODE_SIZE (GET_MODE (value)))
8683 2 : destmem = emit_memset (destmem, destptr, vec_value, i);
8684 : else
8685 4 : destmem = emit_memset (destmem, destptr, value, i);
8686 : }
8687 : else
8688 13 : destmem = emit_memmov (destmem, &srcmem, destptr, srcptr, i);
8689 19 : ix86_adjust_counter (count, i);
8690 19 : emit_label (label);
8691 19 : LABEL_NUSES (label) = 1;
8692 19 : set_mem_align (destmem, i * 2 * BITS_PER_UNIT);
8693 : }
8694 : }
8695 7 : return destmem;
8696 : }
8697 :
8698 : /* Test if COUNT&SIZE is nonzero and if so, expand movme
8699 : or setmem sequence that is valid for SIZE..2*SIZE-1 bytes
8700 : and jump to DONE_LABEL. */
8701 : static void
8702 2638 : expand_small_cpymem_or_setmem (rtx destmem, rtx srcmem,
8703 : rtx destptr, rtx srcptr,
8704 : rtx value, rtx vec_value,
8705 : rtx count, int size,
8706 : rtx done_label, bool issetmem)
8707 : {
8708 2638 : rtx_code_label *label = ix86_expand_aligntest (count, size, false);
8709 2638 : machine_mode mode = int_mode_for_size (size * BITS_PER_UNIT, 1).else_blk ();
8710 2638 : rtx modesize;
8711 2638 : rtx scalar_value = value;
8712 2638 : int n;
8713 :
8714 : /* If we do not have vector value to copy, we must reduce size. */
8715 2638 : if (issetmem)
8716 : {
8717 889 : if (!vec_value)
8718 : {
8719 4 : if (GET_MODE (value) == VOIDmode && size > 8)
8720 0 : mode = Pmode;
8721 12 : else if (GET_MODE_SIZE (mode) > GET_MODE_SIZE (GET_MODE (value)))
8722 0 : mode = GET_MODE (value);
8723 : }
8724 : else
8725 885 : mode = GET_MODE (vec_value), value = vec_value;
8726 : }
8727 : else
8728 : {
8729 : /* Choose appropriate vector mode. */
8730 1749 : if (size >= 32)
8731 436 : switch (MOVE_MAX)
8732 : {
8733 0 : case 64:
8734 0 : if (size >= 64)
8735 : {
8736 : mode = V64QImode;
8737 : break;
8738 : }
8739 : /* FALLTHRU */
8740 0 : case 32:
8741 0 : mode = V32QImode;
8742 0 : break;
8743 : case 16:
8744 : mode = V16QImode;
8745 : break;
8746 : case 8:
8747 : mode = DImode;
8748 : break;
8749 0 : default:
8750 0 : gcc_unreachable ();
8751 : }
8752 1313 : else if (size >= 16)
8753 436 : mode = TARGET_SSE ? V16QImode : DImode;
8754 1749 : srcmem = change_address (srcmem, mode, srcptr);
8755 : }
8756 3523 : if (issetmem && vec_value && GET_MODE_SIZE (mode) > size)
8757 : {
8758 : /* For memset with vector and the size is smaller than the vector
8759 : size, first try the narrower vector, otherwise, use the
8760 : original value. */
8761 447 : machine_mode inner_mode = GET_MODE_INNER (mode);
8762 447 : unsigned int nunits = size / GET_MODE_SIZE (inner_mode);
8763 447 : if (nunits > 1)
8764 : {
8765 324 : mode = mode_for_vector (GET_MODE_INNER (mode),
8766 324 : nunits).require ();
8767 162 : value = gen_rtx_SUBREG (mode, value, 0);
8768 : }
8769 : else
8770 : {
8771 285 : scalar_int_mode smode
8772 285 : = smallest_int_mode_for_size (size * BITS_PER_UNIT).require ();
8773 855 : gcc_assert (GET_MODE_SIZE (GET_MODE (scalar_value))
8774 : >= GET_MODE_SIZE (smode));
8775 285 : mode = smode;
8776 285 : if (GET_MODE (scalar_value) == mode)
8777 : value = scalar_value;
8778 : else
8779 66 : value = gen_rtx_SUBREG (mode, scalar_value, 0);
8780 : }
8781 : }
8782 2638 : destmem = change_address (destmem, mode, destptr);
8783 5276 : modesize = GEN_INT (GET_MODE_SIZE (mode));
8784 5276 : gcc_assert (GET_MODE_SIZE (mode) <= size);
8785 11862 : for (n = 0; n * GET_MODE_SIZE (mode) < size; n++)
8786 : {
8787 3293 : if (issetmem)
8788 1108 : emit_move_insn (destmem, gen_lowpart (mode, value));
8789 : else
8790 : {
8791 2185 : emit_move_insn (destmem, srcmem);
8792 4370 : srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
8793 : }
8794 6586 : destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
8795 : }
8796 :
8797 2638 : destmem = offset_address (destmem, count, 1);
8798 5276 : destmem = offset_address (destmem, GEN_INT (-2 * size),
8799 2638 : GET_MODE_SIZE (mode));
8800 2638 : if (!issetmem)
8801 : {
8802 1749 : srcmem = offset_address (srcmem, count, 1);
8803 3498 : srcmem = offset_address (srcmem, GEN_INT (-2 * size),
8804 1749 : GET_MODE_SIZE (mode));
8805 : }
8806 11862 : for (n = 0; n * GET_MODE_SIZE (mode) < size; n++)
8807 : {
8808 3293 : if (issetmem)
8809 1108 : emit_move_insn (destmem, gen_lowpart (mode, value));
8810 : else
8811 : {
8812 2185 : emit_move_insn (destmem, srcmem);
8813 4370 : srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
8814 : }
8815 6586 : destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
8816 : }
8817 2638 : emit_jump_insn (gen_jump (done_label));
8818 2638 : emit_barrier ();
8819 :
8820 2638 : emit_label (label);
8821 2638 : LABEL_NUSES (label) = 1;
8822 2638 : }
8823 :
8824 : /* Handle small memcpy (up to SIZE that is supposed to be small power of 2.
8825 : and get ready for the main memcpy loop by copying initial DESIRED_ALIGN-ALIGN
8826 : bytes and last SIZE bytes adjusitng DESTPTR/SRCPTR/COUNT in a way we can
8827 : proceed with an loop copying SIZE bytes at once. Do moves in MODE.
8828 : DONE_LABEL is a label after the whole copying sequence. The label is created
8829 : on demand if *DONE_LABEL is NULL.
8830 : MIN_SIZE is minimal size of block copied. This value gets adjusted for new
8831 : bounds after the initial copies.
8832 :
8833 : DESTMEM/SRCMEM are memory expressions pointing to the copies block,
8834 : DESTPTR/SRCPTR are pointers to the block. DYNAMIC_CHECK indicate whether
8835 : we will dispatch to a library call for large blocks.
8836 :
8837 : In pseudocode we do:
8838 :
8839 : if (COUNT < SIZE)
8840 : {
8841 : Assume that SIZE is 4. Bigger sizes are handled analogously
8842 : if (COUNT & 4)
8843 : {
8844 : copy 4 bytes from SRCPTR to DESTPTR
8845 : copy 4 bytes from SRCPTR + COUNT - 4 to DESTPTR + COUNT - 4
8846 : goto done_label
8847 : }
8848 : if (!COUNT)
8849 : goto done_label;
8850 : copy 1 byte from SRCPTR to DESTPTR
8851 : if (COUNT & 2)
8852 : {
8853 : copy 2 bytes from SRCPTR to DESTPTR
8854 : copy 2 bytes from SRCPTR + COUNT - 2 to DESTPTR + COUNT - 2
8855 : }
8856 : }
8857 : else
8858 : {
8859 : copy at least DESIRED_ALIGN-ALIGN bytes from SRCPTR to DESTPTR
8860 : copy SIZE bytes from SRCPTR + COUNT - SIZE to DESTPTR + COUNT -SIZE
8861 :
8862 : OLD_DESPTR = DESTPTR;
8863 : Align DESTPTR up to DESIRED_ALIGN
8864 : SRCPTR += DESTPTR - OLD_DESTPTR
8865 : COUNT -= DEST_PTR - OLD_DESTPTR
8866 : if (DYNAMIC_CHECK)
8867 : Round COUNT down to multiple of SIZE
8868 : << optional caller supplied zero size guard is here >>
8869 : << optional caller supplied dynamic check is here >>
8870 : << caller supplied main copy loop is here >>
8871 : }
8872 : done_label:
8873 : */
8874 : static void
8875 4029 : expand_set_or_cpymem_prologue_epilogue_by_misaligned_moves (rtx destmem, rtx srcmem,
8876 : rtx *destptr, rtx *srcptr,
8877 : machine_mode mode,
8878 : rtx value, rtx vec_value,
8879 : rtx *count,
8880 : rtx_code_label **done_label,
8881 : int size,
8882 : int desired_align,
8883 : int align,
8884 : unsigned HOST_WIDE_INT *min_size,
8885 : bool dynamic_check,
8886 : bool issetmem)
8887 : {
8888 4029 : rtx_code_label *loop_label = NULL, *label;
8889 4029 : int n;
8890 4029 : rtx modesize;
8891 4029 : int prolog_size = 0;
8892 4029 : rtx mode_value;
8893 :
8894 : /* Chose proper value to copy. */
8895 4029 : if (issetmem && VECTOR_MODE_P (mode))
8896 : mode_value = vec_value;
8897 : else
8898 4029 : mode_value = value;
8899 8058 : gcc_assert (GET_MODE_SIZE (mode) <= size);
8900 :
8901 : /* See if block is big or small, handle small blocks. */
8902 4029 : if (!CONST_INT_P (*count) && *min_size < (unsigned HOST_WIDE_INT)size)
8903 : {
8904 664 : int size2 = size;
8905 664 : loop_label = gen_label_rtx ();
8906 :
8907 664 : if (!*done_label)
8908 664 : *done_label = gen_label_rtx ();
8909 :
8910 664 : emit_cmp_and_jump_insns (*count, GEN_INT (size2), GE, 0, GET_MODE (*count),
8911 : 1, loop_label);
8912 664 : size2 >>= 1;
8913 :
8914 : /* Handle sizes > 3. */
8915 3302 : for (;size2 > 2; size2 >>= 1)
8916 2638 : expand_small_cpymem_or_setmem (destmem, srcmem,
8917 : *destptr, *srcptr,
8918 : value, vec_value,
8919 : *count,
8920 : size2, *done_label, issetmem);
8921 : /* Nothing to copy? Jump to DONE_LABEL if so */
8922 664 : emit_cmp_and_jump_insns (*count, const0_rtx, EQ, 0, GET_MODE (*count),
8923 : 1, *done_label);
8924 :
8925 : /* Do a byte copy. */
8926 664 : destmem = change_address (destmem, QImode, *destptr);
8927 664 : if (issetmem)
8928 223 : emit_move_insn (destmem, gen_lowpart (QImode, value));
8929 : else
8930 : {
8931 441 : srcmem = change_address (srcmem, QImode, *srcptr);
8932 441 : emit_move_insn (destmem, srcmem);
8933 : }
8934 :
8935 : /* Handle sizes 2 and 3. */
8936 664 : label = ix86_expand_aligntest (*count, 2, false);
8937 664 : destmem = change_address (destmem, HImode, *destptr);
8938 664 : destmem = offset_address (destmem, *count, 1);
8939 664 : destmem = offset_address (destmem, GEN_INT (-2), 2);
8940 664 : if (issetmem)
8941 223 : emit_move_insn (destmem, gen_lowpart (HImode, value));
8942 : else
8943 : {
8944 441 : srcmem = change_address (srcmem, HImode, *srcptr);
8945 441 : srcmem = offset_address (srcmem, *count, 1);
8946 441 : srcmem = offset_address (srcmem, GEN_INT (-2), 2);
8947 441 : emit_move_insn (destmem, srcmem);
8948 : }
8949 :
8950 664 : emit_label (label);
8951 664 : LABEL_NUSES (label) = 1;
8952 664 : emit_jump_insn (gen_jump (*done_label));
8953 664 : emit_barrier ();
8954 : }
8955 : else
8956 3365 : gcc_assert (*min_size >= (unsigned HOST_WIDE_INT)size
8957 : || UINTVAL (*count) >= (unsigned HOST_WIDE_INT)size);
8958 :
8959 : /* Start memcpy for COUNT >= SIZE. */
8960 664 : if (loop_label)
8961 : {
8962 664 : emit_label (loop_label);
8963 664 : LABEL_NUSES (loop_label) = 1;
8964 : }
8965 :
8966 : /* Copy first desired_align bytes. */
8967 4029 : if (!issetmem)
8968 2102 : srcmem = change_address (srcmem, mode, *srcptr);
8969 4029 : destmem = change_address (destmem, mode, *destptr);
8970 4029 : modesize = GEN_INT (GET_MODE_SIZE (mode));
8971 8075 : for (n = 0; prolog_size < desired_align - align; n++)
8972 : {
8973 17 : if (issetmem)
8974 1 : emit_move_insn (destmem, mode_value);
8975 : else
8976 : {
8977 16 : emit_move_insn (destmem, srcmem);
8978 32 : srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
8979 : }
8980 34 : destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
8981 34 : prolog_size += GET_MODE_SIZE (mode);
8982 : }
8983 :
8984 :
8985 : /* Copy last SIZE bytes. */
8986 4029 : destmem = offset_address (destmem, *count, 1);
8987 4029 : destmem = offset_address (destmem,
8988 4029 : GEN_INT (-size - prolog_size),
8989 : 1);
8990 4029 : if (issetmem)
8991 1927 : emit_move_insn (destmem, mode_value);
8992 : else
8993 : {
8994 2102 : srcmem = offset_address (srcmem, *count, 1);
8995 2102 : srcmem = offset_address (srcmem,
8996 : GEN_INT (-size - prolog_size),
8997 : 1);
8998 2102 : emit_move_insn (destmem, srcmem);
8999 : }
9000 30522 : for (n = 1; n * GET_MODE_SIZE (mode) < size; n++)
9001 : {
9002 11232 : destmem = offset_address (destmem, modesize, 1);
9003 11232 : if (issetmem)
9004 5583 : emit_move_insn (destmem, mode_value);
9005 : else
9006 : {
9007 5649 : srcmem = offset_address (srcmem, modesize, 1);
9008 5649 : emit_move_insn (destmem, srcmem);
9009 : }
9010 : }
9011 :
9012 : /* Align destination. */
9013 4029 : if (desired_align > 1 && desired_align > align)
9014 : {
9015 17 : rtx saveddest = *destptr;
9016 :
9017 17 : gcc_assert (desired_align <= size);
9018 : /* Align destptr up, place it to new register. */
9019 17 : *destptr = expand_simple_binop (GET_MODE (*destptr), PLUS, *destptr,
9020 : GEN_INT (prolog_size),
9021 : NULL_RTX, 1, OPTAB_DIRECT);
9022 17 : if (REG_P (*destptr) && REG_P (saveddest) && REG_POINTER (saveddest))
9023 17 : REG_POINTER (*destptr) = 1;
9024 17 : *destptr = expand_simple_binop (GET_MODE (*destptr), AND, *destptr,
9025 17 : GEN_INT (-desired_align),
9026 : *destptr, 1, OPTAB_DIRECT);
9027 : /* See how many bytes we skipped. */
9028 17 : saveddest = expand_simple_binop (GET_MODE (*destptr), MINUS, saveddest,
9029 : *destptr,
9030 : NULL_RTX, 1, OPTAB_DIRECT);
9031 : /* Adjust srcptr and count. */
9032 17 : if (!issetmem)
9033 16 : *srcptr = expand_simple_binop (GET_MODE (*srcptr), MINUS, *srcptr,
9034 : saveddest, *srcptr, 1, OPTAB_DIRECT);
9035 17 : *count = expand_simple_binop (GET_MODE (*count), PLUS, *count,
9036 : saveddest, *count, 1, OPTAB_DIRECT);
9037 : /* We copied at most size + prolog_size. */
9038 17 : if (*min_size > (unsigned HOST_WIDE_INT)(size + prolog_size))
9039 16 : *min_size
9040 16 : = ROUND_DOWN (*min_size - size, (unsigned HOST_WIDE_INT)size);
9041 : else
9042 : *min_size = 0;
9043 :
9044 : /* Our loops always round down the block size, but for dispatch to
9045 : library we need precise value. */
9046 17 : if (dynamic_check)
9047 17 : *count = expand_simple_binop (GET_MODE (*count), AND, *count,
9048 : GEN_INT (-size), *count, 1, OPTAB_DIRECT);
9049 : }
9050 : else
9051 : {
9052 4012 : gcc_assert (prolog_size == 0);
9053 : /* Decrease count, so we won't end up copying last word twice. */
9054 4012 : if (!CONST_INT_P (*count))
9055 663 : *count = expand_simple_binop (GET_MODE (*count), PLUS, *count,
9056 : constm1_rtx, *count, 1, OPTAB_DIRECT);
9057 : else
9058 3349 : *count = GEN_INT (ROUND_DOWN (UINTVAL (*count) - 1,
9059 : (unsigned HOST_WIDE_INT)size));
9060 4012 : if (*min_size)
9061 3786 : *min_size = ROUND_DOWN (*min_size - 1, (unsigned HOST_WIDE_INT)size);
9062 : }
9063 4029 : }
9064 :
9065 :
9066 : /* This function is like the previous one, except here we know how many bytes
9067 : need to be copied. That allows us to update alignment not only of DST, which
9068 : is returned, but also of SRC, which is passed as a pointer for that
9069 : reason. */
9070 : static rtx
9071 0 : expand_set_or_cpymem_constant_prologue (rtx dst, rtx *srcp, rtx destreg,
9072 : rtx srcreg, rtx value, rtx vec_value,
9073 : int desired_align, int align_bytes,
9074 : bool issetmem)
9075 : {
9076 0 : rtx src = NULL;
9077 0 : rtx orig_dst = dst;
9078 0 : rtx orig_src = NULL;
9079 0 : int piece_size = 1;
9080 0 : int copied_bytes = 0;
9081 :
9082 0 : if (!issetmem)
9083 : {
9084 0 : gcc_assert (srcp != NULL);
9085 0 : src = *srcp;
9086 0 : orig_src = src;
9087 : }
9088 :
9089 0 : for (piece_size = 1;
9090 0 : piece_size <= desired_align && copied_bytes < align_bytes;
9091 0 : piece_size <<= 1)
9092 : {
9093 0 : if (align_bytes & piece_size)
9094 : {
9095 0 : if (issetmem)
9096 : {
9097 0 : if (vec_value && piece_size > GET_MODE_SIZE (GET_MODE (value)))
9098 0 : dst = emit_memset (dst, destreg, vec_value, piece_size);
9099 : else
9100 0 : dst = emit_memset (dst, destreg, value, piece_size);
9101 : }
9102 : else
9103 0 : dst = emit_memmov (dst, &src, destreg, srcreg, piece_size);
9104 0 : copied_bytes += piece_size;
9105 : }
9106 : }
9107 0 : if (MEM_ALIGN (dst) < (unsigned int) desired_align * BITS_PER_UNIT)
9108 0 : set_mem_align (dst, desired_align * BITS_PER_UNIT);
9109 0 : if (MEM_SIZE_KNOWN_P (orig_dst))
9110 0 : set_mem_size (dst, MEM_SIZE (orig_dst) - align_bytes);
9111 :
9112 0 : if (!issetmem)
9113 : {
9114 0 : int src_align_bytes = get_mem_align_offset (src, desired_align
9115 : * BITS_PER_UNIT);
9116 0 : if (src_align_bytes >= 0)
9117 0 : src_align_bytes = desired_align - src_align_bytes;
9118 0 : if (src_align_bytes >= 0)
9119 : {
9120 : unsigned int src_align;
9121 0 : for (src_align = desired_align; src_align >= 2; src_align >>= 1)
9122 : {
9123 0 : if ((src_align_bytes & (src_align - 1))
9124 0 : == (align_bytes & (src_align - 1)))
9125 : break;
9126 : }
9127 0 : if (src_align > (unsigned int) desired_align)
9128 : src_align = desired_align;
9129 0 : if (MEM_ALIGN (src) < src_align * BITS_PER_UNIT)
9130 0 : set_mem_align (src, src_align * BITS_PER_UNIT);
9131 : }
9132 0 : if (MEM_SIZE_KNOWN_P (orig_src))
9133 0 : set_mem_size (src, MEM_SIZE (orig_src) - align_bytes);
9134 0 : *srcp = src;
9135 : }
9136 :
9137 0 : return dst;
9138 : }
9139 :
9140 : /* Return true if ALG can be used in current context.
9141 : Assume we expand memset if MEMSET is true. */
9142 : static bool
9143 1454129 : alg_usable_p (enum stringop_alg alg, bool memset,
9144 : addr_space_t dst_as, addr_space_t src_as)
9145 : {
9146 1454129 : if (alg == no_stringop)
9147 : return false;
9148 : /* It is not possible to use a library call if we have non-default
9149 : address space. We can do better than the generic byte-at-a-time
9150 : loop, used as a fallback. */
9151 1454129 : if (alg == libcall &&
9152 834415 : !(ADDR_SPACE_GENERIC_P (dst_as) && ADDR_SPACE_GENERIC_P (src_as)))
9153 : return false;
9154 1454115 : if (alg == vector_loop)
9155 311140 : return TARGET_SSE || TARGET_AVX;
9156 : /* Algorithms using the rep prefix want at least edi and ecx;
9157 : additionally, memset wants eax and memcpy wants esi. Don't
9158 : consider such algorithms if the user has appropriated those
9159 : registers for their own purposes, or if we have the destination
9160 : in the non-default address space, since string insns cannot
9161 : override the destination segment. */
9162 1142975 : if (alg == rep_prefix_1_byte
9163 : || alg == rep_prefix_4_byte
9164 1142975 : || alg == rep_prefix_8_byte)
9165 : {
9166 38171 : if (fixed_regs[CX_REG]
9167 38167 : || fixed_regs[DI_REG]
9168 38163 : || (memset ? fixed_regs[AX_REG] : fixed_regs[SI_REG])
9169 38159 : || !ADDR_SPACE_GENERIC_P (dst_as)
9170 76330 : || !(ADDR_SPACE_GENERIC_P (src_as) || Pmode == word_mode))
9171 12 : return false;
9172 : }
9173 : return true;
9174 : }
9175 :
9176 : /* Given COUNT and EXPECTED_SIZE, decide on codegen of string operation. */
9177 : static enum stringop_alg
9178 289304 : decide_alg (HOST_WIDE_INT count, HOST_WIDE_INT expected_size,
9179 : unsigned HOST_WIDE_INT min_size, unsigned HOST_WIDE_INT max_size,
9180 : bool memset, bool zero_memset, addr_space_t dst_as,
9181 : addr_space_t src_as, int *dynamic_check, bool *noalign, bool recur)
9182 : {
9183 289304 : const struct stringop_algs *algs;
9184 289304 : bool optimize_for_speed;
9185 289304 : int max = 0;
9186 289304 : const struct processor_costs *cost;
9187 289304 : int i;
9188 289304 : bool any_alg_usable_p = false;
9189 :
9190 289304 : *noalign = false;
9191 289304 : *dynamic_check = -1;
9192 :
9193 : /* Even if the string operation call is cold, we still might spend a lot
9194 : of time processing large blocks. */
9195 289304 : if (optimize_function_for_size_p (cfun)
9196 289304 : || (optimize_insn_for_size_p ()
9197 12330 : && (max_size < 256
9198 4018 : || (expected_size != -1 && expected_size < 256))))
9199 : optimize_for_speed = false;
9200 : else
9201 270370 : optimize_for_speed = true;
9202 :
9203 270370 : cost = optimize_for_speed ? ix86_cost : &ix86_size_cost;
9204 289304 : if (memset)
9205 88353 : algs = &cost->memset[TARGET_64BIT != 0];
9206 : else
9207 209802 : algs = &cost->memcpy[TARGET_64BIT != 0];
9208 :
9209 : /* See maximal size for user defined algorithm. */
9210 1446520 : for (i = 0; i < MAX_STRINGOP_ALGS; i++)
9211 : {
9212 1157216 : enum stringop_alg candidate = algs->size[i].alg;
9213 1157216 : bool usable = alg_usable_p (candidate, memset, dst_as, src_as);
9214 1157216 : any_alg_usable_p |= usable;
9215 :
9216 1157216 : if (candidate != libcall && candidate && usable)
9217 559561 : max = algs->size[i].max;
9218 : }
9219 :
9220 : /* If expected size is not known but max size is small enough
9221 : so inline version is a win, set expected size into
9222 : the range. */
9223 289304 : if (((max > 1 && (unsigned HOST_WIDE_INT) max >= max_size) || max == -1)
9224 38177 : && expected_size == -1)
9225 15550 : expected_size = min_size / 2 + max_size / 2;
9226 :
9227 : /* If user specified the algorithm, honor it if possible. */
9228 289304 : if (ix86_stringop_alg != no_stringop
9229 289304 : && alg_usable_p (ix86_stringop_alg, memset, dst_as, src_as))
9230 : return ix86_stringop_alg;
9231 : /* rep; movq or rep; movl is the smallest variant. */
9232 289166 : else if (!optimize_for_speed)
9233 : {
9234 18851 : *noalign = true;
9235 18851 : if (!count || (count & 3) || (memset && !zero_memset))
9236 5824 : return alg_usable_p (rep_prefix_1_byte, memset, dst_as, src_as)
9237 5824 : ? rep_prefix_1_byte : loop_1_byte;
9238 : else
9239 13027 : return alg_usable_p (rep_prefix_4_byte, memset, dst_as, src_as)
9240 13027 : ? rep_prefix_4_byte : loop;
9241 : }
9242 : /* Very tiny blocks are best handled via the loop, REP is expensive to
9243 : setup. */
9244 270315 : else if (expected_size != -1 && expected_size < 4)
9245 : return loop_1_byte;
9246 268785 : else if (expected_size != -1)
9247 : {
9248 : enum stringop_alg alg = libcall;
9249 : bool alg_noalign = false;
9250 343010 : for (i = 0; i < MAX_STRINGOP_ALGS; i++)
9251 : {
9252 : /* We get here if the algorithms that were not libcall-based
9253 : were rep-prefix based and we are unable to use rep prefixes
9254 : based on global register usage. Break out of the loop and
9255 : use the heuristic below. */
9256 337273 : if (algs->size[i].max == 0)
9257 : break;
9258 337273 : if (algs->size[i].max >= expected_size || algs->size[i].max == -1)
9259 : {
9260 132822 : enum stringop_alg candidate = algs->size[i].alg;
9261 :
9262 132822 : if (candidate != libcall
9263 132822 : && alg_usable_p (candidate, memset, dst_as, src_as))
9264 : {
9265 24835 : alg = candidate;
9266 24835 : alg_noalign = algs->size[i].noalign;
9267 : }
9268 : /* Honor TARGET_INLINE_ALL_STRINGOPS by picking
9269 : last non-libcall inline algorithm. */
9270 132822 : if (TARGET_INLINE_ALL_STRINGOPS)
9271 : {
9272 : /* When the current size is best to be copied by a libcall,
9273 : but we are still forced to inline, run the heuristic below
9274 : that will pick code for medium sized blocks. */
9275 20038 : if (alg != libcall)
9276 : {
9277 8555 : *noalign = alg_noalign;
9278 8555 : return alg;
9279 : }
9280 11483 : else if (!any_alg_usable_p)
9281 : break;
9282 : }
9283 112784 : else if (alg_usable_p (candidate, memset, dst_as, src_as)
9284 112784 : && !(TARGET_PREFER_KNOWN_REP_MOVSB_STOSB
9285 34 : && candidate == rep_prefix_1_byte
9286 : /* NB: If min_size != max_size, size is
9287 : unknown. */
9288 34 : && min_size != max_size))
9289 : {
9290 112759 : *noalign = algs->size[i].noalign;
9291 112759 : return candidate;
9292 : }
9293 : }
9294 : }
9295 : }
9296 : /* When asked to inline the call anyway, try to pick meaningful choice.
9297 : We look for maximal size of block that is faster to copy by hand and
9298 : take blocks of at most of that size guessing that average size will
9299 : be roughly half of the block.
9300 :
9301 : If this turns out to be bad, we might simply specify the preferred
9302 : choice in ix86_costs. */
9303 140282 : if ((TARGET_INLINE_ALL_STRINGOPS || TARGET_INLINE_STRINGOPS_DYNAMICALLY)
9304 147477 : && (algs->unknown_size == libcall
9305 0 : || !alg_usable_p (algs->unknown_size, memset, dst_as, src_as)))
9306 : {
9307 7195 : enum stringop_alg alg;
9308 7195 : HOST_WIDE_INT new_expected_size = (max > 0 ? max : 4096) / 2;
9309 :
9310 : /* If there aren't any usable algorithms or if recursing already,
9311 : then recursing on smaller sizes or same size isn't going to
9312 : find anything. Just return the simple byte-at-a-time copy loop. */
9313 7195 : if (!any_alg_usable_p || recur)
9314 : {
9315 : /* Pick something reasonable. */
9316 0 : if (TARGET_INLINE_STRINGOPS_DYNAMICALLY && !recur)
9317 0 : *dynamic_check = 128;
9318 : return loop_1_byte;
9319 : }
9320 7195 : alg = decide_alg (count, new_expected_size, min_size, max_size,
9321 : memset, zero_memset, dst_as, src_as,
9322 : dynamic_check, noalign, true);
9323 7195 : gcc_assert (*dynamic_check == -1);
9324 7195 : if (TARGET_INLINE_STRINGOPS_DYNAMICALLY)
9325 10 : *dynamic_check = max;
9326 : else
9327 7185 : gcc_assert (alg != libcall);
9328 : return alg;
9329 : }
9330 :
9331 : /* Try to use some reasonable fallback algorithm. Note that for
9332 : non-default address spaces we default to a loop instead of
9333 : a libcall. */
9334 :
9335 140276 : bool have_as = !(ADDR_SPACE_GENERIC_P (dst_as)
9336 : && ADDR_SPACE_GENERIC_P (src_as));
9337 :
9338 140276 : return (alg_usable_p (algs->unknown_size, memset, dst_as, src_as)
9339 140276 : ? algs->unknown_size : have_as ? loop : libcall);
9340 : }
9341 :
9342 : /* Decide on alignment. We know that the operand is already aligned to ALIGN
9343 : (ALIGN can be based on profile feedback and thus it is not 100% guaranteed). */
9344 : static int
9345 27049 : decide_alignment (int align,
9346 : enum stringop_alg alg,
9347 : int expected_size,
9348 : machine_mode move_mode)
9349 : {
9350 27049 : int desired_align = 0;
9351 :
9352 27049 : gcc_assert (alg != no_stringop);
9353 :
9354 27049 : if (alg == libcall)
9355 : return 0;
9356 27049 : if (move_mode == VOIDmode)
9357 : return 0;
9358 :
9359 27049 : desired_align = GET_MODE_SIZE (move_mode);
9360 : /* PentiumPro has special logic triggering for 8 byte aligned blocks.
9361 : copying whole cacheline at once. */
9362 27049 : if (TARGET_CPU_P (PENTIUMPRO)
9363 0 : && (alg == rep_prefix_4_byte || alg == rep_prefix_1_byte))
9364 27049 : desired_align = 8;
9365 :
9366 27049 : if (optimize_size)
9367 10067 : desired_align = 1;
9368 27049 : if (desired_align < align)
9369 : desired_align = align;
9370 27049 : if (expected_size != -1 && expected_size < 4)
9371 1 : desired_align = align;
9372 :
9373 : return desired_align;
9374 : }
9375 :
9376 :
9377 : /* Helper function for memcpy. For QImode value 0xXY produce
9378 : 0xXYXYXYXY of wide specified by MODE. This is essentially
9379 : a * 0x10101010, but we can do slightly better than
9380 : synth_mult by unwinding the sequence by hand on CPUs with
9381 : slow multiply. */
9382 : static rtx
9383 18596 : promote_duplicated_reg (machine_mode mode, rtx val)
9384 : {
9385 18596 : if (val == const0_rtx)
9386 16115 : return copy_to_mode_reg (mode, CONST0_RTX (mode));
9387 :
9388 2481 : machine_mode valmode = GET_MODE (val);
9389 2481 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
9390 : {
9391 : /* Duplicate the scalar value for integer vector. */
9392 1866 : gcc_assert ((val == const0_rtx || val == constm1_rtx)
9393 : || GET_MODE_INNER (mode) == valmode);
9394 937 : rtx dup = gen_reg_rtx (mode);
9395 937 : bool ok = ix86_expand_vector_init_duplicate (false, mode, dup,
9396 : val);
9397 937 : gcc_assert (ok);
9398 : return dup;
9399 : }
9400 :
9401 1544 : rtx tmp;
9402 1544 : int nops = mode == DImode ? 3 : 2;
9403 :
9404 40 : gcc_assert (mode == SImode || mode == DImode);
9405 1544 : if (CONST_INT_P (val))
9406 : {
9407 1233 : HOST_WIDE_INT v = INTVAL (val) & 255;
9408 :
9409 1233 : v |= v << 8;
9410 1233 : v |= v << 16;
9411 1233 : if (mode == DImode)
9412 1205 : v |= (v << 16) << 16;
9413 1233 : return copy_to_mode_reg (mode, gen_int_mode (v, mode));
9414 : }
9415 :
9416 311 : if (valmode == VOIDmode)
9417 : valmode = QImode;
9418 311 : if (valmode != QImode)
9419 0 : val = gen_lowpart (QImode, val);
9420 311 : if (mode == QImode)
9421 : return val;
9422 311 : if (!TARGET_PARTIAL_REG_STALL)
9423 311 : nops--;
9424 311 : if (ix86_cost->mult_init[mode == DImode ? 3 : 2]
9425 311 : + ix86_cost->mult_bit * (mode == DImode ? 8 : 4)
9426 311 : <= (ix86_cost->shift_const + ix86_cost->add) * nops
9427 311 : + (COSTS_N_INSNS (TARGET_PARTIAL_REG_STALL == 0)))
9428 : {
9429 311 : rtx reg = convert_modes (mode, QImode, val, true);
9430 311 : tmp = promote_duplicated_reg (mode, const1_rtx);
9431 311 : return expand_simple_binop (mode, MULT, reg, tmp, NULL, 1,
9432 311 : OPTAB_DIRECT);
9433 : }
9434 : else
9435 : {
9436 0 : rtx reg = convert_modes (mode, QImode, val, true);
9437 :
9438 0 : if (!TARGET_PARTIAL_REG_STALL)
9439 0 : emit_insn (gen_insv_1 (mode, reg, reg));
9440 : else
9441 : {
9442 0 : tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (8),
9443 : NULL, 1, OPTAB_DIRECT);
9444 0 : reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1,
9445 : OPTAB_DIRECT);
9446 : }
9447 0 : tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (16),
9448 : NULL, 1, OPTAB_DIRECT);
9449 0 : reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
9450 0 : if (mode == SImode)
9451 : return reg;
9452 0 : tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (32),
9453 : NULL, 1, OPTAB_DIRECT);
9454 0 : reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
9455 0 : return reg;
9456 : }
9457 : }
9458 :
9459 : /* Duplicate value VAL using promote_duplicated_reg into maximal size that will
9460 : be needed by main loop copying SIZE_NEEDED chunks and prologue getting
9461 : alignment from ALIGN to DESIRED_ALIGN. */
9462 : static rtx
9463 13301 : promote_duplicated_reg_to_size (rtx val, int size_needed, int desired_align,
9464 : int align)
9465 : {
9466 13301 : rtx promoted_val;
9467 :
9468 13301 : if (TARGET_64BIT
9469 11825 : && (size_needed > 4 || (desired_align > align && desired_align > 4)))
9470 3945 : promoted_val = promote_duplicated_reg (DImode, val);
9471 9356 : else if (size_needed > 2 || (desired_align > align && desired_align > 2))
9472 8001 : promoted_val = promote_duplicated_reg (SImode, val);
9473 1355 : else if (size_needed > 1 || (desired_align > align && desired_align > 1))
9474 0 : promoted_val = promote_duplicated_reg (HImode, val);
9475 : else
9476 : promoted_val = val;
9477 :
9478 13301 : return promoted_val;
9479 : }
9480 :
9481 : /* Copy the address to a Pmode register. This is used for x32 to
9482 : truncate DImode TLS address to a SImode register. */
9483 :
9484 : static rtx
9485 65478 : ix86_copy_addr_to_reg (rtx addr)
9486 : {
9487 65478 : rtx reg;
9488 70003 : if (GET_MODE (addr) == Pmode || GET_MODE (addr) == VOIDmode)
9489 : {
9490 65478 : reg = copy_addr_to_reg (addr);
9491 65478 : REG_POINTER (reg) = 1;
9492 65478 : return reg;
9493 : }
9494 : else
9495 : {
9496 0 : gcc_assert (GET_MODE (addr) == DImode && Pmode == SImode);
9497 0 : reg = copy_to_mode_reg (DImode, addr);
9498 0 : REG_POINTER (reg) = 1;
9499 0 : return gen_rtx_SUBREG (SImode, reg, 0);
9500 : }
9501 : }
9502 :
9503 : /* Expand string move (memcpy) ot store (memset) operation. Use i386 string
9504 : operations when profitable. The code depends upon architecture, block size
9505 : and alignment, but always has one of the following overall structures:
9506 :
9507 : Aligned move sequence:
9508 :
9509 : 1) Prologue guard: Conditional that jumps up to epilogues for small
9510 : blocks that can be handled by epilogue alone. This is faster
9511 : but also needed for correctness, since prologue assume the block
9512 : is larger than the desired alignment.
9513 :
9514 : Optional dynamic check for size and libcall for large
9515 : blocks is emitted here too, with -minline-stringops-dynamically.
9516 :
9517 : 2) Prologue: copy first few bytes in order to get destination
9518 : aligned to DESIRED_ALIGN. It is emitted only when ALIGN is less
9519 : than DESIRED_ALIGN and up to DESIRED_ALIGN - ALIGN bytes can be
9520 : copied. We emit either a jump tree on power of two sized
9521 : blocks, or a byte loop.
9522 :
9523 : 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
9524 : with specified algorithm.
9525 :
9526 : 4) Epilogue: code copying tail of the block that is too small to be
9527 : handled by main body (or up to size guarded by prologue guard).
9528 :
9529 : Misaligned move sequence
9530 :
9531 : 1) missaligned move prologue/epilogue containing:
9532 : a) Prologue handling small memory blocks and jumping to done_label
9533 : (skipped if blocks are known to be large enough)
9534 : b) Single move copying first DESIRED_ALIGN-ALIGN bytes if alignment is
9535 : needed by single possibly misaligned move
9536 : (skipped if alignment is not needed)
9537 : c) Copy of last SIZE_NEEDED bytes by possibly misaligned moves
9538 :
9539 : 2) Zero size guard dispatching to done_label, if needed
9540 :
9541 : 3) dispatch to library call, if needed,
9542 :
9543 : 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
9544 : with specified algorithm. */
9545 : bool
9546 153560 : ix86_expand_set_or_cpymem (rtx dst, rtx src, rtx count_exp, rtx val_exp,
9547 : rtx align_exp, rtx expected_align_exp,
9548 : rtx expected_size_exp, rtx min_size_exp,
9549 : rtx max_size_exp, rtx probable_max_size_exp,
9550 : bool issetmem)
9551 : {
9552 153560 : if (TARGET_MISALIGNED_MOVE_STRING_PRO_EPILOGUES)
9553 : {
9554 : /* Expand bounded memset and memcpy as memmove if misaligned moves
9555 : are preferred. Since
9556 :
9557 : commit b41f96465190751561f6909e858604ceab00595b
9558 : Author: H.J. Lu <hjl.tools@gmail.com>
9559 : Date: Mon Oct 20 16:14:34 2025 +0800
9560 :
9561 : x86-64: Inline memmove with overlapping unaligned loads and stores.
9562 :
9563 : inlines memmove with overlapping unaligned and stores, which
9564 : reduces the numbers of branches and memory moves, comparing
9565 : against the regular memset and memcpy inlining. */
9566 153542 : rtx operands[9];
9567 153542 : operands[0] = dst;
9568 153542 : operands[1] = issetmem ? val_exp : src;
9569 153542 : operands[2] = count_exp;
9570 153542 : operands[3] = align_exp;
9571 153542 : operands[4] = expected_align_exp;
9572 153542 : operands[5] = expected_size_exp;
9573 153542 : operands[6] = min_size_exp;
9574 153542 : operands[7] = max_size_exp;
9575 153542 : operands[8] = probable_max_size_exp;
9576 153542 : if (ix86_expand_set_or_movmem (operands, !issetmem, issetmem))
9577 8668 : return true;
9578 : }
9579 :
9580 144892 : rtx destreg;
9581 144892 : rtx srcreg = NULL;
9582 144892 : rtx_code_label *label = NULL;
9583 144892 : rtx tmp;
9584 144892 : rtx_code_label *jump_around_label = NULL;
9585 144892 : HOST_WIDE_INT align = 1;
9586 144892 : unsigned HOST_WIDE_INT count = 0;
9587 144892 : HOST_WIDE_INT expected_size = -1;
9588 144892 : int size_needed = 0, epilogue_size_needed;
9589 144892 : int desired_align = 0, align_bytes = 0;
9590 144892 : enum stringop_alg alg;
9591 144892 : rtx promoted_val = NULL;
9592 144892 : rtx vec_promoted_val = NULL;
9593 144892 : bool force_loopy_epilogue = false;
9594 144892 : int dynamic_check;
9595 144892 : bool need_zero_guard = false;
9596 144892 : bool noalign;
9597 144892 : machine_mode move_mode = VOIDmode;
9598 144892 : int unroll_factor = 1;
9599 : /* TODO: Once value ranges are available, fill in proper data. */
9600 144892 : unsigned HOST_WIDE_INT min_size = HOST_WIDE_INT_0U;
9601 144892 : unsigned HOST_WIDE_INT max_size = HOST_WIDE_INT_M1U;
9602 144892 : unsigned HOST_WIDE_INT probable_max_size = HOST_WIDE_INT_M1U;
9603 144892 : bool misaligned_prologue_used = false;
9604 144892 : addr_space_t dst_as, src_as = ADDR_SPACE_GENERIC;
9605 :
9606 144892 : if (CONST_INT_P (align_exp))
9607 144892 : align = INTVAL (align_exp);
9608 : /* i386 can do misaligned access on reasonably increased cost. */
9609 144892 : if (CONST_INT_P (expected_align_exp)
9610 144892 : && INTVAL (expected_align_exp) > align)
9611 : align = INTVAL (expected_align_exp);
9612 : /* ALIGN is the minimum of destination and source alignment, but we care here
9613 : just about destination alignment. */
9614 138647 : else if (!issetmem
9615 230605 : && MEM_ALIGN (dst) > (unsigned HOST_WIDE_INT) align * BITS_PER_UNIT)
9616 3288 : align = MEM_ALIGN (dst) / BITS_PER_UNIT;
9617 :
9618 144892 : if (CONST_INT_P (count_exp))
9619 : {
9620 70791 : min_size = max_size = probable_max_size = count = expected_size
9621 70791 : = INTVAL (count_exp);
9622 : /* When COUNT is 0, there is nothing to do. */
9623 70791 : if (!count)
9624 : return true;
9625 : }
9626 : else
9627 : {
9628 74101 : if (min_size_exp)
9629 74101 : min_size = INTVAL (min_size_exp);
9630 74101 : if (max_size_exp)
9631 59038 : max_size = INTVAL (max_size_exp);
9632 74101 : if (probable_max_size_exp)
9633 60488 : probable_max_size = INTVAL (probable_max_size_exp);
9634 74101 : if (CONST_INT_P (expected_size_exp))
9635 74101 : expected_size = INTVAL (expected_size_exp);
9636 : }
9637 :
9638 : /* Make sure we don't need to care about overflow later on. */
9639 144892 : if (count > (HOST_WIDE_INT_1U << 30))
9640 : return false;
9641 :
9642 144717 : dst_as = MEM_ADDR_SPACE (dst);
9643 144717 : if (!issetmem)
9644 98089 : src_as = MEM_ADDR_SPACE (src);
9645 :
9646 : /* Step 0: Decide on preferred algorithm, desired alignment and
9647 : size of chunks to be copied by main loop. */
9648 144717 : alg = decide_alg (count, expected_size, min_size, probable_max_size,
9649 46628 : issetmem, issetmem && val_exp == const0_rtx,
9650 : dst_as, src_as, &dynamic_check, &noalign, false);
9651 :
9652 144717 : if (dump_file)
9653 7 : fprintf (dump_file, "Selected stringop expansion strategy: %s\n",
9654 7 : stringop_alg_names[alg]);
9655 :
9656 144717 : if (alg == libcall)
9657 : return false;
9658 27049 : gcc_assert (alg != no_stringop);
9659 :
9660 27049 : if (!count)
9661 5684 : count_exp = copy_to_mode_reg (GET_MODE (count_exp), count_exp);
9662 27049 : destreg = ix86_copy_addr_to_reg (XEXP (dst, 0));
9663 27049 : if (!issetmem)
9664 13748 : srcreg = ix86_copy_addr_to_reg (XEXP (src, 0));
9665 :
9666 27049 : bool aligned_dstmem = false;
9667 27049 : unsigned int nunits = issetmem ? STORE_MAX_PIECES : MOVE_MAX;
9668 27049 : bool single_insn_p = count && count <= nunits;
9669 27049 : if (single_insn_p)
9670 : {
9671 : /* If it can be done with a single instruction, use vector
9672 : instruction and don't align destination. */
9673 7 : alg = vector_loop;
9674 7 : noalign = true;
9675 7 : dynamic_check = -1;
9676 : }
9677 :
9678 27049 : unroll_factor = 1;
9679 27049 : move_mode = word_mode;
9680 27049 : switch (alg)
9681 : {
9682 0 : case libcall:
9683 0 : case no_stringop:
9684 0 : case last_alg:
9685 0 : gcc_unreachable ();
9686 173 : case loop_1_byte:
9687 173 : need_zero_guard = true;
9688 173 : move_mode = QImode;
9689 173 : break;
9690 46 : case loop:
9691 46 : need_zero_guard = true;
9692 46 : break;
9693 19 : case unrolled_loop:
9694 19 : need_zero_guard = true;
9695 19 : unroll_factor = (TARGET_64BIT ? 4 : 2);
9696 : break;
9697 7935 : case vector_loop:
9698 7935 : need_zero_guard = true;
9699 7935 : unroll_factor = 4;
9700 : /* Get the vector mode to move STORE_MAX_PIECES/MOVE_MAX bytes. */
9701 7935 : nunits /= GET_MODE_SIZE (word_mode);
9702 7935 : if (nunits > 1)
9703 : {
9704 7931 : move_mode = mode_for_vector (word_mode, nunits).require ();
9705 7931 : gcc_assert (optab_handler (mov_optab, move_mode)
9706 : != CODE_FOR_nothing);
9707 : }
9708 : break;
9709 25 : case rep_prefix_8_byte:
9710 25 : move_mode = DImode;
9711 25 : break;
9712 13023 : case rep_prefix_4_byte:
9713 13023 : move_mode = SImode;
9714 13023 : break;
9715 5828 : case rep_prefix_1_byte:
9716 5828 : move_mode = QImode;
9717 5828 : break;
9718 : }
9719 27049 : size_needed = GET_MODE_SIZE (move_mode) * unroll_factor;
9720 27049 : epilogue_size_needed = size_needed;
9721 :
9722 : /* If we are going to call any library calls conditionally, make sure any
9723 : pending stack adjustment happen before the first conditional branch,
9724 : otherwise they will be emitted before the library call only and won't
9725 : happen from the other branches. */
9726 27049 : if (dynamic_check != -1)
9727 2 : do_pending_stack_adjust ();
9728 :
9729 27049 : desired_align = decide_alignment (align, alg, expected_size, move_mode);
9730 27049 : if (!TARGET_ALIGN_STRINGOPS || noalign)
9731 26794 : align = desired_align;
9732 :
9733 : /* Step 1: Prologue guard. */
9734 :
9735 : /* Alignment code needs count to be in register. */
9736 27049 : if (CONST_INT_P (count_exp) && desired_align > align)
9737 : {
9738 22 : if (INTVAL (count_exp) > desired_align
9739 22 : && INTVAL (count_exp) > size_needed)
9740 : {
9741 22 : align_bytes
9742 22 : = get_mem_align_offset (dst, desired_align * BITS_PER_UNIT);
9743 22 : if (align_bytes <= 0)
9744 : align_bytes = 0;
9745 : else
9746 0 : align_bytes = desired_align - align_bytes;
9747 : }
9748 0 : if (align_bytes == 0)
9749 44 : count_exp = force_reg (counter_mode (count_exp), count_exp);
9750 : }
9751 27049 : gcc_assert (desired_align >= 1 && align >= 1);
9752 :
9753 27049 : if (!single_insn_p)
9754 : {
9755 : /* Misaligned move sequences handle both prologue and epilogue
9756 : at once. Default code generation results in a smaller code
9757 : for large alignments and also avoids redundant job when sizes
9758 : are known precisely. */
9759 27042 : misaligned_prologue_used
9760 54084 : = (TARGET_MISALIGNED_MOVE_STRING_PRO_EPILOGUES
9761 27036 : && MAX (desired_align, epilogue_size_needed) <= 32
9762 18704 : && desired_align <= epilogue_size_needed
9763 31626 : && ((desired_align > align && !align_bytes)
9764 4567 : || (!count && epilogue_size_needed > 1)));
9765 :
9766 : /* Destination is aligned after the misaligned prologue. */
9767 27042 : aligned_dstmem = misaligned_prologue_used;
9768 :
9769 27042 : if (noalign
9770 26787 : && !misaligned_prologue_used
9771 26787 : && (!count
9772 21274 : || count > (unsigned HOST_WIDE_INT) epilogue_size_needed))
9773 : {
9774 : /* Also use misaligned prologue if count > epilogue size,
9775 : alignment isn't needed and destination isn't aligned.
9776 : Since alignment isn't needed, the destination after
9777 : prologue won't be aligned. */
9778 26737 : aligned_dstmem = (GET_MODE_ALIGNMENT (move_mode)
9779 26737 : <= MEM_ALIGN (dst));
9780 26737 : if (!aligned_dstmem)
9781 4012 : misaligned_prologue_used = true;
9782 : }
9783 : }
9784 :
9785 : /* Do the cheap promotion to allow better CSE across the
9786 : main loop and epilogue (ie one load of the big constant in the
9787 : front of all code.
9788 : For now the misaligned move sequences do not have fast path
9789 : without broadcasting. */
9790 27049 : if (issetmem
9791 13301 : && (alg == vector_loop
9792 7987 : || CONST_INT_P (val_exp)
9793 49 : || misaligned_prologue_used))
9794 : {
9795 7939 : if (alg == vector_loop)
9796 : {
9797 5314 : promoted_val = promote_duplicated_reg_to_size (val_exp,
9798 10628 : GET_MODE_SIZE (word_mode),
9799 : desired_align, align);
9800 : /* Duplicate the promoted scalar value if not 0 nor -1. */
9801 5314 : vec_promoted_val
9802 5314 : = promote_duplicated_reg (move_mode,
9803 5314 : (val_exp == const0_rtx
9804 569 : || val_exp == constm1_rtx)
9805 : ? val_exp : promoted_val);
9806 : }
9807 : else
9808 : {
9809 7939 : promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
9810 : desired_align, align);
9811 : }
9812 : }
9813 : /* Misaligned move sequences handles both prologues and epilogues at once.
9814 : Default code generation results in smaller code for large alignments and
9815 : also avoids redundant job when sizes are known precisely. */
9816 27001 : if (misaligned_prologue_used)
9817 : {
9818 : /* Misaligned move prologue handled small blocks by itself. */
9819 4029 : expand_set_or_cpymem_prologue_epilogue_by_misaligned_moves
9820 4029 : (dst, src, &destreg, &srcreg,
9821 : move_mode, promoted_val, vec_promoted_val,
9822 : &count_exp,
9823 : &jump_around_label,
9824 4029 : desired_align < align
9825 0 : ? MAX (desired_align, epilogue_size_needed) : epilogue_size_needed,
9826 : desired_align, align, &min_size, dynamic_check, issetmem);
9827 4029 : if (!issetmem)
9828 2102 : src = change_address (src, BLKmode, srcreg);
9829 4029 : dst = change_address (dst, BLKmode, destreg);
9830 4029 : if (aligned_dstmem)
9831 17 : set_mem_align (dst, desired_align * BITS_PER_UNIT);
9832 4029 : epilogue_size_needed = 0;
9833 4029 : if (need_zero_guard
9834 3751 : && min_size < (unsigned HOST_WIDE_INT) size_needed)
9835 : {
9836 : /* It is possible that we copied enough so the main loop will not
9837 : execute. */
9838 662 : gcc_assert (size_needed > 1);
9839 662 : if (jump_around_label == NULL_RTX)
9840 0 : jump_around_label = gen_label_rtx ();
9841 1324 : emit_cmp_and_jump_insns (count_exp,
9842 : GEN_INT (size_needed),
9843 : LTU, 0, counter_mode (count_exp), 1, jump_around_label);
9844 662 : if (expected_size == -1
9845 6 : || expected_size < (desired_align - align) / 2 + size_needed)
9846 656 : predict_jump (REG_BR_PROB_BASE * 20 / 100);
9847 : else
9848 6 : predict_jump (REG_BR_PROB_BASE * 60 / 100);
9849 : }
9850 : }
9851 : /* Ensure that alignment prologue won't copy past end of block. */
9852 23020 : else if (size_needed > 1 || (desired_align > 1 && desired_align > align))
9853 : {
9854 17019 : epilogue_size_needed = MAX (size_needed - 1, desired_align - align);
9855 : /* Epilogue always copies COUNT_EXP & EPILOGUE_SIZE_NEEDED bytes.
9856 : Make sure it is power of 2. */
9857 17019 : epilogue_size_needed = 1 << (floor_log2 (epilogue_size_needed) + 1);
9858 :
9859 : /* To improve performance of small blocks, we jump around the VAL
9860 : promoting mode. This mean that if the promoted VAL is not constant,
9861 : we might not use it in the epilogue and have to use byte
9862 : loop variant. */
9863 17019 : if (issetmem && epilogue_size_needed > 2 && !promoted_val)
9864 17019 : force_loopy_epilogue = true;
9865 17019 : if ((count && count < (unsigned HOST_WIDE_INT) epilogue_size_needed)
9866 17009 : || max_size < (unsigned HOST_WIDE_INT) epilogue_size_needed)
9867 : {
9868 : /* If main algorithm works on QImode, no epilogue is needed.
9869 : For small sizes just don't align anything. */
9870 44 : if (size_needed == 1)
9871 0 : desired_align = align;
9872 : else
9873 44 : goto epilogue;
9874 : }
9875 16975 : else if (!count
9876 62 : && min_size < (unsigned HOST_WIDE_INT) epilogue_size_needed)
9877 : {
9878 62 : label = gen_label_rtx ();
9879 124 : emit_cmp_and_jump_insns (count_exp,
9880 : GEN_INT (epilogue_size_needed),
9881 : LTU, 0, counter_mode (count_exp), 1, label);
9882 62 : if (expected_size == -1 || expected_size < epilogue_size_needed)
9883 62 : predict_jump (REG_BR_PROB_BASE * 60 / 100);
9884 : else
9885 0 : predict_jump (REG_BR_PROB_BASE * 20 / 100);
9886 : }
9887 : }
9888 :
9889 : /* Emit code to decide on runtime whether library call or inline should be
9890 : used. */
9891 27005 : if (dynamic_check != -1)
9892 : {
9893 2 : if (!issetmem && CONST_INT_P (count_exp))
9894 : {
9895 1 : if (UINTVAL (count_exp) >= (unsigned HOST_WIDE_INT)dynamic_check)
9896 : {
9897 1 : emit_block_copy_via_libcall (dst, src, count_exp);
9898 1 : count_exp = const0_rtx;
9899 1 : goto epilogue;
9900 : }
9901 : }
9902 : else
9903 : {
9904 1 : rtx_code_label *hot_label = gen_label_rtx ();
9905 1 : if (jump_around_label == NULL_RTX)
9906 1 : jump_around_label = gen_label_rtx ();
9907 2 : emit_cmp_and_jump_insns (count_exp, GEN_INT (dynamic_check - 1),
9908 : LEU, 0, counter_mode (count_exp),
9909 : 1, hot_label);
9910 1 : predict_jump (REG_BR_PROB_BASE * 90 / 100);
9911 1 : if (issetmem)
9912 1 : set_storage_via_libcall (dst, count_exp, val_exp);
9913 : else
9914 0 : emit_block_copy_via_libcall (dst, src, count_exp);
9915 1 : emit_jump (jump_around_label);
9916 1 : emit_label (hot_label);
9917 : }
9918 : }
9919 :
9920 : /* Step 2: Alignment prologue. */
9921 : /* Do the expensive promotion once we branched off the small blocks. */
9922 27004 : if (issetmem && !promoted_val)
9923 48 : promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
9924 : desired_align, align);
9925 :
9926 27004 : if (desired_align > align && !misaligned_prologue_used)
9927 : {
9928 7 : if (align_bytes == 0)
9929 : {
9930 : /* Except for the first move in prologue, we no longer know
9931 : constant offset in aliasing info. It don't seems to worth
9932 : the pain to maintain it for the first move, so throw away
9933 : the info early. */
9934 7 : dst = change_address (dst, BLKmode, destreg);
9935 7 : if (!issetmem)
9936 5 : src = change_address (src, BLKmode, srcreg);
9937 7 : dst = expand_set_or_cpymem_prologue (dst, src, destreg, srcreg,
9938 : promoted_val, vec_promoted_val,
9939 : count_exp, align, desired_align,
9940 : issetmem);
9941 : /* At most desired_align - align bytes are copied. */
9942 7 : if (min_size < (unsigned)(desired_align - align))
9943 0 : min_size = 0;
9944 : else
9945 7 : min_size -= desired_align - align;
9946 : }
9947 : else
9948 : {
9949 : /* If we know how many bytes need to be stored before dst is
9950 : sufficiently aligned, maintain aliasing info accurately. */
9951 0 : dst = expand_set_or_cpymem_constant_prologue (dst, &src, destreg,
9952 : srcreg,
9953 : promoted_val,
9954 : vec_promoted_val,
9955 : desired_align,
9956 : align_bytes,
9957 : issetmem);
9958 :
9959 0 : count_exp = plus_constant (counter_mode (count_exp),
9960 0 : count_exp, -align_bytes);
9961 0 : count -= align_bytes;
9962 0 : min_size -= align_bytes;
9963 0 : max_size -= align_bytes;
9964 : }
9965 7 : if (need_zero_guard
9966 7 : && min_size < (unsigned HOST_WIDE_INT) size_needed
9967 1 : && (count < (unsigned HOST_WIDE_INT) size_needed
9968 0 : || (align_bytes == 0
9969 0 : && count < ((unsigned HOST_WIDE_INT) size_needed
9970 0 : + desired_align - align))))
9971 : {
9972 : /* It is possible that we copied enough so the main loop will not
9973 : execute. */
9974 1 : gcc_assert (size_needed > 1);
9975 1 : if (label == NULL_RTX)
9976 0 : label = gen_label_rtx ();
9977 2 : emit_cmp_and_jump_insns (count_exp,
9978 : GEN_INT (size_needed),
9979 : LTU, 0, counter_mode (count_exp), 1, label);
9980 1 : if (expected_size == -1
9981 0 : || expected_size < (desired_align - align) / 2 + size_needed)
9982 1 : predict_jump (REG_BR_PROB_BASE * 20 / 100);
9983 : else
9984 0 : predict_jump (REG_BR_PROB_BASE * 60 / 100);
9985 : }
9986 : }
9987 27004 : if (label && size_needed == 1)
9988 : {
9989 0 : emit_label (label);
9990 0 : LABEL_NUSES (label) = 1;
9991 0 : label = NULL;
9992 0 : epilogue_size_needed = 1;
9993 0 : if (issetmem)
9994 0 : promoted_val = val_exp;
9995 : }
9996 27004 : else if (label == NULL_RTX && !misaligned_prologue_used)
9997 22914 : epilogue_size_needed = size_needed;
9998 :
9999 : /* Step 3: Main loop. */
10000 :
10001 27004 : switch (alg)
10002 : {
10003 : case libcall:
10004 : case no_stringop:
10005 : case last_alg:
10006 : gcc_unreachable ();
10007 238 : case loop_1_byte:
10008 238 : case loop:
10009 238 : case unrolled_loop:
10010 238 : expand_set_or_cpymem_via_loop (dst, src, destreg, srcreg, promoted_val,
10011 : count_exp, move_mode, unroll_factor,
10012 : expected_size, issetmem);
10013 238 : break;
10014 7890 : case vector_loop:
10015 7890 : expand_set_or_cpymem_via_loop (dst, src, destreg, srcreg,
10016 : vec_promoted_val, count_exp, move_mode,
10017 : unroll_factor, expected_size, issetmem);
10018 7890 : break;
10019 18876 : case rep_prefix_8_byte:
10020 18876 : case rep_prefix_4_byte:
10021 18876 : case rep_prefix_1_byte:
10022 18876 : expand_set_or_cpymem_via_rep (dst, src, destreg, srcreg, promoted_val,
10023 : val_exp, count_exp, move_mode, issetmem);
10024 18876 : break;
10025 : }
10026 : /* Adjust properly the offset of src and dest memory for aliasing. */
10027 27004 : if (CONST_INT_P (count_exp))
10028 : {
10029 21332 : if (!issetmem)
10030 9099 : src = adjust_automodify_address_nv (src, BLKmode, srcreg,
10031 : (count / size_needed) * size_needed);
10032 21332 : dst = adjust_automodify_address_nv (dst, BLKmode, destreg,
10033 : (count / size_needed) * size_needed);
10034 : }
10035 : else
10036 : {
10037 5672 : if (!issetmem)
10038 4646 : src = change_address (src, BLKmode, srcreg);
10039 5672 : dst = change_address (dst, BLKmode, destreg);
10040 : }
10041 :
10042 : /* Step 4: Epilogue to copy the remaining bytes. */
10043 27049 : epilogue:
10044 27049 : if (label)
10045 : {
10046 : /* When the main loop is done, COUNT_EXP might hold original count,
10047 : while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
10048 : Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
10049 : bytes. Compensate if needed. */
10050 :
10051 62 : if (size_needed < epilogue_size_needed)
10052 : {
10053 0 : tmp = expand_simple_binop (counter_mode (count_exp), AND, count_exp,
10054 0 : GEN_INT (size_needed - 1), count_exp, 1,
10055 : OPTAB_DIRECT);
10056 0 : if (tmp != count_exp)
10057 0 : emit_move_insn (count_exp, tmp);
10058 : }
10059 62 : emit_label (label);
10060 62 : LABEL_NUSES (label) = 1;
10061 : }
10062 :
10063 27049 : if (count_exp != const0_rtx && epilogue_size_needed > 1)
10064 : {
10065 17019 : if (force_loopy_epilogue)
10066 0 : expand_setmem_epilogue_via_loop (dst, destreg, val_exp, count_exp,
10067 : epilogue_size_needed);
10068 : else
10069 : {
10070 17019 : if (issetmem)
10071 10019 : expand_setmem_epilogue (dst, destreg, promoted_val,
10072 : vec_promoted_val, count_exp,
10073 : epilogue_size_needed);
10074 : else
10075 7000 : expand_cpymem_epilogue (dst, src, destreg, srcreg, count_exp,
10076 : epilogue_size_needed);
10077 : }
10078 : }
10079 27049 : if (jump_around_label)
10080 665 : emit_label (jump_around_label);
10081 : return true;
10082 : }
10083 :
10084 : /* Fully unroll memmove of known size with up to 8 registers. */
10085 :
10086 : static bool
10087 1884 : ix86_expand_unroll_movmem (rtx dst, rtx src, rtx destreg, rtx srcreg,
10088 : unsigned HOST_WIDE_INT count,
10089 : machine_mode mode)
10090 : {
10091 : /* If 8 registers registers can cover all memory, load them into
10092 : registers and store them together to avoid possible address
10093 : overlap between source and destination. */
10094 1884 : unsigned HOST_WIDE_INT moves = count / GET_MODE_SIZE (mode);
10095 1884 : if (moves == 0)
10096 : {
10097 0 : mode = smallest_int_mode_for_size
10098 0 : (count * BITS_PER_UNIT).require ();
10099 0 : if (count == GET_MODE_SIZE (mode))
10100 : moves = 1;
10101 : else
10102 : {
10103 : /* Reduce the smallest move size by half so that MOVES == 1. */
10104 0 : mode = smallest_int_mode_for_size
10105 0 : (GET_MODE_BITSIZE (mode) / 2).require ();
10106 0 : moves = count / GET_MODE_SIZE (mode);
10107 0 : gcc_assert (moves == 1);
10108 : }
10109 : }
10110 1884 : else if (moves > 8)
10111 : return false;
10112 :
10113 1875 : unsigned int i;
10114 1875 : rtx tmp[9];
10115 :
10116 4318 : for (i = 0; i < moves; i++)
10117 2443 : tmp[i] = gen_reg_rtx (mode);
10118 :
10119 1875 : rtx srcmem = change_address (src, mode, srcreg);
10120 6193 : for (i = 0; i < moves; i++)
10121 : {
10122 2443 : emit_move_insn (tmp[i], srcmem);
10123 4886 : srcmem = offset_address (srcmem,
10124 2443 : GEN_INT (GET_MODE_SIZE (mode)),
10125 2443 : GET_MODE_SIZE (mode));
10126 : }
10127 :
10128 1875 : unsigned int epilogue_size = count & (GET_MODE_SIZE (mode) - 1);
10129 1875 : machine_mode epilogue_mode = VOIDmode;
10130 1875 : if (epilogue_size)
10131 : {
10132 : /* Handle the remaining bytes with overlapping move. */
10133 1702 : epilogue_mode = smallest_int_mode_for_size
10134 1702 : (epilogue_size * BITS_PER_UNIT).require ();
10135 1702 : tmp[8] = gen_reg_rtx (epilogue_mode);
10136 1702 : srcmem = adjust_address (srcmem, epilogue_mode, 0);
10137 1702 : srcmem = offset_address (srcmem, GEN_INT (epilogue_size), 1);
10138 3404 : srcmem = offset_address (srcmem,
10139 1702 : GEN_INT (-GET_MODE_SIZE (epilogue_mode)),
10140 1702 : GET_MODE_SIZE (epilogue_mode));
10141 1702 : emit_move_insn (tmp[8], srcmem);
10142 : }
10143 :
10144 1875 : rtx destmem = change_address (dst, mode, destreg);
10145 6193 : for (i = 0; i < moves; i++)
10146 : {
10147 2443 : emit_move_insn (destmem, tmp[i]);
10148 4886 : destmem = offset_address (destmem,
10149 2443 : GEN_INT (GET_MODE_SIZE (mode)),
10150 2443 : GET_MODE_SIZE (mode));
10151 : }
10152 :
10153 1875 : if (epilogue_size)
10154 : {
10155 : /* Use overlapping move. */
10156 1702 : destmem = adjust_address (destmem, epilogue_mode, 0);
10157 1702 : destmem = offset_address (destmem, GEN_INT (epilogue_size), 1);
10158 3404 : destmem = offset_address (destmem,
10159 1702 : GEN_INT (-GET_MODE_SIZE (epilogue_mode)),
10160 1702 : GET_MODE_SIZE (epilogue_mode));
10161 1702 : emit_move_insn (destmem, tmp[8]);
10162 : }
10163 :
10164 : return true;
10165 : }
10166 :
10167 : /* Value kind in MEMSET_VALS:
10168 :
10169 : memset_val_byte: The value rtx in QImode.
10170 : memset_val_word: The value rtx in word_mode.
10171 : memset_val_vector: The value rtx in QI vector mode.
10172 :
10173 : */
10174 : enum memset_val_kind
10175 : {
10176 : memset_val_byte = 0,
10177 : memset_val_word = 1,
10178 : memset_val_vector = 2,
10179 : memset_val_max = 3
10180 : };
10181 :
10182 : /* Return a value rtx in MODE for memset from MEMSET_VALS. */
10183 :
10184 : static rtx
10185 4278 : ix86_expand_memset_val (rtx *memset_vals, machine_mode mode)
10186 : {
10187 4278 : rtx byte_val = memset_vals[memset_val_byte];
10188 :
10189 4278 : if (mode == QImode)
10190 : return byte_val;
10191 4278 : else if (mode == word_mode)
10192 975 : return memset_vals[memset_val_word];
10193 :
10194 : /* All-zero/all-ones is a property of the original byte value, so
10195 : detect it once here rather than re-deriving it from each slot. */
10196 3303 : if (byte_val == const0_rtx)
10197 1254 : return CONST0_RTX (mode);
10198 2049 : if (byte_val == constm1_rtx)
10199 78 : return CONSTM1_RTX (mode);
10200 :
10201 1971 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
10202 : {
10203 684 : if (GET_MODE (memset_vals[memset_val_vector]) == mode)
10204 : return memset_vals[memset_val_vector];
10205 7 : return gen_rtx_SUBREG (mode, memset_vals[memset_val_vector], 0);
10206 : }
10207 :
10208 1287 : gcc_assert (mode == HImode || mode == SImode);
10209 1287 : return gen_rtx_SUBREG (mode, memset_vals[memset_val_word], 0);
10210 : }
10211 :
10212 : /* Expand memmove of size with MOVES * mode size and MOVES <= 4. If
10213 : FORWARD is true, copy forward. Otherwise copy backward. */
10214 :
10215 : static void
10216 2433 : ix86_expand_n_move_set_or_movmem (rtx destmem, rtx srcmem,
10217 : rtx *memset_vals, machine_mode mode,
10218 : unsigned int moves, bool forward)
10219 : {
10220 2433 : gcc_assert (moves <= 4);
10221 :
10222 2433 : unsigned int i;
10223 2433 : rtx tmp[8];
10224 :
10225 2433 : rtx step;
10226 2433 : if (forward)
10227 2560 : step = GEN_INT (GET_MODE_SIZE (mode));
10228 : else
10229 2306 : step = GEN_INT (-GET_MODE_SIZE (mode));
10230 :
10231 2433 : if (memset_vals)
10232 : {
10233 : /* Expand memset. */
10234 99 : rtx val = ix86_expand_memset_val (memset_vals, mode);
10235 594 : for (i = 0; i < moves; i++)
10236 396 : tmp[i] = val;
10237 : }
10238 : else
10239 : {
10240 : /* Expand memmove. */
10241 11670 : for (i = 0; i < moves; i++)
10242 9336 : tmp[i] = gen_reg_rtx (mode);
10243 :
10244 : /* Load MOVES. */
10245 9336 : for (i = 0; i < moves - 1; i++)
10246 : {
10247 7002 : emit_move_insn (tmp[i], srcmem);
10248 14004 : srcmem = offset_address (srcmem, step, GET_MODE_SIZE (mode));
10249 : }
10250 2334 : emit_move_insn (tmp[i], srcmem);
10251 : }
10252 :
10253 : /* Store MOVES. */
10254 9732 : for (i = 0; i < moves - 1; i++)
10255 : {
10256 7299 : emit_move_insn (destmem, tmp[i]);
10257 14598 : destmem = offset_address (destmem, step, GET_MODE_SIZE (mode));
10258 : }
10259 2433 : emit_move_insn (destmem, tmp[i]);
10260 2433 : }
10261 :
10262 : /* Load MOVES of mode size into REGS. If LAST is true, load the
10263 : last MOVES. Otherwise, load the first MOVES. */
10264 :
10265 : static void
10266 2334 : ix86_expand_load_movmem (rtx src, rtx srcreg, rtx count_exp,
10267 : machine_mode mode, unsigned int moves,
10268 : rtx regs[], bool last)
10269 : {
10270 2334 : unsigned int i;
10271 :
10272 11670 : for (i = 0; i < moves; i++)
10273 9336 : regs[i] = gen_reg_rtx (mode);
10274 :
10275 2334 : rtx srcmem = change_address (src, mode, srcreg);
10276 2334 : rtx step;
10277 2334 : if (last)
10278 : {
10279 1181 : srcmem = offset_address (srcmem, count_exp, 1);
10280 2362 : step = GEN_INT (-GET_MODE_SIZE (mode));
10281 2362 : srcmem = offset_address (srcmem, step, GET_MODE_SIZE (mode));
10282 : }
10283 : else
10284 2306 : step = GEN_INT (GET_MODE_SIZE (mode));
10285 :
10286 9336 : for (i = 0; i < moves - 1; i++)
10287 : {
10288 7002 : emit_move_insn (regs[i], srcmem);
10289 14004 : srcmem = offset_address (srcmem, step, GET_MODE_SIZE (mode));
10290 : }
10291 2334 : emit_move_insn (regs[i], srcmem);
10292 2334 : }
10293 :
10294 : /* Store MOVES of mode size into REGS. If LAST is true, store the
10295 : last MOVES. Otherwise, store the first MOVES. */
10296 :
10297 : static void
10298 2433 : ix86_expand_store_movmem (rtx dst, rtx destreg, rtx count_exp,
10299 : machine_mode mode, unsigned int moves,
10300 : rtx regs[], bool last)
10301 : {
10302 2433 : unsigned int i;
10303 :
10304 2433 : rtx destmem = change_address (dst, mode, destreg);
10305 2433 : rtx step;
10306 2433 : if (last)
10307 : {
10308 1280 : destmem = offset_address (destmem, count_exp, 1);
10309 2560 : step = GEN_INT (-GET_MODE_SIZE (mode));
10310 2560 : destmem = offset_address (destmem, step, GET_MODE_SIZE (mode));
10311 : }
10312 : else
10313 2306 : step = GEN_INT (GET_MODE_SIZE (mode));
10314 :
10315 9732 : for (i = 0; i < moves - 1; i++)
10316 : {
10317 7299 : emit_move_insn (destmem, regs[i]);
10318 14598 : destmem = offset_address (destmem, step, GET_MODE_SIZE (mode));
10319 : }
10320 2433 : emit_move_insn (destmem, regs[i]);
10321 2433 : }
10322 :
10323 : /* Expand memmove of size between (MOVES / 2) * mode size and
10324 : MOVES * mode size with overlapping load and store. MOVES is even.
10325 : MOVES >= 2 and MOVES <= 8. */
10326 :
10327 : static void
10328 43652 : ix86_expand_n_overlapping_move_set_or_movmem (rtx dst, rtx src,
10329 : rtx *memset_vals,
10330 : rtx destreg, rtx srcreg,
10331 : rtx count_exp,
10332 : machine_mode mode,
10333 : unsigned int moves)
10334 : {
10335 43652 : gcc_assert (moves >= 2 && moves <= 8 && (moves & 1) == 0);
10336 :
10337 43652 : unsigned int half_moves = moves / 2;
10338 43652 : unsigned int i, j;
10339 43652 : rtx tmp[8];
10340 :
10341 43652 : if (memset_vals)
10342 : {
10343 : /* Expand memset. */
10344 4080 : rtx val = ix86_expand_memset_val (memset_vals, mode);
10345 18284 : for (i = 0; i < moves; i++)
10346 10124 : tmp[i] = val;
10347 : }
10348 : else
10349 : {
10350 : /* Expand memmove. */
10351 133376 : for (i = 0; i < moves; i++)
10352 93804 : tmp[i] = gen_reg_rtx (mode);
10353 :
10354 39572 : rtx base_srcmem = change_address (src, mode, srcreg);
10355 :
10356 : /* Load the first half. */
10357 39572 : rtx srcmem = base_srcmem;
10358 86474 : for (i = 0; i < half_moves - 1; i++)
10359 : {
10360 7330 : emit_move_insn (tmp[i], srcmem);
10361 14660 : srcmem = offset_address (srcmem,
10362 7330 : GEN_INT (GET_MODE_SIZE (mode)),
10363 7330 : GET_MODE_SIZE (mode));
10364 : }
10365 39572 : emit_move_insn (tmp[i], srcmem);
10366 :
10367 : /* Load the second half. */
10368 39572 : srcmem = offset_address (base_srcmem, count_exp, 1);
10369 39572 : srcmem = offset_address (srcmem,
10370 39572 : GEN_INT (-GET_MODE_SIZE (mode)),
10371 39572 : GET_MODE_SIZE (mode));
10372 86474 : for (j = half_moves, i = 0; i < half_moves - 1; i++, j++)
10373 : {
10374 7330 : emit_move_insn (tmp[j], srcmem);
10375 14660 : srcmem = offset_address (srcmem,
10376 7330 : GEN_INT (-GET_MODE_SIZE (mode)),
10377 7330 : GET_MODE_SIZE (mode));
10378 : }
10379 39572 : emit_move_insn (tmp[j], srcmem);
10380 : }
10381 :
10382 43652 : rtx base_destmem = change_address (dst, mode, destreg);
10383 :
10384 : /* Store the first half. */
10385 43652 : rtx destmem = base_destmem;
10386 95616 : for (i = 0; i < half_moves - 1; i++)
10387 : {
10388 8312 : emit_move_insn (destmem, tmp[i]);
10389 16624 : destmem = offset_address (destmem,
10390 8312 : GEN_INT (GET_MODE_SIZE (mode)),
10391 8312 : GET_MODE_SIZE (mode));
10392 : }
10393 43652 : emit_move_insn (destmem, tmp[i]);
10394 :
10395 : /* Store the second half. */
10396 43652 : destmem = offset_address (base_destmem, count_exp, 1);
10397 87304 : destmem = offset_address (destmem, GEN_INT (-GET_MODE_SIZE (mode)),
10398 43652 : GET_MODE_SIZE (mode));
10399 95616 : for (j = half_moves, i = 0; i < half_moves - 1; i++, j++)
10400 : {
10401 8312 : emit_move_insn (destmem, tmp[j]);
10402 16624 : destmem = offset_address (destmem, GEN_INT (-GET_MODE_SIZE (mode)),
10403 8312 : GET_MODE_SIZE (mode));
10404 : }
10405 43652 : emit_move_insn (destmem, tmp[j]);
10406 43652 : }
10407 :
10408 : /* Expand memmove of size < mode size which is <= 64. */
10409 :
10410 : static void
10411 10789 : ix86_expand_less_move_set_or_movmem (rtx dst, rtx src, rtx *memset_vals,
10412 : rtx destreg, rtx srcreg,
10413 : rtx count_exp,
10414 : unsigned HOST_WIDE_INT min_size,
10415 : machine_mode mode,
10416 : rtx_code_label *done_label)
10417 : {
10418 10789 : bool skip = false;
10419 10789 : machine_mode count_mode = counter_mode (count_exp);
10420 :
10421 10789 : rtx_code_label *between_32_63_label
10422 10789 : = GET_MODE_SIZE (mode) > 32 ? gen_label_rtx () : nullptr;
10423 : /* Jump to BETWEEN_32_64_LABEL if size >= 32 and size < 64. */
10424 7 : if (between_32_63_label)
10425 : {
10426 7 : if (min_size && min_size >= 32)
10427 : {
10428 1 : emit_jump_insn (gen_jump (between_32_63_label));
10429 1 : emit_barrier ();
10430 1 : skip = true;
10431 : }
10432 : else
10433 6 : emit_cmp_and_jump_insns (count_exp, GEN_INT (32), GEU,
10434 : nullptr, count_mode, 1,
10435 : between_32_63_label);
10436 : }
10437 :
10438 7 : rtx_code_label *between_16_31_label
10439 10788 : = (!skip && GET_MODE_SIZE (mode) > 16) ? gen_label_rtx () : nullptr;
10440 : /* Jump to BETWEEN_16_31_LABEL if size >= 16 and size < 31. */
10441 21 : if (between_16_31_label)
10442 : {
10443 21 : if (min_size && min_size >= 16)
10444 : {
10445 2 : emit_jump_insn (gen_jump (between_16_31_label));
10446 2 : emit_barrier ();
10447 2 : skip = true;
10448 : }
10449 : else
10450 19 : emit_cmp_and_jump_insns (count_exp, GEN_INT (16), GEU,
10451 : nullptr, count_mode, 1,
10452 : between_16_31_label);
10453 : }
10454 :
10455 2 : rtx_code_label *between_8_15_label
10456 21573 : = (!skip && GET_MODE_SIZE (mode) > 8) ? gen_label_rtx () : nullptr;
10457 : /* Jump to BETWEEN_8_15_LABEL if size >= 8 and size < 15. */
10458 8599 : if (between_8_15_label)
10459 : {
10460 8599 : if (min_size && min_size >= 8)
10461 : {
10462 473 : emit_jump_insn (gen_jump (between_8_15_label));
10463 473 : emit_barrier ();
10464 473 : skip = true;
10465 : }
10466 : else
10467 8126 : emit_cmp_and_jump_insns (count_exp, GEN_INT (8), GEU,
10468 : nullptr, count_mode, 1,
10469 : between_8_15_label);
10470 : }
10471 :
10472 473 : rtx_code_label *between_4_7_label
10473 20629 : = (!skip && GET_MODE_SIZE (mode) > 4) ? gen_label_rtx () : nullptr;
10474 : /* Jump to BETWEEN_4_7_LABEL if size >= 4 and size < 7. */
10475 9417 : if (between_4_7_label)
10476 : {
10477 9417 : if (min_size && min_size >= 4)
10478 : {
10479 445 : emit_jump_insn (gen_jump (between_4_7_label));
10480 445 : emit_barrier ();
10481 445 : skip = true;
10482 : }
10483 : else
10484 8972 : emit_cmp_and_jump_insns (count_exp, GEN_INT (4), GEU,
10485 : nullptr, count_mode, 1,
10486 : between_4_7_label);
10487 : }
10488 :
10489 445 : rtx_code_label *between_2_3_label
10490 20212 : = (!skip && GET_MODE_SIZE (mode) > 2) ? gen_label_rtx () : nullptr;
10491 : /* Jump to BETWEEN_2_3_LABEL if size >= 2 and size < 3. */
10492 9650 : if (between_2_3_label)
10493 : {
10494 9650 : if (min_size && min_size >= 2)
10495 : {
10496 1923 : emit_jump_insn (gen_jump (between_2_3_label));
10497 1923 : emit_barrier ();
10498 1923 : skip = true;
10499 : }
10500 : else
10501 7727 : emit_cmp_and_jump_insns (count_exp, GEN_INT (1), GT,
10502 : nullptr, count_mode, 1,
10503 : between_2_3_label);
10504 : }
10505 :
10506 10789 : if (!skip)
10507 : {
10508 7945 : rtx_code_label *zero_label
10509 7945 : = min_size == 0 ? gen_label_rtx () : nullptr;
10510 : /* Skip if size == 0. */
10511 2711 : if (zero_label)
10512 2711 : emit_cmp_and_jump_insns (count_exp, GEN_INT (1), LT,
10513 : nullptr, count_mode, 1,
10514 : zero_label,
10515 : profile_probability::unlikely ());
10516 :
10517 : /* Move 1 byte. */
10518 7945 : rtx tmp0;
10519 : /* Use the value rtx in QImode for memset. */
10520 7945 : if (memset_vals)
10521 904 : tmp0 = memset_vals[memset_val_byte];
10522 : else
10523 : {
10524 7041 : tmp0 = gen_reg_rtx (QImode);
10525 7041 : rtx srcmem = change_address (src, QImode, srcreg);
10526 7041 : emit_move_insn (tmp0, srcmem);
10527 : }
10528 7945 : rtx destmem = change_address (dst, QImode, destreg);
10529 7945 : emit_move_insn (destmem, tmp0);
10530 :
10531 7945 : if (zero_label)
10532 2711 : emit_label (zero_label);
10533 :
10534 7945 : emit_jump_insn (gen_jump (done_label));
10535 7945 : emit_barrier ();
10536 : }
10537 :
10538 : /* For each size band, memset uses a QI-vector mode above a word so it
10539 : can broadcast the fill value, while memmove uses the same-size
10540 : scalar integer mode; at and below a word both use the scalar
10541 : integer mode. */
10542 10789 : struct {
10543 : rtx_code_label *label;
10544 : machine_mode set_mode;
10545 : machine_mode move_mode;
10546 10789 : } bands[] = {
10547 : { between_32_63_label, V32QImode, OImode },
10548 : { between_16_31_label, V16QImode, TImode },
10549 : { between_8_15_label, DImode, DImode },
10550 : { between_4_7_label, SImode, SImode },
10551 : { between_2_3_label, HImode, HImode },
10552 10789 : };
10553 :
10554 64734 : for (auto &band : bands)
10555 53945 : if (band.label)
10556 : {
10557 27694 : emit_label (band.label);
10558 27694 : machine_mode bmode = memset_vals ? band.set_mode : band.move_mode;
10559 27694 : ix86_expand_n_overlapping_move_set_or_movmem (dst, src,
10560 : memset_vals,
10561 : destreg, srcreg,
10562 : count_exp, bmode,
10563 : 2);
10564 27694 : emit_jump_insn (gen_jump (done_label));
10565 27694 : emit_barrier ();
10566 : }
10567 10789 : }
10568 :
10569 : /* Expand movmem with overlapping unaligned loads and stores:
10570 : 1. Load all sources into registers and store them together to avoid
10571 : possible address overlap between source and destination.
10572 : 2. For known size, first try to fully unroll with 8 registers.
10573 : 3. For size <= 2 * MOVE_MAX, load all sources into 2 registers first
10574 : and then store them together.
10575 : 4. For size > 2 * MOVE_MAX and size <= 4 * MOVE_MAX, load all sources
10576 : into 4 registers first and then store them together.
10577 : 5. For size > 4 * MOVE_MAX and size <= 8 * MOVE_MAX, load all sources
10578 : into 8 registers first and then store them together.
10579 : 6. For size > 8 * MOVE_MAX,
10580 : a. If address of destination > address of source, copy backward
10581 : with a 4 * MOVE_MAX loop with unaligned loads and stores. Load
10582 : the first 4 * MOVE_MAX into 4 registers before the loop and
10583 : store them after the loop to support overlapping addresses.
10584 : b. Otherwise, copy forward with a 4 * MOVE_MAX loop with unaligned
10585 : loads and stores. Load the last 4 * MOVE_MAX into 4 registers
10586 : before the loop and store them after the loop to support
10587 : overlapping addresses.
10588 : */
10589 :
10590 : bool
10591 171433 : ix86_expand_set_or_movmem (rtx operands[], bool iscpymem, bool issetmem)
10592 : {
10593 : /* Since there are much less registers available in 32-bit mode, don't
10594 : inline movmem in 32-bit mode. */
10595 171433 : if (!TARGET_64BIT || optimize_insn_for_size_p ())
10596 : return false;
10597 :
10598 137583 : rtx dst = operands[0];
10599 137583 : rtx src, memset_val_exp;
10600 137583 : if (issetmem)
10601 : {
10602 35514 : src = nullptr;
10603 35514 : memset_val_exp = operands[1];
10604 : }
10605 : else
10606 : {
10607 102069 : src = operands[1];
10608 102069 : memset_val_exp = nullptr;
10609 : }
10610 137583 : rtx count_exp = operands[2];
10611 137583 : rtx expected_size_exp = operands[5];
10612 137583 : rtx min_size_exp = operands[6];
10613 137583 : rtx max_size_exp = operands[7];
10614 137583 : rtx probable_max_size_exp = operands[8];
10615 137583 : unsigned HOST_WIDE_INT count = HOST_WIDE_INT_0U;
10616 137583 : HOST_WIDE_INT expected_size = HOST_WIDE_INT_M1U;
10617 137583 : unsigned HOST_WIDE_INT min_size = HOST_WIDE_INT_0U;
10618 137583 : unsigned HOST_WIDE_INT max_size = HOST_WIDE_INT_M1U;
10619 137583 : unsigned HOST_WIDE_INT probable_max_size = HOST_WIDE_INT_M1U;
10620 :
10621 137583 : if (CONST_INT_P (count_exp))
10622 : {
10623 54357 : min_size = max_size = probable_max_size = count = expected_size
10624 54357 : = INTVAL (count_exp);
10625 : /* When COUNT is 0, there is nothing to do. */
10626 54357 : if (!count)
10627 : return true;
10628 : }
10629 : else
10630 : {
10631 83226 : if (min_size_exp)
10632 83226 : min_size = INTVAL (min_size_exp);
10633 83226 : if (max_size_exp)
10634 63586 : max_size = INTVAL (max_size_exp);
10635 83226 : if (probable_max_size_exp)
10636 65531 : probable_max_size = INTVAL (probable_max_size_exp);
10637 83226 : if (CONST_INT_P (expected_size_exp))
10638 83226 : expected_size = INTVAL (expected_size_exp);
10639 :
10640 : /* NB: This assert may fail without the fixes for
10641 : https://gcc.gnu.org/bugzilla/show_bug.cgi?id=125977
10642 : */
10643 83226 : gcc_assert (min_size != max_size);
10644 : }
10645 :
10646 : /* Make sure we don't need to care about overflow later on. */
10647 54355 : if (count > (HOST_WIDE_INT_1U << 30))
10648 : return false;
10649 :
10650 137392 : addr_space_t dst_as = MEM_ADDR_SPACE (dst);
10651 137392 : addr_space_t src_as = (issetmem
10652 : ? ADDR_SPACE_GENERIC
10653 137392 : : MEM_ADDR_SPACE (src));
10654 101941 : int dynamic_check;
10655 101941 : bool noalign;
10656 137392 : enum stringop_alg alg = decide_alg (count, expected_size, min_size,
10657 : probable_max_size, issetmem,
10658 : (issetmem
10659 35451 : && memset_val_exp == const0_rtx),
10660 : dst_as, src_as, &dynamic_check,
10661 : &noalign, false);
10662 137392 : if (alg == libcall)
10663 : return false;
10664 :
10665 : /* Expand memcpy and memset like memmove only for bounded size. */
10666 18264 : if (iscpymem || issetmem)
10667 : {
10668 14024 : unsigned HOST_WIDE_INT unbounded
10669 14024 : = GET_MODE_MASK (counter_mode (count_exp));
10670 14024 : if (count != 0 /* Fixed size. */
10671 14024 : || max_size == 0 /* Unbounded size. */
10672 8994 : || max_size == unbounded) /* Unbounded size. */
10673 : return false;
10674 : }
10675 :
10676 12906 : rtx destreg = ix86_copy_addr_to_reg (XEXP (dst, 0));
10677 12906 : rtx srcreg = (issetmem
10678 12906 : ? nullptr
10679 11775 : : ix86_copy_addr_to_reg (XEXP (src, 0)));
10680 :
10681 12906 : unsigned int move_max = MOVE_MAX;
10682 12906 : machine_mode mode = smallest_int_mode_for_size
10683 12906 : (move_max * BITS_PER_UNIT).require ();
10684 12906 : if (probable_max_size && probable_max_size < move_max)
10685 : {
10686 : /* Get a usable MOVE_MAX. */
10687 3802 : mode = smallest_int_mode_for_size
10688 3802 : (probable_max_size * BITS_PER_UNIT).require ();
10689 : /* Reduce MOVE_MAX by half so that MOVE_MAX can be used. */
10690 7604 : if (GET_MODE_SIZE (mode) > probable_max_size)
10691 3274 : mode = smallest_int_mode_for_size
10692 3274 : (GET_MODE_BITSIZE (mode) / 2).require ();
10693 7604 : move_max = GET_MODE_SIZE (mode);
10694 : }
10695 :
10696 : /* Try to fully unroll memmove of known size first. */
10697 12906 : if (count
10698 12906 : && ix86_expand_unroll_movmem (dst, src, destreg, srcreg, count,
10699 : mode))
10700 : return true;
10701 :
10702 11031 : rtx memset_vals[memset_val_max];
10703 11031 : rtx *memset_vals_p;
10704 11031 : if (issetmem)
10705 : {
10706 : /* Use vector mode if MODE size > word size. */
10707 1131 : unsigned int size = GET_MODE_SIZE (mode);
10708 1131 : poly_uint64 nunits;
10709 1131 : if (size > UNITS_PER_WORD)
10710 : {
10711 628 : nunits = size / GET_MODE_SIZE (QImode);
10712 628 : mode = mode_for_vector (QImode, nunits).require ();
10713 : }
10714 :
10715 : /* Populate MEMSET_VALS to expand memset. */
10716 1131 : rtx val_word;
10717 1131 : memset_vals[memset_val_byte] = memset_val_exp;
10718 1131 : if (memset_val_exp == const0_rtx || memset_val_exp == constm1_rtx)
10719 : val_word = memset_val_exp;
10720 : else
10721 657 : val_word = promote_duplicated_reg (word_mode, memset_val_exp);
10722 1131 : memset_vals[memset_val_word] = val_word;
10723 2262 : if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
10724 : {
10725 628 : if (memset_val_exp == const0_rtx)
10726 242 : memset_vals[memset_val_vector] = CONST0_RTX (mode);
10727 386 : else if (memset_val_exp == constm1_rtx)
10728 18 : memset_vals[memset_val_vector] = CONSTM1_RTX (mode);
10729 : else
10730 : {
10731 : /* Use the vector mode based on WORD_MODE to avoid extra
10732 : GPR moves. */
10733 736 : nunits = size / GET_MODE_SIZE (word_mode);
10734 368 : machine_mode vector_mode
10735 368 : = mode_for_vector (word_mode, nunits).require ();
10736 368 : rtx vector = promote_duplicated_reg (vector_mode,
10737 : val_word);
10738 368 : memset_vals[memset_val_vector]
10739 368 : = convert_to_mode (mode, vector, 1);
10740 : }
10741 : }
10742 : else
10743 503 : memset_vals[memset_val_vector] = nullptr;
10744 1131 : memset_vals_p = memset_vals;
10745 : }
10746 : else
10747 : memset_vals_p = nullptr;
10748 :
10749 11031 : rtx_code_label *done_label = gen_label_rtx ();
10750 :
10751 11031 : rtx_code_label *less_vec_label = nullptr;
10752 11031 : if (min_size == 0 || min_size < move_max)
10753 10789 : less_vec_label = gen_label_rtx ();
10754 :
10755 11031 : machine_mode count_mode = counter_mode (count_exp);
10756 :
10757 : /* Jump to LESS_VEC_LABEL if size < MOVE_MAX. */
10758 11031 : if (less_vec_label)
10759 10789 : emit_cmp_and_jump_insns (count_exp, GEN_INT (move_max), LTU,
10760 : nullptr, count_mode, 1,
10761 : less_vec_label);
10762 :
10763 11031 : rtx_code_label *more_2x_vec_label = nullptr;
10764 11031 : if (probable_max_size == 0 || probable_max_size > 2 * move_max)
10765 3311 : more_2x_vec_label = gen_label_rtx ();
10766 :
10767 : /* Jump to MORE_2X_VEC_LABEL if size > 2 * MOVE_MAX. */
10768 3311 : if (more_2x_vec_label)
10769 3311 : emit_cmp_and_jump_insns (count_exp, GEN_INT (2 * move_max), GTU,
10770 : nullptr, count_mode, 1,
10771 : more_2x_vec_label);
10772 :
10773 11031 : if (max_size != 1 && (min_size == 0 || min_size <= 2 * move_max))
10774 : {
10775 : /* Max size != 1, size >= MOVE_MAX and size <= 2 * MOVE_MAX. */
10776 10988 : ix86_expand_n_overlapping_move_set_or_movmem (dst, src,
10777 : memset_vals_p,
10778 : destreg, srcreg,
10779 : count_exp, mode, 2);
10780 10988 : emit_jump_insn (gen_jump (done_label));
10781 10988 : emit_barrier ();
10782 : }
10783 :
10784 11031 : if (less_vec_label)
10785 : {
10786 : /* Size < MOVE_MAX. */
10787 10789 : emit_label (less_vec_label);
10788 10789 : ix86_expand_less_move_set_or_movmem (dst, src, memset_vals_p,
10789 : destreg, srcreg, count_exp,
10790 : min_size, mode, done_label);
10791 10789 : emit_jump_insn (gen_jump (done_label));
10792 10789 : emit_barrier ();
10793 : }
10794 :
10795 11031 : if (more_2x_vec_label)
10796 : {
10797 : /* Size > 2 * MOVE_MAX and destination may overlap with source. */
10798 3311 : emit_label (more_2x_vec_label);
10799 :
10800 3311 : rtx_code_label *more_8x_vec_label = nullptr;
10801 3311 : if (probable_max_size == 0 || probable_max_size > 8 * move_max)
10802 1280 : more_8x_vec_label = gen_label_rtx ();
10803 :
10804 : /* Jump to MORE_8X_VEC_LABEL if size > 8 * MOVE_MAX. */
10805 1280 : if (more_8x_vec_label)
10806 1280 : emit_cmp_and_jump_insns (count_exp, GEN_INT (8 * move_max), GTU,
10807 : nullptr, count_mode, 1,
10808 : more_8x_vec_label);
10809 :
10810 3311 : rtx_code_label *last_4x_vec_label = nullptr;
10811 3311 : if (min_size == 0 || min_size <= 4 * move_max)
10812 3299 : last_4x_vec_label = gen_label_rtx ();
10813 :
10814 : /* Jump to LAST_4X_VEC_LABEL if size <= 4 * MOVE_MAX. */
10815 3299 : if (last_4x_vec_label)
10816 3299 : emit_cmp_and_jump_insns (count_exp, GEN_INT (4 * move_max), LEU,
10817 : nullptr, count_mode, 1,
10818 : last_4x_vec_label);
10819 :
10820 3311 : if (probable_max_size == 0 || probable_max_size > 4 * move_max)
10821 : {
10822 : /* Size > 4 * MOVE_MAX and size <= 8 * MOVE_MAX. */
10823 1671 : ix86_expand_n_overlapping_move_set_or_movmem (dst, src,
10824 : memset_vals_p,
10825 : destreg, srcreg,
10826 : count_exp, mode,
10827 : 8);
10828 1671 : emit_jump_insn (gen_jump (done_label));
10829 1671 : emit_barrier ();
10830 : }
10831 :
10832 3311 : if (last_4x_vec_label)
10833 : {
10834 : /* Size > 2 * MOVE_MAX and size <= 4 * MOVE_MAX. */
10835 3299 : emit_label (last_4x_vec_label);
10836 3299 : ix86_expand_n_overlapping_move_set_or_movmem (dst, src,
10837 : memset_vals_p,
10838 : destreg, srcreg,
10839 : count_exp, mode,
10840 : 4);
10841 3299 : emit_jump_insn (gen_jump (done_label));
10842 3299 : emit_barrier ();
10843 : }
10844 :
10845 3311 : if (more_8x_vec_label)
10846 : {
10847 : /* Size > 8 * MOVE_MAX. */
10848 1280 : emit_label (more_8x_vec_label);
10849 :
10850 1280 : rtx loop_count = gen_reg_rtx (count_mode);
10851 1280 : emit_move_insn (loop_count, count_exp);
10852 :
10853 1280 : rtx_code_label *more_8x_vec_backward_label;
10854 1280 : rtx base_destreg;
10855 1280 : rtx srcmem;
10856 1280 : rtx regs[4];
10857 1280 : if (iscpymem || issetmem)
10858 : {
10859 : /* Always store forward for memcpy and memset. */
10860 127 : more_8x_vec_backward_label = nullptr;
10861 127 : if (iscpymem)
10862 : {
10863 : /* Load the last 4 * MOVE_MAX for memcpy. */
10864 28 : ix86_expand_load_movmem (src, srcreg, count_exp, mode,
10865 : ARRAY_SIZE (regs), regs,
10866 : true);
10867 28 : srcmem = change_address (src, mode, srcreg);
10868 : }
10869 : else
10870 : {
10871 : /* Fill REGS with MEMSET_VALS for memset. */
10872 99 : rtx val = ix86_expand_memset_val (memset_vals, mode);
10873 594 : for (unsigned int i = 0; i < 4; i++)
10874 396 : regs[i] = val;
10875 : srcmem = nullptr;
10876 : }
10877 127 : base_destreg = gen_reg_rtx (GET_MODE (destreg));
10878 127 : emit_move_insn (base_destreg, destreg);
10879 : }
10880 : else
10881 : {
10882 : /* Jump to MORE_8X_VEC_BACKWARD_LABEL if source address is
10883 : lower than destination address. */
10884 1153 : more_8x_vec_backward_label = gen_label_rtx ();
10885 1153 : emit_cmp_and_jump_insns (srcreg, destreg, LTU, nullptr,
10886 1153 : GET_MODE (destreg), 1,
10887 : more_8x_vec_backward_label);
10888 :
10889 : /* Skip if source == destination which is less common. */
10890 1153 : emit_cmp_and_jump_insns (srcreg, destreg, EQ, nullptr,
10891 1153 : GET_MODE (destreg), 1, done_label,
10892 : profile_probability::unlikely ());
10893 :
10894 1153 : base_destreg = gen_reg_rtx (GET_MODE (destreg));
10895 1153 : emit_move_insn (base_destreg, destreg);
10896 :
10897 : /* Load the last 4 * MOVE_MAX. */
10898 1153 : ix86_expand_load_movmem (src, srcreg, count_exp, mode,
10899 : ARRAY_SIZE (regs), regs, true);
10900 :
10901 1153 : srcmem = change_address (src, mode, srcreg);
10902 : }
10903 :
10904 1280 : rtx destmem = change_address (dst, mode, destreg);
10905 :
10906 : /* Copy forward with a 4 * MOVE_MAX loop. */
10907 1280 : rtx_code_label *loop_4x_vec_forward_label = gen_label_rtx ();
10908 1280 : emit_label (loop_4x_vec_forward_label);
10909 :
10910 1280 : ix86_expand_n_move_set_or_movmem (destmem, srcmem,
10911 : memset_vals_p, mode, 4,
10912 : true);
10913 :
10914 1280 : rtx tmp;
10915 1280 : rtx delta = GEN_INT (4 * MOVE_MAX);
10916 :
10917 : /* Decrement LOOP_COUNT by 4 * MOVE_MAX. */
10918 1280 : tmp = expand_simple_binop (GET_MODE (loop_count), MINUS,
10919 : loop_count, delta, nullptr, 1,
10920 : OPTAB_DIRECT);
10921 1280 : if (tmp != loop_count)
10922 1280 : emit_move_insn (loop_count, tmp);
10923 :
10924 : /* Increment DESTREG and SRCREG by 4 * MOVE_MAX. */
10925 1280 : tmp = expand_simple_binop (GET_MODE (destreg), PLUS,
10926 : destreg, delta, nullptr, 1,
10927 : OPTAB_DIRECT);
10928 1280 : if (tmp != destreg)
10929 1280 : emit_move_insn (destreg, tmp);
10930 1280 : if (!issetmem)
10931 : {
10932 1181 : tmp = expand_simple_binop (GET_MODE (srcreg), PLUS,
10933 : srcreg, delta, nullptr, 1,
10934 : OPTAB_DIRECT);
10935 1181 : if (tmp != srcreg)
10936 1181 : emit_move_insn (srcreg, tmp);
10937 : }
10938 :
10939 : /* Stop if LOOP_EXP <= 4 * MOVE_MAX. */
10940 1280 : emit_cmp_and_jump_insns (loop_count, delta, GTU, nullptr,
10941 1280 : GET_MODE (loop_count), 1,
10942 : loop_4x_vec_forward_label);
10943 :
10944 : /* Store the last 4 * MOVE_MAX. */
10945 1280 : ix86_expand_store_movmem (dst, base_destreg, count_exp, mode,
10946 : ARRAY_SIZE (regs), regs, true);
10947 :
10948 1280 : emit_jump_insn (gen_jump (done_label));
10949 1280 : emit_barrier ();
10950 :
10951 1280 : if (more_8x_vec_backward_label)
10952 : {
10953 : /* Copy backward with a 4 * MOVE_MAX loop. */
10954 1153 : emit_label (more_8x_vec_backward_label);
10955 :
10956 1153 : base_destreg = gen_reg_rtx (GET_MODE (destreg));
10957 1153 : emit_move_insn (base_destreg, destreg);
10958 :
10959 : /* Load the first 4 * MOVE_MAX. */
10960 1153 : ix86_expand_load_movmem (src, srcreg, count_exp, mode,
10961 : ARRAY_SIZE (regs), regs, false);
10962 :
10963 : /* Increment DESTREG and SRCREG by COUNT_EXP. */
10964 1153 : tmp = expand_simple_binop (GET_MODE (destreg), PLUS,
10965 : destreg, count_exp, nullptr, 1,
10966 : OPTAB_DIRECT);
10967 1153 : if (tmp != destreg)
10968 1153 : emit_move_insn (destreg, tmp);
10969 1153 : tmp = expand_simple_binop (GET_MODE (srcreg), PLUS, srcreg,
10970 : count_exp, nullptr, 1,
10971 : OPTAB_DIRECT);
10972 1153 : if (tmp != srcreg)
10973 1153 : emit_move_insn (srcreg, tmp);
10974 :
10975 1153 : srcmem = change_address (src, mode, srcreg);
10976 1153 : destmem = change_address (dst, mode, destreg);
10977 2306 : rtx step = GEN_INT (-GET_MODE_SIZE (mode));
10978 1153 : srcmem = offset_address (srcmem, step,
10979 1153 : GET_MODE_SIZE (mode));
10980 1153 : destmem = offset_address (destmem, step,
10981 1153 : GET_MODE_SIZE (mode));
10982 :
10983 1153 : rtx_code_label *loop_4x_vec_backward_label
10984 1153 : = gen_label_rtx ();
10985 1153 : emit_label (loop_4x_vec_backward_label);
10986 :
10987 1153 : ix86_expand_n_move_set_or_movmem (destmem, srcmem,
10988 : memset_vals_p, mode, 4,
10989 : false);
10990 :
10991 : /* Decrement LOOP_COUNT by 4 * MOVE_MAX. */
10992 1153 : tmp = expand_simple_binop (GET_MODE (loop_count), MINUS,
10993 : loop_count, delta, nullptr, 1,
10994 : OPTAB_DIRECT);
10995 1153 : if (tmp != loop_count)
10996 1153 : emit_move_insn (loop_count, tmp);
10997 :
10998 : /* Decrement DESTREG and SRCREG by 4 * MOVE_MAX. */
10999 1153 : tmp = expand_simple_binop (GET_MODE (destreg), MINUS,
11000 : destreg, delta, nullptr, 1,
11001 : OPTAB_DIRECT);
11002 1153 : if (tmp != destreg)
11003 1153 : emit_move_insn (destreg, tmp);
11004 1153 : tmp = expand_simple_binop (GET_MODE (srcreg), MINUS,
11005 : srcreg, delta, nullptr, 1,
11006 : OPTAB_DIRECT);
11007 1153 : if (tmp != srcreg)
11008 1153 : emit_move_insn (srcreg, tmp);
11009 :
11010 : /* Stop if LOOP_EXP <= 4 * MOVE_MAX. */
11011 1153 : emit_cmp_and_jump_insns (loop_count, delta, GTU, nullptr,
11012 1153 : GET_MODE (loop_count), 1,
11013 : loop_4x_vec_backward_label);
11014 :
11015 : /* Store the first 4 * MOVE_MAX. */
11016 1153 : ix86_expand_store_movmem (dst, base_destreg, count_exp,
11017 : mode, ARRAY_SIZE (regs), regs,
11018 : false);
11019 :
11020 1153 : emit_jump_insn (gen_jump (done_label));
11021 1153 : emit_barrier ();
11022 : }
11023 : }
11024 : }
11025 :
11026 11031 : emit_label (done_label);
11027 :
11028 11031 : return true;
11029 : }
11030 :
11031 : /* Expand cmpstrn or memcmp. */
11032 :
11033 : bool
11034 171564 : ix86_expand_cmpstrn_or_cmpmem (rtx result, rtx src1, rtx src2,
11035 : rtx length, rtx align, bool is_cmpstrn)
11036 : {
11037 : /* Expand strncmp and memcmp only with -minline-all-stringops since
11038 : "repz cmpsb" can be much slower than strncmp and memcmp functions
11039 : implemented with vector instructions, see
11040 :
11041 : https://gcc.gnu.org/bugzilla/show_bug.cgi?id=43052
11042 : */
11043 171564 : if (!TARGET_INLINE_ALL_STRINGOPS)
11044 : return false;
11045 :
11046 : /* Can't use this if the user has appropriated ecx, esi or edi. */
11047 5796 : if (fixed_regs[CX_REG] || fixed_regs[SI_REG] || fixed_regs[DI_REG])
11048 : return false;
11049 :
11050 5796 : if (is_cmpstrn)
11051 : {
11052 : /* For strncmp, length is the maximum length, which can be larger
11053 : than actual string lengths. We can expand the cmpstrn pattern
11054 : to "repz cmpsb" only if one of the strings is a constant so
11055 : that expand_builtin_strncmp() can write the length argument to
11056 : be the minimum of the const string length and the actual length
11057 : argument. Otherwise, "repz cmpsb" may pass the 0 byte. */
11058 69 : tree t1 = MEM_EXPR (src1);
11059 69 : tree t2 = MEM_EXPR (src2);
11060 138 : if (!((t1 && TREE_CODE (t1) == MEM_REF
11061 69 : && TREE_CODE (TREE_OPERAND (t1, 0)) == ADDR_EXPR
11062 0 : && (TREE_CODE (TREE_OPERAND (TREE_OPERAND (t1, 0), 0))
11063 : == STRING_CST))
11064 69 : || (t2 && TREE_CODE (t2) == MEM_REF
11065 69 : && TREE_CODE (TREE_OPERAND (t2, 0)) == ADDR_EXPR
11066 69 : && (TREE_CODE (TREE_OPERAND (TREE_OPERAND (t2, 0), 0))
11067 : == STRING_CST))))
11068 : return false;
11069 : }
11070 :
11071 5796 : rtx addr1 = copy_addr_to_reg (XEXP (src1, 0));
11072 5796 : rtx addr2 = copy_addr_to_reg (XEXP (src2, 0));
11073 5796 : if (addr1 != XEXP (src1, 0))
11074 5796 : src1 = replace_equiv_address_nv (src1, addr1);
11075 5796 : if (addr2 != XEXP (src2, 0))
11076 5796 : src2 = replace_equiv_address_nv (src2, addr2);
11077 :
11078 : /* NB: Make a copy of the data length to avoid changing the original
11079 : data length by cmpstrnqi patterns. */
11080 5796 : length = ix86_zero_extend_to_Pmode (length);
11081 8711 : rtx lengthreg = gen_reg_rtx (Pmode);
11082 5796 : emit_move_insn (lengthreg, length);
11083 :
11084 : /* If we are testing strict equality, we can use known alignment to
11085 : good advantage. This may be possible with combine, particularly
11086 : once cc0 is dead. */
11087 5796 : if (CONST_INT_P (length))
11088 : {
11089 0 : if (length == const0_rtx)
11090 : {
11091 0 : emit_move_insn (result, const0_rtx);
11092 0 : return true;
11093 : }
11094 0 : emit_insn (gen_cmpstrnqi_nz_1 (addr1, addr2, lengthreg, align,
11095 : src1, src2));
11096 : }
11097 : else
11098 : {
11099 8711 : emit_insn (gen_cmp_1 (Pmode, lengthreg, lengthreg));
11100 5796 : emit_insn (gen_cmpstrnqi_1 (addr1, addr2, lengthreg, align,
11101 : src1, src2));
11102 : }
11103 :
11104 5796 : rtx out = gen_lowpart (QImode, result);
11105 5796 : emit_insn (gen_cmpintqi (out));
11106 5796 : emit_move_insn (result, gen_rtx_SIGN_EXTEND (SImode, out));
11107 :
11108 5796 : return true;
11109 : }
11110 :
11111 : /* Expand the appropriate insns for doing strlen if not just doing
11112 : repnz; scasb
11113 :
11114 : out = result, initialized with the start address
11115 : align_rtx = alignment of the address.
11116 : scratch = scratch register, initialized with the startaddress when
11117 : not aligned, otherwise undefined
11118 :
11119 : This is just the body. It needs the initializations mentioned above and
11120 : some address computing at the end. These things are done in i386.md. */
11121 :
11122 : static void
11123 11 : ix86_expand_strlensi_unroll_1 (rtx out, rtx src, rtx align_rtx)
11124 : {
11125 11 : int align;
11126 11 : rtx tmp;
11127 11 : rtx_code_label *align_2_label = NULL;
11128 11 : rtx_code_label *align_3_label = NULL;
11129 11 : rtx_code_label *align_4_label = gen_label_rtx ();
11130 11 : rtx_code_label *end_0_label = gen_label_rtx ();
11131 11 : rtx mem;
11132 11 : rtx tmpreg = gen_reg_rtx (SImode);
11133 11 : rtx scratch = gen_reg_rtx (SImode);
11134 11 : rtx cmp;
11135 :
11136 11 : align = 0;
11137 11 : if (CONST_INT_P (align_rtx))
11138 11 : align = INTVAL (align_rtx);
11139 :
11140 : /* Loop to check 1..3 bytes for null to get an aligned pointer. */
11141 :
11142 : /* Is there a known alignment and is it less than 4? */
11143 11 : if (align < 4)
11144 : {
11145 15 : rtx scratch1 = gen_reg_rtx (Pmode);
11146 11 : emit_move_insn (scratch1, out);
11147 : /* Is there a known alignment and is it not 2? */
11148 11 : if (align != 2)
11149 : {
11150 11 : align_3_label = gen_label_rtx (); /* Label when aligned to 3-byte */
11151 11 : align_2_label = gen_label_rtx (); /* Label when aligned to 2-byte */
11152 :
11153 : /* Leave just the 3 lower bits. */
11154 15 : align_rtx = expand_binop (Pmode, and_optab, scratch1, GEN_INT (3),
11155 : NULL_RTX, 0, OPTAB_WIDEN);
11156 :
11157 15 : emit_cmp_and_jump_insns (align_rtx, const0_rtx, EQ, NULL,
11158 11 : Pmode, 1, align_4_label);
11159 15 : emit_cmp_and_jump_insns (align_rtx, const2_rtx, EQ, NULL,
11160 11 : Pmode, 1, align_2_label);
11161 15 : emit_cmp_and_jump_insns (align_rtx, const2_rtx, GTU, NULL,
11162 11 : Pmode, 1, align_3_label);
11163 : }
11164 : else
11165 : {
11166 : /* Since the alignment is 2, we have to check 2 or 0 bytes;
11167 : check if is aligned to 4 - byte. */
11168 :
11169 0 : align_rtx = expand_binop (Pmode, and_optab, scratch1, const2_rtx,
11170 : NULL_RTX, 0, OPTAB_WIDEN);
11171 :
11172 0 : emit_cmp_and_jump_insns (align_rtx, const0_rtx, EQ, NULL,
11173 0 : Pmode, 1, align_4_label);
11174 : }
11175 :
11176 11 : mem = change_address (src, QImode, out);
11177 :
11178 : /* Now compare the bytes. */
11179 :
11180 : /* Compare the first n unaligned byte on a byte per byte basis. */
11181 11 : emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL,
11182 : QImode, 1, end_0_label);
11183 :
11184 : /* Increment the address. */
11185 11 : emit_insn (gen_add2_insn (out, const1_rtx));
11186 :
11187 : /* Not needed with an alignment of 2 */
11188 11 : if (align != 2)
11189 : {
11190 11 : emit_label (align_2_label);
11191 :
11192 11 : emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1,
11193 : end_0_label);
11194 :
11195 11 : emit_insn (gen_add2_insn (out, const1_rtx));
11196 :
11197 11 : emit_label (align_3_label);
11198 : }
11199 :
11200 11 : emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1,
11201 : end_0_label);
11202 :
11203 11 : emit_insn (gen_add2_insn (out, const1_rtx));
11204 : }
11205 :
11206 : /* Generate loop to check 4 bytes at a time. It is not a good idea to
11207 : align this loop. It gives only huge programs, but does not help to
11208 : speed up. */
11209 11 : emit_label (align_4_label);
11210 :
11211 11 : mem = change_address (src, SImode, out);
11212 11 : emit_move_insn (scratch, mem);
11213 11 : emit_insn (gen_add2_insn (out, GEN_INT (4)));
11214 :
11215 : /* This formula yields a nonzero result iff one of the bytes is zero.
11216 : This saves three branches inside loop and many cycles. */
11217 :
11218 11 : emit_insn (gen_addsi3 (tmpreg, scratch, GEN_INT (-0x01010101)));
11219 11 : emit_insn (gen_one_cmplsi2 (scratch, scratch));
11220 11 : emit_insn (gen_andsi3 (tmpreg, tmpreg, scratch));
11221 11 : emit_insn (gen_andsi3 (tmpreg, tmpreg,
11222 : gen_int_mode (0x80808080, SImode)));
11223 11 : emit_cmp_and_jump_insns (tmpreg, const0_rtx, EQ, 0, SImode, 1,
11224 : align_4_label);
11225 :
11226 11 : if (TARGET_CMOVE)
11227 : {
11228 11 : rtx reg = gen_reg_rtx (SImode);
11229 15 : rtx reg2 = gen_reg_rtx (Pmode);
11230 11 : emit_move_insn (reg, tmpreg);
11231 11 : emit_insn (gen_lshrsi3 (reg, reg, GEN_INT (16)));
11232 :
11233 : /* If zero is not in the first two bytes, move two bytes forward. */
11234 11 : emit_insn (gen_testsi_ccno_1 (tmpreg, GEN_INT (0x8080)));
11235 11 : tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
11236 11 : tmp = gen_rtx_EQ (VOIDmode, tmp, const0_rtx);
11237 11 : emit_insn (gen_rtx_SET (tmpreg,
11238 : gen_rtx_IF_THEN_ELSE (SImode, tmp,
11239 : reg,
11240 : tmpreg)));
11241 : /* Emit lea manually to avoid clobbering of flags. */
11242 15 : emit_insn (gen_rtx_SET (reg2, plus_constant (Pmode, out, 2)));
11243 :
11244 11 : tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
11245 11 : tmp = gen_rtx_EQ (VOIDmode, tmp, const0_rtx);
11246 15 : emit_insn (gen_rtx_SET (out,
11247 : gen_rtx_IF_THEN_ELSE (Pmode, tmp,
11248 : reg2,
11249 : out)));
11250 11 : }
11251 : else
11252 : {
11253 0 : rtx_code_label *end_2_label = gen_label_rtx ();
11254 : /* Is zero in the first two bytes? */
11255 :
11256 0 : emit_insn (gen_testsi_ccno_1 (tmpreg, GEN_INT (0x8080)));
11257 0 : tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
11258 0 : tmp = gen_rtx_NE (VOIDmode, tmp, const0_rtx);
11259 0 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
11260 : gen_rtx_LABEL_REF (VOIDmode, end_2_label),
11261 : pc_rtx);
11262 0 : tmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
11263 0 : JUMP_LABEL (tmp) = end_2_label;
11264 :
11265 : /* Not in the first two. Move two bytes forward. */
11266 0 : emit_insn (gen_lshrsi3 (tmpreg, tmpreg, GEN_INT (16)));
11267 0 : emit_insn (gen_add2_insn (out, const2_rtx));
11268 :
11269 0 : emit_label (end_2_label);
11270 :
11271 : }
11272 :
11273 : /* Avoid branch in fixing the byte. */
11274 11 : tmpreg = gen_lowpart (QImode, tmpreg);
11275 11 : emit_insn (gen_addqi3_cconly_overflow (tmpreg, tmpreg));
11276 11 : tmp = gen_rtx_REG (CCmode, FLAGS_REG);
11277 11 : cmp = gen_rtx_LTU (VOIDmode, tmp, const0_rtx);
11278 15 : emit_insn (gen_sub3_carry (Pmode, out, out, GEN_INT (3), tmp, cmp));
11279 :
11280 11 : emit_label (end_0_label);
11281 11 : }
11282 :
11283 : /* Expand strlen. */
11284 :
11285 : bool
11286 14488 : ix86_expand_strlen (rtx out, rtx src, rtx eoschar, rtx align)
11287 : {
11288 14488 : if (TARGET_UNROLL_STRLEN
11289 14488 : && TARGET_INLINE_ALL_STRINGOPS
11290 11 : && eoschar == const0_rtx
11291 11 : && optimize > 1)
11292 : {
11293 : /* The generic case of strlen expander is long. Avoid it's
11294 : expanding unless TARGET_INLINE_ALL_STRINGOPS. */
11295 15 : rtx addr = force_reg (Pmode, XEXP (src, 0));
11296 : /* Well it seems that some optimizer does not combine a call like
11297 : foo(strlen(bar), strlen(bar));
11298 : when the move and the subtraction is done here. It does calculate
11299 : the length just once when these instructions are done inside of
11300 : output_strlen_unroll(). But I think since &bar[strlen(bar)] is
11301 : often used and I use one fewer register for the lifetime of
11302 : output_strlen_unroll() this is better. */
11303 :
11304 11 : emit_move_insn (out, addr);
11305 :
11306 11 : ix86_expand_strlensi_unroll_1 (out, src, align);
11307 :
11308 : /* strlensi_unroll_1 returns the address of the zero at the end of
11309 : the string, like memchr(), so compute the length by subtracting
11310 : the start address. */
11311 11 : emit_insn (gen_sub2_insn (out, addr));
11312 11 : return true;
11313 : }
11314 : else
11315 : return false;
11316 : }
11317 :
11318 : /* For given symbol (function) construct code to compute address of it's PLT
11319 : entry in large x86-64 PIC model. */
11320 :
11321 : static rtx
11322 34 : construct_plt_address (rtx symbol)
11323 : {
11324 34 : rtx tmp, unspec;
11325 :
11326 34 : gcc_assert (SYMBOL_REF_P (symbol));
11327 34 : gcc_assert (ix86_cmodel == CM_LARGE_PIC && !TARGET_PECOFF);
11328 34 : gcc_assert (Pmode == DImode);
11329 :
11330 34 : tmp = gen_reg_rtx (Pmode);
11331 34 : unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, symbol), UNSPEC_PLTOFF);
11332 :
11333 34 : emit_move_insn (tmp, gen_rtx_CONST (Pmode, unspec));
11334 34 : emit_insn (gen_add2_insn (tmp, pic_offset_table_rtx));
11335 34 : return tmp;
11336 : }
11337 :
11338 : rtx_insn *
11339 6397417 : ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
11340 : rtx callarg2,
11341 : rtx pop, bool sibcall)
11342 : {
11343 6397417 : rtx vec[3];
11344 6397417 : rtx use = NULL, call;
11345 6397417 : unsigned int vec_len = 0;
11346 6397417 : tree fndecl;
11347 :
11348 6397417 : if (SYMBOL_REF_P (XEXP (fnaddr, 0)))
11349 : {
11350 6210485 : fndecl = SYMBOL_REF_DECL (XEXP (fnaddr, 0));
11351 6210485 : if (fndecl)
11352 : {
11353 5950516 : if (lookup_attribute ("interrupt",
11354 5950516 : TYPE_ATTRIBUTES (TREE_TYPE (fndecl))))
11355 1 : error ("interrupt service routine cannot be called directly");
11356 5950516 : if (fndecl == current_function_decl
11357 5950516 : && decl_binds_to_current_def_p (fndecl))
11358 11351 : cfun->machine->recursive_function = true;
11359 : }
11360 : }
11361 : else
11362 : fndecl = NULL_TREE;
11363 :
11364 6397417 : if (pop == const0_rtx)
11365 0 : pop = NULL;
11366 6397417 : gcc_assert (!TARGET_64BIT || !pop);
11367 :
11368 6397417 : rtx addr = XEXP (fnaddr, 0);
11369 6397417 : if (TARGET_MACHO && !TARGET_64BIT)
11370 : {
11371 : #if TARGET_MACHO
11372 : if (flag_pic && SYMBOL_REF_P (XEXP (fnaddr, 0)))
11373 : fnaddr = machopic_indirect_call_target (fnaddr);
11374 : #endif
11375 : }
11376 : else
11377 : {
11378 : /* Static functions and indirect calls don't need the pic register. Also,
11379 : check if PLT was explicitly avoided via no-plt or "noplt" attribute, making
11380 : it an indirect call. */
11381 6397417 : if (flag_pic
11382 749633 : && SYMBOL_REF_P (addr)
11383 7119475 : && ix86_call_use_plt_p (addr))
11384 : {
11385 555490 : if (flag_plt
11386 555490 : && (SYMBOL_REF_DECL (addr) == NULL_TREE
11387 555456 : || !lookup_attribute ("noplt",
11388 555456 : DECL_ATTRIBUTES (SYMBOL_REF_DECL (addr)))))
11389 : {
11390 555455 : if (!TARGET_64BIT
11391 376501 : || (ix86_cmodel == CM_LARGE_PIC
11392 : && DEFAULT_ABI != MS_ABI))
11393 : {
11394 536896 : use_reg (&use, gen_rtx_REG (Pmode,
11395 : REAL_PIC_OFFSET_TABLE_REGNUM));
11396 178988 : if (ix86_use_pseudo_pic_reg ())
11397 357942 : emit_move_insn (gen_rtx_REG (Pmode,
11398 178988 : REAL_PIC_OFFSET_TABLE_REGNUM),
11399 : pic_offset_table_rtx);
11400 : }
11401 : }
11402 35 : else if (!TARGET_PECOFF && !TARGET_MACHO)
11403 : {
11404 35 : if (TARGET_64BIT
11405 35 : && ix86_cmodel == CM_LARGE_PIC
11406 : && DEFAULT_ABI != MS_ABI)
11407 : {
11408 1 : fnaddr = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr),
11409 : UNSPEC_GOT);
11410 1 : fnaddr = gen_rtx_CONST (Pmode, fnaddr);
11411 1 : fnaddr = force_reg (Pmode, fnaddr);
11412 1 : fnaddr = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, fnaddr);
11413 : }
11414 34 : else if (TARGET_64BIT)
11415 : {
11416 38 : fnaddr = gen_rtx_UNSPEC (Pmode,
11417 : gen_rtvec (1, addr),
11418 : UNSPEC_GOTPCREL);
11419 38 : fnaddr = gen_rtx_CONST (Pmode, fnaddr);
11420 : }
11421 : else
11422 : {
11423 0 : fnaddr = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr),
11424 : UNSPEC_GOT);
11425 0 : fnaddr = gen_rtx_CONST (Pmode, fnaddr);
11426 0 : fnaddr = gen_rtx_PLUS (Pmode, pic_offset_table_rtx,
11427 : fnaddr);
11428 : }
11429 39 : fnaddr = gen_const_mem (Pmode, fnaddr);
11430 : /* Pmode may not be the same as word_mode for x32, which
11431 : doesn't support indirect branch via 32-bit memory slot.
11432 : Since x32 GOT slot is 64 bit with zero upper 32 bits,
11433 : indirect branch via x32 GOT slot is OK. */
11434 35 : if (GET_MODE (fnaddr) != word_mode)
11435 4 : fnaddr = gen_rtx_ZERO_EXTEND (word_mode, fnaddr);
11436 35 : fnaddr = gen_rtx_MEM (QImode, fnaddr);
11437 : }
11438 : }
11439 : }
11440 :
11441 : /* Skip setting up RAX register for -mskip-rax-setup when there are no
11442 : parameters passed in vector registers. */
11443 6397417 : if (TARGET_64BIT
11444 5557244 : && (INTVAL (callarg2) > 0
11445 5495924 : || (INTVAL (callarg2) == 0
11446 328259 : && (TARGET_SSE || !flag_skip_rax_setup))))
11447 : {
11448 389577 : rtx al = gen_rtx_REG (QImode, AX_REG);
11449 389577 : emit_move_insn (al, callarg2);
11450 389577 : use_reg (&use, al);
11451 : }
11452 :
11453 6397417 : if (ix86_cmodel == CM_LARGE_PIC
11454 : && !TARGET_PECOFF
11455 45 : && MEM_P (fnaddr)
11456 45 : && SYMBOL_REF_P (XEXP (fnaddr, 0))
11457 6397454 : && !local_symbolic_operand (XEXP (fnaddr, 0), VOIDmode))
11458 34 : fnaddr = gen_rtx_MEM (QImode, construct_plt_address (XEXP (fnaddr, 0)));
11459 : /* Since x32 GOT slot is 64 bit with zero upper 32 bits, indirect
11460 : branch via x32 GOT slot is OK. */
11461 6397383 : else if (TARGET_X32
11462 74 : && MEM_P (fnaddr)
11463 74 : && GET_CODE (XEXP (fnaddr, 0)) == ZERO_EXTEND
11464 8 : && GOT_memory_operand (XEXP (XEXP (fnaddr, 0), 0), Pmode)
11465 6397387 : && !TARGET_INDIRECT_BRANCH_REGISTER)
11466 : ;
11467 6397383 : else if (sibcall
11468 6397383 : ? !sibcall_insn_operand (XEXP (fnaddr, 0), word_mode)
11469 6266377 : : !call_insn_operand (XEXP (fnaddr, 0), word_mode))
11470 : {
11471 532 : fnaddr = convert_to_mode (word_mode, XEXP (fnaddr, 0), 1);
11472 532 : fnaddr = gen_rtx_MEM (QImode, copy_to_mode_reg (word_mode, fnaddr));
11473 : }
11474 :
11475 : /* PR100665: Hwasan may tag code pointer which is not supported by LAM,
11476 : mask off code pointers here.
11477 : TODO: also need to handle indirect jump. */
11478 6398435 : if (ix86_memtag_can_tag_addresses () && !fndecl
11479 6397441 : && sanitize_flags_p (SANITIZE_HWADDRESS))
11480 : {
11481 24 : rtx untagged_addr = ix86_memtag_untagged_pointer (XEXP (fnaddr, 0),
11482 : NULL_RTX);
11483 24 : fnaddr = gen_rtx_MEM (QImode, untagged_addr);
11484 : }
11485 :
11486 6397417 : call = gen_rtx_CALL (VOIDmode, fnaddr, callarg1);
11487 :
11488 6397417 : if (retval)
11489 2523974 : call = gen_rtx_SET (retval, call);
11490 6397417 : vec[vec_len++] = call;
11491 :
11492 6397417 : if (pop)
11493 : {
11494 450314 : pop = gen_rtx_PLUS (Pmode, stack_pointer_rtx, pop);
11495 225157 : pop = gen_rtx_SET (stack_pointer_rtx, pop);
11496 225157 : vec[vec_len++] = pop;
11497 : }
11498 :
11499 : /* Set here, but it may get cleared later. */
11500 5557244 : if (TARGET_64BIT_MS_ABI
11501 73441 : && (!callarg2 || INTVAL (callarg2) != -2)
11502 6463721 : && TARGET_CALL_MS2SYSV_XLOGUES)
11503 : {
11504 7046 : if (!TARGET_SSE)
11505 : ;
11506 :
11507 : /* Don't break hot-patched functions. */
11508 7046 : else if (ix86_function_ms_hook_prologue (current_function_decl))
11509 : ;
11510 :
11511 : /* TODO: Cases not yet examined. */
11512 7046 : else if (flag_split_stack)
11513 0 : warn_once_call_ms2sysv_xlogues ("-fsplit-stack");
11514 :
11515 : else
11516 : {
11517 7046 : gcc_assert (!reload_completed);
11518 7046 : cfun->machine->call_ms2sysv = true;
11519 : }
11520 : }
11521 :
11522 6397417 : if (TARGET_MACHO && TARGET_64BIT && !sibcall
11523 : && ((SYMBOL_REF_P (addr) && !SYMBOL_REF_LOCAL_P (addr))
11524 : || !fndecl || TREE_PUBLIC (fndecl)))
11525 : {
11526 : /* We allow public functions defined in a TU to bind locally for PIC
11527 : code (the default) on 64bit Mach-O.
11528 : If such functions are not inlined, we cannot tell at compile-time if
11529 : they will be called via the lazy symbol resolver (this can depend on
11530 : options given at link-time). Therefore, we must assume that the lazy
11531 : resolver could be used which clobbers R11 and R10. */
11532 : clobber_reg (&use, gen_rtx_REG (DImode, R11_REG));
11533 : clobber_reg (&use, gen_rtx_REG (DImode, R10_REG));
11534 : }
11535 :
11536 6397417 : if (vec_len > 1)
11537 225157 : call = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (vec_len, vec));
11538 6397417 : rtx_insn *call_insn = emit_call_insn (call);
11539 6397417 : if (use)
11540 568564 : CALL_INSN_FUNCTION_USAGE (call_insn) = use;
11541 :
11542 6397417 : return call_insn;
11543 : }
11544 :
11545 : /* Split simple return with popping POPC bytes from stack to indirect
11546 : branch with stack adjustment . */
11547 :
11548 : void
11549 0 : ix86_split_simple_return_pop_internal (rtx popc)
11550 : {
11551 0 : struct machine_function *m = cfun->machine;
11552 0 : rtx ecx = gen_rtx_REG (SImode, CX_REG);
11553 0 : rtx_insn *insn;
11554 :
11555 : /* There is no "pascal" calling convention in any 64bit ABI. */
11556 0 : gcc_assert (!TARGET_64BIT);
11557 :
11558 0 : insn = emit_insn (gen_pop (ecx));
11559 0 : m->fs.cfa_offset -= UNITS_PER_WORD;
11560 0 : m->fs.sp_offset -= UNITS_PER_WORD;
11561 :
11562 0 : rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
11563 0 : x = gen_rtx_SET (stack_pointer_rtx, x);
11564 0 : add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
11565 0 : add_reg_note (insn, REG_CFA_REGISTER, gen_rtx_SET (ecx, pc_rtx));
11566 0 : RTX_FRAME_RELATED_P (insn) = 1;
11567 :
11568 0 : x = gen_rtx_PLUS (Pmode, stack_pointer_rtx, popc);
11569 0 : x = gen_rtx_SET (stack_pointer_rtx, x);
11570 0 : insn = emit_insn (x);
11571 0 : add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
11572 0 : RTX_FRAME_RELATED_P (insn) = 1;
11573 :
11574 : /* Now return address is in ECX. */
11575 0 : emit_jump_insn (gen_simple_return_indirect_internal (ecx));
11576 0 : }
11577 :
11578 : /* Errors in the source file can cause expand_expr to return const0_rtx
11579 : where we expect a vector. To avoid crashing, use one of the vector
11580 : clear instructions. */
11581 :
11582 : static rtx
11583 203091 : safe_vector_operand (rtx x, machine_mode mode)
11584 : {
11585 0 : if (x == const0_rtx)
11586 0 : x = CONST0_RTX (mode);
11587 24 : return x;
11588 : }
11589 :
11590 : /* Subroutine of ix86_expand_builtin to take care of binop insns. */
11591 :
11592 : static rtx
11593 8858 : ix86_expand_binop_builtin (enum insn_code icode, tree exp, rtx target)
11594 : {
11595 8858 : rtx pat;
11596 8858 : tree arg0 = CALL_EXPR_ARG (exp, 0);
11597 8858 : tree arg1 = CALL_EXPR_ARG (exp, 1);
11598 8858 : rtx op0 = expand_normal (arg0);
11599 8858 : rtx op1 = expand_normal (arg1);
11600 8858 : machine_mode tmode = insn_data[icode].operand[0].mode;
11601 8858 : machine_mode mode0 = insn_data[icode].operand[1].mode;
11602 8858 : machine_mode mode1 = insn_data[icode].operand[2].mode;
11603 :
11604 8858 : if (VECTOR_MODE_P (mode0))
11605 8847 : op0 = safe_vector_operand (op0, mode0);
11606 8858 : if (VECTOR_MODE_P (mode1))
11607 8711 : op1 = safe_vector_operand (op1, mode1);
11608 :
11609 2794 : if (optimize || !target
11610 2794 : || GET_MODE (target) != tmode
11611 11652 : || !insn_data[icode].operand[0].predicate (target, tmode))
11612 6117 : target = gen_reg_rtx (tmode);
11613 :
11614 8858 : if (GET_MODE (op1) == SImode && mode1 == TImode)
11615 : {
11616 0 : rtx x = gen_reg_rtx (V4SImode);
11617 0 : emit_insn (gen_sse2_loadd (x, op1));
11618 0 : op1 = gen_lowpart (TImode, x);
11619 : }
11620 :
11621 8858 : if (!insn_data[icode].operand[1].predicate (op0, mode0))
11622 1379 : op0 = copy_to_mode_reg (mode0, op0);
11623 8858 : if (!insn_data[icode].operand[2].predicate (op1, mode1))
11624 805 : op1 = copy_to_mode_reg (mode1, op1);
11625 :
11626 8858 : pat = GEN_FCN (icode) (target, op0, op1);
11627 8858 : if (! pat)
11628 : return 0;
11629 :
11630 8858 : emit_insn (pat);
11631 :
11632 8858 : return target;
11633 : }
11634 :
11635 : /* Subroutine of ix86_expand_builtin to take care of 2-4 argument insns. */
11636 :
11637 : static rtx
11638 1813 : ix86_expand_multi_arg_builtin (enum insn_code icode, tree exp, rtx target,
11639 : enum ix86_builtin_func_type m_type,
11640 : enum rtx_code sub_code)
11641 : {
11642 1813 : rtx pat;
11643 1813 : unsigned int i, nargs;
11644 1813 : bool comparison_p = false;
11645 1813 : bool tf_p = false;
11646 1813 : bool last_arg_constant = false;
11647 1813 : int num_memory = 0;
11648 1813 : rtx xops[4];
11649 :
11650 1813 : machine_mode tmode = insn_data[icode].operand[0].mode;
11651 :
11652 1813 : switch (m_type)
11653 : {
11654 : case MULTI_ARG_4_DF2_DI_I:
11655 : case MULTI_ARG_4_DF2_DI_I1:
11656 : case MULTI_ARG_4_SF2_SI_I:
11657 : case MULTI_ARG_4_SF2_SI_I1:
11658 : nargs = 4;
11659 : last_arg_constant = true;
11660 : break;
11661 :
11662 842 : case MULTI_ARG_3_SF:
11663 842 : case MULTI_ARG_3_DF:
11664 842 : case MULTI_ARG_3_SF2:
11665 842 : case MULTI_ARG_3_DF2:
11666 842 : case MULTI_ARG_3_DI:
11667 842 : case MULTI_ARG_3_SI:
11668 842 : case MULTI_ARG_3_SI_DI:
11669 842 : case MULTI_ARG_3_HI:
11670 842 : case MULTI_ARG_3_HI_SI:
11671 842 : case MULTI_ARG_3_QI:
11672 842 : case MULTI_ARG_3_DI2:
11673 842 : case MULTI_ARG_3_SI2:
11674 842 : case MULTI_ARG_3_HI2:
11675 842 : case MULTI_ARG_3_QI2:
11676 842 : nargs = 3;
11677 842 : break;
11678 :
11679 128 : case MULTI_ARG_2_SF:
11680 128 : case MULTI_ARG_2_DF:
11681 128 : case MULTI_ARG_2_DI:
11682 128 : case MULTI_ARG_2_SI:
11683 128 : case MULTI_ARG_2_HI:
11684 128 : case MULTI_ARG_2_QI:
11685 128 : nargs = 2;
11686 128 : break;
11687 :
11688 64 : case MULTI_ARG_2_DI_IMM:
11689 64 : case MULTI_ARG_2_SI_IMM:
11690 64 : case MULTI_ARG_2_HI_IMM:
11691 64 : case MULTI_ARG_2_QI_IMM:
11692 64 : nargs = 2;
11693 64 : last_arg_constant = true;
11694 64 : break;
11695 :
11696 187 : case MULTI_ARG_1_SF:
11697 187 : case MULTI_ARG_1_DF:
11698 187 : case MULTI_ARG_1_SF2:
11699 187 : case MULTI_ARG_1_DF2:
11700 187 : case MULTI_ARG_1_DI:
11701 187 : case MULTI_ARG_1_SI:
11702 187 : case MULTI_ARG_1_HI:
11703 187 : case MULTI_ARG_1_QI:
11704 187 : case MULTI_ARG_1_SI_DI:
11705 187 : case MULTI_ARG_1_HI_DI:
11706 187 : case MULTI_ARG_1_HI_SI:
11707 187 : case MULTI_ARG_1_QI_DI:
11708 187 : case MULTI_ARG_1_QI_SI:
11709 187 : case MULTI_ARG_1_QI_HI:
11710 187 : nargs = 1;
11711 187 : break;
11712 :
11713 384 : case MULTI_ARG_2_DI_CMP:
11714 384 : case MULTI_ARG_2_SI_CMP:
11715 384 : case MULTI_ARG_2_HI_CMP:
11716 384 : case MULTI_ARG_2_QI_CMP:
11717 384 : nargs = 2;
11718 384 : comparison_p = true;
11719 384 : break;
11720 :
11721 128 : case MULTI_ARG_2_SF_TF:
11722 128 : case MULTI_ARG_2_DF_TF:
11723 128 : case MULTI_ARG_2_DI_TF:
11724 128 : case MULTI_ARG_2_SI_TF:
11725 128 : case MULTI_ARG_2_HI_TF:
11726 128 : case MULTI_ARG_2_QI_TF:
11727 128 : nargs = 2;
11728 128 : tf_p = true;
11729 128 : break;
11730 :
11731 0 : default:
11732 0 : gcc_unreachable ();
11733 : }
11734 :
11735 628 : if (optimize || !target
11736 628 : || GET_MODE (target) != tmode
11737 2417 : || !insn_data[icode].operand[0].predicate (target, tmode))
11738 1209 : target = gen_reg_rtx (tmode);
11739 604 : else if (memory_operand (target, tmode))
11740 0 : num_memory++;
11741 :
11742 1813 : gcc_assert (nargs <= ARRAY_SIZE (xops));
11743 :
11744 6246 : for (i = 0; i < nargs; i++)
11745 : {
11746 4441 : tree arg = CALL_EXPR_ARG (exp, i);
11747 4441 : rtx op = expand_normal (arg);
11748 4441 : int adjust = (comparison_p) ? 1 : 0;
11749 4441 : machine_mode mode = insn_data[icode].operand[i+adjust+1].mode;
11750 :
11751 4441 : if (last_arg_constant && i == nargs - 1)
11752 : {
11753 144 : if (!insn_data[icode].operand[i + 1].predicate (op, mode))
11754 : {
11755 30 : enum insn_code new_icode = icode;
11756 30 : switch (icode)
11757 : {
11758 8 : case CODE_FOR_xop_vpermil2v2df3:
11759 8 : case CODE_FOR_xop_vpermil2v4sf3:
11760 8 : case CODE_FOR_xop_vpermil2v4df3:
11761 8 : case CODE_FOR_xop_vpermil2v8sf3:
11762 8 : error ("the last argument must be a 2-bit immediate");
11763 8 : return gen_reg_rtx (tmode);
11764 5 : case CODE_FOR_xop_rotlv2di3:
11765 5 : new_icode = CODE_FOR_rotlv2di3;
11766 5 : goto xop_rotl;
11767 5 : case CODE_FOR_xop_rotlv4si3:
11768 5 : new_icode = CODE_FOR_rotlv4si3;
11769 5 : goto xop_rotl;
11770 6 : case CODE_FOR_xop_rotlv8hi3:
11771 6 : new_icode = CODE_FOR_rotlv8hi3;
11772 6 : goto xop_rotl;
11773 : case CODE_FOR_xop_rotlv16qi3:
11774 : new_icode = CODE_FOR_rotlv16qi3;
11775 22 : xop_rotl:
11776 22 : if (CONST_INT_P (op))
11777 : {
11778 6 : int mask = GET_MODE_UNIT_BITSIZE (tmode) - 1;
11779 6 : op = GEN_INT (INTVAL (op) & mask);
11780 6 : gcc_checking_assert
11781 : (insn_data[icode].operand[i + 1].predicate (op, mode));
11782 : }
11783 : else
11784 : {
11785 16 : gcc_checking_assert
11786 : (nargs == 2
11787 : && insn_data[new_icode].operand[0].mode == tmode
11788 : && insn_data[new_icode].operand[1].mode == tmode
11789 : && insn_data[new_icode].operand[2].mode == mode
11790 : && insn_data[new_icode].operand[0].predicate
11791 : == insn_data[icode].operand[0].predicate
11792 : && insn_data[new_icode].operand[1].predicate
11793 : == insn_data[icode].operand[1].predicate);
11794 16 : icode = new_icode;
11795 16 : goto non_constant;
11796 : }
11797 : break;
11798 0 : default:
11799 0 : gcc_unreachable ();
11800 : }
11801 : }
11802 : }
11803 : else
11804 : {
11805 4297 : non_constant:
11806 4313 : if (VECTOR_MODE_P (mode))
11807 4297 : op = safe_vector_operand (op, mode);
11808 :
11809 : /* If we aren't optimizing, only allow one memory operand to be
11810 : generated. */
11811 4313 : if (memory_operand (op, mode))
11812 826 : num_memory++;
11813 :
11814 4313 : gcc_assert (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode);
11815 :
11816 4313 : if (optimize
11817 1506 : || !insn_data[icode].operand[i+adjust+1].predicate (op, mode)
11818 5741 : || num_memory > 1)
11819 3392 : op = force_reg (mode, op);
11820 : }
11821 :
11822 4433 : xops[i] = op;
11823 : }
11824 :
11825 1805 : switch (nargs)
11826 : {
11827 187 : case 1:
11828 187 : pat = GEN_FCN (icode) (target, xops[0]);
11829 187 : break;
11830 :
11831 704 : case 2:
11832 704 : if (tf_p)
11833 128 : pat = GEN_FCN (icode) (target, xops[0], xops[1],
11834 128 : GEN_INT ((int)sub_code));
11835 576 : else if (! comparison_p)
11836 192 : pat = GEN_FCN (icode) (target, xops[0], xops[1]);
11837 : else
11838 : {
11839 384 : rtx cmp_op = gen_rtx_fmt_ee (sub_code, GET_MODE (target),
11840 : xops[0], xops[1]);
11841 :
11842 384 : pat = GEN_FCN (icode) (target, cmp_op, xops[0], xops[1]);
11843 : }
11844 : break;
11845 :
11846 842 : case 3:
11847 842 : pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2]);
11848 842 : break;
11849 :
11850 72 : case 4:
11851 72 : pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2], xops[3]);
11852 72 : break;
11853 :
11854 : default:
11855 : gcc_unreachable ();
11856 : }
11857 :
11858 1805 : if (! pat)
11859 : return 0;
11860 :
11861 1805 : emit_insn (pat);
11862 1805 : return target;
11863 : }
11864 :
11865 : /* Subroutine of ix86_expand_args_builtin to take care of scalar unop
11866 : insns with vec_merge. */
11867 :
11868 : static rtx
11869 52 : ix86_expand_unop_vec_merge_builtin (enum insn_code icode, tree exp,
11870 : rtx target)
11871 : {
11872 52 : rtx pat;
11873 52 : tree arg0 = CALL_EXPR_ARG (exp, 0);
11874 52 : rtx op1, op0 = expand_normal (arg0);
11875 52 : machine_mode tmode = insn_data[icode].operand[0].mode;
11876 52 : machine_mode mode0 = insn_data[icode].operand[1].mode;
11877 :
11878 16 : if (optimize || !target
11879 16 : || GET_MODE (target) != tmode
11880 68 : || !insn_data[icode].operand[0].predicate (target, tmode))
11881 36 : target = gen_reg_rtx (tmode);
11882 :
11883 52 : if (VECTOR_MODE_P (mode0))
11884 52 : op0 = safe_vector_operand (op0, mode0);
11885 :
11886 36 : if ((optimize && !register_operand (op0, mode0))
11887 88 : || !insn_data[icode].operand[1].predicate (op0, mode0))
11888 0 : op0 = copy_to_mode_reg (mode0, op0);
11889 :
11890 52 : op1 = op0;
11891 52 : if (!insn_data[icode].operand[2].predicate (op1, mode0))
11892 16 : op1 = copy_to_mode_reg (mode0, op1);
11893 :
11894 52 : pat = GEN_FCN (icode) (target, op0, op1);
11895 52 : if (! pat)
11896 : return 0;
11897 52 : emit_insn (pat);
11898 52 : return target;
11899 : }
11900 :
11901 : /* Subroutine of ix86_expand_builtin to take care of comparison insns. */
11902 :
11903 : static rtx
11904 608 : ix86_expand_sse_compare (const struct builtin_description *d,
11905 : tree exp, rtx target, bool swap)
11906 : {
11907 608 : rtx pat;
11908 608 : tree arg0 = CALL_EXPR_ARG (exp, 0);
11909 608 : tree arg1 = CALL_EXPR_ARG (exp, 1);
11910 608 : rtx op0 = expand_normal (arg0);
11911 608 : rtx op1 = expand_normal (arg1);
11912 608 : rtx op2;
11913 608 : machine_mode tmode = insn_data[d->icode].operand[0].mode;
11914 608 : machine_mode mode0 = insn_data[d->icode].operand[1].mode;
11915 608 : machine_mode mode1 = insn_data[d->icode].operand[2].mode;
11916 608 : enum rtx_code comparison = d->comparison;
11917 :
11918 608 : if (VECTOR_MODE_P (mode0))
11919 608 : op0 = safe_vector_operand (op0, mode0);
11920 608 : if (VECTOR_MODE_P (mode1))
11921 608 : op1 = safe_vector_operand (op1, mode1);
11922 :
11923 : /* Swap operands if we have a comparison that isn't available in
11924 : hardware. */
11925 608 : if (swap)
11926 80 : std::swap (op0, op1);
11927 :
11928 202 : if (optimize || !target
11929 202 : || GET_MODE (target) != tmode
11930 810 : || !insn_data[d->icode].operand[0].predicate (target, tmode))
11931 406 : target = gen_reg_rtx (tmode);
11932 :
11933 406 : if ((optimize && !register_operand (op0, mode0))
11934 958 : || !insn_data[d->icode].operand[1].predicate (op0, mode0))
11935 258 : op0 = copy_to_mode_reg (mode0, op0);
11936 406 : if ((optimize && !register_operand (op1, mode1))
11937 958 : || !insn_data[d->icode].operand[2].predicate (op1, mode1))
11938 56 : op1 = copy_to_mode_reg (mode1, op1);
11939 :
11940 608 : op2 = gen_rtx_fmt_ee (comparison, mode0, op0, op1);
11941 608 : pat = GEN_FCN (d->icode) (target, op0, op1, op2);
11942 608 : if (! pat)
11943 : return 0;
11944 608 : emit_insn (pat);
11945 608 : return target;
11946 : }
11947 :
11948 : /* Subroutine of ix86_sse_comi and ix86_sse_comi_round to take care of
11949 : * ordered EQ or unordered NE, generate PF jump. */
11950 :
11951 : static rtx
11952 646 : ix86_ssecom_setcc (const enum rtx_code comparison,
11953 : bool check_unordered, machine_mode mode,
11954 : rtx set_dst, rtx target)
11955 : {
11956 :
11957 646 : rtx_code_label *label = NULL;
11958 :
11959 : /* NB: For ordered EQ or unordered NE, check ZF alone isn't sufficient
11960 : with NAN operands.
11961 : Under TARGET_AVX10_2, VCOMX/VUCOMX are generated instead of
11962 : COMI/UCOMI. VCOMX/VUCOMX will not set ZF for NAN operands. */
11963 646 : if (check_unordered)
11964 : {
11965 122 : gcc_assert (comparison == EQ || comparison == NE);
11966 :
11967 122 : rtx flag = gen_rtx_REG (CCFPmode, FLAGS_REG);
11968 122 : label = gen_label_rtx ();
11969 122 : rtx tmp = gen_rtx_fmt_ee (UNORDERED, VOIDmode, flag, const0_rtx);
11970 122 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
11971 : gen_rtx_LABEL_REF (VOIDmode, label),
11972 : pc_rtx);
11973 122 : emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
11974 : }
11975 :
11976 : /* NB: Set CCFPmode and check a different CCmode which is in subset
11977 : of CCFPmode. */
11978 646 : if (GET_MODE (set_dst) != mode)
11979 : {
11980 200 : gcc_assert (mode == CCAmode || mode == CCCmode
11981 : || mode == CCOmode || mode == CCPmode
11982 : || mode == CCSmode || mode == CCZmode);
11983 200 : set_dst = gen_rtx_REG (mode, FLAGS_REG);
11984 : }
11985 :
11986 646 : emit_insn (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
11987 : gen_rtx_fmt_ee (comparison, QImode,
11988 : set_dst,
11989 : const0_rtx)));
11990 :
11991 646 : if (label)
11992 122 : emit_label (label);
11993 :
11994 646 : return SUBREG_REG (target);
11995 : }
11996 :
11997 : /* Subroutine of ix86_expand_builtin to take care of comi insns. */
11998 :
11999 : static rtx
12000 547 : ix86_expand_sse_comi (const struct builtin_description *d, tree exp,
12001 : rtx target, bool comx_ok)
12002 : {
12003 547 : rtx pat, set_dst;
12004 547 : tree arg0 = CALL_EXPR_ARG (exp, 0);
12005 547 : tree arg1 = CALL_EXPR_ARG (exp, 1);
12006 547 : rtx op0 = expand_normal (arg0);
12007 547 : rtx op1 = expand_normal (arg1);
12008 547 : enum insn_code icode = d->icode;
12009 547 : const struct insn_data_d *insn_p = &insn_data[icode];
12010 547 : machine_mode mode0 = insn_p->operand[0].mode;
12011 547 : machine_mode mode1 = insn_p->operand[1].mode;
12012 :
12013 547 : if (VECTOR_MODE_P (mode0))
12014 547 : op0 = safe_vector_operand (op0, mode0);
12015 547 : if (VECTOR_MODE_P (mode1))
12016 547 : op1 = safe_vector_operand (op1, mode1);
12017 :
12018 547 : enum rtx_code comparison = d->comparison;
12019 547 : rtx const_val = const0_rtx;
12020 :
12021 547 : bool check_unordered = false;
12022 547 : machine_mode mode = CCFPmode;
12023 547 : switch (comparison)
12024 : {
12025 194 : case LE: /* -> GE */
12026 194 : case LT: /* -> GT */
12027 194 : std::swap (op0, op1);
12028 194 : comparison = swap_condition (comparison);
12029 : /* FALLTHRU */
12030 : case GT:
12031 : case GE:
12032 : break;
12033 73 : case EQ:
12034 73 : if (!TARGET_AVX10_2 || !comx_ok)
12035 45 : check_unordered = true;
12036 : mode = CCZmode;
12037 : break;
12038 96 : case NE:
12039 96 : if (!TARGET_AVX10_2 || !comx_ok)
12040 68 : check_unordered = true;
12041 96 : mode = CCZmode;
12042 96 : const_val = const1_rtx;
12043 96 : break;
12044 0 : default:
12045 0 : gcc_unreachable ();
12046 : }
12047 :
12048 547 : target = gen_reg_rtx (SImode);
12049 547 : emit_move_insn (target, const_val);
12050 547 : target = gen_rtx_SUBREG (QImode, target, 0);
12051 :
12052 426 : if ((optimize && !register_operand (op0, mode0))
12053 925 : || !insn_p->operand[0].predicate (op0, mode0))
12054 169 : op0 = copy_to_mode_reg (mode0, op0);
12055 426 : if ((optimize && !register_operand (op1, mode1))
12056 924 : || !insn_p->operand[1].predicate (op1, mode1))
12057 49 : op1 = copy_to_mode_reg (mode1, op1);
12058 :
12059 547 : if ((comparison == EQ || comparison == NE)
12060 169 : && TARGET_AVX10_2 && comx_ok)
12061 : {
12062 56 : switch (icode)
12063 : {
12064 : case CODE_FOR_sse_comi:
12065 : icode = CODE_FOR_avx10_2_comxsf;
12066 : break;
12067 14 : case CODE_FOR_sse_ucomi:
12068 14 : icode = CODE_FOR_avx10_2_ucomxsf;
12069 14 : break;
12070 14 : case CODE_FOR_sse2_comi:
12071 14 : icode = CODE_FOR_avx10_2_comxdf;
12072 14 : break;
12073 14 : case CODE_FOR_sse2_ucomi:
12074 14 : icode = CODE_FOR_avx10_2_ucomxdf;
12075 14 : break;
12076 :
12077 0 : default:
12078 0 : gcc_unreachable ();
12079 : }
12080 : }
12081 547 : pat = GEN_FCN (icode) (op0, op1);
12082 547 : if (! pat)
12083 : return 0;
12084 :
12085 547 : set_dst = SET_DEST (pat);
12086 547 : emit_insn (pat);
12087 547 : return ix86_ssecom_setcc (comparison, check_unordered, mode,
12088 547 : set_dst, target);
12089 : }
12090 :
12091 : /* Subroutines of ix86_expand_args_builtin to take care of round insns. */
12092 :
12093 : static rtx
12094 0 : ix86_expand_sse_round (const struct builtin_description *d, tree exp,
12095 : rtx target)
12096 : {
12097 0 : rtx pat;
12098 0 : tree arg0 = CALL_EXPR_ARG (exp, 0);
12099 0 : rtx op1, op0 = expand_normal (arg0);
12100 0 : machine_mode tmode = insn_data[d->icode].operand[0].mode;
12101 0 : machine_mode mode0 = insn_data[d->icode].operand[1].mode;
12102 :
12103 0 : if (optimize || target == 0
12104 0 : || GET_MODE (target) != tmode
12105 0 : || !insn_data[d->icode].operand[0].predicate (target, tmode))
12106 0 : target = gen_reg_rtx (tmode);
12107 :
12108 0 : if (VECTOR_MODE_P (mode0))
12109 0 : op0 = safe_vector_operand (op0, mode0);
12110 :
12111 0 : if ((optimize && !register_operand (op0, mode0))
12112 0 : || !insn_data[d->icode].operand[0].predicate (op0, mode0))
12113 0 : op0 = copy_to_mode_reg (mode0, op0);
12114 :
12115 0 : op1 = GEN_INT (d->comparison);
12116 :
12117 0 : pat = GEN_FCN (d->icode) (target, op0, op1);
12118 0 : if (! pat)
12119 : return 0;
12120 0 : emit_insn (pat);
12121 0 : return target;
12122 : }
12123 :
12124 : static rtx
12125 12 : ix86_expand_sse_round_vec_pack_sfix (const struct builtin_description *d,
12126 : tree exp, rtx target)
12127 : {
12128 12 : rtx pat;
12129 12 : tree arg0 = CALL_EXPR_ARG (exp, 0);
12130 12 : tree arg1 = CALL_EXPR_ARG (exp, 1);
12131 12 : rtx op0 = expand_normal (arg0);
12132 12 : rtx op1 = expand_normal (arg1);
12133 12 : rtx op2;
12134 12 : machine_mode tmode = insn_data[d->icode].operand[0].mode;
12135 12 : machine_mode mode0 = insn_data[d->icode].operand[1].mode;
12136 12 : machine_mode mode1 = insn_data[d->icode].operand[2].mode;
12137 :
12138 0 : if (optimize || target == 0
12139 0 : || GET_MODE (target) != tmode
12140 12 : || !insn_data[d->icode].operand[0].predicate (target, tmode))
12141 12 : target = gen_reg_rtx (tmode);
12142 :
12143 12 : op0 = safe_vector_operand (op0, mode0);
12144 12 : op1 = safe_vector_operand (op1, mode1);
12145 :
12146 12 : if ((optimize && !register_operand (op0, mode0))
12147 12 : || !insn_data[d->icode].operand[0].predicate (op0, mode0))
12148 12 : op0 = copy_to_mode_reg (mode0, op0);
12149 12 : if ((optimize && !register_operand (op1, mode1))
12150 12 : || !insn_data[d->icode].operand[1].predicate (op1, mode1))
12151 12 : op1 = copy_to_mode_reg (mode1, op1);
12152 :
12153 12 : op2 = GEN_INT (d->comparison);
12154 :
12155 12 : pat = GEN_FCN (d->icode) (target, op0, op1, op2);
12156 12 : if (! pat)
12157 : return 0;
12158 12 : emit_insn (pat);
12159 12 : return target;
12160 : }
12161 :
12162 : /* Subroutine of ix86_expand_builtin to take care of ptest insns. */
12163 :
12164 : static rtx
12165 239 : ix86_expand_sse_ptest (const struct builtin_description *d, tree exp,
12166 : rtx target)
12167 : {
12168 239 : rtx pat;
12169 239 : tree arg0 = CALL_EXPR_ARG (exp, 0);
12170 239 : tree arg1 = CALL_EXPR_ARG (exp, 1);
12171 239 : rtx op0 = expand_normal (arg0);
12172 239 : rtx op1 = expand_normal (arg1);
12173 239 : machine_mode mode0 = insn_data[d->icode].operand[0].mode;
12174 239 : machine_mode mode1 = insn_data[d->icode].operand[1].mode;
12175 239 : enum rtx_code comparison = d->comparison;
12176 239 : rtx result = NULL_RTX;
12177 :
12178 239 : if (VECTOR_MODE_P (mode0))
12179 239 : op0 = safe_vector_operand (op0, mode0);
12180 239 : if (VECTOR_MODE_P (mode1))
12181 239 : op1 = safe_vector_operand (op1, mode1);
12182 :
12183 239 : switch (d->code)
12184 : {
12185 49 : case IX86_BUILTIN_PTESTZ:
12186 49 : case IX86_BUILTIN_PTESTZ256:
12187 : // Returns (OP0 & OP1) == 0
12188 49 : if (rtx_equal_p (op0, CONST0_RTX (mode0))
12189 49 : || rtx_equal_p (op1, CONST0_RTX (mode1)))
12190 2 : result = const1_rtx;
12191 47 : else if (rtx_equal_p (op0, CONSTM1_RTX (mode0)))
12192 : {
12193 1 : op1 = force_reg (mode1, op1);
12194 1 : op0 = op1;
12195 : }
12196 46 : else if (rtx_equal_p (op1, CONSTM1_RTX (mode1)))
12197 : {
12198 1 : op0 = force_reg (mode0, op0);
12199 1 : op1 = op0;
12200 : }
12201 45 : else if (MEM_P (op0) && !MEM_P (op1))
12202 : std::swap (op0, op1);
12203 : break;
12204 :
12205 31 : case IX86_BUILTIN_PTESTC:
12206 31 : case IX86_BUILTIN_PTESTC256:
12207 : // Returns (~OP0 & OP1) == 0
12208 31 : if (rtx_equal_p (op0, CONSTM1_RTX (mode0))
12209 31 : || rtx_equal_p (op1, CONST0_RTX (mode1))
12210 62 : || rtx_equal_p (op0, op1))
12211 2 : result = const1_rtx;
12212 : break;
12213 :
12214 27 : case IX86_BUILTIN_PTESTNZC:
12215 27 : case IX86_BUILTIN_PTESTNZC256:
12216 : // Returns ((OP0 && OP1) != 0) && ((~OP0 && OP1) != 0)
12217 27 : if (rtx_equal_p (op0, CONST0_RTX (mode0))
12218 26 : || rtx_equal_p (op0, CONSTM1_RTX (mode0))
12219 26 : || rtx_equal_p (op1, CONST0_RTX (mode1))
12220 53 : || rtx_equal_p (op0, op1))
12221 1 : result = const0_rtx;
12222 : break;
12223 :
12224 : default:
12225 : break;
12226 : }
12227 :
12228 167 : if ((optimize && !register_operand (op0, mode0))
12229 210 : || !insn_data[d->icode].operand[0].predicate (op0, mode0)
12230 377 : || result)
12231 104 : op0 = copy_to_mode_reg (mode0, op0);
12232 167 : if ((optimize && !register_operand (op1, mode1))
12233 211 : || !insn_data[d->icode].operand[1].predicate (op1, mode1)
12234 450 : || result)
12235 31 : op1 = copy_to_mode_reg (mode1, op1);
12236 :
12237 239 : if (result)
12238 : {
12239 5 : if (!target)
12240 0 : target = gen_reg_rtx (SImode);
12241 5 : emit_move_insn (target, result);
12242 5 : return target;
12243 : }
12244 :
12245 234 : target = gen_reg_rtx (SImode);
12246 234 : emit_move_insn (target, const0_rtx);
12247 234 : target = gen_rtx_SUBREG (QImode, target, 0);
12248 :
12249 234 : pat = GEN_FCN (d->icode) (op0, op1);
12250 234 : if (! pat)
12251 : return 0;
12252 234 : emit_insn (pat);
12253 234 : emit_insn (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
12254 : gen_rtx_fmt_ee (comparison, QImode,
12255 : SET_DEST (pat),
12256 : const0_rtx)));
12257 :
12258 234 : return SUBREG_REG (target);
12259 : }
12260 :
12261 : /* Subroutine of ix86_expand_builtin to take care of pcmpestr[im] insns. */
12262 :
12263 : static rtx
12264 216 : ix86_expand_sse_pcmpestr (const struct builtin_description *d,
12265 : tree exp, rtx target)
12266 : {
12267 216 : rtx pat;
12268 216 : tree arg0 = CALL_EXPR_ARG (exp, 0);
12269 216 : tree arg1 = CALL_EXPR_ARG (exp, 1);
12270 216 : tree arg2 = CALL_EXPR_ARG (exp, 2);
12271 216 : tree arg3 = CALL_EXPR_ARG (exp, 3);
12272 216 : tree arg4 = CALL_EXPR_ARG (exp, 4);
12273 216 : rtx scratch0, scratch1;
12274 216 : rtx op0 = expand_normal (arg0);
12275 216 : rtx op1 = expand_normal (arg1);
12276 216 : rtx op2 = expand_normal (arg2);
12277 216 : rtx op3 = expand_normal (arg3);
12278 216 : rtx op4 = expand_normal (arg4);
12279 216 : machine_mode tmode0, tmode1, modev2, modei3, modev4, modei5, modeimm;
12280 :
12281 216 : tmode0 = insn_data[d->icode].operand[0].mode;
12282 216 : tmode1 = insn_data[d->icode].operand[1].mode;
12283 216 : modev2 = insn_data[d->icode].operand[2].mode;
12284 216 : modei3 = insn_data[d->icode].operand[3].mode;
12285 216 : modev4 = insn_data[d->icode].operand[4].mode;
12286 216 : modei5 = insn_data[d->icode].operand[5].mode;
12287 216 : modeimm = insn_data[d->icode].operand[6].mode;
12288 :
12289 216 : if (VECTOR_MODE_P (modev2))
12290 216 : op0 = safe_vector_operand (op0, modev2);
12291 216 : if (VECTOR_MODE_P (modev4))
12292 216 : op2 = safe_vector_operand (op2, modev4);
12293 :
12294 216 : if (!insn_data[d->icode].operand[2].predicate (op0, modev2))
12295 6 : op0 = copy_to_mode_reg (modev2, op0);
12296 216 : if (!insn_data[d->icode].operand[3].predicate (op1, modei3))
12297 34 : op1 = copy_to_mode_reg (modei3, op1);
12298 160 : if ((optimize && !register_operand (op2, modev4))
12299 371 : || !insn_data[d->icode].operand[4].predicate (op2, modev4))
12300 5 : op2 = copy_to_mode_reg (modev4, op2);
12301 216 : if (!insn_data[d->icode].operand[5].predicate (op3, modei5))
12302 34 : op3 = copy_to_mode_reg (modei5, op3);
12303 :
12304 216 : if (!insn_data[d->icode].operand[6].predicate (op4, modeimm))
12305 : {
12306 21 : error ("the fifth argument must be an 8-bit immediate");
12307 21 : return const0_rtx;
12308 : }
12309 :
12310 195 : if (d->code == IX86_BUILTIN_PCMPESTRI128)
12311 : {
12312 5 : if (optimize || !target
12313 5 : || GET_MODE (target) != tmode0
12314 34 : || !insn_data[d->icode].operand[0].predicate (target, tmode0))
12315 24 : target = gen_reg_rtx (tmode0);
12316 :
12317 29 : scratch1 = gen_reg_rtx (tmode1);
12318 :
12319 29 : pat = GEN_FCN (d->icode) (target, scratch1, op0, op1, op2, op3, op4);
12320 : }
12321 166 : else if (d->code == IX86_BUILTIN_PCMPESTRM128)
12322 : {
12323 5 : if (optimize || !target
12324 5 : || GET_MODE (target) != tmode1
12325 36 : || !insn_data[d->icode].operand[1].predicate (target, tmode1))
12326 26 : target = gen_reg_rtx (tmode1);
12327 :
12328 31 : scratch0 = gen_reg_rtx (tmode0);
12329 :
12330 31 : pat = GEN_FCN (d->icode) (scratch0, target, op0, op1, op2, op3, op4);
12331 : }
12332 : else
12333 : {
12334 135 : gcc_assert (d->flag);
12335 :
12336 135 : scratch0 = gen_reg_rtx (tmode0);
12337 135 : scratch1 = gen_reg_rtx (tmode1);
12338 :
12339 135 : pat = GEN_FCN (d->icode) (scratch0, scratch1, op0, op1, op2, op3, op4);
12340 : }
12341 :
12342 195 : if (! pat)
12343 : return 0;
12344 :
12345 195 : emit_insn (pat);
12346 :
12347 195 : if (d->flag)
12348 : {
12349 135 : target = gen_reg_rtx (SImode);
12350 135 : emit_move_insn (target, const0_rtx);
12351 135 : target = gen_rtx_SUBREG (QImode, target, 0);
12352 :
12353 135 : emit_insn
12354 135 : (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
12355 : gen_rtx_fmt_ee (EQ, QImode,
12356 : gen_rtx_REG ((machine_mode) d->flag,
12357 : FLAGS_REG),
12358 : const0_rtx)));
12359 135 : return SUBREG_REG (target);
12360 : }
12361 : else
12362 : return target;
12363 : }
12364 :
12365 :
12366 : /* Subroutine of ix86_expand_builtin to take care of pcmpistr[im] insns. */
12367 :
12368 : static rtx
12369 275 : ix86_expand_sse_pcmpistr (const struct builtin_description *d,
12370 : tree exp, rtx target)
12371 : {
12372 275 : rtx pat;
12373 275 : tree arg0 = CALL_EXPR_ARG (exp, 0);
12374 275 : tree arg1 = CALL_EXPR_ARG (exp, 1);
12375 275 : tree arg2 = CALL_EXPR_ARG (exp, 2);
12376 275 : rtx scratch0, scratch1;
12377 275 : rtx op0 = expand_normal (arg0);
12378 275 : rtx op1 = expand_normal (arg1);
12379 275 : rtx op2 = expand_normal (arg2);
12380 275 : machine_mode tmode0, tmode1, modev2, modev3, modeimm;
12381 :
12382 275 : tmode0 = insn_data[d->icode].operand[0].mode;
12383 275 : tmode1 = insn_data[d->icode].operand[1].mode;
12384 275 : modev2 = insn_data[d->icode].operand[2].mode;
12385 275 : modev3 = insn_data[d->icode].operand[3].mode;
12386 275 : modeimm = insn_data[d->icode].operand[4].mode;
12387 :
12388 275 : if (VECTOR_MODE_P (modev2))
12389 275 : op0 = safe_vector_operand (op0, modev2);
12390 275 : if (VECTOR_MODE_P (modev3))
12391 275 : op1 = safe_vector_operand (op1, modev3);
12392 :
12393 275 : if (!insn_data[d->icode].operand[2].predicate (op0, modev2))
12394 4 : op0 = copy_to_mode_reg (modev2, op0);
12395 210 : if ((optimize && !register_operand (op1, modev3))
12396 481 : || !insn_data[d->icode].operand[3].predicate (op1, modev3))
12397 4 : op1 = copy_to_mode_reg (modev3, op1);
12398 :
12399 275 : if (!insn_data[d->icode].operand[4].predicate (op2, modeimm))
12400 : {
12401 21 : error ("the third argument must be an 8-bit immediate");
12402 21 : return const0_rtx;
12403 : }
12404 :
12405 254 : if (d->code == IX86_BUILTIN_PCMPISTRI128)
12406 : {
12407 5 : if (optimize || !target
12408 5 : || GET_MODE (target) != tmode0
12409 38 : || !insn_data[d->icode].operand[0].predicate (target, tmode0))
12410 28 : target = gen_reg_rtx (tmode0);
12411 :
12412 33 : scratch1 = gen_reg_rtx (tmode1);
12413 :
12414 33 : pat = GEN_FCN (d->icode) (target, scratch1, op0, op1, op2);
12415 : }
12416 221 : else if (d->code == IX86_BUILTIN_PCMPISTRM128)
12417 : {
12418 8 : if (optimize || !target
12419 8 : || GET_MODE (target) != tmode1
12420 58 : || !insn_data[d->icode].operand[1].predicate (target, tmode1))
12421 42 : target = gen_reg_rtx (tmode1);
12422 :
12423 50 : scratch0 = gen_reg_rtx (tmode0);
12424 :
12425 50 : pat = GEN_FCN (d->icode) (scratch0, target, op0, op1, op2);
12426 : }
12427 : else
12428 : {
12429 171 : gcc_assert (d->flag);
12430 :
12431 171 : scratch0 = gen_reg_rtx (tmode0);
12432 171 : scratch1 = gen_reg_rtx (tmode1);
12433 :
12434 171 : pat = GEN_FCN (d->icode) (scratch0, scratch1, op0, op1, op2);
12435 : }
12436 :
12437 254 : if (! pat)
12438 : return 0;
12439 :
12440 254 : emit_insn (pat);
12441 :
12442 254 : if (d->flag)
12443 : {
12444 171 : target = gen_reg_rtx (SImode);
12445 171 : emit_move_insn (target, const0_rtx);
12446 171 : target = gen_rtx_SUBREG (QImode, target, 0);
12447 :
12448 171 : emit_insn
12449 171 : (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
12450 : gen_rtx_fmt_ee (EQ, QImode,
12451 : gen_rtx_REG ((machine_mode) d->flag,
12452 : FLAGS_REG),
12453 : const0_rtx)));
12454 171 : return SUBREG_REG (target);
12455 : }
12456 : else
12457 : return target;
12458 : }
12459 :
12460 : /* Fixup modeless constants to fit required mode. */
12461 :
12462 : static rtx
12463 267793 : fixup_modeless_constant (rtx x, machine_mode mode)
12464 : {
12465 267793 : if (GET_MODE (x) == VOIDmode)
12466 41901 : x = convert_to_mode (mode, x, 1);
12467 267793 : return x;
12468 : }
12469 :
12470 : /* Expand the outgoing argument ARG to extract unsigned char and short
12471 : integer constants suitable for the predicates and the instruction
12472 : templates which expect the unsigned expanded value. */
12473 :
12474 : static rtx
12475 289044 : ix86_expand_unsigned_small_int_cst_argument (tree arg)
12476 : {
12477 : /* When passing 0xff as an unsigned char function argument with the
12478 : C frontend promotion, expand_normal gets
12479 :
12480 : <integer_cst 0x7fffe6aa23a8 type <integer_type 0x7fffe98225e8 int> constant 255>
12481 :
12482 : and returns the rtx value using the sign-extended representation:
12483 :
12484 : (const_int 255 [0xff])
12485 :
12486 : Without the C frontend promotion, expand_normal gets
12487 :
12488 : <integer_cst 0x7fffe9824018 type <integer_type 0x7fffe9822348 unsigned char > constant 255>
12489 :
12490 : and returns
12491 :
12492 : (const_int -1 [0xffffffffffffffff])
12493 :
12494 : which doesn't work with the predicates nor the instruction templates
12495 : which expect the unsigned expanded value. Extract the unsigned char
12496 : and short integer constants to return
12497 :
12498 : (const_int 255 [0xff])
12499 :
12500 : so that the expanded value is always unsigned, without the C frontend
12501 : promotion. */
12502 :
12503 289044 : if (TREE_CODE (arg) == INTEGER_CST)
12504 : {
12505 60796 : tree type = TREE_TYPE (arg);
12506 60796 : if (INTEGRAL_TYPE_P (type)
12507 60796 : && TYPE_UNSIGNED (type)
12508 83096 : && TYPE_PRECISION (type) < TYPE_PRECISION (integer_type_node))
12509 : {
12510 18758 : HOST_WIDE_INT cst = TREE_INT_CST_LOW (arg);
12511 18758 : return GEN_INT (cst);
12512 : }
12513 : }
12514 :
12515 270286 : return expand_normal (arg);
12516 : }
12517 :
12518 : /* Subroutine of ix86_expand_builtin to take care of insns with
12519 : variable number of operands. */
12520 :
12521 : static rtx
12522 74280 : ix86_expand_args_builtin (const struct builtin_description *d,
12523 : tree exp, rtx target)
12524 : {
12525 74280 : rtx pat, real_target;
12526 74280 : unsigned int i, nargs;
12527 74280 : unsigned int nargs_constant = 0;
12528 74280 : unsigned int mask_pos = 0;
12529 74280 : int num_memory = 0;
12530 74280 : rtx xops[6];
12531 74280 : bool second_arg_count = false;
12532 74280 : enum insn_code icode = d->icode;
12533 74280 : const struct insn_data_d *insn_p = &insn_data[icode];
12534 74280 : machine_mode tmode = insn_p->operand[0].mode;
12535 74280 : machine_mode rmode = VOIDmode;
12536 74280 : bool swap = false;
12537 74280 : enum rtx_code comparison = d->comparison;
12538 :
12539 74280 : switch ((enum ix86_builtin_func_type) d->flag)
12540 : {
12541 0 : case V2DF_FTYPE_V2DF_ROUND:
12542 0 : case V4DF_FTYPE_V4DF_ROUND:
12543 0 : case V8DF_FTYPE_V8DF_ROUND:
12544 0 : case V4SF_FTYPE_V4SF_ROUND:
12545 0 : case V8SF_FTYPE_V8SF_ROUND:
12546 0 : case V16SF_FTYPE_V16SF_ROUND:
12547 0 : case V8HF_FTYPE_V8HF_ROUND:
12548 0 : case V16HF_FTYPE_V16HF_ROUND:
12549 0 : case V32HF_FTYPE_V32HF_ROUND:
12550 0 : case V4SI_FTYPE_V4SF_ROUND:
12551 0 : case V8SI_FTYPE_V8SF_ROUND:
12552 0 : case V16SI_FTYPE_V16SF_ROUND:
12553 0 : return ix86_expand_sse_round (d, exp, target);
12554 12 : case V4SI_FTYPE_V2DF_V2DF_ROUND:
12555 12 : case V8SI_FTYPE_V4DF_V4DF_ROUND:
12556 12 : case V16SI_FTYPE_V8DF_V8DF_ROUND:
12557 12 : return ix86_expand_sse_round_vec_pack_sfix (d, exp, target);
12558 239 : case INT_FTYPE_V8SF_V8SF_PTEST:
12559 239 : case INT_FTYPE_V4DI_V4DI_PTEST:
12560 239 : case INT_FTYPE_V4DF_V4DF_PTEST:
12561 239 : case INT_FTYPE_V4SF_V4SF_PTEST:
12562 239 : case INT_FTYPE_V2DI_V2DI_PTEST:
12563 239 : case INT_FTYPE_V2DF_V2DF_PTEST:
12564 239 : return ix86_expand_sse_ptest (d, exp, target);
12565 : case FLOAT128_FTYPE_FLOAT128:
12566 : case FLOAT_FTYPE_FLOAT:
12567 : case FLOAT_FTYPE_BFLOAT16:
12568 : case INT_FTYPE_INT:
12569 : case UINT_FTYPE_UINT:
12570 : case UINT16_FTYPE_UINT16:
12571 : case UINT64_FTYPE_INT:
12572 : case UINT64_FTYPE_UINT64:
12573 : case INT64_FTYPE_INT64:
12574 : case INT64_FTYPE_V4SF:
12575 : case INT64_FTYPE_V2DF:
12576 : case INT_FTYPE_V16QI:
12577 : case INT_FTYPE_V8QI:
12578 : case INT_FTYPE_V8SF:
12579 : case INT_FTYPE_V4DF:
12580 : case INT_FTYPE_V4SF:
12581 : case INT_FTYPE_V2DF:
12582 : case INT_FTYPE_V32QI:
12583 : case V16QI_FTYPE_V16QI:
12584 : case V8SI_FTYPE_V8SF:
12585 : case V8SI_FTYPE_V4SI:
12586 : case V8HI_FTYPE_V8HI:
12587 : case V8HI_FTYPE_V16QI:
12588 : case V8QI_FTYPE_V8QI:
12589 : case V8SF_FTYPE_V8SF:
12590 : case V8SF_FTYPE_V8SI:
12591 : case V8SF_FTYPE_V4SF:
12592 : case V8SF_FTYPE_V8HI:
12593 : case V4SI_FTYPE_V4SI:
12594 : case V4SI_FTYPE_V16QI:
12595 : case V4SI_FTYPE_V4SF:
12596 : case V4SI_FTYPE_V8SI:
12597 : case V4SI_FTYPE_V8HI:
12598 : case V4SI_FTYPE_V4DF:
12599 : case V4SI_FTYPE_V2DF:
12600 : case V4HI_FTYPE_V4HI:
12601 : case V4DF_FTYPE_V4DF:
12602 : case V4DF_FTYPE_V4SI:
12603 : case V4DF_FTYPE_V4SF:
12604 : case V4DF_FTYPE_V2DF:
12605 : case V4SF_FTYPE_V4SF:
12606 : case V4SF_FTYPE_V4SI:
12607 : case V4SF_FTYPE_V8SF:
12608 : case V4SF_FTYPE_V4DF:
12609 : case V4SF_FTYPE_V8HI:
12610 : case V4SF_FTYPE_V2DF:
12611 : case V2DI_FTYPE_V2DI:
12612 : case V2DI_FTYPE_V16QI:
12613 : case V2DI_FTYPE_V8HI:
12614 : case V2DI_FTYPE_V4SI:
12615 : case V2DF_FTYPE_V2DF:
12616 : case V2DF_FTYPE_V4SI:
12617 : case V2DF_FTYPE_V4DF:
12618 : case V2DF_FTYPE_V4SF:
12619 : case V2DF_FTYPE_V2SI:
12620 : case V2SI_FTYPE_V2SI:
12621 : case V2SI_FTYPE_V4SF:
12622 : case V2SI_FTYPE_V2SF:
12623 : case V2SI_FTYPE_V2DF:
12624 : case V2SF_FTYPE_V2SF:
12625 : case V2SF_FTYPE_V2SI:
12626 : case V32QI_FTYPE_V32QI:
12627 : case V32QI_FTYPE_V16QI:
12628 : case V16HI_FTYPE_V16HI:
12629 : case V16HI_FTYPE_V8HI:
12630 : case V8SI_FTYPE_V8SI:
12631 : case V16HI_FTYPE_V16QI:
12632 : case V8SI_FTYPE_V16QI:
12633 : case V4DI_FTYPE_V16QI:
12634 : case V8SI_FTYPE_V8HI:
12635 : case V4DI_FTYPE_V8HI:
12636 : case V4DI_FTYPE_V4SI:
12637 : case V4DI_FTYPE_V2DI:
12638 : case UQI_FTYPE_UQI:
12639 : case UHI_FTYPE_UHI:
12640 : case USI_FTYPE_USI:
12641 : case USI_FTYPE_UQI:
12642 : case USI_FTYPE_UHI:
12643 : case UDI_FTYPE_UDI:
12644 : case UHI_FTYPE_V16QI:
12645 : case USI_FTYPE_V32QI:
12646 : case UDI_FTYPE_V64QI:
12647 : case V16QI_FTYPE_UHI:
12648 : case V32QI_FTYPE_USI:
12649 : case V64QI_FTYPE_UDI:
12650 : case V8HI_FTYPE_UQI:
12651 : case V16HI_FTYPE_UHI:
12652 : case V32HI_FTYPE_USI:
12653 : case V4SI_FTYPE_UQI:
12654 : case V8SI_FTYPE_UQI:
12655 : case V4SI_FTYPE_UHI:
12656 : case V8SI_FTYPE_UHI:
12657 : case UQI_FTYPE_V8HI:
12658 : case UHI_FTYPE_V16HI:
12659 : case USI_FTYPE_V32HI:
12660 : case UQI_FTYPE_V4SI:
12661 : case UQI_FTYPE_V8SI:
12662 : case UHI_FTYPE_V16SI:
12663 : case UQI_FTYPE_V2DI:
12664 : case UQI_FTYPE_V4DI:
12665 : case UQI_FTYPE_V8DI:
12666 : case V16SI_FTYPE_UHI:
12667 : case V2DI_FTYPE_UQI:
12668 : case V4DI_FTYPE_UQI:
12669 : case V16SI_FTYPE_INT:
12670 : case V16SF_FTYPE_V8SF:
12671 : case V16SI_FTYPE_V8SI:
12672 : case V16SF_FTYPE_V4SF:
12673 : case V16SI_FTYPE_V4SI:
12674 : case V16SI_FTYPE_V16SF:
12675 : case V16SI_FTYPE_V16SI:
12676 : case V64QI_FTYPE_V64QI:
12677 : case V32HI_FTYPE_V32HI:
12678 : case V16SF_FTYPE_V16SF:
12679 : case V8DI_FTYPE_UQI:
12680 : case V8DI_FTYPE_V8DI:
12681 : case V8DF_FTYPE_V4DF:
12682 : case V8DF_FTYPE_V2DF:
12683 : case V8DF_FTYPE_V8DF:
12684 : case V4DI_FTYPE_V4DI:
12685 : case V16BF_FTYPE_V16SF:
12686 : case V8BF_FTYPE_V8SF:
12687 : case V8BF_FTYPE_V4SF:
12688 : case V16QI_FTYPE_V32QI:
12689 : case V32QI_FTYPE_V64QI:
12690 : nargs = 1;
12691 : break;
12692 52 : case V4SF_FTYPE_V4SF_VEC_MERGE:
12693 52 : case V2DF_FTYPE_V2DF_VEC_MERGE:
12694 52 : return ix86_expand_unop_vec_merge_builtin (icode, exp, target);
12695 9386 : case FLOAT128_FTYPE_FLOAT128_FLOAT128:
12696 9386 : case V16QI_FTYPE_V16QI_V16QI:
12697 9386 : case V16QI_FTYPE_V8HI_V8HI:
12698 9386 : case V16HF_FTYPE_V16HF_V16HF:
12699 9386 : case V16SF_FTYPE_V16SF_V16SF:
12700 9386 : case V16SI_FTYPE_V16SI_V16SI:
12701 9386 : case V8QI_FTYPE_V8QI_V8QI:
12702 9386 : case V8QI_FTYPE_V4HI_V4HI:
12703 9386 : case V8HI_FTYPE_V8HI_V8HI:
12704 9386 : case V8HI_FTYPE_V16QI_V16QI:
12705 9386 : case V8HI_FTYPE_V4SI_V4SI:
12706 9386 : case V8HF_FTYPE_V8HF_V8HF:
12707 9386 : case V8SF_FTYPE_V8SF_V8SF:
12708 9386 : case V8SF_FTYPE_V8SF_V8SI:
12709 9386 : case V8DF_FTYPE_V8DF_V8DF:
12710 9386 : case V4SI_FTYPE_V4SI_V4SI:
12711 9386 : case V4SI_FTYPE_V8HI_V8HI:
12712 9386 : case V4SI_FTYPE_V2DF_V2DF:
12713 9386 : case V4HI_FTYPE_V4HI_V4HI:
12714 9386 : case V4HI_FTYPE_V8QI_V8QI:
12715 9386 : case V4HI_FTYPE_V2SI_V2SI:
12716 9386 : case V4DF_FTYPE_V4DF_V4DF:
12717 9386 : case V4DF_FTYPE_V4DF_V4DI:
12718 9386 : case V4SF_FTYPE_V4SF_V4SF:
12719 9386 : case V4SF_FTYPE_V4SF_V4SI:
12720 9386 : case V4SF_FTYPE_V4SF_V2SI:
12721 9386 : case V4SF_FTYPE_V4SF_V2DF:
12722 9386 : case V4SF_FTYPE_V4SF_UINT:
12723 9386 : case V4SF_FTYPE_V4SF_DI:
12724 9386 : case V4SF_FTYPE_V4SF_SI:
12725 9386 : case V4DI_FTYPE_V4DI_V2DI:
12726 9386 : case V2DI_FTYPE_V2DI_V2DI:
12727 9386 : case V2DI_FTYPE_V16QI_V16QI:
12728 9386 : case V2DI_FTYPE_V4SI_V4SI:
12729 9386 : case V2DI_FTYPE_V2DI_V16QI:
12730 9386 : case V2SI_FTYPE_V2SI_V2SI:
12731 9386 : case V2SI_FTYPE_V4HI_V4HI:
12732 9386 : case V2SI_FTYPE_V2SF_V2SF:
12733 9386 : case V2DF_FTYPE_V2DF_V2DF:
12734 9386 : case V2DF_FTYPE_V2DF_V4SF:
12735 9386 : case V2DF_FTYPE_V2DF_V2DI:
12736 9386 : case V2DF_FTYPE_V2DF_DI:
12737 9386 : case V2DF_FTYPE_V2DF_SI:
12738 9386 : case V2DF_FTYPE_V2DF_UINT:
12739 9386 : case V2SF_FTYPE_V2SF_V2SF:
12740 9386 : case V1DI_FTYPE_V1DI_V1DI:
12741 9386 : case V1DI_FTYPE_V8QI_V8QI:
12742 9386 : case V1DI_FTYPE_V2SI_V2SI:
12743 9386 : case V32QI_FTYPE_V16HI_V16HI:
12744 9386 : case V16HI_FTYPE_V8SI_V8SI:
12745 9386 : case V64QI_FTYPE_V64QI_V64QI:
12746 9386 : case V32QI_FTYPE_V32QI_V32QI:
12747 9386 : case V32BF_FTYPE_V32BF_V32BF:
12748 9386 : case V16BF_FTYPE_V16BF_V16BF:
12749 9386 : case V8BF_FTYPE_V8BF_V8BF:
12750 9386 : case V16HI_FTYPE_V32QI_V32QI:
12751 9386 : case V16HI_FTYPE_V16HI_V16HI:
12752 9386 : case V8SI_FTYPE_V4DF_V4DF:
12753 9386 : case V8SI_FTYPE_V8SI_V8SI:
12754 9386 : case V8SI_FTYPE_V16HI_V16HI:
12755 9386 : case V4DI_FTYPE_V4DI_V4DI:
12756 9386 : case V4DI_FTYPE_V8SI_V8SI:
12757 9386 : case V4DI_FTYPE_V32QI_V32QI:
12758 9386 : case V8DI_FTYPE_V64QI_V64QI:
12759 9386 : if (comparison == UNKNOWN)
12760 8858 : return ix86_expand_binop_builtin (icode, exp, target);
12761 : nargs = 2;
12762 : break;
12763 80 : case V4SF_FTYPE_V4SF_V4SF_SWAP:
12764 80 : case V2DF_FTYPE_V2DF_V2DF_SWAP:
12765 80 : gcc_assert (comparison != UNKNOWN);
12766 : nargs = 2;
12767 : swap = true;
12768 : break;
12769 1481 : case V16HI_FTYPE_V16HI_V8HI_COUNT:
12770 1481 : case V16HI_FTYPE_V16HI_SI_COUNT:
12771 1481 : case V8SI_FTYPE_V8SI_V4SI_COUNT:
12772 1481 : case V8SI_FTYPE_V8SI_SI_COUNT:
12773 1481 : case V4DI_FTYPE_V4DI_V2DI_COUNT:
12774 1481 : case V4DI_FTYPE_V4DI_INT_COUNT:
12775 1481 : case V8HI_FTYPE_V8HI_V8HI_COUNT:
12776 1481 : case V8HI_FTYPE_V8HI_SI_COUNT:
12777 1481 : case V4SI_FTYPE_V4SI_V4SI_COUNT:
12778 1481 : case V4SI_FTYPE_V4SI_SI_COUNT:
12779 1481 : case V4HI_FTYPE_V4HI_V4HI_COUNT:
12780 1481 : case V4HI_FTYPE_V4HI_SI_COUNT:
12781 1481 : case V2DI_FTYPE_V2DI_V2DI_COUNT:
12782 1481 : case V2DI_FTYPE_V2DI_SI_COUNT:
12783 1481 : case V2SI_FTYPE_V2SI_V2SI_COUNT:
12784 1481 : case V2SI_FTYPE_V2SI_SI_COUNT:
12785 1481 : case V1DI_FTYPE_V1DI_V1DI_COUNT:
12786 1481 : case V1DI_FTYPE_V1DI_SI_COUNT:
12787 1481 : nargs = 2;
12788 1481 : second_arg_count = true;
12789 1481 : break;
12790 1408 : case V16HI_FTYPE_V16HI_INT_V16HI_UHI_COUNT:
12791 1408 : case V16HI_FTYPE_V16HI_V8HI_V16HI_UHI_COUNT:
12792 1408 : case V16SI_FTYPE_V16SI_INT_V16SI_UHI_COUNT:
12793 1408 : case V16SI_FTYPE_V16SI_V4SI_V16SI_UHI_COUNT:
12794 1408 : case V2DI_FTYPE_V2DI_INT_V2DI_UQI_COUNT:
12795 1408 : case V2DI_FTYPE_V2DI_V2DI_V2DI_UQI_COUNT:
12796 1408 : case V32HI_FTYPE_V32HI_INT_V32HI_USI_COUNT:
12797 1408 : case V32HI_FTYPE_V32HI_V8HI_V32HI_USI_COUNT:
12798 1408 : case V4DI_FTYPE_V4DI_INT_V4DI_UQI_COUNT:
12799 1408 : case V4DI_FTYPE_V4DI_V2DI_V4DI_UQI_COUNT:
12800 1408 : case V4SI_FTYPE_V4SI_INT_V4SI_UQI_COUNT:
12801 1408 : case V4SI_FTYPE_V4SI_V4SI_V4SI_UQI_COUNT:
12802 1408 : case V8DI_FTYPE_V8DI_INT_V8DI_UQI_COUNT:
12803 1408 : case V8DI_FTYPE_V8DI_V2DI_V8DI_UQI_COUNT:
12804 1408 : case V8HI_FTYPE_V8HI_INT_V8HI_UQI_COUNT:
12805 1408 : case V8HI_FTYPE_V8HI_V8HI_V8HI_UQI_COUNT:
12806 1408 : case V8SI_FTYPE_V8SI_INT_V8SI_UQI_COUNT:
12807 1408 : case V8SI_FTYPE_V8SI_V4SI_V8SI_UQI_COUNT:
12808 1408 : nargs = 4;
12809 1408 : second_arg_count = true;
12810 1408 : break;
12811 966 : case UINT64_FTYPE_UINT64_UINT64:
12812 966 : case UINT_FTYPE_UINT_UINT:
12813 966 : case UINT_FTYPE_UINT_USHORT:
12814 966 : case UINT_FTYPE_UINT_UCHAR:
12815 966 : case UINT16_FTYPE_UINT16_INT:
12816 966 : case UINT8_FTYPE_UINT8_INT:
12817 966 : case UQI_FTYPE_UQI_UQI:
12818 966 : case UHI_FTYPE_UHI_UHI:
12819 966 : case USI_FTYPE_USI_USI:
12820 966 : case UDI_FTYPE_UDI_UDI:
12821 966 : case V16SI_FTYPE_V8DF_V8DF:
12822 966 : case V32BF_FTYPE_V16SF_V16SF:
12823 966 : case V16BF_FTYPE_V8SF_V8SF:
12824 966 : case V8BF_FTYPE_V4SF_V4SF:
12825 966 : case V16BF_FTYPE_V16SF_UHI:
12826 966 : case V8BF_FTYPE_V8SF_UQI:
12827 966 : case V8BF_FTYPE_V4SF_UQI:
12828 966 : case V16QI_FTYPE_V16QI_V8HF:
12829 966 : nargs = 2;
12830 966 : break;
12831 651 : case V2DI_FTYPE_V2DI_INT_CONVERT:
12832 651 : nargs = 2;
12833 651 : rmode = V1TImode;
12834 651 : nargs_constant = 1;
12835 651 : break;
12836 42 : case V4DI_FTYPE_V4DI_INT_CONVERT:
12837 42 : nargs = 2;
12838 42 : rmode = V2TImode;
12839 42 : nargs_constant = 1;
12840 42 : break;
12841 16 : case V8DI_FTYPE_V8DI_INT_CONVERT:
12842 16 : nargs = 2;
12843 16 : rmode = V4TImode;
12844 16 : nargs_constant = 1;
12845 16 : break;
12846 2424 : case V8HI_FTYPE_V8HI_INT:
12847 2424 : case V8HI_FTYPE_V8SF_INT:
12848 2424 : case V16HI_FTYPE_V16SF_INT:
12849 2424 : case V8HI_FTYPE_V4SF_INT:
12850 2424 : case V8SF_FTYPE_V8SF_INT:
12851 2424 : case V4SF_FTYPE_V16SF_INT:
12852 2424 : case V16SF_FTYPE_V16SF_INT:
12853 2424 : case V4SI_FTYPE_V4SI_INT:
12854 2424 : case V4SI_FTYPE_V8SI_INT:
12855 2424 : case V4HI_FTYPE_V4HI_INT:
12856 2424 : case V4DF_FTYPE_V4DF_INT:
12857 2424 : case V4DF_FTYPE_V8DF_INT:
12858 2424 : case V4SF_FTYPE_V4SF_INT:
12859 2424 : case V4SF_FTYPE_V8SF_INT:
12860 2424 : case V2DI_FTYPE_V2DI_INT:
12861 2424 : case V2DF_FTYPE_V2DF_INT:
12862 2424 : case V2DF_FTYPE_V4DF_INT:
12863 2424 : case V16HI_FTYPE_V16HI_INT:
12864 2424 : case V8SI_FTYPE_V8SI_INT:
12865 2424 : case V16SI_FTYPE_V16SI_INT:
12866 2424 : case V4SI_FTYPE_V16SI_INT:
12867 2424 : case V4DI_FTYPE_V4DI_INT:
12868 2424 : case V2DI_FTYPE_V4DI_INT:
12869 2424 : case V4DI_FTYPE_V8DI_INT:
12870 2424 : case UQI_FTYPE_UQI_UQI_CONST:
12871 2424 : case UHI_FTYPE_UHI_UQI:
12872 2424 : case USI_FTYPE_USI_UQI:
12873 2424 : case UDI_FTYPE_UDI_UQI:
12874 2424 : nargs = 2;
12875 2424 : nargs_constant = 1;
12876 2424 : break;
12877 19903 : case V16QI_FTYPE_V16QI_V16QI_V16QI:
12878 19903 : case V8SF_FTYPE_V8SF_V8SF_V8SF:
12879 19903 : case V4DF_FTYPE_V4DF_V4DF_V4DF:
12880 19903 : case V4SF_FTYPE_V4SF_V4SF_V4SF:
12881 19903 : case V2DF_FTYPE_V2DF_V2DF_V2DF:
12882 19903 : case V32QI_FTYPE_V32QI_V32QI_V32QI:
12883 19903 : case UHI_FTYPE_V16SI_V16SI_UHI:
12884 19903 : case UQI_FTYPE_V8DI_V8DI_UQI:
12885 19903 : case V16HI_FTYPE_V16SI_V16HI_UHI:
12886 19903 : case V16QI_FTYPE_V16SI_V16QI_UHI:
12887 19903 : case V16QI_FTYPE_V8DI_V16QI_UQI:
12888 19903 : case V32HF_FTYPE_V32HF_V32HF_USI:
12889 19903 : case V16SF_FTYPE_V16SF_V16SF_UHI:
12890 19903 : case V16SF_FTYPE_V4SF_V16SF_UHI:
12891 19903 : case V16SI_FTYPE_SI_V16SI_UHI:
12892 19903 : case V16SI_FTYPE_V16HI_V16SI_UHI:
12893 19903 : case V16SI_FTYPE_V16QI_V16SI_UHI:
12894 19903 : case V8SF_FTYPE_V4SF_V8SF_UQI:
12895 19903 : case V4DF_FTYPE_V2DF_V4DF_UQI:
12896 19903 : case V8SI_FTYPE_V4SI_V8SI_UQI:
12897 19903 : case V8SI_FTYPE_SI_V8SI_UQI:
12898 19903 : case V4SI_FTYPE_V4SI_V4SI_UQI:
12899 19903 : case V4SI_FTYPE_SI_V4SI_UQI:
12900 19903 : case V4DI_FTYPE_V2DI_V4DI_UQI:
12901 19903 : case V4DI_FTYPE_DI_V4DI_UQI:
12902 19903 : case V2DI_FTYPE_V2DI_V2DI_UQI:
12903 19903 : case V2DI_FTYPE_DI_V2DI_UQI:
12904 19903 : case V64QI_FTYPE_V64QI_V64QI_UDI:
12905 19903 : case V64QI_FTYPE_V16QI_V64QI_UDI:
12906 19903 : case V64QI_FTYPE_QI_V64QI_UDI:
12907 19903 : case V32QI_FTYPE_V32QI_V32QI_USI:
12908 19903 : case V32QI_FTYPE_V16QI_V32QI_USI:
12909 19903 : case V32QI_FTYPE_QI_V32QI_USI:
12910 19903 : case V16QI_FTYPE_V16QI_V16QI_UHI:
12911 19903 : case V16QI_FTYPE_QI_V16QI_UHI:
12912 19903 : case V32HI_FTYPE_V8HI_V32HI_USI:
12913 19903 : case V32HI_FTYPE_V32BF_V32HI_USI:
12914 19903 : case V32HI_FTYPE_HI_V32HI_USI:
12915 19903 : case V16HI_FTYPE_V8HI_V16HI_UHI:
12916 19903 : case V16HI_FTYPE_V16BF_V16HI_UHI:
12917 19903 : case V16HI_FTYPE_HI_V16HI_UHI:
12918 19903 : case V8HI_FTYPE_V8HI_V8HI_UQI:
12919 19903 : case V8HI_FTYPE_V8BF_V8HI_UQI:
12920 19903 : case V8BF_FTYPE_V8BF_V8BF_UQI:
12921 19903 : case V8HI_FTYPE_HI_V8HI_UQI:
12922 19903 : case V16HF_FTYPE_V16HF_V16HF_UHI:
12923 19903 : case V8SF_FTYPE_V8HI_V8SF_UQI:
12924 19903 : case V4SF_FTYPE_V8HI_V4SF_UQI:
12925 19903 : case V8SI_FTYPE_V8HF_V8SI_UQI:
12926 19903 : case V8SF_FTYPE_V8HF_V8SF_UQI:
12927 19903 : case V8SI_FTYPE_V8SF_V8SI_UQI:
12928 19903 : case V4SI_FTYPE_V4SF_V4SI_UQI:
12929 19903 : case V4SI_FTYPE_V8HF_V4SI_UQI:
12930 19903 : case V4SF_FTYPE_V8HF_V4SF_UQI:
12931 19903 : case V4DI_FTYPE_V8HF_V4DI_UQI:
12932 19903 : case V4DI_FTYPE_V4SF_V4DI_UQI:
12933 19903 : case V2DI_FTYPE_V8HF_V2DI_UQI:
12934 19903 : case V2DI_FTYPE_V4SF_V2DI_UQI:
12935 19903 : case V8HF_FTYPE_V8HF_V8HF_UQI:
12936 19903 : case V8HF_FTYPE_V8HF_V8HF_V8HF:
12937 19903 : case V8HF_FTYPE_V8HI_V8HF_UQI:
12938 19903 : case V8HF_FTYPE_V8SI_V8HF_UQI:
12939 19903 : case V8HF_FTYPE_V8SF_V8HF_UQI:
12940 19903 : case V8HF_FTYPE_V4SI_V8HF_UQI:
12941 19903 : case V8HF_FTYPE_V4SF_V8HF_UQI:
12942 19903 : case V8HF_FTYPE_V4DI_V8HF_UQI:
12943 19903 : case V8HF_FTYPE_V4DF_V8HF_UQI:
12944 19903 : case V8HF_FTYPE_V2DI_V8HF_UQI:
12945 19903 : case V8HF_FTYPE_V2DF_V8HF_UQI:
12946 19903 : case V4SF_FTYPE_V4DI_V4SF_UQI:
12947 19903 : case V4SF_FTYPE_V2DI_V4SF_UQI:
12948 19903 : case V4DF_FTYPE_V4DI_V4DF_UQI:
12949 19903 : case V4DF_FTYPE_V8HF_V4DF_UQI:
12950 19903 : case V2DF_FTYPE_V8HF_V2DF_UQI:
12951 19903 : case V2DF_FTYPE_V2DI_V2DF_UQI:
12952 19903 : case V16QI_FTYPE_V8HI_V16QI_UQI:
12953 19903 : case V16QI_FTYPE_V16HI_V16QI_UHI:
12954 19903 : case V16QI_FTYPE_V4SI_V16QI_UQI:
12955 19903 : case V16QI_FTYPE_V8SI_V16QI_UQI:
12956 19903 : case V8HI_FTYPE_V8HF_V8HI_UQI:
12957 19903 : case V8HI_FTYPE_V4SI_V8HI_UQI:
12958 19903 : case V8HI_FTYPE_V8SI_V8HI_UQI:
12959 19903 : case V16QI_FTYPE_V2DI_V16QI_UQI:
12960 19903 : case V16QI_FTYPE_V4DI_V16QI_UQI:
12961 19903 : case V8HI_FTYPE_V2DI_V8HI_UQI:
12962 19903 : case V8HI_FTYPE_V4DI_V8HI_UQI:
12963 19903 : case V4SI_FTYPE_V2DI_V4SI_UQI:
12964 19903 : case V4SI_FTYPE_V4DI_V4SI_UQI:
12965 19903 : case V32QI_FTYPE_V32HI_V32QI_USI:
12966 19903 : case UHI_FTYPE_V16QI_V16QI_UHI:
12967 19903 : case USI_FTYPE_V32QI_V32QI_USI:
12968 19903 : case UDI_FTYPE_V64QI_V64QI_UDI:
12969 19903 : case UQI_FTYPE_V8HI_V8HI_UQI:
12970 19903 : case UHI_FTYPE_V16HI_V16HI_UHI:
12971 19903 : case USI_FTYPE_V32HI_V32HI_USI:
12972 19903 : case UQI_FTYPE_V4SI_V4SI_UQI:
12973 19903 : case UQI_FTYPE_V8SI_V8SI_UQI:
12974 19903 : case UQI_FTYPE_V2DI_V2DI_UQI:
12975 19903 : case UQI_FTYPE_V4DI_V4DI_UQI:
12976 19903 : case V4SF_FTYPE_V2DF_V4SF_UQI:
12977 19903 : case V4SF_FTYPE_V4DF_V4SF_UQI:
12978 19903 : case V16SI_FTYPE_V16SI_V16SI_UHI:
12979 19903 : case V16SI_FTYPE_V4SI_V16SI_UHI:
12980 19903 : case V2DI_FTYPE_V4SI_V2DI_UQI:
12981 19903 : case V2DI_FTYPE_V8HI_V2DI_UQI:
12982 19903 : case V2DI_FTYPE_V16QI_V2DI_UQI:
12983 19903 : case V4DI_FTYPE_V4DI_V4DI_UQI:
12984 19903 : case V4DI_FTYPE_V4SI_V4DI_UQI:
12985 19903 : case V4DI_FTYPE_V8HI_V4DI_UQI:
12986 19903 : case V4DI_FTYPE_V16QI_V4DI_UQI:
12987 19903 : case V4DI_FTYPE_V4DF_V4DI_UQI:
12988 19903 : case V2DI_FTYPE_V2DF_V2DI_UQI:
12989 19903 : case V4SI_FTYPE_V4DF_V4SI_UQI:
12990 19903 : case V4SI_FTYPE_V2DF_V4SI_UQI:
12991 19903 : case V4SI_FTYPE_V8HI_V4SI_UQI:
12992 19903 : case V4SI_FTYPE_V16QI_V4SI_UQI:
12993 19903 : case V4DI_FTYPE_V4DI_V4DI_V4DI:
12994 19903 : case V8DF_FTYPE_V2DF_V8DF_UQI:
12995 19903 : case V8DF_FTYPE_V4DF_V8DF_UQI:
12996 19903 : case V8DF_FTYPE_V8DF_V8DF_UQI:
12997 19903 : case V8SF_FTYPE_V8SF_V8SF_UQI:
12998 19903 : case V8SF_FTYPE_V8SI_V8SF_UQI:
12999 19903 : case V4DF_FTYPE_V4DF_V4DF_UQI:
13000 19903 : case V4SF_FTYPE_V4SF_V4SF_UQI:
13001 19903 : case V2DF_FTYPE_V2DF_V2DF_UQI:
13002 19903 : case V2DF_FTYPE_V4SF_V2DF_UQI:
13003 19903 : case V2DF_FTYPE_V4SI_V2DF_UQI:
13004 19903 : case V4SF_FTYPE_V4SI_V4SF_UQI:
13005 19903 : case V4DF_FTYPE_V4SF_V4DF_UQI:
13006 19903 : case V4DF_FTYPE_V4SI_V4DF_UQI:
13007 19903 : case V8SI_FTYPE_V8SI_V8SI_UQI:
13008 19903 : case V8SI_FTYPE_V8HI_V8SI_UQI:
13009 19903 : case V8SI_FTYPE_V16QI_V8SI_UQI:
13010 19903 : case V8DF_FTYPE_V8SI_V8DF_UQI:
13011 19903 : case V8DI_FTYPE_DI_V8DI_UQI:
13012 19903 : case V16SF_FTYPE_V8SF_V16SF_UHI:
13013 19903 : case V16SI_FTYPE_V8SI_V16SI_UHI:
13014 19903 : case V16HF_FTYPE_V16HI_V16HF_UHI:
13015 19903 : case V16HF_FTYPE_V16HF_V16HF_V16HF:
13016 19903 : case V16HI_FTYPE_V16HF_V16HI_UHI:
13017 19903 : case V16HI_FTYPE_V16HI_V16HI_UHI:
13018 19903 : case V16BF_FTYPE_V16BF_V16BF_UHI:
13019 19903 : case V8HI_FTYPE_V16QI_V8HI_UQI:
13020 19903 : case V16HI_FTYPE_V16QI_V16HI_UHI:
13021 19903 : case V32HI_FTYPE_V32HI_V32HI_USI:
13022 19903 : case V32BF_FTYPE_V32BF_V32BF_USI:
13023 19903 : case V32HI_FTYPE_V32QI_V32HI_USI:
13024 19903 : case V8DI_FTYPE_V16QI_V8DI_UQI:
13025 19903 : case V8DI_FTYPE_V2DI_V8DI_UQI:
13026 19903 : case V8DI_FTYPE_V4DI_V8DI_UQI:
13027 19903 : case V8DI_FTYPE_V8DI_V8DI_UQI:
13028 19903 : case V8DI_FTYPE_V8HI_V8DI_UQI:
13029 19903 : case V8DI_FTYPE_V8SI_V8DI_UQI:
13030 19903 : case V8HI_FTYPE_V8DI_V8HI_UQI:
13031 19903 : case V8SI_FTYPE_V8DI_V8SI_UQI:
13032 19903 : case V4SI_FTYPE_V4SI_V4SI_V4SI:
13033 19903 : case V4DI_FTYPE_V4DI_V4DI_V2DI:
13034 19903 : case V16SI_FTYPE_V16SI_V16SI_V16SI:
13035 19903 : case V8DI_FTYPE_V8DI_V8DI_V8DI:
13036 19903 : case V32HI_FTYPE_V32HI_V32HI_V32HI:
13037 19903 : case V2DI_FTYPE_V2DI_V2DI_V2DI:
13038 19903 : case V16HI_FTYPE_V16HI_V16HI_V16HI:
13039 19903 : case V8SI_FTYPE_V8SI_V8SI_V8SI:
13040 19903 : case V8HI_FTYPE_V8HI_V8HI_V8HI:
13041 19903 : case V32BF_FTYPE_V16SF_V16SF_USI:
13042 19903 : case V16BF_FTYPE_V8SF_V8SF_UHI:
13043 19903 : case V8BF_FTYPE_V4SF_V4SF_UQI:
13044 19903 : case V16BF_FTYPE_V16SF_V16BF_UHI:
13045 19903 : case V8BF_FTYPE_V8SF_V8BF_UQI:
13046 19903 : case V8BF_FTYPE_V4SF_V8BF_UQI:
13047 19903 : case V16SF_FTYPE_V16SF_V32BF_V32BF:
13048 19903 : case V8SF_FTYPE_V8SF_V16BF_V16BF:
13049 19903 : case V4SF_FTYPE_V4SF_V8BF_V8BF:
13050 19903 : case V16QI_FTYPE_V16QI_V8HF_V8HF:
13051 19903 : case V32QI_FTYPE_V32QI_V16HF_V16HF:
13052 19903 : case V64QI_FTYPE_V64QI_V32HF_V32HF:
13053 19903 : case V16QI_FTYPE_V8HF_V16QI_UQI:
13054 19903 : case V16QI_FTYPE_V16HF_V16QI_UHI:
13055 19903 : case V32QI_FTYPE_V32HF_V32QI_USI:
13056 19903 : case V8HF_FTYPE_V16QI_V8HF_UQI:
13057 19903 : case V16HF_FTYPE_V16QI_V16HF_UHI:
13058 19903 : case V32HF_FTYPE_V32QI_V32HF_USI:
13059 19903 : case V16SI_FTYPE_V16SF_V16SI_UHI:
13060 19903 : case V32HI_FTYPE_V32HF_V32HI_USI:
13061 19903 : case V8DI_FTYPE_V8SF_V8DI_UQI:
13062 19903 : case V8DI_FTYPE_V8DF_V8DI_UQI:
13063 19903 : case V8SI_FTYPE_V8DF_V8SI_UQI:
13064 19903 : case V16QI_FTYPE_V4SF_V16QI_UQI:
13065 19903 : case V16QI_FTYPE_V8SF_V16QI_UQI:
13066 19903 : case V16QI_FTYPE_V16SF_V16QI_UHI:
13067 19903 : case V4SF_FTYPE_V16QI_V4SF_UQI:
13068 19903 : case V8SF_FTYPE_V16QI_V8SF_UQI:
13069 19903 : case V16SF_FTYPE_V16QI_V16SF_UHI:
13070 19903 : case V64QI_FTYPE_V32QI_V64QI_UDI:
13071 19903 : nargs = 3;
13072 19903 : break;
13073 1481 : case V32QI_FTYPE_V32QI_V32QI_INT:
13074 1481 : case V16HI_FTYPE_V16HI_V16HI_INT:
13075 1481 : case V16QI_FTYPE_V16QI_V16QI_INT:
13076 1481 : case V4DI_FTYPE_V4DI_V4DI_INT:
13077 1481 : case V8HI_FTYPE_V8HI_V8HI_INT:
13078 1481 : case V8SI_FTYPE_V8SI_V8SI_INT:
13079 1481 : case V8SI_FTYPE_V8SI_V4SI_INT:
13080 1481 : case V8SF_FTYPE_V8SF_V8SF_INT:
13081 1481 : case V8SF_FTYPE_V8SF_V4SF_INT:
13082 1481 : case V4SI_FTYPE_V4SI_V4SI_INT:
13083 1481 : case V4DF_FTYPE_V4DF_V4DF_INT:
13084 1481 : case V16SF_FTYPE_V16SF_V16SF_INT:
13085 1481 : case V16SF_FTYPE_V16SF_V4SF_INT:
13086 1481 : case V16SI_FTYPE_V16SI_V4SI_INT:
13087 1481 : case V4DF_FTYPE_V4DF_V2DF_INT:
13088 1481 : case V4SF_FTYPE_V4SF_V4SF_INT:
13089 1481 : case V2DI_FTYPE_V2DI_V2DI_INT:
13090 1481 : case V4DI_FTYPE_V4DI_V2DI_INT:
13091 1481 : case V2DF_FTYPE_V2DF_V2DF_INT:
13092 1481 : case UQI_FTYPE_V8DI_V8UDI_INT:
13093 1481 : case UQI_FTYPE_V8DF_V8DF_INT:
13094 1481 : case UQI_FTYPE_V2DF_V2DF_INT:
13095 1481 : case UQI_FTYPE_V4SF_V4SF_INT:
13096 1481 : case UHI_FTYPE_V16SI_V16SI_INT:
13097 1481 : case UHI_FTYPE_V16SF_V16SF_INT:
13098 1481 : case V64QI_FTYPE_V64QI_V64QI_INT:
13099 1481 : case V32HI_FTYPE_V32HI_V32HI_INT:
13100 1481 : case V16SI_FTYPE_V16SI_V16SI_INT:
13101 1481 : case V8DI_FTYPE_V8DI_V8DI_INT:
13102 1481 : nargs = 3;
13103 1481 : nargs_constant = 1;
13104 1481 : break;
13105 47 : case V4DI_FTYPE_V4DI_V4DI_INT_CONVERT:
13106 47 : nargs = 3;
13107 47 : rmode = V4DImode;
13108 47 : nargs_constant = 1;
13109 47 : break;
13110 80 : case V2DI_FTYPE_V2DI_V2DI_INT_CONVERT:
13111 80 : nargs = 3;
13112 80 : rmode = V2DImode;
13113 80 : nargs_constant = 1;
13114 80 : break;
13115 48 : case V1DI_FTYPE_V1DI_V1DI_INT_CONVERT:
13116 48 : nargs = 3;
13117 48 : rmode = DImode;
13118 48 : nargs_constant = 1;
13119 48 : break;
13120 20 : case V2DI_FTYPE_V2DI_UINT_UINT:
13121 20 : nargs = 3;
13122 20 : nargs_constant = 2;
13123 20 : break;
13124 8 : case V8DI_FTYPE_V8DI_V8DI_INT_CONVERT:
13125 8 : nargs = 3;
13126 8 : rmode = V8DImode;
13127 8 : nargs_constant = 1;
13128 8 : break;
13129 16 : case V8DI_FTYPE_V8DI_V8DI_INT_V8DI_UDI_CONVERT:
13130 16 : nargs = 5;
13131 16 : rmode = V8DImode;
13132 16 : mask_pos = 2;
13133 16 : nargs_constant = 1;
13134 16 : break;
13135 320 : case QI_FTYPE_V8DF_INT_UQI:
13136 320 : case QI_FTYPE_V4DF_INT_UQI:
13137 320 : case QI_FTYPE_V2DF_INT_UQI:
13138 320 : case HI_FTYPE_V16SF_INT_UHI:
13139 320 : case QI_FTYPE_V8SF_INT_UQI:
13140 320 : case QI_FTYPE_V4SF_INT_UQI:
13141 320 : case QI_FTYPE_V8HF_INT_UQI:
13142 320 : case HI_FTYPE_V16HF_INT_UHI:
13143 320 : case SI_FTYPE_V32HF_INT_USI:
13144 320 : case QI_FTYPE_V8BF_INT_UQI:
13145 320 : case HI_FTYPE_V16BF_INT_UHI:
13146 320 : case SI_FTYPE_V32BF_INT_USI:
13147 320 : case V4SI_FTYPE_V4SI_V4SI_UHI:
13148 320 : case V8SI_FTYPE_V8SI_V8SI_UHI:
13149 320 : nargs = 3;
13150 320 : mask_pos = 1;
13151 320 : nargs_constant = 1;
13152 320 : break;
13153 17 : case V4DI_FTYPE_V4DI_V4DI_INT_V4DI_USI_CONVERT:
13154 17 : nargs = 5;
13155 17 : rmode = V4DImode;
13156 17 : mask_pos = 2;
13157 17 : nargs_constant = 1;
13158 17 : break;
13159 17 : case V2DI_FTYPE_V2DI_V2DI_INT_V2DI_UHI_CONVERT:
13160 17 : nargs = 5;
13161 17 : rmode = V2DImode;
13162 17 : mask_pos = 2;
13163 17 : nargs_constant = 1;
13164 17 : break;
13165 17615 : case V32QI_FTYPE_V32QI_V32QI_V32QI_USI:
13166 17615 : case V32HI_FTYPE_V32HI_V32HI_V32HI_USI:
13167 17615 : case V32BF_FTYPE_V32BF_V32BF_V32BF_USI:
13168 17615 : case V32HI_FTYPE_V64QI_V64QI_V32HI_USI:
13169 17615 : case V16SI_FTYPE_V32HI_V32HI_V16SI_UHI:
13170 17615 : case V64QI_FTYPE_V64QI_V64QI_V64QI_UDI:
13171 17615 : case V32HI_FTYPE_V32HI_V8HI_V32HI_USI:
13172 17615 : case V16HI_FTYPE_V16HI_V8HI_V16HI_UHI:
13173 17615 : case V8SI_FTYPE_V8SI_V4SI_V8SI_UQI:
13174 17615 : case V4DI_FTYPE_V4DI_V2DI_V4DI_UQI:
13175 17615 : case V64QI_FTYPE_V32HI_V32HI_V64QI_UDI:
13176 17615 : case V32QI_FTYPE_V16HI_V16HI_V32QI_USI:
13177 17615 : case V16QI_FTYPE_V8HI_V8HI_V16QI_UHI:
13178 17615 : case V32HI_FTYPE_V16SI_V16SI_V32HI_USI:
13179 17615 : case V16HI_FTYPE_V8SI_V8SI_V16HI_UHI:
13180 17615 : case V8HI_FTYPE_V4SI_V4SI_V8HI_UQI:
13181 17615 : case V4DF_FTYPE_V4DF_V4DI_V4DF_UQI:
13182 17615 : case V32HF_FTYPE_V32HF_V32HF_V32HF_USI:
13183 17615 : case V8SF_FTYPE_V8SF_V8SI_V8SF_UQI:
13184 17615 : case V4SF_FTYPE_V4SF_V4SI_V4SF_UQI:
13185 17615 : case V2DF_FTYPE_V2DF_V2DI_V2DF_UQI:
13186 17615 : case V2DI_FTYPE_V4SI_V4SI_V2DI_UQI:
13187 17615 : case V4DI_FTYPE_V8SI_V8SI_V4DI_UQI:
13188 17615 : case V4DF_FTYPE_V4DI_V4DF_V4DF_UQI:
13189 17615 : case V8SF_FTYPE_V8SI_V8SF_V8SF_UQI:
13190 17615 : case V2DF_FTYPE_V2DI_V2DF_V2DF_UQI:
13191 17615 : case V4SF_FTYPE_V4SI_V4SF_V4SF_UQI:
13192 17615 : case V16SF_FTYPE_V16SF_V16SF_V16SF_UHI:
13193 17615 : case V16SF_FTYPE_V16SF_V16SI_V16SF_UHI:
13194 17615 : case V16SF_FTYPE_V16SI_V16SF_V16SF_UHI:
13195 17615 : case V16SI_FTYPE_V16SI_V16SI_V16SI_UHI:
13196 17615 : case V16SI_FTYPE_V16SI_V4SI_V16SI_UHI:
13197 17615 : case V8HI_FTYPE_V8HI_V8HI_V8HI_UQI:
13198 17615 : case V8BF_FTYPE_V8BF_V8BF_V8BF_UQI:
13199 17615 : case V8SI_FTYPE_V8SI_V8SI_V8SI_UQI:
13200 17615 : case V4SI_FTYPE_V4SI_V4SI_V4SI_UQI:
13201 17615 : case V16HF_FTYPE_V16HF_V16HF_V16HF_UQI:
13202 17615 : case V16HF_FTYPE_V16HF_V16HF_V16HF_UHI:
13203 17615 : case V8SF_FTYPE_V8SF_V8SF_V8SF_UQI:
13204 17615 : case V16QI_FTYPE_V16QI_V16QI_V16QI_UHI:
13205 17615 : case V16HI_FTYPE_V16HI_V16HI_V16HI_UHI:
13206 17615 : case V16BF_FTYPE_V16BF_V16BF_V16BF_UHI:
13207 17615 : case V2DI_FTYPE_V2DI_V2DI_V2DI_UQI:
13208 17615 : case V2DF_FTYPE_V2DF_V2DF_V2DF_UQI:
13209 17615 : case V4DI_FTYPE_V4DI_V4DI_V4DI_UQI:
13210 17615 : case V4DF_FTYPE_V4DF_V4DF_V4DF_UQI:
13211 17615 : case V8HF_FTYPE_V8HF_V8HF_V8HF_UQI:
13212 17615 : case V4SF_FTYPE_V4SF_V4SF_V4SF_UQI:
13213 17615 : case V8DF_FTYPE_V8DF_V8DF_V8DF_UQI:
13214 17615 : case V8DF_FTYPE_V8DF_V8DI_V8DF_UQI:
13215 17615 : case V8DF_FTYPE_V8DI_V8DF_V8DF_UQI:
13216 17615 : case V8DI_FTYPE_V16SI_V16SI_V8DI_UQI:
13217 17615 : case V8DI_FTYPE_V8DI_V2DI_V8DI_UQI:
13218 17615 : case V8DI_FTYPE_V8DI_V8DI_V8DI_UQI:
13219 17615 : case V8HI_FTYPE_V16QI_V16QI_V8HI_UQI:
13220 17615 : case V16HI_FTYPE_V32QI_V32QI_V16HI_UHI:
13221 17615 : case V8SI_FTYPE_V16HI_V16HI_V8SI_UQI:
13222 17615 : case V4SI_FTYPE_V8HI_V8HI_V4SI_UQI:
13223 17615 : case V32BF_FTYPE_V16SF_V16SF_V32BF_USI:
13224 17615 : case V16BF_FTYPE_V8SF_V8SF_V16BF_UHI:
13225 17615 : case V8BF_FTYPE_V4SF_V4SF_V8BF_UQI:
13226 17615 : case V32HF_FTYPE_V16SF_V16SF_V32HF_USI:
13227 17615 : case V16HF_FTYPE_V8SF_V8SF_V16HF_UHI:
13228 17615 : case V8HF_FTYPE_V4SF_V4SF_V8HF_UQI:
13229 17615 : case V16QI_FTYPE_V8HF_V8HF_V16QI_UHI:
13230 17615 : case V32QI_FTYPE_V16HF_V16HF_V32QI_USI:
13231 17615 : case V64QI_FTYPE_V32HF_V32HF_V64QI_UDI:
13232 17615 : case V16QI_FTYPE_V16QI_V8HF_V16QI_UHI:
13233 17615 : case V16QI_FTYPE_V32QI_V16HF_V16QI_UHI:
13234 17615 : case V32QI_FTYPE_V64QI_V32HF_V32QI_USI:
13235 17615 : case V16QI_FTYPE_V4SI_V4SF_V16QI_UQI:
13236 17615 : case V16QI_FTYPE_V8SI_V8SF_V16QI_UQI:
13237 17615 : case V16QI_FTYPE_V16SI_V16SF_V16QI_UHI:
13238 17615 : nargs = 4;
13239 17615 : break;
13240 11 : case V2DF_FTYPE_V2DF_V2DF_V2DI_INT:
13241 11 : case V4DF_FTYPE_V4DF_V4DF_V4DI_INT:
13242 11 : case V4SF_FTYPE_V4SF_V4SF_V4SI_INT:
13243 11 : case V8SF_FTYPE_V8SF_V8SF_V8SI_INT:
13244 11 : case V16SF_FTYPE_V16SF_V16SF_V16SI_INT:
13245 11 : case V4SI_FTYPE_V4SI_V4SI_V4SI_INT:
13246 11 : nargs = 4;
13247 11 : nargs_constant = 1;
13248 11 : break;
13249 3718 : case UQI_FTYPE_V4DI_V4DI_INT_UQI:
13250 3718 : case UQI_FTYPE_V8SI_V8SI_INT_UQI:
13251 3718 : case QI_FTYPE_V4DF_V4DF_INT_UQI:
13252 3718 : case QI_FTYPE_V8SF_V8SF_INT_UQI:
13253 3718 : case UHI_FTYPE_V16HF_V16HF_INT_UHI:
13254 3718 : case UQI_FTYPE_V2DI_V2DI_INT_UQI:
13255 3718 : case UQI_FTYPE_V4SI_V4SI_INT_UQI:
13256 3718 : case UQI_FTYPE_V2DF_V2DF_INT_UQI:
13257 3718 : case UQI_FTYPE_V4SF_V4SF_INT_UQI:
13258 3718 : case UQI_FTYPE_V8HF_V8HF_INT_UQI:
13259 3718 : case UDI_FTYPE_V64QI_V64QI_INT_UDI:
13260 3718 : case USI_FTYPE_V32QI_V32QI_INT_USI:
13261 3718 : case UHI_FTYPE_V16QI_V16QI_INT_UHI:
13262 3718 : case USI_FTYPE_V32HI_V32HI_INT_USI:
13263 3718 : case USI_FTYPE_V32BF_V32BF_INT_USI:
13264 3718 : case USI_FTYPE_V32HF_V32HF_INT_USI:
13265 3718 : case UHI_FTYPE_V16HI_V16HI_INT_UHI:
13266 3718 : case UHI_FTYPE_V16BF_V16BF_INT_UHI:
13267 3718 : case UQI_FTYPE_V8HI_V8HI_INT_UQI:
13268 3718 : case UQI_FTYPE_V8BF_V8BF_INT_UQI:
13269 3718 : nargs = 4;
13270 3718 : mask_pos = 1;
13271 3718 : nargs_constant = 1;
13272 3718 : break;
13273 23 : case V2DI_FTYPE_V2DI_V2DI_UINT_UINT:
13274 23 : nargs = 4;
13275 23 : nargs_constant = 2;
13276 23 : break;
13277 69 : case UCHAR_FTYPE_UCHAR_UINT_UINT_PUNSIGNED:
13278 69 : case UCHAR_FTYPE_UCHAR_ULONGLONG_ULONGLONG_PULONGLONG:
13279 69 : case V16SF_FTYPE_V16SF_V32BF_V32BF_UHI:
13280 69 : case V8SF_FTYPE_V8SF_V16BF_V16BF_UQI:
13281 69 : case V4SF_FTYPE_V4SF_V8BF_V8BF_UQI:
13282 69 : nargs = 4;
13283 69 : break;
13284 679 : case UQI_FTYPE_V8DI_V8DI_INT_UQI:
13285 679 : case UHI_FTYPE_V16SI_V16SI_INT_UHI:
13286 679 : mask_pos = 1;
13287 679 : nargs = 4;
13288 679 : nargs_constant = 1;
13289 679 : break;
13290 4056 : case V8SF_FTYPE_V8SF_INT_V8SF_UQI:
13291 4056 : case V4SF_FTYPE_V4SF_INT_V4SF_UQI:
13292 4056 : case V2DF_FTYPE_V4DF_INT_V2DF_UQI:
13293 4056 : case V2DI_FTYPE_V4DI_INT_V2DI_UQI:
13294 4056 : case V8SF_FTYPE_V16SF_INT_V8SF_UQI:
13295 4056 : case V8SI_FTYPE_V16SI_INT_V8SI_UQI:
13296 4056 : case V2DF_FTYPE_V8DF_INT_V2DF_UQI:
13297 4056 : case V2DI_FTYPE_V8DI_INT_V2DI_UQI:
13298 4056 : case V4SF_FTYPE_V8SF_INT_V4SF_UQI:
13299 4056 : case V4SI_FTYPE_V8SI_INT_V4SI_UQI:
13300 4056 : case V8HI_FTYPE_V8SF_INT_V8HI_UQI:
13301 4056 : case V8HI_FTYPE_V4SF_INT_V8HI_UQI:
13302 4056 : case V32HI_FTYPE_V32HI_INT_V32HI_USI:
13303 4056 : case V16HI_FTYPE_V16HI_INT_V16HI_UHI:
13304 4056 : case V8HI_FTYPE_V8HI_INT_V8HI_UQI:
13305 4056 : case V32BF_FTYPE_V32BF_INT_V32BF_USI:
13306 4056 : case V16BF_FTYPE_V16BF_INT_V16BF_UHI:
13307 4056 : case V8BF_FTYPE_V8BF_INT_V8BF_UQI:
13308 4056 : case V4DI_FTYPE_V4DI_INT_V4DI_UQI:
13309 4056 : case V2DI_FTYPE_V2DI_INT_V2DI_UQI:
13310 4056 : case V8SI_FTYPE_V8SI_INT_V8SI_UQI:
13311 4056 : case V4SI_FTYPE_V4SI_INT_V4SI_UQI:
13312 4056 : case V4DF_FTYPE_V4DF_INT_V4DF_UQI:
13313 4056 : case V2DF_FTYPE_V2DF_INT_V2DF_UQI:
13314 4056 : case V8DF_FTYPE_V8DF_INT_V8DF_UQI:
13315 4056 : case V16SF_FTYPE_V16SF_INT_V16SF_UHI:
13316 4056 : case V16HI_FTYPE_V16SF_INT_V16HI_UHI:
13317 4056 : case V16SI_FTYPE_V16SI_INT_V16SI_UHI:
13318 4056 : case V16HF_FTYPE_V16HF_INT_V16HF_UHI:
13319 4056 : case V8HF_FTYPE_V8HF_INT_V8HF_UQI:
13320 4056 : case V4SI_FTYPE_V16SI_INT_V4SI_UQI:
13321 4056 : case V4DI_FTYPE_V8DI_INT_V4DI_UQI:
13322 4056 : case V4DF_FTYPE_V8DF_INT_V4DF_UQI:
13323 4056 : case V4SF_FTYPE_V16SF_INT_V4SF_UQI:
13324 4056 : case V8DI_FTYPE_V8DI_INT_V8DI_UQI:
13325 4056 : case V16QI_FTYPE_V16QI_INT_V16QI_UHI:
13326 4056 : case V32QI_FTYPE_V32QI_INT_V32QI_USI:
13327 4056 : case V64QI_FTYPE_V64QI_INT_V64QI_UDI:
13328 4056 : nargs = 4;
13329 4056 : mask_pos = 2;
13330 4056 : nargs_constant = 1;
13331 4056 : break;
13332 1726 : case V16SF_FTYPE_V16SF_V4SF_INT_V16SF_UHI:
13333 1726 : case V16SI_FTYPE_V16SI_V4SI_INT_V16SI_UHI:
13334 1726 : case V8DF_FTYPE_V8DF_V8DF_INT_V8DF_UQI:
13335 1726 : case V8DI_FTYPE_V8DI_V8DI_INT_V8DI_UQI:
13336 1726 : case V16SF_FTYPE_V16SF_V16SF_INT_V16SF_UHI:
13337 1726 : case V16SI_FTYPE_V16SI_V16SI_INT_V16SI_UHI:
13338 1726 : case V4SF_FTYPE_V4SF_V4SF_INT_V4SF_UQI:
13339 1726 : case V2DF_FTYPE_V2DF_V2DF_INT_V2DF_UQI:
13340 1726 : case V8DF_FTYPE_V8DF_V4DF_INT_V8DF_UQI:
13341 1726 : case V8DI_FTYPE_V8DI_V4DI_INT_V8DI_UQI:
13342 1726 : case V4DF_FTYPE_V4DF_V4DF_INT_V4DF_UQI:
13343 1726 : case V8SF_FTYPE_V8SF_V8SF_INT_V8SF_UQI:
13344 1726 : case V8DF_FTYPE_V8DF_V2DF_INT_V8DF_UQI:
13345 1726 : case V8DI_FTYPE_V8DI_V2DI_INT_V8DI_UQI:
13346 1726 : case V8SI_FTYPE_V8SI_V8SI_INT_V8SI_UQI:
13347 1726 : case V4DI_FTYPE_V4DI_V4DI_INT_V4DI_UQI:
13348 1726 : case V4SI_FTYPE_V4SI_V4SI_INT_V4SI_UQI:
13349 1726 : case V2DI_FTYPE_V2DI_V2DI_INT_V2DI_UQI:
13350 1726 : case V32HI_FTYPE_V64QI_V64QI_INT_V32HI_USI:
13351 1726 : case V16HI_FTYPE_V32QI_V32QI_INT_V16HI_UHI:
13352 1726 : case V8HI_FTYPE_V16QI_V16QI_INT_V8HI_UQI:
13353 1726 : case V16SF_FTYPE_V16SF_V8SF_INT_V16SF_UHI:
13354 1726 : case V16SI_FTYPE_V16SI_V8SI_INT_V16SI_UHI:
13355 1726 : case V8SF_FTYPE_V8SF_V4SF_INT_V8SF_UQI:
13356 1726 : case V8SI_FTYPE_V8SI_V4SI_INT_V8SI_UQI:
13357 1726 : case V4DI_FTYPE_V4DI_V2DI_INT_V4DI_UQI:
13358 1726 : case V4DF_FTYPE_V4DF_V2DF_INT_V4DF_UQI:
13359 1726 : nargs = 5;
13360 1726 : mask_pos = 2;
13361 1726 : nargs_constant = 1;
13362 1726 : break;
13363 268 : case V8DI_FTYPE_V8DI_V8DI_V8DI_INT_UQI:
13364 268 : case V16SI_FTYPE_V16SI_V16SI_V16SI_INT_UHI:
13365 268 : case V2DF_FTYPE_V2DF_V2DF_V2DI_INT_UQI:
13366 268 : case V4SF_FTYPE_V4SF_V4SF_V4SI_INT_UQI:
13367 268 : case V8SF_FTYPE_V8SF_V8SF_V8SI_INT_UQI:
13368 268 : case V8SI_FTYPE_V8SI_V8SI_V8SI_INT_UQI:
13369 268 : case V4DF_FTYPE_V4DF_V4DF_V4DI_INT_UQI:
13370 268 : case V4DI_FTYPE_V4DI_V4DI_V4DI_INT_UQI:
13371 268 : case V4SI_FTYPE_V4SI_V4SI_V4SI_INT_UQI:
13372 268 : case V2DI_FTYPE_V2DI_V2DI_V2DI_INT_UQI:
13373 268 : nargs = 5;
13374 268 : mask_pos = 1;
13375 268 : nargs_constant = 1;
13376 268 : break;
13377 732 : case V64QI_FTYPE_V64QI_V64QI_INT_V64QI_UDI:
13378 732 : case V32QI_FTYPE_V32QI_V32QI_INT_V32QI_USI:
13379 732 : case V16QI_FTYPE_V16QI_V16QI_INT_V16QI_UHI:
13380 732 : case V32HI_FTYPE_V32HI_V32HI_INT_V32HI_INT:
13381 732 : case V16SI_FTYPE_V16SI_V16SI_INT_V16SI_INT:
13382 732 : case V8DI_FTYPE_V8DI_V8DI_INT_V8DI_INT:
13383 732 : case V16HI_FTYPE_V16HI_V16HI_INT_V16HI_INT:
13384 732 : case V8SI_FTYPE_V8SI_V8SI_INT_V8SI_INT:
13385 732 : case V4DI_FTYPE_V4DI_V4DI_INT_V4DI_INT:
13386 732 : case V8HI_FTYPE_V8HI_V8HI_INT_V8HI_INT:
13387 732 : case V4SI_FTYPE_V4SI_V4SI_INT_V4SI_INT:
13388 732 : case V2DI_FTYPE_V2DI_V2DI_INT_V2DI_INT:
13389 732 : case V8BF_FTYPE_V8BF_V8BF_INT_V8BF_UQI:
13390 732 : case V16BF_FTYPE_V16BF_V16BF_INT_V16BF_UHI:
13391 732 : case V32BF_FTYPE_V32BF_V32BF_INT_V32BF_USI:
13392 732 : case V16HF_FTYPE_V16HF_V16HF_INT_V16HF_UHI:
13393 732 : case V8HF_FTYPE_V8HF_V8HF_INT_V8HF_UQI:
13394 732 : nargs = 5;
13395 732 : mask_pos = 1;
13396 732 : nargs_constant = 2;
13397 732 : break;
13398 :
13399 0 : default:
13400 0 : gcc_unreachable ();
13401 : }
13402 :
13403 57842 : gcc_assert (nargs <= ARRAY_SIZE (xops));
13404 :
13405 65119 : if (comparison != UNKNOWN)
13406 : {
13407 608 : gcc_assert (nargs == 2);
13408 608 : return ix86_expand_sse_compare (d, exp, target, swap);
13409 : }
13410 :
13411 64511 : if (rmode == VOIDmode || rmode == tmode)
13412 : {
13413 64326 : if (optimize
13414 18360 : || target == 0
13415 18360 : || GET_MODE (target) != tmode
13416 82484 : || !insn_p->operand[0].predicate (target, tmode))
13417 46256 : target = gen_reg_rtx (tmode);
13418 18070 : else if (memory_operand (target, tmode))
13419 578 : num_memory++;
13420 : real_target = target;
13421 : }
13422 : else
13423 : {
13424 185 : real_target = gen_reg_rtx (tmode);
13425 185 : target = lowpart_subreg (rmode, real_target, tmode);
13426 : }
13427 :
13428 271829 : for (i = 0; i < nargs; i++)
13429 : {
13430 207560 : tree arg = CALL_EXPR_ARG (exp, i);
13431 207560 : rtx op = ix86_expand_unsigned_small_int_cst_argument (arg);
13432 207560 : machine_mode mode = insn_p->operand[i + 1].mode;
13433 : /* Need to fixup modeless constant before testing predicate. */
13434 207560 : op = fixup_modeless_constant (op, mode);
13435 207560 : bool match = insn_p->operand[i + 1].predicate (op, mode);
13436 :
13437 207560 : if (second_arg_count && i == 1)
13438 : {
13439 : /* SIMD shift insns take either an 8-bit immediate or
13440 : register as count. But builtin functions take int as
13441 : count. If count doesn't match, we put it in register.
13442 : The instructions are using 64-bit count, if op is just
13443 : 32-bit, zero-extend it, as negative shift counts
13444 : are undefined behavior and zero-extension is more
13445 : efficient. */
13446 2889 : if (!match)
13447 : {
13448 1750 : if (SCALAR_INT_MODE_P (GET_MODE (op)))
13449 489 : op = convert_modes (mode, GET_MODE (op), op, 1);
13450 : else
13451 1261 : op = lowpart_subreg (mode, op, GET_MODE (op));
13452 1750 : if (!insn_p->operand[i + 1].predicate (op, mode))
13453 190 : op = copy_to_reg (op);
13454 : }
13455 : }
13456 204671 : else if ((mask_pos && (nargs - i - mask_pos) == nargs_constant) ||
13457 156201 : (!mask_pos && (nargs - i) <= nargs_constant))
13458 : {
13459 16439 : if (!match)
13460 233 : switch (icode)
13461 : {
13462 2 : case CODE_FOR_avx_vinsertf128v4di:
13463 2 : case CODE_FOR_avx_vextractf128v4di:
13464 2 : error ("the last argument must be an 1-bit immediate");
13465 2 : return const0_rtx;
13466 :
13467 8 : case CODE_FOR_avx512f_cmpv8di3_mask:
13468 8 : case CODE_FOR_avx512f_cmpv16si3_mask:
13469 8 : case CODE_FOR_avx512f_ucmpv8di3_mask:
13470 8 : case CODE_FOR_avx512f_ucmpv16si3_mask:
13471 8 : case CODE_FOR_avx512vl_cmpv4di3_mask:
13472 8 : case CODE_FOR_avx512vl_cmpv8si3_mask:
13473 8 : case CODE_FOR_avx512vl_ucmpv4di3_mask:
13474 8 : case CODE_FOR_avx512vl_ucmpv8si3_mask:
13475 8 : case CODE_FOR_avx512vl_cmpv2di3_mask:
13476 8 : case CODE_FOR_avx512vl_cmpv4si3_mask:
13477 8 : case CODE_FOR_avx512vl_ucmpv2di3_mask:
13478 8 : case CODE_FOR_avx512vl_ucmpv4si3_mask:
13479 8 : error ("the last argument must be a 3-bit immediate");
13480 8 : return const0_rtx;
13481 :
13482 24 : case CODE_FOR_sse4_1_roundsd:
13483 24 : case CODE_FOR_sse4_1_roundss:
13484 :
13485 24 : case CODE_FOR_sse4_1_roundpd:
13486 24 : case CODE_FOR_sse4_1_roundps:
13487 24 : case CODE_FOR_avx_roundpd256:
13488 24 : case CODE_FOR_avx_roundps256:
13489 :
13490 24 : case CODE_FOR_sse4_1_roundpd_vec_pack_sfix:
13491 24 : case CODE_FOR_sse4_1_roundps_sfix:
13492 24 : case CODE_FOR_avx_roundpd_vec_pack_sfix256:
13493 24 : case CODE_FOR_avx_roundps_sfix256:
13494 :
13495 24 : case CODE_FOR_sse4_1_blendps:
13496 24 : case CODE_FOR_avx_blendpd256:
13497 24 : case CODE_FOR_avx_vpermilv4df:
13498 24 : case CODE_FOR_avx_vpermilv4df_mask:
13499 24 : case CODE_FOR_avx512f_getmantv8df_mask:
13500 24 : case CODE_FOR_avx512f_getmantv16sf_mask:
13501 24 : case CODE_FOR_avx512vl_getmantv16hf_mask:
13502 24 : case CODE_FOR_avx512vl_getmantv8sf_mask:
13503 24 : case CODE_FOR_avx512vl_getmantv4df_mask:
13504 24 : case CODE_FOR_avx512fp16_getmantv8hf_mask:
13505 24 : case CODE_FOR_avx512vl_getmantv4sf_mask:
13506 24 : case CODE_FOR_avx512vl_getmantv2df_mask:
13507 24 : case CODE_FOR_avx512dq_rangepv8df_mask_round:
13508 24 : case CODE_FOR_avx512dq_rangepv16sf_mask_round:
13509 24 : case CODE_FOR_avx512dq_rangepv4df_mask:
13510 24 : case CODE_FOR_avx512dq_rangepv8sf_mask:
13511 24 : case CODE_FOR_avx512dq_rangepv2df_mask:
13512 24 : case CODE_FOR_avx512dq_rangepv4sf_mask:
13513 24 : case CODE_FOR_avx_shufpd256_mask:
13514 24 : error ("the last argument must be a 4-bit immediate");
13515 24 : return const0_rtx;
13516 :
13517 15 : case CODE_FOR_sha1rnds4:
13518 15 : case CODE_FOR_sse4_1_blendpd:
13519 15 : case CODE_FOR_avx_vpermilv2df:
13520 15 : case CODE_FOR_avx_vpermilv2df_mask:
13521 15 : case CODE_FOR_xop_vpermil2v2df3:
13522 15 : case CODE_FOR_xop_vpermil2v4sf3:
13523 15 : case CODE_FOR_xop_vpermil2v4df3:
13524 15 : case CODE_FOR_xop_vpermil2v8sf3:
13525 15 : case CODE_FOR_avx512f_vinsertf32x4_mask:
13526 15 : case CODE_FOR_avx512f_vinserti32x4_mask:
13527 15 : case CODE_FOR_avx512f_vextractf32x4_mask:
13528 15 : case CODE_FOR_avx512f_vextracti32x4_mask:
13529 15 : case CODE_FOR_sse2_shufpd:
13530 15 : case CODE_FOR_sse2_shufpd_mask:
13531 15 : case CODE_FOR_avx512dq_shuf_f64x2_mask:
13532 15 : case CODE_FOR_avx512dq_shuf_i64x2_mask:
13533 15 : case CODE_FOR_avx512vl_shuf_i32x4_mask:
13534 15 : case CODE_FOR_avx512vl_shuf_f32x4_mask:
13535 15 : error ("the last argument must be a 2-bit immediate");
13536 15 : return const0_rtx;
13537 :
13538 30 : case CODE_FOR_avx_vextractf128v4df:
13539 30 : case CODE_FOR_avx_vextractf128v8sf:
13540 30 : case CODE_FOR_avx_vextractf128v8si:
13541 30 : case CODE_FOR_avx_vinsertf128v4df:
13542 30 : case CODE_FOR_avx_vinsertf128v8sf:
13543 30 : case CODE_FOR_avx_vinsertf128v8si:
13544 30 : case CODE_FOR_avx512f_vinsertf64x4_mask:
13545 30 : case CODE_FOR_avx512f_vinserti64x4_mask:
13546 30 : case CODE_FOR_avx512f_vextractf64x4_mask:
13547 30 : case CODE_FOR_avx512f_vextracti64x4_mask:
13548 30 : case CODE_FOR_avx512dq_vinsertf32x8_mask:
13549 30 : case CODE_FOR_avx512dq_vinserti32x8_mask:
13550 30 : case CODE_FOR_avx512vl_vinsertv4df:
13551 30 : case CODE_FOR_avx512vl_vinsertv4di:
13552 30 : case CODE_FOR_avx512vl_vinsertv8sf:
13553 30 : case CODE_FOR_avx512vl_vinsertv8si:
13554 30 : error ("the last argument must be a 1-bit immediate");
13555 30 : return const0_rtx;
13556 :
13557 16 : case CODE_FOR_avx_vmcmpv2df3:
13558 16 : case CODE_FOR_avx_vmcmpv4sf3:
13559 16 : case CODE_FOR_avx_cmpv2df3:
13560 16 : case CODE_FOR_avx_cmpv4sf3:
13561 16 : if (CONST_INT_P (op) && IN_RANGE (INTVAL (op), 8, 31))
13562 : {
13563 4 : error ("'%s' needs isa option %s", d->name, "-mavx");
13564 4 : return const0_rtx;
13565 : }
13566 : /* FALLTHRU */
13567 18 : case CODE_FOR_avx_cmpv4df3:
13568 18 : case CODE_FOR_avx_cmpv8sf3:
13569 18 : case CODE_FOR_avx512f_cmpv8df3_mask:
13570 18 : case CODE_FOR_avx512f_cmpv16sf3_mask:
13571 18 : case CODE_FOR_avx512f_vmcmpv2df3_mask:
13572 18 : case CODE_FOR_avx512f_vmcmpv4sf3_mask:
13573 18 : case CODE_FOR_avx512bw_cmpv32hf3_mask:
13574 18 : case CODE_FOR_avx512vl_cmpv16hf3_mask:
13575 18 : case CODE_FOR_avx512fp16_cmpv8hf3_mask:
13576 18 : error ("the last argument must be a 5-bit immediate");
13577 18 : return const0_rtx;
13578 :
13579 132 : default:
13580 132 : switch (nargs_constant)
13581 : {
13582 8 : case 2:
13583 8 : if ((mask_pos && (nargs - i - mask_pos) == nargs_constant) ||
13584 8 : (!mask_pos && (nargs - i) == nargs_constant))
13585 : {
13586 4 : error ("the next to last argument must be an 8-bit immediate");
13587 4 : break;
13588 : }
13589 : /* FALLTHRU */
13590 128 : case 1:
13591 128 : error ("the last argument must be an 8-bit immediate");
13592 128 : break;
13593 0 : default:
13594 0 : gcc_unreachable ();
13595 : }
13596 132 : return const0_rtx;
13597 : }
13598 16206 : if ((icode == CODE_FOR_vunpackbv16qi_mask
13599 16206 : || icode == CODE_FOR_vunpackbv32qi_mask
13600 : || icode == CODE_FOR_vunpackbv64qi_mask)
13601 108 : && CONST_INT_P (op))
13602 : {
13603 108 : char val = INTVAL (op);
13604 108 : if ((val & 0xc0)
13605 108 : || (!(val & 0x18))
13606 107 : || ((val & 0x02) && ((val & 0x1c) != 0x08))
13607 102 : || ((val & 0x01) && (((val & 0x1c) >> 2) > 0x4)))
13608 : {
13609 9 : error ("the last argument must not use reserved value "
13610 : "immediate");
13611 9 : return const0_rtx;
13612 : }
13613 : }
13614 : }
13615 : else
13616 : {
13617 188232 : if (VECTOR_MODE_P (mode))
13618 135972 : op = safe_vector_operand (op, mode);
13619 :
13620 : /* If we aren't optimizing, only allow one memory operand to
13621 : be generated. */
13622 188232 : if (memory_operand (op, mode))
13623 : {
13624 30581 : num_memory++;
13625 30581 : if (!optimize && num_memory > 1)
13626 13824 : op = copy_to_mode_reg (mode, op);
13627 : }
13628 :
13629 188232 : if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
13630 : {
13631 186100 : if (!match)
13632 43480 : op = copy_to_mode_reg (mode, op);
13633 : }
13634 : else
13635 : {
13636 2132 : op = copy_to_reg (op);
13637 2132 : op = lowpart_subreg (mode, op, GET_MODE (op));
13638 : }
13639 : }
13640 :
13641 207318 : xops[i] = op;
13642 : }
13643 :
13644 64269 : switch (nargs)
13645 : {
13646 6669 : case 1:
13647 6669 : pat = GEN_FCN (icode) (real_target, xops[0]);
13648 6669 : break;
13649 5527 : case 2:
13650 5527 : pat = GEN_FCN (icode) (real_target, xops[0], xops[1]);
13651 5527 : break;
13652 21817 : case 3:
13653 21817 : pat = GEN_FCN (icode) (real_target, xops[0], xops[1], xops[2]);
13654 21817 : break;
13655 27516 : case 4:
13656 27516 : pat = GEN_FCN (icode) (real_target, xops[0], xops[1],
13657 27516 : xops[2], xops[3]);
13658 27516 : break;
13659 2740 : case 5:
13660 2740 : pat = GEN_FCN (icode) (real_target, xops[0], xops[1],
13661 2740 : xops[2], xops[3], xops[4]);
13662 2740 : break;
13663 : case 6:
13664 : pat = GEN_FCN (icode) (real_target, xops[0], xops[1],
13665 : xops[2], xops[3], xops[4], xops[5]);
13666 : break;
13667 : default:
13668 : gcc_unreachable ();
13669 : }
13670 :
13671 64269 : if (! pat)
13672 : return 0;
13673 :
13674 64269 : emit_insn (pat);
13675 64269 : return target;
13676 : }
13677 :
13678 : /* Transform pattern of following layout:
13679 : (set A
13680 : (unspec [B C] UNSPEC_EMBEDDED_ROUNDING))
13681 : )
13682 : into:
13683 : (set (A B)) */
13684 :
13685 : static rtx
13686 4955 : ix86_erase_embedded_rounding (rtx pat)
13687 : {
13688 4955 : if (NONJUMP_INSN_P (pat))
13689 672 : pat = PATTERN (pat);
13690 :
13691 4955 : gcc_assert (GET_CODE (pat) == SET);
13692 4955 : rtx src = SET_SRC (pat);
13693 4955 : gcc_assert (XVECLEN (src, 0) == 2);
13694 4955 : rtx p0 = XVECEXP (src, 0, 0);
13695 4955 : gcc_assert (GET_CODE (src) == UNSPEC
13696 : && XINT (src, 1) == UNSPEC_EMBEDDED_ROUNDING);
13697 4955 : rtx res = gen_rtx_SET (SET_DEST (pat), p0);
13698 4955 : return res;
13699 : }
13700 :
13701 : /* Subroutine of ix86_expand_round_builtin to take care of comi insns
13702 : with rounding. */
13703 : static rtx
13704 103 : ix86_expand_sse_comi_round (const struct builtin_description *d,
13705 : tree exp, rtx target, bool comx_ok)
13706 : {
13707 103 : rtx pat, set_dst;
13708 103 : tree arg0 = CALL_EXPR_ARG (exp, 0);
13709 103 : tree arg1 = CALL_EXPR_ARG (exp, 1);
13710 103 : tree arg2 = CALL_EXPR_ARG (exp, 2);
13711 103 : tree arg3 = CALL_EXPR_ARG (exp, 3);
13712 103 : rtx op0 = expand_normal (arg0);
13713 103 : rtx op1 = expand_normal (arg1);
13714 103 : rtx op2 = expand_normal (arg2);
13715 103 : rtx op3 = expand_normal (arg3);
13716 103 : enum insn_code icode = d->icode;
13717 103 : const struct insn_data_d *insn_p = &insn_data[icode];
13718 103 : machine_mode mode0 = insn_p->operand[0].mode;
13719 103 : machine_mode mode1 = insn_p->operand[1].mode;
13720 :
13721 : /* See avxintrin.h for values. */
13722 103 : static const enum rtx_code comparisons[32] =
13723 : {
13724 : EQ, LT, LE, UNORDERED, NE, UNGE, UNGT, ORDERED,
13725 : UNEQ, UNLT, UNLE, UNORDERED, LTGT, GE, GT, ORDERED,
13726 : EQ, LT, LE, UNORDERED, NE, UNGE, UNGT, ORDERED,
13727 : UNEQ, UNLT, UNLE, UNORDERED, LTGT, GE, GT, ORDERED
13728 : };
13729 103 : static const bool ordereds[32] =
13730 : {
13731 : true, true, true, false, false, false, false, true,
13732 : false, false, false, true, true, true, true, false,
13733 : true, true, true, false, false, false, false, true,
13734 : false, false, false, true, true, true, true, false
13735 : };
13736 103 : static const bool non_signalings[32] =
13737 : {
13738 : true, false, false, true, true, false, false, true,
13739 : true, false, false, true, true, false, false, true,
13740 : false, true, true, false, false, true, true, false,
13741 : false, true, true, false, false, true, true, false
13742 : };
13743 :
13744 103 : if (!CONST_INT_P (op2))
13745 : {
13746 0 : error ("the third argument must be comparison constant");
13747 0 : return const0_rtx;
13748 : }
13749 103 : if (INTVAL (op2) < 0 || INTVAL (op2) >= 32)
13750 : {
13751 0 : error ("incorrect comparison mode");
13752 0 : return const0_rtx;
13753 : }
13754 :
13755 103 : if (!insn_p->operand[2].predicate (op3, SImode))
13756 : {
13757 4 : error ("incorrect rounding operand");
13758 4 : return const0_rtx;
13759 : }
13760 :
13761 99 : if (VECTOR_MODE_P (mode0))
13762 99 : op0 = safe_vector_operand (op0, mode0);
13763 99 : if (VECTOR_MODE_P (mode1))
13764 99 : op1 = safe_vector_operand (op1, mode1);
13765 :
13766 99 : enum rtx_code comparison = comparisons[INTVAL (op2)];
13767 99 : enum rtx_code orig_comp = comparison;
13768 99 : bool ordered = ordereds[INTVAL (op2)];
13769 99 : bool non_signaling = non_signalings[INTVAL (op2)];
13770 99 : rtx const_val = const0_rtx;
13771 :
13772 99 : bool check_unordered = false;
13773 99 : machine_mode mode = CCFPmode;
13774 99 : switch (comparison)
13775 : {
13776 8 : case ORDERED:
13777 8 : if (!ordered)
13778 : {
13779 4 : if (TARGET_AVX10_2 && comx_ok)
13780 : {
13781 : /* Unlike VCOMI{SH,SS,SD}, VCOMX{SH,SS,SD} will set SF
13782 : differently. So directly return true here. */
13783 0 : target = gen_reg_rtx (SImode);
13784 0 : emit_move_insn (target, const1_rtx);
13785 0 : return target;
13786 : }
13787 : else
13788 : {
13789 : /* NB: Use CCSmode/NE for _CMP_TRUE_UQ/_CMP_TRUE_US. */
13790 : if (!non_signaling)
13791 99 : ordered = true;
13792 99 : mode = CCSmode;
13793 : }
13794 : }
13795 : else
13796 : {
13797 : /* NB: Use CCPmode/NE for _CMP_ORD_Q/_CMP_ORD_S. */
13798 : if (non_signaling)
13799 99 : ordered = false;
13800 : mode = CCPmode;
13801 : }
13802 : comparison = NE;
13803 : break;
13804 8 : case UNORDERED:
13805 8 : if (ordered)
13806 : {
13807 4 : if (TARGET_AVX10_2 && comx_ok)
13808 : {
13809 : /* Unlike VCOMI{SH,SS,SD}, VCOMX{SH,SS,SD} will set SF
13810 : differently. So directly return false here. */
13811 0 : target = gen_reg_rtx (SImode);
13812 0 : emit_move_insn (target, const0_rtx);
13813 0 : return target;
13814 : }
13815 : else
13816 : {
13817 : /* NB: Use CCSmode/EQ for _CMP_FALSE_OQ/_CMP_FALSE_OS. */
13818 : if (non_signaling)
13819 : ordered = false;
13820 : mode = CCSmode;
13821 : }
13822 : }
13823 : else
13824 : {
13825 : /* NB: Use CCPmode/NE for _CMP_UNORD_Q/_CMP_UNORD_S. */
13826 : if (!non_signaling)
13827 99 : ordered = true;
13828 99 : mode = CCPmode;
13829 : }
13830 : comparison = EQ;
13831 : break;
13832 :
13833 40 : case LE: /* -> GE */
13834 40 : case LT: /* -> GT */
13835 40 : case UNGE: /* -> UNLE */
13836 40 : case UNGT: /* -> UNLT */
13837 40 : std::swap (op0, op1);
13838 40 : comparison = swap_condition (comparison);
13839 : /* FALLTHRU */
13840 : case GT:
13841 : case GE:
13842 : case UNEQ:
13843 : case UNLT:
13844 : case UNLE:
13845 : case LTGT:
13846 : /* These are supported by CCFPmode. NB: Use ordered/signaling
13847 : COMI or unordered/non-signaling UCOMI. Both set ZF, PF, CF
13848 : with NAN operands. */
13849 : if (ordered == non_signaling)
13850 : ordered = !ordered;
13851 : break;
13852 : /* NB: COMI/UCOMI will set ZF with NAN operands. Use CCZmode for
13853 : _CMP_EQ_OQ/_CMP_EQ_OS.
13854 : Under TARGET_AVX10_2, VCOMX/VUCOMX are always generated instead
13855 : of COMI/UCOMI, VCOMX/VUCOMX will not set ZF with NAN. */
13856 8 : case EQ:
13857 8 : if (!TARGET_AVX10_2 || !comx_ok)
13858 5 : check_unordered = true;
13859 : mode = CCZmode;
13860 : break;
13861 7 : case NE:
13862 : /* NB: COMI/UCOMI will set ZF with NAN operands. Use CCZmode for
13863 : _CMP_NEQ_UQ/_CMP_NEQ_US.
13864 : Under TARGET_AVX10_2, VCOMX/VUCOMX are always generated instead
13865 : of COMI/UCOMI, VCOMX/VUCOMX will not set ZF with NAN. */
13866 7 : gcc_assert (!ordered);
13867 7 : if (!TARGET_AVX10_2 || !comx_ok)
13868 4 : check_unordered = true;
13869 7 : mode = CCZmode;
13870 7 : const_val = const1_rtx;
13871 7 : break;
13872 : default:
13873 : gcc_unreachable ();
13874 : }
13875 :
13876 99 : target = gen_reg_rtx (SImode);
13877 99 : emit_move_insn (target, const_val);
13878 99 : target = gen_rtx_SUBREG (QImode, target, 0);
13879 :
13880 93 : if ((optimize && !register_operand (op0, mode0))
13881 192 : || !insn_p->operand[0].predicate (op0, mode0))
13882 6 : op0 = copy_to_mode_reg (mode0, op0);
13883 93 : if ((optimize && !register_operand (op1, mode1))
13884 192 : || !insn_p->operand[1].predicate (op1, mode1))
13885 6 : op1 = copy_to_mode_reg (mode1, op1);
13886 :
13887 : /* Generate comx instead of comi when EQ/NE to avoid NAN checks.
13888 : Use orig_comp to exclude ORDERED/UNORDERED cases. */
13889 99 : if ((orig_comp == EQ || orig_comp == NE)
13890 15 : && TARGET_AVX10_2 && comx_ok)
13891 : {
13892 6 : switch (icode)
13893 : {
13894 : case CODE_FOR_avx512fp16_comi_round:
13895 99 : icode = CODE_FOR_avx10_2_comxhf_round;
13896 : break;
13897 4 : case CODE_FOR_sse_comi_round:
13898 4 : icode = CODE_FOR_avx10_2_comxsf_round;
13899 4 : break;
13900 2 : case CODE_FOR_sse2_comi_round:
13901 2 : icode = CODE_FOR_avx10_2_comxdf_round;
13902 2 : break;
13903 :
13904 : default:
13905 : break;
13906 : }
13907 : }
13908 :
13909 : /* Generate comi instead of comx when UNEQ/LTGT to avoid NAN checks. */
13910 99 : if ((comparison == UNEQ || comparison == LTGT)
13911 8 : && TARGET_AVX10_2 && comx_ok)
13912 : {
13913 0 : switch (icode)
13914 : {
13915 : case CODE_FOR_avx10_2_comxhf_round:
13916 99 : icode = CODE_FOR_avx512fp16_comi_round;
13917 : break;
13918 0 : case CODE_FOR_avx10_2_comxsf_round:
13919 0 : icode = CODE_FOR_sse_comi_round;
13920 0 : break;
13921 0 : case CODE_FOR_avx10_2_comxdf_round:
13922 0 : icode = CODE_FOR_sse2_comi_round;
13923 0 : break;
13924 :
13925 : default:
13926 : break;
13927 : }
13928 : }
13929 :
13930 : /*
13931 : 1. COMI/VCOMX: ordered and signaling.
13932 : 2. UCOMI/VUCOMX: unordered and non-signaling.
13933 : */
13934 99 : if (non_signaling)
13935 38 : switch (icode)
13936 : {
13937 : case CODE_FOR_sse_comi_round:
13938 : icode = CODE_FOR_sse_ucomi_round;
13939 : break;
13940 17 : case CODE_FOR_sse2_comi_round:
13941 17 : icode = CODE_FOR_sse2_ucomi_round;
13942 17 : break;
13943 0 : case CODE_FOR_avx512fp16_comi_round:
13944 0 : icode = CODE_FOR_avx512fp16_ucomi_round;
13945 0 : break;
13946 3 : case CODE_FOR_avx10_2_comxsf_round:
13947 3 : icode = CODE_FOR_avx10_2_ucomxsf_round;
13948 3 : break;
13949 0 : case CODE_FOR_avx10_2_comxhf_round:
13950 0 : icode = CODE_FOR_avx10_2_ucomxhf_round;
13951 0 : break;
13952 1 : case CODE_FOR_avx10_2_comxdf_round:
13953 1 : icode = CODE_FOR_avx10_2_ucomxdf_round;
13954 1 : break;
13955 0 : default:
13956 0 : gcc_unreachable ();
13957 : }
13958 :
13959 99 : pat = GEN_FCN (icode) (op0, op1, op3);
13960 99 : if (! pat)
13961 : return 0;
13962 :
13963 : /* Rounding operand can be either NO_ROUND or ROUND_SAE at this point. */
13964 99 : if (INTVAL (op3) == NO_ROUND)
13965 : {
13966 1 : pat = ix86_erase_embedded_rounding (pat);
13967 1 : if (! pat)
13968 : return 0;
13969 :
13970 1 : set_dst = SET_DEST (pat);
13971 : }
13972 : else
13973 : {
13974 98 : gcc_assert (GET_CODE (pat) == SET);
13975 98 : set_dst = SET_DEST (pat);
13976 : }
13977 :
13978 99 : emit_insn (pat);
13979 :
13980 99 : return ix86_ssecom_setcc (comparison, check_unordered, mode,
13981 99 : set_dst, target);
13982 : }
13983 :
13984 : static rtx
13985 15637 : ix86_expand_round_builtin (const struct builtin_description *d,
13986 : tree exp, rtx target)
13987 : {
13988 15637 : rtx pat;
13989 15637 : unsigned int i, nargs;
13990 15637 : rtx xops[6];
13991 15637 : enum insn_code icode = d->icode;
13992 15637 : const struct insn_data_d *insn_p = &insn_data[icode];
13993 15637 : machine_mode tmode = insn_p->operand[0].mode;
13994 15637 : unsigned int nargs_constant = 0;
13995 15637 : unsigned int redundant_embed_rnd = 0;
13996 :
13997 15637 : switch ((enum ix86_builtin_func_type) d->flag)
13998 : {
13999 : case UINT64_FTYPE_V2DF_INT:
14000 : case UINT64_FTYPE_V4SF_INT:
14001 : case UINT64_FTYPE_V8HF_INT:
14002 : case UINT_FTYPE_V2DF_INT:
14003 : case UINT_FTYPE_V4SF_INT:
14004 : case UINT_FTYPE_V8HF_INT:
14005 : case INT64_FTYPE_V2DF_INT:
14006 : case INT64_FTYPE_V4SF_INT:
14007 : case INT64_FTYPE_V8HF_INT:
14008 : case INT_FTYPE_V2DF_INT:
14009 : case INT_FTYPE_V4SF_INT:
14010 : case INT_FTYPE_V8HF_INT:
14011 : nargs = 2;
14012 : break;
14013 629 : case V32HF_FTYPE_V32HF_V32HF_INT:
14014 629 : case V8HF_FTYPE_V8HF_V8HF_INT:
14015 629 : case V8HF_FTYPE_V8HF_INT_INT:
14016 629 : case V8HF_FTYPE_V8HF_UINT_INT:
14017 629 : case V8HF_FTYPE_V8HF_INT64_INT:
14018 629 : case V8HF_FTYPE_V8HF_UINT64_INT:
14019 629 : case V4SF_FTYPE_V4SF_UINT_INT:
14020 629 : case V4SF_FTYPE_V4SF_UINT64_INT:
14021 629 : case V2DF_FTYPE_V2DF_UINT64_INT:
14022 629 : case V4SF_FTYPE_V4SF_INT_INT:
14023 629 : case V4SF_FTYPE_V4SF_INT64_INT:
14024 629 : case V2DF_FTYPE_V2DF_INT64_INT:
14025 629 : case V4SF_FTYPE_V4SF_V4SF_INT:
14026 629 : case V2DF_FTYPE_V2DF_V2DF_INT:
14027 629 : case V4SF_FTYPE_V4SF_V2DF_INT:
14028 629 : case V2DF_FTYPE_V2DF_V4SF_INT:
14029 629 : nargs = 3;
14030 629 : break;
14031 4554 : case V8SF_FTYPE_V8DF_V8SF_QI_INT:
14032 4554 : case V8DF_FTYPE_V8DF_V8DF_QI_INT:
14033 4554 : case V32HI_FTYPE_V32HF_V32HI_USI_INT:
14034 4554 : case V32HI_FTYPE_V32BF_V32HI_USI_INT:
14035 4554 : case V8SI_FTYPE_V8DF_V8SI_QI_INT:
14036 4554 : case V8DI_FTYPE_V8HF_V8DI_UQI_INT:
14037 4554 : case V8DI_FTYPE_V8DF_V8DI_QI_INT:
14038 4554 : case V8SF_FTYPE_V8DI_V8SF_QI_INT:
14039 4554 : case V8DF_FTYPE_V8DI_V8DF_QI_INT:
14040 4554 : case V8DF_FTYPE_V8HF_V8DF_UQI_INT:
14041 4554 : case V16SF_FTYPE_V16HF_V16SF_UHI_INT:
14042 4554 : case V32HF_FTYPE_V32HI_V32HF_USI_INT:
14043 4554 : case V32HF_FTYPE_V32HF_V32HF_USI_INT:
14044 4554 : case V32HF_FTYPE_V32HF_V32HF_V32HF_INT:
14045 4554 : case V16SF_FTYPE_V16SF_V16SF_HI_INT:
14046 4554 : case V8DI_FTYPE_V8SF_V8DI_QI_INT:
14047 4554 : case V16SF_FTYPE_V16SI_V16SF_HI_INT:
14048 4554 : case V16SI_FTYPE_V16SF_V16SI_HI_INT:
14049 4554 : case V16SI_FTYPE_V16SF_V16SI_UHI_INT:
14050 4554 : case V16SI_FTYPE_V16HF_V16SI_UHI_INT:
14051 4554 : case V16HF_FTYPE_V16SI_V16HF_UHI_INT:
14052 4554 : case V8DF_FTYPE_V8SF_V8DF_QI_INT:
14053 4554 : case V16SF_FTYPE_V16HI_V16SF_HI_INT:
14054 4554 : case V2DF_FTYPE_V2DF_V2DF_V2DF_INT:
14055 4554 : case V4SF_FTYPE_V4SF_V4SF_V4SF_INT:
14056 4554 : case V8HF_FTYPE_V8DI_V8HF_UQI_INT:
14057 4554 : case V8HF_FTYPE_V8DF_V8HF_UQI_INT:
14058 4554 : case V16HF_FTYPE_V16SF_V16HF_UHI_INT:
14059 4554 : case V16HI_FTYPE_V16BF_V16HI_UHI_INT:
14060 4554 : case V8HF_FTYPE_V8HF_V8HF_V8HF_INT:
14061 4554 : nargs = 4;
14062 4554 : break;
14063 158 : case V4SF_FTYPE_V4SF_V4SF_INT_INT:
14064 158 : case V2DF_FTYPE_V2DF_V2DF_INT_INT:
14065 158 : nargs_constant = 2;
14066 158 : nargs = 4;
14067 158 : break;
14068 103 : case INT_FTYPE_V4SF_V4SF_INT_INT:
14069 103 : case INT_FTYPE_V2DF_V2DF_INT_INT:
14070 103 : return ix86_expand_sse_comi_round (d, exp, target, true);
14071 6211 : case V8DF_FTYPE_V8DF_V8DF_V8DF_UQI_INT:
14072 6211 : case V2DF_FTYPE_V2DF_V2DF_V2DF_UQI_INT:
14073 6211 : case V4SF_FTYPE_V4SF_V4SF_V4SF_UQI_INT:
14074 6211 : case V4SF_FTYPE_V8HF_V4SF_V4SF_UQI_INT:
14075 6211 : case V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT:
14076 6211 : case V32HF_FTYPE_V32HF_V32HF_V32HF_UHI_INT:
14077 6211 : case V32HF_FTYPE_V32HF_V32HF_V32HF_USI_INT:
14078 6211 : case V2DF_FTYPE_V8HF_V2DF_V2DF_UQI_INT:
14079 6211 : case V2DF_FTYPE_V2DF_V2DF_V2DF_QI_INT:
14080 6211 : case V2DF_FTYPE_V2DF_V4SF_V2DF_QI_INT:
14081 6211 : case V2DF_FTYPE_V2DF_V4SF_V2DF_UQI_INT:
14082 6211 : case V4SF_FTYPE_V4SF_V4SF_V4SF_QI_INT:
14083 6211 : case V4SF_FTYPE_V4SF_V2DF_V4SF_QI_INT:
14084 6211 : case V4SF_FTYPE_V4SF_V2DF_V4SF_UQI_INT:
14085 6211 : case V8HF_FTYPE_V8HF_V8HF_V8HF_UQI_INT:
14086 6211 : case V8HF_FTYPE_V2DF_V8HF_V8HF_UQI_INT:
14087 6211 : case V8HF_FTYPE_V4SF_V8HF_V8HF_UQI_INT:
14088 6211 : case V32HF_FTYPE_V16SF_V16SF_V32HF_USI_INT:
14089 6211 : nargs = 5;
14090 6211 : break;
14091 635 : case V32HF_FTYPE_V32HF_INT_V32HF_USI_INT:
14092 635 : case V16SF_FTYPE_V16SF_INT_V16SF_HI_INT:
14093 635 : case V8DF_FTYPE_V8DF_INT_V8DF_QI_INT:
14094 635 : case V8DF_FTYPE_V8DF_INT_V8DF_UQI_INT:
14095 635 : case V16SF_FTYPE_V16SF_INT_V16SF_UHI_INT:
14096 635 : nargs_constant = 4;
14097 635 : nargs = 5;
14098 635 : break;
14099 1181 : case UQI_FTYPE_V8DF_V8DF_INT_UQI_INT:
14100 1181 : case UQI_FTYPE_V2DF_V2DF_INT_UQI_INT:
14101 1181 : case UHI_FTYPE_V16SF_V16SF_INT_UHI_INT:
14102 1181 : case UQI_FTYPE_V4SF_V4SF_INT_UQI_INT:
14103 1181 : case USI_FTYPE_V32HF_V32HF_INT_USI_INT:
14104 1181 : case UQI_FTYPE_V8HF_V8HF_INT_UQI_INT:
14105 1181 : nargs_constant = 3;
14106 1181 : nargs = 5;
14107 1181 : break;
14108 1071 : case V16SF_FTYPE_V16SF_V16SF_INT_V16SF_HI_INT:
14109 1071 : case V8DF_FTYPE_V8DF_V8DF_INT_V8DF_QI_INT:
14110 1071 : case V4SF_FTYPE_V4SF_V4SF_INT_V4SF_QI_INT:
14111 1071 : case V2DF_FTYPE_V2DF_V2DF_INT_V2DF_QI_INT:
14112 1071 : case V2DF_FTYPE_V2DF_V2DF_INT_V2DF_UQI_INT:
14113 1071 : case V4SF_FTYPE_V4SF_V4SF_INT_V4SF_UQI_INT:
14114 1071 : case V8HF_FTYPE_V8HF_V8HF_INT_V8HF_UQI_INT:
14115 1071 : case V8DF_FTYPE_V8DF_V8DF_INT_V8DF_UQI_INT:
14116 1071 : case V32HF_FTYPE_V32HF_V32HF_INT_V32HF_USI_INT:
14117 1071 : case V16SF_FTYPE_V16SF_V16SF_INT_V16SF_UHI_INT:
14118 1071 : nargs = 6;
14119 1071 : nargs_constant = 4;
14120 1071 : break;
14121 252 : case V8DF_FTYPE_V8DF_V8DF_V8DI_INT_QI_INT:
14122 252 : case V16SF_FTYPE_V16SF_V16SF_V16SI_INT_HI_INT:
14123 252 : case V2DF_FTYPE_V2DF_V2DF_V2DI_INT_QI_INT:
14124 252 : case V4SF_FTYPE_V4SF_V4SF_V4SI_INT_QI_INT:
14125 252 : nargs = 6;
14126 252 : nargs_constant = 3;
14127 252 : break;
14128 0 : default:
14129 0 : gcc_unreachable ();
14130 : }
14131 14691 : gcc_assert (nargs <= ARRAY_SIZE (xops));
14132 :
14133 15534 : if (optimize
14134 4293 : || target == 0
14135 4293 : || GET_MODE (target) != tmode
14136 19827 : || !insn_p->operand[0].predicate (target, tmode))
14137 11241 : target = gen_reg_rtx (tmode);
14138 :
14139 85377 : for (i = 0; i < nargs; i++)
14140 : {
14141 70398 : tree arg = CALL_EXPR_ARG (exp, i);
14142 70398 : rtx op = ix86_expand_unsigned_small_int_cst_argument (arg);
14143 70398 : machine_mode mode = insn_p->operand[i + 1].mode;
14144 70398 : bool match = insn_p->operand[i + 1].predicate (op, mode);
14145 :
14146 70398 : if (i == nargs - nargs_constant)
14147 : {
14148 3297 : if (!match)
14149 : {
14150 40 : switch (icode)
14151 : {
14152 12 : case CODE_FOR_avx512f_getmantv8df_mask_round:
14153 12 : case CODE_FOR_avx512f_getmantv16sf_mask_round:
14154 12 : case CODE_FOR_avx512bw_getmantv32hf_mask_round:
14155 12 : case CODE_FOR_avx512f_vgetmantv2df_round:
14156 12 : case CODE_FOR_avx512f_vgetmantv2df_mask_round:
14157 12 : case CODE_FOR_avx512f_vgetmantv4sf_round:
14158 12 : case CODE_FOR_avx512f_vgetmantv4sf_mask_round:
14159 12 : case CODE_FOR_avx512f_vgetmantv8hf_mask_round:
14160 12 : error ("the immediate argument must be a 4-bit immediate");
14161 12 : return const0_rtx;
14162 8 : case CODE_FOR_avx512f_cmpv8df3_mask_round:
14163 8 : case CODE_FOR_avx512f_cmpv16sf3_mask_round:
14164 8 : case CODE_FOR_avx512f_vmcmpv2df3_mask_round:
14165 8 : case CODE_FOR_avx512f_vmcmpv4sf3_mask_round:
14166 8 : case CODE_FOR_avx512f_vmcmpv8hf3_mask_round:
14167 8 : case CODE_FOR_avx512bw_cmpv32hf3_mask_round:
14168 8 : error ("the immediate argument must be a 5-bit immediate");
14169 8 : return const0_rtx;
14170 20 : default:
14171 20 : error ("the immediate argument must be an 8-bit immediate");
14172 20 : return const0_rtx;
14173 : }
14174 : }
14175 : }
14176 67101 : else if (i == nargs-1)
14177 : {
14178 15494 : if (!insn_p->operand[nargs].predicate (op, SImode))
14179 : {
14180 515 : error ("incorrect rounding operand");
14181 515 : return const0_rtx;
14182 : }
14183 :
14184 : /* If there is no rounding use normal version of the pattern. */
14185 14979 : if (INTVAL (op) == NO_ROUND)
14186 : {
14187 : /* Skip erasing embedded rounding for below expanders who
14188 : generates multiple insns. In ix86_erase_embedded_rounding
14189 : the pattern will be transformed to a single set, and emit_insn
14190 : appends the set instead of insert it to chain. So the insns
14191 : emitted inside define_expander would be ignored. */
14192 4986 : switch (icode)
14193 : {
14194 : case CODE_FOR_avx512bw_fmaddc_v32hf_mask1_round:
14195 : case CODE_FOR_avx512bw_fcmaddc_v32hf_mask1_round:
14196 : case CODE_FOR_avx512fp16_fmaddcsh_v8hf_mask1_round:
14197 : case CODE_FOR_avx512fp16_fcmaddcsh_v8hf_mask1_round:
14198 : case CODE_FOR_avx512fp16_fmaddcsh_v8hf_mask3_round:
14199 : case CODE_FOR_avx512fp16_fcmaddcsh_v8hf_mask3_round:
14200 : redundant_embed_rnd = 0;
14201 : break;
14202 4954 : default:
14203 4954 : redundant_embed_rnd = 1;
14204 4954 : break;
14205 : }
14206 : }
14207 : }
14208 : else
14209 : {
14210 51607 : if (VECTOR_MODE_P (mode))
14211 37712 : op = safe_vector_operand (op, mode);
14212 :
14213 51607 : op = fixup_modeless_constant (op, mode);
14214 :
14215 51607 : if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
14216 : {
14217 51607 : if (optimize || !match)
14218 45279 : op = copy_to_mode_reg (mode, op);
14219 : }
14220 : else
14221 : {
14222 0 : op = copy_to_reg (op);
14223 0 : op = lowpart_subreg (mode, op, GET_MODE (op));
14224 : }
14225 : }
14226 :
14227 69843 : xops[i] = op;
14228 : }
14229 :
14230 14979 : switch (nargs)
14231 : {
14232 : case 1:
14233 : pat = GEN_FCN (icode) (target, xops[0]);
14234 : break;
14235 810 : case 2:
14236 810 : pat = GEN_FCN (icode) (target, xops[0], xops[1]);
14237 810 : break;
14238 585 : case 3:
14239 585 : pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2]);
14240 585 : break;
14241 4588 : case 4:
14242 4588 : pat = GEN_FCN (icode) (target, xops[0], xops[1],
14243 4588 : xops[2], xops[3]);
14244 4588 : break;
14245 7723 : case 5:
14246 7723 : pat = GEN_FCN (icode) (target, xops[0], xops[1],
14247 7723 : xops[2], xops[3], xops[4]);
14248 7723 : break;
14249 1273 : case 6:
14250 1273 : pat = GEN_FCN (icode) (target, xops[0], xops[1],
14251 1273 : xops[2], xops[3], xops[4], xops[5]);
14252 1273 : break;
14253 : default:
14254 : gcc_unreachable ();
14255 : }
14256 :
14257 14979 : if (!pat)
14258 : return 0;
14259 :
14260 14979 : if (redundant_embed_rnd)
14261 4954 : pat = ix86_erase_embedded_rounding (pat);
14262 :
14263 14979 : emit_insn (pat);
14264 14979 : return target;
14265 : }
14266 :
14267 : /* Subroutine of ix86_expand_builtin to take care of special insns
14268 : with variable number of operands. */
14269 :
14270 : static rtx
14271 27131 : ix86_expand_special_args_builtin (const struct builtin_description *d,
14272 : tree exp, rtx target)
14273 : {
14274 27131 : tree arg;
14275 27131 : rtx pat, op;
14276 27131 : unsigned int i, nargs, arg_adjust, memory;
14277 27131 : unsigned int constant = 100;
14278 27131 : bool aligned_mem = false;
14279 27131 : rtx xops[4];
14280 27131 : enum insn_code icode = d->icode;
14281 27131 : const struct insn_data_d *insn_p = &insn_data[icode];
14282 27131 : machine_mode tmode = insn_p->operand[0].mode;
14283 27131 : enum { load, store } klass;
14284 :
14285 27131 : switch ((enum ix86_builtin_func_type) d->flag)
14286 : {
14287 15357 : case VOID_FTYPE_VOID:
14288 15357 : emit_insn (GEN_FCN (icode) (target));
14289 15357 : return 0;
14290 : case VOID_FTYPE_UINT64:
14291 : case VOID_FTYPE_UNSIGNED:
14292 : nargs = 0;
14293 : klass = store;
14294 : memory = 0;
14295 : break;
14296 :
14297 7508 : case INT_FTYPE_VOID:
14298 7508 : case USHORT_FTYPE_VOID:
14299 7508 : case UINT64_FTYPE_VOID:
14300 7508 : case UINT_FTYPE_VOID:
14301 7508 : case UINT8_FTYPE_VOID:
14302 7508 : case UNSIGNED_FTYPE_VOID:
14303 7508 : nargs = 0;
14304 7508 : klass = load;
14305 7508 : memory = 0;
14306 7508 : break;
14307 360 : case CHAR_FTYPE_PCCHAR:
14308 360 : case SHORT_FTYPE_PCSHORT:
14309 360 : case INT_FTYPE_PCINT:
14310 360 : case INT64_FTYPE_PCINT64:
14311 360 : case UINT64_FTYPE_PUNSIGNED:
14312 360 : case V2DI_FTYPE_PV2DI:
14313 360 : case V4DI_FTYPE_PV4DI:
14314 360 : case V32QI_FTYPE_PCCHAR:
14315 360 : case V16QI_FTYPE_PCCHAR:
14316 360 : case V8SF_FTYPE_PCV4SF:
14317 360 : case V8SF_FTYPE_PCFLOAT:
14318 360 : case V4SF_FTYPE_PCFLOAT:
14319 360 : case V4SF_FTYPE_PCFLOAT16:
14320 360 : case V4SF_FTYPE_PCBFLOAT16:
14321 360 : case V4SF_FTYPE_PCV8BF:
14322 360 : case V4SF_FTYPE_PCV8HF:
14323 360 : case V8SF_FTYPE_PCFLOAT16:
14324 360 : case V8SF_FTYPE_PCBFLOAT16:
14325 360 : case V8SF_FTYPE_PCV16HF:
14326 360 : case V8SF_FTYPE_PCV16BF:
14327 360 : case V4DF_FTYPE_PCV2DF:
14328 360 : case V4DF_FTYPE_PCDOUBLE:
14329 360 : case V2DF_FTYPE_PCDOUBLE:
14330 360 : case VOID_FTYPE_PVOID:
14331 360 : case V8DI_FTYPE_PV8DI:
14332 360 : nargs = 1;
14333 360 : klass = load;
14334 360 : memory = 0;
14335 360 : switch (icode)
14336 : {
14337 : case CODE_FOR_sse4_1_movntdqa:
14338 : case CODE_FOR_avx2_movntdqa:
14339 : case CODE_FOR_avx512f_movntdqa:
14340 : aligned_mem = true;
14341 : break;
14342 : default:
14343 : break;
14344 : }
14345 : break;
14346 371 : case VOID_FTYPE_PV2SF_V4SF:
14347 371 : case VOID_FTYPE_PV8DI_V8DI:
14348 371 : case VOID_FTYPE_PV4DI_V4DI:
14349 371 : case VOID_FTYPE_PV2DI_V2DI:
14350 371 : case VOID_FTYPE_PCHAR_V32QI:
14351 371 : case VOID_FTYPE_PCHAR_V16QI:
14352 371 : case VOID_FTYPE_PFLOAT_V16SF:
14353 371 : case VOID_FTYPE_PFLOAT_V8SF:
14354 371 : case VOID_FTYPE_PFLOAT_V4SF:
14355 371 : case VOID_FTYPE_PDOUBLE_V8DF:
14356 371 : case VOID_FTYPE_PDOUBLE_V4DF:
14357 371 : case VOID_FTYPE_PDOUBLE_V2DF:
14358 371 : case VOID_FTYPE_PLONGLONG_LONGLONG:
14359 371 : case VOID_FTYPE_PULONGLONG_ULONGLONG:
14360 371 : case VOID_FTYPE_PUNSIGNED_UNSIGNED:
14361 371 : case VOID_FTYPE_PINT_INT:
14362 371 : nargs = 1;
14363 371 : klass = store;
14364 : /* Reserve memory operand for target. */
14365 371 : memory = ARRAY_SIZE (xops);
14366 371 : switch (icode)
14367 : {
14368 : /* These builtins and instructions require the memory
14369 : to be properly aligned. */
14370 : case CODE_FOR_avx_movntv4di:
14371 : case CODE_FOR_sse2_movntv2di:
14372 : case CODE_FOR_avx_movntv8sf:
14373 : case CODE_FOR_sse_movntv4sf:
14374 : case CODE_FOR_sse4a_vmmovntv4sf:
14375 : case CODE_FOR_avx_movntv4df:
14376 : case CODE_FOR_sse2_movntv2df:
14377 : case CODE_FOR_sse4a_vmmovntv2df:
14378 : case CODE_FOR_sse2_movntidi:
14379 : case CODE_FOR_sse_movntq:
14380 : case CODE_FOR_sse2_movntisi:
14381 : case CODE_FOR_avx512f_movntv16sf:
14382 : case CODE_FOR_avx512f_movntv8df:
14383 : case CODE_FOR_avx512f_movntv8di:
14384 : aligned_mem = true;
14385 : break;
14386 : default:
14387 : break;
14388 : }
14389 : break;
14390 0 : case VOID_FTYPE_PVOID_PCVOID:
14391 0 : nargs = 1;
14392 0 : klass = store;
14393 0 : memory = 0;
14394 :
14395 0 : break;
14396 26 : case V4SF_FTYPE_V4SF_PCV2SF:
14397 26 : case V2DF_FTYPE_V2DF_PCDOUBLE:
14398 26 : nargs = 2;
14399 26 : klass = load;
14400 26 : memory = 1;
14401 26 : break;
14402 93 : case V8SF_FTYPE_PCV8SF_V8SI:
14403 93 : case V4DF_FTYPE_PCV4DF_V4DI:
14404 93 : case V4SF_FTYPE_PCV4SF_V4SI:
14405 93 : case V2DF_FTYPE_PCV2DF_V2DI:
14406 93 : case V8SI_FTYPE_PCV8SI_V8SI:
14407 93 : case V4DI_FTYPE_PCV4DI_V4DI:
14408 93 : case V4SI_FTYPE_PCV4SI_V4SI:
14409 93 : case V2DI_FTYPE_PCV2DI_V2DI:
14410 93 : case VOID_FTYPE_INT_INT64:
14411 93 : nargs = 2;
14412 93 : klass = load;
14413 93 : memory = 0;
14414 93 : break;
14415 360 : case VOID_FTYPE_PV8DF_V8DF_UQI:
14416 360 : case VOID_FTYPE_PV4DF_V4DF_UQI:
14417 360 : case VOID_FTYPE_PV2DF_V2DF_UQI:
14418 360 : case VOID_FTYPE_PV16SF_V16SF_UHI:
14419 360 : case VOID_FTYPE_PV8SF_V8SF_UQI:
14420 360 : case VOID_FTYPE_PV4SF_V4SF_UQI:
14421 360 : case VOID_FTYPE_PV8DI_V8DI_UQI:
14422 360 : case VOID_FTYPE_PV4DI_V4DI_UQI:
14423 360 : case VOID_FTYPE_PV2DI_V2DI_UQI:
14424 360 : case VOID_FTYPE_PV16SI_V16SI_UHI:
14425 360 : case VOID_FTYPE_PV8SI_V8SI_UQI:
14426 360 : case VOID_FTYPE_PV4SI_V4SI_UQI:
14427 360 : case VOID_FTYPE_PV64QI_V64QI_UDI:
14428 360 : case VOID_FTYPE_PV32HI_V32HI_USI:
14429 360 : case VOID_FTYPE_PV32QI_V32QI_USI:
14430 360 : case VOID_FTYPE_PV16QI_V16QI_UHI:
14431 360 : case VOID_FTYPE_PV16HI_V16HI_UHI:
14432 360 : case VOID_FTYPE_PV8HI_V8HI_UQI:
14433 360 : switch (icode)
14434 : {
14435 : /* These builtins and instructions require the memory
14436 : to be properly aligned. */
14437 144 : case CODE_FOR_avx512f_storev16sf_mask:
14438 144 : case CODE_FOR_avx512f_storev16si_mask:
14439 144 : case CODE_FOR_avx512f_storev8df_mask:
14440 144 : case CODE_FOR_avx512f_storev8di_mask:
14441 144 : case CODE_FOR_avx512vl_storev8sf_mask:
14442 144 : case CODE_FOR_avx512vl_storev8si_mask:
14443 144 : case CODE_FOR_avx512vl_storev4df_mask:
14444 144 : case CODE_FOR_avx512vl_storev4di_mask:
14445 144 : case CODE_FOR_avx512vl_storev4sf_mask:
14446 144 : case CODE_FOR_avx512vl_storev4si_mask:
14447 144 : case CODE_FOR_avx512vl_storev2df_mask:
14448 144 : case CODE_FOR_avx512vl_storev2di_mask:
14449 144 : aligned_mem = true;
14450 144 : break;
14451 : default:
14452 : break;
14453 : }
14454 : /* FALLTHRU */
14455 : case VOID_FTYPE_PV8SF_V8SI_V8SF:
14456 : case VOID_FTYPE_PV4DF_V4DI_V4DF:
14457 : case VOID_FTYPE_PV4SF_V4SI_V4SF:
14458 : case VOID_FTYPE_PV2DF_V2DI_V2DF:
14459 : case VOID_FTYPE_PV8SI_V8SI_V8SI:
14460 : case VOID_FTYPE_PV4DI_V4DI_V4DI:
14461 : case VOID_FTYPE_PV4SI_V4SI_V4SI:
14462 : case VOID_FTYPE_PV2DI_V2DI_V2DI:
14463 : case VOID_FTYPE_PV8SI_V8DI_UQI:
14464 : case VOID_FTYPE_PV8HI_V8DI_UQI:
14465 : case VOID_FTYPE_PV16HI_V16SI_UHI:
14466 : case VOID_FTYPE_PUDI_V8DI_UQI:
14467 : case VOID_FTYPE_PV16QI_V16SI_UHI:
14468 : case VOID_FTYPE_PV4SI_V4DI_UQI:
14469 : case VOID_FTYPE_PUDI_V2DI_UQI:
14470 : case VOID_FTYPE_PUDI_V4DI_UQI:
14471 : case VOID_FTYPE_PUSI_V2DI_UQI:
14472 : case VOID_FTYPE_PV8HI_V8SI_UQI:
14473 : case VOID_FTYPE_PUDI_V4SI_UQI:
14474 : case VOID_FTYPE_PUSI_V4DI_UQI:
14475 : case VOID_FTYPE_PUHI_V2DI_UQI:
14476 : case VOID_FTYPE_PUDI_V8SI_UQI:
14477 : case VOID_FTYPE_PUSI_V4SI_UQI:
14478 : case VOID_FTYPE_PCHAR_V64QI_UDI:
14479 : case VOID_FTYPE_PCHAR_V32QI_USI:
14480 : case VOID_FTYPE_PCHAR_V16QI_UHI:
14481 : case VOID_FTYPE_PSHORT_V32HI_USI:
14482 : case VOID_FTYPE_PSHORT_V16HI_UHI:
14483 : case VOID_FTYPE_PSHORT_V8HI_UQI:
14484 : case VOID_FTYPE_PINT_V16SI_UHI:
14485 : case VOID_FTYPE_PINT_V8SI_UQI:
14486 : case VOID_FTYPE_PINT_V4SI_UQI:
14487 : case VOID_FTYPE_PINT64_V8DI_UQI:
14488 : case VOID_FTYPE_PINT64_V4DI_UQI:
14489 : case VOID_FTYPE_PINT64_V2DI_UQI:
14490 : case VOID_FTYPE_PDOUBLE_V8DF_UQI:
14491 : case VOID_FTYPE_PDOUBLE_V4DF_UQI:
14492 : case VOID_FTYPE_PDOUBLE_V2DF_UQI:
14493 : case VOID_FTYPE_PFLOAT_V16SF_UHI:
14494 : case VOID_FTYPE_PFLOAT_V8SF_UQI:
14495 : case VOID_FTYPE_PFLOAT_V4SF_UQI:
14496 : case VOID_FTYPE_PCFLOAT16_V8HF_UQI:
14497 : case VOID_FTYPE_PV32QI_V32HI_USI:
14498 : case VOID_FTYPE_PV16QI_V16HI_UHI:
14499 : case VOID_FTYPE_PUDI_V8HI_UQI:
14500 : nargs = 2;
14501 : klass = store;
14502 : /* Reserve memory operand for target. */
14503 : memory = ARRAY_SIZE (xops);
14504 : break;
14505 1243 : case V4SF_FTYPE_PCV4SF_V4SF_UQI:
14506 1243 : case V8SF_FTYPE_PCV8SF_V8SF_UQI:
14507 1243 : case V16SF_FTYPE_PCV16SF_V16SF_UHI:
14508 1243 : case V4SI_FTYPE_PCV4SI_V4SI_UQI:
14509 1243 : case V8SI_FTYPE_PCV8SI_V8SI_UQI:
14510 1243 : case V16SI_FTYPE_PCV16SI_V16SI_UHI:
14511 1243 : case V2DF_FTYPE_PCV2DF_V2DF_UQI:
14512 1243 : case V4DF_FTYPE_PCV4DF_V4DF_UQI:
14513 1243 : case V8DF_FTYPE_PCV8DF_V8DF_UQI:
14514 1243 : case V2DI_FTYPE_PCV2DI_V2DI_UQI:
14515 1243 : case V4DI_FTYPE_PCV4DI_V4DI_UQI:
14516 1243 : case V8DI_FTYPE_PCV8DI_V8DI_UQI:
14517 1243 : case V64QI_FTYPE_PCV64QI_V64QI_UDI:
14518 1243 : case V32HI_FTYPE_PCV32HI_V32HI_USI:
14519 1243 : case V32QI_FTYPE_PCV32QI_V32QI_USI:
14520 1243 : case V16QI_FTYPE_PCV16QI_V16QI_UHI:
14521 1243 : case V16HI_FTYPE_PCV16HI_V16HI_UHI:
14522 1243 : case V8HI_FTYPE_PCV8HI_V8HI_UQI:
14523 1243 : switch (icode)
14524 : {
14525 : /* These builtins and instructions require the memory
14526 : to be properly aligned. */
14527 288 : case CODE_FOR_avx512f_loadv16sf_mask:
14528 288 : case CODE_FOR_avx512f_loadv16si_mask:
14529 288 : case CODE_FOR_avx512f_loadv8df_mask:
14530 288 : case CODE_FOR_avx512f_loadv8di_mask:
14531 288 : case CODE_FOR_avx512vl_loadv8sf_mask:
14532 288 : case CODE_FOR_avx512vl_loadv8si_mask:
14533 288 : case CODE_FOR_avx512vl_loadv4df_mask:
14534 288 : case CODE_FOR_avx512vl_loadv4di_mask:
14535 288 : case CODE_FOR_avx512vl_loadv4sf_mask:
14536 288 : case CODE_FOR_avx512vl_loadv4si_mask:
14537 288 : case CODE_FOR_avx512vl_loadv2df_mask:
14538 288 : case CODE_FOR_avx512vl_loadv2di_mask:
14539 288 : case CODE_FOR_avx512bw_loadv64qi_mask:
14540 288 : case CODE_FOR_avx512vl_loadv32qi_mask:
14541 288 : case CODE_FOR_avx512vl_loadv16qi_mask:
14542 288 : case CODE_FOR_avx512bw_loadv32hi_mask:
14543 288 : case CODE_FOR_avx512vl_loadv16hi_mask:
14544 288 : case CODE_FOR_avx512vl_loadv8hi_mask:
14545 288 : aligned_mem = true;
14546 288 : break;
14547 : default:
14548 : break;
14549 : }
14550 : /* FALLTHRU */
14551 : case V64QI_FTYPE_PCCHAR_V64QI_UDI:
14552 : case V32QI_FTYPE_PCCHAR_V32QI_USI:
14553 : case V16QI_FTYPE_PCCHAR_V16QI_UHI:
14554 : case V32HI_FTYPE_PCSHORT_V32HI_USI:
14555 : case V16HI_FTYPE_PCSHORT_V16HI_UHI:
14556 : case V8HI_FTYPE_PCSHORT_V8HI_UQI:
14557 : case V16SI_FTYPE_PCINT_V16SI_UHI:
14558 : case V8SI_FTYPE_PCINT_V8SI_UQI:
14559 : case V4SI_FTYPE_PCINT_V4SI_UQI:
14560 : case V8DI_FTYPE_PCINT64_V8DI_UQI:
14561 : case V4DI_FTYPE_PCINT64_V4DI_UQI:
14562 : case V2DI_FTYPE_PCINT64_V2DI_UQI:
14563 : case V8DF_FTYPE_PCDOUBLE_V8DF_UQI:
14564 : case V4DF_FTYPE_PCDOUBLE_V4DF_UQI:
14565 : case V2DF_FTYPE_PCDOUBLE_V2DF_UQI:
14566 : case V16SF_FTYPE_PCFLOAT_V16SF_UHI:
14567 : case V8SF_FTYPE_PCFLOAT_V8SF_UQI:
14568 : case V4SF_FTYPE_PCFLOAT_V4SF_UQI:
14569 : case V8HF_FTYPE_PCFLOAT16_V8HF_UQI:
14570 : nargs = 3;
14571 : klass = load;
14572 : memory = 0;
14573 : break;
14574 105 : case INT_FTYPE_PINT_INT_INT_INT:
14575 105 : case LONGLONG_FTYPE_PLONGLONG_LONGLONG_LONGLONG_INT:
14576 105 : nargs = 4;
14577 105 : klass = load;
14578 105 : memory = 0;
14579 105 : constant = 3;
14580 105 : break;
14581 0 : default:
14582 0 : gcc_unreachable ();
14583 : }
14584 :
14585 8267 : gcc_assert (nargs <= ARRAY_SIZE (xops));
14586 :
14587 11774 : if (klass == store)
14588 : {
14589 1911 : arg = CALL_EXPR_ARG (exp, 0);
14590 1911 : op = expand_normal (arg);
14591 1911 : gcc_assert (target == 0);
14592 1911 : if (memory)
14593 : {
14594 1748 : op = ix86_zero_extend_to_Pmode (op);
14595 1748 : target = gen_rtx_MEM (tmode, op);
14596 : /* target at this point has just BITS_PER_UNIT MEM_ALIGN
14597 : on it. Try to improve it using get_pointer_alignment,
14598 : and if the special builtin is one that requires strict
14599 : mode alignment, also from it's GET_MODE_ALIGNMENT.
14600 : Failure to do so could lead to ix86_legitimate_combined_insn
14601 : rejecting all changes to such insns. */
14602 1748 : unsigned int align = get_pointer_alignment (arg);
14603 1748 : if (aligned_mem && align < GET_MODE_ALIGNMENT (tmode))
14604 275 : align = GET_MODE_ALIGNMENT (tmode);
14605 3496 : if (MEM_ALIGN (target) < align)
14606 422 : set_mem_align (target, align);
14607 : }
14608 : else
14609 163 : target = force_reg (tmode, op);
14610 : arg_adjust = 1;
14611 : }
14612 : else
14613 : {
14614 9863 : arg_adjust = 0;
14615 9863 : if (optimize
14616 2919 : || target == 0
14617 2919 : || !register_operand (target, tmode)
14618 12771 : || GET_MODE (target) != tmode)
14619 6955 : target = gen_reg_rtx (tmode);
14620 : }
14621 :
14622 21230 : for (i = 0; i < nargs; i++)
14623 : {
14624 9456 : machine_mode mode = insn_p->operand[i + 1].mode;
14625 :
14626 9456 : arg = CALL_EXPR_ARG (exp, i + arg_adjust);
14627 9456 : op = ix86_expand_unsigned_small_int_cst_argument (arg);
14628 :
14629 9456 : if (i == memory)
14630 : {
14631 : /* This must be the memory operand. */
14632 2355 : op = ix86_zero_extend_to_Pmode (op);
14633 2355 : op = gen_rtx_MEM (mode, op);
14634 : /* op at this point has just BITS_PER_UNIT MEM_ALIGN
14635 : on it. Try to improve it using get_pointer_alignment,
14636 : and if the special builtin is one that requires strict
14637 : mode alignment, also from it's GET_MODE_ALIGNMENT.
14638 : Failure to do so could lead to ix86_legitimate_combined_insn
14639 : rejecting all changes to such insns. */
14640 2355 : unsigned int align = get_pointer_alignment (arg);
14641 2355 : if (aligned_mem && align < GET_MODE_ALIGNMENT (mode))
14642 299 : align = GET_MODE_ALIGNMENT (mode);
14643 4710 : if (MEM_ALIGN (op) < align)
14644 523 : set_mem_align (op, align);
14645 : }
14646 7101 : else if (i == constant)
14647 : {
14648 : /* This must be the constant. */
14649 105 : if (!insn_p->operand[nargs].predicate(op, SImode))
14650 : {
14651 0 : error ("the fourth argument must be one of enum %qs", "_CMPCCX_ENUM");
14652 0 : return const0_rtx;
14653 : }
14654 : }
14655 : else
14656 : {
14657 : /* This must be register. */
14658 6996 : if (VECTOR_MODE_P (mode))
14659 3508 : op = safe_vector_operand (op, mode);
14660 :
14661 6996 : op = fixup_modeless_constant (op, mode);
14662 :
14663 : /* NB: 3-operands load implied it's a mask load or v{p}expand*,
14664 : and that mask operand should be at the end.
14665 : Keep all-ones mask which would be simplified by the expander. */
14666 1771 : if (nargs == 3 && i == 2 && klass == load
14667 1771 : && constm1_operand (op, mode)
14668 7169 : && insn_p->operand[i].predicate (op, mode))
14669 : ;
14670 6996 : else if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
14671 6996 : op = copy_to_mode_reg (mode, op);
14672 : else
14673 : {
14674 0 : op = copy_to_reg (op);
14675 0 : op = lowpart_subreg (mode, op, GET_MODE (op));
14676 : }
14677 : }
14678 :
14679 9456 : xops[i]= op;
14680 : }
14681 :
14682 11774 : switch (nargs)
14683 : {
14684 7671 : case 0:
14685 7671 : pat = GEN_FCN (icode) (target);
14686 7671 : break;
14687 731 : case 1:
14688 731 : pat = GEN_FCN (icode) (target, xops[0]);
14689 731 : break;
14690 1496 : case 2:
14691 1496 : pat = GEN_FCN (icode) (target, xops[0], xops[1]);
14692 1496 : break;
14693 1771 : case 3:
14694 1771 : pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2]);
14695 1771 : break;
14696 105 : case 4:
14697 105 : pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2], xops[3]);
14698 105 : break;
14699 : default:
14700 : gcc_unreachable ();
14701 : }
14702 :
14703 11774 : if (! pat)
14704 : return 0;
14705 :
14706 11774 : emit_insn (pat);
14707 11774 : return klass == store ? 0 : target;
14708 : }
14709 :
14710 : /* Return the integer constant in ARG. Constrain it to be in the range
14711 : of the subparts of VEC_TYPE; issue an error if not. */
14712 :
14713 : static int
14714 604 : get_element_number (tree vec_type, tree arg)
14715 : {
14716 604 : unsigned HOST_WIDE_INT elt, max = TYPE_VECTOR_SUBPARTS (vec_type) - 1;
14717 :
14718 604 : if (!tree_fits_uhwi_p (arg)
14719 604 : || (elt = tree_to_uhwi (arg), elt > max))
14720 : {
14721 0 : error ("selector must be an integer constant in the range "
14722 : "[0, %wi]", max);
14723 0 : return 0;
14724 : }
14725 :
14726 604 : return elt;
14727 : }
14728 :
14729 : /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
14730 : ix86_expand_vector_init. We DO have language-level syntax for this, in
14731 : the form of (type){ init-list }. Except that since we can't place emms
14732 : instructions from inside the compiler, we can't allow the use of MMX
14733 : registers unless the user explicitly asks for it. So we do *not* define
14734 : vec_set/vec_extract/vec_init patterns for MMX modes in mmx.md. Instead
14735 : we have builtins invoked by mmintrin.h that gives us license to emit
14736 : these sorts of instructions. */
14737 :
14738 : static rtx
14739 229 : ix86_expand_vec_init_builtin (tree type, tree exp, rtx target)
14740 : {
14741 229 : machine_mode tmode = TYPE_MODE (type);
14742 229 : machine_mode inner_mode = GET_MODE_INNER (tmode);
14743 229 : int i, n_elt = GET_MODE_NUNITS (tmode);
14744 229 : rtvec v = rtvec_alloc (n_elt);
14745 :
14746 229 : gcc_assert (VECTOR_MODE_P (tmode));
14747 229 : gcc_assert (call_expr_nargs (exp) == n_elt);
14748 :
14749 1203 : for (i = 0; i < n_elt; ++i)
14750 : {
14751 974 : rtx x = expand_normal (CALL_EXPR_ARG (exp, i));
14752 974 : RTVEC_ELT (v, i) = gen_lowpart (inner_mode, x);
14753 : }
14754 :
14755 229 : if (!target || !register_operand (target, tmode))
14756 0 : target = gen_reg_rtx (tmode);
14757 :
14758 229 : ix86_expand_vector_init (true, target, gen_rtx_PARALLEL (tmode, v));
14759 229 : return target;
14760 : }
14761 :
14762 : /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
14763 : ix86_expand_vector_extract. They would be redundant (for non-MMX) if we
14764 : had a language-level syntax for referencing vector elements. */
14765 :
14766 : static rtx
14767 400 : ix86_expand_vec_ext_builtin (tree exp, rtx target)
14768 : {
14769 400 : machine_mode tmode, mode0;
14770 400 : tree arg0, arg1;
14771 400 : int elt;
14772 400 : rtx op0;
14773 :
14774 400 : arg0 = CALL_EXPR_ARG (exp, 0);
14775 400 : arg1 = CALL_EXPR_ARG (exp, 1);
14776 :
14777 400 : op0 = expand_normal (arg0);
14778 400 : elt = get_element_number (TREE_TYPE (arg0), arg1);
14779 :
14780 400 : tmode = TYPE_MODE (TREE_TYPE (TREE_TYPE (arg0)));
14781 400 : mode0 = TYPE_MODE (TREE_TYPE (arg0));
14782 400 : gcc_assert (VECTOR_MODE_P (mode0));
14783 :
14784 400 : op0 = force_reg (mode0, op0);
14785 :
14786 400 : if (optimize || !target || !register_operand (target, tmode))
14787 321 : target = gen_reg_rtx (tmode);
14788 :
14789 400 : ix86_expand_vector_extract (true, target, op0, elt);
14790 :
14791 400 : return target;
14792 : }
14793 :
14794 : /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
14795 : ix86_expand_vector_set. They would be redundant (for non-MMX) if we had
14796 : a language-level syntax for referencing vector elements. */
14797 :
14798 : static rtx
14799 204 : ix86_expand_vec_set_builtin (tree exp)
14800 : {
14801 204 : machine_mode tmode, mode1;
14802 204 : tree arg0, arg1, arg2;
14803 204 : int elt;
14804 204 : rtx op0, op1, target;
14805 :
14806 204 : arg0 = CALL_EXPR_ARG (exp, 0);
14807 204 : arg1 = CALL_EXPR_ARG (exp, 1);
14808 204 : arg2 = CALL_EXPR_ARG (exp, 2);
14809 :
14810 204 : tmode = TYPE_MODE (TREE_TYPE (arg0));
14811 204 : mode1 = TYPE_MODE (TREE_TYPE (TREE_TYPE (arg0)));
14812 204 : gcc_assert (VECTOR_MODE_P (tmode));
14813 :
14814 204 : op0 = expand_expr (arg0, NULL_RTX, tmode, EXPAND_NORMAL);
14815 204 : op1 = expand_expr (arg1, NULL_RTX, mode1, EXPAND_NORMAL);
14816 204 : elt = get_element_number (TREE_TYPE (arg0), arg2);
14817 :
14818 204 : if (GET_MODE (op1) != mode1)
14819 82 : op1 = convert_modes (mode1, GET_MODE (op1), op1, true);
14820 :
14821 204 : op0 = force_reg (tmode, op0);
14822 204 : if (op1 != CONST0_RTX (mode1))
14823 204 : op1 = force_reg (mode1, op1);
14824 :
14825 : /* OP0 is the source of these builtin functions and shouldn't be
14826 : modified. Create a copy, use it and return it as target. */
14827 204 : target = gen_reg_rtx (tmode);
14828 204 : emit_move_insn (target, op0);
14829 204 : ix86_expand_vector_set (true, target, op1, elt);
14830 :
14831 204 : return target;
14832 : }
14833 :
14834 : /* Return true if the necessary isa options for this builtin exist,
14835 : else false.
14836 : fcode = DECL_MD_FUNCTION_CODE (fndecl); */
14837 : bool
14838 1332636 : ix86_check_builtin_isa_match (unsigned int fcode,
14839 : HOST_WIDE_INT* pbisa,
14840 : HOST_WIDE_INT* pbisa2)
14841 : {
14842 1332636 : HOST_WIDE_INT isa = ix86_isa_flags;
14843 1332636 : HOST_WIDE_INT isa2 = ix86_isa_flags2;
14844 1332636 : HOST_WIDE_INT bisa = ix86_builtins_isa[fcode].isa;
14845 1332636 : HOST_WIDE_INT bisa2 = ix86_builtins_isa[fcode].isa2;
14846 1332636 : HOST_WIDE_INT tmp_isa = isa, tmp_isa2 = isa2;
14847 : /* The general case is we require all the ISAs specified in bisa{,2}
14848 : to be enabled.
14849 : The exceptions are:
14850 : OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A
14851 : OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32
14852 : OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4
14853 : (OPTION_MASK_ISA_AVX512VNNI | OPTION_MASK_ISA_AVX512VL) or
14854 : OPTION_MASK_ISA2_AVXVNNI
14855 : (OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL) or
14856 : OPTION_MASK_ISA2_AVXIFMA
14857 : (OPTION_MASK_ISA_AVX512VL | OPTION_MASK_ISA2_AVX512BF16) or
14858 : OPTION_MASK_ISA2_AVXNECONVERT
14859 : OPTION_MASK_ISA_AES or (OPTION_MASK_ISA_AVX512VL | OPTION_MASK_ISA2_VAES)
14860 : OPTION_MASK_ISA2_AVX10_2 or OPTION_MASK_ISA2_AVXVNNIINT8
14861 : OPTION_MASK_ISA2_AVX10_2 or OPTION_MASK_ISA2_AVXVNNIINT16
14862 : where for each such pair it is sufficient if either of the ISAs is
14863 : enabled, plus if it is ored with other options also those others.
14864 : OPTION_MASK_ISA_MMX in bisa is satisfied also if TARGET_MMX_WITH_SSE. */
14865 :
14866 : #define SHARE_BUILTIN(A1, A2, B1, B2) \
14867 : if ((((bisa & (A1)) == (A1) && (bisa2 & (A2)) == (A2)) \
14868 : && ((bisa & (B1)) == (B1) && (bisa2 & (B2)) == (B2))) \
14869 : && (((isa & (A1)) == (A1) && (isa2 & (A2)) == (A2)) \
14870 : || ((isa & (B1)) == (B1) && (isa2 & (B2)) == (B2)))) \
14871 : { \
14872 : tmp_isa |= (A1) | (B1); \
14873 : tmp_isa2 |= (A2) | (B2); \
14874 : }
14875 :
14876 1332636 : SHARE_BUILTIN (OPTION_MASK_ISA_SSE, 0, OPTION_MASK_ISA_3DNOW_A, 0);
14877 1332636 : SHARE_BUILTIN (OPTION_MASK_ISA_SSE4_2, 0, OPTION_MASK_ISA_CRC32, 0);
14878 1332636 : SHARE_BUILTIN (OPTION_MASK_ISA_FMA, 0, OPTION_MASK_ISA_FMA4, 0);
14879 1332636 : SHARE_BUILTIN (OPTION_MASK_ISA_AVX512VNNI | OPTION_MASK_ISA_AVX512VL, 0, 0,
14880 1332636 : OPTION_MASK_ISA2_AVXVNNI);
14881 1332636 : SHARE_BUILTIN (OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, 0, 0,
14882 1332636 : OPTION_MASK_ISA2_AVXIFMA);
14883 1332636 : SHARE_BUILTIN (OPTION_MASK_ISA_AVX512VL, OPTION_MASK_ISA2_AVX512BF16, 0,
14884 1332636 : OPTION_MASK_ISA2_AVXNECONVERT);
14885 1332636 : SHARE_BUILTIN (OPTION_MASK_ISA_AES, 0, OPTION_MASK_ISA_AVX512VL,
14886 1332636 : OPTION_MASK_ISA2_VAES);
14887 1332636 : SHARE_BUILTIN (0, OPTION_MASK_ISA2_AVXVNNIINT8, 0,
14888 1332636 : OPTION_MASK_ISA2_AVX10_2);
14889 1332636 : SHARE_BUILTIN (0, OPTION_MASK_ISA2_AVXVNNIINT16, 0,
14890 1332636 : OPTION_MASK_ISA2_AVX10_2);
14891 1332636 : isa = tmp_isa;
14892 1332636 : isa2 = tmp_isa2;
14893 :
14894 1332636 : if ((bisa & OPTION_MASK_ISA_MMX) && !TARGET_MMX && TARGET_MMX_WITH_SSE
14895 : /* __builtin_ia32_maskmovq requires MMX registers. */
14896 4531 : && fcode != IX86_BUILTIN_MASKMOVQ)
14897 : {
14898 4522 : bisa &= ~OPTION_MASK_ISA_MMX;
14899 4522 : bisa |= OPTION_MASK_ISA_SSE2;
14900 : }
14901 :
14902 1332636 : if (pbisa)
14903 176325 : *pbisa = bisa;
14904 1332636 : if (pbisa2)
14905 176325 : *pbisa2 = bisa2;
14906 :
14907 1332636 : return (bisa & isa) == bisa && (bisa2 & isa2) == bisa2;
14908 : }
14909 :
14910 : /* Emit instructions to set the carry flag from ARG. */
14911 :
14912 : void
14913 13744 : ix86_expand_carry (rtx arg)
14914 : {
14915 13744 : if (!CONST_INT_P (arg) || arg == const0_rtx)
14916 : {
14917 13737 : arg = convert_to_mode (QImode, arg, 1);
14918 13737 : arg = copy_to_mode_reg (QImode, arg);
14919 13737 : emit_insn (gen_addqi3_cconly_overflow (arg, constm1_rtx));
14920 : }
14921 : else
14922 7 : emit_insn (gen_x86_stc ());
14923 13744 : }
14924 :
14925 : /* Expand an expression EXP that calls a built-in function,
14926 : with result going to TARGET if that's convenient
14927 : (and in mode MODE if that's convenient).
14928 : SUBTARGET may be used as the target for computing one of EXP's operands.
14929 : IGNORE is nonzero if the value is to be ignored. */
14930 :
14931 : rtx
14932 177124 : ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
14933 : machine_mode mode, int ignore)
14934 : {
14935 177124 : size_t i;
14936 177124 : enum insn_code icode, icode2;
14937 177124 : tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
14938 177124 : tree arg0, arg1, arg2, arg3, arg4;
14939 177124 : rtx op0, op1, op2, op3, op4, pat, pat2, insn;
14940 177124 : machine_mode mode0, mode1, mode2, mode3, mode4;
14941 177124 : unsigned int fcode = DECL_MD_FUNCTION_CODE (fndecl);
14942 177124 : HOST_WIDE_INT bisa, bisa2;
14943 :
14944 : /* For CPU builtins that can be folded, fold first and expand the fold. */
14945 177124 : switch (fcode)
14946 : {
14947 197 : case IX86_BUILTIN_CPU_INIT:
14948 197 : {
14949 : /* Make it call __cpu_indicator_init in libgcc. */
14950 197 : tree call_expr, fndecl, type;
14951 197 : type = build_function_type_list (integer_type_node, NULL_TREE);
14952 197 : fndecl = build_fn_decl ("__cpu_indicator_init", type);
14953 197 : call_expr = build_call_expr (fndecl, 0);
14954 197 : return expand_expr (call_expr, target, mode, EXPAND_NORMAL);
14955 : }
14956 602 : case IX86_BUILTIN_CPU_IS:
14957 602 : case IX86_BUILTIN_CPU_SUPPORTS:
14958 602 : {
14959 602 : tree arg0 = CALL_EXPR_ARG (exp, 0);
14960 602 : tree fold_expr = fold_builtin_cpu (fndecl, &arg0);
14961 602 : gcc_assert (fold_expr != NULL_TREE);
14962 602 : return expand_expr (fold_expr, target, mode, EXPAND_NORMAL);
14963 : }
14964 : }
14965 :
14966 176325 : if (!ix86_check_builtin_isa_match (fcode, &bisa, &bisa2))
14967 : {
14968 23 : bool add_abi_p = bisa & OPTION_MASK_ISA_64BIT;
14969 23 : if (TARGET_ABI_X32)
14970 0 : bisa |= OPTION_MASK_ABI_X32;
14971 : else
14972 23 : bisa |= OPTION_MASK_ABI_64;
14973 23 : char *opts = ix86_target_string (bisa, bisa2, 0, 0, NULL, NULL,
14974 : (enum fpmath_unit) 0,
14975 : (enum prefer_vector_width) 0,
14976 : PVW_NONE, false, add_abi_p);
14977 23 : if (!opts)
14978 0 : error ("%qE needs unknown isa option", fndecl);
14979 : else
14980 : {
14981 23 : gcc_assert (opts != NULL);
14982 23 : error ("%qE needs isa option %s", fndecl, opts);
14983 23 : free (opts);
14984 : }
14985 23 : return expand_call (exp, target, ignore);
14986 : }
14987 :
14988 176302 : switch (fcode)
14989 : {
14990 35 : case IX86_BUILTIN_MASKMOVQ:
14991 35 : case IX86_BUILTIN_MASKMOVDQU:
14992 34 : icode = (fcode == IX86_BUILTIN_MASKMOVQ
14993 35 : ? CODE_FOR_mmx_maskmovq
14994 : : CODE_FOR_sse2_maskmovdqu);
14995 : /* Note the arg order is different from the operand order. */
14996 35 : arg1 = CALL_EXPR_ARG (exp, 0);
14997 35 : arg2 = CALL_EXPR_ARG (exp, 1);
14998 35 : arg0 = CALL_EXPR_ARG (exp, 2);
14999 35 : op0 = expand_normal (arg0);
15000 35 : op1 = expand_normal (arg1);
15001 35 : op2 = expand_normal (arg2);
15002 35 : mode0 = insn_data[icode].operand[0].mode;
15003 35 : mode1 = insn_data[icode].operand[1].mode;
15004 35 : mode2 = insn_data[icode].operand[2].mode;
15005 :
15006 35 : op0 = ix86_zero_extend_to_Pmode (op0);
15007 35 : op0 = gen_rtx_MEM (mode1, op0);
15008 :
15009 35 : if (!insn_data[icode].operand[0].predicate (op0, mode0))
15010 0 : op0 = copy_to_mode_reg (mode0, op0);
15011 35 : if (!insn_data[icode].operand[1].predicate (op1, mode1))
15012 2 : op1 = copy_to_mode_reg (mode1, op1);
15013 35 : if (!insn_data[icode].operand[2].predicate (op2, mode2))
15014 2 : op2 = copy_to_mode_reg (mode2, op2);
15015 35 : pat = GEN_FCN (icode) (op0, op1, op2);
15016 35 : if (! pat)
15017 56403 : return 0;
15018 35 : emit_insn (pat);
15019 35 : return 0;
15020 :
15021 21937 : case IX86_BUILTIN_LDMXCSR:
15022 21937 : op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
15023 21937 : target = assign_stack_temp (SImode, GET_MODE_SIZE (SImode));
15024 21937 : emit_move_insn (target, op0);
15025 21937 : emit_insn (gen_sse_ldmxcsr (target));
15026 21937 : return 0;
15027 :
15028 14713 : case IX86_BUILTIN_STMXCSR:
15029 14713 : target = assign_stack_temp (SImode, GET_MODE_SIZE (SImode));
15030 14713 : emit_insn (gen_sse_stmxcsr (target));
15031 14713 : return copy_to_mode_reg (SImode, target);
15032 :
15033 11 : case IX86_BUILTIN_CLFLUSH:
15034 11 : arg0 = CALL_EXPR_ARG (exp, 0);
15035 11 : op0 = expand_normal (arg0);
15036 11 : icode = CODE_FOR_sse2_clflush;
15037 11 : if (!insn_data[icode].operand[0].predicate (op0, Pmode))
15038 5 : op0 = ix86_zero_extend_to_Pmode (op0);
15039 :
15040 11 : emit_insn (gen_sse2_clflush (op0));
15041 11 : return 0;
15042 :
15043 19 : case IX86_BUILTIN_CLWB:
15044 19 : arg0 = CALL_EXPR_ARG (exp, 0);
15045 19 : op0 = expand_normal (arg0);
15046 19 : icode = CODE_FOR_clwb;
15047 19 : if (!insn_data[icode].operand[0].predicate (op0, Pmode))
15048 9 : op0 = ix86_zero_extend_to_Pmode (op0);
15049 :
15050 19 : emit_insn (gen_clwb (op0));
15051 19 : return 0;
15052 :
15053 19 : case IX86_BUILTIN_CLFLUSHOPT:
15054 19 : arg0 = CALL_EXPR_ARG (exp, 0);
15055 19 : op0 = expand_normal (arg0);
15056 19 : icode = CODE_FOR_clflushopt;
15057 19 : if (!insn_data[icode].operand[0].predicate (op0, Pmode))
15058 9 : op0 = ix86_zero_extend_to_Pmode (op0);
15059 :
15060 19 : emit_insn (gen_clflushopt (op0));
15061 19 : return 0;
15062 :
15063 47 : case IX86_BUILTIN_MONITOR:
15064 47 : case IX86_BUILTIN_MONITORX:
15065 47 : arg0 = CALL_EXPR_ARG (exp, 0);
15066 47 : arg1 = CALL_EXPR_ARG (exp, 1);
15067 47 : arg2 = CALL_EXPR_ARG (exp, 2);
15068 47 : op0 = expand_normal (arg0);
15069 47 : op1 = expand_normal (arg1);
15070 47 : op2 = expand_normal (arg2);
15071 47 : if (!REG_P (op0))
15072 19 : op0 = ix86_zero_extend_to_Pmode (op0);
15073 47 : if (!REG_P (op1))
15074 22 : op1 = copy_to_mode_reg (SImode, op1);
15075 47 : if (!REG_P (op2))
15076 25 : op2 = copy_to_mode_reg (SImode, op2);
15077 :
15078 47 : emit_insn (fcode == IX86_BUILTIN_MONITOR
15079 26 : ? gen_sse3_monitor (Pmode, op0, op1, op2)
15080 21 : : gen_monitorx (Pmode, op0, op1, op2));
15081 47 : return 0;
15082 :
15083 25 : case IX86_BUILTIN_MWAIT:
15084 25 : arg0 = CALL_EXPR_ARG (exp, 0);
15085 25 : arg1 = CALL_EXPR_ARG (exp, 1);
15086 25 : op0 = expand_normal (arg0);
15087 25 : op1 = expand_normal (arg1);
15088 25 : if (!REG_P (op0))
15089 13 : op0 = copy_to_mode_reg (SImode, op0);
15090 25 : if (!REG_P (op1))
15091 11 : op1 = copy_to_mode_reg (SImode, op1);
15092 25 : emit_insn (gen_sse3_mwait (op0, op1));
15093 25 : return 0;
15094 :
15095 21 : case IX86_BUILTIN_MWAITX:
15096 21 : arg0 = CALL_EXPR_ARG (exp, 0);
15097 21 : arg1 = CALL_EXPR_ARG (exp, 1);
15098 21 : arg2 = CALL_EXPR_ARG (exp, 2);
15099 21 : op0 = expand_normal (arg0);
15100 21 : op1 = expand_normal (arg1);
15101 21 : op2 = expand_normal (arg2);
15102 21 : if (!REG_P (op0))
15103 11 : op0 = copy_to_mode_reg (SImode, op0);
15104 21 : if (!REG_P (op1))
15105 10 : op1 = copy_to_mode_reg (SImode, op1);
15106 21 : if (!REG_P (op2))
15107 11 : op2 = copy_to_mode_reg (SImode, op2);
15108 21 : emit_insn (gen_mwaitx (op0, op1, op2));
15109 21 : return 0;
15110 :
15111 21 : case IX86_BUILTIN_UMONITOR:
15112 21 : arg0 = CALL_EXPR_ARG (exp, 0);
15113 21 : op0 = expand_normal (arg0);
15114 :
15115 21 : op0 = ix86_zero_extend_to_Pmode (op0);
15116 21 : emit_insn (gen_umonitor (Pmode, op0));
15117 21 : return 0;
15118 :
15119 42 : case IX86_BUILTIN_UMWAIT:
15120 42 : case IX86_BUILTIN_TPAUSE:
15121 42 : arg0 = CALL_EXPR_ARG (exp, 0);
15122 42 : arg1 = CALL_EXPR_ARG (exp, 1);
15123 42 : op0 = expand_normal (arg0);
15124 42 : op1 = expand_normal (arg1);
15125 :
15126 42 : if (!REG_P (op0))
15127 20 : op0 = copy_to_mode_reg (SImode, op0);
15128 :
15129 42 : op1 = force_reg (DImode, op1);
15130 :
15131 42 : if (TARGET_64BIT)
15132 : {
15133 42 : op2 = expand_simple_binop (DImode, LSHIFTRT, op1, GEN_INT (32),
15134 : NULL, 1, OPTAB_DIRECT);
15135 42 : switch (fcode)
15136 : {
15137 : case IX86_BUILTIN_UMWAIT:
15138 : icode = CODE_FOR_umwait_rex64;
15139 : break;
15140 21 : case IX86_BUILTIN_TPAUSE:
15141 21 : icode = CODE_FOR_tpause_rex64;
15142 21 : break;
15143 0 : default:
15144 0 : gcc_unreachable ();
15145 : }
15146 :
15147 42 : op2 = gen_lowpart (SImode, op2);
15148 42 : op1 = gen_lowpart (SImode, op1);
15149 42 : pat = GEN_FCN (icode) (op0, op1, op2);
15150 : }
15151 : else
15152 : {
15153 0 : switch (fcode)
15154 : {
15155 : case IX86_BUILTIN_UMWAIT:
15156 : icode = CODE_FOR_umwait;
15157 : break;
15158 0 : case IX86_BUILTIN_TPAUSE:
15159 0 : icode = CODE_FOR_tpause;
15160 0 : break;
15161 0 : default:
15162 0 : gcc_unreachable ();
15163 : }
15164 0 : pat = GEN_FCN (icode) (op0, op1);
15165 : }
15166 :
15167 42 : if (!pat)
15168 : return 0;
15169 :
15170 42 : emit_insn (pat);
15171 :
15172 42 : if (target == 0
15173 42 : || !register_operand (target, QImode))
15174 0 : target = gen_reg_rtx (QImode);
15175 :
15176 42 : pat = gen_rtx_EQ (QImode, gen_rtx_REG (CCCmode, FLAGS_REG),
15177 : const0_rtx);
15178 42 : emit_insn (gen_rtx_SET (target, pat));
15179 :
15180 42 : return target;
15181 :
15182 20 : case IX86_BUILTIN_TESTUI:
15183 20 : emit_insn (gen_testui ());
15184 :
15185 20 : if (target == 0
15186 20 : || !register_operand (target, QImode))
15187 0 : target = gen_reg_rtx (QImode);
15188 :
15189 20 : pat = gen_rtx_LTU (QImode, gen_rtx_REG (CCCmode, FLAGS_REG),
15190 : const0_rtx);
15191 20 : emit_insn (gen_rtx_SET (target, pat));
15192 :
15193 20 : return target;
15194 :
15195 19 : case IX86_BUILTIN_CLZERO:
15196 19 : arg0 = CALL_EXPR_ARG (exp, 0);
15197 19 : op0 = expand_normal (arg0);
15198 19 : if (!REG_P (op0))
15199 9 : op0 = ix86_zero_extend_to_Pmode (op0);
15200 19 : emit_insn (gen_clzero (Pmode, op0));
15201 19 : return 0;
15202 :
15203 19 : case IX86_BUILTIN_CLDEMOTE:
15204 19 : arg0 = CALL_EXPR_ARG (exp, 0);
15205 19 : op0 = expand_normal (arg0);
15206 19 : icode = CODE_FOR_cldemote;
15207 19 : if (!insn_data[icode].operand[0].predicate (op0, Pmode))
15208 9 : op0 = ix86_zero_extend_to_Pmode (op0);
15209 :
15210 19 : emit_insn (gen_cldemote (op0));
15211 19 : return 0;
15212 :
15213 11 : case IX86_BUILTIN_LOADIWKEY:
15214 11 : {
15215 11 : arg0 = CALL_EXPR_ARG (exp, 0);
15216 11 : arg1 = CALL_EXPR_ARG (exp, 1);
15217 11 : arg2 = CALL_EXPR_ARG (exp, 2);
15218 11 : arg3 = CALL_EXPR_ARG (exp, 3);
15219 :
15220 11 : op0 = expand_normal (arg0);
15221 11 : op1 = expand_normal (arg1);
15222 11 : op2 = expand_normal (arg2);
15223 11 : op3 = expand_normal (arg3);
15224 :
15225 11 : if (!REG_P (op0))
15226 5 : op0 = copy_to_mode_reg (V2DImode, op0);
15227 11 : if (!REG_P (op1))
15228 5 : op1 = copy_to_mode_reg (V2DImode, op1);
15229 11 : if (!REG_P (op2))
15230 5 : op2 = copy_to_mode_reg (V2DImode, op2);
15231 11 : if (!REG_P (op3))
15232 5 : op3 = copy_to_mode_reg (SImode, op3);
15233 :
15234 11 : emit_insn (gen_loadiwkey (op0, op1, op2, op3));
15235 :
15236 11 : return 0;
15237 : }
15238 :
15239 12 : case IX86_BUILTIN_AESDEC128KLU8:
15240 12 : icode = CODE_FOR_aesdec128klu8;
15241 12 : goto aesdecenc_expand;
15242 :
15243 12 : case IX86_BUILTIN_AESDEC256KLU8:
15244 12 : icode = CODE_FOR_aesdec256klu8;
15245 12 : goto aesdecenc_expand;
15246 :
15247 12 : case IX86_BUILTIN_AESENC128KLU8:
15248 12 : icode = CODE_FOR_aesenc128klu8;
15249 12 : goto aesdecenc_expand;
15250 :
15251 : case IX86_BUILTIN_AESENC256KLU8:
15252 : icode = CODE_FOR_aesenc256klu8;
15253 :
15254 48 : aesdecenc_expand:
15255 :
15256 48 : arg0 = CALL_EXPR_ARG (exp, 0); // __m128i *odata
15257 48 : arg1 = CALL_EXPR_ARG (exp, 1); // __m128i idata
15258 48 : arg2 = CALL_EXPR_ARG (exp, 2); // const void *p
15259 :
15260 48 : op0 = expand_normal (arg0);
15261 48 : op1 = expand_normal (arg1);
15262 48 : op2 = expand_normal (arg2);
15263 :
15264 48 : if (!address_operand (op0, V2DImode))
15265 : {
15266 16 : op0 = convert_memory_address (Pmode, op0);
15267 16 : op0 = copy_addr_to_reg (op0);
15268 : }
15269 48 : op0 = gen_rtx_MEM (V2DImode, op0);
15270 :
15271 48 : if (!REG_P (op1))
15272 20 : op1 = copy_to_mode_reg (V2DImode, op1);
15273 :
15274 48 : if (!address_operand (op2, VOIDmode))
15275 : {
15276 16 : op2 = convert_memory_address (Pmode, op2);
15277 16 : op2 = copy_addr_to_reg (op2);
15278 : }
15279 48 : op2 = gen_rtx_MEM (BLKmode, op2);
15280 :
15281 48 : emit_insn (GEN_FCN (icode) (op1, op1, op2));
15282 :
15283 48 : if (target == 0)
15284 4 : target = gen_reg_rtx (QImode);
15285 :
15286 : /* NB: For aesenc/aesdec keylocker insn, ZF will be set when runtime
15287 : error occurs. Then the output should be cleared for safety. */
15288 48 : rtx_code_label *ok_label;
15289 48 : rtx tmp;
15290 :
15291 48 : tmp = gen_rtx_REG (CCZmode, FLAGS_REG);
15292 48 : pat = gen_rtx_EQ (QImode, tmp, const0_rtx);
15293 48 : ok_label = gen_label_rtx ();
15294 48 : emit_cmp_and_jump_insns (tmp, const0_rtx, NE, 0, GET_MODE (tmp),
15295 : true, ok_label);
15296 : /* Usually the runtime error seldom occur, so predict OK path as
15297 : hotspot to optimize it as fallthrough block. */
15298 48 : predict_jump (REG_BR_PROB_BASE * 90 / 100);
15299 :
15300 48 : emit_insn (gen_rtx_SET (op1, const0_rtx));
15301 :
15302 48 : emit_label (ok_label);
15303 48 : emit_insn (gen_rtx_SET (target, pat));
15304 48 : emit_insn (gen_rtx_SET (op0, op1));
15305 :
15306 48 : return target;
15307 :
15308 11 : case IX86_BUILTIN_AESDECWIDE128KLU8:
15309 11 : icode = CODE_FOR_aesdecwide128klu8;
15310 11 : goto wideaesdecenc_expand;
15311 :
15312 11 : case IX86_BUILTIN_AESDECWIDE256KLU8:
15313 11 : icode = CODE_FOR_aesdecwide256klu8;
15314 11 : goto wideaesdecenc_expand;
15315 :
15316 11 : case IX86_BUILTIN_AESENCWIDE128KLU8:
15317 11 : icode = CODE_FOR_aesencwide128klu8;
15318 11 : goto wideaesdecenc_expand;
15319 :
15320 : case IX86_BUILTIN_AESENCWIDE256KLU8:
15321 : icode = CODE_FOR_aesencwide256klu8;
15322 :
15323 44 : wideaesdecenc_expand:
15324 :
15325 44 : rtx xmm_regs[8];
15326 44 : rtx op;
15327 :
15328 44 : arg0 = CALL_EXPR_ARG (exp, 0); // __m128i * odata
15329 44 : arg1 = CALL_EXPR_ARG (exp, 1); // const __m128i * idata
15330 44 : arg2 = CALL_EXPR_ARG (exp, 2); // const void *p
15331 :
15332 44 : op0 = expand_normal (arg0);
15333 44 : op1 = expand_normal (arg1);
15334 44 : op2 = expand_normal (arg2);
15335 :
15336 44 : if (GET_MODE (op1) != Pmode)
15337 0 : op1 = convert_to_mode (Pmode, op1, 1);
15338 :
15339 44 : if (!address_operand (op2, VOIDmode))
15340 : {
15341 16 : op2 = convert_memory_address (Pmode, op2);
15342 16 : op2 = copy_addr_to_reg (op2);
15343 : }
15344 44 : op2 = gen_rtx_MEM (BLKmode, op2);
15345 :
15346 440 : for (i = 0; i < 8; i++)
15347 : {
15348 352 : xmm_regs[i] = gen_rtx_REG (V2DImode, GET_SSE_REGNO (i));
15349 :
15350 352 : op = gen_rtx_MEM (V2DImode,
15351 352 : plus_constant (Pmode, op1, (i * 16)));
15352 :
15353 352 : emit_move_insn (xmm_regs[i], op);
15354 : }
15355 :
15356 44 : emit_insn (GEN_FCN (icode) (op2));
15357 :
15358 44 : if (target == 0)
15359 0 : target = gen_reg_rtx (QImode);
15360 :
15361 44 : tmp = gen_rtx_REG (CCZmode, FLAGS_REG);
15362 44 : pat = gen_rtx_EQ (QImode, tmp, const0_rtx);
15363 44 : ok_label = gen_label_rtx ();
15364 44 : emit_cmp_and_jump_insns (tmp, const0_rtx, NE, 0, GET_MODE (tmp),
15365 : true, ok_label);
15366 44 : predict_jump (REG_BR_PROB_BASE * 90 / 100);
15367 :
15368 440 : for (i = 0; i < 8; i++)
15369 352 : emit_insn (gen_rtx_SET (xmm_regs[i], const0_rtx));
15370 :
15371 44 : emit_label (ok_label);
15372 44 : emit_insn (gen_rtx_SET (target, pat));
15373 :
15374 44 : if (GET_MODE (op0) != Pmode)
15375 0 : op0 = convert_to_mode (Pmode, op0, 1);
15376 :
15377 396 : for (i = 0; i < 8; i++)
15378 : {
15379 352 : op = gen_rtx_MEM (V2DImode,
15380 352 : plus_constant (Pmode, op0, (i * 16)));
15381 352 : emit_move_insn (op, xmm_regs[i]);
15382 : }
15383 :
15384 : return target;
15385 :
15386 13 : case IX86_BUILTIN_ENCODEKEY128U32:
15387 13 : {
15388 13 : rtx op, xmm_regs[7];
15389 :
15390 13 : arg0 = CALL_EXPR_ARG (exp, 0); // unsigned int htype
15391 13 : arg1 = CALL_EXPR_ARG (exp, 1); // __m128i key
15392 13 : arg2 = CALL_EXPR_ARG (exp, 2); // void *h
15393 :
15394 13 : op0 = expand_normal (arg0);
15395 13 : op1 = expand_normal (arg1);
15396 13 : op2 = expand_normal (arg2);
15397 :
15398 13 : if (!REG_P (op0))
15399 7 : op0 = copy_to_mode_reg (SImode, op0);
15400 :
15401 13 : if (GET_MODE (op2) != Pmode)
15402 1 : op2 = convert_to_mode (Pmode, op2, 1);
15403 :
15404 13 : op = gen_rtx_REG (V2DImode, GET_SSE_REGNO (0));
15405 13 : emit_move_insn (op, op1);
15406 :
15407 65 : for (i = 0; i < 3; i++)
15408 39 : xmm_regs[i] = gen_rtx_REG (V2DImode, GET_SSE_REGNO (i));
15409 :
15410 13 : if (target == 0 || !register_operand (target, SImode))
15411 2 : target = gen_reg_rtx (SImode);
15412 :
15413 13 : emit_insn (gen_encodekey128u32 (target, op0));
15414 :
15415 65 : for (i = 0; i < 3; i++)
15416 : {
15417 39 : op = gen_rtx_MEM (V2DImode,
15418 39 : plus_constant (Pmode, op2, (i * 16)));
15419 39 : emit_move_insn (op, xmm_regs[i]);
15420 : }
15421 :
15422 13 : return target;
15423 : }
15424 13 : case IX86_BUILTIN_ENCODEKEY256U32:
15425 13 : {
15426 13 : rtx op, xmm_regs[7];
15427 :
15428 13 : arg0 = CALL_EXPR_ARG (exp, 0); // unsigned int htype
15429 13 : arg1 = CALL_EXPR_ARG (exp, 1); // __m128i keylow
15430 13 : arg2 = CALL_EXPR_ARG (exp, 2); // __m128i keyhi
15431 13 : arg3 = CALL_EXPR_ARG (exp, 3); // void *h
15432 :
15433 13 : op0 = expand_normal (arg0);
15434 13 : op1 = expand_normal (arg1);
15435 13 : op2 = expand_normal (arg2);
15436 13 : op3 = expand_normal (arg3);
15437 :
15438 13 : if (!REG_P (op0))
15439 7 : op0 = copy_to_mode_reg (SImode, op0);
15440 :
15441 13 : if (GET_MODE (op3) != Pmode)
15442 1 : op3 = convert_to_mode (Pmode, op3, 1);
15443 :
15444 : /* Force to use xmm0, xmm1 for keylow, keyhi*/
15445 13 : op = gen_rtx_REG (V2DImode, GET_SSE_REGNO (0));
15446 13 : emit_move_insn (op, op1);
15447 13 : op = gen_rtx_REG (V2DImode, GET_SSE_REGNO (1));
15448 13 : emit_move_insn (op, op2);
15449 :
15450 78 : for (i = 0; i < 4; i++)
15451 52 : xmm_regs[i] = gen_rtx_REG (V2DImode, GET_SSE_REGNO (i));
15452 :
15453 13 : if (target == 0 || !register_operand (target, SImode))
15454 2 : target = gen_reg_rtx (SImode);
15455 :
15456 13 : emit_insn (gen_encodekey256u32 (target, op0));
15457 :
15458 78 : for (i = 0; i < 4; i++)
15459 : {
15460 52 : op = gen_rtx_MEM (V2DImode,
15461 52 : plus_constant (Pmode, op3, (i * 16)));
15462 52 : emit_move_insn (op, xmm_regs[i]);
15463 : }
15464 :
15465 13 : return target;
15466 : }
15467 :
15468 48 : case IX86_BUILTIN_PREFETCH:
15469 48 : {
15470 48 : arg0 = CALL_EXPR_ARG (exp, 0); // const void *
15471 48 : arg1 = CALL_EXPR_ARG (exp, 1); // const int
15472 48 : arg2 = CALL_EXPR_ARG (exp, 2); // const int
15473 48 : arg3 = CALL_EXPR_ARG (exp, 3); // const int
15474 :
15475 48 : op0 = expand_normal (arg0);
15476 48 : op1 = expand_normal (arg1);
15477 48 : op2 = expand_normal (arg2);
15478 48 : op3 = expand_normal (arg3);
15479 :
15480 48 : if (!CONST_INT_P (op1) || !CONST_INT_P (op2) || !CONST_INT_P (op3))
15481 : {
15482 0 : error ("second, third and fourth argument must be a const");
15483 0 : return const0_rtx;
15484 : }
15485 :
15486 48 : if (!IN_RANGE (INTVAL (op1), 0, 2))
15487 : {
15488 1 : warning (0, "invalid second argument to"
15489 : " %<__builtin_ia32_prefetch%>; using zero");
15490 1 : op1 = const0_rtx;
15491 : }
15492 :
15493 48 : if (INTVAL (op3) == 1)
15494 : {
15495 4 : if (!IN_RANGE (INTVAL (op2), 2, 3))
15496 : {
15497 1 : error ("invalid third argument");
15498 1 : return const0_rtx;
15499 : }
15500 :
15501 3 : if (TARGET_64BIT && TARGET_PREFETCHI
15502 6 : && local_func_symbolic_operand (op0, GET_MODE (op0)))
15503 2 : emit_insn (gen_prefetchi (op0, op2));
15504 : else
15505 : {
15506 1 : warning (0, "instruction prefetch applies when in 64-bit mode"
15507 : " with RIP-relative addressing and"
15508 : " option %<-mprefetchi%>;"
15509 : " they stay NOPs otherwise");
15510 1 : emit_insn (gen_nop ());
15511 : }
15512 : }
15513 : else
15514 : {
15515 44 : if (INTVAL (op3) != 0)
15516 1 : warning (0, "invalid fourth argument to"
15517 : " %<__builtin_ia32_prefetch%>; using zero");
15518 :
15519 44 : if (!address_operand (op0, VOIDmode))
15520 : {
15521 10 : op0 = convert_memory_address (Pmode, op0);
15522 10 : op0 = copy_addr_to_reg (op0);
15523 : }
15524 :
15525 44 : if (!IN_RANGE (INTVAL (op2), 0, 3))
15526 : {
15527 1 : warning (0, "invalid third argument to %<__builtin_ia32_prefetch%>; using zero");
15528 1 : op2 = const0_rtx;
15529 : }
15530 :
15531 44 : if (TARGET_3DNOW
15532 26 : || TARGET_PREFETCH_SSE
15533 0 : || TARGET_PRFCHW
15534 0 : || TARGET_MOVRS)
15535 44 : emit_insn (gen_prefetch (op0, op1, op2));
15536 0 : else if (!MEM_P (op0) && side_effects_p (op0))
15537 : /* Don't do anything with direct references to volatile memory,
15538 : but generate code to handle other side effects. */
15539 0 : emit_insn (op0);
15540 : }
15541 :
15542 : return 0;
15543 : }
15544 :
15545 21 : case IX86_BUILTIN_PREFETCHI:
15546 21 : {
15547 21 : arg0 = CALL_EXPR_ARG (exp, 0); // const void *
15548 21 : arg1 = CALL_EXPR_ARG (exp, 1); // const int
15549 :
15550 21 : op0 = expand_normal (arg0);
15551 21 : op1 = expand_normal (arg1);
15552 :
15553 21 : if (!CONST_INT_P (op1))
15554 : {
15555 0 : error ("second argument must be a const");
15556 0 : return const0_rtx;
15557 : }
15558 :
15559 : /* GOT/PLT_PIC should not be available for instruction prefetch.
15560 : It must be real instruction address. */
15561 21 : if (TARGET_64BIT
15562 21 : && local_func_symbolic_operand (op0, GET_MODE (op0)))
15563 4 : emit_insn (gen_prefetchi (op0, op1));
15564 : else
15565 : {
15566 : /* Ignore the hint. */
15567 17 : warning (0, "instruction prefetch applies when in 64-bit mode"
15568 : " with RIP-relative addressing and"
15569 : " option %<-mprefetchi%>;"
15570 : " they stay NOPs otherwise");
15571 17 : emit_insn (gen_nop ());
15572 : }
15573 :
15574 : return 0;
15575 : }
15576 :
15577 53 : case IX86_BUILTIN_URDMSR:
15578 53 : case IX86_BUILTIN_UWRMSR:
15579 53 : {
15580 53 : arg0 = CALL_EXPR_ARG (exp, 0);
15581 53 : op0 = expand_normal (arg0);
15582 :
15583 53 : if (CONST_INT_P (op0))
15584 : {
15585 12 : unsigned HOST_WIDE_INT val = UINTVAL (op0);
15586 12 : if (val > 0xffffffff)
15587 2 : op0 = force_reg (DImode, op0);
15588 : }
15589 : else
15590 41 : op0 = force_reg (DImode, op0);
15591 :
15592 53 : if (fcode == IX86_BUILTIN_UWRMSR)
15593 : {
15594 26 : arg1 = CALL_EXPR_ARG (exp, 1);
15595 26 : op1 = expand_normal (arg1);
15596 26 : op1 = force_reg (DImode, op1);
15597 26 : icode = CODE_FOR_uwrmsr;
15598 26 : target = 0;
15599 : }
15600 : else
15601 : {
15602 27 : if (target == 0 || !register_operand (target, DImode))
15603 1 : target = gen_reg_rtx (DImode);
15604 : icode = CODE_FOR_urdmsr;
15605 : op1 = op0;
15606 : op0 = target;
15607 : }
15608 53 : emit_insn (GEN_FCN (icode) (op0, op1));
15609 53 : return target;
15610 : }
15611 :
15612 229 : case IX86_BUILTIN_VEC_INIT_V2SI:
15613 229 : case IX86_BUILTIN_VEC_INIT_V4HI:
15614 229 : case IX86_BUILTIN_VEC_INIT_V8QI:
15615 229 : return ix86_expand_vec_init_builtin (TREE_TYPE (exp), exp, target);
15616 :
15617 400 : case IX86_BUILTIN_VEC_EXT_V2DF:
15618 400 : case IX86_BUILTIN_VEC_EXT_V2DI:
15619 400 : case IX86_BUILTIN_VEC_EXT_V4SF:
15620 400 : case IX86_BUILTIN_VEC_EXT_V4SI:
15621 400 : case IX86_BUILTIN_VEC_EXT_V8HI:
15622 400 : case IX86_BUILTIN_VEC_EXT_V2SI:
15623 400 : case IX86_BUILTIN_VEC_EXT_V4HI:
15624 400 : case IX86_BUILTIN_VEC_EXT_V16QI:
15625 400 : return ix86_expand_vec_ext_builtin (exp, target);
15626 :
15627 204 : case IX86_BUILTIN_VEC_SET_V2DI:
15628 204 : case IX86_BUILTIN_VEC_SET_V4SF:
15629 204 : case IX86_BUILTIN_VEC_SET_V4SI:
15630 204 : case IX86_BUILTIN_VEC_SET_V8HI:
15631 204 : case IX86_BUILTIN_VEC_SET_V4HI:
15632 204 : case IX86_BUILTIN_VEC_SET_V16QI:
15633 204 : return ix86_expand_vec_set_builtin (exp);
15634 :
15635 0 : case IX86_BUILTIN_NANQ:
15636 0 : case IX86_BUILTIN_NANSQ:
15637 0 : return expand_call (exp, target, ignore);
15638 :
15639 18 : case IX86_BUILTIN_RDPID:
15640 :
15641 18 : op0 = gen_reg_rtx (word_mode);
15642 :
15643 18 : if (TARGET_64BIT)
15644 : {
15645 18 : insn = gen_rdpid_rex64 (op0);
15646 18 : op0 = convert_to_mode (SImode, op0, 1);
15647 : }
15648 : else
15649 0 : insn = gen_rdpid (op0);
15650 :
15651 18 : emit_insn (insn);
15652 :
15653 18 : if (target == 0
15654 18 : || !register_operand (target, SImode))
15655 0 : target = gen_reg_rtx (SImode);
15656 :
15657 18 : emit_move_insn (target, op0);
15658 18 : return target;
15659 :
15660 75 : case IX86_BUILTIN_2INTERSECTD512:
15661 75 : case IX86_BUILTIN_2INTERSECTQ512:
15662 75 : case IX86_BUILTIN_2INTERSECTD256:
15663 75 : case IX86_BUILTIN_2INTERSECTQ256:
15664 75 : case IX86_BUILTIN_2INTERSECTD128:
15665 75 : case IX86_BUILTIN_2INTERSECTQ128:
15666 75 : arg0 = CALL_EXPR_ARG (exp, 0);
15667 75 : arg1 = CALL_EXPR_ARG (exp, 1);
15668 75 : arg2 = CALL_EXPR_ARG (exp, 2);
15669 75 : arg3 = CALL_EXPR_ARG (exp, 3);
15670 75 : op0 = expand_normal (arg0);
15671 75 : op1 = expand_normal (arg1);
15672 75 : op2 = expand_normal (arg2);
15673 75 : op3 = expand_normal (arg3);
15674 :
15675 75 : if (!address_operand (op0, VOIDmode))
15676 : {
15677 25 : op0 = convert_memory_address (Pmode, op0);
15678 25 : op0 = copy_addr_to_reg (op0);
15679 : }
15680 75 : if (!address_operand (op1, VOIDmode))
15681 : {
15682 25 : op1 = convert_memory_address (Pmode, op1);
15683 25 : op1 = copy_addr_to_reg (op1);
15684 : }
15685 :
15686 75 : switch (fcode)
15687 : {
15688 : case IX86_BUILTIN_2INTERSECTD512:
15689 : mode4 = P2HImode;
15690 : icode = CODE_FOR_avx512vp2intersect_2intersectv16si;
15691 : break;
15692 : case IX86_BUILTIN_2INTERSECTQ512:
15693 : mode4 = P2QImode;
15694 : icode = CODE_FOR_avx512vp2intersect_2intersectv8di;
15695 : break;
15696 : case IX86_BUILTIN_2INTERSECTD256:
15697 : mode4 = P2QImode;
15698 : icode = CODE_FOR_avx512vp2intersect_2intersectv8si;
15699 : break;
15700 : case IX86_BUILTIN_2INTERSECTQ256:
15701 : mode4 = P2QImode;
15702 : icode = CODE_FOR_avx512vp2intersect_2intersectv4di;
15703 : break;
15704 : case IX86_BUILTIN_2INTERSECTD128:
15705 : mode4 = P2QImode;
15706 : icode = CODE_FOR_avx512vp2intersect_2intersectv4si;
15707 : break;
15708 : case IX86_BUILTIN_2INTERSECTQ128:
15709 : mode4 = P2QImode;
15710 : icode = CODE_FOR_avx512vp2intersect_2intersectv2di;
15711 : break;
15712 0 : default:
15713 0 : gcc_unreachable ();
15714 : }
15715 :
15716 75 : mode2 = insn_data[icode].operand[1].mode;
15717 75 : mode3 = insn_data[icode].operand[2].mode;
15718 75 : if (!insn_data[icode].operand[1].predicate (op2, mode2))
15719 25 : op2 = copy_to_mode_reg (mode2, op2);
15720 75 : if (!insn_data[icode].operand[2].predicate (op3, mode3))
15721 6 : op3 = copy_to_mode_reg (mode3, op3);
15722 :
15723 75 : op4 = gen_reg_rtx (mode4);
15724 75 : emit_insn (GEN_FCN (icode) (op4, op2, op3));
15725 75 : mode0 = mode4 == P2HImode ? HImode : QImode;
15726 75 : emit_move_insn (gen_rtx_MEM (mode0, op0),
15727 75 : gen_lowpart (mode0, op4));
15728 75 : emit_move_insn (gen_rtx_MEM (mode0, op1),
15729 : gen_highpart (mode0, op4));
15730 :
15731 75 : return 0;
15732 :
15733 102 : case IX86_BUILTIN_RDPMC:
15734 102 : case IX86_BUILTIN_RDTSC:
15735 102 : case IX86_BUILTIN_RDTSCP:
15736 102 : case IX86_BUILTIN_XGETBV:
15737 :
15738 102 : op0 = gen_reg_rtx (DImode);
15739 102 : op1 = gen_reg_rtx (DImode);
15740 :
15741 102 : if (fcode == IX86_BUILTIN_RDPMC)
15742 : {
15743 22 : arg0 = CALL_EXPR_ARG (exp, 0);
15744 22 : op2 = expand_normal (arg0);
15745 22 : if (!register_operand (op2, SImode))
15746 11 : op2 = copy_to_mode_reg (SImode, op2);
15747 :
15748 22 : insn = (TARGET_64BIT
15749 22 : ? gen_rdpmc_rex64 (op0, op1, op2)
15750 0 : : gen_rdpmc (op0, op2));
15751 22 : emit_insn (insn);
15752 : }
15753 80 : else if (fcode == IX86_BUILTIN_XGETBV)
15754 : {
15755 22 : arg0 = CALL_EXPR_ARG (exp, 0);
15756 22 : op2 = expand_normal (arg0);
15757 22 : if (!register_operand (op2, SImode))
15758 1 : op2 = copy_to_mode_reg (SImode, op2);
15759 :
15760 22 : insn = (TARGET_64BIT
15761 22 : ? gen_xgetbv_rex64 (op0, op1, op2)
15762 0 : : gen_xgetbv (op0, op2));
15763 22 : emit_insn (insn);
15764 : }
15765 58 : else if (fcode == IX86_BUILTIN_RDTSC)
15766 : {
15767 36 : insn = (TARGET_64BIT
15768 36 : ? gen_rdtsc_rex64 (op0, op1)
15769 2 : : gen_rdtsc (op0));
15770 36 : emit_insn (insn);
15771 : }
15772 : else
15773 : {
15774 22 : op2 = gen_reg_rtx (SImode);
15775 :
15776 22 : insn = (TARGET_64BIT
15777 22 : ? gen_rdtscp_rex64 (op0, op1, op2)
15778 0 : : gen_rdtscp (op0, op2));
15779 22 : emit_insn (insn);
15780 :
15781 22 : arg0 = CALL_EXPR_ARG (exp, 0);
15782 22 : op4 = expand_normal (arg0);
15783 22 : if (!address_operand (op4, VOIDmode))
15784 : {
15785 10 : op4 = convert_memory_address (Pmode, op4);
15786 10 : op4 = copy_addr_to_reg (op4);
15787 : }
15788 22 : emit_move_insn (gen_rtx_MEM (SImode, op4), op2);
15789 : }
15790 :
15791 102 : if (target == 0
15792 102 : || !register_operand (target, DImode))
15793 10 : target = gen_reg_rtx (DImode);
15794 :
15795 102 : if (TARGET_64BIT)
15796 : {
15797 100 : op1 = expand_simple_binop (DImode, ASHIFT, op1, GEN_INT (32),
15798 : op1, 1, OPTAB_DIRECT);
15799 100 : op0 = expand_simple_binop (DImode, IOR, op0, op1,
15800 : op0, 1, OPTAB_DIRECT);
15801 : }
15802 :
15803 102 : emit_move_insn (target, op0);
15804 102 : return target;
15805 :
15806 61 : case IX86_BUILTIN_ENQCMD:
15807 61 : case IX86_BUILTIN_ENQCMDS:
15808 61 : case IX86_BUILTIN_MOVDIR64B:
15809 :
15810 61 : arg0 = CALL_EXPR_ARG (exp, 0);
15811 61 : arg1 = CALL_EXPR_ARG (exp, 1);
15812 61 : op0 = expand_normal (arg0);
15813 61 : op1 = expand_normal (arg1);
15814 :
15815 61 : op0 = ix86_zero_extend_to_Pmode (op0);
15816 61 : if (!address_operand (op1, VOIDmode))
15817 : {
15818 28 : op1 = convert_memory_address (Pmode, op1);
15819 28 : op1 = copy_addr_to_reg (op1);
15820 : }
15821 61 : op1 = gen_rtx_MEM (XImode, op1);
15822 :
15823 61 : if (fcode == IX86_BUILTIN_MOVDIR64B)
15824 : {
15825 24 : emit_insn (gen_movdir64b (Pmode, op0, op1));
15826 23 : return 0;
15827 : }
15828 : else
15829 : {
15830 38 : if (target == 0
15831 38 : || !register_operand (target, SImode))
15832 0 : target = gen_reg_rtx (SImode);
15833 :
15834 38 : emit_move_insn (target, const0_rtx);
15835 38 : target = gen_rtx_SUBREG (QImode, target, 0);
15836 :
15837 19 : int unspecv = (fcode == IX86_BUILTIN_ENQCMD
15838 38 : ? UNSPECV_ENQCMD
15839 : : UNSPECV_ENQCMDS);
15840 38 : icode = code_for_enqcmd (unspecv, Pmode);
15841 38 : emit_insn (GEN_FCN (icode) (op0, op1));
15842 :
15843 38 : emit_insn
15844 38 : (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
15845 : gen_rtx_fmt_ee (EQ, QImode,
15846 : gen_rtx_REG (CCZmode, FLAGS_REG),
15847 : const0_rtx)));
15848 38 : return SUBREG_REG (target);
15849 : }
15850 :
15851 14703 : case IX86_BUILTIN_FXSAVE:
15852 14703 : case IX86_BUILTIN_FXRSTOR:
15853 14703 : case IX86_BUILTIN_FXSAVE64:
15854 14703 : case IX86_BUILTIN_FXRSTOR64:
15855 14703 : case IX86_BUILTIN_FNSTENV:
15856 14703 : case IX86_BUILTIN_FLDENV:
15857 14703 : mode0 = BLKmode;
15858 14703 : switch (fcode)
15859 : {
15860 : case IX86_BUILTIN_FXSAVE:
15861 : icode = CODE_FOR_fxsave;
15862 : break;
15863 19 : case IX86_BUILTIN_FXRSTOR:
15864 19 : icode = CODE_FOR_fxrstor;
15865 19 : break;
15866 23 : case IX86_BUILTIN_FXSAVE64:
15867 23 : icode = CODE_FOR_fxsave64;
15868 23 : break;
15869 21 : case IX86_BUILTIN_FXRSTOR64:
15870 21 : icode = CODE_FOR_fxrstor64;
15871 21 : break;
15872 7258 : case IX86_BUILTIN_FNSTENV:
15873 7258 : icode = CODE_FOR_fnstenv;
15874 7258 : break;
15875 7362 : case IX86_BUILTIN_FLDENV:
15876 7362 : icode = CODE_FOR_fldenv;
15877 7362 : break;
15878 0 : default:
15879 0 : gcc_unreachable ();
15880 : }
15881 :
15882 14703 : arg0 = CALL_EXPR_ARG (exp, 0);
15883 14703 : op0 = expand_normal (arg0);
15884 :
15885 14703 : if (!address_operand (op0, VOIDmode))
15886 : {
15887 36 : op0 = convert_memory_address (Pmode, op0);
15888 36 : op0 = copy_addr_to_reg (op0);
15889 : }
15890 14703 : op0 = gen_rtx_MEM (mode0, op0);
15891 :
15892 14703 : pat = GEN_FCN (icode) (op0);
15893 14703 : if (pat)
15894 14703 : emit_insn (pat);
15895 : return 0;
15896 :
15897 21 : case IX86_BUILTIN_XSETBV:
15898 21 : arg0 = CALL_EXPR_ARG (exp, 0);
15899 21 : arg1 = CALL_EXPR_ARG (exp, 1);
15900 21 : op0 = expand_normal (arg0);
15901 21 : op1 = expand_normal (arg1);
15902 :
15903 21 : if (!REG_P (op0))
15904 1 : op0 = copy_to_mode_reg (SImode, op0);
15905 :
15906 21 : op1 = force_reg (DImode, op1);
15907 :
15908 21 : if (TARGET_64BIT)
15909 : {
15910 21 : op2 = expand_simple_binop (DImode, LSHIFTRT, op1, GEN_INT (32),
15911 : NULL, 1, OPTAB_DIRECT);
15912 :
15913 21 : icode = CODE_FOR_xsetbv_rex64;
15914 :
15915 21 : op2 = gen_lowpart (SImode, op2);
15916 21 : op1 = gen_lowpart (SImode, op1);
15917 21 : pat = GEN_FCN (icode) (op0, op1, op2);
15918 : }
15919 : else
15920 : {
15921 0 : icode = CODE_FOR_xsetbv;
15922 :
15923 0 : pat = GEN_FCN (icode) (op0, op1);
15924 : }
15925 21 : if (pat)
15926 21 : emit_insn (pat);
15927 : return 0;
15928 :
15929 232 : case IX86_BUILTIN_XSAVE:
15930 232 : case IX86_BUILTIN_XRSTOR:
15931 232 : case IX86_BUILTIN_XSAVE64:
15932 232 : case IX86_BUILTIN_XRSTOR64:
15933 232 : case IX86_BUILTIN_XSAVEOPT:
15934 232 : case IX86_BUILTIN_XSAVEOPT64:
15935 232 : case IX86_BUILTIN_XSAVES:
15936 232 : case IX86_BUILTIN_XRSTORS:
15937 232 : case IX86_BUILTIN_XSAVES64:
15938 232 : case IX86_BUILTIN_XRSTORS64:
15939 232 : case IX86_BUILTIN_XSAVEC:
15940 232 : case IX86_BUILTIN_XSAVEC64:
15941 232 : arg0 = CALL_EXPR_ARG (exp, 0);
15942 232 : arg1 = CALL_EXPR_ARG (exp, 1);
15943 232 : op0 = expand_normal (arg0);
15944 232 : op1 = expand_normal (arg1);
15945 :
15946 232 : if (!address_operand (op0, VOIDmode))
15947 : {
15948 108 : op0 = convert_memory_address (Pmode, op0);
15949 108 : op0 = copy_addr_to_reg (op0);
15950 : }
15951 232 : op0 = gen_rtx_MEM (BLKmode, op0);
15952 :
15953 232 : op1 = force_reg (DImode, op1);
15954 :
15955 232 : if (TARGET_64BIT)
15956 : {
15957 232 : op2 = expand_simple_binop (DImode, LSHIFTRT, op1, GEN_INT (32),
15958 : NULL, 1, OPTAB_DIRECT);
15959 232 : switch (fcode)
15960 : {
15961 : case IX86_BUILTIN_XSAVE:
15962 : icode = CODE_FOR_xsave_rex64;
15963 : break;
15964 19 : case IX86_BUILTIN_XRSTOR:
15965 19 : icode = CODE_FOR_xrstor_rex64;
15966 19 : break;
15967 21 : case IX86_BUILTIN_XSAVE64:
15968 21 : icode = CODE_FOR_xsave64;
15969 21 : break;
15970 21 : case IX86_BUILTIN_XRSTOR64:
15971 21 : icode = CODE_FOR_xrstor64;
15972 21 : break;
15973 19 : case IX86_BUILTIN_XSAVEOPT:
15974 19 : icode = CODE_FOR_xsaveopt_rex64;
15975 19 : break;
15976 19 : case IX86_BUILTIN_XSAVEOPT64:
15977 19 : icode = CODE_FOR_xsaveopt64;
15978 19 : break;
15979 19 : case IX86_BUILTIN_XSAVES:
15980 19 : icode = CODE_FOR_xsaves_rex64;
15981 19 : break;
15982 19 : case IX86_BUILTIN_XRSTORS:
15983 19 : icode = CODE_FOR_xrstors_rex64;
15984 19 : break;
15985 19 : case IX86_BUILTIN_XSAVES64:
15986 19 : icode = CODE_FOR_xsaves64;
15987 19 : break;
15988 19 : case IX86_BUILTIN_XRSTORS64:
15989 19 : icode = CODE_FOR_xrstors64;
15990 19 : break;
15991 19 : case IX86_BUILTIN_XSAVEC:
15992 19 : icode = CODE_FOR_xsavec_rex64;
15993 19 : break;
15994 19 : case IX86_BUILTIN_XSAVEC64:
15995 19 : icode = CODE_FOR_xsavec64;
15996 19 : break;
15997 0 : default:
15998 0 : gcc_unreachable ();
15999 : }
16000 :
16001 232 : op2 = gen_lowpart (SImode, op2);
16002 232 : op1 = gen_lowpart (SImode, op1);
16003 232 : pat = GEN_FCN (icode) (op0, op1, op2);
16004 : }
16005 : else
16006 : {
16007 0 : switch (fcode)
16008 : {
16009 : case IX86_BUILTIN_XSAVE:
16010 : icode = CODE_FOR_xsave;
16011 : break;
16012 : case IX86_BUILTIN_XRSTOR:
16013 : icode = CODE_FOR_xrstor;
16014 : break;
16015 : case IX86_BUILTIN_XSAVEOPT:
16016 : icode = CODE_FOR_xsaveopt;
16017 : break;
16018 : case IX86_BUILTIN_XSAVES:
16019 : icode = CODE_FOR_xsaves;
16020 : break;
16021 : case IX86_BUILTIN_XRSTORS:
16022 : icode = CODE_FOR_xrstors;
16023 : break;
16024 : case IX86_BUILTIN_XSAVEC:
16025 : icode = CODE_FOR_xsavec;
16026 : break;
16027 0 : default:
16028 0 : gcc_unreachable ();
16029 : }
16030 0 : pat = GEN_FCN (icode) (op0, op1);
16031 : }
16032 :
16033 232 : if (pat)
16034 232 : emit_insn (pat);
16035 : return 0;
16036 :
16037 137 : case IX86_BUILTIN_LDTILECFG:
16038 137 : case IX86_BUILTIN_STTILECFG:
16039 137 : arg0 = CALL_EXPR_ARG (exp, 0);
16040 137 : op0 = expand_normal (arg0);
16041 :
16042 137 : if (!address_operand (op0, VOIDmode))
16043 : {
16044 8 : op0 = convert_memory_address (Pmode, op0);
16045 8 : op0 = copy_addr_to_reg (op0);
16046 : }
16047 137 : op0 = gen_rtx_MEM (BLKmode, op0);
16048 137 : if (fcode == IX86_BUILTIN_LDTILECFG)
16049 : icode = CODE_FOR_ldtilecfg;
16050 : else
16051 88 : icode = CODE_FOR_sttilecfg;
16052 137 : pat = GEN_FCN (icode) (op0);
16053 137 : emit_insn (pat);
16054 137 : return 0;
16055 :
16056 18 : case IX86_BUILTIN_LLWPCB:
16057 18 : arg0 = CALL_EXPR_ARG (exp, 0);
16058 18 : op0 = expand_normal (arg0);
16059 :
16060 18 : if (!register_operand (op0, Pmode))
16061 9 : op0 = ix86_zero_extend_to_Pmode (op0);
16062 18 : emit_insn (gen_lwp_llwpcb (Pmode, op0));
16063 18 : return 0;
16064 :
16065 18 : case IX86_BUILTIN_SLWPCB:
16066 18 : if (!target
16067 18 : || !register_operand (target, Pmode))
16068 0 : target = gen_reg_rtx (Pmode);
16069 18 : emit_insn (gen_lwp_slwpcb (Pmode, target));
16070 18 : return target;
16071 :
16072 51 : case IX86_BUILTIN_LWPVAL32:
16073 51 : case IX86_BUILTIN_LWPVAL64:
16074 51 : case IX86_BUILTIN_LWPINS32:
16075 51 : case IX86_BUILTIN_LWPINS64:
16076 51 : mode = ((fcode == IX86_BUILTIN_LWPVAL32
16077 51 : || fcode == IX86_BUILTIN_LWPINS32)
16078 51 : ? SImode : DImode);
16079 :
16080 51 : if (fcode == IX86_BUILTIN_LWPVAL32
16081 51 : || fcode == IX86_BUILTIN_LWPVAL64)
16082 26 : icode = code_for_lwp_lwpval (mode);
16083 : else
16084 25 : icode = code_for_lwp_lwpins (mode);
16085 :
16086 51 : arg0 = CALL_EXPR_ARG (exp, 0);
16087 51 : arg1 = CALL_EXPR_ARG (exp, 1);
16088 51 : arg2 = CALL_EXPR_ARG (exp, 2);
16089 51 : op0 = expand_normal (arg0);
16090 51 : op1 = expand_normal (arg1);
16091 51 : op2 = expand_normal (arg2);
16092 51 : mode0 = insn_data[icode].operand[0].mode;
16093 :
16094 51 : if (!insn_data[icode].operand[0].predicate (op0, mode0))
16095 13 : op0 = copy_to_mode_reg (mode0, op0);
16096 51 : if (!insn_data[icode].operand[1].predicate (op1, SImode))
16097 0 : op1 = copy_to_mode_reg (SImode, op1);
16098 :
16099 51 : if (!CONST_INT_P (op2))
16100 : {
16101 0 : error ("the last argument must be a 32-bit immediate");
16102 0 : return const0_rtx;
16103 : }
16104 :
16105 51 : emit_insn (GEN_FCN (icode) (op0, op1, op2));
16106 :
16107 51 : if (fcode == IX86_BUILTIN_LWPINS32
16108 51 : || fcode == IX86_BUILTIN_LWPINS64)
16109 : {
16110 25 : if (target == 0
16111 25 : || !nonimmediate_operand (target, QImode))
16112 0 : target = gen_reg_rtx (QImode);
16113 :
16114 25 : pat = gen_rtx_EQ (QImode, gen_rtx_REG (CCCmode, FLAGS_REG),
16115 : const0_rtx);
16116 25 : emit_insn (gen_rtx_SET (target, pat));
16117 :
16118 25 : return target;
16119 : }
16120 : else
16121 : return 0;
16122 :
16123 18 : case IX86_BUILTIN_BEXTRI32:
16124 18 : case IX86_BUILTIN_BEXTRI64:
16125 18 : mode = (fcode == IX86_BUILTIN_BEXTRI32 ? SImode : DImode);
16126 :
16127 18 : arg0 = CALL_EXPR_ARG (exp, 0);
16128 18 : arg1 = CALL_EXPR_ARG (exp, 1);
16129 18 : op0 = expand_normal (arg0);
16130 18 : op1 = expand_normal (arg1);
16131 :
16132 18 : if (!CONST_INT_P (op1))
16133 : {
16134 0 : error ("last argument must be an immediate");
16135 0 : return const0_rtx;
16136 : }
16137 : else
16138 : {
16139 18 : unsigned char lsb_index = UINTVAL (op1);
16140 18 : unsigned char length = UINTVAL (op1) >> 8;
16141 :
16142 18 : unsigned char bitsize = GET_MODE_BITSIZE (mode);
16143 :
16144 18 : icode = code_for_tbm_bextri (mode);
16145 :
16146 18 : mode1 = insn_data[icode].operand[1].mode;
16147 18 : if (!insn_data[icode].operand[1].predicate (op0, mode1))
16148 12 : op0 = copy_to_mode_reg (mode1, op0);
16149 :
16150 18 : mode0 = insn_data[icode].operand[0].mode;
16151 18 : if (target == 0
16152 18 : || !register_operand (target, mode0))
16153 0 : target = gen_reg_rtx (mode0);
16154 :
16155 18 : if (length == 0 || lsb_index >= bitsize)
16156 : {
16157 8 : emit_move_insn (target, const0_rtx);
16158 8 : return target;
16159 : }
16160 :
16161 10 : if (length + lsb_index > bitsize)
16162 5 : length = bitsize - lsb_index;
16163 :
16164 10 : op1 = GEN_INT (length);
16165 10 : op2 = GEN_INT (lsb_index);
16166 :
16167 10 : emit_insn (GEN_FCN (icode) (target, op0, op1, op2));
16168 10 : return target;
16169 : }
16170 :
16171 21 : case IX86_BUILTIN_RDRAND16_STEP:
16172 21 : mode = HImode;
16173 21 : goto rdrand_step;
16174 :
16175 42 : case IX86_BUILTIN_RDRAND32_STEP:
16176 42 : mode = SImode;
16177 42 : goto rdrand_step;
16178 :
16179 : case IX86_BUILTIN_RDRAND64_STEP:
16180 : mode = DImode;
16181 :
16182 83 : rdrand_step:
16183 83 : arg0 = CALL_EXPR_ARG (exp, 0);
16184 83 : op1 = expand_normal (arg0);
16185 83 : if (!address_operand (op1, VOIDmode))
16186 : {
16187 29 : op1 = convert_memory_address (Pmode, op1);
16188 29 : op1 = copy_addr_to_reg (op1);
16189 : }
16190 :
16191 83 : op0 = gen_reg_rtx (mode);
16192 83 : emit_insn (gen_rdrand (mode, op0));
16193 :
16194 83 : emit_move_insn (gen_rtx_MEM (mode, op1), op0);
16195 :
16196 83 : op1 = force_reg (SImode, const1_rtx);
16197 :
16198 : /* Emit SImode conditional move. */
16199 83 : if (mode == HImode)
16200 : {
16201 21 : if (TARGET_ZERO_EXTEND_WITH_AND
16202 21 : && optimize_function_for_speed_p (cfun))
16203 : {
16204 0 : op2 = force_reg (SImode, const0_rtx);
16205 :
16206 0 : emit_insn (gen_movstricthi
16207 0 : (gen_lowpart (HImode, op2), op0));
16208 : }
16209 : else
16210 : {
16211 21 : op2 = gen_reg_rtx (SImode);
16212 :
16213 21 : emit_insn (gen_zero_extendhisi2 (op2, op0));
16214 : }
16215 : }
16216 62 : else if (mode == SImode)
16217 : op2 = op0;
16218 : else
16219 20 : op2 = gen_rtx_SUBREG (SImode, op0, 0);
16220 :
16221 83 : if (target == 0
16222 83 : || !register_operand (target, SImode))
16223 7 : target = gen_reg_rtx (SImode);
16224 :
16225 83 : pat = gen_rtx_GEU (VOIDmode, gen_rtx_REG (CCCmode, FLAGS_REG),
16226 : const0_rtx);
16227 83 : emit_insn (gen_rtx_SET (target,
16228 : gen_rtx_IF_THEN_ELSE (SImode, pat, op2, op1)));
16229 83 : return target;
16230 :
16231 19 : case IX86_BUILTIN_RDSEED16_STEP:
16232 19 : mode = HImode;
16233 19 : goto rdseed_step;
16234 :
16235 28 : case IX86_BUILTIN_RDSEED32_STEP:
16236 28 : mode = SImode;
16237 28 : goto rdseed_step;
16238 :
16239 : case IX86_BUILTIN_RDSEED64_STEP:
16240 : mode = DImode;
16241 :
16242 66 : rdseed_step:
16243 66 : arg0 = CALL_EXPR_ARG (exp, 0);
16244 66 : op1 = expand_normal (arg0);
16245 66 : if (!address_operand (op1, VOIDmode))
16246 : {
16247 28 : op1 = convert_memory_address (Pmode, op1);
16248 28 : op1 = copy_addr_to_reg (op1);
16249 : }
16250 :
16251 66 : op0 = gen_reg_rtx (mode);
16252 66 : emit_insn (gen_rdseed (mode, op0));
16253 :
16254 66 : emit_move_insn (gen_rtx_MEM (mode, op1), op0);
16255 :
16256 66 : op2 = gen_reg_rtx (QImode);
16257 :
16258 66 : pat = gen_rtx_LTU (QImode, gen_rtx_REG (CCCmode, FLAGS_REG),
16259 : const0_rtx);
16260 66 : emit_insn (gen_rtx_SET (op2, pat));
16261 :
16262 66 : if (target == 0
16263 66 : || !register_operand (target, SImode))
16264 1 : target = gen_reg_rtx (SImode);
16265 :
16266 66 : emit_insn (gen_zero_extendqisi2 (target, op2));
16267 66 : return target;
16268 :
16269 38 : case IX86_BUILTIN_SBB32:
16270 38 : icode = CODE_FOR_subborrowsi;
16271 38 : icode2 = CODE_FOR_subborrowsi_0;
16272 38 : mode0 = SImode;
16273 38 : mode1 = DImode;
16274 38 : mode2 = CCmode;
16275 38 : goto handlecarry;
16276 :
16277 44 : case IX86_BUILTIN_SBB64:
16278 44 : icode = CODE_FOR_subborrowdi;
16279 44 : icode2 = CODE_FOR_subborrowdi_0;
16280 44 : mode0 = DImode;
16281 44 : mode1 = TImode;
16282 44 : mode2 = CCmode;
16283 44 : goto handlecarry;
16284 :
16285 68 : case IX86_BUILTIN_ADDCARRYX32:
16286 68 : icode = CODE_FOR_addcarrysi;
16287 68 : icode2 = CODE_FOR_addcarrysi_0;
16288 68 : mode0 = SImode;
16289 68 : mode1 = DImode;
16290 68 : mode2 = CCCmode;
16291 68 : goto handlecarry;
16292 :
16293 : case IX86_BUILTIN_ADDCARRYX64:
16294 : icode = CODE_FOR_addcarrydi;
16295 : icode2 = CODE_FOR_addcarrydi_0;
16296 : mode0 = DImode;
16297 : mode1 = TImode;
16298 : mode2 = CCCmode;
16299 :
16300 212 : handlecarry:
16301 212 : arg0 = CALL_EXPR_ARG (exp, 0); /* unsigned char c_in. */
16302 212 : arg1 = CALL_EXPR_ARG (exp, 1); /* unsigned int src1. */
16303 212 : arg2 = CALL_EXPR_ARG (exp, 2); /* unsigned int src2. */
16304 212 : arg3 = CALL_EXPR_ARG (exp, 3); /* unsigned int *sum_out. */
16305 :
16306 212 : op1 = expand_normal (arg0);
16307 :
16308 212 : op2 = expand_normal (arg1);
16309 212 : if (!register_operand (op2, mode0))
16310 117 : op2 = copy_to_mode_reg (mode0, op2);
16311 :
16312 212 : op3 = expand_normal (arg2);
16313 212 : if (!register_operand (op3, mode0))
16314 120 : op3 = copy_to_mode_reg (mode0, op3);
16315 :
16316 212 : op4 = expand_normal (arg3);
16317 212 : if (!address_operand (op4, VOIDmode))
16318 : {
16319 67 : op4 = convert_memory_address (Pmode, op4);
16320 67 : op4 = copy_addr_to_reg (op4);
16321 : }
16322 :
16323 212 : op0 = gen_reg_rtx (mode0);
16324 212 : if (op1 == const0_rtx)
16325 : {
16326 : /* If arg0 is 0, optimize right away into add or sub
16327 : instruction that sets CCCmode flags. */
16328 21 : op1 = gen_rtx_REG (mode2, FLAGS_REG);
16329 21 : emit_insn (GEN_FCN (icode2) (op0, op2, op3));
16330 : }
16331 : else
16332 : {
16333 : /* Generate CF from input operand. */
16334 191 : ix86_expand_carry (op1);
16335 :
16336 : /* Generate instruction that consumes CF. */
16337 191 : op1 = gen_rtx_REG (CCCmode, FLAGS_REG);
16338 191 : pat = gen_rtx_LTU (mode1, op1, const0_rtx);
16339 191 : pat2 = gen_rtx_LTU (mode0, op1, const0_rtx);
16340 191 : emit_insn (GEN_FCN (icode) (op0, op2, op3, op1, pat, pat2));
16341 : }
16342 :
16343 : /* Return current CF value. */
16344 212 : if (target == 0)
16345 14 : target = gen_reg_rtx (QImode);
16346 :
16347 212 : pat = gen_rtx_LTU (QImode, op1, const0_rtx);
16348 212 : emit_insn (gen_rtx_SET (target, pat));
16349 :
16350 : /* Store the result. */
16351 212 : emit_move_insn (gen_rtx_MEM (mode0, op4), op0);
16352 :
16353 212 : return target;
16354 :
16355 24 : case IX86_BUILTIN_READ_FLAGS:
16356 24 : if (ignore)
16357 3 : return const0_rtx;
16358 :
16359 21 : emit_insn (gen_pushfl ());
16360 :
16361 21 : if (optimize
16362 11 : || target == NULL_RTX
16363 11 : || !nonimmediate_operand (target, word_mode)
16364 32 : || GET_MODE (target) != word_mode)
16365 10 : target = gen_reg_rtx (word_mode);
16366 :
16367 21 : emit_insn (gen_pop (target));
16368 21 : return target;
16369 :
16370 21 : case IX86_BUILTIN_WRITE_FLAGS:
16371 :
16372 21 : arg0 = CALL_EXPR_ARG (exp, 0);
16373 21 : op0 = expand_normal (arg0);
16374 21 : if (!general_no_elim_operand (op0, word_mode))
16375 0 : op0 = copy_to_mode_reg (word_mode, op0);
16376 :
16377 21 : emit_insn (gen_push (op0));
16378 21 : emit_insn (gen_popfl ());
16379 21 : return 0;
16380 :
16381 22 : case IX86_BUILTIN_KTESTC8:
16382 22 : icode = CODE_FOR_ktestqi;
16383 22 : mode3 = CCCmode;
16384 22 : goto kortest;
16385 :
16386 22 : case IX86_BUILTIN_KTESTZ8:
16387 22 : icode = CODE_FOR_ktestqi;
16388 22 : mode3 = CCZmode;
16389 22 : goto kortest;
16390 :
16391 22 : case IX86_BUILTIN_KTESTC16:
16392 22 : icode = CODE_FOR_ktesthi;
16393 22 : mode3 = CCCmode;
16394 22 : goto kortest;
16395 :
16396 22 : case IX86_BUILTIN_KTESTZ16:
16397 22 : icode = CODE_FOR_ktesthi;
16398 22 : mode3 = CCZmode;
16399 22 : goto kortest;
16400 :
16401 22 : case IX86_BUILTIN_KTESTC32:
16402 22 : icode = CODE_FOR_ktestsi;
16403 22 : mode3 = CCCmode;
16404 22 : goto kortest;
16405 :
16406 22 : case IX86_BUILTIN_KTESTZ32:
16407 22 : icode = CODE_FOR_ktestsi;
16408 22 : mode3 = CCZmode;
16409 22 : goto kortest;
16410 :
16411 22 : case IX86_BUILTIN_KTESTC64:
16412 22 : icode = CODE_FOR_ktestdi;
16413 22 : mode3 = CCCmode;
16414 22 : goto kortest;
16415 :
16416 22 : case IX86_BUILTIN_KTESTZ64:
16417 22 : icode = CODE_FOR_ktestdi;
16418 22 : mode3 = CCZmode;
16419 22 : goto kortest;
16420 :
16421 22 : case IX86_BUILTIN_KORTESTC8:
16422 22 : icode = CODE_FOR_kortestqi;
16423 22 : mode3 = CCCmode;
16424 22 : goto kortest;
16425 :
16426 76 : case IX86_BUILTIN_KORTESTZ8:
16427 76 : icode = CODE_FOR_kortestqi;
16428 76 : mode3 = CCZmode;
16429 76 : goto kortest;
16430 :
16431 38 : case IX86_BUILTIN_KORTESTC16:
16432 38 : icode = CODE_FOR_kortesthi;
16433 38 : mode3 = CCCmode;
16434 38 : goto kortest;
16435 :
16436 91 : case IX86_BUILTIN_KORTESTZ16:
16437 91 : icode = CODE_FOR_kortesthi;
16438 91 : mode3 = CCZmode;
16439 91 : goto kortest;
16440 :
16441 22 : case IX86_BUILTIN_KORTESTC32:
16442 22 : icode = CODE_FOR_kortestsi;
16443 22 : mode3 = CCCmode;
16444 22 : goto kortest;
16445 :
16446 79 : case IX86_BUILTIN_KORTESTZ32:
16447 79 : icode = CODE_FOR_kortestsi;
16448 79 : mode3 = CCZmode;
16449 79 : goto kortest;
16450 :
16451 22 : case IX86_BUILTIN_KORTESTC64:
16452 22 : icode = CODE_FOR_kortestdi;
16453 22 : mode3 = CCCmode;
16454 22 : goto kortest;
16455 :
16456 : case IX86_BUILTIN_KORTESTZ64:
16457 : icode = CODE_FOR_kortestdi;
16458 : mode3 = CCZmode;
16459 :
16460 610 : kortest:
16461 610 : arg0 = CALL_EXPR_ARG (exp, 0); /* Mask reg src1. */
16462 610 : arg1 = CALL_EXPR_ARG (exp, 1); /* Mask reg src2. */
16463 610 : op0 = expand_normal (arg0);
16464 610 : op1 = expand_normal (arg1);
16465 :
16466 610 : mode0 = insn_data[icode].operand[0].mode;
16467 610 : mode1 = insn_data[icode].operand[1].mode;
16468 :
16469 610 : if (GET_MODE (op0) != VOIDmode)
16470 610 : op0 = force_reg (GET_MODE (op0), op0);
16471 :
16472 610 : op0 = gen_lowpart (mode0, op0);
16473 :
16474 610 : if (!insn_data[icode].operand[0].predicate (op0, mode0))
16475 0 : op0 = copy_to_mode_reg (mode0, op0);
16476 :
16477 610 : if (GET_MODE (op1) != VOIDmode)
16478 609 : op1 = force_reg (GET_MODE (op1), op1);
16479 :
16480 610 : op1 = gen_lowpart (mode1, op1);
16481 :
16482 610 : if (!insn_data[icode].operand[1].predicate (op1, mode1))
16483 1 : op1 = copy_to_mode_reg (mode1, op1);
16484 :
16485 610 : target = gen_reg_rtx (QImode);
16486 :
16487 : /* Emit kortest. */
16488 610 : emit_insn (GEN_FCN (icode) (op0, op1));
16489 : /* And use setcc to return result from flags. */
16490 610 : ix86_expand_setcc (target, EQ,
16491 : gen_rtx_REG (mode3, FLAGS_REG), const0_rtx);
16492 610 : return target;
16493 :
16494 24 : case IX86_BUILTIN_GATHERSIV2DF:
16495 24 : icode = CODE_FOR_avx2_gathersiv2df;
16496 24 : goto gather_gen;
16497 18 : case IX86_BUILTIN_GATHERSIV4DF:
16498 18 : icode = CODE_FOR_avx2_gathersiv4df;
16499 18 : goto gather_gen;
16500 21 : case IX86_BUILTIN_GATHERDIV2DF:
16501 21 : icode = CODE_FOR_avx2_gatherdiv2df;
16502 21 : goto gather_gen;
16503 32 : case IX86_BUILTIN_GATHERDIV4DF:
16504 32 : icode = CODE_FOR_avx2_gatherdiv4df;
16505 32 : goto gather_gen;
16506 30 : case IX86_BUILTIN_GATHERSIV4SF:
16507 30 : icode = CODE_FOR_avx2_gathersiv4sf;
16508 30 : goto gather_gen;
16509 37 : case IX86_BUILTIN_GATHERSIV8SF:
16510 37 : icode = CODE_FOR_avx2_gathersiv8sf;
16511 37 : goto gather_gen;
16512 24 : case IX86_BUILTIN_GATHERDIV4SF:
16513 24 : icode = CODE_FOR_avx2_gatherdiv4sf;
16514 24 : goto gather_gen;
16515 18 : case IX86_BUILTIN_GATHERDIV8SF:
16516 18 : icode = CODE_FOR_avx2_gatherdiv8sf;
16517 18 : goto gather_gen;
16518 18 : case IX86_BUILTIN_GATHERSIV2DI:
16519 18 : icode = CODE_FOR_avx2_gathersiv2di;
16520 18 : goto gather_gen;
16521 18 : case IX86_BUILTIN_GATHERSIV4DI:
16522 18 : icode = CODE_FOR_avx2_gathersiv4di;
16523 18 : goto gather_gen;
16524 27 : case IX86_BUILTIN_GATHERDIV2DI:
16525 27 : icode = CODE_FOR_avx2_gatherdiv2di;
16526 27 : goto gather_gen;
16527 29 : case IX86_BUILTIN_GATHERDIV4DI:
16528 29 : icode = CODE_FOR_avx2_gatherdiv4di;
16529 29 : goto gather_gen;
16530 20 : case IX86_BUILTIN_GATHERSIV4SI:
16531 20 : icode = CODE_FOR_avx2_gathersiv4si;
16532 20 : goto gather_gen;
16533 22 : case IX86_BUILTIN_GATHERSIV8SI:
16534 22 : icode = CODE_FOR_avx2_gathersiv8si;
16535 22 : goto gather_gen;
16536 28 : case IX86_BUILTIN_GATHERDIV4SI:
16537 28 : icode = CODE_FOR_avx2_gatherdiv4si;
16538 28 : goto gather_gen;
16539 18 : case IX86_BUILTIN_GATHERDIV8SI:
16540 18 : icode = CODE_FOR_avx2_gatherdiv8si;
16541 18 : goto gather_gen;
16542 20 : case IX86_BUILTIN_GATHERALTSIV4DF:
16543 20 : icode = CODE_FOR_avx2_gathersiv4df;
16544 20 : goto gather_gen;
16545 16 : case IX86_BUILTIN_GATHERALTDIV8SF:
16546 16 : icode = CODE_FOR_avx2_gatherdiv8sf;
16547 16 : goto gather_gen;
16548 4 : case IX86_BUILTIN_GATHERALTSIV4DI:
16549 4 : icode = CODE_FOR_avx2_gathersiv4di;
16550 4 : goto gather_gen;
16551 12 : case IX86_BUILTIN_GATHERALTDIV8SI:
16552 12 : icode = CODE_FOR_avx2_gatherdiv8si;
16553 12 : goto gather_gen;
16554 36 : case IX86_BUILTIN_GATHER3SIV16SF:
16555 36 : icode = CODE_FOR_avx512f_gathersiv16sf;
16556 36 : goto gather_gen;
16557 24 : case IX86_BUILTIN_GATHER3SIV8DF:
16558 24 : icode = CODE_FOR_avx512f_gathersiv8df;
16559 24 : goto gather_gen;
16560 24 : case IX86_BUILTIN_GATHER3DIV16SF:
16561 24 : icode = CODE_FOR_avx512f_gatherdiv16sf;
16562 24 : goto gather_gen;
16563 37 : case IX86_BUILTIN_GATHER3DIV8DF:
16564 37 : icode = CODE_FOR_avx512f_gatherdiv8df;
16565 37 : goto gather_gen;
16566 30 : case IX86_BUILTIN_GATHER3SIV16SI:
16567 30 : icode = CODE_FOR_avx512f_gathersiv16si;
16568 30 : goto gather_gen;
16569 24 : case IX86_BUILTIN_GATHER3SIV8DI:
16570 24 : icode = CODE_FOR_avx512f_gathersiv8di;
16571 24 : goto gather_gen;
16572 24 : case IX86_BUILTIN_GATHER3DIV16SI:
16573 24 : icode = CODE_FOR_avx512f_gatherdiv16si;
16574 24 : goto gather_gen;
16575 38 : case IX86_BUILTIN_GATHER3DIV8DI:
16576 38 : icode = CODE_FOR_avx512f_gatherdiv8di;
16577 38 : goto gather_gen;
16578 16 : case IX86_BUILTIN_GATHER3ALTSIV8DF:
16579 16 : icode = CODE_FOR_avx512f_gathersiv8df;
16580 16 : goto gather_gen;
16581 22 : case IX86_BUILTIN_GATHER3ALTDIV16SF:
16582 22 : icode = CODE_FOR_avx512f_gatherdiv16sf;
16583 22 : goto gather_gen;
16584 14 : case IX86_BUILTIN_GATHER3ALTSIV8DI:
16585 14 : icode = CODE_FOR_avx512f_gathersiv8di;
16586 14 : goto gather_gen;
16587 18 : case IX86_BUILTIN_GATHER3ALTDIV16SI:
16588 18 : icode = CODE_FOR_avx512f_gatherdiv16si;
16589 18 : goto gather_gen;
16590 18 : case IX86_BUILTIN_GATHER3SIV2DF:
16591 18 : icode = CODE_FOR_avx512vl_gathersiv2df;
16592 18 : goto gather_gen;
16593 10 : case IX86_BUILTIN_GATHER3SIV4DF:
16594 10 : icode = CODE_FOR_avx512vl_gathersiv4df;
16595 10 : goto gather_gen;
16596 15 : case IX86_BUILTIN_GATHER3DIV2DF:
16597 15 : icode = CODE_FOR_avx512vl_gatherdiv2df;
16598 15 : goto gather_gen;
16599 16 : case IX86_BUILTIN_GATHER3DIV4DF:
16600 16 : icode = CODE_FOR_avx512vl_gatherdiv4df;
16601 16 : goto gather_gen;
16602 14 : case IX86_BUILTIN_GATHER3SIV4SF:
16603 14 : icode = CODE_FOR_avx512vl_gathersiv4sf;
16604 14 : goto gather_gen;
16605 12 : case IX86_BUILTIN_GATHER3SIV8SF:
16606 12 : icode = CODE_FOR_avx512vl_gathersiv8sf;
16607 12 : goto gather_gen;
16608 22 : case IX86_BUILTIN_GATHER3DIV4SF:
16609 22 : icode = CODE_FOR_avx512vl_gatherdiv4sf;
16610 22 : goto gather_gen;
16611 10 : case IX86_BUILTIN_GATHER3DIV8SF:
16612 10 : icode = CODE_FOR_avx512vl_gatherdiv8sf;
16613 10 : goto gather_gen;
16614 20 : case IX86_BUILTIN_GATHER3SIV2DI:
16615 20 : icode = CODE_FOR_avx512vl_gathersiv2di;
16616 20 : goto gather_gen;
16617 10 : case IX86_BUILTIN_GATHER3SIV4DI:
16618 10 : icode = CODE_FOR_avx512vl_gathersiv4di;
16619 10 : goto gather_gen;
16620 15 : case IX86_BUILTIN_GATHER3DIV2DI:
16621 15 : icode = CODE_FOR_avx512vl_gatherdiv2di;
16622 15 : goto gather_gen;
16623 14 : case IX86_BUILTIN_GATHER3DIV4DI:
16624 14 : icode = CODE_FOR_avx512vl_gatherdiv4di;
16625 14 : goto gather_gen;
16626 14 : case IX86_BUILTIN_GATHER3SIV4SI:
16627 14 : icode = CODE_FOR_avx512vl_gathersiv4si;
16628 14 : goto gather_gen;
16629 12 : case IX86_BUILTIN_GATHER3SIV8SI:
16630 12 : icode = CODE_FOR_avx512vl_gathersiv8si;
16631 12 : goto gather_gen;
16632 24 : case IX86_BUILTIN_GATHER3DIV4SI:
16633 24 : icode = CODE_FOR_avx512vl_gatherdiv4si;
16634 24 : goto gather_gen;
16635 10 : case IX86_BUILTIN_GATHER3DIV8SI:
16636 10 : icode = CODE_FOR_avx512vl_gatherdiv8si;
16637 10 : goto gather_gen;
16638 4 : case IX86_BUILTIN_GATHER3ALTSIV4DF:
16639 4 : icode = CODE_FOR_avx512vl_gathersiv4df;
16640 4 : goto gather_gen;
16641 8 : case IX86_BUILTIN_GATHER3ALTDIV8SF:
16642 8 : icode = CODE_FOR_avx512vl_gatherdiv8sf;
16643 8 : goto gather_gen;
16644 6 : case IX86_BUILTIN_GATHER3ALTSIV4DI:
16645 6 : icode = CODE_FOR_avx512vl_gathersiv4di;
16646 6 : goto gather_gen;
16647 10 : case IX86_BUILTIN_GATHER3ALTDIV8SI:
16648 10 : icode = CODE_FOR_avx512vl_gatherdiv8si;
16649 10 : goto gather_gen;
16650 40 : case IX86_BUILTIN_SCATTERSIV16SF:
16651 40 : icode = CODE_FOR_avx512f_scattersiv16sf;
16652 40 : goto scatter_gen;
16653 27 : case IX86_BUILTIN_SCATTERSIV8DF:
16654 27 : icode = CODE_FOR_avx512f_scattersiv8df;
16655 27 : goto scatter_gen;
16656 24 : case IX86_BUILTIN_SCATTERDIV16SF:
16657 24 : icode = CODE_FOR_avx512f_scatterdiv16sf;
16658 24 : goto scatter_gen;
16659 33 : case IX86_BUILTIN_SCATTERDIV8DF:
16660 33 : icode = CODE_FOR_avx512f_scatterdiv8df;
16661 33 : goto scatter_gen;
16662 30 : case IX86_BUILTIN_SCATTERSIV16SI:
16663 30 : icode = CODE_FOR_avx512f_scattersiv16si;
16664 30 : goto scatter_gen;
16665 24 : case IX86_BUILTIN_SCATTERSIV8DI:
16666 24 : icode = CODE_FOR_avx512f_scattersiv8di;
16667 24 : goto scatter_gen;
16668 24 : case IX86_BUILTIN_SCATTERDIV16SI:
16669 24 : icode = CODE_FOR_avx512f_scatterdiv16si;
16670 24 : goto scatter_gen;
16671 29 : case IX86_BUILTIN_SCATTERDIV8DI:
16672 29 : icode = CODE_FOR_avx512f_scatterdiv8di;
16673 29 : goto scatter_gen;
16674 18 : case IX86_BUILTIN_SCATTERSIV8SF:
16675 18 : icode = CODE_FOR_avx512vl_scattersiv8sf;
16676 18 : goto scatter_gen;
16677 20 : case IX86_BUILTIN_SCATTERSIV4SF:
16678 20 : icode = CODE_FOR_avx512vl_scattersiv4sf;
16679 20 : goto scatter_gen;
16680 16 : case IX86_BUILTIN_SCATTERSIV4DF:
16681 16 : icode = CODE_FOR_avx512vl_scattersiv4df;
16682 16 : goto scatter_gen;
16683 16 : case IX86_BUILTIN_SCATTERSIV2DF:
16684 16 : icode = CODE_FOR_avx512vl_scattersiv2df;
16685 16 : goto scatter_gen;
16686 16 : case IX86_BUILTIN_SCATTERDIV8SF:
16687 16 : icode = CODE_FOR_avx512vl_scatterdiv8sf;
16688 16 : goto scatter_gen;
16689 16 : case IX86_BUILTIN_SCATTERDIV4SF:
16690 16 : icode = CODE_FOR_avx512vl_scatterdiv4sf;
16691 16 : goto scatter_gen;
16692 18 : case IX86_BUILTIN_SCATTERDIV4DF:
16693 18 : icode = CODE_FOR_avx512vl_scatterdiv4df;
16694 18 : goto scatter_gen;
16695 18 : case IX86_BUILTIN_SCATTERDIV2DF:
16696 18 : icode = CODE_FOR_avx512vl_scatterdiv2df;
16697 18 : goto scatter_gen;
16698 22 : case IX86_BUILTIN_SCATTERSIV8SI:
16699 22 : icode = CODE_FOR_avx512vl_scattersiv8si;
16700 22 : goto scatter_gen;
16701 24 : case IX86_BUILTIN_SCATTERSIV4SI:
16702 24 : icode = CODE_FOR_avx512vl_scattersiv4si;
16703 24 : goto scatter_gen;
16704 16 : case IX86_BUILTIN_SCATTERSIV4DI:
16705 16 : icode = CODE_FOR_avx512vl_scattersiv4di;
16706 16 : goto scatter_gen;
16707 16 : case IX86_BUILTIN_SCATTERSIV2DI:
16708 16 : icode = CODE_FOR_avx512vl_scattersiv2di;
16709 16 : goto scatter_gen;
16710 16 : case IX86_BUILTIN_SCATTERDIV8SI:
16711 16 : icode = CODE_FOR_avx512vl_scatterdiv8si;
16712 16 : goto scatter_gen;
16713 16 : case IX86_BUILTIN_SCATTERDIV4SI:
16714 16 : icode = CODE_FOR_avx512vl_scatterdiv4si;
16715 16 : goto scatter_gen;
16716 18 : case IX86_BUILTIN_SCATTERDIV4DI:
16717 18 : icode = CODE_FOR_avx512vl_scatterdiv4di;
16718 18 : goto scatter_gen;
16719 18 : case IX86_BUILTIN_SCATTERDIV2DI:
16720 18 : icode = CODE_FOR_avx512vl_scatterdiv2di;
16721 18 : goto scatter_gen;
16722 16 : case IX86_BUILTIN_SCATTERALTSIV8DF:
16723 16 : icode = CODE_FOR_avx512f_scattersiv8df;
16724 16 : goto scatter_gen;
16725 12 : case IX86_BUILTIN_SCATTERALTDIV16SF:
16726 12 : icode = CODE_FOR_avx512f_scatterdiv16sf;
16727 12 : goto scatter_gen;
16728 8 : case IX86_BUILTIN_SCATTERALTSIV8DI:
16729 8 : icode = CODE_FOR_avx512f_scattersiv8di;
16730 8 : goto scatter_gen;
16731 24 : case IX86_BUILTIN_SCATTERALTDIV16SI:
16732 24 : icode = CODE_FOR_avx512f_scatterdiv16si;
16733 24 : goto scatter_gen;
16734 4 : case IX86_BUILTIN_SCATTERALTSIV4DF:
16735 4 : icode = CODE_FOR_avx512vl_scattersiv4df;
16736 4 : goto scatter_gen;
16737 4 : case IX86_BUILTIN_SCATTERALTDIV8SF:
16738 4 : icode = CODE_FOR_avx512vl_scatterdiv8sf;
16739 4 : goto scatter_gen;
16740 4 : case IX86_BUILTIN_SCATTERALTSIV4DI:
16741 4 : icode = CODE_FOR_avx512vl_scattersiv4di;
16742 4 : goto scatter_gen;
16743 4 : case IX86_BUILTIN_SCATTERALTDIV8SI:
16744 4 : icode = CODE_FOR_avx512vl_scatterdiv8si;
16745 4 : goto scatter_gen;
16746 8 : case IX86_BUILTIN_SCATTERALTSIV2DF:
16747 8 : icode = CODE_FOR_avx512vl_scattersiv2df;
16748 8 : goto scatter_gen;
16749 8 : case IX86_BUILTIN_SCATTERALTDIV4SF:
16750 8 : icode = CODE_FOR_avx512vl_scatterdiv4sf;
16751 8 : goto scatter_gen;
16752 8 : case IX86_BUILTIN_SCATTERALTSIV2DI:
16753 8 : icode = CODE_FOR_avx512vl_scattersiv2di;
16754 8 : goto scatter_gen;
16755 8 : case IX86_BUILTIN_SCATTERALTDIV4SI:
16756 8 : icode = CODE_FOR_avx512vl_scatterdiv4si;
16757 8 : goto scatter_gen;
16758 :
16759 1007 : gather_gen:
16760 1007 : rtx half;
16761 1007 : rtx (*gen) (rtx, rtx);
16762 :
16763 1007 : arg0 = CALL_EXPR_ARG (exp, 0);
16764 1007 : arg1 = CALL_EXPR_ARG (exp, 1);
16765 1007 : arg2 = CALL_EXPR_ARG (exp, 2);
16766 1007 : arg3 = CALL_EXPR_ARG (exp, 3);
16767 1007 : arg4 = CALL_EXPR_ARG (exp, 4);
16768 1007 : op0 = expand_normal (arg0);
16769 1007 : op1 = expand_normal (arg1);
16770 1007 : op2 = expand_normal (arg2);
16771 1007 : op3 = ix86_expand_unsigned_small_int_cst_argument (arg3);
16772 1007 : op4 = expand_normal (arg4);
16773 : /* Note the arg order is different from the operand order. */
16774 1007 : mode0 = insn_data[icode].operand[1].mode;
16775 1007 : mode2 = insn_data[icode].operand[3].mode;
16776 1007 : mode3 = insn_data[icode].operand[4].mode;
16777 1007 : mode4 = insn_data[icode].operand[5].mode;
16778 :
16779 1007 : if (target == NULL_RTX
16780 1007 : || GET_MODE (target) != insn_data[icode].operand[0].mode
16781 1910 : || !insn_data[icode].operand[0].predicate (target,
16782 : GET_MODE (target)))
16783 105 : subtarget = gen_reg_rtx (insn_data[icode].operand[0].mode);
16784 : else
16785 : subtarget = target;
16786 :
16787 1007 : switch (fcode)
16788 : {
16789 30 : case IX86_BUILTIN_GATHER3ALTSIV8DF:
16790 30 : case IX86_BUILTIN_GATHER3ALTSIV8DI:
16791 30 : half = gen_reg_rtx (V8SImode);
16792 30 : if (!nonimmediate_operand (op2, V16SImode))
16793 0 : op2 = copy_to_mode_reg (V16SImode, op2);
16794 30 : emit_insn (gen_vec_extract_lo_v16si (half, op2));
16795 30 : op2 = half;
16796 30 : break;
16797 34 : case IX86_BUILTIN_GATHER3ALTSIV4DF:
16798 34 : case IX86_BUILTIN_GATHER3ALTSIV4DI:
16799 34 : case IX86_BUILTIN_GATHERALTSIV4DF:
16800 34 : case IX86_BUILTIN_GATHERALTSIV4DI:
16801 34 : half = gen_reg_rtx (V4SImode);
16802 34 : if (!nonimmediate_operand (op2, V8SImode))
16803 0 : op2 = copy_to_mode_reg (V8SImode, op2);
16804 34 : emit_insn (gen_vec_extract_lo_v8si (half, op2));
16805 34 : op2 = half;
16806 34 : break;
16807 40 : case IX86_BUILTIN_GATHER3ALTDIV16SF:
16808 40 : case IX86_BUILTIN_GATHER3ALTDIV16SI:
16809 40 : half = gen_reg_rtx (mode0);
16810 40 : if (mode0 == V8SFmode)
16811 : gen = gen_vec_extract_lo_v16sf;
16812 : else
16813 18 : gen = gen_vec_extract_lo_v16si;
16814 40 : if (!nonimmediate_operand (op0, GET_MODE (op0)))
16815 40 : op0 = copy_to_mode_reg (GET_MODE (op0), op0);
16816 40 : emit_insn (gen (half, op0));
16817 40 : op0 = half;
16818 40 : op3 = lowpart_subreg (QImode, op3, HImode);
16819 40 : break;
16820 46 : case IX86_BUILTIN_GATHER3ALTDIV8SF:
16821 46 : case IX86_BUILTIN_GATHER3ALTDIV8SI:
16822 46 : case IX86_BUILTIN_GATHERALTDIV8SF:
16823 46 : case IX86_BUILTIN_GATHERALTDIV8SI:
16824 46 : half = gen_reg_rtx (mode0);
16825 46 : if (mode0 == V4SFmode)
16826 : gen = gen_vec_extract_lo_v8sf;
16827 : else
16828 22 : gen = gen_vec_extract_lo_v8si;
16829 46 : if (!nonimmediate_operand (op0, GET_MODE (op0)))
16830 46 : op0 = copy_to_mode_reg (GET_MODE (op0), op0);
16831 46 : emit_insn (gen (half, op0));
16832 46 : op0 = half;
16833 46 : if (VECTOR_MODE_P (GET_MODE (op3)))
16834 : {
16835 28 : half = gen_reg_rtx (mode0);
16836 28 : if (!nonimmediate_operand (op3, GET_MODE (op3)))
16837 12 : op3 = copy_to_mode_reg (GET_MODE (op3), op3);
16838 28 : emit_insn (gen (half, op3));
16839 28 : op3 = half;
16840 : }
16841 : break;
16842 : default:
16843 : break;
16844 : }
16845 :
16846 : /* Force memory operand only with base register here. But we
16847 : don't want to do it on memory operand for other builtin
16848 : functions. */
16849 1007 : op1 = ix86_zero_extend_to_Pmode (op1);
16850 :
16851 1007 : if (!insn_data[icode].operand[1].predicate (op0, mode0))
16852 406 : op0 = copy_to_mode_reg (mode0, op0);
16853 1012 : if (!insn_data[icode].operand[2].predicate (op1, Pmode))
16854 0 : op1 = copy_to_mode_reg (Pmode, op1);
16855 1007 : if (!insn_data[icode].operand[3].predicate (op2, mode2))
16856 224 : op2 = copy_to_mode_reg (mode2, op2);
16857 :
16858 1007 : op3 = fixup_modeless_constant (op3, mode3);
16859 :
16860 1007 : if (GET_MODE (op3) == mode3 || GET_MODE (op3) == VOIDmode)
16861 : {
16862 1007 : if (!insn_data[icode].operand[4].predicate (op3, mode3))
16863 356 : op3 = copy_to_mode_reg (mode3, op3);
16864 : }
16865 : else
16866 : {
16867 0 : op3 = copy_to_reg (op3);
16868 0 : op3 = lowpart_subreg (mode3, op3, GET_MODE (op3));
16869 : }
16870 1007 : if (!insn_data[icode].operand[5].predicate (op4, mode4))
16871 : {
16872 0 : error ("the last argument must be scale 1, 2, 4, 8");
16873 0 : return const0_rtx;
16874 : }
16875 :
16876 : /* Optimize. If mask is known to have all high bits set,
16877 : replace op0 with pc_rtx to signal that the instruction
16878 : overwrites the whole destination and doesn't use its
16879 : previous contents. */
16880 1007 : if (optimize)
16881 : {
16882 917 : if (TREE_CODE (arg3) == INTEGER_CST)
16883 : {
16884 209 : if (integer_all_onesp (arg3))
16885 201 : op0 = pc_rtx;
16886 : }
16887 708 : else if (TREE_CODE (arg3) == VECTOR_CST)
16888 : {
16889 : unsigned int negative = 0;
16890 755 : for (i = 0; i < VECTOR_CST_NELTS (arg3); ++i)
16891 : {
16892 620 : tree cst = VECTOR_CST_ELT (arg3, i);
16893 620 : if (TREE_CODE (cst) == INTEGER_CST
16894 620 : && tree_int_cst_sign_bit (cst))
16895 286 : negative++;
16896 334 : else if (TREE_CODE (cst) == REAL_CST
16897 334 : && REAL_VALUE_NEGATIVE (TREE_REAL_CST (cst)))
16898 306 : negative++;
16899 : }
16900 135 : if (negative == TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg3)))
16901 121 : op0 = pc_rtx;
16902 : }
16903 573 : else if (TREE_CODE (arg3) == SSA_NAME
16904 573 : && VECTOR_TYPE_P (TREE_TYPE (arg3)))
16905 : {
16906 : /* Recognize also when mask is like:
16907 : __v2df src = _mm_setzero_pd ();
16908 : __v2df mask = _mm_cmpeq_pd (src, src);
16909 : or
16910 : __v8sf src = _mm256_setzero_ps ();
16911 : __v8sf mask = _mm256_cmp_ps (src, src, _CMP_EQ_OQ);
16912 : as that is a cheaper way to load all ones into
16913 : a register than having to load a constant from
16914 : memory. */
16915 259 : gimple *def_stmt = SSA_NAME_DEF_STMT (arg3);
16916 259 : if (is_gimple_call (def_stmt))
16917 : {
16918 76 : tree fndecl = gimple_call_fndecl (def_stmt);
16919 76 : if (fndecl
16920 76 : && fndecl_built_in_p (fndecl, BUILT_IN_MD))
16921 67 : switch (DECL_MD_FUNCTION_CODE (fndecl))
16922 : {
16923 24 : case IX86_BUILTIN_CMPPD:
16924 24 : case IX86_BUILTIN_CMPPS:
16925 24 : case IX86_BUILTIN_CMPPD256:
16926 24 : case IX86_BUILTIN_CMPPS256:
16927 24 : if (!integer_zerop (gimple_call_arg (def_stmt, 2)))
16928 : break;
16929 : /* FALLTHRU */
16930 49 : case IX86_BUILTIN_CMPEQPD:
16931 49 : case IX86_BUILTIN_CMPEQPS:
16932 49 : if (initializer_zerop (gimple_call_arg (def_stmt, 0))
16933 49 : && initializer_zerop (gimple_call_arg (def_stmt,
16934 : 1)))
16935 49 : op0 = pc_rtx;
16936 : break;
16937 : default:
16938 : break;
16939 : }
16940 : }
16941 : }
16942 : }
16943 :
16944 1007 : pat = GEN_FCN (icode) (subtarget, op0, op1, op2, op3, op4);
16945 1007 : if (! pat)
16946 0 : return const0_rtx;
16947 1007 : emit_insn (pat);
16948 :
16949 1007 : switch (fcode)
16950 : {
16951 24 : case IX86_BUILTIN_GATHER3DIV16SF:
16952 24 : if (target == NULL_RTX)
16953 0 : target = gen_reg_rtx (V8SFmode);
16954 24 : emit_insn (gen_vec_extract_lo_v16sf (target, subtarget));
16955 24 : break;
16956 24 : case IX86_BUILTIN_GATHER3DIV16SI:
16957 24 : if (target == NULL_RTX)
16958 0 : target = gen_reg_rtx (V8SImode);
16959 24 : emit_insn (gen_vec_extract_lo_v16si (target, subtarget));
16960 24 : break;
16961 28 : case IX86_BUILTIN_GATHER3DIV8SF:
16962 28 : case IX86_BUILTIN_GATHERDIV8SF:
16963 28 : if (target == NULL_RTX)
16964 0 : target = gen_reg_rtx (V4SFmode);
16965 28 : emit_insn (gen_vec_extract_lo_v8sf (target, subtarget));
16966 28 : break;
16967 28 : case IX86_BUILTIN_GATHER3DIV8SI:
16968 28 : case IX86_BUILTIN_GATHERDIV8SI:
16969 28 : if (target == NULL_RTX)
16970 0 : target = gen_reg_rtx (V4SImode);
16971 28 : emit_insn (gen_vec_extract_lo_v8si (target, subtarget));
16972 28 : break;
16973 : default:
16974 : target = subtarget;
16975 : break;
16976 : }
16977 : return target;
16978 :
16979 623 : scatter_gen:
16980 623 : arg0 = CALL_EXPR_ARG (exp, 0);
16981 623 : arg1 = CALL_EXPR_ARG (exp, 1);
16982 623 : arg2 = CALL_EXPR_ARG (exp, 2);
16983 623 : arg3 = CALL_EXPR_ARG (exp, 3);
16984 623 : arg4 = CALL_EXPR_ARG (exp, 4);
16985 623 : op0 = expand_normal (arg0);
16986 623 : op1 = ix86_expand_unsigned_small_int_cst_argument (arg1);
16987 623 : op2 = expand_normal (arg2);
16988 623 : op3 = expand_normal (arg3);
16989 623 : op4 = expand_normal (arg4);
16990 623 : mode1 = insn_data[icode].operand[1].mode;
16991 623 : mode2 = insn_data[icode].operand[2].mode;
16992 623 : mode3 = insn_data[icode].operand[3].mode;
16993 623 : mode4 = insn_data[icode].operand[4].mode;
16994 :
16995 : /* Scatter instruction stores operand op3 to memory with
16996 : indices from op2 and scale from op4 under writemask op1.
16997 : If index operand op2 has more elements then source operand
16998 : op3 one need to use only its low half. And vice versa. */
16999 623 : switch (fcode)
17000 : {
17001 24 : case IX86_BUILTIN_SCATTERALTSIV8DF:
17002 24 : case IX86_BUILTIN_SCATTERALTSIV8DI:
17003 24 : half = gen_reg_rtx (V8SImode);
17004 24 : if (!nonimmediate_operand (op2, V16SImode))
17005 0 : op2 = copy_to_mode_reg (V16SImode, op2);
17006 24 : emit_insn (gen_vec_extract_lo_v16si (half, op2));
17007 24 : op2 = half;
17008 24 : break;
17009 36 : case IX86_BUILTIN_SCATTERALTDIV16SF:
17010 36 : case IX86_BUILTIN_SCATTERALTDIV16SI:
17011 36 : half = gen_reg_rtx (mode3);
17012 36 : if (mode3 == V8SFmode)
17013 : gen = gen_vec_extract_lo_v16sf;
17014 : else
17015 24 : gen = gen_vec_extract_lo_v16si;
17016 36 : if (!nonimmediate_operand (op3, GET_MODE (op3)))
17017 0 : op3 = copy_to_mode_reg (GET_MODE (op3), op3);
17018 36 : emit_insn (gen (half, op3));
17019 36 : op3 = half;
17020 36 : break;
17021 8 : case IX86_BUILTIN_SCATTERALTSIV4DF:
17022 8 : case IX86_BUILTIN_SCATTERALTSIV4DI:
17023 8 : half = gen_reg_rtx (V4SImode);
17024 8 : if (!nonimmediate_operand (op2, V8SImode))
17025 0 : op2 = copy_to_mode_reg (V8SImode, op2);
17026 8 : emit_insn (gen_vec_extract_lo_v8si (half, op2));
17027 8 : op2 = half;
17028 8 : break;
17029 8 : case IX86_BUILTIN_SCATTERALTDIV8SF:
17030 8 : case IX86_BUILTIN_SCATTERALTDIV8SI:
17031 8 : half = gen_reg_rtx (mode3);
17032 8 : if (mode3 == V4SFmode)
17033 : gen = gen_vec_extract_lo_v8sf;
17034 : else
17035 4 : gen = gen_vec_extract_lo_v8si;
17036 8 : if (!nonimmediate_operand (op3, GET_MODE (op3)))
17037 0 : op3 = copy_to_mode_reg (GET_MODE (op3), op3);
17038 8 : emit_insn (gen (half, op3));
17039 8 : op3 = half;
17040 8 : break;
17041 16 : case IX86_BUILTIN_SCATTERALTSIV2DF:
17042 16 : case IX86_BUILTIN_SCATTERALTSIV2DI:
17043 16 : if (!nonimmediate_operand (op2, V4SImode))
17044 0 : op2 = copy_to_mode_reg (V4SImode, op2);
17045 : break;
17046 16 : case IX86_BUILTIN_SCATTERALTDIV4SF:
17047 16 : case IX86_BUILTIN_SCATTERALTDIV4SI:
17048 16 : if (!nonimmediate_operand (op3, GET_MODE (op3)))
17049 0 : op3 = copy_to_mode_reg (GET_MODE (op3), op3);
17050 : break;
17051 : default:
17052 : break;
17053 : }
17054 :
17055 : /* Force memory operand only with base register here. But we
17056 : don't want to do it on memory operand for other builtin
17057 : functions. */
17058 633 : op0 = force_reg (Pmode, convert_to_mode (Pmode, op0, 1));
17059 :
17060 628 : if (!insn_data[icode].operand[0].predicate (op0, Pmode))
17061 0 : op0 = copy_to_mode_reg (Pmode, op0);
17062 :
17063 623 : op1 = fixup_modeless_constant (op1, mode1);
17064 :
17065 623 : if (GET_MODE (op1) == mode1 || GET_MODE (op1) == VOIDmode)
17066 : {
17067 607 : if (!insn_data[icode].operand[1].predicate (op1, mode1))
17068 273 : op1 = copy_to_mode_reg (mode1, op1);
17069 : }
17070 : else
17071 : {
17072 16 : op1 = copy_to_reg (op1);
17073 16 : op1 = lowpart_subreg (mode1, op1, GET_MODE (op1));
17074 : }
17075 :
17076 623 : if (!insn_data[icode].operand[2].predicate (op2, mode2))
17077 57 : op2 = copy_to_mode_reg (mode2, op2);
17078 :
17079 623 : if (!insn_data[icode].operand[3].predicate (op3, mode3))
17080 82 : op3 = copy_to_mode_reg (mode3, op3);
17081 :
17082 623 : if (!insn_data[icode].operand[4].predicate (op4, mode4))
17083 : {
17084 0 : error ("the last argument must be scale 1, 2, 4, 8");
17085 0 : return const0_rtx;
17086 : }
17087 :
17088 623 : pat = GEN_FCN (icode) (op0, op1, op2, op3, op4);
17089 623 : if (! pat)
17090 0 : return const0_rtx;
17091 :
17092 623 : emit_insn (pat);
17093 623 : return 0;
17094 :
17095 23 : case IX86_BUILTIN_XABORT:
17096 23 : icode = CODE_FOR_xabort;
17097 23 : arg0 = CALL_EXPR_ARG (exp, 0);
17098 23 : op0 = expand_normal (arg0);
17099 23 : mode0 = insn_data[icode].operand[0].mode;
17100 23 : if (!insn_data[icode].operand[0].predicate (op0, mode0))
17101 : {
17102 0 : error ("the argument to %<xabort%> intrinsic must "
17103 : "be an 8-bit immediate");
17104 0 : return const0_rtx;
17105 : }
17106 23 : emit_insn (gen_xabort (op0));
17107 23 : return 0;
17108 :
17109 55 : case IX86_BUILTIN_RDSSPD:
17110 55 : case IX86_BUILTIN_RDSSPQ:
17111 55 : mode = (fcode == IX86_BUILTIN_RDSSPD ? SImode : DImode);
17112 :
17113 55 : if (target == 0
17114 55 : || !register_operand (target, mode))
17115 0 : target = gen_reg_rtx (mode);
17116 :
17117 55 : op0 = force_reg (mode, const0_rtx);
17118 :
17119 55 : emit_insn (gen_rdssp (mode, target, op0));
17120 55 : return target;
17121 :
17122 55 : case IX86_BUILTIN_INCSSPD:
17123 55 : case IX86_BUILTIN_INCSSPQ:
17124 55 : mode = (fcode == IX86_BUILTIN_INCSSPD ? SImode : DImode);
17125 :
17126 55 : arg0 = CALL_EXPR_ARG (exp, 0);
17127 55 : op0 = expand_normal (arg0);
17128 :
17129 55 : op0 = force_reg (mode, op0);
17130 :
17131 55 : emit_insn (gen_incssp (mode, op0));
17132 55 : return 0;
17133 :
17134 20 : case IX86_BUILTIN_HRESET:
17135 20 : icode = CODE_FOR_hreset;
17136 20 : arg0 = CALL_EXPR_ARG (exp, 0);
17137 20 : op0 = expand_normal (arg0);
17138 20 : op0 = force_reg (SImode, op0);
17139 20 : emit_insn (gen_hreset (op0));
17140 20 : return 0;
17141 :
17142 38 : case IX86_BUILTIN_RSTORSSP:
17143 38 : case IX86_BUILTIN_CLRSSBSY:
17144 38 : arg0 = CALL_EXPR_ARG (exp, 0);
17145 38 : op0 = expand_normal (arg0);
17146 19 : icode = (fcode == IX86_BUILTIN_RSTORSSP
17147 38 : ? CODE_FOR_rstorssp
17148 : : CODE_FOR_clrssbsy);
17149 :
17150 38 : if (!address_operand (op0, VOIDmode))
17151 : {
17152 18 : op0 = convert_memory_address (Pmode, op0);
17153 18 : op0 = copy_addr_to_reg (op0);
17154 : }
17155 38 : emit_insn (GEN_FCN (icode) (gen_rtx_MEM (DImode, op0)));
17156 38 : return 0;
17157 :
17158 80 : case IX86_BUILTIN_WRSSD:
17159 80 : case IX86_BUILTIN_WRSSQ:
17160 80 : case IX86_BUILTIN_WRUSSD:
17161 80 : case IX86_BUILTIN_WRUSSQ:
17162 80 : mode = ((fcode == IX86_BUILTIN_WRSSD
17163 80 : || fcode == IX86_BUILTIN_WRUSSD)
17164 80 : ? SImode : DImode);
17165 :
17166 80 : arg0 = CALL_EXPR_ARG (exp, 0);
17167 80 : op0 = expand_normal (arg0);
17168 80 : arg1 = CALL_EXPR_ARG (exp, 1);
17169 80 : op1 = expand_normal (arg1);
17170 :
17171 80 : op0 = force_reg (mode, op0);
17172 :
17173 80 : if (!address_operand (op1, VOIDmode))
17174 : {
17175 36 : op1 = convert_memory_address (Pmode, op1);
17176 36 : op1 = copy_addr_to_reg (op1);
17177 : }
17178 80 : op1 = gen_rtx_MEM (mode, op1);
17179 :
17180 80 : icode = ((fcode == IX86_BUILTIN_WRSSD
17181 80 : || fcode == IX86_BUILTIN_WRSSQ)
17182 80 : ? code_for_wrss (mode)
17183 40 : : code_for_wruss (mode));
17184 80 : emit_insn (GEN_FCN (icode) (op0, op1));
17185 :
17186 80 : return 0;
17187 :
17188 119899 : default:
17189 119899 : break;
17190 : }
17191 :
17192 119899 : if (fcode >= IX86_BUILTIN__BDESC_SPECIAL_ARGS_FIRST
17193 119899 : && fcode <= IX86_BUILTIN__BDESC_SPECIAL_ARGS_LAST)
17194 : {
17195 27000 : i = fcode - IX86_BUILTIN__BDESC_SPECIAL_ARGS_FIRST;
17196 27000 : return ix86_expand_special_args_builtin (bdesc_special_args + i, exp,
17197 27000 : target);
17198 : }
17199 :
17200 92899 : if (fcode >= IX86_BUILTIN__BDESC_PURE_ARGS_FIRST
17201 92899 : && fcode <= IX86_BUILTIN__BDESC_PURE_ARGS_LAST)
17202 : {
17203 93 : i = fcode - IX86_BUILTIN__BDESC_PURE_ARGS_FIRST;
17204 93 : return ix86_expand_special_args_builtin (bdesc_pure_args + i, exp,
17205 93 : target);
17206 : }
17207 :
17208 92806 : if (fcode >= IX86_BUILTIN__BDESC_ARGS_FIRST
17209 92806 : && fcode <= IX86_BUILTIN__BDESC_ARGS_LAST)
17210 : {
17211 74354 : i = fcode - IX86_BUILTIN__BDESC_ARGS_FIRST;
17212 :
17213 74354 : switch (fcode)
17214 : {
17215 0 : case IX86_BUILTIN_RDPID:
17216 0 : return ix86_expand_special_args_builtin (bdesc_args + i, exp,
17217 0 : target);
17218 74 : case IX86_BUILTIN_VCOMISBF16EQ:
17219 74 : case IX86_BUILTIN_VCOMISBF16NE:
17220 74 : case IX86_BUILTIN_VCOMISBF16GT:
17221 74 : case IX86_BUILTIN_VCOMISBF16GE:
17222 74 : case IX86_BUILTIN_VCOMISBF16LT:
17223 74 : case IX86_BUILTIN_VCOMISBF16LE:
17224 74 : return ix86_expand_sse_comi (bdesc_args + i, exp, target, false);
17225 15 : case IX86_BUILTIN_FABSQ:
17226 15 : case IX86_BUILTIN_COPYSIGNQ:
17227 15 : if (!TARGET_SSE)
17228 : /* Emit a normal call if SSE isn't available. */
17229 0 : return expand_call (exp, target, ignore);
17230 : /* FALLTHRU */
17231 74280 : default:
17232 74280 : return ix86_expand_args_builtin (bdesc_args + i, exp, target);
17233 : }
17234 : }
17235 :
17236 18452 : if (fcode >= IX86_BUILTIN__BDESC_COMI_FIRST
17237 18452 : && fcode <= IX86_BUILTIN__BDESC_COMI_LAST)
17238 : {
17239 473 : i = fcode - IX86_BUILTIN__BDESC_COMI_FIRST;
17240 473 : return ix86_expand_sse_comi (bdesc_comi + i, exp, target, true);
17241 : }
17242 :
17243 17979 : if (fcode >= IX86_BUILTIN__BDESC_ROUND_ARGS_FIRST
17244 17979 : && fcode <= IX86_BUILTIN__BDESC_ROUND_ARGS_LAST)
17245 : {
17246 15637 : i = fcode - IX86_BUILTIN__BDESC_ROUND_ARGS_FIRST;
17247 15637 : return ix86_expand_round_builtin (bdesc_round_args + i, exp, target);
17248 : }
17249 :
17250 2342 : if (fcode >= IX86_BUILTIN__BDESC_PCMPESTR_FIRST
17251 2342 : && fcode <= IX86_BUILTIN__BDESC_PCMPESTR_LAST)
17252 : {
17253 216 : i = fcode - IX86_BUILTIN__BDESC_PCMPESTR_FIRST;
17254 216 : return ix86_expand_sse_pcmpestr (bdesc_pcmpestr + i, exp, target);
17255 : }
17256 :
17257 2126 : if (fcode >= IX86_BUILTIN__BDESC_PCMPISTR_FIRST
17258 2126 : && fcode <= IX86_BUILTIN__BDESC_PCMPISTR_LAST)
17259 : {
17260 275 : i = fcode - IX86_BUILTIN__BDESC_PCMPISTR_FIRST;
17261 275 : return ix86_expand_sse_pcmpistr (bdesc_pcmpistr + i, exp, target);
17262 : }
17263 :
17264 1851 : if (fcode >= IX86_BUILTIN__BDESC_MULTI_ARG_FIRST
17265 1851 : && fcode <= IX86_BUILTIN__BDESC_MULTI_ARG_LAST)
17266 : {
17267 1813 : i = fcode - IX86_BUILTIN__BDESC_MULTI_ARG_FIRST;
17268 1813 : const struct builtin_description *d = bdesc_multi_arg + i;
17269 1813 : return ix86_expand_multi_arg_builtin (d->icode, exp, target,
17270 : (enum ix86_builtin_func_type)
17271 1813 : d->flag, d->comparison);
17272 : }
17273 :
17274 38 : if (fcode >= IX86_BUILTIN__BDESC_CET_FIRST
17275 38 : && fcode <= IX86_BUILTIN__BDESC_CET_LAST)
17276 : {
17277 38 : i = fcode - IX86_BUILTIN__BDESC_CET_FIRST;
17278 38 : return ix86_expand_special_args_builtin (bdesc_cet + i, exp,
17279 38 : target);
17280 : }
17281 :
17282 0 : gcc_unreachable ();
17283 : }
17284 :
17285 : /* See below where shifts are handled for explanation of this enum. */
17286 : enum ix86_vec_bcast_alg
17287 : {
17288 : VEC_BCAST_PXOR,
17289 : VEC_BCAST_PCMPEQ,
17290 : VEC_BCAST_PABSB,
17291 : VEC_BCAST_PADDB,
17292 : VEC_BCAST_PSRLW,
17293 : VEC_BCAST_PSRLD,
17294 : VEC_BCAST_PSLLW,
17295 : VEC_BCAST_PSLLD
17296 : };
17297 :
17298 : struct ix86_vec_bcast_map_simode_t
17299 : {
17300 : unsigned int key;
17301 : enum ix86_vec_bcast_alg alg;
17302 : unsigned int arg;
17303 : };
17304 :
17305 : /* This table must be kept sorted as values are looked-up using bsearch. */
17306 : static const ix86_vec_bcast_map_simode_t ix86_vec_bcast_map_simode[] = {
17307 : { 0x00000000, VEC_BCAST_PXOR, 0 },
17308 : { 0x00000001, VEC_BCAST_PSRLD, 31 },
17309 : { 0x00000003, VEC_BCAST_PSRLD, 30 },
17310 : { 0x00000007, VEC_BCAST_PSRLD, 29 },
17311 : { 0x0000000f, VEC_BCAST_PSRLD, 28 },
17312 : { 0x0000001f, VEC_BCAST_PSRLD, 27 },
17313 : { 0x0000003f, VEC_BCAST_PSRLD, 26 },
17314 : { 0x0000007f, VEC_BCAST_PSRLD, 25 },
17315 : { 0x000000ff, VEC_BCAST_PSRLD, 24 },
17316 : { 0x000001ff, VEC_BCAST_PSRLD, 23 },
17317 : { 0x000003ff, VEC_BCAST_PSRLD, 22 },
17318 : { 0x000007ff, VEC_BCAST_PSRLD, 21 },
17319 : { 0x00000fff, VEC_BCAST_PSRLD, 20 },
17320 : { 0x00001fff, VEC_BCAST_PSRLD, 19 },
17321 : { 0x00003fff, VEC_BCAST_PSRLD, 18 },
17322 : { 0x00007fff, VEC_BCAST_PSRLD, 17 },
17323 : { 0x0000ffff, VEC_BCAST_PSRLD, 16 },
17324 : { 0x00010001, VEC_BCAST_PSRLW, 15 },
17325 : { 0x0001ffff, VEC_BCAST_PSRLD, 15 },
17326 : { 0x00030003, VEC_BCAST_PSRLW, 14 },
17327 : { 0x0003ffff, VEC_BCAST_PSRLD, 14 },
17328 : { 0x00070007, VEC_BCAST_PSRLW, 13 },
17329 : { 0x0007ffff, VEC_BCAST_PSRLD, 13 },
17330 : { 0x000f000f, VEC_BCAST_PSRLW, 12 },
17331 : { 0x000fffff, VEC_BCAST_PSRLD, 12 },
17332 : { 0x001f001f, VEC_BCAST_PSRLW, 11 },
17333 : { 0x001fffff, VEC_BCAST_PSRLD, 11 },
17334 : { 0x003f003f, VEC_BCAST_PSRLW, 10 },
17335 : { 0x003fffff, VEC_BCAST_PSRLD, 10 },
17336 : { 0x007f007f, VEC_BCAST_PSRLW, 9 },
17337 : { 0x007fffff, VEC_BCAST_PSRLD, 9 },
17338 : { 0x00ff00ff, VEC_BCAST_PSRLW, 8 },
17339 : { 0x00ffffff, VEC_BCAST_PSRLD, 8 },
17340 : { 0x01010101, VEC_BCAST_PABSB, 0 },
17341 : { 0x01ff01ff, VEC_BCAST_PSRLW, 7 },
17342 : { 0x01ffffff, VEC_BCAST_PSRLD, 7 },
17343 : { 0x03ff03ff, VEC_BCAST_PSRLW, 6 },
17344 : { 0x03ffffff, VEC_BCAST_PSRLD, 6 },
17345 : { 0x07ff07ff, VEC_BCAST_PSRLW, 5 },
17346 : { 0x07ffffff, VEC_BCAST_PSRLD, 5 },
17347 : { 0x0fff0fff, VEC_BCAST_PSRLW, 4 },
17348 : { 0x0fffffff, VEC_BCAST_PSRLD, 4 },
17349 : { 0x1fff1fff, VEC_BCAST_PSRLW, 3 },
17350 : { 0x1fffffff, VEC_BCAST_PSRLD, 3 },
17351 : { 0x3fff3fff, VEC_BCAST_PSRLW, 2 },
17352 : { 0x3fffffff, VEC_BCAST_PSRLD, 2 },
17353 : { 0x7fff7fff, VEC_BCAST_PSRLW, 1 },
17354 : { 0x7fffffff, VEC_BCAST_PSRLD, 1 },
17355 : { 0x80000000, VEC_BCAST_PSLLD, 31 },
17356 : { 0x80008000, VEC_BCAST_PSLLW, 15 },
17357 : { 0xc0000000, VEC_BCAST_PSLLD, 30 },
17358 : { 0xc000c000, VEC_BCAST_PSLLW, 14 },
17359 : { 0xe0000000, VEC_BCAST_PSLLD, 29 },
17360 : { 0xe000e000, VEC_BCAST_PSLLW, 13 },
17361 : { 0xf0000000, VEC_BCAST_PSLLD, 28 },
17362 : { 0xf000f000, VEC_BCAST_PSLLW, 12 },
17363 : { 0xf8000000, VEC_BCAST_PSLLD, 27 },
17364 : { 0xf800f800, VEC_BCAST_PSLLW, 11 },
17365 : { 0xfc000000, VEC_BCAST_PSLLD, 26 },
17366 : { 0xfc00fc00, VEC_BCAST_PSLLW, 10 },
17367 : { 0xfe000000, VEC_BCAST_PSLLD, 25 },
17368 : { 0xfe00fe00, VEC_BCAST_PSLLW, 9 },
17369 : { 0xfefefefe, VEC_BCAST_PADDB, 0 },
17370 : { 0xff000000, VEC_BCAST_PSLLD, 24 },
17371 : { 0xff00ff00, VEC_BCAST_PSLLW, 8 },
17372 : { 0xff800000, VEC_BCAST_PSLLD, 23 },
17373 : { 0xff80ff80, VEC_BCAST_PSLLW, 7 },
17374 : { 0xffc00000, VEC_BCAST_PSLLD, 22 },
17375 : { 0xffc0ffc0, VEC_BCAST_PSLLW, 6 },
17376 : { 0xffe00000, VEC_BCAST_PSLLD, 21 },
17377 : { 0xffe0ffe0, VEC_BCAST_PSLLW, 5 },
17378 : { 0xfff00000, VEC_BCAST_PSLLD, 20 },
17379 : { 0xfff0fff0, VEC_BCAST_PSLLW, 4 },
17380 : { 0xfff80000, VEC_BCAST_PSLLD, 19 },
17381 : { 0xfff8fff8, VEC_BCAST_PSLLW, 3 },
17382 : { 0xfffc0000, VEC_BCAST_PSLLD, 18 },
17383 : { 0xfffcfffc, VEC_BCAST_PSLLW, 2 },
17384 : { 0xfffe0000, VEC_BCAST_PSLLD, 17 },
17385 : { 0xfffefffe, VEC_BCAST_PSLLW, 1 },
17386 : { 0xffff0000, VEC_BCAST_PSLLD, 16 },
17387 : { 0xffff8000, VEC_BCAST_PSLLD, 15 },
17388 : { 0xffffc000, VEC_BCAST_PSLLD, 14 },
17389 : { 0xffffe000, VEC_BCAST_PSLLD, 13 },
17390 : { 0xfffff000, VEC_BCAST_PSLLD, 12 },
17391 : { 0xfffff800, VEC_BCAST_PSLLD, 11 },
17392 : { 0xfffffc00, VEC_BCAST_PSLLD, 10 },
17393 : { 0xfffffe00, VEC_BCAST_PSLLD, 9 },
17394 : { 0xffffff00, VEC_BCAST_PSLLD, 8 },
17395 : { 0xffffff80, VEC_BCAST_PSLLD, 7 },
17396 : { 0xffffffc0, VEC_BCAST_PSLLD, 6 },
17397 : { 0xffffffe0, VEC_BCAST_PSLLD, 5 },
17398 : { 0xfffffff0, VEC_BCAST_PSLLD, 4 },
17399 : { 0xfffffff8, VEC_BCAST_PSLLD, 3 },
17400 : { 0xfffffffc, VEC_BCAST_PSLLD, 2 },
17401 : { 0xfffffffe, VEC_BCAST_PSLLD, 1 },
17402 : { 0xffffffff, VEC_BCAST_PCMPEQ, 0 }
17403 : };
17404 :
17405 : /* Comparator for bsearch on ix86_vec_bcast_map. */
17406 : static int
17407 344216 : ix86_vec_bcast_map_simode_cmp (const void *key, const void *entry)
17408 : {
17409 344216 : return (*(const unsigned int*)key)
17410 344216 : - ((const ix86_vec_bcast_map_simode_t*)entry)->key;
17411 : }
17412 :
17413 : /* A subroutine of ix86_vector_duplicate_value. Tries to efficiently
17414 : materialize V4SImode, V8SImode and V16SImode vectors from SImode
17415 : integer constants. */
17416 : static bool
17417 52097 : ix86_vector_duplicate_simode_const (machine_mode mode, rtx target,
17418 : unsigned int val)
17419 : {
17420 52097 : const ix86_vec_bcast_map_simode_t *entry;
17421 52097 : rtx tmp1, tmp2;
17422 :
17423 52097 : entry = (const ix86_vec_bcast_map_simode_t*)
17424 52097 : bsearch(&val, ix86_vec_bcast_map_simode,
17425 : ARRAY_SIZE (ix86_vec_bcast_map_simode),
17426 : sizeof (ix86_vec_bcast_map_simode_t),
17427 : ix86_vec_bcast_map_simode_cmp);
17428 52097 : if (!entry)
17429 : return false;
17430 :
17431 19472 : switch (entry->alg)
17432 : {
17433 2588 : case VEC_BCAST_PXOR:
17434 2588 : if ((mode == V8SImode && !TARGET_AVX2)
17435 2588 : || (mode == V16SImode && !TARGET_AVX512F))
17436 : return false;
17437 2588 : emit_move_insn (target, CONST0_RTX (mode));
17438 2588 : return true;
17439 :
17440 186 : case VEC_BCAST_PCMPEQ:
17441 186 : if ((mode == V4SImode && !TARGET_SSE2)
17442 185 : || (mode == V8SImode && !TARGET_AVX2)
17443 158 : || (mode == V16SImode && !TARGET_AVX512F))
17444 : return false;
17445 158 : emit_move_insn (target, CONSTM1_RTX (mode));
17446 158 : return true;
17447 :
17448 599 : case VEC_BCAST_PABSB:
17449 599 : if (mode == V4SImode && TARGET_SSE2)
17450 : {
17451 474 : tmp1 = gen_reg_rtx (V16QImode);
17452 474 : emit_move_insn (tmp1, CONSTM1_RTX (V16QImode));
17453 474 : tmp2 = gen_reg_rtx (V16QImode);
17454 474 : emit_insn (gen_absv16qi2 (tmp2, tmp1));
17455 : }
17456 125 : else if (mode == V8SImode && TARGET_AVX2)
17457 : {
17458 68 : tmp1 = gen_reg_rtx (V32QImode);
17459 68 : emit_move_insn (tmp1, CONSTM1_RTX (V32QImode));
17460 68 : tmp2 = gen_reg_rtx (V32QImode);
17461 68 : emit_insn (gen_absv32qi2 (tmp2, tmp1));
17462 : }
17463 57 : else if (mode == V16SImode && TARGET_AVX512BW)
17464 : {
17465 49 : tmp1 = gen_reg_rtx (V64QImode);
17466 49 : emit_move_insn (tmp1, CONSTM1_RTX (V64QImode));
17467 49 : tmp2 = gen_reg_rtx (V64QImode);
17468 49 : emit_insn (gen_absv64qi2 (tmp2, tmp1));
17469 : }
17470 : else
17471 : return false;
17472 : break;
17473 :
17474 103 : case VEC_BCAST_PADDB:
17475 103 : if (mode == V4SImode && TARGET_SSE2)
17476 : {
17477 99 : tmp1 = gen_reg_rtx (V16QImode);
17478 99 : emit_move_insn (tmp1, CONSTM1_RTX (V16QImode));
17479 99 : tmp2 = gen_reg_rtx (V16QImode);
17480 99 : emit_insn (gen_addv16qi3 (tmp2, tmp1, tmp1));
17481 : }
17482 4 : else if (mode == V8SImode && TARGET_AVX2)
17483 : {
17484 1 : tmp1 = gen_reg_rtx (V32QImode);
17485 1 : emit_move_insn (tmp1, CONSTM1_RTX (V32QImode));
17486 1 : tmp2 = gen_reg_rtx (V32QImode);
17487 1 : emit_insn (gen_addv32qi3 (tmp2, tmp1, tmp1));
17488 : }
17489 3 : else if (mode == V16SImode && TARGET_AVX512BW)
17490 : {
17491 3 : tmp1 = gen_reg_rtx (V64QImode);
17492 3 : emit_move_insn (tmp1, CONSTM1_RTX (V64QImode));
17493 3 : tmp2 = gen_reg_rtx (V64QImode);
17494 3 : emit_insn (gen_addv64qi3 (tmp2, tmp1, tmp1));
17495 : }
17496 : else
17497 : return false;
17498 : break;
17499 :
17500 3781 : case VEC_BCAST_PSRLW:
17501 3781 : if (mode == V4SImode && TARGET_SSE2)
17502 : {
17503 3561 : tmp1 = gen_reg_rtx (V8HImode);
17504 3561 : emit_move_insn (tmp1, CONSTM1_RTX (V8HImode));
17505 3561 : tmp2 = gen_reg_rtx (V8HImode);
17506 3561 : emit_insn (gen_lshrv8hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
17507 : }
17508 220 : else if (mode == V8SImode && TARGET_AVX2)
17509 : {
17510 127 : tmp1 = gen_reg_rtx (V16HImode);
17511 127 : emit_move_insn (tmp1, CONSTM1_RTX (V16HImode));
17512 127 : tmp2 = gen_reg_rtx (V16HImode);
17513 127 : emit_insn (gen_lshrv16hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
17514 : }
17515 93 : else if (mode == V16SImode && TARGET_AVX512BW)
17516 : {
17517 90 : tmp1 = gen_reg_rtx (V32HImode);
17518 90 : emit_move_insn (tmp1, CONSTM1_RTX (V32HImode));
17519 90 : tmp2 = gen_reg_rtx (V32HImode);
17520 90 : emit_insn (gen_lshrv32hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
17521 : }
17522 : else
17523 : return false;
17524 : break;
17525 :
17526 10301 : case VEC_BCAST_PSRLD:
17527 10301 : if (mode == V4SImode && TARGET_SSE2)
17528 : {
17529 7453 : tmp1 = gen_reg_rtx (V4SImode);
17530 7453 : emit_move_insn (tmp1, CONSTM1_RTX (V4SImode));
17531 7453 : emit_insn (gen_lshrv4si3 (target, tmp1, GEN_INT (entry->arg)));
17532 7453 : return true;
17533 : }
17534 2848 : else if (mode == V8SImode && TARGET_AVX2)
17535 : {
17536 1057 : tmp1 = gen_reg_rtx (V8SImode);
17537 1057 : emit_move_insn (tmp1, CONSTM1_RTX (V8SImode));
17538 1057 : emit_insn (gen_lshrv8si3 (target, tmp1, GEN_INT (entry->arg)));
17539 1057 : return true;
17540 : }
17541 1791 : else if (mode == V16SImode && TARGET_AVX512F)
17542 : {
17543 952 : tmp1 = gen_reg_rtx (V16SImode);
17544 952 : emit_move_insn (tmp1, CONSTM1_RTX (V16SImode));
17545 952 : emit_insn (gen_lshrv16si3 (target, tmp1, GEN_INT (entry->arg)));
17546 952 : return true;
17547 : }
17548 : else
17549 : return false;
17550 134 : break;
17551 :
17552 134 : case VEC_BCAST_PSLLW:
17553 134 : if (mode == V4SImode && TARGET_SSE2)
17554 : {
17555 104 : tmp1 = gen_reg_rtx (V8HImode);
17556 104 : emit_move_insn (tmp1, CONSTM1_RTX (V8HImode));
17557 104 : tmp2 = gen_reg_rtx (V8HImode);
17558 104 : emit_insn (gen_ashlv8hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
17559 : }
17560 30 : else if (mode == V8SImode && TARGET_AVX2)
17561 : {
17562 21 : tmp1 = gen_reg_rtx (V16HImode);
17563 21 : emit_move_insn (tmp1, CONSTM1_RTX (V16HImode));
17564 21 : tmp2 = gen_reg_rtx (V16HImode);
17565 21 : emit_insn (gen_ashlv16hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
17566 : }
17567 9 : else if (mode == V16SImode && TARGET_AVX512BW)
17568 : {
17569 9 : tmp1 = gen_reg_rtx (V32HImode);
17570 9 : emit_move_insn (tmp1, CONSTM1_RTX (V32HImode));
17571 9 : tmp2 = gen_reg_rtx (V32HImode);
17572 9 : emit_insn (gen_ashlv32hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
17573 : }
17574 : else
17575 : return false;
17576 : break;
17577 :
17578 1780 : case VEC_BCAST_PSLLD:
17579 1780 : if (mode == V4SImode && TARGET_SSE2)
17580 : {
17581 1744 : tmp1 = gen_reg_rtx (V4SImode);
17582 1744 : emit_move_insn (tmp1, CONSTM1_RTX (V4SImode));
17583 1744 : emit_insn (gen_ashlv4si3 (target, tmp1, GEN_INT (entry->arg)));
17584 1744 : return true;
17585 : }
17586 36 : else if (mode == V8SImode && TARGET_AVX2)
17587 : {
17588 18 : tmp1 = gen_reg_rtx (V8SImode);
17589 18 : emit_move_insn (tmp1, CONSTM1_RTX (V8SImode));
17590 18 : emit_insn (gen_ashlv8si3 (target, tmp1, GEN_INT (entry->arg)));
17591 18 : return true;
17592 : }
17593 18 : else if (mode == V16SImode && TARGET_AVX512F)
17594 : {
17595 18 : tmp1 = gen_reg_rtx (V16SImode);
17596 18 : emit_move_insn (tmp1, CONSTM1_RTX (V16SImode));
17597 18 : emit_insn (gen_ashlv16si3 (target, tmp1, GEN_INT (entry->arg)));
17598 18 : return true;
17599 : }
17600 : else
17601 : return false;
17602 :
17603 : default:
17604 : return false;
17605 : }
17606 :
17607 4606 : emit_move_insn (target, gen_lowpart (mode, tmp2));
17608 4606 : return true;
17609 : }
17610 :
17611 : /* A subroutine of ix86_expand_vector_init_duplicate. Tries to
17612 : fill target with val via vec_duplicate. */
17613 :
17614 : static bool
17615 153311 : ix86_vector_duplicate_value (machine_mode mode, rtx target, rtx val)
17616 : {
17617 153311 : bool ok;
17618 153311 : rtx_insn *insn;
17619 153311 : rtx dup;
17620 :
17621 153311 : if ((mode == V4SImode || mode == V8SImode || mode == V16SImode)
17622 60047 : && CONST_INT_P (val)
17623 52097 : && ix86_vector_duplicate_simode_const (mode, target, INTVAL (val)))
17624 : return true;
17625 :
17626 : /* Save/restore recog_data in case this is called from splitters
17627 : or other routines where recog_data needs to stay valid across
17628 : force_reg. See PR106577. */
17629 134717 : recog_data_d recog_data_save = recog_data;
17630 :
17631 : /* First attempt to recognize VAL as-is. */
17632 134717 : dup = gen_vec_duplicate (mode, val);
17633 134717 : insn = emit_insn (gen_rtx_SET (target, dup));
17634 134717 : if (recog_memoized (insn) < 0)
17635 : {
17636 96138 : rtx_insn *seq;
17637 96138 : machine_mode innermode = GET_MODE_INNER (mode);
17638 96138 : rtx reg;
17639 :
17640 : /* If that fails, force VAL into a register or mem. */
17641 :
17642 96138 : start_sequence ();
17643 :
17644 0 : if (!TARGET_PREFER_BCST_FROM_INTEGER && CONST_INT_P (val)
17645 0 : && GET_MODE_BITSIZE (innermode) <= HOST_BITS_PER_WIDE_INT
17646 96138 : && GET_MODE_BITSIZE(mode) >= 128)
17647 0 : reg = validize_mem (force_const_mem (innermode, val));
17648 : else
17649 : {
17650 96138 : reg = force_reg (innermode, val);
17651 96138 : if (GET_MODE (reg) != innermode)
17652 0 : reg = gen_lowpart (innermode, reg);
17653 : }
17654 :
17655 96138 : SET_SRC (PATTERN (insn)) = gen_vec_duplicate (mode, reg);
17656 96138 : seq = end_sequence ();
17657 96138 : if (seq)
17658 96138 : emit_insn_before (seq, insn);
17659 :
17660 96138 : ok = recog_memoized (insn) >= 0;
17661 96138 : gcc_assert (ok);
17662 : }
17663 134717 : recog_data = recog_data_save;
17664 134717 : return true;
17665 : }
17666 :
17667 : /* Get a vector mode of the same size as the original but with elements
17668 : twice as wide. This is only guaranteed to apply to integral vectors. */
17669 :
17670 : static machine_mode
17671 25949 : get_mode_wider_vector (machine_mode o)
17672 : {
17673 : /* ??? Rely on the ordering that genmodes.cc gives to vectors. */
17674 25949 : machine_mode n = GET_MODE_NEXT_MODE (o).require ();
17675 77847 : gcc_assert (GET_MODE_NUNITS (o) == GET_MODE_NUNITS (n) * 2);
17676 77847 : gcc_assert (GET_MODE_SIZE (o) == GET_MODE_SIZE (n));
17677 25949 : return n;
17678 : }
17679 :
17680 : static bool expand_vec_perm_broadcast_1 (struct expand_vec_perm_d *d);
17681 : static bool expand_vec_perm_1 (struct expand_vec_perm_d *d);
17682 :
17683 : /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
17684 : with all elements equal to VAR. Return true if successful. */
17685 :
17686 : bool
17687 180125 : ix86_expand_vector_init_duplicate (bool mmx_ok, machine_mode mode,
17688 : rtx target, rtx val)
17689 : {
17690 180125 : bool ok;
17691 :
17692 180125 : switch (mode)
17693 : {
17694 71031 : case E_V2DImode:
17695 71031 : if (CONST_INT_P (val))
17696 : {
17697 61792 : int tmp = (int)INTVAL (val);
17698 61792 : if (tmp == (int)(INTVAL (val) >> 32))
17699 : {
17700 106 : rtx reg = gen_reg_rtx (V4SImode);
17701 106 : ok = ix86_vector_duplicate_value (V4SImode, reg,
17702 : GEN_INT (tmp));
17703 106 : if (ok)
17704 : {
17705 106 : emit_move_insn (target, gen_lowpart (V2DImode, reg));
17706 106 : return true;
17707 : }
17708 : }
17709 : }
17710 70925 : return ix86_vector_duplicate_value (mode, target, val);
17711 :
17712 1007 : case E_V4DImode:
17713 1007 : if (CONST_INT_P (val))
17714 : {
17715 718 : int tmp = (int)INTVAL (val);
17716 718 : if (tmp == (int)(INTVAL (val) >> 32))
17717 : {
17718 54 : rtx reg = gen_reg_rtx (V8SImode);
17719 54 : ok = ix86_vector_duplicate_value (V8SImode, reg,
17720 : GEN_INT (tmp));
17721 54 : if (ok)
17722 : {
17723 54 : emit_move_insn (target, gen_lowpart (V4DImode, reg));
17724 54 : return true;
17725 : }
17726 : }
17727 : }
17728 953 : return ix86_vector_duplicate_value (mode, target, val);
17729 :
17730 464 : case E_V8DImode:
17731 464 : if (CONST_INT_P (val))
17732 : {
17733 264 : int tmp = (int)INTVAL (val);
17734 264 : if (tmp == (int)(INTVAL (val) >> 32))
17735 : {
17736 24 : rtx reg = gen_reg_rtx (V16SImode);
17737 24 : ok = ix86_vector_duplicate_value (V16SImode, reg,
17738 : GEN_INT (tmp));
17739 24 : if (ok)
17740 : {
17741 24 : emit_move_insn (target, gen_lowpart (V8DImode, reg));
17742 24 : return true;
17743 : }
17744 : }
17745 : }
17746 440 : return ix86_vector_duplicate_value (mode, target, val);
17747 :
17748 2379 : case E_V2SImode:
17749 2379 : case E_V2SFmode:
17750 2379 : if (!mmx_ok)
17751 : return false;
17752 : /* FALLTHRU */
17753 :
17754 79824 : case E_V4DFmode:
17755 79824 : case E_V8SFmode:
17756 79824 : case E_V8SImode:
17757 79824 : case E_V2DFmode:
17758 79824 : case E_V4SFmode:
17759 79824 : case E_V4SImode:
17760 79824 : case E_V16SImode:
17761 79824 : case E_V16SFmode:
17762 79824 : case E_V8DFmode:
17763 79824 : return ix86_vector_duplicate_value (mode, target, val);
17764 :
17765 386 : case E_V4HImode:
17766 386 : if (!mmx_ok)
17767 : return false;
17768 383 : if (TARGET_SSE || TARGET_3DNOW_A)
17769 : {
17770 383 : rtx x;
17771 :
17772 383 : val = gen_lowpart (SImode, val);
17773 383 : if (CONST_INT_P (val))
17774 : return false;
17775 381 : x = gen_rtx_TRUNCATE (HImode, val);
17776 381 : x = gen_rtx_VEC_DUPLICATE (mode, x);
17777 381 : emit_insn (gen_rtx_SET (target, x));
17778 381 : return true;
17779 : }
17780 0 : goto widen;
17781 :
17782 5 : case E_V4HFmode:
17783 5 : case E_V4BFmode:
17784 5 : if (TARGET_MMX_WITH_SSE)
17785 : {
17786 10 : val = force_reg (GET_MODE_INNER (mode), val);
17787 5 : rtx x = gen_rtx_VEC_DUPLICATE (mode, val);
17788 5 : emit_insn (gen_rtx_SET (target, x));
17789 5 : return true;
17790 : }
17791 : return false;
17792 :
17793 129 : case E_V2HImode:
17794 129 : if (TARGET_SSE2)
17795 : {
17796 129 : rtx x;
17797 :
17798 129 : val = gen_lowpart (SImode, val);
17799 129 : if (CONST_INT_P (val))
17800 : return false;
17801 129 : x = gen_rtx_TRUNCATE (HImode, val);
17802 129 : x = gen_rtx_VEC_DUPLICATE (mode, x);
17803 129 : emit_insn (gen_rtx_SET (target, x));
17804 129 : return true;
17805 : }
17806 : return false;
17807 :
17808 5 : case E_V2HFmode:
17809 5 : case E_V2BFmode:
17810 5 : if (TARGET_SSE2)
17811 : {
17812 10 : val = force_reg (GET_MODE_INNER (mode), val);
17813 5 : rtx x = gen_rtx_VEC_DUPLICATE (mode, val);
17814 5 : emit_insn (gen_rtx_SET (target, x));
17815 5 : return true;
17816 : }
17817 : return false;
17818 :
17819 297 : case E_V8QImode:
17820 297 : case E_V4QImode:
17821 297 : if (!mmx_ok)
17822 : return false;
17823 296 : goto widen;
17824 :
17825 13668 : case E_V8HImode:
17826 13668 : if (CONST_INT_P (val))
17827 13126 : goto widen;
17828 : /* FALLTHRU */
17829 :
17830 856 : case E_V8HFmode:
17831 856 : case E_V8BFmode:
17832 856 : if (TARGET_AVX2)
17833 392 : return ix86_vector_duplicate_value (mode, target, val);
17834 :
17835 464 : if (TARGET_SSE2)
17836 : {
17837 1181 : struct expand_vec_perm_d dperm;
17838 1181 : rtx tmp1, tmp2;
17839 :
17840 464 : permute:
17841 1181 : memset (&dperm, 0, sizeof (dperm));
17842 1181 : dperm.target = target;
17843 1181 : dperm.vmode = mode;
17844 1181 : dperm.nelt = GET_MODE_NUNITS (mode);
17845 1181 : dperm.op0 = dperm.op1 = gen_reg_rtx (mode);
17846 1181 : dperm.one_operand_p = true;
17847 :
17848 1181 : if (mode == V8HFmode || mode == V8BFmode)
17849 : {
17850 6 : tmp1 = force_reg (GET_MODE_INNER (mode), val);
17851 3 : tmp2 = gen_reg_rtx (mode);
17852 3 : emit_insn (gen_vec_set_0 (mode, tmp2, CONST0_RTX (mode), tmp1));
17853 3 : tmp1 = gen_lowpart (mode, tmp2);
17854 : }
17855 : else
17856 : {
17857 : /* Extend to SImode using a paradoxical SUBREG. */
17858 1178 : tmp1 = gen_reg_rtx (SImode);
17859 1178 : emit_move_insn (tmp1, gen_lowpart (SImode, val));
17860 :
17861 : /* Insert the SImode value as
17862 : low element of a V4SImode vector. */
17863 1178 : tmp2 = gen_reg_rtx (V4SImode);
17864 1178 : emit_insn (gen_vec_setv4si_0 (tmp2, CONST0_RTX (V4SImode), tmp1));
17865 1178 : tmp1 = gen_lowpart (mode, tmp2);
17866 : }
17867 :
17868 1181 : emit_move_insn (dperm.op0, tmp1);
17869 1181 : ok = (expand_vec_perm_1 (&dperm)
17870 1181 : || expand_vec_perm_broadcast_1 (&dperm));
17871 0 : gcc_assert (ok);
17872 1181 : return ok;
17873 : }
17874 0 : goto widen;
17875 :
17876 9249 : case E_V16QImode:
17877 9249 : if (CONST_INT_P (val))
17878 8472 : goto widen;
17879 777 : if (TARGET_AVX2)
17880 60 : return ix86_vector_duplicate_value (mode, target, val);
17881 :
17882 717 : if (TARGET_SSE2)
17883 717 : goto permute;
17884 0 : goto widen;
17885 :
17886 24327 : widen:
17887 : /* Replicate the value once into the next wider mode and recurse. */
17888 24327 : {
17889 24327 : machine_mode smode, wsmode, wvmode;
17890 24327 : rtx x;
17891 :
17892 24327 : smode = GET_MODE_INNER (mode);
17893 24327 : wvmode = get_mode_wider_vector (mode);
17894 24327 : wsmode = GET_MODE_INNER (wvmode);
17895 :
17896 24327 : val = convert_modes (wsmode, smode, val, true);
17897 :
17898 24327 : if (CONST_INT_P (val))
17899 : {
17900 48064 : x = simplify_binary_operation (ASHIFT, wsmode, val,
17901 24032 : GEN_INT (GET_MODE_BITSIZE (smode)));
17902 24032 : val = simplify_binary_operation (IOR, wsmode, val, x);
17903 : }
17904 295 : else if (smode == QImode && !TARGET_PARTIAL_REG_STALL)
17905 295 : emit_insn (gen_insv_1 (wsmode, val, val));
17906 : else
17907 : {
17908 0 : x = expand_simple_binop (wsmode, ASHIFT, val,
17909 0 : GEN_INT (GET_MODE_BITSIZE (smode)),
17910 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
17911 0 : val = expand_simple_binop (wsmode, IOR, val, x, x, 1,
17912 : OPTAB_LIB_WIDEN);
17913 : }
17914 :
17915 24327 : x = gen_reg_rtx (wvmode);
17916 24327 : ok = ix86_expand_vector_init_duplicate (mmx_ok, wvmode, x, val);
17917 24327 : if (!ok)
17918 : return false;
17919 24326 : emit_move_insn (target, gen_lowpart (GET_MODE (target), x));
17920 24326 : return true;
17921 : }
17922 :
17923 1532 : case E_V16HImode:
17924 1532 : case E_V32QImode:
17925 1532 : if (CONST_INT_P (val))
17926 1242 : goto widen;
17927 : /* FALLTHRU */
17928 :
17929 373 : case E_V16HFmode:
17930 373 : case E_V16BFmode:
17931 373 : if (TARGET_AVX2)
17932 347 : return ix86_vector_duplicate_value (mode, target, val);
17933 : else
17934 : {
17935 26 : machine_mode hvmode;
17936 26 : switch (mode)
17937 : {
17938 : case V16HImode:
17939 : hvmode = V8HImode;
17940 : break;
17941 0 : case V16HFmode:
17942 0 : hvmode = V8HFmode;
17943 0 : break;
17944 1 : case V16BFmode:
17945 1 : hvmode = V8BFmode;
17946 1 : break;
17947 12 : case V32QImode:
17948 12 : hvmode = V16QImode;
17949 12 : break;
17950 : default:
17951 : gcc_unreachable ();
17952 : }
17953 26 : rtx x = gen_reg_rtx (hvmode);
17954 :
17955 26 : ok = ix86_expand_vector_init_duplicate (false, hvmode, x, val);
17956 26 : if (!ok)
17957 : return false;
17958 :
17959 26 : x = gen_rtx_VEC_CONCAT (mode, x, x);
17960 26 : emit_insn (gen_rtx_SET (target, x));
17961 : }
17962 26 : return true;
17963 :
17964 1318 : case E_V32HImode:
17965 1318 : case E_V64QImode:
17966 1318 : if (CONST_INT_P (val))
17967 1191 : goto widen;
17968 : /* FALLTHRU */
17969 :
17970 206 : case E_V32HFmode:
17971 206 : case E_V32BFmode:
17972 206 : if (TARGET_AVX512BW)
17973 186 : return ix86_vector_duplicate_value (mode, target, val);
17974 : else
17975 : {
17976 20 : machine_mode hvmode;
17977 20 : switch (mode)
17978 : {
17979 : case V32HImode:
17980 : hvmode = V16HImode;
17981 : break;
17982 0 : case V32HFmode:
17983 0 : hvmode = V16HFmode;
17984 0 : break;
17985 1 : case V32BFmode:
17986 1 : hvmode = V16BFmode;
17987 1 : break;
17988 10 : case V64QImode:
17989 10 : hvmode = V32QImode;
17990 10 : break;
17991 : default:
17992 : gcc_unreachable ();
17993 : }
17994 20 : rtx x = gen_reg_rtx (hvmode);
17995 :
17996 20 : ok = ix86_expand_vector_init_duplicate (false, hvmode, x, val);
17997 20 : if (!ok)
17998 : return false;
17999 :
18000 20 : x = gen_rtx_VEC_CONCAT (mode, x, x);
18001 20 : emit_insn (gen_rtx_SET (target, x));
18002 : }
18003 20 : return true;
18004 :
18005 : default:
18006 : return false;
18007 : }
18008 : }
18009 :
18010 : /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
18011 : whose ONE_VAR element is VAR, and other elements are zero. Return true
18012 : if successful. */
18013 :
18014 : bool
18015 10609 : ix86_expand_vector_init_one_nonzero (bool mmx_ok, machine_mode mode,
18016 : rtx target, rtx var, int one_var)
18017 : {
18018 10609 : rtx x, tmp;
18019 10609 : bool use_vector_set = false;
18020 10609 : rtx (*gen_vec_set_0) (rtx, rtx, rtx) = NULL;
18021 :
18022 10609 : switch (mode)
18023 : {
18024 8412 : case E_V2DImode:
18025 8412 : if (TARGET_64BIT || MEM_P (var))
18026 : {
18027 8403 : if (!REG_P (var) && !MEM_P (var))
18028 5486 : var = force_reg (DImode, var);
18029 8403 : x = gen_rtx_VEC_CONCAT (V2DImode, var, CONST0_RTX (DImode));
18030 8403 : if (!one_var)
18031 2423 : emit_insn (gen_rtx_SET (target, x));
18032 5980 : else if (TARGET_SSE2)
18033 : {
18034 5980 : tmp = gen_reg_rtx (V2DImode);
18035 5980 : emit_insn (gen_rtx_SET (tmp, x));
18036 5980 : emit_insn (gen_vec_shl_v2di (target, tmp, GEN_INT (64)));
18037 : }
18038 : else
18039 : {
18040 0 : rtx tmp1 = gen_reg_rtx (V2DImode);
18041 0 : emit_insn (gen_rtx_SET (tmp1, x));
18042 0 : rtx tmp2 = gen_reg_rtx (V4SImode);
18043 0 : emit_move_insn (tmp2, gen_lowpart (V4SImode, tmp1));
18044 0 : emit_insn (gen_sse_shufps_v4si (tmp2, tmp2, tmp2,
18045 : GEN_INT (2), GEN_INT (3),
18046 : GEN_INT (4), GEN_INT (5)));
18047 0 : emit_move_insn (target, gen_lowpart (V2DImode, tmp2));
18048 : }
18049 : }
18050 : else
18051 : {
18052 9 : rtx lo = force_reg (SImode, gen_lowpart (SImode, var));
18053 9 : rtx hi = force_reg (SImode, gen_highpart (SImode, var));
18054 9 : tmp = gen_reg_rtx (V4SImode);
18055 9 : if (TARGET_SSE4_1)
18056 : {
18057 0 : rtx tmp1 = gen_reg_rtx (V4SImode);
18058 0 : emit_insn (gen_vec_setv4si_0 (tmp1, CONST0_RTX (V4SImode), lo));
18059 0 : emit_insn (gen_sse4_1_pinsrd (tmp, tmp1, hi, GEN_INT (2)));
18060 : }
18061 : else
18062 : {
18063 9 : rtx ltmp = gen_reg_rtx (V4SImode);
18064 9 : rtx htmp = gen_reg_rtx (V4SImode);
18065 9 : emit_insn (gen_vec_setv4si_0 (ltmp, CONST0_RTX (V4SImode), lo));
18066 9 : emit_insn (gen_vec_setv4si_0 (htmp, CONST0_RTX (V4SImode), hi));
18067 9 : emit_insn (gen_vec_interleave_lowv4si (tmp, ltmp, htmp));
18068 : }
18069 9 : if (!one_var)
18070 3 : emit_move_insn (target, gen_lowpart (V2DImode, tmp));
18071 6 : else if (TARGET_SSE2)
18072 : {
18073 6 : rtx tmp2 = gen_reg_rtx (V2DImode);
18074 6 : emit_move_insn (tmp2, gen_lowpart (V2DImode, tmp));
18075 6 : emit_insn (gen_vec_shl_v2di (target, tmp2, GEN_INT (64)));
18076 : }
18077 : else
18078 : {
18079 0 : rtx tmp2 = gen_reg_rtx (V2DImode);
18080 0 : emit_insn (gen_sse_shufps_v4si (tmp2, tmp, tmp,
18081 : GEN_INT (2), GEN_INT (3),
18082 : GEN_INT (4), GEN_INT (5)));
18083 0 : emit_move_insn (target, gen_lowpart (V2DImode, tmp2));
18084 : }
18085 : }
18086 : return true;
18087 105 : case E_V2DFmode:
18088 105 : if (!REG_P (var) && !MEM_P (var))
18089 0 : var = force_reg (DFmode, var);
18090 105 : x = gen_rtx_VEC_CONCAT (V2DFmode, var, CONST0_RTX (DFmode));
18091 105 : if (one_var)
18092 : {
18093 27 : tmp = gen_reg_rtx (V2DFmode);
18094 27 : emit_insn (gen_rtx_SET (tmp, x));
18095 27 : emit_insn (gen_vec_shl_v2df (target, tmp, GEN_INT (64)));
18096 : }
18097 : else
18098 78 : emit_insn (gen_rtx_SET (target, x));
18099 : return true;
18100 333 : case E_V4SImode:
18101 333 : var = force_reg (SImode, var);
18102 333 : x = gen_rtx_VEC_DUPLICATE (V4SImode, var);
18103 333 : x = gen_rtx_VEC_MERGE (V4SImode, x, CONST0_RTX (V4SImode), const1_rtx);
18104 333 : if (!one_var)
18105 283 : emit_insn (gen_rtx_SET (target, x));
18106 50 : else if (TARGET_SSE2)
18107 : {
18108 47 : rtx tmp = gen_reg_rtx (V4SImode);
18109 47 : emit_insn (gen_rtx_SET (tmp, x));
18110 47 : emit_insn (gen_vec_shl_v4si (target, tmp, GEN_INT (one_var * 32)));
18111 : }
18112 : else
18113 : {
18114 3 : rtx tmp = gen_reg_rtx (V4SImode);
18115 3 : emit_insn (gen_rtx_SET (tmp, x));
18116 10 : emit_insn (gen_sse_shufps_v4si (target, tmp, tmp,
18117 : const1_rtx,
18118 3 : GEN_INT (one_var == 1 ? 0 : 1),
18119 : GEN_INT (one_var == 2 ? 0+4 : 1+4),
18120 : GEN_INT (one_var == 3 ? 0+4 : 1+4)));
18121 : }
18122 : return true;
18123 586 : case E_V4SFmode:
18124 586 : var = force_reg (SFmode, var);
18125 586 : x = gen_rtx_VEC_DUPLICATE (V4SFmode, var);
18126 586 : x = gen_rtx_VEC_MERGE (V4SFmode, x, CONST0_RTX (V4SFmode), const1_rtx);
18127 586 : if (!one_var)
18128 555 : emit_insn (gen_rtx_SET (target, x));
18129 31 : else if (TARGET_SSE2)
18130 : {
18131 28 : rtx tmp = gen_reg_rtx (V4SFmode);
18132 28 : emit_insn (gen_rtx_SET (tmp, x));
18133 28 : emit_insn (gen_vec_shl_v4sf (target, tmp, GEN_INT (one_var * 32)));
18134 : }
18135 : else
18136 : {
18137 3 : rtx tmp = gen_reg_rtx (V4SFmode);
18138 3 : emit_insn (gen_rtx_SET (tmp, x));
18139 10 : emit_insn (gen_sse_shufps_v4sf (target, tmp, tmp,
18140 : const1_rtx,
18141 3 : GEN_INT (one_var == 1 ? 0 : 1),
18142 : GEN_INT (one_var == 2 ? 0+4 : 1+4),
18143 : GEN_INT (one_var == 3 ? 0+4 : 1+4)));
18144 : }
18145 : return true;
18146 7 : case E_V4DImode:
18147 7 : if (TARGET_AVX2 && (TARGET_64BIT || MEM_P (var)))
18148 : {
18149 6 : if (!REG_P (var) && !MEM_P (var))
18150 0 : var = force_reg (DImode, var);
18151 6 : x = gen_rtx_VEC_DUPLICATE (V4DImode, var);
18152 6 : x = gen_rtx_VEC_MERGE (V4DImode, x, CONST0_RTX (V4DImode),
18153 : const1_rtx);
18154 6 : if (one_var)
18155 : {
18156 0 : tmp = gen_reg_rtx (V4DImode);
18157 0 : emit_insn (gen_rtx_SET (tmp, x));
18158 0 : emit_insn (gen_avx2_permv4di_1 (target, tmp,
18159 : const1_rtx,
18160 0 : GEN_INT (one_var == 1 ? 0 : 1),
18161 0 : GEN_INT (one_var == 2 ? 0 : 1),
18162 0 : GEN_INT (one_var == 3 ? 0 : 1)));
18163 : }
18164 : else
18165 6 : emit_insn (gen_rtx_SET (target, x));
18166 : }
18167 : else
18168 : {
18169 1 : tmp = gen_reg_rtx (V2DImode);
18170 1 : if (!ix86_expand_vector_init_one_nonzero (mmx_ok, V2DImode, tmp,
18171 : var, one_var & 1))
18172 0 : gcc_unreachable ();
18173 1 : rtx zero = CONST0_RTX (V2DImode);
18174 1 : if (one_var >= 2)
18175 : {
18176 0 : zero = force_reg (V2DImode, zero);
18177 0 : emit_insn (gen_avx_vec_concatv4di (target, zero, tmp));
18178 : }
18179 : else
18180 1 : emit_insn (gen_avx_vec_concatv4di (target, tmp, zero));
18181 : }
18182 : return true;
18183 13 : case E_V4DFmode:
18184 13 : if (TARGET_AVX2)
18185 : {
18186 13 : if (!REG_P (var) && !MEM_P (var))
18187 0 : var = force_reg (DFmode, var);
18188 13 : x = gen_rtx_VEC_DUPLICATE (V4DFmode, var);
18189 13 : x = gen_rtx_VEC_MERGE (V4DFmode, x, CONST0_RTX (V4DFmode),
18190 : const1_rtx);
18191 13 : if (one_var)
18192 : {
18193 0 : tmp = gen_reg_rtx (V4DFmode);
18194 0 : emit_insn (gen_rtx_SET (tmp, x));
18195 0 : emit_insn (gen_avx2_permv4df_1 (target, tmp,
18196 : const1_rtx,
18197 0 : GEN_INT (one_var == 1 ? 0 : 1),
18198 0 : GEN_INT (one_var == 2 ? 0 : 1),
18199 0 : GEN_INT (one_var == 3 ? 0 : 1)));
18200 : }
18201 : else
18202 13 : emit_insn (gen_rtx_SET (target, x));
18203 : }
18204 : else
18205 : {
18206 0 : tmp = gen_reg_rtx (V2DFmode);
18207 0 : if (!ix86_expand_vector_init_one_nonzero (mmx_ok, V2DFmode, tmp,
18208 : var, one_var & 1))
18209 0 : gcc_unreachable ();
18210 0 : rtx zero = CONST0_RTX (V2DFmode);
18211 0 : if (one_var >= 2)
18212 : {
18213 0 : zero = force_reg (V2DFmode, zero);
18214 0 : emit_insn (gen_avx_vec_concatv4df (target, zero, tmp));
18215 : }
18216 : else
18217 0 : emit_insn (gen_avx_vec_concatv4df (target, tmp, zero));
18218 : }
18219 : return true;
18220 70 : case E_V16QImode:
18221 70 : use_vector_set = TARGET_SSE4_1;
18222 70 : break;
18223 102 : case E_V8HImode:
18224 102 : use_vector_set = TARGET_SSE2;
18225 102 : gen_vec_set_0 = TARGET_AVX512FP16 && one_var == 0
18226 102 : ? gen_vec_setv8hi_0 : NULL;
18227 : break;
18228 3 : case E_V8QImode:
18229 3 : use_vector_set = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
18230 : break;
18231 14 : case E_V4HImode:
18232 14 : case E_V4HFmode:
18233 14 : case E_V4BFmode:
18234 14 : use_vector_set = TARGET_SSE || TARGET_3DNOW_A;
18235 : break;
18236 32 : case E_V4QImode:
18237 32 : use_vector_set = TARGET_SSE4_1;
18238 32 : break;
18239 0 : case E_V32QImode:
18240 0 : use_vector_set = TARGET_AVX;
18241 0 : break;
18242 5 : case E_V16HImode:
18243 5 : use_vector_set = TARGET_AVX;
18244 5 : gen_vec_set_0 = TARGET_AVX512FP16 && one_var == 0
18245 5 : ? gen_vec_setv16hi_0 : NULL;
18246 : break;
18247 5 : case E_V8SImode:
18248 5 : use_vector_set = TARGET_AVX;
18249 5 : gen_vec_set_0 = gen_vec_setv8si_0;
18250 5 : break;
18251 22 : case E_V8SFmode:
18252 22 : use_vector_set = TARGET_AVX;
18253 22 : gen_vec_set_0 = gen_vec_setv8sf_0;
18254 22 : break;
18255 17 : case E_V16SImode:
18256 17 : use_vector_set = TARGET_AVX512F && one_var == 0;
18257 : gen_vec_set_0 = gen_vec_setv16si_0;
18258 : break;
18259 22 : case E_V16SFmode:
18260 22 : use_vector_set = TARGET_AVX512F && one_var == 0;
18261 : gen_vec_set_0 = gen_vec_setv16sf_0;
18262 : break;
18263 0 : case E_V8DFmode:
18264 0 : use_vector_set = TARGET_AVX512F && one_var == 0;
18265 : gen_vec_set_0 = gen_vec_setv8df_0;
18266 : break;
18267 2 : case E_V8DImode:
18268 : /* Use ix86_expand_vector_set in 64bit mode only. */
18269 2 : use_vector_set = TARGET_AVX512F && TARGET_64BIT && one_var == 0;
18270 : gen_vec_set_0 = gen_vec_setv8di_0;
18271 : break;
18272 39 : case E_V8HFmode:
18273 39 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18274 : gen_vec_set_0 = gen_vec_setv8hf_0;
18275 : break;
18276 9 : case E_V16HFmode:
18277 9 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18278 : gen_vec_set_0 = gen_vec_setv16hf_0;
18279 : break;
18280 6 : case E_V32HFmode:
18281 6 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18282 : gen_vec_set_0 = gen_vec_setv32hf_0;
18283 : break;
18284 2 : case E_V8BFmode:
18285 2 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18286 : gen_vec_set_0 = gen_vec_setv8bf_0;
18287 : break;
18288 0 : case E_V16BFmode:
18289 0 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18290 : gen_vec_set_0 = gen_vec_setv16bf_0;
18291 : break;
18292 0 : case E_V32BFmode:
18293 0 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18294 : gen_vec_set_0 = gen_vec_setv32bf_0;
18295 : break;
18296 4 : case E_V32HImode:
18297 4 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18298 : gen_vec_set_0 = gen_vec_setv32hi_0;
18299 : default:
18300 : break;
18301 : }
18302 :
18303 236 : if (use_vector_set)
18304 : {
18305 285 : if (gen_vec_set_0 && one_var == 0)
18306 : {
18307 316 : var = force_reg (GET_MODE_INNER (mode), var);
18308 158 : emit_insn (gen_vec_set_0 (target, CONST0_RTX (mode), var));
18309 158 : return true;
18310 : }
18311 127 : emit_insn (gen_rtx_SET (target, CONST0_RTX (mode)));
18312 254 : var = force_reg (GET_MODE_INNER (mode), var);
18313 127 : ix86_expand_vector_set (mmx_ok, target, var, one_var);
18314 127 : return true;
18315 : }
18316 :
18317 868 : switch (mode)
18318 : {
18319 783 : case E_V2SFmode:
18320 783 : case E_V2SImode:
18321 783 : if (!mmx_ok || one_var != 0)
18322 : return false;
18323 350 : var = force_reg (GET_MODE_INNER (mode), var);
18324 350 : x = gen_rtx_VEC_CONCAT (mode, var, CONST0_RTX (GET_MODE_INNER (mode)));
18325 175 : emit_insn (gen_rtx_SET (target, x));
18326 175 : return true;
18327 :
18328 0 : case E_V2DFmode:
18329 0 : case E_V2DImode:
18330 0 : if (one_var != 0)
18331 : return false;
18332 0 : var = force_reg (GET_MODE_INNER (mode), var);
18333 0 : x = gen_rtx_VEC_CONCAT (mode, var, CONST0_RTX (GET_MODE_INNER (mode)));
18334 0 : emit_insn (gen_rtx_SET (target, x));
18335 0 : return true;
18336 :
18337 0 : case E_V8HImode:
18338 0 : if (one_var > 1 && !TARGET_SSE2)
18339 : return false;
18340 : /* Zero extend the variable element to SImode and recurse. */
18341 0 : var = convert_modes (SImode, HImode, var, true);
18342 0 : var = force_reg (SImode, var);
18343 0 : if (one_var == 1)
18344 : {
18345 0 : var = simplify_gen_binary (ASHIFT, SImode, var, GEN_INT (16));
18346 0 : var = force_reg (SImode, var);
18347 0 : one_var = 0;
18348 : }
18349 : else
18350 0 : one_var *= 16;
18351 0 : x = gen_reg_rtx (V4SImode);
18352 0 : if (!ix86_expand_vector_init_one_nonzero (mmx_ok, V4SImode, x, var, 0))
18353 0 : gcc_unreachable ();
18354 0 : if (one_var)
18355 : {
18356 0 : tmp = gen_reg_rtx (V4SImode);
18357 0 : emit_insn (gen_vec_shl_v4si (tmp, x, GEN_INT (one_var)));
18358 0 : x = tmp;
18359 : }
18360 0 : emit_move_insn (target, gen_lowpart (mode, x));
18361 0 : return true;
18362 :
18363 30 : case E_V16QImode:
18364 30 : if (one_var > 3 && !TARGET_SSE2)
18365 : return false;
18366 : /* Zero extend the variable element to SImode and recurse. */
18367 30 : var = convert_modes (SImode, QImode, var, true);
18368 30 : var = force_reg (SImode, var);
18369 30 : if (one_var < 4)
18370 : {
18371 14 : var = simplify_gen_binary (ASHIFT, SImode, var,
18372 14 : GEN_INT (one_var * 8));
18373 14 : var = force_reg (SImode, var);
18374 14 : one_var = 0;
18375 : }
18376 : else
18377 16 : one_var *= 8;
18378 30 : x = gen_reg_rtx (V4SImode);
18379 30 : if (!ix86_expand_vector_init_one_nonzero (mmx_ok, V4SImode, x, var, 0))
18380 0 : gcc_unreachable ();
18381 30 : if (one_var)
18382 : {
18383 16 : tmp = gen_reg_rtx (V4SImode);
18384 16 : emit_insn (gen_vec_shl_v4si (tmp, x, GEN_INT (one_var)));
18385 16 : x = tmp;
18386 : }
18387 30 : emit_move_insn (target, gen_lowpart (V16QImode, x));
18388 30 : return true;
18389 :
18390 0 : case E_V4HImode:
18391 0 : if (!mmx_ok || one_var > 1)
18392 : return false;
18393 : /* Zero extend the variable element to SImode and recurse. */
18394 0 : var = convert_modes (SImode, HImode, var, true);
18395 0 : var = force_reg (SImode, var);
18396 0 : if (one_var == 1)
18397 : {
18398 0 : var = simplify_gen_binary (ASHIFT, SImode, var, GEN_INT (16));
18399 0 : var = force_reg (SImode, var);
18400 : }
18401 0 : x = gen_reg_rtx (V2SImode);
18402 0 : if (!ix86_expand_vector_init_one_nonzero (true, V2SImode, x, var, 0))
18403 0 : gcc_unreachable ();
18404 0 : emit_move_insn (target, gen_lowpart (V4HImode, x));
18405 0 : return true;
18406 :
18407 2 : case E_V8QImode:
18408 2 : if (!mmx_ok || one_var > 3)
18409 : return false;
18410 : /* Zero extend the variable element to SImode and recurse. */
18411 2 : var = convert_modes (SImode, QImode, var, true);
18412 2 : var = force_reg (SImode, var);
18413 2 : if (one_var)
18414 : {
18415 0 : var = simplify_gen_binary (ASHIFT, SImode, var,
18416 0 : GEN_INT (one_var * 8));
18417 0 : var = force_reg (SImode, var);
18418 : }
18419 2 : x = gen_reg_rtx (V2SImode);
18420 2 : if (!ix86_expand_vector_init_one_nonzero (true, V2SImode, x, var, 0))
18421 0 : gcc_unreachable ();
18422 2 : emit_move_insn (target, gen_lowpart (V8QImode, x));
18423 2 : return true;
18424 :
18425 : default:
18426 : return false;
18427 : }
18428 : }
18429 :
18430 : /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
18431 : consisting of the values in VALS. It is known that all elements
18432 : except ONE_VAR are constants. Return true if successful. */
18433 :
18434 : static bool
18435 1788 : ix86_expand_vector_init_one_var (bool mmx_ok, machine_mode mode,
18436 : rtx target, rtx vals, int one_var)
18437 : {
18438 1788 : rtx var = XVECEXP (vals, 0, one_var);
18439 1788 : machine_mode wmode;
18440 1788 : rtx const_vec, x;
18441 :
18442 1788 : const_vec = copy_rtx (vals);
18443 1788 : XVECEXP (const_vec, 0, one_var) = CONST0_RTX (GET_MODE_INNER (mode));
18444 1788 : const_vec = gen_rtx_CONST_VECTOR (mode, XVEC (const_vec, 0));
18445 :
18446 1788 : switch (mode)
18447 : {
18448 : case E_V2DFmode:
18449 : case E_V2DImode:
18450 : case E_V2SFmode:
18451 : case E_V2SImode:
18452 : /* For the two element vectors, it's just as easy to use
18453 : the general case. */
18454 : return false;
18455 :
18456 3 : case E_V4DImode:
18457 : /* Use ix86_expand_vector_set in 64bit mode only. */
18458 3 : if (!TARGET_64BIT)
18459 : return false;
18460 : /* FALLTHRU */
18461 : case E_V8HFmode:
18462 : case E_V16HFmode:
18463 : case E_V8BFmode:
18464 : case E_V16BFmode:
18465 : case E_V4DFmode:
18466 : case E_V8SFmode:
18467 : case E_V8SImode:
18468 : case E_V16HImode:
18469 : case E_V32QImode:
18470 : case E_V4SFmode:
18471 : case E_V4SImode:
18472 : case E_V8HImode:
18473 : case E_V4HImode:
18474 : case E_V4HFmode:
18475 : case E_V4BFmode:
18476 : break;
18477 :
18478 1 : case E_V16QImode:
18479 1 : if (TARGET_SSE4_1)
18480 : break;
18481 1 : wmode = V8HImode;
18482 1 : goto widen;
18483 0 : case E_V8QImode:
18484 0 : if (TARGET_MMX_WITH_SSE && TARGET_SSE4_1)
18485 : break;
18486 0 : wmode = V4HImode;
18487 0 : goto widen;
18488 38 : case E_V4QImode:
18489 38 : if (TARGET_SSE4_1)
18490 : break;
18491 : wmode = V2HImode;
18492 39 : widen:
18493 : /* There's no way to set one QImode entry easily. Combine
18494 : the variable value with its adjacent constant value, and
18495 : promote to an HImode set. */
18496 39 : x = XVECEXP (vals, 0, one_var ^ 1);
18497 39 : if (one_var & 1)
18498 : {
18499 8 : var = convert_modes (HImode, QImode, var, true);
18500 8 : var = expand_simple_binop (HImode, ASHIFT, var, GEN_INT (8),
18501 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
18502 8 : x = GEN_INT (INTVAL (x) & 0xff);
18503 : }
18504 : else
18505 : {
18506 31 : var = convert_modes (HImode, QImode, var, true);
18507 31 : x = gen_int_mode (UINTVAL (x) << 8, HImode);
18508 : }
18509 39 : if (x != const0_rtx)
18510 7 : var = expand_simple_binop (HImode, IOR, var, x, var,
18511 : 1, OPTAB_LIB_WIDEN);
18512 :
18513 39 : x = gen_reg_rtx (wmode);
18514 39 : emit_move_insn (x, gen_lowpart (wmode, const_vec));
18515 39 : ix86_expand_vector_set (mmx_ok, x, var, one_var >> 1);
18516 :
18517 39 : emit_move_insn (target, gen_lowpart (mode, x));
18518 39 : return true;
18519 :
18520 : default:
18521 : return false;
18522 : }
18523 :
18524 193 : emit_move_insn (target, const_vec);
18525 193 : ix86_expand_vector_set (mmx_ok, target, var, one_var);
18526 193 : return true;
18527 : }
18528 :
18529 : /* A subroutine of ix86_expand_vector_init_general. Use vector
18530 : concatenate to handle the most general case: all values variable,
18531 : and none identical. */
18532 :
18533 : static void
18534 21390 : ix86_expand_vector_init_concat (machine_mode mode,
18535 : rtx target, rtx *ops, int n)
18536 : {
18537 21390 : machine_mode half_mode = VOIDmode;
18538 21390 : rtx half[2];
18539 21390 : rtvec v;
18540 21390 : int i, j;
18541 :
18542 21390 : switch (n)
18543 : {
18544 21217 : case 2:
18545 21217 : switch (mode)
18546 : {
18547 : case E_V32HFmode:
18548 : half_mode = V16HFmode;
18549 : break;
18550 0 : case E_V32BFmode:
18551 0 : half_mode = V16BFmode;
18552 0 : break;
18553 59 : case E_V16SImode:
18554 59 : half_mode = V8SImode;
18555 59 : break;
18556 33 : case E_V16SFmode:
18557 33 : half_mode = V8SFmode;
18558 33 : break;
18559 84 : case E_V8DImode:
18560 84 : half_mode = V4DImode;
18561 84 : break;
18562 59 : case E_V8DFmode:
18563 59 : half_mode = V4DFmode;
18564 59 : break;
18565 0 : case E_V16HFmode:
18566 0 : half_mode = V8HFmode;
18567 0 : break;
18568 0 : case E_V16BFmode:
18569 0 : half_mode = V8BFmode;
18570 0 : break;
18571 37 : case E_V8SImode:
18572 37 : half_mode = V4SImode;
18573 37 : break;
18574 6 : case E_V8SFmode:
18575 6 : half_mode = V4SFmode;
18576 6 : break;
18577 126 : case E_V4DImode:
18578 126 : half_mode = V2DImode;
18579 126 : break;
18580 57 : case E_V4DFmode:
18581 57 : half_mode = V2DFmode;
18582 57 : break;
18583 956 : case E_V4SImode:
18584 956 : half_mode = V2SImode;
18585 956 : break;
18586 279 : case E_V4SFmode:
18587 279 : half_mode = V2SFmode;
18588 279 : break;
18589 0 : case E_V2DImode:
18590 0 : half_mode = DImode;
18591 0 : break;
18592 17657 : case E_V2SImode:
18593 17657 : half_mode = SImode;
18594 17657 : break;
18595 0 : case E_V2DFmode:
18596 0 : half_mode = DFmode;
18597 0 : break;
18598 1864 : case E_V2SFmode:
18599 1864 : half_mode = SFmode;
18600 1864 : break;
18601 0 : default:
18602 0 : gcc_unreachable ();
18603 : }
18604 :
18605 21217 : if (!register_operand (ops[1], half_mode))
18606 9982 : ops[1] = force_reg (half_mode, ops[1]);
18607 21217 : if (!register_operand (ops[0], half_mode))
18608 5999 : ops[0] = force_reg (half_mode, ops[0]);
18609 21217 : emit_insn (gen_rtx_SET (target, gen_rtx_VEC_CONCAT (mode, ops[0],
18610 : ops[1])));
18611 21217 : break;
18612 :
18613 0 : case 4:
18614 0 : switch (mode)
18615 : {
18616 : case E_V4DImode:
18617 : half_mode = V2DImode;
18618 : break;
18619 0 : case E_V4DFmode:
18620 0 : half_mode = V2DFmode;
18621 0 : break;
18622 0 : case E_V4SImode:
18623 0 : half_mode = V2SImode;
18624 0 : break;
18625 0 : case E_V4SFmode:
18626 0 : half_mode = V2SFmode;
18627 0 : break;
18628 0 : default:
18629 0 : gcc_unreachable ();
18630 : }
18631 0 : goto half;
18632 :
18633 102 : case 8:
18634 102 : switch (mode)
18635 : {
18636 : case E_V8DImode:
18637 : half_mode = V4DImode;
18638 : break;
18639 59 : case E_V8DFmode:
18640 59 : half_mode = V4DFmode;
18641 59 : break;
18642 0 : case E_V8SImode:
18643 0 : half_mode = V4SImode;
18644 0 : break;
18645 0 : case E_V8SFmode:
18646 0 : half_mode = V4SFmode;
18647 0 : break;
18648 0 : default:
18649 0 : gcc_unreachable ();
18650 : }
18651 102 : goto half;
18652 :
18653 71 : case 16:
18654 71 : switch (mode)
18655 : {
18656 : case E_V16SImode:
18657 : half_mode = V8SImode;
18658 : break;
18659 33 : case E_V16SFmode:
18660 33 : half_mode = V8SFmode;
18661 33 : break;
18662 0 : default:
18663 0 : gcc_unreachable ();
18664 : }
18665 71 : goto half;
18666 :
18667 173 : half:
18668 : /* FIXME: We process inputs backward to help RA. PR 36222. */
18669 173 : i = n - 1;
18670 519 : for (j = 1; j != -1; j--)
18671 : {
18672 346 : half[j] = gen_reg_rtx (half_mode);
18673 346 : switch (n >> 1)
18674 : {
18675 0 : case 2:
18676 0 : v = gen_rtvec (2, ops[i-1], ops[i]);
18677 0 : i -= 2;
18678 0 : break;
18679 204 : case 4:
18680 204 : v = gen_rtvec (4, ops[i-3], ops[i-2], ops[i-1], ops[i]);
18681 204 : i -= 4;
18682 204 : break;
18683 142 : case 8:
18684 284 : v = gen_rtvec (8, ops[i-7], ops[i-6], ops[i-5], ops[i-4],
18685 142 : ops[i-3], ops[i-2], ops[i-1], ops[i]);
18686 142 : i -= 8;
18687 142 : break;
18688 : default:
18689 : gcc_unreachable ();
18690 : }
18691 346 : ix86_expand_vector_init (false, half[j],
18692 : gen_rtx_PARALLEL (half_mode, v));
18693 : }
18694 :
18695 173 : ix86_expand_vector_init_concat (mode, target, half, 2);
18696 173 : break;
18697 :
18698 0 : default:
18699 0 : gcc_unreachable ();
18700 : }
18701 21390 : }
18702 :
18703 : /* A subroutine of ix86_expand_vector_init_general. Use vector
18704 : interleave to handle the most general case: all values variable,
18705 : and none identical. */
18706 :
18707 : static void
18708 3490 : ix86_expand_vector_init_interleave (machine_mode mode,
18709 : rtx target, rtx *ops, int n)
18710 : {
18711 3490 : machine_mode first_imode, second_imode, third_imode, inner_mode;
18712 3490 : int i, j;
18713 3490 : rtx op, op0, op1;
18714 3490 : rtx (*gen_load_even) (rtx, rtx, rtx);
18715 3490 : rtx (*gen_interleave_first_low) (rtx, rtx, rtx);
18716 3490 : rtx (*gen_interleave_second_low) (rtx, rtx, rtx);
18717 :
18718 3490 : switch (mode)
18719 : {
18720 : case E_V8HFmode:
18721 : gen_load_even = gen_vec_interleave_lowv8hf;
18722 : gen_interleave_first_low = gen_vec_interleave_lowv4si;
18723 : gen_interleave_second_low = gen_vec_interleave_lowv2di;
18724 : inner_mode = HFmode;
18725 : first_imode = V4SImode;
18726 : second_imode = V2DImode;
18727 : third_imode = VOIDmode;
18728 : break;
18729 487 : case E_V8BFmode:
18730 487 : gen_load_even = gen_vec_interleave_lowv8bf;
18731 487 : gen_interleave_first_low = gen_vec_interleave_lowv4si;
18732 487 : gen_interleave_second_low = gen_vec_interleave_lowv2di;
18733 487 : inner_mode = BFmode;
18734 487 : first_imode = V4SImode;
18735 487 : second_imode = V2DImode;
18736 487 : third_imode = VOIDmode;
18737 487 : break;
18738 414 : case E_V8HImode:
18739 414 : gen_load_even = gen_vec_setv8hi;
18740 414 : gen_interleave_first_low = gen_vec_interleave_lowv4si;
18741 414 : gen_interleave_second_low = gen_vec_interleave_lowv2di;
18742 414 : inner_mode = HImode;
18743 414 : first_imode = V4SImode;
18744 414 : second_imode = V2DImode;
18745 414 : third_imode = VOIDmode;
18746 414 : break;
18747 362 : case E_V16QImode:
18748 362 : gen_load_even = gen_vec_setv16qi;
18749 362 : gen_interleave_first_low = gen_vec_interleave_lowv8hi;
18750 362 : gen_interleave_second_low = gen_vec_interleave_lowv4si;
18751 362 : inner_mode = QImode;
18752 362 : first_imode = V8HImode;
18753 362 : second_imode = V4SImode;
18754 362 : third_imode = V2DImode;
18755 362 : break;
18756 0 : default:
18757 0 : gcc_unreachable ();
18758 : }
18759 :
18760 18898 : for (i = 0; i < n; i++)
18761 : {
18762 15408 : op = ops [i + i];
18763 15408 : if (inner_mode == HFmode || inner_mode == BFmode)
18764 : {
18765 10856 : rtx even, odd;
18766 : /* Use vpuncklwd to pack 2 HFmode or BFmode. */
18767 1948 : machine_mode vec_mode =
18768 10856 : (inner_mode == HFmode) ? V8HFmode : V8BFmode;
18769 10856 : op0 = gen_reg_rtx (vec_mode);
18770 10856 : even = lowpart_subreg (vec_mode,
18771 : force_reg (inner_mode, op), inner_mode);
18772 10856 : odd = lowpart_subreg (vec_mode,
18773 10856 : force_reg (inner_mode, ops[i + i + 1]),
18774 : inner_mode);
18775 10856 : emit_insn (gen_load_even (op0, even, odd));
18776 : }
18777 : else
18778 : {
18779 : /* Extend the odd element to SImode using a paradoxical SUBREG. */
18780 4552 : op0 = gen_reg_rtx (SImode);
18781 4552 : emit_move_insn (op0, gen_lowpart (SImode, op));
18782 :
18783 : /* Insert the SImode value as low element of V4SImode vector. */
18784 4552 : op1 = gen_reg_rtx (V4SImode);
18785 4552 : op0 = gen_rtx_VEC_MERGE (V4SImode,
18786 : gen_rtx_VEC_DUPLICATE (V4SImode,
18787 : op0),
18788 : CONST0_RTX (V4SImode),
18789 : const1_rtx);
18790 4552 : emit_insn (gen_rtx_SET (op1, op0));
18791 :
18792 : /* Cast the V4SImode vector back to a vector in original mode. */
18793 4552 : op0 = gen_reg_rtx (mode);
18794 4552 : emit_move_insn (op0, gen_lowpart (mode, op1));
18795 :
18796 : /* Load even elements into the second position. */
18797 4552 : emit_insn (gen_load_even (op0,
18798 : force_reg (inner_mode,
18799 4552 : ops[i + i + 1]),
18800 : const1_rtx));
18801 : }
18802 :
18803 : /* Cast vector to FIRST_IMODE vector. */
18804 15408 : ops[i] = gen_reg_rtx (first_imode);
18805 15408 : emit_move_insn (ops[i], gen_lowpart (first_imode, op0));
18806 : }
18807 :
18808 : /* Interleave low FIRST_IMODE vectors. */
18809 11194 : for (i = j = 0; i < n; i += 2, j++)
18810 : {
18811 7704 : op0 = gen_reg_rtx (first_imode);
18812 7704 : emit_insn (gen_interleave_first_low (op0, ops[i], ops[i + 1]));
18813 :
18814 : /* Cast FIRST_IMODE vector to SECOND_IMODE vector. */
18815 7704 : ops[j] = gen_reg_rtx (second_imode);
18816 7704 : emit_move_insn (ops[j], gen_lowpart (second_imode, op0));
18817 : }
18818 :
18819 : /* Interleave low SECOND_IMODE vectors. */
18820 3490 : switch (second_imode)
18821 : {
18822 : case E_V4SImode:
18823 1086 : for (i = j = 0; i < n / 2; i += 2, j++)
18824 : {
18825 724 : op0 = gen_reg_rtx (second_imode);
18826 724 : emit_insn (gen_interleave_second_low (op0, ops[i],
18827 724 : ops[i + 1]));
18828 :
18829 : /* Cast the SECOND_IMODE vector to the THIRD_IMODE
18830 : vector. */
18831 724 : ops[j] = gen_reg_rtx (third_imode);
18832 724 : emit_move_insn (ops[j], gen_lowpart (third_imode, op0));
18833 : }
18834 : second_imode = V2DImode;
18835 : gen_interleave_second_low = gen_vec_interleave_lowv2di;
18836 : /* FALLTHRU */
18837 :
18838 3490 : case E_V2DImode:
18839 3490 : op0 = gen_reg_rtx (second_imode);
18840 3490 : emit_insn (gen_interleave_second_low (op0, ops[0],
18841 : ops[1]));
18842 :
18843 : /* Cast the SECOND_IMODE vector back to a vector on original
18844 : mode. */
18845 3490 : emit_insn (gen_rtx_SET (target, gen_lowpart (mode, op0)));
18846 3490 : break;
18847 :
18848 : default:
18849 : gcc_unreachable ();
18850 : }
18851 3490 : }
18852 :
18853 : /* Count number of non-zero integer constants in OPS array of length N. */
18854 :
18855 : static int
18856 5418 : nonzero_int_const_count (rtx *ops, int n)
18857 : {
18858 5418 : int result = 0;
18859 5418 : int i;
18860 30282 : for (i = 0; i < n; i++)
18861 24864 : if (CONST_INT_P (ops[i]) && ops[i] != const0_rtx)
18862 201 : result++;
18863 5418 : return result;
18864 : }
18865 :
18866 : /* Count number of non-zero float constants in OPS array of length N. */
18867 :
18868 : static int
18869 1898 : nonzero_float_const_count (rtx *ops, int n)
18870 : {
18871 1898 : int result = 0;
18872 1898 : int i;
18873 10398 : for (i = 0; i < n; i++)
18874 8500 : if (CONST_DOUBLE_P (ops[i]) && ops[i] != CONST0_RTX (SFmode))
18875 12 : result++;
18876 1898 : return result;
18877 : }
18878 :
18879 : /* Count number of non-zero double constants in OPS array of length N. */
18880 :
18881 : static int
18882 486 : nonzero_double_const_count (rtx *ops, int n)
18883 : {
18884 486 : int result = 0;
18885 486 : int i;
18886 2430 : for (i = 0; i < n; i++)
18887 1944 : if (CONST_DOUBLE_P (ops[i]) && ops[i] != CONST0_RTX (DFmode))
18888 2 : result++;
18889 486 : return result;
18890 : }
18891 :
18892 :
18893 : /* A subroutine of ix86_expand_vector_init_general. Handle the most
18894 : general case: all values variable and none identical. */
18895 :
18896 : static void
18897 898 : ix86_expand_vector_init_insert (machine_mode mode, rtx target,
18898 : rtx *ops, int n)
18899 : {
18900 898 : machine_mode inner_mode = GET_MODE_INNER (mode);
18901 898 : rtx (*pinsr)(rtx, rtx, rtx, rtx);
18902 898 : rtx tmp = gen_reg_rtx (mode);
18903 898 : rtx var, x;
18904 898 : int i;
18905 :
18906 898 : var = ops[0];
18907 :
18908 898 : switch (mode)
18909 : {
18910 68 : case E_V16QImode:
18911 68 : if (var != const0_rtx)
18912 : {
18913 68 : var = convert_modes (SImode, QImode, var, true);
18914 68 : var = force_reg (SImode, var);
18915 68 : x = gen_reg_rtx (V4SImode);
18916 68 : emit_insn (gen_vec_setv4si_0 (x, CONST0_RTX (V4SImode), var));
18917 68 : emit_move_insn (tmp, gen_lowpart (V16QImode, x));
18918 : }
18919 : else
18920 0 : emit_move_insn (tmp, CONST0_RTX (mode));
18921 : pinsr = gen_sse4_1_pinsrb;
18922 : break;
18923 511 : case E_V8HImode:
18924 511 : if (var != const0_rtx)
18925 : {
18926 507 : var = convert_modes (SImode, HImode, var, true);
18927 507 : var = force_reg (SImode, var);
18928 507 : x = gen_reg_rtx (V4SImode);
18929 507 : emit_insn (gen_vec_setv4si_0 (x, CONST0_RTX (V4SImode), var));
18930 507 : emit_move_insn (tmp, gen_lowpart (V8HImode, x));
18931 : }
18932 : else
18933 4 : emit_move_insn (tmp, CONST0_RTX (mode));
18934 : pinsr = gen_sse2_pinsrw;
18935 : break;
18936 319 : case E_V4SImode:
18937 319 : if (var != const0_rtx)
18938 : {
18939 316 : if (!REG_P (var) && !MEM_P (var))
18940 4 : var = force_reg (SImode, var);
18941 316 : emit_insn (gen_vec_setv4si_0 (tmp, CONST0_RTX (mode), var));
18942 : }
18943 : else
18944 3 : emit_move_insn (tmp, CONST0_RTX (mode));
18945 : pinsr = gen_sse4_1_pinsrd;
18946 : break;
18947 :
18948 0 : default:
18949 0 : gcc_unreachable ();
18950 : }
18951 :
18952 6452 : for (i=1; i<n; i++)
18953 5554 : if (ops[i] != CONST0_RTX (inner_mode))
18954 : {
18955 5490 : rtx val = ops[i];
18956 5490 : if (!REG_P (val) && !MEM_P (val))
18957 245 : val = force_reg (inner_mode, val);
18958 5490 : emit_insn (pinsr (tmp, tmp, val, GEN_INT (1 << i)));
18959 : }
18960 898 : emit_move_insn (target, tmp);
18961 898 : }
18962 :
18963 : /* Helper function. Determine if the given OPS array of size N
18964 : contains only zeros and one other value (possible repeated).
18965 : If TRUE, *VAR returns the value, PERM[i] contains 0 for
18966 : this value and 1 for a CONST0_RTX. */
18967 :
18968 : static bool
18969 20190 : onevar_perm_p (const rtx *ops, int n, int *perm, rtx *var)
18970 : {
18971 20190 : bool found = false;
18972 20190 : int i;
18973 :
18974 42195 : for (i = 0; i < n; i++)
18975 41863 : if (ops[i] == const0_rtx
18976 41032 : || ops[i] == CONST0_RTX (SFmode)
18977 40616 : || ops[i] == CONST0_RTX (DFmode))
18978 1247 : perm[i] = 1;
18979 40616 : else if (!found)
18980 : {
18981 20190 : *var = ops[i];
18982 20190 : found = true;
18983 20190 : perm[i] = 0;
18984 : }
18985 20426 : else if (rtx_equal_p (*var, ops[i]))
18986 568 : perm[i] = 0;
18987 : else
18988 : return false;
18989 :
18990 : return found;
18991 : }
18992 :
18993 : /* Helper function. Determine if the given OPS array of size N
18994 : contains only zeros and two other values (possible repeated).
18995 : If TRUE, VARS returns the values, PERM[i] contains 0 for
18996 : the first value, 1 for the second value and 2 for CONST0_RTX. */
18997 :
18998 : static bool
18999 6992 : twovar_perm_p (const rtx *ops, int n, int *perm, rtx *vars)
19000 : {
19001 6992 : int count = 0;
19002 6992 : int i;
19003 :
19004 23032 : for (i = 0; i < n; i++)
19005 22325 : if (ops[i] == const0_rtx
19006 21846 : || ops[i] == CONST0_RTX (SFmode)
19007 21708 : || ops[i] == CONST0_RTX (DFmode))
19008 617 : perm[i] = 2;
19009 21708 : else if (count == 0)
19010 : {
19011 6992 : vars[0] = ops[i];
19012 6992 : perm[i] = 0;
19013 6992 : count = 1;
19014 : }
19015 14716 : else if (rtx_equal_p (vars[0], ops[i]))
19016 1096 : perm[i] = 0;
19017 13620 : else if (count == 1)
19018 : {
19019 6992 : vars[1] = ops[i];
19020 6992 : perm[i] = 1;
19021 6992 : count = 2;
19022 : }
19023 6628 : else if (rtx_equal_p (vars[1], ops[i]))
19024 343 : perm[i] = 1;
19025 : else
19026 : return false;
19027 :
19028 707 : return count == 2;
19029 : }
19030 :
19031 : /* A subroutine of ix86_expand_vector_init for V2DImode. */
19032 :
19033 : static void
19034 60687 : ix86_expand_vector_init_v2di (rtx target, rtx *ops)
19035 : {
19036 60687 : if (ops[0] == const0_rtx && ops[1] == const0_rtx)
19037 2 : emit_move_insn (target, CONST0_RTX (V2DImode));
19038 60685 : else if (CONST_INT_P (ops[0]) && CONST_INT_P (ops[1]))
19039 : {
19040 0 : rtx vec = gen_rtx_CONST_VECTOR (V2DImode, gen_rtvec_v (2, ops));
19041 0 : emit_move_insn (target, vec);
19042 0 : }
19043 60685 : else if (rtx_equal_p (ops[0], ops[1]))
19044 : {
19045 6 : rtx val = ops[0];
19046 6 : if (!REG_P (val) && !MEM_P (val))
19047 0 : val = force_reg (DImode, val);
19048 : /* TARGET_SSE has *vec_dupv2di. */
19049 6 : emit_move_insn (target, gen_rtx_VEC_DUPLICATE (V2DImode, val));
19050 : }
19051 : else
19052 : {
19053 60679 : rtx op0 = force_reg (DImode, ops[0]);
19054 60679 : rtx op1 = force_reg (DImode, ops[1]);
19055 60679 : emit_insn (gen_vec_concatv2di (target, op0, op1));
19056 : }
19057 60687 : }
19058 :
19059 : /* A subroutine of ix86_expand_vector_init for V2DFmode. */
19060 :
19061 : static void
19062 3971 : ix86_expand_vector_init_v2df (rtx target, rtx *ops)
19063 : {
19064 3971 : if (ops[0] == CONST0_RTX (DFmode)
19065 0 : && ops[1] == CONST0_RTX (DFmode))
19066 0 : emit_move_insn (target, CONST0_RTX (V2DFmode));
19067 3971 : else if (CONST_DOUBLE_P (ops[0])
19068 32 : && CONST_DOUBLE_P (ops[1]))
19069 : {
19070 0 : rtx vec = gen_rtx_CONST_VECTOR (V2DFmode, gen_rtvec_v (2, ops));
19071 0 : emit_move_insn (target, vec);
19072 0 : }
19073 3971 : else if (TARGET_SSE2
19074 3971 : && rtx_equal_p (ops[0], ops[1]))
19075 : {
19076 96 : rtx val = ops[0];
19077 96 : if (!REG_P (val) && !MEM_P (val))
19078 0 : val = force_reg (DFmode, val);
19079 96 : emit_move_insn (target, gen_rtx_VEC_DUPLICATE (V2DFmode, val));
19080 : }
19081 : else
19082 : {
19083 3875 : rtx op0 = force_reg (DFmode, ops[0]);
19084 3875 : rtx op1 = force_reg (DFmode, ops[1]);
19085 3875 : emit_insn (gen_vec_concatv2df (target, op0, op1));
19086 : }
19087 3971 : }
19088 :
19089 : /* A subroutine of ix86_expand_vector_init for V4SImode. */
19090 :
19091 : static void
19092 32122 : ix86_expand_vector_init_v4si (rtx target, rtx *ops)
19093 : {
19094 32122 : rtx vars[4];
19095 32122 : int perm[4];
19096 :
19097 32122 : if (ops[0] == const0_rtx
19098 316 : && ops[1] == const0_rtx
19099 172 : && ops[2] == const0_rtx
19100 7 : && ops[3] == const0_rtx)
19101 6 : emit_move_insn (target, CONST0_RTX (V4SImode));
19102 32116 : else if (ops[1] == const0_rtx
19103 18223 : && ops[2] == const0_rtx
19104 17998 : && ops[3] == const0_rtx)
19105 : {
19106 17975 : rtx val = ops[0];
19107 17975 : if (!REG_P (val) && !MEM_P (val))
19108 2913 : val = force_reg (SImode, val);
19109 17975 : emit_insn (gen_vec_setv4si_0 (target, CONST0_RTX (V4SImode), val));
19110 17975 : }
19111 14141 : else if (rtx_equal_p (ops[0], ops[1])
19112 307 : && rtx_equal_p (ops[0], ops[2])
19113 14146 : && rtx_equal_p (ops[0], ops[3]))
19114 : {
19115 2 : rtx val = ops[0];
19116 2 : if (!REG_P (val) && !MEM_P (val))
19117 0 : val = force_reg (SImode, val);
19118 2 : emit_move_insn (target, gen_rtx_VEC_DUPLICATE (V4SImode, val));
19119 : }
19120 14139 : else if (CONST_INT_P (ops[0])
19121 423 : && CONST_INT_P (ops[1])
19122 181 : && CONST_INT_P (ops[2])
19123 1 : && CONST_INT_P (ops[3]))
19124 : {
19125 1 : rtx vec = gen_rtx_CONST_VECTOR (V4SImode, gen_rtvec_v (4, ops));
19126 1 : emit_move_insn (target, vec);
19127 1 : }
19128 14138 : else if (onevar_perm_p (ops, 4, perm, vars))
19129 : {
19130 167 : rtx tmp = gen_reg_rtx (V4SImode);
19131 167 : vars[1] = const0_rtx;
19132 167 : vars[2] = const0_rtx;
19133 167 : vars[3] = const0_rtx;
19134 167 : ix86_expand_vector_init_v4si (tmp, vars);
19135 167 : emit_insn (gen_sse_shufps_v4si (target, tmp, tmp,
19136 167 : GEN_INT (perm[0]),
19137 167 : GEN_INT (perm[1]),
19138 167 : GEN_INT (perm[2] + 4),
19139 167 : GEN_INT (perm[3] + 4)));
19140 : }
19141 13971 : else if (ops[2] == const0_rtx && ops[3] == const0_rtx)
19142 : {
19143 8897 : rtx tmp1 = gen_reg_rtx (V4SImode);
19144 8897 : vars[0] = ops[0];
19145 8897 : vars[1] = const0_rtx;
19146 8897 : vars[2] = const0_rtx;
19147 8897 : vars[3] = const0_rtx;
19148 8897 : ix86_expand_vector_init_v4si (tmp1, vars);
19149 :
19150 8897 : if (TARGET_SSE4_1)
19151 : {
19152 63 : rtx val = ops[1];
19153 63 : if (!REG_P (val) && !MEM_P (val))
19154 2 : val = force_reg (SImode, val);
19155 63 : emit_insn (gen_sse4_1_pinsrd (target, tmp1, val, GEN_INT (2)));
19156 : }
19157 : else
19158 : {
19159 8834 : rtx tmp2 = gen_reg_rtx (V4SImode);
19160 8834 : vars[0] = ops[1];
19161 8834 : ix86_expand_vector_init_v4si (tmp2, vars);
19162 8834 : emit_insn (gen_vec_interleave_lowv4si (target, tmp1, tmp2));
19163 : }
19164 : }
19165 5074 : else if (TARGET_SSE4_1
19166 348 : && ops[1] == const0_rtx
19167 4 : && (ops[2] == const0_rtx || ops[3] == const0_rtx))
19168 : /* { a, 0, b, 0 } and { a, 0, 0, b } become mov; pinsrd. */
19169 4 : ix86_expand_vector_init_insert (V4SImode, target, ops, 4);
19170 5070 : else if (twovar_perm_p (ops, 4, perm, vars))
19171 : {
19172 456 : rtx tmp = gen_reg_rtx (V4SImode);
19173 456 : vars[2] = const0_rtx;
19174 456 : vars[3] = const0_rtx;
19175 456 : ix86_expand_vector_init_v4si (tmp, vars);
19176 456 : emit_insn (gen_sse_shufps_v4si (target, tmp, tmp,
19177 456 : GEN_INT (perm[0]),
19178 456 : GEN_INT (perm[1]),
19179 456 : GEN_INT (perm[2] + 4),
19180 456 : GEN_INT (perm[3] + 4)));
19181 : }
19182 4614 : else if (nonzero_int_const_count (ops, 4) >= 2)
19183 : {
19184 : rtx csts[4];
19185 : int i;
19186 445 : for (i = 0; i < 4; i++)
19187 356 : if (CONST_INT_P (ops[i]))
19188 : {
19189 178 : csts[i] = ops[i];
19190 178 : vars[i] = const0_rtx;
19191 : }
19192 : else
19193 : {
19194 178 : csts[i] = const0_rtx;
19195 178 : vars[i] = ops[i];
19196 : }
19197 89 : rtx tmp1 = gen_reg_rtx (V4SImode);
19198 89 : ix86_expand_vector_init_v4si (tmp1, vars);
19199 89 : rtx tmp2 = gen_reg_rtx (V4SImode);
19200 89 : rtx vec = gen_rtx_CONST_VECTOR (V4SImode, gen_rtvec_v (4, csts));
19201 89 : emit_move_insn (tmp2, vec);
19202 89 : emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V4SImode, tmp1, tmp2)));
19203 : }
19204 4525 : else if (TARGET_SSE4_1)
19205 315 : ix86_expand_vector_init_insert (V4SImode, target, ops, 4);
19206 : else
19207 : {
19208 4210 : rtx tmp1 = gen_reg_rtx (V4SImode);
19209 4210 : rtx tmp2 = gen_reg_rtx (V4SImode);
19210 4210 : vars[0] = ops[0];
19211 4210 : vars[1] = ops[1];
19212 4210 : vars[2] = const0_rtx;
19213 4210 : vars[3] = const0_rtx;
19214 4210 : ix86_expand_vector_init_v4si (tmp1, vars);
19215 4210 : vars[0] = ops[2];
19216 4210 : vars[1] = ops[3];
19217 4210 : ix86_expand_vector_init_v4si (tmp2, vars);
19218 4210 : emit_insn (gen_sse_shufps_v4si (target, tmp1, tmp2,
19219 : const0_rtx, const1_rtx,
19220 : GEN_INT (4), GEN_INT (5)));
19221 : }
19222 32122 : }
19223 :
19224 : /* A subroutine of ix86_expand_vector_init for V4SFmode. */
19225 :
19226 : static void
19227 12721 : ix86_expand_vector_init_v4sf (rtx target, rtx *ops)
19228 : {
19229 12721 : rtx vars[4];
19230 12721 : int perm[4];
19231 :
19232 12721 : if (ops[0] == CONST0_RTX (SFmode)
19233 27 : && ops[1] == CONST0_RTX (SFmode)
19234 1 : && ops[2] == CONST0_RTX (SFmode)
19235 0 : && ops[3] == CONST0_RTX (SFmode))
19236 0 : emit_move_insn (target, CONST0_RTX (V4SFmode));
19237 12721 : else if (ops[1] == CONST0_RTX (SFmode)
19238 7224 : && ops[2] == CONST0_RTX (SFmode)
19239 7114 : && ops[3] == CONST0_RTX (SFmode))
19240 : {
19241 7114 : rtx val = ops[0];
19242 7114 : if (!REG_P (val) && !MEM_P (val))
19243 8 : val = force_reg (SFmode, val);
19244 7114 : emit_insn (gen_vec_setv4sf_0 (target, CONST0_RTX (V4SFmode), val));
19245 7114 : }
19246 5607 : else if (rtx_equal_p (ops[0], ops[1])
19247 287 : && rtx_equal_p (ops[0], ops[2])
19248 5670 : && rtx_equal_p (ops[0], ops[3]))
19249 : {
19250 56 : rtx val = ops[0];
19251 56 : if (!REG_P (val) && !MEM_P (val))
19252 0 : val = force_reg (SFmode, val);
19253 56 : emit_move_insn (target, gen_rtx_VEC_DUPLICATE (V4SFmode, val));
19254 : }
19255 5551 : else if (CONST_DOUBLE_P (ops[0])
19256 39 : && CONST_DOUBLE_P (ops[1])
19257 4 : && CONST_DOUBLE_P (ops[2])
19258 2 : && CONST_DOUBLE_P (ops[3]))
19259 : {
19260 2 : rtx vec = gen_rtx_CONST_VECTOR (V4SFmode, gen_rtvec_v (4, ops));
19261 2 : emit_move_insn (target, vec);
19262 2 : }
19263 5549 : else if (onevar_perm_p (ops, 4, perm, vars))
19264 : {
19265 165 : rtx tmp = gen_reg_rtx (V4SFmode);
19266 165 : vars[1] = CONST0_RTX (SFmode);
19267 165 : vars[2] = CONST0_RTX (SFmode);
19268 165 : vars[3] = CONST0_RTX (SFmode);
19269 165 : ix86_expand_vector_init_v4sf (tmp, vars);
19270 165 : emit_insn (gen_sse_shufps_v4sf (target, tmp, tmp,
19271 165 : GEN_INT (perm[0]),
19272 165 : GEN_INT (perm[1]),
19273 165 : GEN_INT (perm[2] + 4),
19274 165 : GEN_INT (perm[3] + 4)));
19275 : }
19276 5384 : else if (ops[2] == CONST0_RTX (SFmode)
19277 3464 : && ops[3] == CONST0_RTX (SFmode))
19278 : {
19279 3462 : rtx tmp1 = gen_reg_rtx (V4SFmode);
19280 3462 : vars[0] = ops[0];
19281 3462 : vars[1] = CONST0_RTX (SFmode);
19282 3462 : vars[2] = CONST0_RTX (SFmode);
19283 3462 : vars[3] = CONST0_RTX (SFmode);
19284 3462 : ix86_expand_vector_init_v4sf (tmp1, vars);
19285 :
19286 3462 : rtx tmp2 = gen_reg_rtx (V4SFmode);
19287 3462 : vars[0] = ops[1];
19288 3462 : ix86_expand_vector_init_v4sf (tmp2, vars);
19289 3462 : emit_insn (gen_vec_interleave_lowv4sf (target, tmp1, tmp2));
19290 3462 : }
19291 1922 : else if (twovar_perm_p (ops, 4, perm, vars))
19292 : {
19293 251 : rtx tmp = gen_reg_rtx (V4SFmode);
19294 251 : vars[2] = CONST0_RTX (SFmode);
19295 251 : vars[3] = CONST0_RTX (SFmode);
19296 251 : ix86_expand_vector_init_v4sf (tmp, vars);
19297 251 : emit_insn (gen_sse_shufps_v4sf (target, tmp, tmp,
19298 251 : GEN_INT (perm[0]),
19299 251 : GEN_INT (perm[1]),
19300 251 : GEN_INT (perm[2] + 4),
19301 251 : GEN_INT (perm[3] + 4)));
19302 : }
19303 1671 : else if (nonzero_float_const_count (ops, 4) >= 2)
19304 : {
19305 : rtx csts[4];
19306 : int i;
19307 10 : for (i = 0; i < 4; i++)
19308 8 : if (CONST_DOUBLE_P (ops[i]))
19309 : {
19310 4 : csts[i] = ops[i];
19311 4 : vars[i] = CONST0_RTX (SFmode);
19312 : }
19313 : else
19314 : {
19315 4 : csts[i] = CONST0_RTX (SFmode);
19316 4 : vars[i] = ops[i];
19317 : }
19318 2 : rtx tmp1 = gen_reg_rtx (V4SFmode);
19319 2 : ix86_expand_vector_init_v4sf (tmp1, vars);
19320 2 : rtx tmp2 = gen_reg_rtx (V4SFmode);
19321 2 : rtx vec = gen_rtx_CONST_VECTOR (V4SFmode, gen_rtvec_v (4, csts));
19322 2 : emit_move_insn (tmp2, vec);
19323 2 : emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V4SFmode, tmp1, tmp2)));
19324 : }
19325 : else
19326 : {
19327 1669 : rtx tmp1 = gen_reg_rtx (V4SFmode);
19328 1669 : rtx tmp2 = gen_reg_rtx (V4SFmode);
19329 1669 : vars[0] = ops[0];
19330 1669 : vars[1] = ops[1];
19331 1669 : vars[2] = CONST0_RTX (SFmode);
19332 1669 : vars[3] = CONST0_RTX (SFmode);
19333 1669 : ix86_expand_vector_init_v4sf (tmp1, vars);
19334 1669 : vars[0] = ops[2];
19335 1669 : vars[1] = ops[3];
19336 1669 : ix86_expand_vector_init_v4sf (tmp2, vars);
19337 1669 : emit_insn (gen_sse_shufps_v4sf (target, tmp1, tmp2,
19338 : const0_rtx, const1_rtx,
19339 : GEN_INT (4), GEN_INT (5)));
19340 : }
19341 12721 : }
19342 :
19343 : /* A subroutine of ix86_expand_vector_init for V8HImode. */
19344 :
19345 : static bool
19346 511 : ix86_expand_vector_init_v8hi (rtx target, rtx *ops)
19347 : {
19348 511 : rtx vars[8];
19349 511 : int i;
19350 :
19351 511 : bool all_zero_p = true;
19352 515 : for (i = 0; i < 8; i++)
19353 515 : if (ops[i] != const0_rtx)
19354 : {
19355 : all_zero_p = false;
19356 : break;
19357 : }
19358 511 : if (all_zero_p)
19359 : {
19360 0 : emit_move_insn (target, CONST0_RTX (V8HImode));
19361 0 : return true;
19362 : }
19363 :
19364 520 : bool all_const_p = true;
19365 520 : for (i = 0; i < 8; i++)
19366 520 : if (!CONST_INT_P (ops[i]))
19367 : {
19368 : all_const_p = false;
19369 : break;
19370 : }
19371 511 : if (all_const_p)
19372 : {
19373 0 : rtx vec = gen_rtx_CONST_VECTOR (V8HImode, gen_rtvec_v (8, ops));
19374 0 : emit_move_insn (target, vec);
19375 0 : return true;
19376 : }
19377 :
19378 511 : if (TARGET_SSE2
19379 511 : && nonzero_int_const_count (ops, 8) >= 2)
19380 : {
19381 : rtx csts[8];
19382 0 : for (i = 0; i < 8; i++)
19383 0 : if (CONST_INT_P (ops[i]))
19384 : {
19385 0 : csts[i] = ops[i];
19386 0 : vars[i] = const0_rtx;
19387 : }
19388 : else
19389 : {
19390 0 : csts[i] = const0_rtx;
19391 0 : vars[i] = ops[i];
19392 : }
19393 0 : rtx tmp1 = gen_reg_rtx (V8HImode);
19394 0 : if (!ix86_expand_vector_init_v8hi (tmp1, vars))
19395 0 : gcc_unreachable ();
19396 0 : rtx tmp2 = gen_reg_rtx (V8HImode);
19397 0 : rtx vec = gen_rtx_CONST_VECTOR (V8HImode, gen_rtvec_v (8, csts));
19398 0 : emit_move_insn (tmp2, vec);
19399 0 : emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V8HImode, tmp1, tmp2)));
19400 0 : return true;
19401 : }
19402 :
19403 511 : if (TARGET_SSE2)
19404 : {
19405 511 : ix86_expand_vector_init_insert (V8HImode, target, ops, 8);
19406 511 : return true;
19407 : }
19408 : return false;
19409 : }
19410 :
19411 : /* A subroutine of ix86_expand_vector_init for V16QImode. */
19412 :
19413 : static bool
19414 392 : ix86_expand_vector_init_v16qi (rtx target, rtx *ops)
19415 : {
19416 392 : rtx vars[16];
19417 392 : int i;
19418 :
19419 392 : bool all_zero_p = true;
19420 397 : for (i = 0; i < 16; i++)
19421 397 : if (ops[i] != const0_rtx)
19422 : {
19423 : all_zero_p = false;
19424 : break;
19425 : }
19426 392 : if (all_zero_p)
19427 : {
19428 0 : emit_move_insn (target, CONST0_RTX (V16QImode));
19429 0 : return true;
19430 : }
19431 :
19432 399 : bool all_const_p = true;
19433 399 : for (i = 0; i < 16; i++)
19434 399 : if (!CONST_INT_P (ops[i]))
19435 : {
19436 : all_const_p = false;
19437 : break;
19438 : }
19439 392 : if (all_const_p)
19440 : {
19441 0 : rtx vec = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, ops));
19442 0 : emit_move_insn (target, vec);
19443 0 : return true;
19444 : }
19445 :
19446 392 : if (TARGET_SSE4_1
19447 392 : && nonzero_int_const_count (ops, 16) >= 2)
19448 : {
19449 : rtx csts[16];
19450 0 : for (i = 0; i < 16; i++)
19451 0 : if (CONST_INT_P (ops[i]))
19452 : {
19453 0 : csts[i] = ops[i];
19454 0 : vars[i] = const0_rtx;
19455 : }
19456 : else
19457 : {
19458 0 : csts[i] = const0_rtx;
19459 0 : vars[i] = ops[i];
19460 : }
19461 0 : rtx tmp1 = gen_reg_rtx (V16QImode);
19462 0 : if (!ix86_expand_vector_init_v16qi (tmp1, vars))
19463 0 : gcc_unreachable ();
19464 0 : rtx tmp2 = gen_reg_rtx (V16QImode);
19465 0 : rtx vec = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, csts));
19466 0 : emit_move_insn (tmp2, vec);
19467 0 : emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V16QImode, tmp1, tmp2)));
19468 0 : return true;
19469 : }
19470 :
19471 392 : if (TARGET_SSE4_1)
19472 : {
19473 68 : ix86_expand_vector_init_insert (V16QImode, target, ops, 16);
19474 68 : return true;
19475 : }
19476 : return false;
19477 : }
19478 :
19479 : /* A subroutine of ix86_expand_vector_init for V4DImode. */
19480 :
19481 : static void
19482 144 : ix86_expand_vector_init_v4di (rtx target, rtx *ops)
19483 : {
19484 144 : rtx vars[4];
19485 144 : int perm[4];
19486 :
19487 144 : if (ops[0] == const0_rtx
19488 0 : && ops[1] == const0_rtx
19489 0 : && ops[2] == const0_rtx
19490 0 : && ops[3] == const0_rtx)
19491 0 : emit_move_insn (target, CONST0_RTX (V4DImode));
19492 144 : else if (ops[1] == const0_rtx
19493 8 : && ops[2] == const0_rtx
19494 0 : && ops[3] == const0_rtx)
19495 : {
19496 0 : rtx val = ops[0];
19497 0 : if (!REG_P (val) && !MEM_P (val))
19498 0 : val = force_reg (DImode, val);
19499 0 : emit_insn (gen_vec_setv4di_0 (target, CONST0_RTX (V4DImode), val));
19500 0 : }
19501 0 : else if ((TARGET_64BIT || MEM_P (ops[0]))
19502 144 : && rtx_equal_p (ops[0], ops[1])
19503 0 : && rtx_equal_p (ops[0], ops[2])
19504 144 : && rtx_equal_p (ops[0], ops[3]))
19505 : {
19506 0 : rtx val = ops[0];
19507 0 : if (!REG_P (val) && !MEM_P (val))
19508 0 : val = force_reg (DImode, val);
19509 0 : emit_insn (gen_vec_dupv4di (target, val));
19510 : }
19511 144 : else if (CONST_INT_P (ops[0])
19512 2 : && CONST_INT_P (ops[1])
19513 0 : && CONST_INT_P (ops[2])
19514 0 : && CONST_INT_P (ops[3]))
19515 : {
19516 0 : rtx vec = gen_rtx_CONST_VECTOR (V4DImode, gen_rtvec_v (4, ops));
19517 0 : emit_move_insn (target, vec);
19518 0 : }
19519 144 : else if (TARGET_AVX2
19520 144 : && onevar_perm_p (ops, 4, perm, vars))
19521 : {
19522 0 : rtx tmp = gen_reg_rtx (V4DImode);
19523 0 : vars[1] = const0_rtx;
19524 0 : vars[2] = const0_rtx;
19525 0 : vars[3] = const0_rtx;
19526 0 : ix86_expand_vector_init_v4di (tmp, vars);
19527 0 : emit_insn (gen_avx2_permv4di_1 (target, tmp,
19528 0 : GEN_INT (perm[0]),
19529 0 : GEN_INT (perm[1]),
19530 0 : GEN_INT (perm[2]),
19531 0 : GEN_INT (perm[3])));
19532 : }
19533 144 : else if (rtx_equal_p (ops[0], ops[2])
19534 144 : && rtx_equal_p (ops[1], ops[3]))
19535 : {
19536 2 : rtx tmp = gen_reg_rtx (V2DImode);
19537 2 : ix86_expand_vector_init_v2di (tmp, ops);
19538 2 : emit_insn (gen_avx_vec_concatv4di (target, tmp, tmp));
19539 : }
19540 142 : else if (nonzero_int_const_count (ops, 4) >= 2)
19541 : {
19542 : rtx csts[4];
19543 : int i;
19544 0 : for (i = 0; i < 4; i++)
19545 0 : if (CONST_INT_P (ops[i]))
19546 : {
19547 0 : csts[i] = ops[i];
19548 0 : vars[i] = const0_rtx;
19549 : }
19550 : else
19551 : {
19552 0 : csts[i] = const0_rtx;
19553 0 : vars[i] = ops[i];
19554 : }
19555 0 : rtx tmp1 = gen_reg_rtx (V4DImode);
19556 0 : ix86_expand_vector_init_v4di (tmp1, vars);
19557 0 : rtx tmp2 = gen_reg_rtx (V4DImode);
19558 0 : rtx vec = gen_rtx_CONST_VECTOR (V4DImode, gen_rtvec_v (4, csts));
19559 0 : emit_move_insn (tmp2, vec);
19560 0 : emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V4DImode, tmp1, tmp2)));
19561 : }
19562 : else
19563 : {
19564 142 : rtx tmp1 = gen_reg_rtx (V2DImode);
19565 142 : ix86_expand_vector_init_v2di (tmp1, ops);
19566 142 : rtx tmp2 = gen_reg_rtx (V2DImode);
19567 142 : ix86_expand_vector_init_v2di (tmp2, ops + 2);
19568 142 : emit_insn (gen_avx_vec_concatv4di (target, tmp1, tmp2));
19569 : }
19570 144 : }
19571 :
19572 : /* A subroutine of ix86_expand_vector_init for V4DFmode. */
19573 :
19574 : static void
19575 502 : ix86_expand_vector_init_v4df (rtx target, rtx *ops)
19576 : {
19577 502 : rtx vars[4];
19578 502 : int perm[4];
19579 :
19580 502 : if (ops[0] == CONST0_RTX (DFmode)
19581 0 : && ops[1] == CONST0_RTX (DFmode)
19582 0 : && ops[2] == CONST0_RTX (DFmode)
19583 0 : && ops[3] == CONST0_RTX (DFmode))
19584 0 : emit_move_insn (target, CONST0_RTX (V4DFmode));
19585 502 : else if (ops[1] == CONST0_RTX (DFmode)
19586 0 : && ops[2] == CONST0_RTX (DFmode)
19587 0 : && ops[3] == CONST0_RTX (DFmode))
19588 : {
19589 0 : rtx val = ops[0];
19590 0 : if (!REG_P (val) && !MEM_P (val))
19591 0 : val = force_reg (DFmode, val);
19592 0 : emit_insn (gen_vec_setv4df_0 (target, CONST0_RTX (V4DFmode), val));
19593 0 : }
19594 502 : else if (rtx_equal_p (ops[0], ops[1])
19595 50 : && rtx_equal_p (ops[0], ops[2])
19596 506 : && rtx_equal_p (ops[0], ops[3]))
19597 : {
19598 0 : rtx val = ops[0];
19599 0 : if (!REG_P (val) && !MEM_P (val))
19600 0 : val = force_reg (DFmode, val);
19601 0 : emit_insn (gen_vec_dupv4df (target, val));
19602 : }
19603 502 : else if (CONST_DOUBLE_P (ops[0])
19604 4 : && CONST_DOUBLE_P (ops[1])
19605 0 : && CONST_DOUBLE_P (ops[2])
19606 0 : && CONST_DOUBLE_P (ops[3]))
19607 : {
19608 0 : rtx vec = gen_rtx_CONST_VECTOR (V4DFmode, gen_rtvec_v (4, ops));
19609 0 : emit_move_insn (target, vec);
19610 0 : }
19611 502 : else if (TARGET_AVX2
19612 502 : && onevar_perm_p (ops, 4, perm, vars))
19613 : {
19614 0 : rtx tmp = gen_reg_rtx (V4DFmode);
19615 0 : vars[1] = CONST0_RTX (DFmode);
19616 0 : vars[2] = CONST0_RTX (DFmode);
19617 0 : vars[3] = CONST0_RTX (DFmode);
19618 0 : ix86_expand_vector_init_v4df (tmp, vars);
19619 0 : emit_insn (gen_avx2_permv4df_1 (target, tmp,
19620 0 : GEN_INT (perm[0]),
19621 0 : GEN_INT (perm[1]),
19622 0 : GEN_INT (perm[2]),
19623 0 : GEN_INT (perm[3])));
19624 : }
19625 502 : else if (rtx_equal_p (ops[0], ops[2])
19626 502 : && rtx_equal_p (ops[1], ops[3]))
19627 : {
19628 16 : rtx tmp = gen_reg_rtx (V2DFmode);
19629 16 : ix86_expand_vector_init_v2df (tmp, ops);
19630 16 : emit_insn (gen_avx_vec_concatv4df (target, tmp, tmp));
19631 : }
19632 486 : else if (nonzero_double_const_count (ops, 4) >= 2)
19633 : {
19634 : rtx csts[4];
19635 : int i;
19636 0 : for (i = 0; i < 4; i++)
19637 0 : if (CONST_DOUBLE_P (ops[i]))
19638 : {
19639 0 : csts[i] = ops[i];
19640 0 : vars[i] = CONST0_RTX (DFmode);
19641 : }
19642 : else
19643 : {
19644 0 : csts[i] = CONST0_RTX (DFmode);
19645 0 : vars[i] = ops[i];
19646 : }
19647 0 : rtx tmp1 = gen_reg_rtx (V4DFmode);
19648 0 : ix86_expand_vector_init_v4df (tmp1, vars);
19649 0 : rtx tmp2 = gen_reg_rtx (V4DFmode);
19650 0 : rtx vec = gen_rtx_CONST_VECTOR (V4DFmode, gen_rtvec_v (4, csts));
19651 0 : emit_move_insn (tmp2, vec);
19652 0 : emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V4DFmode, tmp1, tmp2)));
19653 : }
19654 : else
19655 : {
19656 486 : rtx tmp1 = gen_reg_rtx (V2DFmode);
19657 486 : ix86_expand_vector_init_v2df (tmp1, ops);
19658 486 : rtx tmp2 = gen_reg_rtx (V2DFmode);
19659 486 : ix86_expand_vector_init_v2df (tmp2, ops + 2);
19660 486 : emit_insn (gen_avx_vec_concatv4df (target, tmp1, tmp2));
19661 : }
19662 502 : }
19663 :
19664 : /* A subroutine of ix86_expand_vector_init for V8SImode. */
19665 :
19666 : static void
19667 134 : ix86_expand_vector_init_v8si (rtx target, rtx *ops)
19668 : {
19669 134 : rtx vars[8];
19670 :
19671 134 : if (ops[0] == const0_rtx
19672 7 : && ops[1] == const0_rtx
19673 7 : && ops[2] == const0_rtx
19674 7 : && ops[3] == const0_rtx
19675 6 : && ops[4] == const0_rtx
19676 3 : && ops[5] == const0_rtx
19677 0 : && ops[6] == const0_rtx
19678 0 : && ops[7] == const0_rtx)
19679 0 : emit_move_insn (target, CONST0_RTX (V8SImode));
19680 134 : else if (ops[1] == const0_rtx
19681 7 : && ops[2] == const0_rtx
19682 7 : && ops[3] == const0_rtx
19683 6 : && ops[4] == const0_rtx
19684 3 : && ops[5] == const0_rtx
19685 0 : && ops[6] == const0_rtx
19686 0 : && ops[7] == const0_rtx)
19687 : {
19688 0 : rtx val = ops[0];
19689 0 : if (!REG_P (val) && !MEM_P (val))
19690 0 : val = force_reg (SImode, val);
19691 0 : emit_insn (gen_vec_setv8si_0 (target, CONST0_RTX (V8SImode), val));
19692 0 : }
19693 134 : else if (rtx_equal_p (ops[0], ops[1])
19694 10 : && rtx_equal_p (ops[0], ops[2])
19695 8 : && rtx_equal_p (ops[0], ops[3])
19696 7 : && rtx_equal_p (ops[0], ops[4])
19697 3 : && rtx_equal_p (ops[0], ops[5])
19698 0 : && rtx_equal_p (ops[0], ops[6])
19699 134 : && rtx_equal_p (ops[0], ops[7]))
19700 : {
19701 0 : rtx val = ops[0];
19702 0 : if (!REG_P (val) && !MEM_P (val))
19703 0 : val = force_reg (SImode, val);
19704 0 : emit_insn (gen_vec_dupv8si (target, val));
19705 : }
19706 134 : else if (TARGET_AVX2
19707 121 : && rtx_equal_p (ops[0], ops[2])
19708 32 : && rtx_equal_p (ops[0], ops[4])
19709 28 : && rtx_equal_p (ops[0], ops[6])
19710 24 : && rtx_equal_p (ops[1], ops[3])
19711 24 : && rtx_equal_p (ops[1], ops[5])
19712 158 : && rtx_equal_p (ops[1], ops[7]))
19713 : {
19714 24 : rtx tmp_ops[4] = { ops[0], ops[1], const0_rtx, const0_rtx };
19715 24 : rtx tmp1 = gen_reg_rtx (V4SImode);
19716 24 : ix86_expand_vector_init_v4si (tmp1, tmp_ops);
19717 24 : tmp1 = gen_lowpart (V2DImode, tmp1);
19718 24 : rtx tmp2 = gen_reg_rtx (V4DImode);
19719 24 : emit_insn (gen_avx2_pbroadcastv4di (tmp2, tmp1));
19720 24 : emit_move_insn (target, gen_lowpart (V8SImode, tmp2));
19721 : }
19722 110 : else if (ops[4] == const0_rtx
19723 9 : && ops[5] == const0_rtx
19724 5 : && ops[6] == const0_rtx
19725 5 : && ops[7] == const0_rtx)
19726 : {
19727 5 : rtx tmp = gen_reg_rtx (V4SImode);
19728 5 : ix86_expand_vector_init_v4si (tmp, ops);
19729 5 : emit_insn (gen_avx_vec_concatv8si (target, tmp, CONST0_RTX (V4SImode)));
19730 5 : }
19731 105 : else if (CONST_INT_P (ops[0])
19732 9 : && CONST_INT_P (ops[1])
19733 7 : && CONST_INT_P (ops[2])
19734 7 : && CONST_INT_P (ops[3])
19735 7 : && CONST_INT_P (ops[4])
19736 4 : && CONST_INT_P (ops[5])
19737 0 : && CONST_INT_P (ops[6])
19738 0 : && CONST_INT_P (ops[7]))
19739 : {
19740 0 : rtx vec = gen_rtx_CONST_VECTOR (V8SImode, gen_rtvec_v (8, ops));
19741 0 : emit_move_insn (target, vec);
19742 0 : }
19743 105 : else if (rtx_equal_p (ops[0], ops[4])
19744 26 : && rtx_equal_p (ops[1], ops[5])
19745 22 : && rtx_equal_p (ops[2], ops[6])
19746 127 : && rtx_equal_p (ops[3], ops[7]))
19747 : {
19748 22 : rtx tmp = gen_reg_rtx (V4SImode);
19749 22 : ix86_expand_vector_init_v4si (tmp, ops);
19750 22 : emit_insn (gen_avx_vec_concatv8si (target, tmp, tmp));
19751 : }
19752 83 : else if (nonzero_int_const_count (ops, 8) >= 2)
19753 : {
19754 : rtx csts[8];
19755 : int i;
19756 9 : for (i = 0; i < 8; i++)
19757 8 : if (CONST_INT_P (ops[i]))
19758 : {
19759 6 : csts[i] = ops[i];
19760 6 : vars[i] = const0_rtx;
19761 : }
19762 : else
19763 : {
19764 2 : csts[i] = const0_rtx;
19765 2 : vars[i] = ops[i];
19766 : }
19767 1 : rtx tmp1 = gen_reg_rtx (V8SImode);
19768 1 : ix86_expand_vector_init_v8si (tmp1, vars);
19769 1 : rtx tmp2 = gen_reg_rtx (V8SImode);
19770 1 : rtx vec = gen_rtx_CONST_VECTOR (V8SImode, gen_rtvec_v (8, csts));
19771 1 : emit_move_insn (tmp2, vec);
19772 1 : emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V8SImode, tmp1, tmp2)));
19773 : }
19774 : else
19775 : {
19776 82 : rtx tmp1 = gen_reg_rtx (V4SImode);
19777 82 : ix86_expand_vector_init_v4si (tmp1, ops);
19778 82 : rtx tmp2 = gen_reg_rtx (V4SImode);
19779 82 : ix86_expand_vector_init_v4si (tmp2, ops + 4);
19780 82 : emit_insn (gen_avx_vec_concatv8si (target, tmp1, tmp2));
19781 : }
19782 134 : }
19783 :
19784 : /* A subroutine of ix86_expand_vector_init for V8SFmode. */
19785 :
19786 : static void
19787 249 : ix86_expand_vector_init_v8sf (rtx target, rtx *ops)
19788 : {
19789 249 : rtx vars[8];
19790 :
19791 249 : if (ops[0] == CONST0_RTX (SFmode)
19792 0 : && ops[1] == CONST0_RTX (SFmode)
19793 0 : && ops[2] == CONST0_RTX (SFmode)
19794 0 : && ops[3] == CONST0_RTX (SFmode)
19795 0 : && ops[4] == CONST0_RTX (SFmode)
19796 0 : && ops[5] == CONST0_RTX (SFmode)
19797 0 : && ops[6] == CONST0_RTX (SFmode)
19798 0 : && ops[7] == CONST0_RTX (SFmode))
19799 0 : emit_move_insn (target, CONST0_RTX (V8SFmode));
19800 249 : else if (ops[1] == CONST0_RTX (SFmode)
19801 24 : && ops[2] == CONST0_RTX (SFmode)
19802 0 : && ops[3] == CONST0_RTX (SFmode)
19803 0 : && ops[4] == CONST0_RTX (SFmode)
19804 0 : && ops[5] == CONST0_RTX (SFmode)
19805 0 : && ops[6] == CONST0_RTX (SFmode)
19806 0 : && ops[7] == CONST0_RTX (SFmode))
19807 : {
19808 0 : rtx val = ops[0];
19809 0 : if (!REG_P (val) && !MEM_P (val))
19810 0 : val = force_reg (SFmode, val);
19811 0 : emit_insn (gen_vec_setv8sf_0 (target, CONST0_RTX (V8SFmode), val));
19812 0 : }
19813 249 : else if (TARGET_AVX2
19814 175 : && rtx_equal_p (ops[0], ops[1])
19815 32 : && rtx_equal_p (ops[0], ops[2])
19816 0 : && rtx_equal_p (ops[0], ops[3])
19817 0 : && rtx_equal_p (ops[0], ops[4])
19818 0 : && rtx_equal_p (ops[0], ops[5])
19819 0 : && rtx_equal_p (ops[0], ops[6])
19820 249 : && rtx_equal_p (ops[0], ops[7]))
19821 : {
19822 0 : rtx val = ops[0];
19823 0 : if (!REG_P (val) && !MEM_P (val))
19824 0 : val = force_reg (SFmode, val);
19825 0 : emit_insn (gen_avx2_vec_dupv8sf_1 (target, val));
19826 : }
19827 249 : else if (TARGET_AVX2
19828 175 : && rtx_equal_p (ops[0], ops[2])
19829 32 : && rtx_equal_p (ops[0], ops[4])
19830 32 : && rtx_equal_p (ops[0], ops[6])
19831 32 : && rtx_equal_p (ops[1], ops[3])
19832 0 : && rtx_equal_p (ops[1], ops[5])
19833 249 : && rtx_equal_p (ops[1], ops[7]))
19834 : {
19835 0 : rtx tmp_ops[4] = { ops[0], ops[1], CONST0_RTX (SFmode),
19836 0 : CONST0_RTX (SFmode) };
19837 0 : rtx tmp1 = gen_reg_rtx (V4SFmode);
19838 0 : ix86_expand_vector_init_v4sf (tmp1, tmp_ops);
19839 0 : tmp1 = gen_lowpart (V2DImode, tmp1);
19840 0 : rtx tmp2 = gen_reg_rtx (V4DImode);
19841 0 : emit_insn (gen_avx2_pbroadcastv4di (tmp2, tmp1));
19842 0 : emit_move_insn (target, gen_lowpart (V8SFmode, tmp2));
19843 : }
19844 249 : else if (ops[4] == CONST0_RTX (SFmode)
19845 0 : && ops[5] == CONST0_RTX (SFmode)
19846 0 : && ops[6] == CONST0_RTX (SFmode)
19847 0 : && ops[7] == CONST0_RTX (SFmode))
19848 : {
19849 0 : rtx tmp = gen_reg_rtx (V4SFmode);
19850 0 : ix86_expand_vector_init_v4sf (tmp, ops);
19851 0 : emit_insn (gen_avx_vec_concatv8sf (target, tmp, CONST0_RTX (V4SFmode)));
19852 0 : }
19853 249 : else if (CONST_DOUBLE_P (ops[0])
19854 2 : && CONST_DOUBLE_P (ops[1])
19855 0 : && CONST_DOUBLE_P (ops[2])
19856 0 : && CONST_DOUBLE_P (ops[3])
19857 0 : && CONST_DOUBLE_P (ops[4])
19858 0 : && CONST_DOUBLE_P (ops[5])
19859 0 : && CONST_DOUBLE_P (ops[6])
19860 0 : && CONST_DOUBLE_P (ops[7]))
19861 : {
19862 0 : rtx vec = gen_rtx_CONST_VECTOR (V8SFmode, gen_rtvec_v (8, ops));
19863 0 : emit_move_insn (target, vec);
19864 0 : }
19865 249 : else if (rtx_equal_p (ops[0], ops[4])
19866 78 : && rtx_equal_p (ops[1], ops[5])
19867 22 : && rtx_equal_p (ops[2], ops[6])
19868 271 : && rtx_equal_p (ops[3], ops[7]))
19869 : {
19870 22 : rtx tmp = gen_reg_rtx (V4SFmode);
19871 22 : ix86_expand_vector_init_v4sf (tmp, ops);
19872 22 : emit_insn (gen_avx_vec_concatv8sf (target, tmp, tmp));
19873 : }
19874 227 : else if (nonzero_float_const_count (ops, 8) >= 2)
19875 : {
19876 : rtx csts[8];
19877 : int i;
19878 0 : for (i = 0; i < 8; i++)
19879 0 : if (CONST_DOUBLE_P (ops[i]))
19880 : {
19881 0 : csts[i] = ops[i];
19882 0 : vars[i] = CONST0_RTX (SFmode);
19883 : }
19884 : else
19885 : {
19886 0 : csts[i] = CONST0_RTX (SFmode);
19887 0 : vars[i] = ops[i];
19888 : }
19889 0 : rtx tmp1 = gen_reg_rtx (V8SFmode);
19890 0 : ix86_expand_vector_init_v8sf (tmp1, vars);
19891 0 : rtx tmp2 = gen_reg_rtx (V8SFmode);
19892 0 : rtx vec = gen_rtx_CONST_VECTOR (V8SFmode, gen_rtvec_v (8, csts));
19893 0 : emit_move_insn (tmp2, vec);
19894 0 : emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V8SFmode, tmp1, tmp2)));
19895 : }
19896 : else
19897 : {
19898 227 : rtx tmp1 = gen_reg_rtx (V4SFmode);
19899 227 : ix86_expand_vector_init_v4sf (tmp1, ops);
19900 227 : rtx tmp2 = gen_reg_rtx (V4SFmode);
19901 227 : ix86_expand_vector_init_v4sf (tmp2, ops + 4);
19902 227 : emit_insn (gen_avx_vec_concatv8sf (target, tmp1, tmp2));
19903 : }
19904 249 : }
19905 :
19906 :
19907 : /* A subroutine of ix86_expand_vector_init. Handle the most general case:
19908 : all values variable, and none identical. */
19909 :
19910 : static void
19911 101551 : ix86_expand_vector_init_general (bool mmx_ok, machine_mode mode,
19912 : rtx target, rtx vals)
19913 : {
19914 101551 : rtx ops[64], op0, op1, op2, op3, op4, op5;
19915 101551 : machine_mode half_mode = VOIDmode;
19916 101551 : machine_mode quarter_mode = VOIDmode;
19917 101551 : machine_mode int_inner_mode = VOIDmode;
19918 101551 : int n, i;
19919 :
19920 101551 : switch (mode)
19921 : {
19922 60401 : case E_V2DImode:
19923 60401 : ops[0] = XVECEXP (vals, 0, 0);
19924 60401 : ops[1] = XVECEXP (vals, 0, 1);
19925 60401 : ix86_expand_vector_init_v2di (target, ops);
19926 153408 : return;
19927 :
19928 2983 : case E_V2DFmode:
19929 2983 : ops[0] = XVECEXP (vals, 0, 0);
19930 2983 : ops[1] = XVECEXP (vals, 0, 1);
19931 2983 : ix86_expand_vector_init_v2df (target, ops);
19932 2983 : return;
19933 :
19934 : case E_V4SImode:
19935 25220 : for (i = 0; i < 4; i++)
19936 20176 : ops[i] = XVECEXP (vals, 0, i);
19937 5044 : ix86_expand_vector_init_v4si (target, ops);
19938 5044 : return;
19939 :
19940 : case E_V4SFmode:
19941 7825 : for (i = 0; i < 4; i++)
19942 6260 : ops[i] = XVECEXP (vals, 0, i);
19943 1565 : ix86_expand_vector_init_v4sf (target, ops);
19944 1565 : return;
19945 :
19946 : case E_V8HImode:
19947 4599 : for (i = 0; i < 8; i++)
19948 4088 : ops[i] = XVECEXP (vals, 0, i);
19949 511 : if (ix86_expand_vector_init_v8hi (target, ops))
19950 : return;
19951 : break;
19952 :
19953 : case E_V16QImode:
19954 6664 : for (i = 0; i < 16; i++)
19955 6272 : ops[i] = XVECEXP (vals, 0, i);
19956 392 : if (ix86_expand_vector_init_v16qi (target, ops))
19957 : return;
19958 : break;
19959 :
19960 : case E_V4DImode:
19961 720 : for (i = 0; i < 4; i++)
19962 576 : ops[i] = XVECEXP (vals, 0, i);
19963 144 : ix86_expand_vector_init_v4di (target,ops);
19964 144 : return;
19965 :
19966 : case E_V4DFmode:
19967 2510 : for (i = 0; i < 4; i++)
19968 2008 : ops[i] = XVECEXP (vals, 0, i);
19969 502 : ix86_expand_vector_init_v4df (target,ops);
19970 502 : return;
19971 :
19972 : case E_V8SImode:
19973 1197 : for (i = 0; i < 8; i++)
19974 1064 : ops[i] = XVECEXP (vals, 0, i);
19975 133 : ix86_expand_vector_init_v8si (target,ops);
19976 133 : return;
19977 :
19978 : case E_V8SFmode:
19979 2241 : for (i = 0; i < 8; i++)
19980 1992 : ops[i] = XVECEXP (vals, 0, i);
19981 249 : ix86_expand_vector_init_v8sf (target,ops);
19982 249 : return;
19983 :
19984 19521 : case E_V2SFmode:
19985 19521 : case E_V2SImode:
19986 19521 : if (!mmx_ok && !TARGET_SSE)
19987 : break;
19988 : /* FALLTHRU */
19989 :
19990 19694 : case E_V16SImode:
19991 19694 : case E_V16SFmode:
19992 19694 : case E_V8DFmode:
19993 19694 : case E_V8DImode:
19994 19694 : n = GET_MODE_NUNITS (mode);
19995 60688 : for (i = 0; i < n; i++)
19996 40994 : ops[i] = XVECEXP (vals, 0, i);
19997 19694 : ix86_expand_vector_init_concat (mode, target, ops, n);
19998 19694 : return;
19999 :
20000 : case E_V2TImode:
20001 135 : for (i = 0; i < 2; i++)
20002 90 : ops[i] = gen_lowpart (V2DImode, XVECEXP (vals, 0, i));
20003 45 : op0 = gen_reg_rtx (V4DImode);
20004 45 : ix86_expand_vector_init_concat (V4DImode, op0, ops, 2);
20005 45 : emit_move_insn (target, gen_lowpart (GET_MODE (target), op0));
20006 45 : return;
20007 :
20008 : case E_V4TImode:
20009 195 : for (i = 0; i < 4; i++)
20010 156 : ops[i] = gen_lowpart (V2DImode, XVECEXP (vals, 0, i));
20011 39 : ops[4] = gen_reg_rtx (V4DImode);
20012 39 : ix86_expand_vector_init_concat (V4DImode, ops[4], ops, 2);
20013 39 : ops[5] = gen_reg_rtx (V4DImode);
20014 39 : ix86_expand_vector_init_concat (V4DImode, ops[5], ops + 2, 2);
20015 39 : op0 = gen_reg_rtx (V8DImode);
20016 39 : ix86_expand_vector_init_concat (V8DImode, op0, ops + 4, 2);
20017 39 : emit_move_insn (target, gen_lowpart (GET_MODE (target), op0));
20018 39 : return;
20019 :
20020 69 : case E_V32QImode:
20021 69 : half_mode = V16QImode;
20022 69 : goto half;
20023 :
20024 65 : case E_V16HImode:
20025 65 : half_mode = V8HImode;
20026 65 : goto half;
20027 :
20028 237 : case E_V16HFmode:
20029 237 : half_mode = V8HFmode;
20030 237 : goto half;
20031 :
20032 95 : case E_V16BFmode:
20033 95 : half_mode = V8BFmode;
20034 95 : goto half;
20035 :
20036 466 : half:
20037 466 : n = GET_MODE_NUNITS (mode);
20038 9026 : for (i = 0; i < n; i++)
20039 8560 : ops[i] = XVECEXP (vals, 0, i);
20040 466 : op0 = gen_reg_rtx (half_mode);
20041 466 : op1 = gen_reg_rtx (half_mode);
20042 466 : ix86_expand_vector_init_interleave (half_mode, op0, ops,
20043 : n >> 2);
20044 466 : ix86_expand_vector_init_interleave (half_mode, op1,
20045 466 : &ops [n >> 1], n >> 2);
20046 466 : emit_insn (gen_rtx_SET (target, gen_rtx_VEC_CONCAT (mode, op0, op1)));
20047 466 : return;
20048 :
20049 56 : case E_V64QImode:
20050 56 : quarter_mode = V16QImode;
20051 56 : half_mode = V32QImode;
20052 56 : goto quarter;
20053 :
20054 71 : case E_V32HImode:
20055 71 : quarter_mode = V8HImode;
20056 71 : half_mode = V16HImode;
20057 71 : goto quarter;
20058 :
20059 287 : case E_V32HFmode:
20060 287 : quarter_mode = V8HFmode;
20061 287 : half_mode = V16HFmode;
20062 287 : goto quarter;
20063 :
20064 51 : case E_V32BFmode:
20065 51 : quarter_mode = V8BFmode;
20066 51 : half_mode = V16BFmode;
20067 51 : goto quarter;
20068 :
20069 465 : quarter:
20070 465 : n = GET_MODE_NUNITS (mode);
20071 17137 : for (i = 0; i < n; i++)
20072 16672 : ops[i] = XVECEXP (vals, 0, i);
20073 465 : op0 = gen_reg_rtx (quarter_mode);
20074 465 : op1 = gen_reg_rtx (quarter_mode);
20075 465 : op2 = gen_reg_rtx (quarter_mode);
20076 465 : op3 = gen_reg_rtx (quarter_mode);
20077 465 : op4 = gen_reg_rtx (half_mode);
20078 465 : op5 = gen_reg_rtx (half_mode);
20079 465 : ix86_expand_vector_init_interleave (quarter_mode, op0, ops,
20080 : n >> 3);
20081 465 : ix86_expand_vector_init_interleave (quarter_mode, op1,
20082 465 : &ops [n >> 2], n >> 3);
20083 465 : ix86_expand_vector_init_interleave (quarter_mode, op2,
20084 465 : &ops [n >> 1], n >> 3);
20085 465 : ix86_expand_vector_init_interleave (quarter_mode, op3,
20086 465 : &ops [(n >> 1) | (n >> 2)], n >> 3);
20087 465 : emit_insn (gen_rtx_SET (op4, gen_rtx_VEC_CONCAT (half_mode, op0, op1)));
20088 465 : emit_insn (gen_rtx_SET (op5, gen_rtx_VEC_CONCAT (half_mode, op2, op3)));
20089 465 : emit_insn (gen_rtx_SET (target, gen_rtx_VEC_CONCAT (mode, op4, op5)));
20090 465 : return;
20091 :
20092 698 : case E_V8HFmode:
20093 698 : case E_V8BFmode:
20094 698 : n = GET_MODE_NUNITS (mode);
20095 6282 : for (i = 0; i < n; i++)
20096 5584 : ops[i] = XVECEXP (vals, 0, i);
20097 698 : ix86_expand_vector_init_interleave (mode, target, ops, n >> 1);
20098 698 : return;
20099 :
20100 : case E_V4HFmode:
20101 : case E_V4BFmode:
20102 : case E_V2HFmode:
20103 : case E_V2BFmode:
20104 8544 : int_inner_mode = HImode;
20105 : break;
20106 :
20107 : case E_V4HImode:
20108 : case E_V8QImode:
20109 :
20110 : case E_V2HImode:
20111 : case E_V4QImode:
20112 : break;
20113 :
20114 0 : default:
20115 0 : gcc_unreachable ();
20116 : }
20117 :
20118 8544 : {
20119 8544 : int i, j, n_elts, n_words, n_elt_per_word;
20120 8544 : machine_mode tmp_mode, inner_mode;
20121 8544 : rtx words[4], shift;
20122 :
20123 17162 : tmp_mode = (GET_MODE_SIZE (mode) < UNITS_PER_WORD) ? SImode : word_mode;
20124 :
20125 8544 : inner_mode = GET_MODE_INNER (mode);
20126 8544 : n_elts = GET_MODE_NUNITS (mode);
20127 17088 : n_words = GET_MODE_SIZE (mode) / GET_MODE_SIZE (tmp_mode);
20128 8544 : n_elt_per_word = n_elts / n_words;
20129 8544 : shift = GEN_INT (GET_MODE_BITSIZE (inner_mode));
20130 :
20131 25964 : for (i = 0; i < n_words; ++i)
20132 : {
20133 : rtx word = NULL_RTX;
20134 :
20135 46818 : for (j = 0; j < n_elt_per_word; ++j)
20136 : {
20137 37942 : rtx elt = XVECEXP (vals, 0, (i+1)*n_elt_per_word - j - 1);
20138 37942 : if (int_inner_mode != E_VOIDmode)
20139 : {
20140 310 : gcc_assert (TARGET_SSE2 && int_inner_mode == HImode);
20141 310 : rtx tmp = gen_reg_rtx (int_inner_mode);
20142 310 : elt = lowpart_subreg (int_inner_mode,
20143 : force_reg (inner_mode, elt),
20144 : inner_mode);
20145 310 : emit_move_insn (tmp, elt);
20146 310 : elt = tmp;
20147 : }
20148 37942 : elt = convert_modes (tmp_mode, inner_mode, elt, true);
20149 :
20150 37942 : if (j == 0)
20151 : word = elt;
20152 : else
20153 : {
20154 29066 : word = expand_simple_binop (tmp_mode, ASHIFT, word, shift,
20155 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
20156 29066 : word = expand_simple_binop (tmp_mode, IOR, word, elt,
20157 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
20158 : }
20159 : }
20160 :
20161 8876 : words[i] = word;
20162 : }
20163 :
20164 8544 : if (n_words == 1)
20165 8212 : emit_move_insn (target, gen_lowpart (mode, words[0]));
20166 332 : else if (n_words == 2)
20167 : {
20168 332 : gcc_assert (tmp_mode == DImode || tmp_mode == SImode);
20169 332 : machine_mode concat_mode = tmp_mode == DImode ? V2DImode : V2SImode;
20170 332 : rtx tmp = gen_reg_rtx (concat_mode);
20171 332 : vals = gen_rtx_PARALLEL (concat_mode, gen_rtvec_v (2, words));
20172 332 : ix86_expand_vector_init_general (mmx_ok, concat_mode, tmp, vals);
20173 332 : emit_move_insn (target, gen_lowpart (mode, tmp));
20174 : }
20175 0 : else if (n_words == 4)
20176 : {
20177 0 : rtx tmp = gen_reg_rtx (V4SImode);
20178 0 : gcc_assert (tmp_mode == SImode);
20179 0 : vals = gen_rtx_PARALLEL (V4SImode, gen_rtvec_v (4, words));
20180 0 : ix86_expand_vector_init_general (false, V4SImode, tmp, vals);
20181 0 : emit_move_insn (target, gen_lowpart (mode, tmp));
20182 : }
20183 : else
20184 0 : gcc_unreachable ();
20185 : }
20186 : }
20187 :
20188 : /* Initialize vector TARGET via VALS. Suppress the use of MMX
20189 : instructions unless MMX_OK is true. */
20190 :
20191 : void
20192 119116 : ix86_expand_vector_init (bool mmx_ok, rtx target, rtx vals)
20193 : {
20194 119116 : machine_mode mode = GET_MODE (target);
20195 119116 : machine_mode inner_mode = GET_MODE_INNER (mode);
20196 119116 : int n_elts = GET_MODE_NUNITS (mode);
20197 119116 : int n_var = 0, one_var = -1;
20198 119116 : bool all_same = true, all_const_zero = true;
20199 119116 : int i;
20200 119116 : rtx x;
20201 :
20202 : /* Handle first initialization from vector elts. */
20203 119116 : if (n_elts != XVECLEN (vals, 0))
20204 : {
20205 1361 : rtx subtarget = target;
20206 1361 : x = XVECEXP (vals, 0, 0);
20207 2722 : gcc_assert (GET_MODE_INNER (GET_MODE (x)) == inner_mode);
20208 2722 : if (GET_MODE_NUNITS (GET_MODE (x)) * 2 == n_elts)
20209 : {
20210 1361 : rtx ops[2] = { XVECEXP (vals, 0, 0), XVECEXP (vals, 0, 1) };
20211 1361 : if (inner_mode == QImode
20212 1361 : || inner_mode == HImode
20213 1361 : || inner_mode == TImode
20214 : || inner_mode == HFmode
20215 : || inner_mode == BFmode)
20216 : {
20217 168 : unsigned int n_bits = n_elts * GET_MODE_SIZE (inner_mode);
20218 168 : scalar_mode elt_mode = inner_mode == TImode ? DImode : SImode;
20219 168 : n_bits /= GET_MODE_SIZE (elt_mode);
20220 168 : mode = mode_for_vector (elt_mode, n_bits).require ();
20221 168 : inner_mode = mode_for_vector (elt_mode, n_bits / 2).require ();
20222 168 : ops[0] = gen_lowpart (inner_mode, ops[0]);
20223 168 : ops[1] = gen_lowpart (inner_mode, ops[1]);
20224 168 : subtarget = gen_reg_rtx (mode);
20225 : }
20226 1361 : ix86_expand_vector_init_concat (mode, subtarget, ops, 2);
20227 1361 : if (subtarget != target)
20228 168 : emit_move_insn (target, gen_lowpart (GET_MODE (target), subtarget));
20229 1361 : return;
20230 : }
20231 0 : gcc_unreachable ();
20232 : }
20233 :
20234 441533 : for (i = 0; i < n_elts; ++i)
20235 : {
20236 323778 : x = XVECEXP (vals, 0, i);
20237 626835 : if (!(CONST_SCALAR_INT_P (x)
20238 306933 : || CONST_DOUBLE_P (x)
20239 : || CONST_FIXED_P (x)))
20240 303057 : n_var++, one_var = i;
20241 20721 : else if (x != CONST0_RTX (inner_mode))
20242 3148 : all_const_zero = false;
20243 323778 : if (i > 0 && !rtx_equal_p (x, XVECEXP (vals, 0, 0)))
20244 : all_same = false;
20245 : }
20246 :
20247 : /* Handle the zero vector as special case. */
20248 117755 : if (n_var == 0 && all_const_zero)
20249 : {
20250 3 : emit_move_insn (target, CONST0_RTX (mode));
20251 3 : return;
20252 : }
20253 :
20254 : /* If all values are identical, broadcast the value. */
20255 117752 : if (all_same
20256 124866 : && ix86_expand_vector_init_duplicate (mmx_ok, mode, target,
20257 7114 : XVECEXP (vals, 0, 0)))
20258 : return;
20259 :
20260 : /* Constants are best loaded from the constant pool. */
20261 111378 : if (n_var == 0)
20262 : {
20263 12 : emit_move_insn (target, gen_rtx_CONST_VECTOR (mode, XVEC (vals, 0)));
20264 12 : return;
20265 : }
20266 :
20267 : /* Values where only one field is non-constant are best loaded from
20268 : the pool and overwritten via move later. */
20269 111366 : if (n_var == 1)
20270 : {
20271 11703 : if (all_const_zero
20272 22279 : && ix86_expand_vector_init_one_nonzero (mmx_ok, mode, target,
20273 10576 : XVECEXP (vals, 0, one_var),
20274 : one_var))
20275 : return;
20276 :
20277 1788 : if (ix86_expand_vector_init_one_var (mmx_ok, mode, target, vals, one_var))
20278 : return;
20279 : }
20280 :
20281 101219 : ix86_expand_vector_init_general (mmx_ok, mode, target, vals);
20282 : }
20283 :
20284 : /* Implemented as
20285 : V setg (V v, int idx, T val)
20286 : {
20287 : V idxv = (V){idx, idx, idx, idx, idx, idx, idx, idx};
20288 : V valv = (V){val, val, val, val, val, val, val, val};
20289 : V mask = ((V){0, 1, 2, 3, 4, 5, 6, 7} == idxv);
20290 : v = (v & ~mask) | (valv & mask);
20291 : return v;
20292 : }. */
20293 : void
20294 129 : ix86_expand_vector_set_var (rtx target, rtx val, rtx idx)
20295 : {
20296 129 : rtx vec[64];
20297 129 : machine_mode mode = GET_MODE (target);
20298 129 : machine_mode cmp_mode = mode;
20299 129 : int n_elts = GET_MODE_NUNITS (mode);
20300 129 : rtx valv,idxv,constv,idx_tmp;
20301 129 : bool ok = false;
20302 :
20303 258 : val = force_reg (GET_MODE_INNER (mode), val);
20304 :
20305 : /* 512-bits vector byte/word broadcast and comparison only available
20306 : under TARGET_AVX512BW, break 512-bits vector into two 256-bits vector
20307 : when without TARGET_AVX512BW. */
20308 129 : if ((mode == V32HImode || mode == V32HFmode || mode == V32BFmode
20309 123 : || mode == V64QImode)
20310 10 : && !TARGET_AVX512BW)
20311 : {
20312 3 : gcc_assert (TARGET_AVX512F);
20313 3 : rtx vhi, vlo, idx_hi;
20314 3 : machine_mode half_mode;
20315 3 : rtx (*extract_hi)(rtx, rtx);
20316 3 : rtx (*extract_lo)(rtx, rtx);
20317 :
20318 3 : if (mode == V32HImode)
20319 : {
20320 : half_mode = V16HImode;
20321 : extract_hi = gen_vec_extract_hi_v32hi;
20322 : extract_lo = gen_vec_extract_lo_v32hi;
20323 : }
20324 : else if (mode == V32HFmode)
20325 : {
20326 : half_mode = V16HFmode;
20327 : extract_hi = gen_vec_extract_hi_v32hf;
20328 : extract_lo = gen_vec_extract_lo_v32hf;
20329 : }
20330 : else if (mode == V32BFmode)
20331 : {
20332 : half_mode = V16BFmode;
20333 : extract_hi = gen_vec_extract_hi_v32bf;
20334 : extract_lo = gen_vec_extract_lo_v32bf;
20335 : }
20336 : else
20337 : {
20338 3 : half_mode = V32QImode;
20339 3 : extract_hi = gen_vec_extract_hi_v64qi;
20340 3 : extract_lo = gen_vec_extract_lo_v64qi;
20341 : }
20342 :
20343 3 : vhi = gen_reg_rtx (half_mode);
20344 3 : vlo = gen_reg_rtx (half_mode);
20345 3 : idx_hi = gen_reg_rtx (GET_MODE (idx));
20346 3 : emit_insn (extract_hi (vhi, target));
20347 3 : emit_insn (extract_lo (vlo, target));
20348 3 : vec[0] = idx_hi;
20349 3 : vec[1] = idx;
20350 3 : vec[2] = GEN_INT (n_elts/2);
20351 3 : ix86_expand_binary_operator (MINUS, GET_MODE (idx), vec);
20352 3 : ix86_expand_vector_set_var (vhi, val, idx_hi);
20353 3 : ix86_expand_vector_set_var (vlo, val, idx);
20354 3 : emit_insn (gen_rtx_SET (target, gen_rtx_VEC_CONCAT (mode, vlo, vhi)));
20355 3 : return;
20356 : }
20357 :
20358 504 : if (FLOAT_MODE_P (GET_MODE_INNER (mode)))
20359 : {
20360 42 : switch (mode)
20361 : {
20362 : case E_V2DFmode:
20363 : cmp_mode = V2DImode;
20364 : break;
20365 6 : case E_V4DFmode:
20366 6 : cmp_mode = V4DImode;
20367 6 : break;
20368 4 : case E_V8DFmode:
20369 4 : cmp_mode = V8DImode;
20370 4 : break;
20371 2 : case E_V2SFmode:
20372 2 : cmp_mode = V2SImode;
20373 2 : break;
20374 6 : case E_V4SFmode:
20375 6 : cmp_mode = V4SImode;
20376 6 : break;
20377 6 : case E_V8SFmode:
20378 6 : cmp_mode = V8SImode;
20379 6 : break;
20380 5 : case E_V16SFmode:
20381 5 : cmp_mode = V16SImode;
20382 5 : break;
20383 1 : case E_V2HFmode:
20384 1 : case E_V2BFmode:
20385 1 : cmp_mode = V2HImode;
20386 1 : break;
20387 1 : case E_V4HFmode:
20388 1 : case E_V4BFmode:
20389 1 : cmp_mode = V4HImode;
20390 1 : break;
20391 : case E_V8HFmode:
20392 2 : cmp_mode = V8HImode;
20393 : break;
20394 : case E_V16HFmode:
20395 2 : cmp_mode = V16HImode;
20396 : break;
20397 : case E_V32HFmode:
20398 1 : cmp_mode = V32HImode;
20399 : break;
20400 : case E_V8BFmode:
20401 2 : cmp_mode = V8HImode;
20402 : break;
20403 : case E_V16BFmode:
20404 2 : cmp_mode = V16HImode;
20405 : break;
20406 : case E_V32BFmode:
20407 1 : cmp_mode = V32HImode;
20408 : break;
20409 0 : default:
20410 0 : gcc_unreachable ();
20411 : }
20412 : }
20413 :
20414 1604 : for (int i = 0; i != n_elts; i++)
20415 1478 : vec[i] = GEN_INT (i);
20416 126 : constv = gen_rtx_CONST_VECTOR (cmp_mode, gen_rtvec_v (n_elts, vec));
20417 126 : valv = gen_reg_rtx (mode);
20418 126 : idxv = gen_reg_rtx (cmp_mode);
20419 252 : idx_tmp = convert_to_mode (GET_MODE_INNER (cmp_mode), idx, 1);
20420 :
20421 126 : ok = ix86_expand_vector_init_duplicate (TARGET_MMX_WITH_SSE,
20422 : mode, valv, val);
20423 126 : gcc_assert (ok);
20424 126 : ok = ix86_expand_vector_init_duplicate (TARGET_MMX_WITH_SSE,
20425 : cmp_mode, idxv, idx_tmp);
20426 126 : gcc_assert (ok);
20427 126 : vec[0] = target;
20428 126 : vec[1] = valv;
20429 126 : vec[2] = target;
20430 126 : vec[3] = gen_rtx_EQ (mode, idxv, constv);
20431 126 : vec[4] = idxv;
20432 126 : vec[5] = constv;
20433 126 : ok = ix86_expand_int_vcond (vec);
20434 126 : gcc_assert (ok);
20435 : }
20436 :
20437 : void
20438 6444 : ix86_expand_vector_set (bool mmx_ok, rtx target, rtx val, int elt)
20439 : {
20440 6444 : machine_mode mode = GET_MODE (target);
20441 6444 : machine_mode inner_mode = GET_MODE_INNER (mode);
20442 6444 : machine_mode half_mode;
20443 6444 : bool use_vec_merge = false;
20444 6444 : bool blendm_const = false;
20445 6444 : rtx tmp;
20446 6444 : static rtx (*gen_extract[8][2]) (rtx, rtx)
20447 : = {
20448 : { gen_vec_extract_lo_v32qi, gen_vec_extract_hi_v32qi },
20449 : { gen_vec_extract_lo_v16hi, gen_vec_extract_hi_v16hi },
20450 : { gen_vec_extract_lo_v8si, gen_vec_extract_hi_v8si },
20451 : { gen_vec_extract_lo_v4di, gen_vec_extract_hi_v4di },
20452 : { gen_vec_extract_lo_v8sf, gen_vec_extract_hi_v8sf },
20453 : { gen_vec_extract_lo_v4df, gen_vec_extract_hi_v4df },
20454 : { gen_vec_extract_lo_v16hf, gen_vec_extract_hi_v16hf },
20455 : { gen_vec_extract_lo_v16bf, gen_vec_extract_hi_v16bf }
20456 : };
20457 6444 : static rtx (*gen_insert[8][2]) (rtx, rtx, rtx)
20458 : = {
20459 : { gen_vec_set_lo_v32qi, gen_vec_set_hi_v32qi },
20460 : { gen_vec_set_lo_v16hi, gen_vec_set_hi_v16hi },
20461 : { gen_vec_set_lo_v8si, gen_vec_set_hi_v8si },
20462 : { gen_vec_set_lo_v4di, gen_vec_set_hi_v4di },
20463 : { gen_vec_set_lo_v8sf, gen_vec_set_hi_v8sf },
20464 : { gen_vec_set_lo_v4df, gen_vec_set_hi_v4df },
20465 : { gen_vec_set_lo_v16hf, gen_vec_set_hi_v16hf },
20466 : { gen_vec_set_lo_v16bf, gen_vec_set_hi_v16bf },
20467 : };
20468 6444 : int i, j, n;
20469 6444 : machine_mode mmode = VOIDmode;
20470 6444 : rtx (*gen_blendm) (rtx, rtx, rtx, rtx);
20471 :
20472 12888 : if (!IN_RANGE (elt, 0, GET_MODE_NUNITS (mode)))
20473 : {
20474 0 : emit_move_insn (target, target);
20475 0 : return;
20476 : }
20477 6444 : if (TARGET_SSE4_1 && mode == V4SImode && val == const0_rtx)
20478 : {
20479 27 : emit_insn (gen_sse4_1_insertps_v4si_zero (target, target,
20480 : CONST0_RTX (V4SImode),
20481 27 : GEN_INT ((1 << elt) ^ 15)));
20482 27 : return;
20483 : }
20484 6417 : if (TARGET_SSE4_1 && mode == V4SFmode && val == CONST0_RTX (SFmode))
20485 : {
20486 25 : emit_insn (gen_sse4_1_insertps_v4sf_zero (target, target,
20487 : CONST0_RTX (V4SFmode),
20488 25 : GEN_INT ((1 << elt) ^ 15)));
20489 25 : return;
20490 : }
20491 :
20492 12784 : val = force_reg (GET_MODE_INNER (mode), val);
20493 :
20494 6392 : switch (mode)
20495 : {
20496 200 : case E_V2SImode:
20497 200 : use_vec_merge = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
20498 : if (use_vec_merge)
20499 : break;
20500 : /* FALLTHRU */
20501 :
20502 183 : case E_V2SFmode:
20503 183 : if (mmx_ok)
20504 : {
20505 362 : tmp = gen_reg_rtx (GET_MODE_INNER (mode));
20506 181 : ix86_expand_vector_extract (true, tmp, target, 1 - elt);
20507 181 : if (elt == 0)
20508 9 : tmp = gen_rtx_VEC_CONCAT (mode, val, tmp);
20509 : else
20510 172 : tmp = gen_rtx_VEC_CONCAT (mode, tmp, val);
20511 181 : emit_insn (gen_rtx_SET (target, tmp));
20512 181 : return;
20513 : }
20514 : break;
20515 :
20516 358 : case E_V2DImode:
20517 358 : use_vec_merge = TARGET_SSE4_1 && TARGET_64BIT;
20518 269 : if (use_vec_merge)
20519 : break;
20520 :
20521 538 : tmp = gen_reg_rtx (GET_MODE_INNER (mode));
20522 269 : ix86_expand_vector_extract (false, tmp, target, 1 - elt);
20523 269 : if (elt == 0)
20524 234 : tmp = gen_rtx_VEC_CONCAT (mode, val, tmp);
20525 : else
20526 35 : tmp = gen_rtx_VEC_CONCAT (mode, tmp, val);
20527 269 : emit_insn (gen_rtx_SET (target, tmp));
20528 269 : return;
20529 :
20530 127 : case E_V2DFmode:
20531 : /* NB: For ELT == 0, use standard scalar operation patterns which
20532 : preserve the rest of the vector for combiner:
20533 :
20534 : (vec_merge:V2DF
20535 : (vec_duplicate:V2DF (reg:DF))
20536 : (reg:V2DF)
20537 : (const_int 1))
20538 : */
20539 127 : if (elt == 0)
20540 64 : goto do_vec_merge;
20541 :
20542 63 : {
20543 63 : rtx op0, op1;
20544 :
20545 : /* For the two element vectors, we implement a VEC_CONCAT with
20546 : the extraction of the other element. */
20547 :
20548 63 : tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (1 - elt)));
20549 63 : tmp = gen_rtx_VEC_SELECT (inner_mode, target, tmp);
20550 :
20551 63 : if (elt == 0)
20552 : op0 = val, op1 = tmp;
20553 : else
20554 63 : op0 = tmp, op1 = val;
20555 :
20556 63 : tmp = gen_rtx_VEC_CONCAT (mode, op0, op1);
20557 63 : emit_insn (gen_rtx_SET (target, tmp));
20558 : }
20559 63 : return;
20560 :
20561 76 : case E_V4SFmode:
20562 76 : use_vec_merge = TARGET_SSE4_1;
20563 76 : if (use_vec_merge)
20564 : break;
20565 :
20566 62 : switch (elt)
20567 : {
20568 : case 0:
20569 : use_vec_merge = true;
20570 : break;
20571 :
20572 1 : case 1:
20573 : /* tmp = target = A B C D */
20574 1 : tmp = copy_to_reg (target);
20575 : /* target = A A B B */
20576 1 : emit_insn (gen_vec_interleave_lowv4sf (target, target, target));
20577 : /* target = X A B B */
20578 1 : ix86_expand_vector_set (false, target, val, 0);
20579 : /* target = A X C D */
20580 1 : emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
20581 : const1_rtx, const0_rtx,
20582 : GEN_INT (2+4), GEN_INT (3+4)));
20583 1 : return;
20584 :
20585 0 : case 2:
20586 : /* tmp = target = A B C D */
20587 0 : tmp = copy_to_reg (target);
20588 : /* tmp = X B C D */
20589 0 : ix86_expand_vector_set (false, tmp, val, 0);
20590 : /* target = A B X D */
20591 0 : emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
20592 : const0_rtx, const1_rtx,
20593 : GEN_INT (0+4), GEN_INT (3+4)));
20594 0 : return;
20595 :
20596 4 : case 3:
20597 : /* tmp = target = A B C D */
20598 4 : tmp = copy_to_reg (target);
20599 : /* tmp = X B C D */
20600 4 : ix86_expand_vector_set (false, tmp, val, 0);
20601 : /* target = A B X D */
20602 4 : emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
20603 : const0_rtx, const1_rtx,
20604 : GEN_INT (2+4), GEN_INT (0+4)));
20605 4 : return;
20606 :
20607 0 : default:
20608 0 : gcc_unreachable ();
20609 : }
20610 : break;
20611 :
20612 402 : case E_V4SImode:
20613 402 : use_vec_merge = TARGET_SSE4_1;
20614 402 : if (use_vec_merge)
20615 : break;
20616 :
20617 : /* Element 0 handled by vec_merge below. */
20618 283 : if (elt == 0)
20619 : {
20620 : use_vec_merge = true;
20621 : break;
20622 : }
20623 :
20624 92 : if (TARGET_SSE2)
20625 : {
20626 : /* With SSE2, use integer shuffles to swap element 0 and ELT,
20627 : store into element 0, then shuffle them back. */
20628 :
20629 92 : rtx order[4];
20630 :
20631 92 : order[0] = GEN_INT (elt);
20632 92 : order[1] = const1_rtx;
20633 92 : order[2] = const2_rtx;
20634 92 : order[3] = GEN_INT (3);
20635 92 : order[elt] = const0_rtx;
20636 :
20637 92 : emit_insn (gen_sse2_pshufd_1 (target, target, order[0],
20638 : order[1], order[2], order[3]));
20639 :
20640 92 : ix86_expand_vector_set (false, target, val, 0);
20641 :
20642 92 : emit_insn (gen_sse2_pshufd_1 (target, target, order[0],
20643 : order[1], order[2], order[3]));
20644 : }
20645 : else
20646 : {
20647 : /* For SSE1, we have to reuse the V4SF code. */
20648 0 : rtx t = gen_reg_rtx (V4SFmode);
20649 0 : emit_move_insn (t, gen_lowpart (V4SFmode, target));
20650 0 : ix86_expand_vector_set (false, t, gen_lowpart (SFmode, val), elt);
20651 0 : emit_move_insn (target, gen_lowpart (mode, t));
20652 : }
20653 : return;
20654 :
20655 2028 : case E_V8HImode:
20656 2028 : case E_V8HFmode:
20657 2028 : case E_V8BFmode:
20658 2028 : case E_V2HImode:
20659 2028 : case E_V2HFmode:
20660 2028 : case E_V2BFmode:
20661 2028 : use_vec_merge = TARGET_SSE2;
20662 2028 : break;
20663 51 : case E_V4HImode:
20664 51 : case E_V4HFmode:
20665 51 : case E_V4BFmode:
20666 51 : use_vec_merge = mmx_ok && (TARGET_SSE || TARGET_3DNOW_A);
20667 : break;
20668 :
20669 3003 : case E_V16QImode:
20670 3003 : case E_V4QImode:
20671 3003 : use_vec_merge = TARGET_SSE4_1;
20672 3003 : break;
20673 :
20674 1 : case E_V8QImode:
20675 1 : use_vec_merge = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
20676 : break;
20677 :
20678 3 : case E_V32QImode:
20679 3 : half_mode = V16QImode;
20680 3 : j = 0;
20681 3 : n = 16;
20682 3 : goto half;
20683 :
20684 17 : case E_V16HFmode:
20685 17 : case E_V16BFmode:
20686 : /* For ELT == 0, vec_setv8hf_0 can save 1 vpbroadcastw. */
20687 17 : if (TARGET_AVX2 && elt != 0)
20688 : {
20689 12 : mmode = SImode;
20690 12 : gen_blendm = ((mode == E_V16HFmode) ? gen_avx2_pblendph_1
20691 : : gen_avx2_pblendbf_1);
20692 : blendm_const = true;
20693 : break;
20694 : }
20695 : else
20696 : {
20697 5 : half_mode = ((mode == E_V16HFmode) ? V8HFmode : V8BFmode);
20698 3 : j = ((mode == E_V16HFmode) ? 6 : 7);
20699 5 : n = 8;
20700 5 : goto half;
20701 : }
20702 :
20703 5 : case E_V16HImode:
20704 5 : half_mode = V8HImode;
20705 5 : j = 1;
20706 5 : n = 8;
20707 5 : goto half;
20708 :
20709 15 : case E_V8SImode:
20710 15 : half_mode = V4SImode;
20711 15 : j = 2;
20712 15 : n = 4;
20713 15 : goto half;
20714 :
20715 16 : case E_V4DImode:
20716 16 : half_mode = V2DImode;
20717 16 : j = 3;
20718 16 : n = 2;
20719 16 : goto half;
20720 :
20721 4 : case E_V8SFmode:
20722 4 : half_mode = V4SFmode;
20723 4 : j = 4;
20724 4 : n = 4;
20725 4 : goto half;
20726 :
20727 6 : case E_V4DFmode:
20728 6 : half_mode = V2DFmode;
20729 6 : j = 5;
20730 6 : n = 2;
20731 6 : goto half;
20732 :
20733 54 : half:
20734 : /* Compute offset. */
20735 54 : i = elt / n;
20736 54 : elt %= n;
20737 :
20738 54 : gcc_assert (i <= 1);
20739 :
20740 : /* Extract the half. */
20741 54 : tmp = gen_reg_rtx (half_mode);
20742 54 : emit_insn (gen_extract[j][i] (tmp, target));
20743 :
20744 : /* Put val in tmp at elt. */
20745 54 : ix86_expand_vector_set (false, tmp, val, elt);
20746 :
20747 : /* Put it back. */
20748 54 : emit_insn (gen_insert[j][i] (target, target, tmp));
20749 54 : return;
20750 :
20751 8 : case E_V8DFmode:
20752 8 : if (TARGET_AVX512F)
20753 : {
20754 : mmode = QImode;
20755 : gen_blendm = gen_avx512f_blendmv8df;
20756 : }
20757 : break;
20758 :
20759 8 : case E_V8DImode:
20760 8 : if (TARGET_AVX512F)
20761 : {
20762 : mmode = QImode;
20763 : gen_blendm = gen_avx512f_blendmv8di;
20764 : }
20765 : break;
20766 :
20767 0 : case E_V16SFmode:
20768 0 : if (TARGET_AVX512F)
20769 : {
20770 : mmode = HImode;
20771 : gen_blendm = gen_avx512f_blendmv16sf;
20772 : }
20773 : break;
20774 :
20775 0 : case E_V16SImode:
20776 0 : if (TARGET_AVX512F)
20777 : {
20778 : mmode = HImode;
20779 : gen_blendm = gen_avx512f_blendmv16si;
20780 : }
20781 : break;
20782 :
20783 12 : case E_V32HFmode:
20784 12 : if (TARGET_AVX512BW)
20785 : {
20786 : mmode = SImode;
20787 : gen_blendm = gen_avx512bw_blendmv32hf;
20788 : }
20789 : break;
20790 12 : case E_V32BFmode:
20791 12 : if (TARGET_AVX512BW)
20792 : {
20793 : mmode = SImode;
20794 : gen_blendm = gen_avx512bw_blendmv32bf;
20795 : }
20796 : break;
20797 11 : case E_V32HImode:
20798 11 : if (TARGET_AVX512BW)
20799 : {
20800 : mmode = SImode;
20801 : gen_blendm = gen_avx512bw_blendmv32hi;
20802 : }
20803 7 : else if (TARGET_AVX512F)
20804 : {
20805 7 : half_mode = E_V8HImode;
20806 7 : n = 8;
20807 7 : goto quarter;
20808 : }
20809 : break;
20810 :
20811 12 : case E_V64QImode:
20812 12 : if (TARGET_AVX512BW)
20813 : {
20814 : mmode = DImode;
20815 : gen_blendm = gen_avx512bw_blendmv64qi;
20816 : }
20817 6 : else if (TARGET_AVX512F)
20818 : {
20819 6 : half_mode = E_V16QImode;
20820 6 : n = 16;
20821 6 : goto quarter;
20822 : }
20823 : break;
20824 :
20825 13 : quarter:
20826 : /* Compute offset. */
20827 13 : i = elt / n;
20828 13 : elt %= n;
20829 :
20830 13 : gcc_assert (i <= 3);
20831 :
20832 13 : {
20833 : /* Extract the quarter. */
20834 13 : tmp = gen_reg_rtx (V4SImode);
20835 13 : rtx tmp2 = gen_lowpart (V16SImode, target);
20836 13 : rtx mask = gen_reg_rtx (QImode);
20837 :
20838 13 : emit_move_insn (mask, constm1_rtx);
20839 13 : emit_insn (gen_avx512f_vextracti32x4_mask (tmp, tmp2, GEN_INT (i),
20840 : tmp, mask));
20841 :
20842 13 : tmp2 = gen_reg_rtx (half_mode);
20843 13 : emit_move_insn (tmp2, gen_lowpart (half_mode, tmp));
20844 13 : tmp = tmp2;
20845 :
20846 : /* Put val in tmp at elt. */
20847 13 : ix86_expand_vector_set (false, tmp, val, elt);
20848 :
20849 : /* Put it back. */
20850 13 : tmp2 = gen_reg_rtx (V16SImode);
20851 13 : rtx tmp3 = gen_lowpart (V16SImode, target);
20852 13 : mask = gen_reg_rtx (HImode);
20853 13 : emit_move_insn (mask, constm1_rtx);
20854 13 : tmp = gen_lowpart (V4SImode, tmp);
20855 13 : emit_insn (gen_avx512f_vinserti32x4_mask (tmp2, tmp3, tmp, GEN_INT (i),
20856 : tmp3, mask));
20857 13 : emit_move_insn (target, gen_lowpart (mode, tmp2));
20858 : }
20859 13 : return;
20860 :
20861 : default:
20862 : break;
20863 : }
20864 :
20865 5037 : if (mmode != VOIDmode)
20866 : {
20867 56 : tmp = gen_reg_rtx (mode);
20868 56 : emit_insn (gen_rtx_SET (tmp, gen_rtx_VEC_DUPLICATE (mode, val)));
20869 56 : rtx merge_mask = gen_int_mode (HOST_WIDE_INT_1U << elt, mmode);
20870 : /* The avx512*_blendm<mode> expanders have different operand order
20871 : from VEC_MERGE. In VEC_MERGE, the first input operand is used for
20872 : elements where the mask is set and second input operand otherwise,
20873 : in {sse,avx}*_*blend* the first input operand is used for elements
20874 : where the mask is clear and second input operand otherwise. */
20875 56 : if (!blendm_const)
20876 44 : merge_mask = force_reg (mmode, merge_mask);
20877 56 : emit_insn (gen_blendm (target, target, tmp, merge_mask));
20878 : }
20879 5595 : else if (use_vec_merge)
20880 : {
20881 5583 : do_vec_merge:
20882 5647 : if (!nonimmediate_operand (val, inner_mode))
20883 0 : val = force_reg (inner_mode, val);
20884 5647 : tmp = gen_rtx_VEC_DUPLICATE (mode, val);
20885 5647 : tmp = gen_rtx_VEC_MERGE (mode, tmp, target,
20886 : GEN_INT (HOST_WIDE_INT_1U << elt));
20887 5647 : emit_insn (gen_rtx_SET (target, tmp));
20888 : }
20889 : else
20890 : {
20891 24 : rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode));
20892 :
20893 12 : emit_move_insn (mem, target);
20894 :
20895 24 : tmp = adjust_address (mem, inner_mode, elt * GET_MODE_SIZE (inner_mode));
20896 12 : emit_move_insn (tmp, val);
20897 :
20898 12 : emit_move_insn (target, mem);
20899 : }
20900 : }
20901 :
20902 : void
20903 113753 : ix86_expand_vector_extract (bool mmx_ok, rtx target, rtx vec, int elt)
20904 : {
20905 119191 : machine_mode mode = GET_MODE (vec);
20906 119191 : machine_mode inner_mode = GET_MODE_INNER (mode);
20907 119191 : bool use_vec_extr = false;
20908 119191 : rtx tmp;
20909 :
20910 119191 : switch (mode)
20911 : {
20912 9780 : case E_V2SImode:
20913 9780 : use_vec_extr = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
20914 : if (use_vec_extr)
20915 : break;
20916 : /* FALLTHRU */
20917 :
20918 12475 : case E_V2SFmode:
20919 12475 : if (!mmx_ok)
20920 : break;
20921 : /* FALLTHRU */
20922 :
20923 : case E_V2DFmode:
20924 : case E_V2DImode:
20925 : case E_V2TImode:
20926 : case E_V4TImode:
20927 : use_vec_extr = true;
20928 : break;
20929 :
20930 7670 : case E_V4SFmode:
20931 7670 : use_vec_extr = TARGET_SSE4_1;
20932 7670 : if (use_vec_extr)
20933 : break;
20934 :
20935 3959 : switch (elt)
20936 : {
20937 : case 0:
20938 : tmp = vec;
20939 : break;
20940 :
20941 1725 : case 1:
20942 1725 : case 3:
20943 1725 : tmp = gen_reg_rtx (mode);
20944 1725 : emit_insn (gen_sse_shufps_v4sf (tmp, vec, vec,
20945 : GEN_INT (elt), GEN_INT (elt),
20946 1725 : GEN_INT (elt+4), GEN_INT (elt+4)));
20947 1725 : break;
20948 :
20949 853 : case 2:
20950 853 : tmp = gen_reg_rtx (mode);
20951 853 : emit_insn (gen_vec_interleave_highv4sf (tmp, vec, vec));
20952 853 : break;
20953 :
20954 0 : default:
20955 0 : gcc_unreachable ();
20956 : }
20957 : vec = tmp;
20958 : use_vec_extr = true;
20959 : elt = 0;
20960 : break;
20961 :
20962 25711 : case E_V4SImode:
20963 25711 : use_vec_extr = TARGET_SSE4_1;
20964 25711 : if (use_vec_extr)
20965 : break;
20966 :
20967 19783 : if (TARGET_SSE2)
20968 : {
20969 19779 : switch (elt)
20970 : {
20971 : case 0:
20972 : tmp = vec;
20973 : break;
20974 :
20975 6659 : case 1:
20976 6659 : case 3:
20977 6659 : tmp = gen_reg_rtx (mode);
20978 6659 : emit_insn (gen_sse2_pshufd_1 (tmp, vec,
20979 : GEN_INT (elt), GEN_INT (elt),
20980 : GEN_INT (elt), GEN_INT (elt)));
20981 6659 : break;
20982 :
20983 3348 : case 2:
20984 3348 : tmp = gen_reg_rtx (mode);
20985 3348 : emit_insn (gen_vec_interleave_highv4si (tmp, vec, vec));
20986 3348 : break;
20987 :
20988 0 : default:
20989 0 : gcc_unreachable ();
20990 : }
20991 : vec = tmp;
20992 : use_vec_extr = true;
20993 : elt = 0;
20994 : }
20995 : else
20996 : {
20997 : /* For SSE1, we have to reuse the V4SF code. */
20998 4 : ix86_expand_vector_extract (false, gen_lowpart (SFmode, target),
20999 4 : gen_lowpart (V4SFmode, vec), elt);
21000 4 : return;
21001 : }
21002 : break;
21003 :
21004 6976 : case E_V8HImode:
21005 6976 : case E_V8HFmode:
21006 6976 : case E_V8BFmode:
21007 6976 : case E_V2HImode:
21008 6976 : case E_V2HFmode:
21009 6976 : case E_V2BFmode:
21010 6976 : use_vec_extr = TARGET_SSE2;
21011 6976 : break;
21012 889 : case E_V4HImode:
21013 889 : case E_V4HFmode:
21014 889 : case E_V4BFmode:
21015 889 : use_vec_extr = mmx_ok && (TARGET_SSE || TARGET_3DNOW_A);
21016 : break;
21017 :
21018 8447 : case E_V16QImode:
21019 8447 : use_vec_extr = TARGET_SSE4_1;
21020 8447 : if (!use_vec_extr
21021 6830 : && TARGET_SSE2
21022 6830 : && elt == 0
21023 12373 : && (optimize_insn_for_size_p () || TARGET_INTER_UNIT_MOVES_FROM_VEC))
21024 : {
21025 3925 : tmp = gen_reg_rtx (SImode);
21026 3925 : ix86_expand_vector_extract (false, tmp, gen_lowpart (V4SImode, vec),
21027 : 0);
21028 3925 : emit_insn (gen_rtx_SET (target, gen_lowpart (QImode, tmp)));
21029 3925 : return;
21030 : }
21031 : break;
21032 78 : case E_V4QImode:
21033 78 : use_vec_extr = TARGET_SSE4_1;
21034 78 : break;
21035 :
21036 215 : case E_V8SFmode:
21037 215 : if (TARGET_AVX)
21038 : {
21039 215 : tmp = gen_reg_rtx (V4SFmode);
21040 215 : if (elt < 4)
21041 102 : emit_insn (gen_vec_extract_lo_v8sf (tmp, vec));
21042 : else
21043 113 : emit_insn (gen_vec_extract_hi_v8sf (tmp, vec));
21044 215 : ix86_expand_vector_extract (false, target, tmp, elt & 3);
21045 215 : return;
21046 : }
21047 : break;
21048 :
21049 350 : case E_V4DFmode:
21050 350 : if (TARGET_AVX)
21051 : {
21052 350 : tmp = gen_reg_rtx (V2DFmode);
21053 350 : if (elt < 2)
21054 189 : emit_insn (gen_vec_extract_lo_v4df (tmp, vec));
21055 : else
21056 161 : emit_insn (gen_vec_extract_hi_v4df (tmp, vec));
21057 350 : ix86_expand_vector_extract (false, target, tmp, elt & 1);
21058 350 : return;
21059 : }
21060 : break;
21061 :
21062 253 : case E_V32QImode:
21063 253 : if (TARGET_AVX)
21064 : {
21065 253 : tmp = gen_reg_rtx (V16QImode);
21066 253 : if (elt < 16)
21067 130 : emit_insn (gen_vec_extract_lo_v32qi (tmp, vec));
21068 : else
21069 123 : emit_insn (gen_vec_extract_hi_v32qi (tmp, vec));
21070 253 : ix86_expand_vector_extract (false, target, tmp, elt & 15);
21071 253 : return;
21072 : }
21073 : break;
21074 :
21075 640 : case E_V16HImode:
21076 640 : if (TARGET_AVX)
21077 : {
21078 640 : tmp = gen_reg_rtx (V8HImode);
21079 640 : if (elt < 8)
21080 312 : emit_insn (gen_vec_extract_lo_v16hi (tmp, vec));
21081 : else
21082 328 : emit_insn (gen_vec_extract_hi_v16hi (tmp, vec));
21083 640 : ix86_expand_vector_extract (false, target, tmp, elt & 7);
21084 640 : return;
21085 : }
21086 : break;
21087 :
21088 1102 : case E_V8SImode:
21089 1102 : if (TARGET_AVX)
21090 : {
21091 1102 : tmp = gen_reg_rtx (V4SImode);
21092 1102 : if (elt < 4)
21093 532 : emit_insn (gen_vec_extract_lo_v8si (tmp, vec));
21094 : else
21095 570 : emit_insn (gen_vec_extract_hi_v8si (tmp, vec));
21096 1102 : ix86_expand_vector_extract (false, target, tmp, elt & 3);
21097 1102 : return;
21098 : }
21099 : break;
21100 :
21101 1345 : case E_V4DImode:
21102 1345 : if (TARGET_AVX)
21103 : {
21104 1345 : tmp = gen_reg_rtx (V2DImode);
21105 1345 : if (elt < 2)
21106 709 : emit_insn (gen_vec_extract_lo_v4di (tmp, vec));
21107 : else
21108 636 : emit_insn (gen_vec_extract_hi_v4di (tmp, vec));
21109 1345 : ix86_expand_vector_extract (false, target, tmp, elt & 1);
21110 1345 : return;
21111 : }
21112 : break;
21113 :
21114 8 : case E_V32HImode:
21115 8 : if (TARGET_AVX512BW)
21116 : {
21117 8 : tmp = gen_reg_rtx (V16HImode);
21118 8 : if (elt < 16)
21119 3 : emit_insn (gen_vec_extract_lo_v32hi (tmp, vec));
21120 : else
21121 5 : emit_insn (gen_vec_extract_hi_v32hi (tmp, vec));
21122 8 : ix86_expand_vector_extract (false, target, tmp, elt & 15);
21123 8 : return;
21124 : }
21125 : break;
21126 :
21127 10 : case E_V64QImode:
21128 10 : if (TARGET_AVX512BW)
21129 : {
21130 10 : tmp = gen_reg_rtx (V32QImode);
21131 10 : if (elt < 32)
21132 5 : emit_insn (gen_vec_extract_lo_v64qi (tmp, vec));
21133 : else
21134 5 : emit_insn (gen_vec_extract_hi_v64qi (tmp, vec));
21135 10 : ix86_expand_vector_extract (false, target, tmp, elt & 31);
21136 10 : return;
21137 : }
21138 : break;
21139 :
21140 23 : case E_V16SFmode:
21141 23 : tmp = gen_reg_rtx (V8SFmode);
21142 23 : if (elt < 8)
21143 13 : emit_insn (gen_vec_extract_lo_v16sf (tmp, vec));
21144 : else
21145 10 : emit_insn (gen_vec_extract_hi_v16sf (tmp, vec));
21146 23 : ix86_expand_vector_extract (false, target, tmp, elt & 7);
21147 23 : return;
21148 :
21149 96 : case E_V8DFmode:
21150 96 : tmp = gen_reg_rtx (V4DFmode);
21151 96 : if (elt < 4)
21152 60 : emit_insn (gen_vec_extract_lo_v8df (tmp, vec));
21153 : else
21154 36 : emit_insn (gen_vec_extract_hi_v8df (tmp, vec));
21155 96 : ix86_expand_vector_extract (false, target, tmp, elt & 3);
21156 96 : return;
21157 :
21158 236 : case E_V16SImode:
21159 236 : tmp = gen_reg_rtx (V8SImode);
21160 236 : if (elt < 8)
21161 125 : emit_insn (gen_vec_extract_lo_v16si (tmp, vec));
21162 : else
21163 111 : emit_insn (gen_vec_extract_hi_v16si (tmp, vec));
21164 236 : ix86_expand_vector_extract (false, target, tmp, elt & 7);
21165 236 : return;
21166 :
21167 669 : case E_V8DImode:
21168 669 : tmp = gen_reg_rtx (V4DImode);
21169 669 : if (elt < 4)
21170 382 : emit_insn (gen_vec_extract_lo_v8di (tmp, vec));
21171 : else
21172 287 : emit_insn (gen_vec_extract_hi_v8di (tmp, vec));
21173 669 : ix86_expand_vector_extract (false, target, tmp, elt & 3);
21174 669 : return;
21175 :
21176 45 : case E_V32HFmode:
21177 45 : case E_V32BFmode:
21178 45 : if (TARGET_AVX512BW)
21179 : {
21180 45 : tmp = (mode == E_V32HFmode
21181 45 : ? gen_reg_rtx (V16HFmode)
21182 7 : : gen_reg_rtx (V16BFmode));
21183 45 : if (elt < 16)
21184 31 : emit_insn (gen_vec_extract_lo (mode, tmp, vec));
21185 : else
21186 14 : emit_insn (gen_vec_extract_hi (mode, tmp, vec));
21187 45 : ix86_expand_vector_extract (false, target, tmp, elt & 15);
21188 45 : return;
21189 : }
21190 : break;
21191 :
21192 442 : case E_V16HFmode:
21193 442 : case E_V16BFmode:
21194 442 : if (TARGET_AVX)
21195 : {
21196 442 : tmp = (mode == E_V16HFmode
21197 442 : ? gen_reg_rtx (V8HFmode)
21198 339 : : gen_reg_rtx (V8BFmode));
21199 442 : if (elt < 8)
21200 233 : emit_insn (gen_vec_extract_lo (mode, tmp, vec));
21201 : else
21202 209 : emit_insn (gen_vec_extract_hi (mode, tmp, vec));
21203 442 : ix86_expand_vector_extract (false, target, tmp, elt & 7);
21204 442 : return;
21205 : }
21206 : break;
21207 :
21208 630 : case E_V8QImode:
21209 630 : use_vec_extr = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
21210 : /* ??? Could extract the appropriate HImode element and shift. */
21211 : break;
21212 :
21213 : default:
21214 : break;
21215 : }
21216 :
21217 39992 : if (use_vec_extr)
21218 : {
21219 101096 : tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (elt)));
21220 101096 : tmp = gen_rtx_VEC_SELECT (inner_mode, vec, tmp);
21221 :
21222 : /* Let the rtl optimizers know about the zero extension performed. */
21223 101096 : if (inner_mode == QImode || inner_mode == HImode)
21224 : {
21225 8978 : rtx reg = gen_reg_rtx (SImode);
21226 8978 : tmp = gen_rtx_ZERO_EXTEND (SImode, tmp);
21227 8978 : emit_move_insn (reg, tmp);
21228 8978 : tmp = gen_lowpart (inner_mode, reg);
21229 8978 : SUBREG_PROMOTED_VAR_P (tmp) = 1;
21230 8978 : SUBREG_PROMOTED_SET (tmp, 1);
21231 : }
21232 :
21233 101096 : emit_move_insn (target, tmp);
21234 : }
21235 : else
21236 : {
21237 17464 : rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode));
21238 :
21239 8732 : emit_move_insn (mem, vec);
21240 :
21241 17464 : tmp = adjust_address (mem, inner_mode, elt*GET_MODE_SIZE (inner_mode));
21242 8732 : emit_move_insn (target, tmp);
21243 : }
21244 : }
21245 :
21246 : /* Generate code to copy vector bits i / 2 ... i - 1 from vector SRC
21247 : to bits 0 ... i / 2 - 1 of vector DEST, which has the same mode.
21248 : The upper bits of DEST are undefined, though they shouldn't cause
21249 : exceptions (some bits from src or all zeros are ok). */
21250 :
21251 : static void
21252 42661 : emit_reduc_half (rtx dest, rtx src, int i)
21253 : {
21254 42661 : rtx tem, d = dest;
21255 42661 : switch (GET_MODE (src))
21256 : {
21257 6218 : case E_V4SFmode:
21258 6218 : if (i == 128)
21259 3109 : tem = gen_sse_movhlps (dest, src, src);
21260 : else
21261 3109 : tem = gen_sse_shufps_v4sf (dest, src, src, const1_rtx, const1_rtx,
21262 : GEN_INT (1 + 4), GEN_INT (1 + 4));
21263 : break;
21264 3734 : case E_V2DFmode:
21265 3734 : tem = gen_vec_interleave_highv2df (dest, src, src);
21266 3734 : break;
21267 76 : case E_V4QImode:
21268 76 : d = gen_reg_rtx (V1SImode);
21269 76 : tem = gen_mmx_lshrv1si3 (d, gen_lowpart (V1SImode, src),
21270 76 : GEN_INT (i / 2));
21271 76 : break;
21272 608 : case E_V8QImode:
21273 608 : case E_V4HImode:
21274 608 : d = gen_reg_rtx (V1DImode);
21275 608 : tem = gen_mmx_lshrv1di3 (d, gen_lowpart (V1DImode, src),
21276 608 : GEN_INT (i / 2));
21277 608 : break;
21278 32025 : case E_V16QImode:
21279 32025 : case E_V8HImode:
21280 32025 : case E_V8HFmode:
21281 32025 : case E_V4SImode:
21282 32025 : case E_V2DImode:
21283 32025 : if (TARGET_SSE_REDUCTION_PREFER_PSHUF)
21284 : {
21285 15 : if (i == 128)
21286 : {
21287 9 : d = gen_reg_rtx (V4SImode);
21288 18 : tem = gen_sse2_pshufd_1 (
21289 9 : d, force_reg (V4SImode, gen_lowpart (V4SImode, src)),
21290 : GEN_INT (2), GEN_INT (3), GEN_INT (2), GEN_INT (3));
21291 9 : break;
21292 : }
21293 6 : else if (i == 64)
21294 : {
21295 5 : d = gen_reg_rtx (V4SImode);
21296 10 : tem = gen_sse2_pshufd_1 (
21297 5 : d, force_reg (V4SImode, gen_lowpart (V4SImode, src)),
21298 : GEN_INT (1), GEN_INT (1), GEN_INT (1), GEN_INT (1));
21299 5 : break;
21300 : }
21301 1 : else if (i == 32)
21302 : {
21303 1 : d = gen_reg_rtx (V8HImode);
21304 2 : tem = gen_sse2_pshuflw_1 (
21305 1 : d, force_reg (V8HImode, gen_lowpart (V8HImode, src)),
21306 : GEN_INT (1), GEN_INT (1), GEN_INT (1), GEN_INT (1));
21307 1 : break;
21308 : }
21309 : }
21310 32010 : d = gen_reg_rtx (V1TImode);
21311 32010 : tem = gen_sse2_lshrv1ti3 (d, gen_lowpart (V1TImode, src),
21312 32010 : GEN_INT (i / 2));
21313 32010 : break;
21314 0 : case E_V8SFmode:
21315 0 : if (i == 256)
21316 0 : tem = gen_avx_vperm2f128v8sf3 (dest, src, src, const1_rtx);
21317 : else
21318 0 : tem = gen_avx_shufps256 (dest, src, src,
21319 : GEN_INT (i == 128 ? 2 + (3 << 2) : 1));
21320 : break;
21321 0 : case E_V4DFmode:
21322 0 : if (i == 256)
21323 0 : tem = gen_avx_vperm2f128v4df3 (dest, src, src, const1_rtx);
21324 : else
21325 0 : tem = gen_avx_shufpd256 (dest, src, src, const1_rtx);
21326 : break;
21327 0 : case E_V32QImode:
21328 0 : case E_V16HImode:
21329 0 : case E_V16HFmode:
21330 0 : case E_V8SImode:
21331 0 : case E_V4DImode:
21332 0 : if (i == 256)
21333 : {
21334 0 : if (GET_MODE (dest) != V4DImode)
21335 0 : d = gen_reg_rtx (V4DImode);
21336 0 : tem = gen_avx2_permv2ti (d, gen_lowpart (V4DImode, src),
21337 0 : gen_lowpart (V4DImode, src),
21338 : const1_rtx);
21339 : }
21340 : else
21341 : {
21342 0 : d = gen_reg_rtx (V2TImode);
21343 0 : tem = gen_avx2_lshrv2ti3 (d, gen_lowpart (V2TImode, src),
21344 0 : GEN_INT (i / 2));
21345 : }
21346 : break;
21347 0 : case E_V64QImode:
21348 0 : case E_V32HImode:
21349 0 : case E_V32HFmode:
21350 0 : if (i < 64)
21351 : {
21352 0 : d = gen_reg_rtx (V4TImode);
21353 0 : tem = gen_avx512bw_lshrv4ti3 (d, gen_lowpart (V4TImode, src),
21354 0 : GEN_INT (i / 2));
21355 0 : break;
21356 : }
21357 : /* FALLTHRU */
21358 0 : case E_V16SImode:
21359 0 : case E_V16SFmode:
21360 0 : case E_V8DImode:
21361 0 : case E_V8DFmode:
21362 0 : if (i > 128)
21363 0 : tem = gen_avx512f_shuf_i32x4_1 (gen_lowpart (V16SImode, dest),
21364 0 : gen_lowpart (V16SImode, src),
21365 0 : gen_lowpart (V16SImode, src),
21366 : GEN_INT (0x4 + (i == 512 ? 4 : 0)),
21367 : GEN_INT (0x5 + (i == 512 ? 4 : 0)),
21368 : GEN_INT (0x6 + (i == 512 ? 4 : 0)),
21369 : GEN_INT (0x7 + (i == 512 ? 4 : 0)),
21370 : GEN_INT (0xC), GEN_INT (0xD),
21371 : GEN_INT (0xE), GEN_INT (0xF),
21372 : GEN_INT (0x10), GEN_INT (0x11),
21373 : GEN_INT (0x12), GEN_INT (0x13),
21374 : GEN_INT (0x14), GEN_INT (0x15),
21375 : GEN_INT (0x16), GEN_INT (0x17));
21376 : else
21377 0 : tem = gen_avx512f_pshufd_1 (gen_lowpart (V16SImode, dest),
21378 0 : gen_lowpart (V16SImode, src),
21379 : GEN_INT (i == 128 ? 0x2 : 0x1),
21380 : GEN_INT (0x3),
21381 : GEN_INT (0x3),
21382 : GEN_INT (0x3),
21383 : GEN_INT (i == 128 ? 0x6 : 0x5),
21384 : GEN_INT (0x7),
21385 : GEN_INT (0x7),
21386 : GEN_INT (0x7),
21387 : GEN_INT (i == 128 ? 0xA : 0x9),
21388 : GEN_INT (0xB),
21389 : GEN_INT (0xB),
21390 : GEN_INT (0xB),
21391 : GEN_INT (i == 128 ? 0xE : 0xD),
21392 : GEN_INT (0xF),
21393 : GEN_INT (0xF),
21394 : GEN_INT (0xF));
21395 : break;
21396 0 : default:
21397 0 : gcc_unreachable ();
21398 : }
21399 42661 : emit_insn (tem);
21400 42661 : if (d != dest)
21401 32709 : emit_move_insn (dest, gen_lowpart (GET_MODE (dest), d));
21402 42661 : }
21403 :
21404 : /* Expand a vector reduction. FN is the binary pattern to reduce;
21405 : DEST is the destination; IN is the input vector. */
21406 :
21407 : void
21408 21522 : ix86_expand_reduc (rtx (*fn) (rtx, rtx, rtx), rtx dest, rtx in)
21409 : {
21410 21522 : rtx half, dst, vec = in;
21411 21522 : machine_mode mode = GET_MODE (in);
21412 21522 : int i;
21413 :
21414 : /* SSE4 has a special instruction for V8HImode UMIN reduction. */
21415 21522 : if (TARGET_SSE4_1
21416 10143 : && mode == V8HImode
21417 780 : && fn == gen_uminv8hi3)
21418 : {
21419 4 : emit_insn (gen_sse4_1_phminposuw (dest, in));
21420 4 : return;
21421 : }
21422 :
21423 : /* SSE3 has haddpd, some targets prefer that over movhlpd plus add. */
21424 21518 : if (TARGET_SSE3
21425 10191 : && TARGET_V2DF_REDUCTION_PREFER_HADDPD
21426 10 : && mode == V2DFmode
21427 10 : && fn == gen_addv2df3)
21428 : {
21429 10 : emit_insn (gen_sse3_haddv2df3 (dest, in, in));
21430 10 : return;
21431 : }
21432 :
21433 43016 : for (i = GET_MODE_BITSIZE (mode);
21434 128338 : i > GET_MODE_UNIT_BITSIZE (mode);
21435 42661 : i >>= 1)
21436 : {
21437 42661 : half = gen_reg_rtx (mode);
21438 42661 : emit_reduc_half (half, vec, i);
21439 85322 : if (i == GET_MODE_UNIT_BITSIZE (mode) * 2)
21440 : dst = dest;
21441 : else
21442 21153 : dst = gen_reg_rtx (mode);
21443 42661 : emit_insn (fn (dst, half, vec));
21444 42661 : vec = dst;
21445 : }
21446 : }
21447 :
21448 : /* Output code to perform a conditional jump to LABEL, if C2 flag in
21449 : FP status register is set. */
21450 :
21451 : void
21452 285 : ix86_emit_fp_unordered_jump (rtx label)
21453 : {
21454 285 : rtx reg = gen_reg_rtx (HImode);
21455 285 : rtx_insn *insn;
21456 285 : rtx temp;
21457 :
21458 285 : emit_insn (gen_x86_fnstsw_1 (reg));
21459 :
21460 285 : if (TARGET_SAHF && (TARGET_USE_SAHF || optimize_insn_for_size_p ()))
21461 : {
21462 37 : emit_insn (gen_x86_sahf_1 (reg));
21463 :
21464 37 : temp = gen_rtx_REG (CCmode, FLAGS_REG);
21465 37 : temp = gen_rtx_UNORDERED (VOIDmode, temp, const0_rtx);
21466 : }
21467 : else
21468 : {
21469 248 : emit_insn (gen_testqi_ext_1_ccno (reg, GEN_INT (0x04)));
21470 :
21471 248 : temp = gen_rtx_REG (CCNOmode, FLAGS_REG);
21472 248 : temp = gen_rtx_NE (VOIDmode, temp, const0_rtx);
21473 : }
21474 :
21475 285 : temp = gen_rtx_IF_THEN_ELSE (VOIDmode, temp,
21476 : gen_rtx_LABEL_REF (VOIDmode, label),
21477 : pc_rtx);
21478 285 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, temp));
21479 285 : predict_jump (REG_BR_PROB_BASE * 10 / 100);
21480 285 : JUMP_LABEL (insn) = label;
21481 285 : }
21482 :
21483 : /* Output code to perform an sinh XFmode calculation. */
21484 :
21485 : void
21486 2 : ix86_emit_i387_sinh (rtx op0, rtx op1)
21487 : {
21488 2 : rtx e1 = gen_reg_rtx (XFmode);
21489 2 : rtx e2 = gen_reg_rtx (XFmode);
21490 2 : rtx scratch = gen_reg_rtx (HImode);
21491 2 : rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
21492 2 : rtx half = const_double_from_real_value (dconsthalf, XFmode);
21493 2 : rtx cst1, tmp;
21494 2 : rtx_code_label *jump_label = gen_label_rtx ();
21495 2 : rtx_insn *insn;
21496 :
21497 : /* scratch = fxam (op1) */
21498 2 : emit_insn (gen_fxamxf2_i387 (scratch, op1));
21499 :
21500 : /* e1 = expm1 (|op1|) */
21501 2 : emit_insn (gen_absxf2 (e2, op1));
21502 2 : emit_insn (gen_expm1xf2 (e1, e2));
21503 :
21504 : /* e2 = e1 / (e1 + 1.0) + e1 */
21505 2 : cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
21506 2 : emit_insn (gen_addxf3 (e2, e1, cst1));
21507 2 : emit_insn (gen_divxf3 (e2, e1, e2));
21508 2 : emit_insn (gen_addxf3 (e2, e2, e1));
21509 :
21510 : /* flags = signbit (op1) */
21511 2 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
21512 :
21513 : /* if (flags) then e2 = -e2 */
21514 2 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
21515 : gen_rtx_EQ (VOIDmode, flags, const0_rtx),
21516 : gen_rtx_LABEL_REF (VOIDmode, jump_label),
21517 : pc_rtx);
21518 2 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
21519 2 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
21520 2 : JUMP_LABEL (insn) = jump_label;
21521 :
21522 2 : emit_insn (gen_negxf2 (e2, e2));
21523 :
21524 2 : emit_label (jump_label);
21525 2 : LABEL_NUSES (jump_label) = 1;
21526 :
21527 : /* op0 = 0.5 * e2 */
21528 2 : half = force_reg (XFmode, half);
21529 2 : emit_insn (gen_mulxf3 (op0, e2, half));
21530 2 : }
21531 :
21532 : /* Output code to perform an cosh XFmode calculation. */
21533 :
21534 : void
21535 3 : ix86_emit_i387_cosh (rtx op0, rtx op1)
21536 : {
21537 3 : rtx e1 = gen_reg_rtx (XFmode);
21538 3 : rtx e2 = gen_reg_rtx (XFmode);
21539 3 : rtx half = const_double_from_real_value (dconsthalf, XFmode);
21540 3 : rtx cst1;
21541 :
21542 : /* e1 = exp (op1) */
21543 3 : emit_insn (gen_expxf2 (e1, op1));
21544 :
21545 : /* e2 = e1 + 1.0 / e1 */
21546 3 : cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
21547 3 : emit_insn (gen_divxf3 (e2, cst1, e1));
21548 3 : emit_insn (gen_addxf3 (e2, e1, e2));
21549 :
21550 : /* op0 = 0.5 * e2 */
21551 3 : half = force_reg (XFmode, half);
21552 3 : emit_insn (gen_mulxf3 (op0, e2, half));
21553 3 : }
21554 :
21555 : /* Output code to perform an tanh XFmode calculation. */
21556 :
21557 : void
21558 1 : ix86_emit_i387_tanh (rtx op0, rtx op1)
21559 : {
21560 1 : rtx e1 = gen_reg_rtx (XFmode);
21561 1 : rtx e2 = gen_reg_rtx (XFmode);
21562 1 : rtx scratch = gen_reg_rtx (HImode);
21563 1 : rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
21564 1 : rtx cst2, tmp;
21565 1 : rtx_code_label *jump_label = gen_label_rtx ();
21566 1 : rtx_insn *insn;
21567 :
21568 : /* scratch = fxam (op1) */
21569 1 : emit_insn (gen_fxamxf2_i387 (scratch, op1));
21570 :
21571 : /* e1 = expm1 (-|2 * op1|) */
21572 1 : emit_insn (gen_addxf3 (e2, op1, op1));
21573 1 : emit_insn (gen_absxf2 (e2, e2));
21574 1 : emit_insn (gen_negxf2 (e2, e2));
21575 1 : emit_insn (gen_expm1xf2 (e1, e2));
21576 :
21577 : /* e2 = e1 / (e1 + 2.0) */
21578 1 : cst2 = force_reg (XFmode, CONST2_RTX (XFmode));
21579 1 : emit_insn (gen_addxf3 (e2, e1, cst2));
21580 1 : emit_insn (gen_divxf3 (e2, e1, e2));
21581 :
21582 : /* flags = signbit (op1) */
21583 1 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
21584 :
21585 : /* if (!flags) then e2 = -e2 */
21586 1 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
21587 : gen_rtx_NE (VOIDmode, flags, const0_rtx),
21588 : gen_rtx_LABEL_REF (VOIDmode, jump_label),
21589 : pc_rtx);
21590 1 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
21591 1 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
21592 1 : JUMP_LABEL (insn) = jump_label;
21593 :
21594 1 : emit_insn (gen_negxf2 (e2, e2));
21595 :
21596 1 : emit_label (jump_label);
21597 1 : LABEL_NUSES (jump_label) = 1;
21598 :
21599 1 : emit_move_insn (op0, e2);
21600 1 : }
21601 :
21602 : /* Output code to perform an asinh XFmode calculation. */
21603 :
21604 : void
21605 0 : ix86_emit_i387_asinh (rtx op0, rtx op1)
21606 : {
21607 0 : rtx e1 = gen_reg_rtx (XFmode);
21608 0 : rtx e2 = gen_reg_rtx (XFmode);
21609 0 : rtx scratch = gen_reg_rtx (HImode);
21610 0 : rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
21611 0 : rtx cst1, tmp;
21612 0 : rtx_code_label *jump_label = gen_label_rtx ();
21613 0 : rtx_insn *insn;
21614 :
21615 : /* e2 = sqrt (op1^2 + 1.0) + 1.0 */
21616 0 : emit_insn (gen_mulxf3 (e1, op1, op1));
21617 0 : cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
21618 0 : emit_insn (gen_addxf3 (e2, e1, cst1));
21619 0 : emit_insn (gen_sqrtxf2 (e2, e2));
21620 0 : emit_insn (gen_addxf3 (e2, e2, cst1));
21621 :
21622 : /* e1 = e1 / e2 */
21623 0 : emit_insn (gen_divxf3 (e1, e1, e2));
21624 :
21625 : /* scratch = fxam (op1) */
21626 0 : emit_insn (gen_fxamxf2_i387 (scratch, op1));
21627 :
21628 : /* e1 = e1 + |op1| */
21629 0 : emit_insn (gen_absxf2 (e2, op1));
21630 0 : emit_insn (gen_addxf3 (e1, e1, e2));
21631 :
21632 : /* e2 = log1p (e1) */
21633 0 : ix86_emit_i387_log1p (e2, e1);
21634 :
21635 : /* flags = signbit (op1) */
21636 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
21637 :
21638 : /* if (flags) then e2 = -e2 */
21639 0 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
21640 : gen_rtx_EQ (VOIDmode, flags, const0_rtx),
21641 : gen_rtx_LABEL_REF (VOIDmode, jump_label),
21642 : pc_rtx);
21643 0 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
21644 0 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
21645 0 : JUMP_LABEL (insn) = jump_label;
21646 :
21647 0 : emit_insn (gen_negxf2 (e2, e2));
21648 :
21649 0 : emit_label (jump_label);
21650 0 : LABEL_NUSES (jump_label) = 1;
21651 :
21652 0 : emit_move_insn (op0, e2);
21653 0 : }
21654 :
21655 : /* Output code to perform an acosh XFmode calculation. */
21656 :
21657 : void
21658 0 : ix86_emit_i387_acosh (rtx op0, rtx op1)
21659 : {
21660 0 : rtx e1 = gen_reg_rtx (XFmode);
21661 0 : rtx e2 = gen_reg_rtx (XFmode);
21662 0 : rtx cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
21663 :
21664 : /* e2 = sqrt (op1 + 1.0) */
21665 0 : emit_insn (gen_addxf3 (e2, op1, cst1));
21666 0 : emit_insn (gen_sqrtxf2 (e2, e2));
21667 :
21668 : /* e1 = sqrt (op1 - 1.0) */
21669 0 : emit_insn (gen_subxf3 (e1, op1, cst1));
21670 0 : emit_insn (gen_sqrtxf2 (e1, e1));
21671 :
21672 : /* e1 = e1 * e2 */
21673 0 : emit_insn (gen_mulxf3 (e1, e1, e2));
21674 :
21675 : /* e1 = e1 + op1 */
21676 0 : emit_insn (gen_addxf3 (e1, e1, op1));
21677 :
21678 : /* op0 = log (e1) */
21679 0 : emit_insn (gen_logxf2 (op0, e1));
21680 0 : }
21681 :
21682 : /* Output code to perform an atanh XFmode calculation. */
21683 :
21684 : void
21685 4 : ix86_emit_i387_atanh (rtx op0, rtx op1)
21686 : {
21687 4 : rtx e1 = gen_reg_rtx (XFmode);
21688 4 : rtx e2 = gen_reg_rtx (XFmode);
21689 4 : rtx scratch = gen_reg_rtx (HImode);
21690 4 : rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
21691 4 : rtx half = const_double_from_real_value (dconsthalf, XFmode);
21692 4 : rtx cst1, tmp;
21693 4 : rtx_code_label *jump_label = gen_label_rtx ();
21694 4 : rtx_insn *insn;
21695 :
21696 : /* scratch = fxam (op1) */
21697 4 : emit_insn (gen_fxamxf2_i387 (scratch, op1));
21698 :
21699 : /* e2 = |op1| */
21700 4 : emit_insn (gen_absxf2 (e2, op1));
21701 :
21702 : /* e1 = -(e2 + e2) / (e2 + 1.0) */
21703 4 : cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
21704 4 : emit_insn (gen_addxf3 (e1, e2, cst1));
21705 4 : emit_insn (gen_addxf3 (e2, e2, e2));
21706 4 : emit_insn (gen_negxf2 (e2, e2));
21707 4 : emit_insn (gen_divxf3 (e1, e2, e1));
21708 :
21709 : /* e2 = log1p (e1) */
21710 4 : ix86_emit_i387_log1p (e2, e1);
21711 :
21712 : /* flags = signbit (op1) */
21713 4 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
21714 :
21715 : /* if (!flags) then e2 = -e2 */
21716 4 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
21717 : gen_rtx_NE (VOIDmode, flags, const0_rtx),
21718 : gen_rtx_LABEL_REF (VOIDmode, jump_label),
21719 : pc_rtx);
21720 4 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
21721 4 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
21722 4 : JUMP_LABEL (insn) = jump_label;
21723 :
21724 4 : emit_insn (gen_negxf2 (e2, e2));
21725 :
21726 4 : emit_label (jump_label);
21727 4 : LABEL_NUSES (jump_label) = 1;
21728 :
21729 : /* op0 = 0.5 * e2 */
21730 4 : half = force_reg (XFmode, half);
21731 4 : emit_insn (gen_mulxf3 (op0, e2, half));
21732 4 : }
21733 :
21734 : /* Output code to perform a log1p XFmode calculation. */
21735 :
21736 : void
21737 5 : ix86_emit_i387_log1p (rtx op0, rtx op1)
21738 : {
21739 5 : rtx_code_label *label1 = gen_label_rtx ();
21740 5 : rtx_code_label *label2 = gen_label_rtx ();
21741 :
21742 5 : rtx tmp = gen_reg_rtx (XFmode);
21743 5 : rtx res = gen_reg_rtx (XFmode);
21744 5 : rtx cst, cstln2, cst1;
21745 5 : rtx_insn *insn;
21746 :
21747 : /* The emit_jump call emits pending stack adjust, make sure it is emitted
21748 : before the conditional jump, otherwise the stack adjustment will be
21749 : only conditional. */
21750 5 : do_pending_stack_adjust ();
21751 :
21752 5 : cst = const_double_from_real_value
21753 5 : (REAL_VALUE_ATOF ("0.29289321881345247561810596348408353", XFmode), XFmode);
21754 5 : cstln2 = force_reg (XFmode, standard_80387_constant_rtx (4)); /* fldln2 */
21755 :
21756 5 : emit_insn (gen_absxf2 (tmp, op1));
21757 :
21758 5 : cst = force_reg (XFmode, cst);
21759 5 : ix86_expand_branch (GE, tmp, cst, label1);
21760 5 : predict_jump (REG_BR_PROB_BASE * 10 / 100);
21761 5 : insn = get_last_insn ();
21762 5 : JUMP_LABEL (insn) = label1;
21763 :
21764 5 : emit_insn (gen_fyl2xp1xf3_i387 (res, op1, cstln2));
21765 5 : emit_jump (label2);
21766 :
21767 5 : emit_label (label1);
21768 5 : LABEL_NUSES (label1) = 1;
21769 :
21770 5 : cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
21771 5 : emit_insn (gen_rtx_SET (tmp, gen_rtx_PLUS (XFmode, op1, cst1)));
21772 5 : emit_insn (gen_fyl2xxf3_i387 (res, tmp, cstln2));
21773 :
21774 5 : emit_label (label2);
21775 5 : LABEL_NUSES (label2) = 1;
21776 :
21777 5 : emit_move_insn (op0, res);
21778 5 : }
21779 :
21780 : /* Emit code for round calculation. */
21781 : void
21782 60 : ix86_emit_i387_round (rtx op0, rtx op1)
21783 : {
21784 60 : machine_mode inmode = GET_MODE (op1);
21785 60 : machine_mode outmode = GET_MODE (op0);
21786 60 : rtx e1 = gen_reg_rtx (XFmode);
21787 60 : rtx e2 = gen_reg_rtx (XFmode);
21788 60 : rtx scratch = gen_reg_rtx (HImode);
21789 60 : rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
21790 60 : rtx half = const_double_from_real_value (dconsthalf, XFmode);
21791 60 : rtx res = gen_reg_rtx (outmode);
21792 60 : rtx_code_label *jump_label = gen_label_rtx ();
21793 60 : rtx (*floor_insn) (rtx, rtx);
21794 60 : rtx (*neg_insn) (rtx, rtx);
21795 60 : rtx_insn *insn;
21796 60 : rtx tmp;
21797 :
21798 60 : switch (inmode)
21799 : {
21800 29 : case E_SFmode:
21801 29 : case E_DFmode:
21802 29 : tmp = gen_reg_rtx (XFmode);
21803 :
21804 29 : emit_insn (gen_rtx_SET (tmp, gen_rtx_FLOAT_EXTEND (XFmode, op1)));
21805 29 : op1 = tmp;
21806 29 : break;
21807 : case E_XFmode:
21808 : break;
21809 0 : default:
21810 0 : gcc_unreachable ();
21811 : }
21812 :
21813 60 : switch (outmode)
21814 : {
21815 : case E_SFmode:
21816 : floor_insn = gen_frndintxf2_floor;
21817 : neg_insn = gen_negsf2;
21818 : break;
21819 6 : case E_DFmode:
21820 6 : floor_insn = gen_frndintxf2_floor;
21821 6 : neg_insn = gen_negdf2;
21822 6 : break;
21823 10 : case E_XFmode:
21824 10 : floor_insn = gen_frndintxf2_floor;
21825 10 : neg_insn = gen_negxf2;
21826 10 : break;
21827 0 : case E_HImode:
21828 0 : floor_insn = gen_lfloorxfhi2;
21829 0 : neg_insn = gen_neghi2;
21830 0 : break;
21831 6 : case E_SImode:
21832 6 : floor_insn = gen_lfloorxfsi2;
21833 6 : neg_insn = gen_negsi2;
21834 6 : break;
21835 36 : case E_DImode:
21836 36 : floor_insn = gen_lfloorxfdi2;
21837 36 : neg_insn = gen_negdi2;
21838 36 : break;
21839 0 : default:
21840 0 : gcc_unreachable ();
21841 : }
21842 :
21843 : /* round(a) = sgn(a) * floor(fabs(a) + 0.5) */
21844 :
21845 : /* scratch = fxam(op1) */
21846 60 : emit_insn (gen_fxamxf2_i387 (scratch, op1));
21847 :
21848 : /* e1 = fabs(op1) */
21849 60 : emit_insn (gen_absxf2 (e1, op1));
21850 :
21851 : /* e2 = e1 + 0.5 */
21852 60 : half = force_reg (XFmode, half);
21853 60 : emit_insn (gen_rtx_SET (e2, gen_rtx_PLUS (XFmode, e1, half)));
21854 :
21855 : /* res = floor(e2) */
21856 60 : switch (outmode)
21857 : {
21858 8 : case E_SFmode:
21859 8 : case E_DFmode:
21860 8 : {
21861 8 : tmp = gen_reg_rtx (XFmode);
21862 :
21863 8 : emit_insn (floor_insn (tmp, e2));
21864 8 : emit_insn (gen_rtx_SET (res,
21865 : gen_rtx_UNSPEC (outmode, gen_rtvec (1, tmp),
21866 : UNSPEC_TRUNC_NOOP)));
21867 : }
21868 8 : break;
21869 52 : default:
21870 52 : emit_insn (floor_insn (res, e2));
21871 : }
21872 :
21873 : /* flags = signbit(a) */
21874 60 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
21875 :
21876 : /* if (flags) then res = -res */
21877 60 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
21878 : gen_rtx_EQ (VOIDmode, flags, const0_rtx),
21879 : gen_rtx_LABEL_REF (VOIDmode, jump_label),
21880 : pc_rtx);
21881 60 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
21882 60 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
21883 60 : JUMP_LABEL (insn) = jump_label;
21884 :
21885 60 : emit_insn (neg_insn (res, res));
21886 :
21887 60 : emit_label (jump_label);
21888 60 : LABEL_NUSES (jump_label) = 1;
21889 :
21890 60 : emit_move_insn (op0, res);
21891 60 : }
21892 :
21893 : /* Output code to perform a Newton-Rhapson approximation of a single precision
21894 : floating point divide [http://en.wikipedia.org/wiki/N-th_root_algorithm]. */
21895 :
21896 : void
21897 56 : ix86_emit_swdivsf (rtx res, rtx a, rtx b, machine_mode mode)
21898 : {
21899 56 : rtx x0, x1, e0, e1;
21900 :
21901 56 : x0 = gen_reg_rtx (mode);
21902 56 : e0 = gen_reg_rtx (mode);
21903 56 : e1 = gen_reg_rtx (mode);
21904 56 : x1 = gen_reg_rtx (mode);
21905 :
21906 56 : b = force_reg (mode, b);
21907 :
21908 : /* x0 = rcp(b) estimate */
21909 56 : if (mode == V16SFmode || mode == V8DFmode)
21910 : {
21911 0 : emit_insn (gen_rtx_SET (x0, gen_rtx_UNSPEC (mode, gen_rtvec (1, b),
21912 : UNSPEC_RCP14)));
21913 : }
21914 : else
21915 56 : emit_insn (gen_rtx_SET (x0, gen_rtx_UNSPEC (mode, gen_rtvec (1, b),
21916 : UNSPEC_RCP)));
21917 :
21918 56 : unsigned vector_size = GET_MODE_SIZE (mode);
21919 :
21920 : /* (a - (rcp(b) * a * b)) * rcp(b) + rcp(b) * a
21921 : N-R step with 2 fma implementation. */
21922 56 : if (TARGET_FMA
21923 55 : || (TARGET_AVX512F && vector_size == 64)
21924 55 : || (TARGET_AVX512VL && (vector_size == 32 || vector_size == 16)))
21925 : {
21926 : /* e0 = x0 * a */
21927 1 : emit_insn (gen_rtx_SET (e0, gen_rtx_MULT (mode, x0, a)));
21928 : /* e1 = e0 * b - a */
21929 1 : emit_insn (gen_rtx_SET (e1, gen_rtx_FMA (mode, e0, b,
21930 : gen_rtx_NEG (mode, a))));
21931 : /* res = - e1 * x0 + e0 */
21932 1 : emit_insn (gen_rtx_SET (res, gen_rtx_FMA (mode,
21933 : gen_rtx_NEG (mode, e1),
21934 : x0, e0)));
21935 : }
21936 : else
21937 : /* a / b = a * ((rcp(b) + rcp(b)) - (b * rcp(b) * rcp (b))) */
21938 : {
21939 : /* e0 = x0 * b */
21940 55 : emit_insn (gen_rtx_SET (e0, gen_rtx_MULT (mode, x0, b)));
21941 :
21942 : /* e1 = x0 + x0 */
21943 55 : emit_insn (gen_rtx_SET (e1, gen_rtx_PLUS (mode, x0, x0)));
21944 :
21945 : /* e0 = x0 * e0 */
21946 55 : emit_insn (gen_rtx_SET (e0, gen_rtx_MULT (mode, x0, e0)));
21947 :
21948 : /* x1 = e1 - e0 */
21949 55 : emit_insn (gen_rtx_SET (x1, gen_rtx_MINUS (mode, e1, e0)));
21950 :
21951 : /* res = a * x1 */
21952 55 : emit_insn (gen_rtx_SET (res, gen_rtx_MULT (mode, a, x1)));
21953 : }
21954 56 : }
21955 :
21956 : /* Output code to perform a Newton-Rhapson approximation of a
21957 : single precision floating point [reciprocal] square root. */
21958 :
21959 : void
21960 85 : ix86_emit_swsqrtsf (rtx res, rtx a, machine_mode mode, bool recip)
21961 : {
21962 85 : rtx x0, e0, e1, e2, e3, mthree, mhalf;
21963 85 : REAL_VALUE_TYPE r;
21964 85 : int unspec;
21965 :
21966 85 : x0 = gen_reg_rtx (mode);
21967 85 : e0 = gen_reg_rtx (mode);
21968 85 : e1 = gen_reg_rtx (mode);
21969 85 : e2 = gen_reg_rtx (mode);
21970 85 : e3 = gen_reg_rtx (mode);
21971 :
21972 85 : real_from_integer (&r, VOIDmode, -3, SIGNED);
21973 85 : mthree = const_double_from_real_value (r, SFmode);
21974 :
21975 85 : real_arithmetic (&r, NEGATE_EXPR, &dconsthalf, NULL);
21976 85 : mhalf = const_double_from_real_value (r, SFmode);
21977 85 : unspec = UNSPEC_RSQRT;
21978 :
21979 85 : if (VECTOR_MODE_P (mode))
21980 : {
21981 66 : mthree = ix86_build_const_vector (mode, true, mthree);
21982 66 : mhalf = ix86_build_const_vector (mode, true, mhalf);
21983 : /* There is no 512-bit rsqrt. There is however rsqrt14. */
21984 132 : if (GET_MODE_SIZE (mode) == 64)
21985 0 : unspec = UNSPEC_RSQRT14;
21986 : }
21987 :
21988 : /* sqrt(a) = -0.5 * a * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0)
21989 : rsqrt(a) = -0.5 * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0) */
21990 :
21991 85 : a = force_reg (mode, a);
21992 :
21993 : /* x0 = rsqrt(a) estimate */
21994 85 : emit_insn (gen_rtx_SET (x0, gen_rtx_UNSPEC (mode, gen_rtvec (1, a),
21995 : unspec)));
21996 :
21997 : /* If (a == 0.0) Filter out infinity to prevent NaN for sqrt(0.0). */
21998 85 : if (!recip)
21999 : {
22000 57 : rtx zero = force_reg (mode, CONST0_RTX(mode));
22001 57 : rtx mask;
22002 :
22003 : /* Handle masked compare. */
22004 110 : if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 64)
22005 : {
22006 0 : mask = gen_reg_rtx (HImode);
22007 : /* Imm value 0x4 corresponds to not-equal comparison. */
22008 0 : emit_insn (gen_avx512f_cmpv16sf3 (mask, zero, a, GEN_INT (0x4)));
22009 0 : emit_insn (gen_avx512f_blendmv16sf (x0, zero, x0, mask));
22010 : }
22011 : else
22012 : {
22013 57 : mask = gen_reg_rtx (mode);
22014 57 : emit_insn (gen_rtx_SET (mask, gen_rtx_NE (mode, zero, a)));
22015 57 : emit_insn (gen_rtx_SET (x0, gen_rtx_AND (mode, x0, mask)));
22016 : }
22017 : }
22018 :
22019 85 : mthree = force_reg (mode, mthree);
22020 :
22021 : /* e0 = x0 * a */
22022 85 : emit_insn (gen_rtx_SET (e0, gen_rtx_MULT (mode, x0, a)));
22023 :
22024 85 : unsigned vector_size = GET_MODE_SIZE (mode);
22025 85 : if (TARGET_FMA
22026 77 : || (TARGET_AVX512F && vector_size == 64)
22027 77 : || (TARGET_AVX512VL && (vector_size == 32 || vector_size == 16)))
22028 16 : emit_insn (gen_rtx_SET (e2,
22029 : gen_rtx_FMA (mode, e0, x0, mthree)));
22030 : else
22031 : {
22032 : /* e1 = e0 * x0 */
22033 69 : emit_insn (gen_rtx_SET (e1, gen_rtx_MULT (mode, e0, x0)));
22034 :
22035 : /* e2 = e1 - 3. */
22036 69 : emit_insn (gen_rtx_SET (e2, gen_rtx_PLUS (mode, e1, mthree)));
22037 : }
22038 :
22039 85 : mhalf = force_reg (mode, mhalf);
22040 85 : if (recip)
22041 : /* e3 = -.5 * x0 */
22042 28 : emit_insn (gen_rtx_SET (e3, gen_rtx_MULT (mode, x0, mhalf)));
22043 : else
22044 : /* e3 = -.5 * e0 */
22045 57 : emit_insn (gen_rtx_SET (e3, gen_rtx_MULT (mode, e0, mhalf)));
22046 : /* ret = e2 * e3 */
22047 85 : emit_insn (gen_rtx_SET (res, gen_rtx_MULT (mode, e2, e3)));
22048 85 : }
22049 :
22050 : /* Expand fabs (OP0) and return a new rtx that holds the result. The
22051 : mask for masking out the sign-bit is stored in *SMASK, if that is
22052 : non-null. */
22053 :
22054 : static rtx
22055 1051 : ix86_expand_sse_fabs (rtx op0, rtx *smask)
22056 : {
22057 1051 : machine_mode vmode, mode = GET_MODE (op0);
22058 1051 : rtx xa, mask;
22059 :
22060 1051 : xa = gen_reg_rtx (mode);
22061 1051 : if (mode == SFmode)
22062 : vmode = V4SFmode;
22063 467 : else if (mode == DFmode)
22064 : vmode = V2DFmode;
22065 : else
22066 0 : vmode = mode;
22067 1051 : mask = ix86_build_signbit_mask (vmode, VECTOR_MODE_P (mode), true);
22068 1051 : if (!VECTOR_MODE_P (mode))
22069 : {
22070 : /* We need to generate a scalar mode mask in this case. */
22071 1051 : rtx tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
22072 1051 : tmp = gen_rtx_VEC_SELECT (mode, mask, tmp);
22073 1051 : mask = gen_reg_rtx (mode);
22074 1051 : emit_insn (gen_rtx_SET (mask, tmp));
22075 : }
22076 1051 : emit_insn (gen_rtx_SET (xa, gen_rtx_AND (mode, op0, mask)));
22077 :
22078 1051 : if (smask)
22079 998 : *smask = mask;
22080 :
22081 1051 : return xa;
22082 : }
22083 :
22084 : /* Expands a comparison of OP0 with OP1 using comparison code CODE,
22085 : swapping the operands if SWAP_OPERANDS is true. The expanded
22086 : code is a forward jump to a newly created label in case the
22087 : comparison is true. The generated label rtx is returned. */
22088 : static rtx_code_label *
22089 1067 : ix86_expand_sse_compare_and_jump (enum rtx_code code, rtx op0, rtx op1,
22090 : bool swap_operands)
22091 : {
22092 1067 : bool unordered_compare = ix86_unordered_fp_compare (code);
22093 1067 : rtx_code_label *label;
22094 1067 : rtx tmp, reg;
22095 :
22096 1067 : if (swap_operands)
22097 35 : std::swap (op0, op1);
22098 :
22099 1067 : label = gen_label_rtx ();
22100 1067 : tmp = gen_rtx_COMPARE (CCFPmode, op0, op1);
22101 1067 : if (unordered_compare)
22102 911 : tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_NOTRAP);
22103 1067 : reg = gen_rtx_REG (CCFPmode, FLAGS_REG);
22104 1067 : emit_insn (gen_rtx_SET (reg, tmp));
22105 1067 : tmp = gen_rtx_fmt_ee (code, VOIDmode, reg, const0_rtx);
22106 1067 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
22107 : gen_rtx_LABEL_REF (VOIDmode, label), pc_rtx);
22108 1067 : tmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
22109 1067 : JUMP_LABEL (tmp) = label;
22110 :
22111 1067 : return label;
22112 : }
22113 :
22114 : /* Expand a mask generating SSE comparison instruction comparing OP0 with OP1
22115 : using comparison code CODE. Operands are swapped for the comparison if
22116 : SWAP_OPERANDS is true. Returns a rtx for the generated mask. */
22117 : static rtx
22118 538 : ix86_expand_sse_compare_mask (enum rtx_code code, rtx op0, rtx op1,
22119 : bool swap_operands)
22120 : {
22121 538 : rtx (*insn)(rtx, rtx, rtx, rtx);
22122 538 : machine_mode mode = GET_MODE (op0);
22123 538 : rtx mask = gen_reg_rtx (mode);
22124 :
22125 538 : if (swap_operands)
22126 361 : std::swap (op0, op1);
22127 :
22128 538 : insn = mode == DFmode ? gen_setcc_df_sse : gen_setcc_sf_sse;
22129 :
22130 538 : emit_insn (insn (mask, op0, op1,
22131 : gen_rtx_fmt_ee (code, mode, op0, op1)));
22132 538 : return mask;
22133 : }
22134 :
22135 : /* Expand copysign from SIGN to the positive value ABS_VALUE
22136 : storing in RESULT. If MASK is non-null, it shall be a mask to mask out
22137 : the sign-bit. */
22138 :
22139 : static void
22140 1018 : ix86_sse_copysign_to_positive (rtx result, rtx abs_value, rtx sign, rtx mask)
22141 : {
22142 1018 : machine_mode mode = GET_MODE (sign);
22143 1018 : rtx sgn = gen_reg_rtx (mode);
22144 1018 : if (mask == NULL_RTX)
22145 : {
22146 28 : machine_mode vmode;
22147 :
22148 28 : if (mode == SFmode)
22149 : vmode = V4SFmode;
22150 : else if (mode == DFmode)
22151 : vmode = V2DFmode;
22152 : else if (mode == HFmode)
22153 : vmode = V8HFmode;
22154 : else
22155 28 : vmode = mode;
22156 :
22157 28 : mask = ix86_build_signbit_mask (vmode, VECTOR_MODE_P (mode), false);
22158 28 : if (!VECTOR_MODE_P (mode))
22159 : {
22160 : /* We need to generate a scalar mode mask in this case. */
22161 28 : rtx tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
22162 28 : tmp = gen_rtx_VEC_SELECT (mode, mask, tmp);
22163 28 : mask = gen_reg_rtx (mode);
22164 28 : emit_insn (gen_rtx_SET (mask, tmp));
22165 : }
22166 : }
22167 : else
22168 990 : mask = gen_rtx_NOT (mode, mask);
22169 1018 : emit_insn (gen_rtx_SET (sgn, gen_rtx_AND (mode, mask, sign)));
22170 1018 : emit_insn (gen_rtx_SET (result, gen_rtx_IOR (mode, abs_value, sgn)));
22171 1018 : }
22172 :
22173 : /* Expand SSE sequence for computing lround from OP1 storing
22174 : into OP0. */
22175 :
22176 : void
22177 28 : ix86_expand_lround (rtx op0, rtx op1)
22178 : {
22179 : /* C code for the stuff we're doing below:
22180 : tmp = op1 + copysign (nextafter (0.5, 0.0), op1)
22181 : return (long)tmp;
22182 : */
22183 28 : machine_mode mode = GET_MODE (op1);
22184 28 : const struct real_format *fmt;
22185 28 : REAL_VALUE_TYPE pred_half, half_minus_pred_half;
22186 28 : rtx adj;
22187 :
22188 : /* load nextafter (0.5, 0.0) */
22189 28 : fmt = REAL_MODE_FORMAT (mode);
22190 28 : real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
22191 28 : real_arithmetic (&pred_half, MINUS_EXPR, &dconsthalf, &half_minus_pred_half);
22192 :
22193 : /* adj = copysign (0.5, op1) */
22194 28 : adj = force_reg (mode, const_double_from_real_value (pred_half, mode));
22195 28 : ix86_sse_copysign_to_positive (adj, adj, force_reg (mode, op1), NULL_RTX);
22196 :
22197 : /* adj = op1 + adj */
22198 28 : adj = expand_simple_binop (mode, PLUS, adj, op1, NULL_RTX, 0, OPTAB_DIRECT);
22199 :
22200 : /* op0 = (imode)adj */
22201 28 : expand_fix (op0, adj, 0);
22202 28 : }
22203 :
22204 : /* Expand SSE2 sequence for computing lround from OPERAND1 storing
22205 : into OPERAND0. */
22206 :
22207 : void
22208 69 : ix86_expand_lfloorceil (rtx op0, rtx op1, bool do_floor)
22209 : {
22210 : /* C code for the stuff we're doing below (for do_floor):
22211 : xi = (long)op1;
22212 : xi -= (double)xi > op1 ? 1 : 0;
22213 : return xi;
22214 : */
22215 69 : machine_mode fmode = GET_MODE (op1);
22216 69 : machine_mode imode = GET_MODE (op0);
22217 69 : rtx ireg, freg, tmp;
22218 69 : rtx_code_label *label;
22219 :
22220 : /* reg = (long)op1 */
22221 69 : ireg = gen_reg_rtx (imode);
22222 69 : expand_fix (ireg, op1, 0);
22223 :
22224 : /* freg = (double)reg */
22225 69 : freg = gen_reg_rtx (fmode);
22226 69 : expand_float (freg, ireg, 0);
22227 :
22228 : /* ireg = (freg > op1) ? ireg - 1 : ireg */
22229 69 : label = ix86_expand_sse_compare_and_jump (UNLE,
22230 : freg, op1, !do_floor);
22231 104 : tmp = expand_simple_binop (imode, do_floor ? MINUS : PLUS,
22232 : ireg, const1_rtx, NULL_RTX, 0, OPTAB_DIRECT);
22233 69 : emit_move_insn (ireg, tmp);
22234 :
22235 69 : emit_label (label);
22236 69 : LABEL_NUSES (label) = 1;
22237 :
22238 69 : emit_move_insn (op0, ireg);
22239 69 : }
22240 :
22241 : /* Generate and return a rtx of mode MODE for 2**n where n is the number
22242 : of bits of the mantissa of MODE, which must be one of DFmode or SFmode. */
22243 :
22244 : static rtx
22245 998 : ix86_gen_TWO52 (machine_mode mode)
22246 : {
22247 998 : const struct real_format *fmt;
22248 998 : REAL_VALUE_TYPE TWO52r;
22249 998 : rtx TWO52;
22250 :
22251 998 : fmt = REAL_MODE_FORMAT (mode);
22252 998 : real_2expN (&TWO52r, fmt->p - 1, mode);
22253 998 : TWO52 = const_double_from_real_value (TWO52r, mode);
22254 998 : TWO52 = force_reg (mode, TWO52);
22255 :
22256 998 : return TWO52;
22257 : }
22258 :
22259 : /* Expand rint rounding OPERAND1 and storing the result in OPERAND0. */
22260 :
22261 : void
22262 121 : ix86_expand_rint (rtx operand0, rtx operand1)
22263 : {
22264 : /* C code for the stuff we're doing below:
22265 : xa = fabs (operand1);
22266 : if (!isless (xa, 2**52))
22267 : return operand1;
22268 : two52 = 2**52;
22269 : if (flag_rounding_math)
22270 : {
22271 : two52 = copysign (two52, operand1);
22272 : xa = operand1;
22273 : }
22274 : xa = xa + two52 - two52;
22275 : return copysign (xa, operand1);
22276 : */
22277 121 : machine_mode mode = GET_MODE (operand0);
22278 121 : rtx res, xa, TWO52, mask;
22279 121 : rtx_code_label *label;
22280 :
22281 121 : TWO52 = ix86_gen_TWO52 (mode);
22282 :
22283 : /* Temporary for holding the result, initialized to the input
22284 : operand to ease control flow. */
22285 121 : res = copy_to_reg (operand1);
22286 :
22287 : /* xa = abs (operand1) */
22288 121 : xa = ix86_expand_sse_fabs (res, &mask);
22289 :
22290 : /* if (!isless (xa, TWO52)) goto label; */
22291 121 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
22292 :
22293 121 : if (flag_rounding_math)
22294 : {
22295 53 : ix86_sse_copysign_to_positive (TWO52, TWO52, res, mask);
22296 53 : xa = res;
22297 : }
22298 :
22299 121 : xa = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
22300 121 : xa = expand_simple_binop (mode, MINUS, xa, TWO52, xa, 0, OPTAB_DIRECT);
22301 :
22302 : /* Remove the sign with FE_DOWNWARD, where x - x = -0.0. */
22303 121 : if (HONOR_SIGNED_ZEROS (mode) && flag_rounding_math)
22304 53 : xa = ix86_expand_sse_fabs (xa, NULL);
22305 :
22306 121 : ix86_sse_copysign_to_positive (res, xa, res, mask);
22307 :
22308 121 : emit_label (label);
22309 121 : LABEL_NUSES (label) = 1;
22310 :
22311 121 : emit_move_insn (operand0, res);
22312 121 : }
22313 :
22314 : /* Expand SSE2 sequence for computing floor or ceil
22315 : from OPERAND1 storing into OPERAND0. */
22316 : void
22317 538 : ix86_expand_floorceil (rtx operand0, rtx operand1, bool do_floor)
22318 : {
22319 : /* C code for the stuff we expand below.
22320 : double xa = fabs (x), x2;
22321 : if (!isless (xa, TWO52))
22322 : return x;
22323 : x2 = (double)(long)x;
22324 :
22325 : Compensate. Floor:
22326 : if (x2 > x)
22327 : x2 -= 1;
22328 : Compensate. Ceil:
22329 : if (x2 < x)
22330 : x2 += 1;
22331 :
22332 : if (HONOR_SIGNED_ZEROS (mode))
22333 : return copysign (x2, x);
22334 : return x2;
22335 : */
22336 538 : machine_mode mode = GET_MODE (operand0);
22337 538 : rtx xa, xi, TWO52, tmp, one, res, mask;
22338 538 : rtx_code_label *label;
22339 :
22340 538 : TWO52 = ix86_gen_TWO52 (mode);
22341 :
22342 : /* Temporary for holding the result, initialized to the input
22343 : operand to ease control flow. */
22344 538 : res = copy_to_reg (operand1);
22345 :
22346 : /* xa = abs (operand1) */
22347 538 : xa = ix86_expand_sse_fabs (res, &mask);
22348 :
22349 : /* if (!isless (xa, TWO52)) goto label; */
22350 538 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
22351 :
22352 : /* xa = (double)(long)x */
22353 538 : xi = gen_reg_rtx (int_mode_for_mode (mode).require ());
22354 538 : expand_fix (xi, res, 0);
22355 538 : expand_float (xa, xi, 0);
22356 :
22357 : /* generate 1.0 */
22358 538 : one = force_reg (mode, const_double_from_real_value (dconst1, mode));
22359 :
22360 : /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
22361 538 : tmp = ix86_expand_sse_compare_mask (UNGT, xa, res, !do_floor);
22362 538 : emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, one, tmp)));
22363 899 : tmp = expand_simple_binop (mode, do_floor ? MINUS : PLUS,
22364 : xa, tmp, NULL_RTX, 0, OPTAB_DIRECT);
22365 538 : if (HONOR_SIGNED_ZEROS (mode))
22366 : {
22367 : /* Remove the sign with FE_DOWNWARD, where x - x = -0.0. */
22368 491 : if (do_floor && flag_rounding_math)
22369 0 : tmp = ix86_expand_sse_fabs (tmp, NULL);
22370 :
22371 491 : ix86_sse_copysign_to_positive (tmp, tmp, res, mask);
22372 : }
22373 538 : emit_move_insn (res, tmp);
22374 :
22375 538 : emit_label (label);
22376 538 : LABEL_NUSES (label) = 1;
22377 :
22378 538 : emit_move_insn (operand0, res);
22379 538 : }
22380 :
22381 : /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
22382 : into OPERAND0 without relying on DImode truncation via cvttsd2siq
22383 : that is only available on 64bit targets. */
22384 : void
22385 0 : ix86_expand_floorceildf_32 (rtx operand0, rtx operand1, bool do_floor)
22386 : {
22387 : /* C code for the stuff we expand below.
22388 : double xa = fabs (x), x2;
22389 : if (!isless (xa, TWO52))
22390 : return x;
22391 : xa = xa + TWO52 - TWO52;
22392 : x2 = copysign (xa, x);
22393 :
22394 : Compensate. Floor:
22395 : if (x2 > x)
22396 : x2 -= 1;
22397 : Compensate. Ceil:
22398 : if (x2 < x)
22399 : x2 += 1;
22400 :
22401 : if (HONOR_SIGNED_ZEROS (mode))
22402 : x2 = copysign (x2, x);
22403 : return x2;
22404 : */
22405 0 : machine_mode mode = GET_MODE (operand0);
22406 0 : rtx xa, TWO52, tmp, one, res, mask;
22407 0 : rtx_code_label *label;
22408 :
22409 0 : TWO52 = ix86_gen_TWO52 (mode);
22410 :
22411 : /* Temporary for holding the result, initialized to the input
22412 : operand to ease control flow. */
22413 0 : res = copy_to_reg (operand1);
22414 :
22415 : /* xa = abs (operand1) */
22416 0 : xa = ix86_expand_sse_fabs (res, &mask);
22417 :
22418 : /* if (!isless (xa, TWO52)) goto label; */
22419 0 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
22420 :
22421 : /* xa = xa + TWO52 - TWO52; */
22422 0 : xa = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
22423 0 : xa = expand_simple_binop (mode, MINUS, xa, TWO52, xa, 0, OPTAB_DIRECT);
22424 :
22425 : /* xa = copysign (xa, operand1) */
22426 0 : ix86_sse_copysign_to_positive (xa, xa, res, mask);
22427 :
22428 : /* generate 1.0 */
22429 0 : one = force_reg (mode, const_double_from_real_value (dconst1, mode));
22430 :
22431 : /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
22432 0 : tmp = ix86_expand_sse_compare_mask (UNGT, xa, res, !do_floor);
22433 0 : emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, one, tmp)));
22434 0 : tmp = expand_simple_binop (mode, do_floor ? MINUS : PLUS,
22435 : xa, tmp, NULL_RTX, 0, OPTAB_DIRECT);
22436 0 : if (HONOR_SIGNED_ZEROS (mode))
22437 : {
22438 : /* Remove the sign with FE_DOWNWARD, where x - x = -0.0. */
22439 0 : if (do_floor && flag_rounding_math)
22440 0 : tmp = ix86_expand_sse_fabs (tmp, NULL);
22441 :
22442 0 : ix86_sse_copysign_to_positive (tmp, tmp, res, mask);
22443 : }
22444 0 : emit_move_insn (res, tmp);
22445 :
22446 0 : emit_label (label);
22447 0 : LABEL_NUSES (label) = 1;
22448 :
22449 0 : emit_move_insn (operand0, res);
22450 0 : }
22451 :
22452 : /* Expand SSE sequence for computing trunc
22453 : from OPERAND1 storing into OPERAND0. */
22454 : void
22455 325 : ix86_expand_trunc (rtx operand0, rtx operand1)
22456 : {
22457 : /* C code for SSE variant we expand below.
22458 : double xa = fabs (x), x2;
22459 : if (!isless (xa, TWO52))
22460 : return x;
22461 : x2 = (double)(long)x;
22462 : if (HONOR_SIGNED_ZEROS (mode))
22463 : return copysign (x2, x);
22464 : return x2;
22465 : */
22466 325 : machine_mode mode = GET_MODE (operand0);
22467 325 : rtx xa, xi, TWO52, res, mask;
22468 325 : rtx_code_label *label;
22469 :
22470 325 : TWO52 = ix86_gen_TWO52 (mode);
22471 :
22472 : /* Temporary for holding the result, initialized to the input
22473 : operand to ease control flow. */
22474 325 : res = copy_to_reg (operand1);
22475 :
22476 : /* xa = abs (operand1) */
22477 325 : xa = ix86_expand_sse_fabs (res, &mask);
22478 :
22479 : /* if (!isless (xa, TWO52)) goto label; */
22480 325 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
22481 :
22482 : /* xa = (double)(long)x */
22483 325 : xi = gen_reg_rtx (int_mode_for_mode (mode).require ());
22484 325 : expand_fix (xi, res, 0);
22485 325 : expand_float (xa, xi, 0);
22486 :
22487 325 : if (HONOR_SIGNED_ZEROS (mode))
22488 311 : ix86_sse_copysign_to_positive (xa, xa, res, mask);
22489 :
22490 325 : emit_move_insn (res, xa);
22491 :
22492 325 : emit_label (label);
22493 325 : LABEL_NUSES (label) = 1;
22494 :
22495 325 : emit_move_insn (operand0, res);
22496 325 : }
22497 :
22498 : /* Expand SSE sequence for computing trunc from OPERAND1 storing
22499 : into OPERAND0 without relying on DImode truncation via cvttsd2siq
22500 : that is only available on 64bit targets. */
22501 : void
22502 0 : ix86_expand_truncdf_32 (rtx operand0, rtx operand1)
22503 : {
22504 0 : machine_mode mode = GET_MODE (operand0);
22505 0 : rtx xa, xa2, TWO52, tmp, one, res, mask;
22506 0 : rtx_code_label *label;
22507 :
22508 : /* C code for SSE variant we expand below.
22509 : double xa = fabs (x), x2;
22510 : if (!isless (xa, TWO52))
22511 : return x;
22512 : xa2 = xa + TWO52 - TWO52;
22513 : Compensate:
22514 : if (xa2 > xa)
22515 : xa2 -= 1.0;
22516 : x2 = copysign (xa2, x);
22517 : return x2;
22518 : */
22519 :
22520 0 : TWO52 = ix86_gen_TWO52 (mode);
22521 :
22522 : /* Temporary for holding the result, initialized to the input
22523 : operand to ease control flow. */
22524 0 : res =copy_to_reg (operand1);
22525 :
22526 : /* xa = abs (operand1) */
22527 0 : xa = ix86_expand_sse_fabs (res, &mask);
22528 :
22529 : /* if (!isless (xa, TWO52)) goto label; */
22530 0 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
22531 :
22532 : /* xa2 = xa + TWO52 - TWO52; */
22533 0 : xa2 = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
22534 0 : xa2 = expand_simple_binop (mode, MINUS, xa2, TWO52, xa2, 0, OPTAB_DIRECT);
22535 :
22536 : /* generate 1.0 */
22537 0 : one = force_reg (mode, const_double_from_real_value (dconst1, mode));
22538 :
22539 : /* Compensate: xa2 = xa2 - (xa2 > xa ? 1 : 0) */
22540 0 : tmp = ix86_expand_sse_compare_mask (UNGT, xa2, xa, false);
22541 0 : emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, one, tmp)));
22542 0 : tmp = expand_simple_binop (mode, MINUS,
22543 : xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
22544 : /* Remove the sign with FE_DOWNWARD, where x - x = -0.0. */
22545 0 : if (HONOR_SIGNED_ZEROS (mode) && flag_rounding_math)
22546 0 : tmp = ix86_expand_sse_fabs (tmp, NULL);
22547 :
22548 : /* res = copysign (xa2, operand1) */
22549 0 : ix86_sse_copysign_to_positive (res, tmp, res, mask);
22550 :
22551 0 : emit_label (label);
22552 0 : LABEL_NUSES (label) = 1;
22553 :
22554 0 : emit_move_insn (operand0, res);
22555 0 : }
22556 :
22557 : /* Expand SSE sequence for computing round
22558 : from OPERAND1 storing into OPERAND0. */
22559 : void
22560 14 : ix86_expand_round (rtx operand0, rtx operand1)
22561 : {
22562 : /* C code for the stuff we're doing below:
22563 : double xa = fabs (x);
22564 : if (!isless (xa, TWO52))
22565 : return x;
22566 : xa = (double)(long)(xa + nextafter (0.5, 0.0));
22567 : return copysign (xa, x);
22568 : */
22569 14 : machine_mode mode = GET_MODE (operand0);
22570 14 : rtx res, TWO52, xa, xi, half, mask;
22571 14 : rtx_code_label *label;
22572 14 : const struct real_format *fmt;
22573 14 : REAL_VALUE_TYPE pred_half, half_minus_pred_half;
22574 :
22575 : /* Temporary for holding the result, initialized to the input
22576 : operand to ease control flow. */
22577 14 : res = copy_to_reg (operand1);
22578 :
22579 14 : TWO52 = ix86_gen_TWO52 (mode);
22580 14 : xa = ix86_expand_sse_fabs (res, &mask);
22581 14 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
22582 :
22583 : /* load nextafter (0.5, 0.0) */
22584 14 : fmt = REAL_MODE_FORMAT (mode);
22585 14 : real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
22586 14 : real_arithmetic (&pred_half, MINUS_EXPR, &dconsthalf, &half_minus_pred_half);
22587 :
22588 : /* xa = xa + 0.5 */
22589 14 : half = force_reg (mode, const_double_from_real_value (pred_half, mode));
22590 14 : xa = expand_simple_binop (mode, PLUS, xa, half, NULL_RTX, 0, OPTAB_DIRECT);
22591 :
22592 : /* xa = (double)(int64_t)xa */
22593 14 : xi = gen_reg_rtx (int_mode_for_mode (mode).require ());
22594 14 : expand_fix (xi, xa, 0);
22595 14 : expand_float (xa, xi, 0);
22596 :
22597 : /* res = copysign (xa, operand1) */
22598 14 : ix86_sse_copysign_to_positive (res, xa, res, mask);
22599 :
22600 14 : emit_label (label);
22601 14 : LABEL_NUSES (label) = 1;
22602 :
22603 14 : emit_move_insn (operand0, res);
22604 14 : }
22605 :
22606 : /* Expand SSE sequence for computing round from OPERAND1 storing
22607 : into OPERAND0 without relying on DImode truncation via cvttsd2siq
22608 : that is only available on 64bit targets. */
22609 : void
22610 0 : ix86_expand_rounddf_32 (rtx operand0, rtx operand1)
22611 : {
22612 : /* C code for the stuff we expand below.
22613 : double xa = fabs (x), xa2, x2;
22614 : if (!isless (xa, TWO52))
22615 : return x;
22616 : Using the absolute value and copying back sign makes
22617 : -0.0 -> -0.0 correct.
22618 : xa2 = xa + TWO52 - TWO52;
22619 : Compensate.
22620 : dxa = xa2 - xa;
22621 : if (dxa <= -0.5)
22622 : xa2 += 1;
22623 : else if (dxa > 0.5)
22624 : xa2 -= 1;
22625 : x2 = copysign (xa2, x);
22626 : return x2;
22627 : */
22628 0 : machine_mode mode = GET_MODE (operand0);
22629 0 : rtx xa, xa2, dxa, TWO52, tmp, half, mhalf, one, res, mask;
22630 0 : rtx_code_label *label;
22631 :
22632 0 : TWO52 = ix86_gen_TWO52 (mode);
22633 :
22634 : /* Temporary for holding the result, initialized to the input
22635 : operand to ease control flow. */
22636 0 : res = copy_to_reg (operand1);
22637 :
22638 : /* xa = abs (operand1) */
22639 0 : xa = ix86_expand_sse_fabs (res, &mask);
22640 :
22641 : /* if (!isless (xa, TWO52)) goto label; */
22642 0 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
22643 :
22644 : /* xa2 = xa + TWO52 - TWO52; */
22645 0 : xa2 = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
22646 0 : xa2 = expand_simple_binop (mode, MINUS, xa2, TWO52, xa2, 0, OPTAB_DIRECT);
22647 :
22648 : /* dxa = xa2 - xa; */
22649 0 : dxa = expand_simple_binop (mode, MINUS, xa2, xa, NULL_RTX, 0, OPTAB_DIRECT);
22650 :
22651 : /* generate 0.5, 1.0 and -0.5 */
22652 0 : half = force_reg (mode, const_double_from_real_value (dconsthalf, mode));
22653 0 : one = expand_simple_binop (mode, PLUS, half, half, NULL_RTX, 0, OPTAB_DIRECT);
22654 0 : mhalf = expand_simple_binop (mode, MINUS, half, one, NULL_RTX,
22655 : 0, OPTAB_DIRECT);
22656 :
22657 : /* Compensate. */
22658 : /* xa2 = xa2 - (dxa > 0.5 ? 1 : 0) */
22659 0 : tmp = ix86_expand_sse_compare_mask (UNGT, dxa, half, false);
22660 0 : emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, tmp, one)));
22661 0 : xa2 = expand_simple_binop (mode, MINUS, xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
22662 : /* xa2 = xa2 + (dxa <= -0.5 ? 1 : 0) */
22663 0 : tmp = ix86_expand_sse_compare_mask (UNGE, mhalf, dxa, false);
22664 0 : emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, tmp, one)));
22665 0 : xa2 = expand_simple_binop (mode, PLUS, xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
22666 :
22667 : /* res = copysign (xa2, operand1) */
22668 0 : ix86_sse_copysign_to_positive (res, xa2, res, mask);
22669 :
22670 0 : emit_label (label);
22671 0 : LABEL_NUSES (label) = 1;
22672 :
22673 0 : emit_move_insn (operand0, res);
22674 0 : }
22675 :
22676 : /* Expand SSE sequence for computing round
22677 : from OP1 storing into OP0 using sse4 round insn. */
22678 : void
22679 9 : ix86_expand_round_sse4 (rtx op0, rtx op1)
22680 : {
22681 9 : machine_mode mode = GET_MODE (op0);
22682 9 : rtx e1, e2, res, half;
22683 9 : const struct real_format *fmt;
22684 9 : REAL_VALUE_TYPE pred_half, half_minus_pred_half;
22685 9 : rtx (*gen_copysign) (rtx, rtx, rtx);
22686 9 : rtx (*gen_round) (rtx, rtx, rtx);
22687 :
22688 9 : switch (mode)
22689 : {
22690 : case E_HFmode:
22691 : gen_copysign = gen_copysignhf3;
22692 : gen_round = gen_sse4_1_roundhf2;
22693 : break;
22694 4 : case E_SFmode:
22695 4 : gen_copysign = gen_copysignsf3;
22696 4 : gen_round = gen_sse4_1_roundsf2;
22697 4 : break;
22698 4 : case E_DFmode:
22699 4 : gen_copysign = gen_copysigndf3;
22700 4 : gen_round = gen_sse4_1_rounddf2;
22701 4 : break;
22702 0 : default:
22703 0 : gcc_unreachable ();
22704 : }
22705 :
22706 : /* round (a) = trunc (a + copysign (0.5, a)) */
22707 :
22708 : /* load nextafter (0.5, 0.0) */
22709 9 : fmt = REAL_MODE_FORMAT (mode);
22710 9 : real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
22711 9 : real_arithmetic (&pred_half, MINUS_EXPR, &dconsthalf, &half_minus_pred_half);
22712 9 : half = const_double_from_real_value (pred_half, mode);
22713 :
22714 : /* e1 = copysign (0.5, op1) */
22715 9 : e1 = gen_reg_rtx (mode);
22716 9 : emit_insn (gen_copysign (e1, half, op1));
22717 :
22718 : /* e2 = op1 + e1 */
22719 9 : e2 = expand_simple_binop (mode, PLUS, op1, e1, NULL_RTX, 0, OPTAB_DIRECT);
22720 :
22721 : /* res = trunc (e2) */
22722 9 : res = gen_reg_rtx (mode);
22723 9 : emit_insn (gen_round (res, e2, GEN_INT (ROUND_TRUNC)));
22724 :
22725 9 : emit_move_insn (op0, res);
22726 9 : }
22727 :
22728 : /* A cached (set (nil) (vselect (vconcat (nil) (nil)) (parallel [])))
22729 : insn, so that expand_vselect{,_vconcat} doesn't have to create a fresh
22730 : insn every time. */
22731 :
22732 : static GTY(()) rtx_insn *vselect_insn;
22733 :
22734 : /* Initialize vselect_insn. */
22735 :
22736 : static void
22737 7810 : init_vselect_insn (void)
22738 : {
22739 7810 : unsigned i;
22740 7810 : rtx x;
22741 :
22742 7810 : x = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (MAX_VECT_LEN));
22743 507650 : for (i = 0; i < MAX_VECT_LEN; ++i)
22744 499840 : XVECEXP (x, 0, i) = const0_rtx;
22745 7810 : x = gen_rtx_VEC_SELECT (V2DFmode, gen_rtx_VEC_CONCAT (V4DFmode, const0_rtx,
22746 : const0_rtx), x);
22747 7810 : x = gen_rtx_SET (const0_rtx, x);
22748 7810 : start_sequence ();
22749 7810 : vselect_insn = emit_insn (x);
22750 7810 : end_sequence ();
22751 7810 : }
22752 :
22753 : /* Construct (set target (vec_select op0 (parallel perm))) and
22754 : return true if that's a valid instruction in the active ISA. */
22755 :
22756 : static bool
22757 547734 : expand_vselect (rtx target, rtx op0, const unsigned char *perm,
22758 : unsigned nelt, bool testing_p)
22759 : {
22760 547734 : unsigned int i;
22761 547734 : rtx x, save_vconcat;
22762 547734 : int icode;
22763 :
22764 547734 : if (vselect_insn == NULL_RTX)
22765 1716 : init_vselect_insn ();
22766 :
22767 547734 : x = XEXP (SET_SRC (PATTERN (vselect_insn)), 1);
22768 547734 : PUT_NUM_ELEM (XVEC (x, 0), nelt);
22769 4188260 : for (i = 0; i < nelt; ++i)
22770 3640526 : XVECEXP (x, 0, i) = GEN_INT (perm[i]);
22771 547734 : save_vconcat = XEXP (SET_SRC (PATTERN (vselect_insn)), 0);
22772 547734 : XEXP (SET_SRC (PATTERN (vselect_insn)), 0) = op0;
22773 547734 : PUT_MODE (SET_SRC (PATTERN (vselect_insn)), GET_MODE (target));
22774 547734 : SET_DEST (PATTERN (vselect_insn)) = target;
22775 547734 : icode = recog_memoized (vselect_insn);
22776 :
22777 547734 : if (icode >= 0 && !testing_p)
22778 73314 : emit_insn (copy_rtx (PATTERN (vselect_insn)));
22779 :
22780 547734 : SET_DEST (PATTERN (vselect_insn)) = const0_rtx;
22781 547734 : XEXP (SET_SRC (PATTERN (vselect_insn)), 0) = save_vconcat;
22782 547734 : INSN_CODE (vselect_insn) = -1;
22783 :
22784 547734 : return icode >= 0;
22785 : }
22786 :
22787 : /* Similar, but generate a vec_concat from op0 and op1 as well. */
22788 :
22789 : static bool
22790 481517 : expand_vselect_vconcat (rtx target, rtx op0, rtx op1,
22791 : const unsigned char *perm, unsigned nelt,
22792 : bool testing_p)
22793 : {
22794 481517 : machine_mode v2mode;
22795 481517 : rtx x;
22796 481517 : bool ok;
22797 :
22798 481517 : if (vselect_insn == NULL_RTX)
22799 6094 : init_vselect_insn ();
22800 :
22801 481517 : if (!GET_MODE_2XWIDER_MODE (GET_MODE (op0)).exists (&v2mode))
22802 : return false;
22803 481517 : x = XEXP (SET_SRC (PATTERN (vselect_insn)), 0);
22804 481517 : PUT_MODE (x, v2mode);
22805 481517 : XEXP (x, 0) = op0;
22806 481517 : XEXP (x, 1) = op1;
22807 481517 : ok = expand_vselect (target, x, perm, nelt, testing_p);
22808 481517 : XEXP (x, 0) = const0_rtx;
22809 481517 : XEXP (x, 1) = const0_rtx;
22810 481517 : return ok;
22811 : }
22812 :
22813 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
22814 : using movss or movsd. */
22815 : static bool
22816 305205 : expand_vec_perm_movs (struct expand_vec_perm_d *d)
22817 : {
22818 305205 : machine_mode vmode = d->vmode;
22819 305205 : unsigned i, nelt = d->nelt;
22820 305205 : rtx x;
22821 :
22822 305205 : if (d->one_operand_p)
22823 : return false;
22824 :
22825 279242 : if (!(TARGET_SSE && (vmode == V4SFmode || vmode == V4SImode))
22826 128007 : && !(TARGET_MMX_WITH_SSE && (vmode == V2SFmode || vmode == V2SImode))
22827 86900 : && !(TARGET_SSE2 && (vmode == V2DFmode || vmode == V2DImode)))
22828 : return false;
22829 :
22830 : /* Only the first element is changed. */
22831 201835 : if (d->perm[0] != nelt && d->perm[0] != 0)
22832 : return false;
22833 122971 : for (i = 1; i < nelt; ++i)
22834 108348 : if (d->perm[i] != i + nelt - d->perm[0])
22835 : return false;
22836 :
22837 14623 : if (d->testing_p)
22838 : return true;
22839 :
22840 6839 : if (d->perm[0] == nelt)
22841 0 : x = gen_rtx_VEC_MERGE (vmode, d->op1, d->op0, GEN_INT (1));
22842 : else
22843 6839 : x = gen_rtx_VEC_MERGE (vmode, d->op0, d->op1, GEN_INT (1));
22844 :
22845 6839 : emit_insn (gen_rtx_SET (d->target, x));
22846 :
22847 6839 : return true;
22848 : }
22849 :
22850 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
22851 : using insertps. */
22852 : static bool
22853 290582 : expand_vec_perm_insertps (struct expand_vec_perm_d *d)
22854 : {
22855 290582 : machine_mode vmode = d->vmode;
22856 290582 : unsigned i, cnt_s, nelt = d->nelt;
22857 290582 : int cnt_d = -1;
22858 290582 : rtx src, dst;
22859 :
22860 290582 : if (d->one_operand_p)
22861 : return false;
22862 :
22863 264619 : if (!(TARGET_SSE4_1
22864 38176 : && (vmode == V4SFmode || vmode == V4SImode
22865 27938 : || (TARGET_MMX_WITH_SSE
22866 21558 : && (vmode == V2SFmode || vmode == V2SImode)))))
22867 : return false;
22868 :
22869 56981 : for (i = 0; i < nelt; ++i)
22870 : {
22871 53631 : if (d->perm[i] == i)
22872 10859 : continue;
22873 42772 : if (cnt_d != -1)
22874 : {
22875 : cnt_d = -1;
22876 : break;
22877 : }
22878 23061 : cnt_d = i;
22879 : }
22880 :
22881 23061 : if (cnt_d == -1)
22882 : {
22883 44605 : for (i = 0; i < nelt; ++i)
22884 : {
22885 41665 : if (d->perm[i] == i + nelt)
22886 5183 : continue;
22887 36482 : if (cnt_d != -1)
22888 : return false;
22889 19711 : cnt_d = i;
22890 : }
22891 :
22892 2940 : if (cnt_d == -1)
22893 : return false;
22894 : }
22895 :
22896 6290 : if (d->testing_p)
22897 : return true;
22898 :
22899 570 : gcc_assert (cnt_d != -1);
22900 :
22901 570 : cnt_s = d->perm[cnt_d];
22902 570 : if (cnt_s < nelt)
22903 : {
22904 246 : src = d->op0;
22905 246 : dst = d->op1;
22906 : }
22907 : else
22908 : {
22909 324 : cnt_s -= nelt;
22910 324 : src = d->op1;
22911 324 : dst = d->op0;
22912 : }
22913 570 : gcc_assert (cnt_s < nelt);
22914 :
22915 570 : rtx x = gen_sse4_1_insertps (vmode, d->target, dst, src,
22916 570 : GEN_INT (cnt_s << 6 | cnt_d << 4));
22917 570 : emit_insn (x);
22918 :
22919 570 : return true;
22920 : }
22921 :
22922 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
22923 : in terms of blendp[sd] / pblendw / pblendvb / vpblendd. */
22924 :
22925 : static bool
22926 309792 : expand_vec_perm_blend (struct expand_vec_perm_d *d)
22927 : {
22928 309792 : machine_mode mmode, vmode = d->vmode;
22929 309792 : unsigned i, nelt = d->nelt;
22930 309792 : unsigned HOST_WIDE_INT mask;
22931 309792 : rtx target, op0, op1, maskop, x;
22932 309792 : rtx rperm[32], vperm;
22933 :
22934 309792 : if (d->one_operand_p)
22935 : return false;
22936 6831 : if (TARGET_AVX512F && GET_MODE_SIZE (vmode) == 64
22937 284809 : && (TARGET_AVX512BW
22938 655 : || GET_MODE_UNIT_SIZE (vmode) >= 4))
22939 : ;
22940 296010 : else if (TARGET_AVX2 && GET_MODE_SIZE (vmode) == 32)
22941 : ;
22942 277451 : else if (TARGET_AVX && (vmode == V4DFmode || vmode == V8SFmode))
22943 : ;
22944 272236 : else if (TARGET_SSE4_1
22945 308698 : && (GET_MODE_SIZE (vmode) == 16
22946 30172 : || (TARGET_MMX_WITH_SSE && GET_MODE_SIZE (vmode) == 8)
22947 3700 : || GET_MODE_SIZE (vmode) == 4))
22948 : ;
22949 : else
22950 : return false;
22951 :
22952 : /* This is a blend, not a permute. Elements must stay in their
22953 : respective lanes. */
22954 97766 : for (i = 0; i < nelt; ++i)
22955 : {
22956 93179 : unsigned e = d->perm[i];
22957 93179 : if (!(e == i || e == i + nelt))
22958 : return false;
22959 : }
22960 :
22961 4587 : if (d->testing_p)
22962 : return true;
22963 :
22964 : /* ??? Without SSE4.1, we could implement this with and/andn/or. This
22965 : decision should be extracted elsewhere, so that we only try that
22966 : sequence once all budget==3 options have been tried. */
22967 2745 : target = d->target;
22968 2745 : op0 = d->op0;
22969 2745 : op1 = d->op1;
22970 2745 : mask = 0;
22971 :
22972 2745 : switch (vmode)
22973 : {
22974 : case E_V8DFmode:
22975 : case E_V16SFmode:
22976 : case E_V4DFmode:
22977 : case E_V8SFmode:
22978 : case E_V2DFmode:
22979 : case E_V4SFmode:
22980 : case E_V2SFmode:
22981 : case E_V2HImode:
22982 : case E_V4HImode:
22983 : case E_V8HImode:
22984 : case E_V8SImode:
22985 : case E_V32HImode:
22986 : case E_V64QImode:
22987 : case E_V16SImode:
22988 : case E_V8DImode:
22989 10578 : for (i = 0; i < nelt; ++i)
22990 9126 : mask |= ((unsigned HOST_WIDE_INT) (d->perm[i] >= nelt)) << i;
22991 : break;
22992 :
22993 : case E_V2DImode:
22994 18 : for (i = 0; i < 2; ++i)
22995 18 : mask |= (d->perm[i] >= 2 ? 15 : 0) << (i * 4);
22996 6 : vmode = V8HImode;
22997 6 : goto do_subreg;
22998 :
22999 : case E_V2SImode:
23000 24 : for (i = 0; i < 2; ++i)
23001 24 : mask |= (d->perm[i] >= 2 ? 3 : 0) << (i * 2);
23002 8 : vmode = V4HImode;
23003 8 : goto do_subreg;
23004 :
23005 872 : case E_V4SImode:
23006 872 : if (TARGET_AVX2)
23007 : {
23008 : /* Use vpblendd instead of vpblendw. */
23009 190 : for (i = 0; i < nelt; ++i)
23010 152 : mask |= ((unsigned HOST_WIDE_INT) (d->perm[i] >= nelt)) << i;
23011 : break;
23012 : }
23013 : else
23014 : {
23015 4170 : for (i = 0; i < 4; ++i)
23016 5200 : mask |= (d->perm[i] >= 4 ? 3 : 0) << (i * 2);
23017 834 : vmode = V8HImode;
23018 834 : goto do_subreg;
23019 : }
23020 :
23021 : case E_V16QImode:
23022 : /* See if bytes move in pairs so we can use pblendw with
23023 : an immediate argument, rather than pblendvb with a vector
23024 : argument. */
23025 102 : for (i = 0; i < 16; i += 2)
23026 100 : if (d->perm[i] + 1 != d->perm[i + 1])
23027 : {
23028 290 : use_pblendvb:
23029 3502 : for (i = 0; i < nelt; ++i)
23030 3212 : rperm[i] = (d->perm[i] < nelt ? const0_rtx : constm1_rtx);
23031 :
23032 290 : finish_pblendvb:
23033 291 : vperm = gen_rtx_CONST_VECTOR (vmode, gen_rtvec_v (nelt, rperm));
23034 291 : vperm = force_reg (vmode, vperm);
23035 :
23036 582 : if (GET_MODE_SIZE (vmode) == 4)
23037 135 : emit_insn (gen_mmx_pblendvb_v4qi (target, op0, op1, vperm));
23038 312 : else if (GET_MODE_SIZE (vmode) == 8)
23039 40 : emit_insn (gen_mmx_pblendvb_v8qi (target, op0, op1, vperm));
23040 232 : else if (GET_MODE_SIZE (vmode) == 16)
23041 83 : emit_insn (gen_sse4_1_pblendvb (target, op0, op1, vperm));
23042 : else
23043 33 : emit_insn (gen_avx2_pblendvb (target, op0, op1, vperm));
23044 291 : if (target != d->target)
23045 1 : emit_move_insn (d->target, gen_lowpart (d->vmode, target));
23046 : return true;
23047 : }
23048 :
23049 18 : for (i = 0; i < 8; ++i)
23050 16 : mask |= (d->perm[i * 2] >= 16) << i;
23051 : vmode = V8HImode;
23052 : /* FALLTHRU */
23053 :
23054 932 : do_subreg:
23055 932 : target = gen_reg_rtx (vmode);
23056 932 : op0 = gen_lowpart (vmode, op0);
23057 932 : op1 = gen_lowpart (vmode, op1);
23058 932 : break;
23059 :
23060 : case E_V8QImode:
23061 40 : for (i = 0; i < 8; i += 2)
23062 40 : if (d->perm[i] + 1 != d->perm[i + 1])
23063 40 : goto use_pblendvb;
23064 :
23065 0 : for (i = 0; i < 4; ++i)
23066 0 : mask |= (d->perm[i * 2] >= 8) << i;
23067 0 : vmode = V4HImode;
23068 0 : goto do_subreg;
23069 :
23070 : case E_V4QImode:
23071 153 : for (i = 0; i < 4; i += 2)
23072 150 : if (d->perm[i] + 1 != d->perm[i + 1])
23073 135 : goto use_pblendvb;
23074 :
23075 9 : for (i = 0; i < 2; ++i)
23076 6 : mask |= (d->perm[i * 2] >= 4) << i;
23077 3 : vmode = V2HImode;
23078 3 : goto do_subreg;
23079 :
23080 : case E_V32QImode:
23081 : /* See if bytes move in pairs. If not, vpblendvb must be used. */
23082 916 : for (i = 0; i < 32; i += 2)
23083 864 : if (d->perm[i] + 1 != d->perm[i + 1])
23084 32 : goto use_pblendvb;
23085 : /* See if bytes move in quadruplets. If yes, vpblendd
23086 : with immediate can be used. */
23087 468 : for (i = 0; i < 32; i += 4)
23088 416 : if (d->perm[i] + 2 != d->perm[i + 2])
23089 : break;
23090 52 : if (i < 32)
23091 : {
23092 : /* See if bytes move the same in both lanes. If yes,
23093 : vpblendw with immediate can be used. */
23094 0 : for (i = 0; i < 16; i += 2)
23095 0 : if (d->perm[i] + 16 != d->perm[i + 16])
23096 0 : goto use_pblendvb;
23097 :
23098 : /* Use vpblendw. */
23099 0 : for (i = 0; i < 16; ++i)
23100 0 : mask |= (d->perm[i * 2] >= 32) << i;
23101 0 : vmode = V16HImode;
23102 0 : goto do_subreg;
23103 : }
23104 :
23105 : /* Use vpblendd. */
23106 468 : for (i = 0; i < 8; ++i)
23107 416 : mask |= (d->perm[i * 4] >= 32) << i;
23108 52 : vmode = V8SImode;
23109 52 : goto do_subreg;
23110 :
23111 : case E_V16HImode:
23112 : /* See if words move in pairs. If yes, vpblendd can be used. */
23113 186 : for (i = 0; i < 16; i += 2)
23114 169 : if (d->perm[i] + 1 != d->perm[i + 1])
23115 : break;
23116 50 : if (i < 16)
23117 : {
23118 : /* See if words move the same in both lanes. If not,
23119 : vpblendvb must be used. */
23120 290 : for (i = 0; i < 8; i++)
23121 258 : if (d->perm[i] + 8 != d->perm[i + 8])
23122 : {
23123 : /* Use vpblendvb. */
23124 33 : for (i = 0; i < 32; ++i)
23125 32 : rperm[i] = (d->perm[i / 2] < 16 ? const0_rtx : constm1_rtx);
23126 :
23127 1 : vmode = V32QImode;
23128 1 : nelt = 32;
23129 1 : target = gen_reg_rtx (vmode);
23130 1 : op0 = gen_lowpart (vmode, op0);
23131 1 : op1 = gen_lowpart (vmode, op1);
23132 1 : goto finish_pblendvb;
23133 : }
23134 :
23135 : /* Use vpblendw. */
23136 544 : for (i = 0; i < 16; ++i)
23137 512 : mask |= (d->perm[i] >= 16) << i;
23138 : break;
23139 : }
23140 :
23141 : /* Use vpblendd. */
23142 153 : for (i = 0; i < 8; ++i)
23143 136 : mask |= (d->perm[i * 2] >= 16) << i;
23144 17 : vmode = V8SImode;
23145 17 : goto do_subreg;
23146 :
23147 : case E_V4DImode:
23148 : /* Use vpblendd. */
23149 50 : for (i = 0; i < 4; ++i)
23150 59 : mask |= (d->perm[i] >= 4 ? 3 : 0) << (i * 2);
23151 10 : vmode = V8SImode;
23152 10 : goto do_subreg;
23153 :
23154 0 : default:
23155 0 : gcc_unreachable ();
23156 : }
23157 :
23158 2454 : switch (vmode)
23159 : {
23160 : case E_V8DFmode:
23161 : case E_V8DImode:
23162 : mmode = QImode;
23163 : break;
23164 1 : case E_V16SFmode:
23165 1 : case E_V16SImode:
23166 1 : mmode = HImode;
23167 1 : break;
23168 6 : case E_V32HImode:
23169 6 : mmode = SImode;
23170 6 : break;
23171 1 : case E_V64QImode:
23172 1 : mmode = DImode;
23173 1 : break;
23174 : default:
23175 : mmode = VOIDmode;
23176 : }
23177 :
23178 : /* Canonicalize vec_merge. */
23179 2454 : if (swap_commutative_operands_p (op1, op0)
23180 : /* Two operands have same precedence, then
23181 : first bit of mask select first operand. */
23182 2454 : || (!swap_commutative_operands_p (op0, op1)
23183 2454 : && !(mask & 1)))
23184 : {
23185 2447 : unsigned n_elts = GET_MODE_NUNITS (vmode);
23186 2447 : std::swap (op0, op1);
23187 2447 : unsigned HOST_WIDE_INT mask_all = HOST_WIDE_INT_1U;
23188 2447 : if (n_elts == HOST_BITS_PER_WIDE_INT)
23189 : mask_all = -1;
23190 : else
23191 2446 : mask_all = (HOST_WIDE_INT_1U << n_elts) - 1;
23192 2447 : mask = ~mask & mask_all;
23193 : }
23194 :
23195 2454 : if (mmode != VOIDmode)
23196 14 : maskop = force_reg (mmode, gen_int_mode (mask, mmode));
23197 : else
23198 2440 : maskop = GEN_INT (mask);
23199 :
23200 : /* This matches five different patterns with the different modes. */
23201 2454 : x = gen_rtx_VEC_MERGE (vmode, op1, op0, maskop);
23202 2454 : x = gen_rtx_SET (target, x);
23203 2454 : emit_insn (x);
23204 2454 : if (target != d->target)
23205 932 : emit_move_insn (d->target, gen_lowpart (d->vmode, target));
23206 :
23207 : return true;
23208 : }
23209 :
23210 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
23211 : in terms of the variable form of vpermilps.
23212 :
23213 : Note that we will have already failed the immediate input vpermilps,
23214 : which requires that the high and low part shuffle be identical; the
23215 : variable form doesn't require that. */
23216 :
23217 : static bool
23218 139323 : expand_vec_perm_vpermil (struct expand_vec_perm_d *d)
23219 : {
23220 139323 : rtx rperm[8], vperm;
23221 139323 : unsigned i;
23222 :
23223 139323 : if (!TARGET_AVX || !d->one_operand_p
23224 11100 : || (d->vmode != V8SImode && d->vmode != V8SFmode))
23225 : return false;
23226 :
23227 : /* We can only permute within the 128-bit lane. */
23228 15948 : for (i = 0; i < 8; ++i)
23229 : {
23230 15312 : unsigned e = d->perm[i];
23231 15312 : if (i < 4 ? e >= 4 : e < 4)
23232 : return false;
23233 : }
23234 :
23235 636 : if (d->testing_p)
23236 : return true;
23237 :
23238 531 : for (i = 0; i < 8; ++i)
23239 : {
23240 472 : unsigned e = d->perm[i];
23241 :
23242 : /* Within each 128-bit lane, the elements of op0 are numbered
23243 : from 0 and the elements of op1 are numbered from 4. */
23244 472 : if (e >= 8 + 4)
23245 0 : e -= 8;
23246 472 : else if (e >= 4)
23247 236 : e -= 4;
23248 :
23249 472 : rperm[i] = GEN_INT (e);
23250 : }
23251 :
23252 59 : vperm = gen_rtx_CONST_VECTOR (V8SImode, gen_rtvec_v (8, rperm));
23253 59 : vperm = force_reg (V8SImode, vperm);
23254 59 : rtx target = d->target;
23255 59 : rtx op0 = d->op0;
23256 59 : if (d->vmode == V8SImode)
23257 : {
23258 7 : target = lowpart_subreg (V8SFmode, target, V8SImode);
23259 7 : op0 = lowpart_subreg (V8SFmode, op0, V8SImode);
23260 : }
23261 :
23262 59 : emit_insn (gen_avx_vpermilvarv8sf3 (target, op0, vperm));
23263 :
23264 59 : return true;
23265 : }
23266 :
23267 : /* For V*[QHS]Imode permutations, check if the same permutation
23268 : can't be performed in a 2x, 4x or 8x wider inner mode. */
23269 :
23270 : static bool
23271 164116 : canonicalize_vector_int_perm (const struct expand_vec_perm_d *d,
23272 : struct expand_vec_perm_d *nd)
23273 : {
23274 164116 : int i;
23275 164116 : machine_mode mode = VOIDmode;
23276 :
23277 164116 : switch (d->vmode)
23278 : {
23279 : case E_V8QImode: mode = V4HImode; break;
23280 30498 : case E_V16QImode: mode = V8HImode; break;
23281 707 : case E_V32QImode: mode = V16HImode; break;
23282 267 : case E_V64QImode: mode = V32HImode; break;
23283 12282 : case E_V4HImode: mode = V2SImode; break;
23284 20994 : case E_V8HImode: mode = V4SImode; break;
23285 1006 : case E_V16HImode: mode = V8SImode; break;
23286 397 : case E_V32HImode: mode = V16SImode; break;
23287 41452 : case E_V4SImode: mode = V2DImode; break;
23288 1429 : case E_V8SImode: mode = V4DImode; break;
23289 65 : case E_V16SImode: mode = V8DImode; break;
23290 : default: return false;
23291 : }
23292 208680 : for (i = 0; i < d->nelt; i += 2)
23293 193910 : if ((d->perm[i] & 1) || d->perm[i + 1] != d->perm[i] + 1)
23294 : return false;
23295 14770 : nd->vmode = mode;
23296 14770 : nd->nelt = d->nelt / 2;
23297 96188 : for (i = 0; i < nd->nelt; i++)
23298 81418 : nd->perm[i] = d->perm[2 * i] / 2;
23299 29540 : if (GET_MODE_INNER (mode) != DImode)
23300 13001 : canonicalize_vector_int_perm (nd, nd);
23301 14770 : if (nd != d)
23302 : {
23303 9420 : nd->one_operand_p = d->one_operand_p;
23304 9420 : nd->testing_p = d->testing_p;
23305 9420 : if (d->op0 == d->op1)
23306 2781 : nd->op0 = nd->op1 = gen_lowpart (nd->vmode, d->op0);
23307 : else
23308 : {
23309 6639 : nd->op0 = gen_lowpart (nd->vmode, d->op0);
23310 6639 : nd->op1 = gen_lowpart (nd->vmode, d->op1);
23311 : }
23312 9420 : if (d->testing_p)
23313 6120 : nd->target = gen_raw_REG (nd->vmode, LAST_VIRTUAL_REGISTER + 1);
23314 : else
23315 3300 : nd->target = gen_reg_rtx (nd->vmode);
23316 : }
23317 : return true;
23318 : }
23319 :
23320 : /* Return true if permutation D can be performed as VMODE permutation
23321 : instead. */
23322 :
23323 : static bool
23324 5928 : valid_perm_using_mode_p (machine_mode vmode, struct expand_vec_perm_d *d)
23325 : {
23326 5928 : unsigned int i, j, chunk;
23327 :
23328 5928 : if (GET_MODE_CLASS (vmode) != MODE_VECTOR_INT
23329 5928 : || GET_MODE_CLASS (d->vmode) != MODE_VECTOR_INT
23330 14496 : || GET_MODE_SIZE (vmode) != GET_MODE_SIZE (d->vmode))
23331 : return false;
23332 :
23333 8568 : if (GET_MODE_NUNITS (vmode) >= d->nelt)
23334 : return true;
23335 :
23336 4020 : chunk = d->nelt / GET_MODE_NUNITS (vmode);
23337 5262 : for (i = 0; i < d->nelt; i += chunk)
23338 5015 : if (d->perm[i] & (chunk - 1))
23339 : return false;
23340 : else
23341 7754 : for (j = 1; j < chunk; ++j)
23342 6512 : if (d->perm[i] + j != d->perm[i + j])
23343 : return false;
23344 :
23345 : return true;
23346 : }
23347 :
23348 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
23349 : in terms of pshufb, vpperm, vpermq, vpermd, vpermps or vperm2i128. */
23350 :
23351 : static bool
23352 138687 : expand_vec_perm_pshufb (struct expand_vec_perm_d *d)
23353 : {
23354 138687 : unsigned i, nelt, eltsz, mask;
23355 138687 : unsigned char perm[64];
23356 138687 : machine_mode vmode;
23357 138687 : struct expand_vec_perm_d nd;
23358 138687 : rtx rperm[64], vperm, target, op0, op1;
23359 :
23360 138687 : nelt = d->nelt;
23361 :
23362 138687 : if (!d->one_operand_p)
23363 226928 : switch (GET_MODE_SIZE (d->vmode))
23364 : {
23365 8573 : case 4:
23366 8573 : if (!TARGET_XOP)
23367 : return false;
23368 : vmode = V4QImode;
23369 : break;
23370 :
23371 20397 : case 8:
23372 20397 : if (!TARGET_XOP)
23373 : return false;
23374 : vmode = V8QImode;
23375 : break;
23376 :
23377 75558 : case 16:
23378 75558 : if (!TARGET_XOP)
23379 : return false;
23380 : vmode = V16QImode;
23381 : break;
23382 :
23383 8058 : case 32:
23384 8058 : if (!TARGET_AVX2)
23385 : return false;
23386 :
23387 3996 : if (valid_perm_using_mode_p (V2TImode, d))
23388 : {
23389 56 : if (d->testing_p)
23390 : return true;
23391 :
23392 : /* Use vperm2i128 insn. The pattern uses
23393 : V4DImode instead of V2TImode. */
23394 52 : target = d->target;
23395 52 : if (d->vmode != V4DImode)
23396 12 : target = gen_reg_rtx (V4DImode);
23397 52 : op0 = gen_lowpart (V4DImode, d->op0);
23398 52 : op1 = gen_lowpart (V4DImode, d->op1);
23399 52 : rperm[0]
23400 52 : = GEN_INT ((d->perm[0] / (nelt / 2))
23401 : | ((d->perm[nelt / 2] / (nelt / 2)) * 16));
23402 52 : emit_insn (gen_avx2_permv2ti (target, op0, op1, rperm[0]));
23403 52 : if (target != d->target)
23404 12 : emit_move_insn (d->target, gen_lowpart (d->vmode, target));
23405 : return true;
23406 : }
23407 : /* FALLTHRU */
23408 :
23409 : default:
23410 : return false;
23411 : }
23412 : else
23413 50446 : switch (GET_MODE_SIZE (d->vmode))
23414 : {
23415 3755 : case 4:
23416 3755 : if (!TARGET_SSSE3)
23417 : return false;
23418 : vmode = V4QImode;
23419 : break;
23420 :
23421 2259 : case 8:
23422 2259 : if (!TARGET_SSSE3)
23423 : return false;
23424 : vmode = V8QImode;
23425 : break;
23426 :
23427 14230 : case 16:
23428 14230 : if (!TARGET_SSSE3)
23429 : return false;
23430 : vmode = V16QImode;
23431 : break;
23432 :
23433 4580 : case 32:
23434 4580 : if (!TARGET_AVX2)
23435 : return false;
23436 :
23437 : /* V4DImode should be already handled through
23438 : expand_vselect by vpermq instruction. */
23439 1923 : gcc_assert (d->vmode != V4DImode);
23440 :
23441 1923 : vmode = V32QImode;
23442 1923 : if (d->vmode == V8SImode
23443 1558 : || d->vmode == V16HImode
23444 1342 : || d->vmode == V32QImode)
23445 : {
23446 : /* First see if vpermq can be used for
23447 : V8SImode/V16HImode/V32QImode. */
23448 879 : if (valid_perm_using_mode_p (V4DImode, d))
23449 : {
23450 770 : for (i = 0; i < 4; i++)
23451 616 : perm[i] = (d->perm[i * nelt / 4] * 4 / nelt) & 3;
23452 154 : if (d->testing_p)
23453 : return true;
23454 58 : target = gen_reg_rtx (V4DImode);
23455 58 : if (expand_vselect (target, gen_lowpart (V4DImode, d->op0),
23456 : perm, 4, false))
23457 : {
23458 116 : emit_move_insn (d->target,
23459 58 : gen_lowpart (d->vmode, target));
23460 58 : return true;
23461 : }
23462 : return false;
23463 : }
23464 :
23465 : /* Next see if vpermd can be used. */
23466 725 : if (valid_perm_using_mode_p (V8SImode, d))
23467 : vmode = V8SImode;
23468 : }
23469 : /* Or if vpermps can be used. */
23470 1044 : else if (d->vmode == V8SFmode)
23471 : vmode = V8SImode;
23472 :
23473 : if (vmode == V32QImode)
23474 : {
23475 : /* vpshufb only works intra lanes, it is not
23476 : possible to shuffle bytes in between the lanes. */
23477 6473 : for (i = 0; i < nelt; ++i)
23478 6291 : if ((d->perm[i] ^ i) & (nelt / 2))
23479 : return false;
23480 : }
23481 : break;
23482 :
23483 399 : case 64:
23484 399 : if (!TARGET_AVX512BW)
23485 : return false;
23486 :
23487 : /* If vpermq didn't work, vpshufb won't work either. */
23488 204 : if (d->vmode == V8DFmode || d->vmode == V8DImode)
23489 : return false;
23490 :
23491 175 : vmode = V64QImode;
23492 175 : if (d->vmode == V16SImode
23493 150 : || d->vmode == V32HImode
23494 50 : || d->vmode == V64QImode)
23495 : {
23496 : /* First see if vpermq can be used for
23497 : V16SImode/V32HImode/V64QImode. */
23498 164 : if (valid_perm_using_mode_p (V8DImode, d))
23499 : {
23500 0 : for (i = 0; i < 8; i++)
23501 0 : perm[i] = (d->perm[i * nelt / 8] * 8 / nelt) & 7;
23502 0 : if (d->testing_p)
23503 : return true;
23504 0 : target = gen_reg_rtx (V8DImode);
23505 0 : if (expand_vselect (target, gen_lowpart (V8DImode, d->op0),
23506 : perm, 8, false))
23507 : {
23508 0 : emit_move_insn (d->target,
23509 0 : gen_lowpart (d->vmode, target));
23510 0 : return true;
23511 : }
23512 : return false;
23513 : }
23514 :
23515 : /* Next see if vpermd can be used. */
23516 164 : if (valid_perm_using_mode_p (V16SImode, d))
23517 : vmode = V16SImode;
23518 : }
23519 : /* Or if vpermps can be used. */
23520 11 : else if (d->vmode == V16SFmode)
23521 : vmode = V16SImode;
23522 :
23523 : if (vmode == V64QImode)
23524 : {
23525 : /* vpshufb only works intra lanes, it is not
23526 : possible to shuffle bytes in between the lanes. */
23527 578 : for (i = 0; i < nelt; ++i)
23528 578 : if ((d->perm[i] ^ i) & (3 * nelt / 4))
23529 : return false;
23530 : }
23531 : break;
23532 :
23533 : default:
23534 : return false;
23535 : }
23536 :
23537 11511 : if (d->testing_p)
23538 : return true;
23539 :
23540 : /* Try to avoid variable permutation instruction. */
23541 8790 : if (canonicalize_vector_int_perm (d, &nd) && expand_vec_perm_1 (&nd))
23542 : {
23543 1839 : emit_move_insn (d->target, gen_lowpart (d->vmode, nd.target));
23544 1839 : return true;
23545 : }
23546 :
23547 6951 : if (vmode == V8SImode)
23548 8955 : for (i = 0; i < 8; ++i)
23549 7960 : rperm[i] = GEN_INT ((d->perm[i * nelt / 8] * 8 / nelt) & 7);
23550 5956 : else if (vmode == V16SImode)
23551 612 : for (i = 0; i < 16; ++i)
23552 576 : rperm[i] = GEN_INT ((d->perm[i * nelt / 16] * 16 / nelt) & 15);
23553 : else
23554 : {
23555 5920 : eltsz = GET_MODE_UNIT_SIZE (d->vmode);
23556 5920 : if (!d->one_operand_p)
23557 3212 : mask = 2 * nelt - 1;
23558 2708 : else if (vmode == V64QImode)
23559 0 : mask = nelt / 4 - 1;
23560 2708 : else if (vmode == V32QImode)
23561 176 : mask = nelt / 2 - 1;
23562 : else
23563 2532 : mask = nelt - 1;
23564 :
23565 59156 : for (i = 0; i < nelt; ++i)
23566 : {
23567 53236 : unsigned j, e = d->perm[i] & mask;
23568 148424 : for (j = 0; j < eltsz; ++j)
23569 95188 : rperm[i * eltsz + j] = GEN_INT (e * eltsz + j);
23570 : }
23571 : }
23572 :
23573 6951 : machine_mode vpmode = vmode;
23574 :
23575 6951 : nelt = GET_MODE_SIZE (vmode);
23576 :
23577 : /* Emulate narrow modes with V16QI instructions. */
23578 6951 : if (nelt < 16)
23579 : {
23580 222 : rtx m128 = GEN_INT (-128);
23581 :
23582 : /* Remap elements from the second operand, as we have to
23583 : account for inactive top elements from the first operand. */
23584 222 : if (!d->one_operand_p)
23585 : {
23586 243 : for (i = 0; i < nelt; ++i)
23587 : {
23588 216 : unsigned ival = UINTVAL (rperm[i]);
23589 216 : if (ival >= nelt)
23590 108 : rperm[i] = GEN_INT (ival + 16 - nelt);
23591 : }
23592 : }
23593 :
23594 : /* Fill inactive elements in the top positions with zeros. */
23595 2570 : for (i = nelt; i < 16; ++i)
23596 2348 : rperm[i] = m128;
23597 :
23598 : vpmode = V16QImode;
23599 : }
23600 :
23601 13902 : vperm = gen_rtx_CONST_VECTOR (vpmode,
23602 6951 : gen_rtvec_v (GET_MODE_NUNITS (vpmode), rperm));
23603 6951 : vperm = force_reg (vpmode, vperm);
23604 :
23605 6951 : if (vmode == d->vmode)
23606 2406 : target = d->target;
23607 : else
23608 4545 : target = gen_reg_rtx (vmode);
23609 :
23610 6951 : op0 = gen_lowpart (vmode, d->op0);
23611 :
23612 6951 : if (d->one_operand_p)
23613 : {
23614 3739 : rtx (*gen) (rtx, rtx, rtx);
23615 :
23616 3739 : if (vmode == V4QImode)
23617 : gen = gen_mmx_pshufbv4qi3;
23618 : else if (vmode == V8QImode)
23619 : gen = gen_mmx_pshufbv8qi3;
23620 : else if (vmode == V16QImode)
23621 : gen = gen_ssse3_pshufbv16qi3;
23622 : else if (vmode == V32QImode)
23623 : gen = gen_avx2_pshufbv32qi3;
23624 : else if (vmode == V64QImode)
23625 : gen = gen_avx512bw_pshufbv64qi3;
23626 : else if (vmode == V8SFmode)
23627 : gen = gen_avx2_permvarv8sf;
23628 : else if (vmode == V8SImode)
23629 : gen = gen_avx2_permvarv8si;
23630 : else if (vmode == V16SFmode)
23631 : gen = gen_avx512f_permvarv16sf;
23632 : else if (vmode == V16SImode)
23633 : gen = gen_avx512f_permvarv16si;
23634 : else
23635 : gcc_unreachable ();
23636 :
23637 3739 : emit_insn (gen (target, op0, vperm));
23638 : }
23639 : else
23640 : {
23641 3212 : rtx (*gen) (rtx, rtx, rtx, rtx);
23642 :
23643 3212 : op1 = gen_lowpart (vmode, d->op1);
23644 :
23645 3212 : if (vmode == V4QImode)
23646 : gen = gen_mmx_ppermv32;
23647 : else if (vmode == V8QImode)
23648 : gen = gen_mmx_ppermv64;
23649 : else if (vmode == V16QImode)
23650 : gen = gen_xop_pperm;
23651 : else
23652 0 : gcc_unreachable ();
23653 :
23654 3212 : emit_insn (gen (target, op0, op1, vperm));
23655 : }
23656 :
23657 6951 : if (target != d->target)
23658 4545 : emit_move_insn (d->target, gen_lowpart (d->vmode, target));
23659 :
23660 : return true;
23661 : }
23662 :
23663 : /* Try to expand one-operand permutation with constant mask. */
23664 :
23665 : static bool
23666 126842 : ix86_expand_vec_one_operand_perm_avx512 (struct expand_vec_perm_d *d)
23667 : {
23668 126842 : machine_mode mode = GET_MODE (d->op0);
23669 126842 : machine_mode maskmode = mode;
23670 253684 : unsigned inner_size = GET_MODE_SIZE (GET_MODE_INNER (mode));
23671 126842 : rtx (*gen) (rtx, rtx, rtx) = NULL;
23672 126842 : rtx target, op0, mask;
23673 126842 : rtx vec[64];
23674 :
23675 126842 : if (!rtx_equal_p (d->op0, d->op1))
23676 : return false;
23677 :
23678 17893 : if (!TARGET_AVX512F)
23679 : return false;
23680 :
23681 : /* Accept VNxHImode and VNxQImode now. */
23682 739 : if (!TARGET_AVX512VL && GET_MODE_SIZE (mode) < 64)
23683 : return false;
23684 :
23685 : /* vpermw. */
23686 463 : if (!TARGET_AVX512BW && inner_size == 2)
23687 : return false;
23688 :
23689 : /* vpermb. */
23690 329 : if (!TARGET_AVX512VBMI && inner_size == 1)
23691 : return false;
23692 :
23693 210 : switch (mode)
23694 : {
23695 : case E_V16SImode:
23696 : gen = gen_avx512f_permvarv16si;
23697 : break;
23698 4 : case E_V16SFmode:
23699 4 : gen = gen_avx512f_permvarv16sf;
23700 4 : maskmode = V16SImode;
23701 4 : break;
23702 1 : case E_V8DImode:
23703 1 : gen = gen_avx512f_permvarv8di;
23704 1 : break;
23705 30 : case E_V8DFmode:
23706 30 : gen = gen_avx512f_permvarv8df;
23707 30 : maskmode = V8DImode;
23708 30 : break;
23709 106 : case E_V32HImode:
23710 106 : gen = gen_avx512bw_permvarv32hi;
23711 106 : break;
23712 14 : case E_V16HImode:
23713 14 : gen = gen_avx512vl_permvarv16hi;
23714 14 : break;
23715 6 : case E_V8HImode:
23716 6 : gen = gen_avx512vl_permvarv8hi;
23717 6 : break;
23718 4 : case E_V64QImode:
23719 4 : gen = gen_avx512bw_permvarv64qi;
23720 4 : break;
23721 2 : case E_V32QImode:
23722 2 : gen = gen_avx512vl_permvarv32qi;
23723 2 : break;
23724 0 : case E_V16QImode:
23725 0 : gen = gen_avx512vl_permvarv16qi;
23726 0 : break;
23727 :
23728 : default:
23729 : return false;
23730 : }
23731 :
23732 209 : if (d->testing_p)
23733 : return true;
23734 :
23735 200 : target = d->target;
23736 200 : op0 = d->op0;
23737 5024 : for (int i = 0; i < d->nelt; ++i)
23738 4824 : vec[i] = GEN_INT (d->perm[i]);
23739 200 : mask = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (d->nelt, vec));
23740 200 : emit_insn (gen (target, op0, force_reg (maskmode, mask)));
23741 200 : return true;
23742 : }
23743 :
23744 : static bool expand_vec_perm_palignr (struct expand_vec_perm_d *d, bool);
23745 :
23746 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to instantiate D
23747 : in a single instruction. */
23748 :
23749 : static bool
23750 344185 : expand_vec_perm_1 (struct expand_vec_perm_d *d)
23751 : {
23752 344185 : unsigned i, nelt = d->nelt;
23753 344185 : struct expand_vec_perm_d nd;
23754 :
23755 : /* Check plain VEC_SELECT first, because AVX has instructions that could
23756 : match both SEL and SEL+CONCAT, but the plain SEL will allow a memory
23757 : input where SEL+CONCAT may not. */
23758 344185 : if (d->one_operand_p)
23759 : {
23760 : int mask = nelt - 1;
23761 : bool identity_perm = true;
23762 : bool broadcast_perm = true;
23763 :
23764 511114 : for (i = 0; i < nelt; i++)
23765 : {
23766 448286 : nd.perm[i] = d->perm[i] & mask;
23767 448286 : if (nd.perm[i] != i)
23768 342202 : identity_perm = false;
23769 448286 : if (nd.perm[i])
23770 369334 : broadcast_perm = false;
23771 : }
23772 :
23773 62828 : if (identity_perm)
23774 : {
23775 11 : if (!d->testing_p)
23776 5 : emit_move_insn (d->target, d->op0);
23777 : return true;
23778 : }
23779 62817 : else if (broadcast_perm && TARGET_AVX2)
23780 : {
23781 : /* Use vpbroadcast{b,w,d}. */
23782 381 : rtx (*gen) (rtx, rtx) = NULL;
23783 381 : switch (d->vmode)
23784 : {
23785 1 : case E_V64QImode:
23786 1 : if (TARGET_AVX512BW)
23787 : gen = gen_avx512bw_vec_dupv64qi_1;
23788 : break;
23789 4 : case E_V32QImode:
23790 4 : gen = gen_avx2_pbroadcastv32qi_1;
23791 4 : break;
23792 1 : case E_V32HImode:
23793 1 : if (TARGET_AVX512BW)
23794 : gen = gen_avx512bw_vec_dupv32hi_1;
23795 : break;
23796 4 : case E_V16HImode:
23797 4 : gen = gen_avx2_pbroadcastv16hi_1;
23798 4 : break;
23799 1 : case E_V16SImode:
23800 1 : if (TARGET_AVX512F)
23801 : gen = gen_avx512f_vec_dupv16si_1;
23802 : break;
23803 4 : case E_V8SImode:
23804 4 : gen = gen_avx2_pbroadcastv8si_1;
23805 4 : break;
23806 4 : case E_V16QImode:
23807 4 : gen = gen_avx2_pbroadcastv16qi;
23808 4 : break;
23809 5 : case E_V8HImode:
23810 5 : gen = gen_avx2_pbroadcastv8hi;
23811 5 : break;
23812 0 : case E_V16SFmode:
23813 0 : if (TARGET_AVX512F)
23814 : gen = gen_avx512f_vec_dupv16sf_1;
23815 : break;
23816 : case E_V8SFmode:
23817 : gen = gen_avx2_vec_dupv8sf_1;
23818 : break;
23819 0 : case E_V8DFmode:
23820 0 : if (TARGET_AVX512F)
23821 : gen = gen_avx512f_vec_dupv8df_1;
23822 : break;
23823 0 : case E_V8DImode:
23824 0 : if (TARGET_AVX512F)
23825 : gen = gen_avx512f_vec_dupv8di_1;
23826 : break;
23827 : /* For other modes prefer other shuffles this function creates. */
23828 : default: break;
23829 : }
23830 21 : if (gen != NULL)
23831 : {
23832 24 : if (!d->testing_p)
23833 24 : emit_insn (gen (d->target, d->op0));
23834 : return true;
23835 : }
23836 : }
23837 :
23838 62793 : if (expand_vselect (d->target, d->op0, nd.perm, nelt, d->testing_p))
23839 : return true;
23840 :
23841 : /* There are plenty of patterns in sse.md that are written for
23842 : SEL+CONCAT and are not replicated for a single op. Perhaps
23843 : that should be changed, to avoid the nastiness here. */
23844 :
23845 : /* Recognize interleave style patterns, which means incrementing
23846 : every other permutation operand. */
23847 201908 : for (i = 0; i < nelt; i += 2)
23848 : {
23849 165060 : nd.perm[i] = d->perm[i] & mask;
23850 165060 : nd.perm[i + 1] = (d->perm[i + 1] & mask) + nelt;
23851 : }
23852 36848 : if (expand_vselect_vconcat (d->target, d->op0, d->op0, nd.perm, nelt,
23853 : d->testing_p))
23854 : return true;
23855 :
23856 : /* Recognize shufps, which means adding {0, 0, nelt, nelt}. */
23857 31760 : if (nelt >= 4)
23858 : {
23859 108190 : for (i = 0; i < nelt; i += 4)
23860 : {
23861 76430 : nd.perm[i + 0] = d->perm[i + 0] & mask;
23862 76430 : nd.perm[i + 1] = d->perm[i + 1] & mask;
23863 76430 : nd.perm[i + 2] = (d->perm[i + 2] & mask) + nelt;
23864 76430 : nd.perm[i + 3] = (d->perm[i + 3] & mask) + nelt;
23865 : }
23866 :
23867 31760 : if (expand_vselect_vconcat (d->target, d->op0, d->op0, nd.perm, nelt,
23868 : d->testing_p))
23869 : return true;
23870 : }
23871 : }
23872 :
23873 : /* Try the SSE4.1 blend variable merge instructions. */
23874 307320 : if (expand_vec_perm_blend (d))
23875 : return true;
23876 :
23877 : /* Try movss/movsd instructions. */
23878 305205 : if (expand_vec_perm_movs (d))
23879 : return true;
23880 :
23881 : /* Try the SSE4.1 insertps instruction. */
23882 290582 : if (expand_vec_perm_insertps (d))
23883 : return true;
23884 :
23885 : /* Try the fully general two operand permute. */
23886 284292 : if (expand_vselect_vconcat (d->target, d->op0, d->op1, d->perm, nelt,
23887 : d->testing_p))
23888 : return true;
23889 :
23890 : /* Recognize interleave style patterns with reversed operands. */
23891 139460 : if (!d->one_operand_p)
23892 : {
23893 905373 : for (i = 0; i < nelt; ++i)
23894 : {
23895 791772 : unsigned e = d->perm[i];
23896 791772 : if (e >= nelt)
23897 387843 : e -= nelt;
23898 : else
23899 403929 : e += nelt;
23900 791772 : nd.perm[i] = e;
23901 : }
23902 :
23903 113601 : if (expand_vselect_vconcat (d->target, d->op1, d->op0, nd.perm, nelt,
23904 : d->testing_p))
23905 : return true;
23906 : }
23907 :
23908 : /* Try one of the AVX vpermil variable permutations. */
23909 139323 : if (expand_vec_perm_vpermil (d))
23910 : return true;
23911 :
23912 : /* Try the SSSE3 pshufb or XOP vpperm or AVX2 vperm2i128,
23913 : vpshufb, vpermd, vpermps or vpermq variable permutation. */
23914 138687 : if (expand_vec_perm_pshufb (d))
23915 : return true;
23916 :
23917 : /* Try the AVX2 vpalignr instruction. */
23918 126966 : if (expand_vec_perm_palignr (d, true))
23919 : return true;
23920 :
23921 : /* Try the AVX512F vperm{w,b,s,d} instructions */
23922 126842 : if (ix86_expand_vec_one_operand_perm_avx512 (d))
23923 : return true;
23924 :
23925 : /* Try the AVX512F vpermt2/vpermi2 instructions. */
23926 126633 : if (ix86_expand_vec_perm_vpermt2 (NULL_RTX, NULL_RTX, NULL_RTX, NULL_RTX, d))
23927 : return true;
23928 :
23929 : /* See if we can get the same permutation in different vector integer
23930 : mode. */
23931 125757 : if (canonicalize_vector_int_perm (d, &nd) && expand_vec_perm_1 (&nd))
23932 : {
23933 6805 : if (!d->testing_p)
23934 1234 : emit_move_insn (d->target, gen_lowpart (d->vmode, nd.target));
23935 : return true;
23936 : }
23937 : return false;
23938 : }
23939 :
23940 : /* Canonicalize vec_perm index to make the first index
23941 : always comes from the first vector. */
23942 : static void
23943 8181 : ix86_vec_perm_index_canon (struct expand_vec_perm_d *d)
23944 : {
23945 8181 : unsigned nelt = d->nelt;
23946 8181 : if (d->perm[0] < nelt)
23947 : return;
23948 :
23949 5 : for (unsigned i = 0; i != nelt; i++)
23950 4 : d->perm[i] = (d->perm[i] + nelt) % (2 * nelt);
23951 :
23952 1 : std::swap (d->op0, d->op1);
23953 1 : return;
23954 : }
23955 :
23956 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
23957 : in terms of a pair of shufps+ shufps/pshufd instructions. */
23958 : static bool
23959 87325 : expand_vec_perm_shufps_shufps (struct expand_vec_perm_d *d)
23960 : {
23961 87325 : unsigned char perm1[4];
23962 87325 : machine_mode vmode = d->vmode;
23963 87325 : bool ok;
23964 87325 : unsigned i, j, k, count = 0;
23965 :
23966 87325 : if (d->one_operand_p
23967 81679 : || (vmode != V4SImode && vmode != V4SFmode))
23968 : return false;
23969 :
23970 35611 : if (d->testing_p)
23971 : return true;
23972 :
23973 8181 : ix86_vec_perm_index_canon (d);
23974 49086 : for (i = 0; i < 4; ++i)
23975 51188 : count += d->perm[i] > 3 ? 1 : 0;
23976 :
23977 8181 : gcc_assert (count & 3);
23978 :
23979 8181 : rtx tmp = gen_reg_rtx (vmode);
23980 : /* 2 from op0 and 2 from op1. */
23981 8181 : if (count == 2)
23982 : {
23983 : unsigned char perm2[4];
23984 18335 : for (i = 0, j = 0, k = 2; i < 4; ++i)
23985 14668 : if (d->perm[i] & 4)
23986 : {
23987 7334 : perm1[k++] = d->perm[i];
23988 7334 : perm2[i] = k - 1;
23989 : }
23990 : else
23991 : {
23992 7334 : perm1[j++] = d->perm[i];
23993 7334 : perm2[i] = j - 1;
23994 : }
23995 :
23996 : /* shufps. */
23997 7334 : ok = expand_vselect_vconcat (tmp, d->op0, d->op1,
23998 3667 : perm1, d->nelt, false);
23999 3667 : gcc_assert (ok);
24000 3667 : if (vmode == V4SImode && TARGET_SSE2)
24001 : /* pshufd. */
24002 2102 : ok = expand_vselect (d->target, tmp,
24003 2102 : perm2, d->nelt, false);
24004 : else
24005 : {
24006 : /* shufps. */
24007 1565 : perm2[2] += 4;
24008 1565 : perm2[3] += 4;
24009 1565 : ok = expand_vselect_vconcat (d->target, tmp, tmp,
24010 1565 : perm2, d->nelt, false);
24011 : }
24012 3667 : gcc_assert (ok);
24013 : }
24014 : /* 3 from one op and 1 from another. */
24015 : else
24016 : {
24017 22570 : unsigned pair_idx = 8, lone_idx = 8, shift;
24018 :
24019 : /* Find the lone index. */
24020 22570 : for (i = 0; i < 4; ++i)
24021 18056 : if ((d->perm[i] > 3 && count == 1)
24022 14748 : || (d->perm[i] < 4 && count == 3))
24023 18056 : lone_idx = i;
24024 :
24025 : /* When lone_idx is not 0, it must from second op(count == 1). */
24026 5720 : gcc_assert (count == (lone_idx ? 1 : 3));
24027 :
24028 : /* Find the pair index that sits in the same half as the lone index. */
24029 4514 : shift = lone_idx & 2;
24030 4514 : pair_idx = 1 - lone_idx + 2 * shift;
24031 :
24032 : /* First permutate lone index and pair index into the same vector as
24033 : [ lone, lone, pair, pair ]. */
24034 9028 : perm1[1] = perm1[0]
24035 4514 : = (count == 3) ? d->perm[lone_idx] : d->perm[lone_idx] - 4;
24036 9028 : perm1[3] = perm1[2]
24037 4514 : = (count == 3) ? d->perm[pair_idx] : d->perm[pair_idx] + 4;
24038 :
24039 : /* Always put the vector contains lone indx at the first. */
24040 4514 : if (count == 1)
24041 3308 : std::swap (d->op0, d->op1);
24042 :
24043 : /* shufps. */
24044 9028 : ok = expand_vselect_vconcat (tmp, d->op0, d->op1,
24045 4514 : perm1, d->nelt, false);
24046 4514 : gcc_assert (ok);
24047 :
24048 : /* Refine lone and pair index to original order. */
24049 4514 : perm1[shift] = lone_idx << 1;
24050 4514 : perm1[shift + 1] = pair_idx << 1;
24051 :
24052 : /* Select the remaining 2 elements in another vector. */
24053 13542 : for (i = 2 - shift; i < 4 - shift; ++i)
24054 9028 : perm1[i] = lone_idx == 1 ? d->perm[i] + 4 : d->perm[i];
24055 :
24056 : /* Adjust to original selector. */
24057 4514 : if (lone_idx > 1)
24058 2236 : std::swap (tmp, d->op1);
24059 :
24060 : /* shufps. */
24061 9028 : ok = expand_vselect_vconcat (d->target, tmp, d->op1,
24062 4514 : perm1, d->nelt, false);
24063 :
24064 4514 : gcc_assert (ok);
24065 : }
24066 :
24067 : return true;
24068 : }
24069 :
24070 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
24071 : in terms of a pair of pshuflw + pshufhw instructions. */
24072 :
24073 : static bool
24074 104248 : expand_vec_perm_pshuflw_pshufhw (struct expand_vec_perm_d *d)
24075 : {
24076 104248 : unsigned char perm2[MAX_VECT_LEN];
24077 104248 : unsigned i;
24078 104248 : bool ok;
24079 :
24080 104248 : if (d->vmode != V8HImode || !d->one_operand_p)
24081 : return false;
24082 :
24083 : /* The two permutations only operate in 64-bit lanes. */
24084 12853 : for (i = 0; i < 4; ++i)
24085 10376 : if (d->perm[i] >= 4)
24086 : return false;
24087 12329 : for (i = 4; i < 8; ++i)
24088 9866 : if (d->perm[i] < 4)
24089 : return false;
24090 :
24091 2463 : if (d->testing_p)
24092 : return true;
24093 :
24094 : /* Emit the pshuflw. */
24095 134 : memcpy (perm2, d->perm, 4);
24096 670 : for (i = 4; i < 8; ++i)
24097 536 : perm2[i] = i;
24098 134 : ok = expand_vselect (d->target, d->op0, perm2, 8, d->testing_p);
24099 134 : gcc_assert (ok);
24100 :
24101 : /* Emit the pshufhw. */
24102 134 : memcpy (perm2 + 4, d->perm + 4, 4);
24103 670 : for (i = 0; i < 4; ++i)
24104 536 : perm2[i] = i;
24105 134 : ok = expand_vselect (d->target, d->target, perm2, 8, d->testing_p);
24106 134 : gcc_assert (ok);
24107 :
24108 : return true;
24109 : }
24110 :
24111 : /* Try to permute 2 64-bit vectors by punpckldq + 128-bit vector shuffle. */
24112 : static bool
24113 51714 : expand_vec_perm_punpckldq_pshuf (struct expand_vec_perm_d *d)
24114 : {
24115 51714 : if (GET_MODE_BITSIZE (d->vmode) != 64
24116 16918 : || !TARGET_MMX_WITH_SSE
24117 68632 : || d->one_operand_p)
24118 : return false;
24119 :
24120 15440 : machine_mode widen_vmode;
24121 15440 : switch (d->vmode)
24122 : {
24123 : /* pshufd. */
24124 : case E_V2SImode:
24125 : widen_vmode = V4SImode;
24126 : break;
24127 :
24128 : /* pshufd. */
24129 1469 : case E_V2SFmode:
24130 1469 : widen_vmode = V4SFmode;
24131 1469 : break;
24132 :
24133 5148 : case E_V4HImode:
24134 5148 : widen_vmode = V8HImode;
24135 : /* pshufb. */
24136 5148 : if (!TARGET_SSSE3)
24137 : return false;
24138 : break;
24139 :
24140 5999 : case E_V8QImode:
24141 : /* pshufb. */
24142 5999 : widen_vmode = V16QImode;
24143 5999 : if (!TARGET_SSSE3)
24144 : return false;
24145 : break;
24146 :
24147 : default:
24148 : return false;
24149 : }
24150 :
24151 6392 : if (d->testing_p)
24152 : return true;
24153 :
24154 400 : struct expand_vec_perm_d dperm;
24155 400 : dperm.target = gen_reg_rtx (widen_vmode);
24156 400 : rtx op0 = gen_reg_rtx (widen_vmode);
24157 400 : emit_move_insn (op0, gen_rtx_VEC_CONCAT (widen_vmode, d->op0, d->op1));
24158 400 : dperm.op0 = op0;
24159 400 : dperm.op1 = op0;
24160 400 : dperm.vmode = widen_vmode;
24161 400 : unsigned nelt = GET_MODE_NUNITS (widen_vmode);
24162 400 : dperm.nelt = nelt;
24163 400 : dperm.one_operand_p = true;
24164 400 : dperm.testing_p = false;
24165 :
24166 2134 : for (unsigned i = 0; i != nelt / 2; i++)
24167 : {
24168 1734 : dperm.perm[i] = d->perm[i];
24169 1734 : dperm.perm[i + nelt / 2] = d->perm[i];
24170 : }
24171 :
24172 400 : gcc_assert (expand_vec_perm_1 (&dperm));
24173 400 : emit_move_insn (d->target, lowpart_subreg (d->vmode,
24174 : dperm.target,
24175 : dperm.vmode));
24176 400 : return true;
24177 : }
24178 :
24179 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
24180 : the permutation using the SSSE3 palignr instruction. This succeeds
24181 : when all of the elements in PERM fit within one vector and we merely
24182 : need to shift them down so that a single vector permutation has a
24183 : chance to succeed. If SINGLE_INSN_ONLY_P, succeed if only
24184 : the vpalignr instruction itself can perform the requested permutation. */
24185 :
24186 : static bool
24187 228751 : expand_vec_perm_palignr (struct expand_vec_perm_d *d, bool single_insn_only_p)
24188 : {
24189 228751 : unsigned i, nelt = d->nelt;
24190 228751 : unsigned min, max, minswap, maxswap;
24191 228751 : bool in_order, ok, swap = false;
24192 228751 : rtx shift, target;
24193 228751 : struct expand_vec_perm_d dcopy;
24194 :
24195 : /* Even with AVX, palignr only operates on 128-bit vectors,
24196 : in AVX2 palignr operates on both 128-bit lanes. */
24197 118518 : if ((!TARGET_SSSE3 || GET_MODE_SIZE (d->vmode) != 16)
24198 271651 : && (!TARGET_AVX2 || GET_MODE_SIZE (d->vmode) != 32))
24199 : return false;
24200 :
24201 34455 : min = 2 * nelt;
24202 34455 : max = 0;
24203 34455 : minswap = 2 * nelt;
24204 34455 : maxswap = 0;
24205 238451 : for (i = 0; i < nelt; ++i)
24206 : {
24207 203996 : unsigned e = d->perm[i];
24208 203996 : unsigned eswap = d->perm[i] ^ nelt;
24209 407992 : if (GET_MODE_SIZE (d->vmode) == 32)
24210 : {
24211 69136 : e = (e & ((nelt / 2) - 1)) | ((e & nelt) >> 1);
24212 69136 : eswap = e ^ (nelt / 2);
24213 : }
24214 203996 : if (e < min)
24215 : min = e;
24216 203996 : if (e > max)
24217 : max = e;
24218 203996 : if (eswap < minswap)
24219 : minswap = eswap;
24220 203996 : if (eswap > maxswap)
24221 : maxswap = eswap;
24222 : }
24223 34455 : if (min == 0
24224 50365 : || max - min >= (GET_MODE_SIZE (d->vmode) == 32 ? nelt / 2 : nelt))
24225 : {
24226 31255 : if (d->one_operand_p
24227 30986 : || minswap == 0
24228 66953 : || maxswap - minswap >= (GET_MODE_SIZE (d->vmode) == 32
24229 17849 : ? nelt / 2 : nelt))
24230 : return false;
24231 : swap = true;
24232 : min = minswap;
24233 6420 : max = maxswap;
24234 : }
24235 :
24236 : /* Given that we have SSSE3, we know we'll be able to implement the
24237 : single operand permutation after the palignr with pshufb for
24238 : 128-bit vectors. If SINGLE_INSN_ONLY_P, in_order has to be computed
24239 : first. */
24240 6474 : if (d->testing_p && GET_MODE_SIZE (d->vmode) == 16 && !single_insn_only_p)
24241 : return true;
24242 :
24243 6420 : dcopy = *d;
24244 6420 : if (swap)
24245 : {
24246 3220 : dcopy.op0 = d->op1;
24247 3220 : dcopy.op1 = d->op0;
24248 16172 : for (i = 0; i < nelt; ++i)
24249 12952 : dcopy.perm[i] ^= nelt;
24250 : }
24251 :
24252 6420 : in_order = true;
24253 32668 : for (i = 0; i < nelt; ++i)
24254 : {
24255 26248 : unsigned e = dcopy.perm[i];
24256 26248 : if (GET_MODE_SIZE (d->vmode) == 32
24257 1152 : && e >= nelt
24258 26510 : && (e & (nelt / 2 - 1)) < min)
24259 262 : e = e - min - (nelt / 2);
24260 : else
24261 25986 : e = e - min;
24262 26248 : if (e != i)
24263 19394 : in_order = false;
24264 26248 : dcopy.perm[i] = e;
24265 : }
24266 6420 : dcopy.one_operand_p = true;
24267 :
24268 6420 : if (single_insn_only_p && !in_order)
24269 : return false;
24270 :
24271 : /* For AVX2, test whether we can permute the result in one instruction. */
24272 3271 : if (d->testing_p)
24273 : {
24274 54 : if (in_order)
24275 : return true;
24276 0 : dcopy.op1 = dcopy.op0;
24277 0 : return expand_vec_perm_1 (&dcopy);
24278 : }
24279 :
24280 6434 : shift = GEN_INT (min * GET_MODE_UNIT_BITSIZE (d->vmode));
24281 6434 : if (GET_MODE_SIZE (d->vmode) == 16)
24282 : {
24283 3145 : target = gen_reg_rtx (V1TImode);
24284 3145 : emit_insn (gen_ssse3_palignrv1ti (target,
24285 3145 : gen_lowpart (V1TImode, dcopy.op1),
24286 3145 : gen_lowpart (V1TImode, dcopy.op0),
24287 : shift));
24288 : }
24289 : else
24290 : {
24291 72 : target = gen_reg_rtx (V2TImode);
24292 72 : emit_insn (gen_avx2_palignrv2ti (target,
24293 72 : gen_lowpart (V2TImode, dcopy.op1),
24294 72 : gen_lowpart (V2TImode, dcopy.op0),
24295 : shift));
24296 : }
24297 :
24298 3217 : dcopy.op0 = dcopy.op1 = gen_lowpart (d->vmode, target);
24299 :
24300 : /* Test for the degenerate case where the alignment by itself
24301 : produces the desired permutation. */
24302 3217 : if (in_order)
24303 : {
24304 70 : emit_move_insn (d->target, dcopy.op0);
24305 70 : return true;
24306 : }
24307 :
24308 3147 : ok = expand_vec_perm_1 (&dcopy);
24309 3159 : gcc_assert (ok || GET_MODE_SIZE (d->vmode) == 32);
24310 :
24311 : return ok;
24312 : }
24313 :
24314 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
24315 : the permutation using the SSE4_1 pblendv instruction. Potentially
24316 : reduces permutation from 2 pshufb and or to 1 pshufb and pblendv. */
24317 :
24318 : static bool
24319 92259 : expand_vec_perm_pblendv (struct expand_vec_perm_d *d)
24320 : {
24321 92259 : unsigned i, which, nelt = d->nelt;
24322 92259 : struct expand_vec_perm_d dcopy, dcopy1;
24323 92259 : machine_mode vmode = d->vmode;
24324 92259 : bool ok;
24325 :
24326 : /* Use the same checks as in expand_vec_perm_blend. */
24327 92259 : if (d->one_operand_p)
24328 : return false;
24329 90440 : if (TARGET_AVX2 && GET_MODE_SIZE (vmode) == 32)
24330 : ;
24331 84906 : else if (TARGET_AVX && (vmode == V4DFmode || vmode == V8SFmode))
24332 : ;
24333 81559 : else if (TARGET_SSE4_1
24334 92225 : && (GET_MODE_SIZE (vmode) == 16
24335 9382 : || (TARGET_MMX_WITH_SSE && GET_MODE_SIZE (vmode) == 8)
24336 2777 : || GET_MODE_SIZE (vmode) == 4))
24337 : ;
24338 : else
24339 : return false;
24340 :
24341 : /* Figure out where permutation elements stay not in their
24342 : respective lanes. */
24343 108714 : for (i = 0, which = 0; i < nelt; ++i)
24344 : {
24345 93330 : unsigned e = d->perm[i];
24346 93330 : if (e != i)
24347 128398 : which |= (e < nelt ? 1 : 2);
24348 : }
24349 : /* We can pblend the part where elements stay not in their
24350 : respective lanes only when these elements are all in one
24351 : half of a permutation.
24352 : {0 1 8 3 4 5 9 7} is ok as 8, 9 are at not at their respective
24353 : lanes, but both 8 and 9 >= 8
24354 : {0 1 8 3 4 5 2 7} is not ok as 2 and 8 are not at their
24355 : respective lanes and 8 >= 8, but 2 not. */
24356 15384 : if (which != 1 && which != 2)
24357 : return false;
24358 3211 : if (d->testing_p && GET_MODE_SIZE (vmode) == 16)
24359 : return true;
24360 :
24361 : /* First we apply one operand permutation to the part where
24362 : elements stay not in their respective lanes. */
24363 1976 : dcopy = *d;
24364 1976 : if (which == 2)
24365 1976 : dcopy.op0 = dcopy.op1 = d->op1;
24366 : else
24367 0 : dcopy.op0 = dcopy.op1 = d->op0;
24368 1976 : if (!d->testing_p)
24369 741 : dcopy.target = gen_reg_rtx (vmode);
24370 1976 : dcopy.one_operand_p = true;
24371 :
24372 15852 : for (i = 0; i < nelt; ++i)
24373 13876 : dcopy.perm[i] = d->perm[i] & (nelt - 1);
24374 :
24375 1976 : ok = expand_vec_perm_1 (&dcopy);
24376 3952 : if (GET_MODE_SIZE (vmode) != 16 && !ok)
24377 : return false;
24378 : else
24379 1681 : gcc_assert (ok);
24380 1681 : if (d->testing_p)
24381 : return true;
24382 :
24383 : /* Next we put permuted elements into their positions. */
24384 679 : dcopy1 = *d;
24385 679 : if (which == 2)
24386 679 : dcopy1.op1 = dcopy.target;
24387 : else
24388 0 : dcopy1.op0 = dcopy.target;
24389 :
24390 5751 : for (i = 0; i < nelt; ++i)
24391 5072 : dcopy1.perm[i] = ((d->perm[i] >= nelt) ? (nelt + i) : i);
24392 :
24393 679 : ok = expand_vec_perm_blend (&dcopy1);
24394 679 : gcc_assert (ok);
24395 :
24396 : return true;
24397 : }
24398 :
24399 : static bool expand_vec_perm_interleave3 (struct expand_vec_perm_d *d);
24400 :
24401 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
24402 : a two vector permutation into a single vector permutation by using
24403 : an interleave operation to merge the vectors. */
24404 :
24405 : static bool
24406 98650 : expand_vec_perm_interleave2 (struct expand_vec_perm_d *d)
24407 : {
24408 98650 : struct expand_vec_perm_d dremap, dfinal;
24409 98650 : unsigned i, nelt = d->nelt, nelt2 = nelt / 2;
24410 98650 : unsigned HOST_WIDE_INT contents;
24411 98650 : unsigned char remap[2 * MAX_VECT_LEN];
24412 98650 : rtx_insn *seq;
24413 98650 : bool ok, same_halves = false;
24414 :
24415 98650 : if (GET_MODE_SIZE (d->vmode) == 4
24416 176484 : || GET_MODE_SIZE (d->vmode) == 8
24417 238004 : || GET_MODE_SIZE (d->vmode) == 16)
24418 : {
24419 92090 : if (d->one_operand_p)
24420 : return false;
24421 : }
24422 13120 : else if (GET_MODE_SIZE (d->vmode) == 32)
24423 : {
24424 6243 : if (!TARGET_AVX)
24425 : return false;
24426 : /* For 32-byte modes allow even d->one_operand_p.
24427 : The lack of cross-lane shuffling in some instructions
24428 : might prevent a single insn shuffle. */
24429 6243 : dfinal = *d;
24430 6243 : dfinal.testing_p = true;
24431 : /* If expand_vec_perm_interleave3 can expand this into
24432 : a 3 insn sequence, give up and let it be expanded as
24433 : 3 insn sequence. While that is one insn longer,
24434 : it doesn't need a memory operand and in the common
24435 : case that both interleave low and high permutations
24436 : with the same operands are adjacent needs 4 insns
24437 : for both after CSE. */
24438 6243 : if (expand_vec_perm_interleave3 (&dfinal))
24439 : return false;
24440 : }
24441 : else
24442 : return false;
24443 :
24444 : /* Examine from whence the elements come. */
24445 92541 : contents = 0;
24446 689559 : for (i = 0; i < nelt; ++i)
24447 597018 : contents |= HOST_WIDE_INT_1U << d->perm[i];
24448 :
24449 92541 : memset (remap, 0xff, sizeof (remap));
24450 92541 : dremap = *d;
24451 :
24452 92541 : if (GET_MODE_SIZE (d->vmode) == 4
24453 176545 : || GET_MODE_SIZE (d->vmode) == 8)
24454 : {
24455 25620 : unsigned HOST_WIDE_INT h1, h2, h3, h4;
24456 :
24457 : /* Split the two input vectors into 4 halves. */
24458 25620 : h1 = (HOST_WIDE_INT_1U << nelt2) - 1;
24459 25620 : h2 = h1 << nelt2;
24460 25620 : h3 = h2 << nelt2;
24461 25620 : h4 = h3 << nelt2;
24462 :
24463 : /* If the elements from the low halves use interleave low,
24464 : and similarly for interleave high. */
24465 25620 : if ((contents & (h1 | h3)) == contents)
24466 : {
24467 : /* punpckl* */
24468 2814 : for (i = 0; i < nelt2; ++i)
24469 : {
24470 1924 : remap[i] = i * 2;
24471 1924 : remap[i + nelt] = i * 2 + 1;
24472 1924 : dremap.perm[i * 2] = i;
24473 1924 : dremap.perm[i * 2 + 1] = i + nelt;
24474 : }
24475 : }
24476 24730 : else if ((contents & (h2 | h4)) == contents)
24477 : {
24478 : /* punpckh* */
24479 2277 : for (i = 0; i < nelt2; ++i)
24480 : {
24481 1548 : remap[i + nelt2] = i * 2;
24482 1548 : remap[i + nelt + nelt2] = i * 2 + 1;
24483 1548 : dremap.perm[i * 2] = i + nelt2;
24484 1548 : dremap.perm[i * 2 + 1] = i + nelt + nelt2;
24485 : }
24486 : }
24487 : else
24488 : return false;
24489 : }
24490 133842 : else if (GET_MODE_SIZE (d->vmode) == 16)
24491 : {
24492 60904 : unsigned HOST_WIDE_INT h1, h2, h3, h4;
24493 :
24494 : /* Split the two input vectors into 4 halves. */
24495 60904 : h1 = (HOST_WIDE_INT_1U << nelt2) - 1;
24496 60904 : h2 = h1 << nelt2;
24497 60904 : h3 = h2 << nelt2;
24498 60904 : h4 = h3 << nelt2;
24499 :
24500 : /* If the elements from the low halves use interleave low, and similarly
24501 : for interleave high. If the elements are from mis-matched halves, we
24502 : can use shufps for V4SF/V4SI or do a DImode shuffle. */
24503 60904 : if ((contents & (h1 | h3)) == contents)
24504 : {
24505 : /* punpckl* */
24506 5368 : for (i = 0; i < nelt2; ++i)
24507 : {
24508 3906 : remap[i] = i * 2;
24509 3906 : remap[i + nelt] = i * 2 + 1;
24510 3906 : dremap.perm[i * 2] = i;
24511 3906 : dremap.perm[i * 2 + 1] = i + nelt;
24512 : }
24513 1462 : if (!TARGET_SSE2 && d->vmode == V4SImode)
24514 0 : dremap.vmode = V4SFmode;
24515 : }
24516 59442 : else if ((contents & (h2 | h4)) == contents)
24517 : {
24518 : /* punpckh* */
24519 4563 : for (i = 0; i < nelt2; ++i)
24520 : {
24521 3278 : remap[i + nelt2] = i * 2;
24522 3278 : remap[i + nelt + nelt2] = i * 2 + 1;
24523 3278 : dremap.perm[i * 2] = i + nelt2;
24524 3278 : dremap.perm[i * 2 + 1] = i + nelt + nelt2;
24525 : }
24526 1285 : if (!TARGET_SSE2 && d->vmode == V4SImode)
24527 0 : dremap.vmode = V4SFmode;
24528 : }
24529 58157 : else if ((contents & (h1 | h4)) == contents)
24530 : {
24531 : /* shufps */
24532 3194 : for (i = 0; i < nelt2; ++i)
24533 : {
24534 2368 : remap[i] = i;
24535 2368 : remap[i + nelt + nelt2] = i + nelt2;
24536 2368 : dremap.perm[i] = i;
24537 2368 : dremap.perm[i + nelt2] = i + nelt + nelt2;
24538 : }
24539 826 : if (nelt != 4)
24540 : {
24541 : /* shufpd */
24542 120 : dremap.vmode = V2DImode;
24543 120 : dremap.nelt = 2;
24544 120 : dremap.perm[0] = 0;
24545 120 : dremap.perm[1] = 3;
24546 : }
24547 : }
24548 57331 : else if ((contents & (h2 | h3)) == contents)
24549 : {
24550 : /* shufps */
24551 4672 : for (i = 0; i < nelt2; ++i)
24552 : {
24553 3430 : remap[i + nelt2] = i;
24554 3430 : remap[i + nelt] = i + nelt2;
24555 3430 : dremap.perm[i] = i + nelt2;
24556 3430 : dremap.perm[i + nelt2] = i + nelt;
24557 : }
24558 1242 : if (nelt != 4)
24559 : {
24560 : /* shufpd */
24561 195 : dremap.vmode = V2DImode;
24562 195 : dremap.nelt = 2;
24563 195 : dremap.perm[0] = 1;
24564 195 : dremap.perm[1] = 2;
24565 : }
24566 : }
24567 : else
24568 : return false;
24569 : }
24570 : else
24571 : {
24572 6017 : unsigned int nelt4 = nelt / 4, nzcnt = 0;
24573 6017 : unsigned HOST_WIDE_INT q[8];
24574 6017 : unsigned int nonzero_halves[4];
24575 :
24576 : /* Split the two input vectors into 8 quarters. */
24577 6017 : q[0] = (HOST_WIDE_INT_1U << nelt4) - 1;
24578 48136 : for (i = 1; i < 8; ++i)
24579 42119 : q[i] = q[0] << (nelt4 * i);
24580 30085 : for (i = 0; i < 4; ++i)
24581 24068 : if (((q[2 * i] | q[2 * i + 1]) & contents) != 0)
24582 : {
24583 21646 : nonzero_halves[nzcnt] = i;
24584 21646 : ++nzcnt;
24585 : }
24586 :
24587 6017 : if (nzcnt == 1)
24588 : {
24589 215 : gcc_assert (d->one_operand_p);
24590 215 : nonzero_halves[1] = nonzero_halves[0];
24591 215 : same_halves = true;
24592 : }
24593 5802 : else if (d->one_operand_p)
24594 : {
24595 115 : gcc_assert (nonzero_halves[0] == 0);
24596 115 : gcc_assert (nonzero_halves[1] == 1);
24597 : }
24598 :
24599 6017 : if (nzcnt <= 2)
24600 : {
24601 582 : if (d->perm[0] / nelt2 == nonzero_halves[1])
24602 : {
24603 : /* Attempt to increase the likelihood that dfinal
24604 : shuffle will be intra-lane. */
24605 315 : std::swap (nonzero_halves[0], nonzero_halves[1]);
24606 : }
24607 :
24608 : /* vperm2f128 or vperm2i128. */
24609 3532 : for (i = 0; i < nelt2; ++i)
24610 : {
24611 2950 : remap[i + nonzero_halves[1] * nelt2] = i + nelt2;
24612 2950 : remap[i + nonzero_halves[0] * nelt2] = i;
24613 2950 : dremap.perm[i + nelt2] = i + nonzero_halves[1] * nelt2;
24614 2950 : dremap.perm[i] = i + nonzero_halves[0] * nelt2;
24615 : }
24616 :
24617 582 : if (d->vmode != V8SFmode
24618 : && d->vmode != V4DFmode
24619 : && d->vmode != V8SImode)
24620 : {
24621 188 : dremap.vmode = V8SImode;
24622 188 : dremap.nelt = 8;
24623 940 : for (i = 0; i < 4; ++i)
24624 : {
24625 752 : dremap.perm[i] = i + nonzero_halves[0] * 4;
24626 752 : dremap.perm[i + 4] = i + nonzero_halves[1] * 4;
24627 : }
24628 : }
24629 : }
24630 5435 : else if (d->one_operand_p)
24631 4970 : return false;
24632 5435 : else if (TARGET_AVX2
24633 2132 : && (contents & (q[0] | q[2] | q[4] | q[6])) == contents)
24634 : {
24635 : /* vpunpckl* */
24636 491 : for (i = 0; i < nelt4; ++i)
24637 : {
24638 247 : remap[i] = i * 2;
24639 247 : remap[i + nelt] = i * 2 + 1;
24640 247 : remap[i + nelt2] = i * 2 + nelt2;
24641 247 : remap[i + nelt + nelt2] = i * 2 + nelt2 + 1;
24642 247 : dremap.perm[i * 2] = i;
24643 247 : dremap.perm[i * 2 + 1] = i + nelt;
24644 247 : dremap.perm[i * 2 + nelt2] = i + nelt2;
24645 247 : dremap.perm[i * 2 + nelt2 + 1] = i + nelt + nelt2;
24646 : }
24647 : }
24648 5191 : else if (TARGET_AVX2
24649 1888 : && (contents & (q[1] | q[3] | q[5] | q[7])) == contents)
24650 : {
24651 : /* vpunpckh* */
24652 445 : for (i = 0; i < nelt4; ++i)
24653 : {
24654 224 : remap[i + nelt4] = i * 2;
24655 224 : remap[i + nelt + nelt4] = i * 2 + 1;
24656 224 : remap[i + nelt2 + nelt4] = i * 2 + nelt2;
24657 224 : remap[i + nelt + nelt2 + nelt4] = i * 2 + nelt2 + 1;
24658 224 : dremap.perm[i * 2] = i + nelt4;
24659 224 : dremap.perm[i * 2 + 1] = i + nelt + nelt4;
24660 224 : dremap.perm[i * 2 + nelt2] = i + nelt2 + nelt4;
24661 224 : dremap.perm[i * 2 + nelt2 + 1] = i + nelt + nelt2 + nelt4;
24662 : }
24663 : }
24664 : else
24665 : return false;
24666 : }
24667 :
24668 : /* Use the remapping array set up above to move the elements from their
24669 : swizzled locations into their final destinations. */
24670 7481 : dfinal = *d;
24671 48173 : for (i = 0; i < nelt; ++i)
24672 : {
24673 40692 : unsigned e = remap[d->perm[i]];
24674 40692 : gcc_assert (e < nelt);
24675 : /* If same_halves is true, both halves of the remapped vector are the
24676 : same. Avoid cross-lane accesses if possible. */
24677 40692 : if (same_halves && i >= nelt2)
24678 : {
24679 792 : gcc_assert (e < nelt2);
24680 792 : dfinal.perm[i] = e + nelt2;
24681 : }
24682 : else
24683 39900 : dfinal.perm[i] = e;
24684 : }
24685 7481 : if (!d->testing_p)
24686 : {
24687 2737 : dremap.target = gen_reg_rtx (dremap.vmode);
24688 2737 : dfinal.op0 = gen_lowpart (dfinal.vmode, dremap.target);
24689 : }
24690 7481 : dfinal.op1 = dfinal.op0;
24691 7481 : dfinal.one_operand_p = true;
24692 :
24693 : /* Test if the final remap can be done with a single insn. For V4SFmode or
24694 : V4SImode this *will* succeed. For V8HImode or V16QImode it may not. */
24695 7481 : start_sequence ();
24696 7481 : ok = expand_vec_perm_1 (&dfinal);
24697 7481 : seq = end_sequence ();
24698 :
24699 7481 : if (!ok)
24700 : return false;
24701 :
24702 6223 : if (d->testing_p)
24703 : return true;
24704 :
24705 2698 : if (dremap.vmode != dfinal.vmode)
24706 : {
24707 57 : dremap.op0 = gen_lowpart (dremap.vmode, dremap.op0);
24708 57 : dremap.op1 = gen_lowpart (dremap.vmode, dremap.op1);
24709 : }
24710 :
24711 2698 : ok = expand_vec_perm_1 (&dremap);
24712 2698 : gcc_assert (ok);
24713 :
24714 2698 : emit_insn (seq);
24715 2698 : return true;
24716 : }
24717 :
24718 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
24719 : a single vector cross-lane permutation into vpermq followed
24720 : by any of the single insn permutations. */
24721 :
24722 : static bool
24723 92327 : expand_vec_perm_vpermq_perm_1 (struct expand_vec_perm_d *d)
24724 : {
24725 92327 : struct expand_vec_perm_d dremap, dfinal;
24726 92327 : unsigned i, j, nelt = d->nelt, nelt2 = nelt / 2, nelt4 = nelt / 4;
24727 92327 : unsigned contents[2];
24728 92327 : bool ok;
24729 :
24730 92327 : if (!(TARGET_AVX2
24731 4057 : && (d->vmode == V32QImode || d->vmode == V16HImode)
24732 248 : && d->one_operand_p))
24733 : return false;
24734 :
24735 7 : contents[0] = 0;
24736 7 : contents[1] = 0;
24737 103 : for (i = 0; i < nelt2; ++i)
24738 : {
24739 96 : contents[0] |= 1u << (d->perm[i] / nelt4);
24740 96 : contents[1] |= 1u << (d->perm[i + nelt2] / nelt4);
24741 : }
24742 :
24743 7 : for (i = 0; i < 2; ++i)
24744 : {
24745 : unsigned int cnt = 0;
24746 21 : for (j = 0; j < 4; ++j)
24747 21 : if ((contents[i] & (1u << j)) != 0 && ++cnt > 2)
24748 : return false;
24749 : }
24750 :
24751 0 : if (d->testing_p)
24752 : return true;
24753 :
24754 0 : dremap = *d;
24755 0 : dremap.vmode = V4DImode;
24756 0 : dremap.nelt = 4;
24757 0 : dremap.target = gen_reg_rtx (V4DImode);
24758 0 : dremap.op0 = gen_lowpart (V4DImode, d->op0);
24759 0 : dremap.op1 = dremap.op0;
24760 0 : dremap.one_operand_p = true;
24761 0 : for (i = 0; i < 2; ++i)
24762 : {
24763 : unsigned int cnt = 0;
24764 0 : for (j = 0; j < 4; ++j)
24765 0 : if ((contents[i] & (1u << j)) != 0)
24766 0 : dremap.perm[2 * i + cnt++] = j;
24767 0 : for (; cnt < 2; ++cnt)
24768 0 : dremap.perm[2 * i + cnt] = 0;
24769 : }
24770 :
24771 0 : dfinal = *d;
24772 0 : dfinal.op0 = gen_lowpart (dfinal.vmode, dremap.target);
24773 0 : dfinal.op1 = dfinal.op0;
24774 0 : dfinal.one_operand_p = true;
24775 0 : for (i = 0, j = 0; i < nelt; ++i)
24776 : {
24777 0 : if (i == nelt2)
24778 0 : j = 2;
24779 0 : dfinal.perm[i] = (d->perm[i] & (nelt4 - 1)) | (j ? nelt2 : 0);
24780 0 : if ((d->perm[i] / nelt4) == dremap.perm[j])
24781 : ;
24782 0 : else if ((d->perm[i] / nelt4) == dremap.perm[j + 1])
24783 0 : dfinal.perm[i] |= nelt4;
24784 : else
24785 0 : gcc_unreachable ();
24786 : }
24787 :
24788 0 : ok = expand_vec_perm_1 (&dremap);
24789 0 : gcc_assert (ok);
24790 :
24791 0 : ok = expand_vec_perm_1 (&dfinal);
24792 0 : gcc_assert (ok);
24793 :
24794 : return true;
24795 : }
24796 :
24797 : static bool canonicalize_perm (struct expand_vec_perm_d *d);
24798 :
24799 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to expand
24800 : a vector permutation using two instructions, vperm2f128 resp.
24801 : vperm2i128 followed by any single in-lane permutation. */
24802 :
24803 : static bool
24804 92327 : expand_vec_perm_vperm2f128 (struct expand_vec_perm_d *d)
24805 : {
24806 92327 : struct expand_vec_perm_d dfirst, dsecond;
24807 92327 : unsigned i, j, nelt = d->nelt, nelt2 = nelt / 2, perm;
24808 92327 : bool ok;
24809 :
24810 92327 : if (!TARGET_AVX
24811 22254 : || GET_MODE_SIZE (d->vmode) != 32
24812 97648 : || (d->vmode != V8SFmode && d->vmode != V4DFmode && !TARGET_AVX2))
24813 : return false;
24814 :
24815 5137 : dsecond = *d;
24816 5137 : dsecond.one_operand_p = false;
24817 5137 : dsecond.testing_p = true;
24818 :
24819 : /* ((perm << 2)|perm) & 0x33 is the vperm2[fi]128
24820 : immediate. For perm < 16 the second permutation uses
24821 : d->op0 as first operand, for perm >= 16 it uses d->op1
24822 : as first operand. The second operand is the result of
24823 : vperm2[fi]128. */
24824 168207 : for (perm = 0; perm < 32; perm++)
24825 : {
24826 : /* Ignore permutations which do not move anything cross-lane. */
24827 163153 : if (perm < 16)
24828 : {
24829 : /* The second shuffle for e.g. V4DFmode has
24830 : 0123 and ABCD operands.
24831 : Ignore AB23, as 23 is already in the second lane
24832 : of the first operand. */
24833 81830 : if ((perm & 0xc) == (1 << 2)) continue;
24834 : /* And 01CD, as 01 is in the first lane of the first
24835 : operand. */
24836 61358 : if ((perm & 3) == 0) continue;
24837 : /* And 4567, as then the vperm2[fi]128 doesn't change
24838 : anything on the original 4567 second operand. */
24839 46001 : if ((perm & 0xf) == ((3 << 2) | 2)) continue;
24840 : }
24841 : else
24842 : {
24843 : /* The second shuffle for e.g. V4DFmode has
24844 : 4567 and ABCD operands.
24845 : Ignore AB67, as 67 is already in the second lane
24846 : of the first operand. */
24847 81323 : if ((perm & 0xc) == (3 << 2)) continue;
24848 : /* And 45CD, as 45 is in the first lane of the first
24849 : operand. */
24850 61107 : if ((perm & 3) == 2) continue;
24851 : /* And 0123, as then the vperm2[fi]128 doesn't change
24852 : anything on the original 0123 first operand. */
24853 45854 : if ((perm & 0xf) == (1 << 2)) continue;
24854 : }
24855 :
24856 209920 : for (i = 0; i < nelt; i++)
24857 : {
24858 208793 : j = d->perm[i] / nelt2;
24859 388270 : if (j == ((perm >> (2 * (i >= nelt2))) & 3))
24860 51548 : dsecond.perm[i] = nelt + (i & nelt2) + (d->perm[i] & (nelt2 - 1));
24861 260763 : else if (j == (unsigned) (i >= nelt2) + 2 * (perm >= 16))
24862 76719 : dsecond.perm[i] = d->perm[i] & (nelt - 1);
24863 : else
24864 : break;
24865 : }
24866 :
24867 81653 : if (i == nelt)
24868 : {
24869 1127 : start_sequence ();
24870 1127 : ok = expand_vec_perm_1 (&dsecond);
24871 1127 : end_sequence ();
24872 : }
24873 : else
24874 : ok = false;
24875 :
24876 1127 : if (ok)
24877 : {
24878 68 : if (d->testing_p)
24879 : return true;
24880 :
24881 : /* Found a usable second shuffle. dfirst will be
24882 : vperm2f128 on d->op0 and d->op1. */
24883 46 : dsecond.testing_p = false;
24884 46 : dfirst = *d;
24885 46 : dfirst.target = gen_reg_rtx (d->vmode);
24886 270 : for (i = 0; i < nelt; i++)
24887 448 : dfirst.perm[i] = (i & (nelt2 - 1))
24888 336 : + ((perm >> (2 * (i >= nelt2))) & 3) * nelt2;
24889 :
24890 46 : canonicalize_perm (&dfirst);
24891 46 : ok = expand_vec_perm_1 (&dfirst);
24892 46 : gcc_assert (ok);
24893 :
24894 : /* And dsecond is some single insn shuffle, taking
24895 : d->op0 and result of vperm2f128 (if perm < 16) or
24896 : d->op1 and result of vperm2f128 (otherwise). */
24897 46 : if (perm >= 16)
24898 46 : dsecond.op0 = dsecond.op1;
24899 46 : dsecond.op1 = dfirst.target;
24900 :
24901 46 : ok = expand_vec_perm_1 (&dsecond);
24902 46 : gcc_assert (ok);
24903 :
24904 : return true;
24905 : }
24906 :
24907 : /* For one operand, the only useful vperm2f128 permutation is 0x01
24908 : aka lanes swap. */
24909 81585 : if (d->one_operand_p)
24910 : return false;
24911 : }
24912 :
24913 : return false;
24914 : }
24915 :
24916 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
24917 : a two vector permutation using 2 intra-lane interleave insns
24918 : and cross-lane shuffle for 32-byte vectors. */
24919 :
24920 : static bool
24921 34666 : expand_vec_perm_interleave3 (struct expand_vec_perm_d *d)
24922 : {
24923 34666 : unsigned i, nelt;
24924 34666 : rtx (*gen) (rtx, rtx, rtx);
24925 :
24926 34666 : if (d->one_operand_p)
24927 : return false;
24928 32063 : if (TARGET_AVX2 && GET_MODE_SIZE (d->vmode) == 32)
24929 : ;
24930 25555 : else if (TARGET_AVX && (d->vmode == V8SFmode || d->vmode == V4DFmode))
24931 : ;
24932 : else
24933 : return false;
24934 :
24935 8253 : nelt = d->nelt;
24936 8253 : if (d->perm[0] != 0 && d->perm[0] != nelt / 2)
24937 : return false;
24938 8662 : for (i = 0; i < nelt; i += 2)
24939 8290 : if (d->perm[i] != d->perm[0] + i / 2
24940 7417 : || d->perm[i + 1] != d->perm[0] + i / 2 + nelt)
24941 : return false;
24942 :
24943 372 : if (d->testing_p)
24944 : return true;
24945 :
24946 56 : switch (d->vmode)
24947 : {
24948 32 : case E_V32QImode:
24949 32 : if (d->perm[0])
24950 : gen = gen_vec_interleave_highv32qi;
24951 : else
24952 16 : gen = gen_vec_interleave_lowv32qi;
24953 : break;
24954 18 : case E_V16HImode:
24955 18 : if (d->perm[0])
24956 : gen = gen_vec_interleave_highv16hi;
24957 : else
24958 9 : gen = gen_vec_interleave_lowv16hi;
24959 : break;
24960 0 : case E_V8SImode:
24961 0 : if (d->perm[0])
24962 : gen = gen_vec_interleave_highv8si;
24963 : else
24964 0 : gen = gen_vec_interleave_lowv8si;
24965 : break;
24966 4 : case E_V4DImode:
24967 4 : if (d->perm[0])
24968 : gen = gen_vec_interleave_highv4di;
24969 : else
24970 2 : gen = gen_vec_interleave_lowv4di;
24971 : break;
24972 2 : case E_V8SFmode:
24973 2 : if (d->perm[0])
24974 : gen = gen_vec_interleave_highv8sf;
24975 : else
24976 1 : gen = gen_vec_interleave_lowv8sf;
24977 : break;
24978 0 : case E_V4DFmode:
24979 0 : if (d->perm[0])
24980 : gen = gen_vec_interleave_highv4df;
24981 : else
24982 0 : gen = gen_vec_interleave_lowv4df;
24983 : break;
24984 0 : default:
24985 0 : gcc_unreachable ();
24986 : }
24987 :
24988 56 : emit_insn (gen (d->target, d->op0, d->op1));
24989 56 : return true;
24990 : }
24991 :
24992 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement
24993 : a single vector permutation using a single intra-lane vector
24994 : permutation, vperm2f128 swapping the lanes and vblend* insn blending
24995 : the non-swapped and swapped vectors together. */
24996 :
24997 : static bool
24998 28277 : expand_vec_perm_vperm2f128_vblend (struct expand_vec_perm_d *d)
24999 : {
25000 28277 : struct expand_vec_perm_d dfirst, dsecond;
25001 28277 : unsigned i, j, msk, nelt = d->nelt, nelt2 = nelt / 2;
25002 28277 : rtx_insn *seq;
25003 28277 : bool ok;
25004 28277 : rtx (*blend) (rtx, rtx, rtx, rtx) = NULL;
25005 :
25006 28277 : if (!TARGET_AVX
25007 2893 : || TARGET_AVX2
25008 1814 : || (d->vmode != V8SFmode && d->vmode != V4DFmode)
25009 1630 : || !d->one_operand_p)
25010 : return false;
25011 :
25012 0 : dfirst = *d;
25013 0 : for (i = 0; i < nelt; i++)
25014 0 : dfirst.perm[i] = 0xff;
25015 0 : for (i = 0, msk = 0; i < nelt; i++)
25016 : {
25017 0 : j = (d->perm[i] & nelt2) ? i | nelt2 : i & ~nelt2;
25018 0 : if (dfirst.perm[j] != 0xff && dfirst.perm[j] != d->perm[i])
25019 : return false;
25020 0 : dfirst.perm[j] = d->perm[i];
25021 0 : if (j != i)
25022 0 : msk |= (1 << i);
25023 : }
25024 0 : for (i = 0; i < nelt; i++)
25025 0 : if (dfirst.perm[i] == 0xff)
25026 0 : dfirst.perm[i] = i;
25027 :
25028 0 : if (!d->testing_p)
25029 0 : dfirst.target = gen_reg_rtx (dfirst.vmode);
25030 :
25031 0 : start_sequence ();
25032 0 : ok = expand_vec_perm_1 (&dfirst);
25033 0 : seq = end_sequence ();
25034 :
25035 0 : if (!ok)
25036 : return false;
25037 :
25038 0 : if (d->testing_p)
25039 : return true;
25040 :
25041 0 : emit_insn (seq);
25042 :
25043 0 : dsecond = *d;
25044 0 : dsecond.op0 = dfirst.target;
25045 0 : dsecond.op1 = dfirst.target;
25046 0 : dsecond.one_operand_p = true;
25047 0 : dsecond.target = gen_reg_rtx (dsecond.vmode);
25048 0 : for (i = 0; i < nelt; i++)
25049 0 : dsecond.perm[i] = i ^ nelt2;
25050 :
25051 0 : ok = expand_vec_perm_1 (&dsecond);
25052 0 : gcc_assert (ok);
25053 :
25054 0 : blend = d->vmode == V8SFmode ? gen_avx_blendps256 : gen_avx_blendpd256;
25055 0 : emit_insn (blend (d->target, dfirst.target, dsecond.target, GEN_INT (msk)));
25056 0 : return true;
25057 : }
25058 :
25059 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement
25060 : a two vector permutation using two single vector permutations and
25061 : {,v}{,p}unpckl{ps,pd,bw,wd,dq}. If two_insn, succeed only if one
25062 : of dfirst or dsecond is identity permutation. */
25063 :
25064 : static bool
25065 118855 : expand_vec_perm_2perm_interleave (struct expand_vec_perm_d *d, bool two_insn)
25066 : {
25067 118855 : unsigned i, nelt = d->nelt, nelt2 = nelt / 2, lane = nelt;
25068 118855 : struct expand_vec_perm_d dfirst, dsecond, dfinal;
25069 118855 : bool ident1 = true, ident2 = true;
25070 :
25071 118855 : if (d->one_operand_p)
25072 : return false;
25073 :
25074 215212 : if (GET_MODE_SIZE (d->vmode) == 16)
25075 : {
25076 64435 : if (!TARGET_SSE)
25077 : return false;
25078 64435 : if (d->vmode != V4SFmode && d->vmode != V2DFmode && !TARGET_SSE2)
25079 : return false;
25080 : }
25081 86342 : else if (GET_MODE_SIZE (d->vmode) == 32)
25082 : {
25083 7219 : if (!TARGET_AVX)
25084 : return false;
25085 7219 : if (d->vmode != V8SFmode && d->vmode != V4DFmode && !TARGET_AVX2)
25086 : return false;
25087 : lane = nelt2;
25088 : }
25089 : else
25090 : return false;
25091 :
25092 233064 : for (i = 1; i < nelt; i++)
25093 199532 : if ((d->perm[i] >= nelt) != ((d->perm[0] >= nelt) ^ (i & 1)))
25094 : return false;
25095 :
25096 33532 : dfirst = *d;
25097 33532 : dsecond = *d;
25098 33532 : dfinal = *d;
25099 33532 : dfirst.op1 = dfirst.op0;
25100 33532 : dfirst.one_operand_p = true;
25101 33532 : dsecond.op0 = dsecond.op1;
25102 33532 : dsecond.one_operand_p = true;
25103 :
25104 219180 : for (i = 0; i < nelt; i++)
25105 185648 : if (d->perm[i] >= nelt)
25106 : {
25107 92824 : dsecond.perm[i / 2 + (i >= lane ? lane / 2 : 0)] = d->perm[i] - nelt;
25108 92824 : if (d->perm[i] - nelt != i / 2 + (i >= lane ? lane / 2 : 0))
25109 84488 : ident2 = false;
25110 92824 : dsecond.perm[i / 2 + (i >= lane ? lane : lane / 2)]
25111 92824 : = d->perm[i] - nelt;
25112 : }
25113 : else
25114 : {
25115 92824 : dfirst.perm[i / 2 + (i >= lane ? lane / 2 : 0)] = d->perm[i];
25116 92824 : if (d->perm[i] != i / 2 + (i >= lane ? lane / 2 : 0))
25117 76041 : ident1 = false;
25118 92824 : dfirst.perm[i / 2 + (i >= lane ? lane : lane / 2)] = d->perm[i];
25119 : }
25120 :
25121 33532 : if (two_insn && !ident1 && !ident2)
25122 : return false;
25123 :
25124 3899 : if (!d->testing_p)
25125 : {
25126 216 : if (!ident1)
25127 146 : dfinal.op0 = dfirst.target = gen_reg_rtx (d->vmode);
25128 216 : if (!ident2)
25129 148 : dfinal.op1 = dsecond.target = gen_reg_rtx (d->vmode);
25130 216 : if (d->perm[0] >= nelt)
25131 0 : std::swap (dfinal.op0, dfinal.op1);
25132 : }
25133 :
25134 3899 : bool ok;
25135 3899 : rtx_insn *seq1 = NULL, *seq2 = NULL;
25136 :
25137 3899 : if (!ident1)
25138 : {
25139 2587 : start_sequence ();
25140 2587 : ok = expand_vec_perm_1 (&dfirst);
25141 2587 : seq1 = end_sequence ();
25142 :
25143 2587 : if (!ok)
25144 : return false;
25145 : }
25146 :
25147 2174 : if (!ident2)
25148 : {
25149 2074 : start_sequence ();
25150 2074 : ok = expand_vec_perm_1 (&dsecond);
25151 2074 : seq2 = end_sequence ();
25152 :
25153 2074 : if (!ok)
25154 : return false;
25155 : }
25156 :
25157 608 : if (d->testing_p)
25158 : return true;
25159 :
25160 690 : for (i = 0; i < nelt; i++)
25161 : {
25162 552 : dfinal.perm[i] = i / 2;
25163 552 : if (i >= lane)
25164 4 : dfinal.perm[i] += lane / 2;
25165 552 : if ((i & 1) != 0)
25166 276 : dfinal.perm[i] += nelt;
25167 : }
25168 138 : emit_insn (seq1);
25169 138 : emit_insn (seq2);
25170 138 : ok = expand_vselect_vconcat (dfinal.target, dfinal.op0, dfinal.op1,
25171 : dfinal.perm, dfinal.nelt, false);
25172 138 : gcc_assert (ok);
25173 : return true;
25174 : }
25175 :
25176 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
25177 : the permutation using two single vector permutations and the SSE4_1 pblendv
25178 : instruction. If two_insn, succeed only if one of dfirst or dsecond is
25179 : identity permutation. */
25180 :
25181 : static bool
25182 118247 : expand_vec_perm_2perm_pblendv (struct expand_vec_perm_d *d, bool two_insn)
25183 : {
25184 118247 : unsigned i, nelt = d->nelt;
25185 118247 : struct expand_vec_perm_d dfirst, dsecond, dfinal;
25186 118247 : machine_mode vmode = d->vmode;
25187 118247 : bool ident1 = true, ident2 = true;
25188 :
25189 : /* Use the same checks as in expand_vec_perm_blend. */
25190 118247 : if (d->one_operand_p)
25191 : return false;
25192 110904 : if (TARGET_AVX2 && GET_MODE_SIZE (vmode) == 32)
25193 : ;
25194 105092 : else if (TARGET_AVX && (vmode == V4DFmode || vmode == V8SFmode))
25195 : ;
25196 100395 : else if (TARGET_SSE4_1
25197 110240 : && (GET_MODE_SIZE (vmode) == 16
25198 8980 : || (TARGET_MMX_WITH_SSE && GET_MODE_SIZE (vmode) == 8)
25199 2647 : || GET_MODE_SIZE (vmode) == 4))
25200 : ;
25201 : else
25202 : return false;
25203 :
25204 15776 : dfirst = *d;
25205 15776 : dsecond = *d;
25206 15776 : dfinal = *d;
25207 15776 : dfirst.op1 = dfirst.op0;
25208 15776 : dfirst.one_operand_p = true;
25209 15776 : dsecond.op0 = dsecond.op1;
25210 15776 : dsecond.one_operand_p = true;
25211 :
25212 116822 : for (i = 0; i < nelt; ++i)
25213 101046 : if (d->perm[i] >= nelt)
25214 : {
25215 51008 : dfirst.perm[i] = 0xff;
25216 51008 : dsecond.perm[i] = d->perm[i] - nelt;
25217 51008 : if (d->perm[i] != i + nelt)
25218 101046 : ident2 = false;
25219 : }
25220 : else
25221 : {
25222 50038 : dsecond.perm[i] = 0xff;
25223 50038 : dfirst.perm[i] = d->perm[i];
25224 50038 : if (d->perm[i] != i)
25225 101046 : ident1 = false;
25226 : }
25227 :
25228 15776 : if (two_insn && !ident1 && !ident2)
25229 : return false;
25230 :
25231 : /* For now. Ideally treat 0xff as a wildcard. */
25232 44021 : for (i = 0; i < nelt; ++i)
25233 38648 : if (dfirst.perm[i] == 0xff)
25234 : {
25235 20593 : if (GET_MODE_SIZE (vmode) == 32
25236 20593 : && dfirst.perm[i ^ (nelt / 2)] != 0xff)
25237 11603 : dfirst.perm[i] = dfirst.perm[i ^ (nelt / 2)] ^ (nelt / 2);
25238 : else
25239 8990 : dfirst.perm[i] = i;
25240 : }
25241 : else
25242 : {
25243 18055 : if (GET_MODE_SIZE (vmode) == 32
25244 18055 : && dsecond.perm[i ^ (nelt / 2)] != 0xff)
25245 9858 : dsecond.perm[i] = dsecond.perm[i ^ (nelt / 2)] ^ (nelt / 2);
25246 : else
25247 8197 : dsecond.perm[i] = i;
25248 : }
25249 :
25250 5373 : if (!d->testing_p)
25251 : {
25252 2135 : if (!ident1)
25253 2011 : dfinal.op0 = dfirst.target = gen_reg_rtx (d->vmode);
25254 2135 : if (!ident2)
25255 823 : dfinal.op1 = dsecond.target = gen_reg_rtx (d->vmode);
25256 : }
25257 :
25258 5373 : bool ok;
25259 5373 : rtx_insn *seq1 = NULL, *seq2 = NULL;
25260 :
25261 5373 : if (!ident1)
25262 : {
25263 4784 : start_sequence ();
25264 4784 : ok = expand_vec_perm_1 (&dfirst);
25265 4784 : seq1 = end_sequence ();
25266 :
25267 4784 : if (!ok)
25268 : return false;
25269 : }
25270 :
25271 3986 : if (!ident2)
25272 : {
25273 1101 : start_sequence ();
25274 1101 : ok = expand_vec_perm_1 (&dsecond);
25275 1101 : seq2 = end_sequence ();
25276 :
25277 1101 : if (!ok)
25278 : return false;
25279 : }
25280 :
25281 3397 : if (d->testing_p)
25282 : return true;
25283 :
25284 13749 : for (i = 0; i < nelt; ++i)
25285 11956 : dfinal.perm[i] = (d->perm[i] >= nelt ? i + nelt : i);
25286 :
25287 1793 : emit_insn (seq1);
25288 1793 : emit_insn (seq2);
25289 1793 : ok = expand_vec_perm_blend (&dfinal);
25290 1793 : gcc_assert (ok);
25291 : return true;
25292 : }
25293 :
25294 : /* A subroutine of ix86_expand_vec_perm_const_1.
25295 : Implement a permutation with psrlw, psllw and por.
25296 : It handles case:
25297 : __builtin_shufflevector (v,v,1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14);
25298 : __builtin_shufflevector (v,v,1,0,3,2,5,4,7,6); */
25299 :
25300 : static bool
25301 27525 : expand_vec_perm_psrlw_psllw_por (struct expand_vec_perm_d *d)
25302 : {
25303 27525 : unsigned i;
25304 27525 : rtx (*gen_shr) (rtx, rtx, rtx);
25305 27525 : rtx (*gen_shl) (rtx, rtx, rtx);
25306 27525 : rtx (*gen_or) (rtx, rtx, rtx);
25307 27525 : machine_mode mode = VOIDmode;
25308 :
25309 27525 : if (!TARGET_SSE2 || !d->one_operand_p)
25310 : return false;
25311 :
25312 5603 : switch (d->vmode)
25313 : {
25314 1478 : case E_V8QImode:
25315 1478 : if (!TARGET_MMX_WITH_SSE)
25316 : return false;
25317 : mode = V4HImode;
25318 : gen_shr = gen_lshrv4hi3;
25319 : gen_shl = gen_ashlv4hi3;
25320 : gen_or = gen_iorv4hi3;
25321 : break;
25322 : case E_V16QImode:
25323 : mode = V8HImode;
25324 : gen_shr = gen_lshrv8hi3;
25325 : gen_shl = gen_ashlv8hi3;
25326 : gen_or = gen_iorv8hi3;
25327 : break;
25328 : default: return false;
25329 : }
25330 :
25331 3280 : if (!rtx_equal_p (d->op0, d->op1))
25332 : return false;
25333 :
25334 12448 : for (i = 0; i < d->nelt; i += 2)
25335 10994 : if (d->perm[i] != i + 1 || d->perm[i + 1] != i)
25336 : return false;
25337 :
25338 1454 : if (d->testing_p)
25339 : return true;
25340 :
25341 30 : rtx tmp1 = gen_reg_rtx (mode);
25342 30 : rtx tmp2 = gen_reg_rtx (mode);
25343 30 : rtx op0 = force_reg (d->vmode, d->op0);
25344 :
25345 30 : emit_move_insn (tmp1, lowpart_subreg (mode, op0, d->vmode));
25346 30 : emit_move_insn (tmp2, lowpart_subreg (mode, op0, d->vmode));
25347 30 : emit_insn (gen_shr (tmp1, tmp1, GEN_INT (8)));
25348 30 : emit_insn (gen_shl (tmp2, tmp2, GEN_INT (8)));
25349 30 : emit_insn (gen_or (tmp1, tmp1, tmp2));
25350 30 : emit_move_insn (d->target, lowpart_subreg (d->vmode, tmp1, mode));
25351 :
25352 30 : return true;
25353 : }
25354 :
25355 : /* A subroutine of ix86_expand_vec_perm_const_1. Implement a V4DF
25356 : permutation using two vperm2f128, followed by a vshufpd insn blending
25357 : the two vectors together. */
25358 :
25359 : static bool
25360 31433 : expand_vec_perm_2vperm2f128_vshuf (struct expand_vec_perm_d *d)
25361 : {
25362 31433 : struct expand_vec_perm_d dfirst, dsecond, dthird;
25363 31433 : bool ok;
25364 :
25365 31433 : if (!TARGET_AVX || (d->vmode != V4DFmode))
25366 : return false;
25367 :
25368 1277 : if (d->testing_p)
25369 : return true;
25370 :
25371 206 : dfirst = *d;
25372 206 : dsecond = *d;
25373 206 : dthird = *d;
25374 :
25375 206 : dfirst.perm[0] = (d->perm[0] & ~1);
25376 206 : dfirst.perm[1] = (d->perm[0] & ~1) + 1;
25377 206 : dfirst.perm[2] = (d->perm[2] & ~1);
25378 206 : dfirst.perm[3] = (d->perm[2] & ~1) + 1;
25379 206 : dsecond.perm[0] = (d->perm[1] & ~1);
25380 206 : dsecond.perm[1] = (d->perm[1] & ~1) + 1;
25381 206 : dsecond.perm[2] = (d->perm[3] & ~1);
25382 206 : dsecond.perm[3] = (d->perm[3] & ~1) + 1;
25383 206 : dthird.perm[0] = (d->perm[0] % 2);
25384 206 : dthird.perm[1] = (d->perm[1] % 2) + 4;
25385 206 : dthird.perm[2] = (d->perm[2] % 2) + 2;
25386 206 : dthird.perm[3] = (d->perm[3] % 2) + 6;
25387 :
25388 206 : dfirst.target = gen_reg_rtx (dfirst.vmode);
25389 206 : dsecond.target = gen_reg_rtx (dsecond.vmode);
25390 206 : dthird.op0 = dfirst.target;
25391 206 : dthird.op1 = dsecond.target;
25392 206 : dthird.one_operand_p = false;
25393 :
25394 206 : canonicalize_perm (&dfirst);
25395 206 : canonicalize_perm (&dsecond);
25396 :
25397 206 : ok = expand_vec_perm_1 (&dfirst)
25398 206 : && expand_vec_perm_1 (&dsecond)
25399 412 : && expand_vec_perm_1 (&dthird);
25400 :
25401 0 : gcc_assert (ok);
25402 :
25403 : return true;
25404 : }
25405 :
25406 : static bool ix86_expand_vec_perm_const_1 (struct expand_vec_perm_d *);
25407 :
25408 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement
25409 : a two vector permutation using two intra-lane vector
25410 : permutations, vperm2f128 swapping the lanes and vblend* insn blending
25411 : the non-swapped and swapped vectors together. */
25412 :
25413 : static bool
25414 16568 : expand_vec_perm2_vperm2f128_vblend (struct expand_vec_perm_d *d)
25415 : {
25416 16568 : struct expand_vec_perm_d dfirst, dsecond, dthird;
25417 16568 : unsigned i, j, msk, nelt = d->nelt, nelt2 = nelt / 2, which1 = 0, which2 = 0;
25418 16568 : rtx_insn *seq1, *seq2;
25419 16568 : bool ok;
25420 16568 : rtx (*blend) (rtx, rtx, rtx, rtx) = NULL;
25421 :
25422 16568 : if (!TARGET_AVX
25423 794 : || TARGET_AVX2
25424 530 : || (d->vmode != V8SFmode && d->vmode != V4DFmode)
25425 403 : || d->one_operand_p)
25426 : return false;
25427 :
25428 403 : dfirst = *d;
25429 403 : dsecond = *d;
25430 3627 : for (i = 0; i < nelt; i++)
25431 : {
25432 3224 : dfirst.perm[i] = 0xff;
25433 3224 : dsecond.perm[i] = 0xff;
25434 : }
25435 3627 : for (i = 0, msk = 0; i < nelt; i++)
25436 : {
25437 3224 : j = (d->perm[i] & nelt2) ? i | nelt2 : i & ~nelt2;
25438 3224 : if (j == i)
25439 : {
25440 2498 : dfirst.perm[j] = d->perm[i];
25441 4322 : which1 |= (d->perm[i] < nelt ? 1 : 2);
25442 : }
25443 : else
25444 : {
25445 726 : dsecond.perm[j] = d->perm[i];
25446 726 : which2 |= (d->perm[i] < nelt ? 1 : 2);
25447 726 : msk |= (1U << i);
25448 : }
25449 : }
25450 403 : if (msk == 0 || msk == (1U << nelt) - 1)
25451 : return false;
25452 :
25453 403 : if (!d->testing_p)
25454 : {
25455 40 : dfirst.target = gen_reg_rtx (dfirst.vmode);
25456 40 : dsecond.target = gen_reg_rtx (dsecond.vmode);
25457 : }
25458 :
25459 3627 : for (i = 0; i < nelt; i++)
25460 : {
25461 3224 : if (dfirst.perm[i] == 0xff)
25462 726 : dfirst.perm[i] = (which1 == 2 ? i + nelt : i);
25463 3224 : if (dsecond.perm[i] == 0xff)
25464 2498 : dsecond.perm[i] = (which2 == 2 ? i + nelt : i);
25465 : }
25466 403 : canonicalize_perm (&dfirst);
25467 403 : start_sequence ();
25468 403 : ok = ix86_expand_vec_perm_const_1 (&dfirst);
25469 403 : seq1 = end_sequence ();
25470 :
25471 403 : if (!ok)
25472 : return false;
25473 :
25474 403 : canonicalize_perm (&dsecond);
25475 403 : start_sequence ();
25476 403 : ok = ix86_expand_vec_perm_const_1 (&dsecond);
25477 403 : seq2 = end_sequence ();
25478 :
25479 403 : if (!ok)
25480 : return false;
25481 :
25482 403 : if (d->testing_p)
25483 : return true;
25484 :
25485 40 : emit_insn (seq1);
25486 40 : emit_insn (seq2);
25487 :
25488 40 : dthird = *d;
25489 40 : dthird.op0 = dsecond.target;
25490 40 : dthird.op1 = dsecond.target;
25491 40 : dthird.one_operand_p = true;
25492 40 : dthird.target = gen_reg_rtx (dthird.vmode);
25493 360 : for (i = 0; i < nelt; i++)
25494 320 : dthird.perm[i] = i ^ nelt2;
25495 :
25496 40 : ok = expand_vec_perm_1 (&dthird);
25497 40 : gcc_assert (ok);
25498 :
25499 40 : blend = d->vmode == V8SFmode ? gen_avx_blendps256 : gen_avx_blendpd256;
25500 40 : emit_insn (blend (d->target, dfirst.target, dthird.target, GEN_INT (msk)));
25501 40 : return true;
25502 : }
25503 :
25504 : /* A subroutine of expand_vec_perm_even_odd_1. Implement the double-word
25505 : permutation with two pshufb insns and an ior. We should have already
25506 : failed all two instruction sequences. */
25507 :
25508 : static bool
25509 30177 : expand_vec_perm_pshufb2 (struct expand_vec_perm_d *d)
25510 : {
25511 30177 : rtx rperm[2][16], vperm, l, h, op, m128;
25512 30177 : unsigned int i, nelt, eltsz;
25513 30177 : machine_mode mode;
25514 30177 : rtx (*gen) (rtx, rtx, rtx);
25515 :
25516 34653 : if (!TARGET_SSSE3 || (GET_MODE_SIZE (d->vmode) != 16
25517 8862 : && GET_MODE_SIZE (d->vmode) != 8
25518 8822 : && GET_MODE_SIZE (d->vmode) != 4))
25519 : return false;
25520 1437 : gcc_assert (!d->one_operand_p);
25521 :
25522 1437 : if (d->testing_p)
25523 : return true;
25524 :
25525 202 : switch (GET_MODE_SIZE (d->vmode))
25526 : {
25527 : case 4:
25528 : mode = V4QImode;
25529 : gen = gen_mmx_pshufbv4qi3;
25530 : break;
25531 20 : case 8:
25532 20 : mode = V8QImode;
25533 20 : gen = gen_mmx_pshufbv8qi3;
25534 20 : break;
25535 45 : case 16:
25536 45 : mode = V16QImode;
25537 45 : gen = gen_ssse3_pshufbv16qi3;
25538 45 : break;
25539 0 : default:
25540 0 : gcc_unreachable ();
25541 : }
25542 :
25543 101 : nelt = d->nelt;
25544 101 : eltsz = GET_MODE_UNIT_SIZE (d->vmode);
25545 :
25546 : /* Generate two permutation masks. If the required element is within
25547 : the given vector it is shuffled into the proper lane. If the required
25548 : element is in the other vector, force a zero into the lane by setting
25549 : bit 7 in the permutation mask. */
25550 101 : m128 = GEN_INT (-128);
25551 1130 : for (i = 0; i < nelt; ++i)
25552 : {
25553 928 : unsigned j, k, e = d->perm[i];
25554 928 : unsigned which = (e >= nelt);
25555 928 : if (e >= nelt)
25556 480 : e -= nelt;
25557 :
25558 1952 : for (j = 0; j < eltsz; ++j)
25559 : {
25560 1024 : rperm[which][i*eltsz + j] = GEN_INT (e*eltsz + j);
25561 1024 : rperm[1-which][i*eltsz + j] = m128;
25562 : }
25563 :
25564 9024 : for (k = i*eltsz + j; k < 16; ++k)
25565 8096 : rperm[0][k] = rperm[1][k] = m128;
25566 : }
25567 :
25568 101 : vperm = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, rperm[0]));
25569 101 : vperm = force_reg (V16QImode, vperm);
25570 :
25571 101 : l = gen_reg_rtx (mode);
25572 101 : op = gen_lowpart (mode, d->op0);
25573 101 : emit_insn (gen (l, op, vperm));
25574 :
25575 101 : vperm = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, rperm[1]));
25576 101 : vperm = force_reg (V16QImode, vperm);
25577 :
25578 101 : h = gen_reg_rtx (mode);
25579 101 : op = gen_lowpart (mode, d->op1);
25580 101 : emit_insn (gen (h, op, vperm));
25581 :
25582 101 : op = d->target;
25583 101 : if (d->vmode != mode)
25584 22 : op = gen_reg_rtx (mode);
25585 101 : ix86_emit_vec_binop (IOR, mode, op, l, h);
25586 101 : if (op != d->target)
25587 22 : emit_move_insn (d->target, gen_lowpart (d->vmode, op));
25588 :
25589 : return true;
25590 : }
25591 :
25592 : /* Implement arbitrary permutation of one V32QImode and V16QImode operand
25593 : with two vpshufb insns, vpermq and vpor. We should have already failed
25594 : all two or three instruction sequences. */
25595 :
25596 : static bool
25597 24944 : expand_vec_perm_vpshufb2_vpermq (struct expand_vec_perm_d *d)
25598 : {
25599 24944 : rtx rperm[2][32], vperm, l, h, hp, op, m128;
25600 24944 : unsigned int i, nelt, eltsz;
25601 :
25602 24944 : if (!TARGET_AVX2
25603 374 : || !d->one_operand_p
25604 172 : || (d->vmode != V32QImode && d->vmode != V16HImode))
25605 : return false;
25606 :
25607 7 : if (d->testing_p)
25608 : return true;
25609 :
25610 7 : nelt = d->nelt;
25611 7 : eltsz = GET_MODE_UNIT_SIZE (d->vmode);
25612 :
25613 : /* Generate two permutation masks. If the required element is within
25614 : the same lane, it is shuffled in. If the required element from the
25615 : other lane, force a zero by setting bit 7 in the permutation mask.
25616 : In the other mask the mask has non-negative elements if element
25617 : is requested from the other lane, but also moved to the other lane,
25618 : so that the result of vpshufb can have the two V2TImode halves
25619 : swapped. */
25620 7 : m128 = GEN_INT (-128);
25621 206 : for (i = 0; i < nelt; ++i)
25622 : {
25623 192 : unsigned j, e = d->perm[i] & (nelt / 2 - 1);
25624 192 : unsigned which = ((d->perm[i] ^ i) & (nelt / 2)) * eltsz;
25625 :
25626 416 : for (j = 0; j < eltsz; ++j)
25627 : {
25628 224 : rperm[!!which][(i * eltsz + j) ^ which] = GEN_INT (e * eltsz + j);
25629 224 : rperm[!which][(i * eltsz + j) ^ (which ^ 16)] = m128;
25630 : }
25631 : }
25632 :
25633 7 : vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[1]));
25634 7 : vperm = force_reg (V32QImode, vperm);
25635 :
25636 7 : h = gen_reg_rtx (V32QImode);
25637 7 : op = gen_lowpart (V32QImode, d->op0);
25638 7 : emit_insn (gen_avx2_pshufbv32qi3 (h, op, vperm));
25639 :
25640 : /* Swap the 128-byte lanes of h into hp. */
25641 7 : hp = gen_reg_rtx (V4DImode);
25642 7 : op = gen_lowpart (V4DImode, h);
25643 7 : emit_insn (gen_avx2_permv4di_1 (hp, op, const2_rtx, GEN_INT (3), const0_rtx,
25644 : const1_rtx));
25645 :
25646 7 : vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[0]));
25647 7 : vperm = force_reg (V32QImode, vperm);
25648 :
25649 7 : l = gen_reg_rtx (V32QImode);
25650 7 : op = gen_lowpart (V32QImode, d->op0);
25651 7 : emit_insn (gen_avx2_pshufbv32qi3 (l, op, vperm));
25652 :
25653 7 : op = d->target;
25654 7 : if (d->vmode != V32QImode)
25655 2 : op = gen_reg_rtx (V32QImode);
25656 7 : emit_insn (gen_iorv32qi3 (op, l, gen_lowpart (V32QImode, hp)));
25657 7 : if (op != d->target)
25658 2 : emit_move_insn (d->target, gen_lowpart (d->vmode, op));
25659 :
25660 : return true;
25661 : }
25662 :
25663 : /* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
25664 : and extract-odd permutations of two V32QImode and V16QImode operand
25665 : with two vpshufb insns, vpor and vpermq. We should have already
25666 : failed all two or three instruction sequences. */
25667 :
25668 : static bool
25669 24937 : expand_vec_perm_vpshufb2_vpermq_even_odd (struct expand_vec_perm_d *d)
25670 : {
25671 24937 : rtx rperm[2][32], vperm, l, h, ior, op, m128;
25672 24937 : unsigned int i, nelt, eltsz;
25673 :
25674 24937 : if (!TARGET_AVX2
25675 367 : || d->one_operand_p
25676 202 : || (d->vmode != V32QImode && d->vmode != V16HImode))
25677 : return false;
25678 :
25679 112 : for (i = 0; i < d->nelt; ++i)
25680 112 : if ((d->perm[i] ^ (i * 2)) & (3 * d->nelt / 2))
25681 : return false;
25682 :
25683 0 : if (d->testing_p)
25684 : return true;
25685 :
25686 0 : nelt = d->nelt;
25687 0 : eltsz = GET_MODE_UNIT_SIZE (d->vmode);
25688 :
25689 : /* Generate two permutation masks. In the first permutation mask
25690 : the first quarter will contain indexes for the first half
25691 : of the op0, the second quarter will contain bit 7 set, third quarter
25692 : will contain indexes for the second half of the op0 and the
25693 : last quarter bit 7 set. In the second permutation mask
25694 : the first quarter will contain bit 7 set, the second quarter
25695 : indexes for the first half of the op1, the third quarter bit 7 set
25696 : and last quarter indexes for the second half of the op1.
25697 : I.e. the first mask e.g. for V32QImode extract even will be:
25698 : 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128
25699 : (all values masked with 0xf except for -128) and second mask
25700 : for extract even will be
25701 : -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe. */
25702 0 : m128 = GEN_INT (-128);
25703 0 : for (i = 0; i < nelt; ++i)
25704 : {
25705 0 : unsigned j, e = d->perm[i] & (nelt / 2 - 1);
25706 0 : unsigned which = d->perm[i] >= nelt;
25707 0 : unsigned xorv = (i >= nelt / 4 && i < 3 * nelt / 4) ? 24 : 0;
25708 :
25709 0 : for (j = 0; j < eltsz; ++j)
25710 : {
25711 0 : rperm[which][(i * eltsz + j) ^ xorv] = GEN_INT (e * eltsz + j);
25712 0 : rperm[1 - which][(i * eltsz + j) ^ xorv] = m128;
25713 : }
25714 : }
25715 :
25716 0 : vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[0]));
25717 0 : vperm = force_reg (V32QImode, vperm);
25718 :
25719 0 : l = gen_reg_rtx (V32QImode);
25720 0 : op = gen_lowpart (V32QImode, d->op0);
25721 0 : emit_insn (gen_avx2_pshufbv32qi3 (l, op, vperm));
25722 :
25723 0 : vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[1]));
25724 0 : vperm = force_reg (V32QImode, vperm);
25725 :
25726 0 : h = gen_reg_rtx (V32QImode);
25727 0 : op = gen_lowpart (V32QImode, d->op1);
25728 0 : emit_insn (gen_avx2_pshufbv32qi3 (h, op, vperm));
25729 :
25730 0 : ior = gen_reg_rtx (V32QImode);
25731 0 : emit_insn (gen_iorv32qi3 (ior, l, h));
25732 :
25733 : /* Permute the V4DImode quarters using { 0, 2, 1, 3 } permutation. */
25734 0 : op = gen_reg_rtx (V4DImode);
25735 0 : ior = gen_lowpart (V4DImode, ior);
25736 0 : emit_insn (gen_avx2_permv4di_1 (op, ior, const0_rtx, const2_rtx,
25737 : const1_rtx, GEN_INT (3)));
25738 0 : emit_move_insn (d->target, gen_lowpart (d->vmode, op));
25739 :
25740 0 : return true;
25741 : }
25742 :
25743 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement a
25744 : permutation (which is a bland) with and, andnot and or when pshufb is not available.
25745 :
25746 : It handles case:
25747 : __builtin_shufflevector (v1, v2, 0, 9, 2, 11, 4, 13, 6, 15);
25748 : __builtin_shufflevector (v1, v2, 8, 1, 2, 11, 4, 13, 6, 15);
25749 :
25750 : An element[i] must be chosen between op0[i] and op1[i] to satisfy the
25751 : requirement.
25752 : */
25753 :
25754 : static bool
25755 26071 : expand_vec_perm_pand_pandn_por (struct expand_vec_perm_d *d)
25756 : {
25757 26071 : rtx rperm[16], vperm;
25758 26071 : unsigned int i, nelt = d->nelt;
25759 :
25760 26071 : if (!TARGET_SSE2
25761 26071 : || d->one_operand_p
25762 21922 : || (d->vmode != V16QImode && d->vmode != V8HImode))
25763 : return false;
25764 :
25765 8168 : if (d->perm[0] != 0)
25766 : return false;
25767 :
25768 : /* The dest[i] must select an element between op0[i] and op1[i]. */
25769 17343 : for (i = 1; i < nelt; i++)
25770 16220 : if ((d->perm[i] % nelt) != i)
25771 : return false;
25772 :
25773 1123 : if (d->testing_p)
25774 : return true;
25775 :
25776 : /* Generates a blend mask for the operators AND and ANDNOT. */
25777 134 : machine_mode inner_mode = GET_MODE_INNER (d->vmode);
25778 1526 : for (i = 0; i < nelt; i++)
25779 1999 : rperm[i] = (d->perm[i] < nelt) ? CONSTM1_RTX (inner_mode)
25780 607 : : CONST0_RTX (inner_mode);
25781 :
25782 134 : vperm = gen_rtx_CONST_VECTOR (d->vmode, gen_rtvec_v (nelt, rperm));
25783 134 : vperm = force_reg (d->vmode, vperm);
25784 :
25785 134 : ix86_expand_sse_movcc (d->target, vperm, d->op0, d->op1);
25786 :
25787 134 : return true;
25788 : }
25789 :
25790 : /* Implement permutation with pslldq + psrldq + por when pshufb is not
25791 : available. */
25792 : static bool
25793 45698 : expand_vec_perm_pslldq_psrldq_por (struct expand_vec_perm_d *d, bool pandn)
25794 : {
25795 45698 : unsigned i, nelt = d->nelt;
25796 45698 : unsigned start1, end1 = -1;
25797 45698 : machine_mode vmode = d->vmode, imode;
25798 45698 : int start2 = -1;
25799 45698 : bool clear_op0, clear_op1;
25800 45698 : unsigned inner_size;
25801 45698 : rtx op0, op1, dop1;
25802 45698 : rtx (*gen_vec_shr) (rtx, rtx, rtx);
25803 45698 : rtx (*gen_vec_shl) (rtx, rtx, rtx);
25804 :
25805 : /* pshufd can be used for V4SI/V2DI under TARGET_SSE2. */
25806 45698 : if (!TARGET_SSE2 || (vmode != E_V16QImode && vmode != E_V8HImode))
25807 : return false;
25808 :
25809 14797 : start1 = d->perm[0];
25810 46490 : for (i = 1; i < nelt; i++)
25811 : {
25812 45483 : if (d->perm[i] != d->perm[i-1] + 1
25813 17456 : || d->perm[i] == nelt)
25814 : {
25815 28587 : if (start2 == -1)
25816 : {
25817 14797 : start2 = d->perm[i];
25818 14797 : end1 = d->perm[i-1];
25819 : }
25820 : else
25821 : return false;
25822 : }
25823 : }
25824 :
25825 1007 : clear_op0 = end1 != nelt - 1;
25826 1007 : clear_op1 = start2 % nelt != 0;
25827 : /* pandn/pand is needed to clear upper/lower bits of op0/op1. */
25828 1007 : if (!pandn && (clear_op0 || clear_op1))
25829 : return false;
25830 :
25831 653 : if (d->testing_p)
25832 : return true;
25833 :
25834 89 : gen_vec_shr = vmode == E_V16QImode ? gen_vec_shr_v16qi : gen_vec_shr_v8hi;
25835 28 : gen_vec_shl = vmode == E_V16QImode ? gen_vec_shl_v16qi : gen_vec_shl_v8hi;
25836 89 : imode = GET_MODE_INNER (vmode);
25837 89 : inner_size = GET_MODE_BITSIZE (imode);
25838 89 : op0 = gen_reg_rtx (vmode);
25839 89 : op1 = gen_reg_rtx (vmode);
25840 :
25841 89 : if (start1)
25842 84 : emit_insn (gen_vec_shr (op0, d->op0, GEN_INT (start1 * inner_size)));
25843 : else
25844 5 : emit_move_insn (op0, d->op0);
25845 :
25846 89 : dop1 = d->op1;
25847 89 : if (d->one_operand_p)
25848 64 : dop1 = d->op0;
25849 :
25850 89 : int shl_offset = end1 - start1 + 1 - start2 % nelt;
25851 89 : if (shl_offset)
25852 49 : emit_insn (gen_vec_shl (op1, dop1, GEN_INT (shl_offset * inner_size)));
25853 : else
25854 40 : emit_move_insn (op1, dop1);
25855 :
25856 : /* Clear lower/upper bits for op0/op1. */
25857 89 : if (clear_op0 || clear_op1)
25858 : {
25859 : rtx vec[16];
25860 : rtx const_vec;
25861 : rtx clear;
25862 989 : for (i = 0; i != nelt; i++)
25863 : {
25864 920 : if (i < (end1 - start1 + 1))
25865 346 : vec[i] = gen_int_mode ((HOST_WIDE_INT_1U << inner_size) - 1, imode);
25866 : else
25867 574 : vec[i] = CONST0_RTX (imode);
25868 : }
25869 69 : const_vec = gen_rtx_CONST_VECTOR (vmode, gen_rtvec_v (nelt, vec));
25870 69 : const_vec = validize_mem (force_const_mem (vmode, const_vec));
25871 69 : clear = force_reg (vmode, const_vec);
25872 :
25873 69 : if (clear_op0)
25874 61 : emit_move_insn (op0, gen_rtx_AND (vmode, op0, clear));
25875 69 : if (clear_op1)
25876 56 : emit_move_insn (op1, gen_rtx_AND (vmode,
25877 : gen_rtx_NOT (vmode, clear),
25878 : op1));
25879 : }
25880 :
25881 89 : emit_move_insn (d->target, gen_rtx_IOR (vmode, op0, op1));
25882 89 : return true;
25883 : }
25884 :
25885 : /* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
25886 : and extract-odd permutations of two V8QI, V8HI, V16QI, V16HI or V32QI
25887 : operands with two "and" and "pack" or two "shift" and "pack" insns.
25888 : We should have already failed all two instruction sequences. */
25889 :
25890 : static bool
25891 47677 : expand_vec_perm_even_odd_pack (struct expand_vec_perm_d *d)
25892 : {
25893 47677 : rtx op, dop0, dop1, t;
25894 47677 : unsigned i, odd, c, s, nelt = d->nelt;
25895 47677 : int pblendw_i = 0;
25896 47677 : bool end_perm = false;
25897 47677 : machine_mode half_mode;
25898 47677 : rtx (*gen_and) (rtx, rtx, rtx);
25899 47677 : rtx (*gen_pack) (rtx, rtx, rtx);
25900 47677 : rtx (*gen_shift) (rtx, rtx, rtx);
25901 :
25902 47677 : if (d->one_operand_p)
25903 : return false;
25904 :
25905 42031 : switch (d->vmode)
25906 : {
25907 4642 : case E_V4HImode:
25908 : /* Required for "pack". */
25909 4642 : if (!TARGET_SSE4_1)
25910 : return false;
25911 : c = 0xffff;
25912 : s = 16;
25913 : half_mode = V2SImode;
25914 : gen_and = gen_andv2si3;
25915 : gen_pack = gen_mmx_packusdw;
25916 : gen_shift = gen_lshrv2si3;
25917 : pblendw_i = 0x5;
25918 : break;
25919 6029 : case E_V8HImode:
25920 : /* Required for "pack". */
25921 6029 : if (!TARGET_SSE4_1)
25922 : return false;
25923 : c = 0xffff;
25924 : s = 16;
25925 : half_mode = V4SImode;
25926 : gen_and = gen_andv4si3;
25927 : gen_pack = gen_sse4_1_packusdw;
25928 : gen_shift = gen_lshrv4si3;
25929 : pblendw_i = 0x55;
25930 : break;
25931 : case E_V8QImode:
25932 : /* No check as all instructions are SSE2. */
25933 : c = 0xff;
25934 : s = 8;
25935 : half_mode = V4HImode;
25936 : gen_and = gen_andv4hi3;
25937 : gen_pack = gen_mmx_packuswb;
25938 : gen_shift = gen_lshrv4hi3;
25939 : break;
25940 14768 : case E_V16QImode:
25941 : /* No check as all instructions are SSE2. */
25942 14768 : c = 0xff;
25943 14768 : s = 8;
25944 14768 : half_mode = V8HImode;
25945 14768 : gen_and = gen_andv8hi3;
25946 14768 : gen_pack = gen_sse2_packuswb;
25947 14768 : gen_shift = gen_lshrv8hi3;
25948 14768 : break;
25949 440 : case E_V16HImode:
25950 440 : if (!TARGET_AVX2)
25951 : return false;
25952 : c = 0xffff;
25953 : s = 16;
25954 : half_mode = V8SImode;
25955 : gen_and = gen_andv8si3;
25956 : gen_pack = gen_avx2_packusdw;
25957 : gen_shift = gen_lshrv8si3;
25958 : pblendw_i = 0x5555;
25959 : end_perm = true;
25960 : break;
25961 268 : case E_V32QImode:
25962 268 : if (!TARGET_AVX2)
25963 : return false;
25964 : c = 0xff;
25965 : s = 8;
25966 : half_mode = V16HImode;
25967 : gen_and = gen_andv16hi3;
25968 : gen_pack = gen_avx2_packuswb;
25969 : gen_shift = gen_lshrv16hi3;
25970 : end_perm = true;
25971 : break;
25972 : default:
25973 : /* Only V4HI, V8QI, V8HI, V16QI, V16HI and V32QI modes
25974 : are more profitable than general shuffles. */
25975 : return false;
25976 : }
25977 :
25978 : /* Check that permutation is even or odd. */
25979 20588 : odd = d->perm[0];
25980 20588 : if (odd > 1)
25981 : return false;
25982 :
25983 231632 : for (i = 1; i < nelt; ++i)
25984 215448 : if (d->perm[i] != 2 * i + odd)
25985 : return false;
25986 :
25987 16184 : if (d->testing_p)
25988 : return true;
25989 :
25990 5566 : dop0 = gen_reg_rtx (half_mode);
25991 5566 : dop1 = gen_reg_rtx (half_mode);
25992 5566 : if (odd == 0)
25993 : {
25994 : /* Use pblendw since const_vector 0 should be cheaper than
25995 : const_vector 0xffff. */
25996 4845 : if (d->vmode == V4HImode
25997 : || d->vmode == E_V8HImode
25998 : || d->vmode == E_V16HImode)
25999 : {
26000 872 : rtx dop0_t = gen_reg_rtx (d->vmode);
26001 872 : rtx dop1_t = gen_reg_rtx (d->vmode);
26002 872 : t = gen_reg_rtx (d->vmode);
26003 872 : emit_move_insn (t, CONST0_RTX (d->vmode));
26004 :
26005 872 : emit_move_insn (dop0_t, gen_rtx_VEC_MERGE (d->vmode, d->op0, t,
26006 : GEN_INT (pblendw_i)));
26007 872 : emit_move_insn (dop1_t, gen_rtx_VEC_MERGE (d->vmode, d->op1, t,
26008 : GEN_INT (pblendw_i)));
26009 :
26010 872 : emit_move_insn (dop0, gen_lowpart (half_mode, dop0_t));
26011 872 : emit_move_insn (dop1, gen_lowpart (half_mode, dop1_t));
26012 872 : }
26013 : else
26014 : {
26015 3973 : t = gen_const_vec_duplicate (half_mode, GEN_INT (c));
26016 3973 : t = force_reg (half_mode, t);
26017 3973 : emit_insn (gen_and (dop0, t, gen_lowpart (half_mode, d->op0)));
26018 3973 : emit_insn (gen_and (dop1, t, gen_lowpart (half_mode, d->op1)));
26019 : }
26020 : }
26021 : else
26022 : {
26023 1442 : emit_insn (gen_shift (dop0,
26024 721 : gen_lowpart (half_mode, d->op0),
26025 : GEN_INT (s)));
26026 1442 : emit_insn (gen_shift (dop1,
26027 721 : gen_lowpart (half_mode, d->op1),
26028 : GEN_INT (s)));
26029 : }
26030 : /* In AVX2 for 256 bit case we need to permute pack result. */
26031 5566 : if (TARGET_AVX2 && end_perm)
26032 : {
26033 411 : op = gen_reg_rtx (d->vmode);
26034 411 : t = gen_reg_rtx (V4DImode);
26035 411 : emit_insn (gen_pack (op, dop0, dop1));
26036 822 : emit_insn (gen_avx2_permv4di_1 (t,
26037 411 : gen_lowpart (V4DImode, op),
26038 : const0_rtx,
26039 : const2_rtx,
26040 : const1_rtx,
26041 : GEN_INT (3)));
26042 411 : emit_move_insn (d->target, gen_lowpart (d->vmode, t));
26043 : }
26044 : else
26045 5155 : emit_insn (gen_pack (d->target, dop0, dop1));
26046 :
26047 : return true;
26048 : }
26049 :
26050 : /* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
26051 : and extract-odd permutations of two V64QI operands
26052 : with two "shifts", two "truncs" and one "concat" insns for "odd"
26053 : and two "truncs" and one concat insn for "even."
26054 : Have already failed all two instruction sequences. */
26055 :
26056 : static bool
26057 24973 : expand_vec_perm_even_odd_trunc (struct expand_vec_perm_d *d)
26058 : {
26059 24973 : rtx t1, t2, t3, t4;
26060 24973 : unsigned i, odd, nelt = d->nelt;
26061 :
26062 24973 : if (!TARGET_AVX512BW
26063 66 : || d->one_operand_p
26064 30 : || d->vmode != V64QImode)
26065 : return false;
26066 :
26067 : /* Check that permutation is even or odd. */
26068 30 : odd = d->perm[0];
26069 30 : if (odd > 1)
26070 : return false;
26071 :
26072 1150 : for (i = 1; i < nelt; ++i)
26073 1133 : if (d->perm[i] != 2 * i + odd)
26074 : return false;
26075 :
26076 17 : if (d->testing_p)
26077 : return true;
26078 :
26079 :
26080 17 : if (odd)
26081 : {
26082 1 : t1 = gen_reg_rtx (V32HImode);
26083 1 : t2 = gen_reg_rtx (V32HImode);
26084 2 : emit_insn (gen_lshrv32hi3 (t1,
26085 1 : gen_lowpart (V32HImode, d->op0),
26086 : GEN_INT (8)));
26087 2 : emit_insn (gen_lshrv32hi3 (t2,
26088 1 : gen_lowpart (V32HImode, d->op1),
26089 : GEN_INT (8)));
26090 : }
26091 : else
26092 : {
26093 16 : t1 = gen_lowpart (V32HImode, d->op0);
26094 16 : t2 = gen_lowpart (V32HImode, d->op1);
26095 : }
26096 :
26097 17 : t3 = gen_reg_rtx (V32QImode);
26098 17 : t4 = gen_reg_rtx (V32QImode);
26099 17 : emit_insn (gen_avx512bw_truncatev32hiv32qi2 (t3, t1));
26100 17 : emit_insn (gen_avx512bw_truncatev32hiv32qi2 (t4, t2));
26101 17 : emit_insn (gen_avx_vec_concatv64qi (d->target, t3, t4));
26102 :
26103 17 : return true;
26104 : }
26105 :
26106 : /* A subroutine of ix86_expand_vec_perm_const_1. Implement extract-even
26107 : and extract-odd permutations. */
26108 :
26109 : static bool
26110 13126 : expand_vec_perm_even_odd_1 (struct expand_vec_perm_d *d, unsigned odd)
26111 : {
26112 13126 : rtx t1, t2, t3, t4, t5;
26113 :
26114 13126 : switch (d->vmode)
26115 : {
26116 19 : case E_V4DFmode:
26117 19 : if (d->testing_p)
26118 : break;
26119 1 : t1 = gen_reg_rtx (V4DFmode);
26120 1 : t2 = gen_reg_rtx (V4DFmode);
26121 :
26122 : /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
26123 1 : emit_insn (gen_avx_vperm2f128v4df3 (t1, d->op0, d->op1, GEN_INT (0x20)));
26124 1 : emit_insn (gen_avx_vperm2f128v4df3 (t2, d->op0, d->op1, GEN_INT (0x31)));
26125 :
26126 : /* Now an unpck[lh]pd will produce the result required. */
26127 1 : if (odd)
26128 0 : t3 = gen_avx_unpckhpd256 (d->target, t1, t2);
26129 : else
26130 1 : t3 = gen_avx_unpcklpd256 (d->target, t1, t2);
26131 1 : emit_insn (t3);
26132 1 : break;
26133 :
26134 1214 : case E_V8SFmode:
26135 1214 : {
26136 1214 : int mask = odd ? 0xdd : 0x88;
26137 :
26138 1214 : if (d->testing_p)
26139 : break;
26140 186 : t1 = gen_reg_rtx (V8SFmode);
26141 186 : t2 = gen_reg_rtx (V8SFmode);
26142 186 : t3 = gen_reg_rtx (V8SFmode);
26143 :
26144 : /* Shuffle within the 128-bit lanes to produce:
26145 : { 0 2 8 a 4 6 c e } | { 1 3 9 b 5 7 d f }. */
26146 186 : emit_insn (gen_avx_shufps256 (t1, d->op0, d->op1,
26147 : GEN_INT (mask)));
26148 :
26149 : /* Shuffle the lanes around to produce:
26150 : { 4 6 c e 0 2 8 a } and { 5 7 d f 1 3 9 b }. */
26151 186 : emit_insn (gen_avx_vperm2f128v8sf3 (t2, t1, t1,
26152 : GEN_INT (0x3)));
26153 :
26154 : /* Shuffle within the 128-bit lanes to produce:
26155 : { 0 2 4 6 4 6 0 2 } | { 1 3 5 7 5 7 1 3 }. */
26156 186 : emit_insn (gen_avx_shufps256 (t3, t1, t2, GEN_INT (0x44)));
26157 :
26158 : /* Shuffle within the 128-bit lanes to produce:
26159 : { 8 a c e c e 8 a } | { 9 b d f d f 9 b }. */
26160 186 : emit_insn (gen_avx_shufps256 (t2, t1, t2, GEN_INT (0xee)));
26161 :
26162 : /* Shuffle the lanes around to produce:
26163 : { 0 2 4 6 8 a c e } | { 1 3 5 7 9 b d f }. */
26164 186 : emit_insn (gen_avx_vperm2f128v8sf3 (d->target, t3, t2,
26165 : GEN_INT (0x20)));
26166 : }
26167 186 : break;
26168 :
26169 0 : case E_V2DFmode:
26170 0 : case E_V4SFmode:
26171 0 : case E_V2DImode:
26172 0 : case E_V2SImode:
26173 0 : case E_V4SImode:
26174 0 : case E_V2HImode:
26175 : /* These are always directly implementable by expand_vec_perm_1. */
26176 0 : gcc_unreachable ();
26177 :
26178 0 : case E_V2SFmode:
26179 0 : gcc_assert (TARGET_MMX_WITH_SSE);
26180 : /* We have no suitable instructions. */
26181 0 : if (d->testing_p)
26182 0 : return false;
26183 : break;
26184 :
26185 1590 : case E_V4QImode:
26186 1590 : if (TARGET_SSSE3 && !TARGET_SLOW_PSHUFB)
26187 0 : return expand_vec_perm_pshufb2 (d);
26188 : else
26189 : {
26190 1590 : if (d->testing_p)
26191 : break;
26192 : /* We need 2*log2(N)-1 operations to achieve odd/even
26193 : with interleave. */
26194 178 : t1 = gen_reg_rtx (V4QImode);
26195 178 : emit_insn (gen_mmx_punpckhbw_low (t1, d->op0, d->op1));
26196 178 : emit_insn (gen_mmx_punpcklbw_low (d->target, d->op0, d->op1));
26197 178 : if (odd)
26198 41 : t2 = gen_mmx_punpckhbw_low (d->target, d->target, t1);
26199 : else
26200 137 : t2 = gen_mmx_punpcklbw_low (d->target, d->target, t1);
26201 178 : emit_insn (t2);
26202 : }
26203 178 : break;
26204 :
26205 1536 : case E_V4HImode:
26206 1536 : if (TARGET_SSE4_1)
26207 92 : return expand_vec_perm_even_odd_pack (d);
26208 1444 : else if (TARGET_SSSE3 && !TARGET_SLOW_PSHUFB)
26209 20 : return expand_vec_perm_pshufb2 (d);
26210 : else
26211 : {
26212 1424 : if (d->testing_p)
26213 : break;
26214 : /* We need 2*log2(N)-1 operations to achieve odd/even
26215 : with interleave. */
26216 451 : t1 = gen_reg_rtx (V4HImode);
26217 451 : emit_insn (gen_mmx_punpckhwd (t1, d->op0, d->op1));
26218 451 : emit_insn (gen_mmx_punpcklwd (d->target, d->op0, d->op1));
26219 451 : if (odd)
26220 8 : t2 = gen_mmx_punpckhwd (d->target, d->target, t1);
26221 : else
26222 443 : t2 = gen_mmx_punpcklwd (d->target, d->target, t1);
26223 451 : emit_insn (t2);
26224 : }
26225 451 : break;
26226 :
26227 6724 : case E_V8HImode:
26228 6724 : if (TARGET_SSE4_1)
26229 440 : return expand_vec_perm_even_odd_pack (d);
26230 6284 : else if (TARGET_SSSE3 && !TARGET_SLOW_PSHUFB)
26231 1 : return expand_vec_perm_pshufb2 (d);
26232 : else
26233 : {
26234 6283 : if (d->testing_p)
26235 : break;
26236 : /* We need 2*log2(N)-1 operations to achieve odd/even
26237 : with interleave. */
26238 2775 : t1 = gen_reg_rtx (V8HImode);
26239 2775 : t2 = gen_reg_rtx (V8HImode);
26240 2775 : emit_insn (gen_vec_interleave_highv8hi (t1, d->op0, d->op1));
26241 2775 : emit_insn (gen_vec_interleave_lowv8hi (d->target, d->op0, d->op1));
26242 2775 : emit_insn (gen_vec_interleave_highv8hi (t2, d->target, t1));
26243 2775 : emit_insn (gen_vec_interleave_lowv8hi (d->target, d->target, t1));
26244 2775 : if (odd)
26245 91 : t3 = gen_vec_interleave_highv8hi (d->target, d->target, t2);
26246 : else
26247 2684 : t3 = gen_vec_interleave_lowv8hi (d->target, d->target, t2);
26248 2775 : emit_insn (t3);
26249 : }
26250 2775 : break;
26251 :
26252 1356 : case E_V8QImode:
26253 1356 : case E_V16QImode:
26254 1356 : return expand_vec_perm_even_odd_pack (d);
26255 :
26256 467 : case E_V16HImode:
26257 467 : case E_V32QImode:
26258 467 : return expand_vec_perm_even_odd_pack (d);
26259 :
26260 25 : case E_V64QImode:
26261 25 : return expand_vec_perm_even_odd_trunc (d);
26262 :
26263 19 : case E_V4DImode:
26264 19 : if (!TARGET_AVX2)
26265 : {
26266 19 : struct expand_vec_perm_d d_copy = *d;
26267 19 : d_copy.vmode = V4DFmode;
26268 19 : if (d->testing_p)
26269 18 : d_copy.target = gen_raw_REG (V4DFmode, LAST_VIRTUAL_REGISTER + 1);
26270 : else
26271 1 : d_copy.target = gen_reg_rtx (V4DFmode);
26272 19 : d_copy.op0 = gen_lowpart (V4DFmode, d->op0);
26273 19 : d_copy.op1 = gen_lowpart (V4DFmode, d->op1);
26274 19 : if (expand_vec_perm_even_odd_1 (&d_copy, odd))
26275 : {
26276 19 : if (!d->testing_p)
26277 1 : emit_move_insn (d->target,
26278 1 : gen_lowpart (V4DImode, d_copy.target));
26279 : return true;
26280 : }
26281 : return false;
26282 : }
26283 :
26284 0 : if (d->testing_p)
26285 : break;
26286 :
26287 0 : t1 = gen_reg_rtx (V4DImode);
26288 0 : t2 = gen_reg_rtx (V4DImode);
26289 :
26290 : /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
26291 0 : emit_insn (gen_avx2_permv2ti (t1, d->op0, d->op1, GEN_INT (0x20)));
26292 0 : emit_insn (gen_avx2_permv2ti (t2, d->op0, d->op1, GEN_INT (0x31)));
26293 :
26294 : /* Now an vpunpck[lh]qdq will produce the result required. */
26295 0 : if (odd)
26296 0 : t3 = gen_avx2_interleave_highv4di (d->target, t1, t2);
26297 : else
26298 0 : t3 = gen_avx2_interleave_lowv4di (d->target, t1, t2);
26299 0 : emit_insn (t3);
26300 0 : break;
26301 :
26302 176 : case E_V8SImode:
26303 176 : if (!TARGET_AVX2)
26304 : {
26305 38 : struct expand_vec_perm_d d_copy = *d;
26306 38 : d_copy.vmode = V8SFmode;
26307 38 : if (d->testing_p)
26308 38 : d_copy.target = gen_raw_REG (V8SFmode, LAST_VIRTUAL_REGISTER + 1);
26309 : else
26310 0 : d_copy.target = gen_reg_rtx (V8SFmode);
26311 38 : d_copy.op0 = gen_lowpart (V8SFmode, d->op0);
26312 38 : d_copy.op1 = gen_lowpart (V8SFmode, d->op1);
26313 38 : if (expand_vec_perm_even_odd_1 (&d_copy, odd))
26314 : {
26315 38 : if (!d->testing_p)
26316 0 : emit_move_insn (d->target,
26317 0 : gen_lowpart (V8SImode, d_copy.target));
26318 : return true;
26319 : }
26320 : return false;
26321 : }
26322 :
26323 138 : if (d->testing_p)
26324 : break;
26325 :
26326 138 : t1 = gen_reg_rtx (V8SImode);
26327 138 : t2 = gen_reg_rtx (V8SImode);
26328 138 : t3 = gen_reg_rtx (V4DImode);
26329 138 : t4 = gen_reg_rtx (V4DImode);
26330 138 : t5 = gen_reg_rtx (V4DImode);
26331 :
26332 : /* Shuffle the lanes around into
26333 : { 0 1 2 3 8 9 a b } and { 4 5 6 7 c d e f }. */
26334 276 : emit_insn (gen_avx2_permv2ti (t3, gen_lowpart (V4DImode, d->op0),
26335 138 : gen_lowpart (V4DImode, d->op1),
26336 : GEN_INT (0x20)));
26337 276 : emit_insn (gen_avx2_permv2ti (t4, gen_lowpart (V4DImode, d->op0),
26338 138 : gen_lowpart (V4DImode, d->op1),
26339 : GEN_INT (0x31)));
26340 :
26341 : /* Swap the 2nd and 3rd position in each lane into
26342 : { 0 2 1 3 8 a 9 b } and { 4 6 5 7 c e d f }. */
26343 138 : emit_insn (gen_avx2_pshufdv3 (t1, gen_lowpart (V8SImode, t3),
26344 : GEN_INT (2 * 4 + 1 * 16 + 3 * 64)));
26345 138 : emit_insn (gen_avx2_pshufdv3 (t2, gen_lowpart (V8SImode, t4),
26346 : GEN_INT (2 * 4 + 1 * 16 + 3 * 64)));
26347 :
26348 : /* Now an vpunpck[lh]qdq will produce
26349 : { 0 2 4 6 8 a c e } resp. { 1 3 5 7 9 b d f }. */
26350 138 : if (odd)
26351 0 : t3 = gen_avx2_interleave_highv4di (t5, gen_lowpart (V4DImode, t1),
26352 0 : gen_lowpart (V4DImode, t2));
26353 : else
26354 138 : t3 = gen_avx2_interleave_lowv4di (t5, gen_lowpart (V4DImode, t1),
26355 138 : gen_lowpart (V4DImode, t2));
26356 138 : emit_insn (t3);
26357 138 : emit_move_insn (d->target, gen_lowpart (V8SImode, t5));
26358 138 : break;
26359 :
26360 0 : default:
26361 0 : gcc_unreachable ();
26362 : }
26363 :
26364 : return true;
26365 : }
26366 :
26367 : /* A subroutine of ix86_expand_vec_perm_const_1. Pattern match
26368 : extract-even and extract-odd permutations. */
26369 :
26370 : static bool
26371 24888 : expand_vec_perm_even_odd (struct expand_vec_perm_d *d)
26372 : {
26373 24888 : unsigned i, odd, nelt = d->nelt;
26374 :
26375 24888 : odd = d->perm[0];
26376 24888 : if (odd != 0 && odd != 1)
26377 : return false;
26378 :
26379 65994 : for (i = 1; i < nelt; ++i)
26380 57980 : if (d->perm[i] != 2 * i + odd)
26381 : return false;
26382 :
26383 8014 : if (d->vmode == E_V32HImode
26384 12 : && d->testing_p
26385 12 : && !TARGET_AVX512BW)
26386 : return false;
26387 :
26388 8002 : return expand_vec_perm_even_odd_1 (d, odd);
26389 : }
26390 :
26391 : /* A subroutine of ix86_expand_vec_perm_const_1. Implement broadcast
26392 : permutations. We assume that expand_vec_perm_1 has already failed. */
26393 :
26394 : static bool
26395 1078 : expand_vec_perm_broadcast_1 (struct expand_vec_perm_d *d)
26396 : {
26397 1078 : unsigned elt = d->perm[0], nelt2 = d->nelt / 2;
26398 1078 : machine_mode vmode = d->vmode;
26399 1078 : rtx (*gen) (rtx, rtx, rtx);
26400 1078 : unsigned char perm2[4];
26401 1078 : rtx op0 = d->op0, dest;
26402 1078 : bool ok;
26403 :
26404 1078 : switch (vmode)
26405 : {
26406 0 : case E_V4DFmode:
26407 0 : case E_V8SFmode:
26408 : /* These are special-cased in sse.md so that we can optionally
26409 : use the vbroadcast instruction. They expand to two insns
26410 : if the input happens to be in a register. */
26411 0 : gcc_unreachable ();
26412 :
26413 0 : case E_V2DFmode:
26414 0 : case E_V2SFmode:
26415 0 : case E_V4SFmode:
26416 0 : case E_V2DImode:
26417 0 : case E_V2SImode:
26418 0 : case E_V4SImode:
26419 0 : case E_V2HImode:
26420 0 : case E_V4HImode:
26421 : /* These are always implementable using standard shuffle patterns. */
26422 0 : gcc_unreachable ();
26423 :
26424 16 : case E_V4QImode:
26425 : /* This can be implemented via interleave and pshuflw. */
26426 16 : if (d->testing_p)
26427 : return true;
26428 :
26429 8 : if (elt >= nelt2)
26430 : {
26431 4 : gen = gen_mmx_punpckhbw_low;
26432 4 : elt -= nelt2;
26433 : }
26434 : else
26435 : gen = gen_mmx_punpcklbw_low;
26436 :
26437 8 : dest = gen_reg_rtx (vmode);
26438 8 : emit_insn (gen (dest, op0, op0));
26439 8 : vmode = get_mode_wider_vector (vmode);
26440 8 : op0 = gen_lowpart (vmode, dest);
26441 :
26442 8 : memset (perm2, elt, 2);
26443 8 : dest = gen_reg_rtx (vmode);
26444 8 : ok = expand_vselect (dest, op0, perm2, 2, d->testing_p);
26445 8 : gcc_assert (ok);
26446 :
26447 8 : emit_move_insn (d->target, gen_lowpart (d->vmode, dest));
26448 8 : return true;
26449 :
26450 4 : case E_V8QImode:
26451 : /* This can be implemented via interleave. We save one insn by
26452 : stopping once we have promoted to V2SImode and then use pshufd. */
26453 4 : if (d->testing_p)
26454 : return true;
26455 4 : do
26456 : {
26457 4 : if (elt >= nelt2)
26458 : {
26459 1 : gen = vmode == V8QImode ? gen_mmx_punpckhbw
26460 : : gen_mmx_punpckhwd;
26461 1 : elt -= nelt2;
26462 : }
26463 : else
26464 3 : gen = vmode == V8QImode ? gen_mmx_punpcklbw
26465 : : gen_mmx_punpcklwd;
26466 4 : nelt2 /= 2;
26467 :
26468 4 : dest = gen_reg_rtx (vmode);
26469 4 : emit_insn (gen (dest, op0, op0));
26470 4 : vmode = get_mode_wider_vector (vmode);
26471 4 : op0 = gen_lowpart (vmode, dest);
26472 : }
26473 4 : while (vmode != V2SImode);
26474 :
26475 2 : memset (perm2, elt, 2);
26476 2 : dest = gen_reg_rtx (vmode);
26477 2 : ok = expand_vselect (dest, op0, perm2, 2, d->testing_p);
26478 2 : gcc_assert (ok);
26479 :
26480 2 : emit_move_insn (d->target, gen_lowpart (d->vmode, dest));
26481 2 : return true;
26482 :
26483 1049 : case E_V8HImode:
26484 1049 : case E_V16QImode:
26485 : /* These can be implemented via interleave. We save one insn by
26486 : stopping once we have promoted to V4SImode and then use pshufd. */
26487 1049 : if (d->testing_p)
26488 : return true;
26489 1610 : do
26490 : {
26491 1610 : if (elt >= nelt2)
26492 : {
26493 16 : gen = vmode == V16QImode ? gen_vec_interleave_highv16qi
26494 : : gen_vec_interleave_highv8hi;
26495 16 : elt -= nelt2;
26496 : }
26497 : else
26498 1594 : gen = vmode == V16QImode ? gen_vec_interleave_lowv16qi
26499 : : gen_vec_interleave_lowv8hi;
26500 1610 : nelt2 /= 2;
26501 :
26502 1610 : dest = gen_reg_rtx (vmode);
26503 1610 : emit_insn (gen (dest, op0, op0));
26504 1610 : vmode = get_mode_wider_vector (vmode);
26505 1610 : op0 = gen_lowpart (vmode, dest);
26506 : }
26507 1610 : while (vmode != V4SImode);
26508 :
26509 985 : memset (perm2, elt, 4);
26510 985 : dest = gen_reg_rtx (vmode);
26511 985 : ok = expand_vselect (dest, op0, perm2, 4, d->testing_p);
26512 985 : gcc_assert (ok);
26513 :
26514 985 : emit_move_insn (d->target, gen_lowpart (d->vmode, dest));
26515 985 : return true;
26516 :
26517 1 : case E_V8HFmode:
26518 1 : case E_V8BFmode:
26519 : /* This can be implemented via interleave and pshufd. */
26520 1 : if (d->testing_p)
26521 : return true;
26522 :
26523 1 : rtx (*gen_interleave) (machine_mode, rtx, rtx, rtx);
26524 1 : if (elt >= nelt2)
26525 : {
26526 0 : gen_interleave = gen_vec_interleave_high;
26527 0 : elt -= nelt2;
26528 : }
26529 : else
26530 : gen_interleave = gen_vec_interleave_low;
26531 1 : nelt2 /= 2;
26532 :
26533 1 : dest = gen_reg_rtx (vmode);
26534 1 : emit_insn (gen_interleave (vmode, dest, op0, op0));
26535 :
26536 1 : vmode = V4SImode;
26537 1 : op0 = gen_lowpart (vmode, dest);
26538 :
26539 1 : memset (perm2, elt, 4);
26540 1 : dest = gen_reg_rtx (vmode);
26541 1 : ok = expand_vselect (dest, op0, perm2, 4, d->testing_p);
26542 1 : gcc_assert (ok);
26543 :
26544 1 : emit_move_insn (d->target, gen_lowpart (d->vmode, dest));
26545 1 : return true;
26546 :
26547 0 : case E_V32QImode:
26548 0 : case E_V16HImode:
26549 0 : case E_V8SImode:
26550 0 : case E_V4DImode:
26551 : /* For AVX2 broadcasts of the first element vpbroadcast* or
26552 : vpermq should be used by expand_vec_perm_1. */
26553 0 : gcc_assert (!TARGET_AVX2 || d->perm[0]);
26554 : return false;
26555 :
26556 6 : case E_V64QImode:
26557 6 : gcc_assert (!TARGET_AVX512BW || d->perm[0]);
26558 : return false;
26559 :
26560 2 : case E_V32HImode:
26561 2 : gcc_assert (!TARGET_AVX512BW);
26562 : return false;
26563 :
26564 0 : default:
26565 0 : gcc_unreachable ();
26566 : }
26567 : }
26568 :
26569 : /* A subroutine of ix86_expand_vec_perm_const_1. Pattern match
26570 : broadcast permutations. */
26571 :
26572 : static bool
26573 92427 : expand_vec_perm_broadcast (struct expand_vec_perm_d *d)
26574 : {
26575 92427 : unsigned i, elt, nelt = d->nelt;
26576 :
26577 92427 : if (!d->one_operand_p)
26578 : return false;
26579 :
26580 5750 : elt = d->perm[0];
26581 8340 : for (i = 1; i < nelt; ++i)
26582 8232 : if (d->perm[i] != elt)
26583 : return false;
26584 :
26585 108 : return expand_vec_perm_broadcast_1 (d);
26586 : }
26587 :
26588 : /* Implement arbitrary permutations of two V64QImode operands
26589 : with 2 vperm[it]2w, 2 vpshufb and one vpor instruction. */
26590 : static bool
26591 24937 : expand_vec_perm_vpermt2_vpshub2 (struct expand_vec_perm_d *d)
26592 : {
26593 24937 : if (!TARGET_AVX512BW || !(d->vmode == V64QImode))
26594 : return false;
26595 :
26596 49 : if (d->testing_p)
26597 : return true;
26598 :
26599 49 : struct expand_vec_perm_d ds[2];
26600 49 : rtx rperm[128], vperm, target0, target1;
26601 49 : unsigned int i, nelt;
26602 49 : machine_mode vmode;
26603 :
26604 49 : nelt = d->nelt;
26605 49 : vmode = V64QImode;
26606 :
26607 147 : for (i = 0; i < 2; i++)
26608 : {
26609 98 : ds[i] = *d;
26610 98 : ds[i].vmode = V32HImode;
26611 98 : ds[i].nelt = 32;
26612 98 : ds[i].target = gen_reg_rtx (V32HImode);
26613 98 : ds[i].op0 = gen_lowpart (V32HImode, d->op0);
26614 98 : ds[i].op1 = gen_lowpart (V32HImode, d->op1);
26615 : }
26616 :
26617 : /* Prepare permutations such that the first one takes care of
26618 : putting the even bytes into the right positions or one higher
26619 : positions (ds[0]) and the second one takes care of
26620 : putting the odd bytes into the right positions or one below
26621 : (ds[1]). */
26622 :
26623 3185 : for (i = 0; i < nelt; i++)
26624 : {
26625 3136 : ds[i & 1].perm[i / 2] = d->perm[i] / 2;
26626 3136 : if (i & 1)
26627 : {
26628 1568 : rperm[i] = constm1_rtx;
26629 1568 : rperm[i + 64] = GEN_INT ((i & 14) + (d->perm[i] & 1));
26630 : }
26631 : else
26632 : {
26633 1568 : rperm[i] = GEN_INT ((i & 14) + (d->perm[i] & 1));
26634 1568 : rperm[i + 64] = constm1_rtx;
26635 : }
26636 : }
26637 :
26638 49 : bool ok = expand_vec_perm_1 (&ds[0]);
26639 49 : gcc_assert (ok);
26640 49 : ds[0].target = gen_lowpart (V64QImode, ds[0].target);
26641 :
26642 49 : ok = expand_vec_perm_1 (&ds[1]);
26643 49 : gcc_assert (ok);
26644 49 : ds[1].target = gen_lowpart (V64QImode, ds[1].target);
26645 :
26646 49 : vperm = gen_rtx_CONST_VECTOR (V64QImode, gen_rtvec_v (64, rperm));
26647 49 : vperm = force_reg (vmode, vperm);
26648 49 : target0 = gen_reg_rtx (V64QImode);
26649 49 : emit_insn (gen_avx512bw_pshufbv64qi3 (target0, ds[0].target, vperm));
26650 :
26651 49 : vperm = gen_rtx_CONST_VECTOR (V64QImode, gen_rtvec_v (64, rperm + 64));
26652 49 : vperm = force_reg (vmode, vperm);
26653 49 : target1 = gen_reg_rtx (V64QImode);
26654 49 : emit_insn (gen_avx512bw_pshufbv64qi3 (target1, ds[1].target, vperm));
26655 :
26656 49 : emit_insn (gen_iorv64qi3 (d->target, target0, target1));
26657 49 : return true;
26658 : }
26659 :
26660 : /* Implement arbitrary permutation of two V32QImode and V16QImode operands
26661 : with 4 vpshufb insns, 2 vpermq and 3 vpor. We should have already failed
26662 : all the shorter instruction sequences. */
26663 :
26664 : static bool
26665 16622 : expand_vec_perm_vpshufb4_vpermq2 (struct expand_vec_perm_d *d)
26666 : {
26667 16622 : rtx rperm[4][32], vperm, l[2], h[2], op, m128;
26668 16622 : unsigned int i, nelt, eltsz;
26669 16622 : bool used[4];
26670 :
26671 16622 : if (!TARGET_AVX2
26672 318 : || d->one_operand_p
26673 189 : || (d->vmode != V32QImode && d->vmode != V16HImode))
26674 : return false;
26675 :
26676 54 : if (d->testing_p)
26677 : return true;
26678 :
26679 54 : nelt = d->nelt;
26680 54 : eltsz = GET_MODE_UNIT_SIZE (d->vmode);
26681 :
26682 : /* Generate 4 permutation masks. If the required element is within
26683 : the same lane, it is shuffled in. If the required element from the
26684 : other lane, force a zero by setting bit 7 in the permutation mask.
26685 : In the other mask the mask has non-negative elements if element
26686 : is requested from the other lane, but also moved to the other lane,
26687 : so that the result of vpshufb can have the two V2TImode halves
26688 : swapped. */
26689 54 : m128 = GEN_INT (-128);
26690 1836 : for (i = 0; i < 32; ++i)
26691 : {
26692 1728 : rperm[0][i] = m128;
26693 1728 : rperm[1][i] = m128;
26694 1728 : rperm[2][i] = m128;
26695 1728 : rperm[3][i] = m128;
26696 : }
26697 54 : used[0] = false;
26698 54 : used[1] = false;
26699 54 : used[2] = false;
26700 54 : used[3] = false;
26701 1590 : for (i = 0; i < nelt; ++i)
26702 : {
26703 1536 : unsigned j, e = d->perm[i] & (nelt / 2 - 1);
26704 1536 : unsigned xlane = ((d->perm[i] ^ i) & (nelt / 2)) * eltsz;
26705 2074 : unsigned int which = ((d->perm[i] & nelt) ? 2 : 0) + (xlane ? 1 : 0);
26706 :
26707 3264 : for (j = 0; j < eltsz; ++j)
26708 1728 : rperm[which][(i * eltsz + j) ^ xlane] = GEN_INT (e * eltsz + j);
26709 1536 : used[which] = true;
26710 : }
26711 :
26712 162 : for (i = 0; i < 2; ++i)
26713 : {
26714 108 : if (!used[2 * i + 1])
26715 : {
26716 22 : h[i] = NULL_RTX;
26717 22 : continue;
26718 : }
26719 86 : vperm = gen_rtx_CONST_VECTOR (V32QImode,
26720 86 : gen_rtvec_v (32, rperm[2 * i + 1]));
26721 86 : vperm = force_reg (V32QImode, vperm);
26722 86 : h[i] = gen_reg_rtx (V32QImode);
26723 86 : op = gen_lowpart (V32QImode, i ? d->op1 : d->op0);
26724 86 : emit_insn (gen_avx2_pshufbv32qi3 (h[i], op, vperm));
26725 : }
26726 :
26727 : /* Swap the 128-byte lanes of h[X]. */
26728 162 : for (i = 0; i < 2; ++i)
26729 : {
26730 108 : if (h[i] == NULL_RTX)
26731 22 : continue;
26732 86 : op = gen_reg_rtx (V4DImode);
26733 86 : emit_insn (gen_avx2_permv4di_1 (op, gen_lowpart (V4DImode, h[i]),
26734 : const2_rtx, GEN_INT (3), const0_rtx,
26735 : const1_rtx));
26736 86 : h[i] = gen_lowpart (V32QImode, op);
26737 : }
26738 :
26739 162 : for (i = 0; i < 2; ++i)
26740 : {
26741 108 : if (!used[2 * i])
26742 : {
26743 0 : l[i] = NULL_RTX;
26744 0 : continue;
26745 : }
26746 108 : vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[2 * i]));
26747 108 : vperm = force_reg (V32QImode, vperm);
26748 108 : l[i] = gen_reg_rtx (V32QImode);
26749 108 : op = gen_lowpart (V32QImode, i ? d->op1 : d->op0);
26750 108 : emit_insn (gen_avx2_pshufbv32qi3 (l[i], op, vperm));
26751 : }
26752 :
26753 162 : for (i = 0; i < 2; ++i)
26754 : {
26755 108 : if (h[i] && l[i])
26756 : {
26757 86 : op = gen_reg_rtx (V32QImode);
26758 86 : emit_insn (gen_iorv32qi3 (op, l[i], h[i]));
26759 86 : l[i] = op;
26760 : }
26761 22 : else if (h[i])
26762 0 : l[i] = h[i];
26763 : }
26764 :
26765 54 : gcc_assert (l[0] && l[1]);
26766 54 : op = d->target;
26767 54 : if (d->vmode != V32QImode)
26768 12 : op = gen_reg_rtx (V32QImode);
26769 54 : emit_insn (gen_iorv32qi3 (op, l[0], l[1]));
26770 54 : if (op != d->target)
26771 12 : emit_move_insn (d->target, gen_lowpart (d->vmode, op));
26772 : return true;
26773 : }
26774 :
26775 : /* The guts of ix86_vectorize_vec_perm_const. With all of the interface bits
26776 : taken care of, perform the expansion in D and return true on success. */
26777 :
26778 : static bool
26779 297077 : ix86_expand_vec_perm_const_1 (struct expand_vec_perm_d *d)
26780 : {
26781 : /* Try a single instruction expansion. */
26782 297077 : if (expand_vec_perm_1 (d))
26783 : return true;
26784 :
26785 : /* Try sequences of two instructions. */
26786 :
26787 104248 : if (expand_vec_perm_pshuflw_pshufhw (d))
26788 : return true;
26789 :
26790 101785 : if (expand_vec_perm_palignr (d, false))
26791 : return true;
26792 :
26793 98650 : if (expand_vec_perm_interleave2 (d))
26794 : return true;
26795 :
26796 92427 : if (expand_vec_perm_broadcast (d))
26797 : return true;
26798 :
26799 92327 : if (expand_vec_perm_vpermq_perm_1 (d))
26800 : return true;
26801 :
26802 92327 : if (expand_vec_perm_vperm2f128 (d))
26803 : return true;
26804 :
26805 92259 : if (expand_vec_perm_pblendv (d))
26806 : return true;
26807 :
26808 90578 : if (expand_vec_perm_2perm_interleave (d, true))
26809 : return true;
26810 :
26811 90210 : if (expand_vec_perm_2perm_pblendv (d, true))
26812 : return true;
26813 :
26814 87325 : if (expand_vec_perm_shufps_shufps (d))
26815 : return true;
26816 :
26817 51714 : if (expand_vec_perm_punpckldq_pshuf (d))
26818 : return true;
26819 :
26820 : /* Try sequences of three instructions. */
26821 :
26822 45322 : if (expand_vec_perm_even_odd_pack (d))
26823 : return true;
26824 :
26825 31433 : if (expand_vec_perm_2vperm2f128_vshuf (d))
26826 : return true;
26827 :
26828 30156 : if (expand_vec_perm_pshufb2 (d))
26829 : return true;
26830 :
26831 28740 : if (expand_vec_perm_pslldq_psrldq_por (d, false))
26832 : return true;
26833 :
26834 28423 : if (expand_vec_perm_interleave3 (d))
26835 : return true;
26836 :
26837 28277 : if (expand_vec_perm_vperm2f128_vblend (d))
26838 : return true;
26839 :
26840 28277 : if (expand_vec_perm_2perm_interleave (d, false))
26841 : return true;
26842 :
26843 28037 : if (expand_vec_perm_2perm_pblendv (d, false))
26844 : return true;
26845 :
26846 27525 : if (expand_vec_perm_psrlw_psllw_por (d))
26847 : return true;
26848 :
26849 26071 : if (expand_vec_perm_pand_pandn_por (d))
26850 : return true;
26851 :
26852 : /* Try sequences of four instructions. */
26853 :
26854 24948 : if (expand_vec_perm_even_odd_trunc (d))
26855 : return true;
26856 24944 : if (expand_vec_perm_vpshufb2_vpermq (d))
26857 : return true;
26858 :
26859 24937 : if (expand_vec_perm_vpshufb2_vpermq_even_odd (d))
26860 : return true;
26861 :
26862 24937 : if (expand_vec_perm_vpermt2_vpshub2 (d))
26863 : return true;
26864 :
26865 : /* ??? Look for narrow permutations whose element orderings would
26866 : allow the promotion to a wider mode. */
26867 :
26868 : /* ??? Look for sequences of interleave or a wider permute that place
26869 : the data into the correct lanes for a half-vector shuffle like
26870 : pshuf[lh]w or vpermilps. */
26871 :
26872 : /* ??? Look for sequences of interleave that produce the desired results.
26873 : The combinatorics of punpck[lh] get pretty ugly... */
26874 :
26875 24888 : if (expand_vec_perm_even_odd (d))
26876 : return true;
26877 :
26878 : /* Generate four or five instructions. */
26879 16958 : if (expand_vec_perm_pslldq_psrldq_por (d, true))
26880 : return true;
26881 :
26882 : /* Even longer sequences. */
26883 16622 : if (expand_vec_perm_vpshufb4_vpermq2 (d))
26884 : return true;
26885 :
26886 : /* See if we can get the same permutation in different vector integer
26887 : mode. */
26888 16568 : struct expand_vec_perm_d nd;
26889 16568 : if (canonicalize_vector_int_perm (d, &nd) && expand_vec_perm_1 (&nd))
26890 : {
26891 0 : if (!d->testing_p)
26892 0 : emit_move_insn (d->target, gen_lowpart (d->vmode, nd.target));
26893 : return true;
26894 : }
26895 :
26896 : /* Even longer, including recursion to ix86_expand_vec_perm_const_1. */
26897 16568 : if (expand_vec_perm2_vperm2f128_vblend (d))
26898 : return true;
26899 :
26900 : return false;
26901 : }
26902 :
26903 : /* If a permutation only uses one operand, make it clear. Returns true
26904 : if the permutation references both operands. */
26905 :
26906 : static bool
26907 75420 : canonicalize_perm (struct expand_vec_perm_d *d)
26908 : {
26909 75420 : int i, which, nelt = d->nelt;
26910 :
26911 449988 : for (i = which = 0; i < nelt; ++i)
26912 508866 : which |= (d->perm[i] < nelt ? 1 : 2);
26913 :
26914 75420 : d->one_operand_p = true;
26915 75420 : switch (which)
26916 : {
26917 0 : default:
26918 0 : gcc_unreachable();
26919 :
26920 56089 : case 3:
26921 56089 : if (!rtx_equal_p (d->op0, d->op1))
26922 : {
26923 56060 : d->one_operand_p = false;
26924 56060 : break;
26925 : }
26926 : /* The elements of PERM do not suggest that only the first operand
26927 : is used, but both operands are identical. Allow easier matching
26928 : of the permutation by folding the permutation into the single
26929 : input vector. */
26930 : /* FALLTHRU */
26931 :
26932 315 : case 2:
26933 2635 : for (i = 0; i < nelt; ++i)
26934 2320 : d->perm[i] &= nelt - 1;
26935 315 : d->op0 = d->op1;
26936 315 : break;
26937 :
26938 19045 : case 1:
26939 19045 : d->op1 = d->op0;
26940 19045 : break;
26941 : }
26942 :
26943 75420 : return (which == 3);
26944 : }
26945 :
26946 : /* Implement TARGET_VECTORIZE_VEC_PERM_CONST. */
26947 :
26948 : bool
26949 826465 : ix86_vectorize_vec_perm_const (machine_mode vmode, machine_mode op_mode,
26950 : rtx target, rtx op0, rtx op1,
26951 : const vec_perm_indices &sel)
26952 : {
26953 826465 : if (vmode != op_mode)
26954 : return false;
26955 :
26956 822975 : struct expand_vec_perm_d d;
26957 822975 : unsigned char perm[MAX_VECT_LEN];
26958 822975 : unsigned int i, nelt, which;
26959 822975 : bool two_args;
26960 :
26961 : /* For HF and BF mode vector, convert it to HI using subreg. */
26962 2466484 : if (GET_MODE_INNER (vmode) == HFmode || GET_MODE_INNER (vmode) == BFmode)
26963 : {
26964 2471 : machine_mode orig_mode = vmode;
26965 4942 : vmode = mode_for_vector (HImode,
26966 2471 : GET_MODE_NUNITS (vmode)).require ();
26967 2471 : if (target)
26968 435 : target = lowpart_subreg (vmode, target, orig_mode);
26969 2471 : if (op0)
26970 435 : op0 = lowpart_subreg (vmode, op0, orig_mode);
26971 2471 : if (op1)
26972 435 : op1 = lowpart_subreg (vmode, op1, orig_mode);
26973 : }
26974 :
26975 822975 : d.target = target;
26976 822975 : d.op0 = op0;
26977 822975 : d.op1 = op1;
26978 :
26979 822975 : d.vmode = vmode;
26980 822975 : gcc_assert (VECTOR_MODE_P (d.vmode));
26981 822975 : d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
26982 822975 : d.testing_p = !target;
26983 :
26984 822975 : gcc_assert (sel.length () == nelt);
26985 822975 : gcc_checking_assert (sizeof (d.perm) == sizeof (perm));
26986 :
26987 : /* Given sufficient ISA support we can just return true here
26988 : for selected vector modes. */
26989 822975 : switch (d.vmode)
26990 : {
26991 1604 : case E_V16SFmode:
26992 1604 : case E_V16SImode:
26993 1604 : case E_V8DImode:
26994 1604 : case E_V8DFmode:
26995 1604 : if (!TARGET_AVX512F)
26996 : return false;
26997 : /* All implementable with a single vperm[it]2 insn. */
26998 1604 : if (d.testing_p)
26999 : return true;
27000 : break;
27001 323 : case E_V32HImode:
27002 323 : if (!TARGET_AVX512F)
27003 : return false;
27004 323 : if (d.testing_p && TARGET_AVX512BW)
27005 : /* All implementable with a single vperm[it]2 insn. */
27006 : return true;
27007 : break;
27008 715 : case E_V64QImode:
27009 715 : if (!TARGET_AVX512F)
27010 : return false;
27011 715 : if (d.testing_p && TARGET_AVX512BW)
27012 : /* Implementable with 2 vperm[it]2, 2 vpshufb and 1 or insn. */
27013 : return true;
27014 : break;
27015 11463 : case E_V8SImode:
27016 11463 : case E_V8SFmode:
27017 11463 : case E_V4DFmode:
27018 11463 : case E_V4DImode:
27019 11463 : if (!TARGET_AVX)
27020 : return false;
27021 11463 : if (d.testing_p && TARGET_AVX512VL)
27022 : /* All implementable with a single vperm[it]2 insn. */
27023 : return true;
27024 : break;
27025 612 : case E_V16HImode:
27026 612 : if (!TARGET_SSE2)
27027 : return false;
27028 612 : if (d.testing_p && TARGET_AVX2)
27029 : /* Implementable with 4 vpshufb insns, 2 vpermq and 3 vpor insns. */
27030 : return true;
27031 : break;
27032 685 : case E_V32QImode:
27033 685 : if (!TARGET_SSE2)
27034 : return false;
27035 685 : if (d.testing_p && TARGET_AVX2)
27036 : /* Implementable with 4 vpshufb insns, 2 vpermq and 3 vpor insns. */
27037 : return true;
27038 : break;
27039 39040 : case E_V8HImode:
27040 39040 : case E_V16QImode:
27041 39040 : if (!TARGET_SSE2)
27042 : return false;
27043 : /* Fall through. */
27044 244680 : case E_V4SImode:
27045 244680 : case E_V4SFmode:
27046 244680 : if (!TARGET_SSE)
27047 : return false;
27048 : /* All implementable with a single vpperm insn. */
27049 244680 : if (d.testing_p && TARGET_XOP)
27050 : return true;
27051 : /* All implementable with 2 pshufb + 1 ior. */
27052 244574 : if (d.testing_p && TARGET_SSSE3)
27053 : return true;
27054 : break;
27055 147174 : case E_V2SFmode:
27056 147174 : case E_V2SImode:
27057 147174 : case E_V4HImode:
27058 147174 : case E_V8QImode:
27059 147174 : if (!TARGET_MMX_WITH_SSE)
27060 : return false;
27061 : break;
27062 20424 : case E_V2HImode:
27063 20424 : if (!TARGET_SSE2)
27064 : return false;
27065 : /* All implementable with *punpckwd. */
27066 20424 : if (d.testing_p)
27067 : return true;
27068 : break;
27069 12393 : case E_V4QImode:
27070 12393 : if (!TARGET_SSE2)
27071 : return false;
27072 : break;
27073 379750 : case E_V2DImode:
27074 379750 : case E_V2DFmode:
27075 379750 : if (!TARGET_SSE)
27076 : return false;
27077 : /* All implementable with shufpd or unpck[lh]pd. */
27078 379750 : if (d.testing_p)
27079 : return true;
27080 : break;
27081 : default:
27082 : return false;
27083 : }
27084 :
27085 2304553 : for (i = which = 0; i < nelt; ++i)
27086 : {
27087 1883558 : unsigned char e = sel[i];
27088 1883558 : gcc_assert (e < 2 * nelt);
27089 1883558 : d.perm[i] = e;
27090 1883558 : perm[i] = e;
27091 2524690 : which |= (e < nelt ? 1 : 2);
27092 : }
27093 :
27094 420995 : if (d.testing_p)
27095 : {
27096 : /* For all elements from second vector, fold the elements to first. */
27097 346839 : if (which == 2)
27098 1060 : for (i = 0; i < nelt; ++i)
27099 958 : d.perm[i] -= nelt;
27100 :
27101 : /* Check whether the mask can be applied to the vector type. */
27102 346839 : d.one_operand_p = (which != 3);
27103 :
27104 : /* Implementable with shufps, pshufd or pshuflw. */
27105 346839 : if (d.one_operand_p
27106 : && (d.vmode == V4SFmode || d.vmode == V2SFmode
27107 : || d.vmode == V4SImode || d.vmode == V2SImode
27108 : || d.vmode == V4HImode || d.vmode == V2HImode))
27109 : return true;
27110 :
27111 : /* Otherwise we have to go through the motions and see if we can
27112 : figure out how to generate the requested permutation. */
27113 219070 : d.target = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 1);
27114 219070 : d.op1 = d.op0 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 2);
27115 219070 : if (!d.one_operand_p)
27116 204408 : d.op1 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 3);
27117 :
27118 219070 : start_sequence ();
27119 219070 : bool ret = ix86_expand_vec_perm_const_1 (&d);
27120 219070 : end_sequence ();
27121 :
27122 219070 : return ret;
27123 : }
27124 :
27125 74156 : two_args = canonicalize_perm (&d);
27126 :
27127 : /* If one of the operands is a zero vector, try to match pmovzx. */
27128 74156 : if (two_args && (d.op0 == CONST0_RTX (vmode) || d.op1 == CONST0_RTX (vmode)))
27129 : {
27130 618 : struct expand_vec_perm_d dzero = d;
27131 618 : if (d.op0 == CONST0_RTX (vmode))
27132 : {
27133 387 : d.op1 = dzero.op1 = force_reg (vmode, d.op1);
27134 387 : std::swap (dzero.op0, dzero.op1);
27135 7527 : for (i = 0; i < nelt; ++i)
27136 7140 : dzero.perm[i] ^= nelt;
27137 : }
27138 : else
27139 231 : d.op0 = dzero.op0 = force_reg (vmode, d.op0);
27140 :
27141 618 : if (expand_vselect_vconcat (dzero.target, dzero.op0, dzero.op1,
27142 : dzero.perm, nelt, dzero.testing_p))
27143 128 : return true;
27144 : }
27145 :
27146 : /* Force operands into registers. */
27147 74028 : rtx nop0 = force_reg (vmode, d.op0);
27148 74028 : if (d.op0 == d.op1)
27149 18959 : d.op1 = nop0;
27150 74028 : d.op0 = nop0;
27151 74028 : d.op1 = force_reg (vmode, d.op1);
27152 :
27153 74028 : if (ix86_expand_vec_perm_const_1 (&d))
27154 : return true;
27155 :
27156 : /* If the selector says both arguments are needed, but the operands are the
27157 : same, the above tried to expand with one_operand_p and flattened selector.
27158 : If that didn't work, retry without one_operand_p; we succeeded with that
27159 : during testing. */
27160 0 : if (two_args && d.one_operand_p)
27161 : {
27162 0 : d.one_operand_p = false;
27163 0 : memcpy (d.perm, perm, sizeof (perm));
27164 0 : return ix86_expand_vec_perm_const_1 (&d);
27165 : }
27166 :
27167 : return false;
27168 : }
27169 :
27170 : void
27171 8284 : ix86_expand_vec_extract_even_odd (rtx targ, rtx op0, rtx op1, unsigned odd)
27172 : {
27173 8284 : struct expand_vec_perm_d d;
27174 8284 : unsigned i, nelt;
27175 :
27176 8284 : d.target = targ;
27177 8284 : d.op0 = op0;
27178 8284 : d.op1 = op1;
27179 8284 : d.vmode = GET_MODE (targ);
27180 8284 : d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
27181 8284 : d.one_operand_p = false;
27182 8284 : d.testing_p = false;
27183 :
27184 78572 : for (i = 0; i < nelt; ++i)
27185 70288 : d.perm[i] = i * 2 + odd;
27186 :
27187 : /* We'll either be able to implement the permutation directly... */
27188 8284 : if (expand_vec_perm_1 (&d))
27189 : return;
27190 :
27191 : /* ... or we use the special-case patterns. */
27192 5067 : expand_vec_perm_even_odd_1 (&d, odd);
27193 : }
27194 :
27195 : static void
27196 922 : ix86_expand_vec_interleave (rtx targ, rtx op0, rtx op1, bool high_p)
27197 : {
27198 922 : struct expand_vec_perm_d d;
27199 922 : unsigned i, nelt, base;
27200 922 : bool ok;
27201 :
27202 922 : d.target = targ;
27203 922 : d.op0 = op0;
27204 922 : d.op1 = op1;
27205 922 : d.vmode = GET_MODE (targ);
27206 922 : d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
27207 922 : d.one_operand_p = false;
27208 922 : d.testing_p = false;
27209 :
27210 922 : base = high_p ? nelt / 2 : 0;
27211 3642 : for (i = 0; i < nelt / 2; ++i)
27212 : {
27213 2720 : d.perm[i * 2] = i + base;
27214 2720 : d.perm[i * 2 + 1] = i + base + nelt;
27215 : }
27216 :
27217 : /* Note that for AVX this isn't one instruction. */
27218 922 : ok = ix86_expand_vec_perm_const_1 (&d);
27219 922 : gcc_assert (ok);
27220 922 : }
27221 :
27222 : /* Expand a vector operation shift by constant for a V*QImode in terms of the
27223 : same operation on V*HImode. Return true if success. */
27224 : static bool
27225 374 : ix86_expand_vec_shift_qihi_constant (enum rtx_code code,
27226 : rtx dest, rtx op1, rtx op2)
27227 : {
27228 374 : machine_mode qimode, himode;
27229 374 : HOST_WIDE_INT and_constant, xor_constant;
27230 374 : HOST_WIDE_INT shift_amount;
27231 374 : rtx vec_const_and, vec_const_xor;
27232 374 : rtx tmp, op1_subreg;
27233 374 : rtx (*gen_shift) (rtx, rtx, rtx);
27234 374 : rtx (*gen_and) (rtx, rtx, rtx);
27235 374 : rtx (*gen_xor) (rtx, rtx, rtx);
27236 374 : rtx (*gen_sub) (rtx, rtx, rtx);
27237 :
27238 : /* Only optimize shift by constant. */
27239 374 : if (!CONST_INT_P (op2))
27240 : return false;
27241 :
27242 374 : qimode = GET_MODE (dest);
27243 374 : shift_amount = INTVAL (op2);
27244 : /* Do nothing when shift amount greater equal 8. */
27245 374 : if (shift_amount > 7)
27246 : return false;
27247 :
27248 374 : gcc_assert (code == ASHIFT || code == ASHIFTRT || code == LSHIFTRT);
27249 :
27250 :
27251 374 : if (shift_amount == 7
27252 374 : && code == ASHIFTRT)
27253 : {
27254 47 : if (qimode == V16QImode
27255 8 : || qimode == V32QImode)
27256 : {
27257 46 : rtx zero = gen_reg_rtx (qimode);
27258 46 : emit_move_insn (zero, CONST0_RTX (qimode));
27259 46 : emit_move_insn (dest, gen_rtx_fmt_ee (GT, qimode, zero, op1));
27260 46 : }
27261 : else
27262 : {
27263 1 : gcc_assert (qimode == V64QImode);
27264 1 : rtx kmask = gen_reg_rtx (DImode);
27265 1 : emit_insn (gen_avx512bw_cvtb2maskv64qi (kmask, op1));
27266 1 : emit_insn (gen_avx512bw_cvtmask2bv64qi (dest, kmask));
27267 : }
27268 : return true;
27269 : }
27270 :
27271 : /* Record sign bit. */
27272 327 : xor_constant = 1 << (8 - shift_amount - 1);
27273 :
27274 : /* Zero upper/lower bits shift from left/right element. */
27275 327 : and_constant
27276 327 : = (code == ASHIFT ? 256 - (1 << shift_amount)
27277 296 : : (1 << (8 - shift_amount)) - 1);
27278 :
27279 327 : switch (qimode)
27280 : {
27281 314 : case V16QImode:
27282 314 : himode = V8HImode;
27283 260 : gen_shift =
27284 : ((code == ASHIFT)
27285 314 : ? gen_ashlv8hi3
27286 292 : : (code == ASHIFTRT) ? gen_ashrv8hi3 : gen_lshrv8hi3);
27287 : gen_and = gen_andv16qi3;
27288 : gen_xor = gen_xorv16qi3;
27289 : gen_sub = gen_subv16qi3;
27290 : break;
27291 6 : case V32QImode:
27292 6 : himode = V16HImode;
27293 1 : gen_shift =
27294 : ((code == ASHIFT)
27295 6 : ? gen_ashlv16hi3
27296 2 : : (code == ASHIFTRT) ? gen_ashrv16hi3 : gen_lshrv16hi3);
27297 : gen_and = gen_andv32qi3;
27298 : gen_xor = gen_xorv32qi3;
27299 : gen_sub = gen_subv32qi3;
27300 : break;
27301 7 : case V64QImode:
27302 7 : himode = V32HImode;
27303 1 : gen_shift =
27304 : ((code == ASHIFT)
27305 7 : ? gen_ashlv32hi3
27306 2 : : (code == ASHIFTRT) ? gen_ashrv32hi3 : gen_lshrv32hi3);
27307 : gen_and = gen_andv64qi3;
27308 : gen_xor = gen_xorv64qi3;
27309 : gen_sub = gen_subv64qi3;
27310 : break;
27311 0 : default:
27312 0 : gcc_unreachable ();
27313 : }
27314 :
27315 327 : tmp = gen_reg_rtx (himode);
27316 327 : vec_const_and = gen_reg_rtx (qimode);
27317 327 : op1_subreg = lowpart_subreg (himode, op1, qimode);
27318 :
27319 : /* For ASHIFT and LSHIFTRT, perform operation like
27320 : vpsllw/vpsrlw $shift_amount, %op1, %dest.
27321 : vpand %vec_const_and, %dest. */
27322 327 : emit_insn (gen_shift (tmp, op1_subreg, op2));
27323 327 : emit_move_insn (dest, simplify_gen_subreg (qimode, tmp, himode, 0));
27324 327 : emit_move_insn (vec_const_and,
27325 : ix86_build_const_vector (qimode, true,
27326 327 : gen_int_mode (and_constant, QImode)));
27327 327 : emit_insn (gen_and (dest, dest, vec_const_and));
27328 :
27329 : /* For ASHIFTRT, perform extra operation like
27330 : vpxor %vec_const_xor, %dest, %dest
27331 : vpsubb %vec_const_xor, %dest, %dest */
27332 327 : if (code == ASHIFTRT)
27333 : {
27334 34 : vec_const_xor = gen_reg_rtx (qimode);
27335 34 : emit_move_insn (vec_const_xor,
27336 : ix86_build_const_vector (qimode, true,
27337 34 : gen_int_mode (xor_constant, QImode)));
27338 34 : emit_insn (gen_xor (dest, dest, vec_const_xor));
27339 34 : emit_insn (gen_sub (dest, dest, vec_const_xor));
27340 : }
27341 : return true;
27342 : }
27343 :
27344 : void
27345 1440 : ix86_expand_vecop_qihi_partial (enum rtx_code code, rtx dest, rtx op1, rtx op2)
27346 : {
27347 1440 : machine_mode qimode = GET_MODE (dest);
27348 1440 : rtx qop1, qop2, hop1, hop2, qdest, hdest;
27349 1440 : bool op2vec = GET_MODE_CLASS (GET_MODE (op2)) == MODE_VECTOR_INT;
27350 1440 : bool uns_p = code != ASHIFTRT;
27351 :
27352 1440 : switch (qimode)
27353 : {
27354 1440 : case E_V4QImode:
27355 1440 : case E_V8QImode:
27356 1440 : break;
27357 0 : default:
27358 0 : gcc_unreachable ();
27359 : }
27360 :
27361 1440 : qop1 = lowpart_subreg (V16QImode, force_reg (qimode, op1), qimode);
27362 :
27363 1440 : if (op2vec)
27364 1310 : qop2 = lowpart_subreg (V16QImode, force_reg (qimode, op2), qimode);
27365 : else
27366 : qop2 = op2;
27367 :
27368 1440 : qdest = gen_reg_rtx (V16QImode);
27369 :
27370 1440 : if (CONST_INT_P (op2)
27371 118 : && (code == ASHIFT || code == LSHIFTRT || code == ASHIFTRT)
27372 : /* With AVX512 it's cheaper to do vpmovsxbw/op/vpmovwb.
27373 : Even with SSE4.1 the alternative is better. */
27374 118 : && !TARGET_SSE4_1
27375 1494 : && ix86_expand_vec_shift_qihi_constant (code, qdest, qop1, qop2))
27376 : {
27377 54 : emit_move_insn (dest, gen_lowpart (qimode, qdest));
27378 54 : return;
27379 : }
27380 :
27381 1386 : if (CONST_INT_P (op2)
27382 64 : && code == ASHIFTRT
27383 14 : && INTVAL (op2) == 7)
27384 : {
27385 4 : rtx zero = gen_reg_rtx (qimode);
27386 4 : emit_move_insn (zero, CONST0_RTX (qimode));
27387 4 : emit_move_insn (dest, gen_rtx_fmt_ee (GT, qimode, zero, op1));
27388 4 : return;
27389 : }
27390 :
27391 1382 : switch (code)
27392 : {
27393 1297 : case MULT:
27394 1297 : gcc_assert (op2vec);
27395 1297 : if (!TARGET_SSE4_1)
27396 : {
27397 : /* Unpack data such that we've got a source byte in each low byte
27398 : of each word. We don't care what goes into the high byte of
27399 : each word. Rather than trying to get zero in there, most
27400 : convenient is to let it be a copy of the low byte. */
27401 244 : hop1 = copy_to_reg (qop1);
27402 244 : hop2 = copy_to_reg (qop2);
27403 244 : emit_insn (gen_vec_interleave_lowv16qi (hop1, hop1, hop1));
27404 244 : emit_insn (gen_vec_interleave_lowv16qi (hop2, hop2, hop2));
27405 244 : break;
27406 : }
27407 : /* FALLTHRU */
27408 1138 : case ASHIFT:
27409 1138 : case ASHIFTRT:
27410 1138 : case LSHIFTRT:
27411 1138 : hop1 = gen_reg_rtx (V8HImode);
27412 1138 : ix86_expand_sse_unpack (hop1, qop1, uns_p, false);
27413 : /* mult/vashr/vlshr/vashl */
27414 1138 : if (op2vec)
27415 : {
27416 1066 : hop2 = gen_reg_rtx (V8HImode);
27417 1066 : ix86_expand_sse_unpack (hop2, qop2, uns_p, false);
27418 : }
27419 : else
27420 : hop2 = qop2;
27421 :
27422 : break;
27423 0 : default:
27424 0 : gcc_unreachable ();
27425 : }
27426 :
27427 1382 : if (code != MULT && op2vec)
27428 : {
27429 : /* Expand vashr/vlshr/vashl. */
27430 13 : hdest = gen_reg_rtx (V8HImode);
27431 13 : emit_insn (gen_rtx_SET (hdest,
27432 : simplify_gen_binary (code, V8HImode,
27433 : hop1, hop2)));
27434 : }
27435 : else
27436 : /* Expand mult/ashr/lshr/ashl. */
27437 1369 : hdest = expand_simple_binop (V8HImode, code, hop1, hop2,
27438 : NULL_RTX, 1, OPTAB_DIRECT);
27439 :
27440 1382 : if (TARGET_AVX512BW && TARGET_AVX512VL)
27441 : {
27442 57 : if (qimode == V8QImode)
27443 : qdest = dest;
27444 : else
27445 10 : qdest = gen_reg_rtx (V8QImode);
27446 :
27447 57 : emit_insn (gen_truncv8hiv8qi2 (qdest, hdest));
27448 : }
27449 : else
27450 : {
27451 1325 : struct expand_vec_perm_d d;
27452 1325 : rtx qres = gen_lowpart (V16QImode, hdest);
27453 1325 : bool ok;
27454 1325 : int i;
27455 :
27456 : /* Merge the data back into the right place. */
27457 1325 : d.target = qdest;
27458 1325 : d.op0 = d.op1 = qres;
27459 1325 : d.vmode = V16QImode;
27460 1325 : d.nelt = 16;
27461 1325 : d.one_operand_p = TARGET_SSSE3;
27462 1325 : d.testing_p = false;
27463 :
27464 22525 : for (i = 0; i < d.nelt; ++i)
27465 21200 : d.perm[i] = i * 2;
27466 :
27467 1325 : ok = ix86_expand_vec_perm_const_1 (&d);
27468 1325 : gcc_assert (ok);
27469 : }
27470 :
27471 1382 : if (qdest != dest)
27472 1335 : emit_move_insn (dest, gen_lowpart (qimode, qdest));
27473 : }
27474 :
27475 : /* Emit instruction in 2x wider mode. For example, optimize
27476 : vector MUL generation like
27477 :
27478 : vpmovzxbw ymm2, xmm0
27479 : vpmovzxbw ymm3, xmm1
27480 : vpmullw ymm4, ymm2, ymm3
27481 : vpmovwb xmm0, ymm4
27482 :
27483 : it would take less instructions than ix86_expand_vecop_qihi.
27484 : Return true if success. */
27485 :
27486 : static bool
27487 1177 : ix86_expand_vecop_qihi2 (enum rtx_code code, rtx dest, rtx op1, rtx op2)
27488 : {
27489 1177 : machine_mode himode, qimode = GET_MODE (dest);
27490 1177 : machine_mode wqimode;
27491 1177 : rtx qop1, qop2, hop1, hop2, hdest;
27492 1177 : rtx (*gen_truncate)(rtx, rtx) = NULL;
27493 1177 : bool op2vec = GET_MODE_CLASS (GET_MODE (op2)) == MODE_VECTOR_INT;
27494 1177 : bool uns_p = code != ASHIFTRT;
27495 :
27496 : /* Without VPMOVWB (provided by AVX512BW ISA), the expansion uses the
27497 : generic permutation to merge the data back into the right place. This
27498 : permutation results in VPERMQ, which is slow, so better fall back to
27499 : ix86_expand_vecop_qihi. */
27500 1177 : if (!TARGET_AVX512BW
27501 327 : || (qimode == V16QImode && !TARGET_AVX512VL)
27502 : /* There are no V64HImode instructions. */
27503 327 : || qimode == V64QImode)
27504 : return false;
27505 :
27506 : /* Do not generate ymm/zmm instructions when
27507 : target prefers 128/256 bit vector width. */
27508 317 : if ((qimode == V16QImode && TARGET_PREFER_AVX128)
27509 317 : || (qimode == V32QImode && TARGET_PREFER_AVX256))
27510 : return false;
27511 :
27512 312 : switch (qimode)
27513 : {
27514 : case E_V16QImode:
27515 : himode = V16HImode;
27516 : gen_truncate = gen_truncv16hiv16qi2;
27517 : break;
27518 57 : case E_V32QImode:
27519 57 : himode = V32HImode;
27520 57 : gen_truncate = gen_truncv32hiv32qi2;
27521 57 : break;
27522 0 : default:
27523 0 : gcc_unreachable ();
27524 : }
27525 :
27526 312 : wqimode = GET_MODE_2XWIDER_MODE (qimode).require ();
27527 312 : qop1 = lowpart_subreg (wqimode, force_reg (qimode, op1), qimode);
27528 :
27529 312 : if (op2vec)
27530 312 : qop2 = lowpart_subreg (wqimode, force_reg (qimode, op2), qimode);
27531 : else
27532 : qop2 = op2;
27533 :
27534 312 : hop1 = gen_reg_rtx (himode);
27535 312 : ix86_expand_sse_unpack (hop1, qop1, uns_p, false);
27536 :
27537 312 : if (op2vec)
27538 : {
27539 312 : hop2 = gen_reg_rtx (himode);
27540 312 : ix86_expand_sse_unpack (hop2, qop2, uns_p, false);
27541 : }
27542 : else
27543 : hop2 = qop2;
27544 :
27545 312 : if (code != MULT && op2vec)
27546 : {
27547 : /* Expand vashr/vlshr/vashl. */
27548 14 : hdest = gen_reg_rtx (himode);
27549 14 : emit_insn (gen_rtx_SET (hdest,
27550 : simplify_gen_binary (code, himode,
27551 : hop1, hop2)));
27552 : }
27553 : else
27554 : /* Expand mult/ashr/lshr/ashl. */
27555 298 : hdest = expand_simple_binop (himode, code, hop1, hop2,
27556 : NULL_RTX, 1, OPTAB_DIRECT);
27557 :
27558 312 : emit_insn (gen_truncate (dest, hdest));
27559 312 : return true;
27560 : }
27561 :
27562 : /* Expand a vector operation CODE for a V*QImode in terms of the
27563 : same operation on V*HImode. */
27564 :
27565 : void
27566 1497 : ix86_expand_vecop_qihi (enum rtx_code code, rtx dest, rtx op1, rtx op2)
27567 : {
27568 1497 : machine_mode qimode = GET_MODE (dest);
27569 1497 : machine_mode himode;
27570 1497 : rtx (*gen_il) (rtx, rtx, rtx);
27571 1497 : rtx (*gen_ih) (rtx, rtx, rtx);
27572 1497 : rtx op1_l, op1_h, op2_l, op2_h, res_l, res_h;
27573 1497 : bool op2vec = GET_MODE_CLASS (GET_MODE (op2)) == MODE_VECTOR_INT;
27574 1497 : struct expand_vec_perm_d d;
27575 1497 : bool full_interleave = true;
27576 1497 : bool uns_p = code != ASHIFTRT;
27577 1497 : bool ok;
27578 1497 : int i;
27579 :
27580 1497 : if (CONST_INT_P (op2)
27581 320 : && (code == ASHIFT || code == LSHIFTRT || code == ASHIFTRT)
27582 1817 : && ix86_expand_vec_shift_qihi_constant (code, dest, op1, op2))
27583 632 : return;
27584 :
27585 1177 : if (ix86_expand_vecop_qihi2 (code, dest, op1, op2))
27586 : return;
27587 :
27588 865 : switch (qimode)
27589 : {
27590 : case E_V16QImode:
27591 : himode = V8HImode;
27592 : break;
27593 44 : case E_V32QImode:
27594 44 : himode = V16HImode;
27595 44 : break;
27596 10 : case E_V64QImode:
27597 10 : himode = V32HImode;
27598 10 : break;
27599 0 : default:
27600 0 : gcc_unreachable ();
27601 : }
27602 :
27603 865 : switch (code)
27604 : {
27605 819 : case MULT:
27606 819 : gcc_assert (op2vec);
27607 : /* Unpack data such that we've got a source byte in each low byte of
27608 : each word. We don't care what goes into the high byte of each word.
27609 : Rather than trying to get zero in there, most convenient is to let
27610 : it be a copy of the low byte. */
27611 819 : switch (qimode)
27612 : {
27613 : case E_V16QImode:
27614 : gen_il = gen_vec_interleave_lowv16qi;
27615 : gen_ih = gen_vec_interleave_highv16qi;
27616 : break;
27617 44 : case E_V32QImode:
27618 44 : gen_il = gen_avx2_interleave_lowv32qi;
27619 44 : gen_ih = gen_avx2_interleave_highv32qi;
27620 44 : full_interleave = false;
27621 44 : break;
27622 8 : case E_V64QImode:
27623 8 : gen_il = gen_avx512bw_interleave_lowv64qi;
27624 8 : gen_ih = gen_avx512bw_interleave_highv64qi;
27625 8 : full_interleave = false;
27626 8 : break;
27627 : default:
27628 : gcc_unreachable ();
27629 : }
27630 :
27631 819 : op2_l = gen_reg_rtx (qimode);
27632 819 : op2_h = gen_reg_rtx (qimode);
27633 819 : emit_insn (gen_il (op2_l, op2, op2));
27634 819 : emit_insn (gen_ih (op2_h, op2, op2));
27635 :
27636 819 : op1_l = gen_reg_rtx (qimode);
27637 819 : op1_h = gen_reg_rtx (qimode);
27638 819 : emit_insn (gen_il (op1_l, op1, op1));
27639 819 : emit_insn (gen_ih (op1_h, op1, op1));
27640 819 : break;
27641 :
27642 46 : case ASHIFT:
27643 46 : case ASHIFTRT:
27644 46 : case LSHIFTRT:
27645 46 : op1_l = gen_reg_rtx (himode);
27646 46 : op1_h = gen_reg_rtx (himode);
27647 46 : ix86_expand_sse_unpack (op1_l, op1, uns_p, false);
27648 46 : ix86_expand_sse_unpack (op1_h, op1, uns_p, true);
27649 : /* vashr/vlshr/vashl */
27650 46 : if (op2vec)
27651 : {
27652 2 : rtx tmp = force_reg (qimode, op2);
27653 2 : op2_l = gen_reg_rtx (himode);
27654 2 : op2_h = gen_reg_rtx (himode);
27655 2 : ix86_expand_sse_unpack (op2_l, tmp, uns_p, false);
27656 2 : ix86_expand_sse_unpack (op2_h, tmp, uns_p, true);
27657 : }
27658 : else
27659 : op2_l = op2_h = op2;
27660 :
27661 : break;
27662 0 : default:
27663 0 : gcc_unreachable ();
27664 : }
27665 :
27666 865 : if (code != MULT && op2vec)
27667 : {
27668 : /* Expand vashr/vlshr/vashl. */
27669 2 : res_l = gen_reg_rtx (himode);
27670 2 : res_h = gen_reg_rtx (himode);
27671 2 : emit_insn (gen_rtx_SET (res_l,
27672 : simplify_gen_binary (code, himode,
27673 : op1_l, op2_l)));
27674 2 : emit_insn (gen_rtx_SET (res_h,
27675 : simplify_gen_binary (code, himode,
27676 : op1_h, op2_h)));
27677 : }
27678 : else
27679 : {
27680 : /* Expand mult/ashr/lshr/ashl. */
27681 863 : res_l = expand_simple_binop (himode, code, op1_l, op2_l, NULL_RTX,
27682 : 1, OPTAB_DIRECT);
27683 863 : res_h = expand_simple_binop (himode, code, op1_h, op2_h, NULL_RTX,
27684 : 1, OPTAB_DIRECT);
27685 : }
27686 :
27687 865 : gcc_assert (res_l && res_h);
27688 :
27689 : /* Merge the data back into the right place. */
27690 865 : d.target = dest;
27691 865 : d.op0 = gen_lowpart (qimode, res_l);
27692 865 : d.op1 = gen_lowpart (qimode, res_h);
27693 865 : d.vmode = qimode;
27694 865 : d.nelt = GET_MODE_NUNITS (qimode);
27695 865 : d.one_operand_p = false;
27696 865 : d.testing_p = false;
27697 :
27698 865 : if (full_interleave)
27699 : {
27700 : /* We used the full interleave, the desired
27701 : results are in the even elements. */
27702 13917 : for (i = 0; i < d.nelt; ++i)
27703 13104 : d.perm[i] = i * 2;
27704 : }
27705 : else
27706 : {
27707 : /* For AVX, the interleave used above was not cross-lane. So the
27708 : extraction is evens but with the second and third quarter swapped.
27709 : Happily, that is even one insn shorter than even extraction.
27710 : For AVX512BW we have 4 lanes. We extract evens from within a lane,
27711 : always first from the first and then from the second source operand,
27712 : the index bits above the low 4 bits remains the same.
27713 : Thus, for d.nelt == 32 we want permutation
27714 : 0,2,4,..14, 32,34,36,..46, 16,18,20,..30, 48,50,52,..62
27715 : and for d.nelt == 64 we want permutation
27716 : 0,2,4,..14, 64,66,68,..78, 16,18,20,..30, 80,82,84,..94,
27717 : 32,34,36,..46, 96,98,100,..110, 48,50,52,..62, 112,114,116,..126. */
27718 1972 : for (i = 0; i < d.nelt; ++i)
27719 2880 : d.perm[i] = ((i * 2) & 14) + ((i & 8) ? d.nelt : 0) + (i & ~15);
27720 : }
27721 :
27722 865 : ok = ix86_expand_vec_perm_const_1 (&d);
27723 865 : gcc_assert (ok);
27724 : }
27725 :
27726 : /* Helper function of ix86_expand_mul_widen_evenodd. Return true
27727 : if op is CONST_VECTOR with all odd elements equal to their
27728 : preceding element. */
27729 :
27730 : static bool
27731 8922 : const_vector_equal_evenodd_p (rtx op)
27732 : {
27733 8922 : machine_mode mode = GET_MODE (op);
27734 8922 : int i, nunits = GET_MODE_NUNITS (mode);
27735 8922 : if (!CONST_VECTOR_P (op)
27736 8922 : || nunits != CONST_VECTOR_NUNITS (op))
27737 : return false;
27738 3705 : for (i = 0; i < nunits; i += 2)
27739 2980 : if (CONST_VECTOR_ELT (op, i) != CONST_VECTOR_ELT (op, i + 1))
27740 : return false;
27741 : return true;
27742 : }
27743 :
27744 : void
27745 9029 : ix86_expand_mul_widen_evenodd (rtx dest, rtx op1, rtx op2,
27746 : bool uns_p, bool odd_p)
27747 : {
27748 9029 : machine_mode mode = GET_MODE (op1);
27749 9029 : machine_mode wmode = GET_MODE (dest);
27750 9029 : rtx x;
27751 9029 : rtx orig_op1 = op1, orig_op2 = op2;
27752 :
27753 9029 : if (!nonimmediate_operand (op1, mode))
27754 0 : op1 = force_reg (mode, op1);
27755 9029 : if (!nonimmediate_operand (op2, mode))
27756 3436 : op2 = force_reg (mode, op2);
27757 :
27758 : /* We only play even/odd games with vectors of SImode. */
27759 9029 : gcc_assert (mode == V4SImode || mode == V8SImode || mode == V16SImode);
27760 :
27761 : /* If we're looking for the odd results, shift those members down to
27762 : the even slots. For some cpus this is faster than a PSHUFD. */
27763 9029 : if (odd_p)
27764 : {
27765 : /* For XOP use vpmacsdqh, but only for smult, as it is only
27766 : signed. */
27767 4479 : if (TARGET_XOP && mode == V4SImode && !uns_p)
27768 : {
27769 18 : x = force_reg (wmode, CONST0_RTX (wmode));
27770 18 : emit_insn (gen_xop_pmacsdqh (dest, op1, op2, x));
27771 18 : return;
27772 : }
27773 :
27774 8922 : x = GEN_INT (GET_MODE_UNIT_BITSIZE (mode));
27775 4461 : if (!const_vector_equal_evenodd_p (orig_op1))
27776 4461 : op1 = expand_binop (wmode, lshr_optab, gen_lowpart (wmode, op1),
27777 : x, NULL, 1, OPTAB_DIRECT);
27778 4461 : if (!const_vector_equal_evenodd_p (orig_op2))
27779 3736 : op2 = expand_binop (wmode, lshr_optab, gen_lowpart (wmode, op2),
27780 : x, NULL, 1, OPTAB_DIRECT);
27781 4461 : op1 = gen_lowpart (mode, op1);
27782 4461 : op2 = gen_lowpart (mode, op2);
27783 : }
27784 :
27785 9011 : if (mode == V16SImode)
27786 : {
27787 6 : if (uns_p)
27788 0 : x = gen_vec_widen_umult_even_v16si (dest, op1, op2);
27789 : else
27790 6 : x = gen_vec_widen_smult_even_v16si (dest, op1, op2);
27791 : }
27792 9005 : else if (mode == V8SImode)
27793 : {
27794 139 : if (uns_p)
27795 59 : x = gen_vec_widen_umult_even_v8si (dest, op1, op2);
27796 : else
27797 80 : x = gen_vec_widen_smult_even_v8si (dest, op1, op2);
27798 : }
27799 8866 : else if (uns_p)
27800 7753 : x = gen_vec_widen_umult_even_v4si (dest, op1, op2);
27801 1113 : else if (TARGET_SSE4_1)
27802 367 : x = gen_sse4_1_mulv2siv2di3 (dest, op1, op2);
27803 : else
27804 : {
27805 746 : rtx s1, s2, t0, t1, t2;
27806 :
27807 : /* The easiest way to implement this without PMULDQ is to go through
27808 : the motions as if we are performing a full 64-bit multiply. With
27809 : the exception that we need to do less shuffling of the elements. */
27810 :
27811 : /* Compute the sign-extension, aka highparts, of the two operands. */
27812 746 : s1 = ix86_expand_sse_cmp (gen_reg_rtx (mode), GT, CONST0_RTX (mode),
27813 : op1, pc_rtx, pc_rtx);
27814 746 : s2 = ix86_expand_sse_cmp (gen_reg_rtx (mode), GT, CONST0_RTX (mode),
27815 : op2, pc_rtx, pc_rtx);
27816 :
27817 : /* Multiply LO(A) * HI(B), and vice-versa. */
27818 746 : t1 = gen_reg_rtx (wmode);
27819 746 : t2 = gen_reg_rtx (wmode);
27820 746 : emit_insn (gen_vec_widen_umult_even_v4si (t1, s1, op2));
27821 746 : emit_insn (gen_vec_widen_umult_even_v4si (t2, s2, op1));
27822 :
27823 : /* Multiply LO(A) * LO(B). */
27824 746 : t0 = gen_reg_rtx (wmode);
27825 746 : emit_insn (gen_vec_widen_umult_even_v4si (t0, op1, op2));
27826 :
27827 : /* Combine and shift the highparts into place. */
27828 746 : t1 = expand_binop (wmode, add_optab, t1, t2, t1, 1, OPTAB_DIRECT);
27829 746 : t1 = expand_binop (wmode, ashl_optab, t1, GEN_INT (32), t1,
27830 : 1, OPTAB_DIRECT);
27831 :
27832 : /* Combine high and low parts. */
27833 746 : force_expand_binop (wmode, add_optab, t0, t1, dest, 1, OPTAB_DIRECT);
27834 746 : return;
27835 : }
27836 8265 : emit_insn (x);
27837 : }
27838 :
27839 : void
27840 979 : ix86_expand_mul_widen_hilo (rtx dest, rtx op1, rtx op2,
27841 : bool uns_p, bool high_p)
27842 : {
27843 979 : machine_mode wmode = GET_MODE (dest);
27844 979 : machine_mode mode = GET_MODE (op1);
27845 979 : rtx t1, t2, t3, t4, mask;
27846 :
27847 979 : switch (mode)
27848 : {
27849 297 : case E_V4SImode:
27850 297 : t1 = gen_reg_rtx (mode);
27851 297 : t2 = gen_reg_rtx (mode);
27852 297 : if (TARGET_XOP && !uns_p)
27853 : {
27854 : /* With XOP, we have pmacsdqh, aka mul_widen_odd. In this case,
27855 : shuffle the elements once so that all elements are in the right
27856 : place for immediate use: { A C B D }. */
27857 33 : emit_insn (gen_sse2_pshufd_1 (t1, op1, const0_rtx, const2_rtx,
27858 : const1_rtx, GEN_INT (3)));
27859 33 : emit_insn (gen_sse2_pshufd_1 (t2, op2, const0_rtx, const2_rtx,
27860 : const1_rtx, GEN_INT (3)));
27861 : }
27862 : else
27863 : {
27864 : /* Put the elements into place for the multiply. */
27865 264 : ix86_expand_vec_interleave (t1, op1, op1, high_p);
27866 264 : ix86_expand_vec_interleave (t2, op2, op2, high_p);
27867 264 : high_p = false;
27868 : }
27869 297 : ix86_expand_mul_widen_evenodd (dest, t1, t2, uns_p, high_p);
27870 297 : break;
27871 :
27872 70 : case E_V8SImode:
27873 : /* Shuffle the elements between the lanes. After this we
27874 : have { A B E F | C D G H } for each operand. */
27875 70 : t1 = gen_reg_rtx (V4DImode);
27876 70 : t2 = gen_reg_rtx (V4DImode);
27877 70 : emit_insn (gen_avx2_permv4di_1 (t1, gen_lowpart (V4DImode, op1),
27878 : const0_rtx, const2_rtx,
27879 : const1_rtx, GEN_INT (3)));
27880 70 : emit_insn (gen_avx2_permv4di_1 (t2, gen_lowpart (V4DImode, op2),
27881 : const0_rtx, const2_rtx,
27882 : const1_rtx, GEN_INT (3)));
27883 :
27884 : /* Shuffle the elements within the lanes. After this we
27885 : have { A A B B | C C D D } or { E E F F | G G H H }. */
27886 70 : t3 = gen_reg_rtx (V8SImode);
27887 70 : t4 = gen_reg_rtx (V8SImode);
27888 105 : mask = GEN_INT (high_p
27889 : ? 2 + (2 << 2) + (3 << 4) + (3 << 6)
27890 : : 0 + (0 << 2) + (1 << 4) + (1 << 6));
27891 70 : emit_insn (gen_avx2_pshufdv3 (t3, gen_lowpart (V8SImode, t1), mask));
27892 70 : emit_insn (gen_avx2_pshufdv3 (t4, gen_lowpart (V8SImode, t2), mask));
27893 :
27894 70 : ix86_expand_mul_widen_evenodd (dest, t3, t4, uns_p, false);
27895 70 : break;
27896 :
27897 394 : case E_V8HImode:
27898 394 : case E_V16HImode:
27899 394 : t1 = expand_binop (mode, smul_optab, op1, op2, NULL_RTX,
27900 : uns_p, OPTAB_DIRECT);
27901 626 : t2 = expand_binop (mode,
27902 : uns_p ? umul_highpart_optab : smul_highpart_optab,
27903 : op1, op2, NULL_RTX, uns_p, OPTAB_DIRECT);
27904 394 : gcc_assert (t1 && t2);
27905 :
27906 394 : t3 = gen_reg_rtx (mode);
27907 394 : ix86_expand_vec_interleave (t3, t1, t2, high_p);
27908 394 : emit_move_insn (dest, gen_lowpart (wmode, t3));
27909 394 : break;
27910 :
27911 218 : case E_V16QImode:
27912 218 : case E_V32QImode:
27913 218 : case E_V32HImode:
27914 218 : case E_V16SImode:
27915 218 : case E_V64QImode:
27916 218 : t1 = gen_reg_rtx (wmode);
27917 218 : t2 = gen_reg_rtx (wmode);
27918 218 : ix86_expand_sse_unpack (t1, op1, uns_p, high_p);
27919 218 : ix86_expand_sse_unpack (t2, op2, uns_p, high_p);
27920 :
27921 218 : emit_insn (gen_rtx_SET (dest, gen_rtx_MULT (wmode, t1, t2)));
27922 218 : break;
27923 :
27924 0 : default:
27925 0 : gcc_unreachable ();
27926 : }
27927 979 : }
27928 :
27929 : void
27930 3706 : ix86_expand_sse2_mulv4si3 (rtx op0, rtx op1, rtx op2)
27931 : {
27932 3706 : rtx res_1, res_2, res_3, res_4;
27933 :
27934 3706 : res_1 = gen_reg_rtx (V4SImode);
27935 3706 : res_2 = gen_reg_rtx (V4SImode);
27936 3706 : res_3 = gen_reg_rtx (V2DImode);
27937 3706 : res_4 = gen_reg_rtx (V2DImode);
27938 3706 : ix86_expand_mul_widen_evenodd (res_3, op1, op2, true, false);
27939 3706 : ix86_expand_mul_widen_evenodd (res_4, op1, op2, true, true);
27940 :
27941 : /* Move the results in element 2 down to element 1; we don't care
27942 : what goes in elements 2 and 3. Then we can merge the parts
27943 : back together with an interleave.
27944 :
27945 : Note that two other sequences were tried:
27946 : (1) Use interleaves at the start instead of psrldq, which allows
27947 : us to use a single shufps to merge things back at the end.
27948 : (2) Use shufps here to combine the two vectors, then pshufd to
27949 : put the elements in the correct order.
27950 : In both cases the cost of the reformatting stall was too high
27951 : and the overall sequence slower. */
27952 :
27953 3706 : emit_insn (gen_sse2_pshufd_1 (res_1, gen_lowpart (V4SImode, res_3),
27954 : const0_rtx, const2_rtx,
27955 : const0_rtx, const0_rtx));
27956 3706 : emit_insn (gen_sse2_pshufd_1 (res_2, gen_lowpart (V4SImode, res_4),
27957 : const0_rtx, const2_rtx,
27958 : const0_rtx, const0_rtx));
27959 3706 : res_1 = emit_insn (gen_vec_interleave_lowv4si (op0, res_1, res_2));
27960 :
27961 3706 : set_unique_reg_note (res_1, REG_EQUAL, gen_rtx_MULT (V4SImode, op1, op2));
27962 3706 : }
27963 :
27964 : void
27965 554 : ix86_expand_sse2_mulvxdi3 (rtx op0, rtx op1, rtx op2)
27966 : {
27967 554 : machine_mode mode = GET_MODE (op0);
27968 554 : rtx t1, t2, t3, t4, t5, t6;
27969 :
27970 554 : if (TARGET_AVX512DQ && mode == V8DImode)
27971 32 : emit_insn (gen_avx512dq_mulv8di3 (op0, op1, op2));
27972 522 : else if (TARGET_AVX512DQ && TARGET_AVX512VL && mode == V4DImode)
27973 32 : emit_insn (gen_avx512dq_mulv4di3 (op0, op1, op2));
27974 490 : else if (TARGET_AVX512DQ && TARGET_AVX512VL && mode == V2DImode)
27975 36 : emit_insn (gen_avx512dq_mulv2di3 (op0, op1, op2));
27976 454 : else if (TARGET_XOP && mode == V2DImode)
27977 : {
27978 : /* op1: A,B,C,D, op2: E,F,G,H */
27979 2 : op1 = gen_lowpart (V4SImode, op1);
27980 2 : op2 = gen_lowpart (V4SImode, op2);
27981 :
27982 2 : t1 = gen_reg_rtx (V4SImode);
27983 2 : t2 = gen_reg_rtx (V4SImode);
27984 2 : t3 = gen_reg_rtx (V2DImode);
27985 2 : t4 = gen_reg_rtx (V2DImode);
27986 :
27987 : /* t1: B,A,D,C */
27988 2 : emit_insn (gen_sse2_pshufd_1 (t1, op1,
27989 : GEN_INT (1),
27990 : GEN_INT (0),
27991 : GEN_INT (3),
27992 : GEN_INT (2)));
27993 :
27994 : /* t2: (B*E),(A*F),(D*G),(C*H) */
27995 2 : emit_insn (gen_mulv4si3 (t2, t1, op2));
27996 :
27997 : /* t3: (B*E)+(A*F), (D*G)+(C*H) */
27998 2 : emit_insn (gen_xop_phadddq (t3, t2));
27999 :
28000 : /* t4: ((B*E)+(A*F))<<32, ((D*G)+(C*H))<<32 */
28001 2 : emit_insn (gen_ashlv2di3 (t4, t3, GEN_INT (32)));
28002 :
28003 : /* Multiply lower parts and add all */
28004 2 : t5 = gen_reg_rtx (V2DImode);
28005 2 : emit_insn (gen_vec_widen_umult_even_v4si (t5,
28006 2 : gen_lowpart (V4SImode, op1),
28007 2 : gen_lowpart (V4SImode, op2)));
28008 2 : force_expand_binop (mode, add_optab, t5, t4, op0, 1, OPTAB_DIRECT);
28009 : }
28010 : else
28011 : {
28012 452 : machine_mode nmode;
28013 452 : rtx (*umul) (rtx, rtx, rtx);
28014 :
28015 452 : if (mode == V2DImode)
28016 : {
28017 : umul = gen_vec_widen_umult_even_v4si;
28018 : nmode = V4SImode;
28019 : }
28020 295 : else if (mode == V4DImode)
28021 : {
28022 : umul = gen_vec_widen_umult_even_v8si;
28023 : nmode = V8SImode;
28024 : }
28025 116 : else if (mode == V8DImode)
28026 : {
28027 : umul = gen_vec_widen_umult_even_v16si;
28028 : nmode = V16SImode;
28029 : }
28030 : else
28031 0 : gcc_unreachable ();
28032 :
28033 :
28034 : /* Multiply low parts. */
28035 452 : t1 = gen_reg_rtx (mode);
28036 452 : emit_insn (umul (t1, gen_lowpart (nmode, op1), gen_lowpart (nmode, op2)));
28037 :
28038 : /* Shift input vectors right 32 bits so we can multiply high parts. */
28039 452 : t6 = GEN_INT (32);
28040 452 : t2 = expand_binop (mode, lshr_optab, op1, t6, NULL, 1, OPTAB_DIRECT);
28041 452 : t3 = expand_binop (mode, lshr_optab, op2, t6, NULL, 1, OPTAB_DIRECT);
28042 :
28043 : /* Multiply high parts by low parts. */
28044 452 : t4 = gen_reg_rtx (mode);
28045 452 : t5 = gen_reg_rtx (mode);
28046 452 : emit_insn (umul (t4, gen_lowpart (nmode, t2), gen_lowpart (nmode, op2)));
28047 452 : emit_insn (umul (t5, gen_lowpart (nmode, t3), gen_lowpart (nmode, op1)));
28048 :
28049 : /* Combine and shift the highparts back. */
28050 452 : t4 = expand_binop (mode, add_optab, t4, t5, t4, 1, OPTAB_DIRECT);
28051 452 : t4 = expand_binop (mode, ashl_optab, t4, t6, t4, 1, OPTAB_DIRECT);
28052 :
28053 : /* Combine high and low parts. */
28054 452 : force_expand_binop (mode, add_optab, t1, t4, op0, 1, OPTAB_DIRECT);
28055 : }
28056 :
28057 554 : set_unique_reg_note (get_last_insn (), REG_EQUAL,
28058 : gen_rtx_MULT (mode, op1, op2));
28059 554 : }
28060 :
28061 : /* Return 1 if control transfer instruction INSN
28062 : should be encoded with notrack prefix. */
28063 :
28064 : bool
28065 15209545 : ix86_notrack_prefixed_insn_p (rtx_insn *insn)
28066 : {
28067 15209545 : if (!insn || !((flag_cf_protection & CF_BRANCH)))
28068 : return false;
28069 :
28070 4152816 : if (CALL_P (insn))
28071 : {
28072 1496822 : rtx call = get_call_rtx_from (insn);
28073 1496822 : gcc_assert (call != NULL_RTX);
28074 1496822 : rtx addr = XEXP (call, 0);
28075 :
28076 : /* Do not emit 'notrack' if it's not an indirect call. */
28077 1496822 : if (MEM_P (addr)
28078 1496822 : && SYMBOL_REF_P (XEXP (addr, 0)))
28079 : return false;
28080 : else
28081 67274 : return find_reg_note (insn, REG_CALL_NOCF_CHECK, 0);
28082 : }
28083 :
28084 2655994 : if (JUMP_P (insn) && !flag_cet_switch)
28085 : {
28086 2642367 : rtx target = JUMP_LABEL (insn);
28087 2642367 : if (target == NULL_RTX || ANY_RETURN_P (target))
28088 : return false;
28089 :
28090 : /* Check the jump is a switch table. */
28091 2642329 : rtx_insn *label = as_a<rtx_insn *> (target);
28092 2642329 : rtx_insn *table = next_insn (label);
28093 2642329 : if (table == NULL_RTX || !JUMP_TABLE_DATA_P (table))
28094 : return false;
28095 : else
28096 4265 : return true;
28097 : }
28098 : return false;
28099 : }
28100 :
28101 : /* Calculate integer abs() using only SSE2 instructions. */
28102 :
28103 : void
28104 609 : ix86_expand_sse2_abs (rtx target, rtx input)
28105 : {
28106 609 : machine_mode mode = GET_MODE (target);
28107 609 : rtx tmp0, tmp1, x;
28108 :
28109 609 : switch (mode)
28110 : {
28111 49 : case E_V2DImode:
28112 49 : case E_V4DImode:
28113 : /* For 64-bit signed integer X, with SSE4.2 use
28114 : pxor t0, t0; pcmpgtq X, t0; pxor t0, X; psubq t0, X.
28115 : Otherwise handle it similarly to V4SImode, except use 64 as W instead of
28116 : 32 and use logical instead of arithmetic right shift (which is
28117 : unimplemented) and subtract. */
28118 49 : if (TARGET_SSE4_2)
28119 : {
28120 9 : tmp0 = gen_reg_rtx (mode);
28121 9 : tmp1 = gen_reg_rtx (mode);
28122 9 : emit_move_insn (tmp1, CONST0_RTX (mode));
28123 9 : if (mode == E_V2DImode)
28124 6 : emit_insn (gen_sse4_2_gtv2di3 (tmp0, tmp1, input));
28125 : else
28126 3 : emit_insn (gen_avx2_gtv4di3 (tmp0, tmp1, input));
28127 : }
28128 : else
28129 : {
28130 80 : tmp0 = expand_simple_binop (mode, LSHIFTRT, input,
28131 40 : GEN_INT (GET_MODE_UNIT_BITSIZE (mode)
28132 : - 1), NULL, 0, OPTAB_DIRECT);
28133 40 : tmp0 = expand_simple_unop (mode, NEG, tmp0, NULL, false);
28134 : }
28135 :
28136 49 : tmp1 = expand_simple_binop (mode, XOR, tmp0, input,
28137 : NULL, 0, OPTAB_DIRECT);
28138 49 : x = expand_simple_binop (mode, MINUS, tmp1, tmp0,
28139 : target, 0, OPTAB_DIRECT);
28140 49 : break;
28141 :
28142 76 : case E_V4SImode:
28143 : /* For 32-bit signed integer X, the best way to calculate the absolute
28144 : value of X is (((signed) X >> (W-1)) ^ X) - ((signed) X >> (W-1)). */
28145 76 : tmp0 = expand_simple_binop (mode, ASHIFTRT, input,
28146 76 : GEN_INT (GET_MODE_UNIT_BITSIZE (mode) - 1),
28147 : NULL, 0, OPTAB_DIRECT);
28148 76 : tmp1 = expand_simple_binop (mode, XOR, tmp0, input,
28149 : NULL, 0, OPTAB_DIRECT);
28150 76 : x = expand_simple_binop (mode, MINUS, tmp1, tmp0,
28151 : target, 0, OPTAB_DIRECT);
28152 76 : break;
28153 :
28154 97 : case E_V8HImode:
28155 : /* For 16-bit signed integer X, the best way to calculate the absolute
28156 : value of X is max (X, -X), as SSE2 provides the PMAXSW insn. */
28157 97 : tmp0 = expand_unop (mode, neg_optab, input, NULL_RTX, 0);
28158 :
28159 97 : x = expand_simple_binop (mode, SMAX, tmp0, input,
28160 : target, 0, OPTAB_DIRECT);
28161 97 : break;
28162 :
28163 387 : case E_V16QImode:
28164 : /* For 8-bit signed integer X, the best way to calculate the absolute
28165 : value of X is min ((unsigned char) X, (unsigned char) (-X)),
28166 : as SSE2 provides the PMINUB insn. */
28167 387 : tmp0 = expand_unop (mode, neg_optab, input, NULL_RTX, 0);
28168 :
28169 387 : x = expand_simple_binop (V16QImode, UMIN, tmp0, input,
28170 : target, 0, OPTAB_DIRECT);
28171 387 : break;
28172 :
28173 0 : default:
28174 0 : gcc_unreachable ();
28175 : }
28176 :
28177 609 : if (x != target)
28178 0 : emit_move_insn (target, x);
28179 609 : }
28180 :
28181 : /* Expand an extract from a vector register through pextr insn.
28182 : Return true if successful. */
28183 :
28184 : bool
28185 120959 : ix86_expand_pextr (rtx *operands)
28186 : {
28187 120959 : rtx dst = operands[0];
28188 120959 : rtx src = operands[1];
28189 :
28190 120959 : unsigned int size = INTVAL (operands[2]);
28191 120959 : unsigned int pos = INTVAL (operands[3]);
28192 :
28193 120959 : if (SUBREG_P (dst))
28194 : {
28195 : /* Reject non-lowpart subregs. */
28196 77693 : if (SUBREG_BYTE (dst) > 0)
28197 : return false;
28198 77564 : dst = SUBREG_REG (dst);
28199 : }
28200 :
28201 120830 : if (SUBREG_P (src))
28202 : {
28203 34468 : pos += SUBREG_BYTE (src) * BITS_PER_UNIT;
28204 34468 : src = SUBREG_REG (src);
28205 : }
28206 :
28207 120830 : switch (GET_MODE (src))
28208 : {
28209 0 : case E_V16QImode:
28210 0 : case E_V8HImode:
28211 0 : case E_V4SImode:
28212 0 : case E_V2DImode:
28213 0 : case E_V1TImode:
28214 0 : {
28215 0 : machine_mode srcmode, dstmode;
28216 0 : rtx d, pat;
28217 :
28218 0 : if (!int_mode_for_size (size, 0).exists (&dstmode))
28219 0 : return false;
28220 :
28221 0 : switch (dstmode)
28222 : {
28223 0 : case E_QImode:
28224 0 : if (!TARGET_SSE4_1)
28225 : return false;
28226 : srcmode = V16QImode;
28227 : break;
28228 :
28229 0 : case E_HImode:
28230 0 : if (!TARGET_SSE2)
28231 : return false;
28232 : srcmode = V8HImode;
28233 : break;
28234 :
28235 0 : case E_SImode:
28236 0 : if (!TARGET_SSE4_1)
28237 : return false;
28238 : srcmode = V4SImode;
28239 : break;
28240 :
28241 0 : case E_DImode:
28242 0 : gcc_assert (TARGET_64BIT);
28243 0 : if (!TARGET_SSE4_1)
28244 : return false;
28245 : srcmode = V2DImode;
28246 : break;
28247 :
28248 : default:
28249 : return false;
28250 : }
28251 :
28252 : /* Reject extractions from misaligned positions. */
28253 0 : if (pos & (size-1))
28254 : return false;
28255 :
28256 0 : if (GET_MODE (dst) == dstmode)
28257 : d = dst;
28258 : else
28259 0 : d = gen_reg_rtx (dstmode);
28260 :
28261 : /* Construct insn pattern. */
28262 0 : pat = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (pos / size)));
28263 0 : pat = gen_rtx_VEC_SELECT (dstmode, gen_lowpart (srcmode, src), pat);
28264 :
28265 : /* Let the rtl optimizers know about the zero extension performed. */
28266 0 : if (dstmode == QImode || dstmode == HImode)
28267 : {
28268 0 : pat = gen_rtx_ZERO_EXTEND (SImode, pat);
28269 0 : d = gen_lowpart (SImode, d);
28270 : }
28271 :
28272 0 : emit_insn (gen_rtx_SET (d, pat));
28273 :
28274 0 : if (d != dst)
28275 0 : emit_move_insn (dst, gen_lowpart (GET_MODE (dst), d));
28276 : return true;
28277 : }
28278 :
28279 : default:
28280 : return false;
28281 : }
28282 : }
28283 :
28284 : /* Expand an insert into a vector register through pinsr insn.
28285 : Return true if successful. */
28286 :
28287 : bool
28288 111336 : ix86_expand_pinsr (rtx *operands)
28289 : {
28290 111336 : rtx dst = operands[0];
28291 111336 : rtx src = operands[3];
28292 :
28293 111336 : unsigned int size = INTVAL (operands[1]);
28294 111336 : unsigned int pos = INTVAL (operands[2]);
28295 :
28296 111336 : if (SUBREG_P (dst))
28297 : {
28298 64157 : pos += SUBREG_BYTE (dst) * BITS_PER_UNIT;
28299 64157 : dst = SUBREG_REG (dst);
28300 : }
28301 :
28302 111336 : switch (GET_MODE (dst))
28303 : {
28304 20 : case E_V16QImode:
28305 20 : case E_V8HImode:
28306 20 : case E_V4SImode:
28307 20 : case E_V2DImode:
28308 20 : case E_V1TImode:
28309 20 : {
28310 20 : machine_mode srcmode, dstmode;
28311 20 : rtx (*pinsr)(rtx, rtx, rtx, rtx);
28312 20 : rtx d;
28313 :
28314 20 : if (!int_mode_for_size (size, 0).exists (&srcmode))
28315 0 : return false;
28316 :
28317 20 : switch (srcmode)
28318 : {
28319 1 : case E_QImode:
28320 1 : if (!TARGET_SSE4_1)
28321 : return false;
28322 : dstmode = V16QImode;
28323 : pinsr = gen_sse4_1_pinsrb;
28324 : break;
28325 :
28326 5 : case E_HImode:
28327 5 : if (!TARGET_SSE2)
28328 : return false;
28329 : dstmode = V8HImode;
28330 : pinsr = gen_sse2_pinsrw;
28331 : break;
28332 :
28333 14 : case E_SImode:
28334 14 : if (!TARGET_SSE4_1)
28335 : return false;
28336 : dstmode = V4SImode;
28337 : pinsr = gen_sse4_1_pinsrd;
28338 : break;
28339 :
28340 0 : case E_DImode:
28341 0 : gcc_assert (TARGET_64BIT);
28342 0 : if (!TARGET_SSE4_1)
28343 : return false;
28344 : dstmode = V2DImode;
28345 : pinsr = gen_sse4_1_pinsrq;
28346 : break;
28347 :
28348 : default:
28349 : return false;
28350 : }
28351 :
28352 : /* Reject insertions to misaligned positions. */
28353 7 : if (pos & (size-1))
28354 : return false;
28355 :
28356 7 : if (SUBREG_P (src))
28357 : {
28358 7 : unsigned int srcpos = SUBREG_BYTE (src);
28359 :
28360 7 : if (srcpos > 0)
28361 : {
28362 0 : rtx extr_ops[4];
28363 :
28364 0 : extr_ops[0] = gen_reg_rtx (srcmode);
28365 0 : extr_ops[1] = gen_lowpart (srcmode, SUBREG_REG (src));
28366 0 : extr_ops[2] = GEN_INT (size);
28367 0 : extr_ops[3] = GEN_INT (srcpos * BITS_PER_UNIT);
28368 :
28369 0 : if (!ix86_expand_pextr (extr_ops))
28370 0 : return false;
28371 :
28372 0 : src = extr_ops[0];
28373 : }
28374 : else
28375 7 : src = gen_lowpart (srcmode, SUBREG_REG (src));
28376 : }
28377 :
28378 7 : if (GET_MODE (dst) == dstmode)
28379 : d = dst;
28380 : else
28381 7 : d = gen_reg_rtx (dstmode);
28382 :
28383 7 : emit_insn (pinsr (d, gen_lowpart (dstmode, dst),
28384 7 : gen_lowpart (srcmode, src),
28385 7 : GEN_INT (1 << (pos / size))));
28386 7 : if (d != dst)
28387 7 : emit_move_insn (dst, gen_lowpart (GET_MODE (dst), d));
28388 : return true;
28389 : }
28390 :
28391 : default:
28392 : return false;
28393 : }
28394 : }
28395 :
28396 : /* All CPUs prefer to avoid cross-lane operations so perform reductions
28397 : upper against lower halves up to SSE reg size. */
28398 :
28399 : machine_mode
28400 1988 : ix86_split_reduction (machine_mode mode)
28401 : {
28402 : /* Reduce lowpart against highpart until we reach SSE reg width to
28403 : avoid cross-lane operations. */
28404 1988 : switch (mode)
28405 : {
28406 : case E_V8DImode:
28407 : case E_V4DImode:
28408 : return V2DImode;
28409 9 : case E_V16SImode:
28410 9 : case E_V8SImode:
28411 9 : return V4SImode;
28412 8 : case E_V32HImode:
28413 8 : case E_V16HImode:
28414 8 : return V8HImode;
28415 4 : case E_V64QImode:
28416 4 : case E_V32QImode:
28417 4 : return V16QImode;
28418 5 : case E_V16SFmode:
28419 5 : case E_V8SFmode:
28420 5 : return V4SFmode;
28421 16 : case E_V8DFmode:
28422 16 : case E_V4DFmode:
28423 16 : return V2DFmode;
28424 1941 : default:
28425 1941 : return mode;
28426 : }
28427 : }
28428 :
28429 : /* Generate call to __divmoddi4. */
28430 :
28431 : void
28432 894 : ix86_expand_divmod_libfunc (rtx libfunc, machine_mode mode,
28433 : rtx op0, rtx op1,
28434 : rtx *quot_p, rtx *rem_p)
28435 : {
28436 1788 : rtx rem = assign_stack_temp (mode, GET_MODE_SIZE (mode));
28437 :
28438 894 : rtx quot = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
28439 : mode, op0, mode, op1, mode,
28440 894 : XEXP (rem, 0), Pmode);
28441 894 : *quot_p = quot;
28442 894 : *rem_p = rem;
28443 894 : }
28444 :
28445 : void
28446 80 : ix86_expand_atomic_fetch_op_loop (rtx target, rtx mem, rtx val,
28447 : enum rtx_code code, bool after,
28448 : bool doubleword)
28449 : {
28450 80 : rtx old_reg, new_reg, old_mem, success;
28451 80 : machine_mode mode = GET_MODE (target);
28452 80 : rtx_code_label *loop_label = NULL;
28453 :
28454 80 : old_reg = gen_reg_rtx (mode);
28455 80 : new_reg = old_reg;
28456 80 : old_mem = copy_to_reg (mem);
28457 80 : loop_label = gen_label_rtx ();
28458 80 : emit_label (loop_label);
28459 80 : emit_move_insn (old_reg, old_mem);
28460 :
28461 : /* return value for atomic_fetch_op. */
28462 80 : if (!after)
28463 40 : emit_move_insn (target, old_reg);
28464 :
28465 80 : if (code == NOT)
28466 : {
28467 20 : new_reg = expand_simple_binop (mode, AND, new_reg, val, NULL_RTX,
28468 : true, OPTAB_LIB_WIDEN);
28469 20 : new_reg = expand_simple_unop (mode, code, new_reg, NULL_RTX, true);
28470 : }
28471 : else
28472 60 : new_reg = expand_simple_binop (mode, code, new_reg, val, NULL_RTX,
28473 : true, OPTAB_LIB_WIDEN);
28474 :
28475 : /* return value for atomic_op_fetch. */
28476 80 : if (after)
28477 40 : emit_move_insn (target, new_reg);
28478 :
28479 80 : success = NULL_RTX;
28480 :
28481 80 : ix86_expand_cmpxchg_loop (&success, old_mem, mem, old_reg, new_reg,
28482 : gen_int_mode (MEMMODEL_SYNC_SEQ_CST,
28483 : SImode),
28484 : doubleword, loop_label);
28485 80 : }
28486 :
28487 : /* Relax cmpxchg instruction, param loop_label indicates whether
28488 : the instruction should be relaxed with a pause loop. If not,
28489 : it will be relaxed to an atomic load + compare, and skip
28490 : cmpxchg instruction if mem != exp_input. */
28491 :
28492 : void
28493 90 : ix86_expand_cmpxchg_loop (rtx *ptarget_bool, rtx target_val,
28494 : rtx mem, rtx exp_input, rtx new_input,
28495 : rtx mem_model, bool doubleword,
28496 : rtx_code_label *loop_label)
28497 : {
28498 90 : rtx_code_label *cmp_label = NULL;
28499 90 : rtx_code_label *done_label = NULL;
28500 90 : rtx target_bool = NULL_RTX, new_mem = NULL_RTX;
28501 90 : rtx (*gen) (rtx, rtx, rtx, rtx, rtx) = NULL;
28502 90 : rtx (*gendw) (rtx, rtx, rtx, rtx, rtx, rtx) = NULL;
28503 90 : machine_mode mode = GET_MODE (target_val), hmode = mode;
28504 :
28505 90 : if (*ptarget_bool == NULL)
28506 80 : target_bool = gen_reg_rtx (QImode);
28507 : else
28508 : target_bool = *ptarget_bool;
28509 :
28510 90 : cmp_label = gen_label_rtx ();
28511 90 : done_label = gen_label_rtx ();
28512 :
28513 90 : new_mem = gen_reg_rtx (mode);
28514 : /* Load memory first. */
28515 90 : expand_atomic_load (new_mem, mem, MEMMODEL_SEQ_CST);
28516 :
28517 90 : switch (mode)
28518 : {
28519 : case E_TImode:
28520 : gendw = gen_atomic_compare_and_swapti_doubleword;
28521 : hmode = DImode;
28522 : break;
28523 18 : case E_DImode:
28524 18 : if (doubleword)
28525 : {
28526 : gendw = gen_atomic_compare_and_swapdi_doubleword;
28527 : hmode = SImode;
28528 : }
28529 : else
28530 : gen = gen_atomic_compare_and_swapdi_1;
28531 : break;
28532 18 : case E_SImode:
28533 18 : gen = gen_atomic_compare_and_swapsi_1;
28534 18 : break;
28535 18 : case E_HImode:
28536 18 : gen = gen_atomic_compare_and_swaphi_1;
28537 18 : break;
28538 18 : case E_QImode:
28539 18 : gen = gen_atomic_compare_and_swapqi_1;
28540 18 : break;
28541 0 : default:
28542 0 : gcc_unreachable ();
28543 : }
28544 :
28545 : /* Compare mem value with expected value. */
28546 72 : if (doubleword)
28547 : {
28548 18 : rtx low_new_mem = gen_lowpart (hmode, new_mem);
28549 18 : rtx low_exp_input = gen_lowpart (hmode, exp_input);
28550 18 : rtx high_new_mem = gen_highpart (hmode, new_mem);
28551 18 : rtx high_exp_input = gen_highpart (hmode, exp_input);
28552 18 : emit_cmp_and_jump_insns (low_new_mem, low_exp_input, NE, NULL_RTX,
28553 : hmode, 1, cmp_label,
28554 : profile_probability::guessed_never ());
28555 18 : emit_cmp_and_jump_insns (high_new_mem, high_exp_input, NE, NULL_RTX,
28556 : hmode, 1, cmp_label,
28557 : profile_probability::guessed_never ());
28558 : }
28559 : else
28560 72 : emit_cmp_and_jump_insns (new_mem, exp_input, NE, NULL_RTX,
28561 72 : GET_MODE (exp_input), 1, cmp_label,
28562 : profile_probability::guessed_never ());
28563 :
28564 : /* Directly emits cmpxchg here. */
28565 90 : if (doubleword)
28566 36 : emit_insn (gendw (target_val, mem, exp_input,
28567 18 : gen_lowpart (hmode, new_input),
28568 : gen_highpart (hmode, new_input),
28569 : mem_model));
28570 : else
28571 72 : emit_insn (gen (target_val, mem, exp_input, new_input, mem_model));
28572 :
28573 90 : if (!loop_label)
28574 : {
28575 10 : emit_jump_insn (gen_jump (done_label));
28576 10 : emit_barrier ();
28577 10 : emit_label (cmp_label);
28578 10 : emit_move_insn (target_val, new_mem);
28579 10 : emit_label (done_label);
28580 10 : ix86_expand_setcc (target_bool, EQ, gen_rtx_REG (CCZmode, FLAGS_REG),
28581 : const0_rtx);
28582 : }
28583 : else
28584 : {
28585 80 : ix86_expand_setcc (target_bool, EQ, gen_rtx_REG (CCZmode, FLAGS_REG),
28586 : const0_rtx);
28587 80 : emit_cmp_and_jump_insns (target_bool, const0_rtx, EQ, const0_rtx,
28588 80 : GET_MODE (target_bool), 1, loop_label,
28589 : profile_probability::guessed_never ());
28590 80 : emit_jump_insn (gen_jump (done_label));
28591 80 : emit_barrier ();
28592 :
28593 : /* If mem is not expected, pause and loop back. */
28594 80 : emit_label (cmp_label);
28595 80 : emit_move_insn (target_val, new_mem);
28596 80 : emit_insn (gen_pause ());
28597 80 : emit_jump_insn (gen_jump (loop_label));
28598 80 : emit_barrier ();
28599 80 : emit_label (done_label);
28600 : }
28601 :
28602 90 : *ptarget_bool = target_bool;
28603 90 : }
28604 :
28605 : /* Convert a BFmode VAL to SFmode without signaling sNaNs.
28606 : This is done by returning SF SUBREG of ((HI SUBREG) (VAL)) << 16. */
28607 :
28608 : rtx
28609 4104 : ix86_expand_fast_convert_bf_to_sf (rtx val)
28610 : {
28611 4104 : rtx op = gen_lowpart (HImode, val), ret;
28612 4104 : if (CONST_INT_P (op))
28613 : {
28614 613 : ret = simplify_const_unary_operation (FLOAT_EXTEND, SFmode,
28615 : val, BFmode);
28616 613 : if (ret)
28617 : return ret;
28618 : /* FLOAT_EXTEND simplification will fail if VAL is a sNaN. */
28619 1 : ret = gen_reg_rtx (SImode);
28620 1 : emit_move_insn (ret, GEN_INT (INTVAL (op) & 0xffff));
28621 1 : emit_insn (gen_ashlsi3 (ret, ret, GEN_INT (16)));
28622 1 : return gen_lowpart (SFmode, ret);
28623 : }
28624 :
28625 3491 : ret = gen_reg_rtx (SFmode);
28626 3491 : emit_insn (gen_extendbfsf2_1 (ret, force_reg (BFmode, val)));
28627 3491 : return ret;
28628 : }
28629 :
28630 : rtx
28631 65582 : ix86_gen_ccmp_first (rtx_insn **prep_seq, rtx_insn **gen_seq,
28632 : rtx_code code, tree treeop0, tree treeop1)
28633 : {
28634 65582 : if (!TARGET_APX_CCMP)
28635 : return NULL_RTX;
28636 :
28637 65582 : rtx op0, op1, res;
28638 65582 : machine_mode op_mode;
28639 :
28640 65582 : start_sequence ();
28641 65582 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
28642 :
28643 65582 : op_mode = GET_MODE (op0);
28644 65582 : if (op_mode == VOIDmode)
28645 0 : op_mode = GET_MODE (op1);
28646 :
28647 : /* We only supports following scalar comparisons that use just 1
28648 : instruction: DI/SI/HI/QI/XF/DF/SF/HF.
28649 : Unordered/Ordered compare cannot be correctly identified by
28650 : ccmp so they are not supported. */
28651 32771 : if (!(op_mode == DImode || op_mode == SImode
28652 65582 : || op_mode == HImode || op_mode == QImode
28653 32780 : || ((op_mode == XFmode || op_mode == DFmode || op_mode == SFmode)
28654 10 : && (TARGET_80387
28655 1 : || (SSE_FLOAT_MODE_P (op_mode) && TARGET_SSE_MATH)))
28656 0 : || (op_mode == HFmode && TARGET_AVX512FP16))
28657 32811 : || code == ORDERED
28658 32811 : || code == UNORDERED)
28659 : {
28660 32771 : end_sequence ();
28661 32771 : return NULL_RTX;
28662 : }
28663 :
28664 : /* Canonicalize the operands according to mode. */
28665 32811 : if (SCALAR_INT_MODE_P (op_mode))
28666 : {
28667 32802 : if (!nonimmediate_operand (op0, op_mode))
28668 0 : op0 = force_reg (op_mode, op0);
28669 32802 : if (!x86_64_general_operand (op1, op_mode))
28670 0 : op1 = force_reg (op_mode, op1);
28671 : }
28672 : else
28673 : {
28674 : /* op0/op1 can be canonicallized from expand_fp_compare, so
28675 : just adjust the code to make it generate supported fp
28676 : condition. */
28677 9 : if (ix86_fp_compare_code_to_integer (code) == UNKNOWN)
28678 : {
28679 : /* First try to split condition if we don't need to honor
28680 : NaNs, as the ORDERED/UNORDERED check always fall
28681 : through. */
28682 6 : if (!HONOR_NANS (op_mode))
28683 : {
28684 6 : rtx_code first_code;
28685 6 : split_comparison (code, op_mode, &first_code, &code);
28686 : }
28687 : /* Otherwise try to swap the operand order and check if
28688 : the comparison is supported. */
28689 : else
28690 : {
28691 0 : code = swap_condition (code);
28692 0 : std::swap (op0, op1);
28693 : }
28694 :
28695 6 : if (ix86_fp_compare_code_to_integer (code) == UNKNOWN)
28696 : {
28697 0 : end_sequence ();
28698 0 : return NULL_RTX;
28699 : }
28700 : }
28701 : }
28702 :
28703 32811 : *prep_seq = end_sequence ();
28704 :
28705 32811 : start_sequence ();
28706 :
28707 32811 : res = ix86_expand_compare (code, op0, op1);
28708 :
28709 32811 : if (!res)
28710 : {
28711 : end_sequence ();
28712 : return NULL_RTX;
28713 : }
28714 32811 : *gen_seq = end_sequence ();
28715 :
28716 32811 : return res;
28717 : }
28718 :
28719 : rtx
28720 32814 : ix86_gen_ccmp_next (rtx_insn **prep_seq, rtx_insn **gen_seq, rtx prev,
28721 : rtx_code cmp_code, tree treeop0, tree treeop1,
28722 : rtx_code bit_code)
28723 : {
28724 32814 : if (!TARGET_APX_CCMP)
28725 : return NULL_RTX;
28726 :
28727 32814 : rtx op0, op1, target;
28728 32814 : machine_mode op_mode, cmp_mode, cc_mode = CCmode;
28729 32814 : int unsignedp = TYPE_UNSIGNED (TREE_TYPE (treeop0));
28730 32814 : insn_code icode;
28731 32814 : rtx_code prev_code;
28732 32814 : struct expand_operand ops[5];
28733 32814 : int dfv;
28734 :
28735 : /* Exit early for non integer modes to avoid O(n^2) part of expand_operands. */
28736 32814 : cmp_mode = op_mode = TYPE_MODE (TREE_TYPE (treeop0));
28737 :
28738 32814 : if (!(op_mode == DImode || op_mode == SImode || op_mode == HImode
28739 : || op_mode == QImode))
28740 : return NULL_RTX;
28741 :
28742 34 : push_to_sequence (*prep_seq);
28743 34 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
28744 :
28745 34 : icode = code_for_ccmp (op_mode);
28746 :
28747 34 : op0 = prepare_operand (icode, op0, 2, op_mode, cmp_mode, unsignedp);
28748 34 : op1 = prepare_operand (icode, op1, 3, op_mode, cmp_mode, unsignedp);
28749 34 : if (!op0 || !op1)
28750 : {
28751 0 : end_sequence ();
28752 0 : return NULL_RTX;
28753 : }
28754 :
28755 34 : *prep_seq = end_sequence ();
28756 :
28757 34 : target = gen_rtx_REG (cc_mode, FLAGS_REG);
28758 34 : dfv = ix86_get_flags_cc ((rtx_code) cmp_code);
28759 :
28760 34 : prev_code = GET_CODE (prev);
28761 : /* Fixup FP compare code here. */
28762 34 : if (GET_MODE (XEXP (prev, 0)) == CCFPmode)
28763 9 : prev_code = ix86_fp_compare_code_to_integer (prev_code);
28764 :
28765 34 : if (bit_code != AND)
28766 17 : prev_code = reverse_condition (prev_code);
28767 : else
28768 17 : dfv = (int)(dfv ^ 1);
28769 :
28770 34 : prev = gen_rtx_fmt_ee (prev_code, VOIDmode, XEXP (prev, 0),
28771 : const0_rtx);
28772 :
28773 34 : create_fixed_operand (&ops[0], target);
28774 34 : create_fixed_operand (&ops[1], prev);
28775 34 : create_fixed_operand (&ops[2], op0);
28776 34 : create_fixed_operand (&ops[3], op1);
28777 34 : create_fixed_operand (&ops[4], GEN_INT (dfv));
28778 :
28779 34 : push_to_sequence (*gen_seq);
28780 34 : if (!maybe_expand_insn (icode, 5, ops))
28781 : {
28782 0 : end_sequence ();
28783 0 : return NULL_RTX;
28784 : }
28785 :
28786 34 : *gen_seq = end_sequence ();
28787 :
28788 34 : return gen_rtx_fmt_ee ((rtx_code) cmp_code, VOIDmode, target, const0_rtx);
28789 : }
28790 :
28791 : /* Attempt to convert a CONST_VECTOR into a bcst_mem_operand.
28792 : Returns NULL_RTX if X is cannot be expressed as a suitable
28793 : VEC_DUPLICATE in mode MODE. */
28794 :
28795 : static rtx
28796 48 : ix86_gen_bcst_mem (machine_mode mode, rtx x)
28797 : {
28798 48 : if (!TARGET_AVX512F
28799 48 : || !CONST_VECTOR_P (x)
28800 64 : || (!TARGET_AVX512VL && GET_MODE_SIZE (mode) != 64)
28801 147 : || !VALID_BCST_MODE_P (GET_MODE_INNER (mode))
28802 : /* Disallow HFmode broadcast. */
28803 126 : || GET_MODE_SIZE (GET_MODE_INNER (mode)) < 4)
28804 : return NULL_RTX;
28805 :
28806 21 : rtx cst = CONST_VECTOR_ELT (x, 0);
28807 21 : if (!CONST_SCALAR_INT_P (cst)
28808 15 : && !CONST_DOUBLE_P (cst)
28809 0 : && !CONST_FIXED_P (cst))
28810 : return NULL_RTX;
28811 :
28812 21 : int n_elts = GET_MODE_NUNITS (mode);
28813 42 : if (CONST_VECTOR_NUNITS (x) != n_elts)
28814 : return NULL_RTX;
28815 :
28816 150 : for (int i = 1; i < n_elts; i++)
28817 129 : if (!rtx_equal_p (cst, CONST_VECTOR_ELT (x, i)))
28818 : return NULL_RTX;
28819 :
28820 42 : rtx mem = force_const_mem (GET_MODE_INNER (mode), cst);
28821 21 : return gen_rtx_VEC_DUPLICATE (mode, validize_mem (mem));
28822 : }
28823 :
28824 : /* Determine the ternlog immediate index that implements 3-operand
28825 : ternary logic expression OP. This uses and modifies the 3 element
28826 : array ARGS to record and check the leaves, either 3 REGs, or 2 REGs
28827 : and MEM. Returns an index between 0 and 255 for a valid ternlog,
28828 : or -1 if the expression isn't suitable. */
28829 :
28830 : int
28831 7514810 : ix86_ternlog_idx (rtx op, rtx *args)
28832 : {
28833 7514810 : int idx0, idx1;
28834 :
28835 7514810 : if (!op)
28836 : return -1;
28837 :
28838 7514810 : switch (GET_CODE (op))
28839 : {
28840 756341 : case SUBREG:
28841 756341 : if (!register_operand (op, GET_MODE (op)))
28842 : return -1;
28843 : /* FALLTHRU */
28844 :
28845 3707932 : case REG:
28846 3707932 : if (!args[0])
28847 : {
28848 1911123 : args[0] = op;
28849 1911123 : return 0xf0;
28850 : }
28851 1796809 : if (rtx_equal_p (op, args[0]))
28852 : return 0xf0;
28853 1770254 : if (!args[1])
28854 : {
28855 1497777 : args[1] = op;
28856 1497777 : return 0xcc;
28857 : }
28858 272477 : if (rtx_equal_p (op, args[1]))
28859 : return 0xcc;
28860 255899 : if (!args[2])
28861 : {
28862 232882 : args[2] = op;
28863 232882 : return 0xaa;
28864 : }
28865 23017 : if (rtx_equal_p (op, args[2]))
28866 1461 : return 0xaa;
28867 : return -1;
28868 :
28869 18611 : case VEC_DUPLICATE:
28870 18611 : if (!bcst_mem_operand (op, GET_MODE (op)))
28871 : return -1;
28872 302 : goto do_mem_operand;
28873 :
28874 351516 : case MEM:
28875 351516 : if (!memory_operand (op, GET_MODE (op)))
28876 : return -1;
28877 351353 : if (MEM_P (op)
28878 351353 : && MEM_VOLATILE_P (op)
28879 351447 : && !volatile_ok)
28880 : return -1;
28881 : /* FALLTHRU */
28882 :
28883 464230 : case CONST_VECTOR:
28884 464230 : do_mem_operand:
28885 464230 : if (!args[2])
28886 : {
28887 416844 : args[2] = op;
28888 416844 : return 0xaa;
28889 : }
28890 : /* Maximum of one volatile memory reference per expression. */
28891 47386 : if (side_effects_p (op))
28892 : return -1;
28893 47386 : if (rtx_equal_p (op, args[2]))
28894 : return 0xaa;
28895 : /* Check if CONST_VECTOR is the ones-complement of args[2]. */
28896 47335 : if (CONST_VECTOR_P (op)
28897 3479 : && CONST_VECTOR_P (args[2])
28898 47580 : && rtx_equal_p (simplify_const_unary_operation (NOT, GET_MODE (op),
28899 245 : op, GET_MODE (op)),
28900 : args[2]))
28901 : return 0x55;
28902 47148 : if (!args[0])
28903 : {
28904 45335 : args[0] = op;
28905 45335 : return 0xf0;
28906 : }
28907 1813 : if (rtx_equal_p (op, args[0]))
28908 : return 0xf0;
28909 : /* Check if CONST_VECTOR is the ones-complement of args[0]. */
28910 1813 : if (CONST_VECTOR_P (op)
28911 105 : && CONST_VECTOR_P (args[0])
28912 1855 : && rtx_equal_p (simplify_const_unary_operation (NOT, GET_MODE (op),
28913 42 : op, GET_MODE (op)),
28914 : args[0]))
28915 : return 0x0f;
28916 1771 : if (!args[1])
28917 : {
28918 1743 : args[1] = op;
28919 1743 : return 0xcc;
28920 : }
28921 28 : if (rtx_equal_p (op, args[1]))
28922 : return 0xcc;
28923 : /* Check if CONST_VECTOR is the ones-complement of args[1]. */
28924 28 : if (CONST_VECTOR_P (op)
28925 0 : && CONST_VECTOR_P (args[1])
28926 28 : && rtx_equal_p (simplify_const_unary_operation (NOT, GET_MODE (op),
28927 0 : op, GET_MODE (op)),
28928 : args[1]))
28929 0 : return 0x33;
28930 : return -1;
28931 :
28932 182901 : case NOT:
28933 182901 : idx0 = ix86_ternlog_idx (XEXP (op, 0), args);
28934 182901 : return (idx0 >= 0) ? idx0 ^ 0xff : -1;
28935 :
28936 1345650 : case AND:
28937 1345650 : idx0 = ix86_ternlog_idx (XEXP (op, 0), args);
28938 1345650 : if (idx0 < 0)
28939 : return -1;
28940 1104997 : idx1 = ix86_ternlog_idx (XEXP (op, 1), args);
28941 1104997 : return (idx1 >= 0) ? idx0 & idx1 : -1;
28942 :
28943 960521 : case IOR:
28944 960521 : idx0 = ix86_ternlog_idx (XEXP (op, 0), args);
28945 960521 : if (idx0 < 0)
28946 : return -1;
28947 712198 : idx1 = ix86_ternlog_idx (XEXP (op, 1), args);
28948 712198 : return (idx1 >= 0) ? idx0 | idx1 : -1;
28949 :
28950 439218 : case XOR:
28951 439218 : idx0 = ix86_ternlog_idx (XEXP (op, 0), args);
28952 439218 : if (idx0 < 0)
28953 : return -1;
28954 416732 : if (vector_all_ones_operand (XEXP (op, 1), GET_MODE (op)))
28955 6698 : return idx0 ^ 0xff;
28956 410034 : idx1 = ix86_ternlog_idx (XEXP (op, 1), args);
28957 410034 : return (idx1 >= 0) ? idx0 ^ idx1 : -1;
28958 :
28959 7465 : case UNSPEC:
28960 7465 : if (XINT (op, 1) != UNSPEC_VTERNLOG
28961 0 : || XVECLEN (op, 0) != 4
28962 0 : || !CONST_INT_P (XVECEXP (op, 0, 3)))
28963 : return -1;
28964 :
28965 : /* TODO: Handle permuted operands. */
28966 0 : if (ix86_ternlog_idx (XVECEXP (op, 0, 0), args) != 0xf0
28967 0 : || ix86_ternlog_idx (XVECEXP (op, 0, 1), args) != 0xcc
28968 0 : || ix86_ternlog_idx (XVECEXP (op, 0, 2), args) != 0xaa)
28969 : return -1;
28970 0 : return INTVAL (XVECEXP (op, 0, 3));
28971 :
28972 : default:
28973 : return -1;
28974 : }
28975 : }
28976 :
28977 : /* Return TRUE if OP (in mode MODE) is the leaf of a ternary logic
28978 : expression, such as a register or a memory reference. */
28979 :
28980 : bool
28981 3488686 : ix86_ternlog_leaf_p (rtx op, machine_mode mode)
28982 : {
28983 : /* We can't use memory_operand here, as it may return a different
28984 : value before and after reload (for volatile MEMs) which creates
28985 : problems splitting instructions. */
28986 3488686 : return register_operand (op, mode)
28987 730819 : || MEM_P (op)
28988 393733 : || CONST_VECTOR_P (op)
28989 3777168 : || bcst_mem_operand (op, mode);
28990 : }
28991 :
28992 : /* Test whether OP is a 3-operand ternary logic expression suitable
28993 : for use in a ternlog instruction. */
28994 :
28995 : bool
28996 2323306 : ix86_ternlog_operand_p (rtx op)
28997 : {
28998 2323306 : rtx op0, op1;
28999 2323306 : rtx args[3];
29000 :
29001 2323306 : args[0] = NULL_RTX;
29002 2323306 : args[1] = NULL_RTX;
29003 2323306 : args[2] = NULL_RTX;
29004 2323306 : int idx = ix86_ternlog_idx (op, args);
29005 2323306 : if (idx < 0)
29006 : return false;
29007 :
29008 : /* Don't match simple (binary or unary) expressions. */
29009 1887364 : machine_mode mode = GET_MODE (op);
29010 1887364 : switch (GET_CODE (op))
29011 : {
29012 870492 : case AND:
29013 870492 : op0 = XEXP (op, 0);
29014 870492 : op1 = XEXP (op, 1);
29015 :
29016 : /* Prefer pand. */
29017 870492 : if (ix86_ternlog_leaf_p (op0, mode)
29018 870492 : && ix86_ternlog_leaf_p (op1, mode))
29019 : return false;
29020 : /* Prefer pandn. */
29021 111298 : if (GET_CODE (op0) == NOT
29022 75801 : && register_operand (XEXP (op0, 0), mode)
29023 183855 : && ix86_ternlog_leaf_p (op1, mode))
29024 71120 : return false;
29025 : break;
29026 :
29027 624662 : case IOR:
29028 : /* Prefer por. */
29029 624662 : if (ix86_ternlog_leaf_p (XEXP (op, 0), mode)
29030 624662 : && ix86_ternlog_leaf_p (XEXP (op, 1), mode))
29031 469733 : return false;
29032 : break;
29033 :
29034 356322 : case XOR:
29035 356322 : op1 = XEXP (op, 1);
29036 : /* Prefer pxor, or one_cmpl<vmode>2. */
29037 356322 : if (ix86_ternlog_leaf_p (XEXP (op, 0), mode)
29038 356322 : && ix86_ternlog_leaf_p (XEXP (op, 1), mode))
29039 335724 : return false;
29040 : break;
29041 :
29042 : default:
29043 : break;
29044 : }
29045 : return true;
29046 : }
29047 :
29048 : /* Helper function for ix86_expand_ternlog. */
29049 : static rtx
29050 0 : ix86_expand_ternlog_binop (enum rtx_code code, machine_mode mode,
29051 : rtx op0, rtx op1, rtx target)
29052 : {
29053 0 : if (GET_MODE (op0) != mode)
29054 0 : op0 = gen_lowpart (mode, op0);
29055 0 : if (GET_MODE (op1) != mode)
29056 0 : op1 = gen_lowpart (mode, op1);
29057 :
29058 0 : if (CONST_VECTOR_P (op0))
29059 0 : op0 = validize_mem (force_const_mem (mode, op0));
29060 0 : if (CONST_VECTOR_P (op1))
29061 0 : op1 = validize_mem (force_const_mem (mode, op1));
29062 :
29063 0 : if (!register_operand (op0, mode))
29064 : {
29065 0 : if (!register_operand (op1, mode))
29066 : {
29067 : /* We can't use force_reg (op0, mode). */
29068 0 : rtx reg = gen_reg_rtx (mode);
29069 0 : emit_move_insn (reg, op0);
29070 0 : op0 = reg;
29071 : }
29072 : else
29073 : std::swap (op0, op1);
29074 : }
29075 0 : rtx ops[3] = { target, op0, op1 };
29076 0 : ix86_expand_vector_logical_operator (code, mode, ops);
29077 0 : return target;
29078 : }
29079 :
29080 :
29081 : /* Helper function for ix86_expand_ternlog. */
29082 : static rtx
29083 0 : ix86_expand_ternlog_andnot (machine_mode mode, rtx op0, rtx op1, rtx target)
29084 : {
29085 0 : if (GET_MODE (op0) != mode)
29086 0 : op0 = gen_lowpart (mode, op0);
29087 0 : op0 = gen_rtx_NOT (mode, op0);
29088 0 : if (GET_MODE (op1) != mode)
29089 0 : op1 = gen_lowpart (mode, op1);
29090 0 : if (CONST_VECTOR_P (op1))
29091 0 : op1 = validize_mem (force_const_mem (mode, op1));
29092 0 : emit_move_insn (target, gen_rtx_AND (mode, op0, op1));
29093 0 : return target;
29094 : }
29095 :
29096 : /* Expand a 3-operand ternary logic expression. Return TARGET. */
29097 : rtx
29098 2427 : ix86_expand_ternlog (machine_mode mode, rtx op0, rtx op1, rtx op2, int idx,
29099 : rtx target)
29100 : {
29101 2427 : rtx tmp0, tmp1, tmp2;
29102 :
29103 2427 : if (!target)
29104 3 : target = gen_reg_rtx (mode);
29105 :
29106 : /* Canonicalize ternlog index for degenerate (duplicated) operands. */
29107 2427 : if (rtx_equal_p (op0, op1) && rtx_equal_p (op0, op2))
29108 0 : switch (idx & 0x81)
29109 : {
29110 : case 0x00:
29111 : idx = 0x00;
29112 : break;
29113 : case 0x01:
29114 : idx = 0x0f;
29115 : break;
29116 : case 0x80:
29117 : idx = 0xf0;
29118 : break;
29119 : case 0x81:
29120 : idx = 0xff;
29121 : break;
29122 : }
29123 :
29124 2427 : switch (idx & 0xff)
29125 : {
29126 0 : case 0x00:
29127 0 : if ((!op0 || !side_effects_p (op0))
29128 0 : && (!op1 || !side_effects_p (op1))
29129 0 : && (!op2 || !side_effects_p (op2)))
29130 : {
29131 0 : emit_move_insn (target, CONST0_RTX (mode));
29132 0 : return target;
29133 : }
29134 : break;
29135 :
29136 0 : case 0x0a: /* ~a&c */
29137 0 : if ((!op1 || !side_effects_p (op1))
29138 0 : && op0 && register_operand (op0, mode)
29139 0 : && op2 && ix86_ternlog_leaf_p (op2, mode))
29140 0 : return ix86_expand_ternlog_andnot (mode, op0, op2, target);
29141 : break;
29142 :
29143 0 : case 0x0c: /* ~a&b */
29144 0 : if ((!op2 || !side_effects_p (op2))
29145 0 : && op0 && register_operand (op0, mode)
29146 0 : && op1 && ix86_ternlog_leaf_p (op1, mode))
29147 0 : return ix86_expand_ternlog_andnot (mode, op0, op1, target);
29148 : break;
29149 :
29150 82 : case 0x0f: /* ~a */
29151 0 : if ((!op1 || !side_effects_p (op1))
29152 82 : && (!op2 || !side_effects_p (op2))
29153 164 : && op0)
29154 : {
29155 82 : emit_move_insn (target, gen_rtx_XOR (mode, op0, CONSTM1_RTX (mode)));
29156 82 : return target;
29157 : }
29158 : break;
29159 :
29160 0 : case 0x22: /* ~b&c */
29161 0 : if ((!op0 || !side_effects_p (op0))
29162 0 : && op1 && register_operand (op1, mode)
29163 0 : && op2 && ix86_ternlog_leaf_p (op2, mode))
29164 0 : return ix86_expand_ternlog_andnot (mode, op1, op2, target);
29165 : break;
29166 :
29167 0 : case 0x30: /* ~b&a */
29168 0 : if ((!op2 || !side_effects_p (op2))
29169 0 : && op0 && ix86_ternlog_leaf_p (op0, mode)
29170 0 : && op1 && register_operand (op1, mode))
29171 0 : return ix86_expand_ternlog_andnot (mode, op1, op0, target);
29172 : break;
29173 :
29174 0 : case 0x33: /* ~b */
29175 0 : if ((!op0 || !side_effects_p (op0))
29176 0 : && (!op2 || !side_effects_p (op2))
29177 0 : && op1)
29178 : {
29179 0 : emit_move_insn (target, gen_rtx_XOR (mode, op1, CONSTM1_RTX (mode)));
29180 0 : return target;
29181 : }
29182 : break;
29183 :
29184 0 : case 0x3c: /* a^b */
29185 0 : if (op0 && ix86_ternlog_leaf_p (op0, mode)
29186 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
29187 0 : && (!op2 || !side_effects_p (op2)))
29188 0 : return ix86_expand_ternlog_binop (XOR, mode, op0, op1, target);
29189 : break;
29190 :
29191 0 : case 0x44: /* ~c&b */
29192 0 : if ((!op0 || !side_effects_p (op0))
29193 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
29194 0 : && op2 && register_operand (op2, mode))
29195 0 : return ix86_expand_ternlog_andnot (mode, op2, op1, target);
29196 : break;
29197 :
29198 2 : case 0x50: /* ~c&a */
29199 0 : if ((!op1 || !side_effects_p (op1))
29200 2 : && op0 && ix86_ternlog_leaf_p (op0, mode)
29201 4 : && op2 && register_operand (op2, mode))
29202 0 : return ix86_expand_ternlog_andnot (mode, op2, op0, target);
29203 : break;
29204 :
29205 4 : case 0x55: /* ~c */
29206 1 : if ((!op0 || !side_effects_p (op0))
29207 4 : && (!op1 || !side_effects_p (op1))
29208 8 : && op2)
29209 : {
29210 4 : emit_move_insn (target, gen_rtx_XOR (mode, op2, CONSTM1_RTX (mode)));
29211 4 : return target;
29212 : }
29213 : break;
29214 :
29215 0 : case 0x5a: /* a^c */
29216 0 : if (op0 && ix86_ternlog_leaf_p (op0, mode)
29217 0 : && op2 && ix86_ternlog_leaf_p (op2, mode)
29218 0 : && (!op1 || !side_effects_p (op1)))
29219 0 : return ix86_expand_ternlog_binop (XOR, mode, op0, op2, target);
29220 : break;
29221 :
29222 0 : case 0x66: /* b^c */
29223 0 : if ((!op0 || !side_effects_p (op0))
29224 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
29225 0 : && op2 && ix86_ternlog_leaf_p (op2, mode))
29226 0 : return ix86_expand_ternlog_binop (XOR, mode, op1, op2, target);
29227 : break;
29228 :
29229 0 : case 0x88: /* b&c */
29230 0 : if ((!op0 || !side_effects_p (op0))
29231 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
29232 0 : && op2 && ix86_ternlog_leaf_p (op2, mode))
29233 0 : return ix86_expand_ternlog_binop (AND, mode, op1, op2, target);
29234 : break;
29235 :
29236 0 : case 0xa0: /* a&c */
29237 0 : if ((!op1 || !side_effects_p (op1))
29238 0 : && op0 && ix86_ternlog_leaf_p (op0, mode)
29239 0 : && op2 && ix86_ternlog_leaf_p (op2, mode))
29240 0 : return ix86_expand_ternlog_binop (AND, mode, op0, op2, target);
29241 : break;
29242 :
29243 0 : case 0xaa: /* c */
29244 0 : if ((!op0 || !side_effects_p (op0))
29245 0 : && (!op1 || !side_effects_p (op1))
29246 0 : && op2)
29247 : {
29248 0 : if (GET_MODE (op2) != mode)
29249 0 : op2 = gen_lowpart (mode, op2);
29250 0 : emit_move_insn (target, op2);
29251 0 : return target;
29252 : }
29253 : break;
29254 :
29255 0 : case 0xc0: /* a&b */
29256 0 : if (op0 && ix86_ternlog_leaf_p (op0, mode)
29257 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
29258 0 : && (!op2 || !side_effects_p (op2)))
29259 0 : return ix86_expand_ternlog_binop (AND, mode, op0, op1, target);
29260 : break;
29261 :
29262 0 : case 0xcc: /* b */
29263 0 : if ((!op0 || !side_effects_p (op0))
29264 0 : && op1
29265 0 : && (!op2 || !side_effects_p (op2)))
29266 : {
29267 0 : if (GET_MODE (op1) != mode)
29268 0 : op1 = gen_lowpart (mode, op1);
29269 0 : emit_move_insn (target, op1);
29270 0 : return target;
29271 : }
29272 : break;
29273 :
29274 0 : case 0xee: /* b|c */
29275 0 : if ((!op0 || !side_effects_p (op0))
29276 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
29277 0 : && op2 && ix86_ternlog_leaf_p (op2, mode))
29278 0 : return ix86_expand_ternlog_binop (IOR, mode, op1, op2, target);
29279 : break;
29280 :
29281 6 : case 0xf0: /* a */
29282 6 : if (op0
29283 6 : && (!op1 || !side_effects_p (op1))
29284 12 : && (!op2 || !side_effects_p (op2)))
29285 : {
29286 6 : if (GET_MODE (op0) != mode)
29287 0 : op0 = gen_lowpart (mode, op0);
29288 6 : emit_move_insn (target, op0);
29289 6 : return target;
29290 : }
29291 : break;
29292 :
29293 0 : case 0xfa: /* a|c */
29294 0 : if (op0 && ix86_ternlog_leaf_p (op0, mode)
29295 0 : && op2 && ix86_ternlog_leaf_p (op2, mode)
29296 0 : && (!op1 || !side_effects_p (op1)))
29297 0 : return ix86_expand_ternlog_binop (IOR, mode, op0, op2, target);
29298 : break;
29299 :
29300 0 : case 0xfc: /* a|b */
29301 0 : if (op0 && ix86_ternlog_leaf_p (op0, mode)
29302 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
29303 0 : && (!op2 || !side_effects_p (op2)))
29304 0 : return ix86_expand_ternlog_binop (IOR, mode, op0, op1, target);
29305 : break;
29306 :
29307 0 : case 0xff:
29308 0 : if ((!op0 || !side_effects_p (op0))
29309 0 : && (!op1 || !side_effects_p (op1))
29310 0 : && (!op2 || !side_effects_p (op2)))
29311 : {
29312 0 : emit_move_insn (target, CONSTM1_RTX (mode));
29313 0 : return target;
29314 : }
29315 : break;
29316 : }
29317 :
29318 2335 : if (!register_operand (op0, mode))
29319 : {
29320 : /* We can't use force_reg (mode, op0). */
29321 12 : tmp0 = gen_reg_rtx (GET_MODE (op0));
29322 12 : emit_move_insn (tmp0,op0);
29323 : }
29324 : else
29325 : tmp0 = op0;
29326 2335 : if (GET_MODE (tmp0) != mode)
29327 0 : tmp0 = gen_lowpart (mode, tmp0);
29328 :
29329 2335 : if (!op1 || rtx_equal_p (op0, op1))
29330 6 : tmp1 = copy_rtx (tmp0);
29331 2329 : else if (!register_operand (op1, mode))
29332 : {
29333 : /* We can't use force_reg (mode, op1). */
29334 28 : tmp1 = gen_reg_rtx (GET_MODE (op1));
29335 28 : emit_move_insn (tmp1, op1);
29336 : }
29337 : else
29338 : tmp1 = op1;
29339 2335 : if (GET_MODE (tmp1) != mode)
29340 0 : tmp1 = gen_lowpart (mode, tmp1);
29341 :
29342 2335 : if (!op2 || rtx_equal_p (op0, op2))
29343 73 : tmp2 = copy_rtx (tmp0);
29344 2262 : else if (rtx_equal_p (op1, op2))
29345 0 : tmp2 = copy_rtx (tmp1);
29346 2262 : else if (CONST_VECTOR_P (op2))
29347 : {
29348 43 : if (GET_MODE (op2) != mode)
29349 0 : op2 = gen_lowpart (mode, op2);
29350 43 : tmp2 = ix86_gen_bcst_mem (mode, op2);
29351 43 : if (!tmp2)
29352 : {
29353 25 : machine_mode bcst32_mode = mode;
29354 25 : machine_mode bcst64_mode = mode;
29355 25 : switch (mode)
29356 : {
29357 1 : case V1TImode:
29358 1 : case V4SImode:
29359 1 : case V4SFmode:
29360 1 : case V8HImode:
29361 1 : case V16QImode:
29362 1 : bcst32_mode = V4SImode;
29363 1 : bcst64_mode = V2DImode;
29364 1 : break;
29365 :
29366 0 : case V2TImode:
29367 0 : case V8SImode:
29368 0 : case V8SFmode:
29369 0 : case V16HImode:
29370 0 : case V32QImode:
29371 0 : bcst32_mode = V8SImode;
29372 0 : bcst64_mode = V4DImode;
29373 0 : break;
29374 :
29375 3 : case V4TImode:
29376 3 : case V16SImode:
29377 3 : case V16SFmode:
29378 3 : case V32HImode:
29379 3 : case V64QImode:
29380 3 : bcst32_mode = V16SImode;
29381 3 : bcst64_mode = V8DImode;
29382 3 : break;
29383 :
29384 : default:
29385 : break;
29386 : }
29387 :
29388 25 : if (bcst32_mode != mode)
29389 : {
29390 4 : tmp2 = gen_lowpart (bcst32_mode, op2);
29391 4 : if (ix86_gen_bcst_mem (bcst32_mode, tmp2))
29392 : {
29393 3 : tmp2 = ix86_expand_ternlog (bcst32_mode,
29394 3 : gen_lowpart (bcst32_mode, tmp0),
29395 3 : gen_lowpart (bcst32_mode, tmp1),
29396 : tmp2, idx, NULL_RTX);
29397 3 : emit_move_insn (target, gen_lowpart (mode, tmp2));
29398 3 : return target;
29399 : }
29400 : }
29401 :
29402 22 : if (bcst64_mode != mode)
29403 : {
29404 1 : tmp2 = gen_lowpart (bcst64_mode, op2);
29405 1 : if (ix86_gen_bcst_mem (bcst64_mode, tmp2))
29406 : {
29407 0 : tmp2 = ix86_expand_ternlog (bcst64_mode,
29408 0 : gen_lowpart (bcst64_mode, tmp0),
29409 0 : gen_lowpart (bcst64_mode, tmp1),
29410 : tmp2, idx, NULL_RTX);
29411 0 : emit_move_insn (target, gen_lowpart (mode, tmp2));
29412 0 : return target;
29413 : }
29414 : }
29415 :
29416 22 : tmp2 = force_const_mem (mode, op2);
29417 22 : rtx bcast = ix86_broadcast_from_constant (mode, tmp2);
29418 22 : tmp2 = validize_mem (tmp2);
29419 22 : if (bcast)
29420 : {
29421 12 : rtx reg2 = gen_reg_rtx (mode);
29422 12 : bool ok = ix86_expand_vector_init_duplicate (false, mode,
29423 : reg2, bcast);
29424 12 : if (ok)
29425 2332 : tmp2 = reg2;
29426 : }
29427 : }
29428 : }
29429 : else
29430 : tmp2 = op2;
29431 2332 : if (GET_MODE (tmp2) != mode)
29432 0 : tmp2 = gen_lowpart (mode, tmp2);
29433 : /* Some memory_operands are not vector_memory_operands. */
29434 2332 : if (!bcst_vector_operand (tmp2, mode))
29435 0 : tmp2 = force_reg (mode, tmp2);
29436 :
29437 2332 : rtvec vec = gen_rtvec (4, tmp0, tmp1, tmp2, GEN_INT (idx));
29438 2332 : emit_move_insn (target, gen_rtx_UNSPEC (mode, vec, UNSPEC_VTERNLOG));
29439 2332 : return target;
29440 : }
29441 :
29442 : /* GF2P8AFFINEQB matrixes to implement shift and rotate. */
29443 :
29444 : static const uint64_t matrix_ashift[8] =
29445 : {
29446 : 0,
29447 : 0x0001020408102040, /* 1 l */
29448 : 0x0000010204081020, /* 2 l */
29449 : 0x0000000102040810, /* 3 l */
29450 : 0x0000000001020408, /* 4 l */
29451 : 0x0000000000010204, /* 5 l */
29452 : 0x0000000000000102, /* 6 l */
29453 : 0x0000000000000001 /* 7 l */
29454 : };
29455 :
29456 : static const uint64_t matrix_lshiftrt[8] =
29457 : {
29458 : 0,
29459 : 0x0204081020408000, /* 1 r */
29460 : 0x0408102040800000, /* 2 r */
29461 : 0x0810204080000000, /* 3 r */
29462 : 0x1020408000000000, /* 4 r */
29463 : 0x2040800000000000, /* 5 r */
29464 : 0x4080000000000000, /* 6 r */
29465 : 0x8000000000000000 /* 7 r */
29466 : };
29467 :
29468 : static const uint64_t matrix_ashiftrt[8] =
29469 : {
29470 : 0,
29471 : 0x0204081020408080, /* 1 r */
29472 : 0x0408102040808080, /* 2 r */
29473 : 0x0810204080808080, /* 3 r */
29474 : 0x1020408080808080, /* 4 r */
29475 : 0x2040808080808080, /* 5 r */
29476 : 0x4080808080808080, /* 6 r */
29477 : 0x8080808080808080 /* 7 r */
29478 : };
29479 :
29480 : static const uint64_t matrix_rotate[8] =
29481 : {
29482 : 0,
29483 : 0x8001020408102040, /* 1 rol8 */
29484 : 0x4080010204081020, /* 2 rol8 */
29485 : 0x2040800102040810, /* 3 rol8 */
29486 : 0x1020408001020408, /* 4 rol8 */
29487 : 0x0810204080010204, /* 5 rol8 */
29488 : 0x0408102040800102, /* 6 rol8 */
29489 : 0x0204081020408001 /* 7 rol8 */
29490 : };
29491 :
29492 : static const uint64_t matrix_rotatert[8] =
29493 : {
29494 : 0,
29495 : 0x0204081020408001, /* 1 ror8 */
29496 : 0x0408102040800102, /* 2 ror8 */
29497 : 0x0810204080010204, /* 3 ror8 */
29498 : 0x1020408001020408, /* 4 ror8 */
29499 : 0x2040800102040810, /* 5 ror8 */
29500 : 0x4080010204081020, /* 6 ror8 */
29501 : 0x8001020408102040 /* 7 ror8 */
29502 : };
29503 :
29504 : /* Return rtx to load a 64bit GF2P8AFFINE GP(2) matrix implementing a shift
29505 : for CODE and shift count COUNT into register with vector of size of SRC. */
29506 :
29507 : rtx
29508 202 : ix86_vgf2p8affine_shift_matrix (rtx src, rtx count, enum rtx_code code)
29509 : {
29510 202 : machine_mode mode = GET_MODE (src);
29511 202 : const uint64_t *matrix;
29512 202 : unsigned shift = INTVAL (count) & 7;
29513 202 : gcc_assert (shift > 0 && shift < 8);
29514 :
29515 202 : switch (code)
29516 : {
29517 : case ASHIFT:
29518 : matrix = matrix_ashift;
29519 : break;
29520 27 : case ASHIFTRT:
29521 27 : matrix = matrix_ashiftrt;
29522 27 : break;
29523 30 : case LSHIFTRT:
29524 30 : matrix = matrix_lshiftrt;
29525 30 : break;
29526 34 : case ROTATE:
29527 34 : matrix = matrix_rotate;
29528 34 : break;
29529 35 : case ROTATERT:
29530 35 : matrix = matrix_rotatert;
29531 35 : break;
29532 0 : default:
29533 0 : gcc_unreachable ();
29534 : }
29535 :
29536 202 : int nelts = GET_MODE_NUNITS (mode);
29537 202 : rtvec vec = rtvec_alloc (nelts);
29538 202 : uint64_t ma = matrix[shift];
29539 6922 : for (int i = 0; i < nelts; i++)
29540 6720 : RTVEC_ELT (vec, i) = gen_int_mode ((ma >> ((i % 8) * 8)) & 0xff, QImode);
29541 :
29542 202 : return force_reg (mode, gen_rtx_CONST_VECTOR (mode, vec));
29543 : }
29544 :
29545 : /* Trunc a vector to a narrow vector, like v4di -> v4si. */
29546 :
29547 : void
29548 61 : ix86_expand_trunc_with_avx2_noavx512f (rtx output, rtx input, machine_mode cvt_mode)
29549 : {
29550 61 : machine_mode out_mode = GET_MODE (output);
29551 61 : machine_mode in_mode = GET_MODE (input);
29552 61 : int len = GET_MODE_SIZE (in_mode);
29553 244 : gcc_assert (len == GET_MODE_SIZE (cvt_mode)
29554 : && GET_MODE_INNER (out_mode) == GET_MODE_INNER (cvt_mode)
29555 : && (REG_P (input) || SUBREG_P (input)));
29556 61 : scalar_mode inner_out_mode = GET_MODE_INNER (out_mode);
29557 122 : int in_innersize = GET_MODE_SIZE (GET_MODE_INNER (in_mode));
29558 61 : int out_innersize = GET_MODE_SIZE (inner_out_mode);
29559 :
29560 61 : struct expand_vec_perm_d d;
29561 61 : d.target = gen_reg_rtx (cvt_mode);
29562 61 : d.op0 = lowpart_subreg (cvt_mode, force_reg(in_mode, input), in_mode);
29563 61 : d.op1 = d.op0;
29564 61 : d.vmode = cvt_mode;
29565 61 : d.nelt = GET_MODE_NUNITS (cvt_mode);
29566 61 : d.testing_p = false;
29567 61 : d.one_operand_p = true;
29568 :
29569 : /* Init perm. Put the needed bits of input in order and
29570 : fill the rest of bits by default. */
29571 661 : for (int i = 0; i < d.nelt; ++i)
29572 : {
29573 600 : d.perm[i] = i;
29574 1200 : if (i < GET_MODE_NUNITS (out_mode))
29575 238 : d.perm[i] = i * (in_innersize / out_innersize);
29576 : }
29577 :
29578 61 : bool ok = ix86_expand_vec_perm_const_1(&d);
29579 61 : gcc_assert (ok);
29580 61 : emit_move_insn (output, gen_lowpart (out_mode, d.target));
29581 61 : }
29582 :
29583 : /* Implement truncv8sfv8bf2 with vector permutation. */
29584 : void
29585 8 : ix86_expand_vector_sf2bf_with_vec_perm (rtx dest, rtx src)
29586 : {
29587 8 : machine_mode vperm_mode, src_mode = GET_MODE (src);
29588 8 : switch (src_mode)
29589 : {
29590 : case V16SFmode:
29591 : vperm_mode = V32BFmode;
29592 : break;
29593 2 : case V8SFmode:
29594 2 : vperm_mode = V16BFmode;
29595 2 : break;
29596 4 : case V4SFmode:
29597 4 : vperm_mode = V8BFmode;
29598 4 : break;
29599 0 : default:
29600 0 : gcc_unreachable ();
29601 : }
29602 :
29603 8 : int nelt = GET_MODE_NUNITS (vperm_mode);
29604 8 : vec_perm_builder sel (nelt, nelt, 1);
29605 8 : sel.quick_grow (nelt);
29606 144 : for (int i = 0; i != nelt; i++)
29607 128 : sel[i] = (2 * i + 1) % nelt;
29608 16 : vec_perm_indices indices (sel, 1, nelt);
29609 :
29610 8 : rtx target = gen_reg_rtx (vperm_mode);
29611 8 : rtx op0 = lowpart_subreg (vperm_mode,
29612 : force_reg (src_mode, src),
29613 : src_mode);
29614 8 : bool ok = targetm.vectorize.vec_perm_const (vperm_mode, vperm_mode,
29615 : target, op0, op0, indices);
29616 8 : gcc_assert (ok);
29617 8 : emit_move_insn (dest, lowpart_subreg (GET_MODE (dest), target, vperm_mode));
29618 8 : }
29619 :
29620 : /* Implement extendv8bf2v8sf2 with vector permutation. */
29621 : void
29622 8 : ix86_expand_vector_bf2sf_with_vec_perm (rtx dest, rtx src)
29623 : {
29624 8 : machine_mode vperm_mode, src_mode = GET_MODE (src);
29625 8 : switch (src_mode)
29626 : {
29627 : case V16BFmode:
29628 : vperm_mode = V32BFmode;
29629 : break;
29630 2 : case V8BFmode:
29631 2 : vperm_mode = V16BFmode;
29632 2 : break;
29633 4 : case V4BFmode:
29634 4 : vperm_mode = V8BFmode;
29635 4 : break;
29636 0 : default:
29637 0 : gcc_unreachable ();
29638 : }
29639 :
29640 8 : int nelt = GET_MODE_NUNITS (vperm_mode);
29641 8 : vec_perm_builder sel (nelt, nelt, 1);
29642 8 : sel.quick_grow (nelt);
29643 144 : for (int i = 0, k = 0, j = nelt; i != nelt; i++)
29644 128 : sel[i] = i & 1 ? j++ : k++;
29645 :
29646 16 : vec_perm_indices indices (sel, 2, nelt);
29647 :
29648 8 : rtx target = gen_reg_rtx (vperm_mode);
29649 8 : rtx op1 = lowpart_subreg (vperm_mode,
29650 : force_reg (src_mode, src),
29651 : src_mode);
29652 8 : rtx op0 = CONST0_RTX (vperm_mode);
29653 8 : bool ok = targetm.vectorize.vec_perm_const (vperm_mode, vperm_mode,
29654 : target, op0, op1, indices);
29655 8 : gcc_assert (ok);
29656 8 : emit_move_insn (dest, lowpart_subreg (GET_MODE (dest), target, vperm_mode));
29657 8 : }
29658 :
29659 : /* Implement bitreverse<mode>2 using gf2p8affineqb. */
29660 :
29661 : void
29662 5 : ix86_expand_gfni_bitreverse (rtx dest, rtx src)
29663 : {
29664 5 : machine_mode mode = GET_MODE (dest);
29665 5 : rtx temp;
29666 10 : if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
29667 : {
29668 1 : rtx temp1 = gen_reg_rtx (mode == TImode ? V2DImode : V4SImode);
29669 1 : rtx temp2 = gen_reg_rtx (mode == TImode ? V2DImode : V4SImode);
29670 1 : if (mode == TImode)
29671 : {
29672 1 : temp = lowpart_subreg (DImode, src, TImode);
29673 1 : emit_insn (gen_rtx_SET (temp1, gen_rtx_VEC_CONCAT (V2DImode, temp,
29674 : const0_rtx)));
29675 1 : temp = gen_highpart (DImode, src);
29676 1 : emit_insn (gen_rtx_SET (temp2, gen_rtx_VEC_CONCAT (V2DImode, temp,
29677 : const0_rtx)));
29678 : }
29679 : else
29680 : {
29681 0 : temp = lowpart_subreg (SImode, src, DImode);
29682 0 : emit_insn (gen_vec_setv4si_0 (temp1, CONST0_RTX (V4SImode), temp));
29683 0 : temp = gen_highpart (SImode, src);
29684 0 : emit_insn (gen_vec_setv4si_0 (temp2, CONST0_RTX (V4SImode), temp));
29685 0 : temp1 = lowpart_subreg (V2DImode, temp1, V4SImode);
29686 0 : temp2 = lowpart_subreg (V2DImode, temp2, V4SImode);
29687 : }
29688 1 : temp = gen_reg_rtx (V2DImode);
29689 1 : emit_insn (gen_vec_interleave_lowv2di (temp, temp1, temp2));
29690 : }
29691 4 : else if (mode != DImode)
29692 : {
29693 3 : if (mode != SImode)
29694 : {
29695 2 : src = force_reg (mode, src);
29696 2 : src = lowpart_subreg (SImode, src, mode);
29697 : }
29698 3 : temp = gen_reg_rtx (V4SImode);
29699 3 : emit_insn (gen_vec_setv4si_0 (temp, CONST0_RTX (V4SImode), src));
29700 : }
29701 : else
29702 : {
29703 1 : temp = gen_reg_rtx (V2DImode);
29704 1 : emit_insn (gen_rtx_SET (temp, gen_rtx_VEC_CONCAT (V2DImode, src,
29705 : const0_rtx)));
29706 : }
29707 5 : src = temp;
29708 5 : temp = gen_reg_rtx (V16QImode);
29709 5 : rtx src2 = gen_rtx_CONST_VECTOR (V16QImode,
29710 : gen_rtvec (16, GEN_INT (1), GEN_INT (2),
29711 : GEN_INT (4), GEN_INT (8),
29712 : GEN_INT (16), GEN_INT (32),
29713 : GEN_INT (64), GEN_INT (-128),
29714 : GEN_INT (1), GEN_INT (2),
29715 : GEN_INT (4), GEN_INT (8),
29716 : GEN_INT (16), GEN_INT (32),
29717 : GEN_INT (64), GEN_INT (-128)));
29718 5 : src2 = validize_mem (force_const_mem (V16QImode, src2));
29719 5 : src = lowpart_subreg (V16QImode, src, GET_MODE (src));
29720 5 : emit_insn (gen_vgf2p8affineqb_v16qi (temp, src, src2, const0_rtx));
29721 5 : if (mode == QImode)
29722 : {
29723 1 : rtx temp1 = gen_reg_rtx (SImode);
29724 1 : rtx temp2 = lowpart_subreg (V4SImode, temp, V16QImode);
29725 1 : rtx temp3 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
29726 1 : emit_insn (gen_rtx_SET (temp1,
29727 : gen_rtx_VEC_SELECT (SImode, temp2, temp3)));
29728 1 : emit_move_insn (dest, lowpart_subreg (QImode, temp1, SImode));
29729 1 : return;
29730 : }
29731 11 : rtx target = gen_reg_rtx ((GET_MODE_SIZE (mode) < 4 || !TARGET_64BIT)
29732 3 : ? SImode : mode == TImode ? DImode : mode);
29733 4 : emit_move_insn (target, lowpart_subreg (GET_MODE (target), temp, V16QImode));
29734 8 : if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
29735 : {
29736 1 : rtx temp1 = gen_reg_rtx (GET_MODE (target));
29737 1 : if (mode == TImode || TARGET_SSE4_1)
29738 : {
29739 1 : rtx temp2 = lowpart_subreg (mode == TImode ? V2DImode : V4SImode,
29740 : temp, V16QImode);
29741 1 : rtx temp3 = gen_rtx_PARALLEL (VOIDmode,
29742 : gen_rtvec (1, GEN_INT (mode == TImode
29743 : ? 1 : 2)));
29744 1 : emit_insn (gen_rtx_SET (temp1,
29745 : gen_rtx_VEC_SELECT (GET_MODE (target), temp2,
29746 : temp3)));
29747 1 : }
29748 : else
29749 : {
29750 0 : rtx temp2 = gen_reg_rtx (V4SImode);
29751 0 : rtx temp3 = lowpart_subreg (V4SImode, temp, V16QImode);
29752 0 : emit_insn (gen_sse2_pshufd (temp2, temp3, GEN_INT (0xaa)));
29753 0 : emit_move_insn (temp1, lowpart_subreg (GET_MODE (target), temp2,
29754 : V4SImode));
29755 : }
29756 1 : rtx temp4 = gen_reg_rtx (GET_MODE (target));
29757 1 : rtx temp5 = gen_reg_rtx (GET_MODE (target));
29758 0 : rtx (*gen_bswap) (rtx, rtx)
29759 1 : = mode == TImode ? gen_bswapdi2 : gen_bswapsi2;
29760 1 : emit_insn (gen_bswap (temp4, target));
29761 1 : emit_insn (gen_bswap (temp5, temp1));
29762 1 : temp4 = gen_rtx_ZERO_EXTEND (mode, temp4);
29763 1 : temp5 = gen_rtx_ZERO_EXTEND (mode, temp5);
29764 1 : rtx shift = GEN_INT (GET_MODE_PRECISION (GET_MODE (target)));
29765 1 : temp4 = gen_rtx_ASHIFT (mode, temp4, shift);
29766 1 : emit_insn (gen_rtx_SET (dest, gen_rtx_IOR (mode, temp4, temp5)));
29767 1 : return;
29768 : }
29769 3 : if (mode == HImode)
29770 : {
29771 1 : target = lowpart_subreg (mode, target, SImode);
29772 1 : emit_insn (gen_bswaphi2 (dest, target));
29773 : }
29774 2 : else if (mode == SImode)
29775 1 : emit_insn (gen_bswapsi2 (dest, target));
29776 : else
29777 1 : emit_insn (gen_rtx_SET (dest, gen_rtx_BSWAP (mode, target)));
29778 : }
29779 :
29780 : /* Expand LCP stall or long immediate peephole for INSN. Use the
29781 : previous scratch register if possible. If USE_XOR is true,
29782 : generate "*movsi_xor". */
29783 :
29784 : void
29785 31845 : ix86_expand_lcp_stall_peephole (rtx_insn *insn, rtx *operands,
29786 : bool use_xor)
29787 : {
29788 31845 : rtx imm, scratch;
29789 31845 : machine_mode mode = GET_MODE (operands[0]);
29790 :
29791 : /* Get the immediate operand and the allocated scratch register. */
29792 31845 : if (use_xor)
29793 : {
29794 10573 : imm = const0_rtx;
29795 10573 : scratch = operands[1];
29796 : }
29797 : else
29798 : {
29799 21272 : imm = operands[1];
29800 21272 : scratch = operands[2];
29801 : }
29802 :
29803 31845 : df_ref def;
29804 31845 : rtx_insn *prev, *prev_insn = nullptr;
29805 31845 : rtx set, prev_scratch = nullptr;
29806 :
29807 : /* Scan backward for the previous scratch register def with IMM in
29808 : the same basic block. */
29809 31845 : basic_block bb = BLOCK_FOR_INSN (insn);
29810 967306 : for (prev = PREV_INSN (insn);
29811 967306 : prev != BB_HEAD (bb);
29812 935461 : prev = PREV_INSN (prev))
29813 : {
29814 940282 : if (NONDEBUG_INSN_P (prev))
29815 2074467 : FOR_EACH_INSN_DEF (def, prev)
29816 1746700 : if (!DF_REF_IS_ARTIFICIAL (def)
29817 1746700 : && !DF_REF_FLAGS_IS_SET (def, DF_REF_MAY_CLOBBER)
29818 175339 : && !DF_REF_FLAGS_IS_SET (def, DF_REF_MUST_CLOBBER))
29819 : {
29820 154727 : rtx reg = DF_REF_REG (def);
29821 154727 : if (HARD_REGISTER_P (reg) && REGNO (reg) != FLAGS_REG)
29822 : {
29823 153559 : set = single_set (prev);
29824 153559 : if (!set)
29825 478 : continue;
29826 :
29827 153081 : rtx dest = SET_DEST (set);
29828 :
29829 : /* Reject DEST if a register is not wide enough to
29830 : supply MODE or invalid for QImode. */
29831 133664 : if (!GENERAL_REG_P (dest)
29832 230480 : || (GET_MODE_SIZE (GET_MODE (dest))
29833 115240 : < GET_MODE_SIZE (mode))
29834 267178 : || (mode == QImode
29835 0 : && !ANY_QI_REGNO_P (REGNO (dest))))
29836 38984 : continue;
29837 :
29838 114097 : rtx src = SET_SRC (set);
29839 114097 : if (rtx_equal_p (src, imm))
29840 : {
29841 : /* A previous scratch register is found. */
29842 : prev_scratch = dest;
29843 : prev_insn = prev;
29844 : break;
29845 : }
29846 : }
29847 : }
29848 :
29849 940282 : if (prev_scratch)
29850 : break;
29851 : }
29852 :
29853 31845 : if (prev_scratch)
29854 : {
29855 : /* The previous scratch register is unusable if it is set between
29856 : PREV_INSN and INSN. */
29857 4821 : unsigned int regno = REGNO (prev_scratch);
29858 :
29859 : /* Scan backward for the previous scratch register def. */
29860 4821 : for (prev = PREV_INSN (insn);
29861 130685 : prev != prev_insn;
29862 125864 : prev = PREV_INSN (prev))
29863 : {
29864 127330 : if (NONDEBUG_INSN_P (prev))
29865 265542 : FOR_EACH_INSN_DEF (def, prev)
29866 187570 : if (HARD_REGISTER_P (DF_REF_REAL_REG (def))
29867 187570 : && DF_REF_REGNO (def) == regno)
29868 : {
29869 : /* Since the previous scratch register is set, it is
29870 : unusable. */
29871 1466 : if (dump_file)
29872 : {
29873 0 : fprintf (dump_file,
29874 : "\nThe previous scratch register:\n\n");
29875 0 : print_rtl_single (dump_file, prev_scratch);
29876 0 : fprintf (dump_file, "\nset in:\n\n");
29877 0 : print_rtl_single (dump_file, prev_insn);
29878 0 : fprintf (dump_file,
29879 : "\nis unusable by:\n\n");
29880 0 : print_rtl_single (dump_file, insn);
29881 0 : fprintf (dump_file,
29882 : "\nsince it is overridden by:\n\n");
29883 0 : print_rtl_single (dump_file, prev);
29884 0 : fprintf (dump_file, "\n");
29885 : }
29886 : prev_scratch = nullptr;
29887 : break;
29888 : }
29889 :
29890 125864 : if (!prev_scratch)
29891 : break;
29892 : }
29893 :
29894 4821 : if (prev_scratch)
29895 : {
29896 : /* Ignore the allocated scratch register and use the previous
29897 : scratch register. */
29898 3355 : if (dump_file)
29899 : {
29900 0 : fprintf (dump_file,
29901 : "\nIgnore the allocated scratch register:\n\n");
29902 0 : print_rtl_single (dump_file, scratch);
29903 0 : fprintf (dump_file,
29904 : "\nand use the previous scratch register:\n\n");
29905 0 : print_rtl_single (dump_file, prev_scratch);
29906 0 : fprintf (dump_file, "\nset in:\n\n");
29907 0 : print_rtl_single (dump_file, prev_insn);
29908 0 : fprintf (dump_file, "\nfor:\n\n");
29909 0 : print_rtl_single (dump_file, insn);
29910 0 : fprintf (dump_file, "\n");
29911 : }
29912 3355 : scratch = gen_lowpart (mode, prev_scratch);
29913 3355 : set = gen_rtx_SET (operands[0], scratch);
29914 3355 : emit_insn (set);
29915 3355 : return;
29916 : }
29917 : }
29918 :
29919 : /* If there is no usable previous scratch register, use the allocated
29920 : scratch register. */
29921 28490 : if (use_xor)
29922 : {
29923 : /* Generate "*movsi_xor". */
29924 9052 : rtx x = gen_lowpart (SImode, scratch);
29925 9052 : rtvec p = rtvec_alloc (2);
29926 9052 : set = gen_rtx_SET (x, const0_rtx);
29927 9052 : RTVEC_ELT (p, 0) = set;
29928 9052 : rtx clobber = gen_rtx_REG (CCmode, FLAGS_REG);
29929 9052 : RTVEC_ELT (p, 1) = gen_rtx_CLOBBER (VOIDmode, clobber);
29930 9052 : set = gen_rtx_PARALLEL (VOIDmode, p);
29931 : }
29932 : else
29933 19438 : set = gen_rtx_SET (scratch, imm);
29934 28490 : emit_insn (set);
29935 28490 : set = gen_rtx_SET (operands[0], scratch);
29936 28490 : emit_insn (set);
29937 : }
29938 :
29939 : #include "gt-i386-expand.h"
|
