Line data Source code
1 : /* Copyright (C) 1988-2026 Free Software Foundation, Inc.
2 :
3 : This file is part of GCC.
4 :
5 : GCC is free software; you can redistribute it and/or modify
6 : it under the terms of the GNU General Public License as published by
7 : the Free Software Foundation; either version 3, or (at your option)
8 : any later version.
9 :
10 : GCC is distributed in the hope that it will be useful,
11 : but WITHOUT ANY WARRANTY; without even the implied warranty of
12 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 : GNU General Public License for more details.
14 :
15 : You should have received a copy of the GNU General Public License
16 : along with GCC; see the file COPYING3. If not see
17 : <http://www.gnu.org/licenses/>. */
18 :
19 : #define IN_TARGET_CODE 1
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "memmodel.h"
28 : #include "gimple.h"
29 : #include "cfghooks.h"
30 : #include "cfgloop.h"
31 : #include "df.h"
32 : #include "tm_p.h"
33 : #include "stringpool.h"
34 : #include "expmed.h"
35 : #include "optabs.h"
36 : #include "regs.h"
37 : #include "emit-rtl.h"
38 : #include "recog.h"
39 : #include "cgraph.h"
40 : #include "diagnostic.h"
41 : #include "cfgbuild.h"
42 : #include "alias.h"
43 : #include "fold-const.h"
44 : #include "attribs.h"
45 : #include "calls.h"
46 : #include "stor-layout.h"
47 : #include "varasm.h"
48 : #include "output.h"
49 : #include "insn-attr.h"
50 : #include "flags.h"
51 : #include "except.h"
52 : #include "explow.h"
53 : #include "expr.h"
54 : #include "cfgrtl.h"
55 : #include "common/common-target.h"
56 : #include "langhooks.h"
57 : #include "reload.h"
58 : #include "gimplify.h"
59 : #include "dwarf2.h"
60 : #include "tm-constrs.h"
61 : #include "cselib.h"
62 : #include "sched-int.h"
63 : #include "opts.h"
64 : #include "tree-pass.h"
65 : #include "context.h"
66 : #include "pass_manager.h"
67 : #include "target-globals.h"
68 : #include "gimple-iterator.h"
69 : #include "shrink-wrap.h"
70 : #include "builtins.h"
71 : #include "rtl-iter.h"
72 : #include "tree-iterator.h"
73 : #include "dbgcnt.h"
74 : #include "case-cfn-macros.h"
75 : #include "dojump.h"
76 : #include "fold-const-call.h"
77 : #include "tree-vrp.h"
78 : #include "tree-ssanames.h"
79 : #include "selftest.h"
80 : #include "selftest-rtl.h"
81 : #include "print-rtl.h"
82 : #include "intl.h"
83 : #include "ifcvt.h"
84 : #include "symbol-summary.h"
85 : #include "sreal.h"
86 : #include "ipa-cp.h"
87 : #include "ipa-prop.h"
88 : #include "ipa-fnsummary.h"
89 : #include "wide-int-bitmask.h"
90 : #include "tree-vector-builder.h"
91 : #include "debug.h"
92 : #include "dwarf2out.h"
93 : #include "i386-options.h"
94 : #include "i386-builtins.h"
95 : #include "i386-expand.h"
96 : #include "asan.h"
97 : #include "function-abi.h"
98 :
99 : /* Split one or more double-mode RTL references into pairs of half-mode
100 : references. The RTL can be REG, offsettable MEM, integer constant, or
101 : CONST_DOUBLE. "operands" is a pointer to an array of double-mode RTLs to
102 : split and "num" is its length. lo_half and hi_half are output arrays
103 : that parallel "operands". */
104 :
105 : void
106 4306083 : split_double_mode (machine_mode mode, rtx operands[],
107 : int num, rtx lo_half[], rtx hi_half[])
108 : {
109 4306083 : machine_mode half_mode;
110 4306083 : unsigned int byte;
111 4306083 : rtx mem_op = NULL_RTX;
112 4306083 : int mem_num = 0;
113 :
114 4306083 : switch (mode)
115 : {
116 : case E_TImode:
117 : half_mode = DImode;
118 : break;
119 660318 : case E_DImode:
120 660318 : half_mode = SImode;
121 660318 : break;
122 6 : case E_P2HImode:
123 6 : half_mode = HImode;
124 6 : break;
125 30 : case E_P2QImode:
126 30 : half_mode = QImode;
127 30 : break;
128 0 : default:
129 0 : gcc_unreachable ();
130 : }
131 :
132 4306083 : byte = GET_MODE_SIZE (half_mode);
133 :
134 8858203 : while (num--)
135 : {
136 4552120 : rtx op = operands[num];
137 :
138 : /* simplify_subreg refuse to split volatile memory addresses,
139 : but we still have to handle it. */
140 4552120 : if (MEM_P (op))
141 : {
142 1775944 : if (mem_op && rtx_equal_p (op, mem_op))
143 : {
144 2307 : lo_half[num] = lo_half[mem_num];
145 2307 : hi_half[num] = hi_half[mem_num];
146 : }
147 : else
148 : {
149 1773637 : mem_op = op;
150 1773637 : mem_num = num;
151 1773637 : lo_half[num] = adjust_address (op, half_mode, 0);
152 1773637 : hi_half[num] = adjust_address (op, half_mode, byte);
153 : }
154 : }
155 : else
156 : {
157 2776176 : lo_half[num] = simplify_gen_subreg (half_mode, op,
158 2776176 : GET_MODE (op) == VOIDmode
159 : ? mode : GET_MODE (op), 0);
160 :
161 2776176 : rtx tmp = simplify_gen_subreg (half_mode, op,
162 2776176 : GET_MODE (op) == VOIDmode
163 2776176 : ? mode : GET_MODE (op), byte);
164 : /* simplify_gen_subreg will return NULL RTX for the
165 : high half of the paradoxical subreg. */
166 2776176 : if (tmp)
167 2776175 : hi_half[num] = tmp;
168 1 : else if (can_create_pseudo_p ())
169 0 : hi_half[num] = gen_reg_rtx (half_mode);
170 : else
171 1 : hi_half[num] = CONST0_RTX (half_mode);
172 : }
173 : }
174 4306083 : }
175 :
176 : /* Emit the double word assignment DST = { LO, HI }. */
177 :
178 : void
179 106091 : split_double_concat (machine_mode mode, rtx dst, rtx lo, rtx hi)
180 : {
181 106091 : rtx dlo, dhi;
182 106091 : int deleted_move_count = 0;
183 106091 : split_double_mode (mode, &dst, 1, &dlo, &dhi);
184 : /* Constraints ensure that if both lo and hi are MEMs, then
185 : dst has early-clobber and thus addresses of MEMs don't use
186 : dlo/dhi registers. Otherwise if at least one of li and hi are MEMs,
187 : dlo/dhi are registers. */
188 106091 : if (MEM_P (lo)
189 5707 : && rtx_equal_p (dlo, hi)
190 107066 : && reg_overlap_mentioned_p (dhi, lo))
191 : {
192 : /* If dlo is same as hi and lo's address uses dhi register,
193 : code below would first emit_move_insn (dhi, hi)
194 : and then emit_move_insn (dlo, lo). But the former
195 : would invalidate lo's address. Load into dhi first,
196 : then swap. */
197 187 : emit_move_insn (dhi, lo);
198 187 : lo = dhi;
199 : }
200 105904 : else if (MEM_P (hi)
201 9746 : && !MEM_P (lo)
202 6904 : && !rtx_equal_p (dlo, lo)
203 107462 : && reg_overlap_mentioned_p (dlo, hi))
204 : {
205 : /* In this case, code below would first emit_move_insn (dlo, lo)
206 : and then emit_move_insn (dhi, hi). But the former would
207 : invalidate hi's address. */
208 14 : if (rtx_equal_p (dhi, lo))
209 : {
210 : /* We can't load into dhi first, so load into dlo
211 : first and we'll swap. */
212 5 : emit_move_insn (dlo, hi);
213 5 : hi = dlo;
214 : }
215 : else
216 : {
217 : /* Load into dhi first. */
218 9 : emit_move_insn (dhi, hi);
219 9 : hi = dhi;
220 : }
221 : }
222 106091 : if (!rtx_equal_p (dlo, hi))
223 : {
224 90552 : if (!rtx_equal_p (dlo, lo))
225 37973 : emit_move_insn (dlo, lo);
226 : else
227 : deleted_move_count++;
228 90552 : if (!rtx_equal_p (dhi, hi))
229 83665 : emit_move_insn (dhi, hi);
230 : else
231 6887 : deleted_move_count++;
232 : }
233 15539 : else if (!rtx_equal_p (lo, dhi))
234 : {
235 8175 : if (!rtx_equal_p (dhi, hi))
236 8175 : emit_move_insn (dhi, hi);
237 : else
238 : deleted_move_count++;
239 8175 : if (!rtx_equal_p (dlo, lo))
240 8069 : emit_move_insn (dlo, lo);
241 : else
242 106 : deleted_move_count++;
243 : }
244 7364 : else if (mode == TImode)
245 7343 : emit_insn (gen_swapdi (dlo, dhi));
246 : else
247 21 : emit_insn (gen_swapsi (dlo, dhi));
248 :
249 106091 : if (deleted_move_count == 2)
250 3170 : emit_note (NOTE_INSN_DELETED);
251 106091 : }
252 :
253 :
254 : /* Generate either "mov $0, reg" or "xor reg, reg", as appropriate
255 : for the target. */
256 :
257 : void
258 129534 : ix86_expand_clear (rtx dest)
259 : {
260 129534 : rtx tmp;
261 :
262 : /* We play register width games, which are only valid after reload. */
263 129534 : gcc_assert (reload_completed);
264 :
265 : /* Avoid HImode and its attendant prefix byte. */
266 259068 : if (GET_MODE_SIZE (GET_MODE (dest)) < 4)
267 1028 : dest = gen_rtx_REG (SImode, REGNO (dest));
268 129534 : tmp = gen_rtx_SET (dest, const0_rtx);
269 :
270 129534 : if (!TARGET_USE_MOV0 || optimize_insn_for_size_p ())
271 : {
272 129534 : rtx clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
273 129534 : tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, clob));
274 : }
275 :
276 129534 : emit_insn (tmp);
277 129534 : }
278 :
279 : /* Return true if V can be broadcasted from an integer of WIDTH bits
280 : which is returned in VAL_BROADCAST. Otherwise, return false. */
281 :
282 : static bool
283 4875 : ix86_broadcast (HOST_WIDE_INT v, unsigned int width,
284 : HOST_WIDE_INT &val_broadcast)
285 : {
286 4875 : wide_int val = wi::uhwi (v, HOST_BITS_PER_WIDE_INT);
287 4875 : val_broadcast = wi::extract_uhwi (val, 0, width);
288 6578 : for (unsigned int i = width; i < HOST_BITS_PER_WIDE_INT; i += width)
289 : {
290 5117 : HOST_WIDE_INT each = wi::extract_uhwi (val, i, width);
291 5117 : if (val_broadcast != each)
292 : return false;
293 : }
294 1461 : val_broadcast = sext_hwi (val_broadcast, width);
295 1461 : return true;
296 4875 : }
297 :
298 : /* Convert the CONST_WIDE_INT operand OP to broadcast in MODE. */
299 :
300 : rtx
301 36643 : ix86_convert_const_wide_int_to_broadcast (machine_mode mode, rtx op)
302 : {
303 : /* Don't use integer vector broadcast if we can't move from GPR to SSE
304 : register directly. */
305 36643 : if (!TARGET_INTER_UNIT_MOVES_TO_VEC)
306 : return nullptr;
307 :
308 36643 : unsigned int msize = GET_MODE_SIZE (mode);
309 :
310 : /* Only optimized for vpbroadcast[bwsd]/vbroadcastss with xmm/ymm/zmm. */
311 36643 : if (msize != 16 && msize != 32 && msize != 64)
312 : return nullptr;
313 :
314 : /* Convert CONST_WIDE_INT to a non-standard SSE constant integer
315 : broadcast only if vector broadcast is available. */
316 36643 : if (!TARGET_AVX
317 1674 : || !CONST_WIDE_INT_P (op)
318 1611 : || standard_sse_constant_p (op, mode)
319 38254 : || (CONST_WIDE_INT_NUNITS (op) * HOST_BITS_PER_WIDE_INT
320 1611 : != GET_MODE_BITSIZE (mode)))
321 : return nullptr;
322 :
323 1603 : HOST_WIDE_INT val = CONST_WIDE_INT_ELT (op, 0);
324 1603 : HOST_WIDE_INT val_broadcast;
325 1603 : scalar_int_mode broadcast_mode;
326 : /* vpbroadcastb zmm requires TARGET_AVX512BW. */
327 716 : if ((msize == 64 ? TARGET_AVX512BW : TARGET_AVX2)
328 2100 : && ix86_broadcast (val, GET_MODE_BITSIZE (QImode),
329 : val_broadcast))
330 : broadcast_mode = QImode;
331 658 : else if ((msize == 64 ? TARGET_AVX512BW : TARGET_AVX2)
332 1979 : && ix86_broadcast (val, GET_MODE_BITSIZE (HImode),
333 : val_broadcast))
334 : broadcast_mode = HImode;
335 : /* vbroadcasts[sd] only support memory operand w/o AVX2.
336 : When msize == 16, pshufs is used for vec_duplicate.
337 : when msize == 64, vpbroadcastd is used, and TARGET_AVX512F must be existed. */
338 416 : else if ((msize != 32 || TARGET_AVX2)
339 1779 : && ix86_broadcast (val, GET_MODE_BITSIZE (SImode),
340 : val_broadcast))
341 : broadcast_mode = SImode;
342 1399 : else if (TARGET_64BIT && (msize != 32 || TARGET_AVX2)
343 2656 : && ix86_broadcast (val, GET_MODE_BITSIZE (DImode),
344 : val_broadcast))
345 : broadcast_mode = DImode;
346 : else
347 : return nullptr;
348 :
349 : /* Check if OP can be broadcasted from VAL. */
350 1783 : for (int i = 1; i < CONST_WIDE_INT_NUNITS (op); i++)
351 1568 : if (val != CONST_WIDE_INT_ELT (op, i))
352 : return nullptr;
353 :
354 215 : unsigned int nunits = (GET_MODE_SIZE (mode)
355 215 : / GET_MODE_SIZE (broadcast_mode));
356 215 : machine_mode vector_mode;
357 215 : if (!mode_for_vector (broadcast_mode, nunits).exists (&vector_mode))
358 0 : gcc_unreachable ();
359 215 : rtx target = gen_reg_rtx (vector_mode);
360 215 : bool ok = ix86_expand_vector_init_duplicate (false, vector_mode,
361 : target,
362 : GEN_INT (val_broadcast));
363 215 : if (!ok)
364 : return nullptr;
365 215 : target = lowpart_subreg (mode, target, vector_mode);
366 215 : return target;
367 : }
368 :
369 : void
370 75627176 : ix86_expand_move (machine_mode mode, rtx operands[])
371 : {
372 75627176 : rtx op0, op1;
373 75627176 : rtx tmp, addend = NULL_RTX;
374 75627176 : enum tls_model model;
375 :
376 75627176 : op0 = operands[0];
377 75627176 : op1 = operands[1];
378 :
379 : /* Avoid complex sets of likely spilled hard registers before reload. */
380 75627176 : if (!ix86_hardreg_mov_ok (op0, op1))
381 : {
382 141258 : tmp = gen_reg_rtx (mode);
383 141258 : operands[0] = tmp;
384 141258 : ix86_expand_move (mode, operands);
385 141258 : operands[0] = op0;
386 141258 : operands[1] = tmp;
387 141258 : op1 = tmp;
388 : }
389 :
390 75627176 : switch (GET_CODE (op1))
391 : {
392 355288 : case CONST:
393 355288 : tmp = XEXP (op1, 0);
394 :
395 355288 : if (GET_CODE (tmp) != PLUS
396 343555 : || !SYMBOL_REF_P (XEXP (tmp, 0)))
397 : break;
398 :
399 340881 : op1 = XEXP (tmp, 0);
400 340881 : addend = XEXP (tmp, 1);
401 : /* FALLTHRU */
402 :
403 5161844 : case SYMBOL_REF:
404 5161844 : model = SYMBOL_REF_TLS_MODEL (op1);
405 :
406 5161844 : if (model)
407 10200 : op1 = legitimize_tls_address (op1, model, true);
408 5151644 : else if (ix86_force_load_from_GOT_p (op1))
409 : {
410 : /* Load the external function address via GOT slot to avoid PLT. */
411 24 : op1 = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, op1),
412 : (TARGET_64BIT
413 : ? UNSPEC_GOTPCREL
414 : : UNSPEC_GOT));
415 24 : op1 = gen_rtx_CONST (Pmode, op1);
416 24 : op1 = gen_const_mem (Pmode, op1);
417 20 : set_mem_alias_set (op1, GOT_ALIAS_SET);
418 : }
419 : else
420 : {
421 : #if TARGET_PECOFF
422 : tmp = legitimize_pe_coff_symbol (op1, addend != NULL_RTX);
423 :
424 : if (tmp)
425 : {
426 : op1 = tmp;
427 : if (!addend)
428 : break;
429 : }
430 : else
431 : #endif
432 5151624 : {
433 5151624 : op1 = operands[1];
434 5151624 : break;
435 : }
436 : }
437 :
438 10220 : if (addend)
439 : {
440 2787 : op1 = force_operand (op1, NULL_RTX);
441 2796 : op1 = expand_simple_binop (Pmode, PLUS, op1, addend,
442 : op0, 1, OPTAB_DIRECT);
443 : }
444 : else
445 7433 : op1 = force_operand (op1, op0);
446 :
447 10220 : if (op1 == op0)
448 : return;
449 :
450 1152 : op1 = convert_to_mode (mode, op1, 1);
451 :
452 : default:
453 : break;
454 :
455 1609270 : case SUBREG:
456 : /* Transform TImode paradoxical SUBREG into zero_extendditi2. */
457 1609270 : if (TARGET_64BIT
458 1367124 : && mode == TImode
459 : && SUBREG_P (op1)
460 74382 : && GET_MODE (SUBREG_REG (op1)) == DImode
461 1656088 : && SUBREG_BYTE (op1) == 0)
462 46818 : op1 = gen_rtx_ZERO_EXTEND (TImode, SUBREG_REG (op1));
463 : /* As not all values in XFmode are representable in real_value,
464 : we might be called with unfoldable SUBREGs of constants. */
465 1609270 : if (mode == XFmode
466 3186 : && CONSTANT_P (SUBREG_REG (op1))
467 0 : && can_create_pseudo_p ())
468 : {
469 0 : machine_mode imode = GET_MODE (SUBREG_REG (op1));
470 0 : rtx r = force_const_mem (imode, SUBREG_REG (op1));
471 0 : if (r)
472 0 : r = validize_mem (r);
473 : else
474 0 : r = force_reg (imode, SUBREG_REG (op1));
475 0 : op1 = simplify_gen_subreg (mode, r, imode, SUBREG_BYTE (op1));
476 : }
477 : break;
478 : }
479 :
480 75618108 : if ((flag_pic || MACHOPIC_INDIRECT)
481 75618108 : && symbolic_operand (op1, mode))
482 : {
483 : #if TARGET_MACHO
484 : if (TARGET_MACHO && !TARGET_64BIT)
485 : {
486 : /* dynamic-no-pic */
487 : if (MACHOPIC_INDIRECT)
488 : {
489 : tmp = (op0 && REG_P (op0) && mode == Pmode)
490 : ? op0 : gen_reg_rtx (Pmode);
491 : op1 = machopic_indirect_data_reference (op1, tmp);
492 : if (MACHOPIC_PURE)
493 : op1 = machopic_legitimize_pic_address (op1, mode,
494 : tmp == op1 ? 0 : tmp);
495 : }
496 : if (op0 != op1 && !MEM_P (op0))
497 : {
498 : rtx insn = gen_rtx_SET (op0, op1);
499 : emit_insn (insn);
500 : return;
501 : }
502 : }
503 : #endif
504 :
505 716747 : if (MEM_P (op0))
506 101825 : op1 = force_reg (mode, op1);
507 614922 : else if (!(TARGET_64BIT && x86_64_movabs_operand (op1, DImode)))
508 : {
509 614865 : rtx reg = can_create_pseudo_p () ? NULL_RTX : op0;
510 614865 : op1 = legitimize_pic_address (op1, reg);
511 614865 : if (op0 == op1)
512 : return;
513 614865 : op1 = convert_to_mode (mode, op1, 1);
514 : }
515 : }
516 : else
517 : {
518 74901361 : if (MEM_P (op0)
519 102256028 : && (PUSH_ROUNDING (GET_MODE_SIZE (mode)) != GET_MODE_SIZE (mode)
520 10978671 : || !push_operand (op0, mode))
521 87569873 : && MEM_P (op1))
522 2158821 : op1 = force_reg (mode, op1);
523 :
524 74901361 : if (push_operand (op0, mode)
525 74901361 : && ! general_no_elim_operand (op1, mode))
526 1007 : op1 = copy_to_mode_reg (mode, op1);
527 :
528 : /* Force large constants in 64bit compilation into register
529 : to get them CSEed. */
530 74901361 : if (can_create_pseudo_p ()
531 69099971 : && (mode == DImode) && TARGET_64BIT
532 36186759 : && immediate_operand (op1, mode)
533 7900934 : && !x86_64_zext_immediate_operand (op1, VOIDmode)
534 736754 : && !register_operand (op0, mode)
535 75080348 : && optimize)
536 126541 : op1 = copy_to_mode_reg (mode, op1);
537 :
538 74901361 : if (can_create_pseudo_p ())
539 : {
540 69099971 : if (CONST_DOUBLE_P (op1))
541 : {
542 : /* If we are loading a floating point constant to a
543 : register, force the value to memory now, since we'll
544 : get better code out the back end. */
545 :
546 911466 : op1 = validize_mem (force_const_mem (mode, op1));
547 911466 : if (!register_operand (op0, mode))
548 : {
549 133061 : tmp = gen_reg_rtx (mode);
550 133061 : emit_insn (gen_rtx_SET (tmp, op1));
551 133061 : emit_move_insn (op0, tmp);
552 133061 : return;
553 : }
554 : }
555 : }
556 : }
557 :
558 : /* Special case inserting 64-bit values into a TImode register. */
559 75485047 : if (TARGET_64BIT
560 : /* Disable for -O0 (see PR110587) unless naked (PR110533). */
561 65599762 : && (optimize || ix86_function_naked (current_function_decl))
562 45067629 : && (mode == DImode || mode == DFmode)
563 30842773 : && SUBREG_P (op0)
564 508987 : && GET_MODE (SUBREG_REG (op0)) == TImode
565 417192 : && REG_P (SUBREG_REG (op0))
566 75902239 : && REG_P (op1))
567 : {
568 : /* Use *insvti_lowpart_1 to set lowpart. */
569 187774 : if (SUBREG_BYTE (op0) == 0)
570 : {
571 57244 : wide_int mask = wi::mask (64, true, 128);
572 57244 : tmp = immed_wide_int_const (mask, TImode);
573 57244 : op0 = SUBREG_REG (op0);
574 57244 : tmp = gen_rtx_AND (TImode, copy_rtx (op0), tmp);
575 57244 : if (mode == DFmode)
576 372 : op1 = gen_lowpart (DImode, op1);
577 57244 : op1 = gen_rtx_ZERO_EXTEND (TImode, op1);
578 57244 : op1 = gen_rtx_IOR (TImode, tmp, op1);
579 57244 : }
580 : /* Use *insvti_highpart_1 to set highpart. */
581 130530 : else if (SUBREG_BYTE (op0) == 8)
582 : {
583 130530 : wide_int mask = wi::mask (64, false, 128);
584 130530 : tmp = immed_wide_int_const (mask, TImode);
585 130530 : op0 = SUBREG_REG (op0);
586 130530 : tmp = gen_rtx_AND (TImode, copy_rtx (op0), tmp);
587 130530 : if (mode == DFmode)
588 226 : op1 = gen_lowpart (DImode, op1);
589 130530 : op1 = gen_rtx_ZERO_EXTEND (TImode, op1);
590 130530 : op1 = gen_rtx_ASHIFT (TImode, op1, GEN_INT (64));
591 130530 : op1 = gen_rtx_IOR (TImode, tmp, op1);
592 130530 : }
593 : }
594 :
595 75485047 : emit_insn (gen_rtx_SET (op0, op1));
596 : }
597 :
598 : /* OP is a memref of CONST_VECTOR, return scalar constant mem
599 : if CONST_VECTOR is a vec_duplicate, else return NULL. */
600 : rtx
601 2654036 : ix86_broadcast_from_constant (machine_mode mode, rtx op)
602 : {
603 2654036 : int nunits = GET_MODE_NUNITS (mode);
604 2654036 : if (nunits < 2)
605 : return nullptr;
606 :
607 : /* Don't use integer vector broadcast if we can't move from GPR to SSE
608 : register directly. */
609 2519976 : if (!TARGET_INTER_UNIT_MOVES_TO_VEC
610 8030 : && INTEGRAL_MODE_P (mode))
611 : return nullptr;
612 :
613 : /* Convert CONST_VECTOR to a non-standard SSE constant integer
614 : broadcast only if vector broadcast is available. */
615 2514635 : if (standard_sse_constant_p (op, mode))
616 : return nullptr;
617 :
618 5029270 : if (GET_MODE_INNER (mode) == TImode)
619 : return nullptr;
620 :
621 2514525 : rtx constant = get_pool_constant (XEXP (op, 0));
622 2514525 : if (!CONST_VECTOR_P (constant))
623 : return nullptr;
624 :
625 : /* There could be some rtx like
626 : (mem/u/c:V16QI (symbol_ref/u:DI ("*.LC1")))
627 : but with "*.LC1" refer to V2DI constant vector. */
628 2514525 : if (GET_MODE (constant) != mode)
629 : {
630 776 : constant = simplify_subreg (mode, constant, GET_MODE (constant),
631 : 0);
632 776 : if (constant == nullptr || !CONST_VECTOR_P (constant))
633 : return nullptr;
634 : }
635 :
636 2514525 : rtx first = XVECEXP (constant, 0, 0);
637 :
638 8679868 : for (int i = 1; i < nunits; ++i)
639 : {
640 7967304 : rtx tmp = XVECEXP (constant, 0, i);
641 : /* Vector duplicate value. */
642 7967304 : if (!rtx_equal_p (tmp, first))
643 : return nullptr;
644 : }
645 :
646 : return first;
647 : }
648 :
649 : void
650 4742844 : ix86_expand_vector_move (machine_mode mode, rtx operands[])
651 : {
652 4742844 : rtx op0 = operands[0], op1 = operands[1];
653 : /* Use GET_MODE_BITSIZE instead of GET_MODE_ALIGNMENT for IA MCU
654 : psABI since the biggest alignment is 4 byte for IA MCU psABI. */
655 4742844 : unsigned int align = (TARGET_IAMCU
656 4742844 : ? GET_MODE_BITSIZE (mode)
657 4742844 : : GET_MODE_ALIGNMENT (mode));
658 :
659 4742844 : if (push_operand (op0, VOIDmode))
660 2920 : op0 = emit_move_resolve_push (mode, op0);
661 :
662 : /* Force constants other than zero into memory. We do not know how
663 : the instructions used to build constants modify the upper 64 bits
664 : of the register, once we have that information we may be able
665 : to handle some of them more efficiently. */
666 4742844 : if (can_create_pseudo_p ()
667 4545418 : && (CONSTANT_P (op1)
668 4212316 : || (SUBREG_P (op1)
669 329385 : && CONSTANT_P (SUBREG_REG (op1))))
670 5075960 : && ((register_operand (op0, mode)
671 275336 : && !standard_sse_constant_p (op1, mode))
672 : /* ix86_expand_vector_move_misalign() does not like constants. */
673 : || (SSE_REG_MODE_P (mode)
674 275313 : && MEM_P (op0)
675 42262 : && MEM_ALIGN (op0) < align)))
676 : {
677 4964 : if (SUBREG_P (op1))
678 : {
679 14 : machine_mode imode = GET_MODE (SUBREG_REG (op1));
680 14 : rtx r = force_const_mem (imode, SUBREG_REG (op1));
681 14 : if (r)
682 14 : r = validize_mem (r);
683 : else
684 0 : r = force_reg (imode, SUBREG_REG (op1));
685 14 : op1 = simplify_gen_subreg (mode, r, imode, SUBREG_BYTE (op1));
686 : }
687 : else
688 : {
689 4950 : machine_mode mode = GET_MODE (op0);
690 4950 : rtx tmp = ix86_convert_const_wide_int_to_broadcast
691 4950 : (mode, op1);
692 4950 : if (tmp == nullptr)
693 4929 : op1 = validize_mem (force_const_mem (mode, op1));
694 : else
695 : op1 = tmp;
696 : }
697 : }
698 :
699 4742844 : if (can_create_pseudo_p ()
700 4545418 : && GET_MODE_SIZE (mode) >= 16
701 3824971 : && VECTOR_MODE_P (mode)
702 8354277 : && (MEM_P (op1)
703 761965 : && SYMBOL_REF_P (XEXP (op1, 0))
704 523657 : && CONSTANT_POOL_ADDRESS_P (XEXP (op1, 0))))
705 : {
706 506181 : rtx first = ix86_broadcast_from_constant (mode, op1);
707 : /* Only broadcast floating-point constants when a broadcast instruction
708 : is available (AVX, or AVX2 for 16-bit elements), avoiding sequences
709 : such as movss + shufps. For integer constants, guard broadcasting
710 : with TARGET_PREFER_BCST_FROM_INTEGER. */
711 506181 : if (first != nullptr
712 644442 : && (FLOAT_MODE_P (mode)
713 21008 : ? (GET_MODE_UNIT_SIZE (mode) < 4 ? TARGET_AVX2 : TARGET_AVX)
714 117253 : : TARGET_PREFER_BCST_FROM_INTEGER))
715 : {
716 : /* Broadcast to XMM/YMM/ZMM register from an integer
717 : constant or scalar mem. */
718 122248 : rtx tmp = gen_reg_rtx (mode);
719 122248 : if (FLOAT_MODE_P (mode))
720 9994 : first = force_const_mem (GET_MODE_INNER (mode), first);
721 122248 : bool ok = ix86_expand_vector_init_duplicate (false, mode,
722 : tmp, first);
723 122248 : if (!ok && !TARGET_64BIT && GET_MODE_INNER (mode) == DImode)
724 : {
725 0 : first = force_const_mem (GET_MODE_INNER (mode), first);
726 0 : ok = ix86_expand_vector_init_duplicate (false, mode,
727 : tmp, first);
728 : }
729 122248 : if (ok)
730 : {
731 122248 : emit_move_insn (op0, tmp);
732 122248 : return;
733 : }
734 : }
735 : }
736 :
737 : /* We need to check memory alignment for SSE mode since attribute
738 : can make operands unaligned. */
739 4620596 : if (can_create_pseudo_p ()
740 : && SSE_REG_MODE_P (mode)
741 9328466 : && ((MEM_P (op0) && (MEM_ALIGN (op0) < align))
742 4180615 : || (MEM_P (op1) && (MEM_ALIGN (op1) < align))))
743 : {
744 332314 : rtx tmp[2];
745 :
746 : /* ix86_expand_vector_move_misalign() does not like both
747 : arguments in memory. */
748 332314 : if (!register_operand (op0, mode)
749 332314 : && !register_operand (op1, mode))
750 : {
751 96028 : rtx scratch = gen_reg_rtx (mode);
752 96028 : emit_move_insn (scratch, op1);
753 96028 : op1 = scratch;
754 : }
755 :
756 332314 : tmp[0] = op0; tmp[1] = op1;
757 332314 : ix86_expand_vector_move_misalign (mode, tmp);
758 332314 : return;
759 : }
760 :
761 : /* Special case TImode to 128-bit vector conversions via V2DI. */
762 1175466 : if (VECTOR_MODE_P (mode)
763 4238897 : && GET_MODE_SIZE (mode) == 16
764 3018445 : && SUBREG_P (op1)
765 259367 : && GET_MODE (SUBREG_REG (op1)) == TImode
766 3204 : && TARGET_64BIT && TARGET_SSE
767 4290866 : && ix86_pre_reload_split ())
768 : {
769 2151 : rtx tmp = gen_reg_rtx (V2DImode);
770 2151 : rtx lo = gen_reg_rtx (DImode);
771 2151 : rtx hi = gen_reg_rtx (DImode);
772 2151 : emit_move_insn (lo, gen_lowpart (DImode, SUBREG_REG (op1)));
773 2151 : emit_move_insn (hi, gen_highpart (DImode, SUBREG_REG (op1)));
774 2151 : emit_insn (gen_vec_concatv2di (tmp, lo, hi));
775 2151 : emit_move_insn (op0, gen_lowpart (mode, tmp));
776 2151 : return;
777 : }
778 :
779 : /* If operand0 is a hard register, make operand1 a pseudo. */
780 4286131 : if (can_create_pseudo_p ()
781 8374836 : && !ix86_hardreg_mov_ok (op0, op1))
782 : {
783 159 : rtx tmp = gen_reg_rtx (GET_MODE (op0));
784 159 : emit_move_insn (tmp, op1);
785 159 : emit_move_insn (op0, tmp);
786 159 : return;
787 : }
788 :
789 : /* Make operand1 a register if it isn't already. */
790 4285972 : if (can_create_pseudo_p ()
791 4088546 : && !register_operand (op0, mode)
792 5437392 : && !register_operand (op1, mode))
793 : {
794 225358 : rtx tmp = gen_reg_rtx (GET_MODE (op0));
795 225358 : emit_move_insn (tmp, op1);
796 225358 : emit_move_insn (op0, tmp);
797 225358 : return;
798 : }
799 :
800 4060614 : emit_insn (gen_rtx_SET (op0, op1));
801 : }
802 :
803 : /* Split 32-byte AVX unaligned load and store if needed. */
804 :
805 : static void
806 13541 : ix86_avx256_split_vector_move_misalign (rtx op0, rtx op1)
807 : {
808 13541 : rtx m;
809 13541 : rtx (*extract) (rtx, rtx, rtx);
810 13541 : machine_mode mode;
811 :
812 13541 : if ((MEM_P (op1) && !TARGET_AVX256_SPLIT_UNALIGNED_LOAD)
813 5324 : || (MEM_P (op0) && !TARGET_AVX256_SPLIT_UNALIGNED_STORE))
814 : {
815 13515 : emit_insn (gen_rtx_SET (op0, op1));
816 13515 : return;
817 : }
818 :
819 26 : rtx orig_op0 = NULL_RTX;
820 26 : mode = GET_MODE (op0);
821 26 : switch (GET_MODE_CLASS (mode))
822 : {
823 9 : case MODE_VECTOR_INT:
824 9 : case MODE_INT:
825 9 : if (mode != V32QImode)
826 : {
827 7 : if (!MEM_P (op0))
828 : {
829 3 : orig_op0 = op0;
830 3 : op0 = gen_reg_rtx (V32QImode);
831 : }
832 : else
833 4 : op0 = gen_lowpart (V32QImode, op0);
834 7 : op1 = gen_lowpart (V32QImode, op1);
835 7 : mode = V32QImode;
836 : }
837 : break;
838 : case MODE_VECTOR_FLOAT:
839 : break;
840 0 : default:
841 0 : gcc_unreachable ();
842 : }
843 :
844 26 : switch (mode)
845 : {
846 0 : default:
847 0 : gcc_unreachable ();
848 : case E_V32QImode:
849 : extract = gen_avx_vextractf128v32qi;
850 : mode = V16QImode;
851 : break;
852 1 : case E_V16BFmode:
853 1 : extract = gen_avx_vextractf128v16bf;
854 1 : mode = V8BFmode;
855 1 : break;
856 0 : case E_V16HFmode:
857 0 : extract = gen_avx_vextractf128v16hf;
858 0 : mode = V8HFmode;
859 0 : break;
860 8 : case E_V8SFmode:
861 8 : extract = gen_avx_vextractf128v8sf;
862 8 : mode = V4SFmode;
863 8 : break;
864 8 : case E_V4DFmode:
865 8 : extract = gen_avx_vextractf128v4df;
866 8 : mode = V2DFmode;
867 8 : break;
868 : }
869 :
870 26 : if (MEM_P (op1))
871 : {
872 9 : rtx r = gen_reg_rtx (mode);
873 9 : m = adjust_address (op1, mode, 0);
874 9 : emit_move_insn (r, m);
875 9 : m = adjust_address (op1, mode, 16);
876 9 : r = gen_rtx_VEC_CONCAT (GET_MODE (op0), r, m);
877 9 : emit_move_insn (op0, r);
878 : }
879 17 : else if (MEM_P (op0))
880 : {
881 17 : m = adjust_address (op0, mode, 0);
882 17 : emit_insn (extract (m, op1, const0_rtx));
883 17 : m = adjust_address (op0, mode, 16);
884 17 : emit_insn (extract (m, copy_rtx (op1), const1_rtx));
885 : }
886 : else
887 0 : gcc_unreachable ();
888 :
889 26 : if (orig_op0)
890 3 : emit_move_insn (orig_op0, gen_lowpart (GET_MODE (orig_op0), op0));
891 : }
892 :
893 : /* Implement the movmisalign patterns for SSE. Non-SSE modes go
894 : straight to ix86_expand_vector_move. */
895 : /* Code generation for scalar reg-reg moves of single and double precision data:
896 : if (x86_sse_partial_reg_dependency == true | x86_sse_split_regs == true)
897 : movaps reg, reg
898 : else
899 : movss reg, reg
900 : if (x86_sse_partial_reg_dependency == true)
901 : movapd reg, reg
902 : else
903 : movsd reg, reg
904 :
905 : Code generation for scalar loads of double precision data:
906 : if (x86_sse_split_regs == true)
907 : movlpd mem, reg (gas syntax)
908 : else
909 : movsd mem, reg
910 :
911 : Code generation for unaligned packed loads of single precision data
912 : (x86_sse_unaligned_move_optimal overrides x86_sse_partial_reg_dependency):
913 : if (x86_sse_unaligned_move_optimal)
914 : movups mem, reg
915 :
916 : if (x86_sse_partial_reg_dependency == true)
917 : {
918 : xorps reg, reg
919 : movlps mem, reg
920 : movhps mem+8, reg
921 : }
922 : else
923 : {
924 : movlps mem, reg
925 : movhps mem+8, reg
926 : }
927 :
928 : Code generation for unaligned packed loads of double precision data
929 : (x86_sse_unaligned_move_optimal overrides x86_sse_split_regs):
930 : if (x86_sse_unaligned_move_optimal)
931 : movupd mem, reg
932 :
933 : if (x86_sse_split_regs == true)
934 : {
935 : movlpd mem, reg
936 : movhpd mem+8, reg
937 : }
938 : else
939 : {
940 : movsd mem, reg
941 : movhpd mem+8, reg
942 : }
943 : */
944 :
945 : void
946 660610 : ix86_expand_vector_move_misalign (machine_mode mode, rtx operands[])
947 : {
948 660610 : rtx op0, op1, m;
949 :
950 660610 : op0 = operands[0];
951 660610 : op1 = operands[1];
952 :
953 : /* Use unaligned load/store for AVX512 or when optimizing for size. */
954 1321220 : if (GET_MODE_SIZE (mode) == 64 || optimize_insn_for_size_p ())
955 : {
956 26668 : emit_insn (gen_rtx_SET (op0, op1));
957 26668 : return;
958 : }
959 :
960 633942 : if (TARGET_AVX)
961 : {
962 64462 : if (GET_MODE_SIZE (mode) == 32)
963 13541 : ix86_avx256_split_vector_move_misalign (op0, op1);
964 : else
965 : /* Always use 128-bit mov<mode>_internal pattern for AVX. */
966 18690 : emit_insn (gen_rtx_SET (op0, op1));
967 : return;
968 : }
969 :
970 601711 : if (TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
971 95 : || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL)
972 : {
973 601616 : emit_insn (gen_rtx_SET (op0, op1));
974 601616 : return;
975 : }
976 :
977 : /* ??? If we have typed data, then it would appear that using
978 : movdqu is the only way to get unaligned data loaded with
979 : integer type. */
980 95 : if (TARGET_SSE2 && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
981 : {
982 81 : emit_insn (gen_rtx_SET (op0, op1));
983 81 : return;
984 : }
985 :
986 14 : if (MEM_P (op1))
987 : {
988 6 : if (TARGET_SSE2 && mode == V2DFmode)
989 : {
990 2 : rtx zero;
991 :
992 : /* When SSE registers are split into halves, we can avoid
993 : writing to the top half twice. */
994 2 : if (TARGET_SSE_SPLIT_REGS)
995 : {
996 2 : emit_clobber (op0);
997 2 : zero = op0;
998 : }
999 : else
1000 : {
1001 : /* ??? Not sure about the best option for the Intel chips.
1002 : The following would seem to satisfy; the register is
1003 : entirely cleared, breaking the dependency chain. We
1004 : then store to the upper half, with a dependency depth
1005 : of one. A rumor has it that Intel recommends two movsd
1006 : followed by an unpacklpd, but this is unconfirmed. And
1007 : given that the dependency depth of the unpacklpd would
1008 : still be one, I'm not sure why this would be better. */
1009 0 : zero = CONST0_RTX (V2DFmode);
1010 : }
1011 :
1012 2 : m = adjust_address (op1, DFmode, 0);
1013 2 : emit_insn (gen_sse2_loadlpd (op0, zero, m));
1014 2 : m = adjust_address (op1, DFmode, 8);
1015 2 : emit_insn (gen_sse2_loadhpd (op0, op0, m));
1016 2 : }
1017 : else
1018 : {
1019 4 : rtx t;
1020 :
1021 4 : if (mode != V4SFmode)
1022 0 : t = gen_reg_rtx (V4SFmode);
1023 : else
1024 : t = op0;
1025 :
1026 4 : if (TARGET_SSE_PARTIAL_REG_DEPENDENCY)
1027 2 : emit_move_insn (t, CONST0_RTX (V4SFmode));
1028 : else
1029 2 : emit_clobber (t);
1030 :
1031 4 : m = adjust_address (op1, V2SFmode, 0);
1032 4 : emit_insn (gen_sse_loadlps (t, t, m));
1033 4 : m = adjust_address (op1, V2SFmode, 8);
1034 4 : emit_insn (gen_sse_loadhps (t, t, m));
1035 4 : if (mode != V4SFmode)
1036 0 : emit_move_insn (op0, gen_lowpart (mode, t));
1037 : }
1038 : }
1039 8 : else if (MEM_P (op0))
1040 : {
1041 8 : if (TARGET_SSE2 && mode == V2DFmode)
1042 : {
1043 2 : m = adjust_address (op0, DFmode, 0);
1044 2 : emit_insn (gen_sse2_storelpd (m, op1));
1045 2 : m = adjust_address (op0, DFmode, 8);
1046 2 : emit_insn (gen_sse2_storehpd (m, op1));
1047 : }
1048 : else
1049 : {
1050 6 : if (mode != V4SFmode)
1051 0 : op1 = gen_lowpart (V4SFmode, op1);
1052 :
1053 6 : m = adjust_address (op0, V2SFmode, 0);
1054 6 : emit_insn (gen_sse_storelps (m, op1));
1055 6 : m = adjust_address (op0, V2SFmode, 8);
1056 6 : emit_insn (gen_sse_storehps (m, copy_rtx (op1)));
1057 : }
1058 : }
1059 : else
1060 0 : gcc_unreachable ();
1061 : }
1062 :
1063 : /* Move bits 64:95 to bits 32:63. */
1064 :
1065 : void
1066 972 : ix86_move_vector_high_sse_to_mmx (rtx op)
1067 : {
1068 972 : rtx mask = gen_rtx_PARALLEL (VOIDmode,
1069 : gen_rtvec (4, GEN_INT (0), GEN_INT (2),
1070 : GEN_INT (0), GEN_INT (0)));
1071 972 : rtx dest = lowpart_subreg (V4SImode, op, GET_MODE (op));
1072 972 : op = gen_rtx_VEC_SELECT (V4SImode, dest, mask);
1073 972 : rtx insn = gen_rtx_SET (dest, op);
1074 972 : emit_insn (insn);
1075 972 : }
1076 :
1077 : /* Split MMX pack with signed/unsigned saturation with SSE/SSE2. */
1078 :
1079 : void
1080 882 : ix86_split_mmx_pack (rtx operands[], enum rtx_code code)
1081 : {
1082 882 : rtx op0 = operands[0];
1083 882 : rtx op1 = operands[1];
1084 882 : rtx op2 = operands[2];
1085 882 : rtx src;
1086 :
1087 882 : machine_mode dmode = GET_MODE (op0);
1088 882 : machine_mode smode = GET_MODE (op1);
1089 882 : machine_mode inner_dmode = GET_MODE_INNER (dmode);
1090 882 : machine_mode inner_smode = GET_MODE_INNER (smode);
1091 :
1092 : /* Get the corresponding SSE mode for destination. */
1093 882 : int nunits = 16 / GET_MODE_SIZE (inner_dmode);
1094 1764 : machine_mode sse_dmode = mode_for_vector (GET_MODE_INNER (dmode),
1095 1764 : nunits).require ();
1096 882 : machine_mode sse_half_dmode = mode_for_vector (GET_MODE_INNER (dmode),
1097 1764 : nunits / 2).require ();
1098 :
1099 : /* Get the corresponding SSE mode for source. */
1100 882 : nunits = 16 / GET_MODE_SIZE (inner_smode);
1101 1764 : machine_mode sse_smode = mode_for_vector (GET_MODE_INNER (smode),
1102 1764 : nunits).require ();
1103 :
1104 : /* Generate SSE pack with signed/unsigned saturation. */
1105 882 : rtx dest = lowpart_subreg (sse_dmode, op0, GET_MODE (op0));
1106 882 : op1 = lowpart_subreg (sse_smode, op1, GET_MODE (op1));
1107 882 : op2 = lowpart_subreg (sse_smode, op2, GET_MODE (op2));
1108 :
1109 : /* paskusdw/packuswb does unsigned saturation of a signed source
1110 : which is different from generic us_truncate RTX. */
1111 882 : if (code == US_TRUNCATE)
1112 780 : src = gen_rtx_UNSPEC (sse_dmode,
1113 : gen_rtvec (2, op1, op2),
1114 : UNSPEC_US_TRUNCATE);
1115 : else
1116 : {
1117 102 : op1 = gen_rtx_fmt_e (code, sse_half_dmode, op1);
1118 102 : op2 = gen_rtx_fmt_e (code, sse_half_dmode, op2);
1119 102 : src = gen_rtx_VEC_CONCAT (sse_dmode, op1, op2);
1120 : }
1121 :
1122 882 : emit_move_insn (dest, src);
1123 :
1124 882 : ix86_move_vector_high_sse_to_mmx (op0);
1125 882 : }
1126 :
1127 : /* Split MMX punpcklXX/punpckhXX with SSE punpcklXX. This is also used
1128 : for a full unpack of OPERANDS[1] and OPERANDS[2] into a wider
1129 : OPERANDS[0]. */
1130 :
1131 : void
1132 6722 : ix86_split_mmx_punpck (rtx operands[], bool high_p)
1133 : {
1134 6722 : rtx op0 = operands[0];
1135 6722 : rtx op1 = operands[1];
1136 6722 : rtx op2 = operands[2];
1137 6722 : machine_mode mode = GET_MODE (op1);
1138 6722 : rtx mask;
1139 : /* The corresponding SSE mode. */
1140 6722 : machine_mode sse_mode, double_sse_mode;
1141 :
1142 6722 : switch (mode)
1143 : {
1144 1785 : case E_V8QImode:
1145 1785 : case E_V4QImode:
1146 1785 : case E_V2QImode:
1147 1785 : sse_mode = V16QImode;
1148 1785 : double_sse_mode = V32QImode;
1149 1785 : mask = gen_rtx_PARALLEL (VOIDmode,
1150 : gen_rtvec (16,
1151 : GEN_INT (0), GEN_INT (16),
1152 : GEN_INT (1), GEN_INT (17),
1153 : GEN_INT (2), GEN_INT (18),
1154 : GEN_INT (3), GEN_INT (19),
1155 : GEN_INT (4), GEN_INT (20),
1156 : GEN_INT (5), GEN_INT (21),
1157 : GEN_INT (6), GEN_INT (22),
1158 : GEN_INT (7), GEN_INT (23)));
1159 1785 : break;
1160 :
1161 3600 : case E_V4HImode:
1162 3600 : case E_V2HImode:
1163 3600 : sse_mode = V8HImode;
1164 3600 : double_sse_mode = V16HImode;
1165 3600 : mask = gen_rtx_PARALLEL (VOIDmode,
1166 : gen_rtvec (8,
1167 : GEN_INT (0), GEN_INT (8),
1168 : GEN_INT (1), GEN_INT (9),
1169 : GEN_INT (2), GEN_INT (10),
1170 : GEN_INT (3), GEN_INT (11)));
1171 3600 : break;
1172 :
1173 937 : case E_V2SImode:
1174 937 : sse_mode = V4SImode;
1175 937 : double_sse_mode = V8SImode;
1176 937 : mask = gen_rtx_PARALLEL (VOIDmode,
1177 : gen_rtvec (4,
1178 : GEN_INT (0), GEN_INT (4),
1179 : GEN_INT (1), GEN_INT (5)));
1180 937 : break;
1181 :
1182 400 : case E_V2SFmode:
1183 400 : sse_mode = V4SFmode;
1184 400 : double_sse_mode = V8SFmode;
1185 400 : mask = gen_rtx_PARALLEL (VOIDmode,
1186 : gen_rtvec (4,
1187 : GEN_INT (0), GEN_INT (4),
1188 : GEN_INT (1), GEN_INT (5)));
1189 400 : break;
1190 :
1191 0 : default:
1192 0 : gcc_unreachable ();
1193 : }
1194 :
1195 : /* Generate SSE punpcklXX. */
1196 6722 : rtx dest = lowpart_subreg (sse_mode, op0, GET_MODE (op0));
1197 6722 : op1 = lowpart_subreg (sse_mode, op1, GET_MODE (op1));
1198 6722 : op2 = lowpart_subreg (sse_mode, op2, GET_MODE (op2));
1199 :
1200 6722 : op1 = gen_rtx_VEC_CONCAT (double_sse_mode, op1, op2);
1201 6722 : op2 = gen_rtx_VEC_SELECT (sse_mode, op1, mask);
1202 6722 : rtx insn = gen_rtx_SET (dest, op2);
1203 6722 : emit_insn (insn);
1204 :
1205 : /* Move high bits to low bits. */
1206 6722 : if (high_p)
1207 : {
1208 2525 : if (sse_mode == V4SFmode)
1209 : {
1210 112 : mask = gen_rtx_PARALLEL (VOIDmode,
1211 : gen_rtvec (4, GEN_INT (2), GEN_INT (3),
1212 : GEN_INT (4), GEN_INT (5)));
1213 112 : op2 = gen_rtx_VEC_CONCAT (V8SFmode, dest, dest);
1214 112 : op1 = gen_rtx_VEC_SELECT (V4SFmode, op2, mask);
1215 : }
1216 : else
1217 : {
1218 2413 : int sz = GET_MODE_SIZE (mode);
1219 :
1220 2413 : if (sz == 4)
1221 243 : mask = gen_rtx_PARALLEL (VOIDmode,
1222 : gen_rtvec (4, GEN_INT (1), GEN_INT (0),
1223 : GEN_INT (0), GEN_INT (1)));
1224 2170 : else if (sz == 8)
1225 2170 : mask = gen_rtx_PARALLEL (VOIDmode,
1226 : gen_rtvec (4, GEN_INT (2), GEN_INT (3),
1227 : GEN_INT (0), GEN_INT (1)));
1228 : else
1229 0 : gcc_unreachable ();
1230 :
1231 2413 : dest = lowpart_subreg (V4SImode, dest, GET_MODE (dest));
1232 2413 : op1 = gen_rtx_VEC_SELECT (V4SImode, dest, mask);
1233 : }
1234 :
1235 2525 : insn = gen_rtx_SET (dest, op1);
1236 2525 : emit_insn (insn);
1237 : }
1238 6722 : }
1239 :
1240 : /* Helper function of ix86_fixup_binary_operands to canonicalize
1241 : operand order. Returns true if the operands should be swapped. */
1242 :
1243 : static bool
1244 179256939 : ix86_swap_binary_operands_p (enum rtx_code code, machine_mode mode,
1245 : rtx operands[])
1246 : {
1247 179256939 : rtx dst = operands[0];
1248 179256939 : rtx src1 = operands[1];
1249 179256939 : rtx src2 = operands[2];
1250 :
1251 : /* If the operation is not commutative, we can't do anything. */
1252 179256939 : if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
1253 28309863 : && GET_RTX_CLASS (code) != RTX_COMM_COMPARE)
1254 : return false;
1255 :
1256 : /* Highest priority is that src1 should match dst. */
1257 150959435 : if (rtx_equal_p (dst, src1))
1258 : return false;
1259 109894510 : if (rtx_equal_p (dst, src2))
1260 : return true;
1261 :
1262 : /* Next highest priority is that immediate constants come second. */
1263 109799205 : if (immediate_operand (src2, mode))
1264 : return false;
1265 26812628 : if (immediate_operand (src1, mode))
1266 : return true;
1267 :
1268 : /* Lowest priority is that memory references should come second. */
1269 26812628 : if (MEM_P (src2))
1270 : return false;
1271 25284382 : if (MEM_P (src1))
1272 579775 : return true;
1273 :
1274 : return false;
1275 : }
1276 :
1277 : /* Fix up OPERANDS to satisfy ix86_binary_operator_ok. Return the
1278 : destination to use for the operation. If different from the true
1279 : destination in operands[0], a copy operation will be required except
1280 : under TARGET_APX_NDD. */
1281 :
1282 : rtx
1283 13864672 : ix86_fixup_binary_operands (enum rtx_code code, machine_mode mode,
1284 : rtx operands[], bool use_ndd)
1285 : {
1286 13864672 : rtx dst = operands[0];
1287 13864672 : rtx src1 = operands[1];
1288 13864672 : rtx src2 = operands[2];
1289 :
1290 : /* Canonicalize operand order. */
1291 13864672 : if (ix86_swap_binary_operands_p (code, mode, operands))
1292 : {
1293 : /* It is invalid to swap operands of different modes. */
1294 91223 : gcc_assert (GET_MODE (src1) == GET_MODE (src2));
1295 :
1296 : std::swap (src1, src2);
1297 : }
1298 :
1299 : /* Both source operands cannot be in memory. */
1300 13864672 : if (MEM_P (src1) && MEM_P (src2))
1301 : {
1302 : /* Optimization: Only read from memory once. */
1303 111820 : if (rtx_equal_p (src1, src2))
1304 : {
1305 19 : src2 = force_reg (mode, src2);
1306 19 : src1 = src2;
1307 : }
1308 111801 : else if (rtx_equal_p (dst, src1))
1309 3545 : src2 = force_reg (mode, src2);
1310 : else
1311 108256 : src1 = force_reg (mode, src1);
1312 : }
1313 :
1314 : /* If the destination is memory, and we do not have matching source
1315 : operands, do things in registers. */
1316 13864672 : if (MEM_P (dst) && !rtx_equal_p (dst, src1))
1317 497626 : dst = gen_reg_rtx (mode);
1318 :
1319 : /* Source 1 cannot be a constant. */
1320 13864672 : if (CONSTANT_P (src1))
1321 713 : src1 = force_reg (mode, src1);
1322 :
1323 : /* Source 1 cannot be a non-matching memory. */
1324 13864672 : if (!use_ndd && MEM_P (src1) && !rtx_equal_p (dst, src1))
1325 455848 : src1 = force_reg (mode, src1);
1326 :
1327 : /* Improve address combine. */
1328 13864672 : if (code == PLUS
1329 10264197 : && GET_MODE_CLASS (mode) == MODE_INT
1330 10153411 : && MEM_P (src2))
1331 182810 : src2 = force_reg (mode, src2);
1332 :
1333 13864672 : operands[1] = src1;
1334 13864672 : operands[2] = src2;
1335 13864672 : return dst;
1336 : }
1337 :
1338 : /* Similarly, but assume that the destination has already been
1339 : set up properly. */
1340 :
1341 : void
1342 301105 : ix86_fixup_binary_operands_no_copy (enum rtx_code code,
1343 : machine_mode mode, rtx operands[],
1344 : bool use_ndd)
1345 : {
1346 301105 : rtx dst = ix86_fixup_binary_operands (code, mode, operands, use_ndd);
1347 301105 : gcc_assert (dst == operands[0]);
1348 301105 : }
1349 :
1350 : /* Attempt to expand a binary operator. Make the expansion closer to the
1351 : actual machine, then just general_operand, which will allow 3 separate
1352 : memory references (one output, two input) in a single insn. */
1353 :
1354 : void
1355 13563438 : ix86_expand_binary_operator (enum rtx_code code, machine_mode mode,
1356 : rtx operands[], bool use_ndd)
1357 : {
1358 13563438 : rtx src1, src2, dst, op, clob;
1359 :
1360 13563438 : dst = ix86_fixup_binary_operands (code, mode, operands, use_ndd);
1361 13563438 : src1 = operands[1];
1362 13563438 : src2 = operands[2];
1363 :
1364 : /* Emit the instruction. */
1365 :
1366 13563438 : op = gen_rtx_SET (dst, gen_rtx_fmt_ee (code, mode, src1, src2));
1367 :
1368 13563438 : if (reload_completed
1369 92688 : && code == PLUS
1370 927 : && !rtx_equal_p (dst, src1)
1371 13563438 : && !use_ndd)
1372 : {
1373 : /* This is going to be an LEA; avoid splitting it later. */
1374 0 : emit_insn (op);
1375 : }
1376 : else
1377 : {
1378 13563438 : clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
1379 13563438 : emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
1380 : }
1381 :
1382 : /* Fix up the destination if needed. */
1383 13563438 : if (dst != operands[0])
1384 497617 : emit_move_insn (operands[0], dst);
1385 13563438 : }
1386 :
1387 : /* Expand vector logical operation CODE (AND, IOR, XOR) in MODE with
1388 : the given OPERANDS. */
1389 :
1390 : void
1391 87056 : ix86_expand_vector_logical_operator (enum rtx_code code, machine_mode mode,
1392 : rtx operands[])
1393 : {
1394 87056 : rtx op1 = NULL_RTX, op2 = NULL_RTX;
1395 87056 : if (SUBREG_P (operands[1]))
1396 : {
1397 372 : op1 = operands[1];
1398 372 : op2 = operands[2];
1399 : }
1400 86684 : else if (SUBREG_P (operands[2]))
1401 : {
1402 : op1 = operands[2];
1403 : op2 = operands[1];
1404 : }
1405 : /* Optimize (__m128i) d | (__m128i) e and similar code
1406 : when d and e are float vectors into float vector logical
1407 : insn. In C/C++ without using intrinsics there is no other way
1408 : to express vector logical operation on float vectors than
1409 : to cast them temporarily to integer vectors. */
1410 3218 : if (op1
1411 3218 : && !TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
1412 3218 : && (SUBREG_P (op2) || CONST_VECTOR_P (op2))
1413 358 : && GET_MODE_CLASS (GET_MODE (SUBREG_REG (op1))) == MODE_VECTOR_FLOAT
1414 453 : && GET_MODE_SIZE (GET_MODE (SUBREG_REG (op1))) == GET_MODE_SIZE (mode)
1415 151 : && SUBREG_BYTE (op1) == 0
1416 151 : && (CONST_VECTOR_P (op2)
1417 1 : || (GET_MODE (SUBREG_REG (op1)) == GET_MODE (SUBREG_REG (op2))
1418 1 : && SUBREG_BYTE (op2) == 0))
1419 151 : && can_create_pseudo_p ())
1420 : {
1421 151 : rtx dst;
1422 151 : switch (GET_MODE (SUBREG_REG (op1)))
1423 : {
1424 67 : case E_V4SFmode:
1425 67 : case E_V8SFmode:
1426 67 : case E_V16SFmode:
1427 67 : case E_V2DFmode:
1428 67 : case E_V4DFmode:
1429 67 : case E_V8DFmode:
1430 67 : dst = gen_reg_rtx (GET_MODE (SUBREG_REG (op1)));
1431 67 : if (CONST_VECTOR_P (op2))
1432 : {
1433 66 : op2 = gen_lowpart (GET_MODE (dst), op2);
1434 66 : op2 = force_reg (GET_MODE (dst), op2);
1435 : }
1436 : else
1437 : {
1438 1 : op1 = operands[1];
1439 1 : op2 = SUBREG_REG (operands[2]);
1440 1 : if (!vector_operand (op2, GET_MODE (dst)))
1441 0 : op2 = force_reg (GET_MODE (dst), op2);
1442 : }
1443 67 : op1 = SUBREG_REG (op1);
1444 67 : if (!vector_operand (op1, GET_MODE (dst)))
1445 0 : op1 = force_reg (GET_MODE (dst), op1);
1446 67 : emit_insn (gen_rtx_SET (dst,
1447 : gen_rtx_fmt_ee (code, GET_MODE (dst),
1448 : op1, op2)));
1449 67 : emit_move_insn (operands[0], gen_lowpart (mode, dst));
1450 67 : return;
1451 : default:
1452 : break;
1453 : }
1454 : }
1455 86989 : if (!vector_operand (operands[1], mode))
1456 1 : operands[1] = force_reg (mode, operands[1]);
1457 86989 : if (!vector_operand (operands[2], mode))
1458 13894 : operands[2] = force_reg (mode, operands[2]);
1459 86989 : ix86_fixup_binary_operands_no_copy (code, mode, operands);
1460 86989 : emit_insn (gen_rtx_SET (operands[0],
1461 : gen_rtx_fmt_ee (code, mode, operands[1],
1462 : operands[2])));
1463 : }
1464 :
1465 : /* Return TRUE or FALSE depending on whether the binary operator meets the
1466 : appropriate constraints. */
1467 :
1468 : bool
1469 166525219 : ix86_binary_operator_ok (enum rtx_code code, machine_mode mode,
1470 : rtx operands[3], bool use_ndd)
1471 : {
1472 166525219 : rtx dst = operands[0];
1473 166525219 : rtx src1 = operands[1];
1474 166525219 : rtx src2 = operands[2];
1475 :
1476 : /* Both source operands cannot be in memory. */
1477 158757454 : if ((MEM_P (src1) || bcst_mem_operand (src1, mode))
1478 166525596 : && (MEM_P (src2) || bcst_mem_operand (src2, mode)))
1479 : return false;
1480 :
1481 : /* Canonicalize operand order for commutative operators. */
1482 165392267 : if (ix86_swap_binary_operands_p (code, mode, operands))
1483 583857 : std::swap (src1, src2);
1484 :
1485 : /* If the destination is memory, we must have a matching source operand. */
1486 165392267 : if (MEM_P (dst) && !rtx_equal_p (dst, src1))
1487 : return false;
1488 :
1489 : /* Source 1 cannot be a constant. */
1490 160177116 : if (CONSTANT_P (src1))
1491 : return false;
1492 :
1493 : /* Source 1 cannot be a non-matching memory. */
1494 160174034 : if (!use_ndd && MEM_P (src1) && !rtx_equal_p (dst, src1))
1495 : /* Support "andhi/andsi/anddi" as a zero-extending move. */
1496 4745031 : return (code == AND
1497 586510 : && (mode == HImode
1498 586510 : || mode == SImode
1499 361512 : || (TARGET_64BIT && mode == DImode))
1500 5078245 : && satisfies_constraint_L (src2));
1501 :
1502 : return true;
1503 : }
1504 :
1505 : /* Attempt to expand a unary operator. Make the expansion closer to the
1506 : actual machine, then just general_operand, which will allow 2 separate
1507 : memory references (one output, one input) in a single insn. */
1508 :
1509 : void
1510 134917 : ix86_expand_unary_operator (enum rtx_code code, machine_mode mode,
1511 : rtx operands[], bool use_ndd)
1512 : {
1513 134917 : bool matching_memory = false;
1514 134917 : rtx src, dst, op, clob;
1515 :
1516 134917 : dst = operands[0];
1517 134917 : src = operands[1];
1518 :
1519 : /* If the destination is memory, and we do not have matching source
1520 : operands, do things in registers. */
1521 134917 : if (MEM_P (dst))
1522 : {
1523 12201 : if (rtx_equal_p (dst, src))
1524 : matching_memory = true;
1525 : else
1526 11891 : dst = gen_reg_rtx (mode);
1527 : }
1528 :
1529 : /* When source operand is memory, destination must match. */
1530 134917 : if (!use_ndd && MEM_P (src) && !matching_memory)
1531 4638 : src = force_reg (mode, src);
1532 :
1533 : /* Emit the instruction. */
1534 :
1535 134917 : op = gen_rtx_SET (dst, gen_rtx_fmt_e (code, mode, src));
1536 :
1537 134917 : if (code == NOT)
1538 71954 : emit_insn (op);
1539 : else
1540 : {
1541 62963 : clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
1542 62963 : emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
1543 : }
1544 :
1545 : /* Fix up the destination if needed. */
1546 134917 : if (dst != operands[0])
1547 11891 : emit_move_insn (operands[0], dst);
1548 134917 : }
1549 :
1550 : /* Return TRUE or FALSE depending on whether the unary operator meets the
1551 : appropriate constraints. */
1552 :
1553 : bool
1554 2092578 : ix86_unary_operator_ok (enum rtx_code,
1555 : machine_mode,
1556 : rtx operands[2],
1557 : bool use_ndd)
1558 : {
1559 : /* If one of operands is memory, source and destination must match. */
1560 2092578 : if ((MEM_P (operands[0])
1561 1949512 : || (!use_ndd && MEM_P (operands[1])))
1562 2122927 : && ! rtx_equal_p (operands[0], operands[1]))
1563 : return false;
1564 : return true;
1565 : }
1566 :
1567 : /* Predict just emitted jump instruction to be taken with probability PROB. */
1568 :
1569 : static void
1570 12881 : predict_jump (int prob)
1571 : {
1572 12881 : rtx_insn *insn = get_last_insn ();
1573 12881 : gcc_assert (JUMP_P (insn));
1574 12881 : add_reg_br_prob_note (insn, profile_probability::from_reg_br_prob_base (prob));
1575 12881 : }
1576 :
1577 : /* Split 32bit/64bit divmod with 8bit unsigned divmod if dividend and
1578 : divisor are within the range [0-255]. */
1579 :
1580 : void
1581 27 : ix86_split_idivmod (machine_mode mode, rtx operands[],
1582 : bool unsigned_p)
1583 : {
1584 27 : rtx_code_label *end_label, *qimode_label;
1585 27 : rtx div, mod;
1586 27 : rtx_insn *insn;
1587 27 : rtx scratch, tmp0, tmp1, tmp2;
1588 27 : rtx (*gen_divmod4_1) (rtx, rtx, rtx, rtx);
1589 :
1590 27 : operands[2] = force_reg (mode, operands[2]);
1591 27 : operands[3] = force_reg (mode, operands[3]);
1592 :
1593 27 : switch (mode)
1594 : {
1595 20 : case E_SImode:
1596 20 : if (GET_MODE (operands[0]) == SImode)
1597 : {
1598 16 : if (GET_MODE (operands[1]) == SImode)
1599 14 : gen_divmod4_1 = unsigned_p ? gen_udivmodsi4_1 : gen_divmodsi4_1;
1600 : else
1601 2 : gen_divmod4_1
1602 2 : = unsigned_p ? gen_udivmodsi4_zext_2 : gen_divmodsi4_zext_2;
1603 : }
1604 : else
1605 4 : gen_divmod4_1
1606 4 : = unsigned_p ? gen_udivmodsi4_zext_1 : gen_divmodsi4_zext_1;
1607 : break;
1608 :
1609 7 : case E_DImode:
1610 7 : gen_divmod4_1 = unsigned_p ? gen_udivmoddi4_1 : gen_divmoddi4_1;
1611 : break;
1612 :
1613 0 : default:
1614 0 : gcc_unreachable ();
1615 : }
1616 :
1617 27 : end_label = gen_label_rtx ();
1618 27 : qimode_label = gen_label_rtx ();
1619 :
1620 27 : scratch = gen_reg_rtx (mode);
1621 :
1622 : /* Use 8bit unsigned divimod if dividend and divisor are within
1623 : the range [0-255]. */
1624 27 : emit_move_insn (scratch, operands[2]);
1625 27 : scratch = expand_simple_binop (mode, IOR, scratch, operands[3],
1626 : scratch, 1, OPTAB_DIRECT);
1627 27 : emit_insn (gen_test_ccno_1 (mode, scratch, GEN_INT (-0x100)));
1628 27 : tmp0 = gen_rtx_REG (CCNOmode, FLAGS_REG);
1629 27 : tmp0 = gen_rtx_EQ (VOIDmode, tmp0, const0_rtx);
1630 27 : tmp0 = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp0,
1631 : gen_rtx_LABEL_REF (VOIDmode, qimode_label),
1632 : pc_rtx);
1633 27 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp0));
1634 27 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
1635 27 : JUMP_LABEL (insn) = qimode_label;
1636 :
1637 : /* Generate original signed/unsigned divimod. */
1638 27 : emit_insn (gen_divmod4_1 (operands[0], operands[1],
1639 : operands[2], operands[3]));
1640 :
1641 : /* Branch to the end. */
1642 27 : emit_jump_insn (gen_jump (end_label));
1643 27 : emit_barrier ();
1644 :
1645 : /* Generate 8bit unsigned divide. */
1646 27 : emit_label (qimode_label);
1647 : /* Don't use operands[0] for result of 8bit divide since not all
1648 : registers support QImode ZERO_EXTRACT. */
1649 27 : tmp0 = lowpart_subreg (HImode, scratch, mode);
1650 27 : tmp1 = lowpart_subreg (HImode, operands[2], mode);
1651 27 : tmp2 = lowpart_subreg (QImode, operands[3], mode);
1652 27 : emit_insn (gen_udivmodhiqi3 (tmp0, tmp1, tmp2));
1653 :
1654 27 : if (unsigned_p)
1655 : {
1656 12 : div = gen_rtx_UDIV (mode, operands[2], operands[3]);
1657 12 : mod = gen_rtx_UMOD (mode, operands[2], operands[3]);
1658 : }
1659 : else
1660 : {
1661 15 : div = gen_rtx_DIV (mode, operands[2], operands[3]);
1662 15 : mod = gen_rtx_MOD (mode, operands[2], operands[3]);
1663 : }
1664 27 : if (mode == SImode)
1665 : {
1666 20 : if (GET_MODE (operands[0]) != SImode)
1667 4 : div = gen_rtx_ZERO_EXTEND (DImode, div);
1668 20 : if (GET_MODE (operands[1]) != SImode)
1669 2 : mod = gen_rtx_ZERO_EXTEND (DImode, mod);
1670 : }
1671 :
1672 : /* Extract remainder from AH. */
1673 27 : scratch = gen_lowpart (GET_MODE (operands[1]), scratch);
1674 27 : tmp1 = gen_rtx_ZERO_EXTRACT (GET_MODE (operands[1]), scratch,
1675 : GEN_INT (8), GEN_INT (8));
1676 27 : insn = emit_move_insn (operands[1], tmp1);
1677 27 : set_unique_reg_note (insn, REG_EQUAL, mod);
1678 :
1679 : /* Zero extend quotient from AL. */
1680 27 : tmp1 = gen_lowpart (QImode, tmp0);
1681 27 : insn = emit_insn (gen_extend_insn
1682 27 : (operands[0], tmp1,
1683 27 : GET_MODE (operands[0]), QImode, 1));
1684 27 : set_unique_reg_note (insn, REG_EQUAL, div);
1685 :
1686 27 : emit_label (end_label);
1687 27 : }
1688 :
1689 : /* Emit x86 binary operand CODE in mode MODE, where the first operand
1690 : matches destination. RTX includes clobber of FLAGS_REG. */
1691 :
1692 : void
1693 9077 : ix86_emit_binop (enum rtx_code code, machine_mode mode,
1694 : rtx dst, rtx src)
1695 : {
1696 9077 : rtx op, clob;
1697 :
1698 9077 : op = gen_rtx_SET (dst, gen_rtx_fmt_ee (code, mode, dst, src));
1699 9077 : clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
1700 :
1701 9077 : emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
1702 9077 : }
1703 :
1704 : /* Return true if regno1 def is nearest to the insn. */
1705 :
1706 : static bool
1707 17 : find_nearest_reg_def (rtx_insn *insn, int regno1, int regno2)
1708 : {
1709 17 : rtx_insn *prev = insn;
1710 17 : rtx_insn *start = BB_HEAD (BLOCK_FOR_INSN (insn));
1711 :
1712 17 : if (insn == start)
1713 : return false;
1714 44 : while (prev && prev != start)
1715 : {
1716 34 : if (!INSN_P (prev) || !NONDEBUG_INSN_P (prev))
1717 : {
1718 10 : prev = PREV_INSN (prev);
1719 10 : continue;
1720 : }
1721 24 : if (insn_defines_reg (regno1, INVALID_REGNUM, prev))
1722 : return true;
1723 17 : else if (insn_defines_reg (regno2, INVALID_REGNUM, prev))
1724 : return false;
1725 17 : prev = PREV_INSN (prev);
1726 : }
1727 :
1728 : /* None of the regs is defined in the bb. */
1729 : return false;
1730 : }
1731 :
1732 : /* INSN_UID of the last insn emitted by zero store peephole2s. */
1733 : int ix86_last_zero_store_uid;
1734 :
1735 : /* Split lea instructions into a sequence of instructions
1736 : which are executed on ALU to avoid AGU stalls.
1737 : It is assumed that it is allowed to clobber flags register
1738 : at lea position. */
1739 :
1740 : void
1741 7115 : ix86_split_lea_for_addr (rtx_insn *insn, rtx operands[], machine_mode mode)
1742 : {
1743 7115 : unsigned int regno0, regno1, regno2;
1744 7115 : struct ix86_address parts;
1745 7115 : rtx target, tmp;
1746 7115 : int ok, adds;
1747 :
1748 7115 : ok = ix86_decompose_address (operands[1], &parts);
1749 7115 : gcc_assert (ok);
1750 :
1751 7115 : target = gen_lowpart (mode, operands[0]);
1752 :
1753 7115 : regno0 = true_regnum (target);
1754 7115 : regno1 = INVALID_REGNUM;
1755 7115 : regno2 = INVALID_REGNUM;
1756 :
1757 7115 : if (parts.base)
1758 : {
1759 7108 : parts.base = gen_lowpart (mode, parts.base);
1760 7108 : regno1 = true_regnum (parts.base);
1761 : }
1762 :
1763 7115 : if (parts.index)
1764 : {
1765 7111 : parts.index = gen_lowpart (mode, parts.index);
1766 7111 : regno2 = true_regnum (parts.index);
1767 : }
1768 :
1769 7115 : if (parts.disp)
1770 195 : parts.disp = gen_lowpart (mode, parts.disp);
1771 :
1772 7115 : if (parts.scale > 1)
1773 : {
1774 : /* Case r1 = r1 + ... */
1775 11 : if (regno1 == regno0)
1776 : {
1777 : /* If we have a case r1 = r1 + C * r2 then we
1778 : should use multiplication which is very
1779 : expensive. Assume cost model is wrong if we
1780 : have such case here. */
1781 0 : gcc_assert (regno2 != regno0);
1782 :
1783 0 : for (adds = parts.scale; adds > 0; adds--)
1784 0 : ix86_emit_binop (PLUS, mode, target, parts.index);
1785 : }
1786 : else
1787 : {
1788 : /* r1 = r2 + r3 * C case. Need to move r3 into r1. */
1789 11 : if (regno0 != regno2)
1790 8 : emit_insn (gen_rtx_SET (target, parts.index));
1791 :
1792 : /* Use shift for scaling, but emit it as MULT instead
1793 : to avoid it being immediately peephole2 optimized back
1794 : into lea. */
1795 11 : ix86_emit_binop (MULT, mode, target, GEN_INT (parts.scale));
1796 :
1797 11 : if (parts.base)
1798 4 : ix86_emit_binop (PLUS, mode, target, parts.base);
1799 :
1800 11 : if (parts.disp && parts.disp != const0_rtx)
1801 2 : ix86_emit_binop (PLUS, mode, target, parts.disp);
1802 : }
1803 : }
1804 7104 : else if (!parts.base && !parts.index)
1805 : {
1806 0 : gcc_assert(parts.disp);
1807 0 : emit_insn (gen_rtx_SET (target, parts.disp));
1808 : }
1809 : else
1810 : {
1811 7104 : if (!parts.base)
1812 : {
1813 0 : if (regno0 != regno2)
1814 0 : emit_insn (gen_rtx_SET (target, parts.index));
1815 : }
1816 7104 : else if (!parts.index)
1817 : {
1818 4 : if (regno0 != regno1)
1819 2 : emit_insn (gen_rtx_SET (target, parts.base));
1820 : }
1821 : else
1822 : {
1823 7100 : if (regno0 == regno1)
1824 : tmp = parts.index;
1825 3684 : else if (regno0 == regno2)
1826 : tmp = parts.base;
1827 : else
1828 : {
1829 17 : rtx tmp1;
1830 :
1831 : /* Find better operand for SET instruction, depending
1832 : on which definition is farther from the insn. */
1833 17 : if (find_nearest_reg_def (insn, regno1, regno2))
1834 7 : tmp = parts.index, tmp1 = parts.base;
1835 : else
1836 10 : tmp = parts.base, tmp1 = parts.index;
1837 :
1838 17 : emit_insn (gen_rtx_SET (target, tmp));
1839 :
1840 17 : if (parts.disp && parts.disp != const0_rtx)
1841 0 : ix86_emit_binop (PLUS, mode, target, parts.disp);
1842 :
1843 17 : ix86_emit_binop (PLUS, mode, target, tmp1);
1844 17 : return;
1845 : }
1846 :
1847 7083 : ix86_emit_binop (PLUS, mode, target, tmp);
1848 : }
1849 :
1850 7087 : if (parts.disp && parts.disp != const0_rtx)
1851 5 : ix86_emit_binop (PLUS, mode, target, parts.disp);
1852 : }
1853 : }
1854 :
1855 : /* Post-reload splitter for converting an SF or DFmode value in an
1856 : SSE register into an unsigned SImode. */
1857 :
1858 : void
1859 0 : ix86_split_convert_uns_si_sse (rtx operands[])
1860 : {
1861 0 : machine_mode vecmode;
1862 0 : rtx value, large, zero_or_two31, input, two31, x;
1863 :
1864 0 : large = operands[1];
1865 0 : zero_or_two31 = operands[2];
1866 0 : input = operands[3];
1867 0 : two31 = operands[4];
1868 0 : vecmode = GET_MODE (large);
1869 0 : value = gen_rtx_REG (vecmode, REGNO (operands[0]));
1870 :
1871 : /* Load up the value into the low element. We must ensure that the other
1872 : elements are valid floats -- zero is the easiest such value. */
1873 0 : if (MEM_P (input))
1874 : {
1875 0 : if (vecmode == V4SFmode)
1876 0 : emit_insn (gen_vec_setv4sf_0 (value, CONST0_RTX (V4SFmode), input));
1877 : else
1878 0 : emit_insn (gen_sse2_loadlpd (value, CONST0_RTX (V2DFmode), input));
1879 : }
1880 : else
1881 : {
1882 0 : input = gen_rtx_REG (vecmode, REGNO (input));
1883 0 : emit_move_insn (value, CONST0_RTX (vecmode));
1884 0 : if (vecmode == V4SFmode)
1885 0 : emit_insn (gen_sse_movss_v4sf (value, value, input));
1886 : else
1887 0 : emit_insn (gen_sse2_movsd_v2df (value, value, input));
1888 : }
1889 :
1890 0 : emit_move_insn (large, two31);
1891 0 : emit_move_insn (zero_or_two31, MEM_P (two31) ? large : two31);
1892 :
1893 0 : x = gen_rtx_fmt_ee (LE, vecmode, large, value);
1894 0 : emit_insn (gen_rtx_SET (large, x));
1895 :
1896 0 : x = gen_rtx_AND (vecmode, zero_or_two31, large);
1897 0 : emit_insn (gen_rtx_SET (zero_or_two31, x));
1898 :
1899 0 : x = gen_rtx_MINUS (vecmode, value, zero_or_two31);
1900 0 : emit_insn (gen_rtx_SET (value, x));
1901 :
1902 0 : large = gen_rtx_REG (V4SImode, REGNO (large));
1903 0 : emit_insn (gen_ashlv4si3 (large, large, GEN_INT (31)));
1904 :
1905 0 : x = gen_rtx_REG (V4SImode, REGNO (value));
1906 0 : if (vecmode == V4SFmode)
1907 0 : emit_insn (gen_fix_truncv4sfv4si2 (x, value));
1908 : else
1909 0 : emit_insn (gen_sse2_cvttpd2dq (x, value));
1910 0 : value = x;
1911 :
1912 0 : emit_insn (gen_xorv4si3 (value, value, large));
1913 0 : }
1914 :
1915 : /* Convert an unsigned DImode value into a DFmode, using only SSE.
1916 : Expects the 64-bit DImode to be supplied in a pair of integral
1917 : registers. Requires SSE2; will use SSE3 if available. For x86_32,
1918 : -mfpmath=sse, !optimize_size only. */
1919 :
1920 : void
1921 0 : ix86_expand_convert_uns_didf_sse (rtx target, rtx input)
1922 : {
1923 0 : REAL_VALUE_TYPE bias_lo_rvt, bias_hi_rvt;
1924 0 : rtx int_xmm, fp_xmm;
1925 0 : rtx biases, exponents;
1926 0 : rtx x;
1927 :
1928 0 : int_xmm = gen_reg_rtx (V4SImode);
1929 0 : if (TARGET_INTER_UNIT_MOVES_TO_VEC)
1930 0 : emit_insn (gen_movdi_to_sse (int_xmm, input));
1931 0 : else if (TARGET_SSE_SPLIT_REGS)
1932 : {
1933 0 : emit_clobber (int_xmm);
1934 0 : emit_move_insn (gen_lowpart (DImode, int_xmm), input);
1935 : }
1936 : else
1937 : {
1938 0 : x = gen_reg_rtx (V2DImode);
1939 0 : ix86_expand_vector_init_one_nonzero (false, V2DImode, x, input, 0);
1940 0 : emit_move_insn (int_xmm, gen_lowpart (V4SImode, x));
1941 : }
1942 :
1943 0 : x = gen_rtx_CONST_VECTOR (V4SImode,
1944 : gen_rtvec (4, GEN_INT (0x43300000UL),
1945 : GEN_INT (0x45300000UL),
1946 : const0_rtx, const0_rtx));
1947 0 : exponents = validize_mem (force_const_mem (V4SImode, x));
1948 :
1949 : /* int_xmm = {0x45300000UL, fp_xmm/hi, 0x43300000, fp_xmm/lo } */
1950 0 : emit_insn (gen_vec_interleave_lowv4si (int_xmm, int_xmm, exponents));
1951 :
1952 : /* Concatenating (juxtaposing) (0x43300000UL ## fp_value_low_xmm)
1953 : yields a valid DF value equal to (0x1.0p52 + double(fp_value_lo_xmm)).
1954 : Similarly (0x45300000UL ## fp_value_hi_xmm) yields
1955 : (0x1.0p84 + double(fp_value_hi_xmm)).
1956 : Note these exponents differ by 32. */
1957 :
1958 0 : fp_xmm = copy_to_mode_reg (V2DFmode, gen_lowpart (V2DFmode, int_xmm));
1959 :
1960 : /* Subtract off those 0x1.0p52 and 0x1.0p84 biases, to produce values
1961 : in [0,2**32-1] and [0]+[2**32,2**64-1] respectively. */
1962 0 : real_ldexp (&bias_lo_rvt, &dconst1, 52);
1963 0 : real_ldexp (&bias_hi_rvt, &dconst1, 84);
1964 0 : biases = const_double_from_real_value (bias_lo_rvt, DFmode);
1965 0 : x = const_double_from_real_value (bias_hi_rvt, DFmode);
1966 0 : biases = gen_rtx_CONST_VECTOR (V2DFmode, gen_rtvec (2, biases, x));
1967 0 : biases = validize_mem (force_const_mem (V2DFmode, biases));
1968 0 : emit_insn (gen_subv2df3 (fp_xmm, fp_xmm, biases));
1969 :
1970 : /* Add the upper and lower DFmode values together. */
1971 0 : if (TARGET_SSE3)
1972 0 : emit_insn (gen_sse3_haddv2df3 (fp_xmm, fp_xmm, fp_xmm));
1973 : else
1974 : {
1975 0 : x = copy_to_mode_reg (V2DFmode, fp_xmm);
1976 0 : emit_insn (gen_vec_interleave_highv2df (fp_xmm, fp_xmm, fp_xmm));
1977 0 : emit_insn (gen_addv2df3 (fp_xmm, fp_xmm, x));
1978 : }
1979 :
1980 0 : ix86_expand_vector_extract (false, target, fp_xmm, 0);
1981 0 : }
1982 :
1983 : /* Not used, but eases macroization of patterns. */
1984 : void
1985 0 : ix86_expand_convert_uns_sixf_sse (rtx, rtx)
1986 : {
1987 0 : gcc_unreachable ();
1988 : }
1989 :
1990 : static rtx ix86_expand_sse_fabs (rtx op0, rtx *smask);
1991 :
1992 : /* Convert an unsigned SImode value into a DFmode. Only currently used
1993 : for SSE, but applicable anywhere. */
1994 :
1995 : void
1996 0 : ix86_expand_convert_uns_sidf_sse (rtx target, rtx input)
1997 : {
1998 0 : REAL_VALUE_TYPE TWO31r;
1999 0 : rtx x, fp;
2000 :
2001 0 : x = expand_simple_binop (SImode, PLUS, input, GEN_INT (-2147483647 - 1),
2002 : NULL, 1, OPTAB_DIRECT);
2003 :
2004 0 : fp = gen_reg_rtx (DFmode);
2005 0 : emit_insn (gen_floatsidf2 (fp, x));
2006 :
2007 0 : real_ldexp (&TWO31r, &dconst1, 31);
2008 0 : x = const_double_from_real_value (TWO31r, DFmode);
2009 :
2010 0 : x = expand_simple_binop (DFmode, PLUS, fp, x, target, 0, OPTAB_DIRECT);
2011 :
2012 : /* Remove the sign with FE_DOWNWARD, where x - x = -0.0. */
2013 0 : if (HONOR_SIGNED_ZEROS (DFmode) && flag_rounding_math)
2014 0 : x = ix86_expand_sse_fabs (x, NULL);
2015 :
2016 0 : if (x != target)
2017 0 : emit_move_insn (target, x);
2018 0 : }
2019 :
2020 : /* Convert a signed DImode value into a DFmode. Only used for SSE in
2021 : 32-bit mode; otherwise we have a direct convert instruction. */
2022 :
2023 : void
2024 0 : ix86_expand_convert_sign_didf_sse (rtx target, rtx input)
2025 : {
2026 0 : REAL_VALUE_TYPE TWO32r;
2027 0 : rtx fp_lo, fp_hi, x;
2028 :
2029 0 : fp_lo = gen_reg_rtx (DFmode);
2030 0 : fp_hi = gen_reg_rtx (DFmode);
2031 :
2032 0 : emit_insn (gen_floatsidf2 (fp_hi, gen_highpart (SImode, input)));
2033 :
2034 0 : real_ldexp (&TWO32r, &dconst1, 32);
2035 0 : x = const_double_from_real_value (TWO32r, DFmode);
2036 0 : fp_hi = expand_simple_binop (DFmode, MULT, fp_hi, x, fp_hi, 0, OPTAB_DIRECT);
2037 :
2038 0 : ix86_expand_convert_uns_sidf_sse (fp_lo, gen_lowpart (SImode, input));
2039 :
2040 0 : x = expand_simple_binop (DFmode, PLUS, fp_hi, fp_lo, target,
2041 : 0, OPTAB_DIRECT);
2042 0 : if (x != target)
2043 0 : emit_move_insn (target, x);
2044 0 : }
2045 :
2046 : /* Convert an unsigned SImode value into a SFmode, using only SSE.
2047 : For x86_32, -mfpmath=sse, !optimize_size only. */
2048 : void
2049 0 : ix86_expand_convert_uns_sisf_sse (rtx target, rtx input)
2050 : {
2051 0 : REAL_VALUE_TYPE ONE16r;
2052 0 : rtx fp_hi, fp_lo, int_hi, int_lo, x;
2053 :
2054 0 : real_ldexp (&ONE16r, &dconst1, 16);
2055 0 : x = const_double_from_real_value (ONE16r, SFmode);
2056 0 : int_lo = expand_simple_binop (SImode, AND, input, GEN_INT(0xffff),
2057 : NULL, 0, OPTAB_DIRECT);
2058 0 : int_hi = expand_simple_binop (SImode, LSHIFTRT, input, GEN_INT(16),
2059 : NULL, 0, OPTAB_DIRECT);
2060 0 : fp_hi = gen_reg_rtx (SFmode);
2061 0 : fp_lo = gen_reg_rtx (SFmode);
2062 0 : emit_insn (gen_floatsisf2 (fp_hi, int_hi));
2063 0 : emit_insn (gen_floatsisf2 (fp_lo, int_lo));
2064 0 : if (TARGET_FMA)
2065 : {
2066 0 : x = validize_mem (force_const_mem (SFmode, x));
2067 0 : fp_hi = gen_rtx_FMA (SFmode, fp_hi, x, fp_lo);
2068 0 : emit_move_insn (target, fp_hi);
2069 : }
2070 : else
2071 : {
2072 0 : fp_hi = expand_simple_binop (SFmode, MULT, fp_hi, x, fp_hi,
2073 : 0, OPTAB_DIRECT);
2074 0 : fp_hi = expand_simple_binop (SFmode, PLUS, fp_hi, fp_lo, target,
2075 : 0, OPTAB_DIRECT);
2076 0 : if (!rtx_equal_p (target, fp_hi))
2077 0 : emit_move_insn (target, fp_hi);
2078 : }
2079 0 : }
2080 :
2081 : /* floatunsv{4,8}siv{4,8}sf2 expander. Expand code to convert
2082 : a vector of unsigned ints VAL to vector of floats TARGET. */
2083 :
2084 : void
2085 48 : ix86_expand_vector_convert_uns_vsivsf (rtx target, rtx val)
2086 : {
2087 48 : rtx tmp[8];
2088 48 : REAL_VALUE_TYPE TWO16r;
2089 48 : machine_mode intmode = GET_MODE (val);
2090 48 : machine_mode fltmode = GET_MODE (target);
2091 48 : rtx (*cvt) (rtx, rtx);
2092 :
2093 48 : if (intmode == V4SImode)
2094 : cvt = gen_floatv4siv4sf2;
2095 : else
2096 2 : cvt = gen_floatv8siv8sf2;
2097 48 : tmp[0] = ix86_build_const_vector (intmode, 1, GEN_INT (0xffff));
2098 48 : tmp[0] = force_reg (intmode, tmp[0]);
2099 48 : tmp[1] = expand_simple_binop (intmode, AND, val, tmp[0], NULL_RTX, 1,
2100 : OPTAB_DIRECT);
2101 48 : tmp[2] = expand_simple_binop (intmode, LSHIFTRT, val, GEN_INT (16),
2102 : NULL_RTX, 1, OPTAB_DIRECT);
2103 48 : tmp[3] = gen_reg_rtx (fltmode);
2104 48 : emit_insn (cvt (tmp[3], tmp[1]));
2105 48 : tmp[4] = gen_reg_rtx (fltmode);
2106 48 : emit_insn (cvt (tmp[4], tmp[2]));
2107 48 : real_ldexp (&TWO16r, &dconst1, 16);
2108 48 : tmp[5] = const_double_from_real_value (TWO16r, SFmode);
2109 48 : tmp[5] = force_reg (fltmode, ix86_build_const_vector (fltmode, 1, tmp[5]));
2110 48 : if (TARGET_FMA)
2111 : {
2112 1 : tmp[6] = gen_rtx_FMA (fltmode, tmp[4], tmp[5], tmp[3]);
2113 1 : emit_move_insn (target, tmp[6]);
2114 : }
2115 : else
2116 : {
2117 47 : tmp[6] = expand_simple_binop (fltmode, MULT, tmp[4], tmp[5],
2118 : NULL_RTX, 1, OPTAB_DIRECT);
2119 47 : tmp[7] = expand_simple_binop (fltmode, PLUS, tmp[3], tmp[6],
2120 : target, 1, OPTAB_DIRECT);
2121 47 : if (tmp[7] != target)
2122 0 : emit_move_insn (target, tmp[7]);
2123 : }
2124 48 : }
2125 :
2126 : /* Adjust a V*SFmode/V*DFmode value VAL so that *sfix_trunc* resp. fix_trunc*
2127 : pattern can be used on it instead of fixuns_trunc*.
2128 : This is done by doing just signed conversion if < 0x1p31, and otherwise by
2129 : subtracting 0x1p31 first and xoring in 0x80000000 from *XORP afterwards. */
2130 :
2131 : rtx
2132 326 : ix86_expand_adjust_ufix_to_sfix_si (rtx val, rtx *xorp)
2133 : {
2134 326 : REAL_VALUE_TYPE TWO31r;
2135 326 : rtx two31r, tmp[4];
2136 326 : machine_mode mode = GET_MODE (val);
2137 326 : machine_mode scalarmode = GET_MODE_INNER (mode);
2138 652 : machine_mode intmode = GET_MODE_SIZE (mode) == 32 ? V8SImode : V4SImode;
2139 326 : rtx (*cmp) (rtx, rtx, rtx, rtx);
2140 326 : int i;
2141 :
2142 1304 : for (i = 0; i < 3; i++)
2143 978 : tmp[i] = gen_reg_rtx (mode);
2144 326 : real_ldexp (&TWO31r, &dconst1, 31);
2145 326 : two31r = const_double_from_real_value (TWO31r, scalarmode);
2146 326 : two31r = ix86_build_const_vector (mode, 1, two31r);
2147 326 : two31r = force_reg (mode, two31r);
2148 326 : switch (mode)
2149 : {
2150 : case E_V8SFmode: cmp = gen_avx_maskcmpv8sf3; break;
2151 10 : case E_V4SFmode: cmp = gen_sse_maskcmpv4sf3; break;
2152 16 : case E_V4DFmode: cmp = gen_avx_maskcmpv4df3; break;
2153 300 : case E_V2DFmode: cmp = gen_sse2_maskcmpv2df3; break;
2154 0 : default: gcc_unreachable ();
2155 : }
2156 326 : tmp[3] = gen_rtx_LE (mode, two31r, val);
2157 326 : emit_insn (cmp (tmp[0], two31r, val, tmp[3]));
2158 326 : tmp[1] = expand_simple_binop (mode, AND, tmp[0], two31r, tmp[1],
2159 : 0, OPTAB_DIRECT);
2160 326 : if (intmode == V4SImode || TARGET_AVX2)
2161 652 : *xorp = expand_simple_binop (intmode, ASHIFT,
2162 326 : gen_lowpart (intmode, tmp[0]),
2163 : GEN_INT (31), NULL_RTX, 0,
2164 : OPTAB_DIRECT);
2165 : else
2166 : {
2167 0 : rtx two31 = gen_int_mode (HOST_WIDE_INT_1U << 31, SImode);
2168 0 : two31 = ix86_build_const_vector (intmode, 1, two31);
2169 0 : *xorp = expand_simple_binop (intmode, AND,
2170 0 : gen_lowpart (intmode, tmp[0]),
2171 : two31, NULL_RTX, 0,
2172 : OPTAB_DIRECT);
2173 : }
2174 326 : return expand_simple_binop (mode, MINUS, val, tmp[1], tmp[2],
2175 326 : 0, OPTAB_DIRECT);
2176 : }
2177 :
2178 : /* Generate code for floating point ABS or NEG. */
2179 :
2180 : void
2181 35272 : ix86_expand_fp_absneg_operator (enum rtx_code code, machine_mode mode,
2182 : rtx operands[])
2183 : {
2184 35272 : rtx set, dst, src;
2185 35272 : bool use_sse = false;
2186 35272 : bool vector_mode = VECTOR_MODE_P (mode);
2187 35272 : machine_mode vmode = mode;
2188 35272 : rtvec par;
2189 :
2190 35272 : switch (mode)
2191 : {
2192 : case E_HFmode:
2193 : use_sse = true;
2194 : vmode = V8HFmode;
2195 : break;
2196 0 : case E_BFmode:
2197 0 : use_sse = true;
2198 0 : vmode = V8BFmode;
2199 0 : break;
2200 9505 : case E_SFmode:
2201 9505 : use_sse = TARGET_SSE_MATH && TARGET_SSE;
2202 : vmode = V4SFmode;
2203 : break;
2204 16223 : case E_DFmode:
2205 16223 : use_sse = TARGET_SSE_MATH && TARGET_SSE2;
2206 : vmode = V2DFmode;
2207 : break;
2208 9343 : default:
2209 9343 : use_sse = vector_mode || mode == TFmode;
2210 9343 : break;
2211 : }
2212 :
2213 35272 : dst = operands[0];
2214 35272 : src = operands[1];
2215 :
2216 35272 : set = gen_rtx_fmt_e (code, mode, src);
2217 35272 : set = gen_rtx_SET (dst, set);
2218 :
2219 35272 : if (use_sse)
2220 : {
2221 29522 : rtx mask, use, clob;
2222 :
2223 : /* NEG and ABS performed with SSE use bitwise mask operations.
2224 : Create the appropriate mask now. */
2225 29522 : mask = ix86_build_signbit_mask (vmode, vector_mode, code == ABS);
2226 29522 : use = gen_rtx_USE (VOIDmode, mask);
2227 29522 : if (vector_mode || mode == TFmode)
2228 4968 : par = gen_rtvec (2, set, use);
2229 : else
2230 : {
2231 24554 : clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
2232 24554 : par = gen_rtvec (3, set, use, clob);
2233 : }
2234 : }
2235 : else
2236 : {
2237 5750 : rtx clob;
2238 :
2239 : /* Changing of sign for FP values is doable using integer unit too. */
2240 5750 : clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
2241 5750 : par = gen_rtvec (2, set, clob);
2242 : }
2243 :
2244 35272 : emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
2245 35272 : }
2246 :
2247 : /* Deconstruct a floating point ABS or NEG operation
2248 : with integer registers into integer operations. */
2249 :
2250 : void
2251 24 : ix86_split_fp_absneg_operator (enum rtx_code code, machine_mode mode,
2252 : rtx operands[])
2253 : {
2254 24 : enum rtx_code absneg_op;
2255 24 : rtx dst, set;
2256 :
2257 24 : gcc_assert (operands_match_p (operands[0], operands[1]));
2258 :
2259 24 : switch (mode)
2260 : {
2261 0 : case E_SFmode:
2262 0 : dst = gen_lowpart (SImode, operands[0]);
2263 :
2264 0 : if (code == ABS)
2265 : {
2266 0 : set = gen_int_mode (0x7fffffff, SImode);
2267 0 : absneg_op = AND;
2268 : }
2269 : else
2270 : {
2271 0 : set = gen_int_mode (0x80000000, SImode);
2272 0 : absneg_op = XOR;
2273 : }
2274 0 : set = gen_rtx_fmt_ee (absneg_op, SImode, dst, set);
2275 0 : break;
2276 :
2277 1 : case E_DFmode:
2278 1 : if (TARGET_64BIT)
2279 : {
2280 1 : dst = gen_lowpart (DImode, operands[0]);
2281 1 : dst = gen_rtx_ZERO_EXTRACT (DImode, dst, const1_rtx, GEN_INT (63));
2282 :
2283 1 : if (code == ABS)
2284 0 : set = const0_rtx;
2285 : else
2286 1 : set = gen_rtx_NOT (DImode, dst);
2287 : }
2288 : else
2289 : {
2290 0 : dst = gen_highpart (SImode, operands[0]);
2291 :
2292 0 : if (code == ABS)
2293 : {
2294 0 : set = gen_int_mode (0x7fffffff, SImode);
2295 0 : absneg_op = AND;
2296 : }
2297 : else
2298 : {
2299 0 : set = gen_int_mode (0x80000000, SImode);
2300 0 : absneg_op = XOR;
2301 : }
2302 0 : set = gen_rtx_fmt_ee (absneg_op, SImode, dst, set);
2303 : }
2304 : break;
2305 :
2306 23 : case E_XFmode:
2307 23 : dst = gen_rtx_REG (SImode,
2308 23 : REGNO (operands[0]) + (TARGET_64BIT ? 1 : 2));
2309 23 : if (code == ABS)
2310 : {
2311 1 : set = GEN_INT (0x7fff);
2312 1 : absneg_op = AND;
2313 : }
2314 : else
2315 : {
2316 22 : set = GEN_INT (0x8000);
2317 22 : absneg_op = XOR;
2318 : }
2319 23 : set = gen_rtx_fmt_ee (absneg_op, SImode, dst, set);
2320 23 : break;
2321 :
2322 0 : default:
2323 0 : gcc_unreachable ();
2324 : }
2325 :
2326 24 : set = gen_rtx_SET (dst, set);
2327 :
2328 24 : rtx clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
2329 24 : rtvec par = gen_rtvec (2, set, clob);
2330 :
2331 24 : emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
2332 24 : }
2333 :
2334 : /* Expand a copysign operation. Special case operand 0 being a constant. */
2335 :
2336 : void
2337 23278 : ix86_expand_copysign (rtx operands[])
2338 : {
2339 23278 : machine_mode mode, vmode;
2340 23278 : rtx dest, vdest, op0, op1, mask, op2, op3;
2341 :
2342 23278 : mode = GET_MODE (operands[0]);
2343 :
2344 23278 : switch (mode)
2345 : {
2346 : case E_HFmode:
2347 : vmode = V8HFmode;
2348 : break;
2349 0 : case E_BFmode:
2350 0 : vmode = V8BFmode;
2351 0 : break;
2352 11571 : case E_SFmode:
2353 11571 : vmode = V4SFmode;
2354 11571 : break;
2355 11567 : case E_DFmode:
2356 11567 : vmode = V2DFmode;
2357 11567 : break;
2358 127 : case E_TFmode:
2359 127 : vmode = mode;
2360 127 : break;
2361 0 : default:
2362 0 : gcc_unreachable();
2363 : }
2364 :
2365 23278 : if (rtx_equal_p (operands[1], operands[2]))
2366 : {
2367 0 : emit_move_insn (operands[0], operands[1]);
2368 0 : return;
2369 : }
2370 :
2371 23278 : dest = operands[0];
2372 23278 : vdest = lowpart_subreg (vmode, dest, mode);
2373 23278 : if (vdest == NULL_RTX)
2374 0 : vdest = gen_reg_rtx (vmode);
2375 : else
2376 : dest = NULL_RTX;
2377 23278 : op1 = lowpart_subreg (vmode, force_reg (mode, operands[1]), mode);
2378 46542 : mask = ix86_build_signbit_mask (vmode, TARGET_AVX512F && mode != HFmode, 0);
2379 :
2380 23278 : if (CONST_DOUBLE_P (operands[2]))
2381 : {
2382 88 : if (real_isneg (CONST_DOUBLE_REAL_VALUE (operands[2])))
2383 : /* Simplify b = copysign (a, negative) to b = mask | a. */
2384 85 : op1 = gen_rtx_IOR (vmode, mask, op1);
2385 : else
2386 : {
2387 : /* Simplify b = copysign (a, positive) to b = invert_mask & a. */
2388 3 : rtx invert_mask
2389 3 : = ix86_build_signbit_mask (vmode,
2390 3 : TARGET_AVX512F && mode != HFmode,
2391 : true);
2392 3 : op1 = gen_rtx_AND (vmode, invert_mask, op1);
2393 : }
2394 88 : emit_move_insn (vdest, op1);
2395 88 : if (dest)
2396 0 : emit_move_insn (dest, lowpart_subreg (mode, vdest, vmode));
2397 : return;
2398 : }
2399 : else
2400 23190 : op0 = lowpart_subreg (vmode, force_reg (mode, operands[2]), mode);
2401 :
2402 23190 : op2 = gen_reg_rtx (vmode);
2403 23190 : op3 = gen_reg_rtx (vmode);
2404 23190 : rtx invert_mask;
2405 : /* NB: Generate vmovdqa, vpandn, vpand, vpor for AVX and generate pand,
2406 : pand, por for SSE. */
2407 23190 : if (TARGET_AVX)
2408 32 : invert_mask = gen_rtx_NOT (vmode, mask);
2409 : else
2410 46316 : invert_mask = ix86_build_signbit_mask (vmode,
2411 : TARGET_AVX512F && mode != HFmode,
2412 : true);
2413 23190 : emit_move_insn (op2, gen_rtx_AND (vmode, invert_mask, op1));
2414 23190 : emit_move_insn (op3, gen_rtx_AND (vmode, mask, op0));
2415 23190 : emit_move_insn (vdest, gen_rtx_IOR (vmode, op2, op3));
2416 23190 : if (dest)
2417 0 : emit_move_insn (dest, lowpart_subreg (mode, vdest, vmode));
2418 : }
2419 :
2420 : /* Expand an xorsign operation. */
2421 :
2422 : void
2423 20 : ix86_expand_xorsign (rtx operands[])
2424 : {
2425 20 : machine_mode mode, vmode;
2426 20 : rtx dest, vdest, op0, op1, mask, x, temp;
2427 :
2428 20 : dest = operands[0];
2429 20 : op0 = operands[1];
2430 20 : op1 = operands[2];
2431 :
2432 20 : mode = GET_MODE (dest);
2433 :
2434 20 : switch (mode)
2435 : {
2436 : case E_HFmode:
2437 : vmode = V8HFmode;
2438 : break;
2439 : case E_BFmode:
2440 : vmode = V8BFmode;
2441 : break;
2442 : case E_SFmode:
2443 : vmode = V4SFmode;
2444 : break;
2445 : case E_DFmode:
2446 : vmode = V2DFmode;
2447 : break;
2448 0 : default:
2449 0 : gcc_unreachable ();
2450 20 : break;
2451 : }
2452 :
2453 20 : temp = gen_reg_rtx (vmode);
2454 20 : mask = ix86_build_signbit_mask (vmode, 0, 0);
2455 :
2456 20 : op1 = lowpart_subreg (vmode, force_reg (mode, op1), mode);
2457 20 : x = gen_rtx_AND (vmode, op1, mask);
2458 20 : emit_insn (gen_rtx_SET (temp, x));
2459 :
2460 20 : op0 = lowpart_subreg (vmode, force_reg (mode, op0), mode);
2461 20 : x = gen_rtx_XOR (vmode, temp, op0);
2462 :
2463 20 : vdest = lowpart_subreg (vmode, dest, mode);
2464 20 : if (vdest == NULL_RTX)
2465 0 : vdest = gen_reg_rtx (vmode);
2466 : else
2467 : dest = NULL_RTX;
2468 20 : emit_insn (gen_rtx_SET (vdest, x));
2469 :
2470 20 : if (dest)
2471 0 : emit_move_insn (dest, lowpart_subreg (mode, vdest, vmode));
2472 20 : }
2473 :
2474 : static rtx ix86_expand_compare (enum rtx_code code, rtx op0, rtx op1);
2475 :
2476 : void
2477 6767784 : ix86_expand_branch (enum rtx_code code, rtx op0, rtx op1, rtx label)
2478 : {
2479 6767784 : machine_mode mode = GET_MODE (op0);
2480 6767784 : rtx tmp;
2481 :
2482 : /* Handle special case - vector comparison with boolean result, transform
2483 : it using ptest instruction or vpcmpeq + kortest. */
2484 6767784 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
2485 6748362 : || (mode == TImode && !TARGET_64BIT)
2486 6748362 : || mode == OImode
2487 13516146 : || GET_MODE_SIZE (mode) == 64)
2488 : {
2489 19422 : unsigned msize = GET_MODE_SIZE (mode);
2490 19422 : machine_mode p_mode
2491 19422 : = msize == 64 ? V16SImode : msize == 32 ? V4DImode : V2DImode;
2492 : /* kortest set CF when result is 0xFFFF (op0 == op1). */
2493 19422 : rtx flag = gen_rtx_REG (msize == 64 ? CCCmode : CCZmode, FLAGS_REG);
2494 :
2495 19422 : gcc_assert (code == EQ || code == NE);
2496 :
2497 : /* Using vpcmpeq zmm zmm k + kortest for 512-bit vectors. */
2498 19422 : if (msize == 64)
2499 : {
2500 2451 : if (mode != V16SImode)
2501 : {
2502 2451 : op0 = lowpart_subreg (p_mode, force_reg (mode, op0), mode);
2503 2451 : op1 = lowpart_subreg (p_mode, force_reg (mode, op1), mode);
2504 : }
2505 :
2506 2451 : tmp = gen_reg_rtx (HImode);
2507 2451 : emit_insn (gen_avx512f_cmpv16si3 (tmp, op0, op1, GEN_INT (0)));
2508 2451 : emit_insn (gen_kortesthi_ccc (tmp, tmp));
2509 : }
2510 : /* Using ptest for 128/256-bit vectors. */
2511 : else
2512 : {
2513 16971 : if (GET_MODE_CLASS (mode) != MODE_VECTOR_INT)
2514 : {
2515 0 : op0 = lowpart_subreg (p_mode, force_reg (mode, op0), mode);
2516 0 : op1 = lowpart_subreg (p_mode, force_reg (mode, op1), mode);
2517 0 : mode = p_mode;
2518 : }
2519 :
2520 : /* Generate XOR since we can't check that one operand is zero
2521 : vector. */
2522 16971 : tmp = gen_reg_rtx (mode);
2523 16971 : rtx ops[3] = { tmp, op0, op1 };
2524 16971 : ix86_expand_vector_logical_operator (XOR, mode, ops);
2525 16971 : tmp = gen_lowpart (p_mode, tmp);
2526 16971 : emit_insn (gen_rtx_SET (gen_rtx_REG (CCZmode, FLAGS_REG),
2527 : gen_rtx_UNSPEC (CCZmode,
2528 : gen_rtvec (2, tmp, tmp),
2529 : UNSPEC_PTEST)));
2530 : }
2531 19422 : tmp = gen_rtx_fmt_ee (code, VOIDmode, flag, const0_rtx);
2532 19422 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
2533 : gen_rtx_LABEL_REF (VOIDmode, label),
2534 : pc_rtx);
2535 19422 : emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
2536 19422 : return;
2537 : }
2538 :
2539 6748362 : switch (mode)
2540 : {
2541 6713689 : case E_HFmode:
2542 6713689 : case E_SFmode:
2543 6713689 : case E_DFmode:
2544 6713689 : case E_XFmode:
2545 6713689 : case E_QImode:
2546 6713689 : case E_HImode:
2547 6713689 : case E_SImode:
2548 6713689 : simple:
2549 6713689 : tmp = ix86_expand_compare (code, op0, op1);
2550 6713689 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
2551 : gen_rtx_LABEL_REF (VOIDmode, label),
2552 : pc_rtx);
2553 6713689 : emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
2554 6713689 : return;
2555 :
2556 7 : case E_BFmode:
2557 7 : gcc_assert (TARGET_AVX10_2 && !flag_trapping_math);
2558 7 : goto simple;
2559 :
2560 2718432 : case E_DImode:
2561 2718432 : if (TARGET_64BIT)
2562 2687754 : goto simple;
2563 : /* FALLTHRU */
2564 93911 : case E_TImode:
2565 : /* DI and TI mode equality/inequality comparisons may be performed
2566 : on SSE registers. Avoid splitting them, except when optimizing
2567 : for size. */
2568 93911 : if ((code == EQ || code == NE)
2569 93911 : && !optimize_insn_for_size_p ())
2570 59238 : goto simple;
2571 :
2572 : /* Expand DImode branch into multiple compare+branch. */
2573 34673 : {
2574 34673 : rtx lo[2], hi[2];
2575 34673 : rtx_code_label *label2;
2576 34673 : enum rtx_code code1, code2, code3;
2577 34673 : machine_mode submode;
2578 :
2579 34673 : if (CONSTANT_P (op0) && !CONSTANT_P (op1))
2580 : {
2581 0 : std::swap (op0, op1);
2582 0 : code = swap_condition (code);
2583 : }
2584 :
2585 34673 : split_double_mode (mode, &op0, 1, lo+0, hi+0);
2586 34673 : split_double_mode (mode, &op1, 1, lo+1, hi+1);
2587 :
2588 34673 : submode = mode == DImode ? SImode : DImode;
2589 :
2590 : /* If we are doing less-than or greater-or-equal-than,
2591 : op1 is a constant and the low word is zero, then we can just
2592 : examine the high word. Similarly for low word -1 and
2593 : less-or-equal-than or greater-than. */
2594 :
2595 34673 : if (CONST_INT_P (hi[1]))
2596 21070 : switch (code)
2597 : {
2598 11426 : case LT: case LTU: case GE: case GEU:
2599 11426 : if (lo[1] == const0_rtx)
2600 : {
2601 10993 : ix86_expand_branch (code, hi[0], hi[1], label);
2602 10993 : return;
2603 : }
2604 : break;
2605 8098 : case LE: case LEU: case GT: case GTU:
2606 8098 : if (lo[1] == constm1_rtx)
2607 : {
2608 530 : ix86_expand_branch (code, hi[0], hi[1], label);
2609 530 : return;
2610 : }
2611 : break;
2612 : default:
2613 : break;
2614 : }
2615 :
2616 : /* Emulate comparisons that do not depend on Zero flag with
2617 : double-word subtraction. Note that only Overflow, Sign
2618 : and Carry flags are valid, so swap arguments and condition
2619 : of comparisons that would otherwise test Zero flag. */
2620 :
2621 23150 : switch (code)
2622 : {
2623 12922 : case LE: case LEU: case GT: case GTU:
2624 12922 : std::swap (lo[0], lo[1]);
2625 12922 : std::swap (hi[0], hi[1]);
2626 12922 : code = swap_condition (code);
2627 : /* FALLTHRU */
2628 :
2629 19661 : case LT: case LTU: case GE: case GEU:
2630 19661 : {
2631 19661 : bool uns = (code == LTU || code == GEU);
2632 4113 : rtx (*sbb_insn) (machine_mode, rtx, rtx, rtx)
2633 19661 : = uns ? gen_sub3_carry_ccc : gen_sub3_carry_ccgz;
2634 :
2635 19661 : if (!nonimmediate_operand (lo[0], submode))
2636 7568 : lo[0] = force_reg (submode, lo[0]);
2637 19661 : if (!x86_64_general_operand (lo[1], submode))
2638 0 : lo[1] = force_reg (submode, lo[1]);
2639 :
2640 19661 : if (!register_operand (hi[0], submode))
2641 8353 : hi[0] = force_reg (submode, hi[0]);
2642 15548 : if ((uns && !nonimmediate_operand (hi[1], submode))
2643 19661 : || (!uns && !x86_64_general_operand (hi[1], submode)))
2644 315 : hi[1] = force_reg (submode, hi[1]);
2645 :
2646 19661 : emit_insn (gen_cmp_1 (submode, lo[0], lo[1]));
2647 :
2648 19661 : tmp = gen_rtx_SCRATCH (submode);
2649 19661 : emit_insn (sbb_insn (submode, tmp, hi[0], hi[1]));
2650 :
2651 23774 : tmp = gen_rtx_REG (uns ? CCCmode : CCGZmode, FLAGS_REG);
2652 19661 : ix86_expand_branch (code, tmp, const0_rtx, label);
2653 19661 : return;
2654 : }
2655 :
2656 3489 : default:
2657 3489 : break;
2658 : }
2659 :
2660 : /* Otherwise, we need two or three jumps. */
2661 :
2662 3489 : label2 = gen_label_rtx ();
2663 :
2664 3489 : code1 = code;
2665 3489 : code2 = swap_condition (code);
2666 3489 : code3 = unsigned_condition (code);
2667 :
2668 3489 : switch (code)
2669 : {
2670 : case LT: case GT: case LTU: case GTU:
2671 : break;
2672 :
2673 : case LE: code1 = LT; code2 = GT; break;
2674 : case GE: code1 = GT; code2 = LT; break;
2675 : case LEU: code1 = LTU; code2 = GTU; break;
2676 : case GEU: code1 = GTU; code2 = LTU; break;
2677 :
2678 : case EQ: code1 = UNKNOWN; code2 = NE; break;
2679 : case NE: code2 = UNKNOWN; break;
2680 :
2681 0 : default:
2682 0 : gcc_unreachable ();
2683 : }
2684 :
2685 : /*
2686 : * a < b =>
2687 : * if (hi(a) < hi(b)) goto true;
2688 : * if (hi(a) > hi(b)) goto false;
2689 : * if (lo(a) < lo(b)) goto true;
2690 : * false:
2691 : */
2692 :
2693 : if (code1 != UNKNOWN)
2694 2593 : ix86_expand_branch (code1, hi[0], hi[1], label);
2695 3489 : if (code2 != UNKNOWN)
2696 896 : ix86_expand_branch (code2, hi[0], hi[1], label2);
2697 :
2698 3489 : ix86_expand_branch (code3, lo[0], lo[1], label);
2699 :
2700 3489 : if (code2 != UNKNOWN)
2701 896 : emit_label (label2);
2702 : return;
2703 : }
2704 :
2705 20161 : default:
2706 20161 : gcc_assert (GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC);
2707 20161 : goto simple;
2708 : }
2709 : }
2710 :
2711 : /* Figure out whether to use unordered fp comparisons. */
2712 :
2713 : static bool
2714 1176230 : ix86_unordered_fp_compare (enum rtx_code code)
2715 : {
2716 1176230 : if (!TARGET_IEEE_FP)
2717 : return false;
2718 :
2719 1171646 : switch (code)
2720 : {
2721 : case LT:
2722 : case LE:
2723 : case GT:
2724 : case GE:
2725 : case LTGT:
2726 : return false;
2727 :
2728 : case EQ:
2729 : case NE:
2730 :
2731 : case UNORDERED:
2732 : case ORDERED:
2733 : case UNLT:
2734 : case UNLE:
2735 : case UNGT:
2736 : case UNGE:
2737 : case UNEQ:
2738 : return true;
2739 :
2740 0 : default:
2741 0 : gcc_unreachable ();
2742 : }
2743 : }
2744 :
2745 : /* Return a comparison we can do and that it is equivalent to
2746 : swap_condition (code) apart possibly from orderedness.
2747 : But, never change orderedness if TARGET_IEEE_FP, returning
2748 : UNKNOWN in that case if necessary. */
2749 :
2750 : static enum rtx_code
2751 39957 : ix86_fp_swap_condition (enum rtx_code code)
2752 : {
2753 39957 : switch (code)
2754 : {
2755 2865 : case GT: /* GTU - CF=0 & ZF=0 */
2756 2865 : return TARGET_IEEE_FP ? UNKNOWN : UNLT;
2757 544 : case GE: /* GEU - CF=0 */
2758 544 : return TARGET_IEEE_FP ? UNKNOWN : UNLE;
2759 524 : case UNLT: /* LTU - CF=1 */
2760 524 : return TARGET_IEEE_FP ? UNKNOWN : GT;
2761 6365 : case UNLE: /* LEU - CF=1 | ZF=1 */
2762 6365 : return TARGET_IEEE_FP ? UNKNOWN : GE;
2763 29659 : default:
2764 29659 : return swap_condition (code);
2765 : }
2766 : }
2767 :
2768 : /* Return cost of comparison CODE using the best strategy for performance.
2769 : All following functions do use number of instructions as a cost metrics.
2770 : In future this should be tweaked to compute bytes for optimize_size and
2771 : take into account performance of various instructions on various CPUs. */
2772 :
2773 : static int
2774 1175064 : ix86_fp_comparison_cost (enum rtx_code code)
2775 : {
2776 1175064 : int arith_cost;
2777 :
2778 : /* The cost of code using bit-twiddling on %ah. */
2779 1175064 : switch (code)
2780 : {
2781 : case UNLE:
2782 : case UNLT:
2783 : case LTGT:
2784 : case GT:
2785 : case GE:
2786 : case UNORDERED:
2787 : case ORDERED:
2788 : case UNEQ:
2789 : arith_cost = 4;
2790 : break;
2791 90314 : case LT:
2792 90314 : case NE:
2793 90314 : case EQ:
2794 90314 : case UNGE:
2795 90314 : arith_cost = TARGET_IEEE_FP ? 5 : 4;
2796 : break;
2797 27439 : case LE:
2798 27439 : case UNGT:
2799 1085584 : arith_cost = TARGET_IEEE_FP ? 6 : 4;
2800 : break;
2801 0 : default:
2802 0 : gcc_unreachable ();
2803 : }
2804 :
2805 1175064 : switch (ix86_fp_comparison_strategy (code))
2806 : {
2807 1175064 : case IX86_FPCMP_COMI:
2808 1175064 : return arith_cost > 4 ? 3 : 2;
2809 0 : case IX86_FPCMP_SAHF:
2810 0 : return arith_cost > 4 ? 4 : 3;
2811 : default:
2812 : return arith_cost;
2813 : }
2814 : }
2815 :
2816 : /* Swap, force into registers, or otherwise massage the two operands
2817 : to a fp comparison. The operands are updated in place; the new
2818 : comparison code is returned. */
2819 :
2820 : static enum rtx_code
2821 587532 : ix86_prepare_fp_compare_args (enum rtx_code code, rtx *pop0, rtx *pop1)
2822 : {
2823 587603 : bool unordered_compare = ix86_unordered_fp_compare (code);
2824 587603 : rtx op0 = *pop0, op1 = *pop1;
2825 587603 : machine_mode op_mode = GET_MODE (op0);
2826 587603 : bool is_sse = SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (op_mode);
2827 :
2828 585199 : if (op_mode == BFmode && (!TARGET_AVX10_2 || flag_trapping_math))
2829 : {
2830 71 : rtx op = gen_lowpart (HImode, op0);
2831 71 : if (CONST_INT_P (op))
2832 0 : op = simplify_const_unary_operation (FLOAT_EXTEND, SFmode,
2833 : op0, BFmode);
2834 : else
2835 : {
2836 71 : rtx t1 = gen_reg_rtx (SImode);
2837 71 : emit_insn (gen_zero_extendhisi2 (t1, op));
2838 71 : emit_insn (gen_ashlsi3 (t1, t1, GEN_INT (16)));
2839 71 : op = gen_lowpart (SFmode, t1);
2840 : }
2841 71 : *pop0 = op;
2842 71 : op = gen_lowpart (HImode, op1);
2843 71 : if (CONST_INT_P (op))
2844 6 : op = simplify_const_unary_operation (FLOAT_EXTEND, SFmode,
2845 : op1, BFmode);
2846 : else
2847 : {
2848 65 : rtx t1 = gen_reg_rtx (SImode);
2849 65 : emit_insn (gen_zero_extendhisi2 (t1, op));
2850 65 : emit_insn (gen_ashlsi3 (t1, t1, GEN_INT (16)));
2851 65 : op = gen_lowpart (SFmode, t1);
2852 : }
2853 71 : *pop1 = op;
2854 71 : return ix86_prepare_fp_compare_args (code, pop0, pop1);
2855 : }
2856 :
2857 : /* All of the unordered compare instructions only work on registers.
2858 : The same is true of the fcomi compare instructions. The XFmode
2859 : compare instructions require registers except when comparing
2860 : against zero or when converting operand 1 from fixed point to
2861 : floating point. */
2862 :
2863 587532 : if (!is_sse
2864 587532 : && (unordered_compare
2865 8306 : || (op_mode == XFmode
2866 10507 : && ! (standard_80387_constant_p (op0) == 1
2867 5250 : || standard_80387_constant_p (op1) == 1)
2868 4806 : && GET_CODE (op1) != FLOAT)
2869 3500 : || ix86_fp_comparison_strategy (code) == IX86_FPCMP_COMI))
2870 : {
2871 148637 : op0 = force_reg (op_mode, op0);
2872 148637 : op1 = force_reg (op_mode, op1);
2873 : }
2874 : else
2875 : {
2876 : /* %%% We only allow op1 in memory; op0 must be st(0). So swap
2877 : things around if they appear profitable, otherwise force op0
2878 : into a register. */
2879 :
2880 438895 : if (standard_80387_constant_p (op0) == 0
2881 438895 : || (MEM_P (op0)
2882 59835 : && ! (standard_80387_constant_p (op1) == 0
2883 44076 : || MEM_P (op1))))
2884 : {
2885 39957 : enum rtx_code new_code = ix86_fp_swap_condition (code);
2886 39957 : if (new_code != UNKNOWN)
2887 : {
2888 : std::swap (op0, op1);
2889 438895 : code = new_code;
2890 : }
2891 : }
2892 :
2893 438895 : if (!REG_P (op0))
2894 55502 : op0 = force_reg (op_mode, op0);
2895 :
2896 438895 : if (CONSTANT_P (op1))
2897 : {
2898 195658 : int tmp = standard_80387_constant_p (op1);
2899 195658 : if (tmp == 0)
2900 75517 : op1 = validize_mem (force_const_mem (op_mode, op1));
2901 120141 : else if (tmp == 1)
2902 : {
2903 65965 : if (TARGET_CMOVE)
2904 65965 : op1 = force_reg (op_mode, op1);
2905 : }
2906 : else
2907 54176 : op1 = force_reg (op_mode, op1);
2908 : }
2909 : }
2910 :
2911 : /* Try to rearrange the comparison to make it cheaper. */
2912 587532 : if (ix86_fp_comparison_cost (code)
2913 587532 : > ix86_fp_comparison_cost (swap_condition (code))
2914 587532 : && (REG_P (op1) || can_create_pseudo_p ()))
2915 : {
2916 0 : std::swap (op0, op1);
2917 0 : code = swap_condition (code);
2918 0 : if (!REG_P (op0))
2919 0 : op0 = force_reg (op_mode, op0);
2920 : }
2921 :
2922 587532 : *pop0 = op0;
2923 587532 : *pop1 = op1;
2924 587532 : return code;
2925 : }
2926 :
2927 : /* Generate insn patterns to do a floating point compare of OPERANDS. */
2928 :
2929 : static rtx
2930 587532 : ix86_expand_fp_compare (enum rtx_code code, rtx op0, rtx op1)
2931 : {
2932 587532 : bool unordered_compare = ix86_unordered_fp_compare (code);
2933 587532 : machine_mode cmp_mode;
2934 587532 : rtx tmp, scratch;
2935 :
2936 587532 : code = ix86_prepare_fp_compare_args (code, &op0, &op1);
2937 587532 : machine_mode op_mode = GET_MODE (op0);
2938 :
2939 587532 : tmp = gen_rtx_COMPARE (CCFPmode, op0, op1);
2940 587532 : if (unordered_compare)
2941 507031 : tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_NOTRAP);
2942 :
2943 : /* Do fcomi/sahf based test when profitable. */
2944 587532 : switch (ix86_fp_comparison_strategy (code))
2945 : {
2946 587532 : case IX86_FPCMP_COMI:
2947 587532 : tmp = gen_rtx_COMPARE (CCFPmode, op0, op1);
2948 : /* VCOMX/VUCOMX only have DF/SF/HF mode instructions. */
2949 587532 : if (TARGET_AVX10_2
2950 1792 : && (code == EQ || code == NE)
2951 982 : && (op_mode == HFmode || op_mode == SFmode || op_mode == DFmode))
2952 972 : tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_OPTCOMX);
2953 : /* We only have vcomisbf16, No vcomubf16 nor vcomxbf16 */
2954 587532 : if (op_mode != BFmode && unordered_compare)
2955 507023 : tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_NOTRAP);
2956 587532 : cmp_mode = CCFPmode;
2957 587532 : emit_insn (gen_rtx_SET (gen_rtx_REG (CCFPmode, FLAGS_REG), tmp));
2958 587532 : break;
2959 :
2960 0 : case IX86_FPCMP_SAHF:
2961 0 : cmp_mode = CCFPmode;
2962 0 : tmp = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW);
2963 0 : scratch = gen_reg_rtx (HImode);
2964 0 : emit_insn (gen_rtx_SET (scratch, tmp));
2965 0 : emit_insn (gen_x86_sahf_1 (scratch));
2966 0 : break;
2967 :
2968 0 : case IX86_FPCMP_ARITH:
2969 0 : cmp_mode = CCNOmode;
2970 0 : tmp = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW);
2971 0 : scratch = gen_reg_rtx (HImode);
2972 0 : emit_insn (gen_rtx_SET (scratch, tmp));
2973 :
2974 : /* In the unordered case, we have to check C2 for NaN's, which
2975 : doesn't happen to work out to anything nice combination-wise.
2976 : So do some bit twiddling on the value we've got in AH to come
2977 : up with an appropriate set of condition codes. */
2978 :
2979 0 : switch (code)
2980 : {
2981 0 : case GT:
2982 0 : case UNGT:
2983 0 : if (code == GT || !TARGET_IEEE_FP)
2984 : {
2985 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x45)));
2986 0 : code = EQ;
2987 : }
2988 : else
2989 : {
2990 0 : emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
2991 0 : emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
2992 0 : emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x44)));
2993 0 : cmp_mode = CCmode;
2994 0 : code = GEU;
2995 : }
2996 : break;
2997 0 : case LT:
2998 0 : case UNLT:
2999 0 : if (code == LT && TARGET_IEEE_FP)
3000 : {
3001 0 : emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
3002 0 : emit_insn (gen_cmpqi_ext_3 (scratch, const1_rtx));
3003 0 : cmp_mode = CCmode;
3004 0 : code = EQ;
3005 : }
3006 : else
3007 : {
3008 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, const1_rtx));
3009 0 : code = NE;
3010 : }
3011 : break;
3012 0 : case GE:
3013 0 : case UNGE:
3014 0 : if (code == GE || !TARGET_IEEE_FP)
3015 : {
3016 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x05)));
3017 0 : code = EQ;
3018 : }
3019 : else
3020 : {
3021 0 : emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
3022 0 : emit_insn (gen_xorqi_ext_1_cc (scratch, scratch, const1_rtx));
3023 0 : code = NE;
3024 : }
3025 : break;
3026 0 : case LE:
3027 0 : case UNLE:
3028 0 : if (code == LE && TARGET_IEEE_FP)
3029 : {
3030 0 : emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
3031 0 : emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
3032 0 : emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
3033 0 : cmp_mode = CCmode;
3034 0 : code = LTU;
3035 : }
3036 : else
3037 : {
3038 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x45)));
3039 0 : code = NE;
3040 : }
3041 : break;
3042 0 : case EQ:
3043 0 : case UNEQ:
3044 0 : if (code == EQ && TARGET_IEEE_FP)
3045 : {
3046 0 : emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
3047 0 : emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
3048 0 : cmp_mode = CCmode;
3049 0 : code = EQ;
3050 : }
3051 : else
3052 : {
3053 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x40)));
3054 0 : code = NE;
3055 : }
3056 : break;
3057 0 : case NE:
3058 0 : case LTGT:
3059 0 : if (code == NE && TARGET_IEEE_FP)
3060 : {
3061 0 : emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
3062 0 : emit_insn (gen_xorqi_ext_1_cc (scratch, scratch,
3063 : GEN_INT (0x40)));
3064 0 : code = NE;
3065 : }
3066 : else
3067 : {
3068 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x40)));
3069 0 : code = EQ;
3070 : }
3071 : break;
3072 :
3073 0 : case UNORDERED:
3074 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x04)));
3075 0 : code = NE;
3076 0 : break;
3077 0 : case ORDERED:
3078 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x04)));
3079 0 : code = EQ;
3080 0 : break;
3081 :
3082 0 : default:
3083 0 : gcc_unreachable ();
3084 : }
3085 : break;
3086 :
3087 0 : default:
3088 0 : gcc_unreachable();
3089 : }
3090 :
3091 : /* Return the test that should be put into the flags user, i.e.
3092 : the bcc, scc, or cmov instruction. */
3093 587532 : return gen_rtx_fmt_ee (code, VOIDmode,
3094 : gen_rtx_REG (cmp_mode, FLAGS_REG),
3095 : const0_rtx);
3096 : }
3097 :
3098 : /* Generate insn patterns to do an integer compare of OPERANDS. */
3099 :
3100 : static rtx
3101 7170282 : ix86_expand_int_compare (enum rtx_code code, rtx op0, rtx op1)
3102 : {
3103 7170282 : machine_mode cmpmode;
3104 7170282 : rtx tmp, flags;
3105 :
3106 : /* Swap operands to emit carry flag comparison. */
3107 7170282 : if ((code == GTU || code == LEU)
3108 7170282 : && nonimmediate_operand (op1, VOIDmode))
3109 : {
3110 145211 : std::swap (op0, op1);
3111 145211 : code = swap_condition (code);
3112 : }
3113 :
3114 7170282 : cmpmode = SELECT_CC_MODE (code, op0, op1);
3115 7170282 : flags = gen_rtx_REG (cmpmode, FLAGS_REG);
3116 :
3117 : /* Attempt to use PTEST, if available, when testing vector modes for
3118 : equality/inequality against zero. */
3119 7170282 : if (op1 == const0_rtx
3120 2974141 : && SUBREG_P (op0)
3121 24503 : && cmpmode == CCZmode
3122 11450 : && SUBREG_BYTE (op0) == 0
3123 9775 : && REG_P (SUBREG_REG (op0))
3124 9775 : && VECTOR_MODE_P (GET_MODE (SUBREG_REG (op0)))
3125 8 : && TARGET_SSE4_1
3126 2 : && GET_MODE (op0) == TImode
3127 7170286 : && GET_MODE_SIZE (GET_MODE (SUBREG_REG (op0))) == 16)
3128 : {
3129 2 : tmp = SUBREG_REG (op0);
3130 2 : if (GET_MODE (tmp) == V8HFmode || GET_MODE (tmp) == V8BFmode)
3131 1 : tmp = gen_lowpart (V8HImode, tmp);
3132 2 : tmp = gen_rtx_UNSPEC (CCZmode, gen_rtvec (2, tmp, tmp), UNSPEC_PTEST);
3133 : }
3134 : else
3135 7170280 : tmp = gen_rtx_COMPARE (cmpmode, op0, op1);
3136 :
3137 : /* This is very simple, but making the interface the same as in the
3138 : FP case makes the rest of the code easier. */
3139 7170282 : emit_insn (gen_rtx_SET (flags, tmp));
3140 :
3141 : /* Return the test that should be put into the flags user, i.e.
3142 : the bcc, scc, or cmov instruction. */
3143 7170282 : return gen_rtx_fmt_ee (code, VOIDmode, flags, const0_rtx);
3144 : }
3145 :
3146 : static rtx
3147 7909763 : ix86_expand_compare (enum rtx_code code, rtx op0, rtx op1)
3148 : {
3149 7909763 : rtx ret;
3150 :
3151 7909763 : if (GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC)
3152 153885 : ret = gen_rtx_fmt_ee (code, VOIDmode, op0, op1);
3153 :
3154 7755878 : else if (SCALAR_FLOAT_MODE_P (GET_MODE (op0)))
3155 : {
3156 585596 : gcc_assert (!DECIMAL_FLOAT_MODE_P (GET_MODE (op0)));
3157 585596 : ret = ix86_expand_fp_compare (code, op0, op1);
3158 : }
3159 : else
3160 7170282 : ret = ix86_expand_int_compare (code, op0, op1);
3161 :
3162 7909763 : return ret;
3163 : }
3164 :
3165 : void
3166 629906 : ix86_expand_setcc (rtx dest, enum rtx_code code, rtx op0, rtx op1)
3167 : {
3168 629906 : rtx ret;
3169 :
3170 629906 : gcc_assert (GET_MODE (dest) == QImode);
3171 :
3172 629906 : ret = ix86_expand_compare (code, op0, op1);
3173 629906 : PUT_MODE (ret, QImode);
3174 629906 : emit_insn (gen_rtx_SET (dest, ret));
3175 629906 : }
3176 :
3177 : /* Expand floating point op0 <=> op1, i.e.
3178 : dest = op0 == op1 ? 0 : op0 < op1 ? -1 : op0 > op1 ? 1 : -128. */
3179 :
3180 : void
3181 174 : ix86_expand_fp_spaceship (rtx dest, rtx op0, rtx op1, rtx op2)
3182 : {
3183 174 : gcc_checking_assert (ix86_fp_comparison_strategy (GT) != IX86_FPCMP_ARITH);
3184 174 : rtx zero = NULL_RTX;
3185 174 : if (op2 != const0_rtx
3186 52 : && (TARGET_IEEE_FP || TARGET_ZERO_EXTEND_WITH_AND)
3187 34 : && GET_MODE (dest) == SImode)
3188 34 : zero = force_reg (SImode, const0_rtx);
3189 174 : rtx gt = ix86_expand_fp_compare (GT, op0, op1);
3190 174 : rtx_code_label *l0 = op2 == const0_rtx ? gen_label_rtx () : nullptr;
3191 174 : rtx_code_label *l1 = op2 == const0_rtx ? gen_label_rtx () : nullptr;
3192 174 : rtx_code_label *l2 = TARGET_IEEE_FP ? gen_label_rtx () : nullptr;
3193 174 : rtx_code_label *lend = gen_label_rtx ();
3194 174 : rtx tmp;
3195 174 : rtx_insn *jmp;
3196 174 : if (l2)
3197 : {
3198 137 : rtx un = gen_rtx_fmt_ee (UNORDERED, VOIDmode,
3199 : gen_rtx_REG (CCFPmode, FLAGS_REG), const0_rtx);
3200 137 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, un,
3201 : gen_rtx_LABEL_REF (VOIDmode, l2), pc_rtx);
3202 137 : jmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
3203 137 : add_reg_br_prob_note (jmp, profile_probability:: very_unlikely ());
3204 : }
3205 174 : if (op2 == const0_rtx)
3206 : {
3207 122 : rtx eq = gen_rtx_fmt_ee (UNEQ, VOIDmode,
3208 : gen_rtx_REG (CCFPmode, FLAGS_REG), const0_rtx);
3209 122 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, eq,
3210 : gen_rtx_LABEL_REF (VOIDmode, l0), pc_rtx);
3211 122 : jmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
3212 122 : add_reg_br_prob_note (jmp, profile_probability::unlikely ());
3213 122 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, gt,
3214 : gen_rtx_LABEL_REF (VOIDmode, l1), pc_rtx);
3215 122 : jmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
3216 122 : add_reg_br_prob_note (jmp, profile_probability::even ());
3217 122 : emit_move_insn (dest, constm1_rtx);
3218 122 : emit_jump (lend);
3219 122 : emit_label (l0);
3220 122 : emit_move_insn (dest, const0_rtx);
3221 122 : emit_jump (lend);
3222 122 : emit_label (l1);
3223 122 : emit_move_insn (dest, const1_rtx);
3224 : }
3225 : else
3226 : {
3227 52 : rtx lt_tmp = NULL_RTX;
3228 52 : if (GET_MODE (dest) != SImode || !TARGET_ZERO_EXTEND_WITH_AND)
3229 : {
3230 52 : lt_tmp = gen_reg_rtx (QImode);
3231 52 : ix86_expand_setcc (lt_tmp, UNLT, gen_rtx_REG (CCFPmode, FLAGS_REG),
3232 : const0_rtx);
3233 52 : if (GET_MODE (dest) != QImode)
3234 : {
3235 52 : tmp = gen_reg_rtx (GET_MODE (dest));
3236 52 : emit_insn (gen_rtx_SET (tmp,
3237 : gen_rtx_ZERO_EXTEND (GET_MODE (dest),
3238 : lt_tmp)));
3239 52 : lt_tmp = tmp;
3240 : }
3241 : }
3242 52 : rtx gt_tmp;
3243 52 : if (zero)
3244 : {
3245 : /* If TARGET_IEEE_FP and dest has SImode, emit SImode clear
3246 : before the floating point comparison and use setcc_si_slp
3247 : pattern to hide it from the combiner, so that it doesn't
3248 : undo it. Similarly for TARGET_ZERO_EXTEND_WITH_AND, where
3249 : the ZERO_EXTEND normally emitted would need to be AND
3250 : with flags clobber. */
3251 34 : tmp = ix86_expand_compare (GT, XEXP (gt, 0), const0_rtx);
3252 34 : PUT_MODE (tmp, QImode);
3253 34 : emit_insn (gen_setcc_si_slp (zero, tmp, zero));
3254 34 : gt_tmp = zero;
3255 : }
3256 : else
3257 : {
3258 18 : gt_tmp = gen_reg_rtx (QImode);
3259 18 : ix86_expand_setcc (gt_tmp, GT, XEXP (gt, 0), const0_rtx);
3260 18 : if (GET_MODE (dest) != QImode)
3261 : {
3262 18 : tmp = gen_reg_rtx (GET_MODE (dest));
3263 18 : emit_insn (gen_rtx_SET (tmp,
3264 : gen_rtx_ZERO_EXTEND (GET_MODE (dest),
3265 : gt_tmp)));
3266 18 : gt_tmp = tmp;
3267 : }
3268 : }
3269 52 : if (lt_tmp)
3270 : {
3271 52 : tmp = expand_simple_binop (GET_MODE (dest), MINUS, gt_tmp, lt_tmp,
3272 : dest, 0, OPTAB_DIRECT);
3273 52 : if (!rtx_equal_p (tmp, dest))
3274 0 : emit_move_insn (dest, tmp);
3275 : }
3276 : else
3277 : {
3278 : /* For TARGET_ZERO_EXTEND_WITH_AND emit sbb directly, as we can't
3279 : do ZERO_EXTEND without clobbering flags. */
3280 0 : tmp = ix86_expand_compare (UNLT, XEXP (gt, 0), const0_rtx);
3281 0 : PUT_MODE (tmp, SImode);
3282 0 : emit_insn (gen_subsi3_carry (dest, gt_tmp,
3283 0 : force_reg (GET_MODE (dest), const0_rtx),
3284 : XEXP (gt, 0), tmp));
3285 : }
3286 : }
3287 174 : emit_jump (lend);
3288 174 : if (l2)
3289 : {
3290 137 : emit_label (l2);
3291 137 : emit_move_insn (dest, op2 == const0_rtx ? GEN_INT (-128) : op2);
3292 : }
3293 174 : emit_label (lend);
3294 174 : }
3295 :
3296 : /* Expand integral op0 <=> op1, i.e.
3297 : dest = op0 == op1 ? 0 : op0 < op1 ? -1 : 1. */
3298 :
3299 : void
3300 159 : ix86_expand_int_spaceship (rtx dest, rtx op0, rtx op1, rtx op2)
3301 : {
3302 159 : gcc_assert (INTVAL (op2));
3303 159 : rtx zero1 = NULL_RTX, zero2 = NULL_RTX;
3304 159 : if (TARGET_ZERO_EXTEND_WITH_AND && GET_MODE (dest) == SImode)
3305 : {
3306 0 : zero1 = force_reg (SImode, const0_rtx);
3307 0 : if (INTVAL (op2) != 1)
3308 0 : zero2 = force_reg (SImode, const0_rtx);
3309 : }
3310 :
3311 : /* Not using ix86_expand_int_compare here, so that it doesn't swap
3312 : operands nor optimize CC mode - we need a mode usable for both
3313 : LT and GT resp. LTU and GTU comparisons with the same unswapped
3314 : operands. */
3315 195 : rtx flags = gen_rtx_REG (INTVAL (op2) != 1 ? CCGCmode : CCmode, FLAGS_REG);
3316 159 : rtx tmp = gen_rtx_COMPARE (GET_MODE (flags), op0, op1);
3317 159 : emit_insn (gen_rtx_SET (flags, tmp));
3318 159 : rtx lt_tmp = NULL_RTX;
3319 159 : if (zero2)
3320 : {
3321 : /* For TARGET_ZERO_EXTEND_WITH_AND, emit setcc_si_slp to avoid
3322 : ZERO_EXTEND. */
3323 0 : tmp = ix86_expand_compare (LT, flags, const0_rtx);
3324 0 : PUT_MODE (tmp, QImode);
3325 0 : emit_insn (gen_setcc_si_slp (zero2, tmp, zero2));
3326 0 : lt_tmp = zero2;
3327 : }
3328 159 : else if (!zero1)
3329 : {
3330 159 : lt_tmp = gen_reg_rtx (QImode);
3331 195 : ix86_expand_setcc (lt_tmp, INTVAL (op2) != 1 ? LT : LTU, flags,
3332 : const0_rtx);
3333 159 : if (GET_MODE (dest) != QImode)
3334 : {
3335 159 : tmp = gen_reg_rtx (GET_MODE (dest));
3336 159 : emit_insn (gen_rtx_SET (tmp, gen_rtx_ZERO_EXTEND (GET_MODE (dest),
3337 : lt_tmp)));
3338 159 : lt_tmp = tmp;
3339 : }
3340 : }
3341 159 : rtx gt_tmp;
3342 159 : if (zero1)
3343 : {
3344 : /* For TARGET_ZERO_EXTEND_WITH_AND, emit setcc_si_slp to avoid
3345 : ZERO_EXTEND. */
3346 0 : tmp = ix86_expand_compare (INTVAL (op2) != 1 ? GT : GTU, flags,
3347 : const0_rtx);
3348 0 : PUT_MODE (tmp, QImode);
3349 0 : emit_insn (gen_setcc_si_slp (zero1, tmp, zero1));
3350 0 : gt_tmp = zero1;
3351 : }
3352 : else
3353 : {
3354 159 : gt_tmp = gen_reg_rtx (QImode);
3355 195 : ix86_expand_setcc (gt_tmp, INTVAL (op2) != 1 ? GT : GTU, flags,
3356 : const0_rtx);
3357 159 : if (GET_MODE (dest) != QImode)
3358 : {
3359 159 : tmp = gen_reg_rtx (GET_MODE (dest));
3360 159 : emit_insn (gen_rtx_SET (tmp, gen_rtx_ZERO_EXTEND (GET_MODE (dest),
3361 : gt_tmp)));
3362 159 : gt_tmp = tmp;
3363 : }
3364 : }
3365 159 : if (lt_tmp)
3366 : {
3367 159 : tmp = expand_simple_binop (GET_MODE (dest), MINUS, gt_tmp, lt_tmp, dest,
3368 : 0, OPTAB_DIRECT);
3369 159 : if (!rtx_equal_p (tmp, dest))
3370 0 : emit_move_insn (dest, tmp);
3371 : }
3372 : else
3373 : {
3374 : /* For TARGET_ZERO_EXTEND_WITH_AND emit sbb directly, as we can't
3375 : do ZERO_EXTEND without clobbering flags. */
3376 0 : tmp = ix86_expand_compare (LTU, flags, const0_rtx);
3377 0 : PUT_MODE (tmp, SImode);
3378 0 : emit_insn (gen_subsi3_carry (dest, gt_tmp,
3379 0 : force_reg (GET_MODE (dest), const0_rtx),
3380 : flags, tmp));
3381 : }
3382 159 : }
3383 :
3384 : /* Expand comparison setting or clearing carry flag. Return true when
3385 : successful and set pop for the operation. */
3386 : static bool
3387 23870 : ix86_expand_carry_flag_compare (enum rtx_code code, rtx op0, rtx op1, rtx *pop)
3388 : {
3389 47740 : machine_mode mode
3390 23870 : = GET_MODE (op0) != VOIDmode ? GET_MODE (op0) : GET_MODE (op1);
3391 :
3392 : /* Do not handle double-mode compares that go through special path. */
3393 26022 : if (mode == (TARGET_64BIT ? TImode : DImode))
3394 : return false;
3395 :
3396 23862 : if (SCALAR_FLOAT_MODE_P (mode))
3397 : {
3398 1764 : rtx compare_op;
3399 1764 : rtx_insn *compare_seq;
3400 :
3401 1764 : gcc_assert (!DECIMAL_FLOAT_MODE_P (mode));
3402 :
3403 : /* Shortcut: following common codes never translate
3404 : into carry flag compares. */
3405 1764 : if (code == EQ || code == NE || code == UNEQ || code == LTGT
3406 : || code == ORDERED || code == UNORDERED)
3407 : return false;
3408 :
3409 : /* These comparisons require zero flag; swap operands so they won't. */
3410 : if ((code == GT || code == UNLE || code == LE || code == UNGT)
3411 1696 : && !TARGET_IEEE_FP)
3412 : {
3413 2 : std::swap (op0, op1);
3414 2 : code = swap_condition (code);
3415 : }
3416 :
3417 : /* Try to expand the comparison and verify that we end up with
3418 : carry flag based comparison. This fails to be true only when
3419 : we decide to expand comparison using arithmetic that is not
3420 : too common scenario. */
3421 1762 : start_sequence ();
3422 1762 : compare_op = ix86_expand_fp_compare (code, op0, op1);
3423 1762 : compare_seq = end_sequence ();
3424 :
3425 1762 : if (GET_MODE (XEXP (compare_op, 0)) == CCFPmode)
3426 1762 : code = ix86_fp_compare_code_to_integer (GET_CODE (compare_op));
3427 : else
3428 0 : code = GET_CODE (compare_op);
3429 :
3430 1762 : if (code != LTU && code != GEU)
3431 : return false;
3432 :
3433 66 : emit_insn (compare_seq);
3434 66 : *pop = compare_op;
3435 66 : return true;
3436 : }
3437 :
3438 22098 : if (!INTEGRAL_MODE_P (mode))
3439 : return false;
3440 :
3441 22026 : switch (code)
3442 : {
3443 : case LTU:
3444 : case GEU:
3445 : break;
3446 :
3447 : /* Convert a==0 into (unsigned)a<1. */
3448 18001 : case EQ:
3449 18001 : case NE:
3450 18001 : if (op1 != const0_rtx)
3451 : return false;
3452 9481 : op1 = const1_rtx;
3453 9481 : code = (code == EQ ? LTU : GEU);
3454 : break;
3455 :
3456 : /* Convert a>b into b<a or a>=b-1. */
3457 920 : case GTU:
3458 920 : case LEU:
3459 920 : if (CONST_INT_P (op1))
3460 : {
3461 882 : op1 = gen_int_mode (INTVAL (op1) + 1, GET_MODE (op0));
3462 : /* Bail out on overflow. We still can swap operands but that
3463 : would force loading of the constant into register. */
3464 882 : if (op1 == const0_rtx
3465 882 : || !x86_64_immediate_operand (op1, GET_MODE (op1)))
3466 : return false;
3467 882 : code = (code == GTU ? GEU : LTU);
3468 : }
3469 : else
3470 : {
3471 38 : std::swap (op0, op1);
3472 38 : code = (code == GTU ? LTU : GEU);
3473 : }
3474 : break;
3475 :
3476 : /* Convert a>=0 into (unsigned)a<0x80000000. */
3477 1445 : case LT:
3478 1445 : case GE:
3479 1445 : if (mode == DImode || op1 != const0_rtx)
3480 : return false;
3481 100 : op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
3482 50 : code = (code == LT ? GEU : LTU);
3483 : break;
3484 965 : case LE:
3485 965 : case GT:
3486 965 : if (mode == DImode || op1 != constm1_rtx)
3487 : return false;
3488 0 : op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
3489 0 : code = (code == LE ? GEU : LTU);
3490 : break;
3491 :
3492 : default:
3493 : return false;
3494 : }
3495 : /* Swapping operands may cause constant to appear as first operand. */
3496 11146 : if (!nonimmediate_operand (op0, VOIDmode))
3497 : {
3498 0 : if (!can_create_pseudo_p ())
3499 : return false;
3500 0 : op0 = force_reg (mode, op0);
3501 : }
3502 11146 : *pop = ix86_expand_compare (code, op0, op1);
3503 11146 : gcc_assert (GET_CODE (*pop) == LTU || GET_CODE (*pop) == GEU);
3504 : return true;
3505 : }
3506 :
3507 : /* Expand conditional increment or decrement using adb/sbb instructions.
3508 : The default case using setcc followed by the conditional move can be
3509 : done by generic code. */
3510 : bool
3511 2758 : ix86_expand_int_addcc (rtx operands[])
3512 : {
3513 2758 : enum rtx_code code = GET_CODE (operands[1]);
3514 2758 : rtx flags;
3515 2758 : rtx (*insn) (machine_mode, rtx, rtx, rtx, rtx, rtx);
3516 2758 : rtx compare_op;
3517 2758 : rtx val = const0_rtx;
3518 2758 : bool fpcmp = false;
3519 2758 : machine_mode mode;
3520 2758 : rtx op0 = XEXP (operands[1], 0);
3521 2758 : rtx op1 = XEXP (operands[1], 1);
3522 :
3523 2758 : if (operands[3] != const1_rtx
3524 2338 : && operands[3] != constm1_rtx)
3525 : return false;
3526 1174 : if (!ix86_expand_carry_flag_compare (code, op0, op1, &compare_op))
3527 : return false;
3528 342 : code = GET_CODE (compare_op);
3529 :
3530 342 : flags = XEXP (compare_op, 0);
3531 :
3532 342 : if (GET_MODE (flags) == CCFPmode)
3533 : {
3534 1 : fpcmp = true;
3535 1 : code = ix86_fp_compare_code_to_integer (code);
3536 : }
3537 :
3538 342 : if (code != LTU)
3539 : {
3540 175 : val = constm1_rtx;
3541 175 : if (fpcmp)
3542 1 : PUT_CODE (compare_op,
3543 : reverse_condition_maybe_unordered
3544 : (GET_CODE (compare_op)));
3545 : else
3546 174 : PUT_CODE (compare_op, reverse_condition (GET_CODE (compare_op)));
3547 : }
3548 :
3549 342 : mode = GET_MODE (operands[0]);
3550 :
3551 : /* Construct either adc or sbb insn. */
3552 342 : if ((code == LTU) == (operands[3] == constm1_rtx))
3553 : insn = gen_sub3_carry;
3554 : else
3555 125 : insn = gen_add3_carry;
3556 :
3557 342 : emit_insn (insn (mode, operands[0], operands[2], val, flags, compare_op));
3558 :
3559 342 : return true;
3560 : }
3561 :
3562 : bool
3563 524407 : ix86_expand_int_movcc (rtx operands[])
3564 : {
3565 524407 : enum rtx_code code = GET_CODE (operands[1]), compare_code;
3566 524407 : rtx_insn *compare_seq;
3567 524407 : rtx compare_op;
3568 524407 : machine_mode mode = GET_MODE (operands[0]);
3569 524407 : bool sign_bit_compare_p = false;
3570 524407 : bool negate_cc_compare_p = false;
3571 524407 : rtx op0 = XEXP (operands[1], 0);
3572 524407 : rtx op1 = XEXP (operands[1], 1);
3573 524407 : rtx op2 = operands[2];
3574 524407 : rtx op3 = operands[3];
3575 :
3576 524407 : if (GET_MODE (op0) == TImode
3577 508803 : || (GET_MODE (op0) == DImode
3578 128488 : && !TARGET_64BIT))
3579 : return false;
3580 :
3581 507625 : if (GET_MODE (op0) == BFmode
3582 507625 : && !ix86_fp_comparison_operator (operands[1], VOIDmode))
3583 : return false;
3584 :
3585 507625 : start_sequence ();
3586 507625 : compare_op = ix86_expand_compare (code, op0, op1);
3587 507625 : compare_seq = end_sequence ();
3588 :
3589 507625 : compare_code = GET_CODE (compare_op);
3590 :
3591 507625 : if ((op1 == const0_rtx && (code == GE || code == LT))
3592 463330 : || (op1 == constm1_rtx && (code == GT || code == LE)))
3593 : sign_bit_compare_p = true;
3594 :
3595 : /* op0 == op1 ? op0 : op3 is equivalent to op0 == op1 ? op1 : op3,
3596 : but if op1 is a constant, the latter form allows more optimizations,
3597 : either through the last 2 ops being constant handling, or the one
3598 : constant and one variable cases. On the other side, for cmov the
3599 : former might be better as we don't need to load the constant into
3600 : another register. */
3601 463330 : if (code == EQ && CONST_INT_P (op1) && rtx_equal_p (op0, op2))
3602 : op2 = op1;
3603 : /* Similarly for op0 != op1 ? op2 : op0 and op0 != op1 ? op2 : op1. */
3604 507104 : else if (code == NE && CONST_INT_P (op1) && rtx_equal_p (op0, op3))
3605 : op3 = op1;
3606 :
3607 : /* Don't attempt mode expansion here -- if we had to expand 5 or 6
3608 : HImode insns, we'd be swallowed in word prefix ops. */
3609 :
3610 5054 : if ((mode != HImode || TARGET_FAST_PREFIX)
3611 540865 : && (mode != (TARGET_64BIT ? TImode : DImode))
3612 507625 : && CONST_INT_P (op2)
3613 538103 : && CONST_INT_P (op3))
3614 : {
3615 23704 : rtx out = operands[0];
3616 23704 : HOST_WIDE_INT ct = INTVAL (op2);
3617 23704 : HOST_WIDE_INT cf = INTVAL (op3);
3618 23704 : HOST_WIDE_INT diff;
3619 :
3620 23704 : if ((mode == SImode
3621 8723 : || (TARGET_64BIT && mode == DImode))
3622 20073 : && (GET_MODE (op0) == SImode
3623 14798 : || (TARGET_64BIT && GET_MODE (op0) == DImode)))
3624 : {
3625 : /* Special case x != 0 ? -1 : y. */
3626 15329 : if (code == NE && op1 == const0_rtx && ct == -1)
3627 : {
3628 : negate_cc_compare_p = true;
3629 : std::swap (ct, cf);
3630 : code = EQ;
3631 : }
3632 15205 : else if (code == EQ && op1 == const0_rtx && cf == -1)
3633 23704 : negate_cc_compare_p = true;
3634 : }
3635 :
3636 23704 : diff = (unsigned HOST_WIDE_INT) ct - cf;
3637 : /* Make sure we can represent the difference between the two values. */
3638 23704 : if ((diff > 0) != ((cf < 0) != (ct < 0) ? cf < 0 : cf < ct))
3639 524407 : return false;
3640 :
3641 : /* Sign bit compares are better done using shifts than we do by using
3642 : sbb. */
3643 23626 : if (sign_bit_compare_p
3644 23626 : || negate_cc_compare_p
3645 23626 : || ix86_expand_carry_flag_compare (code, op0, op1, &compare_op))
3646 : {
3647 : /* Detect overlap between destination and compare sources. */
3648 11800 : rtx tmp = out;
3649 :
3650 11800 : if (negate_cc_compare_p)
3651 : {
3652 291 : if (GET_MODE (op0) == DImode)
3653 117 : emit_insn (gen_x86_negdi_ccc (gen_reg_rtx (DImode), op0));
3654 : else
3655 174 : emit_insn (gen_x86_negsi_ccc (gen_reg_rtx (SImode),
3656 174 : gen_lowpart (SImode, op0)));
3657 :
3658 291 : tmp = gen_reg_rtx (mode);
3659 291 : if (mode == DImode)
3660 137 : emit_insn (gen_x86_movdicc_0_m1_neg (tmp));
3661 : else
3662 154 : emit_insn (gen_x86_movsicc_0_m1_neg (gen_lowpart (SImode,
3663 : tmp)));
3664 : }
3665 11509 : else if (!sign_bit_compare_p)
3666 : {
3667 10870 : rtx flags;
3668 10870 : bool fpcmp = false;
3669 :
3670 10870 : compare_code = GET_CODE (compare_op);
3671 :
3672 10870 : flags = XEXP (compare_op, 0);
3673 :
3674 10870 : if (GET_MODE (flags) == CCFPmode)
3675 : {
3676 65 : fpcmp = true;
3677 65 : compare_code
3678 65 : = ix86_fp_compare_code_to_integer (compare_code);
3679 : }
3680 :
3681 : /* To simplify rest of code, restrict to the GEU case. */
3682 10870 : if (compare_code == LTU)
3683 : {
3684 6519 : std::swap (ct, cf);
3685 6519 : compare_code = reverse_condition (compare_code);
3686 6519 : code = reverse_condition (code);
3687 : }
3688 : else
3689 : {
3690 4351 : if (fpcmp)
3691 65 : PUT_CODE (compare_op,
3692 : reverse_condition_maybe_unordered
3693 : (GET_CODE (compare_op)));
3694 : else
3695 4286 : PUT_CODE (compare_op,
3696 : reverse_condition (GET_CODE (compare_op)));
3697 : }
3698 :
3699 10870 : diff = (unsigned HOST_WIDE_INT) ct - cf;
3700 : /* Make sure we can represent the difference
3701 : between the two values. */
3702 10870 : if ((diff > 0) != ((cf < 0) != (ct < 0) ? cf < 0 : cf < ct))
3703 : return false;
3704 :
3705 10869 : if (reg_overlap_mentioned_p (out, compare_op))
3706 0 : tmp = gen_reg_rtx (mode);
3707 :
3708 10869 : if (mode == DImode)
3709 1926 : emit_insn (gen_x86_movdicc_0_m1 (tmp, flags, compare_op));
3710 : else
3711 8943 : emit_insn (gen_x86_movsicc_0_m1 (gen_lowpart (SImode, tmp),
3712 : flags, compare_op));
3713 : }
3714 : else
3715 : {
3716 639 : if (code == GT || code == GE)
3717 383 : code = reverse_condition (code);
3718 : else
3719 : {
3720 256 : std::swap (ct, cf);
3721 :
3722 256 : diff = (unsigned HOST_WIDE_INT) ct - cf;
3723 : /* Make sure we can represent the difference
3724 : between the two values. */
3725 256 : if ((diff > 0) != ((cf < 0) != (ct < 0) ? cf < 0 : cf < ct))
3726 : return false;
3727 : }
3728 636 : tmp = emit_store_flag (tmp, code, op0, op1, VOIDmode, 0, -1);
3729 : }
3730 :
3731 11796 : if (diff == 1)
3732 : {
3733 : /*
3734 : * cmpl op0,op1
3735 : * sbbl dest,dest
3736 : * [addl dest, ct]
3737 : *
3738 : * Size 5 - 8.
3739 : */
3740 717 : if (ct)
3741 555 : tmp = expand_simple_binop (mode, PLUS,
3742 : tmp, GEN_INT (ct),
3743 : copy_rtx (tmp), 1, OPTAB_DIRECT);
3744 : }
3745 11079 : else if (cf == -1)
3746 : {
3747 : /*
3748 : * cmpl op0,op1
3749 : * sbbl dest,dest
3750 : * orl $ct, dest
3751 : *
3752 : * Size 8.
3753 : */
3754 689 : tmp = expand_simple_binop (mode, IOR,
3755 : tmp, GEN_INT (ct),
3756 : copy_rtx (tmp), 1, OPTAB_DIRECT);
3757 : }
3758 10390 : else if (diff == -1 && ct)
3759 : {
3760 : /*
3761 : * cmpl op0,op1
3762 : * sbbl dest,dest
3763 : * notl dest
3764 : * [addl dest, cf]
3765 : *
3766 : * Size 8 - 11.
3767 : */
3768 468 : tmp = expand_simple_unop (mode, NOT, tmp, copy_rtx (tmp), 1);
3769 468 : if (cf)
3770 449 : tmp = expand_simple_binop (mode, PLUS,
3771 : copy_rtx (tmp), GEN_INT (cf),
3772 : copy_rtx (tmp), 1, OPTAB_DIRECT);
3773 : }
3774 : else
3775 : {
3776 : /*
3777 : * cmpl op0,op1
3778 : * sbbl dest,dest
3779 : * [notl dest]
3780 : * andl cf - ct, dest
3781 : * [addl dest, ct]
3782 : *
3783 : * Size 8 - 11.
3784 : */
3785 :
3786 9922 : if (cf == 0)
3787 : {
3788 1305 : cf = ct;
3789 1305 : ct = 0;
3790 1305 : tmp = expand_simple_unop (mode, NOT, tmp, copy_rtx (tmp), 1);
3791 : }
3792 :
3793 9922 : HOST_WIDE_INT ival = (unsigned HOST_WIDE_INT) cf - ct;
3794 : /* Make sure we can represent the difference
3795 : between the two values. */
3796 9922 : if ((ival > 0) != ((ct < 0) != (cf < 0) ? ct < 0 : ct < cf))
3797 524407 : return false;
3798 :
3799 9922 : tmp = expand_simple_binop (mode, AND,
3800 : copy_rtx (tmp),
3801 9922 : gen_int_mode (ival, mode),
3802 : copy_rtx (tmp), 1, OPTAB_DIRECT);
3803 9922 : if (ct)
3804 7152 : tmp = expand_simple_binop (mode, PLUS,
3805 : copy_rtx (tmp), GEN_INT (ct),
3806 : copy_rtx (tmp), 1, OPTAB_DIRECT);
3807 : }
3808 :
3809 11796 : if (!rtx_equal_p (tmp, out))
3810 476 : emit_move_insn (copy_rtx (out), copy_rtx (tmp));
3811 :
3812 : return true;
3813 : }
3814 :
3815 11826 : if (diff < 0)
3816 : {
3817 3630 : machine_mode cmp_mode = GET_MODE (op0);
3818 3630 : enum rtx_code new_code;
3819 :
3820 3630 : if (SCALAR_FLOAT_MODE_P (cmp_mode))
3821 : {
3822 77 : gcc_assert (!DECIMAL_FLOAT_MODE_P (cmp_mode));
3823 :
3824 : /* We may be reversing a non-trapping
3825 : comparison to a trapping comparison. */
3826 150 : if (HONOR_NANS (cmp_mode) && flag_trapping_math
3827 70 : && code != EQ && code != NE
3828 147 : && code != ORDERED && code != UNORDERED)
3829 : new_code = UNKNOWN;
3830 : else
3831 7 : new_code = reverse_condition_maybe_unordered (code);
3832 : }
3833 : else
3834 3553 : new_code = ix86_reverse_condition (code, cmp_mode);
3835 3560 : if (new_code != UNKNOWN)
3836 : {
3837 3560 : std::swap (ct, cf);
3838 :
3839 3560 : diff = (unsigned HOST_WIDE_INT) ct - cf;
3840 : /* Make sure we can represent the difference
3841 : between the two values. */
3842 3560 : if ((diff > 0) != ((cf < 0) != (ct < 0) ? cf < 0 : cf < ct))
3843 : return false;
3844 :
3845 : code = new_code;
3846 : }
3847 : }
3848 :
3849 11826 : compare_code = UNKNOWN;
3850 11826 : if (GET_MODE_CLASS (GET_MODE (op0)) == MODE_INT
3851 10098 : && CONST_INT_P (op1))
3852 : {
3853 3768 : if (op1 == const0_rtx
3854 221 : && (code == LT || code == GE))
3855 : compare_code = code;
3856 3768 : else if (op1 == constm1_rtx)
3857 : {
3858 110 : if (code == LE)
3859 : compare_code = LT;
3860 110 : else if (code == GT)
3861 : compare_code = GE;
3862 : }
3863 : }
3864 :
3865 : /* Optimize dest = (op0 < 0) ? -1 : cf. */
3866 : if (compare_code != UNKNOWN
3867 0 : && GET_MODE (op0) == GET_MODE (out)
3868 0 : && (cf == -1 || ct == -1))
3869 : {
3870 : /* If lea code below could be used, only optimize
3871 : if it results in a 2 insn sequence. */
3872 :
3873 0 : if (! (diff == 1 || diff == 2 || diff == 4 || diff == 8
3874 0 : || diff == 3 || diff == 5 || diff == 9)
3875 0 : || (compare_code == LT && ct == -1)
3876 0 : || (compare_code == GE && cf == -1))
3877 : {
3878 : /*
3879 : * notl op1 (if necessary)
3880 : * sarl $31, op1
3881 : * orl cf, op1
3882 : */
3883 0 : if (ct != -1)
3884 : {
3885 0 : cf = ct;
3886 0 : ct = -1;
3887 0 : code = reverse_condition (code);
3888 : }
3889 :
3890 0 : out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, -1);
3891 :
3892 0 : out = expand_simple_binop (mode, IOR,
3893 : out, GEN_INT (cf),
3894 : out, 1, OPTAB_DIRECT);
3895 0 : if (out != operands[0])
3896 0 : emit_move_insn (operands[0], out);
3897 :
3898 : return true;
3899 : }
3900 : }
3901 :
3902 :
3903 15907 : if ((diff == 1 || diff == 2 || diff == 4 || diff == 8
3904 4081 : || diff == 3 || diff == 5 || diff == 9)
3905 8304 : && ((mode != QImode && mode != HImode) || !TARGET_PARTIAL_REG_STALL)
3906 20130 : && (mode != DImode
3907 2101 : || x86_64_immediate_operand (GEN_INT (cf), VOIDmode)))
3908 : {
3909 : /*
3910 : * xorl dest,dest
3911 : * cmpl op1,op2
3912 : * setcc dest
3913 : * lea cf(dest*(ct-cf)),dest
3914 : *
3915 : * Size 14.
3916 : *
3917 : * This also catches the degenerate setcc-only case.
3918 : */
3919 :
3920 8304 : rtx tmp;
3921 8304 : int nops;
3922 :
3923 8304 : out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, 1);
3924 :
3925 8304 : nops = 0;
3926 : /* On x86_64 the lea instruction operates on Pmode, so we need
3927 : to get arithmetics done in proper mode to match. */
3928 8304 : if (diff == 1)
3929 6894 : tmp = copy_rtx (out);
3930 : else
3931 : {
3932 1410 : rtx out1;
3933 1410 : out1 = copy_rtx (out);
3934 1410 : tmp = gen_rtx_MULT (mode, out1, GEN_INT (diff & ~1));
3935 1410 : nops++;
3936 1410 : if (diff & 1)
3937 : {
3938 440 : tmp = gen_rtx_PLUS (mode, tmp, out1);
3939 440 : nops++;
3940 : }
3941 : }
3942 8304 : if (cf != 0)
3943 : {
3944 7130 : tmp = plus_constant (mode, tmp, cf);
3945 7130 : nops++;
3946 : }
3947 8304 : if (!rtx_equal_p (tmp, out))
3948 : {
3949 7592 : if (nops == 1)
3950 6338 : out = force_operand (tmp, copy_rtx (out));
3951 : else
3952 1254 : emit_insn (gen_rtx_SET (copy_rtx (out), copy_rtx (tmp)));
3953 : }
3954 8304 : if (!rtx_equal_p (out, operands[0]))
3955 620 : emit_move_insn (operands[0], copy_rtx (out));
3956 :
3957 : return true;
3958 : }
3959 :
3960 : /*
3961 : * General case: Jumpful:
3962 : * xorl dest,dest cmpl op1, op2
3963 : * cmpl op1, op2 movl ct, dest
3964 : * setcc dest jcc 1f
3965 : * decl dest movl cf, dest
3966 : * andl (cf-ct),dest 1:
3967 : * addl ct,dest
3968 : *
3969 : * Size 20. Size 14.
3970 : *
3971 : * This is reasonably steep, but branch mispredict costs are
3972 : * high on modern cpus, so consider failing only if optimizing
3973 : * for space.
3974 : */
3975 :
3976 3522 : if ((!TARGET_CMOVE || (mode == QImode && TARGET_PARTIAL_REG_STALL))
3977 3522 : && BRANCH_COST (optimize_insn_for_speed_p (),
3978 : false) >= 2)
3979 : {
3980 0 : if (cf == 0)
3981 : {
3982 0 : machine_mode cmp_mode = GET_MODE (op0);
3983 0 : enum rtx_code new_code;
3984 :
3985 0 : if (SCALAR_FLOAT_MODE_P (cmp_mode))
3986 : {
3987 0 : gcc_assert (!DECIMAL_FLOAT_MODE_P (cmp_mode));
3988 :
3989 : /* We may be reversing a non-trapping
3990 : comparison to a trapping comparison. */
3991 0 : if (HONOR_NANS (cmp_mode) && flag_trapping_math
3992 0 : && code != EQ && code != NE
3993 0 : && code != ORDERED && code != UNORDERED)
3994 : new_code = UNKNOWN;
3995 : else
3996 0 : new_code = reverse_condition_maybe_unordered (code);
3997 :
3998 : }
3999 : else
4000 : {
4001 0 : new_code = ix86_reverse_condition (code, cmp_mode);
4002 0 : if (compare_code != UNKNOWN && new_code != UNKNOWN)
4003 0 : compare_code = reverse_condition (compare_code);
4004 : }
4005 :
4006 0 : if (new_code != UNKNOWN)
4007 : {
4008 0 : cf = ct;
4009 0 : ct = 0;
4010 0 : code = new_code;
4011 : }
4012 : }
4013 :
4014 0 : if (compare_code != UNKNOWN)
4015 : {
4016 : /* notl op1 (if needed)
4017 : sarl $31, op1
4018 : andl (cf-ct), op1
4019 : addl ct, op1
4020 :
4021 : For x < 0 (resp. x <= -1) there will be no notl,
4022 : so if possible swap the constants to get rid of the
4023 : complement.
4024 : True/false will be -1/0 while code below (store flag
4025 : followed by decrement) is 0/-1, so the constants need
4026 : to be exchanged once more. */
4027 :
4028 0 : if (compare_code == GE || !cf)
4029 : {
4030 0 : code = reverse_condition (code);
4031 0 : compare_code = LT;
4032 : }
4033 : else
4034 : std::swap (ct, cf);
4035 :
4036 0 : out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, -1);
4037 : }
4038 : else
4039 : {
4040 0 : out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, 1);
4041 :
4042 0 : out = expand_simple_binop (mode, PLUS, copy_rtx (out),
4043 : constm1_rtx,
4044 : copy_rtx (out), 1, OPTAB_DIRECT);
4045 : }
4046 :
4047 0 : HOST_WIDE_INT ival = (unsigned HOST_WIDE_INT) cf - ct;
4048 : /* Make sure we can represent the difference
4049 : between the two values. */
4050 0 : if ((ival > 0) != ((ct < 0) != (cf < 0) ? ct < 0 : ct < cf))
4051 : return false;
4052 :
4053 0 : out = expand_simple_binop (mode, AND, copy_rtx (out),
4054 0 : gen_int_mode (ival, mode),
4055 : copy_rtx (out), 1, OPTAB_DIRECT);
4056 0 : if (ct)
4057 0 : out = expand_simple_binop (mode, PLUS, copy_rtx (out), GEN_INT (ct),
4058 : copy_rtx (out), 1, OPTAB_DIRECT);
4059 0 : if (!rtx_equal_p (out, operands[0]))
4060 0 : emit_move_insn (operands[0], copy_rtx (out));
4061 :
4062 : return true;
4063 : }
4064 : }
4065 :
4066 487443 : if (!TARGET_CMOVE || (mode == QImode && TARGET_PARTIAL_REG_STALL))
4067 : {
4068 : /* Try a few things more with specific constants and a variable. */
4069 :
4070 0 : optab op;
4071 0 : rtx var, orig_out, out, tmp;
4072 :
4073 0 : if (BRANCH_COST (optimize_insn_for_speed_p (), false) <= 2)
4074 : return false;
4075 :
4076 0 : operands[2] = op2;
4077 0 : operands[3] = op3;
4078 :
4079 : /* If one of the two operands is an interesting constant, load a
4080 : constant with the above and mask it in with a logical operation. */
4081 :
4082 0 : if (CONST_INT_P (operands[2]))
4083 : {
4084 0 : var = operands[3];
4085 0 : if (INTVAL (operands[2]) == 0 && operands[3] != constm1_rtx)
4086 0 : operands[3] = constm1_rtx, op = and_optab;
4087 0 : else if (INTVAL (operands[2]) == -1 && operands[3] != const0_rtx)
4088 0 : operands[3] = const0_rtx, op = ior_optab;
4089 : else
4090 : return false;
4091 : }
4092 0 : else if (CONST_INT_P (operands[3]))
4093 : {
4094 0 : var = operands[2];
4095 0 : if (INTVAL (operands[3]) == 0 && operands[2] != constm1_rtx)
4096 : {
4097 : /* For smin (x, 0), expand as "x < 0 ? x : 0" instead of
4098 : "x <= 0 ? x : 0" to enable sign_bit_compare_p. */
4099 0 : if (code == LE && op1 == const0_rtx && rtx_equal_p (op0, var))
4100 0 : operands[1] = simplify_gen_relational (LT, VOIDmode,
4101 0 : GET_MODE (op0),
4102 : op0, const0_rtx);
4103 :
4104 0 : operands[2] = constm1_rtx;
4105 0 : op = and_optab;
4106 : }
4107 0 : else if (INTVAL (operands[3]) == -1 && operands[3] != const0_rtx)
4108 0 : operands[2] = const0_rtx, op = ior_optab;
4109 : else
4110 : return false;
4111 : }
4112 : else
4113 : return false;
4114 :
4115 0 : orig_out = operands[0];
4116 0 : tmp = gen_reg_rtx (mode);
4117 0 : operands[0] = tmp;
4118 :
4119 : /* Recurse to get the constant loaded. */
4120 0 : if (!ix86_expand_int_movcc (operands))
4121 : return false;
4122 :
4123 : /* Mask in the interesting variable. */
4124 0 : out = expand_binop (mode, op, var, tmp, orig_out, 0,
4125 : OPTAB_WIDEN);
4126 0 : if (!rtx_equal_p (out, orig_out))
4127 0 : emit_move_insn (copy_rtx (orig_out), copy_rtx (out));
4128 :
4129 : return true;
4130 : }
4131 :
4132 : /*
4133 : * For comparison with above,
4134 : *
4135 : * movl cf,dest
4136 : * movl ct,tmp
4137 : * cmpl op1,op2
4138 : * cmovcc tmp,dest
4139 : *
4140 : * Size 15.
4141 : */
4142 :
4143 487443 : if (! nonimmediate_operand (operands[2], mode))
4144 17577 : operands[2] = force_reg (mode, operands[2]);
4145 487443 : if (! nonimmediate_operand (operands[3], mode))
4146 171361 : operands[3] = force_reg (mode, operands[3]);
4147 :
4148 487443 : if (! register_operand (operands[2], VOIDmode)
4149 487443 : && (mode == QImode
4150 1093 : || ! register_operand (operands[3], VOIDmode)))
4151 1564 : operands[2] = force_reg (mode, operands[2]);
4152 :
4153 487443 : if (mode == QImode
4154 487443 : && ! register_operand (operands[3], VOIDmode))
4155 592 : operands[3] = force_reg (mode, operands[3]);
4156 :
4157 487443 : emit_insn (compare_seq);
4158 487443 : emit_insn (gen_rtx_SET (operands[0],
4159 : gen_rtx_IF_THEN_ELSE (mode,
4160 : compare_op, operands[2],
4161 : operands[3])));
4162 487443 : return true;
4163 : }
4164 :
4165 : /* Detect conditional moves that exactly match min/max operational
4166 : semantics. Note that this is IEEE safe, as long as we don't
4167 : interchange the operands.
4168 :
4169 : Returns FALSE if this conditional move doesn't match a MIN/MAX,
4170 : and TRUE if the operation is successful and instructions are emitted. */
4171 :
4172 : static bool
4173 12368 : ix86_expand_sse_fp_minmax (rtx dest, enum rtx_code code, rtx cmp_op0,
4174 : rtx cmp_op1, rtx if_true, rtx if_false)
4175 : {
4176 12368 : machine_mode mode = GET_MODE (dest);
4177 12368 : bool is_min;
4178 12368 : rtx tmp;
4179 :
4180 12368 : if (code == LT)
4181 : ;
4182 3304 : else if (code == LE && !HONOR_NANS (mode))
4183 : {
4184 : /* We can swap LE to GE and then invert to LT. */
4185 : std::swap (cmp_op0, cmp_op1);
4186 : std::swap (if_true, if_false);
4187 : }
4188 3263 : else if (code == UNGE)
4189 : std::swap (if_true, if_false);
4190 : else
4191 : return false;
4192 :
4193 11222 : if (rtx_equal_p (cmp_op0, if_true) && rtx_equal_p (cmp_op1, if_false))
4194 : is_min = true;
4195 7220 : else if (rtx_equal_p (cmp_op1, if_true) && rtx_equal_p (cmp_op0, if_false))
4196 : is_min = false;
4197 : else
4198 : return false;
4199 :
4200 7596 : if (immediate_operand (if_false, mode))
4201 8 : if_false = force_reg (mode, if_false);
4202 7596 : if (immediate_operand (if_true, mode))
4203 0 : if_true = force_reg (mode, if_true);
4204 :
4205 : /* We want to check HONOR_NANS and HONOR_SIGNED_ZEROS here,
4206 : but MODE may be a vector mode and thus not appropriate. */
4207 7596 : if (!flag_finite_math_only || flag_signed_zeros)
4208 : {
4209 7596 : int u = is_min ? UNSPEC_IEEE_MIN : UNSPEC_IEEE_MAX;
4210 7596 : rtvec v;
4211 :
4212 7596 : if_true = force_reg (mode, if_true);
4213 7596 : v = gen_rtvec (2, if_true, if_false);
4214 7596 : tmp = gen_rtx_UNSPEC (mode, v, u);
4215 7596 : }
4216 : else
4217 : {
4218 0 : code = is_min ? SMIN : SMAX;
4219 0 : if (MEM_P (if_true) && MEM_P (if_false))
4220 0 : if_true = force_reg (mode, if_true);
4221 0 : tmp = gen_rtx_fmt_ee (code, mode, if_true, if_false);
4222 : }
4223 :
4224 7596 : emit_insn (gen_rtx_SET (dest, tmp));
4225 7596 : return true;
4226 : }
4227 :
4228 : /* Return true if MODE is valid for vector compare to mask register,
4229 : Same result for conditionl vector move with mask register. */
4230 : static bool
4231 14928 : ix86_valid_mask_cmp_mode (machine_mode mode)
4232 : {
4233 : /* XOP has its own vector conditional movement. */
4234 14928 : if (TARGET_XOP && !TARGET_AVX512F)
4235 : return false;
4236 :
4237 : /* HFmode only supports vcmpsh whose dest is mask register. */
4238 14922 : if (TARGET_AVX512FP16 && mode == HFmode)
4239 : return true;
4240 :
4241 : /* AVX512F is needed for mask operation. */
4242 14830 : if (!(TARGET_AVX512F && VECTOR_MODE_P (mode)))
4243 : return false;
4244 :
4245 : /* AVX512BW is needed for vector QI/HImode,
4246 : AVX512VL is needed for 128/256-bit vector. */
4247 182 : machine_mode inner_mode = GET_MODE_INNER (mode);
4248 182 : int vector_size = GET_MODE_SIZE (mode);
4249 182 : if ((inner_mode == QImode || inner_mode == HImode) && !TARGET_AVX512BW)
4250 : return false;
4251 :
4252 162 : return vector_size == 64 || TARGET_AVX512VL;
4253 : }
4254 :
4255 : /* Return true if integer mask comparison should be used. */
4256 : static bool
4257 59593 : ix86_use_mask_cmp_p (machine_mode mode, machine_mode cmp_mode,
4258 : rtx op_true, rtx op_false)
4259 : {
4260 59593 : int vector_size = GET_MODE_SIZE (mode);
4261 :
4262 59593 : if (cmp_mode == HFmode)
4263 : return true;
4264 59501 : else if (vector_size < 16)
4265 : return false;
4266 49758 : else if (vector_size == 64)
4267 : return true;
4268 99400 : else if (GET_MODE_INNER (cmp_mode) == HFmode)
4269 : return true;
4270 99400 : else if (GET_MODE_INNER (cmp_mode) == BFmode)
4271 : return true;
4272 :
4273 : /* When op_true is NULL, op_false must be NULL, or vice versa. */
4274 49700 : gcc_assert (!op_true == !op_false);
4275 :
4276 : /* When op_true/op_false is NULL or cmp_mode is not valid mask cmp mode,
4277 : vector dest is required. */
4278 49700 : if (!op_true || !ix86_valid_mask_cmp_mode (cmp_mode))
4279 : return false;
4280 :
4281 : /* Exclude those that could be optimized in ix86_expand_sse_movcc. */
4282 48 : if (op_false == CONST0_RTX (mode)
4283 48 : || op_true == CONST0_RTX (mode)
4284 48 : || (INTEGRAL_MODE_P (mode)
4285 40 : && (op_true == CONSTM1_RTX (mode)
4286 40 : || op_false == CONSTM1_RTX (mode))))
4287 0 : return false;
4288 :
4289 : return true;
4290 : }
4291 :
4292 : /* Expand an SSE comparison. Return the register with the result. */
4293 :
4294 : static rtx
4295 41018 : ix86_expand_sse_cmp (rtx dest, enum rtx_code code, rtx cmp_op0, rtx cmp_op1,
4296 : rtx op_true, rtx op_false)
4297 : {
4298 41018 : machine_mode mode = GET_MODE (dest);
4299 41018 : machine_mode cmp_ops_mode = GET_MODE (cmp_op0);
4300 :
4301 : /* In general case result of comparison can differ from operands' type. */
4302 41018 : machine_mode cmp_mode;
4303 :
4304 : /* In AVX512F the result of comparison is an integer mask. */
4305 41018 : bool maskcmp = false;
4306 41018 : rtx x;
4307 :
4308 41018 : if (ix86_use_mask_cmp_p (mode, cmp_ops_mode, op_true, op_false))
4309 : {
4310 145 : unsigned int nbits = GET_MODE_NUNITS (cmp_ops_mode);
4311 145 : maskcmp = true;
4312 145 : cmp_mode = nbits > 8 ? int_mode_for_size (nbits, 0).require () : E_QImode;
4313 : }
4314 : else
4315 : cmp_mode = cmp_ops_mode;
4316 :
4317 41018 : cmp_op0 = force_reg (cmp_ops_mode, cmp_op0);
4318 :
4319 82036 : bool (*op1_predicate)(rtx, machine_mode)
4320 41018 : = VECTOR_MODE_P (cmp_ops_mode) ? vector_operand : nonimmediate_operand;
4321 :
4322 41018 : if (!op1_predicate (cmp_op1, cmp_ops_mode))
4323 0 : cmp_op1 = force_reg (cmp_ops_mode, cmp_op1);
4324 :
4325 41018 : if (optimize
4326 503 : || (maskcmp && cmp_mode != mode)
4327 503 : || (op_true && reg_overlap_mentioned_p (dest, op_true))
4328 41521 : || (op_false && reg_overlap_mentioned_p (dest, op_false)))
4329 80885 : dest = gen_reg_rtx (maskcmp ? cmp_mode : mode);
4330 :
4331 41018 : if (maskcmp)
4332 : {
4333 145 : bool ok = ix86_expand_mask_vec_cmp (dest, code, cmp_op0, cmp_op1);
4334 145 : gcc_assert (ok);
4335 : return dest;
4336 : }
4337 :
4338 40873 : x = gen_rtx_fmt_ee (code, cmp_mode, cmp_op0, cmp_op1);
4339 :
4340 40873 : if (cmp_mode != mode)
4341 : {
4342 8090 : x = force_reg (cmp_ops_mode, x);
4343 8090 : convert_move (dest, x, false);
4344 : }
4345 : else
4346 32783 : emit_insn (gen_rtx_SET (dest, x));
4347 :
4348 : return dest;
4349 : }
4350 :
4351 : /* Emit x86 binary operand CODE in mode MODE for SSE vector
4352 : instructions that can be performed using GP registers. */
4353 :
4354 : static void
4355 9673 : ix86_emit_vec_binop (enum rtx_code code, machine_mode mode,
4356 : rtx dst, rtx src1, rtx src2)
4357 : {
4358 9673 : rtx tmp;
4359 :
4360 9673 : tmp = gen_rtx_SET (dst, gen_rtx_fmt_ee (code, mode, src1, src2));
4361 :
4362 9673 : if (GET_MODE_SIZE (mode) <= GET_MODE_SIZE (SImode)
4363 9673 : && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
4364 : {
4365 94 : rtx clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
4366 94 : tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, clob));
4367 : }
4368 :
4369 9673 : emit_insn (tmp);
4370 9673 : }
4371 :
4372 : /* Expand DEST = CMP ? OP_TRUE : OP_FALSE into a sequence of logical
4373 : operations. This is used for both scalar and vector conditional moves. */
4374 :
4375 : void
4376 13282 : ix86_expand_sse_movcc (rtx dest, rtx cmp, rtx op_true, rtx op_false)
4377 : {
4378 13282 : machine_mode mode = GET_MODE (dest);
4379 13282 : machine_mode cmpmode = GET_MODE (cmp);
4380 13282 : rtx x;
4381 :
4382 : /* Simplify trivial VEC_COND_EXPR to avoid ICE in pr97506. */
4383 13282 : if (rtx_equal_p (op_true, op_false))
4384 : {
4385 0 : emit_move_insn (dest, op_true);
4386 0 : return;
4387 : }
4388 :
4389 : /* If we have an integer mask and FP value then we need
4390 : to cast mask to FP mode. */
4391 13282 : if (mode != cmpmode && VECTOR_MODE_P (cmpmode))
4392 : {
4393 1652 : cmp = force_reg (cmpmode, cmp);
4394 1652 : cmp = gen_rtx_SUBREG (mode, cmp, 0);
4395 : }
4396 :
4397 : /* In AVX512F the result of comparison is an integer mask. */
4398 13282 : if (mode != cmpmode
4399 1797 : && GET_MODE_CLASS (cmpmode) == MODE_INT)
4400 : {
4401 145 : gcc_assert (ix86_valid_mask_cmp_mode (mode));
4402 : /* Using scalar/vector move with mask register. */
4403 145 : cmp = force_reg (cmpmode, cmp);
4404 : /* Optimize for mask zero. */
4405 290 : op_true = (op_true != CONST0_RTX (mode)
4406 145 : ? force_reg (mode, op_true) : op_true);
4407 290 : op_false = (op_false != CONST0_RTX (mode)
4408 145 : ? force_reg (mode, op_false) : op_false);
4409 145 : if (op_true == CONST0_RTX (mode))
4410 : {
4411 0 : if (cmpmode == E_DImode && !TARGET_64BIT)
4412 : {
4413 0 : x = gen_reg_rtx (cmpmode);
4414 0 : emit_insn (gen_knotdi (x, cmp));
4415 : }
4416 : else
4417 0 : x = expand_simple_unop (cmpmode, NOT, cmp, NULL, 1);
4418 : cmp = x;
4419 : /* Reverse op_true op_false. */
4420 : std::swap (op_true, op_false);
4421 : }
4422 :
4423 145 : if (mode == HFmode)
4424 92 : emit_insn (gen_movhf_mask (dest, op_true, op_false, cmp));
4425 : else
4426 53 : emit_insn (gen_rtx_SET (dest,
4427 : gen_rtx_VEC_MERGE (mode,
4428 : op_true, op_false, cmp)));
4429 : return;
4430 : }
4431 :
4432 13137 : if (vector_all_ones_operand (op_true, mode)
4433 13137 : && op_false == CONST0_RTX (mode))
4434 : {
4435 3 : emit_move_insn (dest, cmp);
4436 3 : return;
4437 : }
4438 13134 : else if (op_false == CONST0_RTX (mode))
4439 : {
4440 967 : x = expand_simple_binop (mode, AND, cmp, op_true,
4441 : dest, 1, OPTAB_DIRECT);
4442 967 : if (x != dest)
4443 0 : emit_move_insn (dest, x);
4444 : return;
4445 : }
4446 12167 : else if (op_true == CONST0_RTX (mode))
4447 : {
4448 90 : op_false = force_reg (mode, op_false);
4449 90 : x = gen_rtx_NOT (mode, cmp);
4450 90 : ix86_emit_vec_binop (AND, mode, dest, x, op_false);
4451 90 : return;
4452 : }
4453 12077 : else if (vector_all_ones_operand (op_true, mode))
4454 : {
4455 3 : x = expand_simple_binop (mode, IOR, cmp, op_false,
4456 : dest, 1, OPTAB_DIRECT);
4457 3 : if (x != dest)
4458 0 : emit_move_insn (dest, x);
4459 : return;
4460 : }
4461 :
4462 12074 : if (TARGET_XOP)
4463 : {
4464 65 : op_true = force_reg (mode, op_true);
4465 :
4466 65 : if (GET_MODE_SIZE (mode) < 16
4467 65 : || !nonimmediate_operand (op_false, mode))
4468 50 : op_false = force_reg (mode, op_false);
4469 :
4470 65 : emit_insn (gen_rtx_SET (dest,
4471 : gen_rtx_IF_THEN_ELSE (mode, cmp,
4472 : op_true, op_false)));
4473 65 : return;
4474 : }
4475 :
4476 12009 : rtx (*gen) (rtx, rtx, rtx, rtx) = NULL;
4477 12009 : machine_mode blend_mode = mode;
4478 :
4479 12009 : switch (mode)
4480 : {
4481 43 : case E_V2SFmode:
4482 43 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4483 : gen = gen_mmx_blendvps;
4484 : break;
4485 373 : case E_V4SFmode:
4486 373 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4487 : gen = gen_sse4_1_blendvps;
4488 : break;
4489 236 : case E_V2DFmode:
4490 236 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4491 : gen = gen_sse4_1_blendvpd;
4492 : break;
4493 3440 : case E_SFmode:
4494 3440 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4495 : gen = gen_sse4_1_blendvss;
4496 : break;
4497 1102 : case E_DFmode:
4498 1102 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4499 : gen = gen_sse4_1_blendvsd;
4500 : break;
4501 263 : case E_V8QImode:
4502 263 : case E_V4HImode:
4503 263 : case E_V4HFmode:
4504 263 : case E_V4BFmode:
4505 263 : case E_V2SImode:
4506 263 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4507 : {
4508 : gen = gen_mmx_pblendvb_v8qi;
4509 : blend_mode = V8QImode;
4510 : }
4511 : break;
4512 95 : case E_V4QImode:
4513 95 : case E_V2HImode:
4514 95 : case E_V2HFmode:
4515 95 : case E_V2BFmode:
4516 95 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4517 : {
4518 : gen = gen_mmx_pblendvb_v4qi;
4519 : blend_mode = V4QImode;
4520 : }
4521 : break;
4522 36 : case E_V2QImode:
4523 36 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4524 : gen = gen_mmx_pblendvb_v2qi;
4525 : break;
4526 5768 : case E_V16QImode:
4527 5768 : case E_V8HImode:
4528 5768 : case E_V8HFmode:
4529 5768 : case E_V8BFmode:
4530 5768 : case E_V4SImode:
4531 5768 : case E_V2DImode:
4532 5768 : case E_V1TImode:
4533 5768 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4534 : {
4535 : gen = gen_sse4_1_pblendvb;
4536 : blend_mode = V16QImode;
4537 : }
4538 : break;
4539 83 : case E_V8SFmode:
4540 83 : if (TARGET_AVX && TARGET_SSE_MOVCC_USE_BLENDV)
4541 : gen = gen_avx_blendvps256;
4542 : break;
4543 192 : case E_V4DFmode:
4544 192 : if (TARGET_AVX && TARGET_SSE_MOVCC_USE_BLENDV)
4545 : gen = gen_avx_blendvpd256;
4546 : break;
4547 378 : case E_V32QImode:
4548 378 : case E_V16HImode:
4549 378 : case E_V16HFmode:
4550 378 : case E_V16BFmode:
4551 378 : case E_V8SImode:
4552 378 : case E_V4DImode:
4553 378 : if (TARGET_AVX2 && TARGET_SSE_MOVCC_USE_BLENDV)
4554 : {
4555 : gen = gen_avx2_pblendvb;
4556 : blend_mode = V32QImode;
4557 : }
4558 : break;
4559 :
4560 0 : case E_V64QImode:
4561 0 : gen = gen_avx512bw_blendmv64qi;
4562 0 : break;
4563 0 : case E_V32HImode:
4564 0 : gen = gen_avx512bw_blendmv32hi;
4565 0 : break;
4566 0 : case E_V32HFmode:
4567 0 : gen = gen_avx512bw_blendmv32hf;
4568 0 : break;
4569 0 : case E_V32BFmode:
4570 0 : gen = gen_avx512bw_blendmv32bf;
4571 0 : break;
4572 0 : case E_V16SImode:
4573 0 : gen = gen_avx512f_blendmv16si;
4574 0 : break;
4575 0 : case E_V8DImode:
4576 0 : gen = gen_avx512f_blendmv8di;
4577 0 : break;
4578 0 : case E_V8DFmode:
4579 0 : gen = gen_avx512f_blendmv8df;
4580 0 : break;
4581 : case E_V16SFmode:
4582 : gen = gen_avx512f_blendmv16sf;
4583 : break;
4584 :
4585 : default:
4586 : break;
4587 : }
4588 :
4589 0 : if (gen != NULL)
4590 : {
4591 2083 : if (GET_MODE_SIZE (mode) < 16
4592 2083 : || !vector_operand (op_true, mode))
4593 559 : op_true = force_reg (mode, op_true);
4594 2083 : op_false = force_reg (mode, op_false);
4595 :
4596 2083 : if (blend_mode == mode)
4597 : x = dest;
4598 : else
4599 : {
4600 1015 : x = gen_reg_rtx (blend_mode);
4601 1015 : op_false = gen_lowpart (blend_mode, op_false);
4602 1015 : op_true = gen_lowpart (blend_mode, op_true);
4603 1015 : cmp = gen_lowpart (blend_mode, cmp);
4604 : }
4605 :
4606 2083 : emit_insn (gen (x, op_false, op_true, cmp));
4607 :
4608 2083 : if (x != dest)
4609 1015 : emit_move_insn (dest, gen_lowpart (mode, x));
4610 : }
4611 9926 : else if (CONST_VECTOR_P (op_true) && CONST_VECTOR_P (op_false))
4612 : {
4613 444 : rtx tmp = simplify_const_binary_operation (XOR, mode, op_true, op_false);
4614 444 : tmp = expand_simple_binop (mode, AND, cmp, tmp,
4615 : NULL, 1, OPTAB_DIRECT);
4616 444 : tmp = expand_simple_binop (mode, XOR, tmp, op_false,
4617 : dest, 1, OPTAB_DIRECT);
4618 444 : if (tmp != dest)
4619 0 : emit_move_insn (dest, tmp);
4620 : }
4621 : else
4622 : {
4623 9482 : rtx t2 = expand_simple_binop (mode, AND, cmp, op_true,
4624 : NULL, 1, OPTAB_DIRECT);
4625 :
4626 9482 : rtx t3 = gen_reg_rtx (mode);
4627 9482 : x = gen_rtx_NOT (mode, cmp);
4628 9482 : op_false = force_reg (mode, op_false);
4629 9482 : ix86_emit_vec_binop (AND, mode, t3, x, op_false);
4630 :
4631 9482 : x = expand_simple_binop (mode, IOR, t3, t2,
4632 : dest, 1, OPTAB_DIRECT);
4633 9482 : if (x != dest)
4634 0 : emit_move_insn (dest, x);
4635 : }
4636 : }
4637 :
4638 : /* Swap, force into registers, or otherwise massage the two operands
4639 : to an sse comparison with a mask result. Thus we differ a bit from
4640 : ix86_prepare_fp_compare_args which expects to produce a flags result.
4641 :
4642 : The DEST operand exists to help determine whether to commute commutative
4643 : operators. The POP0/POP1 operands are updated in place. The new
4644 : comparison code is returned, or UNKNOWN if not implementable. */
4645 :
4646 : static enum rtx_code
4647 20442 : ix86_prepare_sse_fp_compare_args (rtx dest, enum rtx_code code,
4648 : rtx *pop0, rtx *pop1)
4649 : {
4650 20442 : switch (code)
4651 : {
4652 54 : case LTGT:
4653 54 : case UNEQ:
4654 : /* AVX supports all the needed comparisons. */
4655 54 : if (TARGET_AVX)
4656 : break;
4657 : /* We have no LTGT as an operator. We could implement it with
4658 : NE & ORDERED, but this requires an extra temporary. It's
4659 : not clear that it's worth it. */
4660 : return UNKNOWN;
4661 :
4662 : case LT:
4663 : case LE:
4664 : case UNGT:
4665 : case UNGE:
4666 : /* These are supported directly. */
4667 : break;
4668 :
4669 6011 : case EQ:
4670 6011 : case NE:
4671 6011 : case UNORDERED:
4672 6011 : case ORDERED:
4673 : /* AVX has 3 operand comparisons, no need to swap anything. */
4674 6011 : if (TARGET_AVX)
4675 : break;
4676 : /* For commutative operators, try to canonicalize the destination
4677 : operand to be first in the comparison - this helps reload to
4678 : avoid extra moves. */
4679 1412 : if (!dest || !rtx_equal_p (dest, *pop1))
4680 : break;
4681 : /* FALLTHRU */
4682 :
4683 13276 : case GE:
4684 13276 : case GT:
4685 13276 : case UNLE:
4686 13276 : case UNLT:
4687 : /* These are not supported directly before AVX, and furthermore
4688 : ix86_expand_sse_fp_minmax only optimizes LT/UNGE. Swap the
4689 : comparison operands to transform into something that is
4690 : supported. */
4691 13276 : std::swap (*pop0, *pop1);
4692 13276 : code = swap_condition (code);
4693 13276 : break;
4694 :
4695 0 : default:
4696 0 : gcc_unreachable ();
4697 : }
4698 :
4699 : return code;
4700 : }
4701 :
4702 : /* Expand a floating-point conditional move. Return true if successful. */
4703 :
4704 : bool
4705 81796 : ix86_expand_fp_movcc (rtx operands[])
4706 : {
4707 81796 : machine_mode mode = GET_MODE (operands[0]);
4708 81796 : enum rtx_code code = GET_CODE (operands[1]);
4709 81796 : rtx tmp, compare_op;
4710 81796 : rtx op0 = XEXP (operands[1], 0);
4711 81796 : rtx op1 = XEXP (operands[1], 1);
4712 :
4713 81796 : if (GET_MODE (op0) == BFmode
4714 81796 : && !ix86_fp_comparison_operator (operands[1], VOIDmode))
4715 : return false;
4716 :
4717 81796 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
4718 : {
4719 67318 : machine_mode cmode;
4720 :
4721 : /* Since we've no cmove for sse registers, don't force bad register
4722 : allocation just to gain access to it. Deny movcc when the
4723 : comparison mode doesn't match the move mode. */
4724 67318 : cmode = GET_MODE (op0);
4725 67318 : if (cmode == VOIDmode)
4726 0 : cmode = GET_MODE (op1);
4727 67318 : if (cmode != mode)
4728 : return false;
4729 :
4730 12378 : code = ix86_prepare_sse_fp_compare_args (operands[0], code, &op0, &op1);
4731 12378 : if (code == UNKNOWN)
4732 : return false;
4733 :
4734 12368 : if (ix86_expand_sse_fp_minmax (operands[0], code, op0, op1,
4735 : operands[2], operands[3]))
4736 : return true;
4737 :
4738 4772 : tmp = ix86_expand_sse_cmp (operands[0], code, op0, op1,
4739 : operands[2], operands[3]);
4740 4772 : ix86_expand_sse_movcc (operands[0], tmp, operands[2], operands[3]);
4741 4772 : return true;
4742 : }
4743 :
4744 14478 : if (GET_MODE (op0) == TImode
4745 14478 : || (GET_MODE (op0) == DImode
4746 72 : && !TARGET_64BIT))
4747 : return false;
4748 :
4749 : /* The floating point conditional move instructions don't directly
4750 : support conditions resulting from a signed integer comparison. */
4751 :
4752 14406 : compare_op = ix86_expand_compare (code, op0, op1);
4753 14406 : if (!fcmov_comparison_operator (compare_op, VOIDmode))
4754 : {
4755 146 : tmp = gen_reg_rtx (QImode);
4756 146 : ix86_expand_setcc (tmp, code, op0, op1);
4757 :
4758 146 : compare_op = ix86_expand_compare (NE, tmp, const0_rtx);
4759 : }
4760 :
4761 14406 : operands[2] = force_reg (mode, operands[2]);
4762 14406 : operands[3] = force_reg (mode, operands[3]);
4763 14406 : emit_insn (gen_rtx_SET (operands[0],
4764 : gen_rtx_IF_THEN_ELSE (mode, compare_op,
4765 : operands[2], operands[3])));
4766 :
4767 14406 : return true;
4768 : }
4769 :
4770 : /* Expand a conditional fma: operands[0] = operands[1]
4771 : ? fma (operands[2], operands[3], operands[4]) : operands[5]. */
4772 :
4773 : void
4774 322 : ix86_expand_cond_fma (rtx operands[], bool neg_mult, bool neg_add)
4775 : {
4776 322 : machine_mode mode = GET_MODE (operands[0]);
4777 322 : rtx mask = operands[1];
4778 322 : rtx op1 = operands[2], op2 = operands[3], op3 = operands[4];
4779 322 : rtx op4 = operands[5];
4780 322 : rtx tmp, fma;
4781 :
4782 : /* The multiplication is commutative, so an else value equal to the
4783 : second operand for mult can be merged just like first operand
4784 : after swapping. */
4785 322 : if (rtx_equal_p (op4, op2))
4786 54 : std::swap (op1, op2);
4787 :
4788 472 : fma = gen_rtx_FMA (mode, neg_mult ? gen_rtx_NEG (mode, op1) : op1,
4789 : op2, neg_add ? gen_rtx_NEG (mode, op3) : op3);
4790 :
4791 : /* Do not emit an extra move. */
4792 322 : if (rtx_equal_p (op4, op1)
4793 246 : || rtx_equal_p (op4, op3)
4794 486 : || op4 == CONST0_RTX (mode))
4795 : {
4796 250 : emit_insn (gen_rtx_SET (operands[0],
4797 : gen_rtx_VEC_MERGE (mode, fma, op4, mask)));
4798 250 : return;
4799 : }
4800 :
4801 72 : tmp = gen_reg_rtx (mode);
4802 72 : emit_insn (gen_rtx_SET (tmp, gen_rtx_VEC_MERGE (mode, fma,
4803 : CONST0_RTX (mode), mask)));
4804 72 : emit_move_insn (operands[0], gen_rtx_VEC_MERGE (mode, tmp, op4, mask));
4805 : }
4806 :
4807 : /* Helper for ix86_cmp_code_to_pcmp_immediate for int modes. */
4808 :
4809 : static int
4810 4985 : ix86_int_cmp_code_to_pcmp_immediate (enum rtx_code code)
4811 : {
4812 4985 : switch (code)
4813 : {
4814 : case EQ:
4815 : return 0;
4816 380 : case LT:
4817 380 : case LTU:
4818 380 : return 1;
4819 220 : case LE:
4820 220 : case LEU:
4821 220 : return 2;
4822 3165 : case NE:
4823 3165 : return 4;
4824 299 : case GE:
4825 299 : case GEU:
4826 299 : return 5;
4827 497 : case GT:
4828 497 : case GTU:
4829 497 : return 6;
4830 0 : default:
4831 0 : gcc_unreachable ();
4832 : }
4833 : }
4834 :
4835 : /* Helper for ix86_cmp_code_to_pcmp_immediate for fp modes. */
4836 :
4837 : static int
4838 1822 : ix86_fp_cmp_code_to_pcmp_immediate (enum rtx_code code)
4839 : {
4840 1822 : switch (code)
4841 : {
4842 : case EQ:
4843 : return 0x00;
4844 374 : case NE:
4845 374 : return 0x04;
4846 510 : case GT:
4847 510 : return 0x0e;
4848 88 : case LE:
4849 88 : return 0x02;
4850 53 : case GE:
4851 53 : return 0x0d;
4852 638 : case LT:
4853 638 : return 0x01;
4854 3 : case UNLE:
4855 3 : return 0x1a;
4856 3 : case UNLT:
4857 3 : return 0x19;
4858 12 : case UNGE:
4859 12 : return 0x15;
4860 45 : case UNGT:
4861 45 : return 0x16;
4862 3 : case UNEQ:
4863 3 : return 0x08;
4864 2 : case LTGT:
4865 2 : return 0x1c;
4866 2 : case ORDERED:
4867 2 : return 0x07;
4868 2 : case UNORDERED:
4869 2 : return 0x03;
4870 0 : default:
4871 0 : gcc_unreachable ();
4872 : }
4873 : }
4874 :
4875 : /* Return immediate value to be used in UNSPEC_PCMP
4876 : for comparison CODE in MODE. */
4877 :
4878 : static int
4879 6807 : ix86_cmp_code_to_pcmp_immediate (enum rtx_code code, machine_mode mode)
4880 : {
4881 6807 : if (FLOAT_MODE_P (mode))
4882 1822 : return ix86_fp_cmp_code_to_pcmp_immediate (code);
4883 4985 : return ix86_int_cmp_code_to_pcmp_immediate (code);
4884 : }
4885 :
4886 : /* Expand AVX-512 vector comparison. */
4887 :
4888 : bool
4889 6807 : ix86_expand_mask_vec_cmp (rtx dest, enum rtx_code code, rtx cmp_op0, rtx cmp_op1)
4890 : {
4891 6807 : machine_mode mask_mode = GET_MODE (dest);
4892 6807 : machine_mode cmp_mode = GET_MODE (cmp_op0);
4893 6807 : rtx imm = GEN_INT (ix86_cmp_code_to_pcmp_immediate (code, cmp_mode));
4894 6807 : int unspec_code;
4895 6807 : rtx unspec;
4896 :
4897 6807 : switch (code)
4898 : {
4899 : case LEU:
4900 : case GTU:
4901 : case GEU:
4902 : case LTU:
4903 : unspec_code = UNSPEC_UNSIGNED_PCMP;
4904 : break;
4905 :
4906 6398 : default:
4907 6398 : unspec_code = UNSPEC_PCMP;
4908 : }
4909 :
4910 6807 : unspec = gen_rtx_UNSPEC (mask_mode, gen_rtvec (3, cmp_op0, cmp_op1, imm),
4911 : unspec_code);
4912 6807 : emit_insn (gen_rtx_SET (dest, unspec));
4913 :
4914 6807 : return true;
4915 : }
4916 :
4917 : /* Expand fp vector comparison. */
4918 :
4919 : bool
4920 8064 : ix86_expand_fp_vec_cmp (rtx operands[])
4921 : {
4922 8064 : enum rtx_code code = GET_CODE (operands[1]);
4923 8064 : rtx cmp;
4924 :
4925 8064 : code = ix86_prepare_sse_fp_compare_args (operands[0], code,
4926 : &operands[2], &operands[3]);
4927 8064 : if (code == UNKNOWN)
4928 : {
4929 26 : rtx temp;
4930 26 : switch (GET_CODE (operands[1]))
4931 : {
4932 4 : case LTGT:
4933 4 : temp = ix86_expand_sse_cmp (operands[0], LT, operands[2],
4934 : operands[3], NULL, NULL);
4935 4 : cmp = ix86_expand_sse_cmp (operands[0], LT, operands[3],
4936 : operands[2], NULL, NULL);
4937 4 : code = IOR;
4938 4 : break;
4939 22 : case UNEQ:
4940 22 : temp = ix86_expand_sse_cmp (operands[0], UNORDERED, operands[2],
4941 : operands[3], NULL, NULL);
4942 22 : cmp = ix86_expand_sse_cmp (operands[0], EQ, operands[2],
4943 : operands[3], NULL, NULL);
4944 22 : code = IOR;
4945 22 : break;
4946 0 : default:
4947 0 : gcc_unreachable ();
4948 : }
4949 26 : cmp = expand_simple_binop (GET_MODE (cmp), code, temp, cmp, cmp, 1,
4950 : OPTAB_DIRECT);
4951 : }
4952 : else
4953 8038 : cmp = ix86_expand_sse_cmp (operands[0], code, operands[2], operands[3],
4954 : NULL, NULL);
4955 :
4956 8064 : if (operands[0] != cmp)
4957 7984 : emit_move_insn (operands[0], cmp);
4958 :
4959 8064 : return true;
4960 : }
4961 :
4962 : static rtx
4963 18719 : ix86_expand_int_sse_cmp (rtx dest, enum rtx_code code, rtx cop0, rtx cop1,
4964 : rtx op_true, rtx op_false, bool *negate)
4965 : {
4966 18719 : machine_mode data_mode = GET_MODE (dest);
4967 18719 : machine_mode mode = GET_MODE (cop0);
4968 18719 : rtx x;
4969 :
4970 18719 : *negate = false;
4971 :
4972 : /* XOP supports all of the comparisons on all 128-bit vector int types. */
4973 18719 : if (TARGET_XOP
4974 147 : && GET_MODE_CLASS (mode) == MODE_VECTOR_INT
4975 18866 : && GET_MODE_SIZE (mode) <= 16)
4976 : ;
4977 : /* AVX512F supports all of the comparisons
4978 : on all 128/256/512-bit vector int types. */
4979 18575 : else if (ix86_use_mask_cmp_p (data_mode, mode, op_true, op_false))
4980 : ;
4981 : else
4982 : {
4983 : /* Canonicalize the comparison to EQ, GT, GTU. */
4984 18522 : switch (code)
4985 : {
4986 : case EQ:
4987 : case GT:
4988 : case GTU:
4989 : break;
4990 :
4991 962 : case LE:
4992 962 : case LEU:
4993 : /* x <= cst can be handled as x < cst + 1 unless there is
4994 : wrap around in cst + 1. */
4995 962 : if (CONST_VECTOR_P (cop1)
4996 1645 : && GET_MODE_INNER (mode) != TImode)
4997 : {
4998 683 : unsigned int n_elts = GET_MODE_NUNITS (mode), i;
4999 683 : machine_mode eltmode = GET_MODE_INNER (mode);
5000 4404 : for (i = 0; i < n_elts; ++i)
5001 : {
5002 3722 : rtx elt = CONST_VECTOR_ELT (cop1, i);
5003 3722 : if (!CONST_INT_P (elt))
5004 : break;
5005 3722 : if (code == LE)
5006 : {
5007 : /* For LE punt if some element is signed maximum. */
5008 2546 : if ((INTVAL (elt) & (GET_MODE_MASK (eltmode) >> 1))
5009 : == (GET_MODE_MASK (eltmode) >> 1))
5010 : break;
5011 : }
5012 : /* For LEU punt if some element is unsigned maximum. */
5013 1176 : else if (elt == constm1_rtx)
5014 : break;
5015 : }
5016 683 : if (i == n_elts)
5017 : {
5018 682 : rtvec v = rtvec_alloc (n_elts);
5019 5084 : for (i = 0; i < n_elts; ++i)
5020 3720 : RTVEC_ELT (v, i)
5021 3720 : = gen_int_mode (INTVAL (CONST_VECTOR_ELT (cop1, i)) + 1,
5022 : eltmode);
5023 682 : cop1 = gen_rtx_CONST_VECTOR (mode, v);
5024 682 : std::swap (cop0, cop1);
5025 682 : code = code == LE ? GT : GTU;
5026 : break;
5027 : }
5028 : }
5029 : /* FALLTHRU */
5030 3394 : case NE:
5031 3394 : code = reverse_condition (code);
5032 3394 : *negate = true;
5033 3394 : break;
5034 :
5035 547 : case GE:
5036 547 : case GEU:
5037 : /* x >= cst can be handled as x > cst - 1 unless there is
5038 : wrap around in cst - 1. */
5039 547 : if (CONST_VECTOR_P (cop1)
5040 842 : && GET_MODE_INNER (mode) != TImode)
5041 : {
5042 295 : unsigned int n_elts = GET_MODE_NUNITS (mode), i;
5043 295 : machine_mode eltmode = GET_MODE_INNER (mode);
5044 2211 : for (i = 0; i < n_elts; ++i)
5045 : {
5046 1964 : rtx elt = CONST_VECTOR_ELT (cop1, i);
5047 1964 : if (!CONST_INT_P (elt))
5048 : break;
5049 1964 : if (code == GE)
5050 : {
5051 : /* For GE punt if some element is signed minimum. */
5052 1916 : if (INTVAL (elt) < 0
5053 136 : && ((INTVAL (elt) & (GET_MODE_MASK (eltmode) >> 1))
5054 : == 0))
5055 : break;
5056 : }
5057 : /* For GEU punt if some element is zero. */
5058 48 : else if (elt == const0_rtx)
5059 : break;
5060 : }
5061 295 : if (i == n_elts)
5062 : {
5063 247 : rtvec v = rtvec_alloc (n_elts);
5064 2410 : for (i = 0; i < n_elts; ++i)
5065 1916 : RTVEC_ELT (v, i)
5066 1916 : = gen_int_mode (INTVAL (CONST_VECTOR_ELT (cop1, i)) - 1,
5067 : eltmode);
5068 247 : cop1 = gen_rtx_CONST_VECTOR (mode, v);
5069 247 : code = code == GE ? GT : GTU;
5070 : break;
5071 : }
5072 : }
5073 300 : code = reverse_condition (code);
5074 300 : *negate = true;
5075 : /* FALLTHRU */
5076 :
5077 1666 : case LT:
5078 1666 : case LTU:
5079 1666 : std::swap (cop0, cop1);
5080 1666 : code = swap_condition (code);
5081 1666 : break;
5082 :
5083 0 : default:
5084 0 : gcc_unreachable ();
5085 : }
5086 :
5087 : /* Only SSE4.1/SSE4.2 supports V2DImode. */
5088 18522 : if (mode == V2DImode)
5089 : {
5090 843 : switch (code)
5091 : {
5092 642 : case EQ:
5093 : /* SSE4.1 supports EQ. */
5094 642 : if (!TARGET_SSE4_1)
5095 18719 : return NULL;
5096 : break;
5097 :
5098 201 : case GT:
5099 201 : case GTU:
5100 : /* SSE4.2 supports GT/GTU. */
5101 201 : if (!TARGET_SSE4_2)
5102 : return NULL;
5103 : break;
5104 :
5105 0 : default:
5106 0 : gcc_unreachable ();
5107 : }
5108 : }
5109 :
5110 18522 : if (CONST_VECTOR_P (cop0))
5111 1349 : cop0 = force_reg (mode, cop0);
5112 17173 : else if (CONST_VECTOR_P (cop1))
5113 7970 : cop1 = force_reg (mode, cop1);
5114 :
5115 18522 : rtx optrue = op_true ? op_true : CONSTM1_RTX (data_mode);
5116 18522 : rtx opfalse = op_false ? op_false : CONST0_RTX (data_mode);
5117 18522 : if (*negate)
5118 3694 : std::swap (optrue, opfalse);
5119 :
5120 : /* Transform x > y ? 0 : -1 (i.e. x <= y ? -1 : 0 or x <= y) when
5121 : not using integer masks into min (x, y) == x ? -1 : 0 (i.e.
5122 : min (x, y) == x). While we add one instruction (the minimum),
5123 : we remove the need for two instructions in the negation, as the
5124 : result is done this way.
5125 : When using masks, do it for SI/DImode element types, as it is shorter
5126 : than the two subtractions. */
5127 18522 : if ((code != EQ
5128 7657 : && GET_MODE_SIZE (mode) != 64
5129 7657 : && vector_all_ones_operand (opfalse, data_mode)
5130 580 : && optrue == CONST0_RTX (data_mode))
5131 25599 : || (code == GTU
5132 2164 : && GET_MODE_SIZE (GET_MODE_INNER (mode)) >= 4
5133 : /* Don't do it if not using integer masks and we'd end up with
5134 : the right values in the registers though. */
5135 805 : && (GET_MODE_SIZE (mode) == 64
5136 805 : || !vector_all_ones_operand (optrue, data_mode)
5137 676 : || opfalse != CONST0_RTX (data_mode))))
5138 : {
5139 709 : rtx (*gen) (rtx, rtx, rtx) = NULL;
5140 :
5141 709 : switch (mode)
5142 : {
5143 0 : case E_V16SImode:
5144 0 : gen = (code == GTU) ? gen_uminv16si3 : gen_sminv16si3;
5145 : break;
5146 0 : case E_V8DImode:
5147 0 : gen = (code == GTU) ? gen_uminv8di3 : gen_sminv8di3;
5148 0 : cop0 = force_reg (mode, cop0);
5149 0 : cop1 = force_reg (mode, cop1);
5150 0 : break;
5151 24 : case E_V32QImode:
5152 24 : if (TARGET_AVX2)
5153 24 : gen = (code == GTU) ? gen_uminv32qi3 : gen_sminv32qi3;
5154 : break;
5155 24 : case E_V16HImode:
5156 24 : if (TARGET_AVX2)
5157 24 : gen = (code == GTU) ? gen_uminv16hi3 : gen_sminv16hi3;
5158 : break;
5159 25 : case E_V8SImode:
5160 25 : if (TARGET_AVX2)
5161 25 : gen = (code == GTU) ? gen_uminv8si3 : gen_sminv8si3;
5162 : break;
5163 20 : case E_V4DImode:
5164 20 : if (TARGET_AVX512VL)
5165 : {
5166 0 : gen = (code == GTU) ? gen_uminv4di3 : gen_sminv4di3;
5167 0 : cop0 = force_reg (mode, cop0);
5168 0 : cop1 = force_reg (mode, cop1);
5169 : }
5170 : break;
5171 64 : case E_V16QImode:
5172 64 : if (code == GTU && TARGET_SSE2)
5173 : gen = gen_uminv16qi3;
5174 28 : else if (code == GT && TARGET_SSE4_1)
5175 : gen = gen_sminv16qi3;
5176 : break;
5177 44 : case E_V8QImode:
5178 44 : if (code == GTU && TARGET_SSE2)
5179 : gen = gen_uminv8qi3;
5180 42 : else if (code == GT && TARGET_SSE4_1)
5181 : gen = gen_sminv8qi3;
5182 : break;
5183 13 : case E_V4QImode:
5184 13 : if (code == GTU && TARGET_SSE2)
5185 : gen = gen_uminv4qi3;
5186 2 : else if (code == GT && TARGET_SSE4_1)
5187 : gen = gen_sminv4qi3;
5188 : break;
5189 8 : case E_V2QImode:
5190 8 : if (code == GTU && TARGET_SSE2)
5191 : gen = gen_uminv2qi3;
5192 6 : else if (code == GT && TARGET_SSE4_1)
5193 : gen = gen_sminv2qi3;
5194 : break;
5195 73 : case E_V8HImode:
5196 73 : if (code == GTU && TARGET_SSE4_1)
5197 : gen = gen_uminv8hi3;
5198 63 : else if (code == GT && TARGET_SSE2)
5199 : gen = gen_sminv8hi3;
5200 : break;
5201 4 : case E_V4HImode:
5202 4 : if (code == GTU && TARGET_SSE4_1)
5203 : gen = gen_uminv4hi3;
5204 4 : else if (code == GT && TARGET_SSE2)
5205 : gen = gen_sminv4hi3;
5206 : break;
5207 16 : case E_V2HImode:
5208 16 : if (code == GTU && TARGET_SSE4_1)
5209 : gen = gen_uminv2hi3;
5210 16 : else if (code == GT && TARGET_SSE2)
5211 : gen = gen_sminv2hi3;
5212 : break;
5213 254 : case E_V4SImode:
5214 254 : if (TARGET_SSE4_1)
5215 52 : gen = (code == GTU) ? gen_uminv4si3 : gen_sminv4si3;
5216 : break;
5217 116 : case E_V2SImode:
5218 116 : if (TARGET_SSE4_1)
5219 0 : gen = (code == GTU) ? gen_uminv2si3 : gen_sminv2si3;
5220 : break;
5221 24 : case E_V2DImode:
5222 24 : if (TARGET_AVX512VL)
5223 : {
5224 0 : gen = (code == GTU) ? gen_uminv2di3 : gen_sminv2di3;
5225 0 : cop0 = force_reg (mode, cop0);
5226 0 : cop1 = force_reg (mode, cop1);
5227 : }
5228 : break;
5229 : default:
5230 : break;
5231 : }
5232 :
5233 0 : if (gen)
5234 : {
5235 280 : rtx tem = gen_reg_rtx (mode);
5236 280 : if (!vector_operand (cop0, mode))
5237 0 : cop0 = force_reg (mode, cop0);
5238 280 : if (!vector_operand (cop1, mode))
5239 0 : cop1 = force_reg (mode, cop1);
5240 280 : *negate = !*negate;
5241 280 : emit_insn (gen (tem, cop0, cop1));
5242 280 : cop1 = tem;
5243 280 : code = EQ;
5244 : }
5245 : }
5246 :
5247 : /* Unsigned parallel compare is not supported by the hardware.
5248 : Play some tricks to turn this into a signed comparison
5249 : against 0. */
5250 18522 : if (code == GTU)
5251 : {
5252 1219 : cop0 = force_reg (mode, cop0);
5253 :
5254 1219 : switch (mode)
5255 : {
5256 911 : case E_V16SImode:
5257 911 : case E_V8DImode:
5258 911 : case E_V8SImode:
5259 911 : case E_V4DImode:
5260 911 : case E_V4SImode:
5261 911 : case E_V2SImode:
5262 911 : case E_V2DImode:
5263 911 : {
5264 911 : rtx t1, t2, mask;
5265 :
5266 : /* Subtract (-(INT MAX) - 1) from both operands to make
5267 : them signed. */
5268 911 : mask = ix86_build_signbit_mask (mode, true, false);
5269 911 : t1 = gen_reg_rtx (mode);
5270 911 : emit_insn (gen_sub3_insn (t1, cop0, mask));
5271 :
5272 911 : t2 = gen_reg_rtx (mode);
5273 911 : emit_insn (gen_sub3_insn (t2, cop1, mask));
5274 :
5275 911 : cop0 = t1;
5276 911 : cop1 = t2;
5277 911 : code = GT;
5278 : }
5279 911 : break;
5280 :
5281 308 : case E_V64QImode:
5282 308 : case E_V32HImode:
5283 308 : case E_V32QImode:
5284 308 : case E_V16HImode:
5285 308 : case E_V16QImode:
5286 308 : case E_V8QImode:
5287 308 : case E_V4QImode:
5288 308 : case E_V2QImode:
5289 308 : case E_V8HImode:
5290 308 : case E_V4HImode:
5291 308 : case E_V2HImode:
5292 : /* Perform a parallel unsigned saturating subtraction. */
5293 308 : x = gen_reg_rtx (mode);
5294 308 : emit_insn (gen_rtx_SET
5295 : (x, gen_rtx_US_MINUS (mode, cop0, cop1)));
5296 308 : cop0 = x;
5297 308 : cop1 = CONST0_RTX (mode);
5298 308 : code = EQ;
5299 308 : *negate = !*negate;
5300 308 : break;
5301 :
5302 0 : default:
5303 0 : gcc_unreachable ();
5304 : }
5305 : }
5306 : }
5307 :
5308 18719 : if (*negate)
5309 3660 : std::swap (op_true, op_false);
5310 :
5311 18719 : if (CONST_VECTOR_P (cop1))
5312 323 : cop1 = force_reg (mode, cop1);
5313 :
5314 : /* Allow the comparison to be done in one mode, but the movcc to
5315 : happen in another mode. */
5316 18719 : if (data_mode == mode)
5317 18677 : x = ix86_expand_sse_cmp (dest, code, cop0, cop1, op_true, op_false);
5318 : else
5319 : {
5320 126 : gcc_assert (GET_MODE_SIZE (data_mode) == GET_MODE_SIZE (mode));
5321 42 : x = ix86_expand_sse_cmp (gen_reg_rtx (mode), code, cop0, cop1,
5322 : op_true, op_false);
5323 42 : if (GET_MODE (x) == mode)
5324 24 : x = gen_lowpart (data_mode, x);
5325 : }
5326 :
5327 : return x;
5328 : }
5329 :
5330 : /* Expand integer vector comparison. */
5331 :
5332 : bool
5333 11754 : ix86_expand_int_vec_cmp (rtx operands[])
5334 : {
5335 11754 : rtx_code code = GET_CODE (operands[1]);
5336 11754 : bool negate = false;
5337 11754 : rtx cmp = ix86_expand_int_sse_cmp (operands[0], code, operands[2],
5338 : operands[3], NULL, NULL, &negate);
5339 :
5340 11754 : if (!cmp)
5341 : return false;
5342 :
5343 11754 : if (negate)
5344 : {
5345 3752 : if (TARGET_AVX512F && GET_MODE_SIZE (GET_MODE (cmp)) >= 16)
5346 93 : cmp = gen_rtx_XOR (GET_MODE (cmp), cmp, CONSTM1_RTX (GET_MODE (cmp)));
5347 : else
5348 : {
5349 7062 : cmp = ix86_expand_int_sse_cmp (operands[0], EQ, cmp,
5350 3531 : CONST0_RTX (GET_MODE (cmp)),
5351 : NULL, NULL, &negate);
5352 3531 : gcc_assert (!negate);
5353 : }
5354 : }
5355 :
5356 11754 : if (operands[0] != cmp)
5357 11459 : emit_move_insn (operands[0], cmp);
5358 :
5359 : return true;
5360 : }
5361 :
5362 : /* Expand a floating-point vector conditional move; a vcond operation
5363 : rather than a movcc operation. */
5364 :
5365 : bool
5366 0 : ix86_expand_fp_vcond (rtx operands[])
5367 : {
5368 0 : enum rtx_code code = GET_CODE (operands[3]);
5369 0 : rtx cmp;
5370 :
5371 0 : code = ix86_prepare_sse_fp_compare_args (operands[0], code,
5372 : &operands[4], &operands[5]);
5373 0 : if (code == UNKNOWN)
5374 : {
5375 0 : rtx temp;
5376 0 : switch (GET_CODE (operands[3]))
5377 : {
5378 0 : case LTGT:
5379 0 : temp = ix86_expand_sse_cmp (operands[0], ORDERED, operands[4],
5380 : operands[5], operands[0], operands[0]);
5381 0 : cmp = ix86_expand_sse_cmp (operands[0], NE, operands[4],
5382 : operands[5], operands[1], operands[2]);
5383 0 : code = AND;
5384 0 : break;
5385 0 : case UNEQ:
5386 0 : temp = ix86_expand_sse_cmp (operands[0], UNORDERED, operands[4],
5387 : operands[5], operands[0], operands[0]);
5388 0 : cmp = ix86_expand_sse_cmp (operands[0], EQ, operands[4],
5389 : operands[5], operands[1], operands[2]);
5390 0 : code = IOR;
5391 0 : break;
5392 0 : default:
5393 0 : gcc_unreachable ();
5394 : }
5395 0 : cmp = expand_simple_binop (GET_MODE (cmp), code, temp, cmp, cmp, 1,
5396 : OPTAB_DIRECT);
5397 0 : ix86_expand_sse_movcc (operands[0], cmp, operands[1], operands[2]);
5398 0 : return true;
5399 : }
5400 :
5401 0 : if (ix86_expand_sse_fp_minmax (operands[0], code, operands[4],
5402 : operands[5], operands[1], operands[2]))
5403 : return true;
5404 :
5405 0 : cmp = ix86_expand_sse_cmp (operands[0], code, operands[4], operands[5],
5406 : operands[1], operands[2]);
5407 0 : ix86_expand_sse_movcc (operands[0], cmp, operands[1], operands[2]);
5408 0 : return true;
5409 : }
5410 :
5411 : /* Expand a signed/unsigned integral vector conditional move. */
5412 :
5413 : bool
5414 3434 : ix86_expand_int_vcond (rtx operands[])
5415 : {
5416 3434 : machine_mode data_mode = GET_MODE (operands[0]);
5417 3434 : machine_mode mode = GET_MODE (operands[4]);
5418 3434 : enum rtx_code code = GET_CODE (operands[3]);
5419 3434 : bool negate = false;
5420 3434 : rtx x, cop0, cop1;
5421 :
5422 3434 : cop0 = operands[4];
5423 3434 : cop1 = operands[5];
5424 :
5425 : /* Try to optimize x < 0 ? -1 : 0 into (signed) x >> 31
5426 : and x < 0 ? 1 : 0 into (unsigned) x >> 31. */
5427 3434 : if ((code == LT || code == GE)
5428 0 : && data_mode == mode
5429 0 : && cop1 == CONST0_RTX (mode)
5430 0 : && operands[1 + (code == LT)] == CONST0_RTX (data_mode)
5431 0 : && GET_MODE_UNIT_SIZE (data_mode) > 1
5432 0 : && GET_MODE_UNIT_SIZE (data_mode) <= 8
5433 3434 : && (GET_MODE_SIZE (data_mode) == 16
5434 0 : || (TARGET_AVX2 && GET_MODE_SIZE (data_mode) == 32)))
5435 : {
5436 0 : rtx negop = operands[2 - (code == LT)];
5437 0 : int shift = GET_MODE_UNIT_BITSIZE (data_mode) - 1;
5438 0 : if (negop == CONST1_RTX (data_mode))
5439 : {
5440 0 : rtx res = expand_simple_binop (mode, LSHIFTRT, cop0, GEN_INT (shift),
5441 : operands[0], 1, OPTAB_DIRECT);
5442 0 : if (res != operands[0])
5443 0 : emit_move_insn (operands[0], res);
5444 : return true;
5445 : }
5446 0 : else if (GET_MODE_INNER (data_mode) != DImode
5447 0 : && vector_all_ones_operand (negop, data_mode))
5448 : {
5449 0 : rtx res = expand_simple_binop (mode, ASHIFTRT, cop0, GEN_INT (shift),
5450 : operands[0], 0, OPTAB_DIRECT);
5451 0 : if (res != operands[0])
5452 0 : emit_move_insn (operands[0], res);
5453 : return true;
5454 : }
5455 : }
5456 :
5457 3434 : if (!nonimmediate_operand (cop1, mode))
5458 126 : cop1 = force_reg (mode, cop1);
5459 3434 : if (!general_operand (operands[1], data_mode))
5460 0 : operands[1] = force_reg (data_mode, operands[1]);
5461 3434 : if (!general_operand (operands[2], data_mode))
5462 0 : operands[2] = force_reg (data_mode, operands[2]);
5463 :
5464 3434 : x = ix86_expand_int_sse_cmp (operands[0], code, cop0, cop1,
5465 : operands[1], operands[2], &negate);
5466 :
5467 3434 : if (!x)
5468 : return false;
5469 :
5470 3434 : ix86_expand_sse_movcc (operands[0], x, operands[1+negate],
5471 3434 : operands[2-negate]);
5472 3434 : return true;
5473 : }
5474 :
5475 : static bool
5476 128101 : ix86_expand_vec_perm_vpermt2 (rtx target, rtx mask, rtx op0, rtx op1,
5477 : struct expand_vec_perm_d *d)
5478 : {
5479 : /* ix86_expand_vec_perm_vpermt2 is called from both const and non-const
5480 : expander, so args are either in d, or in op0, op1 etc. */
5481 128101 : machine_mode mode = GET_MODE (d ? d->op0 : op0);
5482 128101 : machine_mode maskmode = mode;
5483 128101 : rtx (*gen) (rtx, rtx, rtx, rtx) = NULL;
5484 :
5485 128101 : switch (mode)
5486 : {
5487 24232 : case E_V16QImode:
5488 24232 : if (TARGET_AVX512VL && TARGET_AVX512VBMI)
5489 : gen = gen_avx512vl_vpermt2varv16qi3;
5490 : break;
5491 514 : case E_V32QImode:
5492 514 : if (TARGET_AVX512VL && TARGET_AVX512VBMI)
5493 : gen = gen_avx512vl_vpermt2varv32qi3;
5494 : break;
5495 191 : case E_V64QImode:
5496 191 : if (TARGET_AVX512VBMI)
5497 : gen = gen_avx512bw_vpermt2varv64qi3;
5498 : break;
5499 13865 : case E_V8HImode:
5500 13865 : if (TARGET_AVX512VL && TARGET_AVX512BW)
5501 : gen = gen_avx512vl_vpermt2varv8hi3;
5502 : break;
5503 775 : case E_V16HImode:
5504 775 : if (TARGET_AVX512VL && TARGET_AVX512BW)
5505 : gen = gen_avx512vl_vpermt2varv16hi3;
5506 : break;
5507 342 : case E_V32HImode:
5508 342 : if (TARGET_AVX512BW)
5509 : gen = gen_avx512bw_vpermt2varv32hi3;
5510 : break;
5511 34621 : case E_V4SImode:
5512 34621 : if (TARGET_AVX512VL)
5513 : gen = gen_avx512vl_vpermt2varv4si3;
5514 : break;
5515 1137 : case E_V8SImode:
5516 1137 : if (TARGET_AVX512VL)
5517 : gen = gen_avx512vl_vpermt2varv8si3;
5518 : break;
5519 129 : case E_V16SImode:
5520 129 : if (TARGET_AVX512F)
5521 : gen = gen_avx512f_vpermt2varv16si3;
5522 : break;
5523 10776 : case E_V4SFmode:
5524 10776 : if (TARGET_AVX512VL)
5525 : {
5526 : gen = gen_avx512vl_vpermt2varv4sf3;
5527 : maskmode = V4SImode;
5528 : }
5529 : break;
5530 6031 : case E_V8SFmode:
5531 6031 : if (TARGET_AVX512VL)
5532 : {
5533 : gen = gen_avx512vl_vpermt2varv8sf3;
5534 : maskmode = V8SImode;
5535 : }
5536 : break;
5537 207 : case E_V16SFmode:
5538 207 : if (TARGET_AVX512F)
5539 : {
5540 : gen = gen_avx512f_vpermt2varv16sf3;
5541 : maskmode = V16SImode;
5542 : }
5543 : break;
5544 2 : case E_V2DImode:
5545 2 : if (TARGET_AVX512VL)
5546 : gen = gen_avx512vl_vpermt2varv2di3;
5547 : break;
5548 347 : case E_V4DImode:
5549 347 : if (TARGET_AVX512VL)
5550 : gen = gen_avx512vl_vpermt2varv4di3;
5551 : break;
5552 10 : case E_V8DImode:
5553 10 : if (TARGET_AVX512F)
5554 : gen = gen_avx512f_vpermt2varv8di3;
5555 : break;
5556 2 : case E_V2DFmode:
5557 2 : if (TARGET_AVX512VL)
5558 : {
5559 : gen = gen_avx512vl_vpermt2varv2df3;
5560 : maskmode = V2DImode;
5561 : }
5562 : break;
5563 1868 : case E_V4DFmode:
5564 1868 : if (TARGET_AVX512VL)
5565 : {
5566 : gen = gen_avx512vl_vpermt2varv4df3;
5567 : maskmode = V4DImode;
5568 : }
5569 : break;
5570 198 : case E_V8DFmode:
5571 198 : if (TARGET_AVX512F)
5572 : {
5573 : gen = gen_avx512f_vpermt2varv8df3;
5574 : maskmode = V8DImode;
5575 : }
5576 : break;
5577 : default:
5578 : break;
5579 : }
5580 :
5581 : if (gen == NULL)
5582 : return false;
5583 :
5584 903 : if (d && d->testing_p)
5585 : return true;
5586 :
5587 : /* ix86_expand_vec_perm_vpermt2 is called from both const and non-const
5588 : expander, so args are either in d, or in op0, op1 etc. */
5589 893 : if (d)
5590 : {
5591 893 : rtx vec[64];
5592 893 : target = d->target;
5593 893 : op0 = d->op0;
5594 893 : op1 = d->op1;
5595 15617 : for (int i = 0; i < d->nelt; ++i)
5596 14724 : vec[i] = GEN_INT (d->perm[i]);
5597 893 : mask = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (d->nelt, vec));
5598 : }
5599 :
5600 901 : emit_insn (gen (target, force_reg (maskmode, mask), op0, op1));
5601 901 : return true;
5602 : }
5603 :
5604 : /* Expand a variable vector permutation. */
5605 :
5606 : void
5607 18 : ix86_expand_vec_perm (rtx operands[])
5608 : {
5609 18 : rtx target = operands[0];
5610 18 : rtx op0 = operands[1];
5611 18 : rtx op1 = operands[2];
5612 18 : rtx mask = operands[3];
5613 18 : rtx t1, t2, t3, t4, t5, t6, t7, t8, vt, vt2, vec[32];
5614 18 : machine_mode mode = GET_MODE (op0);
5615 18 : machine_mode maskmode = GET_MODE (mask);
5616 18 : int w, e, i;
5617 18 : bool one_operand_shuffle = rtx_equal_p (op0, op1);
5618 :
5619 : /* Number of elements in the vector. */
5620 18 : w = GET_MODE_NUNITS (mode);
5621 18 : e = GET_MODE_UNIT_SIZE (mode);
5622 18 : gcc_assert (w <= 64);
5623 :
5624 : /* For HF mode vector, convert it to HI using subreg. */
5625 36 : if (GET_MODE_INNER (mode) == HFmode)
5626 : {
5627 6 : machine_mode orig_mode = mode;
5628 6 : mode = mode_for_vector (HImode, w).require ();
5629 6 : target = lowpart_subreg (mode, target, orig_mode);
5630 6 : op0 = lowpart_subreg (mode, op0, orig_mode);
5631 6 : op1 = lowpart_subreg (mode, op1, orig_mode);
5632 : }
5633 :
5634 18 : if (TARGET_AVX512F && one_operand_shuffle)
5635 : {
5636 5 : rtx (*gen) (rtx, rtx, rtx) = NULL;
5637 5 : switch (mode)
5638 : {
5639 : case E_V16SImode:
5640 : gen = gen_avx512f_permvarv16si;
5641 : break;
5642 0 : case E_V16SFmode:
5643 0 : gen = gen_avx512f_permvarv16sf;
5644 0 : break;
5645 0 : case E_V8DImode:
5646 0 : gen = gen_avx512f_permvarv8di;
5647 0 : break;
5648 0 : case E_V8DFmode:
5649 0 : gen = gen_avx512f_permvarv8df;
5650 0 : break;
5651 : default:
5652 : break;
5653 : }
5654 0 : if (gen != NULL)
5655 : {
5656 0 : emit_insn (gen (target, op0, mask));
5657 16 : return;
5658 : }
5659 : }
5660 :
5661 18 : if (ix86_expand_vec_perm_vpermt2 (target, mask, op0, op1, NULL))
5662 : return;
5663 :
5664 10 : if (TARGET_AVX2)
5665 : {
5666 5 : if (mode == V4DImode || mode == V4DFmode || mode == V16HImode)
5667 : {
5668 : /* Unfortunately, the VPERMQ and VPERMPD instructions only support
5669 : an constant shuffle operand. With a tiny bit of effort we can
5670 : use VPERMD instead. A re-interpretation stall for V4DFmode is
5671 : unfortunate but there's no avoiding it.
5672 : Similarly for V16HImode we don't have instructions for variable
5673 : shuffling, while for V32QImode we can use after preparing suitable
5674 : masks vpshufb; vpshufb; vpermq; vpor. */
5675 :
5676 : if (mode == V16HImode)
5677 : {
5678 : maskmode = mode = V32QImode;
5679 : w = 32;
5680 : e = 1;
5681 : }
5682 : else
5683 : {
5684 : maskmode = mode = V8SImode;
5685 : w = 8;
5686 : e = 4;
5687 : }
5688 0 : t1 = gen_reg_rtx (maskmode);
5689 :
5690 : /* Replicate the low bits of the V4DImode mask into V8SImode:
5691 : mask = { A B C D }
5692 : t1 = { A A B B C C D D }. */
5693 0 : for (i = 0; i < w / 2; ++i)
5694 0 : vec[i*2 + 1] = vec[i*2] = GEN_INT (i * 2);
5695 0 : vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
5696 0 : vt = force_reg (maskmode, vt);
5697 0 : mask = gen_lowpart (maskmode, mask);
5698 0 : if (maskmode == V8SImode)
5699 0 : emit_insn (gen_avx2_permvarv8si (t1, mask, vt));
5700 : else
5701 0 : emit_insn (gen_avx2_pshufbv32qi3 (t1, mask, vt));
5702 :
5703 : /* Multiply the shuffle indices by two. */
5704 0 : t1 = expand_simple_binop (maskmode, PLUS, t1, t1, t1, 1,
5705 : OPTAB_DIRECT);
5706 :
5707 : /* Add one to the odd shuffle indices:
5708 : t1 = { A*2, A*2+1, B*2, B*2+1, ... }. */
5709 0 : for (i = 0; i < w / 2; ++i)
5710 : {
5711 0 : vec[i * 2] = const0_rtx;
5712 0 : vec[i * 2 + 1] = const1_rtx;
5713 : }
5714 0 : vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
5715 0 : vt = validize_mem (force_const_mem (maskmode, vt));
5716 0 : t1 = expand_simple_binop (maskmode, PLUS, t1, vt, t1, 1,
5717 : OPTAB_DIRECT);
5718 :
5719 : /* Continue as if V8SImode (resp. V32QImode) was used initially. */
5720 0 : operands[3] = mask = t1;
5721 0 : target = gen_reg_rtx (mode);
5722 0 : op0 = gen_lowpart (mode, op0);
5723 0 : op1 = gen_lowpart (mode, op1);
5724 : }
5725 :
5726 5 : switch (mode)
5727 : {
5728 1 : case E_V8SImode:
5729 : /* The VPERMD and VPERMPS instructions already properly ignore
5730 : the high bits of the shuffle elements. No need for us to
5731 : perform an AND ourselves. */
5732 1 : if (one_operand_shuffle)
5733 : {
5734 0 : emit_insn (gen_avx2_permvarv8si (target, op0, mask));
5735 0 : if (target != operands[0])
5736 0 : emit_move_insn (operands[0],
5737 0 : gen_lowpart (GET_MODE (operands[0]), target));
5738 : }
5739 : else
5740 : {
5741 1 : t1 = gen_reg_rtx (V8SImode);
5742 1 : t2 = gen_reg_rtx (V8SImode);
5743 1 : emit_insn (gen_avx2_permvarv8si (t1, op0, mask));
5744 1 : emit_insn (gen_avx2_permvarv8si (t2, op1, mask));
5745 1 : goto merge_two;
5746 : }
5747 : return;
5748 :
5749 0 : case E_V8SFmode:
5750 0 : mask = gen_lowpart (V8SImode, mask);
5751 0 : if (one_operand_shuffle)
5752 0 : emit_insn (gen_avx2_permvarv8sf (target, op0, mask));
5753 : else
5754 : {
5755 0 : t1 = gen_reg_rtx (V8SFmode);
5756 0 : t2 = gen_reg_rtx (V8SFmode);
5757 0 : emit_insn (gen_avx2_permvarv8sf (t1, op0, mask));
5758 0 : emit_insn (gen_avx2_permvarv8sf (t2, op1, mask));
5759 0 : goto merge_two;
5760 : }
5761 0 : return;
5762 :
5763 1 : case E_V4SImode:
5764 1 : if (one_operand_shuffle)
5765 : break; /* Handled below for TARGET_AVX. */
5766 : /* By combining the two 128-bit input vectors into one 256-bit
5767 : input vector, we can use VPERMD and VPERMPS for the full
5768 : two-operand shuffle. */
5769 0 : t1 = gen_reg_rtx (V8SImode);
5770 0 : t2 = gen_reg_rtx (V8SImode);
5771 0 : emit_insn (gen_avx_vec_concatv8si (t1, op0, op1));
5772 0 : emit_insn (gen_avx_vec_concatv8si (t2, mask, mask));
5773 0 : emit_insn (gen_avx2_permvarv8si (t1, t1, t2));
5774 0 : emit_insn (gen_avx_vextractf128v8si (target, t1, const0_rtx));
5775 0 : return;
5776 :
5777 1 : case E_V4SFmode:
5778 1 : if (one_operand_shuffle)
5779 : break; /* Handled below for TARGET_AVX. */
5780 0 : t1 = gen_reg_rtx (V8SFmode);
5781 0 : t2 = gen_reg_rtx (V8SImode);
5782 0 : mask = gen_lowpart (V4SImode, mask);
5783 0 : emit_insn (gen_avx_vec_concatv8sf (t1, op0, op1));
5784 0 : emit_insn (gen_avx_vec_concatv8si (t2, mask, mask));
5785 0 : emit_insn (gen_avx2_permvarv8sf (t1, t1, t2));
5786 0 : emit_insn (gen_avx_vextractf128v8sf (target, t1, const0_rtx));
5787 0 : return;
5788 :
5789 0 : case E_V32QImode:
5790 0 : t1 = gen_reg_rtx (V32QImode);
5791 0 : t2 = gen_reg_rtx (V32QImode);
5792 0 : t3 = gen_reg_rtx (V32QImode);
5793 0 : vt2 = GEN_INT (-128);
5794 0 : vt = gen_const_vec_duplicate (V32QImode, vt2);
5795 0 : vt = force_reg (V32QImode, vt);
5796 0 : for (i = 0; i < 32; i++)
5797 0 : vec[i] = i < 16 ? vt2 : const0_rtx;
5798 0 : vt2 = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, vec));
5799 0 : vt2 = force_reg (V32QImode, vt2);
5800 : /* From mask create two adjusted masks, which contain the same
5801 : bits as mask in the low 7 bits of each vector element.
5802 : The first mask will have the most significant bit clear
5803 : if it requests element from the same 128-bit lane
5804 : and MSB set if it requests element from the other 128-bit lane.
5805 : The second mask will have the opposite values of the MSB,
5806 : and additionally will have its 128-bit lanes swapped.
5807 : E.g. { 07 12 1e 09 ... | 17 19 05 1f ... } mask vector will have
5808 : t1 { 07 92 9e 09 ... | 17 19 85 1f ... } and
5809 : t3 { 97 99 05 9f ... | 87 12 1e 89 ... } where each ...
5810 : stands for other 12 bytes. */
5811 : /* The bit whether element is from the same lane or the other
5812 : lane is bit 4, so shift it up by 3 to the MSB position. */
5813 0 : t5 = gen_reg_rtx (V4DImode);
5814 0 : emit_insn (gen_ashlv4di3 (t5, gen_lowpart (V4DImode, mask),
5815 : GEN_INT (3)));
5816 : /* Clear MSB bits from the mask just in case it had them set. */
5817 0 : emit_insn (gen_avx2_andnotv32qi3 (t2, vt, mask));
5818 : /* After this t1 will have MSB set for elements from other lane. */
5819 0 : emit_insn (gen_xorv32qi3 (t1, gen_lowpart (V32QImode, t5), vt2));
5820 : /* Clear bits other than MSB. */
5821 0 : emit_insn (gen_andv32qi3 (t1, t1, vt));
5822 : /* Or in the lower bits from mask into t3. */
5823 0 : emit_insn (gen_iorv32qi3 (t3, t1, t2));
5824 : /* And invert MSB bits in t1, so MSB is set for elements from the same
5825 : lane. */
5826 0 : emit_insn (gen_xorv32qi3 (t1, t1, vt));
5827 : /* Swap 128-bit lanes in t3. */
5828 0 : t6 = gen_reg_rtx (V4DImode);
5829 0 : emit_insn (gen_avx2_permv4di_1 (t6, gen_lowpart (V4DImode, t3),
5830 : const2_rtx, GEN_INT (3),
5831 : const0_rtx, const1_rtx));
5832 : /* And or in the lower bits from mask into t1. */
5833 0 : emit_insn (gen_iorv32qi3 (t1, t1, t2));
5834 0 : if (one_operand_shuffle)
5835 : {
5836 : /* Each of these shuffles will put 0s in places where
5837 : element from the other 128-bit lane is needed, otherwise
5838 : will shuffle in the requested value. */
5839 0 : emit_insn (gen_avx2_pshufbv32qi3 (t3, op0,
5840 0 : gen_lowpart (V32QImode, t6)));
5841 0 : emit_insn (gen_avx2_pshufbv32qi3 (t1, op0, t1));
5842 : /* For t3 the 128-bit lanes are swapped again. */
5843 0 : t7 = gen_reg_rtx (V4DImode);
5844 0 : emit_insn (gen_avx2_permv4di_1 (t7, gen_lowpart (V4DImode, t3),
5845 : const2_rtx, GEN_INT (3),
5846 : const0_rtx, const1_rtx));
5847 : /* And oring both together leads to the result. */
5848 0 : emit_insn (gen_iorv32qi3 (target, t1,
5849 0 : gen_lowpart (V32QImode, t7)));
5850 0 : if (target != operands[0])
5851 0 : emit_move_insn (operands[0],
5852 0 : gen_lowpart (GET_MODE (operands[0]), target));
5853 : return;
5854 : }
5855 :
5856 0 : t4 = gen_reg_rtx (V32QImode);
5857 : /* Similarly to the above one_operand_shuffle code,
5858 : just for repeated twice for each operand. merge_two:
5859 : code will merge the two results together. */
5860 0 : emit_insn (gen_avx2_pshufbv32qi3 (t4, op0,
5861 0 : gen_lowpart (V32QImode, t6)));
5862 0 : emit_insn (gen_avx2_pshufbv32qi3 (t3, op1,
5863 0 : gen_lowpart (V32QImode, t6)));
5864 0 : emit_insn (gen_avx2_pshufbv32qi3 (t2, op0, t1));
5865 0 : emit_insn (gen_avx2_pshufbv32qi3 (t1, op1, t1));
5866 0 : t7 = gen_reg_rtx (V4DImode);
5867 0 : emit_insn (gen_avx2_permv4di_1 (t7, gen_lowpart (V4DImode, t4),
5868 : const2_rtx, GEN_INT (3),
5869 : const0_rtx, const1_rtx));
5870 0 : t8 = gen_reg_rtx (V4DImode);
5871 0 : emit_insn (gen_avx2_permv4di_1 (t8, gen_lowpart (V4DImode, t3),
5872 : const2_rtx, GEN_INT (3),
5873 : const0_rtx, const1_rtx));
5874 0 : emit_insn (gen_iorv32qi3 (t4, t2, gen_lowpart (V32QImode, t7)));
5875 0 : emit_insn (gen_iorv32qi3 (t3, t1, gen_lowpart (V32QImode, t8)));
5876 0 : t1 = t4;
5877 0 : t2 = t3;
5878 0 : goto merge_two;
5879 :
5880 2 : default:
5881 4 : gcc_assert (GET_MODE_SIZE (mode) <= 16);
5882 : break;
5883 : }
5884 : }
5885 :
5886 9 : if (TARGET_AVX && one_operand_shuffle)
5887 8 : switch (mode)
5888 : {
5889 2 : case V4SImode:
5890 2 : op0 = gen_lowpart (V4SFmode, op0);
5891 2 : t1 = gen_reg_rtx (V4SFmode);
5892 2 : emit_insn (gen_avx_vpermilvarv4sf3 (t1, op0, mask));
5893 2 : emit_move_insn (target, gen_lowpart (mode, t1));
5894 2 : return;
5895 2 : case V4SFmode:
5896 2 : emit_insn (gen_avx_vpermilvarv4sf3 (target, op0, mask));
5897 2 : return;
5898 2 : case V2DImode:
5899 2 : op0 = gen_lowpart (V2DFmode, op0);
5900 2 : t1 = gen_reg_rtx (V2DImode);
5901 2 : t2 = gen_reg_rtx (V2DFmode);
5902 2 : emit_insn (gen_addv2di3 (t1, mask, mask));
5903 2 : emit_insn (gen_avx_vpermilvarv2df3 (t2, op0, t1));
5904 2 : emit_move_insn (target, gen_lowpart (mode, t2));
5905 2 : return;
5906 2 : case V2DFmode:
5907 2 : t1 = gen_reg_rtx (V2DImode);
5908 2 : emit_insn (gen_addv2di3 (t1, mask, mask));
5909 2 : emit_insn (gen_avx_vpermilvarv2df3 (target, op0, t1));
5910 2 : return;
5911 : default:
5912 : break;
5913 : }
5914 :
5915 1 : if (TARGET_XOP)
5916 : {
5917 : /* The XOP VPPERM insn supports three inputs. By ignoring the
5918 : one_operand_shuffle special case, we avoid creating another
5919 : set of constant vectors in memory. */
5920 0 : one_operand_shuffle = false;
5921 :
5922 : /* mask = mask & {2*w-1, ...} */
5923 0 : vt = GEN_INT (2*w - 1);
5924 : }
5925 : else
5926 : {
5927 : /* mask = mask & {w-1, ...} */
5928 1 : vt = GEN_INT (w - 1);
5929 : }
5930 :
5931 1 : vt = gen_const_vec_duplicate (maskmode, vt);
5932 1 : mask = expand_simple_binop (maskmode, AND, mask, vt,
5933 : NULL_RTX, 0, OPTAB_DIRECT);
5934 :
5935 : /* For non-QImode operations, convert the word permutation control
5936 : into a byte permutation control. */
5937 1 : if (mode != V16QImode)
5938 : {
5939 1 : mask = expand_simple_binop (maskmode, ASHIFT, mask,
5940 2 : GEN_INT (exact_log2 (e)),
5941 : NULL_RTX, 0, OPTAB_DIRECT);
5942 :
5943 : /* Convert mask to vector of chars. */
5944 1 : mask = force_reg (V16QImode, gen_lowpart (V16QImode, mask));
5945 :
5946 : /* Replicate each of the input bytes into byte positions:
5947 : (v2di) --> {0,0,0,0,0,0,0,0, 8,8,8,8,8,8,8,8}
5948 : (v4si) --> {0,0,0,0, 4,4,4,4, 8,8,8,8, 12,12,12,12}
5949 : (v8hi) --> {0,0, 2,2, 4,4, 6,6, ...}. */
5950 18 : for (i = 0; i < 16; ++i)
5951 16 : vec[i] = GEN_INT (i/e * e);
5952 1 : vt = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, vec));
5953 1 : vt = validize_mem (force_const_mem (V16QImode, vt));
5954 1 : if (TARGET_XOP)
5955 0 : emit_insn (gen_xop_pperm (mask, mask, mask, vt));
5956 : else
5957 1 : emit_insn (gen_ssse3_pshufbv16qi3 (mask, mask, vt));
5958 :
5959 : /* Convert it into the byte positions by doing
5960 : mask = mask + {0,1,..,16/w, 0,1,..,16/w, ...} */
5961 17 : for (i = 0; i < 16; ++i)
5962 16 : vec[i] = GEN_INT (i % e);
5963 1 : vt = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, vec));
5964 1 : vt = validize_mem (force_const_mem (V16QImode, vt));
5965 1 : emit_insn (gen_addv16qi3 (mask, mask, vt));
5966 : }
5967 :
5968 : /* The actual shuffle operations all operate on V16QImode. */
5969 1 : op0 = gen_lowpart (V16QImode, op0);
5970 1 : op1 = gen_lowpart (V16QImode, op1);
5971 :
5972 1 : if (TARGET_XOP)
5973 : {
5974 0 : if (GET_MODE (target) != V16QImode)
5975 0 : target = gen_reg_rtx (V16QImode);
5976 0 : emit_insn (gen_xop_pperm (target, op0, op1, mask));
5977 0 : if (target != operands[0])
5978 0 : emit_move_insn (operands[0],
5979 0 : gen_lowpart (GET_MODE (operands[0]), target));
5980 : }
5981 1 : else if (one_operand_shuffle)
5982 : {
5983 1 : if (GET_MODE (target) != V16QImode)
5984 1 : target = gen_reg_rtx (V16QImode);
5985 1 : emit_insn (gen_ssse3_pshufbv16qi3 (target, op0, mask));
5986 1 : if (target != operands[0])
5987 1 : emit_move_insn (operands[0],
5988 1 : gen_lowpart (GET_MODE (operands[0]), target));
5989 : }
5990 : else
5991 : {
5992 0 : rtx xops[6];
5993 0 : bool ok;
5994 :
5995 : /* Shuffle the two input vectors independently. */
5996 0 : t1 = gen_reg_rtx (V16QImode);
5997 0 : t2 = gen_reg_rtx (V16QImode);
5998 0 : emit_insn (gen_ssse3_pshufbv16qi3 (t1, op0, mask));
5999 0 : emit_insn (gen_ssse3_pshufbv16qi3 (t2, op1, mask));
6000 :
6001 1 : merge_two:
6002 : /* Then merge them together. The key is whether any given control
6003 : element contained a bit set that indicates the second word. */
6004 1 : mask = operands[3];
6005 1 : vt = GEN_INT (w);
6006 1 : if (maskmode == V2DImode && !TARGET_SSE4_1)
6007 : {
6008 : /* Without SSE4.1, we don't have V2DImode EQ. Perform one
6009 : more shuffle to convert the V2DI input mask into a V4SI
6010 : input mask. At which point the masking that expand_int_vcond
6011 : will work as desired. */
6012 0 : rtx t3 = gen_reg_rtx (V4SImode);
6013 0 : emit_insn (gen_sse2_pshufd_1 (t3, gen_lowpart (V4SImode, mask),
6014 : const0_rtx, const0_rtx,
6015 : const2_rtx, const2_rtx));
6016 0 : mask = t3;
6017 0 : maskmode = V4SImode;
6018 0 : e = w = 4;
6019 : }
6020 :
6021 1 : vt = gen_const_vec_duplicate (maskmode, vt);
6022 1 : vt = force_reg (maskmode, vt);
6023 1 : mask = expand_simple_binop (maskmode, AND, mask, vt,
6024 : NULL_RTX, 0, OPTAB_DIRECT);
6025 :
6026 1 : if (GET_MODE (target) != mode)
6027 0 : target = gen_reg_rtx (mode);
6028 1 : xops[0] = target;
6029 1 : xops[1] = gen_lowpart (mode, t2);
6030 1 : xops[2] = gen_lowpart (mode, t1);
6031 1 : xops[3] = gen_rtx_EQ (maskmode, mask, vt);
6032 1 : xops[4] = mask;
6033 1 : xops[5] = vt;
6034 1 : ok = ix86_expand_int_vcond (xops);
6035 1 : gcc_assert (ok);
6036 1 : if (target != operands[0])
6037 0 : emit_move_insn (operands[0],
6038 0 : gen_lowpart (GET_MODE (operands[0]), target));
6039 : }
6040 : }
6041 :
6042 : /* Extend SRC into next wider integer vector type. UNSIGNED_P is
6043 : true if we should do zero extension, else sign extension. */
6044 :
6045 : void
6046 753 : ix86_expand_sse_extend (rtx dest, rtx src, bool unsigned_p)
6047 : {
6048 753 : machine_mode imode = GET_MODE (src);
6049 753 : rtx ops[3];
6050 :
6051 753 : switch (imode)
6052 : {
6053 753 : case E_V8QImode:
6054 753 : case E_V4QImode:
6055 753 : case E_V2QImode:
6056 753 : case E_V4HImode:
6057 753 : case E_V2HImode:
6058 753 : case E_V2SImode:
6059 753 : break;
6060 0 : default:
6061 0 : gcc_unreachable ();
6062 : }
6063 :
6064 753 : ops[0] = dest;
6065 :
6066 753 : ops[1] = force_reg (imode, src);
6067 :
6068 753 : if (unsigned_p)
6069 236 : ops[2] = force_reg (imode, CONST0_RTX (imode));
6070 : else
6071 517 : ops[2] = ix86_expand_sse_cmp (gen_reg_rtx (imode), GT, CONST0_RTX (imode),
6072 : ops[1], pc_rtx, pc_rtx);
6073 :
6074 753 : ix86_split_mmx_punpck (ops, false);
6075 753 : }
6076 :
6077 : /* Unpack SRC into the next wider integer vector type. UNSIGNED_P is
6078 : true if we should do zero extension, else sign extension. HIGH_P is
6079 : true if we want the N/2 high elements, else the low elements. */
6080 :
6081 : void
6082 18992 : ix86_expand_sse_unpack (rtx dest, rtx src, bool unsigned_p, bool high_p)
6083 : {
6084 18992 : machine_mode imode = GET_MODE (src);
6085 18992 : rtx tmp;
6086 :
6087 18992 : if (TARGET_SSE4_1)
6088 : {
6089 6582 : rtx (*unpack)(rtx, rtx);
6090 6582 : rtx (*extract)(rtx, rtx) = NULL;
6091 6582 : machine_mode halfmode = BLKmode;
6092 :
6093 6582 : switch (imode)
6094 : {
6095 198 : case E_V64QImode:
6096 198 : if (unsigned_p)
6097 : unpack = gen_avx512bw_zero_extendv32qiv32hi2;
6098 : else
6099 64 : unpack = gen_avx512bw_sign_extendv32qiv32hi2;
6100 198 : halfmode = V32QImode;
6101 198 : extract
6102 198 : = high_p ? gen_vec_extract_hi_v64qi : gen_vec_extract_lo_v64qi;
6103 : break;
6104 711 : case E_V32QImode:
6105 711 : if (unsigned_p)
6106 : unpack = gen_avx2_zero_extendv16qiv16hi2;
6107 : else
6108 142 : unpack = gen_avx2_sign_extendv16qiv16hi2;
6109 711 : halfmode = V16QImode;
6110 711 : extract
6111 711 : = high_p ? gen_vec_extract_hi_v32qi : gen_vec_extract_lo_v32qi;
6112 : break;
6113 104 : case E_V32HImode:
6114 104 : if (unsigned_p)
6115 : unpack = gen_avx512f_zero_extendv16hiv16si2;
6116 : else
6117 64 : unpack = gen_avx512f_sign_extendv16hiv16si2;
6118 104 : halfmode = V16HImode;
6119 104 : extract
6120 104 : = high_p ? gen_vec_extract_hi_v32hi : gen_vec_extract_lo_v32hi;
6121 : break;
6122 417 : case E_V16HImode:
6123 417 : if (unsigned_p)
6124 : unpack = gen_avx2_zero_extendv8hiv8si2;
6125 : else
6126 302 : unpack = gen_avx2_sign_extendv8hiv8si2;
6127 417 : halfmode = V8HImode;
6128 417 : extract
6129 417 : = high_p ? gen_vec_extract_hi_v16hi : gen_vec_extract_lo_v16hi;
6130 : break;
6131 106 : case E_V16SImode:
6132 106 : if (unsigned_p)
6133 : unpack = gen_avx512f_zero_extendv8siv8di2;
6134 : else
6135 88 : unpack = gen_avx512f_sign_extendv8siv8di2;
6136 106 : halfmode = V8SImode;
6137 106 : extract
6138 106 : = high_p ? gen_vec_extract_hi_v16si : gen_vec_extract_lo_v16si;
6139 : break;
6140 380 : case E_V8SImode:
6141 380 : if (unsigned_p)
6142 : unpack = gen_avx2_zero_extendv4siv4di2;
6143 : else
6144 318 : unpack = gen_avx2_sign_extendv4siv4di2;
6145 380 : halfmode = V4SImode;
6146 380 : extract
6147 380 : = high_p ? gen_vec_extract_hi_v8si : gen_vec_extract_lo_v8si;
6148 : break;
6149 2661 : case E_V16QImode:
6150 2661 : if (unsigned_p)
6151 : unpack = gen_sse4_1_zero_extendv8qiv8hi2;
6152 : else
6153 308 : unpack = gen_sse4_1_sign_extendv8qiv8hi2;
6154 : break;
6155 1033 : case E_V8HImode:
6156 1033 : if (unsigned_p)
6157 : unpack = gen_sse4_1_zero_extendv4hiv4si2;
6158 : else
6159 804 : unpack = gen_sse4_1_sign_extendv4hiv4si2;
6160 : break;
6161 544 : case E_V4SImode:
6162 544 : if (unsigned_p)
6163 : unpack = gen_sse4_1_zero_extendv2siv2di2;
6164 : else
6165 484 : unpack = gen_sse4_1_sign_extendv2siv2di2;
6166 : break;
6167 127 : case E_V8QImode:
6168 127 : if (unsigned_p)
6169 : unpack = gen_sse4_1_zero_extendv4qiv4hi2;
6170 : else
6171 86 : unpack = gen_sse4_1_sign_extendv4qiv4hi2;
6172 : break;
6173 295 : case E_V4HImode:
6174 295 : if (unsigned_p)
6175 : unpack = gen_sse4_1_zero_extendv2hiv2si2;
6176 : else
6177 236 : unpack = gen_sse4_1_sign_extendv2hiv2si2;
6178 : break;
6179 6 : case E_V4QImode:
6180 6 : if (unsigned_p)
6181 : unpack = gen_sse4_1_zero_extendv2qiv2hi2;
6182 : else
6183 0 : unpack = gen_sse4_1_sign_extendv2qiv2hi2;
6184 : break;
6185 0 : default:
6186 0 : gcc_unreachable ();
6187 : }
6188 :
6189 13164 : if (GET_MODE_SIZE (imode) >= 32)
6190 : {
6191 1916 : tmp = gen_reg_rtx (halfmode);
6192 1916 : emit_insn (extract (tmp, src));
6193 : }
6194 4666 : else if (high_p)
6195 : {
6196 2476 : switch (GET_MODE_SIZE (imode))
6197 : {
6198 1021 : case 16:
6199 : /* Shift higher 8 bytes to lower 8 bytes. */
6200 1021 : tmp = gen_reg_rtx (V1TImode);
6201 1021 : emit_insn (gen_sse2_lshrv1ti3 (tmp, gen_lowpart (V1TImode, src),
6202 : GEN_INT (64)));
6203 1021 : break;
6204 214 : case 8:
6205 : /* Shift higher 4 bytes to lower 4 bytes. */
6206 214 : tmp = gen_reg_rtx (V1DImode);
6207 214 : emit_insn (gen_mmx_lshrv1di3 (tmp, gen_lowpart (V1DImode, src),
6208 : GEN_INT (32)));
6209 214 : break;
6210 3 : case 4:
6211 : /* Shift higher 2 bytes to lower 2 bytes. */
6212 3 : tmp = gen_reg_rtx (V1SImode);
6213 3 : emit_insn (gen_mmx_lshrv1si3 (tmp, gen_lowpart (V1SImode, src),
6214 : GEN_INT (16)));
6215 3 : break;
6216 0 : default:
6217 0 : gcc_unreachable ();
6218 : }
6219 :
6220 1238 : tmp = gen_lowpart (imode, tmp);
6221 : }
6222 : else
6223 : tmp = src;
6224 :
6225 6582 : emit_insn (unpack (dest, tmp));
6226 : }
6227 : else
6228 : {
6229 12410 : rtx (*unpack)(rtx, rtx, rtx);
6230 :
6231 12410 : switch (imode)
6232 : {
6233 3392 : case E_V16QImode:
6234 3392 : if (high_p)
6235 : unpack = gen_vec_interleave_highv16qi;
6236 : else
6237 1701 : unpack = gen_vec_interleave_lowv16qi;
6238 : break;
6239 5126 : case E_V8HImode:
6240 5126 : if (high_p)
6241 : unpack = gen_vec_interleave_highv8hi;
6242 : else
6243 2565 : unpack = gen_vec_interleave_lowv8hi;
6244 : break;
6245 2106 : case E_V4SImode:
6246 2106 : if (high_p)
6247 : unpack = gen_vec_interleave_highv4si;
6248 : else
6249 1054 : unpack = gen_vec_interleave_lowv4si;
6250 : break;
6251 606 : case E_V8QImode:
6252 606 : if (high_p)
6253 : unpack = gen_mmx_punpckhbw;
6254 : else
6255 305 : unpack = gen_mmx_punpcklbw;
6256 : break;
6257 1162 : case E_V4HImode:
6258 1162 : if (high_p)
6259 : unpack = gen_mmx_punpckhwd;
6260 : else
6261 583 : unpack = gen_mmx_punpcklwd;
6262 : break;
6263 18 : case E_V4QImode:
6264 18 : if (high_p)
6265 : unpack = gen_mmx_punpckhbw_low;
6266 : else
6267 9 : unpack = gen_mmx_punpcklbw_low;
6268 : break;
6269 0 : default:
6270 0 : gcc_unreachable ();
6271 : }
6272 :
6273 12410 : if (unsigned_p)
6274 4962 : tmp = force_reg (imode, CONST0_RTX (imode));
6275 : else
6276 7448 : tmp = ix86_expand_sse_cmp (gen_reg_rtx (imode), GT, CONST0_RTX (imode),
6277 : src, pc_rtx, pc_rtx);
6278 :
6279 12410 : rtx tmp2 = gen_reg_rtx (imode);
6280 12410 : emit_insn (unpack (tmp2, src, tmp));
6281 12410 : emit_move_insn (dest, gen_lowpart (GET_MODE (dest), tmp2));
6282 : }
6283 18992 : }
6284 :
6285 : /* Return true if mem is pool constant which contains a const_vector
6286 : perm index, assign the index to PERM. */
6287 : bool
6288 35 : ix86_extract_perm_from_pool_constant (int* perm, rtx mem)
6289 : {
6290 35 : machine_mode mode = GET_MODE (mem);
6291 35 : int nelt = GET_MODE_NUNITS (mode);
6292 :
6293 35 : if (!INTEGRAL_MODE_P (mode))
6294 : return false;
6295 :
6296 : /* Needs to be constant pool. */
6297 35 : if (!(MEM_P (mem))
6298 35 : || !SYMBOL_REF_P (XEXP (mem, 0))
6299 70 : || !CONSTANT_POOL_ADDRESS_P (XEXP (mem, 0)))
6300 : return false;
6301 :
6302 35 : rtx constant = get_pool_constant (XEXP (mem, 0));
6303 :
6304 35 : if (!CONST_VECTOR_P (constant))
6305 : return false;
6306 :
6307 : /* There could be some rtx like
6308 : (mem/u/c:V16QI (symbol_ref/u:DI ("*.LC1")))
6309 : but with "*.LC1" refer to V2DI constant vector. */
6310 35 : if (GET_MODE (constant) != mode)
6311 : {
6312 0 : constant = simplify_subreg (mode, constant, GET_MODE (constant), 0);
6313 :
6314 0 : if (constant == nullptr || !CONST_VECTOR_P (constant))
6315 : return false;
6316 : }
6317 :
6318 771 : for (int i = 0; i != nelt; i++)
6319 736 : perm[i] = UINTVAL (XVECEXP (constant, 0, i));
6320 :
6321 : return true;
6322 : }
6323 :
6324 : /* Split operands 0 and 1 into half-mode parts. Similar to split_double_mode,
6325 : but works for floating pointer parameters and nonoffsetable memories.
6326 : For pushes, it returns just stack offsets; the values will be saved
6327 : in the right order. Maximally three parts are generated. */
6328 :
6329 : static int
6330 4221956 : ix86_split_to_parts (rtx operand, rtx *parts, machine_mode mode)
6331 : {
6332 4221956 : int size;
6333 :
6334 4221956 : if (!TARGET_64BIT)
6335 1610096 : size = mode==XFmode ? 3 : GET_MODE_SIZE (mode) / 4;
6336 : else
6337 6832536 : size = (GET_MODE_SIZE (mode) + 4) / 8;
6338 :
6339 4221956 : gcc_assert (!REG_P (operand) || !MMX_REGNO_P (REGNO (operand)));
6340 4221956 : gcc_assert (size >= 2 && size <= 4);
6341 :
6342 : /* Optimize constant pool reference to immediates. This is used by fp
6343 : moves, that force all constants to memory to allow combining. */
6344 4221956 : if (MEM_P (operand) && MEM_READONLY_P (operand))
6345 38093 : operand = avoid_constant_pool_reference (operand);
6346 :
6347 4221956 : if (MEM_P (operand) && !offsettable_memref_p (operand))
6348 : {
6349 : /* The only non-offsetable memories we handle are pushes. */
6350 184169 : int ok = push_operand (operand, VOIDmode);
6351 :
6352 184169 : gcc_assert (ok);
6353 :
6354 184169 : operand = copy_rtx (operand);
6355 184169 : PUT_MODE (operand, word_mode);
6356 184169 : parts[0] = parts[1] = parts[2] = parts[3] = operand;
6357 184169 : return size;
6358 : }
6359 :
6360 4037787 : if (CONST_VECTOR_P (operand))
6361 : {
6362 41742 : scalar_int_mode imode = int_mode_for_mode (mode).require ();
6363 : /* Caution: if we looked through a constant pool memory above,
6364 : the operand may actually have a different mode now. That's
6365 : ok, since we want to pun this all the way back to an integer. */
6366 41742 : operand = simplify_subreg (imode, operand, GET_MODE (operand), 0);
6367 41742 : gcc_assert (operand != NULL);
6368 41742 : mode = imode;
6369 : }
6370 :
6371 4037787 : if (!TARGET_64BIT)
6372 : {
6373 647651 : if (mode == DImode)
6374 530194 : split_double_mode (mode, &operand, 1, &parts[0], &parts[1]);
6375 : else
6376 : {
6377 117457 : int i;
6378 :
6379 117457 : if (REG_P (operand))
6380 : {
6381 61028 : gcc_assert (reload_completed);
6382 183084 : for (i = 0; i < size; i++)
6383 122056 : parts[i] = gen_rtx_REG (SImode, REGNO (operand) + i);
6384 : }
6385 56429 : else if (offsettable_memref_p (operand))
6386 : {
6387 55085 : operand = adjust_address (operand, SImode, 0);
6388 55085 : parts[0] = operand;
6389 110681 : for (i = 1; i < size; i++)
6390 55596 : parts[i] = adjust_address (operand, SImode, 4 * i);
6391 : }
6392 1344 : else if (CONST_DOUBLE_P (operand))
6393 : {
6394 1344 : const REAL_VALUE_TYPE *r;
6395 1344 : long l[4];
6396 :
6397 1344 : r = CONST_DOUBLE_REAL_VALUE (operand);
6398 1344 : switch (mode)
6399 : {
6400 0 : case E_TFmode:
6401 0 : real_to_target (l, r, mode);
6402 0 : parts[3] = gen_int_mode (l[3], SImode);
6403 0 : parts[2] = gen_int_mode (l[2], SImode);
6404 0 : break;
6405 198 : case E_XFmode:
6406 : /* We can't use REAL_VALUE_TO_TARGET_LONG_DOUBLE since
6407 : long double may not be 80-bit. */
6408 198 : real_to_target (l, r, mode);
6409 198 : parts[2] = gen_int_mode (l[2], SImode);
6410 198 : break;
6411 1146 : case E_DFmode:
6412 1146 : REAL_VALUE_TO_TARGET_DOUBLE (*r, l);
6413 1146 : break;
6414 0 : default:
6415 0 : gcc_unreachable ();
6416 : }
6417 1344 : parts[1] = gen_int_mode (l[1], SImode);
6418 1344 : parts[0] = gen_int_mode (l[0], SImode);
6419 : }
6420 : else
6421 0 : gcc_unreachable ();
6422 : }
6423 : }
6424 : else
6425 : {
6426 3390136 : if (mode == TImode)
6427 3369994 : split_double_mode (mode, &operand, 1, &parts[0], &parts[1]);
6428 3390136 : if (mode == XFmode || mode == TFmode)
6429 : {
6430 20142 : machine_mode upper_mode = mode==XFmode ? SImode : DImode;
6431 20142 : if (REG_P (operand))
6432 : {
6433 1472 : gcc_assert (reload_completed);
6434 1472 : parts[0] = gen_rtx_REG (DImode, REGNO (operand) + 0);
6435 1472 : parts[1] = gen_rtx_REG (upper_mode, REGNO (operand) + 1);
6436 : }
6437 18670 : else if (offsettable_memref_p (operand))
6438 : {
6439 14849 : operand = adjust_address (operand, DImode, 0);
6440 14849 : parts[0] = operand;
6441 14849 : parts[1] = adjust_address (operand, upper_mode, 8);
6442 : }
6443 3821 : else if (CONST_DOUBLE_P (operand))
6444 : {
6445 3821 : long l[4];
6446 :
6447 3821 : real_to_target (l, CONST_DOUBLE_REAL_VALUE (operand), mode);
6448 :
6449 : /* real_to_target puts 32-bit pieces in each long. */
6450 7642 : parts[0] = gen_int_mode ((l[0] & HOST_WIDE_INT_C (0xffffffff))
6451 3821 : | ((l[1] & HOST_WIDE_INT_C (0xffffffff))
6452 3821 : << 32), DImode);
6453 :
6454 3821 : if (upper_mode == SImode)
6455 2793 : parts[1] = gen_int_mode (l[2], SImode);
6456 : else
6457 1028 : parts[1]
6458 1028 : = gen_int_mode ((l[2] & HOST_WIDE_INT_C (0xffffffff))
6459 1028 : | ((l[3] & HOST_WIDE_INT_C (0xffffffff))
6460 1028 : << 32), DImode);
6461 : }
6462 : else
6463 0 : gcc_unreachable ();
6464 : }
6465 : }
6466 :
6467 : return size;
6468 : }
6469 :
6470 : /* Emit insns to perform a move or push of DI, DF, XF, and TF values.
6471 : Return false when normal moves are needed; true when all required
6472 : insns have been emitted. Operands 2-4 contain the input values
6473 : int the correct order; operands 5-7 contain the output values. */
6474 :
6475 : void
6476 2123940 : ix86_split_long_move (rtx operands[])
6477 : {
6478 2123940 : rtx part[2][4];
6479 2123940 : int nparts, i, j;
6480 2123940 : int push = 0;
6481 2123940 : int collisions = 0;
6482 2123940 : machine_mode mode = GET_MODE (operands[0]);
6483 2123940 : bool collisionparts[4];
6484 :
6485 : /* The DFmode expanders may ask us to move double.
6486 : For 64bit target this is single move. By hiding the fact
6487 : here we simplify i386.md splitters. */
6488 3845036 : if (TARGET_64BIT && GET_MODE_SIZE (GET_MODE (operands[0])) == 8)
6489 : {
6490 : /* Optimize constant pool reference to immediates. This is used by
6491 : fp moves, that force all constants to memory to allow combining. */
6492 :
6493 12962 : if (MEM_P (operands[1])
6494 12538 : && SYMBOL_REF_P (XEXP (operands[1], 0))
6495 13606 : && CONSTANT_POOL_ADDRESS_P (XEXP (operands[1], 0)))
6496 155 : operands[1] = get_pool_constant (XEXP (operands[1], 0));
6497 12962 : if (push_operand (operands[0], VOIDmode))
6498 : {
6499 12962 : operands[0] = copy_rtx (operands[0]);
6500 12962 : PUT_MODE (operands[0], word_mode);
6501 : }
6502 : else
6503 0 : operands[0] = gen_lowpart (DImode, operands[0]);
6504 12962 : operands[1] = gen_lowpart (DImode, operands[1]);
6505 12962 : emit_move_insn (operands[0], operands[1]);
6506 12962 : return;
6507 : }
6508 :
6509 : /* The only non-offsettable memory we handle is push. */
6510 2110978 : if (push_operand (operands[0], VOIDmode))
6511 : push = 1;
6512 : else
6513 1926809 : gcc_assert (!MEM_P (operands[0])
6514 : || offsettable_memref_p (operands[0]));
6515 :
6516 2110978 : nparts = ix86_split_to_parts (operands[1], part[1], GET_MODE (operands[0]));
6517 2110978 : ix86_split_to_parts (operands[0], part[0], GET_MODE (operands[0]));
6518 :
6519 : /* When emitting push, take care for source operands on the stack. */
6520 184169 : if (push && MEM_P (operands[1])
6521 2208626 : && reg_overlap_mentioned_p (stack_pointer_rtx, operands[1]))
6522 : {
6523 56120 : rtx src_base = XEXP (part[1][nparts - 1], 0);
6524 :
6525 : /* Compensate for the stack decrement by 4. */
6526 56120 : if (!TARGET_64BIT && nparts == 3
6527 51397 : && mode == XFmode && TARGET_128BIT_LONG_DOUBLE)
6528 0 : src_base = plus_constant (Pmode, src_base, 4);
6529 :
6530 : /* src_base refers to the stack pointer and is
6531 : automatically decreased by emitted push. */
6532 168616 : for (i = 0; i < nparts; i++)
6533 112496 : part[1][i] = change_address (part[1][i],
6534 112496 : GET_MODE (part[1][i]), src_base);
6535 : }
6536 :
6537 : /* We need to do copy in the right order in case an address register
6538 : of the source overlaps the destination. */
6539 2110978 : if (REG_P (part[0][0]) && MEM_P (part[1][0]))
6540 : {
6541 : rtx tmp;
6542 :
6543 2383713 : for (i = 0; i < nparts; i++)
6544 : {
6545 1589142 : collisionparts[i]
6546 1589142 : = reg_overlap_mentioned_p (part[0][i], XEXP (part[1][0], 0));
6547 1589142 : if (collisionparts[i])
6548 16928 : collisions++;
6549 : }
6550 :
6551 : /* Collision in the middle part can be handled by reordering. */
6552 794571 : if (collisions == 1 && nparts == 3 && collisionparts [1])
6553 : {
6554 0 : std::swap (part[0][1], part[0][2]);
6555 0 : std::swap (part[1][1], part[1][2]);
6556 : }
6557 794571 : else if (collisions == 1
6558 794571 : && nparts == 4
6559 0 : && (collisionparts [1] || collisionparts [2]))
6560 : {
6561 0 : if (collisionparts [1])
6562 : {
6563 0 : std::swap (part[0][1], part[0][2]);
6564 0 : std::swap (part[1][1], part[1][2]);
6565 : }
6566 : else
6567 : {
6568 0 : std::swap (part[0][2], part[0][3]);
6569 0 : std::swap (part[1][2], part[1][3]);
6570 : }
6571 : }
6572 :
6573 : /* If there are more collisions, we can't handle it by reordering.
6574 : Do an lea to the last part and use only one colliding move. */
6575 794571 : else if (collisions > 1)
6576 : {
6577 75 : rtx base, addr;
6578 :
6579 75 : collisions = 1;
6580 :
6581 75 : base = part[0][nparts - 1];
6582 :
6583 : /* Handle the case when the last part isn't valid for lea.
6584 : Happens in 64-bit mode storing the 12-byte XFmode. */
6585 106 : if (GET_MODE (base) != Pmode)
6586 0 : base = gen_rtx_REG (Pmode, REGNO (base));
6587 :
6588 75 : addr = XEXP (part[1][0], 0);
6589 75 : if (TARGET_TLS_DIRECT_SEG_REFS)
6590 : {
6591 75 : struct ix86_address parts;
6592 75 : int ok = ix86_decompose_address (addr, &parts);
6593 75 : gcc_assert (ok);
6594 : /* It is not valid to use %gs: or %fs: in lea. */
6595 75 : gcc_assert (parts.seg == ADDR_SPACE_GENERIC);
6596 : }
6597 75 : emit_insn (gen_rtx_SET (base, addr));
6598 75 : part[1][0] = replace_equiv_address (part[1][0], base);
6599 150 : for (i = 1; i < nparts; i++)
6600 : {
6601 137 : tmp = plus_constant (Pmode, base, UNITS_PER_WORD * i);
6602 75 : part[1][i] = replace_equiv_address (part[1][i], tmp);
6603 : }
6604 : }
6605 : }
6606 :
6607 2110978 : if (push)
6608 : {
6609 184169 : if (!TARGET_64BIT)
6610 : {
6611 158037 : if (nparts == 3)
6612 : {
6613 571 : if (TARGET_128BIT_LONG_DOUBLE && mode == XFmode)
6614 0 : emit_insn (gen_add2_insn (stack_pointer_rtx, GEN_INT (-4)));
6615 571 : emit_move_insn (part[0][2], part[1][2]);
6616 : }
6617 157466 : else if (nparts == 4)
6618 : {
6619 0 : emit_move_insn (part[0][3], part[1][3]);
6620 0 : emit_move_insn (part[0][2], part[1][2]);
6621 : }
6622 : }
6623 : else
6624 : {
6625 : /* In 64bit mode we don't have 32bit push available. In case this is
6626 : register, it is OK - we will just use larger counterpart. We also
6627 : retype memory - these comes from attempt to avoid REX prefix on
6628 : moving of second half of TFmode value. */
6629 26132 : if (GET_MODE (part[1][1]) == SImode)
6630 : {
6631 11888 : switch (GET_CODE (part[1][1]))
6632 : {
6633 11459 : case MEM:
6634 11459 : part[1][1] = adjust_address (part[1][1], DImode, 0);
6635 11459 : break;
6636 :
6637 429 : case REG:
6638 429 : part[1][1] = gen_rtx_REG (DImode, REGNO (part[1][1]));
6639 429 : break;
6640 :
6641 0 : default:
6642 0 : gcc_unreachable ();
6643 : }
6644 :
6645 11888 : if (GET_MODE (part[1][0]) == SImode)
6646 0 : part[1][0] = part[1][1];
6647 : }
6648 : }
6649 184169 : emit_move_insn (part[0][1], part[1][1]);
6650 184169 : emit_move_insn (part[0][0], part[1][0]);
6651 184169 : return;
6652 : }
6653 :
6654 : /* Choose correct order to not overwrite the source before it is copied. */
6655 1926809 : if ((REG_P (part[0][0])
6656 1050430 : && REG_P (part[1][1])
6657 93799 : && (REGNO (part[0][0]) == REGNO (part[1][1])
6658 76184 : || (nparts == 3
6659 0 : && REGNO (part[0][0]) == REGNO (part[1][2]))
6660 76184 : || (nparts == 4
6661 0 : && REGNO (part[0][0]) == REGNO (part[1][3]))))
6662 2959624 : || (collisions > 0
6663 16853 : && reg_overlap_mentioned_p (part[0][0], XEXP (part[1][0], 0))))
6664 : {
6665 102651 : for (i = 0, j = nparts - 1; i < nparts; i++, j--)
6666 : {
6667 68434 : operands[2 + i] = part[0][j];
6668 68434 : operands[6 + i] = part[1][j];
6669 : }
6670 : }
6671 : else
6672 : {
6673 5677845 : for (i = 0; i < nparts; i++)
6674 : {
6675 3785253 : operands[2 + i] = part[0][i];
6676 3785253 : operands[6 + i] = part[1][i];
6677 : }
6678 : }
6679 :
6680 : /* Attempt to locally unCSE nonzero constants. */
6681 3853687 : for (j = 0; j < nparts - 1; j++)
6682 1926878 : if (CONST_INT_P (operands[6 + j])
6683 236822 : && operands[6 + j] != const0_rtx
6684 65723 : && REG_P (operands[2 + j]))
6685 116128 : for (i = j; i < nparts - 1; i++)
6686 58064 : if (CONST_INT_P (operands[7 + i])
6687 58064 : && INTVAL (operands[7 + i]) == INTVAL (operands[6 + j]))
6688 22643 : operands[7 + i] = operands[2 + j];
6689 :
6690 5780496 : for (i = 0; i < nparts; i++)
6691 3853687 : emit_move_insn (operands[2 + i], operands[6 + i]);
6692 :
6693 : return;
6694 : }
6695 :
6696 : /* Helper function of ix86_split_ashl used to generate an SImode/DImode
6697 : left shift by a constant, either using a single shift or
6698 : a sequence of add instructions. */
6699 :
6700 : static void
6701 4408 : ix86_expand_ashl_const (rtx operand, int count, machine_mode mode)
6702 : {
6703 4408 : if (count == 1
6704 4408 : || (count * ix86_cost->add <= ix86_cost->shift_const
6705 0 : && !optimize_insn_for_size_p ()))
6706 : {
6707 16 : while (count-- > 0)
6708 8 : emit_insn (gen_add2_insn (operand, operand));
6709 : }
6710 : else
6711 : {
6712 4400 : rtx (*insn)(rtx, rtx, rtx);
6713 :
6714 4400 : insn = mode == DImode ? gen_ashlsi3 : gen_ashldi3;
6715 4400 : emit_insn (insn (operand, operand, GEN_INT (count)));
6716 : }
6717 4408 : }
6718 :
6719 : void
6720 10624 : ix86_split_ashl (rtx *operands, rtx scratch, machine_mode mode)
6721 : {
6722 10624 : rtx (*gen_ashl3)(rtx, rtx, rtx);
6723 10624 : rtx (*gen_shld)(rtx, rtx, rtx);
6724 10624 : int half_width = GET_MODE_BITSIZE (mode) >> 1;
6725 10624 : machine_mode half_mode;
6726 :
6727 10624 : rtx low[2], high[2];
6728 10624 : int count;
6729 :
6730 10624 : if (CONST_INT_P (operands[2]))
6731 : {
6732 8904 : split_double_mode (mode, operands, 2, low, high);
6733 8904 : count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
6734 :
6735 8904 : if (count >= half_width)
6736 : {
6737 2709 : emit_move_insn (high[0], low[1]);
6738 2709 : ix86_expand_clear (low[0]);
6739 :
6740 2709 : if (count > half_width)
6741 153 : ix86_expand_ashl_const (high[0], count - half_width, mode);
6742 : }
6743 6195 : else if (count == 1)
6744 : {
6745 1940 : if (!rtx_equal_p (operands[0], operands[1]))
6746 0 : emit_move_insn (operands[0], operands[1]);
6747 1940 : rtx x3 = gen_rtx_REG (CCCmode, FLAGS_REG);
6748 1940 : rtx x4 = gen_rtx_LTU (mode, x3, const0_rtx);
6749 1940 : half_mode = mode == DImode ? SImode : DImode;
6750 1940 : emit_insn (gen_add3_cc_overflow_1 (half_mode, low[0],
6751 : low[0], low[0]));
6752 1940 : emit_insn (gen_add3_carry (half_mode, high[0], high[0], high[0],
6753 : x3, x4));
6754 : }
6755 : else
6756 : {
6757 4255 : gen_shld = mode == DImode ? gen_x86_shld : gen_x86_64_shld;
6758 :
6759 4255 : if (!rtx_equal_p (operands[0], operands[1]))
6760 0 : emit_move_insn (operands[0], operands[1]);
6761 :
6762 4255 : emit_insn (gen_shld (high[0], low[0], GEN_INT (count)));
6763 4255 : ix86_expand_ashl_const (low[0], count, mode);
6764 : }
6765 9165 : return;
6766 : }
6767 :
6768 1720 : split_double_mode (mode, operands, 1, low, high);
6769 1720 : half_mode = mode == DImode ? SImode : DImode;
6770 :
6771 1720 : gen_ashl3 = mode == DImode ? gen_ashlsi3 : gen_ashldi3;
6772 :
6773 1720 : if (operands[1] == const1_rtx)
6774 : {
6775 : /* Assuming we've chosen a QImode capable registers, then 1 << N
6776 : can be done with two 32/64-bit shifts, no branches, no cmoves. */
6777 261 : if (ANY_QI_REG_P (low[0]) && ANY_QI_REG_P (high[0]))
6778 : {
6779 151 : rtx s, d, flags = gen_rtx_REG (CCZmode, FLAGS_REG);
6780 :
6781 151 : ix86_expand_clear (low[0]);
6782 151 : ix86_expand_clear (high[0]);
6783 151 : emit_insn (gen_testqi_ccz_1 (operands[2], GEN_INT (half_width)));
6784 :
6785 151 : d = gen_lowpart (QImode, low[0]);
6786 151 : d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
6787 151 : s = gen_rtx_EQ (QImode, flags, const0_rtx);
6788 151 : emit_insn (gen_rtx_SET (d, s));
6789 :
6790 151 : d = gen_lowpart (QImode, high[0]);
6791 151 : d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
6792 151 : s = gen_rtx_NE (QImode, flags, const0_rtx);
6793 151 : emit_insn (gen_rtx_SET (d, s));
6794 : }
6795 :
6796 : /* Otherwise, we can get the same results by manually performing
6797 : a bit extract operation on bit 5/6, and then performing the two
6798 : shifts. The two methods of getting 0/1 into low/high are exactly
6799 : the same size. Avoiding the shift in the bit extract case helps
6800 : pentium4 a bit; no one else seems to care much either way. */
6801 : else
6802 : {
6803 110 : rtx (*gen_lshr3)(rtx, rtx, rtx);
6804 110 : rtx (*gen_and3)(rtx, rtx, rtx);
6805 110 : rtx (*gen_xor3)(rtx, rtx, rtx);
6806 110 : HOST_WIDE_INT bits;
6807 110 : rtx x;
6808 :
6809 110 : if (mode == DImode)
6810 : {
6811 : gen_lshr3 = gen_lshrsi3;
6812 : gen_and3 = gen_andsi3;
6813 : gen_xor3 = gen_xorsi3;
6814 : bits = 5;
6815 : }
6816 : else
6817 : {
6818 0 : gen_lshr3 = gen_lshrdi3;
6819 0 : gen_and3 = gen_anddi3;
6820 0 : gen_xor3 = gen_xordi3;
6821 0 : bits = 6;
6822 : }
6823 :
6824 110 : if (TARGET_PARTIAL_REG_STALL && !optimize_insn_for_size_p ())
6825 0 : x = gen_rtx_ZERO_EXTEND (half_mode, operands[2]);
6826 : else
6827 110 : x = gen_lowpart (half_mode, operands[2]);
6828 110 : emit_insn (gen_rtx_SET (high[0], x));
6829 :
6830 110 : emit_insn (gen_lshr3 (high[0], high[0], GEN_INT (bits)));
6831 110 : emit_insn (gen_and3 (high[0], high[0], const1_rtx));
6832 110 : emit_move_insn (low[0], high[0]);
6833 110 : emit_insn (gen_xor3 (low[0], low[0], const1_rtx));
6834 : }
6835 :
6836 261 : emit_insn (gen_ashl3 (low[0], low[0], operands[2]));
6837 261 : emit_insn (gen_ashl3 (high[0], high[0], operands[2]));
6838 261 : return;
6839 : }
6840 :
6841 1459 : if (operands[1] == constm1_rtx)
6842 : {
6843 : /* For -1 << N, we can avoid the shld instruction, because we
6844 : know that we're shifting 0...31/63 ones into a -1. */
6845 109 : emit_move_insn (low[0], constm1_rtx);
6846 109 : if (optimize_insn_for_size_p ())
6847 6 : emit_move_insn (high[0], low[0]);
6848 : else
6849 103 : emit_move_insn (high[0], constm1_rtx);
6850 : }
6851 : else
6852 : {
6853 1350 : gen_shld = mode == DImode ? gen_x86_shld : gen_x86_64_shld;
6854 :
6855 1350 : if (!rtx_equal_p (operands[0], operands[1]))
6856 0 : emit_move_insn (operands[0], operands[1]);
6857 :
6858 1350 : split_double_mode (mode, operands, 1, low, high);
6859 1350 : emit_insn (gen_shld (high[0], low[0], operands[2]));
6860 : }
6861 :
6862 1459 : emit_insn (gen_ashl3 (low[0], low[0], operands[2]));
6863 :
6864 1459 : if (TARGET_CMOVE && scratch)
6865 : {
6866 998 : ix86_expand_clear (scratch);
6867 998 : emit_insn (gen_x86_shift_adj_1
6868 : (half_mode, high[0], low[0], operands[2], scratch));
6869 : }
6870 : else
6871 461 : emit_insn (gen_x86_shift_adj_2 (half_mode, high[0], low[0], operands[2]));
6872 : }
6873 :
6874 : void
6875 6331 : ix86_split_ashr (rtx *operands, rtx scratch, machine_mode mode)
6876 : {
6877 5085 : rtx (*gen_ashr3)(rtx, rtx, rtx)
6878 6331 : = mode == DImode ? gen_ashrsi3 : gen_ashrdi3;
6879 6331 : rtx (*gen_shrd)(rtx, rtx, rtx);
6880 6331 : int half_width = GET_MODE_BITSIZE (mode) >> 1;
6881 :
6882 6331 : rtx low[2], high[2];
6883 6331 : int count;
6884 :
6885 6331 : if (CONST_INT_P (operands[2]))
6886 : {
6887 6157 : split_double_mode (mode, operands, 2, low, high);
6888 6157 : count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
6889 :
6890 12314 : if (count == GET_MODE_BITSIZE (mode) - 1)
6891 : {
6892 93 : emit_move_insn (high[0], high[1]);
6893 93 : emit_insn (gen_ashr3 (high[0], high[0],
6894 93 : GEN_INT (half_width - 1)));
6895 93 : emit_move_insn (low[0], high[0]);
6896 :
6897 : }
6898 6064 : else if (count >= half_width)
6899 : {
6900 1999 : emit_move_insn (low[0], high[1]);
6901 1999 : emit_move_insn (high[0], low[0]);
6902 1999 : emit_insn (gen_ashr3 (high[0], high[0],
6903 1999 : GEN_INT (half_width - 1)));
6904 :
6905 1999 : if (count > half_width)
6906 38 : emit_insn (gen_ashr3 (low[0], low[0],
6907 38 : GEN_INT (count - half_width)));
6908 : }
6909 4065 : else if (count == 1
6910 767 : && (TARGET_USE_RCR || optimize_size > 1))
6911 : {
6912 1 : if (!rtx_equal_p (operands[0], operands[1]))
6913 0 : emit_move_insn (operands[0], operands[1]);
6914 1 : if (mode == DImode)
6915 : {
6916 0 : emit_insn (gen_ashrsi3_carry (high[0], high[0]));
6917 0 : emit_insn (gen_rcrsi2 (low[0], low[0]));
6918 : }
6919 : else
6920 : {
6921 1 : emit_insn (gen_ashrdi3_carry (high[0], high[0]));
6922 1 : emit_insn (gen_rcrdi2 (low[0], low[0]));
6923 : }
6924 : }
6925 : else
6926 : {
6927 4064 : gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
6928 :
6929 4064 : if (!rtx_equal_p (operands[0], operands[1]))
6930 0 : emit_move_insn (operands[0], operands[1]);
6931 :
6932 4064 : emit_insn (gen_shrd (low[0], high[0], GEN_INT (count)));
6933 4064 : emit_insn (gen_ashr3 (high[0], high[0], GEN_INT (count)));
6934 : }
6935 : }
6936 : else
6937 : {
6938 174 : machine_mode half_mode;
6939 :
6940 174 : gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
6941 :
6942 174 : if (!rtx_equal_p (operands[0], operands[1]))
6943 0 : emit_move_insn (operands[0], operands[1]);
6944 :
6945 174 : split_double_mode (mode, operands, 1, low, high);
6946 174 : half_mode = mode == DImode ? SImode : DImode;
6947 :
6948 174 : emit_insn (gen_shrd (low[0], high[0], operands[2]));
6949 174 : emit_insn (gen_ashr3 (high[0], high[0], operands[2]));
6950 :
6951 174 : if (TARGET_CMOVE && scratch)
6952 : {
6953 134 : emit_move_insn (scratch, high[0]);
6954 134 : emit_insn (gen_ashr3 (scratch, scratch,
6955 134 : GEN_INT (half_width - 1)));
6956 134 : emit_insn (gen_x86_shift_adj_1
6957 : (half_mode, low[0], high[0], operands[2], scratch));
6958 : }
6959 : else
6960 40 : emit_insn (gen_x86_shift_adj_3
6961 : (half_mode, low[0], high[0], operands[2]));
6962 : }
6963 6331 : }
6964 :
6965 : void
6966 16830 : ix86_split_lshr (rtx *operands, rtx scratch, machine_mode mode)
6967 : {
6968 6795 : rtx (*gen_lshr3)(rtx, rtx, rtx)
6969 16830 : = mode == DImode ? gen_lshrsi3 : gen_lshrdi3;
6970 16830 : rtx (*gen_shrd)(rtx, rtx, rtx);
6971 16830 : int half_width = GET_MODE_BITSIZE (mode) >> 1;
6972 :
6973 16830 : rtx low[2], high[2];
6974 16830 : int count;
6975 :
6976 16830 : if (CONST_INT_P (operands[2]))
6977 : {
6978 15395 : split_double_mode (mode, operands, 2, low, high);
6979 15395 : count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
6980 :
6981 15395 : if (count >= half_width)
6982 : {
6983 11923 : emit_move_insn (low[0], high[1]);
6984 11923 : ix86_expand_clear (high[0]);
6985 :
6986 11923 : if (count > half_width)
6987 1483 : emit_insn (gen_lshr3 (low[0], low[0],
6988 1483 : GEN_INT (count - half_width)));
6989 : }
6990 3472 : else if (count == 1
6991 649 : && (TARGET_USE_RCR || optimize_size > 1))
6992 : {
6993 1 : if (!rtx_equal_p (operands[0], operands[1]))
6994 0 : emit_move_insn (operands[0], operands[1]);
6995 1 : if (mode == DImode)
6996 : {
6997 0 : emit_insn (gen_lshrsi3_carry (high[0], high[0]));
6998 0 : emit_insn (gen_rcrsi2 (low[0], low[0]));
6999 : }
7000 : else
7001 : {
7002 1 : emit_insn (gen_lshrdi3_carry (high[0], high[0]));
7003 1 : emit_insn (gen_rcrdi2 (low[0], low[0]));
7004 : }
7005 : }
7006 : else
7007 : {
7008 3471 : gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
7009 :
7010 3471 : if (!rtx_equal_p (operands[0], operands[1]))
7011 0 : emit_move_insn (operands[0], operands[1]);
7012 :
7013 3471 : emit_insn (gen_shrd (low[0], high[0], GEN_INT (count)));
7014 3471 : emit_insn (gen_lshr3 (high[0], high[0], GEN_INT (count)));
7015 : }
7016 : }
7017 : else
7018 : {
7019 1435 : machine_mode half_mode;
7020 :
7021 1435 : gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
7022 :
7023 1435 : if (!rtx_equal_p (operands[0], operands[1]))
7024 0 : emit_move_insn (operands[0], operands[1]);
7025 :
7026 1435 : split_double_mode (mode, operands, 1, low, high);
7027 1435 : half_mode = mode == DImode ? SImode : DImode;
7028 :
7029 1435 : emit_insn (gen_shrd (low[0], high[0], operands[2]));
7030 1435 : emit_insn (gen_lshr3 (high[0], high[0], operands[2]));
7031 :
7032 1435 : if (TARGET_CMOVE && scratch)
7033 : {
7034 1150 : ix86_expand_clear (scratch);
7035 1150 : emit_insn (gen_x86_shift_adj_1
7036 : (half_mode, low[0], high[0], operands[2], scratch));
7037 : }
7038 : else
7039 285 : emit_insn (gen_x86_shift_adj_2
7040 : (half_mode, low[0], high[0], operands[2]));
7041 : }
7042 16830 : }
7043 :
7044 : /* Helper function to split TImode ashl under NDD. */
7045 : void
7046 1 : ix86_split_ashl_ndd (rtx *operands, rtx scratch)
7047 : {
7048 1 : gcc_assert (TARGET_APX_NDD);
7049 1 : int half_width = GET_MODE_BITSIZE (TImode) >> 1;
7050 :
7051 1 : rtx low[2], high[2];
7052 1 : int count;
7053 :
7054 1 : split_double_mode (TImode, operands, 2, low, high);
7055 1 : if (CONST_INT_P (operands[2]))
7056 : {
7057 0 : count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (TImode) - 1);
7058 :
7059 0 : if (count >= half_width)
7060 : {
7061 0 : count = count - half_width;
7062 0 : if (count == 0)
7063 : {
7064 0 : if (!rtx_equal_p (high[0], low[1]))
7065 0 : emit_move_insn (high[0], low[1]);
7066 : }
7067 0 : else if (count == 1)
7068 0 : emit_insn (gen_adddi3 (high[0], low[1], low[1]));
7069 : else
7070 0 : emit_insn (gen_ashldi3 (high[0], low[1], GEN_INT (count)));
7071 :
7072 0 : ix86_expand_clear (low[0]);
7073 : }
7074 0 : else if (count == 1)
7075 : {
7076 0 : rtx x3 = gen_rtx_REG (CCCmode, FLAGS_REG);
7077 0 : rtx x4 = gen_rtx_LTU (TImode, x3, const0_rtx);
7078 0 : emit_insn (gen_add3_cc_overflow_1 (DImode, low[0],
7079 : low[1], low[1]));
7080 0 : emit_insn (gen_add3_carry (DImode, high[0], high[1], high[1],
7081 : x3, x4));
7082 : }
7083 : else
7084 : {
7085 0 : emit_insn (gen_x86_64_shld_ndd (high[0], high[1], low[1],
7086 : GEN_INT (count)));
7087 0 : emit_insn (gen_ashldi3 (low[0], low[1], GEN_INT (count)));
7088 : }
7089 : }
7090 : else
7091 : {
7092 1 : emit_insn (gen_x86_64_shld_ndd (high[0], high[1], low[1],
7093 : operands[2]));
7094 1 : emit_insn (gen_ashldi3 (low[0], low[1], operands[2]));
7095 1 : if (TARGET_CMOVE && scratch)
7096 : {
7097 1 : ix86_expand_clear (scratch);
7098 1 : emit_insn (gen_x86_shift_adj_1
7099 : (DImode, high[0], low[0], operands[2], scratch));
7100 : }
7101 : else
7102 0 : emit_insn (gen_x86_shift_adj_2 (DImode, high[0], low[0], operands[2]));
7103 : }
7104 1 : }
7105 :
7106 : /* Helper function to split TImode l/ashr under NDD. */
7107 : void
7108 2 : ix86_split_rshift_ndd (enum rtx_code code, rtx *operands, rtx scratch)
7109 : {
7110 2 : gcc_assert (TARGET_APX_NDD);
7111 2 : int half_width = GET_MODE_BITSIZE (TImode) >> 1;
7112 2 : bool ashr_p = code == ASHIFTRT;
7113 2 : rtx (*gen_shr)(rtx, rtx, rtx) = ashr_p ? gen_ashrdi3
7114 : : gen_lshrdi3;
7115 :
7116 2 : rtx low[2], high[2];
7117 2 : int count;
7118 :
7119 2 : split_double_mode (TImode, operands, 2, low, high);
7120 2 : if (CONST_INT_P (operands[2]))
7121 : {
7122 0 : count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (TImode) - 1);
7123 :
7124 0 : if (ashr_p && (count == GET_MODE_BITSIZE (TImode) - 1))
7125 : {
7126 0 : emit_insn (gen_shr (high[0], high[1],
7127 : GEN_INT (half_width - 1)));
7128 0 : emit_move_insn (low[0], high[0]);
7129 : }
7130 0 : else if (count >= half_width)
7131 : {
7132 0 : if (ashr_p)
7133 0 : emit_insn (gen_shr (high[0], high[1],
7134 : GEN_INT (half_width - 1)));
7135 : else
7136 0 : ix86_expand_clear (high[0]);
7137 :
7138 0 : if (count > half_width)
7139 0 : emit_insn (gen_shr (low[0], high[1],
7140 0 : GEN_INT (count - half_width)));
7141 : else
7142 0 : emit_move_insn (low[0], high[1]);
7143 : }
7144 : else
7145 : {
7146 0 : emit_insn (gen_x86_64_shrd_ndd (low[0], low[1], high[1],
7147 : GEN_INT (count)));
7148 0 : emit_insn (gen_shr (high[0], high[1], GEN_INT (count)));
7149 : }
7150 : }
7151 : else
7152 : {
7153 2 : emit_insn (gen_x86_64_shrd_ndd (low[0], low[1], high[1],
7154 : operands[2]));
7155 2 : emit_insn (gen_shr (high[0], high[1], operands[2]));
7156 :
7157 2 : if (TARGET_CMOVE && scratch)
7158 : {
7159 2 : if (ashr_p)
7160 : {
7161 1 : emit_move_insn (scratch, high[0]);
7162 1 : emit_insn (gen_shr (scratch, scratch,
7163 : GEN_INT (half_width - 1)));
7164 : }
7165 : else
7166 1 : ix86_expand_clear (scratch);
7167 :
7168 2 : emit_insn (gen_x86_shift_adj_1
7169 : (DImode, low[0], high[0], operands[2], scratch));
7170 : }
7171 0 : else if (ashr_p)
7172 0 : emit_insn (gen_x86_shift_adj_3
7173 : (DImode, low[0], high[0], operands[2]));
7174 : else
7175 0 : emit_insn (gen_x86_shift_adj_2
7176 : (DImode, low[0], high[0], operands[2]));
7177 : }
7178 2 : }
7179 :
7180 : /* Expand move of V1TI mode register X to a new TI mode register. */
7181 : static rtx
7182 18 : ix86_expand_v1ti_to_ti (rtx x)
7183 : {
7184 18 : rtx result = gen_reg_rtx (TImode);
7185 18 : if (TARGET_SSE2)
7186 : {
7187 18 : rtx temp = force_reg (V2DImode, gen_lowpart (V2DImode, x));
7188 18 : rtx lo = gen_lowpart (DImode, result);
7189 18 : emit_insn (gen_vec_extractv2didi (lo, temp, const0_rtx));
7190 18 : rtx hi = gen_highpart (DImode, result);
7191 18 : emit_insn (gen_vec_extractv2didi (hi, temp, const1_rtx));
7192 : }
7193 : else
7194 0 : emit_move_insn (result, gen_lowpart (TImode, x));
7195 18 : return result;
7196 : }
7197 :
7198 : /* Expand move of TI mode register X to a new V1TI mode register. */
7199 : static rtx
7200 18 : ix86_expand_ti_to_v1ti (rtx x)
7201 : {
7202 18 : if (TARGET_SSE2)
7203 : {
7204 18 : rtx lo = gen_lowpart (DImode, x);
7205 18 : rtx hi = gen_highpart (DImode, x);
7206 18 : rtx tmp = gen_reg_rtx (V2DImode);
7207 18 : emit_insn (gen_vec_concatv2di (tmp, lo, hi));
7208 18 : return force_reg (V1TImode, gen_lowpart (V1TImode, tmp));
7209 : }
7210 :
7211 0 : return force_reg (V1TImode, gen_lowpart (V1TImode, x));
7212 : }
7213 :
7214 : /* Expand V1TI mode shift (of rtx_code CODE) by constant. */
7215 : void
7216 43 : ix86_expand_v1ti_shift (enum rtx_code code, rtx operands[])
7217 : {
7218 43 : rtx op1 = force_reg (V1TImode, operands[1]);
7219 :
7220 43 : if (!CONST_INT_P (operands[2]))
7221 : {
7222 7 : rtx tmp1 = ix86_expand_v1ti_to_ti (op1);
7223 7 : rtx tmp2 = gen_reg_rtx (TImode);
7224 3 : rtx (*shift) (rtx, rtx, rtx)
7225 7 : = (code == ASHIFT) ? gen_ashlti3 : gen_lshrti3;
7226 7 : emit_insn (shift (tmp2, tmp1, operands[2]));
7227 7 : rtx tmp3 = ix86_expand_ti_to_v1ti (tmp2);
7228 7 : emit_move_insn (operands[0], tmp3);
7229 7 : return;
7230 : }
7231 :
7232 36 : HOST_WIDE_INT bits = INTVAL (operands[2]) & 127;
7233 :
7234 36 : if (bits == 0)
7235 : {
7236 0 : emit_move_insn (operands[0], op1);
7237 0 : return;
7238 : }
7239 :
7240 36 : if ((bits & 7) == 0)
7241 : {
7242 0 : rtx tmp = gen_reg_rtx (V1TImode);
7243 0 : if (code == ASHIFT)
7244 0 : emit_insn (gen_sse2_ashlv1ti3 (tmp, op1, GEN_INT (bits)));
7245 : else
7246 0 : emit_insn (gen_sse2_lshrv1ti3 (tmp, op1, GEN_INT (bits)));
7247 0 : emit_move_insn (operands[0], tmp);
7248 0 : return;
7249 : }
7250 :
7251 36 : rtx tmp1 = gen_reg_rtx (V1TImode);
7252 36 : if (code == ASHIFT)
7253 18 : emit_insn (gen_sse2_ashlv1ti3 (tmp1, op1, GEN_INT (64)));
7254 : else
7255 18 : emit_insn (gen_sse2_lshrv1ti3 (tmp1, op1, GEN_INT (64)));
7256 :
7257 : /* tmp2 is operands[1] shifted by 64, in V2DImode. */
7258 36 : rtx tmp2 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp1));
7259 :
7260 : /* tmp3 will be the V2DImode result. */
7261 36 : rtx tmp3 = gen_reg_rtx (V2DImode);
7262 :
7263 36 : if (bits > 64)
7264 : {
7265 18 : if (code == ASHIFT)
7266 9 : emit_insn (gen_ashlv2di3 (tmp3, tmp2, GEN_INT (bits - 64)));
7267 : else
7268 9 : emit_insn (gen_lshrv2di3 (tmp3, tmp2, GEN_INT (bits - 64)));
7269 : }
7270 : else
7271 : {
7272 : /* tmp4 is operands[1], in V2DImode. */
7273 18 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
7274 :
7275 18 : rtx tmp5 = gen_reg_rtx (V2DImode);
7276 18 : if (code == ASHIFT)
7277 9 : emit_insn (gen_ashlv2di3 (tmp5, tmp4, GEN_INT (bits)));
7278 : else
7279 9 : emit_insn (gen_lshrv2di3 (tmp5, tmp4, GEN_INT (bits)));
7280 :
7281 18 : rtx tmp6 = gen_reg_rtx (V2DImode);
7282 18 : if (code == ASHIFT)
7283 9 : emit_insn (gen_lshrv2di3 (tmp6, tmp2, GEN_INT (64 - bits)));
7284 : else
7285 9 : emit_insn (gen_ashlv2di3 (tmp6, tmp2, GEN_INT (64 - bits)));
7286 :
7287 18 : emit_insn (gen_iorv2di3 (tmp3, tmp5, tmp6));
7288 : }
7289 :
7290 : /* Convert the result back to V1TImode and store in operands[0]. */
7291 36 : rtx tmp7 = force_reg (V1TImode, gen_lowpart (V1TImode, tmp3));
7292 36 : emit_move_insn (operands[0], tmp7);
7293 : }
7294 :
7295 : /* Expand V1TI mode rotate (of rtx_code CODE) by constant. */
7296 : void
7297 39 : ix86_expand_v1ti_rotate (enum rtx_code code, rtx operands[])
7298 : {
7299 39 : rtx op1 = force_reg (V1TImode, operands[1]);
7300 :
7301 39 : if (!CONST_INT_P (operands[2]))
7302 : {
7303 8 : rtx tmp1 = ix86_expand_v1ti_to_ti (op1);
7304 8 : rtx tmp2 = gen_reg_rtx (TImode);
7305 4 : rtx (*rotate) (rtx, rtx, rtx)
7306 8 : = (code == ROTATE) ? gen_rotlti3 : gen_rotrti3;
7307 8 : emit_insn (rotate (tmp2, tmp1, operands[2]));
7308 8 : rtx tmp3 = ix86_expand_ti_to_v1ti (tmp2);
7309 8 : emit_move_insn (operands[0], tmp3);
7310 8 : return;
7311 : }
7312 :
7313 31 : HOST_WIDE_INT bits = INTVAL (operands[2]) & 127;
7314 :
7315 31 : if (bits == 0)
7316 : {
7317 0 : emit_move_insn (operands[0], op1);
7318 0 : return;
7319 : }
7320 :
7321 31 : if (code == ROTATERT)
7322 16 : bits = 128 - bits;
7323 :
7324 31 : if ((bits & 31) == 0)
7325 : {
7326 5 : rtx tmp2 = gen_reg_rtx (V4SImode);
7327 5 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7328 5 : if (bits == 32)
7329 1 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0x93)));
7330 4 : else if (bits == 64)
7331 2 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0x4e)));
7332 : else
7333 2 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0x39)));
7334 5 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp2));
7335 5 : return;
7336 : }
7337 :
7338 26 : if ((bits & 7) == 0)
7339 : {
7340 6 : rtx tmp1 = gen_reg_rtx (V1TImode);
7341 6 : rtx tmp2 = gen_reg_rtx (V1TImode);
7342 6 : rtx tmp3 = gen_reg_rtx (V1TImode);
7343 :
7344 6 : emit_insn (gen_sse2_ashlv1ti3 (tmp1, op1, GEN_INT (bits)));
7345 6 : emit_insn (gen_sse2_lshrv1ti3 (tmp2, op1, GEN_INT (128 - bits)));
7346 6 : emit_insn (gen_iorv1ti3 (tmp3, tmp1, tmp2));
7347 6 : emit_move_insn (operands[0], tmp3);
7348 6 : return;
7349 : }
7350 :
7351 20 : rtx op1_v4si = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7352 :
7353 20 : rtx lobits;
7354 20 : rtx hibits;
7355 :
7356 20 : switch (bits >> 5)
7357 : {
7358 7 : case 0:
7359 7 : lobits = op1_v4si;
7360 7 : hibits = gen_reg_rtx (V4SImode);
7361 7 : emit_insn (gen_sse2_pshufd (hibits, op1_v4si, GEN_INT (0x93)));
7362 7 : break;
7363 :
7364 2 : case 1:
7365 2 : lobits = gen_reg_rtx (V4SImode);
7366 2 : hibits = gen_reg_rtx (V4SImode);
7367 2 : emit_insn (gen_sse2_pshufd (lobits, op1_v4si, GEN_INT (0x93)));
7368 2 : emit_insn (gen_sse2_pshufd (hibits, op1_v4si, GEN_INT (0x4e)));
7369 2 : break;
7370 :
7371 2 : case 2:
7372 2 : lobits = gen_reg_rtx (V4SImode);
7373 2 : hibits = gen_reg_rtx (V4SImode);
7374 2 : emit_insn (gen_sse2_pshufd (lobits, op1_v4si, GEN_INT (0x4e)));
7375 2 : emit_insn (gen_sse2_pshufd (hibits, op1_v4si, GEN_INT (0x39)));
7376 2 : break;
7377 :
7378 9 : default:
7379 9 : lobits = gen_reg_rtx (V4SImode);
7380 9 : emit_insn (gen_sse2_pshufd (lobits, op1_v4si, GEN_INT (0x39)));
7381 9 : hibits = op1_v4si;
7382 9 : break;
7383 : }
7384 :
7385 20 : rtx tmp1 = gen_reg_rtx (V4SImode);
7386 20 : rtx tmp2 = gen_reg_rtx (V4SImode);
7387 20 : rtx tmp3 = gen_reg_rtx (V4SImode);
7388 :
7389 20 : emit_insn (gen_ashlv4si3 (tmp1, lobits, GEN_INT (bits & 31)));
7390 20 : emit_insn (gen_lshrv4si3 (tmp2, hibits, GEN_INT (32 - (bits & 31))));
7391 20 : emit_insn (gen_iorv4si3 (tmp3, tmp1, tmp2));
7392 :
7393 20 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp3));
7394 : }
7395 :
7396 : /* Expand V1TI mode ashiftrt by constant. */
7397 : void
7398 109 : ix86_expand_v1ti_ashiftrt (rtx operands[])
7399 : {
7400 109 : rtx op1 = force_reg (V1TImode, operands[1]);
7401 :
7402 109 : if (!CONST_INT_P (operands[2]))
7403 : {
7404 3 : rtx tmp1 = ix86_expand_v1ti_to_ti (op1);
7405 3 : rtx tmp2 = gen_reg_rtx (TImode);
7406 3 : emit_insn (gen_ashrti3 (tmp2, tmp1, operands[2]));
7407 3 : rtx tmp3 = ix86_expand_ti_to_v1ti (tmp2);
7408 3 : emit_move_insn (operands[0], tmp3);
7409 3 : return;
7410 : }
7411 :
7412 106 : HOST_WIDE_INT bits = INTVAL (operands[2]) & 127;
7413 :
7414 106 : if (bits == 0)
7415 : {
7416 0 : emit_move_insn (operands[0], op1);
7417 0 : return;
7418 : }
7419 :
7420 106 : if (bits == 127)
7421 : {
7422 : /* Two operations. */
7423 3 : rtx tmp1 = force_reg(V4SImode, gen_lowpart (V4SImode, op1));
7424 3 : rtx tmp2 = gen_reg_rtx (V4SImode);
7425 3 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
7426 :
7427 3 : rtx tmp3 = gen_reg_rtx (V4SImode);
7428 3 : emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
7429 :
7430 3 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp3));
7431 3 : return;
7432 : }
7433 :
7434 103 : if (bits == 64)
7435 : {
7436 : /* Three operations. */
7437 3 : rtx tmp1 = force_reg(V4SImode, gen_lowpart (V4SImode, op1));
7438 3 : rtx tmp2 = gen_reg_rtx (V4SImode);
7439 3 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
7440 :
7441 3 : rtx tmp3 = gen_reg_rtx (V4SImode);
7442 3 : emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
7443 :
7444 3 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp1));
7445 3 : rtx tmp5 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
7446 3 : rtx tmp6 = gen_reg_rtx (V2DImode);
7447 3 : emit_insn (gen_vec_interleave_highv2di (tmp6, tmp4, tmp5));
7448 :
7449 3 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp6));
7450 3 : return;
7451 : }
7452 :
7453 100 : if (bits == 96)
7454 : {
7455 : /* Three operations. */
7456 3 : rtx tmp1 = force_reg(V4SImode, gen_lowpart (V4SImode, op1));
7457 3 : rtx tmp2 = gen_reg_rtx (V4SImode);
7458 3 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (31)));
7459 :
7460 3 : rtx tmp3 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp1));
7461 3 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp2));
7462 3 : rtx tmp5 = gen_reg_rtx (V2DImode);
7463 3 : emit_insn (gen_vec_interleave_highv2di (tmp5, tmp3, tmp4));
7464 :
7465 3 : rtx tmp6 = force_reg(V4SImode, gen_lowpart (V4SImode, tmp5));
7466 3 : rtx tmp7 = gen_reg_rtx (V4SImode);
7467 3 : emit_insn (gen_sse2_pshufd (tmp7, tmp6, GEN_INT (0xfd)));
7468 :
7469 3 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp7));
7470 3 : return;
7471 : }
7472 :
7473 97 : if (bits >= 111)
7474 : {
7475 : /* Three operations. */
7476 21 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7477 21 : rtx tmp2 = gen_reg_rtx (V4SImode);
7478 21 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits - 96)));
7479 :
7480 21 : rtx tmp3 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp2));
7481 21 : rtx tmp4 = gen_reg_rtx (V8HImode);
7482 21 : emit_insn (gen_sse2_pshufhw (tmp4, tmp3, GEN_INT (0xfe)));
7483 :
7484 21 : rtx tmp5 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp4));
7485 21 : rtx tmp6 = gen_reg_rtx (V4SImode);
7486 21 : emit_insn (gen_sse2_pshufd (tmp6, tmp5, GEN_INT (0xfe)));
7487 :
7488 21 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp6));
7489 21 : return;
7490 : }
7491 :
7492 76 : if (TARGET_AVX2 || TARGET_SSE4_1)
7493 : {
7494 : /* Three operations. */
7495 50 : if (bits == 32)
7496 : {
7497 2 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7498 2 : rtx tmp2 = gen_reg_rtx (V4SImode);
7499 2 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (31)));
7500 :
7501 2 : rtx tmp3 = gen_reg_rtx (V1TImode);
7502 2 : emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (32)));
7503 :
7504 2 : if (TARGET_AVX2)
7505 : {
7506 1 : rtx tmp4 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp3));
7507 1 : rtx tmp5 = gen_reg_rtx (V4SImode);
7508 1 : emit_insn (gen_avx2_pblenddv4si (tmp5, tmp2, tmp4,
7509 : GEN_INT (7)));
7510 :
7511 1 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp5));
7512 : }
7513 : else
7514 : {
7515 1 : rtx tmp4 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp2));
7516 1 : rtx tmp5 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp3));
7517 1 : rtx tmp6 = gen_reg_rtx (V8HImode);
7518 1 : emit_insn (gen_sse4_1_pblendw (tmp6, tmp4, tmp5,
7519 : GEN_INT (0x3f)));
7520 :
7521 1 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp6));
7522 : }
7523 : return;
7524 : }
7525 :
7526 : /* Three operations. */
7527 : if (bits == 8 || bits == 16 || bits == 24)
7528 : {
7529 6 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7530 6 : rtx tmp2 = gen_reg_rtx (V4SImode);
7531 6 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits)));
7532 :
7533 6 : rtx tmp3 = gen_reg_rtx (V1TImode);
7534 6 : emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (bits)));
7535 :
7536 6 : if (TARGET_AVX2)
7537 : {
7538 3 : rtx tmp4 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp3));
7539 3 : rtx tmp5 = gen_reg_rtx (V4SImode);
7540 3 : emit_insn (gen_avx2_pblenddv4si (tmp5, tmp2, tmp4,
7541 : GEN_INT (7)));
7542 :
7543 3 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp5));
7544 : }
7545 : else
7546 : {
7547 3 : rtx tmp4 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp2));
7548 3 : rtx tmp5 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp3));
7549 3 : rtx tmp6 = gen_reg_rtx (V8HImode);
7550 3 : emit_insn (gen_sse4_1_pblendw (tmp6, tmp4, tmp5,
7551 : GEN_INT (0x3f)));
7552 :
7553 3 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp6));
7554 : }
7555 : return;
7556 : }
7557 : }
7558 :
7559 68 : if (bits > 96)
7560 : {
7561 : /* Four operations. */
7562 3 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7563 3 : rtx tmp2 = gen_reg_rtx (V4SImode);
7564 3 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits - 96)));
7565 :
7566 3 : rtx tmp3 = gen_reg_rtx (V4SImode);
7567 3 : emit_insn (gen_ashrv4si3 (tmp3, tmp1, GEN_INT (31)));
7568 :
7569 3 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp2));
7570 3 : rtx tmp5 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
7571 3 : rtx tmp6 = gen_reg_rtx (V2DImode);
7572 3 : emit_insn (gen_vec_interleave_highv2di (tmp6, tmp4, tmp5));
7573 :
7574 3 : rtx tmp7 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp6));
7575 3 : rtx tmp8 = gen_reg_rtx (V4SImode);
7576 3 : emit_insn (gen_sse2_pshufd (tmp8, tmp7, GEN_INT (0xfd)));
7577 :
7578 3 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp8));
7579 3 : return;
7580 : }
7581 :
7582 65 : if (TARGET_SSE4_1 && (bits == 48 || bits == 80))
7583 : {
7584 : /* Four operations. */
7585 4 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7586 4 : rtx tmp2 = gen_reg_rtx (V4SImode);
7587 4 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
7588 :
7589 4 : rtx tmp3 = gen_reg_rtx (V4SImode);
7590 4 : emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
7591 :
7592 4 : rtx tmp4 = gen_reg_rtx (V1TImode);
7593 4 : emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (bits)));
7594 :
7595 4 : rtx tmp5 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp3));
7596 4 : rtx tmp6 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp4));
7597 4 : rtx tmp7 = gen_reg_rtx (V8HImode);
7598 6 : emit_insn (gen_sse4_1_pblendw (tmp7, tmp5, tmp6,
7599 : GEN_INT (bits == 48 ? 0x1f : 0x07)));
7600 :
7601 4 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp7));
7602 4 : return;
7603 : }
7604 :
7605 61 : if ((bits & 7) == 0)
7606 : {
7607 : /* Five operations. */
7608 9 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7609 9 : rtx tmp2 = gen_reg_rtx (V4SImode);
7610 9 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
7611 :
7612 9 : rtx tmp3 = gen_reg_rtx (V4SImode);
7613 9 : emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
7614 :
7615 9 : rtx tmp4 = gen_reg_rtx (V1TImode);
7616 9 : emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (bits)));
7617 :
7618 9 : rtx tmp5 = force_reg (V1TImode, gen_lowpart (V1TImode, tmp3));
7619 9 : rtx tmp6 = gen_reg_rtx (V1TImode);
7620 9 : emit_insn (gen_sse2_ashlv1ti3 (tmp6, tmp5, GEN_INT (128 - bits)));
7621 :
7622 9 : rtx tmp7 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp4));
7623 9 : rtx tmp8 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp6));
7624 9 : rtx tmp9 = gen_reg_rtx (V2DImode);
7625 9 : emit_insn (gen_iorv2di3 (tmp9, tmp7, tmp8));
7626 :
7627 9 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp9));
7628 9 : return;
7629 : }
7630 :
7631 52 : if (TARGET_AVX2 && bits < 32)
7632 : {
7633 : /* Six operations. */
7634 9 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7635 9 : rtx tmp2 = gen_reg_rtx (V4SImode);
7636 9 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits)));
7637 :
7638 9 : rtx tmp3 = gen_reg_rtx (V1TImode);
7639 9 : emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (64)));
7640 :
7641 9 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
7642 9 : rtx tmp5 = gen_reg_rtx (V2DImode);
7643 9 : emit_insn (gen_lshrv2di3 (tmp5, tmp4, GEN_INT (bits)));
7644 :
7645 9 : rtx tmp6 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
7646 9 : rtx tmp7 = gen_reg_rtx (V2DImode);
7647 9 : emit_insn (gen_ashlv2di3 (tmp7, tmp6, GEN_INT (64 - bits)));
7648 :
7649 9 : rtx tmp8 = gen_reg_rtx (V2DImode);
7650 9 : emit_insn (gen_iorv2di3 (tmp8, tmp5, tmp7));
7651 :
7652 9 : rtx tmp9 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp8));
7653 9 : rtx tmp10 = gen_reg_rtx (V4SImode);
7654 9 : emit_insn (gen_avx2_pblenddv4si (tmp10, tmp2, tmp9, GEN_INT (7)));
7655 :
7656 9 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp10));
7657 9 : return;
7658 : }
7659 :
7660 43 : if (TARGET_SSE4_1 && bits < 15)
7661 : {
7662 : /* Six operations. */
7663 4 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7664 4 : rtx tmp2 = gen_reg_rtx (V4SImode);
7665 4 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits)));
7666 :
7667 4 : rtx tmp3 = gen_reg_rtx (V1TImode);
7668 4 : emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (64)));
7669 :
7670 4 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
7671 4 : rtx tmp5 = gen_reg_rtx (V2DImode);
7672 4 : emit_insn (gen_lshrv2di3 (tmp5, tmp4, GEN_INT (bits)));
7673 :
7674 4 : rtx tmp6 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
7675 4 : rtx tmp7 = gen_reg_rtx (V2DImode);
7676 4 : emit_insn (gen_ashlv2di3 (tmp7, tmp6, GEN_INT (64 - bits)));
7677 :
7678 4 : rtx tmp8 = gen_reg_rtx (V2DImode);
7679 4 : emit_insn (gen_iorv2di3 (tmp8, tmp5, tmp7));
7680 :
7681 4 : rtx tmp9 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp2));
7682 4 : rtx tmp10 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp8));
7683 4 : rtx tmp11 = gen_reg_rtx (V8HImode);
7684 4 : emit_insn (gen_sse4_1_pblendw (tmp11, tmp9, tmp10, GEN_INT (0x3f)));
7685 :
7686 4 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp11));
7687 4 : return;
7688 : }
7689 :
7690 18 : if (bits == 1)
7691 : {
7692 : /* Eight operations. */
7693 1 : rtx tmp1 = gen_reg_rtx (V1TImode);
7694 1 : emit_insn (gen_sse2_lshrv1ti3 (tmp1, op1, GEN_INT (64)));
7695 :
7696 1 : rtx tmp2 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
7697 1 : rtx tmp3 = gen_reg_rtx (V2DImode);
7698 1 : emit_insn (gen_lshrv2di3 (tmp3, tmp2, GEN_INT (1)));
7699 :
7700 1 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp1));
7701 1 : rtx tmp5 = gen_reg_rtx (V2DImode);
7702 1 : emit_insn (gen_ashlv2di3 (tmp5, tmp4, GEN_INT (63)));
7703 :
7704 1 : rtx tmp6 = gen_reg_rtx (V2DImode);
7705 1 : emit_insn (gen_iorv2di3 (tmp6, tmp3, tmp5));
7706 :
7707 1 : rtx tmp7 = gen_reg_rtx (V2DImode);
7708 1 : emit_insn (gen_lshrv2di3 (tmp7, tmp2, GEN_INT (63)));
7709 :
7710 1 : rtx tmp8 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp7));
7711 1 : rtx tmp9 = gen_reg_rtx (V4SImode);
7712 1 : emit_insn (gen_sse2_pshufd (tmp9, tmp8, GEN_INT (0xbf)));
7713 :
7714 1 : rtx tmp10 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp9));
7715 1 : rtx tmp11 = gen_reg_rtx (V2DImode);
7716 1 : emit_insn (gen_ashlv2di3 (tmp11, tmp10, GEN_INT (31)));
7717 :
7718 1 : rtx tmp12 = gen_reg_rtx (V2DImode);
7719 1 : emit_insn (gen_iorv2di3 (tmp12, tmp6, tmp11));
7720 :
7721 1 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp12));
7722 1 : return;
7723 : }
7724 :
7725 38 : if (bits > 64)
7726 : {
7727 : /* Eight operations. */
7728 12 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7729 12 : rtx tmp2 = gen_reg_rtx (V4SImode);
7730 12 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
7731 :
7732 12 : rtx tmp3 = gen_reg_rtx (V4SImode);
7733 12 : emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
7734 :
7735 12 : rtx tmp4 = gen_reg_rtx (V1TImode);
7736 12 : emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (64)));
7737 :
7738 12 : rtx tmp5 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp4));
7739 12 : rtx tmp6 = gen_reg_rtx (V2DImode);
7740 12 : emit_insn (gen_lshrv2di3 (tmp6, tmp5, GEN_INT (bits - 64)));
7741 :
7742 12 : rtx tmp7 = force_reg (V1TImode, gen_lowpart (V1TImode, tmp3));
7743 12 : rtx tmp8 = gen_reg_rtx (V1TImode);
7744 12 : emit_insn (gen_sse2_ashlv1ti3 (tmp8, tmp7, GEN_INT (64)));
7745 :
7746 12 : rtx tmp9 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
7747 12 : rtx tmp10 = gen_reg_rtx (V2DImode);
7748 12 : emit_insn (gen_ashlv2di3 (tmp10, tmp9, GEN_INT (128 - bits)));
7749 :
7750 12 : rtx tmp11 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp8));
7751 12 : rtx tmp12 = gen_reg_rtx (V2DImode);
7752 12 : emit_insn (gen_iorv2di3 (tmp12, tmp10, tmp11));
7753 :
7754 12 : rtx tmp13 = gen_reg_rtx (V2DImode);
7755 12 : emit_insn (gen_iorv2di3 (tmp13, tmp6, tmp12));
7756 :
7757 12 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp13));
7758 : }
7759 : else
7760 : {
7761 : /* Nine operations. */
7762 26 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7763 26 : rtx tmp2 = gen_reg_rtx (V4SImode);
7764 26 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
7765 :
7766 26 : rtx tmp3 = gen_reg_rtx (V4SImode);
7767 26 : emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
7768 :
7769 26 : rtx tmp4 = gen_reg_rtx (V1TImode);
7770 26 : emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (64)));
7771 :
7772 26 : rtx tmp5 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
7773 26 : rtx tmp6 = gen_reg_rtx (V2DImode);
7774 26 : emit_insn (gen_lshrv2di3 (tmp6, tmp5, GEN_INT (bits)));
7775 :
7776 26 : rtx tmp7 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp4));
7777 26 : rtx tmp8 = gen_reg_rtx (V2DImode);
7778 26 : emit_insn (gen_ashlv2di3 (tmp8, tmp7, GEN_INT (64 - bits)));
7779 :
7780 26 : rtx tmp9 = gen_reg_rtx (V2DImode);
7781 26 : emit_insn (gen_iorv2di3 (tmp9, tmp6, tmp8));
7782 :
7783 26 : rtx tmp10 = force_reg (V1TImode, gen_lowpart (V1TImode, tmp3));
7784 26 : rtx tmp11 = gen_reg_rtx (V1TImode);
7785 26 : emit_insn (gen_sse2_ashlv1ti3 (tmp11, tmp10, GEN_INT (64)));
7786 :
7787 26 : rtx tmp12 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp11));
7788 26 : rtx tmp13 = gen_reg_rtx (V2DImode);
7789 26 : emit_insn (gen_ashlv2di3 (tmp13, tmp12, GEN_INT (64 - bits)));
7790 :
7791 26 : rtx tmp14 = gen_reg_rtx (V2DImode);
7792 26 : emit_insn (gen_iorv2di3 (tmp14, tmp9, tmp13));
7793 :
7794 26 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp14));
7795 : }
7796 : }
7797 :
7798 : /* Expand V2DI mode ashiftrt. */
7799 : void
7800 519 : ix86_expand_v2di_ashiftrt (rtx operands[])
7801 : {
7802 519 : if (operands[2] == const0_rtx)
7803 : {
7804 0 : emit_move_insn (operands[0], operands[1]);
7805 0 : return;
7806 : }
7807 :
7808 519 : if (TARGET_SSE4_2
7809 144 : && CONST_INT_P (operands[2])
7810 144 : && UINTVAL (operands[2]) >= 63
7811 527 : && !optimize_insn_for_size_p ())
7812 : {
7813 8 : rtx zero = force_reg (V2DImode, CONST0_RTX (V2DImode));
7814 8 : emit_insn (gen_sse4_2_gtv2di3 (operands[0], zero, operands[1]));
7815 8 : return;
7816 : }
7817 :
7818 511 : if (CONST_INT_P (operands[2])
7819 485 : && (!TARGET_XOP || UINTVAL (operands[2]) >= 63))
7820 : {
7821 389 : vec_perm_builder sel (4, 4, 1);
7822 389 : sel.quick_grow (4);
7823 389 : rtx arg0, arg1;
7824 389 : rtx op1 = lowpart_subreg (V4SImode,
7825 : force_reg (V2DImode, operands[1]),
7826 : V2DImode);
7827 389 : rtx target = gen_reg_rtx (V4SImode);
7828 389 : if (UINTVAL (operands[2]) >= 63)
7829 : {
7830 132 : arg0 = arg1 = gen_reg_rtx (V4SImode);
7831 132 : emit_insn (gen_ashrv4si3 (arg0, op1, GEN_INT (31)));
7832 132 : sel[0] = 1;
7833 132 : sel[1] = 1;
7834 132 : sel[2] = 3;
7835 132 : sel[3] = 3;
7836 : }
7837 257 : else if (INTVAL (operands[2]) > 32)
7838 : {
7839 30 : arg0 = gen_reg_rtx (V4SImode);
7840 30 : arg1 = gen_reg_rtx (V4SImode);
7841 30 : emit_insn (gen_ashrv4si3 (arg1, op1, GEN_INT (31)));
7842 30 : emit_insn (gen_ashrv4si3 (arg0, op1,
7843 30 : GEN_INT (INTVAL (operands[2]) - 32)));
7844 30 : sel[0] = 1;
7845 30 : sel[1] = 5;
7846 30 : sel[2] = 3;
7847 30 : sel[3] = 7;
7848 : }
7849 227 : else if (INTVAL (operands[2]) == 32)
7850 : {
7851 5 : arg0 = op1;
7852 5 : arg1 = gen_reg_rtx (V4SImode);
7853 5 : emit_insn (gen_ashrv4si3 (arg1, op1, GEN_INT (31)));
7854 5 : sel[0] = 1;
7855 5 : sel[1] = 5;
7856 5 : sel[2] = 3;
7857 5 : sel[3] = 7;
7858 : }
7859 : else
7860 : {
7861 222 : arg0 = gen_reg_rtx (V2DImode);
7862 222 : arg1 = gen_reg_rtx (V4SImode);
7863 222 : emit_insn (gen_lshrv2di3 (arg0, operands[1], operands[2]));
7864 222 : emit_insn (gen_ashrv4si3 (arg1, op1, operands[2]));
7865 222 : arg0 = lowpart_subreg (V4SImode, arg0, V2DImode);
7866 222 : sel[0] = 0;
7867 222 : sel[1] = 5;
7868 222 : sel[2] = 2;
7869 222 : sel[3] = 7;
7870 : }
7871 521 : vec_perm_indices indices (sel, arg0 != arg1 ? 2 : 1, 4);
7872 389 : rtx op0 = operands[0];
7873 389 : bool ok = targetm.vectorize.vec_perm_const (V4SImode, V4SImode,
7874 : target, arg0, arg1,
7875 : indices);
7876 389 : gcc_assert (ok);
7877 389 : emit_move_insn (op0, lowpart_subreg (V2DImode, target, V4SImode));
7878 389 : return;
7879 389 : }
7880 122 : if (!TARGET_XOP)
7881 : {
7882 26 : rtx zero = force_reg (V2DImode, CONST0_RTX (V2DImode));
7883 26 : rtx zero_or_all_ones;
7884 26 : if (TARGET_SSE4_2)
7885 : {
7886 0 : zero_or_all_ones = gen_reg_rtx (V2DImode);
7887 0 : emit_insn (gen_sse4_2_gtv2di3 (zero_or_all_ones, zero,
7888 : operands[1]));
7889 : }
7890 : else
7891 : {
7892 26 : rtx temp = gen_reg_rtx (V4SImode);
7893 26 : emit_insn (gen_ashrv4si3 (temp,
7894 : lowpart_subreg (V4SImode,
7895 : force_reg (V2DImode,
7896 : operands[1]),
7897 : V2DImode),
7898 : GEN_INT (31)));
7899 26 : zero_or_all_ones = gen_reg_rtx (V4SImode);
7900 26 : emit_insn (gen_sse2_pshufd_1 (zero_or_all_ones, temp,
7901 : const1_rtx, const1_rtx,
7902 : GEN_INT (3), GEN_INT (3)));
7903 26 : zero_or_all_ones = lowpart_subreg (V2DImode, zero_or_all_ones,
7904 : V4SImode);
7905 : }
7906 26 : rtx lshr_res = gen_reg_rtx (V2DImode);
7907 26 : emit_insn (gen_lshrv2di3 (lshr_res, operands[1], operands[2]));
7908 26 : rtx ashl_res = gen_reg_rtx (V2DImode);
7909 26 : rtx amount;
7910 26 : if (TARGET_64BIT)
7911 : {
7912 26 : amount = gen_reg_rtx (DImode);
7913 26 : emit_insn (gen_subdi3 (amount, force_reg (DImode, GEN_INT (64)),
7914 : operands[2]));
7915 : }
7916 : else
7917 : {
7918 0 : rtx temp = gen_reg_rtx (SImode);
7919 0 : emit_insn (gen_subsi3 (temp, force_reg (SImode, GEN_INT (64)),
7920 : lowpart_subreg (SImode, operands[2],
7921 : DImode)));
7922 0 : amount = gen_reg_rtx (V4SImode);
7923 0 : emit_insn (gen_vec_setv4si_0 (amount, CONST0_RTX (V4SImode),
7924 : temp));
7925 : }
7926 26 : amount = lowpart_subreg (DImode, amount, GET_MODE (amount));
7927 26 : emit_insn (gen_ashlv2di3 (ashl_res, zero_or_all_ones, amount));
7928 26 : emit_insn (gen_iorv2di3 (operands[0], lshr_res, ashl_res));
7929 26 : return;
7930 : }
7931 :
7932 96 : rtx reg = gen_reg_rtx (V2DImode);
7933 96 : rtx par;
7934 96 : bool negate = false;
7935 96 : int i;
7936 :
7937 96 : if (CONST_INT_P (operands[2]))
7938 96 : operands[2] = GEN_INT (-INTVAL (operands[2]));
7939 : else
7940 : negate = true;
7941 :
7942 96 : par = gen_rtx_PARALLEL (V2DImode, rtvec_alloc (2));
7943 288 : for (i = 0; i < 2; i++)
7944 192 : XVECEXP (par, 0, i) = operands[2];
7945 :
7946 96 : emit_insn (gen_vec_initv2didi (reg, par));
7947 :
7948 96 : if (negate)
7949 0 : emit_insn (gen_negv2di2 (reg, reg));
7950 :
7951 96 : emit_insn (gen_xop_shav2di3 (operands[0], operands[1], reg));
7952 : }
7953 :
7954 : /* Replace all occurrences of REG FROM with REG TO in X, including
7955 : occurrences with different modes. */
7956 :
7957 : rtx
7958 43368 : ix86_replace_reg_with_reg (rtx x, rtx from, rtx to)
7959 : {
7960 43368 : gcc_checking_assert (REG_P (from)
7961 : && REG_P (to)
7962 : && GET_MODE (from) == GET_MODE (to));
7963 43368 : if (!reg_overlap_mentioned_p (from, x))
7964 : return x;
7965 144 : rtx ret = copy_rtx (x);
7966 144 : subrtx_ptr_iterator::array_type array;
7967 732 : FOR_EACH_SUBRTX_PTR (iter, array, &ret, NONCONST)
7968 : {
7969 588 : rtx *loc = *iter;
7970 588 : x = *loc;
7971 588 : if (REG_P (x) && REGNO (x) == REGNO (from))
7972 : {
7973 144 : if (x == from)
7974 144 : *loc = to;
7975 : else
7976 : {
7977 0 : gcc_checking_assert (REG_NREGS (x) == 1);
7978 0 : *loc = gen_rtx_REG (GET_MODE (x), REGNO (to));
7979 : }
7980 : }
7981 : }
7982 144 : return ret;
7983 144 : }
7984 :
7985 : /* Return mode for the memcpy/memset loop counter. Prefer SImode over
7986 : DImode for constant loop counts. */
7987 :
7988 : static machine_mode
7989 45193 : counter_mode (rtx count_exp)
7990 : {
7991 709 : if (GET_MODE (count_exp) != VOIDmode)
7992 32908 : return GET_MODE (count_exp);
7993 12285 : if (!CONST_INT_P (count_exp))
7994 0 : return Pmode;
7995 : if (TARGET_64BIT && (INTVAL (count_exp) & ~0xffffffff))
7996 : return DImode;
7997 : return SImode;
7998 : }
7999 :
8000 : /* When ISSETMEM is FALSE, output simple loop to move memory pointer to SRCPTR
8001 : to DESTPTR via chunks of MODE unrolled UNROLL times, overall size is COUNT
8002 : specified in bytes. When ISSETMEM is TRUE, output the equivalent loop to set
8003 : memory by VALUE (supposed to be in MODE).
8004 :
8005 : The size is rounded down to whole number of chunk size moved at once.
8006 : SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info. */
8007 :
8008 :
8009 : static void
8010 8218 : expand_set_or_cpymem_via_loop (rtx destmem, rtx srcmem,
8011 : rtx destptr, rtx srcptr, rtx value,
8012 : rtx count, machine_mode mode, int unroll,
8013 : int expected_size, bool issetmem)
8014 : {
8015 8218 : rtx_code_label *out_label = nullptr;
8016 8218 : rtx_code_label *top_label = nullptr;
8017 8218 : rtx iter, tmp;
8018 8218 : machine_mode iter_mode = counter_mode (count);
8019 8218 : int piece_size_n = GET_MODE_SIZE (mode) * unroll;
8020 8218 : rtx piece_size = GEN_INT (piece_size_n);
8021 16436 : rtx piece_size_mask = GEN_INT (~((GET_MODE_SIZE (mode) * unroll) - 1));
8022 8218 : rtx size;
8023 8218 : int i;
8024 8218 : int loop_count;
8025 :
8026 8218 : if (expected_size != -1 && CONST_INT_P (count))
8027 7234 : loop_count = INTVAL (count) / GET_MODE_SIZE (mode) / unroll;
8028 : else
8029 : loop_count = -1;
8030 :
8031 : /* Don't generate the loop if the loop count is 1. */
8032 7234 : if (loop_count != 1)
8033 : {
8034 8167 : top_label = gen_label_rtx ();
8035 8167 : out_label = gen_label_rtx ();
8036 : }
8037 8218 : iter = gen_reg_rtx (iter_mode);
8038 :
8039 8218 : size = expand_simple_binop (iter_mode, AND, count, piece_size_mask,
8040 : NULL, 1, OPTAB_DIRECT);
8041 : /* Those two should combine. */
8042 8218 : if (piece_size == const1_rtx)
8043 : {
8044 275 : emit_cmp_and_jump_insns (size, const0_rtx, EQ, NULL_RTX, iter_mode,
8045 : true, out_label);
8046 275 : predict_jump (REG_BR_PROB_BASE * 10 / 100);
8047 : }
8048 8218 : emit_move_insn (iter, const0_rtx);
8049 :
8050 8218 : if (loop_count != 1)
8051 8167 : emit_label (top_label);
8052 :
8053 10940 : tmp = convert_modes (Pmode, iter_mode, iter, true);
8054 :
8055 : /* This assert could be relaxed - in this case we'll need to compute
8056 : smallest power of two, containing in PIECE_SIZE_N and pass it to
8057 : offset_address. */
8058 8218 : gcc_assert ((piece_size_n & (piece_size_n - 1)) == 0);
8059 8218 : destmem = offset_address (destmem, tmp, piece_size_n);
8060 8218 : destmem = adjust_address (destmem, mode, 0);
8061 :
8062 8218 : if (!issetmem)
8063 : {
8064 2886 : srcmem = offset_address (srcmem, copy_rtx (tmp), piece_size_n);
8065 2886 : srcmem = adjust_address (srcmem, mode, 0);
8066 :
8067 : /* When unrolling for chips that reorder memory reads and writes,
8068 : we can save registers by using single temporary.
8069 : Also using 4 temporaries is overkill in 32bit mode. */
8070 2886 : if (!TARGET_64BIT && 0)
8071 : {
8072 : for (i = 0; i < unroll; i++)
8073 : {
8074 : if (i)
8075 : {
8076 : destmem = adjust_address (copy_rtx (destmem), mode,
8077 : GET_MODE_SIZE (mode));
8078 : srcmem = adjust_address (copy_rtx (srcmem), mode,
8079 : GET_MODE_SIZE (mode));
8080 : }
8081 : emit_move_insn (destmem, srcmem);
8082 : }
8083 : }
8084 : else
8085 : {
8086 2886 : rtx tmpreg[4];
8087 2886 : gcc_assert (unroll <= 4);
8088 13628 : for (i = 0; i < unroll; i++)
8089 : {
8090 10742 : tmpreg[i] = gen_reg_rtx (mode);
8091 10742 : if (i)
8092 15712 : srcmem = adjust_address (copy_rtx (srcmem), mode,
8093 : GET_MODE_SIZE (mode));
8094 10742 : emit_move_insn (tmpreg[i], srcmem);
8095 : }
8096 13628 : for (i = 0; i < unroll; i++)
8097 : {
8098 10742 : if (i)
8099 15712 : destmem = adjust_address (copy_rtx (destmem), mode,
8100 : GET_MODE_SIZE (mode));
8101 10742 : emit_move_insn (destmem, tmpreg[i]);
8102 : }
8103 : }
8104 : }
8105 : else
8106 26501 : for (i = 0; i < unroll; i++)
8107 : {
8108 21169 : if (i)
8109 31674 : destmem = adjust_address (copy_rtx (destmem), mode,
8110 : GET_MODE_SIZE (mode));
8111 21169 : emit_move_insn (destmem, value);
8112 : }
8113 :
8114 8218 : tmp = expand_simple_binop (iter_mode, PLUS, iter, piece_size, iter,
8115 : true, OPTAB_LIB_WIDEN);
8116 8218 : if (tmp != iter)
8117 0 : emit_move_insn (iter, tmp);
8118 :
8119 8218 : if (loop_count != 1)
8120 : {
8121 8167 : emit_cmp_and_jump_insns (iter, size, LT, NULL_RTX, iter_mode,
8122 : true, top_label);
8123 8167 : if (expected_size != -1)
8124 : {
8125 7291 : expected_size /= GET_MODE_SIZE (mode) * unroll;
8126 7291 : if (expected_size == 0)
8127 0 : predict_jump (0);
8128 7291 : else if (expected_size > REG_BR_PROB_BASE)
8129 2 : predict_jump (REG_BR_PROB_BASE - 1);
8130 : else
8131 7289 : predict_jump (REG_BR_PROB_BASE
8132 7289 : - (REG_BR_PROB_BASE + expected_size / 2)
8133 7289 : / expected_size);
8134 : }
8135 : else
8136 876 : predict_jump (REG_BR_PROB_BASE * 80 / 100);
8137 : }
8138 8218 : iter = ix86_zero_extend_to_Pmode (iter);
8139 10940 : tmp = expand_simple_binop (Pmode, PLUS, destptr, iter, destptr,
8140 : true, OPTAB_LIB_WIDEN);
8141 8218 : if (tmp != destptr)
8142 0 : emit_move_insn (destptr, tmp);
8143 8218 : if (!issetmem)
8144 : {
8145 4208 : tmp = expand_simple_binop (Pmode, PLUS, srcptr, iter, srcptr,
8146 : true, OPTAB_LIB_WIDEN);
8147 2886 : if (tmp != srcptr)
8148 0 : emit_move_insn (srcptr, tmp);
8149 : }
8150 8218 : if (loop_count != 1)
8151 8167 : emit_label (out_label);
8152 8218 : }
8153 :
8154 : /* Divide COUNTREG by SCALE. */
8155 : static rtx
8156 19366 : scale_counter (rtx countreg, int scale)
8157 : {
8158 19366 : rtx sc;
8159 :
8160 19366 : if (scale == 1)
8161 : return countreg;
8162 13351 : if (CONST_INT_P (countreg))
8163 13333 : return GEN_INT (INTVAL (countreg) / scale);
8164 18 : gcc_assert (REG_P (countreg));
8165 :
8166 54 : sc = expand_simple_binop (GET_MODE (countreg), LSHIFTRT, countreg,
8167 36 : GEN_INT (exact_log2 (scale)),
8168 : NULL, 1, OPTAB_DIRECT);
8169 18 : return sc;
8170 : }
8171 :
8172 : /* Output "rep; mov" or "rep; stos" instruction depending on ISSETMEM argument.
8173 : When ISSETMEM is true, arguments SRCMEM and SRCPTR are ignored.
8174 : When ISSETMEM is false, arguments VALUE and ORIG_VALUE are ignored.
8175 : For setmem case, VALUE is a promoted to a wider size ORIG_VALUE.
8176 : ORIG_VALUE is the original value passed to memset to fill the memory with.
8177 : Other arguments have same meaning as for previous function. */
8178 :
8179 : static void
8180 19366 : expand_set_or_cpymem_via_rep (rtx destmem, rtx srcmem,
8181 : rtx destptr, rtx srcptr, rtx value, rtx orig_value,
8182 : rtx count,
8183 : machine_mode mode, bool issetmem)
8184 : {
8185 19366 : rtx destexp;
8186 19366 : rtx srcexp;
8187 19366 : rtx countreg;
8188 19366 : HOST_WIDE_INT rounded_count;
8189 :
8190 : /* If possible, it is shorter to use rep movs.
8191 : TODO: Maybe it is better to move this logic to decide_alg. */
8192 19366 : if (mode == QImode && CONST_INT_P (count) && !(INTVAL (count) & 3)
8193 276 : && !TARGET_PREFER_KNOWN_REP_MOVSB_STOSB
8194 272 : && (!issetmem || orig_value == const0_rtx))
8195 19366 : mode = SImode;
8196 :
8197 19366 : if (destptr != XEXP (destmem, 0) || GET_MODE (destmem) != BLKmode)
8198 19087 : destmem = adjust_automodify_address_nv (destmem, BLKmode, destptr, 0);
8199 :
8200 38732 : countreg = ix86_zero_extend_to_Pmode (scale_counter (count,
8201 19366 : GET_MODE_SIZE (mode)));
8202 19366 : if (mode != QImode)
8203 : {
8204 40340 : destexp = gen_rtx_ASHIFT (Pmode, countreg,
8205 : GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
8206 13638 : destexp = gen_rtx_PLUS (Pmode, destexp, destptr);
8207 : }
8208 : else
8209 6033 : destexp = gen_rtx_PLUS (Pmode, destptr, countreg);
8210 19366 : if ((!issetmem || orig_value == const0_rtx) && CONST_INT_P (count))
8211 : {
8212 14254 : rounded_count
8213 14254 : = ROUND_DOWN (INTVAL (count), (HOST_WIDE_INT) GET_MODE_SIZE (mode));
8214 14254 : destmem = shallow_copy_rtx (destmem);
8215 14254 : set_mem_size (destmem, rounded_count);
8216 : }
8217 5120 : else if (MEM_SIZE_KNOWN_P (destmem))
8218 386 : clear_mem_size (destmem);
8219 :
8220 19366 : if (issetmem)
8221 : {
8222 8230 : value = force_reg (mode, gen_lowpart (mode, value));
8223 8230 : emit_insn (gen_rep_stos (destptr, countreg, destmem, value, destexp));
8224 : }
8225 : else
8226 : {
8227 11136 : if (srcptr != XEXP (srcmem, 0) || GET_MODE (srcmem) != BLKmode)
8228 10919 : srcmem = adjust_automodify_address_nv (srcmem, BLKmode, srcptr, 0);
8229 11136 : if (mode != QImode)
8230 : {
8231 20391 : srcexp = gen_rtx_ASHIFT (Pmode, countreg,
8232 : GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
8233 6937 : srcexp = gen_rtx_PLUS (Pmode, srcexp, srcptr);
8234 : }
8235 : else
8236 4423 : srcexp = gen_rtx_PLUS (Pmode, srcptr, countreg);
8237 11136 : if (CONST_INT_P (count))
8238 : {
8239 7230 : rounded_count
8240 7230 : = ROUND_DOWN (INTVAL (count), (HOST_WIDE_INT) GET_MODE_SIZE (mode));
8241 7230 : srcmem = shallow_copy_rtx (srcmem);
8242 7230 : set_mem_size (srcmem, rounded_count);
8243 : }
8244 : else
8245 : {
8246 3921 : if (MEM_SIZE_KNOWN_P (srcmem))
8247 0 : clear_mem_size (srcmem);
8248 : }
8249 11136 : emit_insn (gen_rep_mov (destptr, destmem, srcptr, srcmem, countreg,
8250 : destexp, srcexp));
8251 : }
8252 19366 : }
8253 :
8254 : /* This function emits moves to copy SIZE_TO_MOVE bytes from SRCMEM to
8255 : DESTMEM.
8256 : SRC is passed by pointer to be updated on return.
8257 : Return value is updated DST. */
8258 : static rtx
8259 13 : emit_memmov (rtx destmem, rtx *srcmem, rtx destptr, rtx srcptr,
8260 : HOST_WIDE_INT size_to_move)
8261 : {
8262 13 : rtx dst = destmem, src = *srcmem, tempreg;
8263 13 : enum insn_code code;
8264 13 : machine_mode move_mode;
8265 13 : int piece_size, i;
8266 :
8267 : /* Find the widest mode in which we could perform moves.
8268 : Start with the biggest power of 2 less than SIZE_TO_MOVE and half
8269 : it until move of such size is supported. */
8270 13 : piece_size = 1 << floor_log2 (size_to_move);
8271 26 : while (!int_mode_for_size (piece_size * BITS_PER_UNIT, 0).exists (&move_mode)
8272 26 : || (code = optab_handler (mov_optab, move_mode)) == CODE_FOR_nothing)
8273 : {
8274 0 : gcc_assert (piece_size > 1);
8275 0 : piece_size >>= 1;
8276 : }
8277 :
8278 : /* Find the corresponding vector mode with the same size as MOVE_MODE.
8279 : MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.). */
8280 26 : if (GET_MODE_SIZE (move_mode) > UNITS_PER_WORD)
8281 : {
8282 0 : int nunits = GET_MODE_SIZE (move_mode) / UNITS_PER_WORD;
8283 0 : if (!mode_for_vector (word_mode, nunits).exists (&move_mode)
8284 0 : || (code = optab_handler (mov_optab, move_mode)) == CODE_FOR_nothing)
8285 : {
8286 0 : move_mode = word_mode;
8287 0 : piece_size = GET_MODE_SIZE (move_mode);
8288 0 : code = optab_handler (mov_optab, move_mode);
8289 : }
8290 : }
8291 13 : gcc_assert (code != CODE_FOR_nothing);
8292 :
8293 13 : dst = adjust_automodify_address_nv (dst, move_mode, destptr, 0);
8294 13 : src = adjust_automodify_address_nv (src, move_mode, srcptr, 0);
8295 :
8296 : /* Emit moves. We'll need SIZE_TO_MOVE/PIECE_SIZES moves. */
8297 13 : gcc_assert (size_to_move % piece_size == 0);
8298 :
8299 26 : for (i = 0; i < size_to_move; i += piece_size)
8300 : {
8301 : /* We move from memory to memory, so we'll need to do it via
8302 : a temporary register. */
8303 13 : tempreg = gen_reg_rtx (move_mode);
8304 13 : emit_insn (GEN_FCN (code) (tempreg, src));
8305 13 : emit_insn (GEN_FCN (code) (dst, tempreg));
8306 :
8307 26 : emit_move_insn (destptr,
8308 13 : plus_constant (Pmode, copy_rtx (destptr), piece_size));
8309 26 : emit_move_insn (srcptr,
8310 13 : plus_constant (Pmode, copy_rtx (srcptr), piece_size));
8311 :
8312 13 : dst = adjust_automodify_address_nv (dst, move_mode, destptr,
8313 : piece_size);
8314 13 : src = adjust_automodify_address_nv (src, move_mode, srcptr,
8315 : piece_size);
8316 : }
8317 :
8318 : /* Update DST and SRC rtx. */
8319 13 : *srcmem = src;
8320 13 : return dst;
8321 : }
8322 :
8323 : /* Helper function for the string operations below. Dest VARIABLE whether
8324 : it is aligned to VALUE bytes. If true, jump to the label. */
8325 :
8326 : static rtx_code_label *
8327 3254 : ix86_expand_aligntest (rtx variable, int value, bool epilogue)
8328 : {
8329 3254 : rtx_code_label *label = gen_label_rtx ();
8330 3254 : rtx tmpcount = gen_reg_rtx (GET_MODE (variable));
8331 3254 : if (GET_MODE (variable) == DImode)
8332 684 : emit_insn (gen_anddi3 (tmpcount, variable, GEN_INT (value)));
8333 : else
8334 2570 : emit_insn (gen_andsi3 (tmpcount, variable, GEN_INT (value)));
8335 3254 : emit_cmp_and_jump_insns (tmpcount, const0_rtx, EQ, 0, GET_MODE (variable),
8336 : 1, label);
8337 3254 : if (epilogue)
8338 0 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
8339 : else
8340 3254 : predict_jump (REG_BR_PROB_BASE * 90 / 100);
8341 3254 : return label;
8342 : }
8343 :
8344 :
8345 : /* Output code to copy at most count & (max_size - 1) bytes from SRC to DEST. */
8346 :
8347 : static void
8348 7289 : expand_cpymem_epilogue (rtx destmem, rtx srcmem,
8349 : rtx destptr, rtx srcptr, rtx count, int max_size)
8350 : {
8351 7289 : rtx src, dest;
8352 7289 : if (CONST_INT_P (count))
8353 : {
8354 7224 : unsigned HOST_WIDE_INT countval = UINTVAL (count);
8355 7224 : unsigned HOST_WIDE_INT epilogue_size = countval % max_size;
8356 7224 : unsigned int destalign = MEM_ALIGN (destmem);
8357 7224 : cfun->machine->by_pieces_in_use = true;
8358 7224 : move_by_pieces (destmem, srcmem, epilogue_size, destalign,
8359 : RETURN_BEGIN);
8360 7224 : cfun->machine->by_pieces_in_use = false;
8361 7224 : return;
8362 : }
8363 65 : if (max_size > 8)
8364 : {
8365 65 : count = expand_simple_binop (GET_MODE (count), AND, count, GEN_INT (max_size - 1),
8366 : count, 1, OPTAB_DIRECT);
8367 65 : expand_set_or_cpymem_via_loop (destmem, srcmem, destptr, srcptr, NULL,
8368 : count, QImode, 1, 4, false);
8369 65 : return;
8370 : }
8371 :
8372 : /* When there are stringops, we can cheaply increase dest and src pointers.
8373 : Otherwise we save code size by maintaining offset (zero is readily
8374 : available from preceding rep operation) and using x86 addressing modes.
8375 : */
8376 0 : if (TARGET_SINGLE_STRINGOP)
8377 : {
8378 0 : if (max_size > 4)
8379 : {
8380 0 : rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
8381 0 : src = change_address (srcmem, SImode, srcptr);
8382 0 : dest = change_address (destmem, SImode, destptr);
8383 0 : emit_insn (gen_strmov (destptr, dest, srcptr, src));
8384 0 : emit_label (label);
8385 0 : LABEL_NUSES (label) = 1;
8386 : }
8387 0 : if (max_size > 2)
8388 : {
8389 0 : rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
8390 0 : src = change_address (srcmem, HImode, srcptr);
8391 0 : dest = change_address (destmem, HImode, destptr);
8392 0 : emit_insn (gen_strmov (destptr, dest, srcptr, src));
8393 0 : emit_label (label);
8394 0 : LABEL_NUSES (label) = 1;
8395 : }
8396 0 : if (max_size > 1)
8397 : {
8398 0 : rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
8399 0 : src = change_address (srcmem, QImode, srcptr);
8400 0 : dest = change_address (destmem, QImode, destptr);
8401 0 : emit_insn (gen_strmov (destptr, dest, srcptr, src));
8402 0 : emit_label (label);
8403 0 : LABEL_NUSES (label) = 1;
8404 : }
8405 : }
8406 : else
8407 : {
8408 0 : rtx offset = force_reg (Pmode, const0_rtx);
8409 0 : rtx tmp;
8410 :
8411 0 : if (max_size > 4)
8412 : {
8413 0 : rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
8414 0 : src = change_address (srcmem, SImode, srcptr);
8415 0 : dest = change_address (destmem, SImode, destptr);
8416 0 : emit_move_insn (dest, src);
8417 0 : tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (4), NULL,
8418 : true, OPTAB_LIB_WIDEN);
8419 0 : if (tmp != offset)
8420 0 : emit_move_insn (offset, tmp);
8421 0 : emit_label (label);
8422 0 : LABEL_NUSES (label) = 1;
8423 : }
8424 0 : if (max_size > 2)
8425 : {
8426 0 : rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
8427 0 : tmp = gen_rtx_PLUS (Pmode, srcptr, offset);
8428 0 : src = change_address (srcmem, HImode, tmp);
8429 0 : tmp = gen_rtx_PLUS (Pmode, destptr, offset);
8430 0 : dest = change_address (destmem, HImode, tmp);
8431 0 : emit_move_insn (dest, src);
8432 0 : tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (2), tmp,
8433 : true, OPTAB_LIB_WIDEN);
8434 0 : if (tmp != offset)
8435 0 : emit_move_insn (offset, tmp);
8436 0 : emit_label (label);
8437 0 : LABEL_NUSES (label) = 1;
8438 : }
8439 0 : if (max_size > 1)
8440 : {
8441 0 : rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
8442 0 : tmp = gen_rtx_PLUS (Pmode, srcptr, offset);
8443 0 : src = change_address (srcmem, QImode, tmp);
8444 0 : tmp = gen_rtx_PLUS (Pmode, destptr, offset);
8445 0 : dest = change_address (destmem, QImode, tmp);
8446 0 : emit_move_insn (dest, src);
8447 0 : emit_label (label);
8448 0 : LABEL_NUSES (label) = 1;
8449 : }
8450 : }
8451 : }
8452 :
8453 : /* This function emits moves to fill SIZE_TO_MOVE bytes starting from DESTMEM
8454 : with value PROMOTED_VAL.
8455 : SRC is passed by pointer to be updated on return.
8456 : Return value is updated DST. */
8457 : static rtx
8458 6 : emit_memset (rtx destmem, rtx destptr, rtx promoted_val,
8459 : HOST_WIDE_INT size_to_move)
8460 : {
8461 6 : rtx dst = destmem;
8462 6 : enum insn_code code;
8463 6 : machine_mode move_mode;
8464 6 : int piece_size, i;
8465 :
8466 : /* Find the widest mode in which we could perform moves.
8467 : Start with the biggest power of 2 less than SIZE_TO_MOVE and half
8468 : it until move of such size is supported. */
8469 6 : move_mode = GET_MODE (promoted_val);
8470 6 : if (move_mode == VOIDmode)
8471 0 : move_mode = QImode;
8472 12 : if (size_to_move < GET_MODE_SIZE (move_mode))
8473 : {
8474 5 : unsigned int move_bits = size_to_move * BITS_PER_UNIT;
8475 5 : move_mode = int_mode_for_size (move_bits, 0).require ();
8476 5 : promoted_val = gen_lowpart (move_mode, promoted_val);
8477 : }
8478 6 : piece_size = GET_MODE_SIZE (move_mode);
8479 6 : code = optab_handler (mov_optab, move_mode);
8480 6 : gcc_assert (code != CODE_FOR_nothing && promoted_val != NULL_RTX);
8481 :
8482 6 : dst = adjust_automodify_address_nv (dst, move_mode, destptr, 0);
8483 :
8484 : /* Emit moves. We'll need SIZE_TO_MOVE/PIECE_SIZES moves. */
8485 6 : gcc_assert (size_to_move % piece_size == 0);
8486 :
8487 12 : for (i = 0; i < size_to_move; i += piece_size)
8488 : {
8489 6 : if (piece_size <= UNITS_PER_WORD)
8490 : {
8491 4 : emit_insn (gen_strset (destptr, dst, promoted_val));
8492 4 : dst = adjust_automodify_address_nv (dst, move_mode, destptr,
8493 : piece_size);
8494 4 : continue;
8495 : }
8496 :
8497 2 : emit_insn (GEN_FCN (code) (dst, promoted_val));
8498 :
8499 4 : emit_move_insn (destptr,
8500 2 : plus_constant (Pmode, copy_rtx (destptr), piece_size));
8501 :
8502 2 : dst = adjust_automodify_address_nv (dst, move_mode, destptr,
8503 : piece_size);
8504 : }
8505 :
8506 : /* Update DST rtx. */
8507 6 : return dst;
8508 : }
8509 : /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
8510 : static void
8511 37 : expand_setmem_epilogue_via_loop (rtx destmem, rtx destptr, rtx value,
8512 : rtx count, int max_size)
8513 : {
8514 74 : count = expand_simple_binop (counter_mode (count), AND, count,
8515 37 : GEN_INT (max_size - 1), count, 1, OPTAB_DIRECT);
8516 37 : expand_set_or_cpymem_via_loop (destmem, NULL, destptr, NULL,
8517 37 : gen_lowpart (QImode, value), count, QImode,
8518 : 1, max_size / 2, true);
8519 37 : }
8520 :
8521 : /* Callback routine for store_by_pieces. Return the RTL of a register
8522 : containing GET_MODE_SIZE (MODE) bytes in the RTL register op_p which
8523 : is an integer or a word vector register. If PREV_P isn't nullptr,
8524 : it has the RTL info from the previous iteration. */
8525 :
8526 : static rtx
8527 4968 : setmem_epilogue_gen_val (void *op_p, void *prev_p, HOST_WIDE_INT,
8528 : fixed_size_mode mode)
8529 : {
8530 4968 : rtx target;
8531 4968 : by_pieces_prev *prev = (by_pieces_prev *) prev_p;
8532 4968 : if (prev)
8533 : {
8534 4968 : rtx prev_op = prev->data;
8535 4968 : if (prev_op)
8536 : {
8537 2880 : machine_mode prev_mode = GET_MODE (prev_op);
8538 2880 : if (prev_mode == mode)
8539 : return prev_op;
8540 54 : if (VECTOR_MODE_P (prev_mode)
8541 1077 : && VECTOR_MODE_P (mode)
8542 1131 : && GET_MODE_INNER (prev_mode) == GET_MODE_INNER (mode))
8543 : {
8544 0 : target = gen_rtx_SUBREG (mode, prev_op, 0);
8545 0 : return target;
8546 : }
8547 : }
8548 : }
8549 :
8550 3219 : rtx op = (rtx) op_p;
8551 3219 : machine_mode op_mode = GET_MODE (op);
8552 :
8553 3219 : if (VECTOR_MODE_P (mode))
8554 : {
8555 3644 : gcc_assert (GET_MODE_INNER (mode) == QImode);
8556 :
8557 1822 : unsigned int op_size = GET_MODE_SIZE (op_mode);
8558 1822 : unsigned int size = GET_MODE_SIZE (mode);
8559 1822 : unsigned int nunits;
8560 1822 : machine_mode vec_mode;
8561 1822 : if (op_size < size)
8562 : {
8563 : /* If OP size is smaller than MODE size, duplicate it. */
8564 1 : nunits = size / GET_MODE_SIZE (QImode);
8565 1 : vec_mode = mode_for_vector (QImode, nunits).require ();
8566 1 : nunits = size / op_size;
8567 1 : gcc_assert (SCALAR_INT_MODE_P (op_mode));
8568 1 : machine_mode dup_mode
8569 1 : = mode_for_vector (as_a <scalar_mode> (op_mode),
8570 2 : nunits).require ();
8571 1 : target = gen_reg_rtx (vec_mode);
8572 1 : op = gen_vec_duplicate (dup_mode, op);
8573 1 : rtx dup_op = gen_reg_rtx (dup_mode);
8574 1 : emit_move_insn (dup_op, op);
8575 1 : op = gen_rtx_SUBREG (vec_mode, dup_op, 0);
8576 1 : emit_move_insn (target, op);
8577 1 : return target;
8578 : }
8579 1821 : nunits = op_size / GET_MODE_SIZE (QImode);
8580 1821 : vec_mode = mode_for_vector (QImode, nunits).require ();
8581 1821 : target = gen_reg_rtx (vec_mode);
8582 1821 : op = gen_rtx_SUBREG (vec_mode, op, 0);
8583 1821 : emit_move_insn (target, op);
8584 1821 : if (op_size == size)
8585 : return target;
8586 :
8587 0 : rtx tmp = gen_reg_rtx (mode);
8588 0 : target = gen_rtx_SUBREG (mode, target, 0);
8589 0 : emit_move_insn (tmp, target);
8590 0 : return tmp;
8591 : }
8592 :
8593 1397 : if (VECTOR_MODE_P (op_mode))
8594 : {
8595 2784 : gcc_assert (GET_MODE_INNER (op_mode) == word_mode);
8596 1392 : target = gen_reg_rtx (word_mode);
8597 1392 : op = gen_rtx_SUBREG (word_mode, op, 0);
8598 1392 : emit_move_insn (target, op);
8599 : }
8600 : else
8601 : target = op;
8602 :
8603 1397 : if (mode == GET_MODE (target))
8604 : return target;
8605 :
8606 246 : rtx tmp = gen_reg_rtx (mode);
8607 246 : target = gen_rtx_SUBREG (mode, target, 0);
8608 246 : emit_move_insn (tmp, target);
8609 246 : return tmp;
8610 : }
8611 :
8612 : /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
8613 : static void
8614 10043 : expand_setmem_epilogue (rtx destmem, rtx destptr, rtx value, rtx vec_value,
8615 : rtx count, int max_size)
8616 : {
8617 10043 : rtx dest;
8618 :
8619 10043 : if (CONST_INT_P (count))
8620 : {
8621 10006 : unsigned HOST_WIDE_INT countval = UINTVAL (count);
8622 10006 : unsigned HOST_WIDE_INT epilogue_size = countval % max_size;
8623 10006 : unsigned int destalign = MEM_ALIGN (destmem);
8624 10006 : cfun->machine->by_pieces_in_use = true;
8625 16581 : store_by_pieces (destmem, epilogue_size, setmem_epilogue_gen_val,
8626 : vec_value ? vec_value : value, destalign, true,
8627 : RETURN_BEGIN);
8628 10006 : cfun->machine->by_pieces_in_use = false;
8629 10006 : return;
8630 : }
8631 37 : if (max_size > 32)
8632 : {
8633 37 : expand_setmem_epilogue_via_loop (destmem, destptr, value, count, max_size);
8634 37 : return;
8635 : }
8636 0 : if (max_size > 16)
8637 : {
8638 0 : rtx_code_label *label = ix86_expand_aligntest (count, 16, true);
8639 0 : if (TARGET_64BIT)
8640 : {
8641 0 : dest = change_address (destmem, DImode, destptr);
8642 0 : emit_insn (gen_strset (destptr, dest, value));
8643 0 : dest = adjust_automodify_address_nv (dest, DImode, destptr, 8);
8644 0 : emit_insn (gen_strset (destptr, dest, value));
8645 : }
8646 : else
8647 : {
8648 0 : dest = change_address (destmem, SImode, destptr);
8649 0 : emit_insn (gen_strset (destptr, dest, value));
8650 0 : dest = adjust_automodify_address_nv (dest, SImode, destptr, 4);
8651 0 : emit_insn (gen_strset (destptr, dest, value));
8652 0 : dest = adjust_automodify_address_nv (dest, SImode, destptr, 8);
8653 0 : emit_insn (gen_strset (destptr, dest, value));
8654 0 : dest = adjust_automodify_address_nv (dest, SImode, destptr, 12);
8655 0 : emit_insn (gen_strset (destptr, dest, value));
8656 : }
8657 0 : emit_label (label);
8658 0 : LABEL_NUSES (label) = 1;
8659 : }
8660 0 : if (max_size > 8)
8661 : {
8662 0 : rtx_code_label *label = ix86_expand_aligntest (count, 8, true);
8663 0 : if (TARGET_64BIT)
8664 : {
8665 0 : dest = change_address (destmem, DImode, destptr);
8666 0 : emit_insn (gen_strset (destptr, dest, value));
8667 : }
8668 : else
8669 : {
8670 0 : dest = change_address (destmem, SImode, destptr);
8671 0 : emit_insn (gen_strset (destptr, dest, value));
8672 0 : dest = adjust_automodify_address_nv (dest, SImode, destptr, 4);
8673 0 : emit_insn (gen_strset (destptr, dest, value));
8674 : }
8675 0 : emit_label (label);
8676 0 : LABEL_NUSES (label) = 1;
8677 : }
8678 0 : if (max_size > 4)
8679 : {
8680 0 : rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
8681 0 : dest = change_address (destmem, SImode, destptr);
8682 0 : emit_insn (gen_strset (destptr, dest, gen_lowpart (SImode, value)));
8683 0 : emit_label (label);
8684 0 : LABEL_NUSES (label) = 1;
8685 : }
8686 0 : if (max_size > 2)
8687 : {
8688 0 : rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
8689 0 : dest = change_address (destmem, HImode, destptr);
8690 0 : emit_insn (gen_strset (destptr, dest, gen_lowpart (HImode, value)));
8691 0 : emit_label (label);
8692 0 : LABEL_NUSES (label) = 1;
8693 : }
8694 0 : if (max_size > 1)
8695 : {
8696 0 : rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
8697 0 : dest = change_address (destmem, QImode, destptr);
8698 0 : emit_insn (gen_strset (destptr, dest, gen_lowpart (QImode, value)));
8699 0 : emit_label (label);
8700 0 : LABEL_NUSES (label) = 1;
8701 : }
8702 : }
8703 :
8704 : /* Adjust COUNTER by the VALUE. */
8705 : static void
8706 19 : ix86_adjust_counter (rtx countreg, HOST_WIDE_INT value)
8707 : {
8708 19 : emit_insn (gen_add2_insn (countreg, GEN_INT (-value)));
8709 19 : }
8710 :
8711 : /* Depending on ISSETMEM, copy enough from SRCMEM to DESTMEM or set enough to
8712 : DESTMEM to align it to DESIRED_ALIGNMENT. Original alignment is ALIGN.
8713 : Depending on ISSETMEM, either arguments SRCMEM/SRCPTR or VALUE/VEC_VALUE are
8714 : ignored.
8715 : Return value is updated DESTMEM. */
8716 :
8717 : static rtx
8718 7 : expand_set_or_cpymem_prologue (rtx destmem, rtx srcmem,
8719 : rtx destptr, rtx srcptr, rtx value,
8720 : rtx vec_value, rtx count, int align,
8721 : int desired_alignment, bool issetmem)
8722 : {
8723 7 : int i;
8724 35 : for (i = 1; i < desired_alignment; i <<= 1)
8725 : {
8726 28 : if (align <= i)
8727 : {
8728 19 : rtx_code_label *label = ix86_expand_aligntest (destptr, i, false);
8729 19 : if (issetmem)
8730 : {
8731 12 : if (vec_value && i > GET_MODE_SIZE (GET_MODE (value)))
8732 2 : destmem = emit_memset (destmem, destptr, vec_value, i);
8733 : else
8734 4 : destmem = emit_memset (destmem, destptr, value, i);
8735 : }
8736 : else
8737 13 : destmem = emit_memmov (destmem, &srcmem, destptr, srcptr, i);
8738 19 : ix86_adjust_counter (count, i);
8739 19 : emit_label (label);
8740 19 : LABEL_NUSES (label) = 1;
8741 19 : set_mem_align (destmem, i * 2 * BITS_PER_UNIT);
8742 : }
8743 : }
8744 7 : return destmem;
8745 : }
8746 :
8747 : /* Test if COUNT&SIZE is nonzero and if so, expand movme
8748 : or setmem sequence that is valid for SIZE..2*SIZE-1 bytes
8749 : and jump to DONE_LABEL. */
8750 : static void
8751 2588 : expand_small_cpymem_or_setmem (rtx destmem, rtx srcmem,
8752 : rtx destptr, rtx srcptr,
8753 : rtx value, rtx vec_value,
8754 : rtx count, int size,
8755 : rtx done_label, bool issetmem)
8756 : {
8757 2588 : rtx_code_label *label = ix86_expand_aligntest (count, size, false);
8758 2588 : machine_mode mode = int_mode_for_size (size * BITS_PER_UNIT, 1).else_blk ();
8759 2588 : rtx modesize;
8760 2588 : rtx scalar_value = value;
8761 2588 : int n;
8762 :
8763 : /* If we do not have vector value to copy, we must reduce size. */
8764 2588 : if (issetmem)
8765 : {
8766 871 : if (!vec_value)
8767 : {
8768 2 : if (GET_MODE (value) == VOIDmode && size > 8)
8769 0 : mode = Pmode;
8770 6 : else if (GET_MODE_SIZE (mode) > GET_MODE_SIZE (GET_MODE (value)))
8771 0 : mode = GET_MODE (value);
8772 : }
8773 : else
8774 869 : mode = GET_MODE (vec_value), value = vec_value;
8775 : }
8776 : else
8777 : {
8778 : /* Choose appropriate vector mode. */
8779 1717 : if (size >= 32)
8780 429 : switch (MOVE_MAX)
8781 : {
8782 0 : case 64:
8783 0 : if (size >= 64)
8784 : {
8785 : mode = V64QImode;
8786 : break;
8787 : }
8788 : /* FALLTHRU */
8789 0 : case 32:
8790 0 : mode = V32QImode;
8791 0 : break;
8792 : case 16:
8793 : mode = V16QImode;
8794 : break;
8795 : case 8:
8796 : mode = DImode;
8797 : break;
8798 0 : default:
8799 0 : gcc_unreachable ();
8800 : }
8801 1288 : else if (size >= 16)
8802 429 : mode = TARGET_SSE ? V16QImode : DImode;
8803 1717 : srcmem = change_address (srcmem, mode, srcptr);
8804 : }
8805 3457 : if (issetmem && vec_value && GET_MODE_SIZE (mode) > size)
8806 : {
8807 : /* For memset with vector and the size is smaller than the vector
8808 : size, first try the narrower vector, otherwise, use the
8809 : original value. */
8810 439 : machine_mode inner_mode = GET_MODE_INNER (mode);
8811 439 : unsigned int nunits = size / GET_MODE_SIZE (inner_mode);
8812 439 : if (nunits > 1)
8813 : {
8814 328 : mode = mode_for_vector (GET_MODE_INNER (mode),
8815 328 : nunits).require ();
8816 164 : value = gen_rtx_SUBREG (mode, value, 0);
8817 : }
8818 : else
8819 : {
8820 275 : scalar_int_mode smode
8821 275 : = smallest_int_mode_for_size (size * BITS_PER_UNIT).require ();
8822 825 : gcc_assert (GET_MODE_SIZE (GET_MODE (scalar_value))
8823 : >= GET_MODE_SIZE (smode));
8824 275 : mode = smode;
8825 275 : if (GET_MODE (scalar_value) == mode)
8826 : value = scalar_value;
8827 : else
8828 60 : value = gen_rtx_SUBREG (mode, scalar_value, 0);
8829 : }
8830 : }
8831 2588 : destmem = change_address (destmem, mode, destptr);
8832 5176 : modesize = GEN_INT (GET_MODE_SIZE (mode));
8833 5176 : gcc_assert (GET_MODE_SIZE (mode) <= size);
8834 11640 : for (n = 0; n * GET_MODE_SIZE (mode) < size; n++)
8835 : {
8836 3232 : if (issetmem)
8837 1086 : emit_move_insn (destmem, gen_lowpart (mode, value));
8838 : else
8839 : {
8840 2146 : emit_move_insn (destmem, srcmem);
8841 4292 : srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
8842 : }
8843 6464 : destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
8844 : }
8845 :
8846 2588 : destmem = offset_address (destmem, count, 1);
8847 5176 : destmem = offset_address (destmem, GEN_INT (-2 * size),
8848 2588 : GET_MODE_SIZE (mode));
8849 2588 : if (!issetmem)
8850 : {
8851 1717 : srcmem = offset_address (srcmem, count, 1);
8852 3434 : srcmem = offset_address (srcmem, GEN_INT (-2 * size),
8853 1717 : GET_MODE_SIZE (mode));
8854 : }
8855 11640 : for (n = 0; n * GET_MODE_SIZE (mode) < size; n++)
8856 : {
8857 3232 : if (issetmem)
8858 1086 : emit_move_insn (destmem, gen_lowpart (mode, value));
8859 : else
8860 : {
8861 2146 : emit_move_insn (destmem, srcmem);
8862 4292 : srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
8863 : }
8864 6464 : destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
8865 : }
8866 2588 : emit_jump_insn (gen_jump (done_label));
8867 2588 : emit_barrier ();
8868 :
8869 2588 : emit_label (label);
8870 2588 : LABEL_NUSES (label) = 1;
8871 2588 : }
8872 :
8873 : /* Handle small memcpy (up to SIZE that is supposed to be small power of 2.
8874 : and get ready for the main memcpy loop by copying initial DESIRED_ALIGN-ALIGN
8875 : bytes and last SIZE bytes adjusitng DESTPTR/SRCPTR/COUNT in a way we can
8876 : proceed with an loop copying SIZE bytes at once. Do moves in MODE.
8877 : DONE_LABEL is a label after the whole copying sequence. The label is created
8878 : on demand if *DONE_LABEL is NULL.
8879 : MIN_SIZE is minimal size of block copied. This value gets adjusted for new
8880 : bounds after the initial copies.
8881 :
8882 : DESTMEM/SRCMEM are memory expressions pointing to the copies block,
8883 : DESTPTR/SRCPTR are pointers to the block. DYNAMIC_CHECK indicate whether
8884 : we will dispatch to a library call for large blocks.
8885 :
8886 : In pseudocode we do:
8887 :
8888 : if (COUNT < SIZE)
8889 : {
8890 : Assume that SIZE is 4. Bigger sizes are handled analogously
8891 : if (COUNT & 4)
8892 : {
8893 : copy 4 bytes from SRCPTR to DESTPTR
8894 : copy 4 bytes from SRCPTR + COUNT - 4 to DESTPTR + COUNT - 4
8895 : goto done_label
8896 : }
8897 : if (!COUNT)
8898 : goto done_label;
8899 : copy 1 byte from SRCPTR to DESTPTR
8900 : if (COUNT & 2)
8901 : {
8902 : copy 2 bytes from SRCPTR to DESTPTR
8903 : copy 2 bytes from SRCPTR + COUNT - 2 to DESTPTR + COUNT - 2
8904 : }
8905 : }
8906 : else
8907 : {
8908 : copy at least DESIRED_ALIGN-ALIGN bytes from SRCPTR to DESTPTR
8909 : copy SIZE bytes from SRCPTR + COUNT - SIZE to DESTPTR + COUNT -SIZE
8910 :
8911 : OLD_DESPTR = DESTPTR;
8912 : Align DESTPTR up to DESIRED_ALIGN
8913 : SRCPTR += DESTPTR - OLD_DESTPTR
8914 : COUNT -= DEST_PTR - OLD_DESTPTR
8915 : if (DYNAMIC_CHECK)
8916 : Round COUNT down to multiple of SIZE
8917 : << optional caller supplied zero size guard is here >>
8918 : << optional caller supplied dynamic check is here >>
8919 : << caller supplied main copy loop is here >>
8920 : }
8921 : done_label:
8922 : */
8923 : static void
8924 4006 : expand_set_or_cpymem_prologue_epilogue_by_misaligned_moves (rtx destmem, rtx srcmem,
8925 : rtx *destptr, rtx *srcptr,
8926 : machine_mode mode,
8927 : rtx value, rtx vec_value,
8928 : rtx *count,
8929 : rtx_code_label **done_label,
8930 : int size,
8931 : int desired_align,
8932 : int align,
8933 : unsigned HOST_WIDE_INT *min_size,
8934 : bool dynamic_check,
8935 : bool issetmem)
8936 : {
8937 4006 : rtx_code_label *loop_label = NULL, *label;
8938 4006 : int n;
8939 4006 : rtx modesize;
8940 4006 : int prolog_size = 0;
8941 4006 : rtx mode_value;
8942 :
8943 : /* Chose proper value to copy. */
8944 4006 : if (issetmem && VECTOR_MODE_P (mode))
8945 : mode_value = vec_value;
8946 : else
8947 4006 : mode_value = value;
8948 8012 : gcc_assert (GET_MODE_SIZE (mode) <= size);
8949 :
8950 : /* See if block is big or small, handle small blocks. */
8951 4006 : if (!CONST_INT_P (*count) && *min_size < (unsigned HOST_WIDE_INT)size)
8952 : {
8953 647 : int size2 = size;
8954 647 : loop_label = gen_label_rtx ();
8955 :
8956 647 : if (!*done_label)
8957 647 : *done_label = gen_label_rtx ();
8958 :
8959 647 : emit_cmp_and_jump_insns (*count, GEN_INT (size2), GE, 0, GET_MODE (*count),
8960 : 1, loop_label);
8961 647 : size2 >>= 1;
8962 :
8963 : /* Handle sizes > 3. */
8964 3235 : for (;size2 > 2; size2 >>= 1)
8965 2588 : expand_small_cpymem_or_setmem (destmem, srcmem,
8966 : *destptr, *srcptr,
8967 : value, vec_value,
8968 : *count,
8969 : size2, *done_label, issetmem);
8970 : /* Nothing to copy? Jump to DONE_LABEL if so */
8971 647 : emit_cmp_and_jump_insns (*count, const0_rtx, EQ, 0, GET_MODE (*count),
8972 : 1, *done_label);
8973 :
8974 : /* Do a byte copy. */
8975 647 : destmem = change_address (destmem, QImode, *destptr);
8976 647 : if (issetmem)
8977 217 : emit_move_insn (destmem, gen_lowpart (QImode, value));
8978 : else
8979 : {
8980 430 : srcmem = change_address (srcmem, QImode, *srcptr);
8981 430 : emit_move_insn (destmem, srcmem);
8982 : }
8983 :
8984 : /* Handle sizes 2 and 3. */
8985 647 : label = ix86_expand_aligntest (*count, 2, false);
8986 647 : destmem = change_address (destmem, HImode, *destptr);
8987 647 : destmem = offset_address (destmem, *count, 1);
8988 647 : destmem = offset_address (destmem, GEN_INT (-2), 2);
8989 647 : if (issetmem)
8990 217 : emit_move_insn (destmem, gen_lowpart (HImode, value));
8991 : else
8992 : {
8993 430 : srcmem = change_address (srcmem, HImode, *srcptr);
8994 430 : srcmem = offset_address (srcmem, *count, 1);
8995 430 : srcmem = offset_address (srcmem, GEN_INT (-2), 2);
8996 430 : emit_move_insn (destmem, srcmem);
8997 : }
8998 :
8999 647 : emit_label (label);
9000 647 : LABEL_NUSES (label) = 1;
9001 647 : emit_jump_insn (gen_jump (*done_label));
9002 647 : emit_barrier ();
9003 : }
9004 : else
9005 3359 : gcc_assert (*min_size >= (unsigned HOST_WIDE_INT)size
9006 : || UINTVAL (*count) >= (unsigned HOST_WIDE_INT)size);
9007 :
9008 : /* Start memcpy for COUNT >= SIZE. */
9009 647 : if (loop_label)
9010 : {
9011 647 : emit_label (loop_label);
9012 647 : LABEL_NUSES (loop_label) = 1;
9013 : }
9014 :
9015 : /* Copy first desired_align bytes. */
9016 4006 : if (!issetmem)
9017 2094 : srcmem = change_address (srcmem, mode, *srcptr);
9018 4006 : destmem = change_address (destmem, mode, *destptr);
9019 4006 : modesize = GEN_INT (GET_MODE_SIZE (mode));
9020 8029 : for (n = 0; prolog_size < desired_align - align; n++)
9021 : {
9022 17 : if (issetmem)
9023 1 : emit_move_insn (destmem, mode_value);
9024 : else
9025 : {
9026 16 : emit_move_insn (destmem, srcmem);
9027 32 : srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
9028 : }
9029 34 : destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
9030 34 : prolog_size += GET_MODE_SIZE (mode);
9031 : }
9032 :
9033 :
9034 : /* Copy last SIZE bytes. */
9035 4006 : destmem = offset_address (destmem, *count, 1);
9036 4006 : destmem = offset_address (destmem,
9037 4006 : GEN_INT (-size - prolog_size),
9038 : 1);
9039 4006 : if (issetmem)
9040 1912 : emit_move_insn (destmem, mode_value);
9041 : else
9042 : {
9043 2094 : srcmem = offset_address (srcmem, *count, 1);
9044 2094 : srcmem = offset_address (srcmem,
9045 : GEN_INT (-size - prolog_size),
9046 : 1);
9047 2094 : emit_move_insn (destmem, srcmem);
9048 : }
9049 30368 : for (n = 1; n * GET_MODE_SIZE (mode) < size; n++)
9050 : {
9051 11178 : destmem = offset_address (destmem, modesize, 1);
9052 11178 : if (issetmem)
9053 5550 : emit_move_insn (destmem, mode_value);
9054 : else
9055 : {
9056 5628 : srcmem = offset_address (srcmem, modesize, 1);
9057 5628 : emit_move_insn (destmem, srcmem);
9058 : }
9059 : }
9060 :
9061 : /* Align destination. */
9062 4006 : if (desired_align > 1 && desired_align > align)
9063 : {
9064 17 : rtx saveddest = *destptr;
9065 :
9066 17 : gcc_assert (desired_align <= size);
9067 : /* Align destptr up, place it to new register. */
9068 17 : *destptr = expand_simple_binop (GET_MODE (*destptr), PLUS, *destptr,
9069 : GEN_INT (prolog_size),
9070 : NULL_RTX, 1, OPTAB_DIRECT);
9071 17 : if (REG_P (*destptr) && REG_P (saveddest) && REG_POINTER (saveddest))
9072 17 : REG_POINTER (*destptr) = 1;
9073 17 : *destptr = expand_simple_binop (GET_MODE (*destptr), AND, *destptr,
9074 17 : GEN_INT (-desired_align),
9075 : *destptr, 1, OPTAB_DIRECT);
9076 : /* See how many bytes we skipped. */
9077 17 : saveddest = expand_simple_binop (GET_MODE (*destptr), MINUS, saveddest,
9078 : *destptr,
9079 : NULL_RTX, 1, OPTAB_DIRECT);
9080 : /* Adjust srcptr and count. */
9081 17 : if (!issetmem)
9082 16 : *srcptr = expand_simple_binop (GET_MODE (*srcptr), MINUS, *srcptr,
9083 : saveddest, *srcptr, 1, OPTAB_DIRECT);
9084 17 : *count = expand_simple_binop (GET_MODE (*count), PLUS, *count,
9085 : saveddest, *count, 1, OPTAB_DIRECT);
9086 : /* We copied at most size + prolog_size. */
9087 17 : if (*min_size > (unsigned HOST_WIDE_INT)(size + prolog_size))
9088 16 : *min_size
9089 16 : = ROUND_DOWN (*min_size - size, (unsigned HOST_WIDE_INT)size);
9090 : else
9091 : *min_size = 0;
9092 :
9093 : /* Our loops always round down the block size, but for dispatch to
9094 : library we need precise value. */
9095 17 : if (dynamic_check)
9096 17 : *count = expand_simple_binop (GET_MODE (*count), AND, *count,
9097 : GEN_INT (-size), *count, 1, OPTAB_DIRECT);
9098 : }
9099 : else
9100 : {
9101 3989 : gcc_assert (prolog_size == 0);
9102 : /* Decrease count, so we won't end up copying last word twice. */
9103 3989 : if (!CONST_INT_P (*count))
9104 646 : *count = expand_simple_binop (GET_MODE (*count), PLUS, *count,
9105 : constm1_rtx, *count, 1, OPTAB_DIRECT);
9106 : else
9107 3343 : *count = GEN_INT (ROUND_DOWN (UINTVAL (*count) - 1,
9108 : (unsigned HOST_WIDE_INT)size));
9109 3989 : if (*min_size)
9110 3777 : *min_size = ROUND_DOWN (*min_size - 1, (unsigned HOST_WIDE_INT)size);
9111 : }
9112 4006 : }
9113 :
9114 :
9115 : /* This function is like the previous one, except here we know how many bytes
9116 : need to be copied. That allows us to update alignment not only of DST, which
9117 : is returned, but also of SRC, which is passed as a pointer for that
9118 : reason. */
9119 : static rtx
9120 0 : expand_set_or_cpymem_constant_prologue (rtx dst, rtx *srcp, rtx destreg,
9121 : rtx srcreg, rtx value, rtx vec_value,
9122 : int desired_align, int align_bytes,
9123 : bool issetmem)
9124 : {
9125 0 : rtx src = NULL;
9126 0 : rtx orig_dst = dst;
9127 0 : rtx orig_src = NULL;
9128 0 : int piece_size = 1;
9129 0 : int copied_bytes = 0;
9130 :
9131 0 : if (!issetmem)
9132 : {
9133 0 : gcc_assert (srcp != NULL);
9134 0 : src = *srcp;
9135 0 : orig_src = src;
9136 : }
9137 :
9138 0 : for (piece_size = 1;
9139 0 : piece_size <= desired_align && copied_bytes < align_bytes;
9140 0 : piece_size <<= 1)
9141 : {
9142 0 : if (align_bytes & piece_size)
9143 : {
9144 0 : if (issetmem)
9145 : {
9146 0 : if (vec_value && piece_size > GET_MODE_SIZE (GET_MODE (value)))
9147 0 : dst = emit_memset (dst, destreg, vec_value, piece_size);
9148 : else
9149 0 : dst = emit_memset (dst, destreg, value, piece_size);
9150 : }
9151 : else
9152 0 : dst = emit_memmov (dst, &src, destreg, srcreg, piece_size);
9153 0 : copied_bytes += piece_size;
9154 : }
9155 : }
9156 0 : if (MEM_ALIGN (dst) < (unsigned int) desired_align * BITS_PER_UNIT)
9157 0 : set_mem_align (dst, desired_align * BITS_PER_UNIT);
9158 0 : if (MEM_SIZE_KNOWN_P (orig_dst))
9159 0 : set_mem_size (dst, MEM_SIZE (orig_dst) - align_bytes);
9160 :
9161 0 : if (!issetmem)
9162 : {
9163 0 : int src_align_bytes = get_mem_align_offset (src, desired_align
9164 : * BITS_PER_UNIT);
9165 0 : if (src_align_bytes >= 0)
9166 0 : src_align_bytes = desired_align - src_align_bytes;
9167 0 : if (src_align_bytes >= 0)
9168 : {
9169 : unsigned int src_align;
9170 0 : for (src_align = desired_align; src_align >= 2; src_align >>= 1)
9171 : {
9172 0 : if ((src_align_bytes & (src_align - 1))
9173 0 : == (align_bytes & (src_align - 1)))
9174 : break;
9175 : }
9176 0 : if (src_align > (unsigned int) desired_align)
9177 : src_align = desired_align;
9178 0 : if (MEM_ALIGN (src) < src_align * BITS_PER_UNIT)
9179 0 : set_mem_align (src, src_align * BITS_PER_UNIT);
9180 : }
9181 0 : if (MEM_SIZE_KNOWN_P (orig_src))
9182 0 : set_mem_size (src, MEM_SIZE (orig_src) - align_bytes);
9183 0 : *srcp = src;
9184 : }
9185 :
9186 0 : return dst;
9187 : }
9188 :
9189 : /* Return true if ALG can be used in current context.
9190 : Assume we expand memset if MEMSET is true. */
9191 : static bool
9192 1450862 : alg_usable_p (enum stringop_alg alg, bool memset,
9193 : addr_space_t dst_as, addr_space_t src_as)
9194 : {
9195 1450862 : if (alg == no_stringop)
9196 : return false;
9197 : /* It is not possible to use a library call if we have non-default
9198 : address space. We can do better than the generic byte-at-a-time
9199 : loop, used as a fallback. */
9200 1450862 : if (alg == libcall &&
9201 832319 : !(ADDR_SPACE_GENERIC_P (dst_as) && ADDR_SPACE_GENERIC_P (src_as)))
9202 : return false;
9203 1450848 : if (alg == vector_loop)
9204 310190 : return TARGET_SSE || TARGET_AVX;
9205 : /* Algorithms using the rep prefix want at least edi and ecx;
9206 : additionally, memset wants eax and memcpy wants esi. Don't
9207 : consider such algorithms if the user has appropriated those
9208 : registers for their own purposes, or if we have the destination
9209 : in the non-default address space, since string insns cannot
9210 : override the destination segment. */
9211 1140658 : if (alg == rep_prefix_1_byte
9212 : || alg == rep_prefix_4_byte
9213 1140658 : || alg == rep_prefix_8_byte)
9214 : {
9215 39137 : if (fixed_regs[CX_REG]
9216 39133 : || fixed_regs[DI_REG]
9217 39129 : || (memset ? fixed_regs[AX_REG] : fixed_regs[SI_REG])
9218 39125 : || !ADDR_SPACE_GENERIC_P (dst_as)
9219 78262 : || !(ADDR_SPACE_GENERIC_P (src_as) || Pmode == word_mode))
9220 12 : return false;
9221 : }
9222 : return true;
9223 : }
9224 :
9225 : /* Given COUNT and EXPECTED_SIZE, decide on codegen of string operation. */
9226 : static enum stringop_alg
9227 288635 : decide_alg (HOST_WIDE_INT count, HOST_WIDE_INT expected_size,
9228 : unsigned HOST_WIDE_INT min_size, unsigned HOST_WIDE_INT max_size,
9229 : bool memset, bool zero_memset, addr_space_t dst_as,
9230 : addr_space_t src_as, int *dynamic_check, bool *noalign, bool recur)
9231 : {
9232 288635 : const struct stringop_algs *algs;
9233 288635 : bool optimize_for_speed;
9234 288635 : int max = 0;
9235 288635 : const struct processor_costs *cost;
9236 288635 : int i;
9237 288635 : bool any_alg_usable_p = false;
9238 :
9239 288635 : *noalign = false;
9240 288635 : *dynamic_check = -1;
9241 :
9242 : /* Even if the string operation call is cold, we still might spend a lot
9243 : of time processing large blocks. */
9244 288635 : if (optimize_function_for_size_p (cfun)
9245 288635 : || (optimize_insn_for_size_p ()
9246 12089 : && (max_size < 256
9247 3825 : || (expected_size != -1 && expected_size < 256))))
9248 : optimize_for_speed = false;
9249 : else
9250 269211 : optimize_for_speed = true;
9251 :
9252 269211 : cost = optimize_for_speed ? ix86_cost : &ix86_size_cost;
9253 288635 : if (memset)
9254 89349 : algs = &cost->memset[TARGET_64BIT != 0];
9255 : else
9256 208160 : algs = &cost->memcpy[TARGET_64BIT != 0];
9257 :
9258 : /* See maximal size for user defined algorithm. */
9259 1443175 : for (i = 0; i < MAX_STRINGOP_ALGS; i++)
9260 : {
9261 1154540 : enum stringop_alg candidate = algs->size[i].alg;
9262 1154540 : bool usable = alg_usable_p (candidate, memset, dst_as, src_as);
9263 1154540 : any_alg_usable_p |= usable;
9264 :
9265 1154540 : if (candidate != libcall && candidate && usable)
9266 557733 : max = algs->size[i].max;
9267 : }
9268 :
9269 : /* If expected size is not known but max size is small enough
9270 : so inline version is a win, set expected size into
9271 : the range. */
9272 288635 : if (((max > 1 && (unsigned HOST_WIDE_INT) max >= max_size) || max == -1)
9273 38792 : && expected_size == -1)
9274 15598 : expected_size = min_size / 2 + max_size / 2;
9275 :
9276 : /* If user specified the algorithm, honor it if possible. */
9277 288635 : if (ix86_stringop_alg != no_stringop
9278 288635 : && alg_usable_p (ix86_stringop_alg, memset, dst_as, src_as))
9279 : return ix86_stringop_alg;
9280 : /* rep; movq or rep; movl is the smallest variant. */
9281 288497 : else if (!optimize_for_speed)
9282 : {
9283 19341 : *noalign = true;
9284 19341 : if (!count || (count & 3) || (memset && !zero_memset))
9285 6012 : return alg_usable_p (rep_prefix_1_byte, memset, dst_as, src_as)
9286 6012 : ? rep_prefix_1_byte : loop_1_byte;
9287 : else
9288 13329 : return alg_usable_p (rep_prefix_4_byte, memset, dst_as, src_as)
9289 13329 : ? rep_prefix_4_byte : loop;
9290 : }
9291 : /* Very tiny blocks are best handled via the loop, REP is expensive to
9292 : setup. */
9293 269156 : else if (expected_size != -1 && expected_size < 4)
9294 : return loop_1_byte;
9295 267608 : else if (expected_size != -1)
9296 : {
9297 : enum stringop_alg alg = libcall;
9298 : bool alg_noalign = false;
9299 345675 : for (i = 0; i < MAX_STRINGOP_ALGS; i++)
9300 : {
9301 : /* We get here if the algorithms that were not libcall-based
9302 : were rep-prefix based and we are unable to use rep prefixes
9303 : based on global register usage. Break out of the loop and
9304 : use the heuristic below. */
9305 339938 : if (algs->size[i].max == 0)
9306 : break;
9307 339938 : if (algs->size[i].max >= expected_size || algs->size[i].max == -1)
9308 : {
9309 133765 : enum stringop_alg candidate = algs->size[i].alg;
9310 :
9311 133765 : if (candidate != libcall
9312 133765 : && alg_usable_p (candidate, memset, dst_as, src_as))
9313 : {
9314 24917 : alg = candidate;
9315 24917 : alg_noalign = algs->size[i].noalign;
9316 : }
9317 : /* Honor TARGET_INLINE_ALL_STRINGOPS by picking
9318 : last non-libcall inline algorithm. */
9319 133765 : if (TARGET_INLINE_ALL_STRINGOPS)
9320 : {
9321 : /* When the current size is best to be copied by a libcall,
9322 : but we are still forced to inline, run the heuristic below
9323 : that will pick code for medium sized blocks. */
9324 20035 : if (alg != libcall)
9325 : {
9326 8552 : *noalign = alg_noalign;
9327 8552 : return alg;
9328 : }
9329 11483 : else if (!any_alg_usable_p)
9330 : break;
9331 : }
9332 113730 : else if (alg_usable_p (candidate, memset, dst_as, src_as)
9333 113730 : && !(TARGET_PREFER_KNOWN_REP_MOVSB_STOSB
9334 34 : && candidate == rep_prefix_1_byte
9335 : /* NB: If min_size != max_size, size is
9336 : unknown. */
9337 34 : && min_size != max_size))
9338 : {
9339 113705 : *noalign = algs->size[i].noalign;
9340 113705 : return candidate;
9341 : }
9342 : }
9343 : }
9344 : }
9345 : /* When asked to inline the call anyway, try to pick meaningful choice.
9346 : We look for maximal size of block that is faster to copy by hand and
9347 : take blocks of at most of that size guessing that average size will
9348 : be roughly half of the block.
9349 :
9350 : If this turns out to be bad, we might simply specify the preferred
9351 : choice in ix86_costs. */
9352 138173 : if ((TARGET_INLINE_ALL_STRINGOPS || TARGET_INLINE_STRINGOPS_DYNAMICALLY)
9353 145357 : && (algs->unknown_size == libcall
9354 0 : || !alg_usable_p (algs->unknown_size, memset, dst_as, src_as)))
9355 : {
9356 7184 : enum stringop_alg alg;
9357 7184 : HOST_WIDE_INT new_expected_size = (max > 0 ? max : 4096) / 2;
9358 :
9359 : /* If there aren't any usable algorithms or if recursing already,
9360 : then recursing on smaller sizes or same size isn't going to
9361 : find anything. Just return the simple byte-at-a-time copy loop. */
9362 7184 : if (!any_alg_usable_p || recur)
9363 : {
9364 : /* Pick something reasonable. */
9365 0 : if (TARGET_INLINE_STRINGOPS_DYNAMICALLY && !recur)
9366 0 : *dynamic_check = 128;
9367 : return loop_1_byte;
9368 : }
9369 7184 : alg = decide_alg (count, new_expected_size, min_size, max_size,
9370 : memset, zero_memset, dst_as, src_as,
9371 : dynamic_check, noalign, true);
9372 7184 : gcc_assert (*dynamic_check == -1);
9373 7184 : if (TARGET_INLINE_STRINGOPS_DYNAMICALLY)
9374 10 : *dynamic_check = max;
9375 : else
9376 7174 : gcc_assert (alg != libcall);
9377 : return alg;
9378 : }
9379 :
9380 : /* Try to use some reasonable fallback algorithm. Note that for
9381 : non-default address spaces we default to a loop instead of
9382 : a libcall. */
9383 :
9384 138167 : bool have_as = !(ADDR_SPACE_GENERIC_P (dst_as)
9385 : && ADDR_SPACE_GENERIC_P (src_as));
9386 :
9387 138167 : return (alg_usable_p (algs->unknown_size, memset, dst_as, src_as)
9388 138167 : ? algs->unknown_size : have_as ? loop : libcall);
9389 : }
9390 :
9391 : /* Decide on alignment. We know that the operand is already aligned to ALIGN
9392 : (ALIGN can be based on profile feedback and thus it is not 100% guaranteed). */
9393 : static int
9394 27527 : decide_alignment (int align,
9395 : enum stringop_alg alg,
9396 : int expected_size,
9397 : machine_mode move_mode)
9398 : {
9399 27527 : int desired_align = 0;
9400 :
9401 27527 : gcc_assert (alg != no_stringop);
9402 :
9403 27527 : if (alg == libcall)
9404 : return 0;
9405 27527 : if (move_mode == VOIDmode)
9406 : return 0;
9407 :
9408 27527 : desired_align = GET_MODE_SIZE (move_mode);
9409 : /* PentiumPro has special logic triggering for 8 byte aligned blocks.
9410 : copying whole cacheline at once. */
9411 27527 : if (TARGET_CPU_P (PENTIUMPRO)
9412 0 : && (alg == rep_prefix_4_byte || alg == rep_prefix_1_byte))
9413 27527 : desired_align = 8;
9414 :
9415 27527 : if (optimize_size)
9416 10586 : desired_align = 1;
9417 27527 : if (desired_align < align)
9418 : desired_align = align;
9419 27527 : if (expected_size != -1 && expected_size < 4)
9420 1 : desired_align = align;
9421 :
9422 : return desired_align;
9423 : }
9424 :
9425 :
9426 : /* Helper function for memcpy. For QImode value 0xXY produce
9427 : 0xXYXYXYXY of wide specified by MODE. This is essentially
9428 : a * 0x10101010, but we can do slightly better than
9429 : synth_mult by unwinding the sequence by hand on CPUs with
9430 : slow multiply. */
9431 : static rtx
9432 18616 : promote_duplicated_reg (machine_mode mode, rtx val)
9433 : {
9434 18616 : if (val == const0_rtx)
9435 16101 : return copy_to_mode_reg (mode, CONST0_RTX (mode));
9436 :
9437 2515 : machine_mode valmode = GET_MODE (val);
9438 2515 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
9439 : {
9440 : /* Duplicate the scalar value for integer vector. */
9441 1899 : gcc_assert ((val == const0_rtx || val == constm1_rtx)
9442 : || GET_MODE_INNER (mode) == valmode);
9443 954 : rtx dup = gen_reg_rtx (mode);
9444 954 : bool ok = ix86_expand_vector_init_duplicate (false, mode, dup,
9445 : val);
9446 954 : gcc_assert (ok);
9447 : return dup;
9448 : }
9449 :
9450 1561 : rtx tmp;
9451 1561 : int nops = mode == DImode ? 3 : 2;
9452 :
9453 41 : gcc_assert (mode == SImode || mode == DImode);
9454 1561 : if (CONST_INT_P (val))
9455 : {
9456 1248 : HOST_WIDE_INT v = INTVAL (val) & 255;
9457 :
9458 1248 : v |= v << 8;
9459 1248 : v |= v << 16;
9460 1248 : if (mode == DImode)
9461 1219 : v |= (v << 16) << 16;
9462 1248 : return copy_to_mode_reg (mode, gen_int_mode (v, mode));
9463 : }
9464 :
9465 313 : if (valmode == VOIDmode)
9466 : valmode = QImode;
9467 313 : if (valmode != QImode)
9468 0 : val = gen_lowpart (QImode, val);
9469 313 : if (mode == QImode)
9470 : return val;
9471 313 : if (!TARGET_PARTIAL_REG_STALL)
9472 313 : nops--;
9473 313 : if (ix86_cost->mult_init[mode == DImode ? 3 : 2]
9474 313 : + ix86_cost->mult_bit * (mode == DImode ? 8 : 4)
9475 313 : <= (ix86_cost->shift_const + ix86_cost->add) * nops
9476 313 : + (COSTS_N_INSNS (TARGET_PARTIAL_REG_STALL == 0)))
9477 : {
9478 313 : rtx reg = convert_modes (mode, QImode, val, true);
9479 313 : tmp = promote_duplicated_reg (mode, const1_rtx);
9480 313 : return expand_simple_binop (mode, MULT, reg, tmp, NULL, 1,
9481 313 : OPTAB_DIRECT);
9482 : }
9483 : else
9484 : {
9485 0 : rtx reg = convert_modes (mode, QImode, val, true);
9486 :
9487 0 : if (!TARGET_PARTIAL_REG_STALL)
9488 0 : emit_insn (gen_insv_1 (mode, reg, reg));
9489 : else
9490 : {
9491 0 : tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (8),
9492 : NULL, 1, OPTAB_DIRECT);
9493 0 : reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1,
9494 : OPTAB_DIRECT);
9495 : }
9496 0 : tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (16),
9497 : NULL, 1, OPTAB_DIRECT);
9498 0 : reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
9499 0 : if (mode == SImode)
9500 : return reg;
9501 0 : tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (32),
9502 : NULL, 1, OPTAB_DIRECT);
9503 0 : reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
9504 0 : return reg;
9505 : }
9506 : }
9507 :
9508 : /* Duplicate value VAL using promote_duplicated_reg into maximal size that will
9509 : be needed by main loop copying SIZE_NEEDED chunks and prologue getting
9510 : alignment from ALIGN to DESIRED_ALIGN. */
9511 : static rtx
9512 13567 : promote_duplicated_reg_to_size (rtx val, int size_needed, int desired_align,
9513 : int align)
9514 : {
9515 13567 : rtx promoted_val;
9516 :
9517 13567 : if (TARGET_64BIT
9518 12087 : && (size_needed > 4 || (desired_align > align && desired_align > 4)))
9519 3942 : promoted_val = promote_duplicated_reg (DImode, val);
9520 9625 : else if (size_needed > 2 || (desired_align > align && desired_align > 2))
9521 8013 : promoted_val = promote_duplicated_reg (SImode, val);
9522 1612 : else if (size_needed > 1 || (desired_align > align && desired_align > 1))
9523 0 : promoted_val = promote_duplicated_reg (HImode, val);
9524 : else
9525 : promoted_val = val;
9526 :
9527 13567 : return promoted_val;
9528 : }
9529 :
9530 : /* Copy the address to a Pmode register. This is used for x32 to
9531 : truncate DImode TLS address to a SImode register. */
9532 :
9533 : static rtx
9534 66435 : ix86_copy_addr_to_reg (rtx addr)
9535 : {
9536 66435 : rtx reg;
9537 71008 : if (GET_MODE (addr) == Pmode || GET_MODE (addr) == VOIDmode)
9538 : {
9539 66435 : reg = copy_addr_to_reg (addr);
9540 66435 : REG_POINTER (reg) = 1;
9541 66435 : return reg;
9542 : }
9543 : else
9544 : {
9545 0 : gcc_assert (GET_MODE (addr) == DImode && Pmode == SImode);
9546 0 : reg = copy_to_mode_reg (DImode, addr);
9547 0 : REG_POINTER (reg) = 1;
9548 0 : return gen_rtx_SUBREG (SImode, reg, 0);
9549 : }
9550 : }
9551 :
9552 : /* Expand string move (memcpy) ot store (memset) operation. Use i386 string
9553 : operations when profitable. The code depends upon architecture, block size
9554 : and alignment, but always has one of the following overall structures:
9555 :
9556 : Aligned move sequence:
9557 :
9558 : 1) Prologue guard: Conditional that jumps up to epilogues for small
9559 : blocks that can be handled by epilogue alone. This is faster
9560 : but also needed for correctness, since prologue assume the block
9561 : is larger than the desired alignment.
9562 :
9563 : Optional dynamic check for size and libcall for large
9564 : blocks is emitted here too, with -minline-stringops-dynamically.
9565 :
9566 : 2) Prologue: copy first few bytes in order to get destination
9567 : aligned to DESIRED_ALIGN. It is emitted only when ALIGN is less
9568 : than DESIRED_ALIGN and up to DESIRED_ALIGN - ALIGN bytes can be
9569 : copied. We emit either a jump tree on power of two sized
9570 : blocks, or a byte loop.
9571 :
9572 : 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
9573 : with specified algorithm.
9574 :
9575 : 4) Epilogue: code copying tail of the block that is too small to be
9576 : handled by main body (or up to size guarded by prologue guard).
9577 :
9578 : Misaligned move sequence
9579 :
9580 : 1) missaligned move prologue/epilogue containing:
9581 : a) Prologue handling small memory blocks and jumping to done_label
9582 : (skipped if blocks are known to be large enough)
9583 : b) Single move copying first DESIRED_ALIGN-ALIGN bytes if alignment is
9584 : needed by single possibly misaligned move
9585 : (skipped if alignment is not needed)
9586 : c) Copy of last SIZE_NEEDED bytes by possibly misaligned moves
9587 :
9588 : 2) Zero size guard dispatching to done_label, if needed
9589 :
9590 : 3) dispatch to library call, if needed,
9591 :
9592 : 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
9593 : with specified algorithm. */
9594 : bool
9595 153513 : ix86_expand_set_or_cpymem (rtx dst, rtx src, rtx count_exp, rtx val_exp,
9596 : rtx align_exp, rtx expected_align_exp,
9597 : rtx expected_size_exp, rtx min_size_exp,
9598 : rtx max_size_exp, rtx probable_max_size_exp,
9599 : bool issetmem)
9600 : {
9601 153513 : if (TARGET_MISALIGNED_MOVE_STRING_PRO_EPILOGUES)
9602 : {
9603 : /* Expand bounded memset and memcpy as memmove if misaligned moves
9604 : are preferred. Since
9605 :
9606 : commit b41f96465190751561f6909e858604ceab00595b
9607 : Author: H.J. Lu <hjl.tools@gmail.com>
9608 : Date: Mon Oct 20 16:14:34 2025 +0800
9609 :
9610 : x86-64: Inline memmove with overlapping unaligned loads and stores.
9611 :
9612 : inlines memmove with overlapping unaligned and stores, which
9613 : reduces the numbers of branches and memory moves, comparing
9614 : against the regular memset and memcpy inlining. */
9615 153495 : rtx operands[9];
9616 153495 : operands[0] = dst;
9617 153495 : operands[1] = issetmem ? val_exp : src;
9618 153495 : operands[2] = count_exp;
9619 153495 : operands[3] = align_exp;
9620 153495 : operands[4] = expected_align_exp;
9621 153495 : operands[5] = expected_size_exp;
9622 153495 : operands[6] = min_size_exp;
9623 153495 : operands[7] = max_size_exp;
9624 153495 : operands[8] = probable_max_size_exp;
9625 153495 : if (ix86_expand_set_or_movmem (operands, !issetmem, issetmem))
9626 8808 : return true;
9627 : }
9628 :
9629 144705 : rtx destreg;
9630 144705 : rtx srcreg = NULL;
9631 144705 : rtx_code_label *label = NULL;
9632 144705 : rtx tmp;
9633 144705 : rtx_code_label *jump_around_label = NULL;
9634 144705 : HOST_WIDE_INT align = 1;
9635 144705 : unsigned HOST_WIDE_INT count = 0;
9636 144705 : HOST_WIDE_INT expected_size = -1;
9637 144705 : int size_needed = 0, epilogue_size_needed;
9638 144705 : int desired_align = 0, align_bytes = 0;
9639 144705 : enum stringop_alg alg;
9640 144705 : rtx promoted_val = NULL;
9641 144705 : rtx vec_promoted_val = NULL;
9642 144705 : bool force_loopy_epilogue = false;
9643 144705 : int dynamic_check;
9644 144705 : bool need_zero_guard = false;
9645 144705 : bool noalign;
9646 144705 : machine_mode move_mode = VOIDmode;
9647 144705 : int unroll_factor = 1;
9648 : /* TODO: Once value ranges are available, fill in proper data. */
9649 144705 : unsigned HOST_WIDE_INT min_size = HOST_WIDE_INT_0U;
9650 144705 : unsigned HOST_WIDE_INT max_size = HOST_WIDE_INT_M1U;
9651 144705 : unsigned HOST_WIDE_INT probable_max_size = HOST_WIDE_INT_M1U;
9652 144705 : bool misaligned_prologue_used = false;
9653 144705 : addr_space_t dst_as, src_as = ADDR_SPACE_GENERIC;
9654 :
9655 144705 : if (CONST_INT_P (align_exp))
9656 144705 : align = INTVAL (align_exp);
9657 : /* i386 can do misaligned access on reasonably increased cost. */
9658 144705 : if (CONST_INT_P (expected_align_exp)
9659 144705 : && INTVAL (expected_align_exp) > align)
9660 : align = INTVAL (expected_align_exp);
9661 : /* ALIGN is the minimum of destination and source alignment, but we care here
9662 : just about destination alignment. */
9663 138231 : else if (!issetmem
9664 229116 : && MEM_ALIGN (dst) > (unsigned HOST_WIDE_INT) align * BITS_PER_UNIT)
9665 3194 : align = MEM_ALIGN (dst) / BITS_PER_UNIT;
9666 :
9667 144705 : if (CONST_INT_P (count_exp))
9668 : {
9669 71802 : min_size = max_size = probable_max_size = count = expected_size
9670 71802 : = INTVAL (count_exp);
9671 : /* When COUNT is 0, there is nothing to do. */
9672 71802 : if (!count)
9673 : return true;
9674 : }
9675 : else
9676 : {
9677 72903 : if (min_size_exp)
9678 72903 : min_size = INTVAL (min_size_exp);
9679 72903 : if (max_size_exp)
9680 57795 : max_size = INTVAL (max_size_exp);
9681 72903 : if (probable_max_size_exp)
9682 59239 : probable_max_size = INTVAL (probable_max_size_exp);
9683 72903 : if (CONST_INT_P (expected_size_exp))
9684 72903 : expected_size = INTVAL (expected_size_exp);
9685 : }
9686 :
9687 : /* Make sure we don't need to care about overflow later on. */
9688 144705 : if (count > (HOST_WIDE_INT_1U << 30))
9689 : return false;
9690 :
9691 144530 : dst_as = MEM_ADDR_SPACE (dst);
9692 144530 : if (!issetmem)
9693 97245 : src_as = MEM_ADDR_SPACE (src);
9694 :
9695 : /* Step 0: Decide on preferred algorithm, desired alignment and
9696 : size of chunks to be copied by main loop. */
9697 144530 : alg = decide_alg (count, expected_size, min_size, probable_max_size,
9698 47285 : issetmem, issetmem && val_exp == const0_rtx,
9699 : dst_as, src_as, &dynamic_check, &noalign, false);
9700 :
9701 144530 : if (dump_file)
9702 13 : fprintf (dump_file, "Selected stringop expansion strategy: %s\n",
9703 13 : stringop_alg_names[alg]);
9704 :
9705 144530 : if (alg == libcall)
9706 : return false;
9707 27527 : gcc_assert (alg != no_stringop);
9708 :
9709 27527 : if (!count)
9710 5602 : count_exp = copy_to_mode_reg (GET_MODE (count_exp), count_exp);
9711 27527 : destreg = ix86_copy_addr_to_reg (XEXP (dst, 0));
9712 27527 : if (!issetmem)
9713 13960 : srcreg = ix86_copy_addr_to_reg (XEXP (src, 0));
9714 :
9715 27527 : bool aligned_dstmem = false;
9716 27527 : unsigned int nunits = issetmem ? STORE_MAX_PIECES : MOVE_MAX;
9717 27527 : bool single_insn_p = count && count <= nunits;
9718 27527 : if (single_insn_p)
9719 : {
9720 : /* If it can be done with a single instruction, use vector
9721 : instruction and don't align destination. */
9722 7 : alg = vector_loop;
9723 7 : noalign = true;
9724 7 : dynamic_check = -1;
9725 : }
9726 :
9727 27527 : unroll_factor = 1;
9728 27527 : move_mode = word_mode;
9729 27527 : switch (alg)
9730 : {
9731 0 : case libcall:
9732 0 : case no_stringop:
9733 0 : case last_alg:
9734 0 : gcc_unreachable ();
9735 173 : case loop_1_byte:
9736 173 : need_zero_guard = true;
9737 173 : move_mode = QImode;
9738 173 : break;
9739 40 : case loop:
9740 40 : need_zero_guard = true;
9741 40 : break;
9742 19 : case unrolled_loop:
9743 19 : need_zero_guard = true;
9744 19 : unroll_factor = (TARGET_64BIT ? 4 : 2);
9745 : break;
9746 7929 : case vector_loop:
9747 7929 : need_zero_guard = true;
9748 7929 : unroll_factor = 4;
9749 : /* Get the vector mode to move STORE_MAX_PIECES/MOVE_MAX bytes. */
9750 7929 : nunits /= UNITS_PER_WORD;
9751 7929 : if (nunits > 1)
9752 : {
9753 7925 : move_mode = mode_for_vector (word_mode, nunits).require ();
9754 7925 : gcc_assert (optab_handler (mov_optab, move_mode)
9755 : != CODE_FOR_nothing);
9756 : }
9757 : break;
9758 25 : case rep_prefix_8_byte:
9759 25 : move_mode = DImode;
9760 25 : break;
9761 13325 : case rep_prefix_4_byte:
9762 13325 : move_mode = SImode;
9763 13325 : break;
9764 6016 : case rep_prefix_1_byte:
9765 6016 : move_mode = QImode;
9766 6016 : break;
9767 : }
9768 27527 : size_needed = GET_MODE_SIZE (move_mode) * unroll_factor;
9769 27527 : epilogue_size_needed = size_needed;
9770 :
9771 : /* If we are going to call any library calls conditionally, make sure any
9772 : pending stack adjustment happen before the first conditional branch,
9773 : otherwise they will be emitted before the library call only and won't
9774 : happen from the other branches. */
9775 27527 : if (dynamic_check != -1)
9776 2 : do_pending_stack_adjust ();
9777 :
9778 27527 : desired_align = decide_alignment (align, alg, expected_size, move_mode);
9779 27527 : if (!TARGET_ALIGN_STRINGOPS || noalign)
9780 27272 : align = desired_align;
9781 :
9782 : /* Step 1: Prologue guard. */
9783 :
9784 : /* Alignment code needs count to be in register. */
9785 27527 : if (CONST_INT_P (count_exp) && desired_align > align)
9786 : {
9787 22 : if (INTVAL (count_exp) > desired_align
9788 22 : && INTVAL (count_exp) > size_needed)
9789 : {
9790 22 : align_bytes
9791 22 : = get_mem_align_offset (dst, desired_align * BITS_PER_UNIT);
9792 22 : if (align_bytes <= 0)
9793 : align_bytes = 0;
9794 : else
9795 0 : align_bytes = desired_align - align_bytes;
9796 : }
9797 0 : if (align_bytes == 0)
9798 44 : count_exp = force_reg (counter_mode (count_exp), count_exp);
9799 : }
9800 27527 : gcc_assert (desired_align >= 1 && align >= 1);
9801 :
9802 27527 : if (!single_insn_p)
9803 : {
9804 : /* Misaligned move sequences handle both prologue and epilogue
9805 : at once. Default code generation results in a smaller code
9806 : for large alignments and also avoids redundant job when sizes
9807 : are known precisely. */
9808 27520 : misaligned_prologue_used
9809 55040 : = (TARGET_MISALIGNED_MOVE_STRING_PRO_EPILOGUES
9810 27514 : && MAX (desired_align, epilogue_size_needed) <= 32
9811 19156 : && desired_align <= epilogue_size_needed
9812 32250 : && ((desired_align > align && !align_bytes)
9813 4713 : || (!count && epilogue_size_needed > 1)));
9814 :
9815 : /* Destination is aligned after the misaligned prologue. */
9816 27520 : aligned_dstmem = misaligned_prologue_used;
9817 :
9818 27520 : if (noalign
9819 27265 : && !misaligned_prologue_used
9820 27265 : && (!count
9821 21834 : || count > (unsigned HOST_WIDE_INT) epilogue_size_needed))
9822 : {
9823 : /* Also use misaligned prologue if count > epilogue size,
9824 : alignment isn't needed and destination isn't aligned.
9825 : Since alignment isn't needed, the destination after
9826 : prologue won't be aligned. */
9827 27215 : aligned_dstmem = (GET_MODE_ALIGNMENT (move_mode)
9828 27215 : <= MEM_ALIGN (dst));
9829 27215 : if (!aligned_dstmem)
9830 3989 : misaligned_prologue_used = true;
9831 : }
9832 : }
9833 :
9834 : /* Do the cheap promotion to allow better CSE across the
9835 : main loop and epilogue (ie one load of the big constant in the
9836 : front of all code.
9837 : For now the misaligned move sequences do not have fast path
9838 : without broadcasting. */
9839 27527 : if (issetmem
9840 13567 : && (alg == vector_loop
9841 8252 : || CONST_INT_P (val_exp)
9842 49 : || misaligned_prologue_used))
9843 : {
9844 8204 : if (alg == vector_loop)
9845 : {
9846 5315 : promoted_val = promote_duplicated_reg_to_size (val_exp,
9847 5315 : UNITS_PER_WORD,
9848 : desired_align, align);
9849 : /* Duplicate the promoted scalar value if not 0 nor -1. */
9850 5315 : vec_promoted_val
9851 5315 : = promote_duplicated_reg (move_mode,
9852 5315 : (val_exp == const0_rtx
9853 582 : || val_exp == constm1_rtx)
9854 : ? val_exp : promoted_val);
9855 : }
9856 : else
9857 : {
9858 8204 : promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
9859 : desired_align, align);
9860 : }
9861 : }
9862 : /* Misaligned move sequences handles both prologues and epilogues at once.
9863 : Default code generation results in smaller code for large alignments and
9864 : also avoids redundant job when sizes are known precisely. */
9865 27479 : if (misaligned_prologue_used)
9866 : {
9867 : /* Misaligned move prologue handled small blocks by itself. */
9868 4006 : expand_set_or_cpymem_prologue_epilogue_by_misaligned_moves
9869 4006 : (dst, src, &destreg, &srcreg,
9870 : move_mode, promoted_val, vec_promoted_val,
9871 : &count_exp,
9872 : &jump_around_label,
9873 4006 : desired_align < align
9874 0 : ? MAX (desired_align, epilogue_size_needed) : epilogue_size_needed,
9875 : desired_align, align, &min_size, dynamic_check, issetmem);
9876 4006 : if (!issetmem)
9877 2094 : src = change_address (src, BLKmode, srcreg);
9878 4006 : dst = change_address (dst, BLKmode, destreg);
9879 4006 : if (aligned_dstmem)
9880 17 : set_mem_align (dst, desired_align * BITS_PER_UNIT);
9881 4006 : epilogue_size_needed = 0;
9882 4006 : if (need_zero_guard
9883 3727 : && min_size < (unsigned HOST_WIDE_INT) size_needed)
9884 : {
9885 : /* It is possible that we copied enough so the main loop will not
9886 : execute. */
9887 645 : gcc_assert (size_needed > 1);
9888 645 : if (jump_around_label == NULL_RTX)
9889 0 : jump_around_label = gen_label_rtx ();
9890 1290 : emit_cmp_and_jump_insns (count_exp,
9891 : GEN_INT (size_needed),
9892 : LTU, 0, counter_mode (count_exp), 1, jump_around_label);
9893 645 : if (expected_size == -1
9894 0 : || expected_size < (desired_align - align) / 2 + size_needed)
9895 645 : predict_jump (REG_BR_PROB_BASE * 20 / 100);
9896 : else
9897 0 : predict_jump (REG_BR_PROB_BASE * 60 / 100);
9898 : }
9899 : }
9900 : /* Ensure that alignment prologue won't copy past end of block. */
9901 23521 : else if (size_needed > 1 || (desired_align > 1 && desired_align > align))
9902 : {
9903 17332 : epilogue_size_needed = MAX (size_needed - 1, desired_align - align);
9904 : /* Epilogue always copies COUNT_EXP & EPILOGUE_SIZE_NEEDED bytes.
9905 : Make sure it is power of 2. */
9906 17332 : epilogue_size_needed = 1 << (floor_log2 (epilogue_size_needed) + 1);
9907 :
9908 : /* To improve performance of small blocks, we jump around the VAL
9909 : promoting mode. This mean that if the promoted VAL is not constant,
9910 : we might not use it in the epilogue and have to use byte
9911 : loop variant. */
9912 17332 : if (issetmem && epilogue_size_needed > 2 && !promoted_val)
9913 17332 : force_loopy_epilogue = true;
9914 17332 : if ((count && count < (unsigned HOST_WIDE_INT) epilogue_size_needed)
9915 17322 : || max_size < (unsigned HOST_WIDE_INT) epilogue_size_needed)
9916 : {
9917 : /* If main algorithm works on QImode, no epilogue is needed.
9918 : For small sizes just don't align anything. */
9919 44 : if (size_needed == 1)
9920 0 : desired_align = align;
9921 : else
9922 44 : goto epilogue;
9923 : }
9924 17288 : else if (!count
9925 62 : && min_size < (unsigned HOST_WIDE_INT) epilogue_size_needed)
9926 : {
9927 62 : label = gen_label_rtx ();
9928 124 : emit_cmp_and_jump_insns (count_exp,
9929 : GEN_INT (epilogue_size_needed),
9930 : LTU, 0, counter_mode (count_exp), 1, label);
9931 62 : if (expected_size == -1 || expected_size < epilogue_size_needed)
9932 62 : predict_jump (REG_BR_PROB_BASE * 60 / 100);
9933 : else
9934 0 : predict_jump (REG_BR_PROB_BASE * 20 / 100);
9935 : }
9936 : }
9937 :
9938 : /* Emit code to decide on runtime whether library call or inline should be
9939 : used. */
9940 27483 : if (dynamic_check != -1)
9941 : {
9942 2 : if (!issetmem && CONST_INT_P (count_exp))
9943 : {
9944 1 : if (UINTVAL (count_exp) >= (unsigned HOST_WIDE_INT)dynamic_check)
9945 : {
9946 1 : emit_block_copy_via_libcall (dst, src, count_exp);
9947 1 : count_exp = const0_rtx;
9948 1 : goto epilogue;
9949 : }
9950 : }
9951 : else
9952 : {
9953 1 : rtx_code_label *hot_label = gen_label_rtx ();
9954 1 : if (jump_around_label == NULL_RTX)
9955 1 : jump_around_label = gen_label_rtx ();
9956 2 : emit_cmp_and_jump_insns (count_exp, GEN_INT (dynamic_check - 1),
9957 : LEU, 0, counter_mode (count_exp),
9958 : 1, hot_label);
9959 1 : predict_jump (REG_BR_PROB_BASE * 90 / 100);
9960 1 : if (issetmem)
9961 1 : set_storage_via_libcall (dst, count_exp, val_exp);
9962 : else
9963 0 : emit_block_copy_via_libcall (dst, src, count_exp);
9964 1 : emit_jump (jump_around_label);
9965 1 : emit_label (hot_label);
9966 : }
9967 : }
9968 :
9969 : /* Step 2: Alignment prologue. */
9970 : /* Do the expensive promotion once we branched off the small blocks. */
9971 27482 : if (issetmem && !promoted_val)
9972 48 : promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
9973 : desired_align, align);
9974 :
9975 27482 : if (desired_align > align && !misaligned_prologue_used)
9976 : {
9977 7 : if (align_bytes == 0)
9978 : {
9979 : /* Except for the first move in prologue, we no longer know
9980 : constant offset in aliasing info. It don't seems to worth
9981 : the pain to maintain it for the first move, so throw away
9982 : the info early. */
9983 7 : dst = change_address (dst, BLKmode, destreg);
9984 7 : if (!issetmem)
9985 5 : src = change_address (src, BLKmode, srcreg);
9986 7 : dst = expand_set_or_cpymem_prologue (dst, src, destreg, srcreg,
9987 : promoted_val, vec_promoted_val,
9988 : count_exp, align, desired_align,
9989 : issetmem);
9990 : /* At most desired_align - align bytes are copied. */
9991 7 : if (min_size < (unsigned)(desired_align - align))
9992 0 : min_size = 0;
9993 : else
9994 7 : min_size -= desired_align - align;
9995 : }
9996 : else
9997 : {
9998 : /* If we know how many bytes need to be stored before dst is
9999 : sufficiently aligned, maintain aliasing info accurately. */
10000 0 : dst = expand_set_or_cpymem_constant_prologue (dst, &src, destreg,
10001 : srcreg,
10002 : promoted_val,
10003 : vec_promoted_val,
10004 : desired_align,
10005 : align_bytes,
10006 : issetmem);
10007 :
10008 0 : count_exp = plus_constant (counter_mode (count_exp),
10009 0 : count_exp, -align_bytes);
10010 0 : count -= align_bytes;
10011 0 : min_size -= align_bytes;
10012 0 : max_size -= align_bytes;
10013 : }
10014 7 : if (need_zero_guard
10015 7 : && min_size < (unsigned HOST_WIDE_INT) size_needed
10016 1 : && (count < (unsigned HOST_WIDE_INT) size_needed
10017 0 : || (align_bytes == 0
10018 0 : && count < ((unsigned HOST_WIDE_INT) size_needed
10019 0 : + desired_align - align))))
10020 : {
10021 : /* It is possible that we copied enough so the main loop will not
10022 : execute. */
10023 1 : gcc_assert (size_needed > 1);
10024 1 : if (label == NULL_RTX)
10025 0 : label = gen_label_rtx ();
10026 2 : emit_cmp_and_jump_insns (count_exp,
10027 : GEN_INT (size_needed),
10028 : LTU, 0, counter_mode (count_exp), 1, label);
10029 1 : if (expected_size == -1
10030 0 : || expected_size < (desired_align - align) / 2 + size_needed)
10031 1 : predict_jump (REG_BR_PROB_BASE * 20 / 100);
10032 : else
10033 0 : predict_jump (REG_BR_PROB_BASE * 60 / 100);
10034 : }
10035 : }
10036 27482 : if (label && size_needed == 1)
10037 : {
10038 0 : emit_label (label);
10039 0 : LABEL_NUSES (label) = 1;
10040 0 : label = NULL;
10041 0 : epilogue_size_needed = 1;
10042 0 : if (issetmem)
10043 0 : promoted_val = val_exp;
10044 : }
10045 27482 : else if (label == NULL_RTX && !misaligned_prologue_used)
10046 23415 : epilogue_size_needed = size_needed;
10047 :
10048 : /* Step 3: Main loop. */
10049 :
10050 27482 : switch (alg)
10051 : {
10052 : case libcall:
10053 : case no_stringop:
10054 : case last_alg:
10055 : gcc_unreachable ();
10056 232 : case loop_1_byte:
10057 232 : case loop:
10058 232 : case unrolled_loop:
10059 232 : expand_set_or_cpymem_via_loop (dst, src, destreg, srcreg, promoted_val,
10060 : count_exp, move_mode, unroll_factor,
10061 : expected_size, issetmem);
10062 232 : break;
10063 7884 : case vector_loop:
10064 7884 : expand_set_or_cpymem_via_loop (dst, src, destreg, srcreg,
10065 : vec_promoted_val, count_exp, move_mode,
10066 : unroll_factor, expected_size, issetmem);
10067 7884 : break;
10068 19366 : case rep_prefix_8_byte:
10069 19366 : case rep_prefix_4_byte:
10070 19366 : case rep_prefix_1_byte:
10071 19366 : expand_set_or_cpymem_via_rep (dst, src, destreg, srcreg, promoted_val,
10072 : val_exp, count_exp, move_mode, issetmem);
10073 19366 : break;
10074 : }
10075 : /* Adjust properly the offset of src and dest memory for aliasing. */
10076 27482 : if (CONST_INT_P (count_exp))
10077 : {
10078 21892 : if (!issetmem)
10079 9395 : src = adjust_automodify_address_nv (src, BLKmode, srcreg,
10080 : (count / size_needed) * size_needed);
10081 21892 : dst = adjust_automodify_address_nv (dst, BLKmode, destreg,
10082 : (count / size_needed) * size_needed);
10083 : }
10084 : else
10085 : {
10086 5590 : if (!issetmem)
10087 4562 : src = change_address (src, BLKmode, srcreg);
10088 5590 : dst = change_address (dst, BLKmode, destreg);
10089 : }
10090 :
10091 : /* Step 4: Epilogue to copy the remaining bytes. */
10092 27527 : epilogue:
10093 27527 : if (label)
10094 : {
10095 : /* When the main loop is done, COUNT_EXP might hold original count,
10096 : while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
10097 : Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
10098 : bytes. Compensate if needed. */
10099 :
10100 62 : if (size_needed < epilogue_size_needed)
10101 : {
10102 0 : tmp = expand_simple_binop (counter_mode (count_exp), AND, count_exp,
10103 0 : GEN_INT (size_needed - 1), count_exp, 1,
10104 : OPTAB_DIRECT);
10105 0 : if (tmp != count_exp)
10106 0 : emit_move_insn (count_exp, tmp);
10107 : }
10108 62 : emit_label (label);
10109 62 : LABEL_NUSES (label) = 1;
10110 : }
10111 :
10112 27527 : if (count_exp != const0_rtx && epilogue_size_needed > 1)
10113 : {
10114 17332 : if (force_loopy_epilogue)
10115 0 : expand_setmem_epilogue_via_loop (dst, destreg, val_exp, count_exp,
10116 : epilogue_size_needed);
10117 : else
10118 : {
10119 17332 : if (issetmem)
10120 10043 : expand_setmem_epilogue (dst, destreg, promoted_val,
10121 : vec_promoted_val, count_exp,
10122 : epilogue_size_needed);
10123 : else
10124 7289 : expand_cpymem_epilogue (dst, src, destreg, srcreg, count_exp,
10125 : epilogue_size_needed);
10126 : }
10127 : }
10128 27527 : if (jump_around_label)
10129 648 : emit_label (jump_around_label);
10130 : return true;
10131 : }
10132 :
10133 : /* Fully unroll memmove of known size with up to 8 registers. */
10134 :
10135 : static bool
10136 1896 : ix86_expand_unroll_movmem (rtx dst, rtx src, rtx destreg, rtx srcreg,
10137 : unsigned HOST_WIDE_INT count,
10138 : machine_mode mode)
10139 : {
10140 : /* If 8 registers registers can cover all memory, load them into
10141 : registers and store them together to avoid possible address
10142 : overlap between source and destination. */
10143 1896 : unsigned HOST_WIDE_INT moves = count / GET_MODE_SIZE (mode);
10144 1896 : if (moves == 0)
10145 : {
10146 0 : mode = smallest_int_mode_for_size
10147 0 : (count * BITS_PER_UNIT).require ();
10148 0 : if (count == GET_MODE_SIZE (mode))
10149 : moves = 1;
10150 : else
10151 : {
10152 : /* Reduce the smallest move size by half so that MOVES == 1. */
10153 0 : mode = smallest_int_mode_for_size
10154 0 : (GET_MODE_BITSIZE (mode) / 2).require ();
10155 0 : moves = count / GET_MODE_SIZE (mode);
10156 0 : gcc_assert (moves == 1);
10157 : }
10158 : }
10159 1896 : else if (moves > 8)
10160 : return false;
10161 :
10162 1887 : unsigned int i;
10163 1887 : rtx tmp[9];
10164 :
10165 4342 : for (i = 0; i < moves; i++)
10166 2455 : tmp[i] = gen_reg_rtx (mode);
10167 :
10168 1887 : rtx srcmem = change_address (src, mode, srcreg);
10169 6229 : for (i = 0; i < moves; i++)
10170 : {
10171 2455 : emit_move_insn (tmp[i], srcmem);
10172 4910 : srcmem = offset_address (srcmem,
10173 2455 : GEN_INT (GET_MODE_SIZE (mode)),
10174 2455 : GET_MODE_SIZE (mode));
10175 : }
10176 :
10177 1887 : unsigned int epilogue_size = count & (GET_MODE_SIZE (mode) - 1);
10178 1887 : machine_mode epilogue_mode = VOIDmode;
10179 1887 : if (epilogue_size)
10180 : {
10181 : /* Handle the remaining bytes with overlapping move. */
10182 1714 : epilogue_mode = smallest_int_mode_for_size
10183 1714 : (epilogue_size * BITS_PER_UNIT).require ();
10184 1714 : tmp[8] = gen_reg_rtx (epilogue_mode);
10185 1714 : srcmem = adjust_address (srcmem, epilogue_mode, 0);
10186 1714 : srcmem = offset_address (srcmem, GEN_INT (epilogue_size), 1);
10187 3428 : srcmem = offset_address (srcmem,
10188 1714 : GEN_INT (-GET_MODE_SIZE (epilogue_mode)),
10189 1714 : GET_MODE_SIZE (epilogue_mode));
10190 1714 : emit_move_insn (tmp[8], srcmem);
10191 : }
10192 :
10193 1887 : rtx destmem = change_address (dst, mode, destreg);
10194 6229 : for (i = 0; i < moves; i++)
10195 : {
10196 2455 : emit_move_insn (destmem, tmp[i]);
10197 4910 : destmem = offset_address (destmem,
10198 2455 : GEN_INT (GET_MODE_SIZE (mode)),
10199 2455 : GET_MODE_SIZE (mode));
10200 : }
10201 :
10202 1887 : if (epilogue_size)
10203 : {
10204 : /* Use overlapping move. */
10205 1714 : destmem = adjust_address (destmem, epilogue_mode, 0);
10206 1714 : destmem = offset_address (destmem, GEN_INT (epilogue_size), 1);
10207 3428 : destmem = offset_address (destmem,
10208 1714 : GEN_INT (-GET_MODE_SIZE (epilogue_mode)),
10209 1714 : GET_MODE_SIZE (epilogue_mode));
10210 1714 : emit_move_insn (destmem, tmp[8]);
10211 : }
10212 :
10213 : return true;
10214 : }
10215 :
10216 : /* Value kind in MEMSET_VALS:
10217 :
10218 : memset_val_byte: The value rtx in QImode.
10219 : memset_val_word: The value rtx in word_mode.
10220 : memset_val_vector: The value rtx in QI vector mode.
10221 :
10222 : */
10223 : enum memset_val_kind
10224 : {
10225 : memset_val_byte = 0,
10226 : memset_val_word = 1,
10227 : memset_val_vector = 2,
10228 : memset_val_max = 3
10229 : };
10230 :
10231 : /* Return a value rtx in MODE for memset from MEMSET_VALS. */
10232 :
10233 : static rtx
10234 4318 : ix86_expand_memset_val (rtx *memset_vals, machine_mode mode)
10235 : {
10236 4318 : rtx byte_val = memset_vals[memset_val_byte];
10237 :
10238 4318 : if (mode == QImode)
10239 : return byte_val;
10240 4318 : else if (mode == word_mode)
10241 985 : return memset_vals[memset_val_word];
10242 :
10243 : /* All-zero/all-ones is a property of the original byte value, so
10244 : detect it once here rather than re-deriving it from each slot. */
10245 3333 : if (byte_val == const0_rtx)
10246 1254 : return CONST0_RTX (mode);
10247 2079 : if (byte_val == constm1_rtx)
10248 96 : return CONSTM1_RTX (mode);
10249 :
10250 1983 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
10251 : {
10252 688 : if (GET_MODE (memset_vals[memset_val_vector]) == mode)
10253 : return memset_vals[memset_val_vector];
10254 7 : return gen_rtx_SUBREG (mode, memset_vals[memset_val_vector], 0);
10255 : }
10256 :
10257 1295 : gcc_assert (mode == HImode || mode == SImode);
10258 1295 : return gen_rtx_SUBREG (mode, memset_vals[memset_val_word], 0);
10259 : }
10260 :
10261 : /* Expand memmove of size with MOVES * mode size and MOVES <= 4. If
10262 : FORWARD is true, copy forward. Otherwise copy backward. */
10263 :
10264 : static void
10265 2419 : ix86_expand_n_move_set_or_movmem (rtx destmem, rtx srcmem,
10266 : rtx *memset_vals, machine_mode mode,
10267 : unsigned int moves, bool forward)
10268 : {
10269 2419 : gcc_assert (moves <= 4);
10270 :
10271 2419 : unsigned int i;
10272 2419 : rtx tmp[8];
10273 :
10274 2419 : rtx step;
10275 2419 : if (forward)
10276 2542 : step = GEN_INT (GET_MODE_SIZE (mode));
10277 : else
10278 2296 : step = GEN_INT (-GET_MODE_SIZE (mode));
10279 :
10280 2419 : if (memset_vals)
10281 : {
10282 : /* Expand memset. */
10283 95 : rtx val = ix86_expand_memset_val (memset_vals, mode);
10284 570 : for (i = 0; i < moves; i++)
10285 380 : tmp[i] = val;
10286 : }
10287 : else
10288 : {
10289 : /* Expand memmove. */
10290 11620 : for (i = 0; i < moves; i++)
10291 9296 : tmp[i] = gen_reg_rtx (mode);
10292 :
10293 : /* Load MOVES. */
10294 9296 : for (i = 0; i < moves - 1; i++)
10295 : {
10296 6972 : emit_move_insn (tmp[i], srcmem);
10297 13944 : srcmem = offset_address (srcmem, step, GET_MODE_SIZE (mode));
10298 : }
10299 2324 : emit_move_insn (tmp[i], srcmem);
10300 : }
10301 :
10302 : /* Store MOVES. */
10303 9676 : for (i = 0; i < moves - 1; i++)
10304 : {
10305 7257 : emit_move_insn (destmem, tmp[i]);
10306 14514 : destmem = offset_address (destmem, step, GET_MODE_SIZE (mode));
10307 : }
10308 2419 : emit_move_insn (destmem, tmp[i]);
10309 2419 : }
10310 :
10311 : /* Load MOVES of mode size into REGS. If LAST is true, load the
10312 : last MOVES. Otherwise, load the first MOVES. */
10313 :
10314 : static void
10315 2324 : ix86_expand_load_movmem (rtx src, rtx srcreg, rtx count_exp,
10316 : machine_mode mode, unsigned int moves,
10317 : rtx regs[], bool last)
10318 : {
10319 2324 : unsigned int i;
10320 :
10321 11620 : for (i = 0; i < moves; i++)
10322 9296 : regs[i] = gen_reg_rtx (mode);
10323 :
10324 2324 : rtx srcmem = change_address (src, mode, srcreg);
10325 2324 : rtx step;
10326 2324 : if (last)
10327 : {
10328 1176 : srcmem = offset_address (srcmem, count_exp, 1);
10329 2352 : step = GEN_INT (-GET_MODE_SIZE (mode));
10330 2352 : srcmem = offset_address (srcmem, step, GET_MODE_SIZE (mode));
10331 : }
10332 : else
10333 2296 : step = GEN_INT (GET_MODE_SIZE (mode));
10334 :
10335 9296 : for (i = 0; i < moves - 1; i++)
10336 : {
10337 6972 : emit_move_insn (regs[i], srcmem);
10338 13944 : srcmem = offset_address (srcmem, step, GET_MODE_SIZE (mode));
10339 : }
10340 2324 : emit_move_insn (regs[i], srcmem);
10341 2324 : }
10342 :
10343 : /* Store MOVES of mode size into REGS. If LAST is true, store the
10344 : last MOVES. Otherwise, store the first MOVES. */
10345 :
10346 : static void
10347 2419 : ix86_expand_store_movmem (rtx dst, rtx destreg, rtx count_exp,
10348 : machine_mode mode, unsigned int moves,
10349 : rtx regs[], bool last)
10350 : {
10351 2419 : unsigned int i;
10352 :
10353 2419 : rtx destmem = change_address (dst, mode, destreg);
10354 2419 : rtx step;
10355 2419 : if (last)
10356 : {
10357 1271 : destmem = offset_address (destmem, count_exp, 1);
10358 2542 : step = GEN_INT (-GET_MODE_SIZE (mode));
10359 2542 : destmem = offset_address (destmem, step, GET_MODE_SIZE (mode));
10360 : }
10361 : else
10362 2296 : step = GEN_INT (GET_MODE_SIZE (mode));
10363 :
10364 9676 : for (i = 0; i < moves - 1; i++)
10365 : {
10366 7257 : emit_move_insn (destmem, regs[i]);
10367 14514 : destmem = offset_address (destmem, step, GET_MODE_SIZE (mode));
10368 : }
10369 2419 : emit_move_insn (destmem, regs[i]);
10370 2419 : }
10371 :
10372 : /* Expand memmove of size between (MOVES / 2) * mode size and
10373 : MOVES * mode size with overlapping load and store. MOVES is even.
10374 : MOVES >= 2 and MOVES <= 8. */
10375 :
10376 : static void
10377 44171 : ix86_expand_n_overlapping_move_set_or_movmem (rtx dst, rtx src,
10378 : rtx *memset_vals,
10379 : rtx destreg, rtx srcreg,
10380 : rtx count_exp,
10381 : machine_mode mode,
10382 : unsigned int moves)
10383 : {
10384 44171 : gcc_assert (moves >= 2 && moves <= 8 && (moves & 1) == 0);
10385 :
10386 44171 : unsigned int half_moves = moves / 2;
10387 44171 : unsigned int i, j;
10388 44171 : rtx tmp[8];
10389 :
10390 44171 : if (memset_vals)
10391 : {
10392 : /* Expand memset. */
10393 4128 : rtx val = ix86_expand_memset_val (memset_vals, mode);
10394 18500 : for (i = 0; i < moves; i++)
10395 10244 : tmp[i] = val;
10396 : }
10397 : else
10398 : {
10399 : /* Expand memmove. */
10400 134899 : for (i = 0; i < moves; i++)
10401 94856 : tmp[i] = gen_reg_rtx (mode);
10402 :
10403 40043 : rtx base_srcmem = change_address (src, mode, srcreg);
10404 :
10405 : /* Load the first half. */
10406 40043 : rtx srcmem = base_srcmem;
10407 87471 : for (i = 0; i < half_moves - 1; i++)
10408 : {
10409 7385 : emit_move_insn (tmp[i], srcmem);
10410 14770 : srcmem = offset_address (srcmem,
10411 7385 : GEN_INT (GET_MODE_SIZE (mode)),
10412 7385 : GET_MODE_SIZE (mode));
10413 : }
10414 40043 : emit_move_insn (tmp[i], srcmem);
10415 :
10416 : /* Load the second half. */
10417 40043 : srcmem = offset_address (base_srcmem, count_exp, 1);
10418 40043 : srcmem = offset_address (srcmem,
10419 40043 : GEN_INT (-GET_MODE_SIZE (mode)),
10420 40043 : GET_MODE_SIZE (mode));
10421 87471 : for (j = half_moves, i = 0; i < half_moves - 1; i++, j++)
10422 : {
10423 7385 : emit_move_insn (tmp[j], srcmem);
10424 14770 : srcmem = offset_address (srcmem,
10425 7385 : GEN_INT (-GET_MODE_SIZE (mode)),
10426 7385 : GET_MODE_SIZE (mode));
10427 : }
10428 40043 : emit_move_insn (tmp[j], srcmem);
10429 : }
10430 :
10431 44171 : rtx base_destmem = change_address (dst, mode, destreg);
10432 :
10433 : /* Store the first half. */
10434 44171 : rtx destmem = base_destmem;
10435 96721 : for (i = 0; i < half_moves - 1; i++)
10436 : {
10437 8379 : emit_move_insn (destmem, tmp[i]);
10438 16758 : destmem = offset_address (destmem,
10439 8379 : GEN_INT (GET_MODE_SIZE (mode)),
10440 8379 : GET_MODE_SIZE (mode));
10441 : }
10442 44171 : emit_move_insn (destmem, tmp[i]);
10443 :
10444 : /* Store the second half. */
10445 44171 : destmem = offset_address (base_destmem, count_exp, 1);
10446 88342 : destmem = offset_address (destmem, GEN_INT (-GET_MODE_SIZE (mode)),
10447 44171 : GET_MODE_SIZE (mode));
10448 96721 : for (j = half_moves, i = 0; i < half_moves - 1; i++, j++)
10449 : {
10450 8379 : emit_move_insn (destmem, tmp[j]);
10451 16758 : destmem = offset_address (destmem, GEN_INT (-GET_MODE_SIZE (mode)),
10452 8379 : GET_MODE_SIZE (mode));
10453 : }
10454 44171 : emit_move_insn (destmem, tmp[j]);
10455 44171 : }
10456 :
10457 : /* Expand memmove of size < mode size which is <= 64. */
10458 :
10459 : static void
10460 10912 : ix86_expand_less_move_set_or_movmem (rtx dst, rtx src, rtx *memset_vals,
10461 : rtx destreg, rtx srcreg,
10462 : rtx count_exp,
10463 : unsigned HOST_WIDE_INT min_size,
10464 : machine_mode mode,
10465 : rtx_code_label *done_label)
10466 : {
10467 10912 : bool skip = false;
10468 10912 : machine_mode count_mode = counter_mode (count_exp);
10469 :
10470 10912 : rtx_code_label *between_32_63_label
10471 10912 : = GET_MODE_SIZE (mode) > 32 ? gen_label_rtx () : nullptr;
10472 : /* Jump to BETWEEN_32_64_LABEL if size >= 32 and size < 64. */
10473 7 : if (between_32_63_label)
10474 : {
10475 7 : if (min_size && min_size >= 32)
10476 : {
10477 1 : emit_jump_insn (gen_jump (between_32_63_label));
10478 1 : emit_barrier ();
10479 1 : skip = true;
10480 : }
10481 : else
10482 6 : emit_cmp_and_jump_insns (count_exp, GEN_INT (32), GEU,
10483 : nullptr, count_mode, 1,
10484 : between_32_63_label);
10485 : }
10486 :
10487 7 : rtx_code_label *between_16_31_label
10488 10911 : = (!skip && GET_MODE_SIZE (mode) > 16) ? gen_label_rtx () : nullptr;
10489 : /* Jump to BETWEEN_16_31_LABEL if size >= 16 and size < 31. */
10490 21 : if (between_16_31_label)
10491 : {
10492 21 : if (min_size && min_size >= 16)
10493 : {
10494 2 : emit_jump_insn (gen_jump (between_16_31_label));
10495 2 : emit_barrier ();
10496 2 : skip = true;
10497 : }
10498 : else
10499 19 : emit_cmp_and_jump_insns (count_exp, GEN_INT (16), GEU,
10500 : nullptr, count_mode, 1,
10501 : between_16_31_label);
10502 : }
10503 :
10504 2 : rtx_code_label *between_8_15_label
10505 21819 : = (!skip && GET_MODE_SIZE (mode) > 8) ? gen_label_rtx () : nullptr;
10506 : /* Jump to BETWEEN_8_15_LABEL if size >= 8 and size < 15. */
10507 8665 : if (between_8_15_label)
10508 : {
10509 8665 : if (min_size && min_size >= 8)
10510 : {
10511 476 : emit_jump_insn (gen_jump (between_8_15_label));
10512 476 : emit_barrier ();
10513 476 : skip = true;
10514 : }
10515 : else
10516 8189 : emit_cmp_and_jump_insns (count_exp, GEN_INT (8), GEU,
10517 : nullptr, count_mode, 1,
10518 : between_8_15_label);
10519 : }
10520 :
10521 476 : rtx_code_label *between_4_7_label
10522 20869 : = (!skip && GET_MODE_SIZE (mode) > 4) ? gen_label_rtx () : nullptr;
10523 : /* Jump to BETWEEN_4_7_LABEL if size >= 4 and size < 7. */
10524 9517 : if (between_4_7_label)
10525 : {
10526 9517 : if (min_size && min_size >= 4)
10527 : {
10528 385 : emit_jump_insn (gen_jump (between_4_7_label));
10529 385 : emit_barrier ();
10530 385 : skip = true;
10531 : }
10532 : else
10533 9132 : emit_cmp_and_jump_insns (count_exp, GEN_INT (4), GEU,
10534 : nullptr, count_mode, 1,
10535 : between_4_7_label);
10536 : }
10537 :
10538 385 : rtx_code_label *between_2_3_label
10539 20575 : = (!skip && GET_MODE_SIZE (mode) > 2) ? gen_label_rtx () : nullptr;
10540 : /* Jump to BETWEEN_2_3_LABEL if size >= 2 and size < 3. */
10541 9822 : if (between_2_3_label)
10542 : {
10543 9822 : if (min_size && min_size >= 2)
10544 : {
10545 2076 : emit_jump_insn (gen_jump (between_2_3_label));
10546 2076 : emit_barrier ();
10547 2076 : skip = true;
10548 : }
10549 : else
10550 7746 : emit_cmp_and_jump_insns (count_exp, GEN_INT (1), GT,
10551 : nullptr, count_mode, 1,
10552 : between_2_3_label);
10553 : }
10554 :
10555 10912 : if (!skip)
10556 : {
10557 7972 : rtx_code_label *zero_label
10558 7972 : = min_size == 0 ? gen_label_rtx () : nullptr;
10559 : /* Skip if size == 0. */
10560 2716 : if (zero_label)
10561 2716 : emit_cmp_and_jump_insns (count_exp, GEN_INT (1), LT,
10562 : nullptr, count_mode, 1,
10563 : zero_label,
10564 : profile_probability::unlikely ());
10565 :
10566 : /* Move 1 byte. */
10567 7972 : rtx tmp0;
10568 : /* Use the value rtx in QImode for memset. */
10569 7972 : if (memset_vals)
10570 912 : tmp0 = memset_vals[memset_val_byte];
10571 : else
10572 : {
10573 7060 : tmp0 = gen_reg_rtx (QImode);
10574 7060 : rtx srcmem = change_address (src, QImode, srcreg);
10575 7060 : emit_move_insn (tmp0, srcmem);
10576 : }
10577 7972 : rtx destmem = change_address (dst, QImode, destreg);
10578 7972 : emit_move_insn (destmem, tmp0);
10579 :
10580 7972 : if (zero_label)
10581 2716 : emit_label (zero_label);
10582 :
10583 7972 : emit_jump_insn (gen_jump (done_label));
10584 7972 : emit_barrier ();
10585 : }
10586 :
10587 : /* For each size band, memset uses a QI-vector mode above a word so it
10588 : can broadcast the fill value, while memmove uses the same-size
10589 : scalar integer mode; at and below a word both use the scalar
10590 : integer mode. */
10591 10912 : struct {
10592 : rtx_code_label *label;
10593 : machine_mode set_mode;
10594 : machine_mode move_mode;
10595 10912 : } bands[] = {
10596 : { between_32_63_label, V32QImode, OImode },
10597 : { between_16_31_label, V16QImode, TImode },
10598 : { between_8_15_label, DImode, DImode },
10599 : { between_4_7_label, SImode, SImode },
10600 : { between_2_3_label, HImode, HImode },
10601 10912 : };
10602 :
10603 65472 : for (auto &band : bands)
10604 54560 : if (band.label)
10605 : {
10606 28032 : emit_label (band.label);
10607 28032 : machine_mode bmode = memset_vals ? band.set_mode : band.move_mode;
10608 28032 : ix86_expand_n_overlapping_move_set_or_movmem (dst, src,
10609 : memset_vals,
10610 : destreg, srcreg,
10611 : count_exp, bmode,
10612 : 2);
10613 28032 : emit_jump_insn (gen_jump (done_label));
10614 28032 : emit_barrier ();
10615 : }
10616 10912 : }
10617 :
10618 : /* Expand movmem with overlapping unaligned loads and stores:
10619 : 1. Load all sources into registers and store them together to avoid
10620 : possible address overlap between source and destination.
10621 : 2. For known size, first try to fully unroll with 8 registers.
10622 : 3. For size <= 2 * MOVE_MAX, load all sources into 2 registers first
10623 : and then store them together.
10624 : 4. For size > 2 * MOVE_MAX and size <= 4 * MOVE_MAX, load all sources
10625 : into 4 registers first and then store them together.
10626 : 5. For size > 4 * MOVE_MAX and size <= 8 * MOVE_MAX, load all sources
10627 : into 8 registers first and then store them together.
10628 : 6. For size > 8 * MOVE_MAX,
10629 : a. If address of destination > address of source, copy backward
10630 : with a 4 * MOVE_MAX loop with unaligned loads and stores. Load
10631 : the first 4 * MOVE_MAX into 4 registers before the loop and
10632 : store them after the loop to support overlapping addresses.
10633 : b. Otherwise, copy forward with a 4 * MOVE_MAX loop with unaligned
10634 : loads and stores. Load the last 4 * MOVE_MAX into 4 registers
10635 : before the loop and store them after the loop to support
10636 : overlapping addresses.
10637 : */
10638 :
10639 : bool
10640 171280 : ix86_expand_set_or_movmem (rtx operands[], bool iscpymem, bool issetmem)
10641 : {
10642 : /* Since there are much less registers available in 32-bit mode, don't
10643 : inline movmem in 32-bit mode. */
10644 171280 : if (!TARGET_64BIT || optimize_insn_for_size_p ())
10645 : return false;
10646 :
10647 137112 : rtx dst = operands[0];
10648 137112 : rtx src, memset_val_exp;
10649 137112 : if (issetmem)
10650 : {
10651 35856 : src = nullptr;
10652 35856 : memset_val_exp = operands[1];
10653 : }
10654 : else
10655 : {
10656 101256 : src = operands[1];
10657 101256 : memset_val_exp = nullptr;
10658 : }
10659 137112 : rtx count_exp = operands[2];
10660 137112 : rtx expected_size_exp = operands[5];
10661 137112 : rtx min_size_exp = operands[6];
10662 137112 : rtx max_size_exp = operands[7];
10663 137112 : rtx probable_max_size_exp = operands[8];
10664 137112 : unsigned HOST_WIDE_INT count = HOST_WIDE_INT_0U;
10665 137112 : HOST_WIDE_INT expected_size = HOST_WIDE_INT_M1U;
10666 137112 : unsigned HOST_WIDE_INT min_size = HOST_WIDE_INT_0U;
10667 137112 : unsigned HOST_WIDE_INT max_size = HOST_WIDE_INT_M1U;
10668 137112 : unsigned HOST_WIDE_INT probable_max_size = HOST_WIDE_INT_M1U;
10669 :
10670 137112 : if (CONST_INT_P (count_exp))
10671 : {
10672 54768 : min_size = max_size = probable_max_size = count = expected_size
10673 54768 : = INTVAL (count_exp);
10674 : /* When COUNT is 0, there is nothing to do. */
10675 54768 : if (!count)
10676 : return true;
10677 : }
10678 : else
10679 : {
10680 82344 : if (min_size_exp)
10681 82344 : min_size = INTVAL (min_size_exp);
10682 82344 : if (max_size_exp)
10683 62892 : max_size = INTVAL (max_size_exp);
10684 82344 : if (probable_max_size_exp)
10685 64839 : probable_max_size = INTVAL (probable_max_size_exp);
10686 82344 : if (CONST_INT_P (expected_size_exp))
10687 82344 : expected_size = INTVAL (expected_size_exp);
10688 :
10689 : /* NB: This assert may fail without the fixes for
10690 : https://gcc.gnu.org/bugzilla/show_bug.cgi?id=125977
10691 : */
10692 82344 : gcc_assert (min_size != max_size);
10693 : }
10694 :
10695 : /* Make sure we don't need to care about overflow later on. */
10696 54766 : if (count > (HOST_WIDE_INT_1U << 30))
10697 : return false;
10698 :
10699 136921 : addr_space_t dst_as = MEM_ADDR_SPACE (dst);
10700 136921 : addr_space_t src_as = (issetmem
10701 136921 : ? ADDR_SPACE_GENERIC
10702 101128 : : MEM_ADDR_SPACE (src));
10703 101128 : int dynamic_check;
10704 101128 : bool noalign;
10705 136921 : enum stringop_alg alg = decide_alg (count, expected_size, min_size,
10706 : probable_max_size, issetmem,
10707 : (issetmem
10708 35793 : && memset_val_exp == const0_rtx),
10709 : dst_as, src_as, &dynamic_check,
10710 : &noalign, false);
10711 136921 : if (alg == libcall)
10712 : return false;
10713 :
10714 : /* Expand memcpy and memset like memmove only for bounded size. */
10715 18376 : if (iscpymem || issetmem)
10716 : {
10717 14137 : unsigned HOST_WIDE_INT unbounded
10718 14137 : = GET_MODE_MASK (counter_mode (count_exp));
10719 14137 : if (count != 0 /* Fixed size. */
10720 14137 : || max_size == 0 /* Unbounded size. */
10721 9118 : || max_size == unbounded) /* Unbounded size. */
10722 : return false;
10723 : }
10724 :
10725 13045 : rtx destreg = ix86_copy_addr_to_reg (XEXP (dst, 0));
10726 13045 : rtx srcreg = (issetmem
10727 13045 : ? nullptr
10728 11903 : : ix86_copy_addr_to_reg (XEXP (src, 0)));
10729 :
10730 13045 : unsigned int move_max = MOVE_MAX;
10731 13045 : machine_mode mode = smallest_int_mode_for_size
10732 13045 : (move_max * BITS_PER_UNIT).require ();
10733 13045 : if (probable_max_size && probable_max_size < move_max)
10734 : {
10735 : /* Get a usable MOVE_MAX. */
10736 3867 : mode = smallest_int_mode_for_size
10737 3867 : (probable_max_size * BITS_PER_UNIT).require ();
10738 : /* Reduce MOVE_MAX by half so that MOVE_MAX can be used. */
10739 7734 : if (GET_MODE_SIZE (mode) > probable_max_size)
10740 3355 : mode = smallest_int_mode_for_size
10741 3355 : (GET_MODE_BITSIZE (mode) / 2).require ();
10742 7734 : move_max = GET_MODE_SIZE (mode);
10743 : }
10744 :
10745 : /* Try to fully unroll memmove of known size first. */
10746 13045 : if (count
10747 13045 : && ix86_expand_unroll_movmem (dst, src, destreg, srcreg, count,
10748 : mode))
10749 : return true;
10750 :
10751 11158 : rtx memset_vals[memset_val_max];
10752 11158 : rtx *memset_vals_p;
10753 11158 : if (issetmem)
10754 : {
10755 : /* Use vector mode if MODE size > word size. */
10756 1142 : unsigned int size = GET_MODE_SIZE (mode);
10757 1142 : poly_uint64 nunits;
10758 1142 : if (size > UNITS_PER_WORD)
10759 : {
10760 639 : nunits = size / GET_MODE_SIZE (QImode);
10761 639 : mode = mode_for_vector (QImode, nunits).require ();
10762 : }
10763 :
10764 : /* Populate MEMSET_VALS to expand memset. */
10765 1142 : rtx val_word;
10766 1142 : memset_vals[memset_val_byte] = memset_val_exp;
10767 1142 : if (memset_val_exp == const0_rtx || memset_val_exp == constm1_rtx)
10768 : val_word = memset_val_exp;
10769 : else
10770 661 : val_word = promote_duplicated_reg (word_mode, memset_val_exp);
10771 1142 : memset_vals[memset_val_word] = val_word;
10772 2284 : if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
10773 : {
10774 639 : if (memset_val_exp == const0_rtx)
10775 243 : memset_vals[memset_val_vector] = CONST0_RTX (mode);
10776 396 : else if (memset_val_exp == constm1_rtx)
10777 24 : memset_vals[memset_val_vector] = CONSTM1_RTX (mode);
10778 : else
10779 : {
10780 : /* Use the vector mode based on WORD_MODE to avoid extra
10781 : GPR moves. */
10782 372 : nunits = size / UNITS_PER_WORD;
10783 372 : machine_mode vector_mode
10784 372 : = mode_for_vector (word_mode, nunits).require ();
10785 372 : rtx vector = promote_duplicated_reg (vector_mode,
10786 : val_word);
10787 372 : memset_vals[memset_val_vector]
10788 372 : = convert_to_mode (mode, vector, 1);
10789 : }
10790 : }
10791 : else
10792 503 : memset_vals[memset_val_vector] = nullptr;
10793 1142 : memset_vals_p = memset_vals;
10794 : }
10795 : else
10796 : memset_vals_p = nullptr;
10797 :
10798 11158 : rtx_code_label *done_label = gen_label_rtx ();
10799 :
10800 11158 : rtx_code_label *less_vec_label = nullptr;
10801 11158 : if (min_size == 0 || min_size < move_max)
10802 10912 : less_vec_label = gen_label_rtx ();
10803 :
10804 11158 : machine_mode count_mode = counter_mode (count_exp);
10805 :
10806 : /* Jump to LESS_VEC_LABEL if size < MOVE_MAX. */
10807 11158 : if (less_vec_label)
10808 10912 : emit_cmp_and_jump_insns (count_exp, GEN_INT (move_max), LTU,
10809 : nullptr, count_mode, 1,
10810 : less_vec_label);
10811 :
10812 11158 : rtx_code_label *more_2x_vec_label = nullptr;
10813 11158 : if (probable_max_size == 0 || probable_max_size > 2 * move_max)
10814 3360 : more_2x_vec_label = gen_label_rtx ();
10815 :
10816 : /* Jump to MORE_2X_VEC_LABEL if size > 2 * MOVE_MAX. */
10817 3360 : if (more_2x_vec_label)
10818 3360 : emit_cmp_and_jump_insns (count_exp, GEN_INT (2 * move_max), GTU,
10819 : nullptr, count_mode, 1,
10820 : more_2x_vec_label);
10821 :
10822 11158 : if (max_size != 1 && (min_size == 0 || min_size <= 2 * move_max))
10823 : {
10824 : /* Max size != 1, size >= MOVE_MAX and size <= 2 * MOVE_MAX. */
10825 11114 : ix86_expand_n_overlapping_move_set_or_movmem (dst, src,
10826 : memset_vals_p,
10827 : destreg, srcreg,
10828 : count_exp, mode, 2);
10829 11114 : emit_jump_insn (gen_jump (done_label));
10830 11114 : emit_barrier ();
10831 : }
10832 :
10833 11158 : if (less_vec_label)
10834 : {
10835 : /* Size < MOVE_MAX. */
10836 10912 : emit_label (less_vec_label);
10837 10912 : ix86_expand_less_move_set_or_movmem (dst, src, memset_vals_p,
10838 : destreg, srcreg, count_exp,
10839 : min_size, mode, done_label);
10840 10912 : emit_jump_insn (gen_jump (done_label));
10841 10912 : emit_barrier ();
10842 : }
10843 :
10844 11158 : if (more_2x_vec_label)
10845 : {
10846 : /* Size > 2 * MOVE_MAX and destination may overlap with source. */
10847 3360 : emit_label (more_2x_vec_label);
10848 :
10849 3360 : rtx_code_label *more_8x_vec_label = nullptr;
10850 3360 : if (probable_max_size == 0 || probable_max_size > 8 * move_max)
10851 1271 : more_8x_vec_label = gen_label_rtx ();
10852 :
10853 : /* Jump to MORE_8X_VEC_LABEL if size > 8 * MOVE_MAX. */
10854 1271 : if (more_8x_vec_label)
10855 1271 : emit_cmp_and_jump_insns (count_exp, GEN_INT (8 * move_max), GTU,
10856 : nullptr, count_mode, 1,
10857 : more_8x_vec_label);
10858 :
10859 3360 : rtx_code_label *last_4x_vec_label = nullptr;
10860 3360 : if (min_size == 0 || min_size <= 4 * move_max)
10861 3348 : last_4x_vec_label = gen_label_rtx ();
10862 :
10863 : /* Jump to LAST_4X_VEC_LABEL if size <= 4 * MOVE_MAX. */
10864 3348 : if (last_4x_vec_label)
10865 3348 : emit_cmp_and_jump_insns (count_exp, GEN_INT (4 * move_max), LEU,
10866 : nullptr, count_mode, 1,
10867 : last_4x_vec_label);
10868 :
10869 3360 : if (probable_max_size == 0 || probable_max_size > 4 * move_max)
10870 : {
10871 : /* Size > 4 * MOVE_MAX and size <= 8 * MOVE_MAX. */
10872 1677 : ix86_expand_n_overlapping_move_set_or_movmem (dst, src,
10873 : memset_vals_p,
10874 : destreg, srcreg,
10875 : count_exp, mode,
10876 : 8);
10877 1677 : emit_jump_insn (gen_jump (done_label));
10878 1677 : emit_barrier ();
10879 : }
10880 :
10881 3360 : if (last_4x_vec_label)
10882 : {
10883 : /* Size > 2 * MOVE_MAX and size <= 4 * MOVE_MAX. */
10884 3348 : emit_label (last_4x_vec_label);
10885 3348 : ix86_expand_n_overlapping_move_set_or_movmem (dst, src,
10886 : memset_vals_p,
10887 : destreg, srcreg,
10888 : count_exp, mode,
10889 : 4);
10890 3348 : emit_jump_insn (gen_jump (done_label));
10891 3348 : emit_barrier ();
10892 : }
10893 :
10894 3360 : if (more_8x_vec_label)
10895 : {
10896 : /* Size > 8 * MOVE_MAX. */
10897 1271 : emit_label (more_8x_vec_label);
10898 :
10899 1271 : rtx loop_count = gen_reg_rtx (count_mode);
10900 1271 : emit_move_insn (loop_count, count_exp);
10901 :
10902 1271 : rtx_code_label *more_8x_vec_backward_label;
10903 1271 : rtx base_destreg;
10904 1271 : rtx srcmem;
10905 1271 : rtx regs[4];
10906 1271 : if (iscpymem || issetmem)
10907 : {
10908 : /* Always store forward for memcpy and memset. */
10909 123 : more_8x_vec_backward_label = nullptr;
10910 123 : if (iscpymem)
10911 : {
10912 : /* Load the last 4 * MOVE_MAX for memcpy. */
10913 28 : ix86_expand_load_movmem (src, srcreg, count_exp, mode,
10914 : ARRAY_SIZE (regs), regs,
10915 : true);
10916 28 : srcmem = change_address (src, mode, srcreg);
10917 : }
10918 : else
10919 : {
10920 : /* Fill REGS with MEMSET_VALS for memset. */
10921 95 : rtx val = ix86_expand_memset_val (memset_vals, mode);
10922 570 : for (unsigned int i = 0; i < 4; i++)
10923 380 : regs[i] = val;
10924 : srcmem = nullptr;
10925 : }
10926 123 : base_destreg = gen_reg_rtx (GET_MODE (destreg));
10927 123 : emit_move_insn (base_destreg, destreg);
10928 : }
10929 : else
10930 : {
10931 : /* Jump to MORE_8X_VEC_BACKWARD_LABEL if source address is
10932 : lower than destination address. */
10933 1148 : more_8x_vec_backward_label = gen_label_rtx ();
10934 1148 : emit_cmp_and_jump_insns (srcreg, destreg, LTU, nullptr,
10935 1148 : GET_MODE (destreg), 1,
10936 : more_8x_vec_backward_label);
10937 :
10938 : /* Skip if source == destination which is less common. */
10939 1148 : emit_cmp_and_jump_insns (srcreg, destreg, EQ, nullptr,
10940 1148 : GET_MODE (destreg), 1, done_label,
10941 : profile_probability::unlikely ());
10942 :
10943 1148 : base_destreg = gen_reg_rtx (GET_MODE (destreg));
10944 1148 : emit_move_insn (base_destreg, destreg);
10945 :
10946 : /* Load the last 4 * MOVE_MAX. */
10947 1148 : ix86_expand_load_movmem (src, srcreg, count_exp, mode,
10948 : ARRAY_SIZE (regs), regs, true);
10949 :
10950 1148 : srcmem = change_address (src, mode, srcreg);
10951 : }
10952 :
10953 1271 : rtx destmem = change_address (dst, mode, destreg);
10954 :
10955 : /* Copy forward with a 4 * MOVE_MAX loop. */
10956 1271 : rtx_code_label *loop_4x_vec_forward_label = gen_label_rtx ();
10957 1271 : emit_label (loop_4x_vec_forward_label);
10958 :
10959 1271 : ix86_expand_n_move_set_or_movmem (destmem, srcmem,
10960 : memset_vals_p, mode, 4,
10961 : true);
10962 :
10963 1271 : rtx tmp;
10964 1271 : rtx delta = GEN_INT (4 * MOVE_MAX);
10965 :
10966 : /* Decrement LOOP_COUNT by 4 * MOVE_MAX. */
10967 1271 : tmp = expand_simple_binop (GET_MODE (loop_count), MINUS,
10968 : loop_count, delta, nullptr, 1,
10969 : OPTAB_DIRECT);
10970 1271 : if (tmp != loop_count)
10971 1271 : emit_move_insn (loop_count, tmp);
10972 :
10973 : /* Increment DESTREG and SRCREG by 4 * MOVE_MAX. */
10974 1271 : tmp = expand_simple_binop (GET_MODE (destreg), PLUS,
10975 : destreg, delta, nullptr, 1,
10976 : OPTAB_DIRECT);
10977 1271 : if (tmp != destreg)
10978 1271 : emit_move_insn (destreg, tmp);
10979 1271 : if (!issetmem)
10980 : {
10981 1176 : tmp = expand_simple_binop (GET_MODE (srcreg), PLUS,
10982 : srcreg, delta, nullptr, 1,
10983 : OPTAB_DIRECT);
10984 1176 : if (tmp != srcreg)
10985 1176 : emit_move_insn (srcreg, tmp);
10986 : }
10987 :
10988 : /* Stop if LOOP_EXP <= 4 * MOVE_MAX. */
10989 1271 : emit_cmp_and_jump_insns (loop_count, delta, GTU, nullptr,
10990 1271 : GET_MODE (loop_count), 1,
10991 : loop_4x_vec_forward_label);
10992 :
10993 : /* Store the last 4 * MOVE_MAX. */
10994 1271 : ix86_expand_store_movmem (dst, base_destreg, count_exp, mode,
10995 : ARRAY_SIZE (regs), regs, true);
10996 :
10997 1271 : emit_jump_insn (gen_jump (done_label));
10998 1271 : emit_barrier ();
10999 :
11000 1271 : if (more_8x_vec_backward_label)
11001 : {
11002 : /* Copy backward with a 4 * MOVE_MAX loop. */
11003 1148 : emit_label (more_8x_vec_backward_label);
11004 :
11005 1148 : base_destreg = gen_reg_rtx (GET_MODE (destreg));
11006 1148 : emit_move_insn (base_destreg, destreg);
11007 :
11008 : /* Load the first 4 * MOVE_MAX. */
11009 1148 : ix86_expand_load_movmem (src, srcreg, count_exp, mode,
11010 : ARRAY_SIZE (regs), regs, false);
11011 :
11012 : /* Increment DESTREG and SRCREG by COUNT_EXP. */
11013 1148 : tmp = expand_simple_binop (GET_MODE (destreg), PLUS,
11014 : destreg, count_exp, nullptr, 1,
11015 : OPTAB_DIRECT);
11016 1148 : if (tmp != destreg)
11017 1148 : emit_move_insn (destreg, tmp);
11018 1148 : tmp = expand_simple_binop (GET_MODE (srcreg), PLUS, srcreg,
11019 : count_exp, nullptr, 1,
11020 : OPTAB_DIRECT);
11021 1148 : if (tmp != srcreg)
11022 1148 : emit_move_insn (srcreg, tmp);
11023 :
11024 1148 : srcmem = change_address (src, mode, srcreg);
11025 1148 : destmem = change_address (dst, mode, destreg);
11026 2296 : rtx step = GEN_INT (-GET_MODE_SIZE (mode));
11027 1148 : srcmem = offset_address (srcmem, step,
11028 1148 : GET_MODE_SIZE (mode));
11029 1148 : destmem = offset_address (destmem, step,
11030 1148 : GET_MODE_SIZE (mode));
11031 :
11032 1148 : rtx_code_label *loop_4x_vec_backward_label
11033 1148 : = gen_label_rtx ();
11034 1148 : emit_label (loop_4x_vec_backward_label);
11035 :
11036 1148 : ix86_expand_n_move_set_or_movmem (destmem, srcmem,
11037 : memset_vals_p, mode, 4,
11038 : false);
11039 :
11040 : /* Decrement LOOP_COUNT by 4 * MOVE_MAX. */
11041 1148 : tmp = expand_simple_binop (GET_MODE (loop_count), MINUS,
11042 : loop_count, delta, nullptr, 1,
11043 : OPTAB_DIRECT);
11044 1148 : if (tmp != loop_count)
11045 1148 : emit_move_insn (loop_count, tmp);
11046 :
11047 : /* Decrement DESTREG and SRCREG by 4 * MOVE_MAX. */
11048 1148 : tmp = expand_simple_binop (GET_MODE (destreg), MINUS,
11049 : destreg, delta, nullptr, 1,
11050 : OPTAB_DIRECT);
11051 1148 : if (tmp != destreg)
11052 1148 : emit_move_insn (destreg, tmp);
11053 1148 : tmp = expand_simple_binop (GET_MODE (srcreg), MINUS,
11054 : srcreg, delta, nullptr, 1,
11055 : OPTAB_DIRECT);
11056 1148 : if (tmp != srcreg)
11057 1148 : emit_move_insn (srcreg, tmp);
11058 :
11059 : /* Stop if LOOP_EXP <= 4 * MOVE_MAX. */
11060 1148 : emit_cmp_and_jump_insns (loop_count, delta, GTU, nullptr,
11061 1148 : GET_MODE (loop_count), 1,
11062 : loop_4x_vec_backward_label);
11063 :
11064 : /* Store the first 4 * MOVE_MAX. */
11065 1148 : ix86_expand_store_movmem (dst, base_destreg, count_exp,
11066 : mode, ARRAY_SIZE (regs), regs,
11067 : false);
11068 :
11069 1148 : emit_jump_insn (gen_jump (done_label));
11070 1148 : emit_barrier ();
11071 : }
11072 : }
11073 : }
11074 :
11075 11158 : emit_label (done_label);
11076 :
11077 11158 : return true;
11078 : }
11079 :
11080 : /* Expand cmpstrn or memcmp. */
11081 :
11082 : bool
11083 171578 : ix86_expand_cmpstrn_or_cmpmem (rtx result, rtx src1, rtx src2,
11084 : rtx length, rtx align, bool is_cmpstrn)
11085 : {
11086 : /* Expand strncmp and memcmp only with -minline-all-stringops since
11087 : "repz cmpsb" can be much slower than strncmp and memcmp functions
11088 : implemented with vector instructions, see
11089 :
11090 : https://gcc.gnu.org/bugzilla/show_bug.cgi?id=43052
11091 : */
11092 171578 : if (!TARGET_INLINE_ALL_STRINGOPS)
11093 : return false;
11094 :
11095 : /* Can't use this if the user has appropriated ecx, esi or edi. */
11096 5802 : if (fixed_regs[CX_REG] || fixed_regs[SI_REG] || fixed_regs[DI_REG])
11097 : return false;
11098 :
11099 5802 : if (is_cmpstrn)
11100 : {
11101 : /* For strncmp, length is the maximum length, which can be larger
11102 : than actual string lengths. We can expand the cmpstrn pattern
11103 : to "repz cmpsb" only if one of the strings is a constant so
11104 : that expand_builtin_strncmp() can write the length argument to
11105 : be the minimum of the const string length and the actual length
11106 : argument. Otherwise, "repz cmpsb" may pass the 0 byte. */
11107 69 : tree t1 = MEM_EXPR (src1);
11108 69 : tree t2 = MEM_EXPR (src2);
11109 138 : if (!((t1 && TREE_CODE (t1) == MEM_REF
11110 69 : && TREE_CODE (TREE_OPERAND (t1, 0)) == ADDR_EXPR
11111 0 : && (TREE_CODE (TREE_OPERAND (TREE_OPERAND (t1, 0), 0))
11112 : == STRING_CST))
11113 69 : || (t2 && TREE_CODE (t2) == MEM_REF
11114 69 : && TREE_CODE (TREE_OPERAND (t2, 0)) == ADDR_EXPR
11115 69 : && (TREE_CODE (TREE_OPERAND (TREE_OPERAND (t2, 0), 0))
11116 : == STRING_CST))))
11117 : return false;
11118 : }
11119 :
11120 5802 : rtx addr1 = copy_addr_to_reg (XEXP (src1, 0));
11121 5802 : rtx addr2 = copy_addr_to_reg (XEXP (src2, 0));
11122 5802 : if (addr1 != XEXP (src1, 0))
11123 5802 : src1 = replace_equiv_address_nv (src1, addr1);
11124 5802 : if (addr2 != XEXP (src2, 0))
11125 5802 : src2 = replace_equiv_address_nv (src2, addr2);
11126 :
11127 : /* NB: Make a copy of the data length to avoid changing the original
11128 : data length by cmpstrnqi patterns. */
11129 5802 : length = ix86_zero_extend_to_Pmode (length);
11130 8720 : rtx lengthreg = gen_reg_rtx (Pmode);
11131 5802 : emit_move_insn (lengthreg, length);
11132 :
11133 : /* If we are testing strict equality, we can use known alignment to
11134 : good advantage. This may be possible with combine, particularly
11135 : once cc0 is dead. */
11136 5802 : if (CONST_INT_P (length))
11137 : {
11138 0 : if (length == const0_rtx)
11139 : {
11140 0 : emit_move_insn (result, const0_rtx);
11141 0 : return true;
11142 : }
11143 0 : emit_insn (gen_cmpstrnqi_nz_1 (addr1, addr2, lengthreg, align,
11144 : src1, src2));
11145 : }
11146 : else
11147 : {
11148 8720 : emit_insn (gen_cmp_1 (Pmode, lengthreg, lengthreg));
11149 5802 : emit_insn (gen_cmpstrnqi_1 (addr1, addr2, lengthreg, align,
11150 : src1, src2));
11151 : }
11152 :
11153 5802 : rtx out = gen_lowpart (QImode, result);
11154 5802 : emit_insn (gen_cmpintqi (out));
11155 5802 : emit_move_insn (result, gen_rtx_SIGN_EXTEND (SImode, out));
11156 :
11157 5802 : return true;
11158 : }
11159 :
11160 : /* Expand the appropriate insns for doing strlen if not just doing
11161 : repnz; scasb
11162 :
11163 : out = result, initialized with the start address
11164 : align_rtx = alignment of the address.
11165 : scratch = scratch register, initialized with the startaddress when
11166 : not aligned, otherwise undefined
11167 :
11168 : This is just the body. It needs the initializations mentioned above and
11169 : some address computing at the end. These things are done in i386.md. */
11170 :
11171 : static void
11172 11 : ix86_expand_strlensi_unroll_1 (rtx out, rtx src, rtx align_rtx)
11173 : {
11174 11 : int align;
11175 11 : rtx tmp;
11176 11 : rtx_code_label *align_2_label = NULL;
11177 11 : rtx_code_label *align_3_label = NULL;
11178 11 : rtx_code_label *align_4_label = gen_label_rtx ();
11179 11 : rtx_code_label *end_0_label = gen_label_rtx ();
11180 11 : rtx mem;
11181 11 : rtx tmpreg = gen_reg_rtx (SImode);
11182 11 : rtx scratch = gen_reg_rtx (SImode);
11183 11 : rtx cmp;
11184 :
11185 11 : align = 0;
11186 11 : if (CONST_INT_P (align_rtx))
11187 11 : align = INTVAL (align_rtx);
11188 :
11189 : /* Loop to check 1..3 bytes for null to get an aligned pointer. */
11190 :
11191 : /* Is there a known alignment and is it less than 4? */
11192 11 : if (align < 4)
11193 : {
11194 15 : rtx scratch1 = gen_reg_rtx (Pmode);
11195 11 : emit_move_insn (scratch1, out);
11196 : /* Is there a known alignment and is it not 2? */
11197 11 : if (align != 2)
11198 : {
11199 11 : align_3_label = gen_label_rtx (); /* Label when aligned to 3-byte */
11200 11 : align_2_label = gen_label_rtx (); /* Label when aligned to 2-byte */
11201 :
11202 : /* Leave just the 3 lower bits. */
11203 15 : align_rtx = expand_binop (Pmode, and_optab, scratch1, GEN_INT (3),
11204 : NULL_RTX, 0, OPTAB_WIDEN);
11205 :
11206 15 : emit_cmp_and_jump_insns (align_rtx, const0_rtx, EQ, NULL,
11207 11 : Pmode, 1, align_4_label);
11208 15 : emit_cmp_and_jump_insns (align_rtx, const2_rtx, EQ, NULL,
11209 11 : Pmode, 1, align_2_label);
11210 15 : emit_cmp_and_jump_insns (align_rtx, const2_rtx, GTU, NULL,
11211 11 : Pmode, 1, align_3_label);
11212 : }
11213 : else
11214 : {
11215 : /* Since the alignment is 2, we have to check 2 or 0 bytes;
11216 : check if is aligned to 4 - byte. */
11217 :
11218 0 : align_rtx = expand_binop (Pmode, and_optab, scratch1, const2_rtx,
11219 : NULL_RTX, 0, OPTAB_WIDEN);
11220 :
11221 0 : emit_cmp_and_jump_insns (align_rtx, const0_rtx, EQ, NULL,
11222 0 : Pmode, 1, align_4_label);
11223 : }
11224 :
11225 11 : mem = change_address (src, QImode, out);
11226 :
11227 : /* Now compare the bytes. */
11228 :
11229 : /* Compare the first n unaligned byte on a byte per byte basis. */
11230 11 : emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL,
11231 : QImode, 1, end_0_label);
11232 :
11233 : /* Increment the address. */
11234 11 : emit_insn (gen_add2_insn (out, const1_rtx));
11235 :
11236 : /* Not needed with an alignment of 2 */
11237 11 : if (align != 2)
11238 : {
11239 11 : emit_label (align_2_label);
11240 :
11241 11 : emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1,
11242 : end_0_label);
11243 :
11244 11 : emit_insn (gen_add2_insn (out, const1_rtx));
11245 :
11246 11 : emit_label (align_3_label);
11247 : }
11248 :
11249 11 : emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1,
11250 : end_0_label);
11251 :
11252 11 : emit_insn (gen_add2_insn (out, const1_rtx));
11253 : }
11254 :
11255 : /* Generate loop to check 4 bytes at a time. It is not a good idea to
11256 : align this loop. It gives only huge programs, but does not help to
11257 : speed up. */
11258 11 : emit_label (align_4_label);
11259 :
11260 11 : mem = change_address (src, SImode, out);
11261 11 : emit_move_insn (scratch, mem);
11262 11 : emit_insn (gen_add2_insn (out, GEN_INT (4)));
11263 :
11264 : /* This formula yields a nonzero result iff one of the bytes is zero.
11265 : This saves three branches inside loop and many cycles. */
11266 :
11267 11 : emit_insn (gen_addsi3 (tmpreg, scratch, GEN_INT (-0x01010101)));
11268 11 : emit_insn (gen_one_cmplsi2 (scratch, scratch));
11269 11 : emit_insn (gen_andsi3 (tmpreg, tmpreg, scratch));
11270 11 : emit_insn (gen_andsi3 (tmpreg, tmpreg,
11271 : gen_int_mode (0x80808080, SImode)));
11272 11 : emit_cmp_and_jump_insns (tmpreg, const0_rtx, EQ, 0, SImode, 1,
11273 : align_4_label);
11274 :
11275 11 : if (TARGET_CMOVE)
11276 : {
11277 11 : rtx reg = gen_reg_rtx (SImode);
11278 15 : rtx reg2 = gen_reg_rtx (Pmode);
11279 11 : emit_move_insn (reg, tmpreg);
11280 11 : emit_insn (gen_lshrsi3 (reg, reg, GEN_INT (16)));
11281 :
11282 : /* If zero is not in the first two bytes, move two bytes forward. */
11283 11 : emit_insn (gen_testsi_ccno_1 (tmpreg, GEN_INT (0x8080)));
11284 11 : tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
11285 11 : tmp = gen_rtx_EQ (VOIDmode, tmp, const0_rtx);
11286 11 : emit_insn (gen_rtx_SET (tmpreg,
11287 : gen_rtx_IF_THEN_ELSE (SImode, tmp,
11288 : reg,
11289 : tmpreg)));
11290 : /* Emit lea manually to avoid clobbering of flags. */
11291 15 : emit_insn (gen_rtx_SET (reg2, plus_constant (Pmode, out, 2)));
11292 :
11293 11 : tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
11294 11 : tmp = gen_rtx_EQ (VOIDmode, tmp, const0_rtx);
11295 15 : emit_insn (gen_rtx_SET (out,
11296 : gen_rtx_IF_THEN_ELSE (Pmode, tmp,
11297 : reg2,
11298 : out)));
11299 11 : }
11300 : else
11301 : {
11302 0 : rtx_code_label *end_2_label = gen_label_rtx ();
11303 : /* Is zero in the first two bytes? */
11304 :
11305 0 : emit_insn (gen_testsi_ccno_1 (tmpreg, GEN_INT (0x8080)));
11306 0 : tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
11307 0 : tmp = gen_rtx_NE (VOIDmode, tmp, const0_rtx);
11308 0 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
11309 : gen_rtx_LABEL_REF (VOIDmode, end_2_label),
11310 : pc_rtx);
11311 0 : rtx_insn *jump = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
11312 0 : JUMP_LABEL (jump) = end_2_label;
11313 :
11314 : /* Not in the first two. Move two bytes forward. */
11315 0 : emit_insn (gen_lshrsi3 (tmpreg, tmpreg, GEN_INT (16)));
11316 0 : emit_insn (gen_add2_insn (out, const2_rtx));
11317 :
11318 0 : emit_label (end_2_label);
11319 :
11320 : }
11321 :
11322 : /* Avoid branch in fixing the byte. */
11323 11 : tmpreg = gen_lowpart (QImode, tmpreg);
11324 11 : emit_insn (gen_addqi3_cconly_overflow (tmpreg, tmpreg));
11325 11 : tmp = gen_rtx_REG (CCmode, FLAGS_REG);
11326 11 : cmp = gen_rtx_LTU (VOIDmode, tmp, const0_rtx);
11327 15 : emit_insn (gen_sub3_carry (Pmode, out, out, GEN_INT (3), tmp, cmp));
11328 :
11329 11 : emit_label (end_0_label);
11330 11 : }
11331 :
11332 : /* Expand strlen. */
11333 :
11334 : bool
11335 13648 : ix86_expand_strlen (rtx out, rtx src, rtx eoschar, rtx align)
11336 : {
11337 13648 : if (TARGET_UNROLL_STRLEN
11338 13648 : && TARGET_INLINE_ALL_STRINGOPS
11339 11 : && eoschar == const0_rtx
11340 11 : && optimize > 1)
11341 : {
11342 : /* The generic case of strlen expander is long. Avoid it's
11343 : expanding unless TARGET_INLINE_ALL_STRINGOPS. */
11344 15 : rtx addr = force_reg (Pmode, XEXP (src, 0));
11345 : /* Well it seems that some optimizer does not combine a call like
11346 : foo(strlen(bar), strlen(bar));
11347 : when the move and the subtraction is done here. It does calculate
11348 : the length just once when these instructions are done inside of
11349 : output_strlen_unroll(). But I think since &bar[strlen(bar)] is
11350 : often used and I use one fewer register for the lifetime of
11351 : output_strlen_unroll() this is better. */
11352 :
11353 11 : emit_move_insn (out, addr);
11354 :
11355 11 : ix86_expand_strlensi_unroll_1 (out, src, align);
11356 :
11357 : /* strlensi_unroll_1 returns the address of the zero at the end of
11358 : the string, like memchr(), so compute the length by subtracting
11359 : the start address. */
11360 11 : emit_insn (gen_sub2_insn (out, addr));
11361 11 : return true;
11362 : }
11363 : else
11364 : return false;
11365 : }
11366 :
11367 : /* For given symbol (function) construct code to compute address of it's PLT
11368 : entry in large x86-64 PIC model. */
11369 :
11370 : static rtx
11371 34 : construct_plt_address (rtx symbol)
11372 : {
11373 34 : rtx tmp, unspec;
11374 :
11375 34 : gcc_assert (SYMBOL_REF_P (symbol));
11376 34 : gcc_assert (ix86_cmodel == CM_LARGE_PIC && !TARGET_PECOFF);
11377 34 : gcc_assert (Pmode == DImode);
11378 :
11379 34 : tmp = gen_reg_rtx (Pmode);
11380 34 : unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, symbol), UNSPEC_PLTOFF);
11381 :
11382 34 : emit_move_insn (tmp, gen_rtx_CONST (Pmode, unspec));
11383 34 : emit_insn (gen_add2_insn (tmp, pic_offset_table_rtx));
11384 34 : return tmp;
11385 : }
11386 :
11387 : rtx_insn *
11388 6425601 : ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
11389 : rtx callarg2,
11390 : rtx pop, bool sibcall)
11391 : {
11392 6425601 : rtx vec[3];
11393 6425601 : rtx use = NULL, call;
11394 6425601 : unsigned int vec_len = 0;
11395 6425601 : tree fndecl;
11396 :
11397 6425601 : if (SYMBOL_REF_P (XEXP (fnaddr, 0)))
11398 : {
11399 6238568 : fndecl = SYMBOL_REF_DECL (XEXP (fnaddr, 0));
11400 6238568 : if (fndecl)
11401 : {
11402 5978590 : if (lookup_attribute ("interrupt",
11403 5978590 : TYPE_ATTRIBUTES (TREE_TYPE (fndecl))))
11404 1 : error ("interrupt service routine cannot be called directly");
11405 5978590 : if (fndecl == current_function_decl
11406 5978590 : && decl_binds_to_current_def_p (fndecl))
11407 11485 : cfun->machine->recursive_function = true;
11408 : }
11409 : }
11410 : else
11411 : fndecl = NULL_TREE;
11412 :
11413 6425601 : if (pop == const0_rtx)
11414 0 : pop = NULL;
11415 6425601 : gcc_assert (!TARGET_64BIT || !pop);
11416 :
11417 6425601 : rtx addr = XEXP (fnaddr, 0);
11418 6425601 : if (TARGET_MACHO && !TARGET_64BIT)
11419 : {
11420 : #if TARGET_MACHO
11421 : if (flag_pic && SYMBOL_REF_P (XEXP (fnaddr, 0)))
11422 : fnaddr = machopic_indirect_call_target (fnaddr);
11423 : #endif
11424 : }
11425 : else
11426 : {
11427 : /* Static functions and indirect calls don't need the pic register. Also,
11428 : check if PLT was explicitly avoided via no-plt or "noplt" attribute, making
11429 : it an indirect call. */
11430 6425601 : if (flag_pic
11431 729096 : && SYMBOL_REF_P (addr)
11432 7127016 : && ix86_call_use_plt_p (addr))
11433 : {
11434 539166 : if (flag_plt
11435 539166 : && (SYMBOL_REF_DECL (addr) == NULL_TREE
11436 539132 : || !lookup_attribute ("noplt",
11437 539132 : DECL_ATTRIBUTES (SYMBOL_REF_DECL (addr)))))
11438 : {
11439 539131 : if (!TARGET_64BIT
11440 360083 : || (ix86_cmodel == CM_LARGE_PIC
11441 : && DEFAULT_ABI != MS_ABI))
11442 : {
11443 537178 : use_reg (&use, gen_rtx_REG (Pmode,
11444 : REAL_PIC_OFFSET_TABLE_REGNUM));
11445 179082 : if (ix86_use_pseudo_pic_reg ())
11446 358130 : emit_move_insn (gen_rtx_REG (Pmode,
11447 179082 : REAL_PIC_OFFSET_TABLE_REGNUM),
11448 : pic_offset_table_rtx);
11449 : }
11450 : }
11451 35 : else if (!TARGET_PECOFF && !TARGET_MACHO)
11452 : {
11453 35 : if (TARGET_64BIT
11454 35 : && ix86_cmodel == CM_LARGE_PIC
11455 : && DEFAULT_ABI != MS_ABI)
11456 : {
11457 1 : fnaddr = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr),
11458 : UNSPEC_GOT);
11459 1 : fnaddr = gen_rtx_CONST (Pmode, fnaddr);
11460 1 : fnaddr = force_reg (Pmode, fnaddr);
11461 1 : fnaddr = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, fnaddr);
11462 : }
11463 34 : else if (TARGET_64BIT)
11464 : {
11465 38 : fnaddr = gen_rtx_UNSPEC (Pmode,
11466 : gen_rtvec (1, addr),
11467 : UNSPEC_GOTPCREL);
11468 38 : fnaddr = gen_rtx_CONST (Pmode, fnaddr);
11469 : }
11470 : else
11471 : {
11472 0 : fnaddr = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr),
11473 : UNSPEC_GOT);
11474 0 : fnaddr = gen_rtx_CONST (Pmode, fnaddr);
11475 0 : fnaddr = gen_rtx_PLUS (Pmode, pic_offset_table_rtx,
11476 : fnaddr);
11477 : }
11478 39 : fnaddr = gen_const_mem (Pmode, fnaddr);
11479 : /* Pmode may not be the same as word_mode for x32, which
11480 : doesn't support indirect branch via 32-bit memory slot.
11481 : Since x32 GOT slot is 64 bit with zero upper 32 bits,
11482 : indirect branch via x32 GOT slot is OK. */
11483 35 : if (GET_MODE (fnaddr) != word_mode)
11484 4 : fnaddr = gen_rtx_ZERO_EXTEND (word_mode, fnaddr);
11485 35 : fnaddr = gen_rtx_MEM (QImode, fnaddr);
11486 : }
11487 : }
11488 : }
11489 :
11490 : /* Skip setting up RAX register for -mskip-rax-setup when there are no
11491 : parameters passed in vector registers. */
11492 6425601 : if (TARGET_64BIT
11493 5584843 : && (INTVAL (callarg2) > 0
11494 5523505 : || (INTVAL (callarg2) == 0
11495 331967 : && (TARGET_SSE || !flag_skip_rax_setup))))
11496 : {
11497 393303 : rtx al = gen_rtx_REG (QImode, AX_REG);
11498 393303 : emit_move_insn (al, callarg2);
11499 393303 : use_reg (&use, al);
11500 : }
11501 :
11502 6425601 : if (ix86_cmodel == CM_LARGE_PIC
11503 : && !TARGET_PECOFF
11504 45 : && MEM_P (fnaddr)
11505 45 : && SYMBOL_REF_P (XEXP (fnaddr, 0))
11506 6425638 : && !local_symbolic_operand (XEXP (fnaddr, 0), VOIDmode))
11507 34 : fnaddr = gen_rtx_MEM (QImode, construct_plt_address (XEXP (fnaddr, 0)));
11508 : /* Since x32 GOT slot is 64 bit with zero upper 32 bits, indirect
11509 : branch via x32 GOT slot is OK. */
11510 6425567 : else if (TARGET_X32
11511 74 : && MEM_P (fnaddr)
11512 74 : && GET_CODE (XEXP (fnaddr, 0)) == ZERO_EXTEND
11513 8 : && GOT_memory_operand (XEXP (XEXP (fnaddr, 0), 0), Pmode)
11514 6425571 : && !TARGET_INDIRECT_BRANCH_REGISTER)
11515 : ;
11516 6425567 : else if (sibcall
11517 6425567 : ? !sibcall_insn_operand (XEXP (fnaddr, 0), word_mode)
11518 6297132 : : !call_insn_operand (XEXP (fnaddr, 0), word_mode))
11519 : {
11520 532 : fnaddr = convert_to_mode (word_mode, XEXP (fnaddr, 0), 1);
11521 532 : fnaddr = gen_rtx_MEM (QImode, copy_to_mode_reg (word_mode, fnaddr));
11522 : }
11523 :
11524 : /* PR100665: Hwasan may tag code pointer which is not supported by LAM,
11525 : mask off code pointers here.
11526 : TODO: also need to handle indirect jump. */
11527 6426613 : if (ix86_memtag_can_tag_addresses () && !fndecl
11528 6425625 : && sanitize_flags_p (SANITIZE_HWADDRESS))
11529 : {
11530 24 : rtx untagged_addr = ix86_memtag_untagged_pointer (XEXP (fnaddr, 0),
11531 : NULL_RTX);
11532 24 : fnaddr = gen_rtx_MEM (QImode, untagged_addr);
11533 : }
11534 :
11535 6425601 : call = gen_rtx_CALL (VOIDmode, fnaddr, callarg1);
11536 :
11537 6425601 : if (retval)
11538 2546061 : call = gen_rtx_SET (retval, call);
11539 6425601 : vec[vec_len++] = call;
11540 :
11541 6425601 : if (pop)
11542 : {
11543 450294 : pop = gen_rtx_PLUS (Pmode, stack_pointer_rtx, pop);
11544 225147 : pop = gen_rtx_SET (stack_pointer_rtx, pop);
11545 225147 : vec[vec_len++] = pop;
11546 : }
11547 :
11548 : /* Set here, but it may get cleared later. */
11549 5584843 : if (TARGET_64BIT_MS_ABI
11550 73446 : && (!callarg2 || INTVAL (callarg2) != -2)
11551 6491909 : && TARGET_CALL_MS2SYSV_XLOGUES)
11552 : {
11553 7046 : if (!TARGET_SSE)
11554 : ;
11555 :
11556 : /* Don't break hot-patched functions. */
11557 7046 : else if (ix86_function_ms_hook_prologue (current_function_decl))
11558 : ;
11559 :
11560 : /* TODO: Cases not yet examined. */
11561 7046 : else if (flag_split_stack)
11562 0 : warn_once_call_ms2sysv_xlogues ("-fsplit-stack");
11563 :
11564 : else
11565 : {
11566 7046 : gcc_assert (!reload_completed);
11567 7046 : cfun->machine->call_ms2sysv = true;
11568 : }
11569 : }
11570 :
11571 6425601 : if (TARGET_MACHO && TARGET_64BIT && !sibcall
11572 : && (!fndecl || TREE_PUBLIC (fndecl)
11573 : || (SYMBOL_REF_P (addr) && !SYMBOL_REF_LOCAL_P (addr)))
11574 : && (!fndecl || !lookup_attribute ("no_caller_saved_registers",
11575 : TYPE_ATTRIBUTES (TREE_TYPE (fndecl)))))
11576 : {
11577 : /* We allow public functions defined in a TU to bind locally for PIC
11578 : code (the default) on 64bit Mach-O.
11579 : If such functions are not inlined, we cannot tell at compile-time if
11580 : they will be called via the lazy symbol resolver (this can depend on
11581 : options given at link-time). Therefore, we must assume that the lazy
11582 : resolver could be used which clobbers R11 and R10.
11583 : User specification of "no caller saved regs" overides. */
11584 : clobber_reg (&use, gen_rtx_REG (DImode, R11_REG));
11585 : clobber_reg (&use, gen_rtx_REG (DImode, R10_REG));
11586 : }
11587 :
11588 6425601 : if (vec_len > 1)
11589 225147 : call = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (vec_len, vec));
11590 6425601 : rtx_insn *call_insn = emit_call_insn (call);
11591 6425601 : if (use)
11592 572384 : CALL_INSN_FUNCTION_USAGE (call_insn) = use;
11593 :
11594 6425601 : return call_insn;
11595 : }
11596 :
11597 : /* Split simple return with popping POPC bytes from stack to indirect
11598 : branch with stack adjustment . */
11599 :
11600 : void
11601 0 : ix86_split_simple_return_pop_internal (rtx popc)
11602 : {
11603 0 : struct machine_function *m = cfun->machine;
11604 0 : rtx ecx = gen_rtx_REG (SImode, CX_REG);
11605 0 : rtx_insn *insn;
11606 :
11607 : /* There is no "pascal" calling convention in any 64bit ABI. */
11608 0 : gcc_assert (!TARGET_64BIT);
11609 :
11610 0 : insn = emit_insn (gen_pop (ecx));
11611 0 : m->fs.cfa_offset -= UNITS_PER_WORD;
11612 0 : m->fs.sp_offset -= UNITS_PER_WORD;
11613 :
11614 0 : rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
11615 0 : x = gen_rtx_SET (stack_pointer_rtx, x);
11616 0 : add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
11617 0 : add_reg_note (insn, REG_CFA_REGISTER, gen_rtx_SET (ecx, pc_rtx));
11618 0 : RTX_FRAME_RELATED_P (insn) = 1;
11619 :
11620 0 : x = gen_rtx_PLUS (Pmode, stack_pointer_rtx, popc);
11621 0 : x = gen_rtx_SET (stack_pointer_rtx, x);
11622 0 : insn = emit_insn (x);
11623 0 : add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
11624 0 : RTX_FRAME_RELATED_P (insn) = 1;
11625 :
11626 : /* Now return address is in ECX. */
11627 0 : emit_jump_insn (gen_simple_return_indirect_internal (ecx));
11628 0 : }
11629 :
11630 : /* Errors in the source file can cause expand_expr to return const0_rtx
11631 : where we expect a vector. To avoid crashing, use one of the vector
11632 : clear instructions. */
11633 :
11634 : static rtx
11635 204065 : safe_vector_operand (rtx x, machine_mode mode)
11636 : {
11637 0 : if (x == const0_rtx)
11638 0 : x = CONST0_RTX (mode);
11639 24 : return x;
11640 : }
11641 :
11642 : /* Subroutine of ix86_expand_builtin to take care of binop insns. */
11643 :
11644 : static rtx
11645 8858 : ix86_expand_binop_builtin (enum insn_code icode, tree exp, rtx target)
11646 : {
11647 8858 : rtx pat;
11648 8858 : tree arg0 = CALL_EXPR_ARG (exp, 0);
11649 8858 : tree arg1 = CALL_EXPR_ARG (exp, 1);
11650 8858 : rtx op0 = expand_normal (arg0);
11651 8858 : rtx op1 = expand_normal (arg1);
11652 8858 : machine_mode tmode = insn_data[icode].operand[0].mode;
11653 8858 : machine_mode mode0 = insn_data[icode].operand[1].mode;
11654 8858 : machine_mode mode1 = insn_data[icode].operand[2].mode;
11655 :
11656 8858 : if (VECTOR_MODE_P (mode0))
11657 8847 : op0 = safe_vector_operand (op0, mode0);
11658 8858 : if (VECTOR_MODE_P (mode1))
11659 8711 : op1 = safe_vector_operand (op1, mode1);
11660 :
11661 2793 : if (optimize || !target
11662 2793 : || GET_MODE (target) != tmode
11663 11651 : || !insn_data[icode].operand[0].predicate (target, tmode))
11664 6118 : target = gen_reg_rtx (tmode);
11665 :
11666 8858 : if (GET_MODE (op1) == SImode && mode1 == TImode)
11667 : {
11668 0 : rtx x = gen_reg_rtx (V4SImode);
11669 0 : emit_insn (gen_sse2_loadd (x, op1));
11670 0 : op1 = gen_lowpart (TImode, x);
11671 : }
11672 :
11673 8858 : if (!insn_data[icode].operand[1].predicate (op0, mode0))
11674 1378 : op0 = copy_to_mode_reg (mode0, op0);
11675 8858 : if (!insn_data[icode].operand[2].predicate (op1, mode1))
11676 805 : op1 = copy_to_mode_reg (mode1, op1);
11677 :
11678 8858 : pat = GEN_FCN (icode) (target, op0, op1);
11679 8858 : if (! pat)
11680 : return 0;
11681 :
11682 8858 : emit_insn (pat);
11683 :
11684 8858 : return target;
11685 : }
11686 :
11687 : /* Subroutine of ix86_expand_builtin to take care of 2-4 argument insns. */
11688 :
11689 : static rtx
11690 1816 : ix86_expand_multi_arg_builtin (enum insn_code icode, tree exp, rtx target,
11691 : enum ix86_builtin_func_type m_type,
11692 : enum rtx_code sub_code)
11693 : {
11694 1816 : rtx pat;
11695 1816 : unsigned int i, nargs;
11696 1816 : bool comparison_p = false;
11697 1816 : bool tf_p = false;
11698 1816 : bool last_arg_constant = false;
11699 1816 : int num_memory = 0;
11700 1816 : rtx xops[4];
11701 :
11702 1816 : machine_mode tmode = insn_data[icode].operand[0].mode;
11703 :
11704 1816 : switch (m_type)
11705 : {
11706 : case MULTI_ARG_4_DF2_DI_I:
11707 : case MULTI_ARG_4_DF2_DI_I1:
11708 : case MULTI_ARG_4_SF2_SI_I:
11709 : case MULTI_ARG_4_SF2_SI_I1:
11710 : nargs = 4;
11711 : last_arg_constant = true;
11712 : break;
11713 :
11714 845 : case MULTI_ARG_3_SF:
11715 845 : case MULTI_ARG_3_DF:
11716 845 : case MULTI_ARG_3_SF2:
11717 845 : case MULTI_ARG_3_DF2:
11718 845 : case MULTI_ARG_3_DI:
11719 845 : case MULTI_ARG_3_SI:
11720 845 : case MULTI_ARG_3_SI_DI:
11721 845 : case MULTI_ARG_3_HI:
11722 845 : case MULTI_ARG_3_HI_SI:
11723 845 : case MULTI_ARG_3_QI:
11724 845 : case MULTI_ARG_3_DI2:
11725 845 : case MULTI_ARG_3_SI2:
11726 845 : case MULTI_ARG_3_HI2:
11727 845 : case MULTI_ARG_3_QI2:
11728 845 : nargs = 3;
11729 845 : break;
11730 :
11731 128 : case MULTI_ARG_2_SF:
11732 128 : case MULTI_ARG_2_DF:
11733 128 : case MULTI_ARG_2_DI:
11734 128 : case MULTI_ARG_2_SI:
11735 128 : case MULTI_ARG_2_HI:
11736 128 : case MULTI_ARG_2_QI:
11737 128 : nargs = 2;
11738 128 : break;
11739 :
11740 64 : case MULTI_ARG_2_DI_IMM:
11741 64 : case MULTI_ARG_2_SI_IMM:
11742 64 : case MULTI_ARG_2_HI_IMM:
11743 64 : case MULTI_ARG_2_QI_IMM:
11744 64 : nargs = 2;
11745 64 : last_arg_constant = true;
11746 64 : break;
11747 :
11748 187 : case MULTI_ARG_1_SF:
11749 187 : case MULTI_ARG_1_DF:
11750 187 : case MULTI_ARG_1_SF2:
11751 187 : case MULTI_ARG_1_DF2:
11752 187 : case MULTI_ARG_1_DI:
11753 187 : case MULTI_ARG_1_SI:
11754 187 : case MULTI_ARG_1_HI:
11755 187 : case MULTI_ARG_1_QI:
11756 187 : case MULTI_ARG_1_SI_DI:
11757 187 : case MULTI_ARG_1_HI_DI:
11758 187 : case MULTI_ARG_1_HI_SI:
11759 187 : case MULTI_ARG_1_QI_DI:
11760 187 : case MULTI_ARG_1_QI_SI:
11761 187 : case MULTI_ARG_1_QI_HI:
11762 187 : nargs = 1;
11763 187 : break;
11764 :
11765 384 : case MULTI_ARG_2_DI_CMP:
11766 384 : case MULTI_ARG_2_SI_CMP:
11767 384 : case MULTI_ARG_2_HI_CMP:
11768 384 : case MULTI_ARG_2_QI_CMP:
11769 384 : nargs = 2;
11770 384 : comparison_p = true;
11771 384 : break;
11772 :
11773 128 : case MULTI_ARG_2_SF_TF:
11774 128 : case MULTI_ARG_2_DF_TF:
11775 128 : case MULTI_ARG_2_DI_TF:
11776 128 : case MULTI_ARG_2_SI_TF:
11777 128 : case MULTI_ARG_2_HI_TF:
11778 128 : case MULTI_ARG_2_QI_TF:
11779 128 : nargs = 2;
11780 128 : tf_p = true;
11781 128 : break;
11782 :
11783 0 : default:
11784 0 : gcc_unreachable ();
11785 : }
11786 :
11787 628 : if (optimize || !target
11788 628 : || GET_MODE (target) != tmode
11789 2420 : || !insn_data[icode].operand[0].predicate (target, tmode))
11790 1212 : target = gen_reg_rtx (tmode);
11791 604 : else if (memory_operand (target, tmode))
11792 0 : num_memory++;
11793 :
11794 1816 : gcc_assert (nargs <= ARRAY_SIZE (xops));
11795 :
11796 6258 : for (i = 0; i < nargs; i++)
11797 : {
11798 4450 : tree arg = CALL_EXPR_ARG (exp, i);
11799 4450 : rtx op = expand_normal (arg);
11800 4450 : int adjust = (comparison_p) ? 1 : 0;
11801 4450 : machine_mode mode = insn_data[icode].operand[i+adjust+1].mode;
11802 :
11803 4450 : if (last_arg_constant && i == nargs - 1)
11804 : {
11805 144 : if (!insn_data[icode].operand[i + 1].predicate (op, mode))
11806 : {
11807 30 : enum insn_code new_icode = icode;
11808 30 : switch (icode)
11809 : {
11810 8 : case CODE_FOR_xop_vpermil2v2df3:
11811 8 : case CODE_FOR_xop_vpermil2v4sf3:
11812 8 : case CODE_FOR_xop_vpermil2v4df3:
11813 8 : case CODE_FOR_xop_vpermil2v8sf3:
11814 8 : error ("the last argument must be a 2-bit immediate");
11815 8 : return gen_reg_rtx (tmode);
11816 5 : case CODE_FOR_xop_rotlv2di3:
11817 5 : new_icode = CODE_FOR_rotlv2di3;
11818 5 : goto xop_rotl;
11819 5 : case CODE_FOR_xop_rotlv4si3:
11820 5 : new_icode = CODE_FOR_rotlv4si3;
11821 5 : goto xop_rotl;
11822 6 : case CODE_FOR_xop_rotlv8hi3:
11823 6 : new_icode = CODE_FOR_rotlv8hi3;
11824 6 : goto xop_rotl;
11825 : case CODE_FOR_xop_rotlv16qi3:
11826 : new_icode = CODE_FOR_rotlv16qi3;
11827 22 : xop_rotl:
11828 22 : if (CONST_INT_P (op))
11829 : {
11830 6 : int mask = GET_MODE_UNIT_BITSIZE (tmode) - 1;
11831 6 : op = GEN_INT (INTVAL (op) & mask);
11832 6 : gcc_checking_assert
11833 : (insn_data[icode].operand[i + 1].predicate (op, mode));
11834 : }
11835 : else
11836 : {
11837 16 : gcc_checking_assert
11838 : (nargs == 2
11839 : && insn_data[new_icode].operand[0].mode == tmode
11840 : && insn_data[new_icode].operand[1].mode == tmode
11841 : && insn_data[new_icode].operand[2].mode == mode
11842 : && insn_data[new_icode].operand[0].predicate
11843 : == insn_data[icode].operand[0].predicate
11844 : && insn_data[new_icode].operand[1].predicate
11845 : == insn_data[icode].operand[1].predicate);
11846 16 : icode = new_icode;
11847 16 : goto non_constant;
11848 : }
11849 : break;
11850 0 : default:
11851 0 : gcc_unreachable ();
11852 : }
11853 : }
11854 : }
11855 : else
11856 : {
11857 4306 : non_constant:
11858 4322 : if (VECTOR_MODE_P (mode))
11859 4306 : op = safe_vector_operand (op, mode);
11860 :
11861 : /* If we aren't optimizing, only allow one memory operand to be
11862 : generated. */
11863 4322 : if (memory_operand (op, mode))
11864 826 : num_memory++;
11865 :
11866 4322 : gcc_assert (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode);
11867 :
11868 4322 : if (optimize
11869 1506 : || !insn_data[icode].operand[i+adjust+1].predicate (op, mode)
11870 5750 : || num_memory > 1)
11871 3401 : op = force_reg (mode, op);
11872 : }
11873 :
11874 4442 : xops[i] = op;
11875 : }
11876 :
11877 1808 : switch (nargs)
11878 : {
11879 187 : case 1:
11880 187 : pat = GEN_FCN (icode) (target, xops[0]);
11881 187 : break;
11882 :
11883 704 : case 2:
11884 704 : if (tf_p)
11885 128 : pat = GEN_FCN (icode) (target, xops[0], xops[1],
11886 128 : GEN_INT ((int)sub_code));
11887 576 : else if (! comparison_p)
11888 192 : pat = GEN_FCN (icode) (target, xops[0], xops[1]);
11889 : else
11890 : {
11891 384 : rtx cmp_op = gen_rtx_fmt_ee (sub_code, GET_MODE (target),
11892 : xops[0], xops[1]);
11893 :
11894 384 : pat = GEN_FCN (icode) (target, cmp_op, xops[0], xops[1]);
11895 : }
11896 : break;
11897 :
11898 845 : case 3:
11899 845 : pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2]);
11900 845 : break;
11901 :
11902 72 : case 4:
11903 72 : pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2], xops[3]);
11904 72 : break;
11905 :
11906 : default:
11907 : gcc_unreachable ();
11908 : }
11909 :
11910 1808 : if (! pat)
11911 : return 0;
11912 :
11913 1808 : emit_insn (pat);
11914 1808 : return target;
11915 : }
11916 :
11917 : /* Subroutine of ix86_expand_args_builtin to take care of scalar unop
11918 : insns with vec_merge. */
11919 :
11920 : static rtx
11921 52 : ix86_expand_unop_vec_merge_builtin (enum insn_code icode, tree exp,
11922 : rtx target)
11923 : {
11924 52 : rtx pat;
11925 52 : tree arg0 = CALL_EXPR_ARG (exp, 0);
11926 52 : rtx op1, op0 = expand_normal (arg0);
11927 52 : machine_mode tmode = insn_data[icode].operand[0].mode;
11928 52 : machine_mode mode0 = insn_data[icode].operand[1].mode;
11929 :
11930 16 : if (optimize || !target
11931 16 : || GET_MODE (target) != tmode
11932 68 : || !insn_data[icode].operand[0].predicate (target, tmode))
11933 36 : target = gen_reg_rtx (tmode);
11934 :
11935 52 : if (VECTOR_MODE_P (mode0))
11936 52 : op0 = safe_vector_operand (op0, mode0);
11937 :
11938 36 : if ((optimize && !register_operand (op0, mode0))
11939 88 : || !insn_data[icode].operand[1].predicate (op0, mode0))
11940 0 : op0 = copy_to_mode_reg (mode0, op0);
11941 :
11942 52 : op1 = op0;
11943 52 : if (!insn_data[icode].operand[2].predicate (op1, mode0))
11944 16 : op1 = copy_to_mode_reg (mode0, op1);
11945 :
11946 52 : pat = GEN_FCN (icode) (target, op0, op1);
11947 52 : if (! pat)
11948 : return 0;
11949 52 : emit_insn (pat);
11950 52 : return target;
11951 : }
11952 :
11953 : /* Subroutine of ix86_expand_builtin to take care of comparison insns. */
11954 :
11955 : static rtx
11956 608 : ix86_expand_sse_compare (const struct builtin_description *d,
11957 : tree exp, rtx target, bool swap)
11958 : {
11959 608 : rtx pat;
11960 608 : tree arg0 = CALL_EXPR_ARG (exp, 0);
11961 608 : tree arg1 = CALL_EXPR_ARG (exp, 1);
11962 608 : rtx op0 = expand_normal (arg0);
11963 608 : rtx op1 = expand_normal (arg1);
11964 608 : rtx op2;
11965 608 : machine_mode tmode = insn_data[d->icode].operand[0].mode;
11966 608 : machine_mode mode0 = insn_data[d->icode].operand[1].mode;
11967 608 : machine_mode mode1 = insn_data[d->icode].operand[2].mode;
11968 608 : enum rtx_code comparison = d->comparison;
11969 :
11970 608 : if (VECTOR_MODE_P (mode0))
11971 608 : op0 = safe_vector_operand (op0, mode0);
11972 608 : if (VECTOR_MODE_P (mode1))
11973 608 : op1 = safe_vector_operand (op1, mode1);
11974 :
11975 : /* Swap operands if we have a comparison that isn't available in
11976 : hardware. */
11977 608 : if (swap)
11978 80 : std::swap (op0, op1);
11979 :
11980 202 : if (optimize || !target
11981 202 : || GET_MODE (target) != tmode
11982 810 : || !insn_data[d->icode].operand[0].predicate (target, tmode))
11983 406 : target = gen_reg_rtx (tmode);
11984 :
11985 406 : if ((optimize && !register_operand (op0, mode0))
11986 958 : || !insn_data[d->icode].operand[1].predicate (op0, mode0))
11987 258 : op0 = copy_to_mode_reg (mode0, op0);
11988 406 : if ((optimize && !register_operand (op1, mode1))
11989 958 : || !insn_data[d->icode].operand[2].predicate (op1, mode1))
11990 56 : op1 = copy_to_mode_reg (mode1, op1);
11991 :
11992 608 : op2 = gen_rtx_fmt_ee (comparison, mode0, op0, op1);
11993 608 : pat = GEN_FCN (d->icode) (target, op0, op1, op2);
11994 608 : if (! pat)
11995 : return 0;
11996 608 : emit_insn (pat);
11997 608 : return target;
11998 : }
11999 :
12000 : /* Subroutine of ix86_sse_comi and ix86_sse_comi_round to take care of
12001 : * ordered EQ or unordered NE, generate PF jump. */
12002 :
12003 : static rtx
12004 646 : ix86_ssecom_setcc (const enum rtx_code comparison,
12005 : bool check_unordered, machine_mode mode,
12006 : rtx set_dst, rtx target)
12007 : {
12008 :
12009 646 : rtx_code_label *label = NULL;
12010 :
12011 : /* NB: For ordered EQ or unordered NE, check ZF alone isn't sufficient
12012 : with NAN operands.
12013 : Under TARGET_AVX10_2, VCOMX/VUCOMX are generated instead of
12014 : COMI/UCOMI. VCOMX/VUCOMX will not set ZF for NAN operands. */
12015 646 : if (check_unordered)
12016 : {
12017 122 : gcc_assert (comparison == EQ || comparison == NE);
12018 :
12019 122 : rtx flag = gen_rtx_REG (CCFPmode, FLAGS_REG);
12020 122 : label = gen_label_rtx ();
12021 122 : rtx tmp = gen_rtx_fmt_ee (UNORDERED, VOIDmode, flag, const0_rtx);
12022 122 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
12023 : gen_rtx_LABEL_REF (VOIDmode, label),
12024 : pc_rtx);
12025 122 : emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
12026 : }
12027 :
12028 : /* NB: Set CCFPmode and check a different CCmode which is in subset
12029 : of CCFPmode. */
12030 646 : if (GET_MODE (set_dst) != mode)
12031 : {
12032 200 : gcc_assert (mode == CCAmode || mode == CCCmode
12033 : || mode == CCOmode || mode == CCPmode
12034 : || mode == CCSmode || mode == CCZmode);
12035 200 : set_dst = gen_rtx_REG (mode, FLAGS_REG);
12036 : }
12037 :
12038 646 : emit_insn (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
12039 : gen_rtx_fmt_ee (comparison, QImode,
12040 : set_dst,
12041 : const0_rtx)));
12042 :
12043 646 : if (label)
12044 122 : emit_label (label);
12045 :
12046 646 : return SUBREG_REG (target);
12047 : }
12048 :
12049 : /* Subroutine of ix86_expand_builtin to take care of comi insns. */
12050 :
12051 : static rtx
12052 547 : ix86_expand_sse_comi (const struct builtin_description *d, tree exp,
12053 : rtx target, bool comx_ok)
12054 : {
12055 547 : rtx pat, set_dst;
12056 547 : tree arg0 = CALL_EXPR_ARG (exp, 0);
12057 547 : tree arg1 = CALL_EXPR_ARG (exp, 1);
12058 547 : rtx op0 = expand_normal (arg0);
12059 547 : rtx op1 = expand_normal (arg1);
12060 547 : enum insn_code icode = d->icode;
12061 547 : const struct insn_data_d *insn_p = &insn_data[icode];
12062 547 : machine_mode mode0 = insn_p->operand[0].mode;
12063 547 : machine_mode mode1 = insn_p->operand[1].mode;
12064 :
12065 547 : if (VECTOR_MODE_P (mode0))
12066 547 : op0 = safe_vector_operand (op0, mode0);
12067 547 : if (VECTOR_MODE_P (mode1))
12068 547 : op1 = safe_vector_operand (op1, mode1);
12069 :
12070 547 : enum rtx_code comparison = d->comparison;
12071 547 : rtx const_val = const0_rtx;
12072 :
12073 547 : bool check_unordered = false;
12074 547 : machine_mode mode = CCFPmode;
12075 547 : switch (comparison)
12076 : {
12077 194 : case LE: /* -> GE */
12078 194 : case LT: /* -> GT */
12079 194 : std::swap (op0, op1);
12080 194 : comparison = swap_condition (comparison);
12081 : /* FALLTHRU */
12082 : case GT:
12083 : case GE:
12084 : break;
12085 73 : case EQ:
12086 73 : if (!TARGET_AVX10_2 || !comx_ok)
12087 45 : check_unordered = true;
12088 : mode = CCZmode;
12089 : break;
12090 96 : case NE:
12091 96 : if (!TARGET_AVX10_2 || !comx_ok)
12092 68 : check_unordered = true;
12093 96 : mode = CCZmode;
12094 96 : const_val = const1_rtx;
12095 96 : break;
12096 0 : default:
12097 0 : gcc_unreachable ();
12098 : }
12099 :
12100 547 : target = gen_reg_rtx (SImode);
12101 547 : emit_move_insn (target, const_val);
12102 547 : target = gen_rtx_SUBREG (QImode, target, 0);
12103 :
12104 426 : if ((optimize && !register_operand (op0, mode0))
12105 925 : || !insn_p->operand[0].predicate (op0, mode0))
12106 169 : op0 = copy_to_mode_reg (mode0, op0);
12107 426 : if ((optimize && !register_operand (op1, mode1))
12108 924 : || !insn_p->operand[1].predicate (op1, mode1))
12109 49 : op1 = copy_to_mode_reg (mode1, op1);
12110 :
12111 547 : if ((comparison == EQ || comparison == NE)
12112 169 : && TARGET_AVX10_2 && comx_ok)
12113 : {
12114 56 : switch (icode)
12115 : {
12116 : case CODE_FOR_sse_comi:
12117 : icode = CODE_FOR_avx10_2_comxsf;
12118 : break;
12119 14 : case CODE_FOR_sse_ucomi:
12120 14 : icode = CODE_FOR_avx10_2_ucomxsf;
12121 14 : break;
12122 14 : case CODE_FOR_sse2_comi:
12123 14 : icode = CODE_FOR_avx10_2_comxdf;
12124 14 : break;
12125 14 : case CODE_FOR_sse2_ucomi:
12126 14 : icode = CODE_FOR_avx10_2_ucomxdf;
12127 14 : break;
12128 :
12129 0 : default:
12130 0 : gcc_unreachable ();
12131 : }
12132 : }
12133 547 : pat = GEN_FCN (icode) (op0, op1);
12134 547 : if (! pat)
12135 : return 0;
12136 :
12137 547 : set_dst = SET_DEST (pat);
12138 547 : emit_insn (pat);
12139 547 : return ix86_ssecom_setcc (comparison, check_unordered, mode,
12140 547 : set_dst, target);
12141 : }
12142 :
12143 : /* Subroutines of ix86_expand_args_builtin to take care of round insns. */
12144 :
12145 : static rtx
12146 0 : ix86_expand_sse_round (const struct builtin_description *d, tree exp,
12147 : rtx target)
12148 : {
12149 0 : rtx pat;
12150 0 : tree arg0 = CALL_EXPR_ARG (exp, 0);
12151 0 : rtx op1, op0 = expand_normal (arg0);
12152 0 : machine_mode tmode = insn_data[d->icode].operand[0].mode;
12153 0 : machine_mode mode0 = insn_data[d->icode].operand[1].mode;
12154 :
12155 0 : if (optimize || target == 0
12156 0 : || GET_MODE (target) != tmode
12157 0 : || !insn_data[d->icode].operand[0].predicate (target, tmode))
12158 0 : target = gen_reg_rtx (tmode);
12159 :
12160 0 : if (VECTOR_MODE_P (mode0))
12161 0 : op0 = safe_vector_operand (op0, mode0);
12162 :
12163 0 : if ((optimize && !register_operand (op0, mode0))
12164 0 : || !insn_data[d->icode].operand[0].predicate (op0, mode0))
12165 0 : op0 = copy_to_mode_reg (mode0, op0);
12166 :
12167 0 : op1 = GEN_INT (d->comparison);
12168 :
12169 0 : pat = GEN_FCN (d->icode) (target, op0, op1);
12170 0 : if (! pat)
12171 : return 0;
12172 0 : emit_insn (pat);
12173 0 : return target;
12174 : }
12175 :
12176 : static rtx
12177 12 : ix86_expand_sse_round_vec_pack_sfix (const struct builtin_description *d,
12178 : tree exp, rtx target)
12179 : {
12180 12 : rtx pat;
12181 12 : tree arg0 = CALL_EXPR_ARG (exp, 0);
12182 12 : tree arg1 = CALL_EXPR_ARG (exp, 1);
12183 12 : rtx op0 = expand_normal (arg0);
12184 12 : rtx op1 = expand_normal (arg1);
12185 12 : rtx op2;
12186 12 : machine_mode tmode = insn_data[d->icode].operand[0].mode;
12187 12 : machine_mode mode0 = insn_data[d->icode].operand[1].mode;
12188 12 : machine_mode mode1 = insn_data[d->icode].operand[2].mode;
12189 :
12190 0 : if (optimize || target == 0
12191 0 : || GET_MODE (target) != tmode
12192 12 : || !insn_data[d->icode].operand[0].predicate (target, tmode))
12193 12 : target = gen_reg_rtx (tmode);
12194 :
12195 12 : op0 = safe_vector_operand (op0, mode0);
12196 12 : op1 = safe_vector_operand (op1, mode1);
12197 :
12198 12 : if ((optimize && !register_operand (op0, mode0))
12199 12 : || !insn_data[d->icode].operand[0].predicate (op0, mode0))
12200 12 : op0 = copy_to_mode_reg (mode0, op0);
12201 12 : if ((optimize && !register_operand (op1, mode1))
12202 12 : || !insn_data[d->icode].operand[1].predicate (op1, mode1))
12203 12 : op1 = copy_to_mode_reg (mode1, op1);
12204 :
12205 12 : op2 = GEN_INT (d->comparison);
12206 :
12207 12 : pat = GEN_FCN (d->icode) (target, op0, op1, op2);
12208 12 : if (! pat)
12209 : return 0;
12210 12 : emit_insn (pat);
12211 12 : return target;
12212 : }
12213 :
12214 : /* Subroutine of ix86_expand_builtin to take care of ptest insns. */
12215 :
12216 : static rtx
12217 239 : ix86_expand_sse_ptest (const struct builtin_description *d, tree exp,
12218 : rtx target)
12219 : {
12220 239 : rtx pat;
12221 239 : tree arg0 = CALL_EXPR_ARG (exp, 0);
12222 239 : tree arg1 = CALL_EXPR_ARG (exp, 1);
12223 239 : rtx op0 = expand_normal (arg0);
12224 239 : rtx op1 = expand_normal (arg1);
12225 239 : machine_mode mode0 = insn_data[d->icode].operand[0].mode;
12226 239 : machine_mode mode1 = insn_data[d->icode].operand[1].mode;
12227 239 : enum rtx_code comparison = d->comparison;
12228 239 : rtx result = NULL_RTX;
12229 :
12230 239 : if (VECTOR_MODE_P (mode0))
12231 239 : op0 = safe_vector_operand (op0, mode0);
12232 239 : if (VECTOR_MODE_P (mode1))
12233 239 : op1 = safe_vector_operand (op1, mode1);
12234 :
12235 239 : switch (d->code)
12236 : {
12237 49 : case IX86_BUILTIN_PTESTZ:
12238 49 : case IX86_BUILTIN_PTESTZ256:
12239 : // Returns (OP0 & OP1) == 0
12240 49 : if (rtx_equal_p (op0, CONST0_RTX (mode0))
12241 49 : || rtx_equal_p (op1, CONST0_RTX (mode1)))
12242 2 : result = const1_rtx;
12243 47 : else if (rtx_equal_p (op0, CONSTM1_RTX (mode0)))
12244 : {
12245 1 : op1 = force_reg (mode1, op1);
12246 1 : op0 = op1;
12247 : }
12248 46 : else if (rtx_equal_p (op1, CONSTM1_RTX (mode1)))
12249 : {
12250 1 : op0 = force_reg (mode0, op0);
12251 1 : op1 = op0;
12252 : }
12253 45 : else if (MEM_P (op0) && !MEM_P (op1))
12254 : std::swap (op0, op1);
12255 : break;
12256 :
12257 31 : case IX86_BUILTIN_PTESTC:
12258 31 : case IX86_BUILTIN_PTESTC256:
12259 : // Returns (~OP0 & OP1) == 0
12260 31 : if (rtx_equal_p (op0, CONSTM1_RTX (mode0))
12261 31 : || rtx_equal_p (op1, CONST0_RTX (mode1))
12262 62 : || rtx_equal_p (op0, op1))
12263 2 : result = const1_rtx;
12264 : break;
12265 :
12266 27 : case IX86_BUILTIN_PTESTNZC:
12267 27 : case IX86_BUILTIN_PTESTNZC256:
12268 : // Returns ((OP0 && OP1) != 0) && ((~OP0 && OP1) != 0)
12269 27 : if (rtx_equal_p (op0, CONST0_RTX (mode0))
12270 26 : || rtx_equal_p (op0, CONSTM1_RTX (mode0))
12271 26 : || rtx_equal_p (op1, CONST0_RTX (mode1))
12272 53 : || rtx_equal_p (op0, op1))
12273 1 : result = const0_rtx;
12274 : break;
12275 :
12276 : default:
12277 : break;
12278 : }
12279 :
12280 167 : if ((optimize && !register_operand (op0, mode0))
12281 210 : || !insn_data[d->icode].operand[0].predicate (op0, mode0)
12282 377 : || result)
12283 104 : op0 = copy_to_mode_reg (mode0, op0);
12284 167 : if ((optimize && !register_operand (op1, mode1))
12285 211 : || !insn_data[d->icode].operand[1].predicate (op1, mode1)
12286 450 : || result)
12287 31 : op1 = copy_to_mode_reg (mode1, op1);
12288 :
12289 239 : if (result)
12290 : {
12291 5 : if (!target)
12292 0 : target = gen_reg_rtx (SImode);
12293 5 : emit_move_insn (target, result);
12294 5 : return target;
12295 : }
12296 :
12297 234 : target = gen_reg_rtx (SImode);
12298 234 : emit_move_insn (target, const0_rtx);
12299 234 : target = gen_rtx_SUBREG (QImode, target, 0);
12300 :
12301 234 : pat = GEN_FCN (d->icode) (op0, op1);
12302 234 : if (! pat)
12303 : return 0;
12304 234 : emit_insn (pat);
12305 234 : emit_insn (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
12306 : gen_rtx_fmt_ee (comparison, QImode,
12307 : SET_DEST (pat),
12308 : const0_rtx)));
12309 :
12310 234 : return SUBREG_REG (target);
12311 : }
12312 :
12313 : /* Subroutine of ix86_expand_builtin to take care of pcmpestr[im] insns. */
12314 :
12315 : static rtx
12316 216 : ix86_expand_sse_pcmpestr (const struct builtin_description *d,
12317 : tree exp, rtx target)
12318 : {
12319 216 : rtx pat;
12320 216 : tree arg0 = CALL_EXPR_ARG (exp, 0);
12321 216 : tree arg1 = CALL_EXPR_ARG (exp, 1);
12322 216 : tree arg2 = CALL_EXPR_ARG (exp, 2);
12323 216 : tree arg3 = CALL_EXPR_ARG (exp, 3);
12324 216 : tree arg4 = CALL_EXPR_ARG (exp, 4);
12325 216 : rtx scratch0, scratch1;
12326 216 : rtx op0 = expand_normal (arg0);
12327 216 : rtx op1 = expand_normal (arg1);
12328 216 : rtx op2 = expand_normal (arg2);
12329 216 : rtx op3 = expand_normal (arg3);
12330 216 : rtx op4 = expand_normal (arg4);
12331 216 : machine_mode tmode0, tmode1, modev2, modei3, modev4, modei5, modeimm;
12332 :
12333 216 : tmode0 = insn_data[d->icode].operand[0].mode;
12334 216 : tmode1 = insn_data[d->icode].operand[1].mode;
12335 216 : modev2 = insn_data[d->icode].operand[2].mode;
12336 216 : modei3 = insn_data[d->icode].operand[3].mode;
12337 216 : modev4 = insn_data[d->icode].operand[4].mode;
12338 216 : modei5 = insn_data[d->icode].operand[5].mode;
12339 216 : modeimm = insn_data[d->icode].operand[6].mode;
12340 :
12341 216 : if (VECTOR_MODE_P (modev2))
12342 216 : op0 = safe_vector_operand (op0, modev2);
12343 216 : if (VECTOR_MODE_P (modev4))
12344 216 : op2 = safe_vector_operand (op2, modev4);
12345 :
12346 216 : if (!insn_data[d->icode].operand[2].predicate (op0, modev2))
12347 6 : op0 = copy_to_mode_reg (modev2, op0);
12348 216 : if (!insn_data[d->icode].operand[3].predicate (op1, modei3))
12349 34 : op1 = copy_to_mode_reg (modei3, op1);
12350 160 : if ((optimize && !register_operand (op2, modev4))
12351 371 : || !insn_data[d->icode].operand[4].predicate (op2, modev4))
12352 5 : op2 = copy_to_mode_reg (modev4, op2);
12353 216 : if (!insn_data[d->icode].operand[5].predicate (op3, modei5))
12354 34 : op3 = copy_to_mode_reg (modei5, op3);
12355 :
12356 216 : if (!insn_data[d->icode].operand[6].predicate (op4, modeimm))
12357 : {
12358 21 : error ("the fifth argument must be an 8-bit immediate");
12359 21 : return const0_rtx;
12360 : }
12361 :
12362 195 : if (d->code == IX86_BUILTIN_PCMPESTRI128)
12363 : {
12364 5 : if (optimize || !target
12365 5 : || GET_MODE (target) != tmode0
12366 34 : || !insn_data[d->icode].operand[0].predicate (target, tmode0))
12367 24 : target = gen_reg_rtx (tmode0);
12368 :
12369 29 : scratch1 = gen_reg_rtx (tmode1);
12370 :
12371 29 : pat = GEN_FCN (d->icode) (target, scratch1, op0, op1, op2, op3, op4);
12372 : }
12373 166 : else if (d->code == IX86_BUILTIN_PCMPESTRM128)
12374 : {
12375 5 : if (optimize || !target
12376 5 : || GET_MODE (target) != tmode1
12377 36 : || !insn_data[d->icode].operand[1].predicate (target, tmode1))
12378 26 : target = gen_reg_rtx (tmode1);
12379 :
12380 31 : scratch0 = gen_reg_rtx (tmode0);
12381 :
12382 31 : pat = GEN_FCN (d->icode) (scratch0, target, op0, op1, op2, op3, op4);
12383 : }
12384 : else
12385 : {
12386 135 : gcc_assert (d->flag);
12387 :
12388 135 : scratch0 = gen_reg_rtx (tmode0);
12389 135 : scratch1 = gen_reg_rtx (tmode1);
12390 :
12391 135 : pat = GEN_FCN (d->icode) (scratch0, scratch1, op0, op1, op2, op3, op4);
12392 : }
12393 :
12394 195 : if (! pat)
12395 : return 0;
12396 :
12397 195 : emit_insn (pat);
12398 :
12399 195 : if (d->flag)
12400 : {
12401 135 : target = gen_reg_rtx (SImode);
12402 135 : emit_move_insn (target, const0_rtx);
12403 135 : target = gen_rtx_SUBREG (QImode, target, 0);
12404 :
12405 135 : emit_insn
12406 135 : (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
12407 : gen_rtx_fmt_ee (EQ, QImode,
12408 : gen_rtx_REG ((machine_mode) d->flag,
12409 : FLAGS_REG),
12410 : const0_rtx)));
12411 135 : return SUBREG_REG (target);
12412 : }
12413 : else
12414 : return target;
12415 : }
12416 :
12417 :
12418 : /* Subroutine of ix86_expand_builtin to take care of pcmpistr[im] insns. */
12419 :
12420 : static rtx
12421 275 : ix86_expand_sse_pcmpistr (const struct builtin_description *d,
12422 : tree exp, rtx target)
12423 : {
12424 275 : rtx pat;
12425 275 : tree arg0 = CALL_EXPR_ARG (exp, 0);
12426 275 : tree arg1 = CALL_EXPR_ARG (exp, 1);
12427 275 : tree arg2 = CALL_EXPR_ARG (exp, 2);
12428 275 : rtx scratch0, scratch1;
12429 275 : rtx op0 = expand_normal (arg0);
12430 275 : rtx op1 = expand_normal (arg1);
12431 275 : rtx op2 = expand_normal (arg2);
12432 275 : machine_mode tmode0, tmode1, modev2, modev3, modeimm;
12433 :
12434 275 : tmode0 = insn_data[d->icode].operand[0].mode;
12435 275 : tmode1 = insn_data[d->icode].operand[1].mode;
12436 275 : modev2 = insn_data[d->icode].operand[2].mode;
12437 275 : modev3 = insn_data[d->icode].operand[3].mode;
12438 275 : modeimm = insn_data[d->icode].operand[4].mode;
12439 :
12440 275 : if (VECTOR_MODE_P (modev2))
12441 275 : op0 = safe_vector_operand (op0, modev2);
12442 275 : if (VECTOR_MODE_P (modev3))
12443 275 : op1 = safe_vector_operand (op1, modev3);
12444 :
12445 275 : if (!insn_data[d->icode].operand[2].predicate (op0, modev2))
12446 4 : op0 = copy_to_mode_reg (modev2, op0);
12447 210 : if ((optimize && !register_operand (op1, modev3))
12448 481 : || !insn_data[d->icode].operand[3].predicate (op1, modev3))
12449 4 : op1 = copy_to_mode_reg (modev3, op1);
12450 :
12451 275 : if (!insn_data[d->icode].operand[4].predicate (op2, modeimm))
12452 : {
12453 21 : error ("the third argument must be an 8-bit immediate");
12454 21 : return const0_rtx;
12455 : }
12456 :
12457 254 : if (d->code == IX86_BUILTIN_PCMPISTRI128)
12458 : {
12459 5 : if (optimize || !target
12460 5 : || GET_MODE (target) != tmode0
12461 38 : || !insn_data[d->icode].operand[0].predicate (target, tmode0))
12462 28 : target = gen_reg_rtx (tmode0);
12463 :
12464 33 : scratch1 = gen_reg_rtx (tmode1);
12465 :
12466 33 : pat = GEN_FCN (d->icode) (target, scratch1, op0, op1, op2);
12467 : }
12468 221 : else if (d->code == IX86_BUILTIN_PCMPISTRM128)
12469 : {
12470 8 : if (optimize || !target
12471 8 : || GET_MODE (target) != tmode1
12472 58 : || !insn_data[d->icode].operand[1].predicate (target, tmode1))
12473 42 : target = gen_reg_rtx (tmode1);
12474 :
12475 50 : scratch0 = gen_reg_rtx (tmode0);
12476 :
12477 50 : pat = GEN_FCN (d->icode) (scratch0, target, op0, op1, op2);
12478 : }
12479 : else
12480 : {
12481 171 : gcc_assert (d->flag);
12482 :
12483 171 : scratch0 = gen_reg_rtx (tmode0);
12484 171 : scratch1 = gen_reg_rtx (tmode1);
12485 :
12486 171 : pat = GEN_FCN (d->icode) (scratch0, scratch1, op0, op1, op2);
12487 : }
12488 :
12489 254 : if (! pat)
12490 : return 0;
12491 :
12492 254 : emit_insn (pat);
12493 :
12494 254 : if (d->flag)
12495 : {
12496 171 : target = gen_reg_rtx (SImode);
12497 171 : emit_move_insn (target, const0_rtx);
12498 171 : target = gen_rtx_SUBREG (QImode, target, 0);
12499 :
12500 171 : emit_insn
12501 171 : (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
12502 : gen_rtx_fmt_ee (EQ, QImode,
12503 : gen_rtx_REG ((machine_mode) d->flag,
12504 : FLAGS_REG),
12505 : const0_rtx)));
12506 171 : return SUBREG_REG (target);
12507 : }
12508 : else
12509 : return target;
12510 : }
12511 :
12512 : /* Fixup modeless constants to fit required mode. */
12513 :
12514 : static rtx
12515 269052 : fixup_modeless_constant (rtx x, machine_mode mode)
12516 : {
12517 269052 : if (GET_MODE (x) == VOIDmode)
12518 41970 : x = convert_to_mode (mode, x, 1);
12519 269052 : return x;
12520 : }
12521 :
12522 : /* Expand the outgoing argument ARG to extract unsigned char and short
12523 : integer constants suitable for the predicates and the instruction
12524 : templates which expect the unsigned expanded value. */
12525 :
12526 : static rtx
12527 290568 : ix86_expand_unsigned_small_int_cst_argument (tree arg)
12528 : {
12529 : /* When passing 0xff as an unsigned char function argument with the
12530 : C frontend promotion, expand_normal gets
12531 :
12532 : <integer_cst 0x7fffe6aa23a8 type <integer_type 0x7fffe98225e8 int> constant 255>
12533 :
12534 : and returns the rtx value using the sign-extended representation:
12535 :
12536 : (const_int 255 [0xff])
12537 :
12538 : Without the C frontend promotion, expand_normal gets
12539 :
12540 : <integer_cst 0x7fffe9824018 type <integer_type 0x7fffe9822348 unsigned char > constant 255>
12541 :
12542 : and returns
12543 :
12544 : (const_int -1 [0xffffffffffffffff])
12545 :
12546 : which doesn't work with the predicates nor the instruction templates
12547 : which expect the unsigned expanded value. Extract the unsigned char
12548 : and short integer constants to return
12549 :
12550 : (const_int 255 [0xff])
12551 :
12552 : so that the expanded value is always unsigned, without the C frontend
12553 : promotion. */
12554 :
12555 290568 : if (TREE_CODE (arg) == INTEGER_CST)
12556 : {
12557 61129 : tree type = TREE_TYPE (arg);
12558 61129 : if (INTEGRAL_TYPE_P (type)
12559 61129 : && TYPE_UNSIGNED (type)
12560 83689 : && TYPE_PRECISION (type) < TYPE_PRECISION (integer_type_node))
12561 : {
12562 18990 : HOST_WIDE_INT cst = TREE_INT_CST_LOW (arg);
12563 18990 : return GEN_INT (cst);
12564 : }
12565 : }
12566 :
12567 271578 : return expand_normal (arg);
12568 : }
12569 :
12570 : /* Subroutine of ix86_expand_builtin to take care of insns with
12571 : variable number of operands. */
12572 :
12573 : static rtx
12574 74502 : ix86_expand_args_builtin (const struct builtin_description *d,
12575 : tree exp, rtx target)
12576 : {
12577 74502 : rtx pat, real_target;
12578 74502 : unsigned int i, nargs;
12579 74502 : unsigned int nargs_constant = 0;
12580 74502 : unsigned int mask_pos = 0;
12581 74502 : int num_memory = 0;
12582 74502 : rtx xops[6];
12583 74502 : bool second_arg_count = false;
12584 74502 : enum insn_code icode = d->icode;
12585 74502 : const struct insn_data_d *insn_p = &insn_data[icode];
12586 74502 : machine_mode tmode = insn_p->operand[0].mode;
12587 74502 : machine_mode rmode = VOIDmode;
12588 74502 : bool swap = false;
12589 74502 : enum rtx_code comparison = d->comparison;
12590 :
12591 74502 : switch ((enum ix86_builtin_func_type) d->flag)
12592 : {
12593 0 : case V2DF_FTYPE_V2DF_ROUND:
12594 0 : case V4DF_FTYPE_V4DF_ROUND:
12595 0 : case V8DF_FTYPE_V8DF_ROUND:
12596 0 : case V4SF_FTYPE_V4SF_ROUND:
12597 0 : case V8SF_FTYPE_V8SF_ROUND:
12598 0 : case V16SF_FTYPE_V16SF_ROUND:
12599 0 : case V8HF_FTYPE_V8HF_ROUND:
12600 0 : case V16HF_FTYPE_V16HF_ROUND:
12601 0 : case V32HF_FTYPE_V32HF_ROUND:
12602 0 : case V4SI_FTYPE_V4SF_ROUND:
12603 0 : case V8SI_FTYPE_V8SF_ROUND:
12604 0 : case V16SI_FTYPE_V16SF_ROUND:
12605 0 : return ix86_expand_sse_round (d, exp, target);
12606 12 : case V4SI_FTYPE_V2DF_V2DF_ROUND:
12607 12 : case V8SI_FTYPE_V4DF_V4DF_ROUND:
12608 12 : case V16SI_FTYPE_V8DF_V8DF_ROUND:
12609 12 : return ix86_expand_sse_round_vec_pack_sfix (d, exp, target);
12610 239 : case INT_FTYPE_V8SF_V8SF_PTEST:
12611 239 : case INT_FTYPE_V4DI_V4DI_PTEST:
12612 239 : case INT_FTYPE_V4DF_V4DF_PTEST:
12613 239 : case INT_FTYPE_V4SF_V4SF_PTEST:
12614 239 : case INT_FTYPE_V2DI_V2DI_PTEST:
12615 239 : case INT_FTYPE_V2DF_V2DF_PTEST:
12616 239 : return ix86_expand_sse_ptest (d, exp, target);
12617 : case FLOAT128_FTYPE_FLOAT128:
12618 : case FLOAT_FTYPE_FLOAT:
12619 : case FLOAT_FTYPE_BFLOAT16:
12620 : case INT_FTYPE_INT:
12621 : case UINT_FTYPE_UINT:
12622 : case UINT16_FTYPE_UINT16:
12623 : case UINT64_FTYPE_INT:
12624 : case UINT64_FTYPE_UINT64:
12625 : case INT64_FTYPE_INT64:
12626 : case INT64_FTYPE_V4SF:
12627 : case INT64_FTYPE_V2DF:
12628 : case INT_FTYPE_V16QI:
12629 : case INT_FTYPE_V8QI:
12630 : case INT_FTYPE_V8SF:
12631 : case INT_FTYPE_V4DF:
12632 : case INT_FTYPE_V4SF:
12633 : case INT_FTYPE_V2DF:
12634 : case INT_FTYPE_V32QI:
12635 : case V16QI_FTYPE_V16QI:
12636 : case V8SI_FTYPE_V8SF:
12637 : case V8SI_FTYPE_V4SI:
12638 : case V8HI_FTYPE_V8HI:
12639 : case V8HI_FTYPE_V16QI:
12640 : case V8QI_FTYPE_V8QI:
12641 : case V8SF_FTYPE_V8SF:
12642 : case V8SF_FTYPE_V8SI:
12643 : case V8SF_FTYPE_V4SF:
12644 : case V8SF_FTYPE_V8HI:
12645 : case V4SI_FTYPE_V4SI:
12646 : case V4SI_FTYPE_V16QI:
12647 : case V4SI_FTYPE_V4SF:
12648 : case V4SI_FTYPE_V8SI:
12649 : case V4SI_FTYPE_V8HI:
12650 : case V4SI_FTYPE_V4DF:
12651 : case V4SI_FTYPE_V2DF:
12652 : case V4HI_FTYPE_V4HI:
12653 : case V4DF_FTYPE_V4DF:
12654 : case V4DF_FTYPE_V4SI:
12655 : case V4DF_FTYPE_V4SF:
12656 : case V4DF_FTYPE_V2DF:
12657 : case V4SF_FTYPE_V4SF:
12658 : case V4SF_FTYPE_V4SI:
12659 : case V4SF_FTYPE_V8SF:
12660 : case V4SF_FTYPE_V4DF:
12661 : case V4SF_FTYPE_V8HI:
12662 : case V4SF_FTYPE_V2DF:
12663 : case V2DI_FTYPE_V2DI:
12664 : case V2DI_FTYPE_V16QI:
12665 : case V2DI_FTYPE_V8HI:
12666 : case V2DI_FTYPE_V4SI:
12667 : case V2DF_FTYPE_V2DF:
12668 : case V2DF_FTYPE_V4SI:
12669 : case V2DF_FTYPE_V4DF:
12670 : case V2DF_FTYPE_V4SF:
12671 : case V2DF_FTYPE_V2SI:
12672 : case V2SI_FTYPE_V2SI:
12673 : case V2SI_FTYPE_V4SF:
12674 : case V2SI_FTYPE_V2SF:
12675 : case V2SI_FTYPE_V2DF:
12676 : case V2SF_FTYPE_V2SF:
12677 : case V2SF_FTYPE_V2SI:
12678 : case V32QI_FTYPE_V32QI:
12679 : case V32QI_FTYPE_V16QI:
12680 : case V16HI_FTYPE_V16HI:
12681 : case V16HI_FTYPE_V8HI:
12682 : case V8SI_FTYPE_V8SI:
12683 : case V16HI_FTYPE_V16QI:
12684 : case V8SI_FTYPE_V16QI:
12685 : case V4DI_FTYPE_V16QI:
12686 : case V8SI_FTYPE_V8HI:
12687 : case V4DI_FTYPE_V8HI:
12688 : case V4DI_FTYPE_V4SI:
12689 : case V4DI_FTYPE_V2DI:
12690 : case UQI_FTYPE_UQI:
12691 : case UHI_FTYPE_UHI:
12692 : case USI_FTYPE_USI:
12693 : case USI_FTYPE_UQI:
12694 : case USI_FTYPE_UHI:
12695 : case UDI_FTYPE_UDI:
12696 : case UHI_FTYPE_V16QI:
12697 : case USI_FTYPE_V32QI:
12698 : case UDI_FTYPE_V64QI:
12699 : case V16QI_FTYPE_UHI:
12700 : case V32QI_FTYPE_USI:
12701 : case V64QI_FTYPE_UDI:
12702 : case V8HI_FTYPE_UQI:
12703 : case V16HI_FTYPE_UHI:
12704 : case V32HI_FTYPE_USI:
12705 : case V4SI_FTYPE_UQI:
12706 : case V8SI_FTYPE_UQI:
12707 : case V4SI_FTYPE_UHI:
12708 : case V8SI_FTYPE_UHI:
12709 : case UQI_FTYPE_V8HI:
12710 : case UHI_FTYPE_V16HI:
12711 : case USI_FTYPE_V32HI:
12712 : case UQI_FTYPE_V4SI:
12713 : case UQI_FTYPE_V8SI:
12714 : case UHI_FTYPE_V16SI:
12715 : case UQI_FTYPE_V2DI:
12716 : case UQI_FTYPE_V4DI:
12717 : case UQI_FTYPE_V8DI:
12718 : case V16SI_FTYPE_UHI:
12719 : case V2DI_FTYPE_UQI:
12720 : case V4DI_FTYPE_UQI:
12721 : case V16SI_FTYPE_INT:
12722 : case V16SF_FTYPE_V8SF:
12723 : case V16SI_FTYPE_V8SI:
12724 : case V16SF_FTYPE_V4SF:
12725 : case V16SI_FTYPE_V4SI:
12726 : case V16SI_FTYPE_V16SF:
12727 : case V16SI_FTYPE_V16SI:
12728 : case V64QI_FTYPE_V64QI:
12729 : case V32HI_FTYPE_V32HI:
12730 : case V16SF_FTYPE_V16SF:
12731 : case V8DI_FTYPE_UQI:
12732 : case V8DI_FTYPE_V8DI:
12733 : case V8DF_FTYPE_V4DF:
12734 : case V8DF_FTYPE_V2DF:
12735 : case V8DF_FTYPE_V8DF:
12736 : case V4DI_FTYPE_V4DI:
12737 : case V16BF_FTYPE_V16SF:
12738 : case V8BF_FTYPE_V8SF:
12739 : case V8BF_FTYPE_V4SF:
12740 : case V16QI_FTYPE_V32QI:
12741 : case V32QI_FTYPE_V64QI:
12742 : nargs = 1;
12743 : break;
12744 52 : case V4SF_FTYPE_V4SF_VEC_MERGE:
12745 52 : case V2DF_FTYPE_V2DF_VEC_MERGE:
12746 52 : return ix86_expand_unop_vec_merge_builtin (icode, exp, target);
12747 9386 : case FLOAT128_FTYPE_FLOAT128_FLOAT128:
12748 9386 : case V16QI_FTYPE_V16QI_V16QI:
12749 9386 : case V16QI_FTYPE_V8HI_V8HI:
12750 9386 : case V16HF_FTYPE_V16HF_V16HF:
12751 9386 : case V16SF_FTYPE_V16SF_V16SF:
12752 9386 : case V16SI_FTYPE_V16SI_V16SI:
12753 9386 : case V8QI_FTYPE_V8QI_V8QI:
12754 9386 : case V8QI_FTYPE_V4HI_V4HI:
12755 9386 : case V8HI_FTYPE_V8HI_V8HI:
12756 9386 : case V8HI_FTYPE_V16QI_V16QI:
12757 9386 : case V8HI_FTYPE_V4SI_V4SI:
12758 9386 : case V8HF_FTYPE_V8HF_V8HF:
12759 9386 : case V8SF_FTYPE_V8SF_V8SF:
12760 9386 : case V8SF_FTYPE_V8SF_V8SI:
12761 9386 : case V8DF_FTYPE_V8DF_V8DF:
12762 9386 : case V4SI_FTYPE_V4SI_V4SI:
12763 9386 : case V4SI_FTYPE_V8HI_V8HI:
12764 9386 : case V4SI_FTYPE_V2DF_V2DF:
12765 9386 : case V4HI_FTYPE_V4HI_V4HI:
12766 9386 : case V4HI_FTYPE_V8QI_V8QI:
12767 9386 : case V4HI_FTYPE_V2SI_V2SI:
12768 9386 : case V4DF_FTYPE_V4DF_V4DF:
12769 9386 : case V4DF_FTYPE_V4DF_V4DI:
12770 9386 : case V4SF_FTYPE_V4SF_V4SF:
12771 9386 : case V4SF_FTYPE_V4SF_V4SI:
12772 9386 : case V4SF_FTYPE_V4SF_V2SI:
12773 9386 : case V4SF_FTYPE_V4SF_V2DF:
12774 9386 : case V4SF_FTYPE_V4SF_UINT:
12775 9386 : case V4SF_FTYPE_V4SF_DI:
12776 9386 : case V4SF_FTYPE_V4SF_SI:
12777 9386 : case V4DI_FTYPE_V4DI_V2DI:
12778 9386 : case V2DI_FTYPE_V2DI_V2DI:
12779 9386 : case V2DI_FTYPE_V16QI_V16QI:
12780 9386 : case V2DI_FTYPE_V4SI_V4SI:
12781 9386 : case V2DI_FTYPE_V2DI_V16QI:
12782 9386 : case V2SI_FTYPE_V2SI_V2SI:
12783 9386 : case V2SI_FTYPE_V4HI_V4HI:
12784 9386 : case V2SI_FTYPE_V2SF_V2SF:
12785 9386 : case V2DF_FTYPE_V2DF_V2DF:
12786 9386 : case V2DF_FTYPE_V2DF_V4SF:
12787 9386 : case V2DF_FTYPE_V2DF_V2DI:
12788 9386 : case V2DF_FTYPE_V2DF_DI:
12789 9386 : case V2DF_FTYPE_V2DF_SI:
12790 9386 : case V2DF_FTYPE_V2DF_UINT:
12791 9386 : case V2SF_FTYPE_V2SF_V2SF:
12792 9386 : case V1DI_FTYPE_V1DI_V1DI:
12793 9386 : case V1DI_FTYPE_V8QI_V8QI:
12794 9386 : case V1DI_FTYPE_V2SI_V2SI:
12795 9386 : case V32QI_FTYPE_V16HI_V16HI:
12796 9386 : case V16HI_FTYPE_V8SI_V8SI:
12797 9386 : case V64QI_FTYPE_V64QI_V64QI:
12798 9386 : case V32QI_FTYPE_V32QI_V32QI:
12799 9386 : case V32BF_FTYPE_V32BF_V32BF:
12800 9386 : case V16BF_FTYPE_V16BF_V16BF:
12801 9386 : case V8BF_FTYPE_V8BF_V8BF:
12802 9386 : case V16HI_FTYPE_V32QI_V32QI:
12803 9386 : case V16HI_FTYPE_V16HI_V16HI:
12804 9386 : case V8SI_FTYPE_V4DF_V4DF:
12805 9386 : case V8SI_FTYPE_V8SI_V8SI:
12806 9386 : case V8SI_FTYPE_V16HI_V16HI:
12807 9386 : case V4DI_FTYPE_V4DI_V4DI:
12808 9386 : case V4DI_FTYPE_V8SI_V8SI:
12809 9386 : case V4DI_FTYPE_V32QI_V32QI:
12810 9386 : case V8DI_FTYPE_V64QI_V64QI:
12811 9386 : if (comparison == UNKNOWN)
12812 8858 : return ix86_expand_binop_builtin (icode, exp, target);
12813 : nargs = 2;
12814 : break;
12815 80 : case V4SF_FTYPE_V4SF_V4SF_SWAP:
12816 80 : case V2DF_FTYPE_V2DF_V2DF_SWAP:
12817 80 : gcc_assert (comparison != UNKNOWN);
12818 : nargs = 2;
12819 : swap = true;
12820 : break;
12821 1481 : case V16HI_FTYPE_V16HI_V8HI_COUNT:
12822 1481 : case V16HI_FTYPE_V16HI_SI_COUNT:
12823 1481 : case V8SI_FTYPE_V8SI_V4SI_COUNT:
12824 1481 : case V8SI_FTYPE_V8SI_SI_COUNT:
12825 1481 : case V4DI_FTYPE_V4DI_V2DI_COUNT:
12826 1481 : case V4DI_FTYPE_V4DI_INT_COUNT:
12827 1481 : case V8HI_FTYPE_V8HI_V8HI_COUNT:
12828 1481 : case V8HI_FTYPE_V8HI_SI_COUNT:
12829 1481 : case V4SI_FTYPE_V4SI_V4SI_COUNT:
12830 1481 : case V4SI_FTYPE_V4SI_SI_COUNT:
12831 1481 : case V4HI_FTYPE_V4HI_V4HI_COUNT:
12832 1481 : case V4HI_FTYPE_V4HI_SI_COUNT:
12833 1481 : case V2DI_FTYPE_V2DI_V2DI_COUNT:
12834 1481 : case V2DI_FTYPE_V2DI_SI_COUNT:
12835 1481 : case V2SI_FTYPE_V2SI_V2SI_COUNT:
12836 1481 : case V2SI_FTYPE_V2SI_SI_COUNT:
12837 1481 : case V1DI_FTYPE_V1DI_V1DI_COUNT:
12838 1481 : case V1DI_FTYPE_V1DI_SI_COUNT:
12839 1481 : nargs = 2;
12840 1481 : second_arg_count = true;
12841 1481 : break;
12842 1408 : case V16HI_FTYPE_V16HI_INT_V16HI_UHI_COUNT:
12843 1408 : case V16HI_FTYPE_V16HI_V8HI_V16HI_UHI_COUNT:
12844 1408 : case V16SI_FTYPE_V16SI_INT_V16SI_UHI_COUNT:
12845 1408 : case V16SI_FTYPE_V16SI_V4SI_V16SI_UHI_COUNT:
12846 1408 : case V2DI_FTYPE_V2DI_INT_V2DI_UQI_COUNT:
12847 1408 : case V2DI_FTYPE_V2DI_V2DI_V2DI_UQI_COUNT:
12848 1408 : case V32HI_FTYPE_V32HI_INT_V32HI_USI_COUNT:
12849 1408 : case V32HI_FTYPE_V32HI_V8HI_V32HI_USI_COUNT:
12850 1408 : case V4DI_FTYPE_V4DI_INT_V4DI_UQI_COUNT:
12851 1408 : case V4DI_FTYPE_V4DI_V2DI_V4DI_UQI_COUNT:
12852 1408 : case V4SI_FTYPE_V4SI_INT_V4SI_UQI_COUNT:
12853 1408 : case V4SI_FTYPE_V4SI_V4SI_V4SI_UQI_COUNT:
12854 1408 : case V8DI_FTYPE_V8DI_INT_V8DI_UQI_COUNT:
12855 1408 : case V8DI_FTYPE_V8DI_V2DI_V8DI_UQI_COUNT:
12856 1408 : case V8HI_FTYPE_V8HI_INT_V8HI_UQI_COUNT:
12857 1408 : case V8HI_FTYPE_V8HI_V8HI_V8HI_UQI_COUNT:
12858 1408 : case V8SI_FTYPE_V8SI_INT_V8SI_UQI_COUNT:
12859 1408 : case V8SI_FTYPE_V8SI_V4SI_V8SI_UQI_COUNT:
12860 1408 : nargs = 4;
12861 1408 : second_arg_count = true;
12862 1408 : break;
12863 977 : case UINT64_FTYPE_UINT64_UINT64:
12864 977 : case UINT_FTYPE_UINT_UINT:
12865 977 : case UINT_FTYPE_UINT_USHORT:
12866 977 : case UINT_FTYPE_UINT_UCHAR:
12867 977 : case UINT16_FTYPE_UINT16_INT:
12868 977 : case UINT8_FTYPE_UINT8_INT:
12869 977 : case UQI_FTYPE_UQI_UQI:
12870 977 : case UHI_FTYPE_UHI_UHI:
12871 977 : case USI_FTYPE_USI_USI:
12872 977 : case UDI_FTYPE_UDI_UDI:
12873 977 : case V16SI_FTYPE_V8DF_V8DF:
12874 977 : case V32BF_FTYPE_V16SF_V16SF:
12875 977 : case V16BF_FTYPE_V8SF_V8SF:
12876 977 : case V8BF_FTYPE_V4SF_V4SF:
12877 977 : case V16BF_FTYPE_V16SF_UHI:
12878 977 : case V8BF_FTYPE_V8SF_UQI:
12879 977 : case V8BF_FTYPE_V4SF_UQI:
12880 977 : case V16QI_FTYPE_V16QI_V8HF:
12881 977 : nargs = 2;
12882 977 : break;
12883 639 : case V2DI_FTYPE_V2DI_INT_CONVERT:
12884 639 : nargs = 2;
12885 639 : rmode = V1TImode;
12886 639 : nargs_constant = 1;
12887 639 : break;
12888 42 : case V4DI_FTYPE_V4DI_INT_CONVERT:
12889 42 : nargs = 2;
12890 42 : rmode = V2TImode;
12891 42 : nargs_constant = 1;
12892 42 : break;
12893 16 : case V8DI_FTYPE_V8DI_INT_CONVERT:
12894 16 : nargs = 2;
12895 16 : rmode = V4TImode;
12896 16 : nargs_constant = 1;
12897 16 : break;
12898 2424 : case V8HI_FTYPE_V8HI_INT:
12899 2424 : case V8HI_FTYPE_V8SF_INT:
12900 2424 : case V16HI_FTYPE_V16SF_INT:
12901 2424 : case V8HI_FTYPE_V4SF_INT:
12902 2424 : case V8SF_FTYPE_V8SF_INT:
12903 2424 : case V4SF_FTYPE_V16SF_INT:
12904 2424 : case V16SF_FTYPE_V16SF_INT:
12905 2424 : case V4SI_FTYPE_V4SI_INT:
12906 2424 : case V4SI_FTYPE_V8SI_INT:
12907 2424 : case V4HI_FTYPE_V4HI_INT:
12908 2424 : case V4DF_FTYPE_V4DF_INT:
12909 2424 : case V4DF_FTYPE_V8DF_INT:
12910 2424 : case V4SF_FTYPE_V4SF_INT:
12911 2424 : case V4SF_FTYPE_V8SF_INT:
12912 2424 : case V2DI_FTYPE_V2DI_INT:
12913 2424 : case V2DF_FTYPE_V2DF_INT:
12914 2424 : case V2DF_FTYPE_V4DF_INT:
12915 2424 : case V16HI_FTYPE_V16HI_INT:
12916 2424 : case V8SI_FTYPE_V8SI_INT:
12917 2424 : case V16SI_FTYPE_V16SI_INT:
12918 2424 : case V4SI_FTYPE_V16SI_INT:
12919 2424 : case V4DI_FTYPE_V4DI_INT:
12920 2424 : case V2DI_FTYPE_V4DI_INT:
12921 2424 : case V4DI_FTYPE_V8DI_INT:
12922 2424 : case UQI_FTYPE_UQI_UQI_CONST:
12923 2424 : case UHI_FTYPE_UHI_UQI:
12924 2424 : case USI_FTYPE_USI_UQI:
12925 2424 : case UDI_FTYPE_UDI_UQI:
12926 2424 : nargs = 2;
12927 2424 : nargs_constant = 1;
12928 2424 : break;
12929 19988 : case V16QI_FTYPE_V16QI_V16QI_V16QI:
12930 19988 : case V8SF_FTYPE_V8SF_V8SF_V8SF:
12931 19988 : case V4DF_FTYPE_V4DF_V4DF_V4DF:
12932 19988 : case V4SF_FTYPE_V4SF_V4SF_V4SF:
12933 19988 : case V2DF_FTYPE_V2DF_V2DF_V2DF:
12934 19988 : case V32QI_FTYPE_V32QI_V32QI_V32QI:
12935 19988 : case UHI_FTYPE_V16SI_V16SI_UHI:
12936 19988 : case UQI_FTYPE_V8DI_V8DI_UQI:
12937 19988 : case V16HI_FTYPE_V16SI_V16HI_UHI:
12938 19988 : case V16QI_FTYPE_V16SI_V16QI_UHI:
12939 19988 : case V16QI_FTYPE_V8DI_V16QI_UQI:
12940 19988 : case V32HF_FTYPE_V32HF_V32HF_USI:
12941 19988 : case V16SF_FTYPE_V16SF_V16SF_UHI:
12942 19988 : case V16SF_FTYPE_V4SF_V16SF_UHI:
12943 19988 : case V16SI_FTYPE_SI_V16SI_UHI:
12944 19988 : case V16SI_FTYPE_V16HI_V16SI_UHI:
12945 19988 : case V16SI_FTYPE_V16QI_V16SI_UHI:
12946 19988 : case V8SF_FTYPE_V4SF_V8SF_UQI:
12947 19988 : case V4DF_FTYPE_V2DF_V4DF_UQI:
12948 19988 : case V8SI_FTYPE_V4SI_V8SI_UQI:
12949 19988 : case V8SI_FTYPE_SI_V8SI_UQI:
12950 19988 : case V4SI_FTYPE_V4SI_V4SI_UQI:
12951 19988 : case V4SI_FTYPE_SI_V4SI_UQI:
12952 19988 : case V4DI_FTYPE_V2DI_V4DI_UQI:
12953 19988 : case V4DI_FTYPE_DI_V4DI_UQI:
12954 19988 : case V2DI_FTYPE_V2DI_V2DI_UQI:
12955 19988 : case V2DI_FTYPE_DI_V2DI_UQI:
12956 19988 : case V64QI_FTYPE_V64QI_V64QI_UDI:
12957 19988 : case V64QI_FTYPE_V16QI_V64QI_UDI:
12958 19988 : case V64QI_FTYPE_QI_V64QI_UDI:
12959 19988 : case V32QI_FTYPE_V32QI_V32QI_USI:
12960 19988 : case V32QI_FTYPE_V16QI_V32QI_USI:
12961 19988 : case V32QI_FTYPE_QI_V32QI_USI:
12962 19988 : case V16QI_FTYPE_V16QI_V16QI_UHI:
12963 19988 : case V16QI_FTYPE_QI_V16QI_UHI:
12964 19988 : case V32HI_FTYPE_V8HI_V32HI_USI:
12965 19988 : case V32HI_FTYPE_V32BF_V32HI_USI:
12966 19988 : case V32HI_FTYPE_HI_V32HI_USI:
12967 19988 : case V16HI_FTYPE_V8HI_V16HI_UHI:
12968 19988 : case V16HI_FTYPE_V16BF_V16HI_UHI:
12969 19988 : case V16HI_FTYPE_HI_V16HI_UHI:
12970 19988 : case V8HI_FTYPE_V8HI_V8HI_UQI:
12971 19988 : case V8HI_FTYPE_V8BF_V8HI_UQI:
12972 19988 : case V8BF_FTYPE_V8BF_V8BF_UQI:
12973 19988 : case V8HI_FTYPE_HI_V8HI_UQI:
12974 19988 : case V16HF_FTYPE_V16HF_V16HF_UHI:
12975 19988 : case V8SF_FTYPE_V8HI_V8SF_UQI:
12976 19988 : case V4SF_FTYPE_V8HI_V4SF_UQI:
12977 19988 : case V8SI_FTYPE_V8HF_V8SI_UQI:
12978 19988 : case V8SF_FTYPE_V8HF_V8SF_UQI:
12979 19988 : case V8SI_FTYPE_V8SF_V8SI_UQI:
12980 19988 : case V4SI_FTYPE_V4SF_V4SI_UQI:
12981 19988 : case V4SI_FTYPE_V8HF_V4SI_UQI:
12982 19988 : case V4SF_FTYPE_V8HF_V4SF_UQI:
12983 19988 : case V4DI_FTYPE_V8HF_V4DI_UQI:
12984 19988 : case V4DI_FTYPE_V4SF_V4DI_UQI:
12985 19988 : case V2DI_FTYPE_V8HF_V2DI_UQI:
12986 19988 : case V2DI_FTYPE_V4SF_V2DI_UQI:
12987 19988 : case V8HF_FTYPE_V8HF_V8HF_UQI:
12988 19988 : case V8HF_FTYPE_V8HF_V8HF_V8HF:
12989 19988 : case V8HF_FTYPE_V8HI_V8HF_UQI:
12990 19988 : case V8HF_FTYPE_V8SI_V8HF_UQI:
12991 19988 : case V8HF_FTYPE_V8SF_V8HF_UQI:
12992 19988 : case V8HF_FTYPE_V4SI_V8HF_UQI:
12993 19988 : case V8HF_FTYPE_V4SF_V8HF_UQI:
12994 19988 : case V8HF_FTYPE_V4DI_V8HF_UQI:
12995 19988 : case V8HF_FTYPE_V4DF_V8HF_UQI:
12996 19988 : case V8HF_FTYPE_V2DI_V8HF_UQI:
12997 19988 : case V8HF_FTYPE_V2DF_V8HF_UQI:
12998 19988 : case V4SF_FTYPE_V4DI_V4SF_UQI:
12999 19988 : case V4SF_FTYPE_V2DI_V4SF_UQI:
13000 19988 : case V4DF_FTYPE_V4DI_V4DF_UQI:
13001 19988 : case V4DF_FTYPE_V8HF_V4DF_UQI:
13002 19988 : case V2DF_FTYPE_V8HF_V2DF_UQI:
13003 19988 : case V2DF_FTYPE_V2DI_V2DF_UQI:
13004 19988 : case V16QI_FTYPE_V8HI_V16QI_UQI:
13005 19988 : case V16QI_FTYPE_V16HI_V16QI_UHI:
13006 19988 : case V16QI_FTYPE_V4SI_V16QI_UQI:
13007 19988 : case V16QI_FTYPE_V8SI_V16QI_UQI:
13008 19988 : case V8HI_FTYPE_V8HF_V8HI_UQI:
13009 19988 : case V8HI_FTYPE_V4SI_V8HI_UQI:
13010 19988 : case V8HI_FTYPE_V8SI_V8HI_UQI:
13011 19988 : case V16QI_FTYPE_V2DI_V16QI_UQI:
13012 19988 : case V16QI_FTYPE_V4DI_V16QI_UQI:
13013 19988 : case V8HI_FTYPE_V2DI_V8HI_UQI:
13014 19988 : case V8HI_FTYPE_V4DI_V8HI_UQI:
13015 19988 : case V4SI_FTYPE_V2DI_V4SI_UQI:
13016 19988 : case V4SI_FTYPE_V4DI_V4SI_UQI:
13017 19988 : case V32QI_FTYPE_V32HI_V32QI_USI:
13018 19988 : case UHI_FTYPE_V16QI_V16QI_UHI:
13019 19988 : case USI_FTYPE_V32QI_V32QI_USI:
13020 19988 : case UDI_FTYPE_V64QI_V64QI_UDI:
13021 19988 : case UQI_FTYPE_V8HI_V8HI_UQI:
13022 19988 : case UHI_FTYPE_V16HI_V16HI_UHI:
13023 19988 : case USI_FTYPE_V32HI_V32HI_USI:
13024 19988 : case UQI_FTYPE_V4SI_V4SI_UQI:
13025 19988 : case UQI_FTYPE_V8SI_V8SI_UQI:
13026 19988 : case UQI_FTYPE_V2DI_V2DI_UQI:
13027 19988 : case UQI_FTYPE_V4DI_V4DI_UQI:
13028 19988 : case V4SF_FTYPE_V2DF_V4SF_UQI:
13029 19988 : case V4SF_FTYPE_V4DF_V4SF_UQI:
13030 19988 : case V16SI_FTYPE_V16SI_V16SI_UHI:
13031 19988 : case V16SI_FTYPE_V4SI_V16SI_UHI:
13032 19988 : case V2DI_FTYPE_V4SI_V2DI_UQI:
13033 19988 : case V2DI_FTYPE_V8HI_V2DI_UQI:
13034 19988 : case V2DI_FTYPE_V16QI_V2DI_UQI:
13035 19988 : case V4DI_FTYPE_V4DI_V4DI_UQI:
13036 19988 : case V4DI_FTYPE_V4SI_V4DI_UQI:
13037 19988 : case V4DI_FTYPE_V8HI_V4DI_UQI:
13038 19988 : case V4DI_FTYPE_V16QI_V4DI_UQI:
13039 19988 : case V4DI_FTYPE_V4DF_V4DI_UQI:
13040 19988 : case V2DI_FTYPE_V2DF_V2DI_UQI:
13041 19988 : case V4SI_FTYPE_V4DF_V4SI_UQI:
13042 19988 : case V4SI_FTYPE_V2DF_V4SI_UQI:
13043 19988 : case V4SI_FTYPE_V8HI_V4SI_UQI:
13044 19988 : case V4SI_FTYPE_V16QI_V4SI_UQI:
13045 19988 : case V4DI_FTYPE_V4DI_V4DI_V4DI:
13046 19988 : case V8DF_FTYPE_V2DF_V8DF_UQI:
13047 19988 : case V8DF_FTYPE_V4DF_V8DF_UQI:
13048 19988 : case V8DF_FTYPE_V8DF_V8DF_UQI:
13049 19988 : case V8SF_FTYPE_V8SF_V8SF_UQI:
13050 19988 : case V8SF_FTYPE_V8SI_V8SF_UQI:
13051 19988 : case V4DF_FTYPE_V4DF_V4DF_UQI:
13052 19988 : case V4SF_FTYPE_V4SF_V4SF_UQI:
13053 19988 : case V2DF_FTYPE_V2DF_V2DF_UQI:
13054 19988 : case V2DF_FTYPE_V4SF_V2DF_UQI:
13055 19988 : case V2DF_FTYPE_V4SI_V2DF_UQI:
13056 19988 : case V4SF_FTYPE_V4SI_V4SF_UQI:
13057 19988 : case V4DF_FTYPE_V4SF_V4DF_UQI:
13058 19988 : case V4DF_FTYPE_V4SI_V4DF_UQI:
13059 19988 : case V8SI_FTYPE_V8SI_V8SI_UQI:
13060 19988 : case V8SI_FTYPE_V8HI_V8SI_UQI:
13061 19988 : case V8SI_FTYPE_V16QI_V8SI_UQI:
13062 19988 : case V8DF_FTYPE_V8SI_V8DF_UQI:
13063 19988 : case V8DI_FTYPE_DI_V8DI_UQI:
13064 19988 : case V16SF_FTYPE_V8SF_V16SF_UHI:
13065 19988 : case V16SI_FTYPE_V8SI_V16SI_UHI:
13066 19988 : case V16HF_FTYPE_V16HI_V16HF_UHI:
13067 19988 : case V16HF_FTYPE_V16HF_V16HF_V16HF:
13068 19988 : case V16HI_FTYPE_V16HF_V16HI_UHI:
13069 19988 : case V16HI_FTYPE_V16HI_V16HI_UHI:
13070 19988 : case V16BF_FTYPE_V16BF_V16BF_UHI:
13071 19988 : case V8HI_FTYPE_V16QI_V8HI_UQI:
13072 19988 : case V16HI_FTYPE_V16QI_V16HI_UHI:
13073 19988 : case V32HI_FTYPE_V32HI_V32HI_USI:
13074 19988 : case V32BF_FTYPE_V32BF_V32BF_USI:
13075 19988 : case V32HI_FTYPE_V32QI_V32HI_USI:
13076 19988 : case V8DI_FTYPE_V16QI_V8DI_UQI:
13077 19988 : case V8DI_FTYPE_V2DI_V8DI_UQI:
13078 19988 : case V8DI_FTYPE_V4DI_V8DI_UQI:
13079 19988 : case V8DI_FTYPE_V8DI_V8DI_UQI:
13080 19988 : case V8DI_FTYPE_V8HI_V8DI_UQI:
13081 19988 : case V8DI_FTYPE_V8SI_V8DI_UQI:
13082 19988 : case V8HI_FTYPE_V8DI_V8HI_UQI:
13083 19988 : case V8SI_FTYPE_V8DI_V8SI_UQI:
13084 19988 : case V4SI_FTYPE_V4SI_V4SI_V4SI:
13085 19988 : case V4DI_FTYPE_V4DI_V4DI_V2DI:
13086 19988 : case V16SI_FTYPE_V16SI_V16SI_V16SI:
13087 19988 : case V8DI_FTYPE_V8DI_V8DI_V8DI:
13088 19988 : case V32HI_FTYPE_V32HI_V32HI_V32HI:
13089 19988 : case V2DI_FTYPE_V2DI_V2DI_V2DI:
13090 19988 : case V16HI_FTYPE_V16HI_V16HI_V16HI:
13091 19988 : case V8SI_FTYPE_V8SI_V8SI_V8SI:
13092 19988 : case V8HI_FTYPE_V8HI_V8HI_V8HI:
13093 19988 : case V32BF_FTYPE_V16SF_V16SF_USI:
13094 19988 : case V16BF_FTYPE_V8SF_V8SF_UHI:
13095 19988 : case V8BF_FTYPE_V4SF_V4SF_UQI:
13096 19988 : case V16BF_FTYPE_V16SF_V16BF_UHI:
13097 19988 : case V8BF_FTYPE_V8SF_V8BF_UQI:
13098 19988 : case V8BF_FTYPE_V4SF_V8BF_UQI:
13099 19988 : case V16SF_FTYPE_V16SF_V32BF_V32BF:
13100 19988 : case V8SF_FTYPE_V8SF_V16BF_V16BF:
13101 19988 : case V4SF_FTYPE_V4SF_V8BF_V8BF:
13102 19988 : case V16QI_FTYPE_V16QI_V8HF_V8HF:
13103 19988 : case V32QI_FTYPE_V32QI_V16HF_V16HF:
13104 19988 : case V64QI_FTYPE_V64QI_V32HF_V32HF:
13105 19988 : case V16QI_FTYPE_V8HF_V16QI_UQI:
13106 19988 : case V16QI_FTYPE_V16HF_V16QI_UHI:
13107 19988 : case V32QI_FTYPE_V32HF_V32QI_USI:
13108 19988 : case V8HF_FTYPE_V16QI_V8HF_UQI:
13109 19988 : case V16HF_FTYPE_V16QI_V16HF_UHI:
13110 19988 : case V32HF_FTYPE_V32QI_V32HF_USI:
13111 19988 : case V16SI_FTYPE_V16SF_V16SI_UHI:
13112 19988 : case V32HI_FTYPE_V32HF_V32HI_USI:
13113 19988 : case V8DI_FTYPE_V8SF_V8DI_UQI:
13114 19988 : case V8DI_FTYPE_V8DF_V8DI_UQI:
13115 19988 : case V8SI_FTYPE_V8DF_V8SI_UQI:
13116 19988 : case V16QI_FTYPE_V4SF_V16QI_UQI:
13117 19988 : case V16QI_FTYPE_V8SF_V16QI_UQI:
13118 19988 : case V16QI_FTYPE_V16SF_V16QI_UHI:
13119 19988 : case V4SF_FTYPE_V16QI_V4SF_UQI:
13120 19988 : case V8SF_FTYPE_V16QI_V8SF_UQI:
13121 19988 : case V16SF_FTYPE_V16QI_V16SF_UHI:
13122 19988 : case V64QI_FTYPE_V32QI_V64QI_UDI:
13123 19988 : nargs = 3;
13124 19988 : break;
13125 1484 : case V32QI_FTYPE_V32QI_V32QI_INT:
13126 1484 : case V16HI_FTYPE_V16HI_V16HI_INT:
13127 1484 : case V16QI_FTYPE_V16QI_V16QI_INT:
13128 1484 : case V4DI_FTYPE_V4DI_V4DI_INT:
13129 1484 : case V8HI_FTYPE_V8HI_V8HI_INT:
13130 1484 : case V8SI_FTYPE_V8SI_V8SI_INT:
13131 1484 : case V8SI_FTYPE_V8SI_V4SI_INT:
13132 1484 : case V8SF_FTYPE_V8SF_V8SF_INT:
13133 1484 : case V8SF_FTYPE_V8SF_V4SF_INT:
13134 1484 : case V4SI_FTYPE_V4SI_V4SI_INT:
13135 1484 : case V4DF_FTYPE_V4DF_V4DF_INT:
13136 1484 : case V16SF_FTYPE_V16SF_V16SF_INT:
13137 1484 : case V16SF_FTYPE_V16SF_V4SF_INT:
13138 1484 : case V16SI_FTYPE_V16SI_V4SI_INT:
13139 1484 : case V4DF_FTYPE_V4DF_V2DF_INT:
13140 1484 : case V4SF_FTYPE_V4SF_V4SF_INT:
13141 1484 : case V2DI_FTYPE_V2DI_V2DI_INT:
13142 1484 : case V4DI_FTYPE_V4DI_V2DI_INT:
13143 1484 : case V2DF_FTYPE_V2DF_V2DF_INT:
13144 1484 : case UQI_FTYPE_V8DI_V8UDI_INT:
13145 1484 : case UQI_FTYPE_V8DF_V8DF_INT:
13146 1484 : case UQI_FTYPE_V2DF_V2DF_INT:
13147 1484 : case UQI_FTYPE_V4SF_V4SF_INT:
13148 1484 : case UHI_FTYPE_V16SI_V16SI_INT:
13149 1484 : case UHI_FTYPE_V16SF_V16SF_INT:
13150 1484 : case V64QI_FTYPE_V64QI_V64QI_INT:
13151 1484 : case V32HI_FTYPE_V32HI_V32HI_INT:
13152 1484 : case V16SI_FTYPE_V16SI_V16SI_INT:
13153 1484 : case V8DI_FTYPE_V8DI_V8DI_INT:
13154 1484 : nargs = 3;
13155 1484 : nargs_constant = 1;
13156 1484 : break;
13157 47 : case V4DI_FTYPE_V4DI_V4DI_INT_CONVERT:
13158 47 : nargs = 3;
13159 47 : rmode = V4DImode;
13160 47 : nargs_constant = 1;
13161 47 : break;
13162 80 : case V2DI_FTYPE_V2DI_V2DI_INT_CONVERT:
13163 80 : nargs = 3;
13164 80 : rmode = V2DImode;
13165 80 : nargs_constant = 1;
13166 80 : break;
13167 48 : case V1DI_FTYPE_V1DI_V1DI_INT_CONVERT:
13168 48 : nargs = 3;
13169 48 : rmode = DImode;
13170 48 : nargs_constant = 1;
13171 48 : break;
13172 20 : case V2DI_FTYPE_V2DI_UINT_UINT:
13173 20 : nargs = 3;
13174 20 : nargs_constant = 2;
13175 20 : break;
13176 8 : case V8DI_FTYPE_V8DI_V8DI_INT_CONVERT:
13177 8 : nargs = 3;
13178 8 : rmode = V8DImode;
13179 8 : nargs_constant = 1;
13180 8 : break;
13181 16 : case V8DI_FTYPE_V8DI_V8DI_INT_V8DI_UDI_CONVERT:
13182 16 : nargs = 5;
13183 16 : rmode = V8DImode;
13184 16 : mask_pos = 2;
13185 16 : nargs_constant = 1;
13186 16 : break;
13187 320 : case QI_FTYPE_V8DF_INT_UQI:
13188 320 : case QI_FTYPE_V4DF_INT_UQI:
13189 320 : case QI_FTYPE_V2DF_INT_UQI:
13190 320 : case HI_FTYPE_V16SF_INT_UHI:
13191 320 : case QI_FTYPE_V8SF_INT_UQI:
13192 320 : case QI_FTYPE_V4SF_INT_UQI:
13193 320 : case QI_FTYPE_V8HF_INT_UQI:
13194 320 : case HI_FTYPE_V16HF_INT_UHI:
13195 320 : case SI_FTYPE_V32HF_INT_USI:
13196 320 : case QI_FTYPE_V8BF_INT_UQI:
13197 320 : case HI_FTYPE_V16BF_INT_UHI:
13198 320 : case SI_FTYPE_V32BF_INT_USI:
13199 320 : case V4SI_FTYPE_V4SI_V4SI_UHI:
13200 320 : case V8SI_FTYPE_V8SI_V8SI_UHI:
13201 320 : nargs = 3;
13202 320 : mask_pos = 1;
13203 320 : nargs_constant = 1;
13204 320 : break;
13205 17 : case V4DI_FTYPE_V4DI_V4DI_INT_V4DI_USI_CONVERT:
13206 17 : nargs = 5;
13207 17 : rmode = V4DImode;
13208 17 : mask_pos = 2;
13209 17 : nargs_constant = 1;
13210 17 : break;
13211 17 : case V2DI_FTYPE_V2DI_V2DI_INT_V2DI_UHI_CONVERT:
13212 17 : nargs = 5;
13213 17 : rmode = V2DImode;
13214 17 : mask_pos = 2;
13215 17 : nargs_constant = 1;
13216 17 : break;
13217 17783 : case V32QI_FTYPE_V32QI_V32QI_V32QI_USI:
13218 17783 : case V32HI_FTYPE_V32HI_V32HI_V32HI_USI:
13219 17783 : case V32BF_FTYPE_V32BF_V32BF_V32BF_USI:
13220 17783 : case V32HI_FTYPE_V64QI_V64QI_V32HI_USI:
13221 17783 : case V16SI_FTYPE_V32HI_V32HI_V16SI_UHI:
13222 17783 : case V64QI_FTYPE_V64QI_V64QI_V64QI_UDI:
13223 17783 : case V32HI_FTYPE_V32HI_V8HI_V32HI_USI:
13224 17783 : case V16HI_FTYPE_V16HI_V8HI_V16HI_UHI:
13225 17783 : case V8SI_FTYPE_V8SI_V4SI_V8SI_UQI:
13226 17783 : case V4DI_FTYPE_V4DI_V2DI_V4DI_UQI:
13227 17783 : case V64QI_FTYPE_V32HI_V32HI_V64QI_UDI:
13228 17783 : case V32QI_FTYPE_V16HI_V16HI_V32QI_USI:
13229 17783 : case V16QI_FTYPE_V8HI_V8HI_V16QI_UHI:
13230 17783 : case V32HI_FTYPE_V16SI_V16SI_V32HI_USI:
13231 17783 : case V16HI_FTYPE_V8SI_V8SI_V16HI_UHI:
13232 17783 : case V8HI_FTYPE_V4SI_V4SI_V8HI_UQI:
13233 17783 : case V4DF_FTYPE_V4DF_V4DI_V4DF_UQI:
13234 17783 : case V32HF_FTYPE_V32HF_V32HF_V32HF_USI:
13235 17783 : case V8SF_FTYPE_V8SF_V8SI_V8SF_UQI:
13236 17783 : case V4SF_FTYPE_V4SF_V4SI_V4SF_UQI:
13237 17783 : case V2DF_FTYPE_V2DF_V2DI_V2DF_UQI:
13238 17783 : case V2DI_FTYPE_V4SI_V4SI_V2DI_UQI:
13239 17783 : case V4DI_FTYPE_V8SI_V8SI_V4DI_UQI:
13240 17783 : case V4DF_FTYPE_V4DI_V4DF_V4DF_UQI:
13241 17783 : case V8SF_FTYPE_V8SI_V8SF_V8SF_UQI:
13242 17783 : case V2DF_FTYPE_V2DI_V2DF_V2DF_UQI:
13243 17783 : case V4SF_FTYPE_V4SI_V4SF_V4SF_UQI:
13244 17783 : case V16SF_FTYPE_V16SF_V16SF_V16SF_UHI:
13245 17783 : case V16SF_FTYPE_V16SF_V16SI_V16SF_UHI:
13246 17783 : case V16SF_FTYPE_V16SI_V16SF_V16SF_UHI:
13247 17783 : case V16SI_FTYPE_V16SI_V16SI_V16SI_UHI:
13248 17783 : case V16SI_FTYPE_V16SI_V4SI_V16SI_UHI:
13249 17783 : case V8HI_FTYPE_V8HI_V8HI_V8HI_UQI:
13250 17783 : case V8BF_FTYPE_V8BF_V8BF_V8BF_UQI:
13251 17783 : case V8SI_FTYPE_V8SI_V8SI_V8SI_UQI:
13252 17783 : case V4SI_FTYPE_V4SI_V4SI_V4SI_UQI:
13253 17783 : case V16HF_FTYPE_V16HF_V16HF_V16HF_UQI:
13254 17783 : case V16HF_FTYPE_V16HF_V16HF_V16HF_UHI:
13255 17783 : case V8SF_FTYPE_V8SF_V8SF_V8SF_UQI:
13256 17783 : case V16QI_FTYPE_V16QI_V16QI_V16QI_UHI:
13257 17783 : case V16HI_FTYPE_V16HI_V16HI_V16HI_UHI:
13258 17783 : case V16BF_FTYPE_V16BF_V16BF_V16BF_UHI:
13259 17783 : case V2DI_FTYPE_V2DI_V2DI_V2DI_UQI:
13260 17783 : case V2DF_FTYPE_V2DF_V2DF_V2DF_UQI:
13261 17783 : case V4DI_FTYPE_V4DI_V4DI_V4DI_UQI:
13262 17783 : case V4DF_FTYPE_V4DF_V4DF_V4DF_UQI:
13263 17783 : case V8HF_FTYPE_V8HF_V8HF_V8HF_UQI:
13264 17783 : case V4SF_FTYPE_V4SF_V4SF_V4SF_UQI:
13265 17783 : case V8DF_FTYPE_V8DF_V8DF_V8DF_UQI:
13266 17783 : case V8DF_FTYPE_V8DF_V8DI_V8DF_UQI:
13267 17783 : case V8DF_FTYPE_V8DI_V8DF_V8DF_UQI:
13268 17783 : case V8DI_FTYPE_V16SI_V16SI_V8DI_UQI:
13269 17783 : case V8DI_FTYPE_V8DI_V2DI_V8DI_UQI:
13270 17783 : case V8DI_FTYPE_V8DI_V8DI_V8DI_UQI:
13271 17783 : case V8HI_FTYPE_V16QI_V16QI_V8HI_UQI:
13272 17783 : case V16HI_FTYPE_V32QI_V32QI_V16HI_UHI:
13273 17783 : case V8SI_FTYPE_V16HI_V16HI_V8SI_UQI:
13274 17783 : case V4SI_FTYPE_V8HI_V8HI_V4SI_UQI:
13275 17783 : case V32BF_FTYPE_V16SF_V16SF_V32BF_USI:
13276 17783 : case V16BF_FTYPE_V8SF_V8SF_V16BF_UHI:
13277 17783 : case V8BF_FTYPE_V4SF_V4SF_V8BF_UQI:
13278 17783 : case V32HF_FTYPE_V16SF_V16SF_V32HF_USI:
13279 17783 : case V16HF_FTYPE_V8SF_V8SF_V16HF_UHI:
13280 17783 : case V8HF_FTYPE_V4SF_V4SF_V8HF_UQI:
13281 17783 : case V16QI_FTYPE_V8HF_V8HF_V16QI_UHI:
13282 17783 : case V32QI_FTYPE_V16HF_V16HF_V32QI_USI:
13283 17783 : case V64QI_FTYPE_V32HF_V32HF_V64QI_UDI:
13284 17783 : case V16QI_FTYPE_V16QI_V8HF_V16QI_UHI:
13285 17783 : case V16QI_FTYPE_V32QI_V16HF_V16QI_UHI:
13286 17783 : case V32QI_FTYPE_V64QI_V32HF_V32QI_USI:
13287 17783 : case V16QI_FTYPE_V4SI_V4SF_V16QI_UQI:
13288 17783 : case V16QI_FTYPE_V8SI_V8SF_V16QI_UQI:
13289 17783 : case V16QI_FTYPE_V16SI_V16SF_V16QI_UHI:
13290 17783 : nargs = 4;
13291 17783 : break;
13292 11 : case V2DF_FTYPE_V2DF_V2DF_V2DI_INT:
13293 11 : case V4DF_FTYPE_V4DF_V4DF_V4DI_INT:
13294 11 : case V4SF_FTYPE_V4SF_V4SF_V4SI_INT:
13295 11 : case V8SF_FTYPE_V8SF_V8SF_V8SI_INT:
13296 11 : case V16SF_FTYPE_V16SF_V16SF_V16SI_INT:
13297 11 : case V4SI_FTYPE_V4SI_V4SI_V4SI_INT:
13298 11 : nargs = 4;
13299 11 : nargs_constant = 1;
13300 11 : break;
13301 3718 : case UQI_FTYPE_V4DI_V4DI_INT_UQI:
13302 3718 : case UQI_FTYPE_V8SI_V8SI_INT_UQI:
13303 3718 : case QI_FTYPE_V4DF_V4DF_INT_UQI:
13304 3718 : case QI_FTYPE_V8SF_V8SF_INT_UQI:
13305 3718 : case UHI_FTYPE_V16HF_V16HF_INT_UHI:
13306 3718 : case UQI_FTYPE_V2DI_V2DI_INT_UQI:
13307 3718 : case UQI_FTYPE_V4SI_V4SI_INT_UQI:
13308 3718 : case UQI_FTYPE_V2DF_V2DF_INT_UQI:
13309 3718 : case UQI_FTYPE_V4SF_V4SF_INT_UQI:
13310 3718 : case UQI_FTYPE_V8HF_V8HF_INT_UQI:
13311 3718 : case UDI_FTYPE_V64QI_V64QI_INT_UDI:
13312 3718 : case USI_FTYPE_V32QI_V32QI_INT_USI:
13313 3718 : case UHI_FTYPE_V16QI_V16QI_INT_UHI:
13314 3718 : case USI_FTYPE_V32HI_V32HI_INT_USI:
13315 3718 : case USI_FTYPE_V32BF_V32BF_INT_USI:
13316 3718 : case USI_FTYPE_V32HF_V32HF_INT_USI:
13317 3718 : case UHI_FTYPE_V16HI_V16HI_INT_UHI:
13318 3718 : case UHI_FTYPE_V16BF_V16BF_INT_UHI:
13319 3718 : case UQI_FTYPE_V8HI_V8HI_INT_UQI:
13320 3718 : case UQI_FTYPE_V8BF_V8BF_INT_UQI:
13321 3718 : nargs = 4;
13322 3718 : mask_pos = 1;
13323 3718 : nargs_constant = 1;
13324 3718 : break;
13325 23 : case V2DI_FTYPE_V2DI_V2DI_UINT_UINT:
13326 23 : nargs = 4;
13327 23 : nargs_constant = 2;
13328 23 : break;
13329 69 : case UCHAR_FTYPE_UCHAR_UINT_UINT_PUNSIGNED:
13330 69 : case UCHAR_FTYPE_UCHAR_ULONGLONG_ULONGLONG_PULONGLONG:
13331 69 : case V16SF_FTYPE_V16SF_V32BF_V32BF_UHI:
13332 69 : case V8SF_FTYPE_V8SF_V16BF_V16BF_UQI:
13333 69 : case V4SF_FTYPE_V4SF_V8BF_V8BF_UQI:
13334 69 : nargs = 4;
13335 69 : break;
13336 679 : case UQI_FTYPE_V8DI_V8DI_INT_UQI:
13337 679 : case UHI_FTYPE_V16SI_V16SI_INT_UHI:
13338 679 : mask_pos = 1;
13339 679 : nargs = 4;
13340 679 : nargs_constant = 1;
13341 679 : break;
13342 4056 : case V8SF_FTYPE_V8SF_INT_V8SF_UQI:
13343 4056 : case V4SF_FTYPE_V4SF_INT_V4SF_UQI:
13344 4056 : case V2DF_FTYPE_V4DF_INT_V2DF_UQI:
13345 4056 : case V2DI_FTYPE_V4DI_INT_V2DI_UQI:
13346 4056 : case V8SF_FTYPE_V16SF_INT_V8SF_UQI:
13347 4056 : case V8SI_FTYPE_V16SI_INT_V8SI_UQI:
13348 4056 : case V2DF_FTYPE_V8DF_INT_V2DF_UQI:
13349 4056 : case V2DI_FTYPE_V8DI_INT_V2DI_UQI:
13350 4056 : case V4SF_FTYPE_V8SF_INT_V4SF_UQI:
13351 4056 : case V4SI_FTYPE_V8SI_INT_V4SI_UQI:
13352 4056 : case V8HI_FTYPE_V8SF_INT_V8HI_UQI:
13353 4056 : case V8HI_FTYPE_V4SF_INT_V8HI_UQI:
13354 4056 : case V32HI_FTYPE_V32HI_INT_V32HI_USI:
13355 4056 : case V16HI_FTYPE_V16HI_INT_V16HI_UHI:
13356 4056 : case V8HI_FTYPE_V8HI_INT_V8HI_UQI:
13357 4056 : case V32BF_FTYPE_V32BF_INT_V32BF_USI:
13358 4056 : case V16BF_FTYPE_V16BF_INT_V16BF_UHI:
13359 4056 : case V8BF_FTYPE_V8BF_INT_V8BF_UQI:
13360 4056 : case V4DI_FTYPE_V4DI_INT_V4DI_UQI:
13361 4056 : case V2DI_FTYPE_V2DI_INT_V2DI_UQI:
13362 4056 : case V8SI_FTYPE_V8SI_INT_V8SI_UQI:
13363 4056 : case V4SI_FTYPE_V4SI_INT_V4SI_UQI:
13364 4056 : case V4DF_FTYPE_V4DF_INT_V4DF_UQI:
13365 4056 : case V2DF_FTYPE_V2DF_INT_V2DF_UQI:
13366 4056 : case V8DF_FTYPE_V8DF_INT_V8DF_UQI:
13367 4056 : case V16SF_FTYPE_V16SF_INT_V16SF_UHI:
13368 4056 : case V16HI_FTYPE_V16SF_INT_V16HI_UHI:
13369 4056 : case V16SI_FTYPE_V16SI_INT_V16SI_UHI:
13370 4056 : case V16HF_FTYPE_V16HF_INT_V16HF_UHI:
13371 4056 : case V8HF_FTYPE_V8HF_INT_V8HF_UQI:
13372 4056 : case V4SI_FTYPE_V16SI_INT_V4SI_UQI:
13373 4056 : case V4DI_FTYPE_V8DI_INT_V4DI_UQI:
13374 4056 : case V4DF_FTYPE_V8DF_INT_V4DF_UQI:
13375 4056 : case V4SF_FTYPE_V16SF_INT_V4SF_UQI:
13376 4056 : case V8DI_FTYPE_V8DI_INT_V8DI_UQI:
13377 4056 : case V16QI_FTYPE_V16QI_INT_V16QI_UHI:
13378 4056 : case V32QI_FTYPE_V32QI_INT_V32QI_USI:
13379 4056 : case V64QI_FTYPE_V64QI_INT_V64QI_UDI:
13380 4056 : nargs = 4;
13381 4056 : mask_pos = 2;
13382 4056 : nargs_constant = 1;
13383 4056 : break;
13384 1726 : case V16SF_FTYPE_V16SF_V4SF_INT_V16SF_UHI:
13385 1726 : case V16SI_FTYPE_V16SI_V4SI_INT_V16SI_UHI:
13386 1726 : case V8DF_FTYPE_V8DF_V8DF_INT_V8DF_UQI:
13387 1726 : case V8DI_FTYPE_V8DI_V8DI_INT_V8DI_UQI:
13388 1726 : case V16SF_FTYPE_V16SF_V16SF_INT_V16SF_UHI:
13389 1726 : case V16SI_FTYPE_V16SI_V16SI_INT_V16SI_UHI:
13390 1726 : case V4SF_FTYPE_V4SF_V4SF_INT_V4SF_UQI:
13391 1726 : case V2DF_FTYPE_V2DF_V2DF_INT_V2DF_UQI:
13392 1726 : case V8DF_FTYPE_V8DF_V4DF_INT_V8DF_UQI:
13393 1726 : case V8DI_FTYPE_V8DI_V4DI_INT_V8DI_UQI:
13394 1726 : case V4DF_FTYPE_V4DF_V4DF_INT_V4DF_UQI:
13395 1726 : case V8SF_FTYPE_V8SF_V8SF_INT_V8SF_UQI:
13396 1726 : case V8DF_FTYPE_V8DF_V2DF_INT_V8DF_UQI:
13397 1726 : case V8DI_FTYPE_V8DI_V2DI_INT_V8DI_UQI:
13398 1726 : case V8SI_FTYPE_V8SI_V8SI_INT_V8SI_UQI:
13399 1726 : case V4DI_FTYPE_V4DI_V4DI_INT_V4DI_UQI:
13400 1726 : case V4SI_FTYPE_V4SI_V4SI_INT_V4SI_UQI:
13401 1726 : case V2DI_FTYPE_V2DI_V2DI_INT_V2DI_UQI:
13402 1726 : case V32HI_FTYPE_V64QI_V64QI_INT_V32HI_USI:
13403 1726 : case V16HI_FTYPE_V32QI_V32QI_INT_V16HI_UHI:
13404 1726 : case V8HI_FTYPE_V16QI_V16QI_INT_V8HI_UQI:
13405 1726 : case V16SF_FTYPE_V16SF_V8SF_INT_V16SF_UHI:
13406 1726 : case V16SI_FTYPE_V16SI_V8SI_INT_V16SI_UHI:
13407 1726 : case V8SF_FTYPE_V8SF_V4SF_INT_V8SF_UQI:
13408 1726 : case V8SI_FTYPE_V8SI_V4SI_INT_V8SI_UQI:
13409 1726 : case V4DI_FTYPE_V4DI_V2DI_INT_V4DI_UQI:
13410 1726 : case V4DF_FTYPE_V4DF_V2DF_INT_V4DF_UQI:
13411 1726 : nargs = 5;
13412 1726 : mask_pos = 2;
13413 1726 : nargs_constant = 1;
13414 1726 : break;
13415 268 : case V8DI_FTYPE_V8DI_V8DI_V8DI_INT_UQI:
13416 268 : case V16SI_FTYPE_V16SI_V16SI_V16SI_INT_UHI:
13417 268 : case V2DF_FTYPE_V2DF_V2DF_V2DI_INT_UQI:
13418 268 : case V4SF_FTYPE_V4SF_V4SF_V4SI_INT_UQI:
13419 268 : case V8SF_FTYPE_V8SF_V8SF_V8SI_INT_UQI:
13420 268 : case V8SI_FTYPE_V8SI_V8SI_V8SI_INT_UQI:
13421 268 : case V4DF_FTYPE_V4DF_V4DF_V4DI_INT_UQI:
13422 268 : case V4DI_FTYPE_V4DI_V4DI_V4DI_INT_UQI:
13423 268 : case V4SI_FTYPE_V4SI_V4SI_V4SI_INT_UQI:
13424 268 : case V2DI_FTYPE_V2DI_V2DI_V2DI_INT_UQI:
13425 268 : nargs = 5;
13426 268 : mask_pos = 1;
13427 268 : nargs_constant = 1;
13428 268 : break;
13429 732 : case V64QI_FTYPE_V64QI_V64QI_INT_V64QI_UDI:
13430 732 : case V32QI_FTYPE_V32QI_V32QI_INT_V32QI_USI:
13431 732 : case V16QI_FTYPE_V16QI_V16QI_INT_V16QI_UHI:
13432 732 : case V32HI_FTYPE_V32HI_V32HI_INT_V32HI_INT:
13433 732 : case V16SI_FTYPE_V16SI_V16SI_INT_V16SI_INT:
13434 732 : case V8DI_FTYPE_V8DI_V8DI_INT_V8DI_INT:
13435 732 : case V16HI_FTYPE_V16HI_V16HI_INT_V16HI_INT:
13436 732 : case V8SI_FTYPE_V8SI_V8SI_INT_V8SI_INT:
13437 732 : case V4DI_FTYPE_V4DI_V4DI_INT_V4DI_INT:
13438 732 : case V8HI_FTYPE_V8HI_V8HI_INT_V8HI_INT:
13439 732 : case V4SI_FTYPE_V4SI_V4SI_INT_V4SI_INT:
13440 732 : case V2DI_FTYPE_V2DI_V2DI_INT_V2DI_INT:
13441 732 : case V8BF_FTYPE_V8BF_V8BF_INT_V8BF_UQI:
13442 732 : case V16BF_FTYPE_V16BF_V16BF_INT_V16BF_UHI:
13443 732 : case V32BF_FTYPE_V32BF_V32BF_INT_V32BF_USI:
13444 732 : case V16HF_FTYPE_V16HF_V16HF_INT_V16HF_UHI:
13445 732 : case V8HF_FTYPE_V8HF_V8HF_INT_V8HF_UQI:
13446 732 : nargs = 5;
13447 732 : mask_pos = 1;
13448 732 : nargs_constant = 2;
13449 732 : break;
13450 :
13451 0 : default:
13452 0 : gcc_unreachable ();
13453 : }
13454 :
13455 58097 : gcc_assert (nargs <= ARRAY_SIZE (xops));
13456 :
13457 65341 : if (comparison != UNKNOWN)
13458 : {
13459 608 : gcc_assert (nargs == 2);
13460 608 : return ix86_expand_sse_compare (d, exp, target, swap);
13461 : }
13462 :
13463 64733 : if (rmode == VOIDmode || rmode == tmode)
13464 : {
13465 64548 : if (optimize
13466 18379 : || target == 0
13467 18379 : || GET_MODE (target) != tmode
13468 82725 : || !insn_p->operand[0].predicate (target, tmode))
13469 46459 : target = gen_reg_rtx (tmode);
13470 18089 : else if (memory_operand (target, tmode))
13471 578 : num_memory++;
13472 : real_target = target;
13473 : }
13474 : else
13475 : {
13476 185 : real_target = gen_reg_rtx (tmode);
13477 185 : target = lowpart_subreg (rmode, real_target, tmode);
13478 : }
13479 :
13480 272952 : for (i = 0; i < nargs; i++)
13481 : {
13482 208461 : tree arg = CALL_EXPR_ARG (exp, i);
13483 208461 : rtx op = ix86_expand_unsigned_small_int_cst_argument (arg);
13484 208461 : machine_mode mode = insn_p->operand[i + 1].mode;
13485 : /* Need to fixup modeless constant before testing predicate. */
13486 208461 : op = fixup_modeless_constant (op, mode);
13487 208461 : bool match = insn_p->operand[i + 1].predicate (op, mode);
13488 :
13489 208461 : if (second_arg_count && i == 1)
13490 : {
13491 : /* SIMD shift insns take either an 8-bit immediate or
13492 : register as count. But builtin functions take int as
13493 : count. If count doesn't match, we put it in register.
13494 : The instructions are using 64-bit count, if op is just
13495 : 32-bit, zero-extend it, as negative shift counts
13496 : are undefined behavior and zero-extension is more
13497 : efficient. */
13498 2889 : if (!match)
13499 : {
13500 1750 : if (SCALAR_INT_MODE_P (GET_MODE (op)))
13501 489 : op = convert_modes (mode, GET_MODE (op), op, 1);
13502 : else
13503 1261 : op = lowpart_subreg (mode, op, GET_MODE (op));
13504 1750 : if (!insn_p->operand[i + 1].predicate (op, mode))
13505 190 : op = copy_to_reg (op);
13506 : }
13507 : }
13508 205572 : else if ((mask_pos && (nargs - i - mask_pos) == nargs_constant) ||
13509 157102 : (!mask_pos && (nargs - i) <= nargs_constant))
13510 : {
13511 16430 : if (!match)
13512 233 : switch (icode)
13513 : {
13514 2 : case CODE_FOR_avx_vinsertf128v4di:
13515 2 : case CODE_FOR_avx_vextractf128v4di:
13516 2 : error ("the last argument must be an 1-bit immediate");
13517 2 : return const0_rtx;
13518 :
13519 8 : case CODE_FOR_avx512f_cmpv8di3_mask:
13520 8 : case CODE_FOR_avx512f_cmpv16si3_mask:
13521 8 : case CODE_FOR_avx512f_ucmpv8di3_mask:
13522 8 : case CODE_FOR_avx512f_ucmpv16si3_mask:
13523 8 : case CODE_FOR_avx512vl_cmpv4di3_mask:
13524 8 : case CODE_FOR_avx512vl_cmpv8si3_mask:
13525 8 : case CODE_FOR_avx512vl_ucmpv4di3_mask:
13526 8 : case CODE_FOR_avx512vl_ucmpv8si3_mask:
13527 8 : case CODE_FOR_avx512vl_cmpv2di3_mask:
13528 8 : case CODE_FOR_avx512vl_cmpv4si3_mask:
13529 8 : case CODE_FOR_avx512vl_ucmpv2di3_mask:
13530 8 : case CODE_FOR_avx512vl_ucmpv4si3_mask:
13531 8 : error ("the last argument must be a 3-bit immediate");
13532 8 : return const0_rtx;
13533 :
13534 24 : case CODE_FOR_sse4_1_roundsd:
13535 24 : case CODE_FOR_sse4_1_roundss:
13536 :
13537 24 : case CODE_FOR_sse4_1_roundpd:
13538 24 : case CODE_FOR_sse4_1_roundps:
13539 24 : case CODE_FOR_avx_roundpd256:
13540 24 : case CODE_FOR_avx_roundps256:
13541 :
13542 24 : case CODE_FOR_sse4_1_roundpd_vec_pack_sfix:
13543 24 : case CODE_FOR_sse4_1_roundps_sfix:
13544 24 : case CODE_FOR_avx_roundpd_vec_pack_sfix256:
13545 24 : case CODE_FOR_avx_roundps_sfix256:
13546 :
13547 24 : case CODE_FOR_sse4_1_blendps:
13548 24 : case CODE_FOR_avx_blendpd256:
13549 24 : case CODE_FOR_avx_vpermilv4df:
13550 24 : case CODE_FOR_avx_vpermilv4df_mask:
13551 24 : case CODE_FOR_avx512f_getmantv8df_mask:
13552 24 : case CODE_FOR_avx512f_getmantv16sf_mask:
13553 24 : case CODE_FOR_avx512vl_getmantv16hf_mask:
13554 24 : case CODE_FOR_avx512vl_getmantv8sf_mask:
13555 24 : case CODE_FOR_avx512vl_getmantv4df_mask:
13556 24 : case CODE_FOR_avx512fp16_getmantv8hf_mask:
13557 24 : case CODE_FOR_avx512vl_getmantv4sf_mask:
13558 24 : case CODE_FOR_avx512vl_getmantv2df_mask:
13559 24 : case CODE_FOR_avx512dq_rangepv8df_mask_round:
13560 24 : case CODE_FOR_avx512dq_rangepv16sf_mask_round:
13561 24 : case CODE_FOR_avx512dq_rangepv4df_mask:
13562 24 : case CODE_FOR_avx512dq_rangepv8sf_mask:
13563 24 : case CODE_FOR_avx512dq_rangepv2df_mask:
13564 24 : case CODE_FOR_avx512dq_rangepv4sf_mask:
13565 24 : case CODE_FOR_avx_shufpd256_mask:
13566 24 : error ("the last argument must be a 4-bit immediate");
13567 24 : return const0_rtx;
13568 :
13569 15 : case CODE_FOR_sha1rnds4:
13570 15 : case CODE_FOR_sse4_1_blendpd:
13571 15 : case CODE_FOR_avx_vpermilv2df:
13572 15 : case CODE_FOR_avx_vpermilv2df_mask:
13573 15 : case CODE_FOR_xop_vpermil2v2df3:
13574 15 : case CODE_FOR_xop_vpermil2v4sf3:
13575 15 : case CODE_FOR_xop_vpermil2v4df3:
13576 15 : case CODE_FOR_xop_vpermil2v8sf3:
13577 15 : case CODE_FOR_avx512f_vinsertf32x4_mask:
13578 15 : case CODE_FOR_avx512f_vinserti32x4_mask:
13579 15 : case CODE_FOR_avx512f_vextractf32x4_mask:
13580 15 : case CODE_FOR_avx512f_vextracti32x4_mask:
13581 15 : case CODE_FOR_sse2_shufpd:
13582 15 : case CODE_FOR_sse2_shufpd_mask:
13583 15 : case CODE_FOR_avx512dq_shuf_f64x2_mask:
13584 15 : case CODE_FOR_avx512dq_shuf_i64x2_mask:
13585 15 : case CODE_FOR_avx512vl_shuf_i32x4_mask:
13586 15 : case CODE_FOR_avx512vl_shuf_f32x4_mask:
13587 15 : error ("the last argument must be a 2-bit immediate");
13588 15 : return const0_rtx;
13589 :
13590 30 : case CODE_FOR_avx_vextractf128v4df:
13591 30 : case CODE_FOR_avx_vextractf128v8sf:
13592 30 : case CODE_FOR_avx_vextractf128v8si:
13593 30 : case CODE_FOR_avx_vinsertf128v4df:
13594 30 : case CODE_FOR_avx_vinsertf128v8sf:
13595 30 : case CODE_FOR_avx_vinsertf128v8si:
13596 30 : case CODE_FOR_avx512f_vinsertf64x4_mask:
13597 30 : case CODE_FOR_avx512f_vinserti64x4_mask:
13598 30 : case CODE_FOR_avx512f_vextractf64x4_mask:
13599 30 : case CODE_FOR_avx512f_vextracti64x4_mask:
13600 30 : case CODE_FOR_avx512dq_vinsertf32x8_mask:
13601 30 : case CODE_FOR_avx512dq_vinserti32x8_mask:
13602 30 : case CODE_FOR_avx512vl_vinsertv4df:
13603 30 : case CODE_FOR_avx512vl_vinsertv4di:
13604 30 : case CODE_FOR_avx512vl_vinsertv8sf:
13605 30 : case CODE_FOR_avx512vl_vinsertv8si:
13606 30 : error ("the last argument must be a 1-bit immediate");
13607 30 : return const0_rtx;
13608 :
13609 16 : case CODE_FOR_avx_vmcmpv2df3:
13610 16 : case CODE_FOR_avx_vmcmpv4sf3:
13611 16 : case CODE_FOR_avx_cmpv2df3:
13612 16 : case CODE_FOR_avx_cmpv4sf3:
13613 16 : if (CONST_INT_P (op) && IN_RANGE (INTVAL (op), 8, 31))
13614 : {
13615 4 : error ("'%s' needs isa option %s", d->name, "-mavx");
13616 4 : return const0_rtx;
13617 : }
13618 : /* FALLTHRU */
13619 18 : case CODE_FOR_avx_cmpv4df3:
13620 18 : case CODE_FOR_avx_cmpv8sf3:
13621 18 : case CODE_FOR_avx512f_cmpv8df3_mask:
13622 18 : case CODE_FOR_avx512f_cmpv16sf3_mask:
13623 18 : case CODE_FOR_avx512f_vmcmpv2df3_mask:
13624 18 : case CODE_FOR_avx512f_vmcmpv4sf3_mask:
13625 18 : case CODE_FOR_avx512bw_cmpv32hf3_mask:
13626 18 : case CODE_FOR_avx512vl_cmpv16hf3_mask:
13627 18 : case CODE_FOR_avx512fp16_cmpv8hf3_mask:
13628 18 : error ("the last argument must be a 5-bit immediate");
13629 18 : return const0_rtx;
13630 :
13631 132 : default:
13632 132 : switch (nargs_constant)
13633 : {
13634 8 : case 2:
13635 8 : if ((mask_pos && (nargs - i - mask_pos) == nargs_constant) ||
13636 8 : (!mask_pos && (nargs - i) == nargs_constant))
13637 : {
13638 4 : error ("the next to last argument must be an 8-bit immediate");
13639 4 : break;
13640 : }
13641 : /* FALLTHRU */
13642 128 : case 1:
13643 128 : error ("the last argument must be an 8-bit immediate");
13644 128 : break;
13645 0 : default:
13646 0 : gcc_unreachable ();
13647 : }
13648 132 : return const0_rtx;
13649 : }
13650 16197 : if ((icode == CODE_FOR_vunpackbv16qi_mask
13651 16197 : || icode == CODE_FOR_vunpackbv32qi_mask
13652 : || icode == CODE_FOR_vunpackbv64qi_mask)
13653 108 : && CONST_INT_P (op))
13654 : {
13655 108 : char val = INTVAL (op);
13656 108 : if ((val & 0xc0)
13657 108 : || (!(val & 0x18))
13658 107 : || ((val & 0x02) && ((val & 0x1c) != 0x08))
13659 102 : || ((val & 0x01) && (((val & 0x1c) >> 2) > 0x4)))
13660 : {
13661 9 : error ("the last argument must not use reserved value "
13662 : "immediate");
13663 9 : return const0_rtx;
13664 : }
13665 : }
13666 : }
13667 : else
13668 : {
13669 189142 : if (VECTOR_MODE_P (mode))
13670 136656 : op = safe_vector_operand (op, mode);
13671 :
13672 : /* If we aren't optimizing, only allow one memory operand to
13673 : be generated. */
13674 189142 : if (memory_operand (op, mode))
13675 : {
13676 30648 : num_memory++;
13677 30648 : if (!optimize && num_memory > 1)
13678 13873 : op = copy_to_mode_reg (mode, op);
13679 : }
13680 :
13681 189142 : if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
13682 : {
13683 187013 : if (!match)
13684 43549 : op = copy_to_mode_reg (mode, op);
13685 : }
13686 : else
13687 : {
13688 2129 : op = copy_to_reg (op);
13689 2129 : op = lowpart_subreg (mode, op, GET_MODE (op));
13690 : }
13691 : }
13692 :
13693 208219 : xops[i] = op;
13694 : }
13695 :
13696 64491 : switch (nargs)
13697 : {
13698 6636 : case 1:
13699 6636 : pat = GEN_FCN (icode) (real_target, xops[0]);
13700 6636 : break;
13701 5526 : case 2:
13702 5526 : pat = GEN_FCN (icode) (real_target, xops[0], xops[1]);
13703 5526 : break;
13704 21905 : case 3:
13705 21905 : pat = GEN_FCN (icode) (real_target, xops[0], xops[1], xops[2]);
13706 21905 : break;
13707 27684 : case 4:
13708 27684 : pat = GEN_FCN (icode) (real_target, xops[0], xops[1],
13709 27684 : xops[2], xops[3]);
13710 27684 : break;
13711 2740 : case 5:
13712 2740 : pat = GEN_FCN (icode) (real_target, xops[0], xops[1],
13713 2740 : xops[2], xops[3], xops[4]);
13714 2740 : break;
13715 : case 6:
13716 : pat = GEN_FCN (icode) (real_target, xops[0], xops[1],
13717 : xops[2], xops[3], xops[4], xops[5]);
13718 : break;
13719 : default:
13720 : gcc_unreachable ();
13721 : }
13722 :
13723 64491 : if (! pat)
13724 : return 0;
13725 :
13726 64491 : emit_insn (pat);
13727 64491 : return target;
13728 : }
13729 :
13730 : /* Transform pattern of following layout:
13731 : (set A
13732 : (unspec [B C] UNSPEC_EMBEDDED_ROUNDING))
13733 : )
13734 : into:
13735 : (set (A B)) */
13736 :
13737 : static rtx
13738 4958 : ix86_erase_embedded_rounding (rtx pat)
13739 : {
13740 4958 : if (NONJUMP_INSN_P (pat))
13741 680 : pat = PATTERN (pat);
13742 :
13743 4958 : gcc_assert (GET_CODE (pat) == SET);
13744 4958 : rtx src = SET_SRC (pat);
13745 4958 : gcc_assert (XVECLEN (src, 0) == 2);
13746 4958 : rtx p0 = XVECEXP (src, 0, 0);
13747 4958 : gcc_assert (GET_CODE (src) == UNSPEC
13748 : && XINT (src, 1) == UNSPEC_EMBEDDED_ROUNDING);
13749 4958 : rtx res = gen_rtx_SET (SET_DEST (pat), p0);
13750 4958 : return res;
13751 : }
13752 :
13753 : /* Subroutine of ix86_expand_round_builtin to take care of comi insns
13754 : with rounding. */
13755 : static rtx
13756 103 : ix86_expand_sse_comi_round (const struct builtin_description *d,
13757 : tree exp, rtx target, bool comx_ok)
13758 : {
13759 103 : rtx pat, set_dst;
13760 103 : tree arg0 = CALL_EXPR_ARG (exp, 0);
13761 103 : tree arg1 = CALL_EXPR_ARG (exp, 1);
13762 103 : tree arg2 = CALL_EXPR_ARG (exp, 2);
13763 103 : tree arg3 = CALL_EXPR_ARG (exp, 3);
13764 103 : rtx op0 = expand_normal (arg0);
13765 103 : rtx op1 = expand_normal (arg1);
13766 103 : rtx op2 = expand_normal (arg2);
13767 103 : rtx op3 = expand_normal (arg3);
13768 103 : enum insn_code icode = d->icode;
13769 103 : const struct insn_data_d *insn_p = &insn_data[icode];
13770 103 : machine_mode mode0 = insn_p->operand[0].mode;
13771 103 : machine_mode mode1 = insn_p->operand[1].mode;
13772 :
13773 : /* See avxintrin.h for values. */
13774 103 : static const enum rtx_code comparisons[32] =
13775 : {
13776 : EQ, LT, LE, UNORDERED, NE, UNGE, UNGT, ORDERED,
13777 : UNEQ, UNLT, UNLE, UNORDERED, LTGT, GE, GT, ORDERED,
13778 : EQ, LT, LE, UNORDERED, NE, UNGE, UNGT, ORDERED,
13779 : UNEQ, UNLT, UNLE, UNORDERED, LTGT, GE, GT, ORDERED
13780 : };
13781 103 : static const bool ordereds[32] =
13782 : {
13783 : true, true, true, false, false, false, false, true,
13784 : false, false, false, true, true, true, true, false,
13785 : true, true, true, false, false, false, false, true,
13786 : false, false, false, true, true, true, true, false
13787 : };
13788 103 : static const bool non_signalings[32] =
13789 : {
13790 : true, false, false, true, true, false, false, true,
13791 : true, false, false, true, true, false, false, true,
13792 : false, true, true, false, false, true, true, false,
13793 : false, true, true, false, false, true, true, false
13794 : };
13795 :
13796 103 : if (!CONST_INT_P (op2))
13797 : {
13798 0 : error ("the third argument must be comparison constant");
13799 0 : return const0_rtx;
13800 : }
13801 103 : if (INTVAL (op2) < 0 || INTVAL (op2) >= 32)
13802 : {
13803 0 : error ("incorrect comparison mode");
13804 0 : return const0_rtx;
13805 : }
13806 :
13807 103 : if (!insn_p->operand[2].predicate (op3, SImode))
13808 : {
13809 4 : error ("incorrect rounding operand");
13810 4 : return const0_rtx;
13811 : }
13812 :
13813 99 : if (VECTOR_MODE_P (mode0))
13814 99 : op0 = safe_vector_operand (op0, mode0);
13815 99 : if (VECTOR_MODE_P (mode1))
13816 99 : op1 = safe_vector_operand (op1, mode1);
13817 :
13818 99 : enum rtx_code comparison = comparisons[INTVAL (op2)];
13819 99 : enum rtx_code orig_comp = comparison;
13820 99 : bool ordered = ordereds[INTVAL (op2)];
13821 99 : bool non_signaling = non_signalings[INTVAL (op2)];
13822 99 : rtx const_val = const0_rtx;
13823 :
13824 99 : bool check_unordered = false;
13825 99 : machine_mode mode = CCFPmode;
13826 99 : switch (comparison)
13827 : {
13828 8 : case ORDERED:
13829 8 : if (!ordered)
13830 : {
13831 4 : if (TARGET_AVX10_2 && comx_ok)
13832 : {
13833 : /* Unlike VCOMI{SH,SS,SD}, VCOMX{SH,SS,SD} will set SF
13834 : differently. So directly return true here. */
13835 0 : target = gen_reg_rtx (SImode);
13836 0 : emit_move_insn (target, const1_rtx);
13837 0 : return target;
13838 : }
13839 : else
13840 : {
13841 : /* NB: Use CCSmode/NE for _CMP_TRUE_UQ/_CMP_TRUE_US. */
13842 : if (!non_signaling)
13843 99 : ordered = true;
13844 99 : mode = CCSmode;
13845 : }
13846 : }
13847 : else
13848 : {
13849 : /* NB: Use CCPmode/NE for _CMP_ORD_Q/_CMP_ORD_S. */
13850 : if (non_signaling)
13851 99 : ordered = false;
13852 : mode = CCPmode;
13853 : }
13854 : comparison = NE;
13855 : break;
13856 8 : case UNORDERED:
13857 8 : if (ordered)
13858 : {
13859 4 : if (TARGET_AVX10_2 && comx_ok)
13860 : {
13861 : /* Unlike VCOMI{SH,SS,SD}, VCOMX{SH,SS,SD} will set SF
13862 : differently. So directly return false here. */
13863 0 : target = gen_reg_rtx (SImode);
13864 0 : emit_move_insn (target, const0_rtx);
13865 0 : return target;
13866 : }
13867 : else
13868 : {
13869 : /* NB: Use CCSmode/EQ for _CMP_FALSE_OQ/_CMP_FALSE_OS. */
13870 : if (non_signaling)
13871 : ordered = false;
13872 : mode = CCSmode;
13873 : }
13874 : }
13875 : else
13876 : {
13877 : /* NB: Use CCPmode/NE for _CMP_UNORD_Q/_CMP_UNORD_S. */
13878 : if (!non_signaling)
13879 99 : ordered = true;
13880 99 : mode = CCPmode;
13881 : }
13882 : comparison = EQ;
13883 : break;
13884 :
13885 40 : case LE: /* -> GE */
13886 40 : case LT: /* -> GT */
13887 40 : case UNGE: /* -> UNLE */
13888 40 : case UNGT: /* -> UNLT */
13889 40 : std::swap (op0, op1);
13890 40 : comparison = swap_condition (comparison);
13891 : /* FALLTHRU */
13892 : case GT:
13893 : case GE:
13894 : case UNEQ:
13895 : case UNLT:
13896 : case UNLE:
13897 : case LTGT:
13898 : /* These are supported by CCFPmode. NB: Use ordered/signaling
13899 : COMI or unordered/non-signaling UCOMI. Both set ZF, PF, CF
13900 : with NAN operands. */
13901 : if (ordered == non_signaling)
13902 : ordered = !ordered;
13903 : break;
13904 : /* NB: COMI/UCOMI will set ZF with NAN operands. Use CCZmode for
13905 : _CMP_EQ_OQ/_CMP_EQ_OS.
13906 : Under TARGET_AVX10_2, VCOMX/VUCOMX are always generated instead
13907 : of COMI/UCOMI, VCOMX/VUCOMX will not set ZF with NAN. */
13908 8 : case EQ:
13909 8 : if (!TARGET_AVX10_2 || !comx_ok)
13910 5 : check_unordered = true;
13911 : mode = CCZmode;
13912 : break;
13913 7 : case NE:
13914 : /* NB: COMI/UCOMI will set ZF with NAN operands. Use CCZmode for
13915 : _CMP_NEQ_UQ/_CMP_NEQ_US.
13916 : Under TARGET_AVX10_2, VCOMX/VUCOMX are always generated instead
13917 : of COMI/UCOMI, VCOMX/VUCOMX will not set ZF with NAN. */
13918 7 : gcc_assert (!ordered);
13919 7 : if (!TARGET_AVX10_2 || !comx_ok)
13920 4 : check_unordered = true;
13921 7 : mode = CCZmode;
13922 7 : const_val = const1_rtx;
13923 7 : break;
13924 : default:
13925 : gcc_unreachable ();
13926 : }
13927 :
13928 99 : target = gen_reg_rtx (SImode);
13929 99 : emit_move_insn (target, const_val);
13930 99 : target = gen_rtx_SUBREG (QImode, target, 0);
13931 :
13932 93 : if ((optimize && !register_operand (op0, mode0))
13933 192 : || !insn_p->operand[0].predicate (op0, mode0))
13934 6 : op0 = copy_to_mode_reg (mode0, op0);
13935 93 : if ((optimize && !register_operand (op1, mode1))
13936 192 : || !insn_p->operand[1].predicate (op1, mode1))
13937 6 : op1 = copy_to_mode_reg (mode1, op1);
13938 :
13939 : /* Generate comx instead of comi when EQ/NE to avoid NAN checks.
13940 : Use orig_comp to exclude ORDERED/UNORDERED cases. */
13941 99 : if ((orig_comp == EQ || orig_comp == NE)
13942 15 : && TARGET_AVX10_2 && comx_ok)
13943 : {
13944 6 : switch (icode)
13945 : {
13946 : case CODE_FOR_avx512fp16_comi_round:
13947 99 : icode = CODE_FOR_avx10_2_comxhf_round;
13948 : break;
13949 4 : case CODE_FOR_sse_comi_round:
13950 4 : icode = CODE_FOR_avx10_2_comxsf_round;
13951 4 : break;
13952 2 : case CODE_FOR_sse2_comi_round:
13953 2 : icode = CODE_FOR_avx10_2_comxdf_round;
13954 2 : break;
13955 :
13956 : default:
13957 : break;
13958 : }
13959 : }
13960 :
13961 : /* Generate comi instead of comx when UNEQ/LTGT to avoid NAN checks. */
13962 99 : if ((comparison == UNEQ || comparison == LTGT)
13963 8 : && TARGET_AVX10_2 && comx_ok)
13964 : {
13965 0 : switch (icode)
13966 : {
13967 : case CODE_FOR_avx10_2_comxhf_round:
13968 99 : icode = CODE_FOR_avx512fp16_comi_round;
13969 : break;
13970 0 : case CODE_FOR_avx10_2_comxsf_round:
13971 0 : icode = CODE_FOR_sse_comi_round;
13972 0 : break;
13973 0 : case CODE_FOR_avx10_2_comxdf_round:
13974 0 : icode = CODE_FOR_sse2_comi_round;
13975 0 : break;
13976 :
13977 : default:
13978 : break;
13979 : }
13980 : }
13981 :
13982 : /*
13983 : 1. COMI/VCOMX: ordered and signaling.
13984 : 2. UCOMI/VUCOMX: unordered and non-signaling.
13985 : */
13986 99 : if (non_signaling)
13987 38 : switch (icode)
13988 : {
13989 : case CODE_FOR_sse_comi_round:
13990 : icode = CODE_FOR_sse_ucomi_round;
13991 : break;
13992 17 : case CODE_FOR_sse2_comi_round:
13993 17 : icode = CODE_FOR_sse2_ucomi_round;
13994 17 : break;
13995 0 : case CODE_FOR_avx512fp16_comi_round:
13996 0 : icode = CODE_FOR_avx512fp16_ucomi_round;
13997 0 : break;
13998 3 : case CODE_FOR_avx10_2_comxsf_round:
13999 3 : icode = CODE_FOR_avx10_2_ucomxsf_round;
14000 3 : break;
14001 0 : case CODE_FOR_avx10_2_comxhf_round:
14002 0 : icode = CODE_FOR_avx10_2_ucomxhf_round;
14003 0 : break;
14004 1 : case CODE_FOR_avx10_2_comxdf_round:
14005 1 : icode = CODE_FOR_avx10_2_ucomxdf_round;
14006 1 : break;
14007 0 : default:
14008 0 : gcc_unreachable ();
14009 : }
14010 :
14011 99 : pat = GEN_FCN (icode) (op0, op1, op3);
14012 99 : if (! pat)
14013 : return 0;
14014 :
14015 : /* Rounding operand can be either NO_ROUND or ROUND_SAE at this point. */
14016 99 : if (INTVAL (op3) == NO_ROUND)
14017 : {
14018 1 : pat = ix86_erase_embedded_rounding (pat);
14019 1 : if (! pat)
14020 : return 0;
14021 :
14022 1 : set_dst = SET_DEST (pat);
14023 : }
14024 : else
14025 : {
14026 98 : gcc_assert (GET_CODE (pat) == SET);
14027 98 : set_dst = SET_DEST (pat);
14028 : }
14029 :
14030 99 : emit_insn (pat);
14031 :
14032 99 : return ix86_ssecom_setcc (comparison, check_unordered, mode,
14033 99 : set_dst, target);
14034 : }
14035 :
14036 : static rtx
14037 15660 : ix86_expand_round_builtin (const struct builtin_description *d,
14038 : tree exp, rtx target)
14039 : {
14040 15660 : rtx pat;
14041 15660 : unsigned int i, nargs;
14042 15660 : rtx xops[6];
14043 15660 : enum insn_code icode = d->icode;
14044 15660 : const struct insn_data_d *insn_p = &insn_data[icode];
14045 15660 : machine_mode tmode = insn_p->operand[0].mode;
14046 15660 : unsigned int nargs_constant = 0;
14047 15660 : unsigned int redundant_embed_rnd = 0;
14048 :
14049 15660 : switch ((enum ix86_builtin_func_type) d->flag)
14050 : {
14051 : case UINT64_FTYPE_V2DF_INT:
14052 : case UINT64_FTYPE_V4SF_INT:
14053 : case UINT64_FTYPE_V8HF_INT:
14054 : case UINT_FTYPE_V2DF_INT:
14055 : case UINT_FTYPE_V4SF_INT:
14056 : case UINT_FTYPE_V8HF_INT:
14057 : case INT64_FTYPE_V2DF_INT:
14058 : case INT64_FTYPE_V4SF_INT:
14059 : case INT64_FTYPE_V8HF_INT:
14060 : case INT_FTYPE_V2DF_INT:
14061 : case INT_FTYPE_V4SF_INT:
14062 : case INT_FTYPE_V8HF_INT:
14063 : nargs = 2;
14064 : break;
14065 637 : case V32HF_FTYPE_V32HF_V32HF_INT:
14066 637 : case V8HF_FTYPE_V8HF_V8HF_INT:
14067 637 : case V8HF_FTYPE_V8HF_INT_INT:
14068 637 : case V8HF_FTYPE_V8HF_UINT_INT:
14069 637 : case V8HF_FTYPE_V8HF_INT64_INT:
14070 637 : case V8HF_FTYPE_V8HF_UINT64_INT:
14071 637 : case V4SF_FTYPE_V4SF_UINT_INT:
14072 637 : case V4SF_FTYPE_V4SF_UINT64_INT:
14073 637 : case V2DF_FTYPE_V2DF_UINT64_INT:
14074 637 : case V4SF_FTYPE_V4SF_INT_INT:
14075 637 : case V4SF_FTYPE_V4SF_INT64_INT:
14076 637 : case V2DF_FTYPE_V2DF_INT64_INT:
14077 637 : case V4SF_FTYPE_V4SF_V4SF_INT:
14078 637 : case V2DF_FTYPE_V2DF_V2DF_INT:
14079 637 : case V4SF_FTYPE_V4SF_V2DF_INT:
14080 637 : case V2DF_FTYPE_V2DF_V4SF_INT:
14081 637 : nargs = 3;
14082 637 : break;
14083 4554 : case V8SF_FTYPE_V8DF_V8SF_QI_INT:
14084 4554 : case V8DF_FTYPE_V8DF_V8DF_QI_INT:
14085 4554 : case V32HI_FTYPE_V32HF_V32HI_USI_INT:
14086 4554 : case V32HI_FTYPE_V32BF_V32HI_USI_INT:
14087 4554 : case V8SI_FTYPE_V8DF_V8SI_QI_INT:
14088 4554 : case V8DI_FTYPE_V8HF_V8DI_UQI_INT:
14089 4554 : case V8DI_FTYPE_V8DF_V8DI_QI_INT:
14090 4554 : case V8SF_FTYPE_V8DI_V8SF_QI_INT:
14091 4554 : case V8DF_FTYPE_V8DI_V8DF_QI_INT:
14092 4554 : case V8DF_FTYPE_V8HF_V8DF_UQI_INT:
14093 4554 : case V16SF_FTYPE_V16HF_V16SF_UHI_INT:
14094 4554 : case V32HF_FTYPE_V32HI_V32HF_USI_INT:
14095 4554 : case V32HF_FTYPE_V32HF_V32HF_USI_INT:
14096 4554 : case V32HF_FTYPE_V32HF_V32HF_V32HF_INT:
14097 4554 : case V16SF_FTYPE_V16SF_V16SF_HI_INT:
14098 4554 : case V8DI_FTYPE_V8SF_V8DI_QI_INT:
14099 4554 : case V16SF_FTYPE_V16SI_V16SF_HI_INT:
14100 4554 : case V16SI_FTYPE_V16SF_V16SI_HI_INT:
14101 4554 : case V16SI_FTYPE_V16SF_V16SI_UHI_INT:
14102 4554 : case V16SI_FTYPE_V16HF_V16SI_UHI_INT:
14103 4554 : case V16HF_FTYPE_V16SI_V16HF_UHI_INT:
14104 4554 : case V8DF_FTYPE_V8SF_V8DF_QI_INT:
14105 4554 : case V16SF_FTYPE_V16HI_V16SF_HI_INT:
14106 4554 : case V2DF_FTYPE_V2DF_V2DF_V2DF_INT:
14107 4554 : case V4SF_FTYPE_V4SF_V4SF_V4SF_INT:
14108 4554 : case V8HF_FTYPE_V8DI_V8HF_UQI_INT:
14109 4554 : case V8HF_FTYPE_V8DF_V8HF_UQI_INT:
14110 4554 : case V16HF_FTYPE_V16SF_V16HF_UHI_INT:
14111 4554 : case V16HI_FTYPE_V16BF_V16HI_UHI_INT:
14112 4554 : case V8HF_FTYPE_V8HF_V8HF_V8HF_INT:
14113 4554 : nargs = 4;
14114 4554 : break;
14115 166 : case V4SF_FTYPE_V4SF_V4SF_INT_INT:
14116 166 : case V2DF_FTYPE_V2DF_V2DF_INT_INT:
14117 166 : nargs_constant = 2;
14118 166 : nargs = 4;
14119 166 : break;
14120 103 : case INT_FTYPE_V4SF_V4SF_INT_INT:
14121 103 : case INT_FTYPE_V2DF_V2DF_INT_INT:
14122 103 : return ix86_expand_sse_comi_round (d, exp, target, true);
14123 6226 : case V8DF_FTYPE_V8DF_V8DF_V8DF_UQI_INT:
14124 6226 : case V2DF_FTYPE_V2DF_V2DF_V2DF_UQI_INT:
14125 6226 : case V4SF_FTYPE_V4SF_V4SF_V4SF_UQI_INT:
14126 6226 : case V4SF_FTYPE_V8HF_V4SF_V4SF_UQI_INT:
14127 6226 : case V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT:
14128 6226 : case V32HF_FTYPE_V32HF_V32HF_V32HF_UHI_INT:
14129 6226 : case V32HF_FTYPE_V32HF_V32HF_V32HF_USI_INT:
14130 6226 : case V2DF_FTYPE_V8HF_V2DF_V2DF_UQI_INT:
14131 6226 : case V2DF_FTYPE_V2DF_V2DF_V2DF_QI_INT:
14132 6226 : case V2DF_FTYPE_V2DF_V4SF_V2DF_QI_INT:
14133 6226 : case V2DF_FTYPE_V2DF_V4SF_V2DF_UQI_INT:
14134 6226 : case V4SF_FTYPE_V4SF_V4SF_V4SF_QI_INT:
14135 6226 : case V4SF_FTYPE_V4SF_V2DF_V4SF_QI_INT:
14136 6226 : case V4SF_FTYPE_V4SF_V2DF_V4SF_UQI_INT:
14137 6226 : case V8HF_FTYPE_V8HF_V8HF_V8HF_UQI_INT:
14138 6226 : case V8HF_FTYPE_V2DF_V8HF_V8HF_UQI_INT:
14139 6226 : case V8HF_FTYPE_V4SF_V8HF_V8HF_UQI_INT:
14140 6226 : case V32HF_FTYPE_V16SF_V16SF_V32HF_USI_INT:
14141 6226 : nargs = 5;
14142 6226 : break;
14143 635 : case V32HF_FTYPE_V32HF_INT_V32HF_USI_INT:
14144 635 : case V16SF_FTYPE_V16SF_INT_V16SF_HI_INT:
14145 635 : case V8DF_FTYPE_V8DF_INT_V8DF_QI_INT:
14146 635 : case V8DF_FTYPE_V8DF_INT_V8DF_UQI_INT:
14147 635 : case V16SF_FTYPE_V16SF_INT_V16SF_UHI_INT:
14148 635 : nargs_constant = 4;
14149 635 : nargs = 5;
14150 635 : break;
14151 1173 : case UQI_FTYPE_V8DF_V8DF_INT_UQI_INT:
14152 1173 : case UQI_FTYPE_V2DF_V2DF_INT_UQI_INT:
14153 1173 : case UHI_FTYPE_V16SF_V16SF_INT_UHI_INT:
14154 1173 : case UQI_FTYPE_V4SF_V4SF_INT_UQI_INT:
14155 1173 : case USI_FTYPE_V32HF_V32HF_INT_USI_INT:
14156 1173 : case UQI_FTYPE_V8HF_V8HF_INT_UQI_INT:
14157 1173 : nargs_constant = 3;
14158 1173 : nargs = 5;
14159 1173 : break;
14160 1071 : case V16SF_FTYPE_V16SF_V16SF_INT_V16SF_HI_INT:
14161 1071 : case V8DF_FTYPE_V8DF_V8DF_INT_V8DF_QI_INT:
14162 1071 : case V4SF_FTYPE_V4SF_V4SF_INT_V4SF_QI_INT:
14163 1071 : case V2DF_FTYPE_V2DF_V2DF_INT_V2DF_QI_INT:
14164 1071 : case V2DF_FTYPE_V2DF_V2DF_INT_V2DF_UQI_INT:
14165 1071 : case V4SF_FTYPE_V4SF_V4SF_INT_V4SF_UQI_INT:
14166 1071 : case V8HF_FTYPE_V8HF_V8HF_INT_V8HF_UQI_INT:
14167 1071 : case V8DF_FTYPE_V8DF_V8DF_INT_V8DF_UQI_INT:
14168 1071 : case V32HF_FTYPE_V32HF_V32HF_INT_V32HF_USI_INT:
14169 1071 : case V16SF_FTYPE_V16SF_V16SF_INT_V16SF_UHI_INT:
14170 1071 : nargs = 6;
14171 1071 : nargs_constant = 4;
14172 1071 : break;
14173 252 : case V8DF_FTYPE_V8DF_V8DF_V8DI_INT_QI_INT:
14174 252 : case V16SF_FTYPE_V16SF_V16SF_V16SI_INT_HI_INT:
14175 252 : case V2DF_FTYPE_V2DF_V2DF_V2DI_INT_QI_INT:
14176 252 : case V4SF_FTYPE_V4SF_V4SF_V4SI_INT_QI_INT:
14177 252 : nargs = 6;
14178 252 : nargs_constant = 3;
14179 252 : break;
14180 0 : default:
14181 0 : gcc_unreachable ();
14182 : }
14183 14714 : gcc_assert (nargs <= ARRAY_SIZE (xops));
14184 :
14185 15557 : if (optimize
14186 4293 : || target == 0
14187 4293 : || GET_MODE (target) != tmode
14188 19850 : || !insn_p->operand[0].predicate (target, tmode))
14189 11264 : target = gen_reg_rtx (tmode);
14190 :
14191 85491 : for (i = 0; i < nargs; i++)
14192 : {
14193 70489 : tree arg = CALL_EXPR_ARG (exp, i);
14194 70489 : rtx op = ix86_expand_unsigned_small_int_cst_argument (arg);
14195 70489 : machine_mode mode = insn_p->operand[i + 1].mode;
14196 70489 : bool match = insn_p->operand[i + 1].predicate (op, mode);
14197 :
14198 70489 : if (i == nargs - nargs_constant)
14199 : {
14200 3297 : if (!match)
14201 : {
14202 40 : switch (icode)
14203 : {
14204 12 : case CODE_FOR_avx512f_getmantv8df_mask_round:
14205 12 : case CODE_FOR_avx512f_getmantv16sf_mask_round:
14206 12 : case CODE_FOR_avx512bw_getmantv32hf_mask_round:
14207 12 : case CODE_FOR_avx512f_vgetmantv2df_round:
14208 12 : case CODE_FOR_avx512f_vgetmantv2df_mask_round:
14209 12 : case CODE_FOR_avx512f_vgetmantv4sf_round:
14210 12 : case CODE_FOR_avx512f_vgetmantv4sf_mask_round:
14211 12 : case CODE_FOR_avx512f_vgetmantv8hf_mask_round:
14212 12 : error ("the immediate argument must be a 4-bit immediate");
14213 12 : return const0_rtx;
14214 8 : case CODE_FOR_avx512f_cmpv8df3_mask_round:
14215 8 : case CODE_FOR_avx512f_cmpv16sf3_mask_round:
14216 8 : case CODE_FOR_avx512f_vmcmpv2df3_mask_round:
14217 8 : case CODE_FOR_avx512f_vmcmpv4sf3_mask_round:
14218 8 : case CODE_FOR_avx512f_vmcmpv8hf3_mask_round:
14219 8 : case CODE_FOR_avx512bw_cmpv32hf3_mask_round:
14220 8 : error ("the immediate argument must be a 5-bit immediate");
14221 8 : return const0_rtx;
14222 20 : default:
14223 20 : error ("the immediate argument must be an 8-bit immediate");
14224 20 : return const0_rtx;
14225 : }
14226 : }
14227 : }
14228 67192 : else if (i == nargs-1)
14229 : {
14230 15517 : if (!insn_p->operand[nargs].predicate (op, SImode))
14231 : {
14232 515 : error ("incorrect rounding operand");
14233 515 : return const0_rtx;
14234 : }
14235 :
14236 : /* If there is no rounding use normal version of the pattern. */
14237 15002 : if (INTVAL (op) == NO_ROUND)
14238 : {
14239 : /* Skip erasing embedded rounding for below expanders who
14240 : generates multiple insns. In ix86_erase_embedded_rounding
14241 : the pattern will be transformed to a single set, and emit_insn
14242 : appends the set instead of insert it to chain. So the insns
14243 : emitted inside define_expander would be ignored. */
14244 4989 : switch (icode)
14245 : {
14246 : case CODE_FOR_avx512bw_fmaddc_v32hf_mask1_round:
14247 : case CODE_FOR_avx512bw_fcmaddc_v32hf_mask1_round:
14248 : case CODE_FOR_avx512fp16_fmaddcsh_v8hf_mask1_round:
14249 : case CODE_FOR_avx512fp16_fcmaddcsh_v8hf_mask1_round:
14250 : case CODE_FOR_avx512fp16_fmaddcsh_v8hf_mask3_round:
14251 : case CODE_FOR_avx512fp16_fcmaddcsh_v8hf_mask3_round:
14252 : redundant_embed_rnd = 0;
14253 : break;
14254 4957 : default:
14255 4957 : redundant_embed_rnd = 1;
14256 4957 : break;
14257 : }
14258 : }
14259 : }
14260 : else
14261 : {
14262 51675 : if (VECTOR_MODE_P (mode))
14263 37773 : op = safe_vector_operand (op, mode);
14264 :
14265 51675 : op = fixup_modeless_constant (op, mode);
14266 :
14267 51675 : if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
14268 : {
14269 51675 : if (optimize || !match)
14270 45347 : op = copy_to_mode_reg (mode, op);
14271 : }
14272 : else
14273 : {
14274 0 : op = copy_to_reg (op);
14275 0 : op = lowpart_subreg (mode, op, GET_MODE (op));
14276 : }
14277 : }
14278 :
14279 69934 : xops[i] = op;
14280 : }
14281 :
14282 15002 : switch (nargs)
14283 : {
14284 : case 1:
14285 : pat = GEN_FCN (icode) (target, xops[0]);
14286 : break;
14287 810 : case 2:
14288 810 : pat = GEN_FCN (icode) (target, xops[0], xops[1]);
14289 810 : break;
14290 593 : case 3:
14291 593 : pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2]);
14292 593 : break;
14293 4596 : case 4:
14294 4596 : pat = GEN_FCN (icode) (target, xops[0], xops[1],
14295 4596 : xops[2], xops[3]);
14296 4596 : break;
14297 7730 : case 5:
14298 7730 : pat = GEN_FCN (icode) (target, xops[0], xops[1],
14299 7730 : xops[2], xops[3], xops[4]);
14300 7730 : break;
14301 1273 : case 6:
14302 1273 : pat = GEN_FCN (icode) (target, xops[0], xops[1],
14303 1273 : xops[2], xops[3], xops[4], xops[5]);
14304 1273 : break;
14305 : default:
14306 : gcc_unreachable ();
14307 : }
14308 :
14309 15002 : if (!pat)
14310 : return 0;
14311 :
14312 15002 : if (redundant_embed_rnd)
14313 4957 : pat = ix86_erase_embedded_rounding (pat);
14314 :
14315 15002 : emit_insn (pat);
14316 15002 : return target;
14317 : }
14318 :
14319 : /* Subroutine of ix86_expand_builtin to take care of special insns
14320 : with variable number of operands. */
14321 :
14322 : static rtx
14323 27160 : ix86_expand_special_args_builtin (const struct builtin_description *d,
14324 : tree exp, rtx target)
14325 : {
14326 27160 : tree arg;
14327 27160 : rtx pat, op;
14328 27160 : unsigned int i, nargs, arg_adjust, memory;
14329 27160 : unsigned int constant = 100;
14330 27160 : bool aligned_mem = false;
14331 27160 : rtx xops[4];
14332 27160 : enum insn_code icode = d->icode;
14333 27160 : const struct insn_data_d *insn_p = &insn_data[icode];
14334 27160 : machine_mode tmode = insn_p->operand[0].mode;
14335 27160 : enum { load, store } klass;
14336 :
14337 27160 : switch ((enum ix86_builtin_func_type) d->flag)
14338 : {
14339 15382 : case VOID_FTYPE_VOID:
14340 15382 : emit_insn (GEN_FCN (icode) (target));
14341 15382 : return 0;
14342 : case VOID_FTYPE_UINT64:
14343 : case VOID_FTYPE_UNSIGNED:
14344 : nargs = 0;
14345 : klass = store;
14346 : memory = 0;
14347 : break;
14348 :
14349 7510 : case INT_FTYPE_VOID:
14350 7510 : case USHORT_FTYPE_VOID:
14351 7510 : case UINT64_FTYPE_VOID:
14352 7510 : case UINT_FTYPE_VOID:
14353 7510 : case UINT8_FTYPE_VOID:
14354 7510 : case UNSIGNED_FTYPE_VOID:
14355 7510 : nargs = 0;
14356 7510 : klass = load;
14357 7510 : memory = 0;
14358 7510 : break;
14359 360 : case CHAR_FTYPE_PCCHAR:
14360 360 : case SHORT_FTYPE_PCSHORT:
14361 360 : case INT_FTYPE_PCINT:
14362 360 : case INT64_FTYPE_PCINT64:
14363 360 : case UINT64_FTYPE_PUNSIGNED:
14364 360 : case V2DI_FTYPE_PV2DI:
14365 360 : case V4DI_FTYPE_PV4DI:
14366 360 : case V32QI_FTYPE_PCCHAR:
14367 360 : case V16QI_FTYPE_PCCHAR:
14368 360 : case V8SF_FTYPE_PCV4SF:
14369 360 : case V8SF_FTYPE_PCFLOAT:
14370 360 : case V4SF_FTYPE_PCFLOAT:
14371 360 : case V4SF_FTYPE_PCFLOAT16:
14372 360 : case V4SF_FTYPE_PCBFLOAT16:
14373 360 : case V4SF_FTYPE_PCV8BF:
14374 360 : case V4SF_FTYPE_PCV8HF:
14375 360 : case V8SF_FTYPE_PCFLOAT16:
14376 360 : case V8SF_FTYPE_PCBFLOAT16:
14377 360 : case V8SF_FTYPE_PCV16HF:
14378 360 : case V8SF_FTYPE_PCV16BF:
14379 360 : case V4DF_FTYPE_PCV2DF:
14380 360 : case V4DF_FTYPE_PCDOUBLE:
14381 360 : case V2DF_FTYPE_PCDOUBLE:
14382 360 : case VOID_FTYPE_PVOID:
14383 360 : case V8DI_FTYPE_PV8DI:
14384 360 : nargs = 1;
14385 360 : klass = load;
14386 360 : memory = 0;
14387 360 : switch (icode)
14388 : {
14389 : case CODE_FOR_sse4_1_movntdqa:
14390 : case CODE_FOR_avx2_movntdqa:
14391 : case CODE_FOR_avx512f_movntdqa:
14392 : aligned_mem = true;
14393 : break;
14394 : default:
14395 : break;
14396 : }
14397 : break;
14398 373 : case VOID_FTYPE_PV2SF_V4SF:
14399 373 : case VOID_FTYPE_PV8DI_V8DI:
14400 373 : case VOID_FTYPE_PV4DI_V4DI:
14401 373 : case VOID_FTYPE_PV2DI_V2DI:
14402 373 : case VOID_FTYPE_PCHAR_V32QI:
14403 373 : case VOID_FTYPE_PCHAR_V16QI:
14404 373 : case VOID_FTYPE_PFLOAT_V16SF:
14405 373 : case VOID_FTYPE_PFLOAT_V8SF:
14406 373 : case VOID_FTYPE_PFLOAT_V4SF:
14407 373 : case VOID_FTYPE_PDOUBLE_V8DF:
14408 373 : case VOID_FTYPE_PDOUBLE_V4DF:
14409 373 : case VOID_FTYPE_PDOUBLE_V2DF:
14410 373 : case VOID_FTYPE_PLONGLONG_LONGLONG:
14411 373 : case VOID_FTYPE_PULONGLONG_ULONGLONG:
14412 373 : case VOID_FTYPE_PUNSIGNED_UNSIGNED:
14413 373 : case VOID_FTYPE_PINT_INT:
14414 373 : nargs = 1;
14415 373 : klass = store;
14416 : /* Reserve memory operand for target. */
14417 373 : memory = ARRAY_SIZE (xops);
14418 373 : switch (icode)
14419 : {
14420 : /* These builtins and instructions require the memory
14421 : to be properly aligned. */
14422 : case CODE_FOR_avx_movntv4di:
14423 : case CODE_FOR_sse2_movntv2di:
14424 : case CODE_FOR_avx_movntv8sf:
14425 : case CODE_FOR_sse_movntv4sf:
14426 : case CODE_FOR_sse4a_vmmovntv4sf:
14427 : case CODE_FOR_avx_movntv4df:
14428 : case CODE_FOR_sse2_movntv2df:
14429 : case CODE_FOR_sse4a_vmmovntv2df:
14430 : case CODE_FOR_sse2_movntidi:
14431 : case CODE_FOR_sse_movntq:
14432 : case CODE_FOR_sse2_movntisi:
14433 : case CODE_FOR_avx512f_movntv16sf:
14434 : case CODE_FOR_avx512f_movntv8df:
14435 : case CODE_FOR_avx512f_movntv8di:
14436 : aligned_mem = true;
14437 : break;
14438 : default:
14439 : break;
14440 : }
14441 : break;
14442 0 : case VOID_FTYPE_PVOID_PCVOID:
14443 0 : nargs = 1;
14444 0 : klass = store;
14445 0 : memory = 0;
14446 :
14447 0 : break;
14448 26 : case V4SF_FTYPE_V4SF_PCV2SF:
14449 26 : case V2DF_FTYPE_V2DF_PCDOUBLE:
14450 26 : nargs = 2;
14451 26 : klass = load;
14452 26 : memory = 1;
14453 26 : break;
14454 93 : case V8SF_FTYPE_PCV8SF_V8SI:
14455 93 : case V4DF_FTYPE_PCV4DF_V4DI:
14456 93 : case V4SF_FTYPE_PCV4SF_V4SI:
14457 93 : case V2DF_FTYPE_PCV2DF_V2DI:
14458 93 : case V8SI_FTYPE_PCV8SI_V8SI:
14459 93 : case V4DI_FTYPE_PCV4DI_V4DI:
14460 93 : case V4SI_FTYPE_PCV4SI_V4SI:
14461 93 : case V2DI_FTYPE_PCV2DI_V2DI:
14462 93 : case VOID_FTYPE_INT_INT64:
14463 93 : nargs = 2;
14464 93 : klass = load;
14465 93 : memory = 0;
14466 93 : break;
14467 360 : case VOID_FTYPE_PV8DF_V8DF_UQI:
14468 360 : case VOID_FTYPE_PV4DF_V4DF_UQI:
14469 360 : case VOID_FTYPE_PV2DF_V2DF_UQI:
14470 360 : case VOID_FTYPE_PV16SF_V16SF_UHI:
14471 360 : case VOID_FTYPE_PV8SF_V8SF_UQI:
14472 360 : case VOID_FTYPE_PV4SF_V4SF_UQI:
14473 360 : case VOID_FTYPE_PV8DI_V8DI_UQI:
14474 360 : case VOID_FTYPE_PV4DI_V4DI_UQI:
14475 360 : case VOID_FTYPE_PV2DI_V2DI_UQI:
14476 360 : case VOID_FTYPE_PV16SI_V16SI_UHI:
14477 360 : case VOID_FTYPE_PV8SI_V8SI_UQI:
14478 360 : case VOID_FTYPE_PV4SI_V4SI_UQI:
14479 360 : case VOID_FTYPE_PV64QI_V64QI_UDI:
14480 360 : case VOID_FTYPE_PV32HI_V32HI_USI:
14481 360 : case VOID_FTYPE_PV32QI_V32QI_USI:
14482 360 : case VOID_FTYPE_PV16QI_V16QI_UHI:
14483 360 : case VOID_FTYPE_PV16HI_V16HI_UHI:
14484 360 : case VOID_FTYPE_PV8HI_V8HI_UQI:
14485 360 : switch (icode)
14486 : {
14487 : /* These builtins and instructions require the memory
14488 : to be properly aligned. */
14489 144 : case CODE_FOR_avx512f_storev16sf_mask:
14490 144 : case CODE_FOR_avx512f_storev16si_mask:
14491 144 : case CODE_FOR_avx512f_storev8df_mask:
14492 144 : case CODE_FOR_avx512f_storev8di_mask:
14493 144 : case CODE_FOR_avx512vl_storev8sf_mask:
14494 144 : case CODE_FOR_avx512vl_storev8si_mask:
14495 144 : case CODE_FOR_avx512vl_storev4df_mask:
14496 144 : case CODE_FOR_avx512vl_storev4di_mask:
14497 144 : case CODE_FOR_avx512vl_storev4sf_mask:
14498 144 : case CODE_FOR_avx512vl_storev4si_mask:
14499 144 : case CODE_FOR_avx512vl_storev2df_mask:
14500 144 : case CODE_FOR_avx512vl_storev2di_mask:
14501 144 : aligned_mem = true;
14502 144 : break;
14503 : default:
14504 : break;
14505 : }
14506 : /* FALLTHRU */
14507 : case VOID_FTYPE_PV8SF_V8SI_V8SF:
14508 : case VOID_FTYPE_PV4DF_V4DI_V4DF:
14509 : case VOID_FTYPE_PV4SF_V4SI_V4SF:
14510 : case VOID_FTYPE_PV2DF_V2DI_V2DF:
14511 : case VOID_FTYPE_PV8SI_V8SI_V8SI:
14512 : case VOID_FTYPE_PV4DI_V4DI_V4DI:
14513 : case VOID_FTYPE_PV4SI_V4SI_V4SI:
14514 : case VOID_FTYPE_PV2DI_V2DI_V2DI:
14515 : case VOID_FTYPE_PV8SI_V8DI_UQI:
14516 : case VOID_FTYPE_PV8HI_V8DI_UQI:
14517 : case VOID_FTYPE_PV16HI_V16SI_UHI:
14518 : case VOID_FTYPE_PUDI_V8DI_UQI:
14519 : case VOID_FTYPE_PV16QI_V16SI_UHI:
14520 : case VOID_FTYPE_PV4SI_V4DI_UQI:
14521 : case VOID_FTYPE_PUDI_V2DI_UQI:
14522 : case VOID_FTYPE_PUDI_V4DI_UQI:
14523 : case VOID_FTYPE_PUSI_V2DI_UQI:
14524 : case VOID_FTYPE_PV8HI_V8SI_UQI:
14525 : case VOID_FTYPE_PUDI_V4SI_UQI:
14526 : case VOID_FTYPE_PUSI_V4DI_UQI:
14527 : case VOID_FTYPE_PUHI_V2DI_UQI:
14528 : case VOID_FTYPE_PUDI_V8SI_UQI:
14529 : case VOID_FTYPE_PUSI_V4SI_UQI:
14530 : case VOID_FTYPE_PCHAR_V64QI_UDI:
14531 : case VOID_FTYPE_PCHAR_V32QI_USI:
14532 : case VOID_FTYPE_PCHAR_V16QI_UHI:
14533 : case VOID_FTYPE_PSHORT_V32HI_USI:
14534 : case VOID_FTYPE_PSHORT_V16HI_UHI:
14535 : case VOID_FTYPE_PSHORT_V8HI_UQI:
14536 : case VOID_FTYPE_PINT_V16SI_UHI:
14537 : case VOID_FTYPE_PINT_V8SI_UQI:
14538 : case VOID_FTYPE_PINT_V4SI_UQI:
14539 : case VOID_FTYPE_PINT64_V8DI_UQI:
14540 : case VOID_FTYPE_PINT64_V4DI_UQI:
14541 : case VOID_FTYPE_PINT64_V2DI_UQI:
14542 : case VOID_FTYPE_PDOUBLE_V8DF_UQI:
14543 : case VOID_FTYPE_PDOUBLE_V4DF_UQI:
14544 : case VOID_FTYPE_PDOUBLE_V2DF_UQI:
14545 : case VOID_FTYPE_PFLOAT_V16SF_UHI:
14546 : case VOID_FTYPE_PFLOAT_V8SF_UQI:
14547 : case VOID_FTYPE_PFLOAT_V4SF_UQI:
14548 : case VOID_FTYPE_PCFLOAT16_V8HF_UQI:
14549 : case VOID_FTYPE_PV32QI_V32HI_USI:
14550 : case VOID_FTYPE_PV16QI_V16HI_UHI:
14551 : case VOID_FTYPE_PUDI_V8HI_UQI:
14552 : nargs = 2;
14553 : klass = store;
14554 : /* Reserve memory operand for target. */
14555 : memory = ARRAY_SIZE (xops);
14556 : break;
14557 1243 : case V4SF_FTYPE_PCV4SF_V4SF_UQI:
14558 1243 : case V8SF_FTYPE_PCV8SF_V8SF_UQI:
14559 1243 : case V16SF_FTYPE_PCV16SF_V16SF_UHI:
14560 1243 : case V4SI_FTYPE_PCV4SI_V4SI_UQI:
14561 1243 : case V8SI_FTYPE_PCV8SI_V8SI_UQI:
14562 1243 : case V16SI_FTYPE_PCV16SI_V16SI_UHI:
14563 1243 : case V2DF_FTYPE_PCV2DF_V2DF_UQI:
14564 1243 : case V4DF_FTYPE_PCV4DF_V4DF_UQI:
14565 1243 : case V8DF_FTYPE_PCV8DF_V8DF_UQI:
14566 1243 : case V2DI_FTYPE_PCV2DI_V2DI_UQI:
14567 1243 : case V4DI_FTYPE_PCV4DI_V4DI_UQI:
14568 1243 : case V8DI_FTYPE_PCV8DI_V8DI_UQI:
14569 1243 : case V64QI_FTYPE_PCV64QI_V64QI_UDI:
14570 1243 : case V32HI_FTYPE_PCV32HI_V32HI_USI:
14571 1243 : case V32QI_FTYPE_PCV32QI_V32QI_USI:
14572 1243 : case V16QI_FTYPE_PCV16QI_V16QI_UHI:
14573 1243 : case V16HI_FTYPE_PCV16HI_V16HI_UHI:
14574 1243 : case V8HI_FTYPE_PCV8HI_V8HI_UQI:
14575 1243 : switch (icode)
14576 : {
14577 : /* These builtins and instructions require the memory
14578 : to be properly aligned. */
14579 288 : case CODE_FOR_avx512f_loadv16sf_mask:
14580 288 : case CODE_FOR_avx512f_loadv16si_mask:
14581 288 : case CODE_FOR_avx512f_loadv8df_mask:
14582 288 : case CODE_FOR_avx512f_loadv8di_mask:
14583 288 : case CODE_FOR_avx512vl_loadv8sf_mask:
14584 288 : case CODE_FOR_avx512vl_loadv8si_mask:
14585 288 : case CODE_FOR_avx512vl_loadv4df_mask:
14586 288 : case CODE_FOR_avx512vl_loadv4di_mask:
14587 288 : case CODE_FOR_avx512vl_loadv4sf_mask:
14588 288 : case CODE_FOR_avx512vl_loadv4si_mask:
14589 288 : case CODE_FOR_avx512vl_loadv2df_mask:
14590 288 : case CODE_FOR_avx512vl_loadv2di_mask:
14591 288 : case CODE_FOR_avx512bw_loadv64qi_mask:
14592 288 : case CODE_FOR_avx512vl_loadv32qi_mask:
14593 288 : case CODE_FOR_avx512vl_loadv16qi_mask:
14594 288 : case CODE_FOR_avx512bw_loadv32hi_mask:
14595 288 : case CODE_FOR_avx512vl_loadv16hi_mask:
14596 288 : case CODE_FOR_avx512vl_loadv8hi_mask:
14597 288 : aligned_mem = true;
14598 288 : break;
14599 : default:
14600 : break;
14601 : }
14602 : /* FALLTHRU */
14603 : case V64QI_FTYPE_PCCHAR_V64QI_UDI:
14604 : case V32QI_FTYPE_PCCHAR_V32QI_USI:
14605 : case V16QI_FTYPE_PCCHAR_V16QI_UHI:
14606 : case V32HI_FTYPE_PCSHORT_V32HI_USI:
14607 : case V16HI_FTYPE_PCSHORT_V16HI_UHI:
14608 : case V8HI_FTYPE_PCSHORT_V8HI_UQI:
14609 : case V16SI_FTYPE_PCINT_V16SI_UHI:
14610 : case V8SI_FTYPE_PCINT_V8SI_UQI:
14611 : case V4SI_FTYPE_PCINT_V4SI_UQI:
14612 : case V8DI_FTYPE_PCINT64_V8DI_UQI:
14613 : case V4DI_FTYPE_PCINT64_V4DI_UQI:
14614 : case V2DI_FTYPE_PCINT64_V2DI_UQI:
14615 : case V8DF_FTYPE_PCDOUBLE_V8DF_UQI:
14616 : case V4DF_FTYPE_PCDOUBLE_V4DF_UQI:
14617 : case V2DF_FTYPE_PCDOUBLE_V2DF_UQI:
14618 : case V16SF_FTYPE_PCFLOAT_V16SF_UHI:
14619 : case V8SF_FTYPE_PCFLOAT_V8SF_UQI:
14620 : case V4SF_FTYPE_PCFLOAT_V4SF_UQI:
14621 : case V8HF_FTYPE_PCFLOAT16_V8HF_UQI:
14622 : nargs = 3;
14623 : klass = load;
14624 : memory = 0;
14625 : break;
14626 105 : case INT_FTYPE_PINT_INT_INT_INT:
14627 105 : case LONGLONG_FTYPE_PLONGLONG_LONGLONG_LONGLONG_INT:
14628 105 : nargs = 4;
14629 105 : klass = load;
14630 105 : memory = 0;
14631 105 : constant = 3;
14632 105 : break;
14633 0 : default:
14634 0 : gcc_unreachable ();
14635 : }
14636 :
14637 8271 : gcc_assert (nargs <= ARRAY_SIZE (xops));
14638 :
14639 11778 : if (klass == store)
14640 : {
14641 1913 : arg = CALL_EXPR_ARG (exp, 0);
14642 1913 : op = expand_normal (arg);
14643 1913 : gcc_assert (target == 0);
14644 1913 : if (memory)
14645 : {
14646 1750 : op = ix86_zero_extend_to_Pmode (op);
14647 1750 : target = gen_rtx_MEM (tmode, op);
14648 : /* target at this point has just BITS_PER_UNIT MEM_ALIGN
14649 : on it. Try to improve it using get_pointer_alignment,
14650 : and if the special builtin is one that requires strict
14651 : mode alignment, also from it's GET_MODE_ALIGNMENT.
14652 : Failure to do so could lead to ix86_legitimate_combined_insn
14653 : rejecting all changes to such insns. */
14654 1750 : unsigned int align = get_pointer_alignment (arg);
14655 1750 : if (aligned_mem && align < GET_MODE_ALIGNMENT (tmode))
14656 275 : align = GET_MODE_ALIGNMENT (tmode);
14657 3500 : if (MEM_ALIGN (target) < align)
14658 422 : set_mem_align (target, align);
14659 : }
14660 : else
14661 163 : target = force_reg (tmode, op);
14662 : arg_adjust = 1;
14663 : }
14664 : else
14665 : {
14666 9865 : arg_adjust = 0;
14667 9865 : if (optimize
14668 2919 : || target == 0
14669 2919 : || !register_operand (target, tmode)
14670 12773 : || GET_MODE (target) != tmode)
14671 6957 : target = gen_reg_rtx (tmode);
14672 : }
14673 :
14674 21236 : for (i = 0; i < nargs; i++)
14675 : {
14676 9458 : machine_mode mode = insn_p->operand[i + 1].mode;
14677 :
14678 9458 : arg = CALL_EXPR_ARG (exp, i + arg_adjust);
14679 9458 : op = ix86_expand_unsigned_small_int_cst_argument (arg);
14680 :
14681 9458 : if (i == memory)
14682 : {
14683 : /* This must be the memory operand. */
14684 2355 : op = ix86_zero_extend_to_Pmode (op);
14685 2355 : op = gen_rtx_MEM (mode, op);
14686 : /* op at this point has just BITS_PER_UNIT MEM_ALIGN
14687 : on it. Try to improve it using get_pointer_alignment,
14688 : and if the special builtin is one that requires strict
14689 : mode alignment, also from it's GET_MODE_ALIGNMENT.
14690 : Failure to do so could lead to ix86_legitimate_combined_insn
14691 : rejecting all changes to such insns. */
14692 2355 : unsigned int align = get_pointer_alignment (arg);
14693 2355 : if (aligned_mem && align < GET_MODE_ALIGNMENT (mode))
14694 299 : align = GET_MODE_ALIGNMENT (mode);
14695 4710 : if (MEM_ALIGN (op) < align)
14696 523 : set_mem_align (op, align);
14697 : }
14698 7103 : else if (i == constant)
14699 : {
14700 : /* This must be the constant. */
14701 105 : if (!insn_p->operand[nargs].predicate(op, SImode))
14702 : {
14703 0 : error ("the fourth argument must be one of enum %qs", "_CMPCCX_ENUM");
14704 0 : return const0_rtx;
14705 : }
14706 : }
14707 : else
14708 : {
14709 : /* This must be register. */
14710 6998 : if (VECTOR_MODE_P (mode))
14711 3508 : op = safe_vector_operand (op, mode);
14712 :
14713 6998 : op = fixup_modeless_constant (op, mode);
14714 :
14715 : /* NB: 3-operands load implied it's a mask load or v{p}expand*,
14716 : and that mask operand should be at the end.
14717 : Keep all-ones mask which would be simplified by the expander. */
14718 1771 : if (nargs == 3 && i == 2 && klass == load
14719 1771 : && constm1_operand (op, mode)
14720 7171 : && insn_p->operand[i].predicate (op, mode))
14721 : ;
14722 6998 : else if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
14723 6998 : op = copy_to_mode_reg (mode, op);
14724 : else
14725 : {
14726 0 : op = copy_to_reg (op);
14727 0 : op = lowpart_subreg (mode, op, GET_MODE (op));
14728 : }
14729 : }
14730 :
14731 9458 : xops[i]= op;
14732 : }
14733 :
14734 11778 : switch (nargs)
14735 : {
14736 7673 : case 0:
14737 7673 : pat = GEN_FCN (icode) (target);
14738 7673 : break;
14739 733 : case 1:
14740 733 : pat = GEN_FCN (icode) (target, xops[0]);
14741 733 : break;
14742 1496 : case 2:
14743 1496 : pat = GEN_FCN (icode) (target, xops[0], xops[1]);
14744 1496 : break;
14745 1771 : case 3:
14746 1771 : pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2]);
14747 1771 : break;
14748 105 : case 4:
14749 105 : pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2], xops[3]);
14750 105 : break;
14751 : default:
14752 : gcc_unreachable ();
14753 : }
14754 :
14755 11778 : if (! pat)
14756 : return 0;
14757 :
14758 11778 : emit_insn (pat);
14759 11778 : return klass == store ? 0 : target;
14760 : }
14761 :
14762 : /* Subroutine of ix86_expand_builtin to take care of amx insns
14763 : with variable number of operands. */
14764 :
14765 : static rtx
14766 192 : ix86_expand_ace_builtin (const struct builtin_description *d, tree exp,
14767 : rtx target)
14768 : {
14769 192 : tree arg;
14770 192 : rtx pat, op;
14771 192 : unsigned int i, nargs, arg_adjust = 0, constant = 100;
14772 192 : bool tmm_src = false;
14773 192 : rtx xops[4];
14774 192 : enum insn_code icode = d->icode;
14775 192 : const struct insn_data_d *insn_p = &insn_data[icode];
14776 :
14777 192 : switch ((enum ix86_builtin_func_type) d->flag)
14778 : {
14779 : case VOID_FTYPE_UQI:
14780 : nargs = 1;
14781 : break;
14782 60 : case V16SF_FTYPE_UQI_SI:
14783 60 : case V32BF_FTYPE_UQI_SI:
14784 60 : case V32HF_FTYPE_UQI_SI:
14785 60 : case V16SI_FTYPE_UQI_SI:
14786 60 : nargs = 2;
14787 60 : tmm_src = true;
14788 60 : break;
14789 70 : case VOID_FTYPE_UQI_V16SI_SI:
14790 70 : case VOID_FTYPE_UQI_V32BF_V32BF:
14791 70 : case VOID_FTYPE_UQI_V64QI_V64QI:
14792 70 : nargs = 3;
14793 70 : break;
14794 50 : case VOID_FTYPE_UQI_V64QI_V64QI_SI:
14795 50 : nargs = 4;
14796 50 : constant = 3;
14797 50 : break;
14798 :
14799 0 : default:
14800 0 : gcc_unreachable ();
14801 : }
14802 :
14803 180 : gcc_assert (nargs <= ARRAY_SIZE (xops));
14804 :
14805 192 : if (tmm_src)
14806 : {
14807 60 : machine_mode tmode = insn_p->operand[0].mode;
14808 60 : arg_adjust = 1;
14809 60 : if (optimize
14810 18 : || target == 0
14811 18 : || !register_operand (target, tmode)
14812 78 : || GET_MODE (target) != tmode)
14813 42 : target = gen_reg_rtx (tmode);
14814 : }
14815 :
14816 732 : for (i = 0; i < nargs; i++)
14817 : {
14818 542 : machine_mode mode = insn_p->operand[i + arg_adjust].mode;
14819 :
14820 542 : arg = CALL_EXPR_ARG (exp, i);
14821 542 : op = ix86_expand_unsigned_small_int_cst_argument (arg);
14822 :
14823 542 : if (i == 0 || i == constant)
14824 : {
14825 242 : if (!insn_p->operand[i + arg_adjust].predicate (op, SImode))
14826 : {
14827 2 : if (i == 0)
14828 : /* This must be the tmm reg number constant. */
14829 2 : error ("the tmm register number argument must be between 0 to 7");
14830 : else
14831 : /* This must be the constant. */
14832 0 : error ("the argument must be constant");
14833 2 : return const0_rtx;
14834 : }
14835 : }
14836 : else
14837 : {
14838 : /* This must be register. */
14839 300 : if (VECTOR_MODE_P (mode))
14840 220 : op = safe_vector_operand (op, mode);
14841 :
14842 300 : op = fixup_modeless_constant (op, mode);
14843 :
14844 300 : if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
14845 300 : op = copy_to_mode_reg (mode, op);
14846 : else
14847 : {
14848 0 : op = copy_to_reg (op);
14849 0 : op = lowpart_subreg (mode, op, GET_MODE (op));
14850 : }
14851 : }
14852 :
14853 540 : xops[i] = op;
14854 : }
14855 :
14856 190 : if (tmm_src)
14857 : {
14858 60 : switch (nargs)
14859 : {
14860 60 : case 2:
14861 60 : pat = GEN_FCN (icode) (target, xops[0], xops[1]);
14862 60 : break;
14863 0 : default:
14864 0 : gcc_unreachable ();
14865 : }
14866 : }
14867 : else
14868 : {
14869 130 : switch (nargs)
14870 : {
14871 10 : case 1:
14872 10 : pat = GEN_FCN (icode) (xops[0]);
14873 10 : break;
14874 70 : case 3:
14875 70 : pat = GEN_FCN (icode) (xops[0], xops[1], xops[2]);
14876 70 : break;
14877 50 : case 4:
14878 50 : pat = GEN_FCN (icode) (xops[0], xops[1], xops[2], xops[3]);
14879 50 : break;
14880 0 : default:
14881 0 : gcc_unreachable ();
14882 : }
14883 : }
14884 :
14885 190 : if (!pat)
14886 : return 0;
14887 :
14888 190 : emit_insn (pat);
14889 190 : return tmm_src ? target : 0;
14890 : }
14891 :
14892 : /* Return the integer constant in ARG. Constrain it to be in the range
14893 : of the subparts of VEC_TYPE; issue an error if not. */
14894 :
14895 : static int
14896 604 : get_element_number (tree vec_type, tree arg)
14897 : {
14898 604 : unsigned HOST_WIDE_INT elt, max = TYPE_VECTOR_SUBPARTS (vec_type) - 1;
14899 :
14900 604 : if (!tree_fits_uhwi_p (arg)
14901 604 : || (elt = tree_to_uhwi (arg), elt > max))
14902 : {
14903 0 : error ("selector must be an integer constant in the range "
14904 : "[0, %wi]", max);
14905 0 : return 0;
14906 : }
14907 :
14908 604 : return elt;
14909 : }
14910 :
14911 : /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
14912 : ix86_expand_vector_init. We DO have language-level syntax for this, in
14913 : the form of (type){ init-list }. Except that since we can't place emms
14914 : instructions from inside the compiler, we can't allow the use of MMX
14915 : registers unless the user explicitly asks for it. So we do *not* define
14916 : vec_set/vec_extract/vec_init patterns for MMX modes in mmx.md. Instead
14917 : we have builtins invoked by mmintrin.h that gives us license to emit
14918 : these sorts of instructions. */
14919 :
14920 : static rtx
14921 229 : ix86_expand_vec_init_builtin (tree type, tree exp, rtx target)
14922 : {
14923 229 : machine_mode tmode = TYPE_MODE (type);
14924 229 : machine_mode inner_mode = GET_MODE_INNER (tmode);
14925 229 : int i, n_elt = GET_MODE_NUNITS (tmode);
14926 229 : rtvec v = rtvec_alloc (n_elt);
14927 :
14928 229 : gcc_assert (VECTOR_MODE_P (tmode));
14929 229 : gcc_assert (call_expr_nargs (exp) == n_elt);
14930 :
14931 1203 : for (i = 0; i < n_elt; ++i)
14932 : {
14933 974 : rtx x = expand_normal (CALL_EXPR_ARG (exp, i));
14934 974 : RTVEC_ELT (v, i) = gen_lowpart (inner_mode, x);
14935 : }
14936 :
14937 229 : if (!target || !register_operand (target, tmode))
14938 0 : target = gen_reg_rtx (tmode);
14939 :
14940 229 : ix86_expand_vector_init (true, target, gen_rtx_PARALLEL (tmode, v));
14941 229 : return target;
14942 : }
14943 :
14944 : /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
14945 : ix86_expand_vector_extract. They would be redundant (for non-MMX) if we
14946 : had a language-level syntax for referencing vector elements. */
14947 :
14948 : static rtx
14949 400 : ix86_expand_vec_ext_builtin (tree exp, rtx target)
14950 : {
14951 400 : machine_mode tmode, mode0;
14952 400 : tree arg0, arg1;
14953 400 : int elt;
14954 400 : rtx op0;
14955 :
14956 400 : arg0 = CALL_EXPR_ARG (exp, 0);
14957 400 : arg1 = CALL_EXPR_ARG (exp, 1);
14958 :
14959 400 : op0 = expand_normal (arg0);
14960 400 : elt = get_element_number (TREE_TYPE (arg0), arg1);
14961 :
14962 400 : tmode = TYPE_MODE (TREE_TYPE (TREE_TYPE (arg0)));
14963 400 : mode0 = TYPE_MODE (TREE_TYPE (arg0));
14964 400 : gcc_assert (VECTOR_MODE_P (mode0));
14965 :
14966 400 : op0 = force_reg (mode0, op0);
14967 :
14968 400 : if (optimize || !target || !register_operand (target, tmode))
14969 321 : target = gen_reg_rtx (tmode);
14970 :
14971 400 : ix86_expand_vector_extract (true, target, op0, elt);
14972 :
14973 400 : return target;
14974 : }
14975 :
14976 : /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
14977 : ix86_expand_vector_set. They would be redundant (for non-MMX) if we had
14978 : a language-level syntax for referencing vector elements. */
14979 :
14980 : static rtx
14981 204 : ix86_expand_vec_set_builtin (tree exp)
14982 : {
14983 204 : machine_mode tmode, mode1;
14984 204 : tree arg0, arg1, arg2;
14985 204 : int elt;
14986 204 : rtx op0, op1, target;
14987 :
14988 204 : arg0 = CALL_EXPR_ARG (exp, 0);
14989 204 : arg1 = CALL_EXPR_ARG (exp, 1);
14990 204 : arg2 = CALL_EXPR_ARG (exp, 2);
14991 :
14992 204 : tmode = TYPE_MODE (TREE_TYPE (arg0));
14993 204 : mode1 = TYPE_MODE (TREE_TYPE (TREE_TYPE (arg0)));
14994 204 : gcc_assert (VECTOR_MODE_P (tmode));
14995 :
14996 204 : op0 = expand_expr (arg0, NULL_RTX, tmode, EXPAND_NORMAL);
14997 204 : op1 = expand_expr (arg1, NULL_RTX, mode1, EXPAND_NORMAL);
14998 204 : elt = get_element_number (TREE_TYPE (arg0), arg2);
14999 :
15000 204 : if (GET_MODE (op1) != mode1)
15001 82 : op1 = convert_modes (mode1, GET_MODE (op1), op1, true);
15002 :
15003 204 : op0 = force_reg (tmode, op0);
15004 204 : if (op1 != CONST0_RTX (mode1))
15005 204 : op1 = force_reg (mode1, op1);
15006 :
15007 : /* OP0 is the source of these builtin functions and shouldn't be
15008 : modified. Create a copy, use it and return it as target. */
15009 204 : target = gen_reg_rtx (tmode);
15010 204 : emit_move_insn (target, op0);
15011 204 : ix86_expand_vector_set (true, target, op1, elt);
15012 :
15013 204 : return target;
15014 : }
15015 :
15016 : /* Return true if the necessary isa options for this builtin exist,
15017 : else false.
15018 : fcode = DECL_MD_FUNCTION_CODE (fndecl); */
15019 : bool
15020 1337694 : ix86_check_builtin_isa_match (unsigned int fcode,
15021 : HOST_WIDE_INT* pbisa,
15022 : HOST_WIDE_INT* pbisa2)
15023 : {
15024 1337694 : HOST_WIDE_INT isa = ix86_isa_flags;
15025 1337694 : HOST_WIDE_INT isa2 = ix86_isa_flags2;
15026 1337694 : HOST_WIDE_INT bisa = ix86_builtins_isa[fcode].isa;
15027 1337694 : HOST_WIDE_INT bisa2 = ix86_builtins_isa[fcode].isa2;
15028 1337694 : HOST_WIDE_INT tmp_isa = isa, tmp_isa2 = isa2;
15029 : /* The general case is we require all the ISAs specified in bisa{,2}
15030 : to be enabled.
15031 : The exceptions are:
15032 : OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A
15033 : OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32
15034 : OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4
15035 : (OPTION_MASK_ISA_AVX512VNNI | OPTION_MASK_ISA_AVX512VL) or
15036 : OPTION_MASK_ISA2_AVXVNNI
15037 : (OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL) or
15038 : OPTION_MASK_ISA2_AVXIFMA
15039 : (OPTION_MASK_ISA_AVX512VL | OPTION_MASK_ISA2_AVX512BF16) or
15040 : OPTION_MASK_ISA2_AVXNECONVERT
15041 : OPTION_MASK_ISA_AES or (OPTION_MASK_ISA_AVX512VL | OPTION_MASK_ISA2_VAES)
15042 : OPTION_MASK_ISA2_AVX10V1AUX or OPTION_MASK_ISA2_AVXVNNIINT8
15043 : OPTION_MASK_ISA2_AVX10V1AUX or OPTION_MASK_ISA2_AVXVNNIINT16
15044 : OPTION_MASK_ISA2_AMX_TILE or OPTION_MASK_ISA2_ACEV1
15045 : where for each such pair it is sufficient if either of the ISAs is
15046 : enabled, plus if it is ored with other options also those others.
15047 : OPTION_MASK_ISA_MMX in bisa is satisfied also if TARGET_MMX_WITH_SSE. */
15048 :
15049 : #define SHARE_BUILTIN(A1, A2, B1, B2) \
15050 : if ((((bisa & (A1)) == (A1) && (bisa2 & (A2)) == (A2)) \
15051 : && ((bisa & (B1)) == (B1) && (bisa2 & (B2)) == (B2))) \
15052 : && (((isa & (A1)) == (A1) && (isa2 & (A2)) == (A2)) \
15053 : || ((isa & (B1)) == (B1) && (isa2 & (B2)) == (B2)))) \
15054 : { \
15055 : tmp_isa |= (A1) | (B1); \
15056 : tmp_isa2 |= (A2) | (B2); \
15057 : }
15058 :
15059 1337694 : SHARE_BUILTIN (OPTION_MASK_ISA_SSE, 0, OPTION_MASK_ISA_3DNOW_A, 0);
15060 1337694 : SHARE_BUILTIN (OPTION_MASK_ISA_SSE4_2, 0, OPTION_MASK_ISA_CRC32, 0);
15061 1337694 : SHARE_BUILTIN (OPTION_MASK_ISA_FMA, 0, OPTION_MASK_ISA_FMA4, 0);
15062 1337694 : SHARE_BUILTIN (OPTION_MASK_ISA_AVX512VNNI | OPTION_MASK_ISA_AVX512VL, 0, 0,
15063 1337694 : OPTION_MASK_ISA2_AVXVNNI);
15064 1337694 : SHARE_BUILTIN (OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, 0, 0,
15065 1337694 : OPTION_MASK_ISA2_AVXIFMA);
15066 1337694 : SHARE_BUILTIN (OPTION_MASK_ISA_AVX512VL, OPTION_MASK_ISA2_AVX512BF16, 0,
15067 1337694 : OPTION_MASK_ISA2_AVXNECONVERT);
15068 1337694 : SHARE_BUILTIN (OPTION_MASK_ISA_AES, 0, OPTION_MASK_ISA_AVX512VL,
15069 1337694 : OPTION_MASK_ISA2_VAES);
15070 1337694 : SHARE_BUILTIN (0, OPTION_MASK_ISA2_AVXVNNIINT8, 0,
15071 1337694 : OPTION_MASK_ISA2_AVX10V1AUX);
15072 1337694 : SHARE_BUILTIN (0, OPTION_MASK_ISA2_AVXVNNIINT16, 0,
15073 1337694 : OPTION_MASK_ISA2_AVX10V1AUX);
15074 1337694 : SHARE_BUILTIN (0, OPTION_MASK_ISA2_AMX_TILE, 0, OPTION_MASK_ISA2_ACEV1);
15075 1337694 : isa = tmp_isa;
15076 1337694 : isa2 = tmp_isa2;
15077 :
15078 1337694 : if ((bisa & OPTION_MASK_ISA_MMX) && !TARGET_MMX && TARGET_MMX_WITH_SSE
15079 : /* __builtin_ia32_maskmovq requires MMX registers. */
15080 4531 : && fcode != IX86_BUILTIN_MASKMOVQ)
15081 : {
15082 4522 : bisa &= ~OPTION_MASK_ISA_MMX;
15083 4522 : bisa |= OPTION_MASK_ISA_SSE2;
15084 : }
15085 :
15086 1337694 : if (pbisa)
15087 176874 : *pbisa = bisa;
15088 1337694 : if (pbisa2)
15089 176874 : *pbisa2 = bisa2;
15090 :
15091 1337694 : return (bisa & isa) == bisa && (bisa2 & isa2) == bisa2;
15092 : }
15093 :
15094 : /* Emit instructions to set the carry flag from ARG. */
15095 :
15096 : void
15097 13894 : ix86_expand_carry (rtx arg)
15098 : {
15099 13894 : if (!CONST_INT_P (arg) || arg == const0_rtx)
15100 : {
15101 13887 : arg = convert_to_mode (QImode, arg, 1);
15102 13887 : arg = copy_to_mode_reg (QImode, arg);
15103 13887 : emit_insn (gen_addqi3_cconly_overflow (arg, constm1_rtx));
15104 : }
15105 : else
15106 7 : emit_insn (gen_x86_stc ());
15107 13894 : }
15108 :
15109 : /* Expand an expression EXP that calls a built-in function,
15110 : with result going to TARGET if that's convenient
15111 : (and in mode MODE if that's convenient).
15112 : SUBTARGET may be used as the target for computing one of EXP's operands.
15113 : IGNORE is nonzero if the value is to be ignored. */
15114 :
15115 : rtx
15116 177680 : ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
15117 : machine_mode mode, int ignore)
15118 : {
15119 177680 : size_t i;
15120 177680 : enum insn_code icode, icode2;
15121 177680 : tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
15122 177680 : tree arg0, arg1, arg2, arg3, arg4;
15123 177680 : rtx op0, op1, op2, op3, op4, pat, pat2, insn;
15124 177680 : machine_mode mode0, mode1, mode2, mode3, mode4;
15125 177680 : unsigned int fcode = DECL_MD_FUNCTION_CODE (fndecl);
15126 177680 : HOST_WIDE_INT bisa, bisa2;
15127 :
15128 : /* For CPU builtins that can be folded, fold first and expand the fold. */
15129 177680 : switch (fcode)
15130 : {
15131 201 : case IX86_BUILTIN_CPU_INIT:
15132 201 : {
15133 : /* Make it call __cpu_indicator_init in libgcc. */
15134 201 : tree call_expr, fndecl, type;
15135 201 : type = build_function_type_list (integer_type_node, NULL_TREE);
15136 201 : fndecl = build_fn_decl ("__cpu_indicator_init", type);
15137 201 : call_expr = build_call_expr (fndecl, 0);
15138 201 : return expand_expr (call_expr, target, mode, EXPAND_NORMAL);
15139 : }
15140 605 : case IX86_BUILTIN_CPU_IS:
15141 605 : case IX86_BUILTIN_CPU_SUPPORTS:
15142 605 : {
15143 605 : tree arg0 = CALL_EXPR_ARG (exp, 0);
15144 605 : tree fold_expr = fold_builtin_cpu (fndecl, &arg0);
15145 605 : gcc_assert (fold_expr != NULL_TREE);
15146 605 : return expand_expr (fold_expr, target, mode, EXPAND_NORMAL);
15147 : }
15148 : }
15149 :
15150 176874 : if (!ix86_check_builtin_isa_match (fcode, &bisa, &bisa2))
15151 : {
15152 23 : bool add_abi_p = bisa & OPTION_MASK_ISA_64BIT;
15153 23 : if (TARGET_ABI_X32)
15154 0 : bisa |= OPTION_MASK_ABI_X32;
15155 : else
15156 23 : bisa |= OPTION_MASK_ABI_64;
15157 23 : char *opts = ix86_target_string (bisa, bisa2, 0, 0, NULL, NULL,
15158 : (enum fpmath_unit) 0,
15159 : (enum prefer_vector_width) 0,
15160 : PVW_NONE, false, add_abi_p);
15161 23 : if (!opts)
15162 0 : error ("%qE needs unknown isa option", fndecl);
15163 : else
15164 : {
15165 23 : gcc_assert (opts != NULL);
15166 23 : error ("%qE needs isa option %s", fndecl, opts);
15167 23 : free (opts);
15168 : }
15169 23 : return expand_call (exp, target, ignore);
15170 : }
15171 :
15172 176851 : switch (fcode)
15173 : {
15174 35 : case IX86_BUILTIN_MASKMOVQ:
15175 35 : case IX86_BUILTIN_MASKMOVDQU:
15176 34 : icode = (fcode == IX86_BUILTIN_MASKMOVQ
15177 35 : ? CODE_FOR_mmx_maskmovq
15178 : : CODE_FOR_sse2_maskmovdqu);
15179 : /* Note the arg order is different from the operand order. */
15180 35 : arg1 = CALL_EXPR_ARG (exp, 0);
15181 35 : arg2 = CALL_EXPR_ARG (exp, 1);
15182 35 : arg0 = CALL_EXPR_ARG (exp, 2);
15183 35 : op0 = expand_normal (arg0);
15184 35 : op1 = expand_normal (arg1);
15185 35 : op2 = expand_normal (arg2);
15186 35 : mode0 = insn_data[icode].operand[0].mode;
15187 35 : mode1 = insn_data[icode].operand[1].mode;
15188 35 : mode2 = insn_data[icode].operand[2].mode;
15189 :
15190 35 : op0 = ix86_zero_extend_to_Pmode (op0);
15191 35 : op0 = gen_rtx_MEM (mode1, op0);
15192 :
15193 35 : if (!insn_data[icode].operand[0].predicate (op0, mode0))
15194 0 : op0 = copy_to_mode_reg (mode0, op0);
15195 35 : if (!insn_data[icode].operand[1].predicate (op1, mode1))
15196 2 : op1 = copy_to_mode_reg (mode1, op1);
15197 35 : if (!insn_data[icode].operand[2].predicate (op2, mode2))
15198 2 : op2 = copy_to_mode_reg (mode2, op2);
15199 35 : pat = GEN_FCN (icode) (op0, op1, op2);
15200 35 : if (! pat)
15201 56483 : return 0;
15202 35 : emit_insn (pat);
15203 35 : return 0;
15204 :
15205 21943 : case IX86_BUILTIN_LDMXCSR:
15206 21943 : op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
15207 21943 : target = assign_stack_temp (SImode, GET_MODE_SIZE (SImode));
15208 21943 : emit_move_insn (target, op0);
15209 21943 : emit_insn (gen_sse_ldmxcsr (target));
15210 21943 : return 0;
15211 :
15212 14717 : case IX86_BUILTIN_STMXCSR:
15213 14717 : target = assign_stack_temp (SImode, GET_MODE_SIZE (SImode));
15214 14717 : emit_insn (gen_sse_stmxcsr (target));
15215 14717 : return copy_to_mode_reg (SImode, target);
15216 :
15217 11 : case IX86_BUILTIN_CLFLUSH:
15218 11 : arg0 = CALL_EXPR_ARG (exp, 0);
15219 11 : op0 = expand_normal (arg0);
15220 11 : icode = CODE_FOR_sse2_clflush;
15221 11 : if (!insn_data[icode].operand[0].predicate (op0, Pmode))
15222 5 : op0 = ix86_zero_extend_to_Pmode (op0);
15223 :
15224 11 : emit_insn (gen_sse2_clflush (op0));
15225 11 : return 0;
15226 :
15227 19 : case IX86_BUILTIN_CLWB:
15228 19 : arg0 = CALL_EXPR_ARG (exp, 0);
15229 19 : op0 = expand_normal (arg0);
15230 19 : icode = CODE_FOR_clwb;
15231 19 : if (!insn_data[icode].operand[0].predicate (op0, Pmode))
15232 9 : op0 = ix86_zero_extend_to_Pmode (op0);
15233 :
15234 19 : emit_insn (gen_clwb (op0));
15235 19 : return 0;
15236 :
15237 19 : case IX86_BUILTIN_CLFLUSHOPT:
15238 19 : arg0 = CALL_EXPR_ARG (exp, 0);
15239 19 : op0 = expand_normal (arg0);
15240 19 : icode = CODE_FOR_clflushopt;
15241 19 : if (!insn_data[icode].operand[0].predicate (op0, Pmode))
15242 9 : op0 = ix86_zero_extend_to_Pmode (op0);
15243 :
15244 19 : emit_insn (gen_clflushopt (op0));
15245 19 : return 0;
15246 :
15247 47 : case IX86_BUILTIN_MONITOR:
15248 47 : case IX86_BUILTIN_MONITORX:
15249 47 : arg0 = CALL_EXPR_ARG (exp, 0);
15250 47 : arg1 = CALL_EXPR_ARG (exp, 1);
15251 47 : arg2 = CALL_EXPR_ARG (exp, 2);
15252 47 : op0 = expand_normal (arg0);
15253 47 : op1 = expand_normal (arg1);
15254 47 : op2 = expand_normal (arg2);
15255 47 : if (!REG_P (op0))
15256 19 : op0 = ix86_zero_extend_to_Pmode (op0);
15257 47 : if (!REG_P (op1))
15258 22 : op1 = copy_to_mode_reg (SImode, op1);
15259 47 : if (!REG_P (op2))
15260 25 : op2 = copy_to_mode_reg (SImode, op2);
15261 :
15262 47 : emit_insn (fcode == IX86_BUILTIN_MONITOR
15263 26 : ? gen_sse3_monitor (Pmode, op0, op1, op2)
15264 21 : : gen_monitorx (Pmode, op0, op1, op2));
15265 47 : return 0;
15266 :
15267 25 : case IX86_BUILTIN_MWAIT:
15268 25 : arg0 = CALL_EXPR_ARG (exp, 0);
15269 25 : arg1 = CALL_EXPR_ARG (exp, 1);
15270 25 : op0 = expand_normal (arg0);
15271 25 : op1 = expand_normal (arg1);
15272 25 : if (!REG_P (op0))
15273 13 : op0 = copy_to_mode_reg (SImode, op0);
15274 25 : if (!REG_P (op1))
15275 11 : op1 = copy_to_mode_reg (SImode, op1);
15276 25 : emit_insn (gen_sse3_mwait (op0, op1));
15277 25 : return 0;
15278 :
15279 21 : case IX86_BUILTIN_MWAITX:
15280 21 : arg0 = CALL_EXPR_ARG (exp, 0);
15281 21 : arg1 = CALL_EXPR_ARG (exp, 1);
15282 21 : arg2 = CALL_EXPR_ARG (exp, 2);
15283 21 : op0 = expand_normal (arg0);
15284 21 : op1 = expand_normal (arg1);
15285 21 : op2 = expand_normal (arg2);
15286 21 : if (!REG_P (op0))
15287 11 : op0 = copy_to_mode_reg (SImode, op0);
15288 21 : if (!REG_P (op1))
15289 10 : op1 = copy_to_mode_reg (SImode, op1);
15290 21 : if (!REG_P (op2))
15291 11 : op2 = copy_to_mode_reg (SImode, op2);
15292 21 : emit_insn (gen_mwaitx (op0, op1, op2));
15293 21 : return 0;
15294 :
15295 21 : case IX86_BUILTIN_UMONITOR:
15296 21 : arg0 = CALL_EXPR_ARG (exp, 0);
15297 21 : op0 = expand_normal (arg0);
15298 :
15299 21 : op0 = ix86_zero_extend_to_Pmode (op0);
15300 21 : emit_insn (gen_umonitor (Pmode, op0));
15301 21 : return 0;
15302 :
15303 42 : case IX86_BUILTIN_UMWAIT:
15304 42 : case IX86_BUILTIN_TPAUSE:
15305 42 : arg0 = CALL_EXPR_ARG (exp, 0);
15306 42 : arg1 = CALL_EXPR_ARG (exp, 1);
15307 42 : op0 = expand_normal (arg0);
15308 42 : op1 = expand_normal (arg1);
15309 :
15310 42 : if (!REG_P (op0))
15311 20 : op0 = copy_to_mode_reg (SImode, op0);
15312 :
15313 42 : op1 = force_reg (DImode, op1);
15314 :
15315 42 : if (TARGET_64BIT)
15316 : {
15317 42 : op2 = expand_simple_binop (DImode, LSHIFTRT, op1, GEN_INT (32),
15318 : NULL, 1, OPTAB_DIRECT);
15319 42 : switch (fcode)
15320 : {
15321 : case IX86_BUILTIN_UMWAIT:
15322 : icode = CODE_FOR_umwait_rex64;
15323 : break;
15324 21 : case IX86_BUILTIN_TPAUSE:
15325 21 : icode = CODE_FOR_tpause_rex64;
15326 21 : break;
15327 0 : default:
15328 0 : gcc_unreachable ();
15329 : }
15330 :
15331 42 : op2 = gen_lowpart (SImode, op2);
15332 42 : op1 = gen_lowpart (SImode, op1);
15333 42 : pat = GEN_FCN (icode) (op0, op1, op2);
15334 : }
15335 : else
15336 : {
15337 0 : switch (fcode)
15338 : {
15339 : case IX86_BUILTIN_UMWAIT:
15340 : icode = CODE_FOR_umwait;
15341 : break;
15342 0 : case IX86_BUILTIN_TPAUSE:
15343 0 : icode = CODE_FOR_tpause;
15344 0 : break;
15345 0 : default:
15346 0 : gcc_unreachable ();
15347 : }
15348 0 : pat = GEN_FCN (icode) (op0, op1);
15349 : }
15350 :
15351 42 : if (!pat)
15352 : return 0;
15353 :
15354 42 : emit_insn (pat);
15355 :
15356 42 : if (target == 0
15357 42 : || !register_operand (target, QImode))
15358 0 : target = gen_reg_rtx (QImode);
15359 :
15360 42 : pat = gen_rtx_EQ (QImode, gen_rtx_REG (CCCmode, FLAGS_REG),
15361 : const0_rtx);
15362 42 : emit_insn (gen_rtx_SET (target, pat));
15363 :
15364 42 : return target;
15365 :
15366 20 : case IX86_BUILTIN_TESTUI:
15367 20 : emit_insn (gen_testui ());
15368 :
15369 20 : if (target == 0
15370 20 : || !register_operand (target, QImode))
15371 0 : target = gen_reg_rtx (QImode);
15372 :
15373 20 : pat = gen_rtx_LTU (QImode, gen_rtx_REG (CCCmode, FLAGS_REG),
15374 : const0_rtx);
15375 20 : emit_insn (gen_rtx_SET (target, pat));
15376 :
15377 20 : return target;
15378 :
15379 19 : case IX86_BUILTIN_CLZERO:
15380 19 : arg0 = CALL_EXPR_ARG (exp, 0);
15381 19 : op0 = expand_normal (arg0);
15382 19 : if (!REG_P (op0))
15383 9 : op0 = ix86_zero_extend_to_Pmode (op0);
15384 19 : emit_insn (gen_clzero (Pmode, op0));
15385 19 : return 0;
15386 :
15387 19 : case IX86_BUILTIN_CLDEMOTE:
15388 19 : arg0 = CALL_EXPR_ARG (exp, 0);
15389 19 : op0 = expand_normal (arg0);
15390 19 : icode = CODE_FOR_cldemote;
15391 19 : if (!insn_data[icode].operand[0].predicate (op0, Pmode))
15392 9 : op0 = ix86_zero_extend_to_Pmode (op0);
15393 :
15394 19 : emit_insn (gen_cldemote (op0));
15395 19 : return 0;
15396 :
15397 11 : case IX86_BUILTIN_LOADIWKEY:
15398 11 : {
15399 11 : arg0 = CALL_EXPR_ARG (exp, 0);
15400 11 : arg1 = CALL_EXPR_ARG (exp, 1);
15401 11 : arg2 = CALL_EXPR_ARG (exp, 2);
15402 11 : arg3 = CALL_EXPR_ARG (exp, 3);
15403 :
15404 11 : op0 = expand_normal (arg0);
15405 11 : op1 = expand_normal (arg1);
15406 11 : op2 = expand_normal (arg2);
15407 11 : op3 = expand_normal (arg3);
15408 :
15409 11 : if (!REG_P (op0))
15410 5 : op0 = copy_to_mode_reg (V2DImode, op0);
15411 11 : if (!REG_P (op1))
15412 5 : op1 = copy_to_mode_reg (V2DImode, op1);
15413 11 : if (!REG_P (op2))
15414 5 : op2 = copy_to_mode_reg (V2DImode, op2);
15415 11 : if (!REG_P (op3))
15416 5 : op3 = copy_to_mode_reg (SImode, op3);
15417 :
15418 11 : emit_insn (gen_loadiwkey (op0, op1, op2, op3));
15419 :
15420 11 : return 0;
15421 : }
15422 :
15423 12 : case IX86_BUILTIN_AESDEC128KLU8:
15424 12 : icode = CODE_FOR_aesdec128klu8;
15425 12 : goto aesdecenc_expand;
15426 :
15427 12 : case IX86_BUILTIN_AESDEC256KLU8:
15428 12 : icode = CODE_FOR_aesdec256klu8;
15429 12 : goto aesdecenc_expand;
15430 :
15431 12 : case IX86_BUILTIN_AESENC128KLU8:
15432 12 : icode = CODE_FOR_aesenc128klu8;
15433 12 : goto aesdecenc_expand;
15434 :
15435 : case IX86_BUILTIN_AESENC256KLU8:
15436 : icode = CODE_FOR_aesenc256klu8;
15437 :
15438 48 : aesdecenc_expand:
15439 :
15440 48 : arg0 = CALL_EXPR_ARG (exp, 0); // __m128i *odata
15441 48 : arg1 = CALL_EXPR_ARG (exp, 1); // __m128i idata
15442 48 : arg2 = CALL_EXPR_ARG (exp, 2); // const void *p
15443 :
15444 48 : op0 = expand_normal (arg0);
15445 48 : op1 = expand_normal (arg1);
15446 48 : op2 = expand_normal (arg2);
15447 :
15448 48 : if (!address_operand (op0, V2DImode))
15449 : {
15450 16 : op0 = convert_memory_address (Pmode, op0);
15451 16 : op0 = copy_addr_to_reg (op0);
15452 : }
15453 48 : op0 = gen_rtx_MEM (V2DImode, op0);
15454 :
15455 48 : if (!REG_P (op1))
15456 20 : op1 = copy_to_mode_reg (V2DImode, op1);
15457 :
15458 48 : if (!address_operand (op2, VOIDmode))
15459 : {
15460 16 : op2 = convert_memory_address (Pmode, op2);
15461 16 : op2 = copy_addr_to_reg (op2);
15462 : }
15463 48 : op2 = gen_rtx_MEM (BLKmode, op2);
15464 :
15465 48 : emit_insn (GEN_FCN (icode) (op1, op1, op2));
15466 :
15467 48 : if (target == 0)
15468 4 : target = gen_reg_rtx (QImode);
15469 :
15470 : /* NB: For aesenc/aesdec keylocker insn, ZF will be set when runtime
15471 : error occurs. Then the output should be cleared for safety. */
15472 48 : rtx_code_label *ok_label;
15473 48 : rtx tmp;
15474 :
15475 48 : tmp = gen_rtx_REG (CCZmode, FLAGS_REG);
15476 48 : pat = gen_rtx_EQ (QImode, tmp, const0_rtx);
15477 48 : ok_label = gen_label_rtx ();
15478 48 : emit_cmp_and_jump_insns (tmp, const0_rtx, NE, 0, GET_MODE (tmp),
15479 : true, ok_label);
15480 : /* Usually the runtime error seldom occur, so predict OK path as
15481 : hotspot to optimize it as fallthrough block. */
15482 48 : predict_jump (REG_BR_PROB_BASE * 90 / 100);
15483 :
15484 48 : emit_insn (gen_rtx_SET (op1, const0_rtx));
15485 :
15486 48 : emit_label (ok_label);
15487 48 : emit_insn (gen_rtx_SET (target, pat));
15488 48 : emit_insn (gen_rtx_SET (op0, op1));
15489 :
15490 48 : return target;
15491 :
15492 11 : case IX86_BUILTIN_AESDECWIDE128KLU8:
15493 11 : icode = CODE_FOR_aesdecwide128klu8;
15494 11 : goto wideaesdecenc_expand;
15495 :
15496 11 : case IX86_BUILTIN_AESDECWIDE256KLU8:
15497 11 : icode = CODE_FOR_aesdecwide256klu8;
15498 11 : goto wideaesdecenc_expand;
15499 :
15500 11 : case IX86_BUILTIN_AESENCWIDE128KLU8:
15501 11 : icode = CODE_FOR_aesencwide128klu8;
15502 11 : goto wideaesdecenc_expand;
15503 :
15504 : case IX86_BUILTIN_AESENCWIDE256KLU8:
15505 : icode = CODE_FOR_aesencwide256klu8;
15506 :
15507 44 : wideaesdecenc_expand:
15508 :
15509 44 : rtx xmm_regs[8];
15510 44 : rtx op;
15511 :
15512 44 : arg0 = CALL_EXPR_ARG (exp, 0); // __m128i * odata
15513 44 : arg1 = CALL_EXPR_ARG (exp, 1); // const __m128i * idata
15514 44 : arg2 = CALL_EXPR_ARG (exp, 2); // const void *p
15515 :
15516 44 : op0 = expand_normal (arg0);
15517 44 : op1 = expand_normal (arg1);
15518 44 : op2 = expand_normal (arg2);
15519 :
15520 44 : if (GET_MODE (op1) != Pmode)
15521 0 : op1 = convert_to_mode (Pmode, op1, 1);
15522 :
15523 44 : if (!address_operand (op2, VOIDmode))
15524 : {
15525 16 : op2 = convert_memory_address (Pmode, op2);
15526 16 : op2 = copy_addr_to_reg (op2);
15527 : }
15528 44 : op2 = gen_rtx_MEM (BLKmode, op2);
15529 :
15530 440 : for (i = 0; i < 8; i++)
15531 : {
15532 352 : xmm_regs[i] = gen_rtx_REG (V2DImode, GET_SSE_REGNO (i));
15533 :
15534 352 : op = gen_rtx_MEM (V2DImode,
15535 352 : plus_constant (Pmode, op1, (i * 16)));
15536 :
15537 352 : emit_move_insn (xmm_regs[i], op);
15538 : }
15539 :
15540 44 : emit_insn (GEN_FCN (icode) (op2));
15541 :
15542 44 : if (target == 0)
15543 0 : target = gen_reg_rtx (QImode);
15544 :
15545 44 : tmp = gen_rtx_REG (CCZmode, FLAGS_REG);
15546 44 : pat = gen_rtx_EQ (QImode, tmp, const0_rtx);
15547 44 : ok_label = gen_label_rtx ();
15548 44 : emit_cmp_and_jump_insns (tmp, const0_rtx, NE, 0, GET_MODE (tmp),
15549 : true, ok_label);
15550 44 : predict_jump (REG_BR_PROB_BASE * 90 / 100);
15551 :
15552 440 : for (i = 0; i < 8; i++)
15553 352 : emit_insn (gen_rtx_SET (xmm_regs[i], const0_rtx));
15554 :
15555 44 : emit_label (ok_label);
15556 44 : emit_insn (gen_rtx_SET (target, pat));
15557 :
15558 44 : if (GET_MODE (op0) != Pmode)
15559 0 : op0 = convert_to_mode (Pmode, op0, 1);
15560 :
15561 396 : for (i = 0; i < 8; i++)
15562 : {
15563 352 : op = gen_rtx_MEM (V2DImode,
15564 352 : plus_constant (Pmode, op0, (i * 16)));
15565 352 : emit_move_insn (op, xmm_regs[i]);
15566 : }
15567 :
15568 : return target;
15569 :
15570 13 : case IX86_BUILTIN_ENCODEKEY128U32:
15571 13 : {
15572 13 : rtx op, xmm_regs[7];
15573 :
15574 13 : arg0 = CALL_EXPR_ARG (exp, 0); // unsigned int htype
15575 13 : arg1 = CALL_EXPR_ARG (exp, 1); // __m128i key
15576 13 : arg2 = CALL_EXPR_ARG (exp, 2); // void *h
15577 :
15578 13 : op0 = expand_normal (arg0);
15579 13 : op1 = expand_normal (arg1);
15580 13 : op2 = expand_normal (arg2);
15581 :
15582 13 : if (!REG_P (op0))
15583 7 : op0 = copy_to_mode_reg (SImode, op0);
15584 :
15585 13 : if (GET_MODE (op2) != Pmode)
15586 1 : op2 = convert_to_mode (Pmode, op2, 1);
15587 :
15588 13 : op = gen_rtx_REG (V2DImode, GET_SSE_REGNO (0));
15589 13 : emit_move_insn (op, op1);
15590 :
15591 65 : for (i = 0; i < 3; i++)
15592 39 : xmm_regs[i] = gen_rtx_REG (V2DImode, GET_SSE_REGNO (i));
15593 :
15594 13 : if (target == 0 || !register_operand (target, SImode))
15595 2 : target = gen_reg_rtx (SImode);
15596 :
15597 13 : emit_insn (gen_encodekey128u32 (target, op0));
15598 :
15599 65 : for (i = 0; i < 3; i++)
15600 : {
15601 39 : op = gen_rtx_MEM (V2DImode,
15602 39 : plus_constant (Pmode, op2, (i * 16)));
15603 39 : emit_move_insn (op, xmm_regs[i]);
15604 : }
15605 :
15606 13 : return target;
15607 : }
15608 13 : case IX86_BUILTIN_ENCODEKEY256U32:
15609 13 : {
15610 13 : rtx op, xmm_regs[7];
15611 :
15612 13 : arg0 = CALL_EXPR_ARG (exp, 0); // unsigned int htype
15613 13 : arg1 = CALL_EXPR_ARG (exp, 1); // __m128i keylow
15614 13 : arg2 = CALL_EXPR_ARG (exp, 2); // __m128i keyhi
15615 13 : arg3 = CALL_EXPR_ARG (exp, 3); // void *h
15616 :
15617 13 : op0 = expand_normal (arg0);
15618 13 : op1 = expand_normal (arg1);
15619 13 : op2 = expand_normal (arg2);
15620 13 : op3 = expand_normal (arg3);
15621 :
15622 13 : if (!REG_P (op0))
15623 7 : op0 = copy_to_mode_reg (SImode, op0);
15624 :
15625 13 : if (GET_MODE (op3) != Pmode)
15626 1 : op3 = convert_to_mode (Pmode, op3, 1);
15627 :
15628 : /* Force to use xmm0, xmm1 for keylow, keyhi*/
15629 13 : op = gen_rtx_REG (V2DImode, GET_SSE_REGNO (0));
15630 13 : emit_move_insn (op, op1);
15631 13 : op = gen_rtx_REG (V2DImode, GET_SSE_REGNO (1));
15632 13 : emit_move_insn (op, op2);
15633 :
15634 78 : for (i = 0; i < 4; i++)
15635 52 : xmm_regs[i] = gen_rtx_REG (V2DImode, GET_SSE_REGNO (i));
15636 :
15637 13 : if (target == 0 || !register_operand (target, SImode))
15638 2 : target = gen_reg_rtx (SImode);
15639 :
15640 13 : emit_insn (gen_encodekey256u32 (target, op0));
15641 :
15642 78 : for (i = 0; i < 4; i++)
15643 : {
15644 52 : op = gen_rtx_MEM (V2DImode,
15645 52 : plus_constant (Pmode, op3, (i * 16)));
15646 52 : emit_move_insn (op, xmm_regs[i]);
15647 : }
15648 :
15649 13 : return target;
15650 : }
15651 :
15652 48 : case IX86_BUILTIN_PREFETCH:
15653 48 : {
15654 48 : arg0 = CALL_EXPR_ARG (exp, 0); // const void *
15655 48 : arg1 = CALL_EXPR_ARG (exp, 1); // const int
15656 48 : arg2 = CALL_EXPR_ARG (exp, 2); // const int
15657 48 : arg3 = CALL_EXPR_ARG (exp, 3); // const int
15658 :
15659 48 : op0 = expand_normal (arg0);
15660 48 : op1 = expand_normal (arg1);
15661 48 : op2 = expand_normal (arg2);
15662 48 : op3 = expand_normal (arg3);
15663 :
15664 48 : if (!CONST_INT_P (op1) || !CONST_INT_P (op2) || !CONST_INT_P (op3))
15665 : {
15666 0 : error ("second, third and fourth argument must be a const");
15667 0 : return const0_rtx;
15668 : }
15669 :
15670 48 : if (!IN_RANGE (INTVAL (op1), 0, 2))
15671 : {
15672 1 : warning (0, "invalid second argument to"
15673 : " %<__builtin_ia32_prefetch%>; using zero");
15674 1 : op1 = const0_rtx;
15675 : }
15676 :
15677 48 : if (INTVAL (op3) == 1)
15678 : {
15679 4 : if (!IN_RANGE (INTVAL (op2), 2, 3))
15680 : {
15681 1 : error ("invalid third argument");
15682 1 : return const0_rtx;
15683 : }
15684 :
15685 3 : if (TARGET_64BIT && TARGET_PREFETCHI
15686 6 : && local_func_symbolic_operand (op0, GET_MODE (op0)))
15687 2 : emit_insn (gen_prefetchi (op0, op2));
15688 : else
15689 : {
15690 1 : warning (0, "instruction prefetch applies when in 64-bit mode"
15691 : " with RIP-relative addressing and"
15692 : " option %<-mprefetchi%>;"
15693 : " they stay NOPs otherwise");
15694 1 : emit_insn (gen_nop ());
15695 : }
15696 : }
15697 : else
15698 : {
15699 44 : if (INTVAL (op3) != 0)
15700 1 : warning (0, "invalid fourth argument to"
15701 : " %<__builtin_ia32_prefetch%>; using zero");
15702 :
15703 44 : if (!address_operand (op0, VOIDmode))
15704 : {
15705 10 : op0 = convert_memory_address (Pmode, op0);
15706 10 : op0 = copy_addr_to_reg (op0);
15707 : }
15708 :
15709 44 : if (!IN_RANGE (INTVAL (op2), 0, 3))
15710 : {
15711 1 : warning (0, "invalid third argument to %<__builtin_ia32_prefetch%>; using zero");
15712 1 : op2 = const0_rtx;
15713 : }
15714 :
15715 44 : if (TARGET_3DNOW
15716 26 : || TARGET_PREFETCH_SSE
15717 0 : || TARGET_PRFCHW
15718 0 : || TARGET_MOVRS)
15719 44 : emit_insn (gen_prefetch (op0, op1, op2));
15720 0 : else if (!MEM_P (op0) && side_effects_p (op0))
15721 : /* Don't do anything with direct references to volatile memory,
15722 : but generate code to handle other side effects. */
15723 0 : emit_insn (op0);
15724 : }
15725 :
15726 : return 0;
15727 : }
15728 :
15729 21 : case IX86_BUILTIN_PREFETCHI:
15730 21 : {
15731 21 : arg0 = CALL_EXPR_ARG (exp, 0); // const void *
15732 21 : arg1 = CALL_EXPR_ARG (exp, 1); // const int
15733 :
15734 21 : op0 = expand_normal (arg0);
15735 21 : op1 = expand_normal (arg1);
15736 :
15737 21 : if (!CONST_INT_P (op1))
15738 : {
15739 0 : error ("second argument must be a const");
15740 0 : return const0_rtx;
15741 : }
15742 :
15743 : /* GOT/PLT_PIC should not be available for instruction prefetch.
15744 : It must be real instruction address. */
15745 21 : if (TARGET_64BIT
15746 21 : && local_func_symbolic_operand (op0, GET_MODE (op0)))
15747 4 : emit_insn (gen_prefetchi (op0, op1));
15748 : else
15749 : {
15750 : /* Ignore the hint. */
15751 17 : warning (0, "instruction prefetch applies when in 64-bit mode"
15752 : " with RIP-relative addressing and"
15753 : " option %<-mprefetchi%>;"
15754 : " they stay NOPs otherwise");
15755 17 : emit_insn (gen_nop ());
15756 : }
15757 :
15758 : return 0;
15759 : }
15760 :
15761 53 : case IX86_BUILTIN_URDMSR:
15762 53 : case IX86_BUILTIN_UWRMSR:
15763 53 : {
15764 53 : arg0 = CALL_EXPR_ARG (exp, 0);
15765 53 : op0 = expand_normal (arg0);
15766 :
15767 53 : if (CONST_INT_P (op0))
15768 : {
15769 12 : unsigned HOST_WIDE_INT val = UINTVAL (op0);
15770 12 : if (val > 0xffffffff)
15771 2 : op0 = force_reg (DImode, op0);
15772 : }
15773 : else
15774 41 : op0 = force_reg (DImode, op0);
15775 :
15776 53 : if (fcode == IX86_BUILTIN_UWRMSR)
15777 : {
15778 26 : arg1 = CALL_EXPR_ARG (exp, 1);
15779 26 : op1 = expand_normal (arg1);
15780 26 : op1 = force_reg (DImode, op1);
15781 26 : icode = CODE_FOR_uwrmsr;
15782 26 : target = 0;
15783 : }
15784 : else
15785 : {
15786 27 : if (target == 0 || !register_operand (target, DImode))
15787 1 : target = gen_reg_rtx (DImode);
15788 : icode = CODE_FOR_urdmsr;
15789 : op1 = op0;
15790 : op0 = target;
15791 : }
15792 53 : emit_insn (GEN_FCN (icode) (op0, op1));
15793 53 : return target;
15794 : }
15795 :
15796 13 : case IX86_BUILTIN_BSR0INIT:
15797 13 : {
15798 13 : target = gen_rtx_REG (V32SImode, BSR0_REG);
15799 13 : emit_insn (gen_bsrinit (target));
15800 13 : return 0;
15801 : }
15802 :
15803 11 : case IX86_BUILTIN_BSR0MOVF:
15804 11 : {
15805 11 : arg0 = CALL_EXPR_ARG (exp, 0);
15806 11 : arg1 = CALL_EXPR_ARG (exp, 1);
15807 11 : op0 = expand_normal (arg0);
15808 11 : op1 = expand_normal (arg1);
15809 :
15810 11 : target = gen_rtx_REG (V32SImode, BSR0_REG);
15811 11 : if (CONST_VECTOR_P (op0) || MEM_P (op0))
15812 1 : op0 = force_reg (V16SImode, op0);
15813 11 : if (CONST_VECTOR_P (op1))
15814 0 : op1 = force_reg (V16SImode, op1);
15815 11 : emit_insn (gen_bsrmovf (target, op0, op1));
15816 11 : return 0;
15817 : }
15818 :
15819 22 : case IX86_BUILTIN_BSR0MOVHINSERT:
15820 22 : case IX86_BUILTIN_BSR0MOVLINSERT:
15821 22 : {
15822 22 : arg0 = CALL_EXPR_ARG (exp, 0);
15823 22 : op0 = expand_normal (arg0);
15824 :
15825 22 : if (fcode == IX86_BUILTIN_BSR0MOVHINSERT)
15826 : icode = CODE_FOR_bsrmovh_load;
15827 : else
15828 11 : icode = CODE_FOR_bsrmovl_load;
15829 22 : target = gen_rtx_REG (V32SImode, BSR0_REG);
15830 22 : if (CONST_VECTOR_P (op0))
15831 0 : op0 = force_reg (V16SImode, op0);
15832 22 : emit_insn (GEN_FCN (icode) (target, op0));
15833 22 : return 0;
15834 : }
15835 :
15836 22 : case IX86_BUILTIN_BSR0MOVHEXTRACT:
15837 22 : case IX86_BUILTIN_BSR0MOVLEXTRACT:
15838 22 : {
15839 22 : op0 = gen_rtx_REG (V32SImode, BSR0_REG);
15840 22 : if (fcode == IX86_BUILTIN_BSR0MOVHEXTRACT)
15841 : icode = CODE_FOR_bsrmovh_store;
15842 : else
15843 11 : icode = CODE_FOR_bsrmovl_store;
15844 22 : if (target == 0 || !register_operand (target, V16SImode))
15845 0 : target = gen_reg_rtx (V16SImode);
15846 22 : emit_insn (GEN_FCN (icode) (target, op0));
15847 22 : return target;
15848 : }
15849 :
15850 229 : case IX86_BUILTIN_VEC_INIT_V2SI:
15851 229 : case IX86_BUILTIN_VEC_INIT_V4HI:
15852 229 : case IX86_BUILTIN_VEC_INIT_V8QI:
15853 229 : return ix86_expand_vec_init_builtin (TREE_TYPE (exp), exp, target);
15854 :
15855 400 : case IX86_BUILTIN_VEC_EXT_V2DF:
15856 400 : case IX86_BUILTIN_VEC_EXT_V2DI:
15857 400 : case IX86_BUILTIN_VEC_EXT_V4SF:
15858 400 : case IX86_BUILTIN_VEC_EXT_V4SI:
15859 400 : case IX86_BUILTIN_VEC_EXT_V8HI:
15860 400 : case IX86_BUILTIN_VEC_EXT_V2SI:
15861 400 : case IX86_BUILTIN_VEC_EXT_V4HI:
15862 400 : case IX86_BUILTIN_VEC_EXT_V16QI:
15863 400 : return ix86_expand_vec_ext_builtin (exp, target);
15864 :
15865 204 : case IX86_BUILTIN_VEC_SET_V2DI:
15866 204 : case IX86_BUILTIN_VEC_SET_V4SF:
15867 204 : case IX86_BUILTIN_VEC_SET_V4SI:
15868 204 : case IX86_BUILTIN_VEC_SET_V8HI:
15869 204 : case IX86_BUILTIN_VEC_SET_V4HI:
15870 204 : case IX86_BUILTIN_VEC_SET_V16QI:
15871 204 : return ix86_expand_vec_set_builtin (exp);
15872 :
15873 0 : case IX86_BUILTIN_NANQ:
15874 0 : case IX86_BUILTIN_NANSQ:
15875 0 : return expand_call (exp, target, ignore);
15876 :
15877 18 : case IX86_BUILTIN_RDPID:
15878 :
15879 18 : op0 = gen_reg_rtx (word_mode);
15880 :
15881 18 : if (TARGET_64BIT)
15882 : {
15883 18 : insn = gen_rdpid_rex64 (op0);
15884 18 : op0 = convert_to_mode (SImode, op0, 1);
15885 : }
15886 : else
15887 0 : insn = gen_rdpid (op0);
15888 :
15889 18 : emit_insn (insn);
15890 :
15891 18 : if (target == 0
15892 18 : || !register_operand (target, SImode))
15893 0 : target = gen_reg_rtx (SImode);
15894 :
15895 18 : emit_move_insn (target, op0);
15896 18 : return target;
15897 :
15898 75 : case IX86_BUILTIN_2INTERSECTD512:
15899 75 : case IX86_BUILTIN_2INTERSECTQ512:
15900 75 : case IX86_BUILTIN_2INTERSECTD256:
15901 75 : case IX86_BUILTIN_2INTERSECTQ256:
15902 75 : case IX86_BUILTIN_2INTERSECTD128:
15903 75 : case IX86_BUILTIN_2INTERSECTQ128:
15904 75 : arg0 = CALL_EXPR_ARG (exp, 0);
15905 75 : arg1 = CALL_EXPR_ARG (exp, 1);
15906 75 : arg2 = CALL_EXPR_ARG (exp, 2);
15907 75 : arg3 = CALL_EXPR_ARG (exp, 3);
15908 75 : op0 = expand_normal (arg0);
15909 75 : op1 = expand_normal (arg1);
15910 75 : op2 = expand_normal (arg2);
15911 75 : op3 = expand_normal (arg3);
15912 :
15913 75 : if (!address_operand (op0, VOIDmode))
15914 : {
15915 25 : op0 = convert_memory_address (Pmode, op0);
15916 25 : op0 = copy_addr_to_reg (op0);
15917 : }
15918 75 : if (!address_operand (op1, VOIDmode))
15919 : {
15920 25 : op1 = convert_memory_address (Pmode, op1);
15921 25 : op1 = copy_addr_to_reg (op1);
15922 : }
15923 :
15924 75 : switch (fcode)
15925 : {
15926 : case IX86_BUILTIN_2INTERSECTD512:
15927 : mode4 = P2HImode;
15928 : icode = CODE_FOR_avx512vp2intersect_2intersectv16si;
15929 : break;
15930 : case IX86_BUILTIN_2INTERSECTQ512:
15931 : mode4 = P2QImode;
15932 : icode = CODE_FOR_avx512vp2intersect_2intersectv8di;
15933 : break;
15934 : case IX86_BUILTIN_2INTERSECTD256:
15935 : mode4 = P2QImode;
15936 : icode = CODE_FOR_avx512vp2intersect_2intersectv8si;
15937 : break;
15938 : case IX86_BUILTIN_2INTERSECTQ256:
15939 : mode4 = P2QImode;
15940 : icode = CODE_FOR_avx512vp2intersect_2intersectv4di;
15941 : break;
15942 : case IX86_BUILTIN_2INTERSECTD128:
15943 : mode4 = P2QImode;
15944 : icode = CODE_FOR_avx512vp2intersect_2intersectv4si;
15945 : break;
15946 : case IX86_BUILTIN_2INTERSECTQ128:
15947 : mode4 = P2QImode;
15948 : icode = CODE_FOR_avx512vp2intersect_2intersectv2di;
15949 : break;
15950 0 : default:
15951 0 : gcc_unreachable ();
15952 : }
15953 :
15954 75 : mode2 = insn_data[icode].operand[1].mode;
15955 75 : mode3 = insn_data[icode].operand[2].mode;
15956 75 : if (!insn_data[icode].operand[1].predicate (op2, mode2))
15957 25 : op2 = copy_to_mode_reg (mode2, op2);
15958 75 : if (!insn_data[icode].operand[2].predicate (op3, mode3))
15959 6 : op3 = copy_to_mode_reg (mode3, op3);
15960 :
15961 75 : op4 = gen_reg_rtx (mode4);
15962 75 : emit_insn (GEN_FCN (icode) (op4, op2, op3));
15963 75 : mode0 = mode4 == P2HImode ? HImode : QImode;
15964 75 : emit_move_insn (gen_rtx_MEM (mode0, op0),
15965 75 : gen_lowpart (mode0, op4));
15966 75 : emit_move_insn (gen_rtx_MEM (mode0, op1),
15967 : gen_highpart (mode0, op4));
15968 :
15969 75 : return 0;
15970 :
15971 102 : case IX86_BUILTIN_RDPMC:
15972 102 : case IX86_BUILTIN_RDTSC:
15973 102 : case IX86_BUILTIN_RDTSCP:
15974 102 : case IX86_BUILTIN_XGETBV:
15975 :
15976 102 : op0 = gen_reg_rtx (DImode);
15977 102 : op1 = gen_reg_rtx (DImode);
15978 :
15979 102 : if (fcode == IX86_BUILTIN_RDPMC)
15980 : {
15981 22 : arg0 = CALL_EXPR_ARG (exp, 0);
15982 22 : op2 = expand_normal (arg0);
15983 22 : if (!register_operand (op2, SImode))
15984 11 : op2 = copy_to_mode_reg (SImode, op2);
15985 :
15986 22 : insn = (TARGET_64BIT
15987 22 : ? gen_rdpmc_rex64 (op0, op1, op2)
15988 0 : : gen_rdpmc (op0, op2));
15989 22 : emit_insn (insn);
15990 : }
15991 80 : else if (fcode == IX86_BUILTIN_XGETBV)
15992 : {
15993 22 : arg0 = CALL_EXPR_ARG (exp, 0);
15994 22 : op2 = expand_normal (arg0);
15995 22 : if (!register_operand (op2, SImode))
15996 1 : op2 = copy_to_mode_reg (SImode, op2);
15997 :
15998 22 : insn = (TARGET_64BIT
15999 22 : ? gen_xgetbv_rex64 (op0, op1, op2)
16000 0 : : gen_xgetbv (op0, op2));
16001 22 : emit_insn (insn);
16002 : }
16003 58 : else if (fcode == IX86_BUILTIN_RDTSC)
16004 : {
16005 36 : insn = (TARGET_64BIT
16006 36 : ? gen_rdtsc_rex64 (op0, op1)
16007 2 : : gen_rdtsc (op0));
16008 36 : emit_insn (insn);
16009 : }
16010 : else
16011 : {
16012 22 : op2 = gen_reg_rtx (SImode);
16013 :
16014 22 : insn = (TARGET_64BIT
16015 22 : ? gen_rdtscp_rex64 (op0, op1, op2)
16016 0 : : gen_rdtscp (op0, op2));
16017 22 : emit_insn (insn);
16018 :
16019 22 : arg0 = CALL_EXPR_ARG (exp, 0);
16020 22 : op4 = expand_normal (arg0);
16021 22 : if (!address_operand (op4, VOIDmode))
16022 : {
16023 10 : op4 = convert_memory_address (Pmode, op4);
16024 10 : op4 = copy_addr_to_reg (op4);
16025 : }
16026 22 : emit_move_insn (gen_rtx_MEM (SImode, op4), op2);
16027 : }
16028 :
16029 102 : if (target == 0
16030 102 : || !register_operand (target, DImode))
16031 10 : target = gen_reg_rtx (DImode);
16032 :
16033 102 : if (TARGET_64BIT)
16034 : {
16035 100 : op1 = expand_simple_binop (DImode, ASHIFT, op1, GEN_INT (32),
16036 : op1, 1, OPTAB_DIRECT);
16037 100 : op0 = expand_simple_binop (DImode, IOR, op0, op1,
16038 : op0, 1, OPTAB_DIRECT);
16039 : }
16040 :
16041 102 : emit_move_insn (target, op0);
16042 102 : return target;
16043 :
16044 62 : case IX86_BUILTIN_ENQCMD:
16045 62 : case IX86_BUILTIN_ENQCMDS:
16046 62 : case IX86_BUILTIN_MOVDIR64B:
16047 :
16048 62 : arg0 = CALL_EXPR_ARG (exp, 0);
16049 62 : arg1 = CALL_EXPR_ARG (exp, 1);
16050 62 : op0 = expand_normal (arg0);
16051 62 : op1 = expand_normal (arg1);
16052 :
16053 62 : op0 = ix86_zero_extend_to_Pmode (op0);
16054 62 : if (!address_operand (op1, VOIDmode))
16055 : {
16056 29 : op1 = convert_memory_address (Pmode, op1);
16057 29 : op1 = copy_addr_to_reg (op1);
16058 : }
16059 62 : op1 = gen_rtx_MEM (XImode, op1);
16060 :
16061 62 : if (fcode == IX86_BUILTIN_MOVDIR64B)
16062 : {
16063 25 : emit_insn (gen_movdir64b (Pmode, op0, op1));
16064 24 : return 0;
16065 : }
16066 : else
16067 : {
16068 38 : if (target == 0
16069 38 : || !register_operand (target, SImode))
16070 0 : target = gen_reg_rtx (SImode);
16071 :
16072 38 : emit_move_insn (target, const0_rtx);
16073 38 : target = gen_rtx_SUBREG (QImode, target, 0);
16074 :
16075 19 : int unspecv = (fcode == IX86_BUILTIN_ENQCMD
16076 38 : ? UNSPECV_ENQCMD
16077 : : UNSPECV_ENQCMDS);
16078 38 : icode = code_for_enqcmd (unspecv, Pmode);
16079 38 : emit_insn (GEN_FCN (icode) (op0, op1));
16080 :
16081 38 : emit_insn
16082 38 : (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
16083 : gen_rtx_fmt_ee (EQ, QImode,
16084 : gen_rtx_REG (CCZmode, FLAGS_REG),
16085 : const0_rtx)));
16086 38 : return SUBREG_REG (target);
16087 : }
16088 :
16089 14707 : case IX86_BUILTIN_FXSAVE:
16090 14707 : case IX86_BUILTIN_FXRSTOR:
16091 14707 : case IX86_BUILTIN_FXSAVE64:
16092 14707 : case IX86_BUILTIN_FXRSTOR64:
16093 14707 : case IX86_BUILTIN_FNSTENV:
16094 14707 : case IX86_BUILTIN_FLDENV:
16095 14707 : mode0 = BLKmode;
16096 14707 : switch (fcode)
16097 : {
16098 : case IX86_BUILTIN_FXSAVE:
16099 : icode = CODE_FOR_fxsave;
16100 : break;
16101 19 : case IX86_BUILTIN_FXRSTOR:
16102 19 : icode = CODE_FOR_fxrstor;
16103 19 : break;
16104 23 : case IX86_BUILTIN_FXSAVE64:
16105 23 : icode = CODE_FOR_fxsave64;
16106 23 : break;
16107 21 : case IX86_BUILTIN_FXRSTOR64:
16108 21 : icode = CODE_FOR_fxrstor64;
16109 21 : break;
16110 7260 : case IX86_BUILTIN_FNSTENV:
16111 7260 : icode = CODE_FOR_fnstenv;
16112 7260 : break;
16113 7364 : case IX86_BUILTIN_FLDENV:
16114 7364 : icode = CODE_FOR_fldenv;
16115 7364 : break;
16116 0 : default:
16117 0 : gcc_unreachable ();
16118 : }
16119 :
16120 14707 : arg0 = CALL_EXPR_ARG (exp, 0);
16121 14707 : op0 = expand_normal (arg0);
16122 :
16123 14707 : if (!address_operand (op0, VOIDmode))
16124 : {
16125 36 : op0 = convert_memory_address (Pmode, op0);
16126 36 : op0 = copy_addr_to_reg (op0);
16127 : }
16128 14707 : op0 = gen_rtx_MEM (mode0, op0);
16129 :
16130 14707 : pat = GEN_FCN (icode) (op0);
16131 14707 : if (pat)
16132 14707 : emit_insn (pat);
16133 : return 0;
16134 :
16135 21 : case IX86_BUILTIN_XSETBV:
16136 21 : arg0 = CALL_EXPR_ARG (exp, 0);
16137 21 : arg1 = CALL_EXPR_ARG (exp, 1);
16138 21 : op0 = expand_normal (arg0);
16139 21 : op1 = expand_normal (arg1);
16140 :
16141 21 : if (!REG_P (op0))
16142 1 : op0 = copy_to_mode_reg (SImode, op0);
16143 :
16144 21 : op1 = force_reg (DImode, op1);
16145 :
16146 21 : if (TARGET_64BIT)
16147 : {
16148 21 : op2 = expand_simple_binop (DImode, LSHIFTRT, op1, GEN_INT (32),
16149 : NULL, 1, OPTAB_DIRECT);
16150 :
16151 21 : icode = CODE_FOR_xsetbv_rex64;
16152 :
16153 21 : op2 = gen_lowpart (SImode, op2);
16154 21 : op1 = gen_lowpart (SImode, op1);
16155 21 : pat = GEN_FCN (icode) (op0, op1, op2);
16156 : }
16157 : else
16158 : {
16159 0 : icode = CODE_FOR_xsetbv;
16160 :
16161 0 : pat = GEN_FCN (icode) (op0, op1);
16162 : }
16163 21 : if (pat)
16164 21 : emit_insn (pat);
16165 : return 0;
16166 :
16167 232 : case IX86_BUILTIN_XSAVE:
16168 232 : case IX86_BUILTIN_XRSTOR:
16169 232 : case IX86_BUILTIN_XSAVE64:
16170 232 : case IX86_BUILTIN_XRSTOR64:
16171 232 : case IX86_BUILTIN_XSAVEOPT:
16172 232 : case IX86_BUILTIN_XSAVEOPT64:
16173 232 : case IX86_BUILTIN_XSAVES:
16174 232 : case IX86_BUILTIN_XRSTORS:
16175 232 : case IX86_BUILTIN_XSAVES64:
16176 232 : case IX86_BUILTIN_XRSTORS64:
16177 232 : case IX86_BUILTIN_XSAVEC:
16178 232 : case IX86_BUILTIN_XSAVEC64:
16179 232 : arg0 = CALL_EXPR_ARG (exp, 0);
16180 232 : arg1 = CALL_EXPR_ARG (exp, 1);
16181 232 : op0 = expand_normal (arg0);
16182 232 : op1 = expand_normal (arg1);
16183 :
16184 232 : if (!address_operand (op0, VOIDmode))
16185 : {
16186 108 : op0 = convert_memory_address (Pmode, op0);
16187 108 : op0 = copy_addr_to_reg (op0);
16188 : }
16189 232 : op0 = gen_rtx_MEM (BLKmode, op0);
16190 :
16191 232 : op1 = force_reg (DImode, op1);
16192 :
16193 232 : if (TARGET_64BIT)
16194 : {
16195 232 : op2 = expand_simple_binop (DImode, LSHIFTRT, op1, GEN_INT (32),
16196 : NULL, 1, OPTAB_DIRECT);
16197 232 : switch (fcode)
16198 : {
16199 : case IX86_BUILTIN_XSAVE:
16200 : icode = CODE_FOR_xsave_rex64;
16201 : break;
16202 19 : case IX86_BUILTIN_XRSTOR:
16203 19 : icode = CODE_FOR_xrstor_rex64;
16204 19 : break;
16205 21 : case IX86_BUILTIN_XSAVE64:
16206 21 : icode = CODE_FOR_xsave64;
16207 21 : break;
16208 21 : case IX86_BUILTIN_XRSTOR64:
16209 21 : icode = CODE_FOR_xrstor64;
16210 21 : break;
16211 19 : case IX86_BUILTIN_XSAVEOPT:
16212 19 : icode = CODE_FOR_xsaveopt_rex64;
16213 19 : break;
16214 19 : case IX86_BUILTIN_XSAVEOPT64:
16215 19 : icode = CODE_FOR_xsaveopt64;
16216 19 : break;
16217 19 : case IX86_BUILTIN_XSAVES:
16218 19 : icode = CODE_FOR_xsaves_rex64;
16219 19 : break;
16220 19 : case IX86_BUILTIN_XRSTORS:
16221 19 : icode = CODE_FOR_xrstors_rex64;
16222 19 : break;
16223 19 : case IX86_BUILTIN_XSAVES64:
16224 19 : icode = CODE_FOR_xsaves64;
16225 19 : break;
16226 19 : case IX86_BUILTIN_XRSTORS64:
16227 19 : icode = CODE_FOR_xrstors64;
16228 19 : break;
16229 19 : case IX86_BUILTIN_XSAVEC:
16230 19 : icode = CODE_FOR_xsavec_rex64;
16231 19 : break;
16232 19 : case IX86_BUILTIN_XSAVEC64:
16233 19 : icode = CODE_FOR_xsavec64;
16234 19 : break;
16235 0 : default:
16236 0 : gcc_unreachable ();
16237 : }
16238 :
16239 232 : op2 = gen_lowpart (SImode, op2);
16240 232 : op1 = gen_lowpart (SImode, op1);
16241 232 : pat = GEN_FCN (icode) (op0, op1, op2);
16242 : }
16243 : else
16244 : {
16245 0 : switch (fcode)
16246 : {
16247 : case IX86_BUILTIN_XSAVE:
16248 : icode = CODE_FOR_xsave;
16249 : break;
16250 : case IX86_BUILTIN_XRSTOR:
16251 : icode = CODE_FOR_xrstor;
16252 : break;
16253 : case IX86_BUILTIN_XSAVEOPT:
16254 : icode = CODE_FOR_xsaveopt;
16255 : break;
16256 : case IX86_BUILTIN_XSAVES:
16257 : icode = CODE_FOR_xsaves;
16258 : break;
16259 : case IX86_BUILTIN_XRSTORS:
16260 : icode = CODE_FOR_xrstors;
16261 : break;
16262 : case IX86_BUILTIN_XSAVEC:
16263 : icode = CODE_FOR_xsavec;
16264 : break;
16265 0 : default:
16266 0 : gcc_unreachable ();
16267 : }
16268 0 : pat = GEN_FCN (icode) (op0, op1);
16269 : }
16270 :
16271 232 : if (pat)
16272 232 : emit_insn (pat);
16273 : return 0;
16274 :
16275 139 : case IX86_BUILTIN_LDTILECFG:
16276 139 : case IX86_BUILTIN_STTILECFG:
16277 139 : arg0 = CALL_EXPR_ARG (exp, 0);
16278 139 : op0 = expand_normal (arg0);
16279 :
16280 139 : if (!address_operand (op0, VOIDmode))
16281 : {
16282 8 : op0 = convert_memory_address (Pmode, op0);
16283 8 : op0 = copy_addr_to_reg (op0);
16284 : }
16285 139 : op0 = gen_rtx_MEM (BLKmode, op0);
16286 139 : if (fcode == IX86_BUILTIN_LDTILECFG)
16287 : icode = CODE_FOR_ldtilecfg;
16288 : else
16289 89 : icode = CODE_FOR_sttilecfg;
16290 139 : pat = GEN_FCN (icode) (op0);
16291 139 : emit_insn (pat);
16292 139 : return 0;
16293 :
16294 18 : case IX86_BUILTIN_LLWPCB:
16295 18 : arg0 = CALL_EXPR_ARG (exp, 0);
16296 18 : op0 = expand_normal (arg0);
16297 :
16298 18 : if (!register_operand (op0, Pmode))
16299 9 : op0 = ix86_zero_extend_to_Pmode (op0);
16300 18 : emit_insn (gen_lwp_llwpcb (Pmode, op0));
16301 18 : return 0;
16302 :
16303 18 : case IX86_BUILTIN_SLWPCB:
16304 18 : if (!target
16305 18 : || !register_operand (target, Pmode))
16306 0 : target = gen_reg_rtx (Pmode);
16307 18 : emit_insn (gen_lwp_slwpcb (Pmode, target));
16308 18 : return target;
16309 :
16310 51 : case IX86_BUILTIN_LWPVAL32:
16311 51 : case IX86_BUILTIN_LWPVAL64:
16312 51 : case IX86_BUILTIN_LWPINS32:
16313 51 : case IX86_BUILTIN_LWPINS64:
16314 51 : mode = ((fcode == IX86_BUILTIN_LWPVAL32
16315 51 : || fcode == IX86_BUILTIN_LWPINS32)
16316 51 : ? SImode : DImode);
16317 :
16318 51 : if (fcode == IX86_BUILTIN_LWPVAL32
16319 51 : || fcode == IX86_BUILTIN_LWPVAL64)
16320 26 : icode = code_for_lwp_lwpval (mode);
16321 : else
16322 25 : icode = code_for_lwp_lwpins (mode);
16323 :
16324 51 : arg0 = CALL_EXPR_ARG (exp, 0);
16325 51 : arg1 = CALL_EXPR_ARG (exp, 1);
16326 51 : arg2 = CALL_EXPR_ARG (exp, 2);
16327 51 : op0 = expand_normal (arg0);
16328 51 : op1 = expand_normal (arg1);
16329 51 : op2 = expand_normal (arg2);
16330 51 : mode0 = insn_data[icode].operand[0].mode;
16331 :
16332 51 : if (!insn_data[icode].operand[0].predicate (op0, mode0))
16333 13 : op0 = copy_to_mode_reg (mode0, op0);
16334 51 : if (!insn_data[icode].operand[1].predicate (op1, SImode))
16335 0 : op1 = copy_to_mode_reg (SImode, op1);
16336 :
16337 51 : if (!CONST_INT_P (op2))
16338 : {
16339 0 : error ("the last argument must be a 32-bit immediate");
16340 0 : return const0_rtx;
16341 : }
16342 :
16343 51 : emit_insn (GEN_FCN (icode) (op0, op1, op2));
16344 :
16345 51 : if (fcode == IX86_BUILTIN_LWPINS32
16346 51 : || fcode == IX86_BUILTIN_LWPINS64)
16347 : {
16348 25 : if (target == 0
16349 25 : || !nonimmediate_operand (target, QImode))
16350 0 : target = gen_reg_rtx (QImode);
16351 :
16352 25 : pat = gen_rtx_EQ (QImode, gen_rtx_REG (CCCmode, FLAGS_REG),
16353 : const0_rtx);
16354 25 : emit_insn (gen_rtx_SET (target, pat));
16355 :
16356 25 : return target;
16357 : }
16358 : else
16359 : return 0;
16360 :
16361 18 : case IX86_BUILTIN_BEXTRI32:
16362 18 : case IX86_BUILTIN_BEXTRI64:
16363 18 : mode = (fcode == IX86_BUILTIN_BEXTRI32 ? SImode : DImode);
16364 :
16365 18 : arg0 = CALL_EXPR_ARG (exp, 0);
16366 18 : arg1 = CALL_EXPR_ARG (exp, 1);
16367 18 : op0 = expand_normal (arg0);
16368 18 : op1 = expand_normal (arg1);
16369 :
16370 18 : if (!CONST_INT_P (op1))
16371 : {
16372 0 : error ("last argument must be an immediate");
16373 0 : return const0_rtx;
16374 : }
16375 : else
16376 : {
16377 18 : unsigned char lsb_index = UINTVAL (op1);
16378 18 : unsigned char length = UINTVAL (op1) >> 8;
16379 :
16380 18 : unsigned char bitsize = GET_MODE_BITSIZE (mode);
16381 :
16382 18 : icode = code_for_tbm_bextri (mode);
16383 :
16384 18 : mode1 = insn_data[icode].operand[1].mode;
16385 18 : if (!insn_data[icode].operand[1].predicate (op0, mode1))
16386 12 : op0 = copy_to_mode_reg (mode1, op0);
16387 :
16388 18 : mode0 = insn_data[icode].operand[0].mode;
16389 18 : if (target == 0
16390 18 : || !register_operand (target, mode0))
16391 0 : target = gen_reg_rtx (mode0);
16392 :
16393 18 : if (length == 0 || lsb_index >= bitsize)
16394 : {
16395 8 : emit_move_insn (target, const0_rtx);
16396 8 : return target;
16397 : }
16398 :
16399 10 : if (length + lsb_index > bitsize)
16400 5 : length = bitsize - lsb_index;
16401 :
16402 10 : op1 = GEN_INT (length);
16403 10 : op2 = GEN_INT (lsb_index);
16404 :
16405 10 : emit_insn (GEN_FCN (icode) (target, op0, op1, op2));
16406 10 : return target;
16407 : }
16408 :
16409 21 : case IX86_BUILTIN_RDRAND16_STEP:
16410 21 : mode = HImode;
16411 21 : goto rdrand_step;
16412 :
16413 42 : case IX86_BUILTIN_RDRAND32_STEP:
16414 42 : mode = SImode;
16415 42 : goto rdrand_step;
16416 :
16417 : case IX86_BUILTIN_RDRAND64_STEP:
16418 : mode = DImode;
16419 :
16420 84 : rdrand_step:
16421 84 : arg0 = CALL_EXPR_ARG (exp, 0);
16422 84 : op1 = expand_normal (arg0);
16423 84 : if (!address_operand (op1, VOIDmode))
16424 : {
16425 29 : op1 = convert_memory_address (Pmode, op1);
16426 29 : op1 = copy_addr_to_reg (op1);
16427 : }
16428 :
16429 84 : op0 = gen_reg_rtx (mode);
16430 84 : emit_insn (gen_rdrand (mode, op0));
16431 :
16432 84 : emit_move_insn (gen_rtx_MEM (mode, op1), op0);
16433 :
16434 84 : op1 = force_reg (SImode, const1_rtx);
16435 :
16436 : /* Emit SImode conditional move. */
16437 84 : if (mode == HImode)
16438 : {
16439 21 : if (TARGET_ZERO_EXTEND_WITH_AND
16440 21 : && optimize_function_for_speed_p (cfun))
16441 : {
16442 0 : op2 = force_reg (SImode, const0_rtx);
16443 :
16444 0 : emit_insn (gen_movstricthi
16445 0 : (gen_lowpart (HImode, op2), op0));
16446 : }
16447 : else
16448 : {
16449 21 : op2 = gen_reg_rtx (SImode);
16450 :
16451 21 : emit_insn (gen_zero_extendhisi2 (op2, op0));
16452 : }
16453 : }
16454 63 : else if (mode == SImode)
16455 : op2 = op0;
16456 : else
16457 21 : op2 = gen_rtx_SUBREG (SImode, op0, 0);
16458 :
16459 84 : if (target == 0
16460 84 : || !register_operand (target, SImode))
16461 7 : target = gen_reg_rtx (SImode);
16462 :
16463 84 : pat = gen_rtx_GEU (VOIDmode, gen_rtx_REG (CCCmode, FLAGS_REG),
16464 : const0_rtx);
16465 84 : emit_insn (gen_rtx_SET (target,
16466 : gen_rtx_IF_THEN_ELSE (SImode, pat, op2, op1)));
16467 84 : return target;
16468 :
16469 19 : case IX86_BUILTIN_RDSEED16_STEP:
16470 19 : mode = HImode;
16471 19 : goto rdseed_step;
16472 :
16473 28 : case IX86_BUILTIN_RDSEED32_STEP:
16474 28 : mode = SImode;
16475 28 : goto rdseed_step;
16476 :
16477 : case IX86_BUILTIN_RDSEED64_STEP:
16478 : mode = DImode;
16479 :
16480 67 : rdseed_step:
16481 67 : arg0 = CALL_EXPR_ARG (exp, 0);
16482 67 : op1 = expand_normal (arg0);
16483 67 : if (!address_operand (op1, VOIDmode))
16484 : {
16485 28 : op1 = convert_memory_address (Pmode, op1);
16486 28 : op1 = copy_addr_to_reg (op1);
16487 : }
16488 :
16489 67 : op0 = gen_reg_rtx (mode);
16490 67 : emit_insn (gen_rdseed (mode, op0));
16491 :
16492 67 : emit_move_insn (gen_rtx_MEM (mode, op1), op0);
16493 :
16494 67 : op2 = gen_reg_rtx (QImode);
16495 :
16496 67 : pat = gen_rtx_LTU (QImode, gen_rtx_REG (CCCmode, FLAGS_REG),
16497 : const0_rtx);
16498 67 : emit_insn (gen_rtx_SET (op2, pat));
16499 :
16500 67 : if (target == 0
16501 67 : || !register_operand (target, SImode))
16502 1 : target = gen_reg_rtx (SImode);
16503 :
16504 67 : emit_insn (gen_zero_extendqisi2 (target, op2));
16505 67 : return target;
16506 :
16507 38 : case IX86_BUILTIN_SBB32:
16508 38 : icode = CODE_FOR_subborrowsi;
16509 38 : icode2 = CODE_FOR_subborrowsi_0;
16510 38 : mode0 = SImode;
16511 38 : mode1 = DImode;
16512 38 : mode2 = CCmode;
16513 38 : goto handlecarry;
16514 :
16515 45 : case IX86_BUILTIN_SBB64:
16516 45 : icode = CODE_FOR_subborrowdi;
16517 45 : icode2 = CODE_FOR_subborrowdi_0;
16518 45 : mode0 = DImode;
16519 45 : mode1 = TImode;
16520 45 : mode2 = CCmode;
16521 45 : goto handlecarry;
16522 :
16523 68 : case IX86_BUILTIN_ADDCARRYX32:
16524 68 : icode = CODE_FOR_addcarrysi;
16525 68 : icode2 = CODE_FOR_addcarrysi_0;
16526 68 : mode0 = SImode;
16527 68 : mode1 = DImode;
16528 68 : mode2 = CCCmode;
16529 68 : goto handlecarry;
16530 :
16531 : case IX86_BUILTIN_ADDCARRYX64:
16532 : icode = CODE_FOR_addcarrydi;
16533 : icode2 = CODE_FOR_addcarrydi_0;
16534 : mode0 = DImode;
16535 : mode1 = TImode;
16536 : mode2 = CCCmode;
16537 :
16538 215 : handlecarry:
16539 215 : arg0 = CALL_EXPR_ARG (exp, 0); /* unsigned char c_in. */
16540 215 : arg1 = CALL_EXPR_ARG (exp, 1); /* unsigned int src1. */
16541 215 : arg2 = CALL_EXPR_ARG (exp, 2); /* unsigned int src2. */
16542 215 : arg3 = CALL_EXPR_ARG (exp, 3); /* unsigned int *sum_out. */
16543 :
16544 215 : op1 = expand_normal (arg0);
16545 :
16546 215 : op2 = expand_normal (arg1);
16547 215 : if (!register_operand (op2, mode0))
16548 120 : op2 = copy_to_mode_reg (mode0, op2);
16549 :
16550 215 : op3 = expand_normal (arg2);
16551 215 : if (!register_operand (op3, mode0))
16552 123 : op3 = copy_to_mode_reg (mode0, op3);
16553 :
16554 215 : op4 = expand_normal (arg3);
16555 215 : if (!address_operand (op4, VOIDmode))
16556 : {
16557 68 : op4 = convert_memory_address (Pmode, op4);
16558 68 : op4 = copy_addr_to_reg (op4);
16559 : }
16560 :
16561 215 : op0 = gen_reg_rtx (mode0);
16562 215 : if (op1 == const0_rtx)
16563 : {
16564 : /* If arg0 is 0, optimize right away into add or sub
16565 : instruction that sets CCCmode flags. */
16566 21 : op1 = gen_rtx_REG (mode2, FLAGS_REG);
16567 21 : emit_insn (GEN_FCN (icode2) (op0, op2, op3));
16568 : }
16569 : else
16570 : {
16571 : /* Generate CF from input operand. */
16572 194 : ix86_expand_carry (op1);
16573 :
16574 : /* Generate instruction that consumes CF. */
16575 194 : op1 = gen_rtx_REG (CCCmode, FLAGS_REG);
16576 194 : pat = gen_rtx_LTU (mode1, op1, const0_rtx);
16577 194 : pat2 = gen_rtx_LTU (mode0, op1, const0_rtx);
16578 194 : emit_insn (GEN_FCN (icode) (op0, op2, op3, op1, pat, pat2));
16579 : }
16580 :
16581 : /* Return current CF value. */
16582 215 : if (target == 0)
16583 14 : target = gen_reg_rtx (QImode);
16584 :
16585 215 : pat = gen_rtx_LTU (QImode, op1, const0_rtx);
16586 215 : emit_insn (gen_rtx_SET (target, pat));
16587 :
16588 : /* Store the result. */
16589 215 : emit_move_insn (gen_rtx_MEM (mode0, op4), op0);
16590 :
16591 215 : return target;
16592 :
16593 24 : case IX86_BUILTIN_READ_FLAGS:
16594 24 : if (ignore)
16595 3 : return const0_rtx;
16596 :
16597 21 : emit_insn (gen_pushfl ());
16598 :
16599 21 : if (optimize
16600 11 : || target == NULL_RTX
16601 11 : || !nonimmediate_operand (target, word_mode)
16602 32 : || GET_MODE (target) != word_mode)
16603 10 : target = gen_reg_rtx (word_mode);
16604 :
16605 21 : emit_insn (gen_pop (target));
16606 21 : return target;
16607 :
16608 21 : case IX86_BUILTIN_WRITE_FLAGS:
16609 :
16610 21 : arg0 = CALL_EXPR_ARG (exp, 0);
16611 21 : op0 = expand_normal (arg0);
16612 21 : if (!general_no_elim_operand (op0, word_mode))
16613 0 : op0 = copy_to_mode_reg (word_mode, op0);
16614 :
16615 21 : emit_insn (gen_push (op0));
16616 21 : emit_insn (gen_popfl ());
16617 21 : return 0;
16618 :
16619 22 : case IX86_BUILTIN_KTESTC8:
16620 22 : icode = CODE_FOR_ktestqi;
16621 22 : mode3 = CCCmode;
16622 22 : goto kortest;
16623 :
16624 22 : case IX86_BUILTIN_KTESTZ8:
16625 22 : icode = CODE_FOR_ktestqi;
16626 22 : mode3 = CCZmode;
16627 22 : goto kortest;
16628 :
16629 22 : case IX86_BUILTIN_KTESTC16:
16630 22 : icode = CODE_FOR_ktesthi;
16631 22 : mode3 = CCCmode;
16632 22 : goto kortest;
16633 :
16634 22 : case IX86_BUILTIN_KTESTZ16:
16635 22 : icode = CODE_FOR_ktesthi;
16636 22 : mode3 = CCZmode;
16637 22 : goto kortest;
16638 :
16639 22 : case IX86_BUILTIN_KTESTC32:
16640 22 : icode = CODE_FOR_ktestsi;
16641 22 : mode3 = CCCmode;
16642 22 : goto kortest;
16643 :
16644 22 : case IX86_BUILTIN_KTESTZ32:
16645 22 : icode = CODE_FOR_ktestsi;
16646 22 : mode3 = CCZmode;
16647 22 : goto kortest;
16648 :
16649 22 : case IX86_BUILTIN_KTESTC64:
16650 22 : icode = CODE_FOR_ktestdi;
16651 22 : mode3 = CCCmode;
16652 22 : goto kortest;
16653 :
16654 22 : case IX86_BUILTIN_KTESTZ64:
16655 22 : icode = CODE_FOR_ktestdi;
16656 22 : mode3 = CCZmode;
16657 22 : goto kortest;
16658 :
16659 22 : case IX86_BUILTIN_KORTESTC8:
16660 22 : icode = CODE_FOR_kortestqi;
16661 22 : mode3 = CCCmode;
16662 22 : goto kortest;
16663 :
16664 76 : case IX86_BUILTIN_KORTESTZ8:
16665 76 : icode = CODE_FOR_kortestqi;
16666 76 : mode3 = CCZmode;
16667 76 : goto kortest;
16668 :
16669 38 : case IX86_BUILTIN_KORTESTC16:
16670 38 : icode = CODE_FOR_kortesthi;
16671 38 : mode3 = CCCmode;
16672 38 : goto kortest;
16673 :
16674 91 : case IX86_BUILTIN_KORTESTZ16:
16675 91 : icode = CODE_FOR_kortesthi;
16676 91 : mode3 = CCZmode;
16677 91 : goto kortest;
16678 :
16679 22 : case IX86_BUILTIN_KORTESTC32:
16680 22 : icode = CODE_FOR_kortestsi;
16681 22 : mode3 = CCCmode;
16682 22 : goto kortest;
16683 :
16684 79 : case IX86_BUILTIN_KORTESTZ32:
16685 79 : icode = CODE_FOR_kortestsi;
16686 79 : mode3 = CCZmode;
16687 79 : goto kortest;
16688 :
16689 22 : case IX86_BUILTIN_KORTESTC64:
16690 22 : icode = CODE_FOR_kortestdi;
16691 22 : mode3 = CCCmode;
16692 22 : goto kortest;
16693 :
16694 : case IX86_BUILTIN_KORTESTZ64:
16695 : icode = CODE_FOR_kortestdi;
16696 : mode3 = CCZmode;
16697 :
16698 610 : kortest:
16699 610 : arg0 = CALL_EXPR_ARG (exp, 0); /* Mask reg src1. */
16700 610 : arg1 = CALL_EXPR_ARG (exp, 1); /* Mask reg src2. */
16701 610 : op0 = expand_normal (arg0);
16702 610 : op1 = expand_normal (arg1);
16703 :
16704 610 : mode0 = insn_data[icode].operand[0].mode;
16705 610 : mode1 = insn_data[icode].operand[1].mode;
16706 :
16707 610 : if (GET_MODE (op0) != VOIDmode)
16708 610 : op0 = force_reg (GET_MODE (op0), op0);
16709 :
16710 610 : op0 = gen_lowpart (mode0, op0);
16711 :
16712 610 : if (!insn_data[icode].operand[0].predicate (op0, mode0))
16713 0 : op0 = copy_to_mode_reg (mode0, op0);
16714 :
16715 610 : if (GET_MODE (op1) != VOIDmode)
16716 609 : op1 = force_reg (GET_MODE (op1), op1);
16717 :
16718 610 : op1 = gen_lowpart (mode1, op1);
16719 :
16720 610 : if (!insn_data[icode].operand[1].predicate (op1, mode1))
16721 1 : op1 = copy_to_mode_reg (mode1, op1);
16722 :
16723 610 : target = gen_reg_rtx (QImode);
16724 :
16725 : /* Emit kortest. */
16726 610 : emit_insn (GEN_FCN (icode) (op0, op1));
16727 : /* And use setcc to return result from flags. */
16728 610 : ix86_expand_setcc (target, EQ,
16729 : gen_rtx_REG (mode3, FLAGS_REG), const0_rtx);
16730 610 : return target;
16731 :
16732 20 : case IX86_BUILTIN_GATHERSIV2DF:
16733 20 : icode = CODE_FOR_avx2_gathersiv2df;
16734 20 : goto gather_gen;
16735 18 : case IX86_BUILTIN_GATHERSIV4DF:
16736 18 : icode = CODE_FOR_avx2_gathersiv4df;
16737 18 : goto gather_gen;
16738 21 : case IX86_BUILTIN_GATHERDIV2DF:
16739 21 : icode = CODE_FOR_avx2_gatherdiv2df;
16740 21 : goto gather_gen;
16741 32 : case IX86_BUILTIN_GATHERDIV4DF:
16742 32 : icode = CODE_FOR_avx2_gatherdiv4df;
16743 32 : goto gather_gen;
16744 28 : case IX86_BUILTIN_GATHERSIV4SF:
16745 28 : icode = CODE_FOR_avx2_gathersiv4sf;
16746 28 : goto gather_gen;
16747 35 : case IX86_BUILTIN_GATHERSIV8SF:
16748 35 : icode = CODE_FOR_avx2_gathersiv8sf;
16749 35 : goto gather_gen;
16750 24 : case IX86_BUILTIN_GATHERDIV4SF:
16751 24 : icode = CODE_FOR_avx2_gatherdiv4sf;
16752 24 : goto gather_gen;
16753 18 : case IX86_BUILTIN_GATHERDIV8SF:
16754 18 : icode = CODE_FOR_avx2_gatherdiv8sf;
16755 18 : goto gather_gen;
16756 18 : case IX86_BUILTIN_GATHERSIV2DI:
16757 18 : icode = CODE_FOR_avx2_gathersiv2di;
16758 18 : goto gather_gen;
16759 18 : case IX86_BUILTIN_GATHERSIV4DI:
16760 18 : icode = CODE_FOR_avx2_gathersiv4di;
16761 18 : goto gather_gen;
16762 27 : case IX86_BUILTIN_GATHERDIV2DI:
16763 27 : icode = CODE_FOR_avx2_gatherdiv2di;
16764 27 : goto gather_gen;
16765 29 : case IX86_BUILTIN_GATHERDIV4DI:
16766 29 : icode = CODE_FOR_avx2_gatherdiv4di;
16767 29 : goto gather_gen;
16768 20 : case IX86_BUILTIN_GATHERSIV4SI:
16769 20 : icode = CODE_FOR_avx2_gathersiv4si;
16770 20 : goto gather_gen;
16771 22 : case IX86_BUILTIN_GATHERSIV8SI:
16772 22 : icode = CODE_FOR_avx2_gathersiv8si;
16773 22 : goto gather_gen;
16774 28 : case IX86_BUILTIN_GATHERDIV4SI:
16775 28 : icode = CODE_FOR_avx2_gatherdiv4si;
16776 28 : goto gather_gen;
16777 18 : case IX86_BUILTIN_GATHERDIV8SI:
16778 18 : icode = CODE_FOR_avx2_gatherdiv8si;
16779 18 : goto gather_gen;
16780 16 : case IX86_BUILTIN_GATHERALTSIV4DF:
16781 16 : icode = CODE_FOR_avx2_gathersiv4df;
16782 16 : goto gather_gen;
16783 16 : case IX86_BUILTIN_GATHERALTDIV8SF:
16784 16 : icode = CODE_FOR_avx2_gatherdiv8sf;
16785 16 : goto gather_gen;
16786 4 : case IX86_BUILTIN_GATHERALTSIV4DI:
16787 4 : icode = CODE_FOR_avx2_gathersiv4di;
16788 4 : goto gather_gen;
16789 12 : case IX86_BUILTIN_GATHERALTDIV8SI:
16790 12 : icode = CODE_FOR_avx2_gatherdiv8si;
16791 12 : goto gather_gen;
16792 36 : case IX86_BUILTIN_GATHER3SIV16SF:
16793 36 : icode = CODE_FOR_avx512f_gathersiv16sf;
16794 36 : goto gather_gen;
16795 24 : case IX86_BUILTIN_GATHER3SIV8DF:
16796 24 : icode = CODE_FOR_avx512f_gathersiv8df;
16797 24 : goto gather_gen;
16798 24 : case IX86_BUILTIN_GATHER3DIV16SF:
16799 24 : icode = CODE_FOR_avx512f_gatherdiv16sf;
16800 24 : goto gather_gen;
16801 37 : case IX86_BUILTIN_GATHER3DIV8DF:
16802 37 : icode = CODE_FOR_avx512f_gatherdiv8df;
16803 37 : goto gather_gen;
16804 30 : case IX86_BUILTIN_GATHER3SIV16SI:
16805 30 : icode = CODE_FOR_avx512f_gathersiv16si;
16806 30 : goto gather_gen;
16807 24 : case IX86_BUILTIN_GATHER3SIV8DI:
16808 24 : icode = CODE_FOR_avx512f_gathersiv8di;
16809 24 : goto gather_gen;
16810 24 : case IX86_BUILTIN_GATHER3DIV16SI:
16811 24 : icode = CODE_FOR_avx512f_gatherdiv16si;
16812 24 : goto gather_gen;
16813 38 : case IX86_BUILTIN_GATHER3DIV8DI:
16814 38 : icode = CODE_FOR_avx512f_gatherdiv8di;
16815 38 : goto gather_gen;
16816 16 : case IX86_BUILTIN_GATHER3ALTSIV8DF:
16817 16 : icode = CODE_FOR_avx512f_gathersiv8df;
16818 16 : goto gather_gen;
16819 22 : case IX86_BUILTIN_GATHER3ALTDIV16SF:
16820 22 : icode = CODE_FOR_avx512f_gatherdiv16sf;
16821 22 : goto gather_gen;
16822 14 : case IX86_BUILTIN_GATHER3ALTSIV8DI:
16823 14 : icode = CODE_FOR_avx512f_gathersiv8di;
16824 14 : goto gather_gen;
16825 18 : case IX86_BUILTIN_GATHER3ALTDIV16SI:
16826 18 : icode = CODE_FOR_avx512f_gatherdiv16si;
16827 18 : goto gather_gen;
16828 18 : case IX86_BUILTIN_GATHER3SIV2DF:
16829 18 : icode = CODE_FOR_avx512vl_gathersiv2df;
16830 18 : goto gather_gen;
16831 10 : case IX86_BUILTIN_GATHER3SIV4DF:
16832 10 : icode = CODE_FOR_avx512vl_gathersiv4df;
16833 10 : goto gather_gen;
16834 15 : case IX86_BUILTIN_GATHER3DIV2DF:
16835 15 : icode = CODE_FOR_avx512vl_gatherdiv2df;
16836 15 : goto gather_gen;
16837 16 : case IX86_BUILTIN_GATHER3DIV4DF:
16838 16 : icode = CODE_FOR_avx512vl_gatherdiv4df;
16839 16 : goto gather_gen;
16840 14 : case IX86_BUILTIN_GATHER3SIV4SF:
16841 14 : icode = CODE_FOR_avx512vl_gathersiv4sf;
16842 14 : goto gather_gen;
16843 12 : case IX86_BUILTIN_GATHER3SIV8SF:
16844 12 : icode = CODE_FOR_avx512vl_gathersiv8sf;
16845 12 : goto gather_gen;
16846 22 : case IX86_BUILTIN_GATHER3DIV4SF:
16847 22 : icode = CODE_FOR_avx512vl_gatherdiv4sf;
16848 22 : goto gather_gen;
16849 10 : case IX86_BUILTIN_GATHER3DIV8SF:
16850 10 : icode = CODE_FOR_avx512vl_gatherdiv8sf;
16851 10 : goto gather_gen;
16852 20 : case IX86_BUILTIN_GATHER3SIV2DI:
16853 20 : icode = CODE_FOR_avx512vl_gathersiv2di;
16854 20 : goto gather_gen;
16855 10 : case IX86_BUILTIN_GATHER3SIV4DI:
16856 10 : icode = CODE_FOR_avx512vl_gathersiv4di;
16857 10 : goto gather_gen;
16858 15 : case IX86_BUILTIN_GATHER3DIV2DI:
16859 15 : icode = CODE_FOR_avx512vl_gatherdiv2di;
16860 15 : goto gather_gen;
16861 14 : case IX86_BUILTIN_GATHER3DIV4DI:
16862 14 : icode = CODE_FOR_avx512vl_gatherdiv4di;
16863 14 : goto gather_gen;
16864 14 : case IX86_BUILTIN_GATHER3SIV4SI:
16865 14 : icode = CODE_FOR_avx512vl_gathersiv4si;
16866 14 : goto gather_gen;
16867 12 : case IX86_BUILTIN_GATHER3SIV8SI:
16868 12 : icode = CODE_FOR_avx512vl_gathersiv8si;
16869 12 : goto gather_gen;
16870 24 : case IX86_BUILTIN_GATHER3DIV4SI:
16871 24 : icode = CODE_FOR_avx512vl_gatherdiv4si;
16872 24 : goto gather_gen;
16873 10 : case IX86_BUILTIN_GATHER3DIV8SI:
16874 10 : icode = CODE_FOR_avx512vl_gatherdiv8si;
16875 10 : goto gather_gen;
16876 4 : case IX86_BUILTIN_GATHER3ALTSIV4DF:
16877 4 : icode = CODE_FOR_avx512vl_gathersiv4df;
16878 4 : goto gather_gen;
16879 8 : case IX86_BUILTIN_GATHER3ALTDIV8SF:
16880 8 : icode = CODE_FOR_avx512vl_gatherdiv8sf;
16881 8 : goto gather_gen;
16882 6 : case IX86_BUILTIN_GATHER3ALTSIV4DI:
16883 6 : icode = CODE_FOR_avx512vl_gathersiv4di;
16884 6 : goto gather_gen;
16885 10 : case IX86_BUILTIN_GATHER3ALTDIV8SI:
16886 10 : icode = CODE_FOR_avx512vl_gatherdiv8si;
16887 10 : goto gather_gen;
16888 40 : case IX86_BUILTIN_SCATTERSIV16SF:
16889 40 : icode = CODE_FOR_avx512f_scattersiv16sf;
16890 40 : goto scatter_gen;
16891 27 : case IX86_BUILTIN_SCATTERSIV8DF:
16892 27 : icode = CODE_FOR_avx512f_scattersiv8df;
16893 27 : goto scatter_gen;
16894 24 : case IX86_BUILTIN_SCATTERDIV16SF:
16895 24 : icode = CODE_FOR_avx512f_scatterdiv16sf;
16896 24 : goto scatter_gen;
16897 33 : case IX86_BUILTIN_SCATTERDIV8DF:
16898 33 : icode = CODE_FOR_avx512f_scatterdiv8df;
16899 33 : goto scatter_gen;
16900 30 : case IX86_BUILTIN_SCATTERSIV16SI:
16901 30 : icode = CODE_FOR_avx512f_scattersiv16si;
16902 30 : goto scatter_gen;
16903 24 : case IX86_BUILTIN_SCATTERSIV8DI:
16904 24 : icode = CODE_FOR_avx512f_scattersiv8di;
16905 24 : goto scatter_gen;
16906 24 : case IX86_BUILTIN_SCATTERDIV16SI:
16907 24 : icode = CODE_FOR_avx512f_scatterdiv16si;
16908 24 : goto scatter_gen;
16909 29 : case IX86_BUILTIN_SCATTERDIV8DI:
16910 29 : icode = CODE_FOR_avx512f_scatterdiv8di;
16911 29 : goto scatter_gen;
16912 18 : case IX86_BUILTIN_SCATTERSIV8SF:
16913 18 : icode = CODE_FOR_avx512vl_scattersiv8sf;
16914 18 : goto scatter_gen;
16915 20 : case IX86_BUILTIN_SCATTERSIV4SF:
16916 20 : icode = CODE_FOR_avx512vl_scattersiv4sf;
16917 20 : goto scatter_gen;
16918 16 : case IX86_BUILTIN_SCATTERSIV4DF:
16919 16 : icode = CODE_FOR_avx512vl_scattersiv4df;
16920 16 : goto scatter_gen;
16921 16 : case IX86_BUILTIN_SCATTERSIV2DF:
16922 16 : icode = CODE_FOR_avx512vl_scattersiv2df;
16923 16 : goto scatter_gen;
16924 16 : case IX86_BUILTIN_SCATTERDIV8SF:
16925 16 : icode = CODE_FOR_avx512vl_scatterdiv8sf;
16926 16 : goto scatter_gen;
16927 16 : case IX86_BUILTIN_SCATTERDIV4SF:
16928 16 : icode = CODE_FOR_avx512vl_scatterdiv4sf;
16929 16 : goto scatter_gen;
16930 18 : case IX86_BUILTIN_SCATTERDIV4DF:
16931 18 : icode = CODE_FOR_avx512vl_scatterdiv4df;
16932 18 : goto scatter_gen;
16933 18 : case IX86_BUILTIN_SCATTERDIV2DF:
16934 18 : icode = CODE_FOR_avx512vl_scatterdiv2df;
16935 18 : goto scatter_gen;
16936 22 : case IX86_BUILTIN_SCATTERSIV8SI:
16937 22 : icode = CODE_FOR_avx512vl_scattersiv8si;
16938 22 : goto scatter_gen;
16939 24 : case IX86_BUILTIN_SCATTERSIV4SI:
16940 24 : icode = CODE_FOR_avx512vl_scattersiv4si;
16941 24 : goto scatter_gen;
16942 16 : case IX86_BUILTIN_SCATTERSIV4DI:
16943 16 : icode = CODE_FOR_avx512vl_scattersiv4di;
16944 16 : goto scatter_gen;
16945 16 : case IX86_BUILTIN_SCATTERSIV2DI:
16946 16 : icode = CODE_FOR_avx512vl_scattersiv2di;
16947 16 : goto scatter_gen;
16948 16 : case IX86_BUILTIN_SCATTERDIV8SI:
16949 16 : icode = CODE_FOR_avx512vl_scatterdiv8si;
16950 16 : goto scatter_gen;
16951 16 : case IX86_BUILTIN_SCATTERDIV4SI:
16952 16 : icode = CODE_FOR_avx512vl_scatterdiv4si;
16953 16 : goto scatter_gen;
16954 18 : case IX86_BUILTIN_SCATTERDIV4DI:
16955 18 : icode = CODE_FOR_avx512vl_scatterdiv4di;
16956 18 : goto scatter_gen;
16957 18 : case IX86_BUILTIN_SCATTERDIV2DI:
16958 18 : icode = CODE_FOR_avx512vl_scatterdiv2di;
16959 18 : goto scatter_gen;
16960 16 : case IX86_BUILTIN_SCATTERALTSIV8DF:
16961 16 : icode = CODE_FOR_avx512f_scattersiv8df;
16962 16 : goto scatter_gen;
16963 12 : case IX86_BUILTIN_SCATTERALTDIV16SF:
16964 12 : icode = CODE_FOR_avx512f_scatterdiv16sf;
16965 12 : goto scatter_gen;
16966 8 : case IX86_BUILTIN_SCATTERALTSIV8DI:
16967 8 : icode = CODE_FOR_avx512f_scattersiv8di;
16968 8 : goto scatter_gen;
16969 24 : case IX86_BUILTIN_SCATTERALTDIV16SI:
16970 24 : icode = CODE_FOR_avx512f_scatterdiv16si;
16971 24 : goto scatter_gen;
16972 4 : case IX86_BUILTIN_SCATTERALTSIV4DF:
16973 4 : icode = CODE_FOR_avx512vl_scattersiv4df;
16974 4 : goto scatter_gen;
16975 4 : case IX86_BUILTIN_SCATTERALTDIV8SF:
16976 4 : icode = CODE_FOR_avx512vl_scatterdiv8sf;
16977 4 : goto scatter_gen;
16978 4 : case IX86_BUILTIN_SCATTERALTSIV4DI:
16979 4 : icode = CODE_FOR_avx512vl_scattersiv4di;
16980 4 : goto scatter_gen;
16981 4 : case IX86_BUILTIN_SCATTERALTDIV8SI:
16982 4 : icode = CODE_FOR_avx512vl_scatterdiv8si;
16983 4 : goto scatter_gen;
16984 8 : case IX86_BUILTIN_SCATTERALTSIV2DF:
16985 8 : icode = CODE_FOR_avx512vl_scattersiv2df;
16986 8 : goto scatter_gen;
16987 8 : case IX86_BUILTIN_SCATTERALTDIV4SF:
16988 8 : icode = CODE_FOR_avx512vl_scatterdiv4sf;
16989 8 : goto scatter_gen;
16990 8 : case IX86_BUILTIN_SCATTERALTSIV2DI:
16991 8 : icode = CODE_FOR_avx512vl_scattersiv2di;
16992 8 : goto scatter_gen;
16993 8 : case IX86_BUILTIN_SCATTERALTDIV4SI:
16994 8 : icode = CODE_FOR_avx512vl_scatterdiv4si;
16995 8 : goto scatter_gen;
16996 :
16997 995 : gather_gen:
16998 995 : rtx half;
16999 995 : rtx (*gen) (rtx, rtx);
17000 :
17001 995 : arg0 = CALL_EXPR_ARG (exp, 0);
17002 995 : arg1 = CALL_EXPR_ARG (exp, 1);
17003 995 : arg2 = CALL_EXPR_ARG (exp, 2);
17004 995 : arg3 = CALL_EXPR_ARG (exp, 3);
17005 995 : arg4 = CALL_EXPR_ARG (exp, 4);
17006 995 : op0 = expand_normal (arg0);
17007 995 : op1 = expand_normal (arg1);
17008 995 : op2 = expand_normal (arg2);
17009 995 : op3 = ix86_expand_unsigned_small_int_cst_argument (arg3);
17010 995 : op4 = expand_normal (arg4);
17011 : /* Note the arg order is different from the operand order. */
17012 995 : mode0 = insn_data[icode].operand[1].mode;
17013 995 : mode2 = insn_data[icode].operand[3].mode;
17014 995 : mode3 = insn_data[icode].operand[4].mode;
17015 995 : mode4 = insn_data[icode].operand[5].mode;
17016 :
17017 995 : if (target == NULL_RTX
17018 995 : || GET_MODE (target) != insn_data[icode].operand[0].mode
17019 1886 : || !insn_data[icode].operand[0].predicate (target,
17020 : GET_MODE (target)))
17021 105 : subtarget = gen_reg_rtx (insn_data[icode].operand[0].mode);
17022 : else
17023 : subtarget = target;
17024 :
17025 995 : switch (fcode)
17026 : {
17027 30 : case IX86_BUILTIN_GATHER3ALTSIV8DF:
17028 30 : case IX86_BUILTIN_GATHER3ALTSIV8DI:
17029 30 : half = gen_reg_rtx (V8SImode);
17030 30 : if (!nonimmediate_operand (op2, V16SImode))
17031 0 : op2 = copy_to_mode_reg (V16SImode, op2);
17032 30 : emit_insn (gen_vec_extract_lo_v16si (half, op2));
17033 30 : op2 = half;
17034 30 : break;
17035 30 : case IX86_BUILTIN_GATHER3ALTSIV4DF:
17036 30 : case IX86_BUILTIN_GATHER3ALTSIV4DI:
17037 30 : case IX86_BUILTIN_GATHERALTSIV4DF:
17038 30 : case IX86_BUILTIN_GATHERALTSIV4DI:
17039 30 : half = gen_reg_rtx (V4SImode);
17040 30 : if (!nonimmediate_operand (op2, V8SImode))
17041 0 : op2 = copy_to_mode_reg (V8SImode, op2);
17042 30 : emit_insn (gen_vec_extract_lo_v8si (half, op2));
17043 30 : op2 = half;
17044 30 : break;
17045 40 : case IX86_BUILTIN_GATHER3ALTDIV16SF:
17046 40 : case IX86_BUILTIN_GATHER3ALTDIV16SI:
17047 40 : half = gen_reg_rtx (mode0);
17048 40 : if (mode0 == V8SFmode)
17049 : gen = gen_vec_extract_lo_v16sf;
17050 : else
17051 18 : gen = gen_vec_extract_lo_v16si;
17052 40 : if (!nonimmediate_operand (op0, GET_MODE (op0)))
17053 40 : op0 = copy_to_mode_reg (GET_MODE (op0), op0);
17054 40 : emit_insn (gen (half, op0));
17055 40 : op0 = half;
17056 40 : op3 = lowpart_subreg (QImode, op3, HImode);
17057 40 : break;
17058 46 : case IX86_BUILTIN_GATHER3ALTDIV8SF:
17059 46 : case IX86_BUILTIN_GATHER3ALTDIV8SI:
17060 46 : case IX86_BUILTIN_GATHERALTDIV8SF:
17061 46 : case IX86_BUILTIN_GATHERALTDIV8SI:
17062 46 : half = gen_reg_rtx (mode0);
17063 46 : if (mode0 == V4SFmode)
17064 : gen = gen_vec_extract_lo_v8sf;
17065 : else
17066 22 : gen = gen_vec_extract_lo_v8si;
17067 46 : if (!nonimmediate_operand (op0, GET_MODE (op0)))
17068 46 : op0 = copy_to_mode_reg (GET_MODE (op0), op0);
17069 46 : emit_insn (gen (half, op0));
17070 46 : op0 = half;
17071 46 : if (VECTOR_MODE_P (GET_MODE (op3)))
17072 : {
17073 28 : half = gen_reg_rtx (mode0);
17074 28 : if (!nonimmediate_operand (op3, GET_MODE (op3)))
17075 12 : op3 = copy_to_mode_reg (GET_MODE (op3), op3);
17076 28 : emit_insn (gen (half, op3));
17077 28 : op3 = half;
17078 : }
17079 : break;
17080 : default:
17081 : break;
17082 : }
17083 :
17084 : /* Force memory operand only with base register here. But we
17085 : don't want to do it on memory operand for other builtin
17086 : functions. */
17087 995 : op1 = ix86_zero_extend_to_Pmode (op1);
17088 :
17089 995 : if (!insn_data[icode].operand[1].predicate (op0, mode0))
17090 394 : op0 = copy_to_mode_reg (mode0, op0);
17091 998 : if (!insn_data[icode].operand[2].predicate (op1, Pmode))
17092 0 : op1 = copy_to_mode_reg (Pmode, op1);
17093 995 : if (!insn_data[icode].operand[3].predicate (op2, mode2))
17094 224 : op2 = copy_to_mode_reg (mode2, op2);
17095 :
17096 995 : op3 = fixup_modeless_constant (op3, mode3);
17097 :
17098 995 : if (GET_MODE (op3) == mode3 || GET_MODE (op3) == VOIDmode)
17099 : {
17100 995 : if (!insn_data[icode].operand[4].predicate (op3, mode3))
17101 352 : op3 = copy_to_mode_reg (mode3, op3);
17102 : }
17103 : else
17104 : {
17105 0 : op3 = copy_to_reg (op3);
17106 0 : op3 = lowpart_subreg (mode3, op3, GET_MODE (op3));
17107 : }
17108 995 : if (!insn_data[icode].operand[5].predicate (op4, mode4))
17109 : {
17110 0 : error ("the last argument must be scale 1, 2, 4, 8");
17111 0 : return const0_rtx;
17112 : }
17113 :
17114 : /* Optimize. If mask is known to have all high bits set,
17115 : replace op0 with pc_rtx to signal that the instruction
17116 : overwrites the whole destination and doesn't use its
17117 : previous contents. */
17118 995 : if (optimize)
17119 : {
17120 905 : if (TREE_CODE (arg3) == INTEGER_CST)
17121 : {
17122 209 : if (integer_all_onesp (arg3))
17123 201 : op0 = pc_rtx;
17124 : }
17125 696 : else if (TREE_CODE (arg3) == VECTOR_CST)
17126 : {
17127 : unsigned int negative = 0;
17128 727 : for (i = 0; i < VECTOR_CST_NELTS (arg3); ++i)
17129 : {
17130 596 : tree cst = VECTOR_CST_ELT (arg3, i);
17131 596 : if (TREE_CODE (cst) == INTEGER_CST
17132 596 : && tree_int_cst_sign_bit (cst))
17133 286 : negative++;
17134 310 : else if (TREE_CODE (cst) == REAL_CST
17135 310 : && REAL_VALUE_NEGATIVE (TREE_REAL_CST (cst)))
17136 282 : negative++;
17137 : }
17138 131 : if (negative == TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg3)))
17139 117 : op0 = pc_rtx;
17140 : }
17141 565 : else if (TREE_CODE (arg3) == SSA_NAME
17142 565 : && VECTOR_TYPE_P (TREE_TYPE (arg3)))
17143 : {
17144 : /* Recognize also when mask is like:
17145 : __v2df src = _mm_setzero_pd ();
17146 : __v2df mask = _mm_cmpeq_pd (src, src);
17147 : or
17148 : __v8sf src = _mm256_setzero_ps ();
17149 : __v8sf mask = _mm256_cmp_ps (src, src, _CMP_EQ_OQ);
17150 : as that is a cheaper way to load all ones into
17151 : a register than having to load a constant from
17152 : memory. */
17153 251 : gimple *def_stmt = SSA_NAME_DEF_STMT (arg3);
17154 251 : if (is_gimple_call (def_stmt))
17155 : {
17156 76 : tree fndecl = gimple_call_fndecl (def_stmt);
17157 76 : if (fndecl
17158 76 : && fndecl_built_in_p (fndecl, BUILT_IN_MD))
17159 67 : switch (DECL_MD_FUNCTION_CODE (fndecl))
17160 : {
17161 24 : case IX86_BUILTIN_CMPPD:
17162 24 : case IX86_BUILTIN_CMPPS:
17163 24 : case IX86_BUILTIN_CMPPD256:
17164 24 : case IX86_BUILTIN_CMPPS256:
17165 24 : if (!integer_zerop (gimple_call_arg (def_stmt, 2)))
17166 : break;
17167 : /* FALLTHRU */
17168 49 : case IX86_BUILTIN_CMPEQPD:
17169 49 : case IX86_BUILTIN_CMPEQPS:
17170 49 : if (initializer_zerop (gimple_call_arg (def_stmt, 0))
17171 49 : && initializer_zerop (gimple_call_arg (def_stmt,
17172 : 1)))
17173 49 : op0 = pc_rtx;
17174 : break;
17175 : default:
17176 : break;
17177 : }
17178 : }
17179 : }
17180 : }
17181 :
17182 995 : pat = GEN_FCN (icode) (subtarget, op0, op1, op2, op3, op4);
17183 995 : if (! pat)
17184 0 : return const0_rtx;
17185 995 : emit_insn (pat);
17186 :
17187 995 : switch (fcode)
17188 : {
17189 24 : case IX86_BUILTIN_GATHER3DIV16SF:
17190 24 : if (target == NULL_RTX)
17191 0 : target = gen_reg_rtx (V8SFmode);
17192 24 : emit_insn (gen_vec_extract_lo_v16sf (target, subtarget));
17193 24 : break;
17194 24 : case IX86_BUILTIN_GATHER3DIV16SI:
17195 24 : if (target == NULL_RTX)
17196 0 : target = gen_reg_rtx (V8SImode);
17197 24 : emit_insn (gen_vec_extract_lo_v16si (target, subtarget));
17198 24 : break;
17199 28 : case IX86_BUILTIN_GATHER3DIV8SF:
17200 28 : case IX86_BUILTIN_GATHERDIV8SF:
17201 28 : if (target == NULL_RTX)
17202 0 : target = gen_reg_rtx (V4SFmode);
17203 28 : emit_insn (gen_vec_extract_lo_v8sf (target, subtarget));
17204 28 : break;
17205 28 : case IX86_BUILTIN_GATHER3DIV8SI:
17206 28 : case IX86_BUILTIN_GATHERDIV8SI:
17207 28 : if (target == NULL_RTX)
17208 0 : target = gen_reg_rtx (V4SImode);
17209 28 : emit_insn (gen_vec_extract_lo_v8si (target, subtarget));
17210 28 : break;
17211 : default:
17212 : target = subtarget;
17213 : break;
17214 : }
17215 : return target;
17216 :
17217 623 : scatter_gen:
17218 623 : arg0 = CALL_EXPR_ARG (exp, 0);
17219 623 : arg1 = CALL_EXPR_ARG (exp, 1);
17220 623 : arg2 = CALL_EXPR_ARG (exp, 2);
17221 623 : arg3 = CALL_EXPR_ARG (exp, 3);
17222 623 : arg4 = CALL_EXPR_ARG (exp, 4);
17223 623 : op0 = expand_normal (arg0);
17224 623 : op1 = ix86_expand_unsigned_small_int_cst_argument (arg1);
17225 623 : op2 = expand_normal (arg2);
17226 623 : op3 = expand_normal (arg3);
17227 623 : op4 = expand_normal (arg4);
17228 623 : mode1 = insn_data[icode].operand[1].mode;
17229 623 : mode2 = insn_data[icode].operand[2].mode;
17230 623 : mode3 = insn_data[icode].operand[3].mode;
17231 623 : mode4 = insn_data[icode].operand[4].mode;
17232 :
17233 : /* Scatter instruction stores operand op3 to memory with
17234 : indices from op2 and scale from op4 under writemask op1.
17235 : If index operand op2 has more elements then source operand
17236 : op3 one need to use only its low half. And vice versa. */
17237 623 : switch (fcode)
17238 : {
17239 24 : case IX86_BUILTIN_SCATTERALTSIV8DF:
17240 24 : case IX86_BUILTIN_SCATTERALTSIV8DI:
17241 24 : half = gen_reg_rtx (V8SImode);
17242 24 : if (!nonimmediate_operand (op2, V16SImode))
17243 0 : op2 = copy_to_mode_reg (V16SImode, op2);
17244 24 : emit_insn (gen_vec_extract_lo_v16si (half, op2));
17245 24 : op2 = half;
17246 24 : break;
17247 36 : case IX86_BUILTIN_SCATTERALTDIV16SF:
17248 36 : case IX86_BUILTIN_SCATTERALTDIV16SI:
17249 36 : half = gen_reg_rtx (mode3);
17250 36 : if (mode3 == V8SFmode)
17251 : gen = gen_vec_extract_lo_v16sf;
17252 : else
17253 24 : gen = gen_vec_extract_lo_v16si;
17254 36 : if (!nonimmediate_operand (op3, GET_MODE (op3)))
17255 0 : op3 = copy_to_mode_reg (GET_MODE (op3), op3);
17256 36 : emit_insn (gen (half, op3));
17257 36 : op3 = half;
17258 36 : break;
17259 8 : case IX86_BUILTIN_SCATTERALTSIV4DF:
17260 8 : case IX86_BUILTIN_SCATTERALTSIV4DI:
17261 8 : half = gen_reg_rtx (V4SImode);
17262 8 : if (!nonimmediate_operand (op2, V8SImode))
17263 0 : op2 = copy_to_mode_reg (V8SImode, op2);
17264 8 : emit_insn (gen_vec_extract_lo_v8si (half, op2));
17265 8 : op2 = half;
17266 8 : break;
17267 8 : case IX86_BUILTIN_SCATTERALTDIV8SF:
17268 8 : case IX86_BUILTIN_SCATTERALTDIV8SI:
17269 8 : half = gen_reg_rtx (mode3);
17270 8 : if (mode3 == V4SFmode)
17271 : gen = gen_vec_extract_lo_v8sf;
17272 : else
17273 4 : gen = gen_vec_extract_lo_v8si;
17274 8 : if (!nonimmediate_operand (op3, GET_MODE (op3)))
17275 0 : op3 = copy_to_mode_reg (GET_MODE (op3), op3);
17276 8 : emit_insn (gen (half, op3));
17277 8 : op3 = half;
17278 8 : break;
17279 16 : case IX86_BUILTIN_SCATTERALTSIV2DF:
17280 16 : case IX86_BUILTIN_SCATTERALTSIV2DI:
17281 16 : if (!nonimmediate_operand (op2, V4SImode))
17282 0 : op2 = copy_to_mode_reg (V4SImode, op2);
17283 : break;
17284 16 : case IX86_BUILTIN_SCATTERALTDIV4SF:
17285 16 : case IX86_BUILTIN_SCATTERALTDIV4SI:
17286 16 : if (!nonimmediate_operand (op3, GET_MODE (op3)))
17287 0 : op3 = copy_to_mode_reg (GET_MODE (op3), op3);
17288 : break;
17289 : default:
17290 : break;
17291 : }
17292 :
17293 : /* Force memory operand only with base register here. But we
17294 : don't want to do it on memory operand for other builtin
17295 : functions. */
17296 633 : op0 = force_reg (Pmode, convert_to_mode (Pmode, op0, 1));
17297 :
17298 628 : if (!insn_data[icode].operand[0].predicate (op0, Pmode))
17299 0 : op0 = copy_to_mode_reg (Pmode, op0);
17300 :
17301 623 : op1 = fixup_modeless_constant (op1, mode1);
17302 :
17303 623 : if (GET_MODE (op1) == mode1 || GET_MODE (op1) == VOIDmode)
17304 : {
17305 607 : if (!insn_data[icode].operand[1].predicate (op1, mode1))
17306 273 : op1 = copy_to_mode_reg (mode1, op1);
17307 : }
17308 : else
17309 : {
17310 16 : op1 = copy_to_reg (op1);
17311 16 : op1 = lowpart_subreg (mode1, op1, GET_MODE (op1));
17312 : }
17313 :
17314 623 : if (!insn_data[icode].operand[2].predicate (op2, mode2))
17315 57 : op2 = copy_to_mode_reg (mode2, op2);
17316 :
17317 623 : if (!insn_data[icode].operand[3].predicate (op3, mode3))
17318 82 : op3 = copy_to_mode_reg (mode3, op3);
17319 :
17320 623 : if (!insn_data[icode].operand[4].predicate (op4, mode4))
17321 : {
17322 0 : error ("the last argument must be scale 1, 2, 4, 8");
17323 0 : return const0_rtx;
17324 : }
17325 :
17326 623 : pat = GEN_FCN (icode) (op0, op1, op2, op3, op4);
17327 623 : if (! pat)
17328 0 : return const0_rtx;
17329 :
17330 623 : emit_insn (pat);
17331 623 : return 0;
17332 :
17333 23 : case IX86_BUILTIN_XABORT:
17334 23 : icode = CODE_FOR_xabort;
17335 23 : arg0 = CALL_EXPR_ARG (exp, 0);
17336 23 : op0 = expand_normal (arg0);
17337 23 : mode0 = insn_data[icode].operand[0].mode;
17338 23 : if (!insn_data[icode].operand[0].predicate (op0, mode0))
17339 : {
17340 0 : error ("the argument to %<xabort%> intrinsic must "
17341 : "be an 8-bit immediate");
17342 0 : return const0_rtx;
17343 : }
17344 23 : emit_insn (gen_xabort (op0));
17345 23 : return 0;
17346 :
17347 55 : case IX86_BUILTIN_RDSSPD:
17348 55 : case IX86_BUILTIN_RDSSPQ:
17349 55 : mode = (fcode == IX86_BUILTIN_RDSSPD ? SImode : DImode);
17350 :
17351 55 : if (target == 0
17352 55 : || !register_operand (target, mode))
17353 0 : target = gen_reg_rtx (mode);
17354 :
17355 55 : op0 = force_reg (mode, const0_rtx);
17356 :
17357 55 : emit_insn (gen_rdssp (mode, target, op0));
17358 55 : return target;
17359 :
17360 55 : case IX86_BUILTIN_INCSSPD:
17361 55 : case IX86_BUILTIN_INCSSPQ:
17362 55 : mode = (fcode == IX86_BUILTIN_INCSSPD ? SImode : DImode);
17363 :
17364 55 : arg0 = CALL_EXPR_ARG (exp, 0);
17365 55 : op0 = expand_normal (arg0);
17366 :
17367 55 : op0 = force_reg (mode, op0);
17368 :
17369 55 : emit_insn (gen_incssp (mode, op0));
17370 55 : return 0;
17371 :
17372 20 : case IX86_BUILTIN_HRESET:
17373 20 : icode = CODE_FOR_hreset;
17374 20 : arg0 = CALL_EXPR_ARG (exp, 0);
17375 20 : op0 = expand_normal (arg0);
17376 20 : op0 = force_reg (SImode, op0);
17377 20 : emit_insn (gen_hreset (op0));
17378 20 : return 0;
17379 :
17380 39 : case IX86_BUILTIN_RSTORSSP:
17381 39 : case IX86_BUILTIN_CLRSSBSY:
17382 39 : arg0 = CALL_EXPR_ARG (exp, 0);
17383 39 : op0 = expand_normal (arg0);
17384 20 : icode = (fcode == IX86_BUILTIN_RSTORSSP
17385 39 : ? CODE_FOR_rstorssp
17386 : : CODE_FOR_clrssbsy);
17387 :
17388 39 : if (!address_operand (op0, VOIDmode))
17389 : {
17390 18 : op0 = convert_memory_address (Pmode, op0);
17391 18 : op0 = copy_addr_to_reg (op0);
17392 : }
17393 39 : emit_insn (GEN_FCN (icode) (gen_rtx_MEM (DImode, op0)));
17394 39 : return 0;
17395 :
17396 81 : case IX86_BUILTIN_WRSSD:
17397 81 : case IX86_BUILTIN_WRSSQ:
17398 81 : case IX86_BUILTIN_WRUSSD:
17399 81 : case IX86_BUILTIN_WRUSSQ:
17400 81 : mode = ((fcode == IX86_BUILTIN_WRSSD
17401 81 : || fcode == IX86_BUILTIN_WRUSSD)
17402 81 : ? SImode : DImode);
17403 :
17404 81 : arg0 = CALL_EXPR_ARG (exp, 0);
17405 81 : op0 = expand_normal (arg0);
17406 81 : arg1 = CALL_EXPR_ARG (exp, 1);
17407 81 : op1 = expand_normal (arg1);
17408 :
17409 81 : op0 = force_reg (mode, op0);
17410 :
17411 81 : if (!address_operand (op1, VOIDmode))
17412 : {
17413 36 : op1 = convert_memory_address (Pmode, op1);
17414 36 : op1 = copy_addr_to_reg (op1);
17415 : }
17416 81 : op1 = gen_rtx_MEM (mode, op1);
17417 :
17418 81 : icode = ((fcode == IX86_BUILTIN_WRSSD
17419 81 : || fcode == IX86_BUILTIN_WRSSQ)
17420 81 : ? code_for_wrss (mode)
17421 40 : : code_for_wruss (mode));
17422 81 : emit_insn (GEN_FCN (icode) (op0, op1));
17423 :
17424 81 : return 0;
17425 :
17426 120368 : default:
17427 120368 : break;
17428 : }
17429 :
17430 120368 : if (fcode >= IX86_BUILTIN__BDESC_SPECIAL_ARGS_FIRST
17431 120368 : && fcode <= IX86_BUILTIN__BDESC_SPECIAL_ARGS_LAST)
17432 : {
17433 27029 : i = fcode - IX86_BUILTIN__BDESC_SPECIAL_ARGS_FIRST;
17434 27029 : return ix86_expand_special_args_builtin (bdesc_special_args + i, exp,
17435 27029 : target);
17436 : }
17437 :
17438 93339 : if (fcode >= IX86_BUILTIN__BDESC_PURE_ARGS_FIRST
17439 93339 : && fcode <= IX86_BUILTIN__BDESC_PURE_ARGS_LAST)
17440 : {
17441 93 : i = fcode - IX86_BUILTIN__BDESC_PURE_ARGS_FIRST;
17442 93 : return ix86_expand_special_args_builtin (bdesc_pure_args + i, exp,
17443 93 : target);
17444 : }
17445 :
17446 93246 : if (fcode >= IX86_BUILTIN__BDESC_ARGS_FIRST
17447 93246 : && fcode <= IX86_BUILTIN__BDESC_ARGS_LAST)
17448 : {
17449 74576 : i = fcode - IX86_BUILTIN__BDESC_ARGS_FIRST;
17450 :
17451 74576 : switch (fcode)
17452 : {
17453 0 : case IX86_BUILTIN_RDPID:
17454 0 : return ix86_expand_special_args_builtin (bdesc_args + i, exp,
17455 0 : target);
17456 74 : case IX86_BUILTIN_VCOMISBF16EQ:
17457 74 : case IX86_BUILTIN_VCOMISBF16NE:
17458 74 : case IX86_BUILTIN_VCOMISBF16GT:
17459 74 : case IX86_BUILTIN_VCOMISBF16GE:
17460 74 : case IX86_BUILTIN_VCOMISBF16LT:
17461 74 : case IX86_BUILTIN_VCOMISBF16LE:
17462 74 : return ix86_expand_sse_comi (bdesc_args + i, exp, target, false);
17463 15 : case IX86_BUILTIN_FABSQ:
17464 15 : case IX86_BUILTIN_COPYSIGNQ:
17465 15 : if (!TARGET_SSE)
17466 : /* Emit a normal call if SSE isn't available. */
17467 0 : return expand_call (exp, target, ignore);
17468 : /* FALLTHRU */
17469 74502 : default:
17470 74502 : return ix86_expand_args_builtin (bdesc_args + i, exp, target);
17471 : }
17472 : }
17473 :
17474 18670 : if (fcode >= IX86_BUILTIN__BDESC_COMI_FIRST
17475 18670 : && fcode <= IX86_BUILTIN__BDESC_COMI_LAST)
17476 : {
17477 473 : i = fcode - IX86_BUILTIN__BDESC_COMI_FIRST;
17478 473 : return ix86_expand_sse_comi (bdesc_comi + i, exp, target, true);
17479 : }
17480 :
17481 18197 : if (fcode >= IX86_BUILTIN__BDESC_ROUND_ARGS_FIRST
17482 18197 : && fcode <= IX86_BUILTIN__BDESC_ROUND_ARGS_LAST)
17483 : {
17484 15660 : i = fcode - IX86_BUILTIN__BDESC_ROUND_ARGS_FIRST;
17485 15660 : return ix86_expand_round_builtin (bdesc_round_args + i, exp, target);
17486 : }
17487 :
17488 2537 : if (fcode >= IX86_BUILTIN__BDESC_PCMPESTR_FIRST
17489 2537 : && fcode <= IX86_BUILTIN__BDESC_PCMPESTR_LAST)
17490 : {
17491 216 : i = fcode - IX86_BUILTIN__BDESC_PCMPESTR_FIRST;
17492 216 : return ix86_expand_sse_pcmpestr (bdesc_pcmpestr + i, exp, target);
17493 : }
17494 :
17495 2321 : if (fcode >= IX86_BUILTIN__BDESC_PCMPISTR_FIRST
17496 2321 : && fcode <= IX86_BUILTIN__BDESC_PCMPISTR_LAST)
17497 : {
17498 275 : i = fcode - IX86_BUILTIN__BDESC_PCMPISTR_FIRST;
17499 275 : return ix86_expand_sse_pcmpistr (bdesc_pcmpistr + i, exp, target);
17500 : }
17501 :
17502 2046 : if (fcode >= IX86_BUILTIN__BDESC_MULTI_ARG_FIRST
17503 2046 : && fcode <= IX86_BUILTIN__BDESC_MULTI_ARG_LAST)
17504 : {
17505 1816 : i = fcode - IX86_BUILTIN__BDESC_MULTI_ARG_FIRST;
17506 1816 : const struct builtin_description *d = bdesc_multi_arg + i;
17507 1816 : return ix86_expand_multi_arg_builtin (d->icode, exp, target,
17508 : (enum ix86_builtin_func_type)
17509 1816 : d->flag, d->comparison);
17510 : }
17511 :
17512 230 : if (fcode >= IX86_BUILTIN__BDESC_CET_FIRST
17513 230 : && fcode <= IX86_BUILTIN__BDESC_CET_LAST)
17514 : {
17515 38 : i = fcode - IX86_BUILTIN__BDESC_CET_FIRST;
17516 38 : return ix86_expand_special_args_builtin (bdesc_cet + i, exp,
17517 38 : target);
17518 : }
17519 :
17520 192 : if (fcode >= IX86_BUILTIN__BDESC_ACE_FIRST
17521 192 : && fcode <= IX86_BUILTIN__BDESC_ACE_LAST)
17522 : {
17523 192 : i = fcode - IX86_BUILTIN__BDESC_ACE_FIRST;
17524 192 : return ix86_expand_ace_builtin (bdesc_ace + i, exp, target);
17525 : }
17526 :
17527 0 : gcc_unreachable ();
17528 : }
17529 :
17530 : /* See below where shifts are handled for explanation of this enum. */
17531 : enum ix86_vec_bcast_alg
17532 : {
17533 : VEC_BCAST_PXOR,
17534 : VEC_BCAST_PCMPEQ,
17535 : VEC_BCAST_PABSB,
17536 : VEC_BCAST_PADDB,
17537 : VEC_BCAST_PSRLW,
17538 : VEC_BCAST_PSRLD,
17539 : VEC_BCAST_PSLLW,
17540 : VEC_BCAST_PSLLD
17541 : };
17542 :
17543 : struct ix86_vec_bcast_map_simode_t
17544 : {
17545 : unsigned int key;
17546 : enum ix86_vec_bcast_alg alg;
17547 : unsigned int arg;
17548 : };
17549 :
17550 : /* This table must be kept sorted as values are looked-up using bsearch. */
17551 : static const ix86_vec_bcast_map_simode_t ix86_vec_bcast_map_simode[] = {
17552 : { 0x00000000, VEC_BCAST_PXOR, 0 },
17553 : { 0x00000001, VEC_BCAST_PSRLD, 31 },
17554 : { 0x00000003, VEC_BCAST_PSRLD, 30 },
17555 : { 0x00000007, VEC_BCAST_PSRLD, 29 },
17556 : { 0x0000000f, VEC_BCAST_PSRLD, 28 },
17557 : { 0x0000001f, VEC_BCAST_PSRLD, 27 },
17558 : { 0x0000003f, VEC_BCAST_PSRLD, 26 },
17559 : { 0x0000007f, VEC_BCAST_PSRLD, 25 },
17560 : { 0x000000ff, VEC_BCAST_PSRLD, 24 },
17561 : { 0x000001ff, VEC_BCAST_PSRLD, 23 },
17562 : { 0x000003ff, VEC_BCAST_PSRLD, 22 },
17563 : { 0x000007ff, VEC_BCAST_PSRLD, 21 },
17564 : { 0x00000fff, VEC_BCAST_PSRLD, 20 },
17565 : { 0x00001fff, VEC_BCAST_PSRLD, 19 },
17566 : { 0x00003fff, VEC_BCAST_PSRLD, 18 },
17567 : { 0x00007fff, VEC_BCAST_PSRLD, 17 },
17568 : { 0x0000ffff, VEC_BCAST_PSRLD, 16 },
17569 : { 0x00010001, VEC_BCAST_PSRLW, 15 },
17570 : { 0x0001ffff, VEC_BCAST_PSRLD, 15 },
17571 : { 0x00030003, VEC_BCAST_PSRLW, 14 },
17572 : { 0x0003ffff, VEC_BCAST_PSRLD, 14 },
17573 : { 0x00070007, VEC_BCAST_PSRLW, 13 },
17574 : { 0x0007ffff, VEC_BCAST_PSRLD, 13 },
17575 : { 0x000f000f, VEC_BCAST_PSRLW, 12 },
17576 : { 0x000fffff, VEC_BCAST_PSRLD, 12 },
17577 : { 0x001f001f, VEC_BCAST_PSRLW, 11 },
17578 : { 0x001fffff, VEC_BCAST_PSRLD, 11 },
17579 : { 0x003f003f, VEC_BCAST_PSRLW, 10 },
17580 : { 0x003fffff, VEC_BCAST_PSRLD, 10 },
17581 : { 0x007f007f, VEC_BCAST_PSRLW, 9 },
17582 : { 0x007fffff, VEC_BCAST_PSRLD, 9 },
17583 : { 0x00ff00ff, VEC_BCAST_PSRLW, 8 },
17584 : { 0x00ffffff, VEC_BCAST_PSRLD, 8 },
17585 : { 0x01010101, VEC_BCAST_PABSB, 0 },
17586 : { 0x01ff01ff, VEC_BCAST_PSRLW, 7 },
17587 : { 0x01ffffff, VEC_BCAST_PSRLD, 7 },
17588 : { 0x03ff03ff, VEC_BCAST_PSRLW, 6 },
17589 : { 0x03ffffff, VEC_BCAST_PSRLD, 6 },
17590 : { 0x07ff07ff, VEC_BCAST_PSRLW, 5 },
17591 : { 0x07ffffff, VEC_BCAST_PSRLD, 5 },
17592 : { 0x0fff0fff, VEC_BCAST_PSRLW, 4 },
17593 : { 0x0fffffff, VEC_BCAST_PSRLD, 4 },
17594 : { 0x1fff1fff, VEC_BCAST_PSRLW, 3 },
17595 : { 0x1fffffff, VEC_BCAST_PSRLD, 3 },
17596 : { 0x3fff3fff, VEC_BCAST_PSRLW, 2 },
17597 : { 0x3fffffff, VEC_BCAST_PSRLD, 2 },
17598 : { 0x7fff7fff, VEC_BCAST_PSRLW, 1 },
17599 : { 0x7fffffff, VEC_BCAST_PSRLD, 1 },
17600 : { 0x80000000, VEC_BCAST_PSLLD, 31 },
17601 : { 0x80008000, VEC_BCAST_PSLLW, 15 },
17602 : { 0xc0000000, VEC_BCAST_PSLLD, 30 },
17603 : { 0xc000c000, VEC_BCAST_PSLLW, 14 },
17604 : { 0xe0000000, VEC_BCAST_PSLLD, 29 },
17605 : { 0xe000e000, VEC_BCAST_PSLLW, 13 },
17606 : { 0xf0000000, VEC_BCAST_PSLLD, 28 },
17607 : { 0xf000f000, VEC_BCAST_PSLLW, 12 },
17608 : { 0xf8000000, VEC_BCAST_PSLLD, 27 },
17609 : { 0xf800f800, VEC_BCAST_PSLLW, 11 },
17610 : { 0xfc000000, VEC_BCAST_PSLLD, 26 },
17611 : { 0xfc00fc00, VEC_BCAST_PSLLW, 10 },
17612 : { 0xfe000000, VEC_BCAST_PSLLD, 25 },
17613 : { 0xfe00fe00, VEC_BCAST_PSLLW, 9 },
17614 : { 0xfefefefe, VEC_BCAST_PADDB, 0 },
17615 : { 0xff000000, VEC_BCAST_PSLLD, 24 },
17616 : { 0xff00ff00, VEC_BCAST_PSLLW, 8 },
17617 : { 0xff800000, VEC_BCAST_PSLLD, 23 },
17618 : { 0xff80ff80, VEC_BCAST_PSLLW, 7 },
17619 : { 0xffc00000, VEC_BCAST_PSLLD, 22 },
17620 : { 0xffc0ffc0, VEC_BCAST_PSLLW, 6 },
17621 : { 0xffe00000, VEC_BCAST_PSLLD, 21 },
17622 : { 0xffe0ffe0, VEC_BCAST_PSLLW, 5 },
17623 : { 0xfff00000, VEC_BCAST_PSLLD, 20 },
17624 : { 0xfff0fff0, VEC_BCAST_PSLLW, 4 },
17625 : { 0xfff80000, VEC_BCAST_PSLLD, 19 },
17626 : { 0xfff8fff8, VEC_BCAST_PSLLW, 3 },
17627 : { 0xfffc0000, VEC_BCAST_PSLLD, 18 },
17628 : { 0xfffcfffc, VEC_BCAST_PSLLW, 2 },
17629 : { 0xfffe0000, VEC_BCAST_PSLLD, 17 },
17630 : { 0xfffefffe, VEC_BCAST_PSLLW, 1 },
17631 : { 0xffff0000, VEC_BCAST_PSLLD, 16 },
17632 : { 0xffff8000, VEC_BCAST_PSLLD, 15 },
17633 : { 0xffffc000, VEC_BCAST_PSLLD, 14 },
17634 : { 0xffffe000, VEC_BCAST_PSLLD, 13 },
17635 : { 0xfffff000, VEC_BCAST_PSLLD, 12 },
17636 : { 0xfffff800, VEC_BCAST_PSLLD, 11 },
17637 : { 0xfffffc00, VEC_BCAST_PSLLD, 10 },
17638 : { 0xfffffe00, VEC_BCAST_PSLLD, 9 },
17639 : { 0xffffff00, VEC_BCAST_PSLLD, 8 },
17640 : { 0xffffff80, VEC_BCAST_PSLLD, 7 },
17641 : { 0xffffffc0, VEC_BCAST_PSLLD, 6 },
17642 : { 0xffffffe0, VEC_BCAST_PSLLD, 5 },
17643 : { 0xfffffff0, VEC_BCAST_PSLLD, 4 },
17644 : { 0xfffffff8, VEC_BCAST_PSLLD, 3 },
17645 : { 0xfffffffc, VEC_BCAST_PSLLD, 2 },
17646 : { 0xfffffffe, VEC_BCAST_PSLLD, 1 },
17647 : { 0xffffffff, VEC_BCAST_PCMPEQ, 0 }
17648 : };
17649 :
17650 : /* Comparator for bsearch on ix86_vec_bcast_map. */
17651 : static int
17652 355692 : ix86_vec_bcast_map_simode_cmp (const void *key, const void *entry)
17653 : {
17654 355692 : return (*(const unsigned int*)key)
17655 355692 : - ((const ix86_vec_bcast_map_simode_t*)entry)->key;
17656 : }
17657 :
17658 : /* A subroutine of ix86_vector_duplicate_value. Tries to efficiently
17659 : materialize V4SImode, V8SImode and V16SImode vectors from SImode
17660 : integer constants. */
17661 : static bool
17662 53907 : ix86_vector_duplicate_simode_const (machine_mode mode, rtx target,
17663 : unsigned int val)
17664 : {
17665 53907 : const ix86_vec_bcast_map_simode_t *entry;
17666 53907 : rtx tmp1, tmp2;
17667 :
17668 53907 : entry = (const ix86_vec_bcast_map_simode_t*)
17669 53907 : bsearch(&val, ix86_vec_bcast_map_simode,
17670 : ARRAY_SIZE (ix86_vec_bcast_map_simode),
17671 : sizeof (ix86_vec_bcast_map_simode_t),
17672 : ix86_vec_bcast_map_simode_cmp);
17673 53907 : if (!entry)
17674 : return false;
17675 :
17676 19800 : switch (entry->alg)
17677 : {
17678 2576 : case VEC_BCAST_PXOR:
17679 2576 : if ((mode == V8SImode && !TARGET_AVX2)
17680 2576 : || (mode == V16SImode && !TARGET_AVX512F))
17681 : return false;
17682 2576 : emit_move_insn (target, CONST0_RTX (mode));
17683 2576 : return true;
17684 :
17685 223 : case VEC_BCAST_PCMPEQ:
17686 223 : if ((mode == V4SImode && !TARGET_SSE2)
17687 222 : || (mode == V8SImode && !TARGET_AVX2)
17688 195 : || (mode == V16SImode && !TARGET_AVX512F))
17689 : return false;
17690 195 : emit_move_insn (target, CONSTM1_RTX (mode));
17691 195 : return true;
17692 :
17693 653 : case VEC_BCAST_PABSB:
17694 653 : if (mode == V4SImode && TARGET_SSE2)
17695 : {
17696 526 : tmp1 = gen_reg_rtx (V16QImode);
17697 526 : emit_move_insn (tmp1, CONSTM1_RTX (V16QImode));
17698 526 : tmp2 = gen_reg_rtx (V16QImode);
17699 526 : emit_insn (gen_absv16qi2 (tmp2, tmp1));
17700 : }
17701 127 : else if (mode == V8SImode && TARGET_AVX2)
17702 : {
17703 69 : tmp1 = gen_reg_rtx (V32QImode);
17704 69 : emit_move_insn (tmp1, CONSTM1_RTX (V32QImode));
17705 69 : tmp2 = gen_reg_rtx (V32QImode);
17706 69 : emit_insn (gen_absv32qi2 (tmp2, tmp1));
17707 : }
17708 58 : else if (mode == V16SImode && TARGET_AVX512BW)
17709 : {
17710 50 : tmp1 = gen_reg_rtx (V64QImode);
17711 50 : emit_move_insn (tmp1, CONSTM1_RTX (V64QImode));
17712 50 : tmp2 = gen_reg_rtx (V64QImode);
17713 50 : emit_insn (gen_absv64qi2 (tmp2, tmp1));
17714 : }
17715 : else
17716 : return false;
17717 : break;
17718 :
17719 98 : case VEC_BCAST_PADDB:
17720 98 : if (mode == V4SImode && TARGET_SSE2)
17721 : {
17722 94 : tmp1 = gen_reg_rtx (V16QImode);
17723 94 : emit_move_insn (tmp1, CONSTM1_RTX (V16QImode));
17724 94 : tmp2 = gen_reg_rtx (V16QImode);
17725 94 : emit_insn (gen_addv16qi3 (tmp2, tmp1, tmp1));
17726 : }
17727 4 : else if (mode == V8SImode && TARGET_AVX2)
17728 : {
17729 1 : tmp1 = gen_reg_rtx (V32QImode);
17730 1 : emit_move_insn (tmp1, CONSTM1_RTX (V32QImode));
17731 1 : tmp2 = gen_reg_rtx (V32QImode);
17732 1 : emit_insn (gen_addv32qi3 (tmp2, tmp1, tmp1));
17733 : }
17734 3 : else if (mode == V16SImode && TARGET_AVX512BW)
17735 : {
17736 3 : tmp1 = gen_reg_rtx (V64QImode);
17737 3 : emit_move_insn (tmp1, CONSTM1_RTX (V64QImode));
17738 3 : tmp2 = gen_reg_rtx (V64QImode);
17739 3 : emit_insn (gen_addv64qi3 (tmp2, tmp1, tmp1));
17740 : }
17741 : else
17742 : return false;
17743 : break;
17744 :
17745 3772 : case VEC_BCAST_PSRLW:
17746 3772 : if (mode == V4SImode && TARGET_SSE2)
17747 : {
17748 3549 : tmp1 = gen_reg_rtx (V8HImode);
17749 3549 : emit_move_insn (tmp1, CONSTM1_RTX (V8HImode));
17750 3549 : tmp2 = gen_reg_rtx (V8HImode);
17751 3549 : emit_insn (gen_lshrv8hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
17752 : }
17753 223 : else if (mode == V8SImode && TARGET_AVX2)
17754 : {
17755 129 : tmp1 = gen_reg_rtx (V16HImode);
17756 129 : emit_move_insn (tmp1, CONSTM1_RTX (V16HImode));
17757 129 : tmp2 = gen_reg_rtx (V16HImode);
17758 129 : emit_insn (gen_lshrv16hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
17759 : }
17760 94 : else if (mode == V16SImode && TARGET_AVX512BW)
17761 : {
17762 91 : tmp1 = gen_reg_rtx (V32HImode);
17763 91 : emit_move_insn (tmp1, CONSTM1_RTX (V32HImode));
17764 91 : tmp2 = gen_reg_rtx (V32HImode);
17765 91 : emit_insn (gen_lshrv32hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
17766 : }
17767 : else
17768 : return false;
17769 : break;
17770 :
17771 10507 : case VEC_BCAST_PSRLD:
17772 10507 : if (mode == V4SImode && TARGET_SSE2)
17773 : {
17774 7628 : tmp1 = gen_reg_rtx (V4SImode);
17775 7628 : emit_move_insn (tmp1, CONSTM1_RTX (V4SImode));
17776 7628 : emit_insn (gen_lshrv4si3 (target, tmp1, GEN_INT (entry->arg)));
17777 7628 : return true;
17778 : }
17779 2879 : else if (mode == V8SImode && TARGET_AVX2)
17780 : {
17781 1064 : tmp1 = gen_reg_rtx (V8SImode);
17782 1064 : emit_move_insn (tmp1, CONSTM1_RTX (V8SImode));
17783 1064 : emit_insn (gen_lshrv8si3 (target, tmp1, GEN_INT (entry->arg)));
17784 1064 : return true;
17785 : }
17786 1815 : else if (mode == V16SImode && TARGET_AVX512F)
17787 : {
17788 963 : tmp1 = gen_reg_rtx (V16SImode);
17789 963 : emit_move_insn (tmp1, CONSTM1_RTX (V16SImode));
17790 963 : emit_insn (gen_lshrv16si3 (target, tmp1, GEN_INT (entry->arg)));
17791 963 : return true;
17792 : }
17793 : else
17794 : return false;
17795 137 : break;
17796 :
17797 137 : case VEC_BCAST_PSLLW:
17798 137 : if (mode == V4SImode && TARGET_SSE2)
17799 : {
17800 107 : tmp1 = gen_reg_rtx (V8HImode);
17801 107 : emit_move_insn (tmp1, CONSTM1_RTX (V8HImode));
17802 107 : tmp2 = gen_reg_rtx (V8HImode);
17803 107 : emit_insn (gen_ashlv8hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
17804 : }
17805 30 : else if (mode == V8SImode && TARGET_AVX2)
17806 : {
17807 21 : tmp1 = gen_reg_rtx (V16HImode);
17808 21 : emit_move_insn (tmp1, CONSTM1_RTX (V16HImode));
17809 21 : tmp2 = gen_reg_rtx (V16HImode);
17810 21 : emit_insn (gen_ashlv16hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
17811 : }
17812 9 : else if (mode == V16SImode && TARGET_AVX512BW)
17813 : {
17814 9 : tmp1 = gen_reg_rtx (V32HImode);
17815 9 : emit_move_insn (tmp1, CONSTM1_RTX (V32HImode));
17816 9 : tmp2 = gen_reg_rtx (V32HImode);
17817 9 : emit_insn (gen_ashlv32hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
17818 : }
17819 : else
17820 : return false;
17821 : break;
17822 :
17823 1834 : case VEC_BCAST_PSLLD:
17824 1834 : if (mode == V4SImode && TARGET_SSE2)
17825 : {
17826 1786 : tmp1 = gen_reg_rtx (V4SImode);
17827 1786 : emit_move_insn (tmp1, CONSTM1_RTX (V4SImode));
17828 1786 : emit_insn (gen_ashlv4si3 (target, tmp1, GEN_INT (entry->arg)));
17829 1786 : return true;
17830 : }
17831 48 : else if (mode == V8SImode && TARGET_AVX2)
17832 : {
17833 30 : tmp1 = gen_reg_rtx (V8SImode);
17834 30 : emit_move_insn (tmp1, CONSTM1_RTX (V8SImode));
17835 30 : emit_insn (gen_ashlv8si3 (target, tmp1, GEN_INT (entry->arg)));
17836 30 : return true;
17837 : }
17838 18 : else if (mode == V16SImode && TARGET_AVX512F)
17839 : {
17840 18 : tmp1 = gen_reg_rtx (V16SImode);
17841 18 : emit_move_insn (tmp1, CONSTM1_RTX (V16SImode));
17842 18 : emit_insn (gen_ashlv16si3 (target, tmp1, GEN_INT (entry->arg)));
17843 18 : return true;
17844 : }
17845 : else
17846 : return false;
17847 :
17848 : default:
17849 : return false;
17850 : }
17851 :
17852 4649 : emit_move_insn (target, gen_lowpart (mode, tmp2));
17853 4649 : return true;
17854 : }
17855 :
17856 : /* A subroutine of ix86_expand_vector_init_duplicate. Tries to
17857 : fill target with val via vec_duplicate. */
17858 :
17859 : static bool
17860 146388 : ix86_vector_duplicate_value (machine_mode mode, rtx target, rtx val)
17861 : {
17862 146388 : bool ok;
17863 146388 : rtx_insn *insn;
17864 146388 : rtx dup;
17865 :
17866 146388 : if ((mode == V4SImode || mode == V8SImode || mode == V16SImode)
17867 61897 : && CONST_INT_P (val)
17868 53907 : && ix86_vector_duplicate_simode_const (mode, target, INTVAL (val)))
17869 : return true;
17870 :
17871 : /* Save/restore recog_data in case this is called from splitters
17872 : or other routines where recog_data needs to stay valid across
17873 : force_reg. See PR106577. */
17874 127479 : recog_state_saver recog_save;
17875 :
17876 : /* First attempt to recognize VAL as-is. */
17877 127479 : dup = gen_vec_duplicate (mode, val);
17878 127479 : insn = emit_insn (gen_rtx_SET (target, dup));
17879 127479 : if (recog_memoized (insn) < 0)
17880 : {
17881 98707 : rtx_insn *seq;
17882 98707 : machine_mode innermode = GET_MODE_INNER (mode);
17883 98707 : rtx reg;
17884 :
17885 : /* If that fails, force VAL into a register or mem. */
17886 :
17887 98707 : start_sequence ();
17888 :
17889 0 : if (!TARGET_PREFER_BCST_FROM_INTEGER && CONST_INT_P (val)
17890 0 : && GET_MODE_BITSIZE (innermode) <= HOST_BITS_PER_WIDE_INT
17891 98707 : && GET_MODE_BITSIZE(mode) >= 128)
17892 0 : reg = validize_mem (force_const_mem (innermode, val));
17893 : else
17894 : {
17895 98707 : reg = force_reg (innermode, val);
17896 98707 : if (GET_MODE (reg) != innermode)
17897 0 : reg = gen_lowpart (innermode, reg);
17898 : }
17899 :
17900 98707 : SET_SRC (PATTERN (insn)) = gen_vec_duplicate (mode, reg);
17901 98707 : seq = end_sequence ();
17902 98707 : if (seq)
17903 98707 : emit_insn_before (seq, insn);
17904 :
17905 98707 : ok = recog_memoized (insn) >= 0;
17906 98707 : gcc_assert (ok);
17907 : }
17908 127479 : return true;
17909 127479 : }
17910 :
17911 : /* Get a vector mode of the same size as the original but with elements
17912 : twice as wide. This is only guaranteed to apply to integral vectors. */
17913 :
17914 : static machine_mode
17915 27594 : get_mode_wider_vector (machine_mode o)
17916 : {
17917 : /* ??? Rely on the ordering that genmodes.cc gives to vectors. */
17918 27594 : machine_mode n = GET_MODE_NEXT_MODE (o).require ();
17919 82782 : gcc_assert (GET_MODE_NUNITS (o) == GET_MODE_NUNITS (n) * 2);
17920 82782 : gcc_assert (GET_MODE_SIZE (o) == GET_MODE_SIZE (n));
17921 27594 : return n;
17922 : }
17923 :
17924 : static bool expand_vec_perm_broadcast_1 (struct expand_vec_perm_d *d);
17925 : static bool expand_vec_perm_1 (struct expand_vec_perm_d *d);
17926 :
17927 : /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
17928 : with all elements equal to VAR. Return true if successful. */
17929 :
17930 : bool
17931 174936 : ix86_expand_vector_init_duplicate (bool mmx_ok, machine_mode mode,
17932 : rtx target, rtx val)
17933 : {
17934 174936 : bool ok;
17935 :
17936 174936 : switch (mode)
17937 : {
17938 71993 : case E_V2DImode:
17939 71993 : if (CONST_INT_P (val))
17940 : {
17941 62778 : int tmp = (int)INTVAL (val);
17942 62778 : if (tmp == (int)(INTVAL (val) >> 32))
17943 : {
17944 78 : rtx reg = gen_reg_rtx (V4SImode);
17945 78 : ok = ix86_vector_duplicate_value (V4SImode, reg,
17946 : GEN_INT (tmp));
17947 78 : if (ok)
17948 : {
17949 78 : emit_move_insn (target, gen_lowpart (V2DImode, reg));
17950 78 : return true;
17951 : }
17952 : }
17953 : }
17954 71915 : return ix86_vector_duplicate_value (mode, target, val);
17955 :
17956 1088 : case E_V4DImode:
17957 1088 : if (CONST_INT_P (val))
17958 : {
17959 778 : int tmp = (int)INTVAL (val);
17960 778 : if (tmp == (int)(INTVAL (val) >> 32))
17961 : {
17962 54 : rtx reg = gen_reg_rtx (V8SImode);
17963 54 : ok = ix86_vector_duplicate_value (V8SImode, reg,
17964 : GEN_INT (tmp));
17965 54 : if (ok)
17966 : {
17967 54 : emit_move_insn (target, gen_lowpart (V4DImode, reg));
17968 54 : return true;
17969 : }
17970 : }
17971 : }
17972 1034 : return ix86_vector_duplicate_value (mode, target, val);
17973 :
17974 465 : case E_V8DImode:
17975 465 : if (CONST_INT_P (val))
17976 : {
17977 264 : int tmp = (int)INTVAL (val);
17978 264 : if (tmp == (int)(INTVAL (val) >> 32))
17979 : {
17980 24 : rtx reg = gen_reg_rtx (V16SImode);
17981 24 : ok = ix86_vector_duplicate_value (V16SImode, reg,
17982 : GEN_INT (tmp));
17983 24 : if (ok)
17984 : {
17985 24 : emit_move_insn (target, gen_lowpart (V8DImode, reg));
17986 24 : return true;
17987 : }
17988 : }
17989 : }
17990 441 : return ix86_vector_duplicate_value (mode, target, val);
17991 :
17992 2535 : case E_V2SImode:
17993 2535 : case E_V2SFmode:
17994 2535 : if (!mmx_ok)
17995 : return false;
17996 : /* FALLTHRU */
17997 :
17998 71847 : case E_V4DFmode:
17999 71847 : case E_V8SFmode:
18000 71847 : case E_V8SImode:
18001 71847 : case E_V2DFmode:
18002 71847 : case E_V4SFmode:
18003 71847 : case E_V4SImode:
18004 71847 : case E_V16SImode:
18005 71847 : case E_V16SFmode:
18006 71847 : case E_V8DFmode:
18007 71847 : return ix86_vector_duplicate_value (mode, target, val);
18008 :
18009 388 : case E_V4HImode:
18010 388 : if (!mmx_ok)
18011 : return false;
18012 385 : if (TARGET_SSE || TARGET_3DNOW_A)
18013 : {
18014 385 : rtx x;
18015 :
18016 385 : val = gen_lowpart (SImode, val);
18017 385 : if (CONST_INT_P (val))
18018 : return false;
18019 383 : x = gen_rtx_TRUNCATE (HImode, val);
18020 383 : x = gen_rtx_VEC_DUPLICATE (mode, x);
18021 383 : emit_insn (gen_rtx_SET (target, x));
18022 383 : return true;
18023 : }
18024 0 : goto widen;
18025 :
18026 5 : case E_V4HFmode:
18027 5 : case E_V4BFmode:
18028 5 : if (TARGET_MMX_WITH_SSE)
18029 : {
18030 10 : val = force_reg (GET_MODE_INNER (mode), val);
18031 5 : rtx x = gen_rtx_VEC_DUPLICATE (mode, val);
18032 5 : emit_insn (gen_rtx_SET (target, x));
18033 5 : return true;
18034 : }
18035 : return false;
18036 :
18037 127 : case E_V2HImode:
18038 127 : if (TARGET_SSE2)
18039 : {
18040 127 : rtx x;
18041 :
18042 127 : val = gen_lowpart (SImode, val);
18043 127 : if (CONST_INT_P (val))
18044 : return false;
18045 127 : x = gen_rtx_TRUNCATE (HImode, val);
18046 127 : x = gen_rtx_VEC_DUPLICATE (mode, x);
18047 127 : emit_insn (gen_rtx_SET (target, x));
18048 127 : return true;
18049 : }
18050 : return false;
18051 :
18052 5 : case E_V2HFmode:
18053 5 : case E_V2BFmode:
18054 5 : if (TARGET_SSE2)
18055 : {
18056 10 : val = force_reg (GET_MODE_INNER (mode), val);
18057 5 : rtx x = gen_rtx_VEC_DUPLICATE (mode, val);
18058 5 : emit_insn (gen_rtx_SET (target, x));
18059 5 : return true;
18060 : }
18061 : return false;
18062 :
18063 291 : case E_V8QImode:
18064 291 : case E_V4QImode:
18065 291 : if (!mmx_ok)
18066 : return false;
18067 290 : goto widen;
18068 :
18069 14403 : case E_V8HImode:
18070 14403 : if (CONST_INT_P (val))
18071 13855 : goto widen;
18072 : /* FALLTHRU */
18073 :
18074 862 : case E_V8HFmode:
18075 862 : case E_V8BFmode:
18076 862 : if (TARGET_AVX2)
18077 396 : return ix86_vector_duplicate_value (mode, target, val);
18078 :
18079 466 : if (TARGET_SSE2)
18080 : {
18081 1201 : struct expand_vec_perm_d dperm;
18082 1201 : rtx tmp1, tmp2;
18083 :
18084 466 : permute:
18085 1201 : memset (&dperm, 0, sizeof (dperm));
18086 1201 : dperm.target = target;
18087 1201 : dperm.vmode = mode;
18088 1201 : dperm.nelt = GET_MODE_NUNITS (mode);
18089 1201 : dperm.op0 = dperm.op1 = gen_reg_rtx (mode);
18090 1201 : dperm.one_operand_p = true;
18091 :
18092 1201 : if (mode == V8HFmode || mode == V8BFmode)
18093 : {
18094 6 : tmp1 = force_reg (GET_MODE_INNER (mode), val);
18095 3 : tmp2 = gen_reg_rtx (mode);
18096 3 : emit_insn (gen_vec_set_0 (mode, tmp2, CONST0_RTX (mode), tmp1));
18097 3 : tmp1 = gen_lowpart (mode, tmp2);
18098 : }
18099 : else
18100 : {
18101 : /* Extend to SImode using a paradoxical SUBREG. */
18102 1198 : tmp1 = gen_reg_rtx (SImode);
18103 1198 : emit_move_insn (tmp1, gen_lowpart (SImode, val));
18104 :
18105 : /* Insert the SImode value as
18106 : low element of a V4SImode vector. */
18107 1198 : tmp2 = gen_reg_rtx (V4SImode);
18108 1198 : emit_insn (gen_vec_setv4si_0 (tmp2, CONST0_RTX (V4SImode), tmp1));
18109 1198 : tmp1 = gen_lowpart (mode, tmp2);
18110 : }
18111 :
18112 1201 : emit_move_insn (dperm.op0, tmp1);
18113 1201 : ok = (expand_vec_perm_1 (&dperm)
18114 1201 : || expand_vec_perm_broadcast_1 (&dperm));
18115 0 : gcc_assert (ok);
18116 1201 : return ok;
18117 : }
18118 0 : goto widen;
18119 :
18120 9921 : case E_V16QImode:
18121 9921 : if (CONST_INT_P (val))
18122 9125 : goto widen;
18123 796 : if (TARGET_AVX2)
18124 61 : return ix86_vector_duplicate_value (mode, target, val);
18125 :
18126 735 : if (TARGET_SSE2)
18127 735 : goto permute;
18128 0 : goto widen;
18129 :
18130 25942 : widen:
18131 : /* Replicate the value once into the next wider mode and recurse. */
18132 25942 : {
18133 25942 : machine_mode smode, wsmode, wvmode;
18134 25942 : rtx x;
18135 :
18136 25942 : smode = GET_MODE_INNER (mode);
18137 25942 : wvmode = get_mode_wider_vector (mode);
18138 25942 : wsmode = GET_MODE_INNER (wvmode);
18139 :
18140 25942 : val = convert_modes (wsmode, smode, val, true);
18141 :
18142 25942 : if (CONST_INT_P (val))
18143 : {
18144 51306 : x = simplify_binary_operation (ASHIFT, wsmode, val,
18145 25653 : GEN_INT (GET_MODE_BITSIZE (smode)));
18146 25653 : val = simplify_binary_operation (IOR, wsmode, val, x);
18147 : }
18148 289 : else if (smode == QImode && !TARGET_PARTIAL_REG_STALL)
18149 289 : emit_insn (gen_insv_1 (wsmode, val, val));
18150 : else
18151 : {
18152 0 : x = expand_simple_binop (wsmode, ASHIFT, val,
18153 0 : GEN_INT (GET_MODE_BITSIZE (smode)),
18154 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
18155 0 : val = expand_simple_binop (wsmode, IOR, val, x, x, 1,
18156 : OPTAB_LIB_WIDEN);
18157 : }
18158 :
18159 25942 : x = gen_reg_rtx (wvmode);
18160 25942 : ok = ix86_expand_vector_init_duplicate (mmx_ok, wvmode, x, val);
18161 25942 : if (!ok)
18162 : return false;
18163 25941 : emit_move_insn (target, gen_lowpart (GET_MODE (target), x));
18164 25941 : return true;
18165 : }
18166 :
18167 1729 : case E_V16HImode:
18168 1729 : case E_V32QImode:
18169 1729 : if (CONST_INT_P (val))
18170 1438 : goto widen;
18171 : /* FALLTHRU */
18172 :
18173 374 : case E_V16HFmode:
18174 374 : case E_V16BFmode:
18175 374 : if (TARGET_AVX2)
18176 348 : return ix86_vector_duplicate_value (mode, target, val);
18177 : else
18178 : {
18179 26 : machine_mode hvmode;
18180 26 : switch (mode)
18181 : {
18182 : case V16HImode:
18183 : hvmode = V8HImode;
18184 : break;
18185 0 : case V16HFmode:
18186 0 : hvmode = V8HFmode;
18187 0 : break;
18188 1 : case V16BFmode:
18189 1 : hvmode = V8BFmode;
18190 1 : break;
18191 12 : case V32QImode:
18192 12 : hvmode = V16QImode;
18193 12 : break;
18194 : default:
18195 : gcc_unreachable ();
18196 : }
18197 26 : rtx x = gen_reg_rtx (hvmode);
18198 :
18199 26 : ok = ix86_expand_vector_init_duplicate (false, hvmode, x, val);
18200 26 : if (!ok)
18201 : return false;
18202 :
18203 26 : x = gen_rtx_VEC_CONCAT (mode, x, x);
18204 26 : emit_insn (gen_rtx_SET (target, x));
18205 : }
18206 26 : return true;
18207 :
18208 1363 : case E_V32HImode:
18209 1363 : case E_V64QImode:
18210 1363 : if (CONST_INT_P (val))
18211 1234 : goto widen;
18212 : /* FALLTHRU */
18213 :
18214 210 : case E_V32HFmode:
18215 210 : case E_V32BFmode:
18216 210 : if (TARGET_AVX512BW)
18217 190 : return ix86_vector_duplicate_value (mode, target, val);
18218 : else
18219 : {
18220 20 : machine_mode hvmode;
18221 20 : switch (mode)
18222 : {
18223 : case V32HImode:
18224 : hvmode = V16HImode;
18225 : break;
18226 0 : case V32HFmode:
18227 0 : hvmode = V16HFmode;
18228 0 : break;
18229 1 : case V32BFmode:
18230 1 : hvmode = V16BFmode;
18231 1 : break;
18232 10 : case V64QImode:
18233 10 : hvmode = V32QImode;
18234 10 : break;
18235 : default:
18236 : gcc_unreachable ();
18237 : }
18238 20 : rtx x = gen_reg_rtx (hvmode);
18239 :
18240 20 : ok = ix86_expand_vector_init_duplicate (false, hvmode, x, val);
18241 20 : if (!ok)
18242 : return false;
18243 :
18244 20 : x = gen_rtx_VEC_CONCAT (mode, x, x);
18245 20 : emit_insn (gen_rtx_SET (target, x));
18246 : }
18247 20 : return true;
18248 :
18249 : default:
18250 : return false;
18251 : }
18252 : }
18253 :
18254 : /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
18255 : whose ONE_VAR element is VAR, and other elements are zero. Return true
18256 : if successful. */
18257 :
18258 : bool
18259 13349 : ix86_expand_vector_init_one_nonzero (bool mmx_ok, machine_mode mode,
18260 : rtx target, rtx var, int one_var)
18261 : {
18262 13349 : rtx x, tmp;
18263 13349 : bool use_vector_set = false;
18264 13349 : rtx (*gen_vec_set_0) (rtx, rtx, rtx) = NULL;
18265 :
18266 13349 : switch (mode)
18267 : {
18268 8509 : case E_V2DImode:
18269 8509 : if (TARGET_64BIT || MEM_P (var))
18270 : {
18271 8500 : if (!REG_P (var) && !MEM_P (var))
18272 5612 : var = force_reg (DImode, var);
18273 8500 : x = gen_rtx_VEC_CONCAT (V2DImode, var, CONST0_RTX (DImode));
18274 8500 : if (!one_var)
18275 2350 : emit_insn (gen_rtx_SET (target, x));
18276 6150 : else if (TARGET_SSE2)
18277 : {
18278 6150 : tmp = gen_reg_rtx (V2DImode);
18279 6150 : emit_insn (gen_rtx_SET (tmp, x));
18280 6150 : emit_insn (gen_vec_shl_v2di (target, tmp, GEN_INT (64)));
18281 : }
18282 : else
18283 : {
18284 0 : rtx tmp1 = gen_reg_rtx (V2DImode);
18285 0 : emit_insn (gen_rtx_SET (tmp1, x));
18286 0 : rtx tmp2 = gen_reg_rtx (V4SImode);
18287 0 : emit_move_insn (tmp2, gen_lowpart (V4SImode, tmp1));
18288 0 : emit_insn (gen_sse_shufps_v4si (tmp2, tmp2, tmp2,
18289 : GEN_INT (2), GEN_INT (3),
18290 : GEN_INT (4), GEN_INT (5)));
18291 0 : emit_move_insn (target, gen_lowpart (V2DImode, tmp2));
18292 : }
18293 : }
18294 : else
18295 : {
18296 9 : rtx lo = force_reg (SImode, gen_lowpart (SImode, var));
18297 9 : rtx hi = force_reg (SImode, gen_highpart (SImode, var));
18298 9 : tmp = gen_reg_rtx (V4SImode);
18299 9 : if (TARGET_SSE4_1)
18300 : {
18301 0 : rtx tmp1 = gen_reg_rtx (V4SImode);
18302 0 : emit_insn (gen_vec_setv4si_0 (tmp1, CONST0_RTX (V4SImode), lo));
18303 0 : emit_insn (gen_sse4_1_pinsrd (tmp, tmp1, hi, GEN_INT (2)));
18304 : }
18305 : else
18306 : {
18307 9 : rtx ltmp = gen_reg_rtx (V4SImode);
18308 9 : rtx htmp = gen_reg_rtx (V4SImode);
18309 9 : emit_insn (gen_vec_setv4si_0 (ltmp, CONST0_RTX (V4SImode), lo));
18310 9 : emit_insn (gen_vec_setv4si_0 (htmp, CONST0_RTX (V4SImode), hi));
18311 9 : emit_insn (gen_vec_interleave_lowv4si (tmp, ltmp, htmp));
18312 : }
18313 9 : if (!one_var)
18314 3 : emit_move_insn (target, gen_lowpart (V2DImode, tmp));
18315 6 : else if (TARGET_SSE2)
18316 : {
18317 6 : rtx tmp2 = gen_reg_rtx (V2DImode);
18318 6 : emit_move_insn (tmp2, gen_lowpart (V2DImode, tmp));
18319 6 : emit_insn (gen_vec_shl_v2di (target, tmp2, GEN_INT (64)));
18320 : }
18321 : else
18322 : {
18323 0 : rtx tmp2 = gen_reg_rtx (V2DImode);
18324 0 : emit_insn (gen_sse_shufps_v4si (tmp2, tmp, tmp,
18325 : GEN_INT (2), GEN_INT (3),
18326 : GEN_INT (4), GEN_INT (5)));
18327 0 : emit_move_insn (target, gen_lowpart (V2DImode, tmp2));
18328 : }
18329 : }
18330 : return true;
18331 106 : case E_V2DFmode:
18332 106 : if (!REG_P (var) && !MEM_P (var))
18333 0 : var = force_reg (DFmode, var);
18334 106 : x = gen_rtx_VEC_CONCAT (V2DFmode, var, CONST0_RTX (DFmode));
18335 106 : if (one_var)
18336 : {
18337 27 : tmp = gen_reg_rtx (V2DFmode);
18338 27 : emit_insn (gen_rtx_SET (tmp, x));
18339 27 : emit_insn (gen_vec_shl_v2df (target, tmp, GEN_INT (64)));
18340 : }
18341 : else
18342 79 : emit_insn (gen_rtx_SET (target, x));
18343 : return true;
18344 358 : case E_V4SImode:
18345 358 : var = force_reg (SImode, var);
18346 358 : x = gen_rtx_VEC_DUPLICATE (V4SImode, var);
18347 358 : x = gen_rtx_VEC_MERGE (V4SImode, x, CONST0_RTX (V4SImode), const1_rtx);
18348 358 : if (!one_var)
18349 308 : emit_insn (gen_rtx_SET (target, x));
18350 50 : else if (TARGET_SSE2)
18351 : {
18352 47 : rtx tmp = gen_reg_rtx (V4SImode);
18353 47 : emit_insn (gen_rtx_SET (tmp, x));
18354 47 : emit_insn (gen_vec_shl_v4si (target, tmp, GEN_INT (one_var * 32)));
18355 : }
18356 : else
18357 : {
18358 3 : rtx tmp = gen_reg_rtx (V4SImode);
18359 3 : emit_insn (gen_rtx_SET (tmp, x));
18360 10 : emit_insn (gen_sse_shufps_v4si (target, tmp, tmp,
18361 : const1_rtx,
18362 3 : GEN_INT (one_var == 1 ? 0 : 1),
18363 : GEN_INT (one_var == 2 ? 0+4 : 1+4),
18364 : GEN_INT (one_var == 3 ? 0+4 : 1+4)));
18365 : }
18366 : return true;
18367 3174 : case E_V4SFmode:
18368 3174 : if (TARGET_SSE4_1)
18369 : {
18370 522 : if (!REG_P (var) && !MEM_P (var))
18371 0 : var = force_reg (SFmode, var);
18372 522 : if (one_var)
18373 12 : emit_insn (gen_sse4_1_insertps_v4sf_init (target, var,
18374 : CONST0_RTX (V4SFmode),
18375 12 : GEN_INT (1 << one_var)));
18376 : else
18377 510 : emit_insn (gen_vec_setv4sf_0 (target, CONST0_RTX (V4SFmode), var));
18378 : return true;
18379 : }
18380 2652 : if (TARGET_SSE2 && REG_P (var))
18381 : {
18382 56 : if (one_var == 3)
18383 1 : emit_insn (gen_sse2_insertps_v4sf_3 (target, var,
18384 : CONST0_RTX (V4SFmode)));
18385 : else
18386 : {
18387 55 : rtx tmp = gen_reg_rtx (V4SFmode);
18388 55 : emit_insn (gen_sse2_insertps_v4sf_3 (tmp, var,
18389 : CONST0_RTX (V4SFmode)));
18390 55 : emit_insn (gen_vec_shr_v4sf (target, tmp,
18391 55 : GEN_INT ((3 - one_var) * 32)));
18392 : }
18393 : return true;
18394 : }
18395 2596 : if (!one_var)
18396 2580 : emit_insn (gen_vec_setv4sf_0 (target, CONST0_RTX (V4SFmode), var));
18397 16 : else if (TARGET_SSE2)
18398 : {
18399 13 : rtx tmp = gen_reg_rtx (V4SFmode);
18400 13 : emit_insn (gen_vec_setv4sf_0 (tmp, CONST0_RTX (V4SFmode), var));
18401 13 : emit_insn (gen_vec_shl_v4sf (target, tmp, GEN_INT (one_var * 32)));
18402 : }
18403 : else
18404 : {
18405 3 : rtx tmp = gen_reg_rtx (V4SFmode);
18406 3 : emit_insn (gen_vec_setv4sf_0 (tmp, CONST0_RTX (V4SFmode), var));
18407 10 : emit_insn (gen_sse_shufps_v4sf (target, tmp, tmp,
18408 : const1_rtx,
18409 3 : GEN_INT (one_var == 1 ? 0 : 1),
18410 : GEN_INT (one_var == 2 ? 0+4 : 1+4),
18411 : GEN_INT (one_var == 3 ? 0+4 : 1+4)));
18412 : }
18413 : return true;
18414 7 : case E_V4DImode:
18415 7 : if (TARGET_AVX2 && (TARGET_64BIT || MEM_P (var)))
18416 : {
18417 6 : if (!REG_P (var) && !MEM_P (var))
18418 0 : var = force_reg (DImode, var);
18419 6 : x = gen_rtx_VEC_DUPLICATE (V4DImode, var);
18420 6 : x = gen_rtx_VEC_MERGE (V4DImode, x, CONST0_RTX (V4DImode),
18421 : const1_rtx);
18422 6 : if (one_var)
18423 : {
18424 0 : tmp = gen_reg_rtx (V4DImode);
18425 0 : emit_insn (gen_rtx_SET (tmp, x));
18426 0 : emit_insn (gen_avx2_permv4di_1 (target, tmp,
18427 : const1_rtx,
18428 0 : GEN_INT (one_var == 1 ? 0 : 1),
18429 0 : GEN_INT (one_var == 2 ? 0 : 1),
18430 0 : GEN_INT (one_var == 3 ? 0 : 1)));
18431 : }
18432 : else
18433 6 : emit_insn (gen_rtx_SET (target, x));
18434 : }
18435 : else
18436 : {
18437 1 : tmp = gen_reg_rtx (V2DImode);
18438 1 : if (!ix86_expand_vector_init_one_nonzero (mmx_ok, V2DImode, tmp,
18439 : var, one_var & 1))
18440 0 : gcc_unreachable ();
18441 1 : rtx zero = CONST0_RTX (V2DImode);
18442 1 : if (one_var >= 2)
18443 : {
18444 0 : zero = force_reg (V2DImode, zero);
18445 0 : emit_insn (gen_avx_vec_concatv4di (target, zero, tmp));
18446 : }
18447 : else
18448 1 : emit_insn (gen_avx_vec_concatv4di (target, tmp, zero));
18449 : }
18450 : return true;
18451 13 : case E_V4DFmode:
18452 13 : if (TARGET_AVX2)
18453 : {
18454 13 : if (!REG_P (var) && !MEM_P (var))
18455 0 : var = force_reg (DFmode, var);
18456 13 : x = gen_rtx_VEC_DUPLICATE (V4DFmode, var);
18457 13 : x = gen_rtx_VEC_MERGE (V4DFmode, x, CONST0_RTX (V4DFmode),
18458 : const1_rtx);
18459 13 : if (one_var)
18460 : {
18461 0 : tmp = gen_reg_rtx (V4DFmode);
18462 0 : emit_insn (gen_rtx_SET (tmp, x));
18463 0 : emit_insn (gen_avx2_permv4df_1 (target, tmp,
18464 : const1_rtx,
18465 0 : GEN_INT (one_var == 1 ? 0 : 1),
18466 0 : GEN_INT (one_var == 2 ? 0 : 1),
18467 0 : GEN_INT (one_var == 3 ? 0 : 1)));
18468 : }
18469 : else
18470 13 : emit_insn (gen_rtx_SET (target, x));
18471 : }
18472 : else
18473 : {
18474 0 : tmp = gen_reg_rtx (V2DFmode);
18475 0 : if (!ix86_expand_vector_init_one_nonzero (mmx_ok, V2DFmode, tmp,
18476 : var, one_var & 1))
18477 0 : gcc_unreachable ();
18478 0 : rtx zero = CONST0_RTX (V2DFmode);
18479 0 : if (one_var >= 2)
18480 : {
18481 0 : zero = force_reg (V2DFmode, zero);
18482 0 : emit_insn (gen_avx_vec_concatv4df (target, zero, tmp));
18483 : }
18484 : else
18485 0 : emit_insn (gen_avx_vec_concatv4df (target, tmp, zero));
18486 : }
18487 : return true;
18488 86 : case E_V16QImode:
18489 86 : use_vector_set = TARGET_SSE4_1;
18490 86 : break;
18491 102 : case E_V8HImode:
18492 102 : use_vector_set = TARGET_SSE2;
18493 102 : gen_vec_set_0 = TARGET_AVX512FP16 && one_var == 0
18494 102 : ? gen_vec_setv8hi_0 : NULL;
18495 : break;
18496 15 : case E_V8QImode:
18497 15 : use_vector_set = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
18498 : break;
18499 14 : case E_V4HImode:
18500 14 : case E_V4HFmode:
18501 14 : case E_V4BFmode:
18502 14 : use_vector_set = TARGET_SSE || TARGET_3DNOW_A;
18503 : break;
18504 32 : case E_V4QImode:
18505 32 : use_vector_set = TARGET_SSE4_1;
18506 32 : break;
18507 0 : case E_V32QImode:
18508 0 : use_vector_set = TARGET_AVX;
18509 0 : break;
18510 5 : case E_V16HImode:
18511 5 : use_vector_set = TARGET_AVX;
18512 5 : gen_vec_set_0 = TARGET_AVX512FP16 && one_var == 0
18513 5 : ? gen_vec_setv16hi_0 : NULL;
18514 : break;
18515 5 : case E_V8SImode:
18516 5 : use_vector_set = TARGET_AVX;
18517 5 : gen_vec_set_0 = gen_vec_setv8si_0;
18518 5 : break;
18519 22 : case E_V8SFmode:
18520 22 : use_vector_set = TARGET_AVX;
18521 22 : gen_vec_set_0 = gen_vec_setv8sf_0;
18522 22 : break;
18523 17 : case E_V16SImode:
18524 17 : use_vector_set = TARGET_AVX512F && one_var == 0;
18525 : gen_vec_set_0 = gen_vec_setv16si_0;
18526 : break;
18527 22 : case E_V16SFmode:
18528 22 : use_vector_set = TARGET_AVX512F && one_var == 0;
18529 : gen_vec_set_0 = gen_vec_setv16sf_0;
18530 : break;
18531 0 : case E_V8DFmode:
18532 0 : use_vector_set = TARGET_AVX512F && one_var == 0;
18533 : gen_vec_set_0 = gen_vec_setv8df_0;
18534 : break;
18535 2 : case E_V8DImode:
18536 : /* Use ix86_expand_vector_set in 64bit mode only. */
18537 2 : use_vector_set = TARGET_AVX512F && TARGET_64BIT && one_var == 0;
18538 : gen_vec_set_0 = gen_vec_setv8di_0;
18539 : break;
18540 39 : case E_V8HFmode:
18541 39 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18542 : gen_vec_set_0 = gen_vec_setv8hf_0;
18543 : break;
18544 9 : case E_V16HFmode:
18545 9 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18546 : gen_vec_set_0 = gen_vec_setv16hf_0;
18547 : break;
18548 6 : case E_V32HFmode:
18549 6 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18550 : gen_vec_set_0 = gen_vec_setv32hf_0;
18551 : break;
18552 2 : case E_V8BFmode:
18553 2 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18554 : gen_vec_set_0 = gen_vec_setv8bf_0;
18555 : break;
18556 0 : case E_V16BFmode:
18557 0 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18558 : gen_vec_set_0 = gen_vec_setv16bf_0;
18559 : break;
18560 0 : case E_V32BFmode:
18561 0 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18562 : gen_vec_set_0 = gen_vec_setv32bf_0;
18563 : break;
18564 4 : case E_V32HImode:
18565 4 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18566 : gen_vec_set_0 = gen_vec_setv32hi_0;
18567 : default:
18568 : break;
18569 : }
18570 :
18571 252 : if (use_vector_set)
18572 : {
18573 301 : if (gen_vec_set_0 && one_var == 0)
18574 : {
18575 316 : var = force_reg (GET_MODE_INNER (mode), var);
18576 158 : emit_insn (gen_vec_set_0 (target, CONST0_RTX (mode), var));
18577 158 : return true;
18578 : }
18579 143 : emit_insn (gen_rtx_SET (target, CONST0_RTX (mode)));
18580 286 : var = force_reg (GET_MODE_INNER (mode), var);
18581 143 : ix86_expand_vector_set (mmx_ok, target, var, one_var);
18582 143 : return true;
18583 : }
18584 :
18585 881 : switch (mode)
18586 : {
18587 780 : case E_V2SFmode:
18588 780 : case E_V2SImode:
18589 780 : if (!mmx_ok || one_var != 0)
18590 : return false;
18591 378 : var = force_reg (GET_MODE_INNER (mode), var);
18592 378 : x = gen_rtx_VEC_CONCAT (mode, var, CONST0_RTX (GET_MODE_INNER (mode)));
18593 189 : emit_insn (gen_rtx_SET (target, x));
18594 189 : return true;
18595 :
18596 0 : case E_V2DFmode:
18597 0 : case E_V2DImode:
18598 0 : if (one_var != 0)
18599 : return false;
18600 0 : var = force_reg (GET_MODE_INNER (mode), var);
18601 0 : x = gen_rtx_VEC_CONCAT (mode, var, CONST0_RTX (GET_MODE_INNER (mode)));
18602 0 : emit_insn (gen_rtx_SET (target, x));
18603 0 : return true;
18604 :
18605 0 : case E_V8HImode:
18606 0 : if (one_var > 1 && !TARGET_SSE2)
18607 : return false;
18608 : /* Zero extend the variable element to SImode and recurse. */
18609 0 : var = convert_modes (SImode, HImode, var, true);
18610 0 : var = force_reg (SImode, var);
18611 0 : if (one_var == 1)
18612 : {
18613 0 : var = simplify_gen_binary (ASHIFT, SImode, var, GEN_INT (16));
18614 0 : var = force_reg (SImode, var);
18615 0 : one_var = 0;
18616 : }
18617 : else
18618 0 : one_var *= 16;
18619 0 : x = gen_reg_rtx (V4SImode);
18620 0 : if (!ix86_expand_vector_init_one_nonzero (mmx_ok, V4SImode, x, var, 0))
18621 0 : gcc_unreachable ();
18622 0 : if (one_var)
18623 : {
18624 0 : tmp = gen_reg_rtx (V4SImode);
18625 0 : emit_insn (gen_vec_shl_v4si (tmp, x, GEN_INT (one_var)));
18626 0 : x = tmp;
18627 : }
18628 0 : emit_move_insn (target, gen_lowpart (mode, x));
18629 0 : return true;
18630 :
18631 30 : case E_V16QImode:
18632 30 : if (one_var > 3 && !TARGET_SSE2)
18633 : return false;
18634 : /* Zero extend the variable element to SImode and recurse. */
18635 30 : var = convert_modes (SImode, QImode, var, true);
18636 30 : var = force_reg (SImode, var);
18637 30 : if (one_var < 4)
18638 : {
18639 14 : var = simplify_gen_binary (ASHIFT, SImode, var,
18640 14 : GEN_INT (one_var * 8));
18641 14 : var = force_reg (SImode, var);
18642 14 : one_var = 0;
18643 : }
18644 : else
18645 16 : one_var *= 8;
18646 30 : x = gen_reg_rtx (V4SImode);
18647 30 : if (!ix86_expand_vector_init_one_nonzero (mmx_ok, V4SImode, x, var, 0))
18648 0 : gcc_unreachable ();
18649 30 : if (one_var)
18650 : {
18651 16 : tmp = gen_reg_rtx (V4SImode);
18652 16 : emit_insn (gen_vec_shl_v4si (tmp, x, GEN_INT (one_var)));
18653 16 : x = tmp;
18654 : }
18655 30 : emit_move_insn (target, gen_lowpart (V16QImode, x));
18656 30 : return true;
18657 :
18658 0 : case E_V4HImode:
18659 0 : if (!mmx_ok || one_var > 1)
18660 : return false;
18661 : /* Zero extend the variable element to SImode and recurse. */
18662 0 : var = convert_modes (SImode, HImode, var, true);
18663 0 : var = force_reg (SImode, var);
18664 0 : if (one_var == 1)
18665 : {
18666 0 : var = simplify_gen_binary (ASHIFT, SImode, var, GEN_INT (16));
18667 0 : var = force_reg (SImode, var);
18668 : }
18669 0 : x = gen_reg_rtx (V2SImode);
18670 0 : if (!ix86_expand_vector_init_one_nonzero (true, V2SImode, x, var, 0))
18671 0 : gcc_unreachable ();
18672 0 : emit_move_insn (target, gen_lowpart (V4HImode, x));
18673 0 : return true;
18674 :
18675 14 : case E_V8QImode:
18676 14 : if (!mmx_ok || one_var > 3)
18677 : return false;
18678 : /* Zero extend the variable element to SImode and recurse. */
18679 14 : var = convert_modes (SImode, QImode, var, true);
18680 14 : var = force_reg (SImode, var);
18681 14 : if (one_var)
18682 : {
18683 0 : var = simplify_gen_binary (ASHIFT, SImode, var,
18684 0 : GEN_INT (one_var * 8));
18685 0 : var = force_reg (SImode, var);
18686 : }
18687 14 : x = gen_reg_rtx (V2SImode);
18688 14 : if (!ix86_expand_vector_init_one_nonzero (true, V2SImode, x, var, 0))
18689 0 : gcc_unreachable ();
18690 14 : emit_move_insn (target, gen_lowpart (V8QImode, x));
18691 14 : return true;
18692 :
18693 : default:
18694 : return false;
18695 : }
18696 : }
18697 :
18698 : /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
18699 : consisting of the values in VALS. It is known that all elements
18700 : except ONE_VAR are constants. Return true if successful. */
18701 :
18702 : static bool
18703 1683 : ix86_expand_vector_init_one_var (bool mmx_ok, machine_mode mode,
18704 : rtx target, rtx vals, int one_var)
18705 : {
18706 1683 : rtx var = XVECEXP (vals, 0, one_var);
18707 1683 : machine_mode wmode;
18708 1683 : rtx const_vec, x;
18709 :
18710 1683 : const_vec = copy_rtx (vals);
18711 1683 : XVECEXP (const_vec, 0, one_var) = CONST0_RTX (GET_MODE_INNER (mode));
18712 1683 : const_vec = gen_rtx_CONST_VECTOR (mode, XVEC (const_vec, 0));
18713 :
18714 1683 : switch (mode)
18715 : {
18716 : case E_V2DFmode:
18717 : case E_V2DImode:
18718 : case E_V2SFmode:
18719 : case E_V2SImode:
18720 : /* For the two element vectors, it's just as easy to use
18721 : the general case. */
18722 : return false;
18723 :
18724 3 : case E_V4DImode:
18725 : /* Use ix86_expand_vector_set in 64bit mode only. */
18726 3 : if (!TARGET_64BIT)
18727 : return false;
18728 : /* FALLTHRU */
18729 : case E_V8HFmode:
18730 : case E_V16HFmode:
18731 : case E_V8BFmode:
18732 : case E_V16BFmode:
18733 : case E_V4DFmode:
18734 : case E_V8SFmode:
18735 : case E_V8SImode:
18736 : case E_V16HImode:
18737 : case E_V32QImode:
18738 : case E_V4SFmode:
18739 : case E_V4SImode:
18740 : case E_V8HImode:
18741 : case E_V4HImode:
18742 : case E_V4HFmode:
18743 : case E_V4BFmode:
18744 : break;
18745 :
18746 1 : case E_V16QImode:
18747 1 : if (TARGET_SSE4_1)
18748 : break;
18749 1 : wmode = V8HImode;
18750 1 : goto widen;
18751 0 : case E_V8QImode:
18752 0 : if (TARGET_MMX_WITH_SSE && TARGET_SSE4_1)
18753 : break;
18754 0 : wmode = V4HImode;
18755 0 : goto widen;
18756 38 : case E_V4QImode:
18757 38 : if (TARGET_SSE4_1)
18758 : break;
18759 : wmode = V2HImode;
18760 39 : widen:
18761 : /* There's no way to set one QImode entry easily. Combine
18762 : the variable value with its adjacent constant value, and
18763 : promote to an HImode set. */
18764 39 : x = XVECEXP (vals, 0, one_var ^ 1);
18765 39 : if (one_var & 1)
18766 : {
18767 8 : var = convert_modes (HImode, QImode, var, true);
18768 8 : var = expand_simple_binop (HImode, ASHIFT, var, GEN_INT (8),
18769 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
18770 8 : x = GEN_INT (INTVAL (x) & 0xff);
18771 : }
18772 : else
18773 : {
18774 31 : var = convert_modes (HImode, QImode, var, true);
18775 31 : x = gen_int_mode (UINTVAL (x) << 8, HImode);
18776 : }
18777 39 : if (x != const0_rtx)
18778 7 : var = expand_simple_binop (HImode, IOR, var, x, var,
18779 : 1, OPTAB_LIB_WIDEN);
18780 :
18781 39 : x = gen_reg_rtx (wmode);
18782 39 : emit_move_insn (x, gen_lowpart (wmode, const_vec));
18783 39 : ix86_expand_vector_set (mmx_ok, x, var, one_var >> 1);
18784 :
18785 39 : emit_move_insn (target, gen_lowpart (mode, x));
18786 39 : return true;
18787 :
18788 : default:
18789 : return false;
18790 : }
18791 :
18792 191 : emit_move_insn (target, const_vec);
18793 191 : ix86_expand_vector_set (mmx_ok, target, var, one_var);
18794 191 : return true;
18795 : }
18796 :
18797 : /* A subroutine of ix86_expand_vector_init_general. Use vector
18798 : concatenate to handle the most general case: all values variable,
18799 : and none identical. */
18800 :
18801 : static void
18802 24797 : ix86_expand_vector_init_concat (machine_mode mode,
18803 : rtx target, rtx *ops, int n)
18804 : {
18805 24797 : machine_mode half_mode = VOIDmode;
18806 24797 : rtx half[2];
18807 24797 : rtvec v;
18808 24797 : int i, j;
18809 :
18810 24797 : switch (n)
18811 : {
18812 23758 : case 2:
18813 23758 : switch (mode)
18814 : {
18815 : case E_V32HFmode:
18816 : half_mode = V16HFmode;
18817 : break;
18818 0 : case E_V32BFmode:
18819 0 : half_mode = V16BFmode;
18820 0 : break;
18821 59 : case E_V16SImode:
18822 59 : half_mode = V8SImode;
18823 59 : break;
18824 34 : case E_V16SFmode:
18825 34 : half_mode = V8SFmode;
18826 34 : break;
18827 84 : case E_V8DImode:
18828 84 : half_mode = V4DImode;
18829 84 : break;
18830 62 : case E_V8DFmode:
18831 62 : half_mode = V4DFmode;
18832 62 : break;
18833 0 : case E_V16HFmode:
18834 0 : half_mode = V8HFmode;
18835 0 : break;
18836 0 : case E_V16BFmode:
18837 0 : half_mode = V8BFmode;
18838 0 : break;
18839 39 : case E_V8SImode:
18840 39 : half_mode = V4SImode;
18841 39 : break;
18842 7 : case E_V8SFmode:
18843 7 : half_mode = V4SFmode;
18844 7 : break;
18845 125 : case E_V4DImode:
18846 125 : half_mode = V2DImode;
18847 125 : break;
18848 57 : case E_V4DFmode:
18849 57 : half_mode = V2DFmode;
18850 57 : break;
18851 997 : case E_V4SImode:
18852 997 : half_mode = V2SImode;
18853 997 : break;
18854 1147 : case E_V4SFmode:
18855 1147 : half_mode = V2SFmode;
18856 1147 : break;
18857 0 : case E_V2DImode:
18858 0 : half_mode = DImode;
18859 0 : break;
18860 17387 : case E_V2SImode:
18861 17387 : half_mode = SImode;
18862 17387 : break;
18863 0 : case E_V2DFmode:
18864 0 : half_mode = DFmode;
18865 0 : break;
18866 3760 : case E_V2SFmode:
18867 3760 : half_mode = SFmode;
18868 3760 : break;
18869 0 : default:
18870 0 : gcc_unreachable ();
18871 : }
18872 :
18873 23758 : if (!register_operand (ops[1], half_mode))
18874 10088 : ops[1] = force_reg (half_mode, ops[1]);
18875 23758 : if (!register_operand (ops[0], half_mode))
18876 5982 : ops[0] = force_reg (half_mode, ops[0]);
18877 23758 : emit_insn (gen_rtx_SET (target, gen_rtx_VEC_CONCAT (mode, ops[0],
18878 : ops[1])));
18879 23758 : break;
18880 :
18881 863 : case 4:
18882 863 : switch (mode)
18883 : {
18884 : case E_V4DImode:
18885 : half_mode = V2DImode;
18886 : break;
18887 0 : case E_V4DFmode:
18888 0 : half_mode = V2DFmode;
18889 0 : break;
18890 0 : case E_V4SImode:
18891 0 : half_mode = V2SImode;
18892 0 : break;
18893 863 : case E_V4SFmode:
18894 863 : half_mode = V2SFmode;
18895 863 : break;
18896 0 : default:
18897 0 : gcc_unreachable ();
18898 : }
18899 863 : goto half;
18900 :
18901 105 : case 8:
18902 105 : switch (mode)
18903 : {
18904 : case E_V8DImode:
18905 : half_mode = V4DImode;
18906 : break;
18907 62 : case E_V8DFmode:
18908 62 : half_mode = V4DFmode;
18909 62 : break;
18910 0 : case E_V8SImode:
18911 0 : half_mode = V4SImode;
18912 0 : break;
18913 0 : case E_V8SFmode:
18914 0 : half_mode = V4SFmode;
18915 0 : break;
18916 0 : default:
18917 0 : gcc_unreachable ();
18918 : }
18919 105 : goto half;
18920 :
18921 71 : case 16:
18922 71 : switch (mode)
18923 : {
18924 : case E_V16SImode:
18925 : half_mode = V8SImode;
18926 : break;
18927 33 : case E_V16SFmode:
18928 33 : half_mode = V8SFmode;
18929 33 : break;
18930 0 : default:
18931 0 : gcc_unreachable ();
18932 : }
18933 71 : goto half;
18934 :
18935 1039 : half:
18936 : /* FIXME: We process inputs backward to help RA. PR 36222. */
18937 1039 : i = n - 1;
18938 3117 : for (j = 1; j != -1; j--)
18939 : {
18940 2078 : half[j] = gen_reg_rtx (half_mode);
18941 2078 : switch (n >> 1)
18942 : {
18943 1726 : case 2:
18944 1726 : v = gen_rtvec (2, ops[i-1], ops[i]);
18945 1726 : i -= 2;
18946 1726 : break;
18947 210 : case 4:
18948 210 : v = gen_rtvec (4, ops[i-3], ops[i-2], ops[i-1], ops[i]);
18949 210 : i -= 4;
18950 210 : break;
18951 142 : case 8:
18952 284 : v = gen_rtvec (8, ops[i-7], ops[i-6], ops[i-5], ops[i-4],
18953 142 : ops[i-3], ops[i-2], ops[i-1], ops[i]);
18954 142 : i -= 8;
18955 142 : break;
18956 : default:
18957 : gcc_unreachable ();
18958 : }
18959 2078 : ix86_expand_vector_init (false, half[j],
18960 : gen_rtx_PARALLEL (half_mode, v));
18961 : }
18962 :
18963 1039 : ix86_expand_vector_init_concat (mode, target, half, 2);
18964 1039 : break;
18965 :
18966 0 : default:
18967 0 : gcc_unreachable ();
18968 : }
18969 24797 : }
18970 :
18971 : /* A subroutine of ix86_expand_vector_init_general. Use vector
18972 : interleave to handle the most general case: all values variable,
18973 : and none identical. */
18974 :
18975 : static void
18976 3489 : ix86_expand_vector_init_interleave (machine_mode mode,
18977 : rtx target, rtx *ops, int n)
18978 : {
18979 3489 : machine_mode first_imode, second_imode, third_imode, inner_mode;
18980 3489 : int i, j;
18981 3489 : rtx op, op0, op1;
18982 3489 : rtx (*gen_load_even) (rtx, rtx, rtx);
18983 3489 : rtx (*gen_interleave_first_low) (rtx, rtx, rtx);
18984 3489 : rtx (*gen_interleave_second_low) (rtx, rtx, rtx);
18985 :
18986 3489 : switch (mode)
18987 : {
18988 : case E_V8HFmode:
18989 : gen_load_even = gen_vec_interleave_lowv8hf;
18990 : gen_interleave_first_low = gen_vec_interleave_lowv4si;
18991 : gen_interleave_second_low = gen_vec_interleave_lowv2di;
18992 : inner_mode = HFmode;
18993 : first_imode = V4SImode;
18994 : second_imode = V2DImode;
18995 : third_imode = VOIDmode;
18996 : break;
18997 487 : case E_V8BFmode:
18998 487 : gen_load_even = gen_vec_interleave_lowv8bf;
18999 487 : gen_interleave_first_low = gen_vec_interleave_lowv4si;
19000 487 : gen_interleave_second_low = gen_vec_interleave_lowv2di;
19001 487 : inner_mode = BFmode;
19002 487 : first_imode = V4SImode;
19003 487 : second_imode = V2DImode;
19004 487 : third_imode = VOIDmode;
19005 487 : break;
19006 412 : case E_V8HImode:
19007 412 : gen_load_even = gen_vec_setv8hi;
19008 412 : gen_interleave_first_low = gen_vec_interleave_lowv4si;
19009 412 : gen_interleave_second_low = gen_vec_interleave_lowv2di;
19010 412 : inner_mode = HImode;
19011 412 : first_imode = V4SImode;
19012 412 : second_imode = V2DImode;
19013 412 : third_imode = VOIDmode;
19014 412 : break;
19015 362 : case E_V16QImode:
19016 362 : gen_load_even = gen_vec_setv16qi;
19017 362 : gen_interleave_first_low = gen_vec_interleave_lowv8hi;
19018 362 : gen_interleave_second_low = gen_vec_interleave_lowv4si;
19019 362 : inner_mode = QImode;
19020 362 : first_imode = V8HImode;
19021 362 : second_imode = V4SImode;
19022 362 : third_imode = V2DImode;
19023 362 : break;
19024 0 : default:
19025 0 : gcc_unreachable ();
19026 : }
19027 :
19028 18893 : for (i = 0; i < n; i++)
19029 : {
19030 15404 : op = ops [i + i];
19031 15404 : if (inner_mode == HFmode || inner_mode == BFmode)
19032 : {
19033 10860 : rtx even, odd;
19034 : /* Use vpuncklwd to pack 2 HFmode or BFmode. */
19035 1948 : machine_mode vec_mode =
19036 10860 : (inner_mode == HFmode) ? V8HFmode : V8BFmode;
19037 10860 : op0 = gen_reg_rtx (vec_mode);
19038 10860 : even = lowpart_subreg (vec_mode,
19039 : force_reg (inner_mode, op), inner_mode);
19040 10860 : odd = lowpart_subreg (vec_mode,
19041 10860 : force_reg (inner_mode, ops[i + i + 1]),
19042 : inner_mode);
19043 10860 : emit_insn (gen_load_even (op0, even, odd));
19044 : }
19045 : else
19046 : {
19047 : /* Extend the odd element to SImode using a paradoxical SUBREG. */
19048 4544 : op0 = gen_reg_rtx (SImode);
19049 4544 : emit_move_insn (op0, gen_lowpart (SImode, op));
19050 :
19051 : /* Insert the SImode value as low element of V4SImode vector. */
19052 4544 : op1 = gen_reg_rtx (V4SImode);
19053 4544 : op0 = gen_rtx_VEC_MERGE (V4SImode,
19054 : gen_rtx_VEC_DUPLICATE (V4SImode,
19055 : op0),
19056 : CONST0_RTX (V4SImode),
19057 : const1_rtx);
19058 4544 : emit_insn (gen_rtx_SET (op1, op0));
19059 :
19060 : /* Cast the V4SImode vector back to a vector in original mode. */
19061 4544 : op0 = gen_reg_rtx (mode);
19062 4544 : emit_move_insn (op0, gen_lowpart (mode, op1));
19063 :
19064 : /* Load even elements into the second position. */
19065 4544 : emit_insn (gen_load_even (op0,
19066 : force_reg (inner_mode,
19067 4544 : ops[i + i + 1]),
19068 : const1_rtx));
19069 : }
19070 :
19071 : /* Cast vector to FIRST_IMODE vector. */
19072 15404 : ops[i] = gen_reg_rtx (first_imode);
19073 15404 : emit_move_insn (ops[i], gen_lowpart (first_imode, op0));
19074 : }
19075 :
19076 : /* Interleave low FIRST_IMODE vectors. */
19077 11191 : for (i = j = 0; i < n; i += 2, j++)
19078 : {
19079 7702 : op0 = gen_reg_rtx (first_imode);
19080 7702 : emit_insn (gen_interleave_first_low (op0, ops[i], ops[i + 1]));
19081 :
19082 : /* Cast FIRST_IMODE vector to SECOND_IMODE vector. */
19083 7702 : ops[j] = gen_reg_rtx (second_imode);
19084 7702 : emit_move_insn (ops[j], gen_lowpart (second_imode, op0));
19085 : }
19086 :
19087 : /* Interleave low SECOND_IMODE vectors. */
19088 3489 : switch (second_imode)
19089 : {
19090 : case E_V4SImode:
19091 1086 : for (i = j = 0; i < n / 2; i += 2, j++)
19092 : {
19093 724 : op0 = gen_reg_rtx (second_imode);
19094 724 : emit_insn (gen_interleave_second_low (op0, ops[i],
19095 724 : ops[i + 1]));
19096 :
19097 : /* Cast the SECOND_IMODE vector to the THIRD_IMODE
19098 : vector. */
19099 724 : ops[j] = gen_reg_rtx (third_imode);
19100 724 : emit_move_insn (ops[j], gen_lowpart (third_imode, op0));
19101 : }
19102 : second_imode = V2DImode;
19103 : gen_interleave_second_low = gen_vec_interleave_lowv2di;
19104 : /* FALLTHRU */
19105 :
19106 3489 : case E_V2DImode:
19107 3489 : op0 = gen_reg_rtx (second_imode);
19108 3489 : emit_insn (gen_interleave_second_low (op0, ops[0],
19109 : ops[1]));
19110 :
19111 : /* Cast the SECOND_IMODE vector back to a vector on original
19112 : mode. */
19113 3489 : emit_insn (gen_rtx_SET (target, gen_lowpart (mode, op0)));
19114 3489 : break;
19115 :
19116 : default:
19117 : gcc_unreachable ();
19118 : }
19119 3489 : }
19120 :
19121 : /* Count number of non-zero integer constants in OPS array of length N. */
19122 :
19123 : static int
19124 5866 : nonzero_int_const_count (rtx *ops, int n)
19125 : {
19126 5866 : int result = 0;
19127 5866 : int i;
19128 33070 : for (i = 0; i < n; i++)
19129 27204 : if (CONST_INT_P (ops[i]) && ops[i] != const0_rtx)
19130 201 : result++;
19131 5866 : return result;
19132 : }
19133 :
19134 : /* Count number of non-zero float constants in OPS array of length N. */
19135 :
19136 : static int
19137 1881 : nonzero_float_const_count (rtx *ops, int n)
19138 : {
19139 1881 : int result = 0;
19140 1881 : int i;
19141 10313 : for (i = 0; i < n; i++)
19142 8432 : if (CONST_DOUBLE_P (ops[i]) && ops[i] != CONST0_RTX (SFmode))
19143 12 : result++;
19144 1881 : return result;
19145 : }
19146 :
19147 : /* Count number of non-zero double constants in OPS array of length N. */
19148 :
19149 : static int
19150 495 : nonzero_double_const_count (rtx *ops, int n)
19151 : {
19152 495 : int result = 0;
19153 495 : int i;
19154 2475 : for (i = 0; i < n; i++)
19155 1980 : if (CONST_DOUBLE_P (ops[i]) && ops[i] != CONST0_RTX (DFmode))
19156 2 : result++;
19157 495 : return result;
19158 : }
19159 :
19160 :
19161 : /* A subroutine of ix86_expand_vector_init_general. Handle the most
19162 : general case: all values variable and none identical. */
19163 :
19164 : static void
19165 977 : ix86_expand_vector_init_insert (machine_mode mode, rtx target,
19166 : rtx *ops, int n)
19167 : {
19168 977 : machine_mode inner_mode = GET_MODE_INNER (mode);
19169 977 : rtx (*pinsr)(rtx, rtx, rtx, rtx);
19170 977 : rtx tmp = gen_reg_rtx (mode);
19171 977 : rtx var, x;
19172 977 : int i;
19173 :
19174 977 : var = ops[0];
19175 :
19176 977 : switch (mode)
19177 : {
19178 98 : case E_V16QImode:
19179 98 : if (var != const0_rtx)
19180 : {
19181 98 : if (REG_P (var)
19182 52 : && ops[1] != const0_rtx
19183 43 : && ops[2] != const0_rtx
19184 40 : && ops[3] != const0_rtx)
19185 40 : var = gen_lowpart (SImode, var);
19186 : else
19187 58 : var = convert_modes (SImode, QImode, var, true);
19188 98 : var = force_reg (SImode, var);
19189 98 : x = gen_reg_rtx (V4SImode);
19190 98 : emit_insn (gen_vec_setv4si_0 (x, CONST0_RTX (V4SImode), var));
19191 98 : emit_move_insn (tmp, gen_lowpart (V16QImode, x));
19192 : }
19193 : else
19194 0 : emit_move_insn (tmp, CONST0_RTX (mode));
19195 : pinsr = gen_sse4_1_pinsrb;
19196 : break;
19197 557 : case E_V8HImode:
19198 557 : if (var != const0_rtx)
19199 : {
19200 553 : if (REG_P (var)
19201 368 : && ops[1] != const0_rtx)
19202 368 : var = gen_lowpart (SImode, var);
19203 : else
19204 185 : var = convert_modes (SImode, HImode, var, true);
19205 553 : var = force_reg (SImode, var);
19206 553 : x = gen_reg_rtx (V4SImode);
19207 553 : emit_insn (gen_vec_setv4si_0 (x, CONST0_RTX (V4SImode), var));
19208 553 : emit_move_insn (tmp, gen_lowpart (V8HImode, x));
19209 : }
19210 : else
19211 4 : emit_move_insn (tmp, CONST0_RTX (mode));
19212 : pinsr = gen_sse2_pinsrw;
19213 : break;
19214 322 : case E_V4SImode:
19215 322 : if (var != const0_rtx)
19216 : {
19217 319 : if (!REG_P (var) && !MEM_P (var))
19218 4 : var = force_reg (SImode, var);
19219 319 : emit_insn (gen_vec_setv4si_0 (tmp, CONST0_RTX (mode), var));
19220 : }
19221 : else
19222 3 : emit_move_insn (tmp, CONST0_RTX (mode));
19223 : pinsr = gen_sse4_1_pinsrd;
19224 : break;
19225 :
19226 0 : default:
19227 0 : gcc_unreachable ();
19228 : }
19229 :
19230 7312 : for (i = 1; i < n; i++)
19231 6335 : if (ops[i] != CONST0_RTX (inner_mode))
19232 : {
19233 6007 : rtx val = ops[i];
19234 6007 : if (!REG_P (val) && !MEM_P (val))
19235 245 : val = force_reg (inner_mode, val);
19236 6007 : emit_insn (pinsr (tmp, tmp, val, GEN_INT (1 << i)));
19237 : }
19238 977 : emit_move_insn (target, tmp);
19239 977 : }
19240 :
19241 : /* Helper function. Determine if the given OPS array of size N
19242 : contains only zeros and one other value (possible repeated).
19243 : If TRUE, *VAR returns the value, PERM[i] contains 0 for
19244 : this value and 1 for a CONST0_RTX. */
19245 :
19246 : static bool
19247 19611 : onevar_perm_p (const rtx *ops, int n, int *perm, rtx *var)
19248 : {
19249 19611 : bool found = false;
19250 19611 : int i;
19251 :
19252 40782 : for (i = 0; i < n; i++)
19253 40542 : if (ops[i] == const0_rtx
19254 39654 : || ops[i] == CONST0_RTX (SFmode)
19255 39464 : || ops[i] == CONST0_RTX (DFmode))
19256 1078 : perm[i] = 1;
19257 39464 : else if (!found)
19258 : {
19259 19611 : *var = ops[i];
19260 19611 : found = true;
19261 19611 : perm[i] = 0;
19262 : }
19263 19853 : else if (rtx_equal_p (*var, ops[i]))
19264 482 : perm[i] = 0;
19265 : else
19266 : return false;
19267 :
19268 : return found;
19269 : }
19270 :
19271 : /* Helper function. Determine if the given OPS array of size N
19272 : contains only zeros and two other values (possible repeated).
19273 : If TRUE, VARS returns the values, PERM[i] contains 0 for
19274 : the first value, 1 for the second value and 2 for CONST0_RTX. */
19275 :
19276 : static bool
19277 7371 : twovar_perm_p (const rtx *ops, int n, int *perm, rtx *vars)
19278 : {
19279 7371 : int count = 0;
19280 7371 : int i;
19281 :
19282 24252 : for (i = 0; i < n; i++)
19283 23519 : if (ops[i] == const0_rtx
19284 23030 : || ops[i] == CONST0_RTX (SFmode)
19285 22892 : || ops[i] == CONST0_RTX (DFmode))
19286 627 : perm[i] = 2;
19287 22892 : else if (count == 0)
19288 : {
19289 7371 : vars[0] = ops[i];
19290 7371 : perm[i] = 0;
19291 7371 : count = 1;
19292 : }
19293 15521 : else if (rtx_equal_p (vars[0], ops[i]))
19294 1126 : perm[i] = 0;
19295 14395 : else if (count == 1)
19296 : {
19297 7371 : vars[1] = ops[i];
19298 7371 : perm[i] = 1;
19299 7371 : count = 2;
19300 : }
19301 7024 : else if (rtx_equal_p (vars[1], ops[i]))
19302 386 : perm[i] = 1;
19303 : else
19304 : return false;
19305 :
19306 733 : return count == 2;
19307 : }
19308 :
19309 : /* A subroutine of ix86_expand_vector_init for V2DImode. */
19310 :
19311 : static void
19312 60401 : ix86_expand_vector_init_v2di (rtx target, rtx *ops)
19313 : {
19314 60401 : if (ops[0] == const0_rtx && ops[1] == const0_rtx)
19315 2 : emit_move_insn (target, CONST0_RTX (V2DImode));
19316 60399 : else if (CONST_INT_P (ops[0]) && CONST_INT_P (ops[1]))
19317 : {
19318 0 : rtx vec = gen_rtx_CONST_VECTOR (V2DImode, gen_rtvec_v (2, ops));
19319 0 : emit_move_insn (target, vec);
19320 0 : }
19321 60399 : else if (rtx_equal_p (ops[0], ops[1]))
19322 : {
19323 60 : rtx val = ops[0];
19324 60 : if (!REG_P (val) && !MEM_P (val))
19325 0 : val = force_reg (DImode, val);
19326 : /* TARGET_SSE has *vec_dupv2di. */
19327 60 : emit_move_insn (target, gen_rtx_VEC_DUPLICATE (V2DImode, val));
19328 : }
19329 : else
19330 : {
19331 60339 : rtx op0 = force_reg (DImode, ops[0]);
19332 60339 : rtx op1 = force_reg (DImode, ops[1]);
19333 60339 : emit_insn (gen_vec_concatv2di (target, op0, op1));
19334 : }
19335 60401 : }
19336 :
19337 : /* A subroutine of ix86_expand_vector_init for V2DFmode. */
19338 :
19339 : static void
19340 4053 : ix86_expand_vector_init_v2df (rtx target, rtx *ops)
19341 : {
19342 4053 : if (ops[0] == CONST0_RTX (DFmode)
19343 0 : && ops[1] == CONST0_RTX (DFmode))
19344 0 : emit_move_insn (target, CONST0_RTX (V2DFmode));
19345 4053 : else if (CONST_DOUBLE_P (ops[0])
19346 32 : && CONST_DOUBLE_P (ops[1]))
19347 : {
19348 0 : rtx vec = gen_rtx_CONST_VECTOR (V2DFmode, gen_rtvec_v (2, ops));
19349 0 : emit_move_insn (target, vec);
19350 0 : }
19351 4053 : else if (TARGET_SSE2
19352 4053 : && rtx_equal_p (ops[0], ops[1]))
19353 : {
19354 102 : rtx val = ops[0];
19355 102 : if (!REG_P (val) && !MEM_P (val))
19356 0 : val = force_reg (DFmode, val);
19357 102 : emit_move_insn (target, gen_rtx_VEC_DUPLICATE (V2DFmode, val));
19358 : }
19359 : else
19360 : {
19361 3951 : rtx op0 = force_reg (DFmode, ops[0]);
19362 3951 : rtx op1 = force_reg (DFmode, ops[1]);
19363 3951 : emit_insn (gen_vec_concatv2df (target, op0, op1));
19364 : }
19365 4053 : }
19366 :
19367 : /* A subroutine of ix86_expand_vector_init for V4SImode. */
19368 :
19369 : static void
19370 34786 : ix86_expand_vector_init_v4si (rtx target, rtx *ops)
19371 : {
19372 34786 : rtx vars[4];
19373 34786 : int perm[4];
19374 :
19375 34786 : if (ops[0] == const0_rtx
19376 335 : && ops[1] == const0_rtx
19377 172 : && ops[2] == const0_rtx
19378 7 : && ops[3] == const0_rtx)
19379 6 : emit_move_insn (target, CONST0_RTX (V4SImode));
19380 34780 : else if (ops[1] == const0_rtx
19381 19725 : && ops[2] == const0_rtx
19382 19500 : && ops[3] == const0_rtx)
19383 : {
19384 19477 : rtx val = ops[0];
19385 19477 : if (!REG_P (val) && !MEM_P (val))
19386 2915 : val = force_reg (SImode, val);
19387 19477 : emit_insn (gen_vec_setv4si_0 (target, CONST0_RTX (V4SImode), val));
19388 19477 : }
19389 15303 : else if (rtx_equal_p (ops[0], ops[1])
19390 325 : && rtx_equal_p (ops[0], ops[2])
19391 15308 : && rtx_equal_p (ops[0], ops[3]))
19392 : {
19393 2 : rtx val = ops[0];
19394 2 : if (!REG_P (val) && !MEM_P (val))
19395 0 : val = force_reg (SImode, val);
19396 2 : emit_move_insn (target, gen_rtx_VEC_DUPLICATE (V4SImode, val));
19397 : }
19398 15301 : else if (CONST_INT_P (ops[0])
19399 442 : && CONST_INT_P (ops[1])
19400 181 : && CONST_INT_P (ops[2])
19401 1 : && CONST_INT_P (ops[3]))
19402 : {
19403 1 : rtx vec = gen_rtx_CONST_VECTOR (V4SImode, gen_rtvec_v (4, ops));
19404 1 : emit_move_insn (target, vec);
19405 1 : }
19406 15300 : else if (onevar_perm_p (ops, 4, perm, vars))
19407 : {
19408 186 : rtx tmp = gen_reg_rtx (V4SImode);
19409 186 : vars[1] = const0_rtx;
19410 186 : vars[2] = const0_rtx;
19411 186 : vars[3] = const0_rtx;
19412 186 : ix86_expand_vector_init_v4si (tmp, vars);
19413 186 : emit_insn (gen_sse_shufps_v4si (target, tmp, tmp,
19414 186 : GEN_INT (perm[0]),
19415 186 : GEN_INT (perm[1]),
19416 186 : GEN_INT (perm[2] + 4),
19417 186 : GEN_INT (perm[3] + 4)));
19418 : }
19419 15114 : else if (ops[2] == const0_rtx && ops[3] == const0_rtx)
19420 : {
19421 9644 : rtx tmp1 = gen_reg_rtx (V4SImode);
19422 9644 : vars[0] = ops[0];
19423 9644 : vars[1] = const0_rtx;
19424 9644 : vars[2] = const0_rtx;
19425 9644 : vars[3] = const0_rtx;
19426 9644 : ix86_expand_vector_init_v4si (tmp1, vars);
19427 :
19428 9644 : if (TARGET_SSE4_1)
19429 : {
19430 72 : rtx val = ops[1];
19431 72 : if (!REG_P (val) && !MEM_P (val))
19432 2 : val = force_reg (SImode, val);
19433 72 : emit_insn (gen_sse4_1_pinsrd (target, tmp1, val, GEN_INT (2)));
19434 : }
19435 : else
19436 : {
19437 9572 : rtx tmp2 = gen_reg_rtx (V4SImode);
19438 9572 : vars[0] = ops[1];
19439 9572 : ix86_expand_vector_init_v4si (tmp2, vars);
19440 9572 : emit_insn (gen_vec_interleave_lowv4si (target, tmp1, tmp2));
19441 : }
19442 : }
19443 5470 : else if (TARGET_SSE4_1
19444 360 : && ops[1] == const0_rtx
19445 4 : && (ops[2] == const0_rtx || ops[3] == const0_rtx))
19446 : /* { a, 0, b, 0 } and { a, 0, 0, b } become mov; pinsrd. */
19447 4 : ix86_expand_vector_init_insert (V4SImode, target, ops, 4);
19448 5466 : else if (twovar_perm_p (ops, 4, perm, vars))
19449 : {
19450 482 : rtx tmp = gen_reg_rtx (V4SImode);
19451 482 : vars[2] = const0_rtx;
19452 482 : vars[3] = const0_rtx;
19453 482 : ix86_expand_vector_init_v4si (tmp, vars);
19454 482 : emit_insn (gen_sse_shufps_v4si (target, tmp, tmp,
19455 482 : GEN_INT (perm[0]),
19456 482 : GEN_INT (perm[1]),
19457 482 : GEN_INT (perm[2] + 4),
19458 482 : GEN_INT (perm[3] + 4)));
19459 : }
19460 4984 : else if (nonzero_int_const_count (ops, 4) >= 2)
19461 : {
19462 : rtx csts[4];
19463 : int i;
19464 445 : for (i = 0; i < 4; i++)
19465 356 : if (CONST_INT_P (ops[i]))
19466 : {
19467 178 : csts[i] = ops[i];
19468 178 : vars[i] = const0_rtx;
19469 : }
19470 : else
19471 : {
19472 178 : csts[i] = const0_rtx;
19473 178 : vars[i] = ops[i];
19474 : }
19475 89 : rtx tmp1 = gen_reg_rtx (V4SImode);
19476 89 : ix86_expand_vector_init_v4si (tmp1, vars);
19477 89 : rtx tmp2 = gen_reg_rtx (V4SImode);
19478 89 : rtx vec = gen_rtx_CONST_VECTOR (V4SImode, gen_rtvec_v (4, csts));
19479 89 : emit_move_insn (tmp2, vec);
19480 89 : emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V4SImode, tmp1, tmp2)));
19481 : }
19482 4895 : else if (TARGET_SSE4_1)
19483 318 : ix86_expand_vector_init_insert (V4SImode, target, ops, 4);
19484 : else
19485 : {
19486 4577 : rtx tmp1 = gen_reg_rtx (V4SImode);
19487 4577 : rtx tmp2 = gen_reg_rtx (V4SImode);
19488 4577 : vars[0] = ops[0];
19489 4577 : vars[1] = ops[1];
19490 4577 : vars[2] = const0_rtx;
19491 4577 : vars[3] = const0_rtx;
19492 4577 : ix86_expand_vector_init_v4si (tmp1, vars);
19493 4577 : vars[0] = ops[2];
19494 4577 : vars[1] = ops[3];
19495 4577 : ix86_expand_vector_init_v4si (tmp2, vars);
19496 4577 : emit_insn (gen_sse_shufps_v4si (target, tmp1, tmp2,
19497 : const0_rtx, const1_rtx,
19498 : GEN_INT (4), GEN_INT (5)));
19499 : }
19500 34786 : }
19501 :
19502 : /* A subroutine of ix86_expand_vector_init for V4SFmode. */
19503 :
19504 : static void
19505 6440 : ix86_expand_vector_init_v4sf (rtx target, rtx *ops)
19506 : {
19507 6440 : rtx vars[4];
19508 6440 : int perm[4];
19509 :
19510 6440 : if (ops[0] == CONST0_RTX (SFmode)
19511 21 : && ops[1] == CONST0_RTX (SFmode)
19512 1 : && ops[2] == CONST0_RTX (SFmode)
19513 0 : && ops[3] == CONST0_RTX (SFmode))
19514 0 : emit_move_insn (target, CONST0_RTX (V4SFmode));
19515 6440 : else if (ops[1] == CONST0_RTX (SFmode)
19516 2696 : && ops[2] == CONST0_RTX (SFmode)
19517 2586 : && ops[3] == CONST0_RTX (SFmode))
19518 2586 : ix86_expand_vector_init_one_nonzero (false, V4SFmode, target, ops[0], 0);
19519 3854 : else if (rtx_equal_p (ops[0], ops[1])
19520 180 : && rtx_equal_p (ops[0], ops[2])
19521 3917 : && rtx_equal_p (ops[0], ops[3]))
19522 : {
19523 56 : rtx val = ops[0];
19524 56 : if (!REG_P (val) && !MEM_P (val))
19525 0 : val = force_reg (SFmode, val);
19526 56 : emit_move_insn (target, gen_rtx_VEC_DUPLICATE (V4SFmode, val));
19527 : }
19528 3798 : else if (CONST_DOUBLE_P (ops[0])
19529 29 : && CONST_DOUBLE_P (ops[1])
19530 2 : && CONST_DOUBLE_P (ops[2])
19531 0 : && CONST_DOUBLE_P (ops[3]))
19532 : {
19533 0 : rtx vec = gen_rtx_CONST_VECTOR (V4SFmode, gen_rtvec_v (4, ops));
19534 0 : emit_move_insn (target, vec);
19535 0 : }
19536 3798 : else if (ops[0] == CONST0_RTX (SFmode)
19537 21 : && ops[2] == CONST0_RTX (SFmode)
19538 16 : && ops[3] == CONST0_RTX (SFmode))
19539 0 : ix86_expand_vector_init_one_nonzero (false, V4SFmode, target, ops[1], 1);
19540 3798 : else if (ops[0] == CONST0_RTX (SFmode)
19541 21 : && ops[1] == CONST0_RTX (SFmode)
19542 1 : && ops[3] == CONST0_RTX (SFmode))
19543 0 : ix86_expand_vector_init_one_nonzero (false, V4SFmode, target, ops[2], 2);
19544 3798 : else if (ops[0] == CONST0_RTX (SFmode)
19545 21 : && ops[1] == CONST0_RTX (SFmode)
19546 1 : && ops[2] == CONST0_RTX (SFmode))
19547 0 : ix86_expand_vector_init_one_nonzero (false, V4SFmode, target, ops[3], 3);
19548 3798 : else if (onevar_perm_p (ops, 4, perm, vars))
19549 : {
19550 54 : rtx tmp = gen_reg_rtx (V4SFmode);
19551 54 : if (TARGET_SSE2 && !TARGET_SSE4_1 && REG_P (ops[0]))
19552 : {
19553 25 : emit_insn (gen_sse2_insertps_v4sf_3 (tmp, ops[0],
19554 : CONST0_RTX (V4SFmode)));
19555 100 : emit_insn (gen_sse_shufps_v4sf (target, tmp, tmp,
19556 25 : GEN_INT (perm[0] ? 0 : 3),
19557 25 : GEN_INT (perm[1] ? 0 : 3),
19558 25 : GEN_INT (perm[2] ? 4 : 7),
19559 25 : GEN_INT (perm[3] ? 4 : 7)));
19560 : }
19561 : else
19562 : {
19563 29 : vars[1] = CONST0_RTX (SFmode);
19564 29 : vars[2] = CONST0_RTX (SFmode);
19565 29 : vars[3] = CONST0_RTX (SFmode);
19566 29 : ix86_expand_vector_init_v4sf (tmp, vars);
19567 29 : emit_insn (gen_sse_shufps_v4sf (target, tmp, tmp,
19568 29 : GEN_INT (perm[0]),
19569 29 : GEN_INT (perm[1]),
19570 29 : GEN_INT (perm[2] + 4),
19571 29 : GEN_INT (perm[3] + 4)));
19572 : }
19573 : }
19574 3744 : else if (ops[2] == CONST0_RTX (SFmode)
19575 1841 : && ops[3] == CONST0_RTX (SFmode))
19576 : {
19577 1839 : if (TARGET_SSE4_1)
19578 : {
19579 442 : rtx tmp = gen_reg_rtx (V4SFmode);
19580 442 : rtx val = ops[0];
19581 442 : if (!REG_P (val) && !MEM_P (val))
19582 0 : val = force_reg (SFmode, val);
19583 442 : emit_insn (gen_vec_setv4sf_0 (tmp, CONST0_RTX (V4SFmode), val));
19584 442 : val = ops[1];
19585 442 : if (!REG_P (val) && !MEM_P (val))
19586 0 : val = force_reg (SFmode, val);
19587 442 : emit_insn (gen_vec_setv4sf_sse4_1 (target, tmp, val, GEN_INT (2)));
19588 : }
19589 1397 : else if (MEM_P (ops[0]) && MEM_P (ops[1]))
19590 : {
19591 1278 : rtx tmp1 = gen_reg_rtx (V4SFmode);
19592 1278 : vars[0] = ops[0];
19593 1278 : vars[1] = CONST0_RTX (SFmode);
19594 1278 : vars[2] = CONST0_RTX (SFmode);
19595 1278 : vars[3] = CONST0_RTX (SFmode);
19596 1278 : ix86_expand_vector_init_v4sf (tmp1, vars);
19597 :
19598 1278 : rtx tmp2 = gen_reg_rtx (V4SFmode);
19599 1278 : vars[0] = ops[1];
19600 1278 : ix86_expand_vector_init_v4sf (tmp2, vars);
19601 1278 : emit_insn (gen_vec_interleave_lowv4sf (target, tmp1, tmp2));
19602 1278 : }
19603 119 : else if (TARGET_SSE2)
19604 : {
19605 115 : rtx tmp = gen_reg_rtx (V2SFmode);
19606 115 : ix86_expand_vector_init_concat (V2SFmode, tmp, ops, 2);
19607 115 : tmp = gen_rtx_VEC_CONCAT (V4SFmode, tmp, CONST0_RTX (V2SFmode));
19608 115 : emit_insn (gen_rtx_SET (target, tmp));
19609 : }
19610 : else
19611 : {
19612 4 : rtx tmp1 = gen_reg_rtx (V2SFmode);
19613 4 : ix86_expand_vector_init_concat (V2SFmode, tmp1, ops, 2);
19614 4 : rtx tmp2 = gen_reg_rtx (V2SFmode);
19615 : /* Without both MMX and TARGET_MMX_WITH_SSE there's no movv2sf,
19616 : go through emit_move_insn so the zero can be materialized in
19617 : the corresponding integer mode. */
19618 4 : emit_move_insn (tmp2, CONST0_RTX (V2SFmode));
19619 4 : rtx tmp = gen_rtx_VEC_CONCAT (V4SFmode, tmp1, tmp2);
19620 4 : emit_insn (gen_rtx_SET (target, tmp));
19621 : }
19622 : }
19623 1905 : else if (MEM_P (ops[0])
19624 791 : && ops[1] == CONST0_RTX (SFmode)
19625 0 : && MEM_P (ops[2])
19626 0 : && ops[3] == CONST0_RTX (SFmode))
19627 : {
19628 0 : rtx tmp1 = gen_reg_rtx (V4SFmode);
19629 0 : ix86_expand_vector_init_one_nonzero (false, V4SFmode, tmp1, ops[0], 0);
19630 0 : rtx tmp2 = gen_reg_rtx (V4SFmode);
19631 0 : ix86_expand_vector_init_one_nonzero (false, V4SFmode, tmp2, ops[2], 0);
19632 0 : emit_insn (gen_sse_movlhps (target, tmp1, tmp2));
19633 0 : }
19634 1905 : else if (twovar_perm_p (ops, 4, perm, vars))
19635 : {
19636 251 : rtx tmp = gen_reg_rtx (V4SFmode);
19637 251 : vars[2] = CONST0_RTX (SFmode);
19638 251 : vars[3] = CONST0_RTX (SFmode);
19639 251 : ix86_expand_vector_init_v4sf (tmp, vars);
19640 251 : emit_insn (gen_sse_shufps_v4sf (target, tmp, tmp,
19641 251 : GEN_INT (perm[0]),
19642 251 : GEN_INT (perm[1]),
19643 251 : GEN_INT (perm[2] + 4),
19644 251 : GEN_INT (perm[3] + 4)));
19645 : }
19646 1654 : else if (nonzero_float_const_count (ops, 4) >= 2)
19647 : {
19648 : rtx csts[4];
19649 : int i;
19650 10 : for (i = 0; i < 4; i++)
19651 8 : if (CONST_DOUBLE_P (ops[i]))
19652 : {
19653 4 : csts[i] = ops[i];
19654 4 : vars[i] = CONST0_RTX (SFmode);
19655 : }
19656 : else
19657 : {
19658 4 : csts[i] = CONST0_RTX (SFmode);
19659 4 : vars[i] = ops[i];
19660 : }
19661 2 : rtx tmp1 = gen_reg_rtx (V4SFmode);
19662 2 : ix86_expand_vector_init_v4sf (tmp1, vars);
19663 2 : rtx tmp2 = gen_reg_rtx (V4SFmode);
19664 2 : rtx vec = gen_rtx_CONST_VECTOR (V4SFmode, gen_rtvec_v (4, csts));
19665 2 : emit_move_insn (tmp2, vec);
19666 2 : emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V4SFmode, tmp1, tmp2)));
19667 : }
19668 1652 : else if (TARGET_SSE4_1 && optimize_insn_for_size_p ())
19669 : {
19670 1 : int i;
19671 1 : rtx tmp = gen_reg_rtx (V4SFmode);
19672 1 : bool first_p = true;
19673 6 : for (i = 0; i < 4; i++)
19674 4 : if (ops[i] != CONST0_RTX (SFmode))
19675 : {
19676 4 : rtx idx = GEN_INT (1 << i);
19677 4 : rtx val = ops[i];
19678 4 : if (!REG_P (val) && !MEM_P (val))
19679 0 : val = force_reg (SFmode, val);
19680 4 : rtx pat;
19681 4 : if (!first_p)
19682 3 : pat = gen_vec_setv4sf_sse4_1 (tmp, tmp, val, idx);
19683 1 : else if (i == 0)
19684 1 : pat = gen_vec_setv4sf_0 (tmp, CONST0_RTX (V4SFmode), val);
19685 : else
19686 0 : pat = gen_sse4_1_insertps_v4sf_init (tmp, val,
19687 : CONST0_RTX (V4SFmode), idx);
19688 4 : emit_insn (pat);
19689 4 : first_p = false;
19690 : }
19691 1 : emit_move_insn (target, tmp);
19692 : }
19693 1651 : else if (MEM_P (ops[0])
19694 790 : && MEM_P (ops[1])
19695 789 : && MEM_P (ops[2])
19696 788 : && MEM_P (ops[3]))
19697 : {
19698 788 : rtx tmp1 = gen_reg_rtx (V4SFmode);
19699 788 : rtx tmp2 = gen_reg_rtx (V4SFmode);
19700 788 : vars[0] = ops[0];
19701 788 : vars[1] = ops[1];
19702 788 : vars[2] = CONST0_RTX (SFmode);
19703 788 : vars[3] = CONST0_RTX (SFmode);
19704 788 : ix86_expand_vector_init_v4sf (tmp1, vars);
19705 788 : vars[0] = ops[2];
19706 788 : vars[1] = ops[3];
19707 788 : ix86_expand_vector_init_v4sf (tmp2, vars);
19708 788 : emit_insn (gen_sse_shufps_v4sf (target, tmp1, tmp2,
19709 : const0_rtx, const1_rtx,
19710 : GEN_INT (4), GEN_INT (5)));
19711 788 : }
19712 : else
19713 863 : ix86_expand_vector_init_concat (V4SFmode, target, ops, 4);
19714 6440 : }
19715 :
19716 : /* A subroutine of ix86_expand_vector_init for V8HImode. */
19717 :
19718 : static bool
19719 557 : ix86_expand_vector_init_v8hi (rtx target, rtx *ops)
19720 : {
19721 557 : rtx vars[8];
19722 557 : int i;
19723 :
19724 557 : bool all_zero_p = true;
19725 561 : for (i = 0; i < 8; i++)
19726 561 : if (ops[i] != const0_rtx)
19727 : {
19728 : all_zero_p = false;
19729 : break;
19730 : }
19731 557 : if (all_zero_p)
19732 : {
19733 0 : emit_move_insn (target, CONST0_RTX (V8HImode));
19734 0 : return true;
19735 : }
19736 :
19737 566 : bool all_const_p = true;
19738 566 : for (i = 0; i < 8; i++)
19739 566 : if (!CONST_INT_P (ops[i]))
19740 : {
19741 : all_const_p = false;
19742 : break;
19743 : }
19744 557 : if (all_const_p)
19745 : {
19746 0 : rtx vec = gen_rtx_CONST_VECTOR (V8HImode, gen_rtvec_v (8, ops));
19747 0 : emit_move_insn (target, vec);
19748 0 : return true;
19749 : }
19750 :
19751 557 : if (TARGET_SSE2
19752 557 : && nonzero_int_const_count (ops, 8) >= 2)
19753 : {
19754 : rtx csts[8];
19755 0 : for (i = 0; i < 8; i++)
19756 0 : if (CONST_INT_P (ops[i]))
19757 : {
19758 0 : csts[i] = ops[i];
19759 0 : vars[i] = const0_rtx;
19760 : }
19761 : else
19762 : {
19763 0 : csts[i] = const0_rtx;
19764 0 : vars[i] = ops[i];
19765 : }
19766 0 : rtx tmp1 = gen_reg_rtx (V8HImode);
19767 0 : if (!ix86_expand_vector_init_v8hi (tmp1, vars))
19768 0 : gcc_unreachable ();
19769 0 : rtx tmp2 = gen_reg_rtx (V8HImode);
19770 0 : rtx vec = gen_rtx_CONST_VECTOR (V8HImode, gen_rtvec_v (8, csts));
19771 0 : emit_move_insn (tmp2, vec);
19772 0 : emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V8HImode, tmp1, tmp2)));
19773 0 : return true;
19774 : }
19775 :
19776 557 : if (TARGET_SSE2)
19777 : {
19778 557 : ix86_expand_vector_init_insert (V8HImode, target, ops, 8);
19779 557 : return true;
19780 : }
19781 : return false;
19782 : }
19783 :
19784 : /* A subroutine of ix86_expand_vector_init_v16qi.
19785 : Place OPS[0..3] in an SImode REG or CONST_INT. */
19786 :
19787 : static rtx
19788 0 : ix86_expand_vector_init_qi4 (rtx *ops)
19789 : {
19790 0 : rtx vars[4];
19791 0 : rtx word;
19792 0 : int i;
19793 :
19794 0 : if (CONST_INT_P (ops[0])
19795 0 : && CONST_INT_P (ops[1])
19796 0 : && CONST_INT_P (ops[2])
19797 0 : && CONST_INT_P (ops[3]))
19798 : {
19799 0 : HOST_WIDE_INT val = (UINTVAL (ops[0]) & 0xff)
19800 0 : + ((UINTVAL (ops[1]) & 0xff) << 8)
19801 0 : + ((UINTVAL (ops[2]) & 0xff) << 16)
19802 0 : + ((UINTVAL (ops[3]) & 0xff) << 24);
19803 0 : return gen_int_mode (val, SImode);
19804 : }
19805 :
19806 0 : if (ops[1] == const0_rtx
19807 0 : && ops[2] == const0_rtx
19808 0 : && ops[3] == const0_rtx)
19809 0 : return convert_modes (SImode, QImode, ops[0], true);
19810 :
19811 :
19812 0 : if (rtx_equal_p (ops[0], ops[2])
19813 0 : && rtx_equal_p (ops[1], ops[3]))
19814 : {
19815 0 : vars[0] = ops[0];
19816 0 : vars[1] = ops[1];
19817 0 : vars[2] = const0_rtx;
19818 0 : vars[3] = const0_rtx;
19819 0 : rtx tmp1 = ix86_expand_vector_init_qi4 (vars);
19820 0 : rtx tmp2 = expand_simple_binop (SImode, ASHIFT, tmp1, GEN_INT (16),
19821 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
19822 0 : return expand_simple_binop (SImode, PLUS, tmp2, tmp1,
19823 0 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
19824 : }
19825 :
19826 : HOST_WIDE_INT val = 0;
19827 0 : for (i = 0; i < 4; i++)
19828 0 : if (CONST_INT_P (ops[i]) && ops[i] != const0_rtx)
19829 : {
19830 0 : val += (UINTVAL (ops[i]) & 0xff) << (i * 8);
19831 0 : vars[i] = const0_rtx;
19832 : }
19833 : else
19834 0 : vars[i] = ops[i];
19835 :
19836 0 : if (val != 0)
19837 : {
19838 0 : rtx tmp = ix86_expand_vector_init_qi4 (vars);
19839 0 : return expand_simple_binop (SImode, IOR, tmp,
19840 0 : gen_int_mode (val, SImode),
19841 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
19842 : }
19843 :
19844 0 : if (ops[0] == const0_rtx && ops[1] == const0_rtx)
19845 : {
19846 0 : if (ops[2] == const0_rtx)
19847 : {
19848 0 : word = convert_modes (SImode, QImode, ops[3], true);
19849 0 : return expand_simple_binop (SImode, ASHIFT, word, GEN_INT (24),
19850 0 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
19851 : }
19852 0 : vars[0] = ops[2];
19853 0 : vars[1] = ops[3];
19854 0 : vars[2] = const0_rtx;
19855 0 : vars[3] = const0_rtx;
19856 0 : word = ix86_expand_vector_init_qi4 (vars);
19857 0 : return expand_simple_binop (SImode, ASHIFT, word, GEN_INT (16),
19858 0 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
19859 : }
19860 :
19861 0 : vars[0] = ops[1];
19862 0 : vars[1] = ops[2];
19863 0 : vars[2] = ops[3];
19864 0 : vars[3] = const0_rtx;
19865 0 : word = ix86_expand_vector_init_qi4 (vars);
19866 0 : word = expand_simple_binop (SImode, ASHIFT, word, GEN_INT (8),
19867 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
19868 0 : if (ops[0] != const0_rtx)
19869 : {
19870 0 : rtx elt = convert_modes (SImode, QImode, ops[0], true);
19871 0 : word = expand_simple_binop (SImode, PLUS, word, elt,
19872 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
19873 : }
19874 : return word;
19875 : }
19876 :
19877 : /* A subroutine of ix86_expand_vector_init_v16qi.
19878 : Place OPS[0..7] in an DImode REG or CONST_INT. */
19879 :
19880 : static rtx
19881 4513 : ix86_expand_vector_init_qi8 (rtx *ops)
19882 : {
19883 4513 : rtx vars[8];
19884 4513 : rtx word;
19885 4513 : int i;
19886 :
19887 4513 : if (CONST_INT_P (ops[0])
19888 34 : && CONST_INT_P (ops[1])
19889 12 : && CONST_INT_P (ops[2])
19890 12 : && CONST_INT_P (ops[3])
19891 12 : && CONST_INT_P (ops[4])
19892 12 : && CONST_INT_P (ops[5])
19893 12 : && CONST_INT_P (ops[6])
19894 12 : && CONST_INT_P (ops[7]))
19895 : {
19896 12 : HOST_WIDE_INT val = (UINTVAL (ops[0]) & 0xff)
19897 12 : + ((UINTVAL (ops[1]) & 0xff) << 8)
19898 12 : + ((UINTVAL (ops[2]) & 0xff) << 16)
19899 12 : + ((UINTVAL (ops[3]) & 0xff) << 24)
19900 12 : + ((UINTVAL (ops[4]) & 0xff) << 32)
19901 12 : + ((UINTVAL (ops[5]) & 0xff) << 40)
19902 12 : + ((UINTVAL (ops[6]) & 0xff) << 48)
19903 12 : + ((UINTVAL (ops[7]) & 0xff) << 56);
19904 12 : return gen_int_mode (val, DImode);
19905 : }
19906 :
19907 4501 : if (ops[1] == const0_rtx
19908 640 : && ops[2] == const0_rtx
19909 627 : && ops[3] == const0_rtx
19910 627 : && ops[4] == const0_rtx
19911 626 : && ops[5] == const0_rtx
19912 626 : && ops[6] == const0_rtx
19913 626 : && ops[7] == const0_rtx)
19914 626 : return convert_modes (DImode, QImode, ops[0], true);
19915 :
19916 3875 : if (rtx_equal_p (ops[0], ops[4])
19917 62 : && rtx_equal_p (ops[1], ops[5])
19918 42 : && rtx_equal_p (ops[2], ops[6])
19919 3911 : && rtx_equal_p (ops[3], ops[7]))
19920 : {
19921 30 : vars[0] = ops[0];
19922 30 : vars[1] = ops[1];
19923 30 : vars[2] = ops[2];
19924 30 : vars[3] = ops[3];
19925 30 : vars[4] = const0_rtx;
19926 30 : vars[5] = const0_rtx;
19927 30 : vars[6] = const0_rtx;
19928 30 : vars[7] = const0_rtx;
19929 30 : rtx tmp1 = ix86_expand_vector_init_qi8 (vars);
19930 30 : rtx tmp2 = expand_simple_binop (DImode, ASHIFT, tmp1, GEN_INT (32),
19931 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
19932 30 : return expand_simple_binop (DImode, PLUS, tmp2, tmp1,
19933 30 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
19934 : }
19935 :
19936 : HOST_WIDE_INT val = 0;
19937 34605 : for (i = 0; i < 8; i++)
19938 30760 : if (CONST_INT_P (ops[i]) && ops[i] != const0_rtx)
19939 : {
19940 380 : val += (UINTVAL (ops[i]) & 0xff) << (i * 8);
19941 380 : vars[i] = const0_rtx;
19942 : }
19943 : else
19944 30380 : vars[i] = ops[i];
19945 :
19946 3845 : if (val != 0)
19947 : {
19948 58 : rtx tmp = ix86_expand_vector_init_qi8 (vars);
19949 58 : return expand_simple_binop (DImode, IOR, tmp,
19950 58 : gen_int_mode (val, DImode),
19951 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
19952 : }
19953 :
19954 3787 : if (rtx_equal_p (ops[0], ops[2])
19955 76 : && rtx_equal_p (ops[1], ops[3])
19956 69 : && ops[4] == const0_rtx
19957 46 : && ops[5] == const0_rtx
19958 46 : && ops[6] == const0_rtx
19959 3833 : && ops[7] == const0_rtx)
19960 : {
19961 46 : vars[0] = ops[0];
19962 46 : vars[1] = ops[1];
19963 46 : vars[2] = const0_rtx;
19964 46 : vars[3] = const0_rtx;
19965 46 : vars[4] = const0_rtx;
19966 46 : vars[5] = const0_rtx;
19967 46 : vars[6] = const0_rtx;
19968 46 : vars[7] = const0_rtx;
19969 46 : rtx tmp1 = ix86_expand_vector_init_qi8 (vars);
19970 46 : rtx tmp2 = expand_simple_binop (DImode, ASHIFT, tmp1, GEN_INT (16),
19971 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
19972 46 : return expand_simple_binop (DImode, PLUS, tmp2, tmp1,
19973 46 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
19974 : }
19975 :
19976 3741 : if (ops[0] == const0_rtx && ops[1] == const0_rtx)
19977 : {
19978 : i = 2;
19979 0 : while (ops[i] == const0_rtx)
19980 0 : i++;
19981 0 : vars[0] = ops[i];
19982 0 : vars[1] = i + 1 < 8 ? ops[i + 1] : const0_rtx;
19983 0 : vars[2] = i + 2 < 8 ? ops[i + 2] : const0_rtx;
19984 0 : vars[3] = i + 3 < 8 ? ops[i + 3] : const0_rtx;
19985 0 : vars[4] = i + 4 < 8 ? ops[i + 4] : const0_rtx;
19986 0 : vars[5] = i + 5 < 8 ? ops[i + 5] : const0_rtx;
19987 0 : vars[6] = const0_rtx;
19988 0 : vars[7] = const0_rtx;
19989 0 : word = ix86_expand_vector_init_qi8 (vars);
19990 0 : return expand_simple_binop (DImode, ASHIFT, word, GEN_INT (i * 8),
19991 0 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
19992 : }
19993 :
19994 3741 : vars[0] = ops[1];
19995 3741 : vars[1] = ops[2];
19996 3741 : vars[2] = ops[3];
19997 3741 : vars[3] = ops[4];
19998 3741 : vars[4] = ops[5];
19999 3741 : vars[5] = ops[6];
20000 3741 : vars[6] = ops[7];
20001 3741 : vars[7] = const0_rtx;
20002 3741 : word = ix86_expand_vector_init_qi8 (vars);
20003 3741 : word = expand_simple_binop (DImode, ASHIFT, word, GEN_INT (8),
20004 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
20005 3741 : if (ops[0] != const0_rtx)
20006 : {
20007 3721 : rtx elt = convert_modes (DImode, QImode, ops[0], true);
20008 3721 : word = expand_simple_binop (DImode, PLUS, word, elt,
20009 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
20010 : }
20011 : return word;
20012 : }
20013 :
20014 : /* A subroutine of ix86_expand_vector_init for V16QImode. */
20015 :
20016 : static void
20017 445 : ix86_expand_vector_init_v16qi (rtx target, rtx *ops)
20018 : {
20019 445 : rtx vars[16];
20020 445 : bool ok;
20021 445 : int i;
20022 :
20023 445 : bool all_zero_p = true;
20024 450 : for (i = 0; i < 16; i++)
20025 450 : if (ops[i] != const0_rtx)
20026 : {
20027 : all_zero_p = false;
20028 : break;
20029 : }
20030 445 : if (all_zero_p)
20031 : {
20032 0 : emit_move_insn (target, CONST0_RTX (V16QImode));
20033 1 : return;
20034 : }
20035 :
20036 578 : bool all_same_p = true;
20037 578 : for (i = 1; i < 16; i++)
20038 578 : if (!rtx_equal_p (ops[i], ops[0]))
20039 : {
20040 : all_same_p = false;
20041 : break;
20042 : }
20043 445 : if (all_same_p)
20044 : {
20045 0 : ok = ix86_expand_vector_init_duplicate (false, V16QImode,
20046 : target, ops[0]);
20047 0 : gcc_assert (ok);
20048 : return;
20049 : }
20050 :
20051 : /* abababababababab */
20052 445 : if (TARGET_AVX2
20053 72 : && rtx_equal_p (ops[0], ops[2])
20054 4 : && rtx_equal_p (ops[1], ops[3])
20055 4 : && rtx_equal_p (ops[0], ops[4])
20056 2 : && rtx_equal_p (ops[1], ops[5])
20057 2 : && rtx_equal_p (ops[0], ops[6])
20058 2 : && rtx_equal_p (ops[1], ops[7])
20059 2 : && rtx_equal_p (ops[0], ops[8])
20060 1 : && rtx_equal_p (ops[1], ops[9])
20061 1 : && rtx_equal_p (ops[0], ops[10])
20062 1 : && rtx_equal_p (ops[1], ops[11])
20063 1 : && rtx_equal_p (ops[0], ops[12])
20064 1 : && rtx_equal_p (ops[1], ops[13])
20065 1 : && rtx_equal_p (ops[0], ops[14])
20066 446 : && rtx_equal_p (ops[1], ops[15]))
20067 : {
20068 1 : rtx tmp1 = gen_reg_rtx (V16QImode);
20069 1 : if (REG_P (ops[0]))
20070 : {
20071 1 : rtx op0 = force_reg (SImode, gen_lowpart (SImode, ops[0]));
20072 1 : rtx tmp = gen_reg_rtx (V4SImode);
20073 1 : emit_insn (gen_vec_setv4si_0 (tmp, CONST0_RTX (V4SImode), op0));
20074 1 : emit_move_insn (tmp1, gen_lowpart (V16QImode, tmp));
20075 1 : rtx op1 = ops[1];
20076 1 : if (!REG_P (op1) && !MEM_P (op1))
20077 0 : op1 = force_reg (QImode, op1);
20078 1 : emit_insn (gen_sse4_1_pinsrb (tmp1, tmp1, op1, GEN_INT (2)));
20079 : }
20080 : else
20081 : {
20082 0 : vars[0] = ops[0];
20083 0 : vars[1] = ops[1];
20084 0 : for (i = 2; i < 16; i++)
20085 0 : vars[i] = const0_rtx;
20086 0 : ix86_expand_vector_init_v16qi (tmp1, vars);
20087 : }
20088 1 : tmp1 = gen_lowpart (V8HImode, tmp1);
20089 1 : rtx tmp2 = gen_reg_rtx (V8HImode);
20090 1 : emit_insn (gen_avx2_pbroadcastv8hi (tmp2, tmp1));
20091 1 : emit_move_insn (target, gen_lowpart (V16QImode, tmp2));
20092 1 : return;
20093 : }
20094 :
20095 451 : bool all_const_p = true;
20096 451 : for (i = 0; i < 16; i++)
20097 451 : if (!CONST_INT_P (ops[i]))
20098 : {
20099 : all_const_p = false;
20100 : break;
20101 : }
20102 444 : if (all_const_p)
20103 : {
20104 : int last_nonzero = 15;
20105 0 : while (ops[last_nonzero] == const0_rtx)
20106 0 : last_nonzero--;
20107 0 : if (last_nonzero < 4)
20108 : {
20109 0 : HOST_WIDE_INT val = (UINTVAL (ops[0]) & 0xff)
20110 0 : + ((UINTVAL (ops[1]) & 0xff) << 8)
20111 0 : + ((UINTVAL (ops[2]) & 0xff) << 16)
20112 0 : + ((UINTVAL (ops[3]) & 0xff) << 24);
20113 0 : rtx tmp1 = force_reg (SImode, gen_int_mode (val, SImode));
20114 0 : rtx tmp2 = gen_reg_rtx (V4SImode);
20115 0 : emit_insn (gen_vec_setv4si_0 (tmp2, CONST0_RTX (V4SImode), tmp1));
20116 0 : emit_move_insn (target, gen_lowpart (V16QImode, tmp2));
20117 0 : return;
20118 : }
20119 0 : rtx vec = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, ops));
20120 0 : emit_move_insn (target, vec);
20121 0 : return;
20122 : }
20123 :
20124 444 : if (TARGET_SSE4_1
20125 444 : && nonzero_int_const_count (ops, 16) >= 2)
20126 : {
20127 : rtx csts[16];
20128 : int count = 0;
20129 0 : for (i = 0; i < 16; i++)
20130 0 : if (CONST_INT_P (ops[i]))
20131 : {
20132 0 : csts[i] = ops[i];
20133 0 : vars[i] = const0_rtx;
20134 : }
20135 : else
20136 : {
20137 0 : csts[i] = const0_rtx;
20138 0 : vars[i] = ops[i];
20139 0 : count++;
20140 : }
20141 0 : rtx tmp1 = gen_reg_rtx (V16QImode);
20142 0 : if (count <= 3)
20143 : {
20144 0 : ix86_expand_vector_init_v16qi (tmp1, csts);
20145 0 : for (i = 0; i < 16; i++)
20146 0 : if (vars[i] != const0_rtx)
20147 : {
20148 0 : rtx elt = vars[i];
20149 0 : if (!REG_P (elt) && !MEM_P (elt))
20150 0 : elt = force_reg (QImode, elt);
20151 0 : emit_insn (gen_sse4_1_pinsrb (tmp1, tmp1, elt,
20152 0 : GEN_INT (1 << i)));
20153 : }
20154 0 : emit_move_insn (target, tmp1);
20155 0 : return;
20156 : }
20157 0 : ix86_expand_vector_init_v16qi (tmp1, vars);
20158 0 : rtx tmp2 = gen_reg_rtx (V16QImode);
20159 0 : rtx vec = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, csts));
20160 0 : emit_move_insn (tmp2, vec);
20161 0 : emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V16QImode, tmp1, tmp2)));
20162 : }
20163 444 : else if (TARGET_SSE4_1)
20164 98 : ix86_expand_vector_init_insert (V16QImode, target, ops, 16);
20165 346 : else if (TARGET_64BIT)
20166 : {
20167 346 : rtx tmp = gen_reg_rtx (V2DImode);
20168 346 : vars[0] = ix86_expand_vector_init_qi8 (ops);
20169 346 : if (rtx_equal_p (ops[0], ops[8])
20170 57 : && rtx_equal_p (ops[1], ops[9])
20171 54 : && rtx_equal_p (ops[2], ops[10])
20172 54 : && rtx_equal_p (ops[3], ops[11])
20173 54 : && rtx_equal_p (ops[4], ops[12])
20174 54 : && rtx_equal_p (ops[5], ops[13])
20175 54 : && rtx_equal_p (ops[6], ops[14])
20176 400 : && rtx_equal_p (ops[7], ops[15]))
20177 54 : vars[1] = vars[0];
20178 : else
20179 292 : vars[1] = ix86_expand_vector_init_qi8 (ops + 8);
20180 346 : ix86_expand_vector_init_v2di (tmp, vars);
20181 346 : emit_move_insn (target, gen_lowpart (V16QImode, tmp));
20182 : }
20183 : else
20184 : {
20185 0 : rtx tmp = gen_reg_rtx (V4SImode);
20186 0 : vars[0] = ix86_expand_vector_init_qi4 (ops);
20187 0 : if (rtx_equal_p (ops[0], ops[4])
20188 0 : && rtx_equal_p (ops[1], ops[5])
20189 0 : && rtx_equal_p (ops[2], ops[6])
20190 0 : && rtx_equal_p (ops[3], ops[7]))
20191 0 : vars[1] = vars[0];
20192 : else
20193 0 : vars[1] = ix86_expand_vector_init_qi4 (ops + 4);
20194 0 : if (rtx_equal_p (ops[0], ops[8])
20195 0 : && rtx_equal_p (ops[1], ops[9])
20196 0 : && rtx_equal_p (ops[2], ops[10])
20197 0 : && rtx_equal_p (ops[3], ops[11]))
20198 0 : vars[2] = vars[0];
20199 0 : else if (rtx_equal_p (ops[4], ops[8])
20200 0 : && rtx_equal_p (ops[5], ops[9])
20201 0 : && rtx_equal_p (ops[6], ops[10])
20202 0 : && rtx_equal_p (ops[7], ops[11]))
20203 0 : vars[2] = vars[1];
20204 : else
20205 0 : vars[2] = ix86_expand_vector_init_qi4 (ops + 8);
20206 0 : if (rtx_equal_p (ops[0], ops[12])
20207 0 : && rtx_equal_p (ops[1], ops[13])
20208 0 : && rtx_equal_p (ops[2], ops[14])
20209 0 : && rtx_equal_p (ops[3], ops[15]))
20210 0 : vars[3] = vars[0];
20211 0 : else if (rtx_equal_p (ops[4], ops[12])
20212 0 : && rtx_equal_p (ops[5], ops[13])
20213 0 : && rtx_equal_p (ops[6], ops[14])
20214 0 : && rtx_equal_p (ops[7], ops[15]))
20215 0 : vars[3] = vars[1];
20216 0 : else if (rtx_equal_p (ops[8], ops[12])
20217 0 : && rtx_equal_p (ops[9], ops[13])
20218 0 : && rtx_equal_p (ops[10], ops[14])
20219 0 : && rtx_equal_p (ops[11], ops[15]))
20220 0 : vars[3] = vars[2];
20221 : else
20222 0 : vars[3] = ix86_expand_vector_init_qi4 (ops + 12);
20223 0 : ix86_expand_vector_init_v4si (tmp, vars);
20224 0 : emit_move_insn (target, gen_lowpart (V16QImode, tmp));
20225 : }
20226 : }
20227 :
20228 : /* A subroutine of ix86_expand_vector_init for V4DImode. */
20229 :
20230 : static void
20231 145 : ix86_expand_vector_init_v4di (rtx target, rtx *ops)
20232 : {
20233 145 : rtx vars[4];
20234 145 : int perm[4];
20235 :
20236 145 : if (ops[0] == const0_rtx
20237 0 : && ops[1] == const0_rtx
20238 0 : && ops[2] == const0_rtx
20239 0 : && ops[3] == const0_rtx)
20240 0 : emit_move_insn (target, CONST0_RTX (V4DImode));
20241 145 : else if (ops[1] == const0_rtx
20242 8 : && ops[2] == const0_rtx
20243 0 : && ops[3] == const0_rtx)
20244 : {
20245 0 : rtx val = ops[0];
20246 0 : if (!REG_P (val) && !MEM_P (val))
20247 0 : val = force_reg (DImode, val);
20248 0 : emit_insn (gen_vec_setv4di_0 (target, CONST0_RTX (V4DImode), val));
20249 0 : }
20250 0 : else if ((TARGET_64BIT || MEM_P (ops[0]))
20251 145 : && rtx_equal_p (ops[0], ops[1])
20252 0 : && rtx_equal_p (ops[0], ops[2])
20253 145 : && rtx_equal_p (ops[0], ops[3]))
20254 : {
20255 0 : rtx val = ops[0];
20256 0 : if (!REG_P (val) && !MEM_P (val))
20257 0 : val = force_reg (DImode, val);
20258 0 : emit_insn (gen_vec_dupv4di (target, val));
20259 : }
20260 145 : else if (CONST_INT_P (ops[0])
20261 2 : && CONST_INT_P (ops[1])
20262 0 : && CONST_INT_P (ops[2])
20263 0 : && CONST_INT_P (ops[3]))
20264 : {
20265 0 : rtx vec = gen_rtx_CONST_VECTOR (V4DImode, gen_rtvec_v (4, ops));
20266 0 : emit_move_insn (target, vec);
20267 0 : }
20268 145 : else if (TARGET_AVX2
20269 145 : && onevar_perm_p (ops, 4, perm, vars))
20270 : {
20271 0 : rtx tmp = gen_reg_rtx (V4DImode);
20272 0 : vars[1] = const0_rtx;
20273 0 : vars[2] = const0_rtx;
20274 0 : vars[3] = const0_rtx;
20275 0 : ix86_expand_vector_init_v4di (tmp, vars);
20276 0 : emit_insn (gen_avx2_permv4di_1 (target, tmp,
20277 0 : GEN_INT (perm[0]),
20278 0 : GEN_INT (perm[1]),
20279 0 : GEN_INT (perm[2]),
20280 0 : GEN_INT (perm[3])));
20281 : }
20282 145 : else if (rtx_equal_p (ops[0], ops[2])
20283 145 : && rtx_equal_p (ops[1], ops[3]))
20284 : {
20285 2 : rtx tmp = gen_reg_rtx (V2DImode);
20286 2 : ix86_expand_vector_init_v2di (tmp, ops);
20287 2 : emit_insn (gen_avx_vec_concatv4di (target, tmp, tmp));
20288 : }
20289 143 : else if (nonzero_int_const_count (ops, 4) >= 2)
20290 : {
20291 : rtx csts[4];
20292 : int i;
20293 0 : for (i = 0; i < 4; i++)
20294 0 : if (CONST_INT_P (ops[i]))
20295 : {
20296 0 : csts[i] = ops[i];
20297 0 : vars[i] = const0_rtx;
20298 : }
20299 : else
20300 : {
20301 0 : csts[i] = const0_rtx;
20302 0 : vars[i] = ops[i];
20303 : }
20304 0 : rtx tmp1 = gen_reg_rtx (V4DImode);
20305 0 : ix86_expand_vector_init_v4di (tmp1, vars);
20306 0 : rtx tmp2 = gen_reg_rtx (V4DImode);
20307 0 : rtx vec = gen_rtx_CONST_VECTOR (V4DImode, gen_rtvec_v (4, csts));
20308 0 : emit_move_insn (tmp2, vec);
20309 0 : emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V4DImode, tmp1, tmp2)));
20310 : }
20311 : else
20312 : {
20313 143 : rtx tmp1 = gen_reg_rtx (V2DImode);
20314 143 : ix86_expand_vector_init_v2di (tmp1, ops);
20315 143 : rtx tmp2 = gen_reg_rtx (V2DImode);
20316 143 : ix86_expand_vector_init_v2di (tmp2, ops + 2);
20317 143 : emit_insn (gen_avx_vec_concatv4di (target, tmp1, tmp2));
20318 : }
20319 145 : }
20320 :
20321 : /* A subroutine of ix86_expand_vector_init for V4DFmode. */
20322 :
20323 : static void
20324 511 : ix86_expand_vector_init_v4df (rtx target, rtx *ops)
20325 : {
20326 511 : rtx vars[4];
20327 511 : int perm[4];
20328 :
20329 511 : if (ops[0] == CONST0_RTX (DFmode)
20330 0 : && ops[1] == CONST0_RTX (DFmode)
20331 0 : && ops[2] == CONST0_RTX (DFmode)
20332 0 : && ops[3] == CONST0_RTX (DFmode))
20333 0 : emit_move_insn (target, CONST0_RTX (V4DFmode));
20334 511 : else if (ops[1] == CONST0_RTX (DFmode)
20335 0 : && ops[2] == CONST0_RTX (DFmode)
20336 0 : && ops[3] == CONST0_RTX (DFmode))
20337 : {
20338 0 : rtx val = ops[0];
20339 0 : if (!REG_P (val) && !MEM_P (val))
20340 0 : val = force_reg (DFmode, val);
20341 0 : emit_insn (gen_vec_setv4df_0 (target, CONST0_RTX (V4DFmode), val));
20342 0 : }
20343 511 : else if (rtx_equal_p (ops[0], ops[1])
20344 53 : && rtx_equal_p (ops[0], ops[2])
20345 515 : && rtx_equal_p (ops[0], ops[3]))
20346 : {
20347 0 : rtx val = ops[0];
20348 0 : if (!REG_P (val) && !MEM_P (val))
20349 0 : val = force_reg (DFmode, val);
20350 0 : emit_insn (gen_vec_dupv4df (target, val));
20351 : }
20352 511 : else if (CONST_DOUBLE_P (ops[0])
20353 4 : && CONST_DOUBLE_P (ops[1])
20354 0 : && CONST_DOUBLE_P (ops[2])
20355 0 : && CONST_DOUBLE_P (ops[3]))
20356 : {
20357 0 : rtx vec = gen_rtx_CONST_VECTOR (V4DFmode, gen_rtvec_v (4, ops));
20358 0 : emit_move_insn (target, vec);
20359 0 : }
20360 511 : else if (TARGET_AVX2
20361 511 : && onevar_perm_p (ops, 4, perm, vars))
20362 : {
20363 0 : rtx tmp = gen_reg_rtx (V4DFmode);
20364 0 : vars[1] = CONST0_RTX (DFmode);
20365 0 : vars[2] = CONST0_RTX (DFmode);
20366 0 : vars[3] = CONST0_RTX (DFmode);
20367 0 : ix86_expand_vector_init_v4df (tmp, vars);
20368 0 : emit_insn (gen_avx2_permv4df_1 (target, tmp,
20369 0 : GEN_INT (perm[0]),
20370 0 : GEN_INT (perm[1]),
20371 0 : GEN_INT (perm[2]),
20372 0 : GEN_INT (perm[3])));
20373 : }
20374 511 : else if (rtx_equal_p (ops[0], ops[2])
20375 511 : && rtx_equal_p (ops[1], ops[3]))
20376 : {
20377 16 : rtx tmp = gen_reg_rtx (V2DFmode);
20378 16 : ix86_expand_vector_init_v2df (tmp, ops);
20379 16 : emit_insn (gen_avx_vec_concatv4df (target, tmp, tmp));
20380 : }
20381 495 : else if (nonzero_double_const_count (ops, 4) >= 2)
20382 : {
20383 : rtx csts[4];
20384 : int i;
20385 0 : for (i = 0; i < 4; i++)
20386 0 : if (CONST_DOUBLE_P (ops[i]))
20387 : {
20388 0 : csts[i] = ops[i];
20389 0 : vars[i] = CONST0_RTX (DFmode);
20390 : }
20391 : else
20392 : {
20393 0 : csts[i] = CONST0_RTX (DFmode);
20394 0 : vars[i] = ops[i];
20395 : }
20396 0 : rtx tmp1 = gen_reg_rtx (V4DFmode);
20397 0 : ix86_expand_vector_init_v4df (tmp1, vars);
20398 0 : rtx tmp2 = gen_reg_rtx (V4DFmode);
20399 0 : rtx vec = gen_rtx_CONST_VECTOR (V4DFmode, gen_rtvec_v (4, csts));
20400 0 : emit_move_insn (tmp2, vec);
20401 0 : emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V4DFmode, tmp1, tmp2)));
20402 : }
20403 : else
20404 : {
20405 495 : rtx tmp1 = gen_reg_rtx (V2DFmode);
20406 495 : ix86_expand_vector_init_v2df (tmp1, ops);
20407 495 : rtx tmp2 = gen_reg_rtx (V2DFmode);
20408 495 : ix86_expand_vector_init_v2df (tmp2, ops + 2);
20409 495 : emit_insn (gen_avx_vec_concatv4df (target, tmp1, tmp2));
20410 : }
20411 511 : }
20412 :
20413 : /* A subroutine of ix86_expand_vector_init for V8SImode. */
20414 :
20415 : static void
20416 135 : ix86_expand_vector_init_v8si (rtx target, rtx *ops)
20417 : {
20418 135 : rtx vars[8];
20419 :
20420 135 : if (ops[0] == const0_rtx
20421 7 : && ops[1] == const0_rtx
20422 7 : && ops[2] == const0_rtx
20423 7 : && ops[3] == const0_rtx
20424 6 : && ops[4] == const0_rtx
20425 3 : && ops[5] == const0_rtx
20426 0 : && ops[6] == const0_rtx
20427 0 : && ops[7] == const0_rtx)
20428 0 : emit_move_insn (target, CONST0_RTX (V8SImode));
20429 135 : else if (ops[1] == const0_rtx
20430 7 : && ops[2] == const0_rtx
20431 7 : && ops[3] == const0_rtx
20432 6 : && ops[4] == const0_rtx
20433 3 : && ops[5] == const0_rtx
20434 0 : && ops[6] == const0_rtx
20435 0 : && ops[7] == const0_rtx)
20436 : {
20437 0 : rtx val = ops[0];
20438 0 : if (!REG_P (val) && !MEM_P (val))
20439 0 : val = force_reg (SImode, val);
20440 0 : emit_insn (gen_vec_setv8si_0 (target, CONST0_RTX (V8SImode), val));
20441 0 : }
20442 135 : else if (rtx_equal_p (ops[0], ops[1])
20443 10 : && rtx_equal_p (ops[0], ops[2])
20444 8 : && rtx_equal_p (ops[0], ops[3])
20445 7 : && rtx_equal_p (ops[0], ops[4])
20446 3 : && rtx_equal_p (ops[0], ops[5])
20447 0 : && rtx_equal_p (ops[0], ops[6])
20448 135 : && rtx_equal_p (ops[0], ops[7]))
20449 : {
20450 0 : rtx val = ops[0];
20451 0 : if (!REG_P (val) && !MEM_P (val))
20452 0 : val = force_reg (SImode, val);
20453 0 : emit_insn (gen_vec_dupv8si (target, val));
20454 : }
20455 135 : else if (TARGET_AVX2
20456 122 : && rtx_equal_p (ops[0], ops[2])
20457 32 : && rtx_equal_p (ops[0], ops[4])
20458 28 : && rtx_equal_p (ops[0], ops[6])
20459 24 : && rtx_equal_p (ops[1], ops[3])
20460 24 : && rtx_equal_p (ops[1], ops[5])
20461 159 : && rtx_equal_p (ops[1], ops[7]))
20462 : {
20463 24 : rtx tmp_ops[4] = { ops[0], ops[1], const0_rtx, const0_rtx };
20464 24 : rtx tmp1 = gen_reg_rtx (V4SImode);
20465 24 : ix86_expand_vector_init_v4si (tmp1, tmp_ops);
20466 24 : tmp1 = gen_lowpart (V2DImode, tmp1);
20467 24 : rtx tmp2 = gen_reg_rtx (V4DImode);
20468 24 : emit_insn (gen_avx2_pbroadcastv4di (tmp2, tmp1));
20469 24 : emit_move_insn (target, gen_lowpart (V8SImode, tmp2));
20470 : }
20471 111 : else if (ops[4] == const0_rtx
20472 9 : && ops[5] == const0_rtx
20473 5 : && ops[6] == const0_rtx
20474 5 : && ops[7] == const0_rtx)
20475 : {
20476 5 : rtx tmp = gen_reg_rtx (V4SImode);
20477 5 : ix86_expand_vector_init_v4si (tmp, ops);
20478 5 : emit_insn (gen_avx_vec_concatv8si (target, tmp, CONST0_RTX (V4SImode)));
20479 5 : }
20480 106 : else if (CONST_INT_P (ops[0])
20481 9 : && CONST_INT_P (ops[1])
20482 7 : && CONST_INT_P (ops[2])
20483 7 : && CONST_INT_P (ops[3])
20484 7 : && CONST_INT_P (ops[4])
20485 4 : && CONST_INT_P (ops[5])
20486 0 : && CONST_INT_P (ops[6])
20487 0 : && CONST_INT_P (ops[7]))
20488 : {
20489 0 : rtx vec = gen_rtx_CONST_VECTOR (V8SImode, gen_rtvec_v (8, ops));
20490 0 : emit_move_insn (target, vec);
20491 0 : }
20492 106 : else if (rtx_equal_p (ops[0], ops[4])
20493 26 : && rtx_equal_p (ops[1], ops[5])
20494 22 : && rtx_equal_p (ops[2], ops[6])
20495 128 : && rtx_equal_p (ops[3], ops[7]))
20496 : {
20497 22 : rtx tmp = gen_reg_rtx (V4SImode);
20498 22 : ix86_expand_vector_init_v4si (tmp, ops);
20499 22 : emit_insn (gen_avx_vec_concatv8si (target, tmp, tmp));
20500 : }
20501 84 : else if (nonzero_int_const_count (ops, 8) >= 2)
20502 : {
20503 : rtx csts[8];
20504 : int i;
20505 9 : for (i = 0; i < 8; i++)
20506 8 : if (CONST_INT_P (ops[i]))
20507 : {
20508 6 : csts[i] = ops[i];
20509 6 : vars[i] = const0_rtx;
20510 : }
20511 : else
20512 : {
20513 2 : csts[i] = const0_rtx;
20514 2 : vars[i] = ops[i];
20515 : }
20516 1 : rtx tmp1 = gen_reg_rtx (V8SImode);
20517 1 : ix86_expand_vector_init_v8si (tmp1, vars);
20518 1 : rtx tmp2 = gen_reg_rtx (V8SImode);
20519 1 : rtx vec = gen_rtx_CONST_VECTOR (V8SImode, gen_rtvec_v (8, csts));
20520 1 : emit_move_insn (tmp2, vec);
20521 1 : emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V8SImode, tmp1, tmp2)));
20522 : }
20523 : else
20524 : {
20525 83 : rtx tmp1 = gen_reg_rtx (V4SImode);
20526 83 : ix86_expand_vector_init_v4si (tmp1, ops);
20527 83 : rtx tmp2 = gen_reg_rtx (V4SImode);
20528 83 : ix86_expand_vector_init_v4si (tmp2, ops + 4);
20529 83 : emit_insn (gen_avx_vec_concatv8si (target, tmp1, tmp2));
20530 : }
20531 135 : }
20532 :
20533 : /* A subroutine of ix86_expand_vector_init for V8SFmode. */
20534 :
20535 : static void
20536 249 : ix86_expand_vector_init_v8sf (rtx target, rtx *ops)
20537 : {
20538 249 : rtx vars[8];
20539 :
20540 249 : if (ops[0] == CONST0_RTX (SFmode)
20541 0 : && ops[1] == CONST0_RTX (SFmode)
20542 0 : && ops[2] == CONST0_RTX (SFmode)
20543 0 : && ops[3] == CONST0_RTX (SFmode)
20544 0 : && ops[4] == CONST0_RTX (SFmode)
20545 0 : && ops[5] == CONST0_RTX (SFmode)
20546 0 : && ops[6] == CONST0_RTX (SFmode)
20547 0 : && ops[7] == CONST0_RTX (SFmode))
20548 0 : emit_move_insn (target, CONST0_RTX (V8SFmode));
20549 249 : else if (ops[1] == CONST0_RTX (SFmode)
20550 24 : && ops[2] == CONST0_RTX (SFmode)
20551 0 : && ops[3] == CONST0_RTX (SFmode)
20552 0 : && ops[4] == CONST0_RTX (SFmode)
20553 0 : && ops[5] == CONST0_RTX (SFmode)
20554 0 : && ops[6] == CONST0_RTX (SFmode)
20555 0 : && ops[7] == CONST0_RTX (SFmode))
20556 : {
20557 0 : rtx val = ops[0];
20558 0 : if (!REG_P (val) && !MEM_P (val))
20559 0 : val = force_reg (SFmode, val);
20560 0 : emit_insn (gen_vec_setv8sf_0 (target, CONST0_RTX (V8SFmode), val));
20561 0 : }
20562 249 : else if (TARGET_AVX2
20563 175 : && rtx_equal_p (ops[0], ops[1])
20564 32 : && rtx_equal_p (ops[0], ops[2])
20565 0 : && rtx_equal_p (ops[0], ops[3])
20566 0 : && rtx_equal_p (ops[0], ops[4])
20567 0 : && rtx_equal_p (ops[0], ops[5])
20568 0 : && rtx_equal_p (ops[0], ops[6])
20569 249 : && rtx_equal_p (ops[0], ops[7]))
20570 : {
20571 0 : rtx val = ops[0];
20572 0 : if (!REG_P (val) && !MEM_P (val))
20573 0 : val = force_reg (SFmode, val);
20574 0 : emit_insn (gen_avx2_vec_dupv8sf_1 (target, val));
20575 : }
20576 249 : else if (TARGET_AVX2
20577 175 : && rtx_equal_p (ops[0], ops[2])
20578 32 : && rtx_equal_p (ops[0], ops[4])
20579 32 : && rtx_equal_p (ops[0], ops[6])
20580 32 : && rtx_equal_p (ops[1], ops[3])
20581 0 : && rtx_equal_p (ops[1], ops[5])
20582 249 : && rtx_equal_p (ops[1], ops[7]))
20583 : {
20584 0 : rtx tmp_ops[4] = { ops[0], ops[1], CONST0_RTX (SFmode),
20585 0 : CONST0_RTX (SFmode) };
20586 0 : rtx tmp1 = gen_reg_rtx (V4SFmode);
20587 0 : ix86_expand_vector_init_v4sf (tmp1, tmp_ops);
20588 0 : tmp1 = gen_lowpart (V2DImode, tmp1);
20589 0 : rtx tmp2 = gen_reg_rtx (V4DImode);
20590 0 : emit_insn (gen_avx2_pbroadcastv4di (tmp2, tmp1));
20591 0 : emit_move_insn (target, gen_lowpart (V8SFmode, tmp2));
20592 : }
20593 249 : else if (ops[4] == CONST0_RTX (SFmode)
20594 0 : && ops[5] == CONST0_RTX (SFmode)
20595 0 : && ops[6] == CONST0_RTX (SFmode)
20596 0 : && ops[7] == CONST0_RTX (SFmode))
20597 : {
20598 0 : rtx tmp = gen_reg_rtx (V4SFmode);
20599 0 : ix86_expand_vector_init_v4sf (tmp, ops);
20600 0 : emit_insn (gen_avx_vec_concatv8sf (target, tmp, CONST0_RTX (V4SFmode)));
20601 0 : }
20602 249 : else if (CONST_DOUBLE_P (ops[0])
20603 2 : && CONST_DOUBLE_P (ops[1])
20604 0 : && CONST_DOUBLE_P (ops[2])
20605 0 : && CONST_DOUBLE_P (ops[3])
20606 0 : && CONST_DOUBLE_P (ops[4])
20607 0 : && CONST_DOUBLE_P (ops[5])
20608 0 : && CONST_DOUBLE_P (ops[6])
20609 0 : && CONST_DOUBLE_P (ops[7]))
20610 : {
20611 0 : rtx vec = gen_rtx_CONST_VECTOR (V8SFmode, gen_rtvec_v (8, ops));
20612 0 : emit_move_insn (target, vec);
20613 0 : }
20614 249 : else if (rtx_equal_p (ops[0], ops[4])
20615 78 : && rtx_equal_p (ops[1], ops[5])
20616 22 : && rtx_equal_p (ops[2], ops[6])
20617 271 : && rtx_equal_p (ops[3], ops[7]))
20618 : {
20619 22 : rtx tmp = gen_reg_rtx (V4SFmode);
20620 22 : ix86_expand_vector_init_v4sf (tmp, ops);
20621 22 : emit_insn (gen_avx_vec_concatv8sf (target, tmp, tmp));
20622 : }
20623 227 : else if (nonzero_float_const_count (ops, 8) >= 2)
20624 : {
20625 : rtx csts[8];
20626 : int i;
20627 0 : for (i = 0; i < 8; i++)
20628 0 : if (CONST_DOUBLE_P (ops[i]))
20629 : {
20630 0 : csts[i] = ops[i];
20631 0 : vars[i] = CONST0_RTX (SFmode);
20632 : }
20633 : else
20634 : {
20635 0 : csts[i] = CONST0_RTX (SFmode);
20636 0 : vars[i] = ops[i];
20637 : }
20638 0 : rtx tmp1 = gen_reg_rtx (V8SFmode);
20639 0 : ix86_expand_vector_init_v8sf (tmp1, vars);
20640 0 : rtx tmp2 = gen_reg_rtx (V8SFmode);
20641 0 : rtx vec = gen_rtx_CONST_VECTOR (V8SFmode, gen_rtvec_v (8, csts));
20642 0 : emit_move_insn (tmp2, vec);
20643 0 : emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V8SFmode, tmp1, tmp2)));
20644 : }
20645 : else
20646 : {
20647 227 : rtx tmp1 = gen_reg_rtx (V4SFmode);
20648 227 : ix86_expand_vector_init_v4sf (tmp1, ops);
20649 227 : rtx tmp2 = gen_reg_rtx (V4SFmode);
20650 227 : ix86_expand_vector_init_v4sf (tmp2, ops + 4);
20651 227 : emit_insn (gen_avx_vec_concatv8sf (target, tmp1, tmp2));
20652 : }
20653 249 : }
20654 :
20655 :
20656 : /* A subroutine of ix86_expand_vector_init. Handle the most general case:
20657 : all values variable, and none identical. */
20658 :
20659 : static void
20660 103040 : ix86_expand_vector_init_general (bool mmx_ok, machine_mode mode,
20661 : rtx target, rtx vals)
20662 : {
20663 103040 : rtx ops[64], op0, op1, op2, op3, op4, op5;
20664 103040 : machine_mode half_mode = VOIDmode;
20665 103040 : machine_mode quarter_mode = VOIDmode;
20666 103040 : machine_mode int_inner_mode = VOIDmode;
20667 103040 : int n, i;
20668 :
20669 103040 : switch (mode)
20670 : {
20671 59767 : case E_V2DImode:
20672 59767 : ops[0] = XVECEXP (vals, 0, 0);
20673 59767 : ops[1] = XVECEXP (vals, 0, 1);
20674 59767 : ix86_expand_vector_init_v2di (target, ops);
20675 154530 : return;
20676 :
20677 3047 : case E_V2DFmode:
20678 3047 : ops[0] = XVECEXP (vals, 0, 0);
20679 3047 : ops[1] = XVECEXP (vals, 0, 1);
20680 3047 : ix86_expand_vector_init_v2df (target, ops);
20681 3047 : return;
20682 :
20683 : case E_V4SImode:
20684 27210 : for (i = 0; i < 4; i++)
20685 21768 : ops[i] = XVECEXP (vals, 0, i);
20686 5442 : ix86_expand_vector_init_v4si (target, ops);
20687 5442 : return;
20688 :
20689 : case E_V4SFmode:
20690 7750 : for (i = 0; i < 4; i++)
20691 6200 : ops[i] = XVECEXP (vals, 0, i);
20692 1550 : ix86_expand_vector_init_v4sf (target, ops);
20693 1550 : return;
20694 :
20695 : case E_V8HImode:
20696 5013 : for (i = 0; i < 8; i++)
20697 4456 : ops[i] = XVECEXP (vals, 0, i);
20698 557 : if (ix86_expand_vector_init_v8hi (target, ops))
20699 : return;
20700 : break;
20701 :
20702 : case E_V16QImode:
20703 7565 : for (i = 0; i < 16; i++)
20704 7120 : ops[i] = XVECEXP (vals, 0, i);
20705 445 : ix86_expand_vector_init_v16qi (target, ops);
20706 445 : return;
20707 :
20708 : case E_V4DImode:
20709 725 : for (i = 0; i < 4; i++)
20710 580 : ops[i] = XVECEXP (vals, 0, i);
20711 145 : ix86_expand_vector_init_v4di (target,ops);
20712 145 : return;
20713 :
20714 : case E_V4DFmode:
20715 2555 : for (i = 0; i < 4; i++)
20716 2044 : ops[i] = XVECEXP (vals, 0, i);
20717 511 : ix86_expand_vector_init_v4df (target,ops);
20718 511 : return;
20719 :
20720 : case E_V8SImode:
20721 1206 : for (i = 0; i < 8; i++)
20722 1072 : ops[i] = XVECEXP (vals, 0, i);
20723 134 : ix86_expand_vector_init_v8si (target,ops);
20724 134 : return;
20725 :
20726 : case E_V8SFmode:
20727 2241 : for (i = 0; i < 8; i++)
20728 1992 : ops[i] = XVECEXP (vals, 0, i);
20729 249 : ix86_expand_vector_init_v8sf (target,ops);
20730 249 : return;
20731 :
20732 21028 : case E_V2SFmode:
20733 21028 : case E_V2SImode:
20734 21028 : if (!mmx_ok && !TARGET_SSE)
20735 : break;
20736 : /* FALLTHRU */
20737 :
20738 21204 : case E_V16SImode:
20739 21204 : case E_V16SFmode:
20740 21204 : case E_V8DFmode:
20741 21204 : case E_V8DImode:
20742 21204 : n = GET_MODE_NUNITS (mode);
20743 65236 : for (i = 0; i < n; i++)
20744 44032 : ops[i] = XVECEXP (vals, 0, i);
20745 21204 : ix86_expand_vector_init_concat (mode, target, ops, n);
20746 21204 : return;
20747 :
20748 : case E_V2TImode:
20749 132 : for (i = 0; i < 2; i++)
20750 88 : ops[i] = gen_lowpart (V2DImode, XVECEXP (vals, 0, i));
20751 44 : op0 = gen_reg_rtx (V4DImode);
20752 44 : ix86_expand_vector_init_concat (V4DImode, op0, ops, 2);
20753 44 : emit_move_insn (target, gen_lowpart (GET_MODE (target), op0));
20754 44 : return;
20755 :
20756 : case E_V4TImode:
20757 195 : for (i = 0; i < 4; i++)
20758 156 : ops[i] = gen_lowpart (V2DImode, XVECEXP (vals, 0, i));
20759 39 : ops[4] = gen_reg_rtx (V4DImode);
20760 39 : ix86_expand_vector_init_concat (V4DImode, ops[4], ops, 2);
20761 39 : ops[5] = gen_reg_rtx (V4DImode);
20762 39 : ix86_expand_vector_init_concat (V4DImode, ops[5], ops + 2, 2);
20763 39 : op0 = gen_reg_rtx (V8DImode);
20764 39 : ix86_expand_vector_init_concat (V8DImode, op0, ops + 4, 2);
20765 39 : emit_move_insn (target, gen_lowpart (GET_MODE (target), op0));
20766 39 : return;
20767 :
20768 69 : case E_V32QImode:
20769 69 : half_mode = V16QImode;
20770 69 : goto half;
20771 :
20772 64 : case E_V16HImode:
20773 64 : half_mode = V8HImode;
20774 64 : goto half;
20775 :
20776 237 : case E_V16HFmode:
20777 237 : half_mode = V8HFmode;
20778 237 : goto half;
20779 :
20780 95 : case E_V16BFmode:
20781 95 : half_mode = V8BFmode;
20782 95 : goto half;
20783 :
20784 465 : half:
20785 465 : n = GET_MODE_NUNITS (mode);
20786 9009 : for (i = 0; i < n; i++)
20787 8544 : ops[i] = XVECEXP (vals, 0, i);
20788 465 : op0 = gen_reg_rtx (half_mode);
20789 465 : op1 = gen_reg_rtx (half_mode);
20790 465 : ix86_expand_vector_init_interleave (half_mode, op0, ops,
20791 : n >> 2);
20792 465 : ix86_expand_vector_init_interleave (half_mode, op1,
20793 465 : &ops [n >> 1], n >> 2);
20794 465 : emit_insn (gen_rtx_SET (target, gen_rtx_VEC_CONCAT (mode, op0, op1)));
20795 465 : return;
20796 :
20797 56 : case E_V64QImode:
20798 56 : quarter_mode = V16QImode;
20799 56 : half_mode = V32QImode;
20800 56 : goto quarter;
20801 :
20802 71 : case E_V32HImode:
20803 71 : quarter_mode = V8HImode;
20804 71 : half_mode = V16HImode;
20805 71 : goto quarter;
20806 :
20807 287 : case E_V32HFmode:
20808 287 : quarter_mode = V8HFmode;
20809 287 : half_mode = V16HFmode;
20810 287 : goto quarter;
20811 :
20812 51 : case E_V32BFmode:
20813 51 : quarter_mode = V8BFmode;
20814 51 : half_mode = V16BFmode;
20815 51 : goto quarter;
20816 :
20817 465 : quarter:
20818 465 : n = GET_MODE_NUNITS (mode);
20819 17137 : for (i = 0; i < n; i++)
20820 16672 : ops[i] = XVECEXP (vals, 0, i);
20821 465 : op0 = gen_reg_rtx (quarter_mode);
20822 465 : op1 = gen_reg_rtx (quarter_mode);
20823 465 : op2 = gen_reg_rtx (quarter_mode);
20824 465 : op3 = gen_reg_rtx (quarter_mode);
20825 465 : op4 = gen_reg_rtx (half_mode);
20826 465 : op5 = gen_reg_rtx (half_mode);
20827 465 : ix86_expand_vector_init_interleave (quarter_mode, op0, ops,
20828 : n >> 3);
20829 465 : ix86_expand_vector_init_interleave (quarter_mode, op1,
20830 465 : &ops [n >> 2], n >> 3);
20831 465 : ix86_expand_vector_init_interleave (quarter_mode, op2,
20832 465 : &ops [n >> 1], n >> 3);
20833 465 : ix86_expand_vector_init_interleave (quarter_mode, op3,
20834 465 : &ops [(n >> 1) | (n >> 2)], n >> 3);
20835 465 : emit_insn (gen_rtx_SET (op4, gen_rtx_VEC_CONCAT (half_mode, op0, op1)));
20836 465 : emit_insn (gen_rtx_SET (op5, gen_rtx_VEC_CONCAT (half_mode, op2, op3)));
20837 465 : emit_insn (gen_rtx_SET (target, gen_rtx_VEC_CONCAT (mode, op4, op5)));
20838 465 : return;
20839 :
20840 699 : case E_V8HFmode:
20841 699 : case E_V8BFmode:
20842 699 : n = GET_MODE_NUNITS (mode);
20843 6291 : for (i = 0; i < n; i++)
20844 5592 : ops[i] = XVECEXP (vals, 0, i);
20845 699 : ix86_expand_vector_init_interleave (mode, target, ops, n >> 1);
20846 699 : return;
20847 :
20848 : case E_V4HFmode:
20849 : case E_V4BFmode:
20850 : case E_V2HFmode:
20851 : case E_V2BFmode:
20852 8277 : int_inner_mode = HImode;
20853 : break;
20854 :
20855 : case E_V4HImode:
20856 : case E_V8QImode:
20857 :
20858 : case E_V2HImode:
20859 : case E_V4QImode:
20860 : break;
20861 :
20862 0 : default:
20863 0 : gcc_unreachable ();
20864 : }
20865 :
20866 8277 : {
20867 8277 : int i, j, n_elts, n_words, n_elt_per_word;
20868 8277 : machine_mode tmp_mode, inner_mode;
20869 8277 : rtx words[4], shift;
20870 :
20871 16625 : tmp_mode = (GET_MODE_SIZE (mode) < UNITS_PER_WORD) ? SImode : word_mode;
20872 :
20873 8277 : inner_mode = GET_MODE_INNER (mode);
20874 8277 : n_elts = GET_MODE_NUNITS (mode);
20875 16554 : n_words = GET_MODE_SIZE (mode) / GET_MODE_SIZE (tmp_mode);
20876 8277 : n_elt_per_word = n_elts / n_words;
20877 8277 : shift = GEN_INT (GET_MODE_BITSIZE (inner_mode));
20878 :
20879 24839 : for (i = 0; i < n_words; ++i)
20880 : {
20881 : rtx word = NULL_RTX;
20882 :
20883 41287 : for (j = 0; j < n_elt_per_word; ++j)
20884 : {
20885 33002 : rtx elt = XVECEXP (vals, 0, (i+1)*n_elt_per_word - j - 1);
20886 33002 : if (int_inner_mode != E_VOIDmode)
20887 : {
20888 328 : gcc_assert (TARGET_SSE2 && int_inner_mode == HImode);
20889 328 : rtx tmp = gen_reg_rtx (int_inner_mode);
20890 328 : elt = lowpart_subreg (int_inner_mode,
20891 : force_reg (inner_mode, elt),
20892 : inner_mode);
20893 328 : emit_move_insn (tmp, elt);
20894 328 : elt = tmp;
20895 : }
20896 33002 : elt = convert_modes (tmp_mode, inner_mode, elt, true);
20897 :
20898 33002 : if (j == 0)
20899 : word = elt;
20900 : else
20901 : {
20902 24717 : word = expand_simple_binop (tmp_mode, ASHIFT, word, shift,
20903 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
20904 24717 : word = expand_simple_binop (tmp_mode, IOR, word, elt,
20905 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
20906 : }
20907 : }
20908 :
20909 8285 : words[i] = word;
20910 : }
20911 :
20912 8277 : if (n_words == 1)
20913 8269 : emit_move_insn (target, gen_lowpart (mode, words[0]));
20914 8 : else if (n_words == 2)
20915 : {
20916 8 : gcc_assert (tmp_mode == DImode || tmp_mode == SImode);
20917 8 : machine_mode concat_mode = tmp_mode == DImode ? V2DImode : V2SImode;
20918 8 : rtx tmp = gen_reg_rtx (concat_mode);
20919 8 : vals = gen_rtx_PARALLEL (concat_mode, gen_rtvec_v (2, words));
20920 8 : ix86_expand_vector_init_general (mmx_ok, concat_mode, tmp, vals);
20921 8 : emit_move_insn (target, gen_lowpart (mode, tmp));
20922 : }
20923 0 : else if (n_words == 4)
20924 : {
20925 0 : rtx tmp = gen_reg_rtx (V4SImode);
20926 0 : gcc_assert (tmp_mode == SImode);
20927 0 : vals = gen_rtx_PARALLEL (V4SImode, gen_rtvec_v (4, words));
20928 0 : ix86_expand_vector_init_general (false, V4SImode, tmp, vals);
20929 0 : emit_move_insn (target, gen_lowpart (mode, tmp));
20930 : }
20931 : else
20932 0 : gcc_unreachable ();
20933 : }
20934 : }
20935 :
20936 : /* Initialize vector TARGET via VALS. Suppress the use of MMX
20937 : instructions unless MMX_OK is true. */
20938 :
20939 : void
20940 121237 : ix86_expand_vector_init (bool mmx_ok, rtx target, rtx vals)
20941 : {
20942 121237 : machine_mode mode = GET_MODE (target);
20943 121237 : machine_mode inner_mode = GET_MODE_INNER (mode);
20944 121237 : int n_elts = GET_MODE_NUNITS (mode);
20945 121237 : int n_var = 0, one_var = -1;
20946 121237 : bool all_same = true, all_const_zero = true;
20947 121237 : int i;
20948 121237 : rtx x;
20949 :
20950 : /* Handle first initialization from vector elts. */
20951 121237 : if (n_elts != XVECLEN (vals, 0))
20952 : {
20953 1411 : rtx subtarget = target;
20954 1411 : x = XVECEXP (vals, 0, 0);
20955 2822 : gcc_assert (GET_MODE_INNER (GET_MODE (x)) == inner_mode);
20956 2822 : if (GET_MODE_NUNITS (GET_MODE (x)) * 2 == n_elts)
20957 : {
20958 1411 : rtx ops[2] = { XVECEXP (vals, 0, 0), XVECEXP (vals, 0, 1) };
20959 1411 : if (inner_mode == QImode
20960 1411 : || inner_mode == HImode
20961 1411 : || inner_mode == TImode
20962 : || inner_mode == HFmode
20963 : || inner_mode == BFmode)
20964 : {
20965 197 : unsigned int n_bits = n_elts * GET_MODE_SIZE (inner_mode);
20966 197 : scalar_mode elt_mode = inner_mode == TImode ? DImode : SImode;
20967 197 : n_bits /= GET_MODE_SIZE (elt_mode);
20968 197 : mode = mode_for_vector (elt_mode, n_bits).require ();
20969 197 : inner_mode = mode_for_vector (elt_mode, n_bits / 2).require ();
20970 197 : ops[0] = gen_lowpart (inner_mode, ops[0]);
20971 197 : ops[1] = gen_lowpart (inner_mode, ops[1]);
20972 197 : subtarget = gen_reg_rtx (mode);
20973 : }
20974 1411 : ix86_expand_vector_init_concat (mode, subtarget, ops, 2);
20975 1411 : if (subtarget != target)
20976 197 : emit_move_insn (target, gen_lowpart (GET_MODE (target), subtarget));
20977 1411 : return;
20978 : }
20979 0 : gcc_unreachable ();
20980 : }
20981 :
20982 450124 : for (i = 0; i < n_elts; ++i)
20983 : {
20984 330298 : x = XVECEXP (vals, 0, i);
20985 639005 : if (!(CONST_SCALAR_INT_P (x)
20986 312638 : || CONST_DOUBLE_P (x)
20987 : || CONST_FIXED_P (x)))
20988 308707 : n_var++, one_var = i;
20989 21591 : else if (x != CONST0_RTX (inner_mode))
20990 3040 : all_const_zero = false;
20991 330298 : if (i > 0 && !rtx_equal_p (x, XVECEXP (vals, 0, 0)))
20992 : all_same = false;
20993 : }
20994 :
20995 : /* Handle the zero vector as special case. */
20996 119826 : if (n_var == 0 && all_const_zero)
20997 : {
20998 3 : emit_move_insn (target, CONST0_RTX (mode));
20999 3 : return;
21000 : }
21001 :
21002 : /* If all values are identical, broadcast the value. */
21003 119823 : if (all_same
21004 127139 : && ix86_expand_vector_init_duplicate (mmx_ok, mode, target,
21005 7316 : XVECEXP (vals, 0, 0)))
21006 : return;
21007 :
21008 : /* Constants are best loaded from the constant pool. */
21009 113346 : if (n_var == 0)
21010 : {
21011 14 : emit_move_insn (target, gen_rtx_CONST_VECTOR (mode, XVEC (vals, 0)));
21012 14 : return;
21013 : }
21014 :
21015 : /* Values where only one field is non-constant are best loaded from
21016 : the pool and overwritten via move later. */
21017 113332 : if (n_var == 1)
21018 : {
21019 11753 : if (all_const_zero
21020 22471 : && ix86_expand_vector_init_one_nonzero (mmx_ok, mode, target,
21021 10718 : XVECEXP (vals, 0, one_var),
21022 : one_var))
21023 : return;
21024 :
21025 1683 : if (ix86_expand_vector_init_one_var (mmx_ok, mode, target, vals, one_var))
21026 : return;
21027 : }
21028 :
21029 103032 : ix86_expand_vector_init_general (mmx_ok, mode, target, vals);
21030 : }
21031 :
21032 : /* Implemented as
21033 : V setg (V v, int idx, T val)
21034 : {
21035 : V idxv = (V){idx, idx, idx, idx, idx, idx, idx, idx};
21036 : V valv = (V){val, val, val, val, val, val, val, val};
21037 : V mask = ((V){0, 1, 2, 3, 4, 5, 6, 7} == idxv);
21038 : v = (v & ~mask) | (valv & mask);
21039 : return v;
21040 : }. */
21041 : void
21042 129 : ix86_expand_vector_set_var (rtx target, rtx val, rtx idx)
21043 : {
21044 129 : rtx vec[64];
21045 129 : machine_mode mode = GET_MODE (target);
21046 129 : machine_mode cmp_mode = mode;
21047 129 : int n_elts = GET_MODE_NUNITS (mode);
21048 129 : rtx valv,idxv,constv,idx_tmp;
21049 129 : bool ok = false;
21050 :
21051 258 : val = force_reg (GET_MODE_INNER (mode), val);
21052 :
21053 : /* 512-bits vector byte/word broadcast and comparison only available
21054 : under TARGET_AVX512BW, break 512-bits vector into two 256-bits vector
21055 : when without TARGET_AVX512BW. */
21056 129 : if ((mode == V32HImode || mode == V32HFmode || mode == V32BFmode
21057 123 : || mode == V64QImode)
21058 10 : && !TARGET_AVX512BW)
21059 : {
21060 3 : gcc_assert (TARGET_AVX512F);
21061 3 : rtx vhi, vlo, idx_hi;
21062 3 : machine_mode half_mode;
21063 3 : rtx (*extract_hi)(rtx, rtx);
21064 3 : rtx (*extract_lo)(rtx, rtx);
21065 :
21066 3 : if (mode == V32HImode)
21067 : {
21068 : half_mode = V16HImode;
21069 : extract_hi = gen_vec_extract_hi_v32hi;
21070 : extract_lo = gen_vec_extract_lo_v32hi;
21071 : }
21072 : else if (mode == V32HFmode)
21073 : {
21074 : half_mode = V16HFmode;
21075 : extract_hi = gen_vec_extract_hi_v32hf;
21076 : extract_lo = gen_vec_extract_lo_v32hf;
21077 : }
21078 : else if (mode == V32BFmode)
21079 : {
21080 : half_mode = V16BFmode;
21081 : extract_hi = gen_vec_extract_hi_v32bf;
21082 : extract_lo = gen_vec_extract_lo_v32bf;
21083 : }
21084 : else
21085 : {
21086 3 : half_mode = V32QImode;
21087 3 : extract_hi = gen_vec_extract_hi_v64qi;
21088 3 : extract_lo = gen_vec_extract_lo_v64qi;
21089 : }
21090 :
21091 3 : vhi = gen_reg_rtx (half_mode);
21092 3 : vlo = gen_reg_rtx (half_mode);
21093 3 : idx_hi = gen_reg_rtx (GET_MODE (idx));
21094 3 : emit_insn (extract_hi (vhi, target));
21095 3 : emit_insn (extract_lo (vlo, target));
21096 3 : vec[0] = idx_hi;
21097 3 : vec[1] = idx;
21098 3 : vec[2] = GEN_INT (n_elts/2);
21099 3 : ix86_expand_binary_operator (MINUS, GET_MODE (idx), vec);
21100 3 : ix86_expand_vector_set_var (vhi, val, idx_hi);
21101 3 : ix86_expand_vector_set_var (vlo, val, idx);
21102 3 : emit_insn (gen_rtx_SET (target, gen_rtx_VEC_CONCAT (mode, vlo, vhi)));
21103 3 : return;
21104 : }
21105 :
21106 504 : if (FLOAT_MODE_P (GET_MODE_INNER (mode)))
21107 : {
21108 42 : switch (mode)
21109 : {
21110 : case E_V2DFmode:
21111 : cmp_mode = V2DImode;
21112 : break;
21113 6 : case E_V4DFmode:
21114 6 : cmp_mode = V4DImode;
21115 6 : break;
21116 4 : case E_V8DFmode:
21117 4 : cmp_mode = V8DImode;
21118 4 : break;
21119 2 : case E_V2SFmode:
21120 2 : cmp_mode = V2SImode;
21121 2 : break;
21122 6 : case E_V4SFmode:
21123 6 : cmp_mode = V4SImode;
21124 6 : break;
21125 6 : case E_V8SFmode:
21126 6 : cmp_mode = V8SImode;
21127 6 : break;
21128 5 : case E_V16SFmode:
21129 5 : cmp_mode = V16SImode;
21130 5 : break;
21131 1 : case E_V2HFmode:
21132 1 : case E_V2BFmode:
21133 1 : cmp_mode = V2HImode;
21134 1 : break;
21135 1 : case E_V4HFmode:
21136 1 : case E_V4BFmode:
21137 1 : cmp_mode = V4HImode;
21138 1 : break;
21139 : case E_V8HFmode:
21140 2 : cmp_mode = V8HImode;
21141 : break;
21142 : case E_V16HFmode:
21143 2 : cmp_mode = V16HImode;
21144 : break;
21145 : case E_V32HFmode:
21146 1 : cmp_mode = V32HImode;
21147 : break;
21148 : case E_V8BFmode:
21149 2 : cmp_mode = V8HImode;
21150 : break;
21151 : case E_V16BFmode:
21152 2 : cmp_mode = V16HImode;
21153 : break;
21154 : case E_V32BFmode:
21155 1 : cmp_mode = V32HImode;
21156 : break;
21157 0 : default:
21158 0 : gcc_unreachable ();
21159 : }
21160 : }
21161 :
21162 1604 : for (int i = 0; i != n_elts; i++)
21163 1478 : vec[i] = GEN_INT (i);
21164 126 : constv = gen_rtx_CONST_VECTOR (cmp_mode, gen_rtvec_v (n_elts, vec));
21165 126 : valv = gen_reg_rtx (mode);
21166 126 : idxv = gen_reg_rtx (cmp_mode);
21167 252 : idx_tmp = convert_to_mode (GET_MODE_INNER (cmp_mode), idx, 1);
21168 :
21169 126 : ok = ix86_expand_vector_init_duplicate (TARGET_MMX_WITH_SSE,
21170 : mode, valv, val);
21171 126 : gcc_assert (ok);
21172 126 : ok = ix86_expand_vector_init_duplicate (TARGET_MMX_WITH_SSE,
21173 : cmp_mode, idxv, idx_tmp);
21174 126 : gcc_assert (ok);
21175 126 : vec[0] = target;
21176 126 : vec[1] = valv;
21177 126 : vec[2] = target;
21178 126 : vec[3] = gen_rtx_EQ (mode, idxv, constv);
21179 126 : vec[4] = idxv;
21180 126 : vec[5] = constv;
21181 126 : ok = ix86_expand_int_vcond (vec);
21182 126 : gcc_assert (ok);
21183 : }
21184 :
21185 : void
21186 6245 : ix86_expand_vector_set (bool mmx_ok, rtx target, rtx val, int elt)
21187 : {
21188 6245 : machine_mode mode = GET_MODE (target);
21189 6245 : machine_mode inner_mode = GET_MODE_INNER (mode);
21190 6245 : machine_mode half_mode;
21191 6245 : bool use_vec_merge = false;
21192 6245 : bool blendm_const = false;
21193 6245 : rtx tmp;
21194 6245 : static rtx (*gen_extract[8][2]) (rtx, rtx)
21195 : = {
21196 : { gen_vec_extract_lo_v32qi, gen_vec_extract_hi_v32qi },
21197 : { gen_vec_extract_lo_v16hi, gen_vec_extract_hi_v16hi },
21198 : { gen_vec_extract_lo_v8si, gen_vec_extract_hi_v8si },
21199 : { gen_vec_extract_lo_v4di, gen_vec_extract_hi_v4di },
21200 : { gen_vec_extract_lo_v8sf, gen_vec_extract_hi_v8sf },
21201 : { gen_vec_extract_lo_v4df, gen_vec_extract_hi_v4df },
21202 : { gen_vec_extract_lo_v16hf, gen_vec_extract_hi_v16hf },
21203 : { gen_vec_extract_lo_v16bf, gen_vec_extract_hi_v16bf }
21204 : };
21205 6245 : static rtx (*gen_insert[8][2]) (rtx, rtx, rtx)
21206 : = {
21207 : { gen_vec_set_lo_v32qi, gen_vec_set_hi_v32qi },
21208 : { gen_vec_set_lo_v16hi, gen_vec_set_hi_v16hi },
21209 : { gen_vec_set_lo_v8si, gen_vec_set_hi_v8si },
21210 : { gen_vec_set_lo_v4di, gen_vec_set_hi_v4di },
21211 : { gen_vec_set_lo_v8sf, gen_vec_set_hi_v8sf },
21212 : { gen_vec_set_lo_v4df, gen_vec_set_hi_v4df },
21213 : { gen_vec_set_lo_v16hf, gen_vec_set_hi_v16hf },
21214 : { gen_vec_set_lo_v16bf, gen_vec_set_hi_v16bf },
21215 : };
21216 6245 : int i, j, n;
21217 6245 : machine_mode mmode = VOIDmode;
21218 6245 : rtx (*gen_blendm) (rtx, rtx, rtx, rtx);
21219 :
21220 12490 : if (!IN_RANGE (elt, 0, GET_MODE_NUNITS (mode)))
21221 : {
21222 0 : emit_move_insn (target, target);
21223 0 : return;
21224 : }
21225 6245 : if (TARGET_SSE4_1 && mode == V4SImode && val == const0_rtx)
21226 : {
21227 27 : emit_insn (gen_sse4_1_insertps_v4si_zero (target, target,
21228 : CONST0_RTX (V4SImode),
21229 27 : GEN_INT ((1 << elt) ^ 15)));
21230 27 : return;
21231 : }
21232 6218 : if (TARGET_SSE4_1 && mode == V4SFmode && val == CONST0_RTX (SFmode))
21233 : {
21234 25 : emit_insn (gen_sse4_1_insertps_v4sf_zero (target, target,
21235 : CONST0_RTX (V4SFmode),
21236 25 : GEN_INT ((1 << elt) ^ 15)));
21237 25 : return;
21238 : }
21239 :
21240 12386 : val = force_reg (GET_MODE_INNER (mode), val);
21241 :
21242 6193 : switch (mode)
21243 : {
21244 189 : case E_V2SImode:
21245 189 : use_vec_merge = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
21246 : if (use_vec_merge)
21247 : break;
21248 : /* FALLTHRU */
21249 :
21250 174 : case E_V2SFmode:
21251 174 : if (mmx_ok)
21252 : {
21253 344 : tmp = gen_reg_rtx (GET_MODE_INNER (mode));
21254 172 : ix86_expand_vector_extract (true, tmp, target, 1 - elt);
21255 172 : if (elt == 0)
21256 0 : tmp = gen_rtx_VEC_CONCAT (mode, val, tmp);
21257 : else
21258 172 : tmp = gen_rtx_VEC_CONCAT (mode, tmp, val);
21259 172 : emit_insn (gen_rtx_SET (target, tmp));
21260 172 : return;
21261 : }
21262 : break;
21263 :
21264 160 : case E_V2DImode:
21265 160 : use_vec_merge = TARGET_SSE4_1 && TARGET_64BIT;
21266 82 : if (use_vec_merge)
21267 : break;
21268 :
21269 164 : tmp = gen_reg_rtx (GET_MODE_INNER (mode));
21270 82 : ix86_expand_vector_extract (false, tmp, target, 1 - elt);
21271 82 : if (elt == 0)
21272 65 : tmp = gen_rtx_VEC_CONCAT (mode, val, tmp);
21273 : else
21274 17 : tmp = gen_rtx_VEC_CONCAT (mode, tmp, val);
21275 82 : emit_insn (gen_rtx_SET (target, tmp));
21276 82 : return;
21277 :
21278 127 : case E_V2DFmode:
21279 : /* NB: For ELT == 0, use standard scalar operation patterns which
21280 : preserve the rest of the vector for combiner:
21281 :
21282 : (vec_merge:V2DF
21283 : (vec_duplicate:V2DF (reg:DF))
21284 : (reg:V2DF)
21285 : (const_int 1))
21286 : */
21287 127 : if (elt == 0)
21288 64 : goto do_vec_merge;
21289 :
21290 63 : {
21291 63 : rtx op0, op1;
21292 :
21293 : /* For the two element vectors, we implement a VEC_CONCAT with
21294 : the extraction of the other element. */
21295 :
21296 63 : tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (1 - elt)));
21297 63 : tmp = gen_rtx_VEC_SELECT (inner_mode, target, tmp);
21298 :
21299 63 : if (elt == 0)
21300 : op0 = val, op1 = tmp;
21301 : else
21302 63 : op0 = tmp, op1 = val;
21303 :
21304 63 : tmp = gen_rtx_VEC_CONCAT (mode, op0, op1);
21305 63 : emit_insn (gen_rtx_SET (target, tmp));
21306 : }
21307 63 : return;
21308 :
21309 80 : case E_V4SFmode:
21310 80 : use_vec_merge = TARGET_SSE4_1;
21311 80 : if (use_vec_merge)
21312 : break;
21313 :
21314 66 : switch (elt)
21315 : {
21316 : case 0:
21317 : use_vec_merge = true;
21318 : break;
21319 :
21320 2 : case 1:
21321 : /* tmp = target = A B C D */
21322 2 : tmp = copy_to_reg (target);
21323 : /* target = A A B B */
21324 2 : emit_insn (gen_vec_interleave_lowv4sf (target, target, target));
21325 : /* target = X A B B */
21326 2 : ix86_expand_vector_set (false, target, val, 0);
21327 : /* target = A X C D */
21328 2 : emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
21329 : const1_rtx, const0_rtx,
21330 : GEN_INT (2+4), GEN_INT (3+4)));
21331 2 : return;
21332 :
21333 1 : case 2:
21334 : /* tmp = target = A B C D */
21335 1 : tmp = copy_to_reg (target);
21336 : /* tmp = X B C D */
21337 1 : ix86_expand_vector_set (false, tmp, val, 0);
21338 : /* target = A B X D */
21339 1 : emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
21340 : const0_rtx, const1_rtx,
21341 : GEN_INT (0+4), GEN_INT (3+4)));
21342 1 : return;
21343 :
21344 4 : case 3:
21345 : /* tmp = target = A B C D */
21346 4 : tmp = copy_to_reg (target);
21347 : /* tmp = X B C D */
21348 4 : ix86_expand_vector_set (false, tmp, val, 0);
21349 : /* target = A B X D */
21350 4 : emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
21351 : const0_rtx, const1_rtx,
21352 : GEN_INT (2+4), GEN_INT (0+4)));
21353 4 : return;
21354 :
21355 0 : default:
21356 0 : gcc_unreachable ();
21357 : }
21358 : break;
21359 :
21360 402 : case E_V4SImode:
21361 402 : use_vec_merge = TARGET_SSE4_1;
21362 402 : if (use_vec_merge)
21363 : break;
21364 :
21365 : /* Element 0 handled by vec_merge below. */
21366 283 : if (elt == 0)
21367 : {
21368 : use_vec_merge = true;
21369 : break;
21370 : }
21371 :
21372 92 : if (TARGET_SSE2)
21373 : {
21374 : /* With SSE2, use integer shuffles to swap element 0 and ELT,
21375 : store into element 0, then shuffle them back. */
21376 :
21377 92 : rtx order[4];
21378 :
21379 92 : order[0] = GEN_INT (elt);
21380 92 : order[1] = const1_rtx;
21381 92 : order[2] = const2_rtx;
21382 92 : order[3] = GEN_INT (3);
21383 92 : order[elt] = const0_rtx;
21384 :
21385 92 : emit_insn (gen_sse2_pshufd_1 (target, target, order[0],
21386 : order[1], order[2], order[3]));
21387 :
21388 92 : ix86_expand_vector_set (false, target, val, 0);
21389 :
21390 92 : emit_insn (gen_sse2_pshufd_1 (target, target, order[0],
21391 : order[1], order[2], order[3]));
21392 : }
21393 : else
21394 : {
21395 : /* For SSE1, we have to reuse the V4SF code. */
21396 0 : rtx t = gen_reg_rtx (V4SFmode);
21397 0 : emit_move_insn (t, gen_lowpart (V4SFmode, target));
21398 0 : ix86_expand_vector_set (false, t, gen_lowpart (SFmode, val), elt);
21399 0 : emit_move_insn (target, gen_lowpart (mode, t));
21400 : }
21401 : return;
21402 :
21403 2018 : case E_V8HImode:
21404 2018 : case E_V8HFmode:
21405 2018 : case E_V8BFmode:
21406 2018 : case E_V2HImode:
21407 2018 : case E_V2HFmode:
21408 2018 : case E_V2BFmode:
21409 2018 : use_vec_merge = TARGET_SSE2;
21410 2018 : break;
21411 51 : case E_V4HImode:
21412 51 : case E_V4HFmode:
21413 51 : case E_V4BFmode:
21414 51 : use_vec_merge = mmx_ok && (TARGET_SSE || TARGET_3DNOW_A);
21415 : break;
21416 :
21417 3019 : case E_V16QImode:
21418 3019 : case E_V4QImode:
21419 3019 : use_vec_merge = TARGET_SSE4_1;
21420 3019 : break;
21421 :
21422 1 : case E_V8QImode:
21423 1 : use_vec_merge = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
21424 : break;
21425 :
21426 3 : case E_V32QImode:
21427 3 : half_mode = V16QImode;
21428 3 : j = 0;
21429 3 : n = 16;
21430 3 : goto half;
21431 :
21432 17 : case E_V16HFmode:
21433 17 : case E_V16BFmode:
21434 : /* For ELT == 0, vec_setv8hf_0 can save 1 vpbroadcastw. */
21435 17 : if (TARGET_AVX2 && elt != 0)
21436 : {
21437 12 : mmode = SImode;
21438 12 : gen_blendm = ((mode == E_V16HFmode) ? gen_avx2_pblendph_1
21439 : : gen_avx2_pblendbf_1);
21440 : blendm_const = true;
21441 : break;
21442 : }
21443 : else
21444 : {
21445 5 : half_mode = ((mode == E_V16HFmode) ? V8HFmode : V8BFmode);
21446 3 : j = ((mode == E_V16HFmode) ? 6 : 7);
21447 5 : n = 8;
21448 5 : goto half;
21449 : }
21450 :
21451 5 : case E_V16HImode:
21452 5 : half_mode = V8HImode;
21453 5 : j = 1;
21454 5 : n = 8;
21455 5 : goto half;
21456 :
21457 15 : case E_V8SImode:
21458 15 : half_mode = V4SImode;
21459 15 : j = 2;
21460 15 : n = 4;
21461 15 : goto half;
21462 :
21463 15 : case E_V4DImode:
21464 15 : half_mode = V2DImode;
21465 15 : j = 3;
21466 15 : n = 2;
21467 15 : goto half;
21468 :
21469 4 : case E_V8SFmode:
21470 4 : half_mode = V4SFmode;
21471 4 : j = 4;
21472 4 : n = 4;
21473 4 : goto half;
21474 :
21475 6 : case E_V4DFmode:
21476 6 : half_mode = V2DFmode;
21477 6 : j = 5;
21478 6 : n = 2;
21479 6 : goto half;
21480 :
21481 53 : half:
21482 : /* Compute offset. */
21483 53 : i = elt / n;
21484 53 : elt %= n;
21485 :
21486 53 : gcc_assert (i <= 1);
21487 :
21488 : /* Extract the half. */
21489 53 : tmp = gen_reg_rtx (half_mode);
21490 53 : emit_insn (gen_extract[j][i] (tmp, target));
21491 :
21492 : /* Put val in tmp at elt. */
21493 53 : ix86_expand_vector_set (false, tmp, val, elt);
21494 :
21495 : /* Put it back. */
21496 53 : emit_insn (gen_insert[j][i] (target, target, tmp));
21497 53 : return;
21498 :
21499 8 : case E_V8DFmode:
21500 8 : if (TARGET_AVX512F)
21501 : {
21502 : mmode = QImode;
21503 : gen_blendm = gen_avx512f_blendmv8df;
21504 : }
21505 : break;
21506 :
21507 7 : case E_V8DImode:
21508 7 : if (TARGET_AVX512F)
21509 : {
21510 : mmode = QImode;
21511 : gen_blendm = gen_avx512f_blendmv8di;
21512 : }
21513 : break;
21514 :
21515 0 : case E_V16SFmode:
21516 0 : if (TARGET_AVX512F)
21517 : {
21518 : mmode = HImode;
21519 : gen_blendm = gen_avx512f_blendmv16sf;
21520 : }
21521 : break;
21522 :
21523 0 : case E_V16SImode:
21524 0 : if (TARGET_AVX512F)
21525 : {
21526 : mmode = HImode;
21527 : gen_blendm = gen_avx512f_blendmv16si;
21528 : }
21529 : break;
21530 :
21531 12 : case E_V32HFmode:
21532 12 : if (TARGET_AVX512BW)
21533 : {
21534 : mmode = SImode;
21535 : gen_blendm = gen_avx512bw_blendmv32hf;
21536 : }
21537 : break;
21538 12 : case E_V32BFmode:
21539 12 : if (TARGET_AVX512BW)
21540 : {
21541 : mmode = SImode;
21542 : gen_blendm = gen_avx512bw_blendmv32bf;
21543 : }
21544 : break;
21545 11 : case E_V32HImode:
21546 11 : if (TARGET_AVX512BW)
21547 : {
21548 : mmode = SImode;
21549 : gen_blendm = gen_avx512bw_blendmv32hi;
21550 : }
21551 7 : else if (TARGET_AVX512F)
21552 : {
21553 7 : half_mode = E_V8HImode;
21554 7 : n = 8;
21555 7 : goto quarter;
21556 : }
21557 : break;
21558 :
21559 12 : case E_V64QImode:
21560 12 : if (TARGET_AVX512BW)
21561 : {
21562 : mmode = DImode;
21563 : gen_blendm = gen_avx512bw_blendmv64qi;
21564 : }
21565 6 : else if (TARGET_AVX512F)
21566 : {
21567 6 : half_mode = E_V16QImode;
21568 6 : n = 16;
21569 6 : goto quarter;
21570 : }
21571 : break;
21572 :
21573 13 : quarter:
21574 : /* Compute offset. */
21575 13 : i = elt / n;
21576 13 : elt %= n;
21577 :
21578 13 : gcc_assert (i <= 3);
21579 :
21580 13 : {
21581 : /* Extract the quarter. */
21582 13 : tmp = gen_reg_rtx (V4SImode);
21583 13 : rtx tmp2 = gen_lowpart (V16SImode, target);
21584 13 : rtx mask = gen_reg_rtx (QImode);
21585 :
21586 13 : emit_move_insn (mask, constm1_rtx);
21587 13 : emit_insn (gen_avx512f_vextracti32x4_mask (tmp, tmp2, GEN_INT (i),
21588 : tmp, mask));
21589 :
21590 13 : tmp2 = gen_reg_rtx (half_mode);
21591 13 : emit_move_insn (tmp2, gen_lowpart (half_mode, tmp));
21592 13 : tmp = tmp2;
21593 :
21594 : /* Put val in tmp at elt. */
21595 13 : ix86_expand_vector_set (false, tmp, val, elt);
21596 :
21597 : /* Put it back. */
21598 13 : tmp2 = gen_reg_rtx (V16SImode);
21599 13 : rtx tmp3 = gen_lowpart (V16SImode, target);
21600 13 : mask = gen_reg_rtx (HImode);
21601 13 : emit_move_insn (mask, constm1_rtx);
21602 13 : tmp = gen_lowpart (V4SImode, tmp);
21603 13 : emit_insn (gen_avx512f_vinserti32x4_mask (tmp2, tmp3, tmp, GEN_INT (i),
21604 : tmp3, mask));
21605 13 : emit_move_insn (target, gen_lowpart (mode, tmp2));
21606 : }
21607 13 : return;
21608 :
21609 : default:
21610 : break;
21611 : }
21612 :
21613 5043 : if (mmode != VOIDmode)
21614 : {
21615 55 : tmp = gen_reg_rtx (mode);
21616 55 : emit_insn (gen_rtx_SET (tmp, gen_rtx_VEC_DUPLICATE (mode, val)));
21617 55 : rtx merge_mask = gen_int_mode (HOST_WIDE_INT_1U << elt, mmode);
21618 : /* The avx512*_blendm<mode> expanders have different operand order
21619 : from VEC_MERGE. In VEC_MERGE, the first input operand is used for
21620 : elements where the mask is set and second input operand otherwise,
21621 : in {sse,avx}*_*blend* the first input operand is used for elements
21622 : where the mask is clear and second input operand otherwise. */
21623 55 : if (!blendm_const)
21624 43 : merge_mask = force_reg (mmode, merge_mask);
21625 55 : emit_insn (gen_blendm (target, target, tmp, merge_mask));
21626 : }
21627 5592 : else if (use_vec_merge)
21628 : {
21629 5580 : do_vec_merge:
21630 5644 : if (!nonimmediate_operand (val, inner_mode))
21631 0 : val = force_reg (inner_mode, val);
21632 5644 : tmp = gen_rtx_VEC_DUPLICATE (mode, val);
21633 5644 : tmp = gen_rtx_VEC_MERGE (mode, tmp, target,
21634 : GEN_INT (HOST_WIDE_INT_1U << elt));
21635 5644 : emit_insn (gen_rtx_SET (target, tmp));
21636 : }
21637 : else
21638 : {
21639 24 : rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode));
21640 :
21641 12 : emit_move_insn (mem, target);
21642 :
21643 24 : tmp = adjust_address (mem, inner_mode, elt * GET_MODE_SIZE (inner_mode));
21644 12 : emit_move_insn (tmp, val);
21645 :
21646 12 : emit_move_insn (target, mem);
21647 : }
21648 : }
21649 :
21650 : void
21651 83846 : ix86_expand_vector_extract (bool mmx_ok, rtx target, rtx vec, int elt)
21652 : {
21653 89321 : machine_mode mode = GET_MODE (vec);
21654 89321 : machine_mode inner_mode = GET_MODE_INNER (mode);
21655 89321 : bool use_vec_extr = false;
21656 89321 : rtx tmp;
21657 :
21658 89321 : switch (mode)
21659 : {
21660 5530 : case E_V2SImode:
21661 5530 : use_vec_extr = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
21662 : if (use_vec_extr)
21663 : break;
21664 : /* FALLTHRU */
21665 :
21666 8335 : case E_V2SFmode:
21667 8335 : if (!mmx_ok)
21668 : break;
21669 : /* FALLTHRU */
21670 :
21671 : case E_V2DFmode:
21672 : case E_V2DImode:
21673 : case E_V2TImode:
21674 : case E_V4TImode:
21675 : use_vec_extr = true;
21676 : break;
21677 :
21678 7693 : case E_V4SFmode:
21679 7693 : use_vec_extr = TARGET_SSE4_1;
21680 7693 : if (use_vec_extr)
21681 : break;
21682 :
21683 3968 : switch (elt)
21684 : {
21685 : case 0:
21686 : tmp = vec;
21687 : break;
21688 :
21689 1730 : case 1:
21690 1730 : case 3:
21691 1730 : tmp = gen_reg_rtx (mode);
21692 1730 : emit_insn (gen_sse_shufps_v4sf (tmp, vec, vec,
21693 : GEN_INT (elt), GEN_INT (elt),
21694 1730 : GEN_INT (elt+4), GEN_INT (elt+4)));
21695 1730 : break;
21696 :
21697 856 : case 2:
21698 856 : tmp = gen_reg_rtx (mode);
21699 856 : emit_insn (gen_vec_interleave_highv4sf (tmp, vec, vec));
21700 856 : break;
21701 :
21702 0 : default:
21703 0 : gcc_unreachable ();
21704 : }
21705 : vec = tmp;
21706 : use_vec_extr = true;
21707 : elt = 0;
21708 : break;
21709 :
21710 25612 : case E_V4SImode:
21711 25612 : use_vec_extr = TARGET_SSE4_1;
21712 25612 : if (use_vec_extr)
21713 : break;
21714 :
21715 19693 : if (TARGET_SSE2)
21716 : {
21717 19689 : switch (elt)
21718 : {
21719 : case 0:
21720 : tmp = vec;
21721 : break;
21722 :
21723 6610 : case 1:
21724 6610 : case 3:
21725 6610 : tmp = gen_reg_rtx (mode);
21726 6610 : emit_insn (gen_sse2_pshufd_1 (tmp, vec,
21727 : GEN_INT (elt), GEN_INT (elt),
21728 : GEN_INT (elt), GEN_INT (elt)));
21729 6610 : break;
21730 :
21731 3239 : case 2:
21732 3239 : tmp = gen_reg_rtx (mode);
21733 3239 : emit_insn (gen_vec_interleave_highv4si (tmp, vec, vec));
21734 3239 : break;
21735 :
21736 0 : default:
21737 0 : gcc_unreachable ();
21738 : }
21739 : vec = tmp;
21740 : use_vec_extr = true;
21741 : elt = 0;
21742 : }
21743 : else
21744 : {
21745 : /* For SSE1, we have to reuse the V4SF code. */
21746 4 : ix86_expand_vector_extract (false, gen_lowpart (SFmode, target),
21747 4 : gen_lowpart (V4SFmode, vec), elt);
21748 4 : return;
21749 : }
21750 : break;
21751 :
21752 6993 : case E_V8HImode:
21753 6993 : case E_V8HFmode:
21754 6993 : case E_V8BFmode:
21755 6993 : case E_V2HImode:
21756 6993 : case E_V2HFmode:
21757 6993 : case E_V2BFmode:
21758 6993 : use_vec_extr = TARGET_SSE2;
21759 6993 : break;
21760 1138 : case E_V4HImode:
21761 1138 : case E_V4HFmode:
21762 1138 : case E_V4BFmode:
21763 1138 : use_vec_extr = mmx_ok && (TARGET_SSE || TARGET_3DNOW_A);
21764 : break;
21765 :
21766 8439 : case E_V16QImode:
21767 8439 : use_vec_extr = TARGET_SSE4_1;
21768 8439 : if (!use_vec_extr
21769 6826 : && TARGET_SSE2
21770 6826 : && elt == 0
21771 12367 : && (optimize_insn_for_size_p () || TARGET_INTER_UNIT_MOVES_FROM_VEC))
21772 : {
21773 3927 : tmp = gen_reg_rtx (SImode);
21774 3927 : ix86_expand_vector_extract (false, tmp, gen_lowpart (V4SImode, vec),
21775 : 0);
21776 3927 : emit_insn (gen_rtx_SET (target, gen_lowpart (QImode, tmp)));
21777 3927 : return;
21778 : }
21779 : break;
21780 79 : case E_V4QImode:
21781 79 : use_vec_extr = TARGET_SSE4_1;
21782 79 : break;
21783 :
21784 221 : case E_V8SFmode:
21785 221 : if (TARGET_AVX)
21786 : {
21787 221 : tmp = gen_reg_rtx (V4SFmode);
21788 221 : if (elt < 4)
21789 106 : emit_insn (gen_vec_extract_lo_v8sf (tmp, vec));
21790 : else
21791 115 : emit_insn (gen_vec_extract_hi_v8sf (tmp, vec));
21792 221 : ix86_expand_vector_extract (false, target, tmp, elt & 3);
21793 221 : return;
21794 : }
21795 : break;
21796 :
21797 363 : case E_V4DFmode:
21798 363 : if (TARGET_AVX)
21799 : {
21800 363 : tmp = gen_reg_rtx (V2DFmode);
21801 363 : if (elt < 2)
21802 194 : emit_insn (gen_vec_extract_lo_v4df (tmp, vec));
21803 : else
21804 169 : emit_insn (gen_vec_extract_hi_v4df (tmp, vec));
21805 363 : ix86_expand_vector_extract (false, target, tmp, elt & 1);
21806 363 : return;
21807 : }
21808 : break;
21809 :
21810 253 : case E_V32QImode:
21811 253 : if (TARGET_AVX)
21812 : {
21813 253 : tmp = gen_reg_rtx (V16QImode);
21814 253 : if (elt < 16)
21815 130 : emit_insn (gen_vec_extract_lo_v32qi (tmp, vec));
21816 : else
21817 123 : emit_insn (gen_vec_extract_hi_v32qi (tmp, vec));
21818 253 : ix86_expand_vector_extract (false, target, tmp, elt & 15);
21819 253 : return;
21820 : }
21821 : break;
21822 :
21823 616 : case E_V16HImode:
21824 616 : if (TARGET_AVX)
21825 : {
21826 616 : tmp = gen_reg_rtx (V8HImode);
21827 616 : if (elt < 8)
21828 304 : emit_insn (gen_vec_extract_lo_v16hi (tmp, vec));
21829 : else
21830 312 : emit_insn (gen_vec_extract_hi_v16hi (tmp, vec));
21831 616 : ix86_expand_vector_extract (false, target, tmp, elt & 7);
21832 616 : return;
21833 : }
21834 : break;
21835 :
21836 1123 : case E_V8SImode:
21837 1123 : if (TARGET_AVX)
21838 : {
21839 1123 : tmp = gen_reg_rtx (V4SImode);
21840 1123 : if (elt < 4)
21841 540 : emit_insn (gen_vec_extract_lo_v8si (tmp, vec));
21842 : else
21843 583 : emit_insn (gen_vec_extract_hi_v8si (tmp, vec));
21844 1123 : ix86_expand_vector_extract (false, target, tmp, elt & 3);
21845 1123 : return;
21846 : }
21847 : break;
21848 :
21849 1349 : case E_V4DImode:
21850 1349 : if (TARGET_AVX)
21851 : {
21852 1349 : tmp = gen_reg_rtx (V2DImode);
21853 1349 : if (elt < 2)
21854 711 : emit_insn (gen_vec_extract_lo_v4di (tmp, vec));
21855 : else
21856 638 : emit_insn (gen_vec_extract_hi_v4di (tmp, vec));
21857 1349 : ix86_expand_vector_extract (false, target, tmp, elt & 1);
21858 1349 : return;
21859 : }
21860 : break;
21861 :
21862 8 : case E_V32HImode:
21863 8 : if (TARGET_AVX512BW)
21864 : {
21865 8 : tmp = gen_reg_rtx (V16HImode);
21866 8 : if (elt < 16)
21867 3 : emit_insn (gen_vec_extract_lo_v32hi (tmp, vec));
21868 : else
21869 5 : emit_insn (gen_vec_extract_hi_v32hi (tmp, vec));
21870 8 : ix86_expand_vector_extract (false, target, tmp, elt & 15);
21871 8 : return;
21872 : }
21873 : break;
21874 :
21875 10 : case E_V64QImode:
21876 10 : if (TARGET_AVX512BW)
21877 : {
21878 10 : tmp = gen_reg_rtx (V32QImode);
21879 10 : if (elt < 32)
21880 5 : emit_insn (gen_vec_extract_lo_v64qi (tmp, vec));
21881 : else
21882 5 : emit_insn (gen_vec_extract_hi_v64qi (tmp, vec));
21883 10 : ix86_expand_vector_extract (false, target, tmp, elt & 31);
21884 10 : return;
21885 : }
21886 : break;
21887 :
21888 25 : case E_V16SFmode:
21889 25 : tmp = gen_reg_rtx (V8SFmode);
21890 25 : if (elt < 8)
21891 15 : emit_insn (gen_vec_extract_lo_v16sf (tmp, vec));
21892 : else
21893 10 : emit_insn (gen_vec_extract_hi_v16sf (tmp, vec));
21894 25 : ix86_expand_vector_extract (false, target, tmp, elt & 7);
21895 25 : return;
21896 :
21897 106 : case E_V8DFmode:
21898 106 : tmp = gen_reg_rtx (V4DFmode);
21899 106 : if (elt < 4)
21900 66 : emit_insn (gen_vec_extract_lo_v8df (tmp, vec));
21901 : else
21902 40 : emit_insn (gen_vec_extract_hi_v8df (tmp, vec));
21903 106 : ix86_expand_vector_extract (false, target, tmp, elt & 3);
21904 106 : return;
21905 :
21906 239 : case E_V16SImode:
21907 239 : tmp = gen_reg_rtx (V8SImode);
21908 239 : if (elt < 8)
21909 128 : emit_insn (gen_vec_extract_lo_v16si (tmp, vec));
21910 : else
21911 111 : emit_insn (gen_vec_extract_hi_v16si (tmp, vec));
21912 239 : ix86_expand_vector_extract (false, target, tmp, elt & 7);
21913 239 : return;
21914 :
21915 671 : case E_V8DImode:
21916 671 : tmp = gen_reg_rtx (V4DImode);
21917 671 : if (elt < 4)
21918 384 : emit_insn (gen_vec_extract_lo_v8di (tmp, vec));
21919 : else
21920 287 : emit_insn (gen_vec_extract_hi_v8di (tmp, vec));
21921 671 : ix86_expand_vector_extract (false, target, tmp, elt & 3);
21922 671 : return;
21923 :
21924 45 : case E_V32HFmode:
21925 45 : case E_V32BFmode:
21926 45 : if (TARGET_AVX512BW)
21927 : {
21928 45 : tmp = (mode == E_V32HFmode
21929 45 : ? gen_reg_rtx (V16HFmode)
21930 7 : : gen_reg_rtx (V16BFmode));
21931 45 : if (elt < 16)
21932 31 : emit_insn (gen_vec_extract_lo (mode, tmp, vec));
21933 : else
21934 14 : emit_insn (gen_vec_extract_hi (mode, tmp, vec));
21935 45 : ix86_expand_vector_extract (false, target, tmp, elt & 15);
21936 45 : return;
21937 : }
21938 : break;
21939 :
21940 442 : case E_V16HFmode:
21941 442 : case E_V16BFmode:
21942 442 : if (TARGET_AVX)
21943 : {
21944 442 : tmp = (mode == E_V16HFmode
21945 442 : ? gen_reg_rtx (V8HFmode)
21946 339 : : gen_reg_rtx (V8BFmode));
21947 442 : if (elt < 8)
21948 233 : emit_insn (gen_vec_extract_lo (mode, tmp, vec));
21949 : else
21950 209 : emit_insn (gen_vec_extract_hi (mode, tmp, vec));
21951 442 : ix86_expand_vector_extract (false, target, tmp, elt & 7);
21952 442 : return;
21953 : }
21954 : break;
21955 :
21956 1347 : case E_V8QImode:
21957 1347 : use_vec_extr = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
21958 : /* ??? Could extract the appropriate HImode element and shift. */
21959 : break;
21960 :
21961 : default:
21962 : break;
21963 : }
21964 :
21965 35939 : if (use_vec_extr)
21966 : {
21967 75039 : tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (elt)));
21968 75039 : tmp = gen_rtx_VEC_SELECT (inner_mode, vec, tmp);
21969 :
21970 : /* Let the rtl optimizers know about the zero extension performed. */
21971 75039 : if (inner_mode == QImode || inner_mode == HImode)
21972 : {
21973 9947 : rtx reg = gen_reg_rtx (SImode);
21974 9947 : tmp = gen_rtx_ZERO_EXTEND (SImode, tmp);
21975 9947 : emit_move_insn (reg, tmp);
21976 9947 : tmp = gen_lowpart (inner_mode, reg);
21977 9947 : SUBREG_PROMOTED_VAR_P (tmp) = 1;
21978 9947 : SUBREG_PROMOTED_SET (tmp, 1);
21979 : }
21980 :
21981 75039 : emit_move_insn (target, tmp);
21982 : }
21983 : else
21984 : {
21985 9760 : rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode));
21986 :
21987 4880 : emit_move_insn (mem, vec);
21988 :
21989 9760 : tmp = adjust_address (mem, inner_mode, elt*GET_MODE_SIZE (inner_mode));
21990 4880 : emit_move_insn (target, tmp);
21991 : }
21992 : }
21993 :
21994 : /* Generate code to copy vector bits i / 2 ... i - 1 from vector SRC
21995 : to bits 0 ... i / 2 - 1 of vector DEST, which has the same mode.
21996 : The upper bits of DEST are undefined, though they shouldn't cause
21997 : exceptions (some bits from src or all zeros are ok). */
21998 :
21999 : static void
22000 39216 : emit_reduc_half (rtx dest, rtx src, int i)
22001 : {
22002 39216 : rtx tem, d = dest;
22003 39216 : switch (GET_MODE (src))
22004 : {
22005 6214 : case E_V4SFmode:
22006 6214 : if (i == 128)
22007 3107 : tem = gen_sse_movhlps (dest, src, src);
22008 : else
22009 3107 : tem = gen_sse_shufps_v4sf (dest, src, src, const1_rtx, const1_rtx,
22010 : GEN_INT (1 + 4), GEN_INT (1 + 4));
22011 : break;
22012 3771 : case E_V2DFmode:
22013 3771 : tem = gen_vec_interleave_highv2df (dest, src, src);
22014 3771 : break;
22015 78 : case E_V4QImode:
22016 78 : d = gen_reg_rtx (V1SImode);
22017 78 : tem = gen_mmx_lshrv1si3 (d, gen_lowpart (V1SImode, src),
22018 78 : GEN_INT (i / 2));
22019 78 : break;
22020 602 : case E_V8QImode:
22021 602 : case E_V4HImode:
22022 602 : d = gen_reg_rtx (V1DImode);
22023 602 : tem = gen_mmx_lshrv1di3 (d, gen_lowpart (V1DImode, src),
22024 602 : GEN_INT (i / 2));
22025 602 : break;
22026 28551 : case E_V16QImode:
22027 28551 : case E_V8HImode:
22028 28551 : case E_V8HFmode:
22029 28551 : case E_V4SImode:
22030 28551 : case E_V2DImode:
22031 28551 : if (TARGET_SSE_REDUCTION_PREFER_PSHUF)
22032 : {
22033 11 : if (i == 128)
22034 : {
22035 5 : d = gen_reg_rtx (V4SImode);
22036 10 : tem = gen_sse2_pshufd_1 (
22037 5 : d, force_reg (V4SImode, gen_lowpart (V4SImode, src)),
22038 : GEN_INT (2), GEN_INT (3), GEN_INT (2), GEN_INT (3));
22039 5 : break;
22040 : }
22041 6 : else if (i == 64)
22042 : {
22043 5 : d = gen_reg_rtx (V4SImode);
22044 10 : tem = gen_sse2_pshufd_1 (
22045 5 : d, force_reg (V4SImode, gen_lowpart (V4SImode, src)),
22046 : GEN_INT (1), GEN_INT (1), GEN_INT (1), GEN_INT (1));
22047 5 : break;
22048 : }
22049 1 : else if (i == 32)
22050 : {
22051 1 : d = gen_reg_rtx (V8HImode);
22052 2 : tem = gen_sse2_pshuflw_1 (
22053 1 : d, force_reg (V8HImode, gen_lowpart (V8HImode, src)),
22054 : GEN_INT (1), GEN_INT (1), GEN_INT (1), GEN_INT (1));
22055 1 : break;
22056 : }
22057 : }
22058 28540 : d = gen_reg_rtx (V1TImode);
22059 28540 : tem = gen_sse2_lshrv1ti3 (d, gen_lowpart (V1TImode, src),
22060 28540 : GEN_INT (i / 2));
22061 28540 : break;
22062 0 : case E_V8SFmode:
22063 0 : if (i == 256)
22064 0 : tem = gen_avx_vperm2f128v8sf3 (dest, src, src, const1_rtx);
22065 : else
22066 0 : tem = gen_avx_shufps256 (dest, src, src,
22067 : GEN_INT (i == 128 ? 2 + (3 << 2) : 1));
22068 : break;
22069 0 : case E_V4DFmode:
22070 0 : if (i == 256)
22071 0 : tem = gen_avx_vperm2f128v4df3 (dest, src, src, const1_rtx);
22072 : else
22073 0 : tem = gen_avx_shufpd256 (dest, src, src, const1_rtx);
22074 : break;
22075 0 : case E_V32QImode:
22076 0 : case E_V16HImode:
22077 0 : case E_V16HFmode:
22078 0 : case E_V8SImode:
22079 0 : case E_V4DImode:
22080 0 : if (i == 256)
22081 : {
22082 0 : if (GET_MODE (dest) != V4DImode)
22083 0 : d = gen_reg_rtx (V4DImode);
22084 0 : tem = gen_avx2_permv2ti (d, gen_lowpart (V4DImode, src),
22085 0 : gen_lowpart (V4DImode, src),
22086 : const1_rtx);
22087 : }
22088 : else
22089 : {
22090 0 : d = gen_reg_rtx (V2TImode);
22091 0 : tem = gen_avx2_lshrv2ti3 (d, gen_lowpart (V2TImode, src),
22092 0 : GEN_INT (i / 2));
22093 : }
22094 : break;
22095 0 : case E_V64QImode:
22096 0 : case E_V32HImode:
22097 0 : case E_V32HFmode:
22098 0 : if (i < 64)
22099 : {
22100 0 : d = gen_reg_rtx (V4TImode);
22101 0 : tem = gen_avx512bw_lshrv4ti3 (d, gen_lowpart (V4TImode, src),
22102 0 : GEN_INT (i / 2));
22103 0 : break;
22104 : }
22105 : /* FALLTHRU */
22106 0 : case E_V16SImode:
22107 0 : case E_V16SFmode:
22108 0 : case E_V8DImode:
22109 0 : case E_V8DFmode:
22110 0 : if (i > 128)
22111 0 : tem = gen_avx512f_shuf_i32x4_1 (gen_lowpart (V16SImode, dest),
22112 0 : gen_lowpart (V16SImode, src),
22113 0 : gen_lowpart (V16SImode, src),
22114 : GEN_INT (0x4 + (i == 512 ? 4 : 0)),
22115 : GEN_INT (0x5 + (i == 512 ? 4 : 0)),
22116 : GEN_INT (0x6 + (i == 512 ? 4 : 0)),
22117 : GEN_INT (0x7 + (i == 512 ? 4 : 0)),
22118 : GEN_INT (0xC), GEN_INT (0xD),
22119 : GEN_INT (0xE), GEN_INT (0xF),
22120 : GEN_INT (0x10), GEN_INT (0x11),
22121 : GEN_INT (0x12), GEN_INT (0x13),
22122 : GEN_INT (0x14), GEN_INT (0x15),
22123 : GEN_INT (0x16), GEN_INT (0x17));
22124 : else
22125 0 : tem = gen_avx512f_pshufd_1 (gen_lowpart (V16SImode, dest),
22126 0 : gen_lowpart (V16SImode, src),
22127 : GEN_INT (i == 128 ? 0x2 : 0x1),
22128 : GEN_INT (0x3),
22129 : GEN_INT (0x3),
22130 : GEN_INT (0x3),
22131 : GEN_INT (i == 128 ? 0x6 : 0x5),
22132 : GEN_INT (0x7),
22133 : GEN_INT (0x7),
22134 : GEN_INT (0x7),
22135 : GEN_INT (i == 128 ? 0xA : 0x9),
22136 : GEN_INT (0xB),
22137 : GEN_INT (0xB),
22138 : GEN_INT (0xB),
22139 : GEN_INT (i == 128 ? 0xE : 0xD),
22140 : GEN_INT (0xF),
22141 : GEN_INT (0xF),
22142 : GEN_INT (0xF));
22143 : break;
22144 0 : default:
22145 0 : gcc_unreachable ();
22146 : }
22147 39216 : emit_insn (tem);
22148 39216 : if (d != dest)
22149 29231 : emit_move_insn (dest, gen_lowpart (GET_MODE (dest), d));
22150 39216 : }
22151 :
22152 : /* Expand a vector reduction. FN is the binary pattern to reduce;
22153 : DEST is the destination; IN is the input vector. */
22154 :
22155 : void
22156 17941 : ix86_expand_reduc (rtx (*fn) (rtx, rtx, rtx), rtx dest, rtx in)
22157 : {
22158 17941 : rtx half, dst, vec = in;
22159 17941 : machine_mode mode = GET_MODE (in);
22160 17941 : int i;
22161 :
22162 : /* SSE4 has a special instruction for V8HImode UMIN reduction. */
22163 17941 : if (TARGET_SSE4_1
22164 10106 : && mode == V8HImode
22165 780 : && fn == gen_uminv8hi3)
22166 : {
22167 4 : emit_insn (gen_sse4_1_phminposuw (dest, in));
22168 4 : return;
22169 : }
22170 :
22171 : /* SSE3 has haddpd, some targets prefer that over movhlpd plus add. */
22172 17937 : if (TARGET_SSE3
22173 10154 : && TARGET_V2DF_REDUCTION_PREFER_HADDPD
22174 10 : && mode == V2DFmode
22175 10 : && fn == gen_addv2df3)
22176 : {
22177 10 : emit_insn (gen_sse3_haddv2df3 (dest, in, in));
22178 10 : return;
22179 : }
22180 :
22181 35854 : for (i = GET_MODE_BITSIZE (mode);
22182 114286 : i > GET_MODE_UNIT_BITSIZE (mode);
22183 39216 : i >>= 1)
22184 : {
22185 39216 : half = gen_reg_rtx (mode);
22186 39216 : emit_reduc_half (half, vec, i);
22187 78432 : if (i == GET_MODE_UNIT_BITSIZE (mode) * 2)
22188 : dst = dest;
22189 : else
22190 21289 : dst = gen_reg_rtx (mode);
22191 39216 : emit_insn (fn (dst, half, vec));
22192 39216 : vec = dst;
22193 : }
22194 : }
22195 :
22196 : /* Output code to perform a conditional jump to LABEL, if C2 flag in
22197 : FP status register is set. */
22198 :
22199 : void
22200 285 : ix86_emit_fp_unordered_jump (rtx label)
22201 : {
22202 285 : rtx reg = gen_reg_rtx (HImode);
22203 285 : rtx_insn *insn;
22204 285 : rtx temp;
22205 :
22206 285 : emit_insn (gen_x86_fnstsw_1 (reg));
22207 :
22208 285 : if (TARGET_SAHF && (TARGET_USE_SAHF || optimize_insn_for_size_p ()))
22209 : {
22210 37 : emit_insn (gen_x86_sahf_1 (reg));
22211 :
22212 37 : temp = gen_rtx_REG (CCmode, FLAGS_REG);
22213 37 : temp = gen_rtx_UNORDERED (VOIDmode, temp, const0_rtx);
22214 : }
22215 : else
22216 : {
22217 248 : emit_insn (gen_testqi_ext_1_ccno (reg, GEN_INT (0x04)));
22218 :
22219 248 : temp = gen_rtx_REG (CCNOmode, FLAGS_REG);
22220 248 : temp = gen_rtx_NE (VOIDmode, temp, const0_rtx);
22221 : }
22222 :
22223 285 : temp = gen_rtx_IF_THEN_ELSE (VOIDmode, temp,
22224 : gen_rtx_LABEL_REF (VOIDmode, label),
22225 : pc_rtx);
22226 285 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, temp));
22227 285 : predict_jump (REG_BR_PROB_BASE * 10 / 100);
22228 285 : JUMP_LABEL (insn) = label;
22229 285 : }
22230 :
22231 : /* Output code to perform an sinh XFmode calculation. */
22232 :
22233 : void
22234 2 : ix86_emit_i387_sinh (rtx op0, rtx op1)
22235 : {
22236 2 : rtx e1 = gen_reg_rtx (XFmode);
22237 2 : rtx e2 = gen_reg_rtx (XFmode);
22238 2 : rtx scratch = gen_reg_rtx (HImode);
22239 2 : rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
22240 2 : rtx half = const_double_from_real_value (dconsthalf, XFmode);
22241 2 : rtx cst1, tmp;
22242 2 : rtx_code_label *jump_label = gen_label_rtx ();
22243 2 : rtx_insn *insn;
22244 :
22245 : /* scratch = fxam (op1) */
22246 2 : emit_insn (gen_fxamxf2_i387 (scratch, op1));
22247 :
22248 : /* e1 = expm1 (|op1|) */
22249 2 : emit_insn (gen_absxf2 (e2, op1));
22250 2 : emit_insn (gen_expm1xf2 (e1, e2));
22251 :
22252 : /* e2 = e1 / (e1 + 1.0) + e1 */
22253 2 : cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
22254 2 : emit_insn (gen_addxf3 (e2, e1, cst1));
22255 2 : emit_insn (gen_divxf3 (e2, e1, e2));
22256 2 : emit_insn (gen_addxf3 (e2, e2, e1));
22257 :
22258 : /* flags = signbit (op1) */
22259 2 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
22260 :
22261 : /* if (flags) then e2 = -e2 */
22262 2 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
22263 : gen_rtx_EQ (VOIDmode, flags, const0_rtx),
22264 : gen_rtx_LABEL_REF (VOIDmode, jump_label),
22265 : pc_rtx);
22266 2 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
22267 2 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
22268 2 : JUMP_LABEL (insn) = jump_label;
22269 :
22270 2 : emit_insn (gen_negxf2 (e2, e2));
22271 :
22272 2 : emit_label (jump_label);
22273 2 : LABEL_NUSES (jump_label) = 1;
22274 :
22275 : /* op0 = 0.5 * e2 */
22276 2 : half = force_reg (XFmode, half);
22277 2 : emit_insn (gen_mulxf3 (op0, e2, half));
22278 2 : }
22279 :
22280 : /* Output code to perform an cosh XFmode calculation. */
22281 :
22282 : void
22283 3 : ix86_emit_i387_cosh (rtx op0, rtx op1)
22284 : {
22285 3 : rtx e1 = gen_reg_rtx (XFmode);
22286 3 : rtx e2 = gen_reg_rtx (XFmode);
22287 3 : rtx half = const_double_from_real_value (dconsthalf, XFmode);
22288 3 : rtx cst1;
22289 :
22290 : /* e1 = exp (op1) */
22291 3 : emit_insn (gen_expxf2 (e1, op1));
22292 :
22293 : /* e2 = e1 + 1.0 / e1 */
22294 3 : cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
22295 3 : emit_insn (gen_divxf3 (e2, cst1, e1));
22296 3 : emit_insn (gen_addxf3 (e2, e1, e2));
22297 :
22298 : /* op0 = 0.5 * e2 */
22299 3 : half = force_reg (XFmode, half);
22300 3 : emit_insn (gen_mulxf3 (op0, e2, half));
22301 3 : }
22302 :
22303 : /* Output code to perform an tanh XFmode calculation. */
22304 :
22305 : void
22306 1 : ix86_emit_i387_tanh (rtx op0, rtx op1)
22307 : {
22308 1 : rtx e1 = gen_reg_rtx (XFmode);
22309 1 : rtx e2 = gen_reg_rtx (XFmode);
22310 1 : rtx scratch = gen_reg_rtx (HImode);
22311 1 : rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
22312 1 : rtx cst2, tmp;
22313 1 : rtx_code_label *jump_label = gen_label_rtx ();
22314 1 : rtx_insn *insn;
22315 :
22316 : /* scratch = fxam (op1) */
22317 1 : emit_insn (gen_fxamxf2_i387 (scratch, op1));
22318 :
22319 : /* e1 = expm1 (-|2 * op1|) */
22320 1 : emit_insn (gen_addxf3 (e2, op1, op1));
22321 1 : emit_insn (gen_absxf2 (e2, e2));
22322 1 : emit_insn (gen_negxf2 (e2, e2));
22323 1 : emit_insn (gen_expm1xf2 (e1, e2));
22324 :
22325 : /* e2 = e1 / (e1 + 2.0) */
22326 1 : cst2 = force_reg (XFmode, CONST2_RTX (XFmode));
22327 1 : emit_insn (gen_addxf3 (e2, e1, cst2));
22328 1 : emit_insn (gen_divxf3 (e2, e1, e2));
22329 :
22330 : /* flags = signbit (op1) */
22331 1 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
22332 :
22333 : /* if (!flags) then e2 = -e2 */
22334 1 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
22335 : gen_rtx_NE (VOIDmode, flags, const0_rtx),
22336 : gen_rtx_LABEL_REF (VOIDmode, jump_label),
22337 : pc_rtx);
22338 1 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
22339 1 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
22340 1 : JUMP_LABEL (insn) = jump_label;
22341 :
22342 1 : emit_insn (gen_negxf2 (e2, e2));
22343 :
22344 1 : emit_label (jump_label);
22345 1 : LABEL_NUSES (jump_label) = 1;
22346 :
22347 1 : emit_move_insn (op0, e2);
22348 1 : }
22349 :
22350 : /* Output code to perform an asinh XFmode calculation. */
22351 :
22352 : void
22353 0 : ix86_emit_i387_asinh (rtx op0, rtx op1)
22354 : {
22355 0 : rtx e1 = gen_reg_rtx (XFmode);
22356 0 : rtx e2 = gen_reg_rtx (XFmode);
22357 0 : rtx scratch = gen_reg_rtx (HImode);
22358 0 : rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
22359 0 : rtx cst1, tmp;
22360 0 : rtx_code_label *jump_label = gen_label_rtx ();
22361 0 : rtx_insn *insn;
22362 :
22363 : /* e2 = sqrt (op1^2 + 1.0) + 1.0 */
22364 0 : emit_insn (gen_mulxf3 (e1, op1, op1));
22365 0 : cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
22366 0 : emit_insn (gen_addxf3 (e2, e1, cst1));
22367 0 : emit_insn (gen_sqrtxf2 (e2, e2));
22368 0 : emit_insn (gen_addxf3 (e2, e2, cst1));
22369 :
22370 : /* e1 = e1 / e2 */
22371 0 : emit_insn (gen_divxf3 (e1, e1, e2));
22372 :
22373 : /* scratch = fxam (op1) */
22374 0 : emit_insn (gen_fxamxf2_i387 (scratch, op1));
22375 :
22376 : /* e1 = e1 + |op1| */
22377 0 : emit_insn (gen_absxf2 (e2, op1));
22378 0 : emit_insn (gen_addxf3 (e1, e1, e2));
22379 :
22380 : /* e2 = log1p (e1) */
22381 0 : ix86_emit_i387_log1p (e2, e1);
22382 :
22383 : /* flags = signbit (op1) */
22384 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
22385 :
22386 : /* if (flags) then e2 = -e2 */
22387 0 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
22388 : gen_rtx_EQ (VOIDmode, flags, const0_rtx),
22389 : gen_rtx_LABEL_REF (VOIDmode, jump_label),
22390 : pc_rtx);
22391 0 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
22392 0 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
22393 0 : JUMP_LABEL (insn) = jump_label;
22394 :
22395 0 : emit_insn (gen_negxf2 (e2, e2));
22396 :
22397 0 : emit_label (jump_label);
22398 0 : LABEL_NUSES (jump_label) = 1;
22399 :
22400 0 : emit_move_insn (op0, e2);
22401 0 : }
22402 :
22403 : /* Output code to perform an acosh XFmode calculation. */
22404 :
22405 : void
22406 0 : ix86_emit_i387_acosh (rtx op0, rtx op1)
22407 : {
22408 0 : rtx e1 = gen_reg_rtx (XFmode);
22409 0 : rtx e2 = gen_reg_rtx (XFmode);
22410 0 : rtx cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
22411 :
22412 : /* e2 = sqrt (op1 + 1.0) */
22413 0 : emit_insn (gen_addxf3 (e2, op1, cst1));
22414 0 : emit_insn (gen_sqrtxf2 (e2, e2));
22415 :
22416 : /* e1 = sqrt (op1 - 1.0) */
22417 0 : emit_insn (gen_subxf3 (e1, op1, cst1));
22418 0 : emit_insn (gen_sqrtxf2 (e1, e1));
22419 :
22420 : /* e1 = e1 * e2 */
22421 0 : emit_insn (gen_mulxf3 (e1, e1, e2));
22422 :
22423 : /* e1 = e1 + op1 */
22424 0 : emit_insn (gen_addxf3 (e1, e1, op1));
22425 :
22426 : /* op0 = log (e1) */
22427 0 : emit_insn (gen_logxf2 (op0, e1));
22428 0 : }
22429 :
22430 : /* Output code to perform an atanh XFmode calculation. */
22431 :
22432 : void
22433 4 : ix86_emit_i387_atanh (rtx op0, rtx op1)
22434 : {
22435 4 : rtx e1 = gen_reg_rtx (XFmode);
22436 4 : rtx e2 = gen_reg_rtx (XFmode);
22437 4 : rtx scratch = gen_reg_rtx (HImode);
22438 4 : rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
22439 4 : rtx half = const_double_from_real_value (dconsthalf, XFmode);
22440 4 : rtx cst1, tmp;
22441 4 : rtx_code_label *jump_label = gen_label_rtx ();
22442 4 : rtx_insn *insn;
22443 :
22444 : /* scratch = fxam (op1) */
22445 4 : emit_insn (gen_fxamxf2_i387 (scratch, op1));
22446 :
22447 : /* e2 = |op1| */
22448 4 : emit_insn (gen_absxf2 (e2, op1));
22449 :
22450 : /* e1 = -(e2 + e2) / (e2 + 1.0) */
22451 4 : cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
22452 4 : emit_insn (gen_addxf3 (e1, e2, cst1));
22453 4 : emit_insn (gen_addxf3 (e2, e2, e2));
22454 4 : emit_insn (gen_negxf2 (e2, e2));
22455 4 : emit_insn (gen_divxf3 (e1, e2, e1));
22456 :
22457 : /* e2 = log1p (e1) */
22458 4 : ix86_emit_i387_log1p (e2, e1);
22459 :
22460 : /* flags = signbit (op1) */
22461 4 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
22462 :
22463 : /* if (!flags) then e2 = -e2 */
22464 4 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
22465 : gen_rtx_NE (VOIDmode, flags, const0_rtx),
22466 : gen_rtx_LABEL_REF (VOIDmode, jump_label),
22467 : pc_rtx);
22468 4 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
22469 4 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
22470 4 : JUMP_LABEL (insn) = jump_label;
22471 :
22472 4 : emit_insn (gen_negxf2 (e2, e2));
22473 :
22474 4 : emit_label (jump_label);
22475 4 : LABEL_NUSES (jump_label) = 1;
22476 :
22477 : /* op0 = 0.5 * e2 */
22478 4 : half = force_reg (XFmode, half);
22479 4 : emit_insn (gen_mulxf3 (op0, e2, half));
22480 4 : }
22481 :
22482 : /* Output code to perform a log1p XFmode calculation. */
22483 :
22484 : void
22485 5 : ix86_emit_i387_log1p (rtx op0, rtx op1)
22486 : {
22487 5 : rtx_code_label *label1 = gen_label_rtx ();
22488 5 : rtx_code_label *label2 = gen_label_rtx ();
22489 :
22490 5 : rtx tmp = gen_reg_rtx (XFmode);
22491 5 : rtx res = gen_reg_rtx (XFmode);
22492 5 : rtx cst, cstln2, cst1;
22493 5 : rtx_insn *insn;
22494 :
22495 : /* The emit_jump call emits pending stack adjust, make sure it is emitted
22496 : before the conditional jump, otherwise the stack adjustment will be
22497 : only conditional. */
22498 5 : do_pending_stack_adjust ();
22499 :
22500 5 : cst = const_double_from_real_value
22501 5 : (REAL_VALUE_ATOF ("0.29289321881345247561810596348408353", XFmode), XFmode);
22502 5 : cstln2 = force_reg (XFmode, standard_80387_constant_rtx (4)); /* fldln2 */
22503 :
22504 5 : emit_insn (gen_absxf2 (tmp, op1));
22505 :
22506 5 : cst = force_reg (XFmode, cst);
22507 5 : ix86_expand_branch (GE, tmp, cst, label1);
22508 5 : predict_jump (REG_BR_PROB_BASE * 10 / 100);
22509 5 : insn = get_last_insn ();
22510 5 : JUMP_LABEL (insn) = label1;
22511 :
22512 5 : emit_insn (gen_fyl2xp1xf3_i387 (res, op1, cstln2));
22513 5 : emit_jump (label2);
22514 :
22515 5 : emit_label (label1);
22516 5 : LABEL_NUSES (label1) = 1;
22517 :
22518 5 : cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
22519 5 : emit_insn (gen_rtx_SET (tmp, gen_rtx_PLUS (XFmode, op1, cst1)));
22520 5 : emit_insn (gen_fyl2xxf3_i387 (res, tmp, cstln2));
22521 :
22522 5 : emit_label (label2);
22523 5 : LABEL_NUSES (label2) = 1;
22524 :
22525 5 : emit_move_insn (op0, res);
22526 5 : }
22527 :
22528 : /* Emit code for round calculation. */
22529 : void
22530 60 : ix86_emit_i387_round (rtx op0, rtx op1)
22531 : {
22532 60 : machine_mode inmode = GET_MODE (op1);
22533 60 : machine_mode outmode = GET_MODE (op0);
22534 60 : rtx e1 = gen_reg_rtx (XFmode);
22535 60 : rtx e2 = gen_reg_rtx (XFmode);
22536 60 : rtx scratch = gen_reg_rtx (HImode);
22537 60 : rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
22538 60 : rtx half = const_double_from_real_value (dconsthalf, XFmode);
22539 60 : rtx res = gen_reg_rtx (outmode);
22540 60 : rtx_code_label *jump_label = gen_label_rtx ();
22541 60 : rtx (*floor_insn) (rtx, rtx);
22542 60 : rtx (*neg_insn) (rtx, rtx);
22543 60 : rtx_insn *insn;
22544 60 : rtx tmp;
22545 :
22546 60 : switch (inmode)
22547 : {
22548 29 : case E_SFmode:
22549 29 : case E_DFmode:
22550 29 : tmp = gen_reg_rtx (XFmode);
22551 :
22552 29 : emit_insn (gen_rtx_SET (tmp, gen_rtx_FLOAT_EXTEND (XFmode, op1)));
22553 29 : op1 = tmp;
22554 29 : break;
22555 : case E_XFmode:
22556 : break;
22557 0 : default:
22558 0 : gcc_unreachable ();
22559 : }
22560 :
22561 60 : switch (outmode)
22562 : {
22563 : case E_SFmode:
22564 : floor_insn = gen_frndintxf2_floor;
22565 : neg_insn = gen_negsf2;
22566 : break;
22567 6 : case E_DFmode:
22568 6 : floor_insn = gen_frndintxf2_floor;
22569 6 : neg_insn = gen_negdf2;
22570 6 : break;
22571 10 : case E_XFmode:
22572 10 : floor_insn = gen_frndintxf2_floor;
22573 10 : neg_insn = gen_negxf2;
22574 10 : break;
22575 0 : case E_HImode:
22576 0 : floor_insn = gen_lfloorxfhi2;
22577 0 : neg_insn = gen_neghi2;
22578 0 : break;
22579 6 : case E_SImode:
22580 6 : floor_insn = gen_lfloorxfsi2;
22581 6 : neg_insn = gen_negsi2;
22582 6 : break;
22583 36 : case E_DImode:
22584 36 : floor_insn = gen_lfloorxfdi2;
22585 36 : neg_insn = gen_negdi2;
22586 36 : break;
22587 0 : default:
22588 0 : gcc_unreachable ();
22589 : }
22590 :
22591 : /* round(a) = sgn(a) * floor(fabs(a) + 0.5) */
22592 :
22593 : /* scratch = fxam(op1) */
22594 60 : emit_insn (gen_fxamxf2_i387 (scratch, op1));
22595 :
22596 : /* e1 = fabs(op1) */
22597 60 : emit_insn (gen_absxf2 (e1, op1));
22598 :
22599 : /* e2 = e1 + 0.5 */
22600 60 : half = force_reg (XFmode, half);
22601 60 : emit_insn (gen_rtx_SET (e2, gen_rtx_PLUS (XFmode, e1, half)));
22602 :
22603 : /* res = floor(e2) */
22604 60 : switch (outmode)
22605 : {
22606 8 : case E_SFmode:
22607 8 : case E_DFmode:
22608 8 : {
22609 8 : tmp = gen_reg_rtx (XFmode);
22610 :
22611 8 : emit_insn (floor_insn (tmp, e2));
22612 8 : emit_insn (gen_rtx_SET (res,
22613 : gen_rtx_UNSPEC (outmode, gen_rtvec (1, tmp),
22614 : UNSPEC_TRUNC_NOOP)));
22615 : }
22616 8 : break;
22617 52 : default:
22618 52 : emit_insn (floor_insn (res, e2));
22619 : }
22620 :
22621 : /* flags = signbit(a) */
22622 60 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
22623 :
22624 : /* if (flags) then res = -res */
22625 60 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
22626 : gen_rtx_EQ (VOIDmode, flags, const0_rtx),
22627 : gen_rtx_LABEL_REF (VOIDmode, jump_label),
22628 : pc_rtx);
22629 60 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
22630 60 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
22631 60 : JUMP_LABEL (insn) = jump_label;
22632 :
22633 60 : emit_insn (neg_insn (res, res));
22634 :
22635 60 : emit_label (jump_label);
22636 60 : LABEL_NUSES (jump_label) = 1;
22637 :
22638 60 : emit_move_insn (op0, res);
22639 60 : }
22640 :
22641 : /* Output code to perform a Newton-Rhapson approximation of a single precision
22642 : floating point divide [http://en.wikipedia.org/wiki/N-th_root_algorithm]. */
22643 :
22644 : void
22645 56 : ix86_emit_swdivsf (rtx res, rtx a, rtx b, machine_mode mode)
22646 : {
22647 56 : rtx x0, x1, e0, e1;
22648 :
22649 56 : x0 = gen_reg_rtx (mode);
22650 56 : e0 = gen_reg_rtx (mode);
22651 56 : e1 = gen_reg_rtx (mode);
22652 56 : x1 = gen_reg_rtx (mode);
22653 :
22654 56 : b = force_reg (mode, b);
22655 :
22656 : /* x0 = rcp(b) estimate */
22657 56 : if (mode == V16SFmode || mode == V8DFmode)
22658 : {
22659 0 : emit_insn (gen_rtx_SET (x0, gen_rtx_UNSPEC (mode, gen_rtvec (1, b),
22660 : UNSPEC_RCP14)));
22661 : }
22662 : else
22663 56 : emit_insn (gen_rtx_SET (x0, gen_rtx_UNSPEC (mode, gen_rtvec (1, b),
22664 : UNSPEC_RCP)));
22665 :
22666 56 : unsigned vector_size = GET_MODE_SIZE (mode);
22667 :
22668 : /* (a - (rcp(b) * a * b)) * rcp(b) + rcp(b) * a
22669 : N-R step with 2 fma implementation. */
22670 56 : if (TARGET_FMA
22671 55 : || (TARGET_AVX512F && vector_size == 64)
22672 55 : || (TARGET_AVX512VL && (vector_size == 32 || vector_size == 16)))
22673 : {
22674 : /* e0 = x0 * a */
22675 1 : emit_insn (gen_rtx_SET (e0, gen_rtx_MULT (mode, x0, a)));
22676 : /* e1 = e0 * b - a */
22677 1 : emit_insn (gen_rtx_SET (e1, gen_rtx_FMA (mode, e0, b,
22678 : gen_rtx_NEG (mode, a))));
22679 : /* res = - e1 * x0 + e0 */
22680 1 : emit_insn (gen_rtx_SET (res, gen_rtx_FMA (mode,
22681 : gen_rtx_NEG (mode, e1),
22682 : x0, e0)));
22683 : }
22684 : else
22685 : /* a / b = a * ((rcp(b) + rcp(b)) - (b * rcp(b) * rcp (b))) */
22686 : {
22687 : /* e0 = x0 * b */
22688 55 : emit_insn (gen_rtx_SET (e0, gen_rtx_MULT (mode, x0, b)));
22689 :
22690 : /* e1 = x0 + x0 */
22691 55 : emit_insn (gen_rtx_SET (e1, gen_rtx_PLUS (mode, x0, x0)));
22692 :
22693 : /* e0 = x0 * e0 */
22694 55 : emit_insn (gen_rtx_SET (e0, gen_rtx_MULT (mode, x0, e0)));
22695 :
22696 : /* x1 = e1 - e0 */
22697 55 : emit_insn (gen_rtx_SET (x1, gen_rtx_MINUS (mode, e1, e0)));
22698 :
22699 : /* res = a * x1 */
22700 55 : emit_insn (gen_rtx_SET (res, gen_rtx_MULT (mode, a, x1)));
22701 : }
22702 56 : }
22703 :
22704 : /* Output code to perform a Newton-Rhapson approximation of a
22705 : single precision floating point [reciprocal] square root. */
22706 :
22707 : void
22708 86 : ix86_emit_swsqrtsf (rtx res, rtx a, machine_mode mode, bool recip)
22709 : {
22710 86 : rtx x0, e0, e1, e2, e3, mthree, mhalf;
22711 86 : REAL_VALUE_TYPE r;
22712 86 : int unspec;
22713 :
22714 86 : x0 = gen_reg_rtx (mode);
22715 86 : e0 = gen_reg_rtx (mode);
22716 86 : e1 = gen_reg_rtx (mode);
22717 86 : e2 = gen_reg_rtx (mode);
22718 86 : e3 = gen_reg_rtx (mode);
22719 :
22720 86 : real_from_integer (&r, VOIDmode, -3, SIGNED);
22721 86 : mthree = const_double_from_real_value (r, SFmode);
22722 :
22723 86 : real_arithmetic (&r, NEGATE_EXPR, &dconsthalf, NULL);
22724 86 : mhalf = const_double_from_real_value (r, SFmode);
22725 86 : unspec = UNSPEC_RSQRT;
22726 :
22727 86 : if (VECTOR_MODE_P (mode))
22728 : {
22729 66 : mthree = ix86_build_const_vector (mode, true, mthree);
22730 66 : mhalf = ix86_build_const_vector (mode, true, mhalf);
22731 : /* There is no 512-bit rsqrt. There is however rsqrt14. */
22732 132 : if (GET_MODE_SIZE (mode) == 64)
22733 0 : unspec = UNSPEC_RSQRT14;
22734 : }
22735 :
22736 : /* sqrt(a) = -0.5 * a * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0)
22737 : rsqrt(a) = -0.5 * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0) */
22738 :
22739 86 : a = force_reg (mode, a);
22740 :
22741 : /* x0 = rsqrt(a) estimate */
22742 86 : emit_insn (gen_rtx_SET (x0, gen_rtx_UNSPEC (mode, gen_rtvec (1, a),
22743 : unspec)));
22744 :
22745 : /* If (a == 0.0) Filter out infinity to prevent NaN for sqrt(0.0). */
22746 86 : if (!recip)
22747 : {
22748 57 : rtx zero = force_reg (mode, CONST0_RTX(mode));
22749 57 : rtx mask;
22750 :
22751 : /* Handle masked compare. */
22752 110 : if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 64)
22753 : {
22754 0 : mask = gen_reg_rtx (HImode);
22755 : /* Imm value 0x4 corresponds to not-equal comparison. */
22756 0 : emit_insn (gen_avx512f_cmpv16sf3 (mask, zero, a, GEN_INT (0x4)));
22757 0 : emit_insn (gen_avx512f_blendmv16sf (x0, zero, x0, mask));
22758 : }
22759 : else
22760 : {
22761 57 : mask = gen_reg_rtx (mode);
22762 57 : emit_insn (gen_rtx_SET (mask, gen_rtx_NE (mode, zero, a)));
22763 57 : emit_insn (gen_rtx_SET (x0, gen_rtx_AND (mode, x0, mask)));
22764 : }
22765 : }
22766 :
22767 86 : mthree = force_reg (mode, mthree);
22768 :
22769 : /* e0 = x0 * a */
22770 86 : emit_insn (gen_rtx_SET (e0, gen_rtx_MULT (mode, x0, a)));
22771 :
22772 86 : unsigned vector_size = GET_MODE_SIZE (mode);
22773 86 : if (TARGET_FMA
22774 78 : || (TARGET_AVX512F && vector_size == 64)
22775 78 : || (TARGET_AVX512VL && (vector_size == 32 || vector_size == 16)))
22776 16 : emit_insn (gen_rtx_SET (e2,
22777 : gen_rtx_FMA (mode, e0, x0, mthree)));
22778 : else
22779 : {
22780 : /* e1 = e0 * x0 */
22781 70 : emit_insn (gen_rtx_SET (e1, gen_rtx_MULT (mode, e0, x0)));
22782 :
22783 : /* e2 = e1 - 3. */
22784 70 : emit_insn (gen_rtx_SET (e2, gen_rtx_PLUS (mode, e1, mthree)));
22785 : }
22786 :
22787 86 : mhalf = force_reg (mode, mhalf);
22788 86 : if (recip)
22789 : /* e3 = -.5 * x0 */
22790 29 : emit_insn (gen_rtx_SET (e3, gen_rtx_MULT (mode, x0, mhalf)));
22791 : else
22792 : /* e3 = -.5 * e0 */
22793 57 : emit_insn (gen_rtx_SET (e3, gen_rtx_MULT (mode, e0, mhalf)));
22794 : /* ret = e2 * e3 */
22795 86 : emit_insn (gen_rtx_SET (res, gen_rtx_MULT (mode, e2, e3)));
22796 86 : }
22797 :
22798 : /* Expand fabs (OP0) and return a new rtx that holds the result. The
22799 : mask for masking out the sign-bit is stored in *SMASK, if that is
22800 : non-null. */
22801 :
22802 : static rtx
22803 1091 : ix86_expand_sse_fabs (rtx op0, rtx *smask)
22804 : {
22805 1091 : machine_mode vmode, mode = GET_MODE (op0);
22806 1091 : rtx xa, mask;
22807 :
22808 1091 : xa = gen_reg_rtx (mode);
22809 1091 : if (mode == SFmode)
22810 : vmode = V4SFmode;
22811 495 : else if (mode == DFmode)
22812 : vmode = V2DFmode;
22813 : else
22814 0 : vmode = mode;
22815 1091 : mask = ix86_build_signbit_mask (vmode, VECTOR_MODE_P (mode), true);
22816 1091 : if (!VECTOR_MODE_P (mode))
22817 : {
22818 : /* We need to generate a scalar mode mask in this case. */
22819 1091 : rtx tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
22820 1091 : tmp = gen_rtx_VEC_SELECT (mode, mask, tmp);
22821 1091 : mask = gen_reg_rtx (mode);
22822 1091 : emit_insn (gen_rtx_SET (mask, tmp));
22823 : }
22824 1091 : emit_insn (gen_rtx_SET (xa, gen_rtx_AND (mode, op0, mask)));
22825 :
22826 1091 : if (smask)
22827 1026 : *smask = mask;
22828 :
22829 1091 : return xa;
22830 : }
22831 :
22832 : /* Expands a comparison of OP0 with OP1 using comparison code CODE,
22833 : swapping the operands if SWAP_OPERANDS is true. The expanded
22834 : code is a forward jump to a newly created label in case the
22835 : comparison is true. The generated label rtx is returned. */
22836 : static rtx_code_label *
22837 1095 : ix86_expand_sse_compare_and_jump (enum rtx_code code, rtx op0, rtx op1,
22838 : bool swap_operands)
22839 : {
22840 1095 : bool unordered_compare = ix86_unordered_fp_compare (code);
22841 1095 : rtx_code_label *label;
22842 1095 : rtx tmp, reg;
22843 :
22844 1095 : if (swap_operands)
22845 35 : std::swap (op0, op1);
22846 :
22847 1095 : label = gen_label_rtx ();
22848 1095 : tmp = gen_rtx_COMPARE (CCFPmode, op0, op1);
22849 1095 : if (unordered_compare)
22850 939 : tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_NOTRAP);
22851 1095 : reg = gen_rtx_REG (CCFPmode, FLAGS_REG);
22852 1095 : emit_insn (gen_rtx_SET (reg, tmp));
22853 1095 : tmp = gen_rtx_fmt_ee (code, VOIDmode, reg, const0_rtx);
22854 1095 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
22855 : gen_rtx_LABEL_REF (VOIDmode, label), pc_rtx);
22856 1095 : rtx_insn *jump = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
22857 1095 : JUMP_LABEL (jump) = label;
22858 :
22859 1095 : return label;
22860 : }
22861 :
22862 : /* Expand a mask generating SSE comparison instruction comparing OP0 with OP1
22863 : using comparison code CODE. Operands are swapped for the comparison if
22864 : SWAP_OPERANDS is true. Returns a rtx for the generated mask. */
22865 : static rtx
22866 554 : ix86_expand_sse_compare_mask (enum rtx_code code, rtx op0, rtx op1,
22867 : bool swap_operands)
22868 : {
22869 554 : rtx (*insn)(rtx, rtx, rtx, rtx);
22870 554 : machine_mode mode = GET_MODE (op0);
22871 554 : rtx mask = gen_reg_rtx (mode);
22872 :
22873 554 : if (swap_operands)
22874 365 : std::swap (op0, op1);
22875 :
22876 554 : insn = mode == DFmode ? gen_setcc_df_sse : gen_setcc_sf_sse;
22877 :
22878 554 : emit_insn (insn (mask, op0, op1,
22879 : gen_rtx_fmt_ee (code, mode, op0, op1)));
22880 554 : return mask;
22881 : }
22882 :
22883 : /* Expand copysign from SIGN to the positive value ABS_VALUE
22884 : storing in RESULT. If MASK is non-null, it shall be a mask to mask out
22885 : the sign-bit. */
22886 :
22887 : static void
22888 1058 : ix86_sse_copysign_to_positive (rtx result, rtx abs_value, rtx sign, rtx mask)
22889 : {
22890 1058 : machine_mode mode = GET_MODE (sign);
22891 1058 : rtx sgn = gen_reg_rtx (mode);
22892 1058 : if (mask == NULL_RTX)
22893 : {
22894 28 : machine_mode vmode;
22895 :
22896 28 : if (mode == SFmode)
22897 : vmode = V4SFmode;
22898 : else if (mode == DFmode)
22899 : vmode = V2DFmode;
22900 : else if (mode == HFmode)
22901 : vmode = V8HFmode;
22902 : else
22903 28 : vmode = mode;
22904 :
22905 28 : mask = ix86_build_signbit_mask (vmode, VECTOR_MODE_P (mode), false);
22906 28 : if (!VECTOR_MODE_P (mode))
22907 : {
22908 : /* We need to generate a scalar mode mask in this case. */
22909 28 : rtx tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
22910 28 : tmp = gen_rtx_VEC_SELECT (mode, mask, tmp);
22911 28 : mask = gen_reg_rtx (mode);
22912 28 : emit_insn (gen_rtx_SET (mask, tmp));
22913 : }
22914 : }
22915 : else
22916 1030 : mask = gen_rtx_NOT (mode, mask);
22917 1058 : emit_insn (gen_rtx_SET (sgn, gen_rtx_AND (mode, mask, sign)));
22918 1058 : emit_insn (gen_rtx_SET (result, gen_rtx_IOR (mode, abs_value, sgn)));
22919 1058 : }
22920 :
22921 : /* Expand SSE sequence for computing lround from OP1 storing
22922 : into OP0. */
22923 :
22924 : void
22925 28 : ix86_expand_lround (rtx op0, rtx op1)
22926 : {
22927 : /* C code for the stuff we're doing below:
22928 : tmp = op1 + copysign (nextafter (0.5, 0.0), op1)
22929 : return (long)tmp;
22930 : */
22931 28 : machine_mode mode = GET_MODE (op1);
22932 28 : const struct real_format *fmt;
22933 28 : REAL_VALUE_TYPE pred_half, half_minus_pred_half;
22934 28 : rtx adj;
22935 :
22936 : /* load nextafter (0.5, 0.0) */
22937 28 : fmt = REAL_MODE_FORMAT (mode);
22938 28 : real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
22939 28 : real_arithmetic (&pred_half, MINUS_EXPR, &dconsthalf, &half_minus_pred_half);
22940 :
22941 : /* adj = copysign (0.5, op1) */
22942 28 : adj = force_reg (mode, const_double_from_real_value (pred_half, mode));
22943 28 : ix86_sse_copysign_to_positive (adj, adj, force_reg (mode, op1), NULL_RTX);
22944 :
22945 : /* adj = op1 + adj */
22946 28 : adj = expand_simple_binop (mode, PLUS, adj, op1, NULL_RTX, 0, OPTAB_DIRECT);
22947 :
22948 : /* op0 = (imode)adj */
22949 28 : expand_fix (op0, adj, 0);
22950 28 : }
22951 :
22952 : /* Expand SSE2 sequence for computing lround from OPERAND1 storing
22953 : into OPERAND0. */
22954 :
22955 : void
22956 69 : ix86_expand_lfloorceil (rtx op0, rtx op1, bool do_floor)
22957 : {
22958 : /* C code for the stuff we're doing below (for do_floor):
22959 : xi = (long)op1;
22960 : xi -= (double)xi > op1 ? 1 : 0;
22961 : return xi;
22962 : */
22963 69 : machine_mode fmode = GET_MODE (op1);
22964 69 : machine_mode imode = GET_MODE (op0);
22965 69 : rtx ireg, freg, tmp;
22966 69 : rtx_code_label *label;
22967 :
22968 : /* reg = (long)op1 */
22969 69 : ireg = gen_reg_rtx (imode);
22970 69 : expand_fix (ireg, op1, 0);
22971 :
22972 : /* freg = (double)reg */
22973 69 : freg = gen_reg_rtx (fmode);
22974 69 : expand_float (freg, ireg, 0);
22975 :
22976 : /* ireg = (freg > op1) ? ireg - 1 : ireg */
22977 69 : label = ix86_expand_sse_compare_and_jump (UNLE,
22978 : freg, op1, !do_floor);
22979 104 : tmp = expand_simple_binop (imode, do_floor ? MINUS : PLUS,
22980 : ireg, const1_rtx, NULL_RTX, 0, OPTAB_DIRECT);
22981 69 : emit_move_insn (ireg, tmp);
22982 :
22983 69 : emit_label (label);
22984 69 : LABEL_NUSES (label) = 1;
22985 :
22986 69 : emit_move_insn (op0, ireg);
22987 69 : }
22988 :
22989 : /* Generate and return a rtx of mode MODE for 2**n where n is the number
22990 : of bits of the mantissa of MODE, which must be one of DFmode or SFmode. */
22991 :
22992 : static rtx
22993 1026 : ix86_gen_TWO52 (machine_mode mode)
22994 : {
22995 1026 : const struct real_format *fmt;
22996 1026 : REAL_VALUE_TYPE TWO52r;
22997 1026 : rtx TWO52;
22998 :
22999 1026 : fmt = REAL_MODE_FORMAT (mode);
23000 1026 : real_2expN (&TWO52r, fmt->p - 1, mode);
23001 1026 : TWO52 = const_double_from_real_value (TWO52r, mode);
23002 1026 : TWO52 = force_reg (mode, TWO52);
23003 :
23004 1026 : return TWO52;
23005 : }
23006 :
23007 : /* Expand rint rounding OPERAND1 and storing the result in OPERAND0. */
23008 :
23009 : void
23010 133 : ix86_expand_rint (rtx operand0, rtx operand1)
23011 : {
23012 : /* C code for the stuff we're doing below:
23013 : xa = fabs (operand1);
23014 : if (!isless (xa, 2**52))
23015 : return operand1;
23016 : two52 = 2**52;
23017 : if (flag_rounding_math)
23018 : {
23019 : two52 = copysign (two52, operand1);
23020 : xa = operand1;
23021 : }
23022 : xa = xa + two52 - two52;
23023 : return copysign (xa, operand1);
23024 : */
23025 133 : machine_mode mode = GET_MODE (operand0);
23026 133 : rtx res, xa, TWO52, mask;
23027 133 : rtx_code_label *label;
23028 :
23029 133 : TWO52 = ix86_gen_TWO52 (mode);
23030 :
23031 : /* Temporary for holding the result, initialized to the input
23032 : operand to ease control flow. */
23033 133 : res = copy_to_reg (operand1);
23034 :
23035 : /* xa = abs (operand1) */
23036 133 : xa = ix86_expand_sse_fabs (res, &mask);
23037 :
23038 : /* if (!isless (xa, TWO52)) goto label; */
23039 133 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
23040 :
23041 133 : if (flag_rounding_math)
23042 : {
23043 65 : ix86_sse_copysign_to_positive (TWO52, TWO52, res, mask);
23044 65 : xa = res;
23045 : }
23046 :
23047 133 : xa = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
23048 133 : xa = expand_simple_binop (mode, MINUS, xa, TWO52, xa, 0, OPTAB_DIRECT);
23049 :
23050 : /* Remove the sign with FE_DOWNWARD, where x - x = -0.0. */
23051 133 : if (HONOR_SIGNED_ZEROS (mode) && flag_rounding_math)
23052 65 : xa = ix86_expand_sse_fabs (xa, NULL);
23053 :
23054 133 : ix86_sse_copysign_to_positive (res, xa, res, mask);
23055 :
23056 133 : emit_label (label);
23057 133 : LABEL_NUSES (label) = 1;
23058 :
23059 133 : emit_move_insn (operand0, res);
23060 133 : }
23061 :
23062 : /* Expand SSE2 sequence for computing floor or ceil
23063 : from OPERAND1 storing into OPERAND0. */
23064 : void
23065 554 : ix86_expand_floorceil (rtx operand0, rtx operand1, bool do_floor)
23066 : {
23067 : /* C code for the stuff we expand below.
23068 : double xa = fabs (x), x2;
23069 : if (!isless (xa, TWO52))
23070 : return x;
23071 : x2 = (double)(long)x;
23072 :
23073 : Compensate. Floor:
23074 : if (x2 > x)
23075 : x2 -= 1;
23076 : Compensate. Ceil:
23077 : if (x2 < x)
23078 : x2 += 1;
23079 :
23080 : if (HONOR_SIGNED_ZEROS (mode))
23081 : return copysign (x2, x);
23082 : return x2;
23083 : */
23084 554 : machine_mode mode = GET_MODE (operand0);
23085 554 : rtx xa, xi, TWO52, tmp, one, res, mask;
23086 554 : rtx_code_label *label;
23087 :
23088 554 : TWO52 = ix86_gen_TWO52 (mode);
23089 :
23090 : /* Temporary for holding the result, initialized to the input
23091 : operand to ease control flow. */
23092 554 : res = copy_to_reg (operand1);
23093 :
23094 : /* xa = abs (operand1) */
23095 554 : xa = ix86_expand_sse_fabs (res, &mask);
23096 :
23097 : /* if (!isless (xa, TWO52)) goto label; */
23098 554 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
23099 :
23100 : /* xa = (double)(long)x */
23101 554 : xi = gen_reg_rtx (int_mode_for_mode (mode).require ());
23102 554 : expand_fix (xi, res, 0);
23103 554 : expand_float (xa, xi, 0);
23104 :
23105 : /* generate 1.0 */
23106 554 : one = force_reg (mode, const_double_from_real_value (dconst1, mode));
23107 :
23108 : /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
23109 554 : tmp = ix86_expand_sse_compare_mask (UNGT, xa, res, !do_floor);
23110 554 : emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, one, tmp)));
23111 919 : tmp = expand_simple_binop (mode, do_floor ? MINUS : PLUS,
23112 : xa, tmp, NULL_RTX, 0, OPTAB_DIRECT);
23113 554 : if (HONOR_SIGNED_ZEROS (mode))
23114 : {
23115 : /* Remove the sign with FE_DOWNWARD, where x - x = -0.0. */
23116 507 : if (do_floor && flag_rounding_math)
23117 0 : tmp = ix86_expand_sse_fabs (tmp, NULL);
23118 :
23119 507 : ix86_sse_copysign_to_positive (tmp, tmp, res, mask);
23120 : }
23121 554 : emit_move_insn (res, tmp);
23122 :
23123 554 : emit_label (label);
23124 554 : LABEL_NUSES (label) = 1;
23125 :
23126 554 : emit_move_insn (operand0, res);
23127 554 : }
23128 :
23129 : /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
23130 : into OPERAND0 without relying on DImode truncation via cvttsd2siq
23131 : that is only available on 64bit targets. */
23132 : void
23133 0 : ix86_expand_floorceildf_32 (rtx operand0, rtx operand1, bool do_floor)
23134 : {
23135 : /* C code for the stuff we expand below.
23136 : double xa = fabs (x), x2;
23137 : if (!isless (xa, TWO52))
23138 : return x;
23139 : xa = xa + TWO52 - TWO52;
23140 : x2 = copysign (xa, x);
23141 :
23142 : Compensate. Floor:
23143 : if (x2 > x)
23144 : x2 -= 1;
23145 : Compensate. Ceil:
23146 : if (x2 < x)
23147 : x2 += 1;
23148 :
23149 : if (HONOR_SIGNED_ZEROS (mode))
23150 : x2 = copysign (x2, x);
23151 : return x2;
23152 : */
23153 0 : machine_mode mode = GET_MODE (operand0);
23154 0 : rtx xa, TWO52, tmp, one, res, mask;
23155 0 : rtx_code_label *label;
23156 :
23157 0 : TWO52 = ix86_gen_TWO52 (mode);
23158 :
23159 : /* Temporary for holding the result, initialized to the input
23160 : operand to ease control flow. */
23161 0 : res = copy_to_reg (operand1);
23162 :
23163 : /* xa = abs (operand1) */
23164 0 : xa = ix86_expand_sse_fabs (res, &mask);
23165 :
23166 : /* if (!isless (xa, TWO52)) goto label; */
23167 0 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
23168 :
23169 : /* xa = xa + TWO52 - TWO52; */
23170 0 : xa = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
23171 0 : xa = expand_simple_binop (mode, MINUS, xa, TWO52, xa, 0, OPTAB_DIRECT);
23172 :
23173 : /* xa = copysign (xa, operand1) */
23174 0 : ix86_sse_copysign_to_positive (xa, xa, res, mask);
23175 :
23176 : /* generate 1.0 */
23177 0 : one = force_reg (mode, const_double_from_real_value (dconst1, mode));
23178 :
23179 : /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
23180 0 : tmp = ix86_expand_sse_compare_mask (UNGT, xa, res, !do_floor);
23181 0 : emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, one, tmp)));
23182 0 : tmp = expand_simple_binop (mode, do_floor ? MINUS : PLUS,
23183 : xa, tmp, NULL_RTX, 0, OPTAB_DIRECT);
23184 0 : if (HONOR_SIGNED_ZEROS (mode))
23185 : {
23186 : /* Remove the sign with FE_DOWNWARD, where x - x = -0.0. */
23187 0 : if (do_floor && flag_rounding_math)
23188 0 : tmp = ix86_expand_sse_fabs (tmp, NULL);
23189 :
23190 0 : ix86_sse_copysign_to_positive (tmp, tmp, res, mask);
23191 : }
23192 0 : emit_move_insn (res, tmp);
23193 :
23194 0 : emit_label (label);
23195 0 : LABEL_NUSES (label) = 1;
23196 :
23197 0 : emit_move_insn (operand0, res);
23198 0 : }
23199 :
23200 : /* Expand SSE sequence for computing trunc
23201 : from OPERAND1 storing into OPERAND0. */
23202 : void
23203 325 : ix86_expand_trunc (rtx operand0, rtx operand1)
23204 : {
23205 : /* C code for SSE variant we expand below.
23206 : double xa = fabs (x), x2;
23207 : if (!isless (xa, TWO52))
23208 : return x;
23209 : x2 = (double)(long)x;
23210 : if (HONOR_SIGNED_ZEROS (mode))
23211 : return copysign (x2, x);
23212 : return x2;
23213 : */
23214 325 : machine_mode mode = GET_MODE (operand0);
23215 325 : rtx xa, xi, TWO52, res, mask;
23216 325 : rtx_code_label *label;
23217 :
23218 325 : TWO52 = ix86_gen_TWO52 (mode);
23219 :
23220 : /* Temporary for holding the result, initialized to the input
23221 : operand to ease control flow. */
23222 325 : res = copy_to_reg (operand1);
23223 :
23224 : /* xa = abs (operand1) */
23225 325 : xa = ix86_expand_sse_fabs (res, &mask);
23226 :
23227 : /* if (!isless (xa, TWO52)) goto label; */
23228 325 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
23229 :
23230 : /* xa = (double)(long)x */
23231 325 : xi = gen_reg_rtx (int_mode_for_mode (mode).require ());
23232 325 : expand_fix (xi, res, 0);
23233 325 : expand_float (xa, xi, 0);
23234 :
23235 325 : if (HONOR_SIGNED_ZEROS (mode))
23236 311 : ix86_sse_copysign_to_positive (xa, xa, res, mask);
23237 :
23238 325 : emit_move_insn (res, xa);
23239 :
23240 325 : emit_label (label);
23241 325 : LABEL_NUSES (label) = 1;
23242 :
23243 325 : emit_move_insn (operand0, res);
23244 325 : }
23245 :
23246 : /* Expand SSE sequence for computing trunc from OPERAND1 storing
23247 : into OPERAND0 without relying on DImode truncation via cvttsd2siq
23248 : that is only available on 64bit targets. */
23249 : void
23250 0 : ix86_expand_truncdf_32 (rtx operand0, rtx operand1)
23251 : {
23252 0 : machine_mode mode = GET_MODE (operand0);
23253 0 : rtx xa, xa2, TWO52, tmp, one, res, mask;
23254 0 : rtx_code_label *label;
23255 :
23256 : /* C code for SSE variant we expand below.
23257 : double xa = fabs (x), x2;
23258 : if (!isless (xa, TWO52))
23259 : return x;
23260 : xa2 = xa + TWO52 - TWO52;
23261 : Compensate:
23262 : if (xa2 > xa)
23263 : xa2 -= 1.0;
23264 : x2 = copysign (xa2, x);
23265 : return x2;
23266 : */
23267 :
23268 0 : TWO52 = ix86_gen_TWO52 (mode);
23269 :
23270 : /* Temporary for holding the result, initialized to the input
23271 : operand to ease control flow. */
23272 0 : res =copy_to_reg (operand1);
23273 :
23274 : /* xa = abs (operand1) */
23275 0 : xa = ix86_expand_sse_fabs (res, &mask);
23276 :
23277 : /* if (!isless (xa, TWO52)) goto label; */
23278 0 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
23279 :
23280 : /* xa2 = xa + TWO52 - TWO52; */
23281 0 : xa2 = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
23282 0 : xa2 = expand_simple_binop (mode, MINUS, xa2, TWO52, xa2, 0, OPTAB_DIRECT);
23283 :
23284 : /* generate 1.0 */
23285 0 : one = force_reg (mode, const_double_from_real_value (dconst1, mode));
23286 :
23287 : /* Compensate: xa2 = xa2 - (xa2 > xa ? 1 : 0) */
23288 0 : tmp = ix86_expand_sse_compare_mask (UNGT, xa2, xa, false);
23289 0 : emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, one, tmp)));
23290 0 : tmp = expand_simple_binop (mode, MINUS,
23291 : xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
23292 : /* Remove the sign with FE_DOWNWARD, where x - x = -0.0. */
23293 0 : if (HONOR_SIGNED_ZEROS (mode) && flag_rounding_math)
23294 0 : tmp = ix86_expand_sse_fabs (tmp, NULL);
23295 :
23296 : /* res = copysign (xa2, operand1) */
23297 0 : ix86_sse_copysign_to_positive (res, tmp, res, mask);
23298 :
23299 0 : emit_label (label);
23300 0 : LABEL_NUSES (label) = 1;
23301 :
23302 0 : emit_move_insn (operand0, res);
23303 0 : }
23304 :
23305 : /* Expand truncsfbf2 like vcvtneps2bf16, which doesn't honor SNAN,
23306 : turns sNAN into qNAN quietly, it always rounds to nearest even
23307 : and flushes denormals to zero. We can expand the conversion
23308 : inline as (fromi + 0x7fff + ((fromi >> 16) & 1)) >> 16, flushing
23309 : denormals to zero. */
23310 :
23311 : void
23312 74 : ix86_expand_truncsfbf2 (rtx op0, rtx op1)
23313 : {
23314 74 : rtx set;
23315 :
23316 74 : if (TARGET_AVXNECONVERT
23317 72 : || (TARGET_AVX512BF16 && TARGET_AVX512VL))
23318 : {
23319 71 : set = gen_truncsfbf2_vcvtneps2bf16 (op0, op1);
23320 71 : emit_insn (set);
23321 71 : return;
23322 : }
23323 :
23324 : /* Convert OP1 to REG1_SI in SImode. */
23325 3 : rtx reg1_si = gen_reg_rtx (SImode);
23326 3 : rtx op1_si = gen_lowpart (SImode, op1);
23327 3 : set = gen_rtx_SET (reg1_si, op1_si);
23328 3 : emit_insn (set);
23329 :
23330 : /* Set TMP0 to REG1_SI >> 16. */
23331 3 : rtx tmp0 = expand_simple_binop (SImode, LSHIFTRT, reg1_si,
23332 : GEN_INT(16), nullptr, 0,
23333 : OPTAB_DIRECT);
23334 :
23335 3 : rtx_code_label *zero_label = gen_label_rtx ();
23336 3 : rtx_code_label *cast_label = gen_label_rtx ();
23337 :
23338 3 : rtx result = gen_reg_rtx (SImode);
23339 :
23340 : /* OP1 is zero or denormal if (REG1_SI & 0x7f800000) == 0. */
23341 3 : rtx tmp1 = expand_simple_binop (SImode, AND, reg1_si,
23342 : GEN_INT(0x7f800000), nullptr, 0,
23343 : OPTAB_DIRECT);
23344 :
23345 3 : emit_cmp_and_jump_insns (tmp1, const0_rtx, EQ, nullptr, SImode,
23346 : true, zero_label,
23347 : profile_probability::unlikely ());
23348 :
23349 : /* Set TMP1 to TMP0 & 1. */
23350 3 : tmp1 = expand_simple_binop (SImode, AND, tmp0, const1_rtx, nullptr,
23351 : 0, OPTAB_DIRECT);
23352 :
23353 : /* Set TMP2 to REG1_SI + 0x7fff. */
23354 3 : rtx tmp2 = expand_simple_binop (SImode, PLUS, reg1_si,
23355 : GEN_INT (0x7fff), nullptr, 0,
23356 : OPTAB_DIRECT);
23357 : /* Set TMP1 to TMP2 + TMP1. */
23358 3 : tmp1 = expand_simple_binop (SImode, PLUS, tmp2, tmp1, nullptr, 0,
23359 : OPTAB_DIRECT);
23360 :
23361 : /* Set TMP1 to TMP1 >> 16. */
23362 3 : tmp1 = expand_simple_binop (SImode, LSHIFTRT, tmp1, GEN_INT(16),
23363 : nullptr, 0, OPTAB_DIRECT);
23364 :
23365 : /* Move TMP1 to RESULT. */
23366 3 : emit_move_insn (result, tmp1);
23367 :
23368 3 : emit_jump_insn (gen_jump (cast_label));
23369 3 : emit_barrier ();
23370 :
23371 3 : emit_label (zero_label);
23372 :
23373 : /* Flush TMP0 to zero while keeping the sign bit for zero and
23374 : denormal. */
23375 3 : tmp1 = expand_simple_binop (SImode, AND, tmp0, GEN_INT(0x8000),
23376 : nullptr, 0, OPTAB_DIRECT);
23377 :
23378 : /* Move TMP1 to RESULT. */
23379 3 : emit_move_insn (result, tmp1);
23380 :
23381 3 : emit_label (cast_label);
23382 :
23383 : /* Cast RESULT to TMP2 in HImode. */
23384 3 : tmp2 = gen_reg_rtx (HImode);
23385 3 : tmp0 = gen_lowpart (HImode, result);
23386 3 : set = gen_rtx_SET (tmp2, tmp0);
23387 3 : emit_insn (set);
23388 :
23389 : /* Convert TMP2 to OP0. */
23390 3 : tmp0 = gen_lowpart (BFmode, tmp2);
23391 3 : set = gen_rtx_SET (op0, tmp0);
23392 3 : emit_insn (set);
23393 : }
23394 :
23395 : /* Expand SSE sequence for computing round
23396 : from OPERAND1 storing into OPERAND0. */
23397 : void
23398 14 : ix86_expand_round (rtx operand0, rtx operand1)
23399 : {
23400 : /* C code for the stuff we're doing below:
23401 : double xa = fabs (x);
23402 : if (!isless (xa, TWO52))
23403 : return x;
23404 : xa = (double)(long)(xa + nextafter (0.5, 0.0));
23405 : return copysign (xa, x);
23406 : */
23407 14 : machine_mode mode = GET_MODE (operand0);
23408 14 : rtx res, TWO52, xa, xi, half, mask;
23409 14 : rtx_code_label *label;
23410 14 : const struct real_format *fmt;
23411 14 : REAL_VALUE_TYPE pred_half, half_minus_pred_half;
23412 :
23413 : /* Temporary for holding the result, initialized to the input
23414 : operand to ease control flow. */
23415 14 : res = copy_to_reg (operand1);
23416 :
23417 14 : TWO52 = ix86_gen_TWO52 (mode);
23418 14 : xa = ix86_expand_sse_fabs (res, &mask);
23419 14 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
23420 :
23421 : /* load nextafter (0.5, 0.0) */
23422 14 : fmt = REAL_MODE_FORMAT (mode);
23423 14 : real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
23424 14 : real_arithmetic (&pred_half, MINUS_EXPR, &dconsthalf, &half_minus_pred_half);
23425 :
23426 : /* xa = xa + 0.5 */
23427 14 : half = force_reg (mode, const_double_from_real_value (pred_half, mode));
23428 14 : xa = expand_simple_binop (mode, PLUS, xa, half, NULL_RTX, 0, OPTAB_DIRECT);
23429 :
23430 : /* xa = (double)(int64_t)xa */
23431 14 : xi = gen_reg_rtx (int_mode_for_mode (mode).require ());
23432 14 : expand_fix (xi, xa, 0);
23433 14 : expand_float (xa, xi, 0);
23434 :
23435 : /* res = copysign (xa, operand1) */
23436 14 : ix86_sse_copysign_to_positive (res, xa, res, mask);
23437 :
23438 14 : emit_label (label);
23439 14 : LABEL_NUSES (label) = 1;
23440 :
23441 14 : emit_move_insn (operand0, res);
23442 14 : }
23443 :
23444 : /* Expand SSE sequence for computing round from OPERAND1 storing
23445 : into OPERAND0 without relying on DImode truncation via cvttsd2siq
23446 : that is only available on 64bit targets. */
23447 : void
23448 0 : ix86_expand_rounddf_32 (rtx operand0, rtx operand1)
23449 : {
23450 : /* C code for the stuff we expand below.
23451 : double xa = fabs (x), xa2, x2;
23452 : if (!isless (xa, TWO52))
23453 : return x;
23454 : Using the absolute value and copying back sign makes
23455 : -0.0 -> -0.0 correct.
23456 : xa2 = xa + TWO52 - TWO52;
23457 : Compensate.
23458 : dxa = xa2 - xa;
23459 : if (dxa <= -0.5)
23460 : xa2 += 1;
23461 : else if (dxa > 0.5)
23462 : xa2 -= 1;
23463 : x2 = copysign (xa2, x);
23464 : return x2;
23465 : */
23466 0 : machine_mode mode = GET_MODE (operand0);
23467 0 : rtx xa, xa2, dxa, TWO52, tmp, half, mhalf, one, res, mask;
23468 0 : rtx_code_label *label;
23469 :
23470 0 : TWO52 = ix86_gen_TWO52 (mode);
23471 :
23472 : /* Temporary for holding the result, initialized to the input
23473 : operand to ease control flow. */
23474 0 : res = copy_to_reg (operand1);
23475 :
23476 : /* xa = abs (operand1) */
23477 0 : xa = ix86_expand_sse_fabs (res, &mask);
23478 :
23479 : /* if (!isless (xa, TWO52)) goto label; */
23480 0 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
23481 :
23482 : /* xa2 = xa + TWO52 - TWO52; */
23483 0 : xa2 = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
23484 0 : xa2 = expand_simple_binop (mode, MINUS, xa2, TWO52, xa2, 0, OPTAB_DIRECT);
23485 :
23486 : /* dxa = xa2 - xa; */
23487 0 : dxa = expand_simple_binop (mode, MINUS, xa2, xa, NULL_RTX, 0, OPTAB_DIRECT);
23488 :
23489 : /* generate 0.5, 1.0 and -0.5 */
23490 0 : half = force_reg (mode, const_double_from_real_value (dconsthalf, mode));
23491 0 : one = expand_simple_binop (mode, PLUS, half, half, NULL_RTX, 0, OPTAB_DIRECT);
23492 0 : mhalf = expand_simple_binop (mode, MINUS, half, one, NULL_RTX,
23493 : 0, OPTAB_DIRECT);
23494 :
23495 : /* Compensate. */
23496 : /* xa2 = xa2 - (dxa > 0.5 ? 1 : 0) */
23497 0 : tmp = ix86_expand_sse_compare_mask (UNGT, dxa, half, false);
23498 0 : emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, tmp, one)));
23499 0 : xa2 = expand_simple_binop (mode, MINUS, xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
23500 : /* xa2 = xa2 + (dxa <= -0.5 ? 1 : 0) */
23501 0 : tmp = ix86_expand_sse_compare_mask (UNGE, mhalf, dxa, false);
23502 0 : emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, tmp, one)));
23503 0 : xa2 = expand_simple_binop (mode, PLUS, xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
23504 :
23505 : /* res = copysign (xa2, operand1) */
23506 0 : ix86_sse_copysign_to_positive (res, xa2, res, mask);
23507 :
23508 0 : emit_label (label);
23509 0 : LABEL_NUSES (label) = 1;
23510 :
23511 0 : emit_move_insn (operand0, res);
23512 0 : }
23513 :
23514 : /* Expand SSE sequence for computing round
23515 : from OP1 storing into OP0 using sse4 round insn. */
23516 : void
23517 10 : ix86_expand_round_sse4 (rtx op0, rtx op1)
23518 : {
23519 10 : machine_mode mode = GET_MODE (op0);
23520 10 : rtx e1, e2, res, half;
23521 10 : const struct real_format *fmt;
23522 10 : REAL_VALUE_TYPE pred_half, half_minus_pred_half;
23523 10 : rtx (*gen_copysign) (rtx, rtx, rtx);
23524 10 : rtx (*gen_round) (rtx, rtx, rtx);
23525 :
23526 10 : switch (mode)
23527 : {
23528 : case E_HFmode:
23529 : gen_copysign = gen_copysignhf3;
23530 : gen_round = gen_sse4_1_roundhf2;
23531 : break;
23532 4 : case E_SFmode:
23533 4 : gen_copysign = gen_copysignsf3;
23534 4 : gen_round = gen_sse4_1_roundsf2;
23535 4 : break;
23536 4 : case E_DFmode:
23537 4 : gen_copysign = gen_copysigndf3;
23538 4 : gen_round = gen_sse4_1_rounddf2;
23539 4 : break;
23540 0 : default:
23541 0 : gcc_unreachable ();
23542 : }
23543 :
23544 : /* round (a) = trunc (a + copysign (0.5, a)) */
23545 :
23546 : /* load nextafter (0.5, 0.0) */
23547 10 : fmt = REAL_MODE_FORMAT (mode);
23548 10 : real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
23549 10 : real_arithmetic (&pred_half, MINUS_EXPR, &dconsthalf, &half_minus_pred_half);
23550 10 : half = const_double_from_real_value (pred_half, mode);
23551 :
23552 : /* e1 = copysign (0.5, op1) */
23553 10 : e1 = gen_reg_rtx (mode);
23554 10 : emit_insn (gen_copysign (e1, half, op1));
23555 :
23556 : /* e2 = op1 + e1 */
23557 10 : e2 = expand_simple_binop (mode, PLUS, op1, e1, NULL_RTX, 0, OPTAB_DIRECT);
23558 :
23559 : /* res = trunc (e2) */
23560 10 : res = gen_reg_rtx (mode);
23561 10 : emit_insn (gen_round (res, e2, GEN_INT (ROUND_TRUNC)));
23562 :
23563 10 : emit_move_insn (op0, res);
23564 10 : }
23565 :
23566 : /* A cached (set (nil) (vselect (vconcat (nil) (nil)) (parallel [])))
23567 : insn, so that expand_vselect{,_vconcat} doesn't have to create a fresh
23568 : insn every time. */
23569 :
23570 : static GTY(()) rtx_insn *vselect_insn;
23571 :
23572 : /* Initialize vselect_insn. */
23573 :
23574 : static void
23575 7910 : init_vselect_insn (void)
23576 : {
23577 7910 : unsigned i;
23578 7910 : rtx x;
23579 :
23580 7910 : x = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (MAX_VECT_LEN));
23581 514150 : for (i = 0; i < MAX_VECT_LEN; ++i)
23582 506240 : XVECEXP (x, 0, i) = const0_rtx;
23583 7910 : x = gen_rtx_VEC_SELECT (V2DFmode, gen_rtx_VEC_CONCAT (V4DFmode, const0_rtx,
23584 : const0_rtx), x);
23585 7910 : x = gen_rtx_SET (const0_rtx, x);
23586 7910 : start_sequence ();
23587 7910 : vselect_insn = emit_insn (x);
23588 7910 : end_sequence ();
23589 7910 : }
23590 :
23591 : /* Construct (set target (vec_select op0 (parallel perm))) and
23592 : return true if that's a valid instruction in the active ISA. */
23593 :
23594 : static bool
23595 556003 : expand_vselect (rtx target, rtx op0, const unsigned char *perm,
23596 : unsigned nelt, bool testing_p)
23597 : {
23598 556003 : unsigned int i;
23599 556003 : rtx x, save_vconcat;
23600 556003 : int icode;
23601 :
23602 556003 : if (vselect_insn == NULL_RTX)
23603 1677 : init_vselect_insn ();
23604 :
23605 556003 : x = XEXP (SET_SRC (PATTERN (vselect_insn)), 1);
23606 556003 : PUT_NUM_ELEM (XVEC (x, 0), nelt);
23607 4266571 : for (i = 0; i < nelt; ++i)
23608 3710568 : XVECEXP (x, 0, i) = GEN_INT (perm[i]);
23609 556003 : save_vconcat = XEXP (SET_SRC (PATTERN (vselect_insn)), 0);
23610 556003 : XEXP (SET_SRC (PATTERN (vselect_insn)), 0) = op0;
23611 556003 : PUT_MODE (SET_SRC (PATTERN (vselect_insn)), GET_MODE (target));
23612 556003 : SET_DEST (PATTERN (vselect_insn)) = target;
23613 556003 : icode = recog_memoized (vselect_insn);
23614 :
23615 556003 : if (icode >= 0 && !testing_p)
23616 72927 : emit_insn (copy_rtx (PATTERN (vselect_insn)));
23617 :
23618 556003 : SET_DEST (PATTERN (vselect_insn)) = const0_rtx;
23619 556003 : XEXP (SET_SRC (PATTERN (vselect_insn)), 0) = save_vconcat;
23620 556003 : INSN_CODE (vselect_insn) = -1;
23621 :
23622 556003 : return icode >= 0;
23623 : }
23624 :
23625 : /* Similar, but generate a vec_concat from op0 and op1 as well. */
23626 :
23627 : static bool
23628 489042 : expand_vselect_vconcat (rtx target, rtx op0, rtx op1,
23629 : const unsigned char *perm, unsigned nelt,
23630 : bool testing_p)
23631 : {
23632 489042 : machine_mode v2mode;
23633 489042 : rtx x;
23634 489042 : bool ok;
23635 :
23636 489042 : if (vselect_insn == NULL_RTX)
23637 6233 : init_vselect_insn ();
23638 :
23639 489042 : if (!GET_MODE_2XWIDER_MODE (GET_MODE (op0)).exists (&v2mode))
23640 : return false;
23641 489042 : x = XEXP (SET_SRC (PATTERN (vselect_insn)), 0);
23642 489042 : PUT_MODE (x, v2mode);
23643 489042 : XEXP (x, 0) = op0;
23644 489042 : XEXP (x, 1) = op1;
23645 489042 : ok = expand_vselect (target, x, perm, nelt, testing_p);
23646 489042 : XEXP (x, 0) = const0_rtx;
23647 489042 : XEXP (x, 1) = const0_rtx;
23648 489042 : return ok;
23649 : }
23650 :
23651 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
23652 : using movss or movsd. */
23653 : static bool
23654 311637 : expand_vec_perm_movs (struct expand_vec_perm_d *d)
23655 : {
23656 311637 : machine_mode vmode = d->vmode;
23657 311637 : unsigned i, nelt = d->nelt;
23658 311637 : rtx x;
23659 :
23660 311637 : if (d->one_operand_p)
23661 : return false;
23662 :
23663 285670 : if (!(TARGET_SSE && (vmode == V4SFmode || vmode == V4SImode))
23664 132393 : && !(TARGET_MMX_WITH_SSE && (vmode == V2SFmode || vmode == V2SImode))
23665 91389 : && !(TARGET_SSE2 && (vmode == V2DFmode || vmode == V2DImode)))
23666 : return false;
23667 :
23668 : /* Only the first element is changed. */
23669 203895 : if (d->perm[0] != nelt && d->perm[0] != 0)
23670 : return false;
23671 124540 : for (i = 1; i < nelt; ++i)
23672 109703 : if (d->perm[i] != i + nelt - d->perm[0])
23673 : return false;
23674 :
23675 14837 : if (d->testing_p)
23676 : return true;
23677 :
23678 7060 : if (d->perm[0] == nelt)
23679 0 : x = gen_rtx_VEC_MERGE (vmode, d->op1, d->op0, GEN_INT (1));
23680 : else
23681 7060 : x = gen_rtx_VEC_MERGE (vmode, d->op0, d->op1, GEN_INT (1));
23682 :
23683 7060 : emit_insn (gen_rtx_SET (d->target, x));
23684 :
23685 7060 : return true;
23686 : }
23687 :
23688 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
23689 : using insertps. */
23690 : static bool
23691 296800 : expand_vec_perm_insertps (struct expand_vec_perm_d *d)
23692 : {
23693 296800 : machine_mode vmode = d->vmode;
23694 296800 : unsigned i, cnt_s, nelt = d->nelt;
23695 296800 : int cnt_d = -1;
23696 296800 : rtx src, dst;
23697 :
23698 296800 : if (d->one_operand_p)
23699 : return false;
23700 :
23701 270833 : if (!(TARGET_SSE4_1
23702 37318 : && (vmode == V4SFmode || vmode == V4SImode
23703 27169 : || (TARGET_MMX_WITH_SSE
23704 20821 : && (vmode == V2SFmode || vmode == V2SImode)))))
23705 : return false;
23706 :
23707 54969 : for (i = 0; i < nelt; ++i)
23708 : {
23709 51939 : if (d->perm[i] == i)
23710 10489 : continue;
23711 41450 : if (cnt_d != -1)
23712 : {
23713 : cnt_d = -1;
23714 : break;
23715 : }
23716 22240 : cnt_d = i;
23717 : }
23718 :
23719 22240 : if (cnt_d == -1)
23720 : {
23721 43292 : for (i = 0; i < nelt; ++i)
23722 : {
23723 40672 : if (d->perm[i] == i + nelt)
23724 4872 : continue;
23725 35800 : if (cnt_d != -1)
23726 : return false;
23727 19210 : cnt_d = i;
23728 : }
23729 :
23730 2620 : if (cnt_d == -1)
23731 : return false;
23732 : }
23733 :
23734 5650 : if (d->testing_p)
23735 : return true;
23736 :
23737 570 : gcc_assert (cnt_d != -1);
23738 :
23739 570 : cnt_s = d->perm[cnt_d];
23740 570 : if (cnt_s < nelt)
23741 : {
23742 246 : src = d->op0;
23743 246 : dst = d->op1;
23744 : }
23745 : else
23746 : {
23747 324 : cnt_s -= nelt;
23748 324 : src = d->op1;
23749 324 : dst = d->op0;
23750 : }
23751 570 : gcc_assert (cnt_s < nelt);
23752 :
23753 570 : rtx x = gen_sse4_1_insertps (vmode, d->target, dst, src,
23754 570 : GEN_INT (cnt_s << 6 | cnt_d << 4));
23755 570 : emit_insn (x);
23756 :
23757 570 : return true;
23758 : }
23759 :
23760 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
23761 : in terms of blendp[sd] / pblendw / pblendvb / vpblendd. */
23762 :
23763 : static bool
23764 317330 : expand_vec_perm_blend (struct expand_vec_perm_d *d)
23765 : {
23766 317330 : machine_mode mmode, vmode = d->vmode;
23767 317330 : unsigned i, nelt = d->nelt;
23768 317330 : unsigned HOST_WIDE_INT mask;
23769 317330 : rtx target, op0, op1, maskop, x;
23770 317330 : rtx rperm[32], vperm;
23771 :
23772 317330 : if (d->one_operand_p)
23773 : return false;
23774 6874 : if (TARGET_AVX512F && GET_MODE_SIZE (vmode) == 64
23775 292364 : && (TARGET_AVX512BW
23776 660 : || GET_MODE_UNIT_SIZE (vmode) >= 4))
23777 : ;
23778 303589 : else if (TARGET_AVX2 && GET_MODE_SIZE (vmode) == 32)
23779 : ;
23780 284961 : else if (TARGET_AVX && (vmode == V4DFmode || vmode == V8SFmode))
23781 : ;
23782 279498 : else if (TARGET_SSE4_1
23783 315936 : && (GET_MODE_SIZE (vmode) == 16
23784 30280 : || (TARGET_MMX_WITH_SSE && GET_MODE_SIZE (vmode) == 8)
23785 3699 : || GET_MODE_SIZE (vmode) == 4))
23786 : ;
23787 : else
23788 : return false;
23789 :
23790 : /* This is a blend, not a permute. Elements must stay in their
23791 : respective lanes. */
23792 100406 : for (i = 0; i < nelt; ++i)
23793 : {
23794 94713 : unsigned e = d->perm[i];
23795 94713 : if (!(e == i || e == i + nelt))
23796 : return false;
23797 : }
23798 :
23799 5693 : if (d->testing_p)
23800 : return true;
23801 :
23802 : /* ??? Without SSE4.1, we could implement this with and/andn/or. This
23803 : decision should be extracted elsewhere, so that we only try that
23804 : sequence once all budget==3 options have been tried. */
23805 2752 : target = d->target;
23806 2752 : op0 = d->op0;
23807 2752 : op1 = d->op1;
23808 2752 : mask = 0;
23809 :
23810 2752 : switch (vmode)
23811 : {
23812 : case E_V8DFmode:
23813 : case E_V16SFmode:
23814 : case E_V4DFmode:
23815 : case E_V8SFmode:
23816 : case E_V2DFmode:
23817 : case E_V4SFmode:
23818 : case E_V2SFmode:
23819 : case E_V2HImode:
23820 : case E_V4HImode:
23821 : case E_V8HImode:
23822 : case E_V8SImode:
23823 : case E_V32HImode:
23824 : case E_V64QImode:
23825 : case E_V16SImode:
23826 : case E_V8DImode:
23827 10595 : for (i = 0; i < nelt; ++i)
23828 9142 : mask |= ((unsigned HOST_WIDE_INT) (d->perm[i] >= nelt)) << i;
23829 : break;
23830 :
23831 : case E_V2DImode:
23832 18 : for (i = 0; i < 2; ++i)
23833 18 : mask |= (d->perm[i] >= 2 ? 15 : 0) << (i * 4);
23834 6 : vmode = V8HImode;
23835 6 : goto do_subreg;
23836 :
23837 : case E_V2SImode:
23838 24 : for (i = 0; i < 2; ++i)
23839 24 : mask |= (d->perm[i] >= 2 ? 3 : 0) << (i * 2);
23840 8 : vmode = V4HImode;
23841 8 : goto do_subreg;
23842 :
23843 879 : case E_V4SImode:
23844 879 : if (TARGET_AVX2)
23845 : {
23846 : /* Use vpblendd instead of vpblendw. */
23847 225 : for (i = 0; i < nelt; ++i)
23848 180 : mask |= ((unsigned HOST_WIDE_INT) (d->perm[i] >= nelt)) << i;
23849 : break;
23850 : }
23851 : else
23852 : {
23853 4170 : for (i = 0; i < 4; ++i)
23854 5200 : mask |= (d->perm[i] >= 4 ? 3 : 0) << (i * 2);
23855 834 : vmode = V8HImode;
23856 834 : goto do_subreg;
23857 : }
23858 :
23859 : case E_V16QImode:
23860 : /* See if bytes move in pairs so we can use pblendw with
23861 : an immediate argument, rather than pblendvb with a vector
23862 : argument. */
23863 102 : for (i = 0; i < 16; i += 2)
23864 100 : if (d->perm[i] + 1 != d->perm[i + 1])
23865 : {
23866 290 : use_pblendvb:
23867 3502 : for (i = 0; i < nelt; ++i)
23868 3212 : rperm[i] = (d->perm[i] < nelt ? const0_rtx : constm1_rtx);
23869 :
23870 290 : finish_pblendvb:
23871 291 : vperm = gen_rtx_CONST_VECTOR (vmode, gen_rtvec_v (nelt, rperm));
23872 291 : vperm = force_reg (vmode, vperm);
23873 :
23874 582 : if (GET_MODE_SIZE (vmode) == 4)
23875 135 : emit_insn (gen_mmx_pblendvb_v4qi (target, op0, op1, vperm));
23876 312 : else if (GET_MODE_SIZE (vmode) == 8)
23877 40 : emit_insn (gen_mmx_pblendvb_v8qi (target, op0, op1, vperm));
23878 232 : else if (GET_MODE_SIZE (vmode) == 16)
23879 83 : emit_insn (gen_sse4_1_pblendvb (target, op0, op1, vperm));
23880 : else
23881 33 : emit_insn (gen_avx2_pblendvb (target, op0, op1, vperm));
23882 291 : if (target != d->target)
23883 1 : emit_move_insn (d->target, gen_lowpart (d->vmode, target));
23884 : return true;
23885 : }
23886 :
23887 18 : for (i = 0; i < 8; ++i)
23888 16 : mask |= (d->perm[i * 2] >= 16) << i;
23889 : vmode = V8HImode;
23890 : /* FALLTHRU */
23891 :
23892 931 : do_subreg:
23893 931 : target = gen_reg_rtx (vmode);
23894 931 : op0 = gen_lowpart (vmode, op0);
23895 931 : op1 = gen_lowpart (vmode, op1);
23896 931 : break;
23897 :
23898 : case E_V8QImode:
23899 40 : for (i = 0; i < 8; i += 2)
23900 40 : if (d->perm[i] + 1 != d->perm[i + 1])
23901 40 : goto use_pblendvb;
23902 :
23903 0 : for (i = 0; i < 4; ++i)
23904 0 : mask |= (d->perm[i * 2] >= 8) << i;
23905 0 : vmode = V4HImode;
23906 0 : goto do_subreg;
23907 :
23908 : case E_V4QImode:
23909 153 : for (i = 0; i < 4; i += 2)
23910 150 : if (d->perm[i] + 1 != d->perm[i + 1])
23911 135 : goto use_pblendvb;
23912 :
23913 9 : for (i = 0; i < 2; ++i)
23914 6 : mask |= (d->perm[i * 2] >= 4) << i;
23915 3 : vmode = V2HImode;
23916 3 : goto do_subreg;
23917 :
23918 : case E_V32QImode:
23919 : /* See if bytes move in pairs. If not, vpblendvb must be used. */
23920 916 : for (i = 0; i < 32; i += 2)
23921 864 : if (d->perm[i] + 1 != d->perm[i + 1])
23922 32 : goto use_pblendvb;
23923 : /* See if bytes move in quadruplets. If yes, vpblendd
23924 : with immediate can be used. */
23925 468 : for (i = 0; i < 32; i += 4)
23926 416 : if (d->perm[i] + 2 != d->perm[i + 2])
23927 : break;
23928 52 : if (i < 32)
23929 : {
23930 : /* See if bytes move the same in both lanes. If yes,
23931 : vpblendw with immediate can be used. */
23932 0 : for (i = 0; i < 16; i += 2)
23933 0 : if (d->perm[i] + 16 != d->perm[i + 16])
23934 0 : goto use_pblendvb;
23935 :
23936 : /* Use vpblendw. */
23937 0 : for (i = 0; i < 16; ++i)
23938 0 : mask |= (d->perm[i * 2] >= 32) << i;
23939 0 : vmode = V16HImode;
23940 0 : goto do_subreg;
23941 : }
23942 :
23943 : /* Use vpblendd. */
23944 468 : for (i = 0; i < 8; ++i)
23945 416 : mask |= (d->perm[i * 4] >= 32) << i;
23946 52 : vmode = V8SImode;
23947 52 : goto do_subreg;
23948 :
23949 : case E_V16HImode:
23950 : /* See if words move in pairs. If yes, vpblendd can be used. */
23951 177 : for (i = 0; i < 16; i += 2)
23952 161 : if (d->perm[i] + 1 != d->perm[i + 1])
23953 : break;
23954 49 : if (i < 16)
23955 : {
23956 : /* See if words move the same in both lanes. If not,
23957 : vpblendvb must be used. */
23958 290 : for (i = 0; i < 8; i++)
23959 258 : if (d->perm[i] + 8 != d->perm[i + 8])
23960 : {
23961 : /* Use vpblendvb. */
23962 33 : for (i = 0; i < 32; ++i)
23963 32 : rperm[i] = (d->perm[i / 2] < 16 ? const0_rtx : constm1_rtx);
23964 :
23965 1 : vmode = V32QImode;
23966 1 : nelt = 32;
23967 1 : target = gen_reg_rtx (vmode);
23968 1 : op0 = gen_lowpart (vmode, op0);
23969 1 : op1 = gen_lowpart (vmode, op1);
23970 1 : goto finish_pblendvb;
23971 : }
23972 :
23973 : /* Use vpblendw. */
23974 544 : for (i = 0; i < 16; ++i)
23975 512 : mask |= (d->perm[i] >= 16) << i;
23976 : break;
23977 : }
23978 :
23979 : /* Use vpblendd. */
23980 144 : for (i = 0; i < 8; ++i)
23981 128 : mask |= (d->perm[i * 2] >= 16) << i;
23982 16 : vmode = V8SImode;
23983 16 : goto do_subreg;
23984 :
23985 : case E_V4DImode:
23986 : /* Use vpblendd. */
23987 50 : for (i = 0; i < 4; ++i)
23988 59 : mask |= (d->perm[i] >= 4 ? 3 : 0) << (i * 2);
23989 10 : vmode = V8SImode;
23990 10 : goto do_subreg;
23991 :
23992 0 : default:
23993 0 : gcc_unreachable ();
23994 : }
23995 :
23996 2461 : switch (vmode)
23997 : {
23998 : case E_V8DFmode:
23999 : case E_V8DImode:
24000 : mmode = QImode;
24001 : break;
24002 1 : case E_V16SFmode:
24003 1 : case E_V16SImode:
24004 1 : mmode = HImode;
24005 1 : break;
24006 6 : case E_V32HImode:
24007 6 : mmode = SImode;
24008 6 : break;
24009 1 : case E_V64QImode:
24010 1 : mmode = DImode;
24011 1 : break;
24012 : default:
24013 : mmode = VOIDmode;
24014 : }
24015 :
24016 : /* Canonicalize vec_merge. */
24017 2461 : if (swap_commutative_operands_p (op1, op0)
24018 : /* Two operands have same precedence, then
24019 : first bit of mask select first operand. */
24020 2461 : || (!swap_commutative_operands_p (op0, op1)
24021 2461 : && !(mask & 1)))
24022 : {
24023 2454 : unsigned n_elts = GET_MODE_NUNITS (vmode);
24024 2454 : std::swap (op0, op1);
24025 2454 : unsigned HOST_WIDE_INT mask_all = HOST_WIDE_INT_1U;
24026 2454 : if (n_elts == HOST_BITS_PER_WIDE_INT)
24027 : mask_all = -1;
24028 : else
24029 2453 : mask_all = (HOST_WIDE_INT_1U << n_elts) - 1;
24030 2454 : mask = ~mask & mask_all;
24031 : }
24032 :
24033 2461 : if (mmode != VOIDmode)
24034 17 : maskop = force_reg (mmode, gen_int_mode (mask, mmode));
24035 : else
24036 2444 : maskop = GEN_INT (mask);
24037 :
24038 : /* This matches five different patterns with the different modes. */
24039 2461 : x = gen_rtx_VEC_MERGE (vmode, op1, op0, maskop);
24040 2461 : x = gen_rtx_SET (target, x);
24041 2461 : emit_insn (x);
24042 2461 : if (target != d->target)
24043 931 : emit_move_insn (d->target, gen_lowpart (d->vmode, target));
24044 :
24045 : return true;
24046 : }
24047 :
24048 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
24049 : in terms of the variable form of vpermilps.
24050 :
24051 : Note that we will have already failed the immediate input vpermilps,
24052 : which requires that the high and low part shuffle be identical; the
24053 : variable form doesn't require that. */
24054 :
24055 : static bool
24056 140739 : expand_vec_perm_vpermil (struct expand_vec_perm_d *d)
24057 : {
24058 140739 : rtx rperm[8], vperm;
24059 140739 : unsigned i;
24060 :
24061 140739 : if (!TARGET_AVX || !d->one_operand_p
24062 10984 : || (d->vmode != V8SImode && d->vmode != V8SFmode))
24063 : return false;
24064 :
24065 : /* We can only permute within the 128-bit lane. */
24066 15665 : for (i = 0; i < 8; ++i)
24067 : {
24068 15077 : unsigned e = d->perm[i];
24069 15077 : if (i < 4 ? e >= 4 : e < 4)
24070 : return false;
24071 : }
24072 :
24073 588 : if (d->testing_p)
24074 : return true;
24075 :
24076 531 : for (i = 0; i < 8; ++i)
24077 : {
24078 472 : unsigned e = d->perm[i];
24079 :
24080 : /* Within each 128-bit lane, the elements of op0 are numbered
24081 : from 0 and the elements of op1 are numbered from 4. */
24082 472 : if (e >= 8 + 4)
24083 0 : e -= 8;
24084 472 : else if (e >= 4)
24085 236 : e -= 4;
24086 :
24087 472 : rperm[i] = GEN_INT (e);
24088 : }
24089 :
24090 59 : vperm = gen_rtx_CONST_VECTOR (V8SImode, gen_rtvec_v (8, rperm));
24091 59 : vperm = force_reg (V8SImode, vperm);
24092 59 : rtx target = d->target;
24093 59 : rtx op0 = d->op0;
24094 59 : if (d->vmode == V8SImode)
24095 : {
24096 7 : target = lowpart_subreg (V8SFmode, target, V8SImode);
24097 7 : op0 = lowpart_subreg (V8SFmode, op0, V8SImode);
24098 : }
24099 :
24100 59 : emit_insn (gen_avx_vpermilvarv8sf3 (target, op0, vperm));
24101 :
24102 59 : return true;
24103 : }
24104 :
24105 : /* For V*[QHS]Imode permutations, check if the same permutation
24106 : can't be performed in a 2x, 4x or 8x wider inner mode. */
24107 :
24108 : static bool
24109 165687 : canonicalize_vector_int_perm (const struct expand_vec_perm_d *d,
24110 : struct expand_vec_perm_d *nd)
24111 : {
24112 165687 : int i;
24113 165687 : machine_mode mode = VOIDmode;
24114 :
24115 165687 : switch (d->vmode)
24116 : {
24117 : case E_V8QImode: mode = V4HImode; break;
24118 30582 : case E_V16QImode: mode = V8HImode; break;
24119 708 : case E_V32QImode: mode = V16HImode; break;
24120 268 : case E_V64QImode: mode = V32HImode; break;
24121 12653 : case E_V4HImode: mode = V2SImode; break;
24122 21502 : case E_V8HImode: mode = V4SImode; break;
24123 1006 : case E_V16HImode: mode = V8SImode; break;
24124 397 : case E_V32HImode: mode = V16SImode; break;
24125 41805 : case E_V4SImode: mode = V2DImode; break;
24126 1429 : case E_V8SImode: mode = V4DImode; break;
24127 65 : case E_V16SImode: mode = V8DImode; break;
24128 : default: return false;
24129 : }
24130 210359 : for (i = 0; i < d->nelt; i += 2)
24131 195624 : if ((d->perm[i] & 1) || d->perm[i + 1] != d->perm[i] + 1)
24132 : return false;
24133 14735 : nd->vmode = mode;
24134 14735 : nd->nelt = d->nelt / 2;
24135 96159 : for (i = 0; i < nd->nelt; i++)
24136 81424 : nd->perm[i] = d->perm[2 * i] / 2;
24137 29470 : if (GET_MODE_INNER (mode) != DImode)
24138 12972 : canonicalize_vector_int_perm (nd, nd);
24139 14735 : if (nd != d)
24140 : {
24141 9391 : nd->one_operand_p = d->one_operand_p;
24142 9391 : nd->testing_p = d->testing_p;
24143 9391 : if (d->op0 == d->op1)
24144 2782 : nd->op0 = nd->op1 = gen_lowpart (nd->vmode, d->op0);
24145 : else
24146 : {
24147 6609 : nd->op0 = gen_lowpart (nd->vmode, d->op0);
24148 6609 : nd->op1 = gen_lowpart (nd->vmode, d->op1);
24149 : }
24150 9391 : if (d->testing_p)
24151 6089 : nd->target = gen_raw_REG (nd->vmode, LAST_VIRTUAL_REGISTER + 1);
24152 : else
24153 3302 : nd->target = gen_reg_rtx (nd->vmode);
24154 : }
24155 : return true;
24156 : }
24157 :
24158 : /* Return true if permutation D can be performed as VMODE permutation
24159 : instead. */
24160 :
24161 : static bool
24162 5930 : valid_perm_using_mode_p (machine_mode vmode, struct expand_vec_perm_d *d)
24163 : {
24164 5930 : unsigned int i, j, chunk;
24165 :
24166 5930 : if (GET_MODE_CLASS (vmode) != MODE_VECTOR_INT
24167 5930 : || GET_MODE_CLASS (d->vmode) != MODE_VECTOR_INT
24168 14502 : || GET_MODE_SIZE (vmode) != GET_MODE_SIZE (d->vmode))
24169 : return false;
24170 :
24171 8572 : if (GET_MODE_NUNITS (vmode) >= d->nelt)
24172 : return true;
24173 :
24174 4022 : chunk = d->nelt / GET_MODE_NUNITS (vmode);
24175 5216 : for (i = 0; i < d->nelt; i += chunk)
24176 4969 : if (d->perm[i] & (chunk - 1))
24177 : return false;
24178 : else
24179 7516 : for (j = 1; j < chunk; ++j)
24180 6322 : if (d->perm[i] + j != d->perm[i + j])
24181 : return false;
24182 :
24183 : return true;
24184 : }
24185 :
24186 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
24187 : in terms of pshufb, vpperm, vpermq, vpermd, vpermps or vperm2i128. */
24188 :
24189 : static bool
24190 140151 : expand_vec_perm_pshufb (struct expand_vec_perm_d *d)
24191 : {
24192 140151 : unsigned i, nelt, eltsz, mask;
24193 140151 : unsigned char perm[64];
24194 140151 : machine_mode vmode;
24195 140151 : struct expand_vec_perm_d nd;
24196 140151 : rtx rperm[64], vperm, target, op0, op1;
24197 :
24198 140151 : nelt = d->nelt;
24199 :
24200 140151 : if (!d->one_operand_p)
24201 229748 : switch (GET_MODE_SIZE (d->vmode))
24202 : {
24203 8471 : case 4:
24204 8471 : if (!TARGET_XOP)
24205 : return false;
24206 : vmode = V4QImode;
24207 : break;
24208 :
24209 21048 : case 8:
24210 21048 : if (!TARGET_XOP)
24211 : return false;
24212 : vmode = V8QImode;
24213 : break;
24214 :
24215 76463 : case 16:
24216 76463 : if (!TARGET_XOP)
24217 : return false;
24218 : vmode = V16QImode;
24219 : break;
24220 :
24221 7996 : case 32:
24222 7996 : if (!TARGET_AVX2)
24223 : return false;
24224 :
24225 3998 : if (valid_perm_using_mode_p (V2TImode, d))
24226 : {
24227 56 : if (d->testing_p)
24228 : return true;
24229 :
24230 : /* Use vperm2i128 insn. The pattern uses
24231 : V4DImode instead of V2TImode. */
24232 52 : target = d->target;
24233 52 : if (d->vmode != V4DImode)
24234 12 : target = gen_reg_rtx (V4DImode);
24235 52 : op0 = gen_lowpart (V4DImode, d->op0);
24236 52 : op1 = gen_lowpart (V4DImode, d->op1);
24237 52 : rperm[0]
24238 52 : = GEN_INT ((d->perm[0] / (nelt / 2))
24239 : | ((d->perm[nelt / 2] / (nelt / 2)) * 16));
24240 52 : emit_insn (gen_avx2_permv2ti (target, op0, op1, rperm[0]));
24241 52 : if (target != d->target)
24242 12 : emit_move_insn (d->target, gen_lowpart (d->vmode, target));
24243 : return true;
24244 : }
24245 : /* FALLTHRU */
24246 :
24247 : default:
24248 : return false;
24249 : }
24250 : else
24251 50554 : switch (GET_MODE_SIZE (d->vmode))
24252 : {
24253 3716 : case 4:
24254 3716 : if (!TARGET_SSSE3)
24255 : return false;
24256 : vmode = V4QImode;
24257 : break;
24258 :
24259 2277 : case 8:
24260 2277 : if (!TARGET_SSSE3)
24261 : return false;
24262 : vmode = V8QImode;
24263 : break;
24264 :
24265 14381 : case 16:
24266 14381 : if (!TARGET_SSSE3)
24267 : return false;
24268 : vmode = V16QImode;
24269 : break;
24270 :
24271 4501 : case 32:
24272 4501 : if (!TARGET_AVX2)
24273 : return false;
24274 :
24275 : /* V4DImode should be already handled through
24276 : expand_vselect by vpermq instruction. */
24277 1924 : gcc_assert (d->vmode != V4DImode);
24278 :
24279 1924 : vmode = V32QImode;
24280 1924 : if (d->vmode == V8SImode
24281 1559 : || d->vmode == V16HImode
24282 1343 : || d->vmode == V32QImode)
24283 : {
24284 : /* First see if vpermq can be used for
24285 : V8SImode/V16HImode/V32QImode. */
24286 879 : if (valid_perm_using_mode_p (V4DImode, d))
24287 : {
24288 770 : for (i = 0; i < 4; i++)
24289 616 : perm[i] = (d->perm[i * nelt / 4] * 4 / nelt) & 3;
24290 154 : if (d->testing_p)
24291 : return true;
24292 58 : target = gen_reg_rtx (V4DImode);
24293 58 : if (expand_vselect (target, gen_lowpart (V4DImode, d->op0),
24294 : perm, 4, false))
24295 : {
24296 116 : emit_move_insn (d->target,
24297 58 : gen_lowpart (d->vmode, target));
24298 58 : return true;
24299 : }
24300 : return false;
24301 : }
24302 :
24303 : /* Next see if vpermd can be used. */
24304 725 : if (valid_perm_using_mode_p (V8SImode, d))
24305 : vmode = V8SImode;
24306 : }
24307 : /* Or if vpermps can be used. */
24308 1045 : else if (d->vmode == V8SFmode)
24309 : vmode = V8SImode;
24310 :
24311 : if (vmode == V32QImode)
24312 : {
24313 : /* vpshufb only works intra lanes, it is not
24314 : possible to shuffle bytes in between the lanes. */
24315 6473 : for (i = 0; i < nelt; ++i)
24316 6291 : if ((d->perm[i] ^ i) & (nelt / 2))
24317 : return false;
24318 : }
24319 : break;
24320 :
24321 402 : case 64:
24322 402 : if (!TARGET_AVX512BW)
24323 : return false;
24324 :
24325 : /* If vpermq didn't work, vpshufb won't work either. */
24326 204 : if (d->vmode == V8DFmode || d->vmode == V8DImode)
24327 : return false;
24328 :
24329 175 : vmode = V64QImode;
24330 175 : if (d->vmode == V16SImode
24331 150 : || d->vmode == V32HImode
24332 50 : || d->vmode == V64QImode)
24333 : {
24334 : /* First see if vpermq can be used for
24335 : V16SImode/V32HImode/V64QImode. */
24336 164 : if (valid_perm_using_mode_p (V8DImode, d))
24337 : {
24338 0 : for (i = 0; i < 8; i++)
24339 0 : perm[i] = (d->perm[i * nelt / 8] * 8 / nelt) & 7;
24340 0 : if (d->testing_p)
24341 : return true;
24342 0 : target = gen_reg_rtx (V8DImode);
24343 0 : if (expand_vselect (target, gen_lowpart (V8DImode, d->op0),
24344 : perm, 8, false))
24345 : {
24346 0 : emit_move_insn (d->target,
24347 0 : gen_lowpart (d->vmode, target));
24348 0 : return true;
24349 : }
24350 : return false;
24351 : }
24352 :
24353 : /* Next see if vpermd can be used. */
24354 164 : if (valid_perm_using_mode_p (V16SImode, d))
24355 : vmode = V16SImode;
24356 : }
24357 : /* Or if vpermps can be used. */
24358 11 : else if (d->vmode == V16SFmode)
24359 : vmode = V16SImode;
24360 :
24361 : if (vmode == V64QImode)
24362 : {
24363 : /* vpshufb only works intra lanes, it is not
24364 : possible to shuffle bytes in between the lanes. */
24365 578 : for (i = 0; i < nelt; ++i)
24366 578 : if ((d->perm[i] ^ i) & (3 * nelt / 4))
24367 : return false;
24368 : }
24369 : break;
24370 :
24371 : default:
24372 : return false;
24373 : }
24374 :
24375 11523 : if (d->testing_p)
24376 : return true;
24377 :
24378 : /* Try to avoid variable permutation instruction. */
24379 8800 : if (canonicalize_vector_int_perm (d, &nd) && expand_vec_perm_1 (&nd))
24380 : {
24381 1839 : emit_move_insn (d->target, gen_lowpart (d->vmode, nd.target));
24382 1839 : return true;
24383 : }
24384 :
24385 6961 : if (vmode == V8SImode)
24386 8964 : for (i = 0; i < 8; ++i)
24387 7968 : rperm[i] = GEN_INT ((d->perm[i * nelt / 8] * 8 / nelt) & 7);
24388 5965 : else if (vmode == V16SImode)
24389 612 : for (i = 0; i < 16; ++i)
24390 576 : rperm[i] = GEN_INT ((d->perm[i * nelt / 16] * 16 / nelt) & 15);
24391 : else
24392 : {
24393 5929 : eltsz = GET_MODE_UNIT_SIZE (d->vmode);
24394 5929 : if (!d->one_operand_p)
24395 3212 : mask = 2 * nelt - 1;
24396 2717 : else if (vmode == V64QImode)
24397 0 : mask = nelt / 4 - 1;
24398 2717 : else if (vmode == V32QImode)
24399 176 : mask = nelt / 2 - 1;
24400 : else
24401 2541 : mask = nelt - 1;
24402 :
24403 59309 : for (i = 0; i < nelt; ++i)
24404 : {
24405 53380 : unsigned j, e = d->perm[i] & mask;
24406 148712 : for (j = 0; j < eltsz; ++j)
24407 95332 : rperm[i * eltsz + j] = GEN_INT (e * eltsz + j);
24408 : }
24409 : }
24410 :
24411 6961 : machine_mode vpmode = vmode;
24412 :
24413 6961 : nelt = GET_MODE_SIZE (vmode);
24414 :
24415 : /* Emulate narrow modes with V16QI instructions. */
24416 6961 : if (nelt < 16)
24417 : {
24418 222 : rtx m128 = GEN_INT (-128);
24419 :
24420 : /* Remap elements from the second operand, as we have to
24421 : account for inactive top elements from the first operand. */
24422 222 : if (!d->one_operand_p)
24423 : {
24424 243 : for (i = 0; i < nelt; ++i)
24425 : {
24426 216 : unsigned ival = UINTVAL (rperm[i]);
24427 216 : if (ival >= nelt)
24428 108 : rperm[i] = GEN_INT (ival + 16 - nelt);
24429 : }
24430 : }
24431 :
24432 : /* Fill inactive elements in the top positions with zeros. */
24433 2570 : for (i = nelt; i < 16; ++i)
24434 2348 : rperm[i] = m128;
24435 :
24436 : vpmode = V16QImode;
24437 : }
24438 :
24439 13922 : vperm = gen_rtx_CONST_VECTOR (vpmode,
24440 6961 : gen_rtvec_v (GET_MODE_NUNITS (vpmode), rperm));
24441 6961 : vperm = force_reg (vpmode, vperm);
24442 :
24443 6961 : if (vmode == d->vmode)
24444 2415 : target = d->target;
24445 : else
24446 4546 : target = gen_reg_rtx (vmode);
24447 :
24448 6961 : op0 = gen_lowpart (vmode, d->op0);
24449 :
24450 6961 : if (d->one_operand_p)
24451 : {
24452 3749 : rtx (*gen) (rtx, rtx, rtx);
24453 :
24454 3749 : if (vmode == V4QImode)
24455 : gen = gen_mmx_pshufbv4qi3;
24456 : else if (vmode == V8QImode)
24457 : gen = gen_mmx_pshufbv8qi3;
24458 : else if (vmode == V16QImode)
24459 : gen = gen_ssse3_pshufbv16qi3;
24460 : else if (vmode == V32QImode)
24461 : gen = gen_avx2_pshufbv32qi3;
24462 : else if (vmode == V64QImode)
24463 : gen = gen_avx512bw_pshufbv64qi3;
24464 : else if (vmode == V8SFmode)
24465 : gen = gen_avx2_permvarv8sf;
24466 : else if (vmode == V8SImode)
24467 : gen = gen_avx2_permvarv8si;
24468 : else if (vmode == V16SFmode)
24469 : gen = gen_avx512f_permvarv16sf;
24470 : else if (vmode == V16SImode)
24471 : gen = gen_avx512f_permvarv16si;
24472 : else
24473 : gcc_unreachable ();
24474 :
24475 3749 : emit_insn (gen (target, op0, vperm));
24476 : }
24477 : else
24478 : {
24479 3212 : rtx (*gen) (rtx, rtx, rtx, rtx);
24480 :
24481 3212 : op1 = gen_lowpart (vmode, d->op1);
24482 :
24483 3212 : if (vmode == V4QImode)
24484 : gen = gen_mmx_ppermv32;
24485 : else if (vmode == V8QImode)
24486 : gen = gen_mmx_ppermv64;
24487 : else if (vmode == V16QImode)
24488 : gen = gen_xop_pperm;
24489 : else
24490 0 : gcc_unreachable ();
24491 :
24492 3212 : emit_insn (gen (target, op0, op1, vperm));
24493 : }
24494 :
24495 6961 : if (target != d->target)
24496 4546 : emit_move_insn (d->target, gen_lowpart (d->vmode, target));
24497 :
24498 : return true;
24499 : }
24500 :
24501 : /* Try to expand one-operand permutation with constant mask. */
24502 :
24503 : static bool
24504 128294 : ix86_expand_vec_one_operand_perm_avx512 (struct expand_vec_perm_d *d)
24505 : {
24506 128294 : machine_mode mode = GET_MODE (d->op0);
24507 128294 : machine_mode maskmode = mode;
24508 256588 : unsigned inner_size = GET_MODE_SIZE (GET_MODE_INNER (mode));
24509 128294 : rtx (*gen) (rtx, rtx, rtx) = NULL;
24510 128294 : rtx target, op0, mask;
24511 128294 : rtx vec[64];
24512 :
24513 128294 : if (!rtx_equal_p (d->op0, d->op1))
24514 : return false;
24515 :
24516 17927 : if (!TARGET_AVX512F)
24517 : return false;
24518 :
24519 : /* Accept VNxHImode and VNxQImode now. */
24520 744 : if (!TARGET_AVX512VL && GET_MODE_SIZE (mode) < 64)
24521 : return false;
24522 :
24523 : /* vpermw. */
24524 465 : if (!TARGET_AVX512BW && inner_size == 2)
24525 : return false;
24526 :
24527 : /* vpermb. */
24528 330 : if (!TARGET_AVX512VBMI && inner_size == 1)
24529 : return false;
24530 :
24531 211 : switch (mode)
24532 : {
24533 : case E_V16SImode:
24534 : gen = gen_avx512f_permvarv16si;
24535 : break;
24536 5 : case E_V16SFmode:
24537 5 : gen = gen_avx512f_permvarv16sf;
24538 5 : maskmode = V16SImode;
24539 5 : break;
24540 1 : case E_V8DImode:
24541 1 : gen = gen_avx512f_permvarv8di;
24542 1 : break;
24543 30 : case E_V8DFmode:
24544 30 : gen = gen_avx512f_permvarv8df;
24545 30 : maskmode = V8DImode;
24546 30 : break;
24547 106 : case E_V32HImode:
24548 106 : gen = gen_avx512bw_permvarv32hi;
24549 106 : break;
24550 14 : case E_V16HImode:
24551 14 : gen = gen_avx512vl_permvarv16hi;
24552 14 : break;
24553 6 : case E_V8HImode:
24554 6 : gen = gen_avx512vl_permvarv8hi;
24555 6 : break;
24556 4 : case E_V64QImode:
24557 4 : gen = gen_avx512bw_permvarv64qi;
24558 4 : break;
24559 2 : case E_V32QImode:
24560 2 : gen = gen_avx512vl_permvarv32qi;
24561 2 : break;
24562 0 : case E_V16QImode:
24563 0 : gen = gen_avx512vl_permvarv16qi;
24564 0 : break;
24565 :
24566 : default:
24567 : return false;
24568 : }
24569 :
24570 211 : if (d->testing_p)
24571 : return true;
24572 :
24573 201 : target = d->target;
24574 201 : op0 = d->op0;
24575 5041 : for (int i = 0; i < d->nelt; ++i)
24576 4840 : vec[i] = GEN_INT (d->perm[i]);
24577 201 : mask = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (d->nelt, vec));
24578 201 : emit_insn (gen (target, op0, force_reg (maskmode, mask)));
24579 201 : return true;
24580 : }
24581 :
24582 : static bool expand_vec_perm_palignr (struct expand_vec_perm_d *d, bool);
24583 :
24584 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to instantiate D
24585 : in a single instruction. */
24586 :
24587 : static bool
24588 352398 : expand_vec_perm_1 (struct expand_vec_perm_d *d)
24589 : {
24590 352398 : unsigned i, nelt = d->nelt;
24591 352398 : struct expand_vec_perm_d nd;
24592 :
24593 : /* Check plain VEC_SELECT first, because AVX has instructions that could
24594 : match both SEL and SEL+CONCAT, but the plain SEL will allow a memory
24595 : input where SEL+CONCAT may not. */
24596 352398 : if (d->one_operand_p)
24597 : {
24598 : int mask = nelt - 1;
24599 : bool identity_perm = true;
24600 : bool broadcast_perm = true;
24601 :
24602 520044 : for (i = 0; i < nelt; i++)
24603 : {
24604 456528 : nd.perm[i] = d->perm[i] & mask;
24605 456528 : if (nd.perm[i] != i)
24606 349876 : identity_perm = false;
24607 456528 : if (nd.perm[i])
24608 376998 : broadcast_perm = false;
24609 : }
24610 :
24611 63516 : if (identity_perm)
24612 : {
24613 11 : if (!d->testing_p)
24614 5 : emit_move_insn (d->target, d->op0);
24615 : return true;
24616 : }
24617 63505 : else if (broadcast_perm && TARGET_AVX2)
24618 : {
24619 : /* Use vpbroadcast{b,w,d}. */
24620 381 : rtx (*gen) (rtx, rtx) = NULL;
24621 381 : switch (d->vmode)
24622 : {
24623 1 : case E_V64QImode:
24624 1 : if (TARGET_AVX512BW)
24625 : gen = gen_avx512bw_vec_dupv64qi_1;
24626 : break;
24627 4 : case E_V32QImode:
24628 4 : gen = gen_avx2_pbroadcastv32qi_1;
24629 4 : break;
24630 1 : case E_V32HImode:
24631 1 : if (TARGET_AVX512BW)
24632 : gen = gen_avx512bw_vec_dupv32hi_1;
24633 : break;
24634 4 : case E_V16HImode:
24635 4 : gen = gen_avx2_pbroadcastv16hi_1;
24636 4 : break;
24637 1 : case E_V16SImode:
24638 1 : if (TARGET_AVX512F)
24639 : gen = gen_avx512f_vec_dupv16si_1;
24640 : break;
24641 4 : case E_V8SImode:
24642 4 : gen = gen_avx2_pbroadcastv8si_1;
24643 4 : break;
24644 4 : case E_V16QImode:
24645 4 : gen = gen_avx2_pbroadcastv16qi;
24646 4 : break;
24647 5 : case E_V8HImode:
24648 5 : gen = gen_avx2_pbroadcastv8hi;
24649 5 : break;
24650 0 : case E_V16SFmode:
24651 0 : if (TARGET_AVX512F)
24652 : gen = gen_avx512f_vec_dupv16sf_1;
24653 : break;
24654 : case E_V8SFmode:
24655 : gen = gen_avx2_vec_dupv8sf_1;
24656 : break;
24657 0 : case E_V8DFmode:
24658 0 : if (TARGET_AVX512F)
24659 : gen = gen_avx512f_vec_dupv8df_1;
24660 : break;
24661 0 : case E_V8DImode:
24662 0 : if (TARGET_AVX512F)
24663 : gen = gen_avx512f_vec_dupv8di_1;
24664 : break;
24665 : /* For other modes prefer other shuffles this function creates. */
24666 : default: break;
24667 : }
24668 21 : if (gen != NULL)
24669 : {
24670 24 : if (!d->testing_p)
24671 24 : emit_insn (gen (d->target, d->op0));
24672 : return true;
24673 : }
24674 : }
24675 :
24676 63481 : if (expand_vselect (d->target, d->op0, nd.perm, nelt, d->testing_p))
24677 : return true;
24678 :
24679 : /* There are plenty of patterns in sse.md that are written for
24680 : SEL+CONCAT and are not replicated for a single op. Perhaps
24681 : that should be changed, to avoid the nastiness here. */
24682 :
24683 : /* Recognize interleave style patterns, which means incrementing
24684 : every other permutation operand. */
24685 200489 : for (i = 0; i < nelt; i += 2)
24686 : {
24687 164499 : nd.perm[i] = d->perm[i] & mask;
24688 164499 : nd.perm[i + 1] = (d->perm[i + 1] & mask) + nelt;
24689 : }
24690 35990 : if (expand_vselect_vconcat (d->target, d->op0, d->op0, nd.perm, nelt,
24691 : d->testing_p))
24692 : return true;
24693 :
24694 : /* Recognize shufps, which means adding {0, 0, nelt, nelt}. */
24695 31768 : if (nelt >= 4)
24696 : {
24697 108349 : for (i = 0; i < nelt; i += 4)
24698 : {
24699 76581 : nd.perm[i + 0] = d->perm[i + 0] & mask;
24700 76581 : nd.perm[i + 1] = d->perm[i + 1] & mask;
24701 76581 : nd.perm[i + 2] = (d->perm[i + 2] & mask) + nelt;
24702 76581 : nd.perm[i + 3] = (d->perm[i + 3] & mask) + nelt;
24703 : }
24704 :
24705 31768 : if (expand_vselect_vconcat (d->target, d->op0, d->op0, nd.perm, nelt,
24706 : d->testing_p))
24707 : return true;
24708 : }
24709 : }
24710 :
24711 : /* Try the SSE4.1 blend variable merge instructions. */
24712 314849 : if (expand_vec_perm_blend (d))
24713 : return true;
24714 :
24715 : /* Try movss/movsd instructions. */
24716 311637 : if (expand_vec_perm_movs (d))
24717 : return true;
24718 :
24719 : /* Try the SSE4.1 insertps instruction. */
24720 296800 : if (expand_vec_perm_insertps (d))
24721 : return true;
24722 :
24723 : /* Try the fully general two operand permute. */
24724 291150 : if (expand_vselect_vconcat (d->target, d->op0, d->op1, d->perm, nelt,
24725 : d->testing_p))
24726 : return true;
24727 :
24728 : /* Recognize interleave style patterns with reversed operands. */
24729 140894 : if (!d->one_operand_p)
24730 : {
24731 915531 : for (i = 0; i < nelt; ++i)
24732 : {
24733 800502 : unsigned e = d->perm[i];
24734 800502 : if (e >= nelt)
24735 391744 : e -= nelt;
24736 : else
24737 408758 : e += nelt;
24738 800502 : nd.perm[i] = e;
24739 : }
24740 :
24741 115029 : if (expand_vselect_vconcat (d->target, d->op1, d->op0, nd.perm, nelt,
24742 : d->testing_p))
24743 : return true;
24744 : }
24745 :
24746 : /* Try one of the AVX vpermil variable permutations. */
24747 140739 : if (expand_vec_perm_vpermil (d))
24748 : return true;
24749 :
24750 : /* Try the SSSE3 pshufb or XOP vpperm or AVX2 vperm2i128,
24751 : vpshufb, vpermd, vpermps or vpermq variable permutation. */
24752 140151 : if (expand_vec_perm_pshufb (d))
24753 : return true;
24754 :
24755 : /* Try the AVX2 vpalignr instruction. */
24756 128418 : if (expand_vec_perm_palignr (d, true))
24757 : return true;
24758 :
24759 : /* Try the AVX512F vperm{w,b,s,d} instructions */
24760 128294 : if (ix86_expand_vec_one_operand_perm_avx512 (d))
24761 : return true;
24762 :
24763 : /* Try the AVX512F vpermt2/vpermi2 instructions. */
24764 128083 : if (ix86_expand_vec_perm_vpermt2 (NULL_RTX, NULL_RTX, NULL_RTX, NULL_RTX, d))
24765 : return true;
24766 :
24767 : /* See if we can get the same permutation in different vector integer
24768 : mode. */
24769 127188 : if (canonicalize_vector_int_perm (d, &nd) && expand_vec_perm_1 (&nd))
24770 : {
24771 6850 : if (!d->testing_p)
24772 1242 : emit_move_insn (d->target, gen_lowpart (d->vmode, nd.target));
24773 : return true;
24774 : }
24775 : return false;
24776 : }
24777 :
24778 : /* Canonicalize vec_perm index to make the first index
24779 : always comes from the first vector. */
24780 : static void
24781 8209 : ix86_vec_perm_index_canon (struct expand_vec_perm_d *d)
24782 : {
24783 8209 : unsigned nelt = d->nelt;
24784 8209 : if (d->perm[0] < nelt)
24785 : return;
24786 :
24787 5 : for (unsigned i = 0; i != nelt; i++)
24788 4 : d->perm[i] = (d->perm[i] + nelt) % (2 * nelt);
24789 :
24790 1 : std::swap (d->op0, d->op1);
24791 1 : return;
24792 : }
24793 :
24794 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
24795 : in terms of a pair of shufps+ shufps/pshufd instructions. */
24796 : static bool
24797 88420 : expand_vec_perm_shufps_shufps (struct expand_vec_perm_d *d)
24798 : {
24799 88420 : unsigned char perm1[4];
24800 88420 : machine_mode vmode = d->vmode;
24801 88420 : bool ok;
24802 88420 : unsigned i, j, k, count = 0;
24803 :
24804 88420 : if (d->one_operand_p
24805 82775 : || (vmode != V4SImode && vmode != V4SFmode))
24806 : return false;
24807 :
24808 36292 : if (d->testing_p)
24809 : return true;
24810 :
24811 8209 : ix86_vec_perm_index_canon (d);
24812 49254 : for (i = 0; i < 4; ++i)
24813 51356 : count += d->perm[i] > 3 ? 1 : 0;
24814 :
24815 8209 : gcc_assert (count & 3);
24816 :
24817 8209 : rtx tmp = gen_reg_rtx (vmode);
24818 : /* 2 from op0 and 2 from op1. */
24819 8209 : if (count == 2)
24820 : {
24821 : unsigned char perm2[4];
24822 18515 : for (i = 0, j = 0, k = 2; i < 4; ++i)
24823 14812 : if (d->perm[i] & 4)
24824 : {
24825 7406 : perm1[k++] = d->perm[i];
24826 7406 : perm2[i] = k - 1;
24827 : }
24828 : else
24829 : {
24830 7406 : perm1[j++] = d->perm[i];
24831 7406 : perm2[i] = j - 1;
24832 : }
24833 :
24834 : /* shufps. */
24835 7406 : ok = expand_vselect_vconcat (tmp, d->op0, d->op1,
24836 3703 : perm1, d->nelt, false);
24837 3703 : gcc_assert (ok);
24838 3703 : if (vmode == V4SImode && TARGET_SSE2)
24839 : /* pshufd. */
24840 2138 : ok = expand_vselect (d->target, tmp,
24841 2138 : perm2, d->nelt, false);
24842 : else
24843 : {
24844 : /* shufps. */
24845 1565 : perm2[2] += 4;
24846 1565 : perm2[3] += 4;
24847 1565 : ok = expand_vselect_vconcat (d->target, tmp, tmp,
24848 1565 : perm2, d->nelt, false);
24849 : }
24850 3703 : gcc_assert (ok);
24851 : }
24852 : /* 3 from one op and 1 from another. */
24853 : else
24854 : {
24855 22530 : unsigned pair_idx = 8, lone_idx = 8, shift;
24856 :
24857 : /* Find the lone index. */
24858 22530 : for (i = 0; i < 4; ++i)
24859 18024 : if ((d->perm[i] > 3 && count == 1)
24860 14720 : || (d->perm[i] < 4 && count == 3))
24861 18024 : lone_idx = i;
24862 :
24863 : /* When lone_idx is not 0, it must from second op(count == 1). */
24864 5708 : gcc_assert (count == (lone_idx ? 1 : 3));
24865 :
24866 : /* Find the pair index that sits in the same half as the lone index. */
24867 4506 : shift = lone_idx & 2;
24868 4506 : pair_idx = 1 - lone_idx + 2 * shift;
24869 :
24870 : /* First permutate lone index and pair index into the same vector as
24871 : [ lone, lone, pair, pair ]. */
24872 9012 : perm1[1] = perm1[0]
24873 4506 : = (count == 3) ? d->perm[lone_idx] : d->perm[lone_idx] - 4;
24874 9012 : perm1[3] = perm1[2]
24875 4506 : = (count == 3) ? d->perm[pair_idx] : d->perm[pair_idx] + 4;
24876 :
24877 : /* Always put the vector contains lone indx at the first. */
24878 4506 : if (count == 1)
24879 3304 : std::swap (d->op0, d->op1);
24880 :
24881 : /* shufps. */
24882 9012 : ok = expand_vselect_vconcat (tmp, d->op0, d->op1,
24883 4506 : perm1, d->nelt, false);
24884 4506 : gcc_assert (ok);
24885 :
24886 : /* Refine lone and pair index to original order. */
24887 4506 : perm1[shift] = lone_idx << 1;
24888 4506 : perm1[shift + 1] = pair_idx << 1;
24889 :
24890 : /* Select the remaining 2 elements in another vector. */
24891 13518 : for (i = 2 - shift; i < 4 - shift; ++i)
24892 9012 : perm1[i] = lone_idx == 1 ? d->perm[i] + 4 : d->perm[i];
24893 :
24894 : /* Adjust to original selector. */
24895 4506 : if (lone_idx > 1)
24896 2232 : std::swap (tmp, d->op1);
24897 :
24898 : /* shufps. */
24899 9012 : ok = expand_vselect_vconcat (d->target, tmp, d->op1,
24900 4506 : perm1, d->nelt, false);
24901 :
24902 4506 : gcc_assert (ok);
24903 : }
24904 :
24905 : return true;
24906 : }
24907 :
24908 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
24909 : in terms of a pair of pshuflw + pshufhw instructions. */
24910 :
24911 : static bool
24912 105328 : expand_vec_perm_pshuflw_pshufhw (struct expand_vec_perm_d *d)
24913 : {
24914 105328 : unsigned char perm2[MAX_VECT_LEN];
24915 105328 : unsigned i;
24916 105328 : bool ok;
24917 :
24918 105328 : if (d->vmode != V8HImode || !d->one_operand_p)
24919 : return false;
24920 :
24921 : /* The two permutations only operate in 64-bit lanes. */
24922 12993 : for (i = 0; i < 4; ++i)
24923 10488 : if (d->perm[i] >= 4)
24924 : return false;
24925 12469 : for (i = 4; i < 8; ++i)
24926 9978 : if (d->perm[i] < 4)
24927 : return false;
24928 :
24929 2491 : if (d->testing_p)
24930 : return true;
24931 :
24932 : /* Emit the pshuflw. */
24933 136 : memcpy (perm2, d->perm, 4);
24934 680 : for (i = 4; i < 8; ++i)
24935 544 : perm2[i] = i;
24936 136 : ok = expand_vselect (d->target, d->op0, perm2, 8, d->testing_p);
24937 136 : gcc_assert (ok);
24938 :
24939 : /* Emit the pshufhw. */
24940 136 : memcpy (perm2 + 4, d->perm + 4, 4);
24941 680 : for (i = 0; i < 4; ++i)
24942 544 : perm2[i] = i;
24943 136 : ok = expand_vselect (d->target, d->target, perm2, 8, d->testing_p);
24944 136 : gcc_assert (ok);
24945 :
24946 : return true;
24947 : }
24948 :
24949 : /* Try to permute 2 64-bit vectors by punpckldq + 128-bit vector shuffle. */
24950 : static bool
24951 52128 : expand_vec_perm_punpckldq_pshuf (struct expand_vec_perm_d *d)
24952 : {
24953 52128 : if (GET_MODE_BITSIZE (d->vmode) != 64
24954 17524 : || !TARGET_MMX_WITH_SSE
24955 69652 : || d->one_operand_p)
24956 : return false;
24957 :
24958 16030 : machine_mode widen_vmode;
24959 16030 : switch (d->vmode)
24960 : {
24961 : /* pshufd. */
24962 : case E_V2SImode:
24963 : widen_vmode = V4SImode;
24964 : break;
24965 :
24966 : /* pshufd. */
24967 1501 : case E_V2SFmode:
24968 1501 : widen_vmode = V4SFmode;
24969 1501 : break;
24970 :
24971 5293 : case E_V4HImode:
24972 5293 : widen_vmode = V8HImode;
24973 : /* pshufb. */
24974 5293 : if (!TARGET_SSSE3)
24975 : return false;
24976 : break;
24977 :
24978 6292 : case E_V8QImode:
24979 : /* pshufb. */
24980 6292 : widen_vmode = V16QImode;
24981 6292 : if (!TARGET_SSSE3)
24982 : return false;
24983 : break;
24984 :
24985 : default:
24986 : return false;
24987 : }
24988 :
24989 6544 : if (d->testing_p)
24990 : return true;
24991 :
24992 403 : struct expand_vec_perm_d dperm;
24993 403 : dperm.target = gen_reg_rtx (widen_vmode);
24994 403 : rtx op0 = gen_reg_rtx (widen_vmode);
24995 403 : emit_move_insn (op0, gen_rtx_VEC_CONCAT (widen_vmode, d->op0, d->op1));
24996 403 : dperm.op0 = op0;
24997 403 : dperm.op1 = op0;
24998 403 : dperm.vmode = widen_vmode;
24999 403 : unsigned nelt = GET_MODE_NUNITS (widen_vmode);
25000 403 : dperm.nelt = nelt;
25001 403 : dperm.one_operand_p = true;
25002 403 : dperm.testing_p = false;
25003 :
25004 2143 : for (unsigned i = 0; i != nelt / 2; i++)
25005 : {
25006 1740 : dperm.perm[i] = d->perm[i];
25007 1740 : dperm.perm[i + nelt / 2] = d->perm[i];
25008 : }
25009 :
25010 403 : gcc_assert (expand_vec_perm_1 (&dperm));
25011 403 : emit_move_insn (d->target, lowpart_subreg (d->vmode,
25012 : dperm.target,
25013 : dperm.vmode));
25014 403 : return true;
25015 : }
25016 :
25017 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
25018 : the permutation using the SSSE3 palignr instruction. This succeeds
25019 : when all of the elements in PERM fit within one vector and we merely
25020 : need to shift them down so that a single vector permutation has a
25021 : chance to succeed. If SINGLE_INSN_ONLY_P, succeed if only
25022 : the vpalignr instruction itself can perform the requested permutation. */
25023 :
25024 : static bool
25025 231255 : expand_vec_perm_palignr (struct expand_vec_perm_d *d, bool single_insn_only_p)
25026 : {
25027 231255 : unsigned i, nelt = d->nelt;
25028 231255 : unsigned min, max, minswap, maxswap;
25029 231255 : bool in_order, ok, swap = false;
25030 231255 : rtx shift, target;
25031 231255 : struct expand_vec_perm_d dcopy;
25032 :
25033 : /* Even with AVX, palignr only operates on 128-bit vectors,
25034 : in AVX2 palignr operates on both 128-bit lanes. */
25035 117286 : if ((!TARGET_SSSE3 || GET_MODE_SIZE (d->vmode) != 16)
25036 273623 : && (!TARGET_AVX2 || GET_MODE_SIZE (d->vmode) != 32))
25037 : return false;
25038 :
25039 34402 : min = 2 * nelt;
25040 34402 : max = 0;
25041 34402 : minswap = 2 * nelt;
25042 34402 : maxswap = 0;
25043 238478 : for (i = 0; i < nelt; ++i)
25044 : {
25045 204076 : unsigned e = d->perm[i];
25046 204076 : unsigned eswap = d->perm[i] ^ nelt;
25047 408152 : if (GET_MODE_SIZE (d->vmode) == 32)
25048 : {
25049 69192 : e = (e & ((nelt / 2) - 1)) | ((e & nelt) >> 1);
25050 69192 : eswap = e ^ (nelt / 2);
25051 : }
25052 204076 : if (e < min)
25053 : min = e;
25054 204076 : if (e > max)
25055 : max = e;
25056 204076 : if (eswap < minswap)
25057 : minswap = eswap;
25058 204076 : if (eswap > maxswap)
25059 : maxswap = eswap;
25060 : }
25061 34402 : if (min == 0
25062 50666 : || max - min >= (GET_MODE_SIZE (d->vmode) == 32 ? nelt / 2 : nelt))
25063 : {
25064 31202 : if (d->one_operand_p
25065 30933 : || minswap == 0
25066 66072 : || maxswap - minswap >= (GET_MODE_SIZE (d->vmode) == 32
25067 17435 : ? nelt / 2 : nelt))
25068 : return false;
25069 : swap = true;
25070 : min = minswap;
25071 6420 : max = maxswap;
25072 : }
25073 :
25074 : /* Given that we have SSSE3, we know we'll be able to implement the
25075 : single operand permutation after the palignr with pshufb for
25076 : 128-bit vectors. If SINGLE_INSN_ONLY_P, in_order has to be computed
25077 : first. */
25078 6474 : if (d->testing_p && GET_MODE_SIZE (d->vmode) == 16 && !single_insn_only_p)
25079 : return true;
25080 :
25081 6420 : dcopy = *d;
25082 6420 : if (swap)
25083 : {
25084 3220 : dcopy.op0 = d->op1;
25085 3220 : dcopy.op1 = d->op0;
25086 16172 : for (i = 0; i < nelt; ++i)
25087 12952 : dcopy.perm[i] ^= nelt;
25088 : }
25089 :
25090 6420 : in_order = true;
25091 32668 : for (i = 0; i < nelt; ++i)
25092 : {
25093 26248 : unsigned e = dcopy.perm[i];
25094 26248 : if (GET_MODE_SIZE (d->vmode) == 32
25095 1152 : && e >= nelt
25096 26510 : && (e & (nelt / 2 - 1)) < min)
25097 262 : e = e - min - (nelt / 2);
25098 : else
25099 25986 : e = e - min;
25100 26248 : if (e != i)
25101 19394 : in_order = false;
25102 26248 : dcopy.perm[i] = e;
25103 : }
25104 6420 : dcopy.one_operand_p = true;
25105 :
25106 6420 : if (single_insn_only_p && !in_order)
25107 : return false;
25108 :
25109 : /* For AVX2, test whether we can permute the result in one instruction. */
25110 3271 : if (d->testing_p)
25111 : {
25112 54 : if (in_order)
25113 : return true;
25114 0 : dcopy.op1 = dcopy.op0;
25115 0 : return expand_vec_perm_1 (&dcopy);
25116 : }
25117 :
25118 6434 : shift = GEN_INT (min * GET_MODE_UNIT_BITSIZE (d->vmode));
25119 6434 : if (GET_MODE_SIZE (d->vmode) == 16)
25120 : {
25121 3145 : target = gen_reg_rtx (V1TImode);
25122 3145 : emit_insn (gen_ssse3_palignrv1ti (target,
25123 3145 : gen_lowpart (V1TImode, dcopy.op1),
25124 3145 : gen_lowpart (V1TImode, dcopy.op0),
25125 : shift));
25126 : }
25127 : else
25128 : {
25129 72 : target = gen_reg_rtx (V2TImode);
25130 72 : emit_insn (gen_avx2_palignrv2ti (target,
25131 72 : gen_lowpart (V2TImode, dcopy.op1),
25132 72 : gen_lowpart (V2TImode, dcopy.op0),
25133 : shift));
25134 : }
25135 :
25136 3217 : dcopy.op0 = dcopy.op1 = gen_lowpart (d->vmode, target);
25137 :
25138 : /* Test for the degenerate case where the alignment by itself
25139 : produces the desired permutation. */
25140 3217 : if (in_order)
25141 : {
25142 70 : emit_move_insn (d->target, dcopy.op0);
25143 70 : return true;
25144 : }
25145 :
25146 3147 : ok = expand_vec_perm_1 (&dcopy);
25147 3159 : gcc_assert (ok || GET_MODE_SIZE (d->vmode) == 32);
25148 :
25149 : return ok;
25150 : }
25151 :
25152 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
25153 : the permutation using the SSE4_1 pblendv instruction. Potentially
25154 : reduces permutation from 2 pshufb and or to 1 pshufb and pblendv. */
25155 :
25156 : static bool
25157 93355 : expand_vec_perm_pblendv (struct expand_vec_perm_d *d)
25158 : {
25159 93355 : unsigned i, which, nelt = d->nelt;
25160 93355 : struct expand_vec_perm_d dcopy, dcopy1;
25161 93355 : machine_mode vmode = d->vmode;
25162 93355 : bool ok;
25163 :
25164 : /* Use the same checks as in expand_vec_perm_blend. */
25165 93355 : if (d->one_operand_p)
25166 : return false;
25167 91549 : if (TARGET_AVX2 && GET_MODE_SIZE (vmode) == 32)
25168 : ;
25169 86003 : else if (TARGET_AVX && (vmode == V4DFmode || vmode == V8SFmode))
25170 : ;
25171 82912 : else if (TARGET_SSE4_1
25172 93503 : && (GET_MODE_SIZE (vmode) == 16
25173 9198 : || (TARGET_MMX_WITH_SSE && GET_MODE_SIZE (vmode) == 8)
25174 2777 : || GET_MODE_SIZE (vmode) == 4))
25175 : ;
25176 : else
25177 : return false;
25178 :
25179 : /* Figure out where permutation elements stay not in their
25180 : respective lanes. */
25181 106827 : for (i = 0, which = 0; i < nelt; ++i)
25182 : {
25183 91774 : unsigned e = d->perm[i];
25184 91774 : if (e != i)
25185 124373 : which |= (e < nelt ? 1 : 2);
25186 : }
25187 : /* We can pblend the part where elements stay not in their
25188 : respective lanes only when these elements are all in one
25189 : half of a permutation.
25190 : {0 1 8 3 4 5 9 7} is ok as 8, 9 are at not at their respective
25191 : lanes, but both 8 and 9 >= 8
25192 : {0 1 8 3 4 5 2 7} is not ok as 2 and 8 are not at their
25193 : respective lanes and 8 >= 8, but 2 not. */
25194 15053 : if (which != 1 && which != 2)
25195 : return false;
25196 4815 : if (d->testing_p && GET_MODE_SIZE (vmode) == 16)
25197 : return true;
25198 :
25199 : /* First we apply one operand permutation to the part where
25200 : elements stay not in their respective lanes. */
25201 2778 : dcopy = *d;
25202 2778 : if (which == 2)
25203 2778 : dcopy.op0 = dcopy.op1 = d->op1;
25204 : else
25205 0 : dcopy.op0 = dcopy.op1 = d->op0;
25206 2778 : if (!d->testing_p)
25207 741 : dcopy.target = gen_reg_rtx (vmode);
25208 2778 : dcopy.one_operand_p = true;
25209 :
25210 21494 : for (i = 0; i < nelt; ++i)
25211 18716 : dcopy.perm[i] = d->perm[i] & (nelt - 1);
25212 :
25213 2778 : ok = expand_vec_perm_1 (&dcopy);
25214 5556 : if (GET_MODE_SIZE (vmode) != 16 && !ok)
25215 : return false;
25216 : else
25217 2435 : gcc_assert (ok);
25218 2435 : if (d->testing_p)
25219 : return true;
25220 :
25221 : /* Next we put permuted elements into their positions. */
25222 679 : dcopy1 = *d;
25223 679 : if (which == 2)
25224 679 : dcopy1.op1 = dcopy.target;
25225 : else
25226 0 : dcopy1.op0 = dcopy.target;
25227 :
25228 5751 : for (i = 0; i < nelt; ++i)
25229 5072 : dcopy1.perm[i] = ((d->perm[i] >= nelt) ? (nelt + i) : i);
25230 :
25231 679 : ok = expand_vec_perm_blend (&dcopy1);
25232 679 : gcc_assert (ok);
25233 :
25234 : return true;
25235 : }
25236 :
25237 : static bool expand_vec_perm_interleave3 (struct expand_vec_perm_d *d);
25238 :
25239 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
25240 : a two vector permutation into a single vector permutation by using
25241 : an interleave operation to merge the vectors. */
25242 :
25243 : static bool
25244 99702 : expand_vec_perm_interleave2 (struct expand_vec_perm_d *d)
25245 : {
25246 99702 : struct expand_vec_perm_d dremap, dfinal;
25247 99702 : unsigned i, nelt = d->nelt, nelt2 = nelt / 2;
25248 99702 : unsigned HOST_WIDE_INT contents;
25249 99702 : unsigned char remap[2 * MAX_VECT_LEN];
25250 99702 : rtx_insn *seq;
25251 99702 : bool ok, same_halves = false;
25252 :
25253 99702 : if (GET_MODE_SIZE (d->vmode) == 4
25254 178874 : || GET_MODE_SIZE (d->vmode) == 8
25255 240222 : || GET_MODE_SIZE (d->vmode) == 16)
25256 : {
25257 93398 : if (d->one_operand_p)
25258 : return false;
25259 : }
25260 12608 : else if (GET_MODE_SIZE (d->vmode) == 32)
25261 : {
25262 5987 : if (!TARGET_AVX)
25263 : return false;
25264 : /* For 32-byte modes allow even d->one_operand_p.
25265 : The lack of cross-lane shuffling in some instructions
25266 : might prevent a single insn shuffle. */
25267 5987 : dfinal = *d;
25268 5987 : dfinal.testing_p = true;
25269 : /* If expand_vec_perm_interleave3 can expand this into
25270 : a 3 insn sequence, give up and let it be expanded as
25271 : 3 insn sequence. While that is one insn longer,
25272 : it doesn't need a memory operand and in the common
25273 : case that both interleave low and high permutations
25274 : with the same operands are adjacent needs 4 insns
25275 : for both after CSE. */
25276 5987 : if (expand_vec_perm_interleave3 (&dfinal))
25277 : return false;
25278 : }
25279 : else
25280 : return false;
25281 :
25282 : /* Examine from whence the elements come. */
25283 93594 : contents = 0;
25284 695944 : for (i = 0; i < nelt; ++i)
25285 602350 : contents |= HOST_WIDE_INT_1U << d->perm[i];
25286 :
25287 93594 : memset (remap, 0xff, sizeof (remap));
25288 93594 : dremap = *d;
25289 :
25290 93594 : if (GET_MODE_SIZE (d->vmode) == 4
25291 178755 : || GET_MODE_SIZE (d->vmode) == 8)
25292 : {
25293 26112 : unsigned HOST_WIDE_INT h1, h2, h3, h4;
25294 :
25295 : /* Split the two input vectors into 4 halves. */
25296 26112 : h1 = (HOST_WIDE_INT_1U << nelt2) - 1;
25297 26112 : h2 = h1 << nelt2;
25298 26112 : h3 = h2 << nelt2;
25299 26112 : h4 = h3 << nelt2;
25300 :
25301 : /* If the elements from the low halves use interleave low,
25302 : and similarly for interleave high. */
25303 26112 : if ((contents & (h1 | h3)) == contents)
25304 : {
25305 : /* punpckl* */
25306 2832 : for (i = 0; i < nelt2; ++i)
25307 : {
25308 1936 : remap[i] = i * 2;
25309 1936 : remap[i + nelt] = i * 2 + 1;
25310 1936 : dremap.perm[i * 2] = i;
25311 1936 : dremap.perm[i * 2 + 1] = i + nelt;
25312 : }
25313 : }
25314 25216 : else if ((contents & (h2 | h4)) == contents)
25315 : {
25316 : /* punpckh* */
25317 2277 : for (i = 0; i < nelt2; ++i)
25318 : {
25319 1548 : remap[i + nelt2] = i * 2;
25320 1548 : remap[i + nelt + nelt2] = i * 2 + 1;
25321 1548 : dremap.perm[i * 2] = i + nelt2;
25322 1548 : dremap.perm[i * 2 + 1] = i + nelt + nelt2;
25323 : }
25324 : }
25325 : else
25326 : return false;
25327 : }
25328 134964 : else if (GET_MODE_SIZE (d->vmode) == 16)
25329 : {
25330 61721 : unsigned HOST_WIDE_INT h1, h2, h3, h4;
25331 :
25332 : /* Split the two input vectors into 4 halves. */
25333 61721 : h1 = (HOST_WIDE_INT_1U << nelt2) - 1;
25334 61721 : h2 = h1 << nelt2;
25335 61721 : h3 = h2 << nelt2;
25336 61721 : h4 = h3 << nelt2;
25337 :
25338 : /* If the elements from the low halves use interleave low, and similarly
25339 : for interleave high. If the elements are from mis-matched halves, we
25340 : can use shufps for V4SF/V4SI or do a DImode shuffle. */
25341 61721 : if ((contents & (h1 | h3)) == contents)
25342 : {
25343 : /* punpckl* */
25344 5449 : for (i = 0; i < nelt2; ++i)
25345 : {
25346 3976 : remap[i] = i * 2;
25347 3976 : remap[i + nelt] = i * 2 + 1;
25348 3976 : dremap.perm[i * 2] = i;
25349 3976 : dremap.perm[i * 2 + 1] = i + nelt;
25350 : }
25351 1473 : if (!TARGET_SSE2 && d->vmode == V4SImode)
25352 0 : dremap.vmode = V4SFmode;
25353 : }
25354 60248 : else if ((contents & (h2 | h4)) == contents)
25355 : {
25356 : /* punpckh* */
25357 4446 : for (i = 0; i < nelt2; ++i)
25358 : {
25359 3200 : remap[i + nelt2] = i * 2;
25360 3200 : remap[i + nelt + nelt2] = i * 2 + 1;
25361 3200 : dremap.perm[i * 2] = i + nelt2;
25362 3200 : dremap.perm[i * 2 + 1] = i + nelt + nelt2;
25363 : }
25364 1246 : if (!TARGET_SSE2 && d->vmode == V4SImode)
25365 0 : dremap.vmode = V4SFmode;
25366 : }
25367 59002 : else if ((contents & (h1 | h4)) == contents)
25368 : {
25369 : /* shufps */
25370 3188 : for (i = 0; i < nelt2; ++i)
25371 : {
25372 2364 : remap[i] = i;
25373 2364 : remap[i + nelt + nelt2] = i + nelt2;
25374 2364 : dremap.perm[i] = i;
25375 2364 : dremap.perm[i + nelt2] = i + nelt + nelt2;
25376 : }
25377 824 : if (nelt != 4)
25378 : {
25379 : /* shufpd */
25380 120 : dremap.vmode = V2DImode;
25381 120 : dremap.nelt = 2;
25382 120 : dremap.perm[0] = 0;
25383 120 : dremap.perm[1] = 3;
25384 : }
25385 : }
25386 58178 : else if ((contents & (h2 | h3)) == contents)
25387 : {
25388 : /* shufps */
25389 4684 : for (i = 0; i < nelt2; ++i)
25390 : {
25391 3438 : remap[i + nelt2] = i;
25392 3438 : remap[i + nelt] = i + nelt2;
25393 3438 : dremap.perm[i] = i + nelt2;
25394 3438 : dremap.perm[i + nelt2] = i + nelt;
25395 : }
25396 1246 : if (nelt != 4)
25397 : {
25398 : /* shufpd */
25399 195 : dremap.vmode = V2DImode;
25400 195 : dremap.nelt = 2;
25401 195 : dremap.perm[0] = 1;
25402 195 : dremap.perm[1] = 2;
25403 : }
25404 : }
25405 : else
25406 : return false;
25407 : }
25408 : else
25409 : {
25410 5761 : unsigned int nelt4 = nelt / 4, nzcnt = 0;
25411 5761 : unsigned HOST_WIDE_INT q[8];
25412 5761 : unsigned int nonzero_halves[4];
25413 :
25414 : /* Split the two input vectors into 8 quarters. */
25415 5761 : q[0] = (HOST_WIDE_INT_1U << nelt4) - 1;
25416 46088 : for (i = 1; i < 8; ++i)
25417 40327 : q[i] = q[0] << (nelt4 * i);
25418 28805 : for (i = 0; i < 4; ++i)
25419 23044 : if (((q[2 * i] | q[2 * i + 1]) & contents) != 0)
25420 : {
25421 20526 : nonzero_halves[nzcnt] = i;
25422 20526 : ++nzcnt;
25423 : }
25424 :
25425 5761 : if (nzcnt == 1)
25426 : {
25427 215 : gcc_assert (d->one_operand_p);
25428 215 : nonzero_halves[1] = nonzero_halves[0];
25429 215 : same_halves = true;
25430 : }
25431 5546 : else if (d->one_operand_p)
25432 : {
25433 115 : gcc_assert (nonzero_halves[0] == 0);
25434 115 : gcc_assert (nonzero_halves[1] == 1);
25435 : }
25436 :
25437 5761 : if (nzcnt <= 2)
25438 : {
25439 582 : if (d->perm[0] / nelt2 == nonzero_halves[1])
25440 : {
25441 : /* Attempt to increase the likelihood that dfinal
25442 : shuffle will be intra-lane. */
25443 315 : std::swap (nonzero_halves[0], nonzero_halves[1]);
25444 : }
25445 :
25446 : /* vperm2f128 or vperm2i128. */
25447 3532 : for (i = 0; i < nelt2; ++i)
25448 : {
25449 2950 : remap[i + nonzero_halves[1] * nelt2] = i + nelt2;
25450 2950 : remap[i + nonzero_halves[0] * nelt2] = i;
25451 2950 : dremap.perm[i + nelt2] = i + nonzero_halves[1] * nelt2;
25452 2950 : dremap.perm[i] = i + nonzero_halves[0] * nelt2;
25453 : }
25454 :
25455 582 : if (d->vmode != V8SFmode
25456 : && d->vmode != V4DFmode
25457 : && d->vmode != V8SImode)
25458 : {
25459 188 : dremap.vmode = V8SImode;
25460 188 : dremap.nelt = 8;
25461 940 : for (i = 0; i < 4; ++i)
25462 : {
25463 752 : dremap.perm[i] = i + nonzero_halves[0] * 4;
25464 752 : dremap.perm[i + 4] = i + nonzero_halves[1] * 4;
25465 : }
25466 : }
25467 : }
25468 5179 : else if (d->one_operand_p)
25469 4714 : return false;
25470 5179 : else if (TARGET_AVX2
25471 2132 : && (contents & (q[0] | q[2] | q[4] | q[6])) == contents)
25472 : {
25473 : /* vpunpckl* */
25474 491 : for (i = 0; i < nelt4; ++i)
25475 : {
25476 247 : remap[i] = i * 2;
25477 247 : remap[i + nelt] = i * 2 + 1;
25478 247 : remap[i + nelt2] = i * 2 + nelt2;
25479 247 : remap[i + nelt + nelt2] = i * 2 + nelt2 + 1;
25480 247 : dremap.perm[i * 2] = i;
25481 247 : dremap.perm[i * 2 + 1] = i + nelt;
25482 247 : dremap.perm[i * 2 + nelt2] = i + nelt2;
25483 247 : dremap.perm[i * 2 + nelt2 + 1] = i + nelt + nelt2;
25484 : }
25485 : }
25486 4935 : else if (TARGET_AVX2
25487 1888 : && (contents & (q[1] | q[3] | q[5] | q[7])) == contents)
25488 : {
25489 : /* vpunpckh* */
25490 445 : for (i = 0; i < nelt4; ++i)
25491 : {
25492 224 : remap[i + nelt4] = i * 2;
25493 224 : remap[i + nelt + nelt4] = i * 2 + 1;
25494 224 : remap[i + nelt2 + nelt4] = i * 2 + nelt2;
25495 224 : remap[i + nelt + nelt2 + nelt4] = i * 2 + nelt2 + 1;
25496 224 : dremap.perm[i * 2] = i + nelt4;
25497 224 : dremap.perm[i * 2 + 1] = i + nelt + nelt4;
25498 224 : dremap.perm[i * 2 + nelt2] = i + nelt2 + nelt4;
25499 224 : dremap.perm[i * 2 + nelt2 + 1] = i + nelt + nelt2 + nelt4;
25500 : }
25501 : }
25502 : else
25503 : return false;
25504 : }
25505 :
25506 : /* Use the remapping array set up above to move the elements from their
25507 : swizzled locations into their final destinations. */
25508 7461 : dfinal = *d;
25509 48169 : for (i = 0; i < nelt; ++i)
25510 : {
25511 40708 : unsigned e = remap[d->perm[i]];
25512 40708 : gcc_assert (e < nelt);
25513 : /* If same_halves is true, both halves of the remapped vector are the
25514 : same. Avoid cross-lane accesses if possible. */
25515 40708 : if (same_halves && i >= nelt2)
25516 : {
25517 792 : gcc_assert (e < nelt2);
25518 792 : dfinal.perm[i] = e + nelt2;
25519 : }
25520 : else
25521 39916 : dfinal.perm[i] = e;
25522 : }
25523 7461 : if (!d->testing_p)
25524 : {
25525 2689 : dremap.target = gen_reg_rtx (dremap.vmode);
25526 2689 : dfinal.op0 = gen_lowpart (dfinal.vmode, dremap.target);
25527 : }
25528 7461 : dfinal.op1 = dfinal.op0;
25529 7461 : dfinal.one_operand_p = true;
25530 :
25531 : /* Test if the final remap can be done with a single insn. For V4SFmode or
25532 : V4SImode this *will* succeed. For V8HImode or V16QImode it may not. */
25533 7461 : start_sequence ();
25534 7461 : ok = expand_vec_perm_1 (&dfinal);
25535 7461 : seq = end_sequence ();
25536 :
25537 7461 : if (!ok)
25538 : return false;
25539 :
25540 6179 : if (d->testing_p)
25541 : return true;
25542 :
25543 2650 : if (dremap.vmode != dfinal.vmode)
25544 : {
25545 57 : dremap.op0 = gen_lowpart (dremap.vmode, dremap.op0);
25546 57 : dremap.op1 = gen_lowpart (dremap.vmode, dremap.op1);
25547 : }
25548 :
25549 2650 : ok = expand_vec_perm_1 (&dremap);
25550 2650 : gcc_assert (ok);
25551 :
25552 2650 : emit_insn (seq);
25553 2650 : return true;
25554 : }
25555 :
25556 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
25557 : a single vector cross-lane permutation into vpermq followed
25558 : by any of the single insn permutations. */
25559 :
25560 : static bool
25561 93423 : expand_vec_perm_vpermq_perm_1 (struct expand_vec_perm_d *d)
25562 : {
25563 93423 : struct expand_vec_perm_d dremap, dfinal;
25564 93423 : unsigned i, j, nelt = d->nelt, nelt2 = nelt / 2, nelt4 = nelt / 4;
25565 93423 : unsigned contents[2];
25566 93423 : bool ok;
25567 :
25568 93423 : if (!(TARGET_AVX2
25569 4069 : && (d->vmode == V32QImode || d->vmode == V16HImode)
25570 248 : && d->one_operand_p))
25571 : return false;
25572 :
25573 7 : contents[0] = 0;
25574 7 : contents[1] = 0;
25575 103 : for (i = 0; i < nelt2; ++i)
25576 : {
25577 96 : contents[0] |= 1u << (d->perm[i] / nelt4);
25578 96 : contents[1] |= 1u << (d->perm[i + nelt2] / nelt4);
25579 : }
25580 :
25581 7 : for (i = 0; i < 2; ++i)
25582 : {
25583 : unsigned int cnt = 0;
25584 21 : for (j = 0; j < 4; ++j)
25585 21 : if ((contents[i] & (1u << j)) != 0 && ++cnt > 2)
25586 : return false;
25587 : }
25588 :
25589 0 : if (d->testing_p)
25590 : return true;
25591 :
25592 0 : dremap = *d;
25593 0 : dremap.vmode = V4DImode;
25594 0 : dremap.nelt = 4;
25595 0 : dremap.target = gen_reg_rtx (V4DImode);
25596 0 : dremap.op0 = gen_lowpart (V4DImode, d->op0);
25597 0 : dremap.op1 = dremap.op0;
25598 0 : dremap.one_operand_p = true;
25599 0 : for (i = 0; i < 2; ++i)
25600 : {
25601 : unsigned int cnt = 0;
25602 0 : for (j = 0; j < 4; ++j)
25603 0 : if ((contents[i] & (1u << j)) != 0)
25604 0 : dremap.perm[2 * i + cnt++] = j;
25605 0 : for (; cnt < 2; ++cnt)
25606 0 : dremap.perm[2 * i + cnt] = 0;
25607 : }
25608 :
25609 0 : dfinal = *d;
25610 0 : dfinal.op0 = gen_lowpart (dfinal.vmode, dremap.target);
25611 0 : dfinal.op1 = dfinal.op0;
25612 0 : dfinal.one_operand_p = true;
25613 0 : for (i = 0, j = 0; i < nelt; ++i)
25614 : {
25615 0 : if (i == nelt2)
25616 0 : j = 2;
25617 0 : dfinal.perm[i] = (d->perm[i] & (nelt4 - 1)) | (j ? nelt2 : 0);
25618 0 : if ((d->perm[i] / nelt4) == dremap.perm[j])
25619 : ;
25620 0 : else if ((d->perm[i] / nelt4) == dremap.perm[j + 1])
25621 0 : dfinal.perm[i] |= nelt4;
25622 : else
25623 0 : gcc_unreachable ();
25624 : }
25625 :
25626 0 : ok = expand_vec_perm_1 (&dremap);
25627 0 : gcc_assert (ok);
25628 :
25629 0 : ok = expand_vec_perm_1 (&dfinal);
25630 0 : gcc_assert (ok);
25631 :
25632 : return true;
25633 : }
25634 :
25635 : static bool canonicalize_perm (struct expand_vec_perm_d *d);
25636 :
25637 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to expand
25638 : a vector permutation using two instructions, vperm2f128 resp.
25639 : vperm2i128 followed by any single in-lane permutation. */
25640 :
25641 : static bool
25642 93423 : expand_vec_perm_vperm2f128 (struct expand_vec_perm_d *d)
25643 : {
25644 93423 : struct expand_vec_perm_d dfirst, dsecond;
25645 93423 : unsigned i, j, nelt = d->nelt, nelt2 = nelt / 2, perm;
25646 93423 : bool ok;
25647 :
25648 93423 : if (!TARGET_AVX
25649 21590 : || GET_MODE_SIZE (d->vmode) != 32
25650 98488 : || (d->vmode != V8SFmode && d->vmode != V4DFmode && !TARGET_AVX2))
25651 : return false;
25652 :
25653 4881 : dsecond = *d;
25654 4881 : dsecond.one_operand_p = false;
25655 4881 : dsecond.testing_p = true;
25656 :
25657 : /* ((perm << 2)|perm) & 0x33 is the vperm2[fi]128
25658 : immediate. For perm < 16 the second permutation uses
25659 : d->op0 as first operand, for perm >= 16 it uses d->op1
25660 : as first operand. The second operand is the result of
25661 : vperm2[fi]128. */
25662 159747 : for (perm = 0; perm < 32; perm++)
25663 : {
25664 : /* Ignore permutations which do not move anything cross-lane. */
25665 154949 : if (perm < 16)
25666 : {
25667 : /* The second shuffle for e.g. V4DFmode has
25668 : 0123 and ABCD operands.
25669 : Ignore AB23, as 23 is already in the second lane
25670 : of the first operand. */
25671 77729 : if ((perm & 0xc) == (1 << 2)) continue;
25672 : /* And 01CD, as 01 is in the first lane of the first
25673 : operand. */
25674 58281 : if ((perm & 3) == 0) continue;
25675 : /* And 4567, as then the vperm2[fi]128 doesn't change
25676 : anything on the original 4567 second operand. */
25677 43693 : if ((perm & 0xf) == ((3 << 2) | 2)) continue;
25678 : }
25679 : else
25680 : {
25681 : /* The second shuffle for e.g. V4DFmode has
25682 : 4567 and ABCD operands.
25683 : Ignore AB67, as 67 is already in the second lane
25684 : of the first operand. */
25685 77220 : if ((perm & 0xc) == (3 << 2)) continue;
25686 : /* And 45CD, as 45 is in the first lane of the first
25687 : operand. */
25688 58028 : if ((perm & 3) == 2) continue;
25689 : /* And 0123, as then the vperm2[fi]128 doesn't change
25690 : anything on the original 0123 first operand. */
25691 43544 : if ((perm & 0xf) == (1 << 2)) continue;
25692 : }
25693 :
25694 203615 : for (i = 0; i < nelt; i++)
25695 : {
25696 202296 : j = d->perm[i] / nelt2;
25697 374984 : if (j == ((perm >> (2 * (i >= nelt2))) & 3))
25698 48950 : dsecond.perm[i] = nelt + (i & nelt2) + (d->perm[i] & (nelt2 - 1));
25699 257136 : else if (j == (unsigned) (i >= nelt2) + 2 * (perm >= 16))
25700 77116 : dsecond.perm[i] = d->perm[i] & (nelt - 1);
25701 : else
25702 : break;
25703 : }
25704 :
25705 77549 : if (i == nelt)
25706 : {
25707 1319 : start_sequence ();
25708 1319 : ok = expand_vec_perm_1 (&dsecond);
25709 1319 : end_sequence ();
25710 : }
25711 : else
25712 : ok = false;
25713 :
25714 1319 : if (ok)
25715 : {
25716 68 : if (d->testing_p)
25717 : return true;
25718 :
25719 : /* Found a usable second shuffle. dfirst will be
25720 : vperm2f128 on d->op0 and d->op1. */
25721 46 : dsecond.testing_p = false;
25722 46 : dfirst = *d;
25723 46 : dfirst.target = gen_reg_rtx (d->vmode);
25724 270 : for (i = 0; i < nelt; i++)
25725 448 : dfirst.perm[i] = (i & (nelt2 - 1))
25726 336 : + ((perm >> (2 * (i >= nelt2))) & 3) * nelt2;
25727 :
25728 46 : canonicalize_perm (&dfirst);
25729 46 : ok = expand_vec_perm_1 (&dfirst);
25730 46 : gcc_assert (ok);
25731 :
25732 : /* And dsecond is some single insn shuffle, taking
25733 : d->op0 and result of vperm2f128 (if perm < 16) or
25734 : d->op1 and result of vperm2f128 (otherwise). */
25735 46 : if (perm >= 16)
25736 46 : dsecond.op0 = dsecond.op1;
25737 46 : dsecond.op1 = dfirst.target;
25738 :
25739 46 : ok = expand_vec_perm_1 (&dsecond);
25740 46 : gcc_assert (ok);
25741 :
25742 : return true;
25743 : }
25744 :
25745 : /* For one operand, the only useful vperm2f128 permutation is 0x01
25746 : aka lanes swap. */
25747 77481 : if (d->one_operand_p)
25748 : return false;
25749 : }
25750 :
25751 : return false;
25752 : }
25753 :
25754 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
25755 : a two vector permutation using 2 intra-lane interleave insns
25756 : and cross-lane shuffle for 32-byte vectors. */
25757 :
25758 : static bool
25759 34511 : expand_vec_perm_interleave3 (struct expand_vec_perm_d *d)
25760 : {
25761 34511 : unsigned i, nelt;
25762 34511 : rtx (*gen) (rtx, rtx, rtx);
25763 :
25764 34511 : if (d->one_operand_p)
25765 : return false;
25766 31909 : if (TARGET_AVX2 && GET_MODE_SIZE (d->vmode) == 32)
25767 : ;
25768 25401 : else if (TARGET_AVX && (d->vmode == V8SFmode || d->vmode == V4DFmode))
25769 : ;
25770 : else
25771 : return false;
25772 :
25773 7869 : nelt = d->nelt;
25774 7869 : if (d->perm[0] != 0 && d->perm[0] != nelt / 2)
25775 : return false;
25776 7726 : for (i = 0; i < nelt; i += 2)
25777 7354 : if (d->perm[i] != d->perm[0] + i / 2
25778 6359 : || d->perm[i + 1] != d->perm[0] + i / 2 + nelt)
25779 : return false;
25780 :
25781 372 : if (d->testing_p)
25782 : return true;
25783 :
25784 56 : switch (d->vmode)
25785 : {
25786 32 : case E_V32QImode:
25787 32 : if (d->perm[0])
25788 : gen = gen_vec_interleave_highv32qi;
25789 : else
25790 16 : gen = gen_vec_interleave_lowv32qi;
25791 : break;
25792 18 : case E_V16HImode:
25793 18 : if (d->perm[0])
25794 : gen = gen_vec_interleave_highv16hi;
25795 : else
25796 9 : gen = gen_vec_interleave_lowv16hi;
25797 : break;
25798 0 : case E_V8SImode:
25799 0 : if (d->perm[0])
25800 : gen = gen_vec_interleave_highv8si;
25801 : else
25802 0 : gen = gen_vec_interleave_lowv8si;
25803 : break;
25804 4 : case E_V4DImode:
25805 4 : if (d->perm[0])
25806 : gen = gen_vec_interleave_highv4di;
25807 : else
25808 2 : gen = gen_vec_interleave_lowv4di;
25809 : break;
25810 2 : case E_V8SFmode:
25811 2 : if (d->perm[0])
25812 : gen = gen_vec_interleave_highv8sf;
25813 : else
25814 1 : gen = gen_vec_interleave_lowv8sf;
25815 : break;
25816 0 : case E_V4DFmode:
25817 0 : if (d->perm[0])
25818 : gen = gen_vec_interleave_highv4df;
25819 : else
25820 0 : gen = gen_vec_interleave_lowv4df;
25821 : break;
25822 0 : default:
25823 0 : gcc_unreachable ();
25824 : }
25825 :
25826 56 : emit_insn (gen (d->target, d->op0, d->op1));
25827 56 : return true;
25828 : }
25829 :
25830 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement
25831 : a single vector permutation using a single intra-lane vector
25832 : permutation, vperm2f128 swapping the lanes and vblend* insn blending
25833 : the non-swapped and swapped vectors together. */
25834 :
25835 : static bool
25836 28378 : expand_vec_perm_vperm2f128_vblend (struct expand_vec_perm_d *d)
25837 : {
25838 28378 : struct expand_vec_perm_d dfirst, dsecond;
25839 28378 : unsigned i, j, msk, nelt = d->nelt, nelt2 = nelt / 2;
25840 28378 : rtx_insn *seq;
25841 28378 : bool ok;
25842 28378 : rtx (*blend) (rtx, rtx, rtx, rtx) = NULL;
25843 :
25844 28378 : if (!TARGET_AVX
25845 2765 : || TARGET_AVX2
25846 1686 : || (d->vmode != V8SFmode && d->vmode != V4DFmode)
25847 1502 : || !d->one_operand_p)
25848 : return false;
25849 :
25850 0 : dfirst = *d;
25851 0 : for (i = 0; i < nelt; i++)
25852 0 : dfirst.perm[i] = 0xff;
25853 0 : for (i = 0, msk = 0; i < nelt; i++)
25854 : {
25855 0 : j = (d->perm[i] & nelt2) ? i | nelt2 : i & ~nelt2;
25856 0 : if (dfirst.perm[j] != 0xff && dfirst.perm[j] != d->perm[i])
25857 : return false;
25858 0 : dfirst.perm[j] = d->perm[i];
25859 0 : if (j != i)
25860 0 : msk |= (1 << i);
25861 : }
25862 0 : for (i = 0; i < nelt; i++)
25863 0 : if (dfirst.perm[i] == 0xff)
25864 0 : dfirst.perm[i] = i;
25865 :
25866 0 : if (!d->testing_p)
25867 0 : dfirst.target = gen_reg_rtx (dfirst.vmode);
25868 :
25869 0 : start_sequence ();
25870 0 : ok = expand_vec_perm_1 (&dfirst);
25871 0 : seq = end_sequence ();
25872 :
25873 0 : if (!ok)
25874 : return false;
25875 :
25876 0 : if (d->testing_p)
25877 : return true;
25878 :
25879 0 : emit_insn (seq);
25880 :
25881 0 : dsecond = *d;
25882 0 : dsecond.op0 = dfirst.target;
25883 0 : dsecond.op1 = dfirst.target;
25884 0 : dsecond.one_operand_p = true;
25885 0 : dsecond.target = gen_reg_rtx (dsecond.vmode);
25886 0 : for (i = 0; i < nelt; i++)
25887 0 : dsecond.perm[i] = i ^ nelt2;
25888 :
25889 0 : ok = expand_vec_perm_1 (&dsecond);
25890 0 : gcc_assert (ok);
25891 :
25892 0 : blend = d->vmode == V8SFmode ? gen_avx_blendps256 : gen_avx_blendpd256;
25893 0 : emit_insn (blend (d->target, dfirst.target, dsecond.target, GEN_INT (msk)));
25894 0 : return true;
25895 : }
25896 :
25897 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement
25898 : a two vector permutation using two single vector permutations and
25899 : {,v}{,p}unpckl{ps,pd,bw,wd,dq}. If two_insn, succeed only if one
25900 : of dfirst or dsecond is identity permutation. */
25901 :
25902 : static bool
25903 119298 : expand_vec_perm_2perm_interleave (struct expand_vec_perm_d *d, bool two_insn)
25904 : {
25905 119298 : unsigned i, nelt = d->nelt, nelt2 = nelt / 2, lane = nelt;
25906 119298 : struct expand_vec_perm_d dfirst, dsecond, dfinal;
25907 119298 : bool ident1 = true, ident2 = true;
25908 :
25909 119298 : if (d->one_operand_p)
25910 : return false;
25911 :
25912 216102 : if (GET_MODE_SIZE (d->vmode) == 16)
25913 : {
25914 65490 : if (!TARGET_SSE)
25915 : return false;
25916 65490 : if (d->vmode != V4SFmode && d->vmode != V2DFmode && !TARGET_SSE2)
25917 : return false;
25918 : }
25919 85122 : else if (GET_MODE_SIZE (d->vmode) == 32)
25920 : {
25921 6593 : if (!TARGET_AVX)
25922 : return false;
25923 6593 : if (d->vmode != V8SFmode && d->vmode != V4DFmode && !TARGET_AVX2)
25924 : return false;
25925 : lane = nelt2;
25926 : }
25927 : else
25928 : return false;
25929 :
25930 233792 : for (i = 1; i < nelt; i++)
25931 199953 : if ((d->perm[i] >= nelt) != ((d->perm[0] >= nelt) ^ (i & 1)))
25932 : return false;
25933 :
25934 33839 : dfirst = *d;
25935 33839 : dsecond = *d;
25936 33839 : dfinal = *d;
25937 33839 : dfirst.op1 = dfirst.op0;
25938 33839 : dfirst.one_operand_p = true;
25939 33839 : dsecond.op0 = dsecond.op1;
25940 33839 : dsecond.one_operand_p = true;
25941 :
25942 219787 : for (i = 0; i < nelt; i++)
25943 185948 : if (d->perm[i] >= nelt)
25944 : {
25945 92974 : dsecond.perm[i / 2 + (i >= lane ? lane / 2 : 0)] = d->perm[i] - nelt;
25946 92974 : if (d->perm[i] - nelt != i / 2 + (i >= lane ? lane / 2 : 0))
25947 85074 : ident2 = false;
25948 92974 : dsecond.perm[i / 2 + (i >= lane ? lane : lane / 2)]
25949 92974 : = d->perm[i] - nelt;
25950 : }
25951 : else
25952 : {
25953 92974 : dfirst.perm[i / 2 + (i >= lane ? lane / 2 : 0)] = d->perm[i];
25954 92974 : if (d->perm[i] != i / 2 + (i >= lane ? lane / 2 : 0))
25955 76972 : ident1 = false;
25956 92974 : dfirst.perm[i / 2 + (i >= lane ? lane : lane / 2)] = d->perm[i];
25957 : }
25958 :
25959 33839 : if (two_insn && !ident1 && !ident2)
25960 : return false;
25961 :
25962 3893 : if (!d->testing_p)
25963 : {
25964 214 : if (!ident1)
25965 146 : dfinal.op0 = dfirst.target = gen_reg_rtx (d->vmode);
25966 214 : if (!ident2)
25967 146 : dfinal.op1 = dsecond.target = gen_reg_rtx (d->vmode);
25968 214 : if (d->perm[0] >= nelt)
25969 0 : std::swap (dfinal.op0, dfinal.op1);
25970 : }
25971 :
25972 3893 : bool ok;
25973 3893 : rtx_insn *seq1 = NULL, *seq2 = NULL;
25974 :
25975 3893 : if (!ident1)
25976 : {
25977 2781 : start_sequence ();
25978 2781 : ok = expand_vec_perm_1 (&dfirst);
25979 2781 : seq1 = end_sequence ();
25980 :
25981 2781 : if (!ok)
25982 : return false;
25983 : }
25984 :
25985 2212 : if (!ident2)
25986 : {
25987 1922 : start_sequence ();
25988 1922 : ok = expand_vec_perm_1 (&dsecond);
25989 1922 : seq2 = end_sequence ();
25990 :
25991 1922 : if (!ok)
25992 : return false;
25993 : }
25994 :
25995 550 : if (d->testing_p)
25996 : return true;
25997 :
25998 680 : for (i = 0; i < nelt; i++)
25999 : {
26000 544 : dfinal.perm[i] = i / 2;
26001 544 : if (i >= lane)
26002 4 : dfinal.perm[i] += lane / 2;
26003 544 : if ((i & 1) != 0)
26004 272 : dfinal.perm[i] += nelt;
26005 : }
26006 136 : emit_insn (seq1);
26007 136 : emit_insn (seq2);
26008 136 : ok = expand_vselect_vconcat (dfinal.target, dfinal.op0, dfinal.op1,
26009 : dfinal.perm, dfinal.nelt, false);
26010 136 : gcc_assert (ok);
26011 : return true;
26012 : }
26013 :
26014 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
26015 : the permutation using two single vector permutations and the SSE4_1 pblendv
26016 : instruction. If two_insn, succeed only if one of dfirst or dsecond is
26017 : identity permutation. */
26018 :
26019 : static bool
26020 118748 : expand_vec_perm_2perm_pblendv (struct expand_vec_perm_d *d, bool two_insn)
26021 : {
26022 118748 : unsigned i, nelt = d->nelt;
26023 118748 : struct expand_vec_perm_d dfirst, dsecond, dfinal;
26024 118748 : machine_mode vmode = d->vmode;
26025 118748 : bool ident1 = true, ident2 = true;
26026 :
26027 : /* Use the same checks as in expand_vec_perm_blend. */
26028 118748 : if (d->one_operand_p)
26029 : return false;
26030 111051 : if (TARGET_AVX2 && GET_MODE_SIZE (vmode) == 32)
26031 : ;
26032 105787 : else if (TARGET_AVX && (vmode == V4DFmode || vmode == V8SFmode))
26033 : ;
26034 101474 : else if (TARGET_SSE4_1
26035 110732 : && (GET_MODE_SIZE (vmode) == 16
26036 8092 : || (TARGET_MMX_WITH_SSE && GET_MODE_SIZE (vmode) == 8)
26037 2327 : || GET_MODE_SIZE (vmode) == 4))
26038 : ;
26039 : else
26040 : return false;
26041 :
26042 14613 : dfirst = *d;
26043 14613 : dsecond = *d;
26044 14613 : dfinal = *d;
26045 14613 : dfirst.op1 = dfirst.op0;
26046 14613 : dfirst.one_operand_p = true;
26047 14613 : dsecond.op0 = dsecond.op1;
26048 14613 : dsecond.one_operand_p = true;
26049 :
26050 109015 : for (i = 0; i < nelt; ++i)
26051 94402 : if (d->perm[i] >= nelt)
26052 : {
26053 47494 : dfirst.perm[i] = 0xff;
26054 47494 : dsecond.perm[i] = d->perm[i] - nelt;
26055 47494 : if (d->perm[i] != i + nelt)
26056 94402 : ident2 = false;
26057 : }
26058 : else
26059 : {
26060 46908 : dsecond.perm[i] = 0xff;
26061 46908 : dfirst.perm[i] = d->perm[i];
26062 46908 : if (d->perm[i] != i)
26063 94402 : ident1 = false;
26064 : }
26065 :
26066 14613 : if (two_insn && !ident1 && !ident2)
26067 : return false;
26068 :
26069 : /* For now. Ideally treat 0xff as a wildcard. */
26070 38146 : for (i = 0; i < nelt; ++i)
26071 33596 : if (dfirst.perm[i] == 0xff)
26072 : {
26073 17971 : if (GET_MODE_SIZE (vmode) == 32
26074 17971 : && dfirst.perm[i ^ (nelt / 2)] != 0xff)
26075 10707 : dfirst.perm[i] = dfirst.perm[i ^ (nelt / 2)] ^ (nelt / 2);
26076 : else
26077 7264 : dfirst.perm[i] = i;
26078 : }
26079 : else
26080 : {
26081 15625 : if (GET_MODE_SIZE (vmode) == 32
26082 15625 : && dsecond.perm[i ^ (nelt / 2)] != 0xff)
26083 9058 : dsecond.perm[i] = dsecond.perm[i ^ (nelt / 2)] ^ (nelt / 2);
26084 : else
26085 6567 : dsecond.perm[i] = i;
26086 : }
26087 :
26088 4550 : if (!d->testing_p)
26089 : {
26090 2144 : if (!ident1)
26091 2020 : dfinal.op0 = dfirst.target = gen_reg_rtx (d->vmode);
26092 2144 : if (!ident2)
26093 823 : dfinal.op1 = dsecond.target = gen_reg_rtx (d->vmode);
26094 : }
26095 :
26096 4550 : bool ok;
26097 4550 : rtx_insn *seq1 = NULL, *seq2 = NULL;
26098 :
26099 4550 : if (!ident1)
26100 : {
26101 3865 : start_sequence ();
26102 3865 : ok = expand_vec_perm_1 (&dfirst);
26103 3865 : seq1 = end_sequence ();
26104 :
26105 3865 : if (!ok)
26106 : return false;
26107 : }
26108 :
26109 3435 : if (!ident2)
26110 : {
26111 1293 : start_sequence ();
26112 1293 : ok = expand_vec_perm_1 (&dsecond);
26113 1293 : seq2 = end_sequence ();
26114 :
26115 1293 : if (!ok)
26116 : return false;
26117 : }
26118 :
26119 2702 : if (d->testing_p)
26120 : return true;
26121 :
26122 13794 : for (i = 0; i < nelt; ++i)
26123 11992 : dfinal.perm[i] = (d->perm[i] >= nelt ? i + nelt : i);
26124 :
26125 1802 : emit_insn (seq1);
26126 1802 : emit_insn (seq2);
26127 1802 : ok = expand_vec_perm_blend (&dfinal);
26128 1802 : gcc_assert (ok);
26129 : return true;
26130 : }
26131 :
26132 : /* A subroutine of ix86_expand_vec_perm_const_1.
26133 : Implement a permutation with psrlw, psllw and por.
26134 : It handles case:
26135 : __builtin_shufflevector (v,v,1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14);
26136 : __builtin_shufflevector (v,v,1,0,3,2,5,4,7,6); */
26137 :
26138 : static bool
26139 27626 : expand_vec_perm_psrlw_psllw_por (struct expand_vec_perm_d *d)
26140 : {
26141 27626 : unsigned i;
26142 27626 : rtx (*gen_shr) (rtx, rtx, rtx);
26143 27626 : rtx (*gen_shl) (rtx, rtx, rtx);
26144 27626 : rtx (*gen_or) (rtx, rtx, rtx);
26145 27626 : machine_mode mode = VOIDmode;
26146 :
26147 27626 : if (!TARGET_SSE2 || !d->one_operand_p)
26148 : return false;
26149 :
26150 5602 : switch (d->vmode)
26151 : {
26152 1494 : case E_V8QImode:
26153 1494 : if (!TARGET_MMX_WITH_SSE)
26154 : return false;
26155 : mode = V4HImode;
26156 : gen_shr = gen_lshrv4hi3;
26157 : gen_shl = gen_ashlv4hi3;
26158 : gen_or = gen_iorv4hi3;
26159 : break;
26160 : case E_V16QImode:
26161 : mode = V8HImode;
26162 : gen_shr = gen_lshrv8hi3;
26163 : gen_shl = gen_ashlv8hi3;
26164 : gen_or = gen_iorv8hi3;
26165 : break;
26166 : default: return false;
26167 : }
26168 :
26169 3318 : if (!rtx_equal_p (d->op0, d->op1))
26170 : return false;
26171 :
26172 12486 : for (i = 0; i < d->nelt; i += 2)
26173 11032 : if (d->perm[i] != i + 1 || d->perm[i + 1] != i)
26174 : return false;
26175 :
26176 1454 : if (d->testing_p)
26177 : return true;
26178 :
26179 30 : rtx tmp1 = gen_reg_rtx (mode);
26180 30 : rtx tmp2 = gen_reg_rtx (mode);
26181 30 : rtx op0 = force_reg (d->vmode, d->op0);
26182 :
26183 30 : emit_move_insn (tmp1, lowpart_subreg (mode, op0, d->vmode));
26184 30 : emit_move_insn (tmp2, lowpart_subreg (mode, op0, d->vmode));
26185 30 : emit_insn (gen_shr (tmp1, tmp1, GEN_INT (8)));
26186 30 : emit_insn (gen_shl (tmp2, tmp2, GEN_INT (8)));
26187 30 : emit_insn (gen_or (tmp1, tmp1, tmp2));
26188 30 : emit_move_insn (d->target, lowpart_subreg (d->vmode, tmp1, mode));
26189 :
26190 30 : return true;
26191 : }
26192 :
26193 : /* A subroutine of ix86_expand_vec_perm_const_1. Implement a V4DF
26194 : permutation using two vperm2f128, followed by a vshufpd insn blending
26195 : the two vectors together. */
26196 :
26197 : static bool
26198 31402 : expand_vec_perm_2vperm2f128_vshuf (struct expand_vec_perm_d *d)
26199 : {
26200 31402 : struct expand_vec_perm_d dfirst, dsecond, dthird;
26201 31402 : bool ok;
26202 :
26203 31402 : if (!TARGET_AVX || (d->vmode != V4DFmode))
26204 : return false;
26205 :
26206 1149 : if (d->testing_p)
26207 : return true;
26208 :
26209 206 : dfirst = *d;
26210 206 : dsecond = *d;
26211 206 : dthird = *d;
26212 :
26213 206 : dfirst.perm[0] = (d->perm[0] & ~1);
26214 206 : dfirst.perm[1] = (d->perm[0] & ~1) + 1;
26215 206 : dfirst.perm[2] = (d->perm[2] & ~1);
26216 206 : dfirst.perm[3] = (d->perm[2] & ~1) + 1;
26217 206 : dsecond.perm[0] = (d->perm[1] & ~1);
26218 206 : dsecond.perm[1] = (d->perm[1] & ~1) + 1;
26219 206 : dsecond.perm[2] = (d->perm[3] & ~1);
26220 206 : dsecond.perm[3] = (d->perm[3] & ~1) + 1;
26221 206 : dthird.perm[0] = (d->perm[0] % 2);
26222 206 : dthird.perm[1] = (d->perm[1] % 2) + 4;
26223 206 : dthird.perm[2] = (d->perm[2] % 2) + 2;
26224 206 : dthird.perm[3] = (d->perm[3] % 2) + 6;
26225 :
26226 206 : dfirst.target = gen_reg_rtx (dfirst.vmode);
26227 206 : dsecond.target = gen_reg_rtx (dsecond.vmode);
26228 206 : dthird.op0 = dfirst.target;
26229 206 : dthird.op1 = dsecond.target;
26230 206 : dthird.one_operand_p = false;
26231 :
26232 206 : canonicalize_perm (&dfirst);
26233 206 : canonicalize_perm (&dsecond);
26234 :
26235 206 : ok = expand_vec_perm_1 (&dfirst)
26236 206 : && expand_vec_perm_1 (&dsecond)
26237 412 : && expand_vec_perm_1 (&dthird);
26238 :
26239 0 : gcc_assert (ok);
26240 :
26241 : return true;
26242 : }
26243 :
26244 : static bool ix86_expand_vec_perm_const_1 (struct expand_vec_perm_d *);
26245 :
26246 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement
26247 : a two vector permutation using two intra-lane vector
26248 : permutations, vperm2f128 swapping the lanes and vblend* insn blending
26249 : the non-swapped and swapped vectors together. */
26250 :
26251 : static bool
26252 16727 : expand_vec_perm2_vperm2f128_vblend (struct expand_vec_perm_d *d)
26253 : {
26254 16727 : struct expand_vec_perm_d dfirst, dsecond, dthird;
26255 16727 : unsigned i, j, msk, nelt = d->nelt, nelt2 = nelt / 2, which1 = 0, which2 = 0;
26256 16727 : rtx_insn *seq1, *seq2;
26257 16727 : bool ok;
26258 16727 : rtx (*blend) (rtx, rtx, rtx, rtx) = NULL;
26259 :
26260 16727 : if (!TARGET_AVX
26261 794 : || TARGET_AVX2
26262 530 : || (d->vmode != V8SFmode && d->vmode != V4DFmode)
26263 403 : || d->one_operand_p)
26264 : return false;
26265 :
26266 403 : dfirst = *d;
26267 403 : dsecond = *d;
26268 3627 : for (i = 0; i < nelt; i++)
26269 : {
26270 3224 : dfirst.perm[i] = 0xff;
26271 3224 : dsecond.perm[i] = 0xff;
26272 : }
26273 3627 : for (i = 0, msk = 0; i < nelt; i++)
26274 : {
26275 3224 : j = (d->perm[i] & nelt2) ? i | nelt2 : i & ~nelt2;
26276 3224 : if (j == i)
26277 : {
26278 2498 : dfirst.perm[j] = d->perm[i];
26279 4130 : which1 |= (d->perm[i] < nelt ? 1 : 2);
26280 : }
26281 : else
26282 : {
26283 726 : dsecond.perm[j] = d->perm[i];
26284 726 : which2 |= (d->perm[i] < nelt ? 1 : 2);
26285 726 : msk |= (1U << i);
26286 : }
26287 : }
26288 403 : if (msk == 0 || msk == (1U << nelt) - 1)
26289 : return false;
26290 :
26291 403 : if (!d->testing_p)
26292 : {
26293 40 : dfirst.target = gen_reg_rtx (dfirst.vmode);
26294 40 : dsecond.target = gen_reg_rtx (dsecond.vmode);
26295 : }
26296 :
26297 3627 : for (i = 0; i < nelt; i++)
26298 : {
26299 3224 : if (dfirst.perm[i] == 0xff)
26300 726 : dfirst.perm[i] = (which1 == 2 ? i + nelt : i);
26301 3224 : if (dsecond.perm[i] == 0xff)
26302 2498 : dsecond.perm[i] = (which2 == 2 ? i + nelt : i);
26303 : }
26304 403 : canonicalize_perm (&dfirst);
26305 403 : start_sequence ();
26306 403 : ok = ix86_expand_vec_perm_const_1 (&dfirst);
26307 403 : seq1 = end_sequence ();
26308 :
26309 403 : if (!ok)
26310 : return false;
26311 :
26312 403 : canonicalize_perm (&dsecond);
26313 403 : start_sequence ();
26314 403 : ok = ix86_expand_vec_perm_const_1 (&dsecond);
26315 403 : seq2 = end_sequence ();
26316 :
26317 403 : if (!ok)
26318 : return false;
26319 :
26320 403 : if (d->testing_p)
26321 : return true;
26322 :
26323 40 : emit_insn (seq1);
26324 40 : emit_insn (seq2);
26325 :
26326 40 : dthird = *d;
26327 40 : dthird.op0 = dsecond.target;
26328 40 : dthird.op1 = dsecond.target;
26329 40 : dthird.one_operand_p = true;
26330 40 : dthird.target = gen_reg_rtx (dthird.vmode);
26331 360 : for (i = 0; i < nelt; i++)
26332 320 : dthird.perm[i] = i ^ nelt2;
26333 :
26334 40 : ok = expand_vec_perm_1 (&dthird);
26335 40 : gcc_assert (ok);
26336 :
26337 40 : blend = d->vmode == V8SFmode ? gen_avx_blendps256 : gen_avx_blendpd256;
26338 40 : emit_insn (blend (d->target, dfirst.target, dthird.target, GEN_INT (msk)));
26339 40 : return true;
26340 : }
26341 :
26342 : /* A subroutine of expand_vec_perm_even_odd_1. Implement the double-word
26343 : permutation with two pshufb insns and an ior. We should have already
26344 : failed all two instruction sequences. */
26345 :
26346 : static bool
26347 30274 : expand_vec_perm_pshufb2 (struct expand_vec_perm_d *d)
26348 : {
26349 30274 : rtx rperm[2][16], vperm, l, h, op, m128;
26350 30274 : unsigned int i, nelt, eltsz;
26351 30274 : machine_mode mode;
26352 30274 : rtx (*gen) (rtx, rtx, rtx);
26353 :
26354 34622 : if (!TARGET_SSSE3 || (GET_MODE_SIZE (d->vmode) != 16
26355 8606 : && GET_MODE_SIZE (d->vmode) != 8
26356 8566 : && GET_MODE_SIZE (d->vmode) != 4))
26357 : return false;
26358 1437 : gcc_assert (!d->one_operand_p);
26359 :
26360 1437 : if (d->testing_p)
26361 : return true;
26362 :
26363 202 : switch (GET_MODE_SIZE (d->vmode))
26364 : {
26365 : case 4:
26366 : mode = V4QImode;
26367 : gen = gen_mmx_pshufbv4qi3;
26368 : break;
26369 20 : case 8:
26370 20 : mode = V8QImode;
26371 20 : gen = gen_mmx_pshufbv8qi3;
26372 20 : break;
26373 45 : case 16:
26374 45 : mode = V16QImode;
26375 45 : gen = gen_ssse3_pshufbv16qi3;
26376 45 : break;
26377 0 : default:
26378 0 : gcc_unreachable ();
26379 : }
26380 :
26381 101 : nelt = d->nelt;
26382 101 : eltsz = GET_MODE_UNIT_SIZE (d->vmode);
26383 :
26384 : /* Generate two permutation masks. If the required element is within
26385 : the given vector it is shuffled into the proper lane. If the required
26386 : element is in the other vector, force a zero into the lane by setting
26387 : bit 7 in the permutation mask. */
26388 101 : m128 = GEN_INT (-128);
26389 1130 : for (i = 0; i < nelt; ++i)
26390 : {
26391 928 : unsigned j, k, e = d->perm[i];
26392 928 : unsigned which = (e >= nelt);
26393 928 : if (e >= nelt)
26394 480 : e -= nelt;
26395 :
26396 1952 : for (j = 0; j < eltsz; ++j)
26397 : {
26398 1024 : rperm[which][i*eltsz + j] = GEN_INT (e*eltsz + j);
26399 1024 : rperm[1-which][i*eltsz + j] = m128;
26400 : }
26401 :
26402 9024 : for (k = i*eltsz + j; k < 16; ++k)
26403 8096 : rperm[0][k] = rperm[1][k] = m128;
26404 : }
26405 :
26406 101 : vperm = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, rperm[0]));
26407 101 : vperm = force_reg (V16QImode, vperm);
26408 :
26409 101 : l = gen_reg_rtx (mode);
26410 101 : op = gen_lowpart (mode, d->op0);
26411 101 : emit_insn (gen (l, op, vperm));
26412 :
26413 101 : vperm = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, rperm[1]));
26414 101 : vperm = force_reg (V16QImode, vperm);
26415 :
26416 101 : h = gen_reg_rtx (mode);
26417 101 : op = gen_lowpart (mode, d->op1);
26418 101 : emit_insn (gen (h, op, vperm));
26419 :
26420 101 : op = d->target;
26421 101 : if (d->vmode != mode)
26422 22 : op = gen_reg_rtx (mode);
26423 101 : ix86_emit_vec_binop (IOR, mode, op, l, h);
26424 101 : if (op != d->target)
26425 22 : emit_move_insn (d->target, gen_lowpart (d->vmode, op));
26426 :
26427 : return true;
26428 : }
26429 :
26430 : /* Implement arbitrary permutation of one V32QImode and V16QImode operand
26431 : with two vpshufb insns, vpermq and vpor. We should have already failed
26432 : all two or three instruction sequences. */
26433 :
26434 : static bool
26435 25063 : expand_vec_perm_vpshufb2_vpermq (struct expand_vec_perm_d *d)
26436 : {
26437 25063 : rtx rperm[2][32], vperm, l, h, hp, op, m128;
26438 25063 : unsigned int i, nelt, eltsz;
26439 :
26440 25063 : if (!TARGET_AVX2
26441 374 : || !d->one_operand_p
26442 172 : || (d->vmode != V32QImode && d->vmode != V16HImode))
26443 : return false;
26444 :
26445 7 : if (d->testing_p)
26446 : return true;
26447 :
26448 7 : nelt = d->nelt;
26449 7 : eltsz = GET_MODE_UNIT_SIZE (d->vmode);
26450 :
26451 : /* Generate two permutation masks. If the required element is within
26452 : the same lane, it is shuffled in. If the required element from the
26453 : other lane, force a zero by setting bit 7 in the permutation mask.
26454 : In the other mask the mask has non-negative elements if element
26455 : is requested from the other lane, but also moved to the other lane,
26456 : so that the result of vpshufb can have the two V2TImode halves
26457 : swapped. */
26458 7 : m128 = GEN_INT (-128);
26459 206 : for (i = 0; i < nelt; ++i)
26460 : {
26461 192 : unsigned j, e = d->perm[i] & (nelt / 2 - 1);
26462 192 : unsigned which = ((d->perm[i] ^ i) & (nelt / 2)) * eltsz;
26463 :
26464 416 : for (j = 0; j < eltsz; ++j)
26465 : {
26466 224 : rperm[!!which][(i * eltsz + j) ^ which] = GEN_INT (e * eltsz + j);
26467 224 : rperm[!which][(i * eltsz + j) ^ (which ^ 16)] = m128;
26468 : }
26469 : }
26470 :
26471 7 : vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[1]));
26472 7 : vperm = force_reg (V32QImode, vperm);
26473 :
26474 7 : h = gen_reg_rtx (V32QImode);
26475 7 : op = gen_lowpart (V32QImode, d->op0);
26476 7 : emit_insn (gen_avx2_pshufbv32qi3 (h, op, vperm));
26477 :
26478 : /* Swap the 128-byte lanes of h into hp. */
26479 7 : hp = gen_reg_rtx (V4DImode);
26480 7 : op = gen_lowpart (V4DImode, h);
26481 7 : emit_insn (gen_avx2_permv4di_1 (hp, op, const2_rtx, GEN_INT (3), const0_rtx,
26482 : const1_rtx));
26483 :
26484 7 : vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[0]));
26485 7 : vperm = force_reg (V32QImode, vperm);
26486 :
26487 7 : l = gen_reg_rtx (V32QImode);
26488 7 : op = gen_lowpart (V32QImode, d->op0);
26489 7 : emit_insn (gen_avx2_pshufbv32qi3 (l, op, vperm));
26490 :
26491 7 : op = d->target;
26492 7 : if (d->vmode != V32QImode)
26493 2 : op = gen_reg_rtx (V32QImode);
26494 7 : emit_insn (gen_iorv32qi3 (op, l, gen_lowpart (V32QImode, hp)));
26495 7 : if (op != d->target)
26496 2 : emit_move_insn (d->target, gen_lowpart (d->vmode, op));
26497 :
26498 : return true;
26499 : }
26500 :
26501 : /* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
26502 : and extract-odd permutations of two V32QImode and V16QImode operand
26503 : with two vpshufb insns, vpor and vpermq. We should have already
26504 : failed all two or three instruction sequences. */
26505 :
26506 : static bool
26507 25056 : expand_vec_perm_vpshufb2_vpermq_even_odd (struct expand_vec_perm_d *d)
26508 : {
26509 25056 : rtx rperm[2][32], vperm, l, h, ior, op, m128;
26510 25056 : unsigned int i, nelt, eltsz;
26511 :
26512 25056 : if (!TARGET_AVX2
26513 367 : || d->one_operand_p
26514 202 : || (d->vmode != V32QImode && d->vmode != V16HImode))
26515 : return false;
26516 :
26517 112 : for (i = 0; i < d->nelt; ++i)
26518 112 : if ((d->perm[i] ^ (i * 2)) & (3 * d->nelt / 2))
26519 : return false;
26520 :
26521 0 : if (d->testing_p)
26522 : return true;
26523 :
26524 0 : nelt = d->nelt;
26525 0 : eltsz = GET_MODE_UNIT_SIZE (d->vmode);
26526 :
26527 : /* Generate two permutation masks. In the first permutation mask
26528 : the first quarter will contain indexes for the first half
26529 : of the op0, the second quarter will contain bit 7 set, third quarter
26530 : will contain indexes for the second half of the op0 and the
26531 : last quarter bit 7 set. In the second permutation mask
26532 : the first quarter will contain bit 7 set, the second quarter
26533 : indexes for the first half of the op1, the third quarter bit 7 set
26534 : and last quarter indexes for the second half of the op1.
26535 : I.e. the first mask e.g. for V32QImode extract even will be:
26536 : 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128
26537 : (all values masked with 0xf except for -128) and second mask
26538 : for extract even will be
26539 : -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe. */
26540 0 : m128 = GEN_INT (-128);
26541 0 : for (i = 0; i < nelt; ++i)
26542 : {
26543 0 : unsigned j, e = d->perm[i] & (nelt / 2 - 1);
26544 0 : unsigned which = d->perm[i] >= nelt;
26545 0 : unsigned xorv = (i >= nelt / 4 && i < 3 * nelt / 4) ? 24 : 0;
26546 :
26547 0 : for (j = 0; j < eltsz; ++j)
26548 : {
26549 0 : rperm[which][(i * eltsz + j) ^ xorv] = GEN_INT (e * eltsz + j);
26550 0 : rperm[1 - which][(i * eltsz + j) ^ xorv] = m128;
26551 : }
26552 : }
26553 :
26554 0 : vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[0]));
26555 0 : vperm = force_reg (V32QImode, vperm);
26556 :
26557 0 : l = gen_reg_rtx (V32QImode);
26558 0 : op = gen_lowpart (V32QImode, d->op0);
26559 0 : emit_insn (gen_avx2_pshufbv32qi3 (l, op, vperm));
26560 :
26561 0 : vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[1]));
26562 0 : vperm = force_reg (V32QImode, vperm);
26563 :
26564 0 : h = gen_reg_rtx (V32QImode);
26565 0 : op = gen_lowpart (V32QImode, d->op1);
26566 0 : emit_insn (gen_avx2_pshufbv32qi3 (h, op, vperm));
26567 :
26568 0 : ior = gen_reg_rtx (V32QImode);
26569 0 : emit_insn (gen_iorv32qi3 (ior, l, h));
26570 :
26571 : /* Permute the V4DImode quarters using { 0, 2, 1, 3 } permutation. */
26572 0 : op = gen_reg_rtx (V4DImode);
26573 0 : ior = gen_lowpart (V4DImode, ior);
26574 0 : emit_insn (gen_avx2_permv4di_1 (op, ior, const0_rtx, const2_rtx,
26575 : const1_rtx, GEN_INT (3)));
26576 0 : emit_move_insn (d->target, gen_lowpart (d->vmode, op));
26577 :
26578 0 : return true;
26579 : }
26580 :
26581 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement a
26582 : permutation (which is a bland) with and, andnot and or when pshufb is not available.
26583 :
26584 : It handles case:
26585 : __builtin_shufflevector (v1, v2, 0, 9, 2, 11, 4, 13, 6, 15);
26586 : __builtin_shufflevector (v1, v2, 8, 1, 2, 11, 4, 13, 6, 15);
26587 :
26588 : An element[i] must be chosen between op0[i] and op1[i] to satisfy the
26589 : requirement.
26590 : */
26591 :
26592 : static bool
26593 26172 : expand_vec_perm_pand_pandn_por (struct expand_vec_perm_d *d)
26594 : {
26595 26172 : rtx rperm[16], vperm;
26596 26172 : unsigned int i, nelt = d->nelt;
26597 :
26598 26172 : if (!TARGET_SSE2
26599 26172 : || d->one_operand_p
26600 22024 : || (d->vmode != V16QImode && d->vmode != V8HImode))
26601 : return false;
26602 :
26603 8356 : if (d->perm[0] != 0)
26604 : return false;
26605 :
26606 : /* The dest[i] must select an element between op0[i] and op1[i]. */
26607 17225 : for (i = 1; i < nelt; i++)
26608 16120 : if ((d->perm[i] % nelt) != i)
26609 : return false;
26610 :
26611 1105 : if (d->testing_p)
26612 : return true;
26613 :
26614 : /* Generates a blend mask for the operators AND and ANDNOT. */
26615 112 : machine_mode inner_mode = GET_MODE_INNER (d->vmode);
26616 1216 : for (i = 0; i < nelt; i++)
26617 1667 : rperm[i] = (d->perm[i] < nelt) ? CONSTM1_RTX (inner_mode)
26618 563 : : CONST0_RTX (inner_mode);
26619 :
26620 112 : vperm = gen_rtx_CONST_VECTOR (d->vmode, gen_rtvec_v (nelt, rperm));
26621 112 : vperm = force_reg (d->vmode, vperm);
26622 :
26623 112 : ix86_expand_sse_movcc (d->target, vperm, d->op0, d->op1);
26624 :
26625 112 : return true;
26626 : }
26627 :
26628 : /* Implement permutation with pslldq + psrldq + por when pshufb is not
26629 : available. */
26630 : static bool
26631 45954 : expand_vec_perm_pslldq_psrldq_por (struct expand_vec_perm_d *d, bool pandn)
26632 : {
26633 45954 : unsigned i, nelt = d->nelt;
26634 45954 : unsigned start1, end1 = -1;
26635 45954 : machine_mode vmode = d->vmode, imode;
26636 45954 : int start2 = -1;
26637 45954 : bool clear_op0, clear_op1;
26638 45954 : unsigned inner_size;
26639 45954 : rtx op0, op1, dop1;
26640 45954 : rtx (*gen_vec_shr) (rtx, rtx, rtx);
26641 45954 : rtx (*gen_vec_shl) (rtx, rtx, rtx);
26642 :
26643 : /* pshufd can be used for V4SI/V2DI under TARGET_SSE2. */
26644 45954 : if (!TARGET_SSE2 || (vmode != E_V16QImode && vmode != E_V8HImode))
26645 : return false;
26646 :
26647 15147 : start1 = d->perm[0];
26648 47340 : for (i = 1; i < nelt; i++)
26649 : {
26650 46333 : if (d->perm[i] != d->perm[i-1] + 1
26651 17654 : || d->perm[i] == nelt)
26652 : {
26653 29287 : if (start2 == -1)
26654 : {
26655 15147 : start2 = d->perm[i];
26656 15147 : end1 = d->perm[i-1];
26657 : }
26658 : else
26659 : return false;
26660 : }
26661 : }
26662 :
26663 1007 : clear_op0 = end1 != nelt - 1;
26664 1007 : clear_op1 = start2 % nelt != 0;
26665 : /* pandn/pand is needed to clear upper/lower bits of op0/op1. */
26666 1007 : if (!pandn && (clear_op0 || clear_op1))
26667 : return false;
26668 :
26669 649 : if (d->testing_p)
26670 : return true;
26671 :
26672 89 : gen_vec_shr = vmode == E_V16QImode ? gen_vec_shr_v16qi : gen_vec_shr_v8hi;
26673 28 : gen_vec_shl = vmode == E_V16QImode ? gen_vec_shl_v16qi : gen_vec_shl_v8hi;
26674 89 : imode = GET_MODE_INNER (vmode);
26675 89 : inner_size = GET_MODE_BITSIZE (imode);
26676 89 : op0 = gen_reg_rtx (vmode);
26677 89 : op1 = gen_reg_rtx (vmode);
26678 :
26679 89 : if (start1)
26680 84 : emit_insn (gen_vec_shr (op0, d->op0, GEN_INT (start1 * inner_size)));
26681 : else
26682 5 : emit_move_insn (op0, d->op0);
26683 :
26684 89 : dop1 = d->op1;
26685 89 : if (d->one_operand_p)
26686 64 : dop1 = d->op0;
26687 :
26688 89 : int shl_offset = end1 - start1 + 1 - start2 % nelt;
26689 89 : if (shl_offset)
26690 49 : emit_insn (gen_vec_shl (op1, dop1, GEN_INT (shl_offset * inner_size)));
26691 : else
26692 40 : emit_move_insn (op1, dop1);
26693 :
26694 : /* Clear lower/upper bits for op0/op1. */
26695 89 : if (clear_op0 || clear_op1)
26696 : {
26697 : rtx vec[16];
26698 : rtx const_vec;
26699 : rtx clear;
26700 989 : for (i = 0; i != nelt; i++)
26701 : {
26702 920 : if (i < (end1 - start1 + 1))
26703 346 : vec[i] = gen_int_mode ((HOST_WIDE_INT_1U << inner_size) - 1, imode);
26704 : else
26705 574 : vec[i] = CONST0_RTX (imode);
26706 : }
26707 69 : const_vec = gen_rtx_CONST_VECTOR (vmode, gen_rtvec_v (nelt, vec));
26708 69 : const_vec = validize_mem (force_const_mem (vmode, const_vec));
26709 69 : clear = force_reg (vmode, const_vec);
26710 :
26711 69 : if (clear_op0)
26712 61 : emit_move_insn (op0, gen_rtx_AND (vmode, op0, clear));
26713 69 : if (clear_op1)
26714 56 : emit_move_insn (op1, gen_rtx_AND (vmode,
26715 : gen_rtx_NOT (vmode, clear),
26716 : op1));
26717 : }
26718 :
26719 89 : emit_move_insn (d->target, gen_rtx_IOR (vmode, op0, op1));
26720 89 : return true;
26721 : }
26722 :
26723 : /* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
26724 : and extract-odd permutations of two V8QI, V8HI, V16QI, V16HI or V32QI
26725 : operands with two "and" and "pack" or two "shift" and "pack" insns.
26726 : We should have already failed all two instruction sequences. */
26727 :
26728 : static bool
26729 47988 : expand_vec_perm_even_odd_pack (struct expand_vec_perm_d *d)
26730 : {
26731 47988 : rtx op, dop0, dop1, t;
26732 47988 : unsigned i, odd, c, s, nelt = d->nelt;
26733 47988 : int pblendw_i = 0;
26734 47988 : bool end_perm = false;
26735 47988 : machine_mode half_mode;
26736 47988 : rtx (*gen_and) (rtx, rtx, rtx);
26737 47988 : rtx (*gen_pack) (rtx, rtx, rtx);
26738 47988 : rtx (*gen_shift) (rtx, rtx, rtx);
26739 :
26740 47988 : if (d->one_operand_p)
26741 : return false;
26742 :
26743 42343 : switch (d->vmode)
26744 : {
26745 4787 : case E_V4HImode:
26746 : /* Required for "pack". */
26747 4787 : if (!TARGET_SSE4_1)
26748 : return false;
26749 : c = 0xffff;
26750 : s = 16;
26751 : half_mode = V2SImode;
26752 : gen_and = gen_andv2si3;
26753 : gen_pack = gen_mmx_packusdw;
26754 : gen_shift = gen_lshrv2si3;
26755 : pblendw_i = 0x5;
26756 : break;
26757 6215 : case E_V8HImode:
26758 : /* Required for "pack". */
26759 6215 : if (!TARGET_SSE4_1)
26760 : return false;
26761 : c = 0xffff;
26762 : s = 16;
26763 : half_mode = V4SImode;
26764 : gen_and = gen_andv4si3;
26765 : gen_pack = gen_sse4_1_packusdw;
26766 : gen_shift = gen_lshrv4si3;
26767 : pblendw_i = 0x55;
26768 : break;
26769 : case E_V8QImode:
26770 : /* No check as all instructions are SSE2. */
26771 : c = 0xff;
26772 : s = 8;
26773 : half_mode = V4HImode;
26774 : gen_and = gen_andv4hi3;
26775 : gen_pack = gen_mmx_packuswb;
26776 : gen_shift = gen_lshrv4hi3;
26777 : break;
26778 14773 : case E_V16QImode:
26779 : /* No check as all instructions are SSE2. */
26780 14773 : c = 0xff;
26781 14773 : s = 8;
26782 14773 : half_mode = V8HImode;
26783 14773 : gen_and = gen_andv8hi3;
26784 14773 : gen_pack = gen_sse2_packuswb;
26785 14773 : gen_shift = gen_lshrv8hi3;
26786 14773 : break;
26787 440 : case E_V16HImode:
26788 440 : if (!TARGET_AVX2)
26789 : return false;
26790 : c = 0xffff;
26791 : s = 16;
26792 : half_mode = V8SImode;
26793 : gen_and = gen_andv8si3;
26794 : gen_pack = gen_avx2_packusdw;
26795 : gen_shift = gen_lshrv8si3;
26796 : pblendw_i = 0x5555;
26797 : end_perm = true;
26798 : break;
26799 269 : case E_V32QImode:
26800 269 : if (!TARGET_AVX2)
26801 : return false;
26802 : c = 0xff;
26803 : s = 8;
26804 : half_mode = V16HImode;
26805 : gen_and = gen_andv16hi3;
26806 : gen_pack = gen_avx2_packuswb;
26807 : gen_shift = gen_lshrv16hi3;
26808 : end_perm = true;
26809 : break;
26810 : default:
26811 : /* Only V4HI, V8QI, V8HI, V16QI, V16HI and V32QI modes
26812 : are more profitable than general shuffles. */
26813 : return false;
26814 : }
26815 :
26816 : /* Check that permutation is even or odd. */
26817 20933 : odd = d->perm[0];
26818 20933 : if (odd > 1)
26819 : return false;
26820 :
26821 234419 : for (i = 1; i < nelt; ++i)
26822 217893 : if (d->perm[i] != 2 * i + odd)
26823 : return false;
26824 :
26825 16526 : if (d->testing_p)
26826 : return true;
26827 :
26828 5714 : dop0 = gen_reg_rtx (half_mode);
26829 5714 : dop1 = gen_reg_rtx (half_mode);
26830 5714 : if (odd == 0)
26831 : {
26832 : /* Use pblendw since const_vector 0 should be cheaper than
26833 : const_vector 0xffff. */
26834 4993 : if (d->vmode == V4HImode
26835 : || d->vmode == E_V8HImode
26836 : || d->vmode == E_V16HImode)
26837 : {
26838 876 : rtx dop0_t = gen_reg_rtx (d->vmode);
26839 876 : rtx dop1_t = gen_reg_rtx (d->vmode);
26840 876 : t = gen_reg_rtx (d->vmode);
26841 876 : emit_move_insn (t, CONST0_RTX (d->vmode));
26842 :
26843 876 : emit_move_insn (dop0_t, gen_rtx_VEC_MERGE (d->vmode, d->op0, t,
26844 : GEN_INT (pblendw_i)));
26845 876 : emit_move_insn (dop1_t, gen_rtx_VEC_MERGE (d->vmode, d->op1, t,
26846 : GEN_INT (pblendw_i)));
26847 :
26848 876 : emit_move_insn (dop0, gen_lowpart (half_mode, dop0_t));
26849 876 : emit_move_insn (dop1, gen_lowpart (half_mode, dop1_t));
26850 876 : }
26851 : else
26852 : {
26853 4117 : t = gen_const_vec_duplicate (half_mode, GEN_INT (c));
26854 4117 : t = force_reg (half_mode, t);
26855 4117 : emit_insn (gen_and (dop0, t, gen_lowpart (half_mode, d->op0)));
26856 4117 : emit_insn (gen_and (dop1, t, gen_lowpart (half_mode, d->op1)));
26857 : }
26858 : }
26859 : else
26860 : {
26861 1442 : emit_insn (gen_shift (dop0,
26862 721 : gen_lowpart (half_mode, d->op0),
26863 : GEN_INT (s)));
26864 1442 : emit_insn (gen_shift (dop1,
26865 721 : gen_lowpart (half_mode, d->op1),
26866 : GEN_INT (s)));
26867 : }
26868 : /* In AVX2 for 256 bit case we need to permute pack result. */
26869 5714 : if (TARGET_AVX2 && end_perm)
26870 : {
26871 412 : op = gen_reg_rtx (d->vmode);
26872 412 : t = gen_reg_rtx (V4DImode);
26873 412 : emit_insn (gen_pack (op, dop0, dop1));
26874 824 : emit_insn (gen_avx2_permv4di_1 (t,
26875 412 : gen_lowpart (V4DImode, op),
26876 : const0_rtx,
26877 : const2_rtx,
26878 : const1_rtx,
26879 : GEN_INT (3)));
26880 412 : emit_move_insn (d->target, gen_lowpart (d->vmode, t));
26881 : }
26882 : else
26883 5302 : emit_insn (gen_pack (d->target, dop0, dop1));
26884 :
26885 : return true;
26886 : }
26887 :
26888 : /* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
26889 : and extract-odd permutations of two V64QI operands
26890 : with two "shifts", two "truncs" and one "concat" insns for "odd"
26891 : and two "truncs" and one concat insn for "even."
26892 : Have already failed all two instruction sequences. */
26893 :
26894 : static bool
26895 25093 : expand_vec_perm_even_odd_trunc (struct expand_vec_perm_d *d)
26896 : {
26897 25093 : rtx t1, t2, t3, t4;
26898 25093 : unsigned i, odd, nelt = d->nelt;
26899 :
26900 25093 : if (!TARGET_AVX512BW
26901 67 : || d->one_operand_p
26902 31 : || d->vmode != V64QImode)
26903 : return false;
26904 :
26905 : /* Check that permutation is even or odd. */
26906 31 : odd = d->perm[0];
26907 31 : if (odd > 1)
26908 : return false;
26909 :
26910 1214 : for (i = 1; i < nelt; ++i)
26911 1196 : if (d->perm[i] != 2 * i + odd)
26912 : return false;
26913 :
26914 18 : if (d->testing_p)
26915 : return true;
26916 :
26917 :
26918 18 : if (odd)
26919 : {
26920 1 : t1 = gen_reg_rtx (V32HImode);
26921 1 : t2 = gen_reg_rtx (V32HImode);
26922 2 : emit_insn (gen_lshrv32hi3 (t1,
26923 1 : gen_lowpart (V32HImode, d->op0),
26924 : GEN_INT (8)));
26925 2 : emit_insn (gen_lshrv32hi3 (t2,
26926 1 : gen_lowpart (V32HImode, d->op1),
26927 : GEN_INT (8)));
26928 : }
26929 : else
26930 : {
26931 17 : t1 = gen_lowpart (V32HImode, d->op0);
26932 17 : t2 = gen_lowpart (V32HImode, d->op1);
26933 : }
26934 :
26935 18 : t3 = gen_reg_rtx (V32QImode);
26936 18 : t4 = gen_reg_rtx (V32QImode);
26937 18 : emit_insn (gen_avx512bw_truncatev32hiv32qi2 (t3, t1));
26938 18 : emit_insn (gen_avx512bw_truncatev32hiv32qi2 (t4, t2));
26939 18 : emit_insn (gen_avx_vec_concatv64qi (d->target, t3, t4));
26940 :
26941 18 : return true;
26942 : }
26943 :
26944 : /* A subroutine of ix86_expand_vec_perm_const_1. Implement extract-even
26945 : and extract-odd permutations. */
26946 :
26947 : static bool
26948 13262 : expand_vec_perm_even_odd_1 (struct expand_vec_perm_d *d, unsigned odd)
26949 : {
26950 13262 : rtx t1, t2, t3, t4, t5;
26951 :
26952 13262 : switch (d->vmode)
26953 : {
26954 19 : case E_V4DFmode:
26955 19 : if (d->testing_p)
26956 : break;
26957 1 : t1 = gen_reg_rtx (V4DFmode);
26958 1 : t2 = gen_reg_rtx (V4DFmode);
26959 :
26960 : /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
26961 1 : emit_insn (gen_avx_vperm2f128v4df3 (t1, d->op0, d->op1, GEN_INT (0x20)));
26962 1 : emit_insn (gen_avx_vperm2f128v4df3 (t2, d->op0, d->op1, GEN_INT (0x31)));
26963 :
26964 : /* Now an unpck[lh]pd will produce the result required. */
26965 1 : if (odd)
26966 0 : t3 = gen_avx_unpckhpd256 (d->target, t1, t2);
26967 : else
26968 1 : t3 = gen_avx_unpcklpd256 (d->target, t1, t2);
26969 1 : emit_insn (t3);
26970 1 : break;
26971 :
26972 1086 : case E_V8SFmode:
26973 1086 : {
26974 1086 : int mask = odd ? 0xdd : 0x88;
26975 :
26976 1086 : if (d->testing_p)
26977 : break;
26978 186 : t1 = gen_reg_rtx (V8SFmode);
26979 186 : t2 = gen_reg_rtx (V8SFmode);
26980 186 : t3 = gen_reg_rtx (V8SFmode);
26981 :
26982 : /* Shuffle within the 128-bit lanes to produce:
26983 : { 0 2 8 a 4 6 c e } | { 1 3 9 b 5 7 d f }. */
26984 186 : emit_insn (gen_avx_shufps256 (t1, d->op0, d->op1,
26985 : GEN_INT (mask)));
26986 :
26987 : /* Shuffle the lanes around to produce:
26988 : { 4 6 c e 0 2 8 a } and { 5 7 d f 1 3 9 b }. */
26989 186 : emit_insn (gen_avx_vperm2f128v8sf3 (t2, t1, t1,
26990 : GEN_INT (0x3)));
26991 :
26992 : /* Shuffle within the 128-bit lanes to produce:
26993 : { 0 2 4 6 4 6 0 2 } | { 1 3 5 7 5 7 1 3 }. */
26994 186 : emit_insn (gen_avx_shufps256 (t3, t1, t2, GEN_INT (0x44)));
26995 :
26996 : /* Shuffle within the 128-bit lanes to produce:
26997 : { 8 a c e c e 8 a } | { 9 b d f d f 9 b }. */
26998 186 : emit_insn (gen_avx_shufps256 (t2, t1, t2, GEN_INT (0xee)));
26999 :
27000 : /* Shuffle the lanes around to produce:
27001 : { 0 2 4 6 8 a c e } | { 1 3 5 7 9 b d f }. */
27002 186 : emit_insn (gen_avx_vperm2f128v8sf3 (d->target, t3, t2,
27003 : GEN_INT (0x20)));
27004 : }
27005 186 : break;
27006 :
27007 0 : case E_V2DFmode:
27008 0 : case E_V4SFmode:
27009 0 : case E_V2DImode:
27010 0 : case E_V2SImode:
27011 0 : case E_V4SImode:
27012 0 : case E_V2HImode:
27013 : /* These are always directly implementable by expand_vec_perm_1. */
27014 0 : gcc_unreachable ();
27015 :
27016 0 : case E_V2SFmode:
27017 0 : gcc_assert (TARGET_MMX_WITH_SSE);
27018 : /* We have no suitable instructions. */
27019 0 : if (d->testing_p)
27020 0 : return false;
27021 : break;
27022 :
27023 1580 : case E_V4QImode:
27024 1580 : if (TARGET_SSSE3 && !TARGET_SLOW_PSHUFB)
27025 0 : return expand_vec_perm_pshufb2 (d);
27026 : else
27027 : {
27028 1580 : if (d->testing_p)
27029 : break;
27030 : /* We need 2*log2(N)-1 operations to achieve odd/even
27031 : with interleave. */
27032 180 : t1 = gen_reg_rtx (V4QImode);
27033 180 : emit_insn (gen_mmx_punpckhbw_low (t1, d->op0, d->op1));
27034 180 : emit_insn (gen_mmx_punpcklbw_low (d->target, d->op0, d->op1));
27035 180 : if (odd)
27036 41 : t2 = gen_mmx_punpckhbw_low (d->target, d->target, t1);
27037 : else
27038 139 : t2 = gen_mmx_punpcklbw_low (d->target, d->target, t1);
27039 180 : emit_insn (t2);
27040 : }
27041 180 : break;
27042 :
27043 1596 : case E_V4HImode:
27044 1596 : if (TARGET_SSE4_1)
27045 92 : return expand_vec_perm_even_odd_pack (d);
27046 1504 : else if (TARGET_SSSE3 && !TARGET_SLOW_PSHUFB)
27047 20 : return expand_vec_perm_pshufb2 (d);
27048 : else
27049 : {
27050 1484 : if (d->testing_p)
27051 : break;
27052 : /* We need 2*log2(N)-1 operations to achieve odd/even
27053 : with interleave. */
27054 495 : t1 = gen_reg_rtx (V4HImode);
27055 495 : emit_insn (gen_mmx_punpckhwd (t1, d->op0, d->op1));
27056 495 : emit_insn (gen_mmx_punpcklwd (d->target, d->op0, d->op1));
27057 495 : if (odd)
27058 8 : t2 = gen_mmx_punpckhwd (d->target, d->target, t1);
27059 : else
27060 487 : t2 = gen_mmx_punpcklwd (d->target, d->target, t1);
27061 495 : emit_insn (t2);
27062 : }
27063 495 : break;
27064 :
27065 6892 : case E_V8HImode:
27066 6892 : if (TARGET_SSE4_1)
27067 444 : return expand_vec_perm_even_odd_pack (d);
27068 6448 : else if (TARGET_SSSE3 && !TARGET_SLOW_PSHUFB)
27069 1 : return expand_vec_perm_pshufb2 (d);
27070 : else
27071 : {
27072 6447 : if (d->testing_p)
27073 : break;
27074 : /* We need 2*log2(N)-1 operations to achieve odd/even
27075 : with interleave. */
27076 2867 : t1 = gen_reg_rtx (V8HImode);
27077 2867 : t2 = gen_reg_rtx (V8HImode);
27078 2867 : emit_insn (gen_vec_interleave_highv8hi (t1, d->op0, d->op1));
27079 2867 : emit_insn (gen_vec_interleave_lowv8hi (d->target, d->op0, d->op1));
27080 2867 : emit_insn (gen_vec_interleave_highv8hi (t2, d->target, t1));
27081 2867 : emit_insn (gen_vec_interleave_lowv8hi (d->target, d->target, t1));
27082 2867 : if (odd)
27083 97 : t3 = gen_vec_interleave_highv8hi (d->target, d->target, t2);
27084 : else
27085 2770 : t3 = gen_vec_interleave_lowv8hi (d->target, d->target, t2);
27086 2867 : emit_insn (t3);
27087 : }
27088 2867 : break;
27089 :
27090 1400 : case E_V8QImode:
27091 1400 : case E_V16QImode:
27092 1400 : return expand_vec_perm_even_odd_pack (d);
27093 :
27094 468 : case E_V16HImode:
27095 468 : case E_V32QImode:
27096 468 : return expand_vec_perm_even_odd_pack (d);
27097 :
27098 26 : case E_V64QImode:
27099 26 : return expand_vec_perm_even_odd_trunc (d);
27100 :
27101 19 : case E_V4DImode:
27102 19 : if (!TARGET_AVX2)
27103 : {
27104 19 : struct expand_vec_perm_d d_copy = *d;
27105 19 : d_copy.vmode = V4DFmode;
27106 19 : if (d->testing_p)
27107 18 : d_copy.target = gen_raw_REG (V4DFmode, LAST_VIRTUAL_REGISTER + 1);
27108 : else
27109 1 : d_copy.target = gen_reg_rtx (V4DFmode);
27110 19 : d_copy.op0 = gen_lowpart (V4DFmode, d->op0);
27111 19 : d_copy.op1 = gen_lowpart (V4DFmode, d->op1);
27112 19 : if (expand_vec_perm_even_odd_1 (&d_copy, odd))
27113 : {
27114 19 : if (!d->testing_p)
27115 1 : emit_move_insn (d->target,
27116 1 : gen_lowpart (V4DImode, d_copy.target));
27117 : return true;
27118 : }
27119 : return false;
27120 : }
27121 :
27122 0 : if (d->testing_p)
27123 : break;
27124 :
27125 0 : t1 = gen_reg_rtx (V4DImode);
27126 0 : t2 = gen_reg_rtx (V4DImode);
27127 :
27128 : /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
27129 0 : emit_insn (gen_avx2_permv2ti (t1, d->op0, d->op1, GEN_INT (0x20)));
27130 0 : emit_insn (gen_avx2_permv2ti (t2, d->op0, d->op1, GEN_INT (0x31)));
27131 :
27132 : /* Now an vpunpck[lh]qdq will produce the result required. */
27133 0 : if (odd)
27134 0 : t3 = gen_avx2_interleave_highv4di (d->target, t1, t2);
27135 : else
27136 0 : t3 = gen_avx2_interleave_lowv4di (d->target, t1, t2);
27137 0 : emit_insn (t3);
27138 0 : break;
27139 :
27140 176 : case E_V8SImode:
27141 176 : if (!TARGET_AVX2)
27142 : {
27143 38 : struct expand_vec_perm_d d_copy = *d;
27144 38 : d_copy.vmode = V8SFmode;
27145 38 : if (d->testing_p)
27146 38 : d_copy.target = gen_raw_REG (V8SFmode, LAST_VIRTUAL_REGISTER + 1);
27147 : else
27148 0 : d_copy.target = gen_reg_rtx (V8SFmode);
27149 38 : d_copy.op0 = gen_lowpart (V8SFmode, d->op0);
27150 38 : d_copy.op1 = gen_lowpart (V8SFmode, d->op1);
27151 38 : if (expand_vec_perm_even_odd_1 (&d_copy, odd))
27152 : {
27153 38 : if (!d->testing_p)
27154 0 : emit_move_insn (d->target,
27155 0 : gen_lowpart (V8SImode, d_copy.target));
27156 : return true;
27157 : }
27158 : return false;
27159 : }
27160 :
27161 138 : if (d->testing_p)
27162 : break;
27163 :
27164 138 : t1 = gen_reg_rtx (V8SImode);
27165 138 : t2 = gen_reg_rtx (V8SImode);
27166 138 : t3 = gen_reg_rtx (V4DImode);
27167 138 : t4 = gen_reg_rtx (V4DImode);
27168 138 : t5 = gen_reg_rtx (V4DImode);
27169 :
27170 : /* Shuffle the lanes around into
27171 : { 0 1 2 3 8 9 a b } and { 4 5 6 7 c d e f }. */
27172 276 : emit_insn (gen_avx2_permv2ti (t3, gen_lowpart (V4DImode, d->op0),
27173 138 : gen_lowpart (V4DImode, d->op1),
27174 : GEN_INT (0x20)));
27175 276 : emit_insn (gen_avx2_permv2ti (t4, gen_lowpart (V4DImode, d->op0),
27176 138 : gen_lowpart (V4DImode, d->op1),
27177 : GEN_INT (0x31)));
27178 :
27179 : /* Swap the 2nd and 3rd position in each lane into
27180 : { 0 2 1 3 8 a 9 b } and { 4 6 5 7 c e d f }. */
27181 138 : emit_insn (gen_avx2_pshufdv3 (t1, gen_lowpart (V8SImode, t3),
27182 : GEN_INT (2 * 4 + 1 * 16 + 3 * 64)));
27183 138 : emit_insn (gen_avx2_pshufdv3 (t2, gen_lowpart (V8SImode, t4),
27184 : GEN_INT (2 * 4 + 1 * 16 + 3 * 64)));
27185 :
27186 : /* Now an vpunpck[lh]qdq will produce
27187 : { 0 2 4 6 8 a c e } resp. { 1 3 5 7 9 b d f }. */
27188 138 : if (odd)
27189 0 : t3 = gen_avx2_interleave_highv4di (t5, gen_lowpart (V4DImode, t1),
27190 0 : gen_lowpart (V4DImode, t2));
27191 : else
27192 138 : t3 = gen_avx2_interleave_lowv4di (t5, gen_lowpart (V4DImode, t1),
27193 138 : gen_lowpart (V4DImode, t2));
27194 138 : emit_insn (t3);
27195 138 : emit_move_insn (d->target, gen_lowpart (V8SImode, t5));
27196 138 : break;
27197 :
27198 0 : default:
27199 0 : gcc_unreachable ();
27200 : }
27201 :
27202 : return true;
27203 : }
27204 :
27205 : /* A subroutine of ix86_expand_vec_perm_const_1. Pattern match
27206 : extract-even and extract-odd permutations. */
27207 :
27208 : static bool
27209 25007 : expand_vec_perm_even_odd (struct expand_vec_perm_d *d)
27210 : {
27211 25007 : unsigned i, odd, nelt = d->nelt;
27212 :
27213 25007 : odd = d->perm[0];
27214 25007 : if (odd != 0 && odd != 1)
27215 : return false;
27216 :
27217 65727 : for (i = 1; i < nelt; ++i)
27218 57753 : if (d->perm[i] != 2 * i + odd)
27219 : return false;
27220 :
27221 7974 : if (d->vmode == E_V32HImode
27222 12 : && d->testing_p
27223 12 : && !TARGET_AVX512BW)
27224 : return false;
27225 :
27226 7962 : return expand_vec_perm_even_odd_1 (d, odd);
27227 : }
27228 :
27229 : /* A subroutine of ix86_expand_vec_perm_const_1. Implement broadcast
27230 : permutations. We assume that expand_vec_perm_1 has already failed. */
27231 :
27232 : static bool
27233 1094 : expand_vec_perm_broadcast_1 (struct expand_vec_perm_d *d)
27234 : {
27235 1094 : unsigned elt = d->perm[0], nelt2 = d->nelt / 2;
27236 1094 : machine_mode vmode = d->vmode;
27237 1094 : rtx (*gen) (rtx, rtx, rtx);
27238 1094 : unsigned char perm2[4];
27239 1094 : rtx op0 = d->op0, dest;
27240 1094 : bool ok;
27241 :
27242 1094 : switch (vmode)
27243 : {
27244 0 : case E_V4DFmode:
27245 0 : case E_V8SFmode:
27246 : /* These are special-cased in sse.md so that we can optionally
27247 : use the vbroadcast instruction. They expand to two insns
27248 : if the input happens to be in a register. */
27249 0 : gcc_unreachable ();
27250 :
27251 0 : case E_V2DFmode:
27252 0 : case E_V2SFmode:
27253 0 : case E_V4SFmode:
27254 0 : case E_V2DImode:
27255 0 : case E_V2SImode:
27256 0 : case E_V4SImode:
27257 0 : case E_V2HImode:
27258 0 : case E_V4HImode:
27259 : /* These are always implementable using standard shuffle patterns. */
27260 0 : gcc_unreachable ();
27261 :
27262 16 : case E_V4QImode:
27263 : /* This can be implemented via interleave and pshuflw. */
27264 16 : if (d->testing_p)
27265 : return true;
27266 :
27267 8 : if (elt >= nelt2)
27268 : {
27269 4 : gen = gen_mmx_punpckhbw_low;
27270 4 : elt -= nelt2;
27271 : }
27272 : else
27273 : gen = gen_mmx_punpcklbw_low;
27274 :
27275 8 : dest = gen_reg_rtx (vmode);
27276 8 : emit_insn (gen (dest, op0, op0));
27277 8 : vmode = get_mode_wider_vector (vmode);
27278 8 : op0 = gen_lowpart (vmode, dest);
27279 :
27280 8 : memset (perm2, elt, 2);
27281 8 : dest = gen_reg_rtx (vmode);
27282 8 : ok = expand_vselect (dest, op0, perm2, 2, d->testing_p);
27283 8 : gcc_assert (ok);
27284 :
27285 8 : emit_move_insn (d->target, gen_lowpart (d->vmode, dest));
27286 8 : return true;
27287 :
27288 4 : case E_V8QImode:
27289 : /* This can be implemented via interleave. We save one insn by
27290 : stopping once we have promoted to V2SImode and then use pshufd. */
27291 4 : if (d->testing_p)
27292 : return true;
27293 4 : do
27294 : {
27295 4 : if (elt >= nelt2)
27296 : {
27297 1 : gen = vmode == V8QImode ? gen_mmx_punpckhbw
27298 : : gen_mmx_punpckhwd;
27299 1 : elt -= nelt2;
27300 : }
27301 : else
27302 3 : gen = vmode == V8QImode ? gen_mmx_punpcklbw
27303 : : gen_mmx_punpcklwd;
27304 4 : nelt2 /= 2;
27305 :
27306 4 : dest = gen_reg_rtx (vmode);
27307 4 : emit_insn (gen (dest, op0, op0));
27308 4 : vmode = get_mode_wider_vector (vmode);
27309 4 : op0 = gen_lowpart (vmode, dest);
27310 : }
27311 4 : while (vmode != V2SImode);
27312 :
27313 2 : memset (perm2, elt, 2);
27314 2 : dest = gen_reg_rtx (vmode);
27315 2 : ok = expand_vselect (dest, op0, perm2, 2, d->testing_p);
27316 2 : gcc_assert (ok);
27317 :
27318 2 : emit_move_insn (d->target, gen_lowpart (d->vmode, dest));
27319 2 : return true;
27320 :
27321 1065 : case E_V8HImode:
27322 1065 : case E_V16QImode:
27323 : /* These can be implemented via interleave. We save one insn by
27324 : stopping once we have promoted to V4SImode and then use pshufd. */
27325 1065 : if (d->testing_p)
27326 : return true;
27327 1640 : do
27328 : {
27329 1640 : if (elt >= nelt2)
27330 : {
27331 16 : gen = vmode == V16QImode ? gen_vec_interleave_highv16qi
27332 : : gen_vec_interleave_highv8hi;
27333 16 : elt -= nelt2;
27334 : }
27335 : else
27336 1624 : gen = vmode == V16QImode ? gen_vec_interleave_lowv16qi
27337 : : gen_vec_interleave_lowv8hi;
27338 1640 : nelt2 /= 2;
27339 :
27340 1640 : dest = gen_reg_rtx (vmode);
27341 1640 : emit_insn (gen (dest, op0, op0));
27342 1640 : vmode = get_mode_wider_vector (vmode);
27343 1640 : op0 = gen_lowpart (vmode, dest);
27344 : }
27345 1640 : while (vmode != V4SImode);
27346 :
27347 1001 : memset (perm2, elt, 4);
27348 1001 : dest = gen_reg_rtx (vmode);
27349 1001 : ok = expand_vselect (dest, op0, perm2, 4, d->testing_p);
27350 1001 : gcc_assert (ok);
27351 :
27352 1001 : emit_move_insn (d->target, gen_lowpart (d->vmode, dest));
27353 1001 : return true;
27354 :
27355 1 : case E_V8HFmode:
27356 1 : case E_V8BFmode:
27357 : /* This can be implemented via interleave and pshufd. */
27358 1 : if (d->testing_p)
27359 : return true;
27360 :
27361 1 : rtx (*gen_interleave) (machine_mode, rtx, rtx, rtx);
27362 1 : if (elt >= nelt2)
27363 : {
27364 0 : gen_interleave = gen_vec_interleave_high;
27365 0 : elt -= nelt2;
27366 : }
27367 : else
27368 : gen_interleave = gen_vec_interleave_low;
27369 1 : nelt2 /= 2;
27370 :
27371 1 : dest = gen_reg_rtx (vmode);
27372 1 : emit_insn (gen_interleave (vmode, dest, op0, op0));
27373 :
27374 1 : vmode = V4SImode;
27375 1 : op0 = gen_lowpart (vmode, dest);
27376 :
27377 1 : memset (perm2, elt, 4);
27378 1 : dest = gen_reg_rtx (vmode);
27379 1 : ok = expand_vselect (dest, op0, perm2, 4, d->testing_p);
27380 1 : gcc_assert (ok);
27381 :
27382 1 : emit_move_insn (d->target, gen_lowpart (d->vmode, dest));
27383 1 : return true;
27384 :
27385 0 : case E_V32QImode:
27386 0 : case E_V16HImode:
27387 0 : case E_V8SImode:
27388 0 : case E_V4DImode:
27389 : /* For AVX2 broadcasts of the first element vpbroadcast* or
27390 : vpermq should be used by expand_vec_perm_1. */
27391 0 : gcc_assert (!TARGET_AVX2 || d->perm[0]);
27392 : return false;
27393 :
27394 6 : case E_V64QImode:
27395 6 : gcc_assert (!TARGET_AVX512BW || d->perm[0]);
27396 : return false;
27397 :
27398 2 : case E_V32HImode:
27399 2 : gcc_assert (!TARGET_AVX512BW);
27400 : return false;
27401 :
27402 0 : default:
27403 0 : gcc_unreachable ();
27404 : }
27405 : }
27406 :
27407 : /* A subroutine of ix86_expand_vec_perm_const_1. Pattern match
27408 : broadcast permutations. */
27409 :
27410 : static bool
27411 93523 : expand_vec_perm_broadcast (struct expand_vec_perm_d *d)
27412 : {
27413 93523 : unsigned i, elt, nelt = d->nelt;
27414 :
27415 93523 : if (!d->one_operand_p)
27416 : return false;
27417 :
27418 5749 : elt = d->perm[0];
27419 8303 : for (i = 1; i < nelt; ++i)
27420 8195 : if (d->perm[i] != elt)
27421 : return false;
27422 :
27423 108 : return expand_vec_perm_broadcast_1 (d);
27424 : }
27425 :
27426 : /* Implement arbitrary permutations of two V64QImode operands
27427 : with 2 vperm[it]2w, 2 vpshufb and one vpor instruction. */
27428 : static bool
27429 25056 : expand_vec_perm_vpermt2_vpshub2 (struct expand_vec_perm_d *d)
27430 : {
27431 25056 : if (!TARGET_AVX512BW || !(d->vmode == V64QImode))
27432 : return false;
27433 :
27434 49 : if (d->testing_p)
27435 : return true;
27436 :
27437 49 : struct expand_vec_perm_d ds[2];
27438 49 : rtx rperm[128], vperm, target0, target1;
27439 49 : unsigned int i, nelt;
27440 49 : machine_mode vmode;
27441 :
27442 49 : nelt = d->nelt;
27443 49 : vmode = V64QImode;
27444 :
27445 147 : for (i = 0; i < 2; i++)
27446 : {
27447 98 : ds[i] = *d;
27448 98 : ds[i].vmode = V32HImode;
27449 98 : ds[i].nelt = 32;
27450 98 : ds[i].target = gen_reg_rtx (V32HImode);
27451 98 : ds[i].op0 = gen_lowpart (V32HImode, d->op0);
27452 98 : ds[i].op1 = gen_lowpart (V32HImode, d->op1);
27453 : }
27454 :
27455 : /* Prepare permutations such that the first one takes care of
27456 : putting the even bytes into the right positions or one higher
27457 : positions (ds[0]) and the second one takes care of
27458 : putting the odd bytes into the right positions or one below
27459 : (ds[1]). */
27460 :
27461 3185 : for (i = 0; i < nelt; i++)
27462 : {
27463 3136 : ds[i & 1].perm[i / 2] = d->perm[i] / 2;
27464 3136 : if (i & 1)
27465 : {
27466 1568 : rperm[i] = constm1_rtx;
27467 1568 : rperm[i + 64] = GEN_INT ((i & 14) + (d->perm[i] & 1));
27468 : }
27469 : else
27470 : {
27471 1568 : rperm[i] = GEN_INT ((i & 14) + (d->perm[i] & 1));
27472 1568 : rperm[i + 64] = constm1_rtx;
27473 : }
27474 : }
27475 :
27476 49 : bool ok = expand_vec_perm_1 (&ds[0]);
27477 49 : gcc_assert (ok);
27478 49 : ds[0].target = gen_lowpart (V64QImode, ds[0].target);
27479 :
27480 49 : ok = expand_vec_perm_1 (&ds[1]);
27481 49 : gcc_assert (ok);
27482 49 : ds[1].target = gen_lowpart (V64QImode, ds[1].target);
27483 :
27484 49 : vperm = gen_rtx_CONST_VECTOR (V64QImode, gen_rtvec_v (64, rperm));
27485 49 : vperm = force_reg (vmode, vperm);
27486 49 : target0 = gen_reg_rtx (V64QImode);
27487 49 : emit_insn (gen_avx512bw_pshufbv64qi3 (target0, ds[0].target, vperm));
27488 :
27489 49 : vperm = gen_rtx_CONST_VECTOR (V64QImode, gen_rtvec_v (64, rperm + 64));
27490 49 : vperm = force_reg (vmode, vperm);
27491 49 : target1 = gen_reg_rtx (V64QImode);
27492 49 : emit_insn (gen_avx512bw_pshufbv64qi3 (target1, ds[1].target, vperm));
27493 :
27494 49 : emit_insn (gen_iorv64qi3 (d->target, target0, target1));
27495 49 : return true;
27496 : }
27497 :
27498 : /* Implement arbitrary permutation of two V32QImode and V16QImode operands
27499 : with 4 vpshufb insns, 2 vpermq and 3 vpor. We should have already failed
27500 : all the shorter instruction sequences. */
27501 :
27502 : static bool
27503 16781 : expand_vec_perm_vpshufb4_vpermq2 (struct expand_vec_perm_d *d)
27504 : {
27505 16781 : rtx rperm[4][32], vperm, l[2], h[2], op, m128;
27506 16781 : unsigned int i, nelt, eltsz;
27507 16781 : bool used[4];
27508 :
27509 16781 : if (!TARGET_AVX2
27510 318 : || d->one_operand_p
27511 189 : || (d->vmode != V32QImode && d->vmode != V16HImode))
27512 : return false;
27513 :
27514 54 : if (d->testing_p)
27515 : return true;
27516 :
27517 54 : nelt = d->nelt;
27518 54 : eltsz = GET_MODE_UNIT_SIZE (d->vmode);
27519 :
27520 : /* Generate 4 permutation masks. If the required element is within
27521 : the same lane, it is shuffled in. If the required element from the
27522 : other lane, force a zero by setting bit 7 in the permutation mask.
27523 : In the other mask the mask has non-negative elements if element
27524 : is requested from the other lane, but also moved to the other lane,
27525 : so that the result of vpshufb can have the two V2TImode halves
27526 : swapped. */
27527 54 : m128 = GEN_INT (-128);
27528 1836 : for (i = 0; i < 32; ++i)
27529 : {
27530 1728 : rperm[0][i] = m128;
27531 1728 : rperm[1][i] = m128;
27532 1728 : rperm[2][i] = m128;
27533 1728 : rperm[3][i] = m128;
27534 : }
27535 54 : used[0] = false;
27536 54 : used[1] = false;
27537 54 : used[2] = false;
27538 54 : used[3] = false;
27539 1590 : for (i = 0; i < nelt; ++i)
27540 : {
27541 1536 : unsigned j, e = d->perm[i] & (nelt / 2 - 1);
27542 1536 : unsigned xlane = ((d->perm[i] ^ i) & (nelt / 2)) * eltsz;
27543 2074 : unsigned int which = ((d->perm[i] & nelt) ? 2 : 0) + (xlane ? 1 : 0);
27544 :
27545 3264 : for (j = 0; j < eltsz; ++j)
27546 1728 : rperm[which][(i * eltsz + j) ^ xlane] = GEN_INT (e * eltsz + j);
27547 1536 : used[which] = true;
27548 : }
27549 :
27550 162 : for (i = 0; i < 2; ++i)
27551 : {
27552 108 : if (!used[2 * i + 1])
27553 : {
27554 22 : h[i] = NULL_RTX;
27555 22 : continue;
27556 : }
27557 86 : vperm = gen_rtx_CONST_VECTOR (V32QImode,
27558 86 : gen_rtvec_v (32, rperm[2 * i + 1]));
27559 86 : vperm = force_reg (V32QImode, vperm);
27560 86 : h[i] = gen_reg_rtx (V32QImode);
27561 86 : op = gen_lowpart (V32QImode, i ? d->op1 : d->op0);
27562 86 : emit_insn (gen_avx2_pshufbv32qi3 (h[i], op, vperm));
27563 : }
27564 :
27565 : /* Swap the 128-byte lanes of h[X]. */
27566 162 : for (i = 0; i < 2; ++i)
27567 : {
27568 108 : if (h[i] == NULL_RTX)
27569 22 : continue;
27570 86 : op = gen_reg_rtx (V4DImode);
27571 86 : emit_insn (gen_avx2_permv4di_1 (op, gen_lowpart (V4DImode, h[i]),
27572 : const2_rtx, GEN_INT (3), const0_rtx,
27573 : const1_rtx));
27574 86 : h[i] = gen_lowpart (V32QImode, op);
27575 : }
27576 :
27577 162 : for (i = 0; i < 2; ++i)
27578 : {
27579 108 : if (!used[2 * i])
27580 : {
27581 0 : l[i] = NULL_RTX;
27582 0 : continue;
27583 : }
27584 108 : vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[2 * i]));
27585 108 : vperm = force_reg (V32QImode, vperm);
27586 108 : l[i] = gen_reg_rtx (V32QImode);
27587 108 : op = gen_lowpart (V32QImode, i ? d->op1 : d->op0);
27588 108 : emit_insn (gen_avx2_pshufbv32qi3 (l[i], op, vperm));
27589 : }
27590 :
27591 162 : for (i = 0; i < 2; ++i)
27592 : {
27593 108 : if (h[i] && l[i])
27594 : {
27595 86 : op = gen_reg_rtx (V32QImode);
27596 86 : emit_insn (gen_iorv32qi3 (op, l[i], h[i]));
27597 86 : l[i] = op;
27598 : }
27599 22 : else if (h[i])
27600 0 : l[i] = h[i];
27601 : }
27602 :
27603 54 : gcc_assert (l[0] && l[1]);
27604 54 : op = d->target;
27605 54 : if (d->vmode != V32QImode)
27606 12 : op = gen_reg_rtx (V32QImode);
27607 54 : emit_insn (gen_iorv32qi3 (op, l[0], l[1]));
27608 54 : if (op != d->target)
27609 12 : emit_move_insn (d->target, gen_lowpart (d->vmode, op));
27610 : return true;
27611 : }
27612 :
27613 : /* The guts of ix86_vectorize_vec_perm_const. With all of the interface bits
27614 : taken care of, perform the expansion in D and return true on success. */
27615 :
27616 : static bool
27617 304851 : ix86_expand_vec_perm_const_1 (struct expand_vec_perm_d *d)
27618 : {
27619 : /* Try a single instruction expansion. */
27620 304851 : if (expand_vec_perm_1 (d))
27621 : return true;
27622 :
27623 : /* Try sequences of two instructions. */
27624 :
27625 105328 : if (expand_vec_perm_pshuflw_pshufhw (d))
27626 : return true;
27627 :
27628 102837 : if (expand_vec_perm_palignr (d, false))
27629 : return true;
27630 :
27631 99702 : if (expand_vec_perm_interleave2 (d))
27632 : return true;
27633 :
27634 93523 : if (expand_vec_perm_broadcast (d))
27635 : return true;
27636 :
27637 93423 : if (expand_vec_perm_vpermq_perm_1 (d))
27638 : return true;
27639 :
27640 93423 : if (expand_vec_perm_vperm2f128 (d))
27641 : return true;
27642 :
27643 93355 : if (expand_vec_perm_pblendv (d))
27644 : return true;
27645 :
27646 90920 : if (expand_vec_perm_2perm_interleave (d, true))
27647 : return true;
27648 :
27649 90562 : if (expand_vec_perm_2perm_pblendv (d, true))
27650 : return true;
27651 :
27652 88420 : if (expand_vec_perm_shufps_shufps (d))
27653 : return true;
27654 :
27655 52128 : if (expand_vec_perm_punpckldq_pshuf (d))
27656 : return true;
27657 :
27658 : /* Try sequences of three instructions. */
27659 :
27660 45584 : if (expand_vec_perm_even_odd_pack (d))
27661 : return true;
27662 :
27663 31402 : if (expand_vec_perm_2vperm2f128_vshuf (d))
27664 : return true;
27665 :
27666 30253 : if (expand_vec_perm_pshufb2 (d))
27667 : return true;
27668 :
27669 28837 : if (expand_vec_perm_pslldq_psrldq_por (d, false))
27670 : return true;
27671 :
27672 28524 : if (expand_vec_perm_interleave3 (d))
27673 : return true;
27674 :
27675 28378 : if (expand_vec_perm_vperm2f128_vblend (d))
27676 : return true;
27677 :
27678 28378 : if (expand_vec_perm_2perm_interleave (d, false))
27679 : return true;
27680 :
27681 28186 : if (expand_vec_perm_2perm_pblendv (d, false))
27682 : return true;
27683 :
27684 27626 : if (expand_vec_perm_psrlw_psllw_por (d))
27685 : return true;
27686 :
27687 26172 : if (expand_vec_perm_pand_pandn_por (d))
27688 : return true;
27689 :
27690 : /* Try sequences of four instructions. */
27691 :
27692 25067 : if (expand_vec_perm_even_odd_trunc (d))
27693 : return true;
27694 25063 : if (expand_vec_perm_vpshufb2_vpermq (d))
27695 : return true;
27696 :
27697 25056 : if (expand_vec_perm_vpshufb2_vpermq_even_odd (d))
27698 : return true;
27699 :
27700 25056 : if (expand_vec_perm_vpermt2_vpshub2 (d))
27701 : return true;
27702 :
27703 : /* ??? Look for narrow permutations whose element orderings would
27704 : allow the promotion to a wider mode. */
27705 :
27706 : /* ??? Look for sequences of interleave or a wider permute that place
27707 : the data into the correct lanes for a half-vector shuffle like
27708 : pshuf[lh]w or vpermilps. */
27709 :
27710 : /* ??? Look for sequences of interleave that produce the desired results.
27711 : The combinatorics of punpck[lh] get pretty ugly... */
27712 :
27713 25007 : if (expand_vec_perm_even_odd (d))
27714 : return true;
27715 :
27716 : /* Generate four or five instructions. */
27717 17117 : if (expand_vec_perm_pslldq_psrldq_por (d, true))
27718 : return true;
27719 :
27720 : /* Even longer sequences. */
27721 16781 : if (expand_vec_perm_vpshufb4_vpermq2 (d))
27722 : return true;
27723 :
27724 : /* See if we can get the same permutation in different vector integer
27725 : mode. */
27726 16727 : struct expand_vec_perm_d nd;
27727 16727 : if (canonicalize_vector_int_perm (d, &nd) && expand_vec_perm_1 (&nd))
27728 : {
27729 0 : if (!d->testing_p)
27730 0 : emit_move_insn (d->target, gen_lowpart (d->vmode, nd.target));
27731 : return true;
27732 : }
27733 :
27734 : /* Even longer, including recursion to ix86_expand_vec_perm_const_1. */
27735 16727 : if (expand_vec_perm2_vperm2f128_vblend (d))
27736 : return true;
27737 :
27738 : return false;
27739 : }
27740 :
27741 : /* If a permutation only uses one operand, make it clear. Returns true
27742 : if the permutation references both operands. */
27743 :
27744 : static bool
27745 75302 : canonicalize_perm (struct expand_vec_perm_d *d)
27746 : {
27747 75302 : int i, which, nelt = d->nelt;
27748 :
27749 450858 : for (i = which = 0; i < nelt; ++i)
27750 511692 : which |= (d->perm[i] < nelt ? 1 : 2);
27751 :
27752 75302 : d->one_operand_p = true;
27753 75302 : switch (which)
27754 : {
27755 0 : default:
27756 0 : gcc_unreachable();
27757 :
27758 56721 : case 3:
27759 56721 : if (!rtx_equal_p (d->op0, d->op1))
27760 : {
27761 56688 : d->one_operand_p = false;
27762 56688 : break;
27763 : }
27764 : /* The elements of PERM do not suggest that only the first operand
27765 : is used, but both operands are identical. Allow easier matching
27766 : of the permutation by folding the permutation into the single
27767 : input vector. */
27768 : /* FALLTHRU */
27769 :
27770 415 : case 2:
27771 3517 : for (i = 0; i < nelt; ++i)
27772 3102 : d->perm[i] &= nelt - 1;
27773 415 : d->op0 = d->op1;
27774 415 : break;
27775 :
27776 18199 : case 1:
27777 18199 : d->op1 = d->op0;
27778 18199 : break;
27779 : }
27780 :
27781 75302 : return (which == 3);
27782 : }
27783 :
27784 : /* Implement TARGET_VECTORIZE_VEC_PERM_CONST. */
27785 :
27786 : bool
27787 838087 : ix86_vectorize_vec_perm_const (machine_mode vmode, machine_mode op_mode,
27788 : rtx target, rtx op0, rtx op1,
27789 : const vec_perm_indices &sel)
27790 : {
27791 838087 : if (vmode != op_mode)
27792 : return false;
27793 :
27794 830363 : struct expand_vec_perm_d d;
27795 830363 : unsigned char perm[MAX_VECT_LEN];
27796 830363 : unsigned int i, nelt, which;
27797 830363 : bool two_args;
27798 :
27799 : /* For HF and BF mode vector, convert it to HI using subreg. */
27800 2488634 : if (GET_MODE_INNER (vmode) == HFmode || GET_MODE_INNER (vmode) == BFmode)
27801 : {
27802 2485 : machine_mode orig_mode = vmode;
27803 4970 : vmode = mode_for_vector (HImode,
27804 2485 : GET_MODE_NUNITS (vmode)).require ();
27805 2485 : if (target)
27806 436 : target = lowpart_subreg (vmode, target, orig_mode);
27807 2485 : if (op0)
27808 436 : op0 = lowpart_subreg (vmode, op0, orig_mode);
27809 2485 : if (op1)
27810 436 : op1 = lowpart_subreg (vmode, op1, orig_mode);
27811 : }
27812 :
27813 830363 : d.target = target;
27814 830363 : d.op0 = op0;
27815 830363 : d.op1 = op1;
27816 :
27817 830363 : d.vmode = vmode;
27818 830363 : gcc_assert (VECTOR_MODE_P (d.vmode));
27819 830363 : d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
27820 830363 : d.testing_p = !target;
27821 :
27822 830363 : gcc_assert (sel.length () == nelt);
27823 830363 : gcc_checking_assert (sizeof (d.perm) == sizeof (perm));
27824 :
27825 : /* Given sufficient ISA support we can just return true here
27826 : for selected vector modes. */
27827 830363 : switch (d.vmode)
27828 : {
27829 1612 : case E_V16SFmode:
27830 1612 : case E_V16SImode:
27831 1612 : case E_V8DImode:
27832 1612 : case E_V8DFmode:
27833 1612 : if (!TARGET_AVX512F)
27834 : return false;
27835 : /* All implementable with a single vperm[it]2 insn. */
27836 1612 : if (d.testing_p)
27837 : return true;
27838 : break;
27839 323 : case E_V32HImode:
27840 323 : if (!TARGET_AVX512F)
27841 : return false;
27842 323 : if (d.testing_p && TARGET_AVX512BW)
27843 : /* All implementable with a single vperm[it]2 insn. */
27844 : return true;
27845 : break;
27846 715 : case E_V64QImode:
27847 715 : if (!TARGET_AVX512F)
27848 : return false;
27849 715 : if (d.testing_p && TARGET_AVX512BW)
27850 : /* Implementable with 2 vperm[it]2, 2 vpshufb and 1 or insn. */
27851 : return true;
27852 : break;
27853 12799 : case E_V8SImode:
27854 12799 : case E_V8SFmode:
27855 12799 : case E_V4DFmode:
27856 12799 : case E_V4DImode:
27857 12799 : if (!TARGET_AVX)
27858 : return false;
27859 12799 : if (d.testing_p && TARGET_AVX512VL)
27860 : /* All implementable with a single vperm[it]2 insn. */
27861 : return true;
27862 : break;
27863 613 : case E_V16HImode:
27864 613 : if (!TARGET_SSE2)
27865 : return false;
27866 613 : if (d.testing_p && TARGET_AVX2)
27867 : /* Implementable with 4 vpshufb insns, 2 vpermq and 3 vpor insns. */
27868 : return true;
27869 : break;
27870 685 : case E_V32QImode:
27871 685 : if (!TARGET_SSE2)
27872 : return false;
27873 685 : if (d.testing_p && TARGET_AVX2)
27874 : /* Implementable with 4 vpshufb insns, 2 vpermq and 3 vpor insns. */
27875 : return true;
27876 : break;
27877 40689 : case E_V8HImode:
27878 40689 : case E_V16QImode:
27879 40689 : if (!TARGET_SSE2)
27880 : return false;
27881 : /* Fall through. */
27882 249755 : case E_V4SImode:
27883 249755 : case E_V4SFmode:
27884 249755 : if (!TARGET_SSE)
27885 : return false;
27886 : /* All implementable with a single vpperm insn. */
27887 249755 : if (d.testing_p && TARGET_XOP)
27888 : return true;
27889 : /* All implementable with 2 pshufb + 1 ior. */
27890 249649 : if (d.testing_p && TARGET_SSSE3)
27891 : return true;
27892 : break;
27893 152632 : case E_V2SFmode:
27894 152632 : case E_V2SImode:
27895 152632 : case E_V4HImode:
27896 152632 : case E_V8QImode:
27897 152632 : if (!TARGET_MMX_WITH_SSE)
27898 : return false;
27899 : break;
27900 21376 : case E_V2HImode:
27901 21376 : if (!TARGET_SSE2)
27902 : return false;
27903 : /* All implementable with *punpckwd. */
27904 21376 : if (d.testing_p)
27905 : return true;
27906 : break;
27907 13018 : case E_V4QImode:
27908 13018 : if (!TARGET_SSE2)
27909 : return false;
27910 : break;
27911 374325 : case E_V2DImode:
27912 374325 : case E_V2DFmode:
27913 374325 : if (!TARGET_SSE)
27914 : return false;
27915 : /* All implementable with shufpd or unpck[lh]pd. */
27916 374325 : if (d.testing_p)
27917 : return true;
27918 : break;
27919 : default:
27920 : return false;
27921 : }
27922 :
27923 2385215 : for (i = which = 0; i < nelt; ++i)
27924 : {
27925 1952068 : unsigned char e = sel[i];
27926 1952068 : gcc_assert (e < 2 * nelt);
27927 1952068 : d.perm[i] = e;
27928 1952068 : perm[i] = e;
27929 2619472 : which |= (e < nelt ? 1 : 2);
27930 : }
27931 :
27932 433147 : if (d.testing_p)
27933 : {
27934 : /* For all elements from second vector, fold the elements to first. */
27935 359109 : if (which == 2)
27936 1357 : for (i = 0; i < nelt; ++i)
27937 1204 : d.perm[i] -= nelt;
27938 :
27939 : /* Check whether the mask can be applied to the vector type. */
27940 359109 : d.one_operand_p = (which != 3);
27941 :
27942 : /* Implementable with shufps, pshufd or pshuflw. */
27943 359109 : if (d.one_operand_p
27944 : && (d.vmode == V4SFmode || d.vmode == V2SFmode
27945 : || d.vmode == V4SImode || d.vmode == V2SImode
27946 : || d.vmode == V4HImode || d.vmode == V2HImode))
27947 : return true;
27948 :
27949 : /* Otherwise we have to go through the motions and see if we can
27950 : figure out how to generate the requested permutation. */
27951 227053 : d.target = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 1);
27952 227053 : d.op1 = d.op0 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 2);
27953 227053 : if (!d.one_operand_p)
27954 211083 : d.op1 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 3);
27955 :
27956 227053 : start_sequence ();
27957 227053 : bool ret = ix86_expand_vec_perm_const_1 (&d);
27958 227053 : end_sequence ();
27959 :
27960 227053 : return ret;
27961 : }
27962 :
27963 74038 : two_args = canonicalize_perm (&d);
27964 :
27965 : /* If one of the operands is a zero vector, try to match pmovzx. */
27966 74038 : if (two_args && (d.op0 == CONST0_RTX (vmode) || d.op1 == CONST0_RTX (vmode)))
27967 : {
27968 689 : struct expand_vec_perm_d dzero = d;
27969 689 : if (d.op0 == CONST0_RTX (vmode))
27970 : {
27971 452 : d.op1 = dzero.op1 = force_reg (vmode, d.op1);
27972 452 : std::swap (dzero.op0, dzero.op1);
27973 7852 : for (i = 0; i < nelt; ++i)
27974 7400 : dzero.perm[i] ^= nelt;
27975 : }
27976 : else
27977 237 : d.op0 = dzero.op0 = force_reg (vmode, d.op0);
27978 :
27979 689 : if (expand_vselect_vconcat (dzero.target, dzero.op0, dzero.op1,
27980 : dzero.perm, nelt, dzero.testing_p))
27981 261 : return true;
27982 : }
27983 :
27984 : /* Force operands into registers. */
27985 73777 : rtx nop0 = force_reg (vmode, d.op0);
27986 73777 : if (d.op0 == d.op1)
27987 18213 : d.op1 = nop0;
27988 73777 : d.op0 = nop0;
27989 73777 : d.op1 = force_reg (vmode, d.op1);
27990 :
27991 73777 : if (ix86_expand_vec_perm_const_1 (&d))
27992 : return true;
27993 :
27994 : /* If the selector says both arguments are needed, but the operands are the
27995 : same, the above tried to expand with one_operand_p and flattened selector.
27996 : If that didn't work, retry without one_operand_p; we succeeded with that
27997 : during testing. */
27998 0 : if (two_args && d.one_operand_p)
27999 : {
28000 0 : d.one_operand_p = false;
28001 0 : memcpy (d.perm, perm, sizeof (perm));
28002 0 : return ix86_expand_vec_perm_const_1 (&d);
28003 : }
28004 :
28005 : return false;
28006 : }
28007 :
28008 : void
28009 8488 : ix86_expand_vec_extract_even_odd (rtx targ, rtx op0, rtx op1, unsigned odd)
28010 : {
28011 8488 : struct expand_vec_perm_d d;
28012 8488 : unsigned i, nelt;
28013 :
28014 8488 : d.target = targ;
28015 8488 : d.op0 = op0;
28016 8488 : d.op1 = op1;
28017 8488 : d.vmode = GET_MODE (targ);
28018 8488 : d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
28019 8488 : d.one_operand_p = false;
28020 8488 : d.testing_p = false;
28021 :
28022 80584 : for (i = 0; i < nelt; ++i)
28023 72096 : d.perm[i] = i * 2 + odd;
28024 :
28025 : /* We'll either be able to implement the permutation directly... */
28026 8488 : if (expand_vec_perm_1 (&d))
28027 : return;
28028 :
28029 : /* ... or we use the special-case patterns. */
28030 5243 : expand_vec_perm_even_odd_1 (&d, odd);
28031 : }
28032 :
28033 : static void
28034 944 : ix86_expand_vec_interleave (rtx targ, rtx op0, rtx op1, bool high_p)
28035 : {
28036 944 : struct expand_vec_perm_d d;
28037 944 : unsigned i, nelt, base;
28038 944 : bool ok;
28039 :
28040 944 : d.target = targ;
28041 944 : d.op0 = op0;
28042 944 : d.op1 = op1;
28043 944 : d.vmode = GET_MODE (targ);
28044 944 : d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
28045 944 : d.one_operand_p = false;
28046 944 : d.testing_p = false;
28047 :
28048 944 : base = high_p ? nelt / 2 : 0;
28049 3744 : for (i = 0; i < nelt / 2; ++i)
28050 : {
28051 2800 : d.perm[i * 2] = i + base;
28052 2800 : d.perm[i * 2 + 1] = i + base + nelt;
28053 : }
28054 :
28055 : /* Note that for AVX this isn't one instruction. */
28056 944 : ok = ix86_expand_vec_perm_const_1 (&d);
28057 944 : gcc_assert (ok);
28058 944 : }
28059 :
28060 : /* Expand a vector operation shift by constant for a V*QImode in terms of the
28061 : same operation on V*HImode. Return true if success. */
28062 : static bool
28063 459 : ix86_expand_vec_shift_qihi_constant (enum rtx_code code,
28064 : rtx dest, rtx op1, rtx op2)
28065 : {
28066 459 : machine_mode qimode, himode;
28067 459 : HOST_WIDE_INT and_constant, xor_constant;
28068 459 : HOST_WIDE_INT shift_amount;
28069 459 : rtx vec_const_and, vec_const_xor;
28070 459 : rtx tmp, op1_subreg;
28071 459 : rtx (*gen_shift) (rtx, rtx, rtx);
28072 459 : rtx (*gen_and) (rtx, rtx, rtx);
28073 459 : rtx (*gen_xor) (rtx, rtx, rtx);
28074 459 : rtx (*gen_sub) (rtx, rtx, rtx);
28075 :
28076 : /* Only optimize shift by constant. */
28077 459 : if (!CONST_INT_P (op2))
28078 : return false;
28079 :
28080 459 : qimode = GET_MODE (dest);
28081 459 : shift_amount = INTVAL (op2);
28082 : /* Do nothing when shift amount greater equal 8. */
28083 459 : if (shift_amount > 7)
28084 : return false;
28085 :
28086 459 : gcc_assert (code == ASHIFT || code == ASHIFTRT || code == LSHIFTRT);
28087 :
28088 :
28089 459 : if (shift_amount == 7
28090 459 : && code == ASHIFTRT)
28091 : {
28092 84 : if (qimode == V16QImode
28093 9 : || qimode == V32QImode)
28094 : {
28095 83 : rtx zero = gen_reg_rtx (qimode);
28096 83 : emit_move_insn (zero, CONST0_RTX (qimode));
28097 83 : emit_move_insn (dest, gen_rtx_fmt_ee (GT, qimode, zero, op1));
28098 83 : }
28099 : else
28100 : {
28101 1 : gcc_assert (qimode == V64QImode);
28102 1 : rtx kmask = gen_reg_rtx (DImode);
28103 1 : emit_insn (gen_avx512bw_cvtb2maskv64qi (kmask, op1));
28104 1 : emit_insn (gen_avx512bw_cvtmask2bv64qi (dest, kmask));
28105 : }
28106 : return true;
28107 : }
28108 :
28109 : /* Record sign bit. */
28110 375 : xor_constant = 1 << (8 - shift_amount - 1);
28111 :
28112 : /* Zero upper/lower bits shift from left/right element. */
28113 375 : and_constant
28114 375 : = (code == ASHIFT ? 256 - (1 << shift_amount)
28115 344 : : (1 << (8 - shift_amount)) - 1);
28116 :
28117 375 : switch (qimode)
28118 : {
28119 362 : case V16QImode:
28120 362 : himode = V8HImode;
28121 260 : gen_shift =
28122 : ((code == ASHIFT)
28123 362 : ? gen_ashlv8hi3
28124 340 : : (code == ASHIFTRT) ? gen_ashrv8hi3 : gen_lshrv8hi3);
28125 : gen_and = gen_andv16qi3;
28126 : gen_xor = gen_xorv16qi3;
28127 : gen_sub = gen_subv16qi3;
28128 : break;
28129 6 : case V32QImode:
28130 6 : himode = V16HImode;
28131 1 : gen_shift =
28132 : ((code == ASHIFT)
28133 6 : ? gen_ashlv16hi3
28134 2 : : (code == ASHIFTRT) ? gen_ashrv16hi3 : gen_lshrv16hi3);
28135 : gen_and = gen_andv32qi3;
28136 : gen_xor = gen_xorv32qi3;
28137 : gen_sub = gen_subv32qi3;
28138 : break;
28139 7 : case V64QImode:
28140 7 : himode = V32HImode;
28141 1 : gen_shift =
28142 : ((code == ASHIFT)
28143 7 : ? gen_ashlv32hi3
28144 2 : : (code == ASHIFTRT) ? gen_ashrv32hi3 : gen_lshrv32hi3);
28145 : gen_and = gen_andv64qi3;
28146 : gen_xor = gen_xorv64qi3;
28147 : gen_sub = gen_subv64qi3;
28148 : break;
28149 0 : default:
28150 0 : gcc_unreachable ();
28151 : }
28152 :
28153 375 : tmp = gen_reg_rtx (himode);
28154 375 : vec_const_and = gen_reg_rtx (qimode);
28155 375 : op1_subreg = lowpart_subreg (himode, op1, qimode);
28156 :
28157 : /* For ASHIFT and LSHIFTRT, perform operation like
28158 : vpsllw/vpsrlw $shift_amount, %op1, %dest.
28159 : vpand %vec_const_and, %dest. */
28160 375 : emit_insn (gen_shift (tmp, op1_subreg, op2));
28161 375 : emit_move_insn (dest, simplify_gen_subreg (qimode, tmp, himode, 0));
28162 375 : emit_move_insn (vec_const_and,
28163 : ix86_build_const_vector (qimode, true,
28164 375 : gen_int_mode (and_constant, QImode)));
28165 375 : emit_insn (gen_and (dest, dest, vec_const_and));
28166 :
28167 : /* For ASHIFTRT, perform extra operation like
28168 : vpxor %vec_const_xor, %dest, %dest
28169 : vpsubb %vec_const_xor, %dest, %dest */
28170 375 : if (code == ASHIFTRT)
28171 : {
28172 82 : vec_const_xor = gen_reg_rtx (qimode);
28173 82 : emit_move_insn (vec_const_xor,
28174 : ix86_build_const_vector (qimode, true,
28175 82 : gen_int_mode (xor_constant, QImode)));
28176 82 : emit_insn (gen_xor (dest, dest, vec_const_xor));
28177 82 : emit_insn (gen_sub (dest, dest, vec_const_xor));
28178 : }
28179 : return true;
28180 : }
28181 :
28182 : void
28183 1451 : ix86_expand_vecop_qihi_partial (enum rtx_code code, rtx dest, rtx op1, rtx op2)
28184 : {
28185 1451 : machine_mode qimode = GET_MODE (dest);
28186 1451 : rtx qop1, qop2, hop1, hop2, qdest, hdest;
28187 1451 : bool op2vec = GET_MODE_CLASS (GET_MODE (op2)) == MODE_VECTOR_INT;
28188 1451 : bool uns_p = code != ASHIFTRT;
28189 :
28190 1451 : switch (qimode)
28191 : {
28192 1451 : case E_V4QImode:
28193 1451 : case E_V8QImode:
28194 1451 : break;
28195 0 : default:
28196 0 : gcc_unreachable ();
28197 : }
28198 :
28199 1451 : qop1 = lowpart_subreg (V16QImode, force_reg (qimode, op1), qimode);
28200 :
28201 1451 : if (op2vec)
28202 1317 : qop2 = lowpart_subreg (V16QImode, force_reg (qimode, op2), qimode);
28203 : else
28204 : qop2 = op2;
28205 :
28206 1451 : qdest = gen_reg_rtx (V16QImode);
28207 :
28208 1451 : if (CONST_INT_P (op2)
28209 122 : && (code == ASHIFT || code == LSHIFTRT || code == ASHIFTRT)
28210 : /* With AVX512 it's cheaper to do vpmovsxbw/op/vpmovwb.
28211 : Even with SSE4.1 the alternative is better. */
28212 122 : && !TARGET_SSE4_1
28213 1505 : && ix86_expand_vec_shift_qihi_constant (code, qdest, qop1, qop2))
28214 : {
28215 54 : emit_move_insn (dest, gen_lowpart (qimode, qdest));
28216 54 : return;
28217 : }
28218 :
28219 1397 : if (CONST_INT_P (op2)
28220 68 : && code == ASHIFTRT
28221 14 : && INTVAL (op2) == 7)
28222 : {
28223 4 : rtx zero = gen_reg_rtx (qimode);
28224 4 : emit_move_insn (zero, CONST0_RTX (qimode));
28225 4 : emit_move_insn (dest, gen_rtx_fmt_ee (GT, qimode, zero, op1));
28226 4 : return;
28227 : }
28228 :
28229 1393 : switch (code)
28230 : {
28231 1304 : case MULT:
28232 1304 : gcc_assert (op2vec);
28233 1304 : if (!TARGET_SSE4_1)
28234 : {
28235 : /* Unpack data such that we've got a source byte in each low byte
28236 : of each word. We don't care what goes into the high byte of
28237 : each word. Rather than trying to get zero in there, most
28238 : convenient is to let it be a copy of the low byte. */
28239 251 : hop1 = copy_to_reg (qop1);
28240 251 : hop2 = copy_to_reg (qop2);
28241 251 : emit_insn (gen_vec_interleave_lowv16qi (hop1, hop1, hop1));
28242 251 : emit_insn (gen_vec_interleave_lowv16qi (hop2, hop2, hop2));
28243 251 : break;
28244 : }
28245 : /* FALLTHRU */
28246 1142 : case ASHIFT:
28247 1142 : case ASHIFTRT:
28248 1142 : case LSHIFTRT:
28249 1142 : hop1 = gen_reg_rtx (V8HImode);
28250 1142 : ix86_expand_sse_unpack (hop1, qop1, uns_p, false);
28251 : /* mult/vashr/vlshr/vashl */
28252 1142 : if (op2vec)
28253 : {
28254 1066 : hop2 = gen_reg_rtx (V8HImode);
28255 1066 : ix86_expand_sse_unpack (hop2, qop2, uns_p, false);
28256 : }
28257 : else
28258 : hop2 = qop2;
28259 :
28260 : break;
28261 0 : default:
28262 0 : gcc_unreachable ();
28263 : }
28264 :
28265 1393 : if (code != MULT && op2vec)
28266 : {
28267 : /* Expand vashr/vlshr/vashl. */
28268 13 : hdest = gen_reg_rtx (V8HImode);
28269 13 : emit_insn (gen_rtx_SET (hdest,
28270 : simplify_gen_binary (code, V8HImode,
28271 : hop1, hop2)));
28272 : }
28273 : else
28274 : /* Expand mult/ashr/lshr/ashl. */
28275 1380 : hdest = expand_simple_binop (V8HImode, code, hop1, hop2,
28276 : NULL_RTX, 1, OPTAB_DIRECT);
28277 :
28278 1393 : if (TARGET_AVX512BW && TARGET_AVX512VL)
28279 : {
28280 57 : if (qimode == V8QImode)
28281 : qdest = dest;
28282 : else
28283 10 : qdest = gen_reg_rtx (V8QImode);
28284 :
28285 57 : emit_insn (gen_truncv8hiv8qi2 (qdest, hdest));
28286 : }
28287 : else
28288 : {
28289 1336 : struct expand_vec_perm_d d;
28290 1336 : rtx qres = gen_lowpart (V16QImode, hdest);
28291 1336 : bool ok;
28292 1336 : int i;
28293 :
28294 : /* Merge the data back into the right place. */
28295 1336 : d.target = qdest;
28296 1336 : d.op0 = d.op1 = qres;
28297 1336 : d.vmode = V16QImode;
28298 1336 : d.nelt = 16;
28299 1336 : d.one_operand_p = TARGET_SSSE3;
28300 1336 : d.testing_p = false;
28301 :
28302 22712 : for (i = 0; i < d.nelt; ++i)
28303 21376 : d.perm[i] = i * 2;
28304 :
28305 1336 : ok = ix86_expand_vec_perm_const_1 (&d);
28306 1336 : gcc_assert (ok);
28307 : }
28308 :
28309 1393 : if (qdest != dest)
28310 1346 : emit_move_insn (dest, gen_lowpart (qimode, qdest));
28311 : }
28312 :
28313 : /* Emit instruction in 2x wider mode. For example, optimize
28314 : vector MUL generation like
28315 :
28316 : vpmovzxbw ymm2, xmm0
28317 : vpmovzxbw ymm3, xmm1
28318 : vpmullw ymm4, ymm2, ymm3
28319 : vpmovwb xmm0, ymm4
28320 :
28321 : it would take less instructions than ix86_expand_vecop_qihi.
28322 : Return true if success. */
28323 :
28324 : static bool
28325 1185 : ix86_expand_vecop_qihi2 (enum rtx_code code, rtx dest, rtx op1, rtx op2)
28326 : {
28327 1185 : machine_mode himode, qimode = GET_MODE (dest);
28328 1185 : machine_mode wqimode;
28329 1185 : rtx qop1, qop2, hop1, hop2, hdest;
28330 1185 : rtx (*gen_truncate)(rtx, rtx) = NULL;
28331 1185 : bool op2vec = GET_MODE_CLASS (GET_MODE (op2)) == MODE_VECTOR_INT;
28332 1185 : bool uns_p = code != ASHIFTRT;
28333 :
28334 : /* Without VPMOVWB (provided by AVX512BW ISA), the expansion uses the
28335 : generic permutation to merge the data back into the right place. This
28336 : permutation results in VPERMQ, which is slow, so better fall back to
28337 : ix86_expand_vecop_qihi. */
28338 1185 : if (!TARGET_AVX512BW
28339 327 : || (qimode == V16QImode && !TARGET_AVX512VL)
28340 : /* There are no V64HImode instructions. */
28341 327 : || qimode == V64QImode)
28342 : return false;
28343 :
28344 : /* Do not generate ymm/zmm instructions when
28345 : target prefers 128/256 bit vector width. */
28346 317 : if ((qimode == V16QImode && TARGET_PREFER_AVX128)
28347 317 : || (qimode == V32QImode && TARGET_PREFER_AVX256))
28348 : return false;
28349 :
28350 312 : switch (qimode)
28351 : {
28352 : case E_V16QImode:
28353 : himode = V16HImode;
28354 : gen_truncate = gen_truncv16hiv16qi2;
28355 : break;
28356 57 : case E_V32QImode:
28357 57 : himode = V32HImode;
28358 57 : gen_truncate = gen_truncv32hiv32qi2;
28359 57 : break;
28360 0 : default:
28361 0 : gcc_unreachable ();
28362 : }
28363 :
28364 312 : wqimode = GET_MODE_2XWIDER_MODE (qimode).require ();
28365 312 : qop1 = lowpart_subreg (wqimode, force_reg (qimode, op1), qimode);
28366 :
28367 312 : if (op2vec)
28368 312 : qop2 = lowpart_subreg (wqimode, force_reg (qimode, op2), qimode);
28369 : else
28370 : qop2 = op2;
28371 :
28372 312 : hop1 = gen_reg_rtx (himode);
28373 312 : ix86_expand_sse_unpack (hop1, qop1, uns_p, false);
28374 :
28375 312 : if (op2vec)
28376 : {
28377 312 : hop2 = gen_reg_rtx (himode);
28378 312 : ix86_expand_sse_unpack (hop2, qop2, uns_p, false);
28379 : }
28380 : else
28381 : hop2 = qop2;
28382 :
28383 312 : if (code != MULT && op2vec)
28384 : {
28385 : /* Expand vashr/vlshr/vashl. */
28386 14 : hdest = gen_reg_rtx (himode);
28387 14 : emit_insn (gen_rtx_SET (hdest,
28388 : simplify_gen_binary (code, himode,
28389 : hop1, hop2)));
28390 : }
28391 : else
28392 : /* Expand mult/ashr/lshr/ashl. */
28393 298 : hdest = expand_simple_binop (himode, code, hop1, hop2,
28394 : NULL_RTX, 1, OPTAB_DIRECT);
28395 :
28396 312 : emit_insn (gen_truncate (dest, hdest));
28397 312 : return true;
28398 : }
28399 :
28400 : /* Expand a vector operation CODE for a V*QImode in terms of the
28401 : same operation on V*HImode. */
28402 :
28403 : void
28404 1590 : ix86_expand_vecop_qihi (enum rtx_code code, rtx dest, rtx op1, rtx op2)
28405 : {
28406 1590 : machine_mode qimode = GET_MODE (dest);
28407 1590 : machine_mode himode;
28408 1590 : rtx (*gen_il) (rtx, rtx, rtx);
28409 1590 : rtx (*gen_ih) (rtx, rtx, rtx);
28410 1590 : rtx op1_l, op1_h, op2_l, op2_h, res_l, res_h;
28411 1590 : bool op2vec = GET_MODE_CLASS (GET_MODE (op2)) == MODE_VECTOR_INT;
28412 1590 : struct expand_vec_perm_d d;
28413 1590 : bool full_interleave = true;
28414 1590 : bool uns_p = code != ASHIFTRT;
28415 1590 : bool ok;
28416 1590 : int i;
28417 :
28418 1590 : if (CONST_INT_P (op2)
28419 405 : && (code == ASHIFT || code == LSHIFTRT || code == ASHIFTRT)
28420 1995 : && ix86_expand_vec_shift_qihi_constant (code, dest, op1, op2))
28421 717 : return;
28422 :
28423 1185 : if (ix86_expand_vecop_qihi2 (code, dest, op1, op2))
28424 : return;
28425 :
28426 873 : switch (qimode)
28427 : {
28428 : case E_V16QImode:
28429 : himode = V8HImode;
28430 : break;
28431 44 : case E_V32QImode:
28432 44 : himode = V16HImode;
28433 44 : break;
28434 10 : case E_V64QImode:
28435 10 : himode = V32HImode;
28436 10 : break;
28437 0 : default:
28438 0 : gcc_unreachable ();
28439 : }
28440 :
28441 873 : switch (code)
28442 : {
28443 819 : case MULT:
28444 819 : gcc_assert (op2vec);
28445 : /* Unpack data such that we've got a source byte in each low byte of
28446 : each word. We don't care what goes into the high byte of each word.
28447 : Rather than trying to get zero in there, most convenient is to let
28448 : it be a copy of the low byte. */
28449 819 : switch (qimode)
28450 : {
28451 : case E_V16QImode:
28452 : gen_il = gen_vec_interleave_lowv16qi;
28453 : gen_ih = gen_vec_interleave_highv16qi;
28454 : break;
28455 44 : case E_V32QImode:
28456 44 : gen_il = gen_avx2_interleave_lowv32qi;
28457 44 : gen_ih = gen_avx2_interleave_highv32qi;
28458 44 : full_interleave = false;
28459 44 : break;
28460 8 : case E_V64QImode:
28461 8 : gen_il = gen_avx512bw_interleave_lowv64qi;
28462 8 : gen_ih = gen_avx512bw_interleave_highv64qi;
28463 8 : full_interleave = false;
28464 8 : break;
28465 : default:
28466 : gcc_unreachable ();
28467 : }
28468 :
28469 819 : op2_l = gen_reg_rtx (qimode);
28470 819 : op2_h = gen_reg_rtx (qimode);
28471 819 : emit_insn (gen_il (op2_l, op2, op2));
28472 819 : emit_insn (gen_ih (op2_h, op2, op2));
28473 :
28474 819 : op1_l = gen_reg_rtx (qimode);
28475 819 : op1_h = gen_reg_rtx (qimode);
28476 819 : emit_insn (gen_il (op1_l, op1, op1));
28477 819 : emit_insn (gen_ih (op1_h, op1, op1));
28478 819 : break;
28479 :
28480 54 : case ASHIFT:
28481 54 : case ASHIFTRT:
28482 54 : case LSHIFTRT:
28483 54 : op1_l = gen_reg_rtx (himode);
28484 54 : op1_h = gen_reg_rtx (himode);
28485 54 : ix86_expand_sse_unpack (op1_l, op1, uns_p, false);
28486 54 : ix86_expand_sse_unpack (op1_h, op1, uns_p, true);
28487 : /* vashr/vlshr/vashl */
28488 54 : if (op2vec)
28489 : {
28490 2 : rtx tmp = force_reg (qimode, op2);
28491 2 : op2_l = gen_reg_rtx (himode);
28492 2 : op2_h = gen_reg_rtx (himode);
28493 2 : ix86_expand_sse_unpack (op2_l, tmp, uns_p, false);
28494 2 : ix86_expand_sse_unpack (op2_h, tmp, uns_p, true);
28495 : }
28496 : else
28497 : op2_l = op2_h = op2;
28498 :
28499 : break;
28500 0 : default:
28501 0 : gcc_unreachable ();
28502 : }
28503 :
28504 873 : if (code != MULT && op2vec)
28505 : {
28506 : /* Expand vashr/vlshr/vashl. */
28507 2 : res_l = gen_reg_rtx (himode);
28508 2 : res_h = gen_reg_rtx (himode);
28509 2 : emit_insn (gen_rtx_SET (res_l,
28510 : simplify_gen_binary (code, himode,
28511 : op1_l, op2_l)));
28512 2 : emit_insn (gen_rtx_SET (res_h,
28513 : simplify_gen_binary (code, himode,
28514 : op1_h, op2_h)));
28515 : }
28516 : else
28517 : {
28518 : /* Expand mult/ashr/lshr/ashl. */
28519 871 : res_l = expand_simple_binop (himode, code, op1_l, op2_l, NULL_RTX,
28520 : 1, OPTAB_DIRECT);
28521 871 : res_h = expand_simple_binop (himode, code, op1_h, op2_h, NULL_RTX,
28522 : 1, OPTAB_DIRECT);
28523 : }
28524 :
28525 873 : gcc_assert (res_l && res_h);
28526 :
28527 : /* Merge the data back into the right place. */
28528 873 : d.target = dest;
28529 873 : d.op0 = gen_lowpart (qimode, res_l);
28530 873 : d.op1 = gen_lowpart (qimode, res_h);
28531 873 : d.vmode = qimode;
28532 873 : d.nelt = GET_MODE_NUNITS (qimode);
28533 873 : d.one_operand_p = false;
28534 873 : d.testing_p = false;
28535 :
28536 873 : if (full_interleave)
28537 : {
28538 : /* We used the full interleave, the desired
28539 : results are in the even elements. */
28540 14053 : for (i = 0; i < d.nelt; ++i)
28541 13232 : d.perm[i] = i * 2;
28542 : }
28543 : else
28544 : {
28545 : /* For AVX, the interleave used above was not cross-lane. So the
28546 : extraction is evens but with the second and third quarter swapped.
28547 : Happily, that is even one insn shorter than even extraction.
28548 : For AVX512BW we have 4 lanes. We extract evens from within a lane,
28549 : always first from the first and then from the second source operand,
28550 : the index bits above the low 4 bits remains the same.
28551 : Thus, for d.nelt == 32 we want permutation
28552 : 0,2,4,..14, 32,34,36,..46, 16,18,20,..30, 48,50,52,..62
28553 : and for d.nelt == 64 we want permutation
28554 : 0,2,4,..14, 64,66,68,..78, 16,18,20,..30, 80,82,84,..94,
28555 : 32,34,36,..46, 96,98,100,..110, 48,50,52,..62, 112,114,116,..126. */
28556 1972 : for (i = 0; i < d.nelt; ++i)
28557 2880 : d.perm[i] = ((i * 2) & 14) + ((i & 8) ? d.nelt : 0) + (i & ~15);
28558 : }
28559 :
28560 873 : ok = ix86_expand_vec_perm_const_1 (&d);
28561 873 : gcc_assert (ok);
28562 : }
28563 :
28564 : /* Helper function of ix86_expand_mul_widen_evenodd. Return true
28565 : if op is CONST_VECTOR with all odd elements equal to their
28566 : preceding element. */
28567 :
28568 : static bool
28569 9132 : const_vector_equal_evenodd_p (rtx op)
28570 : {
28571 9132 : machine_mode mode = GET_MODE (op);
28572 9132 : int i, nunits = GET_MODE_NUNITS (mode);
28573 9132 : if (!CONST_VECTOR_P (op)
28574 9132 : || nunits != CONST_VECTOR_NUNITS (op))
28575 : return false;
28576 3642 : for (i = 0; i < nunits; i += 2)
28577 2933 : if (CONST_VECTOR_ELT (op, i) != CONST_VECTOR_ELT (op, i + 1))
28578 : return false;
28579 : return true;
28580 : }
28581 :
28582 : void
28583 9253 : ix86_expand_mul_widen_evenodd (rtx dest, rtx op1, rtx op2,
28584 : bool uns_p, bool odd_p)
28585 : {
28586 9253 : machine_mode mode = GET_MODE (op1);
28587 9253 : machine_mode wmode = GET_MODE (dest);
28588 9253 : rtx x;
28589 9253 : rtx orig_op1 = op1, orig_op2 = op2;
28590 :
28591 9253 : if (!nonimmediate_operand (op1, mode))
28592 0 : op1 = force_reg (mode, op1);
28593 9253 : if (!nonimmediate_operand (op2, mode))
28594 3414 : op2 = force_reg (mode, op2);
28595 :
28596 : /* We only play even/odd games with vectors of SImode. */
28597 9253 : gcc_assert (mode == V4SImode || mode == V8SImode || mode == V16SImode);
28598 :
28599 : /* If we're looking for the odd results, shift those members down to
28600 : the even slots. For some cpus this is faster than a PSHUFD. */
28601 9253 : if (odd_p)
28602 : {
28603 : /* For XOP use vpmacsdqh, but only for smult, as it is only
28604 : signed. */
28605 4584 : if (TARGET_XOP && mode == V4SImode && !uns_p)
28606 : {
28607 18 : x = force_reg (wmode, CONST0_RTX (wmode));
28608 18 : emit_insn (gen_xop_pmacsdqh (dest, op1, op2, x));
28609 18 : return;
28610 : }
28611 :
28612 9132 : x = GEN_INT (GET_MODE_UNIT_BITSIZE (mode));
28613 4566 : if (!const_vector_equal_evenodd_p (orig_op1))
28614 4566 : op1 = expand_binop (wmode, lshr_optab, gen_lowpart (wmode, op1),
28615 : x, NULL, 1, OPTAB_DIRECT);
28616 4566 : if (!const_vector_equal_evenodd_p (orig_op2))
28617 3857 : op2 = expand_binop (wmode, lshr_optab, gen_lowpart (wmode, op2),
28618 : x, NULL, 1, OPTAB_DIRECT);
28619 4566 : op1 = gen_lowpart (mode, op1);
28620 4566 : op2 = gen_lowpart (mode, op2);
28621 : }
28622 :
28623 9235 : if (mode == V16SImode)
28624 : {
28625 2 : if (uns_p)
28626 0 : x = gen_vec_widen_umult_even_v16si (dest, op1, op2);
28627 : else
28628 2 : x = gen_vec_widen_smult_even_v16si (dest, op1, op2);
28629 : }
28630 9233 : else if (mode == V8SImode)
28631 : {
28632 139 : if (uns_p)
28633 59 : x = gen_vec_widen_umult_even_v8si (dest, op1, op2);
28634 : else
28635 80 : x = gen_vec_widen_smult_even_v8si (dest, op1, op2);
28636 : }
28637 9094 : else if (uns_p)
28638 7991 : x = gen_vec_widen_umult_even_v4si (dest, op1, op2);
28639 1103 : else if (TARGET_SSE4_1)
28640 367 : x = gen_sse4_1_mulv2siv2di3 (dest, op1, op2);
28641 : else
28642 : {
28643 736 : rtx s1, s2, t0, t1, t2;
28644 :
28645 : /* The easiest way to implement this without PMULDQ is to go through
28646 : the motions as if we are performing a full 64-bit multiply. With
28647 : the exception that we need to do less shuffling of the elements. */
28648 :
28649 : /* Compute the sign-extension, aka highparts, of the two operands. */
28650 736 : s1 = ix86_expand_sse_cmp (gen_reg_rtx (mode), GT, CONST0_RTX (mode),
28651 : op1, pc_rtx, pc_rtx);
28652 736 : s2 = ix86_expand_sse_cmp (gen_reg_rtx (mode), GT, CONST0_RTX (mode),
28653 : op2, pc_rtx, pc_rtx);
28654 :
28655 : /* Multiply LO(A) * HI(B), and vice-versa. */
28656 736 : t1 = gen_reg_rtx (wmode);
28657 736 : t2 = gen_reg_rtx (wmode);
28658 736 : emit_insn (gen_vec_widen_umult_even_v4si (t1, s1, op2));
28659 736 : emit_insn (gen_vec_widen_umult_even_v4si (t2, s2, op1));
28660 :
28661 : /* Multiply LO(A) * LO(B). */
28662 736 : t0 = gen_reg_rtx (wmode);
28663 736 : emit_insn (gen_vec_widen_umult_even_v4si (t0, op1, op2));
28664 :
28665 : /* Combine and shift the highparts into place. */
28666 736 : t1 = expand_binop (wmode, add_optab, t1, t2, t1, 1, OPTAB_DIRECT);
28667 736 : t1 = expand_binop (wmode, ashl_optab, t1, GEN_INT (32), t1,
28668 : 1, OPTAB_DIRECT);
28669 :
28670 : /* Combine high and low parts. */
28671 736 : force_expand_binop (wmode, add_optab, t0, t1, dest, 1, OPTAB_DIRECT);
28672 736 : return;
28673 : }
28674 8499 : emit_insn (x);
28675 : }
28676 :
28677 : void
28678 1011 : ix86_expand_mul_widen_hilo (rtx dest, rtx op1, rtx op2,
28679 : bool uns_p, bool high_p)
28680 : {
28681 1011 : machine_mode wmode = GET_MODE (dest);
28682 1011 : machine_mode mode = GET_MODE (op1);
28683 1011 : rtx t1, t2, t3, t4, mask;
28684 :
28685 1011 : switch (mode)
28686 : {
28687 299 : case E_V4SImode:
28688 299 : t1 = gen_reg_rtx (mode);
28689 299 : t2 = gen_reg_rtx (mode);
28690 299 : if (TARGET_XOP && !uns_p)
28691 : {
28692 : /* With XOP, we have pmacsdqh, aka mul_widen_odd. In this case,
28693 : shuffle the elements once so that all elements are in the right
28694 : place for immediate use: { A C B D }. */
28695 33 : emit_insn (gen_sse2_pshufd_1 (t1, op1, const0_rtx, const2_rtx,
28696 : const1_rtx, GEN_INT (3)));
28697 33 : emit_insn (gen_sse2_pshufd_1 (t2, op2, const0_rtx, const2_rtx,
28698 : const1_rtx, GEN_INT (3)));
28699 : }
28700 : else
28701 : {
28702 : /* Put the elements into place for the multiply. */
28703 266 : ix86_expand_vec_interleave (t1, op1, op1, high_p);
28704 266 : ix86_expand_vec_interleave (t2, op2, op2, high_p);
28705 266 : high_p = false;
28706 : }
28707 299 : ix86_expand_mul_widen_evenodd (dest, t1, t2, uns_p, high_p);
28708 299 : break;
28709 :
28710 70 : case E_V8SImode:
28711 : /* Shuffle the elements between the lanes. After this we
28712 : have { A B E F | C D G H } for each operand. */
28713 70 : t1 = gen_reg_rtx (V4DImode);
28714 70 : t2 = gen_reg_rtx (V4DImode);
28715 70 : emit_insn (gen_avx2_permv4di_1 (t1, gen_lowpart (V4DImode, op1),
28716 : const0_rtx, const2_rtx,
28717 : const1_rtx, GEN_INT (3)));
28718 70 : emit_insn (gen_avx2_permv4di_1 (t2, gen_lowpart (V4DImode, op2),
28719 : const0_rtx, const2_rtx,
28720 : const1_rtx, GEN_INT (3)));
28721 :
28722 : /* Shuffle the elements within the lanes. After this we
28723 : have { A A B B | C C D D } or { E E F F | G G H H }. */
28724 70 : t3 = gen_reg_rtx (V8SImode);
28725 70 : t4 = gen_reg_rtx (V8SImode);
28726 105 : mask = GEN_INT (high_p
28727 : ? 2 + (2 << 2) + (3 << 4) + (3 << 6)
28728 : : 0 + (0 << 2) + (1 << 4) + (1 << 6));
28729 70 : emit_insn (gen_avx2_pshufdv3 (t3, gen_lowpart (V8SImode, t1), mask));
28730 70 : emit_insn (gen_avx2_pshufdv3 (t4, gen_lowpart (V8SImode, t2), mask));
28731 :
28732 70 : ix86_expand_mul_widen_evenodd (dest, t3, t4, uns_p, false);
28733 70 : break;
28734 :
28735 412 : case E_V8HImode:
28736 412 : case E_V16HImode:
28737 412 : t1 = expand_binop (mode, smul_optab, op1, op2, NULL_RTX,
28738 : uns_p, OPTAB_DIRECT);
28739 662 : t2 = expand_binop (mode,
28740 : uns_p ? umul_highpart_optab : smul_highpart_optab,
28741 : op1, op2, NULL_RTX, uns_p, OPTAB_DIRECT);
28742 412 : gcc_assert (t1 && t2);
28743 :
28744 412 : t3 = gen_reg_rtx (mode);
28745 412 : ix86_expand_vec_interleave (t3, t1, t2, high_p);
28746 412 : emit_move_insn (dest, gen_lowpart (wmode, t3));
28747 412 : break;
28748 :
28749 230 : case E_V16QImode:
28750 230 : case E_V32QImode:
28751 230 : case E_V32HImode:
28752 230 : case E_V16SImode:
28753 230 : case E_V64QImode:
28754 230 : t1 = gen_reg_rtx (wmode);
28755 230 : t2 = gen_reg_rtx (wmode);
28756 230 : ix86_expand_sse_unpack (t1, op1, uns_p, high_p);
28757 230 : ix86_expand_sse_unpack (t2, op2, uns_p, high_p);
28758 :
28759 230 : emit_insn (gen_rtx_SET (dest, gen_rtx_MULT (wmode, t1, t2)));
28760 230 : break;
28761 :
28762 0 : default:
28763 0 : gcc_unreachable ();
28764 : }
28765 1011 : }
28766 :
28767 : void
28768 3829 : ix86_expand_sse2_mulv4si3 (rtx op0, rtx op1, rtx op2)
28769 : {
28770 3829 : rtx res_1, res_2, res_3, res_4;
28771 :
28772 3829 : res_1 = gen_reg_rtx (V4SImode);
28773 3829 : res_2 = gen_reg_rtx (V4SImode);
28774 3829 : res_3 = gen_reg_rtx (V2DImode);
28775 3829 : res_4 = gen_reg_rtx (V2DImode);
28776 3829 : ix86_expand_mul_widen_evenodd (res_3, op1, op2, true, false);
28777 3829 : ix86_expand_mul_widen_evenodd (res_4, op1, op2, true, true);
28778 :
28779 : /* Move the results in element 2 down to element 1; we don't care
28780 : what goes in elements 2 and 3. Then we can merge the parts
28781 : back together with an interleave.
28782 :
28783 : Note that two other sequences were tried:
28784 : (1) Use interleaves at the start instead of psrldq, which allows
28785 : us to use a single shufps to merge things back at the end.
28786 : (2) Use shufps here to combine the two vectors, then pshufd to
28787 : put the elements in the correct order.
28788 : In both cases the cost of the reformatting stall was too high
28789 : and the overall sequence slower. */
28790 :
28791 3829 : emit_insn (gen_sse2_pshufd_1 (res_1, gen_lowpart (V4SImode, res_3),
28792 : const0_rtx, const2_rtx,
28793 : const0_rtx, const0_rtx));
28794 3829 : emit_insn (gen_sse2_pshufd_1 (res_2, gen_lowpart (V4SImode, res_4),
28795 : const0_rtx, const2_rtx,
28796 : const0_rtx, const0_rtx));
28797 3829 : rtx_insn *insn = emit_insn (gen_vec_interleave_lowv4si (op0, res_1, res_2));
28798 :
28799 3829 : set_unique_reg_note (insn, REG_EQUAL, gen_rtx_MULT (V4SImode, op1, op2));
28800 3829 : }
28801 :
28802 : void
28803 612 : ix86_expand_sse2_mulvxdi3 (rtx op0, rtx op1, rtx op2)
28804 : {
28805 612 : machine_mode mode = GET_MODE (op0);
28806 612 : rtx t1, t2, t3, t4, t5, t6;
28807 :
28808 612 : if (TARGET_AVX512DQ && mode == V8DImode)
28809 32 : emit_insn (gen_avx512dq_mulv8di3 (op0, op1, op2));
28810 580 : else if (TARGET_AVX512DQ && TARGET_AVX512VL && mode == V4DImode)
28811 32 : emit_insn (gen_avx512dq_mulv4di3 (op0, op1, op2));
28812 548 : else if (TARGET_AVX512DQ && TARGET_AVX512VL && mode == V2DImode)
28813 37 : emit_insn (gen_avx512dq_mulv2di3 (op0, op1, op2));
28814 511 : else if (TARGET_XOP && mode == V2DImode)
28815 : {
28816 : /* op1: A,B,C,D, op2: E,F,G,H */
28817 2 : op1 = gen_lowpart (V4SImode, op1);
28818 2 : op2 = gen_lowpart (V4SImode, op2);
28819 :
28820 2 : t1 = gen_reg_rtx (V4SImode);
28821 2 : t2 = gen_reg_rtx (V4SImode);
28822 2 : t3 = gen_reg_rtx (V2DImode);
28823 2 : t4 = gen_reg_rtx (V2DImode);
28824 :
28825 : /* t1: B,A,D,C */
28826 2 : emit_insn (gen_sse2_pshufd_1 (t1, op1,
28827 : GEN_INT (1),
28828 : GEN_INT (0),
28829 : GEN_INT (3),
28830 : GEN_INT (2)));
28831 :
28832 : /* t2: (B*E),(A*F),(D*G),(C*H) */
28833 2 : emit_insn (gen_mulv4si3 (t2, t1, op2));
28834 :
28835 : /* t3: (B*E)+(A*F), (D*G)+(C*H) */
28836 2 : emit_insn (gen_xop_phadddq (t3, t2));
28837 :
28838 : /* t4: ((B*E)+(A*F))<<32, ((D*G)+(C*H))<<32 */
28839 2 : emit_insn (gen_ashlv2di3 (t4, t3, GEN_INT (32)));
28840 :
28841 : /* Multiply lower parts and add all */
28842 2 : t5 = gen_reg_rtx (V2DImode);
28843 2 : emit_insn (gen_vec_widen_umult_even_v4si (t5,
28844 2 : gen_lowpart (V4SImode, op1),
28845 2 : gen_lowpart (V4SImode, op2)));
28846 2 : force_expand_binop (mode, add_optab, t5, t4, op0, 1, OPTAB_DIRECT);
28847 : }
28848 : else
28849 : {
28850 509 : machine_mode nmode;
28851 509 : rtx (*umul) (rtx, rtx, rtx);
28852 :
28853 509 : if (mode == V2DImode)
28854 : {
28855 : umul = gen_vec_widen_umult_even_v4si;
28856 : nmode = V4SImode;
28857 : }
28858 319 : else if (mode == V4DImode)
28859 : {
28860 : umul = gen_vec_widen_umult_even_v8si;
28861 : nmode = V8SImode;
28862 : }
28863 116 : else if (mode == V8DImode)
28864 : {
28865 : umul = gen_vec_widen_umult_even_v16si;
28866 : nmode = V16SImode;
28867 : }
28868 : else
28869 0 : gcc_unreachable ();
28870 :
28871 :
28872 : /* Multiply low parts. */
28873 509 : t1 = gen_reg_rtx (mode);
28874 509 : emit_insn (umul (t1, gen_lowpart (nmode, op1), gen_lowpart (nmode, op2)));
28875 :
28876 : /* Shift input vectors right 32 bits so we can multiply high parts. */
28877 509 : t6 = GEN_INT (32);
28878 509 : t2 = expand_binop (mode, lshr_optab, op1, t6, NULL, 1, OPTAB_DIRECT);
28879 509 : t3 = expand_binop (mode, lshr_optab, op2, t6, NULL, 1, OPTAB_DIRECT);
28880 :
28881 : /* Multiply high parts by low parts. */
28882 509 : t4 = gen_reg_rtx (mode);
28883 509 : t5 = gen_reg_rtx (mode);
28884 509 : emit_insn (umul (t4, gen_lowpart (nmode, t2), gen_lowpart (nmode, op2)));
28885 509 : emit_insn (umul (t5, gen_lowpart (nmode, t3), gen_lowpart (nmode, op1)));
28886 :
28887 : /* Combine and shift the highparts back. */
28888 509 : t4 = expand_binop (mode, add_optab, t4, t5, t4, 1, OPTAB_DIRECT);
28889 509 : t4 = expand_binop (mode, ashl_optab, t4, t6, t4, 1, OPTAB_DIRECT);
28890 :
28891 : /* Combine high and low parts. */
28892 509 : force_expand_binop (mode, add_optab, t1, t4, op0, 1, OPTAB_DIRECT);
28893 : }
28894 :
28895 612 : set_unique_reg_note (get_last_insn (), REG_EQUAL,
28896 : gen_rtx_MULT (mode, op1, op2));
28897 612 : }
28898 :
28899 : void
28900 30 : ix86_expand_umulvndi_highpart (rtx op0, rtx op1, rtx op2)
28901 : {
28902 : /* The following sequence can be used with SSE2. */
28903 30 : machine_mode mode = GET_MODE (op0);
28904 30 : rtx (*umul) (rtx, rtx, rtx);
28905 30 : machine_mode nmode;
28906 30 : if (mode == V2DImode)
28907 : {
28908 : umul = gen_vec_widen_umult_even_v4si;
28909 : nmode = V4SImode;
28910 : }
28911 : else if (mode == V4DImode)
28912 : {
28913 : umul = gen_vec_widen_umult_even_v8si;
28914 : nmode = V8SImode;
28915 : }
28916 : else if (mode == V8DImode)
28917 : {
28918 : umul = gen_vec_widen_umult_even_v16si;
28919 : nmode = V16SImode;
28920 : }
28921 : else
28922 0 : gcc_unreachable ();
28923 :
28924 30 : rtx t32 = GEN_INT (32);
28925 :
28926 : /* Shift input vectors right 32 bits so we can get high parts of operands. */
28927 30 : rtx op1_hi = expand_binop (mode, lshr_optab, op1, t32, NULL, 1, OPTAB_DIRECT);
28928 30 : rtx op2_hi = expand_binop (mode, lshr_optab, op2, t32, NULL, 1, OPTAB_DIRECT);
28929 :
28930 : /* Multiply low parts. */
28931 30 : rtx p0 = gen_reg_rtx (mode);
28932 30 : emit_insn (umul (p0, gen_lowpart (nmode, op1), gen_lowpart (nmode, op2)));
28933 :
28934 : /* Multiply high parts by low parts. */
28935 30 : rtx p1 = gen_reg_rtx (mode);
28936 30 : emit_insn (umul (p1, gen_lowpart (nmode, op1_hi), gen_lowpart (nmode, op2)));
28937 30 : rtx p2 = gen_reg_rtx (mode);
28938 30 : emit_insn (umul (p2, gen_lowpart (nmode, op1), gen_lowpart (nmode, op2_hi)));
28939 :
28940 : /* Multiply high parts. */
28941 30 : rtx p3 = gen_reg_rtx (mode);
28942 30 : emit_insn (umul (p3, gen_lowpart (nmode, op1_hi),
28943 30 : gen_lowpart (nmode, op2_hi)));
28944 :
28945 : /* Fold the carry in once cross product at a time.
28946 : t = (p0 >> 32) + p1 */
28947 30 : rtx p0_hi = expand_binop (mode, lshr_optab, p0, t32, NULL, 1, OPTAB_DIRECT);
28948 30 : rtx t = expand_binop (mode, add_optab, p0_hi, p1, NULL, 1, OPTAB_DIRECT);
28949 :
28950 : /* Extract low 32 bits of products by masking with 0xffffffff. */
28951 30 : rtx mask = gen_reg_rtx (mode);
28952 30 : emit_insn (gen_rtx_SET (mask, CONSTM1_RTX (mode)));
28953 30 : mask = expand_binop (mode, lshr_optab, mask, t32, NULL, 1, OPTAB_DIRECT);
28954 :
28955 : /* u = (t & MASK) + p2 */
28956 30 : rtx u = expand_binop (mode, and_optab, t, mask, NULL, 1, OPTAB_DIRECT);
28957 30 : u = expand_binop (mode, add_optab, u, p2, NULL, 1, OPTAB_DIRECT);
28958 :
28959 : /* high = p3 + (t >> 32) + (u >> 32) */
28960 30 : rtx t_hi = expand_binop (mode, lshr_optab, t, t32, NULL, 1, OPTAB_DIRECT);
28961 30 : rtx u_hi = expand_binop (mode, lshr_optab, u, t32, NULL, 1, OPTAB_DIRECT);
28962 30 : rtx high = expand_binop (mode, add_optab, p3, t_hi, NULL, 1, OPTAB_DIRECT);
28963 30 : force_expand_binop (mode, add_optab, high, u_hi, op0, 1, OPTAB_DIRECT);
28964 30 : }
28965 :
28966 : /* Return 1 if control transfer instruction INSN
28967 : should be encoded with notrack prefix. */
28968 :
28969 : bool
28970 15216135 : ix86_notrack_prefixed_insn_p (rtx_insn *insn)
28971 : {
28972 15216135 : if (!insn || !((flag_cf_protection & CF_BRANCH)))
28973 : return false;
28974 :
28975 4117626 : if (CALL_P (insn))
28976 : {
28977 1492206 : rtx call = get_call_rtx_from (insn);
28978 1492206 : gcc_assert (call != NULL_RTX);
28979 1492206 : rtx addr = XEXP (call, 0);
28980 :
28981 : /* Do not emit 'notrack' if it's not an indirect call. */
28982 1492206 : if (MEM_P (addr)
28983 1492206 : && SYMBOL_REF_P (XEXP (addr, 0)))
28984 : return false;
28985 : else
28986 65987 : return find_reg_note (insn, REG_CALL_NOCF_CHECK, 0);
28987 : }
28988 :
28989 2625420 : if (JUMP_P (insn) && !flag_cet_switch)
28990 : {
28991 2611766 : rtx target = JUMP_LABEL (insn);
28992 2611766 : if (target == NULL_RTX || ANY_RETURN_P (target))
28993 : return false;
28994 :
28995 : /* Check the jump is a switch table. */
28996 2611728 : rtx_insn *label = as_a<rtx_insn *> (target);
28997 2611728 : rtx_insn *table = next_insn (label);
28998 2611728 : if (table == NULL_RTX || !JUMP_TABLE_DATA_P (table))
28999 : return false;
29000 : else
29001 5474 : return true;
29002 : }
29003 : return false;
29004 : }
29005 :
29006 : /* Calculate integer abs() using only SSE2 instructions. */
29007 :
29008 : void
29009 653 : ix86_expand_sse2_abs (rtx target, rtx input)
29010 : {
29011 653 : machine_mode mode = GET_MODE (target);
29012 653 : rtx tmp0, tmp1, x;
29013 :
29014 653 : switch (mode)
29015 : {
29016 50 : case E_V2DImode:
29017 50 : case E_V4DImode:
29018 : /* For 64-bit signed integer X, with SSE4.2 use
29019 : pxor t0, t0; pcmpgtq X, t0; pxor t0, X; psubq t0, X.
29020 : Otherwise handle it similarly to V4SImode, except use 64 as W instead of
29021 : 32 and use logical instead of arithmetic right shift (which is
29022 : unimplemented) and subtract. */
29023 50 : if (TARGET_SSE4_2)
29024 : {
29025 9 : tmp0 = gen_reg_rtx (mode);
29026 9 : tmp1 = gen_reg_rtx (mode);
29027 9 : emit_move_insn (tmp1, CONST0_RTX (mode));
29028 9 : if (mode == E_V2DImode)
29029 6 : emit_insn (gen_sse4_2_gtv2di3 (tmp0, tmp1, input));
29030 : else
29031 3 : emit_insn (gen_avx2_gtv4di3 (tmp0, tmp1, input));
29032 : }
29033 : else
29034 : {
29035 82 : tmp0 = expand_simple_binop (mode, LSHIFTRT, input,
29036 41 : GEN_INT (GET_MODE_UNIT_BITSIZE (mode)
29037 : - 1), NULL, 0, OPTAB_DIRECT);
29038 41 : tmp0 = expand_simple_unop (mode, NEG, tmp0, NULL, false);
29039 : }
29040 :
29041 50 : tmp1 = expand_simple_binop (mode, XOR, tmp0, input,
29042 : NULL, 0, OPTAB_DIRECT);
29043 50 : x = expand_simple_binop (mode, MINUS, tmp1, tmp0,
29044 : target, 0, OPTAB_DIRECT);
29045 50 : break;
29046 :
29047 76 : case E_V4SImode:
29048 : /* For 32-bit signed integer X, the best way to calculate the absolute
29049 : value of X is (((signed) X >> (W-1)) ^ X) - ((signed) X >> (W-1)). */
29050 76 : tmp0 = expand_simple_binop (mode, ASHIFTRT, input,
29051 76 : GEN_INT (GET_MODE_UNIT_BITSIZE (mode) - 1),
29052 : NULL, 0, OPTAB_DIRECT);
29053 76 : tmp1 = expand_simple_binop (mode, XOR, tmp0, input,
29054 : NULL, 0, OPTAB_DIRECT);
29055 76 : x = expand_simple_binop (mode, MINUS, tmp1, tmp0,
29056 : target, 0, OPTAB_DIRECT);
29057 76 : break;
29058 :
29059 97 : case E_V8HImode:
29060 : /* For 16-bit signed integer X, the best way to calculate the absolute
29061 : value of X is max (X, -X), as SSE2 provides the PMAXSW insn. */
29062 97 : tmp0 = expand_unop (mode, neg_optab, input, NULL_RTX, 0);
29063 :
29064 97 : x = expand_simple_binop (mode, SMAX, tmp0, input,
29065 : target, 0, OPTAB_DIRECT);
29066 97 : break;
29067 :
29068 430 : case E_V16QImode:
29069 : /* For 8-bit signed integer X, the best way to calculate the absolute
29070 : value of X is min ((unsigned char) X, (unsigned char) (-X)),
29071 : as SSE2 provides the PMINUB insn. */
29072 430 : tmp0 = expand_unop (mode, neg_optab, input, NULL_RTX, 0);
29073 :
29074 430 : x = expand_simple_binop (V16QImode, UMIN, tmp0, input,
29075 : target, 0, OPTAB_DIRECT);
29076 430 : break;
29077 :
29078 0 : default:
29079 0 : gcc_unreachable ();
29080 : }
29081 :
29082 653 : if (x != target)
29083 0 : emit_move_insn (target, x);
29084 653 : }
29085 :
29086 : /* Expand an extract from a vector register through pextr insn.
29087 : Return true if successful. */
29088 :
29089 : bool
29090 120648 : ix86_expand_pextr (rtx *operands)
29091 : {
29092 120648 : rtx dst = operands[0];
29093 120648 : rtx src = operands[1];
29094 :
29095 120648 : unsigned int size = INTVAL (operands[2]);
29096 120648 : unsigned int pos = INTVAL (operands[3]);
29097 :
29098 120648 : if (SUBREG_P (dst))
29099 : {
29100 : /* Reject non-lowpart subregs. */
29101 77236 : if (SUBREG_BYTE (dst) > 0)
29102 : return false;
29103 77107 : dst = SUBREG_REG (dst);
29104 : }
29105 :
29106 120519 : if (SUBREG_P (src))
29107 : {
29108 34489 : pos += SUBREG_BYTE (src) * BITS_PER_UNIT;
29109 34489 : src = SUBREG_REG (src);
29110 : }
29111 :
29112 120519 : switch (GET_MODE (src))
29113 : {
29114 0 : case E_V16QImode:
29115 0 : case E_V8HImode:
29116 0 : case E_V4SImode:
29117 0 : case E_V2DImode:
29118 0 : case E_V1TImode:
29119 0 : {
29120 0 : machine_mode srcmode, dstmode;
29121 0 : rtx d, pat;
29122 :
29123 0 : if (!int_mode_for_size (size, 0).exists (&dstmode))
29124 0 : return false;
29125 :
29126 0 : switch (dstmode)
29127 : {
29128 0 : case E_QImode:
29129 0 : if (!TARGET_SSE4_1)
29130 : return false;
29131 : srcmode = V16QImode;
29132 : break;
29133 :
29134 0 : case E_HImode:
29135 0 : if (!TARGET_SSE2)
29136 : return false;
29137 : srcmode = V8HImode;
29138 : break;
29139 :
29140 0 : case E_SImode:
29141 0 : if (!TARGET_SSE4_1)
29142 : return false;
29143 : srcmode = V4SImode;
29144 : break;
29145 :
29146 0 : case E_DImode:
29147 0 : gcc_assert (TARGET_64BIT);
29148 0 : if (!TARGET_SSE4_1)
29149 : return false;
29150 : srcmode = V2DImode;
29151 : break;
29152 :
29153 : default:
29154 : return false;
29155 : }
29156 :
29157 : /* Reject extractions from misaligned positions. */
29158 0 : if (pos & (size-1))
29159 : return false;
29160 :
29161 0 : if (GET_MODE (dst) == dstmode)
29162 : d = dst;
29163 : else
29164 0 : d = gen_reg_rtx (dstmode);
29165 :
29166 : /* Construct insn pattern. */
29167 0 : pat = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (pos / size)));
29168 0 : pat = gen_rtx_VEC_SELECT (dstmode, gen_lowpart (srcmode, src), pat);
29169 :
29170 : /* Let the rtl optimizers know about the zero extension performed. */
29171 0 : if (dstmode == QImode || dstmode == HImode)
29172 : {
29173 0 : pat = gen_rtx_ZERO_EXTEND (SImode, pat);
29174 0 : d = gen_lowpart (SImode, d);
29175 : }
29176 :
29177 0 : emit_insn (gen_rtx_SET (d, pat));
29178 :
29179 0 : if (d != dst)
29180 0 : emit_move_insn (dst, gen_lowpart (GET_MODE (dst), d));
29181 : return true;
29182 : }
29183 :
29184 : default:
29185 : return false;
29186 : }
29187 : }
29188 :
29189 : /* Expand an insert into a vector register through pinsr insn.
29190 : Return true if successful. */
29191 :
29192 : bool
29193 110159 : ix86_expand_pinsr (rtx *operands)
29194 : {
29195 110159 : rtx dst = operands[0];
29196 110159 : rtx src = operands[3];
29197 :
29198 110159 : unsigned int size = INTVAL (operands[1]);
29199 110159 : unsigned int pos = INTVAL (operands[2]);
29200 :
29201 110159 : if (SUBREG_P (dst))
29202 : {
29203 63134 : pos += SUBREG_BYTE (dst) * BITS_PER_UNIT;
29204 63134 : dst = SUBREG_REG (dst);
29205 : }
29206 :
29207 110159 : switch (GET_MODE (dst))
29208 : {
29209 20 : case E_V16QImode:
29210 20 : case E_V8HImode:
29211 20 : case E_V4SImode:
29212 20 : case E_V2DImode:
29213 20 : case E_V1TImode:
29214 20 : {
29215 20 : machine_mode srcmode, dstmode;
29216 20 : rtx (*pinsr)(rtx, rtx, rtx, rtx);
29217 20 : rtx d;
29218 :
29219 20 : if (!int_mode_for_size (size, 0).exists (&srcmode))
29220 0 : return false;
29221 :
29222 20 : switch (srcmode)
29223 : {
29224 1 : case E_QImode:
29225 1 : if (!TARGET_SSE4_1)
29226 : return false;
29227 : dstmode = V16QImode;
29228 : pinsr = gen_sse4_1_pinsrb;
29229 : break;
29230 :
29231 5 : case E_HImode:
29232 5 : if (!TARGET_SSE2)
29233 : return false;
29234 : dstmode = V8HImode;
29235 : pinsr = gen_sse2_pinsrw;
29236 : break;
29237 :
29238 14 : case E_SImode:
29239 14 : if (!TARGET_SSE4_1)
29240 : return false;
29241 : dstmode = V4SImode;
29242 : pinsr = gen_sse4_1_pinsrd;
29243 : break;
29244 :
29245 0 : case E_DImode:
29246 0 : gcc_assert (TARGET_64BIT);
29247 0 : if (!TARGET_SSE4_1)
29248 : return false;
29249 : dstmode = V2DImode;
29250 : pinsr = gen_sse4_1_pinsrq;
29251 : break;
29252 :
29253 : default:
29254 : return false;
29255 : }
29256 :
29257 : /* Reject insertions to misaligned positions. */
29258 7 : if (pos & (size-1))
29259 : return false;
29260 :
29261 7 : if (SUBREG_P (src))
29262 : {
29263 7 : unsigned int srcpos = SUBREG_BYTE (src);
29264 :
29265 7 : if (srcpos > 0)
29266 : {
29267 0 : rtx extr_ops[4];
29268 :
29269 0 : extr_ops[0] = gen_reg_rtx (srcmode);
29270 0 : extr_ops[1] = gen_lowpart (srcmode, SUBREG_REG (src));
29271 0 : extr_ops[2] = GEN_INT (size);
29272 0 : extr_ops[3] = GEN_INT (srcpos * BITS_PER_UNIT);
29273 :
29274 0 : if (!ix86_expand_pextr (extr_ops))
29275 0 : return false;
29276 :
29277 0 : src = extr_ops[0];
29278 : }
29279 : else
29280 7 : src = gen_lowpart (srcmode, SUBREG_REG (src));
29281 : }
29282 :
29283 7 : if (GET_MODE (dst) == dstmode)
29284 : d = dst;
29285 : else
29286 7 : d = gen_reg_rtx (dstmode);
29287 :
29288 7 : emit_insn (pinsr (d, gen_lowpart (dstmode, dst),
29289 7 : gen_lowpart (srcmode, src),
29290 7 : GEN_INT (1 << (pos / size))));
29291 7 : if (d != dst)
29292 7 : emit_move_insn (dst, gen_lowpart (GET_MODE (dst), d));
29293 : return true;
29294 : }
29295 :
29296 : default:
29297 : return false;
29298 : }
29299 : }
29300 :
29301 : /* All CPUs prefer to avoid cross-lane operations so perform reductions
29302 : upper against lower halves up to SSE reg size. */
29303 :
29304 : machine_mode
29305 1847 : ix86_split_reduction (machine_mode mode)
29306 : {
29307 : /* Reduce lowpart against highpart until we reach SSE reg width to
29308 : avoid cross-lane operations. */
29309 1847 : switch (mode)
29310 : {
29311 : case E_V8DImode:
29312 : case E_V4DImode:
29313 : return V2DImode;
29314 9 : case E_V16SImode:
29315 9 : case E_V8SImode:
29316 9 : return V4SImode;
29317 8 : case E_V32HImode:
29318 8 : case E_V16HImode:
29319 8 : return V8HImode;
29320 4 : case E_V64QImode:
29321 4 : case E_V32QImode:
29322 4 : return V16QImode;
29323 5 : case E_V16SFmode:
29324 5 : case E_V8SFmode:
29325 5 : return V4SFmode;
29326 16 : case E_V8DFmode:
29327 16 : case E_V4DFmode:
29328 16 : return V2DFmode;
29329 1800 : default:
29330 1800 : return mode;
29331 : }
29332 : }
29333 :
29334 : /* Generate call to __divmoddi4. */
29335 :
29336 : void
29337 892 : ix86_expand_divmod_libfunc (rtx libfunc, machine_mode mode,
29338 : rtx op0, rtx op1,
29339 : rtx *quot_p, rtx *rem_p)
29340 : {
29341 1784 : rtx rem = assign_stack_temp (mode, GET_MODE_SIZE (mode));
29342 :
29343 892 : rtx quot = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
29344 : mode, op0, mode, op1, mode,
29345 892 : XEXP (rem, 0), Pmode);
29346 892 : *quot_p = quot;
29347 892 : *rem_p = rem;
29348 892 : }
29349 :
29350 : void
29351 80 : ix86_expand_atomic_fetch_op_loop (rtx target, rtx mem, rtx val,
29352 : enum rtx_code code, bool after,
29353 : bool doubleword)
29354 : {
29355 80 : rtx old_reg, new_reg, old_mem, success;
29356 80 : machine_mode mode = GET_MODE (target);
29357 80 : rtx_code_label *loop_label = NULL;
29358 :
29359 80 : old_reg = gen_reg_rtx (mode);
29360 80 : new_reg = old_reg;
29361 80 : old_mem = copy_to_reg (mem);
29362 80 : loop_label = gen_label_rtx ();
29363 80 : emit_label (loop_label);
29364 80 : emit_move_insn (old_reg, old_mem);
29365 :
29366 : /* return value for atomic_fetch_op. */
29367 80 : if (!after)
29368 40 : emit_move_insn (target, old_reg);
29369 :
29370 80 : if (code == NOT)
29371 : {
29372 20 : new_reg = expand_simple_binop (mode, AND, new_reg, val, NULL_RTX,
29373 : true, OPTAB_LIB_WIDEN);
29374 20 : new_reg = expand_simple_unop (mode, code, new_reg, NULL_RTX, true);
29375 : }
29376 : else
29377 60 : new_reg = expand_simple_binop (mode, code, new_reg, val, NULL_RTX,
29378 : true, OPTAB_LIB_WIDEN);
29379 :
29380 : /* return value for atomic_op_fetch. */
29381 80 : if (after)
29382 40 : emit_move_insn (target, new_reg);
29383 :
29384 80 : success = NULL_RTX;
29385 :
29386 80 : ix86_expand_cmpxchg_loop (&success, old_mem, mem, old_reg, new_reg,
29387 : gen_int_mode (MEMMODEL_SYNC_SEQ_CST,
29388 : SImode),
29389 : doubleword, loop_label);
29390 80 : }
29391 :
29392 : /* Relax cmpxchg instruction, param loop_label indicates whether
29393 : the instruction should be relaxed with a pause loop. If not,
29394 : it will be relaxed to an atomic load + compare, and skip
29395 : cmpxchg instruction if mem != exp_input. */
29396 :
29397 : void
29398 90 : ix86_expand_cmpxchg_loop (rtx *ptarget_bool, rtx target_val,
29399 : rtx mem, rtx exp_input, rtx new_input,
29400 : rtx mem_model, bool doubleword,
29401 : rtx_code_label *loop_label)
29402 : {
29403 90 : rtx_code_label *cmp_label = NULL;
29404 90 : rtx_code_label *done_label = NULL;
29405 90 : rtx target_bool = NULL_RTX, new_mem = NULL_RTX;
29406 90 : rtx (*gen) (rtx, rtx, rtx, rtx, rtx) = NULL;
29407 90 : rtx (*gendw) (rtx, rtx, rtx, rtx, rtx, rtx) = NULL;
29408 90 : machine_mode mode = GET_MODE (target_val), hmode = mode;
29409 :
29410 90 : if (*ptarget_bool == NULL)
29411 80 : target_bool = gen_reg_rtx (QImode);
29412 : else
29413 : target_bool = *ptarget_bool;
29414 :
29415 90 : cmp_label = gen_label_rtx ();
29416 90 : done_label = gen_label_rtx ();
29417 :
29418 90 : new_mem = gen_reg_rtx (mode);
29419 : /* Load memory first. */
29420 90 : expand_atomic_load (new_mem, mem, MEMMODEL_SEQ_CST);
29421 :
29422 90 : switch (mode)
29423 : {
29424 : case E_TImode:
29425 : gendw = gen_atomic_compare_and_swapti_doubleword;
29426 : hmode = DImode;
29427 : break;
29428 18 : case E_DImode:
29429 18 : if (doubleword)
29430 : {
29431 : gendw = gen_atomic_compare_and_swapdi_doubleword;
29432 : hmode = SImode;
29433 : }
29434 : else
29435 : gen = gen_atomic_compare_and_swapdi_1;
29436 : break;
29437 18 : case E_SImode:
29438 18 : gen = gen_atomic_compare_and_swapsi_1;
29439 18 : break;
29440 18 : case E_HImode:
29441 18 : gen = gen_atomic_compare_and_swaphi_1;
29442 18 : break;
29443 18 : case E_QImode:
29444 18 : gen = gen_atomic_compare_and_swapqi_1;
29445 18 : break;
29446 0 : default:
29447 0 : gcc_unreachable ();
29448 : }
29449 :
29450 : /* Compare mem value with expected value. */
29451 72 : if (doubleword)
29452 : {
29453 18 : rtx low_new_mem = gen_lowpart (hmode, new_mem);
29454 18 : rtx low_exp_input = gen_lowpart (hmode, exp_input);
29455 18 : rtx high_new_mem = gen_highpart (hmode, new_mem);
29456 18 : rtx high_exp_input = gen_highpart (hmode, exp_input);
29457 18 : emit_cmp_and_jump_insns (low_new_mem, low_exp_input, NE, NULL_RTX,
29458 : hmode, 1, cmp_label,
29459 : profile_probability::guessed_never ());
29460 18 : emit_cmp_and_jump_insns (high_new_mem, high_exp_input, NE, NULL_RTX,
29461 : hmode, 1, cmp_label,
29462 : profile_probability::guessed_never ());
29463 : }
29464 : else
29465 72 : emit_cmp_and_jump_insns (new_mem, exp_input, NE, NULL_RTX,
29466 72 : GET_MODE (exp_input), 1, cmp_label,
29467 : profile_probability::guessed_never ());
29468 :
29469 : /* Directly emits cmpxchg here. */
29470 90 : if (doubleword)
29471 36 : emit_insn (gendw (target_val, mem, exp_input,
29472 18 : gen_lowpart (hmode, new_input),
29473 : gen_highpart (hmode, new_input),
29474 : mem_model));
29475 : else
29476 72 : emit_insn (gen (target_val, mem, exp_input, new_input, mem_model));
29477 :
29478 90 : if (!loop_label)
29479 : {
29480 10 : emit_jump_insn (gen_jump (done_label));
29481 10 : emit_barrier ();
29482 10 : emit_label (cmp_label);
29483 10 : emit_move_insn (target_val, new_mem);
29484 10 : emit_label (done_label);
29485 10 : ix86_expand_setcc (target_bool, EQ, gen_rtx_REG (CCZmode, FLAGS_REG),
29486 : const0_rtx);
29487 : }
29488 : else
29489 : {
29490 80 : ix86_expand_setcc (target_bool, EQ, gen_rtx_REG (CCZmode, FLAGS_REG),
29491 : const0_rtx);
29492 80 : emit_cmp_and_jump_insns (target_bool, const0_rtx, EQ, const0_rtx,
29493 80 : GET_MODE (target_bool), 1, loop_label,
29494 : profile_probability::guessed_never ());
29495 80 : emit_jump_insn (gen_jump (done_label));
29496 80 : emit_barrier ();
29497 :
29498 : /* If mem is not expected, pause and loop back. */
29499 80 : emit_label (cmp_label);
29500 80 : emit_move_insn (target_val, new_mem);
29501 80 : emit_insn (gen_pause ());
29502 80 : emit_jump_insn (gen_jump (loop_label));
29503 80 : emit_barrier ();
29504 80 : emit_label (done_label);
29505 : }
29506 :
29507 90 : *ptarget_bool = target_bool;
29508 90 : }
29509 :
29510 : /* Convert a BFmode VAL to SFmode without signaling sNaNs.
29511 : This is done by returning SF SUBREG of ((HI SUBREG) (VAL)) << 16. */
29512 :
29513 : rtx
29514 4136 : ix86_expand_fast_convert_bf_to_sf (rtx val)
29515 : {
29516 4136 : rtx op = gen_lowpart (HImode, val), ret;
29517 4136 : if (CONST_INT_P (op))
29518 : {
29519 613 : ret = simplify_const_unary_operation (FLOAT_EXTEND, SFmode,
29520 : val, BFmode);
29521 613 : if (ret)
29522 : return ret;
29523 : /* FLOAT_EXTEND simplification will fail if VAL is a sNaN. */
29524 1 : ret = gen_reg_rtx (SImode);
29525 1 : emit_move_insn (ret, GEN_INT (INTVAL (op) & 0xffff));
29526 1 : emit_insn (gen_ashlsi3 (ret, ret, GEN_INT (16)));
29527 1 : return gen_lowpart (SFmode, ret);
29528 : }
29529 :
29530 3523 : ret = gen_reg_rtx (SFmode);
29531 3523 : emit_insn (gen_extendbfsf2_1 (ret, force_reg (BFmode, val)));
29532 3523 : return ret;
29533 : }
29534 :
29535 : rtx
29536 65582 : ix86_gen_ccmp_first (rtx_insn **prep_seq, rtx_insn **gen_seq,
29537 : rtx_code code, tree treeop0, tree treeop1)
29538 : {
29539 65582 : if (!TARGET_APX_CCMP)
29540 : return NULL_RTX;
29541 :
29542 65582 : rtx op0, op1, res;
29543 65582 : machine_mode op_mode;
29544 :
29545 65582 : start_sequence ();
29546 65582 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
29547 :
29548 65582 : op_mode = GET_MODE (op0);
29549 65582 : if (op_mode == VOIDmode)
29550 0 : op_mode = GET_MODE (op1);
29551 :
29552 : /* We only supports following scalar comparisons that use just 1
29553 : instruction: DI/SI/HI/QI/XF/DF/SF/HF.
29554 : Unordered/Ordered compare cannot be correctly identified by
29555 : ccmp so they are not supported. */
29556 32771 : if (!(op_mode == DImode || op_mode == SImode
29557 65582 : || op_mode == HImode || op_mode == QImode
29558 32780 : || ((op_mode == XFmode || op_mode == DFmode || op_mode == SFmode)
29559 10 : && (TARGET_80387
29560 1 : || (SSE_FLOAT_MODE_P (op_mode) && TARGET_SSE_MATH)))
29561 0 : || (op_mode == HFmode && TARGET_AVX512FP16))
29562 32811 : || code == ORDERED
29563 32811 : || code == UNORDERED)
29564 : {
29565 32771 : end_sequence ();
29566 32771 : return NULL_RTX;
29567 : }
29568 :
29569 : /* Canonicalize the operands according to mode. */
29570 32811 : if (SCALAR_INT_MODE_P (op_mode))
29571 : {
29572 32802 : if (!nonimmediate_operand (op0, op_mode))
29573 0 : op0 = force_reg (op_mode, op0);
29574 32802 : if (!x86_64_general_operand (op1, op_mode))
29575 0 : op1 = force_reg (op_mode, op1);
29576 : }
29577 : else
29578 : {
29579 : /* op0/op1 can be canonicallized from expand_fp_compare, so
29580 : just adjust the code to make it generate supported fp
29581 : condition. */
29582 9 : if (ix86_fp_compare_code_to_integer (code) == UNKNOWN)
29583 : {
29584 : /* First try to split condition if we don't need to honor
29585 : NaNs, as the ORDERED/UNORDERED check always fall
29586 : through. */
29587 6 : if (!HONOR_NANS (op_mode))
29588 : {
29589 6 : rtx_code first_code;
29590 6 : split_comparison (code, op_mode, &first_code, &code);
29591 : }
29592 : /* Otherwise try to swap the operand order and check if
29593 : the comparison is supported. */
29594 : else
29595 : {
29596 0 : code = swap_condition (code);
29597 0 : std::swap (op0, op1);
29598 : }
29599 :
29600 6 : if (ix86_fp_compare_code_to_integer (code) == UNKNOWN)
29601 : {
29602 0 : end_sequence ();
29603 0 : return NULL_RTX;
29604 : }
29605 : }
29606 : }
29607 :
29608 32811 : *prep_seq = end_sequence ();
29609 :
29610 32811 : start_sequence ();
29611 :
29612 32811 : res = ix86_expand_compare (code, op0, op1);
29613 :
29614 32811 : if (!res)
29615 : {
29616 : end_sequence ();
29617 : return NULL_RTX;
29618 : }
29619 32811 : *gen_seq = end_sequence ();
29620 :
29621 32811 : return res;
29622 : }
29623 :
29624 : rtx
29625 32814 : ix86_gen_ccmp_next (rtx_insn **prep_seq, rtx_insn **gen_seq, rtx prev,
29626 : rtx_code cmp_code, tree treeop0, tree treeop1,
29627 : rtx_code bit_code)
29628 : {
29629 32814 : if (!TARGET_APX_CCMP)
29630 : return NULL_RTX;
29631 :
29632 32814 : rtx op0, op1, target;
29633 32814 : machine_mode op_mode, cmp_mode, cc_mode = CCmode;
29634 32814 : int unsignedp = TYPE_UNSIGNED (TREE_TYPE (treeop0));
29635 32814 : insn_code icode;
29636 32814 : rtx_code prev_code;
29637 32814 : struct expand_operand ops[5];
29638 32814 : int dfv;
29639 :
29640 : /* Exit early for non integer modes to avoid O(n^2) part of expand_operands. */
29641 32814 : cmp_mode = op_mode = TYPE_MODE (TREE_TYPE (treeop0));
29642 :
29643 32814 : if (!(op_mode == DImode || op_mode == SImode || op_mode == HImode
29644 : || op_mode == QImode))
29645 : return NULL_RTX;
29646 :
29647 34 : push_to_sequence (*prep_seq);
29648 34 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
29649 :
29650 34 : icode = code_for_ccmp (op_mode);
29651 :
29652 34 : op0 = prepare_operand (icode, op0, 2, op_mode, cmp_mode, unsignedp);
29653 34 : op1 = prepare_operand (icode, op1, 3, op_mode, cmp_mode, unsignedp);
29654 34 : if (!op0 || !op1)
29655 : {
29656 0 : end_sequence ();
29657 0 : return NULL_RTX;
29658 : }
29659 :
29660 34 : *prep_seq = end_sequence ();
29661 :
29662 34 : target = gen_rtx_REG (cc_mode, FLAGS_REG);
29663 34 : dfv = ix86_get_flags_cc ((rtx_code) cmp_code);
29664 :
29665 34 : prev_code = GET_CODE (prev);
29666 : /* Fixup FP compare code here. */
29667 34 : if (GET_MODE (XEXP (prev, 0)) == CCFPmode)
29668 9 : prev_code = ix86_fp_compare_code_to_integer (prev_code);
29669 :
29670 34 : if (bit_code != AND)
29671 17 : prev_code = reverse_condition (prev_code);
29672 : else
29673 17 : dfv = (int)(dfv ^ 1);
29674 :
29675 34 : prev = gen_rtx_fmt_ee (prev_code, VOIDmode, XEXP (prev, 0),
29676 : const0_rtx);
29677 :
29678 34 : create_fixed_operand (&ops[0], target);
29679 34 : create_fixed_operand (&ops[1], prev);
29680 34 : create_fixed_operand (&ops[2], op0);
29681 34 : create_fixed_operand (&ops[3], op1);
29682 34 : create_fixed_operand (&ops[4], GEN_INT (dfv));
29683 :
29684 34 : push_to_sequence (*gen_seq);
29685 34 : if (!maybe_expand_insn (icode, 5, ops))
29686 : {
29687 0 : end_sequence ();
29688 0 : return NULL_RTX;
29689 : }
29690 :
29691 34 : *gen_seq = end_sequence ();
29692 :
29693 34 : return gen_rtx_fmt_ee ((rtx_code) cmp_code, VOIDmode, target, const0_rtx);
29694 : }
29695 :
29696 : /* Attempt to convert a CONST_VECTOR into a bcst_mem_operand.
29697 : Returns NULL_RTX if X is cannot be expressed as a suitable
29698 : VEC_DUPLICATE in mode MODE. */
29699 :
29700 : static rtx
29701 48 : ix86_gen_bcst_mem (machine_mode mode, rtx x)
29702 : {
29703 48 : if (!TARGET_AVX512F
29704 48 : || !CONST_VECTOR_P (x)
29705 64 : || (!TARGET_AVX512VL && GET_MODE_SIZE (mode) != 64)
29706 147 : || !VALID_BCST_MODE_P (GET_MODE_INNER (mode))
29707 : /* Disallow HFmode broadcast. */
29708 126 : || GET_MODE_SIZE (GET_MODE_INNER (mode)) < 4)
29709 : return NULL_RTX;
29710 :
29711 21 : rtx cst = CONST_VECTOR_ELT (x, 0);
29712 21 : if (!CONST_SCALAR_INT_P (cst)
29713 15 : && !CONST_DOUBLE_P (cst)
29714 0 : && !CONST_FIXED_P (cst))
29715 : return NULL_RTX;
29716 :
29717 21 : int n_elts = GET_MODE_NUNITS (mode);
29718 42 : if (CONST_VECTOR_NUNITS (x) != n_elts)
29719 : return NULL_RTX;
29720 :
29721 150 : for (int i = 1; i < n_elts; i++)
29722 129 : if (!rtx_equal_p (cst, CONST_VECTOR_ELT (x, i)))
29723 : return NULL_RTX;
29724 :
29725 42 : rtx mem = force_const_mem (GET_MODE_INNER (mode), cst);
29726 21 : return gen_rtx_VEC_DUPLICATE (mode, validize_mem (mem));
29727 : }
29728 :
29729 : /* Determine the ternlog immediate index that implements 3-operand
29730 : ternary logic expression OP. This uses and modifies the 3 element
29731 : array ARGS to record and check the leaves, either 3 REGs, or 2 REGs
29732 : and MEM. Returns an index between 0 and 255 for a valid ternlog,
29733 : or -1 if the expression isn't suitable. */
29734 :
29735 : int
29736 7592958 : ix86_ternlog_idx (rtx op, rtx *args)
29737 : {
29738 7592958 : int idx0, idx1;
29739 :
29740 7592958 : if (!op)
29741 : return -1;
29742 :
29743 7592958 : switch (GET_CODE (op))
29744 : {
29745 748676 : case SUBREG:
29746 748676 : if (!register_operand (op, GET_MODE (op)))
29747 : return -1;
29748 : /* FALLTHRU */
29749 :
29750 3746169 : case REG:
29751 3746169 : if (!args[0])
29752 : {
29753 1930205 : args[0] = op;
29754 1930205 : return 0xf0;
29755 : }
29756 1815964 : if (rtx_equal_p (op, args[0]))
29757 : return 0xf0;
29758 1789437 : if (!args[1])
29759 : {
29760 1512563 : args[1] = op;
29761 1512563 : return 0xcc;
29762 : }
29763 276874 : if (rtx_equal_p (op, args[1]))
29764 : return 0xcc;
29765 260376 : if (!args[2])
29766 : {
29767 237153 : args[2] = op;
29768 237153 : return 0xaa;
29769 : }
29770 23223 : if (rtx_equal_p (op, args[2]))
29771 1461 : return 0xaa;
29772 : return -1;
29773 :
29774 19671 : case VEC_DUPLICATE:
29775 19671 : if (!bcst_mem_operand (op, GET_MODE (op)))
29776 : return -1;
29777 308 : goto do_mem_operand;
29778 :
29779 355062 : case MEM:
29780 355062 : if (!memory_operand (op, GET_MODE (op)))
29781 : return -1;
29782 354919 : if (MEM_P (op)
29783 354919 : && MEM_VOLATILE_P (op)
29784 355013 : && !volatile_ok)
29785 : return -1;
29786 : /* FALLTHRU */
29787 :
29788 470864 : case CONST_VECTOR:
29789 470864 : do_mem_operand:
29790 470864 : if (!args[2])
29791 : {
29792 423350 : args[2] = op;
29793 423350 : return 0xaa;
29794 : }
29795 : /* Maximum of one volatile memory reference per expression. */
29796 47514 : if (side_effects_p (op))
29797 : return -1;
29798 47514 : if (rtx_equal_p (op, args[2]))
29799 : return 0xaa;
29800 : /* Check if CONST_VECTOR is the ones-complement of args[2]. */
29801 47463 : if (CONST_VECTOR_P (op)
29802 3655 : && CONST_VECTOR_P (args[2])
29803 47708 : && rtx_equal_p (simplify_const_unary_operation (NOT, GET_MODE (op),
29804 245 : op, GET_MODE (op)),
29805 : args[2]))
29806 : return 0x55;
29807 47276 : if (!args[0])
29808 : {
29809 45452 : args[0] = op;
29810 45452 : return 0xf0;
29811 : }
29812 1824 : if (rtx_equal_p (op, args[0]))
29813 : return 0xf0;
29814 : /* Check if CONST_VECTOR is the ones-complement of args[0]. */
29815 1824 : if (CONST_VECTOR_P (op)
29816 111 : && CONST_VECTOR_P (args[0])
29817 1866 : && rtx_equal_p (simplify_const_unary_operation (NOT, GET_MODE (op),
29818 42 : op, GET_MODE (op)),
29819 : args[0]))
29820 : return 0x0f;
29821 1782 : if (!args[1])
29822 : {
29823 1740 : args[1] = op;
29824 1740 : return 0xcc;
29825 : }
29826 42 : if (rtx_equal_p (op, args[1]))
29827 : return 0xcc;
29828 : /* Check if CONST_VECTOR is the ones-complement of args[1]. */
29829 42 : if (CONST_VECTOR_P (op)
29830 0 : && CONST_VECTOR_P (args[1])
29831 42 : && rtx_equal_p (simplify_const_unary_operation (NOT, GET_MODE (op),
29832 0 : op, GET_MODE (op)),
29833 : args[1]))
29834 0 : return 0x33;
29835 : return -1;
29836 :
29837 181198 : case NOT:
29838 181198 : idx0 = ix86_ternlog_idx (XEXP (op, 0), args);
29839 181198 : return (idx0 >= 0) ? idx0 ^ 0xff : -1;
29840 :
29841 1364388 : case AND:
29842 1364388 : idx0 = ix86_ternlog_idx (XEXP (op, 0), args);
29843 1364388 : if (idx0 < 0)
29844 : return -1;
29845 1122653 : idx1 = ix86_ternlog_idx (XEXP (op, 1), args);
29846 1122653 : return (idx1 >= 0) ? idx0 & idx1 : -1;
29847 :
29848 955519 : case IOR:
29849 955519 : idx0 = ix86_ternlog_idx (XEXP (op, 0), args);
29850 955519 : if (idx0 < 0)
29851 : return -1;
29852 708958 : idx1 = ix86_ternlog_idx (XEXP (op, 1), args);
29853 708958 : return (idx1 >= 0) ? idx0 | idx1 : -1;
29854 :
29855 456814 : case XOR:
29856 456814 : idx0 = ix86_ternlog_idx (XEXP (op, 0), args);
29857 456814 : if (idx0 < 0)
29858 : return -1;
29859 428395 : if (vector_all_ones_operand (XEXP (op, 1), GET_MODE (op)))
29860 6711 : return idx0 ^ 0xff;
29861 421684 : idx1 = ix86_ternlog_idx (XEXP (op, 1), args);
29862 421684 : return (idx1 >= 0) ? idx0 ^ idx1 : -1;
29863 :
29864 7878 : case UNSPEC:
29865 7878 : if (XINT (op, 1) != UNSPEC_VTERNLOG
29866 0 : || XVECLEN (op, 0) != 4
29867 0 : || !CONST_INT_P (XVECEXP (op, 0, 3)))
29868 : return -1;
29869 :
29870 : /* TODO: Handle permuted operands. */
29871 0 : if (ix86_ternlog_idx (XVECEXP (op, 0, 0), args) != 0xf0
29872 0 : || ix86_ternlog_idx (XVECEXP (op, 0, 1), args) != 0xcc
29873 0 : || ix86_ternlog_idx (XVECEXP (op, 0, 2), args) != 0xaa)
29874 : return -1;
29875 0 : return INTVAL (XVECEXP (op, 0, 3));
29876 :
29877 : default:
29878 : return -1;
29879 : }
29880 : }
29881 :
29882 : /* Return TRUE if OP (in mode MODE) is the leaf of a ternary logic
29883 : expression, such as a register or a memory reference. */
29884 :
29885 : bool
29886 3520270 : ix86_ternlog_leaf_p (rtx op, machine_mode mode)
29887 : {
29888 : /* We can't use memory_operand here, as it may return a different
29889 : value before and after reload (for volatile MEMs) which creates
29890 : problems splitting instructions. */
29891 3520270 : return register_operand (op, mode)
29892 740319 : || MEM_P (op)
29893 401118 : || CONST_VECTOR_P (op)
29894 3813675 : || bcst_mem_operand (op, mode);
29895 : }
29896 :
29897 : /* Test whether OP is a 3-operand ternary logic expression suitable
29898 : for use in a ternlog instruction. */
29899 :
29900 : bool
29901 2345732 : ix86_ternlog_operand_p (rtx op)
29902 : {
29903 2345732 : rtx op0, op1;
29904 2345732 : rtx args[3];
29905 :
29906 2345732 : args[0] = NULL_RTX;
29907 2345732 : args[1] = NULL_RTX;
29908 2345732 : args[2] = NULL_RTX;
29909 2345732 : int idx = ix86_ternlog_idx (op, args);
29910 2345732 : if (idx < 0)
29911 : return false;
29912 :
29913 : /* Don't match simple (binary or unary) expressions. */
29914 1905922 : machine_mode mode = GET_MODE (op);
29915 1905922 : switch (GET_CODE (op))
29916 : {
29917 883592 : case AND:
29918 883592 : op0 = XEXP (op, 0);
29919 883592 : op1 = XEXP (op, 1);
29920 :
29921 : /* Prefer pand. */
29922 883592 : if (ix86_ternlog_leaf_p (op0, mode)
29923 883592 : && ix86_ternlog_leaf_p (op1, mode))
29924 : return false;
29925 : /* Prefer pandn. */
29926 110375 : if (GET_CODE (op0) == NOT
29927 74582 : && register_operand (XEXP (op0, 0), mode)
29928 181767 : && ix86_ternlog_leaf_p (op1, mode))
29929 69905 : return false;
29930 : break;
29931 :
29932 621615 : case IOR:
29933 : /* Prefer por. */
29934 621615 : if (ix86_ternlog_leaf_p (XEXP (op, 0), mode)
29935 621615 : && ix86_ternlog_leaf_p (XEXP (op, 1), mode))
29936 462770 : return false;
29937 : break;
29938 :
29939 365080 : case XOR:
29940 365080 : op1 = XEXP (op, 1);
29941 : /* Prefer pxor, or one_cmpl<vmode>2. */
29942 365080 : if (ix86_ternlog_leaf_p (XEXP (op, 0), mode)
29943 365080 : && ix86_ternlog_leaf_p (XEXP (op, 1), mode))
29944 342602 : return false;
29945 : break;
29946 :
29947 : default:
29948 : break;
29949 : }
29950 : return true;
29951 : }
29952 :
29953 : /* Helper function for ix86_expand_ternlog. */
29954 : static rtx
29955 0 : ix86_expand_ternlog_binop (enum rtx_code code, machine_mode mode,
29956 : rtx op0, rtx op1, rtx target)
29957 : {
29958 0 : if (GET_MODE (op0) != mode)
29959 0 : op0 = gen_lowpart (mode, op0);
29960 0 : if (GET_MODE (op1) != mode)
29961 0 : op1 = gen_lowpart (mode, op1);
29962 :
29963 0 : if (CONST_VECTOR_P (op0))
29964 0 : op0 = validize_mem (force_const_mem (mode, op0));
29965 0 : if (CONST_VECTOR_P (op1))
29966 0 : op1 = validize_mem (force_const_mem (mode, op1));
29967 :
29968 0 : if (!register_operand (op0, mode))
29969 : {
29970 0 : if (!register_operand (op1, mode))
29971 : {
29972 : /* We can't use force_reg (op0, mode). */
29973 0 : rtx reg = gen_reg_rtx (mode);
29974 0 : emit_move_insn (reg, op0);
29975 0 : op0 = reg;
29976 : }
29977 : else
29978 : std::swap (op0, op1);
29979 : }
29980 0 : rtx ops[3] = { target, op0, op1 };
29981 0 : ix86_expand_vector_logical_operator (code, mode, ops);
29982 0 : return target;
29983 : }
29984 :
29985 :
29986 : /* Helper function for ix86_expand_ternlog. */
29987 : static rtx
29988 0 : ix86_expand_ternlog_andnot (machine_mode mode, rtx op0, rtx op1, rtx target)
29989 : {
29990 0 : if (GET_MODE (op0) != mode)
29991 0 : op0 = gen_lowpart (mode, op0);
29992 0 : op0 = gen_rtx_NOT (mode, op0);
29993 0 : if (GET_MODE (op1) != mode)
29994 0 : op1 = gen_lowpart (mode, op1);
29995 0 : if (CONST_VECTOR_P (op1))
29996 0 : op1 = validize_mem (force_const_mem (mode, op1));
29997 0 : emit_move_insn (target, gen_rtx_AND (mode, op0, op1));
29998 0 : return target;
29999 : }
30000 :
30001 : /* Expand a 3-operand ternary logic expression. Return TARGET. */
30002 : rtx
30003 2430 : ix86_expand_ternlog (machine_mode mode, rtx op0, rtx op1, rtx op2, int idx,
30004 : rtx target)
30005 : {
30006 2430 : rtx tmp0, tmp1, tmp2;
30007 :
30008 2430 : if (!target)
30009 3 : target = gen_reg_rtx (mode);
30010 :
30011 : /* Canonicalize ternlog index for degenerate (duplicated) operands. */
30012 2430 : if (rtx_equal_p (op0, op1) && rtx_equal_p (op0, op2))
30013 0 : switch (idx & 0x81)
30014 : {
30015 : case 0x00:
30016 : idx = 0x00;
30017 : break;
30018 : case 0x01:
30019 : idx = 0x0f;
30020 : break;
30021 : case 0x80:
30022 : idx = 0xf0;
30023 : break;
30024 : case 0x81:
30025 : idx = 0xff;
30026 : break;
30027 : }
30028 :
30029 2430 : switch (idx & 0xff)
30030 : {
30031 0 : case 0x00:
30032 0 : if ((!op0 || !side_effects_p (op0))
30033 0 : && (!op1 || !side_effects_p (op1))
30034 0 : && (!op2 || !side_effects_p (op2)))
30035 : {
30036 0 : emit_move_insn (target, CONST0_RTX (mode));
30037 0 : return target;
30038 : }
30039 : break;
30040 :
30041 0 : case 0x0a: /* ~a&c */
30042 0 : if ((!op1 || !side_effects_p (op1))
30043 0 : && op0 && register_operand (op0, mode)
30044 0 : && op2 && ix86_ternlog_leaf_p (op2, mode))
30045 0 : return ix86_expand_ternlog_andnot (mode, op0, op2, target);
30046 : break;
30047 :
30048 0 : case 0x0c: /* ~a&b */
30049 0 : if ((!op2 || !side_effects_p (op2))
30050 0 : && op0 && register_operand (op0, mode)
30051 0 : && op1 && ix86_ternlog_leaf_p (op1, mode))
30052 0 : return ix86_expand_ternlog_andnot (mode, op0, op1, target);
30053 : break;
30054 :
30055 82 : case 0x0f: /* ~a */
30056 0 : if ((!op1 || !side_effects_p (op1))
30057 82 : && (!op2 || !side_effects_p (op2))
30058 164 : && op0)
30059 : {
30060 82 : emit_move_insn (target, gen_rtx_XOR (mode, op0, CONSTM1_RTX (mode)));
30061 82 : return target;
30062 : }
30063 : break;
30064 :
30065 0 : case 0x22: /* ~b&c */
30066 0 : if ((!op0 || !side_effects_p (op0))
30067 0 : && op1 && register_operand (op1, mode)
30068 0 : && op2 && ix86_ternlog_leaf_p (op2, mode))
30069 0 : return ix86_expand_ternlog_andnot (mode, op1, op2, target);
30070 : break;
30071 :
30072 0 : case 0x30: /* ~b&a */
30073 0 : if ((!op2 || !side_effects_p (op2))
30074 0 : && op0 && ix86_ternlog_leaf_p (op0, mode)
30075 0 : && op1 && register_operand (op1, mode))
30076 0 : return ix86_expand_ternlog_andnot (mode, op1, op0, target);
30077 : break;
30078 :
30079 0 : case 0x33: /* ~b */
30080 0 : if ((!op0 || !side_effects_p (op0))
30081 0 : && (!op2 || !side_effects_p (op2))
30082 0 : && op1)
30083 : {
30084 0 : emit_move_insn (target, gen_rtx_XOR (mode, op1, CONSTM1_RTX (mode)));
30085 0 : return target;
30086 : }
30087 : break;
30088 :
30089 0 : case 0x3c: /* a^b */
30090 0 : if (op0 && ix86_ternlog_leaf_p (op0, mode)
30091 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
30092 0 : && (!op2 || !side_effects_p (op2)))
30093 0 : return ix86_expand_ternlog_binop (XOR, mode, op0, op1, target);
30094 : break;
30095 :
30096 0 : case 0x44: /* ~c&b */
30097 0 : if ((!op0 || !side_effects_p (op0))
30098 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
30099 0 : && op2 && register_operand (op2, mode))
30100 0 : return ix86_expand_ternlog_andnot (mode, op2, op1, target);
30101 : break;
30102 :
30103 2 : case 0x50: /* ~c&a */
30104 0 : if ((!op1 || !side_effects_p (op1))
30105 2 : && op0 && ix86_ternlog_leaf_p (op0, mode)
30106 4 : && op2 && register_operand (op2, mode))
30107 0 : return ix86_expand_ternlog_andnot (mode, op2, op0, target);
30108 : break;
30109 :
30110 4 : case 0x55: /* ~c */
30111 1 : if ((!op0 || !side_effects_p (op0))
30112 4 : && (!op1 || !side_effects_p (op1))
30113 8 : && op2)
30114 : {
30115 4 : emit_move_insn (target, gen_rtx_XOR (mode, op2, CONSTM1_RTX (mode)));
30116 4 : return target;
30117 : }
30118 : break;
30119 :
30120 0 : case 0x5a: /* a^c */
30121 0 : if (op0 && ix86_ternlog_leaf_p (op0, mode)
30122 0 : && op2 && ix86_ternlog_leaf_p (op2, mode)
30123 0 : && (!op1 || !side_effects_p (op1)))
30124 0 : return ix86_expand_ternlog_binop (XOR, mode, op0, op2, target);
30125 : break;
30126 :
30127 0 : case 0x66: /* b^c */
30128 0 : if ((!op0 || !side_effects_p (op0))
30129 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
30130 0 : && op2 && ix86_ternlog_leaf_p (op2, mode))
30131 0 : return ix86_expand_ternlog_binop (XOR, mode, op1, op2, target);
30132 : break;
30133 :
30134 0 : case 0x88: /* b&c */
30135 0 : if ((!op0 || !side_effects_p (op0))
30136 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
30137 0 : && op2 && ix86_ternlog_leaf_p (op2, mode))
30138 0 : return ix86_expand_ternlog_binop (AND, mode, op1, op2, target);
30139 : break;
30140 :
30141 0 : case 0xa0: /* a&c */
30142 0 : if ((!op1 || !side_effects_p (op1))
30143 0 : && op0 && ix86_ternlog_leaf_p (op0, mode)
30144 0 : && op2 && ix86_ternlog_leaf_p (op2, mode))
30145 0 : return ix86_expand_ternlog_binop (AND, mode, op0, op2, target);
30146 : break;
30147 :
30148 0 : case 0xaa: /* c */
30149 0 : if ((!op0 || !side_effects_p (op0))
30150 0 : && (!op1 || !side_effects_p (op1))
30151 0 : && op2)
30152 : {
30153 0 : if (GET_MODE (op2) != mode)
30154 0 : op2 = gen_lowpart (mode, op2);
30155 0 : emit_move_insn (target, op2);
30156 0 : return target;
30157 : }
30158 : break;
30159 :
30160 0 : case 0xc0: /* a&b */
30161 0 : if (op0 && ix86_ternlog_leaf_p (op0, mode)
30162 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
30163 0 : && (!op2 || !side_effects_p (op2)))
30164 0 : return ix86_expand_ternlog_binop (AND, mode, op0, op1, target);
30165 : break;
30166 :
30167 0 : case 0xcc: /* b */
30168 0 : if ((!op0 || !side_effects_p (op0))
30169 0 : && op1
30170 0 : && (!op2 || !side_effects_p (op2)))
30171 : {
30172 0 : if (GET_MODE (op1) != mode)
30173 0 : op1 = gen_lowpart (mode, op1);
30174 0 : emit_move_insn (target, op1);
30175 0 : return target;
30176 : }
30177 : break;
30178 :
30179 0 : case 0xee: /* b|c */
30180 0 : if ((!op0 || !side_effects_p (op0))
30181 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
30182 0 : && op2 && ix86_ternlog_leaf_p (op2, mode))
30183 0 : return ix86_expand_ternlog_binop (IOR, mode, op1, op2, target);
30184 : break;
30185 :
30186 6 : case 0xf0: /* a */
30187 6 : if (op0
30188 6 : && (!op1 || !side_effects_p (op1))
30189 12 : && (!op2 || !side_effects_p (op2)))
30190 : {
30191 6 : if (GET_MODE (op0) != mode)
30192 0 : op0 = gen_lowpart (mode, op0);
30193 6 : emit_move_insn (target, op0);
30194 6 : return target;
30195 : }
30196 : break;
30197 :
30198 0 : case 0xfa: /* a|c */
30199 0 : if (op0 && ix86_ternlog_leaf_p (op0, mode)
30200 0 : && op2 && ix86_ternlog_leaf_p (op2, mode)
30201 0 : && (!op1 || !side_effects_p (op1)))
30202 0 : return ix86_expand_ternlog_binop (IOR, mode, op0, op2, target);
30203 : break;
30204 :
30205 0 : case 0xfc: /* a|b */
30206 0 : if (op0 && ix86_ternlog_leaf_p (op0, mode)
30207 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
30208 0 : && (!op2 || !side_effects_p (op2)))
30209 0 : return ix86_expand_ternlog_binop (IOR, mode, op0, op1, target);
30210 : break;
30211 :
30212 0 : case 0xff:
30213 0 : if ((!op0 || !side_effects_p (op0))
30214 0 : && (!op1 || !side_effects_p (op1))
30215 0 : && (!op2 || !side_effects_p (op2)))
30216 : {
30217 0 : emit_move_insn (target, CONSTM1_RTX (mode));
30218 0 : return target;
30219 : }
30220 : break;
30221 : }
30222 :
30223 2338 : if (!register_operand (op0, mode))
30224 : {
30225 : /* We can't use force_reg (mode, op0). */
30226 12 : tmp0 = gen_reg_rtx (GET_MODE (op0));
30227 12 : emit_move_insn (tmp0,op0);
30228 : }
30229 : else
30230 : tmp0 = op0;
30231 2338 : if (GET_MODE (tmp0) != mode)
30232 0 : tmp0 = gen_lowpart (mode, tmp0);
30233 :
30234 2338 : if (!op1 || rtx_equal_p (op0, op1))
30235 6 : tmp1 = copy_rtx (tmp0);
30236 2332 : else if (!register_operand (op1, mode))
30237 : {
30238 : /* We can't use force_reg (mode, op1). */
30239 28 : tmp1 = gen_reg_rtx (GET_MODE (op1));
30240 28 : emit_move_insn (tmp1, op1);
30241 : }
30242 : else
30243 : tmp1 = op1;
30244 2338 : if (GET_MODE (tmp1) != mode)
30245 0 : tmp1 = gen_lowpart (mode, tmp1);
30246 :
30247 2338 : if (!op2 || rtx_equal_p (op0, op2))
30248 74 : tmp2 = copy_rtx (tmp0);
30249 2264 : else if (rtx_equal_p (op1, op2))
30250 0 : tmp2 = copy_rtx (tmp1);
30251 2264 : else if (CONST_VECTOR_P (op2))
30252 : {
30253 43 : if (GET_MODE (op2) != mode)
30254 0 : op2 = gen_lowpart (mode, op2);
30255 43 : tmp2 = ix86_gen_bcst_mem (mode, op2);
30256 43 : if (!tmp2)
30257 : {
30258 25 : machine_mode bcst32_mode = mode;
30259 25 : machine_mode bcst64_mode = mode;
30260 25 : switch (mode)
30261 : {
30262 1 : case V1TImode:
30263 1 : case V4SImode:
30264 1 : case V4SFmode:
30265 1 : case V8HImode:
30266 1 : case V16QImode:
30267 1 : bcst32_mode = V4SImode;
30268 1 : bcst64_mode = V2DImode;
30269 1 : break;
30270 :
30271 0 : case V2TImode:
30272 0 : case V8SImode:
30273 0 : case V8SFmode:
30274 0 : case V16HImode:
30275 0 : case V32QImode:
30276 0 : bcst32_mode = V8SImode;
30277 0 : bcst64_mode = V4DImode;
30278 0 : break;
30279 :
30280 3 : case V4TImode:
30281 3 : case V16SImode:
30282 3 : case V16SFmode:
30283 3 : case V32HImode:
30284 3 : case V64QImode:
30285 3 : bcst32_mode = V16SImode;
30286 3 : bcst64_mode = V8DImode;
30287 3 : break;
30288 :
30289 : default:
30290 : break;
30291 : }
30292 :
30293 25 : if (bcst32_mode != mode)
30294 : {
30295 4 : tmp2 = gen_lowpart (bcst32_mode, op2);
30296 4 : if (ix86_gen_bcst_mem (bcst32_mode, tmp2))
30297 : {
30298 3 : tmp2 = ix86_expand_ternlog (bcst32_mode,
30299 3 : gen_lowpart (bcst32_mode, tmp0),
30300 3 : gen_lowpart (bcst32_mode, tmp1),
30301 : tmp2, idx, NULL_RTX);
30302 3 : emit_move_insn (target, gen_lowpart (mode, tmp2));
30303 3 : return target;
30304 : }
30305 : }
30306 :
30307 22 : if (bcst64_mode != mode)
30308 : {
30309 1 : tmp2 = gen_lowpart (bcst64_mode, op2);
30310 1 : if (ix86_gen_bcst_mem (bcst64_mode, tmp2))
30311 : {
30312 0 : tmp2 = ix86_expand_ternlog (bcst64_mode,
30313 0 : gen_lowpart (bcst64_mode, tmp0),
30314 0 : gen_lowpart (bcst64_mode, tmp1),
30315 : tmp2, idx, NULL_RTX);
30316 0 : emit_move_insn (target, gen_lowpart (mode, tmp2));
30317 0 : return target;
30318 : }
30319 : }
30320 :
30321 22 : tmp2 = force_const_mem (mode, op2);
30322 22 : rtx bcast = ix86_broadcast_from_constant (mode, tmp2);
30323 22 : tmp2 = validize_mem (tmp2);
30324 22 : if (bcast)
30325 : {
30326 12 : rtx reg2 = gen_reg_rtx (mode);
30327 12 : bool ok = ix86_expand_vector_init_duplicate (false, mode,
30328 : reg2, bcast);
30329 12 : if (ok)
30330 2335 : tmp2 = reg2;
30331 : }
30332 : }
30333 : }
30334 : else
30335 : tmp2 = op2;
30336 2335 : if (GET_MODE (tmp2) != mode)
30337 0 : tmp2 = gen_lowpart (mode, tmp2);
30338 : /* Some memory_operands are not vector_memory_operands. */
30339 2335 : if (!bcst_vector_operand (tmp2, mode))
30340 0 : tmp2 = force_reg (mode, tmp2);
30341 :
30342 2335 : rtvec vec = gen_rtvec (4, tmp0, tmp1, tmp2, GEN_INT (idx));
30343 2335 : emit_move_insn (target, gen_rtx_UNSPEC (mode, vec, UNSPEC_VTERNLOG));
30344 2335 : return target;
30345 : }
30346 :
30347 : /* GF2P8AFFINEQB matrixes to implement shift and rotate. */
30348 :
30349 : static const uint64_t matrix_ashift[8] =
30350 : {
30351 : 0,
30352 : 0x0001020408102040, /* 1 l */
30353 : 0x0000010204081020, /* 2 l */
30354 : 0x0000000102040810, /* 3 l */
30355 : 0x0000000001020408, /* 4 l */
30356 : 0x0000000000010204, /* 5 l */
30357 : 0x0000000000000102, /* 6 l */
30358 : 0x0000000000000001 /* 7 l */
30359 : };
30360 :
30361 : static const uint64_t matrix_lshiftrt[8] =
30362 : {
30363 : 0,
30364 : 0x0204081020408000, /* 1 r */
30365 : 0x0408102040800000, /* 2 r */
30366 : 0x0810204080000000, /* 3 r */
30367 : 0x1020408000000000, /* 4 r */
30368 : 0x2040800000000000, /* 5 r */
30369 : 0x4080000000000000, /* 6 r */
30370 : 0x8000000000000000 /* 7 r */
30371 : };
30372 :
30373 : static const uint64_t matrix_ashiftrt[8] =
30374 : {
30375 : 0,
30376 : 0x0204081020408080, /* 1 r */
30377 : 0x0408102040808080, /* 2 r */
30378 : 0x0810204080808080, /* 3 r */
30379 : 0x1020408080808080, /* 4 r */
30380 : 0x2040808080808080, /* 5 r */
30381 : 0x4080808080808080, /* 6 r */
30382 : 0x8080808080808080 /* 7 r */
30383 : };
30384 :
30385 : static const uint64_t matrix_rotate[8] =
30386 : {
30387 : 0,
30388 : 0x8001020408102040, /* 1 rol8 */
30389 : 0x4080010204081020, /* 2 rol8 */
30390 : 0x2040800102040810, /* 3 rol8 */
30391 : 0x1020408001020408, /* 4 rol8 */
30392 : 0x0810204080010204, /* 5 rol8 */
30393 : 0x0408102040800102, /* 6 rol8 */
30394 : 0x0204081020408001 /* 7 rol8 */
30395 : };
30396 :
30397 : static const uint64_t matrix_rotatert[8] =
30398 : {
30399 : 0,
30400 : 0x0204081020408001, /* 1 ror8 */
30401 : 0x0408102040800102, /* 2 ror8 */
30402 : 0x0810204080010204, /* 3 ror8 */
30403 : 0x1020408001020408, /* 4 ror8 */
30404 : 0x2040800102040810, /* 5 ror8 */
30405 : 0x4080010204081020, /* 6 ror8 */
30406 : 0x8001020408102040 /* 7 ror8 */
30407 : };
30408 :
30409 : /* Return rtx to load a 64bit GF2P8AFFINE GP(2) matrix implementing a shift
30410 : for CODE and shift count COUNT into register with vector of size of SRC. */
30411 :
30412 : rtx
30413 217 : ix86_vgf2p8affine_shift_matrix (rtx src, rtx count, enum rtx_code code)
30414 : {
30415 217 : machine_mode mode = GET_MODE (src);
30416 217 : const uint64_t *matrix;
30417 217 : unsigned shift = INTVAL (count) & 7;
30418 217 : gcc_assert (shift > 0 && shift < 8);
30419 :
30420 217 : switch (code)
30421 : {
30422 : case ASHIFT:
30423 : matrix = matrix_ashift;
30424 : break;
30425 30 : case ASHIFTRT:
30426 30 : matrix = matrix_ashiftrt;
30427 30 : break;
30428 34 : case LSHIFTRT:
30429 34 : matrix = matrix_lshiftrt;
30430 34 : break;
30431 34 : case ROTATE:
30432 34 : matrix = matrix_rotate;
30433 34 : break;
30434 35 : case ROTATERT:
30435 35 : matrix = matrix_rotatert;
30436 35 : break;
30437 0 : default:
30438 0 : gcc_unreachable ();
30439 : }
30440 :
30441 217 : int nelts = GET_MODE_NUNITS (mode);
30442 217 : rtvec vec = rtvec_alloc (nelts);
30443 217 : uint64_t ma = matrix[shift];
30444 7417 : for (int i = 0; i < nelts; i++)
30445 7200 : RTVEC_ELT (vec, i) = gen_int_mode ((ma >> ((i % 8) * 8)) & 0xff, QImode);
30446 :
30447 217 : return force_reg (mode, gen_rtx_CONST_VECTOR (mode, vec));
30448 : }
30449 :
30450 : /* Trunc a vector to a narrow vector, like v4di -> v4si. */
30451 :
30452 : void
30453 62 : ix86_expand_trunc_with_avx2_noavx512f (rtx output, rtx input, machine_mode cvt_mode)
30454 : {
30455 62 : machine_mode out_mode = GET_MODE (output);
30456 62 : machine_mode in_mode = GET_MODE (input);
30457 62 : int len = GET_MODE_SIZE (in_mode);
30458 248 : gcc_assert (len == GET_MODE_SIZE (cvt_mode)
30459 : && GET_MODE_INNER (out_mode) == GET_MODE_INNER (cvt_mode)
30460 : && (REG_P (input) || SUBREG_P (input)));
30461 62 : scalar_mode inner_out_mode = GET_MODE_INNER (out_mode);
30462 124 : int in_innersize = GET_MODE_SIZE (GET_MODE_INNER (in_mode));
30463 62 : int out_innersize = GET_MODE_SIZE (inner_out_mode);
30464 :
30465 62 : struct expand_vec_perm_d d;
30466 62 : d.target = gen_reg_rtx (cvt_mode);
30467 62 : d.op0 = lowpart_subreg (cvt_mode, force_reg(in_mode, input), in_mode);
30468 62 : d.op1 = d.op0;
30469 62 : d.vmode = cvt_mode;
30470 62 : d.nelt = GET_MODE_NUNITS (cvt_mode);
30471 62 : d.testing_p = false;
30472 62 : d.one_operand_p = true;
30473 :
30474 : /* Init perm. Put the needed bits of input in order and
30475 : fill the rest of bits by default. */
30476 678 : for (int i = 0; i < d.nelt; ++i)
30477 : {
30478 616 : d.perm[i] = i;
30479 1232 : if (i < GET_MODE_NUNITS (out_mode))
30480 242 : d.perm[i] = i * (in_innersize / out_innersize);
30481 : }
30482 :
30483 62 : bool ok = ix86_expand_vec_perm_const_1(&d);
30484 62 : gcc_assert (ok);
30485 62 : emit_move_insn (output, gen_lowpart (out_mode, d.target));
30486 62 : }
30487 :
30488 : /* Implement truncv8sfv8bf2 with vector permutation. */
30489 : void
30490 8 : ix86_expand_vector_sf2bf_with_vec_perm (rtx dest, rtx src)
30491 : {
30492 8 : machine_mode vperm_mode, src_mode = GET_MODE (src);
30493 8 : switch (src_mode)
30494 : {
30495 : case V16SFmode:
30496 : vperm_mode = V32BFmode;
30497 : break;
30498 2 : case V8SFmode:
30499 2 : vperm_mode = V16BFmode;
30500 2 : break;
30501 4 : case V4SFmode:
30502 4 : vperm_mode = V8BFmode;
30503 4 : break;
30504 0 : default:
30505 0 : gcc_unreachable ();
30506 : }
30507 :
30508 8 : int nelt = GET_MODE_NUNITS (vperm_mode);
30509 8 : vec_perm_builder sel (nelt, nelt, 1);
30510 8 : sel.quick_grow (nelt);
30511 144 : for (int i = 0; i != nelt; i++)
30512 128 : sel[i] = (2 * i + 1) % nelt;
30513 16 : vec_perm_indices indices (sel, 1, nelt);
30514 :
30515 8 : rtx target = gen_reg_rtx (vperm_mode);
30516 8 : rtx op0 = lowpart_subreg (vperm_mode,
30517 : force_reg (src_mode, src),
30518 : src_mode);
30519 8 : bool ok = targetm.vectorize.vec_perm_const (vperm_mode, vperm_mode,
30520 : target, op0, op0, indices);
30521 8 : gcc_assert (ok);
30522 8 : emit_move_insn (dest, lowpart_subreg (GET_MODE (dest), target, vperm_mode));
30523 8 : }
30524 :
30525 : /* Implement extendv8bf2v8sf2 with vector permutation. */
30526 : void
30527 8 : ix86_expand_vector_bf2sf_with_vec_perm (rtx dest, rtx src)
30528 : {
30529 8 : machine_mode vperm_mode, src_mode = GET_MODE (src);
30530 8 : switch (src_mode)
30531 : {
30532 : case V16BFmode:
30533 : vperm_mode = V32BFmode;
30534 : break;
30535 2 : case V8BFmode:
30536 2 : vperm_mode = V16BFmode;
30537 2 : break;
30538 4 : case V4BFmode:
30539 4 : vperm_mode = V8BFmode;
30540 4 : break;
30541 0 : default:
30542 0 : gcc_unreachable ();
30543 : }
30544 :
30545 8 : int nelt = GET_MODE_NUNITS (vperm_mode);
30546 8 : vec_perm_builder sel (nelt, nelt, 1);
30547 8 : sel.quick_grow (nelt);
30548 144 : for (int i = 0, k = 0, j = nelt; i != nelt; i++)
30549 128 : sel[i] = i & 1 ? j++ : k++;
30550 :
30551 16 : vec_perm_indices indices (sel, 2, nelt);
30552 :
30553 8 : rtx target = gen_reg_rtx (vperm_mode);
30554 8 : rtx op1 = lowpart_subreg (vperm_mode,
30555 : force_reg (src_mode, src),
30556 : src_mode);
30557 8 : rtx op0 = CONST0_RTX (vperm_mode);
30558 8 : bool ok = targetm.vectorize.vec_perm_const (vperm_mode, vperm_mode,
30559 : target, op0, op1, indices);
30560 8 : gcc_assert (ok);
30561 8 : emit_move_insn (dest, lowpart_subreg (GET_MODE (dest), target, vperm_mode));
30562 8 : }
30563 :
30564 : /* Implement bitreverse<mode>2 using vbitrevb or gf2p8affineqb. */
30565 :
30566 : void
30567 13 : ix86_expand_bitreverse (rtx dest, rtx src)
30568 : {
30569 13 : machine_mode mode = GET_MODE (dest);
30570 13 : rtx temp;
30571 26 : if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
30572 : {
30573 1 : rtx temp1 = gen_reg_rtx (mode == TImode ? V2DImode : V4SImode);
30574 1 : rtx temp2 = gen_reg_rtx (mode == TImode ? V2DImode : V4SImode);
30575 1 : if (mode == TImode)
30576 : {
30577 1 : temp = lowpart_subreg (DImode, src, TImode);
30578 1 : emit_insn (gen_rtx_SET (temp1, gen_rtx_VEC_CONCAT (V2DImode, temp,
30579 : const0_rtx)));
30580 1 : temp = gen_highpart (DImode, src);
30581 1 : emit_insn (gen_rtx_SET (temp2, gen_rtx_VEC_CONCAT (V2DImode, temp,
30582 : const0_rtx)));
30583 : }
30584 : else
30585 : {
30586 0 : temp = lowpart_subreg (SImode, src, DImode);
30587 0 : emit_insn (gen_vec_setv4si_0 (temp1, CONST0_RTX (V4SImode), temp));
30588 0 : temp = gen_highpart (SImode, src);
30589 0 : emit_insn (gen_vec_setv4si_0 (temp2, CONST0_RTX (V4SImode), temp));
30590 0 : temp1 = lowpart_subreg (V2DImode, temp1, V4SImode);
30591 0 : temp2 = lowpart_subreg (V2DImode, temp2, V4SImode);
30592 : }
30593 1 : temp = gen_reg_rtx (V2DImode);
30594 1 : emit_insn (gen_vec_interleave_lowv2di (temp, temp1, temp2));
30595 : }
30596 12 : else if (mode != DImode)
30597 : {
30598 9 : if (mode != SImode)
30599 : {
30600 6 : src = force_reg (mode, src);
30601 6 : src = lowpart_subreg (SImode, src, mode);
30602 : }
30603 9 : temp = gen_reg_rtx (V4SImode);
30604 9 : emit_insn (gen_vec_setv4si_0 (temp, CONST0_RTX (V4SImode), src));
30605 : }
30606 : else
30607 : {
30608 3 : temp = gen_reg_rtx (V2DImode);
30609 3 : emit_insn (gen_rtx_SET (temp, gen_rtx_VEC_CONCAT (V2DImode, src,
30610 : const0_rtx)));
30611 : }
30612 13 : src = temp;
30613 13 : temp = gen_reg_rtx (V16QImode);
30614 13 : src = lowpart_subreg (V16QImode, src, GET_MODE (src));
30615 13 : if (TARGET_AVX512BMM && TARGET_AVX512VL)
30616 4 : emit_insn (gen_avx512bmm_vbitrevb_v16qi (temp, src));
30617 9 : else if (TARGET_GFNI)
30618 : {
30619 9 : rtx src2 = gen_rtx_CONST_VECTOR (V16QImode,
30620 : gen_rtvec (16, GEN_INT (1), GEN_INT (2),
30621 : GEN_INT (4), GEN_INT (8),
30622 : GEN_INT (16), GEN_INT (32),
30623 : GEN_INT (64), GEN_INT (-128),
30624 : GEN_INT (1), GEN_INT (2),
30625 : GEN_INT (4), GEN_INT (8),
30626 : GEN_INT (16), GEN_INT (32),
30627 : GEN_INT (64),
30628 : GEN_INT (-128)));
30629 9 : src2 = validize_mem (force_const_mem (V16QImode, src2));
30630 9 : emit_insn (gen_vgf2p8affineqb_v16qi (temp, src, src2, const0_rtx));
30631 : }
30632 13 : if (mode == QImode)
30633 : {
30634 3 : rtx temp1 = gen_reg_rtx (SImode);
30635 3 : rtx temp2 = lowpart_subreg (V4SImode, temp, V16QImode);
30636 3 : rtx temp3 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
30637 3 : emit_insn (gen_rtx_SET (temp1,
30638 : gen_rtx_VEC_SELECT (SImode, temp2, temp3)));
30639 3 : emit_move_insn (dest, lowpart_subreg (QImode, temp1, SImode));
30640 3 : return;
30641 : }
30642 27 : rtx target = gen_reg_rtx ((GET_MODE_SIZE (mode) < 4 || !TARGET_64BIT)
30643 7 : ? SImode : mode == TImode ? DImode : mode);
30644 10 : emit_move_insn (target, lowpart_subreg (GET_MODE (target), temp, V16QImode));
30645 20 : if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
30646 : {
30647 1 : rtx temp1 = gen_reg_rtx (GET_MODE (target));
30648 1 : if (mode == TImode || TARGET_SSE4_1)
30649 : {
30650 1 : rtx temp2 = lowpart_subreg (mode == TImode ? V2DImode : V4SImode,
30651 : temp, V16QImode);
30652 1 : rtx temp3 = gen_rtx_PARALLEL (VOIDmode,
30653 : gen_rtvec (1, GEN_INT (mode == TImode
30654 : ? 1 : 2)));
30655 1 : emit_insn (gen_rtx_SET (temp1,
30656 : gen_rtx_VEC_SELECT (GET_MODE (target), temp2,
30657 : temp3)));
30658 1 : }
30659 : else
30660 : {
30661 0 : rtx temp2 = gen_reg_rtx (V4SImode);
30662 0 : rtx temp3 = lowpart_subreg (V4SImode, temp, V16QImode);
30663 0 : emit_insn (gen_sse2_pshufd (temp2, temp3, GEN_INT (0xaa)));
30664 0 : emit_move_insn (temp1, lowpart_subreg (GET_MODE (target), temp2,
30665 : V4SImode));
30666 : }
30667 1 : rtx temp4 = gen_reg_rtx (GET_MODE (target));
30668 1 : rtx temp5 = gen_reg_rtx (GET_MODE (target));
30669 0 : rtx (*gen_bswap) (rtx, rtx)
30670 1 : = mode == TImode ? gen_bswapdi2 : gen_bswapsi2;
30671 1 : emit_insn (gen_bswap (temp4, target));
30672 1 : emit_insn (gen_bswap (temp5, temp1));
30673 1 : temp4 = gen_rtx_ZERO_EXTEND (mode, temp4);
30674 1 : temp5 = gen_rtx_ZERO_EXTEND (mode, temp5);
30675 1 : rtx shift = GEN_INT (GET_MODE_PRECISION (GET_MODE (target)));
30676 1 : temp4 = gen_rtx_ASHIFT (mode, temp4, shift);
30677 1 : emit_insn (gen_rtx_SET (dest, gen_rtx_IOR (mode, temp4, temp5)));
30678 1 : return;
30679 : }
30680 9 : if (mode == HImode)
30681 : {
30682 3 : target = lowpart_subreg (mode, target, SImode);
30683 3 : emit_insn (gen_bswaphi2 (dest, target));
30684 : }
30685 6 : else if (mode == SImode)
30686 3 : emit_insn (gen_bswapsi2 (dest, target));
30687 : else
30688 3 : emit_insn (gen_rtx_SET (dest, gen_rtx_BSWAP (mode, target)));
30689 : }
30690 :
30691 : /* Expand LCP stall or long immediate peephole for INSN. Use the
30692 : previous scratch register if possible. If USE_XOR is true,
30693 : generate "*movsi_xor". */
30694 :
30695 : void
30696 31679 : ix86_expand_lcp_stall_peephole (rtx_insn *insn, rtx *operands,
30697 : bool use_xor)
30698 : {
30699 31679 : rtx imm, scratch;
30700 31679 : machine_mode mode = GET_MODE (operands[0]);
30701 :
30702 : /* Get the immediate operand and the allocated scratch register. */
30703 31679 : if (use_xor)
30704 : {
30705 10207 : imm = const0_rtx;
30706 10207 : scratch = operands[1];
30707 : }
30708 : else
30709 : {
30710 21472 : imm = operands[1];
30711 21472 : scratch = operands[2];
30712 : }
30713 :
30714 31679 : df_ref def;
30715 31679 : rtx_insn *prev, *prev_insn = nullptr;
30716 31679 : rtx set, prev_scratch = nullptr;
30717 :
30718 : /* Scan backward for the previous scratch register def with IMM in
30719 : the same basic block. */
30720 31679 : basic_block bb = BLOCK_FOR_INSN (insn);
30721 949459 : for (prev = PREV_INSN (insn);
30722 949459 : prev != BB_HEAD (bb);
30723 917780 : prev = PREV_INSN (prev))
30724 : {
30725 922678 : if (NONDEBUG_INSN_P (prev))
30726 2084474 : FOR_EACH_INSN_DEF (def, prev)
30727 1758277 : if (!DF_REF_IS_ARTIFICIAL (def)
30728 1758277 : && !DF_REF_FLAGS_IS_SET (def, DF_REF_MAY_CLOBBER)
30729 174633 : && !DF_REF_FLAGS_IS_SET (def, DF_REF_MUST_CLOBBER))
30730 : {
30731 153892 : rtx reg = DF_REF_REG (def);
30732 153892 : if (HARD_REGISTER_P (reg) && REGNO (reg) != FLAGS_REG)
30733 : {
30734 152739 : set = single_set (prev);
30735 152739 : if (!set)
30736 484 : continue;
30737 :
30738 152255 : rtx dest = SET_DEST (set);
30739 :
30740 : /* Reject DEST if a register is not wide enough to
30741 : supply MODE or invalid for QImode. */
30742 133058 : if (!GENERAL_REG_P (dest)
30743 230436 : || (GET_MODE_SIZE (GET_MODE (dest))
30744 115218 : < GET_MODE_SIZE (mode))
30745 266311 : || (mode == QImode
30746 0 : && !ANY_QI_REGNO_P (REGNO (dest))))
30747 38199 : continue;
30748 :
30749 114056 : rtx src = SET_SRC (set);
30750 114056 : if (rtx_equal_p (src, imm))
30751 : {
30752 : /* A previous scratch register is found. */
30753 : prev_scratch = dest;
30754 : prev_insn = prev;
30755 : break;
30756 : }
30757 : }
30758 : }
30759 :
30760 922678 : if (prev_scratch)
30761 : break;
30762 : }
30763 :
30764 31679 : if (prev_scratch)
30765 : {
30766 : /* The previous scratch register is unusable if it is set between
30767 : PREV_INSN and INSN. */
30768 4898 : unsigned int regno = REGNO (prev_scratch);
30769 :
30770 : /* Scan backward for the previous scratch register def. */
30771 4898 : for (prev = PREV_INSN (insn);
30772 128881 : prev != prev_insn;
30773 123983 : prev = PREV_INSN (prev))
30774 : {
30775 125490 : if (NONDEBUG_INSN_P (prev))
30776 253677 : FOR_EACH_INSN_DEF (def, prev)
30777 176504 : if (HARD_REGISTER_P (DF_REF_REAL_REG (def))
30778 176504 : && DF_REF_REGNO (def) == regno)
30779 : {
30780 : /* Since the previous scratch register is set, it is
30781 : unusable. */
30782 1507 : if (dump_file)
30783 : {
30784 0 : fprintf (dump_file,
30785 : "\nThe previous scratch register:\n\n");
30786 0 : print_rtl_single (dump_file, prev_scratch);
30787 0 : fprintf (dump_file, "\nset in:\n\n");
30788 0 : print_rtl_single (dump_file, prev_insn);
30789 0 : fprintf (dump_file,
30790 : "\nis unusable by:\n\n");
30791 0 : print_rtl_single (dump_file, insn);
30792 0 : fprintf (dump_file,
30793 : "\nsince it is overridden by:\n\n");
30794 0 : print_rtl_single (dump_file, prev);
30795 0 : fprintf (dump_file, "\n");
30796 : }
30797 : prev_scratch = nullptr;
30798 : break;
30799 : }
30800 :
30801 123983 : if (!prev_scratch)
30802 : break;
30803 : }
30804 :
30805 4898 : if (prev_scratch)
30806 : {
30807 : /* Ignore the allocated scratch register and use the previous
30808 : scratch register. */
30809 3391 : if (dump_file)
30810 : {
30811 0 : fprintf (dump_file,
30812 : "\nIgnore the allocated scratch register:\n\n");
30813 0 : print_rtl_single (dump_file, scratch);
30814 0 : fprintf (dump_file,
30815 : "\nand use the previous scratch register:\n\n");
30816 0 : print_rtl_single (dump_file, prev_scratch);
30817 0 : fprintf (dump_file, "\nset in:\n\n");
30818 0 : print_rtl_single (dump_file, prev_insn);
30819 0 : fprintf (dump_file, "\nfor:\n\n");
30820 0 : print_rtl_single (dump_file, insn);
30821 0 : fprintf (dump_file, "\n");
30822 : }
30823 3391 : scratch = gen_lowpart (mode, prev_scratch);
30824 3391 : set = gen_rtx_SET (operands[0], scratch);
30825 3391 : emit_insn (set);
30826 3391 : return;
30827 : }
30828 : }
30829 :
30830 : /* If there is no usable previous scratch register, use the allocated
30831 : scratch register. */
30832 28288 : if (use_xor)
30833 : {
30834 : /* Generate "*movsi_xor". */
30835 8686 : rtx x = gen_lowpart (SImode, scratch);
30836 8686 : rtvec p = rtvec_alloc (2);
30837 8686 : set = gen_rtx_SET (x, const0_rtx);
30838 8686 : RTVEC_ELT (p, 0) = set;
30839 8686 : rtx clobber = gen_rtx_REG (CCmode, FLAGS_REG);
30840 8686 : RTVEC_ELT (p, 1) = gen_rtx_CLOBBER (VOIDmode, clobber);
30841 8686 : set = gen_rtx_PARALLEL (VOIDmode, p);
30842 : }
30843 : else
30844 19602 : set = gen_rtx_SET (scratch, imm);
30845 28288 : emit_insn (set);
30846 28288 : set = gen_rtx_SET (operands[0], scratch);
30847 28288 : emit_insn (set);
30848 : }
30849 :
30850 : /* Generate assembly to calculate CRC using pclmulqdq instruction.
30851 : OPERANDS[1] is input CRC,
30852 : OPERANDS[2] is data (message),
30853 : OPERANDS[3] is the polynomial without the leading 1. */
30854 :
30855 : void
30856 14 : ix86_expand_crc_using_pclmul (scalar_mode crc_mode,
30857 : scalar_mode data_mode,
30858 : rtx *operands)
30859 : {
30860 : /* Check and keep arguments. */
30861 14 : gcc_assert (!CONST_INT_P (operands[0]));
30862 14 : gcc_assert (CONST_INT_P (operands[3]));
30863 14 : rtx crc = operands[1];
30864 14 : rtx data = operands[2];
30865 14 : rtx polynomial = operands[3];
30866 :
30867 14 : unsigned HOST_WIDE_INT crc_size = GET_MODE_BITSIZE (crc_mode);
30868 14 : unsigned HOST_WIDE_INT data_size = GET_MODE_BITSIZE (data_mode);
30869 14 : gcc_assert (crc_size <= 64);
30870 14 : gcc_assert (data_size <= crc_size);
30871 :
30872 14 : machine_mode op_mode = crc_size == 64 ? DImode : SImode;
30873 :
30874 : /* Calculate the quotient. */
30875 14 : unsigned HOST_WIDE_INT
30876 14 : q = gf2n_poly_long_div_quotient (UINTVAL (polynomial), crc_size);
30877 :
30878 14 : if (crc_size > data_size)
30879 4 : crc = expand_shift (RSHIFT_EXPR, op_mode, crc, crc_size - data_size,
30880 : NULL_RTX, 1);
30881 :
30882 : /* Keep the quotient and polynomial in V2DImode. */
30883 14 : rtvec q_poly_constants = gen_rtvec (2, gen_int_mode (q, DImode),
30884 14 : gen_int_mode (UINTVAL (polynomial),
30885 : DImode));
30886 14 : rtx constants_v2di = gen_rtx_CONST_VECTOR (V2DImode, q_poly_constants);
30887 14 : rtx constants_reg = gen_reg_rtx (V2DImode);
30888 14 : emit_move_insn (constants_reg, constants_v2di);
30889 :
30890 : /* crc ^ data and keep in V2DImode. */
30891 14 : rtx cd_xor = expand_binop (op_mode, xor_optab, crc, data,
30892 : NULL_RTX, 1, OPTAB_WIDEN);
30893 14 : rtx crc_temp;
30894 14 : if (op_mode == DImode)
30895 : {
30896 6 : crc_temp = gen_reg_rtx (V2DImode);
30897 6 : emit_insn (gen_vec_concatv2di (crc_temp, cd_xor, cd_xor));
30898 : }
30899 : else
30900 : {
30901 8 : crc_temp = gen_reg_rtx (V4SImode);
30902 8 : emit_insn (gen_vec_setv4si_0 (crc_temp, CONST0_RTX (V4SImode), cd_xor));
30903 8 : crc_temp = gen_lowpart (V2DImode, crc_temp);
30904 : }
30905 :
30906 : /* Perform carry-less multiplication with quotient. */
30907 14 : rtx res = gen_reg_rtx (V2DImode);
30908 14 : emit_insn (gen_pclmulqdq (res, crc_temp, constants_reg, const0_rtx));
30909 :
30910 14 : if (op_mode == DImode)
30911 : {
30912 : /* Perform res ^ crc_temp to handle 65th bit of quotient. */
30913 6 : res = expand_binop (V2DImode, xor_optab, res, crc_temp,
30914 : NULL_RTX, 1, OPTAB_DIRECT);
30915 : }
30916 : else
30917 8 : res = expand_shift (RSHIFT_EXPR, V2DImode, res, crc_size, NULL_RTX, 1);
30918 :
30919 : /* Perform carry-less multiplication with polynomial and get low part. */
30920 22 : emit_insn (gen_pclmulqdq (res, res, constants_reg,
30921 : GEN_INT (op_mode == DImode ? 0x11 : 0x10)));
30922 :
30923 14 : rtx crc_part = gen_reg_rtx (op_mode);
30924 14 : if (op_mode == DImode)
30925 6 : emit_insn (gen_vec_extractv2didi (crc_part, res, const0_rtx));
30926 : else
30927 : {
30928 8 : res = gen_lowpart (V4SImode, res);
30929 8 : emit_insn (gen_vec_extractv4sisi (crc_part, res, const0_rtx));
30930 : }
30931 :
30932 14 : if (crc_size > data_size)
30933 : {
30934 4 : rtx shift = expand_shift (LSHIFT_EXPR, op_mode, operands[1], data_size,
30935 : NULL_RTX, 1);
30936 4 : crc_part = expand_binop (op_mode, xor_optab, crc_part, shift,
30937 : NULL_RTX, 1, OPTAB_DIRECT);
30938 : }
30939 14 : emit_move_insn (operands[0], gen_lowpart (crc_mode, crc_part));
30940 14 : }
30941 :
30942 : /* Generate assembly to calculate reversed CRC using pclmulqdq instruction.
30943 : OPERANDS[1] is input CRC,
30944 : OPERANDS[2] is data (message),
30945 : OPERANDS[3] is the polynomial without the leading 1. */
30946 :
30947 : void
30948 18 : ix86_expand_reversed_crc_using_pclmul (scalar_mode crc_mode,
30949 : scalar_mode data_mode,
30950 : rtx *operands)
30951 : {
30952 : /* Check and keep arguments. */
30953 18 : gcc_assert (!CONST_INT_P (operands[0]));
30954 18 : gcc_assert (CONST_INT_P (operands[3]));
30955 18 : rtx crc = operands[1];
30956 18 : rtx data = operands[2];
30957 18 : rtx polynomial = operands[3];
30958 :
30959 18 : unsigned HOST_WIDE_INT crc_size = GET_MODE_BITSIZE (crc_mode);
30960 18 : unsigned HOST_WIDE_INT data_size = GET_MODE_BITSIZE (data_mode);
30961 18 : gcc_assert (crc_size <= 64);
30962 18 : gcc_assert (data_size <= crc_size);
30963 :
30964 18 : machine_mode op_mode = crc_size == 64 ? DImode : SImode;
30965 :
30966 : /* Calculate the quotient. */
30967 18 : unsigned HOST_WIDE_INT
30968 18 : q = gf2n_poly_long_div_quotient (UINTVAL (polynomial), crc_size);
30969 : /* Reflect the calculated quotient. */
30970 18 : q = (reflect_hwi (q, crc_size) << 1) + 1;
30971 :
30972 18 : unsigned HOST_WIDE_INT reflected_op3 = reflect_hwi (UINTVAL (polynomial),
30973 : crc_size);
30974 : /* Keep the quotient and reflected polynomial in V2DImode. */
30975 18 : rtvec q_poly_constants = gen_rtvec (2, gen_int_mode (q, DImode),
30976 18 : gen_int_mode (reflected_op3 << 1,
30977 : DImode));
30978 18 : rtx constants_v2di = gen_rtx_CONST_VECTOR (V2DImode, q_poly_constants);
30979 18 : rtx constants_reg = gen_reg_rtx (V2DImode);
30980 18 : emit_move_insn (constants_reg, constants_v2di);
30981 :
30982 : /* crc ^ data and keep in V2DImode. */
30983 18 : rtx cd_xor = expand_binop (op_mode, xor_optab, crc, data,
30984 : NULL_RTX, 1, OPTAB_WIDEN);
30985 18 : rtx crc_temp;
30986 18 : if (op_mode == DImode)
30987 : {
30988 14 : crc_temp = gen_reg_rtx (V2DImode);
30989 14 : emit_insn (gen_vec_concatv2di (crc_temp, cd_xor, const0_rtx));
30990 : }
30991 : else
30992 : {
30993 4 : crc_temp = gen_reg_rtx (V4SImode);
30994 4 : emit_insn (gen_vec_setv4si_0 (crc_temp, CONST0_RTX (V4SImode), cd_xor));
30995 4 : crc_temp = gen_lowpart (V2DImode, crc_temp);
30996 : }
30997 :
30998 : /* Perform carry-less multiplication with quotient. */
30999 18 : emit_insn (gen_pclmulqdq (crc_temp, crc_temp, constants_reg, const0_rtx));
31000 18 : if (data_size < 64)
31001 12 : crc_temp = expand_shift (LSHIFT_EXPR, V2DImode, crc_temp, 64 - data_size,
31002 : NULL_RTX, 0);
31003 :
31004 18 : rtx res = gen_reg_rtx (V2DImode);
31005 : /* Perform carry-less multiplication with polynomial. */
31006 18 : emit_insn (gen_pclmulqdq (res, crc_temp, constants_reg, GEN_INT (0x10)));
31007 18 : if (op_mode == DImode && (UINTVAL (polynomial) & 1))
31008 : {
31009 : /* Perform xor of res[127:64] with crc_temp[63:0].
31010 : This is needed when the 65th bit of reflected polynomial is 1. */
31011 8 : rtx tmp = gen_reg_rtx (V2DImode);
31012 8 : emit_insn (gen_vec_interleave_lowv2di (tmp, crc_temp, crc_temp));
31013 8 : res = expand_binop (V2DImode, xor_optab, res, tmp,
31014 : NULL_RTX, 1, OPTAB_DIRECT);
31015 : }
31016 :
31017 18 : rtx res_high = gen_reg_rtx (op_mode);
31018 18 : if (op_mode == DImode)
31019 14 : emit_insn (gen_vec_extractv2didi (res_high, res, const1_rtx));
31020 : else
31021 : {
31022 4 : res = gen_lowpart (V4SImode, res);
31023 4 : emit_insn (gen_vec_extractv4sisi (res_high, res, const2_rtx));
31024 : }
31025 :
31026 18 : if (crc_size > data_size)
31027 : {
31028 8 : rtx shift = expand_shift (RSHIFT_EXPR, op_mode, crc, data_size,
31029 : NULL_RTX, 1);
31030 8 : res_high = expand_binop (op_mode, xor_optab, res_high, shift,
31031 : NULL_RTX, 1, OPTAB_DIRECT);
31032 : }
31033 18 : emit_move_insn (operands[0], gen_lowpart (crc_mode, res_high));
31034 18 : }
31035 :
31036 : #include "gt-i386-expand.h"
|