Line data Source code
1 : /* Copyright (C) 1988-2026 Free Software Foundation, Inc.
2 :
3 : This file is part of GCC.
4 :
5 : GCC is free software; you can redistribute it and/or modify
6 : it under the terms of the GNU General Public License as published by
7 : the Free Software Foundation; either version 3, or (at your option)
8 : any later version.
9 :
10 : GCC is distributed in the hope that it will be useful,
11 : but WITHOUT ANY WARRANTY; without even the implied warranty of
12 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 : GNU General Public License for more details.
14 :
15 : You should have received a copy of the GNU General Public License
16 : along with GCC; see the file COPYING3. If not see
17 : <http://www.gnu.org/licenses/>. */
18 :
19 : #define IN_TARGET_CODE 1
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "memmodel.h"
28 : #include "gimple.h"
29 : #include "cfghooks.h"
30 : #include "cfgloop.h"
31 : #include "df.h"
32 : #include "tm_p.h"
33 : #include "stringpool.h"
34 : #include "expmed.h"
35 : #include "optabs.h"
36 : #include "regs.h"
37 : #include "emit-rtl.h"
38 : #include "recog.h"
39 : #include "cgraph.h"
40 : #include "diagnostic.h"
41 : #include "cfgbuild.h"
42 : #include "alias.h"
43 : #include "fold-const.h"
44 : #include "attribs.h"
45 : #include "calls.h"
46 : #include "stor-layout.h"
47 : #include "varasm.h"
48 : #include "output.h"
49 : #include "insn-attr.h"
50 : #include "flags.h"
51 : #include "except.h"
52 : #include "explow.h"
53 : #include "expr.h"
54 : #include "cfgrtl.h"
55 : #include "common/common-target.h"
56 : #include "langhooks.h"
57 : #include "reload.h"
58 : #include "gimplify.h"
59 : #include "dwarf2.h"
60 : #include "tm-constrs.h"
61 : #include "cselib.h"
62 : #include "sched-int.h"
63 : #include "opts.h"
64 : #include "tree-pass.h"
65 : #include "context.h"
66 : #include "pass_manager.h"
67 : #include "target-globals.h"
68 : #include "gimple-iterator.h"
69 : #include "shrink-wrap.h"
70 : #include "builtins.h"
71 : #include "rtl-iter.h"
72 : #include "tree-iterator.h"
73 : #include "dbgcnt.h"
74 : #include "case-cfn-macros.h"
75 : #include "dojump.h"
76 : #include "fold-const-call.h"
77 : #include "tree-vrp.h"
78 : #include "tree-ssanames.h"
79 : #include "selftest.h"
80 : #include "selftest-rtl.h"
81 : #include "print-rtl.h"
82 : #include "intl.h"
83 : #include "ifcvt.h"
84 : #include "symbol-summary.h"
85 : #include "sreal.h"
86 : #include "ipa-cp.h"
87 : #include "ipa-prop.h"
88 : #include "ipa-fnsummary.h"
89 : #include "wide-int-bitmask.h"
90 : #include "tree-vector-builder.h"
91 : #include "debug.h"
92 : #include "dwarf2out.h"
93 : #include "i386-options.h"
94 : #include "i386-builtins.h"
95 : #include "i386-expand.h"
96 : #include "asan.h"
97 : #include "function-abi.h"
98 :
99 : /* Split one or more double-mode RTL references into pairs of half-mode
100 : references. The RTL can be REG, offsettable MEM, integer constant, or
101 : CONST_DOUBLE. "operands" is a pointer to an array of double-mode RTLs to
102 : split and "num" is its length. lo_half and hi_half are output arrays
103 : that parallel "operands". */
104 :
105 : void
106 4195086 : split_double_mode (machine_mode mode, rtx operands[],
107 : int num, rtx lo_half[], rtx hi_half[])
108 : {
109 4195086 : machine_mode half_mode;
110 4195086 : unsigned int byte;
111 4195086 : rtx mem_op = NULL_RTX;
112 4195086 : int mem_num = 0;
113 :
114 4195086 : switch (mode)
115 : {
116 : case E_TImode:
117 : half_mode = DImode;
118 : break;
119 605429 : case E_DImode:
120 605429 : half_mode = SImode;
121 605429 : break;
122 6 : case E_P2HImode:
123 6 : half_mode = HImode;
124 6 : break;
125 30 : case E_P2QImode:
126 30 : half_mode = QImode;
127 30 : break;
128 0 : default:
129 0 : gcc_unreachable ();
130 : }
131 :
132 4195086 : byte = GET_MODE_SIZE (half_mode);
133 :
134 8605251 : while (num--)
135 : {
136 4410165 : rtx op = operands[num];
137 :
138 : /* simplify_subreg refuse to split volatile memory addresses,
139 : but we still have to handle it. */
140 4410165 : if (MEM_P (op))
141 : {
142 1750897 : if (mem_op && rtx_equal_p (op, mem_op))
143 : {
144 2401 : lo_half[num] = lo_half[mem_num];
145 2401 : hi_half[num] = hi_half[mem_num];
146 : }
147 : else
148 : {
149 1748496 : mem_op = op;
150 1748496 : mem_num = num;
151 1748496 : lo_half[num] = adjust_address (op, half_mode, 0);
152 1748496 : hi_half[num] = adjust_address (op, half_mode, byte);
153 : }
154 : }
155 : else
156 : {
157 2659268 : lo_half[num] = simplify_gen_subreg (half_mode, op,
158 2659268 : GET_MODE (op) == VOIDmode
159 : ? mode : GET_MODE (op), 0);
160 :
161 2659268 : rtx tmp = simplify_gen_subreg (half_mode, op,
162 2659268 : GET_MODE (op) == VOIDmode
163 2659268 : ? mode : GET_MODE (op), byte);
164 : /* simplify_gen_subreg will return NULL RTX for the
165 : high half of the paradoxical subreg. */
166 2659268 : hi_half[num] = tmp ? tmp : gen_reg_rtx (half_mode);
167 : }
168 : }
169 4195086 : }
170 :
171 : /* Emit the double word assignment DST = { LO, HI }. */
172 :
173 : void
174 102720 : split_double_concat (machine_mode mode, rtx dst, rtx lo, rtx hi)
175 : {
176 102720 : rtx dlo, dhi;
177 102720 : int deleted_move_count = 0;
178 102720 : split_double_mode (mode, &dst, 1, &dlo, &dhi);
179 : /* Constraints ensure that if both lo and hi are MEMs, then
180 : dst has early-clobber and thus addresses of MEMs don't use
181 : dlo/dhi registers. Otherwise if at least one of li and hi are MEMs,
182 : dlo/dhi are registers. */
183 102720 : if (MEM_P (lo)
184 5580 : && rtx_equal_p (dlo, hi)
185 103690 : && reg_overlap_mentioned_p (dhi, lo))
186 : {
187 : /* If dlo is same as hi and lo's address uses dhi register,
188 : code below would first emit_move_insn (dhi, hi)
189 : and then emit_move_insn (dlo, lo). But the former
190 : would invalidate lo's address. Load into dhi first,
191 : then swap. */
192 193 : emit_move_insn (dhi, lo);
193 193 : lo = dhi;
194 : }
195 102527 : else if (MEM_P (hi)
196 9607 : && !MEM_P (lo)
197 6771 : && !rtx_equal_p (dlo, lo)
198 103885 : && reg_overlap_mentioned_p (dlo, hi))
199 : {
200 : /* In this case, code below would first emit_move_insn (dlo, lo)
201 : and then emit_move_insn (dhi, hi). But the former would
202 : invalidate hi's address. */
203 11 : if (rtx_equal_p (dhi, lo))
204 : {
205 : /* We can't load into dhi first, so load into dlo
206 : first and we'll swap. */
207 5 : emit_move_insn (dlo, hi);
208 5 : hi = dlo;
209 : }
210 : else
211 : {
212 : /* Load into dhi first. */
213 6 : emit_move_insn (dhi, hi);
214 6 : hi = dhi;
215 : }
216 : }
217 102720 : if (!rtx_equal_p (dlo, hi))
218 : {
219 88499 : if (!rtx_equal_p (dlo, lo))
220 38717 : emit_move_insn (dlo, lo);
221 : else
222 : deleted_move_count++;
223 88499 : if (!rtx_equal_p (dhi, hi))
224 82354 : emit_move_insn (dhi, hi);
225 : else
226 6145 : deleted_move_count++;
227 : }
228 14221 : else if (!rtx_equal_p (lo, dhi))
229 : {
230 7221 : if (!rtx_equal_p (dhi, hi))
231 7221 : emit_move_insn (dhi, hi);
232 : else
233 : deleted_move_count++;
234 7221 : if (!rtx_equal_p (dlo, lo))
235 7119 : emit_move_insn (dlo, lo);
236 : else
237 102 : deleted_move_count++;
238 : }
239 7000 : else if (mode == TImode)
240 6980 : emit_insn (gen_swapdi (dlo, dhi));
241 : else
242 20 : emit_insn (gen_swapsi (dlo, dhi));
243 :
244 102720 : if (deleted_move_count == 2)
245 3136 : emit_note (NOTE_INSN_DELETED);
246 102720 : }
247 :
248 :
249 : /* Generate either "mov $0, reg" or "xor reg, reg", as appropriate
250 : for the target. */
251 :
252 : void
253 116601 : ix86_expand_clear (rtx dest)
254 : {
255 116601 : rtx tmp;
256 :
257 : /* We play register width games, which are only valid after reload. */
258 116601 : gcc_assert (reload_completed);
259 :
260 : /* Avoid HImode and its attendant prefix byte. */
261 233202 : if (GET_MODE_SIZE (GET_MODE (dest)) < 4)
262 955 : dest = gen_rtx_REG (SImode, REGNO (dest));
263 116601 : tmp = gen_rtx_SET (dest, const0_rtx);
264 :
265 116601 : if (!TARGET_USE_MOV0 || optimize_insn_for_size_p ())
266 : {
267 116601 : rtx clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
268 116601 : tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, clob));
269 : }
270 :
271 116601 : emit_insn (tmp);
272 116601 : }
273 :
274 : /* Return true if V can be broadcasted from an integer of WIDTH bits
275 : which is returned in VAL_BROADCAST. Otherwise, return false. */
276 :
277 : static bool
278 4851 : ix86_broadcast (HOST_WIDE_INT v, unsigned int width,
279 : HOST_WIDE_INT &val_broadcast)
280 : {
281 4851 : wide_int val = wi::uhwi (v, HOST_BITS_PER_WIDE_INT);
282 4851 : val_broadcast = wi::extract_uhwi (val, 0, width);
283 6543 : for (unsigned int i = width; i < HOST_BITS_PER_WIDE_INT; i += width)
284 : {
285 5089 : HOST_WIDE_INT each = wi::extract_uhwi (val, i, width);
286 5089 : if (val_broadcast != each)
287 : return false;
288 : }
289 1454 : val_broadcast = sext_hwi (val_broadcast, width);
290 1454 : return true;
291 4851 : }
292 :
293 : /* Convert the CONST_WIDE_INT operand OP to broadcast in MODE. */
294 :
295 : rtx
296 36034 : ix86_convert_const_wide_int_to_broadcast (machine_mode mode, rtx op)
297 : {
298 : /* Don't use integer vector broadcast if we can't move from GPR to SSE
299 : register directly. */
300 36034 : if (!TARGET_INTER_UNIT_MOVES_TO_VEC)
301 : return nullptr;
302 :
303 36034 : unsigned int msize = GET_MODE_SIZE (mode);
304 :
305 : /* Only optimized for vpbroadcast[bwsd]/vbroadcastss with xmm/ymm/zmm. */
306 36034 : if (msize != 16 && msize != 32 && msize != 64)
307 : return nullptr;
308 :
309 : /* Convert CONST_WIDE_INT to a non-standard SSE constant integer
310 : broadcast only if vector broadcast is available. */
311 36034 : if (!TARGET_AVX
312 1662 : || !CONST_WIDE_INT_P (op)
313 1603 : || standard_sse_constant_p (op, mode)
314 37637 : || (CONST_WIDE_INT_NUNITS (op) * HOST_BITS_PER_WIDE_INT
315 1603 : != GET_MODE_BITSIZE (mode)))
316 34439 : return nullptr;
317 :
318 1595 : HOST_WIDE_INT val = CONST_WIDE_INT_ELT (op, 0);
319 1595 : HOST_WIDE_INT val_broadcast;
320 1595 : scalar_int_mode broadcast_mode;
321 : /* vpbroadcastb zmm requires TARGET_AVX512BW. */
322 712 : if ((msize == 64 ? TARGET_AVX512BW : TARGET_AVX2)
323 2089 : && ix86_broadcast (val, GET_MODE_BITSIZE (QImode),
324 : val_broadcast))
325 : broadcast_mode = QImode;
326 654 : else if ((msize == 64 ? TARGET_AVX512BW : TARGET_AVX2)
327 1968 : && ix86_broadcast (val, GET_MODE_BITSIZE (HImode),
328 : val_broadcast))
329 : broadcast_mode = HImode;
330 : /* vbroadcasts[sd] only support memory operand w/o AVX2.
331 : When msize == 16, pshufs is used for vec_duplicate.
332 : when msize == 64, vpbroadcastd is used, and TARGET_AVX512F must be existed. */
333 412 : else if ((msize != 32 || TARGET_AVX2)
334 1768 : && ix86_broadcast (val, GET_MODE_BITSIZE (SImode),
335 : val_broadcast))
336 : broadcast_mode = SImode;
337 1391 : else if (TARGET_64BIT && (msize != 32 || TARGET_AVX2)
338 2641 : && ix86_broadcast (val, GET_MODE_BITSIZE (DImode),
339 : val_broadcast))
340 : broadcast_mode = DImode;
341 : else
342 141 : return nullptr;
343 :
344 : /* Check if OP can be broadcasted from VAL. */
345 1776 : for (int i = 1; i < CONST_WIDE_INT_NUNITS (op); i++)
346 1561 : if (val != CONST_WIDE_INT_ELT (op, i))
347 : return nullptr;
348 :
349 215 : unsigned int nunits = (GET_MODE_SIZE (mode)
350 215 : / GET_MODE_SIZE (broadcast_mode));
351 215 : machine_mode vector_mode;
352 215 : if (!mode_for_vector (broadcast_mode, nunits).exists (&vector_mode))
353 0 : gcc_unreachable ();
354 215 : rtx target = gen_reg_rtx (vector_mode);
355 215 : bool ok = ix86_expand_vector_init_duplicate (false, vector_mode,
356 : target,
357 : GEN_INT (val_broadcast));
358 215 : if (!ok)
359 : return nullptr;
360 215 : target = lowpart_subreg (mode, target, vector_mode);
361 215 : return target;
362 : }
363 :
364 : void
365 74359025 : ix86_expand_move (machine_mode mode, rtx operands[])
366 : {
367 74359025 : rtx op0, op1;
368 74359025 : rtx tmp, addend = NULL_RTX;
369 74359025 : enum tls_model model;
370 :
371 74359025 : op0 = operands[0];
372 74359025 : op1 = operands[1];
373 :
374 : /* Avoid complex sets of likely spilled hard registers before reload. */
375 74359025 : if (!ix86_hardreg_mov_ok (op0, op1))
376 : {
377 140352 : tmp = gen_reg_rtx (mode);
378 140352 : operands[0] = tmp;
379 140352 : ix86_expand_move (mode, operands);
380 140352 : operands[0] = op0;
381 140352 : operands[1] = tmp;
382 140352 : op1 = tmp;
383 : }
384 :
385 74359025 : switch (GET_CODE (op1))
386 : {
387 349677 : case CONST:
388 349677 : tmp = XEXP (op1, 0);
389 :
390 349677 : if (GET_CODE (tmp) != PLUS
391 337942 : || !SYMBOL_REF_P (XEXP (tmp, 0)))
392 : break;
393 :
394 335287 : op1 = XEXP (tmp, 0);
395 335287 : addend = XEXP (tmp, 1);
396 : /* FALLTHRU */
397 :
398 5010924 : case SYMBOL_REF:
399 5010924 : model = SYMBOL_REF_TLS_MODEL (op1);
400 :
401 5010924 : if (model)
402 10166 : op1 = legitimize_tls_address (op1, model, true);
403 5000758 : else if (ix86_force_load_from_GOT_p (op1))
404 : {
405 : /* Load the external function address via GOT slot to avoid PLT. */
406 24 : op1 = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, op1),
407 : (TARGET_64BIT
408 : ? UNSPEC_GOTPCREL
409 : : UNSPEC_GOT));
410 24 : op1 = gen_rtx_CONST (Pmode, op1);
411 24 : op1 = gen_const_mem (Pmode, op1);
412 20 : set_mem_alias_set (op1, GOT_ALIAS_SET);
413 : }
414 : else
415 : {
416 : #if TARGET_PECOFF
417 : tmp = legitimize_pe_coff_symbol (op1, addend != NULL_RTX);
418 :
419 : if (tmp)
420 : {
421 : op1 = tmp;
422 : if (!addend)
423 : break;
424 : }
425 : else
426 : #endif
427 5000738 : {
428 5000738 : op1 = operands[1];
429 5000738 : break;
430 : }
431 : }
432 :
433 10186 : if (addend)
434 : {
435 2787 : op1 = force_operand (op1, NULL_RTX);
436 2796 : op1 = expand_simple_binop (Pmode, PLUS, op1, addend,
437 : op0, 1, OPTAB_DIRECT);
438 : }
439 : else
440 7399 : op1 = force_operand (op1, op0);
441 :
442 10186 : if (op1 == op0)
443 : return;
444 :
445 1152 : op1 = convert_to_mode (mode, op1, 1);
446 :
447 : default:
448 : break;
449 :
450 1576113 : case SUBREG:
451 : /* Transform TImode paradoxical SUBREG into zero_extendditi2. */
452 1576113 : if (TARGET_64BIT
453 1348319 : && mode == TImode
454 : && SUBREG_P (op1)
455 74977 : && GET_MODE (SUBREG_REG (op1)) == DImode
456 1622646 : && SUBREG_BYTE (op1) == 0)
457 46533 : op1 = gen_rtx_ZERO_EXTEND (TImode, SUBREG_REG (op1));
458 : /* As not all values in XFmode are representable in real_value,
459 : we might be called with unfoldable SUBREGs of constants. */
460 1576113 : if (mode == XFmode
461 3128 : && CONSTANT_P (SUBREG_REG (op1))
462 0 : && can_create_pseudo_p ())
463 : {
464 0 : machine_mode imode = GET_MODE (SUBREG_REG (op1));
465 0 : rtx r = force_const_mem (imode, SUBREG_REG (op1));
466 0 : if (r)
467 0 : r = validize_mem (r);
468 : else
469 0 : r = force_reg (imode, SUBREG_REG (op1));
470 0 : op1 = simplify_gen_subreg (mode, r, imode, SUBREG_BYTE (op1));
471 : }
472 : break;
473 : }
474 :
475 74349991 : if ((flag_pic || MACHOPIC_INDIRECT)
476 74349991 : && symbolic_operand (op1, mode))
477 : {
478 : #if TARGET_MACHO
479 : if (TARGET_MACHO && !TARGET_64BIT)
480 : {
481 : /* dynamic-no-pic */
482 : if (MACHOPIC_INDIRECT)
483 : {
484 : tmp = (op0 && REG_P (op0) && mode == Pmode)
485 : ? op0 : gen_reg_rtx (Pmode);
486 : op1 = machopic_indirect_data_reference (op1, tmp);
487 : if (MACHOPIC_PURE)
488 : op1 = machopic_legitimize_pic_address (op1, mode,
489 : tmp == op1 ? 0 : tmp);
490 : }
491 : if (op0 != op1 && !MEM_P (op0))
492 : {
493 : rtx insn = gen_rtx_SET (op0, op1);
494 : emit_insn (insn);
495 : return;
496 : }
497 : }
498 : #endif
499 :
500 335093 : if (MEM_P (op0))
501 87578 : op1 = force_reg (mode, op1);
502 247515 : else if (!(TARGET_64BIT && x86_64_movabs_operand (op1, DImode)))
503 : {
504 247458 : rtx reg = can_create_pseudo_p () ? NULL_RTX : op0;
505 247458 : op1 = legitimize_pic_address (op1, reg);
506 247458 : if (op0 == op1)
507 : return;
508 247458 : op1 = convert_to_mode (mode, op1, 1);
509 : }
510 : }
511 : else
512 : {
513 74014898 : if (MEM_P (op0)
514 100945291 : && (PUSH_ROUNDING (GET_MODE_SIZE (mode)) != GET_MODE_SIZE (mode)
515 10867464 : || !push_operand (op0, mode))
516 86467710 : && MEM_P (op1))
517 2147023 : op1 = force_reg (mode, op1);
518 :
519 74014898 : if (push_operand (op0, mode)
520 74014898 : && ! general_no_elim_operand (op1, mode))
521 1004 : op1 = copy_to_mode_reg (mode, op1);
522 :
523 : /* Force large constants in 64bit compilation into register
524 : to get them CSEed. */
525 74014898 : if (can_create_pseudo_p ()
526 68273738 : && (mode == DImode) && TARGET_64BIT
527 35827235 : && immediate_operand (op1, mode)
528 8073695 : && !x86_64_zext_immediate_operand (op1, VOIDmode)
529 728409 : && !register_operand (op0, mode)
530 74192574 : && optimize)
531 125654 : op1 = copy_to_mode_reg (mode, op1);
532 :
533 74014898 : if (can_create_pseudo_p ())
534 : {
535 68273738 : if (CONST_DOUBLE_P (op1))
536 : {
537 : /* If we are loading a floating point constant to a
538 : register, force the value to memory now, since we'll
539 : get better code out the back end. */
540 :
541 905050 : op1 = validize_mem (force_const_mem (mode, op1));
542 905050 : if (!register_operand (op0, mode))
543 : {
544 130648 : tmp = gen_reg_rtx (mode);
545 130648 : emit_insn (gen_rtx_SET (tmp, op1));
546 130648 : emit_move_insn (op0, tmp);
547 130648 : return;
548 : }
549 : }
550 : }
551 : }
552 :
553 : /* Special case inserting 64-bit values into a TImode register. */
554 74219343 : if (TARGET_64BIT
555 : /* Disable for -O0 (see PR110587) unless naked (PR110533). */
556 64496151 : && (optimize || ix86_function_naked (current_function_decl))
557 44244984 : && (mode == DImode || mode == DFmode)
558 30283373 : && SUBREG_P (op0)
559 509922 : && GET_MODE (SUBREG_REG (op0)) == TImode
560 411356 : && REG_P (SUBREG_REG (op0))
561 74630699 : && REG_P (op1))
562 : {
563 : /* Use *insvti_lowpart_1 to set lowpart. */
564 182555 : if (SUBREG_BYTE (op0) == 0)
565 : {
566 54769 : wide_int mask = wi::mask (64, true, 128);
567 54769 : tmp = immed_wide_int_const (mask, TImode);
568 54769 : op0 = SUBREG_REG (op0);
569 54769 : tmp = gen_rtx_AND (TImode, copy_rtx (op0), tmp);
570 54769 : if (mode == DFmode)
571 375 : op1 = gen_lowpart (DImode, op1);
572 54769 : op1 = gen_rtx_ZERO_EXTEND (TImode, op1);
573 54769 : op1 = gen_rtx_IOR (TImode, tmp, op1);
574 54769 : }
575 : /* Use *insvti_highpart_1 to set highpart. */
576 127786 : else if (SUBREG_BYTE (op0) == 8)
577 : {
578 127786 : wide_int mask = wi::mask (64, false, 128);
579 127786 : tmp = immed_wide_int_const (mask, TImode);
580 127786 : op0 = SUBREG_REG (op0);
581 127786 : tmp = gen_rtx_AND (TImode, copy_rtx (op0), tmp);
582 127786 : if (mode == DFmode)
583 226 : op1 = gen_lowpart (DImode, op1);
584 127786 : op1 = gen_rtx_ZERO_EXTEND (TImode, op1);
585 127786 : op1 = gen_rtx_ASHIFT (TImode, op1, GEN_INT (64));
586 127786 : op1 = gen_rtx_IOR (TImode, tmp, op1);
587 127786 : }
588 : }
589 :
590 74219343 : emit_insn (gen_rtx_SET (op0, op1));
591 : }
592 :
593 : /* OP is a memref of CONST_VECTOR, return scalar constant mem
594 : if CONST_VECTOR is a vec_duplicate, else return NULL. */
595 : rtx
596 2544587 : ix86_broadcast_from_constant (machine_mode mode, rtx op)
597 : {
598 2544587 : int nunits = GET_MODE_NUNITS (mode);
599 2544587 : if (nunits < 2)
600 : return nullptr;
601 :
602 : /* Don't use integer vector broadcast if we can't move from GPR to SSE
603 : register directly. */
604 2414244 : if (!TARGET_INTER_UNIT_MOVES_TO_VEC
605 8020 : && INTEGRAL_MODE_P (mode))
606 : return nullptr;
607 :
608 : /* Convert CONST_VECTOR to a non-standard SSE constant integer
609 : broadcast only if vector broadcast is available. */
610 2408834 : if (standard_sse_constant_p (op, mode))
611 : return nullptr;
612 :
613 4817662 : if (GET_MODE_INNER (mode) == TImode)
614 : return nullptr;
615 :
616 2408721 : rtx constant = get_pool_constant (XEXP (op, 0));
617 2408721 : if (!CONST_VECTOR_P (constant))
618 : return nullptr;
619 :
620 : /* There could be some rtx like
621 : (mem/u/c:V16QI (symbol_ref/u:DI ("*.LC1")))
622 : but with "*.LC1" refer to V2DI constant vector. */
623 2408721 : if (GET_MODE (constant) != mode)
624 : {
625 779 : constant = simplify_subreg (mode, constant, GET_MODE (constant),
626 : 0);
627 779 : if (constant == nullptr || !CONST_VECTOR_P (constant))
628 : return nullptr;
629 : }
630 :
631 2408721 : rtx first = XVECEXP (constant, 0, 0);
632 :
633 8289903 : for (int i = 1; i < nunits; ++i)
634 : {
635 7637670 : rtx tmp = XVECEXP (constant, 0, i);
636 : /* Vector duplicate value. */
637 7637670 : if (!rtx_equal_p (tmp, first))
638 : return nullptr;
639 : }
640 :
641 : return first;
642 : }
643 :
644 : void
645 4747092 : ix86_expand_vector_move (machine_mode mode, rtx operands[])
646 : {
647 4747092 : rtx op0 = operands[0], op1 = operands[1];
648 : /* Use GET_MODE_BITSIZE instead of GET_MODE_ALIGNMENT for IA MCU
649 : psABI since the biggest alignment is 4 byte for IA MCU psABI. */
650 4747092 : unsigned int align = (TARGET_IAMCU
651 4747092 : ? GET_MODE_BITSIZE (mode)
652 4747092 : : GET_MODE_ALIGNMENT (mode));
653 :
654 4747092 : if (push_operand (op0, VOIDmode))
655 2918 : op0 = emit_move_resolve_push (mode, op0);
656 :
657 : /* Force constants other than zero into memory. We do not know how
658 : the instructions used to build constants modify the upper 64 bits
659 : of the register, once we have that information we may be able
660 : to handle some of them more efficiently. */
661 4747092 : if (can_create_pseudo_p ()
662 4552020 : && (CONSTANT_P (op1)
663 4221433 : || (SUBREG_P (op1)
664 322783 : && CONSTANT_P (SUBREG_REG (op1))))
665 5077693 : && ((register_operand (op0, mode)
666 272581 : && !standard_sse_constant_p (op1, mode))
667 : /* ix86_expand_vector_move_misalign() does not like constants. */
668 : || (SSE_REG_MODE_P (mode)
669 272880 : && MEM_P (op0)
670 42430 : && MEM_ALIGN (op0) < align)))
671 : {
672 4989 : if (SUBREG_P (op1))
673 : {
674 14 : machine_mode imode = GET_MODE (SUBREG_REG (op1));
675 14 : rtx r = force_const_mem (imode, SUBREG_REG (op1));
676 14 : if (r)
677 14 : r = validize_mem (r);
678 : else
679 0 : r = force_reg (imode, SUBREG_REG (op1));
680 14 : op1 = simplify_gen_subreg (mode, r, imode, SUBREG_BYTE (op1));
681 : }
682 : else
683 : {
684 4975 : machine_mode mode = GET_MODE (op0);
685 4975 : rtx tmp = ix86_convert_const_wide_int_to_broadcast
686 4975 : (mode, op1);
687 4975 : if (tmp == nullptr)
688 4954 : op1 = validize_mem (force_const_mem (mode, op1));
689 : else
690 : op1 = tmp;
691 : }
692 : }
693 :
694 4747092 : if (can_create_pseudo_p ()
695 4552020 : && GET_MODE_SIZE (mode) >= 16
696 3818690 : && VECTOR_MODE_P (mode)
697 8350251 : && (MEM_P (op1)
698 749676 : && SYMBOL_REF_P (XEXP (op1, 0))
699 506531 : && CONSTANT_POOL_ADDRESS_P (XEXP (op1, 0))))
700 : {
701 489932 : rtx first = ix86_broadcast_from_constant (mode, op1);
702 489932 : if (first != nullptr)
703 : {
704 : /* Broadcast to XMM/YMM/ZMM register from an integer
705 : constant or scalar mem. */
706 128924 : rtx tmp = gen_reg_rtx (mode);
707 128924 : if (FLOAT_MODE_P (mode))
708 29496 : first = force_const_mem (GET_MODE_INNER (mode), first);
709 128924 : bool ok = ix86_expand_vector_init_duplicate (false, mode,
710 : tmp, first);
711 128924 : if (!ok && !TARGET_64BIT && GET_MODE_INNER (mode) == DImode)
712 : {
713 0 : first = force_const_mem (GET_MODE_INNER (mode), first);
714 0 : ok = ix86_expand_vector_init_duplicate (false, mode,
715 : tmp, first);
716 : }
717 128924 : if (ok)
718 : {
719 128924 : emit_move_insn (op0, tmp);
720 128924 : return;
721 : }
722 : }
723 : }
724 :
725 : /* We need to check memory alignment for SSE mode since attribute
726 : can make operands unaligned. */
727 4618168 : if (can_create_pseudo_p ()
728 : && SSE_REG_MODE_P (mode)
729 9295951 : && ((MEM_P (op0) && (MEM_ALIGN (op0) < align))
730 4153556 : || (MEM_P (op1) && (MEM_ALIGN (op1) < align))))
731 : {
732 326061 : rtx tmp[2];
733 :
734 : /* ix86_expand_vector_move_misalign() does not like both
735 : arguments in memory. */
736 326061 : if (!register_operand (op0, mode)
737 326061 : && !register_operand (op1, mode))
738 : {
739 96405 : rtx scratch = gen_reg_rtx (mode);
740 96405 : emit_move_insn (scratch, op1);
741 96405 : op1 = scratch;
742 : }
743 :
744 326061 : tmp[0] = op0; tmp[1] = op1;
745 326061 : ix86_expand_vector_move_misalign (mode, tmp);
746 326061 : return;
747 : }
748 :
749 : /* Special case TImode to 128-bit vector conversions via V2DI. */
750 1155396 : if (VECTOR_MODE_P (mode)
751 4240662 : && GET_MODE_SIZE (mode) == 16
752 3014735 : && SUBREG_P (op1)
753 254386 : && GET_MODE (SUBREG_REG (op1)) == TImode
754 3325 : && TARGET_64BIT && TARGET_SSE
755 4294778 : && ix86_pre_reload_split ())
756 : {
757 2557 : rtx tmp = gen_reg_rtx (V2DImode);
758 2557 : rtx lo = gen_reg_rtx (DImode);
759 2557 : rtx hi = gen_reg_rtx (DImode);
760 2557 : emit_move_insn (lo, gen_lowpart (DImode, SUBREG_REG (op1)));
761 2557 : emit_move_insn (hi, gen_highpart (DImode, SUBREG_REG (op1)));
762 2557 : emit_insn (gen_vec_concatv2di (tmp, lo, hi));
763 2557 : emit_move_insn (op0, gen_lowpart (mode, tmp));
764 2557 : return;
765 : }
766 :
767 : /* If operand0 is a hard register, make operand1 a pseudo. */
768 4289550 : if (can_create_pseudo_p ()
769 8384028 : && !ix86_hardreg_mov_ok (op0, op1))
770 : {
771 139 : rtx tmp = gen_reg_rtx (GET_MODE (op0));
772 139 : emit_move_insn (tmp, op1);
773 139 : emit_move_insn (op0, tmp);
774 139 : return;
775 : }
776 :
777 : /* Make operand1 a register if it isn't already. */
778 4289411 : if (can_create_pseudo_p ()
779 4094339 : && !register_operand (op0, mode)
780 5432067 : && !register_operand (op1, mode))
781 : {
782 220038 : rtx tmp = gen_reg_rtx (GET_MODE (op0));
783 220038 : emit_move_insn (tmp, op1);
784 220038 : emit_move_insn (op0, tmp);
785 220038 : return;
786 : }
787 :
788 4069373 : emit_insn (gen_rtx_SET (op0, op1));
789 : }
790 :
791 : /* Split 32-byte AVX unaligned load and store if needed. */
792 :
793 : static void
794 12676 : ix86_avx256_split_vector_move_misalign (rtx op0, rtx op1)
795 : {
796 12676 : rtx m;
797 12676 : rtx (*extract) (rtx, rtx, rtx);
798 12676 : machine_mode mode;
799 :
800 12676 : if ((MEM_P (op1) && !TARGET_AVX256_SPLIT_UNALIGNED_LOAD)
801 4630 : || (MEM_P (op0) && !TARGET_AVX256_SPLIT_UNALIGNED_STORE))
802 : {
803 12650 : emit_insn (gen_rtx_SET (op0, op1));
804 12650 : return;
805 : }
806 :
807 26 : rtx orig_op0 = NULL_RTX;
808 26 : mode = GET_MODE (op0);
809 26 : switch (GET_MODE_CLASS (mode))
810 : {
811 9 : case MODE_VECTOR_INT:
812 9 : case MODE_INT:
813 9 : if (mode != V32QImode)
814 : {
815 7 : if (!MEM_P (op0))
816 : {
817 3 : orig_op0 = op0;
818 3 : op0 = gen_reg_rtx (V32QImode);
819 : }
820 : else
821 4 : op0 = gen_lowpart (V32QImode, op0);
822 7 : op1 = gen_lowpart (V32QImode, op1);
823 7 : mode = V32QImode;
824 : }
825 : break;
826 : case MODE_VECTOR_FLOAT:
827 : break;
828 0 : default:
829 0 : gcc_unreachable ();
830 : }
831 :
832 26 : switch (mode)
833 : {
834 0 : default:
835 0 : gcc_unreachable ();
836 : case E_V32QImode:
837 : extract = gen_avx_vextractf128v32qi;
838 : mode = V16QImode;
839 : break;
840 1 : case E_V16BFmode:
841 1 : extract = gen_avx_vextractf128v16bf;
842 1 : mode = V8BFmode;
843 1 : break;
844 0 : case E_V16HFmode:
845 0 : extract = gen_avx_vextractf128v16hf;
846 0 : mode = V8HFmode;
847 0 : break;
848 8 : case E_V8SFmode:
849 8 : extract = gen_avx_vextractf128v8sf;
850 8 : mode = V4SFmode;
851 8 : break;
852 8 : case E_V4DFmode:
853 8 : extract = gen_avx_vextractf128v4df;
854 8 : mode = V2DFmode;
855 8 : break;
856 : }
857 :
858 26 : if (MEM_P (op1))
859 : {
860 9 : rtx r = gen_reg_rtx (mode);
861 9 : m = adjust_address (op1, mode, 0);
862 9 : emit_move_insn (r, m);
863 9 : m = adjust_address (op1, mode, 16);
864 9 : r = gen_rtx_VEC_CONCAT (GET_MODE (op0), r, m);
865 9 : emit_move_insn (op0, r);
866 : }
867 17 : else if (MEM_P (op0))
868 : {
869 17 : m = adjust_address (op0, mode, 0);
870 17 : emit_insn (extract (m, op1, const0_rtx));
871 17 : m = adjust_address (op0, mode, 16);
872 17 : emit_insn (extract (m, copy_rtx (op1), const1_rtx));
873 : }
874 : else
875 0 : gcc_unreachable ();
876 :
877 26 : if (orig_op0)
878 3 : emit_move_insn (orig_op0, gen_lowpart (GET_MODE (orig_op0), op0));
879 : }
880 :
881 : /* Implement the movmisalign patterns for SSE. Non-SSE modes go
882 : straight to ix86_expand_vector_move. */
883 : /* Code generation for scalar reg-reg moves of single and double precision data:
884 : if (x86_sse_partial_reg_dependency == true | x86_sse_split_regs == true)
885 : movaps reg, reg
886 : else
887 : movss reg, reg
888 : if (x86_sse_partial_reg_dependency == true)
889 : movapd reg, reg
890 : else
891 : movsd reg, reg
892 :
893 : Code generation for scalar loads of double precision data:
894 : if (x86_sse_split_regs == true)
895 : movlpd mem, reg (gas syntax)
896 : else
897 : movsd mem, reg
898 :
899 : Code generation for unaligned packed loads of single precision data
900 : (x86_sse_unaligned_move_optimal overrides x86_sse_partial_reg_dependency):
901 : if (x86_sse_unaligned_move_optimal)
902 : movups mem, reg
903 :
904 : if (x86_sse_partial_reg_dependency == true)
905 : {
906 : xorps reg, reg
907 : movlps mem, reg
908 : movhps mem+8, reg
909 : }
910 : else
911 : {
912 : movlps mem, reg
913 : movhps mem+8, reg
914 : }
915 :
916 : Code generation for unaligned packed loads of double precision data
917 : (x86_sse_unaligned_move_optimal overrides x86_sse_split_regs):
918 : if (x86_sse_unaligned_move_optimal)
919 : movupd mem, reg
920 :
921 : if (x86_sse_split_regs == true)
922 : {
923 : movlpd mem, reg
924 : movhpd mem+8, reg
925 : }
926 : else
927 : {
928 : movsd mem, reg
929 : movhpd mem+8, reg
930 : }
931 : */
932 :
933 : void
934 656888 : ix86_expand_vector_move_misalign (machine_mode mode, rtx operands[])
935 : {
936 656888 : rtx op0, op1, m;
937 :
938 656888 : op0 = operands[0];
939 656888 : op1 = operands[1];
940 :
941 : /* Use unaligned load/store for AVX512 or when optimizing for size. */
942 1313776 : if (GET_MODE_SIZE (mode) == 64 || optimize_insn_for_size_p ())
943 : {
944 25056 : emit_insn (gen_rtx_SET (op0, op1));
945 25056 : return;
946 : }
947 :
948 631832 : if (TARGET_AVX)
949 : {
950 61940 : if (GET_MODE_SIZE (mode) == 32)
951 12676 : ix86_avx256_split_vector_move_misalign (op0, op1);
952 : else
953 : /* Always use 128-bit mov<mode>_internal pattern for AVX. */
954 18294 : emit_insn (gen_rtx_SET (op0, op1));
955 30970 : return;
956 : }
957 :
958 600862 : if (TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
959 95 : || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL)
960 : {
961 600767 : emit_insn (gen_rtx_SET (op0, op1));
962 600767 : return;
963 : }
964 :
965 : /* ??? If we have typed data, then it would appear that using
966 : movdqu is the only way to get unaligned data loaded with
967 : integer type. */
968 95 : if (TARGET_SSE2 && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
969 : {
970 81 : emit_insn (gen_rtx_SET (op0, op1));
971 81 : return;
972 : }
973 :
974 14 : if (MEM_P (op1))
975 : {
976 6 : if (TARGET_SSE2 && mode == V2DFmode)
977 : {
978 2 : rtx zero;
979 :
980 : /* When SSE registers are split into halves, we can avoid
981 : writing to the top half twice. */
982 2 : if (TARGET_SSE_SPLIT_REGS)
983 : {
984 2 : emit_clobber (op0);
985 2 : zero = op0;
986 : }
987 : else
988 : {
989 : /* ??? Not sure about the best option for the Intel chips.
990 : The following would seem to satisfy; the register is
991 : entirely cleared, breaking the dependency chain. We
992 : then store to the upper half, with a dependency depth
993 : of one. A rumor has it that Intel recommends two movsd
994 : followed by an unpacklpd, but this is unconfirmed. And
995 : given that the dependency depth of the unpacklpd would
996 : still be one, I'm not sure why this would be better. */
997 0 : zero = CONST0_RTX (V2DFmode);
998 : }
999 :
1000 2 : m = adjust_address (op1, DFmode, 0);
1001 2 : emit_insn (gen_sse2_loadlpd (op0, zero, m));
1002 2 : m = adjust_address (op1, DFmode, 8);
1003 2 : emit_insn (gen_sse2_loadhpd (op0, op0, m));
1004 2 : }
1005 : else
1006 : {
1007 4 : rtx t;
1008 :
1009 4 : if (mode != V4SFmode)
1010 0 : t = gen_reg_rtx (V4SFmode);
1011 : else
1012 : t = op0;
1013 :
1014 4 : if (TARGET_SSE_PARTIAL_REG_DEPENDENCY)
1015 2 : emit_move_insn (t, CONST0_RTX (V4SFmode));
1016 : else
1017 2 : emit_clobber (t);
1018 :
1019 4 : m = adjust_address (op1, V2SFmode, 0);
1020 4 : emit_insn (gen_sse_loadlps (t, t, m));
1021 4 : m = adjust_address (op1, V2SFmode, 8);
1022 4 : emit_insn (gen_sse_loadhps (t, t, m));
1023 4 : if (mode != V4SFmode)
1024 0 : emit_move_insn (op0, gen_lowpart (mode, t));
1025 : }
1026 : }
1027 8 : else if (MEM_P (op0))
1028 : {
1029 8 : if (TARGET_SSE2 && mode == V2DFmode)
1030 : {
1031 2 : m = adjust_address (op0, DFmode, 0);
1032 2 : emit_insn (gen_sse2_storelpd (m, op1));
1033 2 : m = adjust_address (op0, DFmode, 8);
1034 2 : emit_insn (gen_sse2_storehpd (m, op1));
1035 : }
1036 : else
1037 : {
1038 6 : if (mode != V4SFmode)
1039 0 : op1 = gen_lowpart (V4SFmode, op1);
1040 :
1041 6 : m = adjust_address (op0, V2SFmode, 0);
1042 6 : emit_insn (gen_sse_storelps (m, op1));
1043 6 : m = adjust_address (op0, V2SFmode, 8);
1044 6 : emit_insn (gen_sse_storehps (m, copy_rtx (op1)));
1045 : }
1046 : }
1047 : else
1048 0 : gcc_unreachable ();
1049 : }
1050 :
1051 : /* Move bits 64:95 to bits 32:63. */
1052 :
1053 : void
1054 850 : ix86_move_vector_high_sse_to_mmx (rtx op)
1055 : {
1056 850 : rtx mask = gen_rtx_PARALLEL (VOIDmode,
1057 : gen_rtvec (4, GEN_INT (0), GEN_INT (2),
1058 : GEN_INT (0), GEN_INT (0)));
1059 850 : rtx dest = lowpart_subreg (V4SImode, op, GET_MODE (op));
1060 850 : op = gen_rtx_VEC_SELECT (V4SImode, dest, mask);
1061 850 : rtx insn = gen_rtx_SET (dest, op);
1062 850 : emit_insn (insn);
1063 850 : }
1064 :
1065 : /* Split MMX pack with signed/unsigned saturation with SSE/SSE2. */
1066 :
1067 : void
1068 760 : ix86_split_mmx_pack (rtx operands[], enum rtx_code code)
1069 : {
1070 760 : rtx op0 = operands[0];
1071 760 : rtx op1 = operands[1];
1072 760 : rtx op2 = operands[2];
1073 760 : rtx src;
1074 :
1075 760 : machine_mode dmode = GET_MODE (op0);
1076 760 : machine_mode smode = GET_MODE (op1);
1077 760 : machine_mode inner_dmode = GET_MODE_INNER (dmode);
1078 760 : machine_mode inner_smode = GET_MODE_INNER (smode);
1079 :
1080 : /* Get the corresponding SSE mode for destination. */
1081 760 : int nunits = 16 / GET_MODE_SIZE (inner_dmode);
1082 1520 : machine_mode sse_dmode = mode_for_vector (GET_MODE_INNER (dmode),
1083 1520 : nunits).require ();
1084 760 : machine_mode sse_half_dmode = mode_for_vector (GET_MODE_INNER (dmode),
1085 1520 : nunits / 2).require ();
1086 :
1087 : /* Get the corresponding SSE mode for source. */
1088 760 : nunits = 16 / GET_MODE_SIZE (inner_smode);
1089 1520 : machine_mode sse_smode = mode_for_vector (GET_MODE_INNER (smode),
1090 1520 : nunits).require ();
1091 :
1092 : /* Generate SSE pack with signed/unsigned saturation. */
1093 760 : rtx dest = lowpart_subreg (sse_dmode, op0, GET_MODE (op0));
1094 760 : op1 = lowpart_subreg (sse_smode, op1, GET_MODE (op1));
1095 760 : op2 = lowpart_subreg (sse_smode, op2, GET_MODE (op2));
1096 :
1097 : /* paskusdw/packuswb does unsigned saturation of a signed source
1098 : which is different from generic us_truncate RTX. */
1099 760 : if (code == US_TRUNCATE)
1100 658 : src = gen_rtx_UNSPEC (sse_dmode,
1101 : gen_rtvec (2, op1, op2),
1102 : UNSPEC_US_TRUNCATE);
1103 : else
1104 : {
1105 102 : op1 = gen_rtx_fmt_e (code, sse_half_dmode, op1);
1106 102 : op2 = gen_rtx_fmt_e (code, sse_half_dmode, op2);
1107 102 : src = gen_rtx_VEC_CONCAT (sse_dmode, op1, op2);
1108 : }
1109 :
1110 760 : emit_move_insn (dest, src);
1111 :
1112 760 : ix86_move_vector_high_sse_to_mmx (op0);
1113 760 : }
1114 :
1115 : /* Split MMX punpcklXX/punpckhXX with SSE punpcklXX. This is also used
1116 : for a full unpack of OPERANDS[1] and OPERANDS[2] into a wider
1117 : OPERANDS[0]. */
1118 :
1119 : void
1120 6161 : ix86_split_mmx_punpck (rtx operands[], bool high_p)
1121 : {
1122 6161 : rtx op0 = operands[0];
1123 6161 : rtx op1 = operands[1];
1124 6161 : rtx op2 = operands[2];
1125 6161 : machine_mode mode = GET_MODE (op1);
1126 6161 : rtx mask;
1127 : /* The corresponding SSE mode. */
1128 6161 : machine_mode sse_mode, double_sse_mode;
1129 :
1130 6161 : switch (mode)
1131 : {
1132 1611 : case E_V8QImode:
1133 1611 : case E_V4QImode:
1134 1611 : case E_V2QImode:
1135 1611 : sse_mode = V16QImode;
1136 1611 : double_sse_mode = V32QImode;
1137 1611 : mask = gen_rtx_PARALLEL (VOIDmode,
1138 : gen_rtvec (16,
1139 : GEN_INT (0), GEN_INT (16),
1140 : GEN_INT (1), GEN_INT (17),
1141 : GEN_INT (2), GEN_INT (18),
1142 : GEN_INT (3), GEN_INT (19),
1143 : GEN_INT (4), GEN_INT (20),
1144 : GEN_INT (5), GEN_INT (21),
1145 : GEN_INT (6), GEN_INT (22),
1146 : GEN_INT (7), GEN_INT (23)));
1147 1611 : break;
1148 :
1149 3251 : case E_V4HImode:
1150 3251 : case E_V2HImode:
1151 3251 : sse_mode = V8HImode;
1152 3251 : double_sse_mode = V16HImode;
1153 3251 : mask = gen_rtx_PARALLEL (VOIDmode,
1154 : gen_rtvec (8,
1155 : GEN_INT (0), GEN_INT (8),
1156 : GEN_INT (1), GEN_INT (9),
1157 : GEN_INT (2), GEN_INT (10),
1158 : GEN_INT (3), GEN_INT (11)));
1159 3251 : break;
1160 :
1161 917 : case E_V2SImode:
1162 917 : sse_mode = V4SImode;
1163 917 : double_sse_mode = V8SImode;
1164 917 : mask = gen_rtx_PARALLEL (VOIDmode,
1165 : gen_rtvec (4,
1166 : GEN_INT (0), GEN_INT (4),
1167 : GEN_INT (1), GEN_INT (5)));
1168 917 : break;
1169 :
1170 382 : case E_V2SFmode:
1171 382 : sse_mode = V4SFmode;
1172 382 : double_sse_mode = V8SFmode;
1173 382 : mask = gen_rtx_PARALLEL (VOIDmode,
1174 : gen_rtvec (4,
1175 : GEN_INT (0), GEN_INT (4),
1176 : GEN_INT (1), GEN_INT (5)));
1177 382 : break;
1178 :
1179 0 : default:
1180 0 : gcc_unreachable ();
1181 : }
1182 :
1183 : /* Generate SSE punpcklXX. */
1184 6161 : rtx dest = lowpart_subreg (sse_mode, op0, GET_MODE (op0));
1185 6161 : op1 = lowpart_subreg (sse_mode, op1, GET_MODE (op1));
1186 6161 : op2 = lowpart_subreg (sse_mode, op2, GET_MODE (op2));
1187 :
1188 6161 : op1 = gen_rtx_VEC_CONCAT (double_sse_mode, op1, op2);
1189 6161 : op2 = gen_rtx_VEC_SELECT (sse_mode, op1, mask);
1190 6161 : rtx insn = gen_rtx_SET (dest, op2);
1191 6161 : emit_insn (insn);
1192 :
1193 : /* Move high bits to low bits. */
1194 6161 : if (high_p)
1195 : {
1196 2403 : if (sse_mode == V4SFmode)
1197 : {
1198 104 : mask = gen_rtx_PARALLEL (VOIDmode,
1199 : gen_rtvec (4, GEN_INT (2), GEN_INT (3),
1200 : GEN_INT (4), GEN_INT (5)));
1201 104 : op2 = gen_rtx_VEC_CONCAT (V8SFmode, dest, dest);
1202 104 : op1 = gen_rtx_VEC_SELECT (V4SFmode, op2, mask);
1203 : }
1204 : else
1205 : {
1206 2299 : int sz = GET_MODE_SIZE (mode);
1207 :
1208 2299 : if (sz == 4)
1209 239 : mask = gen_rtx_PARALLEL (VOIDmode,
1210 : gen_rtvec (4, GEN_INT (1), GEN_INT (0),
1211 : GEN_INT (0), GEN_INT (1)));
1212 2060 : else if (sz == 8)
1213 2060 : mask = gen_rtx_PARALLEL (VOIDmode,
1214 : gen_rtvec (4, GEN_INT (2), GEN_INT (3),
1215 : GEN_INT (0), GEN_INT (1)));
1216 : else
1217 0 : gcc_unreachable ();
1218 :
1219 2299 : dest = lowpart_subreg (V4SImode, dest, GET_MODE (dest));
1220 2299 : op1 = gen_rtx_VEC_SELECT (V4SImode, dest, mask);
1221 : }
1222 :
1223 2403 : insn = gen_rtx_SET (dest, op1);
1224 2403 : emit_insn (insn);
1225 : }
1226 6161 : }
1227 :
1228 : /* Helper function of ix86_fixup_binary_operands to canonicalize
1229 : operand order. Returns true if the operands should be swapped. */
1230 :
1231 : static bool
1232 177274903 : ix86_swap_binary_operands_p (enum rtx_code code, machine_mode mode,
1233 : rtx operands[])
1234 : {
1235 177274903 : rtx dst = operands[0];
1236 177274903 : rtx src1 = operands[1];
1237 177274903 : rtx src2 = operands[2];
1238 :
1239 : /* If the operation is not commutative, we can't do anything. */
1240 177274903 : if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
1241 28058664 : && GET_RTX_CLASS (code) != RTX_COMM_COMPARE)
1242 : return false;
1243 :
1244 : /* Highest priority is that src1 should match dst. */
1245 149228859 : if (rtx_equal_p (dst, src1))
1246 : return false;
1247 108839604 : if (rtx_equal_p (dst, src2))
1248 : return true;
1249 :
1250 : /* Next highest priority is that immediate constants come second. */
1251 108752770 : if (immediate_operand (src2, mode))
1252 : return false;
1253 26249066 : if (immediate_operand (src1, mode))
1254 : return true;
1255 :
1256 : /* Lowest priority is that memory references should come second. */
1257 26249066 : if (MEM_P (src2))
1258 : return false;
1259 24785977 : if (MEM_P (src1))
1260 : return true;
1261 :
1262 : return false;
1263 : }
1264 :
1265 : /* Fix up OPERANDS to satisfy ix86_binary_operator_ok. Return the
1266 : destination to use for the operation. If different from the true
1267 : destination in operands[0], a copy operation will be required except
1268 : under TARGET_APX_NDD. */
1269 :
1270 : rtx
1271 13648075 : ix86_fixup_binary_operands (enum rtx_code code, machine_mode mode,
1272 : rtx operands[], bool use_ndd)
1273 : {
1274 13648075 : rtx dst = operands[0];
1275 13648075 : rtx src1 = operands[1];
1276 13648075 : rtx src2 = operands[2];
1277 :
1278 : /* Canonicalize operand order. */
1279 13648075 : if (ix86_swap_binary_operands_p (code, mode, operands))
1280 : {
1281 : /* It is invalid to swap operands of different modes. */
1282 89183 : gcc_assert (GET_MODE (src1) == GET_MODE (src2));
1283 :
1284 : std::swap (src1, src2);
1285 : }
1286 :
1287 : /* Both source operands cannot be in memory. */
1288 13648075 : if (MEM_P (src1) && MEM_P (src2))
1289 : {
1290 : /* Optimization: Only read from memory once. */
1291 113640 : if (rtx_equal_p (src1, src2))
1292 : {
1293 17 : src2 = force_reg (mode, src2);
1294 17 : src1 = src2;
1295 : }
1296 113623 : else if (rtx_equal_p (dst, src1))
1297 3464 : src2 = force_reg (mode, src2);
1298 : else
1299 110159 : src1 = force_reg (mode, src1);
1300 : }
1301 :
1302 : /* If the destination is memory, and we do not have matching source
1303 : operands, do things in registers. */
1304 13648075 : if (MEM_P (dst) && !rtx_equal_p (dst, src1))
1305 492162 : dst = gen_reg_rtx (mode);
1306 :
1307 : /* Source 1 cannot be a constant. */
1308 13648075 : if (CONSTANT_P (src1))
1309 716 : src1 = force_reg (mode, src1);
1310 :
1311 : /* Source 1 cannot be a non-matching memory. */
1312 13648075 : if (!use_ndd && MEM_P (src1) && !rtx_equal_p (dst, src1))
1313 449418 : src1 = force_reg (mode, src1);
1314 :
1315 : /* Improve address combine. */
1316 13648075 : if (code == PLUS
1317 10023522 : && GET_MODE_CLASS (mode) == MODE_INT
1318 9911062 : && MEM_P (src2))
1319 179657 : src2 = force_reg (mode, src2);
1320 :
1321 13648075 : operands[1] = src1;
1322 13648075 : operands[2] = src2;
1323 13648075 : return dst;
1324 : }
1325 :
1326 : /* Similarly, but assume that the destination has already been
1327 : set up properly. */
1328 :
1329 : void
1330 298384 : ix86_fixup_binary_operands_no_copy (enum rtx_code code,
1331 : machine_mode mode, rtx operands[],
1332 : bool use_ndd)
1333 : {
1334 298384 : rtx dst = ix86_fixup_binary_operands (code, mode, operands, use_ndd);
1335 298384 : gcc_assert (dst == operands[0]);
1336 298384 : }
1337 :
1338 : /* Attempt to expand a binary operator. Make the expansion closer to the
1339 : actual machine, then just general_operand, which will allow 3 separate
1340 : memory references (one output, two input) in a single insn. */
1341 :
1342 : void
1343 13349562 : ix86_expand_binary_operator (enum rtx_code code, machine_mode mode,
1344 : rtx operands[], bool use_ndd)
1345 : {
1346 13349562 : rtx src1, src2, dst, op, clob;
1347 :
1348 13349562 : dst = ix86_fixup_binary_operands (code, mode, operands, use_ndd);
1349 13349562 : src1 = operands[1];
1350 13349562 : src2 = operands[2];
1351 :
1352 : /* Emit the instruction. */
1353 :
1354 13349562 : op = gen_rtx_SET (dst, gen_rtx_fmt_ee (code, mode, src1, src2));
1355 :
1356 13349562 : if (reload_completed
1357 82340 : && code == PLUS
1358 908 : && !rtx_equal_p (dst, src1)
1359 13349562 : && !use_ndd)
1360 : {
1361 : /* This is going to be an LEA; avoid splitting it later. */
1362 0 : emit_insn (op);
1363 : }
1364 : else
1365 : {
1366 13349562 : clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
1367 13349562 : emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
1368 : }
1369 :
1370 : /* Fix up the destination if needed. */
1371 13349562 : if (dst != operands[0])
1372 492153 : emit_move_insn (operands[0], dst);
1373 13349562 : }
1374 :
1375 : /* Expand vector logical operation CODE (AND, IOR, XOR) in MODE with
1376 : the given OPERANDS. */
1377 :
1378 : void
1379 85278 : ix86_expand_vector_logical_operator (enum rtx_code code, machine_mode mode,
1380 : rtx operands[])
1381 : {
1382 85278 : rtx op1 = NULL_RTX, op2 = NULL_RTX;
1383 85278 : if (SUBREG_P (operands[1]))
1384 : {
1385 372 : op1 = operands[1];
1386 372 : op2 = operands[2];
1387 : }
1388 84906 : else if (SUBREG_P (operands[2]))
1389 : {
1390 : op1 = operands[2];
1391 : op2 = operands[1];
1392 : }
1393 : /* Optimize (__m128i) d | (__m128i) e and similar code
1394 : when d and e are float vectors into float vector logical
1395 : insn. In C/C++ without using intrinsics there is no other way
1396 : to express vector logical operation on float vectors than
1397 : to cast them temporarily to integer vectors. */
1398 3332 : if (op1
1399 3332 : && !TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
1400 3332 : && (SUBREG_P (op2) || CONST_VECTOR_P (op2))
1401 358 : && GET_MODE_CLASS (GET_MODE (SUBREG_REG (op1))) == MODE_VECTOR_FLOAT
1402 453 : && GET_MODE_SIZE (GET_MODE (SUBREG_REG (op1))) == GET_MODE_SIZE (mode)
1403 151 : && SUBREG_BYTE (op1) == 0
1404 151 : && (CONST_VECTOR_P (op2)
1405 1 : || (GET_MODE (SUBREG_REG (op1)) == GET_MODE (SUBREG_REG (op2))
1406 1 : && SUBREG_BYTE (op2) == 0))
1407 151 : && can_create_pseudo_p ())
1408 : {
1409 151 : rtx dst;
1410 151 : switch (GET_MODE (SUBREG_REG (op1)))
1411 : {
1412 67 : case E_V4SFmode:
1413 67 : case E_V8SFmode:
1414 67 : case E_V16SFmode:
1415 67 : case E_V2DFmode:
1416 67 : case E_V4DFmode:
1417 67 : case E_V8DFmode:
1418 67 : dst = gen_reg_rtx (GET_MODE (SUBREG_REG (op1)));
1419 67 : if (CONST_VECTOR_P (op2))
1420 : {
1421 66 : op2 = gen_lowpart (GET_MODE (dst), op2);
1422 66 : op2 = force_reg (GET_MODE (dst), op2);
1423 : }
1424 : else
1425 : {
1426 1 : op1 = operands[1];
1427 1 : op2 = SUBREG_REG (operands[2]);
1428 1 : if (!vector_operand (op2, GET_MODE (dst)))
1429 0 : op2 = force_reg (GET_MODE (dst), op2);
1430 : }
1431 67 : op1 = SUBREG_REG (op1);
1432 67 : if (!vector_operand (op1, GET_MODE (dst)))
1433 0 : op1 = force_reg (GET_MODE (dst), op1);
1434 67 : emit_insn (gen_rtx_SET (dst,
1435 : gen_rtx_fmt_ee (code, GET_MODE (dst),
1436 : op1, op2)));
1437 67 : emit_move_insn (operands[0], gen_lowpart (mode, dst));
1438 67 : return;
1439 : default:
1440 : break;
1441 : }
1442 : }
1443 85211 : if (!vector_operand (operands[1], mode))
1444 1 : operands[1] = force_reg (mode, operands[1]);
1445 85211 : if (!vector_operand (operands[2], mode))
1446 12885 : operands[2] = force_reg (mode, operands[2]);
1447 85211 : ix86_fixup_binary_operands_no_copy (code, mode, operands);
1448 85211 : emit_insn (gen_rtx_SET (operands[0],
1449 : gen_rtx_fmt_ee (code, mode, operands[1],
1450 : operands[2])));
1451 : }
1452 :
1453 : /* Return TRUE or FALSE depending on whether the binary operator meets the
1454 : appropriate constraints. */
1455 :
1456 : bool
1457 164676046 : ix86_binary_operator_ok (enum rtx_code code, machine_mode mode,
1458 : rtx operands[3], bool use_ndd)
1459 : {
1460 164676046 : rtx dst = operands[0];
1461 164676046 : rtx src1 = operands[1];
1462 164676046 : rtx src2 = operands[2];
1463 :
1464 : /* Both source operands cannot be in memory. */
1465 157203646 : if ((MEM_P (src1) || bcst_mem_operand (src1, mode))
1466 164676435 : && (MEM_P (src2) || bcst_mem_operand (src2, mode)))
1467 1049218 : return false;
1468 :
1469 : /* Canonicalize operand order for commutative operators. */
1470 163626828 : if (ix86_swap_binary_operands_p (code, mode, operands))
1471 549884 : std::swap (src1, src2);
1472 :
1473 : /* If the destination is memory, we must have a matching source operand. */
1474 163626828 : if (MEM_P (dst) && !rtx_equal_p (dst, src1))
1475 : return false;
1476 :
1477 : /* Source 1 cannot be a constant. */
1478 158490410 : if (CONSTANT_P (src1))
1479 : return false;
1480 :
1481 : /* Source 1 cannot be a non-matching memory. */
1482 158487361 : if (!use_ndd && MEM_P (src1) && !rtx_equal_p (dst, src1))
1483 : /* Support "andhi/andsi/anddi" as a zero-extending move. */
1484 4582654 : return (code == AND
1485 532248 : && (mode == HImode
1486 532248 : || mode == SImode
1487 321201 : || (TARGET_64BIT && mode == DImode))
1488 4890530 : && satisfies_constraint_L (src2));
1489 :
1490 : return true;
1491 : }
1492 :
1493 : /* Attempt to expand a unary operator. Make the expansion closer to the
1494 : actual machine, then just general_operand, which will allow 2 separate
1495 : memory references (one output, one input) in a single insn. */
1496 :
1497 : void
1498 131916 : ix86_expand_unary_operator (enum rtx_code code, machine_mode mode,
1499 : rtx operands[], bool use_ndd)
1500 : {
1501 131916 : bool matching_memory = false;
1502 131916 : rtx src, dst, op, clob;
1503 :
1504 131916 : dst = operands[0];
1505 131916 : src = operands[1];
1506 :
1507 : /* If the destination is memory, and we do not have matching source
1508 : operands, do things in registers. */
1509 131916 : if (MEM_P (dst))
1510 : {
1511 11495 : if (rtx_equal_p (dst, src))
1512 : matching_memory = true;
1513 : else
1514 11179 : dst = gen_reg_rtx (mode);
1515 : }
1516 :
1517 : /* When source operand is memory, destination must match. */
1518 131916 : if (!use_ndd && MEM_P (src) && !matching_memory)
1519 4595 : src = force_reg (mode, src);
1520 :
1521 : /* Emit the instruction. */
1522 :
1523 131916 : op = gen_rtx_SET (dst, gen_rtx_fmt_e (code, mode, src));
1524 :
1525 131916 : if (code == NOT)
1526 71469 : emit_insn (op);
1527 : else
1528 : {
1529 60447 : clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
1530 60447 : emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
1531 : }
1532 :
1533 : /* Fix up the destination if needed. */
1534 131916 : if (dst != operands[0])
1535 11179 : emit_move_insn (operands[0], dst);
1536 131916 : }
1537 :
1538 : /* Return TRUE or FALSE depending on whether the unary operator meets the
1539 : appropriate constraints. */
1540 :
1541 : bool
1542 2070870 : ix86_unary_operator_ok (enum rtx_code,
1543 : machine_mode,
1544 : rtx operands[2],
1545 : bool use_ndd)
1546 : {
1547 : /* If one of operands is memory, source and destination must match. */
1548 2070870 : if ((MEM_P (operands[0])
1549 1935507 : || (!use_ndd && MEM_P (operands[1])))
1550 2099989 : && ! rtx_equal_p (operands[0], operands[1]))
1551 : return false;
1552 : return true;
1553 : }
1554 :
1555 : /* Predict just emitted jump instruction to be taken with probability PROB. */
1556 :
1557 : static void
1558 13105 : predict_jump (int prob)
1559 : {
1560 13105 : rtx_insn *insn = get_last_insn ();
1561 13105 : gcc_assert (JUMP_P (insn));
1562 13105 : add_reg_br_prob_note (insn, profile_probability::from_reg_br_prob_base (prob));
1563 13105 : }
1564 :
1565 : /* Split 32bit/64bit divmod with 8bit unsigned divmod if dividend and
1566 : divisor are within the range [0-255]. */
1567 :
1568 : void
1569 27 : ix86_split_idivmod (machine_mode mode, rtx operands[],
1570 : bool unsigned_p)
1571 : {
1572 27 : rtx_code_label *end_label, *qimode_label;
1573 27 : rtx div, mod;
1574 27 : rtx_insn *insn;
1575 27 : rtx scratch, tmp0, tmp1, tmp2;
1576 27 : rtx (*gen_divmod4_1) (rtx, rtx, rtx, rtx);
1577 :
1578 27 : operands[2] = force_reg (mode, operands[2]);
1579 27 : operands[3] = force_reg (mode, operands[3]);
1580 :
1581 27 : switch (mode)
1582 : {
1583 20 : case E_SImode:
1584 20 : if (GET_MODE (operands[0]) == SImode)
1585 : {
1586 16 : if (GET_MODE (operands[1]) == SImode)
1587 14 : gen_divmod4_1 = unsigned_p ? gen_udivmodsi4_1 : gen_divmodsi4_1;
1588 : else
1589 2 : gen_divmod4_1
1590 2 : = unsigned_p ? gen_udivmodsi4_zext_2 : gen_divmodsi4_zext_2;
1591 : }
1592 : else
1593 4 : gen_divmod4_1
1594 4 : = unsigned_p ? gen_udivmodsi4_zext_1 : gen_divmodsi4_zext_1;
1595 : break;
1596 :
1597 7 : case E_DImode:
1598 7 : gen_divmod4_1 = unsigned_p ? gen_udivmoddi4_1 : gen_divmoddi4_1;
1599 : break;
1600 :
1601 0 : default:
1602 0 : gcc_unreachable ();
1603 : }
1604 :
1605 27 : end_label = gen_label_rtx ();
1606 27 : qimode_label = gen_label_rtx ();
1607 :
1608 27 : scratch = gen_reg_rtx (mode);
1609 :
1610 : /* Use 8bit unsigned divimod if dividend and divisor are within
1611 : the range [0-255]. */
1612 27 : emit_move_insn (scratch, operands[2]);
1613 27 : scratch = expand_simple_binop (mode, IOR, scratch, operands[3],
1614 : scratch, 1, OPTAB_DIRECT);
1615 27 : emit_insn (gen_test_ccno_1 (mode, scratch, GEN_INT (-0x100)));
1616 27 : tmp0 = gen_rtx_REG (CCNOmode, FLAGS_REG);
1617 27 : tmp0 = gen_rtx_EQ (VOIDmode, tmp0, const0_rtx);
1618 27 : tmp0 = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp0,
1619 : gen_rtx_LABEL_REF (VOIDmode, qimode_label),
1620 : pc_rtx);
1621 27 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp0));
1622 27 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
1623 27 : JUMP_LABEL (insn) = qimode_label;
1624 :
1625 : /* Generate original signed/unsigned divimod. */
1626 27 : emit_insn (gen_divmod4_1 (operands[0], operands[1],
1627 : operands[2], operands[3]));
1628 :
1629 : /* Branch to the end. */
1630 27 : emit_jump_insn (gen_jump (end_label));
1631 27 : emit_barrier ();
1632 :
1633 : /* Generate 8bit unsigned divide. */
1634 27 : emit_label (qimode_label);
1635 : /* Don't use operands[0] for result of 8bit divide since not all
1636 : registers support QImode ZERO_EXTRACT. */
1637 27 : tmp0 = lowpart_subreg (HImode, scratch, mode);
1638 27 : tmp1 = lowpart_subreg (HImode, operands[2], mode);
1639 27 : tmp2 = lowpart_subreg (QImode, operands[3], mode);
1640 27 : emit_insn (gen_udivmodhiqi3 (tmp0, tmp1, tmp2));
1641 :
1642 27 : if (unsigned_p)
1643 : {
1644 12 : div = gen_rtx_UDIV (mode, operands[2], operands[3]);
1645 12 : mod = gen_rtx_UMOD (mode, operands[2], operands[3]);
1646 : }
1647 : else
1648 : {
1649 15 : div = gen_rtx_DIV (mode, operands[2], operands[3]);
1650 15 : mod = gen_rtx_MOD (mode, operands[2], operands[3]);
1651 : }
1652 27 : if (mode == SImode)
1653 : {
1654 20 : if (GET_MODE (operands[0]) != SImode)
1655 4 : div = gen_rtx_ZERO_EXTEND (DImode, div);
1656 20 : if (GET_MODE (operands[1]) != SImode)
1657 2 : mod = gen_rtx_ZERO_EXTEND (DImode, mod);
1658 : }
1659 :
1660 : /* Extract remainder from AH. */
1661 27 : scratch = gen_lowpart (GET_MODE (operands[1]), scratch);
1662 27 : tmp1 = gen_rtx_ZERO_EXTRACT (GET_MODE (operands[1]), scratch,
1663 : GEN_INT (8), GEN_INT (8));
1664 27 : insn = emit_move_insn (operands[1], tmp1);
1665 27 : set_unique_reg_note (insn, REG_EQUAL, mod);
1666 :
1667 : /* Zero extend quotient from AL. */
1668 27 : tmp1 = gen_lowpart (QImode, tmp0);
1669 27 : insn = emit_insn (gen_extend_insn
1670 27 : (operands[0], tmp1,
1671 27 : GET_MODE (operands[0]), QImode, 1));
1672 27 : set_unique_reg_note (insn, REG_EQUAL, div);
1673 :
1674 27 : emit_label (end_label);
1675 27 : }
1676 :
1677 : /* Emit x86 binary operand CODE in mode MODE, where the first operand
1678 : matches destination. RTX includes clobber of FLAGS_REG. */
1679 :
1680 : void
1681 7768 : ix86_emit_binop (enum rtx_code code, machine_mode mode,
1682 : rtx dst, rtx src)
1683 : {
1684 7768 : rtx op, clob;
1685 :
1686 7768 : op = gen_rtx_SET (dst, gen_rtx_fmt_ee (code, mode, dst, src));
1687 7768 : clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
1688 :
1689 7768 : emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
1690 7768 : }
1691 :
1692 : /* Return true if regno1 def is nearest to the insn. */
1693 :
1694 : static bool
1695 15 : find_nearest_reg_def (rtx_insn *insn, int regno1, int regno2)
1696 : {
1697 15 : rtx_insn *prev = insn;
1698 15 : rtx_insn *start = BB_HEAD (BLOCK_FOR_INSN (insn));
1699 :
1700 15 : if (insn == start)
1701 : return false;
1702 40 : while (prev && prev != start)
1703 : {
1704 30 : if (!INSN_P (prev) || !NONDEBUG_INSN_P (prev))
1705 : {
1706 10 : prev = PREV_INSN (prev);
1707 10 : continue;
1708 : }
1709 20 : if (insn_defines_reg (regno1, INVALID_REGNUM, prev))
1710 : return true;
1711 15 : else if (insn_defines_reg (regno2, INVALID_REGNUM, prev))
1712 : return false;
1713 15 : prev = PREV_INSN (prev);
1714 : }
1715 :
1716 : /* None of the regs is defined in the bb. */
1717 : return false;
1718 : }
1719 :
1720 : /* INSN_UID of the last insn emitted by zero store peephole2s. */
1721 : int ix86_last_zero_store_uid;
1722 :
1723 : /* Split lea instructions into a sequence of instructions
1724 : which are executed on ALU to avoid AGU stalls.
1725 : It is assumed that it is allowed to clobber flags register
1726 : at lea position. */
1727 :
1728 : void
1729 5944 : ix86_split_lea_for_addr (rtx_insn *insn, rtx operands[], machine_mode mode)
1730 : {
1731 5944 : unsigned int regno0, regno1, regno2;
1732 5944 : struct ix86_address parts;
1733 5944 : rtx target, tmp;
1734 5944 : int ok, adds;
1735 :
1736 5944 : ok = ix86_decompose_address (operands[1], &parts);
1737 5944 : gcc_assert (ok);
1738 :
1739 5944 : target = gen_lowpart (mode, operands[0]);
1740 :
1741 5944 : regno0 = true_regnum (target);
1742 5944 : regno1 = INVALID_REGNUM;
1743 5944 : regno2 = INVALID_REGNUM;
1744 :
1745 5944 : if (parts.base)
1746 : {
1747 5936 : parts.base = gen_lowpart (mode, parts.base);
1748 5936 : regno1 = true_regnum (parts.base);
1749 : }
1750 :
1751 5944 : if (parts.index)
1752 : {
1753 5940 : parts.index = gen_lowpart (mode, parts.index);
1754 5940 : regno2 = true_regnum (parts.index);
1755 : }
1756 :
1757 5944 : if (parts.disp)
1758 217 : parts.disp = gen_lowpart (mode, parts.disp);
1759 :
1760 5944 : if (parts.scale > 1)
1761 : {
1762 : /* Case r1 = r1 + ... */
1763 11 : if (regno1 == regno0)
1764 : {
1765 : /* If we have a case r1 = r1 + C * r2 then we
1766 : should use multiplication which is very
1767 : expensive. Assume cost model is wrong if we
1768 : have such case here. */
1769 0 : gcc_assert (regno2 != regno0);
1770 :
1771 0 : for (adds = parts.scale; adds > 0; adds--)
1772 0 : ix86_emit_binop (PLUS, mode, target, parts.index);
1773 : }
1774 : else
1775 : {
1776 : /* r1 = r2 + r3 * C case. Need to move r3 into r1. */
1777 11 : if (regno0 != regno2)
1778 8 : emit_insn (gen_rtx_SET (target, parts.index));
1779 :
1780 : /* Use shift for scaling, but emit it as MULT instead
1781 : to avoid it being immediately peephole2 optimized back
1782 : into lea. */
1783 11 : ix86_emit_binop (MULT, mode, target, GEN_INT (parts.scale));
1784 :
1785 11 : if (parts.base)
1786 3 : ix86_emit_binop (PLUS, mode, target, parts.base);
1787 :
1788 11 : if (parts.disp && parts.disp != const0_rtx)
1789 2 : ix86_emit_binop (PLUS, mode, target, parts.disp);
1790 : }
1791 : }
1792 5933 : else if (!parts.base && !parts.index)
1793 : {
1794 0 : gcc_assert(parts.disp);
1795 0 : emit_insn (gen_rtx_SET (target, parts.disp));
1796 : }
1797 : else
1798 : {
1799 5933 : if (!parts.base)
1800 : {
1801 0 : if (regno0 != regno2)
1802 0 : emit_insn (gen_rtx_SET (target, parts.index));
1803 : }
1804 5933 : else if (!parts.index)
1805 : {
1806 4 : if (regno0 != regno1)
1807 2 : emit_insn (gen_rtx_SET (target, parts.base));
1808 : }
1809 : else
1810 : {
1811 5929 : if (regno0 == regno1)
1812 : tmp = parts.index;
1813 2900 : else if (regno0 == regno2)
1814 : tmp = parts.base;
1815 : else
1816 : {
1817 15 : rtx tmp1;
1818 :
1819 : /* Find better operand for SET instruction, depending
1820 : on which definition is farther from the insn. */
1821 15 : if (find_nearest_reg_def (insn, regno1, regno2))
1822 5 : tmp = parts.index, tmp1 = parts.base;
1823 : else
1824 10 : tmp = parts.base, tmp1 = parts.index;
1825 :
1826 15 : emit_insn (gen_rtx_SET (target, tmp));
1827 :
1828 15 : if (parts.disp && parts.disp != const0_rtx)
1829 0 : ix86_emit_binop (PLUS, mode, target, parts.disp);
1830 :
1831 15 : ix86_emit_binop (PLUS, mode, target, tmp1);
1832 15 : return;
1833 : }
1834 :
1835 5914 : ix86_emit_binop (PLUS, mode, target, tmp);
1836 : }
1837 :
1838 5918 : if (parts.disp && parts.disp != const0_rtx)
1839 5 : ix86_emit_binop (PLUS, mode, target, parts.disp);
1840 : }
1841 : }
1842 :
1843 : /* Post-reload splitter for converting an SF or DFmode value in an
1844 : SSE register into an unsigned SImode. */
1845 :
1846 : void
1847 0 : ix86_split_convert_uns_si_sse (rtx operands[])
1848 : {
1849 0 : machine_mode vecmode;
1850 0 : rtx value, large, zero_or_two31, input, two31, x;
1851 :
1852 0 : large = operands[1];
1853 0 : zero_or_two31 = operands[2];
1854 0 : input = operands[3];
1855 0 : two31 = operands[4];
1856 0 : vecmode = GET_MODE (large);
1857 0 : value = gen_rtx_REG (vecmode, REGNO (operands[0]));
1858 :
1859 : /* Load up the value into the low element. We must ensure that the other
1860 : elements are valid floats -- zero is the easiest such value. */
1861 0 : if (MEM_P (input))
1862 : {
1863 0 : if (vecmode == V4SFmode)
1864 0 : emit_insn (gen_vec_setv4sf_0 (value, CONST0_RTX (V4SFmode), input));
1865 : else
1866 0 : emit_insn (gen_sse2_loadlpd (value, CONST0_RTX (V2DFmode), input));
1867 : }
1868 : else
1869 : {
1870 0 : input = gen_rtx_REG (vecmode, REGNO (input));
1871 0 : emit_move_insn (value, CONST0_RTX (vecmode));
1872 0 : if (vecmode == V4SFmode)
1873 0 : emit_insn (gen_sse_movss_v4sf (value, value, input));
1874 : else
1875 0 : emit_insn (gen_sse2_movsd_v2df (value, value, input));
1876 : }
1877 :
1878 0 : emit_move_insn (large, two31);
1879 0 : emit_move_insn (zero_or_two31, MEM_P (two31) ? large : two31);
1880 :
1881 0 : x = gen_rtx_fmt_ee (LE, vecmode, large, value);
1882 0 : emit_insn (gen_rtx_SET (large, x));
1883 :
1884 0 : x = gen_rtx_AND (vecmode, zero_or_two31, large);
1885 0 : emit_insn (gen_rtx_SET (zero_or_two31, x));
1886 :
1887 0 : x = gen_rtx_MINUS (vecmode, value, zero_or_two31);
1888 0 : emit_insn (gen_rtx_SET (value, x));
1889 :
1890 0 : large = gen_rtx_REG (V4SImode, REGNO (large));
1891 0 : emit_insn (gen_ashlv4si3 (large, large, GEN_INT (31)));
1892 :
1893 0 : x = gen_rtx_REG (V4SImode, REGNO (value));
1894 0 : if (vecmode == V4SFmode)
1895 0 : emit_insn (gen_fix_truncv4sfv4si2 (x, value));
1896 : else
1897 0 : emit_insn (gen_sse2_cvttpd2dq (x, value));
1898 0 : value = x;
1899 :
1900 0 : emit_insn (gen_xorv4si3 (value, value, large));
1901 0 : }
1902 :
1903 : /* Convert an unsigned DImode value into a DFmode, using only SSE.
1904 : Expects the 64-bit DImode to be supplied in a pair of integral
1905 : registers. Requires SSE2; will use SSE3 if available. For x86_32,
1906 : -mfpmath=sse, !optimize_size only. */
1907 :
1908 : void
1909 0 : ix86_expand_convert_uns_didf_sse (rtx target, rtx input)
1910 : {
1911 0 : REAL_VALUE_TYPE bias_lo_rvt, bias_hi_rvt;
1912 0 : rtx int_xmm, fp_xmm;
1913 0 : rtx biases, exponents;
1914 0 : rtx x;
1915 :
1916 0 : int_xmm = gen_reg_rtx (V4SImode);
1917 0 : if (TARGET_INTER_UNIT_MOVES_TO_VEC)
1918 0 : emit_insn (gen_movdi_to_sse (int_xmm, input));
1919 0 : else if (TARGET_SSE_SPLIT_REGS)
1920 : {
1921 0 : emit_clobber (int_xmm);
1922 0 : emit_move_insn (gen_lowpart (DImode, int_xmm), input);
1923 : }
1924 : else
1925 : {
1926 0 : x = gen_reg_rtx (V2DImode);
1927 0 : ix86_expand_vector_init_one_nonzero (false, V2DImode, x, input, 0);
1928 0 : emit_move_insn (int_xmm, gen_lowpart (V4SImode, x));
1929 : }
1930 :
1931 0 : x = gen_rtx_CONST_VECTOR (V4SImode,
1932 : gen_rtvec (4, GEN_INT (0x43300000UL),
1933 : GEN_INT (0x45300000UL),
1934 : const0_rtx, const0_rtx));
1935 0 : exponents = validize_mem (force_const_mem (V4SImode, x));
1936 :
1937 : /* int_xmm = {0x45300000UL, fp_xmm/hi, 0x43300000, fp_xmm/lo } */
1938 0 : emit_insn (gen_vec_interleave_lowv4si (int_xmm, int_xmm, exponents));
1939 :
1940 : /* Concatenating (juxtaposing) (0x43300000UL ## fp_value_low_xmm)
1941 : yields a valid DF value equal to (0x1.0p52 + double(fp_value_lo_xmm)).
1942 : Similarly (0x45300000UL ## fp_value_hi_xmm) yields
1943 : (0x1.0p84 + double(fp_value_hi_xmm)).
1944 : Note these exponents differ by 32. */
1945 :
1946 0 : fp_xmm = copy_to_mode_reg (V2DFmode, gen_lowpart (V2DFmode, int_xmm));
1947 :
1948 : /* Subtract off those 0x1.0p52 and 0x1.0p84 biases, to produce values
1949 : in [0,2**32-1] and [0]+[2**32,2**64-1] respectively. */
1950 0 : real_ldexp (&bias_lo_rvt, &dconst1, 52);
1951 0 : real_ldexp (&bias_hi_rvt, &dconst1, 84);
1952 0 : biases = const_double_from_real_value (bias_lo_rvt, DFmode);
1953 0 : x = const_double_from_real_value (bias_hi_rvt, DFmode);
1954 0 : biases = gen_rtx_CONST_VECTOR (V2DFmode, gen_rtvec (2, biases, x));
1955 0 : biases = validize_mem (force_const_mem (V2DFmode, biases));
1956 0 : emit_insn (gen_subv2df3 (fp_xmm, fp_xmm, biases));
1957 :
1958 : /* Add the upper and lower DFmode values together. */
1959 0 : if (TARGET_SSE3)
1960 0 : emit_insn (gen_sse3_haddv2df3 (fp_xmm, fp_xmm, fp_xmm));
1961 : else
1962 : {
1963 0 : x = copy_to_mode_reg (V2DFmode, fp_xmm);
1964 0 : emit_insn (gen_vec_interleave_highv2df (fp_xmm, fp_xmm, fp_xmm));
1965 0 : emit_insn (gen_addv2df3 (fp_xmm, fp_xmm, x));
1966 : }
1967 :
1968 0 : ix86_expand_vector_extract (false, target, fp_xmm, 0);
1969 0 : }
1970 :
1971 : /* Not used, but eases macroization of patterns. */
1972 : void
1973 0 : ix86_expand_convert_uns_sixf_sse (rtx, rtx)
1974 : {
1975 0 : gcc_unreachable ();
1976 : }
1977 :
1978 : static rtx ix86_expand_sse_fabs (rtx op0, rtx *smask);
1979 :
1980 : /* Convert an unsigned SImode value into a DFmode. Only currently used
1981 : for SSE, but applicable anywhere. */
1982 :
1983 : void
1984 0 : ix86_expand_convert_uns_sidf_sse (rtx target, rtx input)
1985 : {
1986 0 : REAL_VALUE_TYPE TWO31r;
1987 0 : rtx x, fp;
1988 :
1989 0 : x = expand_simple_binop (SImode, PLUS, input, GEN_INT (-2147483647 - 1),
1990 : NULL, 1, OPTAB_DIRECT);
1991 :
1992 0 : fp = gen_reg_rtx (DFmode);
1993 0 : emit_insn (gen_floatsidf2 (fp, x));
1994 :
1995 0 : real_ldexp (&TWO31r, &dconst1, 31);
1996 0 : x = const_double_from_real_value (TWO31r, DFmode);
1997 :
1998 0 : x = expand_simple_binop (DFmode, PLUS, fp, x, target, 0, OPTAB_DIRECT);
1999 :
2000 : /* Remove the sign with FE_DOWNWARD, where x - x = -0.0. */
2001 0 : if (HONOR_SIGNED_ZEROS (DFmode) && flag_rounding_math)
2002 0 : x = ix86_expand_sse_fabs (x, NULL);
2003 :
2004 0 : if (x != target)
2005 0 : emit_move_insn (target, x);
2006 0 : }
2007 :
2008 : /* Convert a signed DImode value into a DFmode. Only used for SSE in
2009 : 32-bit mode; otherwise we have a direct convert instruction. */
2010 :
2011 : void
2012 0 : ix86_expand_convert_sign_didf_sse (rtx target, rtx input)
2013 : {
2014 0 : REAL_VALUE_TYPE TWO32r;
2015 0 : rtx fp_lo, fp_hi, x;
2016 :
2017 0 : fp_lo = gen_reg_rtx (DFmode);
2018 0 : fp_hi = gen_reg_rtx (DFmode);
2019 :
2020 0 : emit_insn (gen_floatsidf2 (fp_hi, gen_highpart (SImode, input)));
2021 :
2022 0 : real_ldexp (&TWO32r, &dconst1, 32);
2023 0 : x = const_double_from_real_value (TWO32r, DFmode);
2024 0 : fp_hi = expand_simple_binop (DFmode, MULT, fp_hi, x, fp_hi, 0, OPTAB_DIRECT);
2025 :
2026 0 : ix86_expand_convert_uns_sidf_sse (fp_lo, gen_lowpart (SImode, input));
2027 :
2028 0 : x = expand_simple_binop (DFmode, PLUS, fp_hi, fp_lo, target,
2029 : 0, OPTAB_DIRECT);
2030 0 : if (x != target)
2031 0 : emit_move_insn (target, x);
2032 0 : }
2033 :
2034 : /* Convert an unsigned SImode value into a SFmode, using only SSE.
2035 : For x86_32, -mfpmath=sse, !optimize_size only. */
2036 : void
2037 0 : ix86_expand_convert_uns_sisf_sse (rtx target, rtx input)
2038 : {
2039 0 : REAL_VALUE_TYPE ONE16r;
2040 0 : rtx fp_hi, fp_lo, int_hi, int_lo, x;
2041 :
2042 0 : real_ldexp (&ONE16r, &dconst1, 16);
2043 0 : x = const_double_from_real_value (ONE16r, SFmode);
2044 0 : int_lo = expand_simple_binop (SImode, AND, input, GEN_INT(0xffff),
2045 : NULL, 0, OPTAB_DIRECT);
2046 0 : int_hi = expand_simple_binop (SImode, LSHIFTRT, input, GEN_INT(16),
2047 : NULL, 0, OPTAB_DIRECT);
2048 0 : fp_hi = gen_reg_rtx (SFmode);
2049 0 : fp_lo = gen_reg_rtx (SFmode);
2050 0 : emit_insn (gen_floatsisf2 (fp_hi, int_hi));
2051 0 : emit_insn (gen_floatsisf2 (fp_lo, int_lo));
2052 0 : if (TARGET_FMA)
2053 : {
2054 0 : x = validize_mem (force_const_mem (SFmode, x));
2055 0 : fp_hi = gen_rtx_FMA (SFmode, fp_hi, x, fp_lo);
2056 0 : emit_move_insn (target, fp_hi);
2057 : }
2058 : else
2059 : {
2060 0 : fp_hi = expand_simple_binop (SFmode, MULT, fp_hi, x, fp_hi,
2061 : 0, OPTAB_DIRECT);
2062 0 : fp_hi = expand_simple_binop (SFmode, PLUS, fp_hi, fp_lo, target,
2063 : 0, OPTAB_DIRECT);
2064 0 : if (!rtx_equal_p (target, fp_hi))
2065 0 : emit_move_insn (target, fp_hi);
2066 : }
2067 0 : }
2068 :
2069 : /* floatunsv{4,8}siv{4,8}sf2 expander. Expand code to convert
2070 : a vector of unsigned ints VAL to vector of floats TARGET. */
2071 :
2072 : void
2073 54 : ix86_expand_vector_convert_uns_vsivsf (rtx target, rtx val)
2074 : {
2075 54 : rtx tmp[8];
2076 54 : REAL_VALUE_TYPE TWO16r;
2077 54 : machine_mode intmode = GET_MODE (val);
2078 54 : machine_mode fltmode = GET_MODE (target);
2079 54 : rtx (*cvt) (rtx, rtx);
2080 :
2081 54 : if (intmode == V4SImode)
2082 : cvt = gen_floatv4siv4sf2;
2083 : else
2084 2 : cvt = gen_floatv8siv8sf2;
2085 54 : tmp[0] = ix86_build_const_vector (intmode, 1, GEN_INT (0xffff));
2086 54 : tmp[0] = force_reg (intmode, tmp[0]);
2087 54 : tmp[1] = expand_simple_binop (intmode, AND, val, tmp[0], NULL_RTX, 1,
2088 : OPTAB_DIRECT);
2089 54 : tmp[2] = expand_simple_binop (intmode, LSHIFTRT, val, GEN_INT (16),
2090 : NULL_RTX, 1, OPTAB_DIRECT);
2091 54 : tmp[3] = gen_reg_rtx (fltmode);
2092 54 : emit_insn (cvt (tmp[3], tmp[1]));
2093 54 : tmp[4] = gen_reg_rtx (fltmode);
2094 54 : emit_insn (cvt (tmp[4], tmp[2]));
2095 54 : real_ldexp (&TWO16r, &dconst1, 16);
2096 54 : tmp[5] = const_double_from_real_value (TWO16r, SFmode);
2097 54 : tmp[5] = force_reg (fltmode, ix86_build_const_vector (fltmode, 1, tmp[5]));
2098 54 : if (TARGET_FMA)
2099 : {
2100 1 : tmp[6] = gen_rtx_FMA (fltmode, tmp[4], tmp[5], tmp[3]);
2101 1 : emit_move_insn (target, tmp[6]);
2102 : }
2103 : else
2104 : {
2105 53 : tmp[6] = expand_simple_binop (fltmode, MULT, tmp[4], tmp[5],
2106 : NULL_RTX, 1, OPTAB_DIRECT);
2107 53 : tmp[7] = expand_simple_binop (fltmode, PLUS, tmp[3], tmp[6],
2108 : target, 1, OPTAB_DIRECT);
2109 53 : if (tmp[7] != target)
2110 0 : emit_move_insn (target, tmp[7]);
2111 : }
2112 54 : }
2113 :
2114 : /* Adjust a V*SFmode/V*DFmode value VAL so that *sfix_trunc* resp. fix_trunc*
2115 : pattern can be used on it instead of fixuns_trunc*.
2116 : This is done by doing just signed conversion if < 0x1p31, and otherwise by
2117 : subtracting 0x1p31 first and xoring in 0x80000000 from *XORP afterwards. */
2118 :
2119 : rtx
2120 308 : ix86_expand_adjust_ufix_to_sfix_si (rtx val, rtx *xorp)
2121 : {
2122 308 : REAL_VALUE_TYPE TWO31r;
2123 308 : rtx two31r, tmp[4];
2124 308 : machine_mode mode = GET_MODE (val);
2125 308 : machine_mode scalarmode = GET_MODE_INNER (mode);
2126 616 : machine_mode intmode = GET_MODE_SIZE (mode) == 32 ? V8SImode : V4SImode;
2127 308 : rtx (*cmp) (rtx, rtx, rtx, rtx);
2128 308 : int i;
2129 :
2130 1232 : for (i = 0; i < 3; i++)
2131 924 : tmp[i] = gen_reg_rtx (mode);
2132 308 : real_ldexp (&TWO31r, &dconst1, 31);
2133 308 : two31r = const_double_from_real_value (TWO31r, scalarmode);
2134 308 : two31r = ix86_build_const_vector (mode, 1, two31r);
2135 308 : two31r = force_reg (mode, two31r);
2136 308 : switch (mode)
2137 : {
2138 : case E_V8SFmode: cmp = gen_avx_maskcmpv8sf3; break;
2139 10 : case E_V4SFmode: cmp = gen_sse_maskcmpv4sf3; break;
2140 16 : case E_V4DFmode: cmp = gen_avx_maskcmpv4df3; break;
2141 282 : case E_V2DFmode: cmp = gen_sse2_maskcmpv2df3; break;
2142 0 : default: gcc_unreachable ();
2143 : }
2144 308 : tmp[3] = gen_rtx_LE (mode, two31r, val);
2145 308 : emit_insn (cmp (tmp[0], two31r, val, tmp[3]));
2146 308 : tmp[1] = expand_simple_binop (mode, AND, tmp[0], two31r, tmp[1],
2147 : 0, OPTAB_DIRECT);
2148 308 : if (intmode == V4SImode || TARGET_AVX2)
2149 616 : *xorp = expand_simple_binop (intmode, ASHIFT,
2150 308 : gen_lowpart (intmode, tmp[0]),
2151 : GEN_INT (31), NULL_RTX, 0,
2152 : OPTAB_DIRECT);
2153 : else
2154 : {
2155 0 : rtx two31 = gen_int_mode (HOST_WIDE_INT_1U << 31, SImode);
2156 0 : two31 = ix86_build_const_vector (intmode, 1, two31);
2157 0 : *xorp = expand_simple_binop (intmode, AND,
2158 0 : gen_lowpart (intmode, tmp[0]),
2159 : two31, NULL_RTX, 0,
2160 : OPTAB_DIRECT);
2161 : }
2162 308 : return expand_simple_binop (mode, MINUS, val, tmp[1], tmp[2],
2163 308 : 0, OPTAB_DIRECT);
2164 : }
2165 :
2166 : /* Generate code for floating point ABS or NEG. */
2167 :
2168 : void
2169 35085 : ix86_expand_fp_absneg_operator (enum rtx_code code, machine_mode mode,
2170 : rtx operands[])
2171 : {
2172 35085 : rtx set, dst, src;
2173 35085 : bool use_sse = false;
2174 35085 : bool vector_mode = VECTOR_MODE_P (mode);
2175 35085 : machine_mode vmode = mode;
2176 35085 : rtvec par;
2177 :
2178 35085 : switch (mode)
2179 : {
2180 : case E_HFmode:
2181 : use_sse = true;
2182 : vmode = V8HFmode;
2183 : break;
2184 0 : case E_BFmode:
2185 0 : use_sse = true;
2186 0 : vmode = V8BFmode;
2187 0 : break;
2188 9543 : case E_SFmode:
2189 9543 : use_sse = TARGET_SSE_MATH && TARGET_SSE;
2190 : vmode = V4SFmode;
2191 : break;
2192 16147 : case E_DFmode:
2193 16147 : use_sse = TARGET_SSE_MATH && TARGET_SSE2;
2194 : vmode = V2DFmode;
2195 : break;
2196 9196 : default:
2197 9196 : use_sse = vector_mode || mode == TFmode;
2198 9196 : break;
2199 : }
2200 :
2201 35085 : dst = operands[0];
2202 35085 : src = operands[1];
2203 :
2204 35085 : set = gen_rtx_fmt_e (code, mode, src);
2205 35085 : set = gen_rtx_SET (dst, set);
2206 :
2207 35085 : if (use_sse)
2208 : {
2209 29378 : rtx mask, use, clob;
2210 :
2211 : /* NEG and ABS performed with SSE use bitwise mask operations.
2212 : Create the appropriate mask now. */
2213 29378 : mask = ix86_build_signbit_mask (vmode, vector_mode, code == ABS);
2214 29378 : use = gen_rtx_USE (VOIDmode, mask);
2215 29378 : if (vector_mode || mode == TFmode)
2216 4842 : par = gen_rtvec (2, set, use);
2217 : else
2218 : {
2219 24536 : clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
2220 24536 : par = gen_rtvec (3, set, use, clob);
2221 : }
2222 : }
2223 : else
2224 : {
2225 5707 : rtx clob;
2226 :
2227 : /* Changing of sign for FP values is doable using integer unit too. */
2228 5707 : clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
2229 5707 : par = gen_rtvec (2, set, clob);
2230 : }
2231 :
2232 35085 : emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
2233 35085 : }
2234 :
2235 : /* Deconstruct a floating point ABS or NEG operation
2236 : with integer registers into integer operations. */
2237 :
2238 : void
2239 24 : ix86_split_fp_absneg_operator (enum rtx_code code, machine_mode mode,
2240 : rtx operands[])
2241 : {
2242 24 : enum rtx_code absneg_op;
2243 24 : rtx dst, set;
2244 :
2245 24 : gcc_assert (operands_match_p (operands[0], operands[1]));
2246 :
2247 24 : switch (mode)
2248 : {
2249 0 : case E_SFmode:
2250 0 : dst = gen_lowpart (SImode, operands[0]);
2251 :
2252 0 : if (code == ABS)
2253 : {
2254 0 : set = gen_int_mode (0x7fffffff, SImode);
2255 0 : absneg_op = AND;
2256 : }
2257 : else
2258 : {
2259 0 : set = gen_int_mode (0x80000000, SImode);
2260 0 : absneg_op = XOR;
2261 : }
2262 0 : set = gen_rtx_fmt_ee (absneg_op, SImode, dst, set);
2263 0 : break;
2264 :
2265 1 : case E_DFmode:
2266 1 : if (TARGET_64BIT)
2267 : {
2268 1 : dst = gen_lowpart (DImode, operands[0]);
2269 1 : dst = gen_rtx_ZERO_EXTRACT (DImode, dst, const1_rtx, GEN_INT (63));
2270 :
2271 1 : if (code == ABS)
2272 0 : set = const0_rtx;
2273 : else
2274 1 : set = gen_rtx_NOT (DImode, dst);
2275 : }
2276 : else
2277 : {
2278 0 : dst = gen_highpart (SImode, operands[0]);
2279 :
2280 0 : if (code == ABS)
2281 : {
2282 0 : set = gen_int_mode (0x7fffffff, SImode);
2283 0 : absneg_op = AND;
2284 : }
2285 : else
2286 : {
2287 0 : set = gen_int_mode (0x80000000, SImode);
2288 0 : absneg_op = XOR;
2289 : }
2290 0 : set = gen_rtx_fmt_ee (absneg_op, SImode, dst, set);
2291 : }
2292 : break;
2293 :
2294 23 : case E_XFmode:
2295 23 : dst = gen_rtx_REG (SImode,
2296 23 : REGNO (operands[0]) + (TARGET_64BIT ? 1 : 2));
2297 23 : if (code == ABS)
2298 : {
2299 1 : set = GEN_INT (0x7fff);
2300 1 : absneg_op = AND;
2301 : }
2302 : else
2303 : {
2304 22 : set = GEN_INT (0x8000);
2305 22 : absneg_op = XOR;
2306 : }
2307 23 : set = gen_rtx_fmt_ee (absneg_op, SImode, dst, set);
2308 23 : break;
2309 :
2310 0 : default:
2311 0 : gcc_unreachable ();
2312 : }
2313 :
2314 24 : set = gen_rtx_SET (dst, set);
2315 :
2316 24 : rtx clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
2317 24 : rtvec par = gen_rtvec (2, set, clob);
2318 :
2319 24 : emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
2320 24 : }
2321 :
2322 : /* Expand a copysign operation. Special case operand 0 being a constant. */
2323 :
2324 : void
2325 23244 : ix86_expand_copysign (rtx operands[])
2326 : {
2327 23244 : machine_mode mode, vmode;
2328 23244 : rtx dest, vdest, op0, op1, mask, op2, op3;
2329 :
2330 23244 : mode = GET_MODE (operands[0]);
2331 :
2332 23244 : switch (mode)
2333 : {
2334 : case E_HFmode:
2335 : vmode = V8HFmode;
2336 : break;
2337 0 : case E_BFmode:
2338 0 : vmode = V8BFmode;
2339 0 : break;
2340 11565 : case E_SFmode:
2341 11565 : vmode = V4SFmode;
2342 11565 : break;
2343 11540 : case E_DFmode:
2344 11540 : vmode = V2DFmode;
2345 11540 : break;
2346 127 : case E_TFmode:
2347 127 : vmode = mode;
2348 127 : break;
2349 0 : default:
2350 0 : gcc_unreachable();
2351 : }
2352 :
2353 23244 : if (rtx_equal_p (operands[1], operands[2]))
2354 : {
2355 0 : emit_move_insn (operands[0], operands[1]);
2356 0 : return;
2357 : }
2358 :
2359 23244 : dest = operands[0];
2360 23244 : vdest = lowpart_subreg (vmode, dest, mode);
2361 23244 : if (vdest == NULL_RTX)
2362 0 : vdest = gen_reg_rtx (vmode);
2363 : else
2364 : dest = NULL_RTX;
2365 23244 : op1 = lowpart_subreg (vmode, force_reg (mode, operands[1]), mode);
2366 46474 : mask = ix86_build_signbit_mask (vmode, TARGET_AVX512F && mode != HFmode, 0);
2367 :
2368 23244 : if (CONST_DOUBLE_P (operands[2]))
2369 : {
2370 82 : if (real_isneg (CONST_DOUBLE_REAL_VALUE (operands[2])))
2371 : /* Simplify b = copysign (a, negative) to b = mask | a. */
2372 79 : op1 = gen_rtx_IOR (vmode, mask, op1);
2373 : else
2374 : {
2375 : /* Simplify b = copysign (a, positive) to b = invert_mask & a. */
2376 3 : rtx invert_mask
2377 3 : = ix86_build_signbit_mask (vmode,
2378 3 : TARGET_AVX512F && mode != HFmode,
2379 : true);
2380 3 : op1 = gen_rtx_AND (vmode, invert_mask, op1);
2381 : }
2382 82 : emit_move_insn (vdest, op1);
2383 82 : if (dest)
2384 0 : emit_move_insn (dest, lowpart_subreg (mode, vdest, vmode));
2385 82 : return;
2386 : }
2387 : else
2388 23162 : op0 = lowpart_subreg (vmode, force_reg (mode, operands[2]), mode);
2389 :
2390 23162 : op2 = gen_reg_rtx (vmode);
2391 23162 : op3 = gen_reg_rtx (vmode);
2392 23162 : rtx invert_mask;
2393 : /* NB: Generate vmovdqa, vpandn, vpand, vpor for AVX and generate pand,
2394 : pand, por for SSE. */
2395 23162 : if (TARGET_AVX)
2396 31 : invert_mask = gen_rtx_NOT (vmode, mask);
2397 : else
2398 23131 : invert_mask = ix86_build_signbit_mask (vmode,
2399 23131 : TARGET_AVX512F && mode != HFmode,
2400 : true);
2401 23162 : emit_move_insn (op2, gen_rtx_AND (vmode, invert_mask, op1));
2402 23162 : emit_move_insn (op3, gen_rtx_AND (vmode, mask, op0));
2403 23162 : emit_move_insn (vdest, gen_rtx_IOR (vmode, op2, op3));
2404 23162 : if (dest)
2405 0 : emit_move_insn (dest, lowpart_subreg (mode, vdest, vmode));
2406 : }
2407 :
2408 : /* Expand an xorsign operation. */
2409 :
2410 : void
2411 20 : ix86_expand_xorsign (rtx operands[])
2412 : {
2413 20 : machine_mode mode, vmode;
2414 20 : rtx dest, vdest, op0, op1, mask, x, temp;
2415 :
2416 20 : dest = operands[0];
2417 20 : op0 = operands[1];
2418 20 : op1 = operands[2];
2419 :
2420 20 : mode = GET_MODE (dest);
2421 :
2422 20 : switch (mode)
2423 : {
2424 : case E_HFmode:
2425 : vmode = V8HFmode;
2426 : break;
2427 : case E_BFmode:
2428 : vmode = V8BFmode;
2429 : break;
2430 : case E_SFmode:
2431 : vmode = V4SFmode;
2432 : break;
2433 : case E_DFmode:
2434 : vmode = V2DFmode;
2435 : break;
2436 0 : default:
2437 0 : gcc_unreachable ();
2438 20 : break;
2439 : }
2440 :
2441 20 : temp = gen_reg_rtx (vmode);
2442 20 : mask = ix86_build_signbit_mask (vmode, 0, 0);
2443 :
2444 20 : op1 = lowpart_subreg (vmode, force_reg (mode, op1), mode);
2445 20 : x = gen_rtx_AND (vmode, op1, mask);
2446 20 : emit_insn (gen_rtx_SET (temp, x));
2447 :
2448 20 : op0 = lowpart_subreg (vmode, force_reg (mode, op0), mode);
2449 20 : x = gen_rtx_XOR (vmode, temp, op0);
2450 :
2451 20 : vdest = lowpart_subreg (vmode, dest, mode);
2452 20 : if (vdest == NULL_RTX)
2453 0 : vdest = gen_reg_rtx (vmode);
2454 : else
2455 : dest = NULL_RTX;
2456 20 : emit_insn (gen_rtx_SET (vdest, x));
2457 :
2458 20 : if (dest)
2459 0 : emit_move_insn (dest, lowpart_subreg (mode, vdest, vmode));
2460 20 : }
2461 :
2462 : static rtx ix86_expand_compare (enum rtx_code code, rtx op0, rtx op1);
2463 :
2464 : void
2465 6719383 : ix86_expand_branch (enum rtx_code code, rtx op0, rtx op1, rtx label)
2466 : {
2467 6719383 : machine_mode mode = GET_MODE (op0);
2468 6719383 : rtx tmp;
2469 :
2470 : /* Handle special case - vector comparison with boolean result, transform
2471 : it using ptest instruction or vpcmpeq + kortest. */
2472 6719383 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
2473 6700033 : || (mode == TImode && !TARGET_64BIT)
2474 6700033 : || mode == OImode
2475 13419416 : || GET_MODE_SIZE (mode) == 64)
2476 : {
2477 19350 : unsigned msize = GET_MODE_SIZE (mode);
2478 19350 : machine_mode p_mode
2479 19350 : = msize == 64 ? V16SImode : msize == 32 ? V4DImode : V2DImode;
2480 : /* kortest set CF when result is 0xFFFF (op0 == op1). */
2481 19350 : rtx flag = gen_rtx_REG (msize == 64 ? CCCmode : CCZmode, FLAGS_REG);
2482 :
2483 19350 : gcc_assert (code == EQ || code == NE);
2484 :
2485 : /* Using vpcmpeq zmm zmm k + kortest for 512-bit vectors. */
2486 19350 : if (msize == 64)
2487 : {
2488 2431 : if (mode != V16SImode)
2489 : {
2490 2431 : op0 = lowpart_subreg (p_mode, force_reg (mode, op0), mode);
2491 2431 : op1 = lowpart_subreg (p_mode, force_reg (mode, op1), mode);
2492 : }
2493 :
2494 2431 : tmp = gen_reg_rtx (HImode);
2495 2431 : emit_insn (gen_avx512f_cmpv16si3 (tmp, op0, op1, GEN_INT (0)));
2496 2431 : emit_insn (gen_kortesthi_ccc (tmp, tmp));
2497 : }
2498 : /* Using ptest for 128/256-bit vectors. */
2499 : else
2500 : {
2501 16919 : if (GET_MODE_CLASS (mode) != MODE_VECTOR_INT)
2502 : {
2503 0 : op0 = lowpart_subreg (p_mode, force_reg (mode, op0), mode);
2504 0 : op1 = lowpart_subreg (p_mode, force_reg (mode, op1), mode);
2505 0 : mode = p_mode;
2506 : }
2507 :
2508 : /* Generate XOR since we can't check that one operand is zero
2509 : vector. */
2510 16919 : tmp = gen_reg_rtx (mode);
2511 16919 : rtx ops[3] = { tmp, op0, op1 };
2512 16919 : ix86_expand_vector_logical_operator (XOR, mode, ops);
2513 16919 : tmp = gen_lowpart (p_mode, tmp);
2514 16919 : emit_insn (gen_rtx_SET (gen_rtx_REG (CCZmode, FLAGS_REG),
2515 : gen_rtx_UNSPEC (CCZmode,
2516 : gen_rtvec (2, tmp, tmp),
2517 : UNSPEC_PTEST)));
2518 : }
2519 19350 : tmp = gen_rtx_fmt_ee (code, VOIDmode, flag, const0_rtx);
2520 19350 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
2521 : gen_rtx_LABEL_REF (VOIDmode, label),
2522 : pc_rtx);
2523 19350 : emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
2524 19350 : return;
2525 : }
2526 :
2527 6700033 : switch (mode)
2528 : {
2529 6668386 : case E_HFmode:
2530 6668386 : case E_SFmode:
2531 6668386 : case E_DFmode:
2532 6668386 : case E_XFmode:
2533 6668386 : case E_QImode:
2534 6668386 : case E_HImode:
2535 6668386 : case E_SImode:
2536 6668386 : simple:
2537 6668386 : tmp = ix86_expand_compare (code, op0, op1);
2538 6668386 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
2539 : gen_rtx_LABEL_REF (VOIDmode, label),
2540 : pc_rtx);
2541 6668386 : emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
2542 6668386 : return;
2543 :
2544 7 : case E_BFmode:
2545 7 : gcc_assert (TARGET_AVX10_2 && !flag_trapping_math);
2546 7 : goto simple;
2547 :
2548 2712067 : case E_DImode:
2549 2712067 : if (TARGET_64BIT)
2550 2683593 : goto simple;
2551 : /* FALLTHRU */
2552 90221 : case E_TImode:
2553 : /* DI and TI mode equality/inequality comparisons may be performed
2554 : on SSE registers. Avoid splitting them, except when optimizing
2555 : for size. */
2556 90221 : if ((code == EQ || code == NE)
2557 90221 : && !optimize_insn_for_size_p ())
2558 58574 : goto simple;
2559 :
2560 : /* Expand DImode branch into multiple compare+branch. */
2561 31647 : {
2562 31647 : rtx lo[2], hi[2];
2563 31647 : rtx_code_label *label2;
2564 31647 : enum rtx_code code1, code2, code3;
2565 31647 : machine_mode submode;
2566 :
2567 31647 : if (CONSTANT_P (op0) && !CONSTANT_P (op1))
2568 : {
2569 0 : std::swap (op0, op1);
2570 0 : code = swap_condition (code);
2571 : }
2572 :
2573 31647 : split_double_mode (mode, &op0, 1, lo+0, hi+0);
2574 31647 : split_double_mode (mode, &op1, 1, lo+1, hi+1);
2575 :
2576 31647 : submode = mode == DImode ? SImode : DImode;
2577 :
2578 : /* If we are doing less-than or greater-or-equal-than,
2579 : op1 is a constant and the low word is zero, then we can just
2580 : examine the high word. Similarly for low word -1 and
2581 : less-or-equal-than or greater-than. */
2582 :
2583 31647 : if (CONST_INT_P (hi[1]))
2584 20750 : switch (code)
2585 : {
2586 11345 : case LT: case LTU: case GE: case GEU:
2587 11345 : if (lo[1] == const0_rtx)
2588 : {
2589 10921 : ix86_expand_branch (code, hi[0], hi[1], label);
2590 10921 : return;
2591 : }
2592 : break;
2593 7853 : case LE: case LEU: case GT: case GTU:
2594 7853 : if (lo[1] == constm1_rtx)
2595 : {
2596 528 : ix86_expand_branch (code, hi[0], hi[1], label);
2597 528 : return;
2598 : }
2599 : break;
2600 : default:
2601 : break;
2602 : }
2603 :
2604 : /* Emulate comparisons that do not depend on Zero flag with
2605 : double-word subtraction. Note that only Overflow, Sign
2606 : and Carry flags are valid, so swap arguments and condition
2607 : of comparisons that would otherwise test Zero flag. */
2608 :
2609 20198 : switch (code)
2610 : {
2611 12706 : case LE: case LEU: case GT: case GTU:
2612 12706 : std::swap (lo[0], lo[1]);
2613 12706 : std::swap (hi[0], hi[1]);
2614 12706 : code = swap_condition (code);
2615 : /* FALLTHRU */
2616 :
2617 17169 : case LT: case LTU: case GE: case GEU:
2618 17169 : {
2619 17169 : bool uns = (code == LTU || code == GEU);
2620 4053 : rtx (*sbb_insn) (machine_mode, rtx, rtx, rtx)
2621 17169 : = uns ? gen_sub3_carry_ccc : gen_sub3_carry_ccgz;
2622 :
2623 17169 : if (!nonimmediate_operand (lo[0], submode))
2624 7325 : lo[0] = force_reg (submode, lo[0]);
2625 17169 : if (!x86_64_general_operand (lo[1], submode))
2626 0 : lo[1] = force_reg (submode, lo[1]);
2627 :
2628 17169 : if (!register_operand (hi[0], submode))
2629 8108 : hi[0] = force_reg (submode, hi[0]);
2630 13116 : if ((uns && !nonimmediate_operand (hi[1], submode))
2631 17169 : || (!uns && !x86_64_general_operand (hi[1], submode)))
2632 315 : hi[1] = force_reg (submode, hi[1]);
2633 :
2634 17169 : emit_insn (gen_cmp_1 (submode, lo[0], lo[1]));
2635 :
2636 17169 : tmp = gen_rtx_SCRATCH (submode);
2637 17169 : emit_insn (sbb_insn (submode, tmp, hi[0], hi[1]));
2638 :
2639 21222 : tmp = gen_rtx_REG (uns ? CCCmode : CCGZmode, FLAGS_REG);
2640 17169 : ix86_expand_branch (code, tmp, const0_rtx, label);
2641 17169 : return;
2642 : }
2643 :
2644 3029 : default:
2645 3029 : break;
2646 : }
2647 :
2648 : /* Otherwise, we need two or three jumps. */
2649 :
2650 3029 : label2 = gen_label_rtx ();
2651 :
2652 3029 : code1 = code;
2653 3029 : code2 = swap_condition (code);
2654 3029 : code3 = unsigned_condition (code);
2655 :
2656 3029 : switch (code)
2657 : {
2658 : case LT: case GT: case LTU: case GTU:
2659 : break;
2660 :
2661 : case LE: code1 = LT; code2 = GT; break;
2662 : case GE: code1 = GT; code2 = LT; break;
2663 0 : case LEU: code1 = LTU; code2 = GTU; break;
2664 0 : case GEU: code1 = GTU; code2 = LTU; break;
2665 :
2666 : case EQ: code1 = UNKNOWN; code2 = NE; break;
2667 : case NE: code2 = UNKNOWN; break;
2668 :
2669 0 : default:
2670 0 : gcc_unreachable ();
2671 : }
2672 :
2673 : /*
2674 : * a < b =>
2675 : * if (hi(a) < hi(b)) goto true;
2676 : * if (hi(a) > hi(b)) goto false;
2677 : * if (lo(a) < lo(b)) goto true;
2678 : * false:
2679 : */
2680 :
2681 0 : if (code1 != UNKNOWN)
2682 2362 : ix86_expand_branch (code1, hi[0], hi[1], label);
2683 3029 : if (code2 != UNKNOWN)
2684 667 : ix86_expand_branch (code2, hi[0], hi[1], label2);
2685 :
2686 3029 : ix86_expand_branch (code3, lo[0], lo[1], label);
2687 :
2688 3029 : if (code2 != UNKNOWN)
2689 667 : emit_label (label2);
2690 : return;
2691 : }
2692 :
2693 17679 : default:
2694 17679 : gcc_assert (GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC);
2695 17679 : goto simple;
2696 : }
2697 : }
2698 :
2699 : /* Figure out whether to use unordered fp comparisons. */
2700 :
2701 : static bool
2702 1158709 : ix86_unordered_fp_compare (enum rtx_code code)
2703 : {
2704 1158709 : if (!TARGET_IEEE_FP)
2705 : return false;
2706 :
2707 1154297 : switch (code)
2708 : {
2709 : case LT:
2710 : case LE:
2711 : case GT:
2712 : case GE:
2713 : case LTGT:
2714 : return false;
2715 :
2716 : case EQ:
2717 : case NE:
2718 :
2719 : case UNORDERED:
2720 : case ORDERED:
2721 : case UNLT:
2722 : case UNLE:
2723 : case UNGT:
2724 : case UNGE:
2725 : case UNEQ:
2726 : return true;
2727 :
2728 0 : default:
2729 0 : gcc_unreachable ();
2730 : }
2731 : }
2732 :
2733 : /* Return a comparison we can do and that it is equivalent to
2734 : swap_condition (code) apart possibly from orderedness.
2735 : But, never change orderedness if TARGET_IEEE_FP, returning
2736 : UNKNOWN in that case if necessary. */
2737 :
2738 : static enum rtx_code
2739 38042 : ix86_fp_swap_condition (enum rtx_code code)
2740 : {
2741 38042 : switch (code)
2742 : {
2743 1888 : case GT: /* GTU - CF=0 & ZF=0 */
2744 1888 : return TARGET_IEEE_FP ? UNKNOWN : UNLT;
2745 544 : case GE: /* GEU - CF=0 */
2746 544 : return TARGET_IEEE_FP ? UNKNOWN : UNLE;
2747 504 : case UNLT: /* LTU - CF=1 */
2748 504 : return TARGET_IEEE_FP ? UNKNOWN : GT;
2749 6362 : case UNLE: /* LEU - CF=1 | ZF=1 */
2750 6362 : return TARGET_IEEE_FP ? UNKNOWN : GE;
2751 28744 : default:
2752 28744 : return swap_condition (code);
2753 : }
2754 : }
2755 :
2756 : /* Return cost of comparison CODE using the best strategy for performance.
2757 : All following functions do use number of instructions as a cost metrics.
2758 : In future this should be tweaked to compute bytes for optimize_size and
2759 : take into account performance of various instructions on various CPUs. */
2760 :
2761 : static int
2762 1157576 : ix86_fp_comparison_cost (enum rtx_code code)
2763 : {
2764 1157576 : int arith_cost;
2765 :
2766 : /* The cost of code using bit-twiddling on %ah. */
2767 1157576 : switch (code)
2768 : {
2769 : case UNLE:
2770 : case UNLT:
2771 : case LTGT:
2772 : case GT:
2773 : case GE:
2774 : case UNORDERED:
2775 : case ORDERED:
2776 : case UNEQ:
2777 : arith_cost = 4;
2778 : break;
2779 85565 : case LT:
2780 85565 : case NE:
2781 85565 : case EQ:
2782 85565 : case UNGE:
2783 85565 : arith_cost = TARGET_IEEE_FP ? 5 : 4;
2784 : break;
2785 25900 : case LE:
2786 25900 : case UNGT:
2787 1072815 : arith_cost = TARGET_IEEE_FP ? 6 : 4;
2788 : break;
2789 0 : default:
2790 0 : gcc_unreachable ();
2791 : }
2792 :
2793 1157576 : switch (ix86_fp_comparison_strategy (code))
2794 : {
2795 1157576 : case IX86_FPCMP_COMI:
2796 1157576 : return arith_cost > 4 ? 3 : 2;
2797 0 : case IX86_FPCMP_SAHF:
2798 0 : return arith_cost > 4 ? 4 : 3;
2799 : default:
2800 : return arith_cost;
2801 : }
2802 : }
2803 :
2804 : /* Swap, force into registers, or otherwise massage the two operands
2805 : to a fp comparison. The operands are updated in place; the new
2806 : comparison code is returned. */
2807 :
2808 : static enum rtx_code
2809 578788 : ix86_prepare_fp_compare_args (enum rtx_code code, rtx *pop0, rtx *pop1)
2810 : {
2811 578859 : bool unordered_compare = ix86_unordered_fp_compare (code);
2812 578859 : rtx op0 = *pop0, op1 = *pop1;
2813 578859 : machine_mode op_mode = GET_MODE (op0);
2814 578859 : bool is_sse = SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (op_mode);
2815 :
2816 576470 : if (op_mode == BFmode && (!TARGET_AVX10_2 || flag_trapping_math))
2817 : {
2818 71 : rtx op = gen_lowpart (HImode, op0);
2819 71 : if (CONST_INT_P (op))
2820 0 : op = simplify_const_unary_operation (FLOAT_EXTEND, SFmode,
2821 : op0, BFmode);
2822 : else
2823 : {
2824 71 : rtx t1 = gen_reg_rtx (SImode);
2825 71 : emit_insn (gen_zero_extendhisi2 (t1, op));
2826 71 : emit_insn (gen_ashlsi3 (t1, t1, GEN_INT (16)));
2827 71 : op = gen_lowpart (SFmode, t1);
2828 : }
2829 71 : *pop0 = op;
2830 71 : op = gen_lowpart (HImode, op1);
2831 71 : if (CONST_INT_P (op))
2832 6 : op = simplify_const_unary_operation (FLOAT_EXTEND, SFmode,
2833 : op1, BFmode);
2834 : else
2835 : {
2836 65 : rtx t1 = gen_reg_rtx (SImode);
2837 65 : emit_insn (gen_zero_extendhisi2 (t1, op));
2838 65 : emit_insn (gen_ashlsi3 (t1, t1, GEN_INT (16)));
2839 65 : op = gen_lowpart (SFmode, t1);
2840 : }
2841 71 : *pop1 = op;
2842 71 : return ix86_prepare_fp_compare_args (code, pop0, pop1);
2843 : }
2844 :
2845 : /* All of the unordered compare instructions only work on registers.
2846 : The same is true of the fcomi compare instructions. The XFmode
2847 : compare instructions require registers except when comparing
2848 : against zero or when converting operand 1 from fixed point to
2849 : floating point. */
2850 :
2851 578788 : if (!is_sse
2852 578788 : && (unordered_compare
2853 8135 : || (op_mode == XFmode
2854 10401 : && ! (standard_80387_constant_p (op0) == 1
2855 5197 : || standard_80387_constant_p (op1) == 1)
2856 4763 : && GET_CODE (op1) != FLOAT)
2857 3372 : || ix86_fp_comparison_strategy (code) == IX86_FPCMP_COMI))
2858 : {
2859 148199 : op0 = force_reg (op_mode, op0);
2860 148199 : op1 = force_reg (op_mode, op1);
2861 : }
2862 : else
2863 : {
2864 : /* %%% We only allow op1 in memory; op0 must be st(0). So swap
2865 : things around if they appear profitable, otherwise force op0
2866 : into a register. */
2867 :
2868 430589 : if (standard_80387_constant_p (op0) == 0
2869 430589 : || (MEM_P (op0)
2870 57385 : && ! (standard_80387_constant_p (op1) == 0
2871 41928 : || MEM_P (op1))))
2872 : {
2873 38042 : enum rtx_code new_code = ix86_fp_swap_condition (code);
2874 38042 : if (new_code != UNKNOWN)
2875 : {
2876 : std::swap (op0, op1);
2877 430589 : code = new_code;
2878 : }
2879 : }
2880 :
2881 430589 : if (!REG_P (op0))
2882 53539 : op0 = force_reg (op_mode, op0);
2883 :
2884 430589 : if (CONSTANT_P (op1))
2885 : {
2886 194949 : int tmp = standard_80387_constant_p (op1);
2887 194949 : if (tmp == 0)
2888 75050 : op1 = validize_mem (force_const_mem (op_mode, op1));
2889 119899 : else if (tmp == 1)
2890 : {
2891 65732 : if (TARGET_CMOVE)
2892 65732 : op1 = force_reg (op_mode, op1);
2893 : }
2894 : else
2895 54167 : op1 = force_reg (op_mode, op1);
2896 : }
2897 : }
2898 :
2899 : /* Try to rearrange the comparison to make it cheaper. */
2900 578788 : if (ix86_fp_comparison_cost (code)
2901 578788 : > ix86_fp_comparison_cost (swap_condition (code))
2902 578788 : && (REG_P (op1) || can_create_pseudo_p ()))
2903 : {
2904 0 : std::swap (op0, op1);
2905 0 : code = swap_condition (code);
2906 0 : if (!REG_P (op0))
2907 0 : op0 = force_reg (op_mode, op0);
2908 : }
2909 :
2910 578788 : *pop0 = op0;
2911 578788 : *pop1 = op1;
2912 578788 : return code;
2913 : }
2914 :
2915 : /* Generate insn patterns to do a floating point compare of OPERANDS. */
2916 :
2917 : static rtx
2918 578788 : ix86_expand_fp_compare (enum rtx_code code, rtx op0, rtx op1)
2919 : {
2920 578788 : bool unordered_compare = ix86_unordered_fp_compare (code);
2921 578788 : machine_mode cmp_mode;
2922 578788 : rtx tmp, scratch;
2923 :
2924 578788 : code = ix86_prepare_fp_compare_args (code, &op0, &op1);
2925 :
2926 578788 : tmp = gen_rtx_COMPARE (CCFPmode, op0, op1);
2927 578788 : if (unordered_compare)
2928 503006 : tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_NOTRAP);
2929 :
2930 : /* Do fcomi/sahf based test when profitable. */
2931 578788 : switch (ix86_fp_comparison_strategy (code))
2932 : {
2933 578788 : case IX86_FPCMP_COMI:
2934 578788 : tmp = gen_rtx_COMPARE (CCFPmode, op0, op1);
2935 : /* We only have vcomisbf16, No vcomubf16 nor vcomxbf16 */
2936 578788 : if (GET_MODE (op0) != E_BFmode)
2937 : {
2938 578760 : if (TARGET_AVX10_2 && (code == EQ || code == NE))
2939 972 : tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_OPTCOMX);
2940 578760 : if (unordered_compare)
2941 502998 : tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_NOTRAP);
2942 : }
2943 578788 : cmp_mode = CCFPmode;
2944 578788 : emit_insn (gen_rtx_SET (gen_rtx_REG (CCFPmode, FLAGS_REG), tmp));
2945 578788 : break;
2946 :
2947 0 : case IX86_FPCMP_SAHF:
2948 0 : cmp_mode = CCFPmode;
2949 0 : tmp = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW);
2950 0 : scratch = gen_reg_rtx (HImode);
2951 0 : emit_insn (gen_rtx_SET (scratch, tmp));
2952 0 : emit_insn (gen_x86_sahf_1 (scratch));
2953 0 : break;
2954 :
2955 0 : case IX86_FPCMP_ARITH:
2956 0 : cmp_mode = CCNOmode;
2957 0 : tmp = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW);
2958 0 : scratch = gen_reg_rtx (HImode);
2959 0 : emit_insn (gen_rtx_SET (scratch, tmp));
2960 :
2961 : /* In the unordered case, we have to check C2 for NaN's, which
2962 : doesn't happen to work out to anything nice combination-wise.
2963 : So do some bit twiddling on the value we've got in AH to come
2964 : up with an appropriate set of condition codes. */
2965 :
2966 0 : switch (code)
2967 : {
2968 0 : case GT:
2969 0 : case UNGT:
2970 0 : if (code == GT || !TARGET_IEEE_FP)
2971 : {
2972 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x45)));
2973 0 : code = EQ;
2974 : }
2975 : else
2976 : {
2977 0 : emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
2978 0 : emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
2979 0 : emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x44)));
2980 0 : cmp_mode = CCmode;
2981 0 : code = GEU;
2982 : }
2983 : break;
2984 0 : case LT:
2985 0 : case UNLT:
2986 0 : if (code == LT && TARGET_IEEE_FP)
2987 : {
2988 0 : emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
2989 0 : emit_insn (gen_cmpqi_ext_3 (scratch, const1_rtx));
2990 0 : cmp_mode = CCmode;
2991 0 : code = EQ;
2992 : }
2993 : else
2994 : {
2995 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, const1_rtx));
2996 0 : code = NE;
2997 : }
2998 : break;
2999 0 : case GE:
3000 0 : case UNGE:
3001 0 : if (code == GE || !TARGET_IEEE_FP)
3002 : {
3003 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x05)));
3004 0 : code = EQ;
3005 : }
3006 : else
3007 : {
3008 0 : emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
3009 0 : emit_insn (gen_xorqi_ext_1_cc (scratch, scratch, const1_rtx));
3010 0 : code = NE;
3011 : }
3012 : break;
3013 0 : case LE:
3014 0 : case UNLE:
3015 0 : if (code == LE && TARGET_IEEE_FP)
3016 : {
3017 0 : emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
3018 0 : emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
3019 0 : emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
3020 0 : cmp_mode = CCmode;
3021 0 : code = LTU;
3022 : }
3023 : else
3024 : {
3025 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x45)));
3026 0 : code = NE;
3027 : }
3028 : break;
3029 0 : case EQ:
3030 0 : case UNEQ:
3031 0 : if (code == EQ && TARGET_IEEE_FP)
3032 : {
3033 0 : emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
3034 0 : emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
3035 0 : cmp_mode = CCmode;
3036 0 : code = EQ;
3037 : }
3038 : else
3039 : {
3040 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x40)));
3041 0 : code = NE;
3042 : }
3043 : break;
3044 0 : case NE:
3045 0 : case LTGT:
3046 0 : if (code == NE && TARGET_IEEE_FP)
3047 : {
3048 0 : emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
3049 0 : emit_insn (gen_xorqi_ext_1_cc (scratch, scratch,
3050 : GEN_INT (0x40)));
3051 0 : code = NE;
3052 : }
3053 : else
3054 : {
3055 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x40)));
3056 0 : code = EQ;
3057 : }
3058 : break;
3059 :
3060 0 : case UNORDERED:
3061 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x04)));
3062 0 : code = NE;
3063 0 : break;
3064 0 : case ORDERED:
3065 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x04)));
3066 0 : code = EQ;
3067 0 : break;
3068 :
3069 0 : default:
3070 0 : gcc_unreachable ();
3071 : }
3072 : break;
3073 :
3074 0 : default:
3075 0 : gcc_unreachable();
3076 : }
3077 :
3078 : /* Return the test that should be put into the flags user, i.e.
3079 : the bcc, scc, or cmov instruction. */
3080 578788 : return gen_rtx_fmt_ee (code, VOIDmode,
3081 : gen_rtx_REG (cmp_mode, FLAGS_REG),
3082 : const0_rtx);
3083 : }
3084 :
3085 : /* Generate insn patterns to do an integer compare of OPERANDS. */
3086 :
3087 : static rtx
3088 7060664 : ix86_expand_int_compare (enum rtx_code code, rtx op0, rtx op1)
3089 : {
3090 7060664 : machine_mode cmpmode;
3091 7060664 : rtx tmp, flags;
3092 :
3093 : /* Swap operands to emit carry flag comparison. */
3094 7060664 : if ((code == GTU || code == LEU)
3095 7060664 : && nonimmediate_operand (op1, VOIDmode))
3096 : {
3097 149736 : std::swap (op0, op1);
3098 149736 : code = swap_condition (code);
3099 : }
3100 :
3101 7060664 : cmpmode = SELECT_CC_MODE (code, op0, op1);
3102 7060664 : flags = gen_rtx_REG (cmpmode, FLAGS_REG);
3103 :
3104 : /* Attempt to use PTEST, if available, when testing vector modes for
3105 : equality/inequality against zero. */
3106 7060664 : if (op1 == const0_rtx
3107 2931856 : && SUBREG_P (op0)
3108 24248 : && cmpmode == CCZmode
3109 11268 : && SUBREG_BYTE (op0) == 0
3110 9598 : && REG_P (SUBREG_REG (op0))
3111 9598 : && VECTOR_MODE_P (GET_MODE (SUBREG_REG (op0)))
3112 8 : && TARGET_SSE4_1
3113 2 : && GET_MODE (op0) == TImode
3114 7060668 : && GET_MODE_SIZE (GET_MODE (SUBREG_REG (op0))) == 16)
3115 : {
3116 2 : tmp = SUBREG_REG (op0);
3117 2 : if (GET_MODE (tmp) == V8HFmode || GET_MODE (tmp) == V8BFmode)
3118 1 : tmp = gen_lowpart (V8HImode, tmp);
3119 2 : tmp = gen_rtx_UNSPEC (CCZmode, gen_rtvec (2, tmp, tmp), UNSPEC_PTEST);
3120 : }
3121 : else
3122 7060662 : tmp = gen_rtx_COMPARE (cmpmode, op0, op1);
3123 :
3124 : /* This is very simple, but making the interface the same as in the
3125 : FP case makes the rest of the code easier. */
3126 7060664 : emit_insn (gen_rtx_SET (flags, tmp));
3127 :
3128 : /* Return the test that should be put into the flags user, i.e.
3129 : the bcc, scc, or cmov instruction. */
3130 7060664 : return gen_rtx_fmt_ee (code, VOIDmode, flags, const0_rtx);
3131 : }
3132 :
3133 : static rtx
3134 7768171 : ix86_expand_compare (enum rtx_code code, rtx op0, rtx op1)
3135 : {
3136 7768171 : rtx ret;
3137 :
3138 7768171 : if (GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC)
3139 130789 : ret = gen_rtx_fmt_ee (code, VOIDmode, op0, op1);
3140 :
3141 7637382 : else if (SCALAR_FLOAT_MODE_P (GET_MODE (op0)))
3142 : {
3143 576718 : gcc_assert (!DECIMAL_FLOAT_MODE_P (GET_MODE (op0)));
3144 576718 : ret = ix86_expand_fp_compare (code, op0, op1);
3145 : }
3146 : else
3147 7060664 : ret = ix86_expand_int_compare (code, op0, op1);
3148 :
3149 7768171 : return ret;
3150 : }
3151 :
3152 : void
3153 631819 : ix86_expand_setcc (rtx dest, enum rtx_code code, rtx op0, rtx op1)
3154 : {
3155 631819 : rtx ret;
3156 :
3157 631819 : gcc_assert (GET_MODE (dest) == QImode);
3158 :
3159 631819 : ret = ix86_expand_compare (code, op0, op1);
3160 631819 : PUT_MODE (ret, QImode);
3161 631819 : emit_insn (gen_rtx_SET (dest, ret));
3162 631819 : }
3163 :
3164 : /* Expand floating point op0 <=> op1, i.e.
3165 : dest = op0 == op1 ? 0 : op0 < op1 ? -1 : op0 > op1 ? 1 : -128. */
3166 :
3167 : void
3168 244 : ix86_expand_fp_spaceship (rtx dest, rtx op0, rtx op1, rtx op2)
3169 : {
3170 244 : gcc_checking_assert (ix86_fp_comparison_strategy (GT) != IX86_FPCMP_ARITH);
3171 244 : rtx zero = NULL_RTX;
3172 244 : if (op2 != const0_rtx
3173 52 : && (TARGET_IEEE_FP || TARGET_ZERO_EXTEND_WITH_AND)
3174 34 : && GET_MODE (dest) == SImode)
3175 34 : zero = force_reg (SImode, const0_rtx);
3176 244 : rtx gt = ix86_expand_fp_compare (GT, op0, op1);
3177 244 : rtx l0 = op2 == const0_rtx ? gen_label_rtx () : NULL_RTX;
3178 244 : rtx l1 = op2 == const0_rtx ? gen_label_rtx () : NULL_RTX;
3179 244 : rtx l2 = TARGET_IEEE_FP ? gen_label_rtx () : NULL_RTX;
3180 244 : rtx lend = gen_label_rtx ();
3181 244 : rtx tmp;
3182 244 : rtx_insn *jmp;
3183 244 : if (l2)
3184 : {
3185 207 : rtx un = gen_rtx_fmt_ee (UNORDERED, VOIDmode,
3186 : gen_rtx_REG (CCFPmode, FLAGS_REG), const0_rtx);
3187 207 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, un,
3188 : gen_rtx_LABEL_REF (VOIDmode, l2), pc_rtx);
3189 207 : jmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
3190 207 : add_reg_br_prob_note (jmp, profile_probability:: very_unlikely ());
3191 : }
3192 244 : if (op2 == const0_rtx)
3193 : {
3194 192 : rtx eq = gen_rtx_fmt_ee (UNEQ, VOIDmode,
3195 : gen_rtx_REG (CCFPmode, FLAGS_REG), const0_rtx);
3196 192 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, eq,
3197 : gen_rtx_LABEL_REF (VOIDmode, l0), pc_rtx);
3198 192 : jmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
3199 192 : add_reg_br_prob_note (jmp, profile_probability::unlikely ());
3200 192 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, gt,
3201 : gen_rtx_LABEL_REF (VOIDmode, l1), pc_rtx);
3202 192 : jmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
3203 192 : add_reg_br_prob_note (jmp, profile_probability::even ());
3204 192 : emit_move_insn (dest, constm1_rtx);
3205 192 : emit_jump (lend);
3206 192 : emit_label (l0);
3207 192 : emit_move_insn (dest, const0_rtx);
3208 192 : emit_jump (lend);
3209 192 : emit_label (l1);
3210 192 : emit_move_insn (dest, const1_rtx);
3211 : }
3212 : else
3213 : {
3214 52 : rtx lt_tmp = NULL_RTX;
3215 52 : if (GET_MODE (dest) != SImode || !TARGET_ZERO_EXTEND_WITH_AND)
3216 : {
3217 52 : lt_tmp = gen_reg_rtx (QImode);
3218 52 : ix86_expand_setcc (lt_tmp, UNLT, gen_rtx_REG (CCFPmode, FLAGS_REG),
3219 : const0_rtx);
3220 52 : if (GET_MODE (dest) != QImode)
3221 : {
3222 52 : tmp = gen_reg_rtx (GET_MODE (dest));
3223 52 : emit_insn (gen_rtx_SET (tmp,
3224 : gen_rtx_ZERO_EXTEND (GET_MODE (dest),
3225 : lt_tmp)));
3226 52 : lt_tmp = tmp;
3227 : }
3228 : }
3229 52 : rtx gt_tmp;
3230 52 : if (zero)
3231 : {
3232 : /* If TARGET_IEEE_FP and dest has SImode, emit SImode clear
3233 : before the floating point comparison and use setcc_si_slp
3234 : pattern to hide it from the combiner, so that it doesn't
3235 : undo it. Similarly for TARGET_ZERO_EXTEND_WITH_AND, where
3236 : the ZERO_EXTEND normally emitted would need to be AND
3237 : with flags clobber. */
3238 34 : tmp = ix86_expand_compare (GT, XEXP (gt, 0), const0_rtx);
3239 34 : PUT_MODE (tmp, QImode);
3240 34 : emit_insn (gen_setcc_si_slp (zero, tmp, zero));
3241 34 : gt_tmp = zero;
3242 : }
3243 : else
3244 : {
3245 18 : gt_tmp = gen_reg_rtx (QImode);
3246 18 : ix86_expand_setcc (gt_tmp, GT, XEXP (gt, 0), const0_rtx);
3247 18 : if (GET_MODE (dest) != QImode)
3248 : {
3249 18 : tmp = gen_reg_rtx (GET_MODE (dest));
3250 18 : emit_insn (gen_rtx_SET (tmp,
3251 : gen_rtx_ZERO_EXTEND (GET_MODE (dest),
3252 : gt_tmp)));
3253 18 : gt_tmp = tmp;
3254 : }
3255 : }
3256 52 : if (lt_tmp)
3257 : {
3258 52 : tmp = expand_simple_binop (GET_MODE (dest), MINUS, gt_tmp, lt_tmp,
3259 : dest, 0, OPTAB_DIRECT);
3260 52 : if (!rtx_equal_p (tmp, dest))
3261 0 : emit_move_insn (dest, tmp);
3262 : }
3263 : else
3264 : {
3265 : /* For TARGET_ZERO_EXTEND_WITH_AND emit sbb directly, as we can't
3266 : do ZERO_EXTEND without clobbering flags. */
3267 0 : tmp = ix86_expand_compare (UNLT, XEXP (gt, 0), const0_rtx);
3268 0 : PUT_MODE (tmp, SImode);
3269 0 : emit_insn (gen_subsi3_carry (dest, gt_tmp,
3270 0 : force_reg (GET_MODE (dest), const0_rtx),
3271 : XEXP (gt, 0), tmp));
3272 : }
3273 : }
3274 244 : emit_jump (lend);
3275 244 : if (l2)
3276 : {
3277 207 : emit_label (l2);
3278 207 : emit_move_insn (dest, op2 == const0_rtx ? GEN_INT (-128) : op2);
3279 : }
3280 244 : emit_label (lend);
3281 244 : }
3282 :
3283 : /* Expand integral op0 <=> op1, i.e.
3284 : dest = op0 == op1 ? 0 : op0 < op1 ? -1 : 1. */
3285 :
3286 : void
3287 164 : ix86_expand_int_spaceship (rtx dest, rtx op0, rtx op1, rtx op2)
3288 : {
3289 164 : gcc_assert (INTVAL (op2));
3290 164 : rtx zero1 = NULL_RTX, zero2 = NULL_RTX;
3291 164 : if (TARGET_ZERO_EXTEND_WITH_AND && GET_MODE (dest) == SImode)
3292 : {
3293 0 : zero1 = force_reg (SImode, const0_rtx);
3294 0 : if (INTVAL (op2) != 1)
3295 0 : zero2 = force_reg (SImode, const0_rtx);
3296 : }
3297 :
3298 : /* Not using ix86_expand_int_compare here, so that it doesn't swap
3299 : operands nor optimize CC mode - we need a mode usable for both
3300 : LT and GT resp. LTU and GTU comparisons with the same unswapped
3301 : operands. */
3302 204 : rtx flags = gen_rtx_REG (INTVAL (op2) != 1 ? CCGCmode : CCmode, FLAGS_REG);
3303 164 : rtx tmp = gen_rtx_COMPARE (GET_MODE (flags), op0, op1);
3304 164 : emit_insn (gen_rtx_SET (flags, tmp));
3305 164 : rtx lt_tmp = NULL_RTX;
3306 164 : if (zero2)
3307 : {
3308 : /* For TARGET_ZERO_EXTEND_WITH_AND, emit setcc_si_slp to avoid
3309 : ZERO_EXTEND. */
3310 0 : tmp = ix86_expand_compare (LT, flags, const0_rtx);
3311 0 : PUT_MODE (tmp, QImode);
3312 0 : emit_insn (gen_setcc_si_slp (zero2, tmp, zero2));
3313 0 : lt_tmp = zero2;
3314 : }
3315 164 : else if (!zero1)
3316 : {
3317 164 : lt_tmp = gen_reg_rtx (QImode);
3318 204 : ix86_expand_setcc (lt_tmp, INTVAL (op2) != 1 ? LT : LTU, flags,
3319 : const0_rtx);
3320 164 : if (GET_MODE (dest) != QImode)
3321 : {
3322 164 : tmp = gen_reg_rtx (GET_MODE (dest));
3323 164 : emit_insn (gen_rtx_SET (tmp, gen_rtx_ZERO_EXTEND (GET_MODE (dest),
3324 : lt_tmp)));
3325 164 : lt_tmp = tmp;
3326 : }
3327 : }
3328 164 : rtx gt_tmp;
3329 164 : if (zero1)
3330 : {
3331 : /* For TARGET_ZERO_EXTEND_WITH_AND, emit setcc_si_slp to avoid
3332 : ZERO_EXTEND. */
3333 0 : tmp = ix86_expand_compare (INTVAL (op2) != 1 ? GT : GTU, flags,
3334 : const0_rtx);
3335 0 : PUT_MODE (tmp, QImode);
3336 0 : emit_insn (gen_setcc_si_slp (zero1, tmp, zero1));
3337 0 : gt_tmp = zero1;
3338 : }
3339 : else
3340 : {
3341 164 : gt_tmp = gen_reg_rtx (QImode);
3342 204 : ix86_expand_setcc (gt_tmp, INTVAL (op2) != 1 ? GT : GTU, flags,
3343 : const0_rtx);
3344 164 : if (GET_MODE (dest) != QImode)
3345 : {
3346 164 : tmp = gen_reg_rtx (GET_MODE (dest));
3347 164 : emit_insn (gen_rtx_SET (tmp, gen_rtx_ZERO_EXTEND (GET_MODE (dest),
3348 : gt_tmp)));
3349 164 : gt_tmp = tmp;
3350 : }
3351 : }
3352 164 : if (lt_tmp)
3353 : {
3354 164 : tmp = expand_simple_binop (GET_MODE (dest), MINUS, gt_tmp, lt_tmp, dest,
3355 : 0, OPTAB_DIRECT);
3356 164 : if (!rtx_equal_p (tmp, dest))
3357 0 : emit_move_insn (dest, tmp);
3358 : }
3359 : else
3360 : {
3361 : /* For TARGET_ZERO_EXTEND_WITH_AND emit sbb directly, as we can't
3362 : do ZERO_EXTEND without clobbering flags. */
3363 0 : tmp = ix86_expand_compare (LTU, flags, const0_rtx);
3364 0 : PUT_MODE (tmp, SImode);
3365 0 : emit_insn (gen_subsi3_carry (dest, gt_tmp,
3366 0 : force_reg (GET_MODE (dest), const0_rtx),
3367 : flags, tmp));
3368 : }
3369 164 : }
3370 :
3371 : /* Expand comparison setting or clearing carry flag. Return true when
3372 : successful and set pop for the operation. */
3373 : static bool
3374 25401 : ix86_expand_carry_flag_compare (enum rtx_code code, rtx op0, rtx op1, rtx *pop)
3375 : {
3376 50802 : machine_mode mode
3377 25401 : = GET_MODE (op0) != VOIDmode ? GET_MODE (op0) : GET_MODE (op1);
3378 :
3379 : /* Do not handle double-mode compares that go through special path. */
3380 27366 : if (mode == (TARGET_64BIT ? TImode : DImode))
3381 : return false;
3382 :
3383 25391 : if (SCALAR_FLOAT_MODE_P (mode))
3384 : {
3385 1828 : rtx compare_op;
3386 1828 : rtx_insn *compare_seq;
3387 :
3388 1828 : gcc_assert (!DECIMAL_FLOAT_MODE_P (mode));
3389 :
3390 : /* Shortcut: following common codes never translate
3391 : into carry flag compares. */
3392 1828 : if (code == EQ || code == NE || code == UNEQ || code == LTGT
3393 : || code == ORDERED || code == UNORDERED)
3394 : return false;
3395 :
3396 : /* These comparisons require zero flag; swap operands so they won't. */
3397 : if ((code == GT || code == UNLE || code == LE || code == UNGT)
3398 1759 : && !TARGET_IEEE_FP)
3399 : {
3400 2 : std::swap (op0, op1);
3401 2 : code = swap_condition (code);
3402 : }
3403 :
3404 : /* Try to expand the comparison and verify that we end up with
3405 : carry flag based comparison. This fails to be true only when
3406 : we decide to expand comparison using arithmetic that is not
3407 : too common scenario. */
3408 1826 : start_sequence ();
3409 1826 : compare_op = ix86_expand_fp_compare (code, op0, op1);
3410 1826 : compare_seq = end_sequence ();
3411 :
3412 1826 : if (GET_MODE (XEXP (compare_op, 0)) == CCFPmode)
3413 1826 : code = ix86_fp_compare_code_to_integer (GET_CODE (compare_op));
3414 : else
3415 0 : code = GET_CODE (compare_op);
3416 :
3417 1826 : if (code != LTU && code != GEU)
3418 : return false;
3419 :
3420 67 : emit_insn (compare_seq);
3421 67 : *pop = compare_op;
3422 67 : return true;
3423 : }
3424 :
3425 23563 : if (!INTEGRAL_MODE_P (mode))
3426 : return false;
3427 :
3428 23492 : switch (code)
3429 : {
3430 : case LTU:
3431 : case GEU:
3432 : break;
3433 :
3434 : /* Convert a==0 into (unsigned)a<1. */
3435 19580 : case EQ:
3436 19580 : case NE:
3437 19580 : if (op1 != const0_rtx)
3438 : return false;
3439 10010 : op1 = const1_rtx;
3440 10010 : code = (code == EQ ? LTU : GEU);
3441 : break;
3442 :
3443 : /* Convert a>b into b<a or a>=b-1. */
3444 779 : case GTU:
3445 779 : case LEU:
3446 779 : if (CONST_INT_P (op1))
3447 : {
3448 741 : op1 = gen_int_mode (INTVAL (op1) + 1, GET_MODE (op0));
3449 : /* Bail out on overflow. We still can swap operands but that
3450 : would force loading of the constant into register. */
3451 741 : if (op1 == const0_rtx
3452 741 : || !x86_64_immediate_operand (op1, GET_MODE (op1)))
3453 0 : return false;
3454 741 : code = (code == GTU ? GEU : LTU);
3455 : }
3456 : else
3457 : {
3458 38 : std::swap (op0, op1);
3459 38 : code = (code == GTU ? LTU : GEU);
3460 : }
3461 : break;
3462 :
3463 : /* Convert a>=0 into (unsigned)a<0x80000000. */
3464 1468 : case LT:
3465 1468 : case GE:
3466 1468 : if (mode == DImode || op1 != const0_rtx)
3467 : return false;
3468 204 : op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
3469 102 : code = (code == LT ? GEU : LTU);
3470 : break;
3471 965 : case LE:
3472 965 : case GT:
3473 965 : if (mode == DImode || op1 != constm1_rtx)
3474 : return false;
3475 0 : op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
3476 0 : code = (code == LE ? GEU : LTU);
3477 : break;
3478 :
3479 : default:
3480 : return false;
3481 : }
3482 : /* Swapping operands may cause constant to appear as first operand. */
3483 11591 : if (!nonimmediate_operand (op0, VOIDmode))
3484 : {
3485 0 : if (!can_create_pseudo_p ())
3486 : return false;
3487 0 : op0 = force_reg (mode, op0);
3488 : }
3489 11591 : *pop = ix86_expand_compare (code, op0, op1);
3490 11591 : gcc_assert (GET_CODE (*pop) == LTU || GET_CODE (*pop) == GEU);
3491 : return true;
3492 : }
3493 :
3494 : /* Expand conditional increment or decrement using adb/sbb instructions.
3495 : The default case using setcc followed by the conditional move can be
3496 : done by generic code. */
3497 : bool
3498 6602 : ix86_expand_int_addcc (rtx operands[])
3499 : {
3500 6602 : enum rtx_code code = GET_CODE (operands[1]);
3501 6602 : rtx flags;
3502 6602 : rtx (*insn) (machine_mode, rtx, rtx, rtx, rtx, rtx);
3503 6602 : rtx compare_op;
3504 6602 : rtx val = const0_rtx;
3505 6602 : bool fpcmp = false;
3506 6602 : machine_mode mode;
3507 6602 : rtx op0 = XEXP (operands[1], 0);
3508 6602 : rtx op1 = XEXP (operands[1], 1);
3509 :
3510 6602 : if (operands[3] != const1_rtx
3511 2912 : && operands[3] != constm1_rtx)
3512 : return false;
3513 4422 : if (!ix86_expand_carry_flag_compare (code, op0, op1, &compare_op))
3514 : return false;
3515 1281 : code = GET_CODE (compare_op);
3516 :
3517 1281 : flags = XEXP (compare_op, 0);
3518 :
3519 1281 : if (GET_MODE (flags) == CCFPmode)
3520 : {
3521 8 : fpcmp = true;
3522 8 : code = ix86_fp_compare_code_to_integer (code);
3523 : }
3524 :
3525 1281 : if (code != LTU)
3526 : {
3527 739 : val = constm1_rtx;
3528 739 : if (fpcmp)
3529 8 : PUT_CODE (compare_op,
3530 : reverse_condition_maybe_unordered
3531 : (GET_CODE (compare_op)));
3532 : else
3533 731 : PUT_CODE (compare_op, reverse_condition (GET_CODE (compare_op)));
3534 : }
3535 :
3536 1281 : mode = GET_MODE (operands[0]);
3537 :
3538 : /* Construct either adc or sbb insn. */
3539 1281 : if ((code == LTU) == (operands[3] == constm1_rtx))
3540 : insn = gen_sub3_carry;
3541 : else
3542 520 : insn = gen_add3_carry;
3543 :
3544 1281 : emit_insn (insn (mode, operands[0], operands[2], val, flags, compare_op));
3545 :
3546 1281 : return true;
3547 : }
3548 :
3549 : bool
3550 425687 : ix86_expand_int_movcc (rtx operands[])
3551 : {
3552 425687 : enum rtx_code code = GET_CODE (operands[1]), compare_code;
3553 425687 : rtx_insn *compare_seq;
3554 425687 : rtx compare_op;
3555 425687 : machine_mode mode = GET_MODE (operands[0]);
3556 425687 : bool sign_bit_compare_p = false;
3557 425687 : bool negate_cc_compare_p = false;
3558 425687 : rtx op0 = XEXP (operands[1], 0);
3559 425687 : rtx op1 = XEXP (operands[1], 1);
3560 425687 : rtx op2 = operands[2];
3561 425687 : rtx op3 = operands[3];
3562 :
3563 425687 : if (GET_MODE (op0) == TImode
3564 410253 : || (GET_MODE (op0) == DImode
3565 105109 : && !TARGET_64BIT))
3566 : return false;
3567 :
3568 409201 : if (GET_MODE (op0) == BFmode
3569 409201 : && !ix86_fp_comparison_operator (operands[1], VOIDmode))
3570 : return false;
3571 :
3572 409201 : start_sequence ();
3573 409201 : compare_op = ix86_expand_compare (code, op0, op1);
3574 409201 : compare_seq = end_sequence ();
3575 :
3576 409201 : compare_code = GET_CODE (compare_op);
3577 :
3578 409201 : if ((op1 == const0_rtx && (code == GE || code == LT))
3579 368352 : || (op1 == constm1_rtx && (code == GT || code == LE)))
3580 : sign_bit_compare_p = true;
3581 :
3582 : /* op0 == op1 ? op0 : op3 is equivalent to op0 == op1 ? op1 : op3,
3583 : but if op1 is a constant, the latter form allows more optimizations,
3584 : either through the last 2 ops being constant handling, or the one
3585 : constant and one variable cases. On the other side, for cmov the
3586 : former might be better as we don't need to load the constant into
3587 : another register. */
3588 368352 : if (code == EQ && CONST_INT_P (op1) && rtx_equal_p (op0, op2))
3589 : op2 = op1;
3590 : /* Similarly for op0 != op1 ? op2 : op0 and op0 != op1 ? op2 : op1. */
3591 408693 : else if (code == NE && CONST_INT_P (op1) && rtx_equal_p (op0, op3))
3592 : op3 = op1;
3593 :
3594 : /* Don't attempt mode expansion here -- if we had to expand 5 or 6
3595 : HImode insns, we'd be swallowed in word prefix ops. */
3596 :
3597 4889 : if ((mode != HImode || TARGET_FAST_PREFIX)
3598 437678 : && (mode != (TARGET_64BIT ? TImode : DImode))
3599 409201 : && CONST_INT_P (op2)
3600 437702 : && CONST_INT_P (op3))
3601 : {
3602 21756 : rtx out = operands[0];
3603 21756 : HOST_WIDE_INT ct = INTVAL (op2);
3604 21756 : HOST_WIDE_INT cf = INTVAL (op3);
3605 21756 : HOST_WIDE_INT diff;
3606 :
3607 21756 : if ((mode == SImode
3608 7828 : || (TARGET_64BIT && mode == DImode))
3609 18379 : && (GET_MODE (op0) == SImode
3610 14292 : || (TARGET_64BIT && GET_MODE (op0) == DImode)))
3611 : {
3612 : /* Special case x != 0 ? -1 : y. */
3613 13725 : if (code == NE && op1 == const0_rtx && ct == -1)
3614 : {
3615 : negate_cc_compare_p = true;
3616 : std::swap (ct, cf);
3617 : code = EQ;
3618 : }
3619 13607 : else if (code == EQ && op1 == const0_rtx && cf == -1)
3620 21756 : negate_cc_compare_p = true;
3621 : }
3622 :
3623 21756 : diff = (unsigned HOST_WIDE_INT) ct - cf;
3624 : /* Make sure we can represent the difference between the two values. */
3625 21756 : if ((diff > 0) != ((cf < 0) != (ct < 0) ? cf < 0 : cf < ct))
3626 425687 : return false;
3627 :
3628 : /* Sign bit compares are better done using shifts than we do by using
3629 : sbb. */
3630 21678 : if (sign_bit_compare_p
3631 21678 : || negate_cc_compare_p
3632 21678 : || ix86_expand_carry_flag_compare (code, op0, op1, &compare_op))
3633 : {
3634 : /* Detect overlap between destination and compare sources. */
3635 11076 : rtx tmp = out;
3636 :
3637 11076 : if (negate_cc_compare_p)
3638 : {
3639 296 : if (GET_MODE (op0) == DImode)
3640 120 : emit_insn (gen_x86_negdi_ccc (gen_reg_rtx (DImode), op0));
3641 : else
3642 176 : emit_insn (gen_x86_negsi_ccc (gen_reg_rtx (SImode),
3643 176 : gen_lowpart (SImode, op0)));
3644 :
3645 296 : tmp = gen_reg_rtx (mode);
3646 296 : if (mode == DImode)
3647 139 : emit_insn (gen_x86_movdicc_0_m1_neg (tmp));
3648 : else
3649 157 : emit_insn (gen_x86_movsicc_0_m1_neg (gen_lowpart (SImode,
3650 : tmp)));
3651 : }
3652 10780 : else if (!sign_bit_compare_p)
3653 : {
3654 10377 : rtx flags;
3655 10377 : bool fpcmp = false;
3656 :
3657 10377 : compare_code = GET_CODE (compare_op);
3658 :
3659 10377 : flags = XEXP (compare_op, 0);
3660 :
3661 10377 : if (GET_MODE (flags) == CCFPmode)
3662 : {
3663 59 : fpcmp = true;
3664 59 : compare_code
3665 59 : = ix86_fp_compare_code_to_integer (compare_code);
3666 : }
3667 :
3668 : /* To simplify rest of code, restrict to the GEU case. */
3669 10377 : if (compare_code == LTU)
3670 : {
3671 6237 : std::swap (ct, cf);
3672 6237 : compare_code = reverse_condition (compare_code);
3673 6237 : code = reverse_condition (code);
3674 : }
3675 : else
3676 : {
3677 4140 : if (fpcmp)
3678 59 : PUT_CODE (compare_op,
3679 : reverse_condition_maybe_unordered
3680 : (GET_CODE (compare_op)));
3681 : else
3682 4081 : PUT_CODE (compare_op,
3683 : reverse_condition (GET_CODE (compare_op)));
3684 : }
3685 :
3686 10377 : diff = (unsigned HOST_WIDE_INT) ct - cf;
3687 : /* Make sure we can represent the difference
3688 : between the two values. */
3689 10377 : if ((diff > 0) != ((cf < 0) != (ct < 0) ? cf < 0 : cf < ct))
3690 : return false;
3691 :
3692 10376 : if (reg_overlap_mentioned_p (out, compare_op))
3693 0 : tmp = gen_reg_rtx (mode);
3694 :
3695 10376 : if (mode == DImode)
3696 1833 : emit_insn (gen_x86_movdicc_0_m1 (tmp, flags, compare_op));
3697 : else
3698 8543 : emit_insn (gen_x86_movsicc_0_m1 (gen_lowpart (SImode, tmp),
3699 : flags, compare_op));
3700 : }
3701 : else
3702 : {
3703 403 : if (code == GT || code == GE)
3704 171 : code = reverse_condition (code);
3705 : else
3706 : {
3707 232 : std::swap (ct, cf);
3708 :
3709 232 : diff = (unsigned HOST_WIDE_INT) ct - cf;
3710 : /* Make sure we can represent the difference
3711 : between the two values. */
3712 232 : if ((diff > 0) != ((cf < 0) != (ct < 0) ? cf < 0 : cf < ct))
3713 : return false;
3714 : }
3715 400 : tmp = emit_store_flag (tmp, code, op0, op1, VOIDmode, 0, -1);
3716 : }
3717 :
3718 11072 : if (diff == 1)
3719 : {
3720 : /*
3721 : * cmpl op0,op1
3722 : * sbbl dest,dest
3723 : * [addl dest, ct]
3724 : *
3725 : * Size 5 - 8.
3726 : */
3727 795 : if (ct)
3728 635 : tmp = expand_simple_binop (mode, PLUS,
3729 : tmp, GEN_INT (ct),
3730 : copy_rtx (tmp), 1, OPTAB_DIRECT);
3731 : }
3732 10277 : else if (cf == -1)
3733 : {
3734 : /*
3735 : * cmpl op0,op1
3736 : * sbbl dest,dest
3737 : * orl $ct, dest
3738 : *
3739 : * Size 8.
3740 : */
3741 684 : tmp = expand_simple_binop (mode, IOR,
3742 : tmp, GEN_INT (ct),
3743 : copy_rtx (tmp), 1, OPTAB_DIRECT);
3744 : }
3745 9593 : else if (diff == -1 && ct)
3746 : {
3747 : /*
3748 : * cmpl op0,op1
3749 : * sbbl dest,dest
3750 : * notl dest
3751 : * [addl dest, cf]
3752 : *
3753 : * Size 8 - 11.
3754 : */
3755 486 : tmp = expand_simple_unop (mode, NOT, tmp, copy_rtx (tmp), 1);
3756 486 : if (cf)
3757 468 : tmp = expand_simple_binop (mode, PLUS,
3758 : copy_rtx (tmp), GEN_INT (cf),
3759 : copy_rtx (tmp), 1, OPTAB_DIRECT);
3760 : }
3761 : else
3762 : {
3763 : /*
3764 : * cmpl op0,op1
3765 : * sbbl dest,dest
3766 : * [notl dest]
3767 : * andl cf - ct, dest
3768 : * [addl dest, ct]
3769 : *
3770 : * Size 8 - 11.
3771 : */
3772 :
3773 9107 : if (cf == 0)
3774 : {
3775 872 : cf = ct;
3776 872 : ct = 0;
3777 872 : tmp = expand_simple_unop (mode, NOT, tmp, copy_rtx (tmp), 1);
3778 : }
3779 :
3780 9107 : HOST_WIDE_INT ival = (unsigned HOST_WIDE_INT) cf - ct;
3781 : /* Make sure we can represent the difference
3782 : between the two values. */
3783 9107 : if ((ival > 0) != ((ct < 0) != (cf < 0) ? ct < 0 : ct < cf))
3784 16568 : return false;
3785 :
3786 9107 : tmp = expand_simple_binop (mode, AND,
3787 : copy_rtx (tmp),
3788 9107 : gen_int_mode (ival, mode),
3789 : copy_rtx (tmp), 1, OPTAB_DIRECT);
3790 9107 : if (ct)
3791 7384 : tmp = expand_simple_binop (mode, PLUS,
3792 : copy_rtx (tmp), GEN_INT (ct),
3793 : copy_rtx (tmp), 1, OPTAB_DIRECT);
3794 : }
3795 :
3796 11072 : if (!rtx_equal_p (tmp, out))
3797 491 : emit_move_insn (copy_rtx (out), copy_rtx (tmp));
3798 :
3799 11072 : return true;
3800 : }
3801 :
3802 10602 : if (diff < 0)
3803 : {
3804 2655 : machine_mode cmp_mode = GET_MODE (op0);
3805 2655 : enum rtx_code new_code;
3806 :
3807 2655 : if (SCALAR_FLOAT_MODE_P (cmp_mode))
3808 : {
3809 55 : gcc_assert (!DECIMAL_FLOAT_MODE_P (cmp_mode));
3810 :
3811 : /* We may be reversing a non-trapping
3812 : comparison to a trapping comparison. */
3813 106 : if (HONOR_NANS (cmp_mode) && flag_trapping_math
3814 48 : && code != EQ && code != NE
3815 103 : && code != ORDERED && code != UNORDERED)
3816 : new_code = UNKNOWN;
3817 : else
3818 7 : new_code = reverse_condition_maybe_unordered (code);
3819 : }
3820 : else
3821 2600 : new_code = ix86_reverse_condition (code, cmp_mode);
3822 2607 : if (new_code != UNKNOWN)
3823 : {
3824 2607 : std::swap (ct, cf);
3825 :
3826 2607 : diff = (unsigned HOST_WIDE_INT) ct - cf;
3827 : /* Make sure we can represent the difference
3828 : between the two values. */
3829 2607 : if ((diff > 0) != ((cf < 0) != (ct < 0) ? cf < 0 : cf < ct))
3830 : return false;
3831 :
3832 : code = new_code;
3833 : }
3834 : }
3835 :
3836 10602 : compare_code = UNKNOWN;
3837 10602 : if (GET_MODE_CLASS (GET_MODE (op0)) == MODE_INT
3838 8916 : && CONST_INT_P (op1))
3839 : {
3840 2869 : if (op1 == const0_rtx
3841 209 : && (code == LT || code == GE))
3842 : compare_code = code;
3843 2869 : else if (op1 == constm1_rtx)
3844 : {
3845 86 : if (code == LE)
3846 : compare_code = LT;
3847 86 : else if (code == GT)
3848 : compare_code = GE;
3849 : }
3850 : }
3851 :
3852 : /* Optimize dest = (op0 < 0) ? -1 : cf. */
3853 : if (compare_code != UNKNOWN
3854 0 : && GET_MODE (op0) == GET_MODE (out)
3855 0 : && (cf == -1 || ct == -1))
3856 : {
3857 : /* If lea code below could be used, only optimize
3858 : if it results in a 2 insn sequence. */
3859 :
3860 0 : if (! (diff == 1 || diff == 2 || diff == 4 || diff == 8
3861 0 : || diff == 3 || diff == 5 || diff == 9)
3862 0 : || (compare_code == LT && ct == -1)
3863 0 : || (compare_code == GE && cf == -1))
3864 : {
3865 : /*
3866 : * notl op1 (if necessary)
3867 : * sarl $31, op1
3868 : * orl cf, op1
3869 : */
3870 0 : if (ct != -1)
3871 : {
3872 0 : cf = ct;
3873 0 : ct = -1;
3874 0 : code = reverse_condition (code);
3875 : }
3876 :
3877 0 : out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, -1);
3878 :
3879 0 : out = expand_simple_binop (mode, IOR,
3880 : out, GEN_INT (cf),
3881 : out, 1, OPTAB_DIRECT);
3882 0 : if (out != operands[0])
3883 0 : emit_move_insn (operands[0], out);
3884 :
3885 0 : return true;
3886 : }
3887 : }
3888 :
3889 :
3890 13865 : if ((diff == 1 || diff == 2 || diff == 4 || diff == 8
3891 3263 : || diff == 3 || diff == 5 || diff == 9)
3892 7705 : && ((mode != QImode && mode != HImode) || !TARGET_PARTIAL_REG_STALL)
3893 18307 : && (mode != DImode
3894 1828 : || x86_64_immediate_operand (GEN_INT (cf), VOIDmode)))
3895 : {
3896 : /*
3897 : * xorl dest,dest
3898 : * cmpl op1,op2
3899 : * setcc dest
3900 : * lea cf(dest*(ct-cf)),dest
3901 : *
3902 : * Size 14.
3903 : *
3904 : * This also catches the degenerate setcc-only case.
3905 : */
3906 :
3907 7705 : rtx tmp;
3908 7705 : int nops;
3909 :
3910 7705 : out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, 1);
3911 :
3912 7705 : nops = 0;
3913 : /* On x86_64 the lea instruction operates on Pmode, so we need
3914 : to get arithmetics done in proper mode to match. */
3915 7705 : if (diff == 1)
3916 6544 : tmp = copy_rtx (out);
3917 : else
3918 : {
3919 1161 : rtx out1;
3920 1161 : out1 = copy_rtx (out);
3921 1161 : tmp = gen_rtx_MULT (mode, out1, GEN_INT (diff & ~1));
3922 1161 : nops++;
3923 1161 : if (diff & 1)
3924 : {
3925 260 : tmp = gen_rtx_PLUS (mode, tmp, out1);
3926 260 : nops++;
3927 : }
3928 : }
3929 7705 : if (cf != 0)
3930 : {
3931 6972 : tmp = plus_constant (mode, tmp, cf);
3932 6972 : nops++;
3933 : }
3934 7705 : if (!rtx_equal_p (tmp, out))
3935 : {
3936 7210 : if (nops == 1)
3937 6145 : out = force_operand (tmp, copy_rtx (out));
3938 : else
3939 1065 : emit_insn (gen_rtx_SET (copy_rtx (out), copy_rtx (tmp)));
3940 : }
3941 7705 : if (!rtx_equal_p (out, operands[0]))
3942 657 : emit_move_insn (operands[0], copy_rtx (out));
3943 :
3944 7705 : return true;
3945 : }
3946 :
3947 : /*
3948 : * General case: Jumpful:
3949 : * xorl dest,dest cmpl op1, op2
3950 : * cmpl op1, op2 movl ct, dest
3951 : * setcc dest jcc 1f
3952 : * decl dest movl cf, dest
3953 : * andl (cf-ct),dest 1:
3954 : * addl ct,dest
3955 : *
3956 : * Size 20. Size 14.
3957 : *
3958 : * This is reasonably steep, but branch mispredict costs are
3959 : * high on modern cpus, so consider failing only if optimizing
3960 : * for space.
3961 : */
3962 :
3963 2897 : if ((!TARGET_CMOVE || (mode == QImode && TARGET_PARTIAL_REG_STALL))
3964 2897 : && BRANCH_COST (optimize_insn_for_speed_p (),
3965 : false) >= 2)
3966 : {
3967 0 : if (cf == 0)
3968 : {
3969 0 : machine_mode cmp_mode = GET_MODE (op0);
3970 0 : enum rtx_code new_code;
3971 :
3972 0 : if (SCALAR_FLOAT_MODE_P (cmp_mode))
3973 : {
3974 0 : gcc_assert (!DECIMAL_FLOAT_MODE_P (cmp_mode));
3975 :
3976 : /* We may be reversing a non-trapping
3977 : comparison to a trapping comparison. */
3978 0 : if (HONOR_NANS (cmp_mode) && flag_trapping_math
3979 0 : && code != EQ && code != NE
3980 0 : && code != ORDERED && code != UNORDERED)
3981 : new_code = UNKNOWN;
3982 : else
3983 0 : new_code = reverse_condition_maybe_unordered (code);
3984 :
3985 : }
3986 : else
3987 : {
3988 0 : new_code = ix86_reverse_condition (code, cmp_mode);
3989 0 : if (compare_code != UNKNOWN && new_code != UNKNOWN)
3990 0 : compare_code = reverse_condition (compare_code);
3991 : }
3992 :
3993 0 : if (new_code != UNKNOWN)
3994 : {
3995 0 : cf = ct;
3996 0 : ct = 0;
3997 0 : code = new_code;
3998 : }
3999 : }
4000 :
4001 0 : if (compare_code != UNKNOWN)
4002 : {
4003 : /* notl op1 (if needed)
4004 : sarl $31, op1
4005 : andl (cf-ct), op1
4006 : addl ct, op1
4007 :
4008 : For x < 0 (resp. x <= -1) there will be no notl,
4009 : so if possible swap the constants to get rid of the
4010 : complement.
4011 : True/false will be -1/0 while code below (store flag
4012 : followed by decrement) is 0/-1, so the constants need
4013 : to be exchanged once more. */
4014 :
4015 0 : if (compare_code == GE || !cf)
4016 : {
4017 0 : code = reverse_condition (code);
4018 0 : compare_code = LT;
4019 : }
4020 : else
4021 : std::swap (ct, cf);
4022 :
4023 0 : out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, -1);
4024 : }
4025 : else
4026 : {
4027 0 : out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, 1);
4028 :
4029 0 : out = expand_simple_binop (mode, PLUS, copy_rtx (out),
4030 : constm1_rtx,
4031 : copy_rtx (out), 1, OPTAB_DIRECT);
4032 : }
4033 :
4034 0 : HOST_WIDE_INT ival = (unsigned HOST_WIDE_INT) cf - ct;
4035 : /* Make sure we can represent the difference
4036 : between the two values. */
4037 0 : if ((ival > 0) != ((ct < 0) != (cf < 0) ? ct < 0 : ct < cf))
4038 : return false;
4039 :
4040 0 : out = expand_simple_binop (mode, AND, copy_rtx (out),
4041 0 : gen_int_mode (ival, mode),
4042 : copy_rtx (out), 1, OPTAB_DIRECT);
4043 0 : if (ct)
4044 0 : out = expand_simple_binop (mode, PLUS, copy_rtx (out), GEN_INT (ct),
4045 : copy_rtx (out), 1, OPTAB_DIRECT);
4046 0 : if (!rtx_equal_p (out, operands[0]))
4047 0 : emit_move_insn (operands[0], copy_rtx (out));
4048 :
4049 0 : return true;
4050 : }
4051 : }
4052 :
4053 390342 : if (!TARGET_CMOVE || (mode == QImode && TARGET_PARTIAL_REG_STALL))
4054 : {
4055 : /* Try a few things more with specific constants and a variable. */
4056 :
4057 0 : optab op;
4058 0 : rtx var, orig_out, out, tmp;
4059 :
4060 0 : if (BRANCH_COST (optimize_insn_for_speed_p (), false) <= 2)
4061 : return false;
4062 :
4063 0 : operands[2] = op2;
4064 0 : operands[3] = op3;
4065 :
4066 : /* If one of the two operands is an interesting constant, load a
4067 : constant with the above and mask it in with a logical operation. */
4068 :
4069 0 : if (CONST_INT_P (operands[2]))
4070 : {
4071 0 : var = operands[3];
4072 0 : if (INTVAL (operands[2]) == 0 && operands[3] != constm1_rtx)
4073 0 : operands[3] = constm1_rtx, op = and_optab;
4074 0 : else if (INTVAL (operands[2]) == -1 && operands[3] != const0_rtx)
4075 0 : operands[3] = const0_rtx, op = ior_optab;
4076 : else
4077 : return false;
4078 : }
4079 0 : else if (CONST_INT_P (operands[3]))
4080 : {
4081 0 : var = operands[2];
4082 0 : if (INTVAL (operands[3]) == 0 && operands[2] != constm1_rtx)
4083 : {
4084 : /* For smin (x, 0), expand as "x < 0 ? x : 0" instead of
4085 : "x <= 0 ? x : 0" to enable sign_bit_compare_p. */
4086 0 : if (code == LE && op1 == const0_rtx && rtx_equal_p (op0, var))
4087 0 : operands[1] = simplify_gen_relational (LT, VOIDmode,
4088 0 : GET_MODE (op0),
4089 : op0, const0_rtx);
4090 :
4091 0 : operands[2] = constm1_rtx;
4092 0 : op = and_optab;
4093 : }
4094 0 : else if (INTVAL (operands[3]) == -1 && operands[3] != const0_rtx)
4095 0 : operands[2] = const0_rtx, op = ior_optab;
4096 : else
4097 : return false;
4098 : }
4099 : else
4100 : return false;
4101 :
4102 0 : orig_out = operands[0];
4103 0 : tmp = gen_reg_rtx (mode);
4104 0 : operands[0] = tmp;
4105 :
4106 : /* Recurse to get the constant loaded. */
4107 0 : if (!ix86_expand_int_movcc (operands))
4108 : return false;
4109 :
4110 : /* Mask in the interesting variable. */
4111 0 : out = expand_binop (mode, op, var, tmp, orig_out, 0,
4112 : OPTAB_WIDEN);
4113 0 : if (!rtx_equal_p (out, orig_out))
4114 0 : emit_move_insn (copy_rtx (orig_out), copy_rtx (out));
4115 :
4116 0 : return true;
4117 : }
4118 :
4119 : /*
4120 : * For comparison with above,
4121 : *
4122 : * movl cf,dest
4123 : * movl ct,tmp
4124 : * cmpl op1,op2
4125 : * cmovcc tmp,dest
4126 : *
4127 : * Size 15.
4128 : */
4129 :
4130 390342 : if (! nonimmediate_operand (operands[2], mode))
4131 17038 : operands[2] = force_reg (mode, operands[2]);
4132 390342 : if (! nonimmediate_operand (operands[3], mode))
4133 166565 : operands[3] = force_reg (mode, operands[3]);
4134 :
4135 390342 : if (! register_operand (operands[2], VOIDmode)
4136 390342 : && (mode == QImode
4137 1093 : || ! register_operand (operands[3], VOIDmode)))
4138 1564 : operands[2] = force_reg (mode, operands[2]);
4139 :
4140 390342 : if (mode == QImode
4141 390342 : && ! register_operand (operands[3], VOIDmode))
4142 592 : operands[3] = force_reg (mode, operands[3]);
4143 :
4144 390342 : emit_insn (compare_seq);
4145 390342 : emit_insn (gen_rtx_SET (operands[0],
4146 : gen_rtx_IF_THEN_ELSE (mode,
4147 : compare_op, operands[2],
4148 : operands[3])));
4149 390342 : return true;
4150 : }
4151 :
4152 : /* Detect conditional moves that exactly match min/max operational
4153 : semantics. Note that this is IEEE safe, as long as we don't
4154 : interchange the operands.
4155 :
4156 : Returns FALSE if this conditional move doesn't match a MIN/MAX,
4157 : and TRUE if the operation is successful and instructions are emitted. */
4158 :
4159 : static bool
4160 9707 : ix86_expand_sse_fp_minmax (rtx dest, enum rtx_code code, rtx cmp_op0,
4161 : rtx cmp_op1, rtx if_true, rtx if_false)
4162 : {
4163 9707 : machine_mode mode = GET_MODE (dest);
4164 9707 : bool is_min;
4165 9707 : rtx tmp;
4166 :
4167 9707 : if (code == LT)
4168 : ;
4169 3230 : else if (code == LE && !HONOR_NANS (mode))
4170 : {
4171 : /* We can swap LE to GE and then invert to LT. */
4172 : std::swap (cmp_op0, cmp_op1);
4173 : std::swap (if_true, if_false);
4174 : }
4175 3189 : else if (code == UNGE)
4176 : std::swap (if_true, if_false);
4177 : else
4178 : return false;
4179 :
4180 8640 : if (rtx_equal_p (cmp_op0, if_true) && rtx_equal_p (cmp_op1, if_false))
4181 : is_min = true;
4182 4650 : else if (rtx_equal_p (cmp_op1, if_true) && rtx_equal_p (cmp_op0, if_false))
4183 : is_min = false;
4184 : else
4185 1051 : return false;
4186 :
4187 7589 : if (immediate_operand (if_false, mode))
4188 8 : if_false = force_reg (mode, if_false);
4189 7589 : if (immediate_operand (if_true, mode))
4190 0 : if_true = force_reg (mode, if_true);
4191 :
4192 : /* We want to check HONOR_NANS and HONOR_SIGNED_ZEROS here,
4193 : but MODE may be a vector mode and thus not appropriate. */
4194 7589 : if (!flag_finite_math_only || flag_signed_zeros)
4195 : {
4196 7589 : int u = is_min ? UNSPEC_IEEE_MIN : UNSPEC_IEEE_MAX;
4197 7589 : rtvec v;
4198 :
4199 7589 : if_true = force_reg (mode, if_true);
4200 7589 : v = gen_rtvec (2, if_true, if_false);
4201 7589 : tmp = gen_rtx_UNSPEC (mode, v, u);
4202 7589 : }
4203 : else
4204 : {
4205 0 : code = is_min ? SMIN : SMAX;
4206 0 : if (MEM_P (if_true) && MEM_P (if_false))
4207 0 : if_true = force_reg (mode, if_true);
4208 0 : tmp = gen_rtx_fmt_ee (code, mode, if_true, if_false);
4209 : }
4210 :
4211 7589 : emit_insn (gen_rtx_SET (dest, tmp));
4212 7589 : return true;
4213 : }
4214 :
4215 : /* Return true if MODE is valid for vector compare to mask register,
4216 : Same result for conditionl vector move with mask register. */
4217 : static bool
4218 15294 : ix86_valid_mask_cmp_mode (machine_mode mode)
4219 : {
4220 : /* XOP has its own vector conditional movement. */
4221 15294 : if (TARGET_XOP && !TARGET_AVX512F)
4222 : return false;
4223 :
4224 : /* HFmode only supports vcmpsh whose dest is mask register. */
4225 15288 : if (TARGET_AVX512FP16 && mode == HFmode)
4226 : return true;
4227 :
4228 : /* AVX512F is needed for mask operation. */
4229 15196 : if (!(TARGET_AVX512F && VECTOR_MODE_P (mode)))
4230 : return false;
4231 :
4232 : /* AVX512BW is needed for vector QI/HImode,
4233 : AVX512VL is needed for 128/256-bit vector. */
4234 182 : machine_mode inner_mode = GET_MODE_INNER (mode);
4235 182 : int vector_size = GET_MODE_SIZE (mode);
4236 182 : if ((inner_mode == QImode || inner_mode == HImode) && !TARGET_AVX512BW)
4237 : return false;
4238 :
4239 162 : return vector_size == 64 || TARGET_AVX512VL;
4240 : }
4241 :
4242 : /* Return true if integer mask comparison should be used. */
4243 : static bool
4244 56497 : ix86_use_mask_cmp_p (machine_mode mode, machine_mode cmp_mode,
4245 : rtx op_true, rtx op_false)
4246 : {
4247 56497 : int vector_size = GET_MODE_SIZE (mode);
4248 :
4249 56497 : if (cmp_mode == HFmode)
4250 : return true;
4251 56405 : else if (vector_size < 16)
4252 : return false;
4253 49617 : else if (vector_size == 64)
4254 : return true;
4255 99118 : else if (GET_MODE_INNER (cmp_mode) == HFmode)
4256 : return true;
4257 99118 : else if (GET_MODE_INNER (cmp_mode) == BFmode)
4258 : return true;
4259 :
4260 : /* When op_true is NULL, op_false must be NULL, or vice versa. */
4261 49559 : gcc_assert (!op_true == !op_false);
4262 :
4263 : /* When op_true/op_false is NULL or cmp_mode is not valid mask cmp mode,
4264 : vector dest is required. */
4265 49559 : if (!op_true || !ix86_valid_mask_cmp_mode (cmp_mode))
4266 : return false;
4267 :
4268 : /* Exclude those that could be optimized in ix86_expand_sse_movcc. */
4269 48 : if (op_false == CONST0_RTX (mode)
4270 48 : || op_true == CONST0_RTX (mode)
4271 48 : || (INTEGRAL_MODE_P (mode)
4272 40 : && (op_true == CONSTM1_RTX (mode)
4273 40 : || op_false == CONSTM1_RTX (mode))))
4274 0 : return false;
4275 :
4276 : return true;
4277 : }
4278 :
4279 : /* Expand an SSE comparison. Return the register with the result. */
4280 :
4281 : static rtx
4282 38232 : ix86_expand_sse_cmp (rtx dest, enum rtx_code code, rtx cmp_op0, rtx cmp_op1,
4283 : rtx op_true, rtx op_false)
4284 : {
4285 38232 : machine_mode mode = GET_MODE (dest);
4286 38232 : machine_mode cmp_ops_mode = GET_MODE (cmp_op0);
4287 :
4288 : /* In general case result of comparison can differ from operands' type. */
4289 38232 : machine_mode cmp_mode;
4290 :
4291 : /* In AVX512F the result of comparison is an integer mask. */
4292 38232 : bool maskcmp = false;
4293 38232 : rtx x;
4294 :
4295 38232 : if (ix86_use_mask_cmp_p (mode, cmp_ops_mode, op_true, op_false))
4296 : {
4297 145 : unsigned int nbits = GET_MODE_NUNITS (cmp_ops_mode);
4298 145 : maskcmp = true;
4299 145 : cmp_mode = nbits > 8 ? int_mode_for_size (nbits, 0).require () : E_QImode;
4300 : }
4301 : else
4302 : cmp_mode = cmp_ops_mode;
4303 :
4304 38232 : cmp_op0 = force_reg (cmp_ops_mode, cmp_op0);
4305 :
4306 76464 : bool (*op1_predicate)(rtx, machine_mode)
4307 38232 : = VECTOR_MODE_P (cmp_ops_mode) ? vector_operand : nonimmediate_operand;
4308 :
4309 38232 : if (!op1_predicate (cmp_op1, cmp_ops_mode))
4310 0 : cmp_op1 = force_reg (cmp_ops_mode, cmp_op1);
4311 :
4312 38232 : if (optimize
4313 503 : || (maskcmp && cmp_mode != mode)
4314 503 : || (op_true && reg_overlap_mentioned_p (dest, op_true))
4315 38735 : || (op_false && reg_overlap_mentioned_p (dest, op_false)))
4316 75313 : dest = gen_reg_rtx (maskcmp ? cmp_mode : mode);
4317 :
4318 38232 : if (maskcmp)
4319 : {
4320 145 : bool ok = ix86_expand_mask_vec_cmp (dest, code, cmp_op0, cmp_op1);
4321 145 : gcc_assert (ok);
4322 : return dest;
4323 : }
4324 :
4325 38087 : x = gen_rtx_fmt_ee (code, cmp_mode, cmp_op0, cmp_op1);
4326 :
4327 38087 : if (cmp_mode != mode)
4328 : {
4329 7987 : x = force_reg (cmp_ops_mode, x);
4330 7987 : convert_move (dest, x, false);
4331 : }
4332 : else
4333 30100 : emit_insn (gen_rtx_SET (dest, x));
4334 :
4335 : return dest;
4336 : }
4337 :
4338 : /* Emit x86 binary operand CODE in mode MODE for SSE vector
4339 : instructions that can be performed using GP registers. */
4340 :
4341 : static void
4342 6803 : ix86_emit_vec_binop (enum rtx_code code, machine_mode mode,
4343 : rtx dst, rtx src1, rtx src2)
4344 : {
4345 6803 : rtx tmp;
4346 :
4347 6803 : tmp = gen_rtx_SET (dst, gen_rtx_fmt_ee (code, mode, src1, src2));
4348 :
4349 6803 : if (GET_MODE_SIZE (mode) <= GET_MODE_SIZE (SImode)
4350 6803 : && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
4351 : {
4352 94 : rtx clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
4353 94 : tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, clob));
4354 : }
4355 :
4356 6803 : emit_insn (tmp);
4357 6803 : }
4358 :
4359 : /* Expand DEST = CMP ? OP_TRUE : OP_FALSE into a sequence of logical
4360 : operations. This is used for both scalar and vector conditional moves. */
4361 :
4362 : void
4363 10458 : ix86_expand_sse_movcc (rtx dest, rtx cmp, rtx op_true, rtx op_false)
4364 : {
4365 10458 : machine_mode mode = GET_MODE (dest);
4366 10458 : machine_mode cmpmode = GET_MODE (cmp);
4367 10458 : rtx x;
4368 :
4369 : /* Simplify trivial VEC_COND_EXPR to avoid ICE in pr97506. */
4370 10458 : if (rtx_equal_p (op_true, op_false))
4371 : {
4372 0 : emit_move_insn (dest, op_true);
4373 0 : return;
4374 : }
4375 :
4376 : /* If we have an integer mask and FP value then we need
4377 : to cast mask to FP mode. */
4378 10458 : if (mode != cmpmode && VECTOR_MODE_P (cmpmode))
4379 : {
4380 1649 : cmp = force_reg (cmpmode, cmp);
4381 1649 : cmp = gen_rtx_SUBREG (mode, cmp, 0);
4382 : }
4383 :
4384 : /* In AVX512F the result of comparison is an integer mask. */
4385 10458 : if (mode != cmpmode
4386 1794 : && GET_MODE_CLASS (cmpmode) == MODE_INT)
4387 : {
4388 145 : gcc_assert (ix86_valid_mask_cmp_mode (mode));
4389 : /* Using scalar/vector move with mask register. */
4390 145 : cmp = force_reg (cmpmode, cmp);
4391 : /* Optimize for mask zero. */
4392 290 : op_true = (op_true != CONST0_RTX (mode)
4393 145 : ? force_reg (mode, op_true) : op_true);
4394 290 : op_false = (op_false != CONST0_RTX (mode)
4395 145 : ? force_reg (mode, op_false) : op_false);
4396 145 : if (op_true == CONST0_RTX (mode))
4397 : {
4398 0 : if (cmpmode == E_DImode && !TARGET_64BIT)
4399 : {
4400 0 : x = gen_reg_rtx (cmpmode);
4401 0 : emit_insn (gen_knotdi (x, cmp));
4402 : }
4403 : else
4404 0 : x = expand_simple_unop (cmpmode, NOT, cmp, NULL, 1);
4405 : cmp = x;
4406 : /* Reverse op_true op_false. */
4407 : std::swap (op_true, op_false);
4408 : }
4409 :
4410 145 : if (mode == HFmode)
4411 92 : emit_insn (gen_movhf_mask (dest, op_true, op_false, cmp));
4412 : else
4413 53 : emit_insn (gen_rtx_SET (dest,
4414 : gen_rtx_VEC_MERGE (mode,
4415 : op_true, op_false, cmp)));
4416 145 : return;
4417 : }
4418 :
4419 10313 : if (vector_all_ones_operand (op_true, mode)
4420 10313 : && op_false == CONST0_RTX (mode))
4421 : {
4422 3 : emit_move_insn (dest, cmp);
4423 3 : return;
4424 : }
4425 10310 : else if (op_false == CONST0_RTX (mode))
4426 : {
4427 982 : x = expand_simple_binop (mode, AND, cmp, op_true,
4428 : dest, 1, OPTAB_DIRECT);
4429 982 : if (x != dest)
4430 0 : emit_move_insn (dest, x);
4431 982 : return;
4432 : }
4433 9328 : else if (op_true == CONST0_RTX (mode))
4434 : {
4435 118 : op_false = force_reg (mode, op_false);
4436 118 : x = gen_rtx_NOT (mode, cmp);
4437 118 : ix86_emit_vec_binop (AND, mode, dest, x, op_false);
4438 118 : return;
4439 : }
4440 9210 : else if (vector_all_ones_operand (op_true, mode))
4441 : {
4442 3 : x = expand_simple_binop (mode, IOR, cmp, op_false,
4443 : dest, 1, OPTAB_DIRECT);
4444 3 : if (x != dest)
4445 0 : emit_move_insn (dest, x);
4446 3 : return;
4447 : }
4448 :
4449 9207 : if (TARGET_XOP)
4450 : {
4451 65 : op_true = force_reg (mode, op_true);
4452 :
4453 65 : if (GET_MODE_SIZE (mode) < 16
4454 65 : || !nonimmediate_operand (op_false, mode))
4455 50 : op_false = force_reg (mode, op_false);
4456 :
4457 65 : emit_insn (gen_rtx_SET (dest,
4458 : gen_rtx_IF_THEN_ELSE (mode, cmp,
4459 : op_true, op_false)));
4460 65 : return;
4461 : }
4462 :
4463 9142 : rtx (*gen) (rtx, rtx, rtx, rtx) = NULL;
4464 9142 : machine_mode blend_mode = mode;
4465 :
4466 9142 : switch (mode)
4467 : {
4468 27 : case E_V2SFmode:
4469 27 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4470 : gen = gen_mmx_blendvps;
4471 : break;
4472 355 : case E_V4SFmode:
4473 355 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4474 : gen = gen_sse4_1_blendvps;
4475 : break;
4476 188 : case E_V2DFmode:
4477 188 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4478 : gen = gen_sse4_1_blendvpd;
4479 : break;
4480 1095 : case E_SFmode:
4481 1095 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4482 : gen = gen_sse4_1_blendvss;
4483 : break;
4484 796 : case E_DFmode:
4485 796 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4486 : gen = gen_sse4_1_blendvsd;
4487 : break;
4488 221 : case E_V8QImode:
4489 221 : case E_V4HImode:
4490 221 : case E_V4HFmode:
4491 221 : case E_V4BFmode:
4492 221 : case E_V2SImode:
4493 221 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4494 : {
4495 : gen = gen_mmx_pblendvb_v8qi;
4496 : blend_mode = V8QImode;
4497 : }
4498 : break;
4499 95 : case E_V4QImode:
4500 95 : case E_V2HImode:
4501 95 : case E_V2HFmode:
4502 95 : case E_V2BFmode:
4503 95 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4504 : {
4505 : gen = gen_mmx_pblendvb_v4qi;
4506 : blend_mode = V4QImode;
4507 : }
4508 : break;
4509 36 : case E_V2QImode:
4510 36 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4511 : gen = gen_mmx_pblendvb_v2qi;
4512 : break;
4513 5660 : case E_V16QImode:
4514 5660 : case E_V8HImode:
4515 5660 : case E_V8HFmode:
4516 5660 : case E_V8BFmode:
4517 5660 : case E_V4SImode:
4518 5660 : case E_V2DImode:
4519 5660 : case E_V1TImode:
4520 5660 : if (TARGET_SSE_MOVCC_USE_BLENDV && TARGET_SSE4_1)
4521 : {
4522 : gen = gen_sse4_1_pblendvb;
4523 : blend_mode = V16QImode;
4524 : }
4525 : break;
4526 99 : case E_V8SFmode:
4527 99 : if (TARGET_AVX && TARGET_SSE_MOVCC_USE_BLENDV)
4528 : gen = gen_avx_blendvps256;
4529 : break;
4530 192 : case E_V4DFmode:
4531 192 : if (TARGET_AVX && TARGET_SSE_MOVCC_USE_BLENDV)
4532 : gen = gen_avx_blendvpd256;
4533 : break;
4534 378 : case E_V32QImode:
4535 378 : case E_V16HImode:
4536 378 : case E_V16HFmode:
4537 378 : case E_V16BFmode:
4538 378 : case E_V8SImode:
4539 378 : case E_V4DImode:
4540 378 : if (TARGET_AVX2 && TARGET_SSE_MOVCC_USE_BLENDV)
4541 : {
4542 : gen = gen_avx2_pblendvb;
4543 : blend_mode = V32QImode;
4544 : }
4545 : break;
4546 :
4547 0 : case E_V64QImode:
4548 0 : gen = gen_avx512bw_blendmv64qi;
4549 0 : break;
4550 0 : case E_V32HImode:
4551 0 : gen = gen_avx512bw_blendmv32hi;
4552 0 : break;
4553 0 : case E_V32HFmode:
4554 0 : gen = gen_avx512bw_blendmv32hf;
4555 0 : break;
4556 0 : case E_V32BFmode:
4557 0 : gen = gen_avx512bw_blendmv32bf;
4558 0 : break;
4559 0 : case E_V16SImode:
4560 0 : gen = gen_avx512f_blendmv16si;
4561 0 : break;
4562 0 : case E_V8DImode:
4563 0 : gen = gen_avx512f_blendmv8di;
4564 0 : break;
4565 0 : case E_V8DFmode:
4566 0 : gen = gen_avx512f_blendmv8df;
4567 0 : break;
4568 : case E_V16SFmode:
4569 : gen = gen_avx512f_blendmv16sf;
4570 : break;
4571 :
4572 : default:
4573 : break;
4574 : }
4575 :
4576 0 : if (gen != NULL)
4577 : {
4578 2086 : if (GET_MODE_SIZE (mode) < 16
4579 2086 : || !vector_operand (op_true, mode))
4580 549 : op_true = force_reg (mode, op_true);
4581 2086 : op_false = force_reg (mode, op_false);
4582 :
4583 2086 : if (blend_mode == mode)
4584 : x = dest;
4585 : else
4586 : {
4587 1036 : x = gen_reg_rtx (blend_mode);
4588 1036 : op_false = gen_lowpart (blend_mode, op_false);
4589 1036 : op_true = gen_lowpart (blend_mode, op_true);
4590 1036 : cmp = gen_lowpart (blend_mode, cmp);
4591 : }
4592 :
4593 2086 : emit_insn (gen (x, op_false, op_true, cmp));
4594 :
4595 2086 : if (x != dest)
4596 1036 : emit_move_insn (dest, gen_lowpart (mode, x));
4597 : }
4598 7056 : else if (CONST_VECTOR_P (op_true) && CONST_VECTOR_P (op_false))
4599 : {
4600 472 : rtx tmp = simplify_const_binary_operation (XOR, mode, op_true, op_false);
4601 472 : tmp = expand_simple_binop (mode, AND, cmp, tmp,
4602 : NULL, 1, OPTAB_DIRECT);
4603 472 : tmp = expand_simple_binop (mode, XOR, tmp, op_false,
4604 : dest, 1, OPTAB_DIRECT);
4605 472 : if (tmp != dest)
4606 0 : emit_move_insn (dest, tmp);
4607 : }
4608 : else
4609 : {
4610 6584 : rtx t2 = expand_simple_binop (mode, AND, cmp, op_true,
4611 : NULL, 1, OPTAB_DIRECT);
4612 :
4613 6584 : rtx t3 = gen_reg_rtx (mode);
4614 6584 : x = gen_rtx_NOT (mode, cmp);
4615 6584 : op_false = force_reg (mode, op_false);
4616 6584 : ix86_emit_vec_binop (AND, mode, t3, x, op_false);
4617 :
4618 6584 : x = expand_simple_binop (mode, IOR, t3, t2,
4619 : dest, 1, OPTAB_DIRECT);
4620 6584 : if (x != dest)
4621 0 : emit_move_insn (dest, x);
4622 : }
4623 : }
4624 :
4625 : /* Swap, force into registers, or otherwise massage the two operands
4626 : to an sse comparison with a mask result. Thus we differ a bit from
4627 : ix86_prepare_fp_compare_args which expects to produce a flags result.
4628 :
4629 : The DEST operand exists to help determine whether to commute commutative
4630 : operators. The POP0/POP1 operands are updated in place. The new
4631 : comparison code is returned, or UNKNOWN if not implementable. */
4632 :
4633 : static enum rtx_code
4634 17694 : ix86_prepare_sse_fp_compare_args (rtx dest, enum rtx_code code,
4635 : rtx *pop0, rtx *pop1)
4636 : {
4637 17694 : switch (code)
4638 : {
4639 67 : case LTGT:
4640 67 : case UNEQ:
4641 : /* AVX supports all the needed comparisons. */
4642 67 : if (TARGET_AVX)
4643 : break;
4644 : /* We have no LTGT as an operator. We could implement it with
4645 : NE & ORDERED, but this requires an extra temporary. It's
4646 : not clear that it's worth it. */
4647 : return UNKNOWN;
4648 :
4649 : case LT:
4650 : case LE:
4651 : case UNGT:
4652 : case UNGE:
4653 : /* These are supported directly. */
4654 : break;
4655 :
4656 6010 : case EQ:
4657 6010 : case NE:
4658 6010 : case UNORDERED:
4659 6010 : case ORDERED:
4660 : /* AVX has 3 operand comparisons, no need to swap anything. */
4661 6010 : if (TARGET_AVX)
4662 : break;
4663 : /* For commutative operators, try to canonicalize the destination
4664 : operand to be first in the comparison - this helps reload to
4665 : avoid extra moves. */
4666 1435 : if (!dest || !rtx_equal_p (dest, *pop1))
4667 : break;
4668 : /* FALLTHRU */
4669 :
4670 10564 : case GE:
4671 10564 : case GT:
4672 10564 : case UNLE:
4673 10564 : case UNLT:
4674 : /* These are not supported directly before AVX, and furthermore
4675 : ix86_expand_sse_fp_minmax only optimizes LT/UNGE. Swap the
4676 : comparison operands to transform into something that is
4677 : supported. */
4678 10564 : std::swap (*pop0, *pop1);
4679 10564 : code = swap_condition (code);
4680 10564 : break;
4681 :
4682 0 : default:
4683 0 : gcc_unreachable ();
4684 : }
4685 :
4686 : return code;
4687 : }
4688 :
4689 : /* Expand a floating-point conditional move. Return true if successful. */
4690 :
4691 : bool
4692 80230 : ix86_expand_fp_movcc (rtx operands[])
4693 : {
4694 80230 : machine_mode mode = GET_MODE (operands[0]);
4695 80230 : enum rtx_code code = GET_CODE (operands[1]);
4696 80230 : rtx tmp, compare_op;
4697 80230 : rtx op0 = XEXP (operands[1], 0);
4698 80230 : rtx op1 = XEXP (operands[1], 1);
4699 :
4700 80230 : if (GET_MODE (op0) == BFmode
4701 80230 : && !ix86_fp_comparison_operator (operands[1], VOIDmode))
4702 : return false;
4703 :
4704 80230 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
4705 : {
4706 65971 : machine_mode cmode;
4707 :
4708 : /* Since we've no cmove for sse registers, don't force bad register
4709 : allocation just to gain access to it. Deny movcc when the
4710 : comparison mode doesn't match the move mode. */
4711 65971 : cmode = GET_MODE (op0);
4712 65971 : if (cmode == VOIDmode)
4713 0 : cmode = GET_MODE (op1);
4714 65971 : if (cmode != mode)
4715 : return false;
4716 :
4717 9727 : code = ix86_prepare_sse_fp_compare_args (operands[0], code, &op0, &op1);
4718 9727 : if (code == UNKNOWN)
4719 : return false;
4720 :
4721 9707 : if (ix86_expand_sse_fp_minmax (operands[0], code, op0, op1,
4722 : operands[2], operands[3]))
4723 : return true;
4724 :
4725 2118 : tmp = ix86_expand_sse_cmp (operands[0], code, op0, op1,
4726 : operands[2], operands[3]);
4727 2118 : ix86_expand_sse_movcc (operands[0], tmp, operands[2], operands[3]);
4728 2118 : return true;
4729 : }
4730 :
4731 14259 : if (GET_MODE (op0) == TImode
4732 14259 : || (GET_MODE (op0) == DImode
4733 72 : && !TARGET_64BIT))
4734 : return false;
4735 :
4736 : /* The floating point conditional move instructions don't directly
4737 : support conditions resulting from a signed integer comparison. */
4738 :
4739 14187 : compare_op = ix86_expand_compare (code, op0, op1);
4740 14187 : if (!fcmov_comparison_operator (compare_op, VOIDmode))
4741 : {
4742 146 : tmp = gen_reg_rtx (QImode);
4743 146 : ix86_expand_setcc (tmp, code, op0, op1);
4744 :
4745 146 : compare_op = ix86_expand_compare (NE, tmp, const0_rtx);
4746 : }
4747 :
4748 14187 : operands[2] = force_reg (mode, operands[2]);
4749 14187 : operands[3] = force_reg (mode, operands[3]);
4750 14187 : emit_insn (gen_rtx_SET (operands[0],
4751 : gen_rtx_IF_THEN_ELSE (mode, compare_op,
4752 : operands[2], operands[3])));
4753 :
4754 14187 : return true;
4755 : }
4756 :
4757 : /* Helper for ix86_cmp_code_to_pcmp_immediate for int modes. */
4758 :
4759 : static int
4760 4940 : ix86_int_cmp_code_to_pcmp_immediate (enum rtx_code code)
4761 : {
4762 4940 : switch (code)
4763 : {
4764 : case EQ:
4765 : return 0;
4766 379 : case LT:
4767 379 : case LTU:
4768 379 : return 1;
4769 220 : case LE:
4770 220 : case LEU:
4771 220 : return 2;
4772 3114 : case NE:
4773 3114 : return 4;
4774 299 : case GE:
4775 299 : case GEU:
4776 299 : return 5;
4777 502 : case GT:
4778 502 : case GTU:
4779 502 : return 6;
4780 0 : default:
4781 0 : gcc_unreachable ();
4782 : }
4783 : }
4784 :
4785 : /* Helper for ix86_cmp_code_to_pcmp_immediate for fp modes. */
4786 :
4787 : static int
4788 1809 : ix86_fp_cmp_code_to_pcmp_immediate (enum rtx_code code)
4789 : {
4790 1809 : switch (code)
4791 : {
4792 : case EQ:
4793 : return 0x00;
4794 378 : case NE:
4795 378 : return 0x04;
4796 514 : case GT:
4797 514 : return 0x0e;
4798 88 : case LE:
4799 88 : return 0x02;
4800 53 : case GE:
4801 53 : return 0x0d;
4802 624 : case LT:
4803 624 : return 0x01;
4804 2 : case UNLE:
4805 2 : return 0x0a;
4806 2 : case UNLT:
4807 2 : return 0x09;
4808 11 : case UNGE:
4809 11 : return 0x05;
4810 44 : case UNGT:
4811 44 : return 0x06;
4812 2 : case UNEQ:
4813 2 : return 0x18;
4814 0 : case LTGT:
4815 0 : return 0x0c;
4816 2 : case ORDERED:
4817 2 : return 0x07;
4818 2 : case UNORDERED:
4819 2 : return 0x03;
4820 0 : default:
4821 0 : gcc_unreachable ();
4822 : }
4823 : }
4824 :
4825 : /* Return immediate value to be used in UNSPEC_PCMP
4826 : for comparison CODE in MODE. */
4827 :
4828 : static int
4829 6749 : ix86_cmp_code_to_pcmp_immediate (enum rtx_code code, machine_mode mode)
4830 : {
4831 6749 : if (FLOAT_MODE_P (mode))
4832 1809 : return ix86_fp_cmp_code_to_pcmp_immediate (code);
4833 4940 : return ix86_int_cmp_code_to_pcmp_immediate (code);
4834 : }
4835 :
4836 : /* Expand AVX-512 vector comparison. */
4837 :
4838 : bool
4839 6749 : ix86_expand_mask_vec_cmp (rtx dest, enum rtx_code code, rtx cmp_op0, rtx cmp_op1)
4840 : {
4841 6749 : machine_mode mask_mode = GET_MODE (dest);
4842 6749 : machine_mode cmp_mode = GET_MODE (cmp_op0);
4843 6749 : rtx imm = GEN_INT (ix86_cmp_code_to_pcmp_immediate (code, cmp_mode));
4844 6749 : int unspec_code;
4845 6749 : rtx unspec;
4846 :
4847 6749 : switch (code)
4848 : {
4849 : case LEU:
4850 : case GTU:
4851 : case GEU:
4852 : case LTU:
4853 : unspec_code = UNSPEC_UNSIGNED_PCMP;
4854 : break;
4855 :
4856 6335 : default:
4857 6335 : unspec_code = UNSPEC_PCMP;
4858 : }
4859 :
4860 6749 : unspec = gen_rtx_UNSPEC (mask_mode, gen_rtvec (3, cmp_op0, cmp_op1, imm),
4861 : unspec_code);
4862 6749 : emit_insn (gen_rtx_SET (dest, unspec));
4863 :
4864 6749 : return true;
4865 : }
4866 :
4867 : /* Expand fp vector comparison. */
4868 :
4869 : bool
4870 7967 : ix86_expand_fp_vec_cmp (rtx operands[])
4871 : {
4872 7967 : enum rtx_code code = GET_CODE (operands[1]);
4873 7967 : rtx cmp;
4874 :
4875 7967 : code = ix86_prepare_sse_fp_compare_args (operands[0], code,
4876 : &operands[2], &operands[3]);
4877 7967 : if (code == UNKNOWN)
4878 : {
4879 20 : rtx temp;
4880 20 : switch (GET_CODE (operands[1]))
4881 : {
4882 2 : case LTGT:
4883 2 : temp = ix86_expand_sse_cmp (operands[0], ORDERED, operands[2],
4884 : operands[3], NULL, NULL);
4885 2 : cmp = ix86_expand_sse_cmp (operands[0], NE, operands[2],
4886 : operands[3], NULL, NULL);
4887 2 : code = AND;
4888 2 : break;
4889 18 : case UNEQ:
4890 18 : temp = ix86_expand_sse_cmp (operands[0], UNORDERED, operands[2],
4891 : operands[3], NULL, NULL);
4892 18 : cmp = ix86_expand_sse_cmp (operands[0], EQ, operands[2],
4893 : operands[3], NULL, NULL);
4894 18 : code = IOR;
4895 18 : break;
4896 0 : default:
4897 0 : gcc_unreachable ();
4898 : }
4899 20 : cmp = expand_simple_binop (GET_MODE (cmp), code, temp, cmp, cmp, 1,
4900 : OPTAB_DIRECT);
4901 : }
4902 : else
4903 7947 : cmp = ix86_expand_sse_cmp (operands[0], code, operands[2], operands[3],
4904 : NULL, NULL);
4905 :
4906 7967 : if (operands[0] != cmp)
4907 7887 : emit_move_insn (operands[0], cmp);
4908 :
4909 7967 : return true;
4910 : }
4911 :
4912 : static rtx
4913 18463 : ix86_expand_int_sse_cmp (rtx dest, enum rtx_code code, rtx cop0, rtx cop1,
4914 : rtx op_true, rtx op_false, bool *negate)
4915 : {
4916 18463 : machine_mode data_mode = GET_MODE (dest);
4917 18463 : machine_mode mode = GET_MODE (cop0);
4918 18463 : rtx x;
4919 :
4920 18463 : *negate = false;
4921 :
4922 : /* XOP supports all of the comparisons on all 128-bit vector int types. */
4923 18463 : if (TARGET_XOP
4924 201 : && GET_MODE_CLASS (mode) == MODE_VECTOR_INT
4925 18664 : && GET_MODE_SIZE (mode) <= 16)
4926 : ;
4927 : /* AVX512F supports all of the comparisons
4928 : on all 128/256/512-bit vector int types. */
4929 18265 : else if (ix86_use_mask_cmp_p (data_mode, mode, op_true, op_false))
4930 : ;
4931 : else
4932 : {
4933 : /* Canonicalize the comparison to EQ, GT, GTU. */
4934 18212 : switch (code)
4935 : {
4936 : case EQ:
4937 : case GT:
4938 : case GTU:
4939 : break;
4940 :
4941 960 : case LE:
4942 960 : case LEU:
4943 : /* x <= cst can be handled as x < cst + 1 unless there is
4944 : wrap around in cst + 1. */
4945 960 : if (CONST_VECTOR_P (cop1)
4946 1637 : && GET_MODE_INNER (mode) != TImode)
4947 : {
4948 677 : unsigned int n_elts = GET_MODE_NUNITS (mode), i;
4949 677 : machine_mode eltmode = GET_MODE_INNER (mode);
4950 4374 : for (i = 0; i < n_elts; ++i)
4951 : {
4952 3698 : rtx elt = CONST_VECTOR_ELT (cop1, i);
4953 3698 : if (!CONST_INT_P (elt))
4954 : break;
4955 3698 : if (code == LE)
4956 : {
4957 : /* For LE punt if some element is signed maximum. */
4958 2522 : if ((INTVAL (elt) & (GET_MODE_MASK (eltmode) >> 1))
4959 : == (GET_MODE_MASK (eltmode) >> 1))
4960 : break;
4961 : }
4962 : /* For LEU punt if some element is unsigned maximum. */
4963 1176 : else if (elt == constm1_rtx)
4964 : break;
4965 : }
4966 677 : if (i == n_elts)
4967 : {
4968 676 : rtvec v = rtvec_alloc (n_elts);
4969 5048 : for (i = 0; i < n_elts; ++i)
4970 3696 : RTVEC_ELT (v, i)
4971 3696 : = gen_int_mode (INTVAL (CONST_VECTOR_ELT (cop1, i)) + 1,
4972 : eltmode);
4973 676 : cop1 = gen_rtx_CONST_VECTOR (mode, v);
4974 676 : std::swap (cop0, cop1);
4975 676 : code = code == LE ? GT : GTU;
4976 : break;
4977 : }
4978 : }
4979 : /* FALLTHRU */
4980 3335 : case NE:
4981 3335 : code = reverse_condition (code);
4982 3335 : *negate = true;
4983 3335 : break;
4984 :
4985 542 : case GE:
4986 542 : case GEU:
4987 : /* x >= cst can be handled as x > cst - 1 unless there is
4988 : wrap around in cst - 1. */
4989 542 : if (CONST_VECTOR_P (cop1)
4990 839 : && GET_MODE_INNER (mode) != TImode)
4991 : {
4992 297 : unsigned int n_elts = GET_MODE_NUNITS (mode), i;
4993 297 : machine_mode eltmode = GET_MODE_INNER (mode);
4994 2221 : for (i = 0; i < n_elts; ++i)
4995 : {
4996 1972 : rtx elt = CONST_VECTOR_ELT (cop1, i);
4997 1972 : if (!CONST_INT_P (elt))
4998 : break;
4999 1972 : if (code == GE)
5000 : {
5001 : /* For GE punt if some element is signed minimum. */
5002 1924 : if (INTVAL (elt) < 0
5003 136 : && ((INTVAL (elt) & (GET_MODE_MASK (eltmode) >> 1))
5004 : == 0))
5005 : break;
5006 : }
5007 : /* For GEU punt if some element is zero. */
5008 48 : else if (elt == const0_rtx)
5009 : break;
5010 : }
5011 297 : if (i == n_elts)
5012 : {
5013 249 : rtvec v = rtvec_alloc (n_elts);
5014 2422 : for (i = 0; i < n_elts; ++i)
5015 1924 : RTVEC_ELT (v, i)
5016 1924 : = gen_int_mode (INTVAL (CONST_VECTOR_ELT (cop1, i)) - 1,
5017 : eltmode);
5018 249 : cop1 = gen_rtx_CONST_VECTOR (mode, v);
5019 249 : code = code == GE ? GT : GTU;
5020 : break;
5021 : }
5022 : }
5023 293 : code = reverse_condition (code);
5024 293 : *negate = true;
5025 : /* FALLTHRU */
5026 :
5027 1662 : case LT:
5028 1662 : case LTU:
5029 1662 : std::swap (cop0, cop1);
5030 1662 : code = swap_condition (code);
5031 1662 : break;
5032 :
5033 0 : default:
5034 0 : gcc_unreachable ();
5035 : }
5036 :
5037 : /* Only SSE4.1/SSE4.2 supports V2DImode. */
5038 18212 : if (mode == V2DImode)
5039 : {
5040 788 : switch (code)
5041 : {
5042 584 : case EQ:
5043 : /* SSE4.1 supports EQ. */
5044 584 : if (!TARGET_SSE4_1)
5045 18463 : return NULL;
5046 : break;
5047 :
5048 204 : case GT:
5049 204 : case GTU:
5050 : /* SSE4.2 supports GT/GTU. */
5051 204 : if (!TARGET_SSE4_2)
5052 : return NULL;
5053 : break;
5054 :
5055 0 : default:
5056 0 : gcc_unreachable ();
5057 : }
5058 : }
5059 :
5060 18212 : if (CONST_VECTOR_P (cop0))
5061 1328 : cop0 = force_reg (mode, cop0);
5062 16884 : else if (CONST_VECTOR_P (cop1))
5063 7661 : cop1 = force_reg (mode, cop1);
5064 :
5065 18212 : rtx optrue = op_true ? op_true : CONSTM1_RTX (data_mode);
5066 18212 : rtx opfalse = op_false ? op_false : CONST0_RTX (data_mode);
5067 18212 : if (*negate)
5068 3628 : std::swap (optrue, opfalse);
5069 :
5070 : /* Transform x > y ? 0 : -1 (i.e. x <= y ? -1 : 0 or x <= y) when
5071 : not using integer masks into min (x, y) == x ? -1 : 0 (i.e.
5072 : min (x, y) == x). While we add one instruction (the minimum),
5073 : we remove the need for two instructions in the negation, as the
5074 : result is done this way.
5075 : When using masks, do it for SI/DImode element types, as it is shorter
5076 : than the two subtractions. */
5077 18212 : if ((code != EQ
5078 7510 : && GET_MODE_SIZE (mode) != 64
5079 7510 : && vector_all_ones_operand (opfalse, data_mode)
5080 577 : && optrue == CONST0_RTX (data_mode))
5081 25145 : || (code == GTU
5082 2082 : && GET_MODE_SIZE (GET_MODE_INNER (mode)) >= 4
5083 : /* Don't do it if not using integer masks and we'd end up with
5084 : the right values in the registers though. */
5085 696 : && (GET_MODE_SIZE (mode) == 64
5086 696 : || !vector_all_ones_operand (optrue, data_mode)
5087 571 : || opfalse != CONST0_RTX (data_mode))))
5088 : {
5089 702 : rtx (*gen) (rtx, rtx, rtx) = NULL;
5090 :
5091 702 : switch (mode)
5092 : {
5093 0 : case E_V16SImode:
5094 0 : gen = (code == GTU) ? gen_uminv16si3 : gen_sminv16si3;
5095 : break;
5096 0 : case E_V8DImode:
5097 0 : gen = (code == GTU) ? gen_uminv8di3 : gen_sminv8di3;
5098 0 : cop0 = force_reg (mode, cop0);
5099 0 : cop1 = force_reg (mode, cop1);
5100 0 : break;
5101 24 : case E_V32QImode:
5102 24 : if (TARGET_AVX2)
5103 24 : gen = (code == GTU) ? gen_uminv32qi3 : gen_sminv32qi3;
5104 : break;
5105 24 : case E_V16HImode:
5106 24 : if (TARGET_AVX2)
5107 24 : gen = (code == GTU) ? gen_uminv16hi3 : gen_sminv16hi3;
5108 : break;
5109 25 : case E_V8SImode:
5110 25 : if (TARGET_AVX2)
5111 25 : gen = (code == GTU) ? gen_uminv8si3 : gen_sminv8si3;
5112 : break;
5113 20 : case E_V4DImode:
5114 20 : if (TARGET_AVX512VL)
5115 : {
5116 0 : gen = (code == GTU) ? gen_uminv4di3 : gen_sminv4di3;
5117 0 : cop0 = force_reg (mode, cop0);
5118 0 : cop1 = force_reg (mode, cop1);
5119 : }
5120 : break;
5121 64 : case E_V16QImode:
5122 64 : if (code == GTU && TARGET_SSE2)
5123 : gen = gen_uminv16qi3;
5124 28 : else if (code == GT && TARGET_SSE4_1)
5125 : gen = gen_sminv16qi3;
5126 : break;
5127 44 : case E_V8QImode:
5128 44 : if (code == GTU && TARGET_SSE2)
5129 : gen = gen_uminv8qi3;
5130 42 : else if (code == GT && TARGET_SSE4_1)
5131 : gen = gen_sminv8qi3;
5132 : break;
5133 13 : case E_V4QImode:
5134 13 : if (code == GTU && TARGET_SSE2)
5135 : gen = gen_uminv4qi3;
5136 2 : else if (code == GT && TARGET_SSE4_1)
5137 : gen = gen_sminv4qi3;
5138 : break;
5139 8 : case E_V2QImode:
5140 8 : if (code == GTU && TARGET_SSE2)
5141 : gen = gen_uminv2qi3;
5142 6 : else if (code == GT && TARGET_SSE4_1)
5143 : gen = gen_sminv2qi3;
5144 : break;
5145 73 : case E_V8HImode:
5146 73 : if (code == GTU && TARGET_SSE4_1)
5147 : gen = gen_uminv8hi3;
5148 63 : else if (code == GT && TARGET_SSE2)
5149 : gen = gen_sminv8hi3;
5150 : break;
5151 4 : case E_V4HImode:
5152 4 : if (code == GTU && TARGET_SSE4_1)
5153 : gen = gen_uminv4hi3;
5154 4 : else if (code == GT && TARGET_SSE2)
5155 : gen = gen_sminv4hi3;
5156 : break;
5157 16 : case E_V2HImode:
5158 16 : if (code == GTU && TARGET_SSE4_1)
5159 : gen = gen_uminv2hi3;
5160 16 : else if (code == GT && TARGET_SSE2)
5161 : gen = gen_sminv2hi3;
5162 : break;
5163 250 : case E_V4SImode:
5164 250 : if (TARGET_SSE4_1)
5165 52 : gen = (code == GTU) ? gen_uminv4si3 : gen_sminv4si3;
5166 : break;
5167 113 : case E_V2SImode:
5168 113 : if (TARGET_SSE4_1)
5169 0 : gen = (code == GTU) ? gen_uminv2si3 : gen_sminv2si3;
5170 : break;
5171 24 : case E_V2DImode:
5172 24 : if (TARGET_AVX512VL)
5173 : {
5174 0 : gen = (code == GTU) ? gen_uminv2di3 : gen_sminv2di3;
5175 0 : cop0 = force_reg (mode, cop0);
5176 0 : cop1 = force_reg (mode, cop1);
5177 : }
5178 : break;
5179 : default:
5180 : break;
5181 : }
5182 :
5183 0 : if (gen)
5184 : {
5185 280 : rtx tem = gen_reg_rtx (mode);
5186 280 : if (!vector_operand (cop0, mode))
5187 0 : cop0 = force_reg (mode, cop0);
5188 280 : if (!vector_operand (cop1, mode))
5189 0 : cop1 = force_reg (mode, cop1);
5190 280 : *negate = !*negate;
5191 280 : emit_insn (gen (tem, cop0, cop1));
5192 280 : cop1 = tem;
5193 280 : code = EQ;
5194 : }
5195 : }
5196 :
5197 : /* Unsigned parallel compare is not supported by the hardware.
5198 : Play some tricks to turn this into a signed comparison
5199 : against 0. */
5200 18212 : if (code == GTU)
5201 : {
5202 1178 : cop0 = force_reg (mode, cop0);
5203 :
5204 1178 : switch (mode)
5205 : {
5206 802 : case E_V16SImode:
5207 802 : case E_V8DImode:
5208 802 : case E_V8SImode:
5209 802 : case E_V4DImode:
5210 802 : case E_V4SImode:
5211 802 : case E_V2SImode:
5212 802 : case E_V2DImode:
5213 802 : {
5214 802 : rtx t1, t2, mask;
5215 :
5216 : /* Subtract (-(INT MAX) - 1) from both operands to make
5217 : them signed. */
5218 802 : mask = ix86_build_signbit_mask (mode, true, false);
5219 802 : t1 = gen_reg_rtx (mode);
5220 802 : emit_insn (gen_sub3_insn (t1, cop0, mask));
5221 :
5222 802 : t2 = gen_reg_rtx (mode);
5223 802 : emit_insn (gen_sub3_insn (t2, cop1, mask));
5224 :
5225 802 : cop0 = t1;
5226 802 : cop1 = t2;
5227 802 : code = GT;
5228 : }
5229 802 : break;
5230 :
5231 376 : case E_V64QImode:
5232 376 : case E_V32HImode:
5233 376 : case E_V32QImode:
5234 376 : case E_V16HImode:
5235 376 : case E_V16QImode:
5236 376 : case E_V8QImode:
5237 376 : case E_V4QImode:
5238 376 : case E_V2QImode:
5239 376 : case E_V8HImode:
5240 376 : case E_V4HImode:
5241 376 : case E_V2HImode:
5242 : /* Perform a parallel unsigned saturating subtraction. */
5243 376 : x = gen_reg_rtx (mode);
5244 376 : emit_insn (gen_rtx_SET
5245 : (x, gen_rtx_US_MINUS (mode, cop0, cop1)));
5246 376 : cop0 = x;
5247 376 : cop1 = CONST0_RTX (mode);
5248 376 : code = EQ;
5249 376 : *negate = !*negate;
5250 376 : break;
5251 :
5252 0 : default:
5253 0 : gcc_unreachable ();
5254 : }
5255 : }
5256 : }
5257 :
5258 18463 : if (*negate)
5259 3662 : std::swap (op_true, op_false);
5260 :
5261 18463 : if (CONST_VECTOR_P (cop1))
5262 445 : cop1 = force_reg (mode, cop1);
5263 :
5264 : /* Allow the comparison to be done in one mode, but the movcc to
5265 : happen in another mode. */
5266 18463 : if (data_mode == mode)
5267 18421 : x = ix86_expand_sse_cmp (dest, code, cop0, cop1, op_true, op_false);
5268 : else
5269 : {
5270 126 : gcc_assert (GET_MODE_SIZE (data_mode) == GET_MODE_SIZE (mode));
5271 42 : x = ix86_expand_sse_cmp (gen_reg_rtx (mode), code, cop0, cop1,
5272 : op_true, op_false);
5273 42 : if (GET_MODE (x) == mode)
5274 24 : x = gen_lowpart (data_mode, x);
5275 : }
5276 :
5277 : return x;
5278 : }
5279 :
5280 : /* Expand integer vector comparison. */
5281 :
5282 : bool
5283 11610 : ix86_expand_int_vec_cmp (rtx operands[])
5284 : {
5285 11610 : rtx_code code = GET_CODE (operands[1]);
5286 11610 : bool negate = false;
5287 11610 : rtx cmp = ix86_expand_int_sse_cmp (operands[0], code, operands[2],
5288 : operands[3], NULL, NULL, &negate);
5289 :
5290 11610 : if (!cmp)
5291 : return false;
5292 :
5293 11610 : if (negate)
5294 : {
5295 3674 : if (TARGET_AVX512F && GET_MODE_SIZE (GET_MODE (cmp)) >= 16)
5296 93 : cmp = gen_rtx_XOR (GET_MODE (cmp), cmp, CONSTM1_RTX (GET_MODE (cmp)));
5297 : else
5298 : {
5299 6906 : cmp = ix86_expand_int_sse_cmp (operands[0], EQ, cmp,
5300 3453 : CONST0_RTX (GET_MODE (cmp)),
5301 : NULL, NULL, &negate);
5302 3453 : gcc_assert (!negate);
5303 : }
5304 : }
5305 :
5306 11610 : if (operands[0] != cmp)
5307 11315 : emit_move_insn (operands[0], cmp);
5308 :
5309 : return true;
5310 : }
5311 :
5312 : /* Expand a floating-point vector conditional move; a vcond operation
5313 : rather than a movcc operation. */
5314 :
5315 : bool
5316 0 : ix86_expand_fp_vcond (rtx operands[])
5317 : {
5318 0 : enum rtx_code code = GET_CODE (operands[3]);
5319 0 : rtx cmp;
5320 :
5321 0 : code = ix86_prepare_sse_fp_compare_args (operands[0], code,
5322 : &operands[4], &operands[5]);
5323 0 : if (code == UNKNOWN)
5324 : {
5325 0 : rtx temp;
5326 0 : switch (GET_CODE (operands[3]))
5327 : {
5328 0 : case LTGT:
5329 0 : temp = ix86_expand_sse_cmp (operands[0], ORDERED, operands[4],
5330 : operands[5], operands[0], operands[0]);
5331 0 : cmp = ix86_expand_sse_cmp (operands[0], NE, operands[4],
5332 : operands[5], operands[1], operands[2]);
5333 0 : code = AND;
5334 0 : break;
5335 0 : case UNEQ:
5336 0 : temp = ix86_expand_sse_cmp (operands[0], UNORDERED, operands[4],
5337 : operands[5], operands[0], operands[0]);
5338 0 : cmp = ix86_expand_sse_cmp (operands[0], EQ, operands[4],
5339 : operands[5], operands[1], operands[2]);
5340 0 : code = IOR;
5341 0 : break;
5342 0 : default:
5343 0 : gcc_unreachable ();
5344 : }
5345 0 : cmp = expand_simple_binop (GET_MODE (cmp), code, temp, cmp, cmp, 1,
5346 : OPTAB_DIRECT);
5347 0 : ix86_expand_sse_movcc (operands[0], cmp, operands[1], operands[2]);
5348 0 : return true;
5349 : }
5350 :
5351 0 : if (ix86_expand_sse_fp_minmax (operands[0], code, operands[4],
5352 : operands[5], operands[1], operands[2]))
5353 : return true;
5354 :
5355 0 : cmp = ix86_expand_sse_cmp (operands[0], code, operands[4], operands[5],
5356 : operands[1], operands[2]);
5357 0 : ix86_expand_sse_movcc (operands[0], cmp, operands[1], operands[2]);
5358 0 : return true;
5359 : }
5360 :
5361 : /* Expand a signed/unsigned integral vector conditional move. */
5362 :
5363 : bool
5364 3400 : ix86_expand_int_vcond (rtx operands[])
5365 : {
5366 3400 : machine_mode data_mode = GET_MODE (operands[0]);
5367 3400 : machine_mode mode = GET_MODE (operands[4]);
5368 3400 : enum rtx_code code = GET_CODE (operands[3]);
5369 3400 : bool negate = false;
5370 3400 : rtx x, cop0, cop1;
5371 :
5372 3400 : cop0 = operands[4];
5373 3400 : cop1 = operands[5];
5374 :
5375 : /* Try to optimize x < 0 ? -1 : 0 into (signed) x >> 31
5376 : and x < 0 ? 1 : 0 into (unsigned) x >> 31. */
5377 3400 : if ((code == LT || code == GE)
5378 0 : && data_mode == mode
5379 0 : && cop1 == CONST0_RTX (mode)
5380 0 : && operands[1 + (code == LT)] == CONST0_RTX (data_mode)
5381 0 : && GET_MODE_UNIT_SIZE (data_mode) > 1
5382 0 : && GET_MODE_UNIT_SIZE (data_mode) <= 8
5383 3400 : && (GET_MODE_SIZE (data_mode) == 16
5384 0 : || (TARGET_AVX2 && GET_MODE_SIZE (data_mode) == 32)))
5385 : {
5386 0 : rtx negop = operands[2 - (code == LT)];
5387 0 : int shift = GET_MODE_UNIT_BITSIZE (data_mode) - 1;
5388 0 : if (negop == CONST1_RTX (data_mode))
5389 : {
5390 0 : rtx res = expand_simple_binop (mode, LSHIFTRT, cop0, GEN_INT (shift),
5391 : operands[0], 1, OPTAB_DIRECT);
5392 0 : if (res != operands[0])
5393 0 : emit_move_insn (operands[0], res);
5394 0 : return true;
5395 : }
5396 0 : else if (GET_MODE_INNER (data_mode) != DImode
5397 0 : && vector_all_ones_operand (negop, data_mode))
5398 : {
5399 0 : rtx res = expand_simple_binop (mode, ASHIFTRT, cop0, GEN_INT (shift),
5400 : operands[0], 0, OPTAB_DIRECT);
5401 0 : if (res != operands[0])
5402 0 : emit_move_insn (operands[0], res);
5403 0 : return true;
5404 : }
5405 : }
5406 :
5407 3400 : if (!nonimmediate_operand (cop1, mode))
5408 126 : cop1 = force_reg (mode, cop1);
5409 3400 : if (!general_operand (operands[1], data_mode))
5410 0 : operands[1] = force_reg (data_mode, operands[1]);
5411 3400 : if (!general_operand (operands[2], data_mode))
5412 0 : operands[2] = force_reg (data_mode, operands[2]);
5413 :
5414 3400 : x = ix86_expand_int_sse_cmp (operands[0], code, cop0, cop1,
5415 : operands[1], operands[2], &negate);
5416 :
5417 3400 : if (!x)
5418 : return false;
5419 :
5420 3400 : ix86_expand_sse_movcc (operands[0], x, operands[1+negate],
5421 3400 : operands[2-negate]);
5422 3400 : return true;
5423 : }
5424 :
5425 : static bool
5426 124698 : ix86_expand_vec_perm_vpermt2 (rtx target, rtx mask, rtx op0, rtx op1,
5427 : struct expand_vec_perm_d *d)
5428 : {
5429 : /* ix86_expand_vec_perm_vpermt2 is called from both const and non-const
5430 : expander, so args are either in d, or in op0, op1 etc. */
5431 124698 : machine_mode mode = GET_MODE (d ? d->op0 : op0);
5432 124698 : machine_mode maskmode = mode;
5433 124698 : rtx (*gen) (rtx, rtx, rtx, rtx) = NULL;
5434 :
5435 124698 : switch (mode)
5436 : {
5437 24001 : case E_V16QImode:
5438 24001 : if (TARGET_AVX512VL && TARGET_AVX512VBMI)
5439 : gen = gen_avx512vl_vpermt2varv16qi3;
5440 : break;
5441 521 : case E_V32QImode:
5442 521 : if (TARGET_AVX512VL && TARGET_AVX512VBMI)
5443 : gen = gen_avx512vl_vpermt2varv32qi3;
5444 : break;
5445 198 : case E_V64QImode:
5446 198 : if (TARGET_AVX512VBMI)
5447 : gen = gen_avx512bw_vpermt2varv64qi3;
5448 : break;
5449 13339 : case E_V8HImode:
5450 13339 : if (TARGET_AVX512VL && TARGET_AVX512BW)
5451 : gen = gen_avx512vl_vpermt2varv8hi3;
5452 : break;
5453 775 : case E_V16HImode:
5454 775 : if (TARGET_AVX512VL && TARGET_AVX512BW)
5455 : gen = gen_avx512vl_vpermt2varv16hi3;
5456 : break;
5457 331 : case E_V32HImode:
5458 331 : if (TARGET_AVX512BW)
5459 : gen = gen_avx512bw_vpermt2varv32hi3;
5460 : break;
5461 33811 : case E_V4SImode:
5462 33811 : if (TARGET_AVX512VL)
5463 : gen = gen_avx512vl_vpermt2varv4si3;
5464 : break;
5465 1138 : case E_V8SImode:
5466 1138 : if (TARGET_AVX512VL)
5467 : gen = gen_avx512vl_vpermt2varv8si3;
5468 : break;
5469 126 : case E_V16SImode:
5470 126 : if (TARGET_AVX512F)
5471 : gen = gen_avx512f_vpermt2varv16si3;
5472 : break;
5473 10387 : case E_V4SFmode:
5474 10387 : if (TARGET_AVX512VL)
5475 : {
5476 : gen = gen_avx512vl_vpermt2varv4sf3;
5477 : maskmode = V4SImode;
5478 : }
5479 : break;
5480 6079 : case E_V8SFmode:
5481 6079 : if (TARGET_AVX512VL)
5482 : {
5483 : gen = gen_avx512vl_vpermt2varv8sf3;
5484 : maskmode = V8SImode;
5485 : }
5486 : break;
5487 239 : case E_V16SFmode:
5488 239 : if (TARGET_AVX512F)
5489 : {
5490 : gen = gen_avx512f_vpermt2varv16sf3;
5491 : maskmode = V16SImode;
5492 : }
5493 : break;
5494 2 : case E_V2DImode:
5495 2 : if (TARGET_AVX512VL)
5496 : gen = gen_avx512vl_vpermt2varv2di3;
5497 : break;
5498 307 : case E_V4DImode:
5499 307 : if (TARGET_AVX512VL)
5500 : gen = gen_avx512vl_vpermt2varv4di3;
5501 : break;
5502 10 : case E_V8DImode:
5503 10 : if (TARGET_AVX512F)
5504 : gen = gen_avx512f_vpermt2varv8di3;
5505 : break;
5506 2 : case E_V2DFmode:
5507 2 : if (TARGET_AVX512VL)
5508 : {
5509 : gen = gen_avx512vl_vpermt2varv2df3;
5510 : maskmode = V2DImode;
5511 : }
5512 : break;
5513 1956 : case E_V4DFmode:
5514 1956 : if (TARGET_AVX512VL)
5515 : {
5516 : gen = gen_avx512vl_vpermt2varv4df3;
5517 : maskmode = V4DImode;
5518 : }
5519 : break;
5520 194 : case E_V8DFmode:
5521 194 : if (TARGET_AVX512F)
5522 : {
5523 : gen = gen_avx512f_vpermt2varv8df3;
5524 : maskmode = V8DImode;
5525 : }
5526 : break;
5527 : default:
5528 : break;
5529 : }
5530 :
5531 : if (gen == NULL)
5532 : return false;
5533 :
5534 918 : if (d && d->testing_p)
5535 : return true;
5536 :
5537 : /* ix86_expand_vec_perm_vpermt2 is called from both const and non-const
5538 : expander, so args are either in d, or in op0, op1 etc. */
5539 907 : if (d)
5540 : {
5541 907 : rtx vec[64];
5542 907 : target = d->target;
5543 907 : op0 = d->op0;
5544 907 : op1 = d->op1;
5545 15703 : for (int i = 0; i < d->nelt; ++i)
5546 14796 : vec[i] = GEN_INT (d->perm[i]);
5547 907 : mask = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (d->nelt, vec));
5548 : }
5549 :
5550 915 : emit_insn (gen (target, force_reg (maskmode, mask), op0, op1));
5551 915 : return true;
5552 : }
5553 :
5554 : /* Expand a variable vector permutation. */
5555 :
5556 : void
5557 18 : ix86_expand_vec_perm (rtx operands[])
5558 : {
5559 18 : rtx target = operands[0];
5560 18 : rtx op0 = operands[1];
5561 18 : rtx op1 = operands[2];
5562 18 : rtx mask = operands[3];
5563 18 : rtx t1, t2, t3, t4, t5, t6, t7, t8, vt, vt2, vec[32];
5564 18 : machine_mode mode = GET_MODE (op0);
5565 18 : machine_mode maskmode = GET_MODE (mask);
5566 18 : int w, e, i;
5567 18 : bool one_operand_shuffle = rtx_equal_p (op0, op1);
5568 :
5569 : /* Number of elements in the vector. */
5570 18 : w = GET_MODE_NUNITS (mode);
5571 18 : e = GET_MODE_UNIT_SIZE (mode);
5572 18 : gcc_assert (w <= 64);
5573 :
5574 : /* For HF mode vector, convert it to HI using subreg. */
5575 36 : if (GET_MODE_INNER (mode) == HFmode)
5576 : {
5577 6 : machine_mode orig_mode = mode;
5578 6 : mode = mode_for_vector (HImode, w).require ();
5579 6 : target = lowpart_subreg (mode, target, orig_mode);
5580 6 : op0 = lowpart_subreg (mode, op0, orig_mode);
5581 6 : op1 = lowpart_subreg (mode, op1, orig_mode);
5582 : }
5583 :
5584 18 : if (TARGET_AVX512F && one_operand_shuffle)
5585 : {
5586 5 : rtx (*gen) (rtx, rtx, rtx) = NULL;
5587 5 : switch (mode)
5588 : {
5589 : case E_V16SImode:
5590 : gen = gen_avx512f_permvarv16si;
5591 : break;
5592 0 : case E_V16SFmode:
5593 0 : gen = gen_avx512f_permvarv16sf;
5594 0 : break;
5595 0 : case E_V8DImode:
5596 0 : gen = gen_avx512f_permvarv8di;
5597 0 : break;
5598 0 : case E_V8DFmode:
5599 0 : gen = gen_avx512f_permvarv8df;
5600 0 : break;
5601 : default:
5602 : break;
5603 : }
5604 0 : if (gen != NULL)
5605 : {
5606 0 : emit_insn (gen (target, op0, mask));
5607 16 : return;
5608 : }
5609 : }
5610 :
5611 18 : if (ix86_expand_vec_perm_vpermt2 (target, mask, op0, op1, NULL))
5612 : return;
5613 :
5614 10 : if (TARGET_AVX2)
5615 : {
5616 5 : if (mode == V4DImode || mode == V4DFmode || mode == V16HImode)
5617 : {
5618 : /* Unfortunately, the VPERMQ and VPERMPD instructions only support
5619 : an constant shuffle operand. With a tiny bit of effort we can
5620 : use VPERMD instead. A re-interpretation stall for V4DFmode is
5621 : unfortunate but there's no avoiding it.
5622 : Similarly for V16HImode we don't have instructions for variable
5623 : shuffling, while for V32QImode we can use after preparing suitable
5624 : masks vpshufb; vpshufb; vpermq; vpor. */
5625 :
5626 : if (mode == V16HImode)
5627 : {
5628 : maskmode = mode = V32QImode;
5629 : w = 32;
5630 : e = 1;
5631 : }
5632 : else
5633 : {
5634 : maskmode = mode = V8SImode;
5635 : w = 8;
5636 : e = 4;
5637 : }
5638 0 : t1 = gen_reg_rtx (maskmode);
5639 :
5640 : /* Replicate the low bits of the V4DImode mask into V8SImode:
5641 : mask = { A B C D }
5642 : t1 = { A A B B C C D D }. */
5643 0 : for (i = 0; i < w / 2; ++i)
5644 0 : vec[i*2 + 1] = vec[i*2] = GEN_INT (i * 2);
5645 0 : vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
5646 0 : vt = force_reg (maskmode, vt);
5647 0 : mask = gen_lowpart (maskmode, mask);
5648 0 : if (maskmode == V8SImode)
5649 0 : emit_insn (gen_avx2_permvarv8si (t1, mask, vt));
5650 : else
5651 0 : emit_insn (gen_avx2_pshufbv32qi3 (t1, mask, vt));
5652 :
5653 : /* Multiply the shuffle indices by two. */
5654 0 : t1 = expand_simple_binop (maskmode, PLUS, t1, t1, t1, 1,
5655 : OPTAB_DIRECT);
5656 :
5657 : /* Add one to the odd shuffle indices:
5658 : t1 = { A*2, A*2+1, B*2, B*2+1, ... }. */
5659 0 : for (i = 0; i < w / 2; ++i)
5660 : {
5661 0 : vec[i * 2] = const0_rtx;
5662 0 : vec[i * 2 + 1] = const1_rtx;
5663 : }
5664 0 : vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
5665 0 : vt = validize_mem (force_const_mem (maskmode, vt));
5666 0 : t1 = expand_simple_binop (maskmode, PLUS, t1, vt, t1, 1,
5667 : OPTAB_DIRECT);
5668 :
5669 : /* Continue as if V8SImode (resp. V32QImode) was used initially. */
5670 0 : operands[3] = mask = t1;
5671 0 : target = gen_reg_rtx (mode);
5672 0 : op0 = gen_lowpart (mode, op0);
5673 0 : op1 = gen_lowpart (mode, op1);
5674 : }
5675 :
5676 5 : switch (mode)
5677 : {
5678 1 : case E_V8SImode:
5679 : /* The VPERMD and VPERMPS instructions already properly ignore
5680 : the high bits of the shuffle elements. No need for us to
5681 : perform an AND ourselves. */
5682 1 : if (one_operand_shuffle)
5683 : {
5684 0 : emit_insn (gen_avx2_permvarv8si (target, op0, mask));
5685 0 : if (target != operands[0])
5686 0 : emit_move_insn (operands[0],
5687 0 : gen_lowpart (GET_MODE (operands[0]), target));
5688 : }
5689 : else
5690 : {
5691 1 : t1 = gen_reg_rtx (V8SImode);
5692 1 : t2 = gen_reg_rtx (V8SImode);
5693 1 : emit_insn (gen_avx2_permvarv8si (t1, op0, mask));
5694 1 : emit_insn (gen_avx2_permvarv8si (t2, op1, mask));
5695 1 : goto merge_two;
5696 : }
5697 0 : return;
5698 :
5699 0 : case E_V8SFmode:
5700 0 : mask = gen_lowpart (V8SImode, mask);
5701 0 : if (one_operand_shuffle)
5702 0 : emit_insn (gen_avx2_permvarv8sf (target, op0, mask));
5703 : else
5704 : {
5705 0 : t1 = gen_reg_rtx (V8SFmode);
5706 0 : t2 = gen_reg_rtx (V8SFmode);
5707 0 : emit_insn (gen_avx2_permvarv8sf (t1, op0, mask));
5708 0 : emit_insn (gen_avx2_permvarv8sf (t2, op1, mask));
5709 0 : goto merge_two;
5710 : }
5711 0 : return;
5712 :
5713 1 : case E_V4SImode:
5714 1 : if (one_operand_shuffle)
5715 : break; /* Handled below for TARGET_AVX. */
5716 : /* By combining the two 128-bit input vectors into one 256-bit
5717 : input vector, we can use VPERMD and VPERMPS for the full
5718 : two-operand shuffle. */
5719 0 : t1 = gen_reg_rtx (V8SImode);
5720 0 : t2 = gen_reg_rtx (V8SImode);
5721 0 : emit_insn (gen_avx_vec_concatv8si (t1, op0, op1));
5722 0 : emit_insn (gen_avx_vec_concatv8si (t2, mask, mask));
5723 0 : emit_insn (gen_avx2_permvarv8si (t1, t1, t2));
5724 0 : emit_insn (gen_avx_vextractf128v8si (target, t1, const0_rtx));
5725 0 : return;
5726 :
5727 1 : case E_V4SFmode:
5728 1 : if (one_operand_shuffle)
5729 : break; /* Handled below for TARGET_AVX. */
5730 0 : t1 = gen_reg_rtx (V8SFmode);
5731 0 : t2 = gen_reg_rtx (V8SImode);
5732 0 : mask = gen_lowpart (V4SImode, mask);
5733 0 : emit_insn (gen_avx_vec_concatv8sf (t1, op0, op1));
5734 0 : emit_insn (gen_avx_vec_concatv8si (t2, mask, mask));
5735 0 : emit_insn (gen_avx2_permvarv8sf (t1, t1, t2));
5736 0 : emit_insn (gen_avx_vextractf128v8sf (target, t1, const0_rtx));
5737 0 : return;
5738 :
5739 0 : case E_V32QImode:
5740 0 : t1 = gen_reg_rtx (V32QImode);
5741 0 : t2 = gen_reg_rtx (V32QImode);
5742 0 : t3 = gen_reg_rtx (V32QImode);
5743 0 : vt2 = GEN_INT (-128);
5744 0 : vt = gen_const_vec_duplicate (V32QImode, vt2);
5745 0 : vt = force_reg (V32QImode, vt);
5746 0 : for (i = 0; i < 32; i++)
5747 0 : vec[i] = i < 16 ? vt2 : const0_rtx;
5748 0 : vt2 = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, vec));
5749 0 : vt2 = force_reg (V32QImode, vt2);
5750 : /* From mask create two adjusted masks, which contain the same
5751 : bits as mask in the low 7 bits of each vector element.
5752 : The first mask will have the most significant bit clear
5753 : if it requests element from the same 128-bit lane
5754 : and MSB set if it requests element from the other 128-bit lane.
5755 : The second mask will have the opposite values of the MSB,
5756 : and additionally will have its 128-bit lanes swapped.
5757 : E.g. { 07 12 1e 09 ... | 17 19 05 1f ... } mask vector will have
5758 : t1 { 07 92 9e 09 ... | 17 19 85 1f ... } and
5759 : t3 { 97 99 05 9f ... | 87 12 1e 89 ... } where each ...
5760 : stands for other 12 bytes. */
5761 : /* The bit whether element is from the same lane or the other
5762 : lane is bit 4, so shift it up by 3 to the MSB position. */
5763 0 : t5 = gen_reg_rtx (V4DImode);
5764 0 : emit_insn (gen_ashlv4di3 (t5, gen_lowpart (V4DImode, mask),
5765 : GEN_INT (3)));
5766 : /* Clear MSB bits from the mask just in case it had them set. */
5767 0 : emit_insn (gen_avx2_andnotv32qi3 (t2, vt, mask));
5768 : /* After this t1 will have MSB set for elements from other lane. */
5769 0 : emit_insn (gen_xorv32qi3 (t1, gen_lowpart (V32QImode, t5), vt2));
5770 : /* Clear bits other than MSB. */
5771 0 : emit_insn (gen_andv32qi3 (t1, t1, vt));
5772 : /* Or in the lower bits from mask into t3. */
5773 0 : emit_insn (gen_iorv32qi3 (t3, t1, t2));
5774 : /* And invert MSB bits in t1, so MSB is set for elements from the same
5775 : lane. */
5776 0 : emit_insn (gen_xorv32qi3 (t1, t1, vt));
5777 : /* Swap 128-bit lanes in t3. */
5778 0 : t6 = gen_reg_rtx (V4DImode);
5779 0 : emit_insn (gen_avx2_permv4di_1 (t6, gen_lowpart (V4DImode, t3),
5780 : const2_rtx, GEN_INT (3),
5781 : const0_rtx, const1_rtx));
5782 : /* And or in the lower bits from mask into t1. */
5783 0 : emit_insn (gen_iorv32qi3 (t1, t1, t2));
5784 0 : if (one_operand_shuffle)
5785 : {
5786 : /* Each of these shuffles will put 0s in places where
5787 : element from the other 128-bit lane is needed, otherwise
5788 : will shuffle in the requested value. */
5789 0 : emit_insn (gen_avx2_pshufbv32qi3 (t3, op0,
5790 0 : gen_lowpart (V32QImode, t6)));
5791 0 : emit_insn (gen_avx2_pshufbv32qi3 (t1, op0, t1));
5792 : /* For t3 the 128-bit lanes are swapped again. */
5793 0 : t7 = gen_reg_rtx (V4DImode);
5794 0 : emit_insn (gen_avx2_permv4di_1 (t7, gen_lowpart (V4DImode, t3),
5795 : const2_rtx, GEN_INT (3),
5796 : const0_rtx, const1_rtx));
5797 : /* And oring both together leads to the result. */
5798 0 : emit_insn (gen_iorv32qi3 (target, t1,
5799 0 : gen_lowpart (V32QImode, t7)));
5800 0 : if (target != operands[0])
5801 0 : emit_move_insn (operands[0],
5802 0 : gen_lowpart (GET_MODE (operands[0]), target));
5803 0 : return;
5804 : }
5805 :
5806 0 : t4 = gen_reg_rtx (V32QImode);
5807 : /* Similarly to the above one_operand_shuffle code,
5808 : just for repeated twice for each operand. merge_two:
5809 : code will merge the two results together. */
5810 0 : emit_insn (gen_avx2_pshufbv32qi3 (t4, op0,
5811 0 : gen_lowpart (V32QImode, t6)));
5812 0 : emit_insn (gen_avx2_pshufbv32qi3 (t3, op1,
5813 0 : gen_lowpart (V32QImode, t6)));
5814 0 : emit_insn (gen_avx2_pshufbv32qi3 (t2, op0, t1));
5815 0 : emit_insn (gen_avx2_pshufbv32qi3 (t1, op1, t1));
5816 0 : t7 = gen_reg_rtx (V4DImode);
5817 0 : emit_insn (gen_avx2_permv4di_1 (t7, gen_lowpart (V4DImode, t4),
5818 : const2_rtx, GEN_INT (3),
5819 : const0_rtx, const1_rtx));
5820 0 : t8 = gen_reg_rtx (V4DImode);
5821 0 : emit_insn (gen_avx2_permv4di_1 (t8, gen_lowpart (V4DImode, t3),
5822 : const2_rtx, GEN_INT (3),
5823 : const0_rtx, const1_rtx));
5824 0 : emit_insn (gen_iorv32qi3 (t4, t2, gen_lowpart (V32QImode, t7)));
5825 0 : emit_insn (gen_iorv32qi3 (t3, t1, gen_lowpart (V32QImode, t8)));
5826 0 : t1 = t4;
5827 0 : t2 = t3;
5828 0 : goto merge_two;
5829 :
5830 2 : default:
5831 4 : gcc_assert (GET_MODE_SIZE (mode) <= 16);
5832 : break;
5833 : }
5834 : }
5835 :
5836 9 : if (TARGET_AVX && one_operand_shuffle)
5837 8 : switch (mode)
5838 : {
5839 2 : case V4SImode:
5840 2 : op0 = gen_lowpart (V4SFmode, op0);
5841 2 : t1 = gen_reg_rtx (V4SFmode);
5842 2 : emit_insn (gen_avx_vpermilvarv4sf3 (t1, op0, mask));
5843 2 : emit_move_insn (target, gen_lowpart (mode, t1));
5844 2 : return;
5845 2 : case V4SFmode:
5846 2 : emit_insn (gen_avx_vpermilvarv4sf3 (target, op0, mask));
5847 2 : return;
5848 2 : case V2DImode:
5849 2 : op0 = gen_lowpart (V2DFmode, op0);
5850 2 : t1 = gen_reg_rtx (V2DImode);
5851 2 : t2 = gen_reg_rtx (V2DFmode);
5852 2 : emit_insn (gen_addv2di3 (t1, mask, mask));
5853 2 : emit_insn (gen_avx_vpermilvarv2df3 (t2, op0, t1));
5854 2 : emit_move_insn (target, gen_lowpart (mode, t2));
5855 2 : return;
5856 2 : case V2DFmode:
5857 2 : t1 = gen_reg_rtx (V2DImode);
5858 2 : emit_insn (gen_addv2di3 (t1, mask, mask));
5859 2 : emit_insn (gen_avx_vpermilvarv2df3 (target, op0, t1));
5860 2 : return;
5861 : default:
5862 : break;
5863 : }
5864 :
5865 1 : if (TARGET_XOP)
5866 : {
5867 : /* The XOP VPPERM insn supports three inputs. By ignoring the
5868 : one_operand_shuffle special case, we avoid creating another
5869 : set of constant vectors in memory. */
5870 0 : one_operand_shuffle = false;
5871 :
5872 : /* mask = mask & {2*w-1, ...} */
5873 0 : vt = GEN_INT (2*w - 1);
5874 : }
5875 : else
5876 : {
5877 : /* mask = mask & {w-1, ...} */
5878 1 : vt = GEN_INT (w - 1);
5879 : }
5880 :
5881 1 : vt = gen_const_vec_duplicate (maskmode, vt);
5882 1 : mask = expand_simple_binop (maskmode, AND, mask, vt,
5883 : NULL_RTX, 0, OPTAB_DIRECT);
5884 :
5885 : /* For non-QImode operations, convert the word permutation control
5886 : into a byte permutation control. */
5887 1 : if (mode != V16QImode)
5888 : {
5889 1 : mask = expand_simple_binop (maskmode, ASHIFT, mask,
5890 2 : GEN_INT (exact_log2 (e)),
5891 : NULL_RTX, 0, OPTAB_DIRECT);
5892 :
5893 : /* Convert mask to vector of chars. */
5894 1 : mask = force_reg (V16QImode, gen_lowpart (V16QImode, mask));
5895 :
5896 : /* Replicate each of the input bytes into byte positions:
5897 : (v2di) --> {0,0,0,0,0,0,0,0, 8,8,8,8,8,8,8,8}
5898 : (v4si) --> {0,0,0,0, 4,4,4,4, 8,8,8,8, 12,12,12,12}
5899 : (v8hi) --> {0,0, 2,2, 4,4, 6,6, ...}. */
5900 18 : for (i = 0; i < 16; ++i)
5901 16 : vec[i] = GEN_INT (i/e * e);
5902 1 : vt = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, vec));
5903 1 : vt = validize_mem (force_const_mem (V16QImode, vt));
5904 1 : if (TARGET_XOP)
5905 0 : emit_insn (gen_xop_pperm (mask, mask, mask, vt));
5906 : else
5907 1 : emit_insn (gen_ssse3_pshufbv16qi3 (mask, mask, vt));
5908 :
5909 : /* Convert it into the byte positions by doing
5910 : mask = mask + {0,1,..,16/w, 0,1,..,16/w, ...} */
5911 17 : for (i = 0; i < 16; ++i)
5912 16 : vec[i] = GEN_INT (i % e);
5913 1 : vt = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, vec));
5914 1 : vt = validize_mem (force_const_mem (V16QImode, vt));
5915 1 : emit_insn (gen_addv16qi3 (mask, mask, vt));
5916 : }
5917 :
5918 : /* The actual shuffle operations all operate on V16QImode. */
5919 1 : op0 = gen_lowpart (V16QImode, op0);
5920 1 : op1 = gen_lowpart (V16QImode, op1);
5921 :
5922 1 : if (TARGET_XOP)
5923 : {
5924 0 : if (GET_MODE (target) != V16QImode)
5925 0 : target = gen_reg_rtx (V16QImode);
5926 0 : emit_insn (gen_xop_pperm (target, op0, op1, mask));
5927 0 : if (target != operands[0])
5928 0 : emit_move_insn (operands[0],
5929 0 : gen_lowpart (GET_MODE (operands[0]), target));
5930 : }
5931 1 : else if (one_operand_shuffle)
5932 : {
5933 1 : if (GET_MODE (target) != V16QImode)
5934 1 : target = gen_reg_rtx (V16QImode);
5935 1 : emit_insn (gen_ssse3_pshufbv16qi3 (target, op0, mask));
5936 1 : if (target != operands[0])
5937 1 : emit_move_insn (operands[0],
5938 1 : gen_lowpart (GET_MODE (operands[0]), target));
5939 : }
5940 : else
5941 : {
5942 0 : rtx xops[6];
5943 0 : bool ok;
5944 :
5945 : /* Shuffle the two input vectors independently. */
5946 0 : t1 = gen_reg_rtx (V16QImode);
5947 0 : t2 = gen_reg_rtx (V16QImode);
5948 0 : emit_insn (gen_ssse3_pshufbv16qi3 (t1, op0, mask));
5949 0 : emit_insn (gen_ssse3_pshufbv16qi3 (t2, op1, mask));
5950 :
5951 1 : merge_two:
5952 : /* Then merge them together. The key is whether any given control
5953 : element contained a bit set that indicates the second word. */
5954 1 : mask = operands[3];
5955 1 : vt = GEN_INT (w);
5956 1 : if (maskmode == V2DImode && !TARGET_SSE4_1)
5957 : {
5958 : /* Without SSE4.1, we don't have V2DImode EQ. Perform one
5959 : more shuffle to convert the V2DI input mask into a V4SI
5960 : input mask. At which point the masking that expand_int_vcond
5961 : will work as desired. */
5962 0 : rtx t3 = gen_reg_rtx (V4SImode);
5963 0 : emit_insn (gen_sse2_pshufd_1 (t3, gen_lowpart (V4SImode, mask),
5964 : const0_rtx, const0_rtx,
5965 : const2_rtx, const2_rtx));
5966 0 : mask = t3;
5967 0 : maskmode = V4SImode;
5968 0 : e = w = 4;
5969 : }
5970 :
5971 1 : vt = gen_const_vec_duplicate (maskmode, vt);
5972 1 : vt = force_reg (maskmode, vt);
5973 1 : mask = expand_simple_binop (maskmode, AND, mask, vt,
5974 : NULL_RTX, 0, OPTAB_DIRECT);
5975 :
5976 1 : if (GET_MODE (target) != mode)
5977 0 : target = gen_reg_rtx (mode);
5978 1 : xops[0] = target;
5979 1 : xops[1] = gen_lowpart (mode, t2);
5980 1 : xops[2] = gen_lowpart (mode, t1);
5981 1 : xops[3] = gen_rtx_EQ (maskmode, mask, vt);
5982 1 : xops[4] = mask;
5983 1 : xops[5] = vt;
5984 1 : ok = ix86_expand_int_vcond (xops);
5985 1 : gcc_assert (ok);
5986 1 : if (target != operands[0])
5987 0 : emit_move_insn (operands[0],
5988 0 : gen_lowpart (GET_MODE (operands[0]), target));
5989 : }
5990 : }
5991 :
5992 : /* Extend SRC into next wider integer vector type. UNSIGNED_P is
5993 : true if we should do zero extension, else sign extension. */
5994 :
5995 : void
5996 489 : ix86_expand_sse_extend (rtx dest, rtx src, bool unsigned_p)
5997 : {
5998 489 : machine_mode imode = GET_MODE (src);
5999 489 : rtx ops[3];
6000 :
6001 489 : switch (imode)
6002 : {
6003 489 : case E_V8QImode:
6004 489 : case E_V4QImode:
6005 489 : case E_V2QImode:
6006 489 : case E_V4HImode:
6007 489 : case E_V2HImode:
6008 489 : case E_V2SImode:
6009 489 : break;
6010 0 : default:
6011 0 : gcc_unreachable ();
6012 : }
6013 :
6014 489 : ops[0] = dest;
6015 :
6016 489 : ops[1] = force_reg (imode, src);
6017 :
6018 489 : if (unsigned_p)
6019 105 : ops[2] = force_reg (imode, CONST0_RTX (imode));
6020 : else
6021 384 : ops[2] = ix86_expand_sse_cmp (gen_reg_rtx (imode), GT, CONST0_RTX (imode),
6022 : ops[1], pc_rtx, pc_rtx);
6023 :
6024 489 : ix86_split_mmx_punpck (ops, false);
6025 489 : }
6026 :
6027 : /* Unpack SRC into the next wider integer vector type. UNSIGNED_P is
6028 : true if we should do zero extension, else sign extension. HIGH_P is
6029 : true if we want the N/2 high elements, else the low elements. */
6030 :
6031 : void
6032 19278 : ix86_expand_sse_unpack (rtx dest, rtx src, bool unsigned_p, bool high_p)
6033 : {
6034 19278 : machine_mode imode = GET_MODE (src);
6035 19278 : rtx tmp;
6036 :
6037 19278 : if (TARGET_SSE4_1)
6038 : {
6039 6476 : rtx (*unpack)(rtx, rtx);
6040 6476 : rtx (*extract)(rtx, rtx) = NULL;
6041 6476 : machine_mode halfmode = BLKmode;
6042 :
6043 6476 : switch (imode)
6044 : {
6045 198 : case E_V64QImode:
6046 198 : if (unsigned_p)
6047 : unpack = gen_avx512bw_zero_extendv32qiv32hi2;
6048 : else
6049 64 : unpack = gen_avx512bw_sign_extendv32qiv32hi2;
6050 198 : halfmode = V32QImode;
6051 198 : extract
6052 198 : = high_p ? gen_vec_extract_hi_v64qi : gen_vec_extract_lo_v64qi;
6053 : break;
6054 719 : case E_V32QImode:
6055 719 : if (unsigned_p)
6056 : unpack = gen_avx2_zero_extendv16qiv16hi2;
6057 : else
6058 150 : unpack = gen_avx2_sign_extendv16qiv16hi2;
6059 719 : halfmode = V16QImode;
6060 719 : extract
6061 719 : = high_p ? gen_vec_extract_hi_v32qi : gen_vec_extract_lo_v32qi;
6062 : break;
6063 104 : case E_V32HImode:
6064 104 : if (unsigned_p)
6065 : unpack = gen_avx512f_zero_extendv16hiv16si2;
6066 : else
6067 64 : unpack = gen_avx512f_sign_extendv16hiv16si2;
6068 104 : halfmode = V16HImode;
6069 104 : extract
6070 104 : = high_p ? gen_vec_extract_hi_v32hi : gen_vec_extract_lo_v32hi;
6071 : break;
6072 429 : case E_V16HImode:
6073 429 : if (unsigned_p)
6074 : unpack = gen_avx2_zero_extendv8hiv8si2;
6075 : else
6076 314 : unpack = gen_avx2_sign_extendv8hiv8si2;
6077 429 : halfmode = V8HImode;
6078 429 : extract
6079 429 : = high_p ? gen_vec_extract_hi_v16hi : gen_vec_extract_lo_v16hi;
6080 : break;
6081 104 : case E_V16SImode:
6082 104 : if (unsigned_p)
6083 : unpack = gen_avx512f_zero_extendv8siv8di2;
6084 : else
6085 86 : unpack = gen_avx512f_sign_extendv8siv8di2;
6086 104 : halfmode = V8SImode;
6087 104 : extract
6088 104 : = high_p ? gen_vec_extract_hi_v16si : gen_vec_extract_lo_v16si;
6089 : break;
6090 382 : case E_V8SImode:
6091 382 : if (unsigned_p)
6092 : unpack = gen_avx2_zero_extendv4siv4di2;
6093 : else
6094 320 : unpack = gen_avx2_sign_extendv4siv4di2;
6095 382 : halfmode = V4SImode;
6096 382 : extract
6097 382 : = high_p ? gen_vec_extract_hi_v8si : gen_vec_extract_lo_v8si;
6098 : break;
6099 2597 : case E_V16QImode:
6100 2597 : if (unsigned_p)
6101 : unpack = gen_sse4_1_zero_extendv8qiv8hi2;
6102 : else
6103 270 : unpack = gen_sse4_1_sign_extendv8qiv8hi2;
6104 : break;
6105 993 : case E_V8HImode:
6106 993 : if (unsigned_p)
6107 : unpack = gen_sse4_1_zero_extendv4hiv4si2;
6108 : else
6109 776 : unpack = gen_sse4_1_sign_extendv4hiv4si2;
6110 : break;
6111 546 : case E_V4SImode:
6112 546 : if (unsigned_p)
6113 : unpack = gen_sse4_1_zero_extendv2siv2di2;
6114 : else
6115 486 : unpack = gen_sse4_1_sign_extendv2siv2di2;
6116 : break;
6117 119 : case E_V8QImode:
6118 119 : if (unsigned_p)
6119 : unpack = gen_sse4_1_zero_extendv4qiv4hi2;
6120 : else
6121 78 : unpack = gen_sse4_1_sign_extendv4qiv4hi2;
6122 : break;
6123 279 : case E_V4HImode:
6124 279 : if (unsigned_p)
6125 : unpack = gen_sse4_1_zero_extendv2hiv2si2;
6126 : else
6127 220 : unpack = gen_sse4_1_sign_extendv2hiv2si2;
6128 : break;
6129 6 : case E_V4QImode:
6130 6 : if (unsigned_p)
6131 : unpack = gen_sse4_1_zero_extendv2qiv2hi2;
6132 : else
6133 0 : unpack = gen_sse4_1_sign_extendv2qiv2hi2;
6134 : break;
6135 0 : default:
6136 0 : gcc_unreachable ();
6137 : }
6138 :
6139 12952 : if (GET_MODE_SIZE (imode) >= 32)
6140 : {
6141 1936 : tmp = gen_reg_rtx (halfmode);
6142 1936 : emit_insn (extract (tmp, src));
6143 : }
6144 4540 : else if (high_p)
6145 : {
6146 2354 : switch (GET_MODE_SIZE (imode))
6147 : {
6148 972 : case 16:
6149 : /* Shift higher 8 bytes to lower 8 bytes. */
6150 972 : tmp = gen_reg_rtx (V1TImode);
6151 972 : emit_insn (gen_sse2_lshrv1ti3 (tmp, gen_lowpart (V1TImode, src),
6152 : GEN_INT (64)));
6153 972 : break;
6154 202 : case 8:
6155 : /* Shift higher 4 bytes to lower 4 bytes. */
6156 202 : tmp = gen_reg_rtx (V1DImode);
6157 202 : emit_insn (gen_mmx_lshrv1di3 (tmp, gen_lowpart (V1DImode, src),
6158 : GEN_INT (32)));
6159 202 : break;
6160 3 : case 4:
6161 : /* Shift higher 2 bytes to lower 2 bytes. */
6162 3 : tmp = gen_reg_rtx (V1SImode);
6163 3 : emit_insn (gen_mmx_lshrv1si3 (tmp, gen_lowpart (V1SImode, src),
6164 : GEN_INT (16)));
6165 3 : break;
6166 0 : default:
6167 0 : gcc_unreachable ();
6168 : }
6169 :
6170 1177 : tmp = gen_lowpart (imode, tmp);
6171 : }
6172 : else
6173 : tmp = src;
6174 :
6175 6476 : emit_insn (unpack (dest, tmp));
6176 : }
6177 : else
6178 : {
6179 12802 : rtx (*unpack)(rtx, rtx, rtx);
6180 :
6181 12802 : switch (imode)
6182 : {
6183 3532 : case E_V16QImode:
6184 3532 : if (high_p)
6185 : unpack = gen_vec_interleave_highv16qi;
6186 : else
6187 1769 : unpack = gen_vec_interleave_lowv16qi;
6188 : break;
6189 5266 : case E_V8HImode:
6190 5266 : if (high_p)
6191 : unpack = gen_vec_interleave_highv8hi;
6192 : else
6193 2633 : unpack = gen_vec_interleave_lowv8hi;
6194 : break;
6195 2348 : case E_V4SImode:
6196 2348 : if (high_p)
6197 : unpack = gen_vec_interleave_highv4si;
6198 : else
6199 1174 : unpack = gen_vec_interleave_lowv4si;
6200 : break;
6201 564 : case E_V8QImode:
6202 564 : if (high_p)
6203 : unpack = gen_mmx_punpckhbw;
6204 : else
6205 282 : unpack = gen_mmx_punpcklbw;
6206 : break;
6207 1078 : case E_V4HImode:
6208 1078 : if (high_p)
6209 : unpack = gen_mmx_punpckhwd;
6210 : else
6211 539 : unpack = gen_mmx_punpcklwd;
6212 : break;
6213 14 : case E_V4QImode:
6214 14 : if (high_p)
6215 : unpack = gen_mmx_punpckhbw_low;
6216 : else
6217 7 : unpack = gen_mmx_punpcklbw_low;
6218 : break;
6219 0 : default:
6220 0 : gcc_unreachable ();
6221 : }
6222 :
6223 12802 : if (unsigned_p)
6224 4990 : tmp = force_reg (imode, CONST0_RTX (imode));
6225 : else
6226 7812 : tmp = ix86_expand_sse_cmp (gen_reg_rtx (imode), GT, CONST0_RTX (imode),
6227 : src, pc_rtx, pc_rtx);
6228 :
6229 12802 : rtx tmp2 = gen_reg_rtx (imode);
6230 12802 : emit_insn (unpack (tmp2, src, tmp));
6231 12802 : emit_move_insn (dest, gen_lowpart (GET_MODE (dest), tmp2));
6232 : }
6233 19278 : }
6234 :
6235 : /* Return true if mem is pool constant which contains a const_vector
6236 : perm index, assign the index to PERM. */
6237 : bool
6238 35 : ix86_extract_perm_from_pool_constant (int* perm, rtx mem)
6239 : {
6240 35 : machine_mode mode = GET_MODE (mem);
6241 35 : int nelt = GET_MODE_NUNITS (mode);
6242 :
6243 35 : if (!INTEGRAL_MODE_P (mode))
6244 : return false;
6245 :
6246 : /* Needs to be constant pool. */
6247 35 : if (!(MEM_P (mem))
6248 35 : || !SYMBOL_REF_P (XEXP (mem, 0))
6249 70 : || !CONSTANT_POOL_ADDRESS_P (XEXP (mem, 0)))
6250 : return false;
6251 :
6252 35 : rtx constant = get_pool_constant (XEXP (mem, 0));
6253 :
6254 35 : if (!CONST_VECTOR_P (constant))
6255 : return false;
6256 :
6257 : /* There could be some rtx like
6258 : (mem/u/c:V16QI (symbol_ref/u:DI ("*.LC1")))
6259 : but with "*.LC1" refer to V2DI constant vector. */
6260 35 : if (GET_MODE (constant) != mode)
6261 : {
6262 0 : constant = simplify_subreg (mode, constant, GET_MODE (constant), 0);
6263 :
6264 0 : if (constant == nullptr || !CONST_VECTOR_P (constant))
6265 : return false;
6266 : }
6267 :
6268 771 : for (int i = 0; i != nelt; i++)
6269 736 : perm[i] = UINTVAL (XVECEXP (constant, 0, i));
6270 :
6271 : return true;
6272 : }
6273 :
6274 : /* Split operands 0 and 1 into half-mode parts. Similar to split_double_mode,
6275 : but works for floating pointer parameters and nonoffsetable memories.
6276 : For pushes, it returns just stack offsets; the values will be saved
6277 : in the right order. Maximally three parts are generated. */
6278 :
6279 : static int
6280 4150262 : ix86_split_to_parts (rtx operand, rtx *parts, machine_mode mode)
6281 : {
6282 4150262 : int size;
6283 :
6284 4150262 : if (!TARGET_64BIT)
6285 1561714 : size = mode==XFmode ? 3 : GET_MODE_SIZE (mode) / 4;
6286 : else
6287 6737484 : size = (GET_MODE_SIZE (mode) + 4) / 8;
6288 :
6289 4150262 : gcc_assert (!REG_P (operand) || !MMX_REGNO_P (REGNO (operand)));
6290 4150262 : gcc_assert (size >= 2 && size <= 4);
6291 :
6292 : /* Optimize constant pool reference to immediates. This is used by fp
6293 : moves, that force all constants to memory to allow combining. */
6294 4150262 : if (MEM_P (operand) && MEM_READONLY_P (operand))
6295 38113 : operand = avoid_constant_pool_reference (operand);
6296 :
6297 4150262 : if (MEM_P (operand) && !offsettable_memref_p (operand))
6298 : {
6299 : /* The only non-offsetable memories we handle are pushes. */
6300 184109 : int ok = push_operand (operand, VOIDmode);
6301 :
6302 184109 : gcc_assert (ok);
6303 :
6304 184109 : operand = copy_rtx (operand);
6305 184109 : PUT_MODE (operand, word_mode);
6306 184109 : parts[0] = parts[1] = parts[2] = parts[3] = operand;
6307 184109 : return size;
6308 : }
6309 :
6310 3966153 : if (CONST_VECTOR_P (operand))
6311 : {
6312 41976 : scalar_int_mode imode = int_mode_for_mode (mode).require ();
6313 : /* Caution: if we looked through a constant pool memory above,
6314 : the operand may actually have a different mode now. That's
6315 : ok, since we want to pun this all the way back to an integer. */
6316 41976 : operand = simplify_subreg (imode, operand, GET_MODE (operand), 0);
6317 41976 : gcc_assert (operand != NULL);
6318 41976 : mode = imode;
6319 : }
6320 :
6321 3966153 : if (!TARGET_64BIT)
6322 : {
6323 622897 : if (mode == DImode)
6324 493776 : split_double_mode (mode, &operand, 1, &parts[0], &parts[1]);
6325 : else
6326 : {
6327 129121 : int i;
6328 :
6329 129121 : if (REG_P (operand))
6330 : {
6331 66946 : gcc_assert (reload_completed);
6332 200838 : for (i = 0; i < size; i++)
6333 133892 : parts[i] = gen_rtx_REG (SImode, REGNO (operand) + i);
6334 : }
6335 62175 : else if (offsettable_memref_p (operand))
6336 : {
6337 60827 : operand = adjust_address (operand, SImode, 0);
6338 60827 : parts[0] = operand;
6339 122188 : for (i = 1; i < size; i++)
6340 61361 : parts[i] = adjust_address (operand, SImode, 4 * i);
6341 : }
6342 1348 : else if (CONST_DOUBLE_P (operand))
6343 : {
6344 1348 : const REAL_VALUE_TYPE *r;
6345 1348 : long l[4];
6346 :
6347 1348 : r = CONST_DOUBLE_REAL_VALUE (operand);
6348 1348 : switch (mode)
6349 : {
6350 0 : case E_TFmode:
6351 0 : real_to_target (l, r, mode);
6352 0 : parts[3] = gen_int_mode (l[3], SImode);
6353 0 : parts[2] = gen_int_mode (l[2], SImode);
6354 0 : break;
6355 198 : case E_XFmode:
6356 : /* We can't use REAL_VALUE_TO_TARGET_LONG_DOUBLE since
6357 : long double may not be 80-bit. */
6358 198 : real_to_target (l, r, mode);
6359 198 : parts[2] = gen_int_mode (l[2], SImode);
6360 198 : break;
6361 1150 : case E_DFmode:
6362 1150 : REAL_VALUE_TO_TARGET_DOUBLE (*r, l);
6363 1150 : break;
6364 0 : default:
6365 0 : gcc_unreachable ();
6366 : }
6367 1348 : parts[1] = gen_int_mode (l[1], SImode);
6368 1348 : parts[0] = gen_int_mode (l[0], SImode);
6369 : }
6370 : else
6371 0 : gcc_unreachable ();
6372 : }
6373 : }
6374 : else
6375 : {
6376 3343256 : if (mode == TImode)
6377 3323094 : split_double_mode (mode, &operand, 1, &parts[0], &parts[1]);
6378 3343256 : if (mode == XFmode || mode == TFmode)
6379 : {
6380 20162 : machine_mode upper_mode = mode==XFmode ? SImode : DImode;
6381 20162 : if (REG_P (operand))
6382 : {
6383 1494 : gcc_assert (reload_completed);
6384 1494 : parts[0] = gen_rtx_REG (DImode, REGNO (operand) + 0);
6385 1494 : parts[1] = gen_rtx_REG (upper_mode, REGNO (operand) + 1);
6386 : }
6387 18668 : else if (offsettable_memref_p (operand))
6388 : {
6389 14571 : operand = adjust_address (operand, DImode, 0);
6390 14571 : parts[0] = operand;
6391 14571 : parts[1] = adjust_address (operand, upper_mode, 8);
6392 : }
6393 4097 : else if (CONST_DOUBLE_P (operand))
6394 : {
6395 4097 : long l[4];
6396 :
6397 4097 : real_to_target (l, CONST_DOUBLE_REAL_VALUE (operand), mode);
6398 :
6399 : /* real_to_target puts 32-bit pieces in each long. */
6400 8194 : parts[0] = gen_int_mode ((l[0] & HOST_WIDE_INT_C (0xffffffff))
6401 4097 : | ((l[1] & HOST_WIDE_INT_C (0xffffffff))
6402 4097 : << 32), DImode);
6403 :
6404 4097 : if (upper_mode == SImode)
6405 2911 : parts[1] = gen_int_mode (l[2], SImode);
6406 : else
6407 1186 : parts[1]
6408 1186 : = gen_int_mode ((l[2] & HOST_WIDE_INT_C (0xffffffff))
6409 1186 : | ((l[3] & HOST_WIDE_INT_C (0xffffffff))
6410 1186 : << 32), DImode);
6411 : }
6412 : else
6413 0 : gcc_unreachable ();
6414 : }
6415 : }
6416 :
6417 : return size;
6418 : }
6419 :
6420 : /* Emit insns to perform a move or push of DI, DF, XF, and TF values.
6421 : Return false when normal moves are needed; true when all required
6422 : insns have been emitted. Operands 2-4 contain the input values
6423 : int the correct order; operands 5-7 contain the output values. */
6424 :
6425 : void
6426 2088091 : ix86_split_long_move (rtx operands[])
6427 : {
6428 2088091 : rtx part[2][4];
6429 2088091 : int nparts, i, j;
6430 2088091 : int push = 0;
6431 2088091 : int collisions = 0;
6432 2088091 : machine_mode mode = GET_MODE (operands[0]);
6433 2088091 : bool collisionparts[4];
6434 :
6435 : /* The DFmode expanders may ask us to move double.
6436 : For 64bit target this is single move. By hiding the fact
6437 : here we simplify i386.md splitters. */
6438 3785422 : if (TARGET_64BIT && GET_MODE_SIZE (GET_MODE (operands[0])) == 8)
6439 : {
6440 : /* Optimize constant pool reference to immediates. This is used by
6441 : fp moves, that force all constants to memory to allow combining. */
6442 :
6443 12960 : if (MEM_P (operands[1])
6444 12537 : && SYMBOL_REF_P (XEXP (operands[1], 0))
6445 13604 : && CONSTANT_POOL_ADDRESS_P (XEXP (operands[1], 0)))
6446 155 : operands[1] = get_pool_constant (XEXP (operands[1], 0));
6447 12960 : if (push_operand (operands[0], VOIDmode))
6448 : {
6449 12960 : operands[0] = copy_rtx (operands[0]);
6450 12960 : PUT_MODE (operands[0], word_mode);
6451 : }
6452 : else
6453 0 : operands[0] = gen_lowpart (DImode, operands[0]);
6454 12960 : operands[1] = gen_lowpart (DImode, operands[1]);
6455 12960 : emit_move_insn (operands[0], operands[1]);
6456 12960 : return;
6457 : }
6458 :
6459 : /* The only non-offsettable memory we handle is push. */
6460 2075131 : if (push_operand (operands[0], VOIDmode))
6461 : push = 1;
6462 : else
6463 1891022 : gcc_assert (!MEM_P (operands[0])
6464 : || offsettable_memref_p (operands[0]));
6465 :
6466 2075131 : nparts = ix86_split_to_parts (operands[1], part[1], GET_MODE (operands[0]));
6467 2075131 : ix86_split_to_parts (operands[0], part[0], GET_MODE (operands[0]));
6468 :
6469 : /* When emitting push, take care for source operands on the stack. */
6470 184109 : if (push && MEM_P (operands[1])
6471 2172615 : && reg_overlap_mentioned_p (stack_pointer_rtx, operands[1]))
6472 : {
6473 56576 : rtx src_base = XEXP (part[1][nparts - 1], 0);
6474 :
6475 : /* Compensate for the stack decrement by 4. */
6476 56576 : if (!TARGET_64BIT && nparts == 3
6477 51875 : && mode == XFmode && TARGET_128BIT_LONG_DOUBLE)
6478 0 : src_base = plus_constant (Pmode, src_base, 4);
6479 :
6480 : /* src_base refers to the stack pointer and is
6481 : automatically decreased by emitted push. */
6482 170007 : for (i = 0; i < nparts; i++)
6483 113431 : part[1][i] = change_address (part[1][i],
6484 113431 : GET_MODE (part[1][i]), src_base);
6485 : }
6486 :
6487 : /* We need to do copy in the right order in case an address register
6488 : of the source overlaps the destination. */
6489 2075131 : if (REG_P (part[0][0]) && MEM_P (part[1][0]))
6490 : {
6491 : rtx tmp;
6492 :
6493 2373597 : for (i = 0; i < nparts; i++)
6494 : {
6495 1582398 : collisionparts[i]
6496 1582398 : = reg_overlap_mentioned_p (part[0][i], XEXP (part[1][0], 0));
6497 1582398 : if (collisionparts[i])
6498 16890 : collisions++;
6499 : }
6500 :
6501 : /* Collision in the middle part can be handled by reordering. */
6502 791199 : if (collisions == 1 && nparts == 3 && collisionparts [1])
6503 : {
6504 0 : std::swap (part[0][1], part[0][2]);
6505 0 : std::swap (part[1][1], part[1][2]);
6506 : }
6507 791199 : else if (collisions == 1
6508 791199 : && nparts == 4
6509 0 : && (collisionparts [1] || collisionparts [2]))
6510 : {
6511 0 : if (collisionparts [1])
6512 : {
6513 0 : std::swap (part[0][1], part[0][2]);
6514 0 : std::swap (part[1][1], part[1][2]);
6515 : }
6516 : else
6517 : {
6518 0 : std::swap (part[0][2], part[0][3]);
6519 0 : std::swap (part[1][2], part[1][3]);
6520 : }
6521 : }
6522 :
6523 : /* If there are more collisions, we can't handle it by reordering.
6524 : Do an lea to the last part and use only one colliding move. */
6525 791199 : else if (collisions > 1)
6526 : {
6527 76 : rtx base, addr;
6528 :
6529 76 : collisions = 1;
6530 :
6531 76 : base = part[0][nparts - 1];
6532 :
6533 : /* Handle the case when the last part isn't valid for lea.
6534 : Happens in 64-bit mode storing the 12-byte XFmode. */
6535 110 : if (GET_MODE (base) != Pmode)
6536 0 : base = gen_rtx_REG (Pmode, REGNO (base));
6537 :
6538 76 : addr = XEXP (part[1][0], 0);
6539 76 : if (TARGET_TLS_DIRECT_SEG_REFS)
6540 : {
6541 76 : struct ix86_address parts;
6542 76 : int ok = ix86_decompose_address (addr, &parts);
6543 76 : gcc_assert (ok);
6544 : /* It is not valid to use %gs: or %fs: in lea. */
6545 76 : gcc_assert (parts.seg == ADDR_SPACE_GENERIC);
6546 : }
6547 76 : emit_insn (gen_rtx_SET (base, addr));
6548 76 : part[1][0] = replace_equiv_address (part[1][0], base);
6549 152 : for (i = 1; i < nparts; i++)
6550 : {
6551 144 : tmp = plus_constant (Pmode, base, UNITS_PER_WORD * i);
6552 76 : part[1][i] = replace_equiv_address (part[1][i], tmp);
6553 : }
6554 : }
6555 : }
6556 :
6557 2075131 : if (push)
6558 : {
6559 184109 : if (!TARGET_64BIT)
6560 : {
6561 158623 : if (nparts == 3)
6562 : {
6563 594 : if (TARGET_128BIT_LONG_DOUBLE && mode == XFmode)
6564 0 : emit_insn (gen_add2_insn (stack_pointer_rtx, GEN_INT (-4)));
6565 594 : emit_move_insn (part[0][2], part[1][2]);
6566 : }
6567 158029 : else if (nparts == 4)
6568 : {
6569 0 : emit_move_insn (part[0][3], part[1][3]);
6570 0 : emit_move_insn (part[0][2], part[1][2]);
6571 : }
6572 : }
6573 : else
6574 : {
6575 : /* In 64bit mode we don't have 32bit push available. In case this is
6576 : register, it is OK - we will just use larger counterpart. We also
6577 : retype memory - these comes from attempt to avoid REX prefix on
6578 : moving of second half of TFmode value. */
6579 25486 : if (GET_MODE (part[1][1]) == SImode)
6580 : {
6581 11313 : switch (GET_CODE (part[1][1]))
6582 : {
6583 10879 : case MEM:
6584 10879 : part[1][1] = adjust_address (part[1][1], DImode, 0);
6585 10879 : break;
6586 :
6587 434 : case REG:
6588 434 : part[1][1] = gen_rtx_REG (DImode, REGNO (part[1][1]));
6589 434 : break;
6590 :
6591 0 : default:
6592 0 : gcc_unreachable ();
6593 : }
6594 :
6595 11313 : if (GET_MODE (part[1][0]) == SImode)
6596 0 : part[1][0] = part[1][1];
6597 : }
6598 : }
6599 184109 : emit_move_insn (part[0][1], part[1][1]);
6600 184109 : emit_move_insn (part[0][0], part[1][0]);
6601 184109 : return;
6602 : }
6603 :
6604 : /* Choose correct order to not overwrite the source before it is copied. */
6605 1891022 : if ((REG_P (part[0][0])
6606 1032617 : && REG_P (part[1][1])
6607 82667 : && (REGNO (part[0][0]) == REGNO (part[1][1])
6608 67340 : || (nparts == 3
6609 0 : && REGNO (part[0][0]) == REGNO (part[1][2]))
6610 67340 : || (nparts == 4
6611 0 : && REGNO (part[0][0]) == REGNO (part[1][3]))))
6612 2908312 : || (collisions > 0
6613 16814 : && reg_overlap_mentioned_p (part[0][0], XEXP (part[1][0], 0))))
6614 : {
6615 95658 : for (i = 0, j = nparts - 1; i < nparts; i++, j--)
6616 : {
6617 63772 : operands[2 + i] = part[0][j];
6618 63772 : operands[6 + i] = part[1][j];
6619 : }
6620 : }
6621 : else
6622 : {
6623 5577477 : for (i = 0; i < nparts; i++)
6624 : {
6625 3718341 : operands[2 + i] = part[0][i];
6626 3718341 : operands[6 + i] = part[1][i];
6627 : }
6628 : }
6629 :
6630 : /* Attempt to locally unCSE nonzero constants. */
6631 3782113 : for (j = 0; j < nparts - 1; j++)
6632 1891091 : if (CONST_INT_P (operands[6 + j])
6633 226021 : && operands[6 + j] != const0_rtx
6634 63535 : && REG_P (operands[2 + j]))
6635 112936 : for (i = j; i < nparts - 1; i++)
6636 56468 : if (CONST_INT_P (operands[7 + i])
6637 56468 : && INTVAL (operands[7 + i]) == INTVAL (operands[6 + j]))
6638 22607 : operands[7 + i] = operands[2 + j];
6639 :
6640 5673135 : for (i = 0; i < nparts; i++)
6641 3782113 : emit_move_insn (operands[2 + i], operands[6 + i]);
6642 :
6643 : return;
6644 : }
6645 :
6646 : /* Helper function of ix86_split_ashl used to generate an SImode/DImode
6647 : left shift by a constant, either using a single shift or
6648 : a sequence of add instructions. */
6649 :
6650 : static void
6651 4364 : ix86_expand_ashl_const (rtx operand, int count, machine_mode mode)
6652 : {
6653 4364 : if (count == 1
6654 4364 : || (count * ix86_cost->add <= ix86_cost->shift_const
6655 0 : && !optimize_insn_for_size_p ()))
6656 : {
6657 16 : while (count-- > 0)
6658 8 : emit_insn (gen_add2_insn (operand, operand));
6659 : }
6660 : else
6661 : {
6662 4356 : rtx (*insn)(rtx, rtx, rtx);
6663 :
6664 4356 : insn = mode == DImode ? gen_ashlsi3 : gen_ashldi3;
6665 4356 : emit_insn (insn (operand, operand, GEN_INT (count)));
6666 : }
6667 4364 : }
6668 :
6669 : void
6670 10520 : ix86_split_ashl (rtx *operands, rtx scratch, machine_mode mode)
6671 : {
6672 10520 : rtx (*gen_ashl3)(rtx, rtx, rtx);
6673 10520 : rtx (*gen_shld)(rtx, rtx, rtx);
6674 10520 : int half_width = GET_MODE_BITSIZE (mode) >> 1;
6675 10520 : machine_mode half_mode;
6676 :
6677 10520 : rtx low[2], high[2];
6678 10520 : int count;
6679 :
6680 10520 : if (CONST_INT_P (operands[2]))
6681 : {
6682 8812 : split_double_mode (mode, operands, 2, low, high);
6683 8812 : count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
6684 :
6685 8812 : if (count >= half_width)
6686 : {
6687 2723 : emit_move_insn (high[0], low[1]);
6688 2723 : ix86_expand_clear (low[0]);
6689 :
6690 2723 : if (count > half_width)
6691 145 : ix86_expand_ashl_const (high[0], count - half_width, mode);
6692 : }
6693 6089 : else if (count == 1)
6694 : {
6695 1870 : if (!rtx_equal_p (operands[0], operands[1]))
6696 0 : emit_move_insn (operands[0], operands[1]);
6697 1870 : rtx x3 = gen_rtx_REG (CCCmode, FLAGS_REG);
6698 1870 : rtx x4 = gen_rtx_LTU (mode, x3, const0_rtx);
6699 1870 : half_mode = mode == DImode ? SImode : DImode;
6700 1870 : emit_insn (gen_add3_cc_overflow_1 (half_mode, low[0],
6701 : low[0], low[0]));
6702 1870 : emit_insn (gen_add3_carry (half_mode, high[0], high[0], high[0],
6703 : x3, x4));
6704 : }
6705 : else
6706 : {
6707 4219 : gen_shld = mode == DImode ? gen_x86_shld : gen_x86_64_shld;
6708 :
6709 4219 : if (!rtx_equal_p (operands[0], operands[1]))
6710 0 : emit_move_insn (operands[0], operands[1]);
6711 :
6712 4219 : emit_insn (gen_shld (high[0], low[0], GEN_INT (count)));
6713 4219 : ix86_expand_ashl_const (low[0], count, mode);
6714 : }
6715 9082 : return;
6716 : }
6717 :
6718 1708 : split_double_mode (mode, operands, 1, low, high);
6719 1708 : half_mode = mode == DImode ? SImode : DImode;
6720 :
6721 1708 : gen_ashl3 = mode == DImode ? gen_ashlsi3 : gen_ashldi3;
6722 :
6723 1708 : if (operands[1] == const1_rtx)
6724 : {
6725 : /* Assuming we've chosen a QImode capable registers, then 1 << N
6726 : can be done with two 32/64-bit shifts, no branches, no cmoves. */
6727 270 : if (ANY_QI_REG_P (low[0]) && ANY_QI_REG_P (high[0]))
6728 : {
6729 163 : rtx s, d, flags = gen_rtx_REG (CCZmode, FLAGS_REG);
6730 :
6731 163 : ix86_expand_clear (low[0]);
6732 163 : ix86_expand_clear (high[0]);
6733 163 : emit_insn (gen_testqi_ccz_1 (operands[2], GEN_INT (half_width)));
6734 :
6735 163 : d = gen_lowpart (QImode, low[0]);
6736 163 : d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
6737 163 : s = gen_rtx_EQ (QImode, flags, const0_rtx);
6738 163 : emit_insn (gen_rtx_SET (d, s));
6739 :
6740 163 : d = gen_lowpart (QImode, high[0]);
6741 163 : d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
6742 163 : s = gen_rtx_NE (QImode, flags, const0_rtx);
6743 163 : emit_insn (gen_rtx_SET (d, s));
6744 : }
6745 :
6746 : /* Otherwise, we can get the same results by manually performing
6747 : a bit extract operation on bit 5/6, and then performing the two
6748 : shifts. The two methods of getting 0/1 into low/high are exactly
6749 : the same size. Avoiding the shift in the bit extract case helps
6750 : pentium4 a bit; no one else seems to care much either way. */
6751 : else
6752 : {
6753 107 : rtx (*gen_lshr3)(rtx, rtx, rtx);
6754 107 : rtx (*gen_and3)(rtx, rtx, rtx);
6755 107 : rtx (*gen_xor3)(rtx, rtx, rtx);
6756 107 : HOST_WIDE_INT bits;
6757 107 : rtx x;
6758 :
6759 107 : if (mode == DImode)
6760 : {
6761 : gen_lshr3 = gen_lshrsi3;
6762 : gen_and3 = gen_andsi3;
6763 : gen_xor3 = gen_xorsi3;
6764 : bits = 5;
6765 : }
6766 : else
6767 : {
6768 0 : gen_lshr3 = gen_lshrdi3;
6769 0 : gen_and3 = gen_anddi3;
6770 0 : gen_xor3 = gen_xordi3;
6771 0 : bits = 6;
6772 : }
6773 :
6774 107 : if (TARGET_PARTIAL_REG_STALL && !optimize_insn_for_size_p ())
6775 0 : x = gen_rtx_ZERO_EXTEND (half_mode, operands[2]);
6776 : else
6777 107 : x = gen_lowpart (half_mode, operands[2]);
6778 107 : emit_insn (gen_rtx_SET (high[0], x));
6779 :
6780 107 : emit_insn (gen_lshr3 (high[0], high[0], GEN_INT (bits)));
6781 107 : emit_insn (gen_and3 (high[0], high[0], const1_rtx));
6782 107 : emit_move_insn (low[0], high[0]);
6783 107 : emit_insn (gen_xor3 (low[0], low[0], const1_rtx));
6784 : }
6785 :
6786 270 : emit_insn (gen_ashl3 (low[0], low[0], operands[2]));
6787 270 : emit_insn (gen_ashl3 (high[0], high[0], operands[2]));
6788 270 : return;
6789 : }
6790 :
6791 1438 : if (operands[1] == constm1_rtx)
6792 : {
6793 : /* For -1 << N, we can avoid the shld instruction, because we
6794 : know that we're shifting 0...31/63 ones into a -1. */
6795 115 : emit_move_insn (low[0], constm1_rtx);
6796 115 : if (optimize_insn_for_size_p ())
6797 6 : emit_move_insn (high[0], low[0]);
6798 : else
6799 109 : emit_move_insn (high[0], constm1_rtx);
6800 : }
6801 : else
6802 : {
6803 1323 : gen_shld = mode == DImode ? gen_x86_shld : gen_x86_64_shld;
6804 :
6805 1323 : if (!rtx_equal_p (operands[0], operands[1]))
6806 0 : emit_move_insn (operands[0], operands[1]);
6807 :
6808 1323 : split_double_mode (mode, operands, 1, low, high);
6809 1323 : emit_insn (gen_shld (high[0], low[0], operands[2]));
6810 : }
6811 :
6812 1438 : emit_insn (gen_ashl3 (low[0], low[0], operands[2]));
6813 :
6814 1438 : if (TARGET_CMOVE && scratch)
6815 : {
6816 963 : ix86_expand_clear (scratch);
6817 963 : emit_insn (gen_x86_shift_adj_1
6818 : (half_mode, high[0], low[0], operands[2], scratch));
6819 : }
6820 : else
6821 475 : emit_insn (gen_x86_shift_adj_2 (half_mode, high[0], low[0], operands[2]));
6822 : }
6823 :
6824 : void
6825 6455 : ix86_split_ashr (rtx *operands, rtx scratch, machine_mode mode)
6826 : {
6827 5211 : rtx (*gen_ashr3)(rtx, rtx, rtx)
6828 6455 : = mode == DImode ? gen_ashrsi3 : gen_ashrdi3;
6829 6455 : rtx (*gen_shrd)(rtx, rtx, rtx);
6830 6455 : int half_width = GET_MODE_BITSIZE (mode) >> 1;
6831 :
6832 6455 : rtx low[2], high[2];
6833 6455 : int count;
6834 :
6835 6455 : if (CONST_INT_P (operands[2]))
6836 : {
6837 6282 : split_double_mode (mode, operands, 2, low, high);
6838 6282 : count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
6839 :
6840 12564 : if (count == GET_MODE_BITSIZE (mode) - 1)
6841 : {
6842 89 : emit_move_insn (high[0], high[1]);
6843 89 : emit_insn (gen_ashr3 (high[0], high[0],
6844 89 : GEN_INT (half_width - 1)));
6845 89 : emit_move_insn (low[0], high[0]);
6846 :
6847 : }
6848 6193 : else if (count >= half_width)
6849 : {
6850 1995 : emit_move_insn (low[0], high[1]);
6851 1995 : emit_move_insn (high[0], low[0]);
6852 1995 : emit_insn (gen_ashr3 (high[0], high[0],
6853 1995 : GEN_INT (half_width - 1)));
6854 :
6855 1995 : if (count > half_width)
6856 38 : emit_insn (gen_ashr3 (low[0], low[0],
6857 38 : GEN_INT (count - half_width)));
6858 : }
6859 4198 : else if (count == 1
6860 766 : && (TARGET_USE_RCR || optimize_size > 1))
6861 : {
6862 1 : if (!rtx_equal_p (operands[0], operands[1]))
6863 0 : emit_move_insn (operands[0], operands[1]);
6864 1 : if (mode == DImode)
6865 : {
6866 0 : emit_insn (gen_ashrsi3_carry (high[0], high[0]));
6867 0 : emit_insn (gen_rcrsi2 (low[0], low[0]));
6868 : }
6869 : else
6870 : {
6871 1 : emit_insn (gen_ashrdi3_carry (high[0], high[0]));
6872 1 : emit_insn (gen_rcrdi2 (low[0], low[0]));
6873 : }
6874 : }
6875 : else
6876 : {
6877 4197 : gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
6878 :
6879 4197 : if (!rtx_equal_p (operands[0], operands[1]))
6880 0 : emit_move_insn (operands[0], operands[1]);
6881 :
6882 4197 : emit_insn (gen_shrd (low[0], high[0], GEN_INT (count)));
6883 4197 : emit_insn (gen_ashr3 (high[0], high[0], GEN_INT (count)));
6884 : }
6885 : }
6886 : else
6887 : {
6888 173 : machine_mode half_mode;
6889 :
6890 173 : gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
6891 :
6892 173 : if (!rtx_equal_p (operands[0], operands[1]))
6893 0 : emit_move_insn (operands[0], operands[1]);
6894 :
6895 173 : split_double_mode (mode, operands, 1, low, high);
6896 173 : half_mode = mode == DImode ? SImode : DImode;
6897 :
6898 173 : emit_insn (gen_shrd (low[0], high[0], operands[2]));
6899 173 : emit_insn (gen_ashr3 (high[0], high[0], operands[2]));
6900 :
6901 173 : if (TARGET_CMOVE && scratch)
6902 : {
6903 133 : emit_move_insn (scratch, high[0]);
6904 133 : emit_insn (gen_ashr3 (scratch, scratch,
6905 133 : GEN_INT (half_width - 1)));
6906 133 : emit_insn (gen_x86_shift_adj_1
6907 : (half_mode, low[0], high[0], operands[2], scratch));
6908 : }
6909 : else
6910 40 : emit_insn (gen_x86_shift_adj_3
6911 : (half_mode, low[0], high[0], operands[2]));
6912 : }
6913 6455 : }
6914 :
6915 : void
6916 13864 : ix86_split_lshr (rtx *operands, rtx scratch, machine_mode mode)
6917 : {
6918 6513 : rtx (*gen_lshr3)(rtx, rtx, rtx)
6919 13864 : = mode == DImode ? gen_lshrsi3 : gen_lshrdi3;
6920 13864 : rtx (*gen_shrd)(rtx, rtx, rtx);
6921 13864 : int half_width = GET_MODE_BITSIZE (mode) >> 1;
6922 :
6923 13864 : rtx low[2], high[2];
6924 13864 : int count;
6925 :
6926 13864 : if (CONST_INT_P (operands[2]))
6927 : {
6928 12444 : split_double_mode (mode, operands, 2, low, high);
6929 12444 : count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
6930 :
6931 12444 : if (count >= half_width)
6932 : {
6933 9035 : emit_move_insn (low[0], high[1]);
6934 9035 : ix86_expand_clear (high[0]);
6935 :
6936 9035 : if (count > half_width)
6937 655 : emit_insn (gen_lshr3 (low[0], low[0],
6938 655 : GEN_INT (count - half_width)));
6939 : }
6940 3409 : else if (count == 1
6941 653 : && (TARGET_USE_RCR || optimize_size > 1))
6942 : {
6943 1 : if (!rtx_equal_p (operands[0], operands[1]))
6944 0 : emit_move_insn (operands[0], operands[1]);
6945 1 : if (mode == DImode)
6946 : {
6947 0 : emit_insn (gen_lshrsi3_carry (high[0], high[0]));
6948 0 : emit_insn (gen_rcrsi2 (low[0], low[0]));
6949 : }
6950 : else
6951 : {
6952 1 : emit_insn (gen_lshrdi3_carry (high[0], high[0]));
6953 1 : emit_insn (gen_rcrdi2 (low[0], low[0]));
6954 : }
6955 : }
6956 : else
6957 : {
6958 3408 : gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
6959 :
6960 3408 : if (!rtx_equal_p (operands[0], operands[1]))
6961 0 : emit_move_insn (operands[0], operands[1]);
6962 :
6963 3408 : emit_insn (gen_shrd (low[0], high[0], GEN_INT (count)));
6964 3408 : emit_insn (gen_lshr3 (high[0], high[0], GEN_INT (count)));
6965 : }
6966 : }
6967 : else
6968 : {
6969 1420 : machine_mode half_mode;
6970 :
6971 1420 : gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
6972 :
6973 1420 : if (!rtx_equal_p (operands[0], operands[1]))
6974 0 : emit_move_insn (operands[0], operands[1]);
6975 :
6976 1420 : split_double_mode (mode, operands, 1, low, high);
6977 1420 : half_mode = mode == DImode ? SImode : DImode;
6978 :
6979 1420 : emit_insn (gen_shrd (low[0], high[0], operands[2]));
6980 1420 : emit_insn (gen_lshr3 (high[0], high[0], operands[2]));
6981 :
6982 1420 : if (TARGET_CMOVE && scratch)
6983 : {
6984 1122 : ix86_expand_clear (scratch);
6985 1122 : emit_insn (gen_x86_shift_adj_1
6986 : (half_mode, low[0], high[0], operands[2], scratch));
6987 : }
6988 : else
6989 298 : emit_insn (gen_x86_shift_adj_2
6990 : (half_mode, low[0], high[0], operands[2]));
6991 : }
6992 13864 : }
6993 :
6994 : /* Helper function to split TImode ashl under NDD. */
6995 : void
6996 1 : ix86_split_ashl_ndd (rtx *operands, rtx scratch)
6997 : {
6998 1 : gcc_assert (TARGET_APX_NDD);
6999 1 : int half_width = GET_MODE_BITSIZE (TImode) >> 1;
7000 :
7001 1 : rtx low[2], high[2];
7002 1 : int count;
7003 :
7004 1 : split_double_mode (TImode, operands, 2, low, high);
7005 1 : if (CONST_INT_P (operands[2]))
7006 : {
7007 0 : count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (TImode) - 1);
7008 :
7009 0 : if (count >= half_width)
7010 : {
7011 0 : count = count - half_width;
7012 0 : if (count == 0)
7013 : {
7014 0 : if (!rtx_equal_p (high[0], low[1]))
7015 0 : emit_move_insn (high[0], low[1]);
7016 : }
7017 0 : else if (count == 1)
7018 0 : emit_insn (gen_adddi3 (high[0], low[1], low[1]));
7019 : else
7020 0 : emit_insn (gen_ashldi3 (high[0], low[1], GEN_INT (count)));
7021 :
7022 0 : ix86_expand_clear (low[0]);
7023 : }
7024 0 : else if (count == 1)
7025 : {
7026 0 : rtx x3 = gen_rtx_REG (CCCmode, FLAGS_REG);
7027 0 : rtx x4 = gen_rtx_LTU (TImode, x3, const0_rtx);
7028 0 : emit_insn (gen_add3_cc_overflow_1 (DImode, low[0],
7029 : low[1], low[1]));
7030 0 : emit_insn (gen_add3_carry (DImode, high[0], high[1], high[1],
7031 : x3, x4));
7032 : }
7033 : else
7034 : {
7035 0 : emit_insn (gen_x86_64_shld_ndd (high[0], high[1], low[1],
7036 : GEN_INT (count)));
7037 0 : emit_insn (gen_ashldi3 (low[0], low[1], GEN_INT (count)));
7038 : }
7039 : }
7040 : else
7041 : {
7042 1 : emit_insn (gen_x86_64_shld_ndd (high[0], high[1], low[1],
7043 : operands[2]));
7044 1 : emit_insn (gen_ashldi3 (low[0], low[1], operands[2]));
7045 1 : if (TARGET_CMOVE && scratch)
7046 : {
7047 1 : ix86_expand_clear (scratch);
7048 1 : emit_insn (gen_x86_shift_adj_1
7049 : (DImode, high[0], low[0], operands[2], scratch));
7050 : }
7051 : else
7052 0 : emit_insn (gen_x86_shift_adj_2 (DImode, high[0], low[0], operands[2]));
7053 : }
7054 1 : }
7055 :
7056 : /* Helper function to split TImode l/ashr under NDD. */
7057 : void
7058 2 : ix86_split_rshift_ndd (enum rtx_code code, rtx *operands, rtx scratch)
7059 : {
7060 2 : gcc_assert (TARGET_APX_NDD);
7061 2 : int half_width = GET_MODE_BITSIZE (TImode) >> 1;
7062 2 : bool ashr_p = code == ASHIFTRT;
7063 2 : rtx (*gen_shr)(rtx, rtx, rtx) = ashr_p ? gen_ashrdi3
7064 : : gen_lshrdi3;
7065 :
7066 2 : rtx low[2], high[2];
7067 2 : int count;
7068 :
7069 2 : split_double_mode (TImode, operands, 2, low, high);
7070 2 : if (CONST_INT_P (operands[2]))
7071 : {
7072 0 : count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (TImode) - 1);
7073 :
7074 0 : if (ashr_p && (count == GET_MODE_BITSIZE (TImode) - 1))
7075 : {
7076 0 : emit_insn (gen_shr (high[0], high[1],
7077 : GEN_INT (half_width - 1)));
7078 0 : emit_move_insn (low[0], high[0]);
7079 : }
7080 0 : else if (count >= half_width)
7081 : {
7082 0 : if (ashr_p)
7083 0 : emit_insn (gen_shr (high[0], high[1],
7084 : GEN_INT (half_width - 1)));
7085 : else
7086 0 : ix86_expand_clear (high[0]);
7087 :
7088 0 : if (count > half_width)
7089 0 : emit_insn (gen_shr (low[0], high[1],
7090 0 : GEN_INT (count - half_width)));
7091 : else
7092 0 : emit_move_insn (low[0], high[1]);
7093 : }
7094 : else
7095 : {
7096 0 : emit_insn (gen_x86_64_shrd_ndd (low[0], low[1], high[1],
7097 : GEN_INT (count)));
7098 0 : emit_insn (gen_shr (high[0], high[1], GEN_INT (count)));
7099 : }
7100 : }
7101 : else
7102 : {
7103 2 : emit_insn (gen_x86_64_shrd_ndd (low[0], low[1], high[1],
7104 : operands[2]));
7105 2 : emit_insn (gen_shr (high[0], high[1], operands[2]));
7106 :
7107 2 : if (TARGET_CMOVE && scratch)
7108 : {
7109 2 : if (ashr_p)
7110 : {
7111 1 : emit_move_insn (scratch, high[0]);
7112 1 : emit_insn (gen_shr (scratch, scratch,
7113 : GEN_INT (half_width - 1)));
7114 : }
7115 : else
7116 1 : ix86_expand_clear (scratch);
7117 :
7118 2 : emit_insn (gen_x86_shift_adj_1
7119 : (DImode, low[0], high[0], operands[2], scratch));
7120 : }
7121 0 : else if (ashr_p)
7122 0 : emit_insn (gen_x86_shift_adj_3
7123 : (DImode, low[0], high[0], operands[2]));
7124 : else
7125 0 : emit_insn (gen_x86_shift_adj_2
7126 : (DImode, low[0], high[0], operands[2]));
7127 : }
7128 2 : }
7129 :
7130 : /* Expand move of V1TI mode register X to a new TI mode register. */
7131 : static rtx
7132 17 : ix86_expand_v1ti_to_ti (rtx x)
7133 : {
7134 17 : rtx result = gen_reg_rtx (TImode);
7135 17 : if (TARGET_SSE2)
7136 : {
7137 17 : rtx temp = force_reg (V2DImode, gen_lowpart (V2DImode, x));
7138 17 : rtx lo = gen_lowpart (DImode, result);
7139 17 : emit_insn (gen_vec_extractv2didi (lo, temp, const0_rtx));
7140 17 : rtx hi = gen_highpart (DImode, result);
7141 17 : emit_insn (gen_vec_extractv2didi (hi, temp, const1_rtx));
7142 : }
7143 : else
7144 0 : emit_move_insn (result, gen_lowpart (TImode, x));
7145 17 : return result;
7146 : }
7147 :
7148 : /* Expand move of TI mode register X to a new V1TI mode register. */
7149 : static rtx
7150 17 : ix86_expand_ti_to_v1ti (rtx x)
7151 : {
7152 17 : if (TARGET_SSE2)
7153 : {
7154 17 : rtx lo = gen_lowpart (DImode, x);
7155 17 : rtx hi = gen_highpart (DImode, x);
7156 17 : rtx tmp = gen_reg_rtx (V2DImode);
7157 17 : emit_insn (gen_vec_concatv2di (tmp, lo, hi));
7158 17 : return force_reg (V1TImode, gen_lowpart (V1TImode, tmp));
7159 : }
7160 :
7161 0 : return force_reg (V1TImode, gen_lowpart (V1TImode, x));
7162 : }
7163 :
7164 : /* Expand V1TI mode shift (of rtx_code CODE) by constant. */
7165 : void
7166 42 : ix86_expand_v1ti_shift (enum rtx_code code, rtx operands[])
7167 : {
7168 42 : rtx op1 = force_reg (V1TImode, operands[1]);
7169 :
7170 42 : if (!CONST_INT_P (operands[2]))
7171 : {
7172 6 : rtx tmp1 = ix86_expand_v1ti_to_ti (op1);
7173 6 : rtx tmp2 = gen_reg_rtx (TImode);
7174 3 : rtx (*shift) (rtx, rtx, rtx)
7175 6 : = (code == ASHIFT) ? gen_ashlti3 : gen_lshrti3;
7176 6 : emit_insn (shift (tmp2, tmp1, operands[2]));
7177 6 : rtx tmp3 = ix86_expand_ti_to_v1ti (tmp2);
7178 6 : emit_move_insn (operands[0], tmp3);
7179 6 : return;
7180 : }
7181 :
7182 36 : HOST_WIDE_INT bits = INTVAL (operands[2]) & 127;
7183 :
7184 36 : if (bits == 0)
7185 : {
7186 0 : emit_move_insn (operands[0], op1);
7187 0 : return;
7188 : }
7189 :
7190 36 : if ((bits & 7) == 0)
7191 : {
7192 0 : rtx tmp = gen_reg_rtx (V1TImode);
7193 0 : if (code == ASHIFT)
7194 0 : emit_insn (gen_sse2_ashlv1ti3 (tmp, op1, GEN_INT (bits)));
7195 : else
7196 0 : emit_insn (gen_sse2_lshrv1ti3 (tmp, op1, GEN_INT (bits)));
7197 0 : emit_move_insn (operands[0], tmp);
7198 0 : return;
7199 : }
7200 :
7201 36 : rtx tmp1 = gen_reg_rtx (V1TImode);
7202 36 : if (code == ASHIFT)
7203 18 : emit_insn (gen_sse2_ashlv1ti3 (tmp1, op1, GEN_INT (64)));
7204 : else
7205 18 : emit_insn (gen_sse2_lshrv1ti3 (tmp1, op1, GEN_INT (64)));
7206 :
7207 : /* tmp2 is operands[1] shifted by 64, in V2DImode. */
7208 36 : rtx tmp2 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp1));
7209 :
7210 : /* tmp3 will be the V2DImode result. */
7211 36 : rtx tmp3 = gen_reg_rtx (V2DImode);
7212 :
7213 36 : if (bits > 64)
7214 : {
7215 18 : if (code == ASHIFT)
7216 9 : emit_insn (gen_ashlv2di3 (tmp3, tmp2, GEN_INT (bits - 64)));
7217 : else
7218 9 : emit_insn (gen_lshrv2di3 (tmp3, tmp2, GEN_INT (bits - 64)));
7219 : }
7220 : else
7221 : {
7222 : /* tmp4 is operands[1], in V2DImode. */
7223 18 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
7224 :
7225 18 : rtx tmp5 = gen_reg_rtx (V2DImode);
7226 18 : if (code == ASHIFT)
7227 9 : emit_insn (gen_ashlv2di3 (tmp5, tmp4, GEN_INT (bits)));
7228 : else
7229 9 : emit_insn (gen_lshrv2di3 (tmp5, tmp4, GEN_INT (bits)));
7230 :
7231 18 : rtx tmp6 = gen_reg_rtx (V2DImode);
7232 18 : if (code == ASHIFT)
7233 9 : emit_insn (gen_lshrv2di3 (tmp6, tmp2, GEN_INT (64 - bits)));
7234 : else
7235 9 : emit_insn (gen_ashlv2di3 (tmp6, tmp2, GEN_INT (64 - bits)));
7236 :
7237 18 : emit_insn (gen_iorv2di3 (tmp3, tmp5, tmp6));
7238 : }
7239 :
7240 : /* Convert the result back to V1TImode and store in operands[0]. */
7241 36 : rtx tmp7 = force_reg (V1TImode, gen_lowpart (V1TImode, tmp3));
7242 36 : emit_move_insn (operands[0], tmp7);
7243 : }
7244 :
7245 : /* Expand V1TI mode rotate (of rtx_code CODE) by constant. */
7246 : void
7247 39 : ix86_expand_v1ti_rotate (enum rtx_code code, rtx operands[])
7248 : {
7249 39 : rtx op1 = force_reg (V1TImode, operands[1]);
7250 :
7251 39 : if (!CONST_INT_P (operands[2]))
7252 : {
7253 8 : rtx tmp1 = ix86_expand_v1ti_to_ti (op1);
7254 8 : rtx tmp2 = gen_reg_rtx (TImode);
7255 4 : rtx (*rotate) (rtx, rtx, rtx)
7256 8 : = (code == ROTATE) ? gen_rotlti3 : gen_rotrti3;
7257 8 : emit_insn (rotate (tmp2, tmp1, operands[2]));
7258 8 : rtx tmp3 = ix86_expand_ti_to_v1ti (tmp2);
7259 8 : emit_move_insn (operands[0], tmp3);
7260 8 : return;
7261 : }
7262 :
7263 31 : HOST_WIDE_INT bits = INTVAL (operands[2]) & 127;
7264 :
7265 31 : if (bits == 0)
7266 : {
7267 0 : emit_move_insn (operands[0], op1);
7268 0 : return;
7269 : }
7270 :
7271 31 : if (code == ROTATERT)
7272 16 : bits = 128 - bits;
7273 :
7274 31 : if ((bits & 31) == 0)
7275 : {
7276 5 : rtx tmp2 = gen_reg_rtx (V4SImode);
7277 5 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7278 5 : if (bits == 32)
7279 1 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0x93)));
7280 4 : else if (bits == 64)
7281 2 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0x4e)));
7282 : else
7283 2 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0x39)));
7284 5 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp2));
7285 5 : return;
7286 : }
7287 :
7288 26 : if ((bits & 7) == 0)
7289 : {
7290 6 : rtx tmp1 = gen_reg_rtx (V1TImode);
7291 6 : rtx tmp2 = gen_reg_rtx (V1TImode);
7292 6 : rtx tmp3 = gen_reg_rtx (V1TImode);
7293 :
7294 6 : emit_insn (gen_sse2_ashlv1ti3 (tmp1, op1, GEN_INT (bits)));
7295 6 : emit_insn (gen_sse2_lshrv1ti3 (tmp2, op1, GEN_INT (128 - bits)));
7296 6 : emit_insn (gen_iorv1ti3 (tmp3, tmp1, tmp2));
7297 6 : emit_move_insn (operands[0], tmp3);
7298 6 : return;
7299 : }
7300 :
7301 20 : rtx op1_v4si = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7302 :
7303 20 : rtx lobits;
7304 20 : rtx hibits;
7305 :
7306 20 : switch (bits >> 5)
7307 : {
7308 7 : case 0:
7309 7 : lobits = op1_v4si;
7310 7 : hibits = gen_reg_rtx (V4SImode);
7311 7 : emit_insn (gen_sse2_pshufd (hibits, op1_v4si, GEN_INT (0x93)));
7312 7 : break;
7313 :
7314 2 : case 1:
7315 2 : lobits = gen_reg_rtx (V4SImode);
7316 2 : hibits = gen_reg_rtx (V4SImode);
7317 2 : emit_insn (gen_sse2_pshufd (lobits, op1_v4si, GEN_INT (0x93)));
7318 2 : emit_insn (gen_sse2_pshufd (hibits, op1_v4si, GEN_INT (0x4e)));
7319 2 : break;
7320 :
7321 2 : case 2:
7322 2 : lobits = gen_reg_rtx (V4SImode);
7323 2 : hibits = gen_reg_rtx (V4SImode);
7324 2 : emit_insn (gen_sse2_pshufd (lobits, op1_v4si, GEN_INT (0x4e)));
7325 2 : emit_insn (gen_sse2_pshufd (hibits, op1_v4si, GEN_INT (0x39)));
7326 2 : break;
7327 :
7328 9 : default:
7329 9 : lobits = gen_reg_rtx (V4SImode);
7330 9 : emit_insn (gen_sse2_pshufd (lobits, op1_v4si, GEN_INT (0x39)));
7331 9 : hibits = op1_v4si;
7332 9 : break;
7333 : }
7334 :
7335 20 : rtx tmp1 = gen_reg_rtx (V4SImode);
7336 20 : rtx tmp2 = gen_reg_rtx (V4SImode);
7337 20 : rtx tmp3 = gen_reg_rtx (V4SImode);
7338 :
7339 20 : emit_insn (gen_ashlv4si3 (tmp1, lobits, GEN_INT (bits & 31)));
7340 20 : emit_insn (gen_lshrv4si3 (tmp2, hibits, GEN_INT (32 - (bits & 31))));
7341 20 : emit_insn (gen_iorv4si3 (tmp3, tmp1, tmp2));
7342 :
7343 20 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp3));
7344 : }
7345 :
7346 : /* Expand V1TI mode ashiftrt by constant. */
7347 : void
7348 109 : ix86_expand_v1ti_ashiftrt (rtx operands[])
7349 : {
7350 109 : rtx op1 = force_reg (V1TImode, operands[1]);
7351 :
7352 109 : if (!CONST_INT_P (operands[2]))
7353 : {
7354 3 : rtx tmp1 = ix86_expand_v1ti_to_ti (op1);
7355 3 : rtx tmp2 = gen_reg_rtx (TImode);
7356 3 : emit_insn (gen_ashrti3 (tmp2, tmp1, operands[2]));
7357 3 : rtx tmp3 = ix86_expand_ti_to_v1ti (tmp2);
7358 3 : emit_move_insn (operands[0], tmp3);
7359 3 : return;
7360 : }
7361 :
7362 106 : HOST_WIDE_INT bits = INTVAL (operands[2]) & 127;
7363 :
7364 106 : if (bits == 0)
7365 : {
7366 0 : emit_move_insn (operands[0], op1);
7367 0 : return;
7368 : }
7369 :
7370 106 : if (bits == 127)
7371 : {
7372 : /* Two operations. */
7373 3 : rtx tmp1 = force_reg(V4SImode, gen_lowpart (V4SImode, op1));
7374 3 : rtx tmp2 = gen_reg_rtx (V4SImode);
7375 3 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
7376 :
7377 3 : rtx tmp3 = gen_reg_rtx (V4SImode);
7378 3 : emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
7379 :
7380 3 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp3));
7381 3 : return;
7382 : }
7383 :
7384 103 : if (bits == 64)
7385 : {
7386 : /* Three operations. */
7387 3 : rtx tmp1 = force_reg(V4SImode, gen_lowpart (V4SImode, op1));
7388 3 : rtx tmp2 = gen_reg_rtx (V4SImode);
7389 3 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
7390 :
7391 3 : rtx tmp3 = gen_reg_rtx (V4SImode);
7392 3 : emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
7393 :
7394 3 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp1));
7395 3 : rtx tmp5 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
7396 3 : rtx tmp6 = gen_reg_rtx (V2DImode);
7397 3 : emit_insn (gen_vec_interleave_highv2di (tmp6, tmp4, tmp5));
7398 :
7399 3 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp6));
7400 3 : return;
7401 : }
7402 :
7403 100 : if (bits == 96)
7404 : {
7405 : /* Three operations. */
7406 3 : rtx tmp1 = force_reg(V4SImode, gen_lowpart (V4SImode, op1));
7407 3 : rtx tmp2 = gen_reg_rtx (V4SImode);
7408 3 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (31)));
7409 :
7410 3 : rtx tmp3 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp1));
7411 3 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp2));
7412 3 : rtx tmp5 = gen_reg_rtx (V2DImode);
7413 3 : emit_insn (gen_vec_interleave_highv2di (tmp5, tmp3, tmp4));
7414 :
7415 3 : rtx tmp6 = force_reg(V4SImode, gen_lowpart (V4SImode, tmp5));
7416 3 : rtx tmp7 = gen_reg_rtx (V4SImode);
7417 3 : emit_insn (gen_sse2_pshufd (tmp7, tmp6, GEN_INT (0xfd)));
7418 :
7419 3 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp7));
7420 3 : return;
7421 : }
7422 :
7423 97 : if (bits >= 111)
7424 : {
7425 : /* Three operations. */
7426 21 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7427 21 : rtx tmp2 = gen_reg_rtx (V4SImode);
7428 21 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits - 96)));
7429 :
7430 21 : rtx tmp3 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp2));
7431 21 : rtx tmp4 = gen_reg_rtx (V8HImode);
7432 21 : emit_insn (gen_sse2_pshufhw (tmp4, tmp3, GEN_INT (0xfe)));
7433 :
7434 21 : rtx tmp5 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp4));
7435 21 : rtx tmp6 = gen_reg_rtx (V4SImode);
7436 21 : emit_insn (gen_sse2_pshufd (tmp6, tmp5, GEN_INT (0xfe)));
7437 :
7438 21 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp6));
7439 21 : return;
7440 : }
7441 :
7442 76 : if (TARGET_AVX2 || TARGET_SSE4_1)
7443 : {
7444 : /* Three operations. */
7445 50 : if (bits == 32)
7446 : {
7447 2 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7448 2 : rtx tmp2 = gen_reg_rtx (V4SImode);
7449 2 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (31)));
7450 :
7451 2 : rtx tmp3 = gen_reg_rtx (V1TImode);
7452 2 : emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (32)));
7453 :
7454 2 : if (TARGET_AVX2)
7455 : {
7456 1 : rtx tmp4 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp3));
7457 1 : rtx tmp5 = gen_reg_rtx (V4SImode);
7458 1 : emit_insn (gen_avx2_pblenddv4si (tmp5, tmp2, tmp4,
7459 : GEN_INT (7)));
7460 :
7461 1 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp5));
7462 : }
7463 : else
7464 : {
7465 1 : rtx tmp4 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp2));
7466 1 : rtx tmp5 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp3));
7467 1 : rtx tmp6 = gen_reg_rtx (V8HImode);
7468 1 : emit_insn (gen_sse4_1_pblendw (tmp6, tmp4, tmp5,
7469 : GEN_INT (0x3f)));
7470 :
7471 1 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp6));
7472 : }
7473 2 : return;
7474 : }
7475 :
7476 : /* Three operations. */
7477 48 : if (bits == 8 || bits == 16 || bits == 24)
7478 : {
7479 6 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7480 6 : rtx tmp2 = gen_reg_rtx (V4SImode);
7481 6 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits)));
7482 :
7483 6 : rtx tmp3 = gen_reg_rtx (V1TImode);
7484 6 : emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (bits)));
7485 :
7486 6 : if (TARGET_AVX2)
7487 : {
7488 3 : rtx tmp4 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp3));
7489 3 : rtx tmp5 = gen_reg_rtx (V4SImode);
7490 3 : emit_insn (gen_avx2_pblenddv4si (tmp5, tmp2, tmp4,
7491 : GEN_INT (7)));
7492 :
7493 3 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp5));
7494 : }
7495 : else
7496 : {
7497 3 : rtx tmp4 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp2));
7498 3 : rtx tmp5 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp3));
7499 3 : rtx tmp6 = gen_reg_rtx (V8HImode);
7500 3 : emit_insn (gen_sse4_1_pblendw (tmp6, tmp4, tmp5,
7501 : GEN_INT (0x3f)));
7502 :
7503 3 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp6));
7504 : }
7505 6 : return;
7506 : }
7507 : }
7508 :
7509 68 : if (bits > 96)
7510 : {
7511 : /* Four operations. */
7512 3 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7513 3 : rtx tmp2 = gen_reg_rtx (V4SImode);
7514 3 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits - 96)));
7515 :
7516 3 : rtx tmp3 = gen_reg_rtx (V4SImode);
7517 3 : emit_insn (gen_ashrv4si3 (tmp3, tmp1, GEN_INT (31)));
7518 :
7519 3 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp2));
7520 3 : rtx tmp5 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
7521 3 : rtx tmp6 = gen_reg_rtx (V2DImode);
7522 3 : emit_insn (gen_vec_interleave_highv2di (tmp6, tmp4, tmp5));
7523 :
7524 3 : rtx tmp7 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp6));
7525 3 : rtx tmp8 = gen_reg_rtx (V4SImode);
7526 3 : emit_insn (gen_sse2_pshufd (tmp8, tmp7, GEN_INT (0xfd)));
7527 :
7528 3 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp8));
7529 3 : return;
7530 : }
7531 :
7532 65 : if (TARGET_SSE4_1 && (bits == 48 || bits == 80))
7533 : {
7534 : /* Four operations. */
7535 4 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7536 4 : rtx tmp2 = gen_reg_rtx (V4SImode);
7537 4 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
7538 :
7539 4 : rtx tmp3 = gen_reg_rtx (V4SImode);
7540 4 : emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
7541 :
7542 4 : rtx tmp4 = gen_reg_rtx (V1TImode);
7543 4 : emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (bits)));
7544 :
7545 4 : rtx tmp5 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp3));
7546 4 : rtx tmp6 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp4));
7547 4 : rtx tmp7 = gen_reg_rtx (V8HImode);
7548 6 : emit_insn (gen_sse4_1_pblendw (tmp7, tmp5, tmp6,
7549 : GEN_INT (bits == 48 ? 0x1f : 0x07)));
7550 :
7551 4 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp7));
7552 4 : return;
7553 : }
7554 :
7555 61 : if ((bits & 7) == 0)
7556 : {
7557 : /* Five operations. */
7558 9 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7559 9 : rtx tmp2 = gen_reg_rtx (V4SImode);
7560 9 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
7561 :
7562 9 : rtx tmp3 = gen_reg_rtx (V4SImode);
7563 9 : emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
7564 :
7565 9 : rtx tmp4 = gen_reg_rtx (V1TImode);
7566 9 : emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (bits)));
7567 :
7568 9 : rtx tmp5 = force_reg (V1TImode, gen_lowpart (V1TImode, tmp3));
7569 9 : rtx tmp6 = gen_reg_rtx (V1TImode);
7570 9 : emit_insn (gen_sse2_ashlv1ti3 (tmp6, tmp5, GEN_INT (128 - bits)));
7571 :
7572 9 : rtx tmp7 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp4));
7573 9 : rtx tmp8 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp6));
7574 9 : rtx tmp9 = gen_reg_rtx (V2DImode);
7575 9 : emit_insn (gen_iorv2di3 (tmp9, tmp7, tmp8));
7576 :
7577 9 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp9));
7578 9 : return;
7579 : }
7580 :
7581 52 : if (TARGET_AVX2 && bits < 32)
7582 : {
7583 : /* Six operations. */
7584 9 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7585 9 : rtx tmp2 = gen_reg_rtx (V4SImode);
7586 9 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits)));
7587 :
7588 9 : rtx tmp3 = gen_reg_rtx (V1TImode);
7589 9 : emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (64)));
7590 :
7591 9 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
7592 9 : rtx tmp5 = gen_reg_rtx (V2DImode);
7593 9 : emit_insn (gen_lshrv2di3 (tmp5, tmp4, GEN_INT (bits)));
7594 :
7595 9 : rtx tmp6 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
7596 9 : rtx tmp7 = gen_reg_rtx (V2DImode);
7597 9 : emit_insn (gen_ashlv2di3 (tmp7, tmp6, GEN_INT (64 - bits)));
7598 :
7599 9 : rtx tmp8 = gen_reg_rtx (V2DImode);
7600 9 : emit_insn (gen_iorv2di3 (tmp8, tmp5, tmp7));
7601 :
7602 9 : rtx tmp9 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp8));
7603 9 : rtx tmp10 = gen_reg_rtx (V4SImode);
7604 9 : emit_insn (gen_avx2_pblenddv4si (tmp10, tmp2, tmp9, GEN_INT (7)));
7605 :
7606 9 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp10));
7607 9 : return;
7608 : }
7609 :
7610 43 : if (TARGET_SSE4_1 && bits < 15)
7611 : {
7612 : /* Six operations. */
7613 4 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7614 4 : rtx tmp2 = gen_reg_rtx (V4SImode);
7615 4 : emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits)));
7616 :
7617 4 : rtx tmp3 = gen_reg_rtx (V1TImode);
7618 4 : emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (64)));
7619 :
7620 4 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
7621 4 : rtx tmp5 = gen_reg_rtx (V2DImode);
7622 4 : emit_insn (gen_lshrv2di3 (tmp5, tmp4, GEN_INT (bits)));
7623 :
7624 4 : rtx tmp6 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
7625 4 : rtx tmp7 = gen_reg_rtx (V2DImode);
7626 4 : emit_insn (gen_ashlv2di3 (tmp7, tmp6, GEN_INT (64 - bits)));
7627 :
7628 4 : rtx tmp8 = gen_reg_rtx (V2DImode);
7629 4 : emit_insn (gen_iorv2di3 (tmp8, tmp5, tmp7));
7630 :
7631 4 : rtx tmp9 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp2));
7632 4 : rtx tmp10 = force_reg (V8HImode, gen_lowpart (V8HImode, tmp8));
7633 4 : rtx tmp11 = gen_reg_rtx (V8HImode);
7634 4 : emit_insn (gen_sse4_1_pblendw (tmp11, tmp9, tmp10, GEN_INT (0x3f)));
7635 :
7636 4 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp11));
7637 4 : return;
7638 : }
7639 :
7640 18 : if (bits == 1)
7641 : {
7642 : /* Eight operations. */
7643 1 : rtx tmp1 = gen_reg_rtx (V1TImode);
7644 1 : emit_insn (gen_sse2_lshrv1ti3 (tmp1, op1, GEN_INT (64)));
7645 :
7646 1 : rtx tmp2 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
7647 1 : rtx tmp3 = gen_reg_rtx (V2DImode);
7648 1 : emit_insn (gen_lshrv2di3 (tmp3, tmp2, GEN_INT (1)));
7649 :
7650 1 : rtx tmp4 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp1));
7651 1 : rtx tmp5 = gen_reg_rtx (V2DImode);
7652 1 : emit_insn (gen_ashlv2di3 (tmp5, tmp4, GEN_INT (63)));
7653 :
7654 1 : rtx tmp6 = gen_reg_rtx (V2DImode);
7655 1 : emit_insn (gen_iorv2di3 (tmp6, tmp3, tmp5));
7656 :
7657 1 : rtx tmp7 = gen_reg_rtx (V2DImode);
7658 1 : emit_insn (gen_lshrv2di3 (tmp7, tmp2, GEN_INT (63)));
7659 :
7660 1 : rtx tmp8 = force_reg (V4SImode, gen_lowpart (V4SImode, tmp7));
7661 1 : rtx tmp9 = gen_reg_rtx (V4SImode);
7662 1 : emit_insn (gen_sse2_pshufd (tmp9, tmp8, GEN_INT (0xbf)));
7663 :
7664 1 : rtx tmp10 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp9));
7665 1 : rtx tmp11 = gen_reg_rtx (V2DImode);
7666 1 : emit_insn (gen_ashlv2di3 (tmp11, tmp10, GEN_INT (31)));
7667 :
7668 1 : rtx tmp12 = gen_reg_rtx (V2DImode);
7669 1 : emit_insn (gen_iorv2di3 (tmp12, tmp6, tmp11));
7670 :
7671 1 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp12));
7672 1 : return;
7673 : }
7674 :
7675 38 : if (bits > 64)
7676 : {
7677 : /* Eight operations. */
7678 12 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7679 12 : rtx tmp2 = gen_reg_rtx (V4SImode);
7680 12 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
7681 :
7682 12 : rtx tmp3 = gen_reg_rtx (V4SImode);
7683 12 : emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
7684 :
7685 12 : rtx tmp4 = gen_reg_rtx (V1TImode);
7686 12 : emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (64)));
7687 :
7688 12 : rtx tmp5 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp4));
7689 12 : rtx tmp6 = gen_reg_rtx (V2DImode);
7690 12 : emit_insn (gen_lshrv2di3 (tmp6, tmp5, GEN_INT (bits - 64)));
7691 :
7692 12 : rtx tmp7 = force_reg (V1TImode, gen_lowpart (V1TImode, tmp3));
7693 12 : rtx tmp8 = gen_reg_rtx (V1TImode);
7694 12 : emit_insn (gen_sse2_ashlv1ti3 (tmp8, tmp7, GEN_INT (64)));
7695 :
7696 12 : rtx tmp9 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp3));
7697 12 : rtx tmp10 = gen_reg_rtx (V2DImode);
7698 12 : emit_insn (gen_ashlv2di3 (tmp10, tmp9, GEN_INT (128 - bits)));
7699 :
7700 12 : rtx tmp11 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp8));
7701 12 : rtx tmp12 = gen_reg_rtx (V2DImode);
7702 12 : emit_insn (gen_iorv2di3 (tmp12, tmp10, tmp11));
7703 :
7704 12 : rtx tmp13 = gen_reg_rtx (V2DImode);
7705 12 : emit_insn (gen_iorv2di3 (tmp13, tmp6, tmp12));
7706 :
7707 12 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp13));
7708 : }
7709 : else
7710 : {
7711 : /* Nine operations. */
7712 26 : rtx tmp1 = force_reg (V4SImode, gen_lowpart (V4SImode, op1));
7713 26 : rtx tmp2 = gen_reg_rtx (V4SImode);
7714 26 : emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
7715 :
7716 26 : rtx tmp3 = gen_reg_rtx (V4SImode);
7717 26 : emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
7718 :
7719 26 : rtx tmp4 = gen_reg_rtx (V1TImode);
7720 26 : emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (64)));
7721 :
7722 26 : rtx tmp5 = force_reg (V2DImode, gen_lowpart (V2DImode, op1));
7723 26 : rtx tmp6 = gen_reg_rtx (V2DImode);
7724 26 : emit_insn (gen_lshrv2di3 (tmp6, tmp5, GEN_INT (bits)));
7725 :
7726 26 : rtx tmp7 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp4));
7727 26 : rtx tmp8 = gen_reg_rtx (V2DImode);
7728 26 : emit_insn (gen_ashlv2di3 (tmp8, tmp7, GEN_INT (64 - bits)));
7729 :
7730 26 : rtx tmp9 = gen_reg_rtx (V2DImode);
7731 26 : emit_insn (gen_iorv2di3 (tmp9, tmp6, tmp8));
7732 :
7733 26 : rtx tmp10 = force_reg (V1TImode, gen_lowpart (V1TImode, tmp3));
7734 26 : rtx tmp11 = gen_reg_rtx (V1TImode);
7735 26 : emit_insn (gen_sse2_ashlv1ti3 (tmp11, tmp10, GEN_INT (64)));
7736 :
7737 26 : rtx tmp12 = force_reg (V2DImode, gen_lowpart (V2DImode, tmp11));
7738 26 : rtx tmp13 = gen_reg_rtx (V2DImode);
7739 26 : emit_insn (gen_ashlv2di3 (tmp13, tmp12, GEN_INT (64 - bits)));
7740 :
7741 26 : rtx tmp14 = gen_reg_rtx (V2DImode);
7742 26 : emit_insn (gen_iorv2di3 (tmp14, tmp9, tmp13));
7743 :
7744 26 : emit_move_insn (operands[0], gen_lowpart (V1TImode, tmp14));
7745 : }
7746 : }
7747 :
7748 : /* Expand V2DI mode ashiftrt. */
7749 : void
7750 439 : ix86_expand_v2di_ashiftrt (rtx operands[])
7751 : {
7752 439 : if (operands[2] == const0_rtx)
7753 : {
7754 0 : emit_move_insn (operands[0], operands[1]);
7755 0 : return;
7756 : }
7757 :
7758 439 : if (TARGET_SSE4_2
7759 135 : && CONST_INT_P (operands[2])
7760 135 : && UINTVAL (operands[2]) >= 63
7761 447 : && !optimize_insn_for_size_p ())
7762 : {
7763 8 : rtx zero = force_reg (V2DImode, CONST0_RTX (V2DImode));
7764 8 : emit_insn (gen_sse4_2_gtv2di3 (operands[0], zero, operands[1]));
7765 8 : return;
7766 : }
7767 :
7768 431 : if (CONST_INT_P (operands[2])
7769 405 : && (!TARGET_XOP || UINTVAL (operands[2]) >= 63))
7770 : {
7771 309 : vec_perm_builder sel (4, 4, 1);
7772 309 : sel.quick_grow (4);
7773 309 : rtx arg0, arg1;
7774 309 : rtx op1 = lowpart_subreg (V4SImode,
7775 : force_reg (V2DImode, operands[1]),
7776 : V2DImode);
7777 309 : rtx target = gen_reg_rtx (V4SImode);
7778 309 : if (UINTVAL (operands[2]) >= 63)
7779 : {
7780 117 : arg0 = arg1 = gen_reg_rtx (V4SImode);
7781 117 : emit_insn (gen_ashrv4si3 (arg0, op1, GEN_INT (31)));
7782 117 : sel[0] = 1;
7783 117 : sel[1] = 1;
7784 117 : sel[2] = 3;
7785 117 : sel[3] = 3;
7786 : }
7787 192 : else if (INTVAL (operands[2]) > 32)
7788 : {
7789 21 : arg0 = gen_reg_rtx (V4SImode);
7790 21 : arg1 = gen_reg_rtx (V4SImode);
7791 21 : emit_insn (gen_ashrv4si3 (arg1, op1, GEN_INT (31)));
7792 21 : emit_insn (gen_ashrv4si3 (arg0, op1,
7793 21 : GEN_INT (INTVAL (operands[2]) - 32)));
7794 21 : sel[0] = 1;
7795 21 : sel[1] = 5;
7796 21 : sel[2] = 3;
7797 21 : sel[3] = 7;
7798 : }
7799 171 : else if (INTVAL (operands[2]) == 32)
7800 : {
7801 5 : arg0 = op1;
7802 5 : arg1 = gen_reg_rtx (V4SImode);
7803 5 : emit_insn (gen_ashrv4si3 (arg1, op1, GEN_INT (31)));
7804 5 : sel[0] = 1;
7805 5 : sel[1] = 5;
7806 5 : sel[2] = 3;
7807 5 : sel[3] = 7;
7808 : }
7809 : else
7810 : {
7811 166 : arg0 = gen_reg_rtx (V2DImode);
7812 166 : arg1 = gen_reg_rtx (V4SImode);
7813 166 : emit_insn (gen_lshrv2di3 (arg0, operands[1], operands[2]));
7814 166 : emit_insn (gen_ashrv4si3 (arg1, op1, operands[2]));
7815 166 : arg0 = lowpart_subreg (V4SImode, arg0, V2DImode);
7816 166 : sel[0] = 0;
7817 166 : sel[1] = 5;
7818 166 : sel[2] = 2;
7819 166 : sel[3] = 7;
7820 : }
7821 426 : vec_perm_indices indices (sel, arg0 != arg1 ? 2 : 1, 4);
7822 309 : rtx op0 = operands[0];
7823 309 : bool ok = targetm.vectorize.vec_perm_const (V4SImode, V4SImode,
7824 : target, arg0, arg1,
7825 : indices);
7826 309 : gcc_assert (ok);
7827 309 : emit_move_insn (op0, lowpart_subreg (V2DImode, target, V4SImode));
7828 309 : return;
7829 309 : }
7830 122 : if (!TARGET_XOP)
7831 : {
7832 26 : rtx zero = force_reg (V2DImode, CONST0_RTX (V2DImode));
7833 26 : rtx zero_or_all_ones;
7834 26 : if (TARGET_SSE4_2)
7835 : {
7836 0 : zero_or_all_ones = gen_reg_rtx (V2DImode);
7837 0 : emit_insn (gen_sse4_2_gtv2di3 (zero_or_all_ones, zero,
7838 : operands[1]));
7839 : }
7840 : else
7841 : {
7842 26 : rtx temp = gen_reg_rtx (V4SImode);
7843 26 : emit_insn (gen_ashrv4si3 (temp,
7844 : lowpart_subreg (V4SImode,
7845 : force_reg (V2DImode,
7846 : operands[1]),
7847 : V2DImode),
7848 : GEN_INT (31)));
7849 26 : zero_or_all_ones = gen_reg_rtx (V4SImode);
7850 26 : emit_insn (gen_sse2_pshufd_1 (zero_or_all_ones, temp,
7851 : const1_rtx, const1_rtx,
7852 : GEN_INT (3), GEN_INT (3)));
7853 26 : zero_or_all_ones = lowpart_subreg (V2DImode, zero_or_all_ones,
7854 : V4SImode);
7855 : }
7856 26 : rtx lshr_res = gen_reg_rtx (V2DImode);
7857 26 : emit_insn (gen_lshrv2di3 (lshr_res, operands[1], operands[2]));
7858 26 : rtx ashl_res = gen_reg_rtx (V2DImode);
7859 26 : rtx amount;
7860 26 : if (TARGET_64BIT)
7861 : {
7862 26 : amount = gen_reg_rtx (DImode);
7863 26 : emit_insn (gen_subdi3 (amount, force_reg (DImode, GEN_INT (64)),
7864 : operands[2]));
7865 : }
7866 : else
7867 : {
7868 0 : rtx temp = gen_reg_rtx (SImode);
7869 0 : emit_insn (gen_subsi3 (temp, force_reg (SImode, GEN_INT (64)),
7870 : lowpart_subreg (SImode, operands[2],
7871 : DImode)));
7872 0 : amount = gen_reg_rtx (V4SImode);
7873 0 : emit_insn (gen_vec_setv4si_0 (amount, CONST0_RTX (V4SImode),
7874 : temp));
7875 : }
7876 26 : amount = lowpart_subreg (DImode, amount, GET_MODE (amount));
7877 26 : emit_insn (gen_ashlv2di3 (ashl_res, zero_or_all_ones, amount));
7878 26 : emit_insn (gen_iorv2di3 (operands[0], lshr_res, ashl_res));
7879 26 : return;
7880 : }
7881 :
7882 96 : rtx reg = gen_reg_rtx (V2DImode);
7883 96 : rtx par;
7884 96 : bool negate = false;
7885 96 : int i;
7886 :
7887 96 : if (CONST_INT_P (operands[2]))
7888 96 : operands[2] = GEN_INT (-INTVAL (operands[2]));
7889 : else
7890 : negate = true;
7891 :
7892 96 : par = gen_rtx_PARALLEL (V2DImode, rtvec_alloc (2));
7893 288 : for (i = 0; i < 2; i++)
7894 192 : XVECEXP (par, 0, i) = operands[2];
7895 :
7896 96 : emit_insn (gen_vec_initv2didi (reg, par));
7897 :
7898 96 : if (negate)
7899 0 : emit_insn (gen_negv2di2 (reg, reg));
7900 :
7901 96 : emit_insn (gen_xop_shav2di3 (operands[0], operands[1], reg));
7902 : }
7903 :
7904 : /* Replace all occurrences of REG FROM with REG TO in X, including
7905 : occurrences with different modes. */
7906 :
7907 : rtx
7908 43025 : ix86_replace_reg_with_reg (rtx x, rtx from, rtx to)
7909 : {
7910 43025 : gcc_checking_assert (REG_P (from)
7911 : && REG_P (to)
7912 : && GET_MODE (from) == GET_MODE (to));
7913 43025 : if (!reg_overlap_mentioned_p (from, x))
7914 : return x;
7915 90 : rtx ret = copy_rtx (x);
7916 90 : subrtx_ptr_iterator::array_type array;
7917 456 : FOR_EACH_SUBRTX_PTR (iter, array, &ret, NONCONST)
7918 : {
7919 366 : rtx *loc = *iter;
7920 366 : x = *loc;
7921 366 : if (REG_P (x) && REGNO (x) == REGNO (from))
7922 : {
7923 90 : if (x == from)
7924 90 : *loc = to;
7925 : else
7926 : {
7927 0 : gcc_checking_assert (REG_NREGS (x) == 1);
7928 0 : *loc = gen_rtx_REG (GET_MODE (x), REGNO (to));
7929 : }
7930 : }
7931 : }
7932 90 : return ret;
7933 90 : }
7934 :
7935 : /* Return mode for the memcpy/memset loop counter. Prefer SImode over
7936 : DImode for constant loop counts. */
7937 :
7938 : static machine_mode
7939 44852 : counter_mode (rtx count_exp)
7940 : {
7941 780 : if (GET_MODE (count_exp) != VOIDmode)
7942 32521 : return GET_MODE (count_exp);
7943 12331 : if (!CONST_INT_P (count_exp))
7944 0 : return Pmode;
7945 : if (TARGET_64BIT && (INTVAL (count_exp) & ~0xffffffff))
7946 : return DImode;
7947 : return SImode;
7948 : }
7949 :
7950 : /* When ISSETMEM is FALSE, output simple loop to move memory pointer to SRCPTR
7951 : to DESTPTR via chunks of MODE unrolled UNROLL times, overall size is COUNT
7952 : specified in bytes. When ISSETMEM is TRUE, output the equivalent loop to set
7953 : memory by VALUE (supposed to be in MODE).
7954 :
7955 : The size is rounded down to whole number of chunk size moved at once.
7956 : SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info. */
7957 :
7958 :
7959 : static void
7960 8233 : expand_set_or_cpymem_via_loop (rtx destmem, rtx srcmem,
7961 : rtx destptr, rtx srcptr, rtx value,
7962 : rtx count, machine_mode mode, int unroll,
7963 : int expected_size, bool issetmem)
7964 : {
7965 8233 : rtx_code_label *out_label = nullptr;
7966 8233 : rtx_code_label *top_label = nullptr;
7967 8233 : rtx iter, tmp;
7968 8233 : machine_mode iter_mode = counter_mode (count);
7969 8233 : int piece_size_n = GET_MODE_SIZE (mode) * unroll;
7970 8233 : rtx piece_size = GEN_INT (piece_size_n);
7971 16466 : rtx piece_size_mask = GEN_INT (~((GET_MODE_SIZE (mode) * unroll) - 1));
7972 8233 : rtx size;
7973 8233 : int i;
7974 8233 : int loop_count;
7975 :
7976 8233 : if (expected_size != -1 && CONST_INT_P (count))
7977 7225 : loop_count = INTVAL (count) / GET_MODE_SIZE (mode) / unroll;
7978 : else
7979 : loop_count = -1;
7980 :
7981 : /* Don't generate the loop if the loop count is 1. */
7982 7225 : if (loop_count != 1)
7983 : {
7984 8231 : top_label = gen_label_rtx ();
7985 8231 : out_label = gen_label_rtx ();
7986 : }
7987 8233 : iter = gen_reg_rtx (iter_mode);
7988 :
7989 8233 : size = expand_simple_binop (iter_mode, AND, count, piece_size_mask,
7990 : NULL, 1, OPTAB_DIRECT);
7991 : /* Those two should combine. */
7992 8233 : if (piece_size == const1_rtx)
7993 : {
7994 277 : emit_cmp_and_jump_insns (size, const0_rtx, EQ, NULL_RTX, iter_mode,
7995 : true, out_label);
7996 277 : predict_jump (REG_BR_PROB_BASE * 10 / 100);
7997 : }
7998 8233 : emit_move_insn (iter, const0_rtx);
7999 :
8000 8233 : if (loop_count != 1)
8001 8231 : emit_label (top_label);
8002 :
8003 10963 : tmp = convert_modes (Pmode, iter_mode, iter, true);
8004 :
8005 : /* This assert could be relaxed - in this case we'll need to compute
8006 : smallest power of two, containing in PIECE_SIZE_N and pass it to
8007 : offset_address. */
8008 8233 : gcc_assert ((piece_size_n & (piece_size_n - 1)) == 0);
8009 8233 : destmem = offset_address (destmem, tmp, piece_size_n);
8010 8233 : destmem = adjust_address (destmem, mode, 0);
8011 :
8012 8233 : if (!issetmem)
8013 : {
8014 2903 : srcmem = offset_address (srcmem, copy_rtx (tmp), piece_size_n);
8015 2903 : srcmem = adjust_address (srcmem, mode, 0);
8016 :
8017 : /* When unrolling for chips that reorder memory reads and writes,
8018 : we can save registers by using single temporary.
8019 : Also using 4 temporaries is overkill in 32bit mode. */
8020 2903 : if (!TARGET_64BIT && 0)
8021 : {
8022 : for (i = 0; i < unroll; i++)
8023 : {
8024 : if (i)
8025 : {
8026 : destmem = adjust_address (copy_rtx (destmem), mode,
8027 : GET_MODE_SIZE (mode));
8028 : srcmem = adjust_address (copy_rtx (srcmem), mode,
8029 : GET_MODE_SIZE (mode));
8030 : }
8031 : emit_move_insn (destmem, srcmem);
8032 : }
8033 : }
8034 : else
8035 : {
8036 2903 : rtx tmpreg[4];
8037 2903 : gcc_assert (unroll <= 4);
8038 13701 : for (i = 0; i < unroll; i++)
8039 : {
8040 10798 : tmpreg[i] = gen_reg_rtx (mode);
8041 10798 : if (i)
8042 15790 : srcmem = adjust_address (copy_rtx (srcmem), mode,
8043 : GET_MODE_SIZE (mode));
8044 10798 : emit_move_insn (tmpreg[i], srcmem);
8045 : }
8046 13701 : for (i = 0; i < unroll; i++)
8047 : {
8048 10798 : if (i)
8049 15790 : destmem = adjust_address (copy_rtx (destmem), mode,
8050 : GET_MODE_SIZE (mode));
8051 10798 : emit_move_insn (destmem, tmpreg[i]);
8052 : }
8053 : }
8054 : }
8055 : else
8056 26479 : for (i = 0; i < unroll; i++)
8057 : {
8058 21149 : if (i)
8059 31638 : destmem = adjust_address (copy_rtx (destmem), mode,
8060 : GET_MODE_SIZE (mode));
8061 21149 : emit_move_insn (destmem, value);
8062 : }
8063 :
8064 8233 : tmp = expand_simple_binop (iter_mode, PLUS, iter, piece_size, iter,
8065 : true, OPTAB_LIB_WIDEN);
8066 8233 : if (tmp != iter)
8067 0 : emit_move_insn (iter, tmp);
8068 :
8069 8233 : if (loop_count != 1)
8070 : {
8071 8231 : emit_cmp_and_jump_insns (iter, size, LT, NULL_RTX, iter_mode,
8072 : true, top_label);
8073 8231 : if (expected_size != -1)
8074 : {
8075 7339 : expected_size /= GET_MODE_SIZE (mode) * unroll;
8076 7339 : if (expected_size == 0)
8077 0 : predict_jump (0);
8078 7339 : else if (expected_size > REG_BR_PROB_BASE)
8079 2 : predict_jump (REG_BR_PROB_BASE - 1);
8080 : else
8081 7337 : predict_jump (REG_BR_PROB_BASE
8082 7337 : - (REG_BR_PROB_BASE + expected_size / 2)
8083 7337 : / expected_size);
8084 : }
8085 : else
8086 892 : predict_jump (REG_BR_PROB_BASE * 80 / 100);
8087 : }
8088 8233 : iter = ix86_zero_extend_to_Pmode (iter);
8089 10963 : tmp = expand_simple_binop (Pmode, PLUS, destptr, iter, destptr,
8090 : true, OPTAB_LIB_WIDEN);
8091 8233 : if (tmp != destptr)
8092 0 : emit_move_insn (destptr, tmp);
8093 8233 : if (!issetmem)
8094 : {
8095 4234 : tmp = expand_simple_binop (Pmode, PLUS, srcptr, iter, srcptr,
8096 : true, OPTAB_LIB_WIDEN);
8097 2903 : if (tmp != srcptr)
8098 0 : emit_move_insn (srcptr, tmp);
8099 : }
8100 8233 : if (loop_count != 1)
8101 8231 : emit_label (out_label);
8102 8233 : }
8103 :
8104 : /* Divide COUNTREG by SCALE. */
8105 : static rtx
8106 18645 : scale_counter (rtx countreg, int scale)
8107 : {
8108 18645 : rtx sc;
8109 :
8110 18645 : if (scale == 1)
8111 : return countreg;
8112 12876 : if (CONST_INT_P (countreg))
8113 12858 : return GEN_INT (INTVAL (countreg) / scale);
8114 18 : gcc_assert (REG_P (countreg));
8115 :
8116 54 : sc = expand_simple_binop (GET_MODE (countreg), LSHIFTRT, countreg,
8117 36 : GEN_INT (exact_log2 (scale)),
8118 : NULL, 1, OPTAB_DIRECT);
8119 18 : return sc;
8120 : }
8121 :
8122 : /* Output "rep; mov" or "rep; stos" instruction depending on ISSETMEM argument.
8123 : When ISSETMEM is true, arguments SRCMEM and SRCPTR are ignored.
8124 : When ISSETMEM is false, arguments VALUE and ORIG_VALUE are ignored.
8125 : For setmem case, VALUE is a promoted to a wider size ORIG_VALUE.
8126 : ORIG_VALUE is the original value passed to memset to fill the memory with.
8127 : Other arguments have same meaning as for previous function. */
8128 :
8129 : static void
8130 18645 : expand_set_or_cpymem_via_rep (rtx destmem, rtx srcmem,
8131 : rtx destptr, rtx srcptr, rtx value, rtx orig_value,
8132 : rtx count,
8133 : machine_mode mode, bool issetmem)
8134 : {
8135 18645 : rtx destexp;
8136 18645 : rtx srcexp;
8137 18645 : rtx countreg;
8138 18645 : HOST_WIDE_INT rounded_count;
8139 :
8140 : /* If possible, it is shorter to use rep movs.
8141 : TODO: Maybe it is better to move this logic to decide_alg. */
8142 18645 : if (mode == QImode && CONST_INT_P (count) && !(INTVAL (count) & 3)
8143 255 : && !TARGET_PREFER_KNOWN_REP_MOVSB_STOSB
8144 251 : && (!issetmem || orig_value == const0_rtx))
8145 18645 : mode = SImode;
8146 :
8147 18645 : if (destptr != XEXP (destmem, 0) || GET_MODE (destmem) != BLKmode)
8148 18361 : destmem = adjust_automodify_address_nv (destmem, BLKmode, destptr, 0);
8149 :
8150 37290 : countreg = ix86_zero_extend_to_Pmode (scale_counter (count,
8151 18645 : GET_MODE_SIZE (mode)));
8152 18645 : if (mode != QImode)
8153 : {
8154 38888 : destexp = gen_rtx_ASHIFT (Pmode, countreg,
8155 : GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
8156 13136 : destexp = gen_rtx_PLUS (Pmode, destexp, destptr);
8157 : }
8158 : else
8159 5788 : destexp = gen_rtx_PLUS (Pmode, destptr, countreg);
8160 18645 : if ((!issetmem || orig_value == const0_rtx) && CONST_INT_P (count))
8161 : {
8162 13546 : rounded_count
8163 13546 : = ROUND_DOWN (INTVAL (count), (HOST_WIDE_INT) GET_MODE_SIZE (mode));
8164 13546 : destmem = shallow_copy_rtx (destmem);
8165 13546 : set_mem_size (destmem, rounded_count);
8166 : }
8167 5107 : else if (MEM_SIZE_KNOWN_P (destmem))
8168 346 : clear_mem_size (destmem);
8169 :
8170 18645 : if (issetmem)
8171 : {
8172 7938 : value = force_reg (mode, gen_lowpart (mode, value));
8173 7938 : emit_insn (gen_rep_stos (destptr, countreg, destmem, value, destexp));
8174 : }
8175 : else
8176 : {
8177 10707 : if (srcptr != XEXP (srcmem, 0) || GET_MODE (srcmem) != BLKmode)
8178 10493 : srcmem = adjust_automodify_address_nv (srcmem, BLKmode, srcptr, 0);
8179 10707 : if (mode != QImode)
8180 : {
8181 18988 : srcexp = gen_rtx_ASHIFT (Pmode, countreg,
8182 : GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
8183 6452 : srcexp = gen_rtx_PLUS (Pmode, srcexp, srcptr);
8184 : }
8185 : else
8186 4454 : srcexp = gen_rtx_PLUS (Pmode, srcptr, countreg);
8187 10707 : if (CONST_INT_P (count))
8188 : {
8189 6763 : rounded_count
8190 6763 : = ROUND_DOWN (INTVAL (count), (HOST_WIDE_INT) GET_MODE_SIZE (mode));
8191 6763 : srcmem = shallow_copy_rtx (srcmem);
8192 6763 : set_mem_size (srcmem, rounded_count);
8193 : }
8194 : else
8195 : {
8196 3959 : if (MEM_SIZE_KNOWN_P (srcmem))
8197 0 : clear_mem_size (srcmem);
8198 : }
8199 10707 : emit_insn (gen_rep_mov (destptr, destmem, srcptr, srcmem, countreg,
8200 : destexp, srcexp));
8201 : }
8202 18645 : }
8203 :
8204 : /* This function emits moves to copy SIZE_TO_MOVE bytes from SRCMEM to
8205 : DESTMEM.
8206 : SRC is passed by pointer to be updated on return.
8207 : Return value is updated DST. */
8208 : static rtx
8209 13 : emit_memmov (rtx destmem, rtx *srcmem, rtx destptr, rtx srcptr,
8210 : HOST_WIDE_INT size_to_move)
8211 : {
8212 13 : rtx dst = destmem, src = *srcmem, tempreg;
8213 13 : enum insn_code code;
8214 13 : machine_mode move_mode;
8215 13 : int piece_size, i;
8216 :
8217 : /* Find the widest mode in which we could perform moves.
8218 : Start with the biggest power of 2 less than SIZE_TO_MOVE and half
8219 : it until move of such size is supported. */
8220 13 : piece_size = 1 << floor_log2 (size_to_move);
8221 26 : while (!int_mode_for_size (piece_size * BITS_PER_UNIT, 0).exists (&move_mode)
8222 26 : || (code = optab_handler (mov_optab, move_mode)) == CODE_FOR_nothing)
8223 : {
8224 0 : gcc_assert (piece_size > 1);
8225 0 : piece_size >>= 1;
8226 : }
8227 :
8228 : /* Find the corresponding vector mode with the same size as MOVE_MODE.
8229 : MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.). */
8230 39 : if (GET_MODE_SIZE (move_mode) > GET_MODE_SIZE (word_mode))
8231 : {
8232 0 : int nunits = GET_MODE_SIZE (move_mode) / GET_MODE_SIZE (word_mode);
8233 0 : if (!mode_for_vector (word_mode, nunits).exists (&move_mode)
8234 0 : || (code = optab_handler (mov_optab, move_mode)) == CODE_FOR_nothing)
8235 : {
8236 0 : move_mode = word_mode;
8237 0 : piece_size = GET_MODE_SIZE (move_mode);
8238 0 : code = optab_handler (mov_optab, move_mode);
8239 : }
8240 : }
8241 13 : gcc_assert (code != CODE_FOR_nothing);
8242 :
8243 13 : dst = adjust_automodify_address_nv (dst, move_mode, destptr, 0);
8244 13 : src = adjust_automodify_address_nv (src, move_mode, srcptr, 0);
8245 :
8246 : /* Emit moves. We'll need SIZE_TO_MOVE/PIECE_SIZES moves. */
8247 13 : gcc_assert (size_to_move % piece_size == 0);
8248 :
8249 26 : for (i = 0; i < size_to_move; i += piece_size)
8250 : {
8251 : /* We move from memory to memory, so we'll need to do it via
8252 : a temporary register. */
8253 13 : tempreg = gen_reg_rtx (move_mode);
8254 13 : emit_insn (GEN_FCN (code) (tempreg, src));
8255 13 : emit_insn (GEN_FCN (code) (dst, tempreg));
8256 :
8257 26 : emit_move_insn (destptr,
8258 13 : plus_constant (Pmode, copy_rtx (destptr), piece_size));
8259 26 : emit_move_insn (srcptr,
8260 13 : plus_constant (Pmode, copy_rtx (srcptr), piece_size));
8261 :
8262 13 : dst = adjust_automodify_address_nv (dst, move_mode, destptr,
8263 : piece_size);
8264 13 : src = adjust_automodify_address_nv (src, move_mode, srcptr,
8265 : piece_size);
8266 : }
8267 :
8268 : /* Update DST and SRC rtx. */
8269 13 : *srcmem = src;
8270 13 : return dst;
8271 : }
8272 :
8273 : /* Helper function for the string operations below. Dest VARIABLE whether
8274 : it is aligned to VALUE bytes. If true, jump to the label. */
8275 :
8276 : static rtx_code_label *
8277 3341 : ix86_expand_aligntest (rtx variable, int value, bool epilogue)
8278 : {
8279 3341 : rtx_code_label *label = gen_label_rtx ();
8280 3341 : rtx tmpcount = gen_reg_rtx (GET_MODE (variable));
8281 3341 : if (GET_MODE (variable) == DImode)
8282 746 : emit_insn (gen_anddi3 (tmpcount, variable, GEN_INT (value)));
8283 : else
8284 2595 : emit_insn (gen_andsi3 (tmpcount, variable, GEN_INT (value)));
8285 3341 : emit_cmp_and_jump_insns (tmpcount, const0_rtx, EQ, 0, GET_MODE (variable),
8286 : 1, label);
8287 3341 : if (epilogue)
8288 0 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
8289 : else
8290 3341 : predict_jump (REG_BR_PROB_BASE * 90 / 100);
8291 3341 : return label;
8292 : }
8293 :
8294 :
8295 : /* Output code to copy at most count & (max_size - 1) bytes from SRC to DEST. */
8296 :
8297 : static void
8298 6839 : expand_cpymem_epilogue (rtx destmem, rtx srcmem,
8299 : rtx destptr, rtx srcptr, rtx count, int max_size)
8300 : {
8301 6839 : rtx src, dest;
8302 6839 : if (CONST_INT_P (count))
8303 : {
8304 6774 : unsigned HOST_WIDE_INT countval = UINTVAL (count);
8305 6774 : unsigned HOST_WIDE_INT epilogue_size = countval % max_size;
8306 6774 : unsigned int destalign = MEM_ALIGN (destmem);
8307 6774 : cfun->machine->by_pieces_in_use = true;
8308 6774 : move_by_pieces (destmem, srcmem, epilogue_size, destalign,
8309 : RETURN_BEGIN);
8310 6774 : cfun->machine->by_pieces_in_use = false;
8311 6774 : return;
8312 : }
8313 65 : if (max_size > 8)
8314 : {
8315 65 : count = expand_simple_binop (GET_MODE (count), AND, count, GEN_INT (max_size - 1),
8316 : count, 1, OPTAB_DIRECT);
8317 65 : expand_set_or_cpymem_via_loop (destmem, srcmem, destptr, srcptr, NULL,
8318 : count, QImode, 1, 4, false);
8319 65 : return;
8320 : }
8321 :
8322 : /* When there are stringops, we can cheaply increase dest and src pointers.
8323 : Otherwise we save code size by maintaining offset (zero is readily
8324 : available from preceding rep operation) and using x86 addressing modes.
8325 : */
8326 0 : if (TARGET_SINGLE_STRINGOP)
8327 : {
8328 0 : if (max_size > 4)
8329 : {
8330 0 : rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
8331 0 : src = change_address (srcmem, SImode, srcptr);
8332 0 : dest = change_address (destmem, SImode, destptr);
8333 0 : emit_insn (gen_strmov (destptr, dest, srcptr, src));
8334 0 : emit_label (label);
8335 0 : LABEL_NUSES (label) = 1;
8336 : }
8337 0 : if (max_size > 2)
8338 : {
8339 0 : rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
8340 0 : src = change_address (srcmem, HImode, srcptr);
8341 0 : dest = change_address (destmem, HImode, destptr);
8342 0 : emit_insn (gen_strmov (destptr, dest, srcptr, src));
8343 0 : emit_label (label);
8344 0 : LABEL_NUSES (label) = 1;
8345 : }
8346 0 : if (max_size > 1)
8347 : {
8348 0 : rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
8349 0 : src = change_address (srcmem, QImode, srcptr);
8350 0 : dest = change_address (destmem, QImode, destptr);
8351 0 : emit_insn (gen_strmov (destptr, dest, srcptr, src));
8352 0 : emit_label (label);
8353 0 : LABEL_NUSES (label) = 1;
8354 : }
8355 : }
8356 : else
8357 : {
8358 0 : rtx offset = force_reg (Pmode, const0_rtx);
8359 0 : rtx tmp;
8360 :
8361 0 : if (max_size > 4)
8362 : {
8363 0 : rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
8364 0 : src = change_address (srcmem, SImode, srcptr);
8365 0 : dest = change_address (destmem, SImode, destptr);
8366 0 : emit_move_insn (dest, src);
8367 0 : tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (4), NULL,
8368 : true, OPTAB_LIB_WIDEN);
8369 0 : if (tmp != offset)
8370 0 : emit_move_insn (offset, tmp);
8371 0 : emit_label (label);
8372 0 : LABEL_NUSES (label) = 1;
8373 : }
8374 0 : if (max_size > 2)
8375 : {
8376 0 : rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
8377 0 : tmp = gen_rtx_PLUS (Pmode, srcptr, offset);
8378 0 : src = change_address (srcmem, HImode, tmp);
8379 0 : tmp = gen_rtx_PLUS (Pmode, destptr, offset);
8380 0 : dest = change_address (destmem, HImode, tmp);
8381 0 : emit_move_insn (dest, src);
8382 0 : tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (2), tmp,
8383 : true, OPTAB_LIB_WIDEN);
8384 0 : if (tmp != offset)
8385 0 : emit_move_insn (offset, tmp);
8386 0 : emit_label (label);
8387 0 : LABEL_NUSES (label) = 1;
8388 : }
8389 0 : if (max_size > 1)
8390 : {
8391 0 : rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
8392 0 : tmp = gen_rtx_PLUS (Pmode, srcptr, offset);
8393 0 : src = change_address (srcmem, QImode, tmp);
8394 0 : tmp = gen_rtx_PLUS (Pmode, destptr, offset);
8395 0 : dest = change_address (destmem, QImode, tmp);
8396 0 : emit_move_insn (dest, src);
8397 0 : emit_label (label);
8398 0 : LABEL_NUSES (label) = 1;
8399 : }
8400 : }
8401 : }
8402 :
8403 : /* This function emits moves to fill SIZE_TO_MOVE bytes starting from DESTMEM
8404 : with value PROMOTED_VAL.
8405 : SRC is passed by pointer to be updated on return.
8406 : Return value is updated DST. */
8407 : static rtx
8408 6 : emit_memset (rtx destmem, rtx destptr, rtx promoted_val,
8409 : HOST_WIDE_INT size_to_move)
8410 : {
8411 6 : rtx dst = destmem;
8412 6 : enum insn_code code;
8413 6 : machine_mode move_mode;
8414 6 : int piece_size, i;
8415 :
8416 : /* Find the widest mode in which we could perform moves.
8417 : Start with the biggest power of 2 less than SIZE_TO_MOVE and half
8418 : it until move of such size is supported. */
8419 6 : move_mode = GET_MODE (promoted_val);
8420 6 : if (move_mode == VOIDmode)
8421 0 : move_mode = QImode;
8422 12 : if (size_to_move < GET_MODE_SIZE (move_mode))
8423 : {
8424 5 : unsigned int move_bits = size_to_move * BITS_PER_UNIT;
8425 5 : move_mode = int_mode_for_size (move_bits, 0).require ();
8426 5 : promoted_val = gen_lowpart (move_mode, promoted_val);
8427 : }
8428 6 : piece_size = GET_MODE_SIZE (move_mode);
8429 6 : code = optab_handler (mov_optab, move_mode);
8430 6 : gcc_assert (code != CODE_FOR_nothing && promoted_val != NULL_RTX);
8431 :
8432 6 : dst = adjust_automodify_address_nv (dst, move_mode, destptr, 0);
8433 :
8434 : /* Emit moves. We'll need SIZE_TO_MOVE/PIECE_SIZES moves. */
8435 6 : gcc_assert (size_to_move % piece_size == 0);
8436 :
8437 12 : for (i = 0; i < size_to_move; i += piece_size)
8438 : {
8439 12 : if (piece_size <= GET_MODE_SIZE (word_mode))
8440 : {
8441 4 : emit_insn (gen_strset (destptr, dst, promoted_val));
8442 4 : dst = adjust_automodify_address_nv (dst, move_mode, destptr,
8443 : piece_size);
8444 4 : continue;
8445 : }
8446 :
8447 2 : emit_insn (GEN_FCN (code) (dst, promoted_val));
8448 :
8449 4 : emit_move_insn (destptr,
8450 2 : plus_constant (Pmode, copy_rtx (destptr), piece_size));
8451 :
8452 2 : dst = adjust_automodify_address_nv (dst, move_mode, destptr,
8453 : piece_size);
8454 : }
8455 :
8456 : /* Update DST rtx. */
8457 6 : return dst;
8458 : }
8459 : /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
8460 : static void
8461 39 : expand_setmem_epilogue_via_loop (rtx destmem, rtx destptr, rtx value,
8462 : rtx count, int max_size)
8463 : {
8464 78 : count = expand_simple_binop (counter_mode (count), AND, count,
8465 39 : GEN_INT (max_size - 1), count, 1, OPTAB_DIRECT);
8466 39 : expand_set_or_cpymem_via_loop (destmem, NULL, destptr, NULL,
8467 39 : gen_lowpart (QImode, value), count, QImode,
8468 : 1, max_size / 2, true);
8469 39 : }
8470 :
8471 : /* Callback routine for store_by_pieces. Return the RTL of a register
8472 : containing GET_MODE_SIZE (MODE) bytes in the RTL register op_p which
8473 : is an integer or a word vector register. If PREV_P isn't nullptr,
8474 : it has the RTL info from the previous iteration. */
8475 :
8476 : static rtx
8477 5048 : setmem_epilogue_gen_val (void *op_p, void *prev_p, HOST_WIDE_INT,
8478 : fixed_size_mode mode)
8479 : {
8480 5048 : rtx target;
8481 5048 : by_pieces_prev *prev = (by_pieces_prev *) prev_p;
8482 5048 : if (prev)
8483 : {
8484 5048 : rtx prev_op = prev->data;
8485 5048 : if (prev_op)
8486 : {
8487 2924 : machine_mode prev_mode = GET_MODE (prev_op);
8488 2924 : if (prev_mode == mode)
8489 : return prev_op;
8490 54 : if (VECTOR_MODE_P (prev_mode)
8491 1098 : && VECTOR_MODE_P (mode)
8492 1152 : && GET_MODE_INNER (prev_mode) == GET_MODE_INNER (mode))
8493 : {
8494 0 : target = gen_rtx_SUBREG (mode, prev_op, 0);
8495 0 : return target;
8496 : }
8497 : }
8498 : }
8499 :
8500 3276 : rtx op = (rtx) op_p;
8501 3276 : machine_mode op_mode = GET_MODE (op);
8502 :
8503 3276 : if (VECTOR_MODE_P (mode))
8504 : {
8505 3712 : gcc_assert (GET_MODE_INNER (mode) == QImode);
8506 :
8507 1856 : unsigned int op_size = GET_MODE_SIZE (op_mode);
8508 1856 : unsigned int size = GET_MODE_SIZE (mode);
8509 1856 : unsigned int nunits;
8510 1856 : machine_mode vec_mode;
8511 1856 : if (op_size < size)
8512 : {
8513 : /* If OP size is smaller than MODE size, duplicate it. */
8514 1 : nunits = size / GET_MODE_SIZE (QImode);
8515 1 : vec_mode = mode_for_vector (QImode, nunits).require ();
8516 1 : nunits = size / op_size;
8517 1 : gcc_assert (SCALAR_INT_MODE_P (op_mode));
8518 1 : machine_mode dup_mode
8519 1 : = mode_for_vector (as_a <scalar_mode> (op_mode),
8520 2 : nunits).require ();
8521 1 : target = gen_reg_rtx (vec_mode);
8522 1 : op = gen_vec_duplicate (dup_mode, op);
8523 1 : rtx dup_op = gen_reg_rtx (dup_mode);
8524 1 : emit_move_insn (dup_op, op);
8525 1 : op = gen_rtx_SUBREG (vec_mode, dup_op, 0);
8526 1 : emit_move_insn (target, op);
8527 1 : return target;
8528 : }
8529 1855 : nunits = op_size / GET_MODE_SIZE (QImode);
8530 1855 : vec_mode = mode_for_vector (QImode, nunits).require ();
8531 1855 : target = gen_reg_rtx (vec_mode);
8532 1855 : op = gen_rtx_SUBREG (vec_mode, op, 0);
8533 1855 : emit_move_insn (target, op);
8534 1855 : if (op_size == size)
8535 : return target;
8536 :
8537 0 : rtx tmp = gen_reg_rtx (mode);
8538 0 : target = gen_rtx_SUBREG (mode, target, 0);
8539 0 : emit_move_insn (tmp, target);
8540 0 : return tmp;
8541 : }
8542 :
8543 1420 : if (VECTOR_MODE_P (op_mode))
8544 : {
8545 2830 : gcc_assert (GET_MODE_INNER (op_mode) == word_mode);
8546 1415 : target = gen_reg_rtx (word_mode);
8547 1415 : op = gen_rtx_SUBREG (word_mode, op, 0);
8548 1415 : emit_move_insn (target, op);
8549 : }
8550 : else
8551 : target = op;
8552 :
8553 1420 : if (mode == GET_MODE (target))
8554 : return target;
8555 :
8556 245 : rtx tmp = gen_reg_rtx (mode);
8557 245 : target = gen_rtx_SUBREG (mode, target, 0);
8558 245 : emit_move_insn (tmp, target);
8559 245 : return tmp;
8560 : }
8561 :
8562 : /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
8563 : static void
8564 9997 : expand_setmem_epilogue (rtx destmem, rtx destptr, rtx value, rtx vec_value,
8565 : rtx count, int max_size)
8566 : {
8567 9997 : rtx dest;
8568 :
8569 9997 : if (CONST_INT_P (count))
8570 : {
8571 9958 : unsigned HOST_WIDE_INT countval = UINTVAL (count);
8572 9958 : unsigned HOST_WIDE_INT epilogue_size = countval % max_size;
8573 9958 : unsigned int destalign = MEM_ALIGN (destmem);
8574 9958 : cfun->machine->by_pieces_in_use = true;
8575 16509 : store_by_pieces (destmem, epilogue_size, setmem_epilogue_gen_val,
8576 : vec_value ? vec_value : value, destalign, true,
8577 : RETURN_BEGIN);
8578 9958 : cfun->machine->by_pieces_in_use = false;
8579 9958 : return;
8580 : }
8581 39 : if (max_size > 32)
8582 : {
8583 39 : expand_setmem_epilogue_via_loop (destmem, destptr, value, count, max_size);
8584 39 : return;
8585 : }
8586 0 : if (max_size > 16)
8587 : {
8588 0 : rtx_code_label *label = ix86_expand_aligntest (count, 16, true);
8589 0 : if (TARGET_64BIT)
8590 : {
8591 0 : dest = change_address (destmem, DImode, destptr);
8592 0 : emit_insn (gen_strset (destptr, dest, value));
8593 0 : dest = adjust_automodify_address_nv (dest, DImode, destptr, 8);
8594 0 : emit_insn (gen_strset (destptr, dest, value));
8595 : }
8596 : else
8597 : {
8598 0 : dest = change_address (destmem, SImode, destptr);
8599 0 : emit_insn (gen_strset (destptr, dest, value));
8600 0 : dest = adjust_automodify_address_nv (dest, SImode, destptr, 4);
8601 0 : emit_insn (gen_strset (destptr, dest, value));
8602 0 : dest = adjust_automodify_address_nv (dest, SImode, destptr, 8);
8603 0 : emit_insn (gen_strset (destptr, dest, value));
8604 0 : dest = adjust_automodify_address_nv (dest, SImode, destptr, 12);
8605 0 : emit_insn (gen_strset (destptr, dest, value));
8606 : }
8607 0 : emit_label (label);
8608 0 : LABEL_NUSES (label) = 1;
8609 : }
8610 0 : if (max_size > 8)
8611 : {
8612 0 : rtx_code_label *label = ix86_expand_aligntest (count, 8, true);
8613 0 : if (TARGET_64BIT)
8614 : {
8615 0 : dest = change_address (destmem, DImode, destptr);
8616 0 : emit_insn (gen_strset (destptr, dest, value));
8617 : }
8618 : else
8619 : {
8620 0 : dest = change_address (destmem, SImode, destptr);
8621 0 : emit_insn (gen_strset (destptr, dest, value));
8622 0 : dest = adjust_automodify_address_nv (dest, SImode, destptr, 4);
8623 0 : emit_insn (gen_strset (destptr, dest, value));
8624 : }
8625 0 : emit_label (label);
8626 0 : LABEL_NUSES (label) = 1;
8627 : }
8628 0 : if (max_size > 4)
8629 : {
8630 0 : rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
8631 0 : dest = change_address (destmem, SImode, destptr);
8632 0 : emit_insn (gen_strset (destptr, dest, gen_lowpart (SImode, value)));
8633 0 : emit_label (label);
8634 0 : LABEL_NUSES (label) = 1;
8635 : }
8636 0 : if (max_size > 2)
8637 : {
8638 0 : rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
8639 0 : dest = change_address (destmem, HImode, destptr);
8640 0 : emit_insn (gen_strset (destptr, dest, gen_lowpart (HImode, value)));
8641 0 : emit_label (label);
8642 0 : LABEL_NUSES (label) = 1;
8643 : }
8644 0 : if (max_size > 1)
8645 : {
8646 0 : rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
8647 0 : dest = change_address (destmem, QImode, destptr);
8648 0 : emit_insn (gen_strset (destptr, dest, gen_lowpart (QImode, value)));
8649 0 : emit_label (label);
8650 0 : LABEL_NUSES (label) = 1;
8651 : }
8652 : }
8653 :
8654 : /* Adjust COUNTER by the VALUE. */
8655 : static void
8656 19 : ix86_adjust_counter (rtx countreg, HOST_WIDE_INT value)
8657 : {
8658 19 : emit_insn (gen_add2_insn (countreg, GEN_INT (-value)));
8659 19 : }
8660 :
8661 : /* Depending on ISSETMEM, copy enough from SRCMEM to DESTMEM or set enough to
8662 : DESTMEM to align it to DESIRED_ALIGNMENT. Original alignment is ALIGN.
8663 : Depending on ISSETMEM, either arguments SRCMEM/SRCPTR or VALUE/VEC_VALUE are
8664 : ignored.
8665 : Return value is updated DESTMEM. */
8666 :
8667 : static rtx
8668 7 : expand_set_or_cpymem_prologue (rtx destmem, rtx srcmem,
8669 : rtx destptr, rtx srcptr, rtx value,
8670 : rtx vec_value, rtx count, int align,
8671 : int desired_alignment, bool issetmem)
8672 : {
8673 7 : int i;
8674 35 : for (i = 1; i < desired_alignment; i <<= 1)
8675 : {
8676 28 : if (align <= i)
8677 : {
8678 19 : rtx_code_label *label = ix86_expand_aligntest (destptr, i, false);
8679 19 : if (issetmem)
8680 : {
8681 12 : if (vec_value && i > GET_MODE_SIZE (GET_MODE (value)))
8682 2 : destmem = emit_memset (destmem, destptr, vec_value, i);
8683 : else
8684 4 : destmem = emit_memset (destmem, destptr, value, i);
8685 : }
8686 : else
8687 13 : destmem = emit_memmov (destmem, &srcmem, destptr, srcptr, i);
8688 19 : ix86_adjust_counter (count, i);
8689 19 : emit_label (label);
8690 19 : LABEL_NUSES (label) = 1;
8691 19 : set_mem_align (destmem, i * 2 * BITS_PER_UNIT);
8692 : }
8693 : }
8694 7 : return destmem;
8695 : }
8696 :
8697 : /* Test if COUNT&SIZE is nonzero and if so, expand movme
8698 : or setmem sequence that is valid for SIZE..2*SIZE-1 bytes
8699 : and jump to DONE_LABEL. */
8700 : static void
8701 2654 : expand_small_cpymem_or_setmem (rtx destmem, rtx srcmem,
8702 : rtx destptr, rtx srcptr,
8703 : rtx value, rtx vec_value,
8704 : rtx count, int size,
8705 : rtx done_label, bool issetmem)
8706 : {
8707 2654 : rtx_code_label *label = ix86_expand_aligntest (count, size, false);
8708 2654 : machine_mode mode = int_mode_for_size (size * BITS_PER_UNIT, 1).else_blk ();
8709 2654 : rtx modesize;
8710 2654 : rtx scalar_value = value;
8711 2654 : int n;
8712 :
8713 : /* If we do not have vector value to copy, we must reduce size. */
8714 2654 : if (issetmem)
8715 : {
8716 881 : if (!vec_value)
8717 : {
8718 4 : if (GET_MODE (value) == VOIDmode && size > 8)
8719 0 : mode = Pmode;
8720 12 : else if (GET_MODE_SIZE (mode) > GET_MODE_SIZE (GET_MODE (value)))
8721 0 : mode = GET_MODE (value);
8722 : }
8723 : else
8724 877 : mode = GET_MODE (vec_value), value = vec_value;
8725 : }
8726 : else
8727 : {
8728 : /* Choose appropriate vector mode. */
8729 1773 : if (size >= 32)
8730 442 : switch (MOVE_MAX)
8731 : {
8732 0 : case 64:
8733 0 : if (size >= 64)
8734 : {
8735 : mode = V64QImode;
8736 : break;
8737 : }
8738 : /* FALLTHRU */
8739 0 : case 32:
8740 0 : mode = V32QImode;
8741 0 : break;
8742 : case 16:
8743 : mode = V16QImode;
8744 : break;
8745 : case 8:
8746 : mode = DImode;
8747 : break;
8748 0 : default:
8749 0 : gcc_unreachable ();
8750 : }
8751 1331 : else if (size >= 16)
8752 442 : mode = TARGET_SSE ? V16QImode : DImode;
8753 1773 : srcmem = change_address (srcmem, mode, srcptr);
8754 : }
8755 3531 : if (issetmem && vec_value && GET_MODE_SIZE (mode) > size)
8756 : {
8757 : /* For memset with vector and the size is smaller than the vector
8758 : size, first try the narrower vector, otherwise, use the
8759 : original value. */
8760 443 : machine_mode inner_mode = GET_MODE_INNER (mode);
8761 443 : unsigned int nunits = size / GET_MODE_SIZE (inner_mode);
8762 443 : if (nunits > 1)
8763 : {
8764 320 : mode = mode_for_vector (GET_MODE_INNER (mode),
8765 320 : nunits).require ();
8766 160 : value = gen_rtx_SUBREG (mode, value, 0);
8767 : }
8768 : else
8769 : {
8770 283 : scalar_int_mode smode
8771 283 : = smallest_int_mode_for_size (size * BITS_PER_UNIT).require ();
8772 849 : gcc_assert (GET_MODE_SIZE (GET_MODE (scalar_value))
8773 : >= GET_MODE_SIZE (smode));
8774 283 : mode = smode;
8775 283 : if (GET_MODE (scalar_value) == mode)
8776 : value = scalar_value;
8777 : else
8778 66 : value = gen_rtx_SUBREG (mode, scalar_value, 0);
8779 : }
8780 : }
8781 2654 : destmem = change_address (destmem, mode, destptr);
8782 5308 : modesize = GEN_INT (GET_MODE_SIZE (mode));
8783 5308 : gcc_assert (GET_MODE_SIZE (mode) <= size);
8784 11934 : for (n = 0; n * GET_MODE_SIZE (mode) < size; n++)
8785 : {
8786 3313 : if (issetmem)
8787 1098 : emit_move_insn (destmem, gen_lowpart (mode, value));
8788 : else
8789 : {
8790 2215 : emit_move_insn (destmem, srcmem);
8791 4430 : srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
8792 : }
8793 6626 : destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
8794 : }
8795 :
8796 2654 : destmem = offset_address (destmem, count, 1);
8797 5308 : destmem = offset_address (destmem, GEN_INT (-2 * size),
8798 2654 : GET_MODE_SIZE (mode));
8799 2654 : if (!issetmem)
8800 : {
8801 1773 : srcmem = offset_address (srcmem, count, 1);
8802 3546 : srcmem = offset_address (srcmem, GEN_INT (-2 * size),
8803 1773 : GET_MODE_SIZE (mode));
8804 : }
8805 11934 : for (n = 0; n * GET_MODE_SIZE (mode) < size; n++)
8806 : {
8807 3313 : if (issetmem)
8808 1098 : emit_move_insn (destmem, gen_lowpart (mode, value));
8809 : else
8810 : {
8811 2215 : emit_move_insn (destmem, srcmem);
8812 4430 : srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
8813 : }
8814 6626 : destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
8815 : }
8816 2654 : emit_jump_insn (gen_jump (done_label));
8817 2654 : emit_barrier ();
8818 :
8819 2654 : emit_label (label);
8820 2654 : LABEL_NUSES (label) = 1;
8821 2654 : }
8822 :
8823 : /* Handle small memcpy (up to SIZE that is supposed to be small power of 2.
8824 : and get ready for the main memcpy loop by copying initial DESIRED_ALIGN-ALIGN
8825 : bytes and last SIZE bytes adjusitng DESTPTR/SRCPTR/COUNT in a way we can
8826 : proceed with an loop copying SIZE bytes at once. Do moves in MODE.
8827 : DONE_LABEL is a label after the whole copying sequence. The label is created
8828 : on demand if *DONE_LABEL is NULL.
8829 : MIN_SIZE is minimal size of block copied. This value gets adjusted for new
8830 : bounds after the initial copies.
8831 :
8832 : DESTMEM/SRCMEM are memory expressions pointing to the copies block,
8833 : DESTPTR/SRCPTR are pointers to the block. DYNAMIC_CHECK indicate whether
8834 : we will dispatch to a library call for large blocks.
8835 :
8836 : In pseudocode we do:
8837 :
8838 : if (COUNT < SIZE)
8839 : {
8840 : Assume that SIZE is 4. Bigger sizes are handled analogously
8841 : if (COUNT & 4)
8842 : {
8843 : copy 4 bytes from SRCPTR to DESTPTR
8844 : copy 4 bytes from SRCPTR + COUNT - 4 to DESTPTR + COUNT - 4
8845 : goto done_label
8846 : }
8847 : if (!COUNT)
8848 : goto done_label;
8849 : copy 1 byte from SRCPTR to DESTPTR
8850 : if (COUNT & 2)
8851 : {
8852 : copy 2 bytes from SRCPTR to DESTPTR
8853 : copy 2 bytes from SRCPTR + COUNT - 2 to DESTPTR + COUNT - 2
8854 : }
8855 : }
8856 : else
8857 : {
8858 : copy at least DESIRED_ALIGN-ALIGN bytes from SRCPTR to DESTPTR
8859 : copy SIZE bytes from SRCPTR + COUNT - SIZE to DESTPTR + COUNT -SIZE
8860 :
8861 : OLD_DESPTR = DESTPTR;
8862 : Align DESTPTR up to DESIRED_ALIGN
8863 : SRCPTR += DESTPTR - OLD_DESTPTR
8864 : COUNT -= DEST_PTR - OLD_DESTPTR
8865 : if (DYNAMIC_CHECK)
8866 : Round COUNT down to multiple of SIZE
8867 : << optional caller supplied zero size guard is here >>
8868 : << optional caller supplied dynamic check is here >>
8869 : << caller supplied main copy loop is here >>
8870 : }
8871 : done_label:
8872 : */
8873 : static void
8874 4040 : expand_set_or_cpymem_prologue_epilogue_by_misaligned_moves (rtx destmem, rtx srcmem,
8875 : rtx *destptr, rtx *srcptr,
8876 : machine_mode mode,
8877 : rtx value, rtx vec_value,
8878 : rtx *count,
8879 : rtx_code_label **done_label,
8880 : int size,
8881 : int desired_align,
8882 : int align,
8883 : unsigned HOST_WIDE_INT *min_size,
8884 : bool dynamic_check,
8885 : bool issetmem)
8886 : {
8887 4040 : rtx_code_label *loop_label = NULL, *label;
8888 4040 : int n;
8889 4040 : rtx modesize;
8890 4040 : int prolog_size = 0;
8891 4040 : rtx mode_value;
8892 :
8893 : /* Chose proper value to copy. */
8894 4040 : if (issetmem && VECTOR_MODE_P (mode))
8895 : mode_value = vec_value;
8896 : else
8897 4040 : mode_value = value;
8898 8080 : gcc_assert (GET_MODE_SIZE (mode) <= size);
8899 :
8900 : /* See if block is big or small, handle small blocks. */
8901 4040 : if (!CONST_INT_P (*count) && *min_size < (unsigned HOST_WIDE_INT)size)
8902 : {
8903 668 : int size2 = size;
8904 668 : loop_label = gen_label_rtx ();
8905 :
8906 668 : if (!*done_label)
8907 668 : *done_label = gen_label_rtx ();
8908 :
8909 668 : emit_cmp_and_jump_insns (*count, GEN_INT (size2), GE, 0, GET_MODE (*count),
8910 : 1, loop_label);
8911 668 : size2 >>= 1;
8912 :
8913 : /* Handle sizes > 3. */
8914 3322 : for (;size2 > 2; size2 >>= 1)
8915 2654 : expand_small_cpymem_or_setmem (destmem, srcmem,
8916 : *destptr, *srcptr,
8917 : value, vec_value,
8918 : *count,
8919 : size2, *done_label, issetmem);
8920 : /* Nothing to copy? Jump to DONE_LABEL if so */
8921 668 : emit_cmp_and_jump_insns (*count, const0_rtx, EQ, 0, GET_MODE (*count),
8922 : 1, *done_label);
8923 :
8924 : /* Do a byte copy. */
8925 668 : destmem = change_address (destmem, QImode, *destptr);
8926 668 : if (issetmem)
8927 221 : emit_move_insn (destmem, gen_lowpart (QImode, value));
8928 : else
8929 : {
8930 447 : srcmem = change_address (srcmem, QImode, *srcptr);
8931 447 : emit_move_insn (destmem, srcmem);
8932 : }
8933 :
8934 : /* Handle sizes 2 and 3. */
8935 668 : label = ix86_expand_aligntest (*count, 2, false);
8936 668 : destmem = change_address (destmem, HImode, *destptr);
8937 668 : destmem = offset_address (destmem, *count, 1);
8938 668 : destmem = offset_address (destmem, GEN_INT (-2), 2);
8939 668 : if (issetmem)
8940 221 : emit_move_insn (destmem, gen_lowpart (HImode, value));
8941 : else
8942 : {
8943 447 : srcmem = change_address (srcmem, HImode, *srcptr);
8944 447 : srcmem = offset_address (srcmem, *count, 1);
8945 447 : srcmem = offset_address (srcmem, GEN_INT (-2), 2);
8946 447 : emit_move_insn (destmem, srcmem);
8947 : }
8948 :
8949 668 : emit_label (label);
8950 668 : LABEL_NUSES (label) = 1;
8951 668 : emit_jump_insn (gen_jump (*done_label));
8952 668 : emit_barrier ();
8953 : }
8954 : else
8955 3372 : gcc_assert (*min_size >= (unsigned HOST_WIDE_INT)size
8956 : || UINTVAL (*count) >= (unsigned HOST_WIDE_INT)size);
8957 :
8958 : /* Start memcpy for COUNT >= SIZE. */
8959 668 : if (loop_label)
8960 : {
8961 668 : emit_label (loop_label);
8962 668 : LABEL_NUSES (loop_label) = 1;
8963 : }
8964 :
8965 : /* Copy first desired_align bytes. */
8966 4040 : if (!issetmem)
8967 2102 : srcmem = change_address (srcmem, mode, *srcptr);
8968 4040 : destmem = change_address (destmem, mode, *destptr);
8969 4040 : modesize = GEN_INT (GET_MODE_SIZE (mode));
8970 8097 : for (n = 0; prolog_size < desired_align - align; n++)
8971 : {
8972 17 : if (issetmem)
8973 1 : emit_move_insn (destmem, mode_value);
8974 : else
8975 : {
8976 16 : emit_move_insn (destmem, srcmem);
8977 32 : srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
8978 : }
8979 34 : destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
8980 34 : prolog_size += GET_MODE_SIZE (mode);
8981 : }
8982 :
8983 :
8984 : /* Copy last SIZE bytes. */
8985 4040 : destmem = offset_address (destmem, *count, 1);
8986 4040 : destmem = offset_address (destmem,
8987 4040 : GEN_INT (-size - prolog_size),
8988 : 1);
8989 4040 : if (issetmem)
8990 1938 : emit_move_insn (destmem, mode_value);
8991 : else
8992 : {
8993 2102 : srcmem = offset_address (srcmem, *count, 1);
8994 2102 : srcmem = offset_address (srcmem,
8995 : GEN_INT (-size - prolog_size),
8996 : 1);
8997 2102 : emit_move_insn (destmem, srcmem);
8998 : }
8999 30574 : for (n = 1; n * GET_MODE_SIZE (mode) < size; n++)
9000 : {
9001 11247 : destmem = offset_address (destmem, modesize, 1);
9002 11247 : if (issetmem)
9003 5598 : emit_move_insn (destmem, mode_value);
9004 : else
9005 : {
9006 5649 : srcmem = offset_address (srcmem, modesize, 1);
9007 5649 : emit_move_insn (destmem, srcmem);
9008 : }
9009 : }
9010 :
9011 : /* Align destination. */
9012 4040 : if (desired_align > 1 && desired_align > align)
9013 : {
9014 17 : rtx saveddest = *destptr;
9015 :
9016 17 : gcc_assert (desired_align <= size);
9017 : /* Align destptr up, place it to new register. */
9018 17 : *destptr = expand_simple_binop (GET_MODE (*destptr), PLUS, *destptr,
9019 : GEN_INT (prolog_size),
9020 : NULL_RTX, 1, OPTAB_DIRECT);
9021 17 : if (REG_P (*destptr) && REG_P (saveddest) && REG_POINTER (saveddest))
9022 17 : REG_POINTER (*destptr) = 1;
9023 17 : *destptr = expand_simple_binop (GET_MODE (*destptr), AND, *destptr,
9024 17 : GEN_INT (-desired_align),
9025 : *destptr, 1, OPTAB_DIRECT);
9026 : /* See how many bytes we skipped. */
9027 17 : saveddest = expand_simple_binop (GET_MODE (*destptr), MINUS, saveddest,
9028 : *destptr,
9029 : NULL_RTX, 1, OPTAB_DIRECT);
9030 : /* Adjust srcptr and count. */
9031 17 : if (!issetmem)
9032 16 : *srcptr = expand_simple_binop (GET_MODE (*srcptr), MINUS, *srcptr,
9033 : saveddest, *srcptr, 1, OPTAB_DIRECT);
9034 17 : *count = expand_simple_binop (GET_MODE (*count), PLUS, *count,
9035 : saveddest, *count, 1, OPTAB_DIRECT);
9036 : /* We copied at most size + prolog_size. */
9037 17 : if (*min_size > (unsigned HOST_WIDE_INT)(size + prolog_size))
9038 16 : *min_size
9039 16 : = ROUND_DOWN (*min_size - size, (unsigned HOST_WIDE_INT)size);
9040 : else
9041 1 : *min_size = 0;
9042 :
9043 : /* Our loops always round down the block size, but for dispatch to
9044 : library we need precise value. */
9045 17 : if (dynamic_check)
9046 17 : *count = expand_simple_binop (GET_MODE (*count), AND, *count,
9047 : GEN_INT (-size), *count, 1, OPTAB_DIRECT);
9048 : }
9049 : else
9050 : {
9051 4023 : gcc_assert (prolog_size == 0);
9052 : /* Decrease count, so we won't end up copying last word twice. */
9053 4023 : if (!CONST_INT_P (*count))
9054 667 : *count = expand_simple_binop (GET_MODE (*count), PLUS, *count,
9055 : constm1_rtx, *count, 1, OPTAB_DIRECT);
9056 : else
9057 3356 : *count = GEN_INT (ROUND_DOWN (UINTVAL (*count) - 1,
9058 : (unsigned HOST_WIDE_INT)size));
9059 4023 : if (*min_size)
9060 3797 : *min_size = ROUND_DOWN (*min_size - 1, (unsigned HOST_WIDE_INT)size);
9061 : }
9062 4040 : }
9063 :
9064 :
9065 : /* This function is like the previous one, except here we know how many bytes
9066 : need to be copied. That allows us to update alignment not only of DST, which
9067 : is returned, but also of SRC, which is passed as a pointer for that
9068 : reason. */
9069 : static rtx
9070 0 : expand_set_or_cpymem_constant_prologue (rtx dst, rtx *srcp, rtx destreg,
9071 : rtx srcreg, rtx value, rtx vec_value,
9072 : int desired_align, int align_bytes,
9073 : bool issetmem)
9074 : {
9075 0 : rtx src = NULL;
9076 0 : rtx orig_dst = dst;
9077 0 : rtx orig_src = NULL;
9078 0 : int piece_size = 1;
9079 0 : int copied_bytes = 0;
9080 :
9081 0 : if (!issetmem)
9082 : {
9083 0 : gcc_assert (srcp != NULL);
9084 0 : src = *srcp;
9085 0 : orig_src = src;
9086 : }
9087 :
9088 0 : for (piece_size = 1;
9089 0 : piece_size <= desired_align && copied_bytes < align_bytes;
9090 0 : piece_size <<= 1)
9091 : {
9092 0 : if (align_bytes & piece_size)
9093 : {
9094 0 : if (issetmem)
9095 : {
9096 0 : if (vec_value && piece_size > GET_MODE_SIZE (GET_MODE (value)))
9097 0 : dst = emit_memset (dst, destreg, vec_value, piece_size);
9098 : else
9099 0 : dst = emit_memset (dst, destreg, value, piece_size);
9100 : }
9101 : else
9102 0 : dst = emit_memmov (dst, &src, destreg, srcreg, piece_size);
9103 0 : copied_bytes += piece_size;
9104 : }
9105 : }
9106 0 : if (MEM_ALIGN (dst) < (unsigned int) desired_align * BITS_PER_UNIT)
9107 0 : set_mem_align (dst, desired_align * BITS_PER_UNIT);
9108 0 : if (MEM_SIZE_KNOWN_P (orig_dst))
9109 0 : set_mem_size (dst, MEM_SIZE (orig_dst) - align_bytes);
9110 :
9111 0 : if (!issetmem)
9112 : {
9113 0 : int src_align_bytes = get_mem_align_offset (src, desired_align
9114 : * BITS_PER_UNIT);
9115 0 : if (src_align_bytes >= 0)
9116 0 : src_align_bytes = desired_align - src_align_bytes;
9117 0 : if (src_align_bytes >= 0)
9118 : {
9119 : unsigned int src_align;
9120 0 : for (src_align = desired_align; src_align >= 2; src_align >>= 1)
9121 : {
9122 0 : if ((src_align_bytes & (src_align - 1))
9123 0 : == (align_bytes & (src_align - 1)))
9124 : break;
9125 : }
9126 0 : if (src_align > (unsigned int) desired_align)
9127 : src_align = desired_align;
9128 0 : if (MEM_ALIGN (src) < src_align * BITS_PER_UNIT)
9129 0 : set_mem_align (src, src_align * BITS_PER_UNIT);
9130 : }
9131 0 : if (MEM_SIZE_KNOWN_P (orig_src))
9132 0 : set_mem_size (src, MEM_SIZE (orig_src) - align_bytes);
9133 0 : *srcp = src;
9134 : }
9135 :
9136 0 : return dst;
9137 : }
9138 :
9139 : /* Return true if ALG can be used in current context.
9140 : Assume we expand memset if MEMSET is true. */
9141 : static bool
9142 1440730 : alg_usable_p (enum stringop_alg alg, bool memset,
9143 : addr_space_t dst_as, addr_space_t src_as)
9144 : {
9145 1440730 : if (alg == no_stringop)
9146 : return false;
9147 : /* It is not possible to use a library call if we have non-default
9148 : address space. We can do better than the generic byte-at-a-time
9149 : loop, used as a fallback. */
9150 1440730 : if (alg == libcall &&
9151 826410 : !(ADDR_SPACE_GENERIC_P (dst_as) && ADDR_SPACE_GENERIC_P (src_as)))
9152 : return false;
9153 1440716 : if (alg == vector_loop)
9154 617225 : return TARGET_SSE || TARGET_AVX;
9155 : /* Algorithms using the rep prefix want at least edi and ecx;
9156 : additionally, memset wants eax and memcpy wants esi. Don't
9157 : consider such algorithms if the user has appropriated those
9158 : registers for their own purposes, or if we have the destination
9159 : in the non-default address space, since string insns cannot
9160 : override the destination segment. */
9161 1132063 : if (alg == rep_prefix_1_byte
9162 : || alg == rep_prefix_4_byte
9163 1132063 : || alg == rep_prefix_8_byte)
9164 : {
9165 37709 : if (fixed_regs[CX_REG]
9166 37705 : || fixed_regs[DI_REG]
9167 37701 : || (memset ? fixed_regs[AX_REG] : fixed_regs[SI_REG])
9168 37697 : || !ADDR_SPACE_GENERIC_P (dst_as)
9169 75406 : || !(ADDR_SPACE_GENERIC_P (src_as) || Pmode == word_mode))
9170 12 : return false;
9171 : }
9172 : return true;
9173 : }
9174 :
9175 : /* Given COUNT and EXPECTED_SIZE, decide on codegen of string operation. */
9176 : static enum stringop_alg
9177 286628 : decide_alg (HOST_WIDE_INT count, HOST_WIDE_INT expected_size,
9178 : unsigned HOST_WIDE_INT min_size, unsigned HOST_WIDE_INT max_size,
9179 : bool memset, bool zero_memset, addr_space_t dst_as,
9180 : addr_space_t src_as, int *dynamic_check, bool *noalign, bool recur)
9181 : {
9182 286628 : const struct stringop_algs *algs;
9183 286628 : bool optimize_for_speed;
9184 286628 : int max = 0;
9185 286628 : const struct processor_costs *cost;
9186 286628 : int i;
9187 286628 : bool any_alg_usable_p = false;
9188 :
9189 286628 : *noalign = false;
9190 286628 : *dynamic_check = -1;
9191 :
9192 : /* Even if the string operation call is cold, we still might spend a lot
9193 : of time processing large blocks. */
9194 286628 : if (optimize_function_for_size_p (cfun)
9195 286628 : || (optimize_insn_for_size_p ()
9196 12317 : && (max_size < 256
9197 4054 : || (expected_size != -1 && expected_size < 256))))
9198 : optimize_for_speed = false;
9199 : else
9200 267925 : optimize_for_speed = true;
9201 :
9202 267925 : cost = optimize_for_speed ? ix86_cost : &ix86_size_cost;
9203 286628 : if (memset)
9204 87837 : algs = &cost->memset[TARGET_64BIT != 0];
9205 : else
9206 207676 : algs = &cost->memcpy[TARGET_64BIT != 0];
9207 :
9208 : /* See maximal size for user defined algorithm. */
9209 1433140 : for (i = 0; i < MAX_STRINGOP_ALGS; i++)
9210 : {
9211 1146512 : enum stringop_alg candidate = algs->size[i].alg;
9212 1146512 : bool usable = alg_usable_p (candidate, memset, dst_as, src_as);
9213 1146512 : any_alg_usable_p |= usable;
9214 :
9215 1146512 : if (candidate != libcall && candidate && usable)
9216 554440 : max = algs->size[i].max;
9217 : }
9218 :
9219 : /* If expected size is not known but max size is small enough
9220 : so inline version is a win, set expected size into
9221 : the range. */
9222 286628 : if (((max > 1 && (unsigned HOST_WIDE_INT) max >= max_size) || max == -1)
9223 37923 : && expected_size == -1)
9224 15502 : expected_size = min_size / 2 + max_size / 2;
9225 :
9226 : /* If user specified the algorithm, honor it if possible. */
9227 286628 : if (ix86_stringop_alg != no_stringop
9228 286628 : && alg_usable_p (ix86_stringop_alg, memset, dst_as, src_as))
9229 : return ix86_stringop_alg;
9230 : /* rep; movq or rep; movl is the smallest variant. */
9231 286490 : else if (!optimize_for_speed)
9232 : {
9233 18620 : *noalign = true;
9234 18620 : if (!count || (count & 3) || (memset && !zero_memset))
9235 5766 : return alg_usable_p (rep_prefix_1_byte, memset, dst_as, src_as)
9236 5766 : ? rep_prefix_1_byte : loop_1_byte;
9237 : else
9238 12854 : return alg_usable_p (rep_prefix_4_byte, memset, dst_as, src_as)
9239 12854 : ? rep_prefix_4_byte : loop;
9240 : }
9241 : /* Very tiny blocks are best handled via the loop, REP is expensive to
9242 : setup. */
9243 267870 : else if (expected_size != -1 && expected_size < 4)
9244 : return loop_1_byte;
9245 266342 : else if (expected_size != -1)
9246 : {
9247 : enum stringop_alg alg = libcall;
9248 : bool alg_noalign = false;
9249 342272 : for (i = 0; i < MAX_STRINGOP_ALGS; i++)
9250 : {
9251 : /* We get here if the algorithms that were not libcall-based
9252 : were rep-prefix based and we are unable to use rep prefixes
9253 : based on global register usage. Break out of the loop and
9254 : use the heuristic below. */
9255 336535 : if (algs->size[i].max == 0)
9256 : break;
9257 336535 : if (algs->size[i].max >= expected_size || algs->size[i].max == -1)
9258 : {
9259 132562 : enum stringop_alg candidate = algs->size[i].alg;
9260 :
9261 132562 : if (candidate != libcall
9262 132562 : && alg_usable_p (candidate, memset, dst_as, src_as))
9263 : {
9264 24814 : alg = candidate;
9265 24814 : alg_noalign = algs->size[i].noalign;
9266 : }
9267 : /* Honor TARGET_INLINE_ALL_STRINGOPS by picking
9268 : last non-libcall inline algorithm. */
9269 132562 : if (TARGET_INLINE_ALL_STRINGOPS)
9270 : {
9271 : /* When the current size is best to be copied by a libcall,
9272 : but we are still forced to inline, run the heuristic below
9273 : that will pick code for medium sized blocks. */
9274 20038 : if (alg != libcall)
9275 : {
9276 8555 : *noalign = alg_noalign;
9277 8555 : return alg;
9278 : }
9279 11483 : else if (!any_alg_usable_p)
9280 : break;
9281 : }
9282 112524 : else if (alg_usable_p (candidate, memset, dst_as, src_as)
9283 112524 : && !(TARGET_PREFER_KNOWN_REP_MOVSB_STOSB
9284 34 : && candidate == rep_prefix_1_byte
9285 : /* NB: If min_size != max_size, size is
9286 : unknown. */
9287 34 : && min_size != max_size))
9288 : {
9289 112499 : *noalign = algs->size[i].noalign;
9290 112499 : return candidate;
9291 : }
9292 : }
9293 : }
9294 : }
9295 : /* When asked to inline the call anyway, try to pick meaningful choice.
9296 : We look for maximal size of block that is faster to copy by hand and
9297 : take blocks of at most of that size guessing that average size will
9298 : be roughly half of the block.
9299 :
9300 : If this turns out to be bad, we might simply specify the preferred
9301 : choice in ix86_costs. */
9302 138099 : if ((TARGET_INLINE_ALL_STRINGOPS || TARGET_INLINE_STRINGOPS_DYNAMICALLY)
9303 145294 : && (algs->unknown_size == libcall
9304 0 : || !alg_usable_p (algs->unknown_size, memset, dst_as, src_as)))
9305 : {
9306 7195 : enum stringop_alg alg;
9307 7195 : HOST_WIDE_INT new_expected_size = (max > 0 ? max : 4096) / 2;
9308 :
9309 : /* If there aren't any usable algorithms or if recursing already,
9310 : then recursing on smaller sizes or same size isn't going to
9311 : find anything. Just return the simple byte-at-a-time copy loop. */
9312 7195 : if (!any_alg_usable_p || recur)
9313 : {
9314 : /* Pick something reasonable. */
9315 0 : if (TARGET_INLINE_STRINGOPS_DYNAMICALLY && !recur)
9316 0 : *dynamic_check = 128;
9317 0 : return loop_1_byte;
9318 : }
9319 7195 : alg = decide_alg (count, new_expected_size, min_size, max_size,
9320 : memset, zero_memset, dst_as, src_as,
9321 : dynamic_check, noalign, true);
9322 7195 : gcc_assert (*dynamic_check == -1);
9323 7195 : if (TARGET_INLINE_STRINGOPS_DYNAMICALLY)
9324 10 : *dynamic_check = max;
9325 : else
9326 7185 : gcc_assert (alg != libcall);
9327 7195 : return alg;
9328 : }
9329 :
9330 : /* Try to use some reasonable fallback algorithm. Note that for
9331 : non-default address spaces we default to a loop instead of
9332 : a libcall. */
9333 :
9334 138093 : bool have_as = !(ADDR_SPACE_GENERIC_P (dst_as)
9335 : && ADDR_SPACE_GENERIC_P (src_as));
9336 :
9337 138093 : return (alg_usable_p (algs->unknown_size, memset, dst_as, src_as)
9338 138093 : ? algs->unknown_size : have_as ? loop : libcall);
9339 : }
9340 :
9341 : /* Decide on alignment. We know that the operand is already aligned to ALIGN
9342 : (ALIGN can be based on profile feedback and thus it is not 100% guaranteed). */
9343 : static int
9344 26819 : decide_alignment (int align,
9345 : enum stringop_alg alg,
9346 : int expected_size,
9347 : machine_mode move_mode)
9348 : {
9349 26819 : int desired_align = 0;
9350 :
9351 26819 : gcc_assert (alg != no_stringop);
9352 :
9353 26819 : if (alg == libcall)
9354 : return 0;
9355 26819 : if (move_mode == VOIDmode)
9356 : return 0;
9357 :
9358 26819 : desired_align = GET_MODE_SIZE (move_mode);
9359 : /* PentiumPro has special logic triggering for 8 byte aligned blocks.
9360 : copying whole cacheline at once. */
9361 26819 : if (TARGET_CPU_P (PENTIUMPRO)
9362 0 : && (alg == rep_prefix_4_byte || alg == rep_prefix_1_byte))
9363 26819 : desired_align = 8;
9364 :
9365 26819 : if (optimize_size)
9366 9884 : desired_align = 1;
9367 26819 : if (desired_align < align)
9368 : desired_align = align;
9369 26819 : if (expected_size != -1 && expected_size < 4)
9370 1 : desired_align = align;
9371 :
9372 : return desired_align;
9373 : }
9374 :
9375 :
9376 : /* Helper function for memcpy. For QImode value 0xXY produce
9377 : 0xXYXYXYXY of wide specified by MODE. This is essentially
9378 : a * 0x10101010, but we can do slightly better than
9379 : synth_mult by unwinding the sequence by hand on CPUs with
9380 : slow multiply. */
9381 : static rtx
9382 18580 : promote_duplicated_reg (machine_mode mode, rtx val)
9383 : {
9384 18580 : if (val == const0_rtx)
9385 16101 : return copy_to_mode_reg (mode, CONST0_RTX (mode));
9386 :
9387 2479 : machine_mode valmode = GET_MODE (val);
9388 2479 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
9389 : {
9390 : /* Duplicate the scalar value for integer vector. */
9391 1864 : gcc_assert ((val == const0_rtx || val == constm1_rtx)
9392 : || GET_MODE_INNER (mode) == valmode);
9393 936 : rtx dup = gen_reg_rtx (mode);
9394 936 : bool ok = ix86_expand_vector_init_duplicate (false, mode, dup,
9395 : val);
9396 936 : gcc_assert (ok);
9397 : return dup;
9398 : }
9399 :
9400 1543 : rtx tmp;
9401 1543 : int nops = mode == DImode ? 3 : 2;
9402 :
9403 40 : gcc_assert (mode == SImode || mode == DImode);
9404 1543 : if (CONST_INT_P (val))
9405 : {
9406 1232 : HOST_WIDE_INT v = INTVAL (val) & 255;
9407 :
9408 1232 : v |= v << 8;
9409 1232 : v |= v << 16;
9410 1232 : if (mode == DImode)
9411 1204 : v |= (v << 16) << 16;
9412 1232 : return copy_to_mode_reg (mode, gen_int_mode (v, mode));
9413 : }
9414 :
9415 311 : if (valmode == VOIDmode)
9416 : valmode = QImode;
9417 311 : if (valmode != QImode)
9418 0 : val = gen_lowpart (QImode, val);
9419 311 : if (mode == QImode)
9420 : return val;
9421 311 : if (!TARGET_PARTIAL_REG_STALL)
9422 311 : nops--;
9423 311 : if (ix86_cost->mult_init[mode == DImode ? 3 : 2]
9424 311 : + ix86_cost->mult_bit * (mode == DImode ? 8 : 4)
9425 311 : <= (ix86_cost->shift_const + ix86_cost->add) * nops
9426 311 : + (COSTS_N_INSNS (TARGET_PARTIAL_REG_STALL == 0)))
9427 : {
9428 311 : rtx reg = convert_modes (mode, QImode, val, true);
9429 311 : tmp = promote_duplicated_reg (mode, const1_rtx);
9430 311 : return expand_simple_binop (mode, MULT, reg, tmp, NULL, 1,
9431 311 : OPTAB_DIRECT);
9432 : }
9433 : else
9434 : {
9435 0 : rtx reg = convert_modes (mode, QImode, val, true);
9436 :
9437 0 : if (!TARGET_PARTIAL_REG_STALL)
9438 0 : emit_insn (gen_insv_1 (mode, reg, reg));
9439 : else
9440 : {
9441 0 : tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (8),
9442 : NULL, 1, OPTAB_DIRECT);
9443 0 : reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1,
9444 : OPTAB_DIRECT);
9445 : }
9446 0 : tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (16),
9447 : NULL, 1, OPTAB_DIRECT);
9448 0 : reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
9449 0 : if (mode == SImode)
9450 : return reg;
9451 0 : tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (32),
9452 : NULL, 1, OPTAB_DIRECT);
9453 0 : reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
9454 0 : return reg;
9455 : }
9456 : }
9457 :
9458 : /* Duplicate value VAL using promote_duplicated_reg into maximal size that will
9459 : be needed by main loop copying SIZE_NEEDED chunks and prologue getting
9460 : alignment from ALIGN to DESIRED_ALIGN. */
9461 : static rtx
9462 13271 : promote_duplicated_reg_to_size (rtx val, int size_needed, int desired_align,
9463 : int align)
9464 : {
9465 13271 : rtx promoted_val;
9466 :
9467 13271 : if (TARGET_64BIT
9468 11794 : && (size_needed > 4 || (desired_align > align && desired_align > 4)))
9469 3940 : promoted_val = promote_duplicated_reg (DImode, val);
9470 9331 : else if (size_needed > 2 || (desired_align > align && desired_align > 2))
9471 7995 : promoted_val = promote_duplicated_reg (SImode, val);
9472 1336 : else if (size_needed > 1 || (desired_align > align && desired_align > 1))
9473 0 : promoted_val = promote_duplicated_reg (HImode, val);
9474 : else
9475 : promoted_val = val;
9476 :
9477 13271 : return promoted_val;
9478 : }
9479 :
9480 : /* Copy the address to a Pmode register. This is used for x32 to
9481 : truncate DImode TLS address to a SImode register. */
9482 :
9483 : static rtx
9484 65010 : ix86_copy_addr_to_reg (rtx addr)
9485 : {
9486 65010 : rtx reg;
9487 69548 : if (GET_MODE (addr) == Pmode || GET_MODE (addr) == VOIDmode)
9488 : {
9489 65010 : reg = copy_addr_to_reg (addr);
9490 65010 : REG_POINTER (reg) = 1;
9491 65010 : return reg;
9492 : }
9493 : else
9494 : {
9495 0 : gcc_assert (GET_MODE (addr) == DImode && Pmode == SImode);
9496 0 : reg = copy_to_mode_reg (DImode, addr);
9497 0 : REG_POINTER (reg) = 1;
9498 0 : return gen_rtx_SUBREG (SImode, reg, 0);
9499 : }
9500 : }
9501 :
9502 : /* Expand string move (memcpy) ot store (memset) operation. Use i386 string
9503 : operations when profitable. The code depends upon architecture, block size
9504 : and alignment, but always has one of the following overall structures:
9505 :
9506 : Aligned move sequence:
9507 :
9508 : 1) Prologue guard: Conditional that jumps up to epilogues for small
9509 : blocks that can be handled by epilogue alone. This is faster
9510 : but also needed for correctness, since prologue assume the block
9511 : is larger than the desired alignment.
9512 :
9513 : Optional dynamic check for size and libcall for large
9514 : blocks is emitted here too, with -minline-stringops-dynamically.
9515 :
9516 : 2) Prologue: copy first few bytes in order to get destination
9517 : aligned to DESIRED_ALIGN. It is emitted only when ALIGN is less
9518 : than DESIRED_ALIGN and up to DESIRED_ALIGN - ALIGN bytes can be
9519 : copied. We emit either a jump tree on power of two sized
9520 : blocks, or a byte loop.
9521 :
9522 : 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
9523 : with specified algorithm.
9524 :
9525 : 4) Epilogue: code copying tail of the block that is too small to be
9526 : handled by main body (or up to size guarded by prologue guard).
9527 :
9528 : Misaligned move sequence
9529 :
9530 : 1) missaligned move prologue/epilogue containing:
9531 : a) Prologue handling small memory blocks and jumping to done_label
9532 : (skipped if blocks are known to be large enough)
9533 : b) Single move copying first DESIRED_ALIGN-ALIGN bytes if alignment is
9534 : needed by single possibly misaligned move
9535 : (skipped if alignment is not needed)
9536 : c) Copy of last SIZE_NEEDED bytes by possibly misaligned moves
9537 :
9538 : 2) Zero size guard dispatching to done_label, if needed
9539 :
9540 : 3) dispatch to library call, if needed,
9541 :
9542 : 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
9543 : with specified algorithm. */
9544 : bool
9545 152281 : ix86_expand_set_or_cpymem (rtx dst, rtx src, rtx count_exp, rtx val_exp,
9546 : rtx align_exp, rtx expected_align_exp,
9547 : rtx expected_size_exp, rtx min_size_exp,
9548 : rtx max_size_exp, rtx probable_max_size_exp,
9549 : bool issetmem)
9550 : {
9551 152281 : if (TARGET_MISALIGNED_MOVE_STRING_PRO_EPILOGUES)
9552 : {
9553 : /* Expand bounded memset and memcpy as memmove if misaligned moves
9554 : are preferred. Since
9555 :
9556 : commit b41f96465190751561f6909e858604ceab00595b
9557 : Author: H.J. Lu <hjl.tools@gmail.com>
9558 : Date: Mon Oct 20 16:14:34 2025 +0800
9559 :
9560 : x86-64: Inline memmove with overlapping unaligned loads and stores.
9561 :
9562 : inlines memmove with overlapping unaligned and stores, which
9563 : reduces the numbers of branches and memory moves, comparing
9564 : against the regular memset and memcpy inlining. */
9565 152263 : rtx operands[9];
9566 152263 : operands[0] = dst;
9567 152263 : operands[1] = issetmem ? val_exp : src;
9568 152263 : operands[2] = count_exp;
9569 152263 : operands[3] = align_exp;
9570 152263 : operands[4] = expected_align_exp;
9571 152263 : operands[5] = expected_size_exp;
9572 152263 : operands[6] = min_size_exp;
9573 152263 : operands[7] = max_size_exp;
9574 152263 : operands[8] = probable_max_size_exp;
9575 152263 : if (ix86_expand_set_or_movmem (operands, !issetmem, issetmem))
9576 8644 : return true;
9577 : }
9578 :
9579 143637 : rtx destreg;
9580 143637 : rtx srcreg = NULL;
9581 143637 : rtx_code_label *label = NULL;
9582 143637 : rtx tmp;
9583 143637 : rtx_code_label *jump_around_label = NULL;
9584 143637 : HOST_WIDE_INT align = 1;
9585 143637 : unsigned HOST_WIDE_INT count = 0;
9586 143637 : HOST_WIDE_INT expected_size = -1;
9587 143637 : int size_needed = 0, epilogue_size_needed;
9588 143637 : int desired_align = 0, align_bytes = 0;
9589 143637 : enum stringop_alg alg;
9590 143637 : rtx promoted_val = NULL;
9591 143637 : rtx vec_promoted_val = NULL;
9592 143637 : bool force_loopy_epilogue = false;
9593 143637 : int dynamic_check;
9594 143637 : bool need_zero_guard = false;
9595 143637 : bool noalign;
9596 143637 : machine_mode move_mode = VOIDmode;
9597 143637 : int unroll_factor = 1;
9598 : /* TODO: Once value ranges are available, fill in proper data. */
9599 143637 : unsigned HOST_WIDE_INT min_size = HOST_WIDE_INT_0U;
9600 143637 : unsigned HOST_WIDE_INT max_size = HOST_WIDE_INT_M1U;
9601 143637 : unsigned HOST_WIDE_INT probable_max_size = HOST_WIDE_INT_M1U;
9602 143637 : bool misaligned_prologue_used = false;
9603 143637 : addr_space_t dst_as, src_as = ADDR_SPACE_GENERIC;
9604 :
9605 143637 : if (CONST_INT_P (align_exp))
9606 143637 : align = INTVAL (align_exp);
9607 : /* i386 can do misaligned access on reasonably increased cost. */
9608 143637 : if (CONST_INT_P (expected_align_exp)
9609 143637 : && INTVAL (expected_align_exp) > align)
9610 : align = INTVAL (expected_align_exp);
9611 : /* ALIGN is the minimum of destination and source alignment, but we care here
9612 : just about destination alignment. */
9613 137508 : else if (!issetmem
9614 228601 : && MEM_ALIGN (dst) > (unsigned HOST_WIDE_INT) align * BITS_PER_UNIT)
9615 3269 : align = MEM_ALIGN (dst) / BITS_PER_UNIT;
9616 :
9617 143637 : if (CONST_INT_P (count_exp))
9618 : {
9619 70472 : min_size = max_size = probable_max_size = count = expected_size
9620 70472 : = INTVAL (count_exp);
9621 : /* When COUNT is 0, there is nothing to do. */
9622 70472 : if (!count)
9623 : return true;
9624 : }
9625 : else
9626 : {
9627 73165 : if (min_size_exp)
9628 73165 : min_size = INTVAL (min_size_exp);
9629 73165 : if (max_size_exp)
9630 58563 : max_size = INTVAL (max_size_exp);
9631 73165 : if (probable_max_size_exp)
9632 60055 : probable_max_size = INTVAL (probable_max_size_exp);
9633 73165 : if (CONST_INT_P (expected_size_exp))
9634 73165 : expected_size = INTVAL (expected_size_exp);
9635 : }
9636 :
9637 : /* Make sure we don't need to care about overflow later on. */
9638 143637 : if (count > (HOST_WIDE_INT_1U << 30))
9639 : return false;
9640 :
9641 143463 : dst_as = MEM_ADDR_SPACE (dst);
9642 143463 : if (!issetmem)
9643 97109 : src_as = MEM_ADDR_SPACE (src);
9644 :
9645 : /* Step 0: Decide on preferred algorithm, desired alignment and
9646 : size of chunks to be copied by main loop. */
9647 143463 : alg = decide_alg (count, expected_size, min_size, probable_max_size,
9648 46354 : issetmem, issetmem && val_exp == const0_rtx,
9649 : dst_as, src_as, &dynamic_check, &noalign, false);
9650 :
9651 143463 : if (dump_file)
9652 7 : fprintf (dump_file, "Selected stringop expansion strategy: %s\n",
9653 7 : stringop_alg_names[alg]);
9654 :
9655 143463 : if (alg == libcall)
9656 : return false;
9657 26819 : gcc_assert (alg != no_stringop);
9658 :
9659 26819 : if (!count)
9660 5653 : count_exp = copy_to_mode_reg (GET_MODE (count_exp), count_exp);
9661 26819 : destreg = ix86_copy_addr_to_reg (XEXP (dst, 0));
9662 26819 : if (!issetmem)
9663 13548 : srcreg = ix86_copy_addr_to_reg (XEXP (src, 0));
9664 :
9665 26819 : bool aligned_dstmem = false;
9666 26819 : unsigned int nunits = issetmem ? STORE_MAX_PIECES : MOVE_MAX;
9667 26819 : bool single_insn_p = count && count <= nunits;
9668 26819 : if (single_insn_p)
9669 : {
9670 : /* If it can be done with a single instruction, use vector
9671 : instruction and don't align destination. */
9672 7 : alg = vector_loop;
9673 7 : noalign = true;
9674 7 : dynamic_check = -1;
9675 : }
9676 :
9677 26819 : unroll_factor = 1;
9678 26819 : move_mode = word_mode;
9679 26819 : switch (alg)
9680 : {
9681 0 : case libcall:
9682 0 : case no_stringop:
9683 0 : case last_alg:
9684 0 : gcc_unreachable ();
9685 173 : case loop_1_byte:
9686 173 : need_zero_guard = true;
9687 173 : move_mode = QImode;
9688 173 : break;
9689 46 : case loop:
9690 46 : need_zero_guard = true;
9691 46 : break;
9692 19 : case unrolled_loop:
9693 19 : need_zero_guard = true;
9694 19 : unroll_factor = (TARGET_64BIT ? 4 : 2);
9695 : break;
9696 7936 : case vector_loop:
9697 7936 : need_zero_guard = true;
9698 7936 : unroll_factor = 4;
9699 : /* Get the vector mode to move STORE_MAX_PIECES/MOVE_MAX bytes. */
9700 7936 : nunits /= GET_MODE_SIZE (word_mode);
9701 7936 : if (nunits > 1)
9702 : {
9703 7932 : move_mode = mode_for_vector (word_mode, nunits).require ();
9704 7932 : gcc_assert (optab_handler (mov_optab, move_mode)
9705 : != CODE_FOR_nothing);
9706 : }
9707 : break;
9708 25 : case rep_prefix_8_byte:
9709 25 : move_mode = DImode;
9710 25 : break;
9711 12850 : case rep_prefix_4_byte:
9712 12850 : move_mode = SImode;
9713 12850 : break;
9714 5770 : case rep_prefix_1_byte:
9715 5770 : move_mode = QImode;
9716 5770 : break;
9717 : }
9718 26819 : size_needed = GET_MODE_SIZE (move_mode) * unroll_factor;
9719 26819 : epilogue_size_needed = size_needed;
9720 :
9721 : /* If we are going to call any library calls conditionally, make sure any
9722 : pending stack adjustment happen before the first conditional branch,
9723 : otherwise they will be emitted before the library call only and won't
9724 : happen from the other branches. */
9725 26819 : if (dynamic_check != -1)
9726 2 : do_pending_stack_adjust ();
9727 :
9728 26819 : desired_align = decide_alignment (align, alg, expected_size, move_mode);
9729 26819 : if (!TARGET_ALIGN_STRINGOPS || noalign)
9730 26564 : align = desired_align;
9731 :
9732 : /* Step 1: Prologue guard. */
9733 :
9734 : /* Alignment code needs count to be in register. */
9735 26819 : if (CONST_INT_P (count_exp) && desired_align > align)
9736 : {
9737 22 : if (INTVAL (count_exp) > desired_align
9738 22 : && INTVAL (count_exp) > size_needed)
9739 : {
9740 22 : align_bytes
9741 22 : = get_mem_align_offset (dst, desired_align * BITS_PER_UNIT);
9742 22 : if (align_bytes <= 0)
9743 : align_bytes = 0;
9744 : else
9745 0 : align_bytes = desired_align - align_bytes;
9746 : }
9747 0 : if (align_bytes == 0)
9748 44 : count_exp = force_reg (counter_mode (count_exp), count_exp);
9749 : }
9750 26819 : gcc_assert (desired_align >= 1 && align >= 1);
9751 :
9752 26819 : if (!single_insn_p)
9753 : {
9754 : /* Misaligned move sequences handle both prologue and epilogue
9755 : at once. Default code generation results in a smaller code
9756 : for large alignments and also avoids redundant job when sizes
9757 : are known precisely. */
9758 26812 : misaligned_prologue_used
9759 53624 : = (TARGET_MISALIGNED_MOVE_STRING_PRO_EPILOGUES
9760 26806 : && MAX (desired_align, epilogue_size_needed) <= 32
9761 18550 : && desired_align <= epilogue_size_needed
9762 31404 : && ((desired_align > align && !align_bytes)
9763 4575 : || (!count && epilogue_size_needed > 1)));
9764 :
9765 : /* Destination is aligned after the misaligned prologue. */
9766 26812 : aligned_dstmem = misaligned_prologue_used;
9767 :
9768 26812 : if (noalign && !misaligned_prologue_used)
9769 : {
9770 : /* Also use misaligned prologue if alignment isn't needed and
9771 : destination isn't aligned. Since alignment isn't needed,
9772 : the destination after prologue won't be aligned. */
9773 26557 : aligned_dstmem = (GET_MODE_ALIGNMENT (move_mode)
9774 26557 : <= MEM_ALIGN (dst));
9775 26557 : if (!aligned_dstmem)
9776 4023 : misaligned_prologue_used = true;
9777 : }
9778 : }
9779 :
9780 : /* Do the cheap promotion to allow better CSE across the
9781 : main loop and epilogue (ie one load of the big constant in the
9782 : front of all code.
9783 : For now the misaligned move sequences do not have fast path
9784 : without broadcasting. */
9785 26819 : if (issetmem
9786 13271 : && (alg == vector_loop
9787 7962 : || CONST_INT_P (val_exp)
9788 49 : || misaligned_prologue_used))
9789 : {
9790 7914 : if (alg == vector_loop)
9791 : {
9792 5309 : promoted_val = promote_duplicated_reg_to_size (val_exp,
9793 10618 : GET_MODE_SIZE (word_mode),
9794 : desired_align, align);
9795 : /* Duplicate the promoted scalar value if not 0 nor -1. */
9796 5309 : vec_promoted_val
9797 5309 : = promote_duplicated_reg (move_mode,
9798 5309 : (val_exp == const0_rtx
9799 568 : || val_exp == constm1_rtx)
9800 : ? val_exp : promoted_val);
9801 : }
9802 : else
9803 : {
9804 7914 : promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
9805 : desired_align, align);
9806 : }
9807 : }
9808 : /* Misaligned move sequences handles both prologues and epilogues at once.
9809 : Default code generation results in smaller code for large alignments and
9810 : also avoids redundant job when sizes are known precisely. */
9811 26771 : if (misaligned_prologue_used)
9812 : {
9813 : /* Misaligned move prologue handled small blocks by itself. */
9814 4040 : expand_set_or_cpymem_prologue_epilogue_by_misaligned_moves
9815 4040 : (dst, src, &destreg, &srcreg,
9816 : move_mode, promoted_val, vec_promoted_val,
9817 : &count_exp,
9818 : &jump_around_label,
9819 4040 : desired_align < align
9820 0 : ? MAX (desired_align, epilogue_size_needed) : epilogue_size_needed,
9821 : desired_align, align, &min_size, dynamic_check, issetmem);
9822 4040 : if (!issetmem)
9823 2102 : src = change_address (src, BLKmode, srcreg);
9824 4040 : dst = change_address (dst, BLKmode, destreg);
9825 4040 : if (aligned_dstmem)
9826 17 : set_mem_align (dst, desired_align * BITS_PER_UNIT);
9827 4040 : epilogue_size_needed = 0;
9828 4040 : if (need_zero_guard
9829 3756 : && min_size < (unsigned HOST_WIDE_INT) size_needed)
9830 : {
9831 : /* It is possible that we copied enough so the main loop will not
9832 : execute. */
9833 715 : gcc_assert (size_needed > 1);
9834 715 : if (jump_around_label == NULL_RTX)
9835 49 : jump_around_label = gen_label_rtx ();
9836 1430 : emit_cmp_and_jump_insns (count_exp,
9837 : GEN_INT (size_needed),
9838 : LTU, 0, counter_mode (count_exp), 1, jump_around_label);
9839 715 : if (expected_size == -1
9840 55 : || expected_size < (desired_align - align) / 2 + size_needed)
9841 660 : predict_jump (REG_BR_PROB_BASE * 20 / 100);
9842 : else
9843 55 : predict_jump (REG_BR_PROB_BASE * 60 / 100);
9844 : }
9845 : }
9846 : /* Ensure that alignment prologue won't copy past end of block. */
9847 22779 : else if (size_needed > 1 || (desired_align > 1 && desired_align > align))
9848 : {
9849 16836 : epilogue_size_needed = MAX (size_needed - 1, desired_align - align);
9850 : /* Epilogue always copies COUNT_EXP & EPILOGUE_SIZE_NEEDED bytes.
9851 : Make sure it is power of 2. */
9852 16836 : epilogue_size_needed = 1 << (floor_log2 (epilogue_size_needed) + 1);
9853 :
9854 : /* To improve performance of small blocks, we jump around the VAL
9855 : promoting mode. This mean that if the promoted VAL is not constant,
9856 : we might not use it in the epilogue and have to use byte
9857 : loop variant. */
9858 16836 : if (issetmem && epilogue_size_needed > 2 && !promoted_val)
9859 16836 : force_loopy_epilogue = true;
9860 16836 : if ((count && count < (unsigned HOST_WIDE_INT) epilogue_size_needed)
9861 16827 : || max_size < (unsigned HOST_WIDE_INT) epilogue_size_needed)
9862 : {
9863 : /* If main algorithm works on QImode, no epilogue is needed.
9864 : For small sizes just don't align anything. */
9865 44 : if (size_needed == 1)
9866 0 : desired_align = align;
9867 : else
9868 44 : goto epilogue;
9869 : }
9870 16792 : else if (!count
9871 63 : && min_size < (unsigned HOST_WIDE_INT) epilogue_size_needed)
9872 : {
9873 63 : label = gen_label_rtx ();
9874 126 : emit_cmp_and_jump_insns (count_exp,
9875 : GEN_INT (epilogue_size_needed),
9876 : LTU, 0, counter_mode (count_exp), 1, label);
9877 63 : if (expected_size == -1 || expected_size < epilogue_size_needed)
9878 63 : predict_jump (REG_BR_PROB_BASE * 60 / 100);
9879 : else
9880 0 : predict_jump (REG_BR_PROB_BASE * 20 / 100);
9881 : }
9882 : }
9883 :
9884 : /* Emit code to decide on runtime whether library call or inline should be
9885 : used. */
9886 26775 : if (dynamic_check != -1)
9887 : {
9888 2 : if (!issetmem && CONST_INT_P (count_exp))
9889 : {
9890 1 : if (UINTVAL (count_exp) >= (unsigned HOST_WIDE_INT)dynamic_check)
9891 : {
9892 1 : emit_block_copy_via_libcall (dst, src, count_exp);
9893 1 : count_exp = const0_rtx;
9894 1 : goto epilogue;
9895 : }
9896 : }
9897 : else
9898 : {
9899 1 : rtx_code_label *hot_label = gen_label_rtx ();
9900 1 : if (jump_around_label == NULL_RTX)
9901 1 : jump_around_label = gen_label_rtx ();
9902 2 : emit_cmp_and_jump_insns (count_exp, GEN_INT (dynamic_check - 1),
9903 : LEU, 0, counter_mode (count_exp),
9904 : 1, hot_label);
9905 1 : predict_jump (REG_BR_PROB_BASE * 90 / 100);
9906 1 : if (issetmem)
9907 1 : set_storage_via_libcall (dst, count_exp, val_exp);
9908 : else
9909 0 : emit_block_copy_via_libcall (dst, src, count_exp);
9910 1 : emit_jump (jump_around_label);
9911 1 : emit_label (hot_label);
9912 : }
9913 : }
9914 :
9915 : /* Step 2: Alignment prologue. */
9916 : /* Do the expensive promotion once we branched off the small blocks. */
9917 26774 : if (issetmem && !promoted_val)
9918 48 : promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
9919 : desired_align, align);
9920 :
9921 26774 : if (desired_align > align && !misaligned_prologue_used)
9922 : {
9923 7 : if (align_bytes == 0)
9924 : {
9925 : /* Except for the first move in prologue, we no longer know
9926 : constant offset in aliasing info. It don't seems to worth
9927 : the pain to maintain it for the first move, so throw away
9928 : the info early. */
9929 7 : dst = change_address (dst, BLKmode, destreg);
9930 7 : if (!issetmem)
9931 5 : src = change_address (src, BLKmode, srcreg);
9932 7 : dst = expand_set_or_cpymem_prologue (dst, src, destreg, srcreg,
9933 : promoted_val, vec_promoted_val,
9934 : count_exp, align, desired_align,
9935 : issetmem);
9936 : /* At most desired_align - align bytes are copied. */
9937 7 : if (min_size < (unsigned)(desired_align - align))
9938 0 : min_size = 0;
9939 : else
9940 7 : min_size -= desired_align - align;
9941 : }
9942 : else
9943 : {
9944 : /* If we know how many bytes need to be stored before dst is
9945 : sufficiently aligned, maintain aliasing info accurately. */
9946 0 : dst = expand_set_or_cpymem_constant_prologue (dst, &src, destreg,
9947 : srcreg,
9948 : promoted_val,
9949 : vec_promoted_val,
9950 : desired_align,
9951 : align_bytes,
9952 : issetmem);
9953 :
9954 0 : count_exp = plus_constant (counter_mode (count_exp),
9955 0 : count_exp, -align_bytes);
9956 0 : count -= align_bytes;
9957 0 : min_size -= align_bytes;
9958 0 : max_size -= align_bytes;
9959 : }
9960 7 : if (need_zero_guard
9961 7 : && min_size < (unsigned HOST_WIDE_INT) size_needed
9962 1 : && (count < (unsigned HOST_WIDE_INT) size_needed
9963 0 : || (align_bytes == 0
9964 0 : && count < ((unsigned HOST_WIDE_INT) size_needed
9965 0 : + desired_align - align))))
9966 : {
9967 : /* It is possible that we copied enough so the main loop will not
9968 : execute. */
9969 1 : gcc_assert (size_needed > 1);
9970 1 : if (label == NULL_RTX)
9971 0 : label = gen_label_rtx ();
9972 2 : emit_cmp_and_jump_insns (count_exp,
9973 : GEN_INT (size_needed),
9974 : LTU, 0, counter_mode (count_exp), 1, label);
9975 1 : if (expected_size == -1
9976 0 : || expected_size < (desired_align - align) / 2 + size_needed)
9977 1 : predict_jump (REG_BR_PROB_BASE * 20 / 100);
9978 : else
9979 0 : predict_jump (REG_BR_PROB_BASE * 60 / 100);
9980 : }
9981 : }
9982 26774 : if (label && size_needed == 1)
9983 : {
9984 0 : emit_label (label);
9985 0 : LABEL_NUSES (label) = 1;
9986 0 : label = NULL;
9987 0 : epilogue_size_needed = 1;
9988 0 : if (issetmem)
9989 0 : promoted_val = val_exp;
9990 : }
9991 26774 : else if (label == NULL_RTX && !misaligned_prologue_used)
9992 22672 : epilogue_size_needed = size_needed;
9993 :
9994 : /* Step 3: Main loop. */
9995 :
9996 26774 : switch (alg)
9997 : {
9998 0 : case libcall:
9999 0 : case no_stringop:
10000 0 : case last_alg:
10001 0 : gcc_unreachable ();
10002 238 : case loop_1_byte:
10003 238 : case loop:
10004 238 : case unrolled_loop:
10005 238 : expand_set_or_cpymem_via_loop (dst, src, destreg, srcreg, promoted_val,
10006 : count_exp, move_mode, unroll_factor,
10007 : expected_size, issetmem);
10008 238 : break;
10009 7891 : case vector_loop:
10010 7891 : expand_set_or_cpymem_via_loop (dst, src, destreg, srcreg,
10011 : vec_promoted_val, count_exp, move_mode,
10012 : unroll_factor, expected_size, issetmem);
10013 7891 : break;
10014 18645 : case rep_prefix_8_byte:
10015 18645 : case rep_prefix_4_byte:
10016 18645 : case rep_prefix_1_byte:
10017 18645 : expand_set_or_cpymem_via_rep (dst, src, destreg, srcreg, promoted_val,
10018 : val_exp, count_exp, move_mode, issetmem);
10019 18645 : break;
10020 : }
10021 : /* Adjust properly the offset of src and dest memory for aliasing. */
10022 26774 : if (CONST_INT_P (count_exp))
10023 : {
10024 21134 : if (!issetmem)
10025 8928 : src = adjust_automodify_address_nv (src, BLKmode, srcreg,
10026 : (count / size_needed) * size_needed);
10027 21134 : dst = adjust_automodify_address_nv (dst, BLKmode, destreg,
10028 : (count / size_needed) * size_needed);
10029 : }
10030 : else
10031 : {
10032 5640 : if (!issetmem)
10033 4617 : src = change_address (src, BLKmode, srcreg);
10034 5640 : dst = change_address (dst, BLKmode, destreg);
10035 : }
10036 :
10037 : /* Step 4: Epilogue to copy the remaining bytes. */
10038 26819 : epilogue:
10039 26819 : if (label)
10040 : {
10041 : /* When the main loop is done, COUNT_EXP might hold original count,
10042 : while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
10043 : Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
10044 : bytes. Compensate if needed. */
10045 :
10046 63 : if (size_needed < epilogue_size_needed)
10047 : {
10048 0 : tmp = expand_simple_binop (counter_mode (count_exp), AND, count_exp,
10049 0 : GEN_INT (size_needed - 1), count_exp, 1,
10050 : OPTAB_DIRECT);
10051 0 : if (tmp != count_exp)
10052 0 : emit_move_insn (count_exp, tmp);
10053 : }
10054 63 : emit_label (label);
10055 63 : LABEL_NUSES (label) = 1;
10056 : }
10057 :
10058 26819 : if (count_exp != const0_rtx && epilogue_size_needed > 1)
10059 : {
10060 16836 : if (force_loopy_epilogue)
10061 0 : expand_setmem_epilogue_via_loop (dst, destreg, val_exp, count_exp,
10062 : epilogue_size_needed);
10063 : else
10064 : {
10065 16836 : if (issetmem)
10066 9997 : expand_setmem_epilogue (dst, destreg, promoted_val,
10067 : vec_promoted_val, count_exp,
10068 : epilogue_size_needed);
10069 : else
10070 6839 : expand_cpymem_epilogue (dst, src, destreg, srcreg, count_exp,
10071 : epilogue_size_needed);
10072 : }
10073 : }
10074 26819 : if (jump_around_label)
10075 718 : emit_label (jump_around_label);
10076 : return true;
10077 : }
10078 :
10079 : /* Fully unroll memmove of known size with up to 8 registers. */
10080 :
10081 : static bool
10082 1882 : ix86_expand_unroll_movmem (rtx dst, rtx src, rtx destreg, rtx srcreg,
10083 : unsigned HOST_WIDE_INT count,
10084 : machine_mode mode)
10085 : {
10086 : /* If 8 registers registers can cover all memory, load them into
10087 : registers and store them together to avoid possible address
10088 : overlap between source and destination. */
10089 1882 : unsigned HOST_WIDE_INT moves = count / GET_MODE_SIZE (mode);
10090 1882 : if (moves == 0)
10091 : {
10092 0 : mode = smallest_int_mode_for_size
10093 0 : (count * BITS_PER_UNIT).require ();
10094 0 : if (count == GET_MODE_SIZE (mode))
10095 : moves = 1;
10096 : else
10097 : {
10098 : /* Reduce the smallest move size by half so that MOVES == 1. */
10099 0 : mode = smallest_int_mode_for_size
10100 0 : (GET_MODE_BITSIZE (mode) / 2).require ();
10101 0 : moves = count / GET_MODE_SIZE (mode);
10102 0 : gcc_assert (moves == 1);
10103 : }
10104 : }
10105 1882 : else if (moves > 8)
10106 : return false;
10107 :
10108 1873 : unsigned int i;
10109 1873 : rtx tmp[9];
10110 :
10111 4314 : for (i = 0; i < moves; i++)
10112 2441 : tmp[i] = gen_reg_rtx (mode);
10113 :
10114 1873 : rtx srcmem = change_address (src, mode, srcreg);
10115 6187 : for (i = 0; i < moves; i++)
10116 : {
10117 2441 : emit_move_insn (tmp[i], srcmem);
10118 4882 : srcmem = offset_address (srcmem,
10119 2441 : GEN_INT (GET_MODE_SIZE (mode)),
10120 2441 : GET_MODE_SIZE (mode));
10121 : }
10122 :
10123 1873 : unsigned int epilogue_size = count & (GET_MODE_SIZE (mode) - 1);
10124 1873 : machine_mode epilogue_mode = VOIDmode;
10125 1873 : if (epilogue_size)
10126 : {
10127 : /* Handle the remaining bytes with overlapping move. */
10128 1700 : epilogue_mode = smallest_int_mode_for_size
10129 1700 : (epilogue_size * BITS_PER_UNIT).require ();
10130 1700 : tmp[8] = gen_reg_rtx (epilogue_mode);
10131 1700 : srcmem = adjust_address (srcmem, epilogue_mode, 0);
10132 1700 : srcmem = offset_address (srcmem, GEN_INT (epilogue_size), 1);
10133 3400 : srcmem = offset_address (srcmem,
10134 1700 : GEN_INT (-GET_MODE_SIZE (epilogue_mode)),
10135 1700 : GET_MODE_SIZE (epilogue_mode));
10136 1700 : emit_move_insn (tmp[8], srcmem);
10137 : }
10138 :
10139 1873 : rtx destmem = change_address (dst, mode, destreg);
10140 6187 : for (i = 0; i < moves; i++)
10141 : {
10142 2441 : emit_move_insn (destmem, tmp[i]);
10143 4882 : destmem = offset_address (destmem,
10144 2441 : GEN_INT (GET_MODE_SIZE (mode)),
10145 2441 : GET_MODE_SIZE (mode));
10146 : }
10147 :
10148 1873 : if (epilogue_size)
10149 : {
10150 : /* Use overlapping move. */
10151 1700 : destmem = adjust_address (destmem, epilogue_mode, 0);
10152 1700 : destmem = offset_address (destmem, GEN_INT (epilogue_size), 1);
10153 3400 : destmem = offset_address (destmem,
10154 1700 : GEN_INT (-GET_MODE_SIZE (epilogue_mode)),
10155 1700 : GET_MODE_SIZE (epilogue_mode));
10156 1700 : emit_move_insn (destmem, tmp[8]);
10157 : }
10158 :
10159 : return true;
10160 : }
10161 :
10162 : /* Value kind in MEMSET_VALS:
10163 :
10164 : memset_val_byte: The value rtx in QImode.
10165 : memset_val_word: The value rtx in word_mode.
10166 : memset_val_vector: The value rtx in QI vector mode.
10167 :
10168 : */
10169 : enum memset_val_kind
10170 : {
10171 : memset_val_byte = 0,
10172 : memset_val_word = 1,
10173 : memset_val_vector = 2,
10174 : memset_val_max = 3
10175 : };
10176 :
10177 : /* Return a value rtx in MODE for memset from MEMSET_VALS. */
10178 :
10179 : static rtx
10180 4281 : ix86_expand_memset_val (rtx *memset_vals, machine_mode mode)
10181 : {
10182 4281 : rtx byte_val = memset_vals[memset_val_byte];
10183 :
10184 4281 : if (mode == QImode)
10185 : return byte_val;
10186 4281 : else if (mode == word_mode)
10187 976 : return memset_vals[memset_val_word];
10188 :
10189 : /* All-zero/all-ones is a property of the original byte value, so
10190 : detect it once here rather than re-deriving it from each slot. */
10191 3305 : if (byte_val == const0_rtx)
10192 1254 : return CONST0_RTX (mode);
10193 2051 : if (byte_val == constm1_rtx)
10194 78 : return CONSTM1_RTX (mode);
10195 :
10196 1973 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
10197 : {
10198 684 : if (GET_MODE (memset_vals[memset_val_vector]) == mode)
10199 : return memset_vals[memset_val_vector];
10200 7 : return gen_rtx_SUBREG (mode, memset_vals[memset_val_vector], 0);
10201 : }
10202 :
10203 1289 : gcc_assert (mode == HImode || mode == SImode);
10204 1289 : return gen_rtx_SUBREG (mode, memset_vals[memset_val_word], 0);
10205 : }
10206 :
10207 : /* Expand memmove of size with MOVES * mode size and MOVES <= 4. If
10208 : FORWARD is true, copy forward. Otherwise copy backward. */
10209 :
10210 : static void
10211 2433 : ix86_expand_n_move_set_or_movmem (rtx destmem, rtx srcmem,
10212 : rtx *memset_vals, machine_mode mode,
10213 : unsigned int moves, bool forward)
10214 : {
10215 2433 : gcc_assert (moves <= 4);
10216 :
10217 2433 : unsigned int i;
10218 2433 : rtx tmp[8];
10219 :
10220 2433 : rtx step;
10221 2433 : if (forward)
10222 2560 : step = GEN_INT (GET_MODE_SIZE (mode));
10223 : else
10224 2306 : step = GEN_INT (-GET_MODE_SIZE (mode));
10225 :
10226 2433 : if (memset_vals)
10227 : {
10228 : /* Expand memset. */
10229 99 : rtx val = ix86_expand_memset_val (memset_vals, mode);
10230 594 : for (i = 0; i < moves; i++)
10231 396 : tmp[i] = val;
10232 : }
10233 : else
10234 : {
10235 : /* Expand memmove. */
10236 11670 : for (i = 0; i < moves; i++)
10237 9336 : tmp[i] = gen_reg_rtx (mode);
10238 :
10239 : /* Load MOVES. */
10240 9336 : for (i = 0; i < moves - 1; i++)
10241 : {
10242 7002 : emit_move_insn (tmp[i], srcmem);
10243 14004 : srcmem = offset_address (srcmem, step, GET_MODE_SIZE (mode));
10244 : }
10245 2334 : emit_move_insn (tmp[i], srcmem);
10246 : }
10247 :
10248 : /* Store MOVES. */
10249 9732 : for (i = 0; i < moves - 1; i++)
10250 : {
10251 7299 : emit_move_insn (destmem, tmp[i]);
10252 14598 : destmem = offset_address (destmem, step, GET_MODE_SIZE (mode));
10253 : }
10254 2433 : emit_move_insn (destmem, tmp[i]);
10255 2433 : }
10256 :
10257 : /* Load MOVES of mode size into REGS. If LAST is true, load the
10258 : last MOVES. Otherwise, load the first MOVES. */
10259 :
10260 : static void
10261 2334 : ix86_expand_load_movmem (rtx src, rtx srcreg, rtx count_exp,
10262 : machine_mode mode, unsigned int moves,
10263 : rtx regs[], bool last)
10264 : {
10265 2334 : unsigned int i;
10266 :
10267 11670 : for (i = 0; i < moves; i++)
10268 9336 : regs[i] = gen_reg_rtx (mode);
10269 :
10270 2334 : rtx srcmem = change_address (src, mode, srcreg);
10271 2334 : rtx step;
10272 2334 : if (last)
10273 : {
10274 1181 : srcmem = offset_address (srcmem, count_exp, 1);
10275 2362 : step = GEN_INT (-GET_MODE_SIZE (mode));
10276 2362 : srcmem = offset_address (srcmem, step, GET_MODE_SIZE (mode));
10277 : }
10278 : else
10279 2306 : step = GEN_INT (GET_MODE_SIZE (mode));
10280 :
10281 9336 : for (i = 0; i < moves - 1; i++)
10282 : {
10283 7002 : emit_move_insn (regs[i], srcmem);
10284 14004 : srcmem = offset_address (srcmem, step, GET_MODE_SIZE (mode));
10285 : }
10286 2334 : emit_move_insn (regs[i], srcmem);
10287 2334 : }
10288 :
10289 : /* Store MOVES of mode size into REGS. If LAST is true, store the
10290 : last MOVES. Otherwise, store the first MOVES. */
10291 :
10292 : static void
10293 2433 : ix86_expand_store_movmem (rtx dst, rtx destreg, rtx count_exp,
10294 : machine_mode mode, unsigned int moves,
10295 : rtx regs[], bool last)
10296 : {
10297 2433 : unsigned int i;
10298 :
10299 2433 : rtx destmem = change_address (dst, mode, destreg);
10300 2433 : rtx step;
10301 2433 : if (last)
10302 : {
10303 1280 : destmem = offset_address (destmem, count_exp, 1);
10304 2560 : step = GEN_INT (-GET_MODE_SIZE (mode));
10305 2560 : destmem = offset_address (destmem, step, GET_MODE_SIZE (mode));
10306 : }
10307 : else
10308 2306 : step = GEN_INT (GET_MODE_SIZE (mode));
10309 :
10310 9732 : for (i = 0; i < moves - 1; i++)
10311 : {
10312 7299 : emit_move_insn (destmem, regs[i]);
10313 14598 : destmem = offset_address (destmem, step, GET_MODE_SIZE (mode));
10314 : }
10315 2433 : emit_move_insn (destmem, regs[i]);
10316 2433 : }
10317 :
10318 : /* Expand memmove of size between (MOVES / 2) * mode size and
10319 : MOVES * mode size with overlapping load and store. MOVES is even.
10320 : MOVES >= 2 and MOVES <= 8. */
10321 :
10322 : static void
10323 43574 : ix86_expand_n_overlapping_move_set_or_movmem (rtx dst, rtx src,
10324 : rtx *memset_vals,
10325 : rtx destreg, rtx srcreg,
10326 : rtx count_exp,
10327 : machine_mode mode,
10328 : unsigned int moves)
10329 : {
10330 43574 : gcc_assert (moves >= 2 && moves <= 8 && (moves & 1) == 0);
10331 :
10332 43574 : unsigned int half_moves = moves / 2;
10333 43574 : unsigned int i, j;
10334 43574 : rtx tmp[8];
10335 :
10336 43574 : if (memset_vals)
10337 : {
10338 : /* Expand memset. */
10339 4083 : rtx val = ix86_expand_memset_val (memset_vals, mode);
10340 18296 : for (i = 0; i < moves; i++)
10341 10130 : tmp[i] = val;
10342 : }
10343 : else
10344 : {
10345 : /* Expand memmove. */
10346 133139 : for (i = 0; i < moves; i++)
10347 93648 : tmp[i] = gen_reg_rtx (mode);
10348 :
10349 39491 : rtx base_srcmem = change_address (src, mode, srcreg);
10350 :
10351 : /* Load the first half. */
10352 39491 : rtx srcmem = base_srcmem;
10353 86315 : for (i = 0; i < half_moves - 1; i++)
10354 : {
10355 7333 : emit_move_insn (tmp[i], srcmem);
10356 14666 : srcmem = offset_address (srcmem,
10357 7333 : GEN_INT (GET_MODE_SIZE (mode)),
10358 7333 : GET_MODE_SIZE (mode));
10359 : }
10360 39491 : emit_move_insn (tmp[i], srcmem);
10361 :
10362 : /* Load the second half. */
10363 39491 : srcmem = offset_address (base_srcmem, count_exp, 1);
10364 39491 : srcmem = offset_address (srcmem,
10365 39491 : GEN_INT (-GET_MODE_SIZE (mode)),
10366 39491 : GET_MODE_SIZE (mode));
10367 86315 : for (j = half_moves, i = 0; i < half_moves - 1; i++, j++)
10368 : {
10369 7333 : emit_move_insn (tmp[j], srcmem);
10370 14666 : srcmem = offset_address (srcmem,
10371 7333 : GEN_INT (-GET_MODE_SIZE (mode)),
10372 7333 : GET_MODE_SIZE (mode));
10373 : }
10374 39491 : emit_move_insn (tmp[j], srcmem);
10375 : }
10376 :
10377 43574 : rtx base_destmem = change_address (dst, mode, destreg);
10378 :
10379 : /* Store the first half. */
10380 43574 : rtx destmem = base_destmem;
10381 95463 : for (i = 0; i < half_moves - 1; i++)
10382 : {
10383 8315 : emit_move_insn (destmem, tmp[i]);
10384 16630 : destmem = offset_address (destmem,
10385 8315 : GEN_INT (GET_MODE_SIZE (mode)),
10386 8315 : GET_MODE_SIZE (mode));
10387 : }
10388 43574 : emit_move_insn (destmem, tmp[i]);
10389 :
10390 : /* Store the second half. */
10391 43574 : destmem = offset_address (base_destmem, count_exp, 1);
10392 87148 : destmem = offset_address (destmem, GEN_INT (-GET_MODE_SIZE (mode)),
10393 43574 : GET_MODE_SIZE (mode));
10394 95463 : for (j = half_moves, i = 0; i < half_moves - 1; i++, j++)
10395 : {
10396 8315 : emit_move_insn (destmem, tmp[j]);
10397 16630 : destmem = offset_address (destmem, GEN_INT (-GET_MODE_SIZE (mode)),
10398 8315 : GET_MODE_SIZE (mode));
10399 : }
10400 43574 : emit_move_insn (destmem, tmp[j]);
10401 43574 : }
10402 :
10403 : /* Expand memmove of size < mode size which is <= 64. */
10404 :
10405 : static void
10406 10769 : ix86_expand_less_move_set_or_movmem (rtx dst, rtx src, rtx *memset_vals,
10407 : rtx destreg, rtx srcreg,
10408 : rtx count_exp,
10409 : unsigned HOST_WIDE_INT min_size,
10410 : machine_mode mode,
10411 : rtx_code_label *done_label)
10412 : {
10413 10769 : bool skip = false;
10414 10769 : machine_mode count_mode = counter_mode (count_exp);
10415 :
10416 10769 : rtx_code_label *between_32_63_label
10417 10769 : = GET_MODE_SIZE (mode) > 32 ? gen_label_rtx () : nullptr;
10418 : /* Jump to BETWEEN_32_64_LABEL if size >= 32 and size < 64. */
10419 7 : if (between_32_63_label)
10420 : {
10421 7 : if (min_size && min_size >= 32)
10422 : {
10423 1 : emit_jump_insn (gen_jump (between_32_63_label));
10424 1 : emit_barrier ();
10425 1 : skip = true;
10426 : }
10427 : else
10428 6 : emit_cmp_and_jump_insns (count_exp, GEN_INT (32), GEU,
10429 : nullptr, count_mode, 1,
10430 : between_32_63_label);
10431 : }
10432 :
10433 7 : rtx_code_label *between_16_31_label
10434 10768 : = (!skip && GET_MODE_SIZE (mode) > 16) ? gen_label_rtx () : nullptr;
10435 : /* Jump to BETWEEN_16_31_LABEL if size >= 16 and size < 31. */
10436 21 : if (between_16_31_label)
10437 : {
10438 21 : if (min_size && min_size >= 16)
10439 : {
10440 2 : emit_jump_insn (gen_jump (between_16_31_label));
10441 2 : emit_barrier ();
10442 2 : skip = true;
10443 : }
10444 : else
10445 19 : emit_cmp_and_jump_insns (count_exp, GEN_INT (16), GEU,
10446 : nullptr, count_mode, 1,
10447 : between_16_31_label);
10448 : }
10449 :
10450 2 : rtx_code_label *between_8_15_label
10451 21533 : = (!skip && GET_MODE_SIZE (mode) > 8) ? gen_label_rtx () : nullptr;
10452 : /* Jump to BETWEEN_8_15_LABEL if size >= 8 and size < 15. */
10453 8587 : if (between_8_15_label)
10454 : {
10455 8587 : if (min_size && min_size >= 8)
10456 : {
10457 481 : emit_jump_insn (gen_jump (between_8_15_label));
10458 481 : emit_barrier ();
10459 481 : skip = true;
10460 : }
10461 : else
10462 8106 : emit_cmp_and_jump_insns (count_exp, GEN_INT (8), GEU,
10463 : nullptr, count_mode, 1,
10464 : between_8_15_label);
10465 : }
10466 :
10467 481 : rtx_code_label *between_4_7_label
10468 20573 : = (!skip && GET_MODE_SIZE (mode) > 4) ? gen_label_rtx () : nullptr;
10469 : /* Jump to BETWEEN_4_7_LABEL if size >= 4 and size < 7. */
10470 9391 : if (between_4_7_label)
10471 : {
10472 9391 : if (min_size && min_size >= 4)
10473 : {
10474 443 : emit_jump_insn (gen_jump (between_4_7_label));
10475 443 : emit_barrier ();
10476 443 : skip = true;
10477 : }
10478 : else
10479 8948 : emit_cmp_and_jump_insns (count_exp, GEN_INT (4), GEU,
10480 : nullptr, count_mode, 1,
10481 : between_4_7_label);
10482 : }
10483 :
10484 443 : rtx_code_label *between_2_3_label
10485 20168 : = (!skip && GET_MODE_SIZE (mode) > 2) ? gen_label_rtx () : nullptr;
10486 : /* Jump to BETWEEN_2_3_LABEL if size >= 2 and size < 3. */
10487 9624 : if (between_2_3_label)
10488 : {
10489 9624 : if (min_size && min_size >= 2)
10490 : {
10491 1919 : emit_jump_insn (gen_jump (between_2_3_label));
10492 1919 : emit_barrier ();
10493 1919 : skip = true;
10494 : }
10495 : else
10496 7705 : emit_cmp_and_jump_insns (count_exp, GEN_INT (1), GT,
10497 : nullptr, count_mode, 1,
10498 : between_2_3_label);
10499 : }
10500 :
10501 10769 : if (!skip)
10502 : {
10503 7923 : rtx_code_label *zero_label
10504 7923 : = min_size == 0 ? gen_label_rtx () : nullptr;
10505 : /* Skip if size == 0. */
10506 2711 : if (zero_label)
10507 2711 : emit_cmp_and_jump_insns (count_exp, GEN_INT (1), LT,
10508 : nullptr, count_mode, 1,
10509 : zero_label,
10510 : profile_probability::unlikely ());
10511 :
10512 : /* Move 1 byte. */
10513 7923 : rtx tmp0;
10514 : /* Use the value rtx in QImode for memset. */
10515 7923 : if (memset_vals)
10516 904 : tmp0 = memset_vals[memset_val_byte];
10517 : else
10518 : {
10519 7019 : tmp0 = gen_reg_rtx (QImode);
10520 7019 : rtx srcmem = change_address (src, QImode, srcreg);
10521 7019 : emit_move_insn (tmp0, srcmem);
10522 : }
10523 7923 : rtx destmem = change_address (dst, QImode, destreg);
10524 7923 : emit_move_insn (destmem, tmp0);
10525 :
10526 7923 : if (zero_label)
10527 2711 : emit_label (zero_label);
10528 :
10529 7923 : emit_jump_insn (gen_jump (done_label));
10530 7923 : emit_barrier ();
10531 : }
10532 :
10533 : /* For each size band, memset uses a QI-vector mode above a word so it
10534 : can broadcast the fill value, while memmove uses the same-size
10535 : scalar integer mode; at and below a word both use the scalar
10536 : integer mode. */
10537 10769 : struct {
10538 : rtx_code_label *label;
10539 : machine_mode set_mode;
10540 : machine_mode move_mode;
10541 10769 : } bands[] = {
10542 : { between_32_63_label, V32QImode, OImode },
10543 : { between_16_31_label, V16QImode, TImode },
10544 : { between_8_15_label, DImode, DImode },
10545 : { between_4_7_label, SImode, SImode },
10546 : { between_2_3_label, HImode, HImode },
10547 10769 : };
10548 :
10549 64614 : for (auto &band : bands)
10550 53845 : if (band.label)
10551 : {
10552 27630 : emit_label (band.label);
10553 27630 : machine_mode bmode = memset_vals ? band.set_mode : band.move_mode;
10554 27630 : ix86_expand_n_overlapping_move_set_or_movmem (dst, src,
10555 : memset_vals,
10556 : destreg, srcreg,
10557 : count_exp, bmode,
10558 : 2);
10559 27630 : emit_jump_insn (gen_jump (done_label));
10560 27630 : emit_barrier ();
10561 : }
10562 10769 : }
10563 :
10564 : /* Expand movmem with overlapping unaligned loads and stores:
10565 : 1. Load all sources into registers and store them together to avoid
10566 : possible address overlap between source and destination.
10567 : 2. For known size, first try to fully unroll with 8 registers.
10568 : 3. For size <= 2 * MOVE_MAX, load all sources into 2 registers first
10569 : and then store them together.
10570 : 4. For size > 2 * MOVE_MAX and size <= 4 * MOVE_MAX, load all sources
10571 : into 4 registers first and then store them together.
10572 : 5. For size > 4 * MOVE_MAX and size <= 8 * MOVE_MAX, load all sources
10573 : into 8 registers first and then store them together.
10574 : 6. For size > 8 * MOVE_MAX,
10575 : a. If address of destination > address of source, copy backward
10576 : with a 4 * MOVE_MAX loop with unaligned loads and stores. Load
10577 : the first 4 * MOVE_MAX into 4 registers before the loop and
10578 : store them after the loop to support overlapping addresses.
10579 : b. Otherwise, copy forward with a 4 * MOVE_MAX loop with unaligned
10580 : loads and stores. Load the last 4 * MOVE_MAX into 4 registers
10581 : before the loop and store them after the loop to support
10582 : overlapping addresses.
10583 : */
10584 :
10585 : bool
10586 169843 : ix86_expand_set_or_movmem (rtx operands[], bool iscpymem, bool issetmem)
10587 : {
10588 : /* Since there are much less registers available in 32-bit mode, don't
10589 : inline movmem in 32-bit mode. */
10590 169843 : if (!TARGET_64BIT || optimize_insn_for_size_p ())
10591 33683 : return false;
10592 :
10593 136160 : rtx dst = operands[0];
10594 136160 : rtx src, memset_val_exp;
10595 136160 : if (issetmem)
10596 : {
10597 35279 : src = nullptr;
10598 35279 : memset_val_exp = operands[1];
10599 : }
10600 : else
10601 : {
10602 100881 : src = operands[1];
10603 100881 : memset_val_exp = nullptr;
10604 : }
10605 136160 : rtx count_exp = operands[2];
10606 136160 : rtx expected_size_exp = operands[5];
10607 136160 : rtx min_size_exp = operands[6];
10608 136160 : rtx max_size_exp = operands[7];
10609 136160 : rtx probable_max_size_exp = operands[8];
10610 136160 : unsigned HOST_WIDE_INT count = HOST_WIDE_INT_0U;
10611 136160 : HOST_WIDE_INT expected_size = HOST_WIDE_INT_M1U;
10612 136160 : unsigned HOST_WIDE_INT min_size = HOST_WIDE_INT_0U;
10613 136160 : unsigned HOST_WIDE_INT max_size = HOST_WIDE_INT_M1U;
10614 136160 : unsigned HOST_WIDE_INT probable_max_size = HOST_WIDE_INT_M1U;
10615 :
10616 136160 : if (CONST_INT_P (count_exp))
10617 : {
10618 54229 : min_size = max_size = probable_max_size = count = expected_size
10619 54229 : = INTVAL (count_exp);
10620 : /* When COUNT is 0, there is nothing to do. */
10621 54229 : if (!count)
10622 : return true;
10623 : }
10624 : else
10625 : {
10626 81931 : if (min_size_exp)
10627 81931 : min_size = INTVAL (min_size_exp);
10628 81931 : if (max_size_exp)
10629 62874 : max_size = INTVAL (max_size_exp);
10630 81931 : if (probable_max_size_exp)
10631 64860 : probable_max_size = INTVAL (probable_max_size_exp);
10632 81931 : if (CONST_INT_P (expected_size_exp))
10633 81931 : expected_size = INTVAL (expected_size_exp);
10634 :
10635 : /* NB: This assert may fail without the fixes for
10636 : https://gcc.gnu.org/bugzilla/show_bug.cgi?id=125977
10637 : */
10638 81931 : gcc_assert (min_size != max_size);
10639 : }
10640 :
10641 : /* Make sure we don't need to care about overflow later on. */
10642 54227 : if (count > (HOST_WIDE_INT_1U << 30))
10643 : return false;
10644 :
10645 135970 : addr_space_t dst_as = MEM_ADDR_SPACE (dst);
10646 135970 : addr_space_t src_as = (issetmem
10647 : ? ADDR_SPACE_GENERIC
10648 135970 : : MEM_ADDR_SPACE (src));
10649 135970 : int dynamic_check;
10650 135970 : bool noalign;
10651 135970 : enum stringop_alg alg = decide_alg (count, expected_size, min_size,
10652 : probable_max_size, issetmem,
10653 : (issetmem
10654 35216 : && memset_val_exp == const0_rtx),
10655 : dst_as, src_as, &dynamic_check,
10656 : &noalign, false);
10657 135970 : if (alg == libcall)
10658 : return false;
10659 :
10660 : /* Expand memcpy and memset like memmove only for bounded size. */
10661 18240 : if (iscpymem || issetmem)
10662 : {
10663 13995 : unsigned HOST_WIDE_INT unbounded
10664 13995 : = GET_MODE_MASK (counter_mode (count_exp));
10665 13995 : if (count != 0 /* Fixed size. */
10666 13995 : || max_size == 0 /* Unbounded size. */
10667 8970 : || max_size == unbounded) /* Unbounded size. */
10668 : return false;
10669 : }
10670 :
10671 12887 : rtx destreg = ix86_copy_addr_to_reg (XEXP (dst, 0));
10672 12887 : rtx srcreg = (issetmem
10673 12887 : ? nullptr
10674 11756 : : ix86_copy_addr_to_reg (XEXP (src, 0)));
10675 :
10676 12887 : unsigned int move_max = MOVE_MAX;
10677 12887 : machine_mode mode = smallest_int_mode_for_size
10678 12887 : (move_max * BITS_PER_UNIT).require ();
10679 12887 : if (probable_max_size && probable_max_size < move_max)
10680 : {
10681 : /* Get a usable MOVE_MAX. */
10682 3794 : mode = smallest_int_mode_for_size
10683 3794 : (probable_max_size * BITS_PER_UNIT).require ();
10684 : /* Reduce MOVE_MAX by half so that MOVE_MAX can be used. */
10685 7588 : if (GET_MODE_SIZE (mode) > probable_max_size)
10686 3266 : mode = smallest_int_mode_for_size
10687 3266 : (GET_MODE_BITSIZE (mode) / 2).require ();
10688 7588 : move_max = GET_MODE_SIZE (mode);
10689 : }
10690 :
10691 : /* Try to fully unroll memmove of known size first. */
10692 12887 : if (count
10693 12887 : && ix86_expand_unroll_movmem (dst, src, destreg, srcreg, count,
10694 : mode))
10695 : return true;
10696 :
10697 11014 : rtx memset_vals[memset_val_max];
10698 11014 : rtx *memset_vals_p;
10699 11014 : if (issetmem)
10700 : {
10701 : /* Use vector mode if MODE size > word size. */
10702 1131 : unsigned int size = GET_MODE_SIZE (mode);
10703 1131 : poly_uint64 nunits;
10704 1131 : if (size > UNITS_PER_WORD)
10705 : {
10706 628 : nunits = size / GET_MODE_SIZE (QImode);
10707 628 : mode = mode_for_vector (QImode, nunits).require ();
10708 : }
10709 :
10710 : /* Populate MEMSET_VALS to expand memset. */
10711 1131 : rtx val_word;
10712 1131 : memset_vals[memset_val_byte] = memset_val_exp;
10713 1131 : if (memset_val_exp == const0_rtx || memset_val_exp == constm1_rtx)
10714 : val_word = memset_val_exp;
10715 : else
10716 657 : val_word = promote_duplicated_reg (word_mode, memset_val_exp);
10717 1131 : memset_vals[memset_val_word] = val_word;
10718 2262 : if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
10719 : {
10720 628 : if (memset_val_exp == const0_rtx)
10721 242 : memset_vals[memset_val_vector] = CONST0_RTX (mode);
10722 386 : else if (memset_val_exp == constm1_rtx)
10723 18 : memset_vals[memset_val_vector] = CONSTM1_RTX (mode);
10724 : else
10725 : {
10726 : /* Use the vector mode based on WORD_MODE to avoid extra
10727 : GPR moves. */
10728 736 : nunits = size / GET_MODE_SIZE (word_mode);
10729 368 : machine_mode vector_mode
10730 368 : = mode_for_vector (word_mode, nunits).require ();
10731 368 : rtx vector = promote_duplicated_reg (vector_mode,
10732 : val_word);
10733 368 : memset_vals[memset_val_vector]
10734 368 : = convert_to_mode (mode, vector, 1);
10735 : }
10736 : }
10737 : else
10738 503 : memset_vals[memset_val_vector] = nullptr;
10739 1131 : memset_vals_p = memset_vals;
10740 : }
10741 : else
10742 : memset_vals_p = nullptr;
10743 :
10744 11014 : rtx_code_label *done_label = gen_label_rtx ();
10745 :
10746 11014 : rtx_code_label *less_vec_label = nullptr;
10747 11014 : if (min_size == 0 || min_size < move_max)
10748 10769 : less_vec_label = gen_label_rtx ();
10749 :
10750 11014 : machine_mode count_mode = counter_mode (count_exp);
10751 :
10752 : /* Jump to LESS_VEC_LABEL if size < MOVE_MAX. */
10753 11014 : if (less_vec_label)
10754 10769 : emit_cmp_and_jump_insns (count_exp, GEN_INT (move_max), LTU,
10755 : nullptr, count_mode, 1,
10756 : less_vec_label);
10757 :
10758 11014 : rtx_code_label *more_2x_vec_label = nullptr;
10759 11014 : if (probable_max_size == 0 || probable_max_size > 2 * move_max)
10760 3314 : more_2x_vec_label = gen_label_rtx ();
10761 :
10762 : /* Jump to MORE_2X_VEC_LABEL if size > 2 * MOVE_MAX. */
10763 3314 : if (more_2x_vec_label)
10764 3314 : emit_cmp_and_jump_insns (count_exp, GEN_INT (2 * move_max), GTU,
10765 : nullptr, count_mode, 1,
10766 : more_2x_vec_label);
10767 :
10768 11014 : if (max_size != 1 && (min_size == 0 || min_size <= 2 * move_max))
10769 : {
10770 : /* Max size != 1, size >= MOVE_MAX and size <= 2 * MOVE_MAX. */
10771 10971 : ix86_expand_n_overlapping_move_set_or_movmem (dst, src,
10772 : memset_vals_p,
10773 : destreg, srcreg,
10774 : count_exp, mode, 2);
10775 10971 : emit_jump_insn (gen_jump (done_label));
10776 10971 : emit_barrier ();
10777 : }
10778 :
10779 11014 : if (less_vec_label)
10780 : {
10781 : /* Size < MOVE_MAX. */
10782 10769 : emit_label (less_vec_label);
10783 10769 : ix86_expand_less_move_set_or_movmem (dst, src, memset_vals_p,
10784 : destreg, srcreg, count_exp,
10785 : min_size, mode, done_label);
10786 10769 : emit_jump_insn (gen_jump (done_label));
10787 10769 : emit_barrier ();
10788 : }
10789 :
10790 11014 : if (more_2x_vec_label)
10791 : {
10792 : /* Size > 2 * MOVE_MAX and destination may overlap with source. */
10793 3314 : emit_label (more_2x_vec_label);
10794 :
10795 3314 : rtx_code_label *more_8x_vec_label = nullptr;
10796 3314 : if (probable_max_size == 0 || probable_max_size > 8 * move_max)
10797 1280 : more_8x_vec_label = gen_label_rtx ();
10798 :
10799 : /* Jump to MORE_8X_VEC_LABEL if size > 8 * MOVE_MAX. */
10800 1280 : if (more_8x_vec_label)
10801 1280 : emit_cmp_and_jump_insns (count_exp, GEN_INT (8 * move_max), GTU,
10802 : nullptr, count_mode, 1,
10803 : more_8x_vec_label);
10804 :
10805 3314 : rtx_code_label *last_4x_vec_label = nullptr;
10806 3314 : if (min_size == 0 || min_size <= 4 * move_max)
10807 3302 : last_4x_vec_label = gen_label_rtx ();
10808 :
10809 : /* Jump to LAST_4X_VEC_LABEL if size <= 4 * MOVE_MAX. */
10810 3302 : if (last_4x_vec_label)
10811 3302 : emit_cmp_and_jump_insns (count_exp, GEN_INT (4 * move_max), LEU,
10812 : nullptr, count_mode, 1,
10813 : last_4x_vec_label);
10814 :
10815 3314 : if (probable_max_size == 0 || probable_max_size > 4 * move_max)
10816 : {
10817 : /* Size > 4 * MOVE_MAX and size <= 8 * MOVE_MAX. */
10818 1671 : ix86_expand_n_overlapping_move_set_or_movmem (dst, src,
10819 : memset_vals_p,
10820 : destreg, srcreg,
10821 : count_exp, mode,
10822 : 8);
10823 1671 : emit_jump_insn (gen_jump (done_label));
10824 1671 : emit_barrier ();
10825 : }
10826 :
10827 3314 : if (last_4x_vec_label)
10828 : {
10829 : /* Size > 2 * MOVE_MAX and size <= 4 * MOVE_MAX. */
10830 3302 : emit_label (last_4x_vec_label);
10831 3302 : ix86_expand_n_overlapping_move_set_or_movmem (dst, src,
10832 : memset_vals_p,
10833 : destreg, srcreg,
10834 : count_exp, mode,
10835 : 4);
10836 3302 : emit_jump_insn (gen_jump (done_label));
10837 3302 : emit_barrier ();
10838 : }
10839 :
10840 3314 : if (more_8x_vec_label)
10841 : {
10842 : /* Size > 8 * MOVE_MAX. */
10843 1280 : emit_label (more_8x_vec_label);
10844 :
10845 1280 : rtx loop_count = gen_reg_rtx (count_mode);
10846 1280 : emit_move_insn (loop_count, count_exp);
10847 :
10848 1280 : rtx_code_label *more_8x_vec_backward_label;
10849 1280 : rtx base_destreg;
10850 1280 : rtx srcmem;
10851 1280 : rtx regs[4];
10852 1280 : if (iscpymem || issetmem)
10853 : {
10854 : /* Always store forward for memcpy and memset. */
10855 127 : more_8x_vec_backward_label = nullptr;
10856 127 : if (iscpymem)
10857 : {
10858 : /* Load the last 4 * MOVE_MAX for memcpy. */
10859 28 : ix86_expand_load_movmem (src, srcreg, count_exp, mode,
10860 : ARRAY_SIZE (regs), regs,
10861 : true);
10862 28 : srcmem = change_address (src, mode, srcreg);
10863 : }
10864 : else
10865 : {
10866 : /* Fill REGS with MEMSET_VALS for memset. */
10867 99 : rtx val = ix86_expand_memset_val (memset_vals, mode);
10868 495 : for (unsigned int i = 0; i < 4; i++)
10869 396 : regs[i] = val;
10870 : srcmem = nullptr;
10871 : }
10872 127 : base_destreg = gen_reg_rtx (GET_MODE (destreg));
10873 127 : emit_move_insn (base_destreg, destreg);
10874 : }
10875 : else
10876 : {
10877 : /* Jump to MORE_8X_VEC_BACKWARD_LABEL if source address is
10878 : lower than destination address. */
10879 1153 : more_8x_vec_backward_label = gen_label_rtx ();
10880 1153 : emit_cmp_and_jump_insns (srcreg, destreg, LTU, nullptr,
10881 1153 : GET_MODE (destreg), 1,
10882 : more_8x_vec_backward_label);
10883 :
10884 : /* Skip if source == destination which is less common. */
10885 1153 : emit_cmp_and_jump_insns (srcreg, destreg, EQ, nullptr,
10886 1153 : GET_MODE (destreg), 1, done_label,
10887 : profile_probability::unlikely ());
10888 :
10889 1153 : base_destreg = gen_reg_rtx (GET_MODE (destreg));
10890 1153 : emit_move_insn (base_destreg, destreg);
10891 :
10892 : /* Load the last 4 * MOVE_MAX. */
10893 1153 : ix86_expand_load_movmem (src, srcreg, count_exp, mode,
10894 : ARRAY_SIZE (regs), regs, true);
10895 :
10896 1153 : srcmem = change_address (src, mode, srcreg);
10897 : }
10898 :
10899 1280 : rtx destmem = change_address (dst, mode, destreg);
10900 :
10901 : /* Copy forward with a 4 * MOVE_MAX loop. */
10902 1280 : rtx_code_label *loop_4x_vec_forward_label = gen_label_rtx ();
10903 1280 : emit_label (loop_4x_vec_forward_label);
10904 :
10905 1280 : ix86_expand_n_move_set_or_movmem (destmem, srcmem,
10906 : memset_vals_p, mode, 4,
10907 : true);
10908 :
10909 1280 : rtx tmp;
10910 1280 : rtx delta = GEN_INT (4 * MOVE_MAX);
10911 :
10912 : /* Decrement LOOP_COUNT by 4 * MOVE_MAX. */
10913 1280 : tmp = expand_simple_binop (GET_MODE (loop_count), MINUS,
10914 : loop_count, delta, nullptr, 1,
10915 : OPTAB_DIRECT);
10916 1280 : if (tmp != loop_count)
10917 1280 : emit_move_insn (loop_count, tmp);
10918 :
10919 : /* Increment DESTREG and SRCREG by 4 * MOVE_MAX. */
10920 1280 : tmp = expand_simple_binop (GET_MODE (destreg), PLUS,
10921 : destreg, delta, nullptr, 1,
10922 : OPTAB_DIRECT);
10923 1280 : if (tmp != destreg)
10924 1280 : emit_move_insn (destreg, tmp);
10925 1280 : if (!issetmem)
10926 : {
10927 1181 : tmp = expand_simple_binop (GET_MODE (srcreg), PLUS,
10928 : srcreg, delta, nullptr, 1,
10929 : OPTAB_DIRECT);
10930 1181 : if (tmp != srcreg)
10931 1181 : emit_move_insn (srcreg, tmp);
10932 : }
10933 :
10934 : /* Stop if LOOP_EXP <= 4 * MOVE_MAX. */
10935 1280 : emit_cmp_and_jump_insns (loop_count, delta, GTU, nullptr,
10936 1280 : GET_MODE (loop_count), 1,
10937 : loop_4x_vec_forward_label);
10938 :
10939 : /* Store the last 4 * MOVE_MAX. */
10940 1280 : ix86_expand_store_movmem (dst, base_destreg, count_exp, mode,
10941 : ARRAY_SIZE (regs), regs, true);
10942 :
10943 1280 : emit_jump_insn (gen_jump (done_label));
10944 1280 : emit_barrier ();
10945 :
10946 1280 : if (more_8x_vec_backward_label)
10947 : {
10948 : /* Copy backward with a 4 * MOVE_MAX loop. */
10949 1153 : emit_label (more_8x_vec_backward_label);
10950 :
10951 1153 : base_destreg = gen_reg_rtx (GET_MODE (destreg));
10952 1153 : emit_move_insn (base_destreg, destreg);
10953 :
10954 : /* Load the first 4 * MOVE_MAX. */
10955 1153 : ix86_expand_load_movmem (src, srcreg, count_exp, mode,
10956 : ARRAY_SIZE (regs), regs, false);
10957 :
10958 : /* Increment DESTREG and SRCREG by COUNT_EXP. */
10959 1153 : tmp = expand_simple_binop (GET_MODE (destreg), PLUS,
10960 : destreg, count_exp, nullptr, 1,
10961 : OPTAB_DIRECT);
10962 1153 : if (tmp != destreg)
10963 1153 : emit_move_insn (destreg, tmp);
10964 1153 : tmp = expand_simple_binop (GET_MODE (srcreg), PLUS, srcreg,
10965 : count_exp, nullptr, 1,
10966 : OPTAB_DIRECT);
10967 1153 : if (tmp != srcreg)
10968 1153 : emit_move_insn (srcreg, tmp);
10969 :
10970 1153 : srcmem = change_address (src, mode, srcreg);
10971 1153 : destmem = change_address (dst, mode, destreg);
10972 2306 : rtx step = GEN_INT (-GET_MODE_SIZE (mode));
10973 1153 : srcmem = offset_address (srcmem, step,
10974 1153 : GET_MODE_SIZE (mode));
10975 1153 : destmem = offset_address (destmem, step,
10976 1153 : GET_MODE_SIZE (mode));
10977 :
10978 1153 : rtx_code_label *loop_4x_vec_backward_label
10979 1153 : = gen_label_rtx ();
10980 1153 : emit_label (loop_4x_vec_backward_label);
10981 :
10982 1153 : ix86_expand_n_move_set_or_movmem (destmem, srcmem,
10983 : memset_vals_p, mode, 4,
10984 : false);
10985 :
10986 : /* Decrement LOOP_COUNT by 4 * MOVE_MAX. */
10987 1153 : tmp = expand_simple_binop (GET_MODE (loop_count), MINUS,
10988 : loop_count, delta, nullptr, 1,
10989 : OPTAB_DIRECT);
10990 1153 : if (tmp != loop_count)
10991 1153 : emit_move_insn (loop_count, tmp);
10992 :
10993 : /* Decrement DESTREG and SRCREG by 4 * MOVE_MAX. */
10994 1153 : tmp = expand_simple_binop (GET_MODE (destreg), MINUS,
10995 : destreg, delta, nullptr, 1,
10996 : OPTAB_DIRECT);
10997 1153 : if (tmp != destreg)
10998 1153 : emit_move_insn (destreg, tmp);
10999 1153 : tmp = expand_simple_binop (GET_MODE (srcreg), MINUS,
11000 : srcreg, delta, nullptr, 1,
11001 : OPTAB_DIRECT);
11002 1153 : if (tmp != srcreg)
11003 1153 : emit_move_insn (srcreg, tmp);
11004 :
11005 : /* Stop if LOOP_EXP <= 4 * MOVE_MAX. */
11006 1153 : emit_cmp_and_jump_insns (loop_count, delta, GTU, nullptr,
11007 1153 : GET_MODE (loop_count), 1,
11008 : loop_4x_vec_backward_label);
11009 :
11010 : /* Store the first 4 * MOVE_MAX. */
11011 1153 : ix86_expand_store_movmem (dst, base_destreg, count_exp,
11012 : mode, ARRAY_SIZE (regs), regs,
11013 : false);
11014 :
11015 1153 : emit_jump_insn (gen_jump (done_label));
11016 1153 : emit_barrier ();
11017 : }
11018 : }
11019 : }
11020 :
11021 11014 : emit_label (done_label);
11022 :
11023 11014 : return true;
11024 : }
11025 :
11026 : /* Expand cmpstrn or memcmp. */
11027 :
11028 : bool
11029 171425 : ix86_expand_cmpstrn_or_cmpmem (rtx result, rtx src1, rtx src2,
11030 : rtx length, rtx align, bool is_cmpstrn)
11031 : {
11032 : /* Expand strncmp and memcmp only with -minline-all-stringops since
11033 : "repz cmpsb" can be much slower than strncmp and memcmp functions
11034 : implemented with vector instructions, see
11035 :
11036 : https://gcc.gnu.org/bugzilla/show_bug.cgi?id=43052
11037 : */
11038 171425 : if (!TARGET_INLINE_ALL_STRINGOPS)
11039 : return false;
11040 :
11041 : /* Can't use this if the user has appropriated ecx, esi or edi. */
11042 5796 : if (fixed_regs[CX_REG] || fixed_regs[SI_REG] || fixed_regs[DI_REG])
11043 : return false;
11044 :
11045 5796 : if (is_cmpstrn)
11046 : {
11047 : /* For strncmp, length is the maximum length, which can be larger
11048 : than actual string lengths. We can expand the cmpstrn pattern
11049 : to "repz cmpsb" only if one of the strings is a constant so
11050 : that expand_builtin_strncmp() can write the length argument to
11051 : be the minimum of the const string length and the actual length
11052 : argument. Otherwise, "repz cmpsb" may pass the 0 byte. */
11053 69 : tree t1 = MEM_EXPR (src1);
11054 69 : tree t2 = MEM_EXPR (src2);
11055 138 : if (!((t1 && TREE_CODE (t1) == MEM_REF
11056 69 : && TREE_CODE (TREE_OPERAND (t1, 0)) == ADDR_EXPR
11057 0 : && (TREE_CODE (TREE_OPERAND (TREE_OPERAND (t1, 0), 0))
11058 : == STRING_CST))
11059 69 : || (t2 && TREE_CODE (t2) == MEM_REF
11060 69 : && TREE_CODE (TREE_OPERAND (t2, 0)) == ADDR_EXPR
11061 69 : && (TREE_CODE (TREE_OPERAND (TREE_OPERAND (t2, 0), 0))
11062 : == STRING_CST))))
11063 : return false;
11064 : }
11065 :
11066 5796 : rtx addr1 = copy_addr_to_reg (XEXP (src1, 0));
11067 5796 : rtx addr2 = copy_addr_to_reg (XEXP (src2, 0));
11068 5796 : if (addr1 != XEXP (src1, 0))
11069 5796 : src1 = replace_equiv_address_nv (src1, addr1);
11070 5796 : if (addr2 != XEXP (src2, 0))
11071 5796 : src2 = replace_equiv_address_nv (src2, addr2);
11072 :
11073 : /* NB: Make a copy of the data length to avoid changing the original
11074 : data length by cmpstrnqi patterns. */
11075 5796 : length = ix86_zero_extend_to_Pmode (length);
11076 8711 : rtx lengthreg = gen_reg_rtx (Pmode);
11077 5796 : emit_move_insn (lengthreg, length);
11078 :
11079 : /* If we are testing strict equality, we can use known alignment to
11080 : good advantage. This may be possible with combine, particularly
11081 : once cc0 is dead. */
11082 5796 : if (CONST_INT_P (length))
11083 : {
11084 0 : if (length == const0_rtx)
11085 : {
11086 0 : emit_move_insn (result, const0_rtx);
11087 0 : return true;
11088 : }
11089 0 : emit_insn (gen_cmpstrnqi_nz_1 (addr1, addr2, lengthreg, align,
11090 : src1, src2));
11091 : }
11092 : else
11093 : {
11094 8711 : emit_insn (gen_cmp_1 (Pmode, lengthreg, lengthreg));
11095 5796 : emit_insn (gen_cmpstrnqi_1 (addr1, addr2, lengthreg, align,
11096 : src1, src2));
11097 : }
11098 :
11099 5796 : rtx out = gen_lowpart (QImode, result);
11100 5796 : emit_insn (gen_cmpintqi (out));
11101 5796 : emit_move_insn (result, gen_rtx_SIGN_EXTEND (SImode, out));
11102 :
11103 5796 : return true;
11104 : }
11105 :
11106 : /* Expand the appropriate insns for doing strlen if not just doing
11107 : repnz; scasb
11108 :
11109 : out = result, initialized with the start address
11110 : align_rtx = alignment of the address.
11111 : scratch = scratch register, initialized with the startaddress when
11112 : not aligned, otherwise undefined
11113 :
11114 : This is just the body. It needs the initializations mentioned above and
11115 : some address computing at the end. These things are done in i386.md. */
11116 :
11117 : static void
11118 11 : ix86_expand_strlensi_unroll_1 (rtx out, rtx src, rtx align_rtx)
11119 : {
11120 11 : int align;
11121 11 : rtx tmp;
11122 11 : rtx_code_label *align_2_label = NULL;
11123 11 : rtx_code_label *align_3_label = NULL;
11124 11 : rtx_code_label *align_4_label = gen_label_rtx ();
11125 11 : rtx_code_label *end_0_label = gen_label_rtx ();
11126 11 : rtx mem;
11127 11 : rtx tmpreg = gen_reg_rtx (SImode);
11128 11 : rtx scratch = gen_reg_rtx (SImode);
11129 11 : rtx cmp;
11130 :
11131 11 : align = 0;
11132 11 : if (CONST_INT_P (align_rtx))
11133 11 : align = INTVAL (align_rtx);
11134 :
11135 : /* Loop to check 1..3 bytes for null to get an aligned pointer. */
11136 :
11137 : /* Is there a known alignment and is it less than 4? */
11138 11 : if (align < 4)
11139 : {
11140 15 : rtx scratch1 = gen_reg_rtx (Pmode);
11141 11 : emit_move_insn (scratch1, out);
11142 : /* Is there a known alignment and is it not 2? */
11143 11 : if (align != 2)
11144 : {
11145 11 : align_3_label = gen_label_rtx (); /* Label when aligned to 3-byte */
11146 11 : align_2_label = gen_label_rtx (); /* Label when aligned to 2-byte */
11147 :
11148 : /* Leave just the 3 lower bits. */
11149 15 : align_rtx = expand_binop (Pmode, and_optab, scratch1, GEN_INT (3),
11150 : NULL_RTX, 0, OPTAB_WIDEN);
11151 :
11152 15 : emit_cmp_and_jump_insns (align_rtx, const0_rtx, EQ, NULL,
11153 11 : Pmode, 1, align_4_label);
11154 15 : emit_cmp_and_jump_insns (align_rtx, const2_rtx, EQ, NULL,
11155 11 : Pmode, 1, align_2_label);
11156 15 : emit_cmp_and_jump_insns (align_rtx, const2_rtx, GTU, NULL,
11157 11 : Pmode, 1, align_3_label);
11158 : }
11159 : else
11160 : {
11161 : /* Since the alignment is 2, we have to check 2 or 0 bytes;
11162 : check if is aligned to 4 - byte. */
11163 :
11164 0 : align_rtx = expand_binop (Pmode, and_optab, scratch1, const2_rtx,
11165 : NULL_RTX, 0, OPTAB_WIDEN);
11166 :
11167 0 : emit_cmp_and_jump_insns (align_rtx, const0_rtx, EQ, NULL,
11168 0 : Pmode, 1, align_4_label);
11169 : }
11170 :
11171 11 : mem = change_address (src, QImode, out);
11172 :
11173 : /* Now compare the bytes. */
11174 :
11175 : /* Compare the first n unaligned byte on a byte per byte basis. */
11176 11 : emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL,
11177 : QImode, 1, end_0_label);
11178 :
11179 : /* Increment the address. */
11180 11 : emit_insn (gen_add2_insn (out, const1_rtx));
11181 :
11182 : /* Not needed with an alignment of 2 */
11183 11 : if (align != 2)
11184 : {
11185 11 : emit_label (align_2_label);
11186 :
11187 11 : emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1,
11188 : end_0_label);
11189 :
11190 11 : emit_insn (gen_add2_insn (out, const1_rtx));
11191 :
11192 11 : emit_label (align_3_label);
11193 : }
11194 :
11195 11 : emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1,
11196 : end_0_label);
11197 :
11198 11 : emit_insn (gen_add2_insn (out, const1_rtx));
11199 : }
11200 :
11201 : /* Generate loop to check 4 bytes at a time. It is not a good idea to
11202 : align this loop. It gives only huge programs, but does not help to
11203 : speed up. */
11204 11 : emit_label (align_4_label);
11205 :
11206 11 : mem = change_address (src, SImode, out);
11207 11 : emit_move_insn (scratch, mem);
11208 11 : emit_insn (gen_add2_insn (out, GEN_INT (4)));
11209 :
11210 : /* This formula yields a nonzero result iff one of the bytes is zero.
11211 : This saves three branches inside loop and many cycles. */
11212 :
11213 11 : emit_insn (gen_addsi3 (tmpreg, scratch, GEN_INT (-0x01010101)));
11214 11 : emit_insn (gen_one_cmplsi2 (scratch, scratch));
11215 11 : emit_insn (gen_andsi3 (tmpreg, tmpreg, scratch));
11216 11 : emit_insn (gen_andsi3 (tmpreg, tmpreg,
11217 : gen_int_mode (0x80808080, SImode)));
11218 11 : emit_cmp_and_jump_insns (tmpreg, const0_rtx, EQ, 0, SImode, 1,
11219 : align_4_label);
11220 :
11221 11 : if (TARGET_CMOVE)
11222 : {
11223 11 : rtx reg = gen_reg_rtx (SImode);
11224 15 : rtx reg2 = gen_reg_rtx (Pmode);
11225 11 : emit_move_insn (reg, tmpreg);
11226 11 : emit_insn (gen_lshrsi3 (reg, reg, GEN_INT (16)));
11227 :
11228 : /* If zero is not in the first two bytes, move two bytes forward. */
11229 11 : emit_insn (gen_testsi_ccno_1 (tmpreg, GEN_INT (0x8080)));
11230 11 : tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
11231 11 : tmp = gen_rtx_EQ (VOIDmode, tmp, const0_rtx);
11232 11 : emit_insn (gen_rtx_SET (tmpreg,
11233 : gen_rtx_IF_THEN_ELSE (SImode, tmp,
11234 : reg,
11235 : tmpreg)));
11236 : /* Emit lea manually to avoid clobbering of flags. */
11237 15 : emit_insn (gen_rtx_SET (reg2, plus_constant (Pmode, out, 2)));
11238 :
11239 11 : tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
11240 11 : tmp = gen_rtx_EQ (VOIDmode, tmp, const0_rtx);
11241 15 : emit_insn (gen_rtx_SET (out,
11242 : gen_rtx_IF_THEN_ELSE (Pmode, tmp,
11243 : reg2,
11244 : out)));
11245 11 : }
11246 : else
11247 : {
11248 0 : rtx_code_label *end_2_label = gen_label_rtx ();
11249 : /* Is zero in the first two bytes? */
11250 :
11251 0 : emit_insn (gen_testsi_ccno_1 (tmpreg, GEN_INT (0x8080)));
11252 0 : tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
11253 0 : tmp = gen_rtx_NE (VOIDmode, tmp, const0_rtx);
11254 0 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
11255 : gen_rtx_LABEL_REF (VOIDmode, end_2_label),
11256 : pc_rtx);
11257 0 : tmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
11258 0 : JUMP_LABEL (tmp) = end_2_label;
11259 :
11260 : /* Not in the first two. Move two bytes forward. */
11261 0 : emit_insn (gen_lshrsi3 (tmpreg, tmpreg, GEN_INT (16)));
11262 0 : emit_insn (gen_add2_insn (out, const2_rtx));
11263 :
11264 0 : emit_label (end_2_label);
11265 :
11266 : }
11267 :
11268 : /* Avoid branch in fixing the byte. */
11269 11 : tmpreg = gen_lowpart (QImode, tmpreg);
11270 11 : emit_insn (gen_addqi3_cconly_overflow (tmpreg, tmpreg));
11271 11 : tmp = gen_rtx_REG (CCmode, FLAGS_REG);
11272 11 : cmp = gen_rtx_LTU (VOIDmode, tmp, const0_rtx);
11273 15 : emit_insn (gen_sub3_carry (Pmode, out, out, GEN_INT (3), tmp, cmp));
11274 :
11275 11 : emit_label (end_0_label);
11276 11 : }
11277 :
11278 : /* Expand strlen. */
11279 :
11280 : bool
11281 14609 : ix86_expand_strlen (rtx out, rtx src, rtx eoschar, rtx align)
11282 : {
11283 14609 : if (TARGET_UNROLL_STRLEN
11284 14609 : && TARGET_INLINE_ALL_STRINGOPS
11285 11 : && eoschar == const0_rtx
11286 11 : && optimize > 1)
11287 : {
11288 : /* The generic case of strlen expander is long. Avoid it's
11289 : expanding unless TARGET_INLINE_ALL_STRINGOPS. */
11290 15 : rtx addr = force_reg (Pmode, XEXP (src, 0));
11291 : /* Well it seems that some optimizer does not combine a call like
11292 : foo(strlen(bar), strlen(bar));
11293 : when the move and the subtraction is done here. It does calculate
11294 : the length just once when these instructions are done inside of
11295 : output_strlen_unroll(). But I think since &bar[strlen(bar)] is
11296 : often used and I use one fewer register for the lifetime of
11297 : output_strlen_unroll() this is better. */
11298 :
11299 11 : emit_move_insn (out, addr);
11300 :
11301 11 : ix86_expand_strlensi_unroll_1 (out, src, align);
11302 :
11303 : /* strlensi_unroll_1 returns the address of the zero at the end of
11304 : the string, like memchr(), so compute the length by subtracting
11305 : the start address. */
11306 11 : emit_insn (gen_sub2_insn (out, addr));
11307 11 : return true;
11308 : }
11309 : else
11310 : return false;
11311 : }
11312 :
11313 : /* For given symbol (function) construct code to compute address of it's PLT
11314 : entry in large x86-64 PIC model. */
11315 :
11316 : static rtx
11317 34 : construct_plt_address (rtx symbol)
11318 : {
11319 34 : rtx tmp, unspec;
11320 :
11321 34 : gcc_assert (SYMBOL_REF_P (symbol));
11322 34 : gcc_assert (ix86_cmodel == CM_LARGE_PIC && !TARGET_PECOFF);
11323 34 : gcc_assert (Pmode == DImode);
11324 :
11325 34 : tmp = gen_reg_rtx (Pmode);
11326 34 : unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, symbol), UNSPEC_PLTOFF);
11327 :
11328 34 : emit_move_insn (tmp, gen_rtx_CONST (Pmode, unspec));
11329 34 : emit_insn (gen_add2_insn (tmp, pic_offset_table_rtx));
11330 34 : return tmp;
11331 : }
11332 :
11333 : rtx_insn *
11334 6360428 : ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
11335 : rtx callarg2,
11336 : rtx pop, bool sibcall)
11337 : {
11338 6360428 : rtx vec[3];
11339 6360428 : rtx use = NULL, call;
11340 6360428 : unsigned int vec_len = 0;
11341 6360428 : tree fndecl;
11342 :
11343 6360428 : if (SYMBOL_REF_P (XEXP (fnaddr, 0)))
11344 : {
11345 6174329 : fndecl = SYMBOL_REF_DECL (XEXP (fnaddr, 0));
11346 6174329 : if (fndecl)
11347 : {
11348 5914309 : if (lookup_attribute ("interrupt",
11349 5914309 : TYPE_ATTRIBUTES (TREE_TYPE (fndecl))))
11350 1 : error ("interrupt service routine cannot be called directly");
11351 5914309 : if (fndecl == current_function_decl
11352 5914309 : && decl_binds_to_current_def_p (fndecl))
11353 11316 : cfun->machine->recursive_function = true;
11354 : }
11355 : }
11356 : else
11357 : fndecl = NULL_TREE;
11358 :
11359 6360428 : if (pop == const0_rtx)
11360 0 : pop = NULL;
11361 6360428 : gcc_assert (!TARGET_64BIT || !pop);
11362 :
11363 6360428 : rtx addr = XEXP (fnaddr, 0);
11364 6360428 : if (TARGET_MACHO && !TARGET_64BIT)
11365 : {
11366 : #if TARGET_MACHO
11367 : if (flag_pic && SYMBOL_REF_P (XEXP (fnaddr, 0)))
11368 : fnaddr = machopic_indirect_call_target (fnaddr);
11369 : #endif
11370 : }
11371 : else
11372 : {
11373 : /* Static functions and indirect calls don't need the pic register. Also,
11374 : check if PLT was explicitly avoided via no-plt or "noplt" attribute, making
11375 : it an indirect call. */
11376 6360428 : if (flag_pic
11377 530877 : && SYMBOL_REF_P (addr)
11378 6864341 : && ix86_call_use_plt_p (addr))
11379 : {
11380 402541 : if (flag_plt
11381 402541 : && (SYMBOL_REF_DECL (addr) == NULL_TREE
11382 402507 : || !lookup_attribute ("noplt",
11383 402507 : DECL_ATTRIBUTES (SYMBOL_REF_DECL (addr)))))
11384 : {
11385 402506 : if (!TARGET_64BIT
11386 223651 : || (ix86_cmodel == CM_LARGE_PIC
11387 : && DEFAULT_ABI != MS_ABI))
11388 : {
11389 536599 : use_reg (&use, gen_rtx_REG (Pmode,
11390 : REAL_PIC_OFFSET_TABLE_REGNUM));
11391 178889 : if (ix86_use_pseudo_pic_reg ())
11392 357744 : emit_move_insn (gen_rtx_REG (Pmode,
11393 178889 : REAL_PIC_OFFSET_TABLE_REGNUM),
11394 : pic_offset_table_rtx);
11395 : }
11396 : }
11397 35 : else if (!TARGET_PECOFF && !TARGET_MACHO)
11398 : {
11399 35 : if (TARGET_64BIT
11400 35 : && ix86_cmodel == CM_LARGE_PIC
11401 : && DEFAULT_ABI != MS_ABI)
11402 : {
11403 1 : fnaddr = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr),
11404 : UNSPEC_GOT);
11405 1 : fnaddr = gen_rtx_CONST (Pmode, fnaddr);
11406 1 : fnaddr = force_reg (Pmode, fnaddr);
11407 1 : fnaddr = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, fnaddr);
11408 : }
11409 34 : else if (TARGET_64BIT)
11410 : {
11411 38 : fnaddr = gen_rtx_UNSPEC (Pmode,
11412 : gen_rtvec (1, addr),
11413 : UNSPEC_GOTPCREL);
11414 38 : fnaddr = gen_rtx_CONST (Pmode, fnaddr);
11415 : }
11416 : else
11417 : {
11418 0 : fnaddr = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr),
11419 : UNSPEC_GOT);
11420 0 : fnaddr = gen_rtx_CONST (Pmode, fnaddr);
11421 0 : fnaddr = gen_rtx_PLUS (Pmode, pic_offset_table_rtx,
11422 : fnaddr);
11423 : }
11424 39 : fnaddr = gen_const_mem (Pmode, fnaddr);
11425 : /* Pmode may not be the same as word_mode for x32, which
11426 : doesn't support indirect branch via 32-bit memory slot.
11427 : Since x32 GOT slot is 64 bit with zero upper 32 bits,
11428 : indirect branch via x32 GOT slot is OK. */
11429 35 : if (GET_MODE (fnaddr) != word_mode)
11430 4 : fnaddr = gen_rtx_ZERO_EXTEND (word_mode, fnaddr);
11431 35 : fnaddr = gen_rtx_MEM (QImode, fnaddr);
11432 : }
11433 : }
11434 : }
11435 :
11436 : /* Skip setting up RAX register for -mskip-rax-setup when there are no
11437 : parameters passed in vector registers. */
11438 6360428 : if (TARGET_64BIT
11439 5520120 : && (INTVAL (callarg2) > 0
11440 5458800 : || (INTVAL (callarg2) == 0
11441 326718 : && (TARGET_SSE || !flag_skip_rax_setup))))
11442 : {
11443 388036 : rtx al = gen_rtx_REG (QImode, AX_REG);
11444 388036 : emit_move_insn (al, callarg2);
11445 388036 : use_reg (&use, al);
11446 : }
11447 :
11448 6360428 : if (ix86_cmodel == CM_LARGE_PIC
11449 : && !TARGET_PECOFF
11450 45 : && MEM_P (fnaddr)
11451 45 : && SYMBOL_REF_P (XEXP (fnaddr, 0))
11452 6360465 : && !local_symbolic_operand (XEXP (fnaddr, 0), VOIDmode))
11453 34 : fnaddr = gen_rtx_MEM (QImode, construct_plt_address (XEXP (fnaddr, 0)));
11454 : /* Since x32 GOT slot is 64 bit with zero upper 32 bits, indirect
11455 : branch via x32 GOT slot is OK. */
11456 6360394 : else if (TARGET_X32
11457 74 : && MEM_P (fnaddr)
11458 74 : && GET_CODE (XEXP (fnaddr, 0)) == ZERO_EXTEND
11459 8 : && GOT_memory_operand (XEXP (XEXP (fnaddr, 0), 0), Pmode)
11460 6360398 : && !TARGET_INDIRECT_BRANCH_REGISTER)
11461 : ;
11462 6360394 : else if (sibcall
11463 6360394 : ? !sibcall_insn_operand (XEXP (fnaddr, 0), word_mode)
11464 6229910 : : !call_insn_operand (XEXP (fnaddr, 0), word_mode))
11465 : {
11466 532 : fnaddr = convert_to_mode (word_mode, XEXP (fnaddr, 0), 1);
11467 532 : fnaddr = gen_rtx_MEM (QImode, copy_to_mode_reg (word_mode, fnaddr));
11468 : }
11469 :
11470 : /* PR100665: Hwasan may tag code pointer which is not supported by LAM,
11471 : mask off code pointers here.
11472 : TODO: also need to handle indirect jump. */
11473 6361456 : if (ix86_memtag_can_tag_addresses () && !fndecl
11474 6360452 : && sanitize_flags_p (SANITIZE_HWADDRESS))
11475 : {
11476 24 : rtx untagged_addr = ix86_memtag_untagged_pointer (XEXP (fnaddr, 0),
11477 : NULL_RTX);
11478 24 : fnaddr = gen_rtx_MEM (QImode, untagged_addr);
11479 : }
11480 :
11481 6360428 : call = gen_rtx_CALL (VOIDmode, fnaddr, callarg1);
11482 :
11483 6360428 : if (retval)
11484 2507173 : call = gen_rtx_SET (retval, call);
11485 6360428 : vec[vec_len++] = call;
11486 :
11487 6360428 : if (pop)
11488 : {
11489 450660 : pop = gen_rtx_PLUS (Pmode, stack_pointer_rtx, pop);
11490 225330 : pop = gen_rtx_SET (stack_pointer_rtx, pop);
11491 225330 : vec[vec_len++] = pop;
11492 : }
11493 :
11494 : /* Set here, but it may get cleared later. */
11495 5520120 : if (TARGET_64BIT_MS_ABI
11496 73426 : && (!callarg2 || INTVAL (callarg2) != -2)
11497 6426731 : && TARGET_CALL_MS2SYSV_XLOGUES)
11498 : {
11499 7046 : if (!TARGET_SSE)
11500 : ;
11501 :
11502 : /* Don't break hot-patched functions. */
11503 7046 : else if (ix86_function_ms_hook_prologue (current_function_decl))
11504 : ;
11505 :
11506 : /* TODO: Cases not yet examined. */
11507 7046 : else if (flag_split_stack)
11508 0 : warn_once_call_ms2sysv_xlogues ("-fsplit-stack");
11509 :
11510 : else
11511 : {
11512 7046 : gcc_assert (!reload_completed);
11513 7046 : cfun->machine->call_ms2sysv = true;
11514 : }
11515 : }
11516 :
11517 6360428 : if (TARGET_MACHO && TARGET_64BIT && !sibcall
11518 : && ((SYMBOL_REF_P (addr) && !SYMBOL_REF_LOCAL_P (addr))
11519 : || !fndecl || TREE_PUBLIC (fndecl)))
11520 : {
11521 : /* We allow public functions defined in a TU to bind locally for PIC
11522 : code (the default) on 64bit Mach-O.
11523 : If such functions are not inlined, we cannot tell at compile-time if
11524 : they will be called via the lazy symbol resolver (this can depend on
11525 : options given at link-time). Therefore, we must assume that the lazy
11526 : resolver could be used which clobbers R11 and R10. */
11527 : clobber_reg (&use, gen_rtx_REG (DImode, R11_REG));
11528 : clobber_reg (&use, gen_rtx_REG (DImode, R10_REG));
11529 : }
11530 :
11531 6360428 : if (vec_len > 1)
11532 225330 : call = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (vec_len, vec));
11533 6360428 : rtx_insn *call_insn = emit_call_insn (call);
11534 6360428 : if (use)
11535 566924 : CALL_INSN_FUNCTION_USAGE (call_insn) = use;
11536 :
11537 6360428 : return call_insn;
11538 : }
11539 :
11540 : /* Split simple return with popping POPC bytes from stack to indirect
11541 : branch with stack adjustment . */
11542 :
11543 : void
11544 0 : ix86_split_simple_return_pop_internal (rtx popc)
11545 : {
11546 0 : struct machine_function *m = cfun->machine;
11547 0 : rtx ecx = gen_rtx_REG (SImode, CX_REG);
11548 0 : rtx_insn *insn;
11549 :
11550 : /* There is no "pascal" calling convention in any 64bit ABI. */
11551 0 : gcc_assert (!TARGET_64BIT);
11552 :
11553 0 : insn = emit_insn (gen_pop (ecx));
11554 0 : m->fs.cfa_offset -= UNITS_PER_WORD;
11555 0 : m->fs.sp_offset -= UNITS_PER_WORD;
11556 :
11557 0 : rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
11558 0 : x = gen_rtx_SET (stack_pointer_rtx, x);
11559 0 : add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
11560 0 : add_reg_note (insn, REG_CFA_REGISTER, gen_rtx_SET (ecx, pc_rtx));
11561 0 : RTX_FRAME_RELATED_P (insn) = 1;
11562 :
11563 0 : x = gen_rtx_PLUS (Pmode, stack_pointer_rtx, popc);
11564 0 : x = gen_rtx_SET (stack_pointer_rtx, x);
11565 0 : insn = emit_insn (x);
11566 0 : add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
11567 0 : RTX_FRAME_RELATED_P (insn) = 1;
11568 :
11569 : /* Now return address is in ECX. */
11570 0 : emit_jump_insn (gen_simple_return_indirect_internal (ecx));
11571 0 : }
11572 :
11573 : /* Errors in the source file can cause expand_expr to return const0_rtx
11574 : where we expect a vector. To avoid crashing, use one of the vector
11575 : clear instructions. */
11576 :
11577 : static rtx
11578 199734 : safe_vector_operand (rtx x, machine_mode mode)
11579 : {
11580 0 : if (x == const0_rtx)
11581 0 : x = CONST0_RTX (mode);
11582 24 : return x;
11583 : }
11584 :
11585 : /* Subroutine of ix86_expand_builtin to take care of binop insns. */
11586 :
11587 : static rtx
11588 8991 : ix86_expand_binop_builtin (enum insn_code icode, tree exp, rtx target)
11589 : {
11590 8991 : rtx pat;
11591 8991 : tree arg0 = CALL_EXPR_ARG (exp, 0);
11592 8991 : tree arg1 = CALL_EXPR_ARG (exp, 1);
11593 8991 : rtx op0 = expand_normal (arg0);
11594 8991 : rtx op1 = expand_normal (arg1);
11595 8991 : machine_mode tmode = insn_data[icode].operand[0].mode;
11596 8991 : machine_mode mode0 = insn_data[icode].operand[1].mode;
11597 8991 : machine_mode mode1 = insn_data[icode].operand[2].mode;
11598 :
11599 8991 : if (VECTOR_MODE_P (mode0))
11600 8980 : op0 = safe_vector_operand (op0, mode0);
11601 8991 : if (VECTOR_MODE_P (mode1))
11602 8844 : op1 = safe_vector_operand (op1, mode1);
11603 :
11604 2836 : if (optimize || !target
11605 2836 : || GET_MODE (target) != tmode
11606 11827 : || !insn_data[icode].operand[0].predicate (target, tmode))
11607 6208 : target = gen_reg_rtx (tmode);
11608 :
11609 8991 : if (GET_MODE (op1) == SImode && mode1 == TImode)
11610 : {
11611 0 : rtx x = gen_reg_rtx (V4SImode);
11612 0 : emit_insn (gen_sse2_loadd (x, op1));
11613 0 : op1 = gen_lowpart (TImode, x);
11614 : }
11615 :
11616 8991 : if (!insn_data[icode].operand[1].predicate (op0, mode0))
11617 1393 : op0 = copy_to_mode_reg (mode0, op0);
11618 8991 : if (!insn_data[icode].operand[2].predicate (op1, mode1))
11619 818 : op1 = copy_to_mode_reg (mode1, op1);
11620 :
11621 8991 : pat = GEN_FCN (icode) (target, op0, op1);
11622 8991 : if (! pat)
11623 : return 0;
11624 :
11625 8991 : emit_insn (pat);
11626 :
11627 8991 : return target;
11628 : }
11629 :
11630 : /* Subroutine of ix86_expand_builtin to take care of 2-4 argument insns. */
11631 :
11632 : static rtx
11633 1813 : ix86_expand_multi_arg_builtin (enum insn_code icode, tree exp, rtx target,
11634 : enum ix86_builtin_func_type m_type,
11635 : enum rtx_code sub_code)
11636 : {
11637 1813 : rtx pat;
11638 1813 : unsigned int i, nargs;
11639 1813 : bool comparison_p = false;
11640 1813 : bool tf_p = false;
11641 1813 : bool last_arg_constant = false;
11642 1813 : int num_memory = 0;
11643 1813 : rtx xops[4];
11644 :
11645 1813 : machine_mode tmode = insn_data[icode].operand[0].mode;
11646 :
11647 1813 : switch (m_type)
11648 : {
11649 : case MULTI_ARG_4_DF2_DI_I:
11650 : case MULTI_ARG_4_DF2_DI_I1:
11651 : case MULTI_ARG_4_SF2_SI_I:
11652 : case MULTI_ARG_4_SF2_SI_I1:
11653 : nargs = 4;
11654 : last_arg_constant = true;
11655 : break;
11656 :
11657 842 : case MULTI_ARG_3_SF:
11658 842 : case MULTI_ARG_3_DF:
11659 842 : case MULTI_ARG_3_SF2:
11660 842 : case MULTI_ARG_3_DF2:
11661 842 : case MULTI_ARG_3_DI:
11662 842 : case MULTI_ARG_3_SI:
11663 842 : case MULTI_ARG_3_SI_DI:
11664 842 : case MULTI_ARG_3_HI:
11665 842 : case MULTI_ARG_3_HI_SI:
11666 842 : case MULTI_ARG_3_QI:
11667 842 : case MULTI_ARG_3_DI2:
11668 842 : case MULTI_ARG_3_SI2:
11669 842 : case MULTI_ARG_3_HI2:
11670 842 : case MULTI_ARG_3_QI2:
11671 842 : nargs = 3;
11672 842 : break;
11673 :
11674 128 : case MULTI_ARG_2_SF:
11675 128 : case MULTI_ARG_2_DF:
11676 128 : case MULTI_ARG_2_DI:
11677 128 : case MULTI_ARG_2_SI:
11678 128 : case MULTI_ARG_2_HI:
11679 128 : case MULTI_ARG_2_QI:
11680 128 : nargs = 2;
11681 128 : break;
11682 :
11683 64 : case MULTI_ARG_2_DI_IMM:
11684 64 : case MULTI_ARG_2_SI_IMM:
11685 64 : case MULTI_ARG_2_HI_IMM:
11686 64 : case MULTI_ARG_2_QI_IMM:
11687 64 : nargs = 2;
11688 64 : last_arg_constant = true;
11689 64 : break;
11690 :
11691 187 : case MULTI_ARG_1_SF:
11692 187 : case MULTI_ARG_1_DF:
11693 187 : case MULTI_ARG_1_SF2:
11694 187 : case MULTI_ARG_1_DF2:
11695 187 : case MULTI_ARG_1_DI:
11696 187 : case MULTI_ARG_1_SI:
11697 187 : case MULTI_ARG_1_HI:
11698 187 : case MULTI_ARG_1_QI:
11699 187 : case MULTI_ARG_1_SI_DI:
11700 187 : case MULTI_ARG_1_HI_DI:
11701 187 : case MULTI_ARG_1_HI_SI:
11702 187 : case MULTI_ARG_1_QI_DI:
11703 187 : case MULTI_ARG_1_QI_SI:
11704 187 : case MULTI_ARG_1_QI_HI:
11705 187 : nargs = 1;
11706 187 : break;
11707 :
11708 384 : case MULTI_ARG_2_DI_CMP:
11709 384 : case MULTI_ARG_2_SI_CMP:
11710 384 : case MULTI_ARG_2_HI_CMP:
11711 384 : case MULTI_ARG_2_QI_CMP:
11712 384 : nargs = 2;
11713 384 : comparison_p = true;
11714 384 : break;
11715 :
11716 128 : case MULTI_ARG_2_SF_TF:
11717 128 : case MULTI_ARG_2_DF_TF:
11718 128 : case MULTI_ARG_2_DI_TF:
11719 128 : case MULTI_ARG_2_SI_TF:
11720 128 : case MULTI_ARG_2_HI_TF:
11721 128 : case MULTI_ARG_2_QI_TF:
11722 128 : nargs = 2;
11723 128 : tf_p = true;
11724 128 : break;
11725 :
11726 0 : default:
11727 0 : gcc_unreachable ();
11728 : }
11729 :
11730 628 : if (optimize || !target
11731 628 : || GET_MODE (target) != tmode
11732 2417 : || !insn_data[icode].operand[0].predicate (target, tmode))
11733 1209 : target = gen_reg_rtx (tmode);
11734 604 : else if (memory_operand (target, tmode))
11735 0 : num_memory++;
11736 :
11737 1813 : gcc_assert (nargs <= ARRAY_SIZE (xops));
11738 :
11739 6246 : for (i = 0; i < nargs; i++)
11740 : {
11741 4441 : tree arg = CALL_EXPR_ARG (exp, i);
11742 4441 : rtx op = expand_normal (arg);
11743 4441 : int adjust = (comparison_p) ? 1 : 0;
11744 4441 : machine_mode mode = insn_data[icode].operand[i+adjust+1].mode;
11745 :
11746 4441 : if (last_arg_constant && i == nargs - 1)
11747 : {
11748 144 : if (!insn_data[icode].operand[i + 1].predicate (op, mode))
11749 : {
11750 30 : enum insn_code new_icode = icode;
11751 30 : switch (icode)
11752 : {
11753 8 : case CODE_FOR_xop_vpermil2v2df3:
11754 8 : case CODE_FOR_xop_vpermil2v4sf3:
11755 8 : case CODE_FOR_xop_vpermil2v4df3:
11756 8 : case CODE_FOR_xop_vpermil2v8sf3:
11757 8 : error ("the last argument must be a 2-bit immediate");
11758 8 : return gen_reg_rtx (tmode);
11759 5 : case CODE_FOR_xop_rotlv2di3:
11760 5 : new_icode = CODE_FOR_rotlv2di3;
11761 5 : goto xop_rotl;
11762 5 : case CODE_FOR_xop_rotlv4si3:
11763 5 : new_icode = CODE_FOR_rotlv4si3;
11764 5 : goto xop_rotl;
11765 6 : case CODE_FOR_xop_rotlv8hi3:
11766 6 : new_icode = CODE_FOR_rotlv8hi3;
11767 6 : goto xop_rotl;
11768 : case CODE_FOR_xop_rotlv16qi3:
11769 : new_icode = CODE_FOR_rotlv16qi3;
11770 22 : xop_rotl:
11771 22 : if (CONST_INT_P (op))
11772 : {
11773 6 : int mask = GET_MODE_UNIT_BITSIZE (tmode) - 1;
11774 6 : op = GEN_INT (INTVAL (op) & mask);
11775 6 : gcc_checking_assert
11776 : (insn_data[icode].operand[i + 1].predicate (op, mode));
11777 : }
11778 : else
11779 : {
11780 16 : gcc_checking_assert
11781 : (nargs == 2
11782 : && insn_data[new_icode].operand[0].mode == tmode
11783 : && insn_data[new_icode].operand[1].mode == tmode
11784 : && insn_data[new_icode].operand[2].mode == mode
11785 : && insn_data[new_icode].operand[0].predicate
11786 : == insn_data[icode].operand[0].predicate
11787 : && insn_data[new_icode].operand[1].predicate
11788 : == insn_data[icode].operand[1].predicate);
11789 16 : icode = new_icode;
11790 16 : goto non_constant;
11791 : }
11792 : break;
11793 0 : default:
11794 0 : gcc_unreachable ();
11795 : }
11796 : }
11797 : }
11798 : else
11799 : {
11800 4297 : non_constant:
11801 4313 : if (VECTOR_MODE_P (mode))
11802 4297 : op = safe_vector_operand (op, mode);
11803 :
11804 : /* If we aren't optimizing, only allow one memory operand to be
11805 : generated. */
11806 4313 : if (memory_operand (op, mode))
11807 826 : num_memory++;
11808 :
11809 4313 : gcc_assert (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode);
11810 :
11811 4313 : if (optimize
11812 1506 : || !insn_data[icode].operand[i+adjust+1].predicate (op, mode)
11813 5741 : || num_memory > 1)
11814 3392 : op = force_reg (mode, op);
11815 : }
11816 :
11817 4433 : xops[i] = op;
11818 : }
11819 :
11820 1805 : switch (nargs)
11821 : {
11822 187 : case 1:
11823 187 : pat = GEN_FCN (icode) (target, xops[0]);
11824 187 : break;
11825 :
11826 704 : case 2:
11827 704 : if (tf_p)
11828 128 : pat = GEN_FCN (icode) (target, xops[0], xops[1],
11829 128 : GEN_INT ((int)sub_code));
11830 576 : else if (! comparison_p)
11831 192 : pat = GEN_FCN (icode) (target, xops[0], xops[1]);
11832 : else
11833 : {
11834 384 : rtx cmp_op = gen_rtx_fmt_ee (sub_code, GET_MODE (target),
11835 : xops[0], xops[1]);
11836 :
11837 384 : pat = GEN_FCN (icode) (target, cmp_op, xops[0], xops[1]);
11838 : }
11839 : break;
11840 :
11841 842 : case 3:
11842 842 : pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2]);
11843 842 : break;
11844 :
11845 72 : case 4:
11846 72 : pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2], xops[3]);
11847 72 : break;
11848 :
11849 : default:
11850 : gcc_unreachable ();
11851 : }
11852 :
11853 1805 : if (! pat)
11854 : return 0;
11855 :
11856 1805 : emit_insn (pat);
11857 1805 : return target;
11858 : }
11859 :
11860 : /* Subroutine of ix86_expand_args_builtin to take care of scalar unop
11861 : insns with vec_merge. */
11862 :
11863 : static rtx
11864 52 : ix86_expand_unop_vec_merge_builtin (enum insn_code icode, tree exp,
11865 : rtx target)
11866 : {
11867 52 : rtx pat;
11868 52 : tree arg0 = CALL_EXPR_ARG (exp, 0);
11869 52 : rtx op1, op0 = expand_normal (arg0);
11870 52 : machine_mode tmode = insn_data[icode].operand[0].mode;
11871 52 : machine_mode mode0 = insn_data[icode].operand[1].mode;
11872 :
11873 16 : if (optimize || !target
11874 16 : || GET_MODE (target) != tmode
11875 68 : || !insn_data[icode].operand[0].predicate (target, tmode))
11876 36 : target = gen_reg_rtx (tmode);
11877 :
11878 52 : if (VECTOR_MODE_P (mode0))
11879 52 : op0 = safe_vector_operand (op0, mode0);
11880 :
11881 36 : if ((optimize && !register_operand (op0, mode0))
11882 88 : || !insn_data[icode].operand[1].predicate (op0, mode0))
11883 0 : op0 = copy_to_mode_reg (mode0, op0);
11884 :
11885 52 : op1 = op0;
11886 52 : if (!insn_data[icode].operand[2].predicate (op1, mode0))
11887 16 : op1 = copy_to_mode_reg (mode0, op1);
11888 :
11889 52 : pat = GEN_FCN (icode) (target, op0, op1);
11890 52 : if (! pat)
11891 : return 0;
11892 52 : emit_insn (pat);
11893 52 : return target;
11894 : }
11895 :
11896 : /* Subroutine of ix86_expand_builtin to take care of comparison insns. */
11897 :
11898 : static rtx
11899 616 : ix86_expand_sse_compare (const struct builtin_description *d,
11900 : tree exp, rtx target, bool swap)
11901 : {
11902 616 : rtx pat;
11903 616 : tree arg0 = CALL_EXPR_ARG (exp, 0);
11904 616 : tree arg1 = CALL_EXPR_ARG (exp, 1);
11905 616 : rtx op0 = expand_normal (arg0);
11906 616 : rtx op1 = expand_normal (arg1);
11907 616 : rtx op2;
11908 616 : machine_mode tmode = insn_data[d->icode].operand[0].mode;
11909 616 : machine_mode mode0 = insn_data[d->icode].operand[1].mode;
11910 616 : machine_mode mode1 = insn_data[d->icode].operand[2].mode;
11911 616 : enum rtx_code comparison = d->comparison;
11912 :
11913 616 : if (VECTOR_MODE_P (mode0))
11914 616 : op0 = safe_vector_operand (op0, mode0);
11915 616 : if (VECTOR_MODE_P (mode1))
11916 616 : op1 = safe_vector_operand (op1, mode1);
11917 :
11918 : /* Swap operands if we have a comparison that isn't available in
11919 : hardware. */
11920 616 : if (swap)
11921 80 : std::swap (op0, op1);
11922 :
11923 202 : if (optimize || !target
11924 202 : || GET_MODE (target) != tmode
11925 818 : || !insn_data[d->icode].operand[0].predicate (target, tmode))
11926 414 : target = gen_reg_rtx (tmode);
11927 :
11928 414 : if ((optimize && !register_operand (op0, mode0))
11929 958 : || !insn_data[d->icode].operand[1].predicate (op0, mode0))
11930 274 : op0 = copy_to_mode_reg (mode0, op0);
11931 414 : if ((optimize && !register_operand (op1, mode1))
11932 974 : || !insn_data[d->icode].operand[2].predicate (op1, mode1))
11933 56 : op1 = copy_to_mode_reg (mode1, op1);
11934 :
11935 616 : op2 = gen_rtx_fmt_ee (comparison, mode0, op0, op1);
11936 616 : pat = GEN_FCN (d->icode) (target, op0, op1, op2);
11937 616 : if (! pat)
11938 : return 0;
11939 616 : emit_insn (pat);
11940 616 : return target;
11941 : }
11942 :
11943 : /* Subroutine of ix86_sse_comi and ix86_sse_comi_round to take care of
11944 : * ordered EQ or unordered NE, generate PF jump. */
11945 :
11946 : static rtx
11947 646 : ix86_ssecom_setcc (const enum rtx_code comparison,
11948 : bool check_unordered, machine_mode mode,
11949 : rtx set_dst, rtx target)
11950 : {
11951 :
11952 646 : rtx_code_label *label = NULL;
11953 :
11954 : /* NB: For ordered EQ or unordered NE, check ZF alone isn't sufficient
11955 : with NAN operands.
11956 : Under TARGET_AVX10_2, VCOMX/VUCOMX are generated instead of
11957 : COMI/UCOMI. VCOMX/VUCOMX will not set ZF for NAN operands. */
11958 646 : if (check_unordered)
11959 : {
11960 122 : gcc_assert (comparison == EQ || comparison == NE);
11961 :
11962 122 : rtx flag = gen_rtx_REG (CCFPmode, FLAGS_REG);
11963 122 : label = gen_label_rtx ();
11964 122 : rtx tmp = gen_rtx_fmt_ee (UNORDERED, VOIDmode, flag, const0_rtx);
11965 122 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
11966 : gen_rtx_LABEL_REF (VOIDmode, label),
11967 : pc_rtx);
11968 122 : emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
11969 : }
11970 :
11971 : /* NB: Set CCFPmode and check a different CCmode which is in subset
11972 : of CCFPmode. */
11973 646 : if (GET_MODE (set_dst) != mode)
11974 : {
11975 200 : gcc_assert (mode == CCAmode || mode == CCCmode
11976 : || mode == CCOmode || mode == CCPmode
11977 : || mode == CCSmode || mode == CCZmode);
11978 200 : set_dst = gen_rtx_REG (mode, FLAGS_REG);
11979 : }
11980 :
11981 646 : emit_insn (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
11982 : gen_rtx_fmt_ee (comparison, QImode,
11983 : set_dst,
11984 : const0_rtx)));
11985 :
11986 646 : if (label)
11987 122 : emit_label (label);
11988 :
11989 646 : return SUBREG_REG (target);
11990 : }
11991 :
11992 : /* Subroutine of ix86_expand_builtin to take care of comi insns. */
11993 :
11994 : static rtx
11995 547 : ix86_expand_sse_comi (const struct builtin_description *d, tree exp,
11996 : rtx target, bool comx_ok)
11997 : {
11998 547 : rtx pat, set_dst;
11999 547 : tree arg0 = CALL_EXPR_ARG (exp, 0);
12000 547 : tree arg1 = CALL_EXPR_ARG (exp, 1);
12001 547 : rtx op0 = expand_normal (arg0);
12002 547 : rtx op1 = expand_normal (arg1);
12003 547 : enum insn_code icode = d->icode;
12004 547 : const struct insn_data_d *insn_p = &insn_data[icode];
12005 547 : machine_mode mode0 = insn_p->operand[0].mode;
12006 547 : machine_mode mode1 = insn_p->operand[1].mode;
12007 :
12008 547 : if (VECTOR_MODE_P (mode0))
12009 547 : op0 = safe_vector_operand (op0, mode0);
12010 547 : if (VECTOR_MODE_P (mode1))
12011 547 : op1 = safe_vector_operand (op1, mode1);
12012 :
12013 547 : enum rtx_code comparison = d->comparison;
12014 547 : rtx const_val = const0_rtx;
12015 :
12016 547 : bool check_unordered = false;
12017 547 : machine_mode mode = CCFPmode;
12018 547 : switch (comparison)
12019 : {
12020 194 : case LE: /* -> GE */
12021 194 : case LT: /* -> GT */
12022 194 : std::swap (op0, op1);
12023 194 : comparison = swap_condition (comparison);
12024 : /* FALLTHRU */
12025 : case GT:
12026 : case GE:
12027 : break;
12028 73 : case EQ:
12029 73 : if (!TARGET_AVX10_2 || !comx_ok)
12030 45 : check_unordered = true;
12031 : mode = CCZmode;
12032 : break;
12033 96 : case NE:
12034 96 : if (!TARGET_AVX10_2 || !comx_ok)
12035 68 : check_unordered = true;
12036 96 : mode = CCZmode;
12037 96 : const_val = const1_rtx;
12038 96 : break;
12039 0 : default:
12040 0 : gcc_unreachable ();
12041 : }
12042 :
12043 547 : target = gen_reg_rtx (SImode);
12044 547 : emit_move_insn (target, const_val);
12045 547 : target = gen_rtx_SUBREG (QImode, target, 0);
12046 :
12047 426 : if ((optimize && !register_operand (op0, mode0))
12048 925 : || !insn_p->operand[0].predicate (op0, mode0))
12049 169 : op0 = copy_to_mode_reg (mode0, op0);
12050 426 : if ((optimize && !register_operand (op1, mode1))
12051 924 : || !insn_p->operand[1].predicate (op1, mode1))
12052 49 : op1 = copy_to_mode_reg (mode1, op1);
12053 :
12054 547 : if ((comparison == EQ || comparison == NE)
12055 169 : && TARGET_AVX10_2 && comx_ok)
12056 : {
12057 56 : switch (icode)
12058 : {
12059 : case CODE_FOR_sse_comi:
12060 : icode = CODE_FOR_avx10_2_comxsf;
12061 : break;
12062 14 : case CODE_FOR_sse_ucomi:
12063 14 : icode = CODE_FOR_avx10_2_ucomxsf;
12064 14 : break;
12065 14 : case CODE_FOR_sse2_comi:
12066 14 : icode = CODE_FOR_avx10_2_comxdf;
12067 14 : break;
12068 14 : case CODE_FOR_sse2_ucomi:
12069 14 : icode = CODE_FOR_avx10_2_ucomxdf;
12070 14 : break;
12071 :
12072 0 : default:
12073 0 : gcc_unreachable ();
12074 : }
12075 : }
12076 547 : pat = GEN_FCN (icode) (op0, op1);
12077 547 : if (! pat)
12078 : return 0;
12079 :
12080 547 : set_dst = SET_DEST (pat);
12081 547 : emit_insn (pat);
12082 547 : return ix86_ssecom_setcc (comparison, check_unordered, mode,
12083 547 : set_dst, target);
12084 : }
12085 :
12086 : /* Subroutines of ix86_expand_args_builtin to take care of round insns. */
12087 :
12088 : static rtx
12089 0 : ix86_expand_sse_round (const struct builtin_description *d, tree exp,
12090 : rtx target)
12091 : {
12092 0 : rtx pat;
12093 0 : tree arg0 = CALL_EXPR_ARG (exp, 0);
12094 0 : rtx op1, op0 = expand_normal (arg0);
12095 0 : machine_mode tmode = insn_data[d->icode].operand[0].mode;
12096 0 : machine_mode mode0 = insn_data[d->icode].operand[1].mode;
12097 :
12098 0 : if (optimize || target == 0
12099 0 : || GET_MODE (target) != tmode
12100 0 : || !insn_data[d->icode].operand[0].predicate (target, tmode))
12101 0 : target = gen_reg_rtx (tmode);
12102 :
12103 0 : if (VECTOR_MODE_P (mode0))
12104 0 : op0 = safe_vector_operand (op0, mode0);
12105 :
12106 0 : if ((optimize && !register_operand (op0, mode0))
12107 0 : || !insn_data[d->icode].operand[0].predicate (op0, mode0))
12108 0 : op0 = copy_to_mode_reg (mode0, op0);
12109 :
12110 0 : op1 = GEN_INT (d->comparison);
12111 :
12112 0 : pat = GEN_FCN (d->icode) (target, op0, op1);
12113 0 : if (! pat)
12114 : return 0;
12115 0 : emit_insn (pat);
12116 0 : return target;
12117 : }
12118 :
12119 : static rtx
12120 12 : ix86_expand_sse_round_vec_pack_sfix (const struct builtin_description *d,
12121 : tree exp, rtx target)
12122 : {
12123 12 : rtx pat;
12124 12 : tree arg0 = CALL_EXPR_ARG (exp, 0);
12125 12 : tree arg1 = CALL_EXPR_ARG (exp, 1);
12126 12 : rtx op0 = expand_normal (arg0);
12127 12 : rtx op1 = expand_normal (arg1);
12128 12 : rtx op2;
12129 12 : machine_mode tmode = insn_data[d->icode].operand[0].mode;
12130 12 : machine_mode mode0 = insn_data[d->icode].operand[1].mode;
12131 12 : machine_mode mode1 = insn_data[d->icode].operand[2].mode;
12132 :
12133 0 : if (optimize || target == 0
12134 0 : || GET_MODE (target) != tmode
12135 12 : || !insn_data[d->icode].operand[0].predicate (target, tmode))
12136 12 : target = gen_reg_rtx (tmode);
12137 :
12138 12 : op0 = safe_vector_operand (op0, mode0);
12139 12 : op1 = safe_vector_operand (op1, mode1);
12140 :
12141 12 : if ((optimize && !register_operand (op0, mode0))
12142 12 : || !insn_data[d->icode].operand[0].predicate (op0, mode0))
12143 12 : op0 = copy_to_mode_reg (mode0, op0);
12144 12 : if ((optimize && !register_operand (op1, mode1))
12145 12 : || !insn_data[d->icode].operand[1].predicate (op1, mode1))
12146 12 : op1 = copy_to_mode_reg (mode1, op1);
12147 :
12148 12 : op2 = GEN_INT (d->comparison);
12149 :
12150 12 : pat = GEN_FCN (d->icode) (target, op0, op1, op2);
12151 12 : if (! pat)
12152 : return 0;
12153 12 : emit_insn (pat);
12154 12 : return target;
12155 : }
12156 :
12157 : /* Subroutine of ix86_expand_builtin to take care of ptest insns. */
12158 :
12159 : static rtx
12160 239 : ix86_expand_sse_ptest (const struct builtin_description *d, tree exp,
12161 : rtx target)
12162 : {
12163 239 : rtx pat;
12164 239 : tree arg0 = CALL_EXPR_ARG (exp, 0);
12165 239 : tree arg1 = CALL_EXPR_ARG (exp, 1);
12166 239 : rtx op0 = expand_normal (arg0);
12167 239 : rtx op1 = expand_normal (arg1);
12168 239 : machine_mode mode0 = insn_data[d->icode].operand[0].mode;
12169 239 : machine_mode mode1 = insn_data[d->icode].operand[1].mode;
12170 239 : enum rtx_code comparison = d->comparison;
12171 239 : rtx result = NULL_RTX;
12172 :
12173 239 : if (VECTOR_MODE_P (mode0))
12174 239 : op0 = safe_vector_operand (op0, mode0);
12175 239 : if (VECTOR_MODE_P (mode1))
12176 239 : op1 = safe_vector_operand (op1, mode1);
12177 :
12178 239 : switch (d->code)
12179 : {
12180 49 : case IX86_BUILTIN_PTESTZ:
12181 49 : case IX86_BUILTIN_PTESTZ256:
12182 : // Returns (OP0 & OP1) == 0
12183 49 : if (rtx_equal_p (op0, CONST0_RTX (mode0))
12184 49 : || rtx_equal_p (op1, CONST0_RTX (mode1)))
12185 2 : result = const1_rtx;
12186 47 : else if (rtx_equal_p (op0, CONSTM1_RTX (mode0)))
12187 : {
12188 1 : op1 = force_reg (mode1, op1);
12189 1 : op0 = op1;
12190 : }
12191 46 : else if (rtx_equal_p (op1, CONSTM1_RTX (mode1)))
12192 : {
12193 1 : op0 = force_reg (mode0, op0);
12194 1 : op1 = op0;
12195 : }
12196 45 : else if (MEM_P (op0) && !MEM_P (op1))
12197 : std::swap (op0, op1);
12198 : break;
12199 :
12200 31 : case IX86_BUILTIN_PTESTC:
12201 31 : case IX86_BUILTIN_PTESTC256:
12202 : // Returns (~OP0 & OP1) == 0
12203 31 : if (rtx_equal_p (op0, CONSTM1_RTX (mode0))
12204 31 : || rtx_equal_p (op1, CONST0_RTX (mode1))
12205 62 : || rtx_equal_p (op0, op1))
12206 2 : result = const1_rtx;
12207 : break;
12208 :
12209 27 : case IX86_BUILTIN_PTESTNZC:
12210 27 : case IX86_BUILTIN_PTESTNZC256:
12211 : // Returns ((OP0 && OP1) != 0) && ((~OP0 && OP1) != 0)
12212 27 : if (rtx_equal_p (op0, CONST0_RTX (mode0))
12213 26 : || rtx_equal_p (op0, CONSTM1_RTX (mode0))
12214 26 : || rtx_equal_p (op1, CONST0_RTX (mode1))
12215 53 : || rtx_equal_p (op0, op1))
12216 1 : result = const0_rtx;
12217 : break;
12218 :
12219 : default:
12220 : break;
12221 : }
12222 :
12223 167 : if ((optimize && !register_operand (op0, mode0))
12224 210 : || !insn_data[d->icode].operand[0].predicate (op0, mode0)
12225 377 : || result)
12226 104 : op0 = copy_to_mode_reg (mode0, op0);
12227 167 : if ((optimize && !register_operand (op1, mode1))
12228 211 : || !insn_data[d->icode].operand[1].predicate (op1, mode1)
12229 450 : || result)
12230 31 : op1 = copy_to_mode_reg (mode1, op1);
12231 :
12232 239 : if (result)
12233 : {
12234 5 : if (!target)
12235 0 : target = gen_reg_rtx (SImode);
12236 5 : emit_move_insn (target, result);
12237 5 : return target;
12238 : }
12239 :
12240 234 : target = gen_reg_rtx (SImode);
12241 234 : emit_move_insn (target, const0_rtx);
12242 234 : target = gen_rtx_SUBREG (QImode, target, 0);
12243 :
12244 234 : pat = GEN_FCN (d->icode) (op0, op1);
12245 234 : if (! pat)
12246 : return 0;
12247 234 : emit_insn (pat);
12248 234 : emit_insn (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
12249 : gen_rtx_fmt_ee (comparison, QImode,
12250 : SET_DEST (pat),
12251 : const0_rtx)));
12252 :
12253 234 : return SUBREG_REG (target);
12254 : }
12255 :
12256 : /* Subroutine of ix86_expand_builtin to take care of pcmpestr[im] insns. */
12257 :
12258 : static rtx
12259 216 : ix86_expand_sse_pcmpestr (const struct builtin_description *d,
12260 : tree exp, rtx target)
12261 : {
12262 216 : rtx pat;
12263 216 : tree arg0 = CALL_EXPR_ARG (exp, 0);
12264 216 : tree arg1 = CALL_EXPR_ARG (exp, 1);
12265 216 : tree arg2 = CALL_EXPR_ARG (exp, 2);
12266 216 : tree arg3 = CALL_EXPR_ARG (exp, 3);
12267 216 : tree arg4 = CALL_EXPR_ARG (exp, 4);
12268 216 : rtx scratch0, scratch1;
12269 216 : rtx op0 = expand_normal (arg0);
12270 216 : rtx op1 = expand_normal (arg1);
12271 216 : rtx op2 = expand_normal (arg2);
12272 216 : rtx op3 = expand_normal (arg3);
12273 216 : rtx op4 = expand_normal (arg4);
12274 216 : machine_mode tmode0, tmode1, modev2, modei3, modev4, modei5, modeimm;
12275 :
12276 216 : tmode0 = insn_data[d->icode].operand[0].mode;
12277 216 : tmode1 = insn_data[d->icode].operand[1].mode;
12278 216 : modev2 = insn_data[d->icode].operand[2].mode;
12279 216 : modei3 = insn_data[d->icode].operand[3].mode;
12280 216 : modev4 = insn_data[d->icode].operand[4].mode;
12281 216 : modei5 = insn_data[d->icode].operand[5].mode;
12282 216 : modeimm = insn_data[d->icode].operand[6].mode;
12283 :
12284 216 : if (VECTOR_MODE_P (modev2))
12285 216 : op0 = safe_vector_operand (op0, modev2);
12286 216 : if (VECTOR_MODE_P (modev4))
12287 216 : op2 = safe_vector_operand (op2, modev4);
12288 :
12289 216 : if (!insn_data[d->icode].operand[2].predicate (op0, modev2))
12290 6 : op0 = copy_to_mode_reg (modev2, op0);
12291 216 : if (!insn_data[d->icode].operand[3].predicate (op1, modei3))
12292 34 : op1 = copy_to_mode_reg (modei3, op1);
12293 160 : if ((optimize && !register_operand (op2, modev4))
12294 371 : || !insn_data[d->icode].operand[4].predicate (op2, modev4))
12295 5 : op2 = copy_to_mode_reg (modev4, op2);
12296 216 : if (!insn_data[d->icode].operand[5].predicate (op3, modei5))
12297 34 : op3 = copy_to_mode_reg (modei5, op3);
12298 :
12299 216 : if (!insn_data[d->icode].operand[6].predicate (op4, modeimm))
12300 : {
12301 21 : error ("the fifth argument must be an 8-bit immediate");
12302 21 : return const0_rtx;
12303 : }
12304 :
12305 195 : if (d->code == IX86_BUILTIN_PCMPESTRI128)
12306 : {
12307 5 : if (optimize || !target
12308 5 : || GET_MODE (target) != tmode0
12309 34 : || !insn_data[d->icode].operand[0].predicate (target, tmode0))
12310 24 : target = gen_reg_rtx (tmode0);
12311 :
12312 29 : scratch1 = gen_reg_rtx (tmode1);
12313 :
12314 29 : pat = GEN_FCN (d->icode) (target, scratch1, op0, op1, op2, op3, op4);
12315 : }
12316 166 : else if (d->code == IX86_BUILTIN_PCMPESTRM128)
12317 : {
12318 5 : if (optimize || !target
12319 5 : || GET_MODE (target) != tmode1
12320 36 : || !insn_data[d->icode].operand[1].predicate (target, tmode1))
12321 26 : target = gen_reg_rtx (tmode1);
12322 :
12323 31 : scratch0 = gen_reg_rtx (tmode0);
12324 :
12325 31 : pat = GEN_FCN (d->icode) (scratch0, target, op0, op1, op2, op3, op4);
12326 : }
12327 : else
12328 : {
12329 135 : gcc_assert (d->flag);
12330 :
12331 135 : scratch0 = gen_reg_rtx (tmode0);
12332 135 : scratch1 = gen_reg_rtx (tmode1);
12333 :
12334 135 : pat = GEN_FCN (d->icode) (scratch0, scratch1, op0, op1, op2, op3, op4);
12335 : }
12336 :
12337 195 : if (! pat)
12338 : return 0;
12339 :
12340 195 : emit_insn (pat);
12341 :
12342 195 : if (d->flag)
12343 : {
12344 135 : target = gen_reg_rtx (SImode);
12345 135 : emit_move_insn (target, const0_rtx);
12346 135 : target = gen_rtx_SUBREG (QImode, target, 0);
12347 :
12348 135 : emit_insn
12349 135 : (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
12350 : gen_rtx_fmt_ee (EQ, QImode,
12351 : gen_rtx_REG ((machine_mode) d->flag,
12352 : FLAGS_REG),
12353 : const0_rtx)));
12354 135 : return SUBREG_REG (target);
12355 : }
12356 : else
12357 : return target;
12358 : }
12359 :
12360 :
12361 : /* Subroutine of ix86_expand_builtin to take care of pcmpistr[im] insns. */
12362 :
12363 : static rtx
12364 275 : ix86_expand_sse_pcmpistr (const struct builtin_description *d,
12365 : tree exp, rtx target)
12366 : {
12367 275 : rtx pat;
12368 275 : tree arg0 = CALL_EXPR_ARG (exp, 0);
12369 275 : tree arg1 = CALL_EXPR_ARG (exp, 1);
12370 275 : tree arg2 = CALL_EXPR_ARG (exp, 2);
12371 275 : rtx scratch0, scratch1;
12372 275 : rtx op0 = expand_normal (arg0);
12373 275 : rtx op1 = expand_normal (arg1);
12374 275 : rtx op2 = expand_normal (arg2);
12375 275 : machine_mode tmode0, tmode1, modev2, modev3, modeimm;
12376 :
12377 275 : tmode0 = insn_data[d->icode].operand[0].mode;
12378 275 : tmode1 = insn_data[d->icode].operand[1].mode;
12379 275 : modev2 = insn_data[d->icode].operand[2].mode;
12380 275 : modev3 = insn_data[d->icode].operand[3].mode;
12381 275 : modeimm = insn_data[d->icode].operand[4].mode;
12382 :
12383 275 : if (VECTOR_MODE_P (modev2))
12384 275 : op0 = safe_vector_operand (op0, modev2);
12385 275 : if (VECTOR_MODE_P (modev3))
12386 275 : op1 = safe_vector_operand (op1, modev3);
12387 :
12388 275 : if (!insn_data[d->icode].operand[2].predicate (op0, modev2))
12389 4 : op0 = copy_to_mode_reg (modev2, op0);
12390 210 : if ((optimize && !register_operand (op1, modev3))
12391 481 : || !insn_data[d->icode].operand[3].predicate (op1, modev3))
12392 4 : op1 = copy_to_mode_reg (modev3, op1);
12393 :
12394 275 : if (!insn_data[d->icode].operand[4].predicate (op2, modeimm))
12395 : {
12396 21 : error ("the third argument must be an 8-bit immediate");
12397 21 : return const0_rtx;
12398 : }
12399 :
12400 254 : if (d->code == IX86_BUILTIN_PCMPISTRI128)
12401 : {
12402 5 : if (optimize || !target
12403 5 : || GET_MODE (target) != tmode0
12404 38 : || !insn_data[d->icode].operand[0].predicate (target, tmode0))
12405 28 : target = gen_reg_rtx (tmode0);
12406 :
12407 33 : scratch1 = gen_reg_rtx (tmode1);
12408 :
12409 33 : pat = GEN_FCN (d->icode) (target, scratch1, op0, op1, op2);
12410 : }
12411 221 : else if (d->code == IX86_BUILTIN_PCMPISTRM128)
12412 : {
12413 8 : if (optimize || !target
12414 8 : || GET_MODE (target) != tmode1
12415 58 : || !insn_data[d->icode].operand[1].predicate (target, tmode1))
12416 42 : target = gen_reg_rtx (tmode1);
12417 :
12418 50 : scratch0 = gen_reg_rtx (tmode0);
12419 :
12420 50 : pat = GEN_FCN (d->icode) (scratch0, target, op0, op1, op2);
12421 : }
12422 : else
12423 : {
12424 171 : gcc_assert (d->flag);
12425 :
12426 171 : scratch0 = gen_reg_rtx (tmode0);
12427 171 : scratch1 = gen_reg_rtx (tmode1);
12428 :
12429 171 : pat = GEN_FCN (d->icode) (scratch0, scratch1, op0, op1, op2);
12430 : }
12431 :
12432 254 : if (! pat)
12433 : return 0;
12434 :
12435 254 : emit_insn (pat);
12436 :
12437 254 : if (d->flag)
12438 : {
12439 171 : target = gen_reg_rtx (SImode);
12440 171 : emit_move_insn (target, const0_rtx);
12441 171 : target = gen_rtx_SUBREG (QImode, target, 0);
12442 :
12443 171 : emit_insn
12444 171 : (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
12445 : gen_rtx_fmt_ee (EQ, QImode,
12446 : gen_rtx_REG ((machine_mode) d->flag,
12447 : FLAGS_REG),
12448 : const0_rtx)));
12449 171 : return SUBREG_REG (target);
12450 : }
12451 : else
12452 : return target;
12453 : }
12454 :
12455 : /* Fixup modeless constants to fit required mode. */
12456 :
12457 : static rtx
12458 262616 : fixup_modeless_constant (rtx x, machine_mode mode)
12459 : {
12460 262616 : if (GET_MODE (x) == VOIDmode)
12461 41522 : x = convert_to_mode (mode, x, 1);
12462 262616 : return x;
12463 : }
12464 :
12465 : /* Expand the outgoing argument ARG to extract unsigned char and short
12466 : integer constants suitable for the predicates and the instruction
12467 : templates which expect the unsigned expanded value. */
12468 :
12469 : static rtx
12470 283773 : ix86_expand_unsigned_small_int_cst_argument (tree arg)
12471 : {
12472 : /* When passing 0xff as an unsigned char function argument with the
12473 : C frontend promotion, expand_normal gets
12474 :
12475 : <integer_cst 0x7fffe6aa23a8 type <integer_type 0x7fffe98225e8 int> constant 255>
12476 :
12477 : and returns the rtx value using the sign-extended representation:
12478 :
12479 : (const_int 255 [0xff])
12480 :
12481 : Without the C frontend promotion, expand_normal gets
12482 :
12483 : <integer_cst 0x7fffe9824018 type <integer_type 0x7fffe9822348 unsigned char > constant 255>
12484 :
12485 : and returns
12486 :
12487 : (const_int -1 [0xffffffffffffffff])
12488 :
12489 : which doesn't work with the predicates nor the instruction templates
12490 : which expect the unsigned expanded value. Extract the unsigned char
12491 : and short integer constants to return
12492 :
12493 : (const_int 255 [0xff])
12494 :
12495 : so that the expanded value is always unsigned, without the C frontend
12496 : promotion. */
12497 :
12498 283773 : if (TREE_CODE (arg) == INTEGER_CST)
12499 : {
12500 60323 : tree type = TREE_TYPE (arg);
12501 60323 : if (INTEGRAL_TYPE_P (type)
12502 60323 : && TYPE_UNSIGNED (type)
12503 82098 : && TYPE_PRECISION (type) < TYPE_PRECISION (integer_type_node))
12504 : {
12505 18294 : HOST_WIDE_INT cst = TREE_INT_CST_LOW (arg);
12506 18294 : return GEN_INT (cst);
12507 : }
12508 : }
12509 :
12510 265479 : return expand_normal (arg);
12511 : }
12512 :
12513 : /* Subroutine of ix86_expand_builtin to take care of insns with
12514 : variable number of operands. */
12515 :
12516 : static rtx
12517 72912 : ix86_expand_args_builtin (const struct builtin_description *d,
12518 : tree exp, rtx target)
12519 : {
12520 72912 : rtx pat, real_target;
12521 72912 : unsigned int i, nargs;
12522 72912 : unsigned int nargs_constant = 0;
12523 72912 : unsigned int mask_pos = 0;
12524 72912 : int num_memory = 0;
12525 72912 : rtx xops[6];
12526 72912 : bool second_arg_count = false;
12527 72912 : enum insn_code icode = d->icode;
12528 72912 : const struct insn_data_d *insn_p = &insn_data[icode];
12529 72912 : machine_mode tmode = insn_p->operand[0].mode;
12530 72912 : machine_mode rmode = VOIDmode;
12531 72912 : bool swap = false;
12532 72912 : enum rtx_code comparison = d->comparison;
12533 :
12534 72912 : switch ((enum ix86_builtin_func_type) d->flag)
12535 : {
12536 0 : case V2DF_FTYPE_V2DF_ROUND:
12537 0 : case V4DF_FTYPE_V4DF_ROUND:
12538 0 : case V8DF_FTYPE_V8DF_ROUND:
12539 0 : case V4SF_FTYPE_V4SF_ROUND:
12540 0 : case V8SF_FTYPE_V8SF_ROUND:
12541 0 : case V16SF_FTYPE_V16SF_ROUND:
12542 0 : case V8HF_FTYPE_V8HF_ROUND:
12543 0 : case V16HF_FTYPE_V16HF_ROUND:
12544 0 : case V32HF_FTYPE_V32HF_ROUND:
12545 0 : case V4SI_FTYPE_V4SF_ROUND:
12546 0 : case V8SI_FTYPE_V8SF_ROUND:
12547 0 : case V16SI_FTYPE_V16SF_ROUND:
12548 0 : return ix86_expand_sse_round (d, exp, target);
12549 12 : case V4SI_FTYPE_V2DF_V2DF_ROUND:
12550 12 : case V8SI_FTYPE_V4DF_V4DF_ROUND:
12551 12 : case V16SI_FTYPE_V8DF_V8DF_ROUND:
12552 12 : return ix86_expand_sse_round_vec_pack_sfix (d, exp, target);
12553 239 : case INT_FTYPE_V8SF_V8SF_PTEST:
12554 239 : case INT_FTYPE_V4DI_V4DI_PTEST:
12555 239 : case INT_FTYPE_V4DF_V4DF_PTEST:
12556 239 : case INT_FTYPE_V4SF_V4SF_PTEST:
12557 239 : case INT_FTYPE_V2DI_V2DI_PTEST:
12558 239 : case INT_FTYPE_V2DF_V2DF_PTEST:
12559 239 : return ix86_expand_sse_ptest (d, exp, target);
12560 : case FLOAT128_FTYPE_FLOAT128:
12561 : case FLOAT_FTYPE_FLOAT:
12562 : case FLOAT_FTYPE_BFLOAT16:
12563 : case INT_FTYPE_INT:
12564 : case UINT_FTYPE_UINT:
12565 : case UINT16_FTYPE_UINT16:
12566 : case UINT64_FTYPE_INT:
12567 : case UINT64_FTYPE_UINT64:
12568 : case INT64_FTYPE_INT64:
12569 : case INT64_FTYPE_V4SF:
12570 : case INT64_FTYPE_V2DF:
12571 : case INT_FTYPE_V16QI:
12572 : case INT_FTYPE_V8QI:
12573 : case INT_FTYPE_V8SF:
12574 : case INT_FTYPE_V4DF:
12575 : case INT_FTYPE_V4SF:
12576 : case INT_FTYPE_V2DF:
12577 : case INT_FTYPE_V32QI:
12578 : case V16QI_FTYPE_V16QI:
12579 : case V8SI_FTYPE_V8SF:
12580 : case V8SI_FTYPE_V4SI:
12581 : case V8HI_FTYPE_V8HI:
12582 : case V8HI_FTYPE_V16QI:
12583 : case V8QI_FTYPE_V8QI:
12584 : case V8SF_FTYPE_V8SF:
12585 : case V8SF_FTYPE_V8SI:
12586 : case V8SF_FTYPE_V4SF:
12587 : case V8SF_FTYPE_V8HI:
12588 : case V4SI_FTYPE_V4SI:
12589 : case V4SI_FTYPE_V16QI:
12590 : case V4SI_FTYPE_V4SF:
12591 : case V4SI_FTYPE_V8SI:
12592 : case V4SI_FTYPE_V8HI:
12593 : case V4SI_FTYPE_V4DF:
12594 : case V4SI_FTYPE_V2DF:
12595 : case V4HI_FTYPE_V4HI:
12596 : case V4DF_FTYPE_V4DF:
12597 : case V4DF_FTYPE_V4SI:
12598 : case V4DF_FTYPE_V4SF:
12599 : case V4DF_FTYPE_V2DF:
12600 : case V4SF_FTYPE_V4SF:
12601 : case V4SF_FTYPE_V4SI:
12602 : case V4SF_FTYPE_V8SF:
12603 : case V4SF_FTYPE_V4DF:
12604 : case V4SF_FTYPE_V8HI:
12605 : case V4SF_FTYPE_V2DF:
12606 : case V2DI_FTYPE_V2DI:
12607 : case V2DI_FTYPE_V16QI:
12608 : case V2DI_FTYPE_V8HI:
12609 : case V2DI_FTYPE_V4SI:
12610 : case V2DF_FTYPE_V2DF:
12611 : case V2DF_FTYPE_V4SI:
12612 : case V2DF_FTYPE_V4DF:
12613 : case V2DF_FTYPE_V4SF:
12614 : case V2DF_FTYPE_V2SI:
12615 : case V2SI_FTYPE_V2SI:
12616 : case V2SI_FTYPE_V4SF:
12617 : case V2SI_FTYPE_V2SF:
12618 : case V2SI_FTYPE_V2DF:
12619 : case V2SF_FTYPE_V2SF:
12620 : case V2SF_FTYPE_V2SI:
12621 : case V32QI_FTYPE_V32QI:
12622 : case V32QI_FTYPE_V16QI:
12623 : case V16HI_FTYPE_V16HI:
12624 : case V16HI_FTYPE_V8HI:
12625 : case V8SI_FTYPE_V8SI:
12626 : case V16HI_FTYPE_V16QI:
12627 : case V8SI_FTYPE_V16QI:
12628 : case V4DI_FTYPE_V16QI:
12629 : case V8SI_FTYPE_V8HI:
12630 : case V4DI_FTYPE_V8HI:
12631 : case V4DI_FTYPE_V4SI:
12632 : case V4DI_FTYPE_V2DI:
12633 : case UQI_FTYPE_UQI:
12634 : case UHI_FTYPE_UHI:
12635 : case USI_FTYPE_USI:
12636 : case USI_FTYPE_UQI:
12637 : case USI_FTYPE_UHI:
12638 : case UDI_FTYPE_UDI:
12639 : case UHI_FTYPE_V16QI:
12640 : case USI_FTYPE_V32QI:
12641 : case UDI_FTYPE_V64QI:
12642 : case V16QI_FTYPE_UHI:
12643 : case V32QI_FTYPE_USI:
12644 : case V64QI_FTYPE_UDI:
12645 : case V8HI_FTYPE_UQI:
12646 : case V16HI_FTYPE_UHI:
12647 : case V32HI_FTYPE_USI:
12648 : case V4SI_FTYPE_UQI:
12649 : case V8SI_FTYPE_UQI:
12650 : case V4SI_FTYPE_UHI:
12651 : case V8SI_FTYPE_UHI:
12652 : case UQI_FTYPE_V8HI:
12653 : case UHI_FTYPE_V16HI:
12654 : case USI_FTYPE_V32HI:
12655 : case UQI_FTYPE_V4SI:
12656 : case UQI_FTYPE_V8SI:
12657 : case UHI_FTYPE_V16SI:
12658 : case UQI_FTYPE_V2DI:
12659 : case UQI_FTYPE_V4DI:
12660 : case UQI_FTYPE_V8DI:
12661 : case V16SI_FTYPE_UHI:
12662 : case V2DI_FTYPE_UQI:
12663 : case V4DI_FTYPE_UQI:
12664 : case V16SI_FTYPE_INT:
12665 : case V16SF_FTYPE_V8SF:
12666 : case V16SI_FTYPE_V8SI:
12667 : case V16SF_FTYPE_V4SF:
12668 : case V16SI_FTYPE_V4SI:
12669 : case V16SI_FTYPE_V16SF:
12670 : case V16SI_FTYPE_V16SI:
12671 : case V64QI_FTYPE_V64QI:
12672 : case V32HI_FTYPE_V32HI:
12673 : case V16SF_FTYPE_V16SF:
12674 : case V8DI_FTYPE_UQI:
12675 : case V8DI_FTYPE_V8DI:
12676 : case V8DF_FTYPE_V4DF:
12677 : case V8DF_FTYPE_V2DF:
12678 : case V8DF_FTYPE_V8DF:
12679 : case V4DI_FTYPE_V4DI:
12680 : case V16BF_FTYPE_V16SF:
12681 : case V8BF_FTYPE_V8SF:
12682 : case V8BF_FTYPE_V4SF:
12683 : nargs = 1;
12684 : break;
12685 52 : case V4SF_FTYPE_V4SF_VEC_MERGE:
12686 52 : case V2DF_FTYPE_V2DF_VEC_MERGE:
12687 52 : return ix86_expand_unop_vec_merge_builtin (icode, exp, target);
12688 9527 : case FLOAT128_FTYPE_FLOAT128_FLOAT128:
12689 9527 : case V16QI_FTYPE_V16QI_V16QI:
12690 9527 : case V16QI_FTYPE_V8HI_V8HI:
12691 9527 : case V16HF_FTYPE_V16HF_V16HF:
12692 9527 : case V16SF_FTYPE_V16SF_V16SF:
12693 9527 : case V16SI_FTYPE_V16SI_V16SI:
12694 9527 : case V8QI_FTYPE_V8QI_V8QI:
12695 9527 : case V8QI_FTYPE_V4HI_V4HI:
12696 9527 : case V8HI_FTYPE_V8HI_V8HI:
12697 9527 : case V8HI_FTYPE_V16QI_V16QI:
12698 9527 : case V8HI_FTYPE_V4SI_V4SI:
12699 9527 : case V8HF_FTYPE_V8HF_V8HF:
12700 9527 : case V8SF_FTYPE_V8SF_V8SF:
12701 9527 : case V8SF_FTYPE_V8SF_V8SI:
12702 9527 : case V8DF_FTYPE_V8DF_V8DF:
12703 9527 : case V4SI_FTYPE_V4SI_V4SI:
12704 9527 : case V4SI_FTYPE_V8HI_V8HI:
12705 9527 : case V4SI_FTYPE_V2DF_V2DF:
12706 9527 : case V4HI_FTYPE_V4HI_V4HI:
12707 9527 : case V4HI_FTYPE_V8QI_V8QI:
12708 9527 : case V4HI_FTYPE_V2SI_V2SI:
12709 9527 : case V4DF_FTYPE_V4DF_V4DF:
12710 9527 : case V4DF_FTYPE_V4DF_V4DI:
12711 9527 : case V4SF_FTYPE_V4SF_V4SF:
12712 9527 : case V4SF_FTYPE_V4SF_V4SI:
12713 9527 : case V4SF_FTYPE_V4SF_V2SI:
12714 9527 : case V4SF_FTYPE_V4SF_V2DF:
12715 9527 : case V4SF_FTYPE_V4SF_UINT:
12716 9527 : case V4SF_FTYPE_V4SF_DI:
12717 9527 : case V4SF_FTYPE_V4SF_SI:
12718 9527 : case V4DI_FTYPE_V4DI_V2DI:
12719 9527 : case V2DI_FTYPE_V2DI_V2DI:
12720 9527 : case V2DI_FTYPE_V16QI_V16QI:
12721 9527 : case V2DI_FTYPE_V4SI_V4SI:
12722 9527 : case V2DI_FTYPE_V2DI_V16QI:
12723 9527 : case V2SI_FTYPE_V2SI_V2SI:
12724 9527 : case V2SI_FTYPE_V4HI_V4HI:
12725 9527 : case V2SI_FTYPE_V2SF_V2SF:
12726 9527 : case V2DF_FTYPE_V2DF_V2DF:
12727 9527 : case V2DF_FTYPE_V2DF_V4SF:
12728 9527 : case V2DF_FTYPE_V2DF_V2DI:
12729 9527 : case V2DF_FTYPE_V2DF_DI:
12730 9527 : case V2DF_FTYPE_V2DF_SI:
12731 9527 : case V2DF_FTYPE_V2DF_UINT:
12732 9527 : case V2SF_FTYPE_V2SF_V2SF:
12733 9527 : case V1DI_FTYPE_V1DI_V1DI:
12734 9527 : case V1DI_FTYPE_V8QI_V8QI:
12735 9527 : case V1DI_FTYPE_V2SI_V2SI:
12736 9527 : case V32QI_FTYPE_V16HI_V16HI:
12737 9527 : case V16HI_FTYPE_V8SI_V8SI:
12738 9527 : case V64QI_FTYPE_V64QI_V64QI:
12739 9527 : case V32QI_FTYPE_V32QI_V32QI:
12740 9527 : case V32BF_FTYPE_V32BF_V32BF:
12741 9527 : case V16BF_FTYPE_V16BF_V16BF:
12742 9527 : case V8BF_FTYPE_V8BF_V8BF:
12743 9527 : case V16HI_FTYPE_V32QI_V32QI:
12744 9527 : case V16HI_FTYPE_V16HI_V16HI:
12745 9527 : case V8SI_FTYPE_V4DF_V4DF:
12746 9527 : case V8SI_FTYPE_V8SI_V8SI:
12747 9527 : case V8SI_FTYPE_V16HI_V16HI:
12748 9527 : case V4DI_FTYPE_V4DI_V4DI:
12749 9527 : case V4DI_FTYPE_V8SI_V8SI:
12750 9527 : case V4DI_FTYPE_V32QI_V32QI:
12751 9527 : case V8DI_FTYPE_V64QI_V64QI:
12752 9527 : if (comparison == UNKNOWN)
12753 8991 : return ix86_expand_binop_builtin (icode, exp, target);
12754 : nargs = 2;
12755 : break;
12756 80 : case V4SF_FTYPE_V4SF_V4SF_SWAP:
12757 80 : case V2DF_FTYPE_V2DF_V2DF_SWAP:
12758 80 : gcc_assert (comparison != UNKNOWN);
12759 : nargs = 2;
12760 : swap = true;
12761 : break;
12762 1481 : case V16HI_FTYPE_V16HI_V8HI_COUNT:
12763 1481 : case V16HI_FTYPE_V16HI_SI_COUNT:
12764 1481 : case V8SI_FTYPE_V8SI_V4SI_COUNT:
12765 1481 : case V8SI_FTYPE_V8SI_SI_COUNT:
12766 1481 : case V4DI_FTYPE_V4DI_V2DI_COUNT:
12767 1481 : case V4DI_FTYPE_V4DI_INT_COUNT:
12768 1481 : case V8HI_FTYPE_V8HI_V8HI_COUNT:
12769 1481 : case V8HI_FTYPE_V8HI_SI_COUNT:
12770 1481 : case V4SI_FTYPE_V4SI_V4SI_COUNT:
12771 1481 : case V4SI_FTYPE_V4SI_SI_COUNT:
12772 1481 : case V4HI_FTYPE_V4HI_V4HI_COUNT:
12773 1481 : case V4HI_FTYPE_V4HI_SI_COUNT:
12774 1481 : case V2DI_FTYPE_V2DI_V2DI_COUNT:
12775 1481 : case V2DI_FTYPE_V2DI_SI_COUNT:
12776 1481 : case V2SI_FTYPE_V2SI_V2SI_COUNT:
12777 1481 : case V2SI_FTYPE_V2SI_SI_COUNT:
12778 1481 : case V1DI_FTYPE_V1DI_V1DI_COUNT:
12779 1481 : case V1DI_FTYPE_V1DI_SI_COUNT:
12780 1481 : nargs = 2;
12781 1481 : second_arg_count = true;
12782 1481 : break;
12783 1408 : case V16HI_FTYPE_V16HI_INT_V16HI_UHI_COUNT:
12784 1408 : case V16HI_FTYPE_V16HI_V8HI_V16HI_UHI_COUNT:
12785 1408 : case V16SI_FTYPE_V16SI_INT_V16SI_UHI_COUNT:
12786 1408 : case V16SI_FTYPE_V16SI_V4SI_V16SI_UHI_COUNT:
12787 1408 : case V2DI_FTYPE_V2DI_INT_V2DI_UQI_COUNT:
12788 1408 : case V2DI_FTYPE_V2DI_V2DI_V2DI_UQI_COUNT:
12789 1408 : case V32HI_FTYPE_V32HI_INT_V32HI_USI_COUNT:
12790 1408 : case V32HI_FTYPE_V32HI_V8HI_V32HI_USI_COUNT:
12791 1408 : case V4DI_FTYPE_V4DI_INT_V4DI_UQI_COUNT:
12792 1408 : case V4DI_FTYPE_V4DI_V2DI_V4DI_UQI_COUNT:
12793 1408 : case V4SI_FTYPE_V4SI_INT_V4SI_UQI_COUNT:
12794 1408 : case V4SI_FTYPE_V4SI_V4SI_V4SI_UQI_COUNT:
12795 1408 : case V8DI_FTYPE_V8DI_INT_V8DI_UQI_COUNT:
12796 1408 : case V8DI_FTYPE_V8DI_V2DI_V8DI_UQI_COUNT:
12797 1408 : case V8HI_FTYPE_V8HI_INT_V8HI_UQI_COUNT:
12798 1408 : case V8HI_FTYPE_V8HI_V8HI_V8HI_UQI_COUNT:
12799 1408 : case V8SI_FTYPE_V8SI_INT_V8SI_UQI_COUNT:
12800 1408 : case V8SI_FTYPE_V8SI_V4SI_V8SI_UQI_COUNT:
12801 1408 : nargs = 4;
12802 1408 : second_arg_count = true;
12803 1408 : break;
12804 966 : case UINT64_FTYPE_UINT64_UINT64:
12805 966 : case UINT_FTYPE_UINT_UINT:
12806 966 : case UINT_FTYPE_UINT_USHORT:
12807 966 : case UINT_FTYPE_UINT_UCHAR:
12808 966 : case UINT16_FTYPE_UINT16_INT:
12809 966 : case UINT8_FTYPE_UINT8_INT:
12810 966 : case UQI_FTYPE_UQI_UQI:
12811 966 : case UHI_FTYPE_UHI_UHI:
12812 966 : case USI_FTYPE_USI_USI:
12813 966 : case UDI_FTYPE_UDI_UDI:
12814 966 : case V16SI_FTYPE_V8DF_V8DF:
12815 966 : case V32BF_FTYPE_V16SF_V16SF:
12816 966 : case V16BF_FTYPE_V8SF_V8SF:
12817 966 : case V8BF_FTYPE_V4SF_V4SF:
12818 966 : case V16BF_FTYPE_V16SF_UHI:
12819 966 : case V8BF_FTYPE_V8SF_UQI:
12820 966 : case V8BF_FTYPE_V4SF_UQI:
12821 966 : case V16QI_FTYPE_V16QI_V8HF:
12822 966 : nargs = 2;
12823 966 : break;
12824 906 : case V2DI_FTYPE_V2DI_INT_CONVERT:
12825 906 : nargs = 2;
12826 906 : rmode = V1TImode;
12827 906 : nargs_constant = 1;
12828 906 : break;
12829 42 : case V4DI_FTYPE_V4DI_INT_CONVERT:
12830 42 : nargs = 2;
12831 42 : rmode = V2TImode;
12832 42 : nargs_constant = 1;
12833 42 : break;
12834 16 : case V8DI_FTYPE_V8DI_INT_CONVERT:
12835 16 : nargs = 2;
12836 16 : rmode = V4TImode;
12837 16 : nargs_constant = 1;
12838 16 : break;
12839 2424 : case V8HI_FTYPE_V8HI_INT:
12840 2424 : case V8HI_FTYPE_V8SF_INT:
12841 2424 : case V16HI_FTYPE_V16SF_INT:
12842 2424 : case V8HI_FTYPE_V4SF_INT:
12843 2424 : case V8SF_FTYPE_V8SF_INT:
12844 2424 : case V4SF_FTYPE_V16SF_INT:
12845 2424 : case V16SF_FTYPE_V16SF_INT:
12846 2424 : case V4SI_FTYPE_V4SI_INT:
12847 2424 : case V4SI_FTYPE_V8SI_INT:
12848 2424 : case V4HI_FTYPE_V4HI_INT:
12849 2424 : case V4DF_FTYPE_V4DF_INT:
12850 2424 : case V4DF_FTYPE_V8DF_INT:
12851 2424 : case V4SF_FTYPE_V4SF_INT:
12852 2424 : case V4SF_FTYPE_V8SF_INT:
12853 2424 : case V2DI_FTYPE_V2DI_INT:
12854 2424 : case V2DF_FTYPE_V2DF_INT:
12855 2424 : case V2DF_FTYPE_V4DF_INT:
12856 2424 : case V16HI_FTYPE_V16HI_INT:
12857 2424 : case V8SI_FTYPE_V8SI_INT:
12858 2424 : case V16SI_FTYPE_V16SI_INT:
12859 2424 : case V4SI_FTYPE_V16SI_INT:
12860 2424 : case V4DI_FTYPE_V4DI_INT:
12861 2424 : case V2DI_FTYPE_V4DI_INT:
12862 2424 : case V4DI_FTYPE_V8DI_INT:
12863 2424 : case UQI_FTYPE_UQI_UQI_CONST:
12864 2424 : case UHI_FTYPE_UHI_UQI:
12865 2424 : case USI_FTYPE_USI_UQI:
12866 2424 : case UDI_FTYPE_UDI_UQI:
12867 2424 : nargs = 2;
12868 2424 : nargs_constant = 1;
12869 2424 : break;
12870 18720 : case V16QI_FTYPE_V16QI_V16QI_V16QI:
12871 18720 : case V8SF_FTYPE_V8SF_V8SF_V8SF:
12872 18720 : case V4DF_FTYPE_V4DF_V4DF_V4DF:
12873 18720 : case V4SF_FTYPE_V4SF_V4SF_V4SF:
12874 18720 : case V2DF_FTYPE_V2DF_V2DF_V2DF:
12875 18720 : case V32QI_FTYPE_V32QI_V32QI_V32QI:
12876 18720 : case UHI_FTYPE_V16SI_V16SI_UHI:
12877 18720 : case UQI_FTYPE_V8DI_V8DI_UQI:
12878 18720 : case V16HI_FTYPE_V16SI_V16HI_UHI:
12879 18720 : case V16QI_FTYPE_V16SI_V16QI_UHI:
12880 18720 : case V16QI_FTYPE_V8DI_V16QI_UQI:
12881 18720 : case V32HF_FTYPE_V32HF_V32HF_USI:
12882 18720 : case V16SF_FTYPE_V16SF_V16SF_UHI:
12883 18720 : case V16SF_FTYPE_V4SF_V16SF_UHI:
12884 18720 : case V16SI_FTYPE_SI_V16SI_UHI:
12885 18720 : case V16SI_FTYPE_V16HI_V16SI_UHI:
12886 18720 : case V16SI_FTYPE_V16QI_V16SI_UHI:
12887 18720 : case V8SF_FTYPE_V4SF_V8SF_UQI:
12888 18720 : case V4DF_FTYPE_V2DF_V4DF_UQI:
12889 18720 : case V8SI_FTYPE_V4SI_V8SI_UQI:
12890 18720 : case V8SI_FTYPE_SI_V8SI_UQI:
12891 18720 : case V4SI_FTYPE_V4SI_V4SI_UQI:
12892 18720 : case V4SI_FTYPE_SI_V4SI_UQI:
12893 18720 : case V4DI_FTYPE_V2DI_V4DI_UQI:
12894 18720 : case V4DI_FTYPE_DI_V4DI_UQI:
12895 18720 : case V2DI_FTYPE_V2DI_V2DI_UQI:
12896 18720 : case V2DI_FTYPE_DI_V2DI_UQI:
12897 18720 : case V64QI_FTYPE_V64QI_V64QI_UDI:
12898 18720 : case V64QI_FTYPE_V16QI_V64QI_UDI:
12899 18720 : case V64QI_FTYPE_QI_V64QI_UDI:
12900 18720 : case V32QI_FTYPE_V32QI_V32QI_USI:
12901 18720 : case V32QI_FTYPE_V16QI_V32QI_USI:
12902 18720 : case V32QI_FTYPE_QI_V32QI_USI:
12903 18720 : case V16QI_FTYPE_V16QI_V16QI_UHI:
12904 18720 : case V16QI_FTYPE_QI_V16QI_UHI:
12905 18720 : case V32HI_FTYPE_V8HI_V32HI_USI:
12906 18720 : case V32HI_FTYPE_V32BF_V32HI_USI:
12907 18720 : case V32HI_FTYPE_HI_V32HI_USI:
12908 18720 : case V16HI_FTYPE_V8HI_V16HI_UHI:
12909 18720 : case V16HI_FTYPE_V16BF_V16HI_UHI:
12910 18720 : case V16HI_FTYPE_HI_V16HI_UHI:
12911 18720 : case V8HI_FTYPE_V8HI_V8HI_UQI:
12912 18720 : case V8HI_FTYPE_V8BF_V8HI_UQI:
12913 18720 : case V8BF_FTYPE_V8BF_V8BF_UQI:
12914 18720 : case V8HI_FTYPE_HI_V8HI_UQI:
12915 18720 : case V16HF_FTYPE_V16HF_V16HF_UHI:
12916 18720 : case V8SF_FTYPE_V8HI_V8SF_UQI:
12917 18720 : case V4SF_FTYPE_V8HI_V4SF_UQI:
12918 18720 : case V8SI_FTYPE_V8HF_V8SI_UQI:
12919 18720 : case V8SF_FTYPE_V8HF_V8SF_UQI:
12920 18720 : case V8SI_FTYPE_V8SF_V8SI_UQI:
12921 18720 : case V4SI_FTYPE_V4SF_V4SI_UQI:
12922 18720 : case V4SI_FTYPE_V8HF_V4SI_UQI:
12923 18720 : case V4SF_FTYPE_V8HF_V4SF_UQI:
12924 18720 : case V4DI_FTYPE_V8HF_V4DI_UQI:
12925 18720 : case V4DI_FTYPE_V4SF_V4DI_UQI:
12926 18720 : case V2DI_FTYPE_V8HF_V2DI_UQI:
12927 18720 : case V2DI_FTYPE_V4SF_V2DI_UQI:
12928 18720 : case V8HF_FTYPE_V8HF_V8HF_UQI:
12929 18720 : case V8HF_FTYPE_V8HF_V8HF_V8HF:
12930 18720 : case V8HF_FTYPE_V8HI_V8HF_UQI:
12931 18720 : case V8HF_FTYPE_V8SI_V8HF_UQI:
12932 18720 : case V8HF_FTYPE_V8SF_V8HF_UQI:
12933 18720 : case V8HF_FTYPE_V4SI_V8HF_UQI:
12934 18720 : case V8HF_FTYPE_V4SF_V8HF_UQI:
12935 18720 : case V8HF_FTYPE_V4DI_V8HF_UQI:
12936 18720 : case V8HF_FTYPE_V4DF_V8HF_UQI:
12937 18720 : case V8HF_FTYPE_V2DI_V8HF_UQI:
12938 18720 : case V8HF_FTYPE_V2DF_V8HF_UQI:
12939 18720 : case V4SF_FTYPE_V4DI_V4SF_UQI:
12940 18720 : case V4SF_FTYPE_V2DI_V4SF_UQI:
12941 18720 : case V4DF_FTYPE_V4DI_V4DF_UQI:
12942 18720 : case V4DF_FTYPE_V8HF_V4DF_UQI:
12943 18720 : case V2DF_FTYPE_V8HF_V2DF_UQI:
12944 18720 : case V2DF_FTYPE_V2DI_V2DF_UQI:
12945 18720 : case V16QI_FTYPE_V8HI_V16QI_UQI:
12946 18720 : case V16QI_FTYPE_V16HI_V16QI_UHI:
12947 18720 : case V16QI_FTYPE_V4SI_V16QI_UQI:
12948 18720 : case V16QI_FTYPE_V8SI_V16QI_UQI:
12949 18720 : case V8HI_FTYPE_V8HF_V8HI_UQI:
12950 18720 : case V8HI_FTYPE_V4SI_V8HI_UQI:
12951 18720 : case V8HI_FTYPE_V8SI_V8HI_UQI:
12952 18720 : case V16QI_FTYPE_V2DI_V16QI_UQI:
12953 18720 : case V16QI_FTYPE_V4DI_V16QI_UQI:
12954 18720 : case V8HI_FTYPE_V2DI_V8HI_UQI:
12955 18720 : case V8HI_FTYPE_V4DI_V8HI_UQI:
12956 18720 : case V4SI_FTYPE_V2DI_V4SI_UQI:
12957 18720 : case V4SI_FTYPE_V4DI_V4SI_UQI:
12958 18720 : case V32QI_FTYPE_V32HI_V32QI_USI:
12959 18720 : case UHI_FTYPE_V16QI_V16QI_UHI:
12960 18720 : case USI_FTYPE_V32QI_V32QI_USI:
12961 18720 : case UDI_FTYPE_V64QI_V64QI_UDI:
12962 18720 : case UQI_FTYPE_V8HI_V8HI_UQI:
12963 18720 : case UHI_FTYPE_V16HI_V16HI_UHI:
12964 18720 : case USI_FTYPE_V32HI_V32HI_USI:
12965 18720 : case UQI_FTYPE_V4SI_V4SI_UQI:
12966 18720 : case UQI_FTYPE_V8SI_V8SI_UQI:
12967 18720 : case UQI_FTYPE_V2DI_V2DI_UQI:
12968 18720 : case UQI_FTYPE_V4DI_V4DI_UQI:
12969 18720 : case V4SF_FTYPE_V2DF_V4SF_UQI:
12970 18720 : case V4SF_FTYPE_V4DF_V4SF_UQI:
12971 18720 : case V16SI_FTYPE_V16SI_V16SI_UHI:
12972 18720 : case V16SI_FTYPE_V4SI_V16SI_UHI:
12973 18720 : case V2DI_FTYPE_V4SI_V2DI_UQI:
12974 18720 : case V2DI_FTYPE_V8HI_V2DI_UQI:
12975 18720 : case V2DI_FTYPE_V16QI_V2DI_UQI:
12976 18720 : case V4DI_FTYPE_V4DI_V4DI_UQI:
12977 18720 : case V4DI_FTYPE_V4SI_V4DI_UQI:
12978 18720 : case V4DI_FTYPE_V8HI_V4DI_UQI:
12979 18720 : case V4DI_FTYPE_V16QI_V4DI_UQI:
12980 18720 : case V4DI_FTYPE_V4DF_V4DI_UQI:
12981 18720 : case V2DI_FTYPE_V2DF_V2DI_UQI:
12982 18720 : case V4SI_FTYPE_V4DF_V4SI_UQI:
12983 18720 : case V4SI_FTYPE_V2DF_V4SI_UQI:
12984 18720 : case V4SI_FTYPE_V8HI_V4SI_UQI:
12985 18720 : case V4SI_FTYPE_V16QI_V4SI_UQI:
12986 18720 : case V4DI_FTYPE_V4DI_V4DI_V4DI:
12987 18720 : case V8DF_FTYPE_V2DF_V8DF_UQI:
12988 18720 : case V8DF_FTYPE_V4DF_V8DF_UQI:
12989 18720 : case V8DF_FTYPE_V8DF_V8DF_UQI:
12990 18720 : case V8SF_FTYPE_V8SF_V8SF_UQI:
12991 18720 : case V8SF_FTYPE_V8SI_V8SF_UQI:
12992 18720 : case V4DF_FTYPE_V4DF_V4DF_UQI:
12993 18720 : case V4SF_FTYPE_V4SF_V4SF_UQI:
12994 18720 : case V2DF_FTYPE_V2DF_V2DF_UQI:
12995 18720 : case V2DF_FTYPE_V4SF_V2DF_UQI:
12996 18720 : case V2DF_FTYPE_V4SI_V2DF_UQI:
12997 18720 : case V4SF_FTYPE_V4SI_V4SF_UQI:
12998 18720 : case V4DF_FTYPE_V4SF_V4DF_UQI:
12999 18720 : case V4DF_FTYPE_V4SI_V4DF_UQI:
13000 18720 : case V8SI_FTYPE_V8SI_V8SI_UQI:
13001 18720 : case V8SI_FTYPE_V8HI_V8SI_UQI:
13002 18720 : case V8SI_FTYPE_V16QI_V8SI_UQI:
13003 18720 : case V8DF_FTYPE_V8SI_V8DF_UQI:
13004 18720 : case V8DI_FTYPE_DI_V8DI_UQI:
13005 18720 : case V16SF_FTYPE_V8SF_V16SF_UHI:
13006 18720 : case V16SI_FTYPE_V8SI_V16SI_UHI:
13007 18720 : case V16HF_FTYPE_V16HI_V16HF_UHI:
13008 18720 : case V16HF_FTYPE_V16HF_V16HF_V16HF:
13009 18720 : case V16HI_FTYPE_V16HF_V16HI_UHI:
13010 18720 : case V16HI_FTYPE_V16HI_V16HI_UHI:
13011 18720 : case V16BF_FTYPE_V16BF_V16BF_UHI:
13012 18720 : case V8HI_FTYPE_V16QI_V8HI_UQI:
13013 18720 : case V16HI_FTYPE_V16QI_V16HI_UHI:
13014 18720 : case V32HI_FTYPE_V32HI_V32HI_USI:
13015 18720 : case V32BF_FTYPE_V32BF_V32BF_USI:
13016 18720 : case V32HI_FTYPE_V32QI_V32HI_USI:
13017 18720 : case V8DI_FTYPE_V16QI_V8DI_UQI:
13018 18720 : case V8DI_FTYPE_V2DI_V8DI_UQI:
13019 18720 : case V8DI_FTYPE_V4DI_V8DI_UQI:
13020 18720 : case V8DI_FTYPE_V8DI_V8DI_UQI:
13021 18720 : case V8DI_FTYPE_V8HI_V8DI_UQI:
13022 18720 : case V8DI_FTYPE_V8SI_V8DI_UQI:
13023 18720 : case V8HI_FTYPE_V8DI_V8HI_UQI:
13024 18720 : case V8SI_FTYPE_V8DI_V8SI_UQI:
13025 18720 : case V4SI_FTYPE_V4SI_V4SI_V4SI:
13026 18720 : case V4DI_FTYPE_V4DI_V4DI_V2DI:
13027 18720 : case V16SI_FTYPE_V16SI_V16SI_V16SI:
13028 18720 : case V8DI_FTYPE_V8DI_V8DI_V8DI:
13029 18720 : case V32HI_FTYPE_V32HI_V32HI_V32HI:
13030 18720 : case V2DI_FTYPE_V2DI_V2DI_V2DI:
13031 18720 : case V16HI_FTYPE_V16HI_V16HI_V16HI:
13032 18720 : case V8SI_FTYPE_V8SI_V8SI_V8SI:
13033 18720 : case V8HI_FTYPE_V8HI_V8HI_V8HI:
13034 18720 : case V32BF_FTYPE_V16SF_V16SF_USI:
13035 18720 : case V16BF_FTYPE_V8SF_V8SF_UHI:
13036 18720 : case V8BF_FTYPE_V4SF_V4SF_UQI:
13037 18720 : case V16BF_FTYPE_V16SF_V16BF_UHI:
13038 18720 : case V8BF_FTYPE_V8SF_V8BF_UQI:
13039 18720 : case V8BF_FTYPE_V4SF_V8BF_UQI:
13040 18720 : case V16SF_FTYPE_V16SF_V32BF_V32BF:
13041 18720 : case V8SF_FTYPE_V8SF_V16BF_V16BF:
13042 18720 : case V4SF_FTYPE_V4SF_V8BF_V8BF:
13043 18720 : case V16QI_FTYPE_V16QI_V8HF_V8HF:
13044 18720 : case V32QI_FTYPE_V32QI_V16HF_V16HF:
13045 18720 : case V64QI_FTYPE_V64QI_V32HF_V32HF:
13046 18720 : case V16QI_FTYPE_V8HF_V16QI_UQI:
13047 18720 : case V16QI_FTYPE_V16HF_V16QI_UHI:
13048 18720 : case V32QI_FTYPE_V32HF_V32QI_USI:
13049 18720 : case V8HF_FTYPE_V16QI_V8HF_UQI:
13050 18720 : case V16HF_FTYPE_V16QI_V16HF_UHI:
13051 18720 : case V32HF_FTYPE_V32QI_V32HF_USI:
13052 18720 : case V16SI_FTYPE_V16SF_V16SI_UHI:
13053 18720 : case V32HI_FTYPE_V32HF_V32HI_USI:
13054 18720 : case V8DI_FTYPE_V8SF_V8DI_UQI:
13055 18720 : case V8DI_FTYPE_V8DF_V8DI_UQI:
13056 18720 : case V8SI_FTYPE_V8DF_V8SI_UQI:
13057 18720 : nargs = 3;
13058 18720 : break;
13059 1480 : case V32QI_FTYPE_V32QI_V32QI_INT:
13060 1480 : case V16HI_FTYPE_V16HI_V16HI_INT:
13061 1480 : case V16QI_FTYPE_V16QI_V16QI_INT:
13062 1480 : case V4DI_FTYPE_V4DI_V4DI_INT:
13063 1480 : case V8HI_FTYPE_V8HI_V8HI_INT:
13064 1480 : case V8SI_FTYPE_V8SI_V8SI_INT:
13065 1480 : case V8SI_FTYPE_V8SI_V4SI_INT:
13066 1480 : case V8SF_FTYPE_V8SF_V8SF_INT:
13067 1480 : case V8SF_FTYPE_V8SF_V4SF_INT:
13068 1480 : case V4SI_FTYPE_V4SI_V4SI_INT:
13069 1480 : case V4DF_FTYPE_V4DF_V4DF_INT:
13070 1480 : case V16SF_FTYPE_V16SF_V16SF_INT:
13071 1480 : case V16SF_FTYPE_V16SF_V4SF_INT:
13072 1480 : case V16SI_FTYPE_V16SI_V4SI_INT:
13073 1480 : case V4DF_FTYPE_V4DF_V2DF_INT:
13074 1480 : case V4SF_FTYPE_V4SF_V4SF_INT:
13075 1480 : case V2DI_FTYPE_V2DI_V2DI_INT:
13076 1480 : case V4DI_FTYPE_V4DI_V2DI_INT:
13077 1480 : case V2DF_FTYPE_V2DF_V2DF_INT:
13078 1480 : case UQI_FTYPE_V8DI_V8UDI_INT:
13079 1480 : case UQI_FTYPE_V8DF_V8DF_INT:
13080 1480 : case UQI_FTYPE_V2DF_V2DF_INT:
13081 1480 : case UQI_FTYPE_V4SF_V4SF_INT:
13082 1480 : case UHI_FTYPE_V16SI_V16SI_INT:
13083 1480 : case UHI_FTYPE_V16SF_V16SF_INT:
13084 1480 : case V64QI_FTYPE_V64QI_V64QI_INT:
13085 1480 : case V32HI_FTYPE_V32HI_V32HI_INT:
13086 1480 : case V16SI_FTYPE_V16SI_V16SI_INT:
13087 1480 : case V8DI_FTYPE_V8DI_V8DI_INT:
13088 1480 : nargs = 3;
13089 1480 : nargs_constant = 1;
13090 1480 : break;
13091 47 : case V4DI_FTYPE_V4DI_V4DI_INT_CONVERT:
13092 47 : nargs = 3;
13093 47 : rmode = V4DImode;
13094 47 : nargs_constant = 1;
13095 47 : break;
13096 80 : case V2DI_FTYPE_V2DI_V2DI_INT_CONVERT:
13097 80 : nargs = 3;
13098 80 : rmode = V2DImode;
13099 80 : nargs_constant = 1;
13100 80 : break;
13101 48 : case V1DI_FTYPE_V1DI_V1DI_INT_CONVERT:
13102 48 : nargs = 3;
13103 48 : rmode = DImode;
13104 48 : nargs_constant = 1;
13105 48 : break;
13106 20 : case V2DI_FTYPE_V2DI_UINT_UINT:
13107 20 : nargs = 3;
13108 20 : nargs_constant = 2;
13109 20 : break;
13110 8 : case V8DI_FTYPE_V8DI_V8DI_INT_CONVERT:
13111 8 : nargs = 3;
13112 8 : rmode = V8DImode;
13113 8 : nargs_constant = 1;
13114 8 : break;
13115 16 : case V8DI_FTYPE_V8DI_V8DI_INT_V8DI_UDI_CONVERT:
13116 16 : nargs = 5;
13117 16 : rmode = V8DImode;
13118 16 : mask_pos = 2;
13119 16 : nargs_constant = 1;
13120 16 : break;
13121 320 : case QI_FTYPE_V8DF_INT_UQI:
13122 320 : case QI_FTYPE_V4DF_INT_UQI:
13123 320 : case QI_FTYPE_V2DF_INT_UQI:
13124 320 : case HI_FTYPE_V16SF_INT_UHI:
13125 320 : case QI_FTYPE_V8SF_INT_UQI:
13126 320 : case QI_FTYPE_V4SF_INT_UQI:
13127 320 : case QI_FTYPE_V8HF_INT_UQI:
13128 320 : case HI_FTYPE_V16HF_INT_UHI:
13129 320 : case SI_FTYPE_V32HF_INT_USI:
13130 320 : case QI_FTYPE_V8BF_INT_UQI:
13131 320 : case HI_FTYPE_V16BF_INT_UHI:
13132 320 : case SI_FTYPE_V32BF_INT_USI:
13133 320 : case V4SI_FTYPE_V4SI_V4SI_UHI:
13134 320 : case V8SI_FTYPE_V8SI_V8SI_UHI:
13135 320 : nargs = 3;
13136 320 : mask_pos = 1;
13137 320 : nargs_constant = 1;
13138 320 : break;
13139 17 : case V4DI_FTYPE_V4DI_V4DI_INT_V4DI_USI_CONVERT:
13140 17 : nargs = 5;
13141 17 : rmode = V4DImode;
13142 17 : mask_pos = 2;
13143 17 : nargs_constant = 1;
13144 17 : break;
13145 17 : case V2DI_FTYPE_V2DI_V2DI_INT_V2DI_UHI_CONVERT:
13146 17 : nargs = 5;
13147 17 : rmode = V2DImode;
13148 17 : mask_pos = 2;
13149 17 : nargs_constant = 1;
13150 17 : break;
13151 17247 : case V32QI_FTYPE_V32QI_V32QI_V32QI_USI:
13152 17247 : case V32HI_FTYPE_V32HI_V32HI_V32HI_USI:
13153 17247 : case V32BF_FTYPE_V32BF_V32BF_V32BF_USI:
13154 17247 : case V32HI_FTYPE_V64QI_V64QI_V32HI_USI:
13155 17247 : case V16SI_FTYPE_V32HI_V32HI_V16SI_UHI:
13156 17247 : case V64QI_FTYPE_V64QI_V64QI_V64QI_UDI:
13157 17247 : case V32HI_FTYPE_V32HI_V8HI_V32HI_USI:
13158 17247 : case V16HI_FTYPE_V16HI_V8HI_V16HI_UHI:
13159 17247 : case V8SI_FTYPE_V8SI_V4SI_V8SI_UQI:
13160 17247 : case V4DI_FTYPE_V4DI_V2DI_V4DI_UQI:
13161 17247 : case V64QI_FTYPE_V32HI_V32HI_V64QI_UDI:
13162 17247 : case V32QI_FTYPE_V16HI_V16HI_V32QI_USI:
13163 17247 : case V16QI_FTYPE_V8HI_V8HI_V16QI_UHI:
13164 17247 : case V32HI_FTYPE_V16SI_V16SI_V32HI_USI:
13165 17247 : case V16HI_FTYPE_V8SI_V8SI_V16HI_UHI:
13166 17247 : case V8HI_FTYPE_V4SI_V4SI_V8HI_UQI:
13167 17247 : case V4DF_FTYPE_V4DF_V4DI_V4DF_UQI:
13168 17247 : case V32HF_FTYPE_V32HF_V32HF_V32HF_USI:
13169 17247 : case V8SF_FTYPE_V8SF_V8SI_V8SF_UQI:
13170 17247 : case V4SF_FTYPE_V4SF_V4SI_V4SF_UQI:
13171 17247 : case V2DF_FTYPE_V2DF_V2DI_V2DF_UQI:
13172 17247 : case V2DI_FTYPE_V4SI_V4SI_V2DI_UQI:
13173 17247 : case V4DI_FTYPE_V8SI_V8SI_V4DI_UQI:
13174 17247 : case V4DF_FTYPE_V4DI_V4DF_V4DF_UQI:
13175 17247 : case V8SF_FTYPE_V8SI_V8SF_V8SF_UQI:
13176 17247 : case V2DF_FTYPE_V2DI_V2DF_V2DF_UQI:
13177 17247 : case V4SF_FTYPE_V4SI_V4SF_V4SF_UQI:
13178 17247 : case V16SF_FTYPE_V16SF_V16SF_V16SF_UHI:
13179 17247 : case V16SF_FTYPE_V16SF_V16SI_V16SF_UHI:
13180 17247 : case V16SF_FTYPE_V16SI_V16SF_V16SF_UHI:
13181 17247 : case V16SI_FTYPE_V16SI_V16SI_V16SI_UHI:
13182 17247 : case V16SI_FTYPE_V16SI_V4SI_V16SI_UHI:
13183 17247 : case V8HI_FTYPE_V8HI_V8HI_V8HI_UQI:
13184 17247 : case V8BF_FTYPE_V8BF_V8BF_V8BF_UQI:
13185 17247 : case V8SI_FTYPE_V8SI_V8SI_V8SI_UQI:
13186 17247 : case V4SI_FTYPE_V4SI_V4SI_V4SI_UQI:
13187 17247 : case V16HF_FTYPE_V16HF_V16HF_V16HF_UQI:
13188 17247 : case V16HF_FTYPE_V16HF_V16HF_V16HF_UHI:
13189 17247 : case V8SF_FTYPE_V8SF_V8SF_V8SF_UQI:
13190 17247 : case V16QI_FTYPE_V16QI_V16QI_V16QI_UHI:
13191 17247 : case V16HI_FTYPE_V16HI_V16HI_V16HI_UHI:
13192 17247 : case V16BF_FTYPE_V16BF_V16BF_V16BF_UHI:
13193 17247 : case V2DI_FTYPE_V2DI_V2DI_V2DI_UQI:
13194 17247 : case V2DF_FTYPE_V2DF_V2DF_V2DF_UQI:
13195 17247 : case V4DI_FTYPE_V4DI_V4DI_V4DI_UQI:
13196 17247 : case V4DF_FTYPE_V4DF_V4DF_V4DF_UQI:
13197 17247 : case V8HF_FTYPE_V8HF_V8HF_V8HF_UQI:
13198 17247 : case V4SF_FTYPE_V4SF_V4SF_V4SF_UQI:
13199 17247 : case V8DF_FTYPE_V8DF_V8DF_V8DF_UQI:
13200 17247 : case V8DF_FTYPE_V8DF_V8DI_V8DF_UQI:
13201 17247 : case V8DF_FTYPE_V8DI_V8DF_V8DF_UQI:
13202 17247 : case V8DI_FTYPE_V16SI_V16SI_V8DI_UQI:
13203 17247 : case V8DI_FTYPE_V8DI_V2DI_V8DI_UQI:
13204 17247 : case V8DI_FTYPE_V8DI_V8DI_V8DI_UQI:
13205 17247 : case V8HI_FTYPE_V16QI_V16QI_V8HI_UQI:
13206 17247 : case V16HI_FTYPE_V32QI_V32QI_V16HI_UHI:
13207 17247 : case V8SI_FTYPE_V16HI_V16HI_V8SI_UQI:
13208 17247 : case V4SI_FTYPE_V8HI_V8HI_V4SI_UQI:
13209 17247 : case V32BF_FTYPE_V16SF_V16SF_V32BF_USI:
13210 17247 : case V16BF_FTYPE_V8SF_V8SF_V16BF_UHI:
13211 17247 : case V8BF_FTYPE_V4SF_V4SF_V8BF_UQI:
13212 17247 : case V32HF_FTYPE_V16SF_V16SF_V32HF_USI:
13213 17247 : case V16HF_FTYPE_V8SF_V8SF_V16HF_UHI:
13214 17247 : case V8HF_FTYPE_V4SF_V4SF_V8HF_UQI:
13215 17247 : case V16QI_FTYPE_V8HF_V8HF_V16QI_UHI:
13216 17247 : case V32QI_FTYPE_V16HF_V16HF_V32QI_USI:
13217 17247 : case V64QI_FTYPE_V32HF_V32HF_V64QI_UDI:
13218 17247 : case V16QI_FTYPE_V16QI_V8HF_V16QI_UHI:
13219 17247 : case V16QI_FTYPE_V32QI_V16HF_V16QI_UHI:
13220 17247 : case V32QI_FTYPE_V64QI_V32HF_V32QI_USI:
13221 17247 : nargs = 4;
13222 17247 : break;
13223 11 : case V2DF_FTYPE_V2DF_V2DF_V2DI_INT:
13224 11 : case V4DF_FTYPE_V4DF_V4DF_V4DI_INT:
13225 11 : case V4SF_FTYPE_V4SF_V4SF_V4SI_INT:
13226 11 : case V8SF_FTYPE_V8SF_V8SF_V8SI_INT:
13227 11 : case V16SF_FTYPE_V16SF_V16SF_V16SI_INT:
13228 11 : case V4SI_FTYPE_V4SI_V4SI_V4SI_INT:
13229 11 : nargs = 4;
13230 11 : nargs_constant = 1;
13231 11 : break;
13232 3718 : case UQI_FTYPE_V4DI_V4DI_INT_UQI:
13233 3718 : case UQI_FTYPE_V8SI_V8SI_INT_UQI:
13234 3718 : case QI_FTYPE_V4DF_V4DF_INT_UQI:
13235 3718 : case QI_FTYPE_V8SF_V8SF_INT_UQI:
13236 3718 : case UHI_FTYPE_V16HF_V16HF_INT_UHI:
13237 3718 : case UQI_FTYPE_V2DI_V2DI_INT_UQI:
13238 3718 : case UQI_FTYPE_V4SI_V4SI_INT_UQI:
13239 3718 : case UQI_FTYPE_V2DF_V2DF_INT_UQI:
13240 3718 : case UQI_FTYPE_V4SF_V4SF_INT_UQI:
13241 3718 : case UQI_FTYPE_V8HF_V8HF_INT_UQI:
13242 3718 : case UDI_FTYPE_V64QI_V64QI_INT_UDI:
13243 3718 : case USI_FTYPE_V32QI_V32QI_INT_USI:
13244 3718 : case UHI_FTYPE_V16QI_V16QI_INT_UHI:
13245 3718 : case USI_FTYPE_V32HI_V32HI_INT_USI:
13246 3718 : case USI_FTYPE_V32BF_V32BF_INT_USI:
13247 3718 : case USI_FTYPE_V32HF_V32HF_INT_USI:
13248 3718 : case UHI_FTYPE_V16HI_V16HI_INT_UHI:
13249 3718 : case UHI_FTYPE_V16BF_V16BF_INT_UHI:
13250 3718 : case UQI_FTYPE_V8HI_V8HI_INT_UQI:
13251 3718 : case UQI_FTYPE_V8BF_V8BF_INT_UQI:
13252 3718 : nargs = 4;
13253 3718 : mask_pos = 1;
13254 3718 : nargs_constant = 1;
13255 3718 : break;
13256 23 : case V2DI_FTYPE_V2DI_V2DI_UINT_UINT:
13257 23 : nargs = 4;
13258 23 : nargs_constant = 2;
13259 23 : break;
13260 67 : case UCHAR_FTYPE_UCHAR_UINT_UINT_PUNSIGNED:
13261 67 : case UCHAR_FTYPE_UCHAR_ULONGLONG_ULONGLONG_PULONGLONG:
13262 67 : case V16SF_FTYPE_V16SF_V32BF_V32BF_UHI:
13263 67 : case V8SF_FTYPE_V8SF_V16BF_V16BF_UQI:
13264 67 : case V4SF_FTYPE_V4SF_V8BF_V8BF_UQI:
13265 67 : nargs = 4;
13266 67 : break;
13267 679 : case UQI_FTYPE_V8DI_V8DI_INT_UQI:
13268 679 : case UHI_FTYPE_V16SI_V16SI_INT_UHI:
13269 679 : mask_pos = 1;
13270 679 : nargs = 4;
13271 679 : nargs_constant = 1;
13272 679 : break;
13273 3948 : case V8SF_FTYPE_V8SF_INT_V8SF_UQI:
13274 3948 : case V4SF_FTYPE_V4SF_INT_V4SF_UQI:
13275 3948 : case V2DF_FTYPE_V4DF_INT_V2DF_UQI:
13276 3948 : case V2DI_FTYPE_V4DI_INT_V2DI_UQI:
13277 3948 : case V8SF_FTYPE_V16SF_INT_V8SF_UQI:
13278 3948 : case V8SI_FTYPE_V16SI_INT_V8SI_UQI:
13279 3948 : case V2DF_FTYPE_V8DF_INT_V2DF_UQI:
13280 3948 : case V2DI_FTYPE_V8DI_INT_V2DI_UQI:
13281 3948 : case V4SF_FTYPE_V8SF_INT_V4SF_UQI:
13282 3948 : case V4SI_FTYPE_V8SI_INT_V4SI_UQI:
13283 3948 : case V8HI_FTYPE_V8SF_INT_V8HI_UQI:
13284 3948 : case V8HI_FTYPE_V4SF_INT_V8HI_UQI:
13285 3948 : case V32HI_FTYPE_V32HI_INT_V32HI_USI:
13286 3948 : case V16HI_FTYPE_V16HI_INT_V16HI_UHI:
13287 3948 : case V8HI_FTYPE_V8HI_INT_V8HI_UQI:
13288 3948 : case V32BF_FTYPE_V32BF_INT_V32BF_USI:
13289 3948 : case V16BF_FTYPE_V16BF_INT_V16BF_UHI:
13290 3948 : case V8BF_FTYPE_V8BF_INT_V8BF_UQI:
13291 3948 : case V4DI_FTYPE_V4DI_INT_V4DI_UQI:
13292 3948 : case V2DI_FTYPE_V2DI_INT_V2DI_UQI:
13293 3948 : case V8SI_FTYPE_V8SI_INT_V8SI_UQI:
13294 3948 : case V4SI_FTYPE_V4SI_INT_V4SI_UQI:
13295 3948 : case V4DF_FTYPE_V4DF_INT_V4DF_UQI:
13296 3948 : case V2DF_FTYPE_V2DF_INT_V2DF_UQI:
13297 3948 : case V8DF_FTYPE_V8DF_INT_V8DF_UQI:
13298 3948 : case V16SF_FTYPE_V16SF_INT_V16SF_UHI:
13299 3948 : case V16HI_FTYPE_V16SF_INT_V16HI_UHI:
13300 3948 : case V16SI_FTYPE_V16SI_INT_V16SI_UHI:
13301 3948 : case V16HF_FTYPE_V16HF_INT_V16HF_UHI:
13302 3948 : case V8HF_FTYPE_V8HF_INT_V8HF_UQI:
13303 3948 : case V4SI_FTYPE_V16SI_INT_V4SI_UQI:
13304 3948 : case V4DI_FTYPE_V8DI_INT_V4DI_UQI:
13305 3948 : case V4DF_FTYPE_V8DF_INT_V4DF_UQI:
13306 3948 : case V4SF_FTYPE_V16SF_INT_V4SF_UQI:
13307 3948 : case V8DI_FTYPE_V8DI_INT_V8DI_UQI:
13308 3948 : nargs = 4;
13309 3948 : mask_pos = 2;
13310 3948 : nargs_constant = 1;
13311 3948 : break;
13312 1726 : case V16SF_FTYPE_V16SF_V4SF_INT_V16SF_UHI:
13313 1726 : case V16SI_FTYPE_V16SI_V4SI_INT_V16SI_UHI:
13314 1726 : case V8DF_FTYPE_V8DF_V8DF_INT_V8DF_UQI:
13315 1726 : case V8DI_FTYPE_V8DI_V8DI_INT_V8DI_UQI:
13316 1726 : case V16SF_FTYPE_V16SF_V16SF_INT_V16SF_UHI:
13317 1726 : case V16SI_FTYPE_V16SI_V16SI_INT_V16SI_UHI:
13318 1726 : case V4SF_FTYPE_V4SF_V4SF_INT_V4SF_UQI:
13319 1726 : case V2DF_FTYPE_V2DF_V2DF_INT_V2DF_UQI:
13320 1726 : case V8DF_FTYPE_V8DF_V4DF_INT_V8DF_UQI:
13321 1726 : case V8DI_FTYPE_V8DI_V4DI_INT_V8DI_UQI:
13322 1726 : case V4DF_FTYPE_V4DF_V4DF_INT_V4DF_UQI:
13323 1726 : case V8SF_FTYPE_V8SF_V8SF_INT_V8SF_UQI:
13324 1726 : case V8DF_FTYPE_V8DF_V2DF_INT_V8DF_UQI:
13325 1726 : case V8DI_FTYPE_V8DI_V2DI_INT_V8DI_UQI:
13326 1726 : case V8SI_FTYPE_V8SI_V8SI_INT_V8SI_UQI:
13327 1726 : case V4DI_FTYPE_V4DI_V4DI_INT_V4DI_UQI:
13328 1726 : case V4SI_FTYPE_V4SI_V4SI_INT_V4SI_UQI:
13329 1726 : case V2DI_FTYPE_V2DI_V2DI_INT_V2DI_UQI:
13330 1726 : case V32HI_FTYPE_V64QI_V64QI_INT_V32HI_USI:
13331 1726 : case V16HI_FTYPE_V32QI_V32QI_INT_V16HI_UHI:
13332 1726 : case V8HI_FTYPE_V16QI_V16QI_INT_V8HI_UQI:
13333 1726 : case V16SF_FTYPE_V16SF_V8SF_INT_V16SF_UHI:
13334 1726 : case V16SI_FTYPE_V16SI_V8SI_INT_V16SI_UHI:
13335 1726 : case V8SF_FTYPE_V8SF_V4SF_INT_V8SF_UQI:
13336 1726 : case V8SI_FTYPE_V8SI_V4SI_INT_V8SI_UQI:
13337 1726 : case V4DI_FTYPE_V4DI_V2DI_INT_V4DI_UQI:
13338 1726 : case V4DF_FTYPE_V4DF_V2DF_INT_V4DF_UQI:
13339 1726 : nargs = 5;
13340 1726 : mask_pos = 2;
13341 1726 : nargs_constant = 1;
13342 1726 : break;
13343 268 : case V8DI_FTYPE_V8DI_V8DI_V8DI_INT_UQI:
13344 268 : case V16SI_FTYPE_V16SI_V16SI_V16SI_INT_UHI:
13345 268 : case V2DF_FTYPE_V2DF_V2DF_V2DI_INT_UQI:
13346 268 : case V4SF_FTYPE_V4SF_V4SF_V4SI_INT_UQI:
13347 268 : case V8SF_FTYPE_V8SF_V8SF_V8SI_INT_UQI:
13348 268 : case V8SI_FTYPE_V8SI_V8SI_V8SI_INT_UQI:
13349 268 : case V4DF_FTYPE_V4DF_V4DF_V4DI_INT_UQI:
13350 268 : case V4DI_FTYPE_V4DI_V4DI_V4DI_INT_UQI:
13351 268 : case V4SI_FTYPE_V4SI_V4SI_V4SI_INT_UQI:
13352 268 : case V2DI_FTYPE_V2DI_V2DI_V2DI_INT_UQI:
13353 268 : nargs = 5;
13354 268 : mask_pos = 1;
13355 268 : nargs_constant = 1;
13356 268 : break;
13357 732 : case V64QI_FTYPE_V64QI_V64QI_INT_V64QI_UDI:
13358 732 : case V32QI_FTYPE_V32QI_V32QI_INT_V32QI_USI:
13359 732 : case V16QI_FTYPE_V16QI_V16QI_INT_V16QI_UHI:
13360 732 : case V32HI_FTYPE_V32HI_V32HI_INT_V32HI_INT:
13361 732 : case V16SI_FTYPE_V16SI_V16SI_INT_V16SI_INT:
13362 732 : case V8DI_FTYPE_V8DI_V8DI_INT_V8DI_INT:
13363 732 : case V16HI_FTYPE_V16HI_V16HI_INT_V16HI_INT:
13364 732 : case V8SI_FTYPE_V8SI_V8SI_INT_V8SI_INT:
13365 732 : case V4DI_FTYPE_V4DI_V4DI_INT_V4DI_INT:
13366 732 : case V8HI_FTYPE_V8HI_V8HI_INT_V8HI_INT:
13367 732 : case V4SI_FTYPE_V4SI_V4SI_INT_V4SI_INT:
13368 732 : case V2DI_FTYPE_V2DI_V2DI_INT_V2DI_INT:
13369 732 : case V8BF_FTYPE_V8BF_V8BF_INT_V8BF_UQI:
13370 732 : case V16BF_FTYPE_V16BF_V16BF_INT_V16BF_UHI:
13371 732 : case V32BF_FTYPE_V32BF_V32BF_INT_V32BF_USI:
13372 732 : case V16HF_FTYPE_V16HF_V16HF_INT_V16HF_UHI:
13373 732 : case V8HF_FTYPE_V8HF_V8HF_INT_V8HF_UQI:
13374 732 : nargs = 5;
13375 732 : mask_pos = 1;
13376 732 : nargs_constant = 2;
13377 732 : break;
13378 :
13379 0 : default:
13380 0 : gcc_unreachable ();
13381 : }
13382 :
13383 56435 : gcc_assert (nargs <= ARRAY_SIZE (xops));
13384 :
13385 63618 : if (comparison != UNKNOWN)
13386 : {
13387 616 : gcc_assert (nargs == 2);
13388 616 : return ix86_expand_sse_compare (d, exp, target, swap);
13389 : }
13390 :
13391 63002 : if (rmode == VOIDmode || rmode == tmode)
13392 : {
13393 62817 : if (optimize
13394 17705 : || target == 0
13395 17705 : || GET_MODE (target) != tmode
13396 80320 : || !insn_p->operand[0].predicate (target, tmode))
13397 45402 : target = gen_reg_rtx (tmode);
13398 17415 : else if (memory_operand (target, tmode))
13399 578 : num_memory++;
13400 : real_target = target;
13401 : }
13402 : else
13403 : {
13404 185 : real_target = gen_reg_rtx (tmode);
13405 185 : target = lowpart_subreg (rmode, real_target, tmode);
13406 : }
13407 :
13408 265291 : for (i = 0; i < nargs; i++)
13409 : {
13410 202522 : tree arg = CALL_EXPR_ARG (exp, i);
13411 202522 : rtx op = ix86_expand_unsigned_small_int_cst_argument (arg);
13412 202522 : machine_mode mode = insn_p->operand[i + 1].mode;
13413 : /* Need to fixup modeless constant before testing predicate. */
13414 202522 : op = fixup_modeless_constant (op, mode);
13415 202522 : bool match = insn_p->operand[i + 1].predicate (op, mode);
13416 :
13417 202522 : if (second_arg_count && i == 1)
13418 : {
13419 : /* SIMD shift insns take either an 8-bit immediate or
13420 : register as count. But builtin functions take int as
13421 : count. If count doesn't match, we put it in register.
13422 : The instructions are using 64-bit count, if op is just
13423 : 32-bit, zero-extend it, as negative shift counts
13424 : are undefined behavior and zero-extension is more
13425 : efficient. */
13426 2889 : if (!match)
13427 : {
13428 1750 : if (SCALAR_INT_MODE_P (GET_MODE (op)))
13429 489 : op = convert_modes (mode, GET_MODE (op), op, 1);
13430 : else
13431 1261 : op = lowpart_subreg (mode, op, GET_MODE (op));
13432 1750 : if (!insn_p->operand[i + 1].predicate (op, mode))
13433 190 : op = copy_to_reg (op);
13434 : }
13435 : }
13436 199633 : else if ((mask_pos && (nargs - i - mask_pos) == nargs_constant) ||
13437 151577 : (!mask_pos && (nargs - i) <= nargs_constant))
13438 : {
13439 16585 : if (!match)
13440 233 : switch (icode)
13441 : {
13442 2 : case CODE_FOR_avx_vinsertf128v4di:
13443 2 : case CODE_FOR_avx_vextractf128v4di:
13444 2 : error ("the last argument must be an 1-bit immediate");
13445 2 : return const0_rtx;
13446 :
13447 8 : case CODE_FOR_avx512f_cmpv8di3_mask:
13448 8 : case CODE_FOR_avx512f_cmpv16si3_mask:
13449 8 : case CODE_FOR_avx512f_ucmpv8di3_mask:
13450 8 : case CODE_FOR_avx512f_ucmpv16si3_mask:
13451 8 : case CODE_FOR_avx512vl_cmpv4di3_mask:
13452 8 : case CODE_FOR_avx512vl_cmpv8si3_mask:
13453 8 : case CODE_FOR_avx512vl_ucmpv4di3_mask:
13454 8 : case CODE_FOR_avx512vl_ucmpv8si3_mask:
13455 8 : case CODE_FOR_avx512vl_cmpv2di3_mask:
13456 8 : case CODE_FOR_avx512vl_cmpv4si3_mask:
13457 8 : case CODE_FOR_avx512vl_ucmpv2di3_mask:
13458 8 : case CODE_FOR_avx512vl_ucmpv4si3_mask:
13459 8 : error ("the last argument must be a 3-bit immediate");
13460 8 : return const0_rtx;
13461 :
13462 24 : case CODE_FOR_sse4_1_roundsd:
13463 24 : case CODE_FOR_sse4_1_roundss:
13464 :
13465 24 : case CODE_FOR_sse4_1_roundpd:
13466 24 : case CODE_FOR_sse4_1_roundps:
13467 24 : case CODE_FOR_avx_roundpd256:
13468 24 : case CODE_FOR_avx_roundps256:
13469 :
13470 24 : case CODE_FOR_sse4_1_roundpd_vec_pack_sfix:
13471 24 : case CODE_FOR_sse4_1_roundps_sfix:
13472 24 : case CODE_FOR_avx_roundpd_vec_pack_sfix256:
13473 24 : case CODE_FOR_avx_roundps_sfix256:
13474 :
13475 24 : case CODE_FOR_sse4_1_blendps:
13476 24 : case CODE_FOR_avx_blendpd256:
13477 24 : case CODE_FOR_avx_vpermilv4df:
13478 24 : case CODE_FOR_avx_vpermilv4df_mask:
13479 24 : case CODE_FOR_avx512f_getmantv8df_mask:
13480 24 : case CODE_FOR_avx512f_getmantv16sf_mask:
13481 24 : case CODE_FOR_avx512vl_getmantv16hf_mask:
13482 24 : case CODE_FOR_avx512vl_getmantv8sf_mask:
13483 24 : case CODE_FOR_avx512vl_getmantv4df_mask:
13484 24 : case CODE_FOR_avx512fp16_getmantv8hf_mask:
13485 24 : case CODE_FOR_avx512vl_getmantv4sf_mask:
13486 24 : case CODE_FOR_avx512vl_getmantv2df_mask:
13487 24 : case CODE_FOR_avx512dq_rangepv8df_mask_round:
13488 24 : case CODE_FOR_avx512dq_rangepv16sf_mask_round:
13489 24 : case CODE_FOR_avx512dq_rangepv4df_mask:
13490 24 : case CODE_FOR_avx512dq_rangepv8sf_mask:
13491 24 : case CODE_FOR_avx512dq_rangepv2df_mask:
13492 24 : case CODE_FOR_avx512dq_rangepv4sf_mask:
13493 24 : case CODE_FOR_avx_shufpd256_mask:
13494 24 : error ("the last argument must be a 4-bit immediate");
13495 24 : return const0_rtx;
13496 :
13497 15 : case CODE_FOR_sha1rnds4:
13498 15 : case CODE_FOR_sse4_1_blendpd:
13499 15 : case CODE_FOR_avx_vpermilv2df:
13500 15 : case CODE_FOR_avx_vpermilv2df_mask:
13501 15 : case CODE_FOR_xop_vpermil2v2df3:
13502 15 : case CODE_FOR_xop_vpermil2v4sf3:
13503 15 : case CODE_FOR_xop_vpermil2v4df3:
13504 15 : case CODE_FOR_xop_vpermil2v8sf3:
13505 15 : case CODE_FOR_avx512f_vinsertf32x4_mask:
13506 15 : case CODE_FOR_avx512f_vinserti32x4_mask:
13507 15 : case CODE_FOR_avx512f_vextractf32x4_mask:
13508 15 : case CODE_FOR_avx512f_vextracti32x4_mask:
13509 15 : case CODE_FOR_sse2_shufpd:
13510 15 : case CODE_FOR_sse2_shufpd_mask:
13511 15 : case CODE_FOR_avx512dq_shuf_f64x2_mask:
13512 15 : case CODE_FOR_avx512dq_shuf_i64x2_mask:
13513 15 : case CODE_FOR_avx512vl_shuf_i32x4_mask:
13514 15 : case CODE_FOR_avx512vl_shuf_f32x4_mask:
13515 15 : error ("the last argument must be a 2-bit immediate");
13516 15 : return const0_rtx;
13517 :
13518 30 : case CODE_FOR_avx_vextractf128v4df:
13519 30 : case CODE_FOR_avx_vextractf128v8sf:
13520 30 : case CODE_FOR_avx_vextractf128v8si:
13521 30 : case CODE_FOR_avx_vinsertf128v4df:
13522 30 : case CODE_FOR_avx_vinsertf128v8sf:
13523 30 : case CODE_FOR_avx_vinsertf128v8si:
13524 30 : case CODE_FOR_avx512f_vinsertf64x4_mask:
13525 30 : case CODE_FOR_avx512f_vinserti64x4_mask:
13526 30 : case CODE_FOR_avx512f_vextractf64x4_mask:
13527 30 : case CODE_FOR_avx512f_vextracti64x4_mask:
13528 30 : case CODE_FOR_avx512dq_vinsertf32x8_mask:
13529 30 : case CODE_FOR_avx512dq_vinserti32x8_mask:
13530 30 : case CODE_FOR_avx512vl_vinsertv4df:
13531 30 : case CODE_FOR_avx512vl_vinsertv4di:
13532 30 : case CODE_FOR_avx512vl_vinsertv8sf:
13533 30 : case CODE_FOR_avx512vl_vinsertv8si:
13534 30 : error ("the last argument must be a 1-bit immediate");
13535 30 : return const0_rtx;
13536 :
13537 16 : case CODE_FOR_avx_vmcmpv2df3:
13538 16 : case CODE_FOR_avx_vmcmpv4sf3:
13539 16 : case CODE_FOR_avx_cmpv2df3:
13540 16 : case CODE_FOR_avx_cmpv4sf3:
13541 16 : if (CONST_INT_P (op) && IN_RANGE (INTVAL (op), 8, 31))
13542 : {
13543 4 : error ("'%s' needs isa option %s", d->name, "-mavx");
13544 4 : return const0_rtx;
13545 : }
13546 : /* FALLTHRU */
13547 18 : case CODE_FOR_avx_cmpv4df3:
13548 18 : case CODE_FOR_avx_cmpv8sf3:
13549 18 : case CODE_FOR_avx512f_cmpv8df3_mask:
13550 18 : case CODE_FOR_avx512f_cmpv16sf3_mask:
13551 18 : case CODE_FOR_avx512f_vmcmpv2df3_mask:
13552 18 : case CODE_FOR_avx512f_vmcmpv4sf3_mask:
13553 18 : case CODE_FOR_avx512bw_cmpv32hf3_mask:
13554 18 : case CODE_FOR_avx512vl_cmpv16hf3_mask:
13555 18 : case CODE_FOR_avx512fp16_cmpv8hf3_mask:
13556 18 : error ("the last argument must be a 5-bit immediate");
13557 18 : return const0_rtx;
13558 :
13559 132 : default:
13560 132 : switch (nargs_constant)
13561 : {
13562 8 : case 2:
13563 8 : if ((mask_pos && (nargs - i - mask_pos) == nargs_constant) ||
13564 8 : (!mask_pos && (nargs - i) == nargs_constant))
13565 : {
13566 4 : error ("the next to last argument must be an 8-bit immediate");
13567 4 : break;
13568 : }
13569 : /* FALLTHRU */
13570 128 : case 1:
13571 128 : error ("the last argument must be an 8-bit immediate");
13572 128 : break;
13573 0 : default:
13574 0 : gcc_unreachable ();
13575 : }
13576 132 : return const0_rtx;
13577 : }
13578 : }
13579 : else
13580 : {
13581 183048 : if (VECTOR_MODE_P (mode))
13582 132445 : op = safe_vector_operand (op, mode);
13583 :
13584 : /* If we aren't optimizing, only allow one memory operand to
13585 : be generated. */
13586 183048 : if (memory_operand (op, mode))
13587 : {
13588 29812 : num_memory++;
13589 29812 : if (!optimize && num_memory > 1)
13590 13571 : op = copy_to_mode_reg (mode, op);
13591 : }
13592 :
13593 183048 : if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
13594 : {
13595 180638 : if (!match)
13596 42550 : op = copy_to_mode_reg (mode, op);
13597 : }
13598 : else
13599 : {
13600 2410 : op = copy_to_reg (op);
13601 2410 : op = lowpart_subreg (mode, op, GET_MODE (op));
13602 : }
13603 : }
13604 :
13605 202289 : xops[i] = op;
13606 : }
13607 :
13608 62769 : switch (nargs)
13609 : {
13610 6567 : case 1:
13611 6567 : pat = GEN_FCN (icode) (real_target, xops[0]);
13612 6567 : break;
13613 5782 : case 2:
13614 5782 : pat = GEN_FCN (icode) (real_target, xops[0], xops[1]);
13615 5782 : break;
13616 20633 : case 3:
13617 20633 : pat = GEN_FCN (icode) (real_target, xops[0], xops[1], xops[2]);
13618 20633 : break;
13619 27047 : case 4:
13620 27047 : pat = GEN_FCN (icode) (real_target, xops[0], xops[1],
13621 27047 : xops[2], xops[3]);
13622 27047 : break;
13623 2740 : case 5:
13624 2740 : pat = GEN_FCN (icode) (real_target, xops[0], xops[1],
13625 2740 : xops[2], xops[3], xops[4]);
13626 2740 : break;
13627 : case 6:
13628 : pat = GEN_FCN (icode) (real_target, xops[0], xops[1],
13629 : xops[2], xops[3], xops[4], xops[5]);
13630 : break;
13631 : default:
13632 : gcc_unreachable ();
13633 : }
13634 :
13635 62769 : if (! pat)
13636 : return 0;
13637 :
13638 62769 : emit_insn (pat);
13639 62769 : return target;
13640 : }
13641 :
13642 : /* Transform pattern of following layout:
13643 : (set A
13644 : (unspec [B C] UNSPEC_EMBEDDED_ROUNDING))
13645 : )
13646 : into:
13647 : (set (A B)) */
13648 :
13649 : static rtx
13650 4927 : ix86_erase_embedded_rounding (rtx pat)
13651 : {
13652 4927 : if (NONJUMP_INSN_P (pat))
13653 677 : pat = PATTERN (pat);
13654 :
13655 4927 : gcc_assert (GET_CODE (pat) == SET);
13656 4927 : rtx src = SET_SRC (pat);
13657 4927 : gcc_assert (XVECLEN (src, 0) == 2);
13658 4927 : rtx p0 = XVECEXP (src, 0, 0);
13659 4927 : gcc_assert (GET_CODE (src) == UNSPEC
13660 : && XINT (src, 1) == UNSPEC_EMBEDDED_ROUNDING);
13661 4927 : rtx res = gen_rtx_SET (SET_DEST (pat), p0);
13662 4927 : return res;
13663 : }
13664 :
13665 : /* Subroutine of ix86_expand_round_builtin to take care of comi insns
13666 : with rounding. */
13667 : static rtx
13668 103 : ix86_expand_sse_comi_round (const struct builtin_description *d,
13669 : tree exp, rtx target, bool comx_ok)
13670 : {
13671 103 : rtx pat, set_dst;
13672 103 : tree arg0 = CALL_EXPR_ARG (exp, 0);
13673 103 : tree arg1 = CALL_EXPR_ARG (exp, 1);
13674 103 : tree arg2 = CALL_EXPR_ARG (exp, 2);
13675 103 : tree arg3 = CALL_EXPR_ARG (exp, 3);
13676 103 : rtx op0 = expand_normal (arg0);
13677 103 : rtx op1 = expand_normal (arg1);
13678 103 : rtx op2 = expand_normal (arg2);
13679 103 : rtx op3 = expand_normal (arg3);
13680 103 : enum insn_code icode = d->icode;
13681 103 : const struct insn_data_d *insn_p = &insn_data[icode];
13682 103 : machine_mode mode0 = insn_p->operand[0].mode;
13683 103 : machine_mode mode1 = insn_p->operand[1].mode;
13684 :
13685 : /* See avxintrin.h for values. */
13686 103 : static const enum rtx_code comparisons[32] =
13687 : {
13688 : EQ, LT, LE, UNORDERED, NE, UNGE, UNGT, ORDERED,
13689 : UNEQ, UNLT, UNLE, UNORDERED, LTGT, GE, GT, ORDERED,
13690 : EQ, LT, LE, UNORDERED, NE, UNGE, UNGT, ORDERED,
13691 : UNEQ, UNLT, UNLE, UNORDERED, LTGT, GE, GT, ORDERED
13692 : };
13693 103 : static const bool ordereds[32] =
13694 : {
13695 : true, true, true, false, false, false, false, true,
13696 : false, false, false, true, true, true, true, false,
13697 : true, true, true, false, false, false, false, true,
13698 : false, false, false, true, true, true, true, false
13699 : };
13700 103 : static const bool non_signalings[32] =
13701 : {
13702 : true, false, false, true, true, false, false, true,
13703 : true, false, false, true, true, false, false, true,
13704 : false, true, true, false, false, true, true, false,
13705 : false, true, true, false, false, true, true, false
13706 : };
13707 :
13708 103 : if (!CONST_INT_P (op2))
13709 : {
13710 0 : error ("the third argument must be comparison constant");
13711 0 : return const0_rtx;
13712 : }
13713 103 : if (INTVAL (op2) < 0 || INTVAL (op2) >= 32)
13714 : {
13715 0 : error ("incorrect comparison mode");
13716 0 : return const0_rtx;
13717 : }
13718 :
13719 103 : if (!insn_p->operand[2].predicate (op3, SImode))
13720 : {
13721 4 : error ("incorrect rounding operand");
13722 4 : return const0_rtx;
13723 : }
13724 :
13725 99 : if (VECTOR_MODE_P (mode0))
13726 99 : op0 = safe_vector_operand (op0, mode0);
13727 99 : if (VECTOR_MODE_P (mode1))
13728 99 : op1 = safe_vector_operand (op1, mode1);
13729 :
13730 99 : enum rtx_code comparison = comparisons[INTVAL (op2)];
13731 99 : enum rtx_code orig_comp = comparison;
13732 99 : bool ordered = ordereds[INTVAL (op2)];
13733 99 : bool non_signaling = non_signalings[INTVAL (op2)];
13734 99 : rtx const_val = const0_rtx;
13735 :
13736 99 : bool check_unordered = false;
13737 99 : machine_mode mode = CCFPmode;
13738 99 : switch (comparison)
13739 : {
13740 8 : case ORDERED:
13741 8 : if (!ordered)
13742 : {
13743 4 : if (TARGET_AVX10_2 && comx_ok)
13744 : {
13745 : /* Unlike VCOMI{SH,SS,SD}, VCOMX{SH,SS,SD} will set SF
13746 : differently. So directly return true here. */
13747 0 : target = gen_reg_rtx (SImode);
13748 0 : emit_move_insn (target, const1_rtx);
13749 0 : return target;
13750 : }
13751 : else
13752 : {
13753 : /* NB: Use CCSmode/NE for _CMP_TRUE_UQ/_CMP_TRUE_US. */
13754 : if (!non_signaling)
13755 99 : ordered = true;
13756 99 : mode = CCSmode;
13757 : }
13758 : }
13759 : else
13760 : {
13761 : /* NB: Use CCPmode/NE for _CMP_ORD_Q/_CMP_ORD_S. */
13762 : if (non_signaling)
13763 : ordered = false;
13764 : mode = CCPmode;
13765 : }
13766 : comparison = NE;
13767 : break;
13768 8 : case UNORDERED:
13769 8 : if (ordered)
13770 : {
13771 4 : if (TARGET_AVX10_2 && comx_ok)
13772 : {
13773 : /* Unlike VCOMI{SH,SS,SD}, VCOMX{SH,SS,SD} will set SF
13774 : differently. So directly return false here. */
13775 0 : target = gen_reg_rtx (SImode);
13776 0 : emit_move_insn (target, const0_rtx);
13777 0 : return target;
13778 : }
13779 : else
13780 : {
13781 : /* NB: Use CCSmode/EQ for _CMP_FALSE_OQ/_CMP_FALSE_OS. */
13782 : if (non_signaling)
13783 99 : ordered = false;
13784 : mode = CCSmode;
13785 : }
13786 : }
13787 : else
13788 : {
13789 : /* NB: Use CCPmode/NE for _CMP_UNORD_Q/_CMP_UNORD_S. */
13790 : if (!non_signaling)
13791 99 : ordered = true;
13792 99 : mode = CCPmode;
13793 : }
13794 : comparison = EQ;
13795 : break;
13796 :
13797 40 : case LE: /* -> GE */
13798 40 : case LT: /* -> GT */
13799 40 : case UNGE: /* -> UNLE */
13800 40 : case UNGT: /* -> UNLT */
13801 40 : std::swap (op0, op1);
13802 40 : comparison = swap_condition (comparison);
13803 : /* FALLTHRU */
13804 68 : case GT:
13805 68 : case GE:
13806 68 : case UNEQ:
13807 68 : case UNLT:
13808 68 : case UNLE:
13809 68 : case LTGT:
13810 : /* These are supported by CCFPmode. NB: Use ordered/signaling
13811 : COMI or unordered/non-signaling UCOMI. Both set ZF, PF, CF
13812 : with NAN operands. */
13813 68 : if (ordered == non_signaling)
13814 : ordered = !ordered;
13815 : break;
13816 : /* NB: COMI/UCOMI will set ZF with NAN operands. Use CCZmode for
13817 : _CMP_EQ_OQ/_CMP_EQ_OS.
13818 : Under TARGET_AVX10_2, VCOMX/VUCOMX are always generated instead
13819 : of COMI/UCOMI, VCOMX/VUCOMX will not set ZF with NAN. */
13820 8 : case EQ:
13821 8 : if (!TARGET_AVX10_2 || !comx_ok)
13822 5 : check_unordered = true;
13823 : mode = CCZmode;
13824 : break;
13825 7 : case NE:
13826 : /* NB: COMI/UCOMI will set ZF with NAN operands. Use CCZmode for
13827 : _CMP_NEQ_UQ/_CMP_NEQ_US.
13828 : Under TARGET_AVX10_2, VCOMX/VUCOMX are always generated instead
13829 : of COMI/UCOMI, VCOMX/VUCOMX will not set ZF with NAN. */
13830 7 : gcc_assert (!ordered);
13831 7 : if (!TARGET_AVX10_2 || !comx_ok)
13832 4 : check_unordered = true;
13833 7 : mode = CCZmode;
13834 7 : const_val = const1_rtx;
13835 7 : break;
13836 0 : default:
13837 0 : gcc_unreachable ();
13838 : }
13839 :
13840 99 : target = gen_reg_rtx (SImode);
13841 99 : emit_move_insn (target, const_val);
13842 99 : target = gen_rtx_SUBREG (QImode, target, 0);
13843 :
13844 93 : if ((optimize && !register_operand (op0, mode0))
13845 192 : || !insn_p->operand[0].predicate (op0, mode0))
13846 6 : op0 = copy_to_mode_reg (mode0, op0);
13847 93 : if ((optimize && !register_operand (op1, mode1))
13848 192 : || !insn_p->operand[1].predicate (op1, mode1))
13849 6 : op1 = copy_to_mode_reg (mode1, op1);
13850 :
13851 : /* Generate comx instead of comi when EQ/NE to avoid NAN checks.
13852 : Use orig_comp to exclude ORDERED/UNORDERED cases. */
13853 99 : if ((orig_comp == EQ || orig_comp == NE)
13854 15 : && TARGET_AVX10_2 && comx_ok)
13855 : {
13856 6 : switch (icode)
13857 : {
13858 : case CODE_FOR_avx512fp16_comi_round:
13859 99 : icode = CODE_FOR_avx10_2_comxhf_round;
13860 : break;
13861 4 : case CODE_FOR_sse_comi_round:
13862 4 : icode = CODE_FOR_avx10_2_comxsf_round;
13863 4 : break;
13864 2 : case CODE_FOR_sse2_comi_round:
13865 2 : icode = CODE_FOR_avx10_2_comxdf_round;
13866 2 : break;
13867 :
13868 : default:
13869 : break;
13870 : }
13871 : }
13872 :
13873 : /* Generate comi instead of comx when UNEQ/LTGT to avoid NAN checks. */
13874 99 : if ((comparison == UNEQ || comparison == LTGT)
13875 8 : && TARGET_AVX10_2 && comx_ok)
13876 : {
13877 0 : switch (icode)
13878 : {
13879 : case CODE_FOR_avx10_2_comxhf_round:
13880 99 : icode = CODE_FOR_avx512fp16_comi_round;
13881 : break;
13882 0 : case CODE_FOR_avx10_2_comxsf_round:
13883 0 : icode = CODE_FOR_sse_comi_round;
13884 0 : break;
13885 0 : case CODE_FOR_avx10_2_comxdf_round:
13886 0 : icode = CODE_FOR_sse2_comi_round;
13887 0 : break;
13888 :
13889 : default:
13890 : break;
13891 : }
13892 : }
13893 :
13894 : /*
13895 : 1. COMI/VCOMX: ordered and signaling.
13896 : 2. UCOMI/VUCOMX: unordered and non-signaling.
13897 : */
13898 99 : if (non_signaling)
13899 38 : switch (icode)
13900 : {
13901 : case CODE_FOR_sse_comi_round:
13902 : icode = CODE_FOR_sse_ucomi_round;
13903 : break;
13904 17 : case CODE_FOR_sse2_comi_round:
13905 17 : icode = CODE_FOR_sse2_ucomi_round;
13906 17 : break;
13907 0 : case CODE_FOR_avx512fp16_comi_round:
13908 0 : icode = CODE_FOR_avx512fp16_ucomi_round;
13909 0 : break;
13910 3 : case CODE_FOR_avx10_2_comxsf_round:
13911 3 : icode = CODE_FOR_avx10_2_ucomxsf_round;
13912 3 : break;
13913 0 : case CODE_FOR_avx10_2_comxhf_round:
13914 0 : icode = CODE_FOR_avx10_2_ucomxhf_round;
13915 0 : break;
13916 1 : case CODE_FOR_avx10_2_comxdf_round:
13917 1 : icode = CODE_FOR_avx10_2_ucomxdf_round;
13918 1 : break;
13919 0 : default:
13920 0 : gcc_unreachable ();
13921 : }
13922 :
13923 99 : pat = GEN_FCN (icode) (op0, op1, op3);
13924 99 : if (! pat)
13925 : return 0;
13926 :
13927 : /* Rounding operand can be either NO_ROUND or ROUND_SAE at this point. */
13928 99 : if (INTVAL (op3) == NO_ROUND)
13929 : {
13930 1 : pat = ix86_erase_embedded_rounding (pat);
13931 1 : if (! pat)
13932 : return 0;
13933 :
13934 1 : set_dst = SET_DEST (pat);
13935 : }
13936 : else
13937 : {
13938 98 : gcc_assert (GET_CODE (pat) == SET);
13939 98 : set_dst = SET_DEST (pat);
13940 : }
13941 :
13942 99 : emit_insn (pat);
13943 :
13944 99 : return ix86_ssecom_setcc (comparison, check_unordered, mode,
13945 99 : set_dst, target);
13946 : }
13947 :
13948 : static rtx
13949 15538 : ix86_expand_round_builtin (const struct builtin_description *d,
13950 : tree exp, rtx target)
13951 : {
13952 15538 : rtx pat;
13953 15538 : unsigned int i, nargs;
13954 15538 : rtx xops[6];
13955 15538 : enum insn_code icode = d->icode;
13956 15538 : const struct insn_data_d *insn_p = &insn_data[icode];
13957 15538 : machine_mode tmode = insn_p->operand[0].mode;
13958 15538 : unsigned int nargs_constant = 0;
13959 15538 : unsigned int redundant_embed_rnd = 0;
13960 :
13961 15538 : switch ((enum ix86_builtin_func_type) d->flag)
13962 : {
13963 : case UINT64_FTYPE_V2DF_INT:
13964 : case UINT64_FTYPE_V4SF_INT:
13965 : case UINT64_FTYPE_V8HF_INT:
13966 : case UINT_FTYPE_V2DF_INT:
13967 : case UINT_FTYPE_V4SF_INT:
13968 : case UINT_FTYPE_V8HF_INT:
13969 : case INT64_FTYPE_V2DF_INT:
13970 : case INT64_FTYPE_V4SF_INT:
13971 : case INT64_FTYPE_V8HF_INT:
13972 : case INT_FTYPE_V2DF_INT:
13973 : case INT_FTYPE_V4SF_INT:
13974 : case INT_FTYPE_V8HF_INT:
13975 : nargs = 2;
13976 : break;
13977 634 : case V32HF_FTYPE_V32HF_V32HF_INT:
13978 634 : case V8HF_FTYPE_V8HF_V8HF_INT:
13979 634 : case V8HF_FTYPE_V8HF_INT_INT:
13980 634 : case V8HF_FTYPE_V8HF_UINT_INT:
13981 634 : case V8HF_FTYPE_V8HF_INT64_INT:
13982 634 : case V8HF_FTYPE_V8HF_UINT64_INT:
13983 634 : case V4SF_FTYPE_V4SF_UINT_INT:
13984 634 : case V4SF_FTYPE_V4SF_UINT64_INT:
13985 634 : case V2DF_FTYPE_V2DF_UINT64_INT:
13986 634 : case V4SF_FTYPE_V4SF_INT_INT:
13987 634 : case V4SF_FTYPE_V4SF_INT64_INT:
13988 634 : case V2DF_FTYPE_V2DF_INT64_INT:
13989 634 : case V4SF_FTYPE_V4SF_V4SF_INT:
13990 634 : case V2DF_FTYPE_V2DF_V2DF_INT:
13991 634 : case V4SF_FTYPE_V4SF_V2DF_INT:
13992 634 : case V2DF_FTYPE_V2DF_V4SF_INT:
13993 634 : nargs = 3;
13994 634 : break;
13995 4554 : case V8SF_FTYPE_V8DF_V8SF_QI_INT:
13996 4554 : case V8DF_FTYPE_V8DF_V8DF_QI_INT:
13997 4554 : case V32HI_FTYPE_V32HF_V32HI_USI_INT:
13998 4554 : case V32HI_FTYPE_V32BF_V32HI_USI_INT:
13999 4554 : case V8SI_FTYPE_V8DF_V8SI_QI_INT:
14000 4554 : case V8DI_FTYPE_V8HF_V8DI_UQI_INT:
14001 4554 : case V8DI_FTYPE_V8DF_V8DI_QI_INT:
14002 4554 : case V8SF_FTYPE_V8DI_V8SF_QI_INT:
14003 4554 : case V8DF_FTYPE_V8DI_V8DF_QI_INT:
14004 4554 : case V8DF_FTYPE_V8HF_V8DF_UQI_INT:
14005 4554 : case V16SF_FTYPE_V16HF_V16SF_UHI_INT:
14006 4554 : case V32HF_FTYPE_V32HI_V32HF_USI_INT:
14007 4554 : case V32HF_FTYPE_V32HF_V32HF_USI_INT:
14008 4554 : case V32HF_FTYPE_V32HF_V32HF_V32HF_INT:
14009 4554 : case V16SF_FTYPE_V16SF_V16SF_HI_INT:
14010 4554 : case V8DI_FTYPE_V8SF_V8DI_QI_INT:
14011 4554 : case V16SF_FTYPE_V16SI_V16SF_HI_INT:
14012 4554 : case V16SI_FTYPE_V16SF_V16SI_HI_INT:
14013 4554 : case V16SI_FTYPE_V16SF_V16SI_UHI_INT:
14014 4554 : case V16SI_FTYPE_V16HF_V16SI_UHI_INT:
14015 4554 : case V16HF_FTYPE_V16SI_V16HF_UHI_INT:
14016 4554 : case V8DF_FTYPE_V8SF_V8DF_QI_INT:
14017 4554 : case V16SF_FTYPE_V16HI_V16SF_HI_INT:
14018 4554 : case V2DF_FTYPE_V2DF_V2DF_V2DF_INT:
14019 4554 : case V4SF_FTYPE_V4SF_V4SF_V4SF_INT:
14020 4554 : case V8HF_FTYPE_V8DI_V8HF_UQI_INT:
14021 4554 : case V8HF_FTYPE_V8DF_V8HF_UQI_INT:
14022 4554 : case V16HF_FTYPE_V16SF_V16HF_UHI_INT:
14023 4554 : case V16HI_FTYPE_V16BF_V16HI_UHI_INT:
14024 4554 : case V8HF_FTYPE_V8HF_V8HF_V8HF_INT:
14025 4554 : nargs = 4;
14026 4554 : break;
14027 163 : case V4SF_FTYPE_V4SF_V4SF_INT_INT:
14028 163 : case V2DF_FTYPE_V2DF_V2DF_INT_INT:
14029 163 : nargs_constant = 2;
14030 163 : nargs = 4;
14031 163 : break;
14032 103 : case INT_FTYPE_V4SF_V4SF_INT_INT:
14033 103 : case INT_FTYPE_V2DF_V2DF_INT_INT:
14034 103 : return ix86_expand_sse_comi_round (d, exp, target, true);
14035 6216 : case V8DF_FTYPE_V8DF_V8DF_V8DF_UQI_INT:
14036 6216 : case V2DF_FTYPE_V2DF_V2DF_V2DF_UQI_INT:
14037 6216 : case V4SF_FTYPE_V4SF_V4SF_V4SF_UQI_INT:
14038 6216 : case V4SF_FTYPE_V8HF_V4SF_V4SF_UQI_INT:
14039 6216 : case V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT:
14040 6216 : case V32HF_FTYPE_V32HF_V32HF_V32HF_UHI_INT:
14041 6216 : case V32HF_FTYPE_V32HF_V32HF_V32HF_USI_INT:
14042 6216 : case V2DF_FTYPE_V8HF_V2DF_V2DF_UQI_INT:
14043 6216 : case V2DF_FTYPE_V2DF_V2DF_V2DF_QI_INT:
14044 6216 : case V2DF_FTYPE_V2DF_V4SF_V2DF_QI_INT:
14045 6216 : case V2DF_FTYPE_V2DF_V4SF_V2DF_UQI_INT:
14046 6216 : case V4SF_FTYPE_V4SF_V4SF_V4SF_QI_INT:
14047 6216 : case V4SF_FTYPE_V4SF_V2DF_V4SF_QI_INT:
14048 6216 : case V4SF_FTYPE_V4SF_V2DF_V4SF_UQI_INT:
14049 6216 : case V8HF_FTYPE_V8HF_V8HF_V8HF_UQI_INT:
14050 6216 : case V8HF_FTYPE_V2DF_V8HF_V8HF_UQI_INT:
14051 6216 : case V8HF_FTYPE_V4SF_V8HF_V8HF_UQI_INT:
14052 6216 : case V32HF_FTYPE_V16SF_V16SF_V32HF_USI_INT:
14053 6216 : nargs = 5;
14054 6216 : break;
14055 635 : case V32HF_FTYPE_V32HF_INT_V32HF_USI_INT:
14056 635 : case V16SF_FTYPE_V16SF_INT_V16SF_HI_INT:
14057 635 : case V8DF_FTYPE_V8DF_INT_V8DF_QI_INT:
14058 635 : case V8DF_FTYPE_V8DF_INT_V8DF_UQI_INT:
14059 635 : case V16SF_FTYPE_V16SF_INT_V16SF_UHI_INT:
14060 635 : nargs_constant = 4;
14061 635 : nargs = 5;
14062 635 : break;
14063 1181 : case UQI_FTYPE_V8DF_V8DF_INT_UQI_INT:
14064 1181 : case UQI_FTYPE_V2DF_V2DF_INT_UQI_INT:
14065 1181 : case UHI_FTYPE_V16SF_V16SF_INT_UHI_INT:
14066 1181 : case UQI_FTYPE_V4SF_V4SF_INT_UQI_INT:
14067 1181 : case USI_FTYPE_V32HF_V32HF_INT_USI_INT:
14068 1181 : case UQI_FTYPE_V8HF_V8HF_INT_UQI_INT:
14069 1181 : nargs_constant = 3;
14070 1181 : nargs = 5;
14071 1181 : break;
14072 1071 : case V16SF_FTYPE_V16SF_V16SF_INT_V16SF_HI_INT:
14073 1071 : case V8DF_FTYPE_V8DF_V8DF_INT_V8DF_QI_INT:
14074 1071 : case V4SF_FTYPE_V4SF_V4SF_INT_V4SF_QI_INT:
14075 1071 : case V2DF_FTYPE_V2DF_V2DF_INT_V2DF_QI_INT:
14076 1071 : case V2DF_FTYPE_V2DF_V2DF_INT_V2DF_UQI_INT:
14077 1071 : case V4SF_FTYPE_V4SF_V4SF_INT_V4SF_UQI_INT:
14078 1071 : case V8HF_FTYPE_V8HF_V8HF_INT_V8HF_UQI_INT:
14079 1071 : case V8DF_FTYPE_V8DF_V8DF_INT_V8DF_UQI_INT:
14080 1071 : case V32HF_FTYPE_V32HF_V32HF_INT_V32HF_USI_INT:
14081 1071 : case V16SF_FTYPE_V16SF_V16SF_INT_V16SF_UHI_INT:
14082 1071 : nargs = 6;
14083 1071 : nargs_constant = 4;
14084 1071 : break;
14085 252 : case V8DF_FTYPE_V8DF_V8DF_V8DI_INT_QI_INT:
14086 252 : case V16SF_FTYPE_V16SF_V16SF_V16SI_INT_HI_INT:
14087 252 : case V2DF_FTYPE_V2DF_V2DF_V2DI_INT_QI_INT:
14088 252 : case V4SF_FTYPE_V4SF_V4SF_V4SI_INT_QI_INT:
14089 252 : nargs = 6;
14090 252 : nargs_constant = 3;
14091 252 : break;
14092 0 : default:
14093 0 : gcc_unreachable ();
14094 : }
14095 14706 : gcc_assert (nargs <= ARRAY_SIZE (xops));
14096 :
14097 15435 : if (optimize
14098 4265 : || target == 0
14099 4265 : || GET_MODE (target) != tmode
14100 19700 : || !insn_p->operand[0].predicate (target, tmode))
14101 11170 : target = gen_reg_rtx (tmode);
14102 :
14103 85110 : for (i = 0; i < nargs; i++)
14104 : {
14105 70230 : tree arg = CALL_EXPR_ARG (exp, i);
14106 70230 : rtx op = ix86_expand_unsigned_small_int_cst_argument (arg);
14107 70230 : machine_mode mode = insn_p->operand[i + 1].mode;
14108 70230 : bool match = insn_p->operand[i + 1].predicate (op, mode);
14109 :
14110 70230 : if (i == nargs - nargs_constant)
14111 : {
14112 3302 : if (!match)
14113 : {
14114 40 : switch (icode)
14115 : {
14116 12 : case CODE_FOR_avx512f_getmantv8df_mask_round:
14117 12 : case CODE_FOR_avx512f_getmantv16sf_mask_round:
14118 12 : case CODE_FOR_avx512bw_getmantv32hf_mask_round:
14119 12 : case CODE_FOR_avx512f_vgetmantv2df_round:
14120 12 : case CODE_FOR_avx512f_vgetmantv2df_mask_round:
14121 12 : case CODE_FOR_avx512f_vgetmantv4sf_round:
14122 12 : case CODE_FOR_avx512f_vgetmantv4sf_mask_round:
14123 12 : case CODE_FOR_avx512f_vgetmantv8hf_mask_round:
14124 12 : error ("the immediate argument must be a 4-bit immediate");
14125 12 : return const0_rtx;
14126 8 : case CODE_FOR_avx512f_cmpv8df3_mask_round:
14127 8 : case CODE_FOR_avx512f_cmpv16sf3_mask_round:
14128 8 : case CODE_FOR_avx512f_vmcmpv2df3_mask_round:
14129 8 : case CODE_FOR_avx512f_vmcmpv4sf3_mask_round:
14130 8 : case CODE_FOR_avx512f_vmcmpv8hf3_mask_round:
14131 8 : case CODE_FOR_avx512bw_cmpv32hf3_mask_round:
14132 8 : error ("the immediate argument must be a 5-bit immediate");
14133 8 : return const0_rtx;
14134 20 : default:
14135 20 : error ("the immediate argument must be an 8-bit immediate");
14136 20 : return const0_rtx;
14137 : }
14138 : }
14139 : }
14140 66928 : else if (i == nargs-1)
14141 : {
14142 15395 : if (!insn_p->operand[nargs].predicate (op, SImode))
14143 : {
14144 515 : error ("incorrect rounding operand");
14145 515 : return const0_rtx;
14146 : }
14147 :
14148 : /* If there is no rounding use normal version of the pattern. */
14149 14880 : if (INTVAL (op) == NO_ROUND)
14150 : {
14151 : /* Skip erasing embedded rounding for below expanders who
14152 : generates multiple insns. In ix86_erase_embedded_rounding
14153 : the pattern will be transformed to a single set, and emit_insn
14154 : appends the set instead of insert it to chain. So the insns
14155 : emitted inside define_expander would be ignored. */
14156 4958 : switch (icode)
14157 : {
14158 : case CODE_FOR_avx512bw_fmaddc_v32hf_mask1_round:
14159 : case CODE_FOR_avx512bw_fcmaddc_v32hf_mask1_round:
14160 : case CODE_FOR_avx512fp16_fmaddcsh_v8hf_mask1_round:
14161 : case CODE_FOR_avx512fp16_fcmaddcsh_v8hf_mask1_round:
14162 : case CODE_FOR_avx512fp16_fmaddcsh_v8hf_mask3_round:
14163 : case CODE_FOR_avx512fp16_fcmaddcsh_v8hf_mask3_round:
14164 : redundant_embed_rnd = 0;
14165 : break;
14166 4926 : default:
14167 4926 : redundant_embed_rnd = 1;
14168 4926 : break;
14169 : }
14170 : }
14171 : }
14172 : else
14173 : {
14174 51533 : if (VECTOR_MODE_P (mode))
14175 37633 : op = safe_vector_operand (op, mode);
14176 :
14177 51533 : op = fixup_modeless_constant (op, mode);
14178 :
14179 51533 : if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
14180 : {
14181 51533 : if (optimize || !match)
14182 45205 : op = copy_to_mode_reg (mode, op);
14183 : }
14184 : else
14185 : {
14186 0 : op = copy_to_reg (op);
14187 0 : op = lowpart_subreg (mode, op, GET_MODE (op));
14188 : }
14189 : }
14190 :
14191 69675 : xops[i] = op;
14192 : }
14193 :
14194 14880 : switch (nargs)
14195 : {
14196 : case 1:
14197 : pat = GEN_FCN (icode) (target, xops[0]);
14198 : break;
14199 696 : case 2:
14200 696 : pat = GEN_FCN (icode) (target, xops[0], xops[1]);
14201 696 : break;
14202 590 : case 3:
14203 590 : pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2]);
14204 590 : break;
14205 4593 : case 4:
14206 4593 : pat = GEN_FCN (icode) (target, xops[0], xops[1],
14207 4593 : xops[2], xops[3]);
14208 4593 : break;
14209 7728 : case 5:
14210 7728 : pat = GEN_FCN (icode) (target, xops[0], xops[1],
14211 7728 : xops[2], xops[3], xops[4]);
14212 7728 : break;
14213 1273 : case 6:
14214 1273 : pat = GEN_FCN (icode) (target, xops[0], xops[1],
14215 1273 : xops[2], xops[3], xops[4], xops[5]);
14216 1273 : break;
14217 : default:
14218 : gcc_unreachable ();
14219 : }
14220 :
14221 14880 : if (!pat)
14222 : return 0;
14223 :
14224 14880 : if (redundant_embed_rnd)
14225 4926 : pat = ix86_erase_embedded_rounding (pat);
14226 :
14227 14880 : emit_insn (pat);
14228 14880 : return target;
14229 : }
14230 :
14231 : /* Subroutine of ix86_expand_builtin to take care of special insns
14232 : with variable number of operands. */
14233 :
14234 : static rtx
14235 27102 : ix86_expand_special_args_builtin (const struct builtin_description *d,
14236 : tree exp, rtx target)
14237 : {
14238 27102 : tree arg;
14239 27102 : rtx pat, op;
14240 27102 : unsigned int i, nargs, arg_adjust, memory;
14241 27102 : unsigned int constant = 100;
14242 27102 : bool aligned_mem = false;
14243 27102 : rtx xops[4];
14244 27102 : enum insn_code icode = d->icode;
14245 27102 : const struct insn_data_d *insn_p = &insn_data[icode];
14246 27102 : machine_mode tmode = insn_p->operand[0].mode;
14247 27102 : enum { load, store } klass;
14248 :
14249 27102 : switch ((enum ix86_builtin_func_type) d->flag)
14250 : {
14251 15360 : case VOID_FTYPE_VOID:
14252 15360 : emit_insn (GEN_FCN (icode) (target));
14253 15360 : return 0;
14254 : case VOID_FTYPE_UINT64:
14255 : case VOID_FTYPE_UNSIGNED:
14256 : nargs = 0;
14257 : klass = store;
14258 : memory = 0;
14259 : break;
14260 :
14261 7508 : case INT_FTYPE_VOID:
14262 7508 : case USHORT_FTYPE_VOID:
14263 7508 : case UINT64_FTYPE_VOID:
14264 7508 : case UINT_FTYPE_VOID:
14265 7508 : case UINT8_FTYPE_VOID:
14266 7508 : case UNSIGNED_FTYPE_VOID:
14267 7508 : nargs = 0;
14268 7508 : klass = load;
14269 7508 : memory = 0;
14270 7508 : break;
14271 360 : case CHAR_FTYPE_PCCHAR:
14272 360 : case SHORT_FTYPE_PCSHORT:
14273 360 : case INT_FTYPE_PCINT:
14274 360 : case INT64_FTYPE_PCINT64:
14275 360 : case UINT64_FTYPE_PUNSIGNED:
14276 360 : case V2DI_FTYPE_PV2DI:
14277 360 : case V4DI_FTYPE_PV4DI:
14278 360 : case V32QI_FTYPE_PCCHAR:
14279 360 : case V16QI_FTYPE_PCCHAR:
14280 360 : case V8SF_FTYPE_PCV4SF:
14281 360 : case V8SF_FTYPE_PCFLOAT:
14282 360 : case V4SF_FTYPE_PCFLOAT:
14283 360 : case V4SF_FTYPE_PCFLOAT16:
14284 360 : case V4SF_FTYPE_PCBFLOAT16:
14285 360 : case V4SF_FTYPE_PCV8BF:
14286 360 : case V4SF_FTYPE_PCV8HF:
14287 360 : case V8SF_FTYPE_PCFLOAT16:
14288 360 : case V8SF_FTYPE_PCBFLOAT16:
14289 360 : case V8SF_FTYPE_PCV16HF:
14290 360 : case V8SF_FTYPE_PCV16BF:
14291 360 : case V4DF_FTYPE_PCV2DF:
14292 360 : case V4DF_FTYPE_PCDOUBLE:
14293 360 : case V2DF_FTYPE_PCDOUBLE:
14294 360 : case VOID_FTYPE_PVOID:
14295 360 : case V8DI_FTYPE_PV8DI:
14296 360 : nargs = 1;
14297 360 : klass = load;
14298 360 : memory = 0;
14299 360 : switch (icode)
14300 : {
14301 : case CODE_FOR_sse4_1_movntdqa:
14302 : case CODE_FOR_avx2_movntdqa:
14303 : case CODE_FOR_avx512f_movntdqa:
14304 : aligned_mem = true;
14305 : break;
14306 : default:
14307 : break;
14308 : }
14309 : break;
14310 372 : case VOID_FTYPE_PV2SF_V4SF:
14311 372 : case VOID_FTYPE_PV8DI_V8DI:
14312 372 : case VOID_FTYPE_PV4DI_V4DI:
14313 372 : case VOID_FTYPE_PV2DI_V2DI:
14314 372 : case VOID_FTYPE_PCHAR_V32QI:
14315 372 : case VOID_FTYPE_PCHAR_V16QI:
14316 372 : case VOID_FTYPE_PFLOAT_V16SF:
14317 372 : case VOID_FTYPE_PFLOAT_V8SF:
14318 372 : case VOID_FTYPE_PFLOAT_V4SF:
14319 372 : case VOID_FTYPE_PDOUBLE_V8DF:
14320 372 : case VOID_FTYPE_PDOUBLE_V4DF:
14321 372 : case VOID_FTYPE_PDOUBLE_V2DF:
14322 372 : case VOID_FTYPE_PLONGLONG_LONGLONG:
14323 372 : case VOID_FTYPE_PULONGLONG_ULONGLONG:
14324 372 : case VOID_FTYPE_PUNSIGNED_UNSIGNED:
14325 372 : case VOID_FTYPE_PINT_INT:
14326 372 : nargs = 1;
14327 372 : klass = store;
14328 : /* Reserve memory operand for target. */
14329 372 : memory = ARRAY_SIZE (xops);
14330 372 : switch (icode)
14331 : {
14332 : /* These builtins and instructions require the memory
14333 : to be properly aligned. */
14334 : case CODE_FOR_avx_movntv4di:
14335 : case CODE_FOR_sse2_movntv2di:
14336 : case CODE_FOR_avx_movntv8sf:
14337 : case CODE_FOR_sse_movntv4sf:
14338 : case CODE_FOR_sse4a_vmmovntv4sf:
14339 : case CODE_FOR_avx_movntv4df:
14340 : case CODE_FOR_sse2_movntv2df:
14341 : case CODE_FOR_sse4a_vmmovntv2df:
14342 : case CODE_FOR_sse2_movntidi:
14343 : case CODE_FOR_sse_movntq:
14344 : case CODE_FOR_sse2_movntisi:
14345 : case CODE_FOR_avx512f_movntv16sf:
14346 : case CODE_FOR_avx512f_movntv8df:
14347 : case CODE_FOR_avx512f_movntv8di:
14348 : aligned_mem = true;
14349 : break;
14350 : default:
14351 : break;
14352 : }
14353 : break;
14354 0 : case VOID_FTYPE_PVOID_PCVOID:
14355 0 : nargs = 1;
14356 0 : klass = store;
14357 0 : memory = 0;
14358 :
14359 0 : break;
14360 26 : case V4SF_FTYPE_V4SF_PCV2SF:
14361 26 : case V2DF_FTYPE_V2DF_PCDOUBLE:
14362 26 : nargs = 2;
14363 26 : klass = load;
14364 26 : memory = 1;
14365 26 : break;
14366 93 : case V8SF_FTYPE_PCV8SF_V8SI:
14367 93 : case V4DF_FTYPE_PCV4DF_V4DI:
14368 93 : case V4SF_FTYPE_PCV4SF_V4SI:
14369 93 : case V2DF_FTYPE_PCV2DF_V2DI:
14370 93 : case V8SI_FTYPE_PCV8SI_V8SI:
14371 93 : case V4DI_FTYPE_PCV4DI_V4DI:
14372 93 : case V4SI_FTYPE_PCV4SI_V4SI:
14373 93 : case V2DI_FTYPE_PCV2DI_V2DI:
14374 93 : case VOID_FTYPE_INT_INT64:
14375 93 : nargs = 2;
14376 93 : klass = load;
14377 93 : memory = 0;
14378 93 : break;
14379 360 : case VOID_FTYPE_PV8DF_V8DF_UQI:
14380 360 : case VOID_FTYPE_PV4DF_V4DF_UQI:
14381 360 : case VOID_FTYPE_PV2DF_V2DF_UQI:
14382 360 : case VOID_FTYPE_PV16SF_V16SF_UHI:
14383 360 : case VOID_FTYPE_PV8SF_V8SF_UQI:
14384 360 : case VOID_FTYPE_PV4SF_V4SF_UQI:
14385 360 : case VOID_FTYPE_PV8DI_V8DI_UQI:
14386 360 : case VOID_FTYPE_PV4DI_V4DI_UQI:
14387 360 : case VOID_FTYPE_PV2DI_V2DI_UQI:
14388 360 : case VOID_FTYPE_PV16SI_V16SI_UHI:
14389 360 : case VOID_FTYPE_PV8SI_V8SI_UQI:
14390 360 : case VOID_FTYPE_PV4SI_V4SI_UQI:
14391 360 : case VOID_FTYPE_PV64QI_V64QI_UDI:
14392 360 : case VOID_FTYPE_PV32HI_V32HI_USI:
14393 360 : case VOID_FTYPE_PV32QI_V32QI_USI:
14394 360 : case VOID_FTYPE_PV16QI_V16QI_UHI:
14395 360 : case VOID_FTYPE_PV16HI_V16HI_UHI:
14396 360 : case VOID_FTYPE_PV8HI_V8HI_UQI:
14397 360 : switch (icode)
14398 : {
14399 : /* These builtins and instructions require the memory
14400 : to be properly aligned. */
14401 : case CODE_FOR_avx512f_storev16sf_mask:
14402 : case CODE_FOR_avx512f_storev16si_mask:
14403 : case CODE_FOR_avx512f_storev8df_mask:
14404 : case CODE_FOR_avx512f_storev8di_mask:
14405 : case CODE_FOR_avx512vl_storev8sf_mask:
14406 : case CODE_FOR_avx512vl_storev8si_mask:
14407 : case CODE_FOR_avx512vl_storev4df_mask:
14408 : case CODE_FOR_avx512vl_storev4di_mask:
14409 : case CODE_FOR_avx512vl_storev4sf_mask:
14410 : case CODE_FOR_avx512vl_storev4si_mask:
14411 : case CODE_FOR_avx512vl_storev2df_mask:
14412 : case CODE_FOR_avx512vl_storev2di_mask:
14413 11742 : aligned_mem = true;
14414 : break;
14415 : default:
14416 : break;
14417 : }
14418 : /* FALLTHRU */
14419 : case VOID_FTYPE_PV8SF_V8SI_V8SF:
14420 : case VOID_FTYPE_PV4DF_V4DI_V4DF:
14421 : case VOID_FTYPE_PV4SF_V4SI_V4SF:
14422 : case VOID_FTYPE_PV2DF_V2DI_V2DF:
14423 : case VOID_FTYPE_PV8SI_V8SI_V8SI:
14424 : case VOID_FTYPE_PV4DI_V4DI_V4DI:
14425 : case VOID_FTYPE_PV4SI_V4SI_V4SI:
14426 : case VOID_FTYPE_PV2DI_V2DI_V2DI:
14427 : case VOID_FTYPE_PV8SI_V8DI_UQI:
14428 : case VOID_FTYPE_PV8HI_V8DI_UQI:
14429 : case VOID_FTYPE_PV16HI_V16SI_UHI:
14430 : case VOID_FTYPE_PUDI_V8DI_UQI:
14431 : case VOID_FTYPE_PV16QI_V16SI_UHI:
14432 : case VOID_FTYPE_PV4SI_V4DI_UQI:
14433 : case VOID_FTYPE_PUDI_V2DI_UQI:
14434 : case VOID_FTYPE_PUDI_V4DI_UQI:
14435 : case VOID_FTYPE_PUSI_V2DI_UQI:
14436 : case VOID_FTYPE_PV8HI_V8SI_UQI:
14437 : case VOID_FTYPE_PUDI_V4SI_UQI:
14438 : case VOID_FTYPE_PUSI_V4DI_UQI:
14439 : case VOID_FTYPE_PUHI_V2DI_UQI:
14440 : case VOID_FTYPE_PUDI_V8SI_UQI:
14441 : case VOID_FTYPE_PUSI_V4SI_UQI:
14442 : case VOID_FTYPE_PCHAR_V64QI_UDI:
14443 : case VOID_FTYPE_PCHAR_V32QI_USI:
14444 : case VOID_FTYPE_PCHAR_V16QI_UHI:
14445 : case VOID_FTYPE_PSHORT_V32HI_USI:
14446 : case VOID_FTYPE_PSHORT_V16HI_UHI:
14447 : case VOID_FTYPE_PSHORT_V8HI_UQI:
14448 : case VOID_FTYPE_PINT_V16SI_UHI:
14449 : case VOID_FTYPE_PINT_V8SI_UQI:
14450 : case VOID_FTYPE_PINT_V4SI_UQI:
14451 : case VOID_FTYPE_PINT64_V8DI_UQI:
14452 : case VOID_FTYPE_PINT64_V4DI_UQI:
14453 : case VOID_FTYPE_PINT64_V2DI_UQI:
14454 : case VOID_FTYPE_PDOUBLE_V8DF_UQI:
14455 : case VOID_FTYPE_PDOUBLE_V4DF_UQI:
14456 : case VOID_FTYPE_PDOUBLE_V2DF_UQI:
14457 : case VOID_FTYPE_PFLOAT_V16SF_UHI:
14458 : case VOID_FTYPE_PFLOAT_V8SF_UQI:
14459 : case VOID_FTYPE_PFLOAT_V4SF_UQI:
14460 : case VOID_FTYPE_PCFLOAT16_V8HF_UQI:
14461 : case VOID_FTYPE_PV32QI_V32HI_USI:
14462 : case VOID_FTYPE_PV16QI_V16HI_UHI:
14463 : case VOID_FTYPE_PUDI_V8HI_UQI:
14464 : nargs = 2;
14465 : klass = store;
14466 : /* Reserve memory operand for target. */
14467 : memory = ARRAY_SIZE (xops);
14468 : break;
14469 1243 : case V4SF_FTYPE_PCV4SF_V4SF_UQI:
14470 1243 : case V8SF_FTYPE_PCV8SF_V8SF_UQI:
14471 1243 : case V16SF_FTYPE_PCV16SF_V16SF_UHI:
14472 1243 : case V4SI_FTYPE_PCV4SI_V4SI_UQI:
14473 1243 : case V8SI_FTYPE_PCV8SI_V8SI_UQI:
14474 1243 : case V16SI_FTYPE_PCV16SI_V16SI_UHI:
14475 1243 : case V2DF_FTYPE_PCV2DF_V2DF_UQI:
14476 1243 : case V4DF_FTYPE_PCV4DF_V4DF_UQI:
14477 1243 : case V8DF_FTYPE_PCV8DF_V8DF_UQI:
14478 1243 : case V2DI_FTYPE_PCV2DI_V2DI_UQI:
14479 1243 : case V4DI_FTYPE_PCV4DI_V4DI_UQI:
14480 1243 : case V8DI_FTYPE_PCV8DI_V8DI_UQI:
14481 1243 : case V64QI_FTYPE_PCV64QI_V64QI_UDI:
14482 1243 : case V32HI_FTYPE_PCV32HI_V32HI_USI:
14483 1243 : case V32QI_FTYPE_PCV32QI_V32QI_USI:
14484 1243 : case V16QI_FTYPE_PCV16QI_V16QI_UHI:
14485 1243 : case V16HI_FTYPE_PCV16HI_V16HI_UHI:
14486 1243 : case V8HI_FTYPE_PCV8HI_V8HI_UQI:
14487 1243 : switch (icode)
14488 : {
14489 : /* These builtins and instructions require the memory
14490 : to be properly aligned. */
14491 : case CODE_FOR_avx512f_loadv16sf_mask:
14492 : case CODE_FOR_avx512f_loadv16si_mask:
14493 : case CODE_FOR_avx512f_loadv8df_mask:
14494 : case CODE_FOR_avx512f_loadv8di_mask:
14495 : case CODE_FOR_avx512vl_loadv8sf_mask:
14496 : case CODE_FOR_avx512vl_loadv8si_mask:
14497 : case CODE_FOR_avx512vl_loadv4df_mask:
14498 : case CODE_FOR_avx512vl_loadv4di_mask:
14499 : case CODE_FOR_avx512vl_loadv4sf_mask:
14500 : case CODE_FOR_avx512vl_loadv4si_mask:
14501 : case CODE_FOR_avx512vl_loadv2df_mask:
14502 : case CODE_FOR_avx512vl_loadv2di_mask:
14503 : case CODE_FOR_avx512bw_loadv64qi_mask:
14504 : case CODE_FOR_avx512vl_loadv32qi_mask:
14505 : case CODE_FOR_avx512vl_loadv16qi_mask:
14506 : case CODE_FOR_avx512bw_loadv32hi_mask:
14507 : case CODE_FOR_avx512vl_loadv16hi_mask:
14508 : case CODE_FOR_avx512vl_loadv8hi_mask:
14509 11742 : aligned_mem = true;
14510 : break;
14511 : default:
14512 : break;
14513 : }
14514 : /* FALLTHRU */
14515 : case V64QI_FTYPE_PCCHAR_V64QI_UDI:
14516 : case V32QI_FTYPE_PCCHAR_V32QI_USI:
14517 : case V16QI_FTYPE_PCCHAR_V16QI_UHI:
14518 : case V32HI_FTYPE_PCSHORT_V32HI_USI:
14519 : case V16HI_FTYPE_PCSHORT_V16HI_UHI:
14520 : case V8HI_FTYPE_PCSHORT_V8HI_UQI:
14521 : case V16SI_FTYPE_PCINT_V16SI_UHI:
14522 : case V8SI_FTYPE_PCINT_V8SI_UQI:
14523 : case V4SI_FTYPE_PCINT_V4SI_UQI:
14524 : case V8DI_FTYPE_PCINT64_V8DI_UQI:
14525 : case V4DI_FTYPE_PCINT64_V4DI_UQI:
14526 : case V2DI_FTYPE_PCINT64_V2DI_UQI:
14527 : case V8DF_FTYPE_PCDOUBLE_V8DF_UQI:
14528 : case V4DF_FTYPE_PCDOUBLE_V4DF_UQI:
14529 : case V2DF_FTYPE_PCDOUBLE_V2DF_UQI:
14530 : case V16SF_FTYPE_PCFLOAT_V16SF_UHI:
14531 : case V8SF_FTYPE_PCFLOAT_V8SF_UQI:
14532 : case V4SF_FTYPE_PCFLOAT_V4SF_UQI:
14533 : case V8HF_FTYPE_PCFLOAT16_V8HF_UQI:
14534 : nargs = 3;
14535 : klass = load;
14536 : memory = 0;
14537 : break;
14538 105 : case INT_FTYPE_PINT_INT_INT_INT:
14539 105 : case LONGLONG_FTYPE_PLONGLONG_LONGLONG_LONGLONG_INT:
14540 105 : nargs = 4;
14541 105 : klass = load;
14542 105 : memory = 0;
14543 105 : constant = 3;
14544 105 : break;
14545 0 : default:
14546 0 : gcc_unreachable ();
14547 : }
14548 :
14549 8268 : gcc_assert (nargs <= ARRAY_SIZE (xops));
14550 :
14551 11742 : if (klass == store)
14552 : {
14553 1879 : arg = CALL_EXPR_ARG (exp, 0);
14554 1879 : op = expand_normal (arg);
14555 1879 : gcc_assert (target == 0);
14556 1879 : if (memory)
14557 : {
14558 1716 : op = ix86_zero_extend_to_Pmode (op);
14559 1716 : target = gen_rtx_MEM (tmode, op);
14560 : /* target at this point has just BITS_PER_UNIT MEM_ALIGN
14561 : on it. Try to improve it using get_pointer_alignment,
14562 : and if the special builtin is one that requires strict
14563 : mode alignment, also from it's GET_MODE_ALIGNMENT.
14564 : Failure to do so could lead to ix86_legitimate_combined_insn
14565 : rejecting all changes to such insns. */
14566 1716 : unsigned int align = get_pointer_alignment (arg);
14567 1716 : if (aligned_mem && align < GET_MODE_ALIGNMENT (tmode))
14568 275 : align = GET_MODE_ALIGNMENT (tmode);
14569 3432 : if (MEM_ALIGN (target) < align)
14570 422 : set_mem_align (target, align);
14571 : }
14572 : else
14573 163 : target = force_reg (tmode, op);
14574 : arg_adjust = 1;
14575 : }
14576 : else
14577 : {
14578 9863 : arg_adjust = 0;
14579 9863 : if (optimize
14580 2919 : || target == 0
14581 2919 : || !register_operand (target, tmode)
14582 12771 : || GET_MODE (target) != tmode)
14583 6955 : target = gen_reg_rtx (tmode);
14584 : }
14585 :
14586 21133 : for (i = 0; i < nargs; i++)
14587 : {
14588 9391 : machine_mode mode = insn_p->operand[i + 1].mode;
14589 :
14590 9391 : arg = CALL_EXPR_ARG (exp, i + arg_adjust);
14591 9391 : op = ix86_expand_unsigned_small_int_cst_argument (arg);
14592 :
14593 9391 : if (i == memory)
14594 : {
14595 : /* This must be the memory operand. */
14596 2355 : op = ix86_zero_extend_to_Pmode (op);
14597 2355 : op = gen_rtx_MEM (mode, op);
14598 : /* op at this point has just BITS_PER_UNIT MEM_ALIGN
14599 : on it. Try to improve it using get_pointer_alignment,
14600 : and if the special builtin is one that requires strict
14601 : mode alignment, also from it's GET_MODE_ALIGNMENT.
14602 : Failure to do so could lead to ix86_legitimate_combined_insn
14603 : rejecting all changes to such insns. */
14604 2355 : unsigned int align = get_pointer_alignment (arg);
14605 2355 : if (aligned_mem && align < GET_MODE_ALIGNMENT (mode))
14606 299 : align = GET_MODE_ALIGNMENT (mode);
14607 4710 : if (MEM_ALIGN (op) < align)
14608 523 : set_mem_align (op, align);
14609 : }
14610 7036 : else if (i == constant)
14611 : {
14612 : /* This must be the constant. */
14613 105 : if (!insn_p->operand[nargs].predicate(op, SImode))
14614 : {
14615 0 : error ("the fourth argument must be one of enum %qs", "_CMPCCX_ENUM");
14616 0 : return const0_rtx;
14617 : }
14618 : }
14619 : else
14620 : {
14621 : /* This must be register. */
14622 6931 : if (VECTOR_MODE_P (mode))
14623 3475 : op = safe_vector_operand (op, mode);
14624 :
14625 6931 : op = fixup_modeless_constant (op, mode);
14626 :
14627 : /* NB: 3-operands load implied it's a mask load or v{p}expand*,
14628 : and that mask operand should be at the end.
14629 : Keep all-ones mask which would be simplified by the expander. */
14630 1771 : if (nargs == 3 && i == 2 && klass == load
14631 1771 : && constm1_operand (op, mode)
14632 7104 : && insn_p->operand[i].predicate (op, mode))
14633 : ;
14634 6931 : else if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
14635 6931 : op = copy_to_mode_reg (mode, op);
14636 : else
14637 : {
14638 0 : op = copy_to_reg (op);
14639 0 : op = lowpart_subreg (mode, op, GET_MODE (op));
14640 : }
14641 : }
14642 :
14643 9391 : xops[i]= op;
14644 : }
14645 :
14646 11742 : switch (nargs)
14647 : {
14648 7671 : case 0:
14649 7671 : pat = GEN_FCN (icode) (target);
14650 7671 : break;
14651 732 : case 1:
14652 732 : pat = GEN_FCN (icode) (target, xops[0]);
14653 732 : break;
14654 1463 : case 2:
14655 1463 : pat = GEN_FCN (icode) (target, xops[0], xops[1]);
14656 1463 : break;
14657 1771 : case 3:
14658 1771 : pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2]);
14659 1771 : break;
14660 105 : case 4:
14661 105 : pat = GEN_FCN (icode) (target, xops[0], xops[1], xops[2], xops[3]);
14662 105 : break;
14663 : default:
14664 : gcc_unreachable ();
14665 : }
14666 :
14667 11742 : if (! pat)
14668 : return 0;
14669 :
14670 11742 : emit_insn (pat);
14671 11742 : return klass == store ? 0 : target;
14672 : }
14673 :
14674 : /* Return the integer constant in ARG. Constrain it to be in the range
14675 : of the subparts of VEC_TYPE; issue an error if not. */
14676 :
14677 : static int
14678 604 : get_element_number (tree vec_type, tree arg)
14679 : {
14680 604 : unsigned HOST_WIDE_INT elt, max = TYPE_VECTOR_SUBPARTS (vec_type) - 1;
14681 :
14682 604 : if (!tree_fits_uhwi_p (arg)
14683 604 : || (elt = tree_to_uhwi (arg), elt > max))
14684 : {
14685 0 : error ("selector must be an integer constant in the range "
14686 : "[0, %wi]", max);
14687 0 : return 0;
14688 : }
14689 :
14690 604 : return elt;
14691 : }
14692 :
14693 : /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
14694 : ix86_expand_vector_init. We DO have language-level syntax for this, in
14695 : the form of (type){ init-list }. Except that since we can't place emms
14696 : instructions from inside the compiler, we can't allow the use of MMX
14697 : registers unless the user explicitly asks for it. So we do *not* define
14698 : vec_set/vec_extract/vec_init patterns for MMX modes in mmx.md. Instead
14699 : we have builtins invoked by mmintrin.h that gives us license to emit
14700 : these sorts of instructions. */
14701 :
14702 : static rtx
14703 229 : ix86_expand_vec_init_builtin (tree type, tree exp, rtx target)
14704 : {
14705 229 : machine_mode tmode = TYPE_MODE (type);
14706 229 : machine_mode inner_mode = GET_MODE_INNER (tmode);
14707 229 : int i, n_elt = GET_MODE_NUNITS (tmode);
14708 229 : rtvec v = rtvec_alloc (n_elt);
14709 :
14710 229 : gcc_assert (VECTOR_MODE_P (tmode));
14711 229 : gcc_assert (call_expr_nargs (exp) == n_elt);
14712 :
14713 1203 : for (i = 0; i < n_elt; ++i)
14714 : {
14715 974 : rtx x = expand_normal (CALL_EXPR_ARG (exp, i));
14716 974 : RTVEC_ELT (v, i) = gen_lowpart (inner_mode, x);
14717 : }
14718 :
14719 229 : if (!target || !register_operand (target, tmode))
14720 0 : target = gen_reg_rtx (tmode);
14721 :
14722 229 : ix86_expand_vector_init (true, target, gen_rtx_PARALLEL (tmode, v));
14723 229 : return target;
14724 : }
14725 :
14726 : /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
14727 : ix86_expand_vector_extract. They would be redundant (for non-MMX) if we
14728 : had a language-level syntax for referencing vector elements. */
14729 :
14730 : static rtx
14731 400 : ix86_expand_vec_ext_builtin (tree exp, rtx target)
14732 : {
14733 400 : machine_mode tmode, mode0;
14734 400 : tree arg0, arg1;
14735 400 : int elt;
14736 400 : rtx op0;
14737 :
14738 400 : arg0 = CALL_EXPR_ARG (exp, 0);
14739 400 : arg1 = CALL_EXPR_ARG (exp, 1);
14740 :
14741 400 : op0 = expand_normal (arg0);
14742 400 : elt = get_element_number (TREE_TYPE (arg0), arg1);
14743 :
14744 400 : tmode = TYPE_MODE (TREE_TYPE (TREE_TYPE (arg0)));
14745 400 : mode0 = TYPE_MODE (TREE_TYPE (arg0));
14746 400 : gcc_assert (VECTOR_MODE_P (mode0));
14747 :
14748 400 : op0 = force_reg (mode0, op0);
14749 :
14750 400 : if (optimize || !target || !register_operand (target, tmode))
14751 321 : target = gen_reg_rtx (tmode);
14752 :
14753 400 : ix86_expand_vector_extract (true, target, op0, elt);
14754 :
14755 400 : return target;
14756 : }
14757 :
14758 : /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
14759 : ix86_expand_vector_set. They would be redundant (for non-MMX) if we had
14760 : a language-level syntax for referencing vector elements. */
14761 :
14762 : static rtx
14763 204 : ix86_expand_vec_set_builtin (tree exp)
14764 : {
14765 204 : machine_mode tmode, mode1;
14766 204 : tree arg0, arg1, arg2;
14767 204 : int elt;
14768 204 : rtx op0, op1, target;
14769 :
14770 204 : arg0 = CALL_EXPR_ARG (exp, 0);
14771 204 : arg1 = CALL_EXPR_ARG (exp, 1);
14772 204 : arg2 = CALL_EXPR_ARG (exp, 2);
14773 :
14774 204 : tmode = TYPE_MODE (TREE_TYPE (arg0));
14775 204 : mode1 = TYPE_MODE (TREE_TYPE (TREE_TYPE (arg0)));
14776 204 : gcc_assert (VECTOR_MODE_P (tmode));
14777 :
14778 204 : op0 = expand_expr (arg0, NULL_RTX, tmode, EXPAND_NORMAL);
14779 204 : op1 = expand_expr (arg1, NULL_RTX, mode1, EXPAND_NORMAL);
14780 204 : elt = get_element_number (TREE_TYPE (arg0), arg2);
14781 :
14782 204 : if (GET_MODE (op1) != mode1)
14783 82 : op1 = convert_modes (mode1, GET_MODE (op1), op1, true);
14784 :
14785 204 : op0 = force_reg (tmode, op0);
14786 204 : op1 = force_reg (mode1, op1);
14787 :
14788 : /* OP0 is the source of these builtin functions and shouldn't be
14789 : modified. Create a copy, use it and return it as target. */
14790 204 : target = gen_reg_rtx (tmode);
14791 204 : emit_move_insn (target, op0);
14792 204 : ix86_expand_vector_set (true, target, op1, elt);
14793 :
14794 204 : return target;
14795 : }
14796 :
14797 : /* Return true if the necessary isa options for this builtin exist,
14798 : else false.
14799 : fcode = DECL_MD_FUNCTION_CODE (fndecl); */
14800 : bool
14801 1328589 : ix86_check_builtin_isa_match (unsigned int fcode,
14802 : HOST_WIDE_INT* pbisa,
14803 : HOST_WIDE_INT* pbisa2)
14804 : {
14805 1328589 : HOST_WIDE_INT isa = ix86_isa_flags;
14806 1328589 : HOST_WIDE_INT isa2 = ix86_isa_flags2;
14807 1328589 : HOST_WIDE_INT bisa = ix86_builtins_isa[fcode].isa;
14808 1328589 : HOST_WIDE_INT bisa2 = ix86_builtins_isa[fcode].isa2;
14809 1328589 : HOST_WIDE_INT tmp_isa = isa, tmp_isa2 = isa2;
14810 : /* The general case is we require all the ISAs specified in bisa{,2}
14811 : to be enabled.
14812 : The exceptions are:
14813 : OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A
14814 : OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32
14815 : OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4
14816 : (OPTION_MASK_ISA_AVX512VNNI | OPTION_MASK_ISA_AVX512VL) or
14817 : OPTION_MASK_ISA2_AVXVNNI
14818 : (OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL) or
14819 : OPTION_MASK_ISA2_AVXIFMA
14820 : (OPTION_MASK_ISA_AVX512VL | OPTION_MASK_ISA2_AVX512BF16) or
14821 : OPTION_MASK_ISA2_AVXNECONVERT
14822 : OPTION_MASK_ISA_AES or (OPTION_MASK_ISA_AVX512VL | OPTION_MASK_ISA2_VAES)
14823 : OPTION_MASK_ISA2_AVX10_2 or OPTION_MASK_ISA2_AVXVNNIINT8
14824 : OPTION_MASK_ISA2_AVX10_2 or OPTION_MASK_ISA2_AVXVNNIINT16
14825 : where for each such pair it is sufficient if either of the ISAs is
14826 : enabled, plus if it is ored with other options also those others.
14827 : OPTION_MASK_ISA_MMX in bisa is satisfied also if TARGET_MMX_WITH_SSE. */
14828 :
14829 : #define SHARE_BUILTIN(A1, A2, B1, B2) \
14830 : if ((((bisa & (A1)) == (A1) && (bisa2 & (A2)) == (A2)) \
14831 : && ((bisa & (B1)) == (B1) && (bisa2 & (B2)) == (B2))) \
14832 : && (((isa & (A1)) == (A1) && (isa2 & (A2)) == (A2)) \
14833 : || ((isa & (B1)) == (B1) && (isa2 & (B2)) == (B2)))) \
14834 : { \
14835 : tmp_isa |= (A1) | (B1); \
14836 : tmp_isa2 |= (A2) | (B2); \
14837 : }
14838 :
14839 1328589 : SHARE_BUILTIN (OPTION_MASK_ISA_SSE, 0, OPTION_MASK_ISA_3DNOW_A, 0);
14840 1328589 : SHARE_BUILTIN (OPTION_MASK_ISA_SSE4_2, 0, OPTION_MASK_ISA_CRC32, 0);
14841 1328589 : SHARE_BUILTIN (OPTION_MASK_ISA_FMA, 0, OPTION_MASK_ISA_FMA4, 0);
14842 1328589 : SHARE_BUILTIN (OPTION_MASK_ISA_AVX512VNNI | OPTION_MASK_ISA_AVX512VL, 0, 0,
14843 1328589 : OPTION_MASK_ISA2_AVXVNNI);
14844 1328589 : SHARE_BUILTIN (OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, 0, 0,
14845 1328589 : OPTION_MASK_ISA2_AVXIFMA);
14846 1328589 : SHARE_BUILTIN (OPTION_MASK_ISA_AVX512VL, OPTION_MASK_ISA2_AVX512BF16, 0,
14847 1328589 : OPTION_MASK_ISA2_AVXNECONVERT);
14848 1328589 : SHARE_BUILTIN (OPTION_MASK_ISA_AES, 0, OPTION_MASK_ISA_AVX512VL,
14849 1328589 : OPTION_MASK_ISA2_VAES);
14850 1328589 : SHARE_BUILTIN (0, OPTION_MASK_ISA2_AVXVNNIINT8, 0,
14851 1328589 : OPTION_MASK_ISA2_AVX10_2);
14852 1328589 : SHARE_BUILTIN (0, OPTION_MASK_ISA2_AVXVNNIINT16, 0,
14853 1328589 : OPTION_MASK_ISA2_AVX10_2);
14854 1328589 : isa = tmp_isa;
14855 1328589 : isa2 = tmp_isa2;
14856 :
14857 1328589 : if ((bisa & OPTION_MASK_ISA_MMX) && !TARGET_MMX && TARGET_MMX_WITH_SSE
14858 : /* __builtin_ia32_maskmovq requires MMX registers. */
14859 4563 : && fcode != IX86_BUILTIN_MASKMOVQ)
14860 : {
14861 4554 : bisa &= ~OPTION_MASK_ISA_MMX;
14862 4554 : bisa |= OPTION_MASK_ISA_SSE2;
14863 : }
14864 :
14865 1328589 : if (pbisa)
14866 174830 : *pbisa = bisa;
14867 1328589 : if (pbisa2)
14868 174830 : *pbisa2 = bisa2;
14869 :
14870 1328589 : return (bisa & isa) == bisa && (bisa2 & isa2) == bisa2;
14871 : }
14872 :
14873 : /* Emit instructions to set the carry flag from ARG. */
14874 :
14875 : void
14876 13556 : ix86_expand_carry (rtx arg)
14877 : {
14878 13556 : if (!CONST_INT_P (arg) || arg == const0_rtx)
14879 : {
14880 13550 : arg = convert_to_mode (QImode, arg, 1);
14881 13550 : arg = copy_to_mode_reg (QImode, arg);
14882 13550 : emit_insn (gen_addqi3_cconly_overflow (arg, constm1_rtx));
14883 : }
14884 : else
14885 6 : emit_insn (gen_x86_stc ());
14886 13556 : }
14887 :
14888 : /* Expand an expression EXP that calls a built-in function,
14889 : with result going to TARGET if that's convenient
14890 : (and in mode MODE if that's convenient).
14891 : SUBTARGET may be used as the target for computing one of EXP's operands.
14892 : IGNORE is nonzero if the value is to be ignored. */
14893 :
14894 : rtx
14895 175629 : ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
14896 : machine_mode mode, int ignore)
14897 : {
14898 175629 : size_t i;
14899 175629 : enum insn_code icode, icode2;
14900 175629 : tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
14901 175629 : tree arg0, arg1, arg2, arg3, arg4;
14902 175629 : rtx op0, op1, op2, op3, op4, pat, pat2, insn;
14903 175629 : machine_mode mode0, mode1, mode2, mode3, mode4;
14904 175629 : unsigned int fcode = DECL_MD_FUNCTION_CODE (fndecl);
14905 175629 : HOST_WIDE_INT bisa, bisa2;
14906 :
14907 : /* For CPU builtins that can be folded, fold first and expand the fold. */
14908 175629 : switch (fcode)
14909 : {
14910 197 : case IX86_BUILTIN_CPU_INIT:
14911 197 : {
14912 : /* Make it call __cpu_indicator_init in libgcc. */
14913 197 : tree call_expr, fndecl, type;
14914 197 : type = build_function_type_list (integer_type_node, NULL_TREE);
14915 197 : fndecl = build_fn_decl ("__cpu_indicator_init", type);
14916 197 : call_expr = build_call_expr (fndecl, 0);
14917 197 : return expand_expr (call_expr, target, mode, EXPAND_NORMAL);
14918 : }
14919 602 : case IX86_BUILTIN_CPU_IS:
14920 602 : case IX86_BUILTIN_CPU_SUPPORTS:
14921 602 : {
14922 602 : tree arg0 = CALL_EXPR_ARG (exp, 0);
14923 602 : tree fold_expr = fold_builtin_cpu (fndecl, &arg0);
14924 602 : gcc_assert (fold_expr != NULL_TREE);
14925 602 : return expand_expr (fold_expr, target, mode, EXPAND_NORMAL);
14926 : }
14927 : }
14928 :
14929 174830 : if (!ix86_check_builtin_isa_match (fcode, &bisa, &bisa2))
14930 : {
14931 23 : bool add_abi_p = bisa & OPTION_MASK_ISA_64BIT;
14932 23 : if (TARGET_ABI_X32)
14933 0 : bisa |= OPTION_MASK_ABI_X32;
14934 : else
14935 23 : bisa |= OPTION_MASK_ABI_64;
14936 23 : char *opts = ix86_target_string (bisa, bisa2, 0, 0, NULL, NULL,
14937 : (enum fpmath_unit) 0,
14938 : (enum prefer_vector_width) 0,
14939 : PVW_NONE, false, add_abi_p);
14940 23 : if (!opts)
14941 0 : error ("%qE needs unknown isa option", fndecl);
14942 : else
14943 : {
14944 23 : gcc_assert (opts != NULL);
14945 23 : error ("%qE needs isa option %s", fndecl, opts);
14946 23 : free (opts);
14947 : }
14948 23 : return expand_call (exp, target, ignore);
14949 : }
14950 :
14951 174807 : switch (fcode)
14952 : {
14953 35 : case IX86_BUILTIN_MASKMOVQ:
14954 35 : case IX86_BUILTIN_MASKMOVDQU:
14955 34 : icode = (fcode == IX86_BUILTIN_MASKMOVQ
14956 35 : ? CODE_FOR_mmx_maskmovq
14957 : : CODE_FOR_sse2_maskmovdqu);
14958 : /* Note the arg order is different from the operand order. */
14959 35 : arg1 = CALL_EXPR_ARG (exp, 0);
14960 35 : arg2 = CALL_EXPR_ARG (exp, 1);
14961 35 : arg0 = CALL_EXPR_ARG (exp, 2);
14962 35 : op0 = expand_normal (arg0);
14963 35 : op1 = expand_normal (arg1);
14964 35 : op2 = expand_normal (arg2);
14965 35 : mode0 = insn_data[icode].operand[0].mode;
14966 35 : mode1 = insn_data[icode].operand[1].mode;
14967 35 : mode2 = insn_data[icode].operand[2].mode;
14968 :
14969 35 : op0 = ix86_zero_extend_to_Pmode (op0);
14970 35 : op0 = gen_rtx_MEM (mode1, op0);
14971 :
14972 35 : if (!insn_data[icode].operand[0].predicate (op0, mode0))
14973 0 : op0 = copy_to_mode_reg (mode0, op0);
14974 35 : if (!insn_data[icode].operand[1].predicate (op1, mode1))
14975 2 : op1 = copy_to_mode_reg (mode1, op1);
14976 35 : if (!insn_data[icode].operand[2].predicate (op2, mode2))
14977 2 : op2 = copy_to_mode_reg (mode2, op2);
14978 35 : pat = GEN_FCN (icode) (op0, op1, op2);
14979 35 : if (! pat)
14980 56404 : return 0;
14981 35 : emit_insn (pat);
14982 35 : return 0;
14983 :
14984 21937 : case IX86_BUILTIN_LDMXCSR:
14985 21937 : op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
14986 21937 : target = assign_stack_temp (SImode, GET_MODE_SIZE (SImode));
14987 21937 : emit_move_insn (target, op0);
14988 21937 : emit_insn (gen_sse_ldmxcsr (target));
14989 21937 : return 0;
14990 :
14991 14713 : case IX86_BUILTIN_STMXCSR:
14992 14713 : target = assign_stack_temp (SImode, GET_MODE_SIZE (SImode));
14993 14713 : emit_insn (gen_sse_stmxcsr (target));
14994 14713 : return copy_to_mode_reg (SImode, target);
14995 :
14996 11 : case IX86_BUILTIN_CLFLUSH:
14997 11 : arg0 = CALL_EXPR_ARG (exp, 0);
14998 11 : op0 = expand_normal (arg0);
14999 11 : icode = CODE_FOR_sse2_clflush;
15000 11 : if (!insn_data[icode].operand[0].predicate (op0, Pmode))
15001 5 : op0 = ix86_zero_extend_to_Pmode (op0);
15002 :
15003 11 : emit_insn (gen_sse2_clflush (op0));
15004 11 : return 0;
15005 :
15006 19 : case IX86_BUILTIN_CLWB:
15007 19 : arg0 = CALL_EXPR_ARG (exp, 0);
15008 19 : op0 = expand_normal (arg0);
15009 19 : icode = CODE_FOR_clwb;
15010 19 : if (!insn_data[icode].operand[0].predicate (op0, Pmode))
15011 9 : op0 = ix86_zero_extend_to_Pmode (op0);
15012 :
15013 19 : emit_insn (gen_clwb (op0));
15014 19 : return 0;
15015 :
15016 19 : case IX86_BUILTIN_CLFLUSHOPT:
15017 19 : arg0 = CALL_EXPR_ARG (exp, 0);
15018 19 : op0 = expand_normal (arg0);
15019 19 : icode = CODE_FOR_clflushopt;
15020 19 : if (!insn_data[icode].operand[0].predicate (op0, Pmode))
15021 9 : op0 = ix86_zero_extend_to_Pmode (op0);
15022 :
15023 19 : emit_insn (gen_clflushopt (op0));
15024 19 : return 0;
15025 :
15026 47 : case IX86_BUILTIN_MONITOR:
15027 47 : case IX86_BUILTIN_MONITORX:
15028 47 : arg0 = CALL_EXPR_ARG (exp, 0);
15029 47 : arg1 = CALL_EXPR_ARG (exp, 1);
15030 47 : arg2 = CALL_EXPR_ARG (exp, 2);
15031 47 : op0 = expand_normal (arg0);
15032 47 : op1 = expand_normal (arg1);
15033 47 : op2 = expand_normal (arg2);
15034 47 : if (!REG_P (op0))
15035 19 : op0 = ix86_zero_extend_to_Pmode (op0);
15036 47 : if (!REG_P (op1))
15037 22 : op1 = copy_to_mode_reg (SImode, op1);
15038 47 : if (!REG_P (op2))
15039 25 : op2 = copy_to_mode_reg (SImode, op2);
15040 :
15041 47 : emit_insn (fcode == IX86_BUILTIN_MONITOR
15042 26 : ? gen_sse3_monitor (Pmode, op0, op1, op2)
15043 21 : : gen_monitorx (Pmode, op0, op1, op2));
15044 47 : return 0;
15045 :
15046 25 : case IX86_BUILTIN_MWAIT:
15047 25 : arg0 = CALL_EXPR_ARG (exp, 0);
15048 25 : arg1 = CALL_EXPR_ARG (exp, 1);
15049 25 : op0 = expand_normal (arg0);
15050 25 : op1 = expand_normal (arg1);
15051 25 : if (!REG_P (op0))
15052 13 : op0 = copy_to_mode_reg (SImode, op0);
15053 25 : if (!REG_P (op1))
15054 11 : op1 = copy_to_mode_reg (SImode, op1);
15055 25 : emit_insn (gen_sse3_mwait (op0, op1));
15056 25 : return 0;
15057 :
15058 21 : case IX86_BUILTIN_MWAITX:
15059 21 : arg0 = CALL_EXPR_ARG (exp, 0);
15060 21 : arg1 = CALL_EXPR_ARG (exp, 1);
15061 21 : arg2 = CALL_EXPR_ARG (exp, 2);
15062 21 : op0 = expand_normal (arg0);
15063 21 : op1 = expand_normal (arg1);
15064 21 : op2 = expand_normal (arg2);
15065 21 : if (!REG_P (op0))
15066 11 : op0 = copy_to_mode_reg (SImode, op0);
15067 21 : if (!REG_P (op1))
15068 10 : op1 = copy_to_mode_reg (SImode, op1);
15069 21 : if (!REG_P (op2))
15070 11 : op2 = copy_to_mode_reg (SImode, op2);
15071 21 : emit_insn (gen_mwaitx (op0, op1, op2));
15072 21 : return 0;
15073 :
15074 21 : case IX86_BUILTIN_UMONITOR:
15075 21 : arg0 = CALL_EXPR_ARG (exp, 0);
15076 21 : op0 = expand_normal (arg0);
15077 :
15078 21 : op0 = ix86_zero_extend_to_Pmode (op0);
15079 21 : emit_insn (gen_umonitor (Pmode, op0));
15080 21 : return 0;
15081 :
15082 42 : case IX86_BUILTIN_UMWAIT:
15083 42 : case IX86_BUILTIN_TPAUSE:
15084 42 : arg0 = CALL_EXPR_ARG (exp, 0);
15085 42 : arg1 = CALL_EXPR_ARG (exp, 1);
15086 42 : op0 = expand_normal (arg0);
15087 42 : op1 = expand_normal (arg1);
15088 :
15089 42 : if (!REG_P (op0))
15090 20 : op0 = copy_to_mode_reg (SImode, op0);
15091 :
15092 42 : op1 = force_reg (DImode, op1);
15093 :
15094 42 : if (TARGET_64BIT)
15095 : {
15096 42 : op2 = expand_simple_binop (DImode, LSHIFTRT, op1, GEN_INT (32),
15097 : NULL, 1, OPTAB_DIRECT);
15098 42 : switch (fcode)
15099 : {
15100 : case IX86_BUILTIN_UMWAIT:
15101 : icode = CODE_FOR_umwait_rex64;
15102 : break;
15103 21 : case IX86_BUILTIN_TPAUSE:
15104 21 : icode = CODE_FOR_tpause_rex64;
15105 21 : break;
15106 0 : default:
15107 0 : gcc_unreachable ();
15108 : }
15109 :
15110 42 : op2 = gen_lowpart (SImode, op2);
15111 42 : op1 = gen_lowpart (SImode, op1);
15112 42 : pat = GEN_FCN (icode) (op0, op1, op2);
15113 : }
15114 : else
15115 : {
15116 0 : switch (fcode)
15117 : {
15118 : case IX86_BUILTIN_UMWAIT:
15119 : icode = CODE_FOR_umwait;
15120 : break;
15121 0 : case IX86_BUILTIN_TPAUSE:
15122 0 : icode = CODE_FOR_tpause;
15123 0 : break;
15124 0 : default:
15125 0 : gcc_unreachable ();
15126 : }
15127 0 : pat = GEN_FCN (icode) (op0, op1);
15128 : }
15129 :
15130 42 : if (!pat)
15131 : return 0;
15132 :
15133 42 : emit_insn (pat);
15134 :
15135 42 : if (target == 0
15136 42 : || !register_operand (target, QImode))
15137 0 : target = gen_reg_rtx (QImode);
15138 :
15139 42 : pat = gen_rtx_EQ (QImode, gen_rtx_REG (CCCmode, FLAGS_REG),
15140 : const0_rtx);
15141 42 : emit_insn (gen_rtx_SET (target, pat));
15142 :
15143 42 : return target;
15144 :
15145 20 : case IX86_BUILTIN_TESTUI:
15146 20 : emit_insn (gen_testui ());
15147 :
15148 20 : if (target == 0
15149 20 : || !register_operand (target, QImode))
15150 0 : target = gen_reg_rtx (QImode);
15151 :
15152 20 : pat = gen_rtx_LTU (QImode, gen_rtx_REG (CCCmode, FLAGS_REG),
15153 : const0_rtx);
15154 20 : emit_insn (gen_rtx_SET (target, pat));
15155 :
15156 20 : return target;
15157 :
15158 19 : case IX86_BUILTIN_CLZERO:
15159 19 : arg0 = CALL_EXPR_ARG (exp, 0);
15160 19 : op0 = expand_normal (arg0);
15161 19 : if (!REG_P (op0))
15162 9 : op0 = ix86_zero_extend_to_Pmode (op0);
15163 19 : emit_insn (gen_clzero (Pmode, op0));
15164 19 : return 0;
15165 :
15166 19 : case IX86_BUILTIN_CLDEMOTE:
15167 19 : arg0 = CALL_EXPR_ARG (exp, 0);
15168 19 : op0 = expand_normal (arg0);
15169 19 : icode = CODE_FOR_cldemote;
15170 19 : if (!insn_data[icode].operand[0].predicate (op0, Pmode))
15171 9 : op0 = ix86_zero_extend_to_Pmode (op0);
15172 :
15173 19 : emit_insn (gen_cldemote (op0));
15174 19 : return 0;
15175 :
15176 11 : case IX86_BUILTIN_LOADIWKEY:
15177 11 : {
15178 11 : arg0 = CALL_EXPR_ARG (exp, 0);
15179 11 : arg1 = CALL_EXPR_ARG (exp, 1);
15180 11 : arg2 = CALL_EXPR_ARG (exp, 2);
15181 11 : arg3 = CALL_EXPR_ARG (exp, 3);
15182 :
15183 11 : op0 = expand_normal (arg0);
15184 11 : op1 = expand_normal (arg1);
15185 11 : op2 = expand_normal (arg2);
15186 11 : op3 = expand_normal (arg3);
15187 :
15188 11 : if (!REG_P (op0))
15189 5 : op0 = copy_to_mode_reg (V2DImode, op0);
15190 11 : if (!REG_P (op1))
15191 5 : op1 = copy_to_mode_reg (V2DImode, op1);
15192 11 : if (!REG_P (op2))
15193 5 : op2 = copy_to_mode_reg (V2DImode, op2);
15194 11 : if (!REG_P (op3))
15195 5 : op3 = copy_to_mode_reg (SImode, op3);
15196 :
15197 11 : emit_insn (gen_loadiwkey (op0, op1, op2, op3));
15198 :
15199 11 : return 0;
15200 : }
15201 :
15202 12 : case IX86_BUILTIN_AESDEC128KLU8:
15203 12 : icode = CODE_FOR_aesdec128klu8;
15204 12 : goto aesdecenc_expand;
15205 :
15206 12 : case IX86_BUILTIN_AESDEC256KLU8:
15207 12 : icode = CODE_FOR_aesdec256klu8;
15208 12 : goto aesdecenc_expand;
15209 :
15210 12 : case IX86_BUILTIN_AESENC128KLU8:
15211 12 : icode = CODE_FOR_aesenc128klu8;
15212 12 : goto aesdecenc_expand;
15213 :
15214 : case IX86_BUILTIN_AESENC256KLU8:
15215 : icode = CODE_FOR_aesenc256klu8;
15216 :
15217 48 : aesdecenc_expand:
15218 :
15219 48 : arg0 = CALL_EXPR_ARG (exp, 0); // __m128i *odata
15220 48 : arg1 = CALL_EXPR_ARG (exp, 1); // __m128i idata
15221 48 : arg2 = CALL_EXPR_ARG (exp, 2); // const void *p
15222 :
15223 48 : op0 = expand_normal (arg0);
15224 48 : op1 = expand_normal (arg1);
15225 48 : op2 = expand_normal (arg2);
15226 :
15227 48 : if (!address_operand (op0, V2DImode))
15228 : {
15229 16 : op0 = convert_memory_address (Pmode, op0);
15230 16 : op0 = copy_addr_to_reg (op0);
15231 : }
15232 48 : op0 = gen_rtx_MEM (V2DImode, op0);
15233 :
15234 48 : if (!REG_P (op1))
15235 20 : op1 = copy_to_mode_reg (V2DImode, op1);
15236 :
15237 48 : if (!address_operand (op2, VOIDmode))
15238 : {
15239 16 : op2 = convert_memory_address (Pmode, op2);
15240 16 : op2 = copy_addr_to_reg (op2);
15241 : }
15242 48 : op2 = gen_rtx_MEM (BLKmode, op2);
15243 :
15244 48 : emit_insn (GEN_FCN (icode) (op1, op1, op2));
15245 :
15246 48 : if (target == 0)
15247 4 : target = gen_reg_rtx (QImode);
15248 :
15249 : /* NB: For aesenc/aesdec keylocker insn, ZF will be set when runtime
15250 : error occurs. Then the output should be cleared for safety. */
15251 48 : rtx_code_label *ok_label;
15252 48 : rtx tmp;
15253 :
15254 48 : tmp = gen_rtx_REG (CCZmode, FLAGS_REG);
15255 48 : pat = gen_rtx_EQ (QImode, tmp, const0_rtx);
15256 48 : ok_label = gen_label_rtx ();
15257 48 : emit_cmp_and_jump_insns (tmp, const0_rtx, NE, 0, GET_MODE (tmp),
15258 : true, ok_label);
15259 : /* Usually the runtime error seldom occur, so predict OK path as
15260 : hotspot to optimize it as fallthrough block. */
15261 48 : predict_jump (REG_BR_PROB_BASE * 90 / 100);
15262 :
15263 48 : emit_insn (gen_rtx_SET (op1, const0_rtx));
15264 :
15265 48 : emit_label (ok_label);
15266 48 : emit_insn (gen_rtx_SET (target, pat));
15267 48 : emit_insn (gen_rtx_SET (op0, op1));
15268 :
15269 48 : return target;
15270 :
15271 11 : case IX86_BUILTIN_AESDECWIDE128KLU8:
15272 11 : icode = CODE_FOR_aesdecwide128klu8;
15273 11 : goto wideaesdecenc_expand;
15274 :
15275 11 : case IX86_BUILTIN_AESDECWIDE256KLU8:
15276 11 : icode = CODE_FOR_aesdecwide256klu8;
15277 11 : goto wideaesdecenc_expand;
15278 :
15279 11 : case IX86_BUILTIN_AESENCWIDE128KLU8:
15280 11 : icode = CODE_FOR_aesencwide128klu8;
15281 11 : goto wideaesdecenc_expand;
15282 :
15283 : case IX86_BUILTIN_AESENCWIDE256KLU8:
15284 : icode = CODE_FOR_aesencwide256klu8;
15285 :
15286 44 : wideaesdecenc_expand:
15287 :
15288 44 : rtx xmm_regs[8];
15289 44 : rtx op;
15290 :
15291 44 : arg0 = CALL_EXPR_ARG (exp, 0); // __m128i * odata
15292 44 : arg1 = CALL_EXPR_ARG (exp, 1); // const __m128i * idata
15293 44 : arg2 = CALL_EXPR_ARG (exp, 2); // const void *p
15294 :
15295 44 : op0 = expand_normal (arg0);
15296 44 : op1 = expand_normal (arg1);
15297 44 : op2 = expand_normal (arg2);
15298 :
15299 44 : if (GET_MODE (op1) != Pmode)
15300 0 : op1 = convert_to_mode (Pmode, op1, 1);
15301 :
15302 44 : if (!address_operand (op2, VOIDmode))
15303 : {
15304 16 : op2 = convert_memory_address (Pmode, op2);
15305 16 : op2 = copy_addr_to_reg (op2);
15306 : }
15307 44 : op2 = gen_rtx_MEM (BLKmode, op2);
15308 :
15309 440 : for (i = 0; i < 8; i++)
15310 : {
15311 352 : xmm_regs[i] = gen_rtx_REG (V2DImode, GET_SSE_REGNO (i));
15312 :
15313 352 : op = gen_rtx_MEM (V2DImode,
15314 352 : plus_constant (Pmode, op1, (i * 16)));
15315 :
15316 352 : emit_move_insn (xmm_regs[i], op);
15317 : }
15318 :
15319 44 : emit_insn (GEN_FCN (icode) (op2));
15320 :
15321 44 : if (target == 0)
15322 0 : target = gen_reg_rtx (QImode);
15323 :
15324 44 : tmp = gen_rtx_REG (CCZmode, FLAGS_REG);
15325 44 : pat = gen_rtx_EQ (QImode, tmp, const0_rtx);
15326 44 : ok_label = gen_label_rtx ();
15327 44 : emit_cmp_and_jump_insns (tmp, const0_rtx, NE, 0, GET_MODE (tmp),
15328 : true, ok_label);
15329 44 : predict_jump (REG_BR_PROB_BASE * 90 / 100);
15330 :
15331 440 : for (i = 0; i < 8; i++)
15332 352 : emit_insn (gen_rtx_SET (xmm_regs[i], const0_rtx));
15333 :
15334 44 : emit_label (ok_label);
15335 44 : emit_insn (gen_rtx_SET (target, pat));
15336 :
15337 44 : if (GET_MODE (op0) != Pmode)
15338 0 : op0 = convert_to_mode (Pmode, op0, 1);
15339 :
15340 396 : for (i = 0; i < 8; i++)
15341 : {
15342 352 : op = gen_rtx_MEM (V2DImode,
15343 352 : plus_constant (Pmode, op0, (i * 16)));
15344 352 : emit_move_insn (op, xmm_regs[i]);
15345 : }
15346 :
15347 : return target;
15348 :
15349 13 : case IX86_BUILTIN_ENCODEKEY128U32:
15350 13 : {
15351 13 : rtx op, xmm_regs[7];
15352 :
15353 13 : arg0 = CALL_EXPR_ARG (exp, 0); // unsigned int htype
15354 13 : arg1 = CALL_EXPR_ARG (exp, 1); // __m128i key
15355 13 : arg2 = CALL_EXPR_ARG (exp, 2); // void *h
15356 :
15357 13 : op0 = expand_normal (arg0);
15358 13 : op1 = expand_normal (arg1);
15359 13 : op2 = expand_normal (arg2);
15360 :
15361 13 : if (!REG_P (op0))
15362 7 : op0 = copy_to_mode_reg (SImode, op0);
15363 :
15364 13 : if (GET_MODE (op2) != Pmode)
15365 1 : op2 = convert_to_mode (Pmode, op2, 1);
15366 :
15367 13 : op = gen_rtx_REG (V2DImode, GET_SSE_REGNO (0));
15368 13 : emit_move_insn (op, op1);
15369 :
15370 65 : for (i = 0; i < 3; i++)
15371 39 : xmm_regs[i] = gen_rtx_REG (V2DImode, GET_SSE_REGNO (i));
15372 :
15373 13 : if (target == 0 || !register_operand (target, SImode))
15374 2 : target = gen_reg_rtx (SImode);
15375 :
15376 13 : emit_insn (gen_encodekey128u32 (target, op0));
15377 :
15378 65 : for (i = 0; i < 3; i++)
15379 : {
15380 39 : op = gen_rtx_MEM (V2DImode,
15381 39 : plus_constant (Pmode, op2, (i * 16)));
15382 39 : emit_move_insn (op, xmm_regs[i]);
15383 : }
15384 :
15385 13 : return target;
15386 : }
15387 13 : case IX86_BUILTIN_ENCODEKEY256U32:
15388 13 : {
15389 13 : rtx op, xmm_regs[7];
15390 :
15391 13 : arg0 = CALL_EXPR_ARG (exp, 0); // unsigned int htype
15392 13 : arg1 = CALL_EXPR_ARG (exp, 1); // __m128i keylow
15393 13 : arg2 = CALL_EXPR_ARG (exp, 2); // __m128i keyhi
15394 13 : arg3 = CALL_EXPR_ARG (exp, 3); // void *h
15395 :
15396 13 : op0 = expand_normal (arg0);
15397 13 : op1 = expand_normal (arg1);
15398 13 : op2 = expand_normal (arg2);
15399 13 : op3 = expand_normal (arg3);
15400 :
15401 13 : if (!REG_P (op0))
15402 7 : op0 = copy_to_mode_reg (SImode, op0);
15403 :
15404 13 : if (GET_MODE (op3) != Pmode)
15405 1 : op3 = convert_to_mode (Pmode, op3, 1);
15406 :
15407 : /* Force to use xmm0, xmm1 for keylow, keyhi*/
15408 13 : op = gen_rtx_REG (V2DImode, GET_SSE_REGNO (0));
15409 13 : emit_move_insn (op, op1);
15410 13 : op = gen_rtx_REG (V2DImode, GET_SSE_REGNO (1));
15411 13 : emit_move_insn (op, op2);
15412 :
15413 78 : for (i = 0; i < 4; i++)
15414 52 : xmm_regs[i] = gen_rtx_REG (V2DImode, GET_SSE_REGNO (i));
15415 :
15416 13 : if (target == 0 || !register_operand (target, SImode))
15417 2 : target = gen_reg_rtx (SImode);
15418 :
15419 13 : emit_insn (gen_encodekey256u32 (target, op0));
15420 :
15421 78 : for (i = 0; i < 4; i++)
15422 : {
15423 52 : op = gen_rtx_MEM (V2DImode,
15424 52 : plus_constant (Pmode, op3, (i * 16)));
15425 52 : emit_move_insn (op, xmm_regs[i]);
15426 : }
15427 :
15428 13 : return target;
15429 : }
15430 :
15431 48 : case IX86_BUILTIN_PREFETCH:
15432 48 : {
15433 48 : arg0 = CALL_EXPR_ARG (exp, 0); // const void *
15434 48 : arg1 = CALL_EXPR_ARG (exp, 1); // const int
15435 48 : arg2 = CALL_EXPR_ARG (exp, 2); // const int
15436 48 : arg3 = CALL_EXPR_ARG (exp, 3); // const int
15437 :
15438 48 : op0 = expand_normal (arg0);
15439 48 : op1 = expand_normal (arg1);
15440 48 : op2 = expand_normal (arg2);
15441 48 : op3 = expand_normal (arg3);
15442 :
15443 48 : if (!CONST_INT_P (op1) || !CONST_INT_P (op2) || !CONST_INT_P (op3))
15444 : {
15445 0 : error ("second, third and fourth argument must be a const");
15446 0 : return const0_rtx;
15447 : }
15448 :
15449 48 : if (!IN_RANGE (INTVAL (op1), 0, 2))
15450 : {
15451 1 : warning (0, "invalid second argument to"
15452 : " %<__builtin_ia32_prefetch%>; using zero");
15453 1 : op1 = const0_rtx;
15454 : }
15455 :
15456 48 : if (INTVAL (op3) == 1)
15457 : {
15458 4 : if (!IN_RANGE (INTVAL (op2), 2, 3))
15459 : {
15460 1 : error ("invalid third argument");
15461 1 : return const0_rtx;
15462 : }
15463 :
15464 3 : if (TARGET_64BIT && TARGET_PREFETCHI
15465 6 : && local_func_symbolic_operand (op0, GET_MODE (op0)))
15466 2 : emit_insn (gen_prefetchi (op0, op2));
15467 : else
15468 : {
15469 1 : warning (0, "instruction prefetch applies when in 64-bit mode"
15470 : " with RIP-relative addressing and"
15471 : " option %<-mprefetchi%>;"
15472 : " they stay NOPs otherwise");
15473 1 : emit_insn (gen_nop ());
15474 : }
15475 : }
15476 : else
15477 : {
15478 44 : if (INTVAL (op3) != 0)
15479 1 : warning (0, "invalid fourth argument to"
15480 : " %<__builtin_ia32_prefetch%>; using zero");
15481 :
15482 44 : if (!address_operand (op0, VOIDmode))
15483 : {
15484 10 : op0 = convert_memory_address (Pmode, op0);
15485 10 : op0 = copy_addr_to_reg (op0);
15486 : }
15487 :
15488 44 : if (!IN_RANGE (INTVAL (op2), 0, 3))
15489 : {
15490 1 : warning (0, "invalid third argument to %<__builtin_ia32_prefetch%>; using zero");
15491 1 : op2 = const0_rtx;
15492 : }
15493 :
15494 44 : if (TARGET_3DNOW
15495 26 : || TARGET_PREFETCH_SSE
15496 0 : || TARGET_PRFCHW
15497 0 : || TARGET_MOVRS)
15498 44 : emit_insn (gen_prefetch (op0, op1, op2));
15499 0 : else if (!MEM_P (op0) && side_effects_p (op0))
15500 : /* Don't do anything with direct references to volatile memory,
15501 : but generate code to handle other side effects. */
15502 0 : emit_insn (op0);
15503 : }
15504 :
15505 : return 0;
15506 : }
15507 :
15508 21 : case IX86_BUILTIN_PREFETCHI:
15509 21 : {
15510 21 : arg0 = CALL_EXPR_ARG (exp, 0); // const void *
15511 21 : arg1 = CALL_EXPR_ARG (exp, 1); // const int
15512 :
15513 21 : op0 = expand_normal (arg0);
15514 21 : op1 = expand_normal (arg1);
15515 :
15516 21 : if (!CONST_INT_P (op1))
15517 : {
15518 0 : error ("second argument must be a const");
15519 0 : return const0_rtx;
15520 : }
15521 :
15522 : /* GOT/PLT_PIC should not be available for instruction prefetch.
15523 : It must be real instruction address. */
15524 21 : if (TARGET_64BIT
15525 21 : && local_func_symbolic_operand (op0, GET_MODE (op0)))
15526 4 : emit_insn (gen_prefetchi (op0, op1));
15527 : else
15528 : {
15529 : /* Ignore the hint. */
15530 17 : warning (0, "instruction prefetch applies when in 64-bit mode"
15531 : " with RIP-relative addressing and"
15532 : " option %<-mprefetchi%>;"
15533 : " they stay NOPs otherwise");
15534 17 : emit_insn (gen_nop ());
15535 : }
15536 :
15537 : return 0;
15538 : }
15539 :
15540 53 : case IX86_BUILTIN_URDMSR:
15541 53 : case IX86_BUILTIN_UWRMSR:
15542 53 : {
15543 53 : arg0 = CALL_EXPR_ARG (exp, 0);
15544 53 : op0 = expand_normal (arg0);
15545 :
15546 53 : if (CONST_INT_P (op0))
15547 : {
15548 12 : unsigned HOST_WIDE_INT val = UINTVAL (op0);
15549 12 : if (val > 0xffffffff)
15550 2 : op0 = force_reg (DImode, op0);
15551 : }
15552 : else
15553 41 : op0 = force_reg (DImode, op0);
15554 :
15555 53 : if (fcode == IX86_BUILTIN_UWRMSR)
15556 : {
15557 26 : arg1 = CALL_EXPR_ARG (exp, 1);
15558 26 : op1 = expand_normal (arg1);
15559 26 : op1 = force_reg (DImode, op1);
15560 26 : icode = CODE_FOR_uwrmsr;
15561 26 : target = 0;
15562 : }
15563 : else
15564 : {
15565 27 : if (target == 0 || !register_operand (target, DImode))
15566 1 : target = gen_reg_rtx (DImode);
15567 : icode = CODE_FOR_urdmsr;
15568 : op1 = op0;
15569 : op0 = target;
15570 : }
15571 53 : emit_insn (GEN_FCN (icode) (op0, op1));
15572 53 : return target;
15573 : }
15574 :
15575 229 : case IX86_BUILTIN_VEC_INIT_V2SI:
15576 229 : case IX86_BUILTIN_VEC_INIT_V4HI:
15577 229 : case IX86_BUILTIN_VEC_INIT_V8QI:
15578 229 : return ix86_expand_vec_init_builtin (TREE_TYPE (exp), exp, target);
15579 :
15580 400 : case IX86_BUILTIN_VEC_EXT_V2DF:
15581 400 : case IX86_BUILTIN_VEC_EXT_V2DI:
15582 400 : case IX86_BUILTIN_VEC_EXT_V4SF:
15583 400 : case IX86_BUILTIN_VEC_EXT_V4SI:
15584 400 : case IX86_BUILTIN_VEC_EXT_V8HI:
15585 400 : case IX86_BUILTIN_VEC_EXT_V2SI:
15586 400 : case IX86_BUILTIN_VEC_EXT_V4HI:
15587 400 : case IX86_BUILTIN_VEC_EXT_V16QI:
15588 400 : return ix86_expand_vec_ext_builtin (exp, target);
15589 :
15590 204 : case IX86_BUILTIN_VEC_SET_V2DI:
15591 204 : case IX86_BUILTIN_VEC_SET_V4SF:
15592 204 : case IX86_BUILTIN_VEC_SET_V4SI:
15593 204 : case IX86_BUILTIN_VEC_SET_V8HI:
15594 204 : case IX86_BUILTIN_VEC_SET_V4HI:
15595 204 : case IX86_BUILTIN_VEC_SET_V16QI:
15596 204 : return ix86_expand_vec_set_builtin (exp);
15597 :
15598 0 : case IX86_BUILTIN_NANQ:
15599 0 : case IX86_BUILTIN_NANSQ:
15600 0 : return expand_call (exp, target, ignore);
15601 :
15602 18 : case IX86_BUILTIN_RDPID:
15603 :
15604 18 : op0 = gen_reg_rtx (word_mode);
15605 :
15606 18 : if (TARGET_64BIT)
15607 : {
15608 18 : insn = gen_rdpid_rex64 (op0);
15609 18 : op0 = convert_to_mode (SImode, op0, 1);
15610 : }
15611 : else
15612 0 : insn = gen_rdpid (op0);
15613 :
15614 18 : emit_insn (insn);
15615 :
15616 18 : if (target == 0
15617 18 : || !register_operand (target, SImode))
15618 0 : target = gen_reg_rtx (SImode);
15619 :
15620 18 : emit_move_insn (target, op0);
15621 18 : return target;
15622 :
15623 75 : case IX86_BUILTIN_2INTERSECTD512:
15624 75 : case IX86_BUILTIN_2INTERSECTQ512:
15625 75 : case IX86_BUILTIN_2INTERSECTD256:
15626 75 : case IX86_BUILTIN_2INTERSECTQ256:
15627 75 : case IX86_BUILTIN_2INTERSECTD128:
15628 75 : case IX86_BUILTIN_2INTERSECTQ128:
15629 75 : arg0 = CALL_EXPR_ARG (exp, 0);
15630 75 : arg1 = CALL_EXPR_ARG (exp, 1);
15631 75 : arg2 = CALL_EXPR_ARG (exp, 2);
15632 75 : arg3 = CALL_EXPR_ARG (exp, 3);
15633 75 : op0 = expand_normal (arg0);
15634 75 : op1 = expand_normal (arg1);
15635 75 : op2 = expand_normal (arg2);
15636 75 : op3 = expand_normal (arg3);
15637 :
15638 75 : if (!address_operand (op0, VOIDmode))
15639 : {
15640 25 : op0 = convert_memory_address (Pmode, op0);
15641 25 : op0 = copy_addr_to_reg (op0);
15642 : }
15643 75 : if (!address_operand (op1, VOIDmode))
15644 : {
15645 25 : op1 = convert_memory_address (Pmode, op1);
15646 25 : op1 = copy_addr_to_reg (op1);
15647 : }
15648 :
15649 75 : switch (fcode)
15650 : {
15651 : case IX86_BUILTIN_2INTERSECTD512:
15652 : mode4 = P2HImode;
15653 : icode = CODE_FOR_avx512vp2intersect_2intersectv16si;
15654 : break;
15655 : case IX86_BUILTIN_2INTERSECTQ512:
15656 : mode4 = P2QImode;
15657 : icode = CODE_FOR_avx512vp2intersect_2intersectv8di;
15658 : break;
15659 : case IX86_BUILTIN_2INTERSECTD256:
15660 : mode4 = P2QImode;
15661 : icode = CODE_FOR_avx512vp2intersect_2intersectv8si;
15662 : break;
15663 : case IX86_BUILTIN_2INTERSECTQ256:
15664 : mode4 = P2QImode;
15665 : icode = CODE_FOR_avx512vp2intersect_2intersectv4di;
15666 : break;
15667 : case IX86_BUILTIN_2INTERSECTD128:
15668 : mode4 = P2QImode;
15669 : icode = CODE_FOR_avx512vp2intersect_2intersectv4si;
15670 : break;
15671 : case IX86_BUILTIN_2INTERSECTQ128:
15672 : mode4 = P2QImode;
15673 : icode = CODE_FOR_avx512vp2intersect_2intersectv2di;
15674 : break;
15675 0 : default:
15676 0 : gcc_unreachable ();
15677 : }
15678 :
15679 75 : mode2 = insn_data[icode].operand[1].mode;
15680 75 : mode3 = insn_data[icode].operand[2].mode;
15681 75 : if (!insn_data[icode].operand[1].predicate (op2, mode2))
15682 25 : op2 = copy_to_mode_reg (mode2, op2);
15683 75 : if (!insn_data[icode].operand[2].predicate (op3, mode3))
15684 6 : op3 = copy_to_mode_reg (mode3, op3);
15685 :
15686 75 : op4 = gen_reg_rtx (mode4);
15687 75 : emit_insn (GEN_FCN (icode) (op4, op2, op3));
15688 75 : mode0 = mode4 == P2HImode ? HImode : QImode;
15689 75 : emit_move_insn (gen_rtx_MEM (mode0, op0),
15690 75 : gen_lowpart (mode0, op4));
15691 75 : emit_move_insn (gen_rtx_MEM (mode0, op1),
15692 : gen_highpart (mode0, op4));
15693 :
15694 75 : return 0;
15695 :
15696 102 : case IX86_BUILTIN_RDPMC:
15697 102 : case IX86_BUILTIN_RDTSC:
15698 102 : case IX86_BUILTIN_RDTSCP:
15699 102 : case IX86_BUILTIN_XGETBV:
15700 :
15701 102 : op0 = gen_reg_rtx (DImode);
15702 102 : op1 = gen_reg_rtx (DImode);
15703 :
15704 102 : if (fcode == IX86_BUILTIN_RDPMC)
15705 : {
15706 22 : arg0 = CALL_EXPR_ARG (exp, 0);
15707 22 : op2 = expand_normal (arg0);
15708 22 : if (!register_operand (op2, SImode))
15709 11 : op2 = copy_to_mode_reg (SImode, op2);
15710 :
15711 22 : insn = (TARGET_64BIT
15712 22 : ? gen_rdpmc_rex64 (op0, op1, op2)
15713 0 : : gen_rdpmc (op0, op2));
15714 22 : emit_insn (insn);
15715 : }
15716 80 : else if (fcode == IX86_BUILTIN_XGETBV)
15717 : {
15718 22 : arg0 = CALL_EXPR_ARG (exp, 0);
15719 22 : op2 = expand_normal (arg0);
15720 22 : if (!register_operand (op2, SImode))
15721 1 : op2 = copy_to_mode_reg (SImode, op2);
15722 :
15723 22 : insn = (TARGET_64BIT
15724 22 : ? gen_xgetbv_rex64 (op0, op1, op2)
15725 0 : : gen_xgetbv (op0, op2));
15726 22 : emit_insn (insn);
15727 : }
15728 58 : else if (fcode == IX86_BUILTIN_RDTSC)
15729 : {
15730 36 : insn = (TARGET_64BIT
15731 36 : ? gen_rdtsc_rex64 (op0, op1)
15732 2 : : gen_rdtsc (op0));
15733 36 : emit_insn (insn);
15734 : }
15735 : else
15736 : {
15737 22 : op2 = gen_reg_rtx (SImode);
15738 :
15739 22 : insn = (TARGET_64BIT
15740 22 : ? gen_rdtscp_rex64 (op0, op1, op2)
15741 0 : : gen_rdtscp (op0, op2));
15742 22 : emit_insn (insn);
15743 :
15744 22 : arg0 = CALL_EXPR_ARG (exp, 0);
15745 22 : op4 = expand_normal (arg0);
15746 22 : if (!address_operand (op4, VOIDmode))
15747 : {
15748 10 : op4 = convert_memory_address (Pmode, op4);
15749 10 : op4 = copy_addr_to_reg (op4);
15750 : }
15751 22 : emit_move_insn (gen_rtx_MEM (SImode, op4), op2);
15752 : }
15753 :
15754 102 : if (target == 0
15755 102 : || !register_operand (target, DImode))
15756 10 : target = gen_reg_rtx (DImode);
15757 :
15758 102 : if (TARGET_64BIT)
15759 : {
15760 100 : op1 = expand_simple_binop (DImode, ASHIFT, op1, GEN_INT (32),
15761 : op1, 1, OPTAB_DIRECT);
15762 100 : op0 = expand_simple_binop (DImode, IOR, op0, op1,
15763 : op0, 1, OPTAB_DIRECT);
15764 : }
15765 :
15766 102 : emit_move_insn (target, op0);
15767 102 : return target;
15768 :
15769 62 : case IX86_BUILTIN_ENQCMD:
15770 62 : case IX86_BUILTIN_ENQCMDS:
15771 62 : case IX86_BUILTIN_MOVDIR64B:
15772 :
15773 62 : arg0 = CALL_EXPR_ARG (exp, 0);
15774 62 : arg1 = CALL_EXPR_ARG (exp, 1);
15775 62 : op0 = expand_normal (arg0);
15776 62 : op1 = expand_normal (arg1);
15777 :
15778 62 : op0 = ix86_zero_extend_to_Pmode (op0);
15779 62 : if (!address_operand (op1, VOIDmode))
15780 : {
15781 29 : op1 = convert_memory_address (Pmode, op1);
15782 29 : op1 = copy_addr_to_reg (op1);
15783 : }
15784 62 : op1 = gen_rtx_MEM (XImode, op1);
15785 :
15786 62 : if (fcode == IX86_BUILTIN_MOVDIR64B)
15787 : {
15788 25 : emit_insn (gen_movdir64b (Pmode, op0, op1));
15789 24 : return 0;
15790 : }
15791 : else
15792 : {
15793 38 : if (target == 0
15794 38 : || !register_operand (target, SImode))
15795 0 : target = gen_reg_rtx (SImode);
15796 :
15797 38 : emit_move_insn (target, const0_rtx);
15798 38 : target = gen_rtx_SUBREG (QImode, target, 0);
15799 :
15800 19 : int unspecv = (fcode == IX86_BUILTIN_ENQCMD
15801 38 : ? UNSPECV_ENQCMD
15802 : : UNSPECV_ENQCMDS);
15803 38 : icode = code_for_enqcmd (unspecv, Pmode);
15804 38 : emit_insn (GEN_FCN (icode) (op0, op1));
15805 :
15806 38 : emit_insn
15807 38 : (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
15808 : gen_rtx_fmt_ee (EQ, QImode,
15809 : gen_rtx_REG (CCZmode, FLAGS_REG),
15810 : const0_rtx)));
15811 38 : return SUBREG_REG (target);
15812 : }
15813 :
15814 14703 : case IX86_BUILTIN_FXSAVE:
15815 14703 : case IX86_BUILTIN_FXRSTOR:
15816 14703 : case IX86_BUILTIN_FXSAVE64:
15817 14703 : case IX86_BUILTIN_FXRSTOR64:
15818 14703 : case IX86_BUILTIN_FNSTENV:
15819 14703 : case IX86_BUILTIN_FLDENV:
15820 14703 : mode0 = BLKmode;
15821 14703 : switch (fcode)
15822 : {
15823 : case IX86_BUILTIN_FXSAVE:
15824 : icode = CODE_FOR_fxsave;
15825 : break;
15826 19 : case IX86_BUILTIN_FXRSTOR:
15827 19 : icode = CODE_FOR_fxrstor;
15828 19 : break;
15829 23 : case IX86_BUILTIN_FXSAVE64:
15830 23 : icode = CODE_FOR_fxsave64;
15831 23 : break;
15832 21 : case IX86_BUILTIN_FXRSTOR64:
15833 21 : icode = CODE_FOR_fxrstor64;
15834 21 : break;
15835 7258 : case IX86_BUILTIN_FNSTENV:
15836 7258 : icode = CODE_FOR_fnstenv;
15837 7258 : break;
15838 7362 : case IX86_BUILTIN_FLDENV:
15839 7362 : icode = CODE_FOR_fldenv;
15840 7362 : break;
15841 0 : default:
15842 0 : gcc_unreachable ();
15843 : }
15844 :
15845 14703 : arg0 = CALL_EXPR_ARG (exp, 0);
15846 14703 : op0 = expand_normal (arg0);
15847 :
15848 14703 : if (!address_operand (op0, VOIDmode))
15849 : {
15850 36 : op0 = convert_memory_address (Pmode, op0);
15851 36 : op0 = copy_addr_to_reg (op0);
15852 : }
15853 14703 : op0 = gen_rtx_MEM (mode0, op0);
15854 :
15855 14703 : pat = GEN_FCN (icode) (op0);
15856 14703 : if (pat)
15857 14703 : emit_insn (pat);
15858 : return 0;
15859 :
15860 21 : case IX86_BUILTIN_XSETBV:
15861 21 : arg0 = CALL_EXPR_ARG (exp, 0);
15862 21 : arg1 = CALL_EXPR_ARG (exp, 1);
15863 21 : op0 = expand_normal (arg0);
15864 21 : op1 = expand_normal (arg1);
15865 :
15866 21 : if (!REG_P (op0))
15867 1 : op0 = copy_to_mode_reg (SImode, op0);
15868 :
15869 21 : op1 = force_reg (DImode, op1);
15870 :
15871 21 : if (TARGET_64BIT)
15872 : {
15873 21 : op2 = expand_simple_binop (DImode, LSHIFTRT, op1, GEN_INT (32),
15874 : NULL, 1, OPTAB_DIRECT);
15875 :
15876 21 : icode = CODE_FOR_xsetbv_rex64;
15877 :
15878 21 : op2 = gen_lowpart (SImode, op2);
15879 21 : op1 = gen_lowpart (SImode, op1);
15880 21 : pat = GEN_FCN (icode) (op0, op1, op2);
15881 : }
15882 : else
15883 : {
15884 0 : icode = CODE_FOR_xsetbv;
15885 :
15886 0 : pat = GEN_FCN (icode) (op0, op1);
15887 : }
15888 21 : if (pat)
15889 21 : emit_insn (pat);
15890 : return 0;
15891 :
15892 232 : case IX86_BUILTIN_XSAVE:
15893 232 : case IX86_BUILTIN_XRSTOR:
15894 232 : case IX86_BUILTIN_XSAVE64:
15895 232 : case IX86_BUILTIN_XRSTOR64:
15896 232 : case IX86_BUILTIN_XSAVEOPT:
15897 232 : case IX86_BUILTIN_XSAVEOPT64:
15898 232 : case IX86_BUILTIN_XSAVES:
15899 232 : case IX86_BUILTIN_XRSTORS:
15900 232 : case IX86_BUILTIN_XSAVES64:
15901 232 : case IX86_BUILTIN_XRSTORS64:
15902 232 : case IX86_BUILTIN_XSAVEC:
15903 232 : case IX86_BUILTIN_XSAVEC64:
15904 232 : arg0 = CALL_EXPR_ARG (exp, 0);
15905 232 : arg1 = CALL_EXPR_ARG (exp, 1);
15906 232 : op0 = expand_normal (arg0);
15907 232 : op1 = expand_normal (arg1);
15908 :
15909 232 : if (!address_operand (op0, VOIDmode))
15910 : {
15911 108 : op0 = convert_memory_address (Pmode, op0);
15912 108 : op0 = copy_addr_to_reg (op0);
15913 : }
15914 232 : op0 = gen_rtx_MEM (BLKmode, op0);
15915 :
15916 232 : op1 = force_reg (DImode, op1);
15917 :
15918 232 : if (TARGET_64BIT)
15919 : {
15920 232 : op2 = expand_simple_binop (DImode, LSHIFTRT, op1, GEN_INT (32),
15921 : NULL, 1, OPTAB_DIRECT);
15922 232 : switch (fcode)
15923 : {
15924 : case IX86_BUILTIN_XSAVE:
15925 : icode = CODE_FOR_xsave_rex64;
15926 : break;
15927 19 : case IX86_BUILTIN_XRSTOR:
15928 19 : icode = CODE_FOR_xrstor_rex64;
15929 19 : break;
15930 21 : case IX86_BUILTIN_XSAVE64:
15931 21 : icode = CODE_FOR_xsave64;
15932 21 : break;
15933 21 : case IX86_BUILTIN_XRSTOR64:
15934 21 : icode = CODE_FOR_xrstor64;
15935 21 : break;
15936 19 : case IX86_BUILTIN_XSAVEOPT:
15937 19 : icode = CODE_FOR_xsaveopt_rex64;
15938 19 : break;
15939 19 : case IX86_BUILTIN_XSAVEOPT64:
15940 19 : icode = CODE_FOR_xsaveopt64;
15941 19 : break;
15942 19 : case IX86_BUILTIN_XSAVES:
15943 19 : icode = CODE_FOR_xsaves_rex64;
15944 19 : break;
15945 19 : case IX86_BUILTIN_XRSTORS:
15946 19 : icode = CODE_FOR_xrstors_rex64;
15947 19 : break;
15948 19 : case IX86_BUILTIN_XSAVES64:
15949 19 : icode = CODE_FOR_xsaves64;
15950 19 : break;
15951 19 : case IX86_BUILTIN_XRSTORS64:
15952 19 : icode = CODE_FOR_xrstors64;
15953 19 : break;
15954 19 : case IX86_BUILTIN_XSAVEC:
15955 19 : icode = CODE_FOR_xsavec_rex64;
15956 19 : break;
15957 19 : case IX86_BUILTIN_XSAVEC64:
15958 19 : icode = CODE_FOR_xsavec64;
15959 19 : break;
15960 0 : default:
15961 0 : gcc_unreachable ();
15962 : }
15963 :
15964 232 : op2 = gen_lowpart (SImode, op2);
15965 232 : op1 = gen_lowpart (SImode, op1);
15966 232 : pat = GEN_FCN (icode) (op0, op1, op2);
15967 : }
15968 : else
15969 : {
15970 0 : switch (fcode)
15971 : {
15972 : case IX86_BUILTIN_XSAVE:
15973 : icode = CODE_FOR_xsave;
15974 : break;
15975 : case IX86_BUILTIN_XRSTOR:
15976 : icode = CODE_FOR_xrstor;
15977 : break;
15978 : case IX86_BUILTIN_XSAVEOPT:
15979 : icode = CODE_FOR_xsaveopt;
15980 : break;
15981 : case IX86_BUILTIN_XSAVES:
15982 : icode = CODE_FOR_xsaves;
15983 : break;
15984 : case IX86_BUILTIN_XRSTORS:
15985 : icode = CODE_FOR_xrstors;
15986 : break;
15987 : case IX86_BUILTIN_XSAVEC:
15988 : icode = CODE_FOR_xsavec;
15989 : break;
15990 0 : default:
15991 0 : gcc_unreachable ();
15992 : }
15993 0 : pat = GEN_FCN (icode) (op0, op1);
15994 : }
15995 :
15996 232 : if (pat)
15997 232 : emit_insn (pat);
15998 : return 0;
15999 :
16000 137 : case IX86_BUILTIN_LDTILECFG:
16001 137 : case IX86_BUILTIN_STTILECFG:
16002 137 : arg0 = CALL_EXPR_ARG (exp, 0);
16003 137 : op0 = expand_normal (arg0);
16004 :
16005 137 : if (!address_operand (op0, VOIDmode))
16006 : {
16007 8 : op0 = convert_memory_address (Pmode, op0);
16008 8 : op0 = copy_addr_to_reg (op0);
16009 : }
16010 137 : op0 = gen_rtx_MEM (BLKmode, op0);
16011 137 : if (fcode == IX86_BUILTIN_LDTILECFG)
16012 : icode = CODE_FOR_ldtilecfg;
16013 : else
16014 88 : icode = CODE_FOR_sttilecfg;
16015 137 : pat = GEN_FCN (icode) (op0);
16016 137 : emit_insn (pat);
16017 137 : return 0;
16018 :
16019 18 : case IX86_BUILTIN_LLWPCB:
16020 18 : arg0 = CALL_EXPR_ARG (exp, 0);
16021 18 : op0 = expand_normal (arg0);
16022 :
16023 18 : if (!register_operand (op0, Pmode))
16024 9 : op0 = ix86_zero_extend_to_Pmode (op0);
16025 18 : emit_insn (gen_lwp_llwpcb (Pmode, op0));
16026 18 : return 0;
16027 :
16028 18 : case IX86_BUILTIN_SLWPCB:
16029 18 : if (!target
16030 18 : || !register_operand (target, Pmode))
16031 0 : target = gen_reg_rtx (Pmode);
16032 18 : emit_insn (gen_lwp_slwpcb (Pmode, target));
16033 18 : return target;
16034 :
16035 51 : case IX86_BUILTIN_LWPVAL32:
16036 51 : case IX86_BUILTIN_LWPVAL64:
16037 51 : case IX86_BUILTIN_LWPINS32:
16038 51 : case IX86_BUILTIN_LWPINS64:
16039 51 : mode = ((fcode == IX86_BUILTIN_LWPVAL32
16040 51 : || fcode == IX86_BUILTIN_LWPINS32)
16041 51 : ? SImode : DImode);
16042 :
16043 51 : if (fcode == IX86_BUILTIN_LWPVAL32
16044 51 : || fcode == IX86_BUILTIN_LWPVAL64)
16045 26 : icode = code_for_lwp_lwpval (mode);
16046 : else
16047 25 : icode = code_for_lwp_lwpins (mode);
16048 :
16049 51 : arg0 = CALL_EXPR_ARG (exp, 0);
16050 51 : arg1 = CALL_EXPR_ARG (exp, 1);
16051 51 : arg2 = CALL_EXPR_ARG (exp, 2);
16052 51 : op0 = expand_normal (arg0);
16053 51 : op1 = expand_normal (arg1);
16054 51 : op2 = expand_normal (arg2);
16055 51 : mode0 = insn_data[icode].operand[0].mode;
16056 :
16057 51 : if (!insn_data[icode].operand[0].predicate (op0, mode0))
16058 13 : op0 = copy_to_mode_reg (mode0, op0);
16059 51 : if (!insn_data[icode].operand[1].predicate (op1, SImode))
16060 0 : op1 = copy_to_mode_reg (SImode, op1);
16061 :
16062 51 : if (!CONST_INT_P (op2))
16063 : {
16064 0 : error ("the last argument must be a 32-bit immediate");
16065 0 : return const0_rtx;
16066 : }
16067 :
16068 51 : emit_insn (GEN_FCN (icode) (op0, op1, op2));
16069 :
16070 51 : if (fcode == IX86_BUILTIN_LWPINS32
16071 51 : || fcode == IX86_BUILTIN_LWPINS64)
16072 : {
16073 25 : if (target == 0
16074 25 : || !nonimmediate_operand (target, QImode))
16075 0 : target = gen_reg_rtx (QImode);
16076 :
16077 25 : pat = gen_rtx_EQ (QImode, gen_rtx_REG (CCCmode, FLAGS_REG),
16078 : const0_rtx);
16079 25 : emit_insn (gen_rtx_SET (target, pat));
16080 :
16081 25 : return target;
16082 : }
16083 : else
16084 : return 0;
16085 :
16086 18 : case IX86_BUILTIN_BEXTRI32:
16087 18 : case IX86_BUILTIN_BEXTRI64:
16088 18 : mode = (fcode == IX86_BUILTIN_BEXTRI32 ? SImode : DImode);
16089 :
16090 18 : arg0 = CALL_EXPR_ARG (exp, 0);
16091 18 : arg1 = CALL_EXPR_ARG (exp, 1);
16092 18 : op0 = expand_normal (arg0);
16093 18 : op1 = expand_normal (arg1);
16094 :
16095 18 : if (!CONST_INT_P (op1))
16096 : {
16097 0 : error ("last argument must be an immediate");
16098 0 : return const0_rtx;
16099 : }
16100 : else
16101 : {
16102 18 : unsigned char lsb_index = UINTVAL (op1);
16103 18 : unsigned char length = UINTVAL (op1) >> 8;
16104 :
16105 18 : unsigned char bitsize = GET_MODE_BITSIZE (mode);
16106 :
16107 18 : icode = code_for_tbm_bextri (mode);
16108 :
16109 18 : mode1 = insn_data[icode].operand[1].mode;
16110 18 : if (!insn_data[icode].operand[1].predicate (op0, mode1))
16111 12 : op0 = copy_to_mode_reg (mode1, op0);
16112 :
16113 18 : mode0 = insn_data[icode].operand[0].mode;
16114 18 : if (target == 0
16115 18 : || !register_operand (target, mode0))
16116 0 : target = gen_reg_rtx (mode0);
16117 :
16118 18 : if (length == 0 || lsb_index >= bitsize)
16119 : {
16120 8 : emit_move_insn (target, const0_rtx);
16121 8 : return target;
16122 : }
16123 :
16124 10 : if (length + lsb_index > bitsize)
16125 5 : length = bitsize - lsb_index;
16126 :
16127 10 : op1 = GEN_INT (length);
16128 10 : op2 = GEN_INT (lsb_index);
16129 :
16130 10 : emit_insn (GEN_FCN (icode) (target, op0, op1, op2));
16131 10 : return target;
16132 : }
16133 :
16134 21 : case IX86_BUILTIN_RDRAND16_STEP:
16135 21 : mode = HImode;
16136 21 : goto rdrand_step;
16137 :
16138 42 : case IX86_BUILTIN_RDRAND32_STEP:
16139 42 : mode = SImode;
16140 42 : goto rdrand_step;
16141 :
16142 : case IX86_BUILTIN_RDRAND64_STEP:
16143 : mode = DImode;
16144 :
16145 83 : rdrand_step:
16146 83 : arg0 = CALL_EXPR_ARG (exp, 0);
16147 83 : op1 = expand_normal (arg0);
16148 83 : if (!address_operand (op1, VOIDmode))
16149 : {
16150 29 : op1 = convert_memory_address (Pmode, op1);
16151 29 : op1 = copy_addr_to_reg (op1);
16152 : }
16153 :
16154 83 : op0 = gen_reg_rtx (mode);
16155 83 : emit_insn (gen_rdrand (mode, op0));
16156 :
16157 83 : emit_move_insn (gen_rtx_MEM (mode, op1), op0);
16158 :
16159 83 : op1 = force_reg (SImode, const1_rtx);
16160 :
16161 : /* Emit SImode conditional move. */
16162 83 : if (mode == HImode)
16163 : {
16164 21 : if (TARGET_ZERO_EXTEND_WITH_AND
16165 21 : && optimize_function_for_speed_p (cfun))
16166 : {
16167 0 : op2 = force_reg (SImode, const0_rtx);
16168 :
16169 0 : emit_insn (gen_movstricthi
16170 0 : (gen_lowpart (HImode, op2), op0));
16171 : }
16172 : else
16173 : {
16174 21 : op2 = gen_reg_rtx (SImode);
16175 :
16176 21 : emit_insn (gen_zero_extendhisi2 (op2, op0));
16177 : }
16178 : }
16179 62 : else if (mode == SImode)
16180 : op2 = op0;
16181 : else
16182 20 : op2 = gen_rtx_SUBREG (SImode, op0, 0);
16183 :
16184 83 : if (target == 0
16185 83 : || !register_operand (target, SImode))
16186 7 : target = gen_reg_rtx (SImode);
16187 :
16188 83 : pat = gen_rtx_GEU (VOIDmode, gen_rtx_REG (CCCmode, FLAGS_REG),
16189 : const0_rtx);
16190 83 : emit_insn (gen_rtx_SET (target,
16191 : gen_rtx_IF_THEN_ELSE (SImode, pat, op2, op1)));
16192 83 : return target;
16193 :
16194 19 : case IX86_BUILTIN_RDSEED16_STEP:
16195 19 : mode = HImode;
16196 19 : goto rdseed_step;
16197 :
16198 28 : case IX86_BUILTIN_RDSEED32_STEP:
16199 28 : mode = SImode;
16200 28 : goto rdseed_step;
16201 :
16202 : case IX86_BUILTIN_RDSEED64_STEP:
16203 : mode = DImode;
16204 :
16205 66 : rdseed_step:
16206 66 : arg0 = CALL_EXPR_ARG (exp, 0);
16207 66 : op1 = expand_normal (arg0);
16208 66 : if (!address_operand (op1, VOIDmode))
16209 : {
16210 28 : op1 = convert_memory_address (Pmode, op1);
16211 28 : op1 = copy_addr_to_reg (op1);
16212 : }
16213 :
16214 66 : op0 = gen_reg_rtx (mode);
16215 66 : emit_insn (gen_rdseed (mode, op0));
16216 :
16217 66 : emit_move_insn (gen_rtx_MEM (mode, op1), op0);
16218 :
16219 66 : op2 = gen_reg_rtx (QImode);
16220 :
16221 66 : pat = gen_rtx_LTU (QImode, gen_rtx_REG (CCCmode, FLAGS_REG),
16222 : const0_rtx);
16223 66 : emit_insn (gen_rtx_SET (op2, pat));
16224 :
16225 66 : if (target == 0
16226 66 : || !register_operand (target, SImode))
16227 1 : target = gen_reg_rtx (SImode);
16228 :
16229 66 : emit_insn (gen_zero_extendqisi2 (target, op2));
16230 66 : return target;
16231 :
16232 38 : case IX86_BUILTIN_SBB32:
16233 38 : icode = CODE_FOR_subborrowsi;
16234 38 : icode2 = CODE_FOR_subborrowsi_0;
16235 38 : mode0 = SImode;
16236 38 : mode1 = DImode;
16237 38 : mode2 = CCmode;
16238 38 : goto handlecarry;
16239 :
16240 44 : case IX86_BUILTIN_SBB64:
16241 44 : icode = CODE_FOR_subborrowdi;
16242 44 : icode2 = CODE_FOR_subborrowdi_0;
16243 44 : mode0 = DImode;
16244 44 : mode1 = TImode;
16245 44 : mode2 = CCmode;
16246 44 : goto handlecarry;
16247 :
16248 68 : case IX86_BUILTIN_ADDCARRYX32:
16249 68 : icode = CODE_FOR_addcarrysi;
16250 68 : icode2 = CODE_FOR_addcarrysi_0;
16251 68 : mode0 = SImode;
16252 68 : mode1 = DImode;
16253 68 : mode2 = CCCmode;
16254 68 : goto handlecarry;
16255 :
16256 : case IX86_BUILTIN_ADDCARRYX64:
16257 : icode = CODE_FOR_addcarrydi;
16258 : icode2 = CODE_FOR_addcarrydi_0;
16259 : mode0 = DImode;
16260 : mode1 = TImode;
16261 : mode2 = CCCmode;
16262 :
16263 212 : handlecarry:
16264 212 : arg0 = CALL_EXPR_ARG (exp, 0); /* unsigned char c_in. */
16265 212 : arg1 = CALL_EXPR_ARG (exp, 1); /* unsigned int src1. */
16266 212 : arg2 = CALL_EXPR_ARG (exp, 2); /* unsigned int src2. */
16267 212 : arg3 = CALL_EXPR_ARG (exp, 3); /* unsigned int *sum_out. */
16268 :
16269 212 : op1 = expand_normal (arg0);
16270 :
16271 212 : op2 = expand_normal (arg1);
16272 212 : if (!register_operand (op2, mode0))
16273 117 : op2 = copy_to_mode_reg (mode0, op2);
16274 :
16275 212 : op3 = expand_normal (arg2);
16276 212 : if (!register_operand (op3, mode0))
16277 120 : op3 = copy_to_mode_reg (mode0, op3);
16278 :
16279 212 : op4 = expand_normal (arg3);
16280 212 : if (!address_operand (op4, VOIDmode))
16281 : {
16282 67 : op4 = convert_memory_address (Pmode, op4);
16283 67 : op4 = copy_addr_to_reg (op4);
16284 : }
16285 :
16286 212 : op0 = gen_reg_rtx (mode0);
16287 212 : if (op1 == const0_rtx)
16288 : {
16289 : /* If arg0 is 0, optimize right away into add or sub
16290 : instruction that sets CCCmode flags. */
16291 21 : op1 = gen_rtx_REG (mode2, FLAGS_REG);
16292 21 : emit_insn (GEN_FCN (icode2) (op0, op2, op3));
16293 : }
16294 : else
16295 : {
16296 : /* Generate CF from input operand. */
16297 191 : ix86_expand_carry (op1);
16298 :
16299 : /* Generate instruction that consumes CF. */
16300 191 : op1 = gen_rtx_REG (CCCmode, FLAGS_REG);
16301 191 : pat = gen_rtx_LTU (mode1, op1, const0_rtx);
16302 191 : pat2 = gen_rtx_LTU (mode0, op1, const0_rtx);
16303 191 : emit_insn (GEN_FCN (icode) (op0, op2, op3, op1, pat, pat2));
16304 : }
16305 :
16306 : /* Return current CF value. */
16307 212 : if (target == 0)
16308 14 : target = gen_reg_rtx (QImode);
16309 :
16310 212 : pat = gen_rtx_LTU (QImode, op1, const0_rtx);
16311 212 : emit_insn (gen_rtx_SET (target, pat));
16312 :
16313 : /* Store the result. */
16314 212 : emit_move_insn (gen_rtx_MEM (mode0, op4), op0);
16315 :
16316 212 : return target;
16317 :
16318 24 : case IX86_BUILTIN_READ_FLAGS:
16319 24 : if (ignore)
16320 3 : return const0_rtx;
16321 :
16322 21 : emit_insn (gen_pushfl ());
16323 :
16324 21 : if (optimize
16325 11 : || target == NULL_RTX
16326 11 : || !nonimmediate_operand (target, word_mode)
16327 32 : || GET_MODE (target) != word_mode)
16328 10 : target = gen_reg_rtx (word_mode);
16329 :
16330 21 : emit_insn (gen_pop (target));
16331 21 : return target;
16332 :
16333 21 : case IX86_BUILTIN_WRITE_FLAGS:
16334 :
16335 21 : arg0 = CALL_EXPR_ARG (exp, 0);
16336 21 : op0 = expand_normal (arg0);
16337 21 : if (!general_no_elim_operand (op0, word_mode))
16338 0 : op0 = copy_to_mode_reg (word_mode, op0);
16339 :
16340 21 : emit_insn (gen_push (op0));
16341 21 : emit_insn (gen_popfl ());
16342 21 : return 0;
16343 :
16344 22 : case IX86_BUILTIN_KTESTC8:
16345 22 : icode = CODE_FOR_ktestqi;
16346 22 : mode3 = CCCmode;
16347 22 : goto kortest;
16348 :
16349 22 : case IX86_BUILTIN_KTESTZ8:
16350 22 : icode = CODE_FOR_ktestqi;
16351 22 : mode3 = CCZmode;
16352 22 : goto kortest;
16353 :
16354 22 : case IX86_BUILTIN_KTESTC16:
16355 22 : icode = CODE_FOR_ktesthi;
16356 22 : mode3 = CCCmode;
16357 22 : goto kortest;
16358 :
16359 22 : case IX86_BUILTIN_KTESTZ16:
16360 22 : icode = CODE_FOR_ktesthi;
16361 22 : mode3 = CCZmode;
16362 22 : goto kortest;
16363 :
16364 22 : case IX86_BUILTIN_KTESTC32:
16365 22 : icode = CODE_FOR_ktestsi;
16366 22 : mode3 = CCCmode;
16367 22 : goto kortest;
16368 :
16369 22 : case IX86_BUILTIN_KTESTZ32:
16370 22 : icode = CODE_FOR_ktestsi;
16371 22 : mode3 = CCZmode;
16372 22 : goto kortest;
16373 :
16374 22 : case IX86_BUILTIN_KTESTC64:
16375 22 : icode = CODE_FOR_ktestdi;
16376 22 : mode3 = CCCmode;
16377 22 : goto kortest;
16378 :
16379 22 : case IX86_BUILTIN_KTESTZ64:
16380 22 : icode = CODE_FOR_ktestdi;
16381 22 : mode3 = CCZmode;
16382 22 : goto kortest;
16383 :
16384 22 : case IX86_BUILTIN_KORTESTC8:
16385 22 : icode = CODE_FOR_kortestqi;
16386 22 : mode3 = CCCmode;
16387 22 : goto kortest;
16388 :
16389 76 : case IX86_BUILTIN_KORTESTZ8:
16390 76 : icode = CODE_FOR_kortestqi;
16391 76 : mode3 = CCZmode;
16392 76 : goto kortest;
16393 :
16394 38 : case IX86_BUILTIN_KORTESTC16:
16395 38 : icode = CODE_FOR_kortesthi;
16396 38 : mode3 = CCCmode;
16397 38 : goto kortest;
16398 :
16399 91 : case IX86_BUILTIN_KORTESTZ16:
16400 91 : icode = CODE_FOR_kortesthi;
16401 91 : mode3 = CCZmode;
16402 91 : goto kortest;
16403 :
16404 22 : case IX86_BUILTIN_KORTESTC32:
16405 22 : icode = CODE_FOR_kortestsi;
16406 22 : mode3 = CCCmode;
16407 22 : goto kortest;
16408 :
16409 79 : case IX86_BUILTIN_KORTESTZ32:
16410 79 : icode = CODE_FOR_kortestsi;
16411 79 : mode3 = CCZmode;
16412 79 : goto kortest;
16413 :
16414 22 : case IX86_BUILTIN_KORTESTC64:
16415 22 : icode = CODE_FOR_kortestdi;
16416 22 : mode3 = CCCmode;
16417 22 : goto kortest;
16418 :
16419 : case IX86_BUILTIN_KORTESTZ64:
16420 : icode = CODE_FOR_kortestdi;
16421 : mode3 = CCZmode;
16422 :
16423 610 : kortest:
16424 610 : arg0 = CALL_EXPR_ARG (exp, 0); /* Mask reg src1. */
16425 610 : arg1 = CALL_EXPR_ARG (exp, 1); /* Mask reg src2. */
16426 610 : op0 = expand_normal (arg0);
16427 610 : op1 = expand_normal (arg1);
16428 :
16429 610 : mode0 = insn_data[icode].operand[0].mode;
16430 610 : mode1 = insn_data[icode].operand[1].mode;
16431 :
16432 610 : if (GET_MODE (op0) != VOIDmode)
16433 610 : op0 = force_reg (GET_MODE (op0), op0);
16434 :
16435 610 : op0 = gen_lowpart (mode0, op0);
16436 :
16437 610 : if (!insn_data[icode].operand[0].predicate (op0, mode0))
16438 0 : op0 = copy_to_mode_reg (mode0, op0);
16439 :
16440 610 : if (GET_MODE (op1) != VOIDmode)
16441 609 : op1 = force_reg (GET_MODE (op1), op1);
16442 :
16443 610 : op1 = gen_lowpart (mode1, op1);
16444 :
16445 610 : if (!insn_data[icode].operand[1].predicate (op1, mode1))
16446 1 : op1 = copy_to_mode_reg (mode1, op1);
16447 :
16448 610 : target = gen_reg_rtx (QImode);
16449 :
16450 : /* Emit kortest. */
16451 610 : emit_insn (GEN_FCN (icode) (op0, op1));
16452 : /* And use setcc to return result from flags. */
16453 610 : ix86_expand_setcc (target, EQ,
16454 : gen_rtx_REG (mode3, FLAGS_REG), const0_rtx);
16455 610 : return target;
16456 :
16457 24 : case IX86_BUILTIN_GATHERSIV2DF:
16458 24 : icode = CODE_FOR_avx2_gathersiv2df;
16459 24 : goto gather_gen;
16460 18 : case IX86_BUILTIN_GATHERSIV4DF:
16461 18 : icode = CODE_FOR_avx2_gathersiv4df;
16462 18 : goto gather_gen;
16463 21 : case IX86_BUILTIN_GATHERDIV2DF:
16464 21 : icode = CODE_FOR_avx2_gatherdiv2df;
16465 21 : goto gather_gen;
16466 32 : case IX86_BUILTIN_GATHERDIV4DF:
16467 32 : icode = CODE_FOR_avx2_gatherdiv4df;
16468 32 : goto gather_gen;
16469 30 : case IX86_BUILTIN_GATHERSIV4SF:
16470 30 : icode = CODE_FOR_avx2_gathersiv4sf;
16471 30 : goto gather_gen;
16472 37 : case IX86_BUILTIN_GATHERSIV8SF:
16473 37 : icode = CODE_FOR_avx2_gathersiv8sf;
16474 37 : goto gather_gen;
16475 24 : case IX86_BUILTIN_GATHERDIV4SF:
16476 24 : icode = CODE_FOR_avx2_gatherdiv4sf;
16477 24 : goto gather_gen;
16478 18 : case IX86_BUILTIN_GATHERDIV8SF:
16479 18 : icode = CODE_FOR_avx2_gatherdiv8sf;
16480 18 : goto gather_gen;
16481 18 : case IX86_BUILTIN_GATHERSIV2DI:
16482 18 : icode = CODE_FOR_avx2_gathersiv2di;
16483 18 : goto gather_gen;
16484 18 : case IX86_BUILTIN_GATHERSIV4DI:
16485 18 : icode = CODE_FOR_avx2_gathersiv4di;
16486 18 : goto gather_gen;
16487 27 : case IX86_BUILTIN_GATHERDIV2DI:
16488 27 : icode = CODE_FOR_avx2_gatherdiv2di;
16489 27 : goto gather_gen;
16490 29 : case IX86_BUILTIN_GATHERDIV4DI:
16491 29 : icode = CODE_FOR_avx2_gatherdiv4di;
16492 29 : goto gather_gen;
16493 20 : case IX86_BUILTIN_GATHERSIV4SI:
16494 20 : icode = CODE_FOR_avx2_gathersiv4si;
16495 20 : goto gather_gen;
16496 22 : case IX86_BUILTIN_GATHERSIV8SI:
16497 22 : icode = CODE_FOR_avx2_gathersiv8si;
16498 22 : goto gather_gen;
16499 28 : case IX86_BUILTIN_GATHERDIV4SI:
16500 28 : icode = CODE_FOR_avx2_gatherdiv4si;
16501 28 : goto gather_gen;
16502 18 : case IX86_BUILTIN_GATHERDIV8SI:
16503 18 : icode = CODE_FOR_avx2_gatherdiv8si;
16504 18 : goto gather_gen;
16505 20 : case IX86_BUILTIN_GATHERALTSIV4DF:
16506 20 : icode = CODE_FOR_avx2_gathersiv4df;
16507 20 : goto gather_gen;
16508 16 : case IX86_BUILTIN_GATHERALTDIV8SF:
16509 16 : icode = CODE_FOR_avx2_gatherdiv8sf;
16510 16 : goto gather_gen;
16511 4 : case IX86_BUILTIN_GATHERALTSIV4DI:
16512 4 : icode = CODE_FOR_avx2_gathersiv4di;
16513 4 : goto gather_gen;
16514 12 : case IX86_BUILTIN_GATHERALTDIV8SI:
16515 12 : icode = CODE_FOR_avx2_gatherdiv8si;
16516 12 : goto gather_gen;
16517 36 : case IX86_BUILTIN_GATHER3SIV16SF:
16518 36 : icode = CODE_FOR_avx512f_gathersiv16sf;
16519 36 : goto gather_gen;
16520 24 : case IX86_BUILTIN_GATHER3SIV8DF:
16521 24 : icode = CODE_FOR_avx512f_gathersiv8df;
16522 24 : goto gather_gen;
16523 24 : case IX86_BUILTIN_GATHER3DIV16SF:
16524 24 : icode = CODE_FOR_avx512f_gatherdiv16sf;
16525 24 : goto gather_gen;
16526 37 : case IX86_BUILTIN_GATHER3DIV8DF:
16527 37 : icode = CODE_FOR_avx512f_gatherdiv8df;
16528 37 : goto gather_gen;
16529 30 : case IX86_BUILTIN_GATHER3SIV16SI:
16530 30 : icode = CODE_FOR_avx512f_gathersiv16si;
16531 30 : goto gather_gen;
16532 24 : case IX86_BUILTIN_GATHER3SIV8DI:
16533 24 : icode = CODE_FOR_avx512f_gathersiv8di;
16534 24 : goto gather_gen;
16535 24 : case IX86_BUILTIN_GATHER3DIV16SI:
16536 24 : icode = CODE_FOR_avx512f_gatherdiv16si;
16537 24 : goto gather_gen;
16538 38 : case IX86_BUILTIN_GATHER3DIV8DI:
16539 38 : icode = CODE_FOR_avx512f_gatherdiv8di;
16540 38 : goto gather_gen;
16541 16 : case IX86_BUILTIN_GATHER3ALTSIV8DF:
16542 16 : icode = CODE_FOR_avx512f_gathersiv8df;
16543 16 : goto gather_gen;
16544 22 : case IX86_BUILTIN_GATHER3ALTDIV16SF:
16545 22 : icode = CODE_FOR_avx512f_gatherdiv16sf;
16546 22 : goto gather_gen;
16547 14 : case IX86_BUILTIN_GATHER3ALTSIV8DI:
16548 14 : icode = CODE_FOR_avx512f_gathersiv8di;
16549 14 : goto gather_gen;
16550 18 : case IX86_BUILTIN_GATHER3ALTDIV16SI:
16551 18 : icode = CODE_FOR_avx512f_gatherdiv16si;
16552 18 : goto gather_gen;
16553 18 : case IX86_BUILTIN_GATHER3SIV2DF:
16554 18 : icode = CODE_FOR_avx512vl_gathersiv2df;
16555 18 : goto gather_gen;
16556 10 : case IX86_BUILTIN_GATHER3SIV4DF:
16557 10 : icode = CODE_FOR_avx512vl_gathersiv4df;
16558 10 : goto gather_gen;
16559 15 : case IX86_BUILTIN_GATHER3DIV2DF:
16560 15 : icode = CODE_FOR_avx512vl_gatherdiv2df;
16561 15 : goto gather_gen;
16562 16 : case IX86_BUILTIN_GATHER3DIV4DF:
16563 16 : icode = CODE_FOR_avx512vl_gatherdiv4df;
16564 16 : goto gather_gen;
16565 14 : case IX86_BUILTIN_GATHER3SIV4SF:
16566 14 : icode = CODE_FOR_avx512vl_gathersiv4sf;
16567 14 : goto gather_gen;
16568 12 : case IX86_BUILTIN_GATHER3SIV8SF:
16569 12 : icode = CODE_FOR_avx512vl_gathersiv8sf;
16570 12 : goto gather_gen;
16571 22 : case IX86_BUILTIN_GATHER3DIV4SF:
16572 22 : icode = CODE_FOR_avx512vl_gatherdiv4sf;
16573 22 : goto gather_gen;
16574 10 : case IX86_BUILTIN_GATHER3DIV8SF:
16575 10 : icode = CODE_FOR_avx512vl_gatherdiv8sf;
16576 10 : goto gather_gen;
16577 20 : case IX86_BUILTIN_GATHER3SIV2DI:
16578 20 : icode = CODE_FOR_avx512vl_gathersiv2di;
16579 20 : goto gather_gen;
16580 10 : case IX86_BUILTIN_GATHER3SIV4DI:
16581 10 : icode = CODE_FOR_avx512vl_gathersiv4di;
16582 10 : goto gather_gen;
16583 15 : case IX86_BUILTIN_GATHER3DIV2DI:
16584 15 : icode = CODE_FOR_avx512vl_gatherdiv2di;
16585 15 : goto gather_gen;
16586 14 : case IX86_BUILTIN_GATHER3DIV4DI:
16587 14 : icode = CODE_FOR_avx512vl_gatherdiv4di;
16588 14 : goto gather_gen;
16589 14 : case IX86_BUILTIN_GATHER3SIV4SI:
16590 14 : icode = CODE_FOR_avx512vl_gathersiv4si;
16591 14 : goto gather_gen;
16592 12 : case IX86_BUILTIN_GATHER3SIV8SI:
16593 12 : icode = CODE_FOR_avx512vl_gathersiv8si;
16594 12 : goto gather_gen;
16595 24 : case IX86_BUILTIN_GATHER3DIV4SI:
16596 24 : icode = CODE_FOR_avx512vl_gatherdiv4si;
16597 24 : goto gather_gen;
16598 10 : case IX86_BUILTIN_GATHER3DIV8SI:
16599 10 : icode = CODE_FOR_avx512vl_gatherdiv8si;
16600 10 : goto gather_gen;
16601 4 : case IX86_BUILTIN_GATHER3ALTSIV4DF:
16602 4 : icode = CODE_FOR_avx512vl_gathersiv4df;
16603 4 : goto gather_gen;
16604 8 : case IX86_BUILTIN_GATHER3ALTDIV8SF:
16605 8 : icode = CODE_FOR_avx512vl_gatherdiv8sf;
16606 8 : goto gather_gen;
16607 6 : case IX86_BUILTIN_GATHER3ALTSIV4DI:
16608 6 : icode = CODE_FOR_avx512vl_gathersiv4di;
16609 6 : goto gather_gen;
16610 10 : case IX86_BUILTIN_GATHER3ALTDIV8SI:
16611 10 : icode = CODE_FOR_avx512vl_gatherdiv8si;
16612 10 : goto gather_gen;
16613 40 : case IX86_BUILTIN_SCATTERSIV16SF:
16614 40 : icode = CODE_FOR_avx512f_scattersiv16sf;
16615 40 : goto scatter_gen;
16616 27 : case IX86_BUILTIN_SCATTERSIV8DF:
16617 27 : icode = CODE_FOR_avx512f_scattersiv8df;
16618 27 : goto scatter_gen;
16619 24 : case IX86_BUILTIN_SCATTERDIV16SF:
16620 24 : icode = CODE_FOR_avx512f_scatterdiv16sf;
16621 24 : goto scatter_gen;
16622 33 : case IX86_BUILTIN_SCATTERDIV8DF:
16623 33 : icode = CODE_FOR_avx512f_scatterdiv8df;
16624 33 : goto scatter_gen;
16625 30 : case IX86_BUILTIN_SCATTERSIV16SI:
16626 30 : icode = CODE_FOR_avx512f_scattersiv16si;
16627 30 : goto scatter_gen;
16628 24 : case IX86_BUILTIN_SCATTERSIV8DI:
16629 24 : icode = CODE_FOR_avx512f_scattersiv8di;
16630 24 : goto scatter_gen;
16631 24 : case IX86_BUILTIN_SCATTERDIV16SI:
16632 24 : icode = CODE_FOR_avx512f_scatterdiv16si;
16633 24 : goto scatter_gen;
16634 29 : case IX86_BUILTIN_SCATTERDIV8DI:
16635 29 : icode = CODE_FOR_avx512f_scatterdiv8di;
16636 29 : goto scatter_gen;
16637 18 : case IX86_BUILTIN_SCATTERSIV8SF:
16638 18 : icode = CODE_FOR_avx512vl_scattersiv8sf;
16639 18 : goto scatter_gen;
16640 20 : case IX86_BUILTIN_SCATTERSIV4SF:
16641 20 : icode = CODE_FOR_avx512vl_scattersiv4sf;
16642 20 : goto scatter_gen;
16643 16 : case IX86_BUILTIN_SCATTERSIV4DF:
16644 16 : icode = CODE_FOR_avx512vl_scattersiv4df;
16645 16 : goto scatter_gen;
16646 16 : case IX86_BUILTIN_SCATTERSIV2DF:
16647 16 : icode = CODE_FOR_avx512vl_scattersiv2df;
16648 16 : goto scatter_gen;
16649 16 : case IX86_BUILTIN_SCATTERDIV8SF:
16650 16 : icode = CODE_FOR_avx512vl_scatterdiv8sf;
16651 16 : goto scatter_gen;
16652 16 : case IX86_BUILTIN_SCATTERDIV4SF:
16653 16 : icode = CODE_FOR_avx512vl_scatterdiv4sf;
16654 16 : goto scatter_gen;
16655 18 : case IX86_BUILTIN_SCATTERDIV4DF:
16656 18 : icode = CODE_FOR_avx512vl_scatterdiv4df;
16657 18 : goto scatter_gen;
16658 18 : case IX86_BUILTIN_SCATTERDIV2DF:
16659 18 : icode = CODE_FOR_avx512vl_scatterdiv2df;
16660 18 : goto scatter_gen;
16661 22 : case IX86_BUILTIN_SCATTERSIV8SI:
16662 22 : icode = CODE_FOR_avx512vl_scattersiv8si;
16663 22 : goto scatter_gen;
16664 24 : case IX86_BUILTIN_SCATTERSIV4SI:
16665 24 : icode = CODE_FOR_avx512vl_scattersiv4si;
16666 24 : goto scatter_gen;
16667 16 : case IX86_BUILTIN_SCATTERSIV4DI:
16668 16 : icode = CODE_FOR_avx512vl_scattersiv4di;
16669 16 : goto scatter_gen;
16670 16 : case IX86_BUILTIN_SCATTERSIV2DI:
16671 16 : icode = CODE_FOR_avx512vl_scattersiv2di;
16672 16 : goto scatter_gen;
16673 16 : case IX86_BUILTIN_SCATTERDIV8SI:
16674 16 : icode = CODE_FOR_avx512vl_scatterdiv8si;
16675 16 : goto scatter_gen;
16676 16 : case IX86_BUILTIN_SCATTERDIV4SI:
16677 16 : icode = CODE_FOR_avx512vl_scatterdiv4si;
16678 16 : goto scatter_gen;
16679 18 : case IX86_BUILTIN_SCATTERDIV4DI:
16680 18 : icode = CODE_FOR_avx512vl_scatterdiv4di;
16681 18 : goto scatter_gen;
16682 18 : case IX86_BUILTIN_SCATTERDIV2DI:
16683 18 : icode = CODE_FOR_avx512vl_scatterdiv2di;
16684 18 : goto scatter_gen;
16685 16 : case IX86_BUILTIN_SCATTERALTSIV8DF:
16686 16 : icode = CODE_FOR_avx512f_scattersiv8df;
16687 16 : goto scatter_gen;
16688 12 : case IX86_BUILTIN_SCATTERALTDIV16SF:
16689 12 : icode = CODE_FOR_avx512f_scatterdiv16sf;
16690 12 : goto scatter_gen;
16691 8 : case IX86_BUILTIN_SCATTERALTSIV8DI:
16692 8 : icode = CODE_FOR_avx512f_scattersiv8di;
16693 8 : goto scatter_gen;
16694 24 : case IX86_BUILTIN_SCATTERALTDIV16SI:
16695 24 : icode = CODE_FOR_avx512f_scatterdiv16si;
16696 24 : goto scatter_gen;
16697 4 : case IX86_BUILTIN_SCATTERALTSIV4DF:
16698 4 : icode = CODE_FOR_avx512vl_scattersiv4df;
16699 4 : goto scatter_gen;
16700 4 : case IX86_BUILTIN_SCATTERALTDIV8SF:
16701 4 : icode = CODE_FOR_avx512vl_scatterdiv8sf;
16702 4 : goto scatter_gen;
16703 4 : case IX86_BUILTIN_SCATTERALTSIV4DI:
16704 4 : icode = CODE_FOR_avx512vl_scattersiv4di;
16705 4 : goto scatter_gen;
16706 4 : case IX86_BUILTIN_SCATTERALTDIV8SI:
16707 4 : icode = CODE_FOR_avx512vl_scatterdiv8si;
16708 4 : goto scatter_gen;
16709 8 : case IX86_BUILTIN_SCATTERALTSIV2DF:
16710 8 : icode = CODE_FOR_avx512vl_scattersiv2df;
16711 8 : goto scatter_gen;
16712 8 : case IX86_BUILTIN_SCATTERALTDIV4SF:
16713 8 : icode = CODE_FOR_avx512vl_scatterdiv4sf;
16714 8 : goto scatter_gen;
16715 8 : case IX86_BUILTIN_SCATTERALTSIV2DI:
16716 8 : icode = CODE_FOR_avx512vl_scattersiv2di;
16717 8 : goto scatter_gen;
16718 8 : case IX86_BUILTIN_SCATTERALTDIV4SI:
16719 8 : icode = CODE_FOR_avx512vl_scatterdiv4si;
16720 8 : goto scatter_gen;
16721 :
16722 1007 : gather_gen:
16723 1007 : rtx half;
16724 1007 : rtx (*gen) (rtx, rtx);
16725 :
16726 1007 : arg0 = CALL_EXPR_ARG (exp, 0);
16727 1007 : arg1 = CALL_EXPR_ARG (exp, 1);
16728 1007 : arg2 = CALL_EXPR_ARG (exp, 2);
16729 1007 : arg3 = CALL_EXPR_ARG (exp, 3);
16730 1007 : arg4 = CALL_EXPR_ARG (exp, 4);
16731 1007 : op0 = expand_normal (arg0);
16732 1007 : op1 = expand_normal (arg1);
16733 1007 : op2 = expand_normal (arg2);
16734 1007 : op3 = ix86_expand_unsigned_small_int_cst_argument (arg3);
16735 1007 : op4 = expand_normal (arg4);
16736 : /* Note the arg order is different from the operand order. */
16737 1007 : mode0 = insn_data[icode].operand[1].mode;
16738 1007 : mode2 = insn_data[icode].operand[3].mode;
16739 1007 : mode3 = insn_data[icode].operand[4].mode;
16740 1007 : mode4 = insn_data[icode].operand[5].mode;
16741 :
16742 1007 : if (target == NULL_RTX
16743 1007 : || GET_MODE (target) != insn_data[icode].operand[0].mode
16744 1910 : || !insn_data[icode].operand[0].predicate (target,
16745 : GET_MODE (target)))
16746 105 : subtarget = gen_reg_rtx (insn_data[icode].operand[0].mode);
16747 : else
16748 : subtarget = target;
16749 :
16750 1007 : switch (fcode)
16751 : {
16752 30 : case IX86_BUILTIN_GATHER3ALTSIV8DF:
16753 30 : case IX86_BUILTIN_GATHER3ALTSIV8DI:
16754 30 : half = gen_reg_rtx (V8SImode);
16755 30 : if (!nonimmediate_operand (op2, V16SImode))
16756 0 : op2 = copy_to_mode_reg (V16SImode, op2);
16757 30 : emit_insn (gen_vec_extract_lo_v16si (half, op2));
16758 30 : op2 = half;
16759 30 : break;
16760 34 : case IX86_BUILTIN_GATHER3ALTSIV4DF:
16761 34 : case IX86_BUILTIN_GATHER3ALTSIV4DI:
16762 34 : case IX86_BUILTIN_GATHERALTSIV4DF:
16763 34 : case IX86_BUILTIN_GATHERALTSIV4DI:
16764 34 : half = gen_reg_rtx (V4SImode);
16765 34 : if (!nonimmediate_operand (op2, V8SImode))
16766 0 : op2 = copy_to_mode_reg (V8SImode, op2);
16767 34 : emit_insn (gen_vec_extract_lo_v8si (half, op2));
16768 34 : op2 = half;
16769 34 : break;
16770 40 : case IX86_BUILTIN_GATHER3ALTDIV16SF:
16771 40 : case IX86_BUILTIN_GATHER3ALTDIV16SI:
16772 40 : half = gen_reg_rtx (mode0);
16773 40 : if (mode0 == V8SFmode)
16774 : gen = gen_vec_extract_lo_v16sf;
16775 : else
16776 18 : gen = gen_vec_extract_lo_v16si;
16777 40 : if (!nonimmediate_operand (op0, GET_MODE (op0)))
16778 40 : op0 = copy_to_mode_reg (GET_MODE (op0), op0);
16779 40 : emit_insn (gen (half, op0));
16780 40 : op0 = half;
16781 40 : op3 = lowpart_subreg (QImode, op3, HImode);
16782 40 : break;
16783 46 : case IX86_BUILTIN_GATHER3ALTDIV8SF:
16784 46 : case IX86_BUILTIN_GATHER3ALTDIV8SI:
16785 46 : case IX86_BUILTIN_GATHERALTDIV8SF:
16786 46 : case IX86_BUILTIN_GATHERALTDIV8SI:
16787 46 : half = gen_reg_rtx (mode0);
16788 46 : if (mode0 == V4SFmode)
16789 : gen = gen_vec_extract_lo_v8sf;
16790 : else
16791 22 : gen = gen_vec_extract_lo_v8si;
16792 46 : if (!nonimmediate_operand (op0, GET_MODE (op0)))
16793 46 : op0 = copy_to_mode_reg (GET_MODE (op0), op0);
16794 46 : emit_insn (gen (half, op0));
16795 46 : op0 = half;
16796 46 : if (VECTOR_MODE_P (GET_MODE (op3)))
16797 : {
16798 28 : half = gen_reg_rtx (mode0);
16799 28 : if (!nonimmediate_operand (op3, GET_MODE (op3)))
16800 12 : op3 = copy_to_mode_reg (GET_MODE (op3), op3);
16801 28 : emit_insn (gen (half, op3));
16802 28 : op3 = half;
16803 : }
16804 : break;
16805 : default:
16806 : break;
16807 : }
16808 :
16809 : /* Force memory operand only with base register here. But we
16810 : don't want to do it on memory operand for other builtin
16811 : functions. */
16812 1007 : op1 = ix86_zero_extend_to_Pmode (op1);
16813 :
16814 1007 : if (!insn_data[icode].operand[1].predicate (op0, mode0))
16815 406 : op0 = copy_to_mode_reg (mode0, op0);
16816 1012 : if (!insn_data[icode].operand[2].predicate (op1, Pmode))
16817 0 : op1 = copy_to_mode_reg (Pmode, op1);
16818 1007 : if (!insn_data[icode].operand[3].predicate (op2, mode2))
16819 224 : op2 = copy_to_mode_reg (mode2, op2);
16820 :
16821 1007 : op3 = fixup_modeless_constant (op3, mode3);
16822 :
16823 1007 : if (GET_MODE (op3) == mode3 || GET_MODE (op3) == VOIDmode)
16824 : {
16825 1007 : if (!insn_data[icode].operand[4].predicate (op3, mode3))
16826 356 : op3 = copy_to_mode_reg (mode3, op3);
16827 : }
16828 : else
16829 : {
16830 0 : op3 = copy_to_reg (op3);
16831 0 : op3 = lowpart_subreg (mode3, op3, GET_MODE (op3));
16832 : }
16833 1007 : if (!insn_data[icode].operand[5].predicate (op4, mode4))
16834 : {
16835 0 : error ("the last argument must be scale 1, 2, 4, 8");
16836 0 : return const0_rtx;
16837 : }
16838 :
16839 : /* Optimize. If mask is known to have all high bits set,
16840 : replace op0 with pc_rtx to signal that the instruction
16841 : overwrites the whole destination and doesn't use its
16842 : previous contents. */
16843 1007 : if (optimize)
16844 : {
16845 917 : if (TREE_CODE (arg3) == INTEGER_CST)
16846 : {
16847 209 : if (integer_all_onesp (arg3))
16848 201 : op0 = pc_rtx;
16849 : }
16850 708 : else if (TREE_CODE (arg3) == VECTOR_CST)
16851 : {
16852 : unsigned int negative = 0;
16853 755 : for (i = 0; i < VECTOR_CST_NELTS (arg3); ++i)
16854 : {
16855 620 : tree cst = VECTOR_CST_ELT (arg3, i);
16856 620 : if (TREE_CODE (cst) == INTEGER_CST
16857 620 : && tree_int_cst_sign_bit (cst))
16858 286 : negative++;
16859 334 : else if (TREE_CODE (cst) == REAL_CST
16860 334 : && REAL_VALUE_NEGATIVE (TREE_REAL_CST (cst)))
16861 306 : negative++;
16862 : }
16863 135 : if (negative == TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg3)))
16864 121 : op0 = pc_rtx;
16865 : }
16866 573 : else if (TREE_CODE (arg3) == SSA_NAME
16867 573 : && VECTOR_TYPE_P (TREE_TYPE (arg3)))
16868 : {
16869 : /* Recognize also when mask is like:
16870 : __v2df src = _mm_setzero_pd ();
16871 : __v2df mask = _mm_cmpeq_pd (src, src);
16872 : or
16873 : __v8sf src = _mm256_setzero_ps ();
16874 : __v8sf mask = _mm256_cmp_ps (src, src, _CMP_EQ_OQ);
16875 : as that is a cheaper way to load all ones into
16876 : a register than having to load a constant from
16877 : memory. */
16878 259 : gimple *def_stmt = SSA_NAME_DEF_STMT (arg3);
16879 259 : if (is_gimple_call (def_stmt))
16880 : {
16881 76 : tree fndecl = gimple_call_fndecl (def_stmt);
16882 76 : if (fndecl
16883 76 : && fndecl_built_in_p (fndecl, BUILT_IN_MD))
16884 67 : switch (DECL_MD_FUNCTION_CODE (fndecl))
16885 : {
16886 24 : case IX86_BUILTIN_CMPPD:
16887 24 : case IX86_BUILTIN_CMPPS:
16888 24 : case IX86_BUILTIN_CMPPD256:
16889 24 : case IX86_BUILTIN_CMPPS256:
16890 24 : if (!integer_zerop (gimple_call_arg (def_stmt, 2)))
16891 : break;
16892 : /* FALLTHRU */
16893 49 : case IX86_BUILTIN_CMPEQPD:
16894 49 : case IX86_BUILTIN_CMPEQPS:
16895 49 : if (initializer_zerop (gimple_call_arg (def_stmt, 0))
16896 49 : && initializer_zerop (gimple_call_arg (def_stmt,
16897 : 1)))
16898 49 : op0 = pc_rtx;
16899 : break;
16900 : default:
16901 : break;
16902 : }
16903 : }
16904 : }
16905 : }
16906 :
16907 1007 : pat = GEN_FCN (icode) (subtarget, op0, op1, op2, op3, op4);
16908 1007 : if (! pat)
16909 0 : return const0_rtx;
16910 1007 : emit_insn (pat);
16911 :
16912 1007 : switch (fcode)
16913 : {
16914 24 : case IX86_BUILTIN_GATHER3DIV16SF:
16915 24 : if (target == NULL_RTX)
16916 0 : target = gen_reg_rtx (V8SFmode);
16917 24 : emit_insn (gen_vec_extract_lo_v16sf (target, subtarget));
16918 24 : break;
16919 24 : case IX86_BUILTIN_GATHER3DIV16SI:
16920 24 : if (target == NULL_RTX)
16921 0 : target = gen_reg_rtx (V8SImode);
16922 24 : emit_insn (gen_vec_extract_lo_v16si (target, subtarget));
16923 24 : break;
16924 28 : case IX86_BUILTIN_GATHER3DIV8SF:
16925 28 : case IX86_BUILTIN_GATHERDIV8SF:
16926 28 : if (target == NULL_RTX)
16927 0 : target = gen_reg_rtx (V4SFmode);
16928 28 : emit_insn (gen_vec_extract_lo_v8sf (target, subtarget));
16929 28 : break;
16930 28 : case IX86_BUILTIN_GATHER3DIV8SI:
16931 28 : case IX86_BUILTIN_GATHERDIV8SI:
16932 28 : if (target == NULL_RTX)
16933 0 : target = gen_reg_rtx (V4SImode);
16934 28 : emit_insn (gen_vec_extract_lo_v8si (target, subtarget));
16935 28 : break;
16936 : default:
16937 : target = subtarget;
16938 : break;
16939 : }
16940 : return target;
16941 :
16942 623 : scatter_gen:
16943 623 : arg0 = CALL_EXPR_ARG (exp, 0);
16944 623 : arg1 = CALL_EXPR_ARG (exp, 1);
16945 623 : arg2 = CALL_EXPR_ARG (exp, 2);
16946 623 : arg3 = CALL_EXPR_ARG (exp, 3);
16947 623 : arg4 = CALL_EXPR_ARG (exp, 4);
16948 623 : op0 = expand_normal (arg0);
16949 623 : op1 = ix86_expand_unsigned_small_int_cst_argument (arg1);
16950 623 : op2 = expand_normal (arg2);
16951 623 : op3 = expand_normal (arg3);
16952 623 : op4 = expand_normal (arg4);
16953 623 : mode1 = insn_data[icode].operand[1].mode;
16954 623 : mode2 = insn_data[icode].operand[2].mode;
16955 623 : mode3 = insn_data[icode].operand[3].mode;
16956 623 : mode4 = insn_data[icode].operand[4].mode;
16957 :
16958 : /* Scatter instruction stores operand op3 to memory with
16959 : indices from op2 and scale from op4 under writemask op1.
16960 : If index operand op2 has more elements then source operand
16961 : op3 one need to use only its low half. And vice versa. */
16962 623 : switch (fcode)
16963 : {
16964 24 : case IX86_BUILTIN_SCATTERALTSIV8DF:
16965 24 : case IX86_BUILTIN_SCATTERALTSIV8DI:
16966 24 : half = gen_reg_rtx (V8SImode);
16967 24 : if (!nonimmediate_operand (op2, V16SImode))
16968 0 : op2 = copy_to_mode_reg (V16SImode, op2);
16969 24 : emit_insn (gen_vec_extract_lo_v16si (half, op2));
16970 24 : op2 = half;
16971 24 : break;
16972 36 : case IX86_BUILTIN_SCATTERALTDIV16SF:
16973 36 : case IX86_BUILTIN_SCATTERALTDIV16SI:
16974 36 : half = gen_reg_rtx (mode3);
16975 36 : if (mode3 == V8SFmode)
16976 : gen = gen_vec_extract_lo_v16sf;
16977 : else
16978 24 : gen = gen_vec_extract_lo_v16si;
16979 36 : if (!nonimmediate_operand (op3, GET_MODE (op3)))
16980 0 : op3 = copy_to_mode_reg (GET_MODE (op3), op3);
16981 36 : emit_insn (gen (half, op3));
16982 36 : op3 = half;
16983 36 : break;
16984 8 : case IX86_BUILTIN_SCATTERALTSIV4DF:
16985 8 : case IX86_BUILTIN_SCATTERALTSIV4DI:
16986 8 : half = gen_reg_rtx (V4SImode);
16987 8 : if (!nonimmediate_operand (op2, V8SImode))
16988 0 : op2 = copy_to_mode_reg (V8SImode, op2);
16989 8 : emit_insn (gen_vec_extract_lo_v8si (half, op2));
16990 8 : op2 = half;
16991 8 : break;
16992 8 : case IX86_BUILTIN_SCATTERALTDIV8SF:
16993 8 : case IX86_BUILTIN_SCATTERALTDIV8SI:
16994 8 : half = gen_reg_rtx (mode3);
16995 8 : if (mode3 == V4SFmode)
16996 : gen = gen_vec_extract_lo_v8sf;
16997 : else
16998 4 : gen = gen_vec_extract_lo_v8si;
16999 8 : if (!nonimmediate_operand (op3, GET_MODE (op3)))
17000 0 : op3 = copy_to_mode_reg (GET_MODE (op3), op3);
17001 8 : emit_insn (gen (half, op3));
17002 8 : op3 = half;
17003 8 : break;
17004 16 : case IX86_BUILTIN_SCATTERALTSIV2DF:
17005 16 : case IX86_BUILTIN_SCATTERALTSIV2DI:
17006 16 : if (!nonimmediate_operand (op2, V4SImode))
17007 0 : op2 = copy_to_mode_reg (V4SImode, op2);
17008 : break;
17009 16 : case IX86_BUILTIN_SCATTERALTDIV4SF:
17010 16 : case IX86_BUILTIN_SCATTERALTDIV4SI:
17011 16 : if (!nonimmediate_operand (op3, GET_MODE (op3)))
17012 0 : op3 = copy_to_mode_reg (GET_MODE (op3), op3);
17013 : break;
17014 : default:
17015 : break;
17016 : }
17017 :
17018 : /* Force memory operand only with base register here. But we
17019 : don't want to do it on memory operand for other builtin
17020 : functions. */
17021 633 : op0 = force_reg (Pmode, convert_to_mode (Pmode, op0, 1));
17022 :
17023 628 : if (!insn_data[icode].operand[0].predicate (op0, Pmode))
17024 0 : op0 = copy_to_mode_reg (Pmode, op0);
17025 :
17026 623 : op1 = fixup_modeless_constant (op1, mode1);
17027 :
17028 623 : if (GET_MODE (op1) == mode1 || GET_MODE (op1) == VOIDmode)
17029 : {
17030 607 : if (!insn_data[icode].operand[1].predicate (op1, mode1))
17031 273 : op1 = copy_to_mode_reg (mode1, op1);
17032 : }
17033 : else
17034 : {
17035 16 : op1 = copy_to_reg (op1);
17036 16 : op1 = lowpart_subreg (mode1, op1, GET_MODE (op1));
17037 : }
17038 :
17039 623 : if (!insn_data[icode].operand[2].predicate (op2, mode2))
17040 57 : op2 = copy_to_mode_reg (mode2, op2);
17041 :
17042 623 : if (!insn_data[icode].operand[3].predicate (op3, mode3))
17043 82 : op3 = copy_to_mode_reg (mode3, op3);
17044 :
17045 623 : if (!insn_data[icode].operand[4].predicate (op4, mode4))
17046 : {
17047 0 : error ("the last argument must be scale 1, 2, 4, 8");
17048 0 : return const0_rtx;
17049 : }
17050 :
17051 623 : pat = GEN_FCN (icode) (op0, op1, op2, op3, op4);
17052 623 : if (! pat)
17053 0 : return const0_rtx;
17054 :
17055 623 : emit_insn (pat);
17056 623 : return 0;
17057 :
17058 23 : case IX86_BUILTIN_XABORT:
17059 23 : icode = CODE_FOR_xabort;
17060 23 : arg0 = CALL_EXPR_ARG (exp, 0);
17061 23 : op0 = expand_normal (arg0);
17062 23 : mode0 = insn_data[icode].operand[0].mode;
17063 23 : if (!insn_data[icode].operand[0].predicate (op0, mode0))
17064 : {
17065 0 : error ("the argument to %<xabort%> intrinsic must "
17066 : "be an 8-bit immediate");
17067 0 : return const0_rtx;
17068 : }
17069 23 : emit_insn (gen_xabort (op0));
17070 23 : return 0;
17071 :
17072 55 : case IX86_BUILTIN_RDSSPD:
17073 55 : case IX86_BUILTIN_RDSSPQ:
17074 55 : mode = (fcode == IX86_BUILTIN_RDSSPD ? SImode : DImode);
17075 :
17076 55 : if (target == 0
17077 55 : || !register_operand (target, mode))
17078 0 : target = gen_reg_rtx (mode);
17079 :
17080 55 : op0 = force_reg (mode, const0_rtx);
17081 :
17082 55 : emit_insn (gen_rdssp (mode, target, op0));
17083 55 : return target;
17084 :
17085 55 : case IX86_BUILTIN_INCSSPD:
17086 55 : case IX86_BUILTIN_INCSSPQ:
17087 55 : mode = (fcode == IX86_BUILTIN_INCSSPD ? SImode : DImode);
17088 :
17089 55 : arg0 = CALL_EXPR_ARG (exp, 0);
17090 55 : op0 = expand_normal (arg0);
17091 :
17092 55 : op0 = force_reg (mode, op0);
17093 :
17094 55 : emit_insn (gen_incssp (mode, op0));
17095 55 : return 0;
17096 :
17097 20 : case IX86_BUILTIN_HRESET:
17098 20 : icode = CODE_FOR_hreset;
17099 20 : arg0 = CALL_EXPR_ARG (exp, 0);
17100 20 : op0 = expand_normal (arg0);
17101 20 : op0 = force_reg (SImode, op0);
17102 20 : emit_insn (gen_hreset (op0));
17103 20 : return 0;
17104 :
17105 38 : case IX86_BUILTIN_RSTORSSP:
17106 38 : case IX86_BUILTIN_CLRSSBSY:
17107 38 : arg0 = CALL_EXPR_ARG (exp, 0);
17108 38 : op0 = expand_normal (arg0);
17109 19 : icode = (fcode == IX86_BUILTIN_RSTORSSP
17110 38 : ? CODE_FOR_rstorssp
17111 : : CODE_FOR_clrssbsy);
17112 :
17113 38 : if (!address_operand (op0, VOIDmode))
17114 : {
17115 18 : op0 = convert_memory_address (Pmode, op0);
17116 18 : op0 = copy_addr_to_reg (op0);
17117 : }
17118 38 : emit_insn (GEN_FCN (icode) (gen_rtx_MEM (DImode, op0)));
17119 38 : return 0;
17120 :
17121 80 : case IX86_BUILTIN_WRSSD:
17122 80 : case IX86_BUILTIN_WRSSQ:
17123 80 : case IX86_BUILTIN_WRUSSD:
17124 80 : case IX86_BUILTIN_WRUSSQ:
17125 80 : mode = ((fcode == IX86_BUILTIN_WRSSD
17126 80 : || fcode == IX86_BUILTIN_WRUSSD)
17127 80 : ? SImode : DImode);
17128 :
17129 80 : arg0 = CALL_EXPR_ARG (exp, 0);
17130 80 : op0 = expand_normal (arg0);
17131 80 : arg1 = CALL_EXPR_ARG (exp, 1);
17132 80 : op1 = expand_normal (arg1);
17133 :
17134 80 : op0 = force_reg (mode, op0);
17135 :
17136 80 : if (!address_operand (op1, VOIDmode))
17137 : {
17138 36 : op1 = convert_memory_address (Pmode, op1);
17139 36 : op1 = copy_addr_to_reg (op1);
17140 : }
17141 80 : op1 = gen_rtx_MEM (mode, op1);
17142 :
17143 80 : icode = ((fcode == IX86_BUILTIN_WRSSD
17144 80 : || fcode == IX86_BUILTIN_WRSSQ)
17145 80 : ? code_for_wrss (mode)
17146 40 : : code_for_wruss (mode));
17147 80 : emit_insn (GEN_FCN (icode) (op0, op1));
17148 :
17149 80 : return 0;
17150 :
17151 118403 : default:
17152 118403 : break;
17153 : }
17154 :
17155 118403 : if (fcode >= IX86_BUILTIN__BDESC_SPECIAL_ARGS_FIRST
17156 118403 : && fcode <= IX86_BUILTIN__BDESC_SPECIAL_ARGS_LAST)
17157 : {
17158 26971 : i = fcode - IX86_BUILTIN__BDESC_SPECIAL_ARGS_FIRST;
17159 26971 : return ix86_expand_special_args_builtin (bdesc_special_args + i, exp,
17160 26971 : target);
17161 : }
17162 :
17163 91432 : if (fcode >= IX86_BUILTIN__BDESC_PURE_ARGS_FIRST
17164 91432 : && fcode <= IX86_BUILTIN__BDESC_PURE_ARGS_LAST)
17165 : {
17166 93 : i = fcode - IX86_BUILTIN__BDESC_PURE_ARGS_FIRST;
17167 93 : return ix86_expand_special_args_builtin (bdesc_pure_args + i, exp,
17168 93 : target);
17169 : }
17170 :
17171 91339 : if (fcode >= IX86_BUILTIN__BDESC_ARGS_FIRST
17172 91339 : && fcode <= IX86_BUILTIN__BDESC_ARGS_LAST)
17173 : {
17174 72986 : i = fcode - IX86_BUILTIN__BDESC_ARGS_FIRST;
17175 :
17176 72986 : switch (fcode)
17177 : {
17178 0 : case IX86_BUILTIN_RDPID:
17179 0 : return ix86_expand_special_args_builtin (bdesc_args + i, exp,
17180 0 : target);
17181 74 : case IX86_BUILTIN_VCOMISBF16EQ:
17182 74 : case IX86_BUILTIN_VCOMISBF16NE:
17183 74 : case IX86_BUILTIN_VCOMISBF16GT:
17184 74 : case IX86_BUILTIN_VCOMISBF16GE:
17185 74 : case IX86_BUILTIN_VCOMISBF16LT:
17186 74 : case IX86_BUILTIN_VCOMISBF16LE:
17187 74 : return ix86_expand_sse_comi (bdesc_args + i, exp, target, false);
17188 15 : case IX86_BUILTIN_FABSQ:
17189 15 : case IX86_BUILTIN_COPYSIGNQ:
17190 15 : if (!TARGET_SSE)
17191 : /* Emit a normal call if SSE isn't available. */
17192 0 : return expand_call (exp, target, ignore);
17193 : /* FALLTHRU */
17194 72912 : default:
17195 72912 : return ix86_expand_args_builtin (bdesc_args + i, exp, target);
17196 : }
17197 : }
17198 :
17199 18353 : if (fcode >= IX86_BUILTIN__BDESC_COMI_FIRST
17200 18353 : && fcode <= IX86_BUILTIN__BDESC_COMI_LAST)
17201 : {
17202 473 : i = fcode - IX86_BUILTIN__BDESC_COMI_FIRST;
17203 473 : return ix86_expand_sse_comi (bdesc_comi + i, exp, target, true);
17204 : }
17205 :
17206 17880 : if (fcode >= IX86_BUILTIN__BDESC_ROUND_ARGS_FIRST
17207 17880 : && fcode <= IX86_BUILTIN__BDESC_ROUND_ARGS_LAST)
17208 : {
17209 15538 : i = fcode - IX86_BUILTIN__BDESC_ROUND_ARGS_FIRST;
17210 15538 : return ix86_expand_round_builtin (bdesc_round_args + i, exp, target);
17211 : }
17212 :
17213 2342 : if (fcode >= IX86_BUILTIN__BDESC_PCMPESTR_FIRST
17214 2342 : && fcode <= IX86_BUILTIN__BDESC_PCMPESTR_LAST)
17215 : {
17216 216 : i = fcode - IX86_BUILTIN__BDESC_PCMPESTR_FIRST;
17217 216 : return ix86_expand_sse_pcmpestr (bdesc_pcmpestr + i, exp, target);
17218 : }
17219 :
17220 2126 : if (fcode >= IX86_BUILTIN__BDESC_PCMPISTR_FIRST
17221 2126 : && fcode <= IX86_BUILTIN__BDESC_PCMPISTR_LAST)
17222 : {
17223 275 : i = fcode - IX86_BUILTIN__BDESC_PCMPISTR_FIRST;
17224 275 : return ix86_expand_sse_pcmpistr (bdesc_pcmpistr + i, exp, target);
17225 : }
17226 :
17227 1851 : if (fcode >= IX86_BUILTIN__BDESC_MULTI_ARG_FIRST
17228 1851 : && fcode <= IX86_BUILTIN__BDESC_MULTI_ARG_LAST)
17229 : {
17230 1813 : i = fcode - IX86_BUILTIN__BDESC_MULTI_ARG_FIRST;
17231 1813 : const struct builtin_description *d = bdesc_multi_arg + i;
17232 1813 : return ix86_expand_multi_arg_builtin (d->icode, exp, target,
17233 : (enum ix86_builtin_func_type)
17234 1813 : d->flag, d->comparison);
17235 : }
17236 :
17237 38 : if (fcode >= IX86_BUILTIN__BDESC_CET_FIRST
17238 38 : && fcode <= IX86_BUILTIN__BDESC_CET_LAST)
17239 : {
17240 38 : i = fcode - IX86_BUILTIN__BDESC_CET_FIRST;
17241 38 : return ix86_expand_special_args_builtin (bdesc_cet + i, exp,
17242 38 : target);
17243 : }
17244 :
17245 0 : gcc_unreachable ();
17246 : }
17247 :
17248 : /* See below where shifts are handled for explanation of this enum. */
17249 : enum ix86_vec_bcast_alg
17250 : {
17251 : VEC_BCAST_PXOR,
17252 : VEC_BCAST_PCMPEQ,
17253 : VEC_BCAST_PABSB,
17254 : VEC_BCAST_PADDB,
17255 : VEC_BCAST_PSRLW,
17256 : VEC_BCAST_PSRLD,
17257 : VEC_BCAST_PSLLW,
17258 : VEC_BCAST_PSLLD
17259 : };
17260 :
17261 : struct ix86_vec_bcast_map_simode_t
17262 : {
17263 : unsigned int key;
17264 : enum ix86_vec_bcast_alg alg;
17265 : unsigned int arg;
17266 : };
17267 :
17268 : /* This table must be kept sorted as values are looked-up using bsearch. */
17269 : static const ix86_vec_bcast_map_simode_t ix86_vec_bcast_map_simode[] = {
17270 : { 0x00000000, VEC_BCAST_PXOR, 0 },
17271 : { 0x00000001, VEC_BCAST_PSRLD, 31 },
17272 : { 0x00000003, VEC_BCAST_PSRLD, 30 },
17273 : { 0x00000007, VEC_BCAST_PSRLD, 29 },
17274 : { 0x0000000f, VEC_BCAST_PSRLD, 28 },
17275 : { 0x0000001f, VEC_BCAST_PSRLD, 27 },
17276 : { 0x0000003f, VEC_BCAST_PSRLD, 26 },
17277 : { 0x0000007f, VEC_BCAST_PSRLD, 25 },
17278 : { 0x000000ff, VEC_BCAST_PSRLD, 24 },
17279 : { 0x000001ff, VEC_BCAST_PSRLD, 23 },
17280 : { 0x000003ff, VEC_BCAST_PSRLD, 22 },
17281 : { 0x000007ff, VEC_BCAST_PSRLD, 21 },
17282 : { 0x00000fff, VEC_BCAST_PSRLD, 20 },
17283 : { 0x00001fff, VEC_BCAST_PSRLD, 19 },
17284 : { 0x00003fff, VEC_BCAST_PSRLD, 18 },
17285 : { 0x00007fff, VEC_BCAST_PSRLD, 17 },
17286 : { 0x0000ffff, VEC_BCAST_PSRLD, 16 },
17287 : { 0x00010001, VEC_BCAST_PSRLW, 15 },
17288 : { 0x0001ffff, VEC_BCAST_PSRLD, 15 },
17289 : { 0x00030003, VEC_BCAST_PSRLW, 14 },
17290 : { 0x0003ffff, VEC_BCAST_PSRLD, 14 },
17291 : { 0x00070007, VEC_BCAST_PSRLW, 13 },
17292 : { 0x0007ffff, VEC_BCAST_PSRLD, 13 },
17293 : { 0x000f000f, VEC_BCAST_PSRLW, 12 },
17294 : { 0x000fffff, VEC_BCAST_PSRLD, 12 },
17295 : { 0x001f001f, VEC_BCAST_PSRLW, 11 },
17296 : { 0x001fffff, VEC_BCAST_PSRLD, 11 },
17297 : { 0x003f003f, VEC_BCAST_PSRLW, 10 },
17298 : { 0x003fffff, VEC_BCAST_PSRLD, 10 },
17299 : { 0x007f007f, VEC_BCAST_PSRLW, 9 },
17300 : { 0x007fffff, VEC_BCAST_PSRLD, 9 },
17301 : { 0x00ff00ff, VEC_BCAST_PSRLW, 8 },
17302 : { 0x00ffffff, VEC_BCAST_PSRLD, 8 },
17303 : { 0x01010101, VEC_BCAST_PABSB, 0 },
17304 : { 0x01ff01ff, VEC_BCAST_PSRLW, 7 },
17305 : { 0x01ffffff, VEC_BCAST_PSRLD, 7 },
17306 : { 0x03ff03ff, VEC_BCAST_PSRLW, 6 },
17307 : { 0x03ffffff, VEC_BCAST_PSRLD, 6 },
17308 : { 0x07ff07ff, VEC_BCAST_PSRLW, 5 },
17309 : { 0x07ffffff, VEC_BCAST_PSRLD, 5 },
17310 : { 0x0fff0fff, VEC_BCAST_PSRLW, 4 },
17311 : { 0x0fffffff, VEC_BCAST_PSRLD, 4 },
17312 : { 0x1fff1fff, VEC_BCAST_PSRLW, 3 },
17313 : { 0x1fffffff, VEC_BCAST_PSRLD, 3 },
17314 : { 0x3fff3fff, VEC_BCAST_PSRLW, 2 },
17315 : { 0x3fffffff, VEC_BCAST_PSRLD, 2 },
17316 : { 0x7fff7fff, VEC_BCAST_PSRLW, 1 },
17317 : { 0x7fffffff, VEC_BCAST_PSRLD, 1 },
17318 : { 0x80000000, VEC_BCAST_PSLLD, 31 },
17319 : { 0x80008000, VEC_BCAST_PSLLW, 15 },
17320 : { 0xc0000000, VEC_BCAST_PSLLD, 30 },
17321 : { 0xc000c000, VEC_BCAST_PSLLW, 14 },
17322 : { 0xe0000000, VEC_BCAST_PSLLD, 29 },
17323 : { 0xe000e000, VEC_BCAST_PSLLW, 13 },
17324 : { 0xf0000000, VEC_BCAST_PSLLD, 28 },
17325 : { 0xf000f000, VEC_BCAST_PSLLW, 12 },
17326 : { 0xf8000000, VEC_BCAST_PSLLD, 27 },
17327 : { 0xf800f800, VEC_BCAST_PSLLW, 11 },
17328 : { 0xfc000000, VEC_BCAST_PSLLD, 26 },
17329 : { 0xfc00fc00, VEC_BCAST_PSLLW, 10 },
17330 : { 0xfe000000, VEC_BCAST_PSLLD, 25 },
17331 : { 0xfe00fe00, VEC_BCAST_PSLLW, 9 },
17332 : { 0xfefefefe, VEC_BCAST_PADDB, 0 },
17333 : { 0xff000000, VEC_BCAST_PSLLD, 24 },
17334 : { 0xff00ff00, VEC_BCAST_PSLLW, 8 },
17335 : { 0xff800000, VEC_BCAST_PSLLD, 23 },
17336 : { 0xff80ff80, VEC_BCAST_PSLLW, 7 },
17337 : { 0xffc00000, VEC_BCAST_PSLLD, 22 },
17338 : { 0xffc0ffc0, VEC_BCAST_PSLLW, 6 },
17339 : { 0xffe00000, VEC_BCAST_PSLLD, 21 },
17340 : { 0xffe0ffe0, VEC_BCAST_PSLLW, 5 },
17341 : { 0xfff00000, VEC_BCAST_PSLLD, 20 },
17342 : { 0xfff0fff0, VEC_BCAST_PSLLW, 4 },
17343 : { 0xfff80000, VEC_BCAST_PSLLD, 19 },
17344 : { 0xfff8fff8, VEC_BCAST_PSLLW, 3 },
17345 : { 0xfffc0000, VEC_BCAST_PSLLD, 18 },
17346 : { 0xfffcfffc, VEC_BCAST_PSLLW, 2 },
17347 : { 0xfffe0000, VEC_BCAST_PSLLD, 17 },
17348 : { 0xfffefffe, VEC_BCAST_PSLLW, 1 },
17349 : { 0xffff0000, VEC_BCAST_PSLLD, 16 },
17350 : { 0xffff8000, VEC_BCAST_PSLLD, 15 },
17351 : { 0xffffc000, VEC_BCAST_PSLLD, 14 },
17352 : { 0xffffe000, VEC_BCAST_PSLLD, 13 },
17353 : { 0xfffff000, VEC_BCAST_PSLLD, 12 },
17354 : { 0xfffff800, VEC_BCAST_PSLLD, 11 },
17355 : { 0xfffffc00, VEC_BCAST_PSLLD, 10 },
17356 : { 0xfffffe00, VEC_BCAST_PSLLD, 9 },
17357 : { 0xffffff00, VEC_BCAST_PSLLD, 8 },
17358 : { 0xffffff80, VEC_BCAST_PSLLD, 7 },
17359 : { 0xffffffc0, VEC_BCAST_PSLLD, 6 },
17360 : { 0xffffffe0, VEC_BCAST_PSLLD, 5 },
17361 : { 0xfffffff0, VEC_BCAST_PSLLD, 4 },
17362 : { 0xfffffff8, VEC_BCAST_PSLLD, 3 },
17363 : { 0xfffffffc, VEC_BCAST_PSLLD, 2 },
17364 : { 0xfffffffe, VEC_BCAST_PSLLD, 1 },
17365 : { 0xffffffff, VEC_BCAST_PCMPEQ, 0 }
17366 : };
17367 :
17368 : /* Comparator for bsearch on ix86_vec_bcast_map. */
17369 : static int
17370 343338 : ix86_vec_bcast_map_simode_cmp (const void *key, const void *entry)
17371 : {
17372 343338 : return (*(const unsigned int*)key)
17373 343338 : - ((const ix86_vec_bcast_map_simode_t*)entry)->key;
17374 : }
17375 :
17376 : /* A subroutine of ix86_vector_duplicate_value. Tries to efficiently
17377 : materialize V4SImode, V8SImode and V16SImode vectors from SImode
17378 : integer constants. */
17379 : static bool
17380 51973 : ix86_vector_duplicate_simode_const (machine_mode mode, rtx target,
17381 : unsigned int val)
17382 : {
17383 51973 : const ix86_vec_bcast_map_simode_t *entry;
17384 51973 : rtx tmp1, tmp2;
17385 :
17386 51973 : entry = (const ix86_vec_bcast_map_simode_t*)
17387 51973 : bsearch(&val, ix86_vec_bcast_map_simode,
17388 : ARRAY_SIZE (ix86_vec_bcast_map_simode),
17389 : sizeof (ix86_vec_bcast_map_simode_t),
17390 : ix86_vec_bcast_map_simode_cmp);
17391 51973 : if (!entry)
17392 : return false;
17393 :
17394 19531 : switch (entry->alg)
17395 : {
17396 2584 : case VEC_BCAST_PXOR:
17397 2584 : if ((mode == V8SImode && !TARGET_AVX2)
17398 2584 : || (mode == V16SImode && !TARGET_AVX512F))
17399 : return false;
17400 2584 : emit_move_insn (target, CONST0_RTX (mode));
17401 2584 : return true;
17402 :
17403 299 : case VEC_BCAST_PCMPEQ:
17404 299 : if ((mode == V4SImode && !TARGET_SSE2)
17405 298 : || (mode == V8SImode && !TARGET_AVX2)
17406 271 : || (mode == V16SImode && !TARGET_AVX512F))
17407 : return false;
17408 271 : emit_move_insn (target, CONSTM1_RTX (mode));
17409 271 : return true;
17410 :
17411 616 : case VEC_BCAST_PABSB:
17412 616 : if (mode == V4SImode && TARGET_SSE2)
17413 : {
17414 491 : tmp1 = gen_reg_rtx (V16QImode);
17415 491 : emit_move_insn (tmp1, CONSTM1_RTX (V16QImode));
17416 491 : tmp2 = gen_reg_rtx (V16QImode);
17417 491 : emit_insn (gen_absv16qi2 (tmp2, tmp1));
17418 : }
17419 125 : else if (mode == V8SImode && TARGET_AVX2)
17420 : {
17421 68 : tmp1 = gen_reg_rtx (V32QImode);
17422 68 : emit_move_insn (tmp1, CONSTM1_RTX (V32QImode));
17423 68 : tmp2 = gen_reg_rtx (V32QImode);
17424 68 : emit_insn (gen_absv32qi2 (tmp2, tmp1));
17425 : }
17426 57 : else if (mode == V16SImode && TARGET_AVX512BW)
17427 : {
17428 49 : tmp1 = gen_reg_rtx (V64QImode);
17429 49 : emit_move_insn (tmp1, CONSTM1_RTX (V64QImode));
17430 49 : tmp2 = gen_reg_rtx (V64QImode);
17431 49 : emit_insn (gen_absv64qi2 (tmp2, tmp1));
17432 : }
17433 : else
17434 : return false;
17435 : break;
17436 :
17437 103 : case VEC_BCAST_PADDB:
17438 103 : if (mode == V4SImode && TARGET_SSE2)
17439 : {
17440 99 : tmp1 = gen_reg_rtx (V16QImode);
17441 99 : emit_move_insn (tmp1, CONSTM1_RTX (V16QImode));
17442 99 : tmp2 = gen_reg_rtx (V16QImode);
17443 99 : emit_insn (gen_addv16qi3 (tmp2, tmp1, tmp1));
17444 : }
17445 4 : else if (mode == V8SImode && TARGET_AVX2)
17446 : {
17447 1 : tmp1 = gen_reg_rtx (V32QImode);
17448 1 : emit_move_insn (tmp1, CONSTM1_RTX (V32QImode));
17449 1 : tmp2 = gen_reg_rtx (V32QImode);
17450 1 : emit_insn (gen_addv32qi3 (tmp2, tmp1, tmp1));
17451 : }
17452 3 : else if (mode == V16SImode && TARGET_AVX512BW)
17453 : {
17454 3 : tmp1 = gen_reg_rtx (V64QImode);
17455 3 : emit_move_insn (tmp1, CONSTM1_RTX (V64QImode));
17456 3 : tmp2 = gen_reg_rtx (V64QImode);
17457 3 : emit_insn (gen_addv64qi3 (tmp2, tmp1, tmp1));
17458 : }
17459 : else
17460 : return false;
17461 : break;
17462 :
17463 3764 : case VEC_BCAST_PSRLW:
17464 3764 : if (mode == V4SImode && TARGET_SSE2)
17465 : {
17466 3540 : tmp1 = gen_reg_rtx (V8HImode);
17467 3540 : emit_move_insn (tmp1, CONSTM1_RTX (V8HImode));
17468 3540 : tmp2 = gen_reg_rtx (V8HImode);
17469 3540 : emit_insn (gen_lshrv8hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
17470 : }
17471 224 : else if (mode == V8SImode && TARGET_AVX2)
17472 : {
17473 131 : tmp1 = gen_reg_rtx (V16HImode);
17474 131 : emit_move_insn (tmp1, CONSTM1_RTX (V16HImode));
17475 131 : tmp2 = gen_reg_rtx (V16HImode);
17476 131 : emit_insn (gen_lshrv16hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
17477 : }
17478 93 : else if (mode == V16SImode && TARGET_AVX512BW)
17479 : {
17480 90 : tmp1 = gen_reg_rtx (V32HImode);
17481 90 : emit_move_insn (tmp1, CONSTM1_RTX (V32HImode));
17482 90 : tmp2 = gen_reg_rtx (V32HImode);
17483 90 : emit_insn (gen_lshrv32hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
17484 : }
17485 : else
17486 : return false;
17487 : break;
17488 :
17489 10287 : case VEC_BCAST_PSRLD:
17490 10287 : if (mode == V4SImode && TARGET_SSE2)
17491 : {
17492 7441 : tmp1 = gen_reg_rtx (V4SImode);
17493 7441 : emit_move_insn (tmp1, CONSTM1_RTX (V4SImode));
17494 7441 : emit_insn (gen_lshrv4si3 (target, tmp1, GEN_INT (entry->arg)));
17495 7441 : return true;
17496 : }
17497 2846 : else if (mode == V8SImode && TARGET_AVX2)
17498 : {
17499 1055 : tmp1 = gen_reg_rtx (V8SImode);
17500 1055 : emit_move_insn (tmp1, CONSTM1_RTX (V8SImode));
17501 1055 : emit_insn (gen_lshrv8si3 (target, tmp1, GEN_INT (entry->arg)));
17502 1055 : return true;
17503 : }
17504 1791 : else if (mode == V16SImode && TARGET_AVX512F)
17505 : {
17506 952 : tmp1 = gen_reg_rtx (V16SImode);
17507 952 : emit_move_insn (tmp1, CONSTM1_RTX (V16SImode));
17508 952 : emit_insn (gen_lshrv16si3 (target, tmp1, GEN_INT (entry->arg)));
17509 952 : return true;
17510 : }
17511 : else
17512 : return false;
17513 134 : break;
17514 :
17515 134 : case VEC_BCAST_PSLLW:
17516 134 : if (mode == V4SImode && TARGET_SSE2)
17517 : {
17518 104 : tmp1 = gen_reg_rtx (V8HImode);
17519 104 : emit_move_insn (tmp1, CONSTM1_RTX (V8HImode));
17520 104 : tmp2 = gen_reg_rtx (V8HImode);
17521 104 : emit_insn (gen_ashlv8hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
17522 : }
17523 30 : else if (mode == V8SImode && TARGET_AVX2)
17524 : {
17525 21 : tmp1 = gen_reg_rtx (V16HImode);
17526 21 : emit_move_insn (tmp1, CONSTM1_RTX (V16HImode));
17527 21 : tmp2 = gen_reg_rtx (V16HImode);
17528 21 : emit_insn (gen_ashlv16hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
17529 : }
17530 9 : else if (mode == V16SImode && TARGET_AVX512BW)
17531 : {
17532 9 : tmp1 = gen_reg_rtx (V32HImode);
17533 9 : emit_move_insn (tmp1, CONSTM1_RTX (V32HImode));
17534 9 : tmp2 = gen_reg_rtx (V32HImode);
17535 9 : emit_insn (gen_ashlv32hi3 (tmp2, tmp1, GEN_INT (entry->arg)));
17536 : }
17537 : else
17538 : return false;
17539 : break;
17540 :
17541 1744 : case VEC_BCAST_PSLLD:
17542 1744 : if (mode == V4SImode && TARGET_SSE2)
17543 : {
17544 1708 : tmp1 = gen_reg_rtx (V4SImode);
17545 1708 : emit_move_insn (tmp1, CONSTM1_RTX (V4SImode));
17546 1708 : emit_insn (gen_ashlv4si3 (target, tmp1, GEN_INT (entry->arg)));
17547 1708 : return true;
17548 : }
17549 36 : else if (mode == V8SImode && TARGET_AVX2)
17550 : {
17551 18 : tmp1 = gen_reg_rtx (V8SImode);
17552 18 : emit_move_insn (tmp1, CONSTM1_RTX (V8SImode));
17553 18 : emit_insn (gen_ashlv8si3 (target, tmp1, GEN_INT (entry->arg)));
17554 18 : return true;
17555 : }
17556 18 : else if (mode == V16SImode && TARGET_AVX512F)
17557 : {
17558 18 : tmp1 = gen_reg_rtx (V16SImode);
17559 18 : emit_move_insn (tmp1, CONSTM1_RTX (V16SImode));
17560 18 : emit_insn (gen_ashlv16si3 (target, tmp1, GEN_INT (entry->arg)));
17561 18 : return true;
17562 : }
17563 : else
17564 : return false;
17565 :
17566 : default:
17567 : return false;
17568 : }
17569 :
17570 4606 : emit_move_insn (target, gen_lowpart (mode, tmp2));
17571 4606 : return true;
17572 : }
17573 :
17574 : /* A subroutine of ix86_expand_vector_init_duplicate. Tries to
17575 : fill target with val via vec_duplicate. */
17576 :
17577 : static bool
17578 152696 : ix86_vector_duplicate_value (machine_mode mode, rtx target, rtx val)
17579 : {
17580 152696 : bool ok;
17581 152696 : rtx_insn *insn;
17582 152696 : rtx dup;
17583 :
17584 152696 : if ((mode == V4SImode || mode == V8SImode || mode == V16SImode)
17585 59915 : && CONST_INT_P (val)
17586 51973 : && ix86_vector_duplicate_simode_const (mode, target, INTVAL (val)))
17587 : return true;
17588 :
17589 : /* Save/restore recog_data in case this is called from splitters
17590 : or other routines where recog_data needs to stay valid across
17591 : force_reg. See PR106577. */
17592 134043 : recog_data_d recog_data_save = recog_data;
17593 :
17594 : /* First attempt to recognize VAL as-is. */
17595 134043 : dup = gen_vec_duplicate (mode, val);
17596 134043 : insn = emit_insn (gen_rtx_SET (target, dup));
17597 134043 : if (recog_memoized (insn) < 0)
17598 : {
17599 95887 : rtx_insn *seq;
17600 95887 : machine_mode innermode = GET_MODE_INNER (mode);
17601 95887 : rtx reg;
17602 :
17603 : /* If that fails, force VAL into a register or mem. */
17604 :
17605 95887 : start_sequence ();
17606 :
17607 0 : if (!TARGET_PREFER_BCST_FROM_INTEGER && CONST_INT_P (val)
17608 0 : && GET_MODE_BITSIZE (innermode) <= HOST_BITS_PER_WIDE_INT
17609 95887 : && GET_MODE_BITSIZE(mode) >= 128)
17610 0 : reg = validize_mem (force_const_mem (innermode, val));
17611 : else
17612 : {
17613 95887 : reg = force_reg (innermode, val);
17614 95887 : if (GET_MODE (reg) != innermode)
17615 0 : reg = gen_lowpart (innermode, reg);
17616 : }
17617 :
17618 95887 : SET_SRC (PATTERN (insn)) = gen_vec_duplicate (mode, reg);
17619 95887 : seq = end_sequence ();
17620 95887 : if (seq)
17621 95887 : emit_insn_before (seq, insn);
17622 :
17623 95887 : ok = recog_memoized (insn) >= 0;
17624 95887 : gcc_assert (ok);
17625 : }
17626 134043 : recog_data = recog_data_save;
17627 134043 : return true;
17628 : }
17629 :
17630 : /* Get a vector mode of the same size as the original but with elements
17631 : twice as wide. This is only guaranteed to apply to integral vectors. */
17632 :
17633 : static machine_mode
17634 25949 : get_mode_wider_vector (machine_mode o)
17635 : {
17636 : /* ??? Rely on the ordering that genmodes.cc gives to vectors. */
17637 25949 : machine_mode n = GET_MODE_NEXT_MODE (o).require ();
17638 77847 : gcc_assert (GET_MODE_NUNITS (o) == GET_MODE_NUNITS (n) * 2);
17639 77847 : gcc_assert (GET_MODE_SIZE (o) == GET_MODE_SIZE (n));
17640 25949 : return n;
17641 : }
17642 :
17643 : static bool expand_vec_perm_broadcast_1 (struct expand_vec_perm_d *d);
17644 : static bool expand_vec_perm_1 (struct expand_vec_perm_d *d);
17645 :
17646 : /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
17647 : with all elements equal to VAR. Return true if successful. */
17648 :
17649 : bool
17650 179922 : ix86_expand_vector_init_duplicate (bool mmx_ok, machine_mode mode,
17651 : rtx target, rtx val)
17652 : {
17653 179922 : bool ok;
17654 :
17655 179922 : switch (mode)
17656 : {
17657 70922 : case E_V2DImode:
17658 70922 : if (CONST_INT_P (val))
17659 : {
17660 61711 : int tmp = (int)INTVAL (val);
17661 61711 : if (tmp == (int)(INTVAL (val) >> 32))
17662 : {
17663 93 : rtx reg = gen_reg_rtx (V4SImode);
17664 93 : ok = ix86_vector_duplicate_value (V4SImode, reg,
17665 : GEN_INT (tmp));
17666 93 : if (ok)
17667 : {
17668 93 : emit_move_insn (target, gen_lowpart (V2DImode, reg));
17669 93 : return true;
17670 : }
17671 : }
17672 : }
17673 70829 : return ix86_vector_duplicate_value (mode, target, val);
17674 :
17675 1007 : case E_V4DImode:
17676 1007 : if (CONST_INT_P (val))
17677 : {
17678 718 : int tmp = (int)INTVAL (val);
17679 718 : if (tmp == (int)(INTVAL (val) >> 32))
17680 : {
17681 54 : rtx reg = gen_reg_rtx (V8SImode);
17682 54 : ok = ix86_vector_duplicate_value (V8SImode, reg,
17683 : GEN_INT (tmp));
17684 54 : if (ok)
17685 : {
17686 54 : emit_move_insn (target, gen_lowpart (V4DImode, reg));
17687 54 : return true;
17688 : }
17689 : }
17690 : }
17691 953 : return ix86_vector_duplicate_value (mode, target, val);
17692 :
17693 464 : case E_V8DImode:
17694 464 : if (CONST_INT_P (val))
17695 : {
17696 264 : int tmp = (int)INTVAL (val);
17697 264 : if (tmp == (int)(INTVAL (val) >> 32))
17698 : {
17699 24 : rtx reg = gen_reg_rtx (V16SImode);
17700 24 : ok = ix86_vector_duplicate_value (V16SImode, reg,
17701 : GEN_INT (tmp));
17702 24 : if (ok)
17703 : {
17704 24 : emit_move_insn (target, gen_lowpart (V8DImode, reg));
17705 24 : return true;
17706 : }
17707 : }
17708 : }
17709 440 : return ix86_vector_duplicate_value (mode, target, val);
17710 :
17711 2684 : case E_V2SImode:
17712 2684 : case E_V2SFmode:
17713 2684 : if (!mmx_ok)
17714 : return false;
17715 : /* FALLTHRU */
17716 :
17717 79317 : case E_V4DFmode:
17718 79317 : case E_V8SFmode:
17719 79317 : case E_V8SImode:
17720 79317 : case E_V2DFmode:
17721 79317 : case E_V4SFmode:
17722 79317 : case E_V4SImode:
17723 79317 : case E_V16SImode:
17724 79317 : case E_V16SFmode:
17725 79317 : case E_V8DFmode:
17726 79317 : return ix86_vector_duplicate_value (mode, target, val);
17727 :
17728 387 : case E_V4HImode:
17729 387 : if (!mmx_ok)
17730 : return false;
17731 384 : if (TARGET_SSE || TARGET_3DNOW_A)
17732 : {
17733 384 : rtx x;
17734 :
17735 384 : val = gen_lowpart (SImode, val);
17736 384 : if (CONST_INT_P (val))
17737 : return false;
17738 382 : x = gen_rtx_TRUNCATE (HImode, val);
17739 382 : x = gen_rtx_VEC_DUPLICATE (mode, x);
17740 382 : emit_insn (gen_rtx_SET (target, x));
17741 382 : return true;
17742 : }
17743 0 : goto widen;
17744 :
17745 5 : case E_V4HFmode:
17746 5 : case E_V4BFmode:
17747 5 : if (TARGET_MMX_WITH_SSE)
17748 : {
17749 10 : val = force_reg (GET_MODE_INNER (mode), val);
17750 5 : rtx x = gen_rtx_VEC_DUPLICATE (mode, val);
17751 5 : emit_insn (gen_rtx_SET (target, x));
17752 5 : return true;
17753 : }
17754 : return false;
17755 :
17756 125 : case E_V2HImode:
17757 125 : if (TARGET_SSE2)
17758 : {
17759 125 : rtx x;
17760 :
17761 125 : val = gen_lowpart (SImode, val);
17762 125 : if (CONST_INT_P (val))
17763 : return false;
17764 125 : x = gen_rtx_TRUNCATE (HImode, val);
17765 125 : x = gen_rtx_VEC_DUPLICATE (mode, x);
17766 125 : emit_insn (gen_rtx_SET (target, x));
17767 125 : return true;
17768 : }
17769 : return false;
17770 :
17771 5 : case E_V2HFmode:
17772 5 : case E_V2BFmode:
17773 5 : if (TARGET_SSE2)
17774 : {
17775 10 : val = force_reg (GET_MODE_INNER (mode), val);
17776 5 : rtx x = gen_rtx_VEC_DUPLICATE (mode, val);
17777 5 : emit_insn (gen_rtx_SET (target, x));
17778 5 : return true;
17779 : }
17780 : return false;
17781 :
17782 297 : case E_V8QImode:
17783 297 : case E_V4QImode:
17784 297 : if (!mmx_ok)
17785 : return false;
17786 296 : goto widen;
17787 :
17788 13668 : case E_V8HImode:
17789 13668 : if (CONST_INT_P (val))
17790 13124 : goto widen;
17791 : /* FALLTHRU */
17792 :
17793 858 : case E_V8HFmode:
17794 858 : case E_V8BFmode:
17795 858 : if (TARGET_AVX2)
17796 392 : return ix86_vector_duplicate_value (mode, target, val);
17797 :
17798 466 : if (TARGET_SSE2)
17799 : {
17800 1182 : struct expand_vec_perm_d dperm;
17801 1182 : rtx tmp1, tmp2;
17802 :
17803 466 : permute:
17804 1182 : memset (&dperm, 0, sizeof (dperm));
17805 1182 : dperm.target = target;
17806 1182 : dperm.vmode = mode;
17807 1182 : dperm.nelt = GET_MODE_NUNITS (mode);
17808 1182 : dperm.op0 = dperm.op1 = gen_reg_rtx (mode);
17809 1182 : dperm.one_operand_p = true;
17810 :
17811 1182 : if (mode == V8HFmode || mode == V8BFmode)
17812 : {
17813 3 : tmp1 = force_reg (GET_MODE_INNER (mode), val);
17814 3 : tmp2 = gen_reg_rtx (mode);
17815 3 : emit_insn (gen_vec_set_0 (mode, tmp2, CONST0_RTX (mode), tmp1));
17816 3 : tmp1 = gen_lowpart (mode, tmp2);
17817 : }
17818 : else
17819 : {
17820 : /* Extend to SImode using a paradoxical SUBREG. */
17821 1179 : tmp1 = gen_reg_rtx (SImode);
17822 1179 : emit_move_insn (tmp1, gen_lowpart (SImode, val));
17823 :
17824 : /* Insert the SImode value as
17825 : low element of a V4SImode vector. */
17826 1179 : tmp2 = gen_reg_rtx (V4SImode);
17827 1179 : emit_insn (gen_vec_setv4si_0 (tmp2, CONST0_RTX (V4SImode), tmp1));
17828 1179 : tmp1 = gen_lowpart (mode, tmp2);
17829 : }
17830 :
17831 1182 : emit_move_insn (dperm.op0, tmp1);
17832 1182 : ok = (expand_vec_perm_1 (&dperm)
17833 1182 : || expand_vec_perm_broadcast_1 (&dperm));
17834 0 : gcc_assert (ok);
17835 1182 : return ok;
17836 : }
17837 0 : goto widen;
17838 :
17839 9262 : case E_V16QImode:
17840 9262 : if (CONST_INT_P (val))
17841 8486 : goto widen;
17842 776 : if (TARGET_AVX2)
17843 60 : return ix86_vector_duplicate_value (mode, target, val);
17844 :
17845 716 : if (TARGET_SSE2)
17846 716 : goto permute;
17847 0 : goto widen;
17848 :
17849 24325 : widen:
17850 : /* Replicate the value once into the next wider mode and recurse. */
17851 24325 : {
17852 24325 : machine_mode smode, wsmode, wvmode;
17853 24325 : rtx x;
17854 :
17855 24325 : smode = GET_MODE_INNER (mode);
17856 24325 : wvmode = get_mode_wider_vector (mode);
17857 24325 : wsmode = GET_MODE_INNER (wvmode);
17858 :
17859 24325 : val = convert_modes (wsmode, smode, val, true);
17860 :
17861 24325 : if (CONST_INT_P (val))
17862 : {
17863 48060 : x = simplify_binary_operation (ASHIFT, wsmode, val,
17864 24030 : GEN_INT (GET_MODE_BITSIZE (smode)));
17865 24030 : val = simplify_binary_operation (IOR, wsmode, val, x);
17866 : }
17867 295 : else if (smode == QImode && !TARGET_PARTIAL_REG_STALL)
17868 295 : emit_insn (gen_insv_1 (wsmode, val, val));
17869 : else
17870 : {
17871 0 : x = expand_simple_binop (wsmode, ASHIFT, val,
17872 0 : GEN_INT (GET_MODE_BITSIZE (smode)),
17873 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
17874 0 : val = expand_simple_binop (wsmode, IOR, val, x, x, 1,
17875 : OPTAB_LIB_WIDEN);
17876 : }
17877 :
17878 24325 : x = gen_reg_rtx (wvmode);
17879 24325 : ok = ix86_expand_vector_init_duplicate (mmx_ok, wvmode, x, val);
17880 24325 : if (!ok)
17881 : return false;
17882 24324 : emit_move_insn (target, gen_lowpart (GET_MODE (target), x));
17883 24324 : return true;
17884 : }
17885 :
17886 1538 : case E_V16HImode:
17887 1538 : case E_V32QImode:
17888 1538 : if (CONST_INT_P (val))
17889 1246 : goto widen;
17890 : /* FALLTHRU */
17891 :
17892 375 : case E_V16HFmode:
17893 375 : case E_V16BFmode:
17894 375 : if (TARGET_AVX2)
17895 347 : return ix86_vector_duplicate_value (mode, target, val);
17896 : else
17897 : {
17898 28 : machine_mode hvmode;
17899 28 : switch (mode)
17900 : {
17901 : case V16HImode:
17902 : hvmode = V8HImode;
17903 : break;
17904 0 : case V16HFmode:
17905 0 : hvmode = V8HFmode;
17906 0 : break;
17907 1 : case V16BFmode:
17908 1 : hvmode = V8BFmode;
17909 1 : break;
17910 14 : case V32QImode:
17911 14 : hvmode = V16QImode;
17912 14 : break;
17913 0 : default:
17914 0 : gcc_unreachable ();
17915 : }
17916 28 : rtx x = gen_reg_rtx (hvmode);
17917 :
17918 28 : ok = ix86_expand_vector_init_duplicate (false, hvmode, x, val);
17919 28 : if (!ok)
17920 : return false;
17921 :
17922 28 : x = gen_rtx_VEC_CONCAT (mode, x, x);
17923 28 : emit_insn (gen_rtx_SET (target, x));
17924 : }
17925 28 : return true;
17926 :
17927 1301 : case E_V32HImode:
17928 1301 : case E_V64QImode:
17929 1301 : if (CONST_INT_P (val))
17930 1173 : goto widen;
17931 : /* FALLTHRU */
17932 :
17933 207 : case E_V32HFmode:
17934 207 : case E_V32BFmode:
17935 207 : if (TARGET_AVX512BW)
17936 187 : return ix86_vector_duplicate_value (mode, target, val);
17937 : else
17938 : {
17939 20 : machine_mode hvmode;
17940 20 : switch (mode)
17941 : {
17942 : case V32HImode:
17943 : hvmode = V16HImode;
17944 : break;
17945 0 : case V32HFmode:
17946 0 : hvmode = V16HFmode;
17947 0 : break;
17948 1 : case V32BFmode:
17949 1 : hvmode = V16BFmode;
17950 1 : break;
17951 10 : case V64QImode:
17952 10 : hvmode = V32QImode;
17953 10 : break;
17954 0 : default:
17955 0 : gcc_unreachable ();
17956 : }
17957 20 : rtx x = gen_reg_rtx (hvmode);
17958 :
17959 20 : ok = ix86_expand_vector_init_duplicate (false, hvmode, x, val);
17960 20 : if (!ok)
17961 : return false;
17962 :
17963 20 : x = gen_rtx_VEC_CONCAT (mode, x, x);
17964 20 : emit_insn (gen_rtx_SET (target, x));
17965 : }
17966 20 : return true;
17967 :
17968 : default:
17969 : return false;
17970 : }
17971 : }
17972 :
17973 : /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
17974 : whose ONE_VAR element is VAR, and other elements are zero. Return true
17975 : if successful. */
17976 :
17977 : bool
17978 10861 : ix86_expand_vector_init_one_nonzero (bool mmx_ok, machine_mode mode,
17979 : rtx target, rtx var, int one_var)
17980 : {
17981 10861 : machine_mode vsimode;
17982 10861 : rtx new_target;
17983 10861 : rtx x, tmp;
17984 10861 : bool use_vector_set = false;
17985 10861 : rtx (*gen_vec_set_0) (rtx, rtx, rtx) = NULL;
17986 :
17987 10861 : switch (mode)
17988 : {
17989 8375 : case E_V2DImode:
17990 : /* For SSE4.1, we normally use vector set. But if the second
17991 : element is zero and inter-unit moves are OK, we use movq
17992 : instead. */
17993 8366 : use_vector_set = (TARGET_64BIT && TARGET_SSE4_1
17994 8498 : && !(TARGET_INTER_UNIT_MOVES_TO_VEC
17995 : && one_var == 0));
17996 : break;
17997 871 : case E_V16QImode:
17998 871 : case E_V4SImode:
17999 871 : case E_V4SFmode:
18000 871 : use_vector_set = TARGET_SSE4_1;
18001 871 : break;
18002 86 : case E_V8HImode:
18003 86 : use_vector_set = TARGET_SSE2;
18004 86 : gen_vec_set_0 = TARGET_AVX512FP16 && one_var == 0
18005 86 : ? gen_vec_setv8hi_0 : NULL;
18006 : break;
18007 4 : case E_V8QImode:
18008 4 : use_vector_set = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
18009 : break;
18010 14 : case E_V4HImode:
18011 14 : case E_V4HFmode:
18012 14 : case E_V4BFmode:
18013 14 : use_vector_set = TARGET_SSE || TARGET_3DNOW_A;
18014 : break;
18015 32 : case E_V4QImode:
18016 32 : use_vector_set = TARGET_SSE4_1;
18017 32 : break;
18018 0 : case E_V32QImode:
18019 0 : use_vector_set = TARGET_AVX;
18020 0 : break;
18021 5 : case E_V16HImode:
18022 5 : use_vector_set = TARGET_AVX;
18023 5 : gen_vec_set_0 = TARGET_AVX512FP16 && one_var == 0
18024 5 : ? gen_vec_setv16hi_0 : NULL;
18025 : break;
18026 5 : case E_V8SImode:
18027 5 : use_vector_set = TARGET_AVX;
18028 5 : gen_vec_set_0 = gen_vec_setv8si_0;
18029 5 : break;
18030 22 : case E_V8SFmode:
18031 22 : use_vector_set = TARGET_AVX;
18032 22 : gen_vec_set_0 = gen_vec_setv8sf_0;
18033 22 : break;
18034 13 : case E_V4DFmode:
18035 13 : use_vector_set = TARGET_AVX;
18036 13 : gen_vec_set_0 = gen_vec_setv4df_0;
18037 13 : break;
18038 7 : case E_V4DImode:
18039 : /* Use ix86_expand_vector_set in 64bit mode only. */
18040 7 : use_vector_set = TARGET_AVX && TARGET_64BIT;
18041 : gen_vec_set_0 = gen_vec_setv4di_0;
18042 : break;
18043 17 : case E_V16SImode:
18044 17 : use_vector_set = TARGET_AVX512F && one_var == 0;
18045 : gen_vec_set_0 = gen_vec_setv16si_0;
18046 : break;
18047 22 : case E_V16SFmode:
18048 22 : use_vector_set = TARGET_AVX512F && one_var == 0;
18049 : gen_vec_set_0 = gen_vec_setv16sf_0;
18050 : break;
18051 0 : case E_V8DFmode:
18052 0 : use_vector_set = TARGET_AVX512F && one_var == 0;
18053 : gen_vec_set_0 = gen_vec_setv8df_0;
18054 : break;
18055 2 : case E_V8DImode:
18056 : /* Use ix86_expand_vector_set in 64bit mode only. */
18057 2 : use_vector_set = TARGET_AVX512F && TARGET_64BIT && one_var == 0;
18058 : gen_vec_set_0 = gen_vec_setv8di_0;
18059 : break;
18060 39 : case E_V8HFmode:
18061 39 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18062 : gen_vec_set_0 = gen_vec_setv8hf_0;
18063 : break;
18064 9 : case E_V16HFmode:
18065 9 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18066 : gen_vec_set_0 = gen_vec_setv16hf_0;
18067 : break;
18068 6 : case E_V32HFmode:
18069 6 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18070 : gen_vec_set_0 = gen_vec_setv32hf_0;
18071 : break;
18072 2 : case E_V8BFmode:
18073 2 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18074 : gen_vec_set_0 = gen_vec_setv8bf_0;
18075 : break;
18076 0 : case E_V16BFmode:
18077 0 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18078 : gen_vec_set_0 = gen_vec_setv16bf_0;
18079 : break;
18080 0 : case E_V32BFmode:
18081 0 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18082 : gen_vec_set_0 = gen_vec_setv32bf_0;
18083 : break;
18084 4 : case E_V32HImode:
18085 4 : use_vector_set = TARGET_AVX512FP16 && one_var == 0;
18086 : gen_vec_set_0 = gen_vec_setv32hi_0;
18087 : default:
18088 : break;
18089 : }
18090 :
18091 9423 : if (use_vector_set)
18092 : {
18093 858 : if (gen_vec_set_0 && one_var == 0)
18094 : {
18095 354 : var = force_reg (GET_MODE_INNER (mode), var);
18096 177 : emit_insn (gen_vec_set_0 (target, CONST0_RTX (mode), var));
18097 177 : return true;
18098 : }
18099 681 : emit_insn (gen_rtx_SET (target, CONST0_RTX (mode)));
18100 1362 : var = force_reg (GET_MODE_INNER (mode), var);
18101 681 : ix86_expand_vector_set (mmx_ok, target, var, one_var);
18102 681 : return true;
18103 : }
18104 :
18105 10003 : switch (mode)
18106 : {
18107 1216 : case E_V2SFmode:
18108 1216 : case E_V2SImode:
18109 1216 : if (!mmx_ok)
18110 : return false;
18111 : /* FALLTHRU */
18112 :
18113 8633 : case E_V2DFmode:
18114 8633 : case E_V2DImode:
18115 8633 : if (one_var != 0)
18116 : return false;
18117 5268 : var = force_reg (GET_MODE_INNER (mode), var);
18118 5268 : x = gen_rtx_VEC_CONCAT (mode, var, CONST0_RTX (GET_MODE_INNER (mode)));
18119 2634 : emit_insn (gen_rtx_SET (target, x));
18120 2634 : return true;
18121 :
18122 306 : case E_V4SFmode:
18123 306 : case E_V4SImode:
18124 306 : if (!REG_P (target) || REGNO (target) < FIRST_PSEUDO_REGISTER)
18125 0 : new_target = gen_reg_rtx (mode);
18126 : else
18127 : new_target = target;
18128 612 : var = force_reg (GET_MODE_INNER (mode), var);
18129 306 : x = gen_rtx_VEC_DUPLICATE (mode, var);
18130 306 : x = gen_rtx_VEC_MERGE (mode, x, CONST0_RTX (mode), const1_rtx);
18131 306 : emit_insn (gen_rtx_SET (new_target, x));
18132 306 : if (one_var != 0)
18133 : {
18134 : /* We need to shuffle the value to the correct position, so
18135 : create a new pseudo to store the intermediate result. */
18136 :
18137 : /* With SSE2, we can use the integer shuffle insns. */
18138 44 : if (mode != V4SFmode && TARGET_SSE2)
18139 : {
18140 31 : emit_insn (gen_sse2_pshufd_1 (new_target, new_target,
18141 : const1_rtx,
18142 31 : GEN_INT (one_var == 1 ? 0 : 1),
18143 31 : GEN_INT (one_var == 2 ? 0 : 1),
18144 31 : GEN_INT (one_var == 3 ? 0 : 1)));
18145 31 : if (target != new_target)
18146 0 : emit_move_insn (target, new_target);
18147 31 : return true;
18148 : }
18149 :
18150 : /* Otherwise convert the intermediate result to V4SFmode and
18151 : use the SSE1 shuffle instructions. */
18152 0 : if (mode != V4SFmode)
18153 : {
18154 0 : tmp = gen_reg_rtx (V4SFmode);
18155 0 : emit_move_insn (tmp, gen_lowpart (V4SFmode, new_target));
18156 : }
18157 : else
18158 : tmp = new_target;
18159 :
18160 43 : emit_insn (gen_sse_shufps_v4sf (tmp, tmp, tmp,
18161 : const1_rtx,
18162 13 : GEN_INT (one_var == 1 ? 0 : 1),
18163 : GEN_INT (one_var == 2 ? 0+4 : 1+4),
18164 : GEN_INT (one_var == 3 ? 0+4 : 1+4)));
18165 :
18166 13 : if (mode != V4SFmode)
18167 0 : emit_move_insn (target, gen_lowpart (V4SImode, tmp));
18168 13 : else if (tmp != target)
18169 0 : emit_move_insn (target, tmp);
18170 : }
18171 262 : else if (target != new_target)
18172 0 : emit_move_insn (target, new_target);
18173 : return true;
18174 :
18175 14 : case E_V8HImode:
18176 14 : case E_V16QImode:
18177 14 : vsimode = V4SImode;
18178 14 : goto widen;
18179 3 : case E_V4HImode:
18180 3 : case E_V8QImode:
18181 3 : if (!mmx_ok)
18182 : return false;
18183 3 : vsimode = V2SImode;
18184 3 : goto widen;
18185 17 : widen:
18186 17 : if (one_var != 0)
18187 : return false;
18188 :
18189 : /* Zero extend the variable element to SImode and recurse. */
18190 18 : var = convert_modes (SImode, GET_MODE_INNER (mode), var, true);
18191 :
18192 9 : x = gen_reg_rtx (vsimode);
18193 9 : if (!ix86_expand_vector_init_one_nonzero (mmx_ok, vsimode, x,
18194 : var, one_var))
18195 0 : gcc_unreachable ();
18196 :
18197 9 : emit_move_insn (target, gen_lowpart (mode, x));
18198 9 : return true;
18199 :
18200 : default:
18201 : return false;
18202 : }
18203 : }
18204 :
18205 : /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
18206 : consisting of the values in VALS. It is known that all elements
18207 : except ONE_VAR are constants. Return true if successful. */
18208 :
18209 : static bool
18210 8367 : ix86_expand_vector_init_one_var (bool mmx_ok, machine_mode mode,
18211 : rtx target, rtx vals, int one_var)
18212 : {
18213 8367 : rtx var = XVECEXP (vals, 0, one_var);
18214 8367 : machine_mode wmode;
18215 8367 : rtx const_vec, x;
18216 :
18217 8367 : const_vec = copy_rtx (vals);
18218 8367 : XVECEXP (const_vec, 0, one_var) = CONST0_RTX (GET_MODE_INNER (mode));
18219 8367 : const_vec = gen_rtx_CONST_VECTOR (mode, XVEC (const_vec, 0));
18220 :
18221 8367 : switch (mode)
18222 : {
18223 : case E_V2DFmode:
18224 : case E_V2DImode:
18225 : case E_V2SFmode:
18226 : case E_V2SImode:
18227 : /* For the two element vectors, it's just as easy to use
18228 : the general case. */
18229 : return false;
18230 :
18231 3 : case E_V4DImode:
18232 : /* Use ix86_expand_vector_set in 64bit mode only. */
18233 3 : if (!TARGET_64BIT)
18234 : return false;
18235 : /* FALLTHRU */
18236 : case E_V8HFmode:
18237 : case E_V16HFmode:
18238 : case E_V8BFmode:
18239 : case E_V16BFmode:
18240 : case E_V4DFmode:
18241 : case E_V8SFmode:
18242 : case E_V8SImode:
18243 : case E_V16HImode:
18244 : case E_V32QImode:
18245 : case E_V4SFmode:
18246 : case E_V4SImode:
18247 : case E_V8HImode:
18248 : case E_V4HImode:
18249 : case E_V4HFmode:
18250 : case E_V4BFmode:
18251 : break;
18252 :
18253 8 : case E_V16QImode:
18254 8 : if (TARGET_SSE4_1)
18255 : break;
18256 8 : wmode = V8HImode;
18257 8 : goto widen;
18258 1 : case E_V8QImode:
18259 1 : if (TARGET_MMX_WITH_SSE && TARGET_SSE4_1)
18260 : break;
18261 1 : wmode = V4HImode;
18262 1 : goto widen;
18263 38 : case E_V4QImode:
18264 38 : if (TARGET_SSE4_1)
18265 : break;
18266 : wmode = V2HImode;
18267 47 : widen:
18268 : /* There's no way to set one QImode entry easily. Combine
18269 : the variable value with its adjacent constant value, and
18270 : promote to an HImode set. */
18271 47 : x = XVECEXP (vals, 0, one_var ^ 1);
18272 47 : if (one_var & 1)
18273 : {
18274 13 : var = convert_modes (HImode, QImode, var, true);
18275 13 : var = expand_simple_binop (HImode, ASHIFT, var, GEN_INT (8),
18276 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
18277 13 : x = GEN_INT (INTVAL (x) & 0xff);
18278 : }
18279 : else
18280 : {
18281 34 : var = convert_modes (HImode, QImode, var, true);
18282 34 : x = gen_int_mode (UINTVAL (x) << 8, HImode);
18283 : }
18284 47 : if (x != const0_rtx)
18285 7 : var = expand_simple_binop (HImode, IOR, var, x, var,
18286 : 1, OPTAB_LIB_WIDEN);
18287 :
18288 47 : x = gen_reg_rtx (wmode);
18289 47 : emit_move_insn (x, gen_lowpart (wmode, const_vec));
18290 47 : ix86_expand_vector_set (mmx_ok, x, var, one_var >> 1);
18291 :
18292 47 : emit_move_insn (target, gen_lowpart (mode, x));
18293 47 : return true;
18294 :
18295 : default:
18296 : return false;
18297 : }
18298 :
18299 199 : emit_move_insn (target, const_vec);
18300 199 : ix86_expand_vector_set (mmx_ok, target, var, one_var);
18301 199 : return true;
18302 : }
18303 :
18304 : /* A subroutine of ix86_expand_vector_init_general. Use vector
18305 : concatenate to handle the most general case: all values variable,
18306 : and none identical. */
18307 :
18308 : static void
18309 121010 : ix86_expand_vector_init_concat (machine_mode mode,
18310 : rtx target, rtx *ops, int n)
18311 : {
18312 121010 : machine_mode half_mode = VOIDmode;
18313 121010 : rtx half[2];
18314 121010 : rtvec v;
18315 121010 : int i, j;
18316 :
18317 121010 : switch (n)
18318 : {
18319 112622 : case 2:
18320 112622 : switch (mode)
18321 : {
18322 : case E_V32HFmode:
18323 : half_mode = V16HFmode;
18324 : break;
18325 0 : case E_V32BFmode:
18326 0 : half_mode = V16BFmode;
18327 0 : break;
18328 59 : case E_V16SImode:
18329 59 : half_mode = V8SImode;
18330 59 : break;
18331 33 : case E_V16SFmode:
18332 33 : half_mode = V8SFmode;
18333 33 : break;
18334 84 : case E_V8DImode:
18335 84 : half_mode = V4DImode;
18336 84 : break;
18337 59 : case E_V8DFmode:
18338 59 : half_mode = V4DFmode;
18339 59 : break;
18340 0 : case E_V16HFmode:
18341 0 : half_mode = V8HFmode;
18342 0 : break;
18343 0 : case E_V16BFmode:
18344 0 : half_mode = V8BFmode;
18345 0 : break;
18346 170 : case E_V8SImode:
18347 170 : half_mode = V4SImode;
18348 170 : break;
18349 258 : case E_V8SFmode:
18350 258 : half_mode = V4SFmode;
18351 258 : break;
18352 272 : case E_V4DImode:
18353 272 : half_mode = V2DImode;
18354 272 : break;
18355 551 : case E_V4DFmode:
18356 551 : half_mode = V2DFmode;
18357 551 : break;
18358 6198 : case E_V4SImode:
18359 6198 : half_mode = V2SImode;
18360 6198 : break;
18361 2227 : case E_V4SFmode:
18362 2227 : half_mode = V2SFmode;
18363 2227 : break;
18364 66071 : case E_V2DImode:
18365 66071 : half_mode = DImode;
18366 66071 : break;
18367 27714 : case E_V2SImode:
18368 27714 : half_mode = SImode;
18369 27714 : break;
18370 3565 : case E_V2DFmode:
18371 3565 : half_mode = DFmode;
18372 3565 : break;
18373 5361 : case E_V2SFmode:
18374 5361 : half_mode = SFmode;
18375 5361 : break;
18376 0 : default:
18377 0 : gcc_unreachable ();
18378 : }
18379 :
18380 112622 : if (!register_operand (ops[1], half_mode))
18381 47829 : ops[1] = force_reg (half_mode, ops[1]);
18382 112622 : if (!register_operand (ops[0], half_mode))
18383 36355 : ops[0] = force_reg (half_mode, ops[0]);
18384 112622 : emit_insn (gen_rtx_SET (target, gen_rtx_VEC_CONCAT (mode, ops[0],
18385 : ops[1])));
18386 112622 : break;
18387 :
18388 7830 : case 4:
18389 7830 : switch (mode)
18390 : {
18391 : case E_V4DImode:
18392 : half_mode = V2DImode;
18393 : break;
18394 494 : case E_V4DFmode:
18395 494 : half_mode = V2DFmode;
18396 494 : break;
18397 5242 : case E_V4SImode:
18398 5242 : half_mode = V2SImode;
18399 5242 : break;
18400 1948 : case E_V4SFmode:
18401 1948 : half_mode = V2SFmode;
18402 1948 : break;
18403 0 : default:
18404 0 : gcc_unreachable ();
18405 : }
18406 7830 : goto half;
18407 :
18408 487 : case 8:
18409 487 : switch (mode)
18410 : {
18411 : case E_V8DImode:
18412 : half_mode = V4DImode;
18413 : break;
18414 59 : case E_V8DFmode:
18415 59 : half_mode = V4DFmode;
18416 59 : break;
18417 133 : case E_V8SImode:
18418 133 : half_mode = V4SImode;
18419 133 : break;
18420 252 : case E_V8SFmode:
18421 252 : half_mode = V4SFmode;
18422 252 : break;
18423 0 : default:
18424 0 : gcc_unreachable ();
18425 : }
18426 487 : goto half;
18427 :
18428 71 : case 16:
18429 71 : switch (mode)
18430 : {
18431 : case E_V16SImode:
18432 : half_mode = V8SImode;
18433 : break;
18434 33 : case E_V16SFmode:
18435 33 : half_mode = V8SFmode;
18436 33 : break;
18437 0 : default:
18438 0 : gcc_unreachable ();
18439 : }
18440 71 : goto half;
18441 :
18442 8388 : half:
18443 : /* FIXME: We process inputs backward to help RA. PR 36222. */
18444 8388 : i = n - 1;
18445 25164 : for (j = 1; j != -1; j--)
18446 : {
18447 16776 : half[j] = gen_reg_rtx (half_mode);
18448 16776 : switch (n >> 1)
18449 : {
18450 15660 : case 2:
18451 15660 : v = gen_rtvec (2, ops[i-1], ops[i]);
18452 15660 : i -= 2;
18453 15660 : break;
18454 974 : case 4:
18455 974 : v = gen_rtvec (4, ops[i-3], ops[i-2], ops[i-1], ops[i]);
18456 974 : i -= 4;
18457 974 : break;
18458 142 : case 8:
18459 284 : v = gen_rtvec (8, ops[i-7], ops[i-6], ops[i-5], ops[i-4],
18460 142 : ops[i-3], ops[i-2], ops[i-1], ops[i]);
18461 142 : i -= 8;
18462 142 : break;
18463 0 : default:
18464 0 : gcc_unreachable ();
18465 : }
18466 16776 : ix86_expand_vector_init (false, half[j],
18467 : gen_rtx_PARALLEL (half_mode, v));
18468 : }
18469 :
18470 8388 : ix86_expand_vector_init_concat (mode, target, half, 2);
18471 8388 : break;
18472 :
18473 0 : default:
18474 0 : gcc_unreachable ();
18475 : }
18476 121010 : }
18477 :
18478 : /* A subroutine of ix86_expand_vector_init_general. Use vector
18479 : interleave to handle the most general case: all values variable,
18480 : and none identical. */
18481 :
18482 : static void
18483 3939 : ix86_expand_vector_init_interleave (machine_mode mode,
18484 : rtx target, rtx *ops, int n)
18485 : {
18486 3939 : machine_mode first_imode, second_imode, third_imode, inner_mode;
18487 3939 : int i, j;
18488 3939 : rtx op, op0, op1;
18489 3939 : rtx (*gen_load_even) (rtx, rtx, rtx);
18490 3939 : rtx (*gen_interleave_first_low) (rtx, rtx, rtx);
18491 3939 : rtx (*gen_interleave_second_low) (rtx, rtx, rtx);
18492 :
18493 3939 : switch (mode)
18494 : {
18495 : case E_V8HFmode:
18496 : gen_load_even = gen_vec_interleave_lowv8hf;
18497 : gen_interleave_first_low = gen_vec_interleave_lowv4si;
18498 : gen_interleave_second_low = gen_vec_interleave_lowv2di;
18499 : inner_mode = HFmode;
18500 : first_imode = V4SImode;
18501 : second_imode = V2DImode;
18502 : third_imode = VOIDmode;
18503 : break;
18504 487 : case E_V8BFmode:
18505 487 : gen_load_even = gen_vec_interleave_lowv8bf;
18506 487 : gen_interleave_first_low = gen_vec_interleave_lowv4si;
18507 487 : gen_interleave_second_low = gen_vec_interleave_lowv2di;
18508 487 : inner_mode = BFmode;
18509 487 : first_imode = V4SImode;
18510 487 : second_imode = V2DImode;
18511 487 : third_imode = VOIDmode;
18512 487 : break;
18513 851 : case E_V8HImode:
18514 851 : gen_load_even = gen_vec_setv8hi;
18515 851 : gen_interleave_first_low = gen_vec_interleave_lowv4si;
18516 851 : gen_interleave_second_low = gen_vec_interleave_lowv2di;
18517 851 : inner_mode = HImode;
18518 851 : first_imode = V4SImode;
18519 851 : second_imode = V2DImode;
18520 851 : third_imode = VOIDmode;
18521 851 : break;
18522 374 : case E_V16QImode:
18523 374 : gen_load_even = gen_vec_setv16qi;
18524 374 : gen_interleave_first_low = gen_vec_interleave_lowv8hi;
18525 374 : gen_interleave_second_low = gen_vec_interleave_lowv4si;
18526 374 : inner_mode = QImode;
18527 374 : first_imode = V8HImode;
18528 374 : second_imode = V4SImode;
18529 374 : third_imode = V2DImode;
18530 374 : break;
18531 0 : default:
18532 0 : gcc_unreachable ();
18533 : }
18534 :
18535 21191 : for (i = 0; i < n; i++)
18536 : {
18537 17252 : op = ops [i + i];
18538 17252 : if (inner_mode == HFmode || inner_mode == BFmode)
18539 : {
18540 10856 : rtx even, odd;
18541 : /* Use vpuncklwd to pack 2 HFmode or BFmode. */
18542 1948 : machine_mode vec_mode =
18543 10856 : (inner_mode == HFmode) ? V8HFmode : V8BFmode;
18544 10856 : op0 = gen_reg_rtx (vec_mode);
18545 10856 : even = lowpart_subreg (vec_mode,
18546 : force_reg (inner_mode, op), inner_mode);
18547 10856 : odd = lowpart_subreg (vec_mode,
18548 10856 : force_reg (inner_mode, ops[i + i + 1]),
18549 : inner_mode);
18550 10856 : emit_insn (gen_load_even (op0, even, odd));
18551 : }
18552 : else
18553 : {
18554 : /* Extend the odd element to SImode using a paradoxical SUBREG. */
18555 6396 : op0 = gen_reg_rtx (SImode);
18556 6396 : emit_move_insn (op0, gen_lowpart (SImode, op));
18557 :
18558 : /* Insert the SImode value as low element of V4SImode vector. */
18559 6396 : op1 = gen_reg_rtx (V4SImode);
18560 6396 : op0 = gen_rtx_VEC_MERGE (V4SImode,
18561 : gen_rtx_VEC_DUPLICATE (V4SImode,
18562 : op0),
18563 : CONST0_RTX (V4SImode),
18564 : const1_rtx);
18565 6396 : emit_insn (gen_rtx_SET (op1, op0));
18566 :
18567 : /* Cast the V4SImode vector back to a vector in original mode. */
18568 6396 : op0 = gen_reg_rtx (mode);
18569 6396 : emit_move_insn (op0, gen_lowpart (mode, op1));
18570 :
18571 : /* Load even elements into the second position. */
18572 6396 : emit_insn (gen_load_even (op0,
18573 : force_reg (inner_mode,
18574 6396 : ops[i + i + 1]),
18575 : const1_rtx));
18576 : }
18577 :
18578 : /* Cast vector to FIRST_IMODE vector. */
18579 17252 : ops[i] = gen_reg_rtx (first_imode);
18580 17252 : emit_move_insn (ops[i], gen_lowpart (first_imode, op0));
18581 : }
18582 :
18583 : /* Interleave low FIRST_IMODE vectors. */
18584 12565 : for (i = j = 0; i < n; i += 2, j++)
18585 : {
18586 8626 : op0 = gen_reg_rtx (first_imode);
18587 8626 : emit_insn (gen_interleave_first_low (op0, ops[i], ops[i + 1]));
18588 :
18589 : /* Cast FIRST_IMODE vector to SECOND_IMODE vector. */
18590 8626 : ops[j] = gen_reg_rtx (second_imode);
18591 8626 : emit_move_insn (ops[j], gen_lowpart (second_imode, op0));
18592 : }
18593 :
18594 : /* Interleave low SECOND_IMODE vectors. */
18595 3939 : switch (second_imode)
18596 : {
18597 : case E_V4SImode:
18598 1122 : for (i = j = 0; i < n / 2; i += 2, j++)
18599 : {
18600 748 : op0 = gen_reg_rtx (second_imode);
18601 748 : emit_insn (gen_interleave_second_low (op0, ops[i],
18602 748 : ops[i + 1]));
18603 :
18604 : /* Cast the SECOND_IMODE vector to the THIRD_IMODE
18605 : vector. */
18606 748 : ops[j] = gen_reg_rtx (third_imode);
18607 748 : emit_move_insn (ops[j], gen_lowpart (third_imode, op0));
18608 : }
18609 : second_imode = V2DImode;
18610 : gen_interleave_second_low = gen_vec_interleave_lowv2di;
18611 : /* FALLTHRU */
18612 :
18613 3939 : case E_V2DImode:
18614 3939 : op0 = gen_reg_rtx (second_imode);
18615 3939 : emit_insn (gen_interleave_second_low (op0, ops[0],
18616 : ops[1]));
18617 :
18618 : /* Cast the SECOND_IMODE vector back to a vector on original
18619 : mode. */
18620 3939 : emit_insn (gen_rtx_SET (target, gen_lowpart (mode, op0)));
18621 3939 : break;
18622 :
18623 : default:
18624 : gcc_unreachable ();
18625 : }
18626 3939 : }
18627 :
18628 : /* A subroutine of ix86_expand_vector_init. Handle the most general case:
18629 : all values variable, and none identical. */
18630 :
18631 : static void
18632 121830 : ix86_expand_vector_init_general (bool mmx_ok, machine_mode mode,
18633 : rtx target, rtx vals)
18634 : {
18635 121830 : rtx ops[64], op0, op1, op2, op3, op4, op5;
18636 121830 : machine_mode half_mode = VOIDmode;
18637 121830 : machine_mode quarter_mode = VOIDmode;
18638 121830 : machine_mode int_inner_mode = VOIDmode;
18639 121830 : int n, i;
18640 :
18641 121830 : switch (mode)
18642 : {
18643 33075 : case E_V2SFmode:
18644 33075 : case E_V2SImode:
18645 33075 : if (!mmx_ok && !TARGET_SSE)
18646 : break;
18647 : /* FALLTHRU */
18648 :
18649 111099 : case E_V16SImode:
18650 111099 : case E_V16SFmode:
18651 111099 : case E_V8DFmode:
18652 111099 : case E_V8DImode:
18653 111099 : case E_V8SFmode:
18654 111099 : case E_V8SImode:
18655 111099 : case E_V4DFmode:
18656 111099 : case E_V4DImode:
18657 111099 : case E_V4SFmode:
18658 111099 : case E_V4SImode:
18659 111099 : case E_V2DFmode:
18660 111099 : case E_V2DImode:
18661 111099 : n = GET_MODE_NUNITS (mode);
18662 352873 : for (i = 0; i < n; i++)
18663 241774 : ops[i] = XVECEXP (vals, 0, i);
18664 111099 : ix86_expand_vector_init_concat (mode, target, ops, n);
18665 224361 : return;
18666 :
18667 : case E_V2TImode:
18668 135 : for (i = 0; i < 2; i++)
18669 90 : ops[i] = gen_lowpart (V2DImode, XVECEXP (vals, 0, i));
18670 45 : op0 = gen_reg_rtx (V4DImode);
18671 45 : ix86_expand_vector_init_concat (V4DImode, op0, ops, 2);
18672 45 : emit_move_insn (target, gen_lowpart (GET_MODE (target), op0));
18673 45 : return;
18674 :
18675 : case E_V4TImode:
18676 195 : for (i = 0; i < 4; i++)
18677 156 : ops[i] = gen_lowpart (V2DImode, XVECEXP (vals, 0, i));
18678 39 : ops[4] = gen_reg_rtx (V4DImode);
18679 39 : ix86_expand_vector_init_concat (V4DImode, ops[4], ops, 2);
18680 39 : ops[5] = gen_reg_rtx (V4DImode);
18681 39 : ix86_expand_vector_init_concat (V4DImode, ops[5], ops + 2, 2);
18682 39 : op0 = gen_reg_rtx (V8DImode);
18683 39 : ix86_expand_vector_init_concat (V8DImode, op0, ops + 4, 2);
18684 39 : emit_move_insn (target, gen_lowpart (GET_MODE (target), op0));
18685 39 : return;
18686 :
18687 69 : case E_V32QImode:
18688 69 : half_mode = V16QImode;
18689 69 : goto half;
18690 :
18691 64 : case E_V16HImode:
18692 64 : half_mode = V8HImode;
18693 64 : goto half;
18694 :
18695 237 : case E_V16HFmode:
18696 237 : half_mode = V8HFmode;
18697 237 : goto half;
18698 :
18699 95 : case E_V16BFmode:
18700 95 : half_mode = V8BFmode;
18701 95 : goto half;
18702 :
18703 465 : half:
18704 465 : n = GET_MODE_NUNITS (mode);
18705 9009 : for (i = 0; i < n; i++)
18706 8544 : ops[i] = XVECEXP (vals, 0, i);
18707 465 : op0 = gen_reg_rtx (half_mode);
18708 465 : op1 = gen_reg_rtx (half_mode);
18709 465 : ix86_expand_vector_init_interleave (half_mode, op0, ops,
18710 : n >> 2);
18711 465 : ix86_expand_vector_init_interleave (half_mode, op1,
18712 465 : &ops [n >> 1], n >> 2);
18713 465 : emit_insn (gen_rtx_SET (target, gen_rtx_VEC_CONCAT (mode, op0, op1)));
18714 465 : return;
18715 :
18716 56 : case E_V64QImode:
18717 56 : quarter_mode = V16QImode;
18718 56 : half_mode = V32QImode;
18719 56 : goto quarter;
18720 :
18721 71 : case E_V32HImode:
18722 71 : quarter_mode = V8HImode;
18723 71 : half_mode = V16HImode;
18724 71 : goto quarter;
18725 :
18726 287 : case E_V32HFmode:
18727 287 : quarter_mode = V8HFmode;
18728 287 : half_mode = V16HFmode;
18729 287 : goto quarter;
18730 :
18731 51 : case E_V32BFmode:
18732 51 : quarter_mode = V8BFmode;
18733 51 : half_mode = V16BFmode;
18734 51 : goto quarter;
18735 :
18736 465 : quarter:
18737 465 : n = GET_MODE_NUNITS (mode);
18738 17137 : for (i = 0; i < n; i++)
18739 16672 : ops[i] = XVECEXP (vals, 0, i);
18740 465 : op0 = gen_reg_rtx (quarter_mode);
18741 465 : op1 = gen_reg_rtx (quarter_mode);
18742 465 : op2 = gen_reg_rtx (quarter_mode);
18743 465 : op3 = gen_reg_rtx (quarter_mode);
18744 465 : op4 = gen_reg_rtx (half_mode);
18745 465 : op5 = gen_reg_rtx (half_mode);
18746 465 : ix86_expand_vector_init_interleave (quarter_mode, op0, ops,
18747 : n >> 3);
18748 465 : ix86_expand_vector_init_interleave (quarter_mode, op1,
18749 465 : &ops [n >> 2], n >> 3);
18750 465 : ix86_expand_vector_init_interleave (quarter_mode, op2,
18751 465 : &ops [n >> 1], n >> 3);
18752 465 : ix86_expand_vector_init_interleave (quarter_mode, op3,
18753 465 : &ops [(n >> 1) | (n >> 2)], n >> 3);
18754 465 : emit_insn (gen_rtx_SET (op4, gen_rtx_VEC_CONCAT (half_mode, op0, op1)));
18755 465 : emit_insn (gen_rtx_SET (op5, gen_rtx_VEC_CONCAT (half_mode, op2, op3)));
18756 465 : emit_insn (gen_rtx_SET (target, gen_rtx_VEC_CONCAT (mode, op4, op5)));
18757 465 : return;
18758 :
18759 389 : case E_V16QImode:
18760 389 : if (!TARGET_SSE4_1)
18761 : break;
18762 : /* FALLTHRU */
18763 :
18764 575 : case E_V8HImode:
18765 575 : if (!TARGET_SSE2)
18766 : break;
18767 :
18768 : /* Don't use ix86_expand_vector_init_interleave if we can't
18769 : move from GPR to SSE register directly. */
18770 575 : if (!TARGET_INTER_UNIT_MOVES_TO_VEC)
18771 : break;
18772 : /* FALLTHRU */
18773 :
18774 1149 : case E_V8HFmode:
18775 1149 : case E_V8BFmode:
18776 :
18777 1149 : n = GET_MODE_NUNITS (mode);
18778 10437 : for (i = 0; i < n; i++)
18779 9288 : ops[i] = XVECEXP (vals, 0, i);
18780 1149 : ix86_expand_vector_init_interleave (mode, target, ops, n >> 1);
18781 1149 : return;
18782 :
18783 : case E_V4HFmode:
18784 : case E_V4BFmode:
18785 : case E_V2HFmode:
18786 : case E_V2BFmode:
18787 8568 : int_inner_mode = HImode;
18788 : break;
18789 :
18790 : case E_V4HImode:
18791 : case E_V8QImode:
18792 :
18793 : case E_V2HImode:
18794 : case E_V4QImode:
18795 : break;
18796 :
18797 0 : default:
18798 0 : gcc_unreachable ();
18799 : }
18800 :
18801 8568 : {
18802 8568 : int i, j, n_elts, n_words, n_elt_per_word;
18803 8568 : machine_mode tmp_mode, inner_mode;
18804 8568 : rtx words[4], shift;
18805 :
18806 17210 : tmp_mode = (GET_MODE_SIZE (mode) < UNITS_PER_WORD) ? SImode : word_mode;
18807 :
18808 8568 : inner_mode = GET_MODE_INNER (mode);
18809 8568 : n_elts = GET_MODE_NUNITS (mode);
18810 17136 : n_words = GET_MODE_SIZE (mode) / GET_MODE_SIZE (tmp_mode);
18811 8568 : n_elt_per_word = n_elts / n_words;
18812 8568 : shift = GEN_INT (GET_MODE_BITSIZE (inner_mode));
18813 :
18814 17591 : for (i = 0; i < n_words; ++i)
18815 : {
18816 : rtx word = NULL_RTX;
18817 :
18818 48129 : for (j = 0; j < n_elt_per_word; ++j)
18819 : {
18820 39106 : rtx elt = XVECEXP (vals, 0, (i+1)*n_elt_per_word - j - 1);
18821 39106 : if (int_inner_mode != E_VOIDmode)
18822 : {
18823 310 : gcc_assert (TARGET_SSE2 && int_inner_mode == HImode);
18824 310 : rtx tmp = gen_reg_rtx (int_inner_mode);
18825 310 : elt = lowpart_subreg (int_inner_mode,
18826 : force_reg (inner_mode, elt),
18827 : inner_mode);
18828 310 : emit_move_insn (tmp, elt);
18829 310 : elt = tmp;
18830 : }
18831 39106 : elt = convert_modes (tmp_mode, inner_mode, elt, true);
18832 :
18833 39106 : if (j == 0)
18834 : word = elt;
18835 : else
18836 : {
18837 30083 : word = expand_simple_binop (tmp_mode, ASHIFT, word, shift,
18838 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
18839 30083 : word = expand_simple_binop (tmp_mode, IOR, word, elt,
18840 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
18841 : }
18842 : }
18843 :
18844 9023 : words[i] = word;
18845 : }
18846 :
18847 8568 : if (n_words == 1)
18848 8113 : emit_move_insn (target, gen_lowpart (mode, words[0]));
18849 455 : else if (n_words == 2)
18850 : {
18851 455 : gcc_assert (tmp_mode == DImode || tmp_mode == SImode);
18852 455 : machine_mode concat_mode = tmp_mode == DImode ? V2DImode : V2SImode;
18853 455 : rtx tmp = gen_reg_rtx (concat_mode);
18854 455 : vals = gen_rtx_PARALLEL (concat_mode, gen_rtvec_v (2, words));
18855 455 : ix86_expand_vector_init_general (mmx_ok, concat_mode, tmp, vals);
18856 455 : emit_move_insn (target, gen_lowpart (mode, tmp));
18857 : }
18858 0 : else if (n_words == 4)
18859 : {
18860 0 : rtx tmp = gen_reg_rtx (V4SImode);
18861 0 : gcc_assert (tmp_mode == SImode);
18862 0 : vals = gen_rtx_PARALLEL (V4SImode, gen_rtvec_v (4, words));
18863 0 : ix86_expand_vector_init_general (false, V4SImode, tmp, vals);
18864 0 : emit_move_insn (target, gen_lowpart (mode, tmp));
18865 : }
18866 : else
18867 0 : gcc_unreachable ();
18868 : }
18869 : }
18870 :
18871 : /* Initialize vector TARGET via VALS. Suppress the use of MMX
18872 : instructions unless MMX_OK is true. */
18873 :
18874 : void
18875 133332 : ix86_expand_vector_init (bool mmx_ok, rtx target, rtx vals)
18876 : {
18877 133332 : machine_mode mode = GET_MODE (target);
18878 133332 : machine_mode inner_mode = GET_MODE_INNER (mode);
18879 133332 : int n_elts = GET_MODE_NUNITS (mode);
18880 133332 : int n_var = 0, one_var = -1;
18881 133332 : bool all_same = true, all_const_zero = true;
18882 133332 : int i;
18883 133332 : rtx x;
18884 :
18885 : /* Handle first initialization from vector elts. */
18886 133332 : if (n_elts != XVECLEN (vals, 0))
18887 : {
18888 1361 : rtx subtarget = target;
18889 1361 : x = XVECEXP (vals, 0, 0);
18890 2722 : gcc_assert (GET_MODE_INNER (GET_MODE (x)) == inner_mode);
18891 2722 : if (GET_MODE_NUNITS (GET_MODE (x)) * 2 == n_elts)
18892 : {
18893 1361 : rtx ops[2] = { XVECEXP (vals, 0, 0), XVECEXP (vals, 0, 1) };
18894 1361 : if (inner_mode == QImode
18895 1361 : || inner_mode == HImode
18896 1361 : || inner_mode == TImode
18897 : || inner_mode == HFmode
18898 : || inner_mode == BFmode)
18899 : {
18900 168 : unsigned int n_bits = n_elts * GET_MODE_SIZE (inner_mode);
18901 168 : scalar_mode elt_mode = inner_mode == TImode ? DImode : SImode;
18902 168 : n_bits /= GET_MODE_SIZE (elt_mode);
18903 168 : mode = mode_for_vector (elt_mode, n_bits).require ();
18904 168 : inner_mode = mode_for_vector (elt_mode, n_bits / 2).require ();
18905 168 : ops[0] = gen_lowpart (inner_mode, ops[0]);
18906 168 : ops[1] = gen_lowpart (inner_mode, ops[1]);
18907 168 : subtarget = gen_reg_rtx (mode);
18908 : }
18909 1361 : ix86_expand_vector_init_concat (mode, subtarget, ops, 2);
18910 1361 : if (subtarget != target)
18911 168 : emit_move_insn (target, gen_lowpart (GET_MODE (target), subtarget));
18912 1361 : return;
18913 : }
18914 0 : gcc_unreachable ();
18915 : }
18916 :
18917 484459 : for (i = 0; i < n_elts; ++i)
18918 : {
18919 352488 : x = XVECEXP (vals, 0, i);
18920 683924 : if (!(CONST_SCALAR_INT_P (x)
18921 335418 : || CONST_DOUBLE_P (x)
18922 : || CONST_FIXED_P (x)))
18923 331436 : n_var++, one_var = i;
18924 21052 : else if (x != CONST0_RTX (inner_mode))
18925 3404 : all_const_zero = false;
18926 352488 : if (i > 0 && !rtx_equal_p (x, XVECEXP (vals, 0, 0)))
18927 : all_same = false;
18928 : }
18929 :
18930 : /* Handle the zero vector as special case. */
18931 131971 : if (n_var == 0 && all_const_zero)
18932 : {
18933 310 : emit_move_insn (target, CONST0_RTX (mode));
18934 310 : return;
18935 : }
18936 :
18937 : /* If all values are identical, broadcast the value. */
18938 131661 : if (all_same
18939 139014 : && ix86_expand_vector_init_duplicate (mmx_ok, mode, target,
18940 7353 : XVECEXP (vals, 0, 0)))
18941 : return;
18942 :
18943 : /* Constants are best loaded from the constant pool. */
18944 125462 : if (n_var == 0)
18945 : {
18946 43 : emit_move_insn (target, gen_rtx_CONST_VECTOR (mode, XVEC (vals, 0)));
18947 43 : return;
18948 : }
18949 :
18950 : /* Values where only one field is non-constant are best loaded from
18951 : the pool and overwritten via move later. */
18952 125419 : if (n_var == 1)
18953 : {
18954 12165 : if (all_const_zero
18955 23017 : && ix86_expand_vector_init_one_nonzero (mmx_ok, mode, target,
18956 10852 : XVECEXP (vals, 0, one_var),
18957 : one_var))
18958 : return;
18959 :
18960 8367 : if (ix86_expand_vector_init_one_var (mmx_ok, mode, target, vals, one_var))
18961 : return;
18962 : }
18963 :
18964 121375 : ix86_expand_vector_init_general (mmx_ok, mode, target, vals);
18965 : }
18966 :
18967 : /* Implemented as
18968 : V setg (V v, int idx, T val)
18969 : {
18970 : V idxv = (V){idx, idx, idx, idx, idx, idx, idx, idx};
18971 : V valv = (V){val, val, val, val, val, val, val, val};
18972 : V mask = ((V){0, 1, 2, 3, 4, 5, 6, 7} == idxv);
18973 : v = (v & ~mask) | (valv & mask);
18974 : return v;
18975 : }. */
18976 : void
18977 129 : ix86_expand_vector_set_var (rtx target, rtx val, rtx idx)
18978 : {
18979 129 : rtx vec[64];
18980 129 : machine_mode mode = GET_MODE (target);
18981 129 : machine_mode cmp_mode = mode;
18982 129 : int n_elts = GET_MODE_NUNITS (mode);
18983 129 : rtx valv,idxv,constv,idx_tmp;
18984 129 : bool ok = false;
18985 :
18986 : /* 512-bits vector byte/word broadcast and comparison only available
18987 : under TARGET_AVX512BW, break 512-bits vector into two 256-bits vector
18988 : when without TARGET_AVX512BW. */
18989 129 : if ((mode == V32HImode || mode == V32HFmode || mode == V32BFmode
18990 123 : || mode == V64QImode)
18991 10 : && !TARGET_AVX512BW)
18992 : {
18993 3 : gcc_assert (TARGET_AVX512F);
18994 3 : rtx vhi, vlo, idx_hi;
18995 3 : machine_mode half_mode;
18996 3 : rtx (*extract_hi)(rtx, rtx);
18997 3 : rtx (*extract_lo)(rtx, rtx);
18998 :
18999 3 : if (mode == V32HImode)
19000 : {
19001 : half_mode = V16HImode;
19002 : extract_hi = gen_vec_extract_hi_v32hi;
19003 : extract_lo = gen_vec_extract_lo_v32hi;
19004 : }
19005 : else if (mode == V32HFmode)
19006 : {
19007 : half_mode = V16HFmode;
19008 : extract_hi = gen_vec_extract_hi_v32hf;
19009 : extract_lo = gen_vec_extract_lo_v32hf;
19010 : }
19011 : else if (mode == V32BFmode)
19012 : {
19013 : half_mode = V16BFmode;
19014 : extract_hi = gen_vec_extract_hi_v32bf;
19015 : extract_lo = gen_vec_extract_lo_v32bf;
19016 : }
19017 : else
19018 : {
19019 3 : half_mode = V32QImode;
19020 3 : extract_hi = gen_vec_extract_hi_v64qi;
19021 3 : extract_lo = gen_vec_extract_lo_v64qi;
19022 : }
19023 :
19024 3 : vhi = gen_reg_rtx (half_mode);
19025 3 : vlo = gen_reg_rtx (half_mode);
19026 3 : idx_hi = gen_reg_rtx (GET_MODE (idx));
19027 3 : emit_insn (extract_hi (vhi, target));
19028 3 : emit_insn (extract_lo (vlo, target));
19029 3 : vec[0] = idx_hi;
19030 3 : vec[1] = idx;
19031 3 : vec[2] = GEN_INT (n_elts/2);
19032 3 : ix86_expand_binary_operator (MINUS, GET_MODE (idx), vec);
19033 3 : ix86_expand_vector_set_var (vhi, val, idx_hi);
19034 3 : ix86_expand_vector_set_var (vlo, val, idx);
19035 3 : emit_insn (gen_rtx_SET (target, gen_rtx_VEC_CONCAT (mode, vlo, vhi)));
19036 3 : return;
19037 : }
19038 :
19039 504 : if (FLOAT_MODE_P (GET_MODE_INNER (mode)))
19040 : {
19041 42 : switch (mode)
19042 : {
19043 : case E_V2DFmode:
19044 : cmp_mode = V2DImode;
19045 : break;
19046 6 : case E_V4DFmode:
19047 6 : cmp_mode = V4DImode;
19048 6 : break;
19049 4 : case E_V8DFmode:
19050 4 : cmp_mode = V8DImode;
19051 4 : break;
19052 2 : case E_V2SFmode:
19053 2 : cmp_mode = V2SImode;
19054 2 : break;
19055 6 : case E_V4SFmode:
19056 6 : cmp_mode = V4SImode;
19057 6 : break;
19058 6 : case E_V8SFmode:
19059 6 : cmp_mode = V8SImode;
19060 6 : break;
19061 5 : case E_V16SFmode:
19062 5 : cmp_mode = V16SImode;
19063 5 : break;
19064 1 : case E_V2HFmode:
19065 1 : case E_V2BFmode:
19066 1 : cmp_mode = V2HImode;
19067 1 : break;
19068 1 : case E_V4HFmode:
19069 1 : case E_V4BFmode:
19070 1 : cmp_mode = V4HImode;
19071 1 : break;
19072 : case E_V8HFmode:
19073 2 : cmp_mode = V8HImode;
19074 : break;
19075 : case E_V16HFmode:
19076 2 : cmp_mode = V16HImode;
19077 : break;
19078 : case E_V32HFmode:
19079 1 : cmp_mode = V32HImode;
19080 : break;
19081 : case E_V8BFmode:
19082 2 : cmp_mode = V8HImode;
19083 : break;
19084 : case E_V16BFmode:
19085 2 : cmp_mode = V16HImode;
19086 : break;
19087 : case E_V32BFmode:
19088 1 : cmp_mode = V32HImode;
19089 : break;
19090 0 : default:
19091 0 : gcc_unreachable ();
19092 : }
19093 : }
19094 :
19095 1604 : for (int i = 0; i != n_elts; i++)
19096 1478 : vec[i] = GEN_INT (i);
19097 126 : constv = gen_rtx_CONST_VECTOR (cmp_mode, gen_rtvec_v (n_elts, vec));
19098 126 : valv = gen_reg_rtx (mode);
19099 126 : idxv = gen_reg_rtx (cmp_mode);
19100 252 : idx_tmp = convert_to_mode (GET_MODE_INNER (cmp_mode), idx, 1);
19101 :
19102 126 : ok = ix86_expand_vector_init_duplicate (TARGET_MMX_WITH_SSE,
19103 : mode, valv, val);
19104 126 : gcc_assert (ok);
19105 126 : ok = ix86_expand_vector_init_duplicate (TARGET_MMX_WITH_SSE,
19106 : cmp_mode, idxv, idx_tmp);
19107 126 : gcc_assert (ok);
19108 126 : vec[0] = target;
19109 126 : vec[1] = valv;
19110 126 : vec[2] = target;
19111 126 : vec[3] = gen_rtx_EQ (mode, idxv, constv);
19112 126 : vec[4] = idxv;
19113 126 : vec[5] = constv;
19114 126 : ok = ix86_expand_int_vcond (vec);
19115 126 : gcc_assert (ok);
19116 : }
19117 :
19118 : void
19119 8606 : ix86_expand_vector_set (bool mmx_ok, rtx target, rtx val, int elt)
19120 : {
19121 8606 : machine_mode mode = GET_MODE (target);
19122 8606 : machine_mode inner_mode = GET_MODE_INNER (mode);
19123 8606 : machine_mode half_mode;
19124 8606 : bool use_vec_merge = false;
19125 8606 : bool blendm_const = false;
19126 8606 : rtx tmp;
19127 8606 : static rtx (*gen_extract[8][2]) (rtx, rtx)
19128 : = {
19129 : { gen_vec_extract_lo_v32qi, gen_vec_extract_hi_v32qi },
19130 : { gen_vec_extract_lo_v16hi, gen_vec_extract_hi_v16hi },
19131 : { gen_vec_extract_lo_v8si, gen_vec_extract_hi_v8si },
19132 : { gen_vec_extract_lo_v4di, gen_vec_extract_hi_v4di },
19133 : { gen_vec_extract_lo_v8sf, gen_vec_extract_hi_v8sf },
19134 : { gen_vec_extract_lo_v4df, gen_vec_extract_hi_v4df },
19135 : { gen_vec_extract_lo_v16hf, gen_vec_extract_hi_v16hf },
19136 : { gen_vec_extract_lo_v16bf, gen_vec_extract_hi_v16bf }
19137 : };
19138 8606 : static rtx (*gen_insert[8][2]) (rtx, rtx, rtx)
19139 : = {
19140 : { gen_vec_set_lo_v32qi, gen_vec_set_hi_v32qi },
19141 : { gen_vec_set_lo_v16hi, gen_vec_set_hi_v16hi },
19142 : { gen_vec_set_lo_v8si, gen_vec_set_hi_v8si },
19143 : { gen_vec_set_lo_v4di, gen_vec_set_hi_v4di },
19144 : { gen_vec_set_lo_v8sf, gen_vec_set_hi_v8sf },
19145 : { gen_vec_set_lo_v4df, gen_vec_set_hi_v4df },
19146 : { gen_vec_set_lo_v16hf, gen_vec_set_hi_v16hf },
19147 : { gen_vec_set_lo_v16bf, gen_vec_set_hi_v16bf },
19148 : };
19149 8606 : int i, j, n;
19150 8606 : machine_mode mmode = VOIDmode;
19151 8606 : rtx (*gen_blendm) (rtx, rtx, rtx, rtx);
19152 :
19153 8606 : switch (mode)
19154 : {
19155 187 : case E_V2SImode:
19156 187 : use_vec_merge = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
19157 : if (use_vec_merge)
19158 : break;
19159 : /* FALLTHRU */
19160 :
19161 170 : case E_V2SFmode:
19162 170 : if (mmx_ok)
19163 : {
19164 168 : tmp = gen_reg_rtx (GET_MODE_INNER (mode));
19165 168 : ix86_expand_vector_extract (true, tmp, target, 1 - elt);
19166 168 : if (elt == 0)
19167 0 : tmp = gen_rtx_VEC_CONCAT (mode, val, tmp);
19168 : else
19169 168 : tmp = gen_rtx_VEC_CONCAT (mode, tmp, val);
19170 168 : emit_insn (gen_rtx_SET (target, tmp));
19171 168 : return;
19172 : }
19173 : break;
19174 :
19175 234 : case E_V2DImode:
19176 234 : use_vec_merge = TARGET_SSE4_1 && TARGET_64BIT;
19177 88 : if (use_vec_merge)
19178 : break;
19179 :
19180 88 : tmp = gen_reg_rtx (GET_MODE_INNER (mode));
19181 88 : ix86_expand_vector_extract (false, tmp, target, 1 - elt);
19182 88 : if (elt == 0)
19183 53 : tmp = gen_rtx_VEC_CONCAT (mode, val, tmp);
19184 : else
19185 35 : tmp = gen_rtx_VEC_CONCAT (mode, tmp, val);
19186 88 : emit_insn (gen_rtx_SET (target, tmp));
19187 88 : return;
19188 :
19189 126 : case E_V2DFmode:
19190 : /* NB: For ELT == 0, use standard scalar operation patterns which
19191 : preserve the rest of the vector for combiner:
19192 :
19193 : (vec_merge:V2DF
19194 : (vec_duplicate:V2DF (reg:DF))
19195 : (reg:V2DF)
19196 : (const_int 1))
19197 : */
19198 126 : if (elt == 0)
19199 64 : goto do_vec_merge;
19200 :
19201 62 : {
19202 62 : rtx op0, op1;
19203 :
19204 : /* For the two element vectors, we implement a VEC_CONCAT with
19205 : the extraction of the other element. */
19206 :
19207 62 : tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (1 - elt)));
19208 62 : tmp = gen_rtx_VEC_SELECT (inner_mode, target, tmp);
19209 :
19210 62 : if (elt == 0)
19211 : op0 = val, op1 = tmp;
19212 : else
19213 62 : op0 = tmp, op1 = val;
19214 :
19215 62 : tmp = gen_rtx_VEC_CONCAT (mode, op0, op1);
19216 62 : emit_insn (gen_rtx_SET (target, tmp));
19217 : }
19218 62 : return;
19219 :
19220 574 : case E_V4SFmode:
19221 574 : use_vec_merge = TARGET_SSE4_1;
19222 574 : if (use_vec_merge)
19223 : break;
19224 :
19225 62 : switch (elt)
19226 : {
19227 : case 0:
19228 : use_vec_merge = true;
19229 : break;
19230 :
19231 1 : case 1:
19232 : /* tmp = target = A B C D */
19233 1 : tmp = copy_to_reg (target);
19234 : /* target = A A B B */
19235 1 : emit_insn (gen_vec_interleave_lowv4sf (target, target, target));
19236 : /* target = X A B B */
19237 1 : ix86_expand_vector_set (false, target, val, 0);
19238 : /* target = A X C D */
19239 1 : emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
19240 : const1_rtx, const0_rtx,
19241 : GEN_INT (2+4), GEN_INT (3+4)));
19242 1 : return;
19243 :
19244 0 : case 2:
19245 : /* tmp = target = A B C D */
19246 0 : tmp = copy_to_reg (target);
19247 : /* tmp = X B C D */
19248 0 : ix86_expand_vector_set (false, tmp, val, 0);
19249 : /* target = A B X D */
19250 0 : emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
19251 : const0_rtx, const1_rtx,
19252 : GEN_INT (0+4), GEN_INT (3+4)));
19253 0 : return;
19254 :
19255 4 : case 3:
19256 : /* tmp = target = A B C D */
19257 4 : tmp = copy_to_reg (target);
19258 : /* tmp = X B C D */
19259 4 : ix86_expand_vector_set (false, tmp, val, 0);
19260 : /* target = A B X D */
19261 4 : emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
19262 : const0_rtx, const1_rtx,
19263 : GEN_INT (2+4), GEN_INT (0+4)));
19264 4 : return;
19265 :
19266 0 : default:
19267 0 : gcc_unreachable ();
19268 : }
19269 : break;
19270 :
19271 455 : case E_V4SImode:
19272 455 : use_vec_merge = TARGET_SSE4_1;
19273 455 : if (use_vec_merge)
19274 : break;
19275 :
19276 : /* Element 0 handled by vec_merge below. */
19277 289 : if (elt == 0)
19278 : {
19279 : use_vec_merge = true;
19280 : break;
19281 : }
19282 :
19283 92 : if (TARGET_SSE2)
19284 : {
19285 : /* With SSE2, use integer shuffles to swap element 0 and ELT,
19286 : store into element 0, then shuffle them back. */
19287 :
19288 92 : rtx order[4];
19289 :
19290 92 : order[0] = GEN_INT (elt);
19291 92 : order[1] = const1_rtx;
19292 92 : order[2] = const2_rtx;
19293 92 : order[3] = GEN_INT (3);
19294 92 : order[elt] = const0_rtx;
19295 :
19296 92 : emit_insn (gen_sse2_pshufd_1 (target, target, order[0],
19297 : order[1], order[2], order[3]));
19298 :
19299 92 : ix86_expand_vector_set (false, target, val, 0);
19300 :
19301 92 : emit_insn (gen_sse2_pshufd_1 (target, target, order[0],
19302 : order[1], order[2], order[3]));
19303 : }
19304 : else
19305 : {
19306 : /* For SSE1, we have to reuse the V4SF code. */
19307 0 : rtx t = gen_reg_rtx (V4SFmode);
19308 0 : emit_move_insn (t, gen_lowpart (V4SFmode, target));
19309 0 : ix86_expand_vector_set (false, t, gen_lowpart (SFmode, val), elt);
19310 0 : emit_move_insn (target, gen_lowpart (mode, t));
19311 : }
19312 : return;
19313 :
19314 3767 : case E_V8HImode:
19315 3767 : case E_V8HFmode:
19316 3767 : case E_V8BFmode:
19317 3767 : case E_V2HImode:
19318 3767 : case E_V2HFmode:
19319 3767 : case E_V2BFmode:
19320 3767 : use_vec_merge = TARGET_SSE2;
19321 3767 : break;
19322 52 : case E_V4HImode:
19323 52 : case E_V4HFmode:
19324 52 : case E_V4BFmode:
19325 52 : use_vec_merge = mmx_ok && (TARGET_SSE || TARGET_3DNOW_A);
19326 : break;
19327 :
19328 3067 : case E_V16QImode:
19329 3067 : case E_V4QImode:
19330 3067 : use_vec_merge = TARGET_SSE4_1;
19331 3067 : break;
19332 :
19333 1 : case E_V8QImode:
19334 1 : use_vec_merge = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
19335 : break;
19336 :
19337 3 : case E_V32QImode:
19338 3 : half_mode = V16QImode;
19339 3 : j = 0;
19340 3 : n = 16;
19341 3 : goto half;
19342 :
19343 17 : case E_V16HFmode:
19344 17 : case E_V16BFmode:
19345 : /* For ELT == 0, vec_setv8hf_0 can save 1 vpbroadcastw. */
19346 17 : if (TARGET_AVX2 && elt != 0)
19347 : {
19348 12 : mmode = SImode;
19349 12 : gen_blendm = ((mode == E_V16HFmode) ? gen_avx2_pblendph_1
19350 : : gen_avx2_pblendbf_1);
19351 : blendm_const = true;
19352 : break;
19353 : }
19354 : else
19355 : {
19356 5 : half_mode = ((mode == E_V16HFmode) ? V8HFmode : V8BFmode);
19357 3 : j = ((mode == E_V16HFmode) ? 6 : 7);
19358 5 : n = 8;
19359 5 : goto half;
19360 : }
19361 :
19362 5 : case E_V16HImode:
19363 5 : half_mode = V8HImode;
19364 5 : j = 1;
19365 5 : n = 8;
19366 5 : goto half;
19367 :
19368 15 : case E_V8SImode:
19369 15 : half_mode = V4SImode;
19370 15 : j = 2;
19371 15 : n = 4;
19372 15 : goto half;
19373 :
19374 15 : case E_V4DImode:
19375 15 : half_mode = V2DImode;
19376 15 : j = 3;
19377 15 : n = 2;
19378 15 : goto half;
19379 :
19380 4 : case E_V8SFmode:
19381 4 : half_mode = V4SFmode;
19382 4 : j = 4;
19383 4 : n = 4;
19384 4 : goto half;
19385 :
19386 6 : case E_V4DFmode:
19387 6 : half_mode = V2DFmode;
19388 6 : j = 5;
19389 6 : n = 2;
19390 6 : goto half;
19391 :
19392 53 : half:
19393 : /* Compute offset. */
19394 53 : i = elt / n;
19395 53 : elt %= n;
19396 :
19397 53 : gcc_assert (i <= 1);
19398 :
19399 : /* Extract the half. */
19400 53 : tmp = gen_reg_rtx (half_mode);
19401 53 : emit_insn (gen_extract[j][i] (tmp, target));
19402 :
19403 : /* Put val in tmp at elt. */
19404 53 : ix86_expand_vector_set (false, tmp, val, elt);
19405 :
19406 : /* Put it back. */
19407 53 : emit_insn (gen_insert[j][i] (target, target, tmp));
19408 53 : return;
19409 :
19410 8 : case E_V8DFmode:
19411 8 : if (TARGET_AVX512F)
19412 : {
19413 : mmode = QImode;
19414 : gen_blendm = gen_avx512f_blendmv8df;
19415 : }
19416 : break;
19417 :
19418 6 : case E_V8DImode:
19419 6 : if (TARGET_AVX512F)
19420 : {
19421 : mmode = QImode;
19422 : gen_blendm = gen_avx512f_blendmv8di;
19423 : }
19424 : break;
19425 :
19426 0 : case E_V16SFmode:
19427 0 : if (TARGET_AVX512F)
19428 : {
19429 : mmode = HImode;
19430 : gen_blendm = gen_avx512f_blendmv16sf;
19431 : }
19432 : break;
19433 :
19434 0 : case E_V16SImode:
19435 0 : if (TARGET_AVX512F)
19436 : {
19437 : mmode = HImode;
19438 : gen_blendm = gen_avx512f_blendmv16si;
19439 : }
19440 : break;
19441 :
19442 12 : case E_V32HFmode:
19443 12 : if (TARGET_AVX512BW)
19444 : {
19445 : mmode = SImode;
19446 : gen_blendm = gen_avx512bw_blendmv32hf;
19447 : }
19448 : break;
19449 12 : case E_V32BFmode:
19450 12 : if (TARGET_AVX512BW)
19451 : {
19452 : mmode = SImode;
19453 : gen_blendm = gen_avx512bw_blendmv32bf;
19454 : }
19455 : break;
19456 11 : case E_V32HImode:
19457 11 : if (TARGET_AVX512BW)
19458 : {
19459 : mmode = SImode;
19460 : gen_blendm = gen_avx512bw_blendmv32hi;
19461 : }
19462 7 : else if (TARGET_AVX512F)
19463 : {
19464 7 : half_mode = E_V8HImode;
19465 7 : n = 8;
19466 7 : goto quarter;
19467 : }
19468 : break;
19469 :
19470 12 : case E_V64QImode:
19471 12 : if (TARGET_AVX512BW)
19472 : {
19473 : mmode = DImode;
19474 : gen_blendm = gen_avx512bw_blendmv64qi;
19475 : }
19476 6 : else if (TARGET_AVX512F)
19477 : {
19478 6 : half_mode = E_V16QImode;
19479 6 : n = 16;
19480 6 : goto quarter;
19481 : }
19482 : break;
19483 :
19484 13 : quarter:
19485 : /* Compute offset. */
19486 13 : i = elt / n;
19487 13 : elt %= n;
19488 :
19489 13 : gcc_assert (i <= 3);
19490 :
19491 13 : {
19492 : /* Extract the quarter. */
19493 13 : tmp = gen_reg_rtx (V4SImode);
19494 13 : rtx tmp2 = gen_lowpart (V16SImode, target);
19495 13 : rtx mask = gen_reg_rtx (QImode);
19496 :
19497 13 : emit_move_insn (mask, constm1_rtx);
19498 13 : emit_insn (gen_avx512f_vextracti32x4_mask (tmp, tmp2, GEN_INT (i),
19499 : tmp, mask));
19500 :
19501 13 : tmp2 = gen_reg_rtx (half_mode);
19502 13 : emit_move_insn (tmp2, gen_lowpart (half_mode, tmp));
19503 13 : tmp = tmp2;
19504 :
19505 : /* Put val in tmp at elt. */
19506 13 : ix86_expand_vector_set (false, tmp, val, elt);
19507 :
19508 : /* Put it back. */
19509 13 : tmp2 = gen_reg_rtx (V16SImode);
19510 13 : rtx tmp3 = gen_lowpart (V16SImode, target);
19511 13 : mask = gen_reg_rtx (HImode);
19512 13 : emit_move_insn (mask, constm1_rtx);
19513 13 : tmp = gen_lowpart (V4SImode, tmp);
19514 13 : emit_insn (gen_avx512f_vinserti32x4_mask (tmp2, tmp3, tmp, GEN_INT (i),
19515 : tmp3, mask));
19516 13 : emit_move_insn (target, gen_lowpart (mode, tmp2));
19517 : }
19518 13 : return;
19519 :
19520 : default:
19521 : break;
19522 : }
19523 :
19524 6834 : if (mmode != VOIDmode)
19525 : {
19526 54 : tmp = gen_reg_rtx (mode);
19527 54 : emit_insn (gen_rtx_SET (tmp, gen_rtx_VEC_DUPLICATE (mode, val)));
19528 54 : rtx merge_mask = gen_int_mode (HOST_WIDE_INT_1U << elt, mmode);
19529 : /* The avx512*_blendm<mode> expanders have different operand order
19530 : from VEC_MERGE. In VEC_MERGE, the first input operand is used for
19531 : elements where the mask is set and second input operand otherwise,
19532 : in {sse,avx}*_*blend* the first input operand is used for elements
19533 : where the mask is clear and second input operand otherwise. */
19534 54 : if (!blendm_const)
19535 42 : merge_mask = force_reg (mmode, merge_mask);
19536 54 : emit_insn (gen_blendm (target, target, tmp, merge_mask));
19537 : }
19538 8007 : else if (use_vec_merge)
19539 : {
19540 7995 : do_vec_merge:
19541 8059 : if (!nonimmediate_operand (val, inner_mode))
19542 1 : val = force_reg (inner_mode, val);
19543 8059 : tmp = gen_rtx_VEC_DUPLICATE (mode, val);
19544 8059 : tmp = gen_rtx_VEC_MERGE (mode, tmp, target,
19545 : GEN_INT (HOST_WIDE_INT_1U << elt));
19546 8059 : emit_insn (gen_rtx_SET (target, tmp));
19547 : }
19548 : else
19549 : {
19550 24 : rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode));
19551 :
19552 12 : emit_move_insn (mem, target);
19553 :
19554 24 : tmp = adjust_address (mem, inner_mode, elt * GET_MODE_SIZE (inner_mode));
19555 12 : emit_move_insn (tmp, val);
19556 :
19557 12 : emit_move_insn (target, mem);
19558 : }
19559 : }
19560 :
19561 : void
19562 112541 : ix86_expand_vector_extract (bool mmx_ok, rtx target, rtx vec, int elt)
19563 : {
19564 112541 : machine_mode mode = GET_MODE (vec);
19565 112541 : machine_mode inner_mode = GET_MODE_INNER (mode);
19566 112541 : bool use_vec_extr = false;
19567 112541 : rtx tmp;
19568 :
19569 112541 : switch (mode)
19570 : {
19571 8577 : case E_V2SImode:
19572 8577 : use_vec_extr = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
19573 : if (use_vec_extr)
19574 : break;
19575 : /* FALLTHRU */
19576 :
19577 9790 : case E_V2SFmode:
19578 9790 : if (!mmx_ok)
19579 : break;
19580 : /* FALLTHRU */
19581 :
19582 : case E_V2DFmode:
19583 : case E_V2DImode:
19584 : case E_V2TImode:
19585 : case E_V4TImode:
19586 : use_vec_extr = true;
19587 : break;
19588 :
19589 7931 : case E_V4SFmode:
19590 7931 : use_vec_extr = TARGET_SSE4_1;
19591 7931 : if (use_vec_extr)
19592 : break;
19593 :
19594 3925 : switch (elt)
19595 : {
19596 : case 0:
19597 : tmp = vec;
19598 : break;
19599 :
19600 1709 : case 1:
19601 1709 : case 3:
19602 1709 : tmp = gen_reg_rtx (mode);
19603 1709 : emit_insn (gen_sse_shufps_v4sf (tmp, vec, vec,
19604 : GEN_INT (elt), GEN_INT (elt),
19605 1709 : GEN_INT (elt+4), GEN_INT (elt+4)));
19606 1709 : break;
19607 :
19608 845 : case 2:
19609 845 : tmp = gen_reg_rtx (mode);
19610 845 : emit_insn (gen_vec_interleave_highv4sf (tmp, vec, vec));
19611 845 : break;
19612 :
19613 0 : default:
19614 0 : gcc_unreachable ();
19615 : }
19616 : vec = tmp;
19617 : use_vec_extr = true;
19618 : elt = 0;
19619 : break;
19620 :
19621 25246 : case E_V4SImode:
19622 25246 : use_vec_extr = TARGET_SSE4_1;
19623 25246 : if (use_vec_extr)
19624 : break;
19625 :
19626 19340 : if (TARGET_SSE2)
19627 : {
19628 19336 : switch (elt)
19629 : {
19630 : case 0:
19631 : tmp = vec;
19632 : break;
19633 :
19634 6479 : case 1:
19635 6479 : case 3:
19636 6479 : tmp = gen_reg_rtx (mode);
19637 6479 : emit_insn (gen_sse2_pshufd_1 (tmp, vec,
19638 : GEN_INT (elt), GEN_INT (elt),
19639 : GEN_INT (elt), GEN_INT (elt)));
19640 6479 : break;
19641 :
19642 3257 : case 2:
19643 3257 : tmp = gen_reg_rtx (mode);
19644 3257 : emit_insn (gen_vec_interleave_highv4si (tmp, vec, vec));
19645 3257 : break;
19646 :
19647 0 : default:
19648 0 : gcc_unreachable ();
19649 : }
19650 : vec = tmp;
19651 : use_vec_extr = true;
19652 : elt = 0;
19653 : }
19654 : else
19655 : {
19656 : /* For SSE1, we have to reuse the V4SF code. */
19657 4 : ix86_expand_vector_extract (false, gen_lowpart (SFmode, target),
19658 4 : gen_lowpart (V4SFmode, vec), elt);
19659 4 : return;
19660 : }
19661 : break;
19662 :
19663 6890 : case E_V8HImode:
19664 6890 : case E_V8HFmode:
19665 6890 : case E_V8BFmode:
19666 6890 : case E_V2HImode:
19667 6890 : case E_V2HFmode:
19668 6890 : case E_V2BFmode:
19669 6890 : use_vec_extr = TARGET_SSE2;
19670 6890 : break;
19671 868 : case E_V4HImode:
19672 868 : case E_V4HFmode:
19673 868 : case E_V4BFmode:
19674 868 : use_vec_extr = mmx_ok && (TARGET_SSE || TARGET_3DNOW_A);
19675 : break;
19676 :
19677 8414 : case E_V16QImode:
19678 8414 : use_vec_extr = TARGET_SSE4_1;
19679 8414 : if (!use_vec_extr
19680 6828 : && TARGET_SSE2
19681 6828 : && elt == 0
19682 12338 : && (optimize_insn_for_size_p () || TARGET_INTER_UNIT_MOVES_FROM_VEC))
19683 : {
19684 3923 : tmp = gen_reg_rtx (SImode);
19685 3923 : ix86_expand_vector_extract (false, tmp, gen_lowpart (V4SImode, vec),
19686 : 0);
19687 3923 : emit_insn (gen_rtx_SET (target, gen_lowpart (QImode, tmp)));
19688 3923 : return;
19689 : }
19690 : break;
19691 78 : case E_V4QImode:
19692 78 : use_vec_extr = TARGET_SSE4_1;
19693 78 : break;
19694 :
19695 663 : case E_V8SFmode:
19696 663 : if (TARGET_AVX)
19697 : {
19698 663 : tmp = gen_reg_rtx (V4SFmode);
19699 663 : if (elt < 4)
19700 326 : emit_insn (gen_vec_extract_lo_v8sf (tmp, vec));
19701 : else
19702 337 : emit_insn (gen_vec_extract_hi_v8sf (tmp, vec));
19703 663 : ix86_expand_vector_extract (false, target, tmp, elt & 3);
19704 663 : return;
19705 : }
19706 : break;
19707 :
19708 538 : case E_V4DFmode:
19709 538 : if (TARGET_AVX)
19710 : {
19711 538 : tmp = gen_reg_rtx (V2DFmode);
19712 538 : if (elt < 2)
19713 283 : emit_insn (gen_vec_extract_lo_v4df (tmp, vec));
19714 : else
19715 255 : emit_insn (gen_vec_extract_hi_v4df (tmp, vec));
19716 538 : ix86_expand_vector_extract (false, target, tmp, elt & 1);
19717 538 : return;
19718 : }
19719 : break;
19720 :
19721 253 : case E_V32QImode:
19722 253 : if (TARGET_AVX)
19723 : {
19724 253 : tmp = gen_reg_rtx (V16QImode);
19725 253 : if (elt < 16)
19726 130 : emit_insn (gen_vec_extract_lo_v32qi (tmp, vec));
19727 : else
19728 123 : emit_insn (gen_vec_extract_hi_v32qi (tmp, vec));
19729 253 : ix86_expand_vector_extract (false, target, tmp, elt & 15);
19730 253 : return;
19731 : }
19732 : break;
19733 :
19734 616 : case E_V16HImode:
19735 616 : if (TARGET_AVX)
19736 : {
19737 616 : tmp = gen_reg_rtx (V8HImode);
19738 616 : if (elt < 8)
19739 304 : emit_insn (gen_vec_extract_lo_v16hi (tmp, vec));
19740 : else
19741 312 : emit_insn (gen_vec_extract_hi_v16hi (tmp, vec));
19742 616 : ix86_expand_vector_extract (false, target, tmp, elt & 7);
19743 616 : return;
19744 : }
19745 : break;
19746 :
19747 1105 : case E_V8SImode:
19748 1105 : if (TARGET_AVX)
19749 : {
19750 1105 : tmp = gen_reg_rtx (V4SImode);
19751 1105 : if (elt < 4)
19752 535 : emit_insn (gen_vec_extract_lo_v8si (tmp, vec));
19753 : else
19754 570 : emit_insn (gen_vec_extract_hi_v8si (tmp, vec));
19755 1105 : ix86_expand_vector_extract (false, target, tmp, elt & 3);
19756 1105 : return;
19757 : }
19758 : break;
19759 :
19760 1350 : case E_V4DImode:
19761 1350 : if (TARGET_AVX)
19762 : {
19763 1350 : tmp = gen_reg_rtx (V2DImode);
19764 1350 : if (elt < 2)
19765 713 : emit_insn (gen_vec_extract_lo_v4di (tmp, vec));
19766 : else
19767 637 : emit_insn (gen_vec_extract_hi_v4di (tmp, vec));
19768 1350 : ix86_expand_vector_extract (false, target, tmp, elt & 1);
19769 1350 : return;
19770 : }
19771 : break;
19772 :
19773 8 : case E_V32HImode:
19774 8 : if (TARGET_AVX512BW)
19775 : {
19776 8 : tmp = gen_reg_rtx (V16HImode);
19777 8 : if (elt < 16)
19778 3 : emit_insn (gen_vec_extract_lo_v32hi (tmp, vec));
19779 : else
19780 5 : emit_insn (gen_vec_extract_hi_v32hi (tmp, vec));
19781 8 : ix86_expand_vector_extract (false, target, tmp, elt & 15);
19782 8 : return;
19783 : }
19784 : break;
19785 :
19786 10 : case E_V64QImode:
19787 10 : if (TARGET_AVX512BW)
19788 : {
19789 10 : tmp = gen_reg_rtx (V32QImode);
19790 10 : if (elt < 32)
19791 5 : emit_insn (gen_vec_extract_lo_v64qi (tmp, vec));
19792 : else
19793 5 : emit_insn (gen_vec_extract_hi_v64qi (tmp, vec));
19794 10 : ix86_expand_vector_extract (false, target, tmp, elt & 31);
19795 10 : return;
19796 : }
19797 : break;
19798 :
19799 311 : case E_V16SFmode:
19800 311 : tmp = gen_reg_rtx (V8SFmode);
19801 311 : if (elt < 8)
19802 157 : emit_insn (gen_vec_extract_lo_v16sf (tmp, vec));
19803 : else
19804 154 : emit_insn (gen_vec_extract_hi_v16sf (tmp, vec));
19805 311 : ix86_expand_vector_extract (false, target, tmp, elt & 7);
19806 311 : return;
19807 :
19808 296 : case E_V8DFmode:
19809 296 : tmp = gen_reg_rtx (V4DFmode);
19810 296 : if (elt < 4)
19811 160 : emit_insn (gen_vec_extract_lo_v8df (tmp, vec));
19812 : else
19813 136 : emit_insn (gen_vec_extract_hi_v8df (tmp, vec));
19814 296 : ix86_expand_vector_extract (false, target, tmp, elt & 3);
19815 296 : return;
19816 :
19817 236 : case E_V16SImode:
19818 236 : tmp = gen_reg_rtx (V8SImode);
19819 236 : if (elt < 8)
19820 125 : emit_insn (gen_vec_extract_lo_v16si (tmp, vec));
19821 : else
19822 111 : emit_insn (gen_vec_extract_hi_v16si (tmp, vec));
19823 236 : ix86_expand_vector_extract (false, target, tmp, elt & 7);
19824 236 : return;
19825 :
19826 674 : case E_V8DImode:
19827 674 : tmp = gen_reg_rtx (V4DImode);
19828 674 : if (elt < 4)
19829 387 : emit_insn (gen_vec_extract_lo_v8di (tmp, vec));
19830 : else
19831 287 : emit_insn (gen_vec_extract_hi_v8di (tmp, vec));
19832 674 : ix86_expand_vector_extract (false, target, tmp, elt & 3);
19833 674 : return;
19834 :
19835 45 : case E_V32HFmode:
19836 45 : case E_V32BFmode:
19837 45 : if (TARGET_AVX512BW)
19838 : {
19839 45 : tmp = (mode == E_V32HFmode
19840 45 : ? gen_reg_rtx (V16HFmode)
19841 7 : : gen_reg_rtx (V16BFmode));
19842 45 : if (elt < 16)
19843 31 : emit_insn (gen_vec_extract_lo (mode, tmp, vec));
19844 : else
19845 14 : emit_insn (gen_vec_extract_hi (mode, tmp, vec));
19846 45 : ix86_expand_vector_extract (false, target, tmp, elt & 15);
19847 45 : return;
19848 : }
19849 : break;
19850 :
19851 474 : case E_V16HFmode:
19852 474 : case E_V16BFmode:
19853 474 : if (TARGET_AVX)
19854 : {
19855 474 : tmp = (mode == E_V16HFmode
19856 474 : ? gen_reg_rtx (V8HFmode)
19857 339 : : gen_reg_rtx (V8BFmode));
19858 474 : if (elt < 8)
19859 249 : emit_insn (gen_vec_extract_lo (mode, tmp, vec));
19860 : else
19861 225 : emit_insn (gen_vec_extract_hi (mode, tmp, vec));
19862 474 : ix86_expand_vector_extract (false, target, tmp, elt & 7);
19863 474 : return;
19864 : }
19865 : break;
19866 :
19867 630 : case E_V8QImode:
19868 630 : use_vec_extr = TARGET_MMX_WITH_SSE && TARGET_SSE4_1;
19869 : /* ??? Could extract the appropriate HImode element and shift. */
19870 : break;
19871 :
19872 : default:
19873 : break;
19874 : }
19875 :
19876 27749 : if (use_vec_extr)
19877 : {
19878 93945 : tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (elt)));
19879 93945 : tmp = gen_rtx_VEC_SELECT (inner_mode, vec, tmp);
19880 :
19881 : /* Let the rtl optimizers know about the zero extension performed. */
19882 93945 : if (inner_mode == QImode || inner_mode == HImode)
19883 : {
19884 8818 : rtx reg = gen_reg_rtx (SImode);
19885 8818 : tmp = gen_rtx_ZERO_EXTEND (SImode, tmp);
19886 8818 : emit_move_insn (reg, tmp);
19887 8818 : tmp = gen_lowpart (inner_mode, reg);
19888 8818 : SUBREG_PROMOTED_VAR_P (tmp) = 1;
19889 8818 : SUBREG_PROMOTED_SET (tmp, 1);
19890 : }
19891 :
19892 93945 : emit_move_insn (target, tmp);
19893 : }
19894 : else
19895 : {
19896 16180 : rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode));
19897 :
19898 8090 : emit_move_insn (mem, vec);
19899 :
19900 16180 : tmp = adjust_address (mem, inner_mode, elt*GET_MODE_SIZE (inner_mode));
19901 8090 : emit_move_insn (target, tmp);
19902 : }
19903 : }
19904 :
19905 : /* Generate code to copy vector bits i / 2 ... i - 1 from vector SRC
19906 : to bits 0 ... i / 2 - 1 of vector DEST, which has the same mode.
19907 : The upper bits of DEST are undefined, though they shouldn't cause
19908 : exceptions (some bits from src or all zeros are ok). */
19909 :
19910 : static void
19911 41828 : emit_reduc_half (rtx dest, rtx src, int i)
19912 : {
19913 41828 : rtx tem, d = dest;
19914 41828 : switch (GET_MODE (src))
19915 : {
19916 6214 : case E_V4SFmode:
19917 6214 : if (i == 128)
19918 3107 : tem = gen_sse_movhlps (dest, src, src);
19919 : else
19920 3107 : tem = gen_sse_shufps_v4sf (dest, src, src, const1_rtx, const1_rtx,
19921 : GEN_INT (1 + 4), GEN_INT (1 + 4));
19922 : break;
19923 3355 : case E_V2DFmode:
19924 3355 : tem = gen_vec_interleave_highv2df (dest, src, src);
19925 3355 : break;
19926 76 : case E_V4QImode:
19927 76 : d = gen_reg_rtx (V1SImode);
19928 76 : tem = gen_mmx_lshrv1si3 (d, gen_lowpart (V1SImode, src),
19929 76 : GEN_INT (i / 2));
19930 76 : break;
19931 600 : case E_V8QImode:
19932 600 : case E_V4HImode:
19933 600 : d = gen_reg_rtx (V1DImode);
19934 600 : tem = gen_mmx_lshrv1di3 (d, gen_lowpart (V1DImode, src),
19935 600 : GEN_INT (i / 2));
19936 600 : break;
19937 31583 : case E_V16QImode:
19938 31583 : case E_V8HImode:
19939 31583 : case E_V8HFmode:
19940 31583 : case E_V4SImode:
19941 31583 : case E_V2DImode:
19942 31583 : if (TARGET_SSE_REDUCTION_PREFER_PSHUF)
19943 : {
19944 15 : if (i == 128)
19945 : {
19946 9 : d = gen_reg_rtx (V4SImode);
19947 18 : tem = gen_sse2_pshufd_1 (
19948 9 : d, force_reg (V4SImode, gen_lowpart (V4SImode, src)),
19949 : GEN_INT (2), GEN_INT (3), GEN_INT (2), GEN_INT (3));
19950 9 : break;
19951 : }
19952 6 : else if (i == 64)
19953 : {
19954 5 : d = gen_reg_rtx (V4SImode);
19955 10 : tem = gen_sse2_pshufd_1 (
19956 5 : d, force_reg (V4SImode, gen_lowpart (V4SImode, src)),
19957 : GEN_INT (1), GEN_INT (1), GEN_INT (1), GEN_INT (1));
19958 5 : break;
19959 : }
19960 1 : else if (i == 32)
19961 : {
19962 1 : d = gen_reg_rtx (V8HImode);
19963 2 : tem = gen_sse2_pshuflw_1 (
19964 1 : d, force_reg (V8HImode, gen_lowpart (V8HImode, src)),
19965 : GEN_INT (1), GEN_INT (1), GEN_INT (1), GEN_INT (1));
19966 1 : break;
19967 : }
19968 : }
19969 31568 : d = gen_reg_rtx (V1TImode);
19970 31568 : tem = gen_sse2_lshrv1ti3 (d, gen_lowpart (V1TImode, src),
19971 31568 : GEN_INT (i / 2));
19972 31568 : break;
19973 0 : case E_V8SFmode:
19974 0 : if (i == 256)
19975 0 : tem = gen_avx_vperm2f128v8sf3 (dest, src, src, const1_rtx);
19976 : else
19977 0 : tem = gen_avx_shufps256 (dest, src, src,
19978 : GEN_INT (i == 128 ? 2 + (3 << 2) : 1));
19979 : break;
19980 0 : case E_V4DFmode:
19981 0 : if (i == 256)
19982 0 : tem = gen_avx_vperm2f128v4df3 (dest, src, src, const1_rtx);
19983 : else
19984 0 : tem = gen_avx_shufpd256 (dest, src, src, const1_rtx);
19985 : break;
19986 0 : case E_V32QImode:
19987 0 : case E_V16HImode:
19988 0 : case E_V16HFmode:
19989 0 : case E_V8SImode:
19990 0 : case E_V4DImode:
19991 0 : if (i == 256)
19992 : {
19993 0 : if (GET_MODE (dest) != V4DImode)
19994 0 : d = gen_reg_rtx (V4DImode);
19995 0 : tem = gen_avx2_permv2ti (d, gen_lowpart (V4DImode, src),
19996 0 : gen_lowpart (V4DImode, src),
19997 : const1_rtx);
19998 : }
19999 : else
20000 : {
20001 0 : d = gen_reg_rtx (V2TImode);
20002 0 : tem = gen_avx2_lshrv2ti3 (d, gen_lowpart (V2TImode, src),
20003 0 : GEN_INT (i / 2));
20004 : }
20005 : break;
20006 0 : case E_V64QImode:
20007 0 : case E_V32HImode:
20008 0 : case E_V32HFmode:
20009 0 : if (i < 64)
20010 : {
20011 0 : d = gen_reg_rtx (V4TImode);
20012 0 : tem = gen_avx512bw_lshrv4ti3 (d, gen_lowpart (V4TImode, src),
20013 0 : GEN_INT (i / 2));
20014 0 : break;
20015 : }
20016 : /* FALLTHRU */
20017 0 : case E_V16SImode:
20018 0 : case E_V16SFmode:
20019 0 : case E_V8DImode:
20020 0 : case E_V8DFmode:
20021 0 : if (i > 128)
20022 0 : tem = gen_avx512f_shuf_i32x4_1 (gen_lowpart (V16SImode, dest),
20023 0 : gen_lowpart (V16SImode, src),
20024 0 : gen_lowpart (V16SImode, src),
20025 : GEN_INT (0x4 + (i == 512 ? 4 : 0)),
20026 : GEN_INT (0x5 + (i == 512 ? 4 : 0)),
20027 : GEN_INT (0x6 + (i == 512 ? 4 : 0)),
20028 : GEN_INT (0x7 + (i == 512 ? 4 : 0)),
20029 : GEN_INT (0xC), GEN_INT (0xD),
20030 : GEN_INT (0xE), GEN_INT (0xF),
20031 : GEN_INT (0x10), GEN_INT (0x11),
20032 : GEN_INT (0x12), GEN_INT (0x13),
20033 : GEN_INT (0x14), GEN_INT (0x15),
20034 : GEN_INT (0x16), GEN_INT (0x17));
20035 : else
20036 0 : tem = gen_avx512f_pshufd_1 (gen_lowpart (V16SImode, dest),
20037 0 : gen_lowpart (V16SImode, src),
20038 : GEN_INT (i == 128 ? 0x2 : 0x1),
20039 : GEN_INT (0x3),
20040 : GEN_INT (0x3),
20041 : GEN_INT (0x3),
20042 : GEN_INT (i == 128 ? 0x6 : 0x5),
20043 : GEN_INT (0x7),
20044 : GEN_INT (0x7),
20045 : GEN_INT (0x7),
20046 : GEN_INT (i == 128 ? 0xA : 0x9),
20047 : GEN_INT (0xB),
20048 : GEN_INT (0xB),
20049 : GEN_INT (0xB),
20050 : GEN_INT (i == 128 ? 0xE : 0xD),
20051 : GEN_INT (0xF),
20052 : GEN_INT (0xF),
20053 : GEN_INT (0xF));
20054 : break;
20055 0 : default:
20056 0 : gcc_unreachable ();
20057 : }
20058 41828 : emit_insn (tem);
20059 41828 : if (d != dest)
20060 32259 : emit_move_insn (dest, gen_lowpart (GET_MODE (dest), d));
20061 41828 : }
20062 :
20063 : /* Expand a vector reduction. FN is the binary pattern to reduce;
20064 : DEST is the destination; IN is the input vector. */
20065 :
20066 : void
20067 20760 : ix86_expand_reduc (rtx (*fn) (rtx, rtx, rtx), rtx dest, rtx in)
20068 : {
20069 20760 : rtx half, dst, vec = in;
20070 20760 : machine_mode mode = GET_MODE (in);
20071 20760 : int i;
20072 :
20073 : /* SSE4 has a special instruction for V8HImode UMIN reduction. */
20074 20760 : if (TARGET_SSE4_1
20075 10026 : && mode == V8HImode
20076 780 : && fn == gen_uminv8hi3)
20077 : {
20078 4 : emit_insn (gen_sse4_1_phminposuw (dest, in));
20079 4 : return;
20080 : }
20081 :
20082 41512 : for (i = GET_MODE_BITSIZE (mode);
20083 125168 : i > GET_MODE_UNIT_BITSIZE (mode);
20084 41828 : i >>= 1)
20085 : {
20086 41828 : half = gen_reg_rtx (mode);
20087 41828 : emit_reduc_half (half, vec, i);
20088 83656 : if (i == GET_MODE_UNIT_BITSIZE (mode) * 2)
20089 : dst = dest;
20090 : else
20091 21072 : dst = gen_reg_rtx (mode);
20092 41828 : emit_insn (fn (dst, half, vec));
20093 41828 : vec = dst;
20094 : }
20095 : }
20096 :
20097 : /* Output code to perform a conditional jump to LABEL, if C2 flag in
20098 : FP status register is set. */
20099 :
20100 : void
20101 285 : ix86_emit_fp_unordered_jump (rtx label)
20102 : {
20103 285 : rtx reg = gen_reg_rtx (HImode);
20104 285 : rtx_insn *insn;
20105 285 : rtx temp;
20106 :
20107 285 : emit_insn (gen_x86_fnstsw_1 (reg));
20108 :
20109 285 : if (TARGET_SAHF && (TARGET_USE_SAHF || optimize_insn_for_size_p ()))
20110 : {
20111 37 : emit_insn (gen_x86_sahf_1 (reg));
20112 :
20113 37 : temp = gen_rtx_REG (CCmode, FLAGS_REG);
20114 37 : temp = gen_rtx_UNORDERED (VOIDmode, temp, const0_rtx);
20115 : }
20116 : else
20117 : {
20118 248 : emit_insn (gen_testqi_ext_1_ccno (reg, GEN_INT (0x04)));
20119 :
20120 248 : temp = gen_rtx_REG (CCNOmode, FLAGS_REG);
20121 248 : temp = gen_rtx_NE (VOIDmode, temp, const0_rtx);
20122 : }
20123 :
20124 285 : temp = gen_rtx_IF_THEN_ELSE (VOIDmode, temp,
20125 : gen_rtx_LABEL_REF (VOIDmode, label),
20126 : pc_rtx);
20127 285 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, temp));
20128 285 : predict_jump (REG_BR_PROB_BASE * 10 / 100);
20129 285 : JUMP_LABEL (insn) = label;
20130 285 : }
20131 :
20132 : /* Output code to perform an sinh XFmode calculation. */
20133 :
20134 : void
20135 2 : ix86_emit_i387_sinh (rtx op0, rtx op1)
20136 : {
20137 2 : rtx e1 = gen_reg_rtx (XFmode);
20138 2 : rtx e2 = gen_reg_rtx (XFmode);
20139 2 : rtx scratch = gen_reg_rtx (HImode);
20140 2 : rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
20141 2 : rtx half = const_double_from_real_value (dconsthalf, XFmode);
20142 2 : rtx cst1, tmp;
20143 2 : rtx_code_label *jump_label = gen_label_rtx ();
20144 2 : rtx_insn *insn;
20145 :
20146 : /* scratch = fxam (op1) */
20147 2 : emit_insn (gen_fxamxf2_i387 (scratch, op1));
20148 :
20149 : /* e1 = expm1 (|op1|) */
20150 2 : emit_insn (gen_absxf2 (e2, op1));
20151 2 : emit_insn (gen_expm1xf2 (e1, e2));
20152 :
20153 : /* e2 = e1 / (e1 + 1.0) + e1 */
20154 2 : cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
20155 2 : emit_insn (gen_addxf3 (e2, e1, cst1));
20156 2 : emit_insn (gen_divxf3 (e2, e1, e2));
20157 2 : emit_insn (gen_addxf3 (e2, e2, e1));
20158 :
20159 : /* flags = signbit (op1) */
20160 2 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
20161 :
20162 : /* if (flags) then e2 = -e2 */
20163 2 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
20164 : gen_rtx_EQ (VOIDmode, flags, const0_rtx),
20165 : gen_rtx_LABEL_REF (VOIDmode, jump_label),
20166 : pc_rtx);
20167 2 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
20168 2 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
20169 2 : JUMP_LABEL (insn) = jump_label;
20170 :
20171 2 : emit_insn (gen_negxf2 (e2, e2));
20172 :
20173 2 : emit_label (jump_label);
20174 2 : LABEL_NUSES (jump_label) = 1;
20175 :
20176 : /* op0 = 0.5 * e2 */
20177 2 : half = force_reg (XFmode, half);
20178 2 : emit_insn (gen_mulxf3 (op0, e2, half));
20179 2 : }
20180 :
20181 : /* Output code to perform an cosh XFmode calculation. */
20182 :
20183 : void
20184 3 : ix86_emit_i387_cosh (rtx op0, rtx op1)
20185 : {
20186 3 : rtx e1 = gen_reg_rtx (XFmode);
20187 3 : rtx e2 = gen_reg_rtx (XFmode);
20188 3 : rtx half = const_double_from_real_value (dconsthalf, XFmode);
20189 3 : rtx cst1;
20190 :
20191 : /* e1 = exp (op1) */
20192 3 : emit_insn (gen_expxf2 (e1, op1));
20193 :
20194 : /* e2 = e1 + 1.0 / e1 */
20195 3 : cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
20196 3 : emit_insn (gen_divxf3 (e2, cst1, e1));
20197 3 : emit_insn (gen_addxf3 (e2, e1, e2));
20198 :
20199 : /* op0 = 0.5 * e2 */
20200 3 : half = force_reg (XFmode, half);
20201 3 : emit_insn (gen_mulxf3 (op0, e2, half));
20202 3 : }
20203 :
20204 : /* Output code to perform an tanh XFmode calculation. */
20205 :
20206 : void
20207 1 : ix86_emit_i387_tanh (rtx op0, rtx op1)
20208 : {
20209 1 : rtx e1 = gen_reg_rtx (XFmode);
20210 1 : rtx e2 = gen_reg_rtx (XFmode);
20211 1 : rtx scratch = gen_reg_rtx (HImode);
20212 1 : rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
20213 1 : rtx cst2, tmp;
20214 1 : rtx_code_label *jump_label = gen_label_rtx ();
20215 1 : rtx_insn *insn;
20216 :
20217 : /* scratch = fxam (op1) */
20218 1 : emit_insn (gen_fxamxf2_i387 (scratch, op1));
20219 :
20220 : /* e1 = expm1 (-|2 * op1|) */
20221 1 : emit_insn (gen_addxf3 (e2, op1, op1));
20222 1 : emit_insn (gen_absxf2 (e2, e2));
20223 1 : emit_insn (gen_negxf2 (e2, e2));
20224 1 : emit_insn (gen_expm1xf2 (e1, e2));
20225 :
20226 : /* e2 = e1 / (e1 + 2.0) */
20227 1 : cst2 = force_reg (XFmode, CONST2_RTX (XFmode));
20228 1 : emit_insn (gen_addxf3 (e2, e1, cst2));
20229 1 : emit_insn (gen_divxf3 (e2, e1, e2));
20230 :
20231 : /* flags = signbit (op1) */
20232 1 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
20233 :
20234 : /* if (!flags) then e2 = -e2 */
20235 1 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
20236 : gen_rtx_NE (VOIDmode, flags, const0_rtx),
20237 : gen_rtx_LABEL_REF (VOIDmode, jump_label),
20238 : pc_rtx);
20239 1 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
20240 1 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
20241 1 : JUMP_LABEL (insn) = jump_label;
20242 :
20243 1 : emit_insn (gen_negxf2 (e2, e2));
20244 :
20245 1 : emit_label (jump_label);
20246 1 : LABEL_NUSES (jump_label) = 1;
20247 :
20248 1 : emit_move_insn (op0, e2);
20249 1 : }
20250 :
20251 : /* Output code to perform an asinh XFmode calculation. */
20252 :
20253 : void
20254 0 : ix86_emit_i387_asinh (rtx op0, rtx op1)
20255 : {
20256 0 : rtx e1 = gen_reg_rtx (XFmode);
20257 0 : rtx e2 = gen_reg_rtx (XFmode);
20258 0 : rtx scratch = gen_reg_rtx (HImode);
20259 0 : rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
20260 0 : rtx cst1, tmp;
20261 0 : rtx_code_label *jump_label = gen_label_rtx ();
20262 0 : rtx_insn *insn;
20263 :
20264 : /* e2 = sqrt (op1^2 + 1.0) + 1.0 */
20265 0 : emit_insn (gen_mulxf3 (e1, op1, op1));
20266 0 : cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
20267 0 : emit_insn (gen_addxf3 (e2, e1, cst1));
20268 0 : emit_insn (gen_sqrtxf2 (e2, e2));
20269 0 : emit_insn (gen_addxf3 (e2, e2, cst1));
20270 :
20271 : /* e1 = e1 / e2 */
20272 0 : emit_insn (gen_divxf3 (e1, e1, e2));
20273 :
20274 : /* scratch = fxam (op1) */
20275 0 : emit_insn (gen_fxamxf2_i387 (scratch, op1));
20276 :
20277 : /* e1 = e1 + |op1| */
20278 0 : emit_insn (gen_absxf2 (e2, op1));
20279 0 : emit_insn (gen_addxf3 (e1, e1, e2));
20280 :
20281 : /* e2 = log1p (e1) */
20282 0 : ix86_emit_i387_log1p (e2, e1);
20283 :
20284 : /* flags = signbit (op1) */
20285 0 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
20286 :
20287 : /* if (flags) then e2 = -e2 */
20288 0 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
20289 : gen_rtx_EQ (VOIDmode, flags, const0_rtx),
20290 : gen_rtx_LABEL_REF (VOIDmode, jump_label),
20291 : pc_rtx);
20292 0 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
20293 0 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
20294 0 : JUMP_LABEL (insn) = jump_label;
20295 :
20296 0 : emit_insn (gen_negxf2 (e2, e2));
20297 :
20298 0 : emit_label (jump_label);
20299 0 : LABEL_NUSES (jump_label) = 1;
20300 :
20301 0 : emit_move_insn (op0, e2);
20302 0 : }
20303 :
20304 : /* Output code to perform an acosh XFmode calculation. */
20305 :
20306 : void
20307 0 : ix86_emit_i387_acosh (rtx op0, rtx op1)
20308 : {
20309 0 : rtx e1 = gen_reg_rtx (XFmode);
20310 0 : rtx e2 = gen_reg_rtx (XFmode);
20311 0 : rtx cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
20312 :
20313 : /* e2 = sqrt (op1 + 1.0) */
20314 0 : emit_insn (gen_addxf3 (e2, op1, cst1));
20315 0 : emit_insn (gen_sqrtxf2 (e2, e2));
20316 :
20317 : /* e1 = sqrt (op1 - 1.0) */
20318 0 : emit_insn (gen_subxf3 (e1, op1, cst1));
20319 0 : emit_insn (gen_sqrtxf2 (e1, e1));
20320 :
20321 : /* e1 = e1 * e2 */
20322 0 : emit_insn (gen_mulxf3 (e1, e1, e2));
20323 :
20324 : /* e1 = e1 + op1 */
20325 0 : emit_insn (gen_addxf3 (e1, e1, op1));
20326 :
20327 : /* op0 = log (e1) */
20328 0 : emit_insn (gen_logxf2 (op0, e1));
20329 0 : }
20330 :
20331 : /* Output code to perform an atanh XFmode calculation. */
20332 :
20333 : void
20334 4 : ix86_emit_i387_atanh (rtx op0, rtx op1)
20335 : {
20336 4 : rtx e1 = gen_reg_rtx (XFmode);
20337 4 : rtx e2 = gen_reg_rtx (XFmode);
20338 4 : rtx scratch = gen_reg_rtx (HImode);
20339 4 : rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
20340 4 : rtx half = const_double_from_real_value (dconsthalf, XFmode);
20341 4 : rtx cst1, tmp;
20342 4 : rtx_code_label *jump_label = gen_label_rtx ();
20343 4 : rtx_insn *insn;
20344 :
20345 : /* scratch = fxam (op1) */
20346 4 : emit_insn (gen_fxamxf2_i387 (scratch, op1));
20347 :
20348 : /* e2 = |op1| */
20349 4 : emit_insn (gen_absxf2 (e2, op1));
20350 :
20351 : /* e1 = -(e2 + e2) / (e2 + 1.0) */
20352 4 : cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
20353 4 : emit_insn (gen_addxf3 (e1, e2, cst1));
20354 4 : emit_insn (gen_addxf3 (e2, e2, e2));
20355 4 : emit_insn (gen_negxf2 (e2, e2));
20356 4 : emit_insn (gen_divxf3 (e1, e2, e1));
20357 :
20358 : /* e2 = log1p (e1) */
20359 4 : ix86_emit_i387_log1p (e2, e1);
20360 :
20361 : /* flags = signbit (op1) */
20362 4 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
20363 :
20364 : /* if (!flags) then e2 = -e2 */
20365 4 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
20366 : gen_rtx_NE (VOIDmode, flags, const0_rtx),
20367 : gen_rtx_LABEL_REF (VOIDmode, jump_label),
20368 : pc_rtx);
20369 4 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
20370 4 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
20371 4 : JUMP_LABEL (insn) = jump_label;
20372 :
20373 4 : emit_insn (gen_negxf2 (e2, e2));
20374 :
20375 4 : emit_label (jump_label);
20376 4 : LABEL_NUSES (jump_label) = 1;
20377 :
20378 : /* op0 = 0.5 * e2 */
20379 4 : half = force_reg (XFmode, half);
20380 4 : emit_insn (gen_mulxf3 (op0, e2, half));
20381 4 : }
20382 :
20383 : /* Output code to perform a log1p XFmode calculation. */
20384 :
20385 : void
20386 5 : ix86_emit_i387_log1p (rtx op0, rtx op1)
20387 : {
20388 5 : rtx_code_label *label1 = gen_label_rtx ();
20389 5 : rtx_code_label *label2 = gen_label_rtx ();
20390 :
20391 5 : rtx tmp = gen_reg_rtx (XFmode);
20392 5 : rtx res = gen_reg_rtx (XFmode);
20393 5 : rtx cst, cstln2, cst1;
20394 5 : rtx_insn *insn;
20395 :
20396 : /* The emit_jump call emits pending stack adjust, make sure it is emitted
20397 : before the conditional jump, otherwise the stack adjustment will be
20398 : only conditional. */
20399 5 : do_pending_stack_adjust ();
20400 :
20401 5 : cst = const_double_from_real_value
20402 5 : (REAL_VALUE_ATOF ("0.29289321881345247561810596348408353", XFmode), XFmode);
20403 5 : cstln2 = force_reg (XFmode, standard_80387_constant_rtx (4)); /* fldln2 */
20404 :
20405 5 : emit_insn (gen_absxf2 (tmp, op1));
20406 :
20407 5 : cst = force_reg (XFmode, cst);
20408 5 : ix86_expand_branch (GE, tmp, cst, label1);
20409 5 : predict_jump (REG_BR_PROB_BASE * 10 / 100);
20410 5 : insn = get_last_insn ();
20411 5 : JUMP_LABEL (insn) = label1;
20412 :
20413 5 : emit_insn (gen_fyl2xp1xf3_i387 (res, op1, cstln2));
20414 5 : emit_jump (label2);
20415 :
20416 5 : emit_label (label1);
20417 5 : LABEL_NUSES (label1) = 1;
20418 :
20419 5 : cst1 = force_reg (XFmode, CONST1_RTX (XFmode));
20420 5 : emit_insn (gen_rtx_SET (tmp, gen_rtx_PLUS (XFmode, op1, cst1)));
20421 5 : emit_insn (gen_fyl2xxf3_i387 (res, tmp, cstln2));
20422 :
20423 5 : emit_label (label2);
20424 5 : LABEL_NUSES (label2) = 1;
20425 :
20426 5 : emit_move_insn (op0, res);
20427 5 : }
20428 :
20429 : /* Emit code for round calculation. */
20430 : void
20431 60 : ix86_emit_i387_round (rtx op0, rtx op1)
20432 : {
20433 60 : machine_mode inmode = GET_MODE (op1);
20434 60 : machine_mode outmode = GET_MODE (op0);
20435 60 : rtx e1 = gen_reg_rtx (XFmode);
20436 60 : rtx e2 = gen_reg_rtx (XFmode);
20437 60 : rtx scratch = gen_reg_rtx (HImode);
20438 60 : rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
20439 60 : rtx half = const_double_from_real_value (dconsthalf, XFmode);
20440 60 : rtx res = gen_reg_rtx (outmode);
20441 60 : rtx_code_label *jump_label = gen_label_rtx ();
20442 60 : rtx (*floor_insn) (rtx, rtx);
20443 60 : rtx (*neg_insn) (rtx, rtx);
20444 60 : rtx_insn *insn;
20445 60 : rtx tmp;
20446 :
20447 60 : switch (inmode)
20448 : {
20449 29 : case E_SFmode:
20450 29 : case E_DFmode:
20451 29 : tmp = gen_reg_rtx (XFmode);
20452 :
20453 29 : emit_insn (gen_rtx_SET (tmp, gen_rtx_FLOAT_EXTEND (XFmode, op1)));
20454 29 : op1 = tmp;
20455 29 : break;
20456 : case E_XFmode:
20457 : break;
20458 0 : default:
20459 0 : gcc_unreachable ();
20460 : }
20461 :
20462 60 : switch (outmode)
20463 : {
20464 : case E_SFmode:
20465 : floor_insn = gen_frndintxf2_floor;
20466 : neg_insn = gen_negsf2;
20467 : break;
20468 6 : case E_DFmode:
20469 6 : floor_insn = gen_frndintxf2_floor;
20470 6 : neg_insn = gen_negdf2;
20471 6 : break;
20472 10 : case E_XFmode:
20473 10 : floor_insn = gen_frndintxf2_floor;
20474 10 : neg_insn = gen_negxf2;
20475 10 : break;
20476 0 : case E_HImode:
20477 0 : floor_insn = gen_lfloorxfhi2;
20478 0 : neg_insn = gen_neghi2;
20479 0 : break;
20480 6 : case E_SImode:
20481 6 : floor_insn = gen_lfloorxfsi2;
20482 6 : neg_insn = gen_negsi2;
20483 6 : break;
20484 36 : case E_DImode:
20485 36 : floor_insn = gen_lfloorxfdi2;
20486 36 : neg_insn = gen_negdi2;
20487 36 : break;
20488 0 : default:
20489 0 : gcc_unreachable ();
20490 : }
20491 :
20492 : /* round(a) = sgn(a) * floor(fabs(a) + 0.5) */
20493 :
20494 : /* scratch = fxam(op1) */
20495 60 : emit_insn (gen_fxamxf2_i387 (scratch, op1));
20496 :
20497 : /* e1 = fabs(op1) */
20498 60 : emit_insn (gen_absxf2 (e1, op1));
20499 :
20500 : /* e2 = e1 + 0.5 */
20501 60 : half = force_reg (XFmode, half);
20502 60 : emit_insn (gen_rtx_SET (e2, gen_rtx_PLUS (XFmode, e1, half)));
20503 :
20504 : /* res = floor(e2) */
20505 60 : switch (outmode)
20506 : {
20507 8 : case E_SFmode:
20508 8 : case E_DFmode:
20509 8 : {
20510 8 : tmp = gen_reg_rtx (XFmode);
20511 :
20512 8 : emit_insn (floor_insn (tmp, e2));
20513 8 : emit_insn (gen_rtx_SET (res,
20514 : gen_rtx_UNSPEC (outmode, gen_rtvec (1, tmp),
20515 : UNSPEC_TRUNC_NOOP)));
20516 : }
20517 8 : break;
20518 52 : default:
20519 52 : emit_insn (floor_insn (res, e2));
20520 : }
20521 :
20522 : /* flags = signbit(a) */
20523 60 : emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x02)));
20524 :
20525 : /* if (flags) then res = -res */
20526 60 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
20527 : gen_rtx_EQ (VOIDmode, flags, const0_rtx),
20528 : gen_rtx_LABEL_REF (VOIDmode, jump_label),
20529 : pc_rtx);
20530 60 : insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
20531 60 : predict_jump (REG_BR_PROB_BASE * 50 / 100);
20532 60 : JUMP_LABEL (insn) = jump_label;
20533 :
20534 60 : emit_insn (neg_insn (res, res));
20535 :
20536 60 : emit_label (jump_label);
20537 60 : LABEL_NUSES (jump_label) = 1;
20538 :
20539 60 : emit_move_insn (op0, res);
20540 60 : }
20541 :
20542 : /* Output code to perform a Newton-Rhapson approximation of a single precision
20543 : floating point divide [http://en.wikipedia.org/wiki/N-th_root_algorithm]. */
20544 :
20545 : void
20546 56 : ix86_emit_swdivsf (rtx res, rtx a, rtx b, machine_mode mode)
20547 : {
20548 56 : rtx x0, x1, e0, e1;
20549 :
20550 56 : x0 = gen_reg_rtx (mode);
20551 56 : e0 = gen_reg_rtx (mode);
20552 56 : e1 = gen_reg_rtx (mode);
20553 56 : x1 = gen_reg_rtx (mode);
20554 :
20555 56 : b = force_reg (mode, b);
20556 :
20557 : /* x0 = rcp(b) estimate */
20558 56 : if (mode == V16SFmode || mode == V8DFmode)
20559 : {
20560 0 : emit_insn (gen_rtx_SET (x0, gen_rtx_UNSPEC (mode, gen_rtvec (1, b),
20561 : UNSPEC_RCP14)));
20562 : }
20563 : else
20564 56 : emit_insn (gen_rtx_SET (x0, gen_rtx_UNSPEC (mode, gen_rtvec (1, b),
20565 : UNSPEC_RCP)));
20566 :
20567 56 : unsigned vector_size = GET_MODE_SIZE (mode);
20568 :
20569 : /* (a - (rcp(b) * a * b)) * rcp(b) + rcp(b) * a
20570 : N-R step with 2 fma implementation. */
20571 56 : if (TARGET_FMA
20572 55 : || (TARGET_AVX512F && vector_size == 64)
20573 55 : || (TARGET_AVX512VL && (vector_size == 32 || vector_size == 16)))
20574 : {
20575 : /* e0 = x0 * a */
20576 1 : emit_insn (gen_rtx_SET (e0, gen_rtx_MULT (mode, x0, a)));
20577 : /* e1 = e0 * b - a */
20578 1 : emit_insn (gen_rtx_SET (e1, gen_rtx_FMA (mode, e0, b,
20579 : gen_rtx_NEG (mode, a))));
20580 : /* res = - e1 * x0 + e0 */
20581 1 : emit_insn (gen_rtx_SET (res, gen_rtx_FMA (mode,
20582 : gen_rtx_NEG (mode, e1),
20583 : x0, e0)));
20584 : }
20585 : else
20586 : /* a / b = a * ((rcp(b) + rcp(b)) - (b * rcp(b) * rcp (b))) */
20587 : {
20588 : /* e0 = x0 * b */
20589 55 : emit_insn (gen_rtx_SET (e0, gen_rtx_MULT (mode, x0, b)));
20590 :
20591 : /* e1 = x0 + x0 */
20592 55 : emit_insn (gen_rtx_SET (e1, gen_rtx_PLUS (mode, x0, x0)));
20593 :
20594 : /* e0 = x0 * e0 */
20595 55 : emit_insn (gen_rtx_SET (e0, gen_rtx_MULT (mode, x0, e0)));
20596 :
20597 : /* x1 = e1 - e0 */
20598 55 : emit_insn (gen_rtx_SET (x1, gen_rtx_MINUS (mode, e1, e0)));
20599 :
20600 : /* res = a * x1 */
20601 55 : emit_insn (gen_rtx_SET (res, gen_rtx_MULT (mode, a, x1)));
20602 : }
20603 56 : }
20604 :
20605 : /* Output code to perform a Newton-Rhapson approximation of a
20606 : single precision floating point [reciprocal] square root. */
20607 :
20608 : void
20609 85 : ix86_emit_swsqrtsf (rtx res, rtx a, machine_mode mode, bool recip)
20610 : {
20611 85 : rtx x0, e0, e1, e2, e3, mthree, mhalf;
20612 85 : REAL_VALUE_TYPE r;
20613 85 : int unspec;
20614 :
20615 85 : x0 = gen_reg_rtx (mode);
20616 85 : e0 = gen_reg_rtx (mode);
20617 85 : e1 = gen_reg_rtx (mode);
20618 85 : e2 = gen_reg_rtx (mode);
20619 85 : e3 = gen_reg_rtx (mode);
20620 :
20621 85 : real_from_integer (&r, VOIDmode, -3, SIGNED);
20622 85 : mthree = const_double_from_real_value (r, SFmode);
20623 :
20624 85 : real_arithmetic (&r, NEGATE_EXPR, &dconsthalf, NULL);
20625 85 : mhalf = const_double_from_real_value (r, SFmode);
20626 85 : unspec = UNSPEC_RSQRT;
20627 :
20628 85 : if (VECTOR_MODE_P (mode))
20629 : {
20630 66 : mthree = ix86_build_const_vector (mode, true, mthree);
20631 66 : mhalf = ix86_build_const_vector (mode, true, mhalf);
20632 : /* There is no 512-bit rsqrt. There is however rsqrt14. */
20633 132 : if (GET_MODE_SIZE (mode) == 64)
20634 0 : unspec = UNSPEC_RSQRT14;
20635 : }
20636 :
20637 : /* sqrt(a) = -0.5 * a * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0)
20638 : rsqrt(a) = -0.5 * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0) */
20639 :
20640 85 : a = force_reg (mode, a);
20641 :
20642 : /* x0 = rsqrt(a) estimate */
20643 85 : emit_insn (gen_rtx_SET (x0, gen_rtx_UNSPEC (mode, gen_rtvec (1, a),
20644 : unspec)));
20645 :
20646 : /* If (a == 0.0) Filter out infinity to prevent NaN for sqrt(0.0). */
20647 85 : if (!recip)
20648 : {
20649 57 : rtx zero = force_reg (mode, CONST0_RTX(mode));
20650 57 : rtx mask;
20651 :
20652 : /* Handle masked compare. */
20653 110 : if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 64)
20654 : {
20655 0 : mask = gen_reg_rtx (HImode);
20656 : /* Imm value 0x4 corresponds to not-equal comparison. */
20657 0 : emit_insn (gen_avx512f_cmpv16sf3 (mask, zero, a, GEN_INT (0x4)));
20658 0 : emit_insn (gen_avx512f_blendmv16sf (x0, zero, x0, mask));
20659 : }
20660 : else
20661 : {
20662 57 : mask = gen_reg_rtx (mode);
20663 57 : emit_insn (gen_rtx_SET (mask, gen_rtx_NE (mode, zero, a)));
20664 57 : emit_insn (gen_rtx_SET (x0, gen_rtx_AND (mode, x0, mask)));
20665 : }
20666 : }
20667 :
20668 85 : mthree = force_reg (mode, mthree);
20669 :
20670 : /* e0 = x0 * a */
20671 85 : emit_insn (gen_rtx_SET (e0, gen_rtx_MULT (mode, x0, a)));
20672 :
20673 85 : unsigned vector_size = GET_MODE_SIZE (mode);
20674 85 : if (TARGET_FMA
20675 77 : || (TARGET_AVX512F && vector_size == 64)
20676 77 : || (TARGET_AVX512VL && (vector_size == 32 || vector_size == 16)))
20677 16 : emit_insn (gen_rtx_SET (e2,
20678 : gen_rtx_FMA (mode, e0, x0, mthree)));
20679 : else
20680 : {
20681 : /* e1 = e0 * x0 */
20682 69 : emit_insn (gen_rtx_SET (e1, gen_rtx_MULT (mode, e0, x0)));
20683 :
20684 : /* e2 = e1 - 3. */
20685 69 : emit_insn (gen_rtx_SET (e2, gen_rtx_PLUS (mode, e1, mthree)));
20686 : }
20687 :
20688 85 : mhalf = force_reg (mode, mhalf);
20689 85 : if (recip)
20690 : /* e3 = -.5 * x0 */
20691 28 : emit_insn (gen_rtx_SET (e3, gen_rtx_MULT (mode, x0, mhalf)));
20692 : else
20693 : /* e3 = -.5 * e0 */
20694 57 : emit_insn (gen_rtx_SET (e3, gen_rtx_MULT (mode, e0, mhalf)));
20695 : /* ret = e2 * e3 */
20696 85 : emit_insn (gen_rtx_SET (res, gen_rtx_MULT (mode, e2, e3)));
20697 85 : }
20698 :
20699 : /* Expand fabs (OP0) and return a new rtx that holds the result. The
20700 : mask for masking out the sign-bit is stored in *SMASK, if that is
20701 : non-null. */
20702 :
20703 : static rtx
20704 1047 : ix86_expand_sse_fabs (rtx op0, rtx *smask)
20705 : {
20706 1047 : machine_mode vmode, mode = GET_MODE (op0);
20707 1047 : rtx xa, mask;
20708 :
20709 1047 : xa = gen_reg_rtx (mode);
20710 1047 : if (mode == SFmode)
20711 : vmode = V4SFmode;
20712 465 : else if (mode == DFmode)
20713 : vmode = V2DFmode;
20714 : else
20715 0 : vmode = mode;
20716 1047 : mask = ix86_build_signbit_mask (vmode, VECTOR_MODE_P (mode), true);
20717 1047 : if (!VECTOR_MODE_P (mode))
20718 : {
20719 : /* We need to generate a scalar mode mask in this case. */
20720 1047 : rtx tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
20721 1047 : tmp = gen_rtx_VEC_SELECT (mode, mask, tmp);
20722 1047 : mask = gen_reg_rtx (mode);
20723 1047 : emit_insn (gen_rtx_SET (mask, tmp));
20724 : }
20725 1047 : emit_insn (gen_rtx_SET (xa, gen_rtx_AND (mode, op0, mask)));
20726 :
20727 1047 : if (smask)
20728 994 : *smask = mask;
20729 :
20730 1047 : return xa;
20731 : }
20732 :
20733 : /* Expands a comparison of OP0 with OP1 using comparison code CODE,
20734 : swapping the operands if SWAP_OPERANDS is true. The expanded
20735 : code is a forward jump to a newly created label in case the
20736 : comparison is true. The generated label rtx is returned. */
20737 : static rtx_code_label *
20738 1062 : ix86_expand_sse_compare_and_jump (enum rtx_code code, rtx op0, rtx op1,
20739 : bool swap_operands)
20740 : {
20741 1062 : bool unordered_compare = ix86_unordered_fp_compare (code);
20742 1062 : rtx_code_label *label;
20743 1062 : rtx tmp, reg;
20744 :
20745 1062 : if (swap_operands)
20746 34 : std::swap (op0, op1);
20747 :
20748 1062 : label = gen_label_rtx ();
20749 1062 : tmp = gen_rtx_COMPARE (CCFPmode, op0, op1);
20750 1062 : if (unordered_compare)
20751 906 : tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_NOTRAP);
20752 1062 : reg = gen_rtx_REG (CCFPmode, FLAGS_REG);
20753 1062 : emit_insn (gen_rtx_SET (reg, tmp));
20754 1062 : tmp = gen_rtx_fmt_ee (code, VOIDmode, reg, const0_rtx);
20755 1062 : tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
20756 : gen_rtx_LABEL_REF (VOIDmode, label), pc_rtx);
20757 1062 : tmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
20758 1062 : JUMP_LABEL (tmp) = label;
20759 :
20760 1062 : return label;
20761 : }
20762 :
20763 : /* Expand a mask generating SSE comparison instruction comparing OP0 with OP1
20764 : using comparison code CODE. Operands are swapped for the comparison if
20765 : SWAP_OPERANDS is true. Returns a rtx for the generated mask. */
20766 : static rtx
20767 538 : ix86_expand_sse_compare_mask (enum rtx_code code, rtx op0, rtx op1,
20768 : bool swap_operands)
20769 : {
20770 538 : rtx (*insn)(rtx, rtx, rtx, rtx);
20771 538 : machine_mode mode = GET_MODE (op0);
20772 538 : rtx mask = gen_reg_rtx (mode);
20773 :
20774 538 : if (swap_operands)
20775 361 : std::swap (op0, op1);
20776 :
20777 538 : insn = mode == DFmode ? gen_setcc_df_sse : gen_setcc_sf_sse;
20778 :
20779 538 : emit_insn (insn (mask, op0, op1,
20780 : gen_rtx_fmt_ee (code, mode, op0, op1)));
20781 538 : return mask;
20782 : }
20783 :
20784 : /* Expand copysign from SIGN to the positive value ABS_VALUE
20785 : storing in RESULT. If MASK is non-null, it shall be a mask to mask out
20786 : the sign-bit. */
20787 :
20788 : static void
20789 1014 : ix86_sse_copysign_to_positive (rtx result, rtx abs_value, rtx sign, rtx mask)
20790 : {
20791 1014 : machine_mode mode = GET_MODE (sign);
20792 1014 : rtx sgn = gen_reg_rtx (mode);
20793 1014 : if (mask == NULL_RTX)
20794 : {
20795 28 : machine_mode vmode;
20796 :
20797 28 : if (mode == SFmode)
20798 : vmode = V4SFmode;
20799 : else if (mode == DFmode)
20800 : vmode = V2DFmode;
20801 : else if (mode == HFmode)
20802 : vmode = V8HFmode;
20803 : else
20804 28 : vmode = mode;
20805 :
20806 28 : mask = ix86_build_signbit_mask (vmode, VECTOR_MODE_P (mode), false);
20807 28 : if (!VECTOR_MODE_P (mode))
20808 : {
20809 : /* We need to generate a scalar mode mask in this case. */
20810 28 : rtx tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
20811 28 : tmp = gen_rtx_VEC_SELECT (mode, mask, tmp);
20812 28 : mask = gen_reg_rtx (mode);
20813 28 : emit_insn (gen_rtx_SET (mask, tmp));
20814 : }
20815 : }
20816 : else
20817 986 : mask = gen_rtx_NOT (mode, mask);
20818 1014 : emit_insn (gen_rtx_SET (sgn, gen_rtx_AND (mode, mask, sign)));
20819 1014 : emit_insn (gen_rtx_SET (result, gen_rtx_IOR (mode, abs_value, sgn)));
20820 1014 : }
20821 :
20822 : /* Expand SSE sequence for computing lround from OP1 storing
20823 : into OP0. */
20824 :
20825 : void
20826 28 : ix86_expand_lround (rtx op0, rtx op1)
20827 : {
20828 : /* C code for the stuff we're doing below:
20829 : tmp = op1 + copysign (nextafter (0.5, 0.0), op1)
20830 : return (long)tmp;
20831 : */
20832 28 : machine_mode mode = GET_MODE (op1);
20833 28 : const struct real_format *fmt;
20834 28 : REAL_VALUE_TYPE pred_half, half_minus_pred_half;
20835 28 : rtx adj;
20836 :
20837 : /* load nextafter (0.5, 0.0) */
20838 28 : fmt = REAL_MODE_FORMAT (mode);
20839 28 : real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
20840 28 : real_arithmetic (&pred_half, MINUS_EXPR, &dconsthalf, &half_minus_pred_half);
20841 :
20842 : /* adj = copysign (0.5, op1) */
20843 28 : adj = force_reg (mode, const_double_from_real_value (pred_half, mode));
20844 28 : ix86_sse_copysign_to_positive (adj, adj, force_reg (mode, op1), NULL_RTX);
20845 :
20846 : /* adj = op1 + adj */
20847 28 : adj = expand_simple_binop (mode, PLUS, adj, op1, NULL_RTX, 0, OPTAB_DIRECT);
20848 :
20849 : /* op0 = (imode)adj */
20850 28 : expand_fix (op0, adj, 0);
20851 28 : }
20852 :
20853 : /* Expand SSE2 sequence for computing lround from OPERAND1 storing
20854 : into OPERAND0. */
20855 :
20856 : void
20857 68 : ix86_expand_lfloorceil (rtx op0, rtx op1, bool do_floor)
20858 : {
20859 : /* C code for the stuff we're doing below (for do_floor):
20860 : xi = (long)op1;
20861 : xi -= (double)xi > op1 ? 1 : 0;
20862 : return xi;
20863 : */
20864 68 : machine_mode fmode = GET_MODE (op1);
20865 68 : machine_mode imode = GET_MODE (op0);
20866 68 : rtx ireg, freg, tmp;
20867 68 : rtx_code_label *label;
20868 :
20869 : /* reg = (long)op1 */
20870 68 : ireg = gen_reg_rtx (imode);
20871 68 : expand_fix (ireg, op1, 0);
20872 :
20873 : /* freg = (double)reg */
20874 68 : freg = gen_reg_rtx (fmode);
20875 68 : expand_float (freg, ireg, 0);
20876 :
20877 : /* ireg = (freg > op1) ? ireg - 1 : ireg */
20878 136 : label = ix86_expand_sse_compare_and_jump (UNLE,
20879 68 : freg, op1, !do_floor);
20880 102 : tmp = expand_simple_binop (imode, do_floor ? MINUS : PLUS,
20881 : ireg, const1_rtx, NULL_RTX, 0, OPTAB_DIRECT);
20882 68 : emit_move_insn (ireg, tmp);
20883 :
20884 68 : emit_label (label);
20885 68 : LABEL_NUSES (label) = 1;
20886 :
20887 68 : emit_move_insn (op0, ireg);
20888 68 : }
20889 :
20890 : /* Generate and return a rtx of mode MODE for 2**n where n is the number
20891 : of bits of the mantissa of MODE, which must be one of DFmode or SFmode. */
20892 :
20893 : static rtx
20894 994 : ix86_gen_TWO52 (machine_mode mode)
20895 : {
20896 994 : const struct real_format *fmt;
20897 994 : REAL_VALUE_TYPE TWO52r;
20898 994 : rtx TWO52;
20899 :
20900 994 : fmt = REAL_MODE_FORMAT (mode);
20901 994 : real_2expN (&TWO52r, fmt->p - 1, mode);
20902 994 : TWO52 = const_double_from_real_value (TWO52r, mode);
20903 994 : TWO52 = force_reg (mode, TWO52);
20904 :
20905 994 : return TWO52;
20906 : }
20907 :
20908 : /* Expand rint rounding OPERAND1 and storing the result in OPERAND0. */
20909 :
20910 : void
20911 121 : ix86_expand_rint (rtx operand0, rtx operand1)
20912 : {
20913 : /* C code for the stuff we're doing below:
20914 : xa = fabs (operand1);
20915 : if (!isless (xa, 2**52))
20916 : return operand1;
20917 : two52 = 2**52;
20918 : if (flag_rounding_math)
20919 : {
20920 : two52 = copysign (two52, operand1);
20921 : xa = operand1;
20922 : }
20923 : xa = xa + two52 - two52;
20924 : return copysign (xa, operand1);
20925 : */
20926 121 : machine_mode mode = GET_MODE (operand0);
20927 121 : rtx res, xa, TWO52, mask;
20928 121 : rtx_code_label *label;
20929 :
20930 121 : TWO52 = ix86_gen_TWO52 (mode);
20931 :
20932 : /* Temporary for holding the result, initialized to the input
20933 : operand to ease control flow. */
20934 121 : res = copy_to_reg (operand1);
20935 :
20936 : /* xa = abs (operand1) */
20937 121 : xa = ix86_expand_sse_fabs (res, &mask);
20938 :
20939 : /* if (!isless (xa, TWO52)) goto label; */
20940 121 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
20941 :
20942 121 : if (flag_rounding_math)
20943 : {
20944 53 : ix86_sse_copysign_to_positive (TWO52, TWO52, res, mask);
20945 53 : xa = res;
20946 : }
20947 :
20948 121 : xa = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
20949 121 : xa = expand_simple_binop (mode, MINUS, xa, TWO52, xa, 0, OPTAB_DIRECT);
20950 :
20951 : /* Remove the sign with FE_DOWNWARD, where x - x = -0.0. */
20952 121 : if (HONOR_SIGNED_ZEROS (mode) && flag_rounding_math)
20953 53 : xa = ix86_expand_sse_fabs (xa, NULL);
20954 :
20955 121 : ix86_sse_copysign_to_positive (res, xa, res, mask);
20956 :
20957 121 : emit_label (label);
20958 121 : LABEL_NUSES (label) = 1;
20959 :
20960 121 : emit_move_insn (operand0, res);
20961 121 : }
20962 :
20963 : /* Expand SSE2 sequence for computing floor or ceil
20964 : from OPERAND1 storing into OPERAND0. */
20965 : void
20966 538 : ix86_expand_floorceil (rtx operand0, rtx operand1, bool do_floor)
20967 : {
20968 : /* C code for the stuff we expand below.
20969 : double xa = fabs (x), x2;
20970 : if (!isless (xa, TWO52))
20971 : return x;
20972 : x2 = (double)(long)x;
20973 :
20974 : Compensate. Floor:
20975 : if (x2 > x)
20976 : x2 -= 1;
20977 : Compensate. Ceil:
20978 : if (x2 < x)
20979 : x2 += 1;
20980 :
20981 : if (HONOR_SIGNED_ZEROS (mode))
20982 : return copysign (x2, x);
20983 : return x2;
20984 : */
20985 538 : machine_mode mode = GET_MODE (operand0);
20986 538 : rtx xa, xi, TWO52, tmp, one, res, mask;
20987 538 : rtx_code_label *label;
20988 :
20989 538 : TWO52 = ix86_gen_TWO52 (mode);
20990 :
20991 : /* Temporary for holding the result, initialized to the input
20992 : operand to ease control flow. */
20993 538 : res = copy_to_reg (operand1);
20994 :
20995 : /* xa = abs (operand1) */
20996 538 : xa = ix86_expand_sse_fabs (res, &mask);
20997 :
20998 : /* if (!isless (xa, TWO52)) goto label; */
20999 538 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
21000 :
21001 : /* xa = (double)(long)x */
21002 538 : xi = gen_reg_rtx (int_mode_for_mode (mode).require ());
21003 538 : expand_fix (xi, res, 0);
21004 538 : expand_float (xa, xi, 0);
21005 :
21006 : /* generate 1.0 */
21007 538 : one = force_reg (mode, const_double_from_real_value (dconst1, mode));
21008 :
21009 : /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
21010 538 : tmp = ix86_expand_sse_compare_mask (UNGT, xa, res, !do_floor);
21011 538 : emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, one, tmp)));
21012 899 : tmp = expand_simple_binop (mode, do_floor ? MINUS : PLUS,
21013 : xa, tmp, NULL_RTX, 0, OPTAB_DIRECT);
21014 538 : if (HONOR_SIGNED_ZEROS (mode))
21015 : {
21016 : /* Remove the sign with FE_DOWNWARD, where x - x = -0.0. */
21017 491 : if (do_floor && flag_rounding_math)
21018 0 : tmp = ix86_expand_sse_fabs (tmp, NULL);
21019 :
21020 491 : ix86_sse_copysign_to_positive (tmp, tmp, res, mask);
21021 : }
21022 538 : emit_move_insn (res, tmp);
21023 :
21024 538 : emit_label (label);
21025 538 : LABEL_NUSES (label) = 1;
21026 :
21027 538 : emit_move_insn (operand0, res);
21028 538 : }
21029 :
21030 : /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
21031 : into OPERAND0 without relying on DImode truncation via cvttsd2siq
21032 : that is only available on 64bit targets. */
21033 : void
21034 0 : ix86_expand_floorceildf_32 (rtx operand0, rtx operand1, bool do_floor)
21035 : {
21036 : /* C code for the stuff we expand below.
21037 : double xa = fabs (x), x2;
21038 : if (!isless (xa, TWO52))
21039 : return x;
21040 : xa = xa + TWO52 - TWO52;
21041 : x2 = copysign (xa, x);
21042 :
21043 : Compensate. Floor:
21044 : if (x2 > x)
21045 : x2 -= 1;
21046 : Compensate. Ceil:
21047 : if (x2 < x)
21048 : x2 += 1;
21049 :
21050 : if (HONOR_SIGNED_ZEROS (mode))
21051 : x2 = copysign (x2, x);
21052 : return x2;
21053 : */
21054 0 : machine_mode mode = GET_MODE (operand0);
21055 0 : rtx xa, TWO52, tmp, one, res, mask;
21056 0 : rtx_code_label *label;
21057 :
21058 0 : TWO52 = ix86_gen_TWO52 (mode);
21059 :
21060 : /* Temporary for holding the result, initialized to the input
21061 : operand to ease control flow. */
21062 0 : res = copy_to_reg (operand1);
21063 :
21064 : /* xa = abs (operand1) */
21065 0 : xa = ix86_expand_sse_fabs (res, &mask);
21066 :
21067 : /* if (!isless (xa, TWO52)) goto label; */
21068 0 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
21069 :
21070 : /* xa = xa + TWO52 - TWO52; */
21071 0 : xa = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
21072 0 : xa = expand_simple_binop (mode, MINUS, xa, TWO52, xa, 0, OPTAB_DIRECT);
21073 :
21074 : /* xa = copysign (xa, operand1) */
21075 0 : ix86_sse_copysign_to_positive (xa, xa, res, mask);
21076 :
21077 : /* generate 1.0 */
21078 0 : one = force_reg (mode, const_double_from_real_value (dconst1, mode));
21079 :
21080 : /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
21081 0 : tmp = ix86_expand_sse_compare_mask (UNGT, xa, res, !do_floor);
21082 0 : emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, one, tmp)));
21083 0 : tmp = expand_simple_binop (mode, do_floor ? MINUS : PLUS,
21084 : xa, tmp, NULL_RTX, 0, OPTAB_DIRECT);
21085 0 : if (HONOR_SIGNED_ZEROS (mode))
21086 : {
21087 : /* Remove the sign with FE_DOWNWARD, where x - x = -0.0. */
21088 0 : if (do_floor && flag_rounding_math)
21089 0 : tmp = ix86_expand_sse_fabs (tmp, NULL);
21090 :
21091 0 : ix86_sse_copysign_to_positive (tmp, tmp, res, mask);
21092 : }
21093 0 : emit_move_insn (res, tmp);
21094 :
21095 0 : emit_label (label);
21096 0 : LABEL_NUSES (label) = 1;
21097 :
21098 0 : emit_move_insn (operand0, res);
21099 0 : }
21100 :
21101 : /* Expand SSE sequence for computing trunc
21102 : from OPERAND1 storing into OPERAND0. */
21103 : void
21104 321 : ix86_expand_trunc (rtx operand0, rtx operand1)
21105 : {
21106 : /* C code for SSE variant we expand below.
21107 : double xa = fabs (x), x2;
21108 : if (!isless (xa, TWO52))
21109 : return x;
21110 : x2 = (double)(long)x;
21111 : if (HONOR_SIGNED_ZEROS (mode))
21112 : return copysign (x2, x);
21113 : return x2;
21114 : */
21115 321 : machine_mode mode = GET_MODE (operand0);
21116 321 : rtx xa, xi, TWO52, res, mask;
21117 321 : rtx_code_label *label;
21118 :
21119 321 : TWO52 = ix86_gen_TWO52 (mode);
21120 :
21121 : /* Temporary for holding the result, initialized to the input
21122 : operand to ease control flow. */
21123 321 : res = copy_to_reg (operand1);
21124 :
21125 : /* xa = abs (operand1) */
21126 321 : xa = ix86_expand_sse_fabs (res, &mask);
21127 :
21128 : /* if (!isless (xa, TWO52)) goto label; */
21129 321 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
21130 :
21131 : /* xa = (double)(long)x */
21132 321 : xi = gen_reg_rtx (int_mode_for_mode (mode).require ());
21133 321 : expand_fix (xi, res, 0);
21134 321 : expand_float (xa, xi, 0);
21135 :
21136 321 : if (HONOR_SIGNED_ZEROS (mode))
21137 307 : ix86_sse_copysign_to_positive (xa, xa, res, mask);
21138 :
21139 321 : emit_move_insn (res, xa);
21140 :
21141 321 : emit_label (label);
21142 321 : LABEL_NUSES (label) = 1;
21143 :
21144 321 : emit_move_insn (operand0, res);
21145 321 : }
21146 :
21147 : /* Expand SSE sequence for computing trunc from OPERAND1 storing
21148 : into OPERAND0 without relying on DImode truncation via cvttsd2siq
21149 : that is only available on 64bit targets. */
21150 : void
21151 0 : ix86_expand_truncdf_32 (rtx operand0, rtx operand1)
21152 : {
21153 0 : machine_mode mode = GET_MODE (operand0);
21154 0 : rtx xa, xa2, TWO52, tmp, one, res, mask;
21155 0 : rtx_code_label *label;
21156 :
21157 : /* C code for SSE variant we expand below.
21158 : double xa = fabs (x), x2;
21159 : if (!isless (xa, TWO52))
21160 : return x;
21161 : xa2 = xa + TWO52 - TWO52;
21162 : Compensate:
21163 : if (xa2 > xa)
21164 : xa2 -= 1.0;
21165 : x2 = copysign (xa2, x);
21166 : return x2;
21167 : */
21168 :
21169 0 : TWO52 = ix86_gen_TWO52 (mode);
21170 :
21171 : /* Temporary for holding the result, initialized to the input
21172 : operand to ease control flow. */
21173 0 : res =copy_to_reg (operand1);
21174 :
21175 : /* xa = abs (operand1) */
21176 0 : xa = ix86_expand_sse_fabs (res, &mask);
21177 :
21178 : /* if (!isless (xa, TWO52)) goto label; */
21179 0 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
21180 :
21181 : /* xa2 = xa + TWO52 - TWO52; */
21182 0 : xa2 = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
21183 0 : xa2 = expand_simple_binop (mode, MINUS, xa2, TWO52, xa2, 0, OPTAB_DIRECT);
21184 :
21185 : /* generate 1.0 */
21186 0 : one = force_reg (mode, const_double_from_real_value (dconst1, mode));
21187 :
21188 : /* Compensate: xa2 = xa2 - (xa2 > xa ? 1 : 0) */
21189 0 : tmp = ix86_expand_sse_compare_mask (UNGT, xa2, xa, false);
21190 0 : emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, one, tmp)));
21191 0 : tmp = expand_simple_binop (mode, MINUS,
21192 : xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
21193 : /* Remove the sign with FE_DOWNWARD, where x - x = -0.0. */
21194 0 : if (HONOR_SIGNED_ZEROS (mode) && flag_rounding_math)
21195 0 : tmp = ix86_expand_sse_fabs (tmp, NULL);
21196 :
21197 : /* res = copysign (xa2, operand1) */
21198 0 : ix86_sse_copysign_to_positive (res, tmp, res, mask);
21199 :
21200 0 : emit_label (label);
21201 0 : LABEL_NUSES (label) = 1;
21202 :
21203 0 : emit_move_insn (operand0, res);
21204 0 : }
21205 :
21206 : /* Expand SSE sequence for computing round
21207 : from OPERAND1 storing into OPERAND0. */
21208 : void
21209 14 : ix86_expand_round (rtx operand0, rtx operand1)
21210 : {
21211 : /* C code for the stuff we're doing below:
21212 : double xa = fabs (x);
21213 : if (!isless (xa, TWO52))
21214 : return x;
21215 : xa = (double)(long)(xa + nextafter (0.5, 0.0));
21216 : return copysign (xa, x);
21217 : */
21218 14 : machine_mode mode = GET_MODE (operand0);
21219 14 : rtx res, TWO52, xa, xi, half, mask;
21220 14 : rtx_code_label *label;
21221 14 : const struct real_format *fmt;
21222 14 : REAL_VALUE_TYPE pred_half, half_minus_pred_half;
21223 :
21224 : /* Temporary for holding the result, initialized to the input
21225 : operand to ease control flow. */
21226 14 : res = copy_to_reg (operand1);
21227 :
21228 14 : TWO52 = ix86_gen_TWO52 (mode);
21229 14 : xa = ix86_expand_sse_fabs (res, &mask);
21230 14 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
21231 :
21232 : /* load nextafter (0.5, 0.0) */
21233 14 : fmt = REAL_MODE_FORMAT (mode);
21234 14 : real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
21235 14 : real_arithmetic (&pred_half, MINUS_EXPR, &dconsthalf, &half_minus_pred_half);
21236 :
21237 : /* xa = xa + 0.5 */
21238 14 : half = force_reg (mode, const_double_from_real_value (pred_half, mode));
21239 14 : xa = expand_simple_binop (mode, PLUS, xa, half, NULL_RTX, 0, OPTAB_DIRECT);
21240 :
21241 : /* xa = (double)(int64_t)xa */
21242 14 : xi = gen_reg_rtx (int_mode_for_mode (mode).require ());
21243 14 : expand_fix (xi, xa, 0);
21244 14 : expand_float (xa, xi, 0);
21245 :
21246 : /* res = copysign (xa, operand1) */
21247 14 : ix86_sse_copysign_to_positive (res, xa, res, mask);
21248 :
21249 14 : emit_label (label);
21250 14 : LABEL_NUSES (label) = 1;
21251 :
21252 14 : emit_move_insn (operand0, res);
21253 14 : }
21254 :
21255 : /* Expand SSE sequence for computing round from OPERAND1 storing
21256 : into OPERAND0 without relying on DImode truncation via cvttsd2siq
21257 : that is only available on 64bit targets. */
21258 : void
21259 0 : ix86_expand_rounddf_32 (rtx operand0, rtx operand1)
21260 : {
21261 : /* C code for the stuff we expand below.
21262 : double xa = fabs (x), xa2, x2;
21263 : if (!isless (xa, TWO52))
21264 : return x;
21265 : Using the absolute value and copying back sign makes
21266 : -0.0 -> -0.0 correct.
21267 : xa2 = xa + TWO52 - TWO52;
21268 : Compensate.
21269 : dxa = xa2 - xa;
21270 : if (dxa <= -0.5)
21271 : xa2 += 1;
21272 : else if (dxa > 0.5)
21273 : xa2 -= 1;
21274 : x2 = copysign (xa2, x);
21275 : return x2;
21276 : */
21277 0 : machine_mode mode = GET_MODE (operand0);
21278 0 : rtx xa, xa2, dxa, TWO52, tmp, half, mhalf, one, res, mask;
21279 0 : rtx_code_label *label;
21280 :
21281 0 : TWO52 = ix86_gen_TWO52 (mode);
21282 :
21283 : /* Temporary for holding the result, initialized to the input
21284 : operand to ease control flow. */
21285 0 : res = copy_to_reg (operand1);
21286 :
21287 : /* xa = abs (operand1) */
21288 0 : xa = ix86_expand_sse_fabs (res, &mask);
21289 :
21290 : /* if (!isless (xa, TWO52)) goto label; */
21291 0 : label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
21292 :
21293 : /* xa2 = xa + TWO52 - TWO52; */
21294 0 : xa2 = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
21295 0 : xa2 = expand_simple_binop (mode, MINUS, xa2, TWO52, xa2, 0, OPTAB_DIRECT);
21296 :
21297 : /* dxa = xa2 - xa; */
21298 0 : dxa = expand_simple_binop (mode, MINUS, xa2, xa, NULL_RTX, 0, OPTAB_DIRECT);
21299 :
21300 : /* generate 0.5, 1.0 and -0.5 */
21301 0 : half = force_reg (mode, const_double_from_real_value (dconsthalf, mode));
21302 0 : one = expand_simple_binop (mode, PLUS, half, half, NULL_RTX, 0, OPTAB_DIRECT);
21303 0 : mhalf = expand_simple_binop (mode, MINUS, half, one, NULL_RTX,
21304 : 0, OPTAB_DIRECT);
21305 :
21306 : /* Compensate. */
21307 : /* xa2 = xa2 - (dxa > 0.5 ? 1 : 0) */
21308 0 : tmp = ix86_expand_sse_compare_mask (UNGT, dxa, half, false);
21309 0 : emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, tmp, one)));
21310 0 : xa2 = expand_simple_binop (mode, MINUS, xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
21311 : /* xa2 = xa2 + (dxa <= -0.5 ? 1 : 0) */
21312 0 : tmp = ix86_expand_sse_compare_mask (UNGE, mhalf, dxa, false);
21313 0 : emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, tmp, one)));
21314 0 : xa2 = expand_simple_binop (mode, PLUS, xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
21315 :
21316 : /* res = copysign (xa2, operand1) */
21317 0 : ix86_sse_copysign_to_positive (res, xa2, res, mask);
21318 :
21319 0 : emit_label (label);
21320 0 : LABEL_NUSES (label) = 1;
21321 :
21322 0 : emit_move_insn (operand0, res);
21323 0 : }
21324 :
21325 : /* Expand SSE sequence for computing round
21326 : from OP1 storing into OP0 using sse4 round insn. */
21327 : void
21328 9 : ix86_expand_round_sse4 (rtx op0, rtx op1)
21329 : {
21330 9 : machine_mode mode = GET_MODE (op0);
21331 9 : rtx e1, e2, res, half;
21332 9 : const struct real_format *fmt;
21333 9 : REAL_VALUE_TYPE pred_half, half_minus_pred_half;
21334 9 : rtx (*gen_copysign) (rtx, rtx, rtx);
21335 9 : rtx (*gen_round) (rtx, rtx, rtx);
21336 :
21337 9 : switch (mode)
21338 : {
21339 : case E_HFmode:
21340 : gen_copysign = gen_copysignhf3;
21341 : gen_round = gen_sse4_1_roundhf2;
21342 : break;
21343 4 : case E_SFmode:
21344 4 : gen_copysign = gen_copysignsf3;
21345 4 : gen_round = gen_sse4_1_roundsf2;
21346 4 : break;
21347 4 : case E_DFmode:
21348 4 : gen_copysign = gen_copysigndf3;
21349 4 : gen_round = gen_sse4_1_rounddf2;
21350 4 : break;
21351 0 : default:
21352 0 : gcc_unreachable ();
21353 : }
21354 :
21355 : /* round (a) = trunc (a + copysign (0.5, a)) */
21356 :
21357 : /* load nextafter (0.5, 0.0) */
21358 9 : fmt = REAL_MODE_FORMAT (mode);
21359 9 : real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
21360 9 : real_arithmetic (&pred_half, MINUS_EXPR, &dconsthalf, &half_minus_pred_half);
21361 9 : half = const_double_from_real_value (pred_half, mode);
21362 :
21363 : /* e1 = copysign (0.5, op1) */
21364 9 : e1 = gen_reg_rtx (mode);
21365 9 : emit_insn (gen_copysign (e1, half, op1));
21366 :
21367 : /* e2 = op1 + e1 */
21368 9 : e2 = expand_simple_binop (mode, PLUS, op1, e1, NULL_RTX, 0, OPTAB_DIRECT);
21369 :
21370 : /* res = trunc (e2) */
21371 9 : res = gen_reg_rtx (mode);
21372 9 : emit_insn (gen_round (res, e2, GEN_INT (ROUND_TRUNC)));
21373 :
21374 9 : emit_move_insn (op0, res);
21375 9 : }
21376 :
21377 : /* A cached (set (nil) (vselect (vconcat (nil) (nil)) (parallel [])))
21378 : insn, so that expand_vselect{,_vconcat} doesn't have to create a fresh
21379 : insn every time. */
21380 :
21381 : static GTY(()) rtx_insn *vselect_insn;
21382 :
21383 : /* Initialize vselect_insn. */
21384 :
21385 : static void
21386 7698 : init_vselect_insn (void)
21387 : {
21388 7698 : unsigned i;
21389 7698 : rtx x;
21390 :
21391 7698 : x = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (MAX_VECT_LEN));
21392 500370 : for (i = 0; i < MAX_VECT_LEN; ++i)
21393 492672 : XVECEXP (x, 0, i) = const0_rtx;
21394 7698 : x = gen_rtx_VEC_SELECT (V2DFmode, gen_rtx_VEC_CONCAT (V4DFmode, const0_rtx,
21395 : const0_rtx), x);
21396 7698 : x = gen_rtx_SET (const0_rtx, x);
21397 7698 : start_sequence ();
21398 7698 : vselect_insn = emit_insn (x);
21399 7698 : end_sequence ();
21400 7698 : }
21401 :
21402 : /* Construct (set target (vec_select op0 (parallel perm))) and
21403 : return true if that's a valid instruction in the active ISA. */
21404 :
21405 : static bool
21406 541847 : expand_vselect (rtx target, rtx op0, const unsigned char *perm,
21407 : unsigned nelt, bool testing_p)
21408 : {
21409 541847 : unsigned int i;
21410 541847 : rtx x, save_vconcat;
21411 541847 : int icode;
21412 :
21413 541847 : if (vselect_insn == NULL_RTX)
21414 1701 : init_vselect_insn ();
21415 :
21416 541847 : x = XEXP (SET_SRC (PATTERN (vselect_insn)), 1);
21417 541847 : PUT_NUM_ELEM (XVEC (x, 0), nelt);
21418 4157737 : for (i = 0; i < nelt; ++i)
21419 3615890 : XVECEXP (x, 0, i) = GEN_INT (perm[i]);
21420 541847 : save_vconcat = XEXP (SET_SRC (PATTERN (vselect_insn)), 0);
21421 541847 : XEXP (SET_SRC (PATTERN (vselect_insn)), 0) = op0;
21422 541847 : PUT_MODE (SET_SRC (PATTERN (vselect_insn)), GET_MODE (target));
21423 541847 : SET_DEST (PATTERN (vselect_insn)) = target;
21424 541847 : icode = recog_memoized (vselect_insn);
21425 :
21426 541847 : if (icode >= 0 && !testing_p)
21427 72941 : emit_insn (copy_rtx (PATTERN (vselect_insn)));
21428 :
21429 541847 : SET_DEST (PATTERN (vselect_insn)) = const0_rtx;
21430 541847 : XEXP (SET_SRC (PATTERN (vselect_insn)), 0) = save_vconcat;
21431 541847 : INSN_CODE (vselect_insn) = -1;
21432 :
21433 541847 : return icode >= 0;
21434 : }
21435 :
21436 : /* Similar, but generate a vec_concat from op0 and op1 as well. */
21437 :
21438 : static bool
21439 476373 : expand_vselect_vconcat (rtx target, rtx op0, rtx op1,
21440 : const unsigned char *perm, unsigned nelt,
21441 : bool testing_p)
21442 : {
21443 476373 : machine_mode v2mode;
21444 476373 : rtx x;
21445 476373 : bool ok;
21446 :
21447 476373 : if (vselect_insn == NULL_RTX)
21448 5997 : init_vselect_insn ();
21449 :
21450 476373 : if (!GET_MODE_2XWIDER_MODE (GET_MODE (op0)).exists (&v2mode))
21451 : return false;
21452 476373 : x = XEXP (SET_SRC (PATTERN (vselect_insn)), 0);
21453 476373 : PUT_MODE (x, v2mode);
21454 476373 : XEXP (x, 0) = op0;
21455 476373 : XEXP (x, 1) = op1;
21456 476373 : ok = expand_vselect (target, x, perm, nelt, testing_p);
21457 476373 : XEXP (x, 0) = const0_rtx;
21458 476373 : XEXP (x, 1) = const0_rtx;
21459 476373 : return ok;
21460 : }
21461 :
21462 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
21463 : using movss or movsd. */
21464 : static bool
21465 342494 : expand_vec_perm_movs (struct expand_vec_perm_d *d)
21466 : {
21467 342494 : machine_mode vmode = d->vmode;
21468 342494 : unsigned i, nelt = d->nelt;
21469 342494 : rtx x;
21470 :
21471 342494 : if (d->one_operand_p)
21472 : return false;
21473 :
21474 316772 : if (!(TARGET_SSE && (vmode == V4SFmode || vmode == V4SImode))
21475 166504 : && !(TARGET_MMX_WITH_SSE && (vmode == V2SFmode || vmode == V2SImode))
21476 86057 : && !(TARGET_SSE2 && (vmode == V2DFmode || vmode == V2DImode)))
21477 : return false;
21478 :
21479 : /* Only the first element is changed. */
21480 239942 : if (d->perm[0] != nelt && d->perm[0] != 0)
21481 : return false;
21482 203028 : for (i = 1; i < nelt; ++i)
21483 148055 : if (d->perm[i] != i + nelt - d->perm[0])
21484 : return false;
21485 :
21486 54973 : if (d->testing_p)
21487 : return true;
21488 :
21489 6668 : if (d->perm[0] == nelt)
21490 0 : x = gen_rtx_VEC_MERGE (vmode, d->op1, d->op0, GEN_INT (1));
21491 : else
21492 6668 : x = gen_rtx_VEC_MERGE (vmode, d->op0, d->op1, GEN_INT (1));
21493 :
21494 6668 : emit_insn (gen_rtx_SET (d->target, x));
21495 :
21496 6668 : return true;
21497 : }
21498 :
21499 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
21500 : using insertps. */
21501 : static bool
21502 287521 : expand_vec_perm_insertps (struct expand_vec_perm_d *d)
21503 : {
21504 287521 : machine_mode vmode = d->vmode;
21505 287521 : unsigned i, cnt_s, nelt = d->nelt;
21506 287521 : int cnt_d = -1;
21507 287521 : rtx src, dst;
21508 :
21509 287521 : if (d->one_operand_p)
21510 : return false;
21511 :
21512 261799 : if (!(TARGET_SSE4_1
21513 37901 : && (vmode == V4SFmode || vmode == V4SImode
21514 27663 : || (TARGET_MMX_WITH_SSE
21515 21283 : && (vmode == V2SFmode || vmode == V2SImode)))))
21516 : return false;
21517 :
21518 56282 : for (i = 0; i < nelt; ++i)
21519 : {
21520 53031 : if (d->perm[i] == i)
21521 10744 : continue;
21522 42287 : if (cnt_d != -1)
21523 : {
21524 : cnt_d = -1;
21525 : break;
21526 : }
21527 22769 : cnt_d = i;
21528 : }
21529 :
21530 22769 : if (cnt_d == -1)
21531 : {
21532 44098 : for (i = 0; i < nelt; ++i)
21533 : {
21534 41279 : if (d->perm[i] == i + nelt)
21535 5062 : continue;
21536 36217 : if (cnt_d != -1)
21537 : return false;
21538 19518 : cnt_d = i;
21539 : }
21540 :
21541 2819 : if (cnt_d == -1)
21542 : return false;
21543 : }
21544 :
21545 6070 : if (d->testing_p)
21546 : return true;
21547 :
21548 544 : gcc_assert (cnt_d != -1);
21549 :
21550 544 : cnt_s = d->perm[cnt_d];
21551 544 : if (cnt_s < nelt)
21552 : {
21553 233 : src = d->op0;
21554 233 : dst = d->op1;
21555 : }
21556 : else
21557 : {
21558 311 : cnt_s -= nelt;
21559 311 : src = d->op1;
21560 311 : dst = d->op0;
21561 : }
21562 544 : gcc_assert (cnt_s < nelt);
21563 :
21564 544 : rtx x = gen_sse4_1_insertps (vmode, d->target, dst, src,
21565 544 : GEN_INT (cnt_s << 6 | cnt_d << 4));
21566 544 : emit_insn (x);
21567 :
21568 544 : return true;
21569 : }
21570 :
21571 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
21572 : in terms of blendp[sd] / pblendw / pblendvb / vpblendd. */
21573 :
21574 : static bool
21575 347126 : expand_vec_perm_blend (struct expand_vec_perm_d *d)
21576 : {
21577 347126 : machine_mode mmode, vmode = d->vmode;
21578 347126 : unsigned i, nelt = d->nelt;
21579 347126 : unsigned HOST_WIDE_INT mask;
21580 347126 : rtx target, op0, op1, maskop, x;
21581 347126 : rtx rperm[32], vperm;
21582 :
21583 347126 : if (d->one_operand_p)
21584 : return false;
21585 6628 : if (TARGET_AVX512F && GET_MODE_SIZE (vmode) == 64
21586 322424 : && (TARGET_AVX512BW
21587 655 : || GET_MODE_UNIT_SIZE (vmode) >= 4))
21588 : ;
21589 333329 : else if (TARGET_AVX2 && GET_MODE_SIZE (vmode) == 32)
21590 : ;
21591 314914 : else if (TARGET_AVX && (vmode == V4DFmode || vmode == V8SFmode))
21592 : ;
21593 309699 : else if (TARGET_SSE4_1
21594 345787 : && (GET_MODE_SIZE (vmode) == 16
21595 29508 : || (TARGET_MMX_WITH_SSE && GET_MODE_SIZE (vmode) == 8)
21596 3636 : || GET_MODE_SIZE (vmode) == 4))
21597 : ;
21598 : else
21599 : return false;
21600 :
21601 : /* This is a blend, not a permute. Elements must stay in their
21602 : respective lanes. */
21603 97619 : for (i = 0; i < nelt; ++i)
21604 : {
21605 92987 : unsigned e = d->perm[i];
21606 92987 : if (!(e == i || e == i + nelt))
21607 : return false;
21608 : }
21609 :
21610 4632 : if (d->testing_p)
21611 : return true;
21612 :
21613 : /* ??? Without SSE4.1, we could implement this with and/andn/or. This
21614 : decision should be extracted elsewhere, so that we only try that
21615 : sequence once all budget==3 options have been tried. */
21616 2773 : target = d->target;
21617 2773 : op0 = d->op0;
21618 2773 : op1 = d->op1;
21619 2773 : mask = 0;
21620 :
21621 2773 : switch (vmode)
21622 : {
21623 : case E_V8DFmode:
21624 : case E_V16SFmode:
21625 : case E_V4DFmode:
21626 : case E_V8SFmode:
21627 : case E_V2DFmode:
21628 : case E_V4SFmode:
21629 : case E_V2SFmode:
21630 : case E_V2HImode:
21631 : case E_V4HImode:
21632 : case E_V8HImode:
21633 : case E_V8SImode:
21634 : case E_V32HImode:
21635 : case E_V64QImode:
21636 : case E_V16SImode:
21637 : case E_V8DImode:
21638 10860 : for (i = 0; i < nelt; ++i)
21639 9378 : mask |= ((unsigned HOST_WIDE_INT) (d->perm[i] >= nelt)) << i;
21640 : break;
21641 :
21642 : case E_V2DImode:
21643 18 : for (i = 0; i < 2; ++i)
21644 18 : mask |= (d->perm[i] >= 2 ? 15 : 0) << (i * 4);
21645 6 : vmode = V8HImode;
21646 6 : goto do_subreg;
21647 :
21648 : case E_V2SImode:
21649 24 : for (i = 0; i < 2; ++i)
21650 24 : mask |= (d->perm[i] >= 2 ? 3 : 0) << (i * 2);
21651 8 : vmode = V4HImode;
21652 8 : goto do_subreg;
21653 :
21654 872 : case E_V4SImode:
21655 872 : if (TARGET_AVX2)
21656 : {
21657 : /* Use vpblendd instead of vpblendw. */
21658 190 : for (i = 0; i < nelt; ++i)
21659 152 : mask |= ((unsigned HOST_WIDE_INT) (d->perm[i] >= nelt)) << i;
21660 : break;
21661 : }
21662 : else
21663 : {
21664 4170 : for (i = 0; i < 4; ++i)
21665 5200 : mask |= (d->perm[i] >= 4 ? 3 : 0) << (i * 2);
21666 834 : vmode = V8HImode;
21667 834 : goto do_subreg;
21668 : }
21669 :
21670 : case E_V16QImode:
21671 : /* See if bytes move in pairs so we can use pblendw with
21672 : an immediate argument, rather than pblendvb with a vector
21673 : argument. */
21674 102 : for (i = 0; i < 16; i += 2)
21675 100 : if (d->perm[i] + 1 != d->perm[i + 1])
21676 : {
21677 83 : use_pblendvb:
21678 3502 : for (i = 0; i < nelt; ++i)
21679 3212 : rperm[i] = (d->perm[i] < nelt ? const0_rtx : constm1_rtx);
21680 :
21681 290 : finish_pblendvb:
21682 291 : vperm = gen_rtx_CONST_VECTOR (vmode, gen_rtvec_v (nelt, rperm));
21683 291 : vperm = force_reg (vmode, vperm);
21684 :
21685 582 : if (GET_MODE_SIZE (vmode) == 4)
21686 135 : emit_insn (gen_mmx_pblendvb_v4qi (target, op0, op1, vperm));
21687 312 : else if (GET_MODE_SIZE (vmode) == 8)
21688 40 : emit_insn (gen_mmx_pblendvb_v8qi (target, op0, op1, vperm));
21689 232 : else if (GET_MODE_SIZE (vmode) == 16)
21690 83 : emit_insn (gen_sse4_1_pblendvb (target, op0, op1, vperm));
21691 : else
21692 33 : emit_insn (gen_avx2_pblendvb (target, op0, op1, vperm));
21693 291 : if (target != d->target)
21694 1 : emit_move_insn (d->target, gen_lowpart (d->vmode, target));
21695 291 : return true;
21696 : }
21697 :
21698 18 : for (i = 0; i < 8; ++i)
21699 16 : mask |= (d->perm[i * 2] >= 16) << i;
21700 : vmode = V8HImode;
21701 : /* FALLTHRU */
21702 :
21703 930 : do_subreg:
21704 930 : target = gen_reg_rtx (vmode);
21705 930 : op0 = gen_lowpart (vmode, op0);
21706 930 : op1 = gen_lowpart (vmode, op1);
21707 930 : break;
21708 :
21709 : case E_V8QImode:
21710 40 : for (i = 0; i < 8; i += 2)
21711 40 : if (d->perm[i] + 1 != d->perm[i + 1])
21712 40 : goto use_pblendvb;
21713 :
21714 0 : for (i = 0; i < 4; ++i)
21715 0 : mask |= (d->perm[i * 2] >= 8) << i;
21716 0 : vmode = V4HImode;
21717 0 : goto do_subreg;
21718 :
21719 : case E_V4QImode:
21720 153 : for (i = 0; i < 4; i += 2)
21721 150 : if (d->perm[i] + 1 != d->perm[i + 1])
21722 135 : goto use_pblendvb;
21723 :
21724 9 : for (i = 0; i < 2; ++i)
21725 6 : mask |= (d->perm[i * 2] >= 4) << i;
21726 3 : vmode = V2HImode;
21727 3 : goto do_subreg;
21728 :
21729 : case E_V32QImode:
21730 : /* See if bytes move in pairs. If not, vpblendvb must be used. */
21731 916 : for (i = 0; i < 32; i += 2)
21732 864 : if (d->perm[i] + 1 != d->perm[i + 1])
21733 32 : goto use_pblendvb;
21734 : /* See if bytes move in quadruplets. If yes, vpblendd
21735 : with immediate can be used. */
21736 468 : for (i = 0; i < 32; i += 4)
21737 416 : if (d->perm[i] + 2 != d->perm[i + 2])
21738 : break;
21739 52 : if (i < 32)
21740 : {
21741 : /* See if bytes move the same in both lanes. If yes,
21742 : vpblendw with immediate can be used. */
21743 0 : for (i = 0; i < 16; i += 2)
21744 0 : if (d->perm[i] + 16 != d->perm[i + 16])
21745 0 : goto use_pblendvb;
21746 :
21747 : /* Use vpblendw. */
21748 0 : for (i = 0; i < 16; ++i)
21749 0 : mask |= (d->perm[i * 2] >= 32) << i;
21750 0 : vmode = V16HImode;
21751 0 : goto do_subreg;
21752 : }
21753 :
21754 : /* Use vpblendd. */
21755 468 : for (i = 0; i < 8; ++i)
21756 416 : mask |= (d->perm[i * 4] >= 32) << i;
21757 52 : vmode = V8SImode;
21758 52 : goto do_subreg;
21759 :
21760 : case E_V16HImode:
21761 : /* See if words move in pairs. If yes, vpblendd can be used. */
21762 186 : for (i = 0; i < 16; i += 2)
21763 169 : if (d->perm[i] + 1 != d->perm[i + 1])
21764 : break;
21765 50 : if (i < 16)
21766 : {
21767 : /* See if words move the same in both lanes. If not,
21768 : vpblendvb must be used. */
21769 290 : for (i = 0; i < 8; i++)
21770 258 : if (d->perm[i] + 8 != d->perm[i + 8])
21771 : {
21772 : /* Use vpblendvb. */
21773 33 : for (i = 0; i < 32; ++i)
21774 32 : rperm[i] = (d->perm[i / 2] < 16 ? const0_rtx : constm1_rtx);
21775 :
21776 1 : vmode = V32QImode;
21777 1 : nelt = 32;
21778 1 : target = gen_reg_rtx (vmode);
21779 1 : op0 = gen_lowpart (vmode, op0);
21780 1 : op1 = gen_lowpart (vmode, op1);
21781 1 : goto finish_pblendvb;
21782 : }
21783 :
21784 : /* Use vpblendw. */
21785 544 : for (i = 0; i < 16; ++i)
21786 512 : mask |= (d->perm[i] >= 16) << i;
21787 : break;
21788 : }
21789 :
21790 : /* Use vpblendd. */
21791 153 : for (i = 0; i < 8; ++i)
21792 136 : mask |= (d->perm[i * 2] >= 16) << i;
21793 17 : vmode = V8SImode;
21794 17 : goto do_subreg;
21795 :
21796 : case E_V4DImode:
21797 : /* Use vpblendd. */
21798 40 : for (i = 0; i < 4; ++i)
21799 47 : mask |= (d->perm[i] >= 4 ? 3 : 0) << (i * 2);
21800 8 : vmode = V8SImode;
21801 8 : goto do_subreg;
21802 :
21803 0 : default:
21804 0 : gcc_unreachable ();
21805 : }
21806 :
21807 2482 : switch (vmode)
21808 : {
21809 : case E_V8DFmode:
21810 : case E_V8DImode:
21811 : mmode = QImode;
21812 : break;
21813 1 : case E_V16SFmode:
21814 1 : case E_V16SImode:
21815 1 : mmode = HImode;
21816 1 : break;
21817 6 : case E_V32HImode:
21818 6 : mmode = SImode;
21819 6 : break;
21820 1 : case E_V64QImode:
21821 1 : mmode = DImode;
21822 1 : break;
21823 : default:
21824 : mmode = VOIDmode;
21825 : }
21826 :
21827 : /* Canonicalize vec_merge. */
21828 2482 : if (swap_commutative_operands_p (op1, op0)
21829 : /* Two operands have same precedence, then
21830 : first bit of mask select first operand. */
21831 2482 : || (!swap_commutative_operands_p (op0, op1)
21832 2482 : && !(mask & 1)))
21833 : {
21834 2475 : unsigned n_elts = GET_MODE_NUNITS (vmode);
21835 2475 : std::swap (op0, op1);
21836 2475 : unsigned HOST_WIDE_INT mask_all = HOST_WIDE_INT_1U;
21837 2475 : if (n_elts == HOST_BITS_PER_WIDE_INT)
21838 : mask_all = -1;
21839 : else
21840 2474 : mask_all = (HOST_WIDE_INT_1U << n_elts) - 1;
21841 2475 : mask = ~mask & mask_all;
21842 : }
21843 :
21844 2482 : if (mmode != VOIDmode)
21845 14 : maskop = force_reg (mmode, gen_int_mode (mask, mmode));
21846 : else
21847 2468 : maskop = GEN_INT (mask);
21848 :
21849 : /* This matches five different patterns with the different modes. */
21850 2482 : x = gen_rtx_VEC_MERGE (vmode, op1, op0, maskop);
21851 2482 : x = gen_rtx_SET (target, x);
21852 2482 : emit_insn (x);
21853 2482 : if (target != d->target)
21854 930 : emit_move_insn (d->target, gen_lowpart (d->vmode, target));
21855 :
21856 : return true;
21857 : }
21858 :
21859 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
21860 : in terms of the variable form of vpermilps.
21861 :
21862 : Note that we will have already failed the immediate input vpermilps,
21863 : which requires that the high and low part shuffle be identical; the
21864 : variable form doesn't require that. */
21865 :
21866 : static bool
21867 137406 : expand_vec_perm_vpermil (struct expand_vec_perm_d *d)
21868 : {
21869 137406 : rtx rperm[8], vperm;
21870 137406 : unsigned i;
21871 :
21872 137406 : if (!TARGET_AVX || !d->one_operand_p
21873 11091 : || (d->vmode != V8SImode && d->vmode != V8SFmode))
21874 : return false;
21875 :
21876 : /* We can only permute within the 128-bit lane. */
21877 16076 : for (i = 0; i < 8; ++i)
21878 : {
21879 15440 : unsigned e = d->perm[i];
21880 15440 : if (i < 4 ? e >= 4 : e < 4)
21881 : return false;
21882 : }
21883 :
21884 636 : if (d->testing_p)
21885 : return true;
21886 :
21887 531 : for (i = 0; i < 8; ++i)
21888 : {
21889 472 : unsigned e = d->perm[i];
21890 :
21891 : /* Within each 128-bit lane, the elements of op0 are numbered
21892 : from 0 and the elements of op1 are numbered from 4. */
21893 472 : if (e >= 8 + 4)
21894 0 : e -= 8;
21895 472 : else if (e >= 4)
21896 236 : e -= 4;
21897 :
21898 472 : rperm[i] = GEN_INT (e);
21899 : }
21900 :
21901 59 : vperm = gen_rtx_CONST_VECTOR (V8SImode, gen_rtvec_v (8, rperm));
21902 59 : vperm = force_reg (V8SImode, vperm);
21903 59 : rtx target = d->target;
21904 59 : rtx op0 = d->op0;
21905 59 : if (d->vmode == V8SImode)
21906 : {
21907 7 : target = lowpart_subreg (V8SFmode, target, V8SImode);
21908 7 : op0 = lowpart_subreg (V8SFmode, op0, V8SImode);
21909 : }
21910 :
21911 59 : emit_insn (gen_avx_vpermilvarv8sf3 (target, op0, vperm));
21912 :
21913 59 : return true;
21914 : }
21915 :
21916 : /* For V*[QHS]Imode permutations, check if the same permutation
21917 : can't be performed in a 2x, 4x or 8x wider inner mode. */
21918 :
21919 : static bool
21920 161748 : canonicalize_vector_int_perm (const struct expand_vec_perm_d *d,
21921 : struct expand_vec_perm_d *nd)
21922 : {
21923 161748 : int i;
21924 161748 : machine_mode mode = VOIDmode;
21925 :
21926 161748 : switch (d->vmode)
21927 : {
21928 : case E_V8QImode: mode = V4HImode; break;
21929 30188 : case E_V16QImode: mode = V8HImode; break;
21930 715 : case E_V32QImode: mode = V16HImode; break;
21931 275 : case E_V64QImode: mode = V32HImode; break;
21932 12270 : case E_V4HImode: mode = V2SImode; break;
21933 20804 : case E_V8HImode: mode = V4SImode; break;
21934 1006 : case E_V16HImode: mode = V8SImode; break;
21935 397 : case E_V32HImode: mode = V16SImode; break;
21936 41007 : case E_V4SImode: mode = V2DImode; break;
21937 1429 : case E_V8SImode: mode = V4DImode; break;
21938 65 : case E_V16SImode: mode = V8DImode; break;
21939 : default: return false;
21940 : }
21941 206458 : for (i = 0; i < d->nelt; i += 2)
21942 191688 : if ((d->perm[i] & 1) || d->perm[i + 1] != d->perm[i] + 1)
21943 : return false;
21944 14770 : nd->vmode = mode;
21945 14770 : nd->nelt = d->nelt / 2;
21946 96188 : for (i = 0; i < nd->nelt; i++)
21947 81418 : nd->perm[i] = d->perm[2 * i] / 2;
21948 29540 : if (GET_MODE_INNER (mode) != DImode)
21949 13001 : canonicalize_vector_int_perm (nd, nd);
21950 14770 : if (nd != d)
21951 : {
21952 9420 : nd->one_operand_p = d->one_operand_p;
21953 9420 : nd->testing_p = d->testing_p;
21954 9420 : if (d->op0 == d->op1)
21955 2781 : nd->op0 = nd->op1 = gen_lowpart (nd->vmode, d->op0);
21956 : else
21957 : {
21958 6639 : nd->op0 = gen_lowpart (nd->vmode, d->op0);
21959 6639 : nd->op1 = gen_lowpart (nd->vmode, d->op1);
21960 : }
21961 9420 : if (d->testing_p)
21962 6120 : nd->target = gen_raw_REG (nd->vmode, LAST_VIRTUAL_REGISTER + 1);
21963 : else
21964 3300 : nd->target = gen_reg_rtx (nd->vmode);
21965 : }
21966 : return true;
21967 : }
21968 :
21969 : /* Return true if permutation D can be performed as VMODE permutation
21970 : instead. */
21971 :
21972 : static bool
21973 5970 : valid_perm_using_mode_p (machine_mode vmode, struct expand_vec_perm_d *d)
21974 : {
21975 5970 : unsigned int i, j, chunk;
21976 :
21977 5970 : if (GET_MODE_CLASS (vmode) != MODE_VECTOR_INT
21978 5970 : || GET_MODE_CLASS (d->vmode) != MODE_VECTOR_INT
21979 14558 : || GET_MODE_SIZE (vmode) != GET_MODE_SIZE (d->vmode))
21980 : return false;
21981 :
21982 8588 : if (GET_MODE_NUNITS (vmode) >= d->nelt)
21983 : return true;
21984 :
21985 4030 : chunk = d->nelt / GET_MODE_NUNITS (vmode);
21986 5272 : for (i = 0; i < d->nelt; i += chunk)
21987 5025 : if (d->perm[i] & (chunk - 1))
21988 : return false;
21989 : else
21990 7759 : for (j = 1; j < chunk; ++j)
21991 6517 : if (d->perm[i] + j != d->perm[i + j])
21992 : return false;
21993 :
21994 : return true;
21995 : }
21996 :
21997 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
21998 : in terms of pshufb, vpperm, vpermq, vpermd, vpermps or vperm2i128. */
21999 :
22000 : static bool
22001 136770 : expand_vec_perm_pshufb (struct expand_vec_perm_d *d)
22002 : {
22003 136770 : unsigned i, nelt, eltsz, mask;
22004 136770 : unsigned char perm[64];
22005 136770 : machine_mode vmode;
22006 136770 : struct expand_vec_perm_d nd;
22007 136770 : rtx rperm[64], vperm, target, op0, op1;
22008 :
22009 136770 : nelt = d->nelt;
22010 :
22011 136770 : if (!d->one_operand_p)
22012 223576 : switch (GET_MODE_SIZE (d->vmode))
22013 : {
22014 8327 : case 4:
22015 8327 : if (!TARGET_XOP)
22016 : return false;
22017 : vmode = V4QImode;
22018 : break;
22019 :
22020 19732 : case 8:
22021 19732 : if (!TARGET_XOP)
22022 : return false;
22023 : vmode = V8QImode;
22024 : break;
22025 :
22026 74711 : case 16:
22027 74711 : if (!TARGET_XOP)
22028 : return false;
22029 : vmode = V16QImode;
22030 : break;
22031 :
22032 8100 : case 32:
22033 8100 : if (!TARGET_AVX2)
22034 : return false;
22035 :
22036 4038 : if (valid_perm_using_mode_p (V2TImode, d))
22037 : {
22038 56 : if (d->testing_p)
22039 : return true;
22040 :
22041 : /* Use vperm2i128 insn. The pattern uses
22042 : V4DImode instead of V2TImode. */
22043 52 : target = d->target;
22044 52 : if (d->vmode != V4DImode)
22045 12 : target = gen_reg_rtx (V4DImode);
22046 52 : op0 = gen_lowpart (V4DImode, d->op0);
22047 52 : op1 = gen_lowpart (V4DImode, d->op1);
22048 52 : rperm[0]
22049 52 : = GEN_INT ((d->perm[0] / (nelt / 2))
22050 : | ((d->perm[nelt / 2] / (nelt / 2)) * 16));
22051 52 : emit_insn (gen_avx2_permv2ti (target, op0, op1, rperm[0]));
22052 52 : if (target != d->target)
22053 12 : emit_move_insn (d->target, gen_lowpart (d->vmode, target));
22054 52 : return true;
22055 : }
22056 : /* FALLTHRU */
22057 :
22058 : default:
22059 : return false;
22060 : }
22061 : else
22062 49964 : switch (GET_MODE_SIZE (d->vmode))
22063 : {
22064 3630 : case 4:
22065 3630 : if (!TARGET_SSSE3)
22066 : return false;
22067 : vmode = V4QImode;
22068 : break;
22069 :
22070 2229 : case 8:
22071 2229 : if (!TARGET_SSSE3)
22072 : return false;
22073 : vmode = V8QImode;
22074 : break;
22075 :
22076 14160 : case 16:
22077 14160 : if (!TARGET_SSSE3)
22078 : return false;
22079 : vmode = V16QImode;
22080 : break;
22081 :
22082 4564 : case 32:
22083 4564 : if (!TARGET_AVX2)
22084 : return false;
22085 :
22086 : /* V4DImode should be already handled through
22087 : expand_vselect by vpermq instruction. */
22088 1987 : gcc_assert (d->vmode != V4DImode);
22089 :
22090 1987 : vmode = V32QImode;
22091 1987 : if (d->vmode == V8SImode
22092 1622 : || d->vmode == V16HImode
22093 1406 : || d->vmode == V32QImode)
22094 : {
22095 : /* First see if vpermq can be used for
22096 : V8SImode/V16HImode/V32QImode. */
22097 879 : if (valid_perm_using_mode_p (V4DImode, d))
22098 : {
22099 770 : for (i = 0; i < 4; i++)
22100 616 : perm[i] = (d->perm[i * nelt / 4] * 4 / nelt) & 3;
22101 154 : if (d->testing_p)
22102 : return true;
22103 58 : target = gen_reg_rtx (V4DImode);
22104 58 : if (expand_vselect (target, gen_lowpart (V4DImode, d->op0),
22105 : perm, 4, false))
22106 : {
22107 116 : emit_move_insn (d->target,
22108 58 : gen_lowpart (d->vmode, target));
22109 58 : return true;
22110 : }
22111 : return false;
22112 : }
22113 :
22114 : /* Next see if vpermd can be used. */
22115 725 : if (valid_perm_using_mode_p (V8SImode, d))
22116 : vmode = V8SImode;
22117 : }
22118 : /* Or if vpermps can be used. */
22119 1108 : else if (d->vmode == V8SFmode)
22120 : vmode = V8SImode;
22121 :
22122 : if (vmode == V32QImode)
22123 : {
22124 : /* vpshufb only works intra lanes, it is not
22125 : possible to shuffle bytes in between the lanes. */
22126 6473 : for (i = 0; i < nelt; ++i)
22127 6291 : if ((d->perm[i] ^ i) & (nelt / 2))
22128 : return false;
22129 : }
22130 : break;
22131 :
22132 399 : case 64:
22133 399 : if (!TARGET_AVX512BW)
22134 : return false;
22135 :
22136 : /* If vpermq didn't work, vpshufb won't work either. */
22137 204 : if (d->vmode == V8DFmode || d->vmode == V8DImode)
22138 : return false;
22139 :
22140 175 : vmode = V64QImode;
22141 175 : if (d->vmode == V16SImode
22142 150 : || d->vmode == V32HImode
22143 50 : || d->vmode == V64QImode)
22144 : {
22145 : /* First see if vpermq can be used for
22146 : V16SImode/V32HImode/V64QImode. */
22147 164 : if (valid_perm_using_mode_p (V8DImode, d))
22148 : {
22149 0 : for (i = 0; i < 8; i++)
22150 0 : perm[i] = (d->perm[i * nelt / 8] * 8 / nelt) & 7;
22151 0 : if (d->testing_p)
22152 : return true;
22153 0 : target = gen_reg_rtx (V8DImode);
22154 0 : if (expand_vselect (target, gen_lowpart (V8DImode, d->op0),
22155 : perm, 8, false))
22156 : {
22157 0 : emit_move_insn (d->target,
22158 0 : gen_lowpart (d->vmode, target));
22159 0 : return true;
22160 : }
22161 : return false;
22162 : }
22163 :
22164 : /* Next see if vpermd can be used. */
22165 164 : if (valid_perm_using_mode_p (V16SImode, d))
22166 : vmode = V16SImode;
22167 : }
22168 : /* Or if vpermps can be used. */
22169 11 : else if (d->vmode == V16SFmode)
22170 : vmode = V16SImode;
22171 :
22172 : if (vmode == V64QImode)
22173 : {
22174 : /* vpshufb only works intra lanes, it is not
22175 : possible to shuffle bytes in between the lanes. */
22176 578 : for (i = 0; i < nelt; ++i)
22177 578 : if ((d->perm[i] ^ i) & (3 * nelt / 4))
22178 : return false;
22179 : }
22180 : break;
22181 :
22182 : default:
22183 : return false;
22184 : }
22185 :
22186 11547 : if (d->testing_p)
22187 : return true;
22188 :
22189 : /* Try to avoid variable permutation instruction. */
22190 8846 : if (canonicalize_vector_int_perm (d, &nd) && expand_vec_perm_1 (&nd))
22191 : {
22192 1839 : emit_move_insn (d->target, gen_lowpart (d->vmode, nd.target));
22193 1839 : return true;
22194 : }
22195 :
22196 7007 : if (vmode == V8SImode)
22197 9531 : for (i = 0; i < 8; ++i)
22198 8472 : rperm[i] = GEN_INT ((d->perm[i * nelt / 8] * 8 / nelt) & 7);
22199 5948 : else if (vmode == V16SImode)
22200 612 : for (i = 0; i < 16; ++i)
22201 576 : rperm[i] = GEN_INT ((d->perm[i * nelt / 16] * 16 / nelt) & 15);
22202 : else
22203 : {
22204 5912 : eltsz = GET_MODE_UNIT_SIZE (d->vmode);
22205 5912 : if (!d->one_operand_p)
22206 3212 : mask = 2 * nelt - 1;
22207 2700 : else if (vmode == V64QImode)
22208 0 : mask = nelt / 4 - 1;
22209 2700 : else if (vmode == V32QImode)
22210 176 : mask = nelt / 2 - 1;
22211 : else
22212 2524 : mask = nelt - 1;
22213 :
22214 59020 : for (i = 0; i < nelt; ++i)
22215 : {
22216 53108 : unsigned j, e = d->perm[i] & mask;
22217 148168 : for (j = 0; j < eltsz; ++j)
22218 95060 : rperm[i * eltsz + j] = GEN_INT (e * eltsz + j);
22219 : }
22220 : }
22221 :
22222 7007 : machine_mode vpmode = vmode;
22223 :
22224 7007 : nelt = GET_MODE_SIZE (vmode);
22225 :
22226 : /* Emulate narrow modes with V16QI instructions. */
22227 7007 : if (nelt < 16)
22228 : {
22229 222 : rtx m128 = GEN_INT (-128);
22230 :
22231 : /* Remap elements from the second operand, as we have to
22232 : account for inactive top elements from the first operand. */
22233 222 : if (!d->one_operand_p)
22234 : {
22235 243 : for (i = 0; i < nelt; ++i)
22236 : {
22237 216 : unsigned ival = UINTVAL (rperm[i]);
22238 216 : if (ival >= nelt)
22239 108 : rperm[i] = GEN_INT (ival + 16 - nelt);
22240 : }
22241 : }
22242 :
22243 : /* Fill inactive elements in the top positions with zeros. */
22244 2570 : for (i = nelt; i < 16; ++i)
22245 2348 : rperm[i] = m128;
22246 :
22247 : vpmode = V16QImode;
22248 : }
22249 :
22250 14014 : vperm = gen_rtx_CONST_VECTOR (vpmode,
22251 7007 : gen_rtvec_v (GET_MODE_NUNITS (vpmode), rperm));
22252 7007 : vperm = force_reg (vpmode, vperm);
22253 :
22254 7007 : if (vmode == d->vmode)
22255 2398 : target = d->target;
22256 : else
22257 4609 : target = gen_reg_rtx (vmode);
22258 :
22259 7007 : op0 = gen_lowpart (vmode, d->op0);
22260 :
22261 7007 : if (d->one_operand_p)
22262 : {
22263 3795 : rtx (*gen) (rtx, rtx, rtx);
22264 :
22265 3795 : if (vmode == V4QImode)
22266 : gen = gen_mmx_pshufbv4qi3;
22267 : else if (vmode == V8QImode)
22268 : gen = gen_mmx_pshufbv8qi3;
22269 : else if (vmode == V16QImode)
22270 : gen = gen_ssse3_pshufbv16qi3;
22271 : else if (vmode == V32QImode)
22272 : gen = gen_avx2_pshufbv32qi3;
22273 : else if (vmode == V64QImode)
22274 : gen = gen_avx512bw_pshufbv64qi3;
22275 : else if (vmode == V8SFmode)
22276 : gen = gen_avx2_permvarv8sf;
22277 : else if (vmode == V8SImode)
22278 : gen = gen_avx2_permvarv8si;
22279 : else if (vmode == V16SFmode)
22280 : gen = gen_avx512f_permvarv16sf;
22281 : else if (vmode == V16SImode)
22282 : gen = gen_avx512f_permvarv16si;
22283 : else
22284 : gcc_unreachable ();
22285 :
22286 3795 : emit_insn (gen (target, op0, vperm));
22287 : }
22288 : else
22289 : {
22290 3212 : rtx (*gen) (rtx, rtx, rtx, rtx);
22291 :
22292 3212 : op1 = gen_lowpart (vmode, d->op1);
22293 :
22294 3212 : if (vmode == V4QImode)
22295 : gen = gen_mmx_ppermv32;
22296 : else if (vmode == V8QImode)
22297 : gen = gen_mmx_ppermv64;
22298 : else if (vmode == V16QImode)
22299 : gen = gen_xop_pperm;
22300 : else
22301 0 : gcc_unreachable ();
22302 :
22303 3212 : emit_insn (gen (target, op0, op1, vperm));
22304 : }
22305 :
22306 7007 : if (target != d->target)
22307 4609 : emit_move_insn (d->target, gen_lowpart (d->vmode, target));
22308 :
22309 : return true;
22310 : }
22311 :
22312 : /* Try to expand one-operand permutation with constant mask. */
22313 :
22314 : static bool
22315 124889 : ix86_expand_vec_one_operand_perm_avx512 (struct expand_vec_perm_d *d)
22316 : {
22317 124889 : machine_mode mode = GET_MODE (d->op0);
22318 124889 : machine_mode maskmode = mode;
22319 249778 : unsigned inner_size = GET_MODE_SIZE (GET_MODE_INNER (mode));
22320 124889 : rtx (*gen) (rtx, rtx, rtx) = NULL;
22321 124889 : rtx target, op0, mask;
22322 124889 : rtx vec[64];
22323 :
22324 124889 : if (!rtx_equal_p (d->op0, d->op1))
22325 : return false;
22326 :
22327 17616 : if (!TARGET_AVX512F)
22328 : return false;
22329 :
22330 : /* Accept VNxHImode and VNxQImode now. */
22331 739 : if (!TARGET_AVX512VL && GET_MODE_SIZE (mode) < 64)
22332 : return false;
22333 :
22334 : /* vpermw. */
22335 463 : if (!TARGET_AVX512BW && inner_size == 2)
22336 : return false;
22337 :
22338 : /* vpermb. */
22339 329 : if (!TARGET_AVX512VBMI && inner_size == 1)
22340 : return false;
22341 :
22342 210 : switch (mode)
22343 : {
22344 : case E_V16SImode:
22345 : gen = gen_avx512f_permvarv16si;
22346 : break;
22347 4 : case E_V16SFmode:
22348 4 : gen = gen_avx512f_permvarv16sf;
22349 4 : maskmode = V16SImode;
22350 4 : break;
22351 1 : case E_V8DImode:
22352 1 : gen = gen_avx512f_permvarv8di;
22353 1 : break;
22354 30 : case E_V8DFmode:
22355 30 : gen = gen_avx512f_permvarv8df;
22356 30 : maskmode = V8DImode;
22357 30 : break;
22358 106 : case E_V32HImode:
22359 106 : gen = gen_avx512bw_permvarv32hi;
22360 106 : break;
22361 14 : case E_V16HImode:
22362 14 : gen = gen_avx512vl_permvarv16hi;
22363 14 : break;
22364 6 : case E_V8HImode:
22365 6 : gen = gen_avx512vl_permvarv8hi;
22366 6 : break;
22367 4 : case E_V64QImode:
22368 4 : gen = gen_avx512bw_permvarv64qi;
22369 4 : break;
22370 2 : case E_V32QImode:
22371 2 : gen = gen_avx512vl_permvarv32qi;
22372 2 : break;
22373 0 : case E_V16QImode:
22374 0 : gen = gen_avx512vl_permvarv16qi;
22375 0 : break;
22376 :
22377 : default:
22378 : return false;
22379 : }
22380 :
22381 209 : if (d->testing_p)
22382 : return true;
22383 :
22384 200 : target = d->target;
22385 200 : op0 = d->op0;
22386 5024 : for (int i = 0; i < d->nelt; ++i)
22387 4824 : vec[i] = GEN_INT (d->perm[i]);
22388 200 : mask = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (d->nelt, vec));
22389 200 : emit_insn (gen (target, op0, force_reg (maskmode, mask)));
22390 200 : return true;
22391 : }
22392 :
22393 : static bool expand_vec_perm_palignr (struct expand_vec_perm_d *d, bool);
22394 :
22395 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to instantiate D
22396 : in a single instruction. */
22397 :
22398 : static bool
22399 380993 : expand_vec_perm_1 (struct expand_vec_perm_d *d)
22400 : {
22401 380993 : unsigned i, nelt = d->nelt;
22402 380993 : struct expand_vec_perm_d nd;
22403 :
22404 : /* Check plain VEC_SELECT first, because AVX has instructions that could
22405 : match both SEL and SEL+CONCAT, but the plain SEL will allow a memory
22406 : input where SEL+CONCAT may not. */
22407 380993 : if (d->one_operand_p)
22408 : {
22409 : int mask = nelt - 1;
22410 : bool identity_perm = true;
22411 : bool broadcast_perm = true;
22412 :
22413 506969 : for (i = 0; i < nelt; i++)
22414 : {
22415 444876 : nd.perm[i] = d->perm[i] & mask;
22416 444876 : if (nd.perm[i] != i)
22417 339698 : identity_perm = false;
22418 444876 : if (nd.perm[i])
22419 366672 : broadcast_perm = false;
22420 : }
22421 :
22422 62093 : if (identity_perm)
22423 : {
22424 11 : if (!d->testing_p)
22425 5 : emit_move_insn (d->target, d->op0);
22426 11 : return true;
22427 : }
22428 62082 : else if (broadcast_perm && TARGET_AVX2)
22429 : {
22430 : /* Use vpbroadcast{b,w,d}. */
22431 381 : rtx (*gen) (rtx, rtx) = NULL;
22432 381 : switch (d->vmode)
22433 : {
22434 1 : case E_V64QImode:
22435 1 : if (TARGET_AVX512BW)
22436 : gen = gen_avx512bw_vec_dupv64qi_1;
22437 : break;
22438 4 : case E_V32QImode:
22439 4 : gen = gen_avx2_pbroadcastv32qi_1;
22440 4 : break;
22441 1 : case E_V32HImode:
22442 1 : if (TARGET_AVX512BW)
22443 : gen = gen_avx512bw_vec_dupv32hi_1;
22444 : break;
22445 4 : case E_V16HImode:
22446 4 : gen = gen_avx2_pbroadcastv16hi_1;
22447 4 : break;
22448 1 : case E_V16SImode:
22449 1 : if (TARGET_AVX512F)
22450 : gen = gen_avx512f_vec_dupv16si_1;
22451 : break;
22452 4 : case E_V8SImode:
22453 4 : gen = gen_avx2_pbroadcastv8si_1;
22454 4 : break;
22455 4 : case E_V16QImode:
22456 4 : gen = gen_avx2_pbroadcastv16qi;
22457 4 : break;
22458 5 : case E_V8HImode:
22459 5 : gen = gen_avx2_pbroadcastv8hi;
22460 5 : break;
22461 0 : case E_V16SFmode:
22462 0 : if (TARGET_AVX512F)
22463 : gen = gen_avx512f_vec_dupv16sf_1;
22464 : break;
22465 : case E_V8SFmode:
22466 : gen = gen_avx2_vec_dupv8sf_1;
22467 : break;
22468 0 : case E_V8DFmode:
22469 0 : if (TARGET_AVX512F)
22470 : gen = gen_avx512f_vec_dupv8df_1;
22471 : break;
22472 0 : case E_V8DImode:
22473 0 : if (TARGET_AVX512F)
22474 : gen = gen_avx512f_vec_dupv8di_1;
22475 : break;
22476 : /* For other modes prefer other shuffles this function creates. */
22477 : default: break;
22478 : }
22479 21 : if (gen != NULL)
22480 : {
22481 24 : if (!d->testing_p)
22482 24 : emit_insn (gen (d->target, d->op0));
22483 24 : return true;
22484 : }
22485 : }
22486 :
22487 62058 : if (expand_vselect (d->target, d->op0, nd.perm, nelt, d->testing_p))
22488 : return true;
22489 :
22490 : /* There are plenty of patterns in sse.md that are written for
22491 : SEL+CONCAT and are not replicated for a single op. Perhaps
22492 : that should be changed, to avoid the nastiness here. */
22493 :
22494 : /* Recognize interleave style patterns, which means incrementing
22495 : every other permutation operand. */
22496 200655 : for (i = 0; i < nelt; i += 2)
22497 : {
22498 164142 : nd.perm[i] = d->perm[i] & mask;
22499 164142 : nd.perm[i + 1] = (d->perm[i + 1] & mask) + nelt;
22500 : }
22501 36513 : if (expand_vselect_vconcat (d->target, d->op0, d->op0, nd.perm, nelt,
22502 36513 : d->testing_p))
22503 : return true;
22504 :
22505 : /* Recognize shufps, which means adding {0, 0, nelt, nelt}. */
22506 31489 : if (nelt >= 4)
22507 : {
22508 107492 : for (i = 0; i < nelt; i += 4)
22509 : {
22510 76003 : nd.perm[i + 0] = d->perm[i + 0] & mask;
22511 76003 : nd.perm[i + 1] = d->perm[i + 1] & mask;
22512 76003 : nd.perm[i + 2] = (d->perm[i + 2] & mask) + nelt;
22513 76003 : nd.perm[i + 3] = (d->perm[i + 3] & mask) + nelt;
22514 : }
22515 :
22516 31489 : if (expand_vselect_vconcat (d->target, d->op0, d->op0, nd.perm, nelt,
22517 31489 : d->testing_p))
22518 : return true;
22519 : }
22520 : }
22521 :
22522 : /* Try the SSE4.1 blend variable merge instructions. */
22523 344622 : if (expand_vec_perm_blend (d))
22524 : return true;
22525 :
22526 : /* Try movss/movsd instructions. */
22527 342494 : if (expand_vec_perm_movs (d))
22528 : return true;
22529 :
22530 : /* Try the SSE4.1 insertps instruction. */
22531 287521 : if (expand_vec_perm_insertps (d))
22532 : return true;
22533 :
22534 : /* Try the fully general two operand permute. */
22535 281451 : if (expand_vselect_vconcat (d->target, d->op0, d->op1, d->perm, nelt,
22536 281451 : d->testing_p))
22537 : return true;
22538 :
22539 : /* Recognize interleave style patterns with reversed operands. */
22540 137538 : if (!d->one_operand_p)
22541 : {
22542 897154 : for (i = 0; i < nelt; ++i)
22543 : {
22544 785234 : unsigned e = d->perm[i];
22545 785234 : if (e >= nelt)
22546 385203 : e -= nelt;
22547 : else
22548 400031 : e += nelt;
22549 785234 : nd.perm[i] = e;
22550 : }
22551 :
22552 111920 : if (expand_vselect_vconcat (d->target, d->op1, d->op0, nd.perm, nelt,
22553 111920 : d->testing_p))
22554 : return true;
22555 : }
22556 :
22557 : /* Try one of the AVX vpermil variable permutations. */
22558 137406 : if (expand_vec_perm_vpermil (d))
22559 : return true;
22560 :
22561 : /* Try the SSSE3 pshufb or XOP vpperm or AVX2 vperm2i128,
22562 : vpshufb, vpermd, vpermps or vpermq variable permutation. */
22563 136770 : if (expand_vec_perm_pshufb (d))
22564 : return true;
22565 :
22566 : /* Try the AVX2 vpalignr instruction. */
22567 125013 : if (expand_vec_perm_palignr (d, true))
22568 : return true;
22569 :
22570 : /* Try the AVX512F vperm{w,b,s,d} instructions */
22571 124889 : if (ix86_expand_vec_one_operand_perm_avx512 (d))
22572 : return true;
22573 :
22574 : /* Try the AVX512F vpermt2/vpermi2 instructions. */
22575 124680 : if (ix86_expand_vec_perm_vpermt2 (NULL_RTX, NULL_RTX, NULL_RTX, NULL_RTX, d))
22576 : return true;
22577 :
22578 : /* See if we can get the same permutation in different vector integer
22579 : mode. */
22580 123770 : if (canonicalize_vector_int_perm (d, &nd) && expand_vec_perm_1 (&nd))
22581 : {
22582 6805 : if (!d->testing_p)
22583 1234 : emit_move_insn (d->target, gen_lowpart (d->vmode, nd.target));
22584 6805 : return true;
22585 : }
22586 : return false;
22587 : }
22588 :
22589 : /* Canonicalize vec_perm index to make the first index
22590 : always comes from the first vector. */
22591 : static void
22592 8168 : ix86_vec_perm_index_canon (struct expand_vec_perm_d *d)
22593 : {
22594 8168 : unsigned nelt = d->nelt;
22595 8168 : if (d->perm[0] < nelt)
22596 : return;
22597 :
22598 5 : for (unsigned i = 0; i != nelt; i++)
22599 4 : d->perm[i] = (d->perm[i] + nelt) % (2 * nelt);
22600 :
22601 1 : std::swap (d->op0, d->op1);
22602 1 : return;
22603 : }
22604 :
22605 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
22606 : in terms of a pair of shufps+ shufps/pshufd instructions. */
22607 : static bool
22608 85707 : expand_vec_perm_shufps_shufps (struct expand_vec_perm_d *d)
22609 : {
22610 85707 : unsigned char perm1[4];
22611 85707 : machine_mode vmode = d->vmode;
22612 85707 : bool ok;
22613 85707 : unsigned i, j, k, count = 0;
22614 :
22615 85707 : if (d->one_operand_p
22616 80236 : || (vmode != V4SImode && vmode != V4SFmode))
22617 : return false;
22618 :
22619 35204 : if (d->testing_p)
22620 : return true;
22621 :
22622 8168 : ix86_vec_perm_index_canon (d);
22623 49008 : for (i = 0; i < 4; ++i)
22624 51107 : count += d->perm[i] > 3 ? 1 : 0;
22625 :
22626 8168 : gcc_assert (count & 3);
22627 :
22628 8168 : rtx tmp = gen_reg_rtx (vmode);
22629 : /* 2 from op0 and 2 from op1. */
22630 8168 : if (count == 2)
22631 : {
22632 : unsigned char perm2[4];
22633 18285 : for (i = 0, j = 0, k = 2; i < 4; ++i)
22634 14628 : if (d->perm[i] & 4)
22635 : {
22636 7314 : perm1[k++] = d->perm[i];
22637 7314 : perm2[i] = k - 1;
22638 : }
22639 : else
22640 : {
22641 7314 : perm1[j++] = d->perm[i];
22642 7314 : perm2[i] = j - 1;
22643 : }
22644 :
22645 : /* shufps. */
22646 7314 : ok = expand_vselect_vconcat (tmp, d->op0, d->op1,
22647 3657 : perm1, d->nelt, false);
22648 3657 : gcc_assert (ok);
22649 3657 : if (vmode == V4SImode && TARGET_SSE2)
22650 : /* pshufd. */
22651 2092 : ok = expand_vselect (d->target, tmp,
22652 2092 : perm2, d->nelt, false);
22653 : else
22654 : {
22655 : /* shufps. */
22656 1565 : perm2[2] += 4;
22657 1565 : perm2[3] += 4;
22658 1565 : ok = expand_vselect_vconcat (d->target, tmp, tmp,
22659 1565 : perm2, d->nelt, false);
22660 : }
22661 3657 : gcc_assert (ok);
22662 : }
22663 : /* 3 from one op and 1 from another. */
22664 : else
22665 : {
22666 22555 : unsigned pair_idx = 8, lone_idx = 8, shift;
22667 :
22668 : /* Find the lone index. */
22669 22555 : for (i = 0; i < 4; ++i)
22670 18044 : if ((d->perm[i] > 3 && count == 1)
22671 14739 : || (d->perm[i] < 4 && count == 3))
22672 18044 : lone_idx = i;
22673 :
22674 : /* When lone_idx is not 0, it must from second op(count == 1). */
22675 5717 : gcc_assert (count == (lone_idx ? 1 : 3));
22676 :
22677 : /* Find the pair index that sits in the same half as the lone index. */
22678 4511 : shift = lone_idx & 2;
22679 4511 : pair_idx = 1 - lone_idx + 2 * shift;
22680 :
22681 : /* First permutate lone index and pair index into the same vector as
22682 : [ lone, lone, pair, pair ]. */
22683 9022 : perm1[1] = perm1[0]
22684 4511 : = (count == 3) ? d->perm[lone_idx] : d->perm[lone_idx] - 4;
22685 9022 : perm1[3] = perm1[2]
22686 4511 : = (count == 3) ? d->perm[pair_idx] : d->perm[pair_idx] + 4;
22687 :
22688 : /* Always put the vector contains lone indx at the first. */
22689 4511 : if (count == 1)
22690 3305 : std::swap (d->op0, d->op1);
22691 :
22692 : /* shufps. */
22693 9022 : ok = expand_vselect_vconcat (tmp, d->op0, d->op1,
22694 4511 : perm1, d->nelt, false);
22695 4511 : gcc_assert (ok);
22696 :
22697 : /* Refine lone and pair index to original order. */
22698 4511 : perm1[shift] = lone_idx << 1;
22699 4511 : perm1[shift + 1] = pair_idx << 1;
22700 :
22701 : /* Select the remaining 2 elements in another vector. */
22702 13533 : for (i = 2 - shift; i < 4 - shift; ++i)
22703 9022 : perm1[i] = lone_idx == 1 ? d->perm[i] + 4 : d->perm[i];
22704 :
22705 : /* Adjust to original selector. */
22706 4511 : if (lone_idx > 1)
22707 2233 : std::swap (tmp, d->op1);
22708 :
22709 : /* shufps. */
22710 9022 : ok = expand_vselect_vconcat (d->target, tmp, d->op1,
22711 4511 : perm1, d->nelt, false);
22712 :
22713 4511 : gcc_assert (ok);
22714 : }
22715 :
22716 : return true;
22717 : }
22718 :
22719 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement D
22720 : in terms of a pair of pshuflw + pshufhw instructions. */
22721 :
22722 : static bool
22723 102322 : expand_vec_perm_pshuflw_pshufhw (struct expand_vec_perm_d *d)
22724 : {
22725 102322 : unsigned char perm2[MAX_VECT_LEN];
22726 102322 : unsigned i;
22727 102322 : bool ok;
22728 :
22729 102322 : if (d->vmode != V8HImode || !d->one_operand_p)
22730 : return false;
22731 :
22732 : /* The two permutations only operate in 64-bit lanes. */
22733 12841 : for (i = 0; i < 4; ++i)
22734 10364 : if (d->perm[i] >= 4)
22735 : return false;
22736 12329 : for (i = 4; i < 8; ++i)
22737 9866 : if (d->perm[i] < 4)
22738 : return false;
22739 :
22740 2463 : if (d->testing_p)
22741 : return true;
22742 :
22743 : /* Emit the pshuflw. */
22744 134 : memcpy (perm2, d->perm, 4);
22745 670 : for (i = 4; i < 8; ++i)
22746 536 : perm2[i] = i;
22747 134 : ok = expand_vselect (d->target, d->op0, perm2, 8, d->testing_p);
22748 134 : gcc_assert (ok);
22749 :
22750 : /* Emit the pshufhw. */
22751 134 : memcpy (perm2 + 4, d->perm + 4, 4);
22752 670 : for (i = 0; i < 4; ++i)
22753 536 : perm2[i] = i;
22754 134 : ok = expand_vselect (d->target, d->target, perm2, 8, d->testing_p);
22755 134 : gcc_assert (ok);
22756 :
22757 : return true;
22758 : }
22759 :
22760 : /* Try to permute 2 64-bit vectors by punpckldq + 128-bit vector shuffle. */
22761 : static bool
22762 50503 : expand_vec_perm_punpckldq_pshuf (struct expand_vec_perm_d *d)
22763 : {
22764 50503 : if (GET_MODE_BITSIZE (d->vmode) != 64
22765 16210 : || !TARGET_MMX_WITH_SSE
22766 66713 : || d->one_operand_p)
22767 : return false;
22768 :
22769 14772 : machine_mode widen_vmode;
22770 14772 : switch (d->vmode)
22771 : {
22772 : /* pshufd. */
22773 : case E_V2SImode:
22774 : widen_vmode = V4SImode;
22775 : break;
22776 :
22777 : /* pshufd. */
22778 1173 : case E_V2SFmode:
22779 1173 : widen_vmode = V4SFmode;
22780 1173 : break;
22781 :
22782 5136 : case E_V4HImode:
22783 5136 : widen_vmode = V8HImode;
22784 : /* pshufb. */
22785 5136 : if (!TARGET_SSSE3)
22786 : return false;
22787 : break;
22788 :
22789 5969 : case E_V8QImode:
22790 : /* pshufb. */
22791 5969 : widen_vmode = V16QImode;
22792 5969 : if (!TARGET_SSSE3)
22793 : return false;
22794 : break;
22795 :
22796 : default:
22797 : return false;
22798 : }
22799 :
22800 5736 : if (d->testing_p)
22801 : return true;
22802 :
22803 368 : struct expand_vec_perm_d dperm;
22804 368 : dperm.target = gen_reg_rtx (widen_vmode);
22805 368 : rtx op0 = gen_reg_rtx (widen_vmode);
22806 368 : emit_move_insn (op0, gen_rtx_VEC_CONCAT (widen_vmode, d->op0, d->op1));
22807 368 : dperm.op0 = op0;
22808 368 : dperm.op1 = op0;
22809 368 : dperm.vmode = widen_vmode;
22810 368 : unsigned nelt = GET_MODE_NUNITS (widen_vmode);
22811 368 : dperm.nelt = nelt;
22812 368 : dperm.one_operand_p = true;
22813 368 : dperm.testing_p = false;
22814 :
22815 1990 : for (unsigned i = 0; i != nelt / 2; i++)
22816 : {
22817 1622 : dperm.perm[i] = d->perm[i];
22818 1622 : dperm.perm[i + nelt / 2] = d->perm[i];
22819 : }
22820 :
22821 368 : gcc_assert (expand_vec_perm_1 (&dperm));
22822 368 : emit_move_insn (d->target, lowpart_subreg (d->vmode,
22823 : dperm.target,
22824 : dperm.vmode));
22825 368 : return true;
22826 : }
22827 :
22828 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
22829 : the permutation using the SSSE3 palignr instruction. This succeeds
22830 : when all of the elements in PERM fit within one vector and we merely
22831 : need to shift them down so that a single vector permutation has a
22832 : chance to succeed. If SINGLE_INSN_ONLY_P, succeed if only
22833 : the vpalignr instruction itself can perform the requested permutation. */
22834 :
22835 : static bool
22836 224872 : expand_vec_perm_palignr (struct expand_vec_perm_d *d, bool single_insn_only_p)
22837 : {
22838 224872 : unsigned i, nelt = d->nelt;
22839 224872 : unsigned min, max, minswap, maxswap;
22840 224872 : bool in_order, ok, swap = false;
22841 224872 : rtx shift, target;
22842 224872 : struct expand_vec_perm_d dcopy;
22843 :
22844 : /* Even with AVX, palignr only operates on 128-bit vectors,
22845 : in AVX2 palignr operates on both 128-bit lanes. */
22846 117914 : if ((!TARGET_SSSE3 || GET_MODE_SIZE (d->vmode) != 16)
22847 267492 : && (!TARGET_AVX2 || GET_MODE_SIZE (d->vmode) != 32))
22848 : return false;
22849 :
22850 34521 : min = 2 * nelt;
22851 34521 : max = 0;
22852 34521 : minswap = 2 * nelt;
22853 34521 : maxswap = 0;
22854 239317 : for (i = 0; i < nelt; ++i)
22855 : {
22856 204796 : unsigned e = d->perm[i];
22857 204796 : unsigned eswap = d->perm[i] ^ nelt;
22858 409592 : if (GET_MODE_SIZE (d->vmode) == 32)
22859 : {
22860 70192 : e = (e & ((nelt / 2) - 1)) | ((e & nelt) >> 1);
22861 70192 : eswap = e ^ (nelt / 2);
22862 : }
22863 204796 : if (e < min)
22864 : min = e;
22865 204796 : if (e > max)
22866 : max = e;
22867 204796 : if (eswap < minswap)
22868 : minswap = eswap;
22869 204796 : if (eswap > maxswap)
22870 : maxswap = eswap;
22871 : }
22872 34521 : if (min == 0
22873 50464 : || max - min >= (GET_MODE_SIZE (d->vmode) == 32 ? nelt / 2 : nelt))
22874 : {
22875 31321 : if (d->one_operand_p
22876 31052 : || minswap == 0
22877 67085 : || maxswap - minswap >= (GET_MODE_SIZE (d->vmode) == 32
22878 17882 : ? nelt / 2 : nelt))
22879 : return false;
22880 : swap = true;
22881 : min = minswap;
22882 6420 : max = maxswap;
22883 : }
22884 :
22885 : /* Given that we have SSSE3, we know we'll be able to implement the
22886 : single operand permutation after the palignr with pshufb for
22887 : 128-bit vectors. If SINGLE_INSN_ONLY_P, in_order has to be computed
22888 : first. */
22889 6474 : if (d->testing_p && GET_MODE_SIZE (d->vmode) == 16 && !single_insn_only_p)
22890 : return true;
22891 :
22892 6420 : dcopy = *d;
22893 6420 : if (swap)
22894 : {
22895 3220 : dcopy.op0 = d->op1;
22896 3220 : dcopy.op1 = d->op0;
22897 16172 : for (i = 0; i < nelt; ++i)
22898 12952 : dcopy.perm[i] ^= nelt;
22899 : }
22900 :
22901 : in_order = true;
22902 32668 : for (i = 0; i < nelt; ++i)
22903 : {
22904 26248 : unsigned e = dcopy.perm[i];
22905 26248 : if (GET_MODE_SIZE (d->vmode) == 32
22906 1152 : && e >= nelt
22907 26510 : && (e & (nelt / 2 - 1)) < min)
22908 262 : e = e - min - (nelt / 2);
22909 : else
22910 25986 : e = e - min;
22911 26248 : if (e != i)
22912 19394 : in_order = false;
22913 26248 : dcopy.perm[i] = e;
22914 : }
22915 6420 : dcopy.one_operand_p = true;
22916 :
22917 6420 : if (single_insn_only_p && !in_order)
22918 : return false;
22919 :
22920 : /* For AVX2, test whether we can permute the result in one instruction. */
22921 3271 : if (d->testing_p)
22922 : {
22923 54 : if (in_order)
22924 : return true;
22925 0 : dcopy.op1 = dcopy.op0;
22926 0 : return expand_vec_perm_1 (&dcopy);
22927 : }
22928 :
22929 6434 : shift = GEN_INT (min * GET_MODE_UNIT_BITSIZE (d->vmode));
22930 6434 : if (GET_MODE_SIZE (d->vmode) == 16)
22931 : {
22932 3145 : target = gen_reg_rtx (V1TImode);
22933 3145 : emit_insn (gen_ssse3_palignrv1ti (target,
22934 3145 : gen_lowpart (V1TImode, dcopy.op1),
22935 3145 : gen_lowpart (V1TImode, dcopy.op0),
22936 : shift));
22937 : }
22938 : else
22939 : {
22940 72 : target = gen_reg_rtx (V2TImode);
22941 72 : emit_insn (gen_avx2_palignrv2ti (target,
22942 72 : gen_lowpart (V2TImode, dcopy.op1),
22943 72 : gen_lowpart (V2TImode, dcopy.op0),
22944 : shift));
22945 : }
22946 :
22947 3217 : dcopy.op0 = dcopy.op1 = gen_lowpart (d->vmode, target);
22948 :
22949 : /* Test for the degenerate case where the alignment by itself
22950 : produces the desired permutation. */
22951 3217 : if (in_order)
22952 : {
22953 70 : emit_move_insn (d->target, dcopy.op0);
22954 70 : return true;
22955 : }
22956 :
22957 3147 : ok = expand_vec_perm_1 (&dcopy);
22958 3159 : gcc_assert (ok || GET_MODE_SIZE (d->vmode) == 32);
22959 :
22960 : return ok;
22961 : }
22962 :
22963 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
22964 : the permutation using the SSE4_1 pblendv instruction. Potentially
22965 : reduces permutation from 2 pshufb and or to 1 pshufb and pblendv. */
22966 :
22967 : static bool
22968 90617 : expand_vec_perm_pblendv (struct expand_vec_perm_d *d)
22969 : {
22970 90617 : unsigned i, which, nelt = d->nelt;
22971 90617 : struct expand_vec_perm_d dcopy, dcopy1;
22972 90617 : machine_mode vmode = d->vmode;
22973 90617 : bool ok;
22974 :
22975 : /* Use the same checks as in expand_vec_perm_blend. */
22976 90617 : if (d->one_operand_p)
22977 : return false;
22978 88951 : if (TARGET_AVX2 && GET_MODE_SIZE (vmode) == 32)
22979 : ;
22980 83399 : else if (TARGET_AVX && (vmode == V4DFmode || vmode == V8SFmode))
22981 : ;
22982 80052 : else if (TARGET_SSE4_1
22983 90596 : && (GET_MODE_SIZE (vmode) == 16
22984 9154 : || (TARGET_MMX_WITH_SSE && GET_MODE_SIZE (vmode) == 8)
22985 2733 : || GET_MODE_SIZE (vmode) == 4))
22986 : ;
22987 : else
22988 : return false;
22989 :
22990 : /* Figure out where permutation elements stay not in their
22991 : respective lanes. */
22992 108472 : for (i = 0, which = 0; i < nelt; ++i)
22993 : {
22994 93178 : unsigned e = d->perm[i];
22995 93178 : if (e != i)
22996 128205 : which |= (e < nelt ? 1 : 2);
22997 : }
22998 : /* We can pblend the part where elements stay not in their
22999 : respective lanes only when these elements are all in one
23000 : half of a permutation.
23001 : {0 1 8 3 4 5 9 7} is ok as 8, 9 are at not at their respective
23002 : lanes, but both 8 and 9 >= 8
23003 : {0 1 8 3 4 5 2 7} is not ok as 2 and 8 are not at their
23004 : respective lanes and 8 >= 8, but 2 not. */
23005 15294 : if (which != 1 && which != 2)
23006 : return false;
23007 3175 : if (d->testing_p && GET_MODE_SIZE (vmode) == 16)
23008 : return true;
23009 :
23010 : /* First we apply one operand permutation to the part where
23011 : elements stay not in their respective lanes. */
23012 1958 : dcopy = *d;
23013 1958 : if (which == 2)
23014 1958 : dcopy.op0 = dcopy.op1 = d->op1;
23015 : else
23016 0 : dcopy.op0 = dcopy.op1 = d->op0;
23017 1958 : if (!d->testing_p)
23018 741 : dcopy.target = gen_reg_rtx (vmode);
23019 1958 : dcopy.one_operand_p = true;
23020 :
23021 15762 : for (i = 0; i < nelt; ++i)
23022 13804 : dcopy.perm[i] = d->perm[i] & (nelt - 1);
23023 :
23024 1958 : ok = expand_vec_perm_1 (&dcopy);
23025 3916 : if (GET_MODE_SIZE (vmode) != 16 && !ok)
23026 : return false;
23027 : else
23028 1663 : gcc_assert (ok);
23029 1663 : if (d->testing_p)
23030 : return true;
23031 :
23032 : /* Next we put permuted elements into their positions. */
23033 679 : dcopy1 = *d;
23034 679 : if (which == 2)
23035 679 : dcopy1.op1 = dcopy.target;
23036 : else
23037 0 : dcopy1.op0 = dcopy.target;
23038 :
23039 5751 : for (i = 0; i < nelt; ++i)
23040 5072 : dcopy1.perm[i] = ((d->perm[i] >= nelt) ? (nelt + i) : i);
23041 :
23042 679 : ok = expand_vec_perm_blend (&dcopy1);
23043 679 : gcc_assert (ok);
23044 :
23045 : return true;
23046 : }
23047 :
23048 : static bool expand_vec_perm_interleave3 (struct expand_vec_perm_d *d);
23049 :
23050 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
23051 : a two vector permutation into a single vector permutation by using
23052 : an interleave operation to merge the vectors. */
23053 :
23054 : static bool
23055 96724 : expand_vec_perm_interleave2 (struct expand_vec_perm_d *d)
23056 : {
23057 96724 : struct expand_vec_perm_d dremap, dfinal;
23058 96724 : unsigned i, nelt = d->nelt, nelt2 = nelt / 2;
23059 96724 : unsigned HOST_WIDE_INT contents;
23060 96724 : unsigned char remap[2 * MAX_VECT_LEN];
23061 96724 : rtx_insn *seq;
23062 96724 : bool ok, same_halves = false;
23063 :
23064 96724 : if (GET_MODE_SIZE (d->vmode) == 4
23065 173314 : || GET_MODE_SIZE (d->vmode) == 8
23066 234324 : || GET_MODE_SIZE (d->vmode) == 16)
23067 : {
23068 90196 : if (d->one_operand_p)
23069 : return false;
23070 : }
23071 13056 : else if (GET_MODE_SIZE (d->vmode) == 32)
23072 : {
23073 6203 : if (!TARGET_AVX)
23074 : return false;
23075 : /* For 32-byte modes allow even d->one_operand_p.
23076 : The lack of cross-lane shuffling in some instructions
23077 : might prevent a single insn shuffle. */
23078 6203 : dfinal = *d;
23079 6203 : dfinal.testing_p = true;
23080 : /* If expand_vec_perm_interleave3 can expand this into
23081 : a 3 insn sequence, give up and let it be expanded as
23082 : 3 insn sequence. While that is one insn longer,
23083 : it doesn't need a memory operand and in the common
23084 : case that both interleave low and high permutations
23085 : with the same operands are adjacent needs 4 insns
23086 : for both after CSE. */
23087 6203 : if (expand_vec_perm_interleave3 (&dfinal))
23088 : return false;
23089 : }
23090 : else
23091 : return false;
23092 :
23093 : /* Examine from whence the elements come. */
23094 90782 : contents = 0;
23095 680328 : for (i = 0; i < nelt; ++i)
23096 589546 : contents |= HOST_WIDE_INT_1U << d->perm[i];
23097 :
23098 90782 : memset (remap, 0xff, sizeof (remap));
23099 90782 : dremap = *d;
23100 :
23101 90782 : if (GET_MODE_SIZE (d->vmode) == 4
23102 173273 : || GET_MODE_SIZE (d->vmode) == 8)
23103 : {
23104 24706 : unsigned HOST_WIDE_INT h1, h2, h3, h4;
23105 :
23106 : /* Split the two input vectors into 4 halves. */
23107 24706 : h1 = (HOST_WIDE_INT_1U << nelt2) - 1;
23108 24706 : h2 = h1 << nelt2;
23109 24706 : h3 = h2 << nelt2;
23110 24706 : h4 = h3 << nelt2;
23111 :
23112 : /* If the elements from the low halves use interleave low,
23113 : and similarly for interleave high. */
23114 24706 : if ((contents & (h1 | h3)) == contents)
23115 : {
23116 : /* punpckl* */
23117 2796 : for (i = 0; i < nelt2; ++i)
23118 : {
23119 1912 : remap[i] = i * 2;
23120 1912 : remap[i + nelt] = i * 2 + 1;
23121 1912 : dremap.perm[i * 2] = i;
23122 1912 : dremap.perm[i * 2 + 1] = i + nelt;
23123 : }
23124 : }
23125 23822 : else if ((contents & (h2 | h4)) == contents)
23126 : {
23127 : /* punpckh* */
23128 2277 : for (i = 0; i < nelt2; ++i)
23129 : {
23130 1548 : remap[i + nelt2] = i * 2;
23131 1548 : remap[i + nelt + nelt2] = i * 2 + 1;
23132 1548 : dremap.perm[i * 2] = i + nelt2;
23133 1548 : dremap.perm[i * 2 + 1] = i + nelt + nelt2;
23134 : }
23135 : }
23136 : else
23137 : return false;
23138 : }
23139 132152 : else if (GET_MODE_SIZE (d->vmode) == 16)
23140 : {
23141 60099 : unsigned HOST_WIDE_INT h1, h2, h3, h4;
23142 :
23143 : /* Split the two input vectors into 4 halves. */
23144 60099 : h1 = (HOST_WIDE_INT_1U << nelt2) - 1;
23145 60099 : h2 = h1 << nelt2;
23146 60099 : h3 = h2 << nelt2;
23147 60099 : h4 = h3 << nelt2;
23148 :
23149 : /* If the elements from the low halves use interleave low, and similarly
23150 : for interleave high. If the elements are from mis-matched halves, we
23151 : can use shufps for V4SF/V4SI or do a DImode shuffle. */
23152 60099 : if ((contents & (h1 | h3)) == contents)
23153 : {
23154 : /* punpckl* */
23155 5368 : for (i = 0; i < nelt2; ++i)
23156 : {
23157 3906 : remap[i] = i * 2;
23158 3906 : remap[i + nelt] = i * 2 + 1;
23159 3906 : dremap.perm[i * 2] = i;
23160 3906 : dremap.perm[i * 2 + 1] = i + nelt;
23161 : }
23162 1462 : if (!TARGET_SSE2 && d->vmode == V4SImode)
23163 0 : dremap.vmode = V4SFmode;
23164 : }
23165 58637 : else if ((contents & (h2 | h4)) == contents)
23166 : {
23167 : /* punpckh* */
23168 4563 : for (i = 0; i < nelt2; ++i)
23169 : {
23170 3278 : remap[i + nelt2] = i * 2;
23171 3278 : remap[i + nelt + nelt2] = i * 2 + 1;
23172 3278 : dremap.perm[i * 2] = i + nelt2;
23173 3278 : dremap.perm[i * 2 + 1] = i + nelt + nelt2;
23174 : }
23175 1285 : if (!TARGET_SSE2 && d->vmode == V4SImode)
23176 0 : dremap.vmode = V4SFmode;
23177 : }
23178 57352 : else if ((contents & (h1 | h4)) == contents)
23179 : {
23180 : /* shufps */
23181 2990 : for (i = 0; i < nelt2; ++i)
23182 : {
23183 2232 : remap[i] = i;
23184 2232 : remap[i + nelt + nelt2] = i + nelt2;
23185 2232 : dremap.perm[i] = i;
23186 2232 : dremap.perm[i + nelt2] = i + nelt + nelt2;
23187 : }
23188 758 : if (nelt != 4)
23189 : {
23190 : /* shufpd */
23191 120 : dremap.vmode = V2DImode;
23192 120 : dremap.nelt = 2;
23193 120 : dremap.perm[0] = 0;
23194 120 : dremap.perm[1] = 3;
23195 : }
23196 : }
23197 56594 : else if ((contents & (h2 | h3)) == contents)
23198 : {
23199 : /* shufps */
23200 4002 : for (i = 0; i < nelt2; ++i)
23201 : {
23202 2914 : remap[i + nelt2] = i;
23203 2914 : remap[i + nelt] = i + nelt2;
23204 2914 : dremap.perm[i] = i + nelt2;
23205 2914 : dremap.perm[i + nelt2] = i + nelt;
23206 : }
23207 1088 : if (nelt != 4)
23208 : {
23209 : /* shufpd */
23210 139 : dremap.vmode = V2DImode;
23211 139 : dremap.nelt = 2;
23212 139 : dremap.perm[0] = 1;
23213 139 : dremap.perm[1] = 2;
23214 : }
23215 : }
23216 : else
23217 : return false;
23218 : }
23219 : else
23220 : {
23221 5977 : unsigned int nelt4 = nelt / 4, nzcnt = 0;
23222 5977 : unsigned HOST_WIDE_INT q[8];
23223 5977 : unsigned int nonzero_halves[4];
23224 :
23225 : /* Split the two input vectors into 8 quarters. */
23226 5977 : q[0] = (HOST_WIDE_INT_1U << nelt4) - 1;
23227 47816 : for (i = 1; i < 8; ++i)
23228 41839 : q[i] = q[0] << (nelt4 * i);
23229 29885 : for (i = 0; i < 4; ++i)
23230 23908 : if (((q[2 * i] | q[2 * i + 1]) & contents) != 0)
23231 : {
23232 21650 : nonzero_halves[nzcnt] = i;
23233 21650 : ++nzcnt;
23234 : }
23235 :
23236 5977 : if (nzcnt == 1)
23237 : {
23238 211 : gcc_assert (d->one_operand_p);
23239 211 : nonzero_halves[1] = nonzero_halves[0];
23240 211 : same_halves = true;
23241 : }
23242 5766 : else if (d->one_operand_p)
23243 : {
23244 39 : gcc_assert (nonzero_halves[0] == 0);
23245 39 : gcc_assert (nonzero_halves[1] == 1);
23246 : }
23247 :
23248 5977 : if (nzcnt <= 2)
23249 : {
23250 502 : if (d->perm[0] / nelt2 == nonzero_halves[1])
23251 : {
23252 : /* Attempt to increase the likelihood that dfinal
23253 : shuffle will be intra-lane. */
23254 235 : std::swap (nonzero_halves[0], nonzero_halves[1]);
23255 : }
23256 :
23257 : /* vperm2f128 or vperm2i128. */
23258 3292 : for (i = 0; i < nelt2; ++i)
23259 : {
23260 2790 : remap[i + nonzero_halves[1] * nelt2] = i + nelt2;
23261 2790 : remap[i + nonzero_halves[0] * nelt2] = i;
23262 2790 : dremap.perm[i + nelt2] = i + nonzero_halves[1] * nelt2;
23263 2790 : dremap.perm[i] = i + nonzero_halves[0] * nelt2;
23264 : }
23265 :
23266 502 : if (d->vmode != V8SFmode
23267 : && d->vmode != V4DFmode
23268 : && d->vmode != V8SImode)
23269 : {
23270 148 : dremap.vmode = V8SImode;
23271 148 : dremap.nelt = 8;
23272 740 : for (i = 0; i < 4; ++i)
23273 : {
23274 592 : dremap.perm[i] = i + nonzero_halves[0] * 4;
23275 592 : dremap.perm[i + 4] = i + nonzero_halves[1] * 4;
23276 : }
23277 : }
23278 : }
23279 5475 : else if (d->one_operand_p)
23280 5010 : return false;
23281 5475 : else if (TARGET_AVX2
23282 2172 : && (contents & (q[0] | q[2] | q[4] | q[6])) == contents)
23283 : {
23284 : /* vpunpckl* */
23285 491 : for (i = 0; i < nelt4; ++i)
23286 : {
23287 247 : remap[i] = i * 2;
23288 247 : remap[i + nelt] = i * 2 + 1;
23289 247 : remap[i + nelt2] = i * 2 + nelt2;
23290 247 : remap[i + nelt + nelt2] = i * 2 + nelt2 + 1;
23291 247 : dremap.perm[i * 2] = i;
23292 247 : dremap.perm[i * 2 + 1] = i + nelt;
23293 247 : dremap.perm[i * 2 + nelt2] = i + nelt2;
23294 247 : dremap.perm[i * 2 + nelt2 + 1] = i + nelt + nelt2;
23295 : }
23296 : }
23297 5231 : else if (TARGET_AVX2
23298 1928 : && (contents & (q[1] | q[3] | q[5] | q[7])) == contents)
23299 : {
23300 : /* vpunpckh* */
23301 445 : for (i = 0; i < nelt4; ++i)
23302 : {
23303 224 : remap[i + nelt4] = i * 2;
23304 224 : remap[i + nelt + nelt4] = i * 2 + 1;
23305 224 : remap[i + nelt2 + nelt4] = i * 2 + nelt2;
23306 224 : remap[i + nelt + nelt2 + nelt4] = i * 2 + nelt2 + 1;
23307 224 : dremap.perm[i * 2] = i + nelt4;
23308 224 : dremap.perm[i * 2 + 1] = i + nelt + nelt4;
23309 224 : dremap.perm[i * 2 + nelt2] = i + nelt2 + nelt4;
23310 224 : dremap.perm[i * 2 + nelt2 + 1] = i + nelt + nelt2 + nelt4;
23311 : }
23312 : }
23313 : else
23314 : return false;
23315 : }
23316 :
23317 : /* Use the remapping array set up above to move the elements from their
23318 : swizzled locations into their final destinations. */
23319 7173 : dfinal = *d;
23320 46217 : for (i = 0; i < nelt; ++i)
23321 : {
23322 39044 : unsigned e = remap[d->perm[i]];
23323 39044 : gcc_assert (e < nelt);
23324 : /* If same_halves is true, both halves of the remapped vector are the
23325 : same. Avoid cross-lane accesses if possible. */
23326 39044 : if (same_halves && i >= nelt2)
23327 : {
23328 784 : gcc_assert (e < nelt2);
23329 784 : dfinal.perm[i] = e + nelt2;
23330 : }
23331 : else
23332 38260 : dfinal.perm[i] = e;
23333 : }
23334 7173 : if (!d->testing_p)
23335 : {
23336 2737 : dremap.target = gen_reg_rtx (dremap.vmode);
23337 2737 : dfinal.op0 = gen_lowpart (dfinal.vmode, dremap.target);
23338 : }
23339 7173 : dfinal.op1 = dfinal.op0;
23340 7173 : dfinal.one_operand_p = true;
23341 :
23342 : /* Test if the final remap can be done with a single insn. For V4SFmode or
23343 : V4SImode this *will* succeed. For V8HImode or V16QImode it may not. */
23344 7173 : start_sequence ();
23345 7173 : ok = expand_vec_perm_1 (&dfinal);
23346 7173 : seq = end_sequence ();
23347 :
23348 7173 : if (!ok)
23349 : return false;
23350 :
23351 5939 : if (d->testing_p)
23352 : return true;
23353 :
23354 2698 : if (dremap.vmode != dfinal.vmode)
23355 : {
23356 57 : dremap.op0 = gen_lowpart (dremap.vmode, dremap.op0);
23357 57 : dremap.op1 = gen_lowpart (dremap.vmode, dremap.op1);
23358 : }
23359 :
23360 2698 : ok = expand_vec_perm_1 (&dremap);
23361 2698 : gcc_assert (ok);
23362 :
23363 2698 : emit_insn (seq);
23364 2698 : return true;
23365 : }
23366 :
23367 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
23368 : a single vector cross-lane permutation into vpermq followed
23369 : by any of the single insn permutations. */
23370 :
23371 : static bool
23372 90685 : expand_vec_perm_vpermq_perm_1 (struct expand_vec_perm_d *d)
23373 : {
23374 90685 : struct expand_vec_perm_d dremap, dfinal;
23375 90685 : unsigned i, j, nelt = d->nelt, nelt2 = nelt / 2, nelt4 = nelt / 4;
23376 90685 : unsigned contents[2];
23377 90685 : bool ok;
23378 :
23379 90685 : if (!(TARGET_AVX2
23380 4035 : && (d->vmode == V32QImode || d->vmode == V16HImode)
23381 256 : && d->one_operand_p))
23382 : return false;
23383 :
23384 7 : contents[0] = 0;
23385 7 : contents[1] = 0;
23386 103 : for (i = 0; i < nelt2; ++i)
23387 : {
23388 96 : contents[0] |= 1u << (d->perm[i] / nelt4);
23389 96 : contents[1] |= 1u << (d->perm[i + nelt2] / nelt4);
23390 : }
23391 :
23392 7 : for (i = 0; i < 2; ++i)
23393 : {
23394 : unsigned int cnt = 0;
23395 21 : for (j = 0; j < 4; ++j)
23396 21 : if ((contents[i] & (1u << j)) != 0 && ++cnt > 2)
23397 : return false;
23398 : }
23399 :
23400 0 : if (d->testing_p)
23401 : return true;
23402 :
23403 0 : dremap = *d;
23404 0 : dremap.vmode = V4DImode;
23405 0 : dremap.nelt = 4;
23406 0 : dremap.target = gen_reg_rtx (V4DImode);
23407 0 : dremap.op0 = gen_lowpart (V4DImode, d->op0);
23408 0 : dremap.op1 = dremap.op0;
23409 0 : dremap.one_operand_p = true;
23410 0 : for (i = 0; i < 2; ++i)
23411 : {
23412 : unsigned int cnt = 0;
23413 0 : for (j = 0; j < 4; ++j)
23414 0 : if ((contents[i] & (1u << j)) != 0)
23415 0 : dremap.perm[2 * i + cnt++] = j;
23416 0 : for (; cnt < 2; ++cnt)
23417 0 : dremap.perm[2 * i + cnt] = 0;
23418 : }
23419 :
23420 0 : dfinal = *d;
23421 0 : dfinal.op0 = gen_lowpart (dfinal.vmode, dremap.target);
23422 0 : dfinal.op1 = dfinal.op0;
23423 0 : dfinal.one_operand_p = true;
23424 0 : for (i = 0, j = 0; i < nelt; ++i)
23425 : {
23426 0 : if (i == nelt2)
23427 0 : j = 2;
23428 0 : dfinal.perm[i] = (d->perm[i] & (nelt4 - 1)) | (j ? nelt2 : 0);
23429 0 : if ((d->perm[i] / nelt4) == dremap.perm[j])
23430 : ;
23431 0 : else if ((d->perm[i] / nelt4) == dremap.perm[j + 1])
23432 0 : dfinal.perm[i] |= nelt4;
23433 : else
23434 0 : gcc_unreachable ();
23435 : }
23436 :
23437 0 : ok = expand_vec_perm_1 (&dremap);
23438 0 : gcc_assert (ok);
23439 :
23440 0 : ok = expand_vec_perm_1 (&dfinal);
23441 0 : gcc_assert (ok);
23442 :
23443 : return true;
23444 : }
23445 :
23446 : static bool canonicalize_perm (struct expand_vec_perm_d *d);
23447 :
23448 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to expand
23449 : a vector permutation using two instructions, vperm2f128 resp.
23450 : vperm2i128 followed by any single in-lane permutation. */
23451 :
23452 : static bool
23453 90685 : expand_vec_perm_vperm2f128 (struct expand_vec_perm_d *d)
23454 : {
23455 90685 : struct expand_vec_perm_d dfirst, dsecond;
23456 90685 : unsigned i, j, nelt = d->nelt, nelt2 = nelt / 2, perm;
23457 90685 : bool ok;
23458 :
23459 90685 : if (!TARGET_AVX
23460 22186 : || GET_MODE_SIZE (d->vmode) != 32
23461 96046 : || (d->vmode != V8SFmode && d->vmode != V4DFmode && !TARGET_AVX2))
23462 : return false;
23463 :
23464 5177 : dsecond = *d;
23465 5177 : dsecond.one_operand_p = false;
23466 5177 : dsecond.testing_p = true;
23467 :
23468 : /* ((perm << 2)|perm) & 0x33 is the vperm2[fi]128
23469 : immediate. For perm < 16 the second permutation uses
23470 : d->op0 as first operand, for perm >= 16 it uses d->op1
23471 : as first operand. The second operand is the result of
23472 : vperm2[fi]128. */
23473 169527 : for (perm = 0; perm < 32; perm++)
23474 : {
23475 : /* Ignore permutations which do not move anything cross-lane. */
23476 164433 : if (perm < 16)
23477 : {
23478 : /* The second shuffle for e.g. V4DFmode has
23479 : 0123 and ABCD operands.
23480 : Ignore AB23, as 23 is already in the second lane
23481 : of the first operand. */
23482 82470 : if ((perm & 0xc) == (1 << 2)) continue;
23483 : /* And 01CD, as 01 is in the first lane of the first
23484 : operand. */
23485 61838 : if ((perm & 3) == 0) continue;
23486 : /* And 4567, as then the vperm2[fi]128 doesn't change
23487 : anything on the original 4567 second operand. */
23488 46361 : if ((perm & 0xf) == ((3 << 2) | 2)) continue;
23489 : }
23490 : else
23491 : {
23492 : /* The second shuffle for e.g. V4DFmode has
23493 : 4567 and ABCD operands.
23494 : Ignore AB67, as 67 is already in the second lane
23495 : of the first operand. */
23496 81963 : if ((perm & 0xc) == (3 << 2)) continue;
23497 : /* And 45CD, as 45 is in the first lane of the first
23498 : operand. */
23499 61587 : if ((perm & 3) == 2) continue;
23500 : /* And 0123, as then the vperm2[fi]128 doesn't change
23501 : anything on the original 0123 first operand. */
23502 46214 : if ((perm & 0xf) == (1 << 2)) continue;
23503 : }
23504 :
23505 212352 : for (i = 0; i < nelt; i++)
23506 : {
23507 211225 : j = d->perm[i] / nelt2;
23508 392886 : if (j == ((perm >> (2 * (i >= nelt2))) & 3))
23509 52316 : dsecond.perm[i] = nelt + (i & nelt2) + (d->perm[i] & (nelt2 - 1));
23510 263771 : else if (j == (unsigned) (i >= nelt2) + 2 * (perm >= 16))
23511 77743 : dsecond.perm[i] = d->perm[i] & (nelt - 1);
23512 : else
23513 : break;
23514 : }
23515 :
23516 82293 : if (i == nelt)
23517 : {
23518 1127 : start_sequence ();
23519 1127 : ok = expand_vec_perm_1 (&dsecond);
23520 1127 : end_sequence ();
23521 : }
23522 : else
23523 : ok = false;
23524 :
23525 1127 : if (ok)
23526 : {
23527 68 : if (d->testing_p)
23528 : return true;
23529 :
23530 : /* Found a usable second shuffle. dfirst will be
23531 : vperm2f128 on d->op0 and d->op1. */
23532 46 : dsecond.testing_p = false;
23533 46 : dfirst = *d;
23534 46 : dfirst.target = gen_reg_rtx (d->vmode);
23535 270 : for (i = 0; i < nelt; i++)
23536 448 : dfirst.perm[i] = (i & (nelt2 - 1))
23537 336 : + ((perm >> (2 * (i >= nelt2))) & 3) * nelt2;
23538 :
23539 46 : canonicalize_perm (&dfirst);
23540 46 : ok = expand_vec_perm_1 (&dfirst);
23541 46 : gcc_assert (ok);
23542 :
23543 : /* And dsecond is some single insn shuffle, taking
23544 : d->op0 and result of vperm2f128 (if perm < 16) or
23545 : d->op1 and result of vperm2f128 (otherwise). */
23546 46 : if (perm >= 16)
23547 46 : dsecond.op0 = dsecond.op1;
23548 46 : dsecond.op1 = dfirst.target;
23549 :
23550 46 : ok = expand_vec_perm_1 (&dsecond);
23551 46 : gcc_assert (ok);
23552 :
23553 : return true;
23554 : }
23555 :
23556 : /* For one operand, the only useful vperm2f128 permutation is 0x01
23557 : aka lanes swap. */
23558 82225 : if (d->one_operand_p)
23559 : return false;
23560 : }
23561 :
23562 : return false;
23563 : }
23564 :
23565 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
23566 : a two vector permutation using 2 intra-lane interleave insns
23567 : and cross-lane shuffle for 32-byte vectors. */
23568 :
23569 : static bool
23570 34163 : expand_vec_perm_interleave3 (struct expand_vec_perm_d *d)
23571 : {
23572 34163 : unsigned i, nelt;
23573 34163 : rtx (*gen) (rtx, rtx, rtx);
23574 :
23575 34163 : if (d->one_operand_p)
23576 : return false;
23577 31895 : if (TARGET_AVX2 && GET_MODE_SIZE (d->vmode) == 32)
23578 : ;
23579 25235 : else if (TARGET_AVX && (d->vmode == V8SFmode || d->vmode == V4DFmode))
23580 : ;
23581 : else
23582 : return false;
23583 :
23584 8325 : nelt = d->nelt;
23585 8325 : if (d->perm[0] != 0 && d->perm[0] != nelt / 2)
23586 : return false;
23587 8698 : for (i = 0; i < nelt; i += 2)
23588 8326 : if (d->perm[i] != d->perm[0] + i / 2
23589 7453 : || d->perm[i + 1] != d->perm[0] + i / 2 + nelt)
23590 : return false;
23591 :
23592 372 : if (d->testing_p)
23593 : return true;
23594 :
23595 56 : switch (d->vmode)
23596 : {
23597 32 : case E_V32QImode:
23598 32 : if (d->perm[0])
23599 : gen = gen_vec_interleave_highv32qi;
23600 : else
23601 16 : gen = gen_vec_interleave_lowv32qi;
23602 : break;
23603 18 : case E_V16HImode:
23604 18 : if (d->perm[0])
23605 : gen = gen_vec_interleave_highv16hi;
23606 : else
23607 9 : gen = gen_vec_interleave_lowv16hi;
23608 : break;
23609 0 : case E_V8SImode:
23610 0 : if (d->perm[0])
23611 : gen = gen_vec_interleave_highv8si;
23612 : else
23613 0 : gen = gen_vec_interleave_lowv8si;
23614 : break;
23615 4 : case E_V4DImode:
23616 4 : if (d->perm[0])
23617 : gen = gen_vec_interleave_highv4di;
23618 : else
23619 2 : gen = gen_vec_interleave_lowv4di;
23620 : break;
23621 2 : case E_V8SFmode:
23622 2 : if (d->perm[0])
23623 : gen = gen_vec_interleave_highv8sf;
23624 : else
23625 1 : gen = gen_vec_interleave_lowv8sf;
23626 : break;
23627 0 : case E_V4DFmode:
23628 0 : if (d->perm[0])
23629 : gen = gen_vec_interleave_highv4df;
23630 : else
23631 0 : gen = gen_vec_interleave_lowv4df;
23632 : break;
23633 0 : default:
23634 0 : gcc_unreachable ();
23635 : }
23636 :
23637 56 : emit_insn (gen (d->target, d->op0, d->op1));
23638 56 : return true;
23639 : }
23640 :
23641 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement
23642 : a single vector permutation using a single intra-lane vector
23643 : permutation, vperm2f128 swapping the lanes and vblend* insn blending
23644 : the non-swapped and swapped vectors together. */
23645 :
23646 : static bool
23647 27814 : expand_vec_perm_vperm2f128_vblend (struct expand_vec_perm_d *d)
23648 : {
23649 27814 : struct expand_vec_perm_d dfirst, dsecond;
23650 27814 : unsigned i, j, msk, nelt = d->nelt, nelt2 = nelt / 2;
23651 27814 : rtx_insn *seq;
23652 27814 : bool ok;
23653 27814 : rtx (*blend) (rtx, rtx, rtx, rtx) = NULL;
23654 :
23655 27814 : if (!TARGET_AVX
23656 2933 : || TARGET_AVX2
23657 1814 : || (d->vmode != V8SFmode && d->vmode != V4DFmode)
23658 1630 : || !d->one_operand_p)
23659 : return false;
23660 :
23661 0 : dfirst = *d;
23662 0 : for (i = 0; i < nelt; i++)
23663 0 : dfirst.perm[i] = 0xff;
23664 0 : for (i = 0, msk = 0; i < nelt; i++)
23665 : {
23666 0 : j = (d->perm[i] & nelt2) ? i | nelt2 : i & ~nelt2;
23667 0 : if (dfirst.perm[j] != 0xff && dfirst.perm[j] != d->perm[i])
23668 : return false;
23669 0 : dfirst.perm[j] = d->perm[i];
23670 0 : if (j != i)
23671 0 : msk |= (1 << i);
23672 : }
23673 0 : for (i = 0; i < nelt; i++)
23674 0 : if (dfirst.perm[i] == 0xff)
23675 0 : dfirst.perm[i] = i;
23676 :
23677 0 : if (!d->testing_p)
23678 0 : dfirst.target = gen_reg_rtx (dfirst.vmode);
23679 :
23680 0 : start_sequence ();
23681 0 : ok = expand_vec_perm_1 (&dfirst);
23682 0 : seq = end_sequence ();
23683 :
23684 0 : if (!ok)
23685 : return false;
23686 :
23687 0 : if (d->testing_p)
23688 : return true;
23689 :
23690 0 : emit_insn (seq);
23691 :
23692 0 : dsecond = *d;
23693 0 : dsecond.op0 = dfirst.target;
23694 0 : dsecond.op1 = dfirst.target;
23695 0 : dsecond.one_operand_p = true;
23696 0 : dsecond.target = gen_reg_rtx (dsecond.vmode);
23697 0 : for (i = 0; i < nelt; i++)
23698 0 : dsecond.perm[i] = i ^ nelt2;
23699 :
23700 0 : ok = expand_vec_perm_1 (&dsecond);
23701 0 : gcc_assert (ok);
23702 :
23703 0 : blend = d->vmode == V8SFmode ? gen_avx_blendps256 : gen_avx_blendpd256;
23704 0 : emit_insn (blend (d->target, dfirst.target, dsecond.target, GEN_INT (msk)));
23705 0 : return true;
23706 : }
23707 :
23708 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement
23709 : a two vector permutation using two single vector permutations and
23710 : {,v}{,p}unpckl{ps,pd,bw,wd,dq}. If two_insn, succeed only if one
23711 : of dfirst or dsecond is identity permutation. */
23712 :
23713 : static bool
23714 116768 : expand_vec_perm_2perm_interleave (struct expand_vec_perm_d *d, bool two_insn)
23715 : {
23716 116768 : unsigned i, nelt = d->nelt, nelt2 = nelt / 2, lane = nelt;
23717 116768 : struct expand_vec_perm_d dfirst, dsecond, dfinal;
23718 116768 : bool ident1 = true, ident2 = true;
23719 :
23720 116768 : if (d->one_operand_p)
23721 : return false;
23722 :
23723 211738 : if (GET_MODE_SIZE (d->vmode) == 16)
23724 : {
23725 63700 : if (!TARGET_SSE)
23726 : return false;
23727 63700 : if (d->vmode != V4SFmode && d->vmode != V2DFmode && !TARGET_SSE2)
23728 : return false;
23729 : }
23730 84338 : else if (GET_MODE_SIZE (d->vmode) == 32)
23731 : {
23732 7291 : if (!TARGET_AVX)
23733 : return false;
23734 7291 : if (d->vmode != V8SFmode && d->vmode != V4DFmode && !TARGET_AVX2)
23735 : return false;
23736 : lane = nelt2;
23737 : }
23738 : else
23739 : return false;
23740 :
23741 231856 : for (i = 1; i < nelt; i++)
23742 198459 : if ((d->perm[i] >= nelt) != ((d->perm[0] >= nelt) ^ (i & 1)))
23743 : return false;
23744 :
23745 33397 : dfirst = *d;
23746 33397 : dsecond = *d;
23747 33397 : dfinal = *d;
23748 33397 : dfirst.op1 = dfirst.op0;
23749 33397 : dfirst.one_operand_p = true;
23750 33397 : dsecond.op0 = dsecond.op1;
23751 33397 : dsecond.one_operand_p = true;
23752 :
23753 218505 : for (i = 0; i < nelt; i++)
23754 185108 : if (d->perm[i] >= nelt)
23755 : {
23756 92554 : dsecond.perm[i / 2 + (i >= lane ? lane / 2 : 0)] = d->perm[i] - nelt;
23757 92554 : if (d->perm[i] - nelt != i / 2 + (i >= lane ? lane / 2 : 0))
23758 84218 : ident2 = false;
23759 92554 : dsecond.perm[i / 2 + (i >= lane ? lane : lane / 2)]
23760 92554 : = d->perm[i] - nelt;
23761 : }
23762 : else
23763 : {
23764 92554 : dfirst.perm[i / 2 + (i >= lane ? lane / 2 : 0)] = d->perm[i];
23765 92554 : if (d->perm[i] != i / 2 + (i >= lane ? lane / 2 : 0))
23766 75819 : ident1 = false;
23767 92554 : dfirst.perm[i / 2 + (i >= lane ? lane : lane / 2)] = d->perm[i];
23768 : }
23769 :
23770 33397 : if (two_insn && !ident1 && !ident2)
23771 : return false;
23772 :
23773 3899 : if (!d->testing_p)
23774 : {
23775 216 : if (!ident1)
23776 146 : dfinal.op0 = dfirst.target = gen_reg_rtx (d->vmode);
23777 216 : if (!ident2)
23778 148 : dfinal.op1 = dsecond.target = gen_reg_rtx (d->vmode);
23779 216 : if (d->perm[0] >= nelt)
23780 0 : std::swap (dfinal.op0, dfinal.op1);
23781 : }
23782 :
23783 3899 : bool ok;
23784 3899 : rtx_insn *seq1 = NULL, *seq2 = NULL;
23785 :
23786 3899 : if (!ident1)
23787 : {
23788 2587 : start_sequence ();
23789 2587 : ok = expand_vec_perm_1 (&dfirst);
23790 2587 : seq1 = end_sequence ();
23791 :
23792 2587 : if (!ok)
23793 : return false;
23794 : }
23795 :
23796 2174 : if (!ident2)
23797 : {
23798 2074 : start_sequence ();
23799 2074 : ok = expand_vec_perm_1 (&dsecond);
23800 2074 : seq2 = end_sequence ();
23801 :
23802 2074 : if (!ok)
23803 : return false;
23804 : }
23805 :
23806 608 : if (d->testing_p)
23807 : return true;
23808 :
23809 690 : for (i = 0; i < nelt; i++)
23810 : {
23811 552 : dfinal.perm[i] = i / 2;
23812 552 : if (i >= lane)
23813 4 : dfinal.perm[i] += lane / 2;
23814 552 : if ((i & 1) != 0)
23815 276 : dfinal.perm[i] += nelt;
23816 : }
23817 138 : emit_insn (seq1);
23818 138 : emit_insn (seq2);
23819 138 : ok = expand_vselect_vconcat (dfinal.target, dfinal.op0, dfinal.op1,
23820 : dfinal.perm, dfinal.nelt, false);
23821 138 : gcc_assert (ok);
23822 : return true;
23823 : }
23824 :
23825 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
23826 : the permutation using two single vector permutations and the SSE4_1 pblendv
23827 : instruction. If two_insn, succeed only if one of dfirst or dsecond is
23828 : identity permutation. */
23829 :
23830 : static bool
23831 116160 : expand_vec_perm_2perm_pblendv (struct expand_vec_perm_d *d, bool two_insn)
23832 : {
23833 116160 : unsigned i, nelt = d->nelt;
23834 116160 : struct expand_vec_perm_d dfirst, dsecond, dfinal;
23835 116160 : machine_mode vmode = d->vmode;
23836 116160 : bool ident1 = true, ident2 = true;
23837 :
23838 : /* Use the same checks as in expand_vec_perm_blend. */
23839 116160 : if (d->one_operand_p)
23840 : return false;
23841 109185 : if (TARGET_AVX2 && GET_MODE_SIZE (vmode) == 32)
23842 : ;
23843 103283 : else if (TARGET_AVX && (vmode == V4DFmode || vmode == V8SFmode))
23844 : ;
23845 98586 : else if (TARGET_SSE4_1
23846 108335 : && (GET_MODE_SIZE (vmode) == 16
23847 8804 : || (TARGET_MMX_WITH_SSE && GET_MODE_SIZE (vmode) == 8)
23848 2629 : || GET_MODE_SIZE (vmode) == 4))
23849 : ;
23850 : else
23851 : return false;
23852 :
23853 15736 : dfirst = *d;
23854 15736 : dsecond = *d;
23855 15736 : dfinal = *d;
23856 15736 : dfirst.op1 = dfirst.op0;
23857 15736 : dfirst.one_operand_p = true;
23858 15736 : dsecond.op0 = dsecond.op1;
23859 15736 : dsecond.one_operand_p = true;
23860 :
23861 116958 : for (i = 0; i < nelt; ++i)
23862 101222 : if (d->perm[i] >= nelt)
23863 : {
23864 51081 : dfirst.perm[i] = 0xff;
23865 51081 : dsecond.perm[i] = d->perm[i] - nelt;
23866 51081 : if (d->perm[i] != i + nelt)
23867 101222 : ident2 = false;
23868 : }
23869 : else
23870 : {
23871 50141 : dsecond.perm[i] = 0xff;
23872 50141 : dfirst.perm[i] = d->perm[i];
23873 50141 : if (d->perm[i] != i)
23874 101222 : ident1 = false;
23875 : }
23876 :
23877 15736 : if (two_insn && !ident1 && !ident2)
23878 : return false;
23879 :
23880 : /* For now. Ideally treat 0xff as a wildcard. */
23881 44279 : for (i = 0; i < nelt; ++i)
23882 38880 : if (dfirst.perm[i] == 0xff)
23883 : {
23884 20709 : if (GET_MODE_SIZE (vmode) == 32
23885 20709 : && dfirst.perm[i ^ (nelt / 2)] != 0xff)
23886 11731 : dfirst.perm[i] = dfirst.perm[i ^ (nelt / 2)] ^ (nelt / 2);
23887 : else
23888 8978 : dfirst.perm[i] = i;
23889 : }
23890 : else
23891 : {
23892 18171 : if (GET_MODE_SIZE (vmode) == 32
23893 18171 : && dsecond.perm[i ^ (nelt / 2)] != 0xff)
23894 9986 : dsecond.perm[i] = dsecond.perm[i ^ (nelt / 2)] ^ (nelt / 2);
23895 : else
23896 8185 : dsecond.perm[i] = i;
23897 : }
23898 :
23899 5399 : if (!d->testing_p)
23900 : {
23901 2167 : if (!ident1)
23902 2043 : dfinal.op0 = dfirst.target = gen_reg_rtx (d->vmode);
23903 2167 : if (!ident2)
23904 855 : dfinal.op1 = dsecond.target = gen_reg_rtx (d->vmode);
23905 : }
23906 :
23907 5399 : bool ok;
23908 5399 : rtx_insn *seq1 = NULL, *seq2 = NULL;
23909 :
23910 5399 : if (!ident1)
23911 : {
23912 4810 : start_sequence ();
23913 4810 : ok = expand_vec_perm_1 (&dfirst);
23914 4810 : seq1 = end_sequence ();
23915 :
23916 4810 : if (!ok)
23917 : return false;
23918 : }
23919 :
23920 4012 : if (!ident2)
23921 : {
23922 1133 : start_sequence ();
23923 1133 : ok = expand_vec_perm_1 (&dsecond);
23924 1133 : seq2 = end_sequence ();
23925 :
23926 1133 : if (!ok)
23927 : return false;
23928 : }
23929 :
23930 3423 : if (d->testing_p)
23931 : return true;
23932 :
23933 14037 : for (i = 0; i < nelt; ++i)
23934 12212 : dfinal.perm[i] = (d->perm[i] >= nelt ? i + nelt : i);
23935 :
23936 1825 : emit_insn (seq1);
23937 1825 : emit_insn (seq2);
23938 1825 : ok = expand_vec_perm_blend (&dfinal);
23939 1825 : gcc_assert (ok);
23940 : return true;
23941 : }
23942 :
23943 : /* A subroutine of ix86_expand_vec_perm_const_1.
23944 : Implement a permutation with psrlw, psllw and por.
23945 : It handles case:
23946 : __builtin_shufflevector (v,v,1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14);
23947 : __builtin_shufflevector (v,v,1,0,3,2,5,4,7,6); */
23948 :
23949 : static bool
23950 27030 : expand_vec_perm_psrlw_psllw_por (struct expand_vec_perm_d *d)
23951 : {
23952 27030 : unsigned i;
23953 27030 : rtx (*gen_shr) (rtx, rtx, rtx);
23954 27030 : rtx (*gen_shl) (rtx, rtx, rtx);
23955 27030 : rtx (*gen_or) (rtx, rtx, rtx);
23956 27030 : machine_mode mode = VOIDmode;
23957 :
23958 27030 : if (!TARGET_SSE2 || !d->one_operand_p)
23959 : return false;
23960 :
23961 5428 : switch (d->vmode)
23962 : {
23963 1438 : case E_V8QImode:
23964 1438 : if (!TARGET_MMX_WITH_SSE)
23965 : return false;
23966 : mode = V4HImode;
23967 : gen_shr = gen_lshrv4hi3;
23968 : gen_shl = gen_ashlv4hi3;
23969 : gen_or = gen_iorv4hi3;
23970 : break;
23971 : case E_V16QImode:
23972 : mode = V8HImode;
23973 : gen_shr = gen_lshrv8hi3;
23974 : gen_shl = gen_ashlv8hi3;
23975 : gen_or = gen_iorv8hi3;
23976 : break;
23977 : default: return false;
23978 : }
23979 :
23980 3212 : if (!rtx_equal_p (d->op0, d->op1))
23981 : return false;
23982 :
23983 12380 : for (i = 0; i < d->nelt; i += 2)
23984 10926 : if (d->perm[i] != i + 1 || d->perm[i + 1] != i)
23985 : return false;
23986 :
23987 1454 : if (d->testing_p)
23988 : return true;
23989 :
23990 30 : rtx tmp1 = gen_reg_rtx (mode);
23991 30 : rtx tmp2 = gen_reg_rtx (mode);
23992 30 : rtx op0 = force_reg (d->vmode, d->op0);
23993 :
23994 30 : emit_move_insn (tmp1, lowpart_subreg (mode, op0, d->vmode));
23995 30 : emit_move_insn (tmp2, lowpart_subreg (mode, op0, d->vmode));
23996 30 : emit_insn (gen_shr (tmp1, tmp1, GEN_INT (8)));
23997 30 : emit_insn (gen_shl (tmp2, tmp2, GEN_INT (8)));
23998 30 : emit_insn (gen_or (tmp1, tmp1, tmp2));
23999 30 : emit_move_insn (d->target, lowpart_subreg (d->vmode, tmp1, mode));
24000 :
24001 30 : return true;
24002 : }
24003 :
24004 : /* A subroutine of ix86_expand_vec_perm_const_1. Implement a V4DF
24005 : permutation using two vperm2f128, followed by a vshufpd insn blending
24006 : the two vectors together. */
24007 :
24008 : static bool
24009 30880 : expand_vec_perm_2vperm2f128_vshuf (struct expand_vec_perm_d *d)
24010 : {
24011 30880 : struct expand_vec_perm_d dfirst, dsecond, dthird;
24012 30880 : bool ok;
24013 :
24014 30880 : if (!TARGET_AVX || (d->vmode != V4DFmode))
24015 : return false;
24016 :
24017 1277 : if (d->testing_p)
24018 : return true;
24019 :
24020 206 : dfirst = *d;
24021 206 : dsecond = *d;
24022 206 : dthird = *d;
24023 :
24024 206 : dfirst.perm[0] = (d->perm[0] & ~1);
24025 206 : dfirst.perm[1] = (d->perm[0] & ~1) + 1;
24026 206 : dfirst.perm[2] = (d->perm[2] & ~1);
24027 206 : dfirst.perm[3] = (d->perm[2] & ~1) + 1;
24028 206 : dsecond.perm[0] = (d->perm[1] & ~1);
24029 206 : dsecond.perm[1] = (d->perm[1] & ~1) + 1;
24030 206 : dsecond.perm[2] = (d->perm[3] & ~1);
24031 206 : dsecond.perm[3] = (d->perm[3] & ~1) + 1;
24032 206 : dthird.perm[0] = (d->perm[0] % 2);
24033 206 : dthird.perm[1] = (d->perm[1] % 2) + 4;
24034 206 : dthird.perm[2] = (d->perm[2] % 2) + 2;
24035 206 : dthird.perm[3] = (d->perm[3] % 2) + 6;
24036 :
24037 206 : dfirst.target = gen_reg_rtx (dfirst.vmode);
24038 206 : dsecond.target = gen_reg_rtx (dsecond.vmode);
24039 206 : dthird.op0 = dfirst.target;
24040 206 : dthird.op1 = dsecond.target;
24041 206 : dthird.one_operand_p = false;
24042 :
24043 206 : canonicalize_perm (&dfirst);
24044 206 : canonicalize_perm (&dsecond);
24045 :
24046 206 : ok = expand_vec_perm_1 (&dfirst)
24047 206 : && expand_vec_perm_1 (&dsecond)
24048 412 : && expand_vec_perm_1 (&dthird);
24049 :
24050 0 : gcc_assert (ok);
24051 :
24052 : return true;
24053 : }
24054 :
24055 : static bool ix86_expand_vec_perm_const_1 (struct expand_vec_perm_d *);
24056 :
24057 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement
24058 : a two vector permutation using two intra-lane vector
24059 : permutations, vperm2f128 swapping the lanes and vblend* insn blending
24060 : the non-swapped and swapped vectors together. */
24061 :
24062 : static bool
24063 16131 : expand_vec_perm2_vperm2f128_vblend (struct expand_vec_perm_d *d)
24064 : {
24065 16131 : struct expand_vec_perm_d dfirst, dsecond, dthird;
24066 16131 : unsigned i, j, msk, nelt = d->nelt, nelt2 = nelt / 2, which1 = 0, which2 = 0;
24067 16131 : rtx_insn *seq1, *seq2;
24068 16131 : bool ok;
24069 16131 : rtx (*blend) (rtx, rtx, rtx, rtx) = NULL;
24070 :
24071 16131 : if (!TARGET_AVX
24072 794 : || TARGET_AVX2
24073 530 : || (d->vmode != V8SFmode && d->vmode != V4DFmode)
24074 403 : || d->one_operand_p)
24075 : return false;
24076 :
24077 403 : dfirst = *d;
24078 403 : dsecond = *d;
24079 3627 : for (i = 0; i < nelt; i++)
24080 : {
24081 3224 : dfirst.perm[i] = 0xff;
24082 3224 : dsecond.perm[i] = 0xff;
24083 : }
24084 3627 : for (i = 0, msk = 0; i < nelt; i++)
24085 : {
24086 3224 : j = (d->perm[i] & nelt2) ? i | nelt2 : i & ~nelt2;
24087 3224 : if (j == i)
24088 : {
24089 2498 : dfirst.perm[j] = d->perm[i];
24090 4322 : which1 |= (d->perm[i] < nelt ? 1 : 2);
24091 : }
24092 : else
24093 : {
24094 726 : dsecond.perm[j] = d->perm[i];
24095 726 : which2 |= (d->perm[i] < nelt ? 1 : 2);
24096 726 : msk |= (1U << i);
24097 : }
24098 : }
24099 403 : if (msk == 0 || msk == (1U << nelt) - 1)
24100 : return false;
24101 :
24102 403 : if (!d->testing_p)
24103 : {
24104 40 : dfirst.target = gen_reg_rtx (dfirst.vmode);
24105 40 : dsecond.target = gen_reg_rtx (dsecond.vmode);
24106 : }
24107 :
24108 3627 : for (i = 0; i < nelt; i++)
24109 : {
24110 3224 : if (dfirst.perm[i] == 0xff)
24111 726 : dfirst.perm[i] = (which1 == 2 ? i + nelt : i);
24112 3224 : if (dsecond.perm[i] == 0xff)
24113 2498 : dsecond.perm[i] = (which2 == 2 ? i + nelt : i);
24114 : }
24115 403 : canonicalize_perm (&dfirst);
24116 403 : start_sequence ();
24117 403 : ok = ix86_expand_vec_perm_const_1 (&dfirst);
24118 403 : seq1 = end_sequence ();
24119 :
24120 403 : if (!ok)
24121 : return false;
24122 :
24123 403 : canonicalize_perm (&dsecond);
24124 403 : start_sequence ();
24125 403 : ok = ix86_expand_vec_perm_const_1 (&dsecond);
24126 403 : seq2 = end_sequence ();
24127 :
24128 403 : if (!ok)
24129 : return false;
24130 :
24131 403 : if (d->testing_p)
24132 : return true;
24133 :
24134 40 : emit_insn (seq1);
24135 40 : emit_insn (seq2);
24136 :
24137 40 : dthird = *d;
24138 40 : dthird.op0 = dsecond.target;
24139 40 : dthird.op1 = dsecond.target;
24140 40 : dthird.one_operand_p = true;
24141 40 : dthird.target = gen_reg_rtx (dthird.vmode);
24142 360 : for (i = 0; i < nelt; i++)
24143 320 : dthird.perm[i] = i ^ nelt2;
24144 :
24145 40 : ok = expand_vec_perm_1 (&dthird);
24146 40 : gcc_assert (ok);
24147 :
24148 40 : blend = d->vmode == V8SFmode ? gen_avx_blendps256 : gen_avx_blendpd256;
24149 40 : emit_insn (blend (d->target, dfirst.target, dthird.target, GEN_INT (msk)));
24150 40 : return true;
24151 : }
24152 :
24153 : /* A subroutine of expand_vec_perm_even_odd_1. Implement the double-word
24154 : permutation with two pshufb insns and an ior. We should have already
24155 : failed all two instruction sequences. */
24156 :
24157 : static bool
24158 29624 : expand_vec_perm_pshufb2 (struct expand_vec_perm_d *d)
24159 : {
24160 29624 : rtx rperm[2][16], vperm, l, h, op, m128;
24161 29624 : unsigned int i, nelt, eltsz;
24162 29624 : machine_mode mode;
24163 29624 : rtx (*gen) (rtx, rtx, rtx);
24164 :
24165 34112 : if (!TARGET_SSSE3 || (GET_MODE_SIZE (d->vmode) != 16
24166 8886 : && GET_MODE_SIZE (d->vmode) != 8
24167 8846 : && GET_MODE_SIZE (d->vmode) != 4))
24168 : return false;
24169 1409 : gcc_assert (!d->one_operand_p);
24170 :
24171 1409 : if (d->testing_p)
24172 : return true;
24173 :
24174 202 : switch (GET_MODE_SIZE (d->vmode))
24175 : {
24176 : case 4:
24177 : mode = V4QImode;
24178 : gen = gen_mmx_pshufbv4qi3;
24179 : break;
24180 20 : case 8:
24181 20 : mode = V8QImode;
24182 20 : gen = gen_mmx_pshufbv8qi3;
24183 20 : break;
24184 45 : case 16:
24185 45 : mode = V16QImode;
24186 45 : gen = gen_ssse3_pshufbv16qi3;
24187 45 : break;
24188 0 : default:
24189 0 : gcc_unreachable ();
24190 : }
24191 :
24192 101 : nelt = d->nelt;
24193 101 : eltsz = GET_MODE_UNIT_SIZE (d->vmode);
24194 :
24195 : /* Generate two permutation masks. If the required element is within
24196 : the given vector it is shuffled into the proper lane. If the required
24197 : element is in the other vector, force a zero into the lane by setting
24198 : bit 7 in the permutation mask. */
24199 101 : m128 = GEN_INT (-128);
24200 1029 : for (i = 0; i < nelt; ++i)
24201 : {
24202 928 : unsigned j, k, e = d->perm[i];
24203 928 : unsigned which = (e >= nelt);
24204 928 : if (e >= nelt)
24205 480 : e -= nelt;
24206 :
24207 1952 : for (j = 0; j < eltsz; ++j)
24208 : {
24209 1024 : rperm[which][i*eltsz + j] = GEN_INT (e*eltsz + j);
24210 1024 : rperm[1-which][i*eltsz + j] = m128;
24211 : }
24212 :
24213 9024 : for (k = i*eltsz + j; k < 16; ++k)
24214 8096 : rperm[0][k] = rperm[1][k] = m128;
24215 : }
24216 :
24217 101 : vperm = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, rperm[0]));
24218 101 : vperm = force_reg (V16QImode, vperm);
24219 :
24220 101 : l = gen_reg_rtx (mode);
24221 101 : op = gen_lowpart (mode, d->op0);
24222 101 : emit_insn (gen (l, op, vperm));
24223 :
24224 101 : vperm = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, rperm[1]));
24225 101 : vperm = force_reg (V16QImode, vperm);
24226 :
24227 101 : h = gen_reg_rtx (mode);
24228 101 : op = gen_lowpart (mode, d->op1);
24229 101 : emit_insn (gen (h, op, vperm));
24230 :
24231 101 : op = d->target;
24232 101 : if (d->vmode != mode)
24233 22 : op = gen_reg_rtx (mode);
24234 101 : ix86_emit_vec_binop (IOR, mode, op, l, h);
24235 101 : if (op != d->target)
24236 22 : emit_move_insn (d->target, gen_lowpart (d->vmode, op));
24237 :
24238 : return true;
24239 : }
24240 :
24241 : /* Implement arbitrary permutation of one V32QImode and V16QImode operand
24242 : with two vpshufb insns, vpermq and vpor. We should have already failed
24243 : all two or three instruction sequences. */
24244 :
24245 : static bool
24246 24441 : expand_vec_perm_vpshufb2_vpermq (struct expand_vec_perm_d *d)
24247 : {
24248 24441 : rtx rperm[2][32], vperm, l, h, hp, op, m128;
24249 24441 : unsigned int i, nelt, eltsz;
24250 :
24251 24441 : if (!TARGET_AVX2
24252 374 : || !d->one_operand_p
24253 172 : || (d->vmode != V32QImode && d->vmode != V16HImode))
24254 : return false;
24255 :
24256 7 : if (d->testing_p)
24257 : return true;
24258 :
24259 7 : nelt = d->nelt;
24260 7 : eltsz = GET_MODE_UNIT_SIZE (d->vmode);
24261 :
24262 : /* Generate two permutation masks. If the required element is within
24263 : the same lane, it is shuffled in. If the required element from the
24264 : other lane, force a zero by setting bit 7 in the permutation mask.
24265 : In the other mask the mask has non-negative elements if element
24266 : is requested from the other lane, but also moved to the other lane,
24267 : so that the result of vpshufb can have the two V2TImode halves
24268 : swapped. */
24269 7 : m128 = GEN_INT (-128);
24270 199 : for (i = 0; i < nelt; ++i)
24271 : {
24272 192 : unsigned j, e = d->perm[i] & (nelt / 2 - 1);
24273 192 : unsigned which = ((d->perm[i] ^ i) & (nelt / 2)) * eltsz;
24274 :
24275 416 : for (j = 0; j < eltsz; ++j)
24276 : {
24277 224 : rperm[!!which][(i * eltsz + j) ^ which] = GEN_INT (e * eltsz + j);
24278 224 : rperm[!which][(i * eltsz + j) ^ (which ^ 16)] = m128;
24279 : }
24280 : }
24281 :
24282 7 : vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[1]));
24283 7 : vperm = force_reg (V32QImode, vperm);
24284 :
24285 7 : h = gen_reg_rtx (V32QImode);
24286 7 : op = gen_lowpart (V32QImode, d->op0);
24287 7 : emit_insn (gen_avx2_pshufbv32qi3 (h, op, vperm));
24288 :
24289 : /* Swap the 128-byte lanes of h into hp. */
24290 7 : hp = gen_reg_rtx (V4DImode);
24291 7 : op = gen_lowpart (V4DImode, h);
24292 7 : emit_insn (gen_avx2_permv4di_1 (hp, op, const2_rtx, GEN_INT (3), const0_rtx,
24293 : const1_rtx));
24294 :
24295 7 : vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[0]));
24296 7 : vperm = force_reg (V32QImode, vperm);
24297 :
24298 7 : l = gen_reg_rtx (V32QImode);
24299 7 : op = gen_lowpart (V32QImode, d->op0);
24300 7 : emit_insn (gen_avx2_pshufbv32qi3 (l, op, vperm));
24301 :
24302 7 : op = d->target;
24303 7 : if (d->vmode != V32QImode)
24304 2 : op = gen_reg_rtx (V32QImode);
24305 7 : emit_insn (gen_iorv32qi3 (op, l, gen_lowpart (V32QImode, hp)));
24306 7 : if (op != d->target)
24307 2 : emit_move_insn (d->target, gen_lowpart (d->vmode, op));
24308 :
24309 : return true;
24310 : }
24311 :
24312 : /* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
24313 : and extract-odd permutations of two V32QImode and V16QImode operand
24314 : with two vpshufb insns, vpor and vpermq. We should have already
24315 : failed all two or three instruction sequences. */
24316 :
24317 : static bool
24318 24434 : expand_vec_perm_vpshufb2_vpermq_even_odd (struct expand_vec_perm_d *d)
24319 : {
24320 24434 : rtx rperm[2][32], vperm, l, h, ior, op, m128;
24321 24434 : unsigned int i, nelt, eltsz;
24322 :
24323 24434 : if (!TARGET_AVX2
24324 367 : || d->one_operand_p
24325 202 : || (d->vmode != V32QImode && d->vmode != V16HImode))
24326 : return false;
24327 :
24328 112 : for (i = 0; i < d->nelt; ++i)
24329 112 : if ((d->perm[i] ^ (i * 2)) & (3 * d->nelt / 2))
24330 : return false;
24331 :
24332 0 : if (d->testing_p)
24333 : return true;
24334 :
24335 0 : nelt = d->nelt;
24336 0 : eltsz = GET_MODE_UNIT_SIZE (d->vmode);
24337 :
24338 : /* Generate two permutation masks. In the first permutation mask
24339 : the first quarter will contain indexes for the first half
24340 : of the op0, the second quarter will contain bit 7 set, third quarter
24341 : will contain indexes for the second half of the op0 and the
24342 : last quarter bit 7 set. In the second permutation mask
24343 : the first quarter will contain bit 7 set, the second quarter
24344 : indexes for the first half of the op1, the third quarter bit 7 set
24345 : and last quarter indexes for the second half of the op1.
24346 : I.e. the first mask e.g. for V32QImode extract even will be:
24347 : 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128
24348 : (all values masked with 0xf except for -128) and second mask
24349 : for extract even will be
24350 : -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe. */
24351 0 : m128 = GEN_INT (-128);
24352 0 : for (i = 0; i < nelt; ++i)
24353 : {
24354 0 : unsigned j, e = d->perm[i] & (nelt / 2 - 1);
24355 0 : unsigned which = d->perm[i] >= nelt;
24356 0 : unsigned xorv = (i >= nelt / 4 && i < 3 * nelt / 4) ? 24 : 0;
24357 :
24358 0 : for (j = 0; j < eltsz; ++j)
24359 : {
24360 0 : rperm[which][(i * eltsz + j) ^ xorv] = GEN_INT (e * eltsz + j);
24361 0 : rperm[1 - which][(i * eltsz + j) ^ xorv] = m128;
24362 : }
24363 : }
24364 :
24365 0 : vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[0]));
24366 0 : vperm = force_reg (V32QImode, vperm);
24367 :
24368 0 : l = gen_reg_rtx (V32QImode);
24369 0 : op = gen_lowpart (V32QImode, d->op0);
24370 0 : emit_insn (gen_avx2_pshufbv32qi3 (l, op, vperm));
24371 :
24372 0 : vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[1]));
24373 0 : vperm = force_reg (V32QImode, vperm);
24374 :
24375 0 : h = gen_reg_rtx (V32QImode);
24376 0 : op = gen_lowpart (V32QImode, d->op1);
24377 0 : emit_insn (gen_avx2_pshufbv32qi3 (h, op, vperm));
24378 :
24379 0 : ior = gen_reg_rtx (V32QImode);
24380 0 : emit_insn (gen_iorv32qi3 (ior, l, h));
24381 :
24382 : /* Permute the V4DImode quarters using { 0, 2, 1, 3 } permutation. */
24383 0 : op = gen_reg_rtx (V4DImode);
24384 0 : ior = gen_lowpart (V4DImode, ior);
24385 0 : emit_insn (gen_avx2_permv4di_1 (op, ior, const0_rtx, const2_rtx,
24386 : const1_rtx, GEN_INT (3)));
24387 0 : emit_move_insn (d->target, gen_lowpart (d->vmode, op));
24388 :
24389 0 : return true;
24390 : }
24391 :
24392 : /* A subroutine of ix86_expand_vec_perm_const_1. Try to implement a
24393 : permutation (which is a bland) with and, andnot and or when pshufb is not available.
24394 :
24395 : It handles case:
24396 : __builtin_shufflevector (v1, v2, 0, 9, 2, 11, 4, 13, 6, 15);
24397 : __builtin_shufflevector (v1, v2, 8, 1, 2, 11, 4, 13, 6, 15);
24398 :
24399 : An element[i] must be chosen between op0[i] and op1[i] to satisfy the
24400 : requirement.
24401 : */
24402 :
24403 : static bool
24404 25576 : expand_vec_perm_pand_pandn_por (struct expand_vec_perm_d *d)
24405 : {
24406 25576 : rtx rperm[16], vperm;
24407 25576 : unsigned int i, nelt = d->nelt;
24408 :
24409 25576 : if (!TARGET_SSE2
24410 25576 : || d->one_operand_p
24411 21602 : || (d->vmode != V16QImode && d->vmode != V8HImode))
24412 : return false;
24413 :
24414 8040 : if (d->perm[0] != 0)
24415 : return false;
24416 :
24417 : /* The dest[i] must select an element between op0[i] and op1[i]. */
24418 17326 : for (i = 1; i < nelt; i++)
24419 16203 : if ((d->perm[i] % nelt) != i)
24420 : return false;
24421 :
24422 1123 : if (d->testing_p)
24423 : return true;
24424 :
24425 : /* Generates a blend mask for the operators AND and ANDNOT. */
24426 134 : machine_mode inner_mode = GET_MODE_INNER (d->vmode);
24427 1526 : for (i = 0; i < nelt; i++)
24428 1999 : rperm[i] = (d->perm[i] < nelt) ? CONSTM1_RTX (inner_mode)
24429 607 : : CONST0_RTX (inner_mode);
24430 :
24431 134 : vperm = gen_rtx_CONST_VECTOR (d->vmode, gen_rtvec_v (nelt, rperm));
24432 134 : vperm = force_reg (d->vmode, vperm);
24433 :
24434 134 : ix86_expand_sse_movcc (d->target, vperm, d->op0, d->op1);
24435 :
24436 134 : return true;
24437 : }
24438 :
24439 : /* Implement permutation with pslldq + psrldq + por when pshufb is not
24440 : available. */
24441 : static bool
24442 44736 : expand_vec_perm_pslldq_psrldq_por (struct expand_vec_perm_d *d, bool pandn)
24443 : {
24444 44736 : unsigned i, nelt = d->nelt;
24445 44736 : unsigned start1, end1 = -1;
24446 44736 : machine_mode vmode = d->vmode, imode;
24447 44736 : int start2 = -1;
24448 44736 : bool clear_op0, clear_op1;
24449 44736 : unsigned inner_size;
24450 44736 : rtx op0, op1, dop1;
24451 44736 : rtx (*gen_vec_shr) (rtx, rtx, rtx);
24452 44736 : rtx (*gen_vec_shl) (rtx, rtx, rtx);
24453 :
24454 : /* pshufd can be used for V4SI/V2DI under TARGET_SSE2. */
24455 44736 : if (!TARGET_SSE2 || (vmode != E_V16QImode && vmode != E_V8HImode))
24456 : return false;
24457 :
24458 14411 : start1 = d->perm[0];
24459 43298 : for (i = 1; i < nelt; i++)
24460 : {
24461 42353 : if (d->perm[i] != d->perm[i-1] + 1
24462 14766 : || d->perm[i] == nelt)
24463 : {
24464 27877 : if (start2 == -1)
24465 : {
24466 14411 : start2 = d->perm[i];
24467 14411 : end1 = d->perm[i-1];
24468 : }
24469 : else
24470 : return false;
24471 : }
24472 : }
24473 :
24474 945 : clear_op0 = end1 != nelt - 1;
24475 945 : clear_op1 = start2 % nelt != 0;
24476 : /* pandn/pand is needed to clear upper/lower bits of op0/op1. */
24477 945 : if (!pandn && (clear_op0 || clear_op1))
24478 : return false;
24479 :
24480 591 : if (d->testing_p)
24481 : return true;
24482 :
24483 89 : gen_vec_shr = vmode == E_V16QImode ? gen_vec_shr_v16qi : gen_vec_shr_v8hi;
24484 28 : gen_vec_shl = vmode == E_V16QImode ? gen_vec_shl_v16qi : gen_vec_shl_v8hi;
24485 89 : imode = GET_MODE_INNER (vmode);
24486 89 : inner_size = GET_MODE_BITSIZE (imode);
24487 89 : op0 = gen_reg_rtx (vmode);
24488 89 : op1 = gen_reg_rtx (vmode);
24489 :
24490 89 : if (start1)
24491 84 : emit_insn (gen_vec_shr (op0, d->op0, GEN_INT (start1 * inner_size)));
24492 : else
24493 5 : emit_move_insn (op0, d->op0);
24494 :
24495 89 : dop1 = d->op1;
24496 89 : if (d->one_operand_p)
24497 64 : dop1 = d->op0;
24498 :
24499 89 : int shl_offset = end1 - start1 + 1 - start2 % nelt;
24500 89 : if (shl_offset)
24501 49 : emit_insn (gen_vec_shl (op1, dop1, GEN_INT (shl_offset * inner_size)));
24502 : else
24503 40 : emit_move_insn (op1, dop1);
24504 :
24505 : /* Clear lower/upper bits for op0/op1. */
24506 89 : if (clear_op0 || clear_op1)
24507 : {
24508 : rtx vec[16];
24509 : rtx const_vec;
24510 : rtx clear;
24511 989 : for (i = 0; i != nelt; i++)
24512 : {
24513 920 : if (i < (end1 - start1 + 1))
24514 346 : vec[i] = gen_int_mode ((HOST_WIDE_INT_1U << inner_size) - 1, imode);
24515 : else
24516 574 : vec[i] = CONST0_RTX (imode);
24517 : }
24518 69 : const_vec = gen_rtx_CONST_VECTOR (vmode, gen_rtvec_v (nelt, vec));
24519 69 : const_vec = validize_mem (force_const_mem (vmode, const_vec));
24520 69 : clear = force_reg (vmode, const_vec);
24521 :
24522 69 : if (clear_op0)
24523 61 : emit_move_insn (op0, gen_rtx_AND (vmode, op0, clear));
24524 69 : if (clear_op1)
24525 56 : emit_move_insn (op1, gen_rtx_AND (vmode,
24526 : gen_rtx_NOT (vmode, clear),
24527 : op1));
24528 : }
24529 :
24530 89 : emit_move_insn (d->target, gen_rtx_IOR (vmode, op0, op1));
24531 89 : return true;
24532 : }
24533 :
24534 : /* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
24535 : and extract-odd permutations of two V8QI, V8HI, V16QI, V16HI or V32QI
24536 : operands with two "and" and "pack" or two "shift" and "pack" insns.
24537 : We should have already failed all two instruction sequences. */
24538 :
24539 : static bool
24540 47109 : expand_vec_perm_even_odd_pack (struct expand_vec_perm_d *d)
24541 : {
24542 47109 : rtx op, dop0, dop1, t;
24543 47109 : unsigned i, odd, c, s, nelt = d->nelt;
24544 47109 : int pblendw_i = 0;
24545 47109 : bool end_perm = false;
24546 47109 : machine_mode half_mode;
24547 47109 : rtx (*gen_and) (rtx, rtx, rtx);
24548 47109 : rtx (*gen_pack) (rtx, rtx, rtx);
24549 47109 : rtx (*gen_shift) (rtx, rtx, rtx);
24550 :
24551 47109 : if (d->one_operand_p)
24552 : return false;
24553 :
24554 41638 : switch (d->vmode)
24555 : {
24556 4628 : case E_V4HImode:
24557 : /* Required for "pack". */
24558 4628 : if (!TARGET_SSE4_1)
24559 : return false;
24560 : c = 0xffff;
24561 : s = 16;
24562 : half_mode = V2SImode;
24563 : gen_and = gen_andv2si3;
24564 : gen_pack = gen_mmx_packusdw;
24565 : gen_shift = gen_lshrv2si3;
24566 : pblendw_i = 0x5;
24567 : break;
24568 5953 : case E_V8HImode:
24569 : /* Required for "pack". */
24570 5953 : if (!TARGET_SSE4_1)
24571 : return false;
24572 : c = 0xffff;
24573 : s = 16;
24574 : half_mode = V4SImode;
24575 : gen_and = gen_andv4si3;
24576 : gen_pack = gen_sse4_1_packusdw;
24577 : gen_shift = gen_lshrv4si3;
24578 : pblendw_i = 0x55;
24579 : break;
24580 : case E_V8QImode:
24581 : /* No check as all instructions are SSE2. */
24582 : c = 0xff;
24583 : s = 8;
24584 : half_mode = V4HImode;
24585 : gen_and = gen_andv4hi3;
24586 : gen_pack = gen_mmx_packuswb;
24587 : gen_shift = gen_lshrv4hi3;
24588 : break;
24589 14630 : case E_V16QImode:
24590 : /* No check as all instructions are SSE2. */
24591 14630 : c = 0xff;
24592 14630 : s = 8;
24593 14630 : half_mode = V8HImode;
24594 14630 : gen_and = gen_andv8hi3;
24595 14630 : gen_pack = gen_sse2_packuswb;
24596 14630 : gen_shift = gen_lshrv8hi3;
24597 14630 : break;
24598 440 : case E_V16HImode:
24599 440 : if (!TARGET_AVX2)
24600 : return false;
24601 : c = 0xffff;
24602 : s = 16;
24603 : half_mode = V8SImode;
24604 : gen_and = gen_andv8si3;
24605 : gen_pack = gen_avx2_packusdw;
24606 : gen_shift = gen_lshrv8si3;
24607 : pblendw_i = 0x5555;
24608 : end_perm = true;
24609 : break;
24610 276 : case E_V32QImode:
24611 276 : if (!TARGET_AVX2)
24612 : return false;
24613 : c = 0xff;
24614 : s = 8;
24615 : half_mode = V16HImode;
24616 : gen_and = gen_andv16hi3;
24617 : gen_pack = gen_avx2_packuswb;
24618 : gen_shift = gen_lshrv16hi3;
24619 : end_perm = true;
24620 : break;
24621 : default:
24622 : /* Only V4HI, V8QI, V8HI, V16QI, V16HI and V32QI modes
24623 : are more profitable than general shuffles. */
24624 : return false;
24625 : }
24626 :
24627 : /* Check that permutation is even or odd. */
24628 20461 : odd = d->perm[0];
24629 20461 : if (odd > 1)
24630 : return false;
24631 :
24632 231488 : for (i = 1; i < nelt; ++i)
24633 215319 : if (d->perm[i] != 2 * i + odd)
24634 : return false;
24635 :
24636 16169 : if (d->testing_p)
24637 : return true;
24638 :
24639 5553 : dop0 = gen_reg_rtx (half_mode);
24640 5553 : dop1 = gen_reg_rtx (half_mode);
24641 5553 : if (odd == 0)
24642 : {
24643 : /* Use pblendw since const_vector 0 should be cheaper than
24644 : const_vector 0xffff. */
24645 4832 : if (d->vmode == V4HImode
24646 : || d->vmode == E_V8HImode
24647 : || d->vmode == E_V16HImode)
24648 : {
24649 872 : rtx dop0_t = gen_reg_rtx (d->vmode);
24650 872 : rtx dop1_t = gen_reg_rtx (d->vmode);
24651 872 : t = gen_reg_rtx (d->vmode);
24652 872 : emit_move_insn (t, CONST0_RTX (d->vmode));
24653 :
24654 872 : emit_move_insn (dop0_t, gen_rtx_VEC_MERGE (d->vmode, d->op0, t,
24655 : GEN_INT (pblendw_i)));
24656 872 : emit_move_insn (dop1_t, gen_rtx_VEC_MERGE (d->vmode, d->op1, t,
24657 : GEN_INT (pblendw_i)));
24658 :
24659 872 : emit_move_insn (dop0, gen_lowpart (half_mode, dop0_t));
24660 872 : emit_move_insn (dop1, gen_lowpart (half_mode, dop1_t));
24661 872 : }
24662 : else
24663 : {
24664 3960 : t = gen_const_vec_duplicate (half_mode, GEN_INT (c));
24665 3960 : t = force_reg (half_mode, t);
24666 3960 : emit_insn (gen_and (dop0, t, gen_lowpart (half_mode, d->op0)));
24667 3960 : emit_insn (gen_and (dop1, t, gen_lowpart (half_mode, d->op1)));
24668 : }
24669 : }
24670 : else
24671 : {
24672 1442 : emit_insn (gen_shift (dop0,
24673 721 : gen_lowpart (half_mode, d->op0),
24674 : GEN_INT (s)));
24675 1442 : emit_insn (gen_shift (dop1,
24676 721 : gen_lowpart (half_mode, d->op1),
24677 : GEN_INT (s)));
24678 : }
24679 : /* In AVX2 for 256 bit case we need to permute pack result. */
24680 5553 : if (TARGET_AVX2 && end_perm)
24681 : {
24682 419 : op = gen_reg_rtx (d->vmode);
24683 419 : t = gen_reg_rtx (V4DImode);
24684 419 : emit_insn (gen_pack (op, dop0, dop1));
24685 838 : emit_insn (gen_avx2_permv4di_1 (t,
24686 419 : gen_lowpart (V4DImode, op),
24687 : const0_rtx,
24688 : const2_rtx,
24689 : const1_rtx,
24690 : GEN_INT (3)));
24691 419 : emit_move_insn (d->target, gen_lowpart (d->vmode, t));
24692 : }
24693 : else
24694 5134 : emit_insn (gen_pack (d->target, dop0, dop1));
24695 :
24696 : return true;
24697 : }
24698 :
24699 : /* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
24700 : and extract-odd permutations of two V64QI operands
24701 : with two "shifts", two "truncs" and one "concat" insns for "odd"
24702 : and two "truncs" and one concat insn for "even."
24703 : Have already failed all two instruction sequences. */
24704 :
24705 : static bool
24706 24478 : expand_vec_perm_even_odd_trunc (struct expand_vec_perm_d *d)
24707 : {
24708 24478 : rtx t1, t2, t3, t4;
24709 24478 : unsigned i, odd, nelt = d->nelt;
24710 :
24711 24478 : if (!TARGET_AVX512BW
24712 74 : || d->one_operand_p
24713 38 : || d->vmode != V64QImode)
24714 : return false;
24715 :
24716 : /* Check that permutation is even or odd. */
24717 38 : odd = d->perm[0];
24718 38 : if (odd > 1)
24719 : return false;
24720 :
24721 1662 : for (i = 1; i < nelt; ++i)
24722 1637 : if (d->perm[i] != 2 * i + odd)
24723 : return false;
24724 :
24725 25 : if (d->testing_p)
24726 : return true;
24727 :
24728 :
24729 25 : if (odd)
24730 : {
24731 5 : t1 = gen_reg_rtx (V32HImode);
24732 5 : t2 = gen_reg_rtx (V32HImode);
24733 10 : emit_insn (gen_lshrv32hi3 (t1,
24734 5 : gen_lowpart (V32HImode, d->op0),
24735 : GEN_INT (8)));
24736 10 : emit_insn (gen_lshrv32hi3 (t2,
24737 5 : gen_lowpart (V32HImode, d->op1),
24738 : GEN_INT (8)));
24739 : }
24740 : else
24741 : {
24742 20 : t1 = gen_lowpart (V32HImode, d->op0);
24743 20 : t2 = gen_lowpart (V32HImode, d->op1);
24744 : }
24745 :
24746 25 : t3 = gen_reg_rtx (V32QImode);
24747 25 : t4 = gen_reg_rtx (V32QImode);
24748 25 : emit_insn (gen_avx512bw_truncatev32hiv32qi2 (t3, t1));
24749 25 : emit_insn (gen_avx512bw_truncatev32hiv32qi2 (t4, t2));
24750 25 : emit_insn (gen_avx_vec_concatv64qi (d->target, t3, t4));
24751 :
24752 25 : return true;
24753 : }
24754 :
24755 : /* A subroutine of ix86_expand_vec_perm_const_1. Implement extract-even
24756 : and extract-odd permutations. */
24757 :
24758 : static bool
24759 13021 : expand_vec_perm_even_odd_1 (struct expand_vec_perm_d *d, unsigned odd)
24760 : {
24761 13021 : rtx t1, t2, t3, t4, t5;
24762 :
24763 13021 : switch (d->vmode)
24764 : {
24765 19 : case E_V4DFmode:
24766 19 : if (d->testing_p)
24767 : break;
24768 1 : t1 = gen_reg_rtx (V4DFmode);
24769 1 : t2 = gen_reg_rtx (V4DFmode);
24770 :
24771 : /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
24772 1 : emit_insn (gen_avx_vperm2f128v4df3 (t1, d->op0, d->op1, GEN_INT (0x20)));
24773 1 : emit_insn (gen_avx_vperm2f128v4df3 (t2, d->op0, d->op1, GEN_INT (0x31)));
24774 :
24775 : /* Now an unpck[lh]pd will produce the result required. */
24776 1 : if (odd)
24777 0 : t3 = gen_avx_unpckhpd256 (d->target, t1, t2);
24778 : else
24779 1 : t3 = gen_avx_unpcklpd256 (d->target, t1, t2);
24780 1 : emit_insn (t3);
24781 1 : break;
24782 :
24783 1214 : case E_V8SFmode:
24784 1214 : {
24785 1214 : int mask = odd ? 0xdd : 0x88;
24786 :
24787 1214 : if (d->testing_p)
24788 : break;
24789 186 : t1 = gen_reg_rtx (V8SFmode);
24790 186 : t2 = gen_reg_rtx (V8SFmode);
24791 186 : t3 = gen_reg_rtx (V8SFmode);
24792 :
24793 : /* Shuffle within the 128-bit lanes to produce:
24794 : { 0 2 8 a 4 6 c e } | { 1 3 9 b 5 7 d f }. */
24795 186 : emit_insn (gen_avx_shufps256 (t1, d->op0, d->op1,
24796 : GEN_INT (mask)));
24797 :
24798 : /* Shuffle the lanes around to produce:
24799 : { 4 6 c e 0 2 8 a } and { 5 7 d f 1 3 9 b }. */
24800 186 : emit_insn (gen_avx_vperm2f128v8sf3 (t2, t1, t1,
24801 : GEN_INT (0x3)));
24802 :
24803 : /* Shuffle within the 128-bit lanes to produce:
24804 : { 0 2 4 6 4 6 0 2 } | { 1 3 5 7 5 7 1 3 }. */
24805 186 : emit_insn (gen_avx_shufps256 (t3, t1, t2, GEN_INT (0x44)));
24806 :
24807 : /* Shuffle within the 128-bit lanes to produce:
24808 : { 8 a c e c e 8 a } | { 9 b d f d f 9 b }. */
24809 186 : emit_insn (gen_avx_shufps256 (t2, t1, t2, GEN_INT (0xee)));
24810 :
24811 : /* Shuffle the lanes around to produce:
24812 : { 0 2 4 6 8 a c e } | { 1 3 5 7 9 b d f }. */
24813 186 : emit_insn (gen_avx_vperm2f128v8sf3 (d->target, t3, t2,
24814 : GEN_INT (0x20)));
24815 : }
24816 186 : break;
24817 :
24818 0 : case E_V2DFmode:
24819 0 : case E_V4SFmode:
24820 0 : case E_V2DImode:
24821 0 : case E_V2SImode:
24822 0 : case E_V4SImode:
24823 0 : case E_V2HImode:
24824 : /* These are always directly implementable by expand_vec_perm_1. */
24825 0 : gcc_unreachable ();
24826 :
24827 0 : case E_V2SFmode:
24828 0 : gcc_assert (TARGET_MMX_WITH_SSE);
24829 : /* We have no suitable instructions. */
24830 0 : if (d->testing_p)
24831 : return false;
24832 : break;
24833 :
24834 1548 : case E_V4QImode:
24835 1548 : if (TARGET_SSSE3 && !TARGET_SLOW_PSHUFB)
24836 0 : return expand_vec_perm_pshufb2 (d);
24837 : else
24838 : {
24839 1548 : if (d->testing_p)
24840 : break;
24841 : /* We need 2*log2(N)-1 operations to achieve odd/even
24842 : with interleave. */
24843 178 : t1 = gen_reg_rtx (V4QImode);
24844 178 : emit_insn (gen_mmx_punpckhbw_low (t1, d->op0, d->op1));
24845 178 : emit_insn (gen_mmx_punpcklbw_low (d->target, d->op0, d->op1));
24846 178 : if (odd)
24847 41 : t2 = gen_mmx_punpckhbw_low (d->target, d->target, t1);
24848 : else
24849 137 : t2 = gen_mmx_punpcklbw_low (d->target, d->target, t1);
24850 178 : emit_insn (t2);
24851 : }
24852 178 : break;
24853 :
24854 1526 : case E_V4HImode:
24855 1526 : if (TARGET_SSE4_1)
24856 92 : return expand_vec_perm_even_odd_pack (d);
24857 1434 : else if (TARGET_SSSE3 && !TARGET_SLOW_PSHUFB)
24858 20 : return expand_vec_perm_pshufb2 (d);
24859 : else
24860 : {
24861 1414 : if (d->testing_p)
24862 : break;
24863 : /* We need 2*log2(N)-1 operations to achieve odd/even
24864 : with interleave. */
24865 453 : t1 = gen_reg_rtx (V4HImode);
24866 453 : emit_insn (gen_mmx_punpckhwd (t1, d->op0, d->op1));
24867 453 : emit_insn (gen_mmx_punpcklwd (d->target, d->op0, d->op1));
24868 453 : if (odd)
24869 8 : t2 = gen_mmx_punpckhwd (d->target, d->target, t1);
24870 : else
24871 445 : t2 = gen_mmx_punpcklwd (d->target, d->target, t1);
24872 453 : emit_insn (t2);
24873 : }
24874 453 : break;
24875 :
24876 6684 : case E_V8HImode:
24877 6684 : if (TARGET_SSE4_1)
24878 440 : return expand_vec_perm_even_odd_pack (d);
24879 6244 : else if (TARGET_SSSE3 && !TARGET_SLOW_PSHUFB)
24880 1 : return expand_vec_perm_pshufb2 (d);
24881 : else
24882 : {
24883 6243 : if (d->testing_p)
24884 : break;
24885 : /* We need 2*log2(N)-1 operations to achieve odd/even
24886 : with interleave. */
24887 2747 : t1 = gen_reg_rtx (V8HImode);
24888 2747 : t2 = gen_reg_rtx (V8HImode);
24889 2747 : emit_insn (gen_vec_interleave_highv8hi (t1, d->op0, d->op1));
24890 2747 : emit_insn (gen_vec_interleave_lowv8hi (d->target, d->op0, d->op1));
24891 2747 : emit_insn (gen_vec_interleave_highv8hi (t2, d->target, t1));
24892 2747 : emit_insn (gen_vec_interleave_lowv8hi (d->target, d->target, t1));
24893 2747 : if (odd)
24894 91 : t3 = gen_vec_interleave_highv8hi (d->target, d->target, t2);
24895 : else
24896 2656 : t3 = gen_vec_interleave_lowv8hi (d->target, d->target, t2);
24897 2747 : emit_insn (t3);
24898 : }
24899 2747 : break;
24900 :
24901 1343 : case E_V8QImode:
24902 1343 : case E_V16QImode:
24903 1343 : return expand_vec_perm_even_odd_pack (d);
24904 :
24905 467 : case E_V16HImode:
24906 467 : case E_V32QImode:
24907 467 : return expand_vec_perm_even_odd_pack (d);
24908 :
24909 25 : case E_V64QImode:
24910 25 : return expand_vec_perm_even_odd_trunc (d);
24911 :
24912 19 : case E_V4DImode:
24913 19 : if (!TARGET_AVX2)
24914 : {
24915 19 : struct expand_vec_perm_d d_copy = *d;
24916 19 : d_copy.vmode = V4DFmode;
24917 19 : if (d->testing_p)
24918 18 : d_copy.target = gen_raw_REG (V4DFmode, LAST_VIRTUAL_REGISTER + 1);
24919 : else
24920 1 : d_copy.target = gen_reg_rtx (V4DFmode);
24921 19 : d_copy.op0 = gen_lowpart (V4DFmode, d->op0);
24922 19 : d_copy.op1 = gen_lowpart (V4DFmode, d->op1);
24923 19 : if (expand_vec_perm_even_odd_1 (&d_copy, odd))
24924 : {
24925 19 : if (!d->testing_p)
24926 1 : emit_move_insn (d->target,
24927 1 : gen_lowpart (V4DImode, d_copy.target));
24928 19 : return true;
24929 : }
24930 : return false;
24931 : }
24932 :
24933 0 : if (d->testing_p)
24934 : break;
24935 :
24936 0 : t1 = gen_reg_rtx (V4DImode);
24937 0 : t2 = gen_reg_rtx (V4DImode);
24938 :
24939 : /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
24940 0 : emit_insn (gen_avx2_permv2ti (t1, d->op0, d->op1, GEN_INT (0x20)));
24941 0 : emit_insn (gen_avx2_permv2ti (t2, d->op0, d->op1, GEN_INT (0x31)));
24942 :
24943 : /* Now an vpunpck[lh]qdq will produce the result required. */
24944 0 : if (odd)
24945 0 : t3 = gen_avx2_interleave_highv4di (d->target, t1, t2);
24946 : else
24947 0 : t3 = gen_avx2_interleave_lowv4di (d->target, t1, t2);
24948 0 : emit_insn (t3);
24949 0 : break;
24950 :
24951 176 : case E_V8SImode:
24952 176 : if (!TARGET_AVX2)
24953 : {
24954 38 : struct expand_vec_perm_d d_copy = *d;
24955 38 : d_copy.vmode = V8SFmode;
24956 38 : if (d->testing_p)
24957 38 : d_copy.target = gen_raw_REG (V8SFmode, LAST_VIRTUAL_REGISTER + 1);
24958 : else
24959 0 : d_copy.target = gen_reg_rtx (V8SFmode);
24960 38 : d_copy.op0 = gen_lowpart (V8SFmode, d->op0);
24961 38 : d_copy.op1 = gen_lowpart (V8SFmode, d->op1);
24962 38 : if (expand_vec_perm_even_odd_1 (&d_copy, odd))
24963 : {
24964 38 : if (!d->testing_p)
24965 0 : emit_move_insn (d->target,
24966 0 : gen_lowpart (V8SImode, d_copy.target));
24967 38 : return true;
24968 : }
24969 : return false;
24970 : }
24971 :
24972 138 : if (d->testing_p)
24973 : break;
24974 :
24975 138 : t1 = gen_reg_rtx (V8SImode);
24976 138 : t2 = gen_reg_rtx (V8SImode);
24977 138 : t3 = gen_reg_rtx (V4DImode);
24978 138 : t4 = gen_reg_rtx (V4DImode);
24979 138 : t5 = gen_reg_rtx (V4DImode);
24980 :
24981 : /* Shuffle the lanes around into
24982 : { 0 1 2 3 8 9 a b } and { 4 5 6 7 c d e f }. */
24983 276 : emit_insn (gen_avx2_permv2ti (t3, gen_lowpart (V4DImode, d->op0),
24984 138 : gen_lowpart (V4DImode, d->op1),
24985 : GEN_INT (0x20)));
24986 276 : emit_insn (gen_avx2_permv2ti (t4, gen_lowpart (V4DImode, d->op0),
24987 138 : gen_lowpart (V4DImode, d->op1),
24988 : GEN_INT (0x31)));
24989 :
24990 : /* Swap the 2nd and 3rd position in each lane into
24991 : { 0 2 1 3 8 a 9 b } and { 4 6 5 7 c e d f }. */
24992 138 : emit_insn (gen_avx2_pshufdv3 (t1, gen_lowpart (V8SImode, t3),
24993 : GEN_INT (2 * 4 + 1 * 16 + 3 * 64)));
24994 138 : emit_insn (gen_avx2_pshufdv3 (t2, gen_lowpart (V8SImode, t4),
24995 : GEN_INT (2 * 4 + 1 * 16 + 3 * 64)));
24996 :
24997 : /* Now an vpunpck[lh]qdq will produce
24998 : { 0 2 4 6 8 a c e } resp. { 1 3 5 7 9 b d f }. */
24999 138 : if (odd)
25000 0 : t3 = gen_avx2_interleave_highv4di (t5, gen_lowpart (V4DImode, t1),
25001 0 : gen_lowpart (V4DImode, t2));
25002 : else
25003 138 : t3 = gen_avx2_interleave_lowv4di (t5, gen_lowpart (V4DImode, t1),
25004 138 : gen_lowpart (V4DImode, t2));
25005 138 : emit_insn (t3);
25006 138 : emit_move_insn (d->target, gen_lowpart (V8SImode, t5));
25007 138 : break;
25008 :
25009 0 : default:
25010 0 : gcc_unreachable ();
25011 : }
25012 :
25013 : return true;
25014 : }
25015 :
25016 : /* A subroutine of ix86_expand_vec_perm_const_1. Pattern match
25017 : extract-even and extract-odd permutations. */
25018 :
25019 : static bool
25020 24385 : expand_vec_perm_even_odd (struct expand_vec_perm_d *d)
25021 : {
25022 24385 : unsigned i, odd, nelt = d->nelt;
25023 :
25024 24385 : odd = d->perm[0];
25025 24385 : if (odd != 0 && odd != 1)
25026 : return false;
25027 :
25028 65549 : for (i = 1; i < nelt; ++i)
25029 57601 : if (d->perm[i] != 2 * i + odd)
25030 : return false;
25031 :
25032 7948 : if (d->vmode == E_V32HImode
25033 12 : && d->testing_p
25034 12 : && !TARGET_AVX512BW)
25035 : return false;
25036 :
25037 7936 : return expand_vec_perm_even_odd_1 (d, odd);
25038 : }
25039 :
25040 : /* A subroutine of ix86_expand_vec_perm_const_1. Implement broadcast
25041 : permutations. We assume that expand_vec_perm_1 has already failed. */
25042 :
25043 : static bool
25044 1080 : expand_vec_perm_broadcast_1 (struct expand_vec_perm_d *d)
25045 : {
25046 1080 : unsigned elt = d->perm[0], nelt2 = d->nelt / 2;
25047 1080 : machine_mode vmode = d->vmode;
25048 1080 : rtx (*gen) (rtx, rtx, rtx);
25049 1080 : unsigned char perm2[4];
25050 1080 : rtx op0 = d->op0, dest;
25051 1080 : bool ok;
25052 :
25053 1080 : switch (vmode)
25054 : {
25055 0 : case E_V4DFmode:
25056 0 : case E_V8SFmode:
25057 : /* These are special-cased in sse.md so that we can optionally
25058 : use the vbroadcast instruction. They expand to two insns
25059 : if the input happens to be in a register. */
25060 0 : gcc_unreachable ();
25061 :
25062 0 : case E_V2DFmode:
25063 0 : case E_V2SFmode:
25064 0 : case E_V4SFmode:
25065 0 : case E_V2DImode:
25066 0 : case E_V2SImode:
25067 0 : case E_V4SImode:
25068 0 : case E_V2HImode:
25069 0 : case E_V4HImode:
25070 : /* These are always implementable using standard shuffle patterns. */
25071 0 : gcc_unreachable ();
25072 :
25073 16 : case E_V4QImode:
25074 : /* This can be implemented via interleave and pshuflw. */
25075 16 : if (d->testing_p)
25076 : return true;
25077 :
25078 8 : if (elt >= nelt2)
25079 : {
25080 4 : gen = gen_mmx_punpckhbw_low;
25081 4 : elt -= nelt2;
25082 : }
25083 : else
25084 : gen = gen_mmx_punpcklbw_low;
25085 :
25086 8 : dest = gen_reg_rtx (vmode);
25087 8 : emit_insn (gen (dest, op0, op0));
25088 8 : vmode = get_mode_wider_vector (vmode);
25089 8 : op0 = gen_lowpart (vmode, dest);
25090 :
25091 8 : memset (perm2, elt, 2);
25092 8 : dest = gen_reg_rtx (vmode);
25093 8 : ok = expand_vselect (dest, op0, perm2, 2, d->testing_p);
25094 8 : gcc_assert (ok);
25095 :
25096 8 : emit_move_insn (d->target, gen_lowpart (d->vmode, dest));
25097 8 : return true;
25098 :
25099 4 : case E_V8QImode:
25100 : /* This can be implemented via interleave. We save one insn by
25101 : stopping once we have promoted to V2SImode and then use pshufd. */
25102 4 : if (d->testing_p)
25103 : return true;
25104 4 : do
25105 : {
25106 4 : if (elt >= nelt2)
25107 : {
25108 1 : gen = vmode == V8QImode ? gen_mmx_punpckhbw
25109 : : gen_mmx_punpckhwd;
25110 1 : elt -= nelt2;
25111 : }
25112 : else
25113 3 : gen = vmode == V8QImode ? gen_mmx_punpcklbw
25114 : : gen_mmx_punpcklwd;
25115 4 : nelt2 /= 2;
25116 :
25117 4 : dest = gen_reg_rtx (vmode);
25118 4 : emit_insn (gen (dest, op0, op0));
25119 4 : vmode = get_mode_wider_vector (vmode);
25120 4 : op0 = gen_lowpart (vmode, dest);
25121 : }
25122 4 : while (vmode != V2SImode);
25123 :
25124 2 : memset (perm2, elt, 2);
25125 2 : dest = gen_reg_rtx (vmode);
25126 2 : ok = expand_vselect (dest, op0, perm2, 2, d->testing_p);
25127 2 : gcc_assert (ok);
25128 :
25129 2 : emit_move_insn (d->target, gen_lowpart (d->vmode, dest));
25130 2 : return true;
25131 :
25132 1051 : case E_V8HImode:
25133 1051 : case E_V16QImode:
25134 : /* These can be implemented via interleave. We save one insn by
25135 : stopping once we have promoted to V4SImode and then use pshufd. */
25136 1051 : if (d->testing_p)
25137 : return true;
25138 1612 : do
25139 : {
25140 1612 : if (elt >= nelt2)
25141 : {
25142 16 : gen = vmode == V16QImode ? gen_vec_interleave_highv16qi
25143 : : gen_vec_interleave_highv8hi;
25144 16 : elt -= nelt2;
25145 : }
25146 : else
25147 1596 : gen = vmode == V16QImode ? gen_vec_interleave_lowv16qi
25148 : : gen_vec_interleave_lowv8hi;
25149 1612 : nelt2 /= 2;
25150 :
25151 1612 : dest = gen_reg_rtx (vmode);
25152 1612 : emit_insn (gen (dest, op0, op0));
25153 1612 : vmode = get_mode_wider_vector (vmode);
25154 1612 : op0 = gen_lowpart (vmode, dest);
25155 : }
25156 1612 : while (vmode != V4SImode);
25157 :
25158 987 : memset (perm2, elt, 4);
25159 987 : dest = gen_reg_rtx (vmode);
25160 987 : ok = expand_vselect (dest, op0, perm2, 4, d->testing_p);
25161 987 : gcc_assert (ok);
25162 :
25163 987 : emit_move_insn (d->target, gen_lowpart (d->vmode, dest));
25164 987 : return true;
25165 :
25166 1 : case E_V8HFmode:
25167 1 : case E_V8BFmode:
25168 : /* This can be implemented via interleave and pshufd. */
25169 1 : if (d->testing_p)
25170 : return true;
25171 :
25172 1 : rtx (*gen_interleave) (machine_mode, rtx, rtx, rtx);
25173 1 : if (elt >= nelt2)
25174 : {
25175 0 : gen_interleave = gen_vec_interleave_high;
25176 0 : elt -= nelt2;
25177 : }
25178 : else
25179 : gen_interleave = gen_vec_interleave_low;
25180 1 : nelt2 /= 2;
25181 :
25182 1 : dest = gen_reg_rtx (vmode);
25183 1 : emit_insn (gen_interleave (vmode, dest, op0, op0));
25184 :
25185 1 : vmode = V4SImode;
25186 1 : op0 = gen_lowpart (vmode, dest);
25187 :
25188 1 : memset (perm2, elt, 4);
25189 1 : dest = gen_reg_rtx (vmode);
25190 1 : ok = expand_vselect (dest, op0, perm2, 4, d->testing_p);
25191 1 : gcc_assert (ok);
25192 :
25193 1 : emit_move_insn (d->target, gen_lowpart (d->vmode, dest));
25194 1 : return true;
25195 :
25196 0 : case E_V32QImode:
25197 0 : case E_V16HImode:
25198 0 : case E_V8SImode:
25199 0 : case E_V4DImode:
25200 : /* For AVX2 broadcasts of the first element vpbroadcast* or
25201 : vpermq should be used by expand_vec_perm_1. */
25202 0 : gcc_assert (!TARGET_AVX2 || d->perm[0]);
25203 : return false;
25204 :
25205 6 : case E_V64QImode:
25206 6 : gcc_assert (!TARGET_AVX512BW || d->perm[0]);
25207 : return false;
25208 :
25209 2 : case E_V32HImode:
25210 2 : gcc_assert (!TARGET_AVX512BW);
25211 : return false;
25212 :
25213 0 : default:
25214 0 : gcc_unreachable ();
25215 : }
25216 : }
25217 :
25218 : /* A subroutine of ix86_expand_vec_perm_const_1. Pattern match
25219 : broadcast permutations. */
25220 :
25221 : static bool
25222 90785 : expand_vec_perm_broadcast (struct expand_vec_perm_d *d)
25223 : {
25224 90785 : unsigned i, elt, nelt = d->nelt;
25225 :
25226 90785 : if (!d->one_operand_p)
25227 : return false;
25228 :
25229 5575 : elt = d->perm[0];
25230 8146 : for (i = 1; i < nelt; ++i)
25231 8038 : if (d->perm[i] != elt)
25232 : return false;
25233 :
25234 108 : return expand_vec_perm_broadcast_1 (d);
25235 : }
25236 :
25237 : /* Implement arbitrary permutations of two V64QImode operands
25238 : with 2 vperm[it]2w, 2 vpshufb and one vpor instruction. */
25239 : static bool
25240 24434 : expand_vec_perm_vpermt2_vpshub2 (struct expand_vec_perm_d *d)
25241 : {
25242 24434 : if (!TARGET_AVX512BW || !(d->vmode == V64QImode))
25243 : return false;
25244 :
25245 49 : if (d->testing_p)
25246 : return true;
25247 :
25248 49 : struct expand_vec_perm_d ds[2];
25249 49 : rtx rperm[128], vperm, target0, target1;
25250 49 : unsigned int i, nelt;
25251 49 : machine_mode vmode;
25252 :
25253 49 : nelt = d->nelt;
25254 49 : vmode = V64QImode;
25255 :
25256 147 : for (i = 0; i < 2; i++)
25257 : {
25258 98 : ds[i] = *d;
25259 98 : ds[i].vmode = V32HImode;
25260 98 : ds[i].nelt = 32;
25261 98 : ds[i].target = gen_reg_rtx (V32HImode);
25262 98 : ds[i].op0 = gen_lowpart (V32HImode, d->op0);
25263 98 : ds[i].op1 = gen_lowpart (V32HImode, d->op1);
25264 : }
25265 :
25266 : /* Prepare permutations such that the first one takes care of
25267 : putting the even bytes into the right positions or one higher
25268 : positions (ds[0]) and the second one takes care of
25269 : putting the odd bytes into the right positions or one below
25270 : (ds[1]). */
25271 :
25272 3185 : for (i = 0; i < nelt; i++)
25273 : {
25274 3136 : ds[i & 1].perm[i / 2] = d->perm[i] / 2;
25275 3136 : if (i & 1)
25276 : {
25277 1568 : rperm[i] = constm1_rtx;
25278 1568 : rperm[i + 64] = GEN_INT ((i & 14) + (d->perm[i] & 1));
25279 : }
25280 : else
25281 : {
25282 1568 : rperm[i] = GEN_INT ((i & 14) + (d->perm[i] & 1));
25283 1568 : rperm[i + 64] = constm1_rtx;
25284 : }
25285 : }
25286 :
25287 49 : bool ok = expand_vec_perm_1 (&ds[0]);
25288 49 : gcc_assert (ok);
25289 49 : ds[0].target = gen_lowpart (V64QImode, ds[0].target);
25290 :
25291 49 : ok = expand_vec_perm_1 (&ds[1]);
25292 49 : gcc_assert (ok);
25293 49 : ds[1].target = gen_lowpart (V64QImode, ds[1].target);
25294 :
25295 49 : vperm = gen_rtx_CONST_VECTOR (V64QImode, gen_rtvec_v (64, rperm));
25296 49 : vperm = force_reg (vmode, vperm);
25297 49 : target0 = gen_reg_rtx (V64QImode);
25298 49 : emit_insn (gen_avx512bw_pshufbv64qi3 (target0, ds[0].target, vperm));
25299 :
25300 49 : vperm = gen_rtx_CONST_VECTOR (V64QImode, gen_rtvec_v (64, rperm + 64));
25301 49 : vperm = force_reg (vmode, vperm);
25302 49 : target1 = gen_reg_rtx (V64QImode);
25303 49 : emit_insn (gen_avx512bw_pshufbv64qi3 (target1, ds[1].target, vperm));
25304 :
25305 49 : emit_insn (gen_iorv64qi3 (d->target, target0, target1));
25306 49 : return true;
25307 : }
25308 :
25309 : /* Implement arbitrary permutation of two V32QImode and V16QImode operands
25310 : with 4 vpshufb insns, 2 vpermq and 3 vpor. We should have already failed
25311 : all the shorter instruction sequences. */
25312 :
25313 : static bool
25314 16185 : expand_vec_perm_vpshufb4_vpermq2 (struct expand_vec_perm_d *d)
25315 : {
25316 16185 : rtx rperm[4][32], vperm, l[2], h[2], op, m128;
25317 16185 : unsigned int i, nelt, eltsz;
25318 16185 : bool used[4];
25319 :
25320 16185 : if (!TARGET_AVX2
25321 318 : || d->one_operand_p
25322 189 : || (d->vmode != V32QImode && d->vmode != V16HImode))
25323 : return false;
25324 :
25325 54 : if (d->testing_p)
25326 : return true;
25327 :
25328 54 : nelt = d->nelt;
25329 54 : eltsz = GET_MODE_UNIT_SIZE (d->vmode);
25330 :
25331 : /* Generate 4 permutation masks. If the required element is within
25332 : the same lane, it is shuffled in. If the required element from the
25333 : other lane, force a zero by setting bit 7 in the permutation mask.
25334 : In the other mask the mask has non-negative elements if element
25335 : is requested from the other lane, but also moved to the other lane,
25336 : so that the result of vpshufb can have the two V2TImode halves
25337 : swapped. */
25338 54 : m128 = GEN_INT (-128);
25339 1836 : for (i = 0; i < 32; ++i)
25340 : {
25341 1728 : rperm[0][i] = m128;
25342 1728 : rperm[1][i] = m128;
25343 1728 : rperm[2][i] = m128;
25344 1728 : rperm[3][i] = m128;
25345 : }
25346 54 : used[0] = false;
25347 54 : used[1] = false;
25348 54 : used[2] = false;
25349 54 : used[3] = false;
25350 1590 : for (i = 0; i < nelt; ++i)
25351 : {
25352 1536 : unsigned j, e = d->perm[i] & (nelt / 2 - 1);
25353 1536 : unsigned xlane = ((d->perm[i] ^ i) & (nelt / 2)) * eltsz;
25354 2074 : unsigned int which = ((d->perm[i] & nelt) ? 2 : 0) + (xlane ? 1 : 0);
25355 :
25356 3264 : for (j = 0; j < eltsz; ++j)
25357 1728 : rperm[which][(i * eltsz + j) ^ xlane] = GEN_INT (e * eltsz + j);
25358 1536 : used[which] = true;
25359 : }
25360 :
25361 162 : for (i = 0; i < 2; ++i)
25362 : {
25363 108 : if (!used[2 * i + 1])
25364 : {
25365 22 : h[i] = NULL_RTX;
25366 22 : continue;
25367 : }
25368 86 : vperm = gen_rtx_CONST_VECTOR (V32QImode,
25369 86 : gen_rtvec_v (32, rperm[2 * i + 1]));
25370 86 : vperm = force_reg (V32QImode, vperm);
25371 86 : h[i] = gen_reg_rtx (V32QImode);
25372 86 : op = gen_lowpart (V32QImode, i ? d->op1 : d->op0);
25373 86 : emit_insn (gen_avx2_pshufbv32qi3 (h[i], op, vperm));
25374 : }
25375 :
25376 : /* Swap the 128-byte lanes of h[X]. */
25377 162 : for (i = 0; i < 2; ++i)
25378 : {
25379 108 : if (h[i] == NULL_RTX)
25380 22 : continue;
25381 86 : op = gen_reg_rtx (V4DImode);
25382 86 : emit_insn (gen_avx2_permv4di_1 (op, gen_lowpart (V4DImode, h[i]),
25383 : const2_rtx, GEN_INT (3), const0_rtx,
25384 : const1_rtx));
25385 86 : h[i] = gen_lowpart (V32QImode, op);
25386 : }
25387 :
25388 162 : for (i = 0; i < 2; ++i)
25389 : {
25390 108 : if (!used[2 * i])
25391 : {
25392 0 : l[i] = NULL_RTX;
25393 0 : continue;
25394 : }
25395 108 : vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[2 * i]));
25396 108 : vperm = force_reg (V32QImode, vperm);
25397 108 : l[i] = gen_reg_rtx (V32QImode);
25398 108 : op = gen_lowpart (V32QImode, i ? d->op1 : d->op0);
25399 108 : emit_insn (gen_avx2_pshufbv32qi3 (l[i], op, vperm));
25400 : }
25401 :
25402 162 : for (i = 0; i < 2; ++i)
25403 : {
25404 108 : if (h[i] && l[i])
25405 : {
25406 86 : op = gen_reg_rtx (V32QImode);
25407 86 : emit_insn (gen_iorv32qi3 (op, l[i], h[i]));
25408 86 : l[i] = op;
25409 : }
25410 22 : else if (h[i])
25411 0 : l[i] = h[i];
25412 : }
25413 :
25414 54 : gcc_assert (l[0] && l[1]);
25415 54 : op = d->target;
25416 54 : if (d->vmode != V32QImode)
25417 12 : op = gen_reg_rtx (V32QImode);
25418 54 : emit_insn (gen_iorv32qi3 (op, l[0], l[1]));
25419 54 : if (op != d->target)
25420 12 : emit_move_insn (d->target, gen_lowpart (d->vmode, op));
25421 : return true;
25422 : }
25423 :
25424 : /* The guts of ix86_vectorize_vec_perm_const. With all of the interface bits
25425 : taken care of, perform the expansion in D and return true on success. */
25426 :
25427 : static bool
25428 334231 : ix86_expand_vec_perm_const_1 (struct expand_vec_perm_d *d)
25429 : {
25430 : /* Try a single instruction expansion. */
25431 334231 : if (expand_vec_perm_1 (d))
25432 : return true;
25433 :
25434 : /* Try sequences of two instructions. */
25435 :
25436 102322 : if (expand_vec_perm_pshuflw_pshufhw (d))
25437 : return true;
25438 :
25439 99859 : if (expand_vec_perm_palignr (d, false))
25440 : return true;
25441 :
25442 96724 : if (expand_vec_perm_interleave2 (d))
25443 : return true;
25444 :
25445 90785 : if (expand_vec_perm_broadcast (d))
25446 : return true;
25447 :
25448 90685 : if (expand_vec_perm_vpermq_perm_1 (d))
25449 : return true;
25450 :
25451 90685 : if (expand_vec_perm_vperm2f128 (d))
25452 : return true;
25453 :
25454 90617 : if (expand_vec_perm_pblendv (d))
25455 : return true;
25456 :
25457 88954 : if (expand_vec_perm_2perm_interleave (d, true))
25458 : return true;
25459 :
25460 88586 : if (expand_vec_perm_2perm_pblendv (d, true))
25461 : return true;
25462 :
25463 85707 : if (expand_vec_perm_shufps_shufps (d))
25464 : return true;
25465 :
25466 50503 : if (expand_vec_perm_punpckldq_pshuf (d))
25467 : return true;
25468 :
25469 : /* Try sequences of three instructions. */
25470 :
25471 44767 : if (expand_vec_perm_even_odd_pack (d))
25472 : return true;
25473 :
25474 30880 : if (expand_vec_perm_2vperm2f128_vshuf (d))
25475 : return true;
25476 :
25477 29603 : if (expand_vec_perm_pshufb2 (d))
25478 : return true;
25479 :
25480 28215 : if (expand_vec_perm_pslldq_psrldq_por (d, false))
25481 : return true;
25482 :
25483 27960 : if (expand_vec_perm_interleave3 (d))
25484 : return true;
25485 :
25486 27814 : if (expand_vec_perm_vperm2f128_vblend (d))
25487 : return true;
25488 :
25489 27814 : if (expand_vec_perm_2perm_interleave (d, false))
25490 : return true;
25491 :
25492 27574 : if (expand_vec_perm_2perm_pblendv (d, false))
25493 : return true;
25494 :
25495 27030 : if (expand_vec_perm_psrlw_psllw_por (d))
25496 : return true;
25497 :
25498 25576 : if (expand_vec_perm_pand_pandn_por (d))
25499 : return true;
25500 :
25501 : /* Try sequences of four instructions. */
25502 :
25503 24453 : if (expand_vec_perm_even_odd_trunc (d))
25504 : return true;
25505 24441 : if (expand_vec_perm_vpshufb2_vpermq (d))
25506 : return true;
25507 :
25508 24434 : if (expand_vec_perm_vpshufb2_vpermq_even_odd (d))
25509 : return true;
25510 :
25511 24434 : if (expand_vec_perm_vpermt2_vpshub2 (d))
25512 : return true;
25513 :
25514 : /* ??? Look for narrow permutations whose element orderings would
25515 : allow the promotion to a wider mode. */
25516 :
25517 : /* ??? Look for sequences of interleave or a wider permute that place
25518 : the data into the correct lanes for a half-vector shuffle like
25519 : pshuf[lh]w or vpermilps. */
25520 :
25521 : /* ??? Look for sequences of interleave that produce the desired results.
25522 : The combinatorics of punpck[lh] get pretty ugly... */
25523 :
25524 24385 : if (expand_vec_perm_even_odd (d))
25525 : return true;
25526 :
25527 : /* Generate four or five instructions. */
25528 16521 : if (expand_vec_perm_pslldq_psrldq_por (d, true))
25529 : return true;
25530 :
25531 : /* Even longer sequences. */
25532 16185 : if (expand_vec_perm_vpshufb4_vpermq2 (d))
25533 : return true;
25534 :
25535 : /* See if we can get the same permutation in different vector integer
25536 : mode. */
25537 16131 : struct expand_vec_perm_d nd;
25538 16131 : if (canonicalize_vector_int_perm (d, &nd) && expand_vec_perm_1 (&nd))
25539 : {
25540 0 : if (!d->testing_p)
25541 0 : emit_move_insn (d->target, gen_lowpart (d->vmode, nd.target));
25542 0 : return true;
25543 : }
25544 :
25545 : /* Even longer, including recursion to ix86_expand_vec_perm_const_1. */
25546 16131 : if (expand_vec_perm2_vperm2f128_vblend (d))
25547 : return true;
25548 :
25549 : return false;
25550 : }
25551 :
25552 : /* If a permutation only uses one operand, make it clear. Returns true
25553 : if the permutation references both operands. */
25554 :
25555 : static bool
25556 74931 : canonicalize_perm (struct expand_vec_perm_d *d)
25557 : {
25558 74931 : int i, which, nelt = d->nelt;
25559 :
25560 449593 : for (i = which = 0; i < nelt; ++i)
25561 509229 : which |= (d->perm[i] < nelt ? 1 : 2);
25562 :
25563 74931 : d->one_operand_p = true;
25564 74931 : switch (which)
25565 : {
25566 0 : default:
25567 0 : gcc_unreachable();
25568 :
25569 55784 : case 3:
25570 55784 : if (!rtx_equal_p (d->op0, d->op1))
25571 : {
25572 55755 : d->one_operand_p = false;
25573 55755 : break;
25574 : }
25575 : /* The elements of PERM do not suggest that only the first operand
25576 : is used, but both operands are identical. Allow easier matching
25577 : of the permutation by folding the permutation into the single
25578 : input vector. */
25579 : /* FALLTHRU */
25580 :
25581 : case 2:
25582 2635 : for (i = 0; i < nelt; ++i)
25583 2320 : d->perm[i] &= nelt - 1;
25584 315 : d->op0 = d->op1;
25585 315 : break;
25586 :
25587 18861 : case 1:
25588 18861 : d->op1 = d->op0;
25589 18861 : break;
25590 : }
25591 :
25592 74931 : return (which == 3);
25593 : }
25594 :
25595 : /* Implement TARGET_VECTORIZE_VEC_PERM_CONST. */
25596 :
25597 : bool
25598 818142 : ix86_vectorize_vec_perm_const (machine_mode vmode, machine_mode op_mode,
25599 : rtx target, rtx op0, rtx op1,
25600 : const vec_perm_indices &sel)
25601 : {
25602 818142 : if (vmode != op_mode)
25603 : return false;
25604 :
25605 816283 : struct expand_vec_perm_d d;
25606 816283 : unsigned char perm[MAX_VECT_LEN];
25607 816283 : unsigned int i, nelt, which;
25608 816283 : bool two_args;
25609 :
25610 : /* For HF and BF mode vector, convert it to HI using subreg. */
25611 2446363 : if (GET_MODE_INNER (vmode) == HFmode || GET_MODE_INNER (vmode) == BFmode)
25612 : {
25613 2512 : machine_mode orig_mode = vmode;
25614 5024 : vmode = mode_for_vector (HImode,
25615 2512 : GET_MODE_NUNITS (vmode)).require ();
25616 2512 : if (target)
25617 437 : target = lowpart_subreg (vmode, target, orig_mode);
25618 2512 : if (op0)
25619 437 : op0 = lowpart_subreg (vmode, op0, orig_mode);
25620 2512 : if (op1)
25621 437 : op1 = lowpart_subreg (vmode, op1, orig_mode);
25622 : }
25623 :
25624 816283 : d.target = target;
25625 816283 : d.op0 = op0;
25626 816283 : d.op1 = op1;
25627 :
25628 816283 : d.vmode = vmode;
25629 816283 : gcc_assert (VECTOR_MODE_P (d.vmode));
25630 816283 : d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
25631 816283 : d.testing_p = !target;
25632 :
25633 816283 : gcc_assert (sel.length () == nelt);
25634 816283 : gcc_checking_assert (sizeof (d.perm) == sizeof (perm));
25635 :
25636 : /* Given sufficient ISA support we can just return true here
25637 : for selected vector modes. */
25638 816283 : switch (d.vmode)
25639 : {
25640 1636 : case E_V16SFmode:
25641 1636 : case E_V16SImode:
25642 1636 : case E_V8DImode:
25643 1636 : case E_V8DFmode:
25644 1636 : if (!TARGET_AVX512F)
25645 : return false;
25646 : /* All implementable with a single vperm[it]2 insn. */
25647 1636 : if (d.testing_p)
25648 : return true;
25649 : break;
25650 323 : case E_V32HImode:
25651 323 : if (!TARGET_AVX512F)
25652 : return false;
25653 323 : if (d.testing_p && TARGET_AVX512BW)
25654 : /* All implementable with a single vperm[it]2 insn. */
25655 : return true;
25656 : break;
25657 747 : case E_V64QImode:
25658 747 : if (!TARGET_AVX512F)
25659 : return false;
25660 747 : if (d.testing_p && TARGET_AVX512BW)
25661 : /* Implementable with 2 vperm[it]2, 2 vpshufb and 1 or insn. */
25662 : return true;
25663 : break;
25664 11410 : case E_V8SImode:
25665 11410 : case E_V8SFmode:
25666 11410 : case E_V4DFmode:
25667 11410 : case E_V4DImode:
25668 11410 : if (!TARGET_AVX)
25669 : return false;
25670 11410 : if (d.testing_p && TARGET_AVX512VL)
25671 : /* All implementable with a single vperm[it]2 insn. */
25672 : return true;
25673 : break;
25674 614 : case E_V16HImode:
25675 614 : if (!TARGET_SSE2)
25676 : return false;
25677 614 : if (d.testing_p && TARGET_AVX2)
25678 : /* Implementable with 4 vpshufb insns, 2 vpermq and 3 vpor insns. */
25679 : return true;
25680 : break;
25681 693 : case E_V32QImode:
25682 693 : if (!TARGET_SSE2)
25683 : return false;
25684 693 : if (d.testing_p && TARGET_AVX2)
25685 : /* Implementable with 4 vpshufb insns, 2 vpermq and 3 vpor insns. */
25686 : return true;
25687 : break;
25688 38754 : case E_V8HImode:
25689 38754 : case E_V16QImode:
25690 38754 : if (!TARGET_SSE2)
25691 : return false;
25692 : /* Fall through. */
25693 238738 : case E_V4SImode:
25694 238738 : case E_V4SFmode:
25695 238738 : if (!TARGET_SSE)
25696 : return false;
25697 : /* All implementable with a single vpperm insn. */
25698 238738 : if (d.testing_p && TARGET_XOP)
25699 : return true;
25700 : /* All implementable with 2 pshufb + 1 ior. */
25701 238632 : if (d.testing_p && TARGET_SSSE3)
25702 : return true;
25703 : break;
25704 178499 : case E_V2SFmode:
25705 178499 : case E_V2SImode:
25706 178499 : case E_V4HImode:
25707 178499 : case E_V8QImode:
25708 178499 : if (!TARGET_MMX_WITH_SSE)
25709 : return false;
25710 : break;
25711 23016 : case E_V2HImode:
25712 23016 : if (!TARGET_SSE2)
25713 : return false;
25714 : /* All implementable with *punpckwd. */
25715 23016 : if (d.testing_p)
25716 : return true;
25717 : break;
25718 11958 : case E_V4QImode:
25719 11958 : if (!TARGET_SSE2)
25720 : return false;
25721 : break;
25722 346403 : case E_V2DImode:
25723 346403 : case E_V2DFmode:
25724 346403 : if (!TARGET_SSE)
25725 : return false;
25726 : /* All implementable with shufpd or unpck[lh]pd. */
25727 346403 : if (d.testing_p)
25728 : return true;
25729 : break;
25730 : default:
25731 : return false;
25732 : }
25733 :
25734 2365509 : for (i = which = 0; i < nelt; ++i)
25735 : {
25736 1918976 : unsigned char e = sel[i];
25737 1918976 : gcc_assert (e < 2 * nelt);
25738 1918976 : d.perm[i] = e;
25739 1918976 : perm[i] = e;
25740 2596380 : which |= (e < nelt ? 1 : 2);
25741 : }
25742 :
25743 446533 : if (d.testing_p)
25744 : {
25745 : /* For all elements from second vector, fold the elements to first. */
25746 372866 : if (which == 2)
25747 1060 : for (i = 0; i < nelt; ++i)
25748 958 : d.perm[i] -= nelt;
25749 :
25750 : /* Check whether the mask can be applied to the vector type. */
25751 372866 : d.one_operand_p = (which != 3);
25752 :
25753 : /* Implementable with shufps, pshufd or pshuflw. */
25754 372866 : if (d.one_operand_p
25755 : && (d.vmode == V4SFmode || d.vmode == V2SFmode
25756 : || d.vmode == V4SImode || d.vmode == V2SImode
25757 : || d.vmode == V4HImode || d.vmode == V2HImode))
25758 : return true;
25759 :
25760 : /* Otherwise we have to go through the motions and see if we can
25761 : figure out how to generate the requested permutation. */
25762 256712 : d.target = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 1);
25763 256712 : d.op1 = d.op0 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 2);
25764 256712 : if (!d.one_operand_p)
25765 242303 : d.op1 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 3);
25766 :
25767 256712 : start_sequence ();
25768 256712 : bool ret = ix86_expand_vec_perm_const_1 (&d);
25769 256712 : end_sequence ();
25770 :
25771 256712 : return ret;
25772 : }
25773 :
25774 73667 : two_args = canonicalize_perm (&d);
25775 :
25776 : /* If one of the operands is a zero vector, try to match pmovzx. */
25777 73667 : if (two_args && (d.op0 == CONST0_RTX (vmode) || d.op1 == CONST0_RTX (vmode)))
25778 : {
25779 618 : struct expand_vec_perm_d dzero = d;
25780 618 : if (d.op0 == CONST0_RTX (vmode))
25781 : {
25782 387 : d.op1 = dzero.op1 = force_reg (vmode, d.op1);
25783 387 : std::swap (dzero.op0, dzero.op1);
25784 7527 : for (i = 0; i < nelt; ++i)
25785 7140 : dzero.perm[i] ^= nelt;
25786 : }
25787 : else
25788 231 : d.op0 = dzero.op0 = force_reg (vmode, d.op0);
25789 :
25790 618 : if (expand_vselect_vconcat (dzero.target, dzero.op0, dzero.op1,
25791 618 : dzero.perm, nelt, dzero.testing_p))
25792 128 : return true;
25793 : }
25794 :
25795 : /* Force operands into registers. */
25796 73539 : rtx nop0 = force_reg (vmode, d.op0);
25797 73539 : if (d.op0 == d.op1)
25798 18775 : d.op1 = nop0;
25799 73539 : d.op0 = nop0;
25800 73539 : d.op1 = force_reg (vmode, d.op1);
25801 :
25802 73539 : if (ix86_expand_vec_perm_const_1 (&d))
25803 : return true;
25804 :
25805 : /* If the selector says both arguments are needed, but the operands are the
25806 : same, the above tried to expand with one_operand_p and flattened selector.
25807 : If that didn't work, retry without one_operand_p; we succeeded with that
25808 : during testing. */
25809 0 : if (two_args && d.one_operand_p)
25810 : {
25811 0 : d.one_operand_p = false;
25812 0 : memcpy (d.perm, perm, sizeof (perm));
25813 0 : return ix86_expand_vec_perm_const_1 (&d);
25814 : }
25815 :
25816 : return false;
25817 : }
25818 :
25819 : void
25820 8237 : ix86_expand_vec_extract_even_odd (rtx targ, rtx op0, rtx op1, unsigned odd)
25821 : {
25822 8237 : struct expand_vec_perm_d d;
25823 8237 : unsigned i, nelt;
25824 :
25825 8237 : d.target = targ;
25826 8237 : d.op0 = op0;
25827 8237 : d.op1 = op1;
25828 8237 : d.vmode = GET_MODE (targ);
25829 8237 : d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
25830 8237 : d.one_operand_p = false;
25831 8237 : d.testing_p = false;
25832 :
25833 78061 : for (i = 0; i < nelt; ++i)
25834 69824 : d.perm[i] = i * 2 + odd;
25835 :
25836 : /* We'll either be able to implement the permutation directly... */
25837 8237 : if (expand_vec_perm_1 (&d))
25838 3209 : return;
25839 :
25840 : /* ... or we use the special-case patterns. */
25841 5028 : expand_vec_perm_even_odd_1 (&d, odd);
25842 : }
25843 :
25844 : static void
25845 922 : ix86_expand_vec_interleave (rtx targ, rtx op0, rtx op1, bool high_p)
25846 : {
25847 922 : struct expand_vec_perm_d d;
25848 922 : unsigned i, nelt, base;
25849 922 : bool ok;
25850 :
25851 922 : d.target = targ;
25852 922 : d.op0 = op0;
25853 922 : d.op1 = op1;
25854 922 : d.vmode = GET_MODE (targ);
25855 922 : d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
25856 922 : d.one_operand_p = false;
25857 922 : d.testing_p = false;
25858 :
25859 922 : base = high_p ? nelt / 2 : 0;
25860 3642 : for (i = 0; i < nelt / 2; ++i)
25861 : {
25862 2720 : d.perm[i * 2] = i + base;
25863 2720 : d.perm[i * 2 + 1] = i + base + nelt;
25864 : }
25865 :
25866 : /* Note that for AVX this isn't one instruction. */
25867 922 : ok = ix86_expand_vec_perm_const_1 (&d);
25868 922 : gcc_assert (ok);
25869 922 : }
25870 :
25871 : /* Expand a vector operation shift by constant for a V*QImode in terms of the
25872 : same operation on V*HImode. Return true if success. */
25873 : static bool
25874 395 : ix86_expand_vec_shift_qihi_constant (enum rtx_code code,
25875 : rtx dest, rtx op1, rtx op2)
25876 : {
25877 395 : machine_mode qimode, himode;
25878 395 : HOST_WIDE_INT and_constant, xor_constant;
25879 395 : HOST_WIDE_INT shift_amount;
25880 395 : rtx vec_const_and, vec_const_xor;
25881 395 : rtx tmp, op1_subreg;
25882 395 : rtx (*gen_shift) (rtx, rtx, rtx);
25883 395 : rtx (*gen_and) (rtx, rtx, rtx);
25884 395 : rtx (*gen_xor) (rtx, rtx, rtx);
25885 395 : rtx (*gen_sub) (rtx, rtx, rtx);
25886 :
25887 : /* Only optimize shift by constant. */
25888 395 : if (!CONST_INT_P (op2))
25889 : return false;
25890 :
25891 395 : qimode = GET_MODE (dest);
25892 395 : shift_amount = INTVAL (op2);
25893 : /* Do nothing when shift amount greater equal 8. */
25894 395 : if (shift_amount > 7)
25895 : return false;
25896 :
25897 395 : gcc_assert (code == ASHIFT || code == ASHIFTRT || code == LSHIFTRT);
25898 :
25899 :
25900 395 : if (shift_amount == 7
25901 395 : && code == ASHIFTRT)
25902 : {
25903 47 : if (qimode == V16QImode
25904 8 : || qimode == V32QImode)
25905 : {
25906 46 : rtx zero = gen_reg_rtx (qimode);
25907 46 : emit_move_insn (zero, CONST0_RTX (qimode));
25908 46 : emit_move_insn (dest, gen_rtx_fmt_ee (GT, qimode, zero, op1));
25909 46 : }
25910 : else
25911 : {
25912 1 : gcc_assert (qimode == V64QImode);
25913 1 : rtx kmask = gen_reg_rtx (DImode);
25914 1 : emit_insn (gen_avx512bw_cvtb2maskv64qi (kmask, op1));
25915 1 : emit_insn (gen_avx512bw_cvtmask2bv64qi (dest, kmask));
25916 : }
25917 47 : return true;
25918 : }
25919 :
25920 : /* Record sign bit. */
25921 348 : xor_constant = 1 << (8 - shift_amount - 1);
25922 :
25923 : /* Zero upper/lower bits shift from left/right element. */
25924 348 : and_constant
25925 348 : = (code == ASHIFT ? 256 - (1 << shift_amount)
25926 317 : : (1 << (8 - shift_amount)) - 1);
25927 :
25928 348 : switch (qimode)
25929 : {
25930 335 : case V16QImode:
25931 335 : himode = V8HImode;
25932 281 : gen_shift =
25933 : ((code == ASHIFT)
25934 335 : ? gen_ashlv8hi3
25935 313 : : (code == ASHIFTRT) ? gen_ashrv8hi3 : gen_lshrv8hi3);
25936 : gen_and = gen_andv16qi3;
25937 : gen_xor = gen_xorv16qi3;
25938 : gen_sub = gen_subv16qi3;
25939 : break;
25940 6 : case V32QImode:
25941 6 : himode = V16HImode;
25942 1 : gen_shift =
25943 : ((code == ASHIFT)
25944 6 : ? gen_ashlv16hi3
25945 2 : : (code == ASHIFTRT) ? gen_ashrv16hi3 : gen_lshrv16hi3);
25946 : gen_and = gen_andv32qi3;
25947 : gen_xor = gen_xorv32qi3;
25948 : gen_sub = gen_subv32qi3;
25949 : break;
25950 7 : case V64QImode:
25951 7 : himode = V32HImode;
25952 1 : gen_shift =
25953 : ((code == ASHIFT)
25954 7 : ? gen_ashlv32hi3
25955 2 : : (code == ASHIFTRT) ? gen_ashrv32hi3 : gen_lshrv32hi3);
25956 : gen_and = gen_andv64qi3;
25957 : gen_xor = gen_xorv64qi3;
25958 : gen_sub = gen_subv64qi3;
25959 : break;
25960 0 : default:
25961 0 : gcc_unreachable ();
25962 : }
25963 :
25964 348 : tmp = gen_reg_rtx (himode);
25965 348 : vec_const_and = gen_reg_rtx (qimode);
25966 348 : op1_subreg = lowpart_subreg (himode, op1, qimode);
25967 :
25968 : /* For ASHIFT and LSHIFTRT, perform operation like
25969 : vpsllw/vpsrlw $shift_amount, %op1, %dest.
25970 : vpand %vec_const_and, %dest. */
25971 348 : emit_insn (gen_shift (tmp, op1_subreg, op2));
25972 348 : emit_move_insn (dest, simplify_gen_subreg (qimode, tmp, himode, 0));
25973 348 : emit_move_insn (vec_const_and,
25974 : ix86_build_const_vector (qimode, true,
25975 348 : gen_int_mode (and_constant, QImode)));
25976 348 : emit_insn (gen_and (dest, dest, vec_const_and));
25977 :
25978 : /* For ASHIFTRT, perform extra operation like
25979 : vpxor %vec_const_xor, %dest, %dest
25980 : vpsubb %vec_const_xor, %dest, %dest */
25981 348 : if (code == ASHIFTRT)
25982 : {
25983 34 : vec_const_xor = gen_reg_rtx (qimode);
25984 34 : emit_move_insn (vec_const_xor,
25985 : ix86_build_const_vector (qimode, true,
25986 34 : gen_int_mode (xor_constant, QImode)));
25987 34 : emit_insn (gen_xor (dest, dest, vec_const_xor));
25988 34 : emit_insn (gen_sub (dest, dest, vec_const_xor));
25989 : }
25990 : return true;
25991 : }
25992 :
25993 : void
25994 1440 : ix86_expand_vecop_qihi_partial (enum rtx_code code, rtx dest, rtx op1, rtx op2)
25995 : {
25996 1440 : machine_mode qimode = GET_MODE (dest);
25997 1440 : rtx qop1, qop2, hop1, hop2, qdest, hdest;
25998 1440 : bool op2vec = GET_MODE_CLASS (GET_MODE (op2)) == MODE_VECTOR_INT;
25999 1440 : bool uns_p = code != ASHIFTRT;
26000 :
26001 1440 : switch (qimode)
26002 : {
26003 1440 : case E_V4QImode:
26004 1440 : case E_V8QImode:
26005 1440 : break;
26006 0 : default:
26007 0 : gcc_unreachable ();
26008 : }
26009 :
26010 1440 : qop1 = lowpart_subreg (V16QImode, force_reg (qimode, op1), qimode);
26011 :
26012 1440 : if (op2vec)
26013 1310 : qop2 = lowpart_subreg (V16QImode, force_reg (qimode, op2), qimode);
26014 : else
26015 : qop2 = op2;
26016 :
26017 1440 : qdest = gen_reg_rtx (V16QImode);
26018 :
26019 1440 : if (CONST_INT_P (op2)
26020 118 : && (code == ASHIFT || code == LSHIFTRT || code == ASHIFTRT)
26021 : /* With AVX512 it's cheaper to do vpmovsxbw/op/vpmovwb.
26022 : Even with SSE4.1 the alternative is better. */
26023 118 : && !TARGET_SSE4_1
26024 1494 : && ix86_expand_vec_shift_qihi_constant (code, qdest, qop1, qop2))
26025 : {
26026 54 : emit_move_insn (dest, gen_lowpart (qimode, qdest));
26027 54 : return;
26028 : }
26029 :
26030 1386 : if (CONST_INT_P (op2)
26031 64 : && code == ASHIFTRT
26032 14 : && INTVAL (op2) == 7)
26033 : {
26034 4 : rtx zero = gen_reg_rtx (qimode);
26035 4 : emit_move_insn (zero, CONST0_RTX (qimode));
26036 4 : emit_move_insn (dest, gen_rtx_fmt_ee (GT, qimode, zero, op1));
26037 4 : return;
26038 : }
26039 :
26040 1382 : switch (code)
26041 : {
26042 1297 : case MULT:
26043 1297 : gcc_assert (op2vec);
26044 1297 : if (!TARGET_SSE4_1)
26045 : {
26046 : /* Unpack data such that we've got a source byte in each low byte
26047 : of each word. We don't care what goes into the high byte of
26048 : each word. Rather than trying to get zero in there, most
26049 : convenient is to let it be a copy of the low byte. */
26050 244 : hop1 = copy_to_reg (qop1);
26051 244 : hop2 = copy_to_reg (qop2);
26052 244 : emit_insn (gen_vec_interleave_lowv16qi (hop1, hop1, hop1));
26053 244 : emit_insn (gen_vec_interleave_lowv16qi (hop2, hop2, hop2));
26054 244 : break;
26055 : }
26056 : /* FALLTHRU */
26057 1138 : case ASHIFT:
26058 1138 : case ASHIFTRT:
26059 1138 : case LSHIFTRT:
26060 1138 : hop1 = gen_reg_rtx (V8HImode);
26061 1138 : ix86_expand_sse_unpack (hop1, qop1, uns_p, false);
26062 : /* mult/vashr/vlshr/vashl */
26063 1138 : if (op2vec)
26064 : {
26065 1066 : hop2 = gen_reg_rtx (V8HImode);
26066 1066 : ix86_expand_sse_unpack (hop2, qop2, uns_p, false);
26067 : }
26068 : else
26069 : hop2 = qop2;
26070 :
26071 : break;
26072 0 : default:
26073 0 : gcc_unreachable ();
26074 : }
26075 :
26076 1382 : if (code != MULT && op2vec)
26077 : {
26078 : /* Expand vashr/vlshr/vashl. */
26079 13 : hdest = gen_reg_rtx (V8HImode);
26080 13 : emit_insn (gen_rtx_SET (hdest,
26081 : simplify_gen_binary (code, V8HImode,
26082 : hop1, hop2)));
26083 : }
26084 : else
26085 : /* Expand mult/ashr/lshr/ashl. */
26086 1369 : hdest = expand_simple_binop (V8HImode, code, hop1, hop2,
26087 : NULL_RTX, 1, OPTAB_DIRECT);
26088 :
26089 1382 : if (TARGET_AVX512BW && TARGET_AVX512VL)
26090 : {
26091 57 : if (qimode == V8QImode)
26092 : qdest = dest;
26093 : else
26094 10 : qdest = gen_reg_rtx (V8QImode);
26095 :
26096 57 : emit_insn (gen_truncv8hiv8qi2 (qdest, hdest));
26097 : }
26098 : else
26099 : {
26100 1325 : struct expand_vec_perm_d d;
26101 1325 : rtx qres = gen_lowpart (V16QImode, hdest);
26102 1325 : bool ok;
26103 1325 : int i;
26104 :
26105 : /* Merge the data back into the right place. */
26106 1325 : d.target = qdest;
26107 1325 : d.op0 = d.op1 = qres;
26108 1325 : d.vmode = V16QImode;
26109 1325 : d.nelt = 16;
26110 1325 : d.one_operand_p = TARGET_SSSE3;
26111 1325 : d.testing_p = false;
26112 :
26113 22525 : for (i = 0; i < d.nelt; ++i)
26114 21200 : d.perm[i] = i * 2;
26115 :
26116 1325 : ok = ix86_expand_vec_perm_const_1 (&d);
26117 1325 : gcc_assert (ok);
26118 : }
26119 :
26120 1382 : if (qdest != dest)
26121 1335 : emit_move_insn (dest, gen_lowpart (qimode, qdest));
26122 : }
26123 :
26124 : /* Emit instruction in 2x wider mode. For example, optimize
26125 : vector MUL generation like
26126 :
26127 : vpmovzxbw ymm2, xmm0
26128 : vpmovzxbw ymm3, xmm1
26129 : vpmullw ymm4, ymm2, ymm3
26130 : vpmovwb xmm0, ymm4
26131 :
26132 : it would take less instructions than ix86_expand_vecop_qihi.
26133 : Return true if success. */
26134 :
26135 : static bool
26136 1177 : ix86_expand_vecop_qihi2 (enum rtx_code code, rtx dest, rtx op1, rtx op2)
26137 : {
26138 1177 : machine_mode himode, qimode = GET_MODE (dest);
26139 1177 : machine_mode wqimode;
26140 1177 : rtx qop1, qop2, hop1, hop2, hdest;
26141 1177 : rtx (*gen_truncate)(rtx, rtx) = NULL;
26142 1177 : bool op2vec = GET_MODE_CLASS (GET_MODE (op2)) == MODE_VECTOR_INT;
26143 1177 : bool uns_p = code != ASHIFTRT;
26144 :
26145 : /* Without VPMOVWB (provided by AVX512BW ISA), the expansion uses the
26146 : generic permutation to merge the data back into the right place. This
26147 : permutation results in VPERMQ, which is slow, so better fall back to
26148 : ix86_expand_vecop_qihi. */
26149 1177 : if (!TARGET_AVX512BW
26150 327 : || (qimode == V16QImode && !TARGET_AVX512VL)
26151 : /* There are no V64HImode instructions. */
26152 327 : || qimode == V64QImode)
26153 : return false;
26154 :
26155 : /* Do not generate ymm/zmm instructions when
26156 : target prefers 128/256 bit vector width. */
26157 317 : if ((qimode == V16QImode && TARGET_PREFER_AVX128)
26158 317 : || (qimode == V32QImode && TARGET_PREFER_AVX256))
26159 : return false;
26160 :
26161 312 : switch (qimode)
26162 : {
26163 : case E_V16QImode:
26164 : himode = V16HImode;
26165 : gen_truncate = gen_truncv16hiv16qi2;
26166 : break;
26167 57 : case E_V32QImode:
26168 57 : himode = V32HImode;
26169 57 : gen_truncate = gen_truncv32hiv32qi2;
26170 57 : break;
26171 0 : default:
26172 0 : gcc_unreachable ();
26173 : }
26174 :
26175 312 : wqimode = GET_MODE_2XWIDER_MODE (qimode).require ();
26176 312 : qop1 = lowpart_subreg (wqimode, force_reg (qimode, op1), qimode);
26177 :
26178 312 : if (op2vec)
26179 312 : qop2 = lowpart_subreg (wqimode, force_reg (qimode, op2), qimode);
26180 : else
26181 : qop2 = op2;
26182 :
26183 312 : hop1 = gen_reg_rtx (himode);
26184 312 : ix86_expand_sse_unpack (hop1, qop1, uns_p, false);
26185 :
26186 312 : if (op2vec)
26187 : {
26188 312 : hop2 = gen_reg_rtx (himode);
26189 312 : ix86_expand_sse_unpack (hop2, qop2, uns_p, false);
26190 : }
26191 : else
26192 : hop2 = qop2;
26193 :
26194 312 : if (code != MULT && op2vec)
26195 : {
26196 : /* Expand vashr/vlshr/vashl. */
26197 14 : hdest = gen_reg_rtx (himode);
26198 14 : emit_insn (gen_rtx_SET (hdest,
26199 : simplify_gen_binary (code, himode,
26200 : hop1, hop2)));
26201 : }
26202 : else
26203 : /* Expand mult/ashr/lshr/ashl. */
26204 298 : hdest = expand_simple_binop (himode, code, hop1, hop2,
26205 : NULL_RTX, 1, OPTAB_DIRECT);
26206 :
26207 312 : emit_insn (gen_truncate (dest, hdest));
26208 312 : return true;
26209 : }
26210 :
26211 : /* Expand a vector operation CODE for a V*QImode in terms of the
26212 : same operation on V*HImode. */
26213 :
26214 : void
26215 1518 : ix86_expand_vecop_qihi (enum rtx_code code, rtx dest, rtx op1, rtx op2)
26216 : {
26217 1518 : machine_mode qimode = GET_MODE (dest);
26218 1518 : machine_mode himode;
26219 1518 : rtx (*gen_il) (rtx, rtx, rtx);
26220 1518 : rtx (*gen_ih) (rtx, rtx, rtx);
26221 1518 : rtx op1_l, op1_h, op2_l, op2_h, res_l, res_h;
26222 1518 : bool op2vec = GET_MODE_CLASS (GET_MODE (op2)) == MODE_VECTOR_INT;
26223 1518 : struct expand_vec_perm_d d;
26224 1518 : bool full_interleave = true;
26225 1518 : bool uns_p = code != ASHIFTRT;
26226 1518 : bool ok;
26227 1518 : int i;
26228 :
26229 1518 : if (CONST_INT_P (op2)
26230 341 : && (code == ASHIFT || code == LSHIFTRT || code == ASHIFTRT)
26231 1859 : && ix86_expand_vec_shift_qihi_constant (code, dest, op1, op2))
26232 653 : return;
26233 :
26234 1177 : if (ix86_expand_vecop_qihi2 (code, dest, op1, op2))
26235 : return;
26236 :
26237 865 : switch (qimode)
26238 : {
26239 : case E_V16QImode:
26240 : himode = V8HImode;
26241 : break;
26242 44 : case E_V32QImode:
26243 44 : himode = V16HImode;
26244 44 : break;
26245 10 : case E_V64QImode:
26246 10 : himode = V32HImode;
26247 10 : break;
26248 0 : default:
26249 0 : gcc_unreachable ();
26250 : }
26251 :
26252 865 : switch (code)
26253 : {
26254 819 : case MULT:
26255 819 : gcc_assert (op2vec);
26256 : /* Unpack data such that we've got a source byte in each low byte of
26257 : each word. We don't care what goes into the high byte of each word.
26258 : Rather than trying to get zero in there, most convenient is to let
26259 : it be a copy of the low byte. */
26260 819 : switch (qimode)
26261 : {
26262 : case E_V16QImode:
26263 : gen_il = gen_vec_interleave_lowv16qi;
26264 : gen_ih = gen_vec_interleave_highv16qi;
26265 : break;
26266 44 : case E_V32QImode:
26267 44 : gen_il = gen_avx2_interleave_lowv32qi;
26268 44 : gen_ih = gen_avx2_interleave_highv32qi;
26269 44 : full_interleave = false;
26270 44 : break;
26271 8 : case E_V64QImode:
26272 8 : gen_il = gen_avx512bw_interleave_lowv64qi;
26273 8 : gen_ih = gen_avx512bw_interleave_highv64qi;
26274 8 : full_interleave = false;
26275 8 : break;
26276 0 : default:
26277 0 : gcc_unreachable ();
26278 : }
26279 :
26280 819 : op2_l = gen_reg_rtx (qimode);
26281 819 : op2_h = gen_reg_rtx (qimode);
26282 819 : emit_insn (gen_il (op2_l, op2, op2));
26283 819 : emit_insn (gen_ih (op2_h, op2, op2));
26284 :
26285 819 : op1_l = gen_reg_rtx (qimode);
26286 819 : op1_h = gen_reg_rtx (qimode);
26287 819 : emit_insn (gen_il (op1_l, op1, op1));
26288 819 : emit_insn (gen_ih (op1_h, op1, op1));
26289 819 : break;
26290 :
26291 46 : case ASHIFT:
26292 46 : case ASHIFTRT:
26293 46 : case LSHIFTRT:
26294 46 : op1_l = gen_reg_rtx (himode);
26295 46 : op1_h = gen_reg_rtx (himode);
26296 46 : ix86_expand_sse_unpack (op1_l, op1, uns_p, false);
26297 46 : ix86_expand_sse_unpack (op1_h, op1, uns_p, true);
26298 : /* vashr/vlshr/vashl */
26299 46 : if (op2vec)
26300 : {
26301 2 : rtx tmp = force_reg (qimode, op2);
26302 2 : op2_l = gen_reg_rtx (himode);
26303 2 : op2_h = gen_reg_rtx (himode);
26304 2 : ix86_expand_sse_unpack (op2_l, tmp, uns_p, false);
26305 2 : ix86_expand_sse_unpack (op2_h, tmp, uns_p, true);
26306 : }
26307 : else
26308 : op2_l = op2_h = op2;
26309 :
26310 : break;
26311 0 : default:
26312 0 : gcc_unreachable ();
26313 : }
26314 :
26315 865 : if (code != MULT && op2vec)
26316 : {
26317 : /* Expand vashr/vlshr/vashl. */
26318 2 : res_l = gen_reg_rtx (himode);
26319 2 : res_h = gen_reg_rtx (himode);
26320 2 : emit_insn (gen_rtx_SET (res_l,
26321 : simplify_gen_binary (code, himode,
26322 : op1_l, op2_l)));
26323 2 : emit_insn (gen_rtx_SET (res_h,
26324 : simplify_gen_binary (code, himode,
26325 : op1_h, op2_h)));
26326 : }
26327 : else
26328 : {
26329 : /* Expand mult/ashr/lshr/ashl. */
26330 863 : res_l = expand_simple_binop (himode, code, op1_l, op2_l, NULL_RTX,
26331 : 1, OPTAB_DIRECT);
26332 863 : res_h = expand_simple_binop (himode, code, op1_h, op2_h, NULL_RTX,
26333 : 1, OPTAB_DIRECT);
26334 : }
26335 :
26336 865 : gcc_assert (res_l && res_h);
26337 :
26338 : /* Merge the data back into the right place. */
26339 865 : d.target = dest;
26340 865 : d.op0 = gen_lowpart (qimode, res_l);
26341 865 : d.op1 = gen_lowpart (qimode, res_h);
26342 865 : d.vmode = qimode;
26343 865 : d.nelt = GET_MODE_NUNITS (qimode);
26344 865 : d.one_operand_p = false;
26345 865 : d.testing_p = false;
26346 :
26347 865 : if (full_interleave)
26348 : {
26349 : /* We used the full interleave, the desired
26350 : results are in the even elements. */
26351 13917 : for (i = 0; i < d.nelt; ++i)
26352 13104 : d.perm[i] = i * 2;
26353 : }
26354 : else
26355 : {
26356 : /* For AVX, the interleave used above was not cross-lane. So the
26357 : extraction is evens but with the second and third quarter swapped.
26358 : Happily, that is even one insn shorter than even extraction.
26359 : For AVX512BW we have 4 lanes. We extract evens from within a lane,
26360 : always first from the first and then from the second source operand,
26361 : the index bits above the low 4 bits remains the same.
26362 : Thus, for d.nelt == 32 we want permutation
26363 : 0,2,4,..14, 32,34,36,..46, 16,18,20,..30, 48,50,52,..62
26364 : and for d.nelt == 64 we want permutation
26365 : 0,2,4,..14, 64,66,68,..78, 16,18,20,..30, 80,82,84,..94,
26366 : 32,34,36,..46, 96,98,100,..110, 48,50,52,..62, 112,114,116,..126. */
26367 1972 : for (i = 0; i < d.nelt; ++i)
26368 2880 : d.perm[i] = ((i * 2) & 14) + ((i & 8) ? d.nelt : 0) + (i & ~15);
26369 : }
26370 :
26371 865 : ok = ix86_expand_vec_perm_const_1 (&d);
26372 865 : gcc_assert (ok);
26373 : }
26374 :
26375 : /* Helper function of ix86_expand_mul_widen_evenodd. Return true
26376 : if op is CONST_VECTOR with all odd elements equal to their
26377 : preceding element. */
26378 :
26379 : static bool
26380 8842 : const_vector_equal_evenodd_p (rtx op)
26381 : {
26382 8842 : machine_mode mode = GET_MODE (op);
26383 8842 : int i, nunits = GET_MODE_NUNITS (mode);
26384 8842 : if (!CONST_VECTOR_P (op)
26385 8842 : || nunits != CONST_VECTOR_NUNITS (op))
26386 : return false;
26387 3639 : for (i = 0; i < nunits; i += 2)
26388 2924 : if (CONST_VECTOR_ELT (op, i) != CONST_VECTOR_ELT (op, i + 1))
26389 : return false;
26390 : return true;
26391 : }
26392 :
26393 : void
26394 8953 : ix86_expand_mul_widen_evenodd (rtx dest, rtx op1, rtx op2,
26395 : bool uns_p, bool odd_p)
26396 : {
26397 8953 : machine_mode mode = GET_MODE (op1);
26398 8953 : machine_mode wmode = GET_MODE (dest);
26399 8953 : rtx x;
26400 8953 : rtx orig_op1 = op1, orig_op2 = op2;
26401 :
26402 8953 : if (!nonimmediate_operand (op1, mode))
26403 0 : op1 = force_reg (mode, op1);
26404 8953 : if (!nonimmediate_operand (op2, mode))
26405 3354 : op2 = force_reg (mode, op2);
26406 :
26407 : /* We only play even/odd games with vectors of SImode. */
26408 8953 : gcc_assert (mode == V4SImode || mode == V8SImode || mode == V16SImode);
26409 :
26410 : /* If we're looking for the odd results, shift those members down to
26411 : the even slots. For some cpus this is faster than a PSHUFD. */
26412 8953 : if (odd_p)
26413 : {
26414 : /* For XOP use vpmacsdqh, but only for smult, as it is only
26415 : signed. */
26416 4439 : if (TARGET_XOP && mode == V4SImode && !uns_p)
26417 : {
26418 18 : x = force_reg (wmode, CONST0_RTX (wmode));
26419 18 : emit_insn (gen_xop_pmacsdqh (dest, op1, op2, x));
26420 18 : return;
26421 : }
26422 :
26423 8842 : x = GEN_INT (GET_MODE_UNIT_BITSIZE (mode));
26424 4421 : if (!const_vector_equal_evenodd_p (orig_op1))
26425 4421 : op1 = expand_binop (wmode, lshr_optab, gen_lowpart (wmode, op1),
26426 : x, NULL, 1, OPTAB_DIRECT);
26427 4421 : if (!const_vector_equal_evenodd_p (orig_op2))
26428 3706 : op2 = expand_binop (wmode, lshr_optab, gen_lowpart (wmode, op2),
26429 : x, NULL, 1, OPTAB_DIRECT);
26430 4421 : op1 = gen_lowpart (mode, op1);
26431 4421 : op2 = gen_lowpart (mode, op2);
26432 : }
26433 :
26434 8935 : if (mode == V16SImode)
26435 : {
26436 6 : if (uns_p)
26437 0 : x = gen_vec_widen_umult_even_v16si (dest, op1, op2);
26438 : else
26439 6 : x = gen_vec_widen_smult_even_v16si (dest, op1, op2);
26440 : }
26441 8929 : else if (mode == V8SImode)
26442 : {
26443 139 : if (uns_p)
26444 59 : x = gen_vec_widen_umult_even_v8si (dest, op1, op2);
26445 : else
26446 80 : x = gen_vec_widen_smult_even_v8si (dest, op1, op2);
26447 : }
26448 8790 : else if (uns_p)
26449 7689 : x = gen_vec_widen_umult_even_v4si (dest, op1, op2);
26450 1101 : else if (TARGET_SSE4_1)
26451 367 : x = gen_sse4_1_mulv2siv2di3 (dest, op1, op2);
26452 : else
26453 : {
26454 734 : rtx s1, s2, t0, t1, t2;
26455 :
26456 : /* The easiest way to implement this without PMULDQ is to go through
26457 : the motions as if we are performing a full 64-bit multiply. With
26458 : the exception that we need to do less shuffling of the elements. */
26459 :
26460 : /* Compute the sign-extension, aka highparts, of the two operands. */
26461 734 : s1 = ix86_expand_sse_cmp (gen_reg_rtx (mode), GT, CONST0_RTX (mode),
26462 : op1, pc_rtx, pc_rtx);
26463 734 : s2 = ix86_expand_sse_cmp (gen_reg_rtx (mode), GT, CONST0_RTX (mode),
26464 : op2, pc_rtx, pc_rtx);
26465 :
26466 : /* Multiply LO(A) * HI(B), and vice-versa. */
26467 734 : t1 = gen_reg_rtx (wmode);
26468 734 : t2 = gen_reg_rtx (wmode);
26469 734 : emit_insn (gen_vec_widen_umult_even_v4si (t1, s1, op2));
26470 734 : emit_insn (gen_vec_widen_umult_even_v4si (t2, s2, op1));
26471 :
26472 : /* Multiply LO(A) * LO(B). */
26473 734 : t0 = gen_reg_rtx (wmode);
26474 734 : emit_insn (gen_vec_widen_umult_even_v4si (t0, op1, op2));
26475 :
26476 : /* Combine and shift the highparts into place. */
26477 734 : t1 = expand_binop (wmode, add_optab, t1, t2, t1, 1, OPTAB_DIRECT);
26478 734 : t1 = expand_binop (wmode, ashl_optab, t1, GEN_INT (32), t1,
26479 : 1, OPTAB_DIRECT);
26480 :
26481 : /* Combine high and low parts. */
26482 734 : force_expand_binop (wmode, add_optab, t0, t1, dest, 1, OPTAB_DIRECT);
26483 734 : return;
26484 : }
26485 8201 : emit_insn (x);
26486 : }
26487 :
26488 : void
26489 977 : ix86_expand_mul_widen_hilo (rtx dest, rtx op1, rtx op2,
26490 : bool uns_p, bool high_p)
26491 : {
26492 977 : machine_mode wmode = GET_MODE (dest);
26493 977 : machine_mode mode = GET_MODE (op1);
26494 977 : rtx t1, t2, t3, t4, mask;
26495 :
26496 977 : switch (mode)
26497 : {
26498 297 : case E_V4SImode:
26499 297 : t1 = gen_reg_rtx (mode);
26500 297 : t2 = gen_reg_rtx (mode);
26501 297 : if (TARGET_XOP && !uns_p)
26502 : {
26503 : /* With XOP, we have pmacsdqh, aka mul_widen_odd. In this case,
26504 : shuffle the elements once so that all elements are in the right
26505 : place for immediate use: { A C B D }. */
26506 33 : emit_insn (gen_sse2_pshufd_1 (t1, op1, const0_rtx, const2_rtx,
26507 : const1_rtx, GEN_INT (3)));
26508 33 : emit_insn (gen_sse2_pshufd_1 (t2, op2, const0_rtx, const2_rtx,
26509 : const1_rtx, GEN_INT (3)));
26510 : }
26511 : else
26512 : {
26513 : /* Put the elements into place for the multiply. */
26514 264 : ix86_expand_vec_interleave (t1, op1, op1, high_p);
26515 264 : ix86_expand_vec_interleave (t2, op2, op2, high_p);
26516 264 : high_p = false;
26517 : }
26518 297 : ix86_expand_mul_widen_evenodd (dest, t1, t2, uns_p, high_p);
26519 297 : break;
26520 :
26521 70 : case E_V8SImode:
26522 : /* Shuffle the elements between the lanes. After this we
26523 : have { A B E F | C D G H } for each operand. */
26524 70 : t1 = gen_reg_rtx (V4DImode);
26525 70 : t2 = gen_reg_rtx (V4DImode);
26526 70 : emit_insn (gen_avx2_permv4di_1 (t1, gen_lowpart (V4DImode, op1),
26527 : const0_rtx, const2_rtx,
26528 : const1_rtx, GEN_INT (3)));
26529 70 : emit_insn (gen_avx2_permv4di_1 (t2, gen_lowpart (V4DImode, op2),
26530 : const0_rtx, const2_rtx,
26531 : const1_rtx, GEN_INT (3)));
26532 :
26533 : /* Shuffle the elements within the lanes. After this we
26534 : have { A A B B | C C D D } or { E E F F | G G H H }. */
26535 70 : t3 = gen_reg_rtx (V8SImode);
26536 70 : t4 = gen_reg_rtx (V8SImode);
26537 105 : mask = GEN_INT (high_p
26538 : ? 2 + (2 << 2) + (3 << 4) + (3 << 6)
26539 : : 0 + (0 << 2) + (1 << 4) + (1 << 6));
26540 70 : emit_insn (gen_avx2_pshufdv3 (t3, gen_lowpart (V8SImode, t1), mask));
26541 70 : emit_insn (gen_avx2_pshufdv3 (t4, gen_lowpart (V8SImode, t2), mask));
26542 :
26543 70 : ix86_expand_mul_widen_evenodd (dest, t3, t4, uns_p, false);
26544 70 : break;
26545 :
26546 394 : case E_V8HImode:
26547 394 : case E_V16HImode:
26548 394 : t1 = expand_binop (mode, smul_optab, op1, op2, NULL_RTX,
26549 : uns_p, OPTAB_DIRECT);
26550 626 : t2 = expand_binop (mode,
26551 : uns_p ? umul_highpart_optab : smul_highpart_optab,
26552 : op1, op2, NULL_RTX, uns_p, OPTAB_DIRECT);
26553 394 : gcc_assert (t1 && t2);
26554 :
26555 394 : t3 = gen_reg_rtx (mode);
26556 394 : ix86_expand_vec_interleave (t3, t1, t2, high_p);
26557 394 : emit_move_insn (dest, gen_lowpart (wmode, t3));
26558 394 : break;
26559 :
26560 216 : case E_V16QImode:
26561 216 : case E_V32QImode:
26562 216 : case E_V32HImode:
26563 216 : case E_V16SImode:
26564 216 : case E_V64QImode:
26565 216 : t1 = gen_reg_rtx (wmode);
26566 216 : t2 = gen_reg_rtx (wmode);
26567 216 : ix86_expand_sse_unpack (t1, op1, uns_p, high_p);
26568 216 : ix86_expand_sse_unpack (t2, op2, uns_p, high_p);
26569 :
26570 216 : emit_insn (gen_rtx_SET (dest, gen_rtx_MULT (wmode, t1, t2)));
26571 216 : break;
26572 :
26573 0 : default:
26574 0 : gcc_unreachable ();
26575 : }
26576 977 : }
26577 :
26578 : void
26579 3676 : ix86_expand_sse2_mulv4si3 (rtx op0, rtx op1, rtx op2)
26580 : {
26581 3676 : rtx res_1, res_2, res_3, res_4;
26582 :
26583 3676 : res_1 = gen_reg_rtx (V4SImode);
26584 3676 : res_2 = gen_reg_rtx (V4SImode);
26585 3676 : res_3 = gen_reg_rtx (V2DImode);
26586 3676 : res_4 = gen_reg_rtx (V2DImode);
26587 3676 : ix86_expand_mul_widen_evenodd (res_3, op1, op2, true, false);
26588 3676 : ix86_expand_mul_widen_evenodd (res_4, op1, op2, true, true);
26589 :
26590 : /* Move the results in element 2 down to element 1; we don't care
26591 : what goes in elements 2 and 3. Then we can merge the parts
26592 : back together with an interleave.
26593 :
26594 : Note that two other sequences were tried:
26595 : (1) Use interleaves at the start instead of psrldq, which allows
26596 : us to use a single shufps to merge things back at the end.
26597 : (2) Use shufps here to combine the two vectors, then pshufd to
26598 : put the elements in the correct order.
26599 : In both cases the cost of the reformatting stall was too high
26600 : and the overall sequence slower. */
26601 :
26602 3676 : emit_insn (gen_sse2_pshufd_1 (res_1, gen_lowpart (V4SImode, res_3),
26603 : const0_rtx, const2_rtx,
26604 : const0_rtx, const0_rtx));
26605 3676 : emit_insn (gen_sse2_pshufd_1 (res_2, gen_lowpart (V4SImode, res_4),
26606 : const0_rtx, const2_rtx,
26607 : const0_rtx, const0_rtx));
26608 3676 : res_1 = emit_insn (gen_vec_interleave_lowv4si (op0, res_1, res_2));
26609 :
26610 3676 : set_unique_reg_note (res_1, REG_EQUAL, gen_rtx_MULT (V4SImode, op1, op2));
26611 3676 : }
26612 :
26613 : void
26614 540 : ix86_expand_sse2_mulvxdi3 (rtx op0, rtx op1, rtx op2)
26615 : {
26616 540 : machine_mode mode = GET_MODE (op0);
26617 540 : rtx t1, t2, t3, t4, t5, t6;
26618 :
26619 540 : if (TARGET_AVX512DQ && mode == V8DImode)
26620 32 : emit_insn (gen_avx512dq_mulv8di3 (op0, op1, op2));
26621 508 : else if (TARGET_AVX512DQ && TARGET_AVX512VL && mode == V4DImode)
26622 32 : emit_insn (gen_avx512dq_mulv4di3 (op0, op1, op2));
26623 476 : else if (TARGET_AVX512DQ && TARGET_AVX512VL && mode == V2DImode)
26624 36 : emit_insn (gen_avx512dq_mulv2di3 (op0, op1, op2));
26625 440 : else if (TARGET_XOP && mode == V2DImode)
26626 : {
26627 : /* op1: A,B,C,D, op2: E,F,G,H */
26628 2 : op1 = gen_lowpart (V4SImode, op1);
26629 2 : op2 = gen_lowpart (V4SImode, op2);
26630 :
26631 2 : t1 = gen_reg_rtx (V4SImode);
26632 2 : t2 = gen_reg_rtx (V4SImode);
26633 2 : t3 = gen_reg_rtx (V2DImode);
26634 2 : t4 = gen_reg_rtx (V2DImode);
26635 :
26636 : /* t1: B,A,D,C */
26637 2 : emit_insn (gen_sse2_pshufd_1 (t1, op1,
26638 : GEN_INT (1),
26639 : GEN_INT (0),
26640 : GEN_INT (3),
26641 : GEN_INT (2)));
26642 :
26643 : /* t2: (B*E),(A*F),(D*G),(C*H) */
26644 2 : emit_insn (gen_mulv4si3 (t2, t1, op2));
26645 :
26646 : /* t3: (B*E)+(A*F), (D*G)+(C*H) */
26647 2 : emit_insn (gen_xop_phadddq (t3, t2));
26648 :
26649 : /* t4: ((B*E)+(A*F))<<32, ((D*G)+(C*H))<<32 */
26650 2 : emit_insn (gen_ashlv2di3 (t4, t3, GEN_INT (32)));
26651 :
26652 : /* Multiply lower parts and add all */
26653 2 : t5 = gen_reg_rtx (V2DImode);
26654 2 : emit_insn (gen_vec_widen_umult_even_v4si (t5,
26655 2 : gen_lowpart (V4SImode, op1),
26656 2 : gen_lowpart (V4SImode, op2)));
26657 2 : force_expand_binop (mode, add_optab, t5, t4, op0, 1, OPTAB_DIRECT);
26658 : }
26659 : else
26660 : {
26661 438 : machine_mode nmode;
26662 438 : rtx (*umul) (rtx, rtx, rtx);
26663 :
26664 438 : if (mode == V2DImode)
26665 : {
26666 : umul = gen_vec_widen_umult_even_v4si;
26667 : nmode = V4SImode;
26668 : }
26669 295 : else if (mode == V4DImode)
26670 : {
26671 : umul = gen_vec_widen_umult_even_v8si;
26672 : nmode = V8SImode;
26673 : }
26674 116 : else if (mode == V8DImode)
26675 : {
26676 : umul = gen_vec_widen_umult_even_v16si;
26677 : nmode = V16SImode;
26678 : }
26679 : else
26680 0 : gcc_unreachable ();
26681 :
26682 :
26683 : /* Multiply low parts. */
26684 438 : t1 = gen_reg_rtx (mode);
26685 438 : emit_insn (umul (t1, gen_lowpart (nmode, op1), gen_lowpart (nmode, op2)));
26686 :
26687 : /* Shift input vectors right 32 bits so we can multiply high parts. */
26688 438 : t6 = GEN_INT (32);
26689 438 : t2 = expand_binop (mode, lshr_optab, op1, t6, NULL, 1, OPTAB_DIRECT);
26690 438 : t3 = expand_binop (mode, lshr_optab, op2, t6, NULL, 1, OPTAB_DIRECT);
26691 :
26692 : /* Multiply high parts by low parts. */
26693 438 : t4 = gen_reg_rtx (mode);
26694 438 : t5 = gen_reg_rtx (mode);
26695 438 : emit_insn (umul (t4, gen_lowpart (nmode, t2), gen_lowpart (nmode, op2)));
26696 438 : emit_insn (umul (t5, gen_lowpart (nmode, t3), gen_lowpart (nmode, op1)));
26697 :
26698 : /* Combine and shift the highparts back. */
26699 438 : t4 = expand_binop (mode, add_optab, t4, t5, t4, 1, OPTAB_DIRECT);
26700 438 : t4 = expand_binop (mode, ashl_optab, t4, t6, t4, 1, OPTAB_DIRECT);
26701 :
26702 : /* Combine high and low parts. */
26703 438 : force_expand_binop (mode, add_optab, t1, t4, op0, 1, OPTAB_DIRECT);
26704 : }
26705 :
26706 540 : set_unique_reg_note (get_last_insn (), REG_EQUAL,
26707 : gen_rtx_MULT (mode, op1, op2));
26708 540 : }
26709 :
26710 : /* Return 1 if control transfer instruction INSN
26711 : should be encoded with notrack prefix. */
26712 :
26713 : bool
26714 15110200 : ix86_notrack_prefixed_insn_p (rtx_insn *insn)
26715 : {
26716 15110200 : if (!insn || !((flag_cf_protection & CF_BRANCH)))
26717 : return false;
26718 :
26719 4090399 : if (CALL_P (insn))
26720 : {
26721 1474511 : rtx call = get_call_rtx_from (insn);
26722 1474511 : gcc_assert (call != NULL_RTX);
26723 1474511 : rtx addr = XEXP (call, 0);
26724 :
26725 : /* Do not emit 'notrack' if it's not an indirect call. */
26726 1474511 : if (MEM_P (addr)
26727 1474511 : && SYMBOL_REF_P (XEXP (addr, 0)))
26728 : return false;
26729 : else
26730 67035 : return find_reg_note (insn, REG_CALL_NOCF_CHECK, 0);
26731 : }
26732 :
26733 2615888 : if (JUMP_P (insn) && !flag_cet_switch)
26734 : {
26735 2602276 : rtx target = JUMP_LABEL (insn);
26736 2602276 : if (target == NULL_RTX || ANY_RETURN_P (target))
26737 : return false;
26738 :
26739 : /* Check the jump is a switch table. */
26740 2602238 : rtx_insn *label = as_a<rtx_insn *> (target);
26741 2602238 : rtx_insn *table = next_insn (label);
26742 2602238 : if (table == NULL_RTX || !JUMP_TABLE_DATA_P (table))
26743 : return false;
26744 : else
26745 : return true;
26746 : }
26747 : return false;
26748 : }
26749 :
26750 : /* Calculate integer abs() using only SSE2 instructions. */
26751 :
26752 : void
26753 625 : ix86_expand_sse2_abs (rtx target, rtx input)
26754 : {
26755 625 : machine_mode mode = GET_MODE (target);
26756 625 : rtx tmp0, tmp1, x;
26757 :
26758 625 : switch (mode)
26759 : {
26760 48 : case E_V2DImode:
26761 48 : case E_V4DImode:
26762 : /* For 64-bit signed integer X, with SSE4.2 use
26763 : pxor t0, t0; pcmpgtq X, t0; pxor t0, X; psubq t0, X.
26764 : Otherwise handle it similarly to V4SImode, except use 64 as W instead of
26765 : 32 and use logical instead of arithmetic right shift (which is
26766 : unimplemented) and subtract. */
26767 48 : if (TARGET_SSE4_2)
26768 : {
26769 9 : tmp0 = gen_reg_rtx (mode);
26770 9 : tmp1 = gen_reg_rtx (mode);
26771 9 : emit_move_insn (tmp1, CONST0_RTX (mode));
26772 9 : if (mode == E_V2DImode)
26773 6 : emit_insn (gen_sse4_2_gtv2di3 (tmp0, tmp1, input));
26774 : else
26775 3 : emit_insn (gen_avx2_gtv4di3 (tmp0, tmp1, input));
26776 : }
26777 : else
26778 : {
26779 78 : tmp0 = expand_simple_binop (mode, LSHIFTRT, input,
26780 39 : GEN_INT (GET_MODE_UNIT_BITSIZE (mode)
26781 : - 1), NULL, 0, OPTAB_DIRECT);
26782 39 : tmp0 = expand_simple_unop (mode, NEG, tmp0, NULL, false);
26783 : }
26784 :
26785 48 : tmp1 = expand_simple_binop (mode, XOR, tmp0, input,
26786 : NULL, 0, OPTAB_DIRECT);
26787 48 : x = expand_simple_binop (mode, MINUS, tmp1, tmp0,
26788 : target, 0, OPTAB_DIRECT);
26789 48 : break;
26790 :
26791 76 : case E_V4SImode:
26792 : /* For 32-bit signed integer X, the best way to calculate the absolute
26793 : value of X is (((signed) X >> (W-1)) ^ X) - ((signed) X >> (W-1)). */
26794 76 : tmp0 = expand_simple_binop (mode, ASHIFTRT, input,
26795 76 : GEN_INT (GET_MODE_UNIT_BITSIZE (mode) - 1),
26796 : NULL, 0, OPTAB_DIRECT);
26797 76 : tmp1 = expand_simple_binop (mode, XOR, tmp0, input,
26798 : NULL, 0, OPTAB_DIRECT);
26799 76 : x = expand_simple_binop (mode, MINUS, tmp1, tmp0,
26800 : target, 0, OPTAB_DIRECT);
26801 76 : break;
26802 :
26803 97 : case E_V8HImode:
26804 : /* For 16-bit signed integer X, the best way to calculate the absolute
26805 : value of X is max (X, -X), as SSE2 provides the PMAXSW insn. */
26806 97 : tmp0 = expand_unop (mode, neg_optab, input, NULL_RTX, 0);
26807 :
26808 97 : x = expand_simple_binop (mode, SMAX, tmp0, input,
26809 : target, 0, OPTAB_DIRECT);
26810 97 : break;
26811 :
26812 404 : case E_V16QImode:
26813 : /* For 8-bit signed integer X, the best way to calculate the absolute
26814 : value of X is min ((unsigned char) X, (unsigned char) (-X)),
26815 : as SSE2 provides the PMINUB insn. */
26816 404 : tmp0 = expand_unop (mode, neg_optab, input, NULL_RTX, 0);
26817 :
26818 404 : x = expand_simple_binop (V16QImode, UMIN, tmp0, input,
26819 : target, 0, OPTAB_DIRECT);
26820 404 : break;
26821 :
26822 0 : default:
26823 0 : gcc_unreachable ();
26824 : }
26825 :
26826 625 : if (x != target)
26827 0 : emit_move_insn (target, x);
26828 625 : }
26829 :
26830 : /* Expand an extract from a vector register through pextr insn.
26831 : Return true if successful. */
26832 :
26833 : bool
26834 119378 : ix86_expand_pextr (rtx *operands)
26835 : {
26836 119378 : rtx dst = operands[0];
26837 119378 : rtx src = operands[1];
26838 :
26839 119378 : unsigned int size = INTVAL (operands[2]);
26840 119378 : unsigned int pos = INTVAL (operands[3]);
26841 :
26842 119378 : if (SUBREG_P (dst))
26843 : {
26844 : /* Reject non-lowpart subregs. */
26845 76150 : if (SUBREG_BYTE (dst) > 0)
26846 : return false;
26847 76021 : dst = SUBREG_REG (dst);
26848 : }
26849 :
26850 119249 : if (SUBREG_P (src))
26851 : {
26852 34025 : pos += SUBREG_BYTE (src) * BITS_PER_UNIT;
26853 34025 : src = SUBREG_REG (src);
26854 : }
26855 :
26856 119249 : switch (GET_MODE (src))
26857 : {
26858 0 : case E_V16QImode:
26859 0 : case E_V8HImode:
26860 0 : case E_V4SImode:
26861 0 : case E_V2DImode:
26862 0 : case E_V1TImode:
26863 0 : {
26864 0 : machine_mode srcmode, dstmode;
26865 0 : rtx d, pat;
26866 :
26867 0 : if (!int_mode_for_size (size, 0).exists (&dstmode))
26868 0 : return false;
26869 :
26870 0 : switch (dstmode)
26871 : {
26872 0 : case E_QImode:
26873 0 : if (!TARGET_SSE4_1)
26874 : return false;
26875 : srcmode = V16QImode;
26876 : break;
26877 :
26878 0 : case E_HImode:
26879 0 : if (!TARGET_SSE2)
26880 : return false;
26881 : srcmode = V8HImode;
26882 : break;
26883 :
26884 0 : case E_SImode:
26885 0 : if (!TARGET_SSE4_1)
26886 : return false;
26887 : srcmode = V4SImode;
26888 : break;
26889 :
26890 0 : case E_DImode:
26891 0 : gcc_assert (TARGET_64BIT);
26892 0 : if (!TARGET_SSE4_1)
26893 : return false;
26894 : srcmode = V2DImode;
26895 : break;
26896 :
26897 : default:
26898 : return false;
26899 : }
26900 :
26901 : /* Reject extractions from misaligned positions. */
26902 0 : if (pos & (size-1))
26903 : return false;
26904 :
26905 0 : if (GET_MODE (dst) == dstmode)
26906 : d = dst;
26907 : else
26908 0 : d = gen_reg_rtx (dstmode);
26909 :
26910 : /* Construct insn pattern. */
26911 0 : pat = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (pos / size)));
26912 0 : pat = gen_rtx_VEC_SELECT (dstmode, gen_lowpart (srcmode, src), pat);
26913 :
26914 : /* Let the rtl optimizers know about the zero extension performed. */
26915 0 : if (dstmode == QImode || dstmode == HImode)
26916 : {
26917 0 : pat = gen_rtx_ZERO_EXTEND (SImode, pat);
26918 0 : d = gen_lowpart (SImode, d);
26919 : }
26920 :
26921 0 : emit_insn (gen_rtx_SET (d, pat));
26922 :
26923 0 : if (d != dst)
26924 0 : emit_move_insn (dst, gen_lowpart (GET_MODE (dst), d));
26925 : return true;
26926 : }
26927 :
26928 : default:
26929 : return false;
26930 : }
26931 : }
26932 :
26933 : /* Expand an insert into a vector register through pinsr insn.
26934 : Return true if successful. */
26935 :
26936 : bool
26937 112545 : ix86_expand_pinsr (rtx *operands)
26938 : {
26939 112545 : rtx dst = operands[0];
26940 112545 : rtx src = operands[3];
26941 :
26942 112545 : unsigned int size = INTVAL (operands[1]);
26943 112545 : unsigned int pos = INTVAL (operands[2]);
26944 :
26945 112545 : if (SUBREG_P (dst))
26946 : {
26947 63957 : pos += SUBREG_BYTE (dst) * BITS_PER_UNIT;
26948 63957 : dst = SUBREG_REG (dst);
26949 : }
26950 :
26951 112545 : switch (GET_MODE (dst))
26952 : {
26953 20 : case E_V16QImode:
26954 20 : case E_V8HImode:
26955 20 : case E_V4SImode:
26956 20 : case E_V2DImode:
26957 20 : case E_V1TImode:
26958 20 : {
26959 20 : machine_mode srcmode, dstmode;
26960 20 : rtx (*pinsr)(rtx, rtx, rtx, rtx);
26961 20 : rtx d;
26962 :
26963 20 : if (!int_mode_for_size (size, 0).exists (&srcmode))
26964 0 : return false;
26965 :
26966 20 : switch (srcmode)
26967 : {
26968 1 : case E_QImode:
26969 1 : if (!TARGET_SSE4_1)
26970 : return false;
26971 : dstmode = V16QImode;
26972 : pinsr = gen_sse4_1_pinsrb;
26973 : break;
26974 :
26975 5 : case E_HImode:
26976 5 : if (!TARGET_SSE2)
26977 : return false;
26978 : dstmode = V8HImode;
26979 : pinsr = gen_sse2_pinsrw;
26980 : break;
26981 :
26982 14 : case E_SImode:
26983 14 : if (!TARGET_SSE4_1)
26984 : return false;
26985 : dstmode = V4SImode;
26986 : pinsr = gen_sse4_1_pinsrd;
26987 : break;
26988 :
26989 0 : case E_DImode:
26990 0 : gcc_assert (TARGET_64BIT);
26991 0 : if (!TARGET_SSE4_1)
26992 : return false;
26993 : dstmode = V2DImode;
26994 : pinsr = gen_sse4_1_pinsrq;
26995 : break;
26996 :
26997 : default:
26998 : return false;
26999 : }
27000 :
27001 : /* Reject insertions to misaligned positions. */
27002 7 : if (pos & (size-1))
27003 : return false;
27004 :
27005 7 : if (SUBREG_P (src))
27006 : {
27007 7 : unsigned int srcpos = SUBREG_BYTE (src);
27008 :
27009 7 : if (srcpos > 0)
27010 : {
27011 0 : rtx extr_ops[4];
27012 :
27013 0 : extr_ops[0] = gen_reg_rtx (srcmode);
27014 0 : extr_ops[1] = gen_lowpart (srcmode, SUBREG_REG (src));
27015 0 : extr_ops[2] = GEN_INT (size);
27016 0 : extr_ops[3] = GEN_INT (srcpos * BITS_PER_UNIT);
27017 :
27018 0 : if (!ix86_expand_pextr (extr_ops))
27019 0 : return false;
27020 :
27021 0 : src = extr_ops[0];
27022 : }
27023 : else
27024 7 : src = gen_lowpart (srcmode, SUBREG_REG (src));
27025 : }
27026 :
27027 7 : if (GET_MODE (dst) == dstmode)
27028 : d = dst;
27029 : else
27030 7 : d = gen_reg_rtx (dstmode);
27031 :
27032 7 : emit_insn (pinsr (d, gen_lowpart (dstmode, dst),
27033 7 : gen_lowpart (srcmode, src),
27034 7 : GEN_INT (1 << (pos / size))));
27035 7 : if (d != dst)
27036 7 : emit_move_insn (dst, gen_lowpart (GET_MODE (dst), d));
27037 : return true;
27038 : }
27039 :
27040 : default:
27041 : return false;
27042 : }
27043 : }
27044 :
27045 : /* All CPUs prefer to avoid cross-lane operations so perform reductions
27046 : upper against lower halves up to SSE reg size. */
27047 :
27048 : machine_mode
27049 1997 : ix86_split_reduction (machine_mode mode)
27050 : {
27051 : /* Reduce lowpart against highpart until we reach SSE reg width to
27052 : avoid cross-lane operations. */
27053 1997 : switch (mode)
27054 : {
27055 : case E_V8DImode:
27056 : case E_V4DImode:
27057 : return V2DImode;
27058 9 : case E_V16SImode:
27059 9 : case E_V8SImode:
27060 9 : return V4SImode;
27061 8 : case E_V32HImode:
27062 8 : case E_V16HImode:
27063 8 : return V8HImode;
27064 4 : case E_V64QImode:
27065 4 : case E_V32QImode:
27066 4 : return V16QImode;
27067 5 : case E_V16SFmode:
27068 5 : case E_V8SFmode:
27069 5 : return V4SFmode;
27070 16 : case E_V8DFmode:
27071 16 : case E_V4DFmode:
27072 16 : return V2DFmode;
27073 1950 : default:
27074 1950 : return mode;
27075 : }
27076 : }
27077 :
27078 : /* Generate call to __divmoddi4. */
27079 :
27080 : void
27081 896 : ix86_expand_divmod_libfunc (rtx libfunc, machine_mode mode,
27082 : rtx op0, rtx op1,
27083 : rtx *quot_p, rtx *rem_p)
27084 : {
27085 1792 : rtx rem = assign_stack_temp (mode, GET_MODE_SIZE (mode));
27086 :
27087 896 : rtx quot = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
27088 : mode, op0, mode, op1, mode,
27089 896 : XEXP (rem, 0), Pmode);
27090 896 : *quot_p = quot;
27091 896 : *rem_p = rem;
27092 896 : }
27093 :
27094 : void
27095 64 : ix86_expand_atomic_fetch_op_loop (rtx target, rtx mem, rtx val,
27096 : enum rtx_code code, bool after,
27097 : bool doubleword)
27098 : {
27099 64 : rtx old_reg, new_reg, old_mem, success;
27100 64 : machine_mode mode = GET_MODE (target);
27101 64 : rtx_code_label *loop_label = NULL;
27102 :
27103 64 : old_reg = gen_reg_rtx (mode);
27104 64 : new_reg = old_reg;
27105 64 : old_mem = copy_to_reg (mem);
27106 64 : loop_label = gen_label_rtx ();
27107 64 : emit_label (loop_label);
27108 64 : emit_move_insn (old_reg, old_mem);
27109 :
27110 : /* return value for atomic_fetch_op. */
27111 64 : if (!after)
27112 32 : emit_move_insn (target, old_reg);
27113 :
27114 64 : if (code == NOT)
27115 : {
27116 16 : new_reg = expand_simple_binop (mode, AND, new_reg, val, NULL_RTX,
27117 : true, OPTAB_LIB_WIDEN);
27118 16 : new_reg = expand_simple_unop (mode, code, new_reg, NULL_RTX, true);
27119 : }
27120 : else
27121 48 : new_reg = expand_simple_binop (mode, code, new_reg, val, NULL_RTX,
27122 : true, OPTAB_LIB_WIDEN);
27123 :
27124 : /* return value for atomic_op_fetch. */
27125 64 : if (after)
27126 32 : emit_move_insn (target, new_reg);
27127 :
27128 64 : success = NULL_RTX;
27129 :
27130 64 : ix86_expand_cmpxchg_loop (&success, old_mem, mem, old_reg, new_reg,
27131 : gen_int_mode (MEMMODEL_SYNC_SEQ_CST,
27132 : SImode),
27133 : doubleword, loop_label);
27134 64 : }
27135 :
27136 : /* Relax cmpxchg instruction, param loop_label indicates whether
27137 : the instruction should be relaxed with a pause loop. If not,
27138 : it will be relaxed to an atomic load + compare, and skip
27139 : cmpxchg instruction if mem != exp_input. */
27140 :
27141 : void
27142 72 : ix86_expand_cmpxchg_loop (rtx *ptarget_bool, rtx target_val,
27143 : rtx mem, rtx exp_input, rtx new_input,
27144 : rtx mem_model, bool doubleword,
27145 : rtx_code_label *loop_label)
27146 : {
27147 72 : rtx_code_label *cmp_label = NULL;
27148 72 : rtx_code_label *done_label = NULL;
27149 72 : rtx target_bool = NULL_RTX, new_mem = NULL_RTX;
27150 72 : rtx (*gen) (rtx, rtx, rtx, rtx, rtx) = NULL;
27151 72 : rtx (*gendw) (rtx, rtx, rtx, rtx, rtx, rtx) = NULL;
27152 72 : machine_mode mode = GET_MODE (target_val), hmode = mode;
27153 :
27154 72 : if (*ptarget_bool == NULL)
27155 64 : target_bool = gen_reg_rtx (QImode);
27156 : else
27157 : target_bool = *ptarget_bool;
27158 :
27159 72 : cmp_label = gen_label_rtx ();
27160 72 : done_label = gen_label_rtx ();
27161 :
27162 72 : new_mem = gen_reg_rtx (mode);
27163 : /* Load memory first. */
27164 72 : expand_atomic_load (new_mem, mem, MEMMODEL_SEQ_CST);
27165 :
27166 72 : switch (mode)
27167 : {
27168 : case E_TImode:
27169 : gendw = gen_atomic_compare_and_swapti_doubleword;
27170 : hmode = DImode;
27171 : break;
27172 18 : case E_DImode:
27173 18 : if (doubleword)
27174 : {
27175 : gendw = gen_atomic_compare_and_swapdi_doubleword;
27176 : hmode = SImode;
27177 : }
27178 : else
27179 : gen = gen_atomic_compare_and_swapdi_1;
27180 : break;
27181 18 : case E_SImode:
27182 18 : gen = gen_atomic_compare_and_swapsi_1;
27183 18 : break;
27184 18 : case E_HImode:
27185 18 : gen = gen_atomic_compare_and_swaphi_1;
27186 18 : break;
27187 18 : case E_QImode:
27188 18 : gen = gen_atomic_compare_and_swapqi_1;
27189 18 : break;
27190 0 : default:
27191 0 : gcc_unreachable ();
27192 : }
27193 :
27194 : /* Compare mem value with expected value. */
27195 54 : if (doubleword)
27196 : {
27197 0 : rtx low_new_mem = gen_lowpart (hmode, new_mem);
27198 0 : rtx low_exp_input = gen_lowpart (hmode, exp_input);
27199 0 : rtx high_new_mem = gen_highpart (hmode, new_mem);
27200 0 : rtx high_exp_input = gen_highpart (hmode, exp_input);
27201 0 : emit_cmp_and_jump_insns (low_new_mem, low_exp_input, NE, NULL_RTX,
27202 : hmode, 1, cmp_label,
27203 : profile_probability::guessed_never ());
27204 0 : emit_cmp_and_jump_insns (high_new_mem, high_exp_input, NE, NULL_RTX,
27205 : hmode, 1, cmp_label,
27206 : profile_probability::guessed_never ());
27207 : }
27208 : else
27209 72 : emit_cmp_and_jump_insns (new_mem, exp_input, NE, NULL_RTX,
27210 72 : GET_MODE (exp_input), 1, cmp_label,
27211 : profile_probability::guessed_never ());
27212 :
27213 : /* Directly emits cmpxchg here. */
27214 72 : if (doubleword)
27215 0 : emit_insn (gendw (target_val, mem, exp_input,
27216 0 : gen_lowpart (hmode, new_input),
27217 : gen_highpart (hmode, new_input),
27218 : mem_model));
27219 : else
27220 72 : emit_insn (gen (target_val, mem, exp_input, new_input, mem_model));
27221 :
27222 72 : if (!loop_label)
27223 : {
27224 8 : emit_jump_insn (gen_jump (done_label));
27225 8 : emit_barrier ();
27226 8 : emit_label (cmp_label);
27227 8 : emit_move_insn (target_val, new_mem);
27228 8 : emit_label (done_label);
27229 8 : ix86_expand_setcc (target_bool, EQ, gen_rtx_REG (CCZmode, FLAGS_REG),
27230 : const0_rtx);
27231 : }
27232 : else
27233 : {
27234 64 : ix86_expand_setcc (target_bool, EQ, gen_rtx_REG (CCZmode, FLAGS_REG),
27235 : const0_rtx);
27236 64 : emit_cmp_and_jump_insns (target_bool, const0_rtx, EQ, const0_rtx,
27237 64 : GET_MODE (target_bool), 1, loop_label,
27238 : profile_probability::guessed_never ());
27239 64 : emit_jump_insn (gen_jump (done_label));
27240 64 : emit_barrier ();
27241 :
27242 : /* If mem is not expected, pause and loop back. */
27243 64 : emit_label (cmp_label);
27244 64 : emit_move_insn (target_val, new_mem);
27245 64 : emit_insn (gen_pause ());
27246 64 : emit_jump_insn (gen_jump (loop_label));
27247 64 : emit_barrier ();
27248 64 : emit_label (done_label);
27249 : }
27250 :
27251 72 : *ptarget_bool = target_bool;
27252 72 : }
27253 :
27254 : /* Convert a BFmode VAL to SFmode without signaling sNaNs.
27255 : This is done by returning SF SUBREG of ((HI SUBREG) (VAL)) << 16. */
27256 :
27257 : rtx
27258 2978 : ix86_expand_fast_convert_bf_to_sf (rtx val)
27259 : {
27260 2978 : rtx op = gen_lowpart (HImode, val), ret;
27261 2978 : if (CONST_INT_P (op))
27262 : {
27263 587 : ret = simplify_const_unary_operation (FLOAT_EXTEND, SFmode,
27264 : val, BFmode);
27265 587 : if (ret)
27266 : return ret;
27267 : /* FLOAT_EXTEND simplification will fail if VAL is a sNaN. */
27268 1 : ret = gen_reg_rtx (SImode);
27269 1 : emit_move_insn (ret, GEN_INT (INTVAL (op) & 0xffff));
27270 1 : emit_insn (gen_ashlsi3 (ret, ret, GEN_INT (16)));
27271 1 : return gen_lowpart (SFmode, ret);
27272 : }
27273 :
27274 2391 : ret = gen_reg_rtx (SFmode);
27275 2391 : emit_insn (gen_extendbfsf2_1 (ret, force_reg (BFmode, val)));
27276 2391 : return ret;
27277 : }
27278 :
27279 : rtx
27280 65578 : ix86_gen_ccmp_first (rtx_insn **prep_seq, rtx_insn **gen_seq,
27281 : rtx_code code, tree treeop0, tree treeop1)
27282 : {
27283 65578 : if (!TARGET_APX_CCMP)
27284 : return NULL_RTX;
27285 :
27286 65578 : rtx op0, op1, res;
27287 65578 : machine_mode op_mode;
27288 :
27289 65578 : start_sequence ();
27290 65578 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
27291 :
27292 65578 : op_mode = GET_MODE (op0);
27293 65578 : if (op_mode == VOIDmode)
27294 0 : op_mode = GET_MODE (op1);
27295 :
27296 : /* We only supports following scalar comparisons that use just 1
27297 : instruction: DI/SI/HI/QI/XF/DF/SF/HF.
27298 : Unordered/Ordered compare cannot be correctly identified by
27299 : ccmp so they are not supported. */
27300 32771 : if (!(op_mode == DImode || op_mode == SImode
27301 65578 : || op_mode == HImode || op_mode == QImode
27302 32778 : || ((op_mode == XFmode || op_mode == DFmode || op_mode == SFmode)
27303 8 : && (TARGET_80387
27304 1 : || (SSE_FLOAT_MODE_P (op_mode) && TARGET_SSE_MATH)))
27305 0 : || (op_mode == HFmode && TARGET_AVX512FP16))
27306 32807 : || code == ORDERED
27307 32807 : || code == UNORDERED)
27308 : {
27309 32771 : end_sequence ();
27310 32771 : return NULL_RTX;
27311 : }
27312 :
27313 : /* Canonicalize the operands according to mode. */
27314 32807 : if (SCALAR_INT_MODE_P (op_mode))
27315 : {
27316 32800 : if (!nonimmediate_operand (op0, op_mode))
27317 0 : op0 = force_reg (op_mode, op0);
27318 32800 : if (!x86_64_general_operand (op1, op_mode))
27319 0 : op1 = force_reg (op_mode, op1);
27320 : }
27321 : else
27322 : {
27323 : /* op0/op1 can be canonicallized from expand_fp_compare, so
27324 : just adjust the code to make it generate supported fp
27325 : condition. */
27326 7 : if (ix86_fp_compare_code_to_integer (code) == UNKNOWN)
27327 : {
27328 : /* First try to split condition if we don't need to honor
27329 : NaNs, as the ORDERED/UNORDERED check always fall
27330 : through. */
27331 6 : if (!HONOR_NANS (op_mode))
27332 : {
27333 6 : rtx_code first_code;
27334 6 : split_comparison (code, op_mode, &first_code, &code);
27335 : }
27336 : /* Otherwise try to swap the operand order and check if
27337 : the comparison is supported. */
27338 : else
27339 : {
27340 0 : code = swap_condition (code);
27341 0 : std::swap (op0, op1);
27342 : }
27343 :
27344 6 : if (ix86_fp_compare_code_to_integer (code) == UNKNOWN)
27345 : {
27346 0 : end_sequence ();
27347 0 : return NULL_RTX;
27348 : }
27349 : }
27350 : }
27351 :
27352 32807 : *prep_seq = end_sequence ();
27353 :
27354 32807 : start_sequence ();
27355 :
27356 32807 : res = ix86_expand_compare (code, op0, op1);
27357 :
27358 32807 : if (!res)
27359 : {
27360 : end_sequence ();
27361 : return NULL_RTX;
27362 : }
27363 32807 : *gen_seq = end_sequence ();
27364 :
27365 32807 : return res;
27366 : }
27367 :
27368 : rtx
27369 32810 : ix86_gen_ccmp_next (rtx_insn **prep_seq, rtx_insn **gen_seq, rtx prev,
27370 : rtx_code cmp_code, tree treeop0, tree treeop1,
27371 : rtx_code bit_code)
27372 : {
27373 32810 : if (!TARGET_APX_CCMP)
27374 : return NULL_RTX;
27375 :
27376 32810 : rtx op0, op1, target;
27377 32810 : machine_mode op_mode, cmp_mode, cc_mode = CCmode;
27378 32810 : int unsignedp = TYPE_UNSIGNED (TREE_TYPE (treeop0));
27379 32810 : insn_code icode;
27380 32810 : rtx_code prev_code;
27381 32810 : struct expand_operand ops[5];
27382 32810 : int dfv;
27383 :
27384 : /* Exit early for non integer modes to avoid O(n^2) part of expand_operands. */
27385 32810 : cmp_mode = op_mode = TYPE_MODE (TREE_TYPE (treeop0));
27386 :
27387 32810 : if (!(op_mode == DImode || op_mode == SImode || op_mode == HImode
27388 : || op_mode == QImode))
27389 : return NULL_RTX;
27390 :
27391 32 : push_to_sequence (*prep_seq);
27392 32 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
27393 :
27394 32 : icode = code_for_ccmp (op_mode);
27395 :
27396 32 : op0 = prepare_operand (icode, op0, 2, op_mode, cmp_mode, unsignedp);
27397 32 : op1 = prepare_operand (icode, op1, 3, op_mode, cmp_mode, unsignedp);
27398 32 : if (!op0 || !op1)
27399 : {
27400 0 : end_sequence ();
27401 0 : return NULL_RTX;
27402 : }
27403 :
27404 32 : *prep_seq = end_sequence ();
27405 :
27406 32 : target = gen_rtx_REG (cc_mode, FLAGS_REG);
27407 32 : dfv = ix86_get_flags_cc ((rtx_code) cmp_code);
27408 :
27409 32 : prev_code = GET_CODE (prev);
27410 : /* Fixup FP compare code here. */
27411 32 : if (GET_MODE (XEXP (prev, 0)) == CCFPmode)
27412 7 : prev_code = ix86_fp_compare_code_to_integer (prev_code);
27413 :
27414 32 : if (bit_code != AND)
27415 17 : prev_code = reverse_condition (prev_code);
27416 : else
27417 15 : dfv = (int)(dfv ^ 1);
27418 :
27419 32 : prev = gen_rtx_fmt_ee (prev_code, VOIDmode, XEXP (prev, 0),
27420 : const0_rtx);
27421 :
27422 32 : create_fixed_operand (&ops[0], target);
27423 32 : create_fixed_operand (&ops[1], prev);
27424 32 : create_fixed_operand (&ops[2], op0);
27425 32 : create_fixed_operand (&ops[3], op1);
27426 32 : create_fixed_operand (&ops[4], GEN_INT (dfv));
27427 :
27428 32 : push_to_sequence (*gen_seq);
27429 32 : if (!maybe_expand_insn (icode, 5, ops))
27430 : {
27431 0 : end_sequence ();
27432 0 : return NULL_RTX;
27433 : }
27434 :
27435 32 : *gen_seq = end_sequence ();
27436 :
27437 32 : return gen_rtx_fmt_ee ((rtx_code) cmp_code, VOIDmode, target, const0_rtx);
27438 : }
27439 :
27440 : /* Attempt to convert a CONST_VECTOR into a bcst_mem_operand.
27441 : Returns NULL_RTX if X is cannot be expressed as a suitable
27442 : VEC_DUPLICATE in mode MODE. */
27443 :
27444 : static rtx
27445 48 : ix86_gen_bcst_mem (machine_mode mode, rtx x)
27446 : {
27447 48 : if (!TARGET_AVX512F
27448 48 : || !CONST_VECTOR_P (x)
27449 64 : || (!TARGET_AVX512VL && GET_MODE_SIZE (mode) != 64)
27450 147 : || !VALID_BCST_MODE_P (GET_MODE_INNER (mode))
27451 : /* Disallow HFmode broadcast. */
27452 126 : || GET_MODE_SIZE (GET_MODE_INNER (mode)) < 4)
27453 : return NULL_RTX;
27454 :
27455 21 : rtx cst = CONST_VECTOR_ELT (x, 0);
27456 21 : if (!CONST_SCALAR_INT_P (cst)
27457 15 : && !CONST_DOUBLE_P (cst)
27458 0 : && !CONST_FIXED_P (cst))
27459 : return NULL_RTX;
27460 :
27461 21 : int n_elts = GET_MODE_NUNITS (mode);
27462 42 : if (CONST_VECTOR_NUNITS (x) != n_elts)
27463 : return NULL_RTX;
27464 :
27465 150 : for (int i = 1; i < n_elts; i++)
27466 129 : if (!rtx_equal_p (cst, CONST_VECTOR_ELT (x, i)))
27467 : return NULL_RTX;
27468 :
27469 42 : rtx mem = force_const_mem (GET_MODE_INNER (mode), cst);
27470 21 : return gen_rtx_VEC_DUPLICATE (mode, validize_mem (mem));
27471 : }
27472 :
27473 : /* Determine the ternlog immediate index that implements 3-operand
27474 : ternary logic expression OP. This uses and modifies the 3 element
27475 : array ARGS to record and check the leaves, either 3 REGs, or 2 REGs
27476 : and MEM. Returns an index between 0 and 255 for a valid ternlog,
27477 : or -1 if the expression isn't suitable. */
27478 :
27479 : int
27480 7507051 : ix86_ternlog_idx (rtx op, rtx *args)
27481 : {
27482 7507051 : int idx0, idx1;
27483 :
27484 7507051 : if (!op)
27485 : return -1;
27486 :
27487 7507051 : switch (GET_CODE (op))
27488 : {
27489 765654 : case SUBREG:
27490 765654 : if (!register_operand (op, GET_MODE (op)))
27491 : return -1;
27492 : /* FALLTHRU */
27493 :
27494 3706319 : case REG:
27495 3706319 : if (!args[0])
27496 : {
27497 1909971 : args[0] = op;
27498 1909971 : return 0xf0;
27499 : }
27500 1796348 : if (rtx_equal_p (op, args[0]))
27501 : return 0xf0;
27502 1770040 : if (!args[1])
27503 : {
27504 1498302 : args[1] = op;
27505 1498302 : return 0xcc;
27506 : }
27507 271738 : if (rtx_equal_p (op, args[1]))
27508 : return 0xcc;
27509 255160 : if (!args[2])
27510 : {
27511 232215 : args[2] = op;
27512 232215 : return 0xaa;
27513 : }
27514 22945 : if (rtx_equal_p (op, args[2]))
27515 : return 0xaa;
27516 : return -1;
27517 :
27518 18502 : case VEC_DUPLICATE:
27519 18502 : if (!bcst_mem_operand (op, GET_MODE (op)))
27520 : return -1;
27521 302 : goto do_mem_operand;
27522 :
27523 349753 : case MEM:
27524 349753 : if (!memory_operand (op, GET_MODE (op)))
27525 : return -1;
27526 349588 : if (MEM_P (op)
27527 349588 : && MEM_VOLATILE_P (op)
27528 349682 : && !volatile_ok)
27529 : return -1;
27530 : /* FALLTHRU */
27531 :
27532 462175 : case CONST_VECTOR:
27533 462175 : do_mem_operand:
27534 462175 : if (!args[2])
27535 : {
27536 414713 : args[2] = op;
27537 414713 : return 0xaa;
27538 : }
27539 : /* Maximum of one volatile memory reference per expression. */
27540 47462 : if (side_effects_p (op))
27541 : return -1;
27542 47462 : if (rtx_equal_p (op, args[2]))
27543 : return 0xaa;
27544 : /* Check if CONST_VECTOR is the ones-complement of args[2]. */
27545 47411 : if (CONST_VECTOR_P (op)
27546 3550 : && CONST_VECTOR_P (args[2])
27547 47656 : && rtx_equal_p (simplify_const_unary_operation (NOT, GET_MODE (op),
27548 245 : op, GET_MODE (op)),
27549 : args[2]))
27550 : return 0x55;
27551 47224 : if (!args[0])
27552 : {
27553 45414 : args[0] = op;
27554 45414 : return 0xf0;
27555 : }
27556 1810 : if (rtx_equal_p (op, args[0]))
27557 : return 0xf0;
27558 : /* Check if CONST_VECTOR is the ones-complement of args[0]. */
27559 1810 : if (CONST_VECTOR_P (op)
27560 105 : && CONST_VECTOR_P (args[0])
27561 1852 : && rtx_equal_p (simplify_const_unary_operation (NOT, GET_MODE (op),
27562 42 : op, GET_MODE (op)),
27563 : args[0]))
27564 : return 0x0f;
27565 1768 : if (!args[1])
27566 : {
27567 1756 : args[1] = op;
27568 1756 : return 0xcc;
27569 : }
27570 12 : if (rtx_equal_p (op, args[1]))
27571 : return 0xcc;
27572 : /* Check if CONST_VECTOR is the ones-complement of args[1]. */
27573 12 : if (CONST_VECTOR_P (op)
27574 0 : && CONST_VECTOR_P (args[1])
27575 12 : && rtx_equal_p (simplify_const_unary_operation (NOT, GET_MODE (op),
27576 0 : op, GET_MODE (op)),
27577 : args[1]))
27578 : return 0x33;
27579 : return -1;
27580 :
27581 184081 : case NOT:
27582 184081 : idx0 = ix86_ternlog_idx (XEXP (op, 0), args);
27583 184081 : return (idx0 >= 0) ? idx0 ^ 0xff : -1;
27584 :
27585 1348365 : case AND:
27586 1348365 : idx0 = ix86_ternlog_idx (XEXP (op, 0), args);
27587 1348365 : if (idx0 < 0)
27588 : return -1;
27589 1108214 : idx1 = ix86_ternlog_idx (XEXP (op, 1), args);
27590 1108214 : return (idx1 >= 0) ? idx0 & idx1 : -1;
27591 :
27592 959190 : case IOR:
27593 959190 : idx0 = ix86_ternlog_idx (XEXP (op, 0), args);
27594 959190 : if (idx0 < 0)
27595 : return -1;
27596 711803 : idx1 = ix86_ternlog_idx (XEXP (op, 1), args);
27597 711803 : return (idx1 >= 0) ? idx0 | idx1 : -1;
27598 :
27599 434096 : case XOR:
27600 434096 : idx0 = ix86_ternlog_idx (XEXP (op, 0), args);
27601 434096 : if (idx0 < 0)
27602 : return -1;
27603 411802 : if (vector_all_ones_operand (XEXP (op, 1), GET_MODE (op)))
27604 6724 : return idx0 ^ 0xff;
27605 405078 : idx1 = ix86_ternlog_idx (XEXP (op, 1), args);
27606 405078 : return (idx1 >= 0) ? idx0 ^ idx1 : -1;
27607 :
27608 7348 : case UNSPEC:
27609 7348 : if (XINT (op, 1) != UNSPEC_VTERNLOG
27610 0 : || XVECLEN (op, 0) != 4
27611 0 : || !CONST_INT_P (XVECEXP (op, 0, 3)))
27612 : return -1;
27613 :
27614 : /* TODO: Handle permuted operands. */
27615 0 : if (ix86_ternlog_idx (XVECEXP (op, 0, 0), args) != 0xf0
27616 0 : || ix86_ternlog_idx (XVECEXP (op, 0, 1), args) != 0xcc
27617 0 : || ix86_ternlog_idx (XVECEXP (op, 0, 2), args) != 0xaa)
27618 0 : return -1;
27619 0 : return INTVAL (XVECEXP (op, 0, 3));
27620 :
27621 : default:
27622 : return -1;
27623 : }
27624 : }
27625 :
27626 : /* Return TRUE if OP (in mode MODE) is the leaf of a ternary logic
27627 : expression, such as a register or a memory reference. */
27628 :
27629 : bool
27630 3486845 : ix86_ternlog_leaf_p (rtx op, machine_mode mode)
27631 : {
27632 : /* We can't use memory_operand here, as it may return a different
27633 : value before and after reload (for volatile MEMs) which creates
27634 : problems splitting instructions. */
27635 3486845 : return register_operand (op, mode)
27636 730423 : || MEM_P (op)
27637 395078 : || CONST_VECTOR_P (op)
27638 3776894 : || bcst_mem_operand (op, mode);
27639 : }
27640 :
27641 : /* Test whether OP is a 3-operand ternary logic expression suitable
27642 : for use in a ternlog instruction. */
27643 :
27644 : bool
27645 2320220 : ix86_ternlog_operand_p (rtx op)
27646 : {
27647 2320220 : rtx op0, op1;
27648 2320220 : rtx args[3];
27649 :
27650 2320220 : args[0] = NULL_RTX;
27651 2320220 : args[1] = NULL_RTX;
27652 2320220 : args[2] = NULL_RTX;
27653 2320220 : int idx = ix86_ternlog_idx (op, args);
27654 2320220 : if (idx < 0)
27655 : return false;
27656 :
27657 : /* Don't match simple (binary or unary) expressions. */
27658 1885899 : machine_mode mode = GET_MODE (op);
27659 1885899 : switch (GET_CODE (op))
27660 : {
27661 874400 : case AND:
27662 874400 : op0 = XEXP (op, 0);
27663 874400 : op1 = XEXP (op, 1);
27664 :
27665 : /* Prefer pand. */
27666 874400 : if (ix86_ternlog_leaf_p (op0, mode)
27667 874400 : && ix86_ternlog_leaf_p (op1, mode))
27668 : return false;
27669 : /* Prefer pandn. */
27670 113251 : if (GET_CODE (op0) == NOT
27671 77860 : && register_operand (XEXP (op0, 0), mode)
27672 187484 : && ix86_ternlog_leaf_p (op1, mode))
27673 : return false;
27674 : break;
27675 :
27676 624633 : case IOR:
27677 : /* Prefer por. */
27678 624633 : if (ix86_ternlog_leaf_p (XEXP (op, 0), mode)
27679 624633 : && ix86_ternlog_leaf_p (XEXP (op, 1), mode))
27680 : return false;
27681 : break;
27682 :
27683 351468 : case XOR:
27684 351468 : op1 = XEXP (op, 1);
27685 : /* Prefer pxor, or one_cmpl<vmode>2. */
27686 351468 : if (ix86_ternlog_leaf_p (XEXP (op, 0), mode)
27687 351468 : && ix86_ternlog_leaf_p (XEXP (op, 1), mode))
27688 : return false;
27689 : break;
27690 :
27691 : default:
27692 : break;
27693 : }
27694 : return true;
27695 : }
27696 :
27697 : /* Helper function for ix86_expand_ternlog. */
27698 : static rtx
27699 0 : ix86_expand_ternlog_binop (enum rtx_code code, machine_mode mode,
27700 : rtx op0, rtx op1, rtx target)
27701 : {
27702 0 : if (GET_MODE (op0) != mode)
27703 0 : op0 = gen_lowpart (mode, op0);
27704 0 : if (GET_MODE (op1) != mode)
27705 0 : op1 = gen_lowpart (mode, op1);
27706 :
27707 0 : if (CONST_VECTOR_P (op0))
27708 0 : op0 = validize_mem (force_const_mem (mode, op0));
27709 0 : if (CONST_VECTOR_P (op1))
27710 0 : op1 = validize_mem (force_const_mem (mode, op1));
27711 :
27712 0 : if (!register_operand (op0, mode))
27713 : {
27714 0 : if (!register_operand (op1, mode))
27715 : {
27716 : /* We can't use force_reg (op0, mode). */
27717 0 : rtx reg = gen_reg_rtx (mode);
27718 0 : emit_move_insn (reg, op0);
27719 0 : op0 = reg;
27720 : }
27721 : else
27722 : std::swap (op0, op1);
27723 : }
27724 0 : rtx ops[3] = { target, op0, op1 };
27725 0 : ix86_expand_vector_logical_operator (code, mode, ops);
27726 0 : return target;
27727 : }
27728 :
27729 :
27730 : /* Helper function for ix86_expand_ternlog. */
27731 : static rtx
27732 0 : ix86_expand_ternlog_andnot (machine_mode mode, rtx op0, rtx op1, rtx target)
27733 : {
27734 0 : if (GET_MODE (op0) != mode)
27735 0 : op0 = gen_lowpart (mode, op0);
27736 0 : op0 = gen_rtx_NOT (mode, op0);
27737 0 : if (GET_MODE (op1) != mode)
27738 0 : op1 = gen_lowpart (mode, op1);
27739 0 : if (CONST_VECTOR_P (op1))
27740 0 : op1 = validize_mem (force_const_mem (mode, op1));
27741 0 : emit_move_insn (target, gen_rtx_AND (mode, op0, op1));
27742 0 : return target;
27743 : }
27744 :
27745 : /* Expand a 3-operand ternary logic expression. Return TARGET. */
27746 : rtx
27747 2422 : ix86_expand_ternlog (machine_mode mode, rtx op0, rtx op1, rtx op2, int idx,
27748 : rtx target)
27749 : {
27750 2422 : rtx tmp0, tmp1, tmp2;
27751 :
27752 2422 : if (!target)
27753 3 : target = gen_reg_rtx (mode);
27754 :
27755 : /* Canonicalize ternlog index for degenerate (duplicated) operands. */
27756 2422 : if (rtx_equal_p (op0, op1) && rtx_equal_p (op0, op2))
27757 0 : switch (idx & 0x81)
27758 : {
27759 : case 0x00:
27760 : idx = 0x00;
27761 : break;
27762 : case 0x01:
27763 : idx = 0x0f;
27764 : break;
27765 : case 0x80:
27766 : idx = 0xf0;
27767 : break;
27768 : case 0x81:
27769 : idx = 0xff;
27770 : break;
27771 : }
27772 :
27773 2422 : switch (idx & 0xff)
27774 : {
27775 0 : case 0x00:
27776 0 : if ((!op0 || !side_effects_p (op0))
27777 0 : && (!op1 || !side_effects_p (op1))
27778 0 : && (!op2 || !side_effects_p (op2)))
27779 : {
27780 0 : emit_move_insn (target, CONST0_RTX (mode));
27781 0 : return target;
27782 : }
27783 : break;
27784 :
27785 0 : case 0x0a: /* ~a&c */
27786 0 : if ((!op1 || !side_effects_p (op1))
27787 0 : && op0 && register_operand (op0, mode)
27788 0 : && op2 && ix86_ternlog_leaf_p (op2, mode))
27789 0 : return ix86_expand_ternlog_andnot (mode, op0, op2, target);
27790 : break;
27791 :
27792 0 : case 0x0c: /* ~a&b */
27793 0 : if ((!op2 || !side_effects_p (op2))
27794 0 : && op0 && register_operand (op0, mode)
27795 0 : && op1 && ix86_ternlog_leaf_p (op1, mode))
27796 0 : return ix86_expand_ternlog_andnot (mode, op0, op1, target);
27797 : break;
27798 :
27799 80 : case 0x0f: /* ~a */
27800 0 : if ((!op1 || !side_effects_p (op1))
27801 80 : && (!op2 || !side_effects_p (op2))
27802 160 : && op0)
27803 : {
27804 80 : emit_move_insn (target, gen_rtx_XOR (mode, op0, CONSTM1_RTX (mode)));
27805 80 : return target;
27806 : }
27807 : break;
27808 :
27809 0 : case 0x22: /* ~b&c */
27810 0 : if ((!op0 || !side_effects_p (op0))
27811 0 : && op1 && register_operand (op1, mode)
27812 0 : && op2 && ix86_ternlog_leaf_p (op2, mode))
27813 0 : return ix86_expand_ternlog_andnot (mode, op1, op2, target);
27814 : break;
27815 :
27816 0 : case 0x30: /* ~b&a */
27817 0 : if ((!op2 || !side_effects_p (op2))
27818 0 : && op0 && ix86_ternlog_leaf_p (op0, mode)
27819 0 : && op1 && register_operand (op1, mode))
27820 0 : return ix86_expand_ternlog_andnot (mode, op1, op0, target);
27821 : break;
27822 :
27823 0 : case 0x33: /* ~b */
27824 0 : if ((!op0 || !side_effects_p (op0))
27825 0 : && (!op2 || !side_effects_p (op2))
27826 0 : && op1)
27827 : {
27828 0 : emit_move_insn (target, gen_rtx_XOR (mode, op1, CONSTM1_RTX (mode)));
27829 0 : return target;
27830 : }
27831 : break;
27832 :
27833 0 : case 0x3c: /* a^b */
27834 0 : if (op0 && ix86_ternlog_leaf_p (op0, mode)
27835 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
27836 0 : && (!op2 || !side_effects_p (op2)))
27837 0 : return ix86_expand_ternlog_binop (XOR, mode, op0, op1, target);
27838 : break;
27839 :
27840 0 : case 0x44: /* ~c&b */
27841 0 : if ((!op0 || !side_effects_p (op0))
27842 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
27843 0 : && op2 && register_operand (op2, mode))
27844 0 : return ix86_expand_ternlog_andnot (mode, op2, op1, target);
27845 : break;
27846 :
27847 2 : case 0x50: /* ~c&a */
27848 0 : if ((!op1 || !side_effects_p (op1))
27849 2 : && op0 && ix86_ternlog_leaf_p (op0, mode)
27850 4 : && op2 && register_operand (op2, mode))
27851 0 : return ix86_expand_ternlog_andnot (mode, op2, op0, target);
27852 : break;
27853 :
27854 4 : case 0x55: /* ~c */
27855 1 : if ((!op0 || !side_effects_p (op0))
27856 4 : && (!op1 || !side_effects_p (op1))
27857 8 : && op2)
27858 : {
27859 4 : emit_move_insn (target, gen_rtx_XOR (mode, op2, CONSTM1_RTX (mode)));
27860 4 : return target;
27861 : }
27862 : break;
27863 :
27864 0 : case 0x5a: /* a^c */
27865 0 : if (op0 && ix86_ternlog_leaf_p (op0, mode)
27866 0 : && op2 && ix86_ternlog_leaf_p (op2, mode)
27867 0 : && (!op1 || !side_effects_p (op1)))
27868 0 : return ix86_expand_ternlog_binop (XOR, mode, op0, op2, target);
27869 : break;
27870 :
27871 0 : case 0x66: /* b^c */
27872 0 : if ((!op0 || !side_effects_p (op0))
27873 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
27874 0 : && op2 && ix86_ternlog_leaf_p (op2, mode))
27875 0 : return ix86_expand_ternlog_binop (XOR, mode, op1, op2, target);
27876 : break;
27877 :
27878 0 : case 0x88: /* b&c */
27879 0 : if ((!op0 || !side_effects_p (op0))
27880 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
27881 0 : && op2 && ix86_ternlog_leaf_p (op2, mode))
27882 0 : return ix86_expand_ternlog_binop (AND, mode, op1, op2, target);
27883 : break;
27884 :
27885 0 : case 0xa0: /* a&c */
27886 0 : if ((!op1 || !side_effects_p (op1))
27887 0 : && op0 && ix86_ternlog_leaf_p (op0, mode)
27888 0 : && op2 && ix86_ternlog_leaf_p (op2, mode))
27889 0 : return ix86_expand_ternlog_binop (AND, mode, op0, op2, target);
27890 : break;
27891 :
27892 0 : case 0xaa: /* c */
27893 0 : if ((!op0 || !side_effects_p (op0))
27894 0 : && (!op1 || !side_effects_p (op1))
27895 0 : && op2)
27896 : {
27897 0 : if (GET_MODE (op2) != mode)
27898 0 : op2 = gen_lowpart (mode, op2);
27899 0 : emit_move_insn (target, op2);
27900 0 : return target;
27901 : }
27902 : break;
27903 :
27904 0 : case 0xc0: /* a&b */
27905 0 : if (op0 && ix86_ternlog_leaf_p (op0, mode)
27906 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
27907 0 : && (!op2 || !side_effects_p (op2)))
27908 0 : return ix86_expand_ternlog_binop (AND, mode, op0, op1, target);
27909 : break;
27910 :
27911 0 : case 0xcc: /* b */
27912 0 : if ((!op0 || !side_effects_p (op0))
27913 0 : && op1
27914 0 : && (!op2 || !side_effects_p (op2)))
27915 : {
27916 0 : if (GET_MODE (op1) != mode)
27917 0 : op1 = gen_lowpart (mode, op1);
27918 0 : emit_move_insn (target, op1);
27919 0 : return target;
27920 : }
27921 : break;
27922 :
27923 0 : case 0xee: /* b|c */
27924 0 : if ((!op0 || !side_effects_p (op0))
27925 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
27926 0 : && op2 && ix86_ternlog_leaf_p (op2, mode))
27927 0 : return ix86_expand_ternlog_binop (IOR, mode, op1, op2, target);
27928 : break;
27929 :
27930 6 : case 0xf0: /* a */
27931 6 : if (op0
27932 6 : && (!op1 || !side_effects_p (op1))
27933 12 : && (!op2 || !side_effects_p (op2)))
27934 : {
27935 6 : if (GET_MODE (op0) != mode)
27936 0 : op0 = gen_lowpart (mode, op0);
27937 6 : emit_move_insn (target, op0);
27938 6 : return target;
27939 : }
27940 : break;
27941 :
27942 0 : case 0xfa: /* a|c */
27943 0 : if (op0 && ix86_ternlog_leaf_p (op0, mode)
27944 0 : && op2 && ix86_ternlog_leaf_p (op2, mode)
27945 0 : && (!op1 || !side_effects_p (op1)))
27946 0 : return ix86_expand_ternlog_binop (IOR, mode, op0, op2, target);
27947 : break;
27948 :
27949 0 : case 0xfc: /* a|b */
27950 0 : if (op0 && ix86_ternlog_leaf_p (op0, mode)
27951 0 : && op1 && ix86_ternlog_leaf_p (op1, mode)
27952 0 : && (!op2 || !side_effects_p (op2)))
27953 0 : return ix86_expand_ternlog_binop (IOR, mode, op0, op1, target);
27954 : break;
27955 :
27956 0 : case 0xff:
27957 0 : if ((!op0 || !side_effects_p (op0))
27958 0 : && (!op1 || !side_effects_p (op1))
27959 0 : && (!op2 || !side_effects_p (op2)))
27960 : {
27961 0 : emit_move_insn (target, CONSTM1_RTX (mode));
27962 0 : return target;
27963 : }
27964 : break;
27965 : }
27966 :
27967 2332 : if (!register_operand (op0, mode))
27968 : {
27969 : /* We can't use force_reg (mode, op0). */
27970 12 : tmp0 = gen_reg_rtx (GET_MODE (op0));
27971 12 : emit_move_insn (tmp0,op0);
27972 : }
27973 : else
27974 : tmp0 = op0;
27975 2332 : if (GET_MODE (tmp0) != mode)
27976 0 : tmp0 = gen_lowpart (mode, tmp0);
27977 :
27978 2332 : if (!op1 || rtx_equal_p (op0, op1))
27979 6 : tmp1 = copy_rtx (tmp0);
27980 2326 : else if (!register_operand (op1, mode))
27981 : {
27982 : /* We can't use force_reg (mode, op1). */
27983 28 : tmp1 = gen_reg_rtx (GET_MODE (op1));
27984 28 : emit_move_insn (tmp1, op1);
27985 : }
27986 : else
27987 : tmp1 = op1;
27988 2332 : if (GET_MODE (tmp1) != mode)
27989 0 : tmp1 = gen_lowpart (mode, tmp1);
27990 :
27991 2332 : if (!op2 || rtx_equal_p (op0, op2))
27992 75 : tmp2 = copy_rtx (tmp0);
27993 2257 : else if (rtx_equal_p (op1, op2))
27994 0 : tmp2 = copy_rtx (tmp1);
27995 2257 : else if (CONST_VECTOR_P (op2))
27996 : {
27997 43 : if (GET_MODE (op2) != mode)
27998 0 : op2 = gen_lowpart (mode, op2);
27999 43 : tmp2 = ix86_gen_bcst_mem (mode, op2);
28000 43 : if (!tmp2)
28001 : {
28002 25 : machine_mode bcst32_mode = mode;
28003 25 : machine_mode bcst64_mode = mode;
28004 25 : switch (mode)
28005 : {
28006 1 : case V1TImode:
28007 1 : case V4SImode:
28008 1 : case V4SFmode:
28009 1 : case V8HImode:
28010 1 : case V16QImode:
28011 1 : bcst32_mode = V4SImode;
28012 1 : bcst64_mode = V2DImode;
28013 1 : break;
28014 :
28015 0 : case V2TImode:
28016 0 : case V8SImode:
28017 0 : case V8SFmode:
28018 0 : case V16HImode:
28019 0 : case V32QImode:
28020 0 : bcst32_mode = V8SImode;
28021 0 : bcst64_mode = V4DImode;
28022 0 : break;
28023 :
28024 3 : case V4TImode:
28025 3 : case V16SImode:
28026 3 : case V16SFmode:
28027 3 : case V32HImode:
28028 3 : case V64QImode:
28029 3 : bcst32_mode = V16SImode;
28030 3 : bcst64_mode = V8DImode;
28031 3 : break;
28032 :
28033 : default:
28034 : break;
28035 : }
28036 :
28037 25 : if (bcst32_mode != mode)
28038 : {
28039 4 : tmp2 = gen_lowpart (bcst32_mode, op2);
28040 4 : if (ix86_gen_bcst_mem (bcst32_mode, tmp2))
28041 : {
28042 3 : tmp2 = ix86_expand_ternlog (bcst32_mode,
28043 3 : gen_lowpart (bcst32_mode, tmp0),
28044 3 : gen_lowpart (bcst32_mode, tmp1),
28045 : tmp2, idx, NULL_RTX);
28046 3 : emit_move_insn (target, gen_lowpart (mode, tmp2));
28047 3 : return target;
28048 : }
28049 : }
28050 :
28051 22 : if (bcst64_mode != mode)
28052 : {
28053 1 : tmp2 = gen_lowpart (bcst64_mode, op2);
28054 1 : if (ix86_gen_bcst_mem (bcst64_mode, tmp2))
28055 : {
28056 0 : tmp2 = ix86_expand_ternlog (bcst64_mode,
28057 0 : gen_lowpart (bcst64_mode, tmp0),
28058 0 : gen_lowpart (bcst64_mode, tmp1),
28059 : tmp2, idx, NULL_RTX);
28060 0 : emit_move_insn (target, gen_lowpart (mode, tmp2));
28061 0 : return target;
28062 : }
28063 : }
28064 :
28065 22 : tmp2 = force_const_mem (mode, op2);
28066 22 : rtx bcast = ix86_broadcast_from_constant (mode, tmp2);
28067 22 : tmp2 = validize_mem (tmp2);
28068 22 : if (bcast)
28069 : {
28070 12 : rtx reg2 = gen_reg_rtx (mode);
28071 12 : bool ok = ix86_expand_vector_init_duplicate (false, mode,
28072 : reg2, bcast);
28073 12 : if (ok)
28074 2329 : tmp2 = reg2;
28075 : }
28076 : }
28077 : }
28078 : else
28079 : tmp2 = op2;
28080 2329 : if (GET_MODE (tmp2) != mode)
28081 0 : tmp2 = gen_lowpart (mode, tmp2);
28082 : /* Some memory_operands are not vector_memory_operands. */
28083 2329 : if (!bcst_vector_operand (tmp2, mode))
28084 0 : tmp2 = force_reg (mode, tmp2);
28085 :
28086 2329 : rtvec vec = gen_rtvec (4, tmp0, tmp1, tmp2, GEN_INT (idx));
28087 2329 : emit_move_insn (target, gen_rtx_UNSPEC (mode, vec, UNSPEC_VTERNLOG));
28088 2329 : return target;
28089 : }
28090 :
28091 : /* GF2P8AFFINEQB matrixes to implement shift and rotate. */
28092 :
28093 : static const uint64_t matrix_ashift[8] =
28094 : {
28095 : 0,
28096 : 0x0001020408102040, /* 1 l */
28097 : 0x0000010204081020, /* 2 l */
28098 : 0x0000000102040810, /* 3 l */
28099 : 0x0000000001020408, /* 4 l */
28100 : 0x0000000000010204, /* 5 l */
28101 : 0x0000000000000102, /* 6 l */
28102 : 0x0000000000000001 /* 7 l */
28103 : };
28104 :
28105 : static const uint64_t matrix_lshiftrt[8] =
28106 : {
28107 : 0,
28108 : 0x0204081020408000, /* 1 r */
28109 : 0x0408102040800000, /* 2 r */
28110 : 0x0810204080000000, /* 3 r */
28111 : 0x1020408000000000, /* 4 r */
28112 : 0x2040800000000000, /* 5 r */
28113 : 0x4080000000000000, /* 6 r */
28114 : 0x8000000000000000 /* 7 r */
28115 : };
28116 :
28117 : static const uint64_t matrix_ashiftrt[8] =
28118 : {
28119 : 0,
28120 : 0x0204081020408080, /* 1 r */
28121 : 0x0408102040808080, /* 2 r */
28122 : 0x0810204080808080, /* 3 r */
28123 : 0x1020408080808080, /* 4 r */
28124 : 0x2040808080808080, /* 5 r */
28125 : 0x4080808080808080, /* 6 r */
28126 : 0x8080808080808080 /* 7 r */
28127 : };
28128 :
28129 : static const uint64_t matrix_rotate[8] =
28130 : {
28131 : 0,
28132 : 0x8001020408102040, /* 1 rol8 */
28133 : 0x4080010204081020, /* 2 rol8 */
28134 : 0x2040800102040810, /* 3 rol8 */
28135 : 0x1020408001020408, /* 4 rol8 */
28136 : 0x0810204080010204, /* 5 rol8 */
28137 : 0x0408102040800102, /* 6 rol8 */
28138 : 0x0204081020408001 /* 7 rol8 */
28139 : };
28140 :
28141 : static const uint64_t matrix_rotatert[8] =
28142 : {
28143 : 0,
28144 : 0x0204081020408001, /* 1 ror8 */
28145 : 0x0408102040800102, /* 2 ror8 */
28146 : 0x0810204080010204, /* 3 ror8 */
28147 : 0x1020408001020408, /* 4 ror8 */
28148 : 0x2040800102040810, /* 5 ror8 */
28149 : 0x4080010204081020, /* 6 ror8 */
28150 : 0x8001020408102040 /* 7 ror8 */
28151 : };
28152 :
28153 : /* Return rtx to load a 64bit GF2P8AFFINE GP(2) matrix implementing a shift
28154 : for CODE and shift count COUNT into register with vector of size of SRC. */
28155 :
28156 : rtx
28157 202 : ix86_vgf2p8affine_shift_matrix (rtx src, rtx count, enum rtx_code code)
28158 : {
28159 202 : machine_mode mode = GET_MODE (src);
28160 202 : const uint64_t *matrix;
28161 202 : unsigned shift = INTVAL (count) & 7;
28162 202 : gcc_assert (shift > 0 && shift < 8);
28163 :
28164 202 : switch (code)
28165 : {
28166 : case ASHIFT:
28167 : matrix = matrix_ashift;
28168 : break;
28169 27 : case ASHIFTRT:
28170 27 : matrix = matrix_ashiftrt;
28171 27 : break;
28172 30 : case LSHIFTRT:
28173 30 : matrix = matrix_lshiftrt;
28174 30 : break;
28175 34 : case ROTATE:
28176 34 : matrix = matrix_rotate;
28177 34 : break;
28178 35 : case ROTATERT:
28179 35 : matrix = matrix_rotatert;
28180 35 : break;
28181 0 : default:
28182 0 : gcc_unreachable ();
28183 : }
28184 :
28185 202 : int nelts = GET_MODE_NUNITS (mode);
28186 202 : rtvec vec = rtvec_alloc (nelts);
28187 202 : uint64_t ma = matrix[shift];
28188 6922 : for (int i = 0; i < nelts; i++)
28189 6720 : RTVEC_ELT (vec, i) = gen_int_mode ((ma >> ((i % 8) * 8)) & 0xff, QImode);
28190 :
28191 202 : return force_reg (mode, gen_rtx_CONST_VECTOR (mode, vec));
28192 : }
28193 :
28194 : /* Trunc a vector to a narrow vector, like v4di -> v4si. */
28195 :
28196 : void
28197 62 : ix86_expand_trunc_with_avx2_noavx512f (rtx output, rtx input, machine_mode cvt_mode)
28198 : {
28199 62 : machine_mode out_mode = GET_MODE (output);
28200 62 : machine_mode in_mode = GET_MODE (input);
28201 62 : int len = GET_MODE_SIZE (in_mode);
28202 248 : gcc_assert (len == GET_MODE_SIZE (cvt_mode)
28203 : && GET_MODE_INNER (out_mode) == GET_MODE_INNER (cvt_mode)
28204 : && (REG_P (input) || SUBREG_P (input)));
28205 62 : scalar_mode inner_out_mode = GET_MODE_INNER (out_mode);
28206 124 : int in_innersize = GET_MODE_SIZE (GET_MODE_INNER (in_mode));
28207 62 : int out_innersize = GET_MODE_SIZE (inner_out_mode);
28208 :
28209 62 : struct expand_vec_perm_d d;
28210 62 : d.target = gen_reg_rtx (cvt_mode);
28211 62 : d.op0 = lowpart_subreg (cvt_mode, force_reg(in_mode, input), in_mode);
28212 62 : d.op1 = d.op0;
28213 62 : d.vmode = cvt_mode;
28214 62 : d.nelt = GET_MODE_NUNITS (cvt_mode);
28215 62 : d.testing_p = false;
28216 62 : d.one_operand_p = true;
28217 :
28218 : /* Init perm. Put the needed bits of input in order and
28219 : fill the rest of bits by default. */
28220 678 : for (int i = 0; i < d.nelt; ++i)
28221 : {
28222 616 : d.perm[i] = i;
28223 1232 : if (i < GET_MODE_NUNITS (out_mode))
28224 242 : d.perm[i] = i * (in_innersize / out_innersize);
28225 : }
28226 :
28227 62 : bool ok = ix86_expand_vec_perm_const_1(&d);
28228 62 : gcc_assert (ok);
28229 62 : emit_move_insn (output, gen_lowpart (out_mode, d.target));
28230 62 : }
28231 :
28232 : /* Implement truncv8sfv8bf2 with vector permutation. */
28233 : void
28234 8 : ix86_expand_vector_sf2bf_with_vec_perm (rtx dest, rtx src)
28235 : {
28236 8 : machine_mode vperm_mode, src_mode = GET_MODE (src);
28237 8 : switch (src_mode)
28238 : {
28239 : case V16SFmode:
28240 : vperm_mode = V32BFmode;
28241 : break;
28242 2 : case V8SFmode:
28243 2 : vperm_mode = V16BFmode;
28244 2 : break;
28245 4 : case V4SFmode:
28246 4 : vperm_mode = V8BFmode;
28247 4 : break;
28248 0 : default:
28249 0 : gcc_unreachable ();
28250 : }
28251 :
28252 8 : int nelt = GET_MODE_NUNITS (vperm_mode);
28253 8 : vec_perm_builder sel (nelt, nelt, 1);
28254 8 : sel.quick_grow (nelt);
28255 136 : for (int i = 0; i != nelt; i++)
28256 128 : sel[i] = (2 * i + 1) % nelt;
28257 16 : vec_perm_indices indices (sel, 1, nelt);
28258 :
28259 8 : rtx target = gen_reg_rtx (vperm_mode);
28260 8 : rtx op0 = lowpart_subreg (vperm_mode,
28261 : force_reg (src_mode, src),
28262 : src_mode);
28263 8 : bool ok = targetm.vectorize.vec_perm_const (vperm_mode, vperm_mode,
28264 : target, op0, op0, indices);
28265 8 : gcc_assert (ok);
28266 8 : emit_move_insn (dest, lowpart_subreg (GET_MODE (dest), target, vperm_mode));
28267 8 : }
28268 :
28269 : /* Implement extendv8bf2v8sf2 with vector permutation. */
28270 : void
28271 8 : ix86_expand_vector_bf2sf_with_vec_perm (rtx dest, rtx src)
28272 : {
28273 8 : machine_mode vperm_mode, src_mode = GET_MODE (src);
28274 8 : switch (src_mode)
28275 : {
28276 : case V16BFmode:
28277 : vperm_mode = V32BFmode;
28278 : break;
28279 2 : case V8BFmode:
28280 2 : vperm_mode = V16BFmode;
28281 2 : break;
28282 4 : case V4BFmode:
28283 4 : vperm_mode = V8BFmode;
28284 4 : break;
28285 0 : default:
28286 0 : gcc_unreachable ();
28287 : }
28288 :
28289 8 : int nelt = GET_MODE_NUNITS (vperm_mode);
28290 8 : vec_perm_builder sel (nelt, nelt, 1);
28291 8 : sel.quick_grow (nelt);
28292 136 : for (int i = 0, k = 0, j = nelt; i != nelt; i++)
28293 128 : sel[i] = i & 1 ? j++ : k++;
28294 :
28295 16 : vec_perm_indices indices (sel, 2, nelt);
28296 :
28297 8 : rtx target = gen_reg_rtx (vperm_mode);
28298 8 : rtx op1 = lowpart_subreg (vperm_mode,
28299 : force_reg (src_mode, src),
28300 : src_mode);
28301 8 : rtx op0 = CONST0_RTX (vperm_mode);
28302 8 : bool ok = targetm.vectorize.vec_perm_const (vperm_mode, vperm_mode,
28303 : target, op0, op1, indices);
28304 8 : gcc_assert (ok);
28305 8 : emit_move_insn (dest, lowpart_subreg (GET_MODE (dest), target, vperm_mode));
28306 8 : }
28307 :
28308 : /* Implement bitreverse<mode>2 using gf2p8affineqb. */
28309 :
28310 : void
28311 5 : ix86_expand_gfni_bitreverse (rtx dest, rtx src)
28312 : {
28313 5 : machine_mode mode = GET_MODE (dest);
28314 5 : rtx temp;
28315 10 : if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
28316 : {
28317 1 : rtx temp1 = gen_reg_rtx (mode == TImode ? V2DImode : V4SImode);
28318 1 : rtx temp2 = gen_reg_rtx (mode == TImode ? V2DImode : V4SImode);
28319 1 : if (mode == TImode)
28320 : {
28321 1 : temp = lowpart_subreg (DImode, src, TImode);
28322 1 : emit_insn (gen_rtx_SET (temp1, gen_rtx_VEC_CONCAT (V2DImode, temp,
28323 : const0_rtx)));
28324 1 : temp = gen_highpart (DImode, src);
28325 1 : emit_insn (gen_rtx_SET (temp2, gen_rtx_VEC_CONCAT (V2DImode, temp,
28326 : const0_rtx)));
28327 : }
28328 : else
28329 : {
28330 0 : temp = lowpart_subreg (SImode, src, DImode);
28331 0 : emit_insn (gen_vec_setv4si_0 (temp1, CONST0_RTX (V4SImode), temp));
28332 0 : temp = gen_highpart (SImode, src);
28333 0 : emit_insn (gen_vec_setv4si_0 (temp2, CONST0_RTX (V4SImode), temp));
28334 0 : temp1 = lowpart_subreg (V2DImode, temp1, V4SImode);
28335 0 : temp2 = lowpart_subreg (V2DImode, temp2, V4SImode);
28336 : }
28337 1 : temp = gen_reg_rtx (V2DImode);
28338 1 : emit_insn (gen_vec_interleave_lowv2di (temp, temp1, temp2));
28339 : }
28340 4 : else if (mode != DImode)
28341 : {
28342 3 : if (mode != SImode)
28343 : {
28344 2 : src = force_reg (mode, src);
28345 2 : src = lowpart_subreg (SImode, src, mode);
28346 : }
28347 3 : temp = gen_reg_rtx (V4SImode);
28348 3 : emit_insn (gen_vec_setv4si_0 (temp, CONST0_RTX (V4SImode), src));
28349 : }
28350 : else
28351 : {
28352 1 : temp = gen_reg_rtx (V2DImode);
28353 1 : emit_insn (gen_rtx_SET (temp, gen_rtx_VEC_CONCAT (V2DImode, src,
28354 : const0_rtx)));
28355 : }
28356 5 : src = temp;
28357 5 : temp = gen_reg_rtx (V16QImode);
28358 5 : rtx src2 = gen_rtx_CONST_VECTOR (V16QImode,
28359 : gen_rtvec (16, GEN_INT (1), GEN_INT (2),
28360 : GEN_INT (4), GEN_INT (8),
28361 : GEN_INT (16), GEN_INT (32),
28362 : GEN_INT (64), GEN_INT (-128),
28363 : GEN_INT (1), GEN_INT (2),
28364 : GEN_INT (4), GEN_INT (8),
28365 : GEN_INT (16), GEN_INT (32),
28366 : GEN_INT (64), GEN_INT (-128)));
28367 5 : src2 = validize_mem (force_const_mem (V16QImode, src2));
28368 5 : src = lowpart_subreg (V16QImode, src, GET_MODE (src));
28369 5 : emit_insn (gen_vgf2p8affineqb_v16qi (temp, src, src2, const0_rtx));
28370 5 : if (mode == QImode)
28371 : {
28372 1 : rtx temp1 = gen_reg_rtx (SImode);
28373 1 : rtx temp2 = lowpart_subreg (V4SImode, temp, V16QImode);
28374 1 : rtx temp3 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
28375 1 : emit_insn (gen_rtx_SET (temp1,
28376 : gen_rtx_VEC_SELECT (SImode, temp2, temp3)));
28377 1 : emit_move_insn (dest, lowpart_subreg (QImode, temp1, SImode));
28378 1 : return;
28379 : }
28380 11 : rtx target = gen_reg_rtx ((GET_MODE_SIZE (mode) < 4 || !TARGET_64BIT)
28381 3 : ? SImode : mode == TImode ? DImode : mode);
28382 4 : emit_move_insn (target, lowpart_subreg (GET_MODE (target), temp, V16QImode));
28383 8 : if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
28384 : {
28385 1 : rtx temp1 = gen_reg_rtx (GET_MODE (target));
28386 1 : if (mode == TImode || TARGET_SSE4_1)
28387 : {
28388 1 : rtx temp2 = lowpart_subreg (mode == TImode ? V2DImode : V4SImode,
28389 : temp, V16QImode);
28390 1 : rtx temp3 = gen_rtx_PARALLEL (VOIDmode,
28391 : gen_rtvec (1, GEN_INT (mode == TImode
28392 : ? 1 : 2)));
28393 1 : emit_insn (gen_rtx_SET (temp1,
28394 : gen_rtx_VEC_SELECT (GET_MODE (target), temp2,
28395 : temp3)));
28396 1 : }
28397 : else
28398 : {
28399 0 : rtx temp2 = gen_reg_rtx (V4SImode);
28400 0 : rtx temp3 = lowpart_subreg (V4SImode, temp, V16QImode);
28401 0 : emit_insn (gen_sse2_pshufd (temp2, temp3, GEN_INT (0xaa)));
28402 0 : emit_move_insn (temp1, lowpart_subreg (GET_MODE (target), temp2,
28403 : V4SImode));
28404 : }
28405 1 : rtx temp4 = gen_reg_rtx (GET_MODE (target));
28406 1 : rtx temp5 = gen_reg_rtx (GET_MODE (target));
28407 0 : rtx (*gen_bswap) (rtx, rtx)
28408 1 : = mode == TImode ? gen_bswapdi2 : gen_bswapsi2;
28409 1 : emit_insn (gen_bswap (temp4, target));
28410 1 : emit_insn (gen_bswap (temp5, temp1));
28411 1 : temp4 = gen_rtx_ZERO_EXTEND (mode, temp4);
28412 1 : temp5 = gen_rtx_ZERO_EXTEND (mode, temp5);
28413 1 : rtx shift = GEN_INT (GET_MODE_PRECISION (GET_MODE (target)));
28414 1 : temp4 = gen_rtx_ASHIFT (mode, temp4, shift);
28415 1 : emit_insn (gen_rtx_SET (dest, gen_rtx_IOR (mode, temp4, temp5)));
28416 1 : return;
28417 : }
28418 3 : if (mode == HImode)
28419 1 : target = lowpart_subreg (mode, target, SImode);
28420 3 : if (mode == SImode)
28421 1 : emit_insn (gen_bswapsi2 (dest, target));
28422 : else
28423 2 : emit_insn (gen_rtx_SET (dest, gen_rtx_BSWAP (mode, target)));
28424 : }
28425 :
28426 : /* Expand LCP stall or long immediate peephole for INSN. Use the
28427 : previous scratch register if possible. If USE_XOR is true,
28428 : generate "*movsi_xor". */
28429 :
28430 : void
28431 31347 : ix86_expand_lcp_stall_peephole (rtx_insn *insn, rtx *operands,
28432 : bool use_xor)
28433 : {
28434 31347 : rtx imm, scratch;
28435 31347 : machine_mode mode = GET_MODE (operands[0]);
28436 :
28437 : /* Get the immediate operand and the allocated scratch register. */
28438 31347 : if (use_xor)
28439 : {
28440 10493 : imm = const0_rtx;
28441 10493 : scratch = operands[1];
28442 : }
28443 : else
28444 : {
28445 20854 : imm = operands[1];
28446 20854 : scratch = operands[2];
28447 : }
28448 :
28449 31347 : df_ref def;
28450 31347 : rtx_insn *prev, *prev_insn = nullptr;
28451 31347 : rtx set, prev_scratch = nullptr;
28452 :
28453 : /* Scan backward for the previous scratch register def with IMM in
28454 : the same basic block. */
28455 31347 : basic_block bb = BLOCK_FOR_INSN (insn);
28456 955497 : for (prev = PREV_INSN (insn);
28457 955497 : prev != BB_HEAD (bb);
28458 924150 : prev = PREV_INSN (prev))
28459 : {
28460 928937 : if (NONDEBUG_INSN_P (prev))
28461 2072688 : FOR_EACH_INSN_DEF (def, prev)
28462 1745806 : if (!DF_REF_IS_ARTIFICIAL (def)
28463 1745806 : && !DF_REF_FLAGS_IS_SET (def, DF_REF_MAY_CLOBBER)
28464 174862 : && !DF_REF_FLAGS_IS_SET (def, DF_REF_MUST_CLOBBER))
28465 : {
28466 154272 : rtx reg = DF_REF_REG (def);
28467 154272 : if (HARD_REGISTER_P (reg) && REGNO (reg) != FLAGS_REG)
28468 : {
28469 153049 : set = single_set (prev);
28470 153049 : if (!set)
28471 479 : continue;
28472 :
28473 152570 : rtx dest = SET_DEST (set);
28474 :
28475 : /* Reject DEST if a register is not wide enough to
28476 : supply MODE or invalid for QImode. */
28477 133333 : if (!GENERAL_REG_P (dest)
28478 229682 : || (GET_MODE_SIZE (GET_MODE (dest))
28479 114841 : < GET_MODE_SIZE (mode))
28480 266277 : || (mode == QImode
28481 0 : && !ANY_QI_REGNO_P (REGNO (dest))))
28482 38863 : continue;
28483 :
28484 113707 : rtx src = SET_SRC (set);
28485 113707 : if (rtx_equal_p (src, imm))
28486 : {
28487 : /* A previous scratch register is found. */
28488 : prev_scratch = dest;
28489 : prev_insn = prev;
28490 : break;
28491 : }
28492 : }
28493 : }
28494 :
28495 928937 : if (prev_scratch)
28496 : break;
28497 : }
28498 :
28499 31347 : if (prev_scratch)
28500 : {
28501 : /* The previous scratch register is unusable if it is set between
28502 : PREV_INSN and INSN. */
28503 4787 : unsigned int regno = REGNO (prev_scratch);
28504 :
28505 : /* Scan backward for the previous scratch register def. */
28506 4787 : for (prev = PREV_INSN (insn);
28507 131544 : prev != prev_insn;
28508 126757 : prev = PREV_INSN (prev))
28509 : {
28510 128217 : if (NONDEBUG_INSN_P (prev))
28511 265425 : FOR_EACH_INSN_DEF (def, prev)
28512 187424 : if (HARD_REGISTER_P (DF_REF_REAL_REG (def))
28513 187424 : && DF_REF_REGNO (def) == regno)
28514 : {
28515 : /* Since the previous scratch register is set, it is
28516 : unusable. */
28517 1460 : if (dump_file)
28518 : {
28519 0 : fprintf (dump_file,
28520 : "\nThe previous scratch register:\n\n");
28521 0 : print_rtl_single (dump_file, prev_scratch);
28522 0 : fprintf (dump_file, "\nset in:\n\n");
28523 0 : print_rtl_single (dump_file, prev_insn);
28524 0 : fprintf (dump_file,
28525 : "\nis unusable by:\n\n");
28526 0 : print_rtl_single (dump_file, insn);
28527 0 : fprintf (dump_file,
28528 : "\nsince it is overridden by:\n\n");
28529 0 : print_rtl_single (dump_file, prev);
28530 0 : fprintf (dump_file, "\n");
28531 : }
28532 : prev_scratch = nullptr;
28533 : break;
28534 : }
28535 :
28536 126757 : if (!prev_scratch)
28537 : break;
28538 : }
28539 :
28540 4787 : if (prev_scratch)
28541 : {
28542 : /* Ignore the allocated scratch register and use the previous
28543 : scratch register. */
28544 3327 : if (dump_file)
28545 : {
28546 0 : fprintf (dump_file,
28547 : "\nIgnore the allocated scratch register:\n\n");
28548 0 : print_rtl_single (dump_file, scratch);
28549 0 : fprintf (dump_file,
28550 : "\nand use the previous scratch register:\n\n");
28551 0 : print_rtl_single (dump_file, prev_scratch);
28552 0 : fprintf (dump_file, "\nset in:\n\n");
28553 0 : print_rtl_single (dump_file, prev_insn);
28554 0 : fprintf (dump_file, "\nfor:\n\n");
28555 0 : print_rtl_single (dump_file, insn);
28556 0 : fprintf (dump_file, "\n");
28557 : }
28558 3327 : scratch = gen_lowpart (mode, prev_scratch);
28559 3327 : set = gen_rtx_SET (operands[0], scratch);
28560 3327 : emit_insn (set);
28561 3327 : return;
28562 : }
28563 : }
28564 :
28565 : /* If there is no usable previous scratch register, use the allocated
28566 : scratch register. */
28567 28020 : if (use_xor)
28568 : {
28569 : /* Generate "*movsi_xor". */
28570 8976 : rtx x = gen_lowpart (SImode, scratch);
28571 8976 : rtvec p = rtvec_alloc (2);
28572 8976 : set = gen_rtx_SET (x, const0_rtx);
28573 8976 : RTVEC_ELT (p, 0) = set;
28574 8976 : rtx clobber = gen_rtx_REG (CCmode, FLAGS_REG);
28575 8976 : RTVEC_ELT (p, 1) = gen_rtx_CLOBBER (VOIDmode, clobber);
28576 8976 : set = gen_rtx_PARALLEL (VOIDmode, p);
28577 : }
28578 : else
28579 19044 : set = gen_rtx_SET (scratch, imm);
28580 28020 : emit_insn (set);
28581 28020 : set = gen_rtx_SET (operands[0], scratch);
28582 28020 : emit_insn (set);
28583 : }
28584 :
28585 : #include "gt-i386-expand.h"
|
