Line data Source code
1 : /* Copyright (C) 1988-2026 Free Software Foundation, Inc.
2 :
3 : This file is part of GCC.
4 :
5 : GCC is free software; you can redistribute it and/or modify
6 : it under the terms of the GNU General Public License as published by
7 : the Free Software Foundation; either version 3, or (at your option)
8 : any later version.
9 :
10 : GCC is distributed in the hope that it will be useful,
11 : but WITHOUT ANY WARRANTY; without even the implied warranty of
12 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 : GNU General Public License for more details.
14 :
15 : You should have received a copy of the GNU General Public License
16 : along with GCC; see the file COPYING3. If not see
17 : <http://www.gnu.org/licenses/>. */
18 :
19 : #define IN_TARGET_CODE 1
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "memmodel.h"
28 : #include "gimple.h"
29 : #include "cfghooks.h"
30 : #include "cfgloop.h"
31 : #include "df.h"
32 : #include "tm_p.h"
33 : #include "stringpool.h"
34 : #include "expmed.h"
35 : #include "optabs.h"
36 : #include "regs.h"
37 : #include "emit-rtl.h"
38 : #include "recog.h"
39 : #include "cgraph.h"
40 : #include "diagnostic.h"
41 : #include "cfgbuild.h"
42 : #include "alias.h"
43 : #include "fold-const.h"
44 : #include "attribs.h"
45 : #include "calls.h"
46 : #include "stor-layout.h"
47 : #include "varasm.h"
48 : #include "output.h"
49 : #include "insn-attr.h"
50 : #include "flags.h"
51 : #include "except.h"
52 : #include "explow.h"
53 : #include "expr.h"
54 : #include "cfgrtl.h"
55 : #include "common/common-target.h"
56 : #include "langhooks.h"
57 : #include "reload.h"
58 : #include "gimplify.h"
59 : #include "dwarf2.h"
60 : #include "tm-constrs.h"
61 : #include "cselib.h"
62 : #include "sched-int.h"
63 : #include "opts.h"
64 : #include "tree-pass.h"
65 : #include "context.h"
66 : #include "pass_manager.h"
67 : #include "target-globals.h"
68 : #include "gimple-iterator.h"
69 : #include "shrink-wrap.h"
70 : #include "builtins.h"
71 : #include "rtl-iter.h"
72 : #include "tree-iterator.h"
73 : #include "dbgcnt.h"
74 : #include "case-cfn-macros.h"
75 : #include "dojump.h"
76 : #include "fold-const-call.h"
77 : #include "tree-vrp.h"
78 : #include "tree-ssanames.h"
79 : #include "selftest.h"
80 : #include "selftest-rtl.h"
81 : #include "print-rtl.h"
82 : #include "intl.h"
83 : #include "ifcvt.h"
84 : #include "symbol-summary.h"
85 : #include "sreal.h"
86 : #include "ipa-cp.h"
87 : #include "ipa-prop.h"
88 : #include "ipa-fnsummary.h"
89 : #include "wide-int-bitmask.h"
90 : #include "tree-vector-builder.h"
91 : #include "debug.h"
92 : #include "dwarf2out.h"
93 : #include "function-abi.h"
94 : #include "i386-builtins.h"
95 : #include "i386-features.h"
96 : #include "i386-expand.h"
97 :
98 : const char * const xlogue_layout::STUB_BASE_NAMES[XLOGUE_STUB_COUNT] = {
99 : "savms64",
100 : "resms64",
101 : "resms64x",
102 : "savms64f",
103 : "resms64f",
104 : "resms64fx"
105 : };
106 :
107 : const unsigned xlogue_layout::REG_ORDER[xlogue_layout::MAX_REGS] = {
108 : /* The below offset values are where each register is stored for the layout
109 : relative to incoming stack pointer. The value of each m_regs[].offset will
110 : be relative to the incoming base pointer (rax or rsi) used by the stub.
111 :
112 : s_instances: 0 1 2 3
113 : Offset: realigned or aligned + 8
114 : Register aligned aligned + 8 aligned w/HFP w/HFP */
115 : XMM15_REG, /* 0x10 0x18 0x10 0x18 */
116 : XMM14_REG, /* 0x20 0x28 0x20 0x28 */
117 : XMM13_REG, /* 0x30 0x38 0x30 0x38 */
118 : XMM12_REG, /* 0x40 0x48 0x40 0x48 */
119 : XMM11_REG, /* 0x50 0x58 0x50 0x58 */
120 : XMM10_REG, /* 0x60 0x68 0x60 0x68 */
121 : XMM9_REG, /* 0x70 0x78 0x70 0x78 */
122 : XMM8_REG, /* 0x80 0x88 0x80 0x88 */
123 : XMM7_REG, /* 0x90 0x98 0x90 0x98 */
124 : XMM6_REG, /* 0xa0 0xa8 0xa0 0xa8 */
125 : SI_REG, /* 0xa8 0xb0 0xa8 0xb0 */
126 : DI_REG, /* 0xb0 0xb8 0xb0 0xb8 */
127 : BX_REG, /* 0xb8 0xc0 0xb8 0xc0 */
128 : BP_REG, /* 0xc0 0xc8 N/A N/A */
129 : R12_REG, /* 0xc8 0xd0 0xc0 0xc8 */
130 : R13_REG, /* 0xd0 0xd8 0xc8 0xd0 */
131 : R14_REG, /* 0xd8 0xe0 0xd0 0xd8 */
132 : R15_REG, /* 0xe0 0xe8 0xd8 0xe0 */
133 : };
134 :
135 : /* Instantiate static const values. */
136 : const HOST_WIDE_INT xlogue_layout::STUB_INDEX_OFFSET;
137 : const unsigned xlogue_layout::MIN_REGS;
138 : const unsigned xlogue_layout::MAX_REGS;
139 : const unsigned xlogue_layout::MAX_EXTRA_REGS;
140 : const unsigned xlogue_layout::VARIANT_COUNT;
141 : const unsigned xlogue_layout::STUB_NAME_MAX_LEN;
142 :
143 : /* Initialize xlogue_layout::s_stub_names to zero. */
144 : char xlogue_layout::s_stub_names[2][XLOGUE_STUB_COUNT][VARIANT_COUNT]
145 : [STUB_NAME_MAX_LEN];
146 :
147 : /* Instantiates all xlogue_layout instances. */
148 : const xlogue_layout xlogue_layout::s_instances[XLOGUE_SET_COUNT] = {
149 : xlogue_layout (0, false),
150 : xlogue_layout (8, false),
151 : xlogue_layout (0, true),
152 : xlogue_layout (8, true)
153 : };
154 :
155 : /* Return an appropriate const instance of xlogue_layout based upon values
156 : in cfun->machine and crtl. */
157 : const class xlogue_layout &
158 49891 : xlogue_layout::get_instance ()
159 : {
160 49891 : enum xlogue_stub_sets stub_set;
161 49891 : bool aligned_plus_8 = cfun->machine->call_ms2sysv_pad_in;
162 :
163 49891 : if (stack_realign_fp)
164 : stub_set = XLOGUE_SET_HFP_ALIGNED_OR_REALIGN;
165 40910 : else if (frame_pointer_needed)
166 25246 : stub_set = aligned_plus_8
167 31552 : ? XLOGUE_SET_HFP_ALIGNED_PLUS_8
168 : : XLOGUE_SET_HFP_ALIGNED_OR_REALIGN;
169 : else
170 9358 : stub_set = aligned_plus_8 ? XLOGUE_SET_ALIGNED_PLUS_8 : XLOGUE_SET_ALIGNED;
171 :
172 49891 : return s_instances[stub_set];
173 : }
174 :
175 : /* Determine how many clobbered registers can be saved by the stub.
176 : Returns the count of registers the stub will save and restore. */
177 : unsigned
178 35225 : xlogue_layout::count_stub_managed_regs ()
179 : {
180 35225 : bool hfp = frame_pointer_needed || stack_realign_fp;
181 35225 : unsigned i, count;
182 35225 : unsigned regno;
183 :
184 93950 : for (count = i = MIN_REGS; i < MAX_REGS; ++i)
185 : {
186 92750 : regno = REG_ORDER[i];
187 92750 : if (regno == BP_REG && hfp)
188 17880 : continue;
189 74870 : if (!ix86_save_reg (regno, false, false))
190 : break;
191 40845 : ++count;
192 : }
193 35225 : return count;
194 : }
195 :
196 : /* Determine if register REGNO is a stub managed register given the
197 : total COUNT of stub managed registers. */
198 : bool
199 2642240 : xlogue_layout::is_stub_managed_reg (unsigned regno, unsigned count)
200 : {
201 2642240 : bool hfp = frame_pointer_needed || stack_realign_fp;
202 2642240 : unsigned i;
203 :
204 34418160 : for (i = 0; i < count; ++i)
205 : {
206 32275013 : gcc_assert (i < MAX_REGS);
207 32275013 : if (REG_ORDER[i] == BP_REG && hfp)
208 510254 : ++count;
209 31764759 : else if (REG_ORDER[i] == regno)
210 : return true;
211 : }
212 : return false;
213 : }
214 :
215 : /* Constructor for xlogue_layout. */
216 1164804 : xlogue_layout::xlogue_layout (HOST_WIDE_INT stack_align_off_in, bool hfp)
217 1164804 : : m_hfp (hfp) , m_nregs (hfp ? 17 : 18),
218 1164804 : m_stack_align_off_in (stack_align_off_in)
219 : {
220 1164804 : HOST_WIDE_INT offset = stack_align_off_in;
221 1164804 : unsigned i, j;
222 :
223 22131276 : for (i = j = 0; i < MAX_REGS; ++i)
224 : {
225 20966472 : unsigned regno = REG_ORDER[i];
226 :
227 20966472 : if (regno == BP_REG && hfp)
228 582402 : continue;
229 20384070 : if (SSE_REGNO_P (regno))
230 : {
231 11648040 : offset += 16;
232 : /* Verify that SSE regs are always aligned. */
233 11648040 : gcc_assert (!((stack_align_off_in + offset) & 15));
234 : }
235 : else
236 8736030 : offset += 8;
237 :
238 20384070 : m_regs[j].regno = regno;
239 20384070 : m_regs[j++].offset = offset - STUB_INDEX_OFFSET;
240 : }
241 1164804 : gcc_assert (j == m_nregs);
242 1164804 : }
243 :
244 : const char *
245 14666 : xlogue_layout::get_stub_name (enum xlogue_stub stub,
246 : unsigned n_extra_regs)
247 : {
248 14666 : const int have_avx = TARGET_AVX;
249 14666 : char *name = s_stub_names[!!have_avx][stub][n_extra_regs];
250 :
251 : /* Lazy init */
252 14666 : if (!*name)
253 : {
254 362 : int res = snprintf (name, STUB_NAME_MAX_LEN, "__%s_%s_%u",
255 : (have_avx ? "avx" : "sse"),
256 181 : STUB_BASE_NAMES[stub],
257 : MIN_REGS + n_extra_regs);
258 181 : gcc_checking_assert (res < (int)STUB_NAME_MAX_LEN);
259 : }
260 :
261 14666 : return name;
262 : }
263 :
264 : /* Return rtx of a symbol ref for the entry point (based upon
265 : cfun->machine->call_ms2sysv_extra_regs) of the specified stub. */
266 : rtx
267 14666 : xlogue_layout::get_stub_rtx (enum xlogue_stub stub)
268 : {
269 14666 : const unsigned n_extra_regs = cfun->machine->call_ms2sysv_extra_regs;
270 14666 : gcc_checking_assert (n_extra_regs <= MAX_EXTRA_REGS);
271 14666 : gcc_assert (stub < XLOGUE_STUB_COUNT);
272 14666 : gcc_assert (crtl->stack_realign_finalized);
273 :
274 14666 : return gen_rtx_SYMBOL_REF (Pmode, get_stub_name (stub, n_extra_regs));
275 : }
276 :
277 : unsigned scalar_chain::max_id = 0;
278 :
279 : namespace {
280 :
281 : /* Initialize new chain. */
282 :
283 6545970 : scalar_chain::scalar_chain (enum machine_mode smode_, enum machine_mode vmode_)
284 : {
285 6545970 : smode = smode_;
286 6545970 : vmode = vmode_;
287 :
288 6545970 : chain_id = ++max_id;
289 :
290 6545970 : if (dump_file)
291 136 : fprintf (dump_file, "Created a new instruction chain #%d\n", chain_id);
292 :
293 6545970 : bitmap_obstack_initialize (NULL);
294 6545970 : insns = BITMAP_ALLOC (NULL);
295 6545970 : defs = BITMAP_ALLOC (NULL);
296 6545970 : defs_conv = BITMAP_ALLOC (NULL);
297 6545970 : insns_conv = BITMAP_ALLOC (NULL);
298 6545970 : queue = NULL;
299 :
300 6545970 : cost_sse_integer = 0;
301 6545970 : weighted_cost_sse_integer = 0 ;
302 6545970 : max_visits = x86_stv_max_visits;
303 6545970 : }
304 :
305 : /* Free chain's data. */
306 :
307 6545970 : scalar_chain::~scalar_chain ()
308 : {
309 6545970 : BITMAP_FREE (insns);
310 6545970 : BITMAP_FREE (defs);
311 6545970 : BITMAP_FREE (defs_conv);
312 6545970 : BITMAP_FREE (insns_conv);
313 6545970 : bitmap_obstack_release (NULL);
314 6545970 : }
315 :
316 : /* Add instruction into chains' queue. */
317 :
318 : void
319 8426813 : scalar_chain::add_to_queue (unsigned insn_uid)
320 : {
321 8426813 : if (!bitmap_set_bit (queue, insn_uid))
322 : return;
323 :
324 6329949 : if (dump_file)
325 141 : fprintf (dump_file, " Adding insn %d into chain's #%d queue\n",
326 : insn_uid, chain_id);
327 : }
328 :
329 : /* For DImode conversion, mark register defined by DEF as requiring
330 : conversion. */
331 :
332 : void
333 9571876 : scalar_chain::mark_dual_mode_def (df_ref def)
334 : {
335 9571876 : gcc_assert (DF_REF_REG_DEF_P (def));
336 :
337 : /* Record the def/insn pair so we can later efficiently iterate over
338 : the defs to convert on insns not in the chain. */
339 9571876 : bool reg_new = bitmap_set_bit (defs_conv, DF_REF_REGNO (def));
340 9571876 : basic_block bb = BLOCK_FOR_INSN (DF_REF_INSN (def));
341 9571876 : profile_count entry_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
342 9571876 : bool speed_p = optimize_bb_for_speed_p (bb);
343 9571876 : int cost = 0;
344 :
345 9571876 : if (!bitmap_bit_p (insns, DF_REF_INSN_UID (def)))
346 : {
347 2856779 : if (!bitmap_set_bit (insns_conv, DF_REF_INSN_UID (def))
348 2856779 : && !reg_new)
349 1468340 : return;
350 :
351 : /* Cost integer to sse moves. */
352 2529964 : if (speed_p)
353 2237332 : cost = COSTS_N_INSNS (ix86_cost->integer_to_sse) / 2;
354 292632 : else if (TARGET_64BIT || smode == SImode)
355 : cost = COSTS_N_BYTES (4);
356 : /* vmovd (4 bytes) + vpinsrd (6 bytes). */
357 18654 : else if (TARGET_SSE4_1)
358 : cost = COSTS_N_BYTES (10);
359 : /* movd (4 bytes) + movd (4 bytes) + unpckldq (4 bytes). */
360 : else
361 8103536 : cost = COSTS_N_BYTES (12);
362 : }
363 : else
364 : {
365 6715097 : if (!reg_new)
366 : return;
367 :
368 : /* Cost sse to integer moves. */
369 5573572 : if (speed_p)
370 5000280 : cost = COSTS_N_INSNS (ix86_cost->sse_to_integer) / 2;
371 573292 : else if (TARGET_64BIT || smode == SImode)
372 : cost = COSTS_N_BYTES (4);
373 : /* vmovd (4 bytes) + vpextrd (6 bytes). */
374 2978 : else if (TARGET_SSE4_1)
375 : cost = COSTS_N_BYTES (10);
376 : /* movd (4 bytes) + psrlq (5 bytes) + movd (4 bytes). */
377 : else
378 8103536 : cost = COSTS_N_BYTES (13);
379 : }
380 :
381 8103536 : if (speed_p)
382 7237612 : weighted_cost_sse_integer += bb->count.to_sreal_scale (entry_count) * cost;
383 :
384 8103536 : cost_sse_integer += cost;
385 :
386 8103536 : if (dump_file)
387 240 : fprintf (dump_file,
388 : " Mark r%d def in insn %d as requiring both modes in chain #%d\n",
389 240 : DF_REF_REGNO (def), DF_REF_INSN_UID (def), chain_id);
390 : }
391 :
392 : /* Check REF's chain to add new insns into a queue
393 : and find registers requiring conversion. Return true if OK, false
394 : if the analysis was aborted. */
395 :
396 : bool
397 18116783 : scalar_chain::analyze_register_chain (bitmap candidates, df_ref ref,
398 : bitmap disallowed)
399 : {
400 18116783 : df_link *chain;
401 18116783 : bool mark_def = false;
402 :
403 18116783 : gcc_checking_assert (bitmap_bit_p (insns, DF_REF_INSN_UID (ref)));
404 :
405 62856155 : for (chain = DF_REF_CHAIN (ref); chain; chain = chain->next)
406 : {
407 44743260 : unsigned uid = DF_REF_INSN_UID (chain->ref);
408 :
409 44743260 : if (!NONDEBUG_INSN_P (DF_REF_INSN (chain->ref)))
410 8087530 : continue;
411 :
412 36655730 : if (--max_visits == 0)
413 : return false;
414 :
415 36655151 : if (!DF_REF_REG_MEM_P (chain->ref))
416 : {
417 30700606 : if (bitmap_bit_p (insns, uid))
418 9723921 : continue;
419 :
420 20976685 : if (bitmap_bit_p (candidates, uid))
421 : {
422 8426813 : add_to_queue (uid);
423 8426813 : continue;
424 : }
425 :
426 : /* If we run into parts of an aborted chain discovery abort. */
427 12549872 : if (bitmap_bit_p (disallowed, uid))
428 : return false;
429 : }
430 :
431 18501108 : if (DF_REF_REG_DEF_P (chain->ref))
432 : {
433 2856779 : if (dump_file)
434 125 : fprintf (dump_file, " r%d def in insn %d isn't convertible\n",
435 : DF_REF_REGNO (chain->ref), uid);
436 2856779 : mark_dual_mode_def (chain->ref);
437 : }
438 : else
439 : {
440 15644329 : if (dump_file)
441 524 : fprintf (dump_file, " r%d use in insn %d isn't convertible\n",
442 : DF_REF_REGNO (chain->ref), uid);
443 : mark_def = true;
444 : }
445 : }
446 :
447 18112895 : if (mark_def)
448 6715097 : mark_dual_mode_def (ref);
449 :
450 : return true;
451 : }
452 :
453 : /* Check whether X is a convertible *concatditi_? variant. X is known
454 : to be any_or_plus:TI, i.e. PLUS:TI, IOR:TI or XOR:TI. */
455 :
456 : static bool
457 30752 : timode_concatdi_p (rtx x)
458 : {
459 30752 : rtx op0 = XEXP (x, 0);
460 30752 : rtx op1 = XEXP (x, 1);
461 :
462 30752 : if (GET_CODE (op1) == ASHIFT)
463 951 : std::swap (op0, op1);
464 :
465 30752 : return GET_CODE (op0) == ASHIFT
466 21778 : && GET_CODE (XEXP (op0, 0)) == ZERO_EXTEND
467 21778 : && GET_MODE (XEXP (XEXP (op0, 0), 0)) == DImode
468 21778 : && REG_P (XEXP (XEXP (op0, 0), 0))
469 21628 : && CONST_INT_P (XEXP (op0, 1))
470 21628 : && INTVAL (XEXP (op0, 1)) == 64
471 21628 : && GET_CODE (op1) == ZERO_EXTEND
472 20677 : && GET_MODE (XEXP (op1, 0)) == DImode
473 51429 : && REG_P (XEXP (op1, 0));
474 : }
475 :
476 :
477 : /* Add instruction into a chain. Return true if OK, false if the search
478 : was aborted. */
479 :
480 : bool
481 12871787 : scalar_chain::add_insn (bitmap candidates, unsigned int insn_uid,
482 : bitmap disallowed)
483 : {
484 12871787 : if (!bitmap_set_bit (insns, insn_uid))
485 : return true;
486 :
487 12871787 : if (dump_file)
488 277 : fprintf (dump_file, " Adding insn %d to chain #%d\n", insn_uid, chain_id);
489 :
490 12871787 : rtx_insn *insn = DF_INSN_UID_GET (insn_uid)->insn;
491 12871787 : rtx def_set = single_set (insn);
492 12871787 : if (def_set && REG_P (SET_DEST (def_set))
493 22769154 : && !HARD_REGISTER_P (SET_DEST (def_set)))
494 9872383 : bitmap_set_bit (defs, REGNO (SET_DEST (def_set)));
495 :
496 : /* ??? The following is quadratic since analyze_register_chain
497 : iterates over all refs to look for dual-mode regs. Instead this
498 : should be done separately for all regs mentioned in the chain once. */
499 12871787 : df_ref ref;
500 26269032 : for (ref = DF_INSN_UID_DEFS (insn_uid); ref; ref = DF_REF_NEXT_LOC (ref))
501 13398661 : if (!HARD_REGISTER_P (DF_REF_REG (ref)))
502 9872383 : if (!analyze_register_chain (candidates, ref, disallowed))
503 : return false;
504 :
505 : /* The operand(s) of VEC_SELECT, ZERO_EXTEND and similar ops don't need
506 : to be converted/convertible. */
507 12870371 : if (def_set)
508 12870371 : switch (GET_CODE (SET_SRC (def_set)))
509 : {
510 3844867 : case REG:
511 3844867 : if (HARD_REGISTER_P (SET_SRC (def_set)))
512 : return true;
513 : break;
514 : case VEC_SELECT:
515 : return true;
516 9957 : case ZERO_EXTEND:
517 9957 : if (GET_MODE (XEXP (SET_SRC (def_set), 0)) == DImode)
518 : return true;
519 : break;
520 2390971 : case PLUS:
521 2390971 : case IOR:
522 2390971 : case XOR:
523 2390971 : if (smode == TImode && timode_concatdi_p (SET_SRC (def_set)))
524 : return true;
525 : break;
526 : default:
527 : break;
528 : }
529 :
530 28168517 : for (ref = DF_INSN_UID_USES (insn_uid); ref; ref = DF_REF_NEXT_LOC (ref))
531 15369574 : if (DF_REF_TYPE (ref) == DF_REF_REG_USE
532 8244403 : && !SUBREG_P (DF_REF_REG (ref)))
533 8244400 : if (!analyze_register_chain (candidates, ref, disallowed))
534 : return false;
535 :
536 : return true;
537 : }
538 :
539 : /* Build new chain starting from insn INSN_UID recursively
540 : adding all dependent uses and definitions. Return true if OK, false
541 : if the chain discovery was aborted. */
542 :
543 : bool
544 6545970 : scalar_chain::build (bitmap candidates, unsigned insn_uid, bitmap disallowed)
545 : {
546 6545970 : queue = BITMAP_ALLOC (NULL);
547 6545970 : bitmap_set_bit (queue, insn_uid);
548 :
549 6545970 : if (dump_file)
550 136 : fprintf (dump_file, "Building chain #%d...\n", chain_id);
551 :
552 19413869 : while (!bitmap_empty_p (queue))
553 : {
554 12871787 : insn_uid = bitmap_first_set_bit (queue);
555 12871787 : bitmap_clear_bit (queue, insn_uid);
556 12871787 : bitmap_clear_bit (candidates, insn_uid);
557 12871787 : if (!add_insn (candidates, insn_uid, disallowed))
558 : {
559 : /* If we aborted the search put sofar found insn on the set of
560 : disallowed insns so that further searches reaching them also
561 : abort and thus we abort the whole but yet undiscovered chain. */
562 3888 : bitmap_ior_into (disallowed, insns);
563 3888 : if (dump_file)
564 0 : fprintf (dump_file, "Aborted chain #%d discovery\n", chain_id);
565 3888 : BITMAP_FREE (queue);
566 3888 : return false;
567 : }
568 : }
569 :
570 6542082 : if (dump_file)
571 : {
572 136 : fprintf (dump_file, "Collected chain #%d...\n", chain_id);
573 136 : fprintf (dump_file, " insns: ");
574 136 : dump_bitmap (dump_file, insns);
575 136 : if (!bitmap_empty_p (defs_conv))
576 : {
577 136 : bitmap_iterator bi;
578 136 : unsigned id;
579 136 : const char *comma = "";
580 136 : fprintf (dump_file, " defs to convert: ");
581 366 : EXECUTE_IF_SET_IN_BITMAP (defs_conv, 0, id, bi)
582 : {
583 230 : fprintf (dump_file, "%sr%d", comma, id);
584 230 : comma = ", ";
585 : }
586 136 : fprintf (dump_file, "\n");
587 : }
588 : }
589 :
590 6542082 : BITMAP_FREE (queue);
591 :
592 6542082 : return true;
593 : }
594 :
595 : /* Return a cost of building a vector constant
596 : instead of using a scalar one. */
597 :
598 : int
599 2664766 : general_scalar_chain::vector_const_cost (rtx exp, basic_block bb)
600 : {
601 2664766 : gcc_assert (CONST_INT_P (exp));
602 :
603 2664766 : if (standard_sse_constant_p (exp, vmode))
604 611269 : return ix86_cost->sse_op;
605 2053497 : if (optimize_bb_for_size_p (bb))
606 : return COSTS_N_BYTES (8);
607 : /* We have separate costs for SImode and DImode, use SImode costs
608 : for smaller modes. */
609 2418517 : return COSTS_N_INSNS (ix86_cost->sse_load[smode == DImode ? 1 : 0]) / 2;
610 : }
611 :
612 : /* Return true if it's cost profitable for chain conversion. */
613 :
614 : bool
615 6017002 : general_scalar_chain::compute_convert_gain ()
616 : {
617 6017002 : bitmap_iterator bi;
618 6017002 : unsigned insn_uid;
619 6017002 : int gain = 0;
620 6017002 : sreal weighted_gain = 0;
621 :
622 6017002 : if (dump_file)
623 136 : fprintf (dump_file, "Computing gain for chain #%d...\n", chain_id);
624 :
625 : /* SSE costs distinguish between SImode and DImode loads/stores, for
626 : int costs factor in the number of GPRs involved. When supporting
627 : smaller modes than SImode the int load/store costs need to be
628 : adjusted as well. */
629 6017002 : unsigned sse_cost_idx = smode == DImode ? 1 : 0;
630 6017002 : int m = smode == DImode ? (TARGET_64BIT ? 1 : 2) : 1;
631 :
632 17829751 : EXECUTE_IF_SET_IN_BITMAP (insns, 0, insn_uid, bi)
633 : {
634 11812749 : rtx_insn *insn = DF_INSN_UID_GET (insn_uid)->insn;
635 11812749 : rtx def_set = single_set (insn);
636 11812749 : rtx src = SET_SRC (def_set);
637 11812749 : rtx dst = SET_DEST (def_set);
638 11812749 : basic_block bb = BLOCK_FOR_INSN (insn);
639 11812749 : int igain = 0;
640 11812749 : profile_count entry_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
641 11812749 : bool speed_p = optimize_bb_for_speed_p (bb);
642 11812749 : sreal bb_freq = bb->count.to_sreal_scale (entry_count);
643 :
644 11812749 : if (REG_P (src) && REG_P (dst))
645 : {
646 930153 : if (!speed_p)
647 : /* reg-reg move is 2 bytes, while SSE 3. */
648 196965 : igain += COSTS_N_BYTES (2 * m - 3);
649 : else
650 : /* Move costs are normalized to reg-reg move having cost 2. */
651 733188 : igain += COSTS_N_INSNS (2 * m - ix86_cost->xmm_move) / 2;
652 : }
653 10882596 : else if (REG_P (src) && MEM_P (dst))
654 : {
655 2367214 : if (!speed_p)
656 : /* Integer load/store is 3+ bytes and SSE 4+. */
657 194350 : igain += COSTS_N_BYTES (3 * m - 4);
658 : else
659 2172864 : igain
660 2172864 : += COSTS_N_INSNS (m * ix86_cost->int_store[2]
661 : - ix86_cost->sse_store[sse_cost_idx]) / 2;
662 : }
663 8515382 : else if (MEM_P (src) && REG_P (dst))
664 : {
665 3827027 : if (!speed_p)
666 356762 : igain += COSTS_N_BYTES (3 * m - 4);
667 : else
668 3470265 : igain += COSTS_N_INSNS (m * ix86_cost->int_load[2]
669 : - ix86_cost->sse_load[sse_cost_idx]) / 2;
670 : }
671 : else
672 : {
673 : /* For operations on memory operands, include the overhead
674 : of explicit load and store instructions. */
675 4688355 : if (MEM_P (dst))
676 : {
677 70820 : if (!speed_p)
678 : /* ??? This probably should account size difference
679 : of SSE and integer load rather than full SSE load. */
680 : igain -= COSTS_N_BYTES (8);
681 : else
682 : {
683 61343 : int cost = (m * (ix86_cost->int_load[2]
684 61343 : + ix86_cost->int_store[2])
685 61343 : - (ix86_cost->sse_load[sse_cost_idx] +
686 61343 : ix86_cost->sse_store[sse_cost_idx]));
687 61343 : igain += COSTS_N_INSNS (cost) / 2;
688 : }
689 : }
690 :
691 4688355 : switch (GET_CODE (src))
692 : {
693 499407 : case ASHIFT:
694 499407 : case ASHIFTRT:
695 499407 : case LSHIFTRT:
696 499407 : if (m == 2)
697 : {
698 16991 : if (INTVAL (XEXP (src, 1)) >= 32)
699 11533 : igain += ix86_cost->add;
700 : /* Gain for extend highpart case. */
701 5458 : else if (GET_CODE (XEXP (src, 0)) == ASHIFT)
702 0 : igain += ix86_cost->shift_const - ix86_cost->sse_op;
703 : else
704 5458 : igain += ix86_cost->shift_const;
705 : }
706 :
707 499407 : igain += ix86_cost->shift_const - ix86_cost->sse_op;
708 :
709 499407 : if (CONST_INT_P (XEXP (src, 0)))
710 0 : igain -= vector_const_cost (XEXP (src, 0), bb);
711 : break;
712 :
713 3786 : case ROTATE:
714 3786 : case ROTATERT:
715 3786 : igain += m * ix86_cost->shift_const;
716 3786 : if (TARGET_AVX512VL)
717 204 : igain -= ix86_cost->sse_op;
718 3582 : else if (smode == DImode)
719 : {
720 634 : int bits = INTVAL (XEXP (src, 1));
721 634 : if ((bits & 0x0f) == 0)
722 106 : igain -= ix86_cost->sse_op;
723 528 : else if ((bits & 0x07) == 0)
724 31 : igain -= 2 * ix86_cost->sse_op;
725 : else
726 497 : igain -= 3 * ix86_cost->sse_op;
727 : }
728 2948 : else if (INTVAL (XEXP (src, 1)) == 16)
729 139 : igain -= ix86_cost->sse_op;
730 : else
731 2809 : igain -= 2 * ix86_cost->sse_op;
732 : break;
733 :
734 2879732 : case AND:
735 2879732 : case IOR:
736 2879732 : case XOR:
737 2879732 : case PLUS:
738 2879732 : case MINUS:
739 2879732 : igain += m * ix86_cost->add - ix86_cost->sse_op;
740 : /* Additional gain for andnot for targets without BMI. */
741 2879732 : if (GET_CODE (XEXP (src, 0)) == NOT
742 3598 : && !TARGET_BMI)
743 3589 : igain += m * ix86_cost->add;
744 :
745 2879732 : if (CONST_INT_P (XEXP (src, 0)))
746 0 : igain -= vector_const_cost (XEXP (src, 0), bb);
747 2879732 : if (CONST_INT_P (XEXP (src, 1)))
748 1706027 : igain -= vector_const_cost (XEXP (src, 1), bb);
749 2879732 : if (MEM_P (XEXP (src, 1)))
750 : {
751 100273 : if (!speed_p)
752 23149 : igain -= COSTS_N_BYTES (m == 2 ? 3 : 5);
753 : else
754 88694 : igain += COSTS_N_INSNS
755 : (m * ix86_cost->int_load[2]
756 : - ix86_cost->sse_load[sse_cost_idx]) / 2;
757 : }
758 : break;
759 :
760 59919 : case NEG:
761 59919 : case NOT:
762 59919 : igain -= ix86_cost->sse_op + COSTS_N_INSNS (1);
763 :
764 59919 : if (GET_CODE (XEXP (src, 0)) != ABS)
765 : {
766 59919 : igain += m * ix86_cost->add;
767 59919 : break;
768 : }
769 : /* FALLTHRU */
770 :
771 988 : case ABS:
772 988 : case SMAX:
773 988 : case SMIN:
774 988 : case UMAX:
775 988 : case UMIN:
776 : /* We do not have any conditional move cost, estimate it as a
777 : reg-reg move. Comparisons are costed as adds. */
778 988 : igain += m * (COSTS_N_INSNS (2) + ix86_cost->add);
779 : /* Integer SSE ops are all costed the same. */
780 988 : igain -= ix86_cost->sse_op;
781 988 : break;
782 :
783 0 : case COMPARE:
784 0 : if (XEXP (src, 1) != const0_rtx)
785 : {
786 : /* cmp vs. pxor;pshufd;ptest. */
787 0 : igain += COSTS_N_INSNS (m - 3);
788 : }
789 0 : else if (GET_CODE (XEXP (src, 0)) != AND)
790 : {
791 : /* test vs. pshufd;ptest. */
792 0 : igain += COSTS_N_INSNS (m - 2);
793 : }
794 0 : else if (GET_CODE (XEXP (XEXP (src, 0), 0)) != NOT)
795 : {
796 : /* and;test vs. pshufd;ptest. */
797 0 : igain += COSTS_N_INSNS (2 * m - 2);
798 : }
799 0 : else if (TARGET_BMI)
800 : {
801 : /* andn;test vs. pandn;pshufd;ptest. */
802 0 : igain += COSTS_N_INSNS (2 * m - 3);
803 : }
804 : else
805 : {
806 : /* not;and;test vs. pandn;pshufd;ptest. */
807 0 : igain += COSTS_N_INSNS (3 * m - 3);
808 : }
809 : break;
810 :
811 1195496 : case CONST_INT:
812 1195496 : if (REG_P (dst))
813 : {
814 1195496 : if (!speed_p)
815 : {
816 : /* xor (2 bytes) vs. xorps (3 bytes). */
817 236757 : if (src == const0_rtx)
818 124252 : igain -= COSTS_N_BYTES (1);
819 : /* movdi_internal vs. movv2di_internal. */
820 : /* => mov (5 bytes) vs. movaps (7 bytes). */
821 112505 : else if (x86_64_immediate_operand (src, SImode))
822 100265 : igain -= COSTS_N_BYTES (2);
823 : else
824 : /* ??? Larger immediate constants are placed in the
825 : constant pool, where the size benefit/impact of
826 : STV conversion is affected by whether and how
827 : often each constant pool entry is shared/reused.
828 : The value below is empirically derived from the
829 : CSiBE benchmark (and the optimal value may drift
830 : over time). */
831 : igain += COSTS_N_BYTES (0);
832 : }
833 : else
834 : {
835 : /* DImode can be immediate for TARGET_64BIT
836 : and SImode always. */
837 958739 : igain += m * COSTS_N_INSNS (1);
838 958739 : igain -= vector_const_cost (src, bb);
839 : }
840 : }
841 0 : else if (MEM_P (dst))
842 : {
843 0 : igain += (m * ix86_cost->int_store[2]
844 0 : - ix86_cost->sse_store[sse_cost_idx]);
845 0 : igain -= vector_const_cost (src, bb);
846 : }
847 : break;
848 :
849 39334 : case VEC_SELECT:
850 39334 : if (XVECEXP (XEXP (src, 1), 0, 0) == const0_rtx)
851 : {
852 : // movd (4 bytes) replaced with movdqa (4 bytes).
853 28875 : if (!!speed_p)
854 26938 : igain += COSTS_N_INSNS (ix86_cost->sse_to_integer
855 : - ix86_cost->xmm_move) / 2;
856 : }
857 : else
858 : {
859 : // pshufd; movd replaced with pshufd.
860 10459 : if (!speed_p)
861 784 : igain += COSTS_N_BYTES (4);
862 : else
863 9675 : igain += ix86_cost->sse_to_integer;
864 : }
865 : break;
866 :
867 9693 : case ZERO_EXTEND:
868 : /* mov eax (6 bytes) vs movd xmm0 (8 bytes). */
869 : /* mov eax; xor edx,edx (7 bytes). */
870 9693 : if (speed_p)
871 8859 : igain += COSTS_N_INSNS (ix86_cost->int_load[2]
872 : - ix86_cost->sse_load[0]) / 2;
873 : else
874 874 : igain += COSTS_N_BYTES (TARGET_64BIT ? -2 : -1);
875 : break;
876 :
877 0 : default:
878 0 : gcc_unreachable ();
879 : }
880 : }
881 :
882 11810812 : if (speed_p)
883 10523565 : weighted_gain += bb_freq * igain;
884 11812749 : gain += igain;
885 :
886 11812749 : if (igain != 0 && dump_file)
887 : {
888 93 : fprintf (dump_file, " Instruction gain %d with bb_freq %.2f for",
889 : igain, bb_freq.to_double ());
890 93 : dump_insn_slim (dump_file, insn);
891 : }
892 : }
893 :
894 6017002 : if (dump_file)
895 : {
896 136 : fprintf (dump_file, " Instruction conversion gain: %d, \n",
897 : gain);
898 136 : fprintf (dump_file, " Registers conversion cost: %d\n",
899 : cost_sse_integer);
900 136 : fprintf (dump_file, " Weighted instruction conversion gain: %.2f, \n",
901 : weighted_gain.to_double ());
902 136 : fprintf (dump_file, " Weighted registers conversion cost: %.2f\n",
903 : weighted_cost_sse_integer.to_double ());
904 : }
905 :
906 6017002 : if (weighted_gain != weighted_cost_sse_integer)
907 4852981 : return weighted_gain > weighted_cost_sse_integer;
908 : else
909 1164021 : return gain > cost_sse_integer;;
910 : }
911 :
912 : /* Insert generated conversion instruction sequence INSNS
913 : after instruction AFTER. New BB may be required in case
914 : instruction has EH region attached. */
915 :
916 : void
917 31231 : scalar_chain::emit_conversion_insns (rtx insns, rtx_insn *after)
918 : {
919 31231 : if (!control_flow_insn_p (after))
920 : {
921 31018 : emit_insn_after (insns, after);
922 31018 : return;
923 : }
924 :
925 213 : basic_block bb = BLOCK_FOR_INSN (after);
926 213 : edge e = find_fallthru_edge (bb->succs);
927 213 : gcc_assert (e);
928 :
929 213 : basic_block new_bb = split_edge (e);
930 213 : emit_insn_after (insns, BB_HEAD (new_bb));
931 : }
932 :
933 : } // anon namespace
934 :
935 : /* Generate the canonical SET_SRC to move GPR to a VMODE vector register,
936 : zeroing the upper parts. */
937 :
938 : static rtx
939 173043 : gen_gpr_to_xmm_move_src (enum machine_mode vmode, rtx gpr)
940 : {
941 346086 : switch (GET_MODE_NUNITS (vmode))
942 : {
943 45 : case 1:
944 45 : return gen_rtx_SUBREG (vmode, gpr, 0);
945 172436 : case 2:
946 344872 : return gen_rtx_VEC_CONCAT (vmode, gpr,
947 : CONST0_RTX (GET_MODE_INNER (vmode)));
948 562 : default:
949 562 : return gen_rtx_VEC_MERGE (vmode, gen_rtx_VEC_DUPLICATE (vmode, gpr),
950 : CONST0_RTX (vmode), GEN_INT (HOST_WIDE_INT_1U));
951 : }
952 : }
953 :
954 : /* Make vector copies for all register REGNO definitions
955 : and replace its uses in a chain. */
956 :
957 : void
958 8347 : scalar_chain::make_vector_copies (rtx_insn *insn, rtx reg)
959 : {
960 8347 : rtx vreg = *defs_map.get (reg);
961 :
962 8347 : start_sequence ();
963 8347 : if (!TARGET_INTER_UNIT_MOVES_TO_VEC)
964 : {
965 0 : rtx tmp = assign_386_stack_local (smode, SLOT_STV_TEMP);
966 0 : if (smode == DImode && !TARGET_64BIT)
967 : {
968 0 : emit_move_insn (adjust_address (tmp, SImode, 0),
969 : gen_rtx_SUBREG (SImode, reg, 0));
970 0 : emit_move_insn (adjust_address (tmp, SImode, 4),
971 : gen_rtx_SUBREG (SImode, reg, 4));
972 : }
973 : else
974 0 : emit_move_insn (copy_rtx (tmp), reg);
975 0 : emit_insn (gen_rtx_SET (gen_rtx_SUBREG (vmode, vreg, 0),
976 : gen_gpr_to_xmm_move_src (vmode, tmp)));
977 : }
978 8347 : else if (!TARGET_64BIT && smode == DImode)
979 : {
980 8211 : if (TARGET_SSE4_1)
981 : {
982 356 : emit_insn (gen_sse2_loadld (gen_rtx_SUBREG (V4SImode, vreg, 0),
983 : CONST0_RTX (V4SImode),
984 : gen_rtx_SUBREG (SImode, reg, 0)));
985 356 : emit_insn (gen_sse4_1_pinsrd (gen_rtx_SUBREG (V4SImode, vreg, 0),
986 : gen_rtx_SUBREG (V4SImode, vreg, 0),
987 : gen_rtx_SUBREG (SImode, reg, 4),
988 : GEN_INT (2)));
989 : }
990 : else
991 : {
992 7855 : rtx tmp = gen_reg_rtx (DImode);
993 7855 : emit_insn (gen_sse2_loadld (gen_rtx_SUBREG (V4SImode, vreg, 0),
994 : CONST0_RTX (V4SImode),
995 : gen_rtx_SUBREG (SImode, reg, 0)));
996 7855 : emit_insn (gen_sse2_loadld (gen_rtx_SUBREG (V4SImode, tmp, 0),
997 : CONST0_RTX (V4SImode),
998 : gen_rtx_SUBREG (SImode, reg, 4)));
999 7855 : emit_insn (gen_vec_interleave_lowv4si
1000 : (gen_rtx_SUBREG (V4SImode, vreg, 0),
1001 : gen_rtx_SUBREG (V4SImode, vreg, 0),
1002 : gen_rtx_SUBREG (V4SImode, tmp, 0)));
1003 : }
1004 : }
1005 : else
1006 136 : emit_insn (gen_rtx_SET (gen_rtx_SUBREG (vmode, vreg, 0),
1007 : gen_gpr_to_xmm_move_src (vmode, reg)));
1008 8347 : rtx_insn *seq = end_sequence ();
1009 8347 : emit_conversion_insns (seq, insn);
1010 :
1011 8347 : if (dump_file)
1012 0 : fprintf (dump_file,
1013 : " Copied r%d to a vector register r%d for insn %d\n",
1014 0 : REGNO (reg), REGNO (vreg), INSN_UID (insn));
1015 8347 : }
1016 :
1017 : /* Copy the definition SRC of INSN inside the chain to DST for
1018 : scalar uses outside of the chain. */
1019 :
1020 : void
1021 22124 : scalar_chain::convert_reg (rtx_insn *insn, rtx dst, rtx src)
1022 : {
1023 22124 : start_sequence ();
1024 22124 : if (!TARGET_INTER_UNIT_MOVES_FROM_VEC)
1025 : {
1026 0 : rtx tmp = assign_386_stack_local (smode, SLOT_STV_TEMP);
1027 0 : emit_move_insn (tmp, src);
1028 0 : if (!TARGET_64BIT && smode == DImode)
1029 : {
1030 0 : emit_move_insn (gen_rtx_SUBREG (SImode, dst, 0),
1031 : adjust_address (tmp, SImode, 0));
1032 0 : emit_move_insn (gen_rtx_SUBREG (SImode, dst, 4),
1033 : adjust_address (tmp, SImode, 4));
1034 : }
1035 : else
1036 0 : emit_move_insn (dst, copy_rtx (tmp));
1037 : }
1038 22124 : else if (!TARGET_64BIT && smode == DImode)
1039 : {
1040 21170 : if (TARGET_SSE4_1)
1041 : {
1042 0 : rtx tmp = gen_rtx_PARALLEL (VOIDmode,
1043 : gen_rtvec (1, const0_rtx));
1044 0 : emit_insn
1045 0 : (gen_rtx_SET
1046 : (gen_rtx_SUBREG (SImode, dst, 0),
1047 : gen_rtx_VEC_SELECT (SImode,
1048 : gen_rtx_SUBREG (V4SImode, src, 0),
1049 : tmp)));
1050 :
1051 0 : tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const1_rtx));
1052 0 : emit_insn
1053 0 : (gen_rtx_SET
1054 : (gen_rtx_SUBREG (SImode, dst, 4),
1055 : gen_rtx_VEC_SELECT (SImode,
1056 : gen_rtx_SUBREG (V4SImode, src, 0),
1057 : tmp)));
1058 : }
1059 : else
1060 : {
1061 21170 : rtx vcopy = gen_reg_rtx (V2DImode);
1062 21170 : emit_move_insn (vcopy, gen_rtx_SUBREG (V2DImode, src, 0));
1063 21170 : emit_move_insn (gen_rtx_SUBREG (SImode, dst, 0),
1064 : gen_rtx_SUBREG (SImode, vcopy, 0));
1065 21170 : emit_move_insn (vcopy,
1066 : gen_rtx_LSHIFTRT (V2DImode,
1067 : vcopy, GEN_INT (32)));
1068 21170 : emit_move_insn (gen_rtx_SUBREG (SImode, dst, 4),
1069 : gen_rtx_SUBREG (SImode, vcopy, 0));
1070 : }
1071 : }
1072 : else
1073 954 : emit_move_insn (dst, src);
1074 :
1075 22124 : rtx_insn *seq = end_sequence ();
1076 22124 : emit_conversion_insns (seq, insn);
1077 :
1078 22124 : if (dump_file)
1079 0 : fprintf (dump_file,
1080 : " Copied r%d to a scalar register r%d for insn %d\n",
1081 0 : REGNO (src), REGNO (dst), INSN_UID (insn));
1082 22124 : }
1083 :
1084 : /* Helper function to convert immediate constant X to vmode. */
1085 : static rtx
1086 36994 : smode_convert_cst (rtx x, enum machine_mode vmode)
1087 : {
1088 : /* Prefer all ones vector in case of -1. */
1089 36994 : if (constm1_operand (x, GET_MODE (x)))
1090 630 : return CONSTM1_RTX (vmode);
1091 :
1092 36364 : unsigned n = GET_MODE_NUNITS (vmode);
1093 36364 : rtx *v = XALLOCAVEC (rtx, n);
1094 36364 : v[0] = x;
1095 42115 : for (unsigned i = 1; i < n; ++i)
1096 5751 : v[i] = const0_rtx;
1097 36364 : return gen_rtx_CONST_VECTOR (vmode, gen_rtvec_v (n, v));
1098 : }
1099 :
1100 : /* Convert operand OP in INSN. We should handle
1101 : memory operands and uninitialized registers.
1102 : All other register uses are converted during
1103 : registers conversion. */
1104 :
1105 : void
1106 247887 : scalar_chain::convert_op (rtx *op, rtx_insn *insn)
1107 : {
1108 247887 : rtx tmp;
1109 :
1110 247887 : if (GET_MODE (*op) == V1TImode)
1111 : return;
1112 :
1113 247704 : *op = copy_rtx_if_shared (*op);
1114 :
1115 247704 : if (GET_CODE (*op) == NOT
1116 247704 : || GET_CODE (*op) == ASHIFT)
1117 : {
1118 3493 : convert_op (&XEXP (*op, 0), insn);
1119 3493 : PUT_MODE (*op, vmode);
1120 : }
1121 : else if (MEM_P (*op))
1122 : {
1123 172907 : rtx_insn *movabs = NULL;
1124 :
1125 : /* Emit MOVABS to load from a 64-bit absolute address to a GPR. */
1126 172907 : if (!memory_operand (*op, GET_MODE (*op)))
1127 : {
1128 0 : tmp = gen_reg_rtx (GET_MODE (*op));
1129 0 : movabs = emit_insn_before (gen_rtx_SET (tmp, *op), insn);
1130 :
1131 0 : *op = tmp;
1132 : }
1133 :
1134 172907 : tmp = gen_rtx_SUBREG (vmode, gen_reg_rtx (GET_MODE (*op)), 0);
1135 :
1136 172907 : rtx_insn *eh_insn
1137 172907 : = emit_insn_before (gen_rtx_SET (copy_rtx (tmp),
1138 : gen_gpr_to_xmm_move_src (vmode, *op)),
1139 172907 : insn);
1140 :
1141 172907 : if (cfun->can_throw_non_call_exceptions)
1142 : {
1143 : /* Handle REG_EH_REGION note. */
1144 168721 : rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
1145 168721 : if (note)
1146 : {
1147 3588 : if (movabs)
1148 0 : eh_insn = movabs;
1149 3588 : control_flow_insns.safe_push (eh_insn);
1150 3588 : add_reg_note (eh_insn, REG_EH_REGION, XEXP (note, 0));
1151 : }
1152 : }
1153 :
1154 172907 : *op = tmp;
1155 :
1156 172907 : if (dump_file)
1157 0 : fprintf (dump_file, " Preloading operand for insn %d into r%d\n",
1158 0 : INSN_UID (insn), reg_or_subregno (tmp));
1159 : }
1160 : else if (REG_P (*op))
1161 65172 : *op = gen_rtx_SUBREG (vmode, *op, 0);
1162 : else if (CONST_SCALAR_INT_P (*op))
1163 : {
1164 6129 : rtx vec_cst = smode_convert_cst (*op, vmode);
1165 :
1166 6129 : if (!standard_sse_constant_p (vec_cst, vmode))
1167 : {
1168 2733 : start_sequence ();
1169 2733 : vec_cst = validize_mem (force_const_mem (vmode, vec_cst));
1170 2733 : rtx_insn *seq = end_sequence ();
1171 2733 : emit_insn_before (seq, insn);
1172 : }
1173 :
1174 6129 : tmp = gen_rtx_SUBREG (vmode, gen_reg_rtx (smode), 0);
1175 :
1176 6129 : emit_insn_before (gen_move_insn (copy_rtx (tmp), vec_cst), insn);
1177 6129 : *op = tmp;
1178 : }
1179 : else
1180 : {
1181 0 : gcc_assert (SUBREG_P (*op));
1182 3 : if (GET_MODE (*op) != vmode)
1183 : {
1184 3 : rtx inner = SUBREG_REG (*op);
1185 3 : poly_uint64 byte = SUBREG_BYTE (*op);
1186 3 : if (targetm.modes_tieable_p (vmode, GET_MODE (inner))
1187 3 : && validate_subreg (vmode, GET_MODE (inner), inner, byte))
1188 3 : *op = gen_lowpart (vmode, *op);
1189 : else
1190 : {
1191 0 : tmp = gen_reg_rtx (GET_MODE (*op));
1192 0 : emit_insn_before (gen_rtx_SET (tmp, *op), insn);
1193 0 : *op = gen_rtx_SUBREG (vmode, tmp, 0);
1194 : }
1195 : }
1196 : }
1197 : }
1198 :
1199 : /* Convert CCZmode COMPARE to vector mode. */
1200 :
1201 : rtx
1202 12 : scalar_chain::convert_compare (rtx op1, rtx op2, rtx_insn *insn)
1203 : {
1204 12 : rtx src, tmp;
1205 :
1206 : /* Handle any REG_EQUAL notes. */
1207 12 : tmp = find_reg_equal_equiv_note (insn);
1208 12 : if (tmp)
1209 : {
1210 1 : if (GET_CODE (XEXP (tmp, 0)) == COMPARE
1211 1 : && GET_MODE (XEXP (tmp, 0)) == CCZmode
1212 1 : && REG_P (XEXP (XEXP (tmp, 0), 0)))
1213 : {
1214 1 : rtx *op = &XEXP (XEXP (tmp, 0), 1);
1215 1 : if (CONST_SCALAR_INT_P (*op))
1216 : {
1217 1 : if (constm1_operand (*op, GET_MODE (*op)))
1218 0 : *op = CONSTM1_RTX (vmode);
1219 : else
1220 : {
1221 1 : unsigned n = GET_MODE_NUNITS (vmode);
1222 1 : rtx *v = XALLOCAVEC (rtx, n);
1223 1 : v[0] = *op;
1224 1 : for (unsigned i = 1; i < n; ++i)
1225 0 : v[i] = const0_rtx;
1226 1 : *op = gen_rtx_CONST_VECTOR (vmode, gen_rtvec_v (n, v));
1227 : }
1228 : tmp = NULL_RTX;
1229 : }
1230 0 : else if (REG_P (*op))
1231 : tmp = NULL_RTX;
1232 : }
1233 :
1234 : if (tmp)
1235 0 : remove_note (insn, tmp);
1236 : }
1237 :
1238 : /* Comparison against anything other than zero, requires an XOR. */
1239 12 : if (op2 != const0_rtx)
1240 : {
1241 6 : convert_op (&op1, insn);
1242 6 : convert_op (&op2, insn);
1243 : /* If both operands are MEMs, explicitly load the OP1 into TMP. */
1244 6 : if (MEM_P (op1) && MEM_P (op2))
1245 : {
1246 0 : tmp = gen_reg_rtx (vmode);
1247 0 : emit_insn_before (gen_rtx_SET (tmp, op1), insn);
1248 0 : src = tmp;
1249 : }
1250 : else
1251 : src = op1;
1252 6 : src = gen_rtx_XOR (vmode, src, op2);
1253 : }
1254 6 : else if (GET_CODE (op1) == AND
1255 0 : && GET_CODE (XEXP (op1, 0)) == NOT)
1256 : {
1257 0 : rtx op11 = XEXP (XEXP (op1, 0), 0);
1258 0 : rtx op12 = XEXP (op1, 1);
1259 0 : convert_op (&op11, insn);
1260 0 : convert_op (&op12, insn);
1261 0 : if (!REG_P (op11))
1262 : {
1263 0 : tmp = gen_reg_rtx (vmode);
1264 0 : emit_insn_before (gen_rtx_SET (tmp, op11), insn);
1265 0 : op11 = tmp;
1266 : }
1267 0 : src = gen_rtx_AND (vmode, gen_rtx_NOT (vmode, op11), op12);
1268 0 : }
1269 6 : else if (GET_CODE (op1) == AND)
1270 : {
1271 0 : rtx op11 = XEXP (op1, 0);
1272 0 : rtx op12 = XEXP (op1, 1);
1273 0 : convert_op (&op11, insn);
1274 0 : convert_op (&op12, insn);
1275 0 : if (!REG_P (op11))
1276 : {
1277 0 : tmp = gen_reg_rtx (vmode);
1278 0 : emit_insn_before (gen_rtx_SET (tmp, op11), insn);
1279 0 : op11 = tmp;
1280 : }
1281 0 : return gen_rtx_UNSPEC (CCZmode, gen_rtvec (2, op11, op12),
1282 : UNSPEC_PTEST);
1283 : }
1284 : else
1285 : {
1286 6 : convert_op (&op1, insn);
1287 6 : src = op1;
1288 : }
1289 :
1290 12 : if (!REG_P (src))
1291 : {
1292 8 : tmp = gen_reg_rtx (vmode);
1293 8 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
1294 8 : src = tmp;
1295 : }
1296 :
1297 12 : if (vmode == V2DImode)
1298 : {
1299 0 : tmp = gen_reg_rtx (vmode);
1300 0 : emit_insn_before (gen_vec_interleave_lowv2di (tmp, src, src), insn);
1301 0 : src = tmp;
1302 : }
1303 12 : else if (vmode == V4SImode)
1304 : {
1305 0 : tmp = gen_reg_rtx (vmode);
1306 0 : emit_insn_before (gen_sse2_pshufd (tmp, src, const0_rtx), insn);
1307 0 : src = tmp;
1308 : }
1309 :
1310 12 : return gen_rtx_UNSPEC (CCZmode, gen_rtvec (2, src, src), UNSPEC_PTEST);
1311 : }
1312 :
1313 : /* Helper function for converting INSN to vector mode. */
1314 :
1315 : void
1316 1368496 : scalar_chain::convert_insn_common (rtx_insn *insn)
1317 : {
1318 : /* Generate copies for out-of-chain uses of defs and adjust debug uses. */
1319 2094558 : for (df_ref ref = DF_INSN_DEFS (insn); ref; ref = DF_REF_NEXT_LOC (ref))
1320 726062 : if (bitmap_bit_p (defs_conv, DF_REF_REGNO (ref)))
1321 : {
1322 23535 : df_link *use;
1323 44281 : for (use = DF_REF_CHAIN (ref); use; use = use->next)
1324 42870 : if (NONDEBUG_INSN_P (DF_REF_INSN (use->ref))
1325 42870 : && (DF_REF_REG_MEM_P (use->ref)
1326 38799 : || !bitmap_bit_p (insns, DF_REF_INSN_UID (use->ref))))
1327 : break;
1328 23535 : if (use)
1329 22124 : convert_reg (insn, DF_REF_REG (ref),
1330 22124 : *defs_map.get (regno_reg_rtx [DF_REF_REGNO (ref)]));
1331 1411 : else if (MAY_HAVE_DEBUG_BIND_INSNS)
1332 : {
1333 : /* If we generated a scalar copy we can leave debug-insns
1334 : as-is, if not, we have to adjust them. */
1335 1289 : auto_vec<rtx_insn *, 5> to_reset_debug_insns;
1336 3859 : for (use = DF_REF_CHAIN (ref); use; use = use->next)
1337 2570 : if (DEBUG_INSN_P (DF_REF_INSN (use->ref)))
1338 : {
1339 817 : rtx_insn *debug_insn = DF_REF_INSN (use->ref);
1340 : /* If there's a reaching definition outside of the
1341 : chain we have to reset. */
1342 817 : df_link *def;
1343 2877 : for (def = DF_REF_CHAIN (use->ref); def; def = def->next)
1344 2229 : if (!bitmap_bit_p (insns, DF_REF_INSN_UID (def->ref)))
1345 : break;
1346 817 : if (def)
1347 169 : to_reset_debug_insns.safe_push (debug_insn);
1348 : else
1349 : {
1350 648 : *DF_REF_REAL_LOC (use->ref)
1351 648 : = *defs_map.get (regno_reg_rtx [DF_REF_REGNO (ref)]);
1352 648 : df_insn_rescan (debug_insn);
1353 : }
1354 : }
1355 : /* Have to do the reset outside of the DF_CHAIN walk to not
1356 : disrupt it. */
1357 2747 : while (!to_reset_debug_insns.is_empty ())
1358 : {
1359 169 : rtx_insn *debug_insn = to_reset_debug_insns.pop ();
1360 169 : INSN_VAR_LOCATION_LOC (debug_insn) = gen_rtx_UNKNOWN_VAR_LOC ();
1361 169 : df_insn_rescan_debug_internal (debug_insn);
1362 : }
1363 1289 : }
1364 : }
1365 :
1366 : /* Replace uses in this insn with the defs we use in the chain. */
1367 3414713 : for (df_ref ref = DF_INSN_USES (insn); ref; ref = DF_REF_NEXT_LOC (ref))
1368 2046217 : if (!DF_REF_REG_MEM_P (ref))
1369 736595 : if (rtx *vreg = defs_map.get (regno_reg_rtx[DF_REF_REGNO (ref)]))
1370 : {
1371 : /* Also update a corresponding REG_DEAD note. */
1372 35338 : rtx note = find_reg_note (insn, REG_DEAD, DF_REF_REG (ref));
1373 35338 : if (note)
1374 23342 : XEXP (note, 0) = *vreg;
1375 35338 : *DF_REF_REAL_LOC (ref) = *vreg;
1376 : }
1377 1368496 : }
1378 :
1379 : /* Convert INSN which is an SImode or DImode rotation by a constant
1380 : to vector mode. CODE is either ROTATE or ROTATERT with operands
1381 : OP0 and OP1. Returns the SET_SRC of the last instruction in the
1382 : resulting sequence, which is emitted before INSN. */
1383 :
1384 : rtx
1385 92 : general_scalar_chain::convert_rotate (enum rtx_code code, rtx op0, rtx op1,
1386 : rtx_insn *insn)
1387 : {
1388 92 : int bits = INTVAL (op1);
1389 92 : rtx pat, result;
1390 :
1391 92 : convert_op (&op0, insn);
1392 92 : if (bits == 0)
1393 0 : return op0;
1394 :
1395 92 : if (smode == DImode)
1396 : {
1397 92 : if (code == ROTATE)
1398 45 : bits = 64 - bits;
1399 92 : if (bits == 32)
1400 : {
1401 0 : rtx tmp1 = gen_reg_rtx (V4SImode);
1402 0 : pat = gen_sse2_pshufd (tmp1, gen_lowpart (V4SImode, op0),
1403 : GEN_INT (225));
1404 0 : emit_insn_before (pat, insn);
1405 0 : result = gen_lowpart (V2DImode, tmp1);
1406 : }
1407 92 : else if (TARGET_AVX512VL)
1408 0 : result = simplify_gen_binary (code, V2DImode, op0, op1);
1409 92 : else if (bits == 16 || bits == 48)
1410 : {
1411 0 : rtx tmp1 = gen_reg_rtx (V8HImode);
1412 0 : pat = gen_sse2_pshuflw (tmp1, gen_lowpart (V8HImode, op0),
1413 : GEN_INT (bits == 16 ? 57 : 147));
1414 0 : emit_insn_before (pat, insn);
1415 0 : result = gen_lowpart (V2DImode, tmp1);
1416 : }
1417 92 : else if ((bits & 0x07) == 0)
1418 : {
1419 0 : rtx tmp1 = gen_reg_rtx (V4SImode);
1420 0 : pat = gen_sse2_pshufd (tmp1, gen_lowpart (V4SImode, op0),
1421 : GEN_INT (68));
1422 0 : emit_insn_before (pat, insn);
1423 0 : rtx tmp2 = gen_reg_rtx (V1TImode);
1424 0 : pat = gen_sse2_lshrv1ti3 (tmp2, gen_lowpart (V1TImode, tmp1),
1425 : GEN_INT (bits));
1426 0 : emit_insn_before (pat, insn);
1427 0 : result = gen_lowpart (V2DImode, tmp2);
1428 : }
1429 : else
1430 : {
1431 92 : rtx tmp1 = gen_reg_rtx (V4SImode);
1432 92 : pat = gen_sse2_pshufd (tmp1, gen_lowpart (V4SImode, op0),
1433 : GEN_INT (20));
1434 92 : emit_insn_before (pat, insn);
1435 92 : rtx tmp2 = gen_reg_rtx (V2DImode);
1436 92 : pat = gen_lshrv2di3 (tmp2, gen_lowpart (V2DImode, tmp1),
1437 : GEN_INT (bits & 31));
1438 92 : emit_insn_before (pat, insn);
1439 92 : rtx tmp3 = gen_reg_rtx (V4SImode);
1440 139 : pat = gen_sse2_pshufd (tmp3, gen_lowpart (V4SImode, tmp2),
1441 : GEN_INT (bits > 32 ? 34 : 136));
1442 92 : emit_insn_before (pat, insn);
1443 92 : result = gen_lowpart (V2DImode, tmp3);
1444 : }
1445 : }
1446 0 : else if (bits == 16)
1447 : {
1448 0 : rtx tmp1 = gen_reg_rtx (V8HImode);
1449 0 : pat = gen_sse2_pshuflw (tmp1, gen_lowpart (V8HImode, op0), GEN_INT (225));
1450 0 : emit_insn_before (pat, insn);
1451 0 : result = gen_lowpart (V4SImode, tmp1);
1452 : }
1453 0 : else if (TARGET_AVX512VL)
1454 0 : result = simplify_gen_binary (code, V4SImode, op0, op1);
1455 : else
1456 : {
1457 0 : if (code == ROTATE)
1458 0 : bits = 32 - bits;
1459 :
1460 0 : rtx tmp1 = gen_reg_rtx (V4SImode);
1461 0 : emit_insn_before (gen_sse2_pshufd (tmp1, op0, GEN_INT (224)), insn);
1462 0 : rtx tmp2 = gen_reg_rtx (V2DImode);
1463 0 : pat = gen_lshrv2di3 (tmp2, gen_lowpart (V2DImode, tmp1),
1464 : GEN_INT (bits));
1465 0 : emit_insn_before (pat, insn);
1466 0 : result = gen_lowpart (V4SImode, tmp2);
1467 : }
1468 :
1469 : return result;
1470 : }
1471 :
1472 : /* Convert INSN to vector mode. */
1473 :
1474 : void
1475 412465 : general_scalar_chain::convert_insn (rtx_insn *insn)
1476 : {
1477 412465 : rtx def_set = single_set (insn);
1478 412465 : rtx src = SET_SRC (def_set);
1479 412465 : rtx dst = SET_DEST (def_set);
1480 412465 : rtx subreg;
1481 :
1482 412465 : if (MEM_P (dst) && !REG_P (src))
1483 : {
1484 : /* There are no scalar integer instructions and therefore
1485 : temporary register usage is required. */
1486 760 : rtx tmp = gen_reg_rtx (smode);
1487 760 : emit_conversion_insns (gen_move_insn (dst, tmp), insn);
1488 760 : dst = gen_rtx_SUBREG (vmode, tmp, 0);
1489 760 : }
1490 411705 : else if (REG_P (dst) && GET_MODE (dst) == smode)
1491 : {
1492 : /* Replace the definition with a SUBREG to the definition we
1493 : use inside the chain. */
1494 216221 : rtx *vdef = defs_map.get (dst);
1495 216221 : if (vdef)
1496 23535 : dst = *vdef;
1497 216221 : dst = gen_rtx_SUBREG (vmode, dst, 0);
1498 : /* IRA doesn't like to have REG_EQUAL/EQUIV notes when the SET_DEST
1499 : is a non-REG_P. So kill those off. */
1500 216221 : rtx note = find_reg_equal_equiv_note (insn);
1501 216221 : if (note)
1502 9726 : remove_note (insn, note);
1503 : }
1504 :
1505 412465 : switch (GET_CODE (src))
1506 : {
1507 30317 : case PLUS:
1508 30317 : case MINUS:
1509 30317 : case IOR:
1510 30317 : case XOR:
1511 30317 : case AND:
1512 30317 : case SMAX:
1513 30317 : case SMIN:
1514 30317 : case UMAX:
1515 30317 : case UMIN:
1516 30317 : convert_op (&XEXP (src, 1), insn);
1517 : /* FALLTHRU */
1518 :
1519 37693 : case ABS:
1520 37693 : case ASHIFT:
1521 37693 : case ASHIFTRT:
1522 37693 : case LSHIFTRT:
1523 37693 : convert_op (&XEXP (src, 0), insn);
1524 37693 : PUT_MODE (src, vmode);
1525 37693 : break;
1526 :
1527 92 : case ROTATE:
1528 92 : case ROTATERT:
1529 92 : src = convert_rotate (GET_CODE (src), XEXP (src, 0), XEXP (src, 1),
1530 : insn);
1531 92 : break;
1532 :
1533 397 : case NEG:
1534 397 : src = XEXP (src, 0);
1535 :
1536 397 : if (GET_CODE (src) == ABS)
1537 : {
1538 0 : src = XEXP (src, 0);
1539 0 : convert_op (&src, insn);
1540 0 : subreg = gen_reg_rtx (vmode);
1541 0 : emit_insn_before (gen_rtx_SET (subreg,
1542 : gen_rtx_ABS (vmode, src)), insn);
1543 0 : src = subreg;
1544 : }
1545 : else
1546 397 : convert_op (&src, insn);
1547 :
1548 397 : subreg = gen_reg_rtx (vmode);
1549 397 : emit_insn_before (gen_move_insn (subreg, CONST0_RTX (vmode)), insn);
1550 397 : src = gen_rtx_MINUS (vmode, subreg, src);
1551 397 : break;
1552 :
1553 250 : case NOT:
1554 250 : src = XEXP (src, 0);
1555 250 : convert_op (&src, insn);
1556 250 : subreg = gen_reg_rtx (vmode);
1557 250 : emit_insn_before (gen_move_insn (subreg, CONSTM1_RTX (vmode)), insn);
1558 250 : src = gen_rtx_XOR (vmode, src, subreg);
1559 250 : break;
1560 :
1561 170741 : case MEM:
1562 170741 : if (!REG_P (dst))
1563 170741 : convert_op (&src, insn);
1564 : break;
1565 :
1566 196786 : case REG:
1567 196786 : if (!MEM_P (dst))
1568 1302 : convert_op (&src, insn);
1569 : break;
1570 :
1571 0 : case SUBREG:
1572 0 : gcc_assert (GET_MODE (src) == vmode);
1573 : break;
1574 :
1575 0 : case COMPARE:
1576 0 : dst = gen_rtx_REG (CCZmode, FLAGS_REG);
1577 0 : src = convert_compare (XEXP (src, 0), XEXP (src, 1), insn);
1578 0 : break;
1579 :
1580 3358 : case CONST_INT:
1581 3358 : convert_op (&src, insn);
1582 3358 : break;
1583 :
1584 2823 : case VEC_SELECT:
1585 2823 : if (XVECEXP (XEXP (src, 1), 0, 0) == const0_rtx)
1586 1642 : src = XEXP (src, 0);
1587 1181 : else if (smode == DImode)
1588 : {
1589 792 : rtx tmp = gen_lowpart (V1TImode, XEXP (src, 0));
1590 792 : dst = gen_lowpart (V1TImode, dst);
1591 792 : src = gen_rtx_LSHIFTRT (V1TImode, tmp, GEN_INT (64));
1592 : }
1593 : else
1594 : {
1595 389 : rtx tmp = XVECEXP (XEXP (src, 1), 0, 0);
1596 389 : rtvec vec = gen_rtvec (4, tmp, tmp, tmp, tmp);
1597 389 : rtx par = gen_rtx_PARALLEL (VOIDmode, vec);
1598 389 : src = gen_rtx_VEC_SELECT (vmode, XEXP (src, 0), par);
1599 : }
1600 : break;
1601 :
1602 325 : case ZERO_EXTEND:
1603 : /* *zero_extendsidi2 becomes *vec_setv2di_0_zero_extendsi_1. */
1604 325 : src = gen_rtx_VEC_CONCAT (V2DImode, src, const0_rtx);
1605 325 : break;
1606 :
1607 0 : default:
1608 0 : gcc_unreachable ();
1609 : }
1610 :
1611 412465 : SET_SRC (def_set) = src;
1612 412465 : SET_DEST (def_set) = dst;
1613 :
1614 : /* Drop possible dead definitions. */
1615 412465 : PATTERN (insn) = def_set;
1616 :
1617 412465 : INSN_CODE (insn) = -1;
1618 412465 : int patt = recog_memoized (insn);
1619 412465 : if (patt == -1)
1620 0 : fatal_insn_not_found (insn);
1621 412465 : df_insn_rescan (insn);
1622 412465 : }
1623 :
1624 : /* Helper function to compute gain for loading an immediate constant.
1625 : Typically, two movabsq for TImode vs. vmovdqa for V1TImode, but
1626 : with numerous special cases. */
1627 :
1628 : static int
1629 19 : timode_immed_const_gain (rtx cst, basic_block bb)
1630 : {
1631 : /* movabsq vs. movabsq+vmovq+vunpacklqdq. */
1632 19 : if (CONST_WIDE_INT_P (cst)
1633 7 : && CONST_WIDE_INT_NUNITS (cst) == 2
1634 26 : && CONST_WIDE_INT_ELT (cst, 0) == CONST_WIDE_INT_ELT (cst, 1))
1635 0 : return optimize_bb_for_size_p (bb) ? -COSTS_N_BYTES (9)
1636 : : -COSTS_N_INSNS (2);
1637 : /* 2x movabsq ~ vmovdqa. */
1638 : return 0;
1639 : }
1640 :
1641 : /* Return true it's cost profitable for for chain conversion. */
1642 :
1643 : bool
1644 525080 : timode_scalar_chain::compute_convert_gain ()
1645 : {
1646 : /* Assume that if we have to move TImode values between units,
1647 : then transforming this chain isn't worth it. */
1648 525080 : if (cost_sse_integer)
1649 : return false;
1650 :
1651 525080 : bitmap_iterator bi;
1652 525080 : unsigned insn_uid;
1653 :
1654 : /* Split ties to prefer V1TImode when not optimizing for size. */
1655 525080 : int gain = optimize_size ? 0 : 1;
1656 525080 : sreal weighted_gain = 0;
1657 :
1658 525080 : if (dump_file)
1659 0 : fprintf (dump_file, "Computing gain for chain #%d...\n", chain_id);
1660 :
1661 1570209 : EXECUTE_IF_SET_IN_BITMAP (insns, 0, insn_uid, bi)
1662 : {
1663 1045129 : rtx_insn *insn = DF_INSN_UID_GET (insn_uid)->insn;
1664 1045129 : rtx def_set = single_set (insn);
1665 1045129 : rtx src = SET_SRC (def_set);
1666 1045129 : rtx dst = SET_DEST (def_set);
1667 1045129 : HOST_WIDE_INT op1val;
1668 1045129 : basic_block bb = BLOCK_FOR_INSN (insn);
1669 1045129 : int scost, vcost;
1670 1045129 : int igain = 0;
1671 1045129 : profile_count entry_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
1672 1045129 : bool speed_p = optimize_bb_for_speed_p (bb);
1673 1045129 : sreal bb_freq = bb->count.to_sreal_scale (entry_count);
1674 :
1675 1045129 : switch (GET_CODE (src))
1676 : {
1677 544371 : case REG:
1678 544371 : if (GENERAL_REGNO_P (REGNO (src)))
1679 : {
1680 24981 : if (TARGET_AVX)
1681 : /* vmovq + vpinsrq */
1682 26 : igain = speed_p ? -ix86_cost->integer_to_sse
1683 : - COSTS_N_INSNS (1)
1684 : : -COSTS_N_BYTES (11);
1685 : else
1686 : /* movq + movq + punpcklqdq */
1687 24955 : igain = speed_p ? -ix86_cost->integer_to_sse
1688 : - COSTS_N_INSNS (2)
1689 : : -COSTS_N_BYTES (14);
1690 : }
1691 519390 : else if (GENERAL_REG_P (dst))
1692 : {
1693 24591 : if (TARGET_AVX)
1694 : /* vpextrq + vmovq */
1695 26 : igain = speed_p ? -ix86_cost->sse_to_integer
1696 : - COSTS_N_INSNS (1)
1697 : : -COSTS_N_BYTES (11);
1698 : else
1699 : /* movhlps + movq + movq */
1700 24565 : igain = speed_p ? -ix86_cost->sse_to_integer
1701 : - COSTS_N_INSNS (2)
1702 : : -COSTS_N_BYTES (13);
1703 : }
1704 494799 : else if (!speed_p)
1705 15229 : igain = MEM_P (dst) ? COSTS_N_BYTES (6) : COSTS_N_BYTES (3);
1706 : else
1707 : igain = COSTS_N_INSNS (1);
1708 : break;
1709 :
1710 453415 : case MEM:
1711 453415 : igain = !speed_p ? COSTS_N_BYTES (7) : COSTS_N_INSNS (1);
1712 : break;
1713 :
1714 10879 : case CONST_INT:
1715 10879 : if (MEM_P (dst)
1716 10879 : && standard_sse_constant_p (src, V1TImode))
1717 10342 : igain = !speed_p ? COSTS_N_BYTES (11) : 1;
1718 : break;
1719 :
1720 31230 : case CONST_WIDE_INT:
1721 : /* 2 x mov vs. vmovdqa. */
1722 31230 : if (MEM_P (dst))
1723 30671 : igain = !speed_p ? COSTS_N_BYTES (3) : COSTS_N_INSNS (1);
1724 : break;
1725 :
1726 78 : case NOT:
1727 78 : if (MEM_P (dst))
1728 67436 : igain = -COSTS_N_INSNS (1);
1729 : break;
1730 :
1731 39 : case AND:
1732 39 : if (!MEM_P (dst))
1733 28 : igain = COSTS_N_INSNS (1);
1734 39 : if (CONST_SCALAR_INT_P (XEXP (src, 1)))
1735 10 : igain += timode_immed_const_gain (XEXP (src, 1), bb);
1736 : break;
1737 :
1738 4499 : case XOR:
1739 4499 : case IOR:
1740 4499 : if (timode_concatdi_p (src))
1741 : {
1742 : /* vmovq;vpinsrq (11 bytes). */
1743 4371 : igain = speed_p ? -ix86_cost->integer_to_sse - COSTS_N_INSNS (1)
1744 : : -COSTS_N_BYTES (11);
1745 : break;
1746 : }
1747 128 : if (!MEM_P (dst))
1748 120 : igain = COSTS_N_INSNS (1);
1749 128 : if (CONST_SCALAR_INT_P (XEXP (src, 1)))
1750 9 : igain += timode_immed_const_gain (XEXP (src, 1), bb);
1751 : break;
1752 :
1753 0 : case PLUS:
1754 0 : if (timode_concatdi_p (src))
1755 : /* vmovq;vpinsrq (11 bytes). */
1756 0 : igain = speed_p ? -ix86_cost->integer_to_sse - COSTS_N_INSNS (1)
1757 : : -COSTS_N_BYTES (11);
1758 : break;
1759 :
1760 206 : case ASHIFT:
1761 206 : case LSHIFTRT:
1762 : /* See ix86_expand_v1ti_shift. */
1763 206 : op1val = INTVAL (XEXP (src, 1));
1764 206 : if (!speed_p)
1765 : {
1766 19 : if (op1val == 64 || op1val == 65)
1767 : scost = COSTS_N_BYTES (5);
1768 13 : else if (op1val >= 66)
1769 : scost = COSTS_N_BYTES (6);
1770 13 : else if (op1val == 1)
1771 : scost = COSTS_N_BYTES (8);
1772 : else
1773 : scost = COSTS_N_BYTES (9);
1774 :
1775 17 : if ((op1val & 7) == 0)
1776 : vcost = COSTS_N_BYTES (5);
1777 13 : else if (op1val > 64)
1778 : vcost = COSTS_N_BYTES (10);
1779 : else
1780 13 : vcost = TARGET_AVX ? COSTS_N_BYTES (19) : COSTS_N_BYTES (23);
1781 : }
1782 : else
1783 : {
1784 187 : scost = COSTS_N_INSNS (2);
1785 187 : if ((op1val & 7) == 0)
1786 : vcost = COSTS_N_INSNS (1);
1787 133 : else if (op1val > 64)
1788 : vcost = COSTS_N_INSNS (2);
1789 : else
1790 133 : vcost = TARGET_AVX ? COSTS_N_INSNS (4) : COSTS_N_INSNS (5);
1791 : }
1792 206 : igain = scost - vcost;
1793 206 : break;
1794 :
1795 123 : case ASHIFTRT:
1796 : /* See ix86_expand_v1ti_ashiftrt. */
1797 123 : op1val = INTVAL (XEXP (src, 1));
1798 123 : if (!speed_p)
1799 : {
1800 9 : if (op1val == 64 || op1val == 127)
1801 : scost = COSTS_N_BYTES (7);
1802 9 : else if (op1val == 1)
1803 : scost = COSTS_N_BYTES (8);
1804 8 : else if (op1val == 65)
1805 : scost = COSTS_N_BYTES (10);
1806 8 : else if (op1val >= 66)
1807 : scost = COSTS_N_BYTES (11);
1808 : else
1809 : scost = COSTS_N_BYTES (9);
1810 :
1811 0 : if (op1val == 127)
1812 : vcost = COSTS_N_BYTES (10);
1813 9 : else if (op1val == 64)
1814 : vcost = COSTS_N_BYTES (14);
1815 9 : else if (op1val == 96)
1816 : vcost = COSTS_N_BYTES (18);
1817 9 : else if (op1val >= 111)
1818 : vcost = COSTS_N_BYTES (15);
1819 9 : else if (TARGET_AVX2 && op1val == 32)
1820 : vcost = COSTS_N_BYTES (16);
1821 9 : else if (TARGET_SSE4_1 && op1val == 32)
1822 : vcost = COSTS_N_BYTES (20);
1823 9 : else if (op1val >= 96)
1824 : vcost = COSTS_N_BYTES (23);
1825 9 : else if ((op1val & 7) == 0)
1826 : vcost = COSTS_N_BYTES (28);
1827 9 : else if (TARGET_AVX2 && op1val < 32)
1828 : vcost = COSTS_N_BYTES (30);
1829 9 : else if (op1val == 1 || op1val >= 64)
1830 : vcost = COSTS_N_BYTES (42);
1831 : else
1832 8 : vcost = COSTS_N_BYTES (47);
1833 : }
1834 : else
1835 : {
1836 114 : if (op1val >= 65 && op1val <= 126)
1837 : scost = COSTS_N_INSNS (3);
1838 : else
1839 114 : scost = COSTS_N_INSNS (2);
1840 :
1841 114 : if (op1val == 127)
1842 : vcost = COSTS_N_INSNS (2);
1843 113 : else if (op1val == 64)
1844 : vcost = COSTS_N_INSNS (3);
1845 113 : else if (op1val == 96)
1846 : vcost = COSTS_N_INSNS (3);
1847 113 : else if (op1val >= 111)
1848 : vcost = COSTS_N_INSNS (3);
1849 113 : else if (TARGET_SSE4_1 && op1val == 32)
1850 : vcost = COSTS_N_INSNS (3);
1851 113 : else if (TARGET_SSE4_1
1852 0 : && (op1val == 8 || op1val == 16 || op1val == 24))
1853 : vcost = COSTS_N_INSNS (3);
1854 113 : else if (op1val >= 96)
1855 : vcost = COSTS_N_INSNS (4);
1856 113 : else if (TARGET_SSE4_1 && (op1val == 28 || op1val == 80))
1857 : vcost = COSTS_N_INSNS (4);
1858 113 : else if ((op1val & 7) == 0)
1859 : vcost = COSTS_N_INSNS (5);
1860 113 : else if (TARGET_AVX2 && op1val < 32)
1861 : vcost = COSTS_N_INSNS (6);
1862 113 : else if (TARGET_SSE4_1 && op1val < 15)
1863 : vcost = COSTS_N_INSNS (6);
1864 113 : else if (op1val == 1 || op1val >= 64)
1865 : vcost = COSTS_N_INSNS (8);
1866 : else
1867 16 : vcost = COSTS_N_INSNS (9);
1868 : }
1869 123 : igain = scost - vcost;
1870 123 : break;
1871 :
1872 6 : case ROTATE:
1873 6 : case ROTATERT:
1874 : /* See ix86_expand_v1ti_rotate. */
1875 6 : op1val = INTVAL (XEXP (src, 1));
1876 6 : if (!speed_p)
1877 : {
1878 0 : scost = COSTS_N_BYTES (13);
1879 0 : if ((op1val & 31) == 0)
1880 : vcost = COSTS_N_BYTES (5);
1881 0 : else if ((op1val & 7) == 0)
1882 0 : vcost = TARGET_AVX ? COSTS_N_BYTES (13) : COSTS_N_BYTES (18);
1883 0 : else if (op1val > 32 && op1val < 96)
1884 : vcost = COSTS_N_BYTES (24);
1885 : else
1886 0 : vcost = COSTS_N_BYTES (19);
1887 : }
1888 : else
1889 : {
1890 6 : scost = COSTS_N_INSNS (3);
1891 6 : if ((op1val & 31) == 0)
1892 : vcost = COSTS_N_INSNS (1);
1893 4 : else if ((op1val & 7) == 0)
1894 1 : vcost = TARGET_AVX ? COSTS_N_INSNS (3) : COSTS_N_INSNS (4);
1895 3 : else if (op1val > 32 && op1val < 96)
1896 : vcost = COSTS_N_INSNS (5);
1897 : else
1898 3 : vcost = COSTS_N_INSNS (4);
1899 : }
1900 6 : igain = scost - vcost;
1901 6 : break;
1902 :
1903 19 : case COMPARE:
1904 19 : if (XEXP (src, 1) == const0_rtx)
1905 : {
1906 8 : if (GET_CODE (XEXP (src, 0)) == AND)
1907 : /* and;and;or (9 bytes) vs. ptest (5 bytes). */
1908 : igain = !speed_p ? COSTS_N_BYTES (4) : COSTS_N_INSNS (2);
1909 : /* or (3 bytes) vs. ptest (5 bytes). */
1910 8 : else if (!speed_p)
1911 0 : igain = -COSTS_N_BYTES (2);
1912 : }
1913 11 : else if (XEXP (src, 1) == const1_rtx)
1914 : /* and;cmp -1 (7 bytes) vs. pcmpeqd;pxor;ptest (13 bytes). */
1915 0 : igain = !speed_p ? -COSTS_N_BYTES (6) : -COSTS_N_INSNS (1);
1916 : break;
1917 :
1918 264 : case ZERO_EXTEND:
1919 264 : if (GET_MODE (XEXP (src, 0)) == DImode)
1920 : /* xor (2 bytes) vs. vmovq (5 bytes). */
1921 264 : igain = speed_p ? COSTS_N_INSNS (1) - ix86_cost->sse_to_integer
1922 : : -COSTS_N_BYTES (3);
1923 : break;
1924 :
1925 : default:
1926 : break;
1927 : }
1928 :
1929 2057247 : gain += igain;
1930 1045121 : if (speed_p)
1931 1012126 : weighted_gain += bb_freq * igain;
1932 :
1933 1045129 : if (igain != 0 && dump_file)
1934 : {
1935 0 : fprintf (dump_file, " Instruction gain %d with bb_freq %.2f for ",
1936 : igain, bb_freq.to_double ());
1937 0 : dump_insn_slim (dump_file, insn);
1938 : }
1939 : }
1940 :
1941 525080 : if (dump_file)
1942 0 : fprintf (dump_file, " Total gain: %d, weighted gain %.2f\n",
1943 : gain, weighted_gain.to_double ());
1944 :
1945 525080 : if (weighted_gain > (sreal) 0)
1946 : return true;
1947 : else
1948 55345 : return gain > 0;
1949 : }
1950 :
1951 : /* Fix uses of converted REG in debug insns. */
1952 :
1953 : void
1954 443225 : timode_scalar_chain::fix_debug_reg_uses (rtx reg)
1955 : {
1956 443225 : if (!flag_var_tracking)
1957 : return;
1958 :
1959 389988 : df_ref ref, next;
1960 798884 : for (ref = DF_REG_USE_CHAIN (REGNO (reg)); ref; ref = next)
1961 : {
1962 408896 : rtx_insn *insn = DF_REF_INSN (ref);
1963 : /* Make sure the next ref is for a different instruction,
1964 : so that we're not affected by the rescan. */
1965 408896 : next = DF_REF_NEXT_REG (ref);
1966 408896 : while (next && DF_REF_INSN (next) == insn)
1967 0 : next = DF_REF_NEXT_REG (next);
1968 :
1969 408896 : if (DEBUG_INSN_P (insn))
1970 : {
1971 : /* It may be a debug insn with a TImode variable in
1972 : register. */
1973 : bool changed = false;
1974 246 : for (; ref != next; ref = DF_REF_NEXT_REG (ref))
1975 : {
1976 123 : rtx *loc = DF_REF_LOC (ref);
1977 123 : if (REG_P (*loc) && GET_MODE (*loc) == V1TImode)
1978 : {
1979 114 : *loc = gen_rtx_SUBREG (TImode, *loc, 0);
1980 114 : changed = true;
1981 : }
1982 : }
1983 123 : if (changed)
1984 114 : df_insn_rescan (insn);
1985 : }
1986 : }
1987 : }
1988 :
1989 : /* Convert SRC, a *concatditi3 pattern, into a vec_concatv2di instruction.
1990 : Insert this before INSN, and return the result as a V1TImode subreg. */
1991 :
1992 : static rtx
1993 266 : timode_convert_concatdi (rtx src, rtx_insn *insn)
1994 : {
1995 266 : rtx hi, lo;
1996 266 : rtx tmp = gen_reg_rtx (V2DImode);
1997 266 : if (GET_CODE (XEXP (src, 0)) == ASHIFT)
1998 : {
1999 266 : hi = XEXP (XEXP (XEXP (src, 0), 0), 0);
2000 266 : lo = XEXP (XEXP (src, 1), 0);
2001 : }
2002 : else
2003 : {
2004 0 : hi = XEXP (XEXP (XEXP (src, 1), 0), 0);
2005 0 : lo = XEXP (XEXP (src, 0), 0);
2006 : }
2007 266 : emit_insn_before (gen_vec_concatv2di (tmp, lo, hi), insn);
2008 266 : return gen_rtx_SUBREG (V1TImode, tmp, 0);
2009 : }
2010 :
2011 : /* Convert INSN from TImode to V1T1mode. */
2012 :
2013 : void
2014 956031 : timode_scalar_chain::convert_insn (rtx_insn *insn)
2015 : {
2016 956031 : rtx def_set = single_set (insn);
2017 956031 : rtx src = SET_SRC (def_set);
2018 956031 : rtx dst = SET_DEST (def_set);
2019 956031 : rtx tmp;
2020 :
2021 956031 : switch (GET_CODE (dst))
2022 : {
2023 443742 : case REG:
2024 443742 : if (GET_MODE (dst) == TImode)
2025 : {
2026 443106 : if (!HARD_REGISTER_NUM_P (REGNO (dst)))
2027 : {
2028 442601 : PUT_MODE (dst, V1TImode);
2029 442601 : fix_debug_reg_uses (dst);
2030 : }
2031 505 : else if (!GENERAL_REGNO_P (REGNO (dst)))
2032 359 : dst = gen_raw_REG (V1TImode, REGNO (dst));
2033 : }
2034 443742 : if (GET_MODE (dst) == V1TImode)
2035 : {
2036 : /* It might potentially be helpful to convert REG_EQUAL notes,
2037 : but for now we just remove them. */
2038 443584 : rtx note = find_reg_equal_equiv_note (insn);
2039 443584 : if (note)
2040 473 : remove_note (insn, note);
2041 : }
2042 : break;
2043 512289 : case MEM:
2044 512289 : PUT_MODE (dst, V1TImode);
2045 512289 : break;
2046 :
2047 0 : default:
2048 0 : gcc_unreachable ();
2049 : }
2050 :
2051 956031 : switch (GET_CODE (src))
2052 : {
2053 472094 : case REG:
2054 472094 : if (GET_MODE (src) == TImode)
2055 : {
2056 830 : if (GENERAL_REGNO_P (REGNO (src)))
2057 : {
2058 201 : rtx lo = gen_reg_rtx (DImode);
2059 201 : rtx hi = gen_reg_rtx (DImode);
2060 201 : emit_insn_before (gen_rtx_SET (lo, gen_lowpart (DImode, src)),
2061 : insn);
2062 201 : emit_insn_before (gen_rtx_SET (hi, gen_highpart (DImode, src)),
2063 : insn);
2064 201 : src = gen_reg_rtx (V2DImode);
2065 201 : emit_insn_before (gen_vec_concatv2di (src, lo, hi), insn);
2066 201 : src = gen_lowpart (V1TImode, src);
2067 : }
2068 629 : else if (!HARD_REGISTER_NUM_P (REGNO (src)))
2069 : {
2070 624 : PUT_MODE (src, V1TImode);
2071 624 : fix_debug_reg_uses (src);
2072 : }
2073 : else
2074 5 : src = gen_raw_REG (V1TImode, REGNO (src));
2075 : }
2076 472094 : if (GENERAL_REG_P (dst))
2077 : {
2078 146 : rtx tmp = gen_reg_rtx (V2DImode);
2079 146 : src = gen_lowpart (V2DImode, src);
2080 146 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
2081 : /* Extracting hi before lo helps register allocation. */
2082 146 : rtx hi = gen_reg_rtx (DImode);
2083 146 : rtx lo = gen_reg_rtx (DImode);
2084 146 : emit_insn_before (gen_vec_extractv2didi (hi, tmp, const1_rtx), insn);
2085 146 : emit_insn_before (gen_vec_extractv2didi (lo, tmp, const0_rtx), insn);
2086 :
2087 : /* Construct *concatditi3 pattern from lo and hi. */
2088 146 : hi = gen_rtx_ZERO_EXTEND (TImode, hi);
2089 146 : hi = gen_rtx_ASHIFT (TImode, hi, GEN_INT (64));
2090 146 : lo = gen_rtx_ZERO_EXTEND (TImode, lo);
2091 146 : src = gen_rtx_PLUS (TImode, hi, lo);
2092 : }
2093 : break;
2094 :
2095 441482 : case MEM:
2096 441482 : PUT_MODE (src, V1TImode);
2097 441482 : break;
2098 :
2099 31059 : case CONST_WIDE_INT:
2100 31059 : if (NONDEBUG_INSN_P (insn))
2101 : {
2102 : /* Since there are no instructions to store 128-bit constant,
2103 : temporary register usage is required. */
2104 31059 : bool use_move;
2105 31059 : start_sequence ();
2106 31059 : tmp = ix86_convert_const_wide_int_to_broadcast (TImode, src);
2107 31059 : if (tmp)
2108 : {
2109 194 : src = lowpart_subreg (V1TImode, tmp, TImode);
2110 194 : use_move = true;
2111 : }
2112 : else
2113 : {
2114 30865 : src = smode_convert_cst (src, V1TImode);
2115 30865 : src = validize_mem (force_const_mem (V1TImode, src));
2116 30865 : use_move = MEM_P (dst);
2117 : }
2118 31059 : rtx_insn *seq = end_sequence ();
2119 31059 : if (seq)
2120 195 : emit_insn_before (seq, insn);
2121 31059 : if (use_move)
2122 : {
2123 30672 : tmp = gen_reg_rtx (V1TImode);
2124 30672 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
2125 30672 : src = tmp;
2126 : }
2127 : }
2128 : break;
2129 :
2130 10879 : case CONST_INT:
2131 10879 : switch (standard_sse_constant_p (src, TImode))
2132 : {
2133 10656 : case 1:
2134 10656 : src = CONST0_RTX (GET_MODE (dst));
2135 10656 : break;
2136 223 : case 2:
2137 223 : src = CONSTM1_RTX (GET_MODE (dst));
2138 223 : break;
2139 0 : default:
2140 0 : gcc_unreachable ();
2141 : }
2142 10879 : if (MEM_P (dst))
2143 : {
2144 10342 : tmp = gen_reg_rtx (V1TImode);
2145 10342 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
2146 10342 : src = tmp;
2147 : }
2148 : break;
2149 :
2150 13 : case AND:
2151 13 : if (GET_CODE (XEXP (src, 0)) == NOT)
2152 : {
2153 0 : convert_op (&XEXP (XEXP (src, 0), 0), insn);
2154 0 : convert_op (&XEXP (src, 1), insn);
2155 0 : PUT_MODE (XEXP (src, 0), V1TImode);
2156 0 : PUT_MODE (src, V1TImode);
2157 0 : break;
2158 : }
2159 13 : convert_op (&XEXP (src, 0), insn);
2160 13 : convert_op (&XEXP (src, 1), insn);
2161 13 : PUT_MODE (src, V1TImode);
2162 13 : if (MEM_P (dst))
2163 : {
2164 10 : tmp = gen_reg_rtx (V1TImode);
2165 10 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
2166 10 : src = tmp;
2167 : }
2168 : break;
2169 :
2170 343 : case XOR:
2171 343 : case IOR:
2172 343 : if (timode_concatdi_p (src))
2173 : {
2174 266 : src = timode_convert_concatdi (src, insn);
2175 266 : break;
2176 : }
2177 77 : convert_op (&XEXP (src, 0), insn);
2178 77 : convert_op (&XEXP (src, 1), insn);
2179 77 : PUT_MODE (src, V1TImode);
2180 77 : if (MEM_P (dst))
2181 : {
2182 8 : tmp = gen_reg_rtx (V1TImode);
2183 8 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
2184 8 : src = tmp;
2185 : }
2186 : break;
2187 :
2188 3 : case NOT:
2189 3 : src = XEXP (src, 0);
2190 3 : convert_op (&src, insn);
2191 3 : tmp = gen_reg_rtx (V1TImode);
2192 3 : emit_insn_before (gen_move_insn (tmp, CONSTM1_RTX (V1TImode)), insn);
2193 3 : src = gen_rtx_XOR (V1TImode, src, tmp);
2194 3 : if (MEM_P (dst))
2195 : {
2196 0 : tmp = gen_reg_rtx (V1TImode);
2197 0 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
2198 0 : src = tmp;
2199 : }
2200 : break;
2201 :
2202 12 : case COMPARE:
2203 12 : dst = gen_rtx_REG (CCZmode, FLAGS_REG);
2204 12 : src = convert_compare (XEXP (src, 0), XEXP (src, 1), insn);
2205 12 : break;
2206 :
2207 43 : case ASHIFT:
2208 43 : case LSHIFTRT:
2209 43 : case ASHIFTRT:
2210 43 : case ROTATERT:
2211 43 : case ROTATE:
2212 43 : convert_op (&XEXP (src, 0), insn);
2213 43 : PUT_MODE (src, V1TImode);
2214 43 : break;
2215 :
2216 103 : case ZERO_EXTEND:
2217 103 : if (GET_MODE (XEXP (src, 0)) == DImode)
2218 : {
2219 : /* Convert to *vec_concatv2di_0. */
2220 103 : rtx tmp = gen_reg_rtx (V2DImode);
2221 103 : rtx pat = gen_rtx_VEC_CONCAT (V2DImode, XEXP (src, 0), const0_rtx);
2222 103 : emit_insn_before (gen_move_insn (tmp, pat), insn);
2223 103 : src = gen_rtx_SUBREG (vmode, tmp, 0);
2224 : }
2225 : else
2226 0 : gcc_unreachable ();
2227 103 : break;
2228 :
2229 0 : case PLUS:
2230 0 : if (timode_concatdi_p (src))
2231 0 : src = timode_convert_concatdi (src, insn);
2232 : else
2233 0 : gcc_unreachable ();
2234 0 : break;
2235 :
2236 0 : default:
2237 0 : gcc_unreachable ();
2238 : }
2239 :
2240 956031 : SET_SRC (def_set) = src;
2241 956031 : SET_DEST (def_set) = dst;
2242 :
2243 : /* Drop possible dead definitions. */
2244 956031 : PATTERN (insn) = def_set;
2245 :
2246 956031 : INSN_CODE (insn) = -1;
2247 956031 : recog_memoized (insn);
2248 956031 : df_insn_rescan (insn);
2249 956031 : }
2250 :
2251 : /* Generate copies from defs used by the chain but not defined therein.
2252 : Also populates defs_map which is used later by convert_insn. */
2253 :
2254 : void
2255 657033 : scalar_chain::convert_registers ()
2256 : {
2257 657033 : bitmap_iterator bi;
2258 657033 : unsigned id;
2259 683214 : EXECUTE_IF_SET_IN_BITMAP (defs_conv, 0, id, bi)
2260 : {
2261 26181 : rtx chain_reg = gen_reg_rtx (smode);
2262 26181 : defs_map.put (regno_reg_rtx[id], chain_reg);
2263 : }
2264 665380 : EXECUTE_IF_SET_IN_BITMAP (insns_conv, 0, id, bi)
2265 20935 : for (df_ref ref = DF_INSN_UID_DEFS (id); ref; ref = DF_REF_NEXT_LOC (ref))
2266 12588 : if (bitmap_bit_p (defs_conv, DF_REF_REGNO (ref)))
2267 8347 : make_vector_copies (DF_REF_INSN (ref), DF_REF_REAL_REG (ref));
2268 657033 : }
2269 :
2270 : /* Convert whole chain creating required register
2271 : conversions and copies. */
2272 :
2273 : int
2274 657033 : scalar_chain::convert ()
2275 : {
2276 657033 : bitmap_iterator bi;
2277 657033 : unsigned id;
2278 657033 : int converted_insns = 0;
2279 :
2280 657033 : if (!dbg_cnt (stv_conversion))
2281 : return 0;
2282 :
2283 657033 : if (dump_file)
2284 0 : fprintf (dump_file, "Converting chain #%d...\n", chain_id);
2285 :
2286 657033 : convert_registers ();
2287 :
2288 2025529 : EXECUTE_IF_SET_IN_BITMAP (insns, 0, id, bi)
2289 : {
2290 1368496 : rtx_insn *insn = DF_INSN_UID_GET (id)->insn;
2291 1368496 : convert_insn_common (insn);
2292 1368496 : convert_insn (insn);
2293 1368496 : converted_insns++;
2294 : }
2295 :
2296 : return converted_insns;
2297 : }
2298 :
2299 : /* Return the SET expression if INSN doesn't reference hard register.
2300 : Return NULL if INSN uses or defines a hard register, excluding
2301 : pseudo register pushes, hard register uses in a memory address,
2302 : clobbers and flags definitions. */
2303 :
2304 : static rtx
2305 350490912 : pseudo_reg_set (rtx_insn *insn)
2306 : {
2307 350490912 : rtx set = single_set (insn);
2308 350490912 : if (!set)
2309 : return NULL;
2310 :
2311 : /* Check pseudo register push first. */
2312 137483508 : machine_mode mode = TARGET_64BIT ? TImode : DImode;
2313 137483508 : if (REG_P (SET_SRC (set))
2314 38887138 : && !HARD_REGISTER_P (SET_SRC (set))
2315 167848864 : && push_operand (SET_DEST (set), mode))
2316 : return set;
2317 :
2318 137230318 : df_ref ref;
2319 222235364 : FOR_EACH_INSN_DEF (ref, insn)
2320 122740429 : if (HARD_REGISTER_P (DF_REF_REAL_REG (ref))
2321 66286387 : && !DF_REF_FLAGS_IS_SET (ref, DF_REF_MUST_CLOBBER)
2322 174158661 : && DF_REF_REGNO (ref) != FLAGS_REG)
2323 : return NULL;
2324 :
2325 190527106 : FOR_EACH_INSN_USE (ref, insn)
2326 116882175 : if (!DF_REF_REG_MEM_P (ref) && HARD_REGISTER_P (DF_REF_REAL_REG (ref)))
2327 : return NULL;
2328 :
2329 : return set;
2330 : }
2331 :
2332 : /* Return true if the register REG is defined in a single DEF chain.
2333 : If it is defined in more than one DEF chains, we may not be able
2334 : to convert it in all chains. */
2335 :
2336 : static bool
2337 1293140 : single_def_chain_p (rtx reg)
2338 : {
2339 1293140 : df_ref ref = DF_REG_DEF_CHAIN (REGNO (reg));
2340 1293140 : if (!ref)
2341 : return false;
2342 1293108 : return DF_REF_NEXT_REG (ref) == nullptr;
2343 : }
2344 :
2345 : /* Check if comparison INSN may be transformed into vector comparison.
2346 : Currently we transform equality/inequality checks which look like:
2347 : (set (reg:CCZ 17 flags) (compare:CCZ (reg:TI x) (reg:TI y))) */
2348 :
2349 : static bool
2350 13059119 : convertible_comparison_p (rtx_insn *insn, enum machine_mode mode)
2351 : {
2352 14471119 : if (mode != (TARGET_64BIT ? TImode : DImode))
2353 : return false;
2354 :
2355 4784397 : if (!TARGET_SSE4_1)
2356 : return false;
2357 :
2358 164232 : rtx def_set = single_set (insn);
2359 :
2360 164232 : gcc_assert (def_set);
2361 :
2362 164232 : rtx src = SET_SRC (def_set);
2363 164232 : rtx dst = SET_DEST (def_set);
2364 :
2365 164232 : gcc_assert (GET_CODE (src) == COMPARE);
2366 :
2367 164232 : if (!REG_P (dst)
2368 164232 : || REGNO (dst) != FLAGS_REG
2369 328464 : || GET_MODE (dst) != CCZmode)
2370 : return false;
2371 :
2372 114925 : rtx op1 = XEXP (src, 0);
2373 114925 : rtx op2 = XEXP (src, 1);
2374 :
2375 : /* *cmp<dwi>_doubleword. */
2376 114925 : if (general_operand (op1, mode)
2377 114925 : && general_operand (op2, mode))
2378 : return true;
2379 :
2380 : /* *testti_doubleword. */
2381 114869 : if (op2 == const0_rtx
2382 38552 : && GET_CODE (op1) == AND
2383 142 : && REG_P (XEXP (op1, 0)))
2384 : {
2385 142 : rtx op12 = XEXP (op1, 1);
2386 142 : return GET_MODE (XEXP (op1, 0)) == TImode
2387 142 : && (CONST_SCALAR_INT_P (op12)
2388 0 : || ((REG_P (op12) || MEM_P (op12))
2389 0 : && GET_MODE (op12) == TImode));
2390 : }
2391 :
2392 : /* *test<dwi>_not_doubleword. */
2393 114727 : if (op2 == const0_rtx
2394 38410 : && GET_CODE (op1) == AND
2395 0 : && GET_CODE (XEXP (op1, 0)) == NOT)
2396 : {
2397 0 : rtx op11 = XEXP (XEXP (op1, 0), 0);
2398 0 : rtx op12 = XEXP (op1, 1);
2399 0 : return (REG_P (op11) || MEM_P (op11))
2400 0 : && (REG_P (op12) || MEM_P (op12))
2401 0 : && GET_MODE (op11) == mode
2402 0 : && GET_MODE (op12) == mode;
2403 : }
2404 :
2405 : return false;
2406 : }
2407 :
2408 : /* The general version of scalar_to_vector_candidate_p. */
2409 :
2410 : static bool
2411 244769796 : general_scalar_to_vector_candidate_p (rtx_insn *insn, enum machine_mode mode)
2412 : {
2413 244769796 : rtx def_set = pseudo_reg_set (insn);
2414 :
2415 244769796 : if (!def_set)
2416 : return false;
2417 :
2418 49966022 : rtx src = SET_SRC (def_set);
2419 49966022 : rtx dst = SET_DEST (def_set);
2420 :
2421 49966022 : if (GET_CODE (src) == COMPARE)
2422 8980722 : return convertible_comparison_p (insn, mode);
2423 :
2424 : /* We are interested in "mode" only. */
2425 40985300 : if ((GET_MODE (src) != mode
2426 27929582 : && !CONST_INT_P (src))
2427 18270339 : || GET_MODE (dst) != mode)
2428 : return false;
2429 :
2430 15352214 : if (!REG_P (dst) && !MEM_P (dst))
2431 : return false;
2432 :
2433 15094432 : switch (GET_CODE (src))
2434 : {
2435 555583 : case ASHIFT:
2436 555583 : case LSHIFTRT:
2437 555583 : case ASHIFTRT:
2438 555583 : case ROTATE:
2439 555583 : case ROTATERT:
2440 555583 : if (!CONST_INT_P (XEXP (src, 1))
2441 1073737 : || !IN_RANGE (INTVAL (XEXP (src, 1)), 0, GET_MODE_BITSIZE (mode)-1))
2442 : return false;
2443 :
2444 : /* Check for extend highpart case. */
2445 518150 : if (mode != DImode
2446 383173 : || GET_CODE (src) != ASHIFTRT
2447 89010 : || GET_CODE (XEXP (src, 0)) != ASHIFT)
2448 : break;
2449 :
2450 3741339 : src = XEXP (src, 0);
2451 : break;
2452 :
2453 100575 : case SMAX:
2454 100575 : case SMIN:
2455 100575 : case UMAX:
2456 100575 : case UMIN:
2457 100575 : if ((mode == DImode && !TARGET_AVX512VL)
2458 18283 : || (mode == SImode && !TARGET_SSE4_1))
2459 : return false;
2460 : /* Fallthru. */
2461 :
2462 3254553 : case AND:
2463 3254553 : case IOR:
2464 3254553 : case XOR:
2465 3254553 : case PLUS:
2466 3254553 : case MINUS:
2467 3254553 : if (!REG_P (XEXP (src, 1))
2468 : && !MEM_P (XEXP (src, 1))
2469 : && !CONST_INT_P (XEXP (src, 1)))
2470 : return false;
2471 :
2472 3161442 : if (GET_MODE (XEXP (src, 1)) != mode
2473 1827751 : && !CONST_INT_P (XEXP (src, 1)))
2474 : return false;
2475 :
2476 : /* Check for andnot case. */
2477 3161442 : if (GET_CODE (src) != AND
2478 172899 : || GET_CODE (XEXP (src, 0)) != NOT)
2479 : break;
2480 :
2481 3741339 : src = XEXP (src, 0);
2482 : /* FALLTHRU */
2483 :
2484 : case NOT:
2485 : break;
2486 :
2487 33248 : case NEG:
2488 : /* Check for nabs case. */
2489 33248 : if (GET_CODE (XEXP (src, 0)) != ABS)
2490 : break;
2491 :
2492 : src = XEXP (src, 0);
2493 : /* FALLTHRU */
2494 :
2495 3835 : case ABS:
2496 3835 : if ((mode == DImode && !TARGET_AVX512VL)
2497 1454 : || (mode == SImode && !TARGET_SSSE3))
2498 : return false;
2499 : break;
2500 :
2501 : case REG:
2502 : return true;
2503 :
2504 6051167 : case MEM:
2505 6051167 : case CONST_INT:
2506 6051167 : return REG_P (dst);
2507 :
2508 59528 : case VEC_SELECT:
2509 : /* Excluding MEM_P (dst) avoids interfering with vpextr[dq]. */
2510 59528 : return REG_P (dst)
2511 49339 : && REG_P (XEXP (src, 0))
2512 57437 : && GET_MODE (XEXP (src, 0)) == (mode == DImode ? V2DImode
2513 : : V4SImode)
2514 39334 : && GET_CODE (XEXP (src, 1)) == PARALLEL
2515 39334 : && XVECLEN (XEXP (src, 1), 0) == 1
2516 98862 : && CONST_INT_P (XVECEXP (XEXP (src, 1), 0, 0));
2517 :
2518 322144 : case ZERO_EXTEND:
2519 : /* *zero_extendsidi2 becomes *vec_setv2di_0_zero_extendsi_1. */
2520 322144 : return mode == DImode
2521 229624 : && REG_P (dst)
2522 226873 : && GET_MODE (XEXP (src, 0)) == SImode
2523 513150 : && MEM_P (XEXP (src, 0));
2524 :
2525 : default:
2526 : return false;
2527 : }
2528 :
2529 3741339 : if (!REG_P (XEXP (src, 0))
2530 : && !MEM_P (XEXP (src, 0))
2531 : && !CONST_INT_P (XEXP (src, 0)))
2532 : return false;
2533 :
2534 3449613 : if (GET_MODE (XEXP (src, 0)) != mode
2535 0 : && !CONST_INT_P (XEXP (src, 0)))
2536 : return false;
2537 :
2538 : return true;
2539 : }
2540 :
2541 : /* Check for a suitable TImode memory operand. */
2542 :
2543 : static bool
2544 1613 : timode_mem_p (rtx x)
2545 : {
2546 1613 : return MEM_P (x)
2547 1613 : && (TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
2548 0 : || !misaligned_operand (x, TImode));
2549 : }
2550 :
2551 : /* The TImode version of scalar_to_vector_candidate_p. */
2552 :
2553 : static bool
2554 105721116 : timode_scalar_to_vector_candidate_p (rtx_insn *insn)
2555 : {
2556 105721116 : rtx def_set = pseudo_reg_set (insn);
2557 :
2558 : /* We allow two exceptions to the pseudo registers only rule.
2559 : Setting a hard register from a pseudo, and setting a pseudo
2560 : from a hard register. */
2561 105721116 : if (!def_set)
2562 : {
2563 81789017 : def_set = single_set (insn);
2564 81789017 : if (def_set)
2565 : {
2566 18362577 : rtx src = SET_SRC (def_set);
2567 18362577 : rtx dst = SET_DEST (def_set);
2568 18362577 : if (GET_MODE (dst) == TImode
2569 223138 : && REG_P (src) && REG_P (dst))
2570 : {
2571 102030 : if (HARD_REGISTER_P (dst)
2572 52616 : && !HARD_REGISTER_P (src)
2573 154646 : && single_def_chain_p (src))
2574 : return true;
2575 73310 : if (HARD_REGISTER_P (src)
2576 49414 : && !HARD_REGISTER_P (dst)
2577 122724 : && single_def_chain_p (dst))
2578 : return true;
2579 : }
2580 : }
2581 : return false;
2582 : }
2583 :
2584 23932099 : rtx src = SET_SRC (def_set);
2585 23932099 : rtx dst = SET_DEST (def_set);
2586 :
2587 23932099 : if (GET_CODE (src) == COMPARE)
2588 4078397 : return convertible_comparison_p (insn, TImode);
2589 :
2590 19853702 : if (GET_MODE (dst) != TImode
2591 1235309 : || (GET_MODE (src) != TImode
2592 60374 : && !CONST_SCALAR_INT_P (src)))
2593 : return false;
2594 :
2595 1235309 : if (!REG_P (dst) && !MEM_P (dst))
2596 : return false;
2597 :
2598 1233856 : if (MEM_P (dst)
2599 549453 : && misaligned_operand (dst, TImode)
2600 1561923 : && !TARGET_SSE_UNALIGNED_STORE_OPTIMAL)
2601 : return false;
2602 :
2603 1233851 : if (REG_P (dst) && !single_def_chain_p (dst))
2604 : return false;
2605 :
2606 1079534 : switch (GET_CODE (src))
2607 : {
2608 506707 : case REG:
2609 506707 : return single_def_chain_p (src);
2610 :
2611 : case CONST_WIDE_INT:
2612 : return true;
2613 :
2614 12841 : case CONST_INT:
2615 : /* ??? Verify performance impact before enabling CONST_INT for
2616 : __int128 store. */
2617 12841 : return standard_sse_constant_p (src, TImode);
2618 :
2619 463582 : case MEM:
2620 : /* Memory must be aligned or unaligned load is optimal. */
2621 463582 : return (REG_P (dst)
2622 463582 : && (!misaligned_operand (src, TImode)
2623 162117 : || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL));
2624 :
2625 3232 : case AND:
2626 3232 : if (!MEM_P (dst)
2627 3191 : && GET_CODE (XEXP (src, 0)) == NOT
2628 0 : && REG_P (XEXP (XEXP (src, 0), 0))
2629 3232 : && (REG_P (XEXP (src, 1))
2630 0 : || CONST_SCALAR_INT_P (XEXP (src, 1))
2631 0 : || timode_mem_p (XEXP (src, 1))))
2632 0 : return true;
2633 3232 : return (REG_P (XEXP (src, 0))
2634 46 : || timode_mem_p (XEXP (src, 0)))
2635 3278 : && (REG_P (XEXP (src, 1))
2636 1363 : || CONST_SCALAR_INT_P (XEXP (src, 1))
2637 35 : || timode_mem_p (XEXP (src, 1)));
2638 :
2639 14362 : case IOR:
2640 14362 : case XOR:
2641 14362 : if (timode_concatdi_p (src))
2642 : return true;
2643 2801 : return (REG_P (XEXP (src, 0))
2644 1462 : || timode_mem_p (XEXP (src, 0)))
2645 2818 : && (REG_P (XEXP (src, 1))
2646 291 : || CONST_SCALAR_INT_P (XEXP (src, 1))
2647 54 : || timode_mem_p (XEXP (src, 1)));
2648 :
2649 513 : case NOT:
2650 513 : return REG_P (XEXP (src, 0)) || timode_mem_p (XEXP (src, 0));
2651 :
2652 11612 : case ASHIFT:
2653 11612 : case LSHIFTRT:
2654 11612 : case ASHIFTRT:
2655 11612 : case ROTATERT:
2656 11612 : case ROTATE:
2657 : /* Handle shifts/rotates by integer constants between 0 and 127. */
2658 11612 : return REG_P (XEXP (src, 0))
2659 11580 : && CONST_INT_P (XEXP (src, 1))
2660 22840 : && (INTVAL (XEXP (src, 1)) & ~0x7f) == 0;
2661 :
2662 7049 : case PLUS:
2663 7049 : return timode_concatdi_p (src);
2664 :
2665 3806 : case ZERO_EXTEND:
2666 3806 : return REG_P (XEXP (src, 0))
2667 3806 : && GET_MODE (XEXP (src, 0)) == DImode;
2668 :
2669 : default:
2670 : return false;
2671 : }
2672 : }
2673 :
2674 : /* For a register REGNO, scan instructions for its defs and uses.
2675 : Put REGNO in REGS if a def or use isn't in CANDIDATES. */
2676 :
2677 : static void
2678 1271705 : timode_check_non_convertible_regs (bitmap candidates, bitmap regs,
2679 : unsigned int regno)
2680 : {
2681 : /* Do nothing if REGNO is already in REGS or is a hard reg. */
2682 1271705 : if (bitmap_bit_p (regs, regno)
2683 1271705 : || HARD_REGISTER_NUM_P (regno))
2684 : return;
2685 :
2686 1263691 : for (df_ref def = DF_REG_DEF_CHAIN (regno);
2687 2516973 : def;
2688 1253282 : def = DF_REF_NEXT_REG (def))
2689 : {
2690 1263671 : if (!bitmap_bit_p (candidates, DF_REF_INSN_UID (def)))
2691 : {
2692 10389 : if (dump_file)
2693 0 : fprintf (dump_file,
2694 : "r%d has non convertible def in insn %d\n",
2695 0 : regno, DF_REF_INSN_UID (def));
2696 :
2697 10389 : bitmap_set_bit (regs, regno);
2698 10389 : break;
2699 : }
2700 : }
2701 :
2702 1263691 : for (df_ref ref = DF_REG_USE_CHAIN (regno);
2703 2797560 : ref;
2704 1533869 : ref = DF_REF_NEXT_REG (ref))
2705 : {
2706 : /* Debug instructions are skipped. */
2707 1603629 : if (NONDEBUG_INSN_P (DF_REF_INSN (ref))
2708 1603629 : && !bitmap_bit_p (candidates, DF_REF_INSN_UID (ref)))
2709 : {
2710 69760 : if (dump_file)
2711 0 : fprintf (dump_file,
2712 : "r%d has non convertible use in insn %d\n",
2713 0 : regno, DF_REF_INSN_UID (ref));
2714 :
2715 69760 : bitmap_set_bit (regs, regno);
2716 69760 : break;
2717 : }
2718 : }
2719 : }
2720 :
2721 : /* For a given bitmap of insn UIDs scans all instructions and
2722 : remove insn from CANDIDATES in case it has both convertible
2723 : and not convertible definitions.
2724 :
2725 : All insns in a bitmap are conversion candidates according to
2726 : scalar_to_vector_candidate_p. Currently it implies all insns
2727 : are single_set. */
2728 :
2729 : static void
2730 844394 : timode_remove_non_convertible_regs (bitmap candidates)
2731 : {
2732 844394 : bitmap_iterator bi;
2733 844394 : unsigned id;
2734 844394 : bitmap regs = BITMAP_ALLOC (NULL);
2735 865605 : bool changed;
2736 :
2737 865605 : do {
2738 865605 : changed = false;
2739 2161590 : EXECUTE_IF_SET_IN_BITMAP (candidates, 0, id, bi)
2740 : {
2741 1295985 : rtx_insn *insn = DF_INSN_UID_GET (id)->insn;
2742 1295985 : df_ref ref;
2743 :
2744 2023173 : FOR_EACH_INSN_DEF (ref, insn)
2745 727188 : if (!DF_REF_REG_MEM_P (ref)
2746 727188 : && GET_MODE (DF_REF_REG (ref)) == TImode)
2747 638883 : timode_check_non_convertible_regs (candidates, regs,
2748 : DF_REF_REGNO (ref));
2749 :
2750 3194347 : FOR_EACH_INSN_USE (ref, insn)
2751 1898362 : if (DF_REF_TYPE (ref) == DF_REF_REG_USE
2752 769562 : && GET_MODE (DF_REF_REG (ref)) == TImode
2753 632827 : && !SUBREG_P (DF_REF_REG (ref)))
2754 632822 : timode_check_non_convertible_regs (candidates, regs,
2755 : DF_REF_REGNO (ref));
2756 : }
2757 :
2758 1041907 : EXECUTE_IF_SET_IN_BITMAP (regs, 0, id, bi)
2759 : {
2760 176302 : for (df_ref def = DF_REG_DEF_CHAIN (id);
2761 358858 : def;
2762 182556 : def = DF_REF_NEXT_REG (def))
2763 182556 : if (bitmap_bit_p (candidates, DF_REF_INSN_UID (def)))
2764 : {
2765 56352 : if (dump_file)
2766 0 : fprintf (dump_file, "Removing insn %d from candidates list\n",
2767 0 : DF_REF_INSN_UID (def));
2768 :
2769 56352 : bitmap_clear_bit (candidates, DF_REF_INSN_UID (def));
2770 56352 : changed = true;
2771 : }
2772 :
2773 176302 : for (df_ref ref = DF_REG_USE_CHAIN (id);
2774 524630 : ref;
2775 348328 : ref = DF_REF_NEXT_REG (ref))
2776 348328 : if (bitmap_bit_p (candidates, DF_REF_INSN_UID (ref)))
2777 : {
2778 16129 : if (dump_file)
2779 0 : fprintf (dump_file, "Removing insn %d from candidates list\n",
2780 0 : DF_REF_INSN_UID (ref));
2781 :
2782 16129 : bitmap_clear_bit (candidates, DF_REF_INSN_UID (ref));
2783 16129 : changed = true;
2784 : }
2785 : }
2786 : } while (changed);
2787 :
2788 844394 : BITMAP_FREE (regs);
2789 844394 : }
2790 :
2791 : /* Main STV pass function. Find and convert scalar
2792 : instructions into vector mode when profitable. */
2793 :
2794 : static unsigned int
2795 1815139 : convert_scalars_to_vector (bool timode_p)
2796 : {
2797 1815139 : basic_block bb;
2798 1815139 : int converted_insns = 0;
2799 1815139 : auto_vec<rtx_insn *> control_flow_insns;
2800 :
2801 1815139 : bitmap_obstack_initialize (NULL);
2802 1815139 : const machine_mode cand_mode[3] = { SImode, DImode, TImode };
2803 1815139 : const machine_mode cand_vmode[3] = { V4SImode, V2DImode, V1TImode };
2804 5445417 : bitmap_head candidates[3]; /* { SImode, DImode, TImode } */
2805 7260556 : for (unsigned i = 0; i < 3; ++i)
2806 5445417 : bitmap_initialize (&candidates[i], &bitmap_default_obstack);
2807 :
2808 1815139 : calculate_dominance_info (CDI_DOMINATORS);
2809 1815139 : df_set_flags (DF_DEFER_INSN_RESCAN | DF_RD_PRUNE_DEAD_DEFS);
2810 1815139 : df_chain_add_problem (DF_DU_CHAIN | DF_UD_CHAIN);
2811 1815139 : df_analyze ();
2812 :
2813 : /* Find all instructions we want to convert into vector mode. */
2814 1815139 : if (dump_file)
2815 44 : fprintf (dump_file, "Searching for mode conversion candidates...\n");
2816 :
2817 20081786 : FOR_EACH_BB_FN (bb, cfun)
2818 : {
2819 18266647 : rtx_insn *insn;
2820 248130784 : FOR_BB_INSNS (bb, insn)
2821 229864137 : if (timode_p
2822 229864137 : && timode_scalar_to_vector_candidate_p (insn))
2823 : {
2824 1117610 : if (dump_file)
2825 0 : fprintf (dump_file, " insn %d is marked as a TImode candidate\n",
2826 0 : INSN_UID (insn));
2827 :
2828 1117610 : bitmap_set_bit (&candidates[2], INSN_UID (insn));
2829 : }
2830 228746527 : else if (!timode_p)
2831 : {
2832 : /* Check {SI,DI}mode. */
2833 357086159 : for (unsigned i = 0; i <= 1; ++i)
2834 244769796 : if (general_scalar_to_vector_candidate_p (insn, cand_mode[i]))
2835 : {
2836 11826658 : if (dump_file)
2837 554 : fprintf (dump_file, " insn %d is marked as a %s candidate\n",
2838 277 : INSN_UID (insn), i == 0 ? "SImode" : "DImode");
2839 :
2840 11826658 : bitmap_set_bit (&candidates[i], INSN_UID (insn));
2841 11826658 : break;
2842 : }
2843 : }
2844 : }
2845 :
2846 1815139 : if (timode_p)
2847 844394 : timode_remove_non_convertible_regs (&candidates[2]);
2848 :
2849 5748659 : for (unsigned i = 0; i <= 2; ++i)
2850 4583042 : if (!bitmap_empty_p (&candidates[i]))
2851 : break;
2852 3933520 : else if (i == 2 && dump_file)
2853 23 : fprintf (dump_file, "There are no candidates for optimization.\n");
2854 :
2855 7260556 : for (unsigned i = 0; i <= 2; ++i)
2856 : {
2857 5445417 : auto_bitmap disallowed;
2858 5445417 : bitmap_tree_view (&candidates[i]);
2859 17436804 : while (!bitmap_empty_p (&candidates[i]))
2860 : {
2861 6545970 : unsigned uid = bitmap_first_set_bit (&candidates[i]);
2862 6545970 : scalar_chain *chain;
2863 :
2864 6545970 : if (cand_mode[i] == TImode)
2865 525080 : chain = new timode_scalar_chain;
2866 : else
2867 6020890 : chain = new general_scalar_chain (cand_mode[i], cand_vmode[i]);
2868 :
2869 : /* Find instructions chain we want to convert to vector mode.
2870 : Check all uses and definitions to estimate all required
2871 : conversions. */
2872 6545970 : if (chain->build (&candidates[i], uid, disallowed))
2873 : {
2874 6542082 : if (chain->compute_convert_gain ())
2875 657033 : converted_insns += chain->convert ();
2876 5885049 : else if (dump_file)
2877 136 : fprintf (dump_file, "Chain #%d conversion is not profitable\n",
2878 : chain->chain_id);
2879 : }
2880 :
2881 6545970 : rtx_insn* iter_insn;
2882 6545970 : unsigned int ii;
2883 6549558 : FOR_EACH_VEC_ELT (chain->control_flow_insns, ii, iter_insn)
2884 3588 : control_flow_insns.safe_push (iter_insn);
2885 :
2886 6545970 : delete chain;
2887 : }
2888 5445417 : }
2889 :
2890 1815139 : if (dump_file)
2891 44 : fprintf (dump_file, "Total insns converted: %d\n", converted_insns);
2892 :
2893 7260556 : for (unsigned i = 0; i <= 2; ++i)
2894 5445417 : bitmap_release (&candidates[i]);
2895 1815139 : bitmap_obstack_release (NULL);
2896 1815139 : df_process_deferred_rescans ();
2897 :
2898 : /* Conversion means we may have 128bit register spills/fills
2899 : which require aligned stack. */
2900 1815139 : if (converted_insns)
2901 : {
2902 106551 : if (crtl->stack_alignment_needed < 128)
2903 2441 : crtl->stack_alignment_needed = 128;
2904 106551 : if (crtl->stack_alignment_estimated < 128)
2905 221 : crtl->stack_alignment_estimated = 128;
2906 :
2907 106551 : crtl->stack_realign_needed
2908 106551 : = INCOMING_STACK_BOUNDARY < crtl->stack_alignment_estimated;
2909 106551 : crtl->stack_realign_tried = crtl->stack_realign_needed;
2910 :
2911 106551 : crtl->stack_realign_processed = true;
2912 :
2913 106551 : if (!crtl->drap_reg)
2914 : {
2915 106365 : rtx drap_rtx = targetm.calls.get_drap_rtx ();
2916 :
2917 : /* stack_realign_drap and drap_rtx must match. */
2918 106365 : gcc_assert ((stack_realign_drap != 0) == (drap_rtx != NULL));
2919 :
2920 : /* Do nothing if NULL is returned,
2921 : which means DRAP is not needed. */
2922 106365 : if (drap_rtx != NULL)
2923 : {
2924 0 : crtl->args.internal_arg_pointer = drap_rtx;
2925 :
2926 : /* Call fixup_tail_calls to clean up
2927 : REG_EQUIV note if DRAP is needed. */
2928 0 : fixup_tail_calls ();
2929 : }
2930 : }
2931 :
2932 : /* Fix up DECL_RTL/DECL_INCOMING_RTL of arguments. */
2933 106551 : if (TARGET_64BIT)
2934 67994 : for (tree parm = DECL_ARGUMENTS (current_function_decl);
2935 186657 : parm; parm = DECL_CHAIN (parm))
2936 : {
2937 118663 : if (TYPE_MODE (TREE_TYPE (parm)) != TImode)
2938 102580 : continue;
2939 16083 : if (DECL_RTL_SET_P (parm)
2940 32166 : && GET_MODE (DECL_RTL (parm)) == V1TImode)
2941 : {
2942 611 : rtx r = DECL_RTL (parm);
2943 611 : if (REG_P (r))
2944 611 : SET_DECL_RTL (parm, gen_rtx_SUBREG (TImode, r, 0));
2945 : }
2946 16083 : if (DECL_INCOMING_RTL (parm)
2947 16083 : && GET_MODE (DECL_INCOMING_RTL (parm)) == V1TImode)
2948 : {
2949 0 : rtx r = DECL_INCOMING_RTL (parm);
2950 0 : if (REG_P (r))
2951 0 : DECL_INCOMING_RTL (parm) = gen_rtx_SUBREG (TImode, r, 0);
2952 : }
2953 : }
2954 :
2955 106551 : if (!control_flow_insns.is_empty ())
2956 : {
2957 1130 : free_dominance_info (CDI_DOMINATORS);
2958 :
2959 1130 : unsigned int i;
2960 1130 : rtx_insn* insn;
2961 5848 : FOR_EACH_VEC_ELT (control_flow_insns, i, insn)
2962 3588 : if (control_flow_insn_p (insn))
2963 : {
2964 : /* Split the block after insn. There will be a fallthru
2965 : edge, which is OK so we keep it. We have to create
2966 : the exception edges ourselves. */
2967 3588 : bb = BLOCK_FOR_INSN (insn);
2968 3588 : split_block (bb, insn);
2969 3588 : rtl_make_eh_edge (NULL, bb, BB_END (bb));
2970 : }
2971 : }
2972 : }
2973 :
2974 1815139 : return 0;
2975 1815139 : }
2976 :
2977 : static unsigned int
2978 74556 : rest_of_handle_insert_vzeroupper (void)
2979 : {
2980 : /* vzeroupper instructions are inserted immediately after reload and
2981 : postreload_cse to clean up after it a little bit to account for possible
2982 : spills from 256bit or 512bit registers. The pass reuses mode switching
2983 : infrastructure by re-running mode insertion pass, so disable entities
2984 : that have already been processed. */
2985 521892 : for (int i = 0; i < MAX_386_ENTITIES; i++)
2986 447336 : ix86_optimize_mode_switching[i] = 0;
2987 :
2988 74556 : ix86_optimize_mode_switching[AVX_U128] = 1;
2989 :
2990 : /* Call optimize_mode_switching. */
2991 74556 : g->get_passes ()->execute_pass_mode_switching ();
2992 :
2993 : /* LRA removes all REG_DEAD/REG_UNUSED notes and normally they
2994 : reappear in the IL only at the start of pass_rtl_dse2, which does
2995 : df_note_add_problem (); df_analyze ();
2996 : The vzeroupper is scheduled after postreload_cse pass and mode
2997 : switching computes the notes as well, the problem is that e.g.
2998 : pass_gcse2 doesn't maintain the notes, see PR113059 and
2999 : PR112760. Remove the notes now to restore status quo ante
3000 : until we figure out how to maintain the notes or what else
3001 : to do. */
3002 74556 : basic_block bb;
3003 74556 : rtx_insn *insn;
3004 405317 : FOR_EACH_BB_FN (bb, cfun)
3005 4256724 : FOR_BB_INSNS (bb, insn)
3006 3925963 : if (NONDEBUG_INSN_P (insn))
3007 : {
3008 2090486 : rtx *pnote = ®_NOTES (insn);
3009 3881136 : while (*pnote != 0)
3010 : {
3011 1790650 : if (REG_NOTE_KIND (*pnote) == REG_DEAD
3012 818441 : || REG_NOTE_KIND (*pnote) == REG_UNUSED)
3013 1283157 : *pnote = XEXP (*pnote, 1);
3014 : else
3015 507493 : pnote = &XEXP (*pnote, 1);
3016 : }
3017 : }
3018 :
3019 74556 : df_remove_problem (df_note);
3020 74556 : df_analyze ();
3021 74556 : return 0;
3022 : }
3023 :
3024 : namespace {
3025 :
3026 : const pass_data pass_data_insert_vzeroupper =
3027 : {
3028 : RTL_PASS, /* type */
3029 : "vzeroupper", /* name */
3030 : OPTGROUP_NONE, /* optinfo_flags */
3031 : TV_MACH_DEP, /* tv_id */
3032 : 0, /* properties_required */
3033 : 0, /* properties_provided */
3034 : 0, /* properties_destroyed */
3035 : 0, /* todo_flags_start */
3036 : TODO_df_finish, /* todo_flags_finish */
3037 : };
3038 :
3039 : class pass_insert_vzeroupper : public rtl_opt_pass
3040 : {
3041 : public:
3042 292371 : pass_insert_vzeroupper(gcc::context *ctxt)
3043 584742 : : rtl_opt_pass(pass_data_insert_vzeroupper, ctxt)
3044 : {}
3045 :
3046 : /* opt_pass methods: */
3047 1504958 : bool gate (function *) final override
3048 : {
3049 1504958 : return TARGET_AVX && TARGET_VZEROUPPER;
3050 : }
3051 :
3052 74556 : unsigned int execute (function *) final override
3053 : {
3054 74556 : return rest_of_handle_insert_vzeroupper ();
3055 : }
3056 :
3057 : }; // class pass_insert_vzeroupper
3058 :
3059 : const pass_data pass_data_stv =
3060 : {
3061 : RTL_PASS, /* type */
3062 : "stv", /* name */
3063 : OPTGROUP_NONE, /* optinfo_flags */
3064 : TV_MACH_DEP, /* tv_id */
3065 : 0, /* properties_required */
3066 : 0, /* properties_provided */
3067 : 0, /* properties_destroyed */
3068 : 0, /* todo_flags_start */
3069 : TODO_df_finish, /* todo_flags_finish */
3070 : };
3071 :
3072 : class pass_stv : public rtl_opt_pass
3073 : {
3074 : public:
3075 584742 : pass_stv (gcc::context *ctxt)
3076 584742 : : rtl_opt_pass (pass_data_stv, ctxt),
3077 1169484 : timode_p (false)
3078 : {}
3079 :
3080 : /* opt_pass methods: */
3081 3009916 : bool gate (function *) final override
3082 : {
3083 1504958 : return ((!timode_p || TARGET_64BIT)
3084 4388202 : && TARGET_STV && TARGET_SSE2 && optimize > 1);
3085 : }
3086 :
3087 1815139 : unsigned int execute (function *) final override
3088 : {
3089 1815139 : return convert_scalars_to_vector (timode_p);
3090 : }
3091 :
3092 292371 : opt_pass *clone () final override
3093 : {
3094 292371 : return new pass_stv (m_ctxt);
3095 : }
3096 :
3097 584742 : void set_pass_param (unsigned int n, bool param) final override
3098 : {
3099 584742 : gcc_assert (n == 0);
3100 584742 : timode_p = param;
3101 584742 : }
3102 :
3103 : private:
3104 : bool timode_p;
3105 : }; // class pass_stv
3106 :
3107 : } // anon namespace
3108 :
3109 : rtl_opt_pass *
3110 292371 : make_pass_insert_vzeroupper (gcc::context *ctxt)
3111 : {
3112 292371 : return new pass_insert_vzeroupper (ctxt);
3113 : }
3114 :
3115 : rtl_opt_pass *
3116 292371 : make_pass_stv (gcc::context *ctxt)
3117 : {
3118 292371 : return new pass_stv (ctxt);
3119 : }
3120 :
3121 : /* Inserting ENDBR and pseudo patchable-area instructions. */
3122 :
3123 : static void
3124 205478 : rest_of_insert_endbr_and_patchable_area (bool need_endbr,
3125 : unsigned int patchable_area_size)
3126 : {
3127 205478 : rtx endbr;
3128 205478 : rtx_insn *insn;
3129 205478 : rtx_insn *endbr_insn = NULL;
3130 205478 : basic_block bb;
3131 :
3132 205478 : if (need_endbr)
3133 : {
3134 : /* Currently emit EB if it's a tracking function, i.e. 'nocf_check'
3135 : is absent among function attributes. Later an optimization will
3136 : be introduced to make analysis if an address of a static function
3137 : is taken. A static function whose address is not taken will get
3138 : a nocf_check attribute. This will allow to reduce the number of
3139 : EB. */
3140 205433 : if (!lookup_attribute ("nocf_check",
3141 205433 : TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
3142 205415 : && (!flag_manual_endbr
3143 8 : || lookup_attribute ("cf_check",
3144 8 : DECL_ATTRIBUTES (cfun->decl)))
3145 410847 : && (!cgraph_node::get (cfun->decl)->only_called_directly_p ()
3146 29794 : || ix86_cmodel == CM_LARGE
3147 29793 : || ix86_cmodel == CM_LARGE_PIC
3148 29792 : || flag_force_indirect_call
3149 29792 : || (TARGET_DLLIMPORT_DECL_ATTRIBUTES
3150 : && DECL_DLLIMPORT_P (cfun->decl))))
3151 : {
3152 175623 : if (crtl->profile && flag_fentry)
3153 : {
3154 : /* Queue ENDBR insertion to x86_function_profiler.
3155 : NB: Any patchable-area insn will be inserted after
3156 : ENDBR. */
3157 6 : cfun->machine->insn_queued_at_entrance = TYPE_ENDBR;
3158 : }
3159 : else
3160 : {
3161 175617 : endbr = gen_nop_endbr ();
3162 175617 : bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
3163 175617 : rtx_insn *insn = BB_HEAD (bb);
3164 175617 : endbr_insn = emit_insn_before (endbr, insn);
3165 : }
3166 : }
3167 : }
3168 :
3169 205478 : if (patchable_area_size)
3170 : {
3171 51 : if (crtl->profile && flag_fentry)
3172 : {
3173 : /* Queue patchable-area insertion to x86_function_profiler.
3174 : NB: If there is a queued ENDBR, x86_function_profiler
3175 : will also handle patchable-area. */
3176 2 : if (!cfun->machine->insn_queued_at_entrance)
3177 1 : cfun->machine->insn_queued_at_entrance = TYPE_PATCHABLE_AREA;
3178 : }
3179 : else
3180 : {
3181 49 : rtx patchable_area
3182 49 : = gen_patchable_area (GEN_INT (patchable_area_size),
3183 49 : GEN_INT (crtl->patch_area_entry == 0));
3184 49 : if (endbr_insn)
3185 3 : emit_insn_after (patchable_area, endbr_insn);
3186 : else
3187 : {
3188 46 : bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
3189 46 : insn = BB_HEAD (bb);
3190 46 : emit_insn_before (patchable_area, insn);
3191 : }
3192 : }
3193 : }
3194 :
3195 205478 : if (!need_endbr)
3196 : return;
3197 :
3198 205433 : bb = 0;
3199 4164622 : FOR_EACH_BB_FN (bb, cfun)
3200 : {
3201 77730934 : for (insn = BB_HEAD (bb); insn != NEXT_INSN (BB_END (bb));
3202 73771745 : insn = NEXT_INSN (insn))
3203 : {
3204 73771745 : if (CALL_P (insn))
3205 : {
3206 1475037 : need_endbr = find_reg_note (insn, REG_SETJMP, NULL) != NULL;
3207 1475037 : if (!need_endbr && !SIBLING_CALL_P (insn))
3208 : {
3209 1422776 : rtx call = get_call_rtx_from (insn);
3210 1422776 : rtx fnaddr = XEXP (call, 0);
3211 1422776 : tree fndecl = NULL_TREE;
3212 :
3213 : /* Also generate ENDBRANCH for non-tail call which
3214 : may return via indirect branch. */
3215 1422776 : if (SYMBOL_REF_P (XEXP (fnaddr, 0)))
3216 1362782 : fndecl = SYMBOL_REF_DECL (XEXP (fnaddr, 0));
3217 1362782 : if (fndecl == NULL_TREE)
3218 60362 : fndecl = MEM_EXPR (fnaddr);
3219 60362 : if (fndecl
3220 1420369 : && TREE_CODE (TREE_TYPE (fndecl)) != FUNCTION_TYPE
3221 601798 : && TREE_CODE (TREE_TYPE (fndecl)) != METHOD_TYPE)
3222 : fndecl = NULL_TREE;
3223 1422776 : if (fndecl && TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
3224 : {
3225 1379794 : tree fntype = TREE_TYPE (fndecl);
3226 1379794 : if (lookup_attribute ("indirect_return",
3227 1379794 : TYPE_ATTRIBUTES (fntype)))
3228 : need_endbr = true;
3229 : }
3230 : }
3231 1475025 : if (!need_endbr)
3232 1475017 : continue;
3233 : /* Generate ENDBRANCH after CALL, which can return more than
3234 : twice, setjmp-like functions. */
3235 :
3236 20 : endbr = gen_nop_endbr ();
3237 20 : emit_insn_after_setloc (endbr, insn, INSN_LOCATION (insn));
3238 20 : continue;
3239 20 : }
3240 :
3241 72296708 : if (JUMP_P (insn) && flag_cet_switch)
3242 : {
3243 9 : rtx target = JUMP_LABEL (insn);
3244 9 : if (target == NULL_RTX || ANY_RETURN_P (target))
3245 5 : continue;
3246 :
3247 : /* Check the jump is a switch table. */
3248 4 : rtx_insn *label = as_a<rtx_insn *> (target);
3249 4 : rtx_insn *table = next_insn (label);
3250 4 : if (table == NULL_RTX || !JUMP_TABLE_DATA_P (table))
3251 2 : continue;
3252 :
3253 : /* For the indirect jump find out all places it jumps and insert
3254 : ENDBRANCH there. It should be done under a special flag to
3255 : control ENDBRANCH generation for switch stmts. */
3256 2 : edge_iterator ei;
3257 2 : edge e;
3258 2 : basic_block dest_blk;
3259 :
3260 24 : FOR_EACH_EDGE (e, ei, bb->succs)
3261 : {
3262 22 : rtx_insn *insn;
3263 :
3264 22 : dest_blk = e->dest;
3265 22 : insn = BB_HEAD (dest_blk);
3266 22 : gcc_assert (LABEL_P (insn));
3267 22 : endbr = gen_nop_endbr ();
3268 22 : emit_insn_after (endbr, insn);
3269 : }
3270 2 : continue;
3271 2 : }
3272 :
3273 72296699 : if (LABEL_P (insn) && LABEL_PRESERVE_P (insn))
3274 : {
3275 150511 : endbr = gen_nop_endbr ();
3276 150511 : emit_insn_after (endbr, insn);
3277 150511 : continue;
3278 : }
3279 : }
3280 : }
3281 :
3282 : return;
3283 : }
3284 :
3285 : namespace {
3286 :
3287 : const pass_data pass_data_insert_endbr_and_patchable_area =
3288 : {
3289 : RTL_PASS, /* type. */
3290 : "endbr_and_patchable_area", /* name. */
3291 : OPTGROUP_NONE, /* optinfo_flags. */
3292 : TV_MACH_DEP, /* tv_id. */
3293 : 0, /* properties_required. */
3294 : 0, /* properties_provided. */
3295 : 0, /* properties_destroyed. */
3296 : 0, /* todo_flags_start. */
3297 : 0, /* todo_flags_finish. */
3298 : };
3299 :
3300 : class pass_insert_endbr_and_patchable_area : public rtl_opt_pass
3301 : {
3302 : public:
3303 292371 : pass_insert_endbr_and_patchable_area (gcc::context *ctxt)
3304 584742 : : rtl_opt_pass (pass_data_insert_endbr_and_patchable_area, ctxt)
3305 : {}
3306 :
3307 : /* opt_pass methods: */
3308 1504958 : bool gate (function *) final override
3309 : {
3310 1504958 : need_endbr = (flag_cf_protection & CF_BRANCH) != 0;
3311 1504958 : patchable_area_size = crtl->patch_area_size - crtl->patch_area_entry;
3312 1504958 : return need_endbr || patchable_area_size;
3313 : }
3314 :
3315 205478 : unsigned int execute (function *) final override
3316 : {
3317 205478 : timevar_push (TV_MACH_DEP);
3318 205478 : rest_of_insert_endbr_and_patchable_area (need_endbr,
3319 : patchable_area_size);
3320 205478 : timevar_pop (TV_MACH_DEP);
3321 205478 : return 0;
3322 : }
3323 :
3324 : private:
3325 : bool need_endbr;
3326 : unsigned int patchable_area_size;
3327 : }; // class pass_insert_endbr_and_patchable_area
3328 :
3329 : } // anon namespace
3330 :
3331 : rtl_opt_pass *
3332 292371 : make_pass_insert_endbr_and_patchable_area (gcc::context *ctxt)
3333 : {
3334 292371 : return new pass_insert_endbr_and_patchable_area (ctxt);
3335 : }
3336 :
3337 : bool
3338 6111191 : ix86_rpad_gate ()
3339 : {
3340 6111191 : return (TARGET_AVX
3341 389627 : && TARGET_SSE_PARTIAL_REG_DEPENDENCY
3342 294696 : && TARGET_SSE_MATH
3343 294402 : && optimize
3344 6400396 : && optimize_function_for_speed_p (cfun));
3345 : }
3346 :
3347 : enum x86_cse_kind
3348 : {
3349 : X86_CSE_CONST0_VECTOR,
3350 : X86_CSE_CONSTM1_VECTOR,
3351 : X86_CSE_CONST_VECTOR,
3352 : X86_CSE_VEC_DUP,
3353 : X86_CSE_TLS_GD,
3354 : X86_CSE_TLS_LD_BASE,
3355 : X86_CSE_TLSDESC
3356 : };
3357 :
3358 157901 : struct redundant_pattern
3359 : {
3360 : /* Bitmap of basic blocks with broadcast instructions. */
3361 : auto_bitmap bbs;
3362 : /* Bitmap of broadcast instructions. */
3363 : auto_bitmap insns;
3364 : /* The broadcast inner scalar. */
3365 : rtx val;
3366 : /* The actual redundant source value for UNSPEC_TLSDESC. */
3367 : rtx tlsdesc_val;
3368 : /* The inner scalar mode. */
3369 : machine_mode mode;
3370 : /* The destination mode which can be changed to the integer mode of
3371 : the same time. */
3372 : machine_mode dest_mode;
3373 : /* The instruction which sets the inner scalar. Nullptr if the inner
3374 : scalar is applied to the whole function, instead of within the same
3375 : block. */
3376 : rtx_insn *def_insn;
3377 : /* The widest broadcast source. */
3378 : rtx broadcast_source;
3379 : /* The widest broadcast register. */
3380 : rtx broadcast_reg;
3381 : /* The basic block of the broadcast instruction. */
3382 : basic_block bb;
3383 : /* The number of broadcast instructions with the same inner scalar. */
3384 : unsigned HOST_WIDE_INT count;
3385 : /* The threshold of broadcast instructions with the same inner
3386 : scalar. */
3387 : unsigned int threshold;
3388 : /* The widest broadcast size in bytes. */
3389 : unsigned int size;
3390 : /* Load kind. */
3391 : x86_cse_kind kind;
3392 : };
3393 :
3394 : /* Generate a vector set, DEST = SRC, at entry of the nearest dominator
3395 : for basic block map BBS, which is in the fake loop that contains the
3396 : whole function, so that there is only a single vector set in the
3397 : whole function. If not nullptr, LOAD is a pointer to the load. */
3398 :
3399 : static void
3400 44805 : ix86_place_single_vector_set (rtx dest, rtx src, bitmap bbs,
3401 : redundant_pattern *load = nullptr)
3402 : {
3403 44805 : basic_block bb = nearest_common_dominator_for_set (CDI_DOMINATORS, bbs);
3404 : /* For X86_CSE_VEC_DUP and X86_CSE_CONST_VECTOR, don't place the vector
3405 : set outside of the loop to avoid extra spills. */
3406 44805 : if (!load
3407 43785 : || (load->kind != X86_CSE_VEC_DUP
3408 43785 : && load->kind != X86_CSE_CONST_VECTOR))
3409 : {
3410 24865 : while (bb->loop_father->latch
3411 24865 : != EXIT_BLOCK_PTR_FOR_FN (cfun))
3412 1437 : bb = get_immediate_dominator (CDI_DOMINATORS,
3413 : bb->loop_father->header);
3414 : }
3415 :
3416 44805 : if (CONST_INT_P (src))
3417 11027 : dest = gen_rtx_SUBREG (load->dest_mode, dest, 0);
3418 33778 : else if (CONST_VECTOR_P (src))
3419 : {
3420 : /* The only possible CONST_VECTORs of SRC are CONST0_RTX and
3421 : CONSTM1_RTX. Otherwise,
3422 :
3423 : rtx set = gen_rtx_SET (dest, src);
3424 :
3425 : won't be a valid instruction. CONST0_RTX always works. It
3426 : can comes from:
3427 :
3428 : 1. remove_partial_avx_dependency with LOAD == NULL.
3429 : 2. X86_CSE_VEC_DUP with
3430 :
3431 : (insn 48 58 16 3 (set (reg:V4HI 123)
3432 : (const_vector:V4HI [
3433 : (const_int 0 [0]) repeated x4
3434 : ])) 2065 {*movv4hi_internal} (nil))
3435 :
3436 : 3. X86_CSE_CONST0_VECTOR.
3437 : */
3438 23428 : machine_mode mode = GET_MODE (dest);
3439 23428 : if (!(src == CONST0_RTX (mode)
3440 1657 : || (src == CONSTM1_RTX (mode)
3441 1657 : && load->kind == X86_CSE_CONSTM1_VECTOR)))
3442 0 : gcc_unreachable ();
3443 : }
3444 44805 : rtx set = gen_rtx_SET (dest, src);
3445 :
3446 44805 : rtx_insn *insn = BB_HEAD (bb);
3447 180897 : while (insn && !NONDEBUG_INSN_P (insn))
3448 : {
3449 136096 : if (insn == BB_END (bb))
3450 : {
3451 : insn = NULL;
3452 : break;
3453 : }
3454 136092 : insn = NEXT_INSN (insn);
3455 : }
3456 :
3457 44805 : rtx_insn *set_insn;
3458 44805 : if (insn == BB_HEAD (bb))
3459 : {
3460 0 : set_insn = emit_insn_before (set, insn);
3461 0 : if (dump_file)
3462 : {
3463 0 : fprintf (dump_file, "\nPlace:\n\n");
3464 0 : print_rtl_single (dump_file, set_insn);
3465 0 : fprintf (dump_file, "\nbefore:\n\n");
3466 0 : print_rtl_single (dump_file, insn);
3467 0 : fprintf (dump_file, "\n");
3468 : }
3469 : }
3470 : else
3471 : {
3472 44805 : rtx_insn *after = insn ? PREV_INSN (insn) : BB_END (bb);
3473 44805 : set_insn = emit_insn_after (set, after);
3474 44805 : if (dump_file)
3475 : {
3476 2 : fprintf (dump_file, "\nPlace:\n\n");
3477 2 : print_rtl_single (dump_file, set_insn);
3478 2 : fprintf (dump_file, "\nafter:\n\n");
3479 2 : print_rtl_single (dump_file, after);
3480 2 : fprintf (dump_file, "\n");
3481 : }
3482 : }
3483 :
3484 44805 : if (load && load->kind == X86_CSE_VEC_DUP)
3485 : {
3486 : /* Get the source from LOAD as (reg:SI 99) in
3487 :
3488 : (vec_duplicate:V4SI (reg:SI 99))
3489 :
3490 : */
3491 10350 : rtx inner_scalar = load->val;
3492 : /* Set the source in (vec_duplicate:V4SI (reg:SI 99)). */
3493 10350 : rtx reg = XEXP (src, 0);
3494 10350 : machine_mode reg_mode = GET_MODE (reg);
3495 10350 : if (reg_mode != GET_MODE (inner_scalar))
3496 : {
3497 10064 : if (REG_P (inner_scalar) || MEM_P (inner_scalar))
3498 0 : inner_scalar = gen_rtx_SUBREG (reg_mode, inner_scalar, 0);
3499 10064 : else if (!SCALAR_INT_MODE_P (reg_mode))
3500 : {
3501 : /* For non-int load with integer constant, generate
3502 :
3503 : (set (subreg:SI (reg/v:SF 105 [ f ]) 0)
3504 : (const_int 1313486336 [0x4e4a3600]))
3505 :
3506 : */
3507 1 : gcc_assert (CONST_INT_P (inner_scalar));
3508 1 : unsigned int bits = GET_MODE_BITSIZE (reg_mode);
3509 1 : machine_mode mode = int_mode_for_size (bits, 0).require ();
3510 1 : reg = gen_rtx_SUBREG (mode, reg, 0);
3511 : }
3512 : }
3513 10350 : rtx set = gen_rtx_SET (reg, inner_scalar);
3514 10350 : insn = emit_insn_before (set, set_insn);
3515 10350 : if (dump_file)
3516 : {
3517 0 : fprintf (dump_file, "\nAdd:\n\n");
3518 0 : print_rtl_single (dump_file, insn);
3519 0 : fprintf (dump_file, "\nbefore:\n\n");
3520 0 : print_rtl_single (dump_file, set_insn);
3521 0 : fprintf (dump_file, "\n");
3522 : }
3523 : }
3524 44805 : }
3525 :
3526 : /* At entry of the nearest common dominator for basic blocks with
3527 : conversions/rcp/sqrt/rsqrt/round, generate a single
3528 : vxorps %xmmN, %xmmN, %xmmN
3529 : for all
3530 : vcvtss2sd op, %xmmN, %xmmX
3531 : vcvtsd2ss op, %xmmN, %xmmX
3532 : vcvtsi2ss op, %xmmN, %xmmX
3533 : vcvtsi2sd op, %xmmN, %xmmX
3534 :
3535 : NB: We want to generate only a single vxorps to cover the whole
3536 : function. The LCM algorithm isn't appropriate here since it may
3537 : place a vxorps inside the loop. */
3538 :
3539 : static unsigned int
3540 33393 : remove_partial_avx_dependency (void)
3541 : {
3542 33393 : timevar_push (TV_MACH_DEP);
3543 :
3544 33393 : bitmap_obstack_initialize (NULL);
3545 33393 : bitmap convert_bbs = BITMAP_ALLOC (NULL);
3546 :
3547 33393 : basic_block bb;
3548 33393 : rtx_insn *insn, *set_insn;
3549 33393 : rtx set;
3550 33393 : rtx v4sf_const0 = NULL_RTX;
3551 :
3552 33393 : auto_vec<rtx_insn *> control_flow_insns;
3553 :
3554 : /* We create invalid RTL initially so defer rescans. */
3555 33393 : df_set_flags (DF_DEFER_INSN_RESCAN);
3556 :
3557 310948 : FOR_EACH_BB_FN (bb, cfun)
3558 : {
3559 3490361 : FOR_BB_INSNS (bb, insn)
3560 : {
3561 3212806 : if (!NONDEBUG_INSN_P (insn))
3562 1435237 : continue;
3563 :
3564 1777569 : set = single_set (insn);
3565 1777569 : if (!set)
3566 70907 : continue;
3567 :
3568 1706662 : if (get_attr_avx_partial_xmm_update (insn)
3569 : != AVX_PARTIAL_XMM_UPDATE_TRUE)
3570 1703509 : continue;
3571 :
3572 : /* Convert PARTIAL_XMM_UPDATE_TRUE insns, DF -> SF, SF -> DF,
3573 : SI -> SF, SI -> DF, DI -> SF, DI -> DF, sqrt, rsqrt, rcp,
3574 : round, to vec_dup and vec_merge with subreg. */
3575 3153 : rtx src = SET_SRC (set);
3576 3153 : rtx dest = SET_DEST (set);
3577 3153 : machine_mode dest_mode = GET_MODE (dest);
3578 3153 : bool convert_p = false;
3579 3153 : switch (GET_CODE (src))
3580 : {
3581 3088 : case FLOAT:
3582 3088 : case FLOAT_EXTEND:
3583 3088 : case FLOAT_TRUNCATE:
3584 3088 : case UNSIGNED_FLOAT:
3585 3088 : convert_p = true;
3586 3088 : break;
3587 : default:
3588 : break;
3589 : }
3590 :
3591 : /* Only handle conversion here. */
3592 3088 : machine_mode src_mode
3593 3088 : = convert_p ? GET_MODE (XEXP (src, 0)) : VOIDmode;
3594 3088 : switch (src_mode)
3595 : {
3596 153 : case E_SFmode:
3597 153 : case E_DFmode:
3598 153 : if (TARGET_USE_VECTOR_FP_CONVERTS
3599 147 : || !TARGET_SSE_PARTIAL_REG_FP_CONVERTS_DEPENDENCY)
3600 8 : continue;
3601 : break;
3602 2935 : case E_SImode:
3603 2935 : case E_DImode:
3604 2935 : if (TARGET_USE_VECTOR_CONVERTS
3605 2923 : || !TARGET_SSE_PARTIAL_REG_CONVERTS_DEPENDENCY)
3606 14 : continue;
3607 : break;
3608 65 : case E_VOIDmode:
3609 65 : gcc_assert (!convert_p);
3610 : break;
3611 0 : default:
3612 0 : gcc_unreachable ();
3613 : }
3614 :
3615 3131 : if (!v4sf_const0)
3616 1020 : v4sf_const0 = gen_reg_rtx (V4SFmode);
3617 :
3618 3131 : rtx zero;
3619 3131 : machine_mode dest_vecmode;
3620 3131 : switch (dest_mode)
3621 : {
3622 50 : case E_HFmode:
3623 50 : dest_vecmode = V8HFmode;
3624 50 : zero = gen_rtx_SUBREG (V8HFmode, v4sf_const0, 0);
3625 50 : break;
3626 : case E_SFmode:
3627 : dest_vecmode = V4SFmode;
3628 : zero = v4sf_const0;
3629 : break;
3630 1152 : case E_DFmode:
3631 1152 : dest_vecmode = V2DFmode;
3632 1152 : zero = gen_rtx_SUBREG (V2DFmode, v4sf_const0, 0);
3633 1152 : break;
3634 0 : default:
3635 0 : gcc_unreachable ();
3636 : }
3637 :
3638 : /* Change source to vector mode. */
3639 3131 : src = gen_rtx_VEC_DUPLICATE (dest_vecmode, src);
3640 3131 : src = gen_rtx_VEC_MERGE (dest_vecmode, src, zero,
3641 : GEN_INT (HOST_WIDE_INT_1U));
3642 : /* Change destination to vector mode. */
3643 3131 : rtx vec = gen_reg_rtx (dest_vecmode);
3644 : /* Generate an XMM vector SET. */
3645 3131 : set = gen_rtx_SET (vec, src);
3646 3131 : set_insn = emit_insn_before (set, insn);
3647 :
3648 3131 : if (cfun->can_throw_non_call_exceptions)
3649 : {
3650 : /* Handle REG_EH_REGION note. */
3651 0 : rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
3652 0 : if (note)
3653 : {
3654 0 : control_flow_insns.safe_push (set_insn);
3655 0 : add_reg_note (set_insn, REG_EH_REGION, XEXP (note, 0));
3656 : }
3657 : }
3658 :
3659 3131 : src = gen_rtx_SUBREG (dest_mode, vec, 0);
3660 3131 : set = gen_rtx_SET (dest, src);
3661 :
3662 : /* Drop possible dead definitions. */
3663 3131 : PATTERN (insn) = set;
3664 :
3665 3131 : INSN_CODE (insn) = -1;
3666 3131 : recog_memoized (insn);
3667 3131 : df_insn_rescan (insn);
3668 3131 : bitmap_set_bit (convert_bbs, bb->index);
3669 : }
3670 : }
3671 :
3672 33393 : if (v4sf_const0)
3673 : {
3674 : /* (Re-)discover loops so that bb->loop_father can be used in the
3675 : analysis below. */
3676 1020 : calculate_dominance_info (CDI_DOMINATORS);
3677 1020 : loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
3678 :
3679 1020 : ix86_place_single_vector_set (v4sf_const0,
3680 : CONST0_RTX (V4SFmode),
3681 : convert_bbs);
3682 :
3683 1020 : loop_optimizer_finalize ();
3684 :
3685 1020 : if (!control_flow_insns.is_empty ())
3686 : {
3687 0 : free_dominance_info (CDI_DOMINATORS);
3688 :
3689 0 : unsigned int i;
3690 0 : FOR_EACH_VEC_ELT (control_flow_insns, i, insn)
3691 0 : if (control_flow_insn_p (insn))
3692 : {
3693 : /* Split the block after insn. There will be a fallthru
3694 : edge, which is OK so we keep it. We have to create
3695 : the exception edges ourselves. */
3696 0 : bb = BLOCK_FOR_INSN (insn);
3697 0 : split_block (bb, insn);
3698 0 : rtl_make_eh_edge (NULL, bb, BB_END (bb));
3699 : }
3700 : }
3701 : }
3702 :
3703 33393 : df_process_deferred_rescans ();
3704 33393 : df_clear_flags (DF_DEFER_INSN_RESCAN);
3705 33393 : bitmap_obstack_release (NULL);
3706 33393 : BITMAP_FREE (convert_bbs);
3707 :
3708 33393 : timevar_pop (TV_MACH_DEP);
3709 33393 : return 0;
3710 33393 : }
3711 :
3712 : namespace {
3713 :
3714 : const pass_data pass_data_remove_partial_avx_dependency =
3715 : {
3716 : RTL_PASS, /* type */
3717 : "rpad", /* name */
3718 : OPTGROUP_NONE, /* optinfo_flags */
3719 : TV_MACH_DEP, /* tv_id */
3720 : 0, /* properties_required */
3721 : 0, /* properties_provided */
3722 : 0, /* properties_destroyed */
3723 : 0, /* todo_flags_start */
3724 : 0, /* todo_flags_finish */
3725 : };
3726 :
3727 : class pass_remove_partial_avx_dependency : public rtl_opt_pass
3728 : {
3729 : public:
3730 292371 : pass_remove_partial_avx_dependency (gcc::context *ctxt)
3731 584742 : : rtl_opt_pass (pass_data_remove_partial_avx_dependency, ctxt)
3732 : {}
3733 :
3734 : /* opt_pass methods: */
3735 1504958 : bool gate (function *) final override
3736 : {
3737 1504958 : return ix86_rpad_gate ();
3738 : }
3739 :
3740 33393 : unsigned int execute (function *) final override
3741 : {
3742 33393 : return remove_partial_avx_dependency ();
3743 : }
3744 : }; // class pass_rpad
3745 :
3746 : } // anon namespace
3747 :
3748 : rtl_opt_pass *
3749 292371 : make_pass_remove_partial_avx_dependency (gcc::context *ctxt)
3750 : {
3751 292371 : return new pass_remove_partial_avx_dependency (ctxt);
3752 : }
3753 :
3754 : /* Return a machine mode suitable for vector SIZE with SMODE inner
3755 : mode. */
3756 :
3757 : static machine_mode
3758 65475 : ix86_get_vector_cse_mode (unsigned int size, machine_mode smode)
3759 : {
3760 : /* Use the inner scalar mode of vector broadcast source in:
3761 :
3762 : (set (reg:V8DF 394)
3763 : (vec_duplicate:V8DF (reg:V2DF 190 [ alpha ])))
3764 :
3765 : to compute the vector mode for broadcast from vector source.
3766 : */
3767 65475 : if (VECTOR_MODE_P (smode))
3768 31361 : smode = GET_MODE_INNER (smode);
3769 65475 : scalar_mode s_mode = as_a <scalar_mode> (smode);
3770 130950 : poly_uint64 nunits = size / GET_MODE_SIZE (smode);
3771 65475 : machine_mode mode = mode_for_vector (s_mode, nunits).require ();
3772 65475 : return mode;
3773 : }
3774 :
3775 : /* Replace the source operand of instructions in VECTOR_INSNS with
3776 : VECTOR_CONST in VECTOR_MODE. */
3777 :
3778 : static void
3779 64961 : replace_vector_const (machine_mode vector_mode, rtx vector_const,
3780 : auto_bitmap &vector_insns,
3781 : machine_mode scalar_mode)
3782 : {
3783 64961 : bitmap_iterator bi;
3784 64961 : unsigned int id;
3785 :
3786 229601 : EXECUTE_IF_SET_IN_BITMAP (vector_insns, 0, id, bi)
3787 : {
3788 164640 : rtx_insn *insn = DF_INSN_UID_GET (id)->insn;
3789 :
3790 : /* Get the single SET instruction. */
3791 164640 : rtx set = single_set (insn);
3792 164640 : rtx src = SET_SRC (set);
3793 164640 : rtx dest = SET_DEST (set);
3794 164640 : machine_mode mode = GET_MODE (dest);
3795 :
3796 164640 : rtx replace;
3797 : /* Replace the source operand with VECTOR_CONST. */
3798 164640 : if (SUBREG_P (src)
3799 164640 : || mode == vector_mode
3800 62012 : || CONST_INT_P (vector_const))
3801 : replace = vector_const;
3802 : else
3803 : {
3804 62005 : unsigned int size = GET_MODE_SIZE (mode);
3805 62005 : if (size < ix86_regmode_natural_size (mode))
3806 : {
3807 : /* If the mode size is smaller than its natural size,
3808 : first insert an extra move with a QI vector SUBREG
3809 : of the same size to avoid validate_subreg failure. */
3810 514 : machine_mode vmode
3811 514 : = ix86_get_vector_cse_mode (size, scalar_mode);
3812 514 : rtx vreg;
3813 514 : if (mode == vmode)
3814 : vreg = vector_const;
3815 : else
3816 : {
3817 70 : vreg = gen_reg_rtx (vmode);
3818 70 : rtx vsubreg = gen_rtx_SUBREG (vmode, vector_const, 0);
3819 70 : rtx pat = gen_rtx_SET (vreg, vsubreg);
3820 70 : rtx_insn *vinsn = emit_insn_before (pat, insn);
3821 70 : if (dump_file)
3822 : {
3823 0 : fprintf (dump_file, "\nInsert an extra move:\n\n");
3824 0 : print_rtl_single (dump_file, vinsn);
3825 0 : fprintf (dump_file, "\nbefore:\n\n");
3826 0 : print_rtl_single (dump_file, insn);
3827 0 : fprintf (dump_file, "\n");
3828 : }
3829 : }
3830 514 : replace = gen_rtx_SUBREG (mode, vreg, 0);
3831 : }
3832 : else
3833 61491 : replace = gen_rtx_SUBREG (mode, vector_const, 0);
3834 : }
3835 :
3836 164640 : if (dump_file)
3837 : {
3838 3 : fprintf (dump_file, "\nReplace:\n\n");
3839 3 : print_rtl_single (dump_file, insn);
3840 : }
3841 164640 : SET_SRC (set) = replace;
3842 164640 : if (CONST_INT_P (replace))
3843 : {
3844 23357 : dest = gen_lowpart (scalar_mode, dest);
3845 23357 : SET_DEST (set) = dest;
3846 : }
3847 : /* Drop possible dead definitions. */
3848 164640 : PATTERN (insn) = set;
3849 164640 : INSN_CODE (insn) = -1;
3850 164640 : recog_memoized (insn);
3851 164640 : if (dump_file)
3852 : {
3853 3 : fprintf (dump_file, "\nwith:\n\n");
3854 3 : print_rtl_single (dump_file, insn);
3855 3 : fprintf (dump_file, "\n");
3856 : }
3857 164640 : df_insn_rescan (insn);
3858 : }
3859 64961 : }
3860 :
3861 : /* Return the inner scalar if OP is a broadcast, else return nullptr. */
3862 :
3863 : static rtx
3864 2178893 : ix86_broadcast_inner (rtx op, machine_mode mode,
3865 : machine_mode *scalar_mode_p,
3866 : x86_cse_kind *kind_p, rtx_insn **insn_p)
3867 : {
3868 2178893 : switch (standard_sse_constant_p (op, mode))
3869 : {
3870 119621 : case 1:
3871 119621 : *scalar_mode_p = QImode;
3872 119621 : *kind_p = X86_CSE_CONST0_VECTOR;
3873 119621 : *insn_p = nullptr;
3874 119621 : return const0_rtx;
3875 12631 : case 2:
3876 12631 : *scalar_mode_p = QImode;
3877 12631 : *kind_p = X86_CSE_CONSTM1_VECTOR;
3878 12631 : *insn_p = nullptr;
3879 12631 : return constm1_rtx;
3880 2046641 : default:
3881 2046641 : break;
3882 : }
3883 :
3884 2046641 : mode = GET_MODE (op);
3885 2046641 : int nunits = GET_MODE_NUNITS (mode);
3886 2046641 : if (nunits < 2)
3887 : return nullptr;
3888 :
3889 1546897 : bool const_vector_p = CONST_VECTOR_P (op);
3890 1546897 : bool duplicated = GET_CODE (op) == VEC_DUPLICATE;
3891 1546897 : rtx orig_op = op;
3892 1546897 : if (!const_vector_p)
3893 : {
3894 : /* Check CONST_VECTOR in REG_EQUAL note. */
3895 1546877 : rtx equal = find_reg_equal_equiv_note (*insn_p);
3896 1546877 : if (equal)
3897 : {
3898 381142 : equal = XEXP (equal, 0);
3899 381142 : const_vector_p = CONST_VECTOR_P (equal);
3900 : /* Use CONST_VECTOR in REG_EQUAL note. */
3901 381142 : if (const_vector_p)
3902 : {
3903 : /* Handle REG_EQUAL note in:
3904 :
3905 : (insn 7 5 12 2 (set (subreg:V8SI (reg:V4DI 100) 0)
3906 : (vec_duplicate:V8SI (reg:SI 102)))
3907 : (expr_list:REG_DEAD (reg:SI 102)
3908 : (expr_list:REG_EQUAL (const_vector:V4DI [
3909 : (const_int -1 [0xffffffffffffffff]) repeated x4]) (nil))))
3910 :
3911 : NB: Don't treat it as CONST_VECTOR since EQUAL isn't
3912 : supported by ISAs as in gcc.target/i386/pr40957.c. */
3913 266373 : if (GET_MODE (equal) != mode)
3914 : const_vector_p = false;
3915 : else
3916 1546897 : op = equal;
3917 : }
3918 : }
3919 : }
3920 :
3921 1546897 : machine_mode inner_mode = GET_MODE_INNER (mode);
3922 :
3923 1546897 : if (const_vector_p)
3924 : {
3925 532712 : bool int_load_p = GET_MODE_SIZE (mode) <= UNITS_PER_WORD;
3926 266356 : *kind_p = X86_CSE_CONST_VECTOR;
3927 266356 : if (int_load_p)
3928 : {
3929 : /* This CONST_VECTOR load can be converted to constant
3930 : integer load. */
3931 35000 : *scalar_mode_p = mode;
3932 35000 : *insn_p = nullptr;
3933 35000 : return op;
3934 : }
3935 :
3936 : /* This CONST_VECTOR is wider than the integer register. */
3937 231356 : rtx first = XVECEXP (op, 0, 0);
3938 :
3939 231356 : if (duplicated)
3940 : {
3941 : /* Check if CONST_VECTOR in REG_EQUAL note is duplicated in
3942 :
3943 : (insn 10 7 12 2 (set (reg:V8SI 128)
3944 : (vec_duplicate:V8SI (vec_select:V2SI (reg:V4SI 180)
3945 : (parallel [(const_int 0 [0])
3946 : (const_int 1 [0x1])]))))
3947 : (expr_list:REG_EQUAL (const_vector:V8SI [
3948 : (const_int 0 [0])
3949 : (const_int 34 [0x22])
3950 : (const_int 0 [0])
3951 : (const_int 34 [0x22])
3952 : (const_int 0 [0])
3953 : (const_int 34 [0x22])
3954 : (const_int 0 [0])
3955 : (const_int 34 [0x22])])(nil)))
3956 :
3957 : */
3958 :
3959 214944 : bool duplicated_const_vector = true;
3960 214944 : for (int i = 1; i < nunits; ++i)
3961 : {
3962 140774 : rtx tmp = XVECEXP (op, 0, i);
3963 140774 : if (!rtx_equal_p (tmp, first))
3964 : {
3965 : duplicated_const_vector = false;
3966 : break;
3967 : }
3968 : }
3969 :
3970 74186 : if (duplicated_const_vector)
3971 : {
3972 74170 : bool const_double_p = CONST_DOUBLE_P (first);
3973 : /* Force the floating point constant to memory. */
3974 74170 : if (const_double_p)
3975 5628 : first = validize_mem (force_const_mem (inner_mode, first));
3976 :
3977 74170 : if (const_double_p || CONST_INT_P (first))
3978 : {
3979 : /* Handle
3980 :
3981 : (insn 7 6 8 2 (set (reg:V4SF 99)
3982 : (vec_duplicate:V4SF (mem/u/c:SF (symbol_ref/u:DI ("*.LC2") [flags 0x2]) [0 S4 A32])))
3983 : (expr_list:REG_EQUAL (const_vector:V4SF [
3984 : (const_double:SF 3.4e+1 [0x0.88p+6]) repeated x4]) (nil)))
3985 :
3986 : and
3987 :
3988 : (insn 14 15 16 3 (set (reg:V4SI 116)
3989 : (vec_duplicate:V4SI (reg:SI 117)))
3990 : (expr_list:REG_EQUAL (const_vector:V4SI [
3991 : (const_int 34 [0x22]) repeated x4]) (nil)))
3992 :
3993 : */
3994 74170 : *kind_p = X86_CSE_VEC_DUP;
3995 74170 : *insn_p = nullptr;
3996 74170 : *scalar_mode_p = inner_mode;
3997 74170 : return first;
3998 : }
3999 : }
4000 :
4001 : op = orig_op;
4002 : }
4003 : else
4004 : {
4005 : /* Only native CONST_VECTOR is allowed. */
4006 157170 : if (orig_op != op)
4007 : return nullptr;
4008 :
4009 : /* Check if VEC_DUPLICATE can be used. */
4010 48 : for (int i = 1; i < nunits; ++i)
4011 : {
4012 48 : rtx tmp = XVECEXP (op, 0, i);
4013 : /* Vector duplicate value. */
4014 48 : if (!rtx_equal_p (tmp, first))
4015 : return nullptr;
4016 : }
4017 :
4018 : /* Use the inner mode to handle
4019 : (const_vector:V2QI [(const_int 0 [0]) repeated x2])
4020 : */
4021 0 : *scalar_mode_p = inner_mode;
4022 0 : *insn_p = nullptr;
4023 0 : return first;
4024 : }
4025 : }
4026 :
4027 1280557 : if (!duplicated)
4028 : return nullptr;
4029 :
4030 23044 : *kind_p = X86_CSE_VEC_DUP;
4031 :
4032 : /* Only
4033 :
4034 : (vec_duplicate:V4SI (reg:SI 99))
4035 : (vec_duplicate:V2DF (mem/u/c:DF (symbol_ref/u:DI ("*.LC1") [flags 0x2]) [0 S8 A64]))
4036 :
4037 : are supported. Set OP to the broadcast source by default. */
4038 23044 : op = XEXP (op, 0);
4039 23044 : rtx reg = op;
4040 23044 : if (SUBREG_P (op)
4041 402 : && SUBREG_BYTE (op) == 0
4042 23446 : && !paradoxical_subreg_p (op))
4043 402 : reg = SUBREG_REG (op);
4044 23044 : if (!REG_P (reg))
4045 : {
4046 2304 : if (MEM_P (op)
4047 2041 : && SYMBOL_REF_P (XEXP (op, 0))
4048 2553 : && CONSTANT_POOL_ADDRESS_P (XEXP (op, 0)))
4049 : {
4050 : /* Handle constant broadcast from memory. */
4051 11 : *scalar_mode_p = inner_mode;
4052 11 : *insn_p = nullptr;
4053 11 : return op;
4054 : }
4055 : return nullptr;
4056 : }
4057 :
4058 20740 : machine_mode orig_mode = mode;
4059 20740 : mode = GET_MODE (op);
4060 :
4061 : /* Only single def chain is supported. */
4062 20740 : df_ref ref = DF_REG_DEF_CHAIN (REGNO (reg));
4063 20740 : if (!ref
4064 20739 : || DF_REF_IS_ARTIFICIAL (ref)
4065 20739 : || DF_REF_NEXT_REG (ref) != nullptr)
4066 : return nullptr;
4067 :
4068 15219 : rtx_insn *insn = DF_REF_INSN (ref);
4069 :
4070 : /* Skip JUMP which happens with asm goto. */
4071 15219 : if (JUMP_P (insn))
4072 : return nullptr;
4073 :
4074 15217 : rtx set = single_set (insn);
4075 15217 : if (!set)
4076 : return nullptr;
4077 :
4078 15178 : rtx src = SET_SRC (set);
4079 :
4080 15178 : if (CONST_INT_P (src))
4081 : {
4082 : /* Handle sequences like
4083 :
4084 : (set (subreg:SI (reg/v:SF 105 [ f ]) 0)
4085 : (const_int 0 [0]))
4086 : (set (reg:V4SF 110)
4087 : (vec_duplicate:V4SF (reg/v:SF 105 [ f ])))
4088 :
4089 : and
4090 :
4091 : (set (reg:SI 99)
4092 : (const_int 34 [0x22]))
4093 : (set (reg:V4SI 98)
4094 : (vec_duplicate:V4SI (reg:SI 99)))
4095 :
4096 : Set *INSN_P to nullptr and return SET_SRC if SET_SRC is an
4097 : integer constant. */
4098 241 : op = src;
4099 241 : if (SCALAR_INT_MODE_P (mode) && mode != GET_MODE (reg))
4100 0 : op = gen_int_mode (INTVAL (src), mode);
4101 241 : if (op == const0_rtx)
4102 : {
4103 6 : if (standard_sse_constant_p (CONST0_RTX (orig_mode),
4104 : orig_mode) == 1)
4105 : {
4106 6 : *scalar_mode_p = QImode;
4107 6 : *kind_p = X86_CSE_CONST0_VECTOR;
4108 6 : *insn_p = nullptr;
4109 6 : return const0_rtx;
4110 : }
4111 0 : op = CONST0_RTX (mode);
4112 : }
4113 235 : else if (op == constm1_rtx
4114 235 : && standard_sse_constant_p (CONSTM1_RTX (orig_mode),
4115 : orig_mode) == 2)
4116 : {
4117 0 : *scalar_mode_p = QImode;
4118 0 : *kind_p = X86_CSE_CONSTM1_VECTOR;
4119 0 : *insn_p = nullptr;
4120 0 : return constm1_rtx;
4121 : }
4122 :
4123 : /* Check if we can convert:
4124 :
4125 : (insn 14 465 412 3 (set (reg:SI 507 [ j_lsm.26 ])
4126 : (const_int 2 [0x2])) "foo.c":10:12 discrim 2 100 {*movsi_internal} (nil))
4127 : ...
4128 : (insn 518 507 434 16 (set (reg:V2SI 493)
4129 : (vec_duplicate:V2SI (reg:SI 507 [ j_lsm.26 ]))) 2395 {*vec_dupv2si} (nil))
4130 :
4131 : to constant integer load:
4132 :
4133 : (insn 566 55 56 6 (set (subreg:DI (reg:V2SI 517) 0)
4134 : (const_int 8589934594 [0x200000002])) -1 (nil))
4135 : ...
4136 : (insn 518 507 434 16 (set (reg:V2SI 493)
4137 : (reg:V2SI 517)) 2066 {*movv2si_internal} (nil))
4138 :
4139 : */
4140 470 : if (GET_MODE_SIZE (orig_mode) <= UNITS_PER_WORD)
4141 13 : *kind_p = X86_CSE_CONST_VECTOR;
4142 :
4143 235 : *insn_p = nullptr;
4144 : }
4145 : else
4146 : {
4147 : /* Handle sequences like
4148 :
4149 : (set (reg:QI 105 [ c ])
4150 : (reg:QI 5 di [ c ]))
4151 : (set (reg:V64QI 102 [ _1 ])
4152 : (vec_duplicate:V64QI (reg:QI 105 [ c ])))
4153 :
4154 : (set (reg/v:SI 116 [ argc ])
4155 : (mem/c:SI (reg:SI 135) [2 argc+0 S4 A32]))
4156 : (set (reg:V4SI 119 [ _45 ])
4157 : (vec_duplicate:V4SI (reg/v:SI 116 [ argc ])))
4158 :
4159 : (set (reg:SI 98 [ _1 ])
4160 : (sign_extend:SI (reg:QI 106 [ c ])))
4161 : (set (reg:V16SI 103 [ _2 ])
4162 : (vec_duplicate:V16SI (reg:SI 98 [ _1 ])))
4163 :
4164 : (set (reg:SI 102 [ cost ])
4165 : (mem/c:SI (symbol_ref:DI ("cost") [flags 0x40])))
4166 : (set (reg:V4HI 103 [ _16 ])
4167 : (vec_duplicate:V4HI (subreg:HI (reg:SI 102 [ cost ]) 0)))
4168 :
4169 : (set (subreg:SI (reg/v:HI 107 [ cr_val ]) 0)
4170 : (ashift:SI (reg:SI 158)
4171 : (subreg:QI (reg:SI 156 [ _2 ]) 0)))
4172 : (set (reg:V16HI 183 [ _61 ])
4173 : (vec_duplicate:V16HI (reg/v:HI 107 [ cr_val ])))
4174 :
4175 : Set *INSN_P to INSN and return the broadcast source otherwise. */
4176 14937 : *insn_p = insn;
4177 : }
4178 :
4179 15172 : *scalar_mode_p = mode;
4180 15172 : return op;
4181 : }
4182 :
4183 : /* Replace CALL instruction in TLS_CALL_INSNS with SET from SRC and
4184 : put the updated instruction in UPDATED_TLS_INSNS. */
4185 :
4186 : static void
4187 319 : replace_tls_call (rtx src, auto_bitmap &tls_call_insns,
4188 : auto_bitmap &updated_tls_insns)
4189 : {
4190 319 : bitmap_iterator bi;
4191 319 : unsigned int id;
4192 :
4193 1751 : EXECUTE_IF_SET_IN_BITMAP (tls_call_insns, 0, id, bi)
4194 : {
4195 1432 : rtx_insn *insn = DF_INSN_UID_GET (id)->insn;
4196 :
4197 : /* If this isn't a CALL, only GNU2 TLS implicit CALL patterns are
4198 : allowed. */
4199 1432 : if (!CALL_P (insn))
4200 : {
4201 47 : attr_tls64 tls64 = get_attr_tls64 (insn);
4202 47 : if (tls64 != TLS64_CALL && tls64 != TLS64_COMBINE)
4203 0 : gcc_unreachable ();
4204 : }
4205 :
4206 1432 : rtx pat = PATTERN (insn);
4207 1432 : gcc_assert (GET_CODE (pat) == PARALLEL);
4208 1432 : rtx set = XVECEXP (pat, 0, 0);
4209 1432 : gcc_assert (GET_CODE (set) == SET);
4210 1432 : rtx dest = SET_DEST (set);
4211 :
4212 1432 : set = gen_rtx_SET (dest, src);
4213 1432 : rtx_insn *set_insn = emit_insn_after (set, insn);
4214 1432 : if (recog_memoized (set_insn) < 0)
4215 0 : gcc_unreachable ();
4216 :
4217 : /* Put SET_INSN in UPDATED_TLS_INSNS. */
4218 1432 : bitmap_set_bit (updated_tls_insns, INSN_UID (set_insn));
4219 :
4220 1432 : if (dump_file)
4221 : {
4222 0 : fprintf (dump_file, "\nReplace:\n\n");
4223 0 : print_rtl_single (dump_file, insn);
4224 0 : fprintf (dump_file, "\nwith:\n\n");
4225 0 : print_rtl_single (dump_file, set_insn);
4226 0 : fprintf (dump_file, "\n");
4227 : }
4228 :
4229 : /* Delete the CALL insn. */
4230 1432 : delete_insn (insn);
4231 :
4232 1432 : df_insn_rescan (set_insn);
4233 : }
4234 319 : }
4235 :
4236 : /* Return the basic block which dominates all basic blocks which set
4237 : hard register REGNO used in basic block BB. */
4238 :
4239 : static basic_block
4240 2 : ix86_get_dominator_for_reg (unsigned int regno, basic_block bb)
4241 : {
4242 2 : basic_block set_bb;
4243 2 : auto_bitmap set_bbs;
4244 :
4245 : /* Get all BBs which set REGNO and dominate the current BB from all
4246 : DEFs of REGNO. */
4247 2 : for (df_ref def = DF_REG_DEF_CHAIN (regno);
4248 18 : def;
4249 16 : def = DF_REF_NEXT_REG (def))
4250 16 : if (!DF_REF_IS_ARTIFICIAL (def)
4251 16 : && !DF_REF_FLAGS_IS_SET (def, DF_REF_MAY_CLOBBER)
4252 6 : && !DF_REF_FLAGS_IS_SET (def, DF_REF_MUST_CLOBBER))
4253 : {
4254 4 : set_bb = DF_REF_BB (def);
4255 4 : if (dominated_by_p (CDI_DOMINATORS, bb, set_bb))
4256 2 : bitmap_set_bit (set_bbs, set_bb->index);
4257 : }
4258 :
4259 2 : bb = nearest_common_dominator_for_set (CDI_DOMINATORS, set_bbs);
4260 2 : return bb;
4261 2 : }
4262 :
4263 : /* Mark FLAGS register as live in DATA, a bitmap of live caller-saved
4264 : registers, if DEST is FLAGS register. */
4265 :
4266 : static void
4267 387 : ix86_check_flags_reg (rtx dest, const_rtx x, void *data)
4268 : {
4269 387 : if (GET_CODE (x) == CLOBBER)
4270 : return;
4271 :
4272 380 : auto_bitmap *live_caller_saved_regs = (auto_bitmap *) data;
4273 380 : if (REG_P (dest) && REGNO (dest) == FLAGS_REG)
4274 0 : bitmap_set_bit (*live_caller_saved_regs, FLAGS_REG);
4275 : }
4276 :
4277 : /* Emit a TLS_SET instruction of KIND in basic block BB. Store the
4278 : insertion point in *BEFORE_P for emit_insn_before or in *AFTER_P
4279 : for emit_insn_after. UPDATED_GNU_TLS_INSNS contains instructions
4280 : which replace the GNU TLS instructions. UPDATED_GNU2_TLS_INSNS
4281 : contains instructions which replace the GNU2 TLS instructions. */
4282 :
4283 : static rtx_insn *
4284 319 : ix86_emit_tls_call (rtx tls_set, x86_cse_kind kind, basic_block bb,
4285 : rtx_insn **before_p, rtx_insn **after_p,
4286 : auto_bitmap &updated_gnu_tls_insns,
4287 : auto_bitmap &updated_gnu2_tls_insns)
4288 : {
4289 321 : rtx_insn *tls_insn;
4290 :
4291 321 : do
4292 : {
4293 321 : rtx_insn *insn = BB_HEAD (bb);
4294 1253 : while (insn && !NONDEBUG_INSN_P (insn))
4295 : {
4296 936 : if (insn == BB_END (bb))
4297 : {
4298 : /* This must be the beginning basic block:
4299 :
4300 : (note 4 0 2 2 [bb 2] NOTE_INSN_BASIC_BLOCK)
4301 : (note 2 4 26 2 NOTE_INSN_FUNCTION_BEG)
4302 :
4303 : or a basic block with only a label:
4304 :
4305 : (code_label 78 11 77 3 14 (nil) [1 uses])
4306 : (note 77 78 54 3 [bb 3] NOTE_INSN_BASIC_BLOCK)
4307 :
4308 : or a basic block with only a debug marker:
4309 :
4310 : (note 3 0 2 2 [bb 2] NOTE_INSN_BASIC_BLOCK)
4311 : (note 2 3 5 2 NOTE_INSN_FUNCTION_BEG)
4312 : (debug_insn 5 2 16 2 (debug_marker) "x.c":6:3 -1 (nil))
4313 :
4314 : or a basic block with only deleted instructions:
4315 :
4316 : (code_label 348 23 349 45 3 (nil) [0 uses])
4317 : (note 349 348 436 45 [bb 45] NOTE_INSN_BASIC_BLOCK)
4318 : (note 436 349 362 45 NOTE_INSN_DELETED)
4319 :
4320 : */
4321 4 : gcc_assert (DEBUG_INSN_P (insn)
4322 : || (NOTE_P (insn)
4323 : && ((NOTE_KIND (insn)
4324 : == NOTE_INSN_FUNCTION_BEG)
4325 : || (NOTE_KIND (insn)
4326 : == NOTE_INSN_DELETED)
4327 : || (NOTE_KIND (insn)
4328 : == NOTE_INSN_BASIC_BLOCK))));
4329 : insn = NULL;
4330 : break;
4331 : }
4332 932 : insn = NEXT_INSN (insn);
4333 : }
4334 :
4335 : /* TLS_GD and TLS_LD_BASE instructions are normal functions which
4336 : clobber caller-saved registers. TLSDESC instructions only
4337 : clobber FLAGS. If any registers clobbered by TLS instructions
4338 : are live in this basic block, we must insert TLS instructions
4339 : after all live registers clobbered are dead. */
4340 :
4341 321 : auto_bitmap live_caller_saved_regs;
4342 642 : bitmap in = df_live ? DF_LIVE_IN (bb) : DF_LR_IN (bb);
4343 :
4344 321 : if (bitmap_bit_p (in, FLAGS_REG))
4345 4 : bitmap_set_bit (live_caller_saved_regs, FLAGS_REG);
4346 :
4347 321 : unsigned int i;
4348 :
4349 : /* Get all live caller-saved registers for TLS_GD and TLS_LD_BASE
4350 : instructions. */
4351 321 : if (kind != X86_CSE_TLSDESC)
4352 : {
4353 299 : const predefined_function_abi &tls_abi
4354 299 : = ix86_tls_get_addr_abi ();
4355 28106 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
4356 27508 : if (tls_abi.clobbers_full_reg_p (i)
4357 24820 : && !fixed_regs[i]
4358 39791 : && bitmap_bit_p (in, i))
4359 350 : bitmap_set_bit (live_caller_saved_regs, i);
4360 : }
4361 :
4362 321 : if (bitmap_empty_p (live_caller_saved_regs))
4363 : {
4364 82 : if (insn == BB_HEAD (bb))
4365 : {
4366 0 : *before_p = insn;
4367 0 : tls_insn = emit_insn_before (tls_set, insn);
4368 : }
4369 : else
4370 : {
4371 : /* Emit the TLS call after NOTE_INSN_FUNCTION_BEG in the
4372 : beginning basic block:
4373 :
4374 : (note 4 0 2 2 [bb 2] NOTE_INSN_BASIC_BLOCK)
4375 : (note 2 4 26 2 NOTE_INSN_FUNCTION_BEG)
4376 :
4377 : or after NOTE_INSN_BASIC_BLOCK in a basic block with
4378 : only a label:
4379 :
4380 : (code_label 78 11 77 3 14 (nil) [1 uses])
4381 : (note 77 78 54 3 [bb 3] NOTE_INSN_BASIC_BLOCK)
4382 :
4383 : or after debug marker in a basic block with only a
4384 : debug marker:
4385 :
4386 : (note 3 0 2 2 [bb 2] NOTE_INSN_BASIC_BLOCK)
4387 : (note 2 3 5 2 NOTE_INSN_FUNCTION_BEG)
4388 : (debug_insn 5 2 16 2 (debug_marker) "x.c":6:3 -1 (nil))
4389 :
4390 : */
4391 82 : insn = insn ? PREV_INSN (insn) : BB_END (bb);
4392 82 : *after_p = insn;
4393 82 : tls_insn = emit_insn_after (tls_set, insn);
4394 : }
4395 82 : return tls_insn;
4396 : }
4397 :
4398 239 : bool repeat = false;
4399 :
4400 : /* Search for REG_DEAD notes in this basic block. */
4401 673 : FOR_BB_INSNS (bb, insn)
4402 : {
4403 673 : if (!NONDEBUG_INSN_P (insn))
4404 289 : continue;
4405 :
4406 : /* NB: Conditional jump is the only instruction which reads
4407 : flags register and changes control flow. We can never
4408 : place the TLS call after unconditional jump. */
4409 384 : if (JUMP_P (insn))
4410 : {
4411 : /* This must be a conditional jump. */
4412 2 : rtx label = JUMP_LABEL (insn);
4413 2 : if (label == nullptr
4414 2 : || ANY_RETURN_P (label)
4415 2 : || !(LABEL_P (label) || SYMBOL_REF_P (label)))
4416 0 : gcc_unreachable ();
4417 :
4418 : /* Place the call before all FLAGS_REG setting BBs since
4419 : we can't place a call before nor after a conditional
4420 : jump. */
4421 2 : bb = ix86_get_dominator_for_reg (FLAGS_REG, bb);
4422 :
4423 : /* Start over again. */
4424 2 : repeat = true;
4425 2 : break;
4426 : }
4427 :
4428 382 : if (bitmap_bit_p (updated_gnu_tls_insns, INSN_UID (insn)))
4429 : {
4430 : /* Insert the __tls_get_addr call before INSN which
4431 : replaces a __tls_get_addr call. */
4432 1 : *before_p = insn;
4433 1 : tls_insn = emit_insn_before (tls_set, insn);
4434 1 : return tls_insn;
4435 : }
4436 :
4437 381 : if (bitmap_bit_p (updated_gnu2_tls_insns, INSN_UID (insn)))
4438 : {
4439 : /* Mark FLAGS register as dead since FLAGS register
4440 : would be clobbered by the GNU2 TLS instruction. */
4441 1 : bitmap_clear_bit (live_caller_saved_regs, FLAGS_REG);
4442 1 : continue;
4443 : }
4444 :
4445 : /* Check if FLAGS register is live. */
4446 380 : note_stores (insn, ix86_check_flags_reg,
4447 : &live_caller_saved_regs);
4448 :
4449 380 : rtx link;
4450 521 : for (link = REG_NOTES (insn); link; link = XEXP (link, 1))
4451 377 : if ((REG_NOTE_KIND (link) == REG_DEAD
4452 9 : || (REG_NOTE_KIND (link) == REG_UNUSED
4453 7 : && REGNO (XEXP (link, 0)) == FLAGS_REG))
4454 384 : && REG_P (XEXP (link, 0)))
4455 : {
4456 : /* Mark the live caller-saved register as dead. */
4457 755 : for (i = REGNO (XEXP (link, 0));
4458 755 : i < END_REGNO (XEXP (link, 0));
4459 : i++)
4460 380 : if (i < FIRST_PSEUDO_REGISTER)
4461 357 : bitmap_clear_bit (live_caller_saved_regs, i);
4462 :
4463 375 : if (bitmap_empty_p (live_caller_saved_regs))
4464 : {
4465 236 : *after_p = insn;
4466 236 : tls_insn = emit_insn_after (tls_set, insn);
4467 236 : return tls_insn;
4468 : }
4469 : }
4470 : }
4471 :
4472 : /* NB: Start over again for conditional jump. */
4473 2 : if (repeat)
4474 2 : continue;
4475 :
4476 0 : gcc_assert (!bitmap_empty_p (live_caller_saved_regs));
4477 :
4478 : /* If any live caller-saved registers aren't dead at the end of
4479 : this basic block, get the basic block which dominates all
4480 : basic blocks which set the remaining live registers. */
4481 0 : auto_bitmap set_bbs;
4482 0 : bitmap_iterator bi;
4483 0 : unsigned int id;
4484 0 : EXECUTE_IF_SET_IN_BITMAP (live_caller_saved_regs, 0, id, bi)
4485 : {
4486 0 : basic_block set_bb = ix86_get_dominator_for_reg (id, bb);
4487 0 : bitmap_set_bit (set_bbs, set_bb->index);
4488 : }
4489 0 : bb = nearest_common_dominator_for_set (CDI_DOMINATORS, set_bbs);
4490 2 : }
4491 : while (true);
4492 : }
4493 :
4494 : /* Generate a TLS call of KIND with VAL and copy the call result to DEST,
4495 : at entry of the nearest dominator for basic block map BBS, which is in
4496 : the fake loop that contains the whole function, so that there is only
4497 : a single TLS CALL of KIND with VAL in the whole function.
4498 : UPDATED_GNU_TLS_INSNS contains instructions which replace the GNU TLS
4499 : instructions. UPDATED_GNU2_TLS_INSNS contains instructions which
4500 : replace the GNU2 TLS instructions. If TLSDESC_SET isn't nullptr,
4501 : insert it before the TLS call. */
4502 :
4503 : static void
4504 319 : ix86_place_single_tls_call (rtx dest, rtx val, x86_cse_kind kind,
4505 : auto_bitmap &bbs,
4506 : auto_bitmap &updated_gnu_tls_insns,
4507 : auto_bitmap &updated_gnu2_tls_insns,
4508 : rtx tlsdesc_set = nullptr)
4509 : {
4510 319 : basic_block bb = nearest_common_dominator_for_set (CDI_DOMINATORS, bbs);
4511 319 : while (bb->loop_father->latch
4512 328 : != EXIT_BLOCK_PTR_FOR_FN (cfun))
4513 9 : bb = get_immediate_dominator (CDI_DOMINATORS,
4514 : bb->loop_father->header);
4515 :
4516 319 : rtx rax = nullptr, rdi;
4517 319 : rtx eqv = nullptr;
4518 319 : rtx caddr;
4519 319 : rtx set;
4520 319 : rtx clob;
4521 319 : rtx symbol;
4522 319 : rtx tls;
4523 :
4524 319 : switch (kind)
4525 : {
4526 264 : case X86_CSE_TLS_GD:
4527 264 : rax = gen_rtx_REG (Pmode, AX_REG);
4528 264 : rdi = gen_rtx_REG (Pmode, DI_REG);
4529 264 : caddr = ix86_tls_get_addr ();
4530 :
4531 264 : symbol = XVECEXP (val, 0, 0);
4532 264 : tls = gen_tls_global_dynamic_64 (Pmode, rax, symbol, caddr, rdi);
4533 264 : CALL_INSN_ABI_ID (tls) = ix86_tls_get_addr_abi ().id ();
4534 :
4535 264 : if (GET_MODE (symbol) != Pmode)
4536 0 : symbol = gen_rtx_ZERO_EXTEND (Pmode, symbol);
4537 : eqv = symbol;
4538 : break;
4539 :
4540 34 : case X86_CSE_TLS_LD_BASE:
4541 34 : rax = gen_rtx_REG (Pmode, AX_REG);
4542 34 : rdi = gen_rtx_REG (Pmode, DI_REG);
4543 34 : caddr = ix86_tls_get_addr ();
4544 :
4545 34 : tls = gen_tls_local_dynamic_base_64 (Pmode, rax, caddr, rdi);
4546 34 : CALL_INSN_ABI_ID (tls) = ix86_tls_get_addr_abi ().id ();
4547 :
4548 : /* Attach a unique REG_EQUAL to DEST, to allow the RTL optimizers
4549 : to share the LD_BASE result with other LD model accesses. */
4550 34 : eqv = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx),
4551 : UNSPEC_TLS_LD_BASE);
4552 :
4553 34 : break;
4554 :
4555 21 : case X86_CSE_TLSDESC:
4556 21 : set = gen_rtx_SET (dest, val);
4557 21 : clob = gen_rtx_CLOBBER (VOIDmode,
4558 : gen_rtx_REG (CCmode, FLAGS_REG));
4559 21 : tls = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, set, clob));
4560 21 : break;
4561 :
4562 0 : default:
4563 0 : gcc_unreachable ();
4564 : }
4565 :
4566 : /* Emit the TLS CALL insn. */
4567 319 : rtx_insn *before = nullptr;
4568 319 : rtx_insn *after = nullptr;
4569 319 : rtx_insn *tls_insn = ix86_emit_tls_call (tls, kind, bb, &before,
4570 : &after,
4571 : updated_gnu_tls_insns,
4572 : updated_gnu2_tls_insns);
4573 :
4574 319 : rtx_insn *tlsdesc_insn = nullptr;
4575 319 : if (tlsdesc_set)
4576 : {
4577 16 : rtx dest = copy_rtx (SET_DEST (tlsdesc_set));
4578 16 : rtx src = copy_rtx (SET_SRC (tlsdesc_set));
4579 16 : tlsdesc_set = gen_rtx_SET (dest, src);
4580 16 : tlsdesc_insn = emit_insn_before (tlsdesc_set, tls_insn);
4581 : }
4582 :
4583 319 : if (kind != X86_CSE_TLSDESC)
4584 : {
4585 298 : RTL_CONST_CALL_P (tls_insn) = 1;
4586 :
4587 : /* Indicate that this function can't jump to non-local gotos. */
4588 298 : make_reg_eh_region_note_nothrow_nononlocal (tls_insn);
4589 : }
4590 :
4591 319 : if (recog_memoized (tls_insn) < 0)
4592 0 : gcc_unreachable ();
4593 :
4594 319 : if (dump_file)
4595 : {
4596 0 : if (after)
4597 : {
4598 0 : fprintf (dump_file, "\nPlace:\n\n");
4599 0 : if (tlsdesc_insn)
4600 0 : print_rtl_single (dump_file, tlsdesc_insn);
4601 0 : print_rtl_single (dump_file, tls_insn);
4602 0 : fprintf (dump_file, "\nafter:\n\n");
4603 0 : print_rtl_single (dump_file, after);
4604 0 : fprintf (dump_file, "\n");
4605 : }
4606 : else
4607 : {
4608 0 : fprintf (dump_file, "\nPlace:\n\n");
4609 0 : if (tlsdesc_insn)
4610 0 : print_rtl_single (dump_file, tlsdesc_insn);
4611 0 : print_rtl_single (dump_file, tls_insn);
4612 0 : fprintf (dump_file, "\nbefore:\n\n");
4613 0 : print_rtl_single (dump_file, before);
4614 0 : fprintf (dump_file, "\n");
4615 : }
4616 : }
4617 :
4618 319 : if (kind != X86_CSE_TLSDESC)
4619 : {
4620 : /* Copy RAX to DEST. */
4621 298 : set = gen_rtx_SET (dest, rax);
4622 298 : rtx_insn *set_insn = emit_insn_after (set, tls_insn);
4623 298 : set_dst_reg_note (set_insn, REG_EQUAL, copy_rtx (eqv), dest);
4624 298 : if (dump_file)
4625 : {
4626 0 : fprintf (dump_file, "\nPlace:\n\n");
4627 0 : print_rtl_single (dump_file, set_insn);
4628 0 : fprintf (dump_file, "\nafter:\n\n");
4629 0 : print_rtl_single (dump_file, tls_insn);
4630 0 : fprintf (dump_file, "\n");
4631 : }
4632 : }
4633 319 : }
4634 :
4635 : namespace {
4636 :
4637 : const pass_data pass_data_x86_cse =
4638 : {
4639 : RTL_PASS, /* type */
4640 : "x86_cse", /* name */
4641 : OPTGROUP_NONE, /* optinfo_flags */
4642 : TV_MACH_DEP, /* tv_id */
4643 : 0, /* properties_required */
4644 : 0, /* properties_provided */
4645 : 0, /* properties_destroyed */
4646 : 0, /* todo_flags_start */
4647 : 0, /* todo_flags_finish */
4648 : };
4649 :
4650 : class pass_x86_cse : public rtl_opt_pass
4651 : {
4652 : public:
4653 292371 : pass_x86_cse (gcc::context *ctxt)
4654 584742 : : rtl_opt_pass (pass_data_x86_cse, ctxt)
4655 : {}
4656 :
4657 : /* opt_pass methods: */
4658 1504958 : bool gate (function *fun) final override
4659 : {
4660 1504958 : return optimize && optimize_function_for_speed_p (fun);
4661 : }
4662 :
4663 991853 : unsigned int execute (function *) final override
4664 : {
4665 991853 : return x86_cse ();
4666 : }
4667 :
4668 : private:
4669 : /* The redundant source value. */
4670 : rtx val;
4671 : /* The actual redundant source value for UNSPEC_TLSDESC. */
4672 : rtx tlsdesc_val;
4673 : /* The instruction which defines the redundant value. */
4674 : rtx_insn *def_insn;
4675 : /* Mode of the destination of the candidate redundant instruction. */
4676 : machine_mode mode;
4677 : /* Mode of the source of the candidate redundant instruction. */
4678 : machine_mode scalar_mode;
4679 : /* The classification of the candidate redundant instruction. */
4680 : x86_cse_kind kind;
4681 :
4682 : unsigned int x86_cse (void);
4683 : bool candidate_gnu_tls_p (rtx_insn *, attr_tls64);
4684 : bool candidate_gnu2_tls_p (rtx, attr_tls64);
4685 : bool candidate_vector_p (rtx, rtx_insn *);
4686 : rtx_insn *tls_set_insn_from_symbol (const_rtx, const_rtx);
4687 : }; // class pass_x86_cse
4688 :
4689 : /* Return the instruction which sets REG from TLS_SYMBOL. */
4690 :
4691 : rtx_insn *
4692 42 : pass_x86_cse::tls_set_insn_from_symbol (const_rtx reg,
4693 : const_rtx tls_symbol)
4694 : {
4695 42 : rtx_insn *set_insn = nullptr;
4696 42 : for (df_ref ref = DF_REG_DEF_CHAIN (REGNO (reg));
4697 111 : ref;
4698 69 : ref = DF_REF_NEXT_REG (ref))
4699 : {
4700 69 : if (DF_REF_IS_ARTIFICIAL (ref))
4701 : return nullptr;
4702 :
4703 69 : set_insn = DF_REF_INSN (ref);
4704 69 : if (get_attr_tls64 (set_insn) != TLS64_LEA)
4705 : return nullptr;
4706 :
4707 69 : rtx tls_set = PATTERN (set_insn);
4708 69 : rtx tls_src = XVECEXP (SET_SRC (tls_set), 0, 0);
4709 69 : if (!rtx_equal_p (tls_symbol, tls_src))
4710 : return nullptr;
4711 : }
4712 :
4713 : return set_insn;
4714 : }
4715 :
4716 : /* Return true and output def_insn, val, mode, scalar_mode and kind if
4717 : INSN is UNSPEC_TLS_GD or UNSPEC_TLS_LD_BASE. */
4718 :
4719 : bool
4720 2196 : pass_x86_cse::candidate_gnu_tls_p (rtx_insn *insn, attr_tls64 tls64)
4721 : {
4722 2196 : if (!TARGET_64BIT || !cfun->machine->tls_descriptor_call_multiple_p)
4723 : return false;
4724 :
4725 : /* Record the redundant TLS CALLs for 64-bit:
4726 :
4727 : (parallel [
4728 : (set (reg:DI 0 ax)
4729 : (call:DI (mem:QI (symbol_ref:DI ("__tls_get_addr")))
4730 : (const_int 0 [0])))
4731 : (unspec:DI [(symbol_ref:DI ("foo") [flags 0x50])
4732 : (reg/f:DI 7 sp)] UNSPEC_TLS_GD)
4733 : (clobber (reg:DI 5 di))])
4734 :
4735 :
4736 : and
4737 :
4738 : (parallel [
4739 : (set (reg:DI 0 ax)
4740 : (call:DI (mem:QI (symbol_ref:DI ("__tls_get_addr")))
4741 : (const_int 0 [0])))
4742 : (unspec:DI [(reg/f:DI 7 sp)] UNSPEC_TLS_LD_BASE)])
4743 :
4744 : */
4745 :
4746 2028 : rtx pat = PATTERN (insn);
4747 2028 : rtx set = XVECEXP (pat, 0, 0);
4748 2028 : gcc_assert (GET_CODE (set) == SET);
4749 2028 : rtx dest = SET_DEST (set);
4750 2028 : scalar_mode = mode = GET_MODE (dest);
4751 2028 : val = XVECEXP (pat, 0, 1);
4752 2028 : gcc_assert (GET_CODE (val) == UNSPEC);
4753 :
4754 2028 : if (tls64 == TLS64_GD)
4755 1919 : kind = X86_CSE_TLS_GD;
4756 : else
4757 109 : kind = X86_CSE_TLS_LD_BASE;
4758 :
4759 2028 : def_insn = nullptr;
4760 2028 : return true;
4761 : }
4762 :
4763 : /* Return true and output def_insn, val, mode, scalar_mode and kind if
4764 : SET is UNSPEC_TLSDESC. */
4765 :
4766 : bool
4767 56 : pass_x86_cse::candidate_gnu2_tls_p (rtx set, attr_tls64 tls64)
4768 : {
4769 56 : if (!TARGET_64BIT || !cfun->machine->tls_descriptor_call_multiple_p)
4770 : return false;
4771 :
4772 54 : rtx tls_symbol;
4773 54 : rtx_insn *set_insn;
4774 54 : rtx src = SET_SRC (set);
4775 54 : val = src;
4776 54 : tlsdesc_val = src;
4777 54 : kind = X86_CSE_TLSDESC;
4778 :
4779 54 : if (tls64 == TLS64_COMBINE)
4780 : {
4781 : /* Record 64-bit TLS64_COMBINE:
4782 :
4783 : (set (reg/f:DI 104)
4784 : (plus:DI (unspec:DI [
4785 : (symbol_ref:DI ("_TLS_MODULE_BASE_") [flags 0x10])
4786 : (reg:DI 114)
4787 : (reg/f:DI 7 sp)] UNSPEC_TLSDESC)
4788 : (const:DI (unspec:DI [
4789 : (symbol_ref:DI ("e") [flags 0x1a])
4790 : ] UNSPEC_DTPOFF))))
4791 :
4792 : (set (reg/f:DI 104)
4793 : (plus:DI (unspec:DI [
4794 : (symbol_ref:DI ("_TLS_MODULE_BASE_") [flags 0x10])
4795 : (unspec:DI [
4796 : (symbol_ref:DI ("_TLS_MODULE_BASE_") [flags 0x10])
4797 : ] UNSPEC_TLSDESC)
4798 : (reg/f:DI 7 sp)] UNSPEC_TLSDESC)
4799 : (const:DI (unspec:DI [
4800 : (symbol_ref:DI ("e") [flags 0x1a])
4801 : ] UNSPEC_DTPOFF))))
4802 : */
4803 :
4804 12 : scalar_mode = mode = GET_MODE (src);
4805 :
4806 : /* Since the first operand of PLUS in the source TLS_COMBINE
4807 : pattern is unused, use the second operand of PLUS:
4808 :
4809 : (const:DI (unspec:DI [
4810 : (symbol_ref:DI ("e") [flags 0x1a])
4811 : ] UNSPEC_DTPOFF))
4812 :
4813 : as VAL to check if 2 TLS_COMBINE patterns have the same
4814 : source. */
4815 12 : val = XEXP (src, 1);
4816 12 : gcc_assert (GET_CODE (val) == CONST
4817 : && GET_CODE (XEXP (val, 0)) == UNSPEC
4818 : && XINT (XEXP (val, 0), 1) == UNSPEC_DTPOFF
4819 : && SYMBOL_REF_P (XVECEXP (XEXP (val, 0), 0, 0)));
4820 12 : def_insn = nullptr;
4821 12 : return true;
4822 : }
4823 :
4824 : /* Record 64-bit TLS_CALL:
4825 :
4826 : (set (reg:DI 101)
4827 : (unspec:DI [(symbol_ref:DI ("foo") [flags 0x50])
4828 : (reg:DI 112)
4829 : (reg/f:DI 7 sp)] UNSPEC_TLSDESC))
4830 :
4831 : */
4832 :
4833 42 : gcc_assert (GET_CODE (src) == UNSPEC);
4834 42 : tls_symbol = XVECEXP (src, 0, 0);
4835 42 : src = XVECEXP (src, 0, 1);
4836 42 : scalar_mode = mode = GET_MODE (src);
4837 42 : gcc_assert (REG_P (src));
4838 :
4839 : /* All definitions of reg:DI 129 in
4840 :
4841 : (set (reg:DI 110)
4842 : (unspec:DI [(symbol_ref:DI ("foo"))
4843 : (reg:DI 129)
4844 : (reg/f:DI 7 sp)] UNSPEC_TLSDESC))
4845 :
4846 : should have the same source as in
4847 :
4848 : (set (reg:DI 129)
4849 : (unspec:DI [(symbol_ref:DI ("foo"))] UNSPEC_TLSDESC))
4850 :
4851 : */
4852 :
4853 42 : set_insn = tls_set_insn_from_symbol (src, tls_symbol);
4854 42 : if (!set_insn)
4855 : return false;
4856 :
4857 : /* Use TLS_SYMBOL as VAL to check if 2 patterns have the same source. */
4858 42 : val = tls_symbol;
4859 42 : def_insn = set_insn;
4860 42 : return true;
4861 : }
4862 :
4863 : /* Return true and output def_insn, val, mode, scalar_mode and kind if
4864 : INSN is a vector broadcast instruction. */
4865 :
4866 : bool
4867 50855157 : pass_x86_cse::candidate_vector_p (rtx set, rtx_insn *insn)
4868 : {
4869 50855157 : rtx src = SET_SRC (set);
4870 50855157 : rtx dest = SET_DEST (set);
4871 50855157 : mode = GET_MODE (dest);
4872 : /* Skip non-vector instruction. */
4873 50855157 : if (!VECTOR_MODE_P (mode))
4874 : return false;
4875 :
4876 : /* Skip non-vector load instruction. */
4877 3654951 : if (!REG_P (dest) && !SUBREG_P (dest))
4878 : return false;
4879 :
4880 2178893 : def_insn = insn;
4881 2178893 : val = ix86_broadcast_inner (src, mode, &scalar_mode, &kind,
4882 : &def_insn);
4883 2178893 : return val ? true : false;
4884 : }
4885 :
4886 : /* At entry of the nearest common dominator for basic blocks with
4887 :
4888 : 1. Vector CONST0_RTX patterns.
4889 : 2. Vector CONSTM1_RTX patterns.
4890 : 3. Vector broadcast patterns.
4891 : 4. UNSPEC_TLS_GD patterns.
4892 : 5. UNSPEC_TLS_LD_BASE patterns.
4893 : 6. UNSPEC_TLSDESC patterns.
4894 :
4895 : generate a single pattern whose destination is used to replace the
4896 : source in all identical patterns.
4897 :
4898 : NB: We want to generate a pattern, which is executed only once, to
4899 : cover the whole function. The LCM algorithm isn't appropriate here
4900 : since it may place a pattern inside the loop. */
4901 :
4902 : unsigned int
4903 991853 : pass_x86_cse::x86_cse (void)
4904 : {
4905 991853 : timevar_push (TV_MACH_DEP);
4906 :
4907 991853 : auto_vec<redundant_pattern *> loads;
4908 991853 : redundant_pattern *load;
4909 991853 : basic_block bb;
4910 991853 : rtx_insn *insn;
4911 991853 : unsigned int i;
4912 991853 : auto_bitmap updated_gnu_tls_insns;
4913 991853 : auto_bitmap updated_gnu2_tls_insns;
4914 991853 : auto_bitmap call_bbs;
4915 :
4916 991853 : df_set_flags (DF_DEFER_INSN_RESCAN);
4917 :
4918 991853 : bool recursive_call_p = cfun->machine->recursive_function;
4919 :
4920 11134099 : FOR_EACH_BB_FN (bb, cfun)
4921 : {
4922 136383315 : FOR_BB_INSNS (bb, insn)
4923 : {
4924 126241069 : if (!NONDEBUG_INSN_P (insn))
4925 71641863 : continue;
4926 :
4927 54599206 : bool matched = false;
4928 : /* Remove redundant patterns if there are more than 2 of
4929 : them. */
4930 54599206 : unsigned int threshold = 2;
4931 :
4932 54599206 : bool call_p = CALL_P (insn);
4933 54599206 : rtx set = single_set (insn);
4934 54599206 : if (!set && !call_p)
4935 1113374 : continue;
4936 :
4937 53485832 : tlsdesc_val = nullptr;
4938 :
4939 53485832 : attr_tls64 tls64 = get_attr_tls64 (insn);
4940 :
4941 : /* NB: TLS calls preserve all registers. */
4942 53485832 : if (call_p && tls64 == TLS64_NONE)
4943 4535573 : bitmap_set_bit (call_bbs, BLOCK_FOR_INSN (insn)->index);
4944 :
4945 53485832 : switch (tls64)
4946 : {
4947 2196 : case TLS64_GD:
4948 2196 : case TLS64_LD_BASE:
4949 : /* Verify UNSPEC_TLS_GD and UNSPEC_TLS_LD_BASE. */
4950 2196 : if (candidate_gnu_tls_p (insn, tls64))
4951 : break;
4952 168 : continue;
4953 :
4954 56 : case TLS64_CALL:
4955 56 : case TLS64_COMBINE:
4956 : /* Verify UNSPEC_TLSDESC. */
4957 56 : if (candidate_gnu2_tls_p (set, tls64))
4958 : break;
4959 2 : continue;
4960 :
4961 36 : case TLS64_LEA:
4962 : /* Skip TLS64_LEA. */
4963 36 : continue;
4964 :
4965 53483544 : case TLS64_NONE:
4966 53483544 : if (!set)
4967 2628387 : continue;
4968 :
4969 : /* Check for vector broadcast. */
4970 50855157 : if (candidate_vector_p (set, insn))
4971 : break;
4972 50598546 : continue;
4973 : }
4974 :
4975 : /* Check if there is a matching redundant load. */
4976 610531 : FOR_EACH_VEC_ELT (loads, i, load)
4977 452630 : if (load->val
4978 452630 : && load->kind == kind
4979 299914 : && load->mode == scalar_mode
4980 264332 : && (load->bb == bb
4981 202319 : || (kind != X86_CSE_VEC_DUP
4982 202319 : && kind != X86_CSE_CONST_VECTOR)
4983 : /* Non all 0s/1s vector load must be in the same
4984 : basic block if it is in a recursive call. */
4985 139265 : || !recursive_call_p)
4986 714835 : && rtx_equal_p (load->val, val))
4987 : {
4988 : /* Record instruction. */
4989 100792 : bitmap_set_bit (load->insns, INSN_UID (insn));
4990 :
4991 : /* Record the maximum vector size. */
4992 100792 : if (kind <= X86_CSE_VEC_DUP
4993 200471 : && load->size < GET_MODE_SIZE (mode))
4994 1018 : load->size = GET_MODE_SIZE (mode);
4995 :
4996 : /* Record the basic block. */
4997 100792 : bitmap_set_bit (load->bbs, bb->index);
4998 :
4999 : /* Increment the count. */
5000 100792 : load->count++;
5001 :
5002 100792 : matched = true;
5003 100792 : break;
5004 : }
5005 :
5006 258693 : if (matched)
5007 100792 : continue;
5008 :
5009 : /* We see this instruction the first time. Record the
5010 : redundant source value, its mode, the destination size,
5011 : instruction which defines the redundant source value,
5012 : instruction basic block and the instruction kind. */
5013 157901 : load = new redundant_pattern;
5014 :
5015 : /* Convert CONST_VECTOR load no larger than integer register
5016 : to constant integer load even if there is no redundant
5017 : CONST_VECTOR load. */
5018 157901 : if (CONST_VECTOR_P (val))
5019 31360 : threshold = 1;
5020 :
5021 157901 : load->val = copy_rtx (val);
5022 157901 : if (tlsdesc_val)
5023 28 : load->tlsdesc_val = copy_rtx (tlsdesc_val);
5024 : else
5025 157873 : load->tlsdesc_val = nullptr;
5026 157901 : load->mode = scalar_mode;
5027 157901 : load->dest_mode = mode;
5028 157901 : load->size = GET_MODE_SIZE (mode);
5029 157901 : load->def_insn = def_insn;
5030 157901 : load->count = 1;
5031 157901 : load->threshold = threshold;
5032 157901 : load->bb = BLOCK_FOR_INSN (insn);
5033 157901 : load->kind = kind;
5034 :
5035 157901 : bitmap_set_bit (load->insns, INSN_UID (insn));
5036 157901 : bitmap_set_bit (load->bbs, bb->index);
5037 :
5038 157901 : loads.safe_push (load);
5039 : }
5040 : }
5041 :
5042 : bool replaced = false;
5043 1149754 : FOR_EACH_VEC_ELT (loads, i, load)
5044 157901 : if (load->count >= load->threshold)
5045 : {
5046 65280 : machine_mode mode;
5047 65280 : rtx reg, broadcast_reg;
5048 65280 : rtx broadcast_source = nullptr;
5049 65280 : replaced = true;
5050 65280 : switch (load->kind)
5051 : {
5052 319 : case X86_CSE_TLS_GD:
5053 319 : case X86_CSE_TLS_LD_BASE:
5054 319 : case X86_CSE_TLSDESC:
5055 319 : broadcast_reg = gen_reg_rtx (load->mode);
5056 319 : replace_tls_call (broadcast_reg, load->insns,
5057 319 : (load->kind == X86_CSE_TLSDESC
5058 : ? updated_gnu2_tls_insns
5059 : : updated_gnu_tls_insns));
5060 319 : load->broadcast_reg = broadcast_reg;
5061 319 : break;
5062 :
5063 11191 : case X86_CSE_VEC_DUP:
5064 11191 : if (CONST_INT_P (load->val)
5065 10064 : && (load->val == CONST0_RTX (load->mode)
5066 10088 : || load->size <= UNITS_PER_WORD))
5067 : {
5068 : /* Generate CONST_VECTOR load. */
5069 31362 : case X86_CSE_CONST_VECTOR:
5070 31362 : mode = ix86_get_vector_cse_mode (load->size,
5071 : load->mode);
5072 :
5073 31362 : if (CONST_VECTOR_P (load->val))
5074 : broadcast_source = load->val;
5075 2 : else if (load->val == CONST0_RTX (load->mode))
5076 0 : broadcast_source = CONST0_RTX (mode);
5077 2 : else if (load->val == CONSTM1_RTX (load->mode))
5078 0 : broadcast_source = CONSTM1_RTX (mode);
5079 : else
5080 : {
5081 2 : int nunits = GET_MODE_NUNITS (mode);
5082 2 : rtvec v = rtvec_alloc (nunits);
5083 6 : for (int j = 0; j < nunits ; j++)
5084 4 : RTVEC_ELT (v, j) = load->val;
5085 2 : if (CONST_INT_P (load->val)
5086 2 : && GET_MODE_CLASS (mode) != MODE_VECTOR_INT)
5087 : {
5088 1 : gcc_assert (load->size <= UNITS_PER_WORD);
5089 1 : machine_mode imode = GET_MODE_INNER (mode);
5090 1 : class scalar_mode i_mode
5091 1 : = int_mode_for_mode (imode).require ();
5092 1 : mode = mode_for_vector (i_mode,
5093 1 : nunits).require ();
5094 : }
5095 :
5096 2 : broadcast_source = gen_rtx_CONST_VECTOR (mode, v);
5097 : }
5098 :
5099 : /* NB: Zero CONST_VECTOR load works for MMX and XMM
5100 : registers. */
5101 32775 : if (load->size <= UNITS_PER_WORD)
5102 : {
5103 : /* Convert CONST_VECTOR load no larger than integer
5104 : register:
5105 :
5106 : (set (reg:V2SI 106)
5107 : (const_vector:V2SI [(const_int 1 [1]) repeated x2]))
5108 :
5109 : to constant integer load:
5110 :
5111 : (set (subreg:DI (reg:V2SI 106 [ _20 ]) 0)
5112 : (const_int 4294967297 [0x100000001]))
5113 : */
5114 31362 : machine_mode int_mode
5115 31362 : = int_mode_for_mode (mode).require ();
5116 31362 : load->dest_mode = int_mode;
5117 31362 : broadcast_source = simplify_subreg (int_mode,
5118 : broadcast_source,
5119 : mode, 0);
5120 31362 : gcc_assert (broadcast_source != nullptr);
5121 :
5122 31362 : bool keep_const_int_load = false;
5123 31362 : if (!bitmap_empty_p (call_bbs))
5124 : {
5125 28096 : bitmap_iterator bi;
5126 28096 : unsigned int id;
5127 36992 : EXECUTE_IF_SET_IN_BITMAP (load->bbs, 0, id, bi)
5128 29231 : if (bitmap_bit_p (call_bbs, id))
5129 : {
5130 : /* NB: Constant integer load is faster
5131 : than save and restore an integer
5132 : register when crossing a function call.
5133 : */
5134 : keep_const_int_load = true;
5135 : break;
5136 : }
5137 : }
5138 :
5139 28096 : if (keep_const_int_load)
5140 : {
5141 : /* Keep constant integer load. */
5142 20335 : replace_vector_const (mode, broadcast_source,
5143 20335 : load->insns, int_mode);
5144 20335 : load->broadcast_source = nullptr;
5145 20335 : load->broadcast_reg = nullptr;
5146 : }
5147 : else
5148 : {
5149 11027 : broadcast_reg = gen_reg_rtx (mode);
5150 11027 : reg = gen_reg_rtx (load->mode);
5151 11027 : replace_vector_const (mode, broadcast_reg,
5152 11027 : load->insns, load->mode);
5153 11027 : load->broadcast_source = broadcast_source;
5154 11027 : load->broadcast_reg = broadcast_reg;
5155 : }
5156 : break;
5157 : }
5158 : }
5159 : /* FALLTHRU */
5160 :
5161 33599 : case X86_CSE_CONST0_VECTOR:
5162 33599 : case X86_CSE_CONSTM1_VECTOR:
5163 33599 : mode = ix86_get_vector_cse_mode (load->size, load->mode);
5164 33599 : broadcast_reg = gen_reg_rtx (mode);
5165 33599 : if (load->def_insn)
5166 : {
5167 : /* Replace redundant vector loads with a single vector
5168 : load in the same basic block. */
5169 841 : reg = load->val;
5170 841 : if (load->mode != GET_MODE (reg))
5171 0 : reg = gen_rtx_SUBREG (load->mode, reg, 0);
5172 841 : broadcast_source = gen_rtx_VEC_DUPLICATE (mode, reg);
5173 : }
5174 : else
5175 : /* This is a constant integer/double vector. If the
5176 : inner scalar is 0 or -1, set vector to CONST0_RTX
5177 : or CONSTM1_RTX directly. */
5178 32758 : switch (load->kind)
5179 : {
5180 20751 : case X86_CSE_CONST0_VECTOR:
5181 20751 : broadcast_source = CONST0_RTX (mode);
5182 20751 : break;
5183 1657 : case X86_CSE_CONSTM1_VECTOR:
5184 1657 : broadcast_source = CONSTM1_RTX (mode);
5185 1657 : break;
5186 10350 : case X86_CSE_CONST_VECTOR:
5187 10350 : case X86_CSE_VEC_DUP:
5188 10350 : if (!broadcast_source)
5189 : {
5190 10350 : reg = gen_reg_rtx (load->mode);
5191 10350 : broadcast_source = gen_rtx_VEC_DUPLICATE (mode,
5192 : reg);
5193 : }
5194 : break;
5195 0 : default:
5196 0 : gcc_unreachable ();
5197 : }
5198 33599 : replace_vector_const (mode, broadcast_reg, load->insns,
5199 : load->mode);
5200 33599 : load->broadcast_source = broadcast_source;
5201 33599 : load->broadcast_reg = broadcast_reg;
5202 33599 : break;
5203 : }
5204 : }
5205 :
5206 991853 : if (replaced)
5207 : {
5208 42553 : auto_vec<rtx_insn *> control_flow_insns;
5209 :
5210 : /* (Re-)discover loops so that bb->loop_father can be used in the
5211 : analysis below. */
5212 42553 : calculate_dominance_info (CDI_DOMINATORS);
5213 42553 : loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
5214 :
5215 128771 : FOR_EACH_VEC_ELT (loads, i, load)
5216 86218 : if (load->count >= load->threshold)
5217 : {
5218 65280 : rtx set;
5219 65280 : if (load->def_insn)
5220 857 : switch (load->kind)
5221 : {
5222 16 : case X86_CSE_TLSDESC:
5223 16 : ix86_place_single_tls_call (load->broadcast_reg,
5224 : load->tlsdesc_val,
5225 : load->kind,
5226 16 : load->bbs,
5227 : updated_gnu_tls_insns,
5228 : updated_gnu2_tls_insns,
5229 16 : PATTERN (load->def_insn));
5230 16 : break;
5231 841 : case X86_CSE_VEC_DUP:
5232 : /* Insert a broadcast after the original scalar
5233 : definition. */
5234 841 : set = gen_rtx_SET (load->broadcast_reg,
5235 : load->broadcast_source);
5236 841 : insn = emit_insn_after (set, load->def_insn);
5237 :
5238 841 : if (cfun->can_throw_non_call_exceptions)
5239 : {
5240 : /* Handle REG_EH_REGION note in DEF_INSN. */
5241 4 : rtx note = find_reg_note (load->def_insn,
5242 : REG_EH_REGION, nullptr);
5243 4 : if (note)
5244 : {
5245 1 : control_flow_insns.safe_push (load->def_insn);
5246 1 : add_reg_note (insn, REG_EH_REGION,
5247 : XEXP (note, 0));
5248 : }
5249 : }
5250 :
5251 841 : if (dump_file)
5252 : {
5253 0 : fprintf (dump_file, "\nAdd:\n\n");
5254 0 : print_rtl_single (dump_file, insn);
5255 0 : fprintf (dump_file, "\nafter:\n\n");
5256 0 : print_rtl_single (dump_file, load->def_insn);
5257 0 : fprintf (dump_file, "\n");
5258 : }
5259 : break;
5260 0 : default:
5261 0 : gcc_unreachable ();
5262 : }
5263 : else
5264 64423 : switch (load->kind)
5265 : {
5266 303 : case X86_CSE_TLS_GD:
5267 303 : case X86_CSE_TLS_LD_BASE:
5268 303 : case X86_CSE_TLSDESC:
5269 303 : ix86_place_single_tls_call (load->broadcast_reg,
5270 : (load->kind == X86_CSE_TLSDESC
5271 : ? load->tlsdesc_val
5272 : : load->val),
5273 : load->kind,
5274 303 : load->bbs,
5275 : updated_gnu_tls_insns,
5276 : updated_gnu2_tls_insns);
5277 303 : break;
5278 41712 : case X86_CSE_CONST_VECTOR:
5279 41712 : case X86_CSE_VEC_DUP:
5280 : /* Keep redundant constant integer load. */
5281 41712 : if (!load->broadcast_reg)
5282 : break;
5283 : /* FALLTHRU */
5284 43785 : case X86_CSE_CONST0_VECTOR:
5285 43785 : case X86_CSE_CONSTM1_VECTOR:
5286 43785 : ix86_place_single_vector_set (load->broadcast_reg,
5287 : load->broadcast_source,
5288 : load->bbs,
5289 : load);
5290 43785 : break;
5291 : }
5292 : }
5293 :
5294 42553 : loop_optimizer_finalize ();
5295 :
5296 42553 : if (!control_flow_insns.is_empty ())
5297 : {
5298 1 : free_dominance_info (CDI_DOMINATORS);
5299 :
5300 3 : FOR_EACH_VEC_ELT (control_flow_insns, i, insn)
5301 1 : if (control_flow_insn_p (insn))
5302 : {
5303 : /* Split the block after insn. There will be a fallthru
5304 : edge, which is OK so we keep it. We have to create
5305 : the exception edges ourselves. */
5306 1 : bb = BLOCK_FOR_INSN (insn);
5307 1 : split_block (bb, insn);
5308 1 : rtl_make_eh_edge (NULL, bb, BB_END (bb));
5309 : }
5310 : }
5311 :
5312 42553 : df_process_deferred_rescans ();
5313 42553 : }
5314 :
5315 1149754 : FOR_EACH_VEC_ELT (loads, i, load)
5316 315802 : delete load;
5317 :
5318 991853 : df_clear_flags (DF_DEFER_INSN_RESCAN);
5319 :
5320 991853 : timevar_pop (TV_MACH_DEP);
5321 991853 : return 0;
5322 991853 : }
5323 :
5324 : } // anon namespace
5325 :
5326 : rtl_opt_pass *
5327 292371 : make_pass_x86_cse (gcc::context *ctxt)
5328 : {
5329 292371 : return new pass_x86_cse (ctxt);
5330 : }
5331 :
5332 : /* Convert legacy instructions that clobbers EFLAGS to APX_NF
5333 : instructions when there are no flag set between a flag
5334 : producer and user. */
5335 :
5336 : static unsigned int
5337 372 : ix86_apx_nf_convert (void)
5338 : {
5339 372 : timevar_push (TV_MACH_DEP);
5340 :
5341 372 : basic_block bb;
5342 372 : rtx_insn *insn;
5343 372 : hash_map <rtx_insn *, rtx> converting_map;
5344 372 : auto_vec <rtx_insn *> current_convert_list;
5345 :
5346 372 : bool converting_seq = false;
5347 372 : rtx cc = gen_rtx_REG (CCmode, FLAGS_REG);
5348 :
5349 794 : FOR_EACH_BB_FN (bb, cfun)
5350 : {
5351 : /* Reset conversion for each bb. */
5352 422 : converting_seq = false;
5353 5081 : FOR_BB_INSNS (bb, insn)
5354 : {
5355 4659 : if (!NONDEBUG_INSN_P (insn))
5356 4984 : continue;
5357 :
5358 3719 : if (recog_memoized (insn) < 0)
5359 338 : continue;
5360 :
5361 : /* Convert candidate insns after cstore, which should
5362 : satisfy the two conditions:
5363 : 1. Is not flag user or producer, only clobbers
5364 : FLAGS_REG.
5365 : 2. Have corresponding nf pattern. */
5366 :
5367 3381 : rtx pat = PATTERN (insn);
5368 :
5369 : /* Starting conversion at first cstorecc. */
5370 3381 : rtx set = NULL_RTX;
5371 3381 : if (!converting_seq
5372 2790 : && (set = single_set (insn))
5373 2714 : && ix86_comparison_operator (SET_SRC (set), VOIDmode)
5374 129 : && reg_overlap_mentioned_p (cc, SET_SRC (set))
5375 3507 : && !reg_overlap_mentioned_p (cc, SET_DEST (set)))
5376 : {
5377 126 : converting_seq = true;
5378 126 : current_convert_list.truncate (0);
5379 : }
5380 : /* Terminate at the next explicit flag set. */
5381 3255 : else if (reg_set_p (cc, pat)
5382 3255 : && GET_CODE (set_of (cc, pat)) != CLOBBER)
5383 : converting_seq = false;
5384 :
5385 3160 : if (!converting_seq)
5386 2766 : continue;
5387 :
5388 615 : if (get_attr_has_nf (insn)
5389 615 : && GET_CODE (pat) == PARALLEL)
5390 : {
5391 : /* Record the insn to candidate map. */
5392 73 : current_convert_list.safe_push (insn);
5393 73 : converting_map.put (insn, pat);
5394 : }
5395 : /* If the insn clobbers flags but has no nf_attr,
5396 : revoke all previous candidates. */
5397 542 : else if (!get_attr_has_nf (insn)
5398 541 : && reg_set_p (cc, pat)
5399 545 : && GET_CODE (set_of (cc, pat)) == CLOBBER)
5400 : {
5401 3 : for (auto item : current_convert_list)
5402 0 : converting_map.remove (item);
5403 3 : converting_seq = false;
5404 : }
5405 : }
5406 : }
5407 :
5408 372 : if (!converting_map.is_empty ())
5409 : {
5410 87 : for (auto iter = converting_map.begin ();
5411 174 : iter != converting_map.end (); ++iter)
5412 : {
5413 73 : rtx_insn *replace = (*iter).first;
5414 73 : rtx pat = (*iter).second;
5415 73 : int i, n = 0, len = XVECLEN (pat, 0);
5416 73 : rtx *new_elems = XALLOCAVEC (rtx, len);
5417 73 : rtx new_pat;
5418 219 : for (i = 0; i < len; i++)
5419 : {
5420 146 : rtx temp = XVECEXP (pat, 0, i);
5421 219 : if (! (GET_CODE (temp) == CLOBBER
5422 73 : && reg_overlap_mentioned_p (cc,
5423 73 : XEXP (temp, 0))))
5424 : {
5425 73 : new_elems[n] = temp;
5426 73 : n++;
5427 : }
5428 : }
5429 :
5430 73 : if (n == 1)
5431 73 : new_pat = new_elems[0];
5432 : else
5433 0 : new_pat =
5434 0 : gen_rtx_PARALLEL (VOIDmode,
5435 : gen_rtvec_v (n,
5436 : new_elems));
5437 :
5438 73 : PATTERN (replace) = new_pat;
5439 73 : INSN_CODE (replace) = -1;
5440 73 : recog_memoized (replace);
5441 73 : df_insn_rescan (replace);
5442 : }
5443 : }
5444 :
5445 372 : timevar_pop (TV_MACH_DEP);
5446 372 : return 0;
5447 372 : }
5448 :
5449 :
5450 : namespace {
5451 :
5452 : const pass_data pass_data_apx_nf_convert =
5453 : {
5454 : RTL_PASS, /* type */
5455 : "apx_nfcvt", /* name */
5456 : OPTGROUP_NONE, /* optinfo_flags */
5457 : TV_MACH_DEP, /* tv_id */
5458 : 0, /* properties_required */
5459 : 0, /* properties_provided */
5460 : 0, /* properties_destroyed */
5461 : 0, /* todo_flags_start */
5462 : 0, /* todo_flags_finish */
5463 : };
5464 :
5465 : class pass_apx_nf_convert : public rtl_opt_pass
5466 : {
5467 : public:
5468 292371 : pass_apx_nf_convert (gcc::context *ctxt)
5469 584742 : : rtl_opt_pass (pass_data_apx_nf_convert, ctxt)
5470 : {}
5471 :
5472 : /* opt_pass methods: */
5473 1504958 : bool gate (function *) final override
5474 : {
5475 1504958 : return (TARGET_APX_NF
5476 466 : && optimize
5477 1505414 : && optimize_function_for_speed_p (cfun));
5478 : }
5479 :
5480 372 : unsigned int execute (function *) final override
5481 : {
5482 372 : return ix86_apx_nf_convert ();
5483 : }
5484 : }; // class pass_apx_nf_convert
5485 :
5486 : } // anon namespace
5487 :
5488 : rtl_opt_pass *
5489 292371 : make_pass_apx_nf_convert (gcc::context *ctxt)
5490 : {
5491 292371 : return new pass_apx_nf_convert (ctxt);
5492 : }
5493 :
5494 : /* When a hot loop can be fit into one cacheline,
5495 : force align the loop without considering the max skip. */
5496 : static void
5497 991374 : ix86_align_loops ()
5498 : {
5499 991374 : basic_block bb;
5500 :
5501 : /* Don't do this when we don't know cache line size. */
5502 991374 : if (ix86_cost->prefetch_block == 0)
5503 9 : return;
5504 :
5505 991365 : loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
5506 991365 : profile_count count_threshold = cfun->cfg->count_max / param_align_threshold;
5507 11592524 : FOR_EACH_BB_FN (bb, cfun)
5508 : {
5509 10601159 : rtx_insn *label = BB_HEAD (bb);
5510 10601159 : bool has_fallthru = 0;
5511 10601159 : edge e;
5512 10601159 : edge_iterator ei;
5513 :
5514 10601159 : if (!LABEL_P (label))
5515 5395821 : continue;
5516 :
5517 5210141 : profile_count fallthru_count = profile_count::zero ();
5518 5210141 : profile_count branch_count = profile_count::zero ();
5519 :
5520 15138218 : FOR_EACH_EDGE (e, ei, bb->preds)
5521 : {
5522 9928077 : if (e->flags & EDGE_FALLTHRU)
5523 2515260 : has_fallthru = 1, fallthru_count += e->count ();
5524 : else
5525 7412817 : branch_count += e->count ();
5526 : }
5527 :
5528 5210141 : if (!fallthru_count.initialized_p () || !branch_count.initialized_p ())
5529 4803 : continue;
5530 :
5531 5205338 : if (bb->loop_father
5532 5205338 : && bb->loop_father->latch != EXIT_BLOCK_PTR_FOR_FN (cfun)
5533 6568790 : && (has_fallthru
5534 1363452 : ? (!(single_succ_p (bb)
5535 147990 : && single_succ (bb) == EXIT_BLOCK_PTR_FOR_FN (cfun))
5536 939471 : && optimize_bb_for_speed_p (bb)
5537 858820 : && branch_count + fallthru_count > count_threshold
5538 737127 : && (branch_count > fallthru_count * param_align_loop_iterations))
5539 : /* In case there'no fallthru for the loop.
5540 : Nops inserted won't be executed. */
5541 423981 : : (branch_count > count_threshold
5542 138660 : || (bb->count > bb->prev_bb->count * 10
5543 12654 : && (bb->prev_bb->count
5544 4653161 : <= ENTRY_BLOCK_PTR_FOR_FN (cfun)->count / 2)))))
5545 : {
5546 564831 : rtx_insn* insn, *end_insn;
5547 564831 : HOST_WIDE_INT size = 0;
5548 564831 : bool padding_p = true;
5549 564831 : basic_block tbb = bb;
5550 564831 : unsigned cond_branch_num = 0;
5551 564831 : bool detect_tight_loop_p = false;
5552 :
5553 888762 : for (unsigned int i = 0; i != bb->loop_father->num_nodes;
5554 323931 : i++, tbb = tbb->next_bb)
5555 : {
5556 : /* Only handle continuous cfg layout. */
5557 888762 : if (bb->loop_father != tbb->loop_father)
5558 : {
5559 : padding_p = false;
5560 : break;
5561 : }
5562 :
5563 10621091 : FOR_BB_INSNS (tbb, insn)
5564 : {
5565 9936938 : if (!NONDEBUG_INSN_P (insn))
5566 5798234 : continue;
5567 4138704 : size += ix86_min_insn_size (insn);
5568 :
5569 : /* We don't know size of inline asm.
5570 : Don't align loop for call. */
5571 4138704 : if (asm_noperands (PATTERN (insn)) >= 0
5572 4138704 : || CALL_P (insn))
5573 : {
5574 : size = -1;
5575 : break;
5576 : }
5577 : }
5578 :
5579 844284 : if (size == -1 || size > ix86_cost->prefetch_block)
5580 : {
5581 : padding_p = false;
5582 : break;
5583 : }
5584 :
5585 1516693 : FOR_EACH_EDGE (e, ei, tbb->succs)
5586 : {
5587 : /* It could be part of the loop. */
5588 1045323 : if (e->dest == bb)
5589 : {
5590 : detect_tight_loop_p = true;
5591 : break;
5592 : }
5593 : }
5594 :
5595 658181 : if (detect_tight_loop_p)
5596 : break;
5597 :
5598 471370 : end_insn = BB_END (tbb);
5599 471370 : if (JUMP_P (end_insn))
5600 : {
5601 : /* For decoded icache:
5602 : 1. Up to two branches are allowed per Way.
5603 : 2. A non-conditional branch is the last micro-op in a Way.
5604 : */
5605 383034 : if (onlyjump_p (end_insn)
5606 383034 : && (any_uncondjump_p (end_insn)
5607 318837 : || single_succ_p (tbb)))
5608 : {
5609 : padding_p = false;
5610 : break;
5611 : }
5612 318837 : else if (++cond_branch_num >= 2)
5613 : {
5614 : padding_p = false;
5615 : break;
5616 : }
5617 : }
5618 :
5619 : }
5620 :
5621 564831 : if (padding_p && detect_tight_loop_p)
5622 : {
5623 373622 : emit_insn_before (gen_max_skip_align (GEN_INT (ceil_log2 (size)),
5624 : GEN_INT (0)), label);
5625 : /* End of function. */
5626 186811 : if (!tbb || tbb == EXIT_BLOCK_PTR_FOR_FN (cfun))
5627 : break;
5628 : /* Skip bb which already fits into one cacheline. */
5629 : bb = tbb;
5630 : }
5631 : }
5632 : }
5633 :
5634 991365 : loop_optimizer_finalize ();
5635 991365 : free_dominance_info (CDI_DOMINATORS);
5636 : }
5637 :
5638 : namespace {
5639 :
5640 : const pass_data pass_data_align_tight_loops =
5641 : {
5642 : RTL_PASS, /* type */
5643 : "align_tight_loops", /* name */
5644 : OPTGROUP_NONE, /* optinfo_flags */
5645 : TV_MACH_DEP, /* tv_id */
5646 : 0, /* properties_required */
5647 : 0, /* properties_provided */
5648 : 0, /* properties_destroyed */
5649 : 0, /* todo_flags_start */
5650 : 0, /* todo_flags_finish */
5651 : };
5652 :
5653 : class pass_align_tight_loops : public rtl_opt_pass
5654 : {
5655 : public:
5656 292371 : pass_align_tight_loops (gcc::context *ctxt)
5657 584742 : : rtl_opt_pass (pass_data_align_tight_loops, ctxt)
5658 : {}
5659 :
5660 : /* opt_pass methods: */
5661 1504958 : bool gate (function *) final override
5662 : {
5663 1504958 : return TARGET_ALIGN_TIGHT_LOOPS
5664 1504472 : && optimize
5665 2561535 : && optimize_function_for_speed_p (cfun);
5666 : }
5667 :
5668 991374 : unsigned int execute (function *) final override
5669 : {
5670 991374 : timevar_push (TV_MACH_DEP);
5671 : #ifdef ASM_OUTPUT_MAX_SKIP_ALIGN
5672 991374 : ix86_align_loops ();
5673 : #endif
5674 991374 : timevar_pop (TV_MACH_DEP);
5675 991374 : return 0;
5676 : }
5677 : }; // class pass_align_tight_loops
5678 :
5679 : } // anon namespace
5680 :
5681 : rtl_opt_pass *
5682 292371 : make_pass_align_tight_loops (gcc::context *ctxt)
5683 : {
5684 292371 : return new pass_align_tight_loops (ctxt);
5685 : }
5686 :
5687 : /* This compares the priority of target features in function DECL1
5688 : and DECL2. It returns positive value if DECL1 is higher priority,
5689 : negative value if DECL2 is higher priority and 0 if they are the
5690 : same. */
5691 :
5692 : int
5693 5827 : ix86_compare_version_priority (tree decl1, tree decl2)
5694 : {
5695 5827 : unsigned int priority1 = get_builtin_code_for_version (decl1, NULL);
5696 5827 : unsigned int priority2 = get_builtin_code_for_version (decl2, NULL);
5697 :
5698 5827 : return (int)priority1 - (int)priority2;
5699 : }
5700 :
5701 : /* V1 and V2 point to function versions with different priorities
5702 : based on the target ISA. This function compares their priorities. */
5703 :
5704 : static int
5705 6882 : feature_compare (const void *v1, const void *v2)
5706 : {
5707 6882 : typedef struct _function_version_info
5708 : {
5709 : tree version_decl;
5710 : tree predicate_chain;
5711 : unsigned int dispatch_priority;
5712 : } function_version_info;
5713 :
5714 6882 : const function_version_info c1 = *(const function_version_info *)v1;
5715 6882 : const function_version_info c2 = *(const function_version_info *)v2;
5716 6882 : return (c2.dispatch_priority - c1.dispatch_priority);
5717 : }
5718 :
5719 : /* This adds a condition to the basic_block NEW_BB in function FUNCTION_DECL
5720 : to return a pointer to VERSION_DECL if the outcome of the expression
5721 : formed by PREDICATE_CHAIN is true. This function will be called during
5722 : version dispatch to decide which function version to execute. It returns
5723 : the basic block at the end, to which more conditions can be added. */
5724 :
5725 : static basic_block
5726 842 : add_condition_to_bb (tree function_decl, tree version_decl,
5727 : tree predicate_chain, basic_block new_bb)
5728 : {
5729 842 : gimple *return_stmt;
5730 842 : tree convert_expr, result_var;
5731 842 : gimple *convert_stmt;
5732 842 : gimple *call_cond_stmt;
5733 842 : gimple *if_else_stmt;
5734 :
5735 842 : basic_block bb1, bb2, bb3;
5736 842 : edge e12, e23;
5737 :
5738 842 : tree cond_var, and_expr_var = NULL_TREE;
5739 842 : gimple_seq gseq;
5740 :
5741 842 : tree predicate_decl, predicate_arg;
5742 :
5743 842 : push_cfun (DECL_STRUCT_FUNCTION (function_decl));
5744 :
5745 842 : gcc_assert (new_bb != NULL);
5746 842 : gseq = bb_seq (new_bb);
5747 :
5748 :
5749 842 : convert_expr = build1 (CONVERT_EXPR, ptr_type_node,
5750 : build_fold_addr_expr (version_decl));
5751 842 : result_var = create_tmp_var (ptr_type_node);
5752 842 : convert_stmt = gimple_build_assign (result_var, convert_expr);
5753 842 : return_stmt = gimple_build_return (result_var);
5754 :
5755 842 : if (predicate_chain == NULL_TREE)
5756 : {
5757 201 : gimple_seq_add_stmt (&gseq, convert_stmt);
5758 201 : gimple_seq_add_stmt (&gseq, return_stmt);
5759 201 : set_bb_seq (new_bb, gseq);
5760 201 : gimple_set_bb (convert_stmt, new_bb);
5761 201 : gimple_set_bb (return_stmt, new_bb);
5762 201 : pop_cfun ();
5763 201 : return new_bb;
5764 : }
5765 :
5766 1321 : while (predicate_chain != NULL)
5767 : {
5768 680 : cond_var = create_tmp_var (integer_type_node);
5769 680 : predicate_decl = TREE_PURPOSE (predicate_chain);
5770 680 : predicate_arg = TREE_VALUE (predicate_chain);
5771 680 : call_cond_stmt = gimple_build_call (predicate_decl, 1, predicate_arg);
5772 680 : gimple_call_set_lhs (call_cond_stmt, cond_var);
5773 :
5774 680 : gimple_set_block (call_cond_stmt, DECL_INITIAL (function_decl));
5775 680 : gimple_set_bb (call_cond_stmt, new_bb);
5776 680 : gimple_seq_add_stmt (&gseq, call_cond_stmt);
5777 :
5778 680 : predicate_chain = TREE_CHAIN (predicate_chain);
5779 :
5780 680 : if (and_expr_var == NULL)
5781 : and_expr_var = cond_var;
5782 : else
5783 : {
5784 39 : gimple *assign_stmt;
5785 : /* Use MIN_EXPR to check if any integer is zero?.
5786 : and_expr_var = min_expr <cond_var, and_expr_var> */
5787 39 : assign_stmt = gimple_build_assign (and_expr_var,
5788 : build2 (MIN_EXPR, integer_type_node,
5789 : cond_var, and_expr_var));
5790 :
5791 39 : gimple_set_block (assign_stmt, DECL_INITIAL (function_decl));
5792 39 : gimple_set_bb (assign_stmt, new_bb);
5793 39 : gimple_seq_add_stmt (&gseq, assign_stmt);
5794 : }
5795 : }
5796 :
5797 641 : if_else_stmt = gimple_build_cond (GT_EXPR, and_expr_var,
5798 : integer_zero_node,
5799 : NULL_TREE, NULL_TREE);
5800 641 : gimple_set_block (if_else_stmt, DECL_INITIAL (function_decl));
5801 641 : gimple_set_bb (if_else_stmt, new_bb);
5802 641 : gimple_seq_add_stmt (&gseq, if_else_stmt);
5803 :
5804 641 : gimple_seq_add_stmt (&gseq, convert_stmt);
5805 641 : gimple_seq_add_stmt (&gseq, return_stmt);
5806 641 : set_bb_seq (new_bb, gseq);
5807 :
5808 641 : bb1 = new_bb;
5809 641 : e12 = split_block (bb1, if_else_stmt);
5810 641 : bb2 = e12->dest;
5811 641 : e12->flags &= ~EDGE_FALLTHRU;
5812 641 : e12->flags |= EDGE_TRUE_VALUE;
5813 :
5814 641 : e23 = split_block (bb2, return_stmt);
5815 :
5816 641 : gimple_set_bb (convert_stmt, bb2);
5817 641 : gimple_set_bb (return_stmt, bb2);
5818 :
5819 641 : bb3 = e23->dest;
5820 641 : make_edge (bb1, bb3, EDGE_FALSE_VALUE);
5821 :
5822 641 : remove_edge (e23);
5823 641 : make_edge (bb2, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
5824 :
5825 641 : pop_cfun ();
5826 :
5827 641 : return bb3;
5828 : }
5829 :
5830 : /* This function generates the dispatch function for
5831 : multi-versioned functions. DISPATCH_DECL is the function which will
5832 : contain the dispatch logic. FNDECLS are the function choices for
5833 : dispatch, and is a tree chain. EMPTY_BB is the basic block pointer
5834 : in DISPATCH_DECL in which the dispatch code is generated. */
5835 :
5836 : static int
5837 201 : dispatch_function_versions (tree dispatch_decl,
5838 : void *fndecls_p,
5839 : basic_block *empty_bb)
5840 : {
5841 201 : tree default_decl;
5842 201 : gimple *ifunc_cpu_init_stmt;
5843 201 : gimple_seq gseq;
5844 201 : int ix;
5845 201 : tree ele;
5846 201 : vec<tree> *fndecls;
5847 201 : unsigned int num_versions = 0;
5848 201 : unsigned int actual_versions = 0;
5849 201 : unsigned int i;
5850 :
5851 201 : struct _function_version_info
5852 : {
5853 : tree version_decl;
5854 : tree predicate_chain;
5855 : unsigned int dispatch_priority;
5856 : }*function_version_info;
5857 :
5858 201 : gcc_assert (dispatch_decl != NULL
5859 : && fndecls_p != NULL
5860 : && empty_bb != NULL);
5861 :
5862 : /*fndecls_p is actually a vector. */
5863 201 : fndecls = static_cast<vec<tree> *> (fndecls_p);
5864 :
5865 : /* At least one more version other than the default. */
5866 201 : num_versions = fndecls->length ();
5867 201 : gcc_assert (num_versions >= 2);
5868 :
5869 201 : function_version_info = (struct _function_version_info *)
5870 201 : XNEWVEC (struct _function_version_info, (num_versions - 1));
5871 :
5872 : /* The first version in the vector is the default decl. */
5873 201 : default_decl = (*fndecls)[0];
5874 :
5875 201 : push_cfun (DECL_STRUCT_FUNCTION (dispatch_decl));
5876 :
5877 201 : gseq = bb_seq (*empty_bb);
5878 : /* Function version dispatch is via IFUNC. IFUNC resolvers fire before
5879 : constructors, so explicitly call __builtin_cpu_init here. */
5880 201 : ifunc_cpu_init_stmt
5881 201 : = gimple_build_call_vec (get_ix86_builtin (IX86_BUILTIN_CPU_INIT), vNULL);
5882 201 : gimple_seq_add_stmt (&gseq, ifunc_cpu_init_stmt);
5883 201 : gimple_set_bb (ifunc_cpu_init_stmt, *empty_bb);
5884 201 : set_bb_seq (*empty_bb, gseq);
5885 :
5886 201 : pop_cfun ();
5887 :
5888 :
5889 999 : for (ix = 1; fndecls->iterate (ix, &ele); ++ix)
5890 : {
5891 798 : tree version_decl = ele;
5892 798 : tree predicate_chain = NULL_TREE;
5893 798 : unsigned int priority;
5894 : /* Get attribute string, parse it and find the right predicate decl.
5895 : The predicate function could be a lengthy combination of many
5896 : features, like arch-type and various isa-variants. */
5897 798 : priority = get_builtin_code_for_version (version_decl,
5898 : &predicate_chain);
5899 :
5900 798 : if (predicate_chain == NULL_TREE)
5901 157 : continue;
5902 :
5903 641 : function_version_info [actual_versions].version_decl = version_decl;
5904 641 : function_version_info [actual_versions].predicate_chain
5905 641 : = predicate_chain;
5906 641 : function_version_info [actual_versions].dispatch_priority = priority;
5907 641 : actual_versions++;
5908 : }
5909 :
5910 : /* Sort the versions according to descending order of dispatch priority. The
5911 : priority is based on the ISA. This is not a perfect solution. There
5912 : could still be ambiguity. If more than one function version is suitable
5913 : to execute, which one should be dispatched? In future, allow the user
5914 : to specify a dispatch priority next to the version. */
5915 201 : qsort (function_version_info, actual_versions,
5916 : sizeof (struct _function_version_info), feature_compare);
5917 :
5918 1043 : for (i = 0; i < actual_versions; ++i)
5919 641 : *empty_bb = add_condition_to_bb (dispatch_decl,
5920 : function_version_info[i].version_decl,
5921 641 : function_version_info[i].predicate_chain,
5922 : *empty_bb);
5923 :
5924 : /* dispatch default version at the end. */
5925 201 : *empty_bb = add_condition_to_bb (dispatch_decl, default_decl,
5926 : NULL, *empty_bb);
5927 :
5928 201 : free (function_version_info);
5929 201 : return 0;
5930 : }
5931 :
5932 : /* This function changes the assembler name for functions that are
5933 : versions. If DECL is a function version and has a "target"
5934 : attribute, it appends the attribute string to its assembler name. */
5935 :
5936 : static tree
5937 1121 : ix86_mangle_function_version_assembler_name (tree decl, tree id)
5938 : {
5939 1121 : tree version_attr;
5940 1121 : char *attr_str;
5941 :
5942 1121 : if (DECL_DECLARED_INLINE_P (decl)
5943 1170 : && lookup_attribute ("gnu_inline",
5944 49 : DECL_ATTRIBUTES (decl)))
5945 0 : error_at (DECL_SOURCE_LOCATION (decl),
5946 : "function versions cannot be marked as %<gnu_inline%>,"
5947 : " bodies have to be generated");
5948 :
5949 1121 : if (DECL_VIRTUAL_P (decl)
5950 2242 : || DECL_VINDEX (decl))
5951 0 : sorry ("virtual function multiversioning not supported");
5952 :
5953 1121 : version_attr = lookup_attribute ("target", DECL_ATTRIBUTES (decl));
5954 :
5955 : /* target attribute string cannot be NULL. */
5956 1121 : gcc_assert (version_attr != NULL_TREE);
5957 :
5958 1121 : attr_str = sorted_attr_string (TREE_VALUE (version_attr));
5959 :
5960 : /* Allow assembler name to be modified if already set. */
5961 1121 : if (DECL_ASSEMBLER_NAME_SET_P (decl))
5962 1106 : SET_DECL_RTL (decl, NULL);
5963 :
5964 1121 : tree ret = clone_identifier (id, attr_str, true);
5965 :
5966 1121 : XDELETEVEC (attr_str);
5967 :
5968 1121 : return ret;
5969 : }
5970 :
5971 : tree
5972 498934109 : ix86_mangle_decl_assembler_name (tree decl, tree id)
5973 : {
5974 : /* For function version, add the target suffix to the assembler name. */
5975 498934109 : if (TREE_CODE (decl) == FUNCTION_DECL)
5976 : {
5977 462483523 : cgraph_node *node = cgraph_node::get (decl);
5978 : /* Mangle all versions when annotated with target_clones, but only
5979 : non-default versions when annotated with target attributes. */
5980 462483523 : if (DECL_FUNCTION_VERSIONED (decl)
5981 462483523 : && (node->is_target_clone
5982 1092 : || !is_function_default_version (node->decl)))
5983 1121 : id = ix86_mangle_function_version_assembler_name (decl, id);
5984 : /* Mangle the dispatched symbol but only in the case of target clones. */
5985 462482402 : else if (node && node->dispatcher_function && !node->is_target_clone)
5986 117 : id = clone_identifier (id, "ifunc");
5987 67634137 : else if (node && node->dispatcher_resolver_function)
5988 201 : id = clone_identifier (id, "resolver");
5989 : }
5990 : #ifdef SUBTARGET_MANGLE_DECL_ASSEMBLER_NAME
5991 : id = SUBTARGET_MANGLE_DECL_ASSEMBLER_NAME (decl, id);
5992 : #endif
5993 :
5994 498934109 : return id;
5995 : }
5996 :
5997 : /* Make a dispatcher declaration for the multi-versioned function DECL.
5998 : Calls to DECL function will be replaced with calls to the dispatcher
5999 : by the front-end. Returns the decl of the dispatcher function. */
6000 :
6001 : tree
6002 327 : ix86_get_function_versions_dispatcher (void *decl)
6003 : {
6004 327 : tree fn = (tree) decl;
6005 327 : struct cgraph_node *node = NULL;
6006 327 : struct cgraph_node *default_node = NULL;
6007 327 : struct cgraph_function_version_info *node_v = NULL;
6008 :
6009 327 : tree dispatch_decl = NULL;
6010 :
6011 327 : struct cgraph_function_version_info *default_version_info = NULL;
6012 :
6013 654 : gcc_assert (fn != NULL && DECL_FUNCTION_VERSIONED (fn));
6014 :
6015 327 : node = cgraph_node::get (fn);
6016 327 : gcc_assert (node != NULL);
6017 :
6018 327 : node_v = node->function_version ();
6019 327 : gcc_assert (node_v != NULL);
6020 :
6021 327 : if (node_v->dispatcher_resolver != NULL)
6022 : return node_v->dispatcher_resolver;
6023 :
6024 : /* The default node is always the beginning of the chain. */
6025 : default_version_info = node_v;
6026 678 : while (default_version_info->prev != NULL)
6027 : default_version_info = default_version_info->prev;
6028 213 : default_node = default_version_info->this_node;
6029 :
6030 : /* If there is no default node, just return NULL. */
6031 213 : if (!is_function_default_version (default_node->decl))
6032 : return NULL;
6033 :
6034 : #if defined (ASM_OUTPUT_TYPE_DIRECTIVE)
6035 204 : if (targetm.has_ifunc_p ())
6036 : {
6037 204 : struct cgraph_function_version_info *it_v = NULL;
6038 :
6039 : /* Right now, the dispatching is done via ifunc. */
6040 204 : dispatch_decl = make_dispatcher_decl (default_node->decl);
6041 :
6042 : /* Set the dispatcher for all the versions. */
6043 204 : it_v = default_version_info;
6044 1413 : while (it_v != NULL)
6045 : {
6046 1005 : it_v->dispatcher_resolver = dispatch_decl;
6047 1005 : it_v = it_v->next;
6048 : }
6049 : }
6050 : else
6051 : #endif
6052 : {
6053 0 : error_at (DECL_SOURCE_LOCATION (default_node->decl),
6054 : "multiversioning needs %<ifunc%> which is not supported "
6055 : "on this target");
6056 : }
6057 :
6058 : return dispatch_decl;
6059 : }
6060 :
6061 : /* Make the resolver function decl to dispatch the versions of
6062 : a multi-versioned function, DEFAULT_DECL. IFUNC_ALIAS_DECL is
6063 : ifunc alias that will point to the created resolver. Create an
6064 : empty basic block in the resolver and store the pointer in
6065 : EMPTY_BB. Return the decl of the resolver function. */
6066 :
6067 : static tree
6068 201 : make_resolver_func (const tree default_decl,
6069 : const tree ifunc_alias_decl,
6070 : basic_block *empty_bb)
6071 : {
6072 201 : tree decl, type, t;
6073 :
6074 : /* The resolver function should return a (void *). */
6075 201 : type = build_function_type_list (ptr_type_node, NULL_TREE);
6076 :
6077 201 : cgraph_node *node = cgraph_node::get (default_decl);
6078 201 : gcc_assert (node && node->function_version ());
6079 :
6080 201 : decl = build_fn_decl (IDENTIFIER_POINTER (DECL_NAME (default_decl)), type);
6081 :
6082 : /* Set the assembler name to prevent cgraph_node attempting to mangle. */
6083 201 : SET_DECL_ASSEMBLER_NAME (decl, DECL_ASSEMBLER_NAME (default_decl));
6084 :
6085 201 : cgraph_node *resolver_node = cgraph_node::get_create (decl);
6086 201 : resolver_node->dispatcher_resolver_function = true;
6087 :
6088 201 : if (node->is_target_clone)
6089 87 : resolver_node->is_target_clone = true;
6090 :
6091 201 : tree id = ix86_mangle_decl_assembler_name
6092 201 : (decl, node->function_version ()->assembler_name);
6093 201 : symtab->change_decl_assembler_name (decl, id);
6094 :
6095 201 : DECL_NAME (decl) = DECL_NAME (default_decl);
6096 201 : TREE_USED (decl) = 1;
6097 201 : DECL_ARTIFICIAL (decl) = 1;
6098 201 : DECL_IGNORED_P (decl) = 1;
6099 201 : TREE_PUBLIC (decl) = 0;
6100 201 : DECL_UNINLINABLE (decl) = 1;
6101 :
6102 : /* Resolver is not external, body is generated. */
6103 201 : DECL_EXTERNAL (decl) = 0;
6104 201 : DECL_EXTERNAL (ifunc_alias_decl) = 0;
6105 :
6106 201 : DECL_CONTEXT (decl) = NULL_TREE;
6107 201 : DECL_INITIAL (decl) = make_node (BLOCK);
6108 201 : DECL_STATIC_CONSTRUCTOR (decl) = 0;
6109 :
6110 201 : if (DECL_COMDAT_GROUP (default_decl)
6111 201 : || TREE_PUBLIC (default_decl))
6112 : {
6113 : /* In this case, each translation unit with a call to this
6114 : versioned function will put out a resolver. Ensure it
6115 : is comdat to keep just one copy. */
6116 177 : DECL_COMDAT (decl) = 1;
6117 177 : make_decl_one_only (decl, DECL_ASSEMBLER_NAME (decl));
6118 : }
6119 : else
6120 24 : TREE_PUBLIC (ifunc_alias_decl) = 0;
6121 :
6122 : /* Build result decl and add to function_decl. */
6123 201 : t = build_decl (UNKNOWN_LOCATION, RESULT_DECL, NULL_TREE, ptr_type_node);
6124 201 : DECL_CONTEXT (t) = decl;
6125 201 : DECL_ARTIFICIAL (t) = 1;
6126 201 : DECL_IGNORED_P (t) = 1;
6127 201 : DECL_RESULT (decl) = t;
6128 :
6129 201 : gimplify_function_tree (decl);
6130 201 : push_cfun (DECL_STRUCT_FUNCTION (decl));
6131 201 : *empty_bb = init_lowered_empty_function (decl, false,
6132 : profile_count::uninitialized ());
6133 :
6134 201 : cgraph_node::add_new_function (decl, true);
6135 201 : symtab->call_cgraph_insertion_hooks (cgraph_node::get_create (decl));
6136 :
6137 201 : pop_cfun ();
6138 :
6139 201 : gcc_assert (ifunc_alias_decl != NULL);
6140 : /* Mark ifunc_alias_decl as "ifunc" with resolver as resolver_name. */
6141 201 : DECL_ATTRIBUTES (ifunc_alias_decl)
6142 201 : = make_attribute ("ifunc", IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)),
6143 201 : DECL_ATTRIBUTES (ifunc_alias_decl));
6144 :
6145 : /* Create the alias for dispatch to resolver here. */
6146 201 : cgraph_node::create_same_body_alias (ifunc_alias_decl, decl);
6147 201 : return decl;
6148 : }
6149 :
6150 : /* Generate the dispatching code body to dispatch multi-versioned function
6151 : DECL. The target hook is called to process the "target" attributes and
6152 : provide the code to dispatch the right function at run-time. NODE points
6153 : to the dispatcher decl whose body will be created. */
6154 :
6155 : tree
6156 201 : ix86_generate_version_dispatcher_body (void *node_p)
6157 : {
6158 201 : tree resolver_decl;
6159 201 : basic_block empty_bb;
6160 201 : tree default_ver_decl;
6161 201 : struct cgraph_node *versn;
6162 201 : struct cgraph_node *node;
6163 :
6164 201 : struct cgraph_function_version_info *node_version_info = NULL;
6165 201 : struct cgraph_function_version_info *versn_info = NULL;
6166 :
6167 201 : node = (cgraph_node *)node_p;
6168 :
6169 201 : node_version_info = node->function_version ();
6170 201 : gcc_assert (node->dispatcher_function
6171 : && node_version_info != NULL);
6172 :
6173 201 : if (node_version_info->dispatcher_resolver)
6174 : return node_version_info->dispatcher_resolver;
6175 :
6176 : /* The first version in the chain corresponds to the default version. */
6177 201 : default_ver_decl = node_version_info->next->this_node->decl;
6178 :
6179 : /* node is going to be an alias, so remove the finalized bit. */
6180 201 : node->definition = false;
6181 :
6182 201 : resolver_decl = make_resolver_func (default_ver_decl,
6183 : node->decl, &empty_bb);
6184 :
6185 201 : node_version_info->dispatcher_resolver = resolver_decl;
6186 :
6187 201 : push_cfun (DECL_STRUCT_FUNCTION (resolver_decl));
6188 :
6189 201 : auto_vec<tree, 2> fn_ver_vec;
6190 :
6191 1200 : for (versn_info = node_version_info->next; versn_info;
6192 999 : versn_info = versn_info->next)
6193 : {
6194 999 : versn = versn_info->this_node;
6195 : /* Check for virtual functions here again, as by this time it should
6196 : have been determined if this function needs a vtable index or
6197 : not. This happens for methods in derived classes that override
6198 : virtual methods in base classes but are not explicitly marked as
6199 : virtual. */
6200 999 : if (DECL_VIRTUAL_P (versn->decl))
6201 0 : sorry ("virtual function multiversioning not supported");
6202 :
6203 999 : fn_ver_vec.safe_push (versn->decl);
6204 : }
6205 :
6206 201 : dispatch_function_versions (resolver_decl, &fn_ver_vec, &empty_bb);
6207 201 : cgraph_edge::rebuild_edges ();
6208 201 : pop_cfun ();
6209 201 : return resolver_decl;
6210 201 : }
6211 :
6212 :
|
