Line data Source code
1 : /* Copyright (C) 1988-2026 Free Software Foundation, Inc.
2 :
3 : This file is part of GCC.
4 :
5 : GCC is free software; you can redistribute it and/or modify
6 : it under the terms of the GNU General Public License as published by
7 : the Free Software Foundation; either version 3, or (at your option)
8 : any later version.
9 :
10 : GCC is distributed in the hope that it will be useful,
11 : but WITHOUT ANY WARRANTY; without even the implied warranty of
12 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 : GNU General Public License for more details.
14 :
15 : You should have received a copy of the GNU General Public License
16 : along with GCC; see the file COPYING3. If not see
17 : <http://www.gnu.org/licenses/>. */
18 :
19 : #define IN_TARGET_CODE 1
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "memmodel.h"
28 : #include "gimple.h"
29 : #include "cfghooks.h"
30 : #include "cfgloop.h"
31 : #include "df.h"
32 : #include "tm_p.h"
33 : #include "stringpool.h"
34 : #include "expmed.h"
35 : #include "optabs.h"
36 : #include "regs.h"
37 : #include "emit-rtl.h"
38 : #include "recog.h"
39 : #include "cgraph.h"
40 : #include "diagnostic.h"
41 : #include "cfgbuild.h"
42 : #include "alias.h"
43 : #include "fold-const.h"
44 : #include "attribs.h"
45 : #include "calls.h"
46 : #include "stor-layout.h"
47 : #include "varasm.h"
48 : #include "output.h"
49 : #include "insn-attr.h"
50 : #include "flags.h"
51 : #include "except.h"
52 : #include "explow.h"
53 : #include "expr.h"
54 : #include "cfgrtl.h"
55 : #include "common/common-target.h"
56 : #include "langhooks.h"
57 : #include "reload.h"
58 : #include "gimplify.h"
59 : #include "dwarf2.h"
60 : #include "tm-constrs.h"
61 : #include "cselib.h"
62 : #include "sched-int.h"
63 : #include "opts.h"
64 : #include "tree-pass.h"
65 : #include "context.h"
66 : #include "pass_manager.h"
67 : #include "target-globals.h"
68 : #include "gimple-iterator.h"
69 : #include "shrink-wrap.h"
70 : #include "builtins.h"
71 : #include "rtl-iter.h"
72 : #include "tree-iterator.h"
73 : #include "dbgcnt.h"
74 : #include "case-cfn-macros.h"
75 : #include "dojump.h"
76 : #include "fold-const-call.h"
77 : #include "tree-vrp.h"
78 : #include "tree-ssanames.h"
79 : #include "selftest.h"
80 : #include "selftest-rtl.h"
81 : #include "print-rtl.h"
82 : #include "intl.h"
83 : #include "ifcvt.h"
84 : #include "symbol-summary.h"
85 : #include "sreal.h"
86 : #include "ipa-cp.h"
87 : #include "ipa-prop.h"
88 : #include "ipa-fnsummary.h"
89 : #include "wide-int-bitmask.h"
90 : #include "tree-vector-builder.h"
91 : #include "debug.h"
92 : #include "dwarf2out.h"
93 : #include "function-abi.h"
94 : #include "i386-builtins.h"
95 : #include "i386-features.h"
96 : #include "i386-expand.h"
97 :
98 : const char * const xlogue_layout::STUB_BASE_NAMES[XLOGUE_STUB_COUNT] = {
99 : "savms64",
100 : "resms64",
101 : "resms64x",
102 : "savms64f",
103 : "resms64f",
104 : "resms64fx"
105 : };
106 :
107 : const unsigned xlogue_layout::REG_ORDER[xlogue_layout::MAX_REGS] = {
108 : /* The below offset values are where each register is stored for the layout
109 : relative to incoming stack pointer. The value of each m_regs[].offset will
110 : be relative to the incoming base pointer (rax or rsi) used by the stub.
111 :
112 : s_instances: 0 1 2 3
113 : Offset: realigned or aligned + 8
114 : Register aligned aligned + 8 aligned w/HFP w/HFP */
115 : XMM15_REG, /* 0x10 0x18 0x10 0x18 */
116 : XMM14_REG, /* 0x20 0x28 0x20 0x28 */
117 : XMM13_REG, /* 0x30 0x38 0x30 0x38 */
118 : XMM12_REG, /* 0x40 0x48 0x40 0x48 */
119 : XMM11_REG, /* 0x50 0x58 0x50 0x58 */
120 : XMM10_REG, /* 0x60 0x68 0x60 0x68 */
121 : XMM9_REG, /* 0x70 0x78 0x70 0x78 */
122 : XMM8_REG, /* 0x80 0x88 0x80 0x88 */
123 : XMM7_REG, /* 0x90 0x98 0x90 0x98 */
124 : XMM6_REG, /* 0xa0 0xa8 0xa0 0xa8 */
125 : SI_REG, /* 0xa8 0xb0 0xa8 0xb0 */
126 : DI_REG, /* 0xb0 0xb8 0xb0 0xb8 */
127 : BX_REG, /* 0xb8 0xc0 0xb8 0xc0 */
128 : BP_REG, /* 0xc0 0xc8 N/A N/A */
129 : R12_REG, /* 0xc8 0xd0 0xc0 0xc8 */
130 : R13_REG, /* 0xd0 0xd8 0xc8 0xd0 */
131 : R14_REG, /* 0xd8 0xe0 0xd0 0xd8 */
132 : R15_REG, /* 0xe0 0xe8 0xd8 0xe0 */
133 : };
134 :
135 : /* Instantiate static const values. */
136 : const HOST_WIDE_INT xlogue_layout::STUB_INDEX_OFFSET;
137 : const unsigned xlogue_layout::MIN_REGS;
138 : const unsigned xlogue_layout::MAX_REGS;
139 : const unsigned xlogue_layout::MAX_EXTRA_REGS;
140 : const unsigned xlogue_layout::VARIANT_COUNT;
141 : const unsigned xlogue_layout::STUB_NAME_MAX_LEN;
142 :
143 : /* Initialize xlogue_layout::s_stub_names to zero. */
144 : char xlogue_layout::s_stub_names[2][XLOGUE_STUB_COUNT][VARIANT_COUNT]
145 : [STUB_NAME_MAX_LEN];
146 :
147 : /* Instantiates all xlogue_layout instances. */
148 : const xlogue_layout xlogue_layout::s_instances[XLOGUE_SET_COUNT] = {
149 : xlogue_layout (0, false),
150 : xlogue_layout (8, false),
151 : xlogue_layout (0, true),
152 : xlogue_layout (8, true)
153 : };
154 :
155 : /* Return an appropriate const instance of xlogue_layout based upon values
156 : in cfun->machine and crtl. */
157 : const class xlogue_layout &
158 49891 : xlogue_layout::get_instance ()
159 : {
160 49891 : enum xlogue_stub_sets stub_set;
161 49891 : bool aligned_plus_8 = cfun->machine->call_ms2sysv_pad_in;
162 :
163 49891 : if (stack_realign_fp)
164 : stub_set = XLOGUE_SET_HFP_ALIGNED_OR_REALIGN;
165 40910 : else if (frame_pointer_needed)
166 25246 : stub_set = aligned_plus_8
167 31552 : ? XLOGUE_SET_HFP_ALIGNED_PLUS_8
168 : : XLOGUE_SET_HFP_ALIGNED_OR_REALIGN;
169 : else
170 9358 : stub_set = aligned_plus_8 ? XLOGUE_SET_ALIGNED_PLUS_8 : XLOGUE_SET_ALIGNED;
171 :
172 49891 : return s_instances[stub_set];
173 : }
174 :
175 : /* Determine how many clobbered registers can be saved by the stub.
176 : Returns the count of registers the stub will save and restore. */
177 : unsigned
178 35225 : xlogue_layout::count_stub_managed_regs ()
179 : {
180 35225 : bool hfp = frame_pointer_needed || stack_realign_fp;
181 35225 : unsigned i, count;
182 35225 : unsigned regno;
183 :
184 93950 : for (count = i = MIN_REGS; i < MAX_REGS; ++i)
185 : {
186 92750 : regno = REG_ORDER[i];
187 92750 : if (regno == BP_REG && hfp)
188 17880 : continue;
189 74870 : if (!ix86_save_reg (regno, false, false))
190 : break;
191 40845 : ++count;
192 : }
193 35225 : return count;
194 : }
195 :
196 : /* Determine if register REGNO is a stub managed register given the
197 : total COUNT of stub managed registers. */
198 : bool
199 2642240 : xlogue_layout::is_stub_managed_reg (unsigned regno, unsigned count)
200 : {
201 2642240 : bool hfp = frame_pointer_needed || stack_realign_fp;
202 2642240 : unsigned i;
203 :
204 34418160 : for (i = 0; i < count; ++i)
205 : {
206 32275013 : gcc_assert (i < MAX_REGS);
207 32275013 : if (REG_ORDER[i] == BP_REG && hfp)
208 510254 : ++count;
209 31764759 : else if (REG_ORDER[i] == regno)
210 : return true;
211 : }
212 : return false;
213 : }
214 :
215 : /* Constructor for xlogue_layout. */
216 1184040 : xlogue_layout::xlogue_layout (HOST_WIDE_INT stack_align_off_in, bool hfp)
217 1184040 : : m_hfp (hfp) , m_nregs (hfp ? 17 : 18),
218 1184040 : m_stack_align_off_in (stack_align_off_in)
219 : {
220 1184040 : HOST_WIDE_INT offset = stack_align_off_in;
221 1184040 : unsigned i, j;
222 :
223 22496760 : for (i = j = 0; i < MAX_REGS; ++i)
224 : {
225 21312720 : unsigned regno = REG_ORDER[i];
226 :
227 21312720 : if (regno == BP_REG && hfp)
228 592020 : continue;
229 20720700 : if (SSE_REGNO_P (regno))
230 : {
231 11840400 : offset += 16;
232 : /* Verify that SSE regs are always aligned. */
233 11840400 : gcc_assert (!((stack_align_off_in + offset) & 15));
234 : }
235 : else
236 8880300 : offset += 8;
237 :
238 20720700 : m_regs[j].regno = regno;
239 20720700 : m_regs[j++].offset = offset - STUB_INDEX_OFFSET;
240 : }
241 1184040 : gcc_assert (j == m_nregs);
242 1184040 : }
243 :
244 : const char *
245 14666 : xlogue_layout::get_stub_name (enum xlogue_stub stub,
246 : unsigned n_extra_regs)
247 : {
248 14666 : const int have_avx = TARGET_AVX;
249 14666 : char *name = s_stub_names[!!have_avx][stub][n_extra_regs];
250 :
251 : /* Lazy init */
252 14666 : if (!*name)
253 : {
254 362 : int res = snprintf (name, STUB_NAME_MAX_LEN, "__%s_%s_%u",
255 : (have_avx ? "avx" : "sse"),
256 181 : STUB_BASE_NAMES[stub],
257 : MIN_REGS + n_extra_regs);
258 181 : gcc_checking_assert (res < (int)STUB_NAME_MAX_LEN);
259 : }
260 :
261 14666 : return name;
262 : }
263 :
264 : /* Return rtx of a symbol ref for the entry point (based upon
265 : cfun->machine->call_ms2sysv_extra_regs) of the specified stub. */
266 : rtx
267 14666 : xlogue_layout::get_stub_rtx (enum xlogue_stub stub)
268 : {
269 14666 : const unsigned n_extra_regs = cfun->machine->call_ms2sysv_extra_regs;
270 14666 : gcc_checking_assert (n_extra_regs <= MAX_EXTRA_REGS);
271 14666 : gcc_assert (stub < XLOGUE_STUB_COUNT);
272 14666 : gcc_assert (crtl->stack_realign_finalized);
273 :
274 14666 : return gen_rtx_SYMBOL_REF (Pmode, get_stub_name (stub, n_extra_regs));
275 : }
276 :
277 : namespace {
278 :
279 : class scalar_chain
280 : {
281 : public:
282 : scalar_chain (enum machine_mode, enum machine_mode);
283 : virtual ~scalar_chain ();
284 :
285 : static unsigned max_id;
286 :
287 : /* Scalar mode. */
288 : enum machine_mode smode;
289 : /* Vector mode. */
290 : enum machine_mode vmode;
291 :
292 : /* ID of a chain. */
293 : unsigned int chain_id;
294 : /* A queue of instructions to be included into a chain. */
295 : bitmap queue;
296 : /* Instructions included into a chain. */
297 : bitmap insns;
298 : /* All registers defined by a chain. */
299 : bitmap defs;
300 : /* Registers used in both vector and scalar modes. */
301 : bitmap defs_conv;
302 :
303 : /* Limit on chain discovery. */
304 : unsigned max_visits;
305 :
306 : bitmap insns_conv;
307 : hash_map<rtx, rtx> defs_map;
308 : /* Cost of inserted conversion between ineteger and sse. */
309 : int cost_sse_integer;
310 : sreal weighted_cost_sse_integer;
311 : auto_vec<rtx_insn *> control_flow_insns;
312 :
313 : bool build (bitmap candidates, unsigned insn_uid, bitmap disallowed);
314 : virtual bool compute_convert_gain () = 0;
315 : int convert ();
316 :
317 : protected:
318 : void add_to_queue (unsigned insn_uid);
319 : void emit_conversion_insns (rtx_insn *insns, rtx_insn *pos);
320 : rtx convert_compare (rtx op1, rtx op2, rtx_insn *insn);
321 : void mark_dual_mode_def (df_ref def);
322 : void convert_reg (rtx_insn *insn, rtx dst, rtx src);
323 : void convert_insn_common (rtx_insn *insn);
324 : void make_vector_copies (rtx_insn *, rtx);
325 : void convert_registers ();
326 : void convert_op (rtx *op, rtx_insn *insn);
327 :
328 : private:
329 : bool add_insn (bitmap candidates, unsigned insn_uid, bitmap disallowed);
330 : bool analyze_register_chain (bitmap candidates, df_ref ref,
331 : bitmap disallowed);
332 : virtual void convert_insn (rtx_insn *insn) = 0;
333 : };
334 :
335 : class general_scalar_chain : public scalar_chain
336 : {
337 : public:
338 6088507 : general_scalar_chain (enum machine_mode smode_, enum machine_mode vmode_)
339 12177014 : : scalar_chain (smode_, vmode_) {}
340 : bool compute_convert_gain () final override;
341 :
342 : private:
343 : void convert_insn (rtx_insn *insn) final override;
344 : int vector_const_cost (rtx exp, basic_block bb);
345 : rtx convert_rotate (enum rtx_code, rtx op0, rtx op1, rtx_insn *insn);
346 : };
347 :
348 : class timode_scalar_chain : public scalar_chain
349 : {
350 : public:
351 1062312 : timode_scalar_chain () : scalar_chain (TImode, V1TImode) {}
352 : bool compute_convert_gain () final override;
353 :
354 : private:
355 : void fix_debug_reg_uses (rtx reg);
356 : void convert_insn (rtx_insn *insn) final override;
357 : };
358 :
359 : } // anon namespace
360 :
361 : unsigned scalar_chain::max_id = 0;
362 :
363 : namespace {
364 :
365 : /* Initialize new chain. */
366 :
367 6619663 : scalar_chain::scalar_chain (enum machine_mode smode_, enum machine_mode vmode_)
368 : {
369 6619663 : smode = smode_;
370 6619663 : vmode = vmode_;
371 :
372 6619663 : chain_id = ++max_id;
373 :
374 6619663 : if (dump_file)
375 139 : fprintf (dump_file, "Created a new instruction chain #%d\n", chain_id);
376 :
377 6619663 : bitmap_obstack_initialize (NULL);
378 6619663 : insns = BITMAP_ALLOC (NULL);
379 6619663 : defs = BITMAP_ALLOC (NULL);
380 6619663 : defs_conv = BITMAP_ALLOC (NULL);
381 6619663 : insns_conv = BITMAP_ALLOC (NULL);
382 6619663 : queue = NULL;
383 :
384 6619663 : cost_sse_integer = 0;
385 6619663 : weighted_cost_sse_integer = 0 ;
386 6619663 : max_visits = x86_stv_max_visits;
387 6619663 : }
388 :
389 : /* Free chain's data. */
390 :
391 6619663 : scalar_chain::~scalar_chain ()
392 : {
393 6619663 : BITMAP_FREE (insns);
394 6619663 : BITMAP_FREE (defs);
395 6619663 : BITMAP_FREE (defs_conv);
396 6619663 : BITMAP_FREE (insns_conv);
397 6619663 : bitmap_obstack_release (NULL);
398 6619663 : }
399 :
400 : /* Add instruction into chains' queue. */
401 :
402 : void
403 8406250 : scalar_chain::add_to_queue (unsigned insn_uid)
404 : {
405 8406250 : if (!bitmap_set_bit (queue, insn_uid))
406 : return;
407 :
408 6444536 : if (dump_file)
409 147 : fprintf (dump_file, " Adding insn %d into chain's #%d queue\n",
410 : insn_uid, chain_id);
411 : }
412 :
413 : /* For DImode conversion, mark register defined by DEF as requiring
414 : conversion. */
415 :
416 : void
417 9742832 : scalar_chain::mark_dual_mode_def (df_ref def)
418 : {
419 9742832 : gcc_assert (DF_REF_REG_DEF_P (def));
420 :
421 : /* Record the def/insn pair so we can later efficiently iterate over
422 : the defs to convert on insns not in the chain. */
423 9742832 : bool reg_new = bitmap_set_bit (defs_conv, DF_REF_REGNO (def));
424 9742832 : basic_block bb = BLOCK_FOR_INSN (DF_REF_INSN (def));
425 9742832 : profile_count entry_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
426 9742832 : bool speed_p = optimize_bb_for_speed_p (bb);
427 9742832 : int cost = 0;
428 :
429 9742832 : if (!bitmap_bit_p (insns, DF_REF_INSN_UID (def)))
430 : {
431 2957075 : if (!bitmap_set_bit (insns_conv, DF_REF_INSN_UID (def))
432 2957075 : && !reg_new)
433 1512282 : return;
434 :
435 : /* Cost integer to sse moves. */
436 2599006 : if (speed_p)
437 2290141 : cost = COSTS_N_INSNS (ix86_cost->integer_to_sse) / 2;
438 308865 : else if (TARGET_64BIT || smode == SImode)
439 : cost = COSTS_N_BYTES (4);
440 : /* vmovd (4 bytes) + vpinsrd (6 bytes). */
441 18767 : else if (TARGET_SSE4_1)
442 : cost = COSTS_N_BYTES (10);
443 : /* movd (4 bytes) + movd (4 bytes) + unpckldq (4 bytes). */
444 : else
445 8230550 : cost = COSTS_N_BYTES (12);
446 : }
447 : else
448 : {
449 6785757 : if (!reg_new)
450 : return;
451 :
452 : /* Cost sse to integer moves. */
453 5631544 : if (speed_p)
454 5042501 : cost = COSTS_N_INSNS (ix86_cost->sse_to_integer) / 2;
455 589043 : else if (TARGET_64BIT || smode == SImode)
456 : cost = COSTS_N_BYTES (4);
457 : /* vmovd (4 bytes) + vpextrd (6 bytes). */
458 2912 : else if (TARGET_SSE4_1)
459 : cost = COSTS_N_BYTES (10);
460 : /* movd (4 bytes) + psrlq (5 bytes) + movd (4 bytes). */
461 : else
462 8230550 : cost = COSTS_N_BYTES (13);
463 : }
464 :
465 8230550 : if (speed_p)
466 7332642 : weighted_cost_sse_integer += bb->count.to_sreal_scale (entry_count) * cost;
467 :
468 8230550 : cost_sse_integer += cost;
469 :
470 8230550 : if (dump_file)
471 236 : fprintf (dump_file,
472 : " Mark r%d def in insn %d as requiring both modes in chain #%d\n",
473 236 : DF_REF_REGNO (def), DF_REF_INSN_UID (def), chain_id);
474 : }
475 :
476 : /* Check REF's chain to add new insns into a queue
477 : and find registers requiring conversion. Return true if OK, false
478 : if the analysis was aborted. */
479 :
480 : bool
481 18424624 : scalar_chain::analyze_register_chain (bitmap candidates, df_ref ref,
482 : bitmap disallowed)
483 : {
484 18424624 : df_link *chain;
485 18424624 : bool mark_def = false;
486 :
487 18424624 : gcc_checking_assert (bitmap_bit_p (insns, DF_REF_INSN_UID (ref)));
488 :
489 62922364 : for (chain = DF_REF_CHAIN (ref); chain; chain = chain->next)
490 : {
491 44499579 : unsigned uid = DF_REF_INSN_UID (chain->ref);
492 :
493 44499579 : if (!NONDEBUG_INSN_P (DF_REF_INSN (chain->ref)))
494 7706238 : continue;
495 :
496 36793341 : if (--max_visits == 0)
497 : return false;
498 :
499 36792820 : if (!DF_REF_REG_MEM_P (chain->ref))
500 : {
501 30848327 : if (bitmap_bit_p (insns, uid))
502 9820892 : continue;
503 :
504 21027435 : if (bitmap_bit_p (candidates, uid))
505 : {
506 8406250 : add_to_queue (uid);
507 8406250 : continue;
508 : }
509 :
510 : /* If we run into parts of an aborted chain discovery abort. */
511 12621185 : if (bitmap_bit_p (disallowed, uid))
512 : return false;
513 : }
514 :
515 18564535 : if (DF_REF_REG_DEF_P (chain->ref))
516 : {
517 : /* An asm goto with outputs defines them on all outgoing edges,
518 : so a conversion copy can't be emitted after it. */
519 2957250 : if (JUMP_P (DF_REF_INSN (chain->ref)))
520 : {
521 175 : if (dump_file)
522 1 : fprintf (dump_file,
523 : " r%d def in jump insn %d, aborting chain\n",
524 : DF_REF_REGNO (chain->ref), uid);
525 : return false;
526 : }
527 :
528 2957075 : if (dump_file)
529 125 : fprintf (dump_file, " r%d def in insn %d isn't convertible\n",
530 : DF_REF_REGNO (chain->ref), uid);
531 2957075 : mark_dual_mode_def (chain->ref);
532 : }
533 : else
534 : {
535 15607285 : if (dump_file)
536 524 : fprintf (dump_file, " r%d use in insn %d isn't convertible\n",
537 : DF_REF_REGNO (chain->ref), uid);
538 : mark_def = true;
539 : }
540 : }
541 :
542 18422785 : if (mark_def)
543 6785757 : mark_dual_mode_def (ref);
544 :
545 : return true;
546 : }
547 :
548 : /* Check whether X is a convertible *concatditi_? variant. X is known
549 : to be any_or_plus:TI, i.e. PLUS:TI, IOR:TI or XOR:TI. */
550 :
551 : static bool
552 32880 : timode_concatdi_p (rtx x)
553 : {
554 32880 : rtx op0 = XEXP (x, 0);
555 32880 : rtx op1 = XEXP (x, 1);
556 :
557 32880 : if (GET_CODE (op1) == ASHIFT)
558 959 : std::swap (op0, op1);
559 :
560 32880 : return GET_CODE (op0) == ASHIFT
561 21955 : && GET_CODE (XEXP (op0, 0)) == ZERO_EXTEND
562 21955 : && GET_MODE (XEXP (XEXP (op0, 0), 0)) == DImode
563 21955 : && REG_P (XEXP (XEXP (op0, 0), 0))
564 21804 : && CONST_INT_P (XEXP (op0, 1))
565 21804 : && INTVAL (XEXP (op0, 1)) == 64
566 21804 : && GET_CODE (op1) == ZERO_EXTEND
567 20845 : && GET_MODE (XEXP (op1, 0)) == DImode
568 53725 : && REG_P (XEXP (op1, 0));
569 : }
570 :
571 :
572 : /* Add instruction into a chain. Return true if OK, false if the search
573 : was aborted. */
574 :
575 : bool
576 13062319 : scalar_chain::add_insn (bitmap candidates, unsigned int insn_uid,
577 : bitmap disallowed)
578 : {
579 13062319 : if (!bitmap_set_bit (insns, insn_uid))
580 : return true;
581 :
582 13062319 : if (dump_file)
583 284 : fprintf (dump_file, " Adding insn %d to chain #%d\n", insn_uid, chain_id);
584 :
585 13062319 : rtx_insn *insn = DF_INSN_UID_GET (insn_uid)->insn;
586 13062319 : rtx def_set = single_set (insn);
587 13062319 : if (def_set && REG_P (SET_DEST (def_set))
588 23102924 : && !HARD_REGISTER_P (SET_DEST (def_set)))
589 10015110 : bitmap_set_bit (defs, REGNO (SET_DEST (def_set)));
590 :
591 : /* ??? The following is quadratic since analyze_register_chain
592 : iterates over all refs to look for dual-mode regs. Instead this
593 : should be done separately for all regs mentioned in the chain once. */
594 13062319 : df_ref ref;
595 26661095 : for (ref = DF_INSN_UID_DEFS (insn_uid); ref; ref = DF_REF_NEXT_LOC (ref))
596 13599567 : if (!HARD_REGISTER_P (DF_REF_REG (ref)))
597 10015110 : if (!analyze_register_chain (candidates, ref, disallowed))
598 : return false;
599 :
600 : /* The operand(s) of VEC_SELECT, ZERO_EXTEND and similar ops don't need
601 : to be converted/convertible. */
602 13061528 : if (def_set)
603 13061528 : switch (GET_CODE (SET_SRC (def_set)))
604 : {
605 3920920 : case REG:
606 3920920 : if (HARD_REGISTER_P (SET_SRC (def_set)))
607 : return true;
608 : break;
609 : case VEC_SELECT:
610 : return true;
611 10349 : case ZERO_EXTEND:
612 10349 : if (GET_MODE (XEXP (SET_SRC (def_set), 0)) == DImode)
613 : return true;
614 : break;
615 2435022 : case PLUS:
616 2435022 : case IOR:
617 2435022 : case XOR:
618 2435022 : if (smode == TImode && timode_concatdi_p (SET_SRC (def_set)))
619 : return true;
620 : break;
621 : default:
622 : break;
623 : }
624 :
625 28614249 : for (ref = DF_INSN_UID_USES (insn_uid); ref; ref = DF_REF_NEXT_LOC (ref))
626 15607279 : if (DF_REF_TYPE (ref) == DF_REF_REG_USE
627 8409517 : && !SUBREG_P (DF_REF_REG (ref)))
628 8409514 : if (!analyze_register_chain (candidates, ref, disallowed))
629 : return false;
630 :
631 : return true;
632 : }
633 :
634 : /* Build new chain starting from insn INSN_UID recursively
635 : adding all dependent uses and definitions. Return true if OK, false
636 : if the chain discovery was aborted. */
637 :
638 : bool
639 6619663 : scalar_chain::build (bitmap candidates, unsigned insn_uid, bitmap disallowed)
640 : {
641 6619663 : queue = BITMAP_ALLOC (NULL);
642 6619663 : bitmap_set_bit (queue, insn_uid);
643 :
644 6619663 : if (dump_file)
645 139 : fprintf (dump_file, "Building chain #%d...\n", chain_id);
646 :
647 19680143 : while (!bitmap_empty_p (queue))
648 : {
649 13062319 : insn_uid = bitmap_first_set_bit (queue);
650 13062319 : bitmap_clear_bit (queue, insn_uid);
651 13062319 : bitmap_clear_bit (candidates, insn_uid);
652 13062319 : if (!add_insn (candidates, insn_uid, disallowed))
653 : {
654 : /* If we aborted the search put sofar found insn on the set of
655 : disallowed insns so that further searches reaching them also
656 : abort and thus we abort the whole but yet undiscovered chain. */
657 1839 : bitmap_ior_into (disallowed, insns);
658 1839 : if (dump_file)
659 3 : fprintf (dump_file, "Aborted chain #%d discovery\n", chain_id);
660 1839 : BITMAP_FREE (queue);
661 1839 : return false;
662 : }
663 : }
664 :
665 6617824 : if (dump_file)
666 : {
667 136 : fprintf (dump_file, "Collected chain #%d...\n", chain_id);
668 136 : fprintf (dump_file, " insns: ");
669 136 : dump_bitmap (dump_file, insns);
670 136 : if (!bitmap_empty_p (defs_conv))
671 : {
672 136 : bitmap_iterator bi;
673 136 : unsigned id;
674 136 : const char *comma = "";
675 136 : fprintf (dump_file, " defs to convert: ");
676 362 : EXECUTE_IF_SET_IN_BITMAP (defs_conv, 0, id, bi)
677 : {
678 226 : fprintf (dump_file, "%sr%d", comma, id);
679 226 : comma = ", ";
680 : }
681 136 : fprintf (dump_file, "\n");
682 : }
683 : }
684 :
685 6617824 : BITMAP_FREE (queue);
686 :
687 6617824 : return true;
688 : }
689 :
690 : /* Return a cost of building a vector constant
691 : instead of using a scalar one. */
692 :
693 : int
694 2706354 : general_scalar_chain::vector_const_cost (rtx exp, basic_block bb)
695 : {
696 2706354 : gcc_assert (CONST_INT_P (exp));
697 :
698 2706354 : if (standard_sse_constant_p (exp, vmode))
699 611583 : return ix86_cost->sse_op;
700 2094771 : if (optimize_bb_for_size_p (bb))
701 : return COSTS_N_BYTES (8);
702 : /* We have separate costs for SImode and DImode, use SImode costs
703 : for smaller modes. */
704 2471632 : return COSTS_N_INSNS (ix86_cost->sse_load[smode == DImode ? 1 : 0]) / 2;
705 : }
706 :
707 : /* Return true if it's cost profitable for chain conversion. */
708 :
709 : bool
710 6086668 : general_scalar_chain::compute_convert_gain ()
711 : {
712 6086668 : bitmap_iterator bi;
713 6086668 : unsigned insn_uid;
714 6086668 : int gain = 0;
715 6086668 : sreal weighted_gain = 0;
716 :
717 6086668 : if (dump_file)
718 136 : fprintf (dump_file, "Computing gain for chain #%d...\n", chain_id);
719 :
720 : /* SSE costs distinguish between SImode and DImode loads/stores, for
721 : int costs factor in the number of GPRs involved. When supporting
722 : smaller modes than SImode the int load/store costs need to be
723 : adjusted as well. */
724 6086668 : unsigned sse_cost_idx = smode == DImode ? 1 : 0;
725 6086668 : int m = smode == DImode ? (TARGET_64BIT ? 1 : 2) : 1;
726 :
727 18083138 : EXECUTE_IF_SET_IN_BITMAP (insns, 0, insn_uid, bi)
728 : {
729 11996470 : rtx_insn *insn = DF_INSN_UID_GET (insn_uid)->insn;
730 11996470 : rtx def_set = single_set (insn);
731 11996470 : rtx src = SET_SRC (def_set);
732 11996470 : rtx dst = SET_DEST (def_set);
733 11996470 : basic_block bb = BLOCK_FOR_INSN (insn);
734 11996470 : int igain = 0;
735 11996470 : profile_count entry_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
736 11996470 : bool speed_p = optimize_bb_for_speed_p (bb);
737 11996470 : sreal bb_freq = bb->count.to_sreal_scale (entry_count);
738 :
739 11996470 : if (REG_P (src) && REG_P (dst))
740 : {
741 964941 : if (!speed_p)
742 : /* reg-reg move is 2 bytes, while SSE 3. */
743 210937 : igain += COSTS_N_BYTES (2 * m - 3);
744 : else
745 : /* Move costs are normalized to reg-reg move having cost 2. */
746 754004 : igain += COSTS_N_INSNS (2 * m - ix86_cost->xmm_move) / 2;
747 : }
748 11031529 : else if (REG_P (src) && MEM_P (dst))
749 : {
750 2406781 : if (!speed_p)
751 : /* Integer load/store is 3+ bytes and SSE 4+. */
752 203013 : igain += COSTS_N_BYTES (3 * m - 4);
753 : else
754 2203768 : igain
755 2203768 : += COSTS_N_INSNS (m * ix86_cost->int_store[2]
756 : - ix86_cost->sse_store[sse_cost_idx]) / 2;
757 : }
758 8624748 : else if (MEM_P (src) && REG_P (dst))
759 : {
760 3872044 : if (!speed_p)
761 365627 : igain += COSTS_N_BYTES (3 * m - 4);
762 : else
763 3506417 : igain += COSTS_N_INSNS (m * ix86_cost->int_load[2]
764 : - ix86_cost->sse_load[sse_cost_idx]) / 2;
765 : }
766 : else
767 : {
768 : /* For operations on memory operands, include the overhead
769 : of explicit load and store instructions. */
770 4752704 : if (MEM_P (dst))
771 : {
772 73276 : if (!speed_p)
773 : /* ??? This probably should account size difference
774 : of SSE and integer load rather than full SSE load. */
775 : igain -= COSTS_N_BYTES (8);
776 : else
777 : {
778 63297 : int cost = (m * (ix86_cost->int_load[2]
779 63297 : + ix86_cost->int_store[2])
780 63297 : - (ix86_cost->sse_load[sse_cost_idx] +
781 63297 : ix86_cost->sse_store[sse_cost_idx]));
782 63297 : igain += COSTS_N_INSNS (cost) / 2;
783 : }
784 : }
785 :
786 4752704 : switch (GET_CODE (src))
787 : {
788 500580 : case ASHIFT:
789 500580 : case ASHIFTRT:
790 500580 : case LSHIFTRT:
791 500580 : if (m == 2)
792 : {
793 17906 : if (INTVAL (XEXP (src, 1)) >= 32)
794 12306 : igain += ix86_cost->add;
795 : /* Gain for extend highpart case. */
796 5600 : else if (GET_CODE (XEXP (src, 0)) == ASHIFT)
797 0 : igain += ix86_cost->shift_const - ix86_cost->sse_op;
798 : else
799 5600 : igain += ix86_cost->shift_const;
800 : }
801 :
802 500580 : igain += ix86_cost->shift_const - ix86_cost->sse_op;
803 :
804 500580 : if (CONST_INT_P (XEXP (src, 0)))
805 0 : igain -= vector_const_cost (XEXP (src, 0), bb);
806 : break;
807 :
808 3942 : case ROTATE:
809 3942 : case ROTATERT:
810 3942 : igain += m * ix86_cost->shift_const;
811 3942 : if (TARGET_AVX512VL)
812 204 : igain -= ix86_cost->sse_op;
813 3738 : else if (smode == DImode)
814 : {
815 798 : int bits = INTVAL (XEXP (src, 1));
816 798 : if ((bits & 0x0f) == 0)
817 193 : igain -= ix86_cost->sse_op;
818 605 : else if ((bits & 0x07) == 0)
819 31 : igain -= 2 * ix86_cost->sse_op;
820 : else
821 574 : igain -= 3 * ix86_cost->sse_op;
822 : }
823 2940 : else if (INTVAL (XEXP (src, 1)) == 16)
824 139 : igain -= ix86_cost->sse_op;
825 : else
826 2801 : igain -= 2 * ix86_cost->sse_op;
827 : break;
828 :
829 2935191 : case AND:
830 2935191 : case IOR:
831 2935191 : case XOR:
832 2935191 : case PLUS:
833 2935191 : case MINUS:
834 2935191 : igain += m * ix86_cost->add - ix86_cost->sse_op;
835 : /* Additional gain for andnot for targets without BMI. */
836 2935191 : if (GET_CODE (XEXP (src, 0)) == NOT
837 3145 : && !TARGET_BMI)
838 3136 : igain += m * ix86_cost->add;
839 :
840 2935191 : if (CONST_INT_P (XEXP (src, 0)))
841 0 : igain -= vector_const_cost (XEXP (src, 0), bb);
842 2935191 : if (CONST_INT_P (XEXP (src, 1)))
843 1729881 : igain -= vector_const_cost (XEXP (src, 1), bb);
844 2935191 : if (MEM_P (XEXP (src, 1)))
845 : {
846 103098 : if (!speed_p)
847 23970 : igain -= COSTS_N_BYTES (m == 2 ? 3 : 5);
848 : else
849 91108 : igain += COSTS_N_INSNS
850 : (m * ix86_cost->int_load[2]
851 : - ix86_cost->sse_load[sse_cost_idx]) / 2;
852 : }
853 : break;
854 :
855 59755 : case NEG:
856 59755 : case NOT:
857 59755 : igain -= ix86_cost->sse_op + COSTS_N_INSNS (1);
858 :
859 59755 : if (GET_CODE (XEXP (src, 0)) != ABS)
860 : {
861 59755 : igain += m * ix86_cost->add;
862 59755 : break;
863 : }
864 : /* FALLTHRU */
865 :
866 968 : case ABS:
867 968 : case SMAX:
868 968 : case SMIN:
869 968 : case UMAX:
870 968 : case UMIN:
871 : /* We do not have any conditional move cost, estimate it as a
872 : reg-reg move. Comparisons are costed as adds. */
873 968 : igain += m * (COSTS_N_INSNS (2) + ix86_cost->add);
874 : /* Integer SSE ops are all costed the same. */
875 968 : igain -= ix86_cost->sse_op;
876 968 : break;
877 :
878 0 : case COMPARE:
879 0 : if (XEXP (src, 1) != const0_rtx)
880 : {
881 : /* cmp vs. pxor;pshufd;ptest. */
882 0 : igain += COSTS_N_INSNS (m - 3);
883 : }
884 0 : else if (GET_CODE (XEXP (src, 0)) != AND)
885 : {
886 : /* test vs. pshufd;ptest. */
887 0 : igain += COSTS_N_INSNS (m - 2);
888 : }
889 0 : else if (GET_CODE (XEXP (XEXP (src, 0), 0)) != NOT)
890 : {
891 : /* and;test vs. pshufd;ptest. */
892 0 : igain += COSTS_N_INSNS (2 * m - 2);
893 : }
894 0 : else if (TARGET_BMI)
895 : {
896 : /* andn;test vs. pandn;pshufd;ptest. */
897 0 : igain += COSTS_N_INSNS (2 * m - 3);
898 : }
899 : else
900 : {
901 : /* not;and;test vs. pandn;pshufd;ptest. */
902 0 : igain += COSTS_N_INSNS (3 * m - 3);
903 : }
904 : break;
905 :
906 1218314 : case CONST_INT:
907 1218314 : if (REG_P (dst))
908 : {
909 1218314 : if (!speed_p)
910 : {
911 : /* xor (2 bytes) vs. xorps (3 bytes). */
912 241841 : if (src == const0_rtx)
913 126012 : igain -= COSTS_N_BYTES (1);
914 : /* movdi_internal vs. movv2di_internal. */
915 : /* => mov (5 bytes) vs. movaps (7 bytes). */
916 115829 : else if (x86_64_immediate_operand (src, SImode))
917 103614 : igain -= COSTS_N_BYTES (2);
918 : else
919 : /* ??? Larger immediate constants are placed in the
920 : constant pool, where the size benefit/impact of
921 : STV conversion is affected by whether and how
922 : often each constant pool entry is shared/reused.
923 : The value below is empirically derived from the
924 : CSiBE benchmark (and the optimal value may drift
925 : over time). */
926 : igain += COSTS_N_BYTES (0);
927 : }
928 : else
929 : {
930 : /* DImode can be immediate for TARGET_64BIT
931 : and SImode always. */
932 976473 : igain += m * COSTS_N_INSNS (1);
933 976473 : igain -= vector_const_cost (src, bb);
934 : }
935 : }
936 0 : else if (MEM_P (dst))
937 : {
938 0 : igain += (m * ix86_cost->int_store[2]
939 0 : - ix86_cost->sse_store[sse_cost_idx]);
940 0 : igain -= vector_const_cost (src, bb);
941 : }
942 : break;
943 :
944 23887 : case VEC_SELECT:
945 23887 : if (XVECEXP (XEXP (src, 1), 0, 0) == const0_rtx)
946 : {
947 : // movd (4 bytes) replaced with movdqa (4 bytes).
948 18911 : if (!!speed_p)
949 17260 : igain += COSTS_N_INSNS (ix86_cost->sse_to_integer
950 : - ix86_cost->xmm_move) / 2;
951 : }
952 : else
953 : {
954 : // pshufd; movd replaced with pshufd.
955 4976 : if (!speed_p)
956 536 : igain += COSTS_N_BYTES (4);
957 : else
958 4440 : igain += ix86_cost->sse_to_integer;
959 : }
960 : break;
961 :
962 10067 : case ZERO_EXTEND:
963 : /* mov eax (6 bytes) vs movd xmm0 (8 bytes). */
964 : /* mov eax; xor edx,edx (7 bytes). */
965 10067 : if (speed_p)
966 8895 : igain += COSTS_N_INSNS (ix86_cost->int_load[2]
967 : - ix86_cost->sse_load[0]) / 2;
968 : else
969 1218 : igain += COSTS_N_BYTES (TARGET_64BIT ? -2 : -1);
970 : break;
971 :
972 0 : default:
973 0 : gcc_unreachable ();
974 : }
975 : }
976 :
977 11994819 : if (speed_p)
978 10660108 : weighted_gain += bb_freq * igain;
979 11996470 : gain += igain;
980 :
981 11996470 : if (igain != 0 && dump_file)
982 : {
983 93 : fprintf (dump_file, " Instruction gain %d with bb_freq %.2f for",
984 : igain, bb_freq.to_double ());
985 93 : dump_insn_slim (dump_file, insn);
986 : }
987 : }
988 :
989 6086668 : if (dump_file)
990 : {
991 136 : fprintf (dump_file, " Instruction conversion gain: %d, \n",
992 : gain);
993 136 : fprintf (dump_file, " Registers conversion cost: %d\n",
994 : cost_sse_integer);
995 136 : fprintf (dump_file, " Weighted instruction conversion gain: %.2f, \n",
996 : weighted_gain.to_double ());
997 136 : fprintf (dump_file, " Weighted registers conversion cost: %.2f\n",
998 : weighted_cost_sse_integer.to_double ());
999 : }
1000 :
1001 6086668 : if (weighted_gain != weighted_cost_sse_integer)
1002 4901491 : return weighted_gain > weighted_cost_sse_integer;
1003 : else
1004 1185177 : return gain > cost_sse_integer;;
1005 : }
1006 :
1007 : /* Insert generated conversion instruction sequence INSNS
1008 : after instruction AFTER. New BB may be required in case
1009 : instruction has EH region attached. */
1010 :
1011 : void
1012 32995 : scalar_chain::emit_conversion_insns (rtx_insn *insns, rtx_insn *after)
1013 : {
1014 32995 : if (!control_flow_insn_p (after))
1015 : {
1016 32787 : emit_insn_after (insns, after);
1017 32787 : return;
1018 : }
1019 :
1020 208 : basic_block bb = BLOCK_FOR_INSN (after);
1021 208 : edge e = find_fallthru_edge (bb->succs);
1022 208 : gcc_assert (e);
1023 :
1024 208 : basic_block new_bb = split_edge (e);
1025 208 : emit_insn_after (insns, BB_HEAD (new_bb));
1026 : }
1027 :
1028 : } // anon namespace
1029 :
1030 : /* Generate the canonical SET_SRC to move GPR to a VMODE vector register,
1031 : zeroing the upper parts. */
1032 :
1033 : static rtx
1034 172732 : gen_gpr_to_xmm_move_src (enum machine_mode vmode, rtx gpr)
1035 : {
1036 345464 : switch (GET_MODE_NUNITS (vmode))
1037 : {
1038 71 : case 1:
1039 71 : return gen_rtx_SUBREG (vmode, gpr, 0);
1040 172086 : case 2:
1041 344172 : return gen_rtx_VEC_CONCAT (vmode, gpr,
1042 : CONST0_RTX (GET_MODE_INNER (vmode)));
1043 575 : default:
1044 575 : return gen_rtx_VEC_MERGE (vmode, gen_rtx_VEC_DUPLICATE (vmode, gpr),
1045 : CONST0_RTX (vmode), GEN_INT (HOST_WIDE_INT_1U));
1046 : }
1047 : }
1048 :
1049 : /* Make vector copies for all register REGNO definitions
1050 : and replace its uses in a chain. */
1051 :
1052 : void
1053 8697 : scalar_chain::make_vector_copies (rtx_insn *insn, rtx reg)
1054 : {
1055 8697 : rtx vreg = *defs_map.get (reg);
1056 :
1057 8697 : start_sequence ();
1058 8697 : if (!TARGET_INTER_UNIT_MOVES_TO_VEC)
1059 : {
1060 0 : rtx tmp = assign_386_stack_local (smode, SLOT_STV_TEMP);
1061 0 : if (smode == DImode && !TARGET_64BIT)
1062 : {
1063 0 : emit_move_insn (adjust_address (tmp, SImode, 0),
1064 : gen_rtx_SUBREG (SImode, reg, 0));
1065 0 : emit_move_insn (adjust_address (tmp, SImode, 4),
1066 : gen_rtx_SUBREG (SImode, reg, 4));
1067 : }
1068 : else
1069 0 : emit_move_insn (copy_rtx (tmp), reg);
1070 0 : emit_insn (gen_rtx_SET (gen_rtx_SUBREG (vmode, vreg, 0),
1071 : gen_gpr_to_xmm_move_src (vmode, tmp)));
1072 : }
1073 8697 : else if (!TARGET_64BIT && smode == DImode)
1074 : {
1075 8636 : if (TARGET_SSE4_1)
1076 : {
1077 280 : emit_insn (gen_sse2_loadld (gen_rtx_SUBREG (V4SImode, vreg, 0),
1078 : CONST0_RTX (V4SImode),
1079 : gen_rtx_SUBREG (SImode, reg, 0)));
1080 280 : emit_insn (gen_sse4_1_pinsrd (gen_rtx_SUBREG (V4SImode, vreg, 0),
1081 : gen_rtx_SUBREG (V4SImode, vreg, 0),
1082 : gen_rtx_SUBREG (SImode, reg, 4),
1083 : GEN_INT (2)));
1084 : }
1085 : else
1086 : {
1087 8356 : rtx tmp = gen_reg_rtx (DImode);
1088 8356 : emit_insn (gen_sse2_loadld (gen_rtx_SUBREG (V4SImode, vreg, 0),
1089 : CONST0_RTX (V4SImode),
1090 : gen_rtx_SUBREG (SImode, reg, 0)));
1091 8356 : emit_insn (gen_sse2_loadld (gen_rtx_SUBREG (V4SImode, tmp, 0),
1092 : CONST0_RTX (V4SImode),
1093 : gen_rtx_SUBREG (SImode, reg, 4)));
1094 8356 : emit_insn (gen_vec_interleave_lowv4si
1095 : (gen_rtx_SUBREG (V4SImode, vreg, 0),
1096 : gen_rtx_SUBREG (V4SImode, vreg, 0),
1097 : gen_rtx_SUBREG (V4SImode, tmp, 0)));
1098 : }
1099 : }
1100 : else
1101 61 : emit_insn (gen_rtx_SET (gen_rtx_SUBREG (vmode, vreg, 0),
1102 : gen_gpr_to_xmm_move_src (vmode, reg)));
1103 8697 : rtx_insn *seq = end_sequence ();
1104 8697 : emit_conversion_insns (seq, insn);
1105 :
1106 8697 : if (dump_file)
1107 0 : fprintf (dump_file,
1108 : " Copied r%d to a vector register r%d for insn %d\n",
1109 0 : REGNO (reg), REGNO (vreg), INSN_UID (insn));
1110 8697 : }
1111 :
1112 : /* Copy the definition SRC of INSN inside the chain to DST for
1113 : scalar uses outside of the chain. */
1114 :
1115 : void
1116 23556 : scalar_chain::convert_reg (rtx_insn *insn, rtx dst, rtx src)
1117 : {
1118 23556 : start_sequence ();
1119 23556 : if (!TARGET_INTER_UNIT_MOVES_FROM_VEC)
1120 : {
1121 0 : rtx tmp = assign_386_stack_local (smode, SLOT_STV_TEMP);
1122 0 : emit_move_insn (tmp, src);
1123 0 : if (!TARGET_64BIT && smode == DImode)
1124 : {
1125 0 : emit_move_insn (gen_rtx_SUBREG (SImode, dst, 0),
1126 : adjust_address (tmp, SImode, 0));
1127 0 : emit_move_insn (gen_rtx_SUBREG (SImode, dst, 4),
1128 : adjust_address (tmp, SImode, 4));
1129 : }
1130 : else
1131 0 : emit_move_insn (dst, copy_rtx (tmp));
1132 : }
1133 23556 : else if (!TARGET_64BIT && smode == DImode)
1134 : {
1135 22531 : if (TARGET_SSE4_1)
1136 : {
1137 0 : rtx tmp = gen_rtx_PARALLEL (VOIDmode,
1138 : gen_rtvec (1, const0_rtx));
1139 0 : emit_insn
1140 0 : (gen_rtx_SET
1141 : (gen_rtx_SUBREG (SImode, dst, 0),
1142 : gen_rtx_VEC_SELECT (SImode,
1143 : gen_rtx_SUBREG (V4SImode, src, 0),
1144 : tmp)));
1145 :
1146 0 : tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const1_rtx));
1147 0 : emit_insn
1148 0 : (gen_rtx_SET
1149 : (gen_rtx_SUBREG (SImode, dst, 4),
1150 : gen_rtx_VEC_SELECT (SImode,
1151 : gen_rtx_SUBREG (V4SImode, src, 0),
1152 : tmp)));
1153 : }
1154 : else
1155 : {
1156 22531 : rtx vcopy = gen_reg_rtx (V2DImode);
1157 22531 : emit_move_insn (vcopy, gen_rtx_SUBREG (V2DImode, src, 0));
1158 22531 : emit_move_insn (gen_rtx_SUBREG (SImode, dst, 0),
1159 : gen_rtx_SUBREG (SImode, vcopy, 0));
1160 22531 : emit_move_insn (vcopy,
1161 : gen_rtx_LSHIFTRT (V2DImode,
1162 : vcopy, GEN_INT (32)));
1163 22531 : emit_move_insn (gen_rtx_SUBREG (SImode, dst, 4),
1164 : gen_rtx_SUBREG (SImode, vcopy, 0));
1165 : }
1166 : }
1167 : else
1168 1025 : emit_move_insn (dst, src);
1169 :
1170 23556 : rtx_insn *seq = end_sequence ();
1171 23556 : emit_conversion_insns (seq, insn);
1172 :
1173 23556 : if (dump_file)
1174 0 : fprintf (dump_file,
1175 : " Copied r%d to a scalar register r%d for insn %d\n",
1176 0 : REGNO (src), REGNO (dst), INSN_UID (insn));
1177 23556 : }
1178 :
1179 : /* Helper function to convert immediate constant X to vmode. */
1180 : static rtx
1181 38020 : smode_convert_cst (rtx x, enum machine_mode vmode)
1182 : {
1183 : /* Prefer all ones vector in case of -1. */
1184 38020 : if (constm1_operand (x, GET_MODE (x)))
1185 625 : return CONSTM1_RTX (vmode);
1186 :
1187 37395 : unsigned n = GET_MODE_NUNITS (vmode);
1188 37395 : rtx *v = XALLOCAVEC (rtx, n);
1189 37395 : v[0] = x;
1190 43432 : for (unsigned i = 1; i < n; ++i)
1191 6037 : v[i] = const0_rtx;
1192 37395 : return gen_rtx_CONST_VECTOR (vmode, gen_rtvec_v (n, v));
1193 : }
1194 :
1195 : /* Convert operand OP in INSN. We should handle
1196 : memory operands and uninitialized registers.
1197 : All other register uses are converted during
1198 : registers conversion. */
1199 :
1200 : void
1201 239534 : scalar_chain::convert_op (rtx *op, rtx_insn *insn)
1202 : {
1203 239534 : rtx tmp;
1204 :
1205 239534 : if (GET_MODE (*op) == V1TImode)
1206 : return;
1207 :
1208 239243 : *op = copy_rtx_if_shared (*op);
1209 :
1210 239243 : if (GET_CODE (*op) == NOT
1211 239243 : || GET_CODE (*op) == ASHIFT)
1212 : {
1213 2892 : convert_op (&XEXP (*op, 0), insn);
1214 2892 : PUT_MODE (*op, vmode);
1215 : }
1216 : else if (MEM_P (*op))
1217 : {
1218 172671 : rtx_insn *movabs = NULL;
1219 :
1220 : /* Emit MOVABS to load from a 64-bit absolute address to a GPR. */
1221 172671 : if (!memory_operand (*op, GET_MODE (*op)))
1222 : {
1223 0 : tmp = gen_reg_rtx (GET_MODE (*op));
1224 0 : movabs = emit_insn_before (gen_rtx_SET (tmp, *op), insn);
1225 :
1226 0 : *op = tmp;
1227 : }
1228 :
1229 172671 : tmp = gen_rtx_SUBREG (vmode, gen_reg_rtx (GET_MODE (*op)), 0);
1230 :
1231 172671 : rtx_insn *eh_insn
1232 172671 : = emit_insn_before (gen_rtx_SET (copy_rtx (tmp),
1233 : gen_gpr_to_xmm_move_src (vmode, *op)),
1234 172671 : insn);
1235 :
1236 172671 : if (cfun->can_throw_non_call_exceptions)
1237 : {
1238 : /* Handle REG_EH_REGION note. */
1239 168367 : rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
1240 168367 : if (note)
1241 : {
1242 3578 : if (movabs)
1243 0 : eh_insn = movabs;
1244 3578 : control_flow_insns.safe_push (eh_insn);
1245 3578 : add_reg_note (eh_insn, REG_EH_REGION, XEXP (note, 0));
1246 : }
1247 : }
1248 :
1249 172671 : *op = tmp;
1250 :
1251 172671 : if (dump_file)
1252 0 : fprintf (dump_file, " Preloading operand for insn %d into r%d\n",
1253 0 : INSN_UID (insn), reg_or_subregno (tmp));
1254 : }
1255 : else if (REG_P (*op))
1256 57156 : *op = gen_rtx_SUBREG (vmode, *op, 0);
1257 : else if (CONST_SCALAR_INT_P (*op))
1258 : {
1259 6521 : rtx vec_cst = smode_convert_cst (*op, vmode);
1260 :
1261 6521 : if (!standard_sse_constant_p (vec_cst, vmode))
1262 : {
1263 2711 : start_sequence ();
1264 2711 : vec_cst = validize_mem (force_const_mem (vmode, vec_cst));
1265 2711 : rtx_insn *seq = end_sequence ();
1266 2711 : emit_insn_before (seq, insn);
1267 : }
1268 :
1269 6521 : tmp = gen_rtx_SUBREG (vmode, gen_reg_rtx (smode), 0);
1270 :
1271 6521 : emit_insn_before (gen_move_insn (copy_rtx (tmp), vec_cst), insn);
1272 6521 : *op = tmp;
1273 : }
1274 : else
1275 : {
1276 0 : gcc_assert (SUBREG_P (*op));
1277 3 : if (GET_MODE (*op) != vmode)
1278 : {
1279 3 : rtx inner = SUBREG_REG (*op);
1280 3 : poly_uint64 byte = SUBREG_BYTE (*op);
1281 3 : if (targetm.modes_tieable_p (vmode, GET_MODE (inner))
1282 3 : && validate_subreg (vmode, GET_MODE (inner), inner, byte))
1283 3 : *op = gen_lowpart (vmode, *op);
1284 : else
1285 : {
1286 0 : tmp = gen_reg_rtx (GET_MODE (*op));
1287 0 : emit_insn_before (gen_rtx_SET (tmp, *op), insn);
1288 0 : *op = gen_rtx_SUBREG (vmode, tmp, 0);
1289 : }
1290 : }
1291 : }
1292 : }
1293 :
1294 : /* Convert CCZmode COMPARE to vector mode. */
1295 :
1296 : rtx
1297 8 : scalar_chain::convert_compare (rtx op1, rtx op2, rtx_insn *insn)
1298 : {
1299 8 : rtx src, tmp;
1300 :
1301 : /* Handle any REG_EQUAL notes. */
1302 8 : tmp = find_reg_equal_equiv_note (insn);
1303 8 : if (tmp)
1304 : {
1305 1 : if (GET_CODE (XEXP (tmp, 0)) == COMPARE
1306 1 : && GET_MODE (XEXP (tmp, 0)) == CCZmode
1307 1 : && REG_P (XEXP (XEXP (tmp, 0), 0)))
1308 : {
1309 1 : rtx *op = &XEXP (XEXP (tmp, 0), 1);
1310 1 : if (CONST_SCALAR_INT_P (*op))
1311 : {
1312 1 : if (constm1_operand (*op, GET_MODE (*op)))
1313 0 : *op = CONSTM1_RTX (vmode);
1314 : else
1315 : {
1316 1 : unsigned n = GET_MODE_NUNITS (vmode);
1317 1 : rtx *v = XALLOCAVEC (rtx, n);
1318 1 : v[0] = *op;
1319 1 : for (unsigned i = 1; i < n; ++i)
1320 0 : v[i] = const0_rtx;
1321 1 : *op = gen_rtx_CONST_VECTOR (vmode, gen_rtvec_v (n, v));
1322 : }
1323 : tmp = NULL_RTX;
1324 : }
1325 0 : else if (REG_P (*op))
1326 : tmp = NULL_RTX;
1327 : }
1328 :
1329 1 : if (tmp)
1330 0 : remove_note (insn, tmp);
1331 : }
1332 :
1333 : /* Comparison against anything other than zero, requires an XOR. */
1334 8 : if (op2 != const0_rtx)
1335 : {
1336 6 : convert_op (&op1, insn);
1337 6 : convert_op (&op2, insn);
1338 : /* If both operands are MEMs, explicitly load the OP1 into TMP. */
1339 6 : if (MEM_P (op1) && MEM_P (op2))
1340 : {
1341 0 : tmp = gen_reg_rtx (vmode);
1342 0 : emit_insn_before (gen_rtx_SET (tmp, op1), insn);
1343 0 : src = tmp;
1344 : }
1345 : else
1346 : src = op1;
1347 6 : src = gen_rtx_XOR (vmode, src, op2);
1348 : }
1349 2 : else if (GET_CODE (op1) == AND
1350 0 : && GET_CODE (XEXP (op1, 0)) == NOT)
1351 : {
1352 0 : rtx op11 = XEXP (XEXP (op1, 0), 0);
1353 0 : rtx op12 = XEXP (op1, 1);
1354 0 : convert_op (&op11, insn);
1355 0 : convert_op (&op12, insn);
1356 0 : if (!REG_P (op11))
1357 : {
1358 0 : tmp = gen_reg_rtx (vmode);
1359 0 : emit_insn_before (gen_rtx_SET (tmp, op11), insn);
1360 0 : op11 = tmp;
1361 : }
1362 0 : src = gen_rtx_AND (vmode, gen_rtx_NOT (vmode, op11), op12);
1363 0 : }
1364 2 : else if (GET_CODE (op1) == AND)
1365 : {
1366 0 : rtx op11 = XEXP (op1, 0);
1367 0 : rtx op12 = XEXP (op1, 1);
1368 0 : convert_op (&op11, insn);
1369 0 : convert_op (&op12, insn);
1370 0 : if (!REG_P (op11))
1371 : {
1372 0 : tmp = gen_reg_rtx (vmode);
1373 0 : emit_insn_before (gen_rtx_SET (tmp, op11), insn);
1374 0 : op11 = tmp;
1375 : }
1376 0 : return gen_rtx_UNSPEC (CCZmode, gen_rtvec (2, op11, op12),
1377 : UNSPEC_PTEST);
1378 : }
1379 : else
1380 : {
1381 2 : convert_op (&op1, insn);
1382 2 : src = op1;
1383 : }
1384 :
1385 8 : if (!REG_P (src))
1386 : {
1387 6 : tmp = gen_reg_rtx (vmode);
1388 6 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
1389 6 : src = tmp;
1390 : }
1391 :
1392 8 : if (vmode == V2DImode)
1393 : {
1394 0 : tmp = gen_reg_rtx (vmode);
1395 0 : emit_insn_before (gen_vec_interleave_lowv2di (tmp, src, src), insn);
1396 0 : src = tmp;
1397 : }
1398 8 : else if (vmode == V4SImode)
1399 : {
1400 0 : tmp = gen_reg_rtx (vmode);
1401 0 : emit_insn_before (gen_sse2_pshufd (tmp, src, const0_rtx), insn);
1402 0 : src = tmp;
1403 : }
1404 :
1405 8 : return gen_rtx_UNSPEC (CCZmode, gen_rtvec (2, src, src), UNSPEC_PTEST);
1406 : }
1407 :
1408 : /* Helper function for converting INSN to vector mode. */
1409 :
1410 : void
1411 1373128 : scalar_chain::convert_insn_common (rtx_insn *insn)
1412 : {
1413 : /* Generate copies for out-of-chain uses of defs and adjust debug uses. */
1414 2093317 : for (df_ref ref = DF_INSN_DEFS (insn); ref; ref = DF_REF_NEXT_LOC (ref))
1415 720189 : if (bitmap_bit_p (defs_conv, DF_REF_REGNO (ref)))
1416 : {
1417 24976 : df_link *use;
1418 49087 : for (use = DF_REF_CHAIN (ref); use; use = use->next)
1419 47667 : if (NONDEBUG_INSN_P (DF_REF_INSN (use->ref))
1420 47667 : && (DF_REF_REG_MEM_P (use->ref)
1421 41719 : || !bitmap_bit_p (insns, DF_REF_INSN_UID (use->ref))))
1422 : break;
1423 24976 : if (use)
1424 23556 : convert_reg (insn, DF_REF_REG (ref),
1425 23556 : *defs_map.get (regno_reg_rtx [DF_REF_REGNO (ref)]));
1426 1420 : else if (MAY_HAVE_DEBUG_BIND_INSNS)
1427 : {
1428 : /* If we generated a scalar copy we can leave debug-insns
1429 : as-is, if not, we have to adjust them. */
1430 1284 : auto_vec<rtx_insn *, 5> to_reset_debug_insns;
1431 3920 : for (use = DF_REF_CHAIN (ref); use; use = use->next)
1432 2636 : if (DEBUG_INSN_P (DF_REF_INSN (use->ref)))
1433 : {
1434 831 : rtx_insn *debug_insn = DF_REF_INSN (use->ref);
1435 : /* If there's a reaching definition outside of the
1436 : chain we have to reset. */
1437 831 : df_link *def;
1438 2909 : for (def = DF_REF_CHAIN (use->ref); def; def = def->next)
1439 2248 : if (!bitmap_bit_p (insns, DF_REF_INSN_UID (def->ref)))
1440 : break;
1441 831 : if (def)
1442 170 : to_reset_debug_insns.safe_push (debug_insn);
1443 : else
1444 : {
1445 661 : *DF_REF_REAL_LOC (use->ref)
1446 661 : = *defs_map.get (regno_reg_rtx [DF_REF_REGNO (ref)]);
1447 661 : df_insn_rescan (debug_insn);
1448 : }
1449 : }
1450 : /* Have to do the reset outside of the DF_CHAIN walk to not
1451 : disrupt it. */
1452 2738 : while (!to_reset_debug_insns.is_empty ())
1453 : {
1454 170 : rtx_insn *debug_insn = to_reset_debug_insns.pop ();
1455 170 : INSN_VAR_LOCATION_LOC (debug_insn) = gen_rtx_UNKNOWN_VAR_LOC ();
1456 170 : df_insn_rescan_debug_internal (debug_insn);
1457 : }
1458 1284 : }
1459 : }
1460 :
1461 : /* Replace uses in this insn with the defs we use in the chain. */
1462 3421591 : for (df_ref ref = DF_INSN_USES (insn); ref; ref = DF_REF_NEXT_LOC (ref))
1463 2048463 : if (!DF_REF_REG_MEM_P (ref))
1464 731677 : if (rtx *vreg = defs_map.get (regno_reg_rtx[DF_REF_REGNO (ref)]))
1465 : {
1466 : /* Also update a corresponding REG_DEAD note. */
1467 36247 : rtx note = find_reg_note (insn, REG_DEAD, DF_REF_REG (ref));
1468 36247 : if (note)
1469 22863 : XEXP (note, 0) = *vreg;
1470 36247 : *DF_REF_REAL_LOC (ref) = *vreg;
1471 : }
1472 1373128 : }
1473 :
1474 : /* Convert INSN which is an SImode or DImode rotation by a constant
1475 : to vector mode. CODE is either ROTATE or ROTATERT with operands
1476 : OP0 and OP1. Returns the SET_SRC of the last instruction in the
1477 : resulting sequence, which is emitted before INSN. */
1478 :
1479 : rtx
1480 92 : general_scalar_chain::convert_rotate (enum rtx_code code, rtx op0, rtx op1,
1481 : rtx_insn *insn)
1482 : {
1483 92 : int bits = INTVAL (op1);
1484 92 : rtx pat, result;
1485 :
1486 92 : convert_op (&op0, insn);
1487 92 : if (bits == 0)
1488 0 : return op0;
1489 :
1490 92 : if (smode == DImode)
1491 : {
1492 92 : if (code == ROTATE)
1493 45 : bits = 64 - bits;
1494 92 : if (bits == 32)
1495 : {
1496 0 : rtx tmp1 = gen_reg_rtx (V4SImode);
1497 0 : pat = gen_sse2_pshufd (tmp1, gen_lowpart (V4SImode, op0),
1498 : GEN_INT (225));
1499 0 : emit_insn_before (pat, insn);
1500 0 : result = gen_lowpart (V2DImode, tmp1);
1501 : }
1502 92 : else if (TARGET_AVX512VL)
1503 0 : result = simplify_gen_binary (code, V2DImode, op0, op1);
1504 92 : else if (bits == 16 || bits == 48)
1505 : {
1506 0 : rtx tmp1 = gen_reg_rtx (V8HImode);
1507 0 : pat = gen_sse2_pshuflw (tmp1, gen_lowpart (V8HImode, op0),
1508 : GEN_INT (bits == 16 ? 57 : 147));
1509 0 : emit_insn_before (pat, insn);
1510 0 : result = gen_lowpart (V2DImode, tmp1);
1511 : }
1512 92 : else if ((bits & 0x07) == 0)
1513 : {
1514 0 : rtx tmp1 = gen_reg_rtx (V4SImode);
1515 0 : pat = gen_sse2_pshufd (tmp1, gen_lowpart (V4SImode, op0),
1516 : GEN_INT (68));
1517 0 : emit_insn_before (pat, insn);
1518 0 : rtx tmp2 = gen_reg_rtx (V1TImode);
1519 0 : pat = gen_sse2_lshrv1ti3 (tmp2, gen_lowpart (V1TImode, tmp1),
1520 : GEN_INT (bits));
1521 0 : emit_insn_before (pat, insn);
1522 0 : result = gen_lowpart (V2DImode, tmp2);
1523 : }
1524 : else
1525 : {
1526 92 : rtx tmp1 = gen_reg_rtx (V4SImode);
1527 92 : pat = gen_sse2_pshufd (tmp1, gen_lowpart (V4SImode, op0),
1528 : GEN_INT (20));
1529 92 : emit_insn_before (pat, insn);
1530 92 : rtx tmp2 = gen_reg_rtx (V2DImode);
1531 92 : pat = gen_lshrv2di3 (tmp2, gen_lowpart (V2DImode, tmp1),
1532 : GEN_INT (bits & 31));
1533 92 : emit_insn_before (pat, insn);
1534 92 : rtx tmp3 = gen_reg_rtx (V4SImode);
1535 139 : pat = gen_sse2_pshufd (tmp3, gen_lowpart (V4SImode, tmp2),
1536 : GEN_INT (bits > 32 ? 34 : 136));
1537 92 : emit_insn_before (pat, insn);
1538 92 : result = gen_lowpart (V2DImode, tmp3);
1539 : }
1540 : }
1541 0 : else if (bits == 16)
1542 : {
1543 0 : rtx tmp1 = gen_reg_rtx (V8HImode);
1544 0 : pat = gen_sse2_pshuflw (tmp1, gen_lowpart (V8HImode, op0), GEN_INT (225));
1545 0 : emit_insn_before (pat, insn);
1546 0 : result = gen_lowpart (V4SImode, tmp1);
1547 : }
1548 0 : else if (TARGET_AVX512VL)
1549 0 : result = simplify_gen_binary (code, V4SImode, op0, op1);
1550 : else
1551 : {
1552 0 : if (code == ROTATE)
1553 0 : bits = 32 - bits;
1554 :
1555 0 : rtx tmp1 = gen_reg_rtx (V4SImode);
1556 0 : emit_insn_before (gen_sse2_pshufd (tmp1, op0, GEN_INT (224)), insn);
1557 0 : rtx tmp2 = gen_reg_rtx (V2DImode);
1558 0 : pat = gen_lshrv2di3 (tmp2, gen_lowpart (V2DImode, tmp1),
1559 : GEN_INT (bits));
1560 0 : emit_insn_before (pat, insn);
1561 0 : result = gen_lowpart (V4SImode, tmp2);
1562 : }
1563 :
1564 : return result;
1565 : }
1566 :
1567 : /* Convert INSN to vector mode. */
1568 :
1569 : void
1570 407478 : general_scalar_chain::convert_insn (rtx_insn *insn)
1571 : {
1572 407478 : rtx def_set = single_set (insn);
1573 407478 : rtx src = SET_SRC (def_set);
1574 407478 : rtx dst = SET_DEST (def_set);
1575 407478 : rtx subreg;
1576 :
1577 407478 : if (MEM_P (dst) && !REG_P (src))
1578 : {
1579 : /* There are no scalar integer instructions and therefore
1580 : temporary register usage is required. */
1581 742 : rtx tmp = gen_reg_rtx (smode);
1582 742 : emit_conversion_insns (gen_move_insn (dst, tmp), insn);
1583 742 : dst = gen_rtx_SUBREG (vmode, tmp, 0);
1584 742 : }
1585 406736 : else if (REG_P (dst) && GET_MODE (dst) == smode)
1586 : {
1587 : /* Replace the definition with a SUBREG to the definition we
1588 : use inside the chain. */
1589 212003 : rtx *vdef = defs_map.get (dst);
1590 212003 : if (vdef)
1591 24976 : dst = *vdef;
1592 212003 : dst = gen_rtx_SUBREG (vmode, dst, 0);
1593 : /* IRA doesn't like to have REG_EQUAL/EQUIV notes when the SET_DEST
1594 : is a non-REG_P. So kill those off. */
1595 212003 : rtx note = find_reg_equal_equiv_note (insn);
1596 212003 : if (note)
1597 9636 : remove_note (insn, note);
1598 : }
1599 :
1600 407478 : switch (GET_CODE (src))
1601 : {
1602 26599 : case PLUS:
1603 26599 : case MINUS:
1604 26599 : case IOR:
1605 26599 : case XOR:
1606 26599 : case AND:
1607 26599 : case SMAX:
1608 26599 : case SMIN:
1609 26599 : case UMAX:
1610 26599 : case UMIN:
1611 26599 : convert_op (&XEXP (src, 1), insn);
1612 : /* FALLTHRU */
1613 :
1614 33200 : case ABS:
1615 33200 : case ASHIFT:
1616 33200 : case ASHIFTRT:
1617 33200 : case LSHIFTRT:
1618 33200 : convert_op (&XEXP (src, 0), insn);
1619 33200 : PUT_MODE (src, vmode);
1620 33200 : break;
1621 :
1622 92 : case ROTATE:
1623 92 : case ROTATERT:
1624 92 : src = convert_rotate (GET_CODE (src), XEXP (src, 0), XEXP (src, 1),
1625 : insn);
1626 92 : break;
1627 :
1628 356 : case NEG:
1629 356 : src = XEXP (src, 0);
1630 :
1631 356 : if (GET_CODE (src) == ABS)
1632 : {
1633 0 : src = XEXP (src, 0);
1634 0 : convert_op (&src, insn);
1635 0 : subreg = gen_reg_rtx (vmode);
1636 0 : emit_insn_before (gen_rtx_SET (subreg,
1637 : gen_rtx_ABS (vmode, src)), insn);
1638 0 : src = subreg;
1639 : }
1640 : else
1641 356 : convert_op (&src, insn);
1642 :
1643 356 : subreg = gen_reg_rtx (vmode);
1644 356 : emit_insn_before (gen_move_insn (subreg, CONST0_RTX (vmode)), insn);
1645 356 : src = gen_rtx_MINUS (vmode, subreg, src);
1646 356 : break;
1647 :
1648 219 : case NOT:
1649 219 : src = XEXP (src, 0);
1650 219 : convert_op (&src, insn);
1651 219 : subreg = gen_reg_rtx (vmode);
1652 219 : emit_insn_before (gen_move_insn (subreg, CONSTM1_RTX (vmode)), insn);
1653 219 : src = gen_rtx_XOR (vmode, src, subreg);
1654 219 : break;
1655 :
1656 170464 : case MEM:
1657 170464 : if (!REG_P (dst))
1658 170464 : convert_op (&src, insn);
1659 : break;
1660 :
1661 196259 : case REG:
1662 196259 : if (!MEM_P (dst))
1663 1526 : convert_op (&src, insn);
1664 : break;
1665 :
1666 0 : case SUBREG:
1667 0 : gcc_assert (GET_MODE (src) == vmode);
1668 : break;
1669 :
1670 0 : case COMPARE:
1671 0 : dst = gen_rtx_REG (CCZmode, FLAGS_REG);
1672 0 : src = convert_compare (XEXP (src, 0), XEXP (src, 1), insn);
1673 0 : break;
1674 :
1675 3799 : case CONST_INT:
1676 3799 : convert_op (&src, insn);
1677 3799 : break;
1678 :
1679 2749 : case VEC_SELECT:
1680 2749 : if (XVECEXP (XEXP (src, 1), 0, 0) == const0_rtx)
1681 1585 : src = XEXP (src, 0);
1682 1164 : else if (smode == DImode)
1683 : {
1684 792 : rtx tmp = gen_lowpart (V1TImode, XEXP (src, 0));
1685 792 : dst = gen_lowpart (V1TImode, dst);
1686 792 : src = gen_rtx_LSHIFTRT (V1TImode, tmp, GEN_INT (64));
1687 : }
1688 : else
1689 : {
1690 372 : rtx tmp = XVECEXP (XEXP (src, 1), 0, 0);
1691 372 : rtvec vec = gen_rtvec (4, tmp, tmp, tmp, tmp);
1692 372 : rtx par = gen_rtx_PARALLEL (VOIDmode, vec);
1693 372 : src = gen_rtx_VEC_SELECT (vmode, XEXP (src, 0), par);
1694 : }
1695 : break;
1696 :
1697 340 : case ZERO_EXTEND:
1698 : /* *zero_extendsidi2 becomes *vec_setv2di_0_zero_extendsi_1. */
1699 340 : src = gen_rtx_VEC_CONCAT (V2DImode, src, const0_rtx);
1700 340 : break;
1701 :
1702 0 : default:
1703 0 : gcc_unreachable ();
1704 : }
1705 :
1706 407478 : SET_SRC (def_set) = src;
1707 407478 : SET_DEST (def_set) = dst;
1708 :
1709 : /* Drop possible dead definitions. */
1710 407478 : PATTERN (insn) = def_set;
1711 :
1712 407478 : INSN_CODE (insn) = -1;
1713 407478 : int patt = recog_memoized (insn);
1714 407478 : if (patt == -1)
1715 0 : fatal_insn_not_found (insn);
1716 407478 : df_insn_rescan (insn);
1717 407478 : }
1718 :
1719 : /* Helper function to compute gain for loading an immediate constant.
1720 : Typically, two movabsq for TImode vs. vmovdqa for V1TImode, but
1721 : with numerous special cases. */
1722 :
1723 : static int
1724 34 : timode_immed_const_gain (rtx cst, basic_block bb)
1725 : {
1726 : /* movabsq vs. movabsq+vmovq+vunpacklqdq. */
1727 34 : if (CONST_WIDE_INT_P (cst)
1728 23 : && CONST_WIDE_INT_NUNITS (cst) == 2
1729 57 : && CONST_WIDE_INT_ELT (cst, 0) == CONST_WIDE_INT_ELT (cst, 1))
1730 0 : return optimize_bb_for_size_p (bb) ? -COSTS_N_BYTES (9)
1731 : : -COSTS_N_INSNS (2);
1732 : /* 2x movabsq ~ vmovdqa. */
1733 : return 0;
1734 : }
1735 :
1736 : /* Return true it's cost profitable for for chain conversion. */
1737 :
1738 : bool
1739 531156 : timode_scalar_chain::compute_convert_gain ()
1740 : {
1741 : /* Assume that if we have to move TImode values between units,
1742 : then transforming this chain isn't worth it. */
1743 531156 : if (cost_sse_integer)
1744 : return false;
1745 :
1746 531156 : bitmap_iterator bi;
1747 531156 : unsigned insn_uid;
1748 :
1749 : /* Split ties to prefer V1TImode when not optimizing for size. */
1750 531156 : int gain = optimize_size ? 0 : 1;
1751 531156 : sreal weighted_gain = 0;
1752 :
1753 531156 : if (dump_file)
1754 0 : fprintf (dump_file, "Computing gain for chain #%d...\n", chain_id);
1755 :
1756 1586613 : EXECUTE_IF_SET_IN_BITMAP (insns, 0, insn_uid, bi)
1757 : {
1758 1055457 : rtx_insn *insn = DF_INSN_UID_GET (insn_uid)->insn;
1759 1055457 : rtx def_set = single_set (insn);
1760 1055457 : rtx src = SET_SRC (def_set);
1761 1055457 : rtx dst = SET_DEST (def_set);
1762 1055457 : HOST_WIDE_INT op1val;
1763 1055457 : basic_block bb = BLOCK_FOR_INSN (insn);
1764 1055457 : int scost, vcost;
1765 1055457 : int igain = 0;
1766 1055457 : profile_count entry_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
1767 1055457 : bool speed_p = optimize_bb_for_speed_p (bb);
1768 1055457 : sreal bb_freq = bb->count.to_sreal_scale (entry_count);
1769 :
1770 1055457 : switch (GET_CODE (src))
1771 : {
1772 548522 : case REG:
1773 548522 : if (GENERAL_REGNO_P (REGNO (src)))
1774 : {
1775 24975 : if (TARGET_SSE4_1)
1776 : /* movq + pinsrq */
1777 27 : igain = speed_p ? -ix86_cost->integer_to_sse
1778 : - COSTS_N_INSNS (1)
1779 : : -COSTS_N_BYTES (11);
1780 : else
1781 : /* movq + movq + punpcklqdq */
1782 24948 : igain = speed_p ? -ix86_cost->integer_to_sse
1783 : - COSTS_N_INSNS (2)
1784 : : -COSTS_N_BYTES (14);
1785 : }
1786 523547 : else if (GENERAL_REG_P (dst))
1787 : {
1788 25108 : if (TARGET_SSE4_1)
1789 : /* pextrq + movq */
1790 27 : igain = speed_p ? -ix86_cost->sse_to_integer
1791 : - COSTS_N_INSNS (1)
1792 : : -COSTS_N_BYTES (11);
1793 : else
1794 : /* movhlps + movq + movq */
1795 25081 : igain = speed_p ? -ix86_cost->sse_to_integer
1796 : - COSTS_N_INSNS (2)
1797 : : -COSTS_N_BYTES (13);
1798 : }
1799 498439 : else if (!speed_p)
1800 15764 : igain = MEM_P (dst) ? COSTS_N_BYTES (6) : COSTS_N_BYTES (3);
1801 : else
1802 : igain = COSTS_N_INSNS (1);
1803 : break;
1804 :
1805 456995 : case MEM:
1806 456995 : igain = !speed_p ? COSTS_N_BYTES (7) : COSTS_N_INSNS (1);
1807 : break;
1808 :
1809 12756 : case CONST_INT:
1810 12756 : if (MEM_P (dst)
1811 12756 : && standard_sse_constant_p (src, V1TImode))
1812 12228 : igain = !speed_p ? COSTS_N_BYTES (11) : 1;
1813 : break;
1814 :
1815 31876 : case CONST_WIDE_INT:
1816 : /* 2 x mov vs. vmovdqa. */
1817 31876 : if (MEM_P (dst))
1818 31305 : igain = !speed_p ? COSTS_N_BYTES (3) : COSTS_N_INSNS (1);
1819 : break;
1820 :
1821 78 : case NOT:
1822 78 : if (MEM_P (dst))
1823 68575 : igain = -COSTS_N_INSNS (1);
1824 : break;
1825 :
1826 69 : case AND:
1827 69 : if (!MEM_P (dst))
1828 : {
1829 58 : igain = COSTS_N_INSNS (1);
1830 58 : if (MEM_P (XEXP (src, 0)) || MEM_P (XEXP (src, 1)))
1831 : /* One less load. */
1832 20 : igain += COSTS_N_INSNS (1);
1833 : }
1834 69 : if (CONST_SCALAR_INT_P (XEXP (src, 1)))
1835 27 : igain += timode_immed_const_gain (XEXP (src, 1), bb);
1836 : break;
1837 :
1838 4506 : case XOR:
1839 4506 : case IOR:
1840 4506 : if (timode_concatdi_p (src))
1841 : {
1842 : /* vmovq;vpinsrq (11 bytes). */
1843 4349 : igain = speed_p ? -ix86_cost->integer_to_sse - COSTS_N_INSNS (1)
1844 : : -COSTS_N_BYTES (11);
1845 : break;
1846 : }
1847 157 : if (!MEM_P (dst))
1848 : {
1849 149 : igain = COSTS_N_INSNS (1);
1850 149 : if (MEM_P (XEXP (src, 0)) || MEM_P (XEXP (src, 1)))
1851 : /* One less load. */
1852 48 : igain += COSTS_N_INSNS (1);
1853 : }
1854 157 : if (CONST_SCALAR_INT_P (XEXP (src, 1)))
1855 7 : igain += timode_immed_const_gain (XEXP (src, 1), bb);
1856 : break;
1857 :
1858 0 : case PLUS:
1859 0 : if (timode_concatdi_p (src))
1860 : /* vmovq;vpinsrq (11 bytes). */
1861 0 : igain = speed_p ? -ix86_cost->integer_to_sse - COSTS_N_INSNS (1)
1862 : : -COSTS_N_BYTES (11);
1863 : break;
1864 :
1865 230 : case ASHIFT:
1866 230 : case LSHIFTRT:
1867 : /* See ix86_expand_v1ti_shift. */
1868 230 : op1val = INTVAL (XEXP (src, 1));
1869 230 : if (!speed_p)
1870 : {
1871 21 : if (op1val == 64 || op1val == 65)
1872 : scost = COSTS_N_BYTES (5);
1873 13 : else if (op1val >= 66)
1874 : scost = COSTS_N_BYTES (6);
1875 13 : else if (op1val == 1)
1876 : scost = COSTS_N_BYTES (8);
1877 : else
1878 : scost = COSTS_N_BYTES (9);
1879 :
1880 19 : if ((op1val & 7) == 0)
1881 : vcost = COSTS_N_BYTES (5);
1882 13 : else if (op1val > 64)
1883 : vcost = COSTS_N_BYTES (10);
1884 : else
1885 13 : vcost = TARGET_AVX ? COSTS_N_BYTES (19) : COSTS_N_BYTES (23);
1886 : }
1887 : else
1888 : {
1889 209 : scost = COSTS_N_INSNS (2);
1890 209 : if ((op1val & 7) == 0)
1891 : vcost = COSTS_N_INSNS (1);
1892 131 : else if (op1val > 64)
1893 : vcost = COSTS_N_INSNS (2);
1894 : else
1895 131 : vcost = TARGET_AVX ? COSTS_N_INSNS (4) : COSTS_N_INSNS (5);
1896 : }
1897 230 : igain = scost - vcost;
1898 230 : break;
1899 :
1900 124 : case ASHIFTRT:
1901 : /* See ix86_expand_v1ti_ashiftrt. */
1902 124 : op1val = INTVAL (XEXP (src, 1));
1903 124 : if (!speed_p)
1904 : {
1905 9 : if (op1val == 64 || op1val == 127)
1906 : scost = COSTS_N_BYTES (7);
1907 9 : else if (op1val == 1)
1908 : scost = COSTS_N_BYTES (8);
1909 8 : else if (op1val == 65)
1910 : scost = COSTS_N_BYTES (10);
1911 8 : else if (op1val >= 66)
1912 : scost = COSTS_N_BYTES (11);
1913 : else
1914 : scost = COSTS_N_BYTES (9);
1915 :
1916 0 : if (op1val == 127)
1917 : vcost = COSTS_N_BYTES (10);
1918 9 : else if (op1val == 64)
1919 : vcost = COSTS_N_BYTES (14);
1920 9 : else if (op1val == 96)
1921 : vcost = COSTS_N_BYTES (18);
1922 9 : else if (op1val >= 111)
1923 : vcost = COSTS_N_BYTES (15);
1924 9 : else if (TARGET_AVX2 && op1val == 32)
1925 : vcost = COSTS_N_BYTES (16);
1926 9 : else if (TARGET_SSE4_1 && op1val == 32)
1927 : vcost = COSTS_N_BYTES (20);
1928 9 : else if (op1val >= 96)
1929 : vcost = COSTS_N_BYTES (23);
1930 9 : else if ((op1val & 7) == 0)
1931 : vcost = COSTS_N_BYTES (28);
1932 9 : else if (TARGET_AVX2 && op1val < 32)
1933 : vcost = COSTS_N_BYTES (30);
1934 9 : else if (op1val == 1 || op1val >= 64)
1935 : vcost = COSTS_N_BYTES (42);
1936 : else
1937 8 : vcost = COSTS_N_BYTES (47);
1938 : }
1939 : else
1940 : {
1941 115 : if (op1val >= 65 && op1val <= 126)
1942 : scost = COSTS_N_INSNS (3);
1943 : else
1944 115 : scost = COSTS_N_INSNS (2);
1945 :
1946 115 : if (op1val == 127)
1947 : vcost = COSTS_N_INSNS (2);
1948 114 : else if (op1val == 64)
1949 : vcost = COSTS_N_INSNS (3);
1950 114 : else if (op1val == 96)
1951 : vcost = COSTS_N_INSNS (3);
1952 114 : else if (op1val >= 111)
1953 : vcost = COSTS_N_INSNS (3);
1954 114 : else if (TARGET_SSE4_1 && op1val == 32)
1955 : vcost = COSTS_N_INSNS (3);
1956 114 : else if (TARGET_SSE4_1
1957 0 : && (op1val == 8 || op1val == 16 || op1val == 24))
1958 : vcost = COSTS_N_INSNS (3);
1959 114 : else if (op1val >= 96)
1960 : vcost = COSTS_N_INSNS (4);
1961 114 : else if (TARGET_SSE4_1 && (op1val == 28 || op1val == 80))
1962 : vcost = COSTS_N_INSNS (4);
1963 114 : else if ((op1val & 7) == 0)
1964 : vcost = COSTS_N_INSNS (5);
1965 114 : else if (TARGET_AVX2 && op1val < 32)
1966 : vcost = COSTS_N_INSNS (6);
1967 114 : else if (TARGET_SSE4_1 && op1val < 15)
1968 : vcost = COSTS_N_INSNS (6);
1969 114 : else if (op1val == 1 || op1val >= 64)
1970 : vcost = COSTS_N_INSNS (8);
1971 : else
1972 17 : vcost = COSTS_N_INSNS (9);
1973 : }
1974 124 : igain = scost - vcost;
1975 124 : break;
1976 :
1977 6 : case ROTATE:
1978 6 : case ROTATERT:
1979 : /* See ix86_expand_v1ti_rotate. */
1980 6 : op1val = INTVAL (XEXP (src, 1));
1981 6 : if (!speed_p)
1982 : {
1983 0 : scost = COSTS_N_BYTES (13);
1984 0 : if ((op1val & 31) == 0)
1985 : vcost = COSTS_N_BYTES (5);
1986 0 : else if ((op1val & 7) == 0)
1987 0 : vcost = TARGET_AVX ? COSTS_N_BYTES (13) : COSTS_N_BYTES (18);
1988 0 : else if (op1val > 32 && op1val < 96)
1989 : vcost = COSTS_N_BYTES (24);
1990 : else
1991 0 : vcost = COSTS_N_BYTES (19);
1992 : }
1993 : else
1994 : {
1995 6 : scost = COSTS_N_INSNS (3);
1996 6 : if ((op1val & 31) == 0)
1997 : vcost = COSTS_N_INSNS (1);
1998 4 : else if ((op1val & 7) == 0)
1999 1 : vcost = TARGET_AVX ? COSTS_N_INSNS (3) : COSTS_N_INSNS (4);
2000 3 : else if (op1val > 32 && op1val < 96)
2001 : vcost = COSTS_N_INSNS (5);
2002 : else
2003 3 : vcost = COSTS_N_INSNS (4);
2004 : }
2005 6 : igain = scost - vcost;
2006 6 : break;
2007 :
2008 13 : case COMPARE:
2009 13 : if (XEXP (src, 1) == const0_rtx)
2010 : {
2011 2 : if (GET_CODE (XEXP (src, 0)) == AND)
2012 : /* and;and;or (9 bytes) vs. ptest (5 bytes). */
2013 : igain = !speed_p ? COSTS_N_BYTES (4) : COSTS_N_INSNS (2);
2014 : /* or (3 bytes) vs. ptest (5 bytes). */
2015 2 : else if (!speed_p)
2016 0 : igain = -COSTS_N_BYTES (2);
2017 : }
2018 11 : else if (XEXP (src, 1) == const1_rtx)
2019 : /* and;cmp -1 (7 bytes) vs. pcmpeqd;pxor;ptest (13 bytes). */
2020 0 : igain = !speed_p ? -COSTS_N_BYTES (6) : -COSTS_N_INSNS (1);
2021 : break;
2022 :
2023 282 : case ZERO_EXTEND:
2024 282 : if (GET_MODE (XEXP (src, 0)) == DImode)
2025 : /* xor (2 bytes) vs. vmovq (5 bytes). */
2026 282 : igain = speed_p ? COSTS_N_INSNS (1) - ix86_cost->sse_to_integer
2027 : : -COSTS_N_BYTES (3);
2028 : break;
2029 :
2030 : default:
2031 : break;
2032 : }
2033 :
2034 2076679 : gain += igain;
2035 1055455 : if (speed_p)
2036 1021224 : weighted_gain += bb_freq * igain;
2037 :
2038 1055457 : if (igain != 0 && dump_file)
2039 : {
2040 0 : fprintf (dump_file, " Instruction gain %d with bb_freq %.2f for ",
2041 : igain, bb_freq.to_double ());
2042 0 : dump_insn_slim (dump_file, insn);
2043 : }
2044 : }
2045 :
2046 531156 : if (dump_file)
2047 0 : fprintf (dump_file, " Total gain: %d, weighted gain %.2f\n",
2048 : gain, weighted_gain.to_double ());
2049 :
2050 531156 : if (weighted_gain > (sreal) 0)
2051 : return true;
2052 : else
2053 56336 : return gain > 0;
2054 : }
2055 :
2056 : /* Fix uses of converted REG in debug insns. */
2057 :
2058 : void
2059 446604 : timode_scalar_chain::fix_debug_reg_uses (rtx reg)
2060 : {
2061 446604 : if (!flag_var_tracking)
2062 : return;
2063 :
2064 391458 : df_ref ref, next;
2065 801875 : for (ref = DF_REG_USE_CHAIN (REGNO (reg)); ref; ref = next)
2066 : {
2067 410417 : rtx_insn *insn = DF_REF_INSN (ref);
2068 : /* Make sure the next ref is for a different instruction,
2069 : so that we're not affected by the rescan. */
2070 410417 : next = DF_REF_NEXT_REG (ref);
2071 410417 : while (next && DF_REF_INSN (next) == insn)
2072 0 : next = DF_REF_NEXT_REG (next);
2073 :
2074 410417 : if (DEBUG_INSN_P (insn))
2075 : {
2076 : /* It may be a debug insn with a TImode variable in
2077 : register. */
2078 : bool changed = false;
2079 246 : for (; ref != next; ref = DF_REF_NEXT_REG (ref))
2080 : {
2081 123 : rtx *loc = DF_REF_LOC (ref);
2082 123 : if (REG_P (*loc) && GET_MODE (*loc) == V1TImode)
2083 : {
2084 114 : *loc = gen_rtx_SUBREG (TImode, *loc, 0);
2085 114 : changed = true;
2086 : }
2087 : }
2088 123 : if (changed)
2089 114 : df_insn_rescan (insn);
2090 : }
2091 : }
2092 : }
2093 :
2094 : /* Convert SRC, a *concatditi3 pattern, into a vec_concatv2di instruction.
2095 : Insert this before INSN, and return the result as a V1TImode subreg. */
2096 :
2097 : static rtx
2098 283 : timode_convert_concatdi (rtx src, rtx_insn *insn)
2099 : {
2100 283 : rtx hi, lo;
2101 283 : rtx tmp = gen_reg_rtx (V2DImode);
2102 283 : if (GET_CODE (XEXP (src, 0)) == ASHIFT)
2103 : {
2104 283 : hi = XEXP (XEXP (XEXP (src, 0), 0), 0);
2105 283 : lo = XEXP (XEXP (src, 1), 0);
2106 : }
2107 : else
2108 : {
2109 0 : hi = XEXP (XEXP (XEXP (src, 1), 0), 0);
2110 0 : lo = XEXP (XEXP (src, 0), 0);
2111 : }
2112 283 : emit_insn_before (gen_vec_concatv2di (tmp, lo, hi), insn);
2113 283 : return gen_rtx_SUBREG (V1TImode, tmp, 0);
2114 : }
2115 :
2116 : /* Convert INSN from TImode to V1T1mode. */
2117 :
2118 : void
2119 965650 : timode_scalar_chain::convert_insn (rtx_insn *insn)
2120 : {
2121 965650 : rtx def_set = single_set (insn);
2122 965650 : rtx src = SET_SRC (def_set);
2123 965650 : rtx dst = SET_DEST (def_set);
2124 965650 : rtx tmp;
2125 :
2126 965650 : switch (GET_CODE (dst))
2127 : {
2128 447125 : case REG:
2129 447125 : if (GET_MODE (dst) == TImode)
2130 : {
2131 446531 : if (!HARD_REGISTER_NUM_P (REGNO (dst)))
2132 : {
2133 446018 : PUT_MODE (dst, V1TImode);
2134 446018 : fix_debug_reg_uses (dst);
2135 : }
2136 513 : else if (!GENERAL_REGNO_P (REGNO (dst)))
2137 359 : dst = gen_raw_REG (V1TImode, REGNO (dst));
2138 : }
2139 447125 : if (GET_MODE (dst) == V1TImode)
2140 : {
2141 : /* It might potentially be helpful to convert REG_EQUAL notes,
2142 : but for now we just remove them. */
2143 446963 : rtx note = find_reg_equal_equiv_note (insn);
2144 446963 : if (note)
2145 483 : remove_note (insn, note);
2146 : }
2147 : break;
2148 518525 : case MEM:
2149 518525 : PUT_MODE (dst, V1TImode);
2150 518525 : break;
2151 :
2152 0 : default:
2153 0 : gcc_unreachable ();
2154 : }
2155 :
2156 965650 : switch (GET_CODE (src))
2157 : {
2158 475840 : case REG:
2159 475840 : if (GET_MODE (src) == TImode)
2160 : {
2161 814 : if (GENERAL_REGNO_P (REGNO (src)))
2162 : {
2163 223 : rtx lo = gen_reg_rtx (DImode);
2164 223 : rtx hi = gen_reg_rtx (DImode);
2165 223 : emit_insn_before (gen_rtx_SET (lo, gen_lowpart (DImode, src)),
2166 : insn);
2167 223 : emit_insn_before (gen_rtx_SET (hi, gen_highpart (DImode, src)),
2168 : insn);
2169 223 : src = gen_reg_rtx (V2DImode);
2170 223 : emit_insn_before (gen_vec_concatv2di (src, lo, hi), insn);
2171 223 : src = gen_lowpart (V1TImode, src);
2172 : }
2173 591 : else if (!HARD_REGISTER_NUM_P (REGNO (src)))
2174 : {
2175 586 : PUT_MODE (src, V1TImode);
2176 586 : fix_debug_reg_uses (src);
2177 : }
2178 : else
2179 5 : src = gen_raw_REG (V1TImode, REGNO (src));
2180 : }
2181 475840 : if (GENERAL_REG_P (dst))
2182 : {
2183 154 : rtx tmp = gen_reg_rtx (V2DImode);
2184 154 : src = gen_lowpart (V2DImode, src);
2185 154 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
2186 : /* Extracting hi before lo helps register allocation. */
2187 154 : rtx hi = gen_reg_rtx (DImode);
2188 154 : rtx lo = gen_reg_rtx (DImode);
2189 154 : emit_insn_before (gen_vec_extractv2didi (hi, tmp, const1_rtx), insn);
2190 154 : emit_insn_before (gen_vec_extractv2didi (lo, tmp, const0_rtx), insn);
2191 :
2192 : /* Construct *concatditi3 pattern from lo and hi. */
2193 154 : hi = gen_rtx_ZERO_EXTEND (TImode, hi);
2194 154 : hi = gen_rtx_ASHIFT (TImode, hi, GEN_INT (64));
2195 154 : lo = gen_rtx_ZERO_EXTEND (TImode, lo);
2196 154 : src = gen_rtx_PLUS (TImode, hi, lo);
2197 : }
2198 : break;
2199 :
2200 444732 : case MEM:
2201 444732 : PUT_MODE (src, V1TImode);
2202 444732 : break;
2203 :
2204 31693 : case CONST_WIDE_INT:
2205 31693 : if (NONDEBUG_INSN_P (insn))
2206 : {
2207 : /* Since there are no instructions to store 128-bit constant,
2208 : temporary register usage is required. */
2209 31693 : bool use_move;
2210 31693 : start_sequence ();
2211 31693 : tmp = ix86_convert_const_wide_int_to_broadcast (TImode, src);
2212 31693 : if (tmp)
2213 : {
2214 194 : src = lowpart_subreg (V1TImode, tmp, TImode);
2215 194 : use_move = true;
2216 : }
2217 : else
2218 : {
2219 31499 : src = smode_convert_cst (src, V1TImode);
2220 31499 : src = validize_mem (force_const_mem (V1TImode, src));
2221 31499 : use_move = MEM_P (dst);
2222 : }
2223 31693 : rtx_insn *seq = end_sequence ();
2224 31693 : if (seq)
2225 195 : emit_insn_before (seq, insn);
2226 31693 : if (use_move)
2227 : {
2228 31306 : tmp = gen_reg_rtx (V1TImode);
2229 31306 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
2230 31306 : src = tmp;
2231 : }
2232 : }
2233 : break;
2234 :
2235 12756 : case CONST_INT:
2236 12756 : switch (standard_sse_constant_p (src, TImode))
2237 : {
2238 12528 : case 1:
2239 12528 : src = CONST0_RTX (GET_MODE (dst));
2240 12528 : break;
2241 228 : case 2:
2242 228 : src = CONSTM1_RTX (GET_MODE (dst));
2243 228 : break;
2244 0 : default:
2245 0 : gcc_unreachable ();
2246 : }
2247 12756 : if (MEM_P (dst))
2248 : {
2249 12228 : tmp = gen_reg_rtx (V1TImode);
2250 12228 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
2251 12228 : src = tmp;
2252 : }
2253 : break;
2254 :
2255 41 : case AND:
2256 41 : if (GET_CODE (XEXP (src, 0)) == NOT)
2257 : {
2258 0 : convert_op (&XEXP (XEXP (src, 0), 0), insn);
2259 0 : convert_op (&XEXP (src, 1), insn);
2260 0 : PUT_MODE (XEXP (src, 0), V1TImode);
2261 0 : PUT_MODE (src, V1TImode);
2262 0 : break;
2263 : }
2264 41 : convert_op (&XEXP (src, 0), insn);
2265 41 : convert_op (&XEXP (src, 1), insn);
2266 41 : PUT_MODE (src, V1TImode);
2267 41 : if (MEM_P (dst))
2268 : {
2269 10 : tmp = gen_reg_rtx (V1TImode);
2270 10 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
2271 10 : src = tmp;
2272 : }
2273 : break;
2274 :
2275 393 : case XOR:
2276 393 : case IOR:
2277 393 : if (timode_concatdi_p (src))
2278 : {
2279 283 : src = timode_convert_concatdi (src, insn);
2280 283 : break;
2281 : }
2282 110 : convert_op (&XEXP (src, 0), insn);
2283 110 : convert_op (&XEXP (src, 1), insn);
2284 110 : PUT_MODE (src, V1TImode);
2285 110 : if (MEM_P (dst))
2286 : {
2287 8 : tmp = gen_reg_rtx (V1TImode);
2288 8 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
2289 8 : src = tmp;
2290 : }
2291 : break;
2292 :
2293 3 : case NOT:
2294 3 : src = XEXP (src, 0);
2295 3 : convert_op (&src, insn);
2296 3 : tmp = gen_reg_rtx (V1TImode);
2297 3 : emit_insn_before (gen_move_insn (tmp, CONSTM1_RTX (V1TImode)), insn);
2298 3 : src = gen_rtx_XOR (V1TImode, src, tmp);
2299 3 : if (MEM_P (dst))
2300 : {
2301 0 : tmp = gen_reg_rtx (V1TImode);
2302 0 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
2303 0 : src = tmp;
2304 : }
2305 : break;
2306 :
2307 8 : case COMPARE:
2308 8 : dst = gen_rtx_REG (CCZmode, FLAGS_REG);
2309 8 : src = convert_compare (XEXP (src, 0), XEXP (src, 1), insn);
2310 8 : break;
2311 :
2312 68 : case ASHIFT:
2313 68 : case LSHIFTRT:
2314 68 : case ASHIFTRT:
2315 68 : case ROTATERT:
2316 68 : case ROTATE:
2317 68 : convert_op (&XEXP (src, 0), insn);
2318 68 : PUT_MODE (src, V1TImode);
2319 68 : break;
2320 :
2321 116 : case ZERO_EXTEND:
2322 116 : if (GET_MODE (XEXP (src, 0)) == DImode)
2323 : {
2324 : /* Convert to *vec_concatv2di_0. */
2325 116 : rtx tmp = gen_reg_rtx (V2DImode);
2326 116 : rtx pat = gen_rtx_VEC_CONCAT (V2DImode, XEXP (src, 0), const0_rtx);
2327 116 : emit_insn_before (gen_move_insn (tmp, pat), insn);
2328 116 : src = gen_rtx_SUBREG (vmode, tmp, 0);
2329 : }
2330 : else
2331 0 : gcc_unreachable ();
2332 116 : break;
2333 :
2334 0 : case PLUS:
2335 0 : if (timode_concatdi_p (src))
2336 0 : src = timode_convert_concatdi (src, insn);
2337 : else
2338 0 : gcc_unreachable ();
2339 0 : break;
2340 :
2341 0 : default:
2342 0 : gcc_unreachable ();
2343 : }
2344 :
2345 965650 : SET_SRC (def_set) = src;
2346 965650 : SET_DEST (def_set) = dst;
2347 :
2348 : /* Drop possible dead definitions. */
2349 965650 : PATTERN (insn) = def_set;
2350 :
2351 965650 : INSN_CODE (insn) = -1;
2352 965650 : recog_memoized (insn);
2353 965650 : df_insn_rescan (insn);
2354 965650 : }
2355 :
2356 : /* Generate copies from defs used by the chain but not defined therein.
2357 : Also populates defs_map which is used later by convert_insn. */
2358 :
2359 : void
2360 662752 : scalar_chain::convert_registers ()
2361 : {
2362 662752 : bitmap_iterator bi;
2363 662752 : unsigned id;
2364 689838 : EXECUTE_IF_SET_IN_BITMAP (defs_conv, 0, id, bi)
2365 : {
2366 27086 : rtx chain_reg = gen_reg_rtx (smode);
2367 27086 : defs_map.put (regno_reg_rtx[id], chain_reg);
2368 : }
2369 671449 : EXECUTE_IF_SET_IN_BITMAP (insns_conv, 0, id, bi)
2370 20952 : for (df_ref ref = DF_INSN_UID_DEFS (id); ref; ref = DF_REF_NEXT_LOC (ref))
2371 12255 : if (bitmap_bit_p (defs_conv, DF_REF_REGNO (ref)))
2372 8697 : make_vector_copies (DF_REF_INSN (ref), DF_REF_REAL_REG (ref));
2373 662752 : }
2374 :
2375 : /* Convert whole chain creating required register
2376 : conversions and copies. */
2377 :
2378 : int
2379 662752 : scalar_chain::convert ()
2380 : {
2381 662752 : bitmap_iterator bi;
2382 662752 : unsigned id;
2383 662752 : int converted_insns = 0;
2384 :
2385 662752 : if (!dbg_cnt (stv_conversion))
2386 : return 0;
2387 :
2388 662752 : if (dump_file)
2389 0 : fprintf (dump_file, "Converting chain #%d...\n", chain_id);
2390 :
2391 662752 : convert_registers ();
2392 :
2393 2035880 : EXECUTE_IF_SET_IN_BITMAP (insns, 0, id, bi)
2394 : {
2395 1373128 : rtx_insn *insn = DF_INSN_UID_GET (id)->insn;
2396 1373128 : convert_insn_common (insn);
2397 1373128 : convert_insn (insn);
2398 1373128 : converted_insns++;
2399 : }
2400 :
2401 : return converted_insns;
2402 : }
2403 :
2404 : /* Return the SET expression if INSN doesn't reference hard register.
2405 : Return NULL if INSN uses or defines a hard register, excluding
2406 : pseudo register pushes, hard register uses in a memory address,
2407 : clobbers and flags definitions. */
2408 :
2409 : static rtx
2410 355249558 : pseudo_reg_set (rtx_insn *insn)
2411 : {
2412 355249558 : rtx set = single_set (insn);
2413 355249558 : if (!set)
2414 : return NULL;
2415 :
2416 : /* Check pseudo register push first. */
2417 139424763 : machine_mode mode = TARGET_64BIT ? TImode : DImode;
2418 139424763 : if (REG_P (SET_SRC (set))
2419 39496911 : && !HARD_REGISTER_P (SET_SRC (set))
2420 170297958 : && push_operand (SET_DEST (set), mode))
2421 : return set;
2422 :
2423 139171574 : df_ref ref;
2424 225570832 : FOR_EACH_INSN_DEF (ref, insn)
2425 124536789 : if (HARD_REGISTER_P (DF_REF_REAL_REG (ref))
2426 67041406 : && !DF_REF_FLAGS_IS_SET (ref, DF_REF_MUST_CLOBBER)
2427 176438611 : && DF_REF_REGNO (ref) != FLAGS_REG)
2428 : return NULL;
2429 :
2430 193202744 : FOR_EACH_INSN_USE (ref, insn)
2431 118262522 : if (!DF_REF_REG_MEM_P (ref) && HARD_REGISTER_P (DF_REF_REAL_REG (ref)))
2432 : return NULL;
2433 :
2434 : return set;
2435 : }
2436 :
2437 : /* Return true if the register REG is defined in a single DEF chain.
2438 : If it is defined in more than one DEF chains, we may not be able
2439 : to convert it in all chains. */
2440 :
2441 : static bool
2442 1312672 : single_def_chain_p (rtx reg)
2443 : {
2444 1312672 : df_ref ref = DF_REG_DEF_CHAIN (REGNO (reg));
2445 1312672 : if (!ref)
2446 : return false;
2447 1312641 : return DF_REF_NEXT_REG (ref) == nullptr;
2448 : }
2449 :
2450 : /* Check if comparison INSN may be transformed into vector comparison.
2451 : Currently we transform equality/inequality checks which look like:
2452 : (set (reg:CCZ 17 flags) (compare:CCZ (reg:TI x) (reg:TI y))) */
2453 :
2454 : static bool
2455 13126547 : convertible_comparison_p (rtx_insn *insn, enum machine_mode mode)
2456 : {
2457 14544269 : if (mode != (TARGET_64BIT ? TImode : DImode))
2458 : return false;
2459 :
2460 4813732 : if (!TARGET_SSE4_1)
2461 : return false;
2462 :
2463 166688 : rtx def_set = single_set (insn);
2464 :
2465 166688 : gcc_assert (def_set);
2466 :
2467 166688 : rtx src = SET_SRC (def_set);
2468 166688 : rtx dst = SET_DEST (def_set);
2469 :
2470 166688 : gcc_assert (GET_CODE (src) == COMPARE);
2471 :
2472 166688 : if (!REG_P (dst)
2473 166688 : || REGNO (dst) != FLAGS_REG
2474 333376 : || GET_MODE (dst) != CCZmode)
2475 : return false;
2476 :
2477 116234 : rtx op1 = XEXP (src, 0);
2478 116234 : rtx op2 = XEXP (src, 1);
2479 :
2480 : /* *cmp<dwi>_doubleword. */
2481 116234 : if (general_operand (op1, mode)
2482 116234 : && general_operand (op2, mode))
2483 : return true;
2484 :
2485 : /* *testti_doubleword. */
2486 116171 : if (op2 == const0_rtx
2487 40019 : && GET_CODE (op1) == AND
2488 142 : && REG_P (XEXP (op1, 0)))
2489 : {
2490 142 : rtx op12 = XEXP (op1, 1);
2491 142 : return GET_MODE (XEXP (op1, 0)) == TImode
2492 142 : && (CONST_SCALAR_INT_P (op12)
2493 0 : || ((REG_P (op12) || MEM_P (op12))
2494 0 : && GET_MODE (op12) == TImode));
2495 : }
2496 :
2497 : /* *test<dwi>_not_doubleword. */
2498 116029 : if (op2 == const0_rtx
2499 39877 : && GET_CODE (op1) == AND
2500 0 : && GET_CODE (XEXP (op1, 0)) == NOT)
2501 : {
2502 0 : rtx op11 = XEXP (XEXP (op1, 0), 0);
2503 0 : rtx op12 = XEXP (op1, 1);
2504 0 : return (REG_P (op11) || MEM_P (op11))
2505 0 : && (REG_P (op12) || MEM_P (op12))
2506 0 : && GET_MODE (op11) == mode
2507 0 : && GET_MODE (op12) == mode;
2508 : }
2509 :
2510 : return false;
2511 : }
2512 :
2513 : /* The general version of scalar_to_vector_candidate_p. */
2514 :
2515 : static bool
2516 247978531 : general_scalar_to_vector_candidate_p (rtx_insn *insn, enum machine_mode mode)
2517 : {
2518 247978531 : rtx def_set = pseudo_reg_set (insn);
2519 :
2520 247978531 : if (!def_set)
2521 : return false;
2522 :
2523 50800179 : rtx src = SET_SRC (def_set);
2524 50800179 : rtx dst = SET_DEST (def_set);
2525 :
2526 50800179 : if (GET_CODE (src) == COMPARE)
2527 9021676 : return convertible_comparison_p (insn, mode);
2528 :
2529 : /* We are interested in "mode" only. */
2530 41778503 : if ((GET_MODE (src) != mode
2531 28489031 : && !CONST_INT_P (src))
2532 18693177 : || GET_MODE (dst) != mode)
2533 : return false;
2534 :
2535 15669382 : if (!REG_P (dst) && !MEM_P (dst))
2536 : return false;
2537 :
2538 15407023 : switch (GET_CODE (src))
2539 : {
2540 557892 : case ASHIFT:
2541 557892 : case LSHIFTRT:
2542 557892 : case ASHIFTRT:
2543 557892 : case ROTATE:
2544 557892 : case ROTATERT:
2545 557892 : if (!CONST_INT_P (XEXP (src, 1))
2546 1077894 : || !IN_RANGE (INTVAL (XEXP (src, 1)), 0, GET_MODE_BITSIZE (mode)-1))
2547 : return false;
2548 :
2549 : /* Check for extend highpart case. */
2550 520002 : if (mode != DImode
2551 383295 : || GET_CODE (src) != ASHIFTRT
2552 88884 : || GET_CODE (XEXP (src, 0)) != ASHIFT)
2553 : break;
2554 :
2555 3807554 : src = XEXP (src, 0);
2556 : break;
2557 :
2558 102396 : case SMAX:
2559 102396 : case SMIN:
2560 102396 : case UMAX:
2561 102396 : case UMIN:
2562 102396 : if ((mode == DImode && !TARGET_AVX512VL)
2563 18005 : || (mode == SImode && !TARGET_SSE4_1))
2564 : return false;
2565 : /* Fallthru. */
2566 :
2567 3319372 : case AND:
2568 3319372 : case IOR:
2569 3319372 : case XOR:
2570 3319372 : case PLUS:
2571 3319372 : case MINUS:
2572 3319372 : if (!REG_P (XEXP (src, 1))
2573 : && !MEM_P (XEXP (src, 1))
2574 : && !CONST_INT_P (XEXP (src, 1)))
2575 : return false;
2576 :
2577 3225819 : if (GET_MODE (XEXP (src, 1)) != mode
2578 1853808 : && !CONST_INT_P (XEXP (src, 1)))
2579 : return false;
2580 :
2581 : /* Check for andnot case. */
2582 3225819 : if (GET_CODE (src) != AND
2583 183811 : || GET_CODE (XEXP (src, 0)) != NOT)
2584 : break;
2585 :
2586 3807554 : src = XEXP (src, 0);
2587 : /* FALLTHRU */
2588 :
2589 : case NOT:
2590 : break;
2591 :
2592 32943 : case NEG:
2593 : /* Check for nabs case. */
2594 32943 : if (GET_CODE (XEXP (src, 0)) != ABS)
2595 : break;
2596 :
2597 : src = XEXP (src, 0);
2598 : /* FALLTHRU */
2599 :
2600 3786 : case ABS:
2601 3786 : if ((mode == DImode && !TARGET_AVX512VL)
2602 1400 : || (mode == SImode && !TARGET_SSSE3))
2603 : return false;
2604 : break;
2605 :
2606 : case REG:
2607 : return true;
2608 :
2609 6173253 : case MEM:
2610 6173253 : case CONST_INT:
2611 6173253 : return REG_P (dst);
2612 :
2613 39913 : case VEC_SELECT:
2614 : /* Excluding MEM_P (dst) avoids interfering with vpextr[dq]. */
2615 39913 : return REG_P (dst)
2616 33297 : && REG_P (XEXP (src, 0))
2617 42569 : && GET_MODE (XEXP (src, 0)) == (mode == DImode ? V2DImode
2618 : : V4SImode)
2619 23898 : && GET_CODE (XEXP (src, 1)) == PARALLEL
2620 23898 : && XVECLEN (XEXP (src, 1), 0) == 1
2621 63811 : && CONST_INT_P (XVECEXP (XEXP (src, 1), 0, 0));
2622 :
2623 321740 : case ZERO_EXTEND:
2624 : /* *zero_extendsidi2 becomes *vec_setv2di_0_zero_extendsi_1. */
2625 321740 : return mode == DImode
2626 226806 : && REG_P (dst)
2627 223972 : && GET_MODE (XEXP (src, 0)) == SImode
2628 506247 : && MEM_P (XEXP (src, 0));
2629 :
2630 : default:
2631 : return false;
2632 : }
2633 :
2634 3807554 : if (!REG_P (XEXP (src, 0))
2635 : && !MEM_P (XEXP (src, 0))
2636 : && !CONST_INT_P (XEXP (src, 0)))
2637 : return false;
2638 :
2639 3509518 : if (GET_MODE (XEXP (src, 0)) != mode
2640 0 : && !CONST_INT_P (XEXP (src, 0)))
2641 : return false;
2642 :
2643 : return true;
2644 : }
2645 :
2646 : /* Check for a suitable TImode memory operand. */
2647 :
2648 : static bool
2649 1642 : timode_mem_p (rtx x)
2650 : {
2651 1642 : return MEM_P (x)
2652 1642 : && (TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
2653 0 : || !misaligned_operand (x, TImode));
2654 : }
2655 :
2656 : /* The TImode version of scalar_to_vector_candidate_p. */
2657 :
2658 : static bool
2659 107271027 : timode_scalar_to_vector_candidate_p (rtx_insn *insn)
2660 : {
2661 107271027 : rtx def_set = pseudo_reg_set (insn);
2662 :
2663 : /* We allow two exceptions to the pseudo registers only rule.
2664 : Setting a hard register from a pseudo, and setting a pseudo
2665 : from a hard register. */
2666 107271027 : if (!def_set)
2667 : {
2668 82877795 : def_set = single_set (insn);
2669 82877795 : if (def_set)
2670 : {
2671 18572852 : rtx src = SET_SRC (def_set);
2672 18572852 : rtx dst = SET_DEST (def_set);
2673 18572852 : if (GET_MODE (dst) == TImode
2674 225067 : && REG_P (src) && REG_P (dst))
2675 : {
2676 102996 : if (HARD_REGISTER_P (dst)
2677 53471 : && !HARD_REGISTER_P (src)
2678 156467 : && single_def_chain_p (src))
2679 : return true;
2680 73477 : if (HARD_REGISTER_P (src)
2681 49525 : && !HARD_REGISTER_P (dst)
2682 123002 : && single_def_chain_p (dst))
2683 : return true;
2684 : }
2685 : }
2686 : return false;
2687 : }
2688 :
2689 24393232 : rtx src = SET_SRC (def_set);
2690 24393232 : rtx dst = SET_DEST (def_set);
2691 :
2692 24393232 : if (GET_CODE (src) == COMPARE)
2693 4104871 : return convertible_comparison_p (insn, TImode);
2694 :
2695 20288361 : if (GET_MODE (dst) != TImode
2696 1256616 : || (GET_MODE (src) != TImode
2697 63644 : && !CONST_SCALAR_INT_P (src)))
2698 : return false;
2699 :
2700 1256616 : if (!REG_P (dst) && !MEM_P (dst))
2701 : return false;
2702 :
2703 1255163 : if (MEM_P (dst)
2704 556590 : && misaligned_operand (dst, TImode)
2705 1586758 : && !TARGET_SSE_UNALIGNED_STORE_OPTIMAL)
2706 : return false;
2707 :
2708 1255158 : if (REG_P (dst) && !single_def_chain_p (dst))
2709 : return false;
2710 :
2711 1095574 : switch (GET_CODE (src))
2712 : {
2713 511103 : case REG:
2714 511103 : return single_def_chain_p (src);
2715 :
2716 : case CONST_WIDE_INT:
2717 : return true;
2718 :
2719 15123 : case CONST_INT:
2720 : /* ??? Verify performance impact before enabling CONST_INT for
2721 : __int128 store. */
2722 15123 : return standard_sse_constant_p (src, TImode);
2723 :
2724 467463 : case MEM:
2725 : /* Memory must be aligned or unaligned load is optimal. */
2726 467463 : return (REG_P (dst)
2727 467463 : && (!misaligned_operand (src, TImode)
2728 164128 : || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL));
2729 :
2730 3849 : case AND:
2731 3849 : if (!MEM_P (dst)
2732 3808 : && GET_CODE (XEXP (src, 0)) == NOT
2733 0 : && REG_P (XEXP (XEXP (src, 0), 0))
2734 3849 : && (REG_P (XEXP (src, 1))
2735 0 : || CONST_SCALAR_INT_P (XEXP (src, 1))
2736 0 : || timode_mem_p (XEXP (src, 1))))
2737 : return true;
2738 3849 : return (REG_P (XEXP (src, 0))
2739 46 : || timode_mem_p (XEXP (src, 0)))
2740 3895 : && (REG_P (XEXP (src, 1))
2741 1499 : || CONST_SCALAR_INT_P (XEXP (src, 1))
2742 45 : || timode_mem_p (XEXP (src, 1)));
2743 :
2744 15831 : case IOR:
2745 15831 : case XOR:
2746 15831 : if (timode_concatdi_p (src))
2747 : return true;
2748 4071 : return (REG_P (XEXP (src, 0))
2749 1467 : || timode_mem_p (XEXP (src, 0)))
2750 4088 : && (REG_P (XEXP (src, 1))
2751 314 : || CONST_SCALAR_INT_P (XEXP (src, 1))
2752 68 : || timode_mem_p (XEXP (src, 1)));
2753 :
2754 524 : case NOT:
2755 524 : return REG_P (XEXP (src, 0)) || timode_mem_p (XEXP (src, 0));
2756 :
2757 11846 : case ASHIFT:
2758 11846 : case LSHIFTRT:
2759 11846 : case ASHIFTRT:
2760 11846 : case ROTATERT:
2761 11846 : case ROTATE:
2762 : /* Handle shifts/rotates by integer constants between 0 and 127. */
2763 11846 : return REG_P (XEXP (src, 0))
2764 11810 : && CONST_INT_P (XEXP (src, 1))
2765 23288 : && (INTVAL (XEXP (src, 1)) & ~0x7f) == 0;
2766 :
2767 7644 : case PLUS:
2768 7644 : return timode_concatdi_p (src);
2769 :
2770 4195 : case ZERO_EXTEND:
2771 4195 : return REG_P (XEXP (src, 0))
2772 4195 : && GET_MODE (XEXP (src, 0)) == DImode;
2773 :
2774 : default:
2775 : return false;
2776 : }
2777 : }
2778 :
2779 : /* For a register REGNO, scan instructions for its defs and uses.
2780 : Put REGNO in REGS if a def or use isn't in CANDIDATES. */
2781 :
2782 : static void
2783 1291030 : timode_check_non_convertible_regs (bitmap candidates, bitmap regs,
2784 : unsigned int regno)
2785 : {
2786 : /* Do nothing if REGNO is already in REGS or is a hard reg. */
2787 1291030 : if (bitmap_bit_p (regs, regno)
2788 1291030 : || HARD_REGISTER_NUM_P (regno))
2789 : return;
2790 :
2791 1282379 : for (df_ref def = DF_REG_DEF_CHAIN (regno);
2792 2551766 : def;
2793 1269387 : def = DF_REF_NEXT_REG (def))
2794 : {
2795 1282359 : if (!bitmap_bit_p (candidates, DF_REF_INSN_UID (def)))
2796 : {
2797 12972 : if (dump_file)
2798 0 : fprintf (dump_file,
2799 : "r%d has non convertible def in insn %d\n",
2800 0 : regno, DF_REF_INSN_UID (def));
2801 :
2802 12972 : bitmap_set_bit (regs, regno);
2803 12972 : break;
2804 : }
2805 : }
2806 :
2807 1282379 : for (df_ref ref = DF_REG_USE_CHAIN (regno);
2808 2841340 : ref;
2809 1558961 : ref = DF_REF_NEXT_REG (ref))
2810 : {
2811 : /* Debug instructions are skipped. */
2812 1631738 : if (NONDEBUG_INSN_P (DF_REF_INSN (ref))
2813 1631738 : && !bitmap_bit_p (candidates, DF_REF_INSN_UID (ref)))
2814 : {
2815 72777 : if (dump_file)
2816 0 : fprintf (dump_file,
2817 : "r%d has non convertible use in insn %d\n",
2818 0 : regno, DF_REF_INSN_UID (ref));
2819 :
2820 72777 : bitmap_set_bit (regs, regno);
2821 72777 : break;
2822 : }
2823 : }
2824 : }
2825 :
2826 : /* For a given bitmap of insn UIDs scans all instructions and
2827 : remove insn from CANDIDATES in case it has both convertible
2828 : and not convertible definitions.
2829 :
2830 : All insns in a bitmap are conversion candidates according to
2831 : scalar_to_vector_candidate_p. Currently it implies all insns
2832 : are single_set. */
2833 :
2834 : static void
2835 850474 : timode_remove_non_convertible_regs (bitmap candidates)
2836 : {
2837 850474 : bitmap_iterator bi;
2838 850474 : unsigned id;
2839 850474 : bitmap regs = BITMAP_ALLOC (NULL);
2840 872318 : bool changed;
2841 :
2842 872318 : do {
2843 872318 : changed = false;
2844 2184916 : EXECUTE_IF_SET_IN_BITMAP (candidates, 0, id, bi)
2845 : {
2846 1312598 : rtx_insn *insn = DF_INSN_UID_GET (id)->insn;
2847 1312598 : df_ref ref;
2848 :
2849 2053103 : FOR_EACH_INSN_DEF (ref, insn)
2850 740505 : if (!DF_REF_REG_MEM_P (ref)
2851 740505 : && GET_MODE (DF_REF_REG (ref)) == TImode)
2852 647334 : timode_check_non_convertible_regs (candidates, regs,
2853 : DF_REF_REGNO (ref));
2854 :
2855 3232632 : FOR_EACH_INSN_USE (ref, insn)
2856 1920034 : if (DF_REF_TYPE (ref) == DF_REF_REG_USE
2857 781291 : && GET_MODE (DF_REF_REG (ref)) == TImode
2858 643701 : && !SUBREG_P (DF_REF_REG (ref)))
2859 643696 : timode_check_non_convertible_regs (candidates, regs,
2860 : DF_REF_REGNO (ref));
2861 : }
2862 :
2863 1063785 : EXECUTE_IF_SET_IN_BITMAP (regs, 0, id, bi)
2864 : {
2865 191467 : for (df_ref def = DF_REG_DEF_CHAIN (id);
2866 392949 : def;
2867 201482 : def = DF_REF_NEXT_REG (def))
2868 201482 : if (bitmap_bit_p (candidates, DF_REF_INSN_UID (def)))
2869 : {
2870 57914 : if (dump_file)
2871 0 : fprintf (dump_file, "Removing insn %d from candidates list\n",
2872 0 : DF_REF_INSN_UID (def));
2873 :
2874 57914 : bitmap_clear_bit (candidates, DF_REF_INSN_UID (def));
2875 57914 : changed = true;
2876 : }
2877 :
2878 191467 : for (df_ref ref = DF_REG_USE_CHAIN (id);
2879 592033 : ref;
2880 400566 : ref = DF_REF_NEXT_REG (ref))
2881 400566 : if (bitmap_bit_p (candidates, DF_REF_INSN_UID (ref)))
2882 : {
2883 18428 : if (dump_file)
2884 0 : fprintf (dump_file, "Removing insn %d from candidates list\n",
2885 0 : DF_REF_INSN_UID (ref));
2886 :
2887 18428 : bitmap_clear_bit (candidates, DF_REF_INSN_UID (ref));
2888 18428 : changed = true;
2889 : }
2890 : }
2891 : } while (changed);
2892 :
2893 850474 : BITMAP_FREE (regs);
2894 850474 : }
2895 :
2896 : /* Main STV pass function. Find and convert scalar
2897 : instructions into vector mode when profitable. */
2898 :
2899 : static unsigned int
2900 1827391 : convert_scalars_to_vector (bool timode_p)
2901 : {
2902 1827391 : basic_block bb;
2903 1827391 : int converted_insns = 0;
2904 1827391 : auto_vec<rtx_insn *> control_flow_insns;
2905 :
2906 1827391 : bitmap_obstack_initialize (NULL);
2907 1827391 : const machine_mode cand_mode[3] = { SImode, DImode, TImode };
2908 1827391 : const machine_mode cand_vmode[3] = { V4SImode, V2DImode, V1TImode };
2909 5482173 : bitmap_head candidates[3]; /* { SImode, DImode, TImode } */
2910 7309564 : for (unsigned i = 0; i < 3; ++i)
2911 5482173 : bitmap_initialize (&candidates[i], &bitmap_default_obstack);
2912 :
2913 1827391 : calculate_dominance_info (CDI_DOMINATORS);
2914 1827391 : df_set_flags (DF_DEFER_INSN_RESCAN | DF_RD_PRUNE_DEAD_DEFS);
2915 1827391 : df_chain_add_problem (DF_DU_CHAIN | DF_UD_CHAIN);
2916 1827391 : df_analyze ();
2917 :
2918 : /* Find all instructions we want to convert into vector mode. */
2919 1827391 : if (dump_file)
2920 45 : fprintf (dump_file, "Searching for mode conversion candidates...\n");
2921 :
2922 20258896 : FOR_EACH_BB_FN (bb, cfun)
2923 : {
2924 18431505 : rtx_insn *insn;
2925 251480844 : FOR_BB_INSNS (bb, insn)
2926 233049339 : if (timode_p
2927 233049339 : && timode_scalar_to_vector_candidate_p (insn))
2928 : {
2929 1131799 : if (dump_file)
2930 0 : fprintf (dump_file, " insn %d is marked as a TImode candidate\n",
2931 0 : INSN_UID (insn));
2932 :
2933 1131799 : bitmap_set_bit (&candidates[2], INSN_UID (insn));
2934 : }
2935 231917540 : else if (!timode_p)
2936 : {
2937 : /* Check {SI,DI}mode. */
2938 361749981 : for (unsigned i = 0; i <= 1; ++i)
2939 247978531 : if (general_scalar_to_vector_candidate_p (insn, cand_mode[i]))
2940 : {
2941 12006862 : if (dump_file)
2942 568 : fprintf (dump_file, " insn %d is marked as a %s candidate\n",
2943 284 : INSN_UID (insn), i == 0 ? "SImode" : "DImode");
2944 :
2945 12006862 : bitmap_set_bit (&candidates[i], INSN_UID (insn));
2946 12006862 : break;
2947 : }
2948 : }
2949 : }
2950 :
2951 1827391 : if (timode_p)
2952 850474 : timode_remove_non_convertible_regs (&candidates[2]);
2953 :
2954 5783191 : for (unsigned i = 0; i <= 2; ++i)
2955 4610345 : if (!bitmap_empty_p (&candidates[i]))
2956 : break;
2957 3955800 : else if (i == 2 && dump_file)
2958 23 : fprintf (dump_file, "There are no candidates for optimization.\n");
2959 :
2960 7309564 : for (unsigned i = 0; i <= 2; ++i)
2961 : {
2962 5482173 : auto_bitmap disallowed;
2963 5482173 : bitmap_tree_view (&candidates[i]);
2964 17584009 : while (!bitmap_empty_p (&candidates[i]))
2965 : {
2966 6619663 : unsigned uid = bitmap_first_set_bit (&candidates[i]);
2967 6619663 : scalar_chain *chain;
2968 :
2969 6619663 : if (cand_mode[i] == TImode)
2970 531156 : chain = new timode_scalar_chain;
2971 : else
2972 6088507 : chain = new general_scalar_chain (cand_mode[i], cand_vmode[i]);
2973 :
2974 : /* Find instructions chain we want to convert to vector mode.
2975 : Check all uses and definitions to estimate all required
2976 : conversions. */
2977 6619663 : if (chain->build (&candidates[i], uid, disallowed))
2978 : {
2979 6617824 : if (chain->compute_convert_gain ())
2980 662752 : converted_insns += chain->convert ();
2981 5955072 : else if (dump_file)
2982 136 : fprintf (dump_file, "Chain #%d conversion is not profitable\n",
2983 : chain->chain_id);
2984 : }
2985 :
2986 6619663 : rtx_insn* iter_insn;
2987 6619663 : unsigned int ii;
2988 6626791 : FOR_EACH_VEC_ELT (chain->control_flow_insns, ii, iter_insn)
2989 3578 : control_flow_insns.safe_push (iter_insn);
2990 :
2991 6619663 : delete chain;
2992 : }
2993 5482173 : }
2994 :
2995 1827391 : if (dump_file)
2996 45 : fprintf (dump_file, "Total insns converted: %d\n", converted_insns);
2997 :
2998 7309564 : for (unsigned i = 0; i <= 2; ++i)
2999 5482173 : bitmap_release (&candidates[i]);
3000 1827391 : bitmap_obstack_release (NULL);
3001 1827391 : df_process_deferred_rescans ();
3002 :
3003 : /* Conversion means we may have 128bit register spills/fills
3004 : which require aligned stack. */
3005 1827391 : if (converted_insns)
3006 : {
3007 107476 : if (crtl->stack_alignment_needed < 128)
3008 2466 : crtl->stack_alignment_needed = 128;
3009 107476 : if (crtl->stack_alignment_estimated < 128)
3010 227 : crtl->stack_alignment_estimated = 128;
3011 :
3012 107476 : crtl->stack_realign_needed
3013 107476 : = INCOMING_STACK_BOUNDARY < crtl->stack_alignment_estimated;
3014 107476 : crtl->stack_realign_tried = crtl->stack_realign_needed;
3015 :
3016 107476 : crtl->stack_realign_processed = true;
3017 :
3018 107476 : if (!crtl->drap_reg)
3019 : {
3020 107286 : rtx drap_rtx = targetm.calls.get_drap_rtx ();
3021 :
3022 : /* stack_realign_drap and drap_rtx must match. */
3023 107286 : gcc_assert ((stack_realign_drap != 0) == (drap_rtx != NULL));
3024 :
3025 : /* Do nothing if NULL is returned,
3026 : which means DRAP is not needed. */
3027 107286 : if (drap_rtx != NULL)
3028 : {
3029 0 : crtl->args.internal_arg_pointer = drap_rtx;
3030 :
3031 : /* Call fixup_tail_calls to clean up
3032 : REG_EQUIV note if DRAP is needed. */
3033 0 : fixup_tail_calls ();
3034 : }
3035 : }
3036 :
3037 : /* Fix up DECL_RTL/DECL_INCOMING_RTL of arguments. */
3038 107476 : if (TARGET_64BIT)
3039 68827 : for (tree parm = DECL_ARGUMENTS (current_function_decl);
3040 188473 : parm; parm = DECL_CHAIN (parm))
3041 : {
3042 119646 : if (TYPE_MODE (TREE_TYPE (parm)) != TImode)
3043 103530 : continue;
3044 16116 : if (DECL_RTL_SET_P (parm)
3045 32232 : && GET_MODE (DECL_RTL (parm)) == V1TImode)
3046 : {
3047 621 : rtx r = DECL_RTL (parm);
3048 621 : if (REG_P (r))
3049 621 : SET_DECL_RTL (parm, gen_rtx_SUBREG (TImode, r, 0));
3050 : }
3051 16116 : if (DECL_INCOMING_RTL (parm)
3052 16116 : && GET_MODE (DECL_INCOMING_RTL (parm)) == V1TImode)
3053 : {
3054 0 : rtx r = DECL_INCOMING_RTL (parm);
3055 0 : if (REG_P (r))
3056 0 : DECL_INCOMING_RTL (parm) = gen_rtx_SUBREG (TImode, r, 0);
3057 : }
3058 : }
3059 :
3060 1828519 : if (!control_flow_insns.is_empty ())
3061 : {
3062 1128 : free_dominance_info (CDI_DOMINATORS);
3063 :
3064 1128 : unsigned int i;
3065 1128 : rtx_insn* insn;
3066 5834 : FOR_EACH_VEC_ELT (control_flow_insns, i, insn)
3067 3578 : if (control_flow_insn_p (insn))
3068 : {
3069 : /* Split the block after insn. There will be a fallthru
3070 : edge, which is OK so we keep it. We have to create
3071 : the exception edges ourselves. */
3072 3578 : bb = BLOCK_FOR_INSN (insn);
3073 3578 : split_block (bb, insn);
3074 3578 : rtl_make_eh_edge (NULL, bb, BB_END (bb));
3075 : }
3076 : }
3077 : }
3078 :
3079 1827391 : return 0;
3080 1827391 : }
3081 :
3082 : static unsigned int
3083 76680 : rest_of_handle_insert_vzeroupper (void)
3084 : {
3085 : /* vzeroupper instructions are inserted immediately after reload and
3086 : postreload_cse to clean up after it a little bit to account for possible
3087 : spills from 256bit or 512bit registers. The pass reuses mode switching
3088 : infrastructure by re-running mode insertion pass, so disable entities
3089 : that have already been processed. */
3090 536760 : for (int i = 0; i < MAX_386_ENTITIES; i++)
3091 460080 : ix86_optimize_mode_switching[i] = 0;
3092 :
3093 76680 : ix86_optimize_mode_switching[AVX_U128] = 1;
3094 :
3095 : /* Call optimize_mode_switching. */
3096 76680 : g->get_passes ()->execute_pass_mode_switching ();
3097 :
3098 : /* LRA removes all REG_DEAD/REG_UNUSED notes and normally they
3099 : reappear in the IL only at the start of pass_rtl_dse2, which does
3100 : df_note_add_problem (); df_analyze ();
3101 : The vzeroupper is scheduled after postreload_cse pass and mode
3102 : switching computes the notes as well, the problem is that e.g.
3103 : pass_gcse2 doesn't maintain the notes, see PR113059 and
3104 : PR112760. Remove the notes now to restore status quo ante
3105 : until we figure out how to maintain the notes or what else
3106 : to do. */
3107 76680 : basic_block bb;
3108 76680 : rtx_insn *insn;
3109 410728 : FOR_EACH_BB_FN (bb, cfun)
3110 4297242 : FOR_BB_INSNS (bb, insn)
3111 3963194 : if (NONDEBUG_INSN_P (insn))
3112 : {
3113 2121222 : rtx *pnote = ®_NOTES (insn);
3114 3945736 : while (*pnote != 0)
3115 : {
3116 1824514 : if (REG_NOTE_KIND (*pnote) == REG_DEAD
3117 834904 : || REG_NOTE_KIND (*pnote) == REG_UNUSED)
3118 1304223 : *pnote = XEXP (*pnote, 1);
3119 : else
3120 520291 : pnote = &XEXP (*pnote, 1);
3121 : }
3122 : }
3123 :
3124 76680 : df_remove_problem (df_note);
3125 76680 : df_analyze ();
3126 76680 : return 0;
3127 : }
3128 :
3129 : namespace {
3130 :
3131 : const pass_data pass_data_insert_vzeroupper =
3132 : {
3133 : RTL_PASS, /* type */
3134 : "vzeroupper", /* name */
3135 : OPTGROUP_NONE, /* optinfo_flags */
3136 : TV_MACH_DEP, /* tv_id */
3137 : 0, /* properties_required */
3138 : 0, /* properties_provided */
3139 : 0, /* properties_destroyed */
3140 : 0, /* todo_flags_start */
3141 : TODO_df_finish, /* todo_flags_finish */
3142 : };
3143 :
3144 : class pass_insert_vzeroupper : public rtl_opt_pass
3145 : {
3146 : public:
3147 297180 : pass_insert_vzeroupper(gcc::context *ctxt)
3148 594360 : : rtl_opt_pass(pass_data_insert_vzeroupper, ctxt)
3149 : {}
3150 :
3151 : /* opt_pass methods: */
3152 1517135 : bool gate (function *) final override
3153 : {
3154 1517135 : return TARGET_AVX && TARGET_VZEROUPPER;
3155 : }
3156 :
3157 76680 : unsigned int execute (function *) final override
3158 : {
3159 76680 : return rest_of_handle_insert_vzeroupper ();
3160 : }
3161 :
3162 : }; // class pass_insert_vzeroupper
3163 :
3164 : const pass_data pass_data_stv =
3165 : {
3166 : RTL_PASS, /* type */
3167 : "stv", /* name */
3168 : OPTGROUP_NONE, /* optinfo_flags */
3169 : TV_MACH_DEP, /* tv_id */
3170 : 0, /* properties_required */
3171 : 0, /* properties_provided */
3172 : 0, /* properties_destroyed */
3173 : 0, /* todo_flags_start */
3174 : TODO_df_finish, /* todo_flags_finish */
3175 : };
3176 :
3177 : class pass_stv : public rtl_opt_pass
3178 : {
3179 : public:
3180 594360 : pass_stv (gcc::context *ctxt)
3181 : : rtl_opt_pass (pass_data_stv, ctxt),
3182 1188720 : timode_p (false)
3183 : {}
3184 :
3185 : /* opt_pass methods: */
3186 3034270 : bool gate (function *) final override
3187 : {
3188 1517135 : return ((!timode_p || TARGET_64BIT)
3189 4424641 : && TARGET_STV && TARGET_SSE2 && optimize > 1);
3190 : }
3191 :
3192 1827391 : unsigned int execute (function *) final override
3193 : {
3194 1827391 : return convert_scalars_to_vector (timode_p);
3195 : }
3196 :
3197 297180 : opt_pass *clone () final override
3198 : {
3199 297180 : return new pass_stv (m_ctxt);
3200 : }
3201 :
3202 594360 : void set_pass_param (unsigned int n, bool param) final override
3203 : {
3204 594360 : gcc_assert (n == 0);
3205 594360 : timode_p = param;
3206 594360 : }
3207 :
3208 : private:
3209 : bool timode_p;
3210 : }; // class pass_stv
3211 :
3212 : } // anon namespace
3213 :
3214 : rtl_opt_pass *
3215 297180 : make_pass_insert_vzeroupper (gcc::context *ctxt)
3216 : {
3217 297180 : return new pass_insert_vzeroupper (ctxt);
3218 : }
3219 :
3220 : rtl_opt_pass *
3221 297180 : make_pass_stv (gcc::context *ctxt)
3222 : {
3223 297180 : return new pass_stv (ctxt);
3224 : }
3225 :
3226 : /* Inserting ENDBR and pseudo patchable-area instructions. */
3227 :
3228 : static void
3229 199164 : rest_of_insert_endbr_and_patchable_area (bool need_endbr,
3230 : unsigned int patchable_area_size)
3231 : {
3232 199164 : rtx endbr;
3233 199164 : rtx_insn *insn;
3234 199164 : rtx_insn *endbr_insn = NULL;
3235 199164 : basic_block bb;
3236 :
3237 199164 : if (need_endbr)
3238 : {
3239 : /* Currently emit EB if it's a tracking function, i.e. 'nocf_check'
3240 : is absent among function attributes. Later an optimization will
3241 : be introduced to make analysis if an address of a static function
3242 : is taken. A static function whose address is not taken will get
3243 : a nocf_check attribute. This will allow to reduce the number of
3244 : EB. */
3245 199119 : if (!lookup_attribute ("nocf_check",
3246 199119 : TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
3247 199119 : && (lookup_attribute ("cf_check", DECL_ATTRIBUTES (cfun->decl))
3248 199090 : || (!flag_manual_endbr
3249 199088 : && (!cgraph_node::get (cfun->decl)->only_called_directly_p ()
3250 28445 : || ix86_cmodel == CM_LARGE
3251 28444 : || ix86_cmodel == CM_LARGE_PIC
3252 28443 : || flag_force_indirect_call
3253 28443 : || (TARGET_DLLIMPORT_DECL_ATTRIBUTES
3254 : && DECL_DLLIMPORT_P (cfun->decl))))))
3255 : {
3256 170657 : if (crtl->profile && flag_fentry)
3257 : {
3258 : /* Queue ENDBR insertion to x86_function_profiler.
3259 : NB: Any patchable-area insn will be inserted after
3260 : ENDBR. */
3261 6 : cfun->machine->insn_queued_at_entrance = TYPE_ENDBR;
3262 : }
3263 : else
3264 : {
3265 170651 : endbr = gen_nop_endbr ();
3266 170651 : bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
3267 170651 : rtx_insn *insn = BB_HEAD (bb);
3268 170651 : endbr_insn = emit_insn_before (endbr, insn);
3269 : }
3270 : }
3271 : }
3272 :
3273 199164 : if (patchable_area_size)
3274 : {
3275 51 : if (crtl->profile && flag_fentry)
3276 : {
3277 : /* Queue patchable-area insertion to x86_function_profiler.
3278 : NB: If there is a queued ENDBR, x86_function_profiler
3279 : will also handle patchable-area. */
3280 2 : if (!cfun->machine->insn_queued_at_entrance)
3281 1 : cfun->machine->insn_queued_at_entrance = TYPE_PATCHABLE_AREA;
3282 : }
3283 : else
3284 : {
3285 49 : rtx patchable_area
3286 49 : = gen_patchable_area (GEN_INT (patchable_area_size),
3287 49 : GEN_INT (crtl->patch_area_entry == 0));
3288 49 : if (endbr_insn)
3289 3 : emit_insn_after (patchable_area, endbr_insn);
3290 : else
3291 : {
3292 46 : bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
3293 46 : insn = BB_HEAD (bb);
3294 46 : emit_insn_before (patchable_area, insn);
3295 : }
3296 : }
3297 : }
3298 :
3299 199164 : if (!need_endbr)
3300 : return;
3301 :
3302 199119 : bb = 0;
3303 4171384 : FOR_EACH_BB_FN (bb, cfun)
3304 : {
3305 77901117 : for (insn = BB_HEAD (bb); insn != NEXT_INSN (BB_END (bb));
3306 73928852 : insn = NEXT_INSN (insn))
3307 : {
3308 73928852 : if (auto call_insn = dyn_cast<rtx_call_insn *> (insn))
3309 : {
3310 1492732 : need_endbr = find_reg_note (call_insn, REG_SETJMP, NULL) != NULL;
3311 1492732 : if (!need_endbr && !SIBLING_CALL_P (call_insn))
3312 : {
3313 1443327 : rtx call = get_call_rtx_from (call_insn);
3314 1443327 : rtx fnaddr = XEXP (call, 0);
3315 1443327 : tree fndecl = NULL_TREE;
3316 :
3317 : /* Also generate ENDBRANCH for non-tail call which
3318 : may return via indirect branch. */
3319 1443327 : if (SYMBOL_REF_P (XEXP (fnaddr, 0)))
3320 1384431 : fndecl = SYMBOL_REF_DECL (XEXP (fnaddr, 0));
3321 1384431 : if (fndecl == NULL_TREE)
3322 59264 : fndecl = MEM_EXPR (fnaddr);
3323 59264 : if (fndecl
3324 1440864 : && TREE_CODE (TREE_TYPE (fndecl)) != FUNCTION_TYPE
3325 600602 : && TREE_CODE (TREE_TYPE (fndecl)) != METHOD_TYPE)
3326 : fndecl = NULL_TREE;
3327 1443327 : if (fndecl && TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
3328 : {
3329 1398312 : tree fntype = TREE_TYPE (fndecl);
3330 1398312 : if (lookup_attribute ("indirect_return",
3331 1398312 : TYPE_ATTRIBUTES (fntype)))
3332 : need_endbr = true;
3333 : }
3334 : }
3335 1492720 : if (!need_endbr)
3336 1492712 : continue;
3337 : /* Generate ENDBRANCH after CALL, which can return more than
3338 : twice, setjmp-like functions. */
3339 :
3340 20 : endbr = gen_nop_endbr ();
3341 20 : emit_insn_after_setloc (endbr, call_insn,
3342 20 : INSN_LOCATION (call_insn));
3343 20 : continue;
3344 20 : }
3345 :
3346 72436120 : if (JUMP_P (insn) && flag_cet_switch)
3347 : {
3348 15 : rtx target = JUMP_LABEL (insn);
3349 15 : if (target == NULL_RTX || ANY_RETURN_P (target))
3350 6 : continue;
3351 :
3352 : /* Check the jump is a switch table. */
3353 9 : rtx_insn *label = as_a<rtx_insn *> (target);
3354 9 : rtx_insn *table = next_insn (label);
3355 9 : if (table == NULL_RTX || !JUMP_TABLE_DATA_P (table))
3356 5 : continue;
3357 :
3358 : /* For the indirect jump find out all places it jumps and insert
3359 : ENDBRANCH there. It should be done under a special flag to
3360 : control ENDBRANCH generation for switch stmts. */
3361 4 : edge_iterator ei;
3362 4 : edge e;
3363 4 : basic_block dest_blk;
3364 :
3365 42 : FOR_EACH_EDGE (e, ei, bb->succs)
3366 : {
3367 38 : rtx_insn *insn;
3368 :
3369 38 : dest_blk = e->dest;
3370 38 : insn = BB_HEAD (dest_blk);
3371 38 : gcc_assert (LABEL_P (insn));
3372 :
3373 : /* Reuse existing ENDBR. */
3374 38 : rtx_insn *next = next_nonnote_nondebug_insn (insn);
3375 38 : if (next && NONJUMP_INSN_P (next)
3376 30 : && GET_CODE (PATTERN (next)) == UNSPEC_VOLATILE
3377 46 : && XINT (PATTERN (next), 1) == UNSPECV_NOP_ENDBR)
3378 8 : continue;
3379 :
3380 30 : endbr = gen_nop_endbr ();
3381 30 : emit_insn_after (endbr, insn);
3382 : }
3383 4 : continue;
3384 4 : }
3385 :
3386 72436105 : if (LABEL_P (insn) && LABEL_PRESERVE_P (insn))
3387 : {
3388 154564 : endbr = gen_nop_endbr ();
3389 154564 : emit_insn_after (endbr, insn);
3390 154564 : continue;
3391 : }
3392 : }
3393 : }
3394 :
3395 : return;
3396 : }
3397 :
3398 : namespace {
3399 :
3400 : const pass_data pass_data_insert_endbr_and_patchable_area =
3401 : {
3402 : RTL_PASS, /* type. */
3403 : "endbr_and_patchable_area", /* name. */
3404 : OPTGROUP_NONE, /* optinfo_flags. */
3405 : TV_MACH_DEP, /* tv_id. */
3406 : 0, /* properties_required. */
3407 : 0, /* properties_provided. */
3408 : 0, /* properties_destroyed. */
3409 : 0, /* todo_flags_start. */
3410 : 0, /* todo_flags_finish. */
3411 : };
3412 :
3413 : class pass_insert_endbr_and_patchable_area : public rtl_opt_pass
3414 : {
3415 : public:
3416 297180 : pass_insert_endbr_and_patchable_area (gcc::context *ctxt)
3417 594360 : : rtl_opt_pass (pass_data_insert_endbr_and_patchable_area, ctxt)
3418 : {}
3419 :
3420 : /* opt_pass methods: */
3421 1517135 : bool gate (function *) final override
3422 : {
3423 1517135 : need_endbr = (flag_cf_protection & CF_BRANCH) != 0;
3424 1517135 : patchable_area_size = crtl->patch_area_size - crtl->patch_area_entry;
3425 1517135 : return need_endbr || patchable_area_size;
3426 : }
3427 :
3428 199164 : unsigned int execute (function *) final override
3429 : {
3430 199164 : timevar_push (TV_MACH_DEP);
3431 199164 : rest_of_insert_endbr_and_patchable_area (need_endbr,
3432 : patchable_area_size);
3433 199164 : timevar_pop (TV_MACH_DEP);
3434 199164 : return 0;
3435 : }
3436 :
3437 : private:
3438 : bool need_endbr;
3439 : unsigned int patchable_area_size;
3440 : }; // class pass_insert_endbr_and_patchable_area
3441 :
3442 : } // anon namespace
3443 :
3444 : rtl_opt_pass *
3445 297180 : make_pass_insert_endbr_and_patchable_area (gcc::context *ctxt)
3446 : {
3447 297180 : return new pass_insert_endbr_and_patchable_area (ctxt);
3448 : }
3449 :
3450 : bool
3451 6119607 : ix86_rpad_gate ()
3452 : {
3453 6119607 : return (TARGET_AVX
3454 392767 : && TARGET_SSE_PARTIAL_REG_DEPENDENCY
3455 294848 : && TARGET_SSE_MATH
3456 294554 : && optimize
3457 6408958 : && optimize_function_for_speed_p (cfun));
3458 : }
3459 :
3460 : enum x86_cse_kind
3461 : {
3462 : X86_CSE_CONST0_VECTOR,
3463 : X86_CSE_CONSTM1_VECTOR,
3464 : X86_CSE_CONST_VECTOR,
3465 : X86_CSE_VEC_DUP,
3466 : X86_CSE_TLS_GD,
3467 : X86_CSE_TLS_LD_BASE,
3468 : X86_CSE_TLSDESC
3469 : };
3470 :
3471 156570 : struct redundant_pattern
3472 : {
3473 : /* Bitmap of basic blocks with broadcast instructions. */
3474 : auto_bitmap bbs;
3475 : /* Bitmap of broadcast instructions. */
3476 : auto_bitmap insns;
3477 : /* The broadcast inner scalar. */
3478 : rtx val;
3479 : /* The actual redundant source value for UNSPEC_TLSDESC. */
3480 : rtx tlsdesc_val;
3481 : /* The inner scalar mode. */
3482 : machine_mode mode;
3483 : /* The destination mode which can be changed to the integer mode of
3484 : the same time. */
3485 : machine_mode dest_mode;
3486 : /* The instruction which sets the inner scalar. Nullptr if the inner
3487 : scalar is applied to the whole function, instead of within the same
3488 : block. */
3489 : rtx_insn *def_insn;
3490 : /* The widest broadcast source. */
3491 : rtx broadcast_source;
3492 : /* The widest broadcast register. */
3493 : rtx broadcast_reg;
3494 : /* The basic block of the broadcast instruction. */
3495 : basic_block bb;
3496 : /* The number of broadcast instructions with the same inner scalar. */
3497 : unsigned HOST_WIDE_INT count;
3498 : /* The threshold of broadcast instructions with the same inner
3499 : scalar. */
3500 : unsigned int threshold;
3501 : /* The widest broadcast size in bytes. */
3502 : unsigned int size;
3503 : /* Load kind. */
3504 : x86_cse_kind kind;
3505 : };
3506 :
3507 : /* Generate a vector set, DEST = SRC, at entry of the nearest dominator
3508 : for basic block map BBS, which is in the fake loop that contains the
3509 : whole function, so that there is only a single vector set in the
3510 : whole function. If not nullptr, LOAD is a pointer to the load. */
3511 :
3512 : static void
3513 45437 : ix86_place_single_vector_set (rtx dest, rtx src, bitmap bbs,
3514 : redundant_pattern *load = nullptr)
3515 : {
3516 45437 : basic_block bb = nearest_common_dominator_for_set (CDI_DOMINATORS, bbs);
3517 : /* For X86_CSE_VEC_DUP and X86_CSE_CONST_VECTOR, don't place the vector
3518 : set outside of the loop to avoid extra spills. */
3519 45437 : if (!load
3520 44409 : || (load->kind != X86_CSE_VEC_DUP
3521 44409 : && load->kind != X86_CSE_CONST_VECTOR))
3522 : {
3523 24959 : while (bb->loop_father->latch
3524 24959 : != EXIT_BLOCK_PTR_FOR_FN (cfun))
3525 1482 : bb = get_immediate_dominator (CDI_DOMINATORS,
3526 : bb->loop_father->header);
3527 : }
3528 :
3529 45437 : if (CONST_INT_P (src))
3530 11347 : dest = gen_rtx_SUBREG (load->dest_mode, dest, 0);
3531 34090 : else if (CONST_VECTOR_P (src))
3532 : {
3533 : /* The only possible CONST_VECTORs of SRC are CONST0_RTX and
3534 : CONSTM1_RTX. Otherwise,
3535 :
3536 : rtx set = gen_rtx_SET (dest, src);
3537 :
3538 : won't be a valid instruction. CONST0_RTX always works. It
3539 : can comes from:
3540 :
3541 : 1. remove_partial_avx_dependency with LOAD == NULL.
3542 : 2. X86_CSE_VEC_DUP with
3543 :
3544 : (insn 48 58 16 3 (set (reg:V4HI 123)
3545 : (const_vector:V4HI [
3546 : (const_int 0 [0]) repeated x4
3547 : ])) 2065 {*movv4hi_internal} (nil))
3548 :
3549 : 3. X86_CSE_CONST0_VECTOR.
3550 : */
3551 23477 : machine_mode mode = GET_MODE (dest);
3552 23477 : if (!(src == CONST0_RTX (mode)
3553 1697 : || (src == CONSTM1_RTX (mode)
3554 1697 : && load->kind == X86_CSE_CONSTM1_VECTOR)))
3555 0 : gcc_unreachable ();
3556 : }
3557 45437 : rtx set = gen_rtx_SET (dest, src);
3558 :
3559 45437 : rtx_insn *insn = BB_HEAD (bb);
3560 184844 : while (insn && !NONDEBUG_INSN_P (insn))
3561 : {
3562 139411 : if (insn == BB_END (bb))
3563 : {
3564 : insn = NULL;
3565 : break;
3566 : }
3567 139407 : insn = NEXT_INSN (insn);
3568 : }
3569 :
3570 45437 : rtx_insn *set_insn;
3571 45437 : if (insn == BB_HEAD (bb))
3572 : {
3573 0 : set_insn = emit_insn_before (set, insn);
3574 0 : if (dump_file)
3575 : {
3576 0 : fprintf (dump_file, "\nPlace:\n\n");
3577 0 : print_rtl_single (dump_file, set_insn);
3578 0 : fprintf (dump_file, "\nbefore:\n\n");
3579 0 : print_rtl_single (dump_file, insn);
3580 0 : fprintf (dump_file, "\n");
3581 : }
3582 : }
3583 : else
3584 : {
3585 45437 : rtx_insn *after = insn ? PREV_INSN (insn) : BB_END (bb);
3586 45437 : set_insn = emit_insn_after (set, after);
3587 45437 : if (dump_file)
3588 : {
3589 0 : fprintf (dump_file, "\nPlace:\n\n");
3590 0 : print_rtl_single (dump_file, set_insn);
3591 0 : fprintf (dump_file, "\nafter:\n\n");
3592 0 : print_rtl_single (dump_file, after);
3593 0 : fprintf (dump_file, "\n");
3594 : }
3595 : }
3596 :
3597 45437 : if (load && load->kind == X86_CSE_VEC_DUP)
3598 : {
3599 : /* Get the source from LOAD as (reg:SI 99) in
3600 :
3601 : (vec_duplicate:V4SI (reg:SI 99))
3602 :
3603 : */
3604 10613 : rtx inner_scalar = load->val;
3605 : /* Set the source in (vec_duplicate:V4SI (reg:SI 99)). */
3606 10613 : rtx reg = XEXP (src, 0);
3607 10613 : machine_mode reg_mode = GET_MODE (reg);
3608 10613 : if (reg_mode != GET_MODE (inner_scalar))
3609 : {
3610 10475 : if (REG_P (inner_scalar) || MEM_P (inner_scalar))
3611 0 : inner_scalar = gen_rtx_SUBREG (reg_mode, inner_scalar, 0);
3612 10475 : else if (!SCALAR_INT_MODE_P (reg_mode))
3613 : {
3614 : /* For non-int load with integer constant, generate
3615 :
3616 : (set (subreg:SI (reg/v:SF 105 [ f ]) 0)
3617 : (const_int 1313486336 [0x4e4a3600]))
3618 :
3619 : */
3620 1 : gcc_assert (CONST_INT_P (inner_scalar));
3621 1 : unsigned int bits = GET_MODE_BITSIZE (reg_mode);
3622 1 : machine_mode mode = int_mode_for_size (bits, 0).require ();
3623 1 : reg = gen_rtx_SUBREG (mode, reg, 0);
3624 : }
3625 : }
3626 10613 : rtx set = gen_rtx_SET (reg, inner_scalar);
3627 10613 : insn = emit_insn_before (set, set_insn);
3628 10613 : if (dump_file)
3629 : {
3630 0 : fprintf (dump_file, "\nAdd:\n\n");
3631 0 : print_rtl_single (dump_file, insn);
3632 0 : fprintf (dump_file, "\nbefore:\n\n");
3633 0 : print_rtl_single (dump_file, set_insn);
3634 0 : fprintf (dump_file, "\n");
3635 : }
3636 : }
3637 45437 : }
3638 :
3639 : /* At entry of the nearest common dominator for basic blocks with
3640 : conversions/rcp/sqrt/rsqrt/round, generate a single
3641 : vxorps %xmmN, %xmmN, %xmmN
3642 : for all
3643 : vcvtss2sd op, %xmmN, %xmmX
3644 : vcvtsd2ss op, %xmmN, %xmmX
3645 : vcvtsi2ss op, %xmmN, %xmmX
3646 : vcvtsi2sd op, %xmmN, %xmmX
3647 :
3648 : NB: We want to generate only a single vxorps to cover the whole
3649 : function. The LCM algorithm isn't appropriate here since it may
3650 : place a vxorps inside the loop. */
3651 :
3652 : static unsigned int
3653 34333 : remove_partial_avx_dependency (void)
3654 : {
3655 34333 : timevar_push (TV_MACH_DEP);
3656 :
3657 34333 : bitmap_obstack_initialize (NULL);
3658 34333 : bitmap convert_bbs = BITMAP_ALLOC (NULL);
3659 :
3660 34333 : basic_block bb;
3661 34333 : rtx_insn *insn, *set_insn;
3662 34333 : rtx set;
3663 34333 : rtx v4sf_const0 = NULL_RTX;
3664 :
3665 34333 : auto_vec<rtx_insn *> control_flow_insns;
3666 :
3667 : /* We create invalid RTL initially so defer rescans. */
3668 34333 : df_set_flags (DF_DEFER_INSN_RESCAN);
3669 :
3670 313703 : FOR_EACH_BB_FN (bb, cfun)
3671 : {
3672 3508123 : FOR_BB_INSNS (bb, insn)
3673 : {
3674 3228753 : if (!NONDEBUG_INSN_P (insn))
3675 1432602 : continue;
3676 :
3677 1796151 : set = single_set (insn);
3678 1796151 : if (!set)
3679 72005 : continue;
3680 :
3681 1724146 : if (get_attr_avx_partial_xmm_update (insn)
3682 : != AVX_PARTIAL_XMM_UPDATE_TRUE)
3683 1720953 : continue;
3684 :
3685 : /* Convert PARTIAL_XMM_UPDATE_TRUE insns, DF -> SF, SF -> DF,
3686 : SI -> SF, SI -> DF, DI -> SF, DI -> DF, sqrt, rsqrt, rcp,
3687 : round, to vec_dup and vec_merge with subreg. */
3688 3193 : rtx src = SET_SRC (set);
3689 3193 : rtx dest = SET_DEST (set);
3690 3193 : machine_mode dest_mode = GET_MODE (dest);
3691 3193 : bool convert_p = false;
3692 3193 : switch (GET_CODE (src))
3693 : {
3694 3128 : case FLOAT:
3695 3128 : case FLOAT_EXTEND:
3696 3128 : case FLOAT_TRUNCATE:
3697 3128 : case UNSIGNED_FLOAT:
3698 3128 : convert_p = true;
3699 3128 : break;
3700 : default:
3701 : break;
3702 : }
3703 :
3704 : /* Only handle conversion here. */
3705 3128 : machine_mode src_mode
3706 3128 : = convert_p ? GET_MODE (XEXP (src, 0)) : VOIDmode;
3707 3128 : switch (src_mode)
3708 : {
3709 153 : case E_SFmode:
3710 153 : case E_DFmode:
3711 153 : if (TARGET_USE_VECTOR_FP_CONVERTS
3712 147 : || !TARGET_SSE_PARTIAL_REG_FP_CONVERTS_DEPENDENCY)
3713 8 : continue;
3714 : break;
3715 2975 : case E_SImode:
3716 2975 : case E_DImode:
3717 2975 : if (TARGET_USE_VECTOR_CONVERTS
3718 2963 : || !TARGET_SSE_PARTIAL_REG_CONVERTS_DEPENDENCY)
3719 14 : continue;
3720 : break;
3721 65 : case E_VOIDmode:
3722 65 : gcc_assert (!convert_p);
3723 : break;
3724 0 : default:
3725 0 : gcc_unreachable ();
3726 : }
3727 :
3728 3171 : if (!v4sf_const0)
3729 1028 : v4sf_const0 = gen_reg_rtx (V4SFmode);
3730 :
3731 3171 : rtx zero;
3732 3171 : machine_mode dest_vecmode;
3733 3171 : switch (dest_mode)
3734 : {
3735 50 : case E_HFmode:
3736 50 : dest_vecmode = V8HFmode;
3737 50 : zero = gen_rtx_SUBREG (V8HFmode, v4sf_const0, 0);
3738 50 : break;
3739 : case E_SFmode:
3740 : dest_vecmode = V4SFmode;
3741 : zero = v4sf_const0;
3742 : break;
3743 1186 : case E_DFmode:
3744 1186 : dest_vecmode = V2DFmode;
3745 1186 : zero = gen_rtx_SUBREG (V2DFmode, v4sf_const0, 0);
3746 1186 : break;
3747 0 : default:
3748 0 : gcc_unreachable ();
3749 : }
3750 :
3751 : /* Change source to vector mode. */
3752 3171 : src = gen_rtx_VEC_DUPLICATE (dest_vecmode, src);
3753 3171 : src = gen_rtx_VEC_MERGE (dest_vecmode, src, zero,
3754 : GEN_INT (HOST_WIDE_INT_1U));
3755 : /* Change destination to vector mode. */
3756 3171 : rtx vec = gen_reg_rtx (dest_vecmode);
3757 : /* Generate an XMM vector SET. */
3758 3171 : set = gen_rtx_SET (vec, src);
3759 3171 : set_insn = emit_insn_before (set, insn);
3760 :
3761 3171 : if (cfun->can_throw_non_call_exceptions)
3762 : {
3763 : /* Handle REG_EH_REGION note. */
3764 0 : rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
3765 0 : if (note)
3766 : {
3767 0 : control_flow_insns.safe_push (set_insn);
3768 0 : add_reg_note (set_insn, REG_EH_REGION, XEXP (note, 0));
3769 : }
3770 : }
3771 :
3772 3171 : src = gen_rtx_SUBREG (dest_mode, vec, 0);
3773 3171 : set = gen_rtx_SET (dest, src);
3774 :
3775 : /* Drop possible dead definitions. */
3776 3171 : PATTERN (insn) = set;
3777 :
3778 3171 : INSN_CODE (insn) = -1;
3779 3171 : recog_memoized (insn);
3780 3171 : df_insn_rescan (insn);
3781 3171 : bitmap_set_bit (convert_bbs, bb->index);
3782 : }
3783 : }
3784 :
3785 34333 : if (v4sf_const0)
3786 : {
3787 : /* (Re-)discover loops so that bb->loop_father can be used in the
3788 : analysis below. */
3789 1028 : calculate_dominance_info (CDI_DOMINATORS);
3790 1028 : loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
3791 :
3792 1028 : ix86_place_single_vector_set (v4sf_const0,
3793 : CONST0_RTX (V4SFmode),
3794 : convert_bbs);
3795 :
3796 1028 : loop_optimizer_finalize ();
3797 :
3798 34333 : if (!control_flow_insns.is_empty ())
3799 : {
3800 0 : free_dominance_info (CDI_DOMINATORS);
3801 :
3802 0 : unsigned int i;
3803 0 : FOR_EACH_VEC_ELT (control_flow_insns, i, insn)
3804 0 : if (control_flow_insn_p (insn))
3805 : {
3806 : /* Split the block after insn. There will be a fallthru
3807 : edge, which is OK so we keep it. We have to create
3808 : the exception edges ourselves. */
3809 0 : bb = BLOCK_FOR_INSN (insn);
3810 0 : split_block (bb, insn);
3811 0 : rtl_make_eh_edge (NULL, bb, BB_END (bb));
3812 : }
3813 : }
3814 : }
3815 :
3816 34333 : df_process_deferred_rescans ();
3817 34333 : df_clear_flags (DF_DEFER_INSN_RESCAN);
3818 34333 : bitmap_obstack_release (NULL);
3819 34333 : BITMAP_FREE (convert_bbs);
3820 :
3821 34333 : timevar_pop (TV_MACH_DEP);
3822 34333 : return 0;
3823 34333 : }
3824 :
3825 : namespace {
3826 :
3827 : const pass_data pass_data_remove_partial_avx_dependency =
3828 : {
3829 : RTL_PASS, /* type */
3830 : "rpad", /* name */
3831 : OPTGROUP_NONE, /* optinfo_flags */
3832 : TV_MACH_DEP, /* tv_id */
3833 : 0, /* properties_required */
3834 : 0, /* properties_provided */
3835 : 0, /* properties_destroyed */
3836 : 0, /* todo_flags_start */
3837 : 0, /* todo_flags_finish */
3838 : };
3839 :
3840 : class pass_remove_partial_avx_dependency : public rtl_opt_pass
3841 : {
3842 : public:
3843 297180 : pass_remove_partial_avx_dependency (gcc::context *ctxt)
3844 594360 : : rtl_opt_pass (pass_data_remove_partial_avx_dependency, ctxt)
3845 : {}
3846 :
3847 : /* opt_pass methods: */
3848 1517135 : bool gate (function *) final override
3849 : {
3850 1517135 : return ix86_rpad_gate ();
3851 : }
3852 :
3853 34333 : unsigned int execute (function *) final override
3854 : {
3855 34333 : return remove_partial_avx_dependency ();
3856 : }
3857 : }; // class pass_rpad
3858 :
3859 : } // anon namespace
3860 :
3861 : rtl_opt_pass *
3862 297180 : make_pass_remove_partial_avx_dependency (gcc::context *ctxt)
3863 : {
3864 297180 : return new pass_remove_partial_avx_dependency (ctxt);
3865 : }
3866 :
3867 : /* Return a machine mode suitable for vector SIZE with SMODE inner
3868 : mode. */
3869 :
3870 : static machine_mode
3871 66760 : ix86_get_vector_cse_mode (unsigned int size, machine_mode smode)
3872 : {
3873 : /* Use the inner scalar mode of vector broadcast source in:
3874 :
3875 : (set (reg:V8DF 394)
3876 : (vec_duplicate:V8DF (reg:V2DF 190 [ alpha ])))
3877 :
3878 : to compute the vector mode for broadcast from vector source.
3879 : */
3880 66760 : if (VECTOR_MODE_P (smode))
3881 32275 : smode = GET_MODE_INNER (smode);
3882 66760 : scalar_mode s_mode = as_a <scalar_mode> (smode);
3883 133520 : poly_uint64 nunits = size / GET_MODE_SIZE (smode);
3884 66760 : machine_mode mode = mode_for_vector (s_mode, nunits).require ();
3885 66760 : return mode;
3886 : }
3887 :
3888 : /* Replace the source operand of instructions in VECTOR_INSNS with
3889 : VECTOR_CONST in VECTOR_MODE. */
3890 :
3891 : static void
3892 66117 : replace_vector_const (machine_mode vector_mode, rtx vector_const,
3893 : auto_bitmap &vector_insns,
3894 : machine_mode scalar_mode)
3895 : {
3896 66117 : bitmap_iterator bi;
3897 66117 : unsigned int id;
3898 :
3899 232594 : EXECUTE_IF_SET_IN_BITMAP (vector_insns, 0, id, bi)
3900 : {
3901 166477 : rtx_insn *insn = DF_INSN_UID_GET (id)->insn;
3902 :
3903 : /* Get the single SET instruction. */
3904 166477 : rtx set = single_set (insn);
3905 166477 : rtx src = SET_SRC (set);
3906 166477 : rtx dest = SET_DEST (set);
3907 166477 : machine_mode mode = GET_MODE (dest);
3908 :
3909 166477 : rtx replace;
3910 : /* Replace the source operand with VECTOR_CONST. */
3911 166477 : if (SUBREG_P (src)
3912 166477 : || mode == vector_mode
3913 61344 : || CONST_INT_P (vector_const))
3914 : replace = vector_const;
3915 : else
3916 : {
3917 61337 : unsigned int size = GET_MODE_SIZE (mode);
3918 61337 : if (size < ix86_regmode_natural_size (mode))
3919 : {
3920 : /* If the mode size is smaller than its natural size,
3921 : first insert an extra move with a QI vector SUBREG
3922 : of the same size to avoid validate_subreg failure. */
3923 643 : machine_mode vmode
3924 643 : = ix86_get_vector_cse_mode (size, scalar_mode);
3925 643 : rtx vreg;
3926 643 : if (mode == vmode)
3927 : vreg = vector_const;
3928 : else
3929 : {
3930 135 : vreg = gen_reg_rtx (vmode);
3931 135 : rtx vsubreg = gen_rtx_SUBREG (vmode, vector_const, 0);
3932 135 : rtx pat = gen_rtx_SET (vreg, vsubreg);
3933 135 : rtx_insn *vinsn = emit_insn_before (pat, insn);
3934 135 : if (dump_file)
3935 : {
3936 0 : fprintf (dump_file, "\nInsert an extra move:\n\n");
3937 0 : print_rtl_single (dump_file, vinsn);
3938 0 : fprintf (dump_file, "\nbefore:\n\n");
3939 0 : print_rtl_single (dump_file, insn);
3940 0 : fprintf (dump_file, "\n");
3941 : }
3942 : }
3943 643 : replace = gen_rtx_SUBREG (mode, vreg, 0);
3944 : }
3945 : else
3946 60694 : replace = gen_rtx_SUBREG (mode, vector_const, 0);
3947 : }
3948 :
3949 166477 : if (dump_file)
3950 : {
3951 0 : fprintf (dump_file, "\nReplace:\n\n");
3952 0 : print_rtl_single (dump_file, insn);
3953 : }
3954 166477 : SET_SRC (set) = replace;
3955 166477 : if (CONST_INT_P (replace))
3956 : {
3957 24430 : dest = gen_lowpart (scalar_mode, dest);
3958 24430 : SET_DEST (set) = dest;
3959 : }
3960 : /* Drop possible dead definitions. */
3961 166477 : PATTERN (insn) = set;
3962 166477 : INSN_CODE (insn) = -1;
3963 166477 : recog_memoized (insn);
3964 : /* Add a REG_EQUAL note with the original CONST_VECTOR so that LRA
3965 : can rematerialize it instead of spilling DEST. */
3966 166477 : if (REG_P (dest) && !CONST_INT_P (replace) && CONST_VECTOR_P (src))
3967 90256 : set_unique_reg_note (insn, REG_EQUAL, copy_rtx (src));
3968 166477 : if (dump_file)
3969 : {
3970 0 : fprintf (dump_file, "\nwith:\n\n");
3971 0 : print_rtl_single (dump_file, insn);
3972 0 : fprintf (dump_file, "\n");
3973 : }
3974 166477 : df_insn_rescan (insn);
3975 : }
3976 66117 : }
3977 :
3978 : /* Return the inner scalar if OP is a broadcast, else return nullptr. */
3979 :
3980 : static rtx
3981 2220655 : ix86_broadcast_inner (rtx op, machine_mode mode,
3982 : machine_mode *scalar_mode_p,
3983 : x86_cse_kind *kind_p, rtx_insn **insn_p)
3984 : {
3985 2220655 : switch (standard_sse_constant_p (op, mode))
3986 : {
3987 119749 : case 1:
3988 119749 : *scalar_mode_p = QImode;
3989 119749 : *kind_p = X86_CSE_CONST0_VECTOR;
3990 119749 : *insn_p = nullptr;
3991 119749 : return const0_rtx;
3992 12914 : case 2:
3993 12914 : *scalar_mode_p = QImode;
3994 12914 : *kind_p = X86_CSE_CONSTM1_VECTOR;
3995 12914 : *insn_p = nullptr;
3996 12914 : return constm1_rtx;
3997 2087992 : default:
3998 2087992 : break;
3999 : }
4000 :
4001 2087992 : mode = GET_MODE (op);
4002 2087992 : int nunits = GET_MODE_NUNITS (mode);
4003 2087992 : if (nunits < 2)
4004 : return nullptr;
4005 :
4006 1581095 : bool const_vector_p = CONST_VECTOR_P (op);
4007 1581095 : bool duplicated = GET_CODE (op) == VEC_DUPLICATE;
4008 1581095 : rtx orig_op = op;
4009 1581095 : if (!const_vector_p)
4010 : {
4011 : /* Check CONST_VECTOR in REG_EQUAL note. */
4012 1581075 : rtx equal = find_reg_equal_equiv_note (*insn_p);
4013 1581075 : if (equal)
4014 : {
4015 386208 : equal = XEXP (equal, 0);
4016 386208 : const_vector_p = CONST_VECTOR_P (equal);
4017 : /* Use CONST_VECTOR in REG_EQUAL note. */
4018 386208 : if (const_vector_p)
4019 : {
4020 : /* Handle REG_EQUAL note in:
4021 :
4022 : (insn 7 5 12 2 (set (subreg:V8SI (reg:V4DI 100) 0)
4023 : (vec_duplicate:V8SI (reg:SI 102)))
4024 : (expr_list:REG_DEAD (reg:SI 102)
4025 : (expr_list:REG_EQUAL (const_vector:V4DI [
4026 : (const_int -1 [0xffffffffffffffff]) repeated x4]) (nil))))
4027 :
4028 : NB: Don't treat it as CONST_VECTOR since EQUAL isn't
4029 : supported by ISAs as in gcc.target/i386/pr40957.c. */
4030 273651 : if (GET_MODE (equal) != mode)
4031 : const_vector_p = false;
4032 : else
4033 1581095 : op = equal;
4034 : }
4035 : }
4036 : }
4037 :
4038 1581095 : machine_mode inner_mode = GET_MODE_INNER (mode);
4039 :
4040 1581095 : if (const_vector_p)
4041 : {
4042 547268 : bool int_load_p = GET_MODE_SIZE (mode) <= UNITS_PER_WORD;
4043 273634 : *kind_p = X86_CSE_CONST_VECTOR;
4044 273634 : if (int_load_p)
4045 : {
4046 : /* This CONST_VECTOR load can be converted to constant
4047 : integer load. */
4048 36420 : *scalar_mode_p = mode;
4049 36420 : *insn_p = nullptr;
4050 36420 : return op;
4051 : }
4052 :
4053 : /* This CONST_VECTOR is wider than the integer register. */
4054 237214 : rtx first = XVECEXP (op, 0, 0);
4055 :
4056 237214 : if (duplicated)
4057 : {
4058 : /* Check if CONST_VECTOR in REG_EQUAL note is duplicated in
4059 :
4060 : (insn 10 7 12 2 (set (reg:V8SI 128)
4061 : (vec_duplicate:V8SI (vec_select:V2SI (reg:V4SI 180)
4062 : (parallel [(const_int 0 [0])
4063 : (const_int 1 [0x1])]))))
4064 : (expr_list:REG_EQUAL (const_vector:V8SI [
4065 : (const_int 0 [0])
4066 : (const_int 34 [0x22])
4067 : (const_int 0 [0])
4068 : (const_int 34 [0x22])
4069 : (const_int 0 [0])
4070 : (const_int 34 [0x22])
4071 : (const_int 0 [0])
4072 : (const_int 34 [0x22])])(nil)))
4073 :
4074 : */
4075 :
4076 205236 : bool duplicated_const_vector = true;
4077 205236 : for (int i = 1; i < nunits; ++i)
4078 : {
4079 133820 : rtx tmp = XVECEXP (op, 0, i);
4080 133820 : if (!rtx_equal_p (tmp, first))
4081 : {
4082 : duplicated_const_vector = false;
4083 : break;
4084 : }
4085 : }
4086 :
4087 71432 : if (duplicated_const_vector)
4088 : {
4089 71416 : bool const_double_p = CONST_DOUBLE_P (first);
4090 : /* Force the floating point constant to memory. */
4091 71416 : if (const_double_p)
4092 1257 : first = validize_mem (force_const_mem (inner_mode, first));
4093 :
4094 71416 : if (const_double_p || CONST_INT_P (first))
4095 : {
4096 : /* Handle
4097 :
4098 : (insn 7 6 8 2 (set (reg:V4SF 99)
4099 : (vec_duplicate:V4SF (mem/u/c:SF (symbol_ref/u:DI ("*.LC2") [flags 0x2]) [0 S4 A32])))
4100 : (expr_list:REG_EQUAL (const_vector:V4SF [
4101 : (const_double:SF 3.4e+1 [0x0.88p+6]) repeated x4]) (nil)))
4102 :
4103 : and
4104 :
4105 : (insn 14 15 16 3 (set (reg:V4SI 116)
4106 : (vec_duplicate:V4SI (reg:SI 117)))
4107 : (expr_list:REG_EQUAL (const_vector:V4SI [
4108 : (const_int 34 [0x22]) repeated x4]) (nil)))
4109 :
4110 : */
4111 71416 : *kind_p = X86_CSE_VEC_DUP;
4112 71416 : *insn_p = nullptr;
4113 71416 : *scalar_mode_p = inner_mode;
4114 71416 : return first;
4115 : }
4116 : }
4117 :
4118 : op = orig_op;
4119 : }
4120 : else
4121 : {
4122 : /* Only native CONST_VECTOR is allowed. */
4123 165782 : if (orig_op != op)
4124 : return nullptr;
4125 :
4126 : /* Check if VEC_DUPLICATE can be used. */
4127 48 : for (int i = 1; i < nunits; ++i)
4128 : {
4129 48 : rtx tmp = XVECEXP (op, 0, i);
4130 : /* Vector duplicate value. */
4131 48 : if (!rtx_equal_p (tmp, first))
4132 : return nullptr;
4133 : }
4134 :
4135 : /* Use the inner mode to handle
4136 : (const_vector:V2QI [(const_int 0 [0]) repeated x2])
4137 : */
4138 0 : *scalar_mode_p = inner_mode;
4139 0 : *insn_p = nullptr;
4140 0 : return first;
4141 : }
4142 : }
4143 :
4144 1307477 : if (!duplicated)
4145 : return nullptr;
4146 :
4147 23493 : *kind_p = X86_CSE_VEC_DUP;
4148 :
4149 : /* Only
4150 :
4151 : (vec_duplicate:V4SI (reg:SI 99))
4152 : (vec_duplicate:V2DF (mem/u/c:DF (symbol_ref/u:DI ("*.LC1") [flags 0x2]) [0 S8 A64]))
4153 :
4154 : are supported. Set OP to the broadcast source by default. */
4155 23493 : op = XEXP (op, 0);
4156 23493 : rtx reg = op;
4157 23493 : if (SUBREG_P (op)
4158 414 : && SUBREG_BYTE (op) == 0
4159 23907 : && !paradoxical_subreg_p (op))
4160 414 : reg = SUBREG_REG (op);
4161 23493 : if (!REG_P (reg))
4162 : {
4163 2354 : if (MEM_P (op)
4164 2084 : && SYMBOL_REF_P (XEXP (op, 0))
4165 2629 : && CONSTANT_POOL_ADDRESS_P (XEXP (op, 0)))
4166 : {
4167 : /* Handle constant broadcast from memory. */
4168 10 : *scalar_mode_p = inner_mode;
4169 10 : *insn_p = nullptr;
4170 10 : return op;
4171 : }
4172 : return nullptr;
4173 : }
4174 :
4175 21139 : machine_mode orig_mode = mode;
4176 21139 : mode = GET_MODE (op);
4177 :
4178 : /* Only single def chain is supported. */
4179 21139 : df_ref ref = DF_REG_DEF_CHAIN (REGNO (reg));
4180 21139 : if (!ref
4181 21139 : || DF_REF_IS_ARTIFICIAL (ref)
4182 21139 : || DF_REF_NEXT_REG (ref) != nullptr)
4183 : return nullptr;
4184 :
4185 15459 : rtx_insn *insn = DF_REF_INSN (ref);
4186 :
4187 : /* Skip JUMP which happens with asm goto. */
4188 15459 : if (JUMP_P (insn))
4189 : return nullptr;
4190 :
4191 15457 : rtx set = single_set (insn);
4192 15457 : if (!set)
4193 : return nullptr;
4194 :
4195 15447 : rtx src = SET_SRC (set);
4196 :
4197 15447 : if (CONST_INT_P (src))
4198 : {
4199 : /* Handle sequences like
4200 :
4201 : (set (subreg:SI (reg/v:SF 105 [ f ]) 0)
4202 : (const_int 0 [0]))
4203 : (set (reg:V4SF 110)
4204 : (vec_duplicate:V4SF (reg/v:SF 105 [ f ])))
4205 :
4206 : and
4207 :
4208 : (set (reg:SI 99)
4209 : (const_int 34 [0x22]))
4210 : (set (reg:V4SI 98)
4211 : (vec_duplicate:V4SI (reg:SI 99)))
4212 :
4213 : Set *INSN_P to nullptr and return SET_SRC if SET_SRC is an
4214 : integer constant. */
4215 231 : op = src;
4216 231 : if (SCALAR_INT_MODE_P (mode) && mode != GET_MODE (reg))
4217 0 : op = gen_int_mode (INTVAL (src), mode);
4218 231 : if (op == const0_rtx)
4219 : {
4220 4 : if (standard_sse_constant_p (CONST0_RTX (orig_mode),
4221 : orig_mode) == 1)
4222 : {
4223 4 : *scalar_mode_p = QImode;
4224 4 : *kind_p = X86_CSE_CONST0_VECTOR;
4225 4 : *insn_p = nullptr;
4226 4 : return const0_rtx;
4227 : }
4228 0 : op = CONST0_RTX (mode);
4229 : }
4230 227 : else if (op == constm1_rtx
4231 227 : && standard_sse_constant_p (CONSTM1_RTX (orig_mode),
4232 : orig_mode) == 2)
4233 : {
4234 0 : *scalar_mode_p = QImode;
4235 0 : *kind_p = X86_CSE_CONSTM1_VECTOR;
4236 0 : *insn_p = nullptr;
4237 0 : return constm1_rtx;
4238 : }
4239 :
4240 : /* Check if we can convert:
4241 :
4242 : (insn 14 465 412 3 (set (reg:SI 507 [ j_lsm.26 ])
4243 : (const_int 2 [0x2])) "foo.c":10:12 discrim 2 100 {*movsi_internal} (nil))
4244 : ...
4245 : (insn 518 507 434 16 (set (reg:V2SI 493)
4246 : (vec_duplicate:V2SI (reg:SI 507 [ j_lsm.26 ]))) 2395 {*vec_dupv2si} (nil))
4247 :
4248 : to constant integer load:
4249 :
4250 : (insn 566 55 56 6 (set (subreg:DI (reg:V2SI 517) 0)
4251 : (const_int 8589934594 [0x200000002])) -1 (nil))
4252 : ...
4253 : (insn 518 507 434 16 (set (reg:V2SI 493)
4254 : (reg:V2SI 517)) 2066 {*movv2si_internal} (nil))
4255 :
4256 : */
4257 454 : if (GET_MODE_SIZE (orig_mode) <= UNITS_PER_WORD)
4258 17 : *kind_p = X86_CSE_CONST_VECTOR;
4259 :
4260 227 : *insn_p = nullptr;
4261 : }
4262 : else
4263 : {
4264 : /* Handle sequences like
4265 :
4266 : (set (reg:QI 105 [ c ])
4267 : (reg:QI 5 di [ c ]))
4268 : (set (reg:V64QI 102 [ _1 ])
4269 : (vec_duplicate:V64QI (reg:QI 105 [ c ])))
4270 :
4271 : (set (reg/v:SI 116 [ argc ])
4272 : (mem/c:SI (reg:SI 135) [2 argc+0 S4 A32]))
4273 : (set (reg:V4SI 119 [ _45 ])
4274 : (vec_duplicate:V4SI (reg/v:SI 116 [ argc ])))
4275 :
4276 : (set (reg:SI 98 [ _1 ])
4277 : (sign_extend:SI (reg:QI 106 [ c ])))
4278 : (set (reg:V16SI 103 [ _2 ])
4279 : (vec_duplicate:V16SI (reg:SI 98 [ _1 ])))
4280 :
4281 : (set (reg:SI 102 [ cost ])
4282 : (mem/c:SI (symbol_ref:DI ("cost") [flags 0x40])))
4283 : (set (reg:V4HI 103 [ _16 ])
4284 : (vec_duplicate:V4HI (subreg:HI (reg:SI 102 [ cost ]) 0)))
4285 :
4286 : (set (subreg:SI (reg/v:HI 107 [ cr_val ]) 0)
4287 : (ashift:SI (reg:SI 158)
4288 : (subreg:QI (reg:SI 156 [ _2 ]) 0)))
4289 : (set (reg:V16HI 183 [ _61 ])
4290 : (vec_duplicate:V16HI (reg/v:HI 107 [ cr_val ])))
4291 :
4292 : Set *INSN_P to INSN and return the broadcast source otherwise. */
4293 15216 : *insn_p = insn;
4294 : }
4295 :
4296 15443 : *scalar_mode_p = mode;
4297 15443 : return op;
4298 : }
4299 :
4300 : /* Replace CALL instruction in TLS_CALL_INSNS with SET from SRC and
4301 : put the updated instruction in UPDATED_TLS_INSNS. */
4302 :
4303 : static void
4304 323 : replace_tls_call (rtx src, auto_bitmap &tls_call_insns,
4305 : auto_bitmap &updated_tls_insns)
4306 : {
4307 323 : bitmap_iterator bi;
4308 323 : unsigned int id;
4309 :
4310 1763 : EXECUTE_IF_SET_IN_BITMAP (tls_call_insns, 0, id, bi)
4311 : {
4312 1440 : rtx_insn *insn = DF_INSN_UID_GET (id)->insn;
4313 :
4314 : /* If this isn't a CALL, only GNU2 TLS implicit CALL patterns are
4315 : allowed. */
4316 1440 : if (!CALL_P (insn))
4317 : {
4318 47 : attr_tls64 tls64 = get_attr_tls64 (insn);
4319 47 : if (tls64 != TLS64_CALL && tls64 != TLS64_COMBINE)
4320 0 : gcc_unreachable ();
4321 : }
4322 :
4323 1440 : rtx pat = PATTERN (insn);
4324 1440 : gcc_assert (GET_CODE (pat) == PARALLEL);
4325 1440 : rtx set = XVECEXP (pat, 0, 0);
4326 1440 : gcc_assert (GET_CODE (set) == SET);
4327 1440 : rtx dest = SET_DEST (set);
4328 :
4329 1440 : set = gen_rtx_SET (dest, src);
4330 1440 : rtx_insn *set_insn = emit_insn_after (set, insn);
4331 1440 : if (recog_memoized (set_insn) < 0)
4332 0 : gcc_unreachable ();
4333 :
4334 : /* Put SET_INSN in UPDATED_TLS_INSNS. */
4335 1440 : bitmap_set_bit (updated_tls_insns, INSN_UID (set_insn));
4336 :
4337 1440 : if (dump_file)
4338 : {
4339 0 : fprintf (dump_file, "\nReplace:\n\n");
4340 0 : print_rtl_single (dump_file, insn);
4341 0 : fprintf (dump_file, "\nwith:\n\n");
4342 0 : print_rtl_single (dump_file, set_insn);
4343 0 : fprintf (dump_file, "\n");
4344 : }
4345 :
4346 : /* Delete the CALL insn. */
4347 1440 : delete_insn (insn);
4348 :
4349 1440 : df_insn_rescan (set_insn);
4350 : }
4351 323 : }
4352 :
4353 : /* Return the basic block which dominates all basic blocks which set
4354 : hard register REGNO used in basic block BB. */
4355 :
4356 : static basic_block
4357 2 : ix86_get_dominator_for_reg (unsigned int regno, basic_block bb)
4358 : {
4359 2 : basic_block set_bb;
4360 2 : auto_bitmap set_bbs;
4361 :
4362 : /* Get all BBs which set REGNO and dominate the current BB from all
4363 : DEFs of REGNO. */
4364 2 : for (df_ref def = DF_REG_DEF_CHAIN (regno);
4365 18 : def;
4366 16 : def = DF_REF_NEXT_REG (def))
4367 16 : if (!DF_REF_IS_ARTIFICIAL (def)
4368 16 : && !DF_REF_FLAGS_IS_SET (def, DF_REF_MAY_CLOBBER)
4369 6 : && !DF_REF_FLAGS_IS_SET (def, DF_REF_MUST_CLOBBER))
4370 : {
4371 4 : set_bb = DF_REF_BB (def);
4372 4 : if (dominated_by_p (CDI_DOMINATORS, bb, set_bb))
4373 2 : bitmap_set_bit (set_bbs, set_bb->index);
4374 : }
4375 :
4376 2 : bb = nearest_common_dominator_for_set (CDI_DOMINATORS, set_bbs);
4377 2 : return bb;
4378 2 : }
4379 :
4380 : /* Mark FLAGS register as live in DATA, a bitmap of live caller-saved
4381 : registers, if DEST is FLAGS register. */
4382 :
4383 : static void
4384 388 : ix86_check_flags_reg (rtx dest, const_rtx x, void *data)
4385 : {
4386 388 : if (GET_CODE (x) == CLOBBER)
4387 : return;
4388 :
4389 381 : auto_bitmap *live_caller_saved_regs = (auto_bitmap *) data;
4390 381 : if (REG_P (dest) && REGNO (dest) == FLAGS_REG)
4391 0 : bitmap_set_bit (*live_caller_saved_regs, FLAGS_REG);
4392 : }
4393 :
4394 : /* Emit a TLS_SET instruction of KIND in basic block BB. Store the
4395 : insertion point in *BEFORE_P for emit_insn_before or in *AFTER_P
4396 : for emit_insn_after. UPDATED_GNU_TLS_INSNS contains instructions
4397 : which replace the GNU TLS instructions. UPDATED_GNU2_TLS_INSNS
4398 : contains instructions which replace the GNU2 TLS instructions. */
4399 :
4400 : static rtx_insn *
4401 323 : ix86_emit_tls_call (rtx tls_set, x86_cse_kind kind, basic_block bb,
4402 : rtx_insn **before_p, rtx_insn **after_p,
4403 : auto_bitmap &updated_gnu_tls_insns,
4404 : auto_bitmap &updated_gnu2_tls_insns)
4405 : {
4406 325 : rtx_insn *tls_insn;
4407 :
4408 325 : do
4409 : {
4410 325 : rtx_insn *insn = BB_HEAD (bb);
4411 1296 : while (insn && !NONDEBUG_INSN_P (insn))
4412 : {
4413 975 : if (insn == BB_END (bb))
4414 : {
4415 : /* This must be the beginning basic block:
4416 :
4417 : (note 4 0 2 2 [bb 2] NOTE_INSN_BASIC_BLOCK)
4418 : (note 2 4 26 2 NOTE_INSN_FUNCTION_BEG)
4419 :
4420 : or a basic block with only a label:
4421 :
4422 : (code_label 78 11 77 3 14 (nil) [1 uses])
4423 : (note 77 78 54 3 [bb 3] NOTE_INSN_BASIC_BLOCK)
4424 :
4425 : or a basic block with only a debug marker:
4426 :
4427 : (note 3 0 2 2 [bb 2] NOTE_INSN_BASIC_BLOCK)
4428 : (note 2 3 5 2 NOTE_INSN_FUNCTION_BEG)
4429 : (debug_insn 5 2 16 2 (debug_marker) "x.c":6:3 -1 (nil))
4430 :
4431 : or a basic block with only deleted instructions:
4432 :
4433 : (code_label 348 23 349 45 3 (nil) [0 uses])
4434 : (note 349 348 436 45 [bb 45] NOTE_INSN_BASIC_BLOCK)
4435 : (note 436 349 362 45 NOTE_INSN_DELETED)
4436 :
4437 : */
4438 4 : gcc_assert (DEBUG_INSN_P (insn)
4439 : || (NOTE_P (insn)
4440 : && ((NOTE_KIND (insn)
4441 : == NOTE_INSN_FUNCTION_BEG)
4442 : || (NOTE_KIND (insn)
4443 : == NOTE_INSN_DELETED)
4444 : || (NOTE_KIND (insn)
4445 : == NOTE_INSN_BASIC_BLOCK))));
4446 : insn = NULL;
4447 : break;
4448 : }
4449 971 : insn = NEXT_INSN (insn);
4450 : }
4451 :
4452 : /* TLS_GD and TLS_LD_BASE instructions are normal functions which
4453 : clobber caller-saved registers. TLSDESC instructions only
4454 : clobber FLAGS. If any registers clobbered by TLS instructions
4455 : are live in this basic block, we must insert TLS instructions
4456 : after all live registers clobbered are dead. */
4457 :
4458 325 : auto_bitmap live_caller_saved_regs;
4459 650 : bitmap in = df_live ? DF_LIVE_IN (bb) : DF_LR_IN (bb);
4460 :
4461 325 : if (bitmap_bit_p (in, FLAGS_REG))
4462 4 : bitmap_set_bit (live_caller_saved_regs, FLAGS_REG);
4463 :
4464 325 : unsigned int i;
4465 :
4466 : /* Get all live caller-saved registers for TLS_GD and TLS_LD_BASE
4467 : instructions. */
4468 325 : if (kind != X86_CSE_TLSDESC)
4469 : {
4470 303 : const predefined_function_abi &tls_abi
4471 303 : = ix86_tls_get_addr_abi ();
4472 29088 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
4473 28482 : if (tls_abi.clobbers_full_reg_p (i)
4474 25758 : && !fixed_regs[i]
4475 40953 : && bitmap_bit_p (in, i))
4476 350 : bitmap_set_bit (live_caller_saved_regs, i);
4477 303 : if (df_live && crtl->drap_reg)
4478 : {
4479 : /* DRAP has no reaching definition at this point, so df_live drops
4480 : it above. Its hard register can also go dead mid-function once
4481 : copied elsewhere (e.g. right after the prologue), so query
4482 : DF_LR_IN per-block rather than treating it as live whenever
4483 : crtl->drap_reg is set. */
4484 1 : i = REGNO (crtl->drap_reg);
4485 2 : if (bitmap_bit_p (DF_LR_IN (bb), i))
4486 1 : bitmap_set_bit (live_caller_saved_regs, i);
4487 : }
4488 : }
4489 :
4490 325 : if (bitmap_empty_p (live_caller_saved_regs))
4491 : {
4492 85 : if (insn == BB_HEAD (bb))
4493 : {
4494 0 : *before_p = insn;
4495 0 : tls_insn = emit_insn_before (tls_set, insn);
4496 : }
4497 : else
4498 : {
4499 : /* Emit the TLS call after NOTE_INSN_FUNCTION_BEG in the
4500 : beginning basic block:
4501 :
4502 : (note 4 0 2 2 [bb 2] NOTE_INSN_BASIC_BLOCK)
4503 : (note 2 4 26 2 NOTE_INSN_FUNCTION_BEG)
4504 :
4505 : or after NOTE_INSN_BASIC_BLOCK in a basic block with
4506 : only a label:
4507 :
4508 : (code_label 78 11 77 3 14 (nil) [1 uses])
4509 : (note 77 78 54 3 [bb 3] NOTE_INSN_BASIC_BLOCK)
4510 :
4511 : or after debug marker in a basic block with only a
4512 : debug marker:
4513 :
4514 : (note 3 0 2 2 [bb 2] NOTE_INSN_BASIC_BLOCK)
4515 : (note 2 3 5 2 NOTE_INSN_FUNCTION_BEG)
4516 : (debug_insn 5 2 16 2 (debug_marker) "x.c":6:3 -1 (nil))
4517 :
4518 : */
4519 85 : insn = insn ? PREV_INSN (insn) : BB_END (bb);
4520 85 : *after_p = insn;
4521 85 : tls_insn = emit_insn_after (tls_set, insn);
4522 : }
4523 : return tls_insn;
4524 : }
4525 :
4526 240 : bool repeat = false;
4527 :
4528 : /* Search for REG_DEAD notes in this basic block. */
4529 675 : FOR_BB_INSNS (bb, insn)
4530 : {
4531 675 : if (!NONDEBUG_INSN_P (insn))
4532 290 : continue;
4533 :
4534 : /* NB: Conditional jump is the only instruction which reads
4535 : flags register and changes control flow. We can never
4536 : place the TLS call after unconditional jump. */
4537 385 : if (JUMP_P (insn))
4538 : {
4539 : /* This must be a conditional jump. */
4540 2 : rtx label = JUMP_LABEL (insn);
4541 2 : if (label == nullptr
4542 2 : || ANY_RETURN_P (label)
4543 2 : || !(LABEL_P (label) || SYMBOL_REF_P (label)))
4544 0 : gcc_unreachable ();
4545 :
4546 : /* Place the call before all FLAGS_REG setting BBs since
4547 : we can't place a call before nor after a conditional
4548 : jump. */
4549 2 : bb = ix86_get_dominator_for_reg (FLAGS_REG, bb);
4550 :
4551 : /* Start over again. */
4552 2 : repeat = true;
4553 2 : break;
4554 : }
4555 :
4556 383 : if (bitmap_bit_p (updated_gnu_tls_insns, INSN_UID (insn)))
4557 : {
4558 : /* Insert the __tls_get_addr call before INSN which
4559 : replaces a __tls_get_addr call. */
4560 1 : *before_p = insn;
4561 1 : tls_insn = emit_insn_before (tls_set, insn);
4562 1 : return tls_insn;
4563 : }
4564 :
4565 382 : if (bitmap_bit_p (updated_gnu2_tls_insns, INSN_UID (insn)))
4566 : {
4567 : /* Mark FLAGS register as dead since FLAGS register
4568 : would be clobbered by the GNU2 TLS instruction. */
4569 1 : bitmap_clear_bit (live_caller_saved_regs, FLAGS_REG);
4570 1 : continue;
4571 : }
4572 :
4573 : /* Check if FLAGS register is live. */
4574 381 : note_stores (insn, ix86_check_flags_reg,
4575 : &live_caller_saved_regs);
4576 :
4577 381 : rtx link;
4578 522 : for (link = REG_NOTES (insn); link; link = XEXP (link, 1))
4579 378 : if ((REG_NOTE_KIND (link) == REG_DEAD
4580 9 : || (REG_NOTE_KIND (link) == REG_UNUSED
4581 7 : && REGNO (XEXP (link, 0)) == FLAGS_REG))
4582 385 : && REG_P (XEXP (link, 0)))
4583 : {
4584 : /* Mark the live caller-saved register as dead. */
4585 757 : for (i = REGNO (XEXP (link, 0));
4586 757 : i < END_REGNO (XEXP (link, 0));
4587 : i++)
4588 381 : if (i < FIRST_PSEUDO_REGISTER)
4589 358 : bitmap_clear_bit (live_caller_saved_regs, i);
4590 :
4591 376 : if (bitmap_empty_p (live_caller_saved_regs))
4592 : {
4593 237 : *after_p = insn;
4594 237 : tls_insn = emit_insn_after (tls_set, insn);
4595 237 : return tls_insn;
4596 : }
4597 : }
4598 : }
4599 :
4600 : /* NB: Start over again for conditional jump. */
4601 2 : if (repeat)
4602 2 : continue;
4603 :
4604 0 : gcc_assert (!bitmap_empty_p (live_caller_saved_regs));
4605 :
4606 : /* If any live caller-saved registers aren't dead at the end of
4607 : this basic block, get the basic block which dominates all
4608 : basic blocks which set the remaining live registers. */
4609 0 : auto_bitmap set_bbs;
4610 0 : bitmap_iterator bi;
4611 0 : unsigned int id;
4612 0 : EXECUTE_IF_SET_IN_BITMAP (live_caller_saved_regs, 0, id, bi)
4613 : {
4614 0 : basic_block set_bb = ix86_get_dominator_for_reg (id, bb);
4615 0 : bitmap_set_bit (set_bbs, set_bb->index);
4616 : }
4617 0 : bb = nearest_common_dominator_for_set (CDI_DOMINATORS, set_bbs);
4618 2 : }
4619 : while (true);
4620 : }
4621 :
4622 : /* Generate a TLS call of KIND with VAL and copy the call result to DEST,
4623 : at entry of the nearest dominator for basic block map BBS, which is in
4624 : the fake loop that contains the whole function, so that there is only
4625 : a single TLS CALL of KIND with VAL in the whole function.
4626 : UPDATED_GNU_TLS_INSNS contains instructions which replace the GNU TLS
4627 : instructions. UPDATED_GNU2_TLS_INSNS contains instructions which
4628 : replace the GNU2 TLS instructions. If TLSDESC_SET isn't nullptr,
4629 : insert it before the TLS call. */
4630 :
4631 : static void
4632 323 : ix86_place_single_tls_call (rtx dest, rtx val, x86_cse_kind kind,
4633 : auto_bitmap &bbs,
4634 : auto_bitmap &updated_gnu_tls_insns,
4635 : auto_bitmap &updated_gnu2_tls_insns,
4636 : rtx tlsdesc_set = nullptr)
4637 : {
4638 323 : basic_block bb = nearest_common_dominator_for_set (CDI_DOMINATORS, bbs);
4639 323 : while (bb->loop_father->latch
4640 333 : != EXIT_BLOCK_PTR_FOR_FN (cfun))
4641 10 : bb = get_immediate_dominator (CDI_DOMINATORS,
4642 : bb->loop_father->header);
4643 :
4644 323 : rtx rax = nullptr, rdi;
4645 323 : rtx eqv = nullptr;
4646 323 : rtx caddr;
4647 323 : rtx set;
4648 323 : rtx clob;
4649 323 : rtx symbol;
4650 323 : rtx tls;
4651 :
4652 323 : switch (kind)
4653 : {
4654 265 : case X86_CSE_TLS_GD:
4655 265 : rax = gen_rtx_REG (Pmode, AX_REG);
4656 265 : rdi = gen_rtx_REG (Pmode, DI_REG);
4657 265 : caddr = ix86_tls_get_addr ();
4658 :
4659 265 : symbol = XVECEXP (val, 0, 0);
4660 265 : tls = gen_tls_global_dynamic_64 (Pmode, rax, symbol, caddr, rdi);
4661 265 : CALL_INSN_ABI_ID (tls) = ix86_tls_get_addr_abi ().id ();
4662 :
4663 265 : if (GET_MODE (symbol) != Pmode)
4664 0 : symbol = gen_rtx_ZERO_EXTEND (Pmode, symbol);
4665 : eqv = symbol;
4666 : break;
4667 :
4668 37 : case X86_CSE_TLS_LD_BASE:
4669 37 : rax = gen_rtx_REG (Pmode, AX_REG);
4670 37 : rdi = gen_rtx_REG (Pmode, DI_REG);
4671 37 : caddr = ix86_tls_get_addr ();
4672 :
4673 37 : tls = gen_tls_local_dynamic_base_64 (Pmode, rax, caddr, rdi);
4674 37 : CALL_INSN_ABI_ID (tls) = ix86_tls_get_addr_abi ().id ();
4675 :
4676 : /* Attach a unique REG_EQUAL to DEST, to allow the RTL optimizers
4677 : to share the LD_BASE result with other LD model accesses. */
4678 37 : eqv = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx),
4679 : UNSPEC_TLS_LD_BASE);
4680 :
4681 37 : break;
4682 :
4683 21 : case X86_CSE_TLSDESC:
4684 21 : set = gen_rtx_SET (dest, val);
4685 21 : clob = gen_rtx_CLOBBER (VOIDmode,
4686 : gen_rtx_REG (CCmode, FLAGS_REG));
4687 21 : tls = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, set, clob));
4688 21 : break;
4689 :
4690 0 : default:
4691 0 : gcc_unreachable ();
4692 : }
4693 :
4694 : /* Emit the TLS CALL insn. */
4695 323 : rtx_insn *before = nullptr;
4696 323 : rtx_insn *after = nullptr;
4697 323 : rtx_insn *tls_insn = ix86_emit_tls_call (tls, kind, bb, &before,
4698 : &after,
4699 : updated_gnu_tls_insns,
4700 : updated_gnu2_tls_insns);
4701 :
4702 323 : rtx_insn *tlsdesc_insn = nullptr;
4703 323 : if (tlsdesc_set)
4704 : {
4705 16 : rtx dest = copy_rtx (SET_DEST (tlsdesc_set));
4706 16 : rtx src = copy_rtx (SET_SRC (tlsdesc_set));
4707 16 : tlsdesc_set = gen_rtx_SET (dest, src);
4708 16 : tlsdesc_insn = emit_insn_before (tlsdesc_set, tls_insn);
4709 : }
4710 :
4711 323 : if (kind != X86_CSE_TLSDESC)
4712 : {
4713 302 : RTL_CONST_CALL_P (tls_insn) = 1;
4714 :
4715 : /* Indicate that this function can't jump to non-local gotos. */
4716 302 : make_reg_eh_region_note_nothrow_nononlocal (tls_insn);
4717 : }
4718 :
4719 323 : if (recog_memoized (tls_insn) < 0)
4720 0 : gcc_unreachable ();
4721 :
4722 323 : if (dump_file)
4723 : {
4724 0 : if (after)
4725 : {
4726 0 : fprintf (dump_file, "\nPlace:\n\n");
4727 0 : if (tlsdesc_insn)
4728 0 : print_rtl_single (dump_file, tlsdesc_insn);
4729 0 : print_rtl_single (dump_file, tls_insn);
4730 0 : fprintf (dump_file, "\nafter:\n\n");
4731 0 : print_rtl_single (dump_file, after);
4732 0 : fprintf (dump_file, "\n");
4733 : }
4734 : else
4735 : {
4736 0 : fprintf (dump_file, "\nPlace:\n\n");
4737 0 : if (tlsdesc_insn)
4738 0 : print_rtl_single (dump_file, tlsdesc_insn);
4739 0 : print_rtl_single (dump_file, tls_insn);
4740 0 : fprintf (dump_file, "\nbefore:\n\n");
4741 0 : print_rtl_single (dump_file, before);
4742 0 : fprintf (dump_file, "\n");
4743 : }
4744 : }
4745 :
4746 323 : if (kind != X86_CSE_TLSDESC)
4747 : {
4748 : /* Copy RAX to DEST. */
4749 302 : set = gen_rtx_SET (dest, rax);
4750 302 : rtx_insn *set_insn = emit_insn_after (set, tls_insn);
4751 302 : set_dst_reg_note (set_insn, REG_EQUAL, copy_rtx (eqv), dest);
4752 302 : if (dump_file)
4753 : {
4754 0 : fprintf (dump_file, "\nPlace:\n\n");
4755 0 : print_rtl_single (dump_file, set_insn);
4756 0 : fprintf (dump_file, "\nafter:\n\n");
4757 0 : print_rtl_single (dump_file, tls_insn);
4758 0 : fprintf (dump_file, "\n");
4759 : }
4760 : }
4761 323 : }
4762 :
4763 : namespace {
4764 :
4765 : const pass_data pass_data_x86_cse =
4766 : {
4767 : RTL_PASS, /* type */
4768 : "x86_cse", /* name */
4769 : OPTGROUP_NONE, /* optinfo_flags */
4770 : TV_MACH_DEP, /* tv_id */
4771 : 0, /* properties_required */
4772 : 0, /* properties_provided */
4773 : 0, /* properties_destroyed */
4774 : 0, /* todo_flags_start */
4775 : 0, /* todo_flags_finish */
4776 : };
4777 :
4778 : class pass_x86_cse : public rtl_opt_pass
4779 : {
4780 : public:
4781 297180 : pass_x86_cse (gcc::context *ctxt)
4782 594360 : : rtl_opt_pass (pass_data_x86_cse, ctxt)
4783 : {}
4784 :
4785 : /* opt_pass methods: */
4786 1517135 : bool gate (function *fun) final override
4787 : {
4788 1517135 : return optimize && optimize_function_for_speed_p (fun);
4789 : }
4790 :
4791 998209 : unsigned int execute (function *) final override
4792 : {
4793 998209 : return x86_cse ();
4794 : }
4795 :
4796 : private:
4797 : /* The redundant source value. */
4798 : rtx val;
4799 : /* The actual redundant source value for UNSPEC_TLSDESC. */
4800 : rtx tlsdesc_val;
4801 : /* The instruction which defines the redundant value. */
4802 : rtx_insn *def_insn;
4803 : /* Mode of the destination of the candidate redundant instruction. */
4804 : machine_mode mode;
4805 : /* Mode of the source of the candidate redundant instruction. */
4806 : machine_mode scalar_mode;
4807 : /* The classification of the candidate redundant instruction. */
4808 : x86_cse_kind kind;
4809 :
4810 : unsigned int x86_cse (void);
4811 : bool candidate_gnu_tls_p (rtx_insn *, attr_tls64);
4812 : bool candidate_gnu2_tls_p (rtx, attr_tls64);
4813 : bool candidate_vector_p (rtx, rtx_insn *);
4814 : rtx_insn *tls_set_insn_from_symbol (const_rtx, const_rtx);
4815 : }; // class pass_x86_cse
4816 :
4817 : /* Return the instruction which sets REG from TLS_SYMBOL. */
4818 :
4819 : rtx_insn *
4820 42 : pass_x86_cse::tls_set_insn_from_symbol (const_rtx reg,
4821 : const_rtx tls_symbol)
4822 : {
4823 42 : rtx_insn *set_insn = nullptr;
4824 42 : for (df_ref ref = DF_REG_DEF_CHAIN (REGNO (reg));
4825 111 : ref;
4826 69 : ref = DF_REF_NEXT_REG (ref))
4827 : {
4828 69 : if (DF_REF_IS_ARTIFICIAL (ref))
4829 : return nullptr;
4830 :
4831 69 : set_insn = DF_REF_INSN (ref);
4832 69 : if (get_attr_tls64 (set_insn) != TLS64_LEA)
4833 : return nullptr;
4834 :
4835 69 : rtx tls_set = PATTERN (set_insn);
4836 69 : rtx tls_src = XVECEXP (SET_SRC (tls_set), 0, 0);
4837 69 : if (!rtx_equal_p (tls_symbol, tls_src))
4838 : return nullptr;
4839 : }
4840 :
4841 : return set_insn;
4842 : }
4843 :
4844 : /* Return true and output def_insn, val, mode, scalar_mode and kind if
4845 : INSN is UNSPEC_TLS_GD or UNSPEC_TLS_LD_BASE. */
4846 :
4847 : bool
4848 2211 : pass_x86_cse::candidate_gnu_tls_p (rtx_insn *insn, attr_tls64 tls64)
4849 : {
4850 2211 : if (!TARGET_64BIT || !cfun->machine->tls_descriptor_call_multiple_p)
4851 : return false;
4852 :
4853 : /* Record the redundant TLS CALLs for 64-bit:
4854 :
4855 : (parallel [
4856 : (set (reg:DI 0 ax)
4857 : (call:DI (mem:QI (symbol_ref:DI ("__tls_get_addr")))
4858 : (const_int 0 [0])))
4859 : (unspec:DI [(symbol_ref:DI ("foo") [flags 0x50])
4860 : (reg/f:DI 7 sp)] UNSPEC_TLS_GD)
4861 : (clobber (reg:DI 5 di))])
4862 :
4863 :
4864 : and
4865 :
4866 : (parallel [
4867 : (set (reg:DI 0 ax)
4868 : (call:DI (mem:QI (symbol_ref:DI ("__tls_get_addr")))
4869 : (const_int 0 [0])))
4870 : (unspec:DI [(reg/f:DI 7 sp)] UNSPEC_TLS_LD_BASE)])
4871 :
4872 : */
4873 :
4874 2037 : rtx pat = PATTERN (insn);
4875 2037 : rtx set = XVECEXP (pat, 0, 0);
4876 2037 : gcc_assert (GET_CODE (set) == SET);
4877 2037 : rtx dest = SET_DEST (set);
4878 2037 : scalar_mode = mode = GET_MODE (dest);
4879 2037 : val = XVECEXP (pat, 0, 1);
4880 2037 : gcc_assert (GET_CODE (val) == UNSPEC);
4881 :
4882 2037 : if (tls64 == TLS64_GD)
4883 : kind = X86_CSE_TLS_GD;
4884 : else
4885 115 : kind = X86_CSE_TLS_LD_BASE;
4886 :
4887 2037 : def_insn = nullptr;
4888 2037 : return true;
4889 : }
4890 :
4891 : /* Return true and output def_insn, val, mode, scalar_mode and kind if
4892 : SET is UNSPEC_TLSDESC. */
4893 :
4894 : bool
4895 56 : pass_x86_cse::candidate_gnu2_tls_p (rtx set, attr_tls64 tls64)
4896 : {
4897 56 : if (!TARGET_64BIT || !cfun->machine->tls_descriptor_call_multiple_p)
4898 : return false;
4899 :
4900 54 : rtx tls_symbol;
4901 54 : rtx_insn *set_insn;
4902 54 : rtx src = SET_SRC (set);
4903 54 : val = src;
4904 54 : tlsdesc_val = src;
4905 54 : kind = X86_CSE_TLSDESC;
4906 :
4907 54 : if (tls64 == TLS64_COMBINE)
4908 : {
4909 : /* Record 64-bit TLS64_COMBINE:
4910 :
4911 : (set (reg/f:DI 104)
4912 : (plus:DI (unspec:DI [
4913 : (symbol_ref:DI ("_TLS_MODULE_BASE_") [flags 0x10])
4914 : (reg:DI 114)
4915 : (reg/f:DI 7 sp)] UNSPEC_TLSDESC)
4916 : (const:DI (unspec:DI [
4917 : (symbol_ref:DI ("e") [flags 0x1a])
4918 : ] UNSPEC_DTPOFF))))
4919 :
4920 : (set (reg/f:DI 104)
4921 : (plus:DI (unspec:DI [
4922 : (symbol_ref:DI ("_TLS_MODULE_BASE_") [flags 0x10])
4923 : (unspec:DI [
4924 : (symbol_ref:DI ("_TLS_MODULE_BASE_") [flags 0x10])
4925 : ] UNSPEC_TLSDESC)
4926 : (reg/f:DI 7 sp)] UNSPEC_TLSDESC)
4927 : (const:DI (unspec:DI [
4928 : (symbol_ref:DI ("e") [flags 0x1a])
4929 : ] UNSPEC_DTPOFF))))
4930 : */
4931 :
4932 12 : scalar_mode = mode = GET_MODE (src);
4933 :
4934 : /* Since the first operand of PLUS in the source TLS_COMBINE
4935 : pattern is unused, use the second operand of PLUS:
4936 :
4937 : (const:DI (unspec:DI [
4938 : (symbol_ref:DI ("e") [flags 0x1a])
4939 : ] UNSPEC_DTPOFF))
4940 :
4941 : as VAL to check if 2 TLS_COMBINE patterns have the same
4942 : source. */
4943 12 : val = XEXP (src, 1);
4944 12 : gcc_assert (GET_CODE (val) == CONST
4945 : && GET_CODE (XEXP (val, 0)) == UNSPEC
4946 : && XINT (XEXP (val, 0), 1) == UNSPEC_DTPOFF
4947 : && SYMBOL_REF_P (XVECEXP (XEXP (val, 0), 0, 0)));
4948 12 : def_insn = nullptr;
4949 12 : return true;
4950 : }
4951 :
4952 : /* Record 64-bit TLS_CALL:
4953 :
4954 : (set (reg:DI 101)
4955 : (unspec:DI [(symbol_ref:DI ("foo") [flags 0x50])
4956 : (reg:DI 112)
4957 : (reg/f:DI 7 sp)] UNSPEC_TLSDESC))
4958 :
4959 : */
4960 :
4961 42 : gcc_assert (GET_CODE (src) == UNSPEC);
4962 42 : tls_symbol = XVECEXP (src, 0, 0);
4963 42 : src = XVECEXP (src, 0, 1);
4964 42 : scalar_mode = mode = GET_MODE (src);
4965 42 : gcc_assert (REG_P (src));
4966 :
4967 : /* All definitions of reg:DI 129 in
4968 :
4969 : (set (reg:DI 110)
4970 : (unspec:DI [(symbol_ref:DI ("foo"))
4971 : (reg:DI 129)
4972 : (reg/f:DI 7 sp)] UNSPEC_TLSDESC))
4973 :
4974 : should have the same source as in
4975 :
4976 : (set (reg:DI 129)
4977 : (unspec:DI [(symbol_ref:DI ("foo"))] UNSPEC_TLSDESC))
4978 :
4979 : */
4980 :
4981 42 : set_insn = tls_set_insn_from_symbol (src, tls_symbol);
4982 42 : if (!set_insn)
4983 : return false;
4984 :
4985 : /* Use TLS_SYMBOL as VAL to check if 2 patterns have the same source. */
4986 42 : val = tls_symbol;
4987 42 : def_insn = set_insn;
4988 42 : return true;
4989 : }
4990 :
4991 : /* Return true and output def_insn, val, mode, scalar_mode and kind if
4992 : INSN is a vector broadcast instruction. */
4993 :
4994 : bool
4995 51538833 : pass_x86_cse::candidate_vector_p (rtx set, rtx_insn *insn)
4996 : {
4997 51538833 : rtx src = SET_SRC (set);
4998 51538833 : rtx dest = SET_DEST (set);
4999 51538833 : mode = GET_MODE (dest);
5000 : /* Skip non-vector instruction. */
5001 51538833 : if (!VECTOR_MODE_P (mode))
5002 : return false;
5003 :
5004 : /* Skip non-vector load instruction. */
5005 3704852 : if (!REG_P (dest) && !SUBREG_P (dest))
5006 : return false;
5007 :
5008 2220655 : def_insn = insn;
5009 2220655 : val = ix86_broadcast_inner (src, mode, &scalar_mode, &kind,
5010 : &def_insn);
5011 2220655 : return val ? true : false;
5012 : }
5013 :
5014 : /* At entry of the nearest common dominator for basic blocks with
5015 :
5016 : 1. Vector CONST0_RTX patterns.
5017 : 2. Vector CONSTM1_RTX patterns.
5018 : 3. Vector broadcast patterns.
5019 : 4. UNSPEC_TLS_GD patterns.
5020 : 5. UNSPEC_TLS_LD_BASE patterns.
5021 : 6. UNSPEC_TLSDESC patterns.
5022 :
5023 : generate a single pattern whose destination is used to replace the
5024 : source in all identical patterns.
5025 :
5026 : NB: We want to generate a pattern, which is executed only once, to
5027 : cover the whole function. The LCM algorithm isn't appropriate here
5028 : since it may place a pattern inside the loop. */
5029 :
5030 : unsigned int
5031 998209 : pass_x86_cse::x86_cse (void)
5032 : {
5033 998209 : timevar_push (TV_MACH_DEP);
5034 :
5035 998209 : auto_vec<redundant_pattern *> loads;
5036 998209 : redundant_pattern *load;
5037 998209 : basic_block bb;
5038 998209 : rtx_insn *insn;
5039 998209 : unsigned int i;
5040 998209 : auto_bitmap updated_gnu_tls_insns;
5041 998209 : auto_bitmap updated_gnu2_tls_insns;
5042 998209 : auto_bitmap call_bbs;
5043 :
5044 998209 : df_set_flags (DF_DEFER_INSN_RESCAN);
5045 :
5046 998209 : bool recursive_call_p = cfun->machine->recursive_function;
5047 :
5048 11219938 : FOR_EACH_BB_FN (bb, cfun)
5049 : {
5050 138089399 : FOR_BB_INSNS (bb, insn)
5051 : {
5052 127867670 : if (!NONDEBUG_INSN_P (insn))
5053 72564163 : continue;
5054 :
5055 55303507 : bool matched = false;
5056 : /* Remove redundant patterns if there are more than 2 of
5057 : them. */
5058 55303507 : unsigned int threshold = 2;
5059 :
5060 55303507 : bool call_p = CALL_P (insn);
5061 55303507 : rtx set = single_set (insn);
5062 55303507 : if (!set && !call_p)
5063 1113504 : continue;
5064 :
5065 54190003 : tlsdesc_val = nullptr;
5066 :
5067 54190003 : attr_tls64 tls64 = get_attr_tls64 (insn);
5068 :
5069 : /* NB: TLS calls preserve all registers. */
5070 54190003 : if (call_p && tls64 == TLS64_NONE)
5071 4584388 : bitmap_set_bit (call_bbs, BLOCK_FOR_INSN (insn)->index);
5072 :
5073 54190003 : switch (tls64)
5074 : {
5075 2211 : case TLS64_GD:
5076 2211 : case TLS64_LD_BASE:
5077 : /* Verify UNSPEC_TLS_GD and UNSPEC_TLS_LD_BASE. */
5078 2211 : if (candidate_gnu_tls_p (insn, tls64))
5079 : break;
5080 174 : continue;
5081 :
5082 56 : case TLS64_CALL:
5083 56 : case TLS64_COMBINE:
5084 : /* Verify UNSPEC_TLSDESC. */
5085 56 : if (candidate_gnu2_tls_p (set, tls64))
5086 : break;
5087 2 : continue;
5088 :
5089 36 : case TLS64_LEA:
5090 : /* Skip TLS64_LEA. */
5091 36 : continue;
5092 :
5093 54187700 : case TLS64_NONE:
5094 54187700 : if (!set)
5095 2648867 : continue;
5096 :
5097 : /* Check for vector broadcast. */
5098 51538833 : if (candidate_vector_p (set, insn))
5099 : break;
5100 51282877 : continue;
5101 : }
5102 :
5103 : /* Check if there is a matching redundant load. */
5104 658868 : FOR_EACH_VEC_ELT (loads, i, load)
5105 442284 : if (load->val
5106 442284 : && load->kind == kind
5107 292304 : && load->mode == scalar_mode
5108 260087 : && (load->bb == bb
5109 198813 : || (kind != X86_CSE_VEC_DUP
5110 198813 : && kind != X86_CSE_CONST_VECTOR)
5111 : /* Non all 0s/1s vector load must be in the same
5112 : basic block if it is in a recursive call. */
5113 135050 : || !recursive_call_p)
5114 700192 : && rtx_equal_p (load->val, val))
5115 : {
5116 : /* Record instruction. */
5117 101477 : bitmap_set_bit (load->insns, INSN_UID (insn));
5118 :
5119 : /* Record the maximum vector size. */
5120 101477 : if (kind <= X86_CSE_VEC_DUP
5121 201837 : && load->size < GET_MODE_SIZE (mode))
5122 836 : load->size = GET_MODE_SIZE (mode);
5123 :
5124 : /* Record the basic block. */
5125 101477 : bitmap_set_bit (load->bbs, bb->index);
5126 :
5127 : /* Increment the count. */
5128 101477 : load->count++;
5129 :
5130 101477 : matched = true;
5131 101477 : break;
5132 : }
5133 :
5134 258047 : if (matched)
5135 101477 : continue;
5136 :
5137 : /* We see this instruction the first time. Record the
5138 : redundant source value, its mode, the destination size,
5139 : instruction which defines the redundant source value,
5140 : instruction basic block and the instruction kind. */
5141 156570 : load = new redundant_pattern;
5142 :
5143 : /* Convert CONST_VECTOR load no larger than integer register
5144 : to constant integer load even if there is no redundant
5145 : CONST_VECTOR load. */
5146 156570 : if (CONST_VECTOR_P (val))
5147 32274 : threshold = 1;
5148 :
5149 156570 : load->val = copy_rtx (val);
5150 156570 : if (tlsdesc_val)
5151 28 : load->tlsdesc_val = copy_rtx (tlsdesc_val);
5152 : else
5153 : load->tlsdesc_val = nullptr;
5154 156570 : load->mode = scalar_mode;
5155 156570 : load->dest_mode = mode;
5156 156570 : load->size = GET_MODE_SIZE (mode);
5157 156570 : load->def_insn = def_insn;
5158 156570 : load->count = 1;
5159 156570 : load->threshold = threshold;
5160 156570 : load->bb = BLOCK_FOR_INSN (insn);
5161 156570 : load->kind = kind;
5162 :
5163 156570 : bitmap_set_bit (load->insns, INSN_UID (insn));
5164 156570 : bitmap_set_bit (load->bbs, bb->index);
5165 :
5166 156570 : loads.safe_push (load);
5167 : }
5168 : }
5169 :
5170 : bool replaced = false;
5171 1251335 : FOR_EACH_VEC_ELT (loads, i, load)
5172 156570 : if (load->count >= load->threshold)
5173 : {
5174 66440 : machine_mode mode;
5175 66440 : rtx reg, broadcast_reg;
5176 66440 : rtx broadcast_source = nullptr;
5177 66440 : replaced = true;
5178 66440 : switch (load->kind)
5179 : {
5180 323 : case X86_CSE_TLS_GD:
5181 323 : case X86_CSE_TLS_LD_BASE:
5182 323 : case X86_CSE_TLSDESC:
5183 323 : broadcast_reg = gen_reg_rtx (load->mode);
5184 323 : replace_tls_call (broadcast_reg, load->insns,
5185 323 : (load->kind == X86_CSE_TLSDESC
5186 : ? updated_gnu2_tls_insns
5187 : : updated_gnu_tls_insns));
5188 323 : load->broadcast_reg = broadcast_reg;
5189 323 : break;
5190 :
5191 11391 : case X86_CSE_VEC_DUP:
5192 11391 : if (CONST_INT_P (load->val)
5193 10475 : && (load->val == CONST0_RTX (load->mode)
5194 10500 : || load->size <= UNITS_PER_WORD))
5195 : {
5196 : /* Generate CONST_VECTOR load. */
5197 32277 : case X86_CSE_CONST_VECTOR:
5198 32277 : mode = ix86_get_vector_cse_mode (load->size,
5199 : load->mode);
5200 :
5201 32277 : if (CONST_VECTOR_P (load->val))
5202 : broadcast_source = load->val;
5203 3 : else if (load->val == CONST0_RTX (load->mode))
5204 0 : broadcast_source = CONST0_RTX (mode);
5205 3 : else if (load->val == CONSTM1_RTX (load->mode))
5206 0 : broadcast_source = CONSTM1_RTX (mode);
5207 : else
5208 : {
5209 3 : int nunits = GET_MODE_NUNITS (mode);
5210 3 : rtvec v = rtvec_alloc (nunits);
5211 12 : for (int j = 0; j < nunits ; j++)
5212 6 : RTVEC_ELT (v, j) = load->val;
5213 3 : if (CONST_INT_P (load->val)
5214 3 : && GET_MODE_CLASS (mode) != MODE_VECTOR_INT)
5215 : {
5216 1 : gcc_assert (load->size <= UNITS_PER_WORD);
5217 1 : machine_mode imode = GET_MODE_INNER (mode);
5218 1 : class scalar_mode i_mode
5219 1 : = int_mode_for_mode (imode).require ();
5220 1 : mode = mode_for_vector (i_mode,
5221 1 : nunits).require ();
5222 : }
5223 :
5224 3 : broadcast_source = gen_rtx_CONST_VECTOR (mode, v);
5225 : }
5226 :
5227 : /* NB: Zero CONST_VECTOR load works for MMX and XMM
5228 : registers. */
5229 33687 : if (load->size <= UNITS_PER_WORD)
5230 : {
5231 : /* Convert CONST_VECTOR load no larger than integer
5232 : register:
5233 :
5234 : (set (reg:V2SI 106)
5235 : (const_vector:V2SI [(const_int 1 [1]) repeated x2]))
5236 :
5237 : to constant integer load:
5238 :
5239 : (set (subreg:DI (reg:V2SI 106 [ _20 ]) 0)
5240 : (const_int 4294967297 [0x100000001]))
5241 : */
5242 32277 : machine_mode int_mode
5243 32277 : = int_mode_for_mode (mode).require ();
5244 32277 : load->dest_mode = int_mode;
5245 32277 : broadcast_source = simplify_subreg (int_mode,
5246 : broadcast_source,
5247 : mode, 0);
5248 32277 : gcc_assert (broadcast_source != nullptr);
5249 :
5250 32277 : bool keep_const_int_load = false;
5251 32277 : if (!bitmap_empty_p (call_bbs))
5252 : {
5253 28853 : bitmap_iterator bi;
5254 28853 : unsigned int id;
5255 38274 : EXECUTE_IF_SET_IN_BITMAP (load->bbs, 0, id, bi)
5256 30351 : if (bitmap_bit_p (call_bbs, id))
5257 : {
5258 : /* NB: Constant integer load is faster
5259 : than save and restore an integer
5260 : register when crossing a function call.
5261 : */
5262 : keep_const_int_load = true;
5263 : break;
5264 : }
5265 : }
5266 :
5267 28853 : if (keep_const_int_load)
5268 : {
5269 : /* Keep constant integer load. */
5270 20930 : replace_vector_const (mode, broadcast_source,
5271 20930 : load->insns, int_mode);
5272 20930 : load->broadcast_source = nullptr;
5273 20930 : load->broadcast_reg = nullptr;
5274 : }
5275 : else
5276 : {
5277 11347 : broadcast_reg = gen_reg_rtx (mode);
5278 11347 : reg = gen_reg_rtx (load->mode);
5279 11347 : replace_vector_const (mode, broadcast_reg,
5280 11347 : load->insns, load->mode);
5281 11347 : load->broadcast_source = broadcast_source;
5282 11347 : load->broadcast_reg = broadcast_reg;
5283 : }
5284 : break;
5285 : }
5286 : }
5287 : /* FALLTHRU */
5288 :
5289 33840 : case X86_CSE_CONST0_VECTOR:
5290 33840 : case X86_CSE_CONSTM1_VECTOR:
5291 33840 : mode = ix86_get_vector_cse_mode (load->size, load->mode);
5292 33840 : broadcast_reg = gen_reg_rtx (mode);
5293 33840 : if (load->def_insn)
5294 : {
5295 : /* Replace redundant vector loads with a single vector
5296 : load in the same basic block. */
5297 778 : reg = load->val;
5298 778 : if (load->mode != GET_MODE (reg))
5299 0 : reg = gen_rtx_SUBREG (load->mode, reg, 0);
5300 778 : broadcast_source = gen_rtx_VEC_DUPLICATE (mode, reg);
5301 : }
5302 : else
5303 : /* This is a constant integer/double vector. If the
5304 : inner scalar is 0 or -1, set vector to CONST0_RTX
5305 : or CONSTM1_RTX directly. */
5306 33062 : switch (load->kind)
5307 : {
5308 20752 : case X86_CSE_CONST0_VECTOR:
5309 20752 : broadcast_source = CONST0_RTX (mode);
5310 20752 : break;
5311 1697 : case X86_CSE_CONSTM1_VECTOR:
5312 1697 : broadcast_source = CONSTM1_RTX (mode);
5313 1697 : break;
5314 10613 : case X86_CSE_CONST_VECTOR:
5315 10613 : case X86_CSE_VEC_DUP:
5316 10613 : if (!broadcast_source)
5317 : {
5318 10613 : reg = gen_reg_rtx (load->mode);
5319 10613 : broadcast_source = gen_rtx_VEC_DUPLICATE (mode,
5320 : reg);
5321 : }
5322 : break;
5323 0 : default:
5324 0 : gcc_unreachable ();
5325 : }
5326 33840 : replace_vector_const (mode, broadcast_reg, load->insns,
5327 : load->mode);
5328 33840 : load->broadcast_source = broadcast_source;
5329 33840 : load->broadcast_reg = broadcast_reg;
5330 33840 : break;
5331 : }
5332 : }
5333 :
5334 998209 : if (replaced)
5335 : {
5336 43210 : auto_vec<rtx_insn *> control_flow_insns;
5337 :
5338 : /* (Re-)discover loops so that bb->loop_father can be used in the
5339 : analysis below. */
5340 43210 : calculate_dominance_info (CDI_DOMINATORS);
5341 43210 : loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
5342 :
5343 216492 : FOR_EACH_VEC_ELT (loads, i, load)
5344 86862 : if (load->count >= load->threshold)
5345 : {
5346 66440 : rtx set;
5347 66440 : if (load->def_insn)
5348 794 : switch (load->kind)
5349 : {
5350 16 : case X86_CSE_TLSDESC:
5351 16 : ix86_place_single_tls_call (load->broadcast_reg,
5352 : load->tlsdesc_val,
5353 : load->kind,
5354 16 : load->bbs,
5355 : updated_gnu_tls_insns,
5356 : updated_gnu2_tls_insns,
5357 16 : PATTERN (load->def_insn));
5358 16 : break;
5359 778 : case X86_CSE_VEC_DUP:
5360 : /* Insert a broadcast after the original scalar
5361 : definition. */
5362 778 : set = gen_rtx_SET (load->broadcast_reg,
5363 : load->broadcast_source);
5364 778 : insn = emit_insn_after (set, load->def_insn);
5365 :
5366 778 : if (cfun->can_throw_non_call_exceptions)
5367 : {
5368 : /* Handle REG_EH_REGION note in DEF_INSN. */
5369 4 : rtx note = find_reg_note (load->def_insn,
5370 : REG_EH_REGION, nullptr);
5371 4 : if (note)
5372 : {
5373 1 : control_flow_insns.safe_push (load->def_insn);
5374 1 : add_reg_note (insn, REG_EH_REGION,
5375 : XEXP (note, 0));
5376 : }
5377 : }
5378 :
5379 778 : if (dump_file)
5380 : {
5381 0 : fprintf (dump_file, "\nAdd:\n\n");
5382 0 : print_rtl_single (dump_file, insn);
5383 0 : fprintf (dump_file, "\nafter:\n\n");
5384 0 : print_rtl_single (dump_file, load->def_insn);
5385 0 : fprintf (dump_file, "\n");
5386 : }
5387 : break;
5388 0 : default:
5389 0 : gcc_unreachable ();
5390 : }
5391 : else
5392 65646 : switch (load->kind)
5393 : {
5394 307 : case X86_CSE_TLS_GD:
5395 307 : case X86_CSE_TLS_LD_BASE:
5396 307 : case X86_CSE_TLSDESC:
5397 307 : ix86_place_single_tls_call (load->broadcast_reg,
5398 : (load->kind == X86_CSE_TLSDESC
5399 : ? load->tlsdesc_val
5400 : : load->val),
5401 : load->kind,
5402 307 : load->bbs,
5403 : updated_gnu_tls_insns,
5404 : updated_gnu2_tls_insns);
5405 307 : break;
5406 42890 : case X86_CSE_CONST_VECTOR:
5407 42890 : case X86_CSE_VEC_DUP:
5408 : /* Keep redundant constant integer load. */
5409 42890 : if (!load->broadcast_reg)
5410 : break;
5411 : /* FALLTHRU */
5412 44409 : case X86_CSE_CONST0_VECTOR:
5413 44409 : case X86_CSE_CONSTM1_VECTOR:
5414 44409 : ix86_place_single_vector_set (load->broadcast_reg,
5415 : load->broadcast_source,
5416 : load->bbs,
5417 : load);
5418 44409 : break;
5419 : }
5420 : }
5421 :
5422 43210 : loop_optimizer_finalize ();
5423 :
5424 43211 : if (!control_flow_insns.is_empty ())
5425 : {
5426 1 : free_dominance_info (CDI_DOMINATORS);
5427 :
5428 3 : FOR_EACH_VEC_ELT (control_flow_insns, i, insn)
5429 1 : if (control_flow_insn_p (insn))
5430 : {
5431 : /* Split the block after insn. There will be a fallthru
5432 : edge, which is OK so we keep it. We have to create
5433 : the exception edges ourselves. */
5434 1 : bb = BLOCK_FOR_INSN (insn);
5435 1 : split_block (bb, insn);
5436 1 : rtl_make_eh_edge (NULL, bb, BB_END (bb));
5437 : }
5438 : }
5439 :
5440 43210 : df_process_deferred_rescans ();
5441 43210 : }
5442 :
5443 1251335 : FOR_EACH_VEC_ELT (loads, i, load)
5444 313140 : delete load;
5445 :
5446 998209 : df_clear_flags (DF_DEFER_INSN_RESCAN);
5447 :
5448 998209 : timevar_pop (TV_MACH_DEP);
5449 998209 : return 0;
5450 998209 : }
5451 :
5452 : } // anon namespace
5453 :
5454 : rtl_opt_pass *
5455 297180 : make_pass_x86_cse (gcc::context *ctxt)
5456 : {
5457 297180 : return new pass_x86_cse (ctxt);
5458 : }
5459 :
5460 : /* Convert legacy instructions that clobbers EFLAGS to APX_NF
5461 : instructions when there are no flag set between a flag
5462 : producer and user. */
5463 :
5464 : static unsigned int
5465 384 : ix86_apx_nf_convert (void)
5466 : {
5467 384 : timevar_push (TV_MACH_DEP);
5468 :
5469 384 : basic_block bb;
5470 384 : rtx_insn *insn;
5471 384 : hash_map <rtx_insn *, rtx> converting_map;
5472 384 : auto_vec <rtx_insn *> current_convert_list;
5473 :
5474 384 : bool converting_seq = false;
5475 384 : rtx cc = gen_rtx_REG (CCmode, FLAGS_REG);
5476 :
5477 842 : FOR_EACH_BB_FN (bb, cfun)
5478 : {
5479 : /* Reset conversion for each bb. */
5480 458 : converting_seq = false;
5481 5352 : FOR_BB_INSNS (bb, insn)
5482 : {
5483 4894 : if (!NONDEBUG_INSN_P (insn))
5484 4277 : continue;
5485 :
5486 3891 : if (recog_memoized (insn) < 0)
5487 346 : continue;
5488 :
5489 : /* Convert candidate insns after cstore, which should
5490 : satisfy the two conditions:
5491 : 1. Is not flag user or producer, only clobbers
5492 : FLAGS_REG.
5493 : 2. Have corresponding nf pattern. */
5494 :
5495 3545 : rtx pat = PATTERN (insn);
5496 :
5497 : /* Starting conversion at first cstorecc. */
5498 3545 : rtx set = NULL_RTX;
5499 3545 : if (!converting_seq
5500 2954 : && (set = single_set (insn))
5501 2876 : && ix86_comparison_operator (SET_SRC (set), VOIDmode)
5502 131 : && reg_overlap_mentioned_p (cc, SET_SRC (set))
5503 3673 : && !reg_overlap_mentioned_p (cc, SET_DEST (set)))
5504 : {
5505 128 : converting_seq = true;
5506 128 : current_convert_list.truncate (0);
5507 : }
5508 : /* Terminate at the next explicit flag set. */
5509 3417 : else if (reg_set_p (cc, pat)
5510 3417 : && GET_CODE (set_of (cc, pat)) != CLOBBER)
5511 : converting_seq = false;
5512 :
5513 3306 : if (!converting_seq)
5514 2928 : continue;
5515 :
5516 617 : if (get_attr_has_nf (insn)
5517 617 : && GET_CODE (pat) == PARALLEL)
5518 : {
5519 : /* Record the insn to candidate map. */
5520 74 : current_convert_list.safe_push (insn);
5521 74 : converting_map.put (insn, pat);
5522 : }
5523 : /* If the insn clobbers flags but has no nf_attr,
5524 : revoke all previous candidates. */
5525 543 : else if (!get_attr_has_nf (insn)
5526 542 : && reg_set_p (cc, pat)
5527 546 : && GET_CODE (set_of (cc, pat)) == CLOBBER)
5528 : {
5529 3 : for (auto item : current_convert_list)
5530 0 : converting_map.remove (item);
5531 3 : converting_seq = false;
5532 : }
5533 : }
5534 : }
5535 :
5536 384 : if (!converting_map.is_empty ())
5537 : {
5538 88 : for (auto iter = converting_map.begin ();
5539 102 : iter != converting_map.end (); ++iter)
5540 : {
5541 74 : rtx_insn *replace = (*iter).first;
5542 74 : rtx pat = (*iter).second;
5543 74 : int i, n = 0, len = XVECLEN (pat, 0);
5544 74 : rtx *new_elems = XALLOCAVEC (rtx, len);
5545 74 : rtx new_pat;
5546 222 : for (i = 0; i < len; i++)
5547 : {
5548 148 : rtx temp = XVECEXP (pat, 0, i);
5549 222 : if (! (GET_CODE (temp) == CLOBBER
5550 74 : && reg_overlap_mentioned_p (cc,
5551 74 : XEXP (temp, 0))))
5552 : {
5553 74 : new_elems[n] = temp;
5554 74 : n++;
5555 : }
5556 : }
5557 :
5558 74 : if (n == 1)
5559 74 : new_pat = new_elems[0];
5560 : else
5561 0 : new_pat =
5562 0 : gen_rtx_PARALLEL (VOIDmode,
5563 : gen_rtvec_v (n,
5564 : new_elems));
5565 :
5566 74 : PATTERN (replace) = new_pat;
5567 74 : INSN_CODE (replace) = -1;
5568 74 : recog_memoized (replace);
5569 74 : df_insn_rescan (replace);
5570 : }
5571 : }
5572 :
5573 384 : timevar_pop (TV_MACH_DEP);
5574 384 : return 0;
5575 384 : }
5576 :
5577 :
5578 : namespace {
5579 :
5580 : const pass_data pass_data_apx_nf_convert =
5581 : {
5582 : RTL_PASS, /* type */
5583 : "apx_nfcvt", /* name */
5584 : OPTGROUP_NONE, /* optinfo_flags */
5585 : TV_MACH_DEP, /* tv_id */
5586 : 0, /* properties_required */
5587 : 0, /* properties_provided */
5588 : 0, /* properties_destroyed */
5589 : 0, /* todo_flags_start */
5590 : 0, /* todo_flags_finish */
5591 : };
5592 :
5593 : class pass_apx_nf_convert : public rtl_opt_pass
5594 : {
5595 : public:
5596 297180 : pass_apx_nf_convert (gcc::context *ctxt)
5597 594360 : : rtl_opt_pass (pass_data_apx_nf_convert, ctxt)
5598 : {}
5599 :
5600 : /* opt_pass methods: */
5601 1517135 : bool gate (function *) final override
5602 : {
5603 1517135 : return (TARGET_APX_NF
5604 480 : && optimize
5605 1517603 : && optimize_function_for_speed_p (cfun));
5606 : }
5607 :
5608 384 : unsigned int execute (function *) final override
5609 : {
5610 384 : return ix86_apx_nf_convert ();
5611 : }
5612 : }; // class pass_apx_nf_convert
5613 :
5614 : } // anon namespace
5615 :
5616 : rtl_opt_pass *
5617 297180 : make_pass_apx_nf_convert (gcc::context *ctxt)
5618 : {
5619 297180 : return new pass_apx_nf_convert (ctxt);
5620 : }
5621 :
5622 : /* When a hot loop can be fit into one cacheline,
5623 : force align the loop without considering the max skip. */
5624 : static void
5625 997723 : ix86_align_loops ()
5626 : {
5627 997723 : basic_block bb;
5628 :
5629 : /* Don't do this when we don't know cache line size. */
5630 997723 : if (ix86_cost->prefetch_block == 0)
5631 9 : return;
5632 :
5633 997714 : loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
5634 997714 : profile_count count_threshold = cfun->cfg->count_max / param_align_threshold;
5635 11666475 : FOR_EACH_BB_FN (bb, cfun)
5636 : {
5637 10668761 : rtx_insn *label = BB_HEAD (bb);
5638 10668761 : bool has_fallthru = 0;
5639 10668761 : edge e;
5640 10668761 : edge_iterator ei;
5641 :
5642 10668761 : if (!LABEL_P (label))
5643 5432103 : continue;
5644 :
5645 5241561 : profile_count fallthru_count = profile_count::zero ();
5646 5241561 : profile_count branch_count = profile_count::zero ();
5647 :
5648 15263565 : FOR_EACH_EDGE (e, ei, bb->preds)
5649 : {
5650 10022004 : if (e->flags & EDGE_FALLTHRU)
5651 2530366 : has_fallthru = 1, fallthru_count += e->count ();
5652 : else
5653 7491638 : branch_count += e->count ();
5654 : }
5655 :
5656 5241561 : if (!fallthru_count.initialized_p () || !branch_count.initialized_p ())
5657 4903 : continue;
5658 :
5659 5236658 : if (bb->loop_father
5660 5236658 : && bb->loop_father->latch != EXIT_BLOCK_PTR_FOR_FN (cfun)
5661 6612635 : && (has_fallthru
5662 1375977 : ? (!(single_succ_p (bb)
5663 153655 : && single_succ (bb) == EXIT_BLOCK_PTR_FOR_FN (cfun))
5664 947588 : && optimize_bb_for_speed_p (bb)
5665 865054 : && branch_count + fallthru_count > count_threshold
5666 738680 : && (branch_count > fallthru_count * param_align_loop_iterations))
5667 : /* In case there'no fallthru for the loop.
5668 : Nops inserted won't be executed. */
5669 428389 : : (branch_count > count_threshold
5670 142735 : || (bb->count > bb->prev_bb->count * 10
5671 12697 : && (bb->prev_bb->count
5672 5249355 : <= ENTRY_BLOCK_PTR_FOR_FN (cfun)->count / 2)))))
5673 : {
5674 563376 : rtx_insn* insn, *end_insn;
5675 563376 : HOST_WIDE_INT size = 0;
5676 563376 : bool padding_p = true;
5677 563376 : basic_block tbb = bb;
5678 563376 : unsigned cond_branch_num = 0;
5679 563376 : bool detect_tight_loop_p = false;
5680 :
5681 878136 : for (unsigned int i = 0; i != bb->loop_father->num_nodes;
5682 314760 : i++, tbb = tbb->next_bb)
5683 : {
5684 : /* Only handle continuous cfg layout. */
5685 878136 : if (bb->loop_father != tbb->loop_father)
5686 : {
5687 : padding_p = false;
5688 : break;
5689 : }
5690 :
5691 10776915 : FOR_BB_INSNS (tbb, insn)
5692 : {
5693 10101367 : if (!NONDEBUG_INSN_P (insn))
5694 5882757 : continue;
5695 4218610 : size += ix86_min_insn_size (insn);
5696 :
5697 : /* We don't know size of inline asm.
5698 : Don't align loop for call. */
5699 4218610 : if (asm_noperands (PATTERN (insn)) >= 0
5700 4218610 : || CALL_P (insn))
5701 : {
5702 : size = -1;
5703 : break;
5704 : }
5705 : }
5706 :
5707 835812 : if (size == -1 || size > ix86_cost->prefetch_block)
5708 : {
5709 : padding_p = false;
5710 : break;
5711 : }
5712 :
5713 1482509 : FOR_EACH_EDGE (e, ei, tbb->succs)
5714 : {
5715 : /* It could be part of the loop. */
5716 1024006 : if (e->dest == bb)
5717 : {
5718 : detect_tight_loop_p = true;
5719 : break;
5720 : }
5721 : }
5722 :
5723 649359 : if (detect_tight_loop_p)
5724 : break;
5725 :
5726 458503 : end_insn = BB_END (tbb);
5727 458503 : if (JUMP_P (end_insn))
5728 : {
5729 : /* For decoded icache:
5730 : 1. Up to two branches are allowed per Way.
5731 : 2. A non-conditional branch is the last micro-op in a Way.
5732 : */
5733 372326 : if (onlyjump_p (end_insn)
5734 372326 : && (any_uncondjump_p (end_insn)
5735 309144 : || single_succ_p (tbb)))
5736 : {
5737 : padding_p = false;
5738 : break;
5739 : }
5740 309144 : else if (++cond_branch_num >= 2)
5741 : {
5742 : padding_p = false;
5743 : break;
5744 : }
5745 : }
5746 :
5747 : }
5748 :
5749 563376 : if (padding_p && detect_tight_loop_p)
5750 : {
5751 381712 : emit_insn_before (gen_max_skip_align (GEN_INT (ceil_log2 (size)),
5752 : GEN_INT (0)), label);
5753 : /* End of function. */
5754 190856 : if (!tbb || tbb == EXIT_BLOCK_PTR_FOR_FN (cfun))
5755 : break;
5756 : /* Skip bb which already fits into one cacheline. */
5757 : bb = tbb;
5758 : }
5759 : }
5760 : }
5761 :
5762 997714 : loop_optimizer_finalize ();
5763 997714 : free_dominance_info (CDI_DOMINATORS);
5764 : }
5765 :
5766 : namespace {
5767 :
5768 : const pass_data pass_data_align_tight_loops =
5769 : {
5770 : RTL_PASS, /* type */
5771 : "align_tight_loops", /* name */
5772 : OPTGROUP_NONE, /* optinfo_flags */
5773 : TV_MACH_DEP, /* tv_id */
5774 : 0, /* properties_required */
5775 : 0, /* properties_provided */
5776 : 0, /* properties_destroyed */
5777 : 0, /* todo_flags_start */
5778 : 0, /* todo_flags_finish */
5779 : };
5780 :
5781 : class pass_align_tight_loops : public rtl_opt_pass
5782 : {
5783 : public:
5784 297180 : pass_align_tight_loops (gcc::context *ctxt)
5785 594360 : : rtl_opt_pass (pass_data_align_tight_loops, ctxt)
5786 : {}
5787 :
5788 : /* opt_pass methods: */
5789 1517135 : bool gate (function *) final override
5790 : {
5791 1517135 : return TARGET_ALIGN_TIGHT_LOOPS
5792 1516642 : && optimize
5793 2581609 : && optimize_function_for_speed_p (cfun);
5794 : }
5795 :
5796 997723 : unsigned int execute (function *) final override
5797 : {
5798 997723 : timevar_push (TV_MACH_DEP);
5799 : #ifdef ASM_OUTPUT_MAX_SKIP_ALIGN
5800 997723 : ix86_align_loops ();
5801 : #endif
5802 997723 : timevar_pop (TV_MACH_DEP);
5803 997723 : return 0;
5804 : }
5805 : }; // class pass_align_tight_loops
5806 :
5807 : } // anon namespace
5808 :
5809 : rtl_opt_pass *
5810 297180 : make_pass_align_tight_loops (gcc::context *ctxt)
5811 : {
5812 297180 : return new pass_align_tight_loops (ctxt);
5813 : }
5814 :
5815 : /* This compares the priority of target features in function DECL1
5816 : and DECL2. It returns positive value if DECL1 is higher priority,
5817 : negative value if DECL2 is higher priority and 0 if they are the
5818 : same. */
5819 :
5820 : int
5821 5830 : ix86_compare_version_priority (tree decl1, tree decl2)
5822 : {
5823 5830 : unsigned int priority1 = get_builtin_code_for_version (decl1, NULL);
5824 5830 : unsigned int priority2 = get_builtin_code_for_version (decl2, NULL);
5825 :
5826 5830 : return (int)priority1 - (int)priority2;
5827 : }
5828 :
5829 : /* V1 and V2 point to function versions with different priorities
5830 : based on the target ISA. This function compares their priorities. */
5831 :
5832 : static int
5833 6882 : feature_compare (const void *v1, const void *v2)
5834 : {
5835 6882 : typedef struct _function_version_info
5836 : {
5837 : tree version_decl;
5838 : tree predicate_chain;
5839 : unsigned int dispatch_priority;
5840 : } function_version_info;
5841 :
5842 6882 : const function_version_info c1 = *(const function_version_info *)v1;
5843 6882 : const function_version_info c2 = *(const function_version_info *)v2;
5844 6882 : return (c2.dispatch_priority - c1.dispatch_priority);
5845 : }
5846 :
5847 : /* This adds a condition to the basic_block NEW_BB in function FUNCTION_DECL
5848 : to return a pointer to VERSION_DECL if the outcome of the expression
5849 : formed by PREDICATE_CHAIN is true. This function will be called during
5850 : version dispatch to decide which function version to execute. It returns
5851 : the basic block at the end, to which more conditions can be added. */
5852 :
5853 : static basic_block
5854 848 : add_condition_to_bb (tree function_decl, tree version_decl,
5855 : tree predicate_chain, basic_block new_bb)
5856 : {
5857 848 : gimple *return_stmt;
5858 848 : tree convert_expr, result_var;
5859 848 : gimple *convert_stmt;
5860 848 : gimple *call_cond_stmt;
5861 848 : gimple *if_else_stmt;
5862 :
5863 848 : basic_block bb1, bb2, bb3;
5864 848 : edge e12, e23;
5865 :
5866 848 : tree cond_var, and_expr_var = NULL_TREE;
5867 848 : gimple_seq gseq;
5868 :
5869 848 : tree predicate_decl, predicate_arg;
5870 :
5871 848 : push_cfun (DECL_STRUCT_FUNCTION (function_decl));
5872 :
5873 848 : gcc_assert (new_bb != NULL);
5874 848 : gseq = bb_seq (new_bb);
5875 :
5876 :
5877 848 : convert_expr = build1 (CONVERT_EXPR, ptr_type_node,
5878 : build_fold_addr_expr (version_decl));
5879 848 : result_var = create_tmp_var (ptr_type_node);
5880 848 : convert_stmt = gimple_build_assign (result_var, convert_expr);
5881 848 : return_stmt = gimple_build_return (result_var);
5882 :
5883 848 : if (predicate_chain == NULL_TREE)
5884 : {
5885 204 : gimple_seq_add_stmt (&gseq, convert_stmt);
5886 204 : gimple_seq_add_stmt (&gseq, return_stmt);
5887 204 : set_bb_seq (new_bb, gseq);
5888 204 : gimple_set_bb (convert_stmt, new_bb);
5889 204 : gimple_set_bb (return_stmt, new_bb);
5890 204 : pop_cfun ();
5891 204 : return new_bb;
5892 : }
5893 :
5894 1327 : while (predicate_chain != NULL)
5895 : {
5896 683 : cond_var = create_tmp_var (integer_type_node);
5897 683 : predicate_decl = TREE_PURPOSE (predicate_chain);
5898 683 : predicate_arg = TREE_VALUE (predicate_chain);
5899 683 : call_cond_stmt = gimple_build_call (predicate_decl, 1, predicate_arg);
5900 683 : gimple_call_set_lhs (call_cond_stmt, cond_var);
5901 :
5902 683 : gimple_set_block (call_cond_stmt, DECL_INITIAL (function_decl));
5903 683 : gimple_set_bb (call_cond_stmt, new_bb);
5904 683 : gimple_seq_add_stmt (&gseq, call_cond_stmt);
5905 :
5906 683 : predicate_chain = TREE_CHAIN (predicate_chain);
5907 :
5908 683 : if (and_expr_var == NULL)
5909 : and_expr_var = cond_var;
5910 : else
5911 : {
5912 39 : gimple *assign_stmt;
5913 : /* Use MIN_EXPR to check if any integer is zero?.
5914 : and_expr_var = min_expr <cond_var, and_expr_var> */
5915 39 : assign_stmt = gimple_build_assign (and_expr_var,
5916 : build2 (MIN_EXPR, integer_type_node,
5917 : cond_var, and_expr_var));
5918 :
5919 39 : gimple_set_block (assign_stmt, DECL_INITIAL (function_decl));
5920 39 : gimple_set_bb (assign_stmt, new_bb);
5921 39 : gimple_seq_add_stmt (&gseq, assign_stmt);
5922 : }
5923 : }
5924 :
5925 644 : if_else_stmt = gimple_build_cond (GT_EXPR, and_expr_var,
5926 : integer_zero_node,
5927 : NULL_TREE, NULL_TREE);
5928 644 : gimple_set_block (if_else_stmt, DECL_INITIAL (function_decl));
5929 644 : gimple_set_bb (if_else_stmt, new_bb);
5930 644 : gimple_seq_add_stmt (&gseq, if_else_stmt);
5931 :
5932 644 : gimple_seq_add_stmt (&gseq, convert_stmt);
5933 644 : gimple_seq_add_stmt (&gseq, return_stmt);
5934 644 : set_bb_seq (new_bb, gseq);
5935 :
5936 644 : bb1 = new_bb;
5937 644 : e12 = split_block (bb1, if_else_stmt);
5938 644 : bb2 = e12->dest;
5939 644 : e12->flags &= ~EDGE_FALLTHRU;
5940 644 : e12->flags |= EDGE_TRUE_VALUE;
5941 :
5942 644 : e23 = split_block (bb2, return_stmt);
5943 :
5944 644 : gimple_set_bb (convert_stmt, bb2);
5945 644 : gimple_set_bb (return_stmt, bb2);
5946 :
5947 644 : bb3 = e23->dest;
5948 644 : make_edge (bb1, bb3, EDGE_FALSE_VALUE);
5949 :
5950 644 : remove_edge (e23);
5951 644 : make_edge (bb2, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
5952 :
5953 644 : pop_cfun ();
5954 :
5955 644 : return bb3;
5956 : }
5957 :
5958 : /* This function generates the dispatch function for
5959 : multi-versioned functions. DISPATCH_DECL is the function which will
5960 : contain the dispatch logic. FNDECLS are the function choices for
5961 : dispatch, and is a tree chain. EMPTY_BB is the basic block pointer
5962 : in DISPATCH_DECL in which the dispatch code is generated. */
5963 :
5964 : static int
5965 204 : dispatch_function_versions (tree dispatch_decl,
5966 : void *fndecls_p,
5967 : basic_block *empty_bb)
5968 : {
5969 204 : tree default_decl;
5970 204 : gimple *ifunc_cpu_init_stmt;
5971 204 : gimple_seq gseq;
5972 204 : int ix;
5973 204 : tree ele;
5974 204 : vec<tree> *fndecls;
5975 204 : unsigned int num_versions = 0;
5976 204 : unsigned int actual_versions = 0;
5977 204 : unsigned int i;
5978 :
5979 204 : struct _function_version_info
5980 : {
5981 : tree version_decl;
5982 : tree predicate_chain;
5983 : unsigned int dispatch_priority;
5984 : }*function_version_info;
5985 :
5986 204 : gcc_assert (dispatch_decl != NULL
5987 : && fndecls_p != NULL
5988 : && empty_bb != NULL);
5989 :
5990 : /*fndecls_p is actually a vector. */
5991 204 : fndecls = static_cast<vec<tree> *> (fndecls_p);
5992 :
5993 : /* At least one more version other than the default. */
5994 204 : num_versions = fndecls->length ();
5995 204 : gcc_assert (num_versions >= 2);
5996 :
5997 204 : function_version_info = (struct _function_version_info *)
5998 204 : XNEWVEC (struct _function_version_info, (num_versions - 1));
5999 :
6000 : /* The first version in the vector is the default decl. */
6001 204 : default_decl = (*fndecls)[0];
6002 :
6003 204 : push_cfun (DECL_STRUCT_FUNCTION (dispatch_decl));
6004 :
6005 204 : gseq = bb_seq (*empty_bb);
6006 : /* Function version dispatch is via IFUNC. IFUNC resolvers fire before
6007 : constructors, so explicitly call __builtin_cpu_init here. */
6008 204 : ifunc_cpu_init_stmt
6009 204 : = gimple_build_call_vec (get_ix86_builtin (IX86_BUILTIN_CPU_INIT), vNULL);
6010 204 : gimple_seq_add_stmt (&gseq, ifunc_cpu_init_stmt);
6011 204 : gimple_set_bb (ifunc_cpu_init_stmt, *empty_bb);
6012 204 : set_bb_seq (*empty_bb, gseq);
6013 :
6014 204 : pop_cfun ();
6015 :
6016 :
6017 1209 : for (ix = 1; fndecls->iterate (ix, &ele); ++ix)
6018 : {
6019 801 : tree version_decl = ele;
6020 801 : tree predicate_chain = NULL_TREE;
6021 801 : unsigned int priority;
6022 : /* Get attribute string, parse it and find the right predicate decl.
6023 : The predicate function could be a lengthy combination of many
6024 : features, like arch-type and various isa-variants. */
6025 801 : priority = get_builtin_code_for_version (version_decl,
6026 : &predicate_chain);
6027 :
6028 801 : if (predicate_chain == NULL_TREE)
6029 157 : continue;
6030 :
6031 644 : function_version_info [actual_versions].version_decl = version_decl;
6032 644 : function_version_info [actual_versions].predicate_chain
6033 644 : = predicate_chain;
6034 644 : function_version_info [actual_versions].dispatch_priority = priority;
6035 644 : actual_versions++;
6036 : }
6037 :
6038 : /* Sort the versions according to descending order of dispatch priority. The
6039 : priority is based on the ISA. This is not a perfect solution. There
6040 : could still be ambiguity. If more than one function version is suitable
6041 : to execute, which one should be dispatched? In future, allow the user
6042 : to specify a dispatch priority next to the version. */
6043 204 : qsort (function_version_info, actual_versions,
6044 : sizeof (struct _function_version_info), feature_compare);
6045 :
6046 1052 : for (i = 0; i < actual_versions; ++i)
6047 644 : *empty_bb = add_condition_to_bb (dispatch_decl,
6048 : function_version_info[i].version_decl,
6049 644 : function_version_info[i].predicate_chain,
6050 : *empty_bb);
6051 :
6052 : /* dispatch default version at the end. */
6053 204 : *empty_bb = add_condition_to_bb (dispatch_decl, default_decl,
6054 : NULL, *empty_bb);
6055 :
6056 204 : free (function_version_info);
6057 204 : return 0;
6058 : }
6059 :
6060 : /* This function changes the assembler name for functions that are
6061 : versions. If DECL is a function version and has a "target"
6062 : attribute, it appends the attribute string to its assembler name. */
6063 :
6064 : static tree
6065 1127 : ix86_mangle_function_version_assembler_name (tree decl, tree id)
6066 : {
6067 1127 : tree version_attr;
6068 1127 : char *attr_str;
6069 :
6070 1127 : if (DECL_DECLARED_INLINE_P (decl)
6071 1176 : && lookup_attribute ("gnu_inline",
6072 49 : DECL_ATTRIBUTES (decl)))
6073 0 : error_at (DECL_SOURCE_LOCATION (decl),
6074 : "function versions cannot be marked as %<gnu_inline%>,"
6075 : " bodies have to be generated");
6076 :
6077 1127 : if (DECL_VIRTUAL_P (decl)
6078 2254 : || DECL_VINDEX (decl))
6079 0 : sorry ("virtual function multiversioning not supported");
6080 :
6081 1127 : version_attr = lookup_attribute ("target", DECL_ATTRIBUTES (decl));
6082 :
6083 : /* target attribute string cannot be NULL. */
6084 1127 : gcc_assert (version_attr != NULL_TREE);
6085 :
6086 1127 : attr_str = sorted_attr_string (TREE_VALUE (version_attr));
6087 :
6088 : /* Allow assembler name to be modified if already set. */
6089 1127 : if (DECL_ASSEMBLER_NAME_SET_P (decl))
6090 1112 : SET_DECL_RTL (decl, NULL);
6091 :
6092 1127 : tree ret = clone_identifier (id, attr_str, true);
6093 :
6094 1127 : XDELETEVEC (attr_str);
6095 :
6096 1127 : return ret;
6097 : }
6098 :
6099 : tree
6100 505736639 : ix86_mangle_decl_assembler_name (tree decl, tree id)
6101 : {
6102 : /* For function version, add the target suffix to the assembler name. */
6103 505736639 : if (TREE_CODE (decl) == FUNCTION_DECL)
6104 : {
6105 469535133 : cgraph_node *node = cgraph_node::get (decl);
6106 : /* Mangle all versions when annotated with target_clones, but only
6107 : non-default versions when annotated with target attributes. */
6108 469535133 : if (DECL_FUNCTION_VERSIONED (decl)
6109 469535133 : && (node->is_target_clone
6110 1092 : || !is_function_default_version (node->decl)))
6111 1127 : id = ix86_mangle_function_version_assembler_name (decl, id);
6112 : /* Mangle the dispatched symbol but only in the case of target clones. */
6113 469534006 : else if (node && node->dispatcher_function && !node->is_target_clone)
6114 117 : id = clone_identifier (id, "ifunc");
6115 67398264 : else if (node && node->dispatcher_resolver_function)
6116 204 : id = clone_identifier (id, "resolver");
6117 : }
6118 : #ifdef SUBTARGET_MANGLE_DECL_ASSEMBLER_NAME
6119 : id = SUBTARGET_MANGLE_DECL_ASSEMBLER_NAME (decl, id);
6120 : #endif
6121 :
6122 505736639 : return id;
6123 : }
6124 :
6125 : /* Make a dispatcher declaration for the multi-versioned function DECL.
6126 : Calls to DECL function will be replaced with calls to the dispatcher
6127 : by the front-end. Returns the decl of the dispatcher function. */
6128 :
6129 : tree
6130 330 : ix86_get_function_versions_dispatcher (void *decl)
6131 : {
6132 330 : tree fn = (tree) decl;
6133 330 : struct cgraph_node *node = NULL;
6134 330 : struct cgraph_node *default_node = NULL;
6135 330 : struct cgraph_function_version_info *node_v = NULL;
6136 :
6137 330 : tree dispatch_decl = NULL;
6138 :
6139 330 : struct cgraph_function_version_info *default_version_info = NULL;
6140 :
6141 660 : gcc_assert (fn != NULL && DECL_FUNCTION_VERSIONED (fn));
6142 :
6143 330 : node = cgraph_node::get (fn);
6144 330 : gcc_assert (node != NULL);
6145 :
6146 330 : node_v = node->function_version ();
6147 330 : gcc_assert (node_v != NULL);
6148 :
6149 330 : if (node_v->dispatcher_resolver != NULL)
6150 : return node_v->dispatcher_resolver;
6151 :
6152 : /* The default node is always the beginning of the chain. */
6153 : default_version_info = node_v;
6154 681 : while (default_version_info->prev != NULL)
6155 : default_version_info = default_version_info->prev;
6156 216 : default_node = default_version_info->this_node;
6157 :
6158 : /* If there is no default node, just return NULL. */
6159 216 : if (!is_function_default_version (default_node->decl))
6160 : return NULL;
6161 :
6162 : #if defined (ASM_OUTPUT_TYPE_DIRECTIVE)
6163 207 : if (targetm.has_ifunc_p ())
6164 : {
6165 207 : struct cgraph_function_version_info *it_v = NULL;
6166 :
6167 : /* Right now, the dispatching is done via ifunc. */
6168 207 : dispatch_decl = make_dispatcher_decl (default_node->decl);
6169 :
6170 : /* Set the dispatcher for all the versions. */
6171 207 : it_v = default_version_info;
6172 1425 : while (it_v != NULL)
6173 : {
6174 1011 : it_v->dispatcher_resolver = dispatch_decl;
6175 1011 : it_v = it_v->next;
6176 : }
6177 : }
6178 : else
6179 : #endif
6180 : {
6181 0 : error_at (DECL_SOURCE_LOCATION (default_node->decl),
6182 : "multiversioning needs %<ifunc%> which is not supported "
6183 : "on this target");
6184 : }
6185 :
6186 : return dispatch_decl;
6187 : }
6188 :
6189 : /* Make the resolver function decl to dispatch the versions of
6190 : a multi-versioned function, DEFAULT_DECL. IFUNC_ALIAS_DECL is
6191 : ifunc alias that will point to the created resolver. Create an
6192 : empty basic block in the resolver and store the pointer in
6193 : EMPTY_BB. Return the decl of the resolver function. */
6194 :
6195 : static tree
6196 204 : make_resolver_func (const tree default_decl,
6197 : const tree ifunc_alias_decl,
6198 : basic_block *empty_bb)
6199 : {
6200 204 : tree decl, type, t;
6201 :
6202 : /* The resolver function should return a (void *). */
6203 204 : type = build_function_type_list (ptr_type_node, NULL_TREE);
6204 :
6205 204 : cgraph_node *node = cgraph_node::get (default_decl);
6206 204 : gcc_assert (node && node->function_version ());
6207 :
6208 204 : decl = build_fn_decl (IDENTIFIER_POINTER (DECL_NAME (default_decl)), type);
6209 :
6210 : /* Set the assembler name to prevent cgraph_node attempting to mangle. */
6211 204 : SET_DECL_ASSEMBLER_NAME (decl, DECL_ASSEMBLER_NAME (default_decl));
6212 :
6213 204 : cgraph_node *resolver_node = cgraph_node::get_create (decl);
6214 204 : resolver_node->dispatcher_resolver_function = true;
6215 :
6216 204 : if (node->is_target_clone)
6217 90 : resolver_node->is_target_clone = true;
6218 :
6219 204 : tree id = ix86_mangle_decl_assembler_name
6220 204 : (decl, node->function_version ()->assembler_name);
6221 204 : symtab->change_decl_assembler_name (decl, id);
6222 :
6223 204 : DECL_NAME (decl) = DECL_NAME (default_decl);
6224 204 : TREE_USED (decl) = 1;
6225 204 : DECL_ARTIFICIAL (decl) = 1;
6226 204 : DECL_IGNORED_P (decl) = 1;
6227 204 : TREE_PUBLIC (decl) = 0;
6228 204 : DECL_UNINLINABLE (decl) = 1;
6229 :
6230 : /* Resolver is not external, body is generated. */
6231 204 : DECL_EXTERNAL (decl) = 0;
6232 204 : DECL_EXTERNAL (ifunc_alias_decl) = 0;
6233 :
6234 204 : DECL_CONTEXT (decl) = NULL_TREE;
6235 204 : DECL_INITIAL (decl) = make_node (BLOCK);
6236 204 : DECL_STATIC_CONSTRUCTOR (decl) = 0;
6237 :
6238 204 : if (DECL_COMDAT_GROUP (default_decl)
6239 204 : || TREE_PUBLIC (default_decl))
6240 : {
6241 : /* In this case, each translation unit with a call to this
6242 : versioned function will put out a resolver. Ensure it
6243 : is comdat to keep just one copy. */
6244 179 : DECL_COMDAT (decl) = 1;
6245 179 : make_decl_one_only (decl, DECL_ASSEMBLER_NAME (decl));
6246 : }
6247 : else
6248 25 : TREE_PUBLIC (ifunc_alias_decl) = 0;
6249 :
6250 : /* Build result decl and add to function_decl. */
6251 204 : t = build_decl (UNKNOWN_LOCATION, RESULT_DECL, NULL_TREE, ptr_type_node);
6252 204 : DECL_CONTEXT (t) = decl;
6253 204 : DECL_ARTIFICIAL (t) = 1;
6254 204 : DECL_IGNORED_P (t) = 1;
6255 204 : DECL_RESULT (decl) = t;
6256 :
6257 204 : gimplify_function_tree (decl);
6258 204 : push_cfun (DECL_STRUCT_FUNCTION (decl));
6259 204 : *empty_bb = init_lowered_empty_function (decl, false,
6260 : profile_count::uninitialized ());
6261 :
6262 204 : cgraph_node::add_new_function (decl, true);
6263 204 : symtab->call_cgraph_insertion_hooks (cgraph_node::get_create (decl));
6264 :
6265 204 : pop_cfun ();
6266 :
6267 204 : gcc_assert (ifunc_alias_decl != NULL);
6268 : /* Mark ifunc_alias_decl as "ifunc" with resolver as resolver_name. */
6269 204 : DECL_ATTRIBUTES (ifunc_alias_decl)
6270 204 : = make_attribute ("ifunc", IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)),
6271 204 : DECL_ATTRIBUTES (ifunc_alias_decl));
6272 :
6273 : /* Create the alias for dispatch to resolver here. */
6274 204 : cgraph_node::create_same_body_alias (ifunc_alias_decl, decl);
6275 204 : return decl;
6276 : }
6277 :
6278 : /* Generate the dispatching code body to dispatch multi-versioned function
6279 : DECL. The target hook is called to process the "target" attributes and
6280 : provide the code to dispatch the right function at run-time. NODE points
6281 : to the dispatcher decl whose body will be created. */
6282 :
6283 : tree
6284 204 : ix86_generate_version_dispatcher_body (void *node_p)
6285 : {
6286 204 : tree resolver_decl;
6287 204 : basic_block empty_bb;
6288 204 : tree default_ver_decl;
6289 204 : struct cgraph_node *versn;
6290 204 : struct cgraph_node *node;
6291 :
6292 204 : struct cgraph_function_version_info *node_version_info = NULL;
6293 204 : struct cgraph_function_version_info *versn_info = NULL;
6294 :
6295 204 : node = (cgraph_node *)node_p;
6296 :
6297 204 : node_version_info = node->function_version ();
6298 204 : gcc_assert (node->dispatcher_function
6299 : && node_version_info != NULL);
6300 :
6301 204 : if (node_version_info->dispatcher_resolver)
6302 : return node_version_info->dispatcher_resolver;
6303 :
6304 : /* The first version in the chain corresponds to the default version. */
6305 204 : default_ver_decl = node_version_info->next->this_node->decl;
6306 :
6307 : /* node is going to be an alias, so remove the finalized bit. */
6308 204 : node->definition = false;
6309 :
6310 204 : resolver_decl = make_resolver_func (default_ver_decl,
6311 : node->decl, &empty_bb);
6312 :
6313 204 : node_version_info->dispatcher_resolver = resolver_decl;
6314 :
6315 204 : push_cfun (DECL_STRUCT_FUNCTION (resolver_decl));
6316 :
6317 204 : auto_vec<tree, 2> fn_ver_vec;
6318 :
6319 1209 : for (versn_info = node_version_info->next; versn_info;
6320 1005 : versn_info = versn_info->next)
6321 : {
6322 1005 : versn = versn_info->this_node;
6323 : /* Check for virtual functions here again, as by this time it should
6324 : have been determined if this function needs a vtable index or
6325 : not. This happens for methods in derived classes that override
6326 : virtual methods in base classes but are not explicitly marked as
6327 : virtual. */
6328 1005 : if (DECL_VIRTUAL_P (versn->decl))
6329 0 : sorry ("virtual function multiversioning not supported");
6330 :
6331 1005 : fn_ver_vec.safe_push (versn->decl);
6332 : }
6333 :
6334 204 : dispatch_function_versions (resolver_decl, &fn_ver_vec, &empty_bb);
6335 204 : cgraph_edge::rebuild_edges ();
6336 204 : pop_cfun ();
6337 204 : return resolver_decl;
6338 204 : }
6339 :
6340 :
|