Line data Source code
1 : /* Copyright (C) 1988-2026 Free Software Foundation, Inc.
2 :
3 : This file is part of GCC.
4 :
5 : GCC is free software; you can redistribute it and/or modify
6 : it under the terms of the GNU General Public License as published by
7 : the Free Software Foundation; either version 3, or (at your option)
8 : any later version.
9 :
10 : GCC is distributed in the hope that it will be useful,
11 : but WITHOUT ANY WARRANTY; without even the implied warranty of
12 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 : GNU General Public License for more details.
14 :
15 : You should have received a copy of the GNU General Public License
16 : along with GCC; see the file COPYING3. If not see
17 : <http://www.gnu.org/licenses/>. */
18 :
19 : #define IN_TARGET_CODE 1
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "memmodel.h"
28 : #include "gimple.h"
29 : #include "cfghooks.h"
30 : #include "cfgloop.h"
31 : #include "df.h"
32 : #include "tm_p.h"
33 : #include "stringpool.h"
34 : #include "expmed.h"
35 : #include "optabs.h"
36 : #include "regs.h"
37 : #include "emit-rtl.h"
38 : #include "recog.h"
39 : #include "cgraph.h"
40 : #include "diagnostic.h"
41 : #include "cfgbuild.h"
42 : #include "alias.h"
43 : #include "fold-const.h"
44 : #include "attribs.h"
45 : #include "calls.h"
46 : #include "stor-layout.h"
47 : #include "varasm.h"
48 : #include "output.h"
49 : #include "insn-attr.h"
50 : #include "flags.h"
51 : #include "except.h"
52 : #include "explow.h"
53 : #include "expr.h"
54 : #include "cfgrtl.h"
55 : #include "common/common-target.h"
56 : #include "langhooks.h"
57 : #include "reload.h"
58 : #include "gimplify.h"
59 : #include "dwarf2.h"
60 : #include "tm-constrs.h"
61 : #include "cselib.h"
62 : #include "sched-int.h"
63 : #include "opts.h"
64 : #include "tree-pass.h"
65 : #include "context.h"
66 : #include "pass_manager.h"
67 : #include "target-globals.h"
68 : #include "gimple-iterator.h"
69 : #include "shrink-wrap.h"
70 : #include "builtins.h"
71 : #include "rtl-iter.h"
72 : #include "tree-iterator.h"
73 : #include "dbgcnt.h"
74 : #include "case-cfn-macros.h"
75 : #include "dojump.h"
76 : #include "fold-const-call.h"
77 : #include "tree-vrp.h"
78 : #include "tree-ssanames.h"
79 : #include "selftest.h"
80 : #include "selftest-rtl.h"
81 : #include "print-rtl.h"
82 : #include "intl.h"
83 : #include "ifcvt.h"
84 : #include "symbol-summary.h"
85 : #include "sreal.h"
86 : #include "ipa-cp.h"
87 : #include "ipa-prop.h"
88 : #include "ipa-fnsummary.h"
89 : #include "wide-int-bitmask.h"
90 : #include "tree-vector-builder.h"
91 : #include "debug.h"
92 : #include "dwarf2out.h"
93 : #include "function-abi.h"
94 : #include "i386-builtins.h"
95 : #include "i386-features.h"
96 : #include "i386-expand.h"
97 :
98 : const char * const xlogue_layout::STUB_BASE_NAMES[XLOGUE_STUB_COUNT] = {
99 : "savms64",
100 : "resms64",
101 : "resms64x",
102 : "savms64f",
103 : "resms64f",
104 : "resms64fx"
105 : };
106 :
107 : const unsigned xlogue_layout::REG_ORDER[xlogue_layout::MAX_REGS] = {
108 : /* The below offset values are where each register is stored for the layout
109 : relative to incoming stack pointer. The value of each m_regs[].offset will
110 : be relative to the incoming base pointer (rax or rsi) used by the stub.
111 :
112 : s_instances: 0 1 2 3
113 : Offset: realigned or aligned + 8
114 : Register aligned aligned + 8 aligned w/HFP w/HFP */
115 : XMM15_REG, /* 0x10 0x18 0x10 0x18 */
116 : XMM14_REG, /* 0x20 0x28 0x20 0x28 */
117 : XMM13_REG, /* 0x30 0x38 0x30 0x38 */
118 : XMM12_REG, /* 0x40 0x48 0x40 0x48 */
119 : XMM11_REG, /* 0x50 0x58 0x50 0x58 */
120 : XMM10_REG, /* 0x60 0x68 0x60 0x68 */
121 : XMM9_REG, /* 0x70 0x78 0x70 0x78 */
122 : XMM8_REG, /* 0x80 0x88 0x80 0x88 */
123 : XMM7_REG, /* 0x90 0x98 0x90 0x98 */
124 : XMM6_REG, /* 0xa0 0xa8 0xa0 0xa8 */
125 : SI_REG, /* 0xa8 0xb0 0xa8 0xb0 */
126 : DI_REG, /* 0xb0 0xb8 0xb0 0xb8 */
127 : BX_REG, /* 0xb8 0xc0 0xb8 0xc0 */
128 : BP_REG, /* 0xc0 0xc8 N/A N/A */
129 : R12_REG, /* 0xc8 0xd0 0xc0 0xc8 */
130 : R13_REG, /* 0xd0 0xd8 0xc8 0xd0 */
131 : R14_REG, /* 0xd8 0xe0 0xd0 0xd8 */
132 : R15_REG, /* 0xe0 0xe8 0xd8 0xe0 */
133 : };
134 :
135 : /* Instantiate static const values. */
136 : const HOST_WIDE_INT xlogue_layout::STUB_INDEX_OFFSET;
137 : const unsigned xlogue_layout::MIN_REGS;
138 : const unsigned xlogue_layout::MAX_REGS;
139 : const unsigned xlogue_layout::MAX_EXTRA_REGS;
140 : const unsigned xlogue_layout::VARIANT_COUNT;
141 : const unsigned xlogue_layout::STUB_NAME_MAX_LEN;
142 :
143 : /* Initialize xlogue_layout::s_stub_names to zero. */
144 : char xlogue_layout::s_stub_names[2][XLOGUE_STUB_COUNT][VARIANT_COUNT]
145 : [STUB_NAME_MAX_LEN];
146 :
147 : /* Instantiates all xlogue_layout instances. */
148 : const xlogue_layout xlogue_layout::s_instances[XLOGUE_SET_COUNT] = {
149 : xlogue_layout (0, false),
150 : xlogue_layout (8, false),
151 : xlogue_layout (0, true),
152 : xlogue_layout (8, true)
153 : };
154 :
155 : /* Return an appropriate const instance of xlogue_layout based upon values
156 : in cfun->machine and crtl. */
157 : const class xlogue_layout &
158 49891 : xlogue_layout::get_instance ()
159 : {
160 49891 : enum xlogue_stub_sets stub_set;
161 49891 : bool aligned_plus_8 = cfun->machine->call_ms2sysv_pad_in;
162 :
163 49891 : if (stack_realign_fp)
164 : stub_set = XLOGUE_SET_HFP_ALIGNED_OR_REALIGN;
165 40910 : else if (frame_pointer_needed)
166 25246 : stub_set = aligned_plus_8
167 31552 : ? XLOGUE_SET_HFP_ALIGNED_PLUS_8
168 : : XLOGUE_SET_HFP_ALIGNED_OR_REALIGN;
169 : else
170 9358 : stub_set = aligned_plus_8 ? XLOGUE_SET_ALIGNED_PLUS_8 : XLOGUE_SET_ALIGNED;
171 :
172 49891 : return s_instances[stub_set];
173 : }
174 :
175 : /* Determine how many clobbered registers can be saved by the stub.
176 : Returns the count of registers the stub will save and restore. */
177 : unsigned
178 35225 : xlogue_layout::count_stub_managed_regs ()
179 : {
180 35225 : bool hfp = frame_pointer_needed || stack_realign_fp;
181 35225 : unsigned i, count;
182 35225 : unsigned regno;
183 :
184 93950 : for (count = i = MIN_REGS; i < MAX_REGS; ++i)
185 : {
186 92750 : regno = REG_ORDER[i];
187 92750 : if (regno == BP_REG && hfp)
188 17880 : continue;
189 74870 : if (!ix86_save_reg (regno, false, false))
190 : break;
191 40845 : ++count;
192 : }
193 35225 : return count;
194 : }
195 :
196 : /* Determine if register REGNO is a stub managed register given the
197 : total COUNT of stub managed registers. */
198 : bool
199 2642240 : xlogue_layout::is_stub_managed_reg (unsigned regno, unsigned count)
200 : {
201 2642240 : bool hfp = frame_pointer_needed || stack_realign_fp;
202 2642240 : unsigned i;
203 :
204 34418160 : for (i = 0; i < count; ++i)
205 : {
206 32275013 : gcc_assert (i < MAX_REGS);
207 32275013 : if (REG_ORDER[i] == BP_REG && hfp)
208 510254 : ++count;
209 31764759 : else if (REG_ORDER[i] == regno)
210 : return true;
211 : }
212 : return false;
213 : }
214 :
215 : /* Constructor for xlogue_layout. */
216 1178548 : xlogue_layout::xlogue_layout (HOST_WIDE_INT stack_align_off_in, bool hfp)
217 1178548 : : m_hfp (hfp) , m_nregs (hfp ? 17 : 18),
218 1178548 : m_stack_align_off_in (stack_align_off_in)
219 : {
220 1178548 : HOST_WIDE_INT offset = stack_align_off_in;
221 1178548 : unsigned i, j;
222 :
223 22392412 : for (i = j = 0; i < MAX_REGS; ++i)
224 : {
225 21213864 : unsigned regno = REG_ORDER[i];
226 :
227 21213864 : if (regno == BP_REG && hfp)
228 589274 : continue;
229 20624590 : if (SSE_REGNO_P (regno))
230 : {
231 11785480 : offset += 16;
232 : /* Verify that SSE regs are always aligned. */
233 11785480 : gcc_assert (!((stack_align_off_in + offset) & 15));
234 : }
235 : else
236 8839110 : offset += 8;
237 :
238 20624590 : m_regs[j].regno = regno;
239 20624590 : m_regs[j++].offset = offset - STUB_INDEX_OFFSET;
240 : }
241 1178548 : gcc_assert (j == m_nregs);
242 1178548 : }
243 :
244 : const char *
245 14666 : xlogue_layout::get_stub_name (enum xlogue_stub stub,
246 : unsigned n_extra_regs)
247 : {
248 14666 : const int have_avx = TARGET_AVX;
249 14666 : char *name = s_stub_names[!!have_avx][stub][n_extra_regs];
250 :
251 : /* Lazy init */
252 14666 : if (!*name)
253 : {
254 362 : int res = snprintf (name, STUB_NAME_MAX_LEN, "__%s_%s_%u",
255 : (have_avx ? "avx" : "sse"),
256 181 : STUB_BASE_NAMES[stub],
257 : MIN_REGS + n_extra_regs);
258 181 : gcc_checking_assert (res < (int)STUB_NAME_MAX_LEN);
259 : }
260 :
261 14666 : return name;
262 : }
263 :
264 : /* Return rtx of a symbol ref for the entry point (based upon
265 : cfun->machine->call_ms2sysv_extra_regs) of the specified stub. */
266 : rtx
267 14666 : xlogue_layout::get_stub_rtx (enum xlogue_stub stub)
268 : {
269 14666 : const unsigned n_extra_regs = cfun->machine->call_ms2sysv_extra_regs;
270 14666 : gcc_checking_assert (n_extra_regs <= MAX_EXTRA_REGS);
271 14666 : gcc_assert (stub < XLOGUE_STUB_COUNT);
272 14666 : gcc_assert (crtl->stack_realign_finalized);
273 :
274 14666 : return gen_rtx_SYMBOL_REF (Pmode, get_stub_name (stub, n_extra_regs));
275 : }
276 :
277 : unsigned scalar_chain::max_id = 0;
278 :
279 : namespace {
280 :
281 : /* Initialize new chain. */
282 :
283 6687765 : scalar_chain::scalar_chain (enum machine_mode smode_, enum machine_mode vmode_)
284 : {
285 6687765 : smode = smode_;
286 6687765 : vmode = vmode_;
287 :
288 6687765 : chain_id = ++max_id;
289 :
290 6687765 : if (dump_file)
291 136 : fprintf (dump_file, "Created a new instruction chain #%d\n", chain_id);
292 :
293 6687765 : bitmap_obstack_initialize (NULL);
294 6687765 : insns = BITMAP_ALLOC (NULL);
295 6687765 : defs = BITMAP_ALLOC (NULL);
296 6687765 : defs_conv = BITMAP_ALLOC (NULL);
297 6687765 : insns_conv = BITMAP_ALLOC (NULL);
298 6687765 : queue = NULL;
299 :
300 6687765 : cost_sse_integer = 0;
301 6687765 : weighted_cost_sse_integer = 0 ;
302 6687765 : max_visits = x86_stv_max_visits;
303 6687765 : }
304 :
305 : /* Free chain's data. */
306 :
307 6687765 : scalar_chain::~scalar_chain ()
308 : {
309 6687765 : BITMAP_FREE (insns);
310 6687765 : BITMAP_FREE (defs);
311 6687765 : BITMAP_FREE (defs_conv);
312 6687765 : BITMAP_FREE (insns_conv);
313 6687765 : bitmap_obstack_release (NULL);
314 6687765 : }
315 :
316 : /* Add instruction into chains' queue. */
317 :
318 : void
319 8449543 : scalar_chain::add_to_queue (unsigned insn_uid)
320 : {
321 8449543 : if (!bitmap_set_bit (queue, insn_uid))
322 : return;
323 :
324 6485904 : if (dump_file)
325 141 : fprintf (dump_file, " Adding insn %d into chain's #%d queue\n",
326 : insn_uid, chain_id);
327 : }
328 :
329 : /* For DImode conversion, mark register defined by DEF as requiring
330 : conversion. */
331 :
332 : void
333 9832324 : scalar_chain::mark_dual_mode_def (df_ref def)
334 : {
335 9832324 : gcc_assert (DF_REF_REG_DEF_P (def));
336 :
337 : /* Record the def/insn pair so we can later efficiently iterate over
338 : the defs to convert on insns not in the chain. */
339 9832324 : bool reg_new = bitmap_set_bit (defs_conv, DF_REF_REGNO (def));
340 9832324 : basic_block bb = BLOCK_FOR_INSN (DF_REF_INSN (def));
341 9832324 : profile_count entry_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
342 9832324 : bool speed_p = optimize_bb_for_speed_p (bb);
343 9832324 : int cost = 0;
344 :
345 9832324 : if (!bitmap_bit_p (insns, DF_REF_INSN_UID (def)))
346 : {
347 2973706 : if (!bitmap_set_bit (insns_conv, DF_REF_INSN_UID (def))
348 2973706 : && !reg_new)
349 1520331 : return;
350 :
351 : /* Cost integer to sse moves. */
352 2614445 : if (speed_p)
353 2302141 : cost = COSTS_N_INSNS (ix86_cost->integer_to_sse) / 2;
354 312304 : else if (TARGET_64BIT || smode == SImode)
355 : cost = COSTS_N_BYTES (4);
356 : /* vmovd (4 bytes) + vpinsrd (6 bytes). */
357 18775 : else if (TARGET_SSE4_1)
358 : cost = COSTS_N_BYTES (10);
359 : /* movd (4 bytes) + movd (4 bytes) + unpckldq (4 bytes). */
360 : else
361 8311993 : cost = COSTS_N_BYTES (12);
362 : }
363 : else
364 : {
365 6858618 : if (!reg_new)
366 : return;
367 :
368 : /* Cost sse to integer moves. */
369 5697548 : if (speed_p)
370 5107770 : cost = COSTS_N_INSNS (ix86_cost->sse_to_integer) / 2;
371 589778 : else if (TARGET_64BIT || smode == SImode)
372 : cost = COSTS_N_BYTES (4);
373 : /* vmovd (4 bytes) + vpextrd (6 bytes). */
374 2940 : else if (TARGET_SSE4_1)
375 : cost = COSTS_N_BYTES (10);
376 : /* movd (4 bytes) + psrlq (5 bytes) + movd (4 bytes). */
377 : else
378 8311993 : cost = COSTS_N_BYTES (13);
379 : }
380 :
381 8311993 : if (speed_p)
382 7409911 : weighted_cost_sse_integer += bb->count.to_sreal_scale (entry_count) * cost;
383 :
384 8311993 : cost_sse_integer += cost;
385 :
386 8311993 : if (dump_file)
387 236 : fprintf (dump_file,
388 : " Mark r%d def in insn %d as requiring both modes in chain #%d\n",
389 236 : DF_REF_REGNO (def), DF_REF_INSN_UID (def), chain_id);
390 : }
391 :
392 : /* Check REF's chain to add new insns into a queue
393 : and find registers requiring conversion. Return true if OK, false
394 : if the analysis was aborted. */
395 :
396 : bool
397 18558365 : scalar_chain::analyze_register_chain (bitmap candidates, df_ref ref,
398 : bitmap disallowed)
399 : {
400 18558365 : df_link *chain;
401 18558365 : bool mark_def = false;
402 :
403 18558365 : gcc_checking_assert (bitmap_bit_p (insns, DF_REF_INSN_UID (ref)));
404 :
405 63615303 : for (chain = DF_REF_CHAIN (ref); chain; chain = chain->next)
406 : {
407 45058503 : unsigned uid = DF_REF_INSN_UID (chain->ref);
408 :
409 45058503 : if (!NONDEBUG_INSN_P (DF_REF_INSN (chain->ref)))
410 8019355 : continue;
411 :
412 37039148 : if (--max_visits == 0)
413 : return false;
414 :
415 37038627 : if (!DF_REF_REG_MEM_P (chain->ref))
416 : {
417 31057557 : if (bitmap_bit_p (insns, uid))
418 9859445 : continue;
419 :
420 21198112 : if (bitmap_bit_p (candidates, uid))
421 : {
422 8449543 : add_to_queue (uid);
423 8449543 : continue;
424 : }
425 :
426 : /* If we run into parts of an aborted chain discovery abort. */
427 12748569 : if (bitmap_bit_p (disallowed, uid))
428 : return false;
429 : }
430 :
431 18728595 : if (DF_REF_REG_DEF_P (chain->ref))
432 : {
433 2973706 : if (dump_file)
434 125 : fprintf (dump_file, " r%d def in insn %d isn't convertible\n",
435 : DF_REF_REGNO (chain->ref), uid);
436 2973706 : mark_dual_mode_def (chain->ref);
437 : }
438 : else
439 : {
440 15754889 : if (dump_file)
441 524 : fprintf (dump_file, " r%d use in insn %d isn't convertible\n",
442 : DF_REF_REGNO (chain->ref), uid);
443 : mark_def = true;
444 : }
445 : }
446 :
447 18556800 : if (mark_def)
448 6858618 : mark_dual_mode_def (ref);
449 :
450 : return true;
451 : }
452 :
453 : /* Check whether X is a convertible *concatditi_? variant. X is known
454 : to be any_or_plus:TI, i.e. PLUS:TI, IOR:TI or XOR:TI. */
455 :
456 : static bool
457 31270 : timode_concatdi_p (rtx x)
458 : {
459 31270 : rtx op0 = XEXP (x, 0);
460 31270 : rtx op1 = XEXP (x, 1);
461 :
462 31270 : if (GET_CODE (op1) == ASHIFT)
463 956 : std::swap (op0, op1);
464 :
465 31270 : return GET_CODE (op0) == ASHIFT
466 21912 : && GET_CODE (XEXP (op0, 0)) == ZERO_EXTEND
467 21912 : && GET_MODE (XEXP (XEXP (op0, 0), 0)) == DImode
468 21912 : && REG_P (XEXP (XEXP (op0, 0), 0))
469 21761 : && CONST_INT_P (XEXP (op0, 1))
470 21761 : && INTVAL (XEXP (op0, 1)) == 64
471 21761 : && GET_CODE (op1) == ZERO_EXTEND
472 20805 : && GET_MODE (XEXP (op1, 0)) == DImode
473 52075 : && REG_P (XEXP (op1, 0));
474 : }
475 :
476 :
477 : /* Add instruction into a chain. Return true if OK, false if the search
478 : was aborted. */
479 :
480 : bool
481 13171933 : scalar_chain::add_insn (bitmap candidates, unsigned int insn_uid,
482 : bitmap disallowed)
483 : {
484 13171933 : if (!bitmap_set_bit (insns, insn_uid))
485 : return true;
486 :
487 13171933 : if (dump_file)
488 277 : fprintf (dump_file, " Adding insn %d to chain #%d\n", insn_uid, chain_id);
489 :
490 13171933 : rtx_insn *insn = DF_INSN_UID_GET (insn_uid)->insn;
491 13171933 : rtx def_set = single_set (insn);
492 13171933 : if (def_set && REG_P (SET_DEST (def_set))
493 23303428 : && !HARD_REGISTER_P (SET_DEST (def_set)))
494 10106134 : bitmap_set_bit (defs, REGNO (SET_DEST (def_set)));
495 :
496 : /* ??? The following is quadratic since analyze_register_chain
497 : iterates over all refs to look for dual-mode regs. Instead this
498 : should be done separately for all regs mentioned in the chain once. */
499 13171933 : df_ref ref;
500 26873909 : for (ref = DF_INSN_UID_DEFS (insn_uid); ref; ref = DF_REF_NEXT_LOC (ref))
501 13702759 : if (!HARD_REGISTER_P (DF_REF_REG (ref)))
502 10106134 : if (!analyze_register_chain (candidates, ref, disallowed))
503 : return false;
504 :
505 : /* The operand(s) of VEC_SELECT, ZERO_EXTEND and similar ops don't need
506 : to be converted/convertible. */
507 13171150 : if (def_set)
508 13171150 : switch (GET_CODE (SET_SRC (def_set)))
509 : {
510 3952333 : case REG:
511 3952333 : if (HARD_REGISTER_P (SET_SRC (def_set)))
512 : return true;
513 : break;
514 : case VEC_SELECT:
515 : return true;
516 9994 : case ZERO_EXTEND:
517 9994 : if (GET_MODE (XEXP (SET_SRC (def_set), 0)) == DImode)
518 : return true;
519 : break;
520 2452002 : case PLUS:
521 2452002 : case IOR:
522 2452002 : case XOR:
523 2452002 : if (smode == TImode && timode_concatdi_p (SET_SRC (def_set)))
524 : return true;
525 : break;
526 : default:
527 : break;
528 : }
529 :
530 28849136 : for (ref = DF_INSN_UID_USES (insn_uid); ref; ref = DF_REF_NEXT_LOC (ref))
531 15732242 : if (DF_REF_TYPE (ref) == DF_REF_REG_USE
532 8452234 : && !SUBREG_P (DF_REF_REG (ref)))
533 8452231 : if (!analyze_register_chain (candidates, ref, disallowed))
534 : return false;
535 :
536 : return true;
537 : }
538 :
539 : /* Build new chain starting from insn INSN_UID recursively
540 : adding all dependent uses and definitions. Return true if OK, false
541 : if the chain discovery was aborted. */
542 :
543 : bool
544 6687765 : scalar_chain::build (bitmap candidates, unsigned insn_uid, bitmap disallowed)
545 : {
546 6687765 : queue = BITMAP_ALLOC (NULL);
547 6687765 : bitmap_set_bit (queue, insn_uid);
548 :
549 6687765 : if (dump_file)
550 136 : fprintf (dump_file, "Building chain #%d...\n", chain_id);
551 :
552 19858133 : while (!bitmap_empty_p (queue))
553 : {
554 13171933 : insn_uid = bitmap_first_set_bit (queue);
555 13171933 : bitmap_clear_bit (queue, insn_uid);
556 13171933 : bitmap_clear_bit (candidates, insn_uid);
557 13171933 : if (!add_insn (candidates, insn_uid, disallowed))
558 : {
559 : /* If we aborted the search put sofar found insn on the set of
560 : disallowed insns so that further searches reaching them also
561 : abort and thus we abort the whole but yet undiscovered chain. */
562 1565 : bitmap_ior_into (disallowed, insns);
563 1565 : if (dump_file)
564 0 : fprintf (dump_file, "Aborted chain #%d discovery\n", chain_id);
565 1565 : BITMAP_FREE (queue);
566 1565 : return false;
567 : }
568 : }
569 :
570 6686200 : if (dump_file)
571 : {
572 136 : fprintf (dump_file, "Collected chain #%d...\n", chain_id);
573 136 : fprintf (dump_file, " insns: ");
574 136 : dump_bitmap (dump_file, insns);
575 136 : if (!bitmap_empty_p (defs_conv))
576 : {
577 136 : bitmap_iterator bi;
578 136 : unsigned id;
579 136 : const char *comma = "";
580 136 : fprintf (dump_file, " defs to convert: ");
581 362 : EXECUTE_IF_SET_IN_BITMAP (defs_conv, 0, id, bi)
582 : {
583 226 : fprintf (dump_file, "%sr%d", comma, id);
584 226 : comma = ", ";
585 : }
586 136 : fprintf (dump_file, "\n");
587 : }
588 : }
589 :
590 6686200 : BITMAP_FREE (queue);
591 :
592 6686200 : return true;
593 : }
594 :
595 : /* Return a cost of building a vector constant
596 : instead of using a scalar one. */
597 :
598 : int
599 2730160 : general_scalar_chain::vector_const_cost (rtx exp, basic_block bb)
600 : {
601 2730160 : gcc_assert (CONST_INT_P (exp));
602 :
603 2730160 : if (standard_sse_constant_p (exp, vmode))
604 615954 : return ix86_cost->sse_op;
605 2114206 : if (optimize_bb_for_size_p (bb))
606 : return COSTS_N_BYTES (8);
607 : /* We have separate costs for SImode and DImode, use SImode costs
608 : for smaller modes. */
609 2488879 : return COSTS_N_INSNS (ix86_cost->sse_load[smode == DImode ? 1 : 0]) / 2;
610 : }
611 :
612 : /* Return true if it's cost profitable for chain conversion. */
613 :
614 : bool
615 6154933 : general_scalar_chain::compute_convert_gain ()
616 : {
617 6154933 : bitmap_iterator bi;
618 6154933 : unsigned insn_uid;
619 6154933 : int gain = 0;
620 6154933 : sreal weighted_gain = 0;
621 :
622 6154933 : if (dump_file)
623 136 : fprintf (dump_file, "Computing gain for chain #%d...\n", chain_id);
624 :
625 : /* SSE costs distinguish between SImode and DImode loads/stores, for
626 : int costs factor in the number of GPRs involved. When supporting
627 : smaller modes than SImode the int load/store costs need to be
628 : adjusted as well. */
629 6154933 : unsigned sse_cost_idx = smode == DImode ? 1 : 0;
630 6154933 : int m = smode == DImode ? (TARGET_64BIT ? 1 : 2) : 1;
631 :
632 18261683 : EXECUTE_IF_SET_IN_BITMAP (insns, 0, insn_uid, bi)
633 : {
634 12106750 : rtx_insn *insn = DF_INSN_UID_GET (insn_uid)->insn;
635 12106750 : rtx def_set = single_set (insn);
636 12106750 : rtx src = SET_SRC (def_set);
637 12106750 : rtx dst = SET_DEST (def_set);
638 12106750 : basic_block bb = BLOCK_FOR_INSN (insn);
639 12106750 : int igain = 0;
640 12106750 : profile_count entry_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
641 12106750 : bool speed_p = optimize_bb_for_speed_p (bb);
642 12106750 : sreal bb_freq = bb->count.to_sreal_scale (entry_count);
643 :
644 12106750 : if (REG_P (src) && REG_P (dst))
645 : {
646 977869 : if (!speed_p)
647 : /* reg-reg move is 2 bytes, while SSE 3. */
648 216538 : igain += COSTS_N_BYTES (2 * m - 3);
649 : else
650 : /* Move costs are normalized to reg-reg move having cost 2. */
651 761331 : igain += COSTS_N_INSNS (2 * m - ix86_cost->xmm_move) / 2;
652 : }
653 11128881 : else if (REG_P (src) && MEM_P (dst))
654 : {
655 2425820 : if (!speed_p)
656 : /* Integer load/store is 3+ bytes and SSE 4+. */
657 204454 : igain += COSTS_N_BYTES (3 * m - 4);
658 : else
659 2221366 : igain
660 2221366 : += COSTS_N_INSNS (m * ix86_cost->int_store[2]
661 : - ix86_cost->sse_store[sse_cost_idx]) / 2;
662 : }
663 8703061 : else if (MEM_P (src) && REG_P (dst))
664 : {
665 3931993 : if (!speed_p)
666 368445 : igain += COSTS_N_BYTES (3 * m - 4);
667 : else
668 3563548 : igain += COSTS_N_INSNS (m * ix86_cost->int_load[2]
669 : - ix86_cost->sse_load[sse_cost_idx]) / 2;
670 : }
671 : else
672 : {
673 : /* For operations on memory operands, include the overhead
674 : of explicit load and store instructions. */
675 4771068 : if (MEM_P (dst))
676 : {
677 72958 : if (!speed_p)
678 : /* ??? This probably should account size difference
679 : of SSE and integer load rather than full SSE load. */
680 : igain -= COSTS_N_BYTES (8);
681 : else
682 : {
683 63184 : int cost = (m * (ix86_cost->int_load[2]
684 63184 : + ix86_cost->int_store[2])
685 63184 : - (ix86_cost->sse_load[sse_cost_idx] +
686 63184 : ix86_cost->sse_store[sse_cost_idx]));
687 63184 : igain += COSTS_N_INSNS (cost) / 2;
688 : }
689 : }
690 :
691 4771068 : switch (GET_CODE (src))
692 : {
693 497772 : case ASHIFT:
694 497772 : case ASHIFTRT:
695 497772 : case LSHIFTRT:
696 497772 : if (m == 2)
697 : {
698 17901 : if (INTVAL (XEXP (src, 1)) >= 32)
699 12306 : igain += ix86_cost->add;
700 : /* Gain for extend highpart case. */
701 5595 : else if (GET_CODE (XEXP (src, 0)) == ASHIFT)
702 0 : igain += ix86_cost->shift_const - ix86_cost->sse_op;
703 : else
704 5595 : igain += ix86_cost->shift_const;
705 : }
706 :
707 497772 : igain += ix86_cost->shift_const - ix86_cost->sse_op;
708 :
709 497772 : if (CONST_INT_P (XEXP (src, 0)))
710 0 : igain -= vector_const_cost (XEXP (src, 0), bb);
711 : break;
712 :
713 3942 : case ROTATE:
714 3942 : case ROTATERT:
715 3942 : igain += m * ix86_cost->shift_const;
716 3942 : if (TARGET_AVX512VL)
717 204 : igain -= ix86_cost->sse_op;
718 3738 : else if (smode == DImode)
719 : {
720 798 : int bits = INTVAL (XEXP (src, 1));
721 798 : if ((bits & 0x0f) == 0)
722 193 : igain -= ix86_cost->sse_op;
723 605 : else if ((bits & 0x07) == 0)
724 31 : igain -= 2 * ix86_cost->sse_op;
725 : else
726 574 : igain -= 3 * ix86_cost->sse_op;
727 : }
728 2940 : else if (INTVAL (XEXP (src, 1)) == 16)
729 139 : igain -= ix86_cost->sse_op;
730 : else
731 2801 : igain -= 2 * ix86_cost->sse_op;
732 : break;
733 :
734 2951601 : case AND:
735 2951601 : case IOR:
736 2951601 : case XOR:
737 2951601 : case PLUS:
738 2951601 : case MINUS:
739 2951601 : igain += m * ix86_cost->add - ix86_cost->sse_op;
740 : /* Additional gain for andnot for targets without BMI. */
741 2951601 : if (GET_CODE (XEXP (src, 0)) == NOT
742 3145 : && !TARGET_BMI)
743 3136 : igain += m * ix86_cost->add;
744 :
745 2951601 : if (CONST_INT_P (XEXP (src, 0)))
746 0 : igain -= vector_const_cost (XEXP (src, 0), bb);
747 2951601 : if (CONST_INT_P (XEXP (src, 1)))
748 1748790 : igain -= vector_const_cost (XEXP (src, 1), bb);
749 2951601 : if (MEM_P (XEXP (src, 1)))
750 : {
751 102617 : if (!speed_p)
752 23614 : igain -= COSTS_N_BYTES (m == 2 ? 3 : 5);
753 : else
754 90805 : igain += COSTS_N_INSNS
755 : (m * ix86_cost->int_load[2]
756 : - ix86_cost->sse_load[sse_cost_idx]) / 2;
757 : }
758 : break;
759 :
760 59264 : case NEG:
761 59264 : case NOT:
762 59264 : igain -= ix86_cost->sse_op + COSTS_N_INSNS (1);
763 :
764 59264 : if (GET_CODE (XEXP (src, 0)) != ABS)
765 : {
766 59264 : igain += m * ix86_cost->add;
767 59264 : break;
768 : }
769 : /* FALLTHRU */
770 :
771 968 : case ABS:
772 968 : case SMAX:
773 968 : case SMIN:
774 968 : case UMAX:
775 968 : case UMIN:
776 : /* We do not have any conditional move cost, estimate it as a
777 : reg-reg move. Comparisons are costed as adds. */
778 968 : igain += m * (COSTS_N_INSNS (2) + ix86_cost->add);
779 : /* Integer SSE ops are all costed the same. */
780 968 : igain -= ix86_cost->sse_op;
781 968 : break;
782 :
783 0 : case COMPARE:
784 0 : if (XEXP (src, 1) != const0_rtx)
785 : {
786 : /* cmp vs. pxor;pshufd;ptest. */
787 0 : igain += COSTS_N_INSNS (m - 3);
788 : }
789 0 : else if (GET_CODE (XEXP (src, 0)) != AND)
790 : {
791 : /* test vs. pshufd;ptest. */
792 0 : igain += COSTS_N_INSNS (m - 2);
793 : }
794 0 : else if (GET_CODE (XEXP (XEXP (src, 0), 0)) != NOT)
795 : {
796 : /* and;test vs. pshufd;ptest. */
797 0 : igain += COSTS_N_INSNS (2 * m - 2);
798 : }
799 0 : else if (TARGET_BMI)
800 : {
801 : /* andn;test vs. pandn;pshufd;ptest. */
802 0 : igain += COSTS_N_INSNS (2 * m - 3);
803 : }
804 : else
805 : {
806 : /* not;and;test vs. pandn;pshufd;ptest. */
807 0 : igain += COSTS_N_INSNS (3 * m - 3);
808 : }
809 : break;
810 :
811 1223959 : case CONST_INT:
812 1223959 : if (REG_P (dst))
813 : {
814 1223959 : if (!speed_p)
815 : {
816 : /* xor (2 bytes) vs. xorps (3 bytes). */
817 242589 : if (src == const0_rtx)
818 126816 : igain -= COSTS_N_BYTES (1);
819 : /* movdi_internal vs. movv2di_internal. */
820 : /* => mov (5 bytes) vs. movaps (7 bytes). */
821 115773 : else if (x86_64_immediate_operand (src, SImode))
822 103495 : igain -= COSTS_N_BYTES (2);
823 : else
824 : /* ??? Larger immediate constants are placed in the
825 : constant pool, where the size benefit/impact of
826 : STV conversion is affected by whether and how
827 : often each constant pool entry is shared/reused.
828 : The value below is empirically derived from the
829 : CSiBE benchmark (and the optimal value may drift
830 : over time). */
831 : igain += COSTS_N_BYTES (0);
832 : }
833 : else
834 : {
835 : /* DImode can be immediate for TARGET_64BIT
836 : and SImode always. */
837 981370 : igain += m * COSTS_N_INSNS (1);
838 981370 : igain -= vector_const_cost (src, bb);
839 : }
840 : }
841 0 : else if (MEM_P (dst))
842 : {
843 0 : igain += (m * ix86_cost->int_store[2]
844 0 : - ix86_cost->sse_store[sse_cost_idx]);
845 0 : igain -= vector_const_cost (src, bb);
846 : }
847 : break;
848 :
849 23838 : case VEC_SELECT:
850 23838 : if (XVECEXP (XEXP (src, 1), 0, 0) == const0_rtx)
851 : {
852 : // movd (4 bytes) replaced with movdqa (4 bytes).
853 18790 : if (!!speed_p)
854 17160 : igain += COSTS_N_INSNS (ix86_cost->sse_to_integer
855 : - ix86_cost->xmm_move) / 2;
856 : }
857 : else
858 : {
859 : // pshufd; movd replaced with pshufd.
860 5048 : if (!speed_p)
861 538 : igain += COSTS_N_BYTES (4);
862 : else
863 4510 : igain += ix86_cost->sse_to_integer;
864 : }
865 : break;
866 :
867 9724 : case ZERO_EXTEND:
868 : /* mov eax (6 bytes) vs movd xmm0 (8 bytes). */
869 : /* mov eax; xor edx,edx (7 bytes). */
870 9724 : if (speed_p)
871 8858 : igain += COSTS_N_INSNS (ix86_cost->int_load[2]
872 : - ix86_cost->sse_load[0]) / 2;
873 : else
874 912 : igain += COSTS_N_BYTES (TARGET_64BIT ? -2 : -1);
875 : break;
876 :
877 0 : default:
878 0 : gcc_unreachable ();
879 : }
880 : }
881 :
882 12105120 : if (speed_p)
883 10763665 : weighted_gain += bb_freq * igain;
884 12106750 : gain += igain;
885 :
886 12106750 : if (igain != 0 && dump_file)
887 : {
888 93 : fprintf (dump_file, " Instruction gain %d with bb_freq %.2f for",
889 : igain, bb_freq.to_double ());
890 93 : dump_insn_slim (dump_file, insn);
891 : }
892 : }
893 :
894 6154933 : if (dump_file)
895 : {
896 136 : fprintf (dump_file, " Instruction conversion gain: %d, \n",
897 : gain);
898 136 : fprintf (dump_file, " Registers conversion cost: %d\n",
899 : cost_sse_integer);
900 136 : fprintf (dump_file, " Weighted instruction conversion gain: %.2f, \n",
901 : weighted_gain.to_double ());
902 136 : fprintf (dump_file, " Weighted registers conversion cost: %.2f\n",
903 : weighted_cost_sse_integer.to_double ());
904 : }
905 :
906 6154933 : if (weighted_gain != weighted_cost_sse_integer)
907 4965832 : return weighted_gain > weighted_cost_sse_integer;
908 : else
909 1189101 : return gain > cost_sse_integer;;
910 : }
911 :
912 : /* Insert generated conversion instruction sequence INSNS
913 : after instruction AFTER. New BB may be required in case
914 : instruction has EH region attached. */
915 :
916 : void
917 32996 : scalar_chain::emit_conversion_insns (rtx insns, rtx_insn *after)
918 : {
919 32996 : if (!control_flow_insn_p (after))
920 : {
921 32788 : emit_insn_after (insns, after);
922 32788 : return;
923 : }
924 :
925 208 : basic_block bb = BLOCK_FOR_INSN (after);
926 208 : edge e = find_fallthru_edge (bb->succs);
927 208 : gcc_assert (e);
928 :
929 208 : basic_block new_bb = split_edge (e);
930 208 : emit_insn_after (insns, BB_HEAD (new_bb));
931 : }
932 :
933 : } // anon namespace
934 :
935 : /* Generate the canonical SET_SRC to move GPR to a VMODE vector register,
936 : zeroing the upper parts. */
937 :
938 : static rtx
939 172670 : gen_gpr_to_xmm_move_src (enum machine_mode vmode, rtx gpr)
940 : {
941 345340 : switch (GET_MODE_NUNITS (vmode))
942 : {
943 71 : case 1:
944 71 : return gen_rtx_SUBREG (vmode, gpr, 0);
945 172025 : case 2:
946 344050 : return gen_rtx_VEC_CONCAT (vmode, gpr,
947 : CONST0_RTX (GET_MODE_INNER (vmode)));
948 574 : default:
949 574 : return gen_rtx_VEC_MERGE (vmode, gen_rtx_VEC_DUPLICATE (vmode, gpr),
950 : CONST0_RTX (vmode), GEN_INT (HOST_WIDE_INT_1U));
951 : }
952 : }
953 :
954 : /* Make vector copies for all register REGNO definitions
955 : and replace its uses in a chain. */
956 :
957 : void
958 8705 : scalar_chain::make_vector_copies (rtx_insn *insn, rtx reg)
959 : {
960 8705 : rtx vreg = *defs_map.get (reg);
961 :
962 8705 : start_sequence ();
963 8705 : if (!TARGET_INTER_UNIT_MOVES_TO_VEC)
964 : {
965 0 : rtx tmp = assign_386_stack_local (smode, SLOT_STV_TEMP);
966 0 : if (smode == DImode && !TARGET_64BIT)
967 : {
968 0 : emit_move_insn (adjust_address (tmp, SImode, 0),
969 : gen_rtx_SUBREG (SImode, reg, 0));
970 0 : emit_move_insn (adjust_address (tmp, SImode, 4),
971 : gen_rtx_SUBREG (SImode, reg, 4));
972 : }
973 : else
974 0 : emit_move_insn (copy_rtx (tmp), reg);
975 0 : emit_insn (gen_rtx_SET (gen_rtx_SUBREG (vmode, vreg, 0),
976 : gen_gpr_to_xmm_move_src (vmode, tmp)));
977 : }
978 8705 : else if (!TARGET_64BIT && smode == DImode)
979 : {
980 8640 : if (TARGET_SSE4_1)
981 : {
982 280 : emit_insn (gen_sse2_loadld (gen_rtx_SUBREG (V4SImode, vreg, 0),
983 : CONST0_RTX (V4SImode),
984 : gen_rtx_SUBREG (SImode, reg, 0)));
985 280 : emit_insn (gen_sse4_1_pinsrd (gen_rtx_SUBREG (V4SImode, vreg, 0),
986 : gen_rtx_SUBREG (V4SImode, vreg, 0),
987 : gen_rtx_SUBREG (SImode, reg, 4),
988 : GEN_INT (2)));
989 : }
990 : else
991 : {
992 8360 : rtx tmp = gen_reg_rtx (DImode);
993 8360 : emit_insn (gen_sse2_loadld (gen_rtx_SUBREG (V4SImode, vreg, 0),
994 : CONST0_RTX (V4SImode),
995 : gen_rtx_SUBREG (SImode, reg, 0)));
996 8360 : emit_insn (gen_sse2_loadld (gen_rtx_SUBREG (V4SImode, tmp, 0),
997 : CONST0_RTX (V4SImode),
998 : gen_rtx_SUBREG (SImode, reg, 4)));
999 8360 : emit_insn (gen_vec_interleave_lowv4si
1000 : (gen_rtx_SUBREG (V4SImode, vreg, 0),
1001 : gen_rtx_SUBREG (V4SImode, vreg, 0),
1002 : gen_rtx_SUBREG (V4SImode, tmp, 0)));
1003 : }
1004 : }
1005 : else
1006 65 : emit_insn (gen_rtx_SET (gen_rtx_SUBREG (vmode, vreg, 0),
1007 : gen_gpr_to_xmm_move_src (vmode, reg)));
1008 8705 : rtx_insn *seq = end_sequence ();
1009 8705 : emit_conversion_insns (seq, insn);
1010 :
1011 8705 : if (dump_file)
1012 0 : fprintf (dump_file,
1013 : " Copied r%d to a vector register r%d for insn %d\n",
1014 0 : REGNO (reg), REGNO (vreg), INSN_UID (insn));
1015 8705 : }
1016 :
1017 : /* Copy the definition SRC of INSN inside the chain to DST for
1018 : scalar uses outside of the chain. */
1019 :
1020 : void
1021 23551 : scalar_chain::convert_reg (rtx_insn *insn, rtx dst, rtx src)
1022 : {
1023 23551 : start_sequence ();
1024 23551 : if (!TARGET_INTER_UNIT_MOVES_FROM_VEC)
1025 : {
1026 0 : rtx tmp = assign_386_stack_local (smode, SLOT_STV_TEMP);
1027 0 : emit_move_insn (tmp, src);
1028 0 : if (!TARGET_64BIT && smode == DImode)
1029 : {
1030 0 : emit_move_insn (gen_rtx_SUBREG (SImode, dst, 0),
1031 : adjust_address (tmp, SImode, 0));
1032 0 : emit_move_insn (gen_rtx_SUBREG (SImode, dst, 4),
1033 : adjust_address (tmp, SImode, 4));
1034 : }
1035 : else
1036 0 : emit_move_insn (dst, copy_rtx (tmp));
1037 : }
1038 23551 : else if (!TARGET_64BIT && smode == DImode)
1039 : {
1040 22535 : if (TARGET_SSE4_1)
1041 : {
1042 0 : rtx tmp = gen_rtx_PARALLEL (VOIDmode,
1043 : gen_rtvec (1, const0_rtx));
1044 0 : emit_insn
1045 0 : (gen_rtx_SET
1046 : (gen_rtx_SUBREG (SImode, dst, 0),
1047 : gen_rtx_VEC_SELECT (SImode,
1048 : gen_rtx_SUBREG (V4SImode, src, 0),
1049 : tmp)));
1050 :
1051 0 : tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const1_rtx));
1052 0 : emit_insn
1053 0 : (gen_rtx_SET
1054 : (gen_rtx_SUBREG (SImode, dst, 4),
1055 : gen_rtx_VEC_SELECT (SImode,
1056 : gen_rtx_SUBREG (V4SImode, src, 0),
1057 : tmp)));
1058 : }
1059 : else
1060 : {
1061 22535 : rtx vcopy = gen_reg_rtx (V2DImode);
1062 22535 : emit_move_insn (vcopy, gen_rtx_SUBREG (V2DImode, src, 0));
1063 22535 : emit_move_insn (gen_rtx_SUBREG (SImode, dst, 0),
1064 : gen_rtx_SUBREG (SImode, vcopy, 0));
1065 22535 : emit_move_insn (vcopy,
1066 : gen_rtx_LSHIFTRT (V2DImode,
1067 : vcopy, GEN_INT (32)));
1068 22535 : emit_move_insn (gen_rtx_SUBREG (SImode, dst, 4),
1069 : gen_rtx_SUBREG (SImode, vcopy, 0));
1070 : }
1071 : }
1072 : else
1073 1016 : emit_move_insn (dst, src);
1074 :
1075 23551 : rtx_insn *seq = end_sequence ();
1076 23551 : emit_conversion_insns (seq, insn);
1077 :
1078 23551 : if (dump_file)
1079 0 : fprintf (dump_file,
1080 : " Copied r%d to a scalar register r%d for insn %d\n",
1081 0 : REGNO (src), REGNO (dst), INSN_UID (insn));
1082 23551 : }
1083 :
1084 : /* Helper function to convert immediate constant X to vmode. */
1085 : static rtx
1086 38247 : smode_convert_cst (rtx x, enum machine_mode vmode)
1087 : {
1088 : /* Prefer all ones vector in case of -1. */
1089 38247 : if (constm1_operand (x, GET_MODE (x)))
1090 626 : return CONSTM1_RTX (vmode);
1091 :
1092 37621 : unsigned n = GET_MODE_NUNITS (vmode);
1093 37621 : rtx *v = XALLOCAVEC (rtx, n);
1094 37621 : v[0] = x;
1095 43610 : for (unsigned i = 1; i < n; ++i)
1096 5989 : v[i] = const0_rtx;
1097 37621 : return gen_rtx_CONST_VECTOR (vmode, gen_rtvec_v (n, v));
1098 : }
1099 :
1100 : /* Convert operand OP in INSN. We should handle
1101 : memory operands and uninitialized registers.
1102 : All other register uses are converted during
1103 : registers conversion. */
1104 :
1105 : void
1106 239391 : scalar_chain::convert_op (rtx *op, rtx_insn *insn)
1107 : {
1108 239391 : rtx tmp;
1109 :
1110 239391 : if (GET_MODE (*op) == V1TImode)
1111 : return;
1112 :
1113 239138 : *op = copy_rtx_if_shared (*op);
1114 :
1115 239138 : if (GET_CODE (*op) == NOT
1116 239138 : || GET_CODE (*op) == ASHIFT)
1117 : {
1118 2892 : convert_op (&XEXP (*op, 0), insn);
1119 2892 : PUT_MODE (*op, vmode);
1120 : }
1121 : else if (MEM_P (*op))
1122 : {
1123 172605 : rtx_insn *movabs = NULL;
1124 :
1125 : /* Emit MOVABS to load from a 64-bit absolute address to a GPR. */
1126 172605 : if (!memory_operand (*op, GET_MODE (*op)))
1127 : {
1128 0 : tmp = gen_reg_rtx (GET_MODE (*op));
1129 0 : movabs = emit_insn_before (gen_rtx_SET (tmp, *op), insn);
1130 :
1131 0 : *op = tmp;
1132 : }
1133 :
1134 172605 : tmp = gen_rtx_SUBREG (vmode, gen_reg_rtx (GET_MODE (*op)), 0);
1135 :
1136 172605 : rtx_insn *eh_insn
1137 172605 : = emit_insn_before (gen_rtx_SET (copy_rtx (tmp),
1138 : gen_gpr_to_xmm_move_src (vmode, *op)),
1139 172605 : insn);
1140 :
1141 172605 : if (cfun->can_throw_non_call_exceptions)
1142 : {
1143 : /* Handle REG_EH_REGION note. */
1144 168371 : rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
1145 168371 : if (note)
1146 : {
1147 3578 : if (movabs)
1148 0 : eh_insn = movabs;
1149 3578 : control_flow_insns.safe_push (eh_insn);
1150 3578 : add_reg_note (eh_insn, REG_EH_REGION, XEXP (note, 0));
1151 : }
1152 : }
1153 :
1154 172605 : *op = tmp;
1155 :
1156 172605 : if (dump_file)
1157 0 : fprintf (dump_file, " Preloading operand for insn %d into r%d\n",
1158 0 : INSN_UID (insn), reg_or_subregno (tmp));
1159 : }
1160 : else if (REG_P (*op))
1161 57164 : *op = gen_rtx_SUBREG (vmode, *op, 0);
1162 : else if (CONST_SCALAR_INT_P (*op))
1163 : {
1164 6474 : rtx vec_cst = smode_convert_cst (*op, vmode);
1165 :
1166 6474 : if (!standard_sse_constant_p (vec_cst, vmode))
1167 : {
1168 2679 : start_sequence ();
1169 2679 : vec_cst = validize_mem (force_const_mem (vmode, vec_cst));
1170 2679 : rtx_insn *seq = end_sequence ();
1171 2679 : emit_insn_before (seq, insn);
1172 : }
1173 :
1174 6474 : tmp = gen_rtx_SUBREG (vmode, gen_reg_rtx (smode), 0);
1175 :
1176 6474 : emit_insn_before (gen_move_insn (copy_rtx (tmp), vec_cst), insn);
1177 6474 : *op = tmp;
1178 : }
1179 : else
1180 : {
1181 0 : gcc_assert (SUBREG_P (*op));
1182 3 : if (GET_MODE (*op) != vmode)
1183 : {
1184 3 : rtx inner = SUBREG_REG (*op);
1185 3 : poly_uint64 byte = SUBREG_BYTE (*op);
1186 3 : if (targetm.modes_tieable_p (vmode, GET_MODE (inner))
1187 3 : && validate_subreg (vmode, GET_MODE (inner), inner, byte))
1188 3 : *op = gen_lowpart (vmode, *op);
1189 : else
1190 : {
1191 0 : tmp = gen_reg_rtx (GET_MODE (*op));
1192 0 : emit_insn_before (gen_rtx_SET (tmp, *op), insn);
1193 0 : *op = gen_rtx_SUBREG (vmode, tmp, 0);
1194 : }
1195 : }
1196 : }
1197 : }
1198 :
1199 : /* Convert CCZmode COMPARE to vector mode. */
1200 :
1201 : rtx
1202 8 : scalar_chain::convert_compare (rtx op1, rtx op2, rtx_insn *insn)
1203 : {
1204 8 : rtx src, tmp;
1205 :
1206 : /* Handle any REG_EQUAL notes. */
1207 8 : tmp = find_reg_equal_equiv_note (insn);
1208 8 : if (tmp)
1209 : {
1210 1 : if (GET_CODE (XEXP (tmp, 0)) == COMPARE
1211 1 : && GET_MODE (XEXP (tmp, 0)) == CCZmode
1212 1 : && REG_P (XEXP (XEXP (tmp, 0), 0)))
1213 : {
1214 1 : rtx *op = &XEXP (XEXP (tmp, 0), 1);
1215 1 : if (CONST_SCALAR_INT_P (*op))
1216 : {
1217 1 : if (constm1_operand (*op, GET_MODE (*op)))
1218 0 : *op = CONSTM1_RTX (vmode);
1219 : else
1220 : {
1221 1 : unsigned n = GET_MODE_NUNITS (vmode);
1222 1 : rtx *v = XALLOCAVEC (rtx, n);
1223 1 : v[0] = *op;
1224 1 : for (unsigned i = 1; i < n; ++i)
1225 0 : v[i] = const0_rtx;
1226 1 : *op = gen_rtx_CONST_VECTOR (vmode, gen_rtvec_v (n, v));
1227 : }
1228 : tmp = NULL_RTX;
1229 : }
1230 0 : else if (REG_P (*op))
1231 : tmp = NULL_RTX;
1232 : }
1233 :
1234 1 : if (tmp)
1235 0 : remove_note (insn, tmp);
1236 : }
1237 :
1238 : /* Comparison against anything other than zero, requires an XOR. */
1239 8 : if (op2 != const0_rtx)
1240 : {
1241 6 : convert_op (&op1, insn);
1242 6 : convert_op (&op2, insn);
1243 : /* If both operands are MEMs, explicitly load the OP1 into TMP. */
1244 6 : if (MEM_P (op1) && MEM_P (op2))
1245 : {
1246 0 : tmp = gen_reg_rtx (vmode);
1247 0 : emit_insn_before (gen_rtx_SET (tmp, op1), insn);
1248 0 : src = tmp;
1249 : }
1250 : else
1251 : src = op1;
1252 6 : src = gen_rtx_XOR (vmode, src, op2);
1253 : }
1254 2 : else if (GET_CODE (op1) == AND
1255 0 : && GET_CODE (XEXP (op1, 0)) == NOT)
1256 : {
1257 0 : rtx op11 = XEXP (XEXP (op1, 0), 0);
1258 0 : rtx op12 = XEXP (op1, 1);
1259 0 : convert_op (&op11, insn);
1260 0 : convert_op (&op12, insn);
1261 0 : if (!REG_P (op11))
1262 : {
1263 0 : tmp = gen_reg_rtx (vmode);
1264 0 : emit_insn_before (gen_rtx_SET (tmp, op11), insn);
1265 0 : op11 = tmp;
1266 : }
1267 0 : src = gen_rtx_AND (vmode, gen_rtx_NOT (vmode, op11), op12);
1268 0 : }
1269 2 : else if (GET_CODE (op1) == AND)
1270 : {
1271 0 : rtx op11 = XEXP (op1, 0);
1272 0 : rtx op12 = XEXP (op1, 1);
1273 0 : convert_op (&op11, insn);
1274 0 : convert_op (&op12, insn);
1275 0 : if (!REG_P (op11))
1276 : {
1277 0 : tmp = gen_reg_rtx (vmode);
1278 0 : emit_insn_before (gen_rtx_SET (tmp, op11), insn);
1279 0 : op11 = tmp;
1280 : }
1281 0 : return gen_rtx_UNSPEC (CCZmode, gen_rtvec (2, op11, op12),
1282 : UNSPEC_PTEST);
1283 : }
1284 : else
1285 : {
1286 2 : convert_op (&op1, insn);
1287 2 : src = op1;
1288 : }
1289 :
1290 8 : if (!REG_P (src))
1291 : {
1292 6 : tmp = gen_reg_rtx (vmode);
1293 6 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
1294 6 : src = tmp;
1295 : }
1296 :
1297 8 : if (vmode == V2DImode)
1298 : {
1299 0 : tmp = gen_reg_rtx (vmode);
1300 0 : emit_insn_before (gen_vec_interleave_lowv2di (tmp, src, src), insn);
1301 0 : src = tmp;
1302 : }
1303 8 : else if (vmode == V4SImode)
1304 : {
1305 0 : tmp = gen_reg_rtx (vmode);
1306 0 : emit_insn_before (gen_sse2_pshufd (tmp, src, const0_rtx), insn);
1307 0 : src = tmp;
1308 : }
1309 :
1310 8 : return gen_rtx_UNSPEC (CCZmode, gen_rtvec (2, src, src), UNSPEC_PTEST);
1311 : }
1312 :
1313 : /* Helper function for converting INSN to vector mode. */
1314 :
1315 : void
1316 1372751 : scalar_chain::convert_insn_common (rtx_insn *insn)
1317 : {
1318 : /* Generate copies for out-of-chain uses of defs and adjust debug uses. */
1319 2092596 : for (df_ref ref = DF_INSN_DEFS (insn); ref; ref = DF_REF_NEXT_LOC (ref))
1320 719845 : if (bitmap_bit_p (defs_conv, DF_REF_REGNO (ref)))
1321 : {
1322 24959 : df_link *use;
1323 49022 : for (use = DF_REF_CHAIN (ref); use; use = use->next)
1324 47614 : if (NONDEBUG_INSN_P (DF_REF_INSN (use->ref))
1325 47614 : && (DF_REF_REG_MEM_P (use->ref)
1326 41689 : || !bitmap_bit_p (insns, DF_REF_INSN_UID (use->ref))))
1327 : break;
1328 24959 : if (use)
1329 23551 : convert_reg (insn, DF_REF_REG (ref),
1330 23551 : *defs_map.get (regno_reg_rtx [DF_REF_REGNO (ref)]));
1331 1408 : else if (MAY_HAVE_DEBUG_BIND_INSNS)
1332 : {
1333 : /* If we generated a scalar copy we can leave debug-insns
1334 : as-is, if not, we have to adjust them. */
1335 1272 : auto_vec<rtx_insn *, 5> to_reset_debug_insns;
1336 3882 : for (use = DF_REF_CHAIN (ref); use; use = use->next)
1337 2610 : if (DEBUG_INSN_P (DF_REF_INSN (use->ref)))
1338 : {
1339 821 : rtx_insn *debug_insn = DF_REF_INSN (use->ref);
1340 : /* If there's a reaching definition outside of the
1341 : chain we have to reset. */
1342 821 : df_link *def;
1343 2883 : for (def = DF_REF_CHAIN (use->ref); def; def = def->next)
1344 2232 : if (!bitmap_bit_p (insns, DF_REF_INSN_UID (def->ref)))
1345 : break;
1346 821 : if (def)
1347 170 : to_reset_debug_insns.safe_push (debug_insn);
1348 : else
1349 : {
1350 651 : *DF_REF_REAL_LOC (use->ref)
1351 651 : = *defs_map.get (regno_reg_rtx [DF_REF_REGNO (ref)]);
1352 651 : df_insn_rescan (debug_insn);
1353 : }
1354 : }
1355 : /* Have to do the reset outside of the DF_CHAIN walk to not
1356 : disrupt it. */
1357 2714 : while (!to_reset_debug_insns.is_empty ())
1358 : {
1359 170 : rtx_insn *debug_insn = to_reset_debug_insns.pop ();
1360 170 : INSN_VAR_LOCATION_LOC (debug_insn) = gen_rtx_UNKNOWN_VAR_LOC ();
1361 170 : df_insn_rescan_debug_internal (debug_insn);
1362 : }
1363 1272 : }
1364 : }
1365 :
1366 : /* Replace uses in this insn with the defs we use in the chain. */
1367 3420827 : for (df_ref ref = DF_INSN_USES (insn); ref; ref = DF_REF_NEXT_LOC (ref))
1368 2048076 : if (!DF_REF_REG_MEM_P (ref))
1369 731293 : if (rtx *vreg = defs_map.get (regno_reg_rtx[DF_REF_REGNO (ref)]))
1370 : {
1371 : /* Also update a corresponding REG_DEAD note. */
1372 36265 : rtx note = find_reg_note (insn, REG_DEAD, DF_REF_REG (ref));
1373 36265 : if (note)
1374 22888 : XEXP (note, 0) = *vreg;
1375 36265 : *DF_REF_REAL_LOC (ref) = *vreg;
1376 : }
1377 1372751 : }
1378 :
1379 : /* Convert INSN which is an SImode or DImode rotation by a constant
1380 : to vector mode. CODE is either ROTATE or ROTATERT with operands
1381 : OP0 and OP1. Returns the SET_SRC of the last instruction in the
1382 : resulting sequence, which is emitted before INSN. */
1383 :
1384 : rtx
1385 92 : general_scalar_chain::convert_rotate (enum rtx_code code, rtx op0, rtx op1,
1386 : rtx_insn *insn)
1387 : {
1388 92 : int bits = INTVAL (op1);
1389 92 : rtx pat, result;
1390 :
1391 92 : convert_op (&op0, insn);
1392 92 : if (bits == 0)
1393 0 : return op0;
1394 :
1395 92 : if (smode == DImode)
1396 : {
1397 92 : if (code == ROTATE)
1398 45 : bits = 64 - bits;
1399 92 : if (bits == 32)
1400 : {
1401 0 : rtx tmp1 = gen_reg_rtx (V4SImode);
1402 0 : pat = gen_sse2_pshufd (tmp1, gen_lowpart (V4SImode, op0),
1403 : GEN_INT (225));
1404 0 : emit_insn_before (pat, insn);
1405 0 : result = gen_lowpart (V2DImode, tmp1);
1406 : }
1407 92 : else if (TARGET_AVX512VL)
1408 0 : result = simplify_gen_binary (code, V2DImode, op0, op1);
1409 92 : else if (bits == 16 || bits == 48)
1410 : {
1411 0 : rtx tmp1 = gen_reg_rtx (V8HImode);
1412 0 : pat = gen_sse2_pshuflw (tmp1, gen_lowpart (V8HImode, op0),
1413 : GEN_INT (bits == 16 ? 57 : 147));
1414 0 : emit_insn_before (pat, insn);
1415 0 : result = gen_lowpart (V2DImode, tmp1);
1416 : }
1417 92 : else if ((bits & 0x07) == 0)
1418 : {
1419 0 : rtx tmp1 = gen_reg_rtx (V4SImode);
1420 0 : pat = gen_sse2_pshufd (tmp1, gen_lowpart (V4SImode, op0),
1421 : GEN_INT (68));
1422 0 : emit_insn_before (pat, insn);
1423 0 : rtx tmp2 = gen_reg_rtx (V1TImode);
1424 0 : pat = gen_sse2_lshrv1ti3 (tmp2, gen_lowpart (V1TImode, tmp1),
1425 : GEN_INT (bits));
1426 0 : emit_insn_before (pat, insn);
1427 0 : result = gen_lowpart (V2DImode, tmp2);
1428 : }
1429 : else
1430 : {
1431 92 : rtx tmp1 = gen_reg_rtx (V4SImode);
1432 92 : pat = gen_sse2_pshufd (tmp1, gen_lowpart (V4SImode, op0),
1433 : GEN_INT (20));
1434 92 : emit_insn_before (pat, insn);
1435 92 : rtx tmp2 = gen_reg_rtx (V2DImode);
1436 92 : pat = gen_lshrv2di3 (tmp2, gen_lowpart (V2DImode, tmp1),
1437 : GEN_INT (bits & 31));
1438 92 : emit_insn_before (pat, insn);
1439 92 : rtx tmp3 = gen_reg_rtx (V4SImode);
1440 139 : pat = gen_sse2_pshufd (tmp3, gen_lowpart (V4SImode, tmp2),
1441 : GEN_INT (bits > 32 ? 34 : 136));
1442 92 : emit_insn_before (pat, insn);
1443 92 : result = gen_lowpart (V2DImode, tmp3);
1444 : }
1445 : }
1446 0 : else if (bits == 16)
1447 : {
1448 0 : rtx tmp1 = gen_reg_rtx (V8HImode);
1449 0 : pat = gen_sse2_pshuflw (tmp1, gen_lowpart (V8HImode, op0), GEN_INT (225));
1450 0 : emit_insn_before (pat, insn);
1451 0 : result = gen_lowpart (V4SImode, tmp1);
1452 : }
1453 0 : else if (TARGET_AVX512VL)
1454 0 : result = simplify_gen_binary (code, V4SImode, op0, op1);
1455 : else
1456 : {
1457 0 : if (code == ROTATE)
1458 0 : bits = 32 - bits;
1459 :
1460 0 : rtx tmp1 = gen_reg_rtx (V4SImode);
1461 0 : emit_insn_before (gen_sse2_pshufd (tmp1, op0, GEN_INT (224)), insn);
1462 0 : rtx tmp2 = gen_reg_rtx (V2DImode);
1463 0 : pat = gen_lshrv2di3 (tmp2, gen_lowpart (V2DImode, tmp1),
1464 : GEN_INT (bits));
1465 0 : emit_insn_before (pat, insn);
1466 0 : result = gen_lowpart (V4SImode, tmp2);
1467 : }
1468 :
1469 : return result;
1470 : }
1471 :
1472 : /* Convert INSN to vector mode. */
1473 :
1474 : void
1475 407309 : general_scalar_chain::convert_insn (rtx_insn *insn)
1476 : {
1477 407309 : rtx def_set = single_set (insn);
1478 407309 : rtx src = SET_SRC (def_set);
1479 407309 : rtx dst = SET_DEST (def_set);
1480 407309 : rtx subreg;
1481 :
1482 407309 : if (MEM_P (dst) && !REG_P (src))
1483 : {
1484 : /* There are no scalar integer instructions and therefore
1485 : temporary register usage is required. */
1486 740 : rtx tmp = gen_reg_rtx (smode);
1487 740 : emit_conversion_insns (gen_move_insn (dst, tmp), insn);
1488 740 : dst = gen_rtx_SUBREG (vmode, tmp, 0);
1489 740 : }
1490 406569 : else if (REG_P (dst) && GET_MODE (dst) == smode)
1491 : {
1492 : /* Replace the definition with a SUBREG to the definition we
1493 : use inside the chain. */
1494 211893 : rtx *vdef = defs_map.get (dst);
1495 211893 : if (vdef)
1496 24959 : dst = *vdef;
1497 211893 : dst = gen_rtx_SUBREG (vmode, dst, 0);
1498 : /* IRA doesn't like to have REG_EQUAL/EQUIV notes when the SET_DEST
1499 : is a non-REG_P. So kill those off. */
1500 211893 : rtx note = find_reg_equal_equiv_note (insn);
1501 211893 : if (note)
1502 9649 : remove_note (insn, note);
1503 : }
1504 :
1505 407309 : switch (GET_CODE (src))
1506 : {
1507 26596 : case PLUS:
1508 26596 : case MINUS:
1509 26596 : case IOR:
1510 26596 : case XOR:
1511 26596 : case AND:
1512 26596 : case SMAX:
1513 26596 : case SMIN:
1514 26596 : case UMAX:
1515 26596 : case UMIN:
1516 26596 : convert_op (&XEXP (src, 1), insn);
1517 : /* FALLTHRU */
1518 :
1519 33209 : case ABS:
1520 33209 : case ASHIFT:
1521 33209 : case ASHIFTRT:
1522 33209 : case LSHIFTRT:
1523 33209 : convert_op (&XEXP (src, 0), insn);
1524 33209 : PUT_MODE (src, vmode);
1525 33209 : break;
1526 :
1527 92 : case ROTATE:
1528 92 : case ROTATERT:
1529 92 : src = convert_rotate (GET_CODE (src), XEXP (src, 0), XEXP (src, 1),
1530 : insn);
1531 92 : break;
1532 :
1533 351 : case NEG:
1534 351 : src = XEXP (src, 0);
1535 :
1536 351 : if (GET_CODE (src) == ABS)
1537 : {
1538 0 : src = XEXP (src, 0);
1539 0 : convert_op (&src, insn);
1540 0 : subreg = gen_reg_rtx (vmode);
1541 0 : emit_insn_before (gen_rtx_SET (subreg,
1542 : gen_rtx_ABS (vmode, src)), insn);
1543 0 : src = subreg;
1544 : }
1545 : else
1546 351 : convert_op (&src, insn);
1547 :
1548 351 : subreg = gen_reg_rtx (vmode);
1549 351 : emit_insn_before (gen_move_insn (subreg, CONST0_RTX (vmode)), insn);
1550 351 : src = gen_rtx_MINUS (vmode, subreg, src);
1551 351 : break;
1552 :
1553 219 : case NOT:
1554 219 : src = XEXP (src, 0);
1555 219 : convert_op (&src, insn);
1556 219 : subreg = gen_reg_rtx (vmode);
1557 219 : emit_insn_before (gen_move_insn (subreg, CONSTM1_RTX (vmode)), insn);
1558 219 : src = gen_rtx_XOR (vmode, src, subreg);
1559 219 : break;
1560 :
1561 170394 : case MEM:
1562 170394 : if (!REG_P (dst))
1563 170394 : convert_op (&src, insn);
1564 : break;
1565 :
1566 196202 : case REG:
1567 196202 : if (!MEM_P (dst))
1568 1526 : convert_op (&src, insn);
1569 : break;
1570 :
1571 0 : case SUBREG:
1572 0 : gcc_assert (GET_MODE (src) == vmode);
1573 : break;
1574 :
1575 0 : case COMPARE:
1576 0 : dst = gen_rtx_REG (CCZmode, FLAGS_REG);
1577 0 : src = convert_compare (XEXP (src, 0), XEXP (src, 1), insn);
1578 0 : break;
1579 :
1580 3763 : case CONST_INT:
1581 3763 : convert_op (&src, insn);
1582 3763 : break;
1583 :
1584 2739 : case VEC_SELECT:
1585 2739 : if (XVECEXP (XEXP (src, 1), 0, 0) == const0_rtx)
1586 1586 : src = XEXP (src, 0);
1587 1153 : else if (smode == DImode)
1588 : {
1589 781 : rtx tmp = gen_lowpart (V1TImode, XEXP (src, 0));
1590 781 : dst = gen_lowpart (V1TImode, dst);
1591 781 : src = gen_rtx_LSHIFTRT (V1TImode, tmp, GEN_INT (64));
1592 : }
1593 : else
1594 : {
1595 372 : rtx tmp = XVECEXP (XEXP (src, 1), 0, 0);
1596 372 : rtvec vec = gen_rtvec (4, tmp, tmp, tmp, tmp);
1597 372 : rtx par = gen_rtx_PARALLEL (VOIDmode, vec);
1598 372 : src = gen_rtx_VEC_SELECT (vmode, XEXP (src, 0), par);
1599 : }
1600 : break;
1601 :
1602 340 : case ZERO_EXTEND:
1603 : /* *zero_extendsidi2 becomes *vec_setv2di_0_zero_extendsi_1. */
1604 340 : src = gen_rtx_VEC_CONCAT (V2DImode, src, const0_rtx);
1605 340 : break;
1606 :
1607 0 : default:
1608 0 : gcc_unreachable ();
1609 : }
1610 :
1611 407309 : SET_SRC (def_set) = src;
1612 407309 : SET_DEST (def_set) = dst;
1613 :
1614 : /* Drop possible dead definitions. */
1615 407309 : PATTERN (insn) = def_set;
1616 :
1617 407309 : INSN_CODE (insn) = -1;
1618 407309 : int patt = recog_memoized (insn);
1619 407309 : if (patt == -1)
1620 0 : fatal_insn_not_found (insn);
1621 407309 : df_insn_rescan (insn);
1622 407309 : }
1623 :
1624 : /* Helper function to compute gain for loading an immediate constant.
1625 : Typically, two movabsq for TImode vs. vmovdqa for V1TImode, but
1626 : with numerous special cases. */
1627 :
1628 : static int
1629 34 : timode_immed_const_gain (rtx cst, basic_block bb)
1630 : {
1631 : /* movabsq vs. movabsq+vmovq+vunpacklqdq. */
1632 34 : if (CONST_WIDE_INT_P (cst)
1633 23 : && CONST_WIDE_INT_NUNITS (cst) == 2
1634 57 : && CONST_WIDE_INT_ELT (cst, 0) == CONST_WIDE_INT_ELT (cst, 1))
1635 0 : return optimize_bb_for_size_p (bb) ? -COSTS_N_BYTES (9)
1636 : : -COSTS_N_INSNS (2);
1637 : /* 2x movabsq ~ vmovdqa. */
1638 : return 0;
1639 : }
1640 :
1641 : /* Return true it's cost profitable for for chain conversion. */
1642 :
1643 : bool
1644 531267 : timode_scalar_chain::compute_convert_gain ()
1645 : {
1646 : /* Assume that if we have to move TImode values between units,
1647 : then transforming this chain isn't worth it. */
1648 531267 : if (cost_sse_integer)
1649 : return false;
1650 :
1651 531267 : bitmap_iterator bi;
1652 531267 : unsigned insn_uid;
1653 :
1654 : /* Split ties to prefer V1TImode when not optimizing for size. */
1655 531267 : int gain = optimize_size ? 0 : 1;
1656 531267 : sreal weighted_gain = 0;
1657 :
1658 531267 : if (dump_file)
1659 0 : fprintf (dump_file, "Computing gain for chain #%d...\n", chain_id);
1660 :
1661 1586428 : EXECUTE_IF_SET_IN_BITMAP (insns, 0, insn_uid, bi)
1662 : {
1663 1055161 : rtx_insn *insn = DF_INSN_UID_GET (insn_uid)->insn;
1664 1055161 : rtx def_set = single_set (insn);
1665 1055161 : rtx src = SET_SRC (def_set);
1666 1055161 : rtx dst = SET_DEST (def_set);
1667 1055161 : HOST_WIDE_INT op1val;
1668 1055161 : basic_block bb = BLOCK_FOR_INSN (insn);
1669 1055161 : int scost, vcost;
1670 1055161 : int igain = 0;
1671 1055161 : profile_count entry_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
1672 1055161 : bool speed_p = optimize_bb_for_speed_p (bb);
1673 1055161 : sreal bb_freq = bb->count.to_sreal_scale (entry_count);
1674 :
1675 1055161 : switch (GET_CODE (src))
1676 : {
1677 548147 : case REG:
1678 548147 : if (GENERAL_REGNO_P (REGNO (src)))
1679 : {
1680 24973 : if (TARGET_SSE4_1)
1681 : /* movq + pinsrq */
1682 27 : igain = speed_p ? -ix86_cost->integer_to_sse
1683 : - COSTS_N_INSNS (1)
1684 : : -COSTS_N_BYTES (11);
1685 : else
1686 : /* movq + movq + punpcklqdq */
1687 24946 : igain = speed_p ? -ix86_cost->integer_to_sse
1688 : - COSTS_N_INSNS (2)
1689 : : -COSTS_N_BYTES (14);
1690 : }
1691 523174 : else if (GENERAL_REG_P (dst))
1692 : {
1693 24974 : if (TARGET_SSE4_1)
1694 : /* pextrq + movq */
1695 27 : igain = speed_p ? -ix86_cost->sse_to_integer
1696 : - COSTS_N_INSNS (1)
1697 : : -COSTS_N_BYTES (11);
1698 : else
1699 : /* movhlps + movq + movq */
1700 24947 : igain = speed_p ? -ix86_cost->sse_to_integer
1701 : - COSTS_N_INSNS (2)
1702 : : -COSTS_N_BYTES (13);
1703 : }
1704 498200 : else if (!speed_p)
1705 15699 : igain = MEM_P (dst) ? COSTS_N_BYTES (6) : COSTS_N_BYTES (3);
1706 : else
1707 : igain = COSTS_N_INSNS (1);
1708 : break;
1709 :
1710 456903 : case MEM:
1711 456903 : igain = !speed_p ? COSTS_N_BYTES (7) : COSTS_N_INSNS (1);
1712 : break;
1713 :
1714 12665 : case CONST_INT:
1715 12665 : if (MEM_P (dst)
1716 12665 : && standard_sse_constant_p (src, V1TImode))
1717 12217 : igain = !speed_p ? COSTS_N_BYTES (11) : 1;
1718 : break;
1719 :
1720 32138 : case CONST_WIDE_INT:
1721 : /* 2 x mov vs. vmovdqa. */
1722 32138 : if (MEM_P (dst))
1723 31579 : igain = !speed_p ? COSTS_N_BYTES (3) : COSTS_N_INSNS (1);
1724 : break;
1725 :
1726 78 : case NOT:
1727 78 : if (MEM_P (dst))
1728 68284 : igain = -COSTS_N_INSNS (1);
1729 : break;
1730 :
1731 69 : case AND:
1732 69 : if (!MEM_P (dst))
1733 : {
1734 58 : igain = COSTS_N_INSNS (1);
1735 58 : if (MEM_P (XEXP (src, 0)) || MEM_P (XEXP (src, 1)))
1736 : /* One less load. */
1737 20 : igain += COSTS_N_INSNS (1);
1738 : }
1739 69 : if (CONST_SCALAR_INT_P (XEXP (src, 1)))
1740 27 : igain += timode_immed_const_gain (XEXP (src, 1), bb);
1741 : break;
1742 :
1743 4528 : case XOR:
1744 4528 : case IOR:
1745 4528 : if (timode_concatdi_p (src))
1746 : {
1747 : /* vmovq;vpinsrq (11 bytes). */
1748 4385 : igain = speed_p ? -ix86_cost->integer_to_sse - COSTS_N_INSNS (1)
1749 : : -COSTS_N_BYTES (11);
1750 : break;
1751 : }
1752 143 : if (!MEM_P (dst))
1753 : {
1754 135 : igain = COSTS_N_INSNS (1);
1755 135 : if (MEM_P (XEXP (src, 0)) || MEM_P (XEXP (src, 1)))
1756 : /* One less load. */
1757 48 : igain += COSTS_N_INSNS (1);
1758 : }
1759 143 : if (CONST_SCALAR_INT_P (XEXP (src, 1)))
1760 7 : igain += timode_immed_const_gain (XEXP (src, 1), bb);
1761 : break;
1762 :
1763 0 : case PLUS:
1764 0 : if (timode_concatdi_p (src))
1765 : /* vmovq;vpinsrq (11 bytes). */
1766 0 : igain = speed_p ? -ix86_cost->integer_to_sse - COSTS_N_INSNS (1)
1767 : : -COSTS_N_BYTES (11);
1768 : break;
1769 :
1770 220 : case ASHIFT:
1771 220 : case LSHIFTRT:
1772 : /* See ix86_expand_v1ti_shift. */
1773 220 : op1val = INTVAL (XEXP (src, 1));
1774 220 : if (!speed_p)
1775 : {
1776 21 : if (op1val == 64 || op1val == 65)
1777 : scost = COSTS_N_BYTES (5);
1778 13 : else if (op1val >= 66)
1779 : scost = COSTS_N_BYTES (6);
1780 13 : else if (op1val == 1)
1781 : scost = COSTS_N_BYTES (8);
1782 : else
1783 : scost = COSTS_N_BYTES (9);
1784 :
1785 19 : if ((op1val & 7) == 0)
1786 : vcost = COSTS_N_BYTES (5);
1787 13 : else if (op1val > 64)
1788 : vcost = COSTS_N_BYTES (10);
1789 : else
1790 13 : vcost = TARGET_AVX ? COSTS_N_BYTES (19) : COSTS_N_BYTES (23);
1791 : }
1792 : else
1793 : {
1794 199 : scost = COSTS_N_INSNS (2);
1795 199 : if ((op1val & 7) == 0)
1796 : vcost = COSTS_N_INSNS (1);
1797 131 : else if (op1val > 64)
1798 : vcost = COSTS_N_INSNS (2);
1799 : else
1800 131 : vcost = TARGET_AVX ? COSTS_N_INSNS (4) : COSTS_N_INSNS (5);
1801 : }
1802 220 : igain = scost - vcost;
1803 220 : break;
1804 :
1805 124 : case ASHIFTRT:
1806 : /* See ix86_expand_v1ti_ashiftrt. */
1807 124 : op1val = INTVAL (XEXP (src, 1));
1808 124 : if (!speed_p)
1809 : {
1810 9 : if (op1val == 64 || op1val == 127)
1811 : scost = COSTS_N_BYTES (7);
1812 9 : else if (op1val == 1)
1813 : scost = COSTS_N_BYTES (8);
1814 8 : else if (op1val == 65)
1815 : scost = COSTS_N_BYTES (10);
1816 8 : else if (op1val >= 66)
1817 : scost = COSTS_N_BYTES (11);
1818 : else
1819 : scost = COSTS_N_BYTES (9);
1820 :
1821 0 : if (op1val == 127)
1822 : vcost = COSTS_N_BYTES (10);
1823 9 : else if (op1val == 64)
1824 : vcost = COSTS_N_BYTES (14);
1825 9 : else if (op1val == 96)
1826 : vcost = COSTS_N_BYTES (18);
1827 9 : else if (op1val >= 111)
1828 : vcost = COSTS_N_BYTES (15);
1829 9 : else if (TARGET_AVX2 && op1val == 32)
1830 : vcost = COSTS_N_BYTES (16);
1831 9 : else if (TARGET_SSE4_1 && op1val == 32)
1832 : vcost = COSTS_N_BYTES (20);
1833 9 : else if (op1val >= 96)
1834 : vcost = COSTS_N_BYTES (23);
1835 9 : else if ((op1val & 7) == 0)
1836 : vcost = COSTS_N_BYTES (28);
1837 9 : else if (TARGET_AVX2 && op1val < 32)
1838 : vcost = COSTS_N_BYTES (30);
1839 9 : else if (op1val == 1 || op1val >= 64)
1840 : vcost = COSTS_N_BYTES (42);
1841 : else
1842 8 : vcost = COSTS_N_BYTES (47);
1843 : }
1844 : else
1845 : {
1846 115 : if (op1val >= 65 && op1val <= 126)
1847 : scost = COSTS_N_INSNS (3);
1848 : else
1849 115 : scost = COSTS_N_INSNS (2);
1850 :
1851 115 : if (op1val == 127)
1852 : vcost = COSTS_N_INSNS (2);
1853 114 : else if (op1val == 64)
1854 : vcost = COSTS_N_INSNS (3);
1855 114 : else if (op1val == 96)
1856 : vcost = COSTS_N_INSNS (3);
1857 114 : else if (op1val >= 111)
1858 : vcost = COSTS_N_INSNS (3);
1859 114 : else if (TARGET_SSE4_1 && op1val == 32)
1860 : vcost = COSTS_N_INSNS (3);
1861 114 : else if (TARGET_SSE4_1
1862 0 : && (op1val == 8 || op1val == 16 || op1val == 24))
1863 : vcost = COSTS_N_INSNS (3);
1864 114 : else if (op1val >= 96)
1865 : vcost = COSTS_N_INSNS (4);
1866 114 : else if (TARGET_SSE4_1 && (op1val == 28 || op1val == 80))
1867 : vcost = COSTS_N_INSNS (4);
1868 114 : else if ((op1val & 7) == 0)
1869 : vcost = COSTS_N_INSNS (5);
1870 114 : else if (TARGET_AVX2 && op1val < 32)
1871 : vcost = COSTS_N_INSNS (6);
1872 114 : else if (TARGET_SSE4_1 && op1val < 15)
1873 : vcost = COSTS_N_INSNS (6);
1874 114 : else if (op1val == 1 || op1val >= 64)
1875 : vcost = COSTS_N_INSNS (8);
1876 : else
1877 17 : vcost = COSTS_N_INSNS (9);
1878 : }
1879 124 : igain = scost - vcost;
1880 124 : break;
1881 :
1882 6 : case ROTATE:
1883 6 : case ROTATERT:
1884 : /* See ix86_expand_v1ti_rotate. */
1885 6 : op1val = INTVAL (XEXP (src, 1));
1886 6 : if (!speed_p)
1887 : {
1888 0 : scost = COSTS_N_BYTES (13);
1889 0 : if ((op1val & 31) == 0)
1890 : vcost = COSTS_N_BYTES (5);
1891 0 : else if ((op1val & 7) == 0)
1892 0 : vcost = TARGET_AVX ? COSTS_N_BYTES (13) : COSTS_N_BYTES (18);
1893 0 : else if (op1val > 32 && op1val < 96)
1894 : vcost = COSTS_N_BYTES (24);
1895 : else
1896 0 : vcost = COSTS_N_BYTES (19);
1897 : }
1898 : else
1899 : {
1900 6 : scost = COSTS_N_INSNS (3);
1901 6 : if ((op1val & 31) == 0)
1902 : vcost = COSTS_N_INSNS (1);
1903 4 : else if ((op1val & 7) == 0)
1904 1 : vcost = TARGET_AVX ? COSTS_N_INSNS (3) : COSTS_N_INSNS (4);
1905 3 : else if (op1val > 32 && op1val < 96)
1906 : vcost = COSTS_N_INSNS (5);
1907 : else
1908 3 : vcost = COSTS_N_INSNS (4);
1909 : }
1910 6 : igain = scost - vcost;
1911 6 : break;
1912 :
1913 13 : case COMPARE:
1914 13 : if (XEXP (src, 1) == const0_rtx)
1915 : {
1916 2 : if (GET_CODE (XEXP (src, 0)) == AND)
1917 : /* and;and;or (9 bytes) vs. ptest (5 bytes). */
1918 : igain = !speed_p ? COSTS_N_BYTES (4) : COSTS_N_INSNS (2);
1919 : /* or (3 bytes) vs. ptest (5 bytes). */
1920 2 : else if (!speed_p)
1921 0 : igain = -COSTS_N_BYTES (2);
1922 : }
1923 11 : else if (XEXP (src, 1) == const1_rtx)
1924 : /* and;cmp -1 (7 bytes) vs. pcmpeqd;pxor;ptest (13 bytes). */
1925 0 : igain = !speed_p ? -COSTS_N_BYTES (6) : -COSTS_N_INSNS (1);
1926 : break;
1927 :
1928 270 : case ZERO_EXTEND:
1929 270 : if (GET_MODE (XEXP (src, 0)) == DImode)
1930 : /* xor (2 bytes) vs. vmovq (5 bytes). */
1931 270 : igain = speed_p ? COSTS_N_INSNS (1) - ix86_cost->sse_to_integer
1932 : : -COSTS_N_BYTES (3);
1933 : break;
1934 :
1935 : default:
1936 : break;
1937 : }
1938 :
1939 2076221 : gain += igain;
1940 1055159 : if (speed_p)
1941 1021062 : weighted_gain += bb_freq * igain;
1942 :
1943 1055161 : if (igain != 0 && dump_file)
1944 : {
1945 0 : fprintf (dump_file, " Instruction gain %d with bb_freq %.2f for ",
1946 : igain, bb_freq.to_double ());
1947 0 : dump_insn_slim (dump_file, insn);
1948 : }
1949 : }
1950 :
1951 531267 : if (dump_file)
1952 0 : fprintf (dump_file, " Total gain: %d, weighted gain %.2f\n",
1953 : gain, weighted_gain.to_double ());
1954 :
1955 531267 : if (weighted_gain > (sreal) 0)
1956 : return true;
1957 : else
1958 56201 : return gain > 0;
1959 : }
1960 :
1961 : /* Fix uses of converted REG in debug insns. */
1962 :
1963 : void
1964 446408 : timode_scalar_chain::fix_debug_reg_uses (rtx reg)
1965 : {
1966 446408 : if (!flag_var_tracking)
1967 : return;
1968 :
1969 391925 : df_ref ref, next;
1970 802874 : for (ref = DF_REG_USE_CHAIN (REGNO (reg)); ref; ref = next)
1971 : {
1972 410949 : rtx_insn *insn = DF_REF_INSN (ref);
1973 : /* Make sure the next ref is for a different instruction,
1974 : so that we're not affected by the rescan. */
1975 410949 : next = DF_REF_NEXT_REG (ref);
1976 410949 : while (next && DF_REF_INSN (next) == insn)
1977 0 : next = DF_REF_NEXT_REG (next);
1978 :
1979 410949 : if (DEBUG_INSN_P (insn))
1980 : {
1981 : /* It may be a debug insn with a TImode variable in
1982 : register. */
1983 : bool changed = false;
1984 246 : for (; ref != next; ref = DF_REF_NEXT_REG (ref))
1985 : {
1986 123 : rtx *loc = DF_REF_LOC (ref);
1987 123 : if (REG_P (*loc) && GET_MODE (*loc) == V1TImode)
1988 : {
1989 114 : *loc = gen_rtx_SUBREG (TImode, *loc, 0);
1990 114 : changed = true;
1991 : }
1992 : }
1993 123 : if (changed)
1994 114 : df_insn_rescan (insn);
1995 : }
1996 : }
1997 : }
1998 :
1999 : /* Convert SRC, a *concatditi3 pattern, into a vec_concatv2di instruction.
2000 : Insert this before INSN, and return the result as a V1TImode subreg. */
2001 :
2002 : static rtx
2003 283 : timode_convert_concatdi (rtx src, rtx_insn *insn)
2004 : {
2005 283 : rtx hi, lo;
2006 283 : rtx tmp = gen_reg_rtx (V2DImode);
2007 283 : if (GET_CODE (XEXP (src, 0)) == ASHIFT)
2008 : {
2009 283 : hi = XEXP (XEXP (XEXP (src, 0), 0), 0);
2010 283 : lo = XEXP (XEXP (src, 1), 0);
2011 : }
2012 : else
2013 : {
2014 0 : hi = XEXP (XEXP (XEXP (src, 1), 0), 0);
2015 0 : lo = XEXP (XEXP (src, 0), 0);
2016 : }
2017 283 : emit_insn_before (gen_vec_concatv2di (tmp, lo, hi), insn);
2018 283 : return gen_rtx_SUBREG (V1TImode, tmp, 0);
2019 : }
2020 :
2021 : /* Convert INSN from TImode to V1T1mode. */
2022 :
2023 : void
2024 965442 : timode_scalar_chain::convert_insn (rtx_insn *insn)
2025 : {
2026 965442 : rtx def_set = single_set (insn);
2027 965442 : rtx src = SET_SRC (def_set);
2028 965442 : rtx dst = SET_DEST (def_set);
2029 965442 : rtx tmp;
2030 :
2031 965442 : switch (GET_CODE (dst))
2032 : {
2033 446927 : case REG:
2034 446927 : if (GET_MODE (dst) == TImode)
2035 : {
2036 446333 : if (!HARD_REGISTER_NUM_P (REGNO (dst)))
2037 : {
2038 445822 : PUT_MODE (dst, V1TImode);
2039 445822 : fix_debug_reg_uses (dst);
2040 : }
2041 511 : else if (!GENERAL_REGNO_P (REGNO (dst)))
2042 359 : dst = gen_raw_REG (V1TImode, REGNO (dst));
2043 : }
2044 446927 : if (GET_MODE (dst) == V1TImode)
2045 : {
2046 : /* It might potentially be helpful to convert REG_EQUAL notes,
2047 : but for now we just remove them. */
2048 446767 : rtx note = find_reg_equal_equiv_note (insn);
2049 446767 : if (note)
2050 483 : remove_note (insn, note);
2051 : }
2052 : break;
2053 518515 : case MEM:
2054 518515 : PUT_MODE (dst, V1TImode);
2055 518515 : break;
2056 :
2057 0 : default:
2058 0 : gcc_unreachable ();
2059 : }
2060 :
2061 965442 : switch (GET_CODE (src))
2062 : {
2063 475565 : case REG:
2064 475565 : if (GET_MODE (src) == TImode)
2065 : {
2066 814 : if (GENERAL_REGNO_P (REGNO (src)))
2067 : {
2068 223 : rtx lo = gen_reg_rtx (DImode);
2069 223 : rtx hi = gen_reg_rtx (DImode);
2070 223 : emit_insn_before (gen_rtx_SET (lo, gen_lowpart (DImode, src)),
2071 : insn);
2072 223 : emit_insn_before (gen_rtx_SET (hi, gen_highpart (DImode, src)),
2073 : insn);
2074 223 : src = gen_reg_rtx (V2DImode);
2075 223 : emit_insn_before (gen_vec_concatv2di (src, lo, hi), insn);
2076 223 : src = gen_lowpart (V1TImode, src);
2077 : }
2078 591 : else if (!HARD_REGISTER_NUM_P (REGNO (src)))
2079 : {
2080 586 : PUT_MODE (src, V1TImode);
2081 586 : fix_debug_reg_uses (src);
2082 : }
2083 : else
2084 5 : src = gen_raw_REG (V1TImode, REGNO (src));
2085 : }
2086 475565 : if (GENERAL_REG_P (dst))
2087 : {
2088 152 : rtx tmp = gen_reg_rtx (V2DImode);
2089 152 : src = gen_lowpart (V2DImode, src);
2090 152 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
2091 : /* Extracting hi before lo helps register allocation. */
2092 152 : rtx hi = gen_reg_rtx (DImode);
2093 152 : rtx lo = gen_reg_rtx (DImode);
2094 152 : emit_insn_before (gen_vec_extractv2didi (hi, tmp, const1_rtx), insn);
2095 152 : emit_insn_before (gen_vec_extractv2didi (lo, tmp, const0_rtx), insn);
2096 :
2097 : /* Construct *concatditi3 pattern from lo and hi. */
2098 152 : hi = gen_rtx_ZERO_EXTEND (TImode, hi);
2099 152 : hi = gen_rtx_ASHIFT (TImode, hi, GEN_INT (64));
2100 152 : lo = gen_rtx_ZERO_EXTEND (TImode, lo);
2101 152 : src = gen_rtx_PLUS (TImode, hi, lo);
2102 : }
2103 : break;
2104 :
2105 444652 : case MEM:
2106 444652 : PUT_MODE (src, V1TImode);
2107 444652 : break;
2108 :
2109 31967 : case CONST_WIDE_INT:
2110 31967 : if (NONDEBUG_INSN_P (insn))
2111 : {
2112 : /* Since there are no instructions to store 128-bit constant,
2113 : temporary register usage is required. */
2114 31967 : bool use_move;
2115 31967 : start_sequence ();
2116 31967 : tmp = ix86_convert_const_wide_int_to_broadcast (TImode, src);
2117 31967 : if (tmp)
2118 : {
2119 194 : src = lowpart_subreg (V1TImode, tmp, TImode);
2120 194 : use_move = true;
2121 : }
2122 : else
2123 : {
2124 31773 : src = smode_convert_cst (src, V1TImode);
2125 31773 : src = validize_mem (force_const_mem (V1TImode, src));
2126 31773 : use_move = MEM_P (dst);
2127 : }
2128 31967 : rtx_insn *seq = end_sequence ();
2129 31967 : if (seq)
2130 195 : emit_insn_before (seq, insn);
2131 31967 : if (use_move)
2132 : {
2133 31580 : tmp = gen_reg_rtx (V1TImode);
2134 31580 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
2135 31580 : src = tmp;
2136 : }
2137 : }
2138 : break;
2139 :
2140 12665 : case CONST_INT:
2141 12665 : switch (standard_sse_constant_p (src, TImode))
2142 : {
2143 12437 : case 1:
2144 12437 : src = CONST0_RTX (GET_MODE (dst));
2145 12437 : break;
2146 228 : case 2:
2147 228 : src = CONSTM1_RTX (GET_MODE (dst));
2148 228 : break;
2149 0 : default:
2150 0 : gcc_unreachable ();
2151 : }
2152 12665 : if (MEM_P (dst))
2153 : {
2154 12217 : tmp = gen_reg_rtx (V1TImode);
2155 12217 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
2156 12217 : src = tmp;
2157 : }
2158 : break;
2159 :
2160 41 : case AND:
2161 41 : if (GET_CODE (XEXP (src, 0)) == NOT)
2162 : {
2163 0 : convert_op (&XEXP (XEXP (src, 0), 0), insn);
2164 0 : convert_op (&XEXP (src, 1), insn);
2165 0 : PUT_MODE (XEXP (src, 0), V1TImode);
2166 0 : PUT_MODE (src, V1TImode);
2167 0 : break;
2168 : }
2169 41 : convert_op (&XEXP (src, 0), insn);
2170 41 : convert_op (&XEXP (src, 1), insn);
2171 41 : PUT_MODE (src, V1TImode);
2172 41 : if (MEM_P (dst))
2173 : {
2174 10 : tmp = gen_reg_rtx (V1TImode);
2175 10 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
2176 10 : src = tmp;
2177 : }
2178 : break;
2179 :
2180 379 : case XOR:
2181 379 : case IOR:
2182 379 : if (timode_concatdi_p (src))
2183 : {
2184 283 : src = timode_convert_concatdi (src, insn);
2185 283 : break;
2186 : }
2187 96 : convert_op (&XEXP (src, 0), insn);
2188 96 : convert_op (&XEXP (src, 1), insn);
2189 96 : PUT_MODE (src, V1TImode);
2190 96 : if (MEM_P (dst))
2191 : {
2192 8 : tmp = gen_reg_rtx (V1TImode);
2193 8 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
2194 8 : src = tmp;
2195 : }
2196 : break;
2197 :
2198 3 : case NOT:
2199 3 : src = XEXP (src, 0);
2200 3 : convert_op (&src, insn);
2201 3 : tmp = gen_reg_rtx (V1TImode);
2202 3 : emit_insn_before (gen_move_insn (tmp, CONSTM1_RTX (V1TImode)), insn);
2203 3 : src = gen_rtx_XOR (V1TImode, src, tmp);
2204 3 : if (MEM_P (dst))
2205 : {
2206 0 : tmp = gen_reg_rtx (V1TImode);
2207 0 : emit_insn_before (gen_rtx_SET (tmp, src), insn);
2208 0 : src = tmp;
2209 : }
2210 : break;
2211 :
2212 8 : case COMPARE:
2213 8 : dst = gen_rtx_REG (CCZmode, FLAGS_REG);
2214 8 : src = convert_compare (XEXP (src, 0), XEXP (src, 1), insn);
2215 8 : break;
2216 :
2217 58 : case ASHIFT:
2218 58 : case LSHIFTRT:
2219 58 : case ASHIFTRT:
2220 58 : case ROTATERT:
2221 58 : case ROTATE:
2222 58 : convert_op (&XEXP (src, 0), insn);
2223 58 : PUT_MODE (src, V1TImode);
2224 58 : break;
2225 :
2226 104 : case ZERO_EXTEND:
2227 104 : if (GET_MODE (XEXP (src, 0)) == DImode)
2228 : {
2229 : /* Convert to *vec_concatv2di_0. */
2230 104 : rtx tmp = gen_reg_rtx (V2DImode);
2231 104 : rtx pat = gen_rtx_VEC_CONCAT (V2DImode, XEXP (src, 0), const0_rtx);
2232 104 : emit_insn_before (gen_move_insn (tmp, pat), insn);
2233 104 : src = gen_rtx_SUBREG (vmode, tmp, 0);
2234 : }
2235 : else
2236 0 : gcc_unreachable ();
2237 104 : break;
2238 :
2239 0 : case PLUS:
2240 0 : if (timode_concatdi_p (src))
2241 0 : src = timode_convert_concatdi (src, insn);
2242 : else
2243 0 : gcc_unreachable ();
2244 0 : break;
2245 :
2246 0 : default:
2247 0 : gcc_unreachable ();
2248 : }
2249 :
2250 965442 : SET_SRC (def_set) = src;
2251 965442 : SET_DEST (def_set) = dst;
2252 :
2253 : /* Drop possible dead definitions. */
2254 965442 : PATTERN (insn) = def_set;
2255 :
2256 965442 : INSN_CODE (insn) = -1;
2257 965442 : recog_memoized (insn);
2258 965442 : df_insn_rescan (insn);
2259 965442 : }
2260 :
2261 : /* Generate copies from defs used by the chain but not defined therein.
2262 : Also populates defs_map which is used later by convert_insn. */
2263 :
2264 : void
2265 662781 : scalar_chain::convert_registers ()
2266 : {
2267 662781 : bitmap_iterator bi;
2268 662781 : unsigned id;
2269 689882 : EXECUTE_IF_SET_IN_BITMAP (defs_conv, 0, id, bi)
2270 : {
2271 27101 : rtx chain_reg = gen_reg_rtx (smode);
2272 27101 : defs_map.put (regno_reg_rtx[id], chain_reg);
2273 : }
2274 671486 : EXECUTE_IF_SET_IN_BITMAP (insns_conv, 0, id, bi)
2275 20977 : for (df_ref ref = DF_INSN_UID_DEFS (id); ref; ref = DF_REF_NEXT_LOC (ref))
2276 12272 : if (bitmap_bit_p (defs_conv, DF_REF_REGNO (ref)))
2277 8705 : make_vector_copies (DF_REF_INSN (ref), DF_REF_REAL_REG (ref));
2278 662781 : }
2279 :
2280 : /* Convert whole chain creating required register
2281 : conversions and copies. */
2282 :
2283 : int
2284 662781 : scalar_chain::convert ()
2285 : {
2286 662781 : bitmap_iterator bi;
2287 662781 : unsigned id;
2288 662781 : int converted_insns = 0;
2289 :
2290 662781 : if (!dbg_cnt (stv_conversion))
2291 : return 0;
2292 :
2293 662781 : if (dump_file)
2294 0 : fprintf (dump_file, "Converting chain #%d...\n", chain_id);
2295 :
2296 662781 : convert_registers ();
2297 :
2298 2035532 : EXECUTE_IF_SET_IN_BITMAP (insns, 0, id, bi)
2299 : {
2300 1372751 : rtx_insn *insn = DF_INSN_UID_GET (id)->insn;
2301 1372751 : convert_insn_common (insn);
2302 1372751 : convert_insn (insn);
2303 1372751 : converted_insns++;
2304 : }
2305 :
2306 : return converted_insns;
2307 : }
2308 :
2309 : /* Return the SET expression if INSN doesn't reference hard register.
2310 : Return NULL if INSN uses or defines a hard register, excluding
2311 : pseudo register pushes, hard register uses in a memory address,
2312 : clobbers and flags definitions. */
2313 :
2314 : static rtx
2315 363207253 : pseudo_reg_set (rtx_insn *insn)
2316 : {
2317 363207253 : rtx set = single_set (insn);
2318 363207253 : if (!set)
2319 : return NULL;
2320 :
2321 : /* Check pseudo register push first. */
2322 140353154 : machine_mode mode = TARGET_64BIT ? TImode : DImode;
2323 140353154 : if (REG_P (SET_SRC (set))
2324 39975579 : && !HARD_REGISTER_P (SET_SRC (set))
2325 171601164 : && push_operand (SET_DEST (set), mode))
2326 : return set;
2327 :
2328 140099981 : df_ref ref;
2329 227049675 : FOR_EACH_INSN_DEF (ref, insn)
2330 125446176 : if (HARD_REGISTER_P (DF_REF_REAL_REG (ref))
2331 67552678 : && !DF_REF_FLAGS_IS_SET (ref, DF_REF_MUST_CLOBBER)
2332 177827914 : && DF_REF_REGNO (ref) != FLAGS_REG)
2333 : return NULL;
2334 :
2335 194257589 : FOR_EACH_INSN_USE (ref, insn)
2336 119004733 : if (!DF_REF_REG_MEM_P (ref) && HARD_REGISTER_P (DF_REF_REAL_REG (ref)))
2337 : return NULL;
2338 :
2339 : return set;
2340 : }
2341 :
2342 : /* Return true if the register REG is defined in a single DEF chain.
2343 : If it is defined in more than one DEF chains, we may not be able
2344 : to convert it in all chains. */
2345 :
2346 : static bool
2347 1305812 : single_def_chain_p (rtx reg)
2348 : {
2349 1305812 : df_ref ref = DF_REG_DEF_CHAIN (REGNO (reg));
2350 1305812 : if (!ref)
2351 : return false;
2352 1305781 : return DF_REF_NEXT_REG (ref) == nullptr;
2353 : }
2354 :
2355 : /* Check if comparison INSN may be transformed into vector comparison.
2356 : Currently we transform equality/inequality checks which look like:
2357 : (set (reg:CCZ 17 flags) (compare:CCZ (reg:TI x) (reg:TI y))) */
2358 :
2359 : static bool
2360 13249333 : convertible_comparison_p (rtx_insn *insn, enum machine_mode mode)
2361 : {
2362 14665823 : if (mode != (TARGET_64BIT ? TImode : DImode))
2363 : return false;
2364 :
2365 4854124 : if (!TARGET_SSE4_1)
2366 : return false;
2367 :
2368 165576 : rtx def_set = single_set (insn);
2369 :
2370 165576 : gcc_assert (def_set);
2371 :
2372 165576 : rtx src = SET_SRC (def_set);
2373 165576 : rtx dst = SET_DEST (def_set);
2374 :
2375 165576 : gcc_assert (GET_CODE (src) == COMPARE);
2376 :
2377 165576 : if (!REG_P (dst)
2378 165576 : || REGNO (dst) != FLAGS_REG
2379 331152 : || GET_MODE (dst) != CCZmode)
2380 : return false;
2381 :
2382 115317 : rtx op1 = XEXP (src, 0);
2383 115317 : rtx op2 = XEXP (src, 1);
2384 :
2385 : /* *cmp<dwi>_doubleword. */
2386 115317 : if (general_operand (op1, mode)
2387 115317 : && general_operand (op2, mode))
2388 : return true;
2389 :
2390 : /* *testti_doubleword. */
2391 115254 : if (op2 == const0_rtx
2392 39800 : && GET_CODE (op1) == AND
2393 142 : && REG_P (XEXP (op1, 0)))
2394 : {
2395 142 : rtx op12 = XEXP (op1, 1);
2396 142 : return GET_MODE (XEXP (op1, 0)) == TImode
2397 142 : && (CONST_SCALAR_INT_P (op12)
2398 0 : || ((REG_P (op12) || MEM_P (op12))
2399 0 : && GET_MODE (op12) == TImode));
2400 : }
2401 :
2402 : /* *test<dwi>_not_doubleword. */
2403 115112 : if (op2 == const0_rtx
2404 39658 : && GET_CODE (op1) == AND
2405 0 : && GET_CODE (XEXP (op1, 0)) == NOT)
2406 : {
2407 0 : rtx op11 = XEXP (XEXP (op1, 0), 0);
2408 0 : rtx op12 = XEXP (op1, 1);
2409 0 : return (REG_P (op11) || MEM_P (op11))
2410 0 : && (REG_P (op12) || MEM_P (op12))
2411 0 : && GET_MODE (op11) == mode
2412 0 : && GET_MODE (op12) == mode;
2413 : }
2414 :
2415 : return false;
2416 : }
2417 :
2418 : /* The general version of scalar_to_vector_candidate_p. */
2419 :
2420 : static bool
2421 253285941 : general_scalar_to_vector_candidate_p (rtx_insn *insn, enum machine_mode mode)
2422 : {
2423 253285941 : rtx def_set = pseudo_reg_set (insn);
2424 :
2425 253285941 : if (!def_set)
2426 : return false;
2427 :
2428 51020453 : rtx src = SET_SRC (def_set);
2429 51020453 : rtx dst = SET_DEST (def_set);
2430 :
2431 51020453 : if (GET_CODE (src) == COMPARE)
2432 9103454 : return convertible_comparison_p (insn, mode);
2433 :
2434 : /* We are interested in "mode" only. */
2435 41916999 : if ((GET_MODE (src) != mode
2436 28532258 : && !CONST_INT_P (src))
2437 18731586 : || GET_MODE (dst) != mode)
2438 : return false;
2439 :
2440 15727426 : if (!REG_P (dst) && !MEM_P (dst))
2441 : return false;
2442 :
2443 15465225 : switch (GET_CODE (src))
2444 : {
2445 554911 : case ASHIFT:
2446 554911 : case LSHIFTRT:
2447 554911 : case ASHIFTRT:
2448 554911 : case ROTATE:
2449 554911 : case ROTATERT:
2450 554911 : if (!CONST_INT_P (XEXP (src, 1))
2451 1072069 : || !IN_RANGE (INTVAL (XEXP (src, 1)), 0, GET_MODE_BITSIZE (mode)-1))
2452 : return false;
2453 :
2454 : /* Check for extend highpart case. */
2455 517158 : if (mode != DImode
2456 381419 : || GET_CODE (src) != ASHIFTRT
2457 89617 : || GET_CODE (XEXP (src, 0)) != ASHIFT)
2458 : break;
2459 :
2460 3817759 : src = XEXP (src, 0);
2461 : break;
2462 :
2463 103237 : case SMAX:
2464 103237 : case SMIN:
2465 103237 : case UMAX:
2466 103237 : case UMIN:
2467 103237 : if ((mode == DImode && !TARGET_AVX512VL)
2468 17996 : || (mode == SImode && !TARGET_SSE4_1))
2469 : return false;
2470 : /* Fallthru. */
2471 :
2472 3331420 : case AND:
2473 3331420 : case IOR:
2474 3331420 : case XOR:
2475 3331420 : case PLUS:
2476 3331420 : case MINUS:
2477 3331420 : if (!REG_P (XEXP (src, 1))
2478 : && !MEM_P (XEXP (src, 1))
2479 : && !CONST_INT_P (XEXP (src, 1)))
2480 : return false;
2481 :
2482 3239376 : if (GET_MODE (XEXP (src, 1)) != mode
2483 1871024 : && !CONST_INT_P (XEXP (src, 1)))
2484 : return false;
2485 :
2486 : /* Check for andnot case. */
2487 3239376 : if (GET_CODE (src) != AND
2488 180533 : || GET_CODE (XEXP (src, 0)) != NOT)
2489 : break;
2490 :
2491 3817759 : src = XEXP (src, 0);
2492 : /* FALLTHRU */
2493 :
2494 : case NOT:
2495 : break;
2496 :
2497 32509 : case NEG:
2498 : /* Check for nabs case. */
2499 32509 : if (GET_CODE (XEXP (src, 0)) != ABS)
2500 : break;
2501 :
2502 : src = XEXP (src, 0);
2503 : /* FALLTHRU */
2504 :
2505 3788 : case ABS:
2506 3788 : if ((mode == DImode && !TARGET_AVX512VL)
2507 1400 : || (mode == SImode && !TARGET_SSSE3))
2508 : return false;
2509 : break;
2510 :
2511 : case REG:
2512 : return true;
2513 :
2514 6195729 : case MEM:
2515 6195729 : case CONST_INT:
2516 6195729 : return REG_P (dst);
2517 :
2518 39862 : case VEC_SELECT:
2519 : /* Excluding MEM_P (dst) avoids interfering with vpextr[dq]. */
2520 39862 : return REG_P (dst)
2521 33240 : && REG_P (XEXP (src, 0))
2522 42330 : && GET_MODE (XEXP (src, 0)) == (mode == DImode ? V2DImode
2523 : : V4SImode)
2524 23840 : && GET_CODE (XEXP (src, 1)) == PARALLEL
2525 23840 : && XVECLEN (XEXP (src, 1), 0) == 1
2526 63702 : && CONST_INT_P (XVECEXP (XEXP (src, 1), 0, 0));
2527 :
2528 319080 : case ZERO_EXTEND:
2529 : /* *zero_extendsidi2 becomes *vec_setv2di_0_zero_extendsi_1. */
2530 319080 : return mode == DImode
2531 224322 : && REG_P (dst)
2532 221538 : && GET_MODE (XEXP (src, 0)) == SImode
2533 503155 : && MEM_P (XEXP (src, 0));
2534 :
2535 : default:
2536 : return false;
2537 : }
2538 :
2539 3817759 : if (!REG_P (XEXP (src, 0))
2540 : && !MEM_P (XEXP (src, 0))
2541 : && !CONST_INT_P (XEXP (src, 0)))
2542 : return false;
2543 :
2544 3522487 : if (GET_MODE (XEXP (src, 0)) != mode
2545 0 : && !CONST_INT_P (XEXP (src, 0)))
2546 : return false;
2547 :
2548 : return true;
2549 : }
2550 :
2551 : /* Check for a suitable TImode memory operand. */
2552 :
2553 : static bool
2554 1635 : timode_mem_p (rtx x)
2555 : {
2556 1635 : return MEM_P (x)
2557 1635 : && (TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
2558 0 : || !misaligned_operand (x, TImode));
2559 : }
2560 :
2561 : /* The TImode version of scalar_to_vector_candidate_p. */
2562 :
2563 : static bool
2564 109921312 : timode_scalar_to_vector_candidate_p (rtx_insn *insn)
2565 : {
2566 109921312 : rtx def_set = pseudo_reg_set (insn);
2567 :
2568 : /* We allow two exceptions to the pseudo registers only rule.
2569 : Setting a hard register from a pseudo, and setting a pseudo
2570 : from a hard register. */
2571 109921312 : if (!def_set)
2572 : {
2573 85435736 : def_set = single_set (insn);
2574 85435736 : if (def_set)
2575 : {
2576 18778567 : rtx src = SET_SRC (def_set);
2577 18778567 : rtx dst = SET_DEST (def_set);
2578 18778567 : if (GET_MODE (dst) == TImode
2579 224862 : && REG_P (src) && REG_P (dst))
2580 : {
2581 102597 : if (HARD_REGISTER_P (dst)
2582 53242 : && !HARD_REGISTER_P (src)
2583 155839 : && single_def_chain_p (src))
2584 : return true;
2585 73394 : if (HARD_REGISTER_P (src)
2586 49355 : && !HARD_REGISTER_P (dst)
2587 122749 : && single_def_chain_p (dst))
2588 : return true;
2589 : }
2590 : }
2591 : return false;
2592 : }
2593 :
2594 24485576 : rtx src = SET_SRC (def_set);
2595 24485576 : rtx dst = SET_DEST (def_set);
2596 :
2597 24485576 : if (GET_CODE (src) == COMPARE)
2598 4145879 : return convertible_comparison_p (insn, TImode);
2599 :
2600 20339697 : if (GET_MODE (dst) != TImode
2601 1250564 : || (GET_MODE (src) != TImode
2602 63557 : && !CONST_SCALAR_INT_P (src)))
2603 : return false;
2604 :
2605 1250564 : if (!REG_P (dst) && !MEM_P (dst))
2606 : return false;
2607 :
2608 1249111 : if (MEM_P (dst)
2609 556486 : && misaligned_operand (dst, TImode)
2610 1581103 : && !TARGET_SSE_UNALIGNED_STORE_OPTIMAL)
2611 : return false;
2612 :
2613 1249106 : if (REG_P (dst) && !single_def_chain_p (dst))
2614 : return false;
2615 :
2616 1091444 : switch (GET_CODE (src))
2617 : {
2618 510590 : case REG:
2619 510590 : return single_def_chain_p (src);
2620 :
2621 : case CONST_WIDE_INT:
2622 : return true;
2623 :
2624 15032 : case CONST_INT:
2625 : /* ??? Verify performance impact before enabling CONST_INT for
2626 : __int128 store. */
2627 15032 : return standard_sse_constant_p (src, TImode);
2628 :
2629 467222 : case MEM:
2630 : /* Memory must be aligned or unaligned load is optimal. */
2631 467222 : return (REG_P (dst)
2632 467222 : && (!misaligned_operand (src, TImode)
2633 164205 : || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL));
2634 :
2635 3333 : case AND:
2636 3333 : if (!MEM_P (dst)
2637 3292 : && GET_CODE (XEXP (src, 0)) == NOT
2638 0 : && REG_P (XEXP (XEXP (src, 0), 0))
2639 3333 : && (REG_P (XEXP (src, 1))
2640 0 : || CONST_SCALAR_INT_P (XEXP (src, 1))
2641 0 : || timode_mem_p (XEXP (src, 1))))
2642 : return true;
2643 3333 : return (REG_P (XEXP (src, 0))
2644 46 : || timode_mem_p (XEXP (src, 0)))
2645 3379 : && (REG_P (XEXP (src, 1))
2646 1483 : || CONST_SCALAR_INT_P (XEXP (src, 1))
2647 45 : || timode_mem_p (XEXP (src, 1)));
2648 :
2649 14526 : case IOR:
2650 14526 : case XOR:
2651 14526 : if (timode_concatdi_p (src))
2652 : return true;
2653 2878 : return (REG_P (XEXP (src, 0))
2654 1461 : || timode_mem_p (XEXP (src, 0)))
2655 2895 : && (REG_P (XEXP (src, 1))
2656 313 : || CONST_SCALAR_INT_P (XEXP (src, 1))
2657 67 : || timode_mem_p (XEXP (src, 1)));
2658 :
2659 523 : case NOT:
2660 523 : return REG_P (XEXP (src, 0)) || timode_mem_p (XEXP (src, 0));
2661 :
2662 11795 : case ASHIFT:
2663 11795 : case LSHIFTRT:
2664 11795 : case ASHIFTRT:
2665 11795 : case ROTATERT:
2666 11795 : case ROTATE:
2667 : /* Handle shifts/rotates by integer constants between 0 and 127. */
2668 11795 : return REG_P (XEXP (src, 0))
2669 11759 : && CONST_INT_P (XEXP (src, 1))
2670 23194 : && (INTVAL (XEXP (src, 1)) & ~0x7f) == 0;
2671 :
2672 7309 : case PLUS:
2673 7309 : return timode_concatdi_p (src);
2674 :
2675 4009 : case ZERO_EXTEND:
2676 4009 : return REG_P (XEXP (src, 0))
2677 4009 : && GET_MODE (XEXP (src, 0)) == DImode;
2678 :
2679 : default:
2680 : return false;
2681 : }
2682 : }
2683 :
2684 : /* For a register REGNO, scan instructions for its defs and uses.
2685 : Put REGNO in REGS if a def or use isn't in CANDIDATES. */
2686 :
2687 : static void
2688 1280600 : timode_check_non_convertible_regs (bitmap candidates, bitmap regs,
2689 : unsigned int regno)
2690 : {
2691 : /* Do nothing if REGNO is already in REGS or is a hard reg. */
2692 1280600 : if (bitmap_bit_p (regs, regno)
2693 1280600 : || HARD_REGISTER_NUM_P (regno))
2694 : return;
2695 :
2696 1272404 : for (df_ref def = DF_REG_DEF_CHAIN (regno);
2697 2533921 : def;
2698 1261517 : def = DF_REF_NEXT_REG (def))
2699 : {
2700 1272384 : if (!bitmap_bit_p (candidates, DF_REF_INSN_UID (def)))
2701 : {
2702 10867 : if (dump_file)
2703 0 : fprintf (dump_file,
2704 : "r%d has non convertible def in insn %d\n",
2705 0 : regno, DF_REF_INSN_UID (def));
2706 :
2707 10867 : bitmap_set_bit (regs, regno);
2708 10867 : break;
2709 : }
2710 : }
2711 :
2712 1272404 : for (df_ref ref = DF_REG_USE_CHAIN (regno);
2713 2815982 : ref;
2714 1543578 : ref = DF_REF_NEXT_REG (ref))
2715 : {
2716 : /* Debug instructions are skipped. */
2717 1614257 : if (NONDEBUG_INSN_P (DF_REF_INSN (ref))
2718 1614257 : && !bitmap_bit_p (candidates, DF_REF_INSN_UID (ref)))
2719 : {
2720 70679 : if (dump_file)
2721 0 : fprintf (dump_file,
2722 : "r%d has non convertible use in insn %d\n",
2723 0 : regno, DF_REF_INSN_UID (ref));
2724 :
2725 70679 : bitmap_set_bit (regs, regno);
2726 70679 : break;
2727 : }
2728 : }
2729 : }
2730 :
2731 : /* For a given bitmap of insn UIDs scans all instructions and
2732 : remove insn from CANDIDATES in case it has both convertible
2733 : and not convertible definitions.
2734 :
2735 : All insns in a bitmap are conversion candidates according to
2736 : scalar_to_vector_candidate_p. Currently it implies all insns
2737 : are single_set. */
2738 :
2739 : static void
2740 857485 : timode_remove_non_convertible_regs (bitmap candidates)
2741 : {
2742 857485 : bitmap_iterator bi;
2743 857485 : unsigned id;
2744 857485 : bitmap regs = BITMAP_ALLOC (NULL);
2745 878953 : bool changed;
2746 :
2747 878953 : do {
2748 878953 : changed = false;
2749 2186758 : EXECUTE_IF_SET_IN_BITMAP (candidates, 0, id, bi)
2750 : {
2751 1307805 : rtx_insn *insn = DF_INSN_UID_GET (id)->insn;
2752 1307805 : df_ref ref;
2753 :
2754 2040589 : FOR_EACH_INSN_DEF (ref, insn)
2755 732784 : if (!DF_REF_REG_MEM_P (ref)
2756 732784 : && GET_MODE (DF_REF_REG (ref)) == TImode)
2757 643096 : timode_check_non_convertible_regs (candidates, regs,
2758 : DF_REF_REGNO (ref));
2759 :
2760 3220545 : FOR_EACH_INSN_USE (ref, insn)
2761 1912740 : if (DF_REF_TYPE (ref) == DF_REF_REG_USE
2762 774242 : && GET_MODE (DF_REF_REG (ref)) == TImode
2763 637509 : && !SUBREG_P (DF_REF_REG (ref)))
2764 637504 : timode_check_non_convertible_regs (candidates, regs,
2765 : DF_REF_REGNO (ref));
2766 : }
2767 :
2768 1056389 : EXECUTE_IF_SET_IN_BITMAP (regs, 0, id, bi)
2769 : {
2770 177436 : for (df_ref def = DF_REG_DEF_CHAIN (id);
2771 361259 : def;
2772 183823 : def = DF_REF_NEXT_REG (def))
2773 183823 : if (bitmap_bit_p (candidates, DF_REF_INSN_UID (def)))
2774 : {
2775 56750 : if (dump_file)
2776 0 : fprintf (dump_file, "Removing insn %d from candidates list\n",
2777 0 : DF_REF_INSN_UID (def));
2778 :
2779 56750 : bitmap_clear_bit (candidates, DF_REF_INSN_UID (def));
2780 56750 : changed = true;
2781 : }
2782 :
2783 177436 : for (df_ref ref = DF_REG_USE_CHAIN (id);
2784 526616 : ref;
2785 349180 : ref = DF_REF_NEXT_REG (ref))
2786 349180 : if (bitmap_bit_p (candidates, DF_REF_INSN_UID (ref)))
2787 : {
2788 16780 : if (dump_file)
2789 0 : fprintf (dump_file, "Removing insn %d from candidates list\n",
2790 0 : DF_REF_INSN_UID (ref));
2791 :
2792 16780 : bitmap_clear_bit (candidates, DF_REF_INSN_UID (ref));
2793 16780 : changed = true;
2794 : }
2795 : }
2796 : } while (changed);
2797 :
2798 857485 : BITMAP_FREE (regs);
2799 857485 : }
2800 :
2801 : /* Main STV pass function. Find and convert scalar
2802 : instructions into vector mode when profitable. */
2803 :
2804 : static unsigned int
2805 1841354 : convert_scalars_to_vector (bool timode_p)
2806 : {
2807 1841354 : basic_block bb;
2808 1841354 : int converted_insns = 0;
2809 1841354 : auto_vec<rtx_insn *> control_flow_insns;
2810 :
2811 1841354 : bitmap_obstack_initialize (NULL);
2812 1841354 : const machine_mode cand_mode[3] = { SImode, DImode, TImode };
2813 1841354 : const machine_mode cand_vmode[3] = { V4SImode, V2DImode, V1TImode };
2814 5524062 : bitmap_head candidates[3]; /* { SImode, DImode, TImode } */
2815 7365416 : for (unsigned i = 0; i < 3; ++i)
2816 5524062 : bitmap_initialize (&candidates[i], &bitmap_default_obstack);
2817 :
2818 1841354 : calculate_dominance_info (CDI_DOMINATORS);
2819 1841354 : df_set_flags (DF_DEFER_INSN_RESCAN | DF_RD_PRUNE_DEAD_DEFS);
2820 1841354 : df_chain_add_problem (DF_DU_CHAIN | DF_UD_CHAIN);
2821 1841354 : df_analyze ();
2822 :
2823 : /* Find all instructions we want to convert into vector mode. */
2824 1841354 : if (dump_file)
2825 44 : fprintf (dump_file, "Searching for mode conversion candidates...\n");
2826 :
2827 20506337 : FOR_EACH_BB_FN (bb, cfun)
2828 : {
2829 18664983 : rtx_insn *insn;
2830 257013690 : FOR_BB_INSNS (bb, insn)
2831 238348707 : if (timode_p
2832 238348707 : && timode_scalar_to_vector_candidate_p (insn))
2833 : {
2834 1128691 : if (dump_file)
2835 0 : fprintf (dump_file, " insn %d is marked as a TImode candidate\n",
2836 0 : INSN_UID (insn));
2837 :
2838 1128691 : bitmap_set_bit (&candidates[2], INSN_UID (insn));
2839 : }
2840 237220016 : else if (!timode_p)
2841 : {
2842 : /* Check {SI,DI}mode. */
2843 369596564 : for (unsigned i = 0; i <= 1; ++i)
2844 253285941 : if (general_scalar_to_vector_candidate_p (insn, cand_mode[i]))
2845 : {
2846 12116772 : if (dump_file)
2847 554 : fprintf (dump_file, " insn %d is marked as a %s candidate\n",
2848 277 : INSN_UID (insn), i == 0 ? "SImode" : "DImode");
2849 :
2850 12116772 : bitmap_set_bit (&candidates[i], INSN_UID (insn));
2851 12116772 : break;
2852 : }
2853 : }
2854 : }
2855 :
2856 1841354 : if (timode_p)
2857 857485 : timode_remove_non_convertible_regs (&candidates[2]);
2858 :
2859 5831862 : for (unsigned i = 0; i <= 2; ++i)
2860 4648997 : if (!bitmap_empty_p (&candidates[i]))
2861 : break;
2862 3990508 : else if (i == 2 && dump_file)
2863 23 : fprintf (dump_file, "There are no candidates for optimization.\n");
2864 :
2865 7365416 : for (unsigned i = 0; i <= 2; ++i)
2866 : {
2867 5524062 : auto_bitmap disallowed;
2868 5524062 : bitmap_tree_view (&candidates[i]);
2869 17735889 : while (!bitmap_empty_p (&candidates[i]))
2870 : {
2871 6687765 : unsigned uid = bitmap_first_set_bit (&candidates[i]);
2872 6687765 : scalar_chain *chain;
2873 :
2874 6687765 : if (cand_mode[i] == TImode)
2875 531267 : chain = new timode_scalar_chain;
2876 : else
2877 6156498 : chain = new general_scalar_chain (cand_mode[i], cand_vmode[i]);
2878 :
2879 : /* Find instructions chain we want to convert to vector mode.
2880 : Check all uses and definitions to estimate all required
2881 : conversions. */
2882 6687765 : if (chain->build (&candidates[i], uid, disallowed))
2883 : {
2884 6686200 : if (chain->compute_convert_gain ())
2885 662781 : converted_insns += chain->convert ();
2886 6023419 : else if (dump_file)
2887 136 : fprintf (dump_file, "Chain #%d conversion is not profitable\n",
2888 : chain->chain_id);
2889 : }
2890 :
2891 6687765 : rtx_insn* iter_insn;
2892 6687765 : unsigned int ii;
2893 6694893 : FOR_EACH_VEC_ELT (chain->control_flow_insns, ii, iter_insn)
2894 3578 : control_flow_insns.safe_push (iter_insn);
2895 :
2896 6687765 : delete chain;
2897 : }
2898 5524062 : }
2899 :
2900 1841354 : if (dump_file)
2901 44 : fprintf (dump_file, "Total insns converted: %d\n", converted_insns);
2902 :
2903 7365416 : for (unsigned i = 0; i <= 2; ++i)
2904 5524062 : bitmap_release (&candidates[i]);
2905 1841354 : bitmap_obstack_release (NULL);
2906 1841354 : df_process_deferred_rescans ();
2907 :
2908 : /* Conversion means we may have 128bit register spills/fills
2909 : which require aligned stack. */
2910 1841354 : if (converted_insns)
2911 : {
2912 107497 : if (crtl->stack_alignment_needed < 128)
2913 2497 : crtl->stack_alignment_needed = 128;
2914 107497 : if (crtl->stack_alignment_estimated < 128)
2915 222 : crtl->stack_alignment_estimated = 128;
2916 :
2917 107497 : crtl->stack_realign_needed
2918 107497 : = INCOMING_STACK_BOUNDARY < crtl->stack_alignment_estimated;
2919 107497 : crtl->stack_realign_tried = crtl->stack_realign_needed;
2920 :
2921 107497 : crtl->stack_realign_processed = true;
2922 :
2923 107497 : if (!crtl->drap_reg)
2924 : {
2925 107307 : rtx drap_rtx = targetm.calls.get_drap_rtx ();
2926 :
2927 : /* stack_realign_drap and drap_rtx must match. */
2928 107307 : gcc_assert ((stack_realign_drap != 0) == (drap_rtx != NULL));
2929 :
2930 : /* Do nothing if NULL is returned,
2931 : which means DRAP is not needed. */
2932 107307 : if (drap_rtx != NULL)
2933 : {
2934 0 : crtl->args.internal_arg_pointer = drap_rtx;
2935 :
2936 : /* Call fixup_tail_calls to clean up
2937 : REG_EQUIV note if DRAP is needed. */
2938 0 : fixup_tail_calls ();
2939 : }
2940 : }
2941 :
2942 : /* Fix up DECL_RTL/DECL_INCOMING_RTL of arguments. */
2943 107497 : if (TARGET_64BIT)
2944 68895 : for (tree parm = DECL_ARGUMENTS (current_function_decl);
2945 189046 : parm; parm = DECL_CHAIN (parm))
2946 : {
2947 120151 : if (TYPE_MODE (TREE_TYPE (parm)) != TImode)
2948 104039 : continue;
2949 16112 : if (DECL_RTL_SET_P (parm)
2950 32224 : && GET_MODE (DECL_RTL (parm)) == V1TImode)
2951 : {
2952 621 : rtx r = DECL_RTL (parm);
2953 621 : if (REG_P (r))
2954 621 : SET_DECL_RTL (parm, gen_rtx_SUBREG (TImode, r, 0));
2955 : }
2956 16112 : if (DECL_INCOMING_RTL (parm)
2957 16112 : && GET_MODE (DECL_INCOMING_RTL (parm)) == V1TImode)
2958 : {
2959 0 : rtx r = DECL_INCOMING_RTL (parm);
2960 0 : if (REG_P (r))
2961 0 : DECL_INCOMING_RTL (parm) = gen_rtx_SUBREG (TImode, r, 0);
2962 : }
2963 : }
2964 :
2965 1842482 : if (!control_flow_insns.is_empty ())
2966 : {
2967 1128 : free_dominance_info (CDI_DOMINATORS);
2968 :
2969 1128 : unsigned int i;
2970 1128 : rtx_insn* insn;
2971 5834 : FOR_EACH_VEC_ELT (control_flow_insns, i, insn)
2972 3578 : if (control_flow_insn_p (insn))
2973 : {
2974 : /* Split the block after insn. There will be a fallthru
2975 : edge, which is OK so we keep it. We have to create
2976 : the exception edges ourselves. */
2977 3578 : bb = BLOCK_FOR_INSN (insn);
2978 3578 : split_block (bb, insn);
2979 3578 : rtl_make_eh_edge (NULL, bb, BB_END (bb));
2980 : }
2981 : }
2982 : }
2983 :
2984 1841354 : return 0;
2985 1841354 : }
2986 :
2987 : static unsigned int
2988 76398 : rest_of_handle_insert_vzeroupper (void)
2989 : {
2990 : /* vzeroupper instructions are inserted immediately after reload and
2991 : postreload_cse to clean up after it a little bit to account for possible
2992 : spills from 256bit or 512bit registers. The pass reuses mode switching
2993 : infrastructure by re-running mode insertion pass, so disable entities
2994 : that have already been processed. */
2995 534786 : for (int i = 0; i < MAX_386_ENTITIES; i++)
2996 458388 : ix86_optimize_mode_switching[i] = 0;
2997 :
2998 76398 : ix86_optimize_mode_switching[AVX_U128] = 1;
2999 :
3000 : /* Call optimize_mode_switching. */
3001 76398 : g->get_passes ()->execute_pass_mode_switching ();
3002 :
3003 : /* LRA removes all REG_DEAD/REG_UNUSED notes and normally they
3004 : reappear in the IL only at the start of pass_rtl_dse2, which does
3005 : df_note_add_problem (); df_analyze ();
3006 : The vzeroupper is scheduled after postreload_cse pass and mode
3007 : switching computes the notes as well, the problem is that e.g.
3008 : pass_gcse2 doesn't maintain the notes, see PR113059 and
3009 : PR112760. Remove the notes now to restore status quo ante
3010 : until we figure out how to maintain the notes or what else
3011 : to do. */
3012 76398 : basic_block bb;
3013 76398 : rtx_insn *insn;
3014 409249 : FOR_EACH_BB_FN (bb, cfun)
3015 4285720 : FOR_BB_INSNS (bb, insn)
3016 3952869 : if (NONDEBUG_INSN_P (insn))
3017 : {
3018 2114308 : rtx *pnote = ®_NOTES (insn);
3019 3923018 : while (*pnote != 0)
3020 : {
3021 1808710 : if (REG_NOTE_KIND (*pnote) == REG_DEAD
3022 822439 : || REG_NOTE_KIND (*pnote) == REG_UNUSED)
3023 1300022 : *pnote = XEXP (*pnote, 1);
3024 : else
3025 508688 : pnote = &XEXP (*pnote, 1);
3026 : }
3027 : }
3028 :
3029 76398 : df_remove_problem (df_note);
3030 76398 : df_analyze ();
3031 76398 : return 0;
3032 : }
3033 :
3034 : namespace {
3035 :
3036 : const pass_data pass_data_insert_vzeroupper =
3037 : {
3038 : RTL_PASS, /* type */
3039 : "vzeroupper", /* name */
3040 : OPTGROUP_NONE, /* optinfo_flags */
3041 : TV_MACH_DEP, /* tv_id */
3042 : 0, /* properties_required */
3043 : 0, /* properties_provided */
3044 : 0, /* properties_destroyed */
3045 : 0, /* todo_flags_start */
3046 : TODO_df_finish, /* todo_flags_finish */
3047 : };
3048 :
3049 : class pass_insert_vzeroupper : public rtl_opt_pass
3050 : {
3051 : public:
3052 295807 : pass_insert_vzeroupper(gcc::context *ctxt)
3053 591614 : : rtl_opt_pass(pass_data_insert_vzeroupper, ctxt)
3054 : {}
3055 :
3056 : /* opt_pass methods: */
3057 1519489 : bool gate (function *) final override
3058 : {
3059 1519489 : return TARGET_AVX && TARGET_VZEROUPPER;
3060 : }
3061 :
3062 76398 : unsigned int execute (function *) final override
3063 : {
3064 76398 : return rest_of_handle_insert_vzeroupper ();
3065 : }
3066 :
3067 : }; // class pass_insert_vzeroupper
3068 :
3069 : const pass_data pass_data_stv =
3070 : {
3071 : RTL_PASS, /* type */
3072 : "stv", /* name */
3073 : OPTGROUP_NONE, /* optinfo_flags */
3074 : TV_MACH_DEP, /* tv_id */
3075 : 0, /* properties_required */
3076 : 0, /* properties_provided */
3077 : 0, /* properties_destroyed */
3078 : 0, /* todo_flags_start */
3079 : TODO_df_finish, /* todo_flags_finish */
3080 : };
3081 :
3082 : class pass_stv : public rtl_opt_pass
3083 : {
3084 : public:
3085 591614 : pass_stv (gcc::context *ctxt)
3086 : : rtl_opt_pass (pass_data_stv, ctxt),
3087 1183228 : timode_p (false)
3088 : {}
3089 :
3090 : /* opt_pass methods: */
3091 3038978 : bool gate (function *) final override
3092 : {
3093 1519489 : return ((!timode_p || TARGET_64BIT)
3094 4431762 : && TARGET_STV && TARGET_SSE2 && optimize > 1);
3095 : }
3096 :
3097 1841354 : unsigned int execute (function *) final override
3098 : {
3099 1841354 : return convert_scalars_to_vector (timode_p);
3100 : }
3101 :
3102 295807 : opt_pass *clone () final override
3103 : {
3104 295807 : return new pass_stv (m_ctxt);
3105 : }
3106 :
3107 591614 : void set_pass_param (unsigned int n, bool param) final override
3108 : {
3109 591614 : gcc_assert (n == 0);
3110 591614 : timode_p = param;
3111 591614 : }
3112 :
3113 : private:
3114 : bool timode_p;
3115 : }; // class pass_stv
3116 :
3117 : } // anon namespace
3118 :
3119 : rtl_opt_pass *
3120 295807 : make_pass_insert_vzeroupper (gcc::context *ctxt)
3121 : {
3122 295807 : return new pass_insert_vzeroupper (ctxt);
3123 : }
3124 :
3125 : rtl_opt_pass *
3126 295807 : make_pass_stv (gcc::context *ctxt)
3127 : {
3128 295807 : return new pass_stv (ctxt);
3129 : }
3130 :
3131 : /* Inserting ENDBR and pseudo patchable-area instructions. */
3132 :
3133 : static void
3134 209221 : rest_of_insert_endbr_and_patchable_area (bool need_endbr,
3135 : unsigned int patchable_area_size)
3136 : {
3137 209221 : rtx endbr;
3138 209221 : rtx_insn *insn;
3139 209221 : rtx_insn *endbr_insn = NULL;
3140 209221 : basic_block bb;
3141 :
3142 209221 : if (need_endbr)
3143 : {
3144 : /* Currently emit EB if it's a tracking function, i.e. 'nocf_check'
3145 : is absent among function attributes. Later an optimization will
3146 : be introduced to make analysis if an address of a static function
3147 : is taken. A static function whose address is not taken will get
3148 : a nocf_check attribute. This will allow to reduce the number of
3149 : EB. */
3150 209176 : if (!lookup_attribute ("nocf_check",
3151 209176 : TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
3152 209158 : && (!flag_manual_endbr
3153 8 : || lookup_attribute ("cf_check",
3154 8 : DECL_ATTRIBUTES (cfun->decl)))
3155 418333 : && (!cgraph_node::get (cfun->decl)->only_called_directly_p ()
3156 30418 : || ix86_cmodel == CM_LARGE
3157 30417 : || ix86_cmodel == CM_LARGE_PIC
3158 30416 : || flag_force_indirect_call
3159 30416 : || (TARGET_DLLIMPORT_DECL_ATTRIBUTES
3160 : && DECL_DLLIMPORT_P (cfun->decl))))
3161 : {
3162 178742 : if (crtl->profile && flag_fentry)
3163 : {
3164 : /* Queue ENDBR insertion to x86_function_profiler.
3165 : NB: Any patchable-area insn will be inserted after
3166 : ENDBR. */
3167 6 : cfun->machine->insn_queued_at_entrance = TYPE_ENDBR;
3168 : }
3169 : else
3170 : {
3171 178736 : endbr = gen_nop_endbr ();
3172 178736 : bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
3173 178736 : rtx_insn *insn = BB_HEAD (bb);
3174 178736 : endbr_insn = emit_insn_before (endbr, insn);
3175 : }
3176 : }
3177 : }
3178 :
3179 209221 : if (patchable_area_size)
3180 : {
3181 51 : if (crtl->profile && flag_fentry)
3182 : {
3183 : /* Queue patchable-area insertion to x86_function_profiler.
3184 : NB: If there is a queued ENDBR, x86_function_profiler
3185 : will also handle patchable-area. */
3186 2 : if (!cfun->machine->insn_queued_at_entrance)
3187 1 : cfun->machine->insn_queued_at_entrance = TYPE_PATCHABLE_AREA;
3188 : }
3189 : else
3190 : {
3191 49 : rtx patchable_area
3192 49 : = gen_patchable_area (GEN_INT (patchable_area_size),
3193 49 : GEN_INT (crtl->patch_area_entry == 0));
3194 49 : if (endbr_insn)
3195 3 : emit_insn_after (patchable_area, endbr_insn);
3196 : else
3197 : {
3198 46 : bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
3199 46 : insn = BB_HEAD (bb);
3200 46 : emit_insn_before (patchable_area, insn);
3201 : }
3202 : }
3203 : }
3204 :
3205 209221 : if (!need_endbr)
3206 : return;
3207 :
3208 209176 : bb = 0;
3209 4327344 : FOR_EACH_BB_FN (bb, cfun)
3210 : {
3211 81026060 : for (insn = BB_HEAD (bb); insn != NEXT_INSN (BB_END (bb));
3212 76907892 : insn = NEXT_INSN (insn))
3213 : {
3214 76907892 : if (CALL_P (insn))
3215 : {
3216 1565343 : need_endbr = find_reg_note (insn, REG_SETJMP, NULL) != NULL;
3217 1565343 : if (!need_endbr && !SIBLING_CALL_P (insn))
3218 : {
3219 1511910 : rtx call = get_call_rtx_from (insn);
3220 1511910 : rtx fnaddr = XEXP (call, 0);
3221 1511910 : tree fndecl = NULL_TREE;
3222 :
3223 : /* Also generate ENDBRANCH for non-tail call which
3224 : may return via indirect branch. */
3225 1511910 : if (SYMBOL_REF_P (XEXP (fnaddr, 0)))
3226 1450573 : fndecl = SYMBOL_REF_DECL (XEXP (fnaddr, 0));
3227 1450573 : if (fndecl == NULL_TREE)
3228 61705 : fndecl = MEM_EXPR (fnaddr);
3229 61705 : if (fndecl
3230 1509447 : && TREE_CODE (TREE_TYPE (fndecl)) != FUNCTION_TYPE
3231 636920 : && TREE_CODE (TREE_TYPE (fndecl)) != METHOD_TYPE)
3232 : fndecl = NULL_TREE;
3233 1511910 : if (fndecl && TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
3234 : {
3235 1466903 : tree fntype = TREE_TYPE (fndecl);
3236 1466903 : if (lookup_attribute ("indirect_return",
3237 1466903 : TYPE_ATTRIBUTES (fntype)))
3238 : need_endbr = true;
3239 : }
3240 : }
3241 1565331 : if (!need_endbr)
3242 1565323 : continue;
3243 : /* Generate ENDBRANCH after CALL, which can return more than
3244 : twice, setjmp-like functions. */
3245 :
3246 20 : endbr = gen_nop_endbr ();
3247 20 : emit_insn_after_setloc (endbr, insn, INSN_LOCATION (insn));
3248 20 : continue;
3249 20 : }
3250 :
3251 75342549 : if (JUMP_P (insn) && flag_cet_switch)
3252 : {
3253 15 : rtx target = JUMP_LABEL (insn);
3254 15 : if (target == NULL_RTX || ANY_RETURN_P (target))
3255 6 : continue;
3256 :
3257 : /* Check the jump is a switch table. */
3258 9 : rtx_insn *label = as_a<rtx_insn *> (target);
3259 9 : rtx_insn *table = next_insn (label);
3260 9 : if (table == NULL_RTX || !JUMP_TABLE_DATA_P (table))
3261 5 : continue;
3262 :
3263 : /* For the indirect jump find out all places it jumps and insert
3264 : ENDBRANCH there. It should be done under a special flag to
3265 : control ENDBRANCH generation for switch stmts. */
3266 4 : edge_iterator ei;
3267 4 : edge e;
3268 4 : basic_block dest_blk;
3269 :
3270 42 : FOR_EACH_EDGE (e, ei, bb->succs)
3271 : {
3272 38 : rtx_insn *insn;
3273 :
3274 38 : dest_blk = e->dest;
3275 38 : insn = BB_HEAD (dest_blk);
3276 38 : gcc_assert (LABEL_P (insn));
3277 :
3278 : /* Reuse existing ENDBR. */
3279 38 : rtx_insn *next = next_nonnote_nondebug_insn (insn);
3280 38 : if (next && NONJUMP_INSN_P (next)
3281 30 : && GET_CODE (PATTERN (next)) == UNSPEC_VOLATILE
3282 46 : && XINT (PATTERN (next), 1) == UNSPECV_NOP_ENDBR)
3283 8 : continue;
3284 :
3285 30 : endbr = gen_nop_endbr ();
3286 30 : emit_insn_after (endbr, insn);
3287 : }
3288 4 : continue;
3289 4 : }
3290 :
3291 75342534 : if (LABEL_P (insn) && LABEL_PRESERVE_P (insn))
3292 : {
3293 163488 : endbr = gen_nop_endbr ();
3294 163488 : emit_insn_after (endbr, insn);
3295 163488 : continue;
3296 : }
3297 : }
3298 : }
3299 :
3300 : return;
3301 : }
3302 :
3303 : namespace {
3304 :
3305 : const pass_data pass_data_insert_endbr_and_patchable_area =
3306 : {
3307 : RTL_PASS, /* type. */
3308 : "endbr_and_patchable_area", /* name. */
3309 : OPTGROUP_NONE, /* optinfo_flags. */
3310 : TV_MACH_DEP, /* tv_id. */
3311 : 0, /* properties_required. */
3312 : 0, /* properties_provided. */
3313 : 0, /* properties_destroyed. */
3314 : 0, /* todo_flags_start. */
3315 : 0, /* todo_flags_finish. */
3316 : };
3317 :
3318 : class pass_insert_endbr_and_patchable_area : public rtl_opt_pass
3319 : {
3320 : public:
3321 295807 : pass_insert_endbr_and_patchable_area (gcc::context *ctxt)
3322 591614 : : rtl_opt_pass (pass_data_insert_endbr_and_patchable_area, ctxt)
3323 : {}
3324 :
3325 : /* opt_pass methods: */
3326 1519489 : bool gate (function *) final override
3327 : {
3328 1519489 : need_endbr = (flag_cf_protection & CF_BRANCH) != 0;
3329 1519489 : patchable_area_size = crtl->patch_area_size - crtl->patch_area_entry;
3330 1519489 : return need_endbr || patchable_area_size;
3331 : }
3332 :
3333 209221 : unsigned int execute (function *) final override
3334 : {
3335 209221 : timevar_push (TV_MACH_DEP);
3336 209221 : rest_of_insert_endbr_and_patchable_area (need_endbr,
3337 : patchable_area_size);
3338 209221 : timevar_pop (TV_MACH_DEP);
3339 209221 : return 0;
3340 : }
3341 :
3342 : private:
3343 : bool need_endbr;
3344 : unsigned int patchable_area_size;
3345 : }; // class pass_insert_endbr_and_patchable_area
3346 :
3347 : } // anon namespace
3348 :
3349 : rtl_opt_pass *
3350 295807 : make_pass_insert_endbr_and_patchable_area (gcc::context *ctxt)
3351 : {
3352 295807 : return new pass_insert_endbr_and_patchable_area (ctxt);
3353 : }
3354 :
3355 : bool
3356 6155988 : ix86_rpad_gate ()
3357 : {
3358 6155988 : return (TARGET_AVX
3359 393173 : && TARGET_SSE_PARTIAL_REG_DEPENDENCY
3360 295254 : && TARGET_SSE_MATH
3361 294960 : && optimize
3362 6445746 : && optimize_function_for_speed_p (cfun));
3363 : }
3364 :
3365 : enum x86_cse_kind
3366 : {
3367 : X86_CSE_CONST0_VECTOR,
3368 : X86_CSE_CONSTM1_VECTOR,
3369 : X86_CSE_CONST_VECTOR,
3370 : X86_CSE_VEC_DUP,
3371 : X86_CSE_TLS_GD,
3372 : X86_CSE_TLS_LD_BASE,
3373 : X86_CSE_TLSDESC
3374 : };
3375 :
3376 156399 : struct redundant_pattern
3377 : {
3378 : /* Bitmap of basic blocks with broadcast instructions. */
3379 : auto_bitmap bbs;
3380 : /* Bitmap of broadcast instructions. */
3381 : auto_bitmap insns;
3382 : /* The broadcast inner scalar. */
3383 : rtx val;
3384 : /* The actual redundant source value for UNSPEC_TLSDESC. */
3385 : rtx tlsdesc_val;
3386 : /* The inner scalar mode. */
3387 : machine_mode mode;
3388 : /* The destination mode which can be changed to the integer mode of
3389 : the same time. */
3390 : machine_mode dest_mode;
3391 : /* The instruction which sets the inner scalar. Nullptr if the inner
3392 : scalar is applied to the whole function, instead of within the same
3393 : block. */
3394 : rtx_insn *def_insn;
3395 : /* The widest broadcast source. */
3396 : rtx broadcast_source;
3397 : /* The widest broadcast register. */
3398 : rtx broadcast_reg;
3399 : /* The basic block of the broadcast instruction. */
3400 : basic_block bb;
3401 : /* The number of broadcast instructions with the same inner scalar. */
3402 : unsigned HOST_WIDE_INT count;
3403 : /* The threshold of broadcast instructions with the same inner
3404 : scalar. */
3405 : unsigned int threshold;
3406 : /* The widest broadcast size in bytes. */
3407 : unsigned int size;
3408 : /* Load kind. */
3409 : x86_cse_kind kind;
3410 : };
3411 :
3412 : /* Generate a vector set, DEST = SRC, at entry of the nearest dominator
3413 : for basic block map BBS, which is in the fake loop that contains the
3414 : whole function, so that there is only a single vector set in the
3415 : whole function. If not nullptr, LOAD is a pointer to the load. */
3416 :
3417 : static void
3418 45540 : ix86_place_single_vector_set (rtx dest, rtx src, bitmap bbs,
3419 : redundant_pattern *load = nullptr)
3420 : {
3421 45540 : basic_block bb = nearest_common_dominator_for_set (CDI_DOMINATORS, bbs);
3422 : /* For X86_CSE_VEC_DUP and X86_CSE_CONST_VECTOR, don't place the vector
3423 : set outside of the loop to avoid extra spills. */
3424 45540 : if (!load
3425 44517 : || (load->kind != X86_CSE_VEC_DUP
3426 44517 : && load->kind != X86_CSE_CONST_VECTOR))
3427 : {
3428 25086 : while (bb->loop_father->latch
3429 25086 : != EXIT_BLOCK_PTR_FOR_FN (cfun))
3430 1459 : bb = get_immediate_dominator (CDI_DOMINATORS,
3431 : bb->loop_father->header);
3432 : }
3433 :
3434 45540 : if (CONST_INT_P (src))
3435 11280 : dest = gen_rtx_SUBREG (load->dest_mode, dest, 0);
3436 34260 : else if (CONST_VECTOR_P (src))
3437 : {
3438 : /* The only possible CONST_VECTORs of SRC are CONST0_RTX and
3439 : CONSTM1_RTX. Otherwise,
3440 :
3441 : rtx set = gen_rtx_SET (dest, src);
3442 :
3443 : won't be a valid instruction. CONST0_RTX always works. It
3444 : can comes from:
3445 :
3446 : 1. remove_partial_avx_dependency with LOAD == NULL.
3447 : 2. X86_CSE_VEC_DUP with
3448 :
3449 : (insn 48 58 16 3 (set (reg:V4HI 123)
3450 : (const_vector:V4HI [
3451 : (const_int 0 [0]) repeated x4
3452 : ])) 2065 {*movv4hi_internal} (nil))
3453 :
3454 : 3. X86_CSE_CONST0_VECTOR.
3455 : */
3456 23627 : machine_mode mode = GET_MODE (dest);
3457 23627 : if (!(src == CONST0_RTX (mode)
3458 1673 : || (src == CONSTM1_RTX (mode)
3459 1673 : && load->kind == X86_CSE_CONSTM1_VECTOR)))
3460 0 : gcc_unreachable ();
3461 : }
3462 45540 : rtx set = gen_rtx_SET (dest, src);
3463 :
3464 45540 : rtx_insn *insn = BB_HEAD (bb);
3465 185370 : while (insn && !NONDEBUG_INSN_P (insn))
3466 : {
3467 139834 : if (insn == BB_END (bb))
3468 : {
3469 : insn = NULL;
3470 : break;
3471 : }
3472 139830 : insn = NEXT_INSN (insn);
3473 : }
3474 :
3475 45540 : rtx_insn *set_insn;
3476 45540 : if (insn == BB_HEAD (bb))
3477 : {
3478 0 : set_insn = emit_insn_before (set, insn);
3479 0 : if (dump_file)
3480 : {
3481 0 : fprintf (dump_file, "\nPlace:\n\n");
3482 0 : print_rtl_single (dump_file, set_insn);
3483 0 : fprintf (dump_file, "\nbefore:\n\n");
3484 0 : print_rtl_single (dump_file, insn);
3485 0 : fprintf (dump_file, "\n");
3486 : }
3487 : }
3488 : else
3489 : {
3490 45540 : rtx_insn *after = insn ? PREV_INSN (insn) : BB_END (bb);
3491 45540 : set_insn = emit_insn_after (set, after);
3492 45540 : if (dump_file)
3493 : {
3494 0 : fprintf (dump_file, "\nPlace:\n\n");
3495 0 : print_rtl_single (dump_file, set_insn);
3496 0 : fprintf (dump_file, "\nafter:\n\n");
3497 0 : print_rtl_single (dump_file, after);
3498 0 : fprintf (dump_file, "\n");
3499 : }
3500 : }
3501 :
3502 45540 : if (load && load->kind == X86_CSE_VEC_DUP)
3503 : {
3504 : /* Get the source from LOAD as (reg:SI 99) in
3505 :
3506 : (vec_duplicate:V4SI (reg:SI 99))
3507 :
3508 : */
3509 10633 : rtx inner_scalar = load->val;
3510 : /* Set the source in (vec_duplicate:V4SI (reg:SI 99)). */
3511 10633 : rtx reg = XEXP (src, 0);
3512 10633 : machine_mode reg_mode = GET_MODE (reg);
3513 10633 : if (reg_mode != GET_MODE (inner_scalar))
3514 : {
3515 10483 : if (REG_P (inner_scalar) || MEM_P (inner_scalar))
3516 0 : inner_scalar = gen_rtx_SUBREG (reg_mode, inner_scalar, 0);
3517 10483 : else if (!SCALAR_INT_MODE_P (reg_mode))
3518 : {
3519 : /* For non-int load with integer constant, generate
3520 :
3521 : (set (subreg:SI (reg/v:SF 105 [ f ]) 0)
3522 : (const_int 1313486336 [0x4e4a3600]))
3523 :
3524 : */
3525 1 : gcc_assert (CONST_INT_P (inner_scalar));
3526 1 : unsigned int bits = GET_MODE_BITSIZE (reg_mode);
3527 1 : machine_mode mode = int_mode_for_size (bits, 0).require ();
3528 1 : reg = gen_rtx_SUBREG (mode, reg, 0);
3529 : }
3530 : }
3531 10633 : rtx set = gen_rtx_SET (reg, inner_scalar);
3532 10633 : insn = emit_insn_before (set, set_insn);
3533 10633 : if (dump_file)
3534 : {
3535 0 : fprintf (dump_file, "\nAdd:\n\n");
3536 0 : print_rtl_single (dump_file, insn);
3537 0 : fprintf (dump_file, "\nbefore:\n\n");
3538 0 : print_rtl_single (dump_file, set_insn);
3539 0 : fprintf (dump_file, "\n");
3540 : }
3541 : }
3542 45540 : }
3543 :
3544 : /* At entry of the nearest common dominator for basic blocks with
3545 : conversions/rcp/sqrt/rsqrt/round, generate a single
3546 : vxorps %xmmN, %xmmN, %xmmN
3547 : for all
3548 : vcvtss2sd op, %xmmN, %xmmX
3549 : vcvtsd2ss op, %xmmN, %xmmX
3550 : vcvtsi2ss op, %xmmN, %xmmX
3551 : vcvtsi2sd op, %xmmN, %xmmX
3552 :
3553 : NB: We want to generate only a single vxorps to cover the whole
3554 : function. The LCM algorithm isn't appropriate here since it may
3555 : place a vxorps inside the loop. */
3556 :
3557 : static unsigned int
3558 34050 : remove_partial_avx_dependency (void)
3559 : {
3560 34050 : timevar_push (TV_MACH_DEP);
3561 :
3562 34050 : bitmap_obstack_initialize (NULL);
3563 34050 : bitmap convert_bbs = BITMAP_ALLOC (NULL);
3564 :
3565 34050 : basic_block bb;
3566 34050 : rtx_insn *insn, *set_insn;
3567 34050 : rtx set;
3568 34050 : rtx v4sf_const0 = NULL_RTX;
3569 :
3570 34050 : auto_vec<rtx_insn *> control_flow_insns;
3571 :
3572 : /* We create invalid RTL initially so defer rescans. */
3573 34050 : df_set_flags (DF_DEFER_INSN_RESCAN);
3574 :
3575 312297 : FOR_EACH_BB_FN (bb, cfun)
3576 : {
3577 3497580 : FOR_BB_INSNS (bb, insn)
3578 : {
3579 3219333 : if (!NONDEBUG_INSN_P (insn))
3580 1430399 : continue;
3581 :
3582 1788934 : set = single_set (insn);
3583 1788934 : if (!set)
3584 71694 : continue;
3585 :
3586 1717240 : if (get_attr_avx_partial_xmm_update (insn)
3587 : != AVX_PARTIAL_XMM_UPDATE_TRUE)
3588 1714056 : continue;
3589 :
3590 : /* Convert PARTIAL_XMM_UPDATE_TRUE insns, DF -> SF, SF -> DF,
3591 : SI -> SF, SI -> DF, DI -> SF, DI -> DF, sqrt, rsqrt, rcp,
3592 : round, to vec_dup and vec_merge with subreg. */
3593 3184 : rtx src = SET_SRC (set);
3594 3184 : rtx dest = SET_DEST (set);
3595 3184 : machine_mode dest_mode = GET_MODE (dest);
3596 3184 : bool convert_p = false;
3597 3184 : switch (GET_CODE (src))
3598 : {
3599 3119 : case FLOAT:
3600 3119 : case FLOAT_EXTEND:
3601 3119 : case FLOAT_TRUNCATE:
3602 3119 : case UNSIGNED_FLOAT:
3603 3119 : convert_p = true;
3604 3119 : break;
3605 : default:
3606 : break;
3607 : }
3608 :
3609 : /* Only handle conversion here. */
3610 3119 : machine_mode src_mode
3611 3119 : = convert_p ? GET_MODE (XEXP (src, 0)) : VOIDmode;
3612 3119 : switch (src_mode)
3613 : {
3614 153 : case E_SFmode:
3615 153 : case E_DFmode:
3616 153 : if (TARGET_USE_VECTOR_FP_CONVERTS
3617 147 : || !TARGET_SSE_PARTIAL_REG_FP_CONVERTS_DEPENDENCY)
3618 8 : continue;
3619 : break;
3620 2966 : case E_SImode:
3621 2966 : case E_DImode:
3622 2966 : if (TARGET_USE_VECTOR_CONVERTS
3623 2954 : || !TARGET_SSE_PARTIAL_REG_CONVERTS_DEPENDENCY)
3624 14 : continue;
3625 : break;
3626 65 : case E_VOIDmode:
3627 65 : gcc_assert (!convert_p);
3628 : break;
3629 0 : default:
3630 0 : gcc_unreachable ();
3631 : }
3632 :
3633 3162 : if (!v4sf_const0)
3634 1023 : v4sf_const0 = gen_reg_rtx (V4SFmode);
3635 :
3636 3162 : rtx zero;
3637 3162 : machine_mode dest_vecmode;
3638 3162 : switch (dest_mode)
3639 : {
3640 50 : case E_HFmode:
3641 50 : dest_vecmode = V8HFmode;
3642 50 : zero = gen_rtx_SUBREG (V8HFmode, v4sf_const0, 0);
3643 50 : break;
3644 : case E_SFmode:
3645 : dest_vecmode = V4SFmode;
3646 : zero = v4sf_const0;
3647 : break;
3648 1184 : case E_DFmode:
3649 1184 : dest_vecmode = V2DFmode;
3650 1184 : zero = gen_rtx_SUBREG (V2DFmode, v4sf_const0, 0);
3651 1184 : break;
3652 0 : default:
3653 0 : gcc_unreachable ();
3654 : }
3655 :
3656 : /* Change source to vector mode. */
3657 3162 : src = gen_rtx_VEC_DUPLICATE (dest_vecmode, src);
3658 3162 : src = gen_rtx_VEC_MERGE (dest_vecmode, src, zero,
3659 : GEN_INT (HOST_WIDE_INT_1U));
3660 : /* Change destination to vector mode. */
3661 3162 : rtx vec = gen_reg_rtx (dest_vecmode);
3662 : /* Generate an XMM vector SET. */
3663 3162 : set = gen_rtx_SET (vec, src);
3664 3162 : set_insn = emit_insn_before (set, insn);
3665 :
3666 3162 : if (cfun->can_throw_non_call_exceptions)
3667 : {
3668 : /* Handle REG_EH_REGION note. */
3669 0 : rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
3670 0 : if (note)
3671 : {
3672 0 : control_flow_insns.safe_push (set_insn);
3673 0 : add_reg_note (set_insn, REG_EH_REGION, XEXP (note, 0));
3674 : }
3675 : }
3676 :
3677 3162 : src = gen_rtx_SUBREG (dest_mode, vec, 0);
3678 3162 : set = gen_rtx_SET (dest, src);
3679 :
3680 : /* Drop possible dead definitions. */
3681 3162 : PATTERN (insn) = set;
3682 :
3683 3162 : INSN_CODE (insn) = -1;
3684 3162 : recog_memoized (insn);
3685 3162 : df_insn_rescan (insn);
3686 3162 : bitmap_set_bit (convert_bbs, bb->index);
3687 : }
3688 : }
3689 :
3690 34050 : if (v4sf_const0)
3691 : {
3692 : /* (Re-)discover loops so that bb->loop_father can be used in the
3693 : analysis below. */
3694 1023 : calculate_dominance_info (CDI_DOMINATORS);
3695 1023 : loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
3696 :
3697 1023 : ix86_place_single_vector_set (v4sf_const0,
3698 : CONST0_RTX (V4SFmode),
3699 : convert_bbs);
3700 :
3701 1023 : loop_optimizer_finalize ();
3702 :
3703 34050 : if (!control_flow_insns.is_empty ())
3704 : {
3705 0 : free_dominance_info (CDI_DOMINATORS);
3706 :
3707 0 : unsigned int i;
3708 0 : FOR_EACH_VEC_ELT (control_flow_insns, i, insn)
3709 0 : if (control_flow_insn_p (insn))
3710 : {
3711 : /* Split the block after insn. There will be a fallthru
3712 : edge, which is OK so we keep it. We have to create
3713 : the exception edges ourselves. */
3714 0 : bb = BLOCK_FOR_INSN (insn);
3715 0 : split_block (bb, insn);
3716 0 : rtl_make_eh_edge (NULL, bb, BB_END (bb));
3717 : }
3718 : }
3719 : }
3720 :
3721 34050 : df_process_deferred_rescans ();
3722 34050 : df_clear_flags (DF_DEFER_INSN_RESCAN);
3723 34050 : bitmap_obstack_release (NULL);
3724 34050 : BITMAP_FREE (convert_bbs);
3725 :
3726 34050 : timevar_pop (TV_MACH_DEP);
3727 34050 : return 0;
3728 34050 : }
3729 :
3730 : namespace {
3731 :
3732 : const pass_data pass_data_remove_partial_avx_dependency =
3733 : {
3734 : RTL_PASS, /* type */
3735 : "rpad", /* name */
3736 : OPTGROUP_NONE, /* optinfo_flags */
3737 : TV_MACH_DEP, /* tv_id */
3738 : 0, /* properties_required */
3739 : 0, /* properties_provided */
3740 : 0, /* properties_destroyed */
3741 : 0, /* todo_flags_start */
3742 : 0, /* todo_flags_finish */
3743 : };
3744 :
3745 : class pass_remove_partial_avx_dependency : public rtl_opt_pass
3746 : {
3747 : public:
3748 295807 : pass_remove_partial_avx_dependency (gcc::context *ctxt)
3749 591614 : : rtl_opt_pass (pass_data_remove_partial_avx_dependency, ctxt)
3750 : {}
3751 :
3752 : /* opt_pass methods: */
3753 1519489 : bool gate (function *) final override
3754 : {
3755 1519489 : return ix86_rpad_gate ();
3756 : }
3757 :
3758 34050 : unsigned int execute (function *) final override
3759 : {
3760 34050 : return remove_partial_avx_dependency ();
3761 : }
3762 : }; // class pass_rpad
3763 :
3764 : } // anon namespace
3765 :
3766 : rtl_opt_pass *
3767 295807 : make_pass_remove_partial_avx_dependency (gcc::context *ctxt)
3768 : {
3769 295807 : return new pass_remove_partial_avx_dependency (ctxt);
3770 : }
3771 :
3772 : /* Return a machine mode suitable for vector SIZE with SMODE inner
3773 : mode. */
3774 :
3775 : static machine_mode
3776 66958 : ix86_get_vector_cse_mode (unsigned int size, machine_mode smode)
3777 : {
3778 : /* Use the inner scalar mode of vector broadcast source in:
3779 :
3780 : (set (reg:V8DF 394)
3781 : (vec_duplicate:V8DF (reg:V2DF 190 [ alpha ])))
3782 :
3783 : to compute the vector mode for broadcast from vector source.
3784 : */
3785 66958 : if (VECTOR_MODE_P (smode))
3786 32287 : smode = GET_MODE_INNER (smode);
3787 66958 : scalar_mode s_mode = as_a <scalar_mode> (smode);
3788 133916 : poly_uint64 nunits = size / GET_MODE_SIZE (smode);
3789 66958 : machine_mode mode = mode_for_vector (s_mode, nunits).require ();
3790 66958 : return mode;
3791 : }
3792 :
3793 : /* Replace the source operand of instructions in VECTOR_INSNS with
3794 : VECTOR_CONST in VECTOR_MODE. */
3795 :
3796 : static void
3797 66307 : replace_vector_const (machine_mode vector_mode, rtx vector_const,
3798 : auto_bitmap &vector_insns,
3799 : machine_mode scalar_mode)
3800 : {
3801 66307 : bitmap_iterator bi;
3802 66307 : unsigned int id;
3803 :
3804 233700 : EXECUTE_IF_SET_IN_BITMAP (vector_insns, 0, id, bi)
3805 : {
3806 167393 : rtx_insn *insn = DF_INSN_UID_GET (id)->insn;
3807 :
3808 : /* Get the single SET instruction. */
3809 167393 : rtx set = single_set (insn);
3810 167393 : rtx src = SET_SRC (set);
3811 167393 : rtx dest = SET_DEST (set);
3812 167393 : machine_mode mode = GET_MODE (dest);
3813 :
3814 167393 : rtx replace;
3815 : /* Replace the source operand with VECTOR_CONST. */
3816 167393 : if (SUBREG_P (src)
3817 167393 : || mode == vector_mode
3818 61696 : || CONST_INT_P (vector_const))
3819 : replace = vector_const;
3820 : else
3821 : {
3822 61689 : unsigned int size = GET_MODE_SIZE (mode);
3823 61689 : if (size < ix86_regmode_natural_size (mode))
3824 : {
3825 : /* If the mode size is smaller than its natural size,
3826 : first insert an extra move with a QI vector SUBREG
3827 : of the same size to avoid validate_subreg failure. */
3828 651 : machine_mode vmode
3829 651 : = ix86_get_vector_cse_mode (size, scalar_mode);
3830 651 : rtx vreg;
3831 651 : if (mode == vmode)
3832 : vreg = vector_const;
3833 : else
3834 : {
3835 135 : vreg = gen_reg_rtx (vmode);
3836 135 : rtx vsubreg = gen_rtx_SUBREG (vmode, vector_const, 0);
3837 135 : rtx pat = gen_rtx_SET (vreg, vsubreg);
3838 135 : rtx_insn *vinsn = emit_insn_before (pat, insn);
3839 135 : if (dump_file)
3840 : {
3841 0 : fprintf (dump_file, "\nInsert an extra move:\n\n");
3842 0 : print_rtl_single (dump_file, vinsn);
3843 0 : fprintf (dump_file, "\nbefore:\n\n");
3844 0 : print_rtl_single (dump_file, insn);
3845 0 : fprintf (dump_file, "\n");
3846 : }
3847 : }
3848 651 : replace = gen_rtx_SUBREG (mode, vreg, 0);
3849 : }
3850 : else
3851 61038 : replace = gen_rtx_SUBREG (mode, vector_const, 0);
3852 : }
3853 :
3854 167393 : if (dump_file)
3855 : {
3856 0 : fprintf (dump_file, "\nReplace:\n\n");
3857 0 : print_rtl_single (dump_file, insn);
3858 : }
3859 167393 : SET_SRC (set) = replace;
3860 167393 : if (CONST_INT_P (replace))
3861 : {
3862 24494 : dest = gen_lowpart (scalar_mode, dest);
3863 24494 : SET_DEST (set) = dest;
3864 : }
3865 : /* Drop possible dead definitions. */
3866 167393 : PATTERN (insn) = set;
3867 167393 : INSN_CODE (insn) = -1;
3868 167393 : recog_memoized (insn);
3869 167393 : if (dump_file)
3870 : {
3871 0 : fprintf (dump_file, "\nwith:\n\n");
3872 0 : print_rtl_single (dump_file, insn);
3873 0 : fprintf (dump_file, "\n");
3874 : }
3875 167393 : df_insn_rescan (insn);
3876 : }
3877 66307 : }
3878 :
3879 : /* Return the inner scalar if OP is a broadcast, else return nullptr. */
3880 :
3881 : static rtx
3882 2226410 : ix86_broadcast_inner (rtx op, machine_mode mode,
3883 : machine_mode *scalar_mode_p,
3884 : x86_cse_kind *kind_p, rtx_insn **insn_p)
3885 : {
3886 2226410 : switch (standard_sse_constant_p (op, mode))
3887 : {
3888 120092 : case 1:
3889 120092 : *scalar_mode_p = QImode;
3890 120092 : *kind_p = X86_CSE_CONST0_VECTOR;
3891 120092 : *insn_p = nullptr;
3892 120092 : return const0_rtx;
3893 12812 : case 2:
3894 12812 : *scalar_mode_p = QImode;
3895 12812 : *kind_p = X86_CSE_CONSTM1_VECTOR;
3896 12812 : *insn_p = nullptr;
3897 12812 : return constm1_rtx;
3898 2093506 : default:
3899 2093506 : break;
3900 : }
3901 :
3902 2093506 : mode = GET_MODE (op);
3903 2093506 : int nunits = GET_MODE_NUNITS (mode);
3904 2093506 : if (nunits < 2)
3905 : return nullptr;
3906 :
3907 1586614 : bool const_vector_p = CONST_VECTOR_P (op);
3908 1586614 : bool duplicated = GET_CODE (op) == VEC_DUPLICATE;
3909 1586614 : rtx orig_op = op;
3910 1586614 : if (!const_vector_p)
3911 : {
3912 : /* Check CONST_VECTOR in REG_EQUAL note. */
3913 1586594 : rtx equal = find_reg_equal_equiv_note (*insn_p);
3914 1586594 : if (equal)
3915 : {
3916 388914 : equal = XEXP (equal, 0);
3917 388914 : const_vector_p = CONST_VECTOR_P (equal);
3918 : /* Use CONST_VECTOR in REG_EQUAL note. */
3919 388914 : if (const_vector_p)
3920 : {
3921 : /* Handle REG_EQUAL note in:
3922 :
3923 : (insn 7 5 12 2 (set (subreg:V8SI (reg:V4DI 100) 0)
3924 : (vec_duplicate:V8SI (reg:SI 102)))
3925 : (expr_list:REG_DEAD (reg:SI 102)
3926 : (expr_list:REG_EQUAL (const_vector:V4DI [
3927 : (const_int -1 [0xffffffffffffffff]) repeated x4]) (nil))))
3928 :
3929 : NB: Don't treat it as CONST_VECTOR since EQUAL isn't
3930 : supported by ISAs as in gcc.target/i386/pr40957.c. */
3931 273470 : if (GET_MODE (equal) != mode)
3932 : const_vector_p = false;
3933 : else
3934 1586614 : op = equal;
3935 : }
3936 : }
3937 : }
3938 :
3939 1586614 : machine_mode inner_mode = GET_MODE_INNER (mode);
3940 :
3941 1586614 : if (const_vector_p)
3942 : {
3943 546906 : bool int_load_p = GET_MODE_SIZE (mode) <= UNITS_PER_WORD;
3944 273453 : *kind_p = X86_CSE_CONST_VECTOR;
3945 273453 : if (int_load_p)
3946 : {
3947 : /* This CONST_VECTOR load can be converted to constant
3948 : integer load. */
3949 36405 : *scalar_mode_p = mode;
3950 36405 : *insn_p = nullptr;
3951 36405 : return op;
3952 : }
3953 :
3954 : /* This CONST_VECTOR is wider than the integer register. */
3955 237048 : rtx first = XVECEXP (op, 0, 0);
3956 :
3957 237048 : if (duplicated)
3958 : {
3959 : /* Check if CONST_VECTOR in REG_EQUAL note is duplicated in
3960 :
3961 : (insn 10 7 12 2 (set (reg:V8SI 128)
3962 : (vec_duplicate:V8SI (vec_select:V2SI (reg:V4SI 180)
3963 : (parallel [(const_int 0 [0])
3964 : (const_int 1 [0x1])]))))
3965 : (expr_list:REG_EQUAL (const_vector:V8SI [
3966 : (const_int 0 [0])
3967 : (const_int 34 [0x22])
3968 : (const_int 0 [0])
3969 : (const_int 34 [0x22])
3970 : (const_int 0 [0])
3971 : (const_int 34 [0x22])
3972 : (const_int 0 [0])
3973 : (const_int 34 [0x22])])(nil)))
3974 :
3975 : */
3976 :
3977 205716 : bool duplicated_const_vector = true;
3978 205716 : for (int i = 1; i < nunits; ++i)
3979 : {
3980 133933 : rtx tmp = XVECEXP (op, 0, i);
3981 133933 : if (!rtx_equal_p (tmp, first))
3982 : {
3983 : duplicated_const_vector = false;
3984 : break;
3985 : }
3986 : }
3987 :
3988 71799 : if (duplicated_const_vector)
3989 : {
3990 71783 : bool const_double_p = CONST_DOUBLE_P (first);
3991 : /* Force the floating point constant to memory. */
3992 71783 : if (const_double_p)
3993 1262 : first = validize_mem (force_const_mem (inner_mode, first));
3994 :
3995 71783 : if (const_double_p || CONST_INT_P (first))
3996 : {
3997 : /* Handle
3998 :
3999 : (insn 7 6 8 2 (set (reg:V4SF 99)
4000 : (vec_duplicate:V4SF (mem/u/c:SF (symbol_ref/u:DI ("*.LC2") [flags 0x2]) [0 S4 A32])))
4001 : (expr_list:REG_EQUAL (const_vector:V4SF [
4002 : (const_double:SF 3.4e+1 [0x0.88p+6]) repeated x4]) (nil)))
4003 :
4004 : and
4005 :
4006 : (insn 14 15 16 3 (set (reg:V4SI 116)
4007 : (vec_duplicate:V4SI (reg:SI 117)))
4008 : (expr_list:REG_EQUAL (const_vector:V4SI [
4009 : (const_int 34 [0x22]) repeated x4]) (nil)))
4010 :
4011 : */
4012 71783 : *kind_p = X86_CSE_VEC_DUP;
4013 71783 : *insn_p = nullptr;
4014 71783 : *scalar_mode_p = inner_mode;
4015 71783 : return first;
4016 : }
4017 : }
4018 :
4019 : op = orig_op;
4020 : }
4021 : else
4022 : {
4023 : /* Only native CONST_VECTOR is allowed. */
4024 165249 : if (orig_op != op)
4025 : return nullptr;
4026 :
4027 : /* Check if VEC_DUPLICATE can be used. */
4028 48 : for (int i = 1; i < nunits; ++i)
4029 : {
4030 48 : rtx tmp = XVECEXP (op, 0, i);
4031 : /* Vector duplicate value. */
4032 48 : if (!rtx_equal_p (tmp, first))
4033 : return nullptr;
4034 : }
4035 :
4036 : /* Use the inner mode to handle
4037 : (const_vector:V2QI [(const_int 0 [0]) repeated x2])
4038 : */
4039 0 : *scalar_mode_p = inner_mode;
4040 0 : *insn_p = nullptr;
4041 0 : return first;
4042 : }
4043 : }
4044 :
4045 1313177 : if (!duplicated)
4046 : return nullptr;
4047 :
4048 23428 : *kind_p = X86_CSE_VEC_DUP;
4049 :
4050 : /* Only
4051 :
4052 : (vec_duplicate:V4SI (reg:SI 99))
4053 : (vec_duplicate:V2DF (mem/u/c:DF (symbol_ref/u:DI ("*.LC1") [flags 0x2]) [0 S8 A64]))
4054 :
4055 : are supported. Set OP to the broadcast source by default. */
4056 23428 : op = XEXP (op, 0);
4057 23428 : rtx reg = op;
4058 23428 : if (SUBREG_P (op)
4059 413 : && SUBREG_BYTE (op) == 0
4060 23841 : && !paradoxical_subreg_p (op))
4061 413 : reg = SUBREG_REG (op);
4062 23428 : if (!REG_P (reg))
4063 : {
4064 2353 : if (MEM_P (op)
4065 2083 : && SYMBOL_REF_P (XEXP (op, 0))
4066 2628 : && CONSTANT_POOL_ADDRESS_P (XEXP (op, 0)))
4067 : {
4068 : /* Handle constant broadcast from memory. */
4069 10 : *scalar_mode_p = inner_mode;
4070 10 : *insn_p = nullptr;
4071 10 : return op;
4072 : }
4073 : return nullptr;
4074 : }
4075 :
4076 21075 : machine_mode orig_mode = mode;
4077 21075 : mode = GET_MODE (op);
4078 :
4079 : /* Only single def chain is supported. */
4080 21075 : df_ref ref = DF_REG_DEF_CHAIN (REGNO (reg));
4081 21075 : if (!ref
4082 21075 : || DF_REF_IS_ARTIFICIAL (ref)
4083 21075 : || DF_REF_NEXT_REG (ref) != nullptr)
4084 : return nullptr;
4085 :
4086 15424 : rtx_insn *insn = DF_REF_INSN (ref);
4087 :
4088 : /* Skip JUMP which happens with asm goto. */
4089 15424 : if (JUMP_P (insn))
4090 : return nullptr;
4091 :
4092 15422 : rtx set = single_set (insn);
4093 15422 : if (!set)
4094 : return nullptr;
4095 :
4096 15412 : rtx src = SET_SRC (set);
4097 :
4098 15412 : if (CONST_INT_P (src))
4099 : {
4100 : /* Handle sequences like
4101 :
4102 : (set (subreg:SI (reg/v:SF 105 [ f ]) 0)
4103 : (const_int 0 [0]))
4104 : (set (reg:V4SF 110)
4105 : (vec_duplicate:V4SF (reg/v:SF 105 [ f ])))
4106 :
4107 : and
4108 :
4109 : (set (reg:SI 99)
4110 : (const_int 34 [0x22]))
4111 : (set (reg:V4SI 98)
4112 : (vec_duplicate:V4SI (reg:SI 99)))
4113 :
4114 : Set *INSN_P to nullptr and return SET_SRC if SET_SRC is an
4115 : integer constant. */
4116 231 : op = src;
4117 231 : if (SCALAR_INT_MODE_P (mode) && mode != GET_MODE (reg))
4118 0 : op = gen_int_mode (INTVAL (src), mode);
4119 231 : if (op == const0_rtx)
4120 : {
4121 4 : if (standard_sse_constant_p (CONST0_RTX (orig_mode),
4122 : orig_mode) == 1)
4123 : {
4124 4 : *scalar_mode_p = QImode;
4125 4 : *kind_p = X86_CSE_CONST0_VECTOR;
4126 4 : *insn_p = nullptr;
4127 4 : return const0_rtx;
4128 : }
4129 0 : op = CONST0_RTX (mode);
4130 : }
4131 227 : else if (op == constm1_rtx
4132 227 : && standard_sse_constant_p (CONSTM1_RTX (orig_mode),
4133 : orig_mode) == 2)
4134 : {
4135 0 : *scalar_mode_p = QImode;
4136 0 : *kind_p = X86_CSE_CONSTM1_VECTOR;
4137 0 : *insn_p = nullptr;
4138 0 : return constm1_rtx;
4139 : }
4140 :
4141 : /* Check if we can convert:
4142 :
4143 : (insn 14 465 412 3 (set (reg:SI 507 [ j_lsm.26 ])
4144 : (const_int 2 [0x2])) "foo.c":10:12 discrim 2 100 {*movsi_internal} (nil))
4145 : ...
4146 : (insn 518 507 434 16 (set (reg:V2SI 493)
4147 : (vec_duplicate:V2SI (reg:SI 507 [ j_lsm.26 ]))) 2395 {*vec_dupv2si} (nil))
4148 :
4149 : to constant integer load:
4150 :
4151 : (insn 566 55 56 6 (set (subreg:DI (reg:V2SI 517) 0)
4152 : (const_int 8589934594 [0x200000002])) -1 (nil))
4153 : ...
4154 : (insn 518 507 434 16 (set (reg:V2SI 493)
4155 : (reg:V2SI 517)) 2066 {*movv2si_internal} (nil))
4156 :
4157 : */
4158 454 : if (GET_MODE_SIZE (orig_mode) <= UNITS_PER_WORD)
4159 17 : *kind_p = X86_CSE_CONST_VECTOR;
4160 :
4161 227 : *insn_p = nullptr;
4162 : }
4163 : else
4164 : {
4165 : /* Handle sequences like
4166 :
4167 : (set (reg:QI 105 [ c ])
4168 : (reg:QI 5 di [ c ]))
4169 : (set (reg:V64QI 102 [ _1 ])
4170 : (vec_duplicate:V64QI (reg:QI 105 [ c ])))
4171 :
4172 : (set (reg/v:SI 116 [ argc ])
4173 : (mem/c:SI (reg:SI 135) [2 argc+0 S4 A32]))
4174 : (set (reg:V4SI 119 [ _45 ])
4175 : (vec_duplicate:V4SI (reg/v:SI 116 [ argc ])))
4176 :
4177 : (set (reg:SI 98 [ _1 ])
4178 : (sign_extend:SI (reg:QI 106 [ c ])))
4179 : (set (reg:V16SI 103 [ _2 ])
4180 : (vec_duplicate:V16SI (reg:SI 98 [ _1 ])))
4181 :
4182 : (set (reg:SI 102 [ cost ])
4183 : (mem/c:SI (symbol_ref:DI ("cost") [flags 0x40])))
4184 : (set (reg:V4HI 103 [ _16 ])
4185 : (vec_duplicate:V4HI (subreg:HI (reg:SI 102 [ cost ]) 0)))
4186 :
4187 : (set (subreg:SI (reg/v:HI 107 [ cr_val ]) 0)
4188 : (ashift:SI (reg:SI 158)
4189 : (subreg:QI (reg:SI 156 [ _2 ]) 0)))
4190 : (set (reg:V16HI 183 [ _61 ])
4191 : (vec_duplicate:V16HI (reg/v:HI 107 [ cr_val ])))
4192 :
4193 : Set *INSN_P to INSN and return the broadcast source otherwise. */
4194 15181 : *insn_p = insn;
4195 : }
4196 :
4197 15408 : *scalar_mode_p = mode;
4198 15408 : return op;
4199 : }
4200 :
4201 : /* Replace CALL instruction in TLS_CALL_INSNS with SET from SRC and
4202 : put the updated instruction in UPDATED_TLS_INSNS. */
4203 :
4204 : static void
4205 320 : replace_tls_call (rtx src, auto_bitmap &tls_call_insns,
4206 : auto_bitmap &updated_tls_insns)
4207 : {
4208 320 : bitmap_iterator bi;
4209 320 : unsigned int id;
4210 :
4211 1754 : EXECUTE_IF_SET_IN_BITMAP (tls_call_insns, 0, id, bi)
4212 : {
4213 1434 : rtx_insn *insn = DF_INSN_UID_GET (id)->insn;
4214 :
4215 : /* If this isn't a CALL, only GNU2 TLS implicit CALL patterns are
4216 : allowed. */
4217 1434 : if (!CALL_P (insn))
4218 : {
4219 47 : attr_tls64 tls64 = get_attr_tls64 (insn);
4220 47 : if (tls64 != TLS64_CALL && tls64 != TLS64_COMBINE)
4221 0 : gcc_unreachable ();
4222 : }
4223 :
4224 1434 : rtx pat = PATTERN (insn);
4225 1434 : gcc_assert (GET_CODE (pat) == PARALLEL);
4226 1434 : rtx set = XVECEXP (pat, 0, 0);
4227 1434 : gcc_assert (GET_CODE (set) == SET);
4228 1434 : rtx dest = SET_DEST (set);
4229 :
4230 1434 : set = gen_rtx_SET (dest, src);
4231 1434 : rtx_insn *set_insn = emit_insn_after (set, insn);
4232 1434 : if (recog_memoized (set_insn) < 0)
4233 0 : gcc_unreachable ();
4234 :
4235 : /* Put SET_INSN in UPDATED_TLS_INSNS. */
4236 1434 : bitmap_set_bit (updated_tls_insns, INSN_UID (set_insn));
4237 :
4238 1434 : if (dump_file)
4239 : {
4240 0 : fprintf (dump_file, "\nReplace:\n\n");
4241 0 : print_rtl_single (dump_file, insn);
4242 0 : fprintf (dump_file, "\nwith:\n\n");
4243 0 : print_rtl_single (dump_file, set_insn);
4244 0 : fprintf (dump_file, "\n");
4245 : }
4246 :
4247 : /* Delete the CALL insn. */
4248 1434 : delete_insn (insn);
4249 :
4250 1434 : df_insn_rescan (set_insn);
4251 : }
4252 320 : }
4253 :
4254 : /* Return the basic block which dominates all basic blocks which set
4255 : hard register REGNO used in basic block BB. */
4256 :
4257 : static basic_block
4258 2 : ix86_get_dominator_for_reg (unsigned int regno, basic_block bb)
4259 : {
4260 2 : basic_block set_bb;
4261 2 : auto_bitmap set_bbs;
4262 :
4263 : /* Get all BBs which set REGNO and dominate the current BB from all
4264 : DEFs of REGNO. */
4265 2 : for (df_ref def = DF_REG_DEF_CHAIN (regno);
4266 18 : def;
4267 16 : def = DF_REF_NEXT_REG (def))
4268 16 : if (!DF_REF_IS_ARTIFICIAL (def)
4269 16 : && !DF_REF_FLAGS_IS_SET (def, DF_REF_MAY_CLOBBER)
4270 6 : && !DF_REF_FLAGS_IS_SET (def, DF_REF_MUST_CLOBBER))
4271 : {
4272 4 : set_bb = DF_REF_BB (def);
4273 4 : if (dominated_by_p (CDI_DOMINATORS, bb, set_bb))
4274 2 : bitmap_set_bit (set_bbs, set_bb->index);
4275 : }
4276 :
4277 2 : bb = nearest_common_dominator_for_set (CDI_DOMINATORS, set_bbs);
4278 2 : return bb;
4279 2 : }
4280 :
4281 : /* Mark FLAGS register as live in DATA, a bitmap of live caller-saved
4282 : registers, if DEST is FLAGS register. */
4283 :
4284 : static void
4285 388 : ix86_check_flags_reg (rtx dest, const_rtx x, void *data)
4286 : {
4287 388 : if (GET_CODE (x) == CLOBBER)
4288 : return;
4289 :
4290 381 : auto_bitmap *live_caller_saved_regs = (auto_bitmap *) data;
4291 381 : if (REG_P (dest) && REGNO (dest) == FLAGS_REG)
4292 0 : bitmap_set_bit (*live_caller_saved_regs, FLAGS_REG);
4293 : }
4294 :
4295 : /* Emit a TLS_SET instruction of KIND in basic block BB. Store the
4296 : insertion point in *BEFORE_P for emit_insn_before or in *AFTER_P
4297 : for emit_insn_after. UPDATED_GNU_TLS_INSNS contains instructions
4298 : which replace the GNU TLS instructions. UPDATED_GNU2_TLS_INSNS
4299 : contains instructions which replace the GNU2 TLS instructions. */
4300 :
4301 : static rtx_insn *
4302 320 : ix86_emit_tls_call (rtx tls_set, x86_cse_kind kind, basic_block bb,
4303 : rtx_insn **before_p, rtx_insn **after_p,
4304 : auto_bitmap &updated_gnu_tls_insns,
4305 : auto_bitmap &updated_gnu2_tls_insns)
4306 : {
4307 322 : rtx_insn *tls_insn;
4308 :
4309 322 : do
4310 : {
4311 322 : rtx_insn *insn = BB_HEAD (bb);
4312 1287 : while (insn && !NONDEBUG_INSN_P (insn))
4313 : {
4314 969 : if (insn == BB_END (bb))
4315 : {
4316 : /* This must be the beginning basic block:
4317 :
4318 : (note 4 0 2 2 [bb 2] NOTE_INSN_BASIC_BLOCK)
4319 : (note 2 4 26 2 NOTE_INSN_FUNCTION_BEG)
4320 :
4321 : or a basic block with only a label:
4322 :
4323 : (code_label 78 11 77 3 14 (nil) [1 uses])
4324 : (note 77 78 54 3 [bb 3] NOTE_INSN_BASIC_BLOCK)
4325 :
4326 : or a basic block with only a debug marker:
4327 :
4328 : (note 3 0 2 2 [bb 2] NOTE_INSN_BASIC_BLOCK)
4329 : (note 2 3 5 2 NOTE_INSN_FUNCTION_BEG)
4330 : (debug_insn 5 2 16 2 (debug_marker) "x.c":6:3 -1 (nil))
4331 :
4332 : or a basic block with only deleted instructions:
4333 :
4334 : (code_label 348 23 349 45 3 (nil) [0 uses])
4335 : (note 349 348 436 45 [bb 45] NOTE_INSN_BASIC_BLOCK)
4336 : (note 436 349 362 45 NOTE_INSN_DELETED)
4337 :
4338 : */
4339 4 : gcc_assert (DEBUG_INSN_P (insn)
4340 : || (NOTE_P (insn)
4341 : && ((NOTE_KIND (insn)
4342 : == NOTE_INSN_FUNCTION_BEG)
4343 : || (NOTE_KIND (insn)
4344 : == NOTE_INSN_DELETED)
4345 : || (NOTE_KIND (insn)
4346 : == NOTE_INSN_BASIC_BLOCK))));
4347 : insn = NULL;
4348 : break;
4349 : }
4350 965 : insn = NEXT_INSN (insn);
4351 : }
4352 :
4353 : /* TLS_GD and TLS_LD_BASE instructions are normal functions which
4354 : clobber caller-saved registers. TLSDESC instructions only
4355 : clobber FLAGS. If any registers clobbered by TLS instructions
4356 : are live in this basic block, we must insert TLS instructions
4357 : after all live registers clobbered are dead. */
4358 :
4359 322 : auto_bitmap live_caller_saved_regs;
4360 644 : bitmap in = df_live ? DF_LIVE_IN (bb) : DF_LR_IN (bb);
4361 :
4362 322 : if (bitmap_bit_p (in, FLAGS_REG))
4363 4 : bitmap_set_bit (live_caller_saved_regs, FLAGS_REG);
4364 :
4365 322 : unsigned int i;
4366 :
4367 : /* Get all live caller-saved registers for TLS_GD and TLS_LD_BASE
4368 : instructions. */
4369 322 : if (kind != X86_CSE_TLSDESC)
4370 : {
4371 300 : const predefined_function_abi &tls_abi
4372 300 : = ix86_tls_get_addr_abi ();
4373 28800 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
4374 28200 : if (tls_abi.clobbers_full_reg_p (i)
4375 25503 : && !fixed_regs[i]
4376 40548 : && bitmap_bit_p (in, i))
4377 350 : bitmap_set_bit (live_caller_saved_regs, i);
4378 300 : if (df_live && crtl->drap_reg)
4379 : {
4380 : /* DRAP has no reaching definition at this point, so df_live drops
4381 : it above. Its hard register can also go dead mid-function once
4382 : copied elsewhere (e.g. right after the prologue), so query
4383 : DF_LR_IN per-block rather than treating it as live whenever
4384 : crtl->drap_reg is set. */
4385 1 : i = REGNO (crtl->drap_reg);
4386 2 : if (bitmap_bit_p (DF_LR_IN (bb), i))
4387 1 : bitmap_set_bit (live_caller_saved_regs, i);
4388 : }
4389 : }
4390 :
4391 322 : if (bitmap_empty_p (live_caller_saved_regs))
4392 : {
4393 82 : if (insn == BB_HEAD (bb))
4394 : {
4395 0 : *before_p = insn;
4396 0 : tls_insn = emit_insn_before (tls_set, insn);
4397 : }
4398 : else
4399 : {
4400 : /* Emit the TLS call after NOTE_INSN_FUNCTION_BEG in the
4401 : beginning basic block:
4402 :
4403 : (note 4 0 2 2 [bb 2] NOTE_INSN_BASIC_BLOCK)
4404 : (note 2 4 26 2 NOTE_INSN_FUNCTION_BEG)
4405 :
4406 : or after NOTE_INSN_BASIC_BLOCK in a basic block with
4407 : only a label:
4408 :
4409 : (code_label 78 11 77 3 14 (nil) [1 uses])
4410 : (note 77 78 54 3 [bb 3] NOTE_INSN_BASIC_BLOCK)
4411 :
4412 : or after debug marker in a basic block with only a
4413 : debug marker:
4414 :
4415 : (note 3 0 2 2 [bb 2] NOTE_INSN_BASIC_BLOCK)
4416 : (note 2 3 5 2 NOTE_INSN_FUNCTION_BEG)
4417 : (debug_insn 5 2 16 2 (debug_marker) "x.c":6:3 -1 (nil))
4418 :
4419 : */
4420 82 : insn = insn ? PREV_INSN (insn) : BB_END (bb);
4421 82 : *after_p = insn;
4422 82 : tls_insn = emit_insn_after (tls_set, insn);
4423 : }
4424 : return tls_insn;
4425 : }
4426 :
4427 240 : bool repeat = false;
4428 :
4429 : /* Search for REG_DEAD notes in this basic block. */
4430 675 : FOR_BB_INSNS (bb, insn)
4431 : {
4432 675 : if (!NONDEBUG_INSN_P (insn))
4433 290 : continue;
4434 :
4435 : /* NB: Conditional jump is the only instruction which reads
4436 : flags register and changes control flow. We can never
4437 : place the TLS call after unconditional jump. */
4438 385 : if (JUMP_P (insn))
4439 : {
4440 : /* This must be a conditional jump. */
4441 2 : rtx label = JUMP_LABEL (insn);
4442 2 : if (label == nullptr
4443 2 : || ANY_RETURN_P (label)
4444 2 : || !(LABEL_P (label) || SYMBOL_REF_P (label)))
4445 0 : gcc_unreachable ();
4446 :
4447 : /* Place the call before all FLAGS_REG setting BBs since
4448 : we can't place a call before nor after a conditional
4449 : jump. */
4450 2 : bb = ix86_get_dominator_for_reg (FLAGS_REG, bb);
4451 :
4452 : /* Start over again. */
4453 2 : repeat = true;
4454 2 : break;
4455 : }
4456 :
4457 383 : if (bitmap_bit_p (updated_gnu_tls_insns, INSN_UID (insn)))
4458 : {
4459 : /* Insert the __tls_get_addr call before INSN which
4460 : replaces a __tls_get_addr call. */
4461 1 : *before_p = insn;
4462 1 : tls_insn = emit_insn_before (tls_set, insn);
4463 1 : return tls_insn;
4464 : }
4465 :
4466 382 : if (bitmap_bit_p (updated_gnu2_tls_insns, INSN_UID (insn)))
4467 : {
4468 : /* Mark FLAGS register as dead since FLAGS register
4469 : would be clobbered by the GNU2 TLS instruction. */
4470 1 : bitmap_clear_bit (live_caller_saved_regs, FLAGS_REG);
4471 1 : continue;
4472 : }
4473 :
4474 : /* Check if FLAGS register is live. */
4475 381 : note_stores (insn, ix86_check_flags_reg,
4476 : &live_caller_saved_regs);
4477 :
4478 381 : rtx link;
4479 522 : for (link = REG_NOTES (insn); link; link = XEXP (link, 1))
4480 378 : if ((REG_NOTE_KIND (link) == REG_DEAD
4481 9 : || (REG_NOTE_KIND (link) == REG_UNUSED
4482 7 : && REGNO (XEXP (link, 0)) == FLAGS_REG))
4483 385 : && REG_P (XEXP (link, 0)))
4484 : {
4485 : /* Mark the live caller-saved register as dead. */
4486 757 : for (i = REGNO (XEXP (link, 0));
4487 757 : i < END_REGNO (XEXP (link, 0));
4488 : i++)
4489 381 : if (i < FIRST_PSEUDO_REGISTER)
4490 358 : bitmap_clear_bit (live_caller_saved_regs, i);
4491 :
4492 376 : if (bitmap_empty_p (live_caller_saved_regs))
4493 : {
4494 237 : *after_p = insn;
4495 237 : tls_insn = emit_insn_after (tls_set, insn);
4496 237 : return tls_insn;
4497 : }
4498 : }
4499 : }
4500 :
4501 : /* NB: Start over again for conditional jump. */
4502 2 : if (repeat)
4503 2 : continue;
4504 :
4505 0 : gcc_assert (!bitmap_empty_p (live_caller_saved_regs));
4506 :
4507 : /* If any live caller-saved registers aren't dead at the end of
4508 : this basic block, get the basic block which dominates all
4509 : basic blocks which set the remaining live registers. */
4510 0 : auto_bitmap set_bbs;
4511 0 : bitmap_iterator bi;
4512 0 : unsigned int id;
4513 0 : EXECUTE_IF_SET_IN_BITMAP (live_caller_saved_regs, 0, id, bi)
4514 : {
4515 0 : basic_block set_bb = ix86_get_dominator_for_reg (id, bb);
4516 0 : bitmap_set_bit (set_bbs, set_bb->index);
4517 : }
4518 0 : bb = nearest_common_dominator_for_set (CDI_DOMINATORS, set_bbs);
4519 2 : }
4520 : while (true);
4521 : }
4522 :
4523 : /* Generate a TLS call of KIND with VAL and copy the call result to DEST,
4524 : at entry of the nearest dominator for basic block map BBS, which is in
4525 : the fake loop that contains the whole function, so that there is only
4526 : a single TLS CALL of KIND with VAL in the whole function.
4527 : UPDATED_GNU_TLS_INSNS contains instructions which replace the GNU TLS
4528 : instructions. UPDATED_GNU2_TLS_INSNS contains instructions which
4529 : replace the GNU2 TLS instructions. If TLSDESC_SET isn't nullptr,
4530 : insert it before the TLS call. */
4531 :
4532 : static void
4533 320 : ix86_place_single_tls_call (rtx dest, rtx val, x86_cse_kind kind,
4534 : auto_bitmap &bbs,
4535 : auto_bitmap &updated_gnu_tls_insns,
4536 : auto_bitmap &updated_gnu2_tls_insns,
4537 : rtx tlsdesc_set = nullptr)
4538 : {
4539 320 : basic_block bb = nearest_common_dominator_for_set (CDI_DOMINATORS, bbs);
4540 320 : while (bb->loop_father->latch
4541 330 : != EXIT_BLOCK_PTR_FOR_FN (cfun))
4542 10 : bb = get_immediate_dominator (CDI_DOMINATORS,
4543 : bb->loop_father->header);
4544 :
4545 320 : rtx rax = nullptr, rdi;
4546 320 : rtx eqv = nullptr;
4547 320 : rtx caddr;
4548 320 : rtx set;
4549 320 : rtx clob;
4550 320 : rtx symbol;
4551 320 : rtx tls;
4552 :
4553 320 : switch (kind)
4554 : {
4555 265 : case X86_CSE_TLS_GD:
4556 265 : rax = gen_rtx_REG (Pmode, AX_REG);
4557 265 : rdi = gen_rtx_REG (Pmode, DI_REG);
4558 265 : caddr = ix86_tls_get_addr ();
4559 :
4560 265 : symbol = XVECEXP (val, 0, 0);
4561 265 : tls = gen_tls_global_dynamic_64 (Pmode, rax, symbol, caddr, rdi);
4562 265 : CALL_INSN_ABI_ID (tls) = ix86_tls_get_addr_abi ().id ();
4563 :
4564 265 : if (GET_MODE (symbol) != Pmode)
4565 0 : symbol = gen_rtx_ZERO_EXTEND (Pmode, symbol);
4566 : eqv = symbol;
4567 : break;
4568 :
4569 34 : case X86_CSE_TLS_LD_BASE:
4570 34 : rax = gen_rtx_REG (Pmode, AX_REG);
4571 34 : rdi = gen_rtx_REG (Pmode, DI_REG);
4572 34 : caddr = ix86_tls_get_addr ();
4573 :
4574 34 : tls = gen_tls_local_dynamic_base_64 (Pmode, rax, caddr, rdi);
4575 34 : CALL_INSN_ABI_ID (tls) = ix86_tls_get_addr_abi ().id ();
4576 :
4577 : /* Attach a unique REG_EQUAL to DEST, to allow the RTL optimizers
4578 : to share the LD_BASE result with other LD model accesses. */
4579 34 : eqv = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx),
4580 : UNSPEC_TLS_LD_BASE);
4581 :
4582 34 : break;
4583 :
4584 21 : case X86_CSE_TLSDESC:
4585 21 : set = gen_rtx_SET (dest, val);
4586 21 : clob = gen_rtx_CLOBBER (VOIDmode,
4587 : gen_rtx_REG (CCmode, FLAGS_REG));
4588 21 : tls = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, set, clob));
4589 21 : break;
4590 :
4591 0 : default:
4592 0 : gcc_unreachable ();
4593 : }
4594 :
4595 : /* Emit the TLS CALL insn. */
4596 320 : rtx_insn *before = nullptr;
4597 320 : rtx_insn *after = nullptr;
4598 320 : rtx_insn *tls_insn = ix86_emit_tls_call (tls, kind, bb, &before,
4599 : &after,
4600 : updated_gnu_tls_insns,
4601 : updated_gnu2_tls_insns);
4602 :
4603 320 : rtx_insn *tlsdesc_insn = nullptr;
4604 320 : if (tlsdesc_set)
4605 : {
4606 16 : rtx dest = copy_rtx (SET_DEST (tlsdesc_set));
4607 16 : rtx src = copy_rtx (SET_SRC (tlsdesc_set));
4608 16 : tlsdesc_set = gen_rtx_SET (dest, src);
4609 16 : tlsdesc_insn = emit_insn_before (tlsdesc_set, tls_insn);
4610 : }
4611 :
4612 320 : if (kind != X86_CSE_TLSDESC)
4613 : {
4614 299 : RTL_CONST_CALL_P (tls_insn) = 1;
4615 :
4616 : /* Indicate that this function can't jump to non-local gotos. */
4617 299 : make_reg_eh_region_note_nothrow_nononlocal (tls_insn);
4618 : }
4619 :
4620 320 : if (recog_memoized (tls_insn) < 0)
4621 0 : gcc_unreachable ();
4622 :
4623 320 : if (dump_file)
4624 : {
4625 0 : if (after)
4626 : {
4627 0 : fprintf (dump_file, "\nPlace:\n\n");
4628 0 : if (tlsdesc_insn)
4629 0 : print_rtl_single (dump_file, tlsdesc_insn);
4630 0 : print_rtl_single (dump_file, tls_insn);
4631 0 : fprintf (dump_file, "\nafter:\n\n");
4632 0 : print_rtl_single (dump_file, after);
4633 0 : fprintf (dump_file, "\n");
4634 : }
4635 : else
4636 : {
4637 0 : fprintf (dump_file, "\nPlace:\n\n");
4638 0 : if (tlsdesc_insn)
4639 0 : print_rtl_single (dump_file, tlsdesc_insn);
4640 0 : print_rtl_single (dump_file, tls_insn);
4641 0 : fprintf (dump_file, "\nbefore:\n\n");
4642 0 : print_rtl_single (dump_file, before);
4643 0 : fprintf (dump_file, "\n");
4644 : }
4645 : }
4646 :
4647 320 : if (kind != X86_CSE_TLSDESC)
4648 : {
4649 : /* Copy RAX to DEST. */
4650 299 : set = gen_rtx_SET (dest, rax);
4651 299 : rtx_insn *set_insn = emit_insn_after (set, tls_insn);
4652 299 : set_dst_reg_note (set_insn, REG_EQUAL, copy_rtx (eqv), dest);
4653 299 : if (dump_file)
4654 : {
4655 0 : fprintf (dump_file, "\nPlace:\n\n");
4656 0 : print_rtl_single (dump_file, set_insn);
4657 0 : fprintf (dump_file, "\nafter:\n\n");
4658 0 : print_rtl_single (dump_file, tls_insn);
4659 0 : fprintf (dump_file, "\n");
4660 : }
4661 : }
4662 320 : }
4663 :
4664 : namespace {
4665 :
4666 : const pass_data pass_data_x86_cse =
4667 : {
4668 : RTL_PASS, /* type */
4669 : "x86_cse", /* name */
4670 : OPTGROUP_NONE, /* optinfo_flags */
4671 : TV_MACH_DEP, /* tv_id */
4672 : 0, /* properties_required */
4673 : 0, /* properties_provided */
4674 : 0, /* properties_destroyed */
4675 : 0, /* todo_flags_start */
4676 : 0, /* todo_flags_finish */
4677 : };
4678 :
4679 : class pass_x86_cse : public rtl_opt_pass
4680 : {
4681 : public:
4682 295807 : pass_x86_cse (gcc::context *ctxt)
4683 591614 : : rtl_opt_pass (pass_data_x86_cse, ctxt)
4684 : {}
4685 :
4686 : /* opt_pass methods: */
4687 1519489 : bool gate (function *fun) final override
4688 : {
4689 1519489 : return optimize && optimize_function_for_speed_p (fun);
4690 : }
4691 :
4692 1005363 : unsigned int execute (function *) final override
4693 : {
4694 1005363 : return x86_cse ();
4695 : }
4696 :
4697 : private:
4698 : /* The redundant source value. */
4699 : rtx val;
4700 : /* The actual redundant source value for UNSPEC_TLSDESC. */
4701 : rtx tlsdesc_val;
4702 : /* The instruction which defines the redundant value. */
4703 : rtx_insn *def_insn;
4704 : /* Mode of the destination of the candidate redundant instruction. */
4705 : machine_mode mode;
4706 : /* Mode of the source of the candidate redundant instruction. */
4707 : machine_mode scalar_mode;
4708 : /* The classification of the candidate redundant instruction. */
4709 : x86_cse_kind kind;
4710 :
4711 : unsigned int x86_cse (void);
4712 : bool candidate_gnu_tls_p (rtx_insn *, attr_tls64);
4713 : bool candidate_gnu2_tls_p (rtx, attr_tls64);
4714 : bool candidate_vector_p (rtx, rtx_insn *);
4715 : rtx_insn *tls_set_insn_from_symbol (const_rtx, const_rtx);
4716 : }; // class pass_x86_cse
4717 :
4718 : /* Return the instruction which sets REG from TLS_SYMBOL. */
4719 :
4720 : rtx_insn *
4721 42 : pass_x86_cse::tls_set_insn_from_symbol (const_rtx reg,
4722 : const_rtx tls_symbol)
4723 : {
4724 42 : rtx_insn *set_insn = nullptr;
4725 42 : for (df_ref ref = DF_REG_DEF_CHAIN (REGNO (reg));
4726 111 : ref;
4727 69 : ref = DF_REF_NEXT_REG (ref))
4728 : {
4729 69 : if (DF_REF_IS_ARTIFICIAL (ref))
4730 : return nullptr;
4731 :
4732 69 : set_insn = DF_REF_INSN (ref);
4733 69 : if (get_attr_tls64 (set_insn) != TLS64_LEA)
4734 : return nullptr;
4735 :
4736 69 : rtx tls_set = PATTERN (set_insn);
4737 69 : rtx tls_src = XVECEXP (SET_SRC (tls_set), 0, 0);
4738 69 : if (!rtx_equal_p (tls_symbol, tls_src))
4739 : return nullptr;
4740 : }
4741 :
4742 : return set_insn;
4743 : }
4744 :
4745 : /* Return true and output def_insn, val, mode, scalar_mode and kind if
4746 : INSN is UNSPEC_TLS_GD or UNSPEC_TLS_LD_BASE. */
4747 :
4748 : bool
4749 2199 : pass_x86_cse::candidate_gnu_tls_p (rtx_insn *insn, attr_tls64 tls64)
4750 : {
4751 2199 : if (!TARGET_64BIT || !cfun->machine->tls_descriptor_call_multiple_p)
4752 : return false;
4753 :
4754 : /* Record the redundant TLS CALLs for 64-bit:
4755 :
4756 : (parallel [
4757 : (set (reg:DI 0 ax)
4758 : (call:DI (mem:QI (symbol_ref:DI ("__tls_get_addr")))
4759 : (const_int 0 [0])))
4760 : (unspec:DI [(symbol_ref:DI ("foo") [flags 0x50])
4761 : (reg/f:DI 7 sp)] UNSPEC_TLS_GD)
4762 : (clobber (reg:DI 5 di))])
4763 :
4764 :
4765 : and
4766 :
4767 : (parallel [
4768 : (set (reg:DI 0 ax)
4769 : (call:DI (mem:QI (symbol_ref:DI ("__tls_get_addr")))
4770 : (const_int 0 [0])))
4771 : (unspec:DI [(reg/f:DI 7 sp)] UNSPEC_TLS_LD_BASE)])
4772 :
4773 : */
4774 :
4775 2031 : rtx pat = PATTERN (insn);
4776 2031 : rtx set = XVECEXP (pat, 0, 0);
4777 2031 : gcc_assert (GET_CODE (set) == SET);
4778 2031 : rtx dest = SET_DEST (set);
4779 2031 : scalar_mode = mode = GET_MODE (dest);
4780 2031 : val = XVECEXP (pat, 0, 1);
4781 2031 : gcc_assert (GET_CODE (val) == UNSPEC);
4782 :
4783 2031 : if (tls64 == TLS64_GD)
4784 : kind = X86_CSE_TLS_GD;
4785 : else
4786 109 : kind = X86_CSE_TLS_LD_BASE;
4787 :
4788 2031 : def_insn = nullptr;
4789 2031 : return true;
4790 : }
4791 :
4792 : /* Return true and output def_insn, val, mode, scalar_mode and kind if
4793 : SET is UNSPEC_TLSDESC. */
4794 :
4795 : bool
4796 56 : pass_x86_cse::candidate_gnu2_tls_p (rtx set, attr_tls64 tls64)
4797 : {
4798 56 : if (!TARGET_64BIT || !cfun->machine->tls_descriptor_call_multiple_p)
4799 : return false;
4800 :
4801 54 : rtx tls_symbol;
4802 54 : rtx_insn *set_insn;
4803 54 : rtx src = SET_SRC (set);
4804 54 : val = src;
4805 54 : tlsdesc_val = src;
4806 54 : kind = X86_CSE_TLSDESC;
4807 :
4808 54 : if (tls64 == TLS64_COMBINE)
4809 : {
4810 : /* Record 64-bit TLS64_COMBINE:
4811 :
4812 : (set (reg/f:DI 104)
4813 : (plus:DI (unspec:DI [
4814 : (symbol_ref:DI ("_TLS_MODULE_BASE_") [flags 0x10])
4815 : (reg:DI 114)
4816 : (reg/f:DI 7 sp)] UNSPEC_TLSDESC)
4817 : (const:DI (unspec:DI [
4818 : (symbol_ref:DI ("e") [flags 0x1a])
4819 : ] UNSPEC_DTPOFF))))
4820 :
4821 : (set (reg/f:DI 104)
4822 : (plus:DI (unspec:DI [
4823 : (symbol_ref:DI ("_TLS_MODULE_BASE_") [flags 0x10])
4824 : (unspec:DI [
4825 : (symbol_ref:DI ("_TLS_MODULE_BASE_") [flags 0x10])
4826 : ] UNSPEC_TLSDESC)
4827 : (reg/f:DI 7 sp)] UNSPEC_TLSDESC)
4828 : (const:DI (unspec:DI [
4829 : (symbol_ref:DI ("e") [flags 0x1a])
4830 : ] UNSPEC_DTPOFF))))
4831 : */
4832 :
4833 12 : scalar_mode = mode = GET_MODE (src);
4834 :
4835 : /* Since the first operand of PLUS in the source TLS_COMBINE
4836 : pattern is unused, use the second operand of PLUS:
4837 :
4838 : (const:DI (unspec:DI [
4839 : (symbol_ref:DI ("e") [flags 0x1a])
4840 : ] UNSPEC_DTPOFF))
4841 :
4842 : as VAL to check if 2 TLS_COMBINE patterns have the same
4843 : source. */
4844 12 : val = XEXP (src, 1);
4845 12 : gcc_assert (GET_CODE (val) == CONST
4846 : && GET_CODE (XEXP (val, 0)) == UNSPEC
4847 : && XINT (XEXP (val, 0), 1) == UNSPEC_DTPOFF
4848 : && SYMBOL_REF_P (XVECEXP (XEXP (val, 0), 0, 0)));
4849 12 : def_insn = nullptr;
4850 12 : return true;
4851 : }
4852 :
4853 : /* Record 64-bit TLS_CALL:
4854 :
4855 : (set (reg:DI 101)
4856 : (unspec:DI [(symbol_ref:DI ("foo") [flags 0x50])
4857 : (reg:DI 112)
4858 : (reg/f:DI 7 sp)] UNSPEC_TLSDESC))
4859 :
4860 : */
4861 :
4862 42 : gcc_assert (GET_CODE (src) == UNSPEC);
4863 42 : tls_symbol = XVECEXP (src, 0, 0);
4864 42 : src = XVECEXP (src, 0, 1);
4865 42 : scalar_mode = mode = GET_MODE (src);
4866 42 : gcc_assert (REG_P (src));
4867 :
4868 : /* All definitions of reg:DI 129 in
4869 :
4870 : (set (reg:DI 110)
4871 : (unspec:DI [(symbol_ref:DI ("foo"))
4872 : (reg:DI 129)
4873 : (reg/f:DI 7 sp)] UNSPEC_TLSDESC))
4874 :
4875 : should have the same source as in
4876 :
4877 : (set (reg:DI 129)
4878 : (unspec:DI [(symbol_ref:DI ("foo"))] UNSPEC_TLSDESC))
4879 :
4880 : */
4881 :
4882 42 : set_insn = tls_set_insn_from_symbol (src, tls_symbol);
4883 42 : if (!set_insn)
4884 : return false;
4885 :
4886 : /* Use TLS_SYMBOL as VAL to check if 2 patterns have the same source. */
4887 42 : val = tls_symbol;
4888 42 : def_insn = set_insn;
4889 42 : return true;
4890 : }
4891 :
4892 : /* Return true and output def_insn, val, mode, scalar_mode and kind if
4893 : INSN is a vector broadcast instruction. */
4894 :
4895 : bool
4896 51856294 : pass_x86_cse::candidate_vector_p (rtx set, rtx_insn *insn)
4897 : {
4898 51856294 : rtx src = SET_SRC (set);
4899 51856294 : rtx dest = SET_DEST (set);
4900 51856294 : mode = GET_MODE (dest);
4901 : /* Skip non-vector instruction. */
4902 51856294 : if (!VECTOR_MODE_P (mode))
4903 : return false;
4904 :
4905 : /* Skip non-vector load instruction. */
4906 3709748 : if (!REG_P (dest) && !SUBREG_P (dest))
4907 : return false;
4908 :
4909 2226410 : def_insn = insn;
4910 2226410 : val = ix86_broadcast_inner (src, mode, &scalar_mode, &kind,
4911 : &def_insn);
4912 2226410 : return val ? true : false;
4913 : }
4914 :
4915 : /* At entry of the nearest common dominator for basic blocks with
4916 :
4917 : 1. Vector CONST0_RTX patterns.
4918 : 2. Vector CONSTM1_RTX patterns.
4919 : 3. Vector broadcast patterns.
4920 : 4. UNSPEC_TLS_GD patterns.
4921 : 5. UNSPEC_TLS_LD_BASE patterns.
4922 : 6. UNSPEC_TLSDESC patterns.
4923 :
4924 : generate a single pattern whose destination is used to replace the
4925 : source in all identical patterns.
4926 :
4927 : NB: We want to generate a pattern, which is executed only once, to
4928 : cover the whole function. The LCM algorithm isn't appropriate here
4929 : since it may place a pattern inside the loop. */
4930 :
4931 : unsigned int
4932 1005363 : pass_x86_cse::x86_cse (void)
4933 : {
4934 1005363 : timevar_push (TV_MACH_DEP);
4935 :
4936 1005363 : auto_vec<redundant_pattern *> loads;
4937 1005363 : redundant_pattern *load;
4938 1005363 : basic_block bb;
4939 1005363 : rtx_insn *insn;
4940 1005363 : unsigned int i;
4941 1005363 : auto_bitmap updated_gnu_tls_insns;
4942 1005363 : auto_bitmap updated_gnu2_tls_insns;
4943 1005363 : auto_bitmap call_bbs;
4944 :
4945 1005363 : df_set_flags (DF_DEFER_INSN_RESCAN);
4946 :
4947 1005363 : bool recursive_call_p = cfun->machine->recursive_function;
4948 :
4949 11345703 : FOR_EACH_BB_FN (bb, cfun)
4950 : {
4951 140899519 : FOR_BB_INSNS (bb, insn)
4952 : {
4953 130559179 : if (!NONDEBUG_INSN_P (insn))
4954 74897365 : continue;
4955 :
4956 55661814 : bool matched = false;
4957 : /* Remove redundant patterns if there are more than 2 of
4958 : them. */
4959 55661814 : unsigned int threshold = 2;
4960 :
4961 55661814 : bool call_p = CALL_P (insn);
4962 55661814 : rtx set = single_set (insn);
4963 55661814 : if (!set && !call_p)
4964 1115094 : continue;
4965 :
4966 54546720 : tlsdesc_val = nullptr;
4967 :
4968 54546720 : attr_tls64 tls64 = get_attr_tls64 (insn);
4969 :
4970 : /* NB: TLS calls preserve all registers. */
4971 54546720 : if (call_p && tls64 == TLS64_NONE)
4972 4644481 : bitmap_set_bit (call_bbs, BLOCK_FOR_INSN (insn)->index);
4973 :
4974 54546720 : switch (tls64)
4975 : {
4976 2199 : case TLS64_GD:
4977 2199 : case TLS64_LD_BASE:
4978 : /* Verify UNSPEC_TLS_GD and UNSPEC_TLS_LD_BASE. */
4979 2199 : if (candidate_gnu_tls_p (insn, tls64))
4980 : break;
4981 168 : continue;
4982 :
4983 56 : case TLS64_CALL:
4984 56 : case TLS64_COMBINE:
4985 : /* Verify UNSPEC_TLSDESC. */
4986 56 : if (candidate_gnu2_tls_p (set, tls64))
4987 : break;
4988 2 : continue;
4989 :
4990 36 : case TLS64_LEA:
4991 : /* Skip TLS64_LEA. */
4992 36 : continue;
4993 :
4994 54544429 : case TLS64_NONE:
4995 54544429 : if (!set)
4996 2688135 : continue;
4997 :
4998 : /* Check for vector broadcast. */
4999 51856294 : if (candidate_vector_p (set, insn))
5000 : break;
5001 51599780 : continue;
5002 : }
5003 :
5004 : /* Check if there is a matching redundant load. */
5005 664201 : FOR_EACH_VEC_ELT (loads, i, load)
5006 447875 : if (load->val
5007 447875 : && load->kind == kind
5008 297389 : && load->mode == scalar_mode
5009 265194 : && (load->bb == bb
5010 203723 : || (kind != X86_CSE_VEC_DUP
5011 203723 : && kind != X86_CSE_CONST_VECTOR)
5012 : /* Non all 0s/1s vector load must be in the same
5013 : basic block if it is in a recursive call. */
5014 139669 : || !recursive_call_p)
5015 710890 : && rtx_equal_p (load->val, val))
5016 : {
5017 : /* Record instruction. */
5018 102200 : bitmap_set_bit (load->insns, INSN_UID (insn));
5019 :
5020 : /* Record the maximum vector size. */
5021 102200 : if (kind <= X86_CSE_VEC_DUP
5022 203286 : && load->size < GET_MODE_SIZE (mode))
5023 836 : load->size = GET_MODE_SIZE (mode);
5024 :
5025 : /* Record the basic block. */
5026 102200 : bitmap_set_bit (load->bbs, bb->index);
5027 :
5028 : /* Increment the count. */
5029 102200 : load->count++;
5030 :
5031 102200 : matched = true;
5032 102200 : break;
5033 : }
5034 :
5035 258599 : if (matched)
5036 102200 : continue;
5037 :
5038 : /* We see this instruction the first time. Record the
5039 : redundant source value, its mode, the destination size,
5040 : instruction which defines the redundant source value,
5041 : instruction basic block and the instruction kind. */
5042 156399 : load = new redundant_pattern;
5043 :
5044 : /* Convert CONST_VECTOR load no larger than integer register
5045 : to constant integer load even if there is no redundant
5046 : CONST_VECTOR load. */
5047 156399 : if (CONST_VECTOR_P (val))
5048 32286 : threshold = 1;
5049 :
5050 156399 : load->val = copy_rtx (val);
5051 156399 : if (tlsdesc_val)
5052 28 : load->tlsdesc_val = copy_rtx (tlsdesc_val);
5053 : else
5054 : load->tlsdesc_val = nullptr;
5055 156399 : load->mode = scalar_mode;
5056 156399 : load->dest_mode = mode;
5057 156399 : load->size = GET_MODE_SIZE (mode);
5058 156399 : load->def_insn = def_insn;
5059 156399 : load->count = 1;
5060 156399 : load->threshold = threshold;
5061 156399 : load->bb = BLOCK_FOR_INSN (insn);
5062 156399 : load->kind = kind;
5063 :
5064 156399 : bitmap_set_bit (load->insns, INSN_UID (insn));
5065 156399 : bitmap_set_bit (load->bbs, bb->index);
5066 :
5067 156399 : loads.safe_push (load);
5068 : }
5069 : }
5070 :
5071 : bool replaced = false;
5072 1258234 : FOR_EACH_VEC_ELT (loads, i, load)
5073 156399 : if (load->count >= load->threshold)
5074 : {
5075 66627 : machine_mode mode;
5076 66627 : rtx reg, broadcast_reg;
5077 66627 : rtx broadcast_source = nullptr;
5078 66627 : replaced = true;
5079 66627 : switch (load->kind)
5080 : {
5081 320 : case X86_CSE_TLS_GD:
5082 320 : case X86_CSE_TLS_LD_BASE:
5083 320 : case X86_CSE_TLSDESC:
5084 320 : broadcast_reg = gen_reg_rtx (load->mode);
5085 320 : replace_tls_call (broadcast_reg, load->insns,
5086 320 : (load->kind == X86_CSE_TLSDESC
5087 : ? updated_gnu2_tls_insns
5088 : : updated_gnu_tls_insns));
5089 320 : load->broadcast_reg = broadcast_reg;
5090 320 : break;
5091 :
5092 11414 : case X86_CSE_VEC_DUP:
5093 11414 : if (CONST_INT_P (load->val)
5094 10483 : && (load->val == CONST0_RTX (load->mode)
5095 10508 : || load->size <= UNITS_PER_WORD))
5096 : {
5097 : /* Generate CONST_VECTOR load. */
5098 32289 : case X86_CSE_CONST_VECTOR:
5099 32289 : mode = ix86_get_vector_cse_mode (load->size,
5100 : load->mode);
5101 :
5102 32289 : if (CONST_VECTOR_P (load->val))
5103 : broadcast_source = load->val;
5104 3 : else if (load->val == CONST0_RTX (load->mode))
5105 0 : broadcast_source = CONST0_RTX (mode);
5106 3 : else if (load->val == CONSTM1_RTX (load->mode))
5107 0 : broadcast_source = CONSTM1_RTX (mode);
5108 : else
5109 : {
5110 3 : int nunits = GET_MODE_NUNITS (mode);
5111 3 : rtvec v = rtvec_alloc (nunits);
5112 12 : for (int j = 0; j < nunits ; j++)
5113 6 : RTVEC_ELT (v, j) = load->val;
5114 3 : if (CONST_INT_P (load->val)
5115 3 : && GET_MODE_CLASS (mode) != MODE_VECTOR_INT)
5116 : {
5117 1 : gcc_assert (load->size <= UNITS_PER_WORD);
5118 1 : machine_mode imode = GET_MODE_INNER (mode);
5119 1 : class scalar_mode i_mode
5120 1 : = int_mode_for_mode (imode).require ();
5121 1 : mode = mode_for_vector (i_mode,
5122 1 : nunits).require ();
5123 : }
5124 :
5125 3 : broadcast_source = gen_rtx_CONST_VECTOR (mode, v);
5126 : }
5127 :
5128 : /* NB: Zero CONST_VECTOR load works for MMX and XMM
5129 : registers. */
5130 33699 : if (load->size <= UNITS_PER_WORD)
5131 : {
5132 : /* Convert CONST_VECTOR load no larger than integer
5133 : register:
5134 :
5135 : (set (reg:V2SI 106)
5136 : (const_vector:V2SI [(const_int 1 [1]) repeated x2]))
5137 :
5138 : to constant integer load:
5139 :
5140 : (set (subreg:DI (reg:V2SI 106 [ _20 ]) 0)
5141 : (const_int 4294967297 [0x100000001]))
5142 : */
5143 32289 : machine_mode int_mode
5144 32289 : = int_mode_for_mode (mode).require ();
5145 32289 : load->dest_mode = int_mode;
5146 32289 : broadcast_source = simplify_subreg (int_mode,
5147 : broadcast_source,
5148 : mode, 0);
5149 32289 : gcc_assert (broadcast_source != nullptr);
5150 :
5151 32289 : bool keep_const_int_load = false;
5152 32289 : if (!bitmap_empty_p (call_bbs))
5153 : {
5154 28881 : bitmap_iterator bi;
5155 28881 : unsigned int id;
5156 38234 : EXECUTE_IF_SET_IN_BITMAP (load->bbs, 0, id, bi)
5157 30362 : if (bitmap_bit_p (call_bbs, id))
5158 : {
5159 : /* NB: Constant integer load is faster
5160 : than save and restore an integer
5161 : register when crossing a function call.
5162 : */
5163 : keep_const_int_load = true;
5164 : break;
5165 : }
5166 : }
5167 :
5168 28881 : if (keep_const_int_load)
5169 : {
5170 : /* Keep constant integer load. */
5171 21009 : replace_vector_const (mode, broadcast_source,
5172 21009 : load->insns, int_mode);
5173 21009 : load->broadcast_source = nullptr;
5174 21009 : load->broadcast_reg = nullptr;
5175 : }
5176 : else
5177 : {
5178 11280 : broadcast_reg = gen_reg_rtx (mode);
5179 11280 : reg = gen_reg_rtx (load->mode);
5180 11280 : replace_vector_const (mode, broadcast_reg,
5181 11280 : load->insns, load->mode);
5182 11280 : load->broadcast_source = broadcast_source;
5183 11280 : load->broadcast_reg = broadcast_reg;
5184 : }
5185 : break;
5186 : }
5187 : }
5188 : /* FALLTHRU */
5189 :
5190 34018 : case X86_CSE_CONST0_VECTOR:
5191 34018 : case X86_CSE_CONSTM1_VECTOR:
5192 34018 : mode = ix86_get_vector_cse_mode (load->size, load->mode);
5193 34018 : broadcast_reg = gen_reg_rtx (mode);
5194 34018 : if (load->def_insn)
5195 : {
5196 : /* Replace redundant vector loads with a single vector
5197 : load in the same basic block. */
5198 781 : reg = load->val;
5199 781 : if (load->mode != GET_MODE (reg))
5200 0 : reg = gen_rtx_SUBREG (load->mode, reg, 0);
5201 781 : broadcast_source = gen_rtx_VEC_DUPLICATE (mode, reg);
5202 : }
5203 : else
5204 : /* This is a constant integer/double vector. If the
5205 : inner scalar is 0 or -1, set vector to CONST0_RTX
5206 : or CONSTM1_RTX directly. */
5207 33237 : switch (load->kind)
5208 : {
5209 20931 : case X86_CSE_CONST0_VECTOR:
5210 20931 : broadcast_source = CONST0_RTX (mode);
5211 20931 : break;
5212 1673 : case X86_CSE_CONSTM1_VECTOR:
5213 1673 : broadcast_source = CONSTM1_RTX (mode);
5214 1673 : break;
5215 10633 : case X86_CSE_CONST_VECTOR:
5216 10633 : case X86_CSE_VEC_DUP:
5217 10633 : if (!broadcast_source)
5218 : {
5219 10633 : reg = gen_reg_rtx (load->mode);
5220 10633 : broadcast_source = gen_rtx_VEC_DUPLICATE (mode,
5221 : reg);
5222 : }
5223 : break;
5224 0 : default:
5225 0 : gcc_unreachable ();
5226 : }
5227 34018 : replace_vector_const (mode, broadcast_reg, load->insns,
5228 : load->mode);
5229 34018 : load->broadcast_source = broadcast_source;
5230 34018 : load->broadcast_reg = broadcast_reg;
5231 34018 : break;
5232 : }
5233 : }
5234 :
5235 1005363 : if (replaced)
5236 : {
5237 43358 : auto_vec<rtx_insn *> control_flow_insns;
5238 :
5239 : /* (Re-)discover loops so that bb->loop_father can be used in the
5240 : analysis below. */
5241 43358 : calculate_dominance_info (CDI_DOMINATORS);
5242 43358 : loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
5243 :
5244 217025 : FOR_EACH_VEC_ELT (loads, i, load)
5245 86951 : if (load->count >= load->threshold)
5246 : {
5247 66627 : rtx set;
5248 66627 : if (load->def_insn)
5249 797 : switch (load->kind)
5250 : {
5251 16 : case X86_CSE_TLSDESC:
5252 16 : ix86_place_single_tls_call (load->broadcast_reg,
5253 : load->tlsdesc_val,
5254 : load->kind,
5255 16 : load->bbs,
5256 : updated_gnu_tls_insns,
5257 : updated_gnu2_tls_insns,
5258 16 : PATTERN (load->def_insn));
5259 16 : break;
5260 781 : case X86_CSE_VEC_DUP:
5261 : /* Insert a broadcast after the original scalar
5262 : definition. */
5263 781 : set = gen_rtx_SET (load->broadcast_reg,
5264 : load->broadcast_source);
5265 781 : insn = emit_insn_after (set, load->def_insn);
5266 :
5267 781 : if (cfun->can_throw_non_call_exceptions)
5268 : {
5269 : /* Handle REG_EH_REGION note in DEF_INSN. */
5270 4 : rtx note = find_reg_note (load->def_insn,
5271 : REG_EH_REGION, nullptr);
5272 4 : if (note)
5273 : {
5274 1 : control_flow_insns.safe_push (load->def_insn);
5275 1 : add_reg_note (insn, REG_EH_REGION,
5276 : XEXP (note, 0));
5277 : }
5278 : }
5279 :
5280 781 : if (dump_file)
5281 : {
5282 0 : fprintf (dump_file, "\nAdd:\n\n");
5283 0 : print_rtl_single (dump_file, insn);
5284 0 : fprintf (dump_file, "\nafter:\n\n");
5285 0 : print_rtl_single (dump_file, load->def_insn);
5286 0 : fprintf (dump_file, "\n");
5287 : }
5288 : break;
5289 0 : default:
5290 0 : gcc_unreachable ();
5291 : }
5292 : else
5293 65830 : switch (load->kind)
5294 : {
5295 304 : case X86_CSE_TLS_GD:
5296 304 : case X86_CSE_TLS_LD_BASE:
5297 304 : case X86_CSE_TLSDESC:
5298 304 : ix86_place_single_tls_call (load->broadcast_reg,
5299 : (load->kind == X86_CSE_TLSDESC
5300 : ? load->tlsdesc_val
5301 : : load->val),
5302 : load->kind,
5303 304 : load->bbs,
5304 : updated_gnu_tls_insns,
5305 : updated_gnu2_tls_insns);
5306 304 : break;
5307 42922 : case X86_CSE_CONST_VECTOR:
5308 42922 : case X86_CSE_VEC_DUP:
5309 : /* Keep redundant constant integer load. */
5310 42922 : if (!load->broadcast_reg)
5311 : break;
5312 : /* FALLTHRU */
5313 44517 : case X86_CSE_CONST0_VECTOR:
5314 44517 : case X86_CSE_CONSTM1_VECTOR:
5315 44517 : ix86_place_single_vector_set (load->broadcast_reg,
5316 : load->broadcast_source,
5317 : load->bbs,
5318 : load);
5319 44517 : break;
5320 : }
5321 : }
5322 :
5323 43358 : loop_optimizer_finalize ();
5324 :
5325 43359 : if (!control_flow_insns.is_empty ())
5326 : {
5327 1 : free_dominance_info (CDI_DOMINATORS);
5328 :
5329 3 : FOR_EACH_VEC_ELT (control_flow_insns, i, insn)
5330 1 : if (control_flow_insn_p (insn))
5331 : {
5332 : /* Split the block after insn. There will be a fallthru
5333 : edge, which is OK so we keep it. We have to create
5334 : the exception edges ourselves. */
5335 1 : bb = BLOCK_FOR_INSN (insn);
5336 1 : split_block (bb, insn);
5337 1 : rtl_make_eh_edge (NULL, bb, BB_END (bb));
5338 : }
5339 : }
5340 :
5341 43358 : df_process_deferred_rescans ();
5342 43358 : }
5343 :
5344 1258234 : FOR_EACH_VEC_ELT (loads, i, load)
5345 312798 : delete load;
5346 :
5347 1005363 : df_clear_flags (DF_DEFER_INSN_RESCAN);
5348 :
5349 1005363 : timevar_pop (TV_MACH_DEP);
5350 1005363 : return 0;
5351 1005363 : }
5352 :
5353 : } // anon namespace
5354 :
5355 : rtl_opt_pass *
5356 295807 : make_pass_x86_cse (gcc::context *ctxt)
5357 : {
5358 295807 : return new pass_x86_cse (ctxt);
5359 : }
5360 :
5361 : /* Convert legacy instructions that clobbers EFLAGS to APX_NF
5362 : instructions when there are no flag set between a flag
5363 : producer and user. */
5364 :
5365 : static unsigned int
5366 382 : ix86_apx_nf_convert (void)
5367 : {
5368 382 : timevar_push (TV_MACH_DEP);
5369 :
5370 382 : basic_block bb;
5371 382 : rtx_insn *insn;
5372 382 : hash_map <rtx_insn *, rtx> converting_map;
5373 382 : auto_vec <rtx_insn *> current_convert_list;
5374 :
5375 382 : bool converting_seq = false;
5376 382 : rtx cc = gen_rtx_REG (CCmode, FLAGS_REG);
5377 :
5378 838 : FOR_EACH_BB_FN (bb, cfun)
5379 : {
5380 : /* Reset conversion for each bb. */
5381 456 : converting_seq = false;
5382 5340 : FOR_BB_INSNS (bb, insn)
5383 : {
5384 4884 : if (!NONDEBUG_INSN_P (insn))
5385 4267 : continue;
5386 :
5387 3885 : if (recog_memoized (insn) < 0)
5388 345 : continue;
5389 :
5390 : /* Convert candidate insns after cstore, which should
5391 : satisfy the two conditions:
5392 : 1. Is not flag user or producer, only clobbers
5393 : FLAGS_REG.
5394 : 2. Have corresponding nf pattern. */
5395 :
5396 3540 : rtx pat = PATTERN (insn);
5397 :
5398 : /* Starting conversion at first cstorecc. */
5399 3540 : rtx set = NULL_RTX;
5400 3540 : if (!converting_seq
5401 2949 : && (set = single_set (insn))
5402 2873 : && ix86_comparison_operator (SET_SRC (set), VOIDmode)
5403 131 : && reg_overlap_mentioned_p (cc, SET_SRC (set))
5404 3668 : && !reg_overlap_mentioned_p (cc, SET_DEST (set)))
5405 : {
5406 128 : converting_seq = true;
5407 128 : current_convert_list.truncate (0);
5408 : }
5409 : /* Terminate at the next explicit flag set. */
5410 3412 : else if (reg_set_p (cc, pat)
5411 3412 : && GET_CODE (set_of (cc, pat)) != CLOBBER)
5412 : converting_seq = false;
5413 :
5414 3301 : if (!converting_seq)
5415 2923 : continue;
5416 :
5417 617 : if (get_attr_has_nf (insn)
5418 617 : && GET_CODE (pat) == PARALLEL)
5419 : {
5420 : /* Record the insn to candidate map. */
5421 74 : current_convert_list.safe_push (insn);
5422 74 : converting_map.put (insn, pat);
5423 : }
5424 : /* If the insn clobbers flags but has no nf_attr,
5425 : revoke all previous candidates. */
5426 543 : else if (!get_attr_has_nf (insn)
5427 542 : && reg_set_p (cc, pat)
5428 546 : && GET_CODE (set_of (cc, pat)) == CLOBBER)
5429 : {
5430 3 : for (auto item : current_convert_list)
5431 0 : converting_map.remove (item);
5432 3 : converting_seq = false;
5433 : }
5434 : }
5435 : }
5436 :
5437 382 : if (!converting_map.is_empty ())
5438 : {
5439 88 : for (auto iter = converting_map.begin ();
5440 102 : iter != converting_map.end (); ++iter)
5441 : {
5442 74 : rtx_insn *replace = (*iter).first;
5443 74 : rtx pat = (*iter).second;
5444 74 : int i, n = 0, len = XVECLEN (pat, 0);
5445 74 : rtx *new_elems = XALLOCAVEC (rtx, len);
5446 74 : rtx new_pat;
5447 222 : for (i = 0; i < len; i++)
5448 : {
5449 148 : rtx temp = XVECEXP (pat, 0, i);
5450 222 : if (! (GET_CODE (temp) == CLOBBER
5451 74 : && reg_overlap_mentioned_p (cc,
5452 74 : XEXP (temp, 0))))
5453 : {
5454 74 : new_elems[n] = temp;
5455 74 : n++;
5456 : }
5457 : }
5458 :
5459 74 : if (n == 1)
5460 74 : new_pat = new_elems[0];
5461 : else
5462 0 : new_pat =
5463 0 : gen_rtx_PARALLEL (VOIDmode,
5464 : gen_rtvec_v (n,
5465 : new_elems));
5466 :
5467 74 : PATTERN (replace) = new_pat;
5468 74 : INSN_CODE (replace) = -1;
5469 74 : recog_memoized (replace);
5470 74 : df_insn_rescan (replace);
5471 : }
5472 : }
5473 :
5474 382 : timevar_pop (TV_MACH_DEP);
5475 382 : return 0;
5476 382 : }
5477 :
5478 :
5479 : namespace {
5480 :
5481 : const pass_data pass_data_apx_nf_convert =
5482 : {
5483 : RTL_PASS, /* type */
5484 : "apx_nfcvt", /* name */
5485 : OPTGROUP_NONE, /* optinfo_flags */
5486 : TV_MACH_DEP, /* tv_id */
5487 : 0, /* properties_required */
5488 : 0, /* properties_provided */
5489 : 0, /* properties_destroyed */
5490 : 0, /* todo_flags_start */
5491 : 0, /* todo_flags_finish */
5492 : };
5493 :
5494 : class pass_apx_nf_convert : public rtl_opt_pass
5495 : {
5496 : public:
5497 295807 : pass_apx_nf_convert (gcc::context *ctxt)
5498 591614 : : rtl_opt_pass (pass_data_apx_nf_convert, ctxt)
5499 : {}
5500 :
5501 : /* opt_pass methods: */
5502 1519489 : bool gate (function *) final override
5503 : {
5504 1519489 : return (TARGET_APX_NF
5505 477 : && optimize
5506 1519955 : && optimize_function_for_speed_p (cfun));
5507 : }
5508 :
5509 382 : unsigned int execute (function *) final override
5510 : {
5511 382 : return ix86_apx_nf_convert ();
5512 : }
5513 : }; // class pass_apx_nf_convert
5514 :
5515 : } // anon namespace
5516 :
5517 : rtl_opt_pass *
5518 295807 : make_pass_apx_nf_convert (gcc::context *ctxt)
5519 : {
5520 295807 : return new pass_apx_nf_convert (ctxt);
5521 : }
5522 :
5523 : /* When a hot loop can be fit into one cacheline,
5524 : force align the loop without considering the max skip. */
5525 : static void
5526 1004877 : ix86_align_loops ()
5527 : {
5528 1004877 : basic_block bb;
5529 :
5530 : /* Don't do this when we don't know cache line size. */
5531 1004877 : if (ix86_cost->prefetch_block == 0)
5532 9 : return;
5533 :
5534 1004868 : loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
5535 1004868 : profile_count count_threshold = cfun->cfg->count_max / param_align_threshold;
5536 11791944 : FOR_EACH_BB_FN (bb, cfun)
5537 : {
5538 10787076 : rtx_insn *label = BB_HEAD (bb);
5539 10787076 : bool has_fallthru = 0;
5540 10787076 : edge e;
5541 10787076 : edge_iterator ei;
5542 :
5543 10787076 : if (!LABEL_P (label))
5544 5491873 : continue;
5545 :
5546 5300039 : profile_count fallthru_count = profile_count::zero ();
5547 5300039 : profile_count branch_count = profile_count::zero ();
5548 :
5549 15414272 : FOR_EACH_EDGE (e, ei, bb->preds)
5550 : {
5551 10114233 : if (e->flags & EDGE_FALLTHRU)
5552 2561158 : has_fallthru = 1, fallthru_count += e->count ();
5553 : else
5554 7553075 : branch_count += e->count ();
5555 : }
5556 :
5557 5300039 : if (!fallthru_count.initialized_p () || !branch_count.initialized_p ())
5558 4836 : continue;
5559 :
5560 5295203 : if (bb->loop_father
5561 5295203 : && bb->loop_father->latch != EXIT_BLOCK_PTR_FOR_FN (cfun)
5562 6689811 : && (has_fallthru
5563 1394608 : ? (!(single_succ_p (bb)
5564 153057 : && single_succ (bb) == EXIT_BLOCK_PTR_FOR_FN (cfun))
5565 956254 : && optimize_bb_for_speed_p (bb)
5566 874243 : && branch_count + fallthru_count > count_threshold
5567 749490 : && (branch_count > fallthru_count * param_align_loop_iterations))
5568 : /* In case there'no fallthru for the loop.
5569 : Nops inserted won't be executed. */
5570 438354 : : (branch_count > count_threshold
5571 143678 : || (bb->count > bb->prev_bb->count * 10
5572 12928 : && (bb->prev_bb->count
5573 5308131 : <= ENTRY_BLOCK_PTR_FOR_FN (cfun)->count / 2)))))
5574 : {
5575 572963 : rtx_insn* insn, *end_insn;
5576 572963 : HOST_WIDE_INT size = 0;
5577 572963 : bool padding_p = true;
5578 572963 : basic_block tbb = bb;
5579 572963 : unsigned cond_branch_num = 0;
5580 572963 : bool detect_tight_loop_p = false;
5581 :
5582 897915 : for (unsigned int i = 0; i != bb->loop_father->num_nodes;
5583 324952 : i++, tbb = tbb->next_bb)
5584 : {
5585 : /* Only handle continuous cfg layout. */
5586 897915 : if (bb->loop_father != tbb->loop_father)
5587 : {
5588 : padding_p = false;
5589 : break;
5590 : }
5591 :
5592 10962800 : FOR_BB_INSNS (tbb, insn)
5593 : {
5594 10274752 : if (!NONDEBUG_INSN_P (insn))
5595 6051261 : continue;
5596 4223491 : size += ix86_min_insn_size (insn);
5597 :
5598 : /* We don't know size of inline asm.
5599 : Don't align loop for call. */
5600 4223491 : if (asm_noperands (PATTERN (insn)) >= 0
5601 4223491 : || CALL_P (insn))
5602 : {
5603 : size = -1;
5604 : break;
5605 : }
5606 : }
5607 :
5608 850260 : if (size == -1 || size > ix86_cost->prefetch_block)
5609 : {
5610 : padding_p = false;
5611 : break;
5612 : }
5613 :
5614 1521234 : FOR_EACH_EDGE (e, ei, tbb->succs)
5615 : {
5616 : /* It could be part of the loop. */
5617 1048726 : if (e->dest == bb)
5618 : {
5619 : detect_tight_loop_p = true;
5620 : break;
5621 : }
5622 : }
5623 :
5624 661931 : if (detect_tight_loop_p)
5625 : break;
5626 :
5627 472508 : end_insn = BB_END (tbb);
5628 472508 : if (JUMP_P (end_insn))
5629 : {
5630 : /* For decoded icache:
5631 : 1. Up to two branches are allowed per Way.
5632 : 2. A non-conditional branch is the last micro-op in a Way.
5633 : */
5634 383164 : if (onlyjump_p (end_insn)
5635 383164 : && (any_uncondjump_p (end_insn)
5636 317548 : || single_succ_p (tbb)))
5637 : {
5638 : padding_p = false;
5639 : break;
5640 : }
5641 317548 : else if (++cond_branch_num >= 2)
5642 : {
5643 : padding_p = false;
5644 : break;
5645 : }
5646 : }
5647 :
5648 : }
5649 :
5650 572963 : if (padding_p && detect_tight_loop_p)
5651 : {
5652 378846 : emit_insn_before (gen_max_skip_align (GEN_INT (ceil_log2 (size)),
5653 : GEN_INT (0)), label);
5654 : /* End of function. */
5655 189423 : if (!tbb || tbb == EXIT_BLOCK_PTR_FOR_FN (cfun))
5656 : break;
5657 : /* Skip bb which already fits into one cacheline. */
5658 : bb = tbb;
5659 : }
5660 : }
5661 : }
5662 :
5663 1004868 : loop_optimizer_finalize ();
5664 1004868 : free_dominance_info (CDI_DOMINATORS);
5665 : }
5666 :
5667 : namespace {
5668 :
5669 : const pass_data pass_data_align_tight_loops =
5670 : {
5671 : RTL_PASS, /* type */
5672 : "align_tight_loops", /* name */
5673 : OPTGROUP_NONE, /* optinfo_flags */
5674 : TV_MACH_DEP, /* tv_id */
5675 : 0, /* properties_required */
5676 : 0, /* properties_provided */
5677 : 0, /* properties_destroyed */
5678 : 0, /* todo_flags_start */
5679 : 0, /* todo_flags_finish */
5680 : };
5681 :
5682 : class pass_align_tight_loops : public rtl_opt_pass
5683 : {
5684 : public:
5685 295807 : pass_align_tight_loops (gcc::context *ctxt)
5686 591614 : : rtl_opt_pass (pass_data_align_tight_loops, ctxt)
5687 : {}
5688 :
5689 : /* opt_pass methods: */
5690 1519489 : bool gate (function *) final override
5691 : {
5692 1519489 : return TARGET_ALIGN_TIGHT_LOOPS
5693 1518996 : && optimize
5694 2590530 : && optimize_function_for_speed_p (cfun);
5695 : }
5696 :
5697 1004877 : unsigned int execute (function *) final override
5698 : {
5699 1004877 : timevar_push (TV_MACH_DEP);
5700 : #ifdef ASM_OUTPUT_MAX_SKIP_ALIGN
5701 1004877 : ix86_align_loops ();
5702 : #endif
5703 1004877 : timevar_pop (TV_MACH_DEP);
5704 1004877 : return 0;
5705 : }
5706 : }; // class pass_align_tight_loops
5707 :
5708 : } // anon namespace
5709 :
5710 : rtl_opt_pass *
5711 295807 : make_pass_align_tight_loops (gcc::context *ctxt)
5712 : {
5713 295807 : return new pass_align_tight_loops (ctxt);
5714 : }
5715 :
5716 : /* This compares the priority of target features in function DECL1
5717 : and DECL2. It returns positive value if DECL1 is higher priority,
5718 : negative value if DECL2 is higher priority and 0 if they are the
5719 : same. */
5720 :
5721 : int
5722 5830 : ix86_compare_version_priority (tree decl1, tree decl2)
5723 : {
5724 5830 : unsigned int priority1 = get_builtin_code_for_version (decl1, NULL);
5725 5830 : unsigned int priority2 = get_builtin_code_for_version (decl2, NULL);
5726 :
5727 5830 : return (int)priority1 - (int)priority2;
5728 : }
5729 :
5730 : /* V1 and V2 point to function versions with different priorities
5731 : based on the target ISA. This function compares their priorities. */
5732 :
5733 : static int
5734 6882 : feature_compare (const void *v1, const void *v2)
5735 : {
5736 6882 : typedef struct _function_version_info
5737 : {
5738 : tree version_decl;
5739 : tree predicate_chain;
5740 : unsigned int dispatch_priority;
5741 : } function_version_info;
5742 :
5743 6882 : const function_version_info c1 = *(const function_version_info *)v1;
5744 6882 : const function_version_info c2 = *(const function_version_info *)v2;
5745 6882 : return (c2.dispatch_priority - c1.dispatch_priority);
5746 : }
5747 :
5748 : /* This adds a condition to the basic_block NEW_BB in function FUNCTION_DECL
5749 : to return a pointer to VERSION_DECL if the outcome of the expression
5750 : formed by PREDICATE_CHAIN is true. This function will be called during
5751 : version dispatch to decide which function version to execute. It returns
5752 : the basic block at the end, to which more conditions can be added. */
5753 :
5754 : static basic_block
5755 848 : add_condition_to_bb (tree function_decl, tree version_decl,
5756 : tree predicate_chain, basic_block new_bb)
5757 : {
5758 848 : gimple *return_stmt;
5759 848 : tree convert_expr, result_var;
5760 848 : gimple *convert_stmt;
5761 848 : gimple *call_cond_stmt;
5762 848 : gimple *if_else_stmt;
5763 :
5764 848 : basic_block bb1, bb2, bb3;
5765 848 : edge e12, e23;
5766 :
5767 848 : tree cond_var, and_expr_var = NULL_TREE;
5768 848 : gimple_seq gseq;
5769 :
5770 848 : tree predicate_decl, predicate_arg;
5771 :
5772 848 : push_cfun (DECL_STRUCT_FUNCTION (function_decl));
5773 :
5774 848 : gcc_assert (new_bb != NULL);
5775 848 : gseq = bb_seq (new_bb);
5776 :
5777 :
5778 848 : convert_expr = build1 (CONVERT_EXPR, ptr_type_node,
5779 : build_fold_addr_expr (version_decl));
5780 848 : result_var = create_tmp_var (ptr_type_node);
5781 848 : convert_stmt = gimple_build_assign (result_var, convert_expr);
5782 848 : return_stmt = gimple_build_return (result_var);
5783 :
5784 848 : if (predicate_chain == NULL_TREE)
5785 : {
5786 204 : gimple_seq_add_stmt (&gseq, convert_stmt);
5787 204 : gimple_seq_add_stmt (&gseq, return_stmt);
5788 204 : set_bb_seq (new_bb, gseq);
5789 204 : gimple_set_bb (convert_stmt, new_bb);
5790 204 : gimple_set_bb (return_stmt, new_bb);
5791 204 : pop_cfun ();
5792 204 : return new_bb;
5793 : }
5794 :
5795 1327 : while (predicate_chain != NULL)
5796 : {
5797 683 : cond_var = create_tmp_var (integer_type_node);
5798 683 : predicate_decl = TREE_PURPOSE (predicate_chain);
5799 683 : predicate_arg = TREE_VALUE (predicate_chain);
5800 683 : call_cond_stmt = gimple_build_call (predicate_decl, 1, predicate_arg);
5801 683 : gimple_call_set_lhs (call_cond_stmt, cond_var);
5802 :
5803 683 : gimple_set_block (call_cond_stmt, DECL_INITIAL (function_decl));
5804 683 : gimple_set_bb (call_cond_stmt, new_bb);
5805 683 : gimple_seq_add_stmt (&gseq, call_cond_stmt);
5806 :
5807 683 : predicate_chain = TREE_CHAIN (predicate_chain);
5808 :
5809 683 : if (and_expr_var == NULL)
5810 : and_expr_var = cond_var;
5811 : else
5812 : {
5813 39 : gimple *assign_stmt;
5814 : /* Use MIN_EXPR to check if any integer is zero?.
5815 : and_expr_var = min_expr <cond_var, and_expr_var> */
5816 39 : assign_stmt = gimple_build_assign (and_expr_var,
5817 : build2 (MIN_EXPR, integer_type_node,
5818 : cond_var, and_expr_var));
5819 :
5820 39 : gimple_set_block (assign_stmt, DECL_INITIAL (function_decl));
5821 39 : gimple_set_bb (assign_stmt, new_bb);
5822 39 : gimple_seq_add_stmt (&gseq, assign_stmt);
5823 : }
5824 : }
5825 :
5826 644 : if_else_stmt = gimple_build_cond (GT_EXPR, and_expr_var,
5827 : integer_zero_node,
5828 : NULL_TREE, NULL_TREE);
5829 644 : gimple_set_block (if_else_stmt, DECL_INITIAL (function_decl));
5830 644 : gimple_set_bb (if_else_stmt, new_bb);
5831 644 : gimple_seq_add_stmt (&gseq, if_else_stmt);
5832 :
5833 644 : gimple_seq_add_stmt (&gseq, convert_stmt);
5834 644 : gimple_seq_add_stmt (&gseq, return_stmt);
5835 644 : set_bb_seq (new_bb, gseq);
5836 :
5837 644 : bb1 = new_bb;
5838 644 : e12 = split_block (bb1, if_else_stmt);
5839 644 : bb2 = e12->dest;
5840 644 : e12->flags &= ~EDGE_FALLTHRU;
5841 644 : e12->flags |= EDGE_TRUE_VALUE;
5842 :
5843 644 : e23 = split_block (bb2, return_stmt);
5844 :
5845 644 : gimple_set_bb (convert_stmt, bb2);
5846 644 : gimple_set_bb (return_stmt, bb2);
5847 :
5848 644 : bb3 = e23->dest;
5849 644 : make_edge (bb1, bb3, EDGE_FALSE_VALUE);
5850 :
5851 644 : remove_edge (e23);
5852 644 : make_edge (bb2, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
5853 :
5854 644 : pop_cfun ();
5855 :
5856 644 : return bb3;
5857 : }
5858 :
5859 : /* This function generates the dispatch function for
5860 : multi-versioned functions. DISPATCH_DECL is the function which will
5861 : contain the dispatch logic. FNDECLS are the function choices for
5862 : dispatch, and is a tree chain. EMPTY_BB is the basic block pointer
5863 : in DISPATCH_DECL in which the dispatch code is generated. */
5864 :
5865 : static int
5866 204 : dispatch_function_versions (tree dispatch_decl,
5867 : void *fndecls_p,
5868 : basic_block *empty_bb)
5869 : {
5870 204 : tree default_decl;
5871 204 : gimple *ifunc_cpu_init_stmt;
5872 204 : gimple_seq gseq;
5873 204 : int ix;
5874 204 : tree ele;
5875 204 : vec<tree> *fndecls;
5876 204 : unsigned int num_versions = 0;
5877 204 : unsigned int actual_versions = 0;
5878 204 : unsigned int i;
5879 :
5880 204 : struct _function_version_info
5881 : {
5882 : tree version_decl;
5883 : tree predicate_chain;
5884 : unsigned int dispatch_priority;
5885 : }*function_version_info;
5886 :
5887 204 : gcc_assert (dispatch_decl != NULL
5888 : && fndecls_p != NULL
5889 : && empty_bb != NULL);
5890 :
5891 : /*fndecls_p is actually a vector. */
5892 204 : fndecls = static_cast<vec<tree> *> (fndecls_p);
5893 :
5894 : /* At least one more version other than the default. */
5895 204 : num_versions = fndecls->length ();
5896 204 : gcc_assert (num_versions >= 2);
5897 :
5898 204 : function_version_info = (struct _function_version_info *)
5899 204 : XNEWVEC (struct _function_version_info, (num_versions - 1));
5900 :
5901 : /* The first version in the vector is the default decl. */
5902 204 : default_decl = (*fndecls)[0];
5903 :
5904 204 : push_cfun (DECL_STRUCT_FUNCTION (dispatch_decl));
5905 :
5906 204 : gseq = bb_seq (*empty_bb);
5907 : /* Function version dispatch is via IFUNC. IFUNC resolvers fire before
5908 : constructors, so explicitly call __builtin_cpu_init here. */
5909 204 : ifunc_cpu_init_stmt
5910 204 : = gimple_build_call_vec (get_ix86_builtin (IX86_BUILTIN_CPU_INIT), vNULL);
5911 204 : gimple_seq_add_stmt (&gseq, ifunc_cpu_init_stmt);
5912 204 : gimple_set_bb (ifunc_cpu_init_stmt, *empty_bb);
5913 204 : set_bb_seq (*empty_bb, gseq);
5914 :
5915 204 : pop_cfun ();
5916 :
5917 :
5918 1209 : for (ix = 1; fndecls->iterate (ix, &ele); ++ix)
5919 : {
5920 801 : tree version_decl = ele;
5921 801 : tree predicate_chain = NULL_TREE;
5922 801 : unsigned int priority;
5923 : /* Get attribute string, parse it and find the right predicate decl.
5924 : The predicate function could be a lengthy combination of many
5925 : features, like arch-type and various isa-variants. */
5926 801 : priority = get_builtin_code_for_version (version_decl,
5927 : &predicate_chain);
5928 :
5929 801 : if (predicate_chain == NULL_TREE)
5930 157 : continue;
5931 :
5932 644 : function_version_info [actual_versions].version_decl = version_decl;
5933 644 : function_version_info [actual_versions].predicate_chain
5934 644 : = predicate_chain;
5935 644 : function_version_info [actual_versions].dispatch_priority = priority;
5936 644 : actual_versions++;
5937 : }
5938 :
5939 : /* Sort the versions according to descending order of dispatch priority. The
5940 : priority is based on the ISA. This is not a perfect solution. There
5941 : could still be ambiguity. If more than one function version is suitable
5942 : to execute, which one should be dispatched? In future, allow the user
5943 : to specify a dispatch priority next to the version. */
5944 204 : qsort (function_version_info, actual_versions,
5945 : sizeof (struct _function_version_info), feature_compare);
5946 :
5947 1052 : for (i = 0; i < actual_versions; ++i)
5948 644 : *empty_bb = add_condition_to_bb (dispatch_decl,
5949 : function_version_info[i].version_decl,
5950 644 : function_version_info[i].predicate_chain,
5951 : *empty_bb);
5952 :
5953 : /* dispatch default version at the end. */
5954 204 : *empty_bb = add_condition_to_bb (dispatch_decl, default_decl,
5955 : NULL, *empty_bb);
5956 :
5957 204 : free (function_version_info);
5958 204 : return 0;
5959 : }
5960 :
5961 : /* This function changes the assembler name for functions that are
5962 : versions. If DECL is a function version and has a "target"
5963 : attribute, it appends the attribute string to its assembler name. */
5964 :
5965 : static tree
5966 1127 : ix86_mangle_function_version_assembler_name (tree decl, tree id)
5967 : {
5968 1127 : tree version_attr;
5969 1127 : char *attr_str;
5970 :
5971 1127 : if (DECL_DECLARED_INLINE_P (decl)
5972 1176 : && lookup_attribute ("gnu_inline",
5973 49 : DECL_ATTRIBUTES (decl)))
5974 0 : error_at (DECL_SOURCE_LOCATION (decl),
5975 : "function versions cannot be marked as %<gnu_inline%>,"
5976 : " bodies have to be generated");
5977 :
5978 1127 : if (DECL_VIRTUAL_P (decl)
5979 2254 : || DECL_VINDEX (decl))
5980 0 : sorry ("virtual function multiversioning not supported");
5981 :
5982 1127 : version_attr = lookup_attribute ("target", DECL_ATTRIBUTES (decl));
5983 :
5984 : /* target attribute string cannot be NULL. */
5985 1127 : gcc_assert (version_attr != NULL_TREE);
5986 :
5987 1127 : attr_str = sorted_attr_string (TREE_VALUE (version_attr));
5988 :
5989 : /* Allow assembler name to be modified if already set. */
5990 1127 : if (DECL_ASSEMBLER_NAME_SET_P (decl))
5991 1112 : SET_DECL_RTL (decl, NULL);
5992 :
5993 1127 : tree ret = clone_identifier (id, attr_str, true);
5994 :
5995 1127 : XDELETEVEC (attr_str);
5996 :
5997 1127 : return ret;
5998 : }
5999 :
6000 : tree
6001 508630546 : ix86_mangle_decl_assembler_name (tree decl, tree id)
6002 : {
6003 : /* For function version, add the target suffix to the assembler name. */
6004 508630546 : if (TREE_CODE (decl) == FUNCTION_DECL)
6005 : {
6006 471756534 : cgraph_node *node = cgraph_node::get (decl);
6007 : /* Mangle all versions when annotated with target_clones, but only
6008 : non-default versions when annotated with target attributes. */
6009 471756534 : if (DECL_FUNCTION_VERSIONED (decl)
6010 471756534 : && (node->is_target_clone
6011 1092 : || !is_function_default_version (node->decl)))
6012 1127 : id = ix86_mangle_function_version_assembler_name (decl, id);
6013 : /* Mangle the dispatched symbol but only in the case of target clones. */
6014 471755407 : else if (node && node->dispatcher_function && !node->is_target_clone)
6015 117 : id = clone_identifier (id, "ifunc");
6016 68358521 : else if (node && node->dispatcher_resolver_function)
6017 204 : id = clone_identifier (id, "resolver");
6018 : }
6019 : #ifdef SUBTARGET_MANGLE_DECL_ASSEMBLER_NAME
6020 : id = SUBTARGET_MANGLE_DECL_ASSEMBLER_NAME (decl, id);
6021 : #endif
6022 :
6023 508630546 : return id;
6024 : }
6025 :
6026 : /* Make a dispatcher declaration for the multi-versioned function DECL.
6027 : Calls to DECL function will be replaced with calls to the dispatcher
6028 : by the front-end. Returns the decl of the dispatcher function. */
6029 :
6030 : tree
6031 330 : ix86_get_function_versions_dispatcher (void *decl)
6032 : {
6033 330 : tree fn = (tree) decl;
6034 330 : struct cgraph_node *node = NULL;
6035 330 : struct cgraph_node *default_node = NULL;
6036 330 : struct cgraph_function_version_info *node_v = NULL;
6037 :
6038 330 : tree dispatch_decl = NULL;
6039 :
6040 330 : struct cgraph_function_version_info *default_version_info = NULL;
6041 :
6042 660 : gcc_assert (fn != NULL && DECL_FUNCTION_VERSIONED (fn));
6043 :
6044 330 : node = cgraph_node::get (fn);
6045 330 : gcc_assert (node != NULL);
6046 :
6047 330 : node_v = node->function_version ();
6048 330 : gcc_assert (node_v != NULL);
6049 :
6050 330 : if (node_v->dispatcher_resolver != NULL)
6051 : return node_v->dispatcher_resolver;
6052 :
6053 : /* The default node is always the beginning of the chain. */
6054 : default_version_info = node_v;
6055 681 : while (default_version_info->prev != NULL)
6056 : default_version_info = default_version_info->prev;
6057 216 : default_node = default_version_info->this_node;
6058 :
6059 : /* If there is no default node, just return NULL. */
6060 216 : if (!is_function_default_version (default_node->decl))
6061 : return NULL;
6062 :
6063 : #if defined (ASM_OUTPUT_TYPE_DIRECTIVE)
6064 207 : if (targetm.has_ifunc_p ())
6065 : {
6066 207 : struct cgraph_function_version_info *it_v = NULL;
6067 :
6068 : /* Right now, the dispatching is done via ifunc. */
6069 207 : dispatch_decl = make_dispatcher_decl (default_node->decl);
6070 :
6071 : /* Set the dispatcher for all the versions. */
6072 207 : it_v = default_version_info;
6073 1425 : while (it_v != NULL)
6074 : {
6075 1011 : it_v->dispatcher_resolver = dispatch_decl;
6076 1011 : it_v = it_v->next;
6077 : }
6078 : }
6079 : else
6080 : #endif
6081 : {
6082 0 : error_at (DECL_SOURCE_LOCATION (default_node->decl),
6083 : "multiversioning needs %<ifunc%> which is not supported "
6084 : "on this target");
6085 : }
6086 :
6087 : return dispatch_decl;
6088 : }
6089 :
6090 : /* Make the resolver function decl to dispatch the versions of
6091 : a multi-versioned function, DEFAULT_DECL. IFUNC_ALIAS_DECL is
6092 : ifunc alias that will point to the created resolver. Create an
6093 : empty basic block in the resolver and store the pointer in
6094 : EMPTY_BB. Return the decl of the resolver function. */
6095 :
6096 : static tree
6097 204 : make_resolver_func (const tree default_decl,
6098 : const tree ifunc_alias_decl,
6099 : basic_block *empty_bb)
6100 : {
6101 204 : tree decl, type, t;
6102 :
6103 : /* The resolver function should return a (void *). */
6104 204 : type = build_function_type_list (ptr_type_node, NULL_TREE);
6105 :
6106 204 : cgraph_node *node = cgraph_node::get (default_decl);
6107 204 : gcc_assert (node && node->function_version ());
6108 :
6109 204 : decl = build_fn_decl (IDENTIFIER_POINTER (DECL_NAME (default_decl)), type);
6110 :
6111 : /* Set the assembler name to prevent cgraph_node attempting to mangle. */
6112 204 : SET_DECL_ASSEMBLER_NAME (decl, DECL_ASSEMBLER_NAME (default_decl));
6113 :
6114 204 : cgraph_node *resolver_node = cgraph_node::get_create (decl);
6115 204 : resolver_node->dispatcher_resolver_function = true;
6116 :
6117 204 : if (node->is_target_clone)
6118 90 : resolver_node->is_target_clone = true;
6119 :
6120 204 : tree id = ix86_mangle_decl_assembler_name
6121 204 : (decl, node->function_version ()->assembler_name);
6122 204 : symtab->change_decl_assembler_name (decl, id);
6123 :
6124 204 : DECL_NAME (decl) = DECL_NAME (default_decl);
6125 204 : TREE_USED (decl) = 1;
6126 204 : DECL_ARTIFICIAL (decl) = 1;
6127 204 : DECL_IGNORED_P (decl) = 1;
6128 204 : TREE_PUBLIC (decl) = 0;
6129 204 : DECL_UNINLINABLE (decl) = 1;
6130 :
6131 : /* Resolver is not external, body is generated. */
6132 204 : DECL_EXTERNAL (decl) = 0;
6133 204 : DECL_EXTERNAL (ifunc_alias_decl) = 0;
6134 :
6135 204 : DECL_CONTEXT (decl) = NULL_TREE;
6136 204 : DECL_INITIAL (decl) = make_node (BLOCK);
6137 204 : DECL_STATIC_CONSTRUCTOR (decl) = 0;
6138 :
6139 204 : if (DECL_COMDAT_GROUP (default_decl)
6140 204 : || TREE_PUBLIC (default_decl))
6141 : {
6142 : /* In this case, each translation unit with a call to this
6143 : versioned function will put out a resolver. Ensure it
6144 : is comdat to keep just one copy. */
6145 179 : DECL_COMDAT (decl) = 1;
6146 179 : make_decl_one_only (decl, DECL_ASSEMBLER_NAME (decl));
6147 : }
6148 : else
6149 25 : TREE_PUBLIC (ifunc_alias_decl) = 0;
6150 :
6151 : /* Build result decl and add to function_decl. */
6152 204 : t = build_decl (UNKNOWN_LOCATION, RESULT_DECL, NULL_TREE, ptr_type_node);
6153 204 : DECL_CONTEXT (t) = decl;
6154 204 : DECL_ARTIFICIAL (t) = 1;
6155 204 : DECL_IGNORED_P (t) = 1;
6156 204 : DECL_RESULT (decl) = t;
6157 :
6158 204 : gimplify_function_tree (decl);
6159 204 : push_cfun (DECL_STRUCT_FUNCTION (decl));
6160 204 : *empty_bb = init_lowered_empty_function (decl, false,
6161 : profile_count::uninitialized ());
6162 :
6163 204 : cgraph_node::add_new_function (decl, true);
6164 204 : symtab->call_cgraph_insertion_hooks (cgraph_node::get_create (decl));
6165 :
6166 204 : pop_cfun ();
6167 :
6168 204 : gcc_assert (ifunc_alias_decl != NULL);
6169 : /* Mark ifunc_alias_decl as "ifunc" with resolver as resolver_name. */
6170 204 : DECL_ATTRIBUTES (ifunc_alias_decl)
6171 204 : = make_attribute ("ifunc", IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)),
6172 204 : DECL_ATTRIBUTES (ifunc_alias_decl));
6173 :
6174 : /* Create the alias for dispatch to resolver here. */
6175 204 : cgraph_node::create_same_body_alias (ifunc_alias_decl, decl);
6176 204 : return decl;
6177 : }
6178 :
6179 : /* Generate the dispatching code body to dispatch multi-versioned function
6180 : DECL. The target hook is called to process the "target" attributes and
6181 : provide the code to dispatch the right function at run-time. NODE points
6182 : to the dispatcher decl whose body will be created. */
6183 :
6184 : tree
6185 204 : ix86_generate_version_dispatcher_body (void *node_p)
6186 : {
6187 204 : tree resolver_decl;
6188 204 : basic_block empty_bb;
6189 204 : tree default_ver_decl;
6190 204 : struct cgraph_node *versn;
6191 204 : struct cgraph_node *node;
6192 :
6193 204 : struct cgraph_function_version_info *node_version_info = NULL;
6194 204 : struct cgraph_function_version_info *versn_info = NULL;
6195 :
6196 204 : node = (cgraph_node *)node_p;
6197 :
6198 204 : node_version_info = node->function_version ();
6199 204 : gcc_assert (node->dispatcher_function
6200 : && node_version_info != NULL);
6201 :
6202 204 : if (node_version_info->dispatcher_resolver)
6203 : return node_version_info->dispatcher_resolver;
6204 :
6205 : /* The first version in the chain corresponds to the default version. */
6206 204 : default_ver_decl = node_version_info->next->this_node->decl;
6207 :
6208 : /* node is going to be an alias, so remove the finalized bit. */
6209 204 : node->definition = false;
6210 :
6211 204 : resolver_decl = make_resolver_func (default_ver_decl,
6212 : node->decl, &empty_bb);
6213 :
6214 204 : node_version_info->dispatcher_resolver = resolver_decl;
6215 :
6216 204 : push_cfun (DECL_STRUCT_FUNCTION (resolver_decl));
6217 :
6218 204 : auto_vec<tree, 2> fn_ver_vec;
6219 :
6220 1209 : for (versn_info = node_version_info->next; versn_info;
6221 1005 : versn_info = versn_info->next)
6222 : {
6223 1005 : versn = versn_info->this_node;
6224 : /* Check for virtual functions here again, as by this time it should
6225 : have been determined if this function needs a vtable index or
6226 : not. This happens for methods in derived classes that override
6227 : virtual methods in base classes but are not explicitly marked as
6228 : virtual. */
6229 1005 : if (DECL_VIRTUAL_P (versn->decl))
6230 0 : sorry ("virtual function multiversioning not supported");
6231 :
6232 1005 : fn_ver_vec.safe_push (versn->decl);
6233 : }
6234 :
6235 204 : dispatch_function_versions (resolver_decl, &fn_ver_vec, &empty_bb);
6236 204 : cgraph_edge::rebuild_edges ();
6237 204 : pop_cfun ();
6238 204 : return resolver_decl;
6239 204 : }
6240 :
6241 :
|
