Line data Source code
1 : /* Subroutines used for code generation on IA-32.
2 : Copyright (C) 1988-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify
7 : it under the terms of the GNU General Public License as published by
8 : the Free Software Foundation; either version 3, or (at your option)
9 : any later version.
10 :
11 : GCC is distributed in the hope that it will be useful,
12 : but WITHOUT ANY WARRANTY; without even the implied warranty of
13 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 : GNU General Public License for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #define INCLUDE_STRING
21 : #define IN_TARGET_CODE 1
22 :
23 : #include "config.h"
24 : #include "system.h"
25 : #include "coretypes.h"
26 : #include "backend.h"
27 : #include "rtl.h"
28 : #include "tree.h"
29 : #include "memmodel.h"
30 : #include "gimple.h"
31 : #include "cfghooks.h"
32 : #include "cfgloop.h"
33 : #include "df.h"
34 : #include "tm_p.h"
35 : #include "stringpool.h"
36 : #include "expmed.h"
37 : #include "optabs.h"
38 : #include "regs.h"
39 : #include "emit-rtl.h"
40 : #include "recog.h"
41 : #include "cgraph.h"
42 : #include "diagnostic.h"
43 : #include "cfgbuild.h"
44 : #include "alias.h"
45 : #include "fold-const.h"
46 : #include "attribs.h"
47 : #include "calls.h"
48 : #include "stor-layout.h"
49 : #include "varasm.h"
50 : #include "output.h"
51 : #include "insn-attr.h"
52 : #include "flags.h"
53 : #include "except.h"
54 : #include "explow.h"
55 : #include "expr.h"
56 : #include "cfgrtl.h"
57 : #include "common/common-target.h"
58 : #include "langhooks.h"
59 : #include "reload.h"
60 : #include "gimplify.h"
61 : #include "dwarf2.h"
62 : #include "tm-constrs.h"
63 : #include "cselib.h"
64 : #include "sched-int.h"
65 : #include "opts.h"
66 : #include "tree-pass.h"
67 : #include "context.h"
68 : #include "pass_manager.h"
69 : #include "target-globals.h"
70 : #include "gimple-iterator.h"
71 : #include "gimple-fold.h"
72 : #include "tree-vectorizer.h"
73 : #include "shrink-wrap.h"
74 : #include "builtins.h"
75 : #include "rtl-iter.h"
76 : #include "tree-iterator.h"
77 : #include "dbgcnt.h"
78 : #include "case-cfn-macros.h"
79 : #include "dojump.h"
80 : #include "fold-const-call.h"
81 : #include "tree-vrp.h"
82 : #include "tree-ssanames.h"
83 : #include "selftest.h"
84 : #include "selftest-rtl.h"
85 : #include "print-rtl.h"
86 : #include "intl.h"
87 : #include "ifcvt.h"
88 : #include "symbol-summary.h"
89 : #include "sreal.h"
90 : #include "ipa-cp.h"
91 : #include "ipa-prop.h"
92 : #include "ipa-fnsummary.h"
93 : #include "wide-int-bitmask.h"
94 : #include "tree-vector-builder.h"
95 : #include "debug.h"
96 : #include "dwarf2out.h"
97 : #include "i386-options.h"
98 : #include "i386-builtins.h"
99 : #include "i386-expand.h"
100 : #include "i386-features.h"
101 : #include "function-abi.h"
102 : #include "rtl-error.h"
103 : #include "gimple-pretty-print.h"
104 :
105 : /* This file should be included last. */
106 : #include "target-def.h"
107 :
108 : static void ix86_print_operand_address_as (FILE *, rtx, addr_space_t, bool);
109 : static void ix86_emit_restore_reg_using_pop (rtx, bool = false);
110 : static const predefined_function_abi & ix86_alternate_abi (void);
111 :
112 :
113 : #ifndef CHECK_STACK_LIMIT
114 : #define CHECK_STACK_LIMIT (-1)
115 : #endif
116 :
117 : /* Return index of given mode in mult and division cost tables. */
118 : #define MODE_INDEX(mode) \
119 : ((mode) == QImode ? 0 \
120 : : (mode) == HImode ? 1 \
121 : : (mode) == SImode ? 2 \
122 : : (mode) == DImode ? 3 \
123 : : 4)
124 :
125 :
126 : /* Set by -mtune. */
127 : const struct processor_costs *ix86_tune_cost = NULL;
128 :
129 : /* Set by -mtune or -Os. */
130 : const struct processor_costs *ix86_cost = NULL;
131 :
132 : /* In case the average insn count for single function invocation is
133 : lower than this constant, emit fast (but longer) prologue and
134 : epilogue code. */
135 : #define FAST_PROLOGUE_INSN_COUNT 20
136 :
137 : /* Names for 8 (low), 8 (high), and 16-bit registers, respectively. */
138 : static const char *const qi_reg_name[] = QI_REGISTER_NAMES;
139 : static const char *const qi_high_reg_name[] = QI_HIGH_REGISTER_NAMES;
140 : static const char *const hi_reg_name[] = HI_REGISTER_NAMES;
141 :
142 : /* Array of the smallest class containing reg number REGNO, indexed by
143 : REGNO. Used by REGNO_REG_CLASS in i386.h. */
144 :
145 : enum reg_class const regclass_map[FIRST_PSEUDO_REGISTER] =
146 : {
147 : /* ax, dx, cx, bx */
148 : AREG, DREG, CREG, BREG,
149 : /* si, di, bp, sp */
150 : SIREG, DIREG, NON_Q_REGS, NON_Q_REGS,
151 : /* FP registers */
152 : FP_TOP_REG, FP_SECOND_REG, FLOAT_REGS, FLOAT_REGS,
153 : FLOAT_REGS, FLOAT_REGS, FLOAT_REGS, FLOAT_REGS,
154 : /* arg pointer, flags, fpsr, frame */
155 : NON_Q_REGS, NO_REGS, NO_REGS, NON_Q_REGS,
156 : /* SSE registers */
157 : SSE_FIRST_REG, SSE_REGS, SSE_REGS, SSE_REGS,
158 : SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS,
159 : /* MMX registers */
160 : MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS,
161 : MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS,
162 : /* REX registers */
163 : GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS,
164 : GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS,
165 : /* SSE REX registers */
166 : SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS,
167 : SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS,
168 : /* AVX-512 SSE registers */
169 : ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS,
170 : ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS,
171 : ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS,
172 : ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS,
173 : /* Mask registers. */
174 : ALL_MASK_REGS, MASK_REGS, MASK_REGS, MASK_REGS,
175 : MASK_REGS, MASK_REGS, MASK_REGS, MASK_REGS,
176 : /* REX2 registers */
177 : GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS,
178 : GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS,
179 : GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS,
180 : GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS,
181 : };
182 :
183 : /* The "default" register map used in 32bit mode. */
184 :
185 : unsigned int const debugger_register_map[FIRST_PSEUDO_REGISTER] =
186 : {
187 : /* general regs */
188 : 0, 2, 1, 3, 6, 7, 4, 5,
189 : /* fp regs */
190 : 12, 13, 14, 15, 16, 17, 18, 19,
191 : /* arg, flags, fpsr, frame */
192 : IGNORED_DWARF_REGNUM, IGNORED_DWARF_REGNUM,
193 : IGNORED_DWARF_REGNUM, IGNORED_DWARF_REGNUM,
194 : /* SSE */
195 : 21, 22, 23, 24, 25, 26, 27, 28,
196 : /* MMX */
197 : 29, 30, 31, 32, 33, 34, 35, 36,
198 : /* extended integer registers */
199 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
200 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
201 : /* extended sse registers */
202 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
203 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
204 : /* AVX-512 registers 16-23 */
205 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
206 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
207 : /* AVX-512 registers 24-31 */
208 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
209 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
210 : /* Mask registers */
211 : 93, 94, 95, 96, 97, 98, 99, 100
212 : };
213 :
214 : /* The "default" register map used in 64bit mode. */
215 :
216 : unsigned int const debugger64_register_map[FIRST_PSEUDO_REGISTER] =
217 : {
218 : /* general regs */
219 : 0, 1, 2, 3, 4, 5, 6, 7,
220 : /* fp regs */
221 : 33, 34, 35, 36, 37, 38, 39, 40,
222 : /* arg, flags, fpsr, frame */
223 : IGNORED_DWARF_REGNUM, IGNORED_DWARF_REGNUM,
224 : IGNORED_DWARF_REGNUM, IGNORED_DWARF_REGNUM,
225 : /* SSE */
226 : 17, 18, 19, 20, 21, 22, 23, 24,
227 : /* MMX */
228 : 41, 42, 43, 44, 45, 46, 47, 48,
229 : /* extended integer registers */
230 : 8, 9, 10, 11, 12, 13, 14, 15,
231 : /* extended SSE registers */
232 : 25, 26, 27, 28, 29, 30, 31, 32,
233 : /* AVX-512 registers 16-23 */
234 : 67, 68, 69, 70, 71, 72, 73, 74,
235 : /* AVX-512 registers 24-31 */
236 : 75, 76, 77, 78, 79, 80, 81, 82,
237 : /* Mask registers */
238 : 118, 119, 120, 121, 122, 123, 124, 125,
239 : /* rex2 extend integer registers */
240 : 130, 131, 132, 133, 134, 135, 136, 137,
241 : 138, 139, 140, 141, 142, 143, 144, 145
242 : };
243 :
244 : /* Define the register numbers to be used in Dwarf debugging information.
245 : The SVR4 reference port C compiler uses the following register numbers
246 : in its Dwarf output code:
247 : 0 for %eax (gcc regno = 0)
248 : 1 for %ecx (gcc regno = 2)
249 : 2 for %edx (gcc regno = 1)
250 : 3 for %ebx (gcc regno = 3)
251 : 4 for %esp (gcc regno = 7)
252 : 5 for %ebp (gcc regno = 6)
253 : 6 for %esi (gcc regno = 4)
254 : 7 for %edi (gcc regno = 5)
255 : The following three DWARF register numbers are never generated by
256 : the SVR4 C compiler or by the GNU compilers, but SDB on x86/svr4
257 : believed these numbers have these meanings.
258 : 8 for %eip (no gcc equivalent)
259 : 9 for %eflags (gcc regno = 17)
260 : 10 for %trapno (no gcc equivalent)
261 : It is not at all clear how we should number the FP stack registers
262 : for the x86 architecture. If the version of SDB on x86/svr4 were
263 : a bit less brain dead with respect to floating-point then we would
264 : have a precedent to follow with respect to DWARF register numbers
265 : for x86 FP registers, but the SDB on x86/svr4 was so completely
266 : broken with respect to FP registers that it is hardly worth thinking
267 : of it as something to strive for compatibility with.
268 : The version of x86/svr4 SDB I had does (partially)
269 : seem to believe that DWARF register number 11 is associated with
270 : the x86 register %st(0), but that's about all. Higher DWARF
271 : register numbers don't seem to be associated with anything in
272 : particular, and even for DWARF regno 11, SDB only seemed to under-
273 : stand that it should say that a variable lives in %st(0) (when
274 : asked via an `=' command) if we said it was in DWARF regno 11,
275 : but SDB still printed garbage when asked for the value of the
276 : variable in question (via a `/' command).
277 : (Also note that the labels SDB printed for various FP stack regs
278 : when doing an `x' command were all wrong.)
279 : Note that these problems generally don't affect the native SVR4
280 : C compiler because it doesn't allow the use of -O with -g and
281 : because when it is *not* optimizing, it allocates a memory
282 : location for each floating-point variable, and the memory
283 : location is what gets described in the DWARF AT_location
284 : attribute for the variable in question.
285 : Regardless of the severe mental illness of the x86/svr4 SDB, we
286 : do something sensible here and we use the following DWARF
287 : register numbers. Note that these are all stack-top-relative
288 : numbers.
289 : 11 for %st(0) (gcc regno = 8)
290 : 12 for %st(1) (gcc regno = 9)
291 : 13 for %st(2) (gcc regno = 10)
292 : 14 for %st(3) (gcc regno = 11)
293 : 15 for %st(4) (gcc regno = 12)
294 : 16 for %st(5) (gcc regno = 13)
295 : 17 for %st(6) (gcc regno = 14)
296 : 18 for %st(7) (gcc regno = 15)
297 : */
298 : unsigned int const svr4_debugger_register_map[FIRST_PSEUDO_REGISTER] =
299 : {
300 : /* general regs */
301 : 0, 2, 1, 3, 6, 7, 5, 4,
302 : /* fp regs */
303 : 11, 12, 13, 14, 15, 16, 17, 18,
304 : /* arg, flags, fpsr, frame */
305 : IGNORED_DWARF_REGNUM, 9,
306 : IGNORED_DWARF_REGNUM, IGNORED_DWARF_REGNUM,
307 : /* SSE registers */
308 : 21, 22, 23, 24, 25, 26, 27, 28,
309 : /* MMX registers */
310 : 29, 30, 31, 32, 33, 34, 35, 36,
311 : /* extended integer registers */
312 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
313 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
314 : /* extended sse registers */
315 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
316 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
317 : /* AVX-512 registers 16-23 */
318 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
319 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
320 : /* AVX-512 registers 24-31 */
321 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
322 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
323 : /* Mask registers */
324 : 93, 94, 95, 96, 97, 98, 99, 100
325 : };
326 :
327 : /* Define parameter passing and return registers. */
328 :
329 : static int const x86_64_int_parameter_registers[6] =
330 : {
331 : DI_REG, SI_REG, DX_REG, CX_REG, R8_REG, R9_REG
332 : };
333 :
334 : static int const x86_64_ms_abi_int_parameter_registers[4] =
335 : {
336 : CX_REG, DX_REG, R8_REG, R9_REG
337 : };
338 :
339 : /* Similar as Clang's preserve_none function parameter passing.
340 : NB: Use DI_REG and SI_REG, see ix86_function_arg_regno_p. */
341 :
342 : static int const x86_64_preserve_none_int_parameter_registers[6] =
343 : {
344 : R12_REG, R13_REG, R14_REG, R15_REG, DI_REG, SI_REG
345 : };
346 :
347 : static int const x86_64_int_return_registers[2] =
348 : {
349 : AX_REG, DX_REG
350 : };
351 :
352 : /* Define the structure for the machine field in struct function. */
353 :
354 : struct GTY(()) stack_local_entry {
355 : unsigned short mode;
356 : unsigned short n;
357 : rtx rtl;
358 : struct stack_local_entry *next;
359 : };
360 :
361 : /* Which cpu are we scheduling for. */
362 : enum attr_cpu ix86_schedule;
363 :
364 : /* Which cpu are we optimizing for. */
365 : enum processor_type ix86_tune;
366 :
367 : /* Which instruction set architecture to use. */
368 : enum processor_type ix86_arch;
369 :
370 : /* True if processor has SSE prefetch instruction. */
371 : unsigned char ix86_prefetch_sse;
372 :
373 : /* Preferred alignment for stack boundary in bits. */
374 : unsigned int ix86_preferred_stack_boundary;
375 :
376 : /* Alignment for incoming stack boundary in bits specified at
377 : command line. */
378 : unsigned int ix86_user_incoming_stack_boundary;
379 :
380 : /* Default alignment for incoming stack boundary in bits. */
381 : unsigned int ix86_default_incoming_stack_boundary;
382 :
383 : /* Alignment for incoming stack boundary in bits. */
384 : unsigned int ix86_incoming_stack_boundary;
385 :
386 : /* True if there is no direct access to extern symbols. */
387 : bool ix86_has_no_direct_extern_access;
388 :
389 : /* Calling abi specific va_list type nodes. */
390 : tree sysv_va_list_type_node;
391 : tree ms_va_list_type_node;
392 :
393 : /* Prefix built by ASM_GENERATE_INTERNAL_LABEL. */
394 : char internal_label_prefix[16];
395 : int internal_label_prefix_len;
396 :
397 : /* Fence to use after loop using movnt. */
398 : tree x86_mfence;
399 :
400 : /* Register class used for passing given 64bit part of the argument.
401 : These represent classes as documented by the PS ABI, with the exception
402 : of SSESF, SSEDF classes, that are basically SSE class, just gcc will
403 : use SF or DFmode move instead of DImode to avoid reformatting penalties.
404 :
405 : Similarly we play games with INTEGERSI_CLASS to use cheaper SImode moves
406 : whenever possible (upper half does contain padding). */
407 : enum x86_64_reg_class
408 : {
409 : X86_64_NO_CLASS,
410 : X86_64_INTEGER_CLASS,
411 : X86_64_INTEGERSI_CLASS,
412 : X86_64_SSE_CLASS,
413 : X86_64_SSEHF_CLASS,
414 : X86_64_SSESF_CLASS,
415 : X86_64_SSEDF_CLASS,
416 : X86_64_SSEUP_CLASS,
417 : X86_64_X87_CLASS,
418 : X86_64_X87UP_CLASS,
419 : X86_64_COMPLEX_X87_CLASS,
420 : X86_64_MEMORY_CLASS
421 : };
422 :
423 : #define MAX_CLASSES 8
424 :
425 : /* Table of constants used by fldpi, fldln2, etc.... */
426 : static REAL_VALUE_TYPE ext_80387_constants_table [5];
427 : static bool ext_80387_constants_init;
428 :
429 :
430 : static rtx ix86_function_value (const_tree, const_tree, bool);
431 : static bool ix86_function_value_regno_p (const unsigned int);
432 : static unsigned int ix86_function_arg_boundary (machine_mode,
433 : const_tree);
434 : static rtx ix86_static_chain (const_tree, bool);
435 : static int ix86_function_regparm (const_tree, const_tree);
436 : static void ix86_compute_frame_layout (void);
437 : static tree ix86_canonical_va_list_type (tree);
438 : static unsigned int split_stack_prologue_scratch_regno (void);
439 : static bool i386_asm_output_addr_const_extra (FILE *, rtx);
440 :
441 : static bool ix86_can_inline_p (tree, tree);
442 : static unsigned int ix86_minimum_incoming_stack_boundary (bool);
443 :
444 : typedef enum ix86_flags_cc
445 : {
446 : X86_CCO = 0, X86_CCNO, X86_CCB, X86_CCNB,
447 : X86_CCE, X86_CCNE, X86_CCBE, X86_CCNBE,
448 : X86_CCS, X86_CCNS, X86_CCP, X86_CCNP,
449 : X86_CCL, X86_CCNL, X86_CCLE, X86_CCNLE
450 : } ix86_cc;
451 :
452 : static const char *ix86_ccmp_dfv_mapping[] =
453 : {
454 : "{dfv=of}", "{dfv=}", "{dfv=cf}", "{dfv=}",
455 : "{dfv=zf}", "{dfv=}", "{dfv=cf, zf}", "{dfv=}",
456 : "{dfv=sf}", "{dfv=}", "{dfv=cf}", "{dfv=}",
457 : "{dfv=sf}", "{dfv=sf, of}", "{dfv=sf, of, zf}", "{dfv=sf, of}"
458 : };
459 :
460 :
461 : /* Whether -mtune= or -march= were specified */
462 : int ix86_tune_defaulted;
463 : int ix86_arch_specified;
464 :
465 : /* Return true if a red-zone is in use. We can't use red-zone when
466 : there are local indirect jumps, like "indirect_jump" or "tablejump",
467 : which jumps to another place in the function, since "call" in the
468 : indirect thunk pushes the return address onto stack, destroying
469 : red-zone.
470 :
471 : NB: Don't use red-zone for functions with no_caller_saved_registers
472 : and 32 GPRs or 16 XMM registers since 128-byte red-zone is too small
473 : for 31 GPRs or 15 GPRs + 16 XMM registers.
474 :
475 : TODO: If we can reserve the first 2 WORDs, for PUSH and, another
476 : for CALL, in red-zone, we can allow local indirect jumps with
477 : indirect thunk. */
478 :
479 : bool
480 9977279 : ix86_using_red_zone (void)
481 : {
482 9977279 : return (TARGET_RED_ZONE
483 9033573 : && !TARGET_64BIT_MS_ABI
484 8730646 : && ((!TARGET_APX_EGPR && !TARGET_SSE)
485 8707502 : || (cfun->machine->call_saved_registers
486 8707502 : != TYPE_NO_CALLER_SAVED_REGISTERS))
487 18707830 : && (!cfun->machine->has_local_indirect_jump
488 49190 : || cfun->machine->indirect_branch_type == indirect_branch_keep));
489 : }
490 :
491 : /* Return true, if profiling code should be emitted before
492 : prologue. Otherwise it returns false.
493 : Note: For x86 with "hotfix" it is sorried. */
494 : static bool
495 4533273 : ix86_profile_before_prologue (void)
496 : {
497 4533273 : return flag_fentry != 0;
498 : }
499 :
500 : /* Update register usage after having seen the compiler flags. */
501 :
502 : static void
503 778760 : ix86_conditional_register_usage (void)
504 : {
505 778760 : int i, c_mask;
506 :
507 : /* For 32-bit targets, disable the REX registers. */
508 778760 : if (! TARGET_64BIT)
509 : {
510 133830 : for (i = FIRST_REX_INT_REG; i <= LAST_REX_INT_REG; i++)
511 118960 : CLEAR_HARD_REG_BIT (accessible_reg_set, i);
512 133830 : for (i = FIRST_REX_SSE_REG; i <= LAST_REX_SSE_REG; i++)
513 118960 : CLEAR_HARD_REG_BIT (accessible_reg_set, i);
514 252790 : for (i = FIRST_EXT_REX_SSE_REG; i <= LAST_EXT_REX_SSE_REG; i++)
515 237920 : CLEAR_HARD_REG_BIT (accessible_reg_set, i);
516 : }
517 :
518 : /* Set up the call-used registers based on the system ABI (ix86_abi).
519 :
520 : See the definition of CALL_USED_REGISTERS in i386.h. */
521 778760 : c_mask = CALL_USED_REGISTERS_MASK (TARGET_64BIT && ix86_abi == MS_ABI);
522 :
523 778760 : CLEAR_HARD_REG_SET (reg_class_contents[(int)CLOBBERED_REGS]);
524 :
525 72424680 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
526 : {
527 : /* Set/reset conditionally defined registers from
528 : CALL_USED_REGISTERS initializer. */
529 71645920 : if (call_used_regs[i] > 1)
530 12460105 : call_used_regs[i] = !!(call_used_regs[i] & c_mask);
531 :
532 : /* Calculate registers of CLOBBERED_REGS register set
533 : as call used registers from GENERAL_REGS register set. */
534 71645920 : if (TEST_HARD_REG_BIT (reg_class_contents[(int)GENERAL_REGS], i)
535 71645920 : && call_used_regs[i])
536 21833750 : SET_HARD_REG_BIT (reg_class_contents[(int)CLOBBERED_REGS], i);
537 : }
538 :
539 : /* If MMX is disabled, disable the registers. */
540 778760 : if (! TARGET_MMX)
541 382328 : accessible_reg_set &= ~reg_class_contents[MMX_REGS];
542 :
543 : /* If SSE is disabled, disable the registers. */
544 778760 : if (! TARGET_SSE)
545 377162 : accessible_reg_set &= ~reg_class_contents[ALL_SSE_REGS];
546 :
547 : /* If the FPU is disabled, disable the registers. */
548 778760 : if (! (TARGET_80387 || TARGET_FLOAT_RETURNS_IN_80387))
549 377588 : accessible_reg_set &= ~reg_class_contents[FLOAT_REGS];
550 :
551 : /* If AVX512F is disabled, disable the registers. */
552 778760 : if (! TARGET_AVX512F)
553 : {
554 9204786 : for (i = FIRST_EXT_REX_SSE_REG; i <= LAST_EXT_REX_SSE_REG; i++)
555 8663328 : CLEAR_HARD_REG_BIT (accessible_reg_set, i);
556 :
557 1082916 : accessible_reg_set &= ~reg_class_contents[ALL_MASK_REGS];
558 : }
559 :
560 : /* If APX is disabled, disable the registers. */
561 778760 : if (! (TARGET_APX_EGPR && TARGET_64BIT))
562 : {
563 13230947 : for (i = FIRST_REX2_INT_REG; i <= LAST_REX2_INT_REG; i++)
564 12452656 : CLEAR_HARD_REG_BIT (accessible_reg_set, i);
565 : }
566 778760 : }
567 :
568 : /* Canonicalize a comparison from one we don't have to one we do have. */
569 :
570 : static void
571 24782708 : ix86_canonicalize_comparison (int *code, rtx *op0, rtx *op1,
572 : bool op0_preserve_value)
573 : {
574 : /* The order of operands in x87 ficom compare is forced by combine in
575 : simplify_comparison () function. Float operator is treated as RTX_OBJ
576 : with a precedence over other operators and is always put in the first
577 : place. Swap condition and operands to match ficom instruction. */
578 24782708 : if (!op0_preserve_value
579 23977363 : && GET_CODE (*op0) == FLOAT && MEM_P (XEXP (*op0, 0)) && REG_P (*op1))
580 : {
581 26 : enum rtx_code scode = swap_condition ((enum rtx_code) *code);
582 :
583 : /* We are called only for compares that are split to SAHF instruction.
584 : Ensure that we have setcc/jcc insn for the swapped condition. */
585 26 : if (ix86_fp_compare_code_to_integer (scode) != UNKNOWN)
586 : {
587 6 : std::swap (*op0, *op1);
588 6 : *code = (int) scode;
589 6 : return;
590 : }
591 : }
592 :
593 : /* SUB (a, b) underflows precisely when a < b. Convert
594 : (compare (minus (a b)) a) to (compare (a b))
595 : to match *sub<mode>_3 pattern. */
596 23977357 : if (!op0_preserve_value
597 23977357 : && (*code == GTU || *code == LEU)
598 1884229 : && GET_CODE (*op0) == MINUS
599 79376 : && rtx_equal_p (XEXP (*op0, 0), *op1))
600 : {
601 487 : *op1 = XEXP (*op0, 1);
602 487 : *op0 = XEXP (*op0, 0);
603 487 : *code = (int) swap_condition ((enum rtx_code) *code);
604 487 : return;
605 : }
606 :
607 : /* Swap operands of GTU comparison to canonicalize
608 : addcarry/subborrow comparison. */
609 24782215 : if (!op0_preserve_value
610 23976870 : && *code == GTU
611 883945 : && GET_CODE (*op0) == PLUS
612 318106 : && ix86_carry_flag_operator (XEXP (*op0, 0), VOIDmode)
613 44696 : && GET_CODE (XEXP (*op0, 1)) == ZERO_EXTEND
614 24822820 : && GET_CODE (*op1) == ZERO_EXTEND)
615 : {
616 37252 : std::swap (*op0, *op1);
617 37252 : *code = (int) swap_condition ((enum rtx_code) *code);
618 37252 : return;
619 : }
620 : }
621 :
622 : /* Hook to determine if one function can safely inline another. */
623 :
624 : static bool
625 10190045 : ix86_can_inline_p (tree caller, tree callee)
626 : {
627 10190045 : tree caller_tree = DECL_FUNCTION_SPECIFIC_TARGET (caller);
628 10190045 : tree callee_tree = DECL_FUNCTION_SPECIFIC_TARGET (callee);
629 :
630 : /* Changes of those flags can be tolerated for always inlines. Lets hope
631 : user knows what he is doing. */
632 10190045 : unsigned HOST_WIDE_INT always_inline_safe_mask
633 : = (MASK_USE_8BIT_IDIV | MASK_ACCUMULATE_OUTGOING_ARGS
634 : | MASK_NO_ALIGN_STRINGOPS | MASK_AVX256_SPLIT_UNALIGNED_LOAD
635 : | MASK_AVX256_SPLIT_UNALIGNED_STORE | MASK_CLD
636 : | MASK_NO_FANCY_MATH_387 | MASK_IEEE_FP | MASK_INLINE_ALL_STRINGOPS
637 : | MASK_INLINE_STRINGOPS_DYNAMICALLY | MASK_RECIP | MASK_STACK_PROBE
638 : | MASK_STV | MASK_TLS_DIRECT_SEG_REFS | MASK_VZEROUPPER
639 : | MASK_NO_PUSH_ARGS | MASK_OMIT_LEAF_FRAME_POINTER);
640 :
641 :
642 10190045 : if (!callee_tree)
643 9592543 : callee_tree = target_option_default_node;
644 10190045 : if (!caller_tree)
645 9592608 : caller_tree = target_option_default_node;
646 10190045 : if (callee_tree == caller_tree)
647 : return true;
648 :
649 5292 : struct cl_target_option *caller_opts = TREE_TARGET_OPTION (caller_tree);
650 5292 : struct cl_target_option *callee_opts = TREE_TARGET_OPTION (callee_tree);
651 5292 : bool ret = false;
652 5292 : bool always_inline
653 5292 : = (DECL_DISREGARD_INLINE_LIMITS (callee)
654 9939 : && lookup_attribute ("always_inline",
655 4647 : DECL_ATTRIBUTES (callee)));
656 :
657 : /* If callee only uses GPRs, ignore MASK_80387. */
658 5292 : if (TARGET_GENERAL_REGS_ONLY_P (callee_opts->x_ix86_target_flags))
659 1030 : always_inline_safe_mask |= MASK_80387;
660 :
661 5292 : cgraph_node *callee_node = cgraph_node::get (callee);
662 : /* Callee's isa options should be a subset of the caller's, i.e. a SSE4
663 : function can inline a SSE2 function but a SSE2 function can't inline
664 : a SSE4 function. */
665 5292 : if (((caller_opts->x_ix86_isa_flags & callee_opts->x_ix86_isa_flags)
666 : != callee_opts->x_ix86_isa_flags)
667 5056 : || ((caller_opts->x_ix86_isa_flags2 & callee_opts->x_ix86_isa_flags2)
668 : != callee_opts->x_ix86_isa_flags2))
669 : ret = false;
670 :
671 : /* See if we have the same non-isa options. */
672 5019 : else if ((!always_inline
673 388 : && caller_opts->x_target_flags != callee_opts->x_target_flags)
674 4975 : || (caller_opts->x_target_flags & ~always_inline_safe_mask)
675 4975 : != (callee_opts->x_target_flags & ~always_inline_safe_mask))
676 : ret = false;
677 :
678 4975 : else if (caller_opts->x_ix86_fpmath != callee_opts->x_ix86_fpmath
679 : /* If the callee doesn't use FP expressions differences in
680 : ix86_fpmath can be ignored. We are called from FEs
681 : for multi-versioning call optimization, so beware of
682 : ipa_fn_summaries not available. */
683 1247 : && (! ipa_fn_summaries
684 1247 : || ipa_fn_summaries->get (callee_node) == NULL
685 1247 : || ipa_fn_summaries->get (callee_node)->fp_expressions))
686 : ret = false;
687 :
688 : /* At this point we cannot identify whether arch or tune setting
689 : comes from target attribute or not. So the most conservative way
690 : is to allow the callee that uses default arch and tune string to
691 : be inlined. */
692 4701 : else if (!strcmp (callee_opts->x_ix86_arch_string, "x86-64")
693 1430 : && !strcmp (callee_opts->x_ix86_tune_string, "generic"))
694 : ret = true;
695 :
696 : /* See if arch, tune, etc. are the same. As previous ISA flags already
697 : checks if callee's ISA is subset of caller's, do not block
698 : always_inline attribute for callee even it has different arch. */
699 3279 : else if (!always_inline && caller_opts->arch != callee_opts->arch)
700 : ret = false;
701 :
702 15 : else if (!always_inline && caller_opts->tune != callee_opts->tune)
703 : ret = false;
704 :
705 3279 : else if (!always_inline
706 15 : && caller_opts->branch_cost != callee_opts->branch_cost)
707 : ret = false;
708 :
709 : else
710 10189454 : ret = true;
711 :
712 : return ret;
713 : }
714 :
715 : /* Return true if this goes in large data/bss. */
716 :
717 : static bool
718 83187442 : ix86_in_large_data_p (tree exp)
719 : {
720 83187442 : if (ix86_cmodel != CM_MEDIUM && ix86_cmodel != CM_MEDIUM_PIC
721 83187204 : && ix86_cmodel != CM_LARGE && ix86_cmodel != CM_LARGE_PIC)
722 : return false;
723 :
724 1147 : if (exp == NULL_TREE)
725 : return false;
726 :
727 : /* Functions are never large data. */
728 1147 : if (TREE_CODE (exp) == FUNCTION_DECL)
729 : return false;
730 :
731 : /* Automatic variables are never large data. */
732 279 : if (VAR_P (exp) && !is_global_var (exp))
733 : return false;
734 :
735 279 : if (VAR_P (exp) && DECL_SECTION_NAME (exp))
736 : {
737 51 : const char *section = DECL_SECTION_NAME (exp);
738 51 : if (strcmp (section, ".ldata") == 0
739 51 : || startswith (section, ".ldata.")
740 51 : || strcmp (section, ".lbss") == 0
741 51 : || startswith (section, ".lbss.")
742 99 : || startswith (section, ".gnu.linkonce.lb."))
743 : return true;
744 : return false;
745 : }
746 : else
747 : {
748 228 : HOST_WIDE_INT size = int_size_in_bytes (TREE_TYPE (exp));
749 :
750 : /* If this is an incomplete type with size 0, then we can't put it
751 : in data because it might be too big when completed. Also,
752 : int_size_in_bytes returns -1 if size can vary or is larger than
753 : an integer in which case also it is safer to assume that it goes in
754 : large data. */
755 228 : if (size <= 0 || size > ix86_section_threshold)
756 : return true;
757 : }
758 :
759 : return false;
760 : }
761 :
762 : /* i386-specific section flag to mark large sections. */
763 : #define SECTION_LARGE SECTION_MACH_DEP
764 :
765 : /* Switch to the appropriate section for output of DECL.
766 : DECL is either a `VAR_DECL' node or a constant of some sort.
767 : RELOC indicates whether forming the initial value of DECL requires
768 : link-time relocations. */
769 :
770 : ATTRIBUTE_UNUSED static section *
771 1672884 : x86_64_elf_select_section (tree decl, int reloc,
772 : unsigned HOST_WIDE_INT align)
773 : {
774 1672884 : if (ix86_in_large_data_p (decl))
775 : {
776 6 : const char *sname = NULL;
777 6 : unsigned int flags = SECTION_WRITE | SECTION_LARGE;
778 6 : switch (categorize_decl_for_section (decl, reloc))
779 : {
780 1 : case SECCAT_DATA:
781 1 : sname = ".ldata";
782 1 : break;
783 0 : case SECCAT_DATA_REL:
784 0 : sname = ".ldata.rel";
785 0 : break;
786 0 : case SECCAT_DATA_REL_LOCAL:
787 0 : sname = ".ldata.rel.local";
788 0 : break;
789 0 : case SECCAT_DATA_REL_RO:
790 0 : sname = ".ldata.rel.ro";
791 0 : break;
792 0 : case SECCAT_DATA_REL_RO_LOCAL:
793 0 : sname = ".ldata.rel.ro.local";
794 0 : break;
795 0 : case SECCAT_BSS:
796 0 : sname = ".lbss";
797 0 : flags |= SECTION_BSS;
798 0 : break;
799 : case SECCAT_RODATA:
800 : case SECCAT_RODATA_MERGE_STR:
801 : case SECCAT_RODATA_MERGE_STR_INIT:
802 : case SECCAT_RODATA_MERGE_CONST:
803 : sname = ".lrodata";
804 : flags &= ~SECTION_WRITE;
805 : break;
806 0 : case SECCAT_SRODATA:
807 0 : case SECCAT_SDATA:
808 0 : case SECCAT_SBSS:
809 0 : gcc_unreachable ();
810 : case SECCAT_TEXT:
811 : case SECCAT_TDATA:
812 : case SECCAT_TBSS:
813 : /* We don't split these for medium model. Place them into
814 : default sections and hope for best. */
815 : break;
816 : }
817 1 : if (sname)
818 : {
819 : /* We might get called with string constants, but get_named_section
820 : doesn't like them as they are not DECLs. Also, we need to set
821 : flags in that case. */
822 6 : if (!DECL_P (decl))
823 3 : return get_section (sname, flags, NULL);
824 3 : return get_named_section (decl, sname, reloc);
825 : }
826 : }
827 1672878 : return default_elf_select_section (decl, reloc, align);
828 : }
829 :
830 : /* Select a set of attributes for section NAME based on the properties
831 : of DECL and whether or not RELOC indicates that DECL's initializer
832 : might contain runtime relocations. */
833 :
834 : static unsigned int ATTRIBUTE_UNUSED
835 69699788 : x86_64_elf_section_type_flags (tree decl, const char *name, int reloc)
836 : {
837 69699788 : unsigned int flags = default_section_type_flags (decl, name, reloc);
838 :
839 69699788 : if (ix86_in_large_data_p (decl))
840 10 : flags |= SECTION_LARGE;
841 :
842 69699788 : if (decl == NULL_TREE
843 375 : && (strcmp (name, ".ldata.rel.ro") == 0
844 375 : || strcmp (name, ".ldata.rel.ro.local") == 0))
845 0 : flags |= SECTION_RELRO;
846 :
847 69699788 : if (strcmp (name, ".lbss") == 0
848 69699784 : || startswith (name, ".lbss.")
849 139399569 : || startswith (name, ".gnu.linkonce.lb."))
850 : {
851 7 : flags |= SECTION_BSS;
852 : /* Clear SECTION_NOTYPE so .lbss etc. are marked @nobits in
853 : default_elf_asm_named_section. */
854 7 : flags &= ~SECTION_NOTYPE;
855 : }
856 :
857 69699788 : return flags;
858 : }
859 :
860 : /* Build up a unique section name, expressed as a
861 : STRING_CST node, and assign it to DECL_SECTION_NAME (decl).
862 : RELOC indicates whether the initial value of EXP requires
863 : link-time relocations. */
864 :
865 : static void ATTRIBUTE_UNUSED
866 1814317 : x86_64_elf_unique_section (tree decl, int reloc)
867 : {
868 1814317 : if (ix86_in_large_data_p (decl))
869 : {
870 3 : const char *prefix = NULL;
871 : /* We only need to use .gnu.linkonce if we don't have COMDAT groups. */
872 3 : bool one_only = DECL_COMDAT_GROUP (decl) && !HAVE_COMDAT_GROUP;
873 :
874 3 : switch (categorize_decl_for_section (decl, reloc))
875 : {
876 0 : case SECCAT_DATA:
877 0 : case SECCAT_DATA_REL:
878 0 : case SECCAT_DATA_REL_LOCAL:
879 0 : case SECCAT_DATA_REL_RO:
880 0 : case SECCAT_DATA_REL_RO_LOCAL:
881 0 : prefix = one_only ? ".ld" : ".ldata";
882 : break;
883 3 : case SECCAT_BSS:
884 3 : prefix = one_only ? ".lb" : ".lbss";
885 : break;
886 : case SECCAT_RODATA:
887 : case SECCAT_RODATA_MERGE_STR:
888 : case SECCAT_RODATA_MERGE_STR_INIT:
889 : case SECCAT_RODATA_MERGE_CONST:
890 : prefix = one_only ? ".lr" : ".lrodata";
891 : break;
892 0 : case SECCAT_SRODATA:
893 0 : case SECCAT_SDATA:
894 0 : case SECCAT_SBSS:
895 0 : gcc_unreachable ();
896 : case SECCAT_TEXT:
897 : case SECCAT_TDATA:
898 : case SECCAT_TBSS:
899 : /* We don't split these for medium model. Place them into
900 : default sections and hope for best. */
901 : break;
902 : }
903 3 : if (prefix)
904 : {
905 3 : const char *name, *linkonce;
906 3 : char *string;
907 :
908 3 : name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
909 3 : name = targetm.strip_name_encoding (name);
910 :
911 : /* If we're using one_only, then there needs to be a .gnu.linkonce
912 : prefix to the section name. */
913 3 : linkonce = one_only ? ".gnu.linkonce" : "";
914 :
915 3 : string = ACONCAT ((linkonce, prefix, ".", name, NULL));
916 :
917 3 : set_decl_section_name (decl, string);
918 3 : return;
919 : }
920 : }
921 1814314 : default_unique_section (decl, reloc);
922 : }
923 :
924 : /* Return true if TYPE has no_callee_saved_registers or preserve_none
925 : attribute. */
926 :
927 : static bool
928 1585766 : ix86_type_no_callee_saved_registers_p (const_tree fntype)
929 : {
930 132328 : auto type = ix86_fntype_call_saved_registers (fntype);
931 1585766 : return type == TYPE_PRESERVE_NONE || type == TYPE_NO_CALLEE_SAVED_REGISTERS;
932 : }
933 :
934 : #ifdef COMMON_ASM_OP
935 :
936 : #ifndef LARGECOMM_SECTION_ASM_OP
937 : #define LARGECOMM_SECTION_ASM_OP "\t.largecomm\t"
938 : #endif
939 :
940 : /* This says how to output assembler code to declare an
941 : uninitialized external linkage data object.
942 :
943 : For medium model x86-64 we need to use LARGECOMM_SECTION_ASM_OP opcode for
944 : large objects. */
945 : void
946 171447 : x86_elf_aligned_decl_common (FILE *file, tree decl,
947 : const char *name, unsigned HOST_WIDE_INT size,
948 : unsigned align)
949 : {
950 171447 : if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC
951 171441 : || ix86_cmodel == CM_LARGE || ix86_cmodel == CM_LARGE_PIC)
952 7 : && size > (unsigned int)ix86_section_threshold)
953 : {
954 1 : switch_to_section (get_named_section (decl, ".lbss", 0));
955 1 : fputs (LARGECOMM_SECTION_ASM_OP, file);
956 : }
957 : else
958 171446 : fputs (COMMON_ASM_OP, file);
959 171447 : assemble_name (file, name);
960 171447 : fprintf (file, "," HOST_WIDE_INT_PRINT_UNSIGNED ",%u\n",
961 : size, align / BITS_PER_UNIT);
962 171447 : }
963 : #endif
964 :
965 : /* Utility function for targets to use in implementing
966 : ASM_OUTPUT_ALIGNED_BSS. */
967 :
968 : void
969 770020 : x86_output_aligned_bss (FILE *file, tree decl, const char *name,
970 : unsigned HOST_WIDE_INT size, unsigned align)
971 : {
972 770020 : if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC
973 770010 : || ix86_cmodel == CM_LARGE || ix86_cmodel == CM_LARGE_PIC)
974 42 : && size > (unsigned int)ix86_section_threshold)
975 3 : switch_to_section (get_named_section (decl, ".lbss", 0));
976 : else
977 770017 : switch_to_section (bss_section);
978 925967 : ASM_OUTPUT_ALIGN (file, floor_log2 (align / BITS_PER_UNIT));
979 : #ifdef ASM_DECLARE_OBJECT_NAME
980 770020 : last_assemble_variable_decl = decl;
981 770020 : ASM_DECLARE_OBJECT_NAME (file, name, decl);
982 : #else
983 : /* Standard thing is just output label for the object. */
984 : ASM_OUTPUT_LABEL (file, name);
985 : #endif /* ASM_DECLARE_OBJECT_NAME */
986 770020 : ASM_OUTPUT_SKIP (file, size ? size : 1);
987 770020 : }
988 :
989 : /* Decide whether we must probe the stack before any space allocation
990 : on this target. It's essentially TARGET_STACK_PROBE except when
991 : -fstack-check causes the stack to be already probed differently. */
992 :
993 : bool
994 877890 : ix86_target_stack_probe (void)
995 : {
996 : /* Do not probe the stack twice if static stack checking is enabled. */
997 877890 : if (flag_stack_check == STATIC_BUILTIN_STACK_CHECK)
998 : return false;
999 :
1000 877890 : return TARGET_STACK_PROBE;
1001 : }
1002 :
1003 : /* Decide whether we can make a sibling call to a function. DECL is the
1004 : declaration of the function being targeted by the call and EXP is the
1005 : CALL_EXPR representing the call. */
1006 :
1007 : static bool
1008 137664 : ix86_function_ok_for_sibcall (tree decl, tree exp)
1009 : {
1010 137664 : tree type, decl_or_type;
1011 137664 : rtx a, b;
1012 137664 : bool bind_global = decl && !targetm.binds_local_p (decl);
1013 :
1014 137664 : if (ix86_function_naked (current_function_decl))
1015 : return false;
1016 :
1017 : /* Sibling call isn't OK if there are no caller-saved registers
1018 : since all registers must be preserved before return. */
1019 137662 : if (cfun->machine->call_saved_registers
1020 137662 : == TYPE_NO_CALLER_SAVED_REGISTERS)
1021 : return false;
1022 :
1023 : /* If we are generating position-independent code, we cannot sibcall
1024 : optimize direct calls to global functions, as the PLT requires
1025 : %ebx be live. (Darwin does not have a PLT.) */
1026 137624 : if (!TARGET_MACHO
1027 137624 : && !TARGET_64BIT
1028 11360 : && flag_pic
1029 8413 : && flag_plt
1030 8413 : && bind_global)
1031 : return false;
1032 :
1033 : /* If we need to align the outgoing stack, then sibcalling would
1034 : unalign the stack, which may break the called function. */
1035 132962 : if (ix86_minimum_incoming_stack_boundary (true)
1036 132962 : < PREFERRED_STACK_BOUNDARY)
1037 : return false;
1038 :
1039 132381 : if (decl)
1040 : {
1041 121473 : decl_or_type = decl;
1042 121473 : type = TREE_TYPE (decl);
1043 : }
1044 : else
1045 : {
1046 : /* We're looking at the CALL_EXPR, we need the type of the function. */
1047 10908 : type = CALL_EXPR_FN (exp); /* pointer expression */
1048 10908 : type = TREE_TYPE (type); /* pointer type */
1049 10908 : type = TREE_TYPE (type); /* function type */
1050 10908 : decl_or_type = type;
1051 : }
1052 :
1053 : /* Sibling call isn't OK if callee has no callee-saved registers
1054 : and the calling function has callee-saved registers. */
1055 132381 : if ((cfun->machine->call_saved_registers
1056 132381 : != TYPE_NO_CALLEE_SAVED_REGISTERS)
1057 132381 : && cfun->machine->call_saved_registers != TYPE_PRESERVE_NONE
1058 264709 : && ix86_type_no_callee_saved_registers_p (type))
1059 : return false;
1060 :
1061 : /* If outgoing reg parm stack space changes, we cannot do sibcall. */
1062 132361 : if ((OUTGOING_REG_PARM_STACK_SPACE (type)
1063 132361 : != OUTGOING_REG_PARM_STACK_SPACE (TREE_TYPE (current_function_decl)))
1064 263964 : || (REG_PARM_STACK_SPACE (decl_or_type)
1065 131603 : != REG_PARM_STACK_SPACE (current_function_decl)))
1066 : {
1067 758 : maybe_complain_about_tail_call (exp,
1068 : "inconsistent size of stack space"
1069 : " allocated for arguments which are"
1070 : " passed in registers");
1071 758 : return false;
1072 : }
1073 :
1074 : /* Check that the return value locations are the same. Like
1075 : if we are returning floats on the 80387 register stack, we cannot
1076 : make a sibcall from a function that doesn't return a float to a
1077 : function that does or, conversely, from a function that does return
1078 : a float to a function that doesn't; the necessary stack adjustment
1079 : would not be executed. This is also the place we notice
1080 : differences in the return value ABI. Note that it is ok for one
1081 : of the functions to have void return type as long as the return
1082 : value of the other is passed in a register. */
1083 131603 : a = ix86_function_value (TREE_TYPE (exp), decl_or_type, false);
1084 131603 : b = ix86_function_value (TREE_TYPE (DECL_RESULT (cfun->decl)),
1085 131603 : cfun->decl, false);
1086 131603 : if (STACK_REG_P (a) || STACK_REG_P (b))
1087 : {
1088 1035 : if (!rtx_equal_p (a, b))
1089 : return false;
1090 : }
1091 130568 : else if (VOID_TYPE_P (TREE_TYPE (DECL_RESULT (cfun->decl))))
1092 : ;
1093 24086 : else if (!rtx_equal_p (a, b))
1094 : return false;
1095 :
1096 131214 : if (TARGET_64BIT)
1097 : {
1098 : /* The SYSV ABI has more call-clobbered registers;
1099 : disallow sibcalls from MS to SYSV. */
1100 124516 : if (cfun->machine->call_abi == MS_ABI
1101 124516 : && ix86_function_type_abi (type) == SYSV_ABI)
1102 : return false;
1103 : }
1104 : else
1105 : {
1106 : /* If this call is indirect, we'll need to be able to use a
1107 : call-clobbered register for the address of the target function.
1108 : Make sure that all such registers are not used for passing
1109 : parameters. Note that DLLIMPORT functions and call to global
1110 : function via GOT slot are indirect. */
1111 6698 : if (!decl
1112 4803 : || (bind_global && flag_pic && !flag_plt)
1113 : || (TARGET_DLLIMPORT_DECL_ATTRIBUTES && DECL_DLLIMPORT_P (decl))
1114 4803 : || flag_force_indirect_call)
1115 : {
1116 : /* Check if regparm >= 3 since arg_reg_available is set to
1117 : false if regparm == 0. If regparm is 1 or 2, there is
1118 : always a call-clobbered register available.
1119 :
1120 : ??? The symbol indirect call doesn't need a call-clobbered
1121 : register. But we don't know if this is a symbol indirect
1122 : call or not here. */
1123 1895 : if (ix86_function_regparm (type, decl) >= 3
1124 1895 : && !cfun->machine->arg_reg_available)
1125 : return false;
1126 : }
1127 : }
1128 :
1129 131214 : if (decl && ix86_use_pseudo_pic_reg ())
1130 : {
1131 : /* When PIC register is used, it must be restored after ifunc
1132 : function returns. */
1133 2066 : cgraph_node *node = cgraph_node::get (decl);
1134 2066 : if (node && node->ifunc_resolver)
1135 : return false;
1136 : }
1137 :
1138 : /* Disable sibcall if callee has indirect_return attribute and
1139 : caller doesn't since callee will return to the caller's caller
1140 : via an indirect jump. */
1141 131214 : if (((flag_cf_protection & (CF_RETURN | CF_BRANCH))
1142 : == (CF_RETURN | CF_BRANCH))
1143 52000 : && lookup_attribute ("indirect_return", TYPE_ATTRIBUTES (type))
1144 131218 : && !lookup_attribute ("indirect_return",
1145 4 : TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl))))
1146 : return false;
1147 :
1148 : /* Otherwise okay. That also includes certain types of indirect calls. */
1149 : return true;
1150 : }
1151 :
1152 : /* This function determines from TYPE the calling-convention. */
1153 :
1154 : unsigned int
1155 6254283 : ix86_get_callcvt (const_tree type)
1156 : {
1157 6254283 : unsigned int ret = 0;
1158 6254283 : bool is_stdarg;
1159 6254283 : tree attrs;
1160 :
1161 6254283 : if (TARGET_64BIT)
1162 : return IX86_CALLCVT_CDECL;
1163 :
1164 3271955 : attrs = TYPE_ATTRIBUTES (type);
1165 3271955 : if (attrs != NULL_TREE)
1166 : {
1167 68241 : if (lookup_attribute ("cdecl", attrs))
1168 : ret |= IX86_CALLCVT_CDECL;
1169 68241 : else if (lookup_attribute ("stdcall", attrs))
1170 : ret |= IX86_CALLCVT_STDCALL;
1171 68241 : else if (lookup_attribute ("fastcall", attrs))
1172 : ret |= IX86_CALLCVT_FASTCALL;
1173 68232 : else if (lookup_attribute ("thiscall", attrs))
1174 : ret |= IX86_CALLCVT_THISCALL;
1175 :
1176 : /* Regparm isn't allowed for thiscall and fastcall. */
1177 : if ((ret & (IX86_CALLCVT_THISCALL | IX86_CALLCVT_FASTCALL)) == 0)
1178 : {
1179 68232 : if (lookup_attribute ("regparm", attrs))
1180 15844 : ret |= IX86_CALLCVT_REGPARM;
1181 68232 : if (lookup_attribute ("sseregparm", attrs))
1182 0 : ret |= IX86_CALLCVT_SSEREGPARM;
1183 : }
1184 :
1185 68241 : if (IX86_BASE_CALLCVT(ret) != 0)
1186 9 : return ret;
1187 : }
1188 :
1189 3271946 : is_stdarg = stdarg_p (type);
1190 3271946 : if (TARGET_RTD && !is_stdarg)
1191 0 : return IX86_CALLCVT_STDCALL | ret;
1192 :
1193 3271946 : if (ret != 0
1194 3271946 : || is_stdarg
1195 3247059 : || TREE_CODE (type) != METHOD_TYPE
1196 3408624 : || ix86_function_type_abi (type) != MS_ABI)
1197 3271946 : return IX86_CALLCVT_CDECL | ret;
1198 :
1199 : return IX86_CALLCVT_THISCALL;
1200 : }
1201 :
1202 : /* Return 0 if the attributes for two types are incompatible, 1 if they
1203 : are compatible, and 2 if they are nearly compatible (which causes a
1204 : warning to be generated). */
1205 :
1206 : static int
1207 1508809 : ix86_comp_type_attributes (const_tree type1, const_tree type2)
1208 : {
1209 1508809 : unsigned int ccvt1, ccvt2;
1210 :
1211 1508809 : if (TREE_CODE (type1) != FUNCTION_TYPE
1212 1508809 : && TREE_CODE (type1) != METHOD_TYPE)
1213 : return 1;
1214 :
1215 1502304 : ccvt1 = ix86_get_callcvt (type1);
1216 1502304 : ccvt2 = ix86_get_callcvt (type2);
1217 1502304 : if (ccvt1 != ccvt2)
1218 : return 0;
1219 2982448 : if (ix86_function_regparm (type1, NULL)
1220 1491224 : != ix86_function_regparm (type2, NULL))
1221 : return 0;
1222 :
1223 726719 : if (ix86_type_no_callee_saved_registers_p (type1)
1224 726719 : != ix86_type_no_callee_saved_registers_p (type2))
1225 : return 0;
1226 :
1227 : /* preserve_none attribute uses a different calling convention is
1228 : only for 64-bit. */
1229 726506 : if (TARGET_64BIT
1230 1452952 : && (lookup_attribute ("preserve_none", TYPE_ATTRIBUTES (type1))
1231 726446 : != lookup_attribute ("preserve_none",
1232 726446 : TYPE_ATTRIBUTES (type2))))
1233 : return 0;
1234 :
1235 : return 1;
1236 : }
1237 :
1238 : /* Return the regparm value for a function with the indicated TYPE and DECL.
1239 : DECL may be NULL when calling function indirectly
1240 : or considering a libcall. */
1241 :
1242 : static int
1243 4254638 : ix86_function_regparm (const_tree type, const_tree decl)
1244 : {
1245 4254638 : tree attr;
1246 4254638 : int regparm;
1247 4254638 : unsigned int ccvt;
1248 :
1249 4254638 : if (TARGET_64BIT)
1250 2982328 : return (ix86_function_type_abi (type) == SYSV_ABI
1251 2982328 : ? X86_64_REGPARM_MAX : X86_64_MS_REGPARM_MAX);
1252 1272310 : ccvt = ix86_get_callcvt (type);
1253 1272310 : regparm = ix86_regparm;
1254 :
1255 1272310 : if ((ccvt & IX86_CALLCVT_REGPARM) != 0)
1256 : {
1257 2020 : attr = lookup_attribute ("regparm", TYPE_ATTRIBUTES (type));
1258 2020 : if (attr)
1259 : {
1260 2020 : regparm = TREE_INT_CST_LOW (TREE_VALUE (TREE_VALUE (attr)));
1261 2020 : return regparm;
1262 : }
1263 : }
1264 1270290 : else if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
1265 : return 2;
1266 1270290 : else if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
1267 : return 1;
1268 :
1269 : /* Use register calling convention for local functions when possible. */
1270 1270290 : if (decl
1271 1205951 : && TREE_CODE (decl) == FUNCTION_DECL)
1272 : {
1273 1195860 : cgraph_node *target = cgraph_node::get (decl);
1274 1195860 : if (target)
1275 1188352 : target = target->function_symbol ();
1276 :
1277 : /* Caller and callee must agree on the calling convention, so
1278 : checking here just optimize means that with
1279 : __attribute__((optimize (...))) caller could use regparm convention
1280 : and callee not, or vice versa. Instead look at whether the callee
1281 : is optimized or not. */
1282 1188352 : if (target && opt_for_fn (target->decl, optimize)
1283 2375812 : && !(profile_flag && !flag_fentry))
1284 : {
1285 1187460 : if (target->local && target->can_change_signature)
1286 : {
1287 139912 : int local_regparm, globals = 0, regno;
1288 :
1289 : /* Make sure no regparm register is taken by a
1290 : fixed register variable. */
1291 139912 : for (local_regparm = 0; local_regparm < REGPARM_MAX;
1292 : local_regparm++)
1293 104934 : if (fixed_regs[local_regparm])
1294 : break;
1295 :
1296 : /* We don't want to use regparm(3) for nested functions as
1297 : these use a static chain pointer in the third argument. */
1298 34978 : if (local_regparm == 3 && DECL_STATIC_CHAIN (target->decl))
1299 : local_regparm = 2;
1300 :
1301 : /* Save a register for the split stack. */
1302 34978 : if (flag_split_stack)
1303 : {
1304 20628 : if (local_regparm == 3)
1305 : local_regparm = 2;
1306 707 : else if (local_regparm == 2
1307 707 : && DECL_STATIC_CHAIN (target->decl))
1308 : local_regparm = 1;
1309 : }
1310 :
1311 : /* Each fixed register usage increases register pressure,
1312 : so less registers should be used for argument passing.
1313 : This functionality can be overridden by an explicit
1314 : regparm value. */
1315 244846 : for (regno = AX_REG; regno <= DI_REG; regno++)
1316 209868 : if (fixed_regs[regno])
1317 0 : globals++;
1318 :
1319 34978 : local_regparm
1320 34978 : = globals < local_regparm ? local_regparm - globals : 0;
1321 :
1322 34978 : if (local_regparm > regparm)
1323 4254638 : regparm = local_regparm;
1324 : }
1325 : }
1326 : }
1327 :
1328 : return regparm;
1329 : }
1330 :
1331 : /* Return 1 or 2, if we can pass up to SSE_REGPARM_MAX SFmode (1) and
1332 : DFmode (2) arguments in SSE registers for a function with the
1333 : indicated TYPE and DECL. DECL may be NULL when calling function
1334 : indirectly or considering a libcall. Return -1 if any FP parameter
1335 : should be rejected by error. This is used in siutation we imply SSE
1336 : calling convention but the function is called from another function with
1337 : SSE disabled. Otherwise return 0. */
1338 :
1339 : static int
1340 1079137 : ix86_function_sseregparm (const_tree type, const_tree decl, bool warn)
1341 : {
1342 1079137 : gcc_assert (!TARGET_64BIT);
1343 :
1344 : /* Use SSE registers to pass SFmode and DFmode arguments if requested
1345 : by the sseregparm attribute. */
1346 1079137 : if (TARGET_SSEREGPARM
1347 1079137 : || (type && lookup_attribute ("sseregparm", TYPE_ATTRIBUTES (type))))
1348 : {
1349 0 : if (!TARGET_SSE)
1350 : {
1351 0 : if (warn)
1352 : {
1353 0 : if (decl)
1354 0 : error ("calling %qD with attribute sseregparm without "
1355 : "SSE/SSE2 enabled", decl);
1356 : else
1357 0 : error ("calling %qT with attribute sseregparm without "
1358 : "SSE/SSE2 enabled", type);
1359 : }
1360 0 : return 0;
1361 : }
1362 :
1363 : return 2;
1364 : }
1365 :
1366 1079137 : if (!decl)
1367 : return 0;
1368 :
1369 979647 : cgraph_node *target = cgraph_node::get (decl);
1370 979647 : if (target)
1371 972146 : target = target->function_symbol ();
1372 :
1373 : /* For local functions, pass up to SSE_REGPARM_MAX SFmode
1374 : (and DFmode for SSE2) arguments in SSE registers. */
1375 972146 : if (target
1376 : /* TARGET_SSE_MATH */
1377 972146 : && (target_opts_for_fn (target->decl)->x_ix86_fpmath & FPMATH_SSE)
1378 1296 : && opt_for_fn (target->decl, optimize)
1379 973442 : && !(profile_flag && !flag_fentry))
1380 : {
1381 1296 : if (target->local && target->can_change_signature)
1382 : {
1383 : /* Refuse to produce wrong code when local function with SSE enabled
1384 : is called from SSE disabled function.
1385 : FIXME: We need a way to detect these cases cross-ltrans partition
1386 : and avoid using SSE calling conventions on local functions called
1387 : from function with SSE disabled. For now at least delay the
1388 : warning until we know we are going to produce wrong code.
1389 : See PR66047 */
1390 0 : if (!TARGET_SSE && warn)
1391 : return -1;
1392 0 : return TARGET_SSE2_P (target_opts_for_fn (target->decl)
1393 0 : ->x_ix86_isa_flags) ? 2 : 1;
1394 : }
1395 : }
1396 :
1397 : return 0;
1398 : }
1399 :
1400 : /* Return true if EAX is live at the start of the function. Used by
1401 : ix86_expand_prologue to determine if we need special help before
1402 : calling allocate_stack_worker. */
1403 :
1404 : static bool
1405 7090 : ix86_eax_live_at_start_p (void)
1406 : {
1407 : /* Cheat. Don't bother working forward from ix86_function_regparm
1408 : to the function type to whether an actual argument is located in
1409 : eax. Instead just look at cfg info, which is still close enough
1410 : to correct at this point. This gives false positives for broken
1411 : functions that might use uninitialized data that happens to be
1412 : allocated in eax, but who cares? */
1413 7090 : return REGNO_REG_SET_P (df_get_live_out (ENTRY_BLOCK_PTR_FOR_FN (cfun)), 0);
1414 : }
1415 :
1416 : static bool
1417 159993 : ix86_keep_aggregate_return_pointer (tree fntype)
1418 : {
1419 159993 : tree attr;
1420 :
1421 159993 : if (!TARGET_64BIT)
1422 : {
1423 159993 : attr = lookup_attribute ("callee_pop_aggregate_return",
1424 159993 : TYPE_ATTRIBUTES (fntype));
1425 159993 : if (attr)
1426 0 : return (TREE_INT_CST_LOW (TREE_VALUE (TREE_VALUE (attr))) == 0);
1427 :
1428 : /* For 32-bit MS-ABI the default is to keep aggregate
1429 : return pointer. */
1430 159993 : if (ix86_function_type_abi (fntype) == MS_ABI)
1431 : return true;
1432 : }
1433 : return KEEP_AGGREGATE_RETURN_POINTER != 0;
1434 : }
1435 :
1436 : /* Value is the number of bytes of arguments automatically
1437 : popped when returning from a subroutine call.
1438 : FUNDECL is the declaration node of the function (as a tree),
1439 : FUNTYPE is the data type of the function (as a tree),
1440 : or for a library call it is an identifier node for the subroutine name.
1441 : SIZE is the number of bytes of arguments passed on the stack.
1442 :
1443 : On the 80386, the RTD insn may be used to pop them if the number
1444 : of args is fixed, but if the number is variable then the caller
1445 : must pop them all. RTD can't be used for library calls now
1446 : because the library is compiled with the Unix compiler.
1447 : Use of RTD is a selectable option, since it is incompatible with
1448 : standard Unix calling sequences. If the option is not selected,
1449 : the caller must always pop the args.
1450 :
1451 : The attribute stdcall is equivalent to RTD on a per module basis. */
1452 :
1453 : static poly_int64
1454 7665318 : ix86_return_pops_args (tree fundecl, tree funtype, poly_int64 size)
1455 : {
1456 7665318 : unsigned int ccvt;
1457 :
1458 : /* None of the 64-bit ABIs pop arguments. */
1459 7665318 : if (TARGET_64BIT)
1460 6791287 : return 0;
1461 :
1462 874031 : ccvt = ix86_get_callcvt (funtype);
1463 :
1464 874031 : if ((ccvt & (IX86_CALLCVT_STDCALL | IX86_CALLCVT_FASTCALL
1465 : | IX86_CALLCVT_THISCALL)) != 0
1466 874031 : && ! stdarg_p (funtype))
1467 3 : return size;
1468 :
1469 : /* Lose any fake structure return argument if it is passed on the stack. */
1470 874028 : if (aggregate_value_p (TREE_TYPE (funtype), fundecl)
1471 874028 : && !ix86_keep_aggregate_return_pointer (funtype))
1472 : {
1473 159993 : int nregs = ix86_function_regparm (funtype, fundecl);
1474 159993 : if (nregs == 0)
1475 459120 : return GET_MODE_SIZE (Pmode);
1476 : }
1477 :
1478 720988 : return 0;
1479 : }
1480 :
1481 : /* Implement the TARGET_LEGITIMATE_COMBINED_INSN hook. */
1482 :
1483 : static bool
1484 10138686 : ix86_legitimate_combined_insn (rtx_insn *insn)
1485 : {
1486 10138686 : int i;
1487 :
1488 : /* Check operand constraints in case hard registers were propagated
1489 : into insn pattern. This check prevents combine pass from
1490 : generating insn patterns with invalid hard register operands.
1491 : These invalid insns can eventually confuse reload to error out
1492 : with a spill failure. See also PRs 46829 and 46843. */
1493 :
1494 10138686 : gcc_assert (INSN_CODE (insn) >= 0);
1495 :
1496 10138686 : extract_insn (insn);
1497 10138686 : preprocess_constraints (insn);
1498 :
1499 10138686 : int n_operands = recog_data.n_operands;
1500 10138686 : int n_alternatives = recog_data.n_alternatives;
1501 34607854 : for (i = 0; i < n_operands; i++)
1502 : {
1503 24472810 : rtx op = recog_data.operand[i];
1504 24472810 : machine_mode mode = GET_MODE (op);
1505 24472810 : const operand_alternative *op_alt;
1506 24472810 : int offset = 0;
1507 24472810 : bool win;
1508 24472810 : int j;
1509 :
1510 : /* A unary operator may be accepted by the predicate, but it
1511 : is irrelevant for matching constraints. */
1512 24472810 : if (UNARY_P (op))
1513 47595 : op = XEXP (op, 0);
1514 :
1515 24472810 : if (SUBREG_P (op))
1516 : {
1517 867134 : if (REG_P (SUBREG_REG (op))
1518 867134 : && REGNO (SUBREG_REG (op)) < FIRST_PSEUDO_REGISTER)
1519 54 : offset = subreg_regno_offset (REGNO (SUBREG_REG (op)),
1520 54 : GET_MODE (SUBREG_REG (op)),
1521 54 : SUBREG_BYTE (op),
1522 54 : GET_MODE (op));
1523 867134 : op = SUBREG_REG (op);
1524 : }
1525 :
1526 24472810 : if (!(REG_P (op) && HARD_REGISTER_P (op)))
1527 24214254 : continue;
1528 :
1529 258556 : op_alt = recog_op_alt;
1530 :
1531 : /* Operand has no constraints, anything is OK. */
1532 258556 : win = !n_alternatives;
1533 :
1534 258556 : alternative_mask preferred = get_preferred_alternatives (insn);
1535 745229 : for (j = 0; j < n_alternatives; j++, op_alt += n_operands)
1536 : {
1537 482954 : if (!TEST_BIT (preferred, j))
1538 140945 : continue;
1539 342009 : if (op_alt[i].anything_ok
1540 203783 : || (op_alt[i].matches != -1
1541 33953 : && operands_match_p
1542 33953 : (recog_data.operand[i],
1543 33953 : recog_data.operand[op_alt[i].matches]))
1544 541661 : || reg_fits_class_p (op, op_alt[i].cl, offset, mode))
1545 : {
1546 : win = true;
1547 : break;
1548 : }
1549 : }
1550 :
1551 258556 : if (!win)
1552 : return false;
1553 : }
1554 :
1555 : return true;
1556 : }
1557 :
1558 : /* Implement the TARGET_ASAN_SHADOW_OFFSET hook. */
1559 :
1560 : static unsigned HOST_WIDE_INT
1561 4832 : ix86_asan_shadow_offset (void)
1562 : {
1563 4832 : return SUBTARGET_SHADOW_OFFSET;
1564 : }
1565 :
1566 : /* Argument support functions. */
1567 :
1568 : /* Return true when register may be used to pass function parameters. */
1569 : bool
1570 1492775274 : ix86_function_arg_regno_p (int regno)
1571 : {
1572 1492775274 : int i;
1573 1492775274 : enum calling_abi call_abi;
1574 1492775274 : const int *parm_regs;
1575 :
1576 1489296383 : if (TARGET_SSE && SSE_REGNO_P (regno)
1577 2468560568 : && regno < FIRST_SSE_REG + SSE_REGPARM_MAX)
1578 : return true;
1579 :
1580 1372254155 : if (!TARGET_64BIT)
1581 129260957 : return (regno < REGPARM_MAX
1582 129260957 : || (TARGET_MMX && MMX_REGNO_P (regno)
1583 11617592 : && regno < FIRST_MMX_REG + MMX_REGPARM_MAX));
1584 :
1585 : /* TODO: The function should depend on current function ABI but
1586 : builtins.cc would need updating then. Therefore we use the
1587 : default ABI. */
1588 1242993198 : call_abi = ix86_cfun_abi ();
1589 :
1590 : /* RAX is used as hidden argument to va_arg functions. */
1591 1242993198 : if (call_abi == SYSV_ABI && regno == AX_REG)
1592 : return true;
1593 :
1594 1228647067 : if (cfun
1595 1228646735 : && cfun->machine->call_saved_registers == TYPE_PRESERVE_NONE)
1596 : parm_regs = x86_64_preserve_none_int_parameter_registers;
1597 1228607109 : else if (call_abi == MS_ABI)
1598 : parm_regs = x86_64_ms_abi_int_parameter_registers;
1599 : else
1600 1190849077 : parm_regs = x86_64_int_parameter_registers;
1601 :
1602 16438379266 : for (i = 0; i < (call_abi == MS_ABI
1603 8219189633 : ? X86_64_MS_REGPARM_MAX : X86_64_REGPARM_MAX); i++)
1604 7078384004 : if (regno == parm_regs[i])
1605 : return true;
1606 : return false;
1607 : }
1608 :
1609 : /* Return if we do not know how to pass ARG solely in registers. */
1610 :
1611 : static bool
1612 416105609 : ix86_must_pass_in_stack (const function_arg_info &arg)
1613 : {
1614 416105609 : if (must_pass_in_stack_var_size_or_pad (arg))
1615 : return true;
1616 :
1617 : /* For 32-bit, we want TImode aggregates to go on the stack. But watch out!
1618 : The layout_type routine is crafty and tries to trick us into passing
1619 : currently unsupported vector types on the stack by using TImode. */
1620 1773458 : return (!TARGET_64BIT && arg.mode == TImode
1621 416105572 : && arg.type && TREE_CODE (arg.type) != VECTOR_TYPE);
1622 : }
1623 :
1624 : /* It returns the size, in bytes, of the area reserved for arguments passed
1625 : in registers for the function represented by fndecl dependent to the used
1626 : abi format. */
1627 : int
1628 10846768 : ix86_reg_parm_stack_space (const_tree fndecl)
1629 : {
1630 10846768 : enum calling_abi call_abi = SYSV_ABI;
1631 10846768 : if (fndecl != NULL_TREE && TREE_CODE (fndecl) == FUNCTION_DECL)
1632 10530685 : call_abi = ix86_function_abi (fndecl);
1633 : else
1634 316083 : call_abi = ix86_function_type_abi (fndecl);
1635 10846768 : if (TARGET_64BIT && call_abi == MS_ABI)
1636 119430 : return 32;
1637 : return 0;
1638 : }
1639 :
1640 : /* We add this as a workaround in order to use libc_has_function
1641 : hook in i386.md. */
1642 : bool
1643 0 : ix86_libc_has_function (enum function_class fn_class)
1644 : {
1645 0 : return targetm.libc_has_function (fn_class, NULL_TREE);
1646 : }
1647 :
1648 : /* Returns value SYSV_ABI, MS_ABI dependent on fntype,
1649 : specifying the call abi used. */
1650 : enum calling_abi
1651 862609192 : ix86_function_type_abi (const_tree fntype)
1652 : {
1653 862609192 : enum calling_abi abi = ix86_abi;
1654 :
1655 862609192 : if (fntype == NULL_TREE || TYPE_ATTRIBUTES (fntype) == NULL_TREE)
1656 : return abi;
1657 :
1658 87604139 : if (abi == SYSV_ABI
1659 87604139 : && lookup_attribute ("ms_abi", TYPE_ATTRIBUTES (fntype)))
1660 : {
1661 4251136 : static int warned;
1662 4251136 : if (TARGET_X32 && !warned)
1663 : {
1664 1 : error ("X32 does not support %<ms_abi%> attribute");
1665 1 : warned = 1;
1666 : }
1667 :
1668 : abi = MS_ABI;
1669 : }
1670 83353003 : else if (abi == MS_ABI
1671 83353003 : && lookup_attribute ("sysv_abi", TYPE_ATTRIBUTES (fntype)))
1672 : abi = SYSV_ABI;
1673 :
1674 : return abi;
1675 : }
1676 :
1677 : enum calling_abi
1678 224461278 : ix86_function_abi (const_tree fndecl)
1679 : {
1680 224461278 : return fndecl ? ix86_function_type_abi (TREE_TYPE (fndecl)) : ix86_abi;
1681 : }
1682 :
1683 : /* Returns value SYSV_ABI, MS_ABI dependent on cfun,
1684 : specifying the call abi used. */
1685 : enum calling_abi
1686 2100265060 : ix86_cfun_abi (void)
1687 : {
1688 2100265060 : return cfun ? cfun->machine->call_abi : ix86_abi;
1689 : }
1690 :
1691 : bool
1692 5052663 : ix86_function_ms_hook_prologue (const_tree fn)
1693 : {
1694 5052663 : if (fn && lookup_attribute ("ms_hook_prologue", DECL_ATTRIBUTES (fn)))
1695 : {
1696 8 : if (decl_function_context (fn) != NULL_TREE)
1697 0 : error_at (DECL_SOURCE_LOCATION (fn),
1698 : "%<ms_hook_prologue%> attribute is not compatible "
1699 : "with nested function");
1700 : else
1701 : return true;
1702 : }
1703 : return false;
1704 : }
1705 :
1706 : bool
1707 124709405 : ix86_function_naked (const_tree fn)
1708 : {
1709 124709405 : if (fn && lookup_attribute ("naked", DECL_ATTRIBUTES (fn)))
1710 : return true;
1711 :
1712 : return false;
1713 : }
1714 :
1715 : /* Write the extra assembler code needed to declare a function properly. */
1716 :
1717 : void
1718 1563758 : ix86_asm_output_function_label (FILE *out_file, const char *fname,
1719 : tree decl)
1720 : {
1721 1563758 : bool is_ms_hook = ix86_function_ms_hook_prologue (decl);
1722 :
1723 1563758 : if (cfun)
1724 1560150 : cfun->machine->function_label_emitted = true;
1725 :
1726 1563758 : if (is_ms_hook)
1727 : {
1728 2 : int i, filler_count = (TARGET_64BIT ? 32 : 16);
1729 2 : unsigned int filler_cc = 0xcccccccc;
1730 :
1731 18 : for (i = 0; i < filler_count; i += 4)
1732 16 : fprintf (out_file, ASM_LONG " %#x\n", filler_cc);
1733 : }
1734 :
1735 : #ifdef SUBTARGET_ASM_UNWIND_INIT
1736 : SUBTARGET_ASM_UNWIND_INIT (out_file);
1737 : #endif
1738 :
1739 1563758 : assemble_function_label_raw (out_file, fname);
1740 :
1741 : /* Output magic byte marker, if hot-patch attribute is set. */
1742 1563758 : if (is_ms_hook)
1743 : {
1744 2 : if (TARGET_64BIT)
1745 : {
1746 : /* leaq [%rsp + 0], %rsp */
1747 2 : fputs (ASM_BYTE "0x48, 0x8d, 0xa4, 0x24, 0x00, 0x00, 0x00, 0x00\n",
1748 : out_file);
1749 : }
1750 : else
1751 : {
1752 : /* movl.s %edi, %edi
1753 : push %ebp
1754 : movl.s %esp, %ebp */
1755 0 : fputs (ASM_BYTE "0x8b, 0xff, 0x55, 0x8b, 0xec\n", out_file);
1756 : }
1757 : }
1758 1563758 : }
1759 :
1760 : /* Output a user-defined label. In AT&T syntax, registers are prefixed
1761 : with %, so labels require no punctuation. In Intel syntax, registers
1762 : are unprefixed, so labels may clash with registers or other operators,
1763 : and require quoting. */
1764 : void
1765 35231480 : ix86_asm_output_labelref (FILE *file, const char *prefix, const char *label)
1766 : {
1767 35231480 : if (ASSEMBLER_DIALECT == ASM_ATT)
1768 35230392 : fprintf (file, "%s%s", prefix, label);
1769 : else
1770 1088 : fprintf (file, "\"%s%s\"", prefix, label);
1771 35231480 : }
1772 :
1773 : /* Implementation of call abi switching target hook. Specific to FNDECL
1774 : the specific call register sets are set. See also
1775 : ix86_conditional_register_usage for more details. */
1776 : void
1777 203660771 : ix86_call_abi_override (const_tree fndecl)
1778 : {
1779 203660771 : cfun->machine->call_abi = ix86_function_abi (fndecl);
1780 203660771 : }
1781 :
1782 : /* Return 1 if pseudo register should be created and used to hold
1783 : GOT address for PIC code. */
1784 : bool
1785 166702577 : ix86_use_pseudo_pic_reg (void)
1786 : {
1787 166702577 : if ((TARGET_64BIT
1788 155644785 : && (ix86_cmodel == CM_SMALL_PIC
1789 : || TARGET_PECOFF))
1790 160996279 : || !flag_pic)
1791 161899083 : return false;
1792 : return true;
1793 : }
1794 :
1795 : /* Initialize large model PIC register. */
1796 :
1797 : static void
1798 56 : ix86_init_large_pic_reg (unsigned int tmp_regno)
1799 : {
1800 56 : rtx_code_label *label;
1801 56 : rtx tmp_reg;
1802 :
1803 56 : gcc_assert (Pmode == DImode);
1804 56 : label = gen_label_rtx ();
1805 56 : emit_label (label);
1806 56 : LABEL_PRESERVE_P (label) = 1;
1807 56 : tmp_reg = gen_rtx_REG (Pmode, tmp_regno);
1808 56 : gcc_assert (REGNO (pic_offset_table_rtx) != tmp_regno);
1809 56 : emit_insn (gen_set_rip_rex64 (pic_offset_table_rtx,
1810 : label));
1811 56 : emit_insn (gen_set_got_offset_rex64 (tmp_reg, label));
1812 56 : emit_insn (gen_add2_insn (pic_offset_table_rtx, tmp_reg));
1813 56 : const char *name = LABEL_NAME (label);
1814 56 : PUT_CODE (label, NOTE);
1815 56 : NOTE_KIND (label) = NOTE_INSN_DELETED_LABEL;
1816 56 : NOTE_DELETED_LABEL_NAME (label) = name;
1817 56 : }
1818 :
1819 : /* Create and initialize PIC register if required. */
1820 : static void
1821 1493940 : ix86_init_pic_reg (void)
1822 : {
1823 1493940 : edge entry_edge;
1824 1493940 : rtx_insn *seq;
1825 :
1826 1493940 : if (!ix86_use_pseudo_pic_reg ())
1827 : return;
1828 :
1829 40530 : start_sequence ();
1830 :
1831 40530 : if (TARGET_64BIT)
1832 : {
1833 69 : if (ix86_cmodel == CM_LARGE_PIC)
1834 53 : ix86_init_large_pic_reg (R11_REG);
1835 : else
1836 16 : emit_insn (gen_set_got_rex64 (pic_offset_table_rtx));
1837 : }
1838 : else
1839 : {
1840 : /* If there is future mcount call in the function it is more profitable
1841 : to emit SET_GOT into ABI defined REAL_PIC_OFFSET_TABLE_REGNUM. */
1842 40461 : rtx reg = crtl->profile
1843 40461 : ? gen_rtx_REG (Pmode, REAL_PIC_OFFSET_TABLE_REGNUM)
1844 40461 : : pic_offset_table_rtx;
1845 40461 : rtx_insn *insn = emit_insn (gen_set_got (reg));
1846 40461 : RTX_FRAME_RELATED_P (insn) = 1;
1847 40461 : if (crtl->profile)
1848 0 : emit_move_insn (pic_offset_table_rtx, reg);
1849 40461 : add_reg_note (insn, REG_CFA_FLUSH_QUEUE, NULL_RTX);
1850 : }
1851 :
1852 40530 : seq = end_sequence ();
1853 :
1854 40530 : entry_edge = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1855 40530 : insert_insn_on_edge (seq, entry_edge);
1856 40530 : commit_one_edge_insertion (entry_edge);
1857 : }
1858 :
1859 : /* Initialize a variable CUM of type CUMULATIVE_ARGS
1860 : for a call to a function whose data type is FNTYPE.
1861 : For a library call, FNTYPE is 0. */
1862 :
1863 : void
1864 10568346 : init_cumulative_args (CUMULATIVE_ARGS *cum, /* Argument info to initialize */
1865 : tree fntype, /* tree ptr for function decl */
1866 : rtx libname, /* SYMBOL_REF of library name or 0 */
1867 : tree fndecl,
1868 : int caller)
1869 : {
1870 10568346 : struct cgraph_node *local_info_node = NULL;
1871 10568346 : struct cgraph_node *target = NULL;
1872 :
1873 : /* Set silent_p to false to raise an error for invalid calls when
1874 : expanding function body. */
1875 10568346 : cfun->machine->silent_p = false;
1876 :
1877 10568346 : memset (cum, 0, sizeof (*cum));
1878 :
1879 10568346 : tree preserve_none_type;
1880 10568346 : if (fndecl)
1881 : {
1882 10222819 : target = cgraph_node::get (fndecl);
1883 10222819 : if (target)
1884 : {
1885 10071270 : target = target->function_symbol ();
1886 10071270 : local_info_node = cgraph_node::local_info_node (target->decl);
1887 10071270 : cum->call_abi = ix86_function_abi (target->decl);
1888 10071270 : preserve_none_type = TREE_TYPE (target->decl);
1889 : }
1890 : else
1891 : {
1892 151549 : cum->call_abi = ix86_function_abi (fndecl);
1893 151549 : preserve_none_type = TREE_TYPE (fndecl);
1894 : }
1895 : }
1896 : else
1897 : {
1898 345527 : cum->call_abi = ix86_function_type_abi (fntype);
1899 345527 : preserve_none_type = fntype;
1900 : }
1901 :
1902 : /* For MS ABI functions, parameter passing scheme is unchanged. */
1903 10568346 : cum->preserve_none_abi
1904 10568346 : = (preserve_none_type
1905 10446872 : && cum->call_abi != MS_ABI
1906 20898108 : && (lookup_attribute ("preserve_none",
1907 10329762 : TYPE_ATTRIBUTES (preserve_none_type))
1908 : != nullptr));
1909 :
1910 10568346 : cum->caller = caller;
1911 :
1912 : /* Set up the number of registers to use for passing arguments. */
1913 10568346 : cum->nregs = ix86_regparm;
1914 10568346 : if (TARGET_64BIT)
1915 : {
1916 9531322 : cum->nregs = (cum->call_abi == SYSV_ABI
1917 9531322 : ? X86_64_REGPARM_MAX
1918 : : X86_64_MS_REGPARM_MAX);
1919 : }
1920 10568346 : if (TARGET_SSE)
1921 : {
1922 10559175 : cum->sse_nregs = SSE_REGPARM_MAX;
1923 10559175 : if (TARGET_64BIT)
1924 : {
1925 9522271 : cum->sse_nregs = (cum->call_abi == SYSV_ABI
1926 9522271 : ? X86_64_SSE_REGPARM_MAX
1927 : : X86_64_MS_SSE_REGPARM_MAX);
1928 : }
1929 : }
1930 10568346 : if (TARGET_MMX)
1931 11392490 : cum->mmx_nregs = MMX_REGPARM_MAX;
1932 10568346 : cum->warn_avx512f = true;
1933 10568346 : cum->warn_avx = true;
1934 10568346 : cum->warn_sse = true;
1935 10568346 : cum->warn_mmx = true;
1936 :
1937 : /* Because type might mismatch in between caller and callee, we need to
1938 : use actual type of function for local calls.
1939 : FIXME: cgraph_analyze can be told to actually record if function uses
1940 : va_start so for local functions maybe_vaarg can be made aggressive
1941 : helping K&R code.
1942 : FIXME: once typesytem is fixed, we won't need this code anymore. */
1943 10568346 : if (local_info_node && local_info_node->local
1944 422329 : && local_info_node->can_change_signature)
1945 399101 : fntype = TREE_TYPE (target->decl);
1946 10568346 : cum->stdarg = stdarg_p (fntype);
1947 21136692 : cum->maybe_vaarg = (fntype
1948 11044781 : ? (!prototype_p (fntype) || stdarg_p (fntype))
1949 121474 : : !libname);
1950 :
1951 10568346 : cum->decl = fndecl;
1952 :
1953 10568346 : cum->warn_empty = !warn_abi || cum->stdarg;
1954 10568346 : if (!cum->warn_empty && fntype)
1955 : {
1956 2786204 : function_args_iterator iter;
1957 2786204 : tree argtype;
1958 2786204 : bool seen_empty_type = false;
1959 7763331 : FOREACH_FUNCTION_ARGS (fntype, argtype, iter)
1960 : {
1961 7763268 : if (argtype == error_mark_node || VOID_TYPE_P (argtype))
1962 : break;
1963 5000989 : if (TYPE_EMPTY_P (argtype))
1964 : seen_empty_type = true;
1965 4917878 : else if (seen_empty_type)
1966 : {
1967 23862 : cum->warn_empty = true;
1968 23862 : break;
1969 : }
1970 : }
1971 : }
1972 :
1973 10568346 : if (!TARGET_64BIT)
1974 : {
1975 : /* If there are variable arguments, then we won't pass anything
1976 : in registers in 32-bit mode. */
1977 1037024 : if (stdarg_p (fntype))
1978 : {
1979 9100 : cum->nregs = 0;
1980 : /* Since in 32-bit, variable arguments are always passed on
1981 : stack, there is scratch register available for indirect
1982 : sibcall. */
1983 9100 : cfun->machine->arg_reg_available = true;
1984 9100 : cum->sse_nregs = 0;
1985 9100 : cum->mmx_nregs = 0;
1986 9100 : cum->warn_avx512f = false;
1987 9100 : cum->warn_avx = false;
1988 9100 : cum->warn_sse = false;
1989 9100 : cum->warn_mmx = false;
1990 9100 : return;
1991 : }
1992 :
1993 : /* Use ecx and edx registers if function has fastcall attribute,
1994 : else look for regparm information. */
1995 1027924 : if (fntype)
1996 : {
1997 1014714 : unsigned int ccvt = ix86_get_callcvt (fntype);
1998 1014714 : if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
1999 : {
2000 0 : cum->nregs = 1;
2001 0 : cum->fastcall = 1; /* Same first register as in fastcall. */
2002 : }
2003 1014714 : else if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
2004 : {
2005 4 : cum->nregs = 2;
2006 4 : cum->fastcall = 1;
2007 : }
2008 : else
2009 1014710 : cum->nregs = ix86_function_regparm (fntype, fndecl);
2010 : }
2011 :
2012 : /* Set up the number of SSE registers used for passing SFmode
2013 : and DFmode arguments. Warn for mismatching ABI. */
2014 1027924 : cum->float_in_sse = ix86_function_sseregparm (fntype, fndecl, true);
2015 : }
2016 :
2017 10559246 : cfun->machine->arg_reg_available = (cum->nregs > 0);
2018 : }
2019 :
2020 : /* Return the "natural" mode for TYPE. In most cases, this is just TYPE_MODE.
2021 : But in the case of vector types, it is some vector mode.
2022 :
2023 : When we have only some of our vector isa extensions enabled, then there
2024 : are some modes for which vector_mode_supported_p is false. For these
2025 : modes, the generic vector support in gcc will choose some non-vector mode
2026 : in order to implement the type. By computing the natural mode, we'll
2027 : select the proper ABI location for the operand and not depend on whatever
2028 : the middle-end decides to do with these vector types.
2029 :
2030 : The midde-end can't deal with the vector types > 16 bytes. In this
2031 : case, we return the original mode and warn ABI change if CUM isn't
2032 : NULL.
2033 :
2034 : If INT_RETURN is true, warn ABI change if the vector mode isn't
2035 : available for function return value. */
2036 :
2037 : static machine_mode
2038 234362049 : type_natural_mode (const_tree type, const CUMULATIVE_ARGS *cum,
2039 : bool in_return)
2040 : {
2041 234362049 : machine_mode mode = TYPE_MODE (type);
2042 :
2043 234362049 : if (VECTOR_TYPE_P (type) && !VECTOR_MODE_P (mode))
2044 : {
2045 498314 : HOST_WIDE_INT size = int_size_in_bytes (type);
2046 498314 : if ((size == 8 || size == 16 || size == 32 || size == 64)
2047 : /* ??? Generic code allows us to create width 1 vectors. Ignore. */
2048 498314 : && TYPE_VECTOR_SUBPARTS (type) > 1)
2049 : {
2050 452264 : machine_mode innermode = TYPE_MODE (TREE_TYPE (type));
2051 :
2052 : /* There are no XFmode vector modes ... */
2053 452264 : if (innermode == XFmode)
2054 : return mode;
2055 :
2056 : /* ... and no decimal float vector modes. */
2057 451711 : if (DECIMAL_FLOAT_MODE_P (innermode))
2058 : return mode;
2059 :
2060 451418 : if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (type)))
2061 : mode = MIN_MODE_VECTOR_FLOAT;
2062 : else
2063 373071 : mode = MIN_MODE_VECTOR_INT;
2064 :
2065 : /* Get the mode which has this inner mode and number of units. */
2066 9478895 : FOR_EACH_MODE_FROM (mode, mode)
2067 19733596 : if (GET_MODE_NUNITS (mode) == TYPE_VECTOR_SUBPARTS (type)
2068 10706119 : && GET_MODE_INNER (mode) == innermode)
2069 : {
2070 451418 : if (size == 64 && !TARGET_AVX512F && !TARGET_IAMCU)
2071 : {
2072 300665 : static bool warnedavx512f;
2073 300665 : static bool warnedavx512f_ret;
2074 :
2075 300665 : if (cum && cum->warn_avx512f && !warnedavx512f)
2076 : {
2077 1361 : if (warning (OPT_Wpsabi, "AVX512F vector argument "
2078 : "without AVX512F enabled changes the ABI"))
2079 2 : warnedavx512f = true;
2080 : }
2081 299304 : else if (in_return && !warnedavx512f_ret)
2082 : {
2083 283951 : if (warning (OPT_Wpsabi, "AVX512F vector return "
2084 : "without AVX512F enabled changes the ABI"))
2085 8 : warnedavx512f_ret = true;
2086 : }
2087 :
2088 300665 : return TYPE_MODE (type);
2089 : }
2090 150753 : else if (size == 32 && !TARGET_AVX && !TARGET_IAMCU)
2091 : {
2092 150198 : static bool warnedavx;
2093 150198 : static bool warnedavx_ret;
2094 :
2095 150198 : if (cum && cum->warn_avx && !warnedavx)
2096 : {
2097 770 : if (warning (OPT_Wpsabi, "AVX vector argument "
2098 : "without AVX enabled changes the ABI"))
2099 5 : warnedavx = true;
2100 : }
2101 149428 : else if (in_return && !warnedavx_ret)
2102 : {
2103 121906 : if (warning (OPT_Wpsabi, "AVX vector return "
2104 : "without AVX enabled changes the ABI"))
2105 13 : warnedavx_ret = true;
2106 : }
2107 :
2108 150198 : return TYPE_MODE (type);
2109 : }
2110 555 : else if (((size == 8 && TARGET_64BIT) || size == 16)
2111 552 : && !TARGET_SSE
2112 140 : && !TARGET_IAMCU)
2113 : {
2114 140 : static bool warnedsse;
2115 140 : static bool warnedsse_ret;
2116 :
2117 140 : if (cum && cum->warn_sse && !warnedsse)
2118 : {
2119 19 : if (warning (OPT_Wpsabi, "SSE vector argument "
2120 : "without SSE enabled changes the ABI"))
2121 6 : warnedsse = true;
2122 : }
2123 121 : else if (!TARGET_64BIT && in_return && !warnedsse_ret)
2124 : {
2125 0 : if (warning (OPT_Wpsabi, "SSE vector return "
2126 : "without SSE enabled changes the ABI"))
2127 0 : warnedsse_ret = true;
2128 : }
2129 : }
2130 415 : else if ((size == 8 && !TARGET_64BIT)
2131 0 : && (!cfun
2132 0 : || cfun->machine->func_type == TYPE_NORMAL)
2133 0 : && !TARGET_MMX
2134 0 : && !TARGET_IAMCU)
2135 : {
2136 0 : static bool warnedmmx;
2137 0 : static bool warnedmmx_ret;
2138 :
2139 0 : if (cum && cum->warn_mmx && !warnedmmx)
2140 : {
2141 0 : if (warning (OPT_Wpsabi, "MMX vector argument "
2142 : "without MMX enabled changes the ABI"))
2143 0 : warnedmmx = true;
2144 : }
2145 0 : else if (in_return && !warnedmmx_ret)
2146 : {
2147 0 : if (warning (OPT_Wpsabi, "MMX vector return "
2148 : "without MMX enabled changes the ABI"))
2149 0 : warnedmmx_ret = true;
2150 : }
2151 : }
2152 555 : return mode;
2153 : }
2154 :
2155 0 : gcc_unreachable ();
2156 : }
2157 : }
2158 :
2159 : return mode;
2160 : }
2161 :
2162 : /* We want to pass a value in REGNO whose "natural" mode is MODE. However,
2163 : this may not agree with the mode that the type system has chosen for the
2164 : register, which is ORIG_MODE. If ORIG_MODE is not BLKmode, then we can
2165 : go ahead and use it. Otherwise we have to build a PARALLEL instead. */
2166 :
2167 : static rtx
2168 36779041 : gen_reg_or_parallel (machine_mode mode, machine_mode orig_mode,
2169 : unsigned int regno)
2170 : {
2171 36779041 : rtx tmp;
2172 :
2173 36779041 : if (orig_mode != BLKmode)
2174 36779012 : tmp = gen_rtx_REG (orig_mode, regno);
2175 : else
2176 : {
2177 29 : tmp = gen_rtx_REG (mode, regno);
2178 29 : tmp = gen_rtx_EXPR_LIST (VOIDmode, tmp, const0_rtx);
2179 29 : tmp = gen_rtx_PARALLEL (orig_mode, gen_rtvec (1, tmp));
2180 : }
2181 :
2182 36779041 : return tmp;
2183 : }
2184 :
2185 : /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
2186 : of this code is to classify each 8bytes of incoming argument by the register
2187 : class and assign registers accordingly. */
2188 :
2189 : /* Return the union class of CLASS1 and CLASS2.
2190 : See the x86-64 PS ABI for details. */
2191 :
2192 : static enum x86_64_reg_class
2193 58251615 : merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
2194 : {
2195 : /* Rule #1: If both classes are equal, this is the resulting class. */
2196 56974124 : if (class1 == class2)
2197 : return class1;
2198 :
2199 : /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
2200 : the other class. */
2201 50218597 : if (class1 == X86_64_NO_CLASS)
2202 : return class2;
2203 51063701 : if (class2 == X86_64_NO_CLASS)
2204 : return class1;
2205 :
2206 : /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
2207 1729706 : if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
2208 : return X86_64_MEMORY_CLASS;
2209 :
2210 : /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
2211 1573048 : if ((class1 == X86_64_INTEGERSI_CLASS
2212 191058 : && (class2 == X86_64_SSESF_CLASS || class2 == X86_64_SSEHF_CLASS))
2213 1571842 : || (class2 == X86_64_INTEGERSI_CLASS
2214 949976 : && (class1 == X86_64_SSESF_CLASS || class1 == X86_64_SSEHF_CLASS)))
2215 : return X86_64_INTEGERSI_CLASS;
2216 1567905 : if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
2217 412847 : || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
2218 : return X86_64_INTEGER_CLASS;
2219 :
2220 : /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
2221 : MEMORY is used. */
2222 100825 : if (class1 == X86_64_X87_CLASS
2223 : || class1 == X86_64_X87UP_CLASS
2224 100825 : || class1 == X86_64_COMPLEX_X87_CLASS
2225 : || class2 == X86_64_X87_CLASS
2226 99920 : || class2 == X86_64_X87UP_CLASS
2227 66968 : || class2 == X86_64_COMPLEX_X87_CLASS)
2228 33857 : return X86_64_MEMORY_CLASS;
2229 :
2230 : /* Rule #6: Otherwise class SSE is used. */
2231 : return X86_64_SSE_CLASS;
2232 : }
2233 :
2234 : /* Classify the argument of type TYPE and mode MODE.
2235 : CLASSES will be filled by the register class used to pass each word
2236 : of the operand. The number of words is returned. In case the parameter
2237 : should be passed in memory, 0 is returned. As a special case for zero
2238 : sized containers, classes[0] will be NO_CLASS and 1 is returned.
2239 :
2240 : BIT_OFFSET is used internally for handling records and specifies offset
2241 : of the offset in bits modulo 512 to avoid overflow cases.
2242 :
2243 : See the x86-64 PS ABI for details.
2244 : */
2245 :
2246 : static int
2247 401912349 : classify_argument (machine_mode mode, const_tree type,
2248 : enum x86_64_reg_class classes[MAX_CLASSES], int bit_offset,
2249 : int &zero_width_bitfields)
2250 : {
2251 401912349 : HOST_WIDE_INT bytes
2252 797461571 : = mode == BLKmode ? int_size_in_bytes (type) : (int) GET_MODE_SIZE (mode);
2253 401912349 : int words = CEIL (bytes + (bit_offset % 64) / 8, UNITS_PER_WORD);
2254 :
2255 : /* Variable sized entities are always passed/returned in memory. */
2256 401912349 : if (bytes < 0)
2257 : return 0;
2258 :
2259 401911150 : if (mode != VOIDmode)
2260 : {
2261 : /* The value of "named" doesn't matter. */
2262 400669711 : function_arg_info arg (const_cast<tree> (type), mode, /*named=*/true);
2263 400669711 : if (targetm.calls.must_pass_in_stack (arg))
2264 37 : return 0;
2265 : }
2266 :
2267 401911113 : if (type && (AGGREGATE_TYPE_P (type)
2268 363801539 : || (BITINT_TYPE_P (type) && words > 1)))
2269 : {
2270 39239159 : int i;
2271 39239159 : tree field;
2272 39239159 : enum x86_64_reg_class subclasses[MAX_CLASSES];
2273 :
2274 : /* On x86-64 we pass structures larger than 64 bytes on the stack. */
2275 39239159 : if (bytes > 64)
2276 : return 0;
2277 :
2278 98435685 : for (i = 0; i < words; i++)
2279 60043511 : classes[i] = X86_64_NO_CLASS;
2280 :
2281 : /* Zero sized arrays or structures are NO_CLASS. We return 0 to
2282 : signalize memory class, so handle it as special case. */
2283 38392174 : if (!words)
2284 : {
2285 82983 : classes[0] = X86_64_NO_CLASS;
2286 82983 : return 1;
2287 : }
2288 :
2289 : /* Classify each field of record and merge classes. */
2290 38309191 : switch (TREE_CODE (type))
2291 : {
2292 36198707 : case RECORD_TYPE:
2293 : /* And now merge the fields of structure. */
2294 1002171696 : for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
2295 : {
2296 966524212 : if (TREE_CODE (field) == FIELD_DECL)
2297 : {
2298 53178475 : int num;
2299 :
2300 53178475 : if (TREE_TYPE (field) == error_mark_node)
2301 4 : continue;
2302 :
2303 : /* Bitfields are always classified as integer. Handle them
2304 : early, since later code would consider them to be
2305 : misaligned integers. */
2306 53178471 : if (DECL_BIT_FIELD (field))
2307 : {
2308 1286710 : if (integer_zerop (DECL_SIZE (field)))
2309 : {
2310 12839 : if (DECL_FIELD_CXX_ZERO_WIDTH_BIT_FIELD (field))
2311 8021 : continue;
2312 4818 : if (zero_width_bitfields != 2)
2313 : {
2314 4284 : zero_width_bitfields = 1;
2315 4284 : continue;
2316 : }
2317 : }
2318 1274405 : for (i = (int_bit_position (field)
2319 1274405 : + (bit_offset % 64)) / 8 / 8;
2320 2551896 : i < ((int_bit_position (field) + (bit_offset % 64))
2321 2551896 : + tree_to_shwi (DECL_SIZE (field))
2322 2551896 : + 63) / 8 / 8; i++)
2323 1277491 : classes[i]
2324 2554982 : = merge_classes (X86_64_INTEGER_CLASS, classes[i]);
2325 : }
2326 : else
2327 : {
2328 51891761 : int pos;
2329 :
2330 51891761 : type = TREE_TYPE (field);
2331 :
2332 : /* Flexible array member is ignored. */
2333 51891761 : if (TYPE_MODE (type) == BLKmode
2334 697589 : && TREE_CODE (type) == ARRAY_TYPE
2335 178027 : && TYPE_SIZE (type) == NULL_TREE
2336 2007 : && TYPE_DOMAIN (type) != NULL_TREE
2337 51893003 : && (TYPE_MAX_VALUE (TYPE_DOMAIN (type))
2338 : == NULL_TREE))
2339 : {
2340 1242 : static bool warned;
2341 :
2342 1242 : if (!warned && warn_psabi)
2343 : {
2344 3 : warned = true;
2345 3 : inform (input_location,
2346 : "the ABI of passing struct with"
2347 : " a flexible array member has"
2348 : " changed in GCC 4.4");
2349 : }
2350 1242 : continue;
2351 1242 : }
2352 51890519 : num = classify_argument (TYPE_MODE (type), type,
2353 : subclasses,
2354 51890519 : (int_bit_position (field)
2355 51890519 : + bit_offset) % 512,
2356 : zero_width_bitfields);
2357 51890519 : if (!num)
2358 : return 0;
2359 51339296 : pos = (int_bit_position (field)
2360 51339296 : + (bit_offset % 64)) / 8 / 8;
2361 106272462 : for (i = 0; i < num && (i + pos) < words; i++)
2362 54933166 : classes[i + pos]
2363 54933166 : = merge_classes (subclasses[i], classes[i + pos]);
2364 : }
2365 : }
2366 : }
2367 : break;
2368 :
2369 491211 : case ARRAY_TYPE:
2370 : /* Arrays are handled as small records. */
2371 491211 : {
2372 491211 : int num;
2373 491211 : num = classify_argument (TYPE_MODE (TREE_TYPE (type)),
2374 491211 : TREE_TYPE (type), subclasses, bit_offset,
2375 : zero_width_bitfields);
2376 491211 : if (!num)
2377 : return 0;
2378 :
2379 : /* The partial classes are now full classes. */
2380 474929 : if (subclasses[0] == X86_64_SSESF_CLASS && bytes != 4)
2381 13919 : subclasses[0] = X86_64_SSE_CLASS;
2382 474929 : if (subclasses[0] == X86_64_SSEHF_CLASS && bytes != 2)
2383 5292 : subclasses[0] = X86_64_SSE_CLASS;
2384 474929 : if (subclasses[0] == X86_64_INTEGERSI_CLASS
2385 174517 : && !((bit_offset % 64) == 0 && bytes == 4))
2386 137405 : subclasses[0] = X86_64_INTEGER_CLASS;
2387 :
2388 1439080 : for (i = 0; i < words; i++)
2389 964151 : classes[i] = subclasses[i % num];
2390 :
2391 : break;
2392 : }
2393 283958 : case UNION_TYPE:
2394 283958 : case QUAL_UNION_TYPE:
2395 : /* Unions are similar to RECORD_TYPE but offset is always 0.
2396 : */
2397 3277632 : for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
2398 : {
2399 3028980 : if (TREE_CODE (field) == FIELD_DECL)
2400 : {
2401 1300153 : int num;
2402 :
2403 1300153 : if (TREE_TYPE (field) == error_mark_node)
2404 10 : continue;
2405 :
2406 1300143 : num = classify_argument (TYPE_MODE (TREE_TYPE (field)),
2407 1300143 : TREE_TYPE (field), subclasses,
2408 : bit_offset, zero_width_bitfields);
2409 1300143 : if (!num)
2410 : return 0;
2411 3305795 : for (i = 0; i < num && i < words; i++)
2412 2040958 : classes[i] = merge_classes (subclasses[i], classes[i]);
2413 : }
2414 : }
2415 : break;
2416 :
2417 1335315 : case BITINT_TYPE:
2418 1335315 : case ENUMERAL_TYPE:
2419 : /* _BitInt(N) for N > 64 is passed as structure containing
2420 : (N + 63) / 64 64-bit elements. */
2421 1335315 : if (words > 2)
2422 : return 0;
2423 77361 : classes[0] = classes[1] = X86_64_INTEGER_CLASS;
2424 77361 : return 2;
2425 :
2426 0 : default:
2427 0 : gcc_unreachable ();
2428 : }
2429 :
2430 36371065 : if (words > 2)
2431 : {
2432 : /* When size > 16 bytes, if the first one isn't
2433 : X86_64_SSE_CLASS or any other ones aren't
2434 : X86_64_SSEUP_CLASS, everything should be passed in
2435 : memory. */
2436 1710068 : if (classes[0] != X86_64_SSE_CLASS)
2437 : return 0;
2438 :
2439 203643 : for (i = 1; i < words; i++)
2440 185456 : if (classes[i] != X86_64_SSEUP_CLASS)
2441 : return 0;
2442 : }
2443 :
2444 : /* Final merger cleanup. */
2445 81417710 : for (i = 0; i < words; i++)
2446 : {
2447 : /* If one class is MEMORY, everything should be passed in
2448 : memory. */
2449 46774040 : if (classes[i] == X86_64_MEMORY_CLASS)
2450 : return 0;
2451 :
2452 : /* The X86_64_SSEUP_CLASS should be always preceded by
2453 : X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
2454 46740896 : if (classes[i] == X86_64_SSEUP_CLASS
2455 297596 : && classes[i - 1] != X86_64_SSE_CLASS
2456 76546 : && classes[i - 1] != X86_64_SSEUP_CLASS)
2457 : {
2458 : /* The first one should never be X86_64_SSEUP_CLASS. */
2459 1916 : gcc_assert (i != 0);
2460 1916 : classes[i] = X86_64_SSE_CLASS;
2461 : }
2462 :
2463 : /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
2464 : everything should be passed in memory. */
2465 46740896 : if (classes[i] == X86_64_X87UP_CLASS
2466 184113 : && (classes[i - 1] != X86_64_X87_CLASS))
2467 : {
2468 2370 : static bool warned;
2469 :
2470 : /* The first one should never be X86_64_X87UP_CLASS. */
2471 2370 : gcc_assert (i != 0);
2472 2370 : if (!warned && warn_psabi)
2473 : {
2474 1 : warned = true;
2475 1 : inform (input_location,
2476 : "the ABI of passing union with %<long double%>"
2477 : " has changed in GCC 4.4");
2478 : }
2479 2370 : return 0;
2480 : }
2481 : }
2482 : return words;
2483 : }
2484 :
2485 : /* Compute alignment needed. We align all types to natural boundaries with
2486 : exception of XFmode that is aligned to 64bits. */
2487 362671954 : if (mode != VOIDmode && mode != BLKmode)
2488 : {
2489 360935714 : int mode_alignment = GET_MODE_BITSIZE (mode);
2490 :
2491 360935714 : if (mode == XFmode)
2492 : mode_alignment = 128;
2493 353842484 : else if (mode == XCmode)
2494 564953 : mode_alignment = 256;
2495 360935714 : if (COMPLEX_MODE_P (mode))
2496 2714147 : mode_alignment /= 2;
2497 : /* Misaligned fields are always returned in memory. */
2498 360935714 : if (bit_offset % mode_alignment)
2499 : return 0;
2500 : }
2501 :
2502 : /* for V1xx modes, just use the base mode */
2503 362664321 : if (VECTOR_MODE_P (mode) && mode != V1DImode && mode != V1TImode
2504 456429581 : && GET_MODE_UNIT_SIZE (mode) == bytes)
2505 6651 : mode = GET_MODE_INNER (mode);
2506 :
2507 : /* Classification of atomic types. */
2508 362664321 : switch (mode)
2509 : {
2510 207987 : case E_SDmode:
2511 207987 : case E_DDmode:
2512 207987 : classes[0] = X86_64_SSE_CLASS;
2513 207987 : return 1;
2514 99137 : case E_TDmode:
2515 99137 : classes[0] = X86_64_SSE_CLASS;
2516 99137 : classes[1] = X86_64_SSEUP_CLASS;
2517 99137 : return 2;
2518 238180590 : case E_DImode:
2519 238180590 : case E_SImode:
2520 238180590 : case E_HImode:
2521 238180590 : case E_QImode:
2522 238180590 : case E_CSImode:
2523 238180590 : case E_CHImode:
2524 238180590 : case E_CQImode:
2525 238180590 : {
2526 238180590 : int size = bit_offset + (int) GET_MODE_BITSIZE (mode);
2527 :
2528 : /* Analyze last 128 bits only. */
2529 238180590 : size = (size - 1) & 0x7f;
2530 :
2531 238180590 : if (size < 32)
2532 : {
2533 105132948 : classes[0] = X86_64_INTEGERSI_CLASS;
2534 105132948 : return 1;
2535 : }
2536 133047642 : else if (size < 64)
2537 : {
2538 121851611 : classes[0] = X86_64_INTEGER_CLASS;
2539 121851611 : return 1;
2540 : }
2541 11196031 : else if (size < 64+32)
2542 : {
2543 4160779 : classes[0] = X86_64_INTEGER_CLASS;
2544 4160779 : classes[1] = X86_64_INTEGERSI_CLASS;
2545 4160779 : return 2;
2546 : }
2547 7035252 : else if (size < 64+64)
2548 : {
2549 7035252 : classes[0] = classes[1] = X86_64_INTEGER_CLASS;
2550 7035252 : return 2;
2551 : }
2552 : else
2553 : gcc_unreachable ();
2554 : }
2555 2524550 : case E_CDImode:
2556 2524550 : case E_TImode:
2557 2524550 : classes[0] = classes[1] = X86_64_INTEGER_CLASS;
2558 2524550 : return 2;
2559 0 : case E_COImode:
2560 0 : case E_OImode:
2561 : /* OImode shouldn't be used directly. */
2562 0 : gcc_unreachable ();
2563 : case E_CTImode:
2564 : return 0;
2565 1013660 : case E_HFmode:
2566 1013660 : case E_BFmode:
2567 1013660 : if (!(bit_offset % 64))
2568 1000446 : classes[0] = X86_64_SSEHF_CLASS;
2569 : else
2570 13214 : classes[0] = X86_64_SSE_CLASS;
2571 : return 1;
2572 10022375 : case E_SFmode:
2573 10022375 : if (!(bit_offset % 64))
2574 9968796 : classes[0] = X86_64_SSESF_CLASS;
2575 : else
2576 53579 : classes[0] = X86_64_SSE_CLASS;
2577 : return 1;
2578 4406147 : case E_DFmode:
2579 4406147 : classes[0] = X86_64_SSEDF_CLASS;
2580 4406147 : return 1;
2581 7092514 : case E_XFmode:
2582 7092514 : classes[0] = X86_64_X87_CLASS;
2583 7092514 : classes[1] = X86_64_X87UP_CLASS;
2584 7092514 : return 2;
2585 1384226 : case E_TFmode:
2586 1384226 : classes[0] = X86_64_SSE_CLASS;
2587 1384226 : classes[1] = X86_64_SSEUP_CLASS;
2588 1384226 : return 2;
2589 104641 : case E_HCmode:
2590 104641 : case E_BCmode:
2591 104641 : classes[0] = X86_64_SSE_CLASS;
2592 104641 : if (!(bit_offset % 64))
2593 : return 1;
2594 : else
2595 : {
2596 98 : classes[1] = X86_64_SSEHF_CLASS;
2597 98 : return 2;
2598 : }
2599 724049 : case E_SCmode:
2600 724049 : classes[0] = X86_64_SSE_CLASS;
2601 724049 : if (!(bit_offset % 64))
2602 : return 1;
2603 : else
2604 : {
2605 1119 : static bool warned;
2606 :
2607 1119 : if (!warned && warn_psabi)
2608 : {
2609 2 : warned = true;
2610 2 : inform (input_location,
2611 : "the ABI of passing structure with %<complex float%>"
2612 : " member has changed in GCC 4.4");
2613 : }
2614 1119 : classes[1] = X86_64_SSESF_CLASS;
2615 1119 : return 2;
2616 : }
2617 733961 : case E_DCmode:
2618 733961 : classes[0] = X86_64_SSEDF_CLASS;
2619 733961 : classes[1] = X86_64_SSEDF_CLASS;
2620 733961 : return 2;
2621 564953 : case E_XCmode:
2622 564953 : classes[0] = X86_64_COMPLEX_X87_CLASS;
2623 564953 : return 1;
2624 : case E_TCmode:
2625 : /* This modes is larger than 16 bytes. */
2626 : return 0;
2627 25382110 : case E_V8SFmode:
2628 25382110 : case E_V8SImode:
2629 25382110 : case E_V32QImode:
2630 25382110 : case E_V16HFmode:
2631 25382110 : case E_V16BFmode:
2632 25382110 : case E_V16HImode:
2633 25382110 : case E_V4DFmode:
2634 25382110 : case E_V4DImode:
2635 25382110 : classes[0] = X86_64_SSE_CLASS;
2636 25382110 : classes[1] = X86_64_SSEUP_CLASS;
2637 25382110 : classes[2] = X86_64_SSEUP_CLASS;
2638 25382110 : classes[3] = X86_64_SSEUP_CLASS;
2639 25382110 : return 4;
2640 27521081 : case E_V8DFmode:
2641 27521081 : case E_V16SFmode:
2642 27521081 : case E_V32HFmode:
2643 27521081 : case E_V32BFmode:
2644 27521081 : case E_V8DImode:
2645 27521081 : case E_V16SImode:
2646 27521081 : case E_V32HImode:
2647 27521081 : case E_V64QImode:
2648 27521081 : classes[0] = X86_64_SSE_CLASS;
2649 27521081 : classes[1] = X86_64_SSEUP_CLASS;
2650 27521081 : classes[2] = X86_64_SSEUP_CLASS;
2651 27521081 : classes[3] = X86_64_SSEUP_CLASS;
2652 27521081 : classes[4] = X86_64_SSEUP_CLASS;
2653 27521081 : classes[5] = X86_64_SSEUP_CLASS;
2654 27521081 : classes[6] = X86_64_SSEUP_CLASS;
2655 27521081 : classes[7] = X86_64_SSEUP_CLASS;
2656 27521081 : return 8;
2657 37533395 : case E_V4SFmode:
2658 37533395 : case E_V4SImode:
2659 37533395 : case E_V16QImode:
2660 37533395 : case E_V8HImode:
2661 37533395 : case E_V8HFmode:
2662 37533395 : case E_V8BFmode:
2663 37533395 : case E_V2DFmode:
2664 37533395 : case E_V2DImode:
2665 37533395 : classes[0] = X86_64_SSE_CLASS;
2666 37533395 : classes[1] = X86_64_SSEUP_CLASS;
2667 37533395 : return 2;
2668 3289360 : case E_V1TImode:
2669 3289360 : case E_V1DImode:
2670 3289360 : case E_V2SFmode:
2671 3289360 : case E_V2SImode:
2672 3289360 : case E_V4HImode:
2673 3289360 : case E_V4HFmode:
2674 3289360 : case E_V4BFmode:
2675 3289360 : case E_V2HFmode:
2676 3289360 : case E_V2BFmode:
2677 3289360 : case E_V8QImode:
2678 3289360 : classes[0] = X86_64_SSE_CLASS;
2679 3289360 : return 1;
2680 : case E_BLKmode:
2681 : case E_VOIDmode:
2682 : return 0;
2683 52806 : default:
2684 52806 : gcc_assert (VECTOR_MODE_P (mode));
2685 :
2686 52806 : if (bytes > 16)
2687 : return 0;
2688 :
2689 75884 : gcc_assert (GET_MODE_CLASS (GET_MODE_INNER (mode)) == MODE_INT);
2690 :
2691 75884 : if (bit_offset + GET_MODE_BITSIZE (mode) <= 32)
2692 37508 : classes[0] = X86_64_INTEGERSI_CLASS;
2693 : else
2694 434 : classes[0] = X86_64_INTEGER_CLASS;
2695 37942 : classes[1] = X86_64_INTEGER_CLASS;
2696 37942 : return 1 + (bytes > 8);
2697 : }
2698 : }
2699 :
2700 : /* Wrapper around classify_argument with the extra zero_width_bitfields
2701 : argument, to diagnose GCC 12.1 ABI differences for C. */
2702 :
2703 : static int
2704 348229942 : classify_argument (machine_mode mode, const_tree type,
2705 : enum x86_64_reg_class classes[MAX_CLASSES], int bit_offset)
2706 : {
2707 348229942 : int zero_width_bitfields = 0;
2708 348229942 : static bool warned = false;
2709 348229942 : int n = classify_argument (mode, type, classes, bit_offset,
2710 : zero_width_bitfields);
2711 348229942 : if (!zero_width_bitfields || warned || !warn_psabi)
2712 : return n;
2713 534 : enum x86_64_reg_class alt_classes[MAX_CLASSES];
2714 534 : zero_width_bitfields = 2;
2715 534 : if (classify_argument (mode, type, alt_classes, bit_offset,
2716 : zero_width_bitfields) != n)
2717 0 : zero_width_bitfields = 3;
2718 : else
2719 1286 : for (int i = 0; i < n; i++)
2720 760 : if (classes[i] != alt_classes[i])
2721 : {
2722 8 : zero_width_bitfields = 3;
2723 8 : break;
2724 : }
2725 534 : if (zero_width_bitfields == 3)
2726 : {
2727 8 : warned = true;
2728 8 : const char *url
2729 : = CHANGES_ROOT_URL "gcc-12/changes.html#zero_width_bitfields";
2730 :
2731 8 : inform (input_location,
2732 : "the ABI of passing C structures with zero-width bit-fields"
2733 : " has changed in GCC %{12.1%}", url);
2734 : }
2735 : return n;
2736 : }
2737 :
2738 : /* Examine the argument and return set number of register required in each
2739 : class. Return true iff parameter should be passed in memory. */
2740 :
2741 : static bool
2742 235668175 : examine_argument (machine_mode mode, const_tree type, bool in_return,
2743 : int *int_nregs, int *sse_nregs)
2744 : {
2745 235668175 : enum x86_64_reg_class regclass[MAX_CLASSES];
2746 235668175 : int n = classify_argument (mode, type, regclass, 0);
2747 :
2748 235668175 : *int_nregs = 0;
2749 235668175 : *sse_nregs = 0;
2750 :
2751 235668175 : if (!n)
2752 : return true;
2753 677621787 : for (n--; n >= 0; n--)
2754 447737712 : switch (regclass[n])
2755 : {
2756 157814586 : case X86_64_INTEGER_CLASS:
2757 157814586 : case X86_64_INTEGERSI_CLASS:
2758 157814586 : (*int_nregs)++;
2759 157814586 : break;
2760 75150412 : case X86_64_SSE_CLASS:
2761 75150412 : case X86_64_SSEHF_CLASS:
2762 75150412 : case X86_64_SSESF_CLASS:
2763 75150412 : case X86_64_SSEDF_CLASS:
2764 75150412 : (*sse_nregs)++;
2765 75150412 : break;
2766 : case X86_64_NO_CLASS:
2767 : case X86_64_SSEUP_CLASS:
2768 : break;
2769 9685631 : case X86_64_X87_CLASS:
2770 9685631 : case X86_64_X87UP_CLASS:
2771 9685631 : case X86_64_COMPLEX_X87_CLASS:
2772 9685631 : if (!in_return)
2773 : return true;
2774 : break;
2775 0 : case X86_64_MEMORY_CLASS:
2776 0 : gcc_unreachable ();
2777 : }
2778 :
2779 : return false;
2780 : }
2781 :
2782 : /* Construct container for the argument used by GCC interface. See
2783 : FUNCTION_ARG for the detailed description. */
2784 :
2785 : static rtx
2786 114180410 : construct_container (machine_mode mode, machine_mode orig_mode,
2787 : const_tree type, bool in_return, int nintregs,
2788 : int nsseregs, const int *intreg, int sse_regno)
2789 : {
2790 : /* The following variables hold the static issued_error state. */
2791 114180410 : static bool issued_sse_arg_error;
2792 114180410 : static bool issued_sse_ret_error;
2793 114180410 : static bool issued_x87_ret_error;
2794 :
2795 114180410 : machine_mode tmpmode;
2796 114180410 : int bytes
2797 227665661 : = mode == BLKmode ? int_size_in_bytes (type) : (int) GET_MODE_SIZE (mode);
2798 114180410 : enum x86_64_reg_class regclass[MAX_CLASSES];
2799 114180410 : int n;
2800 114180410 : int i;
2801 114180410 : int nexps = 0;
2802 114180410 : int needed_sseregs, needed_intregs;
2803 114180410 : rtx exp[MAX_CLASSES];
2804 114180410 : rtx ret;
2805 :
2806 114180410 : if (examine_argument (mode, type, in_return, &needed_intregs,
2807 : &needed_sseregs))
2808 : return NULL;
2809 :
2810 113667993 : if (needed_intregs > nintregs || needed_sseregs > nsseregs)
2811 : return NULL;
2812 :
2813 : /* We allowed the user to turn off SSE for kernel mode. Don't crash if
2814 : some less clueful developer tries to use floating-point anyway. */
2815 112561835 : if (needed_sseregs
2816 37127004 : && (!TARGET_SSE || (VALID_SSE2_TYPE_MODE (mode) && !TARGET_SSE2)))
2817 : {
2818 : /* Return early if we shouldn't raise an error for invalid
2819 : calls. */
2820 68 : if (cfun != NULL && cfun->machine->silent_p)
2821 : return NULL;
2822 36 : if (in_return)
2823 : {
2824 31 : if (!issued_sse_ret_error)
2825 : {
2826 13 : if (VALID_SSE2_TYPE_MODE (mode))
2827 5 : error ("SSE register return with SSE2 disabled");
2828 : else
2829 8 : error ("SSE register return with SSE disabled");
2830 13 : issued_sse_ret_error = true;
2831 : }
2832 : }
2833 5 : else if (!issued_sse_arg_error)
2834 : {
2835 5 : if (VALID_SSE2_TYPE_MODE (mode))
2836 0 : error ("SSE register argument with SSE2 disabled");
2837 : else
2838 5 : error ("SSE register argument with SSE disabled");
2839 5 : issued_sse_arg_error = true;
2840 : }
2841 36 : return NULL;
2842 : }
2843 :
2844 112561767 : n = classify_argument (mode, type, regclass, 0);
2845 112561767 : gcc_assert (n);
2846 :
2847 : /* Likewise, error if the ABI requires us to return values in the
2848 : x87 registers and the user specified -mno-80387. */
2849 112561767 : if (!TARGET_FLOAT_RETURNS_IN_80387 && in_return)
2850 1428805 : for (i = 0; i < n; i++)
2851 754012 : if (regclass[i] == X86_64_X87_CLASS
2852 : || regclass[i] == X86_64_X87UP_CLASS
2853 754012 : || regclass[i] == X86_64_COMPLEX_X87_CLASS)
2854 : {
2855 : /* Return early if we shouldn't raise an error for invalid
2856 : calls. */
2857 13 : if (cfun != NULL && cfun->machine->silent_p)
2858 : return NULL;
2859 10 : if (!issued_x87_ret_error)
2860 : {
2861 5 : error ("x87 register return with x87 disabled");
2862 5 : issued_x87_ret_error = true;
2863 : }
2864 10 : return NULL;
2865 : }
2866 :
2867 : /* First construct simple cases. Avoid SCmode, since we want to use
2868 : single register to pass this type. */
2869 112561754 : if (n == 1 && mode != SCmode && mode != HCmode)
2870 75313228 : switch (regclass[0])
2871 : {
2872 69057657 : case X86_64_INTEGER_CLASS:
2873 69057657 : case X86_64_INTEGERSI_CLASS:
2874 69057657 : return gen_rtx_REG (mode, intreg[0]);
2875 6050171 : case X86_64_SSE_CLASS:
2876 6050171 : case X86_64_SSEHF_CLASS:
2877 6050171 : case X86_64_SSESF_CLASS:
2878 6050171 : case X86_64_SSEDF_CLASS:
2879 6050171 : if (mode != BLKmode)
2880 12099482 : return gen_reg_or_parallel (mode, orig_mode,
2881 12099482 : GET_SSE_REGNO (sse_regno));
2882 : break;
2883 176074 : case X86_64_X87_CLASS:
2884 176074 : case X86_64_COMPLEX_X87_CLASS:
2885 176074 : return gen_rtx_REG (mode, FIRST_STACK_REG);
2886 : case X86_64_NO_CLASS:
2887 : /* Zero sized array, struct or class. */
2888 : return NULL;
2889 0 : default:
2890 0 : gcc_unreachable ();
2891 : }
2892 37248956 : if (n == 2
2893 19407041 : && regclass[0] == X86_64_SSE_CLASS
2894 12973894 : && regclass[1] == X86_64_SSEUP_CLASS
2895 12968291 : && mode != BLKmode)
2896 25936582 : return gen_reg_or_parallel (mode, orig_mode,
2897 25936582 : GET_SSE_REGNO (sse_regno));
2898 24280665 : if (n == 4
2899 8444688 : && regclass[0] == X86_64_SSE_CLASS
2900 8444688 : && regclass[1] == X86_64_SSEUP_CLASS
2901 8444688 : && regclass[2] == X86_64_SSEUP_CLASS
2902 8444688 : && regclass[3] == X86_64_SSEUP_CLASS
2903 8444688 : && mode != BLKmode)
2904 16885998 : return gen_reg_or_parallel (mode, orig_mode,
2905 16885998 : GET_SSE_REGNO (sse_regno));
2906 15837666 : if (n == 8
2907 9143609 : && regclass[0] == X86_64_SSE_CLASS
2908 9143609 : && regclass[1] == X86_64_SSEUP_CLASS
2909 9143609 : && regclass[2] == X86_64_SSEUP_CLASS
2910 9143609 : && regclass[3] == X86_64_SSEUP_CLASS
2911 9143609 : && regclass[4] == X86_64_SSEUP_CLASS
2912 9143609 : && regclass[5] == X86_64_SSEUP_CLASS
2913 9143609 : && regclass[6] == X86_64_SSEUP_CLASS
2914 9143609 : && regclass[7] == X86_64_SSEUP_CLASS
2915 9143609 : && mode != BLKmode)
2916 18282946 : return gen_reg_or_parallel (mode, orig_mode,
2917 18282946 : GET_SSE_REGNO (sse_regno));
2918 6696193 : if (n == 2
2919 6438750 : && regclass[0] == X86_64_X87_CLASS
2920 2314067 : && regclass[1] == X86_64_X87UP_CLASS)
2921 2314067 : return gen_rtx_REG (XFmode, FIRST_STACK_REG);
2922 :
2923 4382126 : if (n == 2
2924 4124683 : && regclass[0] == X86_64_INTEGER_CLASS
2925 3683082 : && regclass[1] == X86_64_INTEGER_CLASS
2926 3674641 : && (mode == CDImode || mode == TImode || mode == BLKmode)
2927 3674641 : && intreg[0] + 1 == intreg[1])
2928 : {
2929 3354223 : if (mode == BLKmode)
2930 : {
2931 : /* Use TImode for BLKmode values in 2 integer registers. */
2932 533044 : exp[0] = gen_rtx_EXPR_LIST (VOIDmode,
2933 266522 : gen_rtx_REG (TImode, intreg[0]),
2934 : GEN_INT (0));
2935 266522 : ret = gen_rtx_PARALLEL (mode, rtvec_alloc (1));
2936 266522 : XVECEXP (ret, 0, 0) = exp[0];
2937 266522 : return ret;
2938 : }
2939 : else
2940 3087701 : return gen_rtx_REG (mode, intreg[0]);
2941 : }
2942 :
2943 : /* Otherwise figure out the entries of the PARALLEL. */
2944 2826266 : for (i = 0; i < n; i++)
2945 : {
2946 1798363 : int pos;
2947 :
2948 1798363 : switch (regclass[i])
2949 : {
2950 : case X86_64_NO_CLASS:
2951 : break;
2952 1010303 : case X86_64_INTEGER_CLASS:
2953 1010303 : case X86_64_INTEGERSI_CLASS:
2954 : /* Merge TImodes on aligned occasions here too. */
2955 1010303 : if (i * 8 + 8 > bytes)
2956 : {
2957 3239 : unsigned int tmpbits = (bytes - i * 8) * BITS_PER_UNIT;
2958 3239 : if (!int_mode_for_size (tmpbits, 0).exists (&tmpmode))
2959 : /* We've requested 24 bytes we
2960 : don't have mode for. Use DImode. */
2961 357 : tmpmode = DImode;
2962 : }
2963 1007064 : else if (regclass[i] == X86_64_INTEGERSI_CLASS)
2964 : tmpmode = SImode;
2965 : else
2966 828153 : tmpmode = DImode;
2967 2020606 : exp [nexps++]
2968 1010303 : = gen_rtx_EXPR_LIST (VOIDmode,
2969 1010303 : gen_rtx_REG (tmpmode, *intreg),
2970 1010303 : GEN_INT (i*8));
2971 1010303 : intreg++;
2972 1010303 : break;
2973 600 : case X86_64_SSEHF_CLASS:
2974 600 : tmpmode = (mode == BFmode ? BFmode : HFmode);
2975 1200 : exp [nexps++]
2976 1200 : = gen_rtx_EXPR_LIST (VOIDmode,
2977 : gen_rtx_REG (tmpmode,
2978 600 : GET_SSE_REGNO (sse_regno)),
2979 600 : GEN_INT (i*8));
2980 600 : sse_regno++;
2981 600 : break;
2982 3060 : case X86_64_SSESF_CLASS:
2983 6120 : exp [nexps++]
2984 6120 : = gen_rtx_EXPR_LIST (VOIDmode,
2985 : gen_rtx_REG (SFmode,
2986 3060 : GET_SSE_REGNO (sse_regno)),
2987 3060 : GEN_INT (i*8));
2988 3060 : sse_regno++;
2989 3060 : break;
2990 507696 : case X86_64_SSEDF_CLASS:
2991 1015392 : exp [nexps++]
2992 1015392 : = gen_rtx_EXPR_LIST (VOIDmode,
2993 : gen_rtx_REG (DFmode,
2994 507696 : GET_SSE_REGNO (sse_regno)),
2995 507696 : GEN_INT (i*8));
2996 507696 : sse_regno++;
2997 507696 : break;
2998 268253 : case X86_64_SSE_CLASS:
2999 268253 : pos = i;
3000 268253 : switch (n)
3001 : {
3002 : case 1:
3003 : tmpmode = DImode;
3004 : break;
3005 11024 : case 2:
3006 11024 : if (i == 0 && regclass[1] == X86_64_SSEUP_CLASS)
3007 : {
3008 0 : tmpmode = TImode;
3009 0 : i++;
3010 : }
3011 : else
3012 : tmpmode = DImode;
3013 : break;
3014 1689 : case 4:
3015 1689 : gcc_assert (i == 0
3016 : && regclass[1] == X86_64_SSEUP_CLASS
3017 : && regclass[2] == X86_64_SSEUP_CLASS
3018 : && regclass[3] == X86_64_SSEUP_CLASS);
3019 : tmpmode = OImode;
3020 : i += 3;
3021 : break;
3022 2136 : case 8:
3023 2136 : gcc_assert (i == 0
3024 : && regclass[1] == X86_64_SSEUP_CLASS
3025 : && regclass[2] == X86_64_SSEUP_CLASS
3026 : && regclass[3] == X86_64_SSEUP_CLASS
3027 : && regclass[4] == X86_64_SSEUP_CLASS
3028 : && regclass[5] == X86_64_SSEUP_CLASS
3029 : && regclass[6] == X86_64_SSEUP_CLASS
3030 : && regclass[7] == X86_64_SSEUP_CLASS);
3031 : tmpmode = XImode;
3032 : i += 7;
3033 : break;
3034 0 : default:
3035 0 : gcc_unreachable ();
3036 : }
3037 536506 : exp [nexps++]
3038 536506 : = gen_rtx_EXPR_LIST (VOIDmode,
3039 : gen_rtx_REG (tmpmode,
3040 268253 : GET_SSE_REGNO (sse_regno)),
3041 268253 : GEN_INT (pos*8));
3042 268253 : sse_regno++;
3043 268253 : break;
3044 0 : default:
3045 0 : gcc_unreachable ();
3046 : }
3047 : }
3048 :
3049 : /* Empty aligned struct, union or class. */
3050 1027903 : if (nexps == 0)
3051 : return NULL;
3052 :
3053 1027648 : ret = gen_rtx_PARALLEL (mode, rtvec_alloc (nexps));
3054 2817560 : for (i = 0; i < nexps; i++)
3055 1789912 : XVECEXP (ret, 0, i) = exp [i];
3056 : return ret;
3057 : }
3058 :
3059 : /* Update the data in CUM to advance over an argument of mode MODE
3060 : and data type TYPE. (TYPE is null for libcalls where that information
3061 : may not be available.)
3062 :
3063 : Return a number of integer registers advanced over. */
3064 :
3065 : static int
3066 2131696 : function_arg_advance_32 (CUMULATIVE_ARGS *cum, machine_mode mode,
3067 : const_tree type, HOST_WIDE_INT bytes,
3068 : HOST_WIDE_INT words)
3069 : {
3070 2131696 : int res = 0;
3071 2131696 : bool error_p = false;
3072 :
3073 2131696 : if (TARGET_IAMCU)
3074 : {
3075 : /* Intel MCU psABI passes scalars and aggregates no larger than 8
3076 : bytes in registers. */
3077 0 : if (!VECTOR_MODE_P (mode) && bytes <= 8)
3078 0 : goto pass_in_reg;
3079 : return res;
3080 : }
3081 :
3082 2131696 : switch (mode)
3083 : {
3084 : default:
3085 : break;
3086 :
3087 93820 : case E_BLKmode:
3088 93820 : if (bytes < 0)
3089 : break;
3090 : /* FALLTHRU */
3091 :
3092 2094767 : case E_DImode:
3093 2094767 : case E_SImode:
3094 2094767 : case E_HImode:
3095 2094767 : case E_QImode:
3096 93820 : pass_in_reg:
3097 2094767 : cum->words += words;
3098 2094767 : cum->nregs -= words;
3099 2094767 : cum->regno += words;
3100 2094767 : if (cum->nregs >= 0)
3101 47322 : res = words;
3102 2094767 : if (cum->nregs <= 0)
3103 : {
3104 2060667 : cum->nregs = 0;
3105 2060667 : cfun->machine->arg_reg_available = false;
3106 2060667 : cum->regno = 0;
3107 : }
3108 : break;
3109 :
3110 0 : case E_OImode:
3111 : /* OImode shouldn't be used directly. */
3112 0 : gcc_unreachable ();
3113 :
3114 4757 : case E_DFmode:
3115 4757 : if (cum->float_in_sse == -1)
3116 0 : error_p = true;
3117 4757 : if (cum->float_in_sse < 2)
3118 : break;
3119 : /* FALLTHRU */
3120 1374 : case E_SFmode:
3121 1374 : if (cum->float_in_sse == -1)
3122 0 : error_p = true;
3123 1374 : if (cum->float_in_sse < 1)
3124 : break;
3125 : /* FALLTHRU */
3126 :
3127 52 : case E_V16HFmode:
3128 52 : case E_V16BFmode:
3129 52 : case E_V8SFmode:
3130 52 : case E_V8SImode:
3131 52 : case E_V64QImode:
3132 52 : case E_V32HImode:
3133 52 : case E_V16SImode:
3134 52 : case E_V8DImode:
3135 52 : case E_V32HFmode:
3136 52 : case E_V32BFmode:
3137 52 : case E_V16SFmode:
3138 52 : case E_V8DFmode:
3139 52 : case E_V32QImode:
3140 52 : case E_V16HImode:
3141 52 : case E_V4DFmode:
3142 52 : case E_V4DImode:
3143 52 : case E_TImode:
3144 52 : case E_V16QImode:
3145 52 : case E_V8HImode:
3146 52 : case E_V4SImode:
3147 52 : case E_V2DImode:
3148 52 : case E_V8HFmode:
3149 52 : case E_V8BFmode:
3150 52 : case E_V4SFmode:
3151 52 : case E_V2DFmode:
3152 52 : if (!type || !AGGREGATE_TYPE_P (type))
3153 : {
3154 52 : cum->sse_words += words;
3155 52 : cum->sse_nregs -= 1;
3156 52 : cum->sse_regno += 1;
3157 52 : if (cum->sse_nregs <= 0)
3158 : {
3159 4 : cum->sse_nregs = 0;
3160 4 : cum->sse_regno = 0;
3161 : }
3162 : }
3163 : break;
3164 :
3165 16 : case E_V8QImode:
3166 16 : case E_V4HImode:
3167 16 : case E_V4HFmode:
3168 16 : case E_V4BFmode:
3169 16 : case E_V2SImode:
3170 16 : case E_V2SFmode:
3171 16 : case E_V1TImode:
3172 16 : case E_V1DImode:
3173 16 : if (!type || !AGGREGATE_TYPE_P (type))
3174 : {
3175 16 : cum->mmx_words += words;
3176 16 : cum->mmx_nregs -= 1;
3177 16 : cum->mmx_regno += 1;
3178 16 : if (cum->mmx_nregs <= 0)
3179 : {
3180 0 : cum->mmx_nregs = 0;
3181 0 : cum->mmx_regno = 0;
3182 : }
3183 : }
3184 : break;
3185 : }
3186 2066850 : if (error_p)
3187 : {
3188 0 : cum->float_in_sse = 0;
3189 0 : error ("calling %qD with SSE calling convention without "
3190 : "SSE/SSE2 enabled", cum->decl);
3191 0 : sorry ("this is a GCC bug that can be worked around by adding "
3192 : "attribute used to function called");
3193 : }
3194 :
3195 : return res;
3196 : }
3197 :
3198 : static int
3199 19405715 : function_arg_advance_64 (CUMULATIVE_ARGS *cum, machine_mode mode,
3200 : const_tree type, HOST_WIDE_INT words, bool named)
3201 : {
3202 19405715 : int int_nregs, sse_nregs;
3203 :
3204 : /* Unnamed 512 and 256bit vector mode parameters are passed on stack. */
3205 19405715 : if (!named && (VALID_AVX512F_REG_MODE (mode)
3206 : || VALID_AVX256_REG_MODE (mode)))
3207 : return 0;
3208 :
3209 19405351 : if (!examine_argument (mode, type, false, &int_nregs, &sse_nregs)
3210 19405351 : && sse_nregs <= cum->sse_nregs && int_nregs <= cum->nregs)
3211 : {
3212 17122813 : cum->nregs -= int_nregs;
3213 17122813 : cum->sse_nregs -= sse_nregs;
3214 17122813 : cum->regno += int_nregs;
3215 17122813 : cum->sse_regno += sse_nregs;
3216 17122813 : return int_nregs;
3217 : }
3218 : else
3219 : {
3220 2282538 : int align = ix86_function_arg_boundary (mode, type) / BITS_PER_WORD;
3221 2282538 : cum->words = ROUND_UP (cum->words, align);
3222 2282538 : cum->words += words;
3223 2282538 : return 0;
3224 : }
3225 : }
3226 :
3227 : static int
3228 447474 : function_arg_advance_ms_64 (CUMULATIVE_ARGS *cum, HOST_WIDE_INT bytes,
3229 : HOST_WIDE_INT words)
3230 : {
3231 : /* Otherwise, this should be passed indirect. */
3232 447474 : gcc_assert (bytes == 1 || bytes == 2 || bytes == 4 || bytes == 8);
3233 :
3234 447474 : cum->words += words;
3235 447474 : if (cum->nregs > 0)
3236 : {
3237 289715 : cum->nregs -= 1;
3238 289715 : cum->regno += 1;
3239 289715 : return 1;
3240 : }
3241 : return 0;
3242 : }
3243 :
3244 : /* Update the data in CUM to advance over argument ARG. */
3245 :
3246 : static void
3247 21985252 : ix86_function_arg_advance (cumulative_args_t cum_v,
3248 : const function_arg_info &arg)
3249 : {
3250 21985252 : CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
3251 21985252 : machine_mode mode = arg.mode;
3252 21985252 : HOST_WIDE_INT bytes, words;
3253 21985252 : int nregs;
3254 :
3255 : /* The argument of interrupt handler is a special case and is
3256 : handled in ix86_function_arg. */
3257 21985252 : if (!cum->caller && cfun->machine->func_type != TYPE_NORMAL)
3258 : return;
3259 :
3260 21984885 : bytes = arg.promoted_size_in_bytes ();
3261 21984885 : words = CEIL (bytes, UNITS_PER_WORD);
3262 :
3263 21984885 : if (arg.type)
3264 21663668 : mode = type_natural_mode (arg.type, NULL, false);
3265 :
3266 21984885 : if (TARGET_64BIT)
3267 : {
3268 19853189 : enum calling_abi call_abi = cum ? cum->call_abi : ix86_abi;
3269 :
3270 19853189 : if (call_abi == MS_ABI)
3271 447474 : nregs = function_arg_advance_ms_64 (cum, bytes, words);
3272 : else
3273 19405715 : nregs = function_arg_advance_64 (cum, mode, arg.type, words,
3274 19405715 : arg.named);
3275 : }
3276 : else
3277 2131696 : nregs = function_arg_advance_32 (cum, mode, arg.type, bytes, words);
3278 :
3279 21984885 : if (!nregs)
3280 : {
3281 : /* Track if there are outgoing arguments on stack. */
3282 5739428 : if (cum->caller)
3283 2728961 : cfun->machine->outgoing_args_on_stack = true;
3284 : }
3285 : }
3286 :
3287 : /* Define where to put the arguments to a function.
3288 : Value is zero to push the argument on the stack,
3289 : or a hard register in which to store the argument.
3290 :
3291 : MODE is the argument's machine mode.
3292 : TYPE is the data type of the argument (as a tree).
3293 : This is null for libcalls where that information may
3294 : not be available.
3295 : CUM is a variable of type CUMULATIVE_ARGS which gives info about
3296 : the preceding args and about the function being called.
3297 : NAMED is nonzero if this argument is a named parameter
3298 : (otherwise it is an extra parameter matching an ellipsis). */
3299 :
3300 : static rtx
3301 2558554 : function_arg_32 (CUMULATIVE_ARGS *cum, machine_mode mode,
3302 : machine_mode orig_mode, const_tree type,
3303 : HOST_WIDE_INT bytes, HOST_WIDE_INT words)
3304 : {
3305 2558554 : bool error_p = false;
3306 :
3307 : /* Avoid the AL settings for the Unix64 ABI. */
3308 2558554 : if (mode == VOIDmode)
3309 742433 : return constm1_rtx;
3310 :
3311 1816121 : if (TARGET_IAMCU)
3312 : {
3313 : /* Intel MCU psABI passes scalars and aggregates no larger than 8
3314 : bytes in registers. */
3315 0 : if (!VECTOR_MODE_P (mode) && bytes <= 8)
3316 0 : goto pass_in_reg;
3317 : return NULL_RTX;
3318 : }
3319 :
3320 1816121 : switch (mode)
3321 : {
3322 : default:
3323 : break;
3324 :
3325 77776 : case E_BLKmode:
3326 77776 : if (bytes < 0)
3327 : break;
3328 : /* FALLTHRU */
3329 1782477 : case E_DImode:
3330 1782477 : case E_SImode:
3331 1782477 : case E_HImode:
3332 1782477 : case E_QImode:
3333 77776 : pass_in_reg:
3334 1782477 : if (words <= cum->nregs)
3335 : {
3336 45490 : int regno = cum->regno;
3337 :
3338 : /* Fastcall allocates the first two DWORD (SImode) or
3339 : smaller arguments to ECX and EDX if it isn't an
3340 : aggregate type . */
3341 45490 : if (cum->fastcall)
3342 : {
3343 6 : if (mode == BLKmode
3344 6 : || mode == DImode
3345 6 : || (type && AGGREGATE_TYPE_P (type)))
3346 : break;
3347 :
3348 : /* ECX not EAX is the first allocated register. */
3349 6 : if (regno == AX_REG)
3350 45490 : regno = CX_REG;
3351 : }
3352 45490 : return gen_rtx_REG (mode, regno);
3353 : }
3354 : break;
3355 :
3356 3367 : case E_DFmode:
3357 3367 : if (cum->float_in_sse == -1)
3358 0 : error_p = true;
3359 3367 : if (cum->float_in_sse < 2)
3360 : break;
3361 : /* FALLTHRU */
3362 974 : case E_SFmode:
3363 974 : if (cum->float_in_sse == -1)
3364 0 : error_p = true;
3365 974 : if (cum->float_in_sse < 1)
3366 : break;
3367 : /* FALLTHRU */
3368 12 : case E_TImode:
3369 : /* In 32bit, we pass TImode in xmm registers. */
3370 12 : case E_V16QImode:
3371 12 : case E_V8HImode:
3372 12 : case E_V4SImode:
3373 12 : case E_V2DImode:
3374 12 : case E_V8HFmode:
3375 12 : case E_V8BFmode:
3376 12 : case E_V4SFmode:
3377 12 : case E_V2DFmode:
3378 12 : if (!type || !AGGREGATE_TYPE_P (type))
3379 : {
3380 12 : if (cum->sse_nregs)
3381 12 : return gen_reg_or_parallel (mode, orig_mode,
3382 12 : cum->sse_regno + FIRST_SSE_REG);
3383 : }
3384 : break;
3385 :
3386 0 : case E_OImode:
3387 0 : case E_XImode:
3388 : /* OImode and XImode shouldn't be used directly. */
3389 0 : gcc_unreachable ();
3390 :
3391 9 : case E_V64QImode:
3392 9 : case E_V32HImode:
3393 9 : case E_V16SImode:
3394 9 : case E_V8DImode:
3395 9 : case E_V32HFmode:
3396 9 : case E_V32BFmode:
3397 9 : case E_V16SFmode:
3398 9 : case E_V8DFmode:
3399 9 : case E_V16HFmode:
3400 9 : case E_V16BFmode:
3401 9 : case E_V8SFmode:
3402 9 : case E_V8SImode:
3403 9 : case E_V32QImode:
3404 9 : case E_V16HImode:
3405 9 : case E_V4DFmode:
3406 9 : case E_V4DImode:
3407 9 : if (!type || !AGGREGATE_TYPE_P (type))
3408 : {
3409 9 : if (cum->sse_nregs)
3410 9 : return gen_reg_or_parallel (mode, orig_mode,
3411 9 : cum->sse_regno + FIRST_SSE_REG);
3412 : }
3413 : break;
3414 :
3415 8 : case E_V8QImode:
3416 8 : case E_V4HImode:
3417 8 : case E_V4HFmode:
3418 8 : case E_V4BFmode:
3419 8 : case E_V2SImode:
3420 8 : case E_V2SFmode:
3421 8 : case E_V1TImode:
3422 8 : case E_V1DImode:
3423 8 : if (!type || !AGGREGATE_TYPE_P (type))
3424 : {
3425 8 : if (cum->mmx_nregs)
3426 8 : return gen_reg_or_parallel (mode, orig_mode,
3427 8 : cum->mmx_regno + FIRST_MMX_REG);
3428 : }
3429 : break;
3430 : }
3431 4341 : if (error_p)
3432 : {
3433 0 : cum->float_in_sse = 0;
3434 0 : error ("calling %qD with SSE calling convention without "
3435 : "SSE/SSE2 enabled", cum->decl);
3436 0 : sorry ("this is a GCC bug that can be worked around by adding "
3437 : "attribute used to function called");
3438 : }
3439 :
3440 : return NULL_RTX;
3441 : }
3442 :
3443 : static rtx
3444 19015962 : function_arg_64 (const CUMULATIVE_ARGS *cum, machine_mode mode,
3445 : machine_mode orig_mode, const_tree type, bool named)
3446 : {
3447 : /* Handle a hidden AL argument containing number of registers
3448 : for varargs x86-64 functions. */
3449 19015962 : if (mode == VOIDmode)
3450 5264089 : return GEN_INT (cum->maybe_vaarg
3451 : ? (cum->sse_nregs < 0
3452 : ? X86_64_SSE_REGPARM_MAX
3453 : : cum->sse_regno)
3454 : : -1);
3455 :
3456 13751873 : switch (mode)
3457 : {
3458 : default:
3459 : break;
3460 :
3461 90093 : case E_V16HFmode:
3462 90093 : case E_V16BFmode:
3463 90093 : case E_V8SFmode:
3464 90093 : case E_V8SImode:
3465 90093 : case E_V32QImode:
3466 90093 : case E_V16HImode:
3467 90093 : case E_V4DFmode:
3468 90093 : case E_V4DImode:
3469 90093 : case E_V32HFmode:
3470 90093 : case E_V32BFmode:
3471 90093 : case E_V16SFmode:
3472 90093 : case E_V16SImode:
3473 90093 : case E_V64QImode:
3474 90093 : case E_V32HImode:
3475 90093 : case E_V8DFmode:
3476 90093 : case E_V8DImode:
3477 : /* Unnamed 256 and 512bit vector mode parameters are passed on stack. */
3478 90093 : if (!named)
3479 : return NULL;
3480 : break;
3481 : }
3482 :
3483 13751509 : const int *parm_regs;
3484 13751509 : if (cum->preserve_none_abi)
3485 : parm_regs = x86_64_preserve_none_int_parameter_registers;
3486 : else
3487 13751309 : parm_regs = x86_64_int_parameter_registers;
3488 :
3489 13751509 : return construct_container (mode, orig_mode, type, false,
3490 13751509 : cum->nregs, cum->sse_nregs,
3491 13751509 : &parm_regs[cum->regno],
3492 13751509 : cum->sse_regno);
3493 : }
3494 :
3495 : static rtx
3496 296679 : function_arg_ms_64 (const CUMULATIVE_ARGS *cum, machine_mode mode,
3497 : machine_mode orig_mode, bool named, const_tree type,
3498 : HOST_WIDE_INT bytes)
3499 : {
3500 296679 : unsigned int regno;
3501 :
3502 : /* We need to add clobber for MS_ABI->SYSV ABI calls in expand_call.
3503 : We use value of -2 to specify that current function call is MSABI. */
3504 296679 : if (mode == VOIDmode)
3505 36329 : return GEN_INT (-2);
3506 :
3507 : /* If we've run out of registers, it goes on the stack. */
3508 260350 : if (cum->nregs == 0)
3509 : return NULL_RTX;
3510 :
3511 176508 : regno = x86_64_ms_abi_int_parameter_registers[cum->regno];
3512 :
3513 : /* Only floating point modes less than 64 bits are passed in anything but
3514 : integer regs. Larger floating point types are excluded as the Windows
3515 : ABI requires vreg args can be shadowed in GPRs (for red zone / varargs). */
3516 176508 : if (TARGET_SSE && (mode == HFmode || mode == SFmode || mode == DFmode))
3517 : {
3518 38309 : if (named)
3519 : {
3520 38309 : if (type == NULL_TREE || !AGGREGATE_TYPE_P (type))
3521 37312 : regno = cum->regno + FIRST_SSE_REG;
3522 : }
3523 : else
3524 : {
3525 0 : rtx t1, t2;
3526 :
3527 : /* Unnamed floating parameters are passed in both the
3528 : SSE and integer registers. */
3529 0 : t1 = gen_rtx_REG (mode, cum->regno + FIRST_SSE_REG);
3530 0 : t2 = gen_rtx_REG (mode, regno);
3531 0 : t1 = gen_rtx_EXPR_LIST (VOIDmode, t1, const0_rtx);
3532 0 : t2 = gen_rtx_EXPR_LIST (VOIDmode, t2, const0_rtx);
3533 0 : return gen_rtx_PARALLEL (mode, gen_rtvec (2, t1, t2));
3534 : }
3535 : }
3536 : /* Handle aggregated types passed in register. */
3537 176508 : if (orig_mode == BLKmode)
3538 : {
3539 0 : if (bytes > 0 && bytes <= 8)
3540 0 : mode = (bytes > 4 ? DImode : SImode);
3541 0 : if (mode == BLKmode)
3542 0 : mode = DImode;
3543 : }
3544 :
3545 176508 : return gen_reg_or_parallel (mode, orig_mode, regno);
3546 : }
3547 :
3548 : /* Return where to put the arguments to a function.
3549 : Return zero to push the argument on the stack, or a hard register in which to store the argument.
3550 :
3551 : ARG describes the argument while CUM gives information about the
3552 : preceding args and about the function being called. */
3553 :
3554 : static rtx
3555 21871382 : ix86_function_arg (cumulative_args_t cum_v, const function_arg_info &arg)
3556 : {
3557 21871382 : CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
3558 21871382 : machine_mode mode = arg.mode;
3559 21871382 : HOST_WIDE_INT bytes, words;
3560 21871382 : rtx reg;
3561 :
3562 21871382 : if (!cum->caller && cfun->machine->func_type != TYPE_NORMAL)
3563 : {
3564 187 : gcc_assert (arg.type != NULL_TREE);
3565 187 : if (POINTER_TYPE_P (arg.type))
3566 : {
3567 : /* This is the pointer argument. */
3568 122 : gcc_assert (TYPE_MODE (arg.type) == ptr_mode);
3569 : /* It is at -WORD(AP) in the current frame in interrupt and
3570 : exception handlers. */
3571 122 : reg = plus_constant (Pmode, arg_pointer_rtx, -UNITS_PER_WORD);
3572 : }
3573 : else
3574 : {
3575 65 : gcc_assert (cfun->machine->func_type == TYPE_EXCEPTION
3576 : && TREE_CODE (arg.type) == INTEGER_TYPE
3577 : && TYPE_MODE (arg.type) == word_mode);
3578 : /* The error code is the word-mode integer argument at
3579 : -2 * WORD(AP) in the current frame of the exception
3580 : handler. */
3581 65 : reg = gen_rtx_MEM (word_mode,
3582 65 : plus_constant (Pmode,
3583 : arg_pointer_rtx,
3584 65 : -2 * UNITS_PER_WORD));
3585 : }
3586 187 : return reg;
3587 : }
3588 :
3589 21871195 : bytes = arg.promoted_size_in_bytes ();
3590 21871195 : words = CEIL (bytes, UNITS_PER_WORD);
3591 :
3592 : /* To simplify the code below, represent vector types with a vector mode
3593 : even if MMX/SSE are not active. */
3594 21871195 : if (arg.type && VECTOR_TYPE_P (arg.type))
3595 172020 : mode = type_natural_mode (arg.type, cum, false);
3596 :
3597 21871195 : if (TARGET_64BIT)
3598 : {
3599 19312641 : if (cum->call_abi == MS_ABI)
3600 296679 : reg = function_arg_ms_64 (cum, mode, arg.mode, arg.named,
3601 296679 : arg.type, bytes);
3602 : else
3603 19015962 : reg = function_arg_64 (cum, mode, arg.mode, arg.type, arg.named);
3604 : }
3605 : else
3606 2558554 : reg = function_arg_32 (cum, mode, arg.mode, arg.type, bytes, words);
3607 :
3608 : /* Track if there are outgoing arguments on stack. */
3609 21871195 : if (reg == NULL_RTX && cum->caller)
3610 2203490 : cfun->machine->outgoing_args_on_stack = true;
3611 :
3612 : return reg;
3613 : }
3614 :
3615 : /* A C expression that indicates when an argument must be passed by
3616 : reference. If nonzero for an argument, a copy of that argument is
3617 : made in memory and a pointer to the argument is passed instead of
3618 : the argument itself. The pointer is passed in whatever way is
3619 : appropriate for passing a pointer to that type. */
3620 :
3621 : static bool
3622 21903556 : ix86_pass_by_reference (cumulative_args_t cum_v, const function_arg_info &arg)
3623 : {
3624 21903556 : CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
3625 :
3626 21903556 : if (TARGET_64BIT)
3627 : {
3628 19782108 : enum calling_abi call_abi = cum ? cum->call_abi : ix86_abi;
3629 :
3630 : /* See Windows x64 Software Convention. */
3631 19782108 : if (call_abi == MS_ABI)
3632 : {
3633 441871 : HOST_WIDE_INT msize = GET_MODE_SIZE (arg.mode);
3634 :
3635 441871 : if (tree type = arg.type)
3636 : {
3637 : /* Arrays are passed by reference. */
3638 441871 : if (TREE_CODE (type) == ARRAY_TYPE)
3639 : return true;
3640 :
3641 441871 : if (RECORD_OR_UNION_TYPE_P (type))
3642 : {
3643 : /* Structs/unions of sizes other than 8, 16, 32, or 64 bits
3644 : are passed by reference. */
3645 15103 : msize = int_size_in_bytes (type);
3646 : }
3647 : }
3648 :
3649 : /* __m128 is passed by reference. */
3650 873737 : return msize != 1 && msize != 2 && msize != 4 && msize != 8;
3651 : }
3652 19340237 : else if (arg.type && int_size_in_bytes (arg.type) == -1)
3653 : return true;
3654 : }
3655 :
3656 : return false;
3657 : }
3658 :
3659 : /* Return true when TYPE should be 128bit aligned for 32bit argument
3660 : passing ABI. XXX: This function is obsolete and is only used for
3661 : checking psABI compatibility with previous versions of GCC. */
3662 :
3663 : static bool
3664 1976687 : ix86_compat_aligned_value_p (const_tree type)
3665 : {
3666 1976687 : machine_mode mode = TYPE_MODE (type);
3667 1976687 : if (((TARGET_SSE && SSE_REG_MODE_P (mode))
3668 1976645 : || mode == TDmode
3669 1976645 : || mode == TFmode
3670 : || mode == TCmode)
3671 1976899 : && (!TYPE_USER_ALIGN (type) || TYPE_ALIGN (type) > 128))
3672 : return true;
3673 1976475 : if (TYPE_ALIGN (type) < 128)
3674 : return false;
3675 :
3676 0 : if (AGGREGATE_TYPE_P (type))
3677 : {
3678 : /* Walk the aggregates recursively. */
3679 0 : switch (TREE_CODE (type))
3680 : {
3681 0 : case RECORD_TYPE:
3682 0 : case UNION_TYPE:
3683 0 : case QUAL_UNION_TYPE:
3684 0 : {
3685 0 : tree field;
3686 :
3687 : /* Walk all the structure fields. */
3688 0 : for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
3689 : {
3690 0 : if (TREE_CODE (field) == FIELD_DECL
3691 0 : && ix86_compat_aligned_value_p (TREE_TYPE (field)))
3692 : return true;
3693 : }
3694 : break;
3695 : }
3696 :
3697 0 : case ARRAY_TYPE:
3698 : /* Just for use if some languages passes arrays by value. */
3699 0 : if (ix86_compat_aligned_value_p (TREE_TYPE (type)))
3700 : return true;
3701 : break;
3702 :
3703 : default:
3704 : gcc_unreachable ();
3705 : }
3706 : }
3707 : return false;
3708 : }
3709 :
3710 : /* Return the alignment boundary for MODE and TYPE with alignment ALIGN.
3711 : XXX: This function is obsolete and is only used for checking psABI
3712 : compatibility with previous versions of GCC. */
3713 :
3714 : static unsigned int
3715 5583805 : ix86_compat_function_arg_boundary (machine_mode mode,
3716 : const_tree type, unsigned int align)
3717 : {
3718 : /* In 32bit, only _Decimal128 and __float128 are aligned to their
3719 : natural boundaries. */
3720 5583805 : if (!TARGET_64BIT && mode != TDmode && mode != TFmode)
3721 : {
3722 : /* i386 ABI defines all arguments to be 4 byte aligned. We have to
3723 : make an exception for SSE modes since these require 128bit
3724 : alignment.
3725 :
3726 : The handling here differs from field_alignment. ICC aligns MMX
3727 : arguments to 4 byte boundaries, while structure fields are aligned
3728 : to 8 byte boundaries. */
3729 1988603 : if (!type)
3730 : {
3731 11916 : if (!(TARGET_SSE && SSE_REG_MODE_P (mode)))
3732 1988391 : align = PARM_BOUNDARY;
3733 : }
3734 : else
3735 : {
3736 1976687 : if (!ix86_compat_aligned_value_p (type))
3737 1988391 : align = PARM_BOUNDARY;
3738 : }
3739 : }
3740 10767407 : if (align > BIGGEST_ALIGNMENT)
3741 104 : align = BIGGEST_ALIGNMENT;
3742 5583805 : return align;
3743 : }
3744 :
3745 : /* Return true when TYPE should be 128bit aligned for 32bit argument
3746 : passing ABI. */
3747 :
3748 : static bool
3749 1979384 : ix86_contains_aligned_value_p (const_tree type)
3750 : {
3751 1979384 : machine_mode mode = TYPE_MODE (type);
3752 :
3753 1979384 : if (mode == XFmode || mode == XCmode)
3754 : return false;
3755 :
3756 1977229 : if (TYPE_ALIGN (type) < 128)
3757 : return false;
3758 :
3759 2909 : if (AGGREGATE_TYPE_P (type))
3760 : {
3761 : /* Walk the aggregates recursively. */
3762 0 : switch (TREE_CODE (type))
3763 : {
3764 0 : case RECORD_TYPE:
3765 0 : case UNION_TYPE:
3766 0 : case QUAL_UNION_TYPE:
3767 0 : {
3768 0 : tree field;
3769 :
3770 : /* Walk all the structure fields. */
3771 0 : for (field = TYPE_FIELDS (type);
3772 0 : field;
3773 0 : field = DECL_CHAIN (field))
3774 : {
3775 0 : if (TREE_CODE (field) == FIELD_DECL
3776 0 : && ix86_contains_aligned_value_p (TREE_TYPE (field)))
3777 : return true;
3778 : }
3779 : break;
3780 : }
3781 :
3782 0 : case ARRAY_TYPE:
3783 : /* Just for use if some languages passes arrays by value. */
3784 0 : if (ix86_contains_aligned_value_p (TREE_TYPE (type)))
3785 : return true;
3786 : break;
3787 :
3788 : default:
3789 : gcc_unreachable ();
3790 : }
3791 : }
3792 : else
3793 2909 : return TYPE_ALIGN (type) >= 128;
3794 :
3795 : return false;
3796 : }
3797 :
3798 : /* Gives the alignment boundary, in bits, of an argument with the
3799 : specified mode and type. */
3800 :
3801 : static unsigned int
3802 10995280 : ix86_function_arg_boundary (machine_mode mode, const_tree type)
3803 : {
3804 10995280 : unsigned int align;
3805 10995280 : if (type)
3806 : {
3807 : /* Since the main variant type is used for call, we convert it to
3808 : the main variant type. */
3809 10955431 : type = TYPE_MAIN_VARIANT (type);
3810 10955431 : align = TYPE_ALIGN (type);
3811 10955431 : if (TYPE_EMPTY_P (type))
3812 25294 : return PARM_BOUNDARY;
3813 : }
3814 : else
3815 39849 : align = GET_MODE_ALIGNMENT (mode);
3816 12996555 : if (align < PARM_BOUNDARY)
3817 4119825 : align = PARM_BOUNDARY;
3818 : else
3819 : {
3820 6850161 : static bool warned;
3821 6850161 : unsigned int saved_align = align;
3822 :
3823 6850161 : if (!TARGET_64BIT)
3824 : {
3825 : /* i386 ABI defines XFmode arguments to be 4 byte aligned. */
3826 2015218 : if (!type)
3827 : {
3828 35834 : if (mode == XFmode || mode == XCmode)
3829 : align = PARM_BOUNDARY;
3830 : }
3831 1979384 : else if (!ix86_contains_aligned_value_p (type))
3832 : align = PARM_BOUNDARY;
3833 :
3834 38743 : if (align < 128)
3835 1988391 : align = PARM_BOUNDARY;
3836 : }
3837 :
3838 6850161 : if (warn_psabi
3839 5592326 : && !warned
3840 12433966 : && align != ix86_compat_function_arg_boundary (mode, type,
3841 : saved_align))
3842 : {
3843 104 : warned = true;
3844 104 : inform (input_location,
3845 : "the ABI for passing parameters with %d-byte"
3846 : " alignment has changed in GCC 4.6",
3847 : align / BITS_PER_UNIT);
3848 : }
3849 : }
3850 :
3851 : return align;
3852 : }
3853 :
3854 : /* Return true if N is a possible register number of function value. */
3855 :
3856 : static bool
3857 4354421 : ix86_function_value_regno_p (const unsigned int regno)
3858 : {
3859 4354421 : switch (regno)
3860 : {
3861 : case AX_REG:
3862 : return true;
3863 104122 : case DX_REG:
3864 104122 : return (!TARGET_64BIT || ix86_cfun_abi () != MS_ABI);
3865 :
3866 : /* Complex values are returned in %st(0)/%st(1) pair. */
3867 16296 : case ST0_REG:
3868 16296 : case ST1_REG:
3869 : /* TODO: An ABI identifier should be passed as a parameter.
3870 : For now, most callers, including those in builtins.cc,
3871 : expect us to use the default ABI. */
3872 16296 : if (TARGET_64BIT && ix86_abi == MS_ABI)
3873 : return false;
3874 16296 : return TARGET_FLOAT_RETURNS_IN_80387;
3875 :
3876 : /* Complex values are returned in %xmm0/%xmm1 pair. */
3877 1285075 : case XMM0_REG:
3878 1285075 : case XMM1_REG:
3879 1285075 : return TARGET_SSE;
3880 :
3881 5472 : case MM0_REG:
3882 5472 : if (TARGET_MACHO || TARGET_64BIT)
3883 : return false;
3884 2502 : return TARGET_MMX;
3885 : }
3886 :
3887 : return false;
3888 : }
3889 :
3890 : /* Check whether the register REGNO should be zeroed on X86.
3891 : When ALL_SSE_ZEROED is true, all SSE registers have been zeroed
3892 : together, no need to zero it again.
3893 : When NEED_ZERO_MMX is true, MMX registers should be cleared. */
3894 :
3895 : static bool
3896 1377 : zero_call_used_regno_p (const unsigned int regno,
3897 : bool all_sse_zeroed,
3898 : bool need_zero_mmx)
3899 : {
3900 835 : return GENERAL_REGNO_P (regno)
3901 819 : || (!all_sse_zeroed && SSE_REGNO_P (regno))
3902 439 : || MASK_REGNO_P (regno)
3903 1800 : || (need_zero_mmx && MMX_REGNO_P (regno));
3904 : }
3905 :
3906 : /* Return the machine_mode that is used to zero register REGNO. */
3907 :
3908 : static machine_mode
3909 954 : zero_call_used_regno_mode (const unsigned int regno)
3910 : {
3911 : /* NB: We only need to zero the lower 32 bits for integer registers
3912 : and the lower 128 bits for vector registers since destination are
3913 : zero-extended to the full register width. */
3914 954 : if (GENERAL_REGNO_P (regno))
3915 : return SImode;
3916 : else if (SSE_REGNO_P (regno))
3917 380 : return V4SFmode;
3918 : else if (MASK_REGNO_P (regno))
3919 : return HImode;
3920 : else if (MMX_REGNO_P (regno))
3921 0 : return V2SImode;
3922 : else
3923 0 : gcc_unreachable ();
3924 : }
3925 :
3926 : /* Generate a rtx to zero all vector registers together if possible,
3927 : otherwise, return NULL. */
3928 :
3929 : static rtx
3930 131 : zero_all_vector_registers (HARD_REG_SET need_zeroed_hardregs)
3931 : {
3932 131 : if (!TARGET_AVX)
3933 : return NULL;
3934 :
3935 372 : for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
3936 368 : if ((LEGACY_SSE_REGNO_P (regno)
3937 336 : || (TARGET_64BIT
3938 336 : && (REX_SSE_REGNO_P (regno)
3939 304 : || (TARGET_AVX512F && EXT_REX_SSE_REGNO_P (regno)))))
3940 432 : && !TEST_HARD_REG_BIT (need_zeroed_hardregs, regno))
3941 : return NULL;
3942 :
3943 4 : return gen_avx_vzeroall ();
3944 : }
3945 :
3946 : /* Generate insns to zero all st registers together.
3947 : Return true when zeroing instructions are generated.
3948 : Assume the number of st registers that are zeroed is num_of_st,
3949 : we will emit the following sequence to zero them together:
3950 : fldz; \
3951 : fldz; \
3952 : ...
3953 : fldz; \
3954 : fstp %%st(0); \
3955 : fstp %%st(0); \
3956 : ...
3957 : fstp %%st(0);
3958 : i.e., num_of_st fldz followed by num_of_st fstp to clear the stack
3959 : mark stack slots empty.
3960 :
3961 : How to compute the num_of_st:
3962 : There is no direct mapping from stack registers to hard register
3963 : numbers. If one stack register needs to be cleared, we don't know
3964 : where in the stack the value remains. So, if any stack register
3965 : needs to be cleared, the whole stack should be cleared. However,
3966 : x87 stack registers that hold the return value should be excluded.
3967 : x87 returns in the top (two for complex values) register, so
3968 : num_of_st should be 7/6 when x87 returns, otherwise it will be 8.
3969 : return the value of num_of_st. */
3970 :
3971 :
3972 : static int
3973 131 : zero_all_st_registers (HARD_REG_SET need_zeroed_hardregs)
3974 : {
3975 :
3976 : /* If the FPU is disabled, no need to zero all st registers. */
3977 131 : if (! (TARGET_80387 || TARGET_FLOAT_RETURNS_IN_80387))
3978 : return 0;
3979 :
3980 10329 : unsigned int num_of_st = 0;
3981 10329 : for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
3982 10220 : if ((STACK_REGNO_P (regno) || MMX_REGNO_P (regno))
3983 10220 : && TEST_HARD_REG_BIT (need_zeroed_hardregs, regno))
3984 : {
3985 : num_of_st++;
3986 : break;
3987 : }
3988 :
3989 130 : if (num_of_st == 0)
3990 : return 0;
3991 :
3992 21 : bool return_with_x87 = false;
3993 42 : return_with_x87 = (crtl->return_rtx
3994 21 : && (STACK_REG_P (crtl->return_rtx)));
3995 :
3996 21 : bool complex_return = false;
3997 42 : complex_return = (crtl->return_rtx
3998 21 : && COMPLEX_MODE_P (GET_MODE (crtl->return_rtx)));
3999 :
4000 21 : if (return_with_x87)
4001 2 : if (complex_return)
4002 : num_of_st = 6;
4003 : else
4004 1 : num_of_st = 7;
4005 : else
4006 : num_of_st = 8;
4007 :
4008 21 : rtx st_reg = gen_rtx_REG (XFmode, FIRST_STACK_REG);
4009 186 : for (unsigned int i = 0; i < num_of_st; i++)
4010 165 : emit_insn (gen_rtx_SET (st_reg, CONST0_RTX (XFmode)));
4011 :
4012 186 : for (unsigned int i = 0; i < num_of_st; i++)
4013 : {
4014 165 : rtx insn;
4015 165 : insn = emit_insn (gen_rtx_SET (st_reg, st_reg));
4016 165 : add_reg_note (insn, REG_DEAD, st_reg);
4017 : }
4018 21 : return num_of_st;
4019 : }
4020 :
4021 :
4022 : /* When the routine exit in MMX mode, if any ST register needs
4023 : to be zeroed, we should clear all MMX registers except the
4024 : RET_MMX_REGNO that holds the return value. */
4025 : static bool
4026 0 : zero_all_mm_registers (HARD_REG_SET need_zeroed_hardregs,
4027 : unsigned int ret_mmx_regno)
4028 : {
4029 0 : bool need_zero_all_mm = false;
4030 0 : for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
4031 0 : if (STACK_REGNO_P (regno)
4032 0 : && TEST_HARD_REG_BIT (need_zeroed_hardregs, regno))
4033 : {
4034 : need_zero_all_mm = true;
4035 : break;
4036 : }
4037 :
4038 0 : if (!need_zero_all_mm)
4039 : return false;
4040 :
4041 : machine_mode mode = V2SImode;
4042 0 : for (unsigned int regno = FIRST_MMX_REG; regno <= LAST_MMX_REG; regno++)
4043 0 : if (regno != ret_mmx_regno)
4044 : {
4045 0 : rtx reg = gen_rtx_REG (mode, regno);
4046 0 : emit_insn (gen_rtx_SET (reg, CONST0_RTX (mode)));
4047 : }
4048 : return true;
4049 : }
4050 :
4051 : /* TARGET_ZERO_CALL_USED_REGS. */
4052 : /* Generate a sequence of instructions that zero registers specified by
4053 : NEED_ZEROED_HARDREGS. Return the ZEROED_HARDREGS that are actually
4054 : zeroed. */
4055 : static HARD_REG_SET
4056 131 : ix86_zero_call_used_regs (HARD_REG_SET need_zeroed_hardregs)
4057 : {
4058 131 : HARD_REG_SET zeroed_hardregs;
4059 131 : bool all_sse_zeroed = false;
4060 131 : int all_st_zeroed_num = 0;
4061 131 : bool all_mm_zeroed = false;
4062 :
4063 131 : CLEAR_HARD_REG_SET (zeroed_hardregs);
4064 :
4065 : /* first, let's see whether we can zero all vector registers together. */
4066 131 : rtx zero_all_vec_insn = zero_all_vector_registers (need_zeroed_hardregs);
4067 131 : if (zero_all_vec_insn)
4068 : {
4069 4 : emit_insn (zero_all_vec_insn);
4070 4 : all_sse_zeroed = true;
4071 4 : if (TARGET_64BIT && TARGET_AVX512F)
4072 : {
4073 2 : rtx zero = CONST0_RTX (V4SFmode);
4074 34 : for (unsigned int regno = XMM16_REG;
4075 34 : regno <= XMM31_REG;
4076 : regno++)
4077 : {
4078 32 : rtx reg = gen_rtx_REG (V4SFmode, regno);
4079 32 : emit_move_insn (reg, zero);
4080 : }
4081 : }
4082 : }
4083 :
4084 : /* mm/st registers are shared registers set, we should follow the following
4085 : rules to clear them:
4086 : MMX exit mode x87 exit mode
4087 : -------------|----------------------|---------------
4088 : uses x87 reg | clear all MMX | clear all x87
4089 : uses MMX reg | clear individual MMX | clear all x87
4090 : x87 + MMX | clear all MMX | clear all x87
4091 :
4092 : first, we should decide which mode (MMX mode or x87 mode) the function
4093 : exit with. */
4094 :
4095 131 : bool exit_with_mmx_mode = (crtl->return_rtx
4096 131 : && (MMX_REG_P (crtl->return_rtx)));
4097 :
4098 131 : if (!exit_with_mmx_mode)
4099 : /* x87 exit mode, we should zero all st registers together. */
4100 : {
4101 131 : all_st_zeroed_num = zero_all_st_registers (need_zeroed_hardregs);
4102 :
4103 131 : if (all_st_zeroed_num > 0)
4104 189 : for (unsigned int regno = FIRST_STACK_REG; regno <= LAST_STACK_REG; regno++)
4105 : /* x87 stack registers that hold the return value should be excluded.
4106 : x87 returns in the top (two for complex values) register. */
4107 168 : if (all_st_zeroed_num == 8
4108 168 : || !((all_st_zeroed_num >= 6 && regno == REGNO (crtl->return_rtx))
4109 : || (all_st_zeroed_num == 6
4110 7 : && (regno == (REGNO (crtl->return_rtx) + 1)))))
4111 165 : SET_HARD_REG_BIT (zeroed_hardregs, regno);
4112 : }
4113 : else
4114 : /* MMX exit mode, check whether we can zero all mm registers. */
4115 : {
4116 0 : unsigned int exit_mmx_regno = REGNO (crtl->return_rtx);
4117 0 : all_mm_zeroed = zero_all_mm_registers (need_zeroed_hardregs,
4118 : exit_mmx_regno);
4119 0 : if (all_mm_zeroed)
4120 0 : for (unsigned int regno = FIRST_MMX_REG; regno <= LAST_MMX_REG; regno++)
4121 0 : if (regno != exit_mmx_regno)
4122 0 : SET_HARD_REG_BIT (zeroed_hardregs, regno);
4123 : }
4124 :
4125 : /* Now, generate instructions to zero all the other registers. */
4126 :
4127 12183 : for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
4128 : {
4129 12052 : if (!TEST_HARD_REG_BIT (need_zeroed_hardregs, regno))
4130 10675 : continue;
4131 1800 : if (!zero_call_used_regno_p (regno, all_sse_zeroed,
4132 1377 : exit_with_mmx_mode && !all_mm_zeroed))
4133 423 : continue;
4134 :
4135 954 : SET_HARD_REG_BIT (zeroed_hardregs, regno);
4136 :
4137 954 : machine_mode mode = zero_call_used_regno_mode (regno);
4138 :
4139 954 : rtx reg = gen_rtx_REG (mode, regno);
4140 954 : rtx tmp = gen_rtx_SET (reg, CONST0_RTX (mode));
4141 :
4142 954 : switch (mode)
4143 : {
4144 558 : case E_SImode:
4145 558 : if (!TARGET_USE_MOV0 || optimize_insn_for_size_p ())
4146 : {
4147 558 : rtx clob = gen_rtx_CLOBBER (VOIDmode,
4148 : gen_rtx_REG (CCmode,
4149 : FLAGS_REG));
4150 558 : tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2,
4151 : tmp,
4152 : clob));
4153 : }
4154 : /* FALLTHRU. */
4155 :
4156 954 : case E_V4SFmode:
4157 954 : case E_HImode:
4158 954 : case E_V2SImode:
4159 954 : emit_insn (tmp);
4160 954 : break;
4161 :
4162 0 : default:
4163 0 : gcc_unreachable ();
4164 : }
4165 : }
4166 131 : return zeroed_hardregs;
4167 : }
4168 :
4169 : /* Define how to find the value returned by a function.
4170 : VALTYPE is the data type of the value (as a tree).
4171 : If the precise function being called is known, FUNC is its FUNCTION_DECL;
4172 : otherwise, FUNC is 0. */
4173 :
4174 : static rtx
4175 3948581 : function_value_32 (machine_mode orig_mode, machine_mode mode,
4176 : const_tree fntype, const_tree fn)
4177 : {
4178 3948581 : unsigned int regno;
4179 :
4180 : /* 8-byte vector modes in %mm0. See ix86_return_in_memory for where
4181 : we normally prevent this case when mmx is not available. However
4182 : some ABIs may require the result to be returned like DImode. */
4183 4216666 : if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 8)
4184 : regno = FIRST_MMX_REG;
4185 :
4186 : /* 16-byte vector modes in %xmm0. See ix86_return_in_memory for where
4187 : we prevent this case when sse is not available. However some ABIs
4188 : may require the result to be returned like integer TImode. */
4189 3939305 : else if (mode == TImode
4190 4198114 : || (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 16))
4191 : regno = FIRST_SSE_REG;
4192 :
4193 : /* 32-byte vector modes in %ymm0. */
4194 3980216 : else if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 32)
4195 : regno = FIRST_SSE_REG;
4196 :
4197 : /* 64-byte vector modes in %zmm0. */
4198 3836018 : else if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 64)
4199 : regno = FIRST_SSE_REG;
4200 :
4201 : /* Floating point return values in %st(0) (unless -mno-fp-ret-in-387). */
4202 3680496 : else if (X87_FLOAT_MODE_P (mode) && TARGET_FLOAT_RETURNS_IN_80387)
4203 : regno = FIRST_FLOAT_REG;
4204 : else
4205 : /* Most things go in %eax. */
4206 3614820 : regno = AX_REG;
4207 :
4208 : /* Return __bf16/ _Float16/_Complex _Foat16 by sse register. */
4209 3948581 : if (mode == HFmode || mode == BFmode)
4210 : {
4211 1917 : if (!TARGET_SSE2)
4212 : {
4213 0 : error ("SSE register return with SSE2 disabled");
4214 0 : regno = AX_REG;
4215 : }
4216 : else
4217 : regno = FIRST_SSE_REG;
4218 : }
4219 :
4220 3948581 : if (mode == HCmode)
4221 : {
4222 129 : if (!TARGET_SSE2)
4223 0 : error ("SSE register return with SSE2 disabled");
4224 :
4225 129 : rtx ret = gen_rtx_PARALLEL (mode, rtvec_alloc(1));
4226 258 : XVECEXP (ret, 0, 0)
4227 258 : = gen_rtx_EXPR_LIST (VOIDmode,
4228 : gen_rtx_REG (SImode,
4229 129 : TARGET_SSE2 ? FIRST_SSE_REG : AX_REG),
4230 : GEN_INT (0));
4231 129 : return ret;
4232 : }
4233 :
4234 : /* Override FP return register with %xmm0 for local functions when
4235 : SSE math is enabled or for functions with sseregparm attribute. */
4236 3948452 : if ((fn || fntype) && (mode == SFmode || mode == DFmode))
4237 : {
4238 51213 : int sse_level = ix86_function_sseregparm (fntype, fn, false);
4239 51213 : if (sse_level == -1)
4240 : {
4241 0 : error ("calling %qD with SSE calling convention without "
4242 : "SSE/SSE2 enabled", fn);
4243 0 : sorry ("this is a GCC bug that can be worked around by adding "
4244 : "attribute used to function called");
4245 : }
4246 51213 : else if ((sse_level >= 1 && mode == SFmode)
4247 51213 : || (sse_level == 2 && mode == DFmode))
4248 : regno = FIRST_SSE_REG;
4249 : }
4250 :
4251 : /* OImode shouldn't be used directly. */
4252 3948452 : gcc_assert (mode != OImode);
4253 :
4254 3948452 : return gen_rtx_REG (orig_mode, regno);
4255 : }
4256 :
4257 : static rtx
4258 100483384 : function_value_64 (machine_mode orig_mode, machine_mode mode,
4259 : const_tree valtype)
4260 : {
4261 100483384 : rtx ret;
4262 :
4263 : /* Handle libcalls, which don't provide a type node. */
4264 100483384 : if (valtype == NULL)
4265 : {
4266 106491 : unsigned int regno;
4267 :
4268 106491 : switch (mode)
4269 : {
4270 : case E_BFmode:
4271 : case E_HFmode:
4272 : case E_HCmode:
4273 : case E_SFmode:
4274 : case E_SCmode:
4275 : case E_DFmode:
4276 : case E_DCmode:
4277 : case E_TFmode:
4278 : case E_SDmode:
4279 : case E_DDmode:
4280 : case E_TDmode:
4281 : regno = FIRST_SSE_REG;
4282 : break;
4283 1040 : case E_XFmode:
4284 1040 : case E_XCmode:
4285 1040 : regno = FIRST_FLOAT_REG;
4286 1040 : break;
4287 : case E_TCmode:
4288 : return NULL;
4289 57198 : default:
4290 57198 : regno = AX_REG;
4291 : }
4292 :
4293 106491 : return gen_rtx_REG (mode, regno);
4294 : }
4295 100376893 : else if (POINTER_TYPE_P (valtype))
4296 : {
4297 : /* Pointers are always returned in word_mode. */
4298 16813356 : mode = word_mode;
4299 : }
4300 :
4301 100376893 : ret = construct_container (mode, orig_mode, valtype, true,
4302 : X86_64_MAX_RETURN_NREGS,
4303 : X86_64_MAX_SSE_RETURN_NREGS,
4304 : x86_64_int_return_registers, 0);
4305 :
4306 : /* For zero sized structures, construct_container returns NULL, but we
4307 : need to keep rest of compiler happy by returning meaningful value. */
4308 100376893 : if (!ret)
4309 204294 : ret = gen_rtx_REG (orig_mode, AX_REG);
4310 :
4311 : return ret;
4312 : }
4313 :
4314 : static rtx
4315 0 : function_value_ms_32 (machine_mode orig_mode, machine_mode mode,
4316 : const_tree fntype, const_tree fn, const_tree valtype)
4317 : {
4318 0 : unsigned int regno;
4319 :
4320 : /* Floating point return values in %st(0)
4321 : (unless -mno-fp-ret-in-387 or aggregate type of up to 8 bytes). */
4322 0 : if (X87_FLOAT_MODE_P (mode) && TARGET_FLOAT_RETURNS_IN_80387
4323 0 : && (GET_MODE_SIZE (mode) > 8
4324 0 : || valtype == NULL_TREE || !AGGREGATE_TYPE_P (valtype)))
4325 : {
4326 0 : regno = FIRST_FLOAT_REG;
4327 0 : return gen_rtx_REG (orig_mode, regno);
4328 : }
4329 : else
4330 0 : return function_value_32(orig_mode, mode, fntype,fn);
4331 : }
4332 :
4333 : static rtx
4334 804596 : function_value_ms_64 (machine_mode orig_mode, machine_mode mode,
4335 : const_tree valtype)
4336 : {
4337 804596 : unsigned int regno = AX_REG;
4338 :
4339 804596 : if (TARGET_SSE)
4340 : {
4341 803867 : unsigned int mode_size = GET_MODE_SIZE (mode);
4342 :
4343 803867 : switch (mode_size)
4344 : {
4345 34397 : case 16:
4346 34397 : case 32:
4347 34397 : case 64:
4348 34397 : if (mode_size == 32 && !TARGET_AVX)
4349 : break;
4350 34397 : if (mode_size == 64 && !TARGET_AVX512F)
4351 : break;
4352 34397 : if (valtype != NULL_TREE
4353 34397 : && !VECTOR_INTEGER_TYPE_P (valtype)
4354 50225 : && !VECTOR_FLOAT_TYPE_P (valtype))
4355 : break;
4356 34397 : if (VECTOR_MODE_P (mode)
4357 : && !COMPLEX_MODE_P (mode))
4358 224364 : regno = FIRST_SSE_REG;
4359 : break;
4360 757831 : case 8:
4361 757831 : case 4:
4362 757831 : case 2:
4363 757831 : if (valtype != NULL_TREE && AGGREGATE_TYPE_P (valtype))
4364 : break;
4365 740537 : if (mode == HFmode || mode == SFmode || mode == DFmode)
4366 224364 : regno = FIRST_SSE_REG;
4367 : break;
4368 : default:
4369 : break;
4370 : }
4371 : }
4372 804596 : return gen_rtx_REG (orig_mode, regno);
4373 : }
4374 :
4375 : static rtx
4376 105236561 : ix86_function_value_1 (const_tree valtype, const_tree fntype_or_decl,
4377 : machine_mode orig_mode, machine_mode mode)
4378 : {
4379 105236561 : const_tree fn, fntype;
4380 :
4381 105236561 : fn = NULL_TREE;
4382 105236561 : if (fntype_or_decl && DECL_P (fntype_or_decl))
4383 3573743 : fn = fntype_or_decl;
4384 3573743 : fntype = fn ? TREE_TYPE (fn) : fntype_or_decl;
4385 :
4386 105236561 : if (ix86_function_type_abi (fntype) == MS_ABI)
4387 : {
4388 804596 : if (TARGET_64BIT)
4389 804596 : return function_value_ms_64 (orig_mode, mode, valtype);
4390 : else
4391 0 : return function_value_ms_32 (orig_mode, mode, fntype, fn, valtype);
4392 : }
4393 104431965 : else if (TARGET_64BIT)
4394 100483384 : return function_value_64 (orig_mode, mode, valtype);
4395 : else
4396 3948581 : return function_value_32 (orig_mode, mode, fntype, fn);
4397 : }
4398 :
4399 : static rtx
4400 105126916 : ix86_function_value (const_tree valtype, const_tree fntype_or_decl, bool)
4401 : {
4402 105126916 : machine_mode mode, orig_mode;
4403 :
4404 105126916 : orig_mode = TYPE_MODE (valtype);
4405 105126916 : mode = type_natural_mode (valtype, NULL, true);
4406 105126916 : return ix86_function_value_1 (valtype, fntype_or_decl, orig_mode, mode);
4407 : }
4408 :
4409 : /* Pointer function arguments and return values are promoted to
4410 : word_mode for normal functions. */
4411 :
4412 : static machine_mode
4413 32466835 : ix86_promote_function_mode (const_tree type, machine_mode mode,
4414 : int *punsignedp, const_tree fntype,
4415 : int for_return)
4416 : {
4417 32466835 : if (cfun->machine->func_type == TYPE_NORMAL
4418 32465812 : && type != NULL_TREE
4419 32431419 : && POINTER_TYPE_P (type))
4420 : {
4421 16309345 : *punsignedp = POINTERS_EXTEND_UNSIGNED;
4422 16309345 : return word_mode;
4423 : }
4424 16157490 : return default_promote_function_mode (type, mode, punsignedp, fntype,
4425 16157490 : for_return);
4426 : }
4427 :
4428 : /* Return true if a structure, union or array with MODE containing FIELD
4429 : should be accessed using BLKmode. */
4430 :
4431 : static bool
4432 148950777 : ix86_member_type_forces_blk (const_tree field, machine_mode mode)
4433 : {
4434 : /* Union with XFmode must be in BLKmode. */
4435 148950777 : return (mode == XFmode
4436 149125996 : && (TREE_CODE (DECL_FIELD_CONTEXT (field)) == UNION_TYPE
4437 133897 : || TREE_CODE (DECL_FIELD_CONTEXT (field)) == QUAL_UNION_TYPE));
4438 : }
4439 :
4440 : rtx
4441 109645 : ix86_libcall_value (machine_mode mode)
4442 : {
4443 109645 : return ix86_function_value_1 (NULL, NULL, mode, mode);
4444 : }
4445 :
4446 : /* Return true iff type is returned in memory. */
4447 :
4448 : static bool
4449 107347409 : ix86_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED)
4450 : {
4451 107347409 : const machine_mode mode = type_natural_mode (type, NULL, true);
4452 107347409 : HOST_WIDE_INT size;
4453 :
4454 107347409 : if (TARGET_64BIT)
4455 : {
4456 102777472 : if (ix86_function_type_abi (fntype) == MS_ABI)
4457 : {
4458 724001 : size = int_size_in_bytes (type);
4459 :
4460 : /* __m128 is returned in xmm0. 256/512-bit vector values are
4461 : returned in ymm0/zmm0 when AVX/AVX512 is enabled. */
4462 724001 : if ((!type || VECTOR_INTEGER_TYPE_P (type)
4463 704430 : || VECTOR_FLOAT_TYPE_P (type))
4464 35399 : && VECTOR_MODE_P (mode)
4465 : && !COMPLEX_MODE_P (mode)
4466 759400 : && ((GET_MODE_SIZE (mode) == 16 || size == 16)
4467 42804 : || (TARGET_AVX && (GET_MODE_SIZE (mode) == 32 || size == 32))
4468 11658 : || (TARGET_AVX512F
4469 11154 : && (GET_MODE_SIZE (mode) == 64 || size == 64))))
4470 : return false;
4471 :
4472 : /* Otherwise, the size must be exactly in [1248]. */
4473 1363112 : return size != 1 && size != 2 && size != 4 && size != 8;
4474 : }
4475 : else
4476 : {
4477 102053471 : int needed_intregs, needed_sseregs;
4478 :
4479 102053471 : return examine_argument (mode, type, true,
4480 : &needed_intregs, &needed_sseregs);
4481 : }
4482 : }
4483 : else
4484 : {
4485 4569937 : size = int_size_in_bytes (type);
4486 :
4487 : /* Intel MCU psABI returns scalars and aggregates no larger than 8
4488 : bytes in registers. */
4489 4569937 : if (TARGET_IAMCU)
4490 0 : return VECTOR_MODE_P (mode) || size < 0 || size > 8;
4491 :
4492 4569937 : if (mode == BLKmode)
4493 : return true;
4494 :
4495 4569937 : if (MS_AGGREGATE_RETURN && AGGREGATE_TYPE_P (type) && size <= 8)
4496 : return false;
4497 :
4498 4569937 : if (VECTOR_MODE_P (mode) || mode == TImode)
4499 : {
4500 : /* User-created vectors small enough to fit in EAX. */
4501 268055 : if (size < 8)
4502 : return false;
4503 :
4504 : /* Unless ABI prescribes otherwise,
4505 : MMX/3dNow values are returned in MM0 if available. */
4506 :
4507 268055 : if (size == 8)
4508 9266 : return TARGET_VECT8_RETURNS || !TARGET_MMX;
4509 :
4510 : /* SSE values are returned in XMM0 if available. */
4511 258789 : if (size == 16)
4512 108939 : return !TARGET_SSE;
4513 :
4514 : /* AVX values are returned in YMM0 if available. */
4515 149850 : if (size == 32)
4516 72090 : return !TARGET_AVX;
4517 :
4518 : /* AVX512F values are returned in ZMM0 if available. */
4519 77760 : if (size == 64)
4520 77760 : return !TARGET_AVX512F;
4521 : }
4522 :
4523 4301882 : if (mode == XFmode)
4524 : return false;
4525 :
4526 4289802 : if (size > 12)
4527 : return true;
4528 :
4529 : /* OImode shouldn't be used directly. */
4530 3294748 : gcc_assert (mode != OImode);
4531 :
4532 : return false;
4533 : }
4534 : }
4535 :
4536 : /* Implement TARGET_PUSH_ARGUMENT. */
4537 :
4538 : static bool
4539 9397485 : ix86_push_argument (unsigned int npush)
4540 : {
4541 : /* If SSE2 is available, use vector move to put large argument onto
4542 : stack. NB: In 32-bit mode, use 8-byte vector move. */
4543 11823956 : return ((!TARGET_SSE2 || npush < (TARGET_64BIT ? 16 : 8))
4544 9132092 : && TARGET_PUSH_ARGS
4545 18529419 : && !ACCUMULATE_OUTGOING_ARGS);
4546 : }
4547 :
4548 :
4549 : /* Create the va_list data type. */
4550 :
4551 : static tree
4552 286366 : ix86_build_builtin_va_list_64 (void)
4553 : {
4554 286366 : tree f_gpr, f_fpr, f_ovf, f_sav, record, type_decl;
4555 :
4556 286366 : record = lang_hooks.types.make_type (RECORD_TYPE);
4557 286366 : type_decl = build_decl (BUILTINS_LOCATION,
4558 : TYPE_DECL, get_identifier ("__va_list_tag"), record);
4559 :
4560 286366 : f_gpr = build_decl (BUILTINS_LOCATION,
4561 : FIELD_DECL, get_identifier ("gp_offset"),
4562 : unsigned_type_node);
4563 286366 : f_fpr = build_decl (BUILTINS_LOCATION,
4564 : FIELD_DECL, get_identifier ("fp_offset"),
4565 : unsigned_type_node);
4566 286366 : f_ovf = build_decl (BUILTINS_LOCATION,
4567 : FIELD_DECL, get_identifier ("overflow_arg_area"),
4568 : ptr_type_node);
4569 286366 : f_sav = build_decl (BUILTINS_LOCATION,
4570 : FIELD_DECL, get_identifier ("reg_save_area"),
4571 : ptr_type_node);
4572 :
4573 286366 : va_list_gpr_counter_field = f_gpr;
4574 286366 : va_list_fpr_counter_field = f_fpr;
4575 :
4576 286366 : DECL_FIELD_CONTEXT (f_gpr) = record;
4577 286366 : DECL_FIELD_CONTEXT (f_fpr) = record;
4578 286366 : DECL_FIELD_CONTEXT (f_ovf) = record;
4579 286366 : DECL_FIELD_CONTEXT (f_sav) = record;
4580 :
4581 286366 : TYPE_STUB_DECL (record) = type_decl;
4582 286366 : TYPE_NAME (record) = type_decl;
4583 286366 : TYPE_FIELDS (record) = f_gpr;
4584 286366 : DECL_CHAIN (f_gpr) = f_fpr;
4585 286366 : DECL_CHAIN (f_fpr) = f_ovf;
4586 286366 : DECL_CHAIN (f_ovf) = f_sav;
4587 286366 : TREE_PUBLIC (type_decl) = 1;
4588 :
4589 286366 : layout_type (record);
4590 :
4591 286366 : TYPE_ATTRIBUTES (record) = tree_cons (get_identifier ("sysv_abi va_list"),
4592 286366 : NULL_TREE, TYPE_ATTRIBUTES (record));
4593 :
4594 : /* The correct type is an array type of one element. */
4595 286366 : return build_array_type (record, build_index_type (size_zero_node));
4596 : }
4597 :
4598 : /* Setup the builtin va_list data type and for 64-bit the additional
4599 : calling convention specific va_list data types. */
4600 :
4601 : static tree
4602 293528 : ix86_build_builtin_va_list (void)
4603 : {
4604 293528 : if (TARGET_64BIT)
4605 : {
4606 : /* Initialize ABI specific va_list builtin types.
4607 :
4608 : In lto1, we can encounter two va_list types:
4609 : - one as a result of the type-merge across TUs, and
4610 : - the one constructed here.
4611 : These two types will not have the same TYPE_MAIN_VARIANT, and therefore
4612 : a type identity check in canonical_va_list_type based on
4613 : TYPE_MAIN_VARIANT (which we used to have) will not work.
4614 : Instead, we tag each va_list_type_node with its unique attribute, and
4615 : look for the attribute in the type identity check in
4616 : canonical_va_list_type.
4617 :
4618 : Tagging sysv_va_list_type_node directly with the attribute is
4619 : problematic since it's a array of one record, which will degrade into a
4620 : pointer to record when used as parameter (see build_va_arg comments for
4621 : an example), dropping the attribute in the process. So we tag the
4622 : record instead. */
4623 :
4624 : /* For SYSV_ABI we use an array of one record. */
4625 286366 : sysv_va_list_type_node = ix86_build_builtin_va_list_64 ();
4626 :
4627 : /* For MS_ABI we use plain pointer to argument area. */
4628 286366 : tree char_ptr_type = build_pointer_type (char_type_node);
4629 286366 : tree attr = tree_cons (get_identifier ("ms_abi va_list"), NULL_TREE,
4630 286366 : TYPE_ATTRIBUTES (char_ptr_type));
4631 286366 : ms_va_list_type_node = build_type_attribute_variant (char_ptr_type, attr);
4632 :
4633 286366 : return ((ix86_abi == MS_ABI)
4634 286366 : ? ms_va_list_type_node
4635 286366 : : sysv_va_list_type_node);
4636 : }
4637 : else
4638 : {
4639 : /* For i386 we use plain pointer to argument area. */
4640 7162 : return build_pointer_type (char_type_node);
4641 : }
4642 : }
4643 :
4644 : /* Worker function for TARGET_SETUP_INCOMING_VARARGS. */
4645 :
4646 : static void
4647 15776 : setup_incoming_varargs_64 (CUMULATIVE_ARGS *cum)
4648 : {
4649 15776 : rtx save_area, mem;
4650 15776 : alias_set_type set;
4651 15776 : int i, max;
4652 :
4653 : /* GPR size of varargs save area. */
4654 15776 : if (cfun->va_list_gpr_size)
4655 15313 : ix86_varargs_gpr_size = X86_64_REGPARM_MAX * UNITS_PER_WORD;
4656 : else
4657 463 : ix86_varargs_gpr_size = 0;
4658 :
4659 : /* FPR size of varargs save area. We don't need it if we don't pass
4660 : anything in SSE registers. */
4661 15776 : if (TARGET_SSE && cfun->va_list_fpr_size)
4662 14712 : ix86_varargs_fpr_size = X86_64_SSE_REGPARM_MAX * 16;
4663 : else
4664 1064 : ix86_varargs_fpr_size = 0;
4665 :
4666 15776 : if (! ix86_varargs_gpr_size && ! ix86_varargs_fpr_size)
4667 : return;
4668 :
4669 15482 : save_area = frame_pointer_rtx;
4670 15482 : set = get_varargs_alias_set ();
4671 :
4672 15482 : max = cum->regno + cfun->va_list_gpr_size / UNITS_PER_WORD;
4673 15482 : if (max > X86_64_REGPARM_MAX)
4674 : max = X86_64_REGPARM_MAX;
4675 :
4676 15482 : const int *parm_regs;
4677 15482 : if (cum->preserve_none_abi)
4678 : parm_regs = x86_64_preserve_none_int_parameter_registers;
4679 : else
4680 15480 : parm_regs = x86_64_int_parameter_registers;
4681 :
4682 86004 : for (i = cum->regno; i < max; i++)
4683 : {
4684 70522 : mem = gen_rtx_MEM (word_mode,
4685 70522 : plus_constant (Pmode, save_area, i * UNITS_PER_WORD));
4686 70522 : MEM_NOTRAP_P (mem) = 1;
4687 70522 : set_mem_alias_set (mem, set);
4688 70522 : emit_move_insn (mem,
4689 70522 : gen_rtx_REG (word_mode, parm_regs[i]));
4690 : }
4691 :
4692 15482 : if (ix86_varargs_fpr_size)
4693 : {
4694 14712 : machine_mode smode;
4695 14712 : rtx_code_label *label;
4696 14712 : rtx test;
4697 :
4698 : /* Now emit code to save SSE registers. The AX parameter contains number
4699 : of SSE parameter registers used to call this function, though all we
4700 : actually check here is the zero/non-zero status. */
4701 :
4702 14712 : label = gen_label_rtx ();
4703 14712 : test = gen_rtx_EQ (VOIDmode, gen_rtx_REG (QImode, AX_REG), const0_rtx);
4704 14712 : emit_jump_insn (gen_cbranchqi4 (test, XEXP (test, 0), XEXP (test, 1),
4705 : label));
4706 :
4707 : /* ??? If !TARGET_SSE_TYPELESS_STORES, would we perform better if
4708 : we used movdqa (i.e. TImode) instead? Perhaps even better would
4709 : be if we could determine the real mode of the data, via a hook
4710 : into pass_stdarg. Ignore all that for now. */
4711 14712 : smode = V4SFmode;
4712 14712 : if (crtl->stack_alignment_needed < GET_MODE_ALIGNMENT (smode))
4713 4207 : crtl->stack_alignment_needed = GET_MODE_ALIGNMENT (smode);
4714 :
4715 14712 : max = cum->sse_regno + cfun->va_list_fpr_size / 16;
4716 14712 : if (max > X86_64_SSE_REGPARM_MAX)
4717 : max = X86_64_SSE_REGPARM_MAX;
4718 :
4719 130784 : for (i = cum->sse_regno; i < max; ++i)
4720 : {
4721 116072 : mem = plus_constant (Pmode, save_area,
4722 116072 : i * 16 + ix86_varargs_gpr_size);
4723 116072 : mem = gen_rtx_MEM (smode, mem);
4724 116072 : MEM_NOTRAP_P (mem) = 1;
4725 116072 : set_mem_alias_set (mem, set);
4726 116072 : set_mem_align (mem, GET_MODE_ALIGNMENT (smode));
4727 :
4728 116072 : emit_move_insn (mem, gen_rtx_REG (smode, GET_SSE_REGNO (i)));
4729 : }
4730 :
4731 14712 : emit_label (label);
4732 : }
4733 : }
4734 :
4735 : static void
4736 5656 : setup_incoming_varargs_ms_64 (CUMULATIVE_ARGS *cum)
4737 : {
4738 5656 : alias_set_type set = get_varargs_alias_set ();
4739 5656 : int i;
4740 :
4741 : /* Reset to zero, as there might be a sysv vaarg used
4742 : before. */
4743 5656 : ix86_varargs_gpr_size = 0;
4744 5656 : ix86_varargs_fpr_size = 0;
4745 :
4746 14170 : for (i = cum->regno; i < X86_64_MS_REGPARM_MAX; i++)
4747 : {
4748 8514 : rtx reg, mem;
4749 :
4750 8514 : mem = gen_rtx_MEM (Pmode,
4751 8514 : plus_constant (Pmode, virtual_incoming_args_rtx,
4752 8514 : i * UNITS_PER_WORD));
4753 8514 : MEM_NOTRAP_P (mem) = 1;
4754 8514 : set_mem_alias_set (mem, set);
4755 :
4756 8514 : reg = gen_rtx_REG (Pmode, x86_64_ms_abi_int_parameter_registers[i]);
4757 8514 : emit_move_insn (mem, reg);
4758 : }
4759 5656 : }
4760 :
4761 : static void
4762 21586 : ix86_setup_incoming_varargs (cumulative_args_t cum_v,
4763 : const function_arg_info &arg,
4764 : int *, int no_rtl)
4765 : {
4766 21586 : CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
4767 21586 : CUMULATIVE_ARGS next_cum;
4768 21586 : tree fntype;
4769 :
4770 : /* This argument doesn't appear to be used anymore. Which is good,
4771 : because the old code here didn't suppress rtl generation. */
4772 21586 : gcc_assert (!no_rtl);
4773 :
4774 21586 : if (!TARGET_64BIT)
4775 154 : return;
4776 :
4777 21432 : fntype = TREE_TYPE (current_function_decl);
4778 :
4779 : /* For varargs, we do not want to skip the dummy va_dcl argument.
4780 : For stdargs, we do want to skip the last named argument. */
4781 21432 : next_cum = *cum;
4782 21432 : if ((!TYPE_NO_NAMED_ARGS_STDARG_P (TREE_TYPE (current_function_decl))
4783 151 : || arg.type != NULL_TREE)
4784 21469 : && stdarg_p (fntype))
4785 21318 : ix86_function_arg_advance (pack_cumulative_args (&next_cum), arg);
4786 :
4787 21432 : if (cum->call_abi == MS_ABI)
4788 5656 : setup_incoming_varargs_ms_64 (&next_cum);
4789 : else
4790 15776 : setup_incoming_varargs_64 (&next_cum);
4791 : }
4792 :
4793 : /* Checks if TYPE is of kind va_list char *. */
4794 :
4795 : static bool
4796 73370 : is_va_list_char_pointer (tree type)
4797 : {
4798 73370 : tree canonic;
4799 :
4800 : /* For 32-bit it is always true. */
4801 73370 : if (!TARGET_64BIT)
4802 : return true;
4803 73208 : canonic = ix86_canonical_va_list_type (type);
4804 73208 : return (canonic == ms_va_list_type_node
4805 73208 : || (ix86_abi == MS_ABI && canonic == va_list_type_node));
4806 : }
4807 :
4808 : /* Implement va_start. */
4809 :
4810 : static void
4811 21074 : ix86_va_start (tree valist, rtx nextarg)
4812 : {
4813 21074 : HOST_WIDE_INT words, n_gpr, n_fpr;
4814 21074 : tree f_gpr, f_fpr, f_ovf, f_sav;
4815 21074 : tree gpr, fpr, ovf, sav, t;
4816 21074 : tree type;
4817 21074 : rtx ovf_rtx;
4818 :
4819 21074 : if (flag_split_stack
4820 12 : && cfun->machine->split_stack_varargs_pointer == NULL_RTX)
4821 : {
4822 12 : unsigned int scratch_regno;
4823 :
4824 : /* When we are splitting the stack, we can't refer to the stack
4825 : arguments using internal_arg_pointer, because they may be on
4826 : the old stack. The split stack prologue will arrange to
4827 : leave a pointer to the old stack arguments in a scratch
4828 : register, which we here copy to a pseudo-register. The split
4829 : stack prologue can't set the pseudo-register directly because
4830 : it (the prologue) runs before any registers have been saved. */
4831 :
4832 12 : scratch_regno = split_stack_prologue_scratch_regno ();
4833 12 : if (scratch_regno != INVALID_REGNUM)
4834 : {
4835 12 : rtx reg;
4836 12 : rtx_insn *seq;
4837 :
4838 16 : reg = gen_reg_rtx (Pmode);
4839 12 : cfun->machine->split_stack_varargs_pointer = reg;
4840 :
4841 12 : start_sequence ();
4842 16 : emit_move_insn (reg, gen_rtx_REG (Pmode, scratch_regno));
4843 12 : seq = end_sequence ();
4844 :
4845 12 : push_topmost_sequence ();
4846 12 : emit_insn_after (seq, entry_of_function ());
4847 12 : pop_topmost_sequence ();
4848 : }
4849 : }
4850 :
4851 : /* Only 64bit target needs something special. */
4852 21074 : if (is_va_list_char_pointer (TREE_TYPE (valist)))
4853 : {
4854 5660 : if (cfun->machine->split_stack_varargs_pointer == NULL_RTX)
4855 5656 : std_expand_builtin_va_start (valist, nextarg);
4856 : else
4857 : {
4858 4 : rtx va_r, next;
4859 :
4860 4 : va_r = expand_expr (valist, NULL_RTX, VOIDmode, EXPAND_WRITE);
4861 8 : next = expand_binop (ptr_mode, add_optab,
4862 4 : cfun->machine->split_stack_varargs_pointer,
4863 : crtl->args.arg_offset_rtx,
4864 : NULL_RTX, 0, OPTAB_LIB_WIDEN);
4865 4 : convert_move (va_r, next, 0);
4866 : }
4867 5660 : return;
4868 : }
4869 :
4870 15414 : f_gpr = TYPE_FIELDS (TREE_TYPE (sysv_va_list_type_node));
4871 15414 : f_fpr = DECL_CHAIN (f_gpr);
4872 15414 : f_ovf = DECL_CHAIN (f_fpr);
4873 15414 : f_sav = DECL_CHAIN (f_ovf);
4874 :
4875 15414 : valist = build_simple_mem_ref (valist);
4876 15414 : TREE_TYPE (valist) = TREE_TYPE (sysv_va_list_type_node);
4877 : /* The following should be folded into the MEM_REF offset. */
4878 15414 : gpr = build3 (COMPONENT_REF, TREE_TYPE (f_gpr), unshare_expr (valist),
4879 : f_gpr, NULL_TREE);
4880 15414 : fpr = build3 (COMPONENT_REF, TREE_TYPE (f_fpr), unshare_expr (valist),
4881 : f_fpr, NULL_TREE);
4882 15414 : ovf = build3 (COMPONENT_REF, TREE_TYPE (f_ovf), unshare_expr (valist),
4883 : f_ovf, NULL_TREE);
4884 15414 : sav = build3 (COMPONENT_REF, TREE_TYPE (f_sav), unshare_expr (valist),
4885 : f_sav, NULL_TREE);
4886 :
4887 : /* Count number of gp and fp argument registers used. */
4888 15414 : words = crtl->args.info.words;
4889 15414 : n_gpr = crtl->args.info.regno;
4890 15414 : n_fpr = crtl->args.info.sse_regno;
4891 :
4892 15414 : if (cfun->va_list_gpr_size)
4893 : {
4894 15167 : type = TREE_TYPE (gpr);
4895 15167 : t = build2 (MODIFY_EXPR, type,
4896 15167 : gpr, build_int_cst (type, n_gpr * 8));
4897 15167 : TREE_SIDE_EFFECTS (t) = 1;
4898 15167 : expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
4899 : }
4900 :
4901 15414 : if (TARGET_SSE && cfun->va_list_fpr_size)
4902 : {
4903 14554 : type = TREE_TYPE (fpr);
4904 14554 : t = build2 (MODIFY_EXPR, type, fpr,
4905 14554 : build_int_cst (type, n_fpr * 16 + 8*X86_64_REGPARM_MAX));
4906 14554 : TREE_SIDE_EFFECTS (t) = 1;
4907 14554 : expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
4908 : }
4909 :
4910 : /* Find the overflow area. */
4911 15414 : type = TREE_TYPE (ovf);
4912 15414 : if (cfun->machine->split_stack_varargs_pointer == NULL_RTX)
4913 15406 : ovf_rtx = crtl->args.internal_arg_pointer;
4914 : else
4915 : ovf_rtx = cfun->machine->split_stack_varargs_pointer;
4916 15414 : t = make_tree (type, ovf_rtx);
4917 15414 : if (words != 0)
4918 492 : t = fold_build_pointer_plus_hwi (t, words * UNITS_PER_WORD);
4919 :
4920 15414 : t = build2 (MODIFY_EXPR, type, ovf, t);
4921 15414 : TREE_SIDE_EFFECTS (t) = 1;
4922 15414 : expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
4923 :
4924 15414 : if (ix86_varargs_gpr_size || ix86_varargs_fpr_size)
4925 : {
4926 : /* Find the register save area.
4927 : Prologue of the function save it right above stack frame. */
4928 15336 : type = TREE_TYPE (sav);
4929 15336 : t = make_tree (type, frame_pointer_rtx);
4930 15336 : if (!ix86_varargs_gpr_size)
4931 169 : t = fold_build_pointer_plus_hwi (t, -8 * X86_64_REGPARM_MAX);
4932 :
4933 15336 : t = build2 (MODIFY_EXPR, type, sav, t);
4934 15336 : TREE_SIDE_EFFECTS (t) = 1;
4935 15336 : expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
4936 : }
4937 : }
4938 :
4939 : /* Implement va_arg. */
4940 :
4941 : static tree
4942 52296 : ix86_gimplify_va_arg (tree valist, tree type, gimple_seq *pre_p,
4943 : gimple_seq *post_p)
4944 : {
4945 52296 : static const int intreg[6] = { 0, 1, 2, 3, 4, 5 };
4946 52296 : tree f_gpr, f_fpr, f_ovf, f_sav;
4947 52296 : tree gpr, fpr, ovf, sav, t;
4948 52296 : int size, rsize;
4949 52296 : tree lab_false, lab_over = NULL_TREE;
4950 52296 : tree addr, t2;
4951 52296 : rtx container;
4952 52296 : int indirect_p = 0;
4953 52296 : tree ptrtype;
4954 52296 : machine_mode nat_mode;
4955 52296 : unsigned int arg_boundary;
4956 52296 : unsigned int type_align;
4957 :
4958 : /* Only 64bit target needs something special. */
4959 52296 : if (is_va_list_char_pointer (TREE_TYPE (valist)))
4960 260 : return std_gimplify_va_arg_expr (valist, type, pre_p, post_p);
4961 :
4962 52036 : f_gpr = TYPE_FIELDS (TREE_TYPE (sysv_va_list_type_node));
4963 52036 : f_fpr = DECL_CHAIN (f_gpr);
4964 52036 : f_ovf = DECL_CHAIN (f_fpr);
4965 52036 : f_sav = DECL_CHAIN (f_ovf);
4966 :
4967 52036 : gpr = build3 (COMPONENT_REF, TREE_TYPE (f_gpr),
4968 : valist, f_gpr, NULL_TREE);
4969 :
4970 52036 : fpr = build3 (COMPONENT_REF, TREE_TYPE (f_fpr), valist, f_fpr, NULL_TREE);
4971 52036 : ovf = build3 (COMPONENT_REF, TREE_TYPE (f_ovf), valist, f_ovf, NULL_TREE);
4972 52036 : sav = build3 (COMPONENT_REF, TREE_TYPE (f_sav), valist, f_sav, NULL_TREE);
4973 :
4974 52036 : indirect_p = pass_va_arg_by_reference (type);
4975 52036 : if (indirect_p)
4976 103 : type = build_pointer_type (type);
4977 52036 : size = arg_int_size_in_bytes (type);
4978 52036 : rsize = CEIL (size, UNITS_PER_WORD);
4979 :
4980 52036 : nat_mode = type_natural_mode (type, NULL, false);
4981 52036 : switch (nat_mode)
4982 : {
4983 28 : case E_V16HFmode:
4984 28 : case E_V16BFmode:
4985 28 : case E_V8SFmode:
4986 28 : case E_V8SImode:
4987 28 : case E_V32QImode:
4988 28 : case E_V16HImode:
4989 28 : case E_V4DFmode:
4990 28 : case E_V4DImode:
4991 28 : case E_V32HFmode:
4992 28 : case E_V32BFmode:
4993 28 : case E_V16SFmode:
4994 28 : case E_V16SImode:
4995 28 : case E_V64QImode:
4996 28 : case E_V32HImode:
4997 28 : case E_V8DFmode:
4998 28 : case E_V8DImode:
4999 : /* Unnamed 256 and 512bit vector mode parameters are passed on stack. */
5000 28 : if (!TARGET_64BIT_MS_ABI)
5001 : {
5002 : container = NULL;
5003 : break;
5004 : }
5005 : /* FALLTHRU */
5006 :
5007 52008 : default:
5008 52008 : container = construct_container (nat_mode, TYPE_MODE (type),
5009 : type, false, X86_64_REGPARM_MAX,
5010 : X86_64_SSE_REGPARM_MAX, intreg, 0);
5011 52008 : break;
5012 : }
5013 :
5014 : /* Pull the value out of the saved registers. */
5015 :
5016 52036 : addr = create_tmp_var (ptr_type_node, "addr");
5017 52036 : type_align = TYPE_ALIGN (type);
5018 :
5019 52036 : if (container)
5020 : {
5021 28943 : int needed_intregs, needed_sseregs;
5022 28943 : bool need_temp;
5023 28943 : tree int_addr, sse_addr;
5024 :
5025 28943 : lab_false = create_artificial_label (UNKNOWN_LOCATION);
5026 28943 : lab_over = create_artificial_label (UNKNOWN_LOCATION);
5027 :
5028 28943 : examine_argument (nat_mode, type, false,
5029 : &needed_intregs, &needed_sseregs);
5030 :
5031 28943 : bool container_in_reg = false;
5032 28943 : if (REG_P (container))
5033 : container_in_reg = true;
5034 1641 : else if (GET_CODE (container) == PARALLEL
5035 1641 : && GET_MODE (container) == BLKmode
5036 580 : && XVECLEN (container, 0) == 1)
5037 : {
5038 : /* Check if it is a PARALLEL BLKmode container of an EXPR_LIST
5039 : expression in a TImode register. In this case, temp isn't
5040 : needed. Otherwise, the TImode variable will be put in the
5041 : GPR save area which guarantees only 8-byte alignment. */
5042 509 : rtx x = XVECEXP (container, 0, 0);
5043 509 : if (GET_CODE (x) == EXPR_LIST
5044 509 : && REG_P (XEXP (x, 0))
5045 509 : && XEXP (x, 1) == const0_rtx)
5046 : container_in_reg = true;
5047 : }
5048 :
5049 680 : need_temp = (!container_in_reg
5050 1150 : && ((needed_intregs && TYPE_ALIGN (type) > 64)
5051 680 : || TYPE_ALIGN (type) > 128));
5052 :
5053 : /* In case we are passing structure, verify that it is consecutive block
5054 : on the register save area. If not we need to do moves. */
5055 680 : if (!need_temp && !container_in_reg)
5056 : {
5057 : /* Verify that all registers are strictly consecutive */
5058 966 : if (SSE_REGNO_P (REGNO (XEXP (XVECEXP (container, 0, 0), 0))))
5059 : {
5060 : int i;
5061 :
5062 815 : for (i = 0; i < XVECLEN (container, 0) && !need_temp; i++)
5063 : {
5064 529 : rtx slot = XVECEXP (container, 0, i);
5065 529 : if (REGNO (XEXP (slot, 0)) != FIRST_SSE_REG + (unsigned int) i
5066 529 : || INTVAL (XEXP (slot, 1)) != i * 16)
5067 : need_temp = true;
5068 : }
5069 : }
5070 : else
5071 : {
5072 : int i;
5073 :
5074 1120 : for (i = 0; i < XVECLEN (container, 0) && !need_temp; i++)
5075 : {
5076 726 : rtx slot = XVECEXP (container, 0, i);
5077 726 : if (REGNO (XEXP (slot, 0)) != (unsigned int) i
5078 726 : || INTVAL (XEXP (slot, 1)) != i * 8)
5079 : need_temp = true;
5080 : }
5081 : }
5082 : }
5083 28943 : if (!need_temp)
5084 : {
5085 : int_addr = addr;
5086 : sse_addr = addr;
5087 : }
5088 : else
5089 : {
5090 877 : int_addr = create_tmp_var (ptr_type_node, "int_addr");
5091 877 : sse_addr = create_tmp_var (ptr_type_node, "sse_addr");
5092 : }
5093 :
5094 : /* First ensure that we fit completely in registers. */
5095 28943 : if (needed_intregs)
5096 : {
5097 18177 : t = build_int_cst (TREE_TYPE (gpr),
5098 18177 : (X86_64_REGPARM_MAX - needed_intregs + 1) * 8);
5099 18177 : t = build2 (GE_EXPR, boolean_type_node, gpr, t);
5100 18177 : t2 = build1 (GOTO_EXPR, void_type_node, lab_false);
5101 18177 : t = build3 (COND_EXPR, void_type_node, t, t2, NULL_TREE);
5102 18177 : gimplify_and_add (t, pre_p);
5103 : }
5104 28943 : if (needed_sseregs)
5105 : {
5106 11158 : t = build_int_cst (TREE_TYPE (fpr),
5107 : (X86_64_SSE_REGPARM_MAX - needed_sseregs + 1) * 16
5108 11158 : + X86_64_REGPARM_MAX * 8);
5109 11158 : t = build2 (GE_EXPR, boolean_type_node, fpr, t);
5110 11158 : t2 = build1 (GOTO_EXPR, void_type_node, lab_false);
5111 11158 : t = build3 (COND_EXPR, void_type_node, t, t2, NULL_TREE);
5112 11158 : gimplify_and_add (t, pre_p);
5113 : }
5114 :
5115 : /* Compute index to start of area used for integer regs. */
5116 28943 : if (needed_intregs)
5117 : {
5118 : /* int_addr = gpr + sav; */
5119 18177 : t = fold_build_pointer_plus (sav, gpr);
5120 18177 : gimplify_assign (int_addr, t, pre_p);
5121 : }
5122 28943 : if (needed_sseregs)
5123 : {
5124 : /* sse_addr = fpr + sav; */
5125 11158 : t = fold_build_pointer_plus (sav, fpr);
5126 11158 : gimplify_assign (sse_addr, t, pre_p);
5127 : }
5128 28943 : if (need_temp)
5129 : {
5130 877 : int i, prev_size = 0;
5131 877 : tree temp = create_tmp_var (type, "va_arg_tmp");
5132 877 : TREE_ADDRESSABLE (temp) = 1;
5133 :
5134 : /* addr = &temp; */
5135 877 : t = build1 (ADDR_EXPR, build_pointer_type (type), temp);
5136 877 : gimplify_assign (addr, t, pre_p);
5137 :
5138 2241 : for (i = 0; i < XVECLEN (container, 0); i++)
5139 : {
5140 1364 : rtx slot = XVECEXP (container, 0, i);
5141 1364 : rtx reg = XEXP (slot, 0);
5142 1364 : machine_mode mode = GET_MODE (reg);
5143 1364 : tree piece_type;
5144 1364 : tree addr_type;
5145 1364 : tree daddr_type;
5146 1364 : tree src_addr, src;
5147 1364 : int src_offset;
5148 1364 : tree dest_addr, dest;
5149 1364 : int cur_size = GET_MODE_SIZE (mode);
5150 :
5151 1364 : gcc_assert (prev_size <= INTVAL (XEXP (slot, 1)));
5152 1364 : prev_size = INTVAL (XEXP (slot, 1));
5153 1364 : if (prev_size + cur_size > size)
5154 : {
5155 30 : cur_size = size - prev_size;
5156 30 : unsigned int nbits = cur_size * BITS_PER_UNIT;
5157 30 : if (!int_mode_for_size (nbits, 1).exists (&mode))
5158 10 : mode = QImode;
5159 : }
5160 1364 : piece_type = lang_hooks.types.type_for_mode (mode, 1);
5161 1364 : if (mode == GET_MODE (reg))
5162 1334 : addr_type = build_pointer_type (piece_type);
5163 : else
5164 30 : addr_type = build_pointer_type_for_mode (piece_type, ptr_mode,
5165 : true);
5166 1364 : daddr_type = build_pointer_type_for_mode (piece_type, ptr_mode,
5167 : true);
5168 :
5169 1364 : if (SSE_REGNO_P (REGNO (reg)))
5170 : {
5171 534 : src_addr = sse_addr;
5172 534 : src_offset = (REGNO (reg) - FIRST_SSE_REG) * 16;
5173 : }
5174 : else
5175 : {
5176 830 : src_addr = int_addr;
5177 830 : src_offset = REGNO (reg) * 8;
5178 : }
5179 1364 : src_addr = fold_convert (addr_type, src_addr);
5180 1364 : src_addr = fold_build_pointer_plus_hwi (src_addr, src_offset);
5181 :
5182 1364 : dest_addr = fold_convert (daddr_type, addr);
5183 1364 : dest_addr = fold_build_pointer_plus_hwi (dest_addr, prev_size);
5184 2728 : if (cur_size == GET_MODE_SIZE (mode))
5185 : {
5186 1354 : src = build_va_arg_indirect_ref (src_addr);
5187 1354 : dest = build_va_arg_indirect_ref (dest_addr);
5188 :
5189 1354 : gimplify_assign (dest, src, pre_p);
5190 : }
5191 : else
5192 : {
5193 10 : tree copy
5194 20 : = build_call_expr (builtin_decl_implicit (BUILT_IN_MEMCPY),
5195 : 3, dest_addr, src_addr,
5196 10 : size_int (cur_size));
5197 10 : gimplify_and_add (copy, pre_p);
5198 : }
5199 1364 : prev_size += cur_size;
5200 : }
5201 : }
5202 :
5203 28943 : if (needed_intregs)
5204 : {
5205 18177 : t = build2 (PLUS_EXPR, TREE_TYPE (gpr), gpr,
5206 18177 : build_int_cst (TREE_TYPE (gpr), needed_intregs * 8));
5207 18177 : gimplify_assign (gpr, t, pre_p);
5208 : /* The GPR save area guarantees only 8-byte alignment. */
5209 18177 : if (!need_temp)
5210 17373 : type_align = MIN (type_align, 64);
5211 : }
5212 :
5213 28943 : if (needed_sseregs)
5214 : {
5215 11158 : t = build2 (PLUS_EXPR, TREE_TYPE (fpr), fpr,
5216 11158 : build_int_cst (TREE_TYPE (fpr), needed_sseregs * 16));
5217 11158 : gimplify_assign (unshare_expr (fpr), t, pre_p);
5218 : }
5219 :
5220 28943 : gimple_seq_add_stmt (pre_p, gimple_build_goto (lab_over));
5221 :
5222 28943 : gimple_seq_add_stmt (pre_p, gimple_build_label (lab_false));
5223 : }
5224 :
5225 : /* ... otherwise out of the overflow area. */
5226 :
5227 : /* When we align parameter on stack for caller, if the parameter
5228 : alignment is beyond MAX_SUPPORTED_STACK_ALIGNMENT, it will be
5229 : aligned at MAX_SUPPORTED_STACK_ALIGNMENT. We will match callee
5230 : here with caller. */
5231 52036 : arg_boundary = ix86_function_arg_boundary (VOIDmode, type);
5232 52036 : if ((unsigned int) arg_boundary > MAX_SUPPORTED_STACK_ALIGNMENT)
5233 : arg_boundary = MAX_SUPPORTED_STACK_ALIGNMENT;
5234 :
5235 : /* Care for on-stack alignment if needed. */
5236 52036 : if (arg_boundary <= 64 || size == 0)
5237 34996 : t = ovf;
5238 : else
5239 : {
5240 17040 : HOST_WIDE_INT align = arg_boundary / 8;
5241 17040 : t = fold_build_pointer_plus_hwi (ovf, align - 1);
5242 17040 : t = build2 (BIT_AND_EXPR, TREE_TYPE (t), t,
5243 17040 : build_int_cst (TREE_TYPE (t), -align));
5244 : }
5245 :
5246 52036 : gimplify_expr (&t, pre_p, NULL, is_gimple_val, fb_rvalue);
5247 52036 : gimplify_assign (addr, t, pre_p);
5248 :
5249 52036 : t = fold_build_pointer_plus_hwi (t, rsize * UNITS_PER_WORD);
5250 52036 : gimplify_assign (unshare_expr (ovf), t, pre_p);
5251 :
5252 52036 : if (container)
5253 28943 : gimple_seq_add_stmt (pre_p, gimple_build_label (lab_over));
5254 :
5255 52036 : type = build_aligned_type (type, type_align);
5256 52036 : ptrtype = build_pointer_type_for_mode (type, ptr_mode, true);
5257 52036 : addr = fold_convert (ptrtype, addr);
5258 :
5259 52036 : if (indirect_p)
5260 103 : addr = build_va_arg_indirect_ref (addr);
5261 52036 : return build_va_arg_indirect_ref (addr);
5262 : }
5263 :
5264 : /* Return true if OPNUM's MEM should be matched
5265 : in movabs* patterns. */
5266 :
5267 : bool
5268 480 : ix86_check_movabs (rtx insn, int opnum)
5269 : {
5270 480 : rtx set, mem;
5271 :
5272 480 : set = PATTERN (insn);
5273 480 : if (GET_CODE (set) == PARALLEL)
5274 0 : set = XVECEXP (set, 0, 0);
5275 480 : gcc_assert (GET_CODE (set) == SET);
5276 480 : mem = XEXP (set, opnum);
5277 480 : while (SUBREG_P (mem))
5278 0 : mem = SUBREG_REG (mem);
5279 480 : gcc_assert (MEM_P (mem));
5280 480 : return volatile_ok || !MEM_VOLATILE_P (mem);
5281 : }
5282 :
5283 : /* Return true if XVECEXP idx of INSN satisfies MOVS arguments. */
5284 : bool
5285 224444 : ix86_check_movs (rtx insn, int idx)
5286 : {
5287 224444 : rtx pat = PATTERN (insn);
5288 224444 : gcc_assert (GET_CODE (pat) == PARALLEL);
5289 :
5290 224444 : rtx set = XVECEXP (pat, 0, idx);
5291 224444 : gcc_assert (GET_CODE (set) == SET);
5292 :
5293 224444 : rtx dst = SET_DEST (set);
5294 224444 : gcc_assert (MEM_P (dst));
5295 :
5296 224444 : rtx src = SET_SRC (set);
5297 224444 : gcc_assert (MEM_P (src));
5298 :
5299 224444 : return (ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (dst))
5300 448888 : && (ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (src))
5301 0 : || Pmode == word_mode));
5302 : }
5303 :
5304 : /* Return false if INSN contains a MEM with a non-default address space. */
5305 : bool
5306 65497 : ix86_check_no_addr_space (rtx insn)
5307 : {
5308 65497 : subrtx_var_iterator::array_type array;
5309 1441386 : FOR_EACH_SUBRTX_VAR (iter, array, PATTERN (insn), ALL)
5310 : {
5311 1375889 : rtx x = *iter;
5312 1506883 : if (MEM_P (x) && !ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (x)))
5313 0 : return false;
5314 : }
5315 65497 : return true;
5316 65497 : }
5317 :
5318 : /* Initialize the table of extra 80387 mathematical constants. */
5319 :
5320 : static void
5321 2381 : init_ext_80387_constants (void)
5322 : {
5323 2381 : static const char * cst[5] =
5324 : {
5325 : "0.3010299956639811952256464283594894482", /* 0: fldlg2 */
5326 : "0.6931471805599453094286904741849753009", /* 1: fldln2 */
5327 : "1.4426950408889634073876517827983434472", /* 2: fldl2e */
5328 : "3.3219280948873623478083405569094566090", /* 3: fldl2t */
5329 : "3.1415926535897932385128089594061862044", /* 4: fldpi */
5330 : };
5331 2381 : int i;
5332 :
5333 14286 : for (i = 0; i < 5; i++)
5334 : {
5335 11905 : real_from_string (&ext_80387_constants_table[i], cst[i]);
5336 : /* Ensure each constant is rounded to XFmode precision. */
5337 11905 : real_convert (&ext_80387_constants_table[i],
5338 23810 : XFmode, &ext_80387_constants_table[i]);
5339 : }
5340 :
5341 2381 : ext_80387_constants_init = 1;
5342 2381 : }
5343 :
5344 : /* Return non-zero if the constant is something that
5345 : can be loaded with a special instruction. */
5346 :
5347 : int
5348 4941090 : standard_80387_constant_p (rtx x)
5349 : {
5350 4941090 : machine_mode mode = GET_MODE (x);
5351 :
5352 4941090 : const REAL_VALUE_TYPE *r;
5353 :
5354 4941090 : if (!(CONST_DOUBLE_P (x) && X87_FLOAT_MODE_P (mode)))
5355 : return -1;
5356 :
5357 4475205 : if (x == CONST0_RTX (mode))
5358 : return 1;
5359 2023136 : if (x == CONST1_RTX (mode))
5360 : return 2;
5361 :
5362 1226435 : r = CONST_DOUBLE_REAL_VALUE (x);
5363 :
5364 : /* For XFmode constants, try to find a special 80387 instruction when
5365 : optimizing for size or on those CPUs that benefit from them. */
5366 1226435 : if (mode == XFmode
5367 791294 : && (optimize_function_for_size_p (cfun) || TARGET_EXT_80387_CONSTANTS)
5368 2017729 : && !flag_rounding_math)
5369 : {
5370 783244 : int i;
5371 :
5372 783244 : if (! ext_80387_constants_init)
5373 2374 : init_ext_80387_constants ();
5374 :
5375 4688090 : for (i = 0; i < 5; i++)
5376 3914503 : if (real_identical (r, &ext_80387_constants_table[i]))
5377 9657 : return i + 3;
5378 : }
5379 :
5380 : /* Load of the constant -0.0 or -1.0 will be split as
5381 : fldz;fchs or fld1;fchs sequence. */
5382 1216778 : if (real_isnegzero (r))
5383 : return 8;
5384 1200350 : if (real_identical (r, &dconstm1))
5385 280185 : return 9;
5386 :
5387 : return 0;
5388 : }
5389 :
5390 : /* Return the opcode of the special instruction to be used to load
5391 : the constant X. */
5392 :
5393 : const char *
5394 44593 : standard_80387_constant_opcode (rtx x)
5395 : {
5396 44593 : switch (standard_80387_constant_p (x))
5397 : {
5398 : case 1:
5399 : return "fldz";
5400 24999 : case 2:
5401 24999 : return "fld1";
5402 1 : case 3:
5403 1 : return "fldlg2";
5404 10 : case 4:
5405 10 : return "fldln2";
5406 12 : case 5:
5407 12 : return "fldl2e";
5408 2 : case 6:
5409 2 : return "fldl2t";
5410 210 : case 7:
5411 210 : return "fldpi";
5412 0 : case 8:
5413 0 : case 9:
5414 0 : return "#";
5415 0 : default:
5416 0 : gcc_unreachable ();
5417 : }
5418 : }
5419 :
5420 : /* Return the CONST_DOUBLE representing the 80387 constant that is
5421 : loaded by the specified special instruction. The argument IDX
5422 : matches the return value from standard_80387_constant_p. */
5423 :
5424 : rtx
5425 24 : standard_80387_constant_rtx (int idx)
5426 : {
5427 24 : int i;
5428 :
5429 24 : if (! ext_80387_constants_init)
5430 7 : init_ext_80387_constants ();
5431 :
5432 24 : switch (idx)
5433 : {
5434 24 : case 3:
5435 24 : case 4:
5436 24 : case 5:
5437 24 : case 6:
5438 24 : case 7:
5439 24 : i = idx - 3;
5440 24 : break;
5441 :
5442 0 : default:
5443 0 : gcc_unreachable ();
5444 : }
5445 :
5446 24 : return const_double_from_real_value (ext_80387_constants_table[i],
5447 24 : XFmode);
5448 : }
5449 :
5450 : /* Return 1 if X is all bits 0, 2 if X is all bits 1
5451 : and 3 if X is all bits 1 with zero extend
5452 : in supported SSE/AVX vector mode. */
5453 :
5454 : int
5455 56569801 : standard_sse_constant_p (rtx x, machine_mode pred_mode)
5456 : {
5457 56569801 : machine_mode mode;
5458 :
5459 56569801 : if (!TARGET_SSE)
5460 : return 0;
5461 :
5462 56399041 : mode = GET_MODE (x);
5463 :
5464 56399041 : if (x == const0_rtx || const0_operand (x, mode))
5465 13597432 : return 1;
5466 :
5467 42801609 : if (x == constm1_rtx
5468 42661481 : || vector_all_ones_operand (x, mode)
5469 84842229 : || ((GET_MODE_CLASS (mode) == MODE_VECTOR_FLOAT
5470 35140510 : || GET_MODE_CLASS (pred_mode) == MODE_VECTOR_FLOAT)
5471 6901151 : && float_vector_all_ones_operand (x, mode)))
5472 : {
5473 : /* VOIDmode integer constant, get mode from the predicate. */
5474 763078 : if (mode == VOIDmode)
5475 140128 : mode = pred_mode;
5476 :
5477 1526156 : switch (GET_MODE_SIZE (mode))
5478 : {
5479 30838 : case 64:
5480 30838 : if (TARGET_AVX512F)
5481 : return 2;
5482 : break;
5483 40247 : case 32:
5484 40247 : if (TARGET_AVX2)
5485 : return 2;
5486 : break;
5487 678631 : case 16:
5488 678631 : if (TARGET_SSE2)
5489 : return 2;
5490 : break;
5491 0 : case 0:
5492 : /* VOIDmode */
5493 0 : gcc_unreachable ();
5494 : default:
5495 : break;
5496 : }
5497 : }
5498 :
5499 42052824 : if (vector_all_ones_zero_extend_half_operand (x, mode)
5500 42052824 : || vector_all_ones_zero_extend_quarter_operand (x, mode))
5501 706 : return 3;
5502 :
5503 : return 0;
5504 : }
5505 :
5506 : /* Return the opcode of the special instruction to be used to load
5507 : the constant operands[1] into operands[0]. */
5508 :
5509 : const char *
5510 471636 : standard_sse_constant_opcode (rtx_insn *insn, rtx *operands)
5511 : {
5512 471636 : machine_mode mode;
5513 471636 : rtx x = operands[1];
5514 :
5515 471636 : gcc_assert (TARGET_SSE);
5516 :
5517 471636 : mode = GET_MODE (x);
5518 :
5519 471636 : if (x == const0_rtx || const0_operand (x, mode))
5520 : {
5521 459314 : switch (get_attr_mode (insn))
5522 : {
5523 441419 : case MODE_TI:
5524 441419 : if (!EXT_REX_SSE_REG_P (operands[0]))
5525 : return "%vpxor\t%0, %d0";
5526 : /* FALLTHRU */
5527 6196 : case MODE_XI:
5528 6196 : case MODE_OI:
5529 6196 : if (EXT_REX_SSE_REG_P (operands[0]))
5530 : {
5531 67 : if (TARGET_AVX512VL)
5532 : return "vpxord\t%x0, %x0, %x0";
5533 : else
5534 28 : return "vpxord\t%g0, %g0, %g0";
5535 : }
5536 : return "vpxor\t%x0, %x0, %x0";
5537 :
5538 2102 : case MODE_V2DF:
5539 2102 : if (!EXT_REX_SSE_REG_P (operands[0]))
5540 : return "%vxorpd\t%0, %d0";
5541 : /* FALLTHRU */
5542 847 : case MODE_V8DF:
5543 847 : case MODE_V4DF:
5544 847 : if (EXT_REX_SSE_REG_P (operands[0]))
5545 : {
5546 4 : if (TARGET_AVX512DQ)
5547 : {
5548 0 : if (TARGET_AVX512VL)
5549 : return "vxorpd\t%x0, %x0, %x0";
5550 : else
5551 0 : return "vxorpd\t%g0, %g0, %g0";
5552 : }
5553 : else
5554 : {
5555 4 : if (TARGET_AVX512VL)
5556 : return "vpxorq\t%x0, %x0, %x0";
5557 : else
5558 4 : return "vpxorq\t%g0, %g0, %g0";
5559 : }
5560 : }
5561 : return "vxorpd\t%x0, %x0, %x0";
5562 :
5563 6793 : case MODE_V4SF:
5564 6793 : if (!EXT_REX_SSE_REG_P (operands[0]))
5565 : return "%vxorps\t%0, %d0";
5566 : /* FALLTHRU */
5567 2021 : case MODE_V16SF:
5568 2021 : case MODE_V8SF:
5569 2021 : if (EXT_REX_SSE_REG_P (operands[0]))
5570 : {
5571 70 : if (TARGET_AVX512DQ)
5572 : {
5573 26 : if (TARGET_AVX512VL)
5574 : return "vxorps\t%x0, %x0, %x0";
5575 : else
5576 0 : return "vxorps\t%g0, %g0, %g0";
5577 : }
5578 : else
5579 : {
5580 44 : if (TARGET_AVX512VL)
5581 : return "vpxord\t%x0, %x0, %x0";
5582 : else
5583 42 : return "vpxord\t%g0, %g0, %g0";
5584 : }
5585 : }
5586 : return "vxorps\t%x0, %x0, %x0";
5587 :
5588 0 : default:
5589 0 : gcc_unreachable ();
5590 : }
5591 : }
5592 12322 : else if (x == constm1_rtx
5593 12311 : || vector_all_ones_operand (x, mode)
5594 12389 : || (GET_MODE_CLASS (mode) == MODE_VECTOR_FLOAT
5595 45 : && float_vector_all_ones_operand (x, mode)))
5596 : {
5597 12300 : enum attr_mode insn_mode = get_attr_mode (insn);
5598 :
5599 12300 : switch (insn_mode)
5600 : {
5601 3 : case MODE_XI:
5602 3 : case MODE_V8DF:
5603 3 : case MODE_V16SF:
5604 3 : gcc_assert (TARGET_AVX512F);
5605 : return "vpternlogd\t{$0xFF, %g0, %g0, %g0|%g0, %g0, %g0, 0xFF}";
5606 :
5607 960 : case MODE_OI:
5608 960 : case MODE_V4DF:
5609 960 : case MODE_V8SF:
5610 960 : gcc_assert (TARGET_AVX2);
5611 : /* FALLTHRU */
5612 12297 : case MODE_TI:
5613 12297 : case MODE_V2DF:
5614 12297 : case MODE_V4SF:
5615 12297 : gcc_assert (TARGET_SSE2);
5616 12297 : if (EXT_REX_SSE_REG_P (operands[0]))
5617 : {
5618 2 : if (TARGET_AVX512VL)
5619 : return "vpternlogd\t{$0xFF, %0, %0, %0|%0, %0, %0, 0xFF}";
5620 : else
5621 0 : return "vpternlogd\t{$0xFF, %g0, %g0, %g0|%g0, %g0, %g0, 0xFF}";
5622 : }
5623 12295 : return (TARGET_AVX
5624 12295 : ? "vpcmpeqd\t%0, %0, %0"
5625 12295 : : "pcmpeqd\t%0, %0");
5626 :
5627 0 : default:
5628 0 : gcc_unreachable ();
5629 : }
5630 : }
5631 22 : else if (vector_all_ones_zero_extend_half_operand (x, mode))
5632 : {
5633 40 : if (GET_MODE_SIZE (mode) == 64)
5634 : {
5635 5 : gcc_assert (TARGET_AVX512F);
5636 : return "vpcmpeqd\t%t0, %t0, %t0";
5637 : }
5638 30 : else if (GET_MODE_SIZE (mode) == 32)
5639 : {
5640 15 : gcc_assert (TARGET_AVX);
5641 : return "vpcmpeqd\t%x0, %x0, %x0";
5642 : }
5643 0 : gcc_unreachable ();
5644 : }
5645 2 : else if (vector_all_ones_zero_extend_quarter_operand (x, mode))
5646 : {
5647 2 : gcc_assert (TARGET_AVX512F);
5648 : return "vpcmpeqd\t%x0, %x0, %x0";
5649 : }
5650 :
5651 0 : gcc_unreachable ();
5652 : }
5653 :
5654 : /* Returns true if INSN can be transformed from a memory load
5655 : to a supported FP constant load. */
5656 :
5657 : bool
5658 2151324 : ix86_standard_x87sse_constant_load_p (const rtx_insn *insn, rtx dst)
5659 : {
5660 2151324 : rtx src = find_constant_src (insn);
5661 :
5662 2151324 : gcc_assert (REG_P (dst));
5663 :
5664 2151324 : if (src == NULL
5665 593132 : || (SSE_REGNO_P (REGNO (dst))
5666 468854 : && standard_sse_constant_p (src, GET_MODE (dst)) != 1)
5667 158319 : || (!TARGET_AVX512VL
5668 158258 : && EXT_REX_SSE_REGNO_P (REGNO (dst))
5669 0 : && standard_sse_constant_p (src, GET_MODE (dst)) == 1)
5670 2309643 : || (STACK_REGNO_P (REGNO (dst))
5671 124278 : && standard_80387_constant_p (src) < 1))
5672 2083693 : return false;
5673 :
5674 : return true;
5675 : }
5676 :
5677 : /* Predicate for pre-reload splitters with associated instructions,
5678 : which can match any time before the split1 pass (usually combine),
5679 : then are unconditionally split in that pass and should not be
5680 : matched again afterwards. */
5681 :
5682 : bool
5683 18389002 : ix86_pre_reload_split (void)
5684 : {
5685 18389002 : return (can_create_pseudo_p ()
5686 28161138 : && !(cfun->curr_properties & PROP_rtl_split_insns));
5687 : }
5688 :
5689 : /* Return the opcode of the TYPE_SSEMOV instruction. To move from
5690 : or to xmm16-xmm31/ymm16-ymm31 registers, we either require
5691 : TARGET_AVX512VL or it is a register to register move which can
5692 : be done with zmm register move. */
5693 :
5694 : static const char *
5695 4111756 : ix86_get_ssemov (rtx *operands, unsigned size,
5696 : enum attr_mode insn_mode, machine_mode mode)
5697 : {
5698 4111756 : char buf[128];
5699 4111756 : bool misaligned_p = (misaligned_operand (operands[0], mode)
5700 4111756 : || misaligned_operand (operands[1], mode));
5701 4111756 : bool evex_reg_p = (size == 64
5702 4025080 : || EXT_REX_SSE_REG_P (operands[0])
5703 8136096 : || EXT_REX_SSE_REG_P (operands[1]));
5704 :
5705 4111756 : bool egpr_p = (TARGET_APX_EGPR
5706 4111756 : && (x86_extended_rex2reg_mentioned_p (operands[0])
5707 181 : || x86_extended_rex2reg_mentioned_p (operands[1])));
5708 197 : bool egpr_vl = egpr_p && TARGET_AVX512VL;
5709 :
5710 4111756 : machine_mode scalar_mode;
5711 :
5712 4111756 : const char *opcode = NULL;
5713 4111756 : enum
5714 : {
5715 : opcode_int,
5716 : opcode_float,
5717 : opcode_double
5718 4111756 : } type = opcode_int;
5719 :
5720 4111756 : switch (insn_mode)
5721 : {
5722 : case MODE_V16SF:
5723 : case MODE_V8SF:
5724 : case MODE_V4SF:
5725 : scalar_mode = E_SFmode;
5726 : type = opcode_float;
5727 : break;
5728 212602 : case MODE_V8DF:
5729 212602 : case MODE_V4DF:
5730 212602 : case MODE_V2DF:
5731 212602 : scalar_mode = E_DFmode;
5732 212602 : type = opcode_double;
5733 212602 : break;
5734 1474551 : case MODE_XI:
5735 1474551 : case MODE_OI:
5736 1474551 : case MODE_TI:
5737 1474551 : scalar_mode = GET_MODE_INNER (mode);
5738 : break;
5739 0 : default:
5740 0 : gcc_unreachable ();
5741 : }
5742 :
5743 : /* NB: To move xmm16-xmm31/ymm16-ymm31 registers without AVX512VL,
5744 : we can only use zmm register move without memory operand. */
5745 4111756 : if (evex_reg_p
5746 88724 : && !TARGET_AVX512VL
5747 4161791 : && GET_MODE_SIZE (mode) < 64)
5748 : {
5749 : /* NB: Even though ix86_hard_regno_mode_ok doesn't allow
5750 : xmm16-xmm31 nor ymm16-ymm31 in 128/256 bit modes when
5751 : AVX512VL is disabled, LRA can still generate reg to
5752 : reg moves with xmm16-xmm31 and ymm16-ymm31 in 128/256 bit
5753 : modes. */
5754 0 : if (memory_operand (operands[0], mode)
5755 0 : || memory_operand (operands[1], mode))
5756 0 : gcc_unreachable ();
5757 0 : size = 64;
5758 0 : switch (type)
5759 : {
5760 0 : case opcode_int:
5761 0 : if (scalar_mode == E_HFmode || scalar_mode == E_BFmode)
5762 0 : opcode = (misaligned_p
5763 0 : ? (TARGET_AVX512BW ? "vmovdqu16" : "vmovdqu64")
5764 : : "vmovdqa64");
5765 : else
5766 0 : opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32";
5767 : break;
5768 0 : case opcode_float:
5769 0 : opcode = misaligned_p ? "vmovups" : "vmovaps";
5770 : break;
5771 0 : case opcode_double:
5772 0 : opcode = misaligned_p ? "vmovupd" : "vmovapd";
5773 : break;
5774 : }
5775 : }
5776 4111756 : else if (SCALAR_FLOAT_MODE_P (scalar_mode))
5777 : {
5778 2814627 : switch (scalar_mode)
5779 : {
5780 36799 : case E_HFmode:
5781 36799 : case E_BFmode:
5782 36799 : if (evex_reg_p || egpr_vl)
5783 11597 : opcode = (misaligned_p
5784 173 : ? (TARGET_AVX512BW
5785 : ? "vmovdqu16"
5786 : : "vmovdqu64")
5787 : : "vmovdqa64");
5788 25202 : else if (egpr_p)
5789 735358 : opcode = (misaligned_p
5790 0 : ? (TARGET_AVX512BW
5791 0 : ? "vmovdqu16"
5792 : : "%vmovups")
5793 : : "%vmovaps");
5794 : else
5795 363381 : opcode = (misaligned_p
5796 25202 : ? (TARGET_AVX512BW && evex_reg_p
5797 : ? "vmovdqu16"
5798 : : "%vmovdqu")
5799 : : "%vmovdqa");
5800 : break;
5801 2424603 : case E_SFmode:
5802 2424603 : opcode = misaligned_p ? "%vmovups" : "%vmovaps";
5803 : break;
5804 212602 : case E_DFmode:
5805 212602 : opcode = misaligned_p ? "%vmovupd" : "%vmovapd";
5806 : break;
5807 140623 : case E_TFmode:
5808 140623 : if (evex_reg_p || egpr_vl)
5809 14 : opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64";
5810 140609 : else if (egpr_p)
5811 0 : opcode = misaligned_p ? "%vmovups" : "%vmovaps";
5812 : else
5813 140609 : opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa";
5814 : break;
5815 0 : default:
5816 0 : gcc_unreachable ();
5817 : }
5818 : }
5819 1297129 : else if (SCALAR_INT_MODE_P (scalar_mode))
5820 : {
5821 1297129 : switch (scalar_mode)
5822 : {
5823 73946 : case E_QImode:
5824 73946 : if (evex_reg_p || egpr_vl)
5825 4122310 : opcode = (misaligned_p
5826 10554 : ? (TARGET_AVX512BW
5827 5264 : ? "vmovdqu8"
5828 : : "vmovdqu64")
5829 : : "vmovdqa64");
5830 63392 : else if (egpr_p)
5831 30 : opcode = (misaligned_p
5832 0 : ? (TARGET_AVX512BW
5833 : ? "vmovdqu8"
5834 : : "%vmovups")
5835 : : "%vmovaps");
5836 : else
5837 63362 : opcode = (misaligned_p
5838 : ? (TARGET_AVX512BW && evex_reg_p
5839 : ? "vmovdqu8"
5840 : : "%vmovdqu")
5841 : : "%vmovdqa");
5842 : break;
5843 43968 : case E_HImode:
5844 43968 : if (evex_reg_p || egpr_vl)
5845 3691 : opcode = (misaligned_p
5846 304 : ? (TARGET_AVX512BW
5847 : ? "vmovdqu16"
5848 : : "vmovdqu64")
5849 : : "vmovdqa64");
5850 40277 : else if (egpr_p)
5851 735358 : opcode = (misaligned_p
5852 27 : ? (TARGET_AVX512BW
5853 0 : ? "vmovdqu16"
5854 : : "%vmovups")
5855 : : "%vmovaps");
5856 : else
5857 338179 : opcode = (misaligned_p
5858 40250 : ? (TARGET_AVX512BW && evex_reg_p
5859 : ? "vmovdqu16"
5860 : : "%vmovdqu")
5861 : : "%vmovdqa");
5862 : break;
5863 186973 : case E_SImode:
5864 186973 : if (evex_reg_p || egpr_vl)
5865 8241 : opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32";
5866 178732 : else if (egpr_p)
5867 14 : opcode = misaligned_p ? "%vmovups" : "%vmovaps";
5868 : else
5869 178718 : opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa";
5870 : break;
5871 980398 : case E_DImode:
5872 980398 : case E_TImode:
5873 980398 : case E_OImode:
5874 980398 : if (evex_reg_p || egpr_vl)
5875 18495 : opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64";
5876 961903 : else if (egpr_p)
5877 26 : opcode = misaligned_p ? "%vmovups" : "%vmovaps";
5878 : else
5879 961877 : opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa";
5880 : break;
5881 11844 : case E_XImode:
5882 49549 : opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64";
5883 : break;
5884 0 : default:
5885 0 : gcc_unreachable ();
5886 : }
5887 : }
5888 : else
5889 0 : gcc_unreachable ();
5890 :
5891 4111756 : switch (size)
5892 : {
5893 86676 : case 64:
5894 86676 : snprintf (buf, sizeof (buf), "%s\t{%%g1, %%g0|%%g0, %%g1}",
5895 : opcode);
5896 86676 : break;
5897 91696 : case 32:
5898 91696 : snprintf (buf, sizeof (buf), "%s\t{%%t1, %%t0|%%t0, %%t1}",
5899 : opcode);
5900 91696 : break;
5901 3933384 : case 16:
5902 3933384 : snprintf (buf, sizeof (buf), "%s\t{%%x1, %%x0|%%x0, %%x1}",
5903 : opcode);
5904 3933384 : break;
5905 0 : default:
5906 0 : gcc_unreachable ();
5907 : }
5908 4111756 : output_asm_insn (buf, operands);
5909 4111756 : return "";
5910 : }
5911 :
5912 : /* Return the template of the TYPE_SSEMOV instruction to move
5913 : operands[1] into operands[0]. */
5914 :
5915 : const char *
5916 6504659 : ix86_output_ssemov (rtx_insn *insn, rtx *operands)
5917 : {
5918 6504659 : machine_mode mode = GET_MODE (operands[0]);
5919 6504659 : if (get_attr_type (insn) != TYPE_SSEMOV
5920 6504659 : || mode != GET_MODE (operands[1]))
5921 0 : gcc_unreachable ();
5922 :
5923 6504659 : enum attr_mode insn_mode = get_attr_mode (insn);
5924 :
5925 6504659 : switch (insn_mode)
5926 : {
5927 86676 : case MODE_XI:
5928 86676 : case MODE_V8DF:
5929 86676 : case MODE_V16SF:
5930 86676 : return ix86_get_ssemov (operands, 64, insn_mode, mode);
5931 :
5932 91696 : case MODE_OI:
5933 91696 : case MODE_V4DF:
5934 91696 : case MODE_V8SF:
5935 91696 : return ix86_get_ssemov (operands, 32, insn_mode, mode);
5936 :
5937 3933384 : case MODE_TI:
5938 3933384 : case MODE_V2DF:
5939 3933384 : case MODE_V4SF:
5940 3933384 : return ix86_get_ssemov (operands, 16, insn_mode, mode);
5941 :
5942 669421 : case MODE_DI:
5943 : /* Handle broken assemblers that require movd instead of movq. */
5944 669421 : if (GENERAL_REG_P (operands[0]))
5945 : {
5946 : if (HAVE_AS_IX86_INTERUNIT_MOVQ)
5947 : return "%vmovq\t{%1, %q0|%q0, %1}";
5948 : else
5949 : return "%vmovd\t{%1, %q0|%q0, %1}";
5950 : }
5951 591704 : else if (GENERAL_REG_P (operands[1]))
5952 : {
5953 : if (HAVE_AS_IX86_INTERUNIT_MOVQ)
5954 : return "%vmovq\t{%q1, %0|%0, %q1}";
5955 : else
5956 : return "%vmovd\t{%q1, %0|%0, %q1}";
5957 : }
5958 : else
5959 420402 : return "%vmovq\t{%1, %0|%0, %1}";
5960 :
5961 210208 : case MODE_SI:
5962 210208 : if (GENERAL_REG_P (operands[0]))
5963 : return "%vmovd\t{%1, %k0|%k0, %1}";
5964 150187 : else if (GENERAL_REG_P (operands[1]))
5965 : return "%vmovd\t{%k1, %0|%0, %k1}";
5966 : else
5967 61593 : return "%vmovd\t{%1, %0|%0, %1}";
5968 :
5969 54076 : case MODE_HI:
5970 54076 : if (GENERAL_REG_P (operands[0]))
5971 : return "vmovw\t{%1, %k0|%k0, %1}";
5972 53913 : else if (GENERAL_REG_P (operands[1]))
5973 : return "vmovw\t{%k1, %0|%0, %k1}";
5974 : else
5975 53679 : return "vmovw\t{%1, %0|%0, %1}";
5976 :
5977 784700 : case MODE_DF:
5978 784700 : if (TARGET_AVX && REG_P (operands[0]) && REG_P (operands[1]))
5979 : return "vmovsd\t{%d1, %0|%0, %d1}";
5980 : else
5981 783885 : return "%vmovsd\t{%1, %0|%0, %1}";
5982 :
5983 670312 : case MODE_SF:
5984 670312 : if (TARGET_AVX && REG_P (operands[0]) && REG_P (operands[1]))
5985 : return "vmovss\t{%d1, %0|%0, %d1}";
5986 : else
5987 669745 : return "%vmovss\t{%1, %0|%0, %1}";
5988 :
5989 96 : case MODE_HF:
5990 96 : case MODE_BF:
5991 96 : if (REG_P (operands[0]) && REG_P (operands[1]))
5992 : return "vmovsh\t{%d1, %0|%0, %d1}";
5993 : else
5994 0 : return "vmovsh\t{%1, %0|%0, %1}";
5995 :
5996 36 : case MODE_V1DF:
5997 36 : gcc_assert (!TARGET_AVX);
5998 : return "movlpd\t{%1, %0|%0, %1}";
5999 :
6000 4054 : case MODE_V2SF:
6001 4054 : if (TARGET_AVX && REG_P (operands[0]))
6002 : return "vmovlps\t{%1, %d0|%d0, %1}";
6003 : else
6004 3981 : return "%vmovlps\t{%1, %0|%0, %1}";
6005 :
6006 0 : default:
6007 0 : gcc_unreachable ();
6008 : }
6009 : }
6010 :
6011 : /* Returns true if OP contains a symbol reference */
6012 :
6013 : bool
6014 578270616 : symbolic_reference_mentioned_p (const_rtx op)
6015 : {
6016 578270616 : const char *fmt;
6017 578270616 : int i;
6018 :
6019 578270616 : if (SYMBOL_REF_P (op) || LABEL_REF_P (op))
6020 : return true;
6021 :
6022 437154362 : fmt = GET_RTX_FORMAT (GET_CODE (op));
6023 741922002 : for (i = GET_RTX_LENGTH (GET_CODE (op)) - 1; i >= 0; i--)
6024 : {
6025 591951693 : if (fmt[i] == 'E')
6026 : {
6027 2030983 : int j;
6028 :
6029 4057918 : for (j = XVECLEN (op, i) - 1; j >= 0; j--)
6030 3339846 : if (symbolic_reference_mentioned_p (XVECEXP (op, i, j)))
6031 : return true;
6032 : }
6033 :
6034 589920710 : else if (fmt[i] == 'e' && symbolic_reference_mentioned_p (XEXP (op, i)))
6035 : return true;
6036 : }
6037 :
6038 : return false;
6039 : }
6040 :
6041 : /* Return true if it is appropriate to emit `ret' instructions in the
6042 : body of a function. Do this only if the epilogue is simple, needing a
6043 : couple of insns. Prior to reloading, we can't tell how many registers
6044 : must be saved, so return false then. Return false if there is no frame
6045 : marker to de-allocate. */
6046 :
6047 : bool
6048 0 : ix86_can_use_return_insn_p (void)
6049 : {
6050 0 : if (ix86_function_ms_hook_prologue (current_function_decl))
6051 : return false;
6052 :
6053 0 : if (ix86_function_naked (current_function_decl))
6054 : return false;
6055 :
6056 : /* Don't use `ret' instruction in interrupt handler. */
6057 0 : if (! reload_completed
6058 0 : || frame_pointer_needed
6059 0 : || cfun->machine->func_type != TYPE_NORMAL)
6060 : return 0;
6061 :
6062 : /* Don't allow more than 32k pop, since that's all we can do
6063 : with one instruction. */
6064 0 : if (crtl->args.pops_args && crtl->args.size >= 32768)
6065 : return 0;
6066 :
6067 0 : struct ix86_frame &frame = cfun->machine->frame;
6068 0 : return (frame.stack_pointer_offset == UNITS_PER_WORD
6069 0 : && (frame.nregs + frame.nsseregs) == 0);
6070 : }
6071 :
6072 : /* Return stack frame size. get_frame_size () returns used stack slots
6073 : during compilation, which may be optimized out later. If stack frame
6074 : is needed, stack_frame_required should be true. */
6075 :
6076 : static HOST_WIDE_INT
6077 8311330 : ix86_get_frame_size (void)
6078 : {
6079 8311330 : if (cfun->machine->stack_frame_required)
6080 8241921 : return get_frame_size ();
6081 : else
6082 : return 0;
6083 : }
6084 :
6085 : /* Value should be nonzero if functions must have frame pointers.
6086 : Zero means the frame pointer need not be set up (and parms may
6087 : be accessed via the stack pointer) in functions that seem suitable. */
6088 :
6089 : static bool
6090 1234286 : ix86_frame_pointer_required (void)
6091 : {
6092 : /* If we accessed previous frames, then the generated code expects
6093 : to be able to access the saved ebp value in our frame. */
6094 1234286 : if (cfun->machine->accesses_prev_frame)
6095 : return true;
6096 :
6097 : /* Several x86 os'es need a frame pointer for other reasons,
6098 : usually pertaining to setjmp. */
6099 1234253 : if (SUBTARGET_FRAME_POINTER_REQUIRED)
6100 : return true;
6101 :
6102 : /* For older 32-bit runtimes setjmp requires valid frame-pointer. */
6103 1234253 : if (TARGET_32BIT_MS_ABI && cfun->calls_setjmp)
6104 : return true;
6105 :
6106 : /* Win64 SEH, very large frames need a frame-pointer as maximum stack
6107 : allocation is 4GB. */
6108 1234253 : if (TARGET_64BIT_MS_ABI && ix86_get_frame_size () > SEH_MAX_FRAME_SIZE)
6109 : return true;
6110 :
6111 : /* SSE saves require frame-pointer when stack is misaligned. */
6112 1234253 : if (TARGET_64BIT_MS_ABI && ix86_incoming_stack_boundary < 128)
6113 : return true;
6114 :
6115 : /* In ix86_option_override_internal, TARGET_OMIT_LEAF_FRAME_POINTER
6116 : turns off the frame pointer by default. Turn it back on now if
6117 : we've not got a leaf function. */
6118 1234252 : if (TARGET_OMIT_LEAF_FRAME_POINTER
6119 1234252 : && (!crtl->is_leaf
6120 0 : || ix86_current_function_calls_tls_descriptor))
6121 0 : return true;
6122 :
6123 : /* Several versions of mcount for the x86 assumes that there is a
6124 : frame, so we cannot allow profiling without a frame pointer. */
6125 1234252 : if (crtl->profile && !flag_fentry)
6126 : return true;
6127 :
6128 : return false;
6129 : }
6130 :
6131 : /* Record that the current function accesses previous call frames. */
6132 :
6133 : void
6134 966 : ix86_setup_frame_addresses (void)
6135 : {
6136 966 : cfun->machine->accesses_prev_frame = 1;
6137 966 : }
6138 :
6139 : #if defined(HAVE_GAS_HIDDEN) && (SUPPORTS_ONE_ONLY - 0)
6140 : # define USE_HIDDEN_LINKONCE 1
6141 : #else
6142 : # define USE_HIDDEN_LINKONCE 0
6143 : #endif
6144 :
6145 : /* Label count for call and return thunks. It is used to make unique
6146 : labels in call and return thunks. */
6147 : static int indirectlabelno;
6148 :
6149 : /* True if call thunk function is needed. */
6150 : static bool indirect_thunk_needed = false;
6151 :
6152 : /* Bit masks of integer registers, which contain branch target, used
6153 : by call thunk functions. */
6154 : static HARD_REG_SET indirect_thunks_used;
6155 :
6156 : /* True if return thunk function is needed. */
6157 : static bool indirect_return_needed = false;
6158 :
6159 : /* True if return thunk function via CX is needed. */
6160 : static bool indirect_return_via_cx;
6161 :
6162 : #ifndef INDIRECT_LABEL
6163 : # define INDIRECT_LABEL "LIND"
6164 : #endif
6165 :
6166 : /* Indicate what prefix is needed for an indirect branch. */
6167 : enum indirect_thunk_prefix
6168 : {
6169 : indirect_thunk_prefix_none,
6170 : indirect_thunk_prefix_nt
6171 : };
6172 :
6173 : /* Return the prefix needed for an indirect branch INSN. */
6174 :
6175 : enum indirect_thunk_prefix
6176 67 : indirect_thunk_need_prefix (rtx_insn *insn)
6177 : {
6178 67 : enum indirect_thunk_prefix need_prefix;
6179 67 : if ((cfun->machine->indirect_branch_type
6180 67 : == indirect_branch_thunk_extern)
6181 67 : && ix86_notrack_prefixed_insn_p (insn))
6182 : {
6183 : /* NOTRACK prefix is only used with external thunk so that it
6184 : can be properly updated to support CET at run-time. */
6185 : need_prefix = indirect_thunk_prefix_nt;
6186 : }
6187 : else
6188 : need_prefix = indirect_thunk_prefix_none;
6189 67 : return need_prefix;
6190 : }
6191 :
6192 : /* Fills in the label name that should be used for the indirect thunk. */
6193 :
6194 : static void
6195 73 : indirect_thunk_name (char name[32], unsigned int regno,
6196 : enum indirect_thunk_prefix need_prefix,
6197 : bool ret_p)
6198 : {
6199 73 : if (regno != INVALID_REGNUM && regno != CX_REG && ret_p)
6200 0 : gcc_unreachable ();
6201 :
6202 73 : if (USE_HIDDEN_LINKONCE)
6203 : {
6204 73 : const char *prefix;
6205 :
6206 73 : if (need_prefix == indirect_thunk_prefix_nt
6207 73 : && regno != INVALID_REGNUM)
6208 : {
6209 : /* NOTRACK prefix is only used with external thunk via
6210 : register so that NOTRACK prefix can be added to indirect
6211 : branch via register to support CET at run-time. */
6212 : prefix = "_nt";
6213 : }
6214 : else
6215 71 : prefix = "";
6216 :
6217 73 : const char *ret = ret_p ? "return" : "indirect";
6218 :
6219 73 : if (regno != INVALID_REGNUM)
6220 : {
6221 55 : const char *reg_prefix;
6222 55 : if (LEGACY_INT_REGNO_P (regno))
6223 53 : reg_prefix = TARGET_64BIT ? "r" : "e";
6224 : else
6225 : reg_prefix = "";
6226 55 : sprintf (name, "__x86_%s_thunk%s_%s%s",
6227 : ret, prefix, reg_prefix, reg_names[regno]);
6228 : }
6229 : else
6230 18 : sprintf (name, "__x86_%s_thunk%s", ret, prefix);
6231 : }
6232 : else
6233 : {
6234 : if (regno != INVALID_REGNUM)
6235 : ASM_GENERATE_INTERNAL_LABEL (name, "LITR", regno);
6236 : else
6237 : {
6238 : if (ret_p)
6239 : ASM_GENERATE_INTERNAL_LABEL (name, "LRT", 0);
6240 : else
6241 73 : ASM_GENERATE_INTERNAL_LABEL (name, "LIT", 0);
6242 : }
6243 : }
6244 73 : }
6245 :
6246 : /* Output a call and return thunk for indirect branch. If REGNO != -1,
6247 : the function address is in REGNO and the call and return thunk looks like:
6248 :
6249 : call L2
6250 : L1:
6251 : pause
6252 : lfence
6253 : jmp L1
6254 : L2:
6255 : mov %REG, (%sp)
6256 : ret
6257 :
6258 : Otherwise, the function address is on the top of stack and the
6259 : call and return thunk looks like:
6260 :
6261 : call L2
6262 : L1:
6263 : pause
6264 : lfence
6265 : jmp L1
6266 : L2:
6267 : lea WORD_SIZE(%sp), %sp
6268 : ret
6269 : */
6270 :
6271 : static void
6272 38 : output_indirect_thunk (unsigned int regno)
6273 : {
6274 38 : char indirectlabel1[32];
6275 38 : char indirectlabel2[32];
6276 :
6277 38 : ASM_GENERATE_INTERNAL_LABEL (indirectlabel1, INDIRECT_LABEL,
6278 : indirectlabelno++);
6279 38 : ASM_GENERATE_INTERNAL_LABEL (indirectlabel2, INDIRECT_LABEL,
6280 : indirectlabelno++);
6281 :
6282 : /* Call */
6283 38 : fputs ("\tcall\t", asm_out_file);
6284 38 : assemble_name_raw (asm_out_file, indirectlabel2);
6285 38 : fputc ('\n', asm_out_file);
6286 :
6287 38 : ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel1);
6288 :
6289 : /* AMD and Intel CPUs prefer each a different instruction as loop filler.
6290 : Usage of both pause + lfence is compromise solution. */
6291 38 : fprintf (asm_out_file, "\tpause\n\tlfence\n");
6292 :
6293 : /* Jump. */
6294 38 : fputs ("\tjmp\t", asm_out_file);
6295 38 : assemble_name_raw (asm_out_file, indirectlabel1);
6296 38 : fputc ('\n', asm_out_file);
6297 :
6298 38 : ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel2);
6299 :
6300 : /* The above call insn pushed a word to stack. Adjust CFI info. */
6301 38 : if (flag_asynchronous_unwind_tables && dwarf2out_do_frame ())
6302 : {
6303 38 : if (! dwarf2out_do_cfi_asm ())
6304 : {
6305 0 : dw_cfi_ref xcfi = ggc_cleared_alloc<dw_cfi_node> ();
6306 0 : xcfi->dw_cfi_opc = DW_CFA_advance_loc4;
6307 0 : xcfi->dw_cfi_oprnd1.dw_cfi_addr = ggc_strdup (indirectlabel2);
6308 0 : vec_safe_push (cfun->fde->dw_fde_cfi, xcfi);
6309 : }
6310 38 : dw_cfi_ref xcfi = ggc_cleared_alloc<dw_cfi_node> ();
6311 38 : xcfi->dw_cfi_opc = DW_CFA_def_cfa_offset;
6312 38 : xcfi->dw_cfi_oprnd1.dw_cfi_offset = 2 * UNITS_PER_WORD;
6313 38 : vec_safe_push (cfun->fde->dw_fde_cfi, xcfi);
6314 38 : dwarf2out_emit_cfi (xcfi);
6315 : }
6316 :
6317 38 : if (regno != INVALID_REGNUM)
6318 : {
6319 : /* MOV. */
6320 27 : rtx xops[2];
6321 27 : xops[0] = gen_rtx_MEM (word_mode, stack_pointer_rtx);
6322 27 : xops[1] = gen_rtx_REG (word_mode, regno);
6323 27 : output_asm_insn ("mov\t{%1, %0|%0, %1}", xops);
6324 : }
6325 : else
6326 : {
6327 : /* LEA. */
6328 11 : rtx xops[2];
6329 11 : xops[0] = stack_pointer_rtx;
6330 11 : xops[1] = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
6331 11 : output_asm_insn ("lea\t{%E1, %0|%0, %E1}", xops);
6332 : }
6333 :
6334 38 : fputs ("\tret\n", asm_out_file);
6335 38 : if ((ix86_harden_sls & harden_sls_return))
6336 1 : fputs ("\tint3\n", asm_out_file);
6337 38 : }
6338 :
6339 : /* Output a function with a call and return thunk for indirect branch.
6340 : If REGNO != INVALID_REGNUM, the function address is in REGNO.
6341 : Otherwise, the function address is on the top of stack. Thunk is
6342 : used for function return if RET_P is true. */
6343 :
6344 : static void
6345 22 : output_indirect_thunk_function (enum indirect_thunk_prefix need_prefix,
6346 : unsigned int regno, bool ret_p)
6347 : {
6348 22 : char name[32];
6349 22 : tree decl;
6350 :
6351 : /* Create __x86_indirect_thunk. */
6352 22 : indirect_thunk_name (name, regno, need_prefix, ret_p);
6353 22 : decl = build_decl (BUILTINS_LOCATION, FUNCTION_DECL,
6354 : get_identifier (name),
6355 : build_function_type_list (void_type_node, NULL_TREE));
6356 22 : DECL_RESULT (decl) = build_decl (BUILTINS_LOCATION, RESULT_DECL,
6357 : NULL_TREE, void_type_node);
6358 22 : TREE_PUBLIC (decl) = 1;
6359 22 : TREE_STATIC (decl) = 1;
6360 22 : DECL_IGNORED_P (decl) = 1;
6361 :
6362 : #if TARGET_MACHO
6363 : if (TARGET_MACHO)
6364 : {
6365 : switch_to_section (darwin_sections[picbase_thunk_section]);
6366 : fputs ("\t.weak_definition\t", asm_out_file);
6367 : assemble_name (asm_out_file, name);
6368 : fputs ("\n\t.private_extern\t", asm_out_file);
6369 : assemble_name (asm_out_file, name);
6370 : putc ('\n', asm_out_file);
6371 : ASM_OUTPUT_LABEL (asm_out_file, name);
6372 : DECL_WEAK (decl) = 1;
6373 : }
6374 : else
6375 : #endif
6376 22 : if (USE_HIDDEN_LINKONCE)
6377 : {
6378 22 : cgraph_node::create (decl)->set_comdat_group (DECL_ASSEMBLER_NAME (decl));
6379 :
6380 22 : targetm.asm_out.unique_section (decl, 0);
6381 22 : switch_to_section (get_named_section (decl, NULL, 0));
6382 :
6383 22 : targetm.asm_out.globalize_label (asm_out_file, name);
6384 22 : fputs ("\t.hidden\t", asm_out_file);
6385 22 : assemble_name (asm_out_file, name);
6386 22 : putc ('\n', asm_out_file);
6387 22 : ASM_DECLARE_FUNCTION_NAME (asm_out_file, name, decl);
6388 : }
6389 : else
6390 : {
6391 : switch_to_section (text_section);
6392 22 : ASM_OUTPUT_LABEL (asm_out_file, name);
6393 : }
6394 :
6395 22 : DECL_INITIAL (decl) = make_node (BLOCK);
6396 22 : current_function_decl = decl;
6397 22 : allocate_struct_function (decl, false);
6398 22 : init_function_start (decl);
6399 : /* We're about to hide the function body from callees of final_* by
6400 : emitting it directly; tell them we're a thunk, if they care. */
6401 22 : cfun->is_thunk = true;
6402 22 : first_function_block_is_cold = false;
6403 : /* Make sure unwind info is emitted for the thunk if needed. */
6404 22 : final_start_function (emit_barrier (), asm_out_file, 1);
6405 :
6406 22 : output_indirect_thunk (regno);
6407 :
6408 22 : final_end_function ();
6409 22 : init_insn_lengths ();
6410 22 : free_after_compilation (cfun);
6411 22 : set_cfun (NULL);
6412 22 : current_function_decl = NULL;
6413 22 : }
6414 :
6415 : static int pic_labels_used;
6416 :
6417 : /* Fills in the label name that should be used for a pc thunk for
6418 : the given register. */
6419 :
6420 : static void
6421 37516 : get_pc_thunk_name (char name[32], unsigned int regno)
6422 : {
6423 37516 : gcc_assert (!TARGET_64BIT);
6424 :
6425 37516 : if (USE_HIDDEN_LINKONCE)
6426 37516 : sprintf (name, "__x86.get_pc_thunk.%s", reg_names[regno]);
6427 : else
6428 37516 : ASM_GENERATE_INTERNAL_LABEL (name, "LPR", regno);
6429 37516 : }
6430 :
6431 :
6432 : /* This function generates code for -fpic that loads %ebx with
6433 : the return address of the caller and then returns. */
6434 :
6435 : static void
6436 236254 : ix86_code_end (void)
6437 : {
6438 236254 : rtx xops[2];
6439 236254 : unsigned int regno;
6440 :
6441 236254 : if (indirect_return_needed)
6442 6 : output_indirect_thunk_function (indirect_thunk_prefix_none,
6443 : INVALID_REGNUM, true);
6444 236254 : if (indirect_return_via_cx)
6445 0 : output_indirect_thunk_function (indirect_thunk_prefix_none,
6446 : CX_REG, true);
6447 236254 : if (indirect_thunk_needed)
6448 0 : output_indirect_thunk_function (indirect_thunk_prefix_none,
6449 : INVALID_REGNUM, false);
6450 :
6451 2126286 : for (regno = FIRST_REX_INT_REG; regno <= LAST_REX_INT_REG; regno++)
6452 : {
6453 1890032 : if (TEST_HARD_REG_BIT (indirect_thunks_used, regno))
6454 0 : output_indirect_thunk_function (indirect_thunk_prefix_none,
6455 : regno, false);
6456 : }
6457 :
6458 4016318 : for (regno = FIRST_REX2_INT_REG; regno <= LAST_REX2_INT_REG; regno++)
6459 : {
6460 3780064 : if (TEST_HARD_REG_BIT (indirect_thunks_used, regno))
6461 0 : output_indirect_thunk_function (indirect_thunk_prefix_none,
6462 : regno, false);
6463 : }
6464 :
6465 2126286 : for (regno = FIRST_INT_REG; regno <= LAST_INT_REG; regno++)
6466 : {
6467 1890032 : char name[32];
6468 1890032 : tree decl;
6469 :
6470 1890032 : if (TEST_HARD_REG_BIT (indirect_thunks_used, regno))
6471 16 : output_indirect_thunk_function (indirect_thunk_prefix_none,
6472 : regno, false);
6473 :
6474 1890032 : if (!(pic_labels_used & (1 << regno)))
6475 1886446 : continue;
6476 :
6477 3586 : get_pc_thunk_name (name, regno);
6478 :
6479 3586 : decl = build_decl (BUILTINS_LOCATION, FUNCTION_DECL,
6480 : get_identifier (name),
6481 : build_function_type_list (void_type_node, NULL_TREE));
6482 3586 : DECL_RESULT (decl) = build_decl (BUILTINS_LOCATION, RESULT_DECL,
6483 : NULL_TREE, void_type_node);
6484 3586 : TREE_PUBLIC (decl) = 1;
6485 3586 : TREE_STATIC (decl) = 1;
6486 3586 : DECL_IGNORED_P (decl) = 1;
6487 :
6488 : #if TARGET_MACHO
6489 : if (TARGET_MACHO)
6490 : {
6491 : switch_to_section (darwin_sections[picbase_thunk_section]);
6492 : fputs ("\t.weak_definition\t", asm_out_file);
6493 : assemble_name (asm_out_file, name);
6494 : fputs ("\n\t.private_extern\t", asm_out_file);
6495 : assemble_name (asm_out_file, name);
6496 : putc ('\n', asm_out_file);
6497 : ASM_OUTPUT_LABEL (asm_out_file, name);
6498 : DECL_WEAK (decl) = 1;
6499 : }
6500 : else
6501 : #endif
6502 3586 : if (USE_HIDDEN_LINKONCE)
6503 : {
6504 3586 : cgraph_node::create (decl)->set_comdat_group (DECL_ASSEMBLER_NAME (decl));
6505 :
6506 3586 : targetm.asm_out.unique_section (decl, 0);
6507 3586 : switch_to_section (get_named_section (decl, NULL, 0));
6508 :
6509 3586 : targetm.asm_out.globalize_label (asm_out_file, name);
6510 3586 : fputs ("\t.hidden\t", asm_out_file);
6511 3586 : assemble_name (asm_out_file, name);
6512 3586 : putc ('\n', asm_out_file);
6513 3586 : ASM_DECLARE_FUNCTION_NAME (asm_out_file, name, decl);
6514 : }
6515 : else
6516 : {
6517 : switch_to_section (text_section);
6518 3586 : ASM_OUTPUT_LABEL (asm_out_file, name);
6519 : }
6520 :
6521 3586 : DECL_INITIAL (decl) = make_node (BLOCK);
6522 3586 : current_function_decl = decl;
6523 3586 : allocate_struct_function (decl, false);
6524 3586 : init_function_start (decl);
6525 : /* We're about to hide the function body from callees of final_* by
6526 : emitting it directly; tell them we're a thunk, if they care. */
6527 3586 : cfun->is_thunk = true;
6528 3586 : first_function_block_is_cold = false;
6529 : /* Make sure unwind info is emitted for the thunk if needed. */
6530 3586 : final_start_function (emit_barrier (), asm_out_file, 1);
6531 :
6532 : /* Pad stack IP move with 4 instructions (two NOPs count
6533 : as one instruction). */
6534 3586 : if (TARGET_PAD_SHORT_FUNCTION)
6535 : {
6536 : int i = 8;
6537 :
6538 0 : while (i--)
6539 0 : fputs ("\tnop\n", asm_out_file);
6540 : }
6541 :
6542 7172 : xops[0] = gen_rtx_REG (Pmode, regno);
6543 7172 : xops[1] = gen_rtx_MEM (Pmode, stack_pointer_rtx);
6544 3586 : output_asm_insn ("mov%z0\t{%1, %0|%0, %1}", xops);
6545 3586 : fputs ("\tret\n", asm_out_file);
6546 3586 : final_end_function ();
6547 3586 : init_insn_lengths ();
6548 3586 : free_after_compilation (cfun);
6549 3586 : set_cfun (NULL);
6550 3586 : current_function_decl = NULL;
6551 : }
6552 :
6553 236254 : if (flag_split_stack)
6554 4712 : file_end_indicate_split_stack ();
6555 236254 : }
6556 :
6557 : /* Emit code for the SET_GOT patterns. */
6558 :
6559 : const char *
6560 33930 : output_set_got (rtx dest, rtx label)
6561 : {
6562 33930 : rtx xops[3];
6563 :
6564 33930 : xops[0] = dest;
6565 :
6566 33930 : if (TARGET_VXWORKS_GOTTPIC && TARGET_VXWORKS_RTP && flag_pic)
6567 : {
6568 : /* Load (*VXWORKS_GOTT_BASE) into the PIC register. */
6569 : xops[2] = gen_rtx_MEM (Pmode,
6570 : gen_rtx_SYMBOL_REF (Pmode, VXWORKS_GOTT_BASE));
6571 : output_asm_insn ("mov{l}\t{%2, %0|%0, %2}", xops);
6572 :
6573 : /* Load (*VXWORKS_GOTT_BASE)[VXWORKS_GOTT_INDEX] into the PIC register.
6574 : Use %P and a local symbol in order to print VXWORKS_GOTT_INDEX as
6575 : an unadorned address. */
6576 : xops[2] = gen_rtx_SYMBOL_REF (Pmode, VXWORKS_GOTT_INDEX);
6577 : SYMBOL_REF_FLAGS (xops[2]) |= SYMBOL_FLAG_LOCAL;
6578 : output_asm_insn ("mov{l}\t{%P2(%0), %0|%0, DWORD PTR %P2[%0]}", xops);
6579 : return "";
6580 : }
6581 :
6582 67860 : xops[1] = gen_rtx_SYMBOL_REF (Pmode, GOT_SYMBOL_NAME);
6583 :
6584 33930 : if (flag_pic)
6585 : {
6586 33930 : char name[32];
6587 33930 : get_pc_thunk_name (name, REGNO (dest));
6588 33930 : pic_labels_used |= 1 << REGNO (dest);
6589 :
6590 67860 : xops[2] = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (name));
6591 33930 : xops[2] = gen_rtx_MEM (QImode, xops[2]);
6592 33930 : output_asm_insn ("%!call\t%X2", xops);
6593 :
6594 : #if TARGET_MACHO
6595 : /* Output the Mach-O "canonical" pic base label name ("Lxx$pb") here.
6596 : This is what will be referenced by the Mach-O PIC subsystem. */
6597 : if (machopic_should_output_picbase_label () || !label)
6598 : ASM_OUTPUT_LABEL (asm_out_file, MACHOPIC_FUNCTION_BASE_NAME);
6599 :
6600 : /* When we are restoring the pic base at the site of a nonlocal label,
6601 : and we decided to emit the pic base above, we will still output a
6602 : local label used for calculating the correction offset (even though
6603 : the offset will be 0 in that case). */
6604 : if (label)
6605 : targetm.asm_out.internal_label (asm_out_file, "L",
6606 : CODE_LABEL_NUMBER (label));
6607 : #endif
6608 : }
6609 : else
6610 : {
6611 0 : if (TARGET_MACHO)
6612 : /* We don't need a pic base, we're not producing pic. */
6613 : gcc_unreachable ();
6614 :
6615 0 : xops[2] = gen_rtx_LABEL_REF (Pmode, label ? label : gen_label_rtx ());
6616 0 : output_asm_insn ("mov%z0\t{%2, %0|%0, %2}", xops);
6617 0 : targetm.asm_out.internal_label (asm_out_file, "L",
6618 0 : CODE_LABEL_NUMBER (XEXP (xops[2], 0)));
6619 : }
6620 :
6621 33930 : if (!TARGET_MACHO)
6622 33930 : output_asm_insn ("add%z0\t{%1, %0|%0, %1}", xops);
6623 :
6624 33930 : return "";
6625 : }
6626 :
6627 : /* Generate an "push" pattern for input ARG. */
6628 :
6629 : rtx
6630 1895000 : gen_push (rtx arg, bool ppx_p)
6631 : {
6632 1895000 : struct machine_function *m = cfun->machine;
6633 :
6634 1895000 : if (m->fs.cfa_reg == stack_pointer_rtx)
6635 1622681 : m->fs.cfa_offset += UNITS_PER_WORD;
6636 1895000 : m->fs.sp_offset += UNITS_PER_WORD;
6637 :
6638 1895000 : if (REG_P (arg) && GET_MODE (arg) != word_mode)
6639 26 : arg = gen_rtx_REG (word_mode, REGNO (arg));
6640 :
6641 1895000 : rtx stack = gen_rtx_MEM (word_mode,
6642 1895000 : gen_rtx_PRE_DEC (Pmode,
6643 : stack_pointer_rtx));
6644 3789911 : return ppx_p ? gen_pushp_di (stack, arg) : gen_rtx_SET (stack, arg);
6645 : }
6646 :
6647 : rtx
6648 21 : gen_pushfl (void)
6649 : {
6650 21 : struct machine_function *m = cfun->machine;
6651 21 : rtx flags, mem;
6652 :
6653 21 : if (m->fs.cfa_reg == stack_pointer_rtx)
6654 0 : m->fs.cfa_offset += UNITS_PER_WORD;
6655 21 : m->fs.sp_offset += UNITS_PER_WORD;
6656 :
6657 21 : flags = gen_rtx_REG (CCmode, FLAGS_REG);
6658 :
6659 21 : mem = gen_rtx_MEM (word_mode,
6660 21 : gen_rtx_PRE_DEC (Pmode, stack_pointer_rtx));
6661 :
6662 21 : return gen_pushfl2 (word_mode, mem, flags);
6663 : }
6664 :
6665 : /* Generate an "pop" pattern for input ARG. */
6666 :
6667 : rtx
6668 1475243 : gen_pop (rtx arg, bool ppx_p)
6669 : {
6670 1475243 : if (REG_P (arg) && GET_MODE (arg) != word_mode)
6671 22 : arg = gen_rtx_REG (word_mode, REGNO (arg));
6672 :
6673 1475243 : rtx stack = gen_rtx_MEM (word_mode,
6674 1475243 : gen_rtx_POST_INC (Pmode,
6675 : stack_pointer_rtx));
6676 :
6677 2950397 : return ppx_p ? gen_popp_di (arg, stack) : gen_rtx_SET (arg, stack);
6678 : }
6679 :
6680 : rtx
6681 21 : gen_popfl (void)
6682 : {
6683 21 : rtx flags, mem;
6684 :
6685 21 : flags = gen_rtx_REG (CCmode, FLAGS_REG);
6686 :
6687 21 : mem = gen_rtx_MEM (word_mode,
6688 21 : gen_rtx_POST_INC (Pmode, stack_pointer_rtx));
6689 :
6690 21 : return gen_popfl1 (word_mode, flags, mem);
6691 : }
6692 :
6693 : /* Generate a "push2" pattern for input ARG. */
6694 : rtx
6695 19 : gen_push2 (rtx mem, rtx reg1, rtx reg2, bool ppx_p = false)
6696 : {
6697 19 : struct machine_function *m = cfun->machine;
6698 19 : const int offset = UNITS_PER_WORD * 2;
6699 :
6700 19 : if (m->fs.cfa_reg == stack_pointer_rtx)
6701 14 : m->fs.cfa_offset += offset;
6702 19 : m->fs.sp_offset += offset;
6703 :
6704 19 : if (REG_P (reg1) && GET_MODE (reg1) != word_mode)
6705 0 : reg1 = gen_rtx_REG (word_mode, REGNO (reg1));
6706 :
6707 19 : if (REG_P (reg2) && GET_MODE (reg2) != word_mode)
6708 0 : reg2 = gen_rtx_REG (word_mode, REGNO (reg2));
6709 :
6710 19 : return ppx_p ? gen_push2p_di (mem, reg1, reg2)
6711 4 : : gen_push2_di (mem, reg1, reg2);
6712 : }
6713 :
6714 : /* Return >= 0 if there is an unused call-clobbered register available
6715 : for the entire function. */
6716 :
6717 : static unsigned int
6718 0 : ix86_select_alt_pic_regnum (void)
6719 : {
6720 0 : if (ix86_use_pseudo_pic_reg ())
6721 : return INVALID_REGNUM;
6722 :
6723 0 : if (crtl->is_leaf
6724 0 : && !crtl->profile
6725 0 : && !ix86_current_function_calls_tls_descriptor)
6726 : {
6727 0 : int i, drap;
6728 : /* Can't use the same register for both PIC and DRAP. */
6729 0 : if (crtl->drap_reg)
6730 0 : drap = REGNO (crtl->drap_reg);
6731 : else
6732 : drap = -1;
6733 0 : for (i = 2; i >= 0; --i)
6734 0 : if (i != drap && !df_regs_ever_live_p (i))
6735 : return i;
6736 : }
6737 :
6738 : return INVALID_REGNUM;
6739 : }
6740 :
6741 : /* Return true if REGNO is used by the epilogue. */
6742 :
6743 : bool
6744 1678667058 : ix86_epilogue_uses (int regno)
6745 : {
6746 : /* If there are no caller-saved registers, we preserve all registers,
6747 : except for MMX and x87 registers which aren't supported when saving
6748 : and restoring registers. Don't explicitly save SP register since
6749 : it is always preserved. */
6750 1678667058 : return (epilogue_completed
6751 265119374 : && (cfun->machine->call_saved_registers
6752 265119374 : == TYPE_NO_CALLER_SAVED_REGISTERS)
6753 29348 : && !fixed_regs[regno]
6754 5441 : && !STACK_REGNO_P (regno)
6755 1678672499 : && !MMX_REGNO_P (regno));
6756 : }
6757 :
6758 : /* Return nonzero if register REGNO can be used as a scratch register
6759 : in peephole2. */
6760 :
6761 : static bool
6762 1229116 : ix86_hard_regno_scratch_ok (unsigned int regno)
6763 : {
6764 : /* If there are no caller-saved registers, we can't use any register
6765 : as a scratch register after epilogue and use REGNO as scratch
6766 : register only if it has been used before to avoid saving and
6767 : restoring it. */
6768 1229116 : return ((cfun->machine->call_saved_registers
6769 1229116 : != TYPE_NO_CALLER_SAVED_REGISTERS)
6770 1229116 : || (!epilogue_completed
6771 0 : && df_regs_ever_live_p (regno)));
6772 : }
6773 :
6774 : /* Return TRUE if we need to save REGNO. */
6775 :
6776 : bool
6777 356205366 : ix86_save_reg (unsigned int regno, bool maybe_eh_return, bool ignore_outlined)
6778 : {
6779 : /* Save and restore DRAP register between prologue and epilogue so
6780 : that stack pointer can be restored. */
6781 356205366 : if (crtl->drap_reg
6782 2266982 : && regno == REGNO (crtl->drap_reg)
6783 356260412 : && !cfun->machine->no_drap_save_restore)
6784 : return true;
6785 :
6786 : /* ??? Treat no_callee_saved_registers as a special case in order
6787 : to cope with -mnoreturn-no-callee-saved-registers, which is not
6788 : reflected in crtl->abi. */
6789 356150320 : if (cfun->machine->call_saved_registers == TYPE_NO_CALLEE_SAVED_REGISTERS
6790 16696 : && regno != HARD_FRAME_POINTER_REGNUM)
6791 : return false;
6792 :
6793 390154745 : if (regno == REAL_PIC_OFFSET_TABLE_REGNUM
6794 10837061 : && pic_offset_table_rtx)
6795 : {
6796 385674 : if (ix86_use_pseudo_pic_reg ())
6797 : {
6798 : /* REAL_PIC_OFFSET_TABLE_REGNUM used by call to
6799 : _mcount in prologue. */
6800 385674 : if (!TARGET_64BIT && flag_pic && crtl->profile)
6801 : return true;
6802 : }
6803 0 : else if (df_regs_ever_live_p (REAL_PIC_OFFSET_TABLE_REGNUM)
6804 0 : || crtl->profile
6805 0 : || crtl->calls_eh_return
6806 0 : || crtl->uses_const_pool
6807 0 : || cfun->has_nonlocal_label)
6808 0 : return ix86_select_alt_pic_regnum () == INVALID_REGNUM;
6809 : }
6810 :
6811 356134126 : if (crtl->calls_eh_return && maybe_eh_return)
6812 : {
6813 : unsigned i;
6814 13237 : for (i = 0; ; i++)
6815 : {
6816 20181 : unsigned test = EH_RETURN_DATA_REGNO (i);
6817 13671 : if (test == INVALID_REGNUM)
6818 : break;
6819 13671 : if (test == regno)
6820 : return true;
6821 13237 : }
6822 : }
6823 :
6824 356133692 : if (ignore_outlined && cfun->machine->call_ms2sysv)
6825 : {
6826 2642240 : unsigned count = cfun->machine->call_ms2sysv_extra_regs
6827 : + xlogue_layout::MIN_REGS;
6828 2642240 : if (xlogue_layout::is_stub_managed_reg (regno, count))
6829 : return false;
6830 : }
6831 :
6832 355634599 : return (df_regs_ever_live_p (regno)
6833 67802983 : && !fixed_regs[regno]
6834 59882195 : && !crtl->abi->clobbers_full_reg_p (regno)
6835 374480036 : && (regno != HARD_FRAME_POINTER_REGNUM || !frame_pointer_needed));
6836 : }
6837 :
6838 : /* Return number of saved general prupose registers. */
6839 :
6840 : static int
6841 8235665 : ix86_nsaved_regs (void)
6842 : {
6843 8235665 : int nregs = 0;
6844 8235665 : int regno;
6845 :
6846 765916845 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
6847 757681180 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true, true))
6848 8252615 : nregs ++;
6849 8235665 : return nregs;
6850 : }
6851 :
6852 : /* Return number of saved SSE registers. */
6853 :
6854 : static int
6855 8270653 : ix86_nsaved_sseregs (void)
6856 : {
6857 8270653 : int nregs = 0;
6858 8270653 : int regno;
6859 :
6860 7467149 : if (!TARGET_64BIT_MS_ABI
6861 8270653 : && (cfun->machine->call_saved_registers
6862 8044875 : != TYPE_NO_CALLER_SAVED_REGISTERS))
6863 : return 0;
6864 21072591 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
6865 20846004 : if (SSE_REGNO_P (regno) && ix86_save_reg (regno, true, true))
6866 1897653 : nregs ++;
6867 : return nregs;
6868 : }
6869 :
6870 : /* Given FROM and TO register numbers, say whether this elimination is
6871 : allowed. If stack alignment is needed, we can only replace argument
6872 : pointer with hard frame pointer, or replace frame pointer with stack
6873 : pointer. Otherwise, frame pointer elimination is automatically
6874 : handled and all other eliminations are valid. */
6875 :
6876 : static bool
6877 48762477 : ix86_can_eliminate (const int from, const int to)
6878 : {
6879 48762477 : if (stack_realign_fp)
6880 1691500 : return ((from == ARG_POINTER_REGNUM
6881 1691500 : && to == HARD_FRAME_POINTER_REGNUM)
6882 1691500 : || (from == FRAME_POINTER_REGNUM
6883 1691500 : && to == STACK_POINTER_REGNUM));
6884 : else
6885 87527426 : return to == STACK_POINTER_REGNUM ? !frame_pointer_needed : true;
6886 : }
6887 :
6888 : /* Return the offset between two registers, one to be eliminated, and the other
6889 : its replacement, at the start of a routine. */
6890 :
6891 : HOST_WIDE_INT
6892 141612728 : ix86_initial_elimination_offset (int from, int to)
6893 : {
6894 141612728 : struct ix86_frame &frame = cfun->machine->frame;
6895 :
6896 141612728 : if (from == ARG_POINTER_REGNUM && to == HARD_FRAME_POINTER_REGNUM)
6897 10489892 : return frame.hard_frame_pointer_offset;
6898 131122836 : else if (from == FRAME_POINTER_REGNUM
6899 131122836 : && to == HARD_FRAME_POINTER_REGNUM)
6900 8200302 : return frame.hard_frame_pointer_offset - frame.frame_pointer_offset;
6901 : else
6902 : {
6903 122922534 : gcc_assert (to == STACK_POINTER_REGNUM);
6904 :
6905 122922534 : if (from == ARG_POINTER_REGNUM)
6906 114722232 : return frame.stack_pointer_offset;
6907 :
6908 8200302 : gcc_assert (from == FRAME_POINTER_REGNUM);
6909 8200302 : return frame.stack_pointer_offset - frame.frame_pointer_offset;
6910 : }
6911 : }
6912 :
6913 : /* Emits a warning for unsupported msabi to sysv pro/epilogues. */
6914 : void
6915 0 : warn_once_call_ms2sysv_xlogues (const char *feature)
6916 : {
6917 0 : static bool warned_once = false;
6918 0 : if (!warned_once)
6919 : {
6920 0 : warning (0, "%<-mcall-ms2sysv-xlogues%> is not compatible with %s",
6921 : feature);
6922 0 : warned_once = true;
6923 : }
6924 0 : }
6925 :
6926 : /* Return the probing interval for -fstack-clash-protection. */
6927 :
6928 : static HOST_WIDE_INT
6929 494 : get_probe_interval (void)
6930 : {
6931 341 : if (flag_stack_clash_protection)
6932 412 : return (HOST_WIDE_INT_1U
6933 412 : << param_stack_clash_protection_probe_interval);
6934 : else
6935 : return (HOST_WIDE_INT_1U << STACK_CHECK_PROBE_INTERVAL_EXP);
6936 : }
6937 :
6938 : /* When using -fsplit-stack, the allocation routines set a field in
6939 : the TCB to the bottom of the stack plus this much space, measured
6940 : in bytes. */
6941 :
6942 : #define SPLIT_STACK_AVAILABLE 256
6943 :
6944 : /* Return true if push2/pop2 can be generated. */
6945 :
6946 : static bool
6947 8236329 : ix86_can_use_push2pop2 (void)
6948 : {
6949 : /* Use push2/pop2 only if the incoming stack is 16-byte aligned. */
6950 8236329 : unsigned int incoming_stack_boundary
6951 8236329 : = (crtl->parm_stack_boundary > ix86_incoming_stack_boundary
6952 8236329 : ? crtl->parm_stack_boundary : ix86_incoming_stack_boundary);
6953 8236329 : return incoming_stack_boundary % 128 == 0;
6954 : }
6955 :
6956 : /* Helper function to determine whether push2/pop2 can be used in prologue or
6957 : epilogue for register save/restore. */
6958 : static bool
6959 8235665 : ix86_pro_and_epilogue_can_use_push2pop2 (int nregs)
6960 : {
6961 8235665 : if (!ix86_can_use_push2pop2 ())
6962 : return false;
6963 8199744 : int aligned = cfun->machine->fs.sp_offset % 16 == 0;
6964 8199744 : return TARGET_APX_PUSH2POP2
6965 2887 : && !cfun->machine->frame.save_regs_using_mov
6966 2875 : && cfun->machine->func_type == TYPE_NORMAL
6967 8202611 : && (nregs + aligned) >= 3;
6968 : }
6969 :
6970 : /* Check if push/pop should be used to save/restore registers. */
6971 : static bool
6972 8966261 : save_regs_using_push_pop (HOST_WIDE_INT to_allocate)
6973 : {
6974 3212430 : return ((!to_allocate && cfun->machine->frame.nregs <= 1)
6975 5996842 : || (TARGET_64BIT && to_allocate >= HOST_WIDE_INT_C (0x80000000))
6976 : /* If static stack checking is enabled and done with probes,
6977 : the registers need to be saved before allocating the frame. */
6978 5996181 : || flag_stack_check == STATIC_BUILTIN_STACK_CHECK
6979 : /* If stack clash probing needs a loop, then it needs a
6980 : scratch register. But the returned register is only guaranteed
6981 : to be safe to use after register saves are complete. So if
6982 : stack clash protections are enabled and the allocated frame is
6983 : larger than the probe interval, then use pushes to save
6984 : callee saved registers. */
6985 14962368 : || (flag_stack_clash_protection
6986 341 : && !ix86_target_stack_probe ()
6987 341 : && to_allocate > get_probe_interval ()));
6988 : }
6989 :
6990 : /* Fill structure ix86_frame about frame of currently computed function. */
6991 :
6992 : static void
6993 8235665 : ix86_compute_frame_layout (void)
6994 : {
6995 8235665 : struct ix86_frame *frame = &cfun->machine->frame;
6996 8235665 : struct machine_function *m = cfun->machine;
6997 8235665 : unsigned HOST_WIDE_INT stack_alignment_needed;
6998 8235665 : HOST_WIDE_INT offset;
6999 8235665 : unsigned HOST_WIDE_INT preferred_alignment;
7000 8235665 : HOST_WIDE_INT size = ix86_get_frame_size ();
7001 8235665 : HOST_WIDE_INT to_allocate;
7002 :
7003 : /* m->call_ms2sysv is initially enabled in ix86_expand_call for all 64-bit
7004 : * ms_abi functions that call a sysv function. We now need to prune away
7005 : * cases where it should be disabled. */
7006 8235665 : if (TARGET_64BIT && m->call_ms2sysv)
7007 : {
7008 35225 : gcc_assert (TARGET_64BIT_MS_ABI);
7009 35225 : gcc_assert (TARGET_CALL_MS2SYSV_XLOGUES);
7010 35225 : gcc_assert (!TARGET_SEH);
7011 35225 : gcc_assert (TARGET_SSE);
7012 35225 : gcc_assert (!ix86_using_red_zone ());
7013 :
7014 35225 : if (crtl->calls_eh_return)
7015 : {
7016 0 : gcc_assert (!reload_completed);
7017 0 : m->call_ms2sysv = false;
7018 0 : warn_once_call_ms2sysv_xlogues ("__builtin_eh_return");
7019 : }
7020 :
7021 35225 : else if (ix86_static_chain_on_stack)
7022 : {
7023 0 : gcc_assert (!reload_completed);
7024 0 : m->call_ms2sysv = false;
7025 0 : warn_once_call_ms2sysv_xlogues ("static call chains");
7026 : }
7027 :
7028 : /* Finally, compute which registers the stub will manage. */
7029 : else
7030 : {
7031 35225 : unsigned count = xlogue_layout::count_stub_managed_regs ();
7032 35225 : m->call_ms2sysv_extra_regs = count - xlogue_layout::MIN_REGS;
7033 35225 : m->call_ms2sysv_pad_in = 0;
7034 : }
7035 : }
7036 :
7037 8235665 : frame->nregs = ix86_nsaved_regs ();
7038 8235665 : frame->nsseregs = ix86_nsaved_sseregs ();
7039 :
7040 : /* 64-bit MS ABI seem to require stack alignment to be always 16,
7041 : except for function prologues, leaf functions and when the default
7042 : incoming stack boundary is overridden at command line or via
7043 : force_align_arg_pointer attribute.
7044 :
7045 : Darwin's ABI specifies 128b alignment for both 32 and 64 bit variants
7046 : at call sites, including profile function calls.
7047 :
7048 : For APX push2/pop2, the stack also requires 128b alignment. */
7049 8235665 : if ((ix86_pro_and_epilogue_can_use_push2pop2 (frame->nregs)
7050 65 : && crtl->preferred_stack_boundary < 128)
7051 8235728 : || (((TARGET_64BIT_MS_ABI || TARGET_MACHO)
7052 225776 : && crtl->preferred_stack_boundary < 128)
7053 0 : && (!crtl->is_leaf || cfun->calls_alloca != 0
7054 0 : || ix86_current_function_calls_tls_descriptor
7055 0 : || (TARGET_MACHO && crtl->profile)
7056 0 : || ix86_incoming_stack_boundary < 128)))
7057 : {
7058 2 : crtl->preferred_stack_boundary = 128;
7059 2 : if (crtl->stack_alignment_needed < 128)
7060 1 : crtl->stack_alignment_needed = 128;
7061 : }
7062 :
7063 8235665 : stack_alignment_needed = crtl->stack_alignment_needed / BITS_PER_UNIT;
7064 8235665 : preferred_alignment = crtl->preferred_stack_boundary / BITS_PER_UNIT;
7065 :
7066 8235665 : gcc_assert (!size || stack_alignment_needed);
7067 9039140 : gcc_assert (preferred_alignment >= STACK_BOUNDARY / BITS_PER_UNIT);
7068 8235665 : gcc_assert (preferred_alignment <= stack_alignment_needed);
7069 :
7070 : /* The only ABI saving SSE regs should be 64-bit ms_abi or with
7071 : no_caller_saved_registers attribute. */
7072 8235665 : gcc_assert (TARGET_64BIT
7073 : || (cfun->machine->call_saved_registers
7074 : == TYPE_NO_CALLER_SAVED_REGISTERS)
7075 : || !frame->nsseregs);
7076 8235665 : if (TARGET_64BIT && m->call_ms2sysv)
7077 : {
7078 35225 : gcc_assert (stack_alignment_needed >= 16);
7079 35225 : gcc_assert ((cfun->machine->call_saved_registers
7080 : == TYPE_NO_CALLER_SAVED_REGISTERS)
7081 : || !frame->nsseregs);
7082 : }
7083 :
7084 : /* For SEH we have to limit the amount of code movement into the prologue.
7085 : At present we do this via a BLOCKAGE, at which point there's very little
7086 : scheduling that can be done, which means that there's very little point
7087 : in doing anything except PUSHs. */
7088 8235665 : if (TARGET_SEH)
7089 : m->use_fast_prologue_epilogue = false;
7090 8235665 : else if (!optimize_bb_for_size_p (ENTRY_BLOCK_PTR_FOR_FN (cfun)))
7091 : {
7092 7898487 : int count = frame->nregs;
7093 7898487 : struct cgraph_node *node = cgraph_node::get (current_function_decl);
7094 :
7095 : /* The fast prologue uses move instead of push to save registers. This
7096 : is significantly longer, but also executes faster as modern hardware
7097 : can execute the moves in parallel, but can't do that for push/pop.
7098 :
7099 : Be careful about choosing what prologue to emit: When function takes
7100 : many instructions to execute we may use slow version as well as in
7101 : case function is known to be outside hot spot (this is known with
7102 : feedback only). Weight the size of function by number of registers
7103 : to save as it is cheap to use one or two push instructions but very
7104 : slow to use many of them.
7105 :
7106 : Calling this hook multiple times with the same frame requirements
7107 : must produce the same layout, since the RA might otherwise be
7108 : unable to reach a fixed point or might fail its final sanity checks.
7109 : This means that once we've assumed that a function does or doesn't
7110 : have a particular size, we have to stick to that assumption
7111 : regardless of how the function has changed since. */
7112 7898487 : if (count)
7113 2618212 : count = (count - 1) * FAST_PROLOGUE_INSN_COUNT;
7114 7898487 : if (node->frequency < NODE_FREQUENCY_NORMAL
7115 7197866 : || (flag_branch_probabilities
7116 966 : && node->frequency < NODE_FREQUENCY_HOT))
7117 700941 : m->use_fast_prologue_epilogue = false;
7118 : else
7119 : {
7120 7197546 : if (count != frame->expensive_count)
7121 : {
7122 286048 : frame->expensive_count = count;
7123 286048 : frame->expensive_p = expensive_function_p (count);
7124 : }
7125 7197546 : m->use_fast_prologue_epilogue = !frame->expensive_p;
7126 : }
7127 : }
7128 :
7129 8235665 : frame->save_regs_using_mov
7130 8235665 : = TARGET_PROLOGUE_USING_MOVE && m->use_fast_prologue_epilogue;
7131 :
7132 : /* Skip return address and error code in exception handler. */
7133 8235665 : offset = INCOMING_FRAME_SP_OFFSET;
7134 :
7135 : /* Skip pushed static chain. */
7136 8235665 : if (ix86_static_chain_on_stack)
7137 0 : offset += UNITS_PER_WORD;
7138 :
7139 : /* Skip saved base pointer. */
7140 8235665 : if (frame_pointer_needed)
7141 2806881 : offset += UNITS_PER_WORD;
7142 8235665 : frame->hfp_save_offset = offset;
7143 :
7144 : /* The traditional frame pointer location is at the top of the frame. */
7145 8235665 : frame->hard_frame_pointer_offset = offset;
7146 :
7147 : /* Register save area */
7148 8235665 : offset += frame->nregs * UNITS_PER_WORD;
7149 8235665 : frame->reg_save_offset = offset;
7150 :
7151 : /* Calculate the size of the va-arg area (not including padding, if any). */
7152 8235665 : frame->va_arg_size = ix86_varargs_gpr_size + ix86_varargs_fpr_size;
7153 :
7154 : /* Also adjust stack_realign_offset for the largest alignment of
7155 : stack slot actually used. */
7156 8235665 : if (stack_realign_fp
7157 7931667 : || (cfun->machine->max_used_stack_alignment != 0
7158 186 : && (offset % cfun->machine->max_used_stack_alignment) != 0))
7159 : {
7160 : /* We may need a 16-byte aligned stack for the remainder of the
7161 : register save area, but the stack frame for the local function
7162 : may require a greater alignment if using AVX/2/512. In order
7163 : to avoid wasting space, we first calculate the space needed for
7164 : the rest of the register saves, add that to the stack pointer,
7165 : and then realign the stack to the boundary of the start of the
7166 : frame for the local function. */
7167 304083 : HOST_WIDE_INT space_needed = 0;
7168 304083 : HOST_WIDE_INT sse_reg_space_needed = 0;
7169 :
7170 304083 : if (TARGET_64BIT)
7171 : {
7172 302298 : if (m->call_ms2sysv)
7173 : {
7174 6415 : m->call_ms2sysv_pad_in = 0;
7175 6415 : space_needed = xlogue_layout::get_instance ().get_stack_space_used ();
7176 : }
7177 :
7178 295883 : else if (frame->nsseregs)
7179 : /* The only ABI that has saved SSE registers (Win64) also has a
7180 : 16-byte aligned default stack. However, many programs violate
7181 : the ABI, and Wine64 forces stack realignment to compensate. */
7182 6447 : space_needed = frame->nsseregs * 16;
7183 :
7184 302298 : sse_reg_space_needed = space_needed = ROUND_UP (space_needed, 16);
7185 :
7186 : /* 64-bit frame->va_arg_size should always be a multiple of 16, but
7187 : rounding to be pedantic. */
7188 302298 : space_needed = ROUND_UP (space_needed + frame->va_arg_size, 16);
7189 : }
7190 : else
7191 1785 : space_needed = frame->va_arg_size;
7192 :
7193 : /* Record the allocation size required prior to the realignment AND. */
7194 304083 : frame->stack_realign_allocate = space_needed;
7195 :
7196 : /* The re-aligned stack starts at frame->stack_realign_offset. Values
7197 : before this point are not directly comparable with values below
7198 : this point. Use sp_valid_at to determine if the stack pointer is
7199 : valid for a given offset, fp_valid_at for the frame pointer, or
7200 : choose_baseaddr to have a base register chosen for you.
7201 :
7202 : Note that the result of (frame->stack_realign_offset
7203 : & (stack_alignment_needed - 1)) may not equal zero. */
7204 304083 : offset = ROUND_UP (offset + space_needed, stack_alignment_needed);
7205 304083 : frame->stack_realign_offset = offset - space_needed;
7206 304083 : frame->sse_reg_save_offset = frame->stack_realign_offset
7207 304083 : + sse_reg_space_needed;
7208 304083 : }
7209 : else
7210 : {
7211 7931582 : frame->stack_realign_offset = offset;
7212 :
7213 7931582 : if (TARGET_64BIT && m->call_ms2sysv)
7214 : {
7215 28810 : m->call_ms2sysv_pad_in = !!(offset & UNITS_PER_WORD);
7216 28810 : offset += xlogue_layout::get_instance ().get_stack_space_used ();
7217 : }
7218 :
7219 : /* Align and set SSE register save area. */
7220 7902772 : else if (frame->nsseregs)
7221 : {
7222 : /* If the incoming stack boundary is at least 16 bytes, or DRAP is
7223 : required and the DRAP re-alignment boundary is at least 16 bytes,
7224 : then we want the SSE register save area properly aligned. */
7225 183315 : if (ix86_incoming_stack_boundary >= 128
7226 6400 : || (stack_realign_drap && stack_alignment_needed >= 16))
7227 183315 : offset = ROUND_UP (offset, 16);
7228 183315 : offset += frame->nsseregs * 16;
7229 : }
7230 7931582 : frame->sse_reg_save_offset = offset;
7231 7931582 : offset += frame->va_arg_size;
7232 : }
7233 :
7234 : /* Align start of frame for local function. When a function call
7235 : is removed, it may become a leaf function. But if argument may
7236 : be passed on stack, we need to align the stack when there is no
7237 : tail call. */
7238 8235665 : if (m->call_ms2sysv
7239 8200440 : || frame->va_arg_size != 0
7240 8121052 : || size != 0
7241 4414325 : || !crtl->is_leaf
7242 2065773 : || (!crtl->tail_call_emit
7243 1747896 : && cfun->machine->outgoing_args_on_stack)
7244 2065723 : || cfun->calls_alloca
7245 10299679 : || ix86_current_function_calls_tls_descriptor)
7246 6172065 : offset = ROUND_UP (offset, stack_alignment_needed);
7247 :
7248 : /* Frame pointer points here. */
7249 8235665 : frame->frame_pointer_offset = offset;
7250 :
7251 8235665 : offset += size;
7252 :
7253 : /* Add outgoing arguments area. Can be skipped if we eliminated
7254 : all the function calls as dead code.
7255 : Skipping is however impossible when function calls alloca. Alloca
7256 : expander assumes that last crtl->outgoing_args_size
7257 : of stack frame are unused. */
7258 8235665 : if (ACCUMULATE_OUTGOING_ARGS
7259 8854266 : && (!crtl->is_leaf || cfun->calls_alloca
7260 392255 : || ix86_current_function_calls_tls_descriptor))
7261 : {
7262 226346 : offset += crtl->outgoing_args_size;
7263 226346 : frame->outgoing_arguments_size = crtl->outgoing_args_size;
7264 : }
7265 : else
7266 8009319 : frame->outgoing_arguments_size = 0;
7267 :
7268 : /* Align stack boundary. Only needed if we're calling another function
7269 : or using alloca. */
7270 2788026 : if (!crtl->is_leaf || cfun->calls_alloca
7271 11020308 : || ix86_current_function_calls_tls_descriptor)
7272 5452832 : offset = ROUND_UP (offset, preferred_alignment);
7273 :
7274 : /* We've reached end of stack frame. */
7275 8235665 : frame->stack_pointer_offset = offset;
7276 :
7277 : /* Size prologue needs to allocate. */
7278 8235665 : to_allocate = offset - frame->sse_reg_save_offset;
7279 :
7280 8235665 : if (save_regs_using_push_pop (to_allocate))
7281 2594980 : frame->save_regs_using_mov = false;
7282 :
7283 8235665 : if (ix86_using_red_zone ()
7284 7206195 : && crtl->sp_is_unchanging
7285 6560321 : && crtl->is_leaf
7286 2688238 : && !cfun->machine->asm_redzone_clobber_seen
7287 2688225 : && !ix86_pc_thunk_call_expanded
7288 10923890 : && !ix86_current_function_calls_tls_descriptor)
7289 : {
7290 2688210 : frame->red_zone_size = to_allocate;
7291 2688210 : if (frame->save_regs_using_mov)
7292 139941 : frame->red_zone_size += frame->nregs * UNITS_PER_WORD;
7293 2688210 : if (frame->red_zone_size > RED_ZONE_SIZE - RED_ZONE_RESERVE)
7294 102669 : frame->red_zone_size = RED_ZONE_SIZE - RED_ZONE_RESERVE;
7295 : }
7296 : else
7297 5547455 : frame->red_zone_size = 0;
7298 8235665 : frame->stack_pointer_offset -= frame->red_zone_size;
7299 :
7300 : /* The SEH frame pointer location is near the bottom of the frame.
7301 : This is enforced by the fact that the difference between the
7302 : stack pointer and the frame pointer is limited to 240 bytes in
7303 : the unwind data structure. */
7304 8235665 : if (TARGET_SEH)
7305 : {
7306 : /* Force the frame pointer to point at or below the lowest register save
7307 : area, see the SEH code in config/i386/winnt.cc for the rationale. */
7308 : frame->hard_frame_pointer_offset = frame->sse_reg_save_offset;
7309 :
7310 : /* If we can leave the frame pointer where it is, do so; however return
7311 : the establisher frame for __builtin_frame_address (0) or else if the
7312 : frame overflows the SEH maximum frame size.
7313 :
7314 : Note that the value returned by __builtin_frame_address (0) is quite
7315 : constrained, because setjmp is piggybacked on the SEH machinery with
7316 : recent versions of MinGW:
7317 :
7318 : # elif defined(__SEH__)
7319 : # if defined(__aarch64__) || defined(_ARM64_)
7320 : # define setjmp(BUF) _setjmp((BUF), __builtin_sponentry())
7321 : # elif (__MINGW_GCC_VERSION < 40702)
7322 : # define setjmp(BUF) _setjmp((BUF), mingw_getsp())
7323 : # else
7324 : # define setjmp(BUF) _setjmp((BUF), __builtin_frame_address (0))
7325 : # endif
7326 :
7327 : and the second argument passed to _setjmp, if not null, is forwarded
7328 : to the TargetFrame parameter of RtlUnwindEx by longjmp (after it has
7329 : built an ExceptionRecord on the fly describing the setjmp buffer). */
7330 : const HOST_WIDE_INT diff
7331 : = frame->stack_pointer_offset - frame->hard_frame_pointer_offset;
7332 : if (diff <= 255 && !crtl->accesses_prior_frames)
7333 : {
7334 : /* The resulting diff will be a multiple of 16 lower than 255,
7335 : i.e. at most 240 as required by the unwind data structure. */
7336 : frame->hard_frame_pointer_offset += (diff & 15);
7337 : }
7338 : else if (diff <= SEH_MAX_FRAME_SIZE && !crtl->accesses_prior_frames)
7339 : {
7340 : /* Ideally we'd determine what portion of the local stack frame
7341 : (within the constraint of the lowest 240) is most heavily used.
7342 : But without that complication, simply bias the frame pointer
7343 : by 128 bytes so as to maximize the amount of the local stack
7344 : frame that is addressable with 8-bit offsets. */
7345 : frame->hard_frame_pointer_offset = frame->stack_pointer_offset - 128;
7346 : }
7347 : else
7348 : frame->hard_frame_pointer_offset = frame->hfp_save_offset;
7349 : }
7350 8235665 : }
7351 :
7352 : /* This is semi-inlined memory_address_length, but simplified
7353 : since we know that we're always dealing with reg+offset, and
7354 : to avoid having to create and discard all that rtl. */
7355 :
7356 : static inline int
7357 1012933 : choose_baseaddr_len (unsigned int regno, HOST_WIDE_INT offset)
7358 : {
7359 1012933 : int len = 4;
7360 :
7361 0 : if (offset == 0)
7362 : {
7363 : /* EBP and R13 cannot be encoded without an offset. */
7364 0 : len = (regno == BP_REG || regno == R13_REG);
7365 : }
7366 1004845 : else if (IN_RANGE (offset, -128, 127))
7367 631716 : len = 1;
7368 :
7369 : /* ESP and R12 must be encoded with a SIB byte. */
7370 0 : if (regno == SP_REG || regno == R12_REG)
7371 0 : len++;
7372 :
7373 1012933 : return len;
7374 : }
7375 :
7376 : /* Determine if the stack pointer is valid for accessing the CFA_OFFSET in
7377 : the frame save area. The register is saved at CFA - CFA_OFFSET. */
7378 :
7379 : static bool
7380 3510082 : sp_valid_at (HOST_WIDE_INT cfa_offset)
7381 : {
7382 3510082 : const struct machine_frame_state &fs = cfun->machine->fs;
7383 3510082 : if (fs.sp_realigned && cfa_offset <= fs.sp_realigned_offset)
7384 : {
7385 : /* Validate that the cfa_offset isn't in a "no-man's land". */
7386 45724 : gcc_assert (cfa_offset <= fs.sp_realigned_fp_last);
7387 : return false;
7388 : }
7389 3464358 : return fs.sp_valid;
7390 : }
7391 :
7392 : /* Determine if the frame pointer is valid for accessing the CFA_OFFSET in
7393 : the frame save area. The register is saved at CFA - CFA_OFFSET. */
7394 :
7395 : static inline bool
7396 1359781 : fp_valid_at (HOST_WIDE_INT cfa_offset)
7397 : {
7398 1359781 : const struct machine_frame_state &fs = cfun->machine->fs;
7399 1359781 : if (fs.sp_realigned && cfa_offset > fs.sp_realigned_fp_last)
7400 : {
7401 : /* Validate that the cfa_offset isn't in a "no-man's land". */
7402 28328 : gcc_assert (cfa_offset >= fs.sp_realigned_offset);
7403 : return false;
7404 : }
7405 1331453 : return fs.fp_valid;
7406 : }
7407 :
7408 : /* Choose a base register based upon alignment requested, speed and/or
7409 : size. */
7410 :
7411 : static void
7412 1359781 : choose_basereg (HOST_WIDE_INT cfa_offset, rtx &base_reg,
7413 : HOST_WIDE_INT &base_offset,
7414 : unsigned int align_reqested, unsigned int *align)
7415 : {
7416 1359781 : const struct machine_function *m = cfun->machine;
7417 1359781 : unsigned int hfp_align;
7418 1359781 : unsigned int drap_align;
7419 1359781 : unsigned int sp_align;
7420 1359781 : bool hfp_ok = fp_valid_at (cfa_offset);
7421 1359781 : bool drap_ok = m->fs.drap_valid;
7422 1359781 : bool sp_ok = sp_valid_at (cfa_offset);
7423 :
7424 1359781 : hfp_align = drap_align = sp_align = INCOMING_STACK_BOUNDARY;
7425 :
7426 : /* Filter out any registers that don't meet the requested alignment
7427 : criteria. */
7428 1359781 : if (align_reqested)
7429 : {
7430 963312 : if (m->fs.realigned)
7431 28160 : hfp_align = drap_align = sp_align = crtl->stack_alignment_needed;
7432 : /* SEH unwind code does do not currently support REG_CFA_EXPRESSION
7433 : notes (which we would need to use a realigned stack pointer),
7434 : so disable on SEH targets. */
7435 935152 : else if (m->fs.sp_realigned)
7436 28328 : sp_align = crtl->stack_alignment_needed;
7437 :
7438 963312 : hfp_ok = hfp_ok && hfp_align >= align_reqested;
7439 963312 : drap_ok = drap_ok && drap_align >= align_reqested;
7440 963312 : sp_ok = sp_ok && sp_align >= align_reqested;
7441 : }
7442 :
7443 1359781 : if (m->use_fast_prologue_epilogue)
7444 : {
7445 : /* Choose the base register most likely to allow the most scheduling
7446 : opportunities. Generally FP is valid throughout the function,
7447 : while DRAP must be reloaded within the epilogue. But choose either
7448 : over the SP due to increased encoding size. */
7449 :
7450 642177 : if (hfp_ok)
7451 : {
7452 118543 : base_reg = hard_frame_pointer_rtx;
7453 118543 : base_offset = m->fs.fp_offset - cfa_offset;
7454 : }
7455 523634 : else if (drap_ok)
7456 : {
7457 0 : base_reg = crtl->drap_reg;
7458 0 : base_offset = 0 - cfa_offset;
7459 : }
7460 523634 : else if (sp_ok)
7461 : {
7462 523634 : base_reg = stack_pointer_rtx;
7463 523634 : base_offset = m->fs.sp_offset - cfa_offset;
7464 : }
7465 : }
7466 : else
7467 : {
7468 717604 : HOST_WIDE_INT toffset;
7469 717604 : int len = 16, tlen;
7470 :
7471 : /* Choose the base register with the smallest address encoding.
7472 : With a tie, choose FP > DRAP > SP. */
7473 717604 : if (sp_ok)
7474 : {
7475 700284 : base_reg = stack_pointer_rtx;
7476 700284 : base_offset = m->fs.sp_offset - cfa_offset;
7477 1392480 : len = choose_baseaddr_len (STACK_POINTER_REGNUM, base_offset);
7478 : }
7479 717604 : if (drap_ok)
7480 : {
7481 0 : toffset = 0 - cfa_offset;
7482 0 : tlen = choose_baseaddr_len (REGNO (crtl->drap_reg), toffset);
7483 0 : if (tlen <= len)
7484 : {
7485 0 : base_reg = crtl->drap_reg;
7486 0 : base_offset = toffset;
7487 0 : len = tlen;
7488 : }
7489 : }
7490 717604 : if (hfp_ok)
7491 : {
7492 312649 : toffset = m->fs.fp_offset - cfa_offset;
7493 312649 : tlen = choose_baseaddr_len (HARD_FRAME_POINTER_REGNUM, toffset);
7494 312649 : if (tlen <= len)
7495 : {
7496 225189 : base_reg = hard_frame_pointer_rtx;
7497 225189 : base_offset = toffset;
7498 : }
7499 : }
7500 : }
7501 :
7502 : /* Set the align return value. */
7503 1359781 : if (align)
7504 : {
7505 963312 : if (base_reg == stack_pointer_rtx)
7506 678380 : *align = sp_align;
7507 284932 : else if (base_reg == crtl->drap_reg)
7508 0 : *align = drap_align;
7509 284932 : else if (base_reg == hard_frame_pointer_rtx)
7510 284932 : *align = hfp_align;
7511 : }
7512 1359781 : }
7513 :
7514 : /* Return an RTX that points to CFA_OFFSET within the stack frame and
7515 : the alignment of address. If ALIGN is non-null, it should point to
7516 : an alignment value (in bits) that is preferred or zero and will
7517 : receive the alignment of the base register that was selected,
7518 : irrespective of rather or not CFA_OFFSET is a multiple of that
7519 : alignment value. If it is possible for the base register offset to be
7520 : non-immediate then SCRATCH_REGNO should specify a scratch register to
7521 : use.
7522 :
7523 : The valid base registers are taken from CFUN->MACHINE->FS. */
7524 :
7525 : static rtx
7526 1359781 : choose_baseaddr (HOST_WIDE_INT cfa_offset, unsigned int *align,
7527 : unsigned int scratch_regno = INVALID_REGNUM)
7528 : {
7529 1359781 : rtx base_reg = NULL;
7530 1359781 : HOST_WIDE_INT base_offset = 0;
7531 :
7532 : /* If a specific alignment is requested, try to get a base register
7533 : with that alignment first. */
7534 1359781 : if (align && *align)
7535 963312 : choose_basereg (cfa_offset, base_reg, base_offset, *align, align);
7536 :
7537 1359781 : if (!base_reg)
7538 396469 : choose_basereg (cfa_offset, base_reg, base_offset, 0, align);
7539 :
7540 1359781 : gcc_assert (base_reg != NULL);
7541 :
7542 1359781 : rtx base_offset_rtx = GEN_INT (base_offset);
7543 :
7544 1410810 : if (!x86_64_immediate_operand (base_offset_rtx, Pmode))
7545 : {
7546 1 : gcc_assert (scratch_regno != INVALID_REGNUM);
7547 :
7548 1 : rtx scratch_reg = gen_rtx_REG (Pmode, scratch_regno);
7549 1 : emit_move_insn (scratch_reg, base_offset_rtx);
7550 :
7551 1 : return gen_rtx_PLUS (Pmode, base_reg, scratch_reg);
7552 : }
7553 :
7554 1410809 : return plus_constant (Pmode, base_reg, base_offset);
7555 : }
7556 :
7557 : /* Emit code to save registers in the prologue. */
7558 :
7559 : static void
7560 429838 : ix86_emit_save_regs (void)
7561 : {
7562 429838 : int regno;
7563 429838 : rtx_insn *insn;
7564 429838 : bool use_ppx = TARGET_APX_PPX && !crtl->calls_eh_return;
7565 :
7566 429838 : if (!TARGET_APX_PUSH2POP2
7567 91 : || !ix86_can_use_push2pop2 ()
7568 429927 : || cfun->machine->func_type != TYPE_NORMAL)
7569 : {
7570 39966750 : for (regno = FIRST_PSEUDO_REGISTER - 1; regno >= 0; regno--)
7571 39537000 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true, true))
7572 : {
7573 1203723 : insn = emit_insn (gen_push (gen_rtx_REG (word_mode, regno),
7574 : use_ppx));
7575 1203723 : RTX_FRAME_RELATED_P (insn) = 1;
7576 : }
7577 : }
7578 : else
7579 : {
7580 88 : int regno_list[2];
7581 88 : regno_list[0] = regno_list[1] = -1;
7582 88 : int loaded_regnum = 0;
7583 88 : bool aligned = cfun->machine->fs.sp_offset % 16 == 0;
7584 :
7585 8184 : for (regno = FIRST_PSEUDO_REGISTER - 1; regno >= 0; regno--)
7586 8096 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true, true))
7587 : {
7588 128 : if (aligned)
7589 : {
7590 45 : regno_list[loaded_regnum++] = regno;
7591 45 : if (loaded_regnum == 2)
7592 : {
7593 19 : gcc_assert (regno_list[0] != -1
7594 : && regno_list[1] != -1
7595 : && regno_list[0] != regno_list[1]);
7596 19 : const int offset = UNITS_PER_WORD * 2;
7597 19 : rtx mem = gen_rtx_MEM (TImode,
7598 19 : gen_rtx_PRE_DEC (Pmode,
7599 : stack_pointer_rtx));
7600 19 : insn = emit_insn (gen_push2 (mem,
7601 : gen_rtx_REG (word_mode,
7602 : regno_list[0]),
7603 : gen_rtx_REG (word_mode,
7604 : regno_list[1]),
7605 : use_ppx));
7606 19 : RTX_FRAME_RELATED_P (insn) = 1;
7607 19 : rtx dwarf = gen_rtx_SEQUENCE (VOIDmode, rtvec_alloc (3));
7608 :
7609 57 : for (int i = 0; i < 2; i++)
7610 : {
7611 76 : rtx dwarf_reg = gen_rtx_REG (word_mode,
7612 38 : regno_list[i]);
7613 38 : rtx sp_offset = plus_constant (Pmode,
7614 : stack_pointer_rtx,
7615 38 : + UNITS_PER_WORD
7616 38 : * (1 - i));
7617 38 : rtx tmp = gen_rtx_SET (gen_frame_mem (DImode,
7618 : sp_offset),
7619 : dwarf_reg);
7620 38 : RTX_FRAME_RELATED_P (tmp) = 1;
7621 38 : XVECEXP (dwarf, 0, i + 1) = tmp;
7622 : }
7623 19 : rtx sp_tmp = gen_rtx_SET (stack_pointer_rtx,
7624 : plus_constant (Pmode,
7625 : stack_pointer_rtx,
7626 : -offset));
7627 19 : RTX_FRAME_RELATED_P (sp_tmp) = 1;
7628 19 : XVECEXP (dwarf, 0, 0) = sp_tmp;
7629 19 : add_reg_note (insn, REG_FRAME_RELATED_EXPR, dwarf);
7630 :
7631 19 : loaded_regnum = 0;
7632 19 : regno_list[0] = regno_list[1] = -1;
7633 : }
7634 : }
7635 : else
7636 : {
7637 83 : insn = emit_insn (gen_push (gen_rtx_REG (word_mode, regno),
7638 : use_ppx));
7639 83 : RTX_FRAME_RELATED_P (insn) = 1;
7640 83 : aligned = true;
7641 : }
7642 : }
7643 88 : if (loaded_regnum == 1)
7644 : {
7645 7 : insn = emit_insn (gen_push (gen_rtx_REG (word_mode,
7646 7 : regno_list[0]),
7647 : use_ppx));
7648 7 : RTX_FRAME_RELATED_P (insn) = 1;
7649 : }
7650 : }
7651 429838 : }
7652 :
7653 : /* Emit a single register save at CFA - CFA_OFFSET. */
7654 :
7655 : static void
7656 609117 : ix86_emit_save_reg_using_mov (machine_mode mode, unsigned int regno,
7657 : HOST_WIDE_INT cfa_offset)
7658 : {
7659 609117 : struct machine_function *m = cfun->machine;
7660 609117 : rtx reg = gen_rtx_REG (mode, regno);
7661 609117 : rtx mem, addr, base, insn;
7662 609117 : unsigned int align = GET_MODE_ALIGNMENT (mode);
7663 :
7664 609117 : addr = choose_baseaddr (cfa_offset, &align);
7665 609117 : mem = gen_frame_mem (mode, addr);
7666 :
7667 : /* The location alignment depends upon the base register. */
7668 609117 : align = MIN (GET_MODE_ALIGNMENT (mode), align);
7669 609117 : gcc_assert (! (cfa_offset & (align / BITS_PER_UNIT - 1)));
7670 609117 : set_mem_align (mem, align);
7671 :
7672 609117 : insn = emit_insn (gen_rtx_SET (mem, reg));
7673 609117 : RTX_FRAME_RELATED_P (insn) = 1;
7674 :
7675 609117 : base = addr;
7676 609117 : if (GET_CODE (base) == PLUS)
7677 597424 : base = XEXP (base, 0);
7678 609117 : gcc_checking_assert (REG_P (base));
7679 :
7680 : /* When saving registers into a re-aligned local stack frame, avoid
7681 : any tricky guessing by dwarf2out. */
7682 609117 : if (m->fs.realigned)
7683 : {
7684 12800 : gcc_checking_assert (stack_realign_drap);
7685 :
7686 12800 : if (regno == REGNO (crtl->drap_reg))
7687 : {
7688 : /* A bit of a hack. We force the DRAP register to be saved in
7689 : the re-aligned stack frame, which provides us with a copy
7690 : of the CFA that will last past the prologue. Install it. */
7691 0 : gcc_checking_assert (cfun->machine->fs.fp_valid);
7692 0 : addr = plus_constant (Pmode, hard_frame_pointer_rtx,
7693 0 : cfun->machine->fs.fp_offset - cfa_offset);
7694 0 : mem = gen_rtx_MEM (mode, addr);
7695 0 : add_reg_note (insn, REG_CFA_DEF_CFA, mem);
7696 : }
7697 : else
7698 : {
7699 : /* The frame pointer is a stable reference within the
7700 : aligned frame. Use it. */
7701 12800 : gcc_checking_assert (cfun->machine->fs.fp_valid);
7702 12800 : addr = plus_constant (Pmode, hard_frame_pointer_rtx,
7703 12800 : cfun->machine->fs.fp_offset - cfa_offset);
7704 12800 : mem = gen_rtx_MEM (mode, addr);
7705 12800 : add_reg_note (insn, REG_CFA_EXPRESSION, gen_rtx_SET (mem, reg));
7706 : }
7707 : }
7708 :
7709 596317 : else if (base == stack_pointer_rtx && m->fs.sp_realigned
7710 12881 : && cfa_offset >= m->fs.sp_realigned_offset)
7711 : {
7712 12881 : gcc_checking_assert (stack_realign_fp);
7713 12881 : add_reg_note (insn, REG_CFA_EXPRESSION, gen_rtx_SET (mem, reg));
7714 : }
7715 :
7716 : /* The memory may not be relative to the current CFA register,
7717 : which means that we may need to generate a new pattern for
7718 : use by the unwind info. */
7719 583436 : else if (base != m->fs.cfa_reg)
7720 : {
7721 43473 : addr = plus_constant (Pmode, m->fs.cfa_reg,
7722 43473 : m->fs.cfa_offset - cfa_offset);
7723 43473 : mem = gen_rtx_MEM (mode, addr);
7724 43473 : add_reg_note (insn, REG_CFA_OFFSET, gen_rtx_SET (mem, reg));
7725 : }
7726 609117 : }
7727 :
7728 : /* Emit code to save registers using MOV insns.
7729 : First register is stored at CFA - CFA_OFFSET. */
7730 : static void
7731 44306 : ix86_emit_save_regs_using_mov (HOST_WIDE_INT cfa_offset)
7732 : {
7733 44306 : unsigned int regno;
7734 :
7735 4120458 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
7736 4076152 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true, true))
7737 : {
7738 : /* Skip registers, already processed by shrink wrap separate. */
7739 187986 : if (!cfun->machine->reg_is_wrapped_separately[regno])
7740 82234 : ix86_emit_save_reg_using_mov (word_mode, regno, cfa_offset);
7741 202698 : cfa_offset -= UNITS_PER_WORD;
7742 : }
7743 44306 : }
7744 :
7745 : /* Emit code to save SSE registers using MOV insns.
7746 : First register is stored at CFA - CFA_OFFSET. */
7747 : static void
7748 33376 : ix86_emit_save_sse_regs_using_mov (HOST_WIDE_INT cfa_offset)
7749 : {
7750 33376 : unsigned int regno;
7751 :
7752 3103968 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
7753 3070592 : if (SSE_REGNO_P (regno) && ix86_save_reg (regno, true, true))
7754 : {
7755 333769 : ix86_emit_save_reg_using_mov (V4SFmode, regno, cfa_offset);
7756 333769 : cfa_offset -= GET_MODE_SIZE (V4SFmode);
7757 : }
7758 33376 : }
7759 :
7760 : static GTY(()) rtx queued_cfa_restores;
7761 :
7762 : /* Add a REG_CFA_RESTORE REG note to INSN or queue them until next stack
7763 : manipulation insn. The value is on the stack at CFA - CFA_OFFSET.
7764 : Don't add the note if the previously saved value will be left untouched
7765 : within stack red-zone till return, as unwinders can find the same value
7766 : in the register and on the stack. */
7767 :
7768 : static void
7769 2297500 : ix86_add_cfa_restore_note (rtx_insn *insn, rtx reg, HOST_WIDE_INT cfa_offset)
7770 : {
7771 2297500 : if (!crtl->shrink_wrapped
7772 2278718 : && cfa_offset <= cfun->machine->fs.red_zone_offset)
7773 : return;
7774 :
7775 770146 : if (insn)
7776 : {
7777 359583 : add_reg_note (insn, REG_CFA_RESTORE, reg);
7778 359583 : RTX_FRAME_RELATED_P (insn) = 1;
7779 : }
7780 : else
7781 410563 : queued_cfa_restores
7782 410563 : = alloc_reg_note (REG_CFA_RESTORE, reg, queued_cfa_restores);
7783 : }
7784 :
7785 : /* Add queued REG_CFA_RESTORE notes if any to INSN. */
7786 :
7787 : static void
7788 2569796 : ix86_add_queued_cfa_restore_notes (rtx insn)
7789 : {
7790 2569796 : rtx last;
7791 2569796 : if (!queued_cfa_restores)
7792 : return;
7793 410563 : for (last = queued_cfa_restores; XEXP (last, 1); last = XEXP (last, 1))
7794 : ;
7795 52951 : XEXP (last, 1) = REG_NOTES (insn);
7796 52951 : REG_NOTES (insn) = queued_cfa_restores;
7797 52951 : queued_cfa_restores = NULL_RTX;
7798 52951 : RTX_FRAME_RELATED_P (insn) = 1;
7799 : }
7800 :
7801 : /* Expand prologue or epilogue stack adjustment.
7802 : The pattern exist to put a dependency on all ebp-based memory accesses.
7803 : STYLE should be negative if instructions should be marked as frame related,
7804 : zero if %r11 register is live and cannot be freely used and positive
7805 : otherwise. */
7806 :
7807 : static rtx
7808 1593813 : pro_epilogue_adjust_stack (rtx dest, rtx src, rtx offset,
7809 : int style, bool set_cfa)
7810 : {
7811 1593813 : struct machine_function *m = cfun->machine;
7812 1593813 : rtx addend = offset;
7813 1593813 : rtx insn;
7814 1593813 : bool add_frame_related_expr = false;
7815 :
7816 1812534 : if (!x86_64_immediate_operand (offset, Pmode))
7817 : {
7818 : /* r11 is used by indirect sibcall return as well, set before the
7819 : epilogue and used after the epilogue. */
7820 199 : if (style)
7821 174 : addend = gen_rtx_REG (Pmode, R11_REG);
7822 : else
7823 : {
7824 25 : gcc_assert (src != hard_frame_pointer_rtx
7825 : && dest != hard_frame_pointer_rtx);
7826 : addend = hard_frame_pointer_rtx;
7827 : }
7828 199 : emit_insn (gen_rtx_SET (addend, offset));
7829 199 : if (style < 0)
7830 88 : add_frame_related_expr = true;
7831 : }
7832 :
7833 : /* Shrink wrap separate may insert prologue between TEST and JMP. In order
7834 : not to affect EFlags, emit add without reg clobbering. */
7835 1593813 : if (crtl->shrink_wrapped_separate)
7836 93623 : insn = emit_insn (gen_pro_epilogue_adjust_stack_add_nocc
7837 93623 : (Pmode, dest, src, addend));
7838 : else
7839 1500190 : insn = emit_insn (gen_pro_epilogue_adjust_stack_add
7840 1500190 : (Pmode, dest, src, addend));
7841 :
7842 1593813 : if (style >= 0)
7843 701880 : ix86_add_queued_cfa_restore_notes (insn);
7844 :
7845 1593813 : if (set_cfa)
7846 : {
7847 1229603 : rtx r;
7848 :
7849 1229603 : gcc_assert (m->fs.cfa_reg == src);
7850 1229603 : m->fs.cfa_offset += INTVAL (offset);
7851 1229603 : m->fs.cfa_reg = dest;
7852 :
7853 1425929 : r = gen_rtx_PLUS (Pmode, src, offset);
7854 1229603 : r = gen_rtx_SET (dest, r);
7855 1229603 : add_reg_note (insn, REG_CFA_ADJUST_CFA, r);
7856 1229603 : RTX_FRAME_RELATED_P (insn) = 1;
7857 : }
7858 364210 : else if (style < 0)
7859 : {
7860 298140 : RTX_FRAME_RELATED_P (insn) = 1;
7861 298140 : if (add_frame_related_expr)
7862 : {
7863 20 : rtx r = gen_rtx_PLUS (Pmode, src, offset);
7864 20 : r = gen_rtx_SET (dest, r);
7865 20 : add_reg_note (insn, REG_FRAME_RELATED_EXPR, r);
7866 : }
7867 : }
7868 :
7869 1593813 : if (dest == stack_pointer_rtx)
7870 : {
7871 1593813 : HOST_WIDE_INT ooffset = m->fs.sp_offset;
7872 1593813 : bool valid = m->fs.sp_valid;
7873 1593813 : bool realigned = m->fs.sp_realigned;
7874 :
7875 1593813 : if (src == hard_frame_pointer_rtx)
7876 : {
7877 29572 : valid = m->fs.fp_valid;
7878 29572 : realigned = false;
7879 29572 : ooffset = m->fs.fp_offset;
7880 : }
7881 1564241 : else if (src == crtl->drap_reg)
7882 : {
7883 0 : valid = m->fs.drap_valid;
7884 0 : realigned = false;
7885 0 : ooffset = 0;
7886 : }
7887 : else
7888 : {
7889 : /* Else there are two possibilities: SP itself, which we set
7890 : up as the default above. Or EH_RETURN_STACKADJ_RTX, which is
7891 : taken care of this by hand along the eh_return path. */
7892 1564241 : gcc_checking_assert (src == stack_pointer_rtx
7893 : || offset == const0_rtx);
7894 : }
7895 :
7896 1593813 : m->fs.sp_offset = ooffset - INTVAL (offset);
7897 1593813 : m->fs.sp_valid = valid;
7898 1593813 : m->fs.sp_realigned = realigned;
7899 : }
7900 1593813 : return insn;
7901 : }
7902 :
7903 : /* Find an available register to be used as dynamic realign argument
7904 : pointer register. Such a register will be written in prologue and
7905 : used in begin of body, so it must not be
7906 : 1. parameter passing register.
7907 : 2. GOT pointer.
7908 : We reuse static-chain register if it is available. Otherwise, we
7909 : use DI for i386 and R13 for x86-64. We chose R13 since it has
7910 : shorter encoding.
7911 :
7912 : Return: the regno of chosen register. */
7913 :
7914 : static unsigned int
7915 7240 : find_drap_reg (void)
7916 : {
7917 7240 : tree decl = cfun->decl;
7918 :
7919 : /* Always use callee-saved register if there are no caller-saved
7920 : registers. */
7921 7240 : if (TARGET_64BIT)
7922 : {
7923 : /* In preserve_none functions, any register can be used for DRAP,
7924 : except AX, R12–R15, DI, SI (argument registers), SP, and BP. */
7925 6955 : if (cfun->machine->call_saved_registers == TYPE_PRESERVE_NONE)
7926 : return R11_REG;
7927 :
7928 : /* Use R13 for nested function or function need static chain.
7929 : Since function with tail call may use any caller-saved
7930 : registers in epilogue, DRAP must not use caller-saved
7931 : register in such case. */
7932 6954 : if (DECL_STATIC_CHAIN (decl)
7933 6912 : || (cfun->machine->call_saved_registers
7934 : == TYPE_NO_CALLER_SAVED_REGISTERS)
7935 13866 : || crtl->tail_call_emit)
7936 191 : return R13_REG;
7937 :
7938 : return R10_REG;
7939 : }
7940 : else
7941 : {
7942 : /* Use DI for nested function or function need static chain.
7943 : Since function with tail call may use any caller-saved
7944 : registers in epilogue, DRAP must not use caller-saved
7945 : register in such case. */
7946 285 : if (DECL_STATIC_CHAIN (decl)
7947 285 : || (cfun->machine->call_saved_registers
7948 285 : == TYPE_NO_CALLER_SAVED_REGISTERS)
7949 285 : || crtl->tail_call_emit
7950 550 : || crtl->calls_eh_return)
7951 : return DI_REG;
7952 :
7953 : /* Reuse static chain register if it isn't used for parameter
7954 : passing. */
7955 265 : if (ix86_function_regparm (TREE_TYPE (decl), decl) <= 2)
7956 : {
7957 265 : unsigned int ccvt = ix86_get_callcvt (TREE_TYPE (decl));
7958 265 : if ((ccvt & (IX86_CALLCVT_FASTCALL | IX86_CALLCVT_THISCALL)) == 0)
7959 : return CX_REG;
7960 : }
7961 0 : return DI_REG;
7962 : }
7963 : }
7964 :
7965 : /* Return minimum incoming stack alignment. */
7966 :
7967 : static unsigned int
7968 1627677 : ix86_minimum_incoming_stack_boundary (bool sibcall)
7969 : {
7970 1627677 : unsigned int incoming_stack_boundary;
7971 :
7972 : /* Stack of interrupt handler is aligned to 128 bits in 64bit mode. */
7973 1627677 : if (cfun->machine->func_type != TYPE_NORMAL)
7974 120 : incoming_stack_boundary = TARGET_64BIT ? 128 : MIN_STACK_BOUNDARY;
7975 : /* Prefer the one specified at command line. */
7976 1627557 : else if (ix86_user_incoming_stack_boundary)
7977 : incoming_stack_boundary = ix86_user_incoming_stack_boundary;
7978 : /* In 32bit, use MIN_STACK_BOUNDARY for incoming stack boundary
7979 : if -mstackrealign is used, it isn't used for sibcall check and
7980 : estimated stack alignment is 128bit. */
7981 1627535 : else if (!sibcall
7982 1494578 : && ix86_force_align_arg_pointer
7983 4572 : && crtl->stack_alignment_estimated == 128)
7984 596 : incoming_stack_boundary = MIN_STACK_BOUNDARY;
7985 : else
7986 1626939 : incoming_stack_boundary = ix86_default_incoming_stack_boundary;
7987 :
7988 : /* Incoming stack alignment can be changed on individual functions
7989 : via force_align_arg_pointer attribute. We use the smallest
7990 : incoming stack boundary. */
7991 1627677 : if (incoming_stack_boundary > MIN_STACK_BOUNDARY
7992 3254748 : && lookup_attribute ("force_align_arg_pointer",
7993 1627071 : TYPE_ATTRIBUTES (TREE_TYPE (current_function_decl))))
7994 5708 : incoming_stack_boundary = MIN_STACK_BOUNDARY;
7995 :
7996 : /* The incoming stack frame has to be aligned at least at
7997 : parm_stack_boundary. */
7998 1627677 : if (incoming_stack_boundary < crtl->parm_stack_boundary)
7999 : incoming_stack_boundary = crtl->parm_stack_boundary;
8000 :
8001 : /* Stack at entrance of main is aligned by runtime. We use the
8002 : smallest incoming stack boundary. */
8003 1627677 : if (incoming_stack_boundary > MAIN_STACK_BOUNDARY
8004 140955 : && DECL_NAME (current_function_decl)
8005 140955 : && MAIN_NAME_P (DECL_NAME (current_function_decl))
8006 1630153 : && DECL_FILE_SCOPE_P (current_function_decl))
8007 2476 : incoming_stack_boundary = MAIN_STACK_BOUNDARY;
8008 :
8009 1627677 : return incoming_stack_boundary;
8010 : }
8011 :
8012 : /* Update incoming stack boundary and estimated stack alignment. */
8013 :
8014 : static void
8015 1494715 : ix86_update_stack_boundary (void)
8016 : {
8017 1494715 : ix86_incoming_stack_boundary
8018 1494715 : = ix86_minimum_incoming_stack_boundary (false);
8019 :
8020 : /* x86_64 vararg needs 16byte stack alignment for register save area. */
8021 1494715 : if (TARGET_64BIT
8022 1368024 : && cfun->stdarg
8023 21433 : && crtl->stack_alignment_estimated < 128)
8024 10255 : crtl->stack_alignment_estimated = 128;
8025 :
8026 : /* __tls_get_addr needs to be called with 16-byte aligned stack. */
8027 1494715 : if (ix86_tls_descriptor_calls_expanded_in_cfun
8028 1085 : && crtl->preferred_stack_boundary < 128)
8029 750 : crtl->preferred_stack_boundary = 128;
8030 :
8031 : /* For 32-bit MS ABI, both the incoming and preferred stack boundaries
8032 : are 32 bits, but if force_align_arg_pointer is specified, it should
8033 : prefer 128 bits for a backward-compatibility reason, which is also
8034 : what the doc suggests. */
8035 1494715 : if (lookup_attribute ("force_align_arg_pointer",
8036 1494715 : TYPE_ATTRIBUTES (TREE_TYPE (current_function_decl)))
8037 1494715 : && crtl->preferred_stack_boundary < 128)
8038 4 : crtl->preferred_stack_boundary = 128;
8039 1494715 : }
8040 :
8041 : /* Handle the TARGET_GET_DRAP_RTX hook. Return NULL if no DRAP is
8042 : needed or an rtx for DRAP otherwise. */
8043 :
8044 : static rtx
8045 1601002 : ix86_get_drap_rtx (void)
8046 : {
8047 : /* We must use DRAP if there are outgoing arguments on stack or
8048 : the stack pointer register is clobbered by asm statement and
8049 : ACCUMULATE_OUTGOING_ARGS is false. */
8050 1601002 : if (ix86_force_drap
8051 1601002 : || ((cfun->machine->outgoing_args_on_stack
8052 1267561 : || crtl->sp_is_clobbered_by_asm)
8053 331496 : && !ACCUMULATE_OUTGOING_ARGS))
8054 311286 : crtl->need_drap = true;
8055 :
8056 1601002 : if (stack_realign_drap)
8057 : {
8058 : /* Assign DRAP to vDRAP and returns vDRAP */
8059 7240 : unsigned int regno = find_drap_reg ();
8060 7240 : rtx drap_vreg;
8061 7240 : rtx arg_ptr;
8062 7240 : rtx_insn *seq, *insn;
8063 :
8064 7525 : arg_ptr = gen_rtx_REG (Pmode, regno);
8065 7240 : crtl->drap_reg = arg_ptr;
8066 :
8067 7240 : start_sequence ();
8068 7240 : drap_vreg = copy_to_reg (arg_ptr);
8069 7240 : seq = end_sequence ();
8070 :
8071 7240 : insn = emit_insn_before (seq, NEXT_INSN (entry_of_function ()));
8072 7240 : if (!optimize)
8073 : {
8074 1897 : add_reg_note (insn, REG_CFA_SET_VDRAP, drap_vreg);
8075 1897 : RTX_FRAME_RELATED_P (insn) = 1;
8076 : }
8077 7240 : return drap_vreg;
8078 : }
8079 : else
8080 : return NULL;
8081 : }
8082 :
8083 : /* Handle the TARGET_INTERNAL_ARG_POINTER hook. */
8084 :
8085 : static rtx
8086 1494717 : ix86_internal_arg_pointer (void)
8087 : {
8088 1494717 : return virtual_incoming_args_rtx;
8089 : }
8090 :
8091 : struct scratch_reg {
8092 : rtx reg;
8093 : bool saved;
8094 : };
8095 :
8096 : /* Return a short-lived scratch register for use on function entry.
8097 : In 32-bit mode, it is valid only after the registers are saved
8098 : in the prologue. This register must be released by means of
8099 : release_scratch_register_on_entry once it is dead. */
8100 :
8101 : static void
8102 25 : get_scratch_register_on_entry (struct scratch_reg *sr)
8103 : {
8104 25 : int regno;
8105 :
8106 25 : sr->saved = false;
8107 :
8108 25 : if (TARGET_64BIT)
8109 : {
8110 : /* We always use R11 in 64-bit mode. */
8111 : regno = R11_REG;
8112 : }
8113 : else
8114 : {
8115 0 : tree decl = current_function_decl, fntype = TREE_TYPE (decl);
8116 0 : bool fastcall_p
8117 0 : = lookup_attribute ("fastcall", TYPE_ATTRIBUTES (fntype)) != NULL_TREE;
8118 0 : bool thiscall_p
8119 0 : = lookup_attribute ("thiscall", TYPE_ATTRIBUTES (fntype)) != NULL_TREE;
8120 0 : bool static_chain_p = DECL_STATIC_CHAIN (decl);
8121 0 : int regparm = ix86_function_regparm (fntype, decl);
8122 0 : int drap_regno
8123 0 : = crtl->drap_reg ? REGNO (crtl->drap_reg) : INVALID_REGNUM;
8124 :
8125 : /* 'fastcall' sets regparm to 2, uses ecx/edx for arguments and eax
8126 : for the static chain register. */
8127 0 : if ((regparm < 1 || (fastcall_p && !static_chain_p))
8128 0 : && drap_regno != AX_REG)
8129 : regno = AX_REG;
8130 : /* 'thiscall' sets regparm to 1, uses ecx for arguments and edx
8131 : for the static chain register. */
8132 0 : else if (thiscall_p && !static_chain_p && drap_regno != AX_REG)
8133 : regno = AX_REG;
8134 0 : else if (regparm < 2 && !thiscall_p && drap_regno != DX_REG)
8135 : regno = DX_REG;
8136 : /* ecx is the static chain register. */
8137 0 : else if (regparm < 3 && !fastcall_p && !thiscall_p
8138 0 : && !static_chain_p
8139 0 : && drap_regno != CX_REG)
8140 : regno = CX_REG;
8141 0 : else if (ix86_save_reg (BX_REG, true, false))
8142 : regno = BX_REG;
8143 : /* esi is the static chain register. */
8144 0 : else if (!(regparm == 3 && static_chain_p)
8145 0 : && ix86_save_reg (SI_REG, true, false))
8146 : regno = SI_REG;
8147 0 : else if (ix86_save_reg (DI_REG, true, false))
8148 : regno = DI_REG;
8149 : else
8150 : {
8151 0 : regno = (drap_regno == AX_REG ? DX_REG : AX_REG);
8152 0 : sr->saved = true;
8153 : }
8154 : }
8155 :
8156 25 : sr->reg = gen_rtx_REG (Pmode, regno);
8157 25 : if (sr->saved)
8158 : {
8159 0 : rtx_insn *insn = emit_insn (gen_push (sr->reg));
8160 0 : RTX_FRAME_RELATED_P (insn) = 1;
8161 : }
8162 25 : }
8163 :
8164 : /* Release a scratch register obtained from the preceding function.
8165 :
8166 : If RELEASE_VIA_POP is true, we just pop the register off the stack
8167 : to release it. This is what non-Linux systems use with -fstack-check.
8168 :
8169 : Otherwise we use OFFSET to locate the saved register and the
8170 : allocated stack space becomes part of the local frame and is
8171 : deallocated by the epilogue. */
8172 :
8173 : static void
8174 25 : release_scratch_register_on_entry (struct scratch_reg *sr, HOST_WIDE_INT offset,
8175 : bool release_via_pop)
8176 : {
8177 25 : if (sr->saved)
8178 : {
8179 0 : if (release_via_pop)
8180 : {
8181 0 : struct machine_function *m = cfun->machine;
8182 0 : rtx x, insn = emit_insn (gen_pop (sr->reg));
8183 :
8184 : /* The RX FRAME_RELATED_P mechanism doesn't know about pop. */
8185 0 : RTX_FRAME_RELATED_P (insn) = 1;
8186 0 : x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
8187 0 : x = gen_rtx_SET (stack_pointer_rtx, x);
8188 0 : add_reg_note (insn, REG_FRAME_RELATED_EXPR, x);
8189 0 : m->fs.sp_offset -= UNITS_PER_WORD;
8190 : }
8191 : else
8192 : {
8193 0 : rtx x = plus_constant (Pmode, stack_pointer_rtx, offset);
8194 0 : x = gen_rtx_SET (sr->reg, gen_rtx_MEM (word_mode, x));
8195 0 : emit_insn (x);
8196 : }
8197 : }
8198 25 : }
8199 :
8200 : /* Emit code to adjust the stack pointer by SIZE bytes while probing it.
8201 :
8202 : If INT_REGISTERS_SAVED is true, then integer registers have already been
8203 : pushed on the stack.
8204 :
8205 : If PROTECTION AREA is true, then probe PROBE_INTERVAL plus a small dope
8206 : beyond SIZE bytes.
8207 :
8208 : This assumes no knowledge of the current probing state, i.e. it is never
8209 : allowed to allocate more than PROBE_INTERVAL bytes of stack space without
8210 : a suitable probe. */
8211 :
8212 : static void
8213 126 : ix86_adjust_stack_and_probe (HOST_WIDE_INT size,
8214 : const bool int_registers_saved,
8215 : const bool protection_area)
8216 : {
8217 126 : struct machine_function *m = cfun->machine;
8218 :
8219 : /* If this function does not statically allocate stack space, then
8220 : no probes are needed. */
8221 126 : if (!size)
8222 : {
8223 : /* However, the allocation of space via pushes for register
8224 : saves could be viewed as allocating space, but without the
8225 : need to probe. */
8226 43 : if (m->frame.nregs || m->frame.nsseregs || frame_pointer_needed)
8227 23 : dump_stack_clash_frame_info (NO_PROBE_SMALL_FRAME, true);
8228 : else
8229 20 : dump_stack_clash_frame_info (NO_PROBE_NO_FRAME, false);
8230 43 : return;
8231 : }
8232 :
8233 : /* If we are a noreturn function, then we have to consider the
8234 : possibility that we're called via a jump rather than a call.
8235 :
8236 : Thus we don't have the implicit probe generated by saving the
8237 : return address into the stack at the call. Thus, the stack
8238 : pointer could be anywhere in the guard page. The safe thing
8239 : to do is emit a probe now.
8240 :
8241 : The probe can be avoided if we have already emitted any callee
8242 : register saves into the stack or have a frame pointer (which will
8243 : have been saved as well). Those saves will function as implicit
8244 : probes.
8245 :
8246 : ?!? This should be revamped to work like aarch64 and s390 where
8247 : we track the offset from the most recent probe. Normally that
8248 : offset would be zero. For a noreturn function we would reset
8249 : it to PROBE_INTERVAL - (STACK_BOUNDARY / BITS_PER_UNIT). Then
8250 : we just probe when we cross PROBE_INTERVAL. */
8251 83 : if (TREE_THIS_VOLATILE (cfun->decl)
8252 15 : && !(m->frame.nregs || m->frame.nsseregs || frame_pointer_needed))
8253 : {
8254 : /* We can safely use any register here since we're just going to push
8255 : its value and immediately pop it back. But we do try and avoid
8256 : argument passing registers so as not to introduce dependencies in
8257 : the pipeline. For 32 bit we use %esi and for 64 bit we use %rax. */
8258 15 : rtx dummy_reg = gen_rtx_REG (word_mode, TARGET_64BIT ? AX_REG : SI_REG);
8259 15 : rtx_insn *insn_push = emit_insn (gen_push (dummy_reg));
8260 15 : rtx_insn *insn_pop = emit_insn (gen_pop (dummy_reg));
8261 15 : m->fs.sp_offset -= UNITS_PER_WORD;
8262 15 : if (m->fs.cfa_reg == stack_pointer_rtx)
8263 : {
8264 15 : m->fs.cfa_offset -= UNITS_PER_WORD;
8265 15 : rtx x = plus_constant (Pmode, stack_pointer_rtx, -UNITS_PER_WORD);
8266 15 : x = gen_rtx_SET (stack_pointer_rtx, x);
8267 15 : add_reg_note (insn_push, REG_CFA_ADJUST_CFA, x);
8268 15 : RTX_FRAME_RELATED_P (insn_push) = 1;
8269 15 : x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
8270 15 : x = gen_rtx_SET (stack_pointer_rtx, x);
8271 15 : add_reg_note (insn_pop, REG_CFA_ADJUST_CFA, x);
8272 15 : RTX_FRAME_RELATED_P (insn_pop) = 1;
8273 : }
8274 15 : emit_insn (gen_blockage ());
8275 : }
8276 :
8277 83 : const HOST_WIDE_INT probe_interval = get_probe_interval ();
8278 83 : const int dope = 4 * UNITS_PER_WORD;
8279 :
8280 : /* If there is protection area, take it into account in the size. */
8281 83 : if (protection_area)
8282 24 : size += probe_interval + dope;
8283 :
8284 : /* If we allocate less than the size of the guard statically,
8285 : then no probing is necessary, but we do need to allocate
8286 : the stack. */
8287 59 : else if (size < (1 << param_stack_clash_protection_guard_size))
8288 : {
8289 38 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
8290 : GEN_INT (-size), -1,
8291 38 : m->fs.cfa_reg == stack_pointer_rtx);
8292 38 : dump_stack_clash_frame_info (NO_PROBE_SMALL_FRAME, true);
8293 38 : return;
8294 : }
8295 :
8296 : /* We're allocating a large enough stack frame that we need to
8297 : emit probes. Either emit them inline or in a loop depending
8298 : on the size. */
8299 45 : if (size <= 4 * probe_interval)
8300 : {
8301 : HOST_WIDE_INT i;
8302 47 : for (i = probe_interval; i <= size; i += probe_interval)
8303 : {
8304 : /* Allocate PROBE_INTERVAL bytes. */
8305 27 : rtx insn
8306 27 : = pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
8307 : GEN_INT (-probe_interval), -1,
8308 27 : m->fs.cfa_reg == stack_pointer_rtx);
8309 27 : add_reg_note (insn, REG_STACK_CHECK, const0_rtx);
8310 :
8311 : /* And probe at *sp. */
8312 27 : emit_stack_probe (stack_pointer_rtx);
8313 27 : emit_insn (gen_blockage ());
8314 : }
8315 :
8316 : /* We need to allocate space for the residual, but we do not need
8317 : to probe the residual... */
8318 20 : HOST_WIDE_INT residual = (i - probe_interval - size);
8319 20 : if (residual)
8320 : {
8321 20 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
8322 : GEN_INT (residual), -1,
8323 20 : m->fs.cfa_reg == stack_pointer_rtx);
8324 :
8325 : /* ...except if there is a protection area to maintain. */
8326 20 : if (protection_area)
8327 11 : emit_stack_probe (stack_pointer_rtx);
8328 : }
8329 :
8330 20 : dump_stack_clash_frame_info (PROBE_INLINE, residual != 0);
8331 : }
8332 : else
8333 : {
8334 : /* We expect the GP registers to be saved when probes are used
8335 : as the probing sequences might need a scratch register and
8336 : the routine to allocate one assumes the integer registers
8337 : have already been saved. */
8338 25 : gcc_assert (int_registers_saved);
8339 :
8340 25 : struct scratch_reg sr;
8341 25 : get_scratch_register_on_entry (&sr);
8342 :
8343 : /* If we needed to save a register, then account for any space
8344 : that was pushed (we are not going to pop the register when
8345 : we do the restore). */
8346 25 : if (sr.saved)
8347 0 : size -= UNITS_PER_WORD;
8348 :
8349 : /* Step 1: round SIZE down to a multiple of the interval. */
8350 25 : HOST_WIDE_INT rounded_size = size & -probe_interval;
8351 :
8352 : /* Step 2: compute final value of the loop counter. Use lea if
8353 : possible. */
8354 25 : rtx addr = plus_constant (Pmode, stack_pointer_rtx, -rounded_size);
8355 25 : rtx insn;
8356 25 : if (address_no_seg_operand (addr, Pmode))
8357 13 : insn = emit_insn (gen_rtx_SET (sr.reg, addr));
8358 : else
8359 : {
8360 12 : emit_move_insn (sr.reg, GEN_INT (-rounded_size));
8361 12 : insn = emit_insn (gen_rtx_SET (sr.reg,
8362 : gen_rtx_PLUS (Pmode, sr.reg,
8363 : stack_pointer_rtx)));
8364 : }
8365 25 : if (m->fs.cfa_reg == stack_pointer_rtx)
8366 : {
8367 22 : add_reg_note (insn, REG_CFA_DEF_CFA,
8368 22 : plus_constant (Pmode, sr.reg,
8369 22 : m->fs.cfa_offset + rounded_size));
8370 22 : RTX_FRAME_RELATED_P (insn) = 1;
8371 : }
8372 :
8373 : /* Step 3: the loop. */
8374 25 : rtx size_rtx = GEN_INT (rounded_size);
8375 25 : insn = emit_insn (gen_adjust_stack_and_probe (Pmode, sr.reg, sr.reg,
8376 : size_rtx));
8377 25 : if (m->fs.cfa_reg == stack_pointer_rtx)
8378 : {
8379 22 : m->fs.cfa_offset += rounded_size;
8380 22 : add_reg_note (insn, REG_CFA_DEF_CFA,
8381 22 : plus_constant (Pmode, stack_pointer_rtx,
8382 22 : m->fs.cfa_offset));
8383 22 : RTX_FRAME_RELATED_P (insn) = 1;
8384 : }
8385 25 : m->fs.sp_offset += rounded_size;
8386 25 : emit_insn (gen_blockage ());
8387 :
8388 : /* Step 4: adjust SP if we cannot assert at compile-time that SIZE
8389 : is equal to ROUNDED_SIZE. */
8390 :
8391 25 : if (size != rounded_size)
8392 : {
8393 25 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
8394 : GEN_INT (rounded_size - size), -1,
8395 25 : m->fs.cfa_reg == stack_pointer_rtx);
8396 :
8397 25 : if (protection_area)
8398 13 : emit_stack_probe (stack_pointer_rtx);
8399 : }
8400 :
8401 25 : dump_stack_clash_frame_info (PROBE_LOOP, size != rounded_size);
8402 :
8403 : /* This does not deallocate the space reserved for the scratch
8404 : register. That will be deallocated in the epilogue. */
8405 25 : release_scratch_register_on_entry (&sr, size, false);
8406 : }
8407 :
8408 : /* Adjust back to account for the protection area. */
8409 45 : if (protection_area)
8410 24 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
8411 24 : GEN_INT (probe_interval + dope), -1,
8412 24 : m->fs.cfa_reg == stack_pointer_rtx);
8413 :
8414 : /* Make sure nothing is scheduled before we are done. */
8415 45 : emit_insn (gen_blockage ());
8416 : }
8417 :
8418 : /* Adjust the stack pointer up to REG while probing it. */
8419 :
8420 : const char *
8421 25 : output_adjust_stack_and_probe (rtx reg)
8422 : {
8423 25 : static int labelno = 0;
8424 25 : char loop_lab[32];
8425 25 : rtx xops[2];
8426 :
8427 25 : ASM_GENERATE_INTERNAL_LABEL (loop_lab, "LPSRL", labelno++);
8428 :
8429 : /* Loop. */
8430 25 : ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, loop_lab);
8431 :
8432 : /* SP = SP + PROBE_INTERVAL. */
8433 25 : xops[0] = stack_pointer_rtx;
8434 37 : xops[1] = GEN_INT (get_probe_interval ());
8435 25 : output_asm_insn ("sub%z0\t{%1, %0|%0, %1}", xops);
8436 :
8437 : /* Probe at SP. */
8438 25 : xops[1] = const0_rtx;
8439 25 : output_asm_insn ("or{b}\t{%1, (%0)|BYTE PTR [%0], %1}", xops);
8440 :
8441 : /* Test if SP == LAST_ADDR. */
8442 25 : xops[0] = stack_pointer_rtx;
8443 25 : xops[1] = reg;
8444 25 : output_asm_insn ("cmp%z0\t{%1, %0|%0, %1}", xops);
8445 :
8446 : /* Branch. */
8447 25 : fputs ("\tjne\t", asm_out_file);
8448 25 : assemble_name_raw (asm_out_file, loop_lab);
8449 25 : fputc ('\n', asm_out_file);
8450 :
8451 25 : return "";
8452 : }
8453 :
8454 : /* Emit code to probe a range of stack addresses from FIRST to FIRST+SIZE,
8455 : inclusive. These are offsets from the current stack pointer.
8456 :
8457 : INT_REGISTERS_SAVED is true if integer registers have already been
8458 : pushed on the stack. */
8459 :
8460 : static void
8461 0 : ix86_emit_probe_stack_range (HOST_WIDE_INT first, HOST_WIDE_INT size,
8462 : const bool int_registers_saved)
8463 : {
8464 0 : const HOST_WIDE_INT probe_interval = get_probe_interval ();
8465 :
8466 : /* See if we have a constant small number of probes to generate. If so,
8467 : that's the easy case. The run-time loop is made up of 6 insns in the
8468 : generic case while the compile-time loop is made up of n insns for n #
8469 : of intervals. */
8470 0 : if (size <= 6 * probe_interval)
8471 : {
8472 : HOST_WIDE_INT i;
8473 :
8474 : /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 1 until
8475 : it exceeds SIZE. If only one probe is needed, this will not
8476 : generate any code. Then probe at FIRST + SIZE. */
8477 0 : for (i = probe_interval; i < size; i += probe_interval)
8478 0 : emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx,
8479 0 : -(first + i)));
8480 :
8481 0 : emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx,
8482 0 : -(first + size)));
8483 : }
8484 :
8485 : /* Otherwise, do the same as above, but in a loop. Note that we must be
8486 : extra careful with variables wrapping around because we might be at
8487 : the very top (or the very bottom) of the address space and we have
8488 : to be able to handle this case properly; in particular, we use an
8489 : equality test for the loop condition. */
8490 : else
8491 : {
8492 : /* We expect the GP registers to be saved when probes are used
8493 : as the probing sequences might need a scratch register and
8494 : the routine to allocate one assumes the integer registers
8495 : have already been saved. */
8496 0 : gcc_assert (int_registers_saved);
8497 :
8498 0 : HOST_WIDE_INT rounded_size, last;
8499 0 : struct scratch_reg sr;
8500 :
8501 0 : get_scratch_register_on_entry (&sr);
8502 :
8503 :
8504 : /* Step 1: round SIZE to the previous multiple of the interval. */
8505 :
8506 0 : rounded_size = ROUND_DOWN (size, probe_interval);
8507 :
8508 :
8509 : /* Step 2: compute initial and final value of the loop counter. */
8510 :
8511 : /* TEST_OFFSET = FIRST. */
8512 0 : emit_move_insn (sr.reg, GEN_INT (-first));
8513 :
8514 : /* LAST_OFFSET = FIRST + ROUNDED_SIZE. */
8515 0 : last = first + rounded_size;
8516 :
8517 :
8518 : /* Step 3: the loop
8519 :
8520 : do
8521 : {
8522 : TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
8523 : probe at TEST_ADDR
8524 : }
8525 : while (TEST_ADDR != LAST_ADDR)
8526 :
8527 : probes at FIRST + N * PROBE_INTERVAL for values of N from 1
8528 : until it is equal to ROUNDED_SIZE. */
8529 :
8530 0 : emit_insn
8531 0 : (gen_probe_stack_range (Pmode, sr.reg, sr.reg, GEN_INT (-last)));
8532 :
8533 :
8534 : /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
8535 : that SIZE is equal to ROUNDED_SIZE. */
8536 :
8537 0 : if (size != rounded_size)
8538 0 : emit_stack_probe (plus_constant (Pmode,
8539 0 : gen_rtx_PLUS (Pmode,
8540 : stack_pointer_rtx,
8541 : sr.reg),
8542 0 : rounded_size - size));
8543 :
8544 0 : release_scratch_register_on_entry (&sr, size, true);
8545 : }
8546 :
8547 : /* Make sure nothing is scheduled before we are done. */
8548 0 : emit_insn (gen_blockage ());
8549 0 : }
8550 :
8551 : /* Probe a range of stack addresses from REG to END, inclusive. These are
8552 : offsets from the current stack pointer. */
8553 :
8554 : const char *
8555 0 : output_probe_stack_range (rtx reg, rtx end)
8556 : {
8557 0 : static int labelno = 0;
8558 0 : char loop_lab[32];
8559 0 : rtx xops[3];
8560 :
8561 0 : ASM_GENERATE_INTERNAL_LABEL (loop_lab, "LPSRL", labelno++);
8562 :
8563 : /* Loop. */
8564 0 : ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, loop_lab);
8565 :
8566 : /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL. */
8567 0 : xops[0] = reg;
8568 0 : xops[1] = GEN_INT (get_probe_interval ());
8569 0 : output_asm_insn ("sub%z0\t{%1, %0|%0, %1}", xops);
8570 :
8571 : /* Probe at TEST_ADDR. */
8572 0 : xops[0] = stack_pointer_rtx;
8573 0 : xops[1] = reg;
8574 0 : xops[2] = const0_rtx;
8575 0 : output_asm_insn ("or{b}\t{%2, (%0,%1)|BYTE PTR [%0+%1], %2}", xops);
8576 :
8577 : /* Test if TEST_ADDR == LAST_ADDR. */
8578 0 : xops[0] = reg;
8579 0 : xops[1] = end;
8580 0 : output_asm_insn ("cmp%z0\t{%1, %0|%0, %1}", xops);
8581 :
8582 : /* Branch. */
8583 0 : fputs ("\tjne\t", asm_out_file);
8584 0 : assemble_name_raw (asm_out_file, loop_lab);
8585 0 : fputc ('\n', asm_out_file);
8586 :
8587 0 : return "";
8588 : }
8589 :
8590 : /* Data passed to ix86_update_stack_alignment. */
8591 : struct stack_access_data
8592 : {
8593 : /* The stack access register. */
8594 : const_rtx reg;
8595 : /* Pointer to stack alignment. */
8596 : unsigned int *stack_alignment;
8597 : };
8598 :
8599 : /* SET stores into OP, a MEM linked to parameter BASE. Return true if
8600 : SET stores BASE's argument register into OP. This is a spill: the
8601 : callee saves its own register argument to the stack. It is not a
8602 : stack argument set up by the caller:
8603 :
8604 : (set (mem/c:V2DI (plus:DI (reg/f:DI 7 sp)
8605 : (const_int -16 [0xfffffffffffffff0])) [4 a1+0 S16 A128])
8606 : (reg:V2DI 20 xmm0 [ a1 ]))
8607 : */
8608 :
8609 : static bool
8610 84993 : ix86_spill_register_argument_p (const_rtx set, const_rtx op, tree base)
8611 : {
8612 84993 : rtx src = SET_SRC (set);
8613 :
8614 : /* Not a hard register store, so not a spill. */
8615 84993 : if (!REG_P (src) || !HARD_REGISTER_P (src))
8616 : return false;
8617 :
8618 24946 : rtx dest = SET_DEST (set);
8619 24946 : tree reg_expr = REG_EXPR (src);
8620 :
8621 : /* If spilling an SSA_NAME into OP, the argument-linked memory is
8622 : also used to store a local variable. */
8623 24946 : return dest == op && (reg_expr == base
8624 2933 : || (reg_expr
8625 154 : && TREE_CODE (reg_expr) == SSA_NAME));
8626 : }
8627 :
8628 : /* Return true if OP, found in PAT, is a stack argument set up by the
8629 : caller. Return false if OP is a register argument that the callee
8630 : spilled to its own stack frame. Both cases share the same MEM_EXPR,
8631 : so we must check PAT to tell them apart. */
8632 :
8633 : static bool
8634 112965 : ix86_argument_passed_on_stack_p (const_rtx op, const_rtx pat)
8635 : {
8636 112965 : tree mem_expr = MEM_EXPR (op);
8637 112965 : if (!mem_expr)
8638 : return false;
8639 :
8640 111083 : tree var = get_base_address (mem_expr);
8641 111083 : if (TREE_CODE (var) != PARM_DECL)
8642 : return false;
8643 :
8644 : /* PAT is always a single SET here: note_stores splits PARALLEL
8645 : patterns into separate SETs before calling this function. */
8646 84993 : if (GET_CODE (pat) == SET)
8647 84993 : return !ix86_spill_register_argument_p (pat, op, var);
8648 :
8649 : return true;
8650 : }
8651 :
8652 : /* Update the maximum stack slot alignment from memory alignment in PAT. */
8653 :
8654 : static void
8655 146947 : ix86_update_stack_alignment (rtx, const_rtx pat, void *data)
8656 : {
8657 : /* This insn may reference stack slot. Update the maximum stack slot
8658 : alignment if the memory is referenced by the stack access register. */
8659 146947 : stack_access_data *p = (stack_access_data *) data;
8660 :
8661 146947 : subrtx_iterator::array_type array;
8662 574907 : FOR_EACH_SUBRTX (iter, array, pat, ALL)
8663 : {
8664 478008 : auto op = *iter;
8665 478008 : if (MEM_P (op))
8666 : {
8667 : /* NB: Ignore arguments passed on stack since caller is
8668 : responsible to align the outgoing stack for arguments
8669 : passed on stack. */
8670 143313 : if (reg_mentioned_p (p->reg, XEXP (op, 0))
8671 143313 : && !ix86_argument_passed_on_stack_p (op, pat))
8672 : {
8673 50048 : unsigned int alignment = MEM_ALIGN (op);
8674 :
8675 50048 : if (alignment > *p->stack_alignment)
8676 49962 : *p->stack_alignment = alignment;
8677 : break;
8678 : }
8679 : else
8680 93265 : iter.skip_subrtxes ();
8681 : }
8682 : }
8683 146947 : }
8684 :
8685 : /* Helper function for ix86_find_all_reg_uses. */
8686 :
8687 : static void
8688 45780731 : ix86_find_all_reg_uses_1 (HARD_REG_SET ®set,
8689 : rtx set, unsigned int regno,
8690 : auto_bitmap &worklist)
8691 : {
8692 45780731 : rtx dest = SET_DEST (set);
8693 :
8694 45780731 : if (!REG_P (dest))
8695 41444210 : return;
8696 :
8697 : /* Reject non-Pmode modes. */
8698 34646667 : if (GET_MODE (dest) != Pmode)
8699 : return;
8700 :
8701 18451857 : unsigned int dst_regno = REGNO (dest);
8702 :
8703 18451857 : if (TEST_HARD_REG_BIT (regset, dst_regno))
8704 : return;
8705 :
8706 4336521 : const_rtx src = SET_SRC (set);
8707 :
8708 4336521 : subrtx_iterator::array_type array;
8709 8626912 : FOR_EACH_SUBRTX (iter, array, src, ALL)
8710 : {
8711 5590054 : auto op = *iter;
8712 :
8713 5590054 : if (MEM_P (op))
8714 3013913 : iter.skip_subrtxes ();
8715 :
8716 5590054 : if (REG_P (op) && REGNO (op) == regno)
8717 : {
8718 : /* Add this register to register set. */
8719 1468294 : add_to_hard_reg_set (®set, Pmode, dst_regno);
8720 1299663 : bitmap_set_bit (worklist, dst_regno);
8721 1299663 : break;
8722 : }
8723 : }
8724 4336521 : }
8725 :
8726 : /* Find all registers defined with register REGNO. */
8727 :
8728 : static void
8729 2327615 : ix86_find_all_reg_uses (HARD_REG_SET ®set,
8730 : unsigned int regno, auto_bitmap &worklist)
8731 : {
8732 2327615 : for (df_ref ref = DF_REG_USE_CHAIN (regno);
8733 82523658 : ref != NULL;
8734 80196043 : ref = DF_REF_NEXT_REG (ref))
8735 : {
8736 80196043 : if (DF_REF_IS_ARTIFICIAL (ref))
8737 16711927 : continue;
8738 :
8739 63484116 : rtx_insn *insn = DF_REF_INSN (ref);
8740 :
8741 63484116 : if (!NONJUMP_INSN_P (insn))
8742 18404247 : continue;
8743 :
8744 45079869 : unsigned int ref_regno = DF_REF_REGNO (ref);
8745 :
8746 45079869 : rtx set = single_set (insn);
8747 45079869 : if (set)
8748 : {
8749 44271752 : ix86_find_all_reg_uses_1 (regset, set,
8750 : ref_regno, worklist);
8751 44271752 : continue;
8752 : }
8753 :
8754 808117 : rtx pat = PATTERN (insn);
8755 808117 : if (GET_CODE (pat) != PARALLEL)
8756 125731 : continue;
8757 :
8758 2663279 : for (int i = 0; i < XVECLEN (pat, 0); i++)
8759 : {
8760 1980893 : rtx exp = XVECEXP (pat, 0, i);
8761 :
8762 1980893 : if (GET_CODE (exp) == SET)
8763 1508979 : ix86_find_all_reg_uses_1 (regset, exp,
8764 : ref_regno, worklist);
8765 : }
8766 : }
8767 2327615 : }
8768 :
8769 : /* Return true if the hard register REGNO used for a stack access is
8770 : defined in a basic block that dominates the block where it is used. */
8771 :
8772 : static bool
8773 40071 : ix86_access_stack_p (unsigned int regno, basic_block bb,
8774 : HARD_REG_SET &set_up_by_prologue,
8775 : HARD_REG_SET &prologue_used,
8776 : auto_bitmap reg_dominate_bbs_known[],
8777 : auto_bitmap reg_dominate_bbs[])
8778 : {
8779 40071 : if (bitmap_bit_p (reg_dominate_bbs_known[regno], bb->index))
8780 11042 : return bitmap_bit_p (reg_dominate_bbs[regno], bb->index);
8781 :
8782 29029 : bitmap_set_bit (reg_dominate_bbs_known[regno], bb->index);
8783 :
8784 : /* Get all BBs which set REGNO and dominate the current BB from all
8785 : DEFs of REGNO. */
8786 29029 : for (df_ref def = DF_REG_DEF_CHAIN (regno);
8787 1524253 : def;
8788 1495224 : def = DF_REF_NEXT_REG (def))
8789 1522789 : if (!DF_REF_IS_ARTIFICIAL (def)
8790 1521086 : && !DF_REF_FLAGS_IS_SET (def, DF_REF_MAY_CLOBBER)
8791 1495890 : && !DF_REF_FLAGS_IS_SET (def, DF_REF_MUST_CLOBBER))
8792 : {
8793 1494076 : basic_block set_bb = DF_REF_BB (def);
8794 1494076 : if (dominated_by_p (CDI_DOMINATORS, bb, set_bb))
8795 : {
8796 85228 : rtx_insn *insn = DF_REF_INSN (def);
8797 : /* Return true if INSN requires stack. */
8798 85228 : if (requires_stack_frame_p (insn, prologue_used,
8799 : set_up_by_prologue))
8800 : {
8801 27565 : bitmap_set_bit (reg_dominate_bbs[regno], bb->index);
8802 27565 : return true;
8803 : }
8804 : }
8805 : }
8806 :
8807 : /* When we get here, REGNO used in the current BB doesn't access
8808 : stack. */
8809 : return false;
8810 : }
8811 :
8812 : /* Return true if OP isn't a memory operand with SYMBOLIC_CONST and
8813 : needs alignment > ALIGNMENT. */
8814 :
8815 : static bool
8816 27857365 : ix86_need_alignment_p_2 (const_rtx op, unsigned int alignment)
8817 : {
8818 27857365 : bool need_alignment = MEM_ALIGN (op) > alignment;
8819 27857365 : tree mem_expr = MEM_EXPR (op);
8820 27857365 : if (!mem_expr)
8821 : return need_alignment;
8822 :
8823 22965399 : tree var = get_base_address (mem_expr);
8824 22965399 : if (!VAR_P (var) || !DECL_RTL_SET_P (var))
8825 : return need_alignment;
8826 :
8827 14541184 : rtx x = DECL_RTL (var);
8828 14541184 : if (!MEM_P (x))
8829 : return need_alignment;
8830 :
8831 14541181 : x = XEXP (x, 0);
8832 14541181 : return !SYMBOLIC_CONST (x) && need_alignment;
8833 : }
8834 :
8835 : /* Return true if SET needs alignment > ALIGNMENT. */
8836 :
8837 : static bool
8838 45780000 : ix86_need_alignment_p_1 (rtx set, unsigned int alignment)
8839 : {
8840 45780000 : rtx dest = SET_DEST (set);
8841 :
8842 45780000 : if (MEM_P (dest))
8843 17285336 : return ix86_need_alignment_p_2 (dest, alignment);
8844 :
8845 28494664 : const_rtx src = SET_SRC (set);
8846 :
8847 28494664 : subrtx_iterator::array_type array;
8848 82702412 : FOR_EACH_SUBRTX (iter, array, src, ALL)
8849 : {
8850 64779777 : auto op = *iter;
8851 :
8852 64779777 : if (MEM_P (op))
8853 10572029 : return ix86_need_alignment_p_2 (op, alignment);
8854 : }
8855 :
8856 17922635 : return false;
8857 28494664 : }
8858 :
8859 : /* Return true if INSN needs alignment > ALIGNMENT. */
8860 :
8861 : static bool
8862 45079869 : ix86_need_alignment_p (rtx_insn *insn, unsigned int alignment)
8863 : {
8864 45079869 : rtx set = single_set (insn);
8865 45079869 : if (set)
8866 44271752 : return ix86_need_alignment_p_1 (set, alignment);
8867 :
8868 808117 : rtx pat = PATTERN (insn);
8869 808117 : if (GET_CODE (pat) != PARALLEL)
8870 : return false;
8871 :
8872 2661340 : for (int i = 0; i < XVECLEN (pat, 0); i++)
8873 : {
8874 1979834 : rtx exp = XVECEXP (pat, 0, i);
8875 :
8876 1979834 : if (GET_CODE (exp) == SET
8877 1979834 : && ix86_need_alignment_p_1 (exp, alignment))
8878 : return true;
8879 : }
8880 :
8881 : return false;
8882 : }
8883 :
8884 : /* Set stack_frame_required to false if stack frame isn't required.
8885 : Update STACK_ALIGNMENT to the largest alignment, in bits, of stack
8886 : slot used if stack frame is required and CHECK_STACK_SLOT is true. */
8887 :
8888 : static void
8889 1493865 : ix86_find_max_used_stack_alignment (unsigned int &stack_alignment,
8890 : bool check_stack_slot)
8891 : {
8892 1493865 : HARD_REG_SET set_up_by_prologue, prologue_used;
8893 1493865 : basic_block bb;
8894 :
8895 5975460 : CLEAR_HARD_REG_SET (prologue_used);
8896 1493865 : CLEAR_HARD_REG_SET (set_up_by_prologue);
8897 1620663 : add_to_hard_reg_set (&set_up_by_prologue, Pmode, STACK_POINTER_REGNUM);
8898 1493865 : add_to_hard_reg_set (&set_up_by_prologue, Pmode, ARG_POINTER_REGNUM);
8899 1493865 : add_to_hard_reg_set (&set_up_by_prologue, Pmode,
8900 : HARD_FRAME_POINTER_REGNUM);
8901 :
8902 1493865 : bool require_stack_frame = false;
8903 :
8904 15974175 : FOR_EACH_BB_FN (bb, cfun)
8905 : {
8906 14480310 : rtx_insn *insn;
8907 91160398 : FOR_BB_INSNS (bb, insn)
8908 84618629 : if (NONDEBUG_INSN_P (insn)
8909 84618629 : && requires_stack_frame_p (insn, prologue_used,
8910 : set_up_by_prologue))
8911 : {
8912 : require_stack_frame = true;
8913 : break;
8914 : }
8915 : }
8916 :
8917 1493865 : cfun->machine->stack_frame_required = require_stack_frame;
8918 :
8919 : /* Stop if we don't need to check stack slot. */
8920 1493865 : if (!check_stack_slot)
8921 794644 : return;
8922 :
8923 : /* The preferred stack alignment is the minimum stack alignment. */
8924 699221 : if (stack_alignment > crtl->preferred_stack_boundary)
8925 142703 : stack_alignment = crtl->preferred_stack_boundary;
8926 :
8927 : HARD_REG_SET stack_slot_access;
8928 699221 : CLEAR_HARD_REG_SET (stack_slot_access);
8929 :
8930 : /* Stack slot can be accessed by stack pointer, frame pointer or
8931 : registers defined by stack pointer or frame pointer. */
8932 699221 : auto_bitmap worklist;
8933 :
8934 758906 : add_to_hard_reg_set (&stack_slot_access, Pmode, STACK_POINTER_REGNUM);
8935 699221 : bitmap_set_bit (worklist, STACK_POINTER_REGNUM);
8936 :
8937 699221 : if (frame_pointer_needed)
8938 : {
8939 337780 : add_to_hard_reg_set (&stack_slot_access, Pmode,
8940 : HARD_FRAME_POINTER_REGNUM);
8941 328731 : bitmap_set_bit (worklist, HARD_FRAME_POINTER_REGNUM);
8942 : }
8943 :
8944 : /* Registers on HARD_STACK_SLOT_ACCESS always access stack. */
8945 699221 : HARD_REG_SET hard_stack_slot_access = stack_slot_access;
8946 :
8947 699221 : calculate_dominance_info (CDI_DOMINATORS);
8948 :
8949 2327615 : unsigned int regno;
8950 :
8951 2327615 : do
8952 : {
8953 2327615 : regno = bitmap_clear_first_set_bit (worklist);
8954 2327615 : ix86_find_all_reg_uses (stack_slot_access, regno, worklist);
8955 : }
8956 2327615 : while (!bitmap_empty_p (worklist));
8957 :
8958 : hard_reg_set_iterator hrsi;
8959 : stack_access_data data;
8960 :
8961 129355885 : auto_bitmap reg_dominate_bbs_known[FIRST_PSEUDO_REGISTER];
8962 129355885 : auto_bitmap reg_dominate_bbs[FIRST_PSEUDO_REGISTER];
8963 :
8964 699221 : data.stack_alignment = &stack_alignment;
8965 :
8966 3026836 : EXECUTE_IF_SET_IN_HARD_REG_SET (stack_slot_access, 0, regno, hrsi)
8967 : {
8968 2327615 : for (df_ref ref = DF_REG_USE_CHAIN (regno);
8969 82523658 : ref != NULL;
8970 80196043 : ref = DF_REF_NEXT_REG (ref))
8971 : {
8972 80196043 : if (DF_REF_IS_ARTIFICIAL (ref))
8973 16711927 : continue;
8974 :
8975 63484116 : rtx_insn *insn = DF_REF_INSN (ref);
8976 :
8977 63484116 : if (!NONJUMP_INSN_P (insn))
8978 18404247 : continue;
8979 :
8980 : /* Call ix86_access_stack_p only if INSN needs alignment >
8981 : STACK_ALIGNMENT. */
8982 45079869 : if (ix86_need_alignment_p (insn, stack_alignment)
8983 45079869 : && (TEST_HARD_REG_BIT (hard_stack_slot_access, regno)
8984 40071 : || ix86_access_stack_p (regno, BLOCK_FOR_INSN (insn),
8985 : set_up_by_prologue,
8986 : prologue_used,
8987 : reg_dominate_bbs_known,
8988 : reg_dominate_bbs)))
8989 : {
8990 : /* Update stack alignment if REGNO is used for stack
8991 : access. */
8992 140077 : data.reg = DF_REF_REG (ref);
8993 140077 : note_stores (insn, ix86_update_stack_alignment, &data);
8994 : }
8995 : }
8996 : }
8997 :
8998 699221 : free_dominance_info (CDI_DOMINATORS);
8999 130754327 : }
9000 :
9001 : /* Finalize stack_realign_needed and frame_pointer_needed flags, which
9002 : will guide prologue/epilogue to be generated in correct form. */
9003 :
9004 : static void
9005 3460226 : ix86_finalize_stack_frame_flags (void)
9006 : {
9007 : /* Check if stack realign is really needed after reload, and
9008 : stores result in cfun */
9009 3460226 : unsigned int incoming_stack_boundary
9010 3460226 : = (crtl->parm_stack_boundary > ix86_incoming_stack_boundary
9011 3460226 : ? crtl->parm_stack_boundary : ix86_incoming_stack_boundary);
9012 3460226 : unsigned int stack_alignment
9013 1189934 : = (crtl->is_leaf && !ix86_current_function_calls_tls_descriptor
9014 4650160 : ? crtl->max_used_stack_slot_alignment
9015 3460226 : : crtl->stack_alignment_needed);
9016 3460226 : unsigned int stack_realign
9017 3460226 : = (incoming_stack_boundary < stack_alignment);
9018 3460226 : bool recompute_frame_layout_p = false;
9019 :
9020 3460226 : if (crtl->stack_realign_finalized)
9021 : {
9022 : /* After stack_realign_needed is finalized, we can't no longer
9023 : change it. */
9024 1966361 : gcc_assert (crtl->stack_realign_needed == stack_realign);
9025 1966361 : return;
9026 : }
9027 :
9028 : /* It is always safe to compute max_used_stack_alignment. We
9029 : compute it only if 128-bit aligned load/store may be generated
9030 : on misaligned stack slot which will lead to segfault. */
9031 2987730 : bool check_stack_slot
9032 1493865 : = (stack_realign || crtl->max_used_stack_slot_alignment >= 128);
9033 1493865 : ix86_find_max_used_stack_alignment (stack_alignment,
9034 : check_stack_slot);
9035 :
9036 : /* If the only reason for frame_pointer_needed is that we conservatively
9037 : assumed stack realignment might be needed or -fno-omit-frame-pointer
9038 : is used, but in the end nothing that needed the stack alignment had
9039 : been spilled nor stack access, clear frame_pointer_needed and say we
9040 : don't need stack realignment.
9041 :
9042 : When vector register is used for piecewise move and store, we don't
9043 : increase stack_alignment_needed as there is no register spill for
9044 : piecewise move and store. Since stack_realign_needed is set to true
9045 : by checking stack_alignment_estimated which is updated by pseudo
9046 : vector register usage, we also need to check stack_realign_needed to
9047 : eliminate frame pointer. */
9048 1493865 : if ((stack_realign
9049 1428096 : || (!flag_omit_frame_pointer && optimize)
9050 1417846 : || crtl->stack_realign_needed)
9051 76744 : && frame_pointer_needed
9052 76744 : && crtl->is_leaf
9053 52563 : && crtl->sp_is_unchanging
9054 52514 : && !ix86_current_function_calls_tls_descriptor
9055 52514 : && !crtl->accesses_prior_frames
9056 52514 : && !cfun->calls_alloca
9057 52514 : && !crtl->calls_eh_return
9058 : /* See ira_setup_eliminable_regset for the rationale. */
9059 52514 : && !(STACK_CHECK_MOVING_SP
9060 52514 : && flag_stack_check
9061 0 : && flag_exceptions
9062 0 : && cfun->can_throw_non_call_exceptions)
9063 52514 : && !ix86_frame_pointer_required ()
9064 52513 : && ix86_get_frame_size () == 0
9065 34988 : && ix86_nsaved_sseregs () == 0
9066 1528853 : && ix86_varargs_gpr_size + ix86_varargs_fpr_size == 0)
9067 : {
9068 34988 : if (cfun->machine->stack_frame_required)
9069 : {
9070 : /* Stack frame is required. If stack alignment needed is less
9071 : than incoming stack boundary, don't realign stack. */
9072 285 : stack_realign = incoming_stack_boundary < stack_alignment;
9073 285 : if (!stack_realign)
9074 : {
9075 285 : crtl->max_used_stack_slot_alignment
9076 285 : = incoming_stack_boundary;
9077 285 : crtl->stack_alignment_needed
9078 285 : = incoming_stack_boundary;
9079 : /* Also update preferred_stack_boundary for leaf
9080 : functions. */
9081 285 : crtl->preferred_stack_boundary
9082 285 : = incoming_stack_boundary;
9083 : }
9084 : }
9085 : else
9086 : {
9087 : /* If drap has been set, but it actually isn't live at the
9088 : start of the function, there is no reason to set it up. */
9089 34703 : if (crtl->drap_reg)
9090 : {
9091 35 : basic_block bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
9092 70 : if (! REGNO_REG_SET_P (DF_LR_IN (bb),
9093 : REGNO (crtl->drap_reg)))
9094 : {
9095 35 : crtl->drap_reg = NULL_RTX;
9096 35 : crtl->need_drap = false;
9097 : }
9098 : }
9099 : else
9100 34668 : cfun->machine->no_drap_save_restore = true;
9101 :
9102 34703 : frame_pointer_needed = false;
9103 34703 : stack_realign = false;
9104 34703 : crtl->max_used_stack_slot_alignment = incoming_stack_boundary;
9105 34703 : crtl->stack_alignment_needed = incoming_stack_boundary;
9106 34703 : crtl->stack_alignment_estimated = incoming_stack_boundary;
9107 34703 : if (crtl->preferred_stack_boundary > incoming_stack_boundary)
9108 1 : crtl->preferred_stack_boundary = incoming_stack_boundary;
9109 34703 : df_finish_pass (true);
9110 34703 : df_scan_alloc (NULL);
9111 34703 : df_scan_blocks ();
9112 34703 : df_compute_regs_ever_live (true);
9113 34703 : df_analyze ();
9114 :
9115 34703 : if (flag_var_tracking)
9116 : {
9117 : /* Since frame pointer is no longer available, replace it with
9118 : stack pointer - UNITS_PER_WORD in debug insns. */
9119 136 : df_ref ref, next;
9120 136 : for (ref = DF_REG_USE_CHAIN (HARD_FRAME_POINTER_REGNUM);
9121 136 : ref; ref = next)
9122 : {
9123 0 : next = DF_REF_NEXT_REG (ref);
9124 0 : if (!DF_REF_INSN_INFO (ref))
9125 0 : continue;
9126 :
9127 : /* Make sure the next ref is for a different instruction,
9128 : so that we're not affected by the rescan. */
9129 0 : rtx_insn *insn = DF_REF_INSN (ref);
9130 0 : while (next && DF_REF_INSN (next) == insn)
9131 0 : next = DF_REF_NEXT_REG (next);
9132 :
9133 0 : if (DEBUG_INSN_P (insn))
9134 : {
9135 : bool changed = false;
9136 0 : for (; ref != next; ref = DF_REF_NEXT_REG (ref))
9137 : {
9138 0 : rtx *loc = DF_REF_LOC (ref);
9139 0 : if (*loc == hard_frame_pointer_rtx)
9140 : {
9141 0 : *loc = plus_constant (Pmode,
9142 : stack_pointer_rtx,
9143 0 : -UNITS_PER_WORD);
9144 0 : changed = true;
9145 : }
9146 : }
9147 0 : if (changed)
9148 0 : df_insn_rescan (insn);
9149 : }
9150 : }
9151 : }
9152 :
9153 : recompute_frame_layout_p = true;
9154 : }
9155 : }
9156 1458877 : else if (crtl->max_used_stack_slot_alignment >= 128
9157 663224 : && cfun->machine->stack_frame_required)
9158 : {
9159 : /* We don't need to realign stack. max_used_stack_alignment is
9160 : used to decide how stack frame should be aligned. This is
9161 : independent of any psABIs nor 32-bit vs 64-bit. */
9162 617750 : cfun->machine->max_used_stack_alignment
9163 617750 : = stack_alignment / BITS_PER_UNIT;
9164 : }
9165 :
9166 1493865 : if (crtl->stack_realign_needed != stack_realign)
9167 35270 : recompute_frame_layout_p = true;
9168 1493865 : crtl->stack_realign_needed = stack_realign;
9169 1493865 : crtl->stack_realign_finalized = true;
9170 1493865 : if (recompute_frame_layout_p)
9171 35363 : ix86_compute_frame_layout ();
9172 : }
9173 :
9174 : /* Delete SET_GOT right after entry block if it is allocated to reg. */
9175 :
9176 : static void
9177 0 : ix86_elim_entry_set_got (rtx reg)
9178 : {
9179 0 : basic_block bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
9180 0 : rtx_insn *c_insn = BB_HEAD (bb);
9181 0 : if (!NONDEBUG_INSN_P (c_insn))
9182 0 : c_insn = next_nonnote_nondebug_insn (c_insn);
9183 0 : if (c_insn && NONJUMP_INSN_P (c_insn))
9184 : {
9185 0 : rtx pat = PATTERN (c_insn);
9186 0 : if (GET_CODE (pat) == PARALLEL)
9187 : {
9188 0 : rtx set = XVECEXP (pat, 0, 0);
9189 0 : if (GET_CODE (set) == SET
9190 0 : && GET_CODE (SET_SRC (set)) == UNSPEC
9191 0 : && XINT (SET_SRC (set), 1) == UNSPEC_SET_GOT
9192 0 : && REGNO (SET_DEST (set)) == REGNO (reg))
9193 0 : delete_insn (c_insn);
9194 : }
9195 : }
9196 0 : }
9197 :
9198 : static rtx
9199 192918 : gen_frame_set (rtx reg, rtx frame_reg, int offset, bool store)
9200 : {
9201 192918 : rtx addr, mem;
9202 :
9203 192918 : if (offset)
9204 184232 : addr = plus_constant (Pmode, frame_reg, offset);
9205 192918 : mem = gen_frame_mem (GET_MODE (reg), offset ? addr : frame_reg);
9206 192918 : return gen_rtx_SET (store ? mem : reg, store ? reg : mem);
9207 : }
9208 :
9209 : static inline rtx
9210 100209 : gen_frame_load (rtx reg, rtx frame_reg, int offset)
9211 : {
9212 100209 : return gen_frame_set (reg, frame_reg, offset, false);
9213 : }
9214 :
9215 : static inline rtx
9216 92709 : gen_frame_store (rtx reg, rtx frame_reg, int offset)
9217 : {
9218 92709 : return gen_frame_set (reg, frame_reg, offset, true);
9219 : }
9220 :
9221 : static void
9222 7045 : ix86_emit_outlined_ms2sysv_save (const struct ix86_frame &frame)
9223 : {
9224 7045 : struct machine_function *m = cfun->machine;
9225 7045 : const unsigned ncregs = NUM_X86_64_MS_CLOBBERED_REGS
9226 7045 : + m->call_ms2sysv_extra_regs;
9227 7045 : rtvec v = rtvec_alloc (ncregs + 1);
9228 7045 : unsigned int align, i, vi = 0;
9229 7045 : rtx_insn *insn;
9230 7045 : rtx sym, addr;
9231 7045 : rtx rax = gen_rtx_REG (word_mode, AX_REG);
9232 7045 : const class xlogue_layout &xlogue = xlogue_layout::get_instance ();
9233 :
9234 : /* AL should only be live with sysv_abi. */
9235 7045 : gcc_assert (!ix86_eax_live_at_start_p ());
9236 7045 : gcc_assert (m->fs.sp_offset >= frame.sse_reg_save_offset);
9237 :
9238 : /* Setup RAX as the stub's base pointer. We use stack_realign_offset rather
9239 : we've actually realigned the stack or not. */
9240 7045 : align = GET_MODE_ALIGNMENT (V4SFmode);
9241 7045 : addr = choose_baseaddr (frame.stack_realign_offset
9242 7045 : + xlogue.get_stub_ptr_offset (), &align, AX_REG);
9243 7045 : gcc_assert (align >= GET_MODE_ALIGNMENT (V4SFmode));
9244 :
9245 7045 : emit_insn (gen_rtx_SET (rax, addr));
9246 :
9247 : /* Get the stub symbol. */
9248 8327 : sym = xlogue.get_stub_rtx (frame_pointer_needed ? XLOGUE_STUB_SAVE_HFP
9249 : : XLOGUE_STUB_SAVE);
9250 7045 : RTVEC_ELT (v, vi++) = gen_rtx_USE (VOIDmode, sym);
9251 :
9252 99754 : for (i = 0; i < ncregs; ++i)
9253 : {
9254 92709 : const xlogue_layout::reginfo &r = xlogue.get_reginfo (i);
9255 92709 : rtx reg = gen_rtx_REG ((SSE_REGNO_P (r.regno) ? V4SFmode : word_mode),
9256 92709 : r.regno);
9257 92709 : RTVEC_ELT (v, vi++) = gen_frame_store (reg, rax, -r.offset);
9258 : }
9259 :
9260 7045 : gcc_assert (vi == (unsigned)GET_NUM_ELEM (v));
9261 :
9262 7045 : insn = emit_insn (gen_rtx_PARALLEL (VOIDmode, v));
9263 7045 : RTX_FRAME_RELATED_P (insn) = true;
9264 7045 : }
9265 :
9266 : /* Generate and return an insn body to AND X with Y. */
9267 :
9268 : static rtx_insn *
9269 31224 : gen_and2_insn (rtx x, rtx y)
9270 : {
9271 31224 : enum insn_code icode = optab_handler (and_optab, GET_MODE (x));
9272 :
9273 31224 : gcc_assert (insn_operand_matches (icode, 0, x));
9274 31224 : gcc_assert (insn_operand_matches (icode, 1, x));
9275 31224 : gcc_assert (insn_operand_matches (icode, 2, y));
9276 :
9277 31224 : return GEN_FCN (icode) (x, x, y);
9278 : }
9279 :
9280 : /* Expand the prologue into a bunch of separate insns. */
9281 :
9282 : void
9283 1538185 : ix86_expand_prologue (void)
9284 : {
9285 1538185 : struct machine_function *m = cfun->machine;
9286 1538185 : rtx insn, t;
9287 1538185 : HOST_WIDE_INT allocate;
9288 1538185 : bool int_registers_saved;
9289 1538185 : bool sse_registers_saved;
9290 1538185 : bool save_stub_call_needed;
9291 1538185 : rtx static_chain = NULL_RTX;
9292 :
9293 1538185 : ix86_last_zero_store_uid = 0;
9294 1538185 : if (ix86_function_naked (current_function_decl))
9295 : {
9296 76 : if (flag_stack_usage_info)
9297 0 : current_function_static_stack_size = 0;
9298 76 : return;
9299 : }
9300 :
9301 1538109 : ix86_finalize_stack_frame_flags ();
9302 :
9303 : /* DRAP should not coexist with stack_realign_fp */
9304 1538109 : gcc_assert (!(crtl->drap_reg && stack_realign_fp));
9305 :
9306 1538109 : memset (&m->fs, 0, sizeof (m->fs));
9307 :
9308 : /* Initialize CFA state for before the prologue. */
9309 1538109 : m->fs.cfa_reg = stack_pointer_rtx;
9310 1538109 : m->fs.cfa_offset = INCOMING_FRAME_SP_OFFSET;
9311 :
9312 : /* Track SP offset to the CFA. We continue tracking this after we've
9313 : swapped the CFA register away from SP. In the case of re-alignment
9314 : this is fudged; we're interested to offsets within the local frame. */
9315 1538109 : m->fs.sp_offset = INCOMING_FRAME_SP_OFFSET;
9316 1538109 : m->fs.sp_valid = true;
9317 1538109 : m->fs.sp_realigned = false;
9318 :
9319 1538109 : const struct ix86_frame &frame = cfun->machine->frame;
9320 :
9321 1538109 : if (!TARGET_64BIT && ix86_function_ms_hook_prologue (current_function_decl))
9322 : {
9323 : /* We should have already generated an error for any use of
9324 : ms_hook on a nested function. */
9325 0 : gcc_checking_assert (!ix86_static_chain_on_stack);
9326 :
9327 : /* Check if profiling is active and we shall use profiling before
9328 : prologue variant. If so sorry. */
9329 0 : if (crtl->profile && flag_fentry != 0)
9330 0 : sorry ("%<ms_hook_prologue%> attribute is not compatible "
9331 : "with %<-mfentry%> for 32-bit");
9332 :
9333 : /* In ix86_asm_output_function_label we emitted:
9334 : 8b ff movl.s %edi,%edi
9335 : 55 push %ebp
9336 : 8b ec movl.s %esp,%ebp
9337 :
9338 : This matches the hookable function prologue in Win32 API
9339 : functions in Microsoft Windows XP Service Pack 2 and newer.
9340 : Wine uses this to enable Windows apps to hook the Win32 API
9341 : functions provided by Wine.
9342 :
9343 : What that means is that we've already set up the frame pointer. */
9344 :
9345 0 : if (frame_pointer_needed
9346 0 : && !(crtl->drap_reg && crtl->stack_realign_needed))
9347 : {
9348 0 : rtx push, mov;
9349 :
9350 : /* We've decided to use the frame pointer already set up.
9351 : Describe this to the unwinder by pretending that both
9352 : push and mov insns happen right here.
9353 :
9354 : Putting the unwind info here at the end of the ms_hook
9355 : is done so that we can make absolutely certain we get
9356 : the required byte sequence at the start of the function,
9357 : rather than relying on an assembler that can produce
9358 : the exact encoding required.
9359 :
9360 : However it does mean (in the unpatched case) that we have
9361 : a 1 insn window where the asynchronous unwind info is
9362 : incorrect. However, if we placed the unwind info at
9363 : its correct location we would have incorrect unwind info
9364 : in the patched case. Which is probably all moot since
9365 : I don't expect Wine generates dwarf2 unwind info for the
9366 : system libraries that use this feature. */
9367 :
9368 0 : insn = emit_insn (gen_blockage ());
9369 :
9370 0 : push = gen_push (hard_frame_pointer_rtx);
9371 0 : mov = gen_rtx_SET (hard_frame_pointer_rtx,
9372 : stack_pointer_rtx);
9373 0 : RTX_FRAME_RELATED_P (push) = 1;
9374 0 : RTX_FRAME_RELATED_P (mov) = 1;
9375 :
9376 0 : RTX_FRAME_RELATED_P (insn) = 1;
9377 0 : add_reg_note (insn, REG_FRAME_RELATED_EXPR,
9378 : gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, push, mov)));
9379 :
9380 : /* Note that gen_push incremented m->fs.cfa_offset, even
9381 : though we didn't emit the push insn here. */
9382 0 : m->fs.cfa_reg = hard_frame_pointer_rtx;
9383 0 : m->fs.fp_offset = m->fs.cfa_offset;
9384 0 : m->fs.fp_valid = true;
9385 0 : }
9386 : else
9387 : {
9388 : /* The frame pointer is not needed so pop %ebp again.
9389 : This leaves us with a pristine state. */
9390 0 : emit_insn (gen_pop (hard_frame_pointer_rtx));
9391 : }
9392 : }
9393 :
9394 : /* The first insn of a function that accepts its static chain on the
9395 : stack is to push the register that would be filled in by a direct
9396 : call. This insn will be skipped by the trampoline. */
9397 1538109 : else if (ix86_static_chain_on_stack)
9398 : {
9399 0 : static_chain = ix86_static_chain (cfun->decl, false);
9400 0 : insn = emit_insn (gen_push (static_chain));
9401 0 : emit_insn (gen_blockage ());
9402 :
9403 : /* We don't want to interpret this push insn as a register save,
9404 : only as a stack adjustment. The real copy of the register as
9405 : a save will be done later, if needed. */
9406 0 : t = plus_constant (Pmode, stack_pointer_rtx, -UNITS_PER_WORD);
9407 0 : t = gen_rtx_SET (stack_pointer_rtx, t);
9408 0 : add_reg_note (insn, REG_CFA_ADJUST_CFA, t);
9409 0 : RTX_FRAME_RELATED_P (insn) = 1;
9410 : }
9411 :
9412 : /* Emit prologue code to adjust stack alignment and setup DRAP, in case
9413 : of DRAP is needed and stack realignment is really needed after reload */
9414 1538109 : if (stack_realign_drap)
9415 : {
9416 7009 : int align_bytes = crtl->stack_alignment_needed / BITS_PER_UNIT;
9417 :
9418 : /* Can't use DRAP in interrupt function. */
9419 7009 : if (cfun->machine->func_type != TYPE_NORMAL)
9420 0 : sorry ("Dynamic Realign Argument Pointer (DRAP) not supported "
9421 : "in interrupt service routine. This may be worked "
9422 : "around by avoiding functions with aggregate return.");
9423 :
9424 : /* Only need to push parameter pointer reg if it is caller saved. */
9425 7009 : if (!call_used_or_fixed_reg_p (REGNO (crtl->drap_reg)))
9426 : {
9427 : /* Push arg pointer reg */
9428 137 : insn = emit_insn (gen_push (crtl->drap_reg));
9429 137 : RTX_FRAME_RELATED_P (insn) = 1;
9430 : }
9431 :
9432 : /* Grab the argument pointer. */
9433 7294 : t = plus_constant (Pmode, stack_pointer_rtx, m->fs.sp_offset);
9434 7009 : insn = emit_insn (gen_rtx_SET (crtl->drap_reg, t));
9435 7009 : RTX_FRAME_RELATED_P (insn) = 1;
9436 7009 : m->fs.cfa_reg = crtl->drap_reg;
9437 7009 : m->fs.cfa_offset = 0;
9438 :
9439 : /* Align the stack. */
9440 7009 : insn = emit_insn (gen_and2_insn (stack_pointer_rtx,
9441 7009 : GEN_INT (-align_bytes)));
9442 7009 : RTX_FRAME_RELATED_P (insn) = 1;
9443 :
9444 : /* Replicate the return address on the stack so that return
9445 : address can be reached via (argp - 1) slot. This is needed
9446 : to implement macro RETURN_ADDR_RTX and intrinsic function
9447 : expand_builtin_return_addr etc. */
9448 7579 : t = plus_constant (Pmode, crtl->drap_reg, -UNITS_PER_WORD);
9449 7009 : t = gen_frame_mem (word_mode, t);
9450 7009 : insn = emit_insn (gen_push (t));
9451 7009 : RTX_FRAME_RELATED_P (insn) = 1;
9452 :
9453 : /* For the purposes of frame and register save area addressing,
9454 : we've started over with a new frame. */
9455 7009 : m->fs.sp_offset = INCOMING_FRAME_SP_OFFSET;
9456 7009 : m->fs.realigned = true;
9457 :
9458 7009 : if (static_chain)
9459 : {
9460 : /* Replicate static chain on the stack so that static chain
9461 : can be reached via (argp - 2) slot. This is needed for
9462 : nested function with stack realignment. */
9463 0 : insn = emit_insn (gen_push (static_chain));
9464 0 : RTX_FRAME_RELATED_P (insn) = 1;
9465 : }
9466 : }
9467 :
9468 1538109 : int_registers_saved = (frame.nregs == 0);
9469 1538109 : sse_registers_saved = (frame.nsseregs == 0);
9470 1538109 : save_stub_call_needed = (m->call_ms2sysv);
9471 1538109 : gcc_assert (sse_registers_saved || !save_stub_call_needed);
9472 :
9473 1538109 : if (frame_pointer_needed && !m->fs.fp_valid)
9474 : {
9475 : /* Note: AT&T enter does NOT have reversed args. Enter is probably
9476 : slower on all targets. Also sdb didn't like it. */
9477 488003 : insn = emit_insn (gen_push (hard_frame_pointer_rtx));
9478 488003 : RTX_FRAME_RELATED_P (insn) = 1;
9479 :
9480 488003 : if (m->fs.sp_offset == frame.hard_frame_pointer_offset)
9481 : {
9482 488003 : insn = emit_move_insn (hard_frame_pointer_rtx, stack_pointer_rtx);
9483 488003 : RTX_FRAME_RELATED_P (insn) = 1;
9484 :
9485 488003 : if (m->fs.cfa_reg == stack_pointer_rtx)
9486 480994 : m->fs.cfa_reg = hard_frame_pointer_rtx;
9487 488003 : m->fs.fp_offset = m->fs.sp_offset;
9488 488003 : m->fs.fp_valid = true;
9489 : }
9490 : }
9491 :
9492 1538109 : if (!int_registers_saved)
9493 : {
9494 : /* If saving registers via PUSH, do so now. */
9495 474144 : if (!frame.save_regs_using_mov)
9496 : {
9497 429838 : ix86_emit_save_regs ();
9498 429838 : m->fs.apx_ppx_used = TARGET_APX_PPX && !crtl->calls_eh_return;
9499 429838 : int_registers_saved = true;
9500 429838 : gcc_assert (m->fs.sp_offset == frame.reg_save_offset);
9501 : }
9502 :
9503 : /* When using red zone we may start register saving before allocating
9504 : the stack frame saving one cycle of the prologue. However, avoid
9505 : doing this if we have to probe the stack; at least on x86_64 the
9506 : stack probe can turn into a call that clobbers a red zone location. */
9507 44306 : else if (ix86_using_red_zone ()
9508 44306 : && (! TARGET_STACK_PROBE
9509 0 : || frame.stack_pointer_offset < CHECK_STACK_LIMIT))
9510 : {
9511 39857 : HOST_WIDE_INT allocate_offset;
9512 39857 : if (crtl->shrink_wrapped_separate)
9513 : {
9514 39801 : allocate_offset = m->fs.sp_offset - frame.stack_pointer_offset;
9515 :
9516 : /* Adjust the total offset at the beginning of the function. */
9517 39801 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
9518 : GEN_INT (allocate_offset), -1,
9519 39801 : m->fs.cfa_reg == stack_pointer_rtx);
9520 39801 : m->fs.sp_offset = cfun->machine->frame.stack_pointer_offset;
9521 : }
9522 :
9523 39857 : ix86_emit_save_regs_using_mov (frame.reg_save_offset);
9524 39857 : int_registers_saved = true;
9525 : }
9526 : }
9527 :
9528 1538109 : if (frame.red_zone_size != 0)
9529 142415 : cfun->machine->red_zone_used = true;
9530 :
9531 1538109 : if (stack_realign_fp)
9532 : {
9533 24215 : int align_bytes = crtl->stack_alignment_needed / BITS_PER_UNIT;
9534 24564 : gcc_assert (align_bytes > MIN_STACK_BOUNDARY / BITS_PER_UNIT);
9535 :
9536 : /* Record last valid frame pointer offset. */
9537 24215 : m->fs.sp_realigned_fp_last = frame.reg_save_offset;
9538 :
9539 : /* The computation of the size of the re-aligned stack frame means
9540 : that we must allocate the size of the register save area before
9541 : performing the actual alignment. Otherwise we cannot guarantee
9542 : that there's enough storage above the realignment point. */
9543 24215 : allocate = frame.reg_save_offset - m->fs.sp_offset
9544 24215 : + frame.stack_realign_allocate;
9545 24215 : if (allocate)
9546 2691 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
9547 : GEN_INT (-allocate), -1, false);
9548 :
9549 : /* Align the stack. */
9550 24215 : emit_insn (gen_and2_insn (stack_pointer_rtx, GEN_INT (-align_bytes)));
9551 24215 : m->fs.sp_offset = ROUND_UP (m->fs.sp_offset, align_bytes);
9552 24215 : m->fs.sp_realigned_offset = m->fs.sp_offset
9553 24215 : - frame.stack_realign_allocate;
9554 : /* The stack pointer may no longer be equal to CFA - m->fs.sp_offset.
9555 : Beyond this point, stack access should be done via choose_baseaddr or
9556 : by using sp_valid_at and fp_valid_at to determine the correct base
9557 : register. Henceforth, any CFA offset should be thought of as logical
9558 : and not physical. */
9559 24215 : gcc_assert (m->fs.sp_realigned_offset >= m->fs.sp_realigned_fp_last);
9560 24215 : gcc_assert (m->fs.sp_realigned_offset == frame.stack_realign_offset);
9561 24215 : m->fs.sp_realigned = true;
9562 :
9563 : /* SEH unwind emit doesn't currently support REG_CFA_EXPRESSION, which
9564 : is needed to describe where a register is saved using a realigned
9565 : stack pointer, so we need to invalidate the stack pointer for that
9566 : target. */
9567 24215 : if (TARGET_SEH)
9568 : m->fs.sp_valid = false;
9569 :
9570 : /* If SP offset is non-immediate after allocation of the stack frame,
9571 : then emit SSE saves or stub call prior to allocating the rest of the
9572 : stack frame. This is less efficient for the out-of-line stub because
9573 : we can't combine allocations across the call barrier, but it's better
9574 : than using a scratch register. */
9575 24215 : else if (!x86_64_immediate_operand (GEN_INT (frame.stack_pointer_offset
9576 : - m->fs.sp_realigned_offset),
9577 24215 : Pmode))
9578 : {
9579 3 : if (!sse_registers_saved)
9580 : {
9581 1 : ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset);
9582 1 : sse_registers_saved = true;
9583 : }
9584 2 : else if (save_stub_call_needed)
9585 : {
9586 1 : ix86_emit_outlined_ms2sysv_save (frame);
9587 1 : save_stub_call_needed = false;
9588 : }
9589 : }
9590 : }
9591 :
9592 1538109 : allocate = frame.stack_pointer_offset - m->fs.sp_offset;
9593 :
9594 1538109 : if (flag_stack_usage_info)
9595 : {
9596 : /* We start to count from ARG_POINTER. */
9597 355 : HOST_WIDE_INT stack_size = frame.stack_pointer_offset;
9598 :
9599 : /* If it was realigned, take into account the fake frame. */
9600 355 : if (stack_realign_drap)
9601 : {
9602 1 : if (ix86_static_chain_on_stack)
9603 0 : stack_size += UNITS_PER_WORD;
9604 :
9605 1 : if (!call_used_or_fixed_reg_p (REGNO (crtl->drap_reg)))
9606 0 : stack_size += UNITS_PER_WORD;
9607 :
9608 : /* This over-estimates by 1 minimal-stack-alignment-unit but
9609 : mitigates that by counting in the new return address slot. */
9610 1 : current_function_dynamic_stack_size
9611 1 : += crtl->stack_alignment_needed / BITS_PER_UNIT;
9612 : }
9613 :
9614 355 : current_function_static_stack_size = stack_size;
9615 : }
9616 :
9617 : /* On SEH target with very large frame size, allocate an area to save
9618 : SSE registers (as the very large allocation won't be described). */
9619 1538109 : if (TARGET_SEH
9620 : && frame.stack_pointer_offset > SEH_MAX_FRAME_SIZE
9621 : && !sse_registers_saved)
9622 : {
9623 : HOST_WIDE_INT sse_size
9624 : = frame.sse_reg_save_offset - frame.reg_save_offset;
9625 :
9626 : gcc_assert (int_registers_saved);
9627 :
9628 : /* No need to do stack checking as the area will be immediately
9629 : written. */
9630 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
9631 : GEN_INT (-sse_size), -1,
9632 : m->fs.cfa_reg == stack_pointer_rtx);
9633 : allocate -= sse_size;
9634 : ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset);
9635 : sse_registers_saved = true;
9636 : }
9637 :
9638 : /* If stack clash protection is requested, then probe the stack, unless it
9639 : is already probed on the target. */
9640 1538109 : if (allocate >= 0
9641 1538105 : && flag_stack_clash_protection
9642 1538207 : && !ix86_target_stack_probe ())
9643 : {
9644 98 : ix86_adjust_stack_and_probe (allocate, int_registers_saved, false);
9645 98 : allocate = 0;
9646 : }
9647 :
9648 : /* The stack has already been decremented by the instruction calling us
9649 : so probe if the size is non-negative to preserve the protection area. */
9650 1538011 : else if (allocate >= 0 && flag_stack_check == STATIC_BUILTIN_STACK_CHECK)
9651 : {
9652 45 : const HOST_WIDE_INT probe_interval = get_probe_interval ();
9653 :
9654 45 : if (STACK_CHECK_MOVING_SP)
9655 : {
9656 45 : if (crtl->is_leaf
9657 18 : && !cfun->calls_alloca
9658 18 : && allocate <= probe_interval)
9659 : ;
9660 :
9661 : else
9662 : {
9663 28 : ix86_adjust_stack_and_probe (allocate, int_registers_saved, true);
9664 28 : allocate = 0;
9665 : }
9666 : }
9667 :
9668 : else
9669 : {
9670 : HOST_WIDE_INT size = allocate;
9671 :
9672 : if (TARGET_64BIT && size >= HOST_WIDE_INT_C (0x80000000))
9673 : size = 0x80000000 - get_stack_check_protect () - 1;
9674 :
9675 : if (TARGET_STACK_PROBE)
9676 : {
9677 : if (crtl->is_leaf && !cfun->calls_alloca)
9678 : {
9679 : if (size > probe_interval)
9680 : ix86_emit_probe_stack_range (0, size, int_registers_saved);
9681 : }
9682 : else
9683 : ix86_emit_probe_stack_range (0,
9684 : size + get_stack_check_protect (),
9685 : int_registers_saved);
9686 : }
9687 : else
9688 : {
9689 : if (crtl->is_leaf && !cfun->calls_alloca)
9690 : {
9691 : if (size > probe_interval
9692 : && size > get_stack_check_protect ())
9693 : ix86_emit_probe_stack_range (get_stack_check_protect (),
9694 : (size
9695 : - get_stack_check_protect ()),
9696 : int_registers_saved);
9697 : }
9698 : else
9699 : ix86_emit_probe_stack_range (get_stack_check_protect (), size,
9700 : int_registers_saved);
9701 : }
9702 : }
9703 : }
9704 :
9705 1538105 : if (allocate == 0)
9706 : ;
9707 849352 : else if (!ix86_target_stack_probe ()
9708 849352 : || frame.stack_pointer_offset < CHECK_STACK_LIMIT)
9709 : {
9710 849307 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
9711 : GEN_INT (-allocate), -1,
9712 849307 : m->fs.cfa_reg == stack_pointer_rtx);
9713 : }
9714 : else
9715 : {
9716 45 : rtx eax = gen_rtx_REG (Pmode, AX_REG);
9717 45 : rtx r10 = NULL;
9718 45 : const bool sp_is_cfa_reg = (m->fs.cfa_reg == stack_pointer_rtx);
9719 45 : bool eax_live = ix86_eax_live_at_start_p ();
9720 45 : bool r10_live = false;
9721 :
9722 45 : if (TARGET_64BIT)
9723 45 : r10_live = (DECL_STATIC_CHAIN (current_function_decl) != 0);
9724 :
9725 45 : if (eax_live)
9726 : {
9727 0 : insn = emit_insn (gen_push (eax));
9728 0 : allocate -= UNITS_PER_WORD;
9729 : /* Note that SEH directives need to continue tracking the stack
9730 : pointer even after the frame pointer has been set up. */
9731 0 : if (sp_is_cfa_reg || TARGET_SEH)
9732 : {
9733 0 : if (sp_is_cfa_reg)
9734 0 : m->fs.cfa_offset += UNITS_PER_WORD;
9735 0 : RTX_FRAME_RELATED_P (insn) = 1;
9736 0 : add_reg_note (insn, REG_FRAME_RELATED_EXPR,
9737 0 : gen_rtx_SET (stack_pointer_rtx,
9738 : plus_constant (Pmode,
9739 : stack_pointer_rtx,
9740 : -UNITS_PER_WORD)));
9741 : }
9742 : }
9743 :
9744 45 : if (r10_live)
9745 : {
9746 0 : r10 = gen_rtx_REG (Pmode, R10_REG);
9747 0 : insn = emit_insn (gen_push (r10));
9748 0 : allocate -= UNITS_PER_WORD;
9749 0 : if (sp_is_cfa_reg || TARGET_SEH)
9750 : {
9751 0 : if (sp_is_cfa_reg)
9752 0 : m->fs.cfa_offset += UNITS_PER_WORD;
9753 0 : RTX_FRAME_RELATED_P (insn) = 1;
9754 0 : add_reg_note (insn, REG_FRAME_RELATED_EXPR,
9755 0 : gen_rtx_SET (stack_pointer_rtx,
9756 : plus_constant (Pmode,
9757 : stack_pointer_rtx,
9758 : -UNITS_PER_WORD)));
9759 : }
9760 : }
9761 :
9762 45 : emit_move_insn (eax, GEN_INT (allocate));
9763 45 : emit_insn (gen_allocate_stack_worker_probe (Pmode, eax, eax));
9764 :
9765 : /* Use the fact that AX still contains ALLOCATE. */
9766 45 : insn = emit_insn (gen_pro_epilogue_adjust_stack_sub
9767 45 : (Pmode, stack_pointer_rtx, stack_pointer_rtx, eax));
9768 :
9769 45 : if (sp_is_cfa_reg || TARGET_SEH)
9770 : {
9771 37 : if (sp_is_cfa_reg)
9772 37 : m->fs.cfa_offset += allocate;
9773 37 : RTX_FRAME_RELATED_P (insn) = 1;
9774 37 : add_reg_note (insn, REG_FRAME_RELATED_EXPR,
9775 37 : gen_rtx_SET (stack_pointer_rtx,
9776 : plus_constant (Pmode, stack_pointer_rtx,
9777 : -allocate)));
9778 : }
9779 45 : m->fs.sp_offset += allocate;
9780 :
9781 : /* Use stack_pointer_rtx for relative addressing so that code works for
9782 : realigned stack. But this means that we need a blockage to prevent
9783 : stores based on the frame pointer from being scheduled before. */
9784 45 : if (r10_live && eax_live)
9785 : {
9786 0 : t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, eax);
9787 0 : emit_move_insn (gen_rtx_REG (word_mode, R10_REG),
9788 : gen_frame_mem (word_mode, t));
9789 0 : t = plus_constant (Pmode, t, UNITS_PER_WORD);
9790 0 : emit_move_insn (gen_rtx_REG (word_mode, AX_REG),
9791 : gen_frame_mem (word_mode, t));
9792 0 : emit_insn (gen_memory_blockage ());
9793 : }
9794 45 : else if (eax_live || r10_live)
9795 : {
9796 0 : t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, eax);
9797 0 : emit_move_insn (gen_rtx_REG (word_mode,
9798 : (eax_live ? AX_REG : R10_REG)),
9799 : gen_frame_mem (word_mode, t));
9800 0 : emit_insn (gen_memory_blockage ());
9801 : }
9802 : }
9803 1538109 : gcc_assert (m->fs.sp_offset == frame.stack_pointer_offset);
9804 :
9805 : /* If we haven't already set up the frame pointer, do so now. */
9806 1538109 : if (frame_pointer_needed && !m->fs.fp_valid)
9807 : {
9808 0 : insn = gen_add3_insn (hard_frame_pointer_rtx, stack_pointer_rtx,
9809 0 : GEN_INT (frame.stack_pointer_offset
9810 : - frame.hard_frame_pointer_offset));
9811 0 : insn = emit_insn (insn);
9812 0 : RTX_FRAME_RELATED_P (insn) = 1;
9813 0 : add_reg_note (insn, REG_CFA_ADJUST_CFA, NULL);
9814 :
9815 0 : if (m->fs.cfa_reg == stack_pointer_rtx)
9816 0 : m->fs.cfa_reg = hard_frame_pointer_rtx;
9817 0 : m->fs.fp_offset = frame.hard_frame_pointer_offset;
9818 0 : m->fs.fp_valid = true;
9819 : }
9820 :
9821 1538109 : if (!int_registers_saved)
9822 4449 : ix86_emit_save_regs_using_mov (frame.reg_save_offset);
9823 1538109 : if (!sse_registers_saved)
9824 33375 : ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset);
9825 1504734 : else if (save_stub_call_needed)
9826 7044 : ix86_emit_outlined_ms2sysv_save (frame);
9827 :
9828 : /* For the mcount profiling on 32 bit PIC mode we need to emit SET_GOT
9829 : in PROLOGUE. */
9830 1538109 : if (!TARGET_64BIT && pic_offset_table_rtx && crtl->profile && !flag_fentry)
9831 : {
9832 0 : rtx pic = gen_rtx_REG (Pmode, REAL_PIC_OFFSET_TABLE_REGNUM);
9833 0 : insn = emit_insn (gen_set_got (pic));
9834 0 : RTX_FRAME_RELATED_P (insn) = 1;
9835 0 : add_reg_note (insn, REG_CFA_FLUSH_QUEUE, NULL_RTX);
9836 0 : emit_insn (gen_prologue_use (pic));
9837 : /* Deleting already emitted SET_GOT if exist and allocated to
9838 : REAL_PIC_OFFSET_TABLE_REGNUM. */
9839 0 : ix86_elim_entry_set_got (pic);
9840 : }
9841 :
9842 1538109 : if (crtl->drap_reg && !crtl->stack_realign_needed)
9843 : {
9844 : /* vDRAP is setup but after reload it turns out stack realign
9845 : isn't necessary, here we will emit prologue to setup DRAP
9846 : without stack realign adjustment */
9847 195 : t = choose_baseaddr (0, NULL);
9848 195 : emit_insn (gen_rtx_SET (crtl->drap_reg, t));
9849 : }
9850 :
9851 : /* Prevent instructions from being scheduled into register save push
9852 : sequence when access to the redzone area is done through frame pointer.
9853 : The offset between the frame pointer and the stack pointer is calculated
9854 : relative to the value of the stack pointer at the end of the function
9855 : prologue, and moving instructions that access redzone area via frame
9856 : pointer inside push sequence violates this assumption. */
9857 1538109 : if (frame_pointer_needed && frame.red_zone_size)
9858 131658 : emit_insn (gen_memory_blockage ());
9859 :
9860 : /* SEH requires that the prologue end within 256 bytes of the start of
9861 : the function. Prevent instruction schedules that would extend that.
9862 : Further, prevent alloca modifications to the stack pointer from being
9863 : combined with prologue modifications. */
9864 : if (TARGET_SEH)
9865 : emit_insn (gen_prologue_use (stack_pointer_rtx));
9866 : }
9867 :
9868 : /* Emit code to restore REG using a POP or POPP insn. */
9869 :
9870 : static void
9871 1475190 : ix86_emit_restore_reg_using_pop (rtx reg, bool ppx_p)
9872 : {
9873 1475190 : struct machine_function *m = cfun->machine;
9874 1475190 : rtx_insn *insn = emit_insn (gen_pop (reg, ppx_p));
9875 :
9876 1475190 : ix86_add_cfa_restore_note (insn, reg, m->fs.sp_offset);
9877 1475190 : m->fs.sp_offset -= UNITS_PER_WORD;
9878 :
9879 1475190 : if (m->fs.cfa_reg == crtl->drap_reg
9880 1475190 : && REGNO (reg) == REGNO (crtl->drap_reg))
9881 : {
9882 : /* Previously we'd represented the CFA as an expression
9883 : like *(%ebp - 8). We've just popped that value from
9884 : the stack, which means we need to reset the CFA to
9885 : the drap register. This will remain until we restore
9886 : the stack pointer. */
9887 3972 : add_reg_note (insn, REG_CFA_DEF_CFA, reg);
9888 3972 : RTX_FRAME_RELATED_P (insn) = 1;
9889 :
9890 : /* This means that the DRAP register is valid for addressing too. */
9891 3972 : m->fs.drap_valid = true;
9892 3972 : return;
9893 : }
9894 :
9895 1471218 : if (m->fs.cfa_reg == stack_pointer_rtx)
9896 : {
9897 1387835 : rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
9898 1023902 : x = gen_rtx_SET (stack_pointer_rtx, x);
9899 1023902 : add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
9900 1023902 : RTX_FRAME_RELATED_P (insn) = 1;
9901 :
9902 1205861 : m->fs.cfa_offset -= UNITS_PER_WORD;
9903 : }
9904 :
9905 : /* When the frame pointer is the CFA, and we pop it, we are
9906 : swapping back to the stack pointer as the CFA. This happens
9907 : for stack frames that don't allocate other data, so we assume
9908 : the stack pointer is now pointing at the return address, i.e.
9909 : the function entry state, which makes the offset be 1 word. */
9910 1471218 : if (reg == hard_frame_pointer_rtx)
9911 : {
9912 241633 : m->fs.fp_valid = false;
9913 241633 : if (m->fs.cfa_reg == hard_frame_pointer_rtx)
9914 : {
9915 237648 : m->fs.cfa_reg = stack_pointer_rtx;
9916 237648 : m->fs.cfa_offset -= UNITS_PER_WORD;
9917 :
9918 237648 : add_reg_note (insn, REG_CFA_DEF_CFA,
9919 237648 : plus_constant (Pmode, stack_pointer_rtx,
9920 237648 : m->fs.cfa_offset));
9921 237648 : RTX_FRAME_RELATED_P (insn) = 1;
9922 : }
9923 : }
9924 : }
9925 :
9926 : /* Emit code to restore REG using a POP2 insn. */
9927 : static void
9928 19 : ix86_emit_restore_reg_using_pop2 (rtx reg1, rtx reg2, bool ppx_p = false)
9929 : {
9930 19 : struct machine_function *m = cfun->machine;
9931 19 : const int offset = UNITS_PER_WORD * 2;
9932 19 : rtx_insn *insn;
9933 :
9934 19 : rtx mem = gen_rtx_MEM (TImode, gen_rtx_POST_INC (Pmode,
9935 : stack_pointer_rtx));
9936 :
9937 19 : if (ppx_p)
9938 15 : insn = emit_insn (gen_pop2p_di (reg1, mem, reg2));
9939 : else
9940 4 : insn = emit_insn (gen_pop2_di (reg1, mem, reg2));
9941 :
9942 19 : RTX_FRAME_RELATED_P (insn) = 1;
9943 :
9944 19 : rtx dwarf = NULL_RTX;
9945 19 : dwarf = alloc_reg_note (REG_CFA_RESTORE, reg1, dwarf);
9946 19 : dwarf = alloc_reg_note (REG_CFA_RESTORE, reg2, dwarf);
9947 19 : REG_NOTES (insn) = dwarf;
9948 19 : m->fs.sp_offset -= offset;
9949 :
9950 19 : if (m->fs.cfa_reg == crtl->drap_reg
9951 19 : && (REGNO (reg1) == REGNO (crtl->drap_reg)
9952 3 : || REGNO (reg2) == REGNO (crtl->drap_reg)))
9953 : {
9954 : /* Previously we'd represented the CFA as an expression
9955 : like *(%ebp - 8). We've just popped that value from
9956 : the stack, which means we need to reset the CFA to
9957 : the drap register. This will remain until we restore
9958 : the stack pointer. */
9959 1 : add_reg_note (insn, REG_CFA_DEF_CFA,
9960 1 : REGNO (reg1) == REGNO (crtl->drap_reg) ? reg1 : reg2);
9961 1 : RTX_FRAME_RELATED_P (insn) = 1;
9962 :
9963 : /* This means that the DRAP register is valid for addressing too. */
9964 1 : m->fs.drap_valid = true;
9965 1 : return;
9966 : }
9967 :
9968 18 : if (m->fs.cfa_reg == stack_pointer_rtx)
9969 : {
9970 14 : rtx x = plus_constant (Pmode, stack_pointer_rtx, offset);
9971 14 : x = gen_rtx_SET (stack_pointer_rtx, x);
9972 14 : add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
9973 14 : RTX_FRAME_RELATED_P (insn) = 1;
9974 :
9975 14 : m->fs.cfa_offset -= offset;
9976 : }
9977 :
9978 : /* When the frame pointer is the CFA, and we pop it, we are
9979 : swapping back to the stack pointer as the CFA. This happens
9980 : for stack frames that don't allocate other data, so we assume
9981 : the stack pointer is now pointing at the return address, i.e.
9982 : the function entry state, which makes the offset be 1 word. */
9983 18 : if (reg1 == hard_frame_pointer_rtx || reg2 == hard_frame_pointer_rtx)
9984 : {
9985 0 : m->fs.fp_valid = false;
9986 0 : if (m->fs.cfa_reg == hard_frame_pointer_rtx)
9987 : {
9988 0 : m->fs.cfa_reg = stack_pointer_rtx;
9989 0 : m->fs.cfa_offset -= offset;
9990 :
9991 0 : add_reg_note (insn, REG_CFA_DEF_CFA,
9992 0 : plus_constant (Pmode, stack_pointer_rtx,
9993 0 : m->fs.cfa_offset));
9994 0 : RTX_FRAME_RELATED_P (insn) = 1;
9995 : }
9996 : }
9997 : }
9998 :
9999 : /* Emit code to restore saved registers using POP insns. */
10000 :
10001 : static void
10002 1364143 : ix86_emit_restore_regs_using_pop (bool ppx_p)
10003 : {
10004 1364143 : unsigned int regno;
10005 :
10006 126865299 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
10007 125501156 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, false, true))
10008 1233235 : ix86_emit_restore_reg_using_pop (gen_rtx_REG (word_mode, regno), ppx_p);
10009 1364143 : }
10010 :
10011 : /* Emit code to restore saved registers using POP2 insns. */
10012 :
10013 : static void
10014 570 : ix86_emit_restore_regs_using_pop2 (void)
10015 : {
10016 570 : int regno;
10017 570 : int regno_list[2];
10018 570 : regno_list[0] = regno_list[1] = -1;
10019 570 : int loaded_regnum = 0;
10020 570 : bool aligned = cfun->machine->fs.sp_offset % 16 == 0;
10021 :
10022 53010 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
10023 52440 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, false, true))
10024 : {
10025 128 : if (aligned)
10026 : {
10027 121 : regno_list[loaded_regnum++] = regno;
10028 121 : if (loaded_regnum == 2)
10029 : {
10030 19 : gcc_assert (regno_list[0] != -1
10031 : && regno_list[1] != -1
10032 : && regno_list[0] != regno_list[1]);
10033 :
10034 19 : ix86_emit_restore_reg_using_pop2 (gen_rtx_REG (word_mode,
10035 : regno_list[0]),
10036 : gen_rtx_REG (word_mode,
10037 : regno_list[1]),
10038 19 : TARGET_APX_PPX);
10039 19 : loaded_regnum = 0;
10040 19 : regno_list[0] = regno_list[1] = -1;
10041 : }
10042 : }
10043 : else
10044 : {
10045 14 : ix86_emit_restore_reg_using_pop (gen_rtx_REG (word_mode, regno),
10046 7 : TARGET_APX_PPX);
10047 7 : aligned = true;
10048 : }
10049 : }
10050 :
10051 570 : if (loaded_regnum == 1)
10052 83 : ix86_emit_restore_reg_using_pop (gen_rtx_REG (word_mode, regno_list[0]),
10053 83 : TARGET_APX_PPX);
10054 570 : }
10055 :
10056 : /* Emit code and notes for the LEAVE instruction. If insn is non-null,
10057 : omits the emit and only attaches the notes. */
10058 :
10059 : static void
10060 247673 : ix86_emit_leave (rtx_insn *insn)
10061 : {
10062 247673 : struct machine_function *m = cfun->machine;
10063 :
10064 247673 : if (!insn)
10065 246710 : insn = emit_insn (gen_leave (word_mode));
10066 :
10067 247673 : ix86_add_queued_cfa_restore_notes (insn);
10068 :
10069 247673 : gcc_assert (m->fs.fp_valid);
10070 247673 : m->fs.sp_valid = true;
10071 247673 : m->fs.sp_realigned = false;
10072 247673 : m->fs.sp_offset = m->fs.fp_offset - UNITS_PER_WORD;
10073 247673 : m->fs.fp_valid = false;
10074 :
10075 247673 : if (m->fs.cfa_reg == hard_frame_pointer_rtx)
10076 : {
10077 244541 : m->fs.cfa_reg = stack_pointer_rtx;
10078 244541 : m->fs.cfa_offset = m->fs.sp_offset;
10079 :
10080 244541 : add_reg_note (insn, REG_CFA_DEF_CFA,
10081 244541 : plus_constant (Pmode, stack_pointer_rtx,
10082 244541 : m->fs.sp_offset));
10083 244541 : RTX_FRAME_RELATED_P (insn) = 1;
10084 : }
10085 247673 : ix86_add_cfa_restore_note (insn, hard_frame_pointer_rtx,
10086 : m->fs.fp_offset);
10087 247673 : }
10088 :
10089 : /* Emit code to restore saved registers using MOV insns.
10090 : First register is restored from CFA - CFA_OFFSET. */
10091 : static void
10092 96108 : ix86_emit_restore_regs_using_mov (HOST_WIDE_INT cfa_offset,
10093 : bool maybe_eh_return)
10094 : {
10095 96108 : struct machine_function *m = cfun->machine;
10096 96108 : unsigned int regno;
10097 :
10098 8938044 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
10099 8841936 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, maybe_eh_return, true))
10100 : {
10101 :
10102 : /* Skip registers, already processed by shrink wrap separate. */
10103 261942 : if (!cfun->machine->reg_is_wrapped_separately[regno])
10104 : {
10105 138031 : rtx reg = gen_rtx_REG (word_mode, regno);
10106 138031 : rtx mem;
10107 138031 : rtx_insn *insn;
10108 :
10109 138031 : mem = choose_baseaddr (cfa_offset, NULL);
10110 138031 : mem = gen_frame_mem (word_mode, mem);
10111 138031 : insn = emit_move_insn (reg, mem);
10112 :
10113 138031 : if (m->fs.cfa_reg == crtl->drap_reg
10114 138031 : && regno == REGNO (crtl->drap_reg))
10115 : {
10116 : /* Previously we'd represented the CFA as an expression
10117 : like *(%ebp - 8). We've just popped that value from
10118 : the stack, which means we need to reset the CFA to
10119 : the drap register. This will remain until we restore
10120 : the stack pointer. */
10121 3132 : add_reg_note (insn, REG_CFA_DEF_CFA, reg);
10122 3132 : RTX_FRAME_RELATED_P (insn) = 1;
10123 :
10124 : /* DRAP register is valid for addressing. */
10125 3132 : m->fs.drap_valid = true;
10126 : }
10127 : else
10128 134899 : ix86_add_cfa_restore_note (NULL, reg, cfa_offset);
10129 : }
10130 282340 : cfa_offset -= UNITS_PER_WORD;
10131 : }
10132 96108 : }
10133 :
10134 : /* Emit code to restore saved registers using MOV insns.
10135 : First register is restored from CFA - CFA_OFFSET. */
10136 : static void
10137 33952 : ix86_emit_restore_sse_regs_using_mov (HOST_WIDE_INT cfa_offset,
10138 : bool maybe_eh_return)
10139 : {
10140 33952 : unsigned int regno;
10141 :
10142 3157536 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
10143 3123584 : if (SSE_REGNO_P (regno) && ix86_save_reg (regno, maybe_eh_return, true))
10144 : {
10145 339529 : rtx reg = gen_rtx_REG (V4SFmode, regno);
10146 339529 : rtx mem;
10147 339529 : unsigned int align = GET_MODE_ALIGNMENT (V4SFmode);
10148 :
10149 339529 : mem = choose_baseaddr (cfa_offset, &align);
10150 339529 : mem = gen_rtx_MEM (V4SFmode, mem);
10151 :
10152 : /* The location alignment depends upon the base register. */
10153 339529 : align = MIN (GET_MODE_ALIGNMENT (V4SFmode), align);
10154 339529 : gcc_assert (! (cfa_offset & (align / BITS_PER_UNIT - 1)));
10155 339529 : set_mem_align (mem, align);
10156 339529 : emit_insn (gen_rtx_SET (reg, mem));
10157 :
10158 339529 : ix86_add_cfa_restore_note (NULL, reg, cfa_offset);
10159 :
10160 339529 : cfa_offset -= GET_MODE_SIZE (V4SFmode);
10161 : }
10162 33952 : }
10163 :
10164 : static void
10165 7621 : ix86_emit_outlined_ms2sysv_restore (const struct ix86_frame &frame,
10166 : bool use_call, int style)
10167 : {
10168 7621 : struct machine_function *m = cfun->machine;
10169 7621 : const unsigned ncregs = NUM_X86_64_MS_CLOBBERED_REGS
10170 7621 : + m->call_ms2sysv_extra_regs;
10171 7621 : rtvec v;
10172 7621 : unsigned int elems_needed, align, i, vi = 0;
10173 7621 : rtx_insn *insn;
10174 7621 : rtx sym, tmp;
10175 7621 : rtx rsi = gen_rtx_REG (word_mode, SI_REG);
10176 7621 : rtx r10 = NULL_RTX;
10177 7621 : const class xlogue_layout &xlogue = xlogue_layout::get_instance ();
10178 7621 : HOST_WIDE_INT stub_ptr_offset = xlogue.get_stub_ptr_offset ();
10179 7621 : HOST_WIDE_INT rsi_offset = frame.stack_realign_offset + stub_ptr_offset;
10180 7621 : rtx rsi_frame_load = NULL_RTX;
10181 7621 : HOST_WIDE_INT rsi_restore_offset = (HOST_WIDE_INT)-1;
10182 7621 : enum xlogue_stub stub;
10183 :
10184 7621 : gcc_assert (!m->fs.fp_valid || frame_pointer_needed);
10185 :
10186 : /* If using a realigned stack, we should never start with padding. */
10187 7621 : gcc_assert (!stack_realign_fp || !xlogue.get_stack_align_off_in ());
10188 :
10189 : /* Setup RSI as the stub's base pointer. */
10190 7621 : align = GET_MODE_ALIGNMENT (V4SFmode);
10191 7621 : tmp = choose_baseaddr (rsi_offset, &align, SI_REG);
10192 7621 : gcc_assert (align >= GET_MODE_ALIGNMENT (V4SFmode));
10193 :
10194 7621 : emit_insn (gen_rtx_SET (rsi, tmp));
10195 :
10196 : /* Get a symbol for the stub. */
10197 7621 : if (frame_pointer_needed)
10198 5955 : stub = use_call ? XLOGUE_STUB_RESTORE_HFP
10199 : : XLOGUE_STUB_RESTORE_HFP_TAIL;
10200 : else
10201 1666 : stub = use_call ? XLOGUE_STUB_RESTORE
10202 : : XLOGUE_STUB_RESTORE_TAIL;
10203 7621 : sym = xlogue.get_stub_rtx (stub);
10204 :
10205 7621 : elems_needed = ncregs;
10206 7621 : if (use_call)
10207 6506 : elems_needed += 1;
10208 : else
10209 1267 : elems_needed += frame_pointer_needed ? 5 : 3;
10210 7621 : v = rtvec_alloc (elems_needed);
10211 :
10212 : /* We call the epilogue stub when we need to pop incoming args or we are
10213 : doing a sibling call as the tail. Otherwise, we will emit a jmp to the
10214 : epilogue stub and it is the tail-call. */
10215 7621 : if (use_call)
10216 6506 : RTVEC_ELT (v, vi++) = gen_rtx_USE (VOIDmode, sym);
10217 : else
10218 : {
10219 1115 : RTVEC_ELT (v, vi++) = ret_rtx;
10220 1115 : RTVEC_ELT (v, vi++) = gen_rtx_USE (VOIDmode, sym);
10221 1115 : if (frame_pointer_needed)
10222 : {
10223 963 : rtx rbp = gen_rtx_REG (DImode, BP_REG);
10224 963 : gcc_assert (m->fs.fp_valid);
10225 963 : gcc_assert (m->fs.cfa_reg == hard_frame_pointer_rtx);
10226 :
10227 963 : tmp = plus_constant (DImode, rbp, 8);
10228 963 : RTVEC_ELT (v, vi++) = gen_rtx_SET (stack_pointer_rtx, tmp);
10229 963 : RTVEC_ELT (v, vi++) = gen_rtx_SET (rbp, gen_rtx_MEM (DImode, rbp));
10230 963 : tmp = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode));
10231 963 : RTVEC_ELT (v, vi++) = gen_rtx_CLOBBER (VOIDmode, tmp);
10232 : }
10233 : else
10234 : {
10235 : /* If no hard frame pointer, we set R10 to the SP restore value. */
10236 152 : gcc_assert (!m->fs.fp_valid);
10237 152 : gcc_assert (m->fs.cfa_reg == stack_pointer_rtx);
10238 152 : gcc_assert (m->fs.sp_valid);
10239 :
10240 152 : r10 = gen_rtx_REG (DImode, R10_REG);
10241 152 : tmp = plus_constant (Pmode, rsi, stub_ptr_offset);
10242 152 : emit_insn (gen_rtx_SET (r10, tmp));
10243 :
10244 152 : RTVEC_ELT (v, vi++) = gen_rtx_SET (stack_pointer_rtx, r10);
10245 : }
10246 : }
10247 :
10248 : /* Generate frame load insns and restore notes. */
10249 107830 : for (i = 0; i < ncregs; ++i)
10250 : {
10251 100209 : const xlogue_layout::reginfo &r = xlogue.get_reginfo (i);
10252 100209 : machine_mode mode = SSE_REGNO_P (r.regno) ? V4SFmode : word_mode;
10253 100209 : rtx reg, frame_load;
10254 :
10255 100209 : reg = gen_rtx_REG (mode, r.regno);
10256 100209 : frame_load = gen_frame_load (reg, rsi, r.offset);
10257 :
10258 : /* Save RSI frame load insn & note to add last. */
10259 100209 : if (r.regno == SI_REG)
10260 : {
10261 7621 : gcc_assert (!rsi_frame_load);
10262 7621 : rsi_frame_load = frame_load;
10263 7621 : rsi_restore_offset = r.offset;
10264 : }
10265 : else
10266 : {
10267 92588 : RTVEC_ELT (v, vi++) = frame_load;
10268 92588 : ix86_add_cfa_restore_note (NULL, reg, r.offset);
10269 : }
10270 : }
10271 :
10272 : /* Add RSI frame load & restore note at the end. */
10273 7621 : gcc_assert (rsi_frame_load);
10274 7621 : gcc_assert (rsi_restore_offset != (HOST_WIDE_INT)-1);
10275 7621 : RTVEC_ELT (v, vi++) = rsi_frame_load;
10276 7621 : ix86_add_cfa_restore_note (NULL, gen_rtx_REG (DImode, SI_REG),
10277 : rsi_restore_offset);
10278 :
10279 : /* Finally, for tail-call w/o a hard frame pointer, set SP to R10. */
10280 7621 : if (!use_call && !frame_pointer_needed)
10281 : {
10282 152 : gcc_assert (m->fs.sp_valid);
10283 152 : gcc_assert (!m->fs.sp_realigned);
10284 :
10285 : /* At this point, R10 should point to frame.stack_realign_offset. */
10286 152 : if (m->fs.cfa_reg == stack_pointer_rtx)
10287 152 : m->fs.cfa_offset += m->fs.sp_offset - frame.stack_realign_offset;
10288 152 : m->fs.sp_offset = frame.stack_realign_offset;
10289 : }
10290 :
10291 7621 : gcc_assert (vi == (unsigned int)GET_NUM_ELEM (v));
10292 7621 : tmp = gen_rtx_PARALLEL (VOIDmode, v);
10293 7621 : if (use_call)
10294 6506 : insn = emit_insn (tmp);
10295 : else
10296 : {
10297 1115 : insn = emit_jump_insn (tmp);
10298 1115 : JUMP_LABEL (insn) = ret_rtx;
10299 :
10300 1115 : if (frame_pointer_needed)
10301 963 : ix86_emit_leave (insn);
10302 : else
10303 : {
10304 : /* Need CFA adjust note. */
10305 152 : tmp = gen_rtx_SET (stack_pointer_rtx, r10);
10306 152 : add_reg_note (insn, REG_CFA_ADJUST_CFA, tmp);
10307 : }
10308 : }
10309 :
10310 7621 : RTX_FRAME_RELATED_P (insn) = true;
10311 7621 : ix86_add_queued_cfa_restore_notes (insn);
10312 :
10313 : /* If we're not doing a tail-call, we need to adjust the stack. */
10314 7621 : if (use_call && m->fs.sp_valid)
10315 : {
10316 3706 : HOST_WIDE_INT dealloc = m->fs.sp_offset - frame.stack_realign_offset;
10317 3706 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
10318 : GEN_INT (dealloc), style,
10319 3706 : m->fs.cfa_reg == stack_pointer_rtx);
10320 : }
10321 7621 : }
10322 :
10323 : /* Restore function stack, frame, and registers. */
10324 :
10325 : void
10326 1662157 : ix86_expand_epilogue (int style)
10327 : {
10328 1662157 : struct machine_function *m = cfun->machine;
10329 1662157 : struct machine_frame_state frame_state_save = m->fs;
10330 1662157 : bool restore_regs_via_mov;
10331 1662157 : bool using_drap;
10332 1662157 : bool restore_stub_is_tail = false;
10333 :
10334 1662157 : if (ix86_function_naked (current_function_decl))
10335 : {
10336 : /* The program should not reach this point. */
10337 76 : emit_insn (gen_ud2 ());
10338 124095 : return;
10339 : }
10340 :
10341 1662081 : ix86_finalize_stack_frame_flags ();
10342 1662081 : const struct ix86_frame &frame = cfun->machine->frame;
10343 :
10344 1662081 : m->fs.sp_realigned = stack_realign_fp;
10345 31359 : m->fs.sp_valid = stack_realign_fp
10346 1637827 : || !frame_pointer_needed
10347 2127160 : || crtl->sp_is_unchanging;
10348 1662081 : gcc_assert (!m->fs.sp_valid
10349 : || m->fs.sp_offset == frame.stack_pointer_offset);
10350 :
10351 : /* The FP must be valid if the frame pointer is present. */
10352 1662081 : gcc_assert (frame_pointer_needed == m->fs.fp_valid);
10353 1662081 : gcc_assert (!m->fs.fp_valid
10354 : || m->fs.fp_offset == frame.hard_frame_pointer_offset);
10355 :
10356 : /* We must have *some* valid pointer to the stack frame. */
10357 1662081 : gcc_assert (m->fs.sp_valid || m->fs.fp_valid);
10358 :
10359 : /* The DRAP is never valid at this point. */
10360 1662081 : gcc_assert (!m->fs.drap_valid);
10361 :
10362 : /* See the comment about red zone and frame
10363 : pointer usage in ix86_expand_prologue. */
10364 1662081 : if (frame_pointer_needed && frame.red_zone_size)
10365 131688 : emit_insn (gen_memory_blockage ());
10366 :
10367 1662081 : using_drap = crtl->drap_reg && crtl->stack_realign_needed;
10368 7105 : gcc_assert (!using_drap || m->fs.cfa_reg == crtl->drap_reg);
10369 :
10370 : /* Determine the CFA offset of the end of the red-zone. */
10371 1662081 : m->fs.red_zone_offset = 0;
10372 1662081 : if (ix86_using_red_zone () && crtl->args.pops_args < 65536)
10373 : {
10374 : /* The red-zone begins below return address and error code in
10375 : exception handler. */
10376 1484449 : m->fs.red_zone_offset = RED_ZONE_SIZE + INCOMING_FRAME_SP_OFFSET;
10377 :
10378 : /* When the register save area is in the aligned portion of
10379 : the stack, determine the maximum runtime displacement that
10380 : matches up with the aligned frame. */
10381 1484449 : if (stack_realign_drap)
10382 8480 : m->fs.red_zone_offset -= (crtl->stack_alignment_needed / BITS_PER_UNIT
10383 4240 : + UNITS_PER_WORD);
10384 : }
10385 :
10386 1662081 : HOST_WIDE_INT reg_save_offset = frame.reg_save_offset;
10387 :
10388 : /* Special care must be taken for the normal return case of a function
10389 : using eh_return: the eax and edx registers are marked as saved, but
10390 : not restored along this path. Adjust the save location to match. */
10391 1662081 : if (crtl->calls_eh_return && style != 2)
10392 37 : reg_save_offset -= 2 * UNITS_PER_WORD;
10393 :
10394 : /* EH_RETURN requires the use of moves to function properly. */
10395 1662081 : if (crtl->calls_eh_return)
10396 : restore_regs_via_mov = true;
10397 : /* SEH requires the use of pops to identify the epilogue. */
10398 1662023 : else if (TARGET_SEH)
10399 : restore_regs_via_mov = false;
10400 : /* If we already save reg with pushp, don't use move at epilogue. */
10401 1662023 : else if (m->fs.apx_ppx_used)
10402 : restore_regs_via_mov = false;
10403 : /* If we're only restoring one register and sp cannot be used then
10404 : using a move instruction to restore the register since it's
10405 : less work than reloading sp and popping the register. */
10406 1661935 : else if (!sp_valid_at (frame.hfp_save_offset) && frame.nregs <= 1)
10407 : restore_regs_via_mov = true;
10408 1601091 : else if (crtl->shrink_wrapped_separate
10409 1549348 : || (TARGET_EPILOGUE_USING_MOVE
10410 56848 : && cfun->machine->use_fast_prologue_epilogue
10411 56792 : && (frame.nregs > 1
10412 56778 : || m->fs.sp_offset != reg_save_offset)))
10413 : restore_regs_via_mov = true;
10414 1549113 : else if (frame_pointer_needed
10415 425873 : && !frame.nregs
10416 330673 : && m->fs.sp_offset != reg_save_offset)
10417 : restore_regs_via_mov = true;
10418 1394913 : else if (frame_pointer_needed
10419 271673 : && TARGET_USE_LEAVE
10420 271598 : && cfun->machine->use_fast_prologue_epilogue
10421 215110 : && frame.nregs == 1)
10422 : restore_regs_via_mov = true;
10423 : else
10424 1662081 : restore_regs_via_mov = false;
10425 :
10426 1662081 : if (crtl->shrink_wrapped_separate)
10427 51774 : gcc_assert (restore_regs_via_mov);
10428 :
10429 1610307 : if (restore_regs_via_mov || frame.nsseregs)
10430 : {
10431 : /* Ensure that the entire register save area is addressable via
10432 : the stack pointer, if we will restore SSE regs via sp. */
10433 330965 : if (TARGET_64BIT
10434 318380 : && m->fs.sp_offset > 0x7fffffff
10435 23 : && sp_valid_at (frame.stack_realign_offset + 1)
10436 330987 : && (frame.nsseregs + frame.nregs) != 0)
10437 : {
10438 6 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
10439 6 : GEN_INT (m->fs.sp_offset
10440 : - frame.sse_reg_save_offset),
10441 : style,
10442 6 : m->fs.cfa_reg == stack_pointer_rtx);
10443 : }
10444 : }
10445 :
10446 : /* If there are any SSE registers to restore, then we have to do it
10447 : via moves, since there's obviously no pop for SSE regs. */
10448 1662081 : if (frame.nsseregs)
10449 33952 : ix86_emit_restore_sse_regs_using_mov (frame.sse_reg_save_offset,
10450 : style == 2);
10451 :
10452 1662081 : if (m->call_ms2sysv)
10453 : {
10454 7621 : int pop_incoming_args = crtl->args.pops_args && crtl->args.size;
10455 :
10456 : /* We cannot use a tail-call for the stub if:
10457 : 1. We have to pop incoming args,
10458 : 2. We have additional int regs to restore, or
10459 : 3. A sibling call will be the tail-call, or
10460 : 4. We are emitting an eh_return_internal epilogue.
10461 :
10462 : TODO: Item 4 has not yet tested!
10463 :
10464 : If any of the above are true, we will call the stub rather than
10465 : jump to it. */
10466 7621 : restore_stub_is_tail = !(pop_incoming_args || frame.nregs || style != 1);
10467 7621 : ix86_emit_outlined_ms2sysv_restore (frame, !restore_stub_is_tail, style);
10468 : }
10469 :
10470 : /* If using out-of-line stub that is a tail-call, then...*/
10471 1662081 : if (m->call_ms2sysv && restore_stub_is_tail)
10472 : {
10473 : /* TODO: parinoid tests. (remove eventually) */
10474 1115 : gcc_assert (m->fs.sp_valid);
10475 1115 : gcc_assert (!m->fs.sp_realigned);
10476 1115 : gcc_assert (!m->fs.fp_valid);
10477 1115 : gcc_assert (!m->fs.realigned);
10478 1115 : gcc_assert (m->fs.sp_offset == UNITS_PER_WORD);
10479 1115 : gcc_assert (!crtl->drap_reg);
10480 1115 : gcc_assert (!frame.nregs);
10481 1115 : gcc_assert (!crtl->shrink_wrapped_separate);
10482 : }
10483 1660966 : else if (restore_regs_via_mov)
10484 : {
10485 296253 : rtx t;
10486 :
10487 296253 : if (frame.nregs)
10488 96108 : ix86_emit_restore_regs_using_mov (reg_save_offset, style == 2);
10489 :
10490 : /* eh_return epilogues need %ecx added to the stack pointer. */
10491 296253 : if (style == 2)
10492 : {
10493 37 : rtx sa = EH_RETURN_STACKADJ_RTX;
10494 29 : rtx_insn *insn;
10495 :
10496 29 : gcc_assert (!crtl->shrink_wrapped_separate);
10497 :
10498 : /* Stack realignment doesn't work with eh_return. */
10499 29 : if (crtl->stack_realign_needed)
10500 0 : sorry ("Stack realignment not supported with "
10501 : "%<__builtin_eh_return%>");
10502 :
10503 : /* regparm nested functions don't work with eh_return. */
10504 29 : if (ix86_static_chain_on_stack)
10505 0 : sorry ("regparm nested function not supported with "
10506 : "%<__builtin_eh_return%>");
10507 :
10508 29 : if (frame_pointer_needed)
10509 : {
10510 35 : t = gen_rtx_PLUS (Pmode, hard_frame_pointer_rtx, sa);
10511 43 : t = plus_constant (Pmode, t, m->fs.fp_offset - UNITS_PER_WORD);
10512 27 : emit_insn (gen_rtx_SET (sa, t));
10513 :
10514 : /* NB: eh_return epilogues must restore the frame pointer
10515 : in word_mode since the upper 32 bits of RBP register
10516 : can have any values. */
10517 27 : t = gen_frame_mem (word_mode, hard_frame_pointer_rtx);
10518 27 : rtx frame_reg = gen_rtx_REG (word_mode,
10519 : HARD_FRAME_POINTER_REGNUM);
10520 27 : insn = emit_move_insn (frame_reg, t);
10521 :
10522 : /* Note that we use SA as a temporary CFA, as the return
10523 : address is at the proper place relative to it. We
10524 : pretend this happens at the FP restore insn because
10525 : prior to this insn the FP would be stored at the wrong
10526 : offset relative to SA, and after this insn we have no
10527 : other reasonable register to use for the CFA. We don't
10528 : bother resetting the CFA to the SP for the duration of
10529 : the return insn, unless the control flow instrumentation
10530 : is done. In this case the SP is used later and we have
10531 : to reset CFA to SP. */
10532 27 : add_reg_note (insn, REG_CFA_DEF_CFA,
10533 35 : plus_constant (Pmode, sa, UNITS_PER_WORD));
10534 27 : ix86_add_queued_cfa_restore_notes (insn);
10535 27 : add_reg_note (insn, REG_CFA_RESTORE, frame_reg);
10536 27 : RTX_FRAME_RELATED_P (insn) = 1;
10537 :
10538 27 : m->fs.cfa_reg = sa;
10539 27 : m->fs.cfa_offset = UNITS_PER_WORD;
10540 27 : m->fs.fp_valid = false;
10541 :
10542 27 : pro_epilogue_adjust_stack (stack_pointer_rtx, sa,
10543 : const0_rtx, style,
10544 27 : flag_cf_protection);
10545 : }
10546 : else
10547 : {
10548 2 : t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, sa);
10549 2 : t = plus_constant (Pmode, t, m->fs.sp_offset - UNITS_PER_WORD);
10550 2 : insn = emit_insn (gen_rtx_SET (stack_pointer_rtx, t));
10551 2 : ix86_add_queued_cfa_restore_notes (insn);
10552 :
10553 2 : gcc_assert (m->fs.cfa_reg == stack_pointer_rtx);
10554 2 : if (m->fs.cfa_offset != UNITS_PER_WORD)
10555 : {
10556 2 : m->fs.cfa_offset = UNITS_PER_WORD;
10557 2 : add_reg_note (insn, REG_CFA_DEF_CFA,
10558 2 : plus_constant (Pmode, stack_pointer_rtx,
10559 2 : UNITS_PER_WORD));
10560 2 : RTX_FRAME_RELATED_P (insn) = 1;
10561 : }
10562 : }
10563 29 : m->fs.sp_offset = UNITS_PER_WORD;
10564 29 : m->fs.sp_valid = true;
10565 29 : m->fs.sp_realigned = false;
10566 : }
10567 : }
10568 : else
10569 : {
10570 : /* SEH requires that the function end with (1) a stack adjustment
10571 : if necessary, (2) a sequence of pops, and (3) a return or
10572 : jump instruction. Prevent insns from the function body from
10573 : being scheduled into this sequence. */
10574 1364713 : if (TARGET_SEH)
10575 : {
10576 : /* Prevent a catch region from being adjacent to the standard
10577 : epilogue sequence. Unfortunately neither crtl->uses_eh_lsda
10578 : nor several other flags that would be interesting to test are
10579 : set up yet. */
10580 : if (flag_non_call_exceptions)
10581 : emit_insn (gen_nops (const1_rtx));
10582 : else
10583 : emit_insn (gen_blockage ());
10584 : }
10585 :
10586 : /* First step is to deallocate the stack frame so that we can
10587 : pop the registers. If the stack pointer was realigned, it needs
10588 : to be restored now. Also do it on SEH target for very large
10589 : frame as the emitted instructions aren't allowed by the ABI
10590 : in epilogues. */
10591 1364713 : if (!m->fs.sp_valid || m->fs.sp_realigned
10592 : || (TARGET_SEH
10593 : && (m->fs.sp_offset - reg_save_offset
10594 : >= SEH_MAX_FRAME_SIZE)))
10595 : {
10596 29560 : pro_epilogue_adjust_stack (stack_pointer_rtx, hard_frame_pointer_rtx,
10597 29560 : GEN_INT (m->fs.fp_offset
10598 : - reg_save_offset),
10599 : style, false);
10600 : }
10601 1335153 : else if (m->fs.sp_offset != reg_save_offset)
10602 : {
10603 619081 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
10604 : GEN_INT (m->fs.sp_offset
10605 : - reg_save_offset),
10606 : style,
10607 619081 : m->fs.cfa_reg == stack_pointer_rtx);
10608 : }
10609 :
10610 1364713 : if (TARGET_APX_PUSH2POP2
10611 573 : && ix86_can_use_push2pop2 ()
10612 1365284 : && m->func_type == TYPE_NORMAL)
10613 570 : ix86_emit_restore_regs_using_pop2 ();
10614 : else
10615 1364143 : ix86_emit_restore_regs_using_pop (TARGET_APX_PPX);
10616 : }
10617 :
10618 : /* If we used a stack pointer and haven't already got rid of it,
10619 : then do so now. */
10620 1662081 : if (m->fs.fp_valid)
10621 : {
10622 : /* If the stack pointer is valid and pointing at the frame
10623 : pointer store address, then we only need a pop. */
10624 488343 : if (sp_valid_at (frame.hfp_save_offset)
10625 488343 : && m->fs.sp_offset == frame.hfp_save_offset)
10626 241621 : ix86_emit_restore_reg_using_pop (hard_frame_pointer_rtx);
10627 : /* Leave results in shorter dependency chains on CPUs that are
10628 : able to grok it fast. */
10629 246722 : else if (TARGET_USE_LEAVE
10630 12 : || optimize_bb_for_size_p (EXIT_BLOCK_PTR_FOR_FN (cfun))
10631 246734 : || !cfun->machine->use_fast_prologue_epilogue)
10632 246710 : ix86_emit_leave (NULL);
10633 : else
10634 : {
10635 12 : pro_epilogue_adjust_stack (stack_pointer_rtx,
10636 : hard_frame_pointer_rtx,
10637 12 : const0_rtx, style, !using_drap);
10638 12 : ix86_emit_restore_reg_using_pop (hard_frame_pointer_rtx);
10639 : }
10640 : }
10641 :
10642 1662081 : if (using_drap)
10643 : {
10644 7105 : int param_ptr_offset = UNITS_PER_WORD;
10645 7105 : rtx_insn *insn;
10646 :
10647 7105 : gcc_assert (stack_realign_drap);
10648 :
10649 7105 : if (ix86_static_chain_on_stack)
10650 0 : param_ptr_offset += UNITS_PER_WORD;
10651 7105 : if (!call_used_or_fixed_reg_p (REGNO (crtl->drap_reg)))
10652 232 : param_ptr_offset += UNITS_PER_WORD;
10653 :
10654 7410 : insn = emit_insn (gen_rtx_SET
10655 : (stack_pointer_rtx,
10656 : plus_constant (Pmode, crtl->drap_reg,
10657 : -param_ptr_offset)));
10658 7105 : m->fs.cfa_reg = stack_pointer_rtx;
10659 7105 : m->fs.cfa_offset = param_ptr_offset;
10660 7105 : m->fs.sp_offset = param_ptr_offset;
10661 7105 : m->fs.realigned = false;
10662 :
10663 7410 : add_reg_note (insn, REG_CFA_DEF_CFA,
10664 7105 : plus_constant (Pmode, stack_pointer_rtx,
10665 7105 : param_ptr_offset));
10666 7105 : RTX_FRAME_RELATED_P (insn) = 1;
10667 :
10668 7105 : if (!call_used_or_fixed_reg_p (REGNO (crtl->drap_reg)))
10669 232 : ix86_emit_restore_reg_using_pop (crtl->drap_reg);
10670 : }
10671 :
10672 : /* At this point the stack pointer must be valid, and we must have
10673 : restored all of the registers. We may not have deallocated the
10674 : entire stack frame. We've delayed this until now because it may
10675 : be possible to merge the local stack deallocation with the
10676 : deallocation forced by ix86_static_chain_on_stack. */
10677 1662081 : gcc_assert (m->fs.sp_valid);
10678 1662081 : gcc_assert (!m->fs.sp_realigned);
10679 1662081 : gcc_assert (!m->fs.fp_valid);
10680 1662081 : gcc_assert (!m->fs.realigned);
10681 1797915 : if (m->fs.sp_offset != UNITS_PER_WORD)
10682 : {
10683 49488 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
10684 : GEN_INT (m->fs.sp_offset - UNITS_PER_WORD),
10685 : style, true);
10686 : }
10687 : else
10688 1612593 : ix86_add_queued_cfa_restore_notes (get_last_insn ());
10689 :
10690 : /* Sibcall epilogues don't want a return instruction. */
10691 1662081 : if (style == 0)
10692 : {
10693 123943 : m->fs = frame_state_save;
10694 123943 : return;
10695 : }
10696 :
10697 1538138 : if (cfun->machine->func_type != TYPE_NORMAL)
10698 120 : emit_jump_insn (gen_interrupt_return ());
10699 1538018 : else if (crtl->args.pops_args && crtl->args.size)
10700 : {
10701 26005 : rtx popc = GEN_INT (crtl->args.pops_args);
10702 :
10703 : /* i386 can only pop 64K bytes. If asked to pop more, pop return
10704 : address, do explicit add, and jump indirectly to the caller. */
10705 :
10706 26005 : if (crtl->args.pops_args >= 65536)
10707 : {
10708 0 : rtx ecx = gen_rtx_REG (SImode, CX_REG);
10709 0 : rtx_insn *insn;
10710 :
10711 : /* There is no "pascal" calling convention in any 64bit ABI. */
10712 0 : gcc_assert (!TARGET_64BIT);
10713 :
10714 0 : insn = emit_insn (gen_pop (ecx));
10715 0 : m->fs.cfa_offset -= UNITS_PER_WORD;
10716 0 : m->fs.sp_offset -= UNITS_PER_WORD;
10717 :
10718 0 : rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
10719 0 : x = gen_rtx_SET (stack_pointer_rtx, x);
10720 0 : add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
10721 0 : add_reg_note (insn, REG_CFA_REGISTER, gen_rtx_SET (ecx, pc_rtx));
10722 0 : RTX_FRAME_RELATED_P (insn) = 1;
10723 :
10724 0 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
10725 : popc, -1, true);
10726 0 : emit_jump_insn (gen_simple_return_indirect_internal (ecx));
10727 : }
10728 : else
10729 26005 : emit_jump_insn (gen_simple_return_pop_internal (popc));
10730 : }
10731 1512013 : else if (!m->call_ms2sysv || !restore_stub_is_tail)
10732 : {
10733 : /* In case of return from EH a simple return cannot be used
10734 : as a return address will be compared with a shadow stack
10735 : return address. Use indirect jump instead. */
10736 1510898 : if (style == 2 && flag_cf_protection)
10737 : {
10738 : /* Register used in indirect jump must be in word_mode. But
10739 : Pmode may not be the same as word_mode for x32. */
10740 17 : rtx ecx = gen_rtx_REG (word_mode, CX_REG);
10741 17 : rtx_insn *insn;
10742 :
10743 17 : insn = emit_insn (gen_pop (ecx));
10744 17 : m->fs.cfa_offset -= UNITS_PER_WORD;
10745 17 : m->fs.sp_offset -= UNITS_PER_WORD;
10746 :
10747 33 : rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
10748 17 : x = gen_rtx_SET (stack_pointer_rtx, x);
10749 17 : add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
10750 17 : add_reg_note (insn, REG_CFA_REGISTER, gen_rtx_SET (ecx, pc_rtx));
10751 17 : RTX_FRAME_RELATED_P (insn) = 1;
10752 :
10753 17 : emit_jump_insn (gen_simple_return_indirect_internal (ecx));
10754 17 : }
10755 : else
10756 1510881 : emit_jump_insn (gen_simple_return_internal ());
10757 : }
10758 :
10759 : /* Restore the state back to the state from the prologue,
10760 : so that it's correct for the next epilogue. */
10761 1538138 : m->fs = frame_state_save;
10762 : }
10763 :
10764 : /* Reset from the function's potential modifications. */
10765 :
10766 : static void
10767 1499320 : ix86_output_function_epilogue (FILE *file ATTRIBUTE_UNUSED)
10768 : {
10769 1499320 : if (pic_offset_table_rtx
10770 1499320 : && !ix86_use_pseudo_pic_reg ())
10771 0 : SET_REGNO (pic_offset_table_rtx, REAL_PIC_OFFSET_TABLE_REGNUM);
10772 :
10773 1499320 : if (TARGET_MACHO)
10774 : {
10775 : rtx_insn *insn = get_last_insn ();
10776 : rtx_insn *deleted_debug_label = NULL;
10777 :
10778 : /* Mach-O doesn't support labels at the end of objects, so if
10779 : it looks like we might want one, take special action.
10780 : First, collect any sequence of deleted debug labels. */
10781 : while (insn
10782 : && NOTE_P (insn)
10783 : && NOTE_KIND (insn) != NOTE_INSN_DELETED_LABEL)
10784 : {
10785 : /* Don't insert a nop for NOTE_INSN_DELETED_DEBUG_LABEL
10786 : notes only, instead set their CODE_LABEL_NUMBER to -1,
10787 : otherwise there would be code generation differences
10788 : in between -g and -g0. */
10789 : if (NOTE_P (insn) && NOTE_KIND (insn)
10790 : == NOTE_INSN_DELETED_DEBUG_LABEL)
10791 : deleted_debug_label = insn;
10792 : insn = PREV_INSN (insn);
10793 : }
10794 :
10795 : /* If we have:
10796 : label:
10797 : barrier
10798 : then this needs to be detected, so skip past the barrier. */
10799 :
10800 : if (insn && BARRIER_P (insn))
10801 : insn = PREV_INSN (insn);
10802 :
10803 : /* Up to now we've only seen notes or barriers. */
10804 : if (insn)
10805 : {
10806 : if (LABEL_P (insn)
10807 : || (NOTE_P (insn)
10808 : && NOTE_KIND (insn) == NOTE_INSN_DELETED_LABEL))
10809 : /* Trailing label. */
10810 : fputs ("\tnop\n", file);
10811 : else if (cfun && ! cfun->is_thunk)
10812 : {
10813 : /* See if we have a completely empty function body, skipping
10814 : the special case of the picbase thunk emitted as asm. */
10815 : while (insn && ! INSN_P (insn))
10816 : insn = PREV_INSN (insn);
10817 : /* If we don't find any insns, we've got an empty function body;
10818 : I.e. completely empty - without a return or branch. This is
10819 : taken as the case where a function body has been removed
10820 : because it contains an inline __builtin_unreachable(). GCC
10821 : declares that reaching __builtin_unreachable() means UB so
10822 : we're not obliged to do anything special; however, we want
10823 : non-zero-sized function bodies. To meet this, and help the
10824 : user out, let's trap the case. */
10825 : if (insn == NULL)
10826 : fputs ("\tud2\n", file);
10827 : }
10828 : }
10829 : else if (deleted_debug_label)
10830 : for (insn = deleted_debug_label; insn; insn = NEXT_INSN (insn))
10831 : if (NOTE_KIND (insn) == NOTE_INSN_DELETED_DEBUG_LABEL)
10832 : CODE_LABEL_NUMBER (insn) = -1;
10833 : }
10834 1499320 : }
10835 :
10836 : /* Implement TARGET_ASM_PRINT_PATCHABLE_FUNCTION_ENTRY. */
10837 :
10838 : void
10839 59 : ix86_print_patchable_function_entry (FILE *file,
10840 : unsigned HOST_WIDE_INT patch_area_size,
10841 : bool record_p)
10842 : {
10843 59 : if (cfun->machine->function_label_emitted)
10844 : {
10845 : /* NB: When ix86_print_patchable_function_entry is called after
10846 : function table has been emitted, we have inserted or queued
10847 : a pseudo UNSPECV_PATCHABLE_AREA instruction at the proper
10848 : place. There is nothing to do here. */
10849 : return;
10850 : }
10851 :
10852 8 : default_print_patchable_function_entry (file, patch_area_size,
10853 : record_p);
10854 : }
10855 :
10856 : /* Output patchable area. NB: default_print_patchable_function_entry
10857 : isn't available in i386.md. */
10858 :
10859 : void
10860 51 : ix86_output_patchable_area (unsigned int patch_area_size,
10861 : bool record_p)
10862 : {
10863 51 : default_print_patchable_function_entry (asm_out_file,
10864 : patch_area_size,
10865 : record_p);
10866 51 : }
10867 :
10868 : /* Return a scratch register to use in the split stack prologue. The
10869 : split stack prologue is used for -fsplit-stack. It is the first
10870 : instructions in the function, even before the regular prologue.
10871 : The scratch register can be any caller-saved register which is not
10872 : used for parameters or for the static chain. */
10873 :
10874 : static unsigned int
10875 24564 : split_stack_prologue_scratch_regno (void)
10876 : {
10877 24564 : if (TARGET_64BIT)
10878 : return R11_REG;
10879 : else
10880 : {
10881 6896 : bool is_fastcall, is_thiscall;
10882 6896 : int regparm;
10883 :
10884 6896 : is_fastcall = (lookup_attribute ("fastcall",
10885 6896 : TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
10886 : != NULL);
10887 6896 : is_thiscall = (lookup_attribute ("thiscall",
10888 6896 : TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
10889 : != NULL);
10890 6896 : regparm = ix86_function_regparm (TREE_TYPE (cfun->decl), cfun->decl);
10891 :
10892 6896 : if (is_fastcall)
10893 : {
10894 0 : if (DECL_STATIC_CHAIN (cfun->decl))
10895 : {
10896 0 : sorry ("%<-fsplit-stack%> does not support fastcall with "
10897 : "nested function");
10898 0 : return INVALID_REGNUM;
10899 : }
10900 : return AX_REG;
10901 : }
10902 6896 : else if (is_thiscall)
10903 : {
10904 0 : if (!DECL_STATIC_CHAIN (cfun->decl))
10905 : return DX_REG;
10906 0 : return AX_REG;
10907 : }
10908 6896 : else if (regparm < 3)
10909 : {
10910 6896 : if (!DECL_STATIC_CHAIN (cfun->decl))
10911 : return CX_REG;
10912 : else
10913 : {
10914 458 : if (regparm >= 2)
10915 : {
10916 0 : sorry ("%<-fsplit-stack%> does not support 2 register "
10917 : "parameters for a nested function");
10918 0 : return INVALID_REGNUM;
10919 : }
10920 : return DX_REG;
10921 : }
10922 : }
10923 : else
10924 : {
10925 : /* FIXME: We could make this work by pushing a register
10926 : around the addition and comparison. */
10927 0 : sorry ("%<-fsplit-stack%> does not support 3 register parameters");
10928 0 : return INVALID_REGNUM;
10929 : }
10930 : }
10931 : }
10932 :
10933 : /* A SYMBOL_REF for the function which allocates new stackspace for
10934 : -fsplit-stack. */
10935 :
10936 : static GTY(()) rtx split_stack_fn;
10937 :
10938 : /* A SYMBOL_REF for the more stack function when using the large model. */
10939 :
10940 : static GTY(()) rtx split_stack_fn_large;
10941 :
10942 : /* Return location of the stack guard value in the TLS block. */
10943 :
10944 : rtx
10945 260046 : ix86_split_stack_guard (void)
10946 : {
10947 260046 : int offset;
10948 260046 : addr_space_t as = DEFAULT_TLS_SEG_REG;
10949 260046 : rtx r;
10950 :
10951 260046 : gcc_assert (flag_split_stack);
10952 :
10953 : #ifdef TARGET_THREAD_SPLIT_STACK_OFFSET
10954 260046 : offset = TARGET_THREAD_SPLIT_STACK_OFFSET;
10955 : #else
10956 : gcc_unreachable ();
10957 : #endif
10958 :
10959 260046 : r = GEN_INT (offset);
10960 358047 : r = gen_const_mem (Pmode, r);
10961 260046 : set_mem_addr_space (r, as);
10962 :
10963 260046 : return r;
10964 : }
10965 :
10966 : /* Handle -fsplit-stack. These are the first instructions in the
10967 : function, even before the regular prologue. */
10968 :
10969 : void
10970 260036 : ix86_expand_split_stack_prologue (void)
10971 : {
10972 260036 : HOST_WIDE_INT allocate;
10973 260036 : unsigned HOST_WIDE_INT args_size;
10974 260036 : rtx_code_label *label;
10975 260036 : rtx limit, current, allocate_rtx, call_fusage;
10976 260036 : rtx_insn *call_insn;
10977 260036 : unsigned int scratch_regno = INVALID_REGNUM;
10978 260036 : rtx scratch_reg = NULL_RTX;
10979 260036 : rtx_code_label *varargs_label = NULL;
10980 260036 : rtx fn;
10981 :
10982 260036 : gcc_assert (flag_split_stack && reload_completed);
10983 :
10984 260036 : ix86_finalize_stack_frame_flags ();
10985 260036 : struct ix86_frame &frame = cfun->machine->frame;
10986 260036 : allocate = frame.stack_pointer_offset - INCOMING_FRAME_SP_OFFSET;
10987 :
10988 : /* This is the label we will branch to if we have enough stack
10989 : space. We expect the basic block reordering pass to reverse this
10990 : branch if optimizing, so that we branch in the unlikely case. */
10991 260036 : label = gen_label_rtx ();
10992 :
10993 : /* We need to compare the stack pointer minus the frame size with
10994 : the stack boundary in the TCB. The stack boundary always gives
10995 : us SPLIT_STACK_AVAILABLE bytes, so if we need less than that we
10996 : can compare directly. Otherwise we need to do an addition. */
10997 :
10998 260036 : limit = ix86_split_stack_guard ();
10999 :
11000 260036 : if (allocate >= SPLIT_STACK_AVAILABLE
11001 235635 : || flag_force_indirect_call)
11002 : {
11003 24416 : scratch_regno = split_stack_prologue_scratch_regno ();
11004 24416 : if (scratch_regno == INVALID_REGNUM)
11005 0 : return;
11006 : }
11007 :
11008 260036 : if (allocate >= SPLIT_STACK_AVAILABLE)
11009 : {
11010 24401 : rtx offset;
11011 :
11012 : /* We need a scratch register to hold the stack pointer minus
11013 : the required frame size. Since this is the very start of the
11014 : function, the scratch register can be any caller-saved
11015 : register which is not used for parameters. */
11016 24401 : offset = GEN_INT (- allocate);
11017 :
11018 31243 : scratch_reg = gen_rtx_REG (Pmode, scratch_regno);
11019 24401 : if (!TARGET_64BIT || x86_64_immediate_operand (offset, Pmode))
11020 : {
11021 : /* We don't use gen_add in this case because it will
11022 : want to split to lea, but when not optimizing the insn
11023 : will not be split after this point. */
11024 31243 : emit_insn (gen_rtx_SET (scratch_reg,
11025 : gen_rtx_PLUS (Pmode, stack_pointer_rtx,
11026 : offset)));
11027 : }
11028 : else
11029 : {
11030 0 : emit_move_insn (scratch_reg, offset);
11031 0 : emit_insn (gen_add2_insn (scratch_reg, stack_pointer_rtx));
11032 : }
11033 : current = scratch_reg;
11034 : }
11035 : else
11036 235635 : current = stack_pointer_rtx;
11037 :
11038 260036 : ix86_expand_branch (GEU, current, limit, label);
11039 260036 : rtx_insn *jump_insn = get_last_insn ();
11040 260036 : JUMP_LABEL (jump_insn) = label;
11041 :
11042 : /* Mark the jump as very likely to be taken. */
11043 260036 : add_reg_br_prob_note (jump_insn, profile_probability::very_likely ());
11044 :
11045 260036 : if (split_stack_fn == NULL_RTX)
11046 : {
11047 5452 : split_stack_fn = gen_rtx_SYMBOL_REF (Pmode, "__morestack");
11048 4348 : SYMBOL_REF_FLAGS (split_stack_fn) |= SYMBOL_FLAG_LOCAL;
11049 : }
11050 260036 : fn = split_stack_fn;
11051 :
11052 : /* Get more stack space. We pass in the desired stack space and the
11053 : size of the arguments to copy to the new stack. In 32-bit mode
11054 : we push the parameters; __morestack will return on a new stack
11055 : anyhow. In 64-bit mode we pass the parameters in r10 and
11056 : r11. */
11057 260036 : allocate_rtx = GEN_INT (allocate);
11058 260036 : args_size = crtl->args.size >= 0 ? (HOST_WIDE_INT) crtl->args.size : 0;
11059 260036 : call_fusage = NULL_RTX;
11060 260036 : rtx pop = NULL_RTX;
11061 260036 : if (TARGET_64BIT)
11062 : {
11063 162035 : rtx reg10, reg11;
11064 :
11065 162035 : reg10 = gen_rtx_REG (DImode, R10_REG);
11066 162035 : reg11 = gen_rtx_REG (DImode, R11_REG);
11067 :
11068 : /* If this function uses a static chain, it will be in %r10.
11069 : Preserve it across the call to __morestack. */
11070 162035 : if (DECL_STATIC_CHAIN (cfun->decl))
11071 : {
11072 7505 : rtx rax;
11073 :
11074 7505 : rax = gen_rtx_REG (word_mode, AX_REG);
11075 7505 : emit_move_insn (rax, gen_rtx_REG (word_mode, R10_REG));
11076 7505 : use_reg (&call_fusage, rax);
11077 : }
11078 :
11079 162035 : if (flag_force_indirect_call
11080 162020 : || ix86_cmodel == CM_LARGE || ix86_cmodel == CM_LARGE_PIC)
11081 : {
11082 16 : HOST_WIDE_INT argval;
11083 :
11084 16 : if (split_stack_fn_large == NULL_RTX)
11085 : {
11086 7 : split_stack_fn_large
11087 7 : = gen_rtx_SYMBOL_REF (Pmode, "__morestack_large_model");
11088 7 : SYMBOL_REF_FLAGS (split_stack_fn_large) |= SYMBOL_FLAG_LOCAL;
11089 : }
11090 :
11091 16 : fn = split_stack_fn_large;
11092 :
11093 16 : if (ix86_cmodel == CM_LARGE_PIC)
11094 : {
11095 3 : rtx_code_label *label;
11096 3 : rtx x;
11097 :
11098 3 : gcc_assert (Pmode == DImode);
11099 :
11100 3 : label = gen_label_rtx ();
11101 3 : emit_label (label);
11102 3 : LABEL_PRESERVE_P (label) = 1;
11103 3 : emit_insn (gen_set_rip_rex64 (reg10, label));
11104 3 : emit_insn (gen_set_got_offset_rex64 (reg11, label));
11105 3 : emit_insn (gen_add2_insn (reg10, reg11));
11106 3 : x = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, fn), UNSPEC_GOT);
11107 3 : x = gen_rtx_CONST (Pmode, x);
11108 3 : emit_move_insn (reg11, x);
11109 3 : x = gen_rtx_PLUS (Pmode, reg10, reg11);
11110 3 : x = gen_const_mem (Pmode, x);
11111 3 : fn = copy_to_suggested_reg (x, reg11, Pmode);
11112 : }
11113 13 : else if (ix86_cmodel == CM_LARGE)
11114 1 : fn = copy_to_suggested_reg (fn, reg11, Pmode);
11115 :
11116 : /* When using the large model we need to load the address
11117 : into a register, and we've run out of registers. So we
11118 : switch to a different calling convention, and we call a
11119 : different function: __morestack_large. We pass the
11120 : argument size in the upper 32 bits of r10 and pass the
11121 : frame size in the lower 32 bits. */
11122 16 : gcc_assert ((allocate & HOST_WIDE_INT_C (0xffffffff)) == allocate);
11123 16 : gcc_assert ((args_size & 0xffffffff) == args_size);
11124 :
11125 16 : argval = ((args_size << 16) << 16) + allocate;
11126 16 : emit_move_insn (reg10, GEN_INT (argval));
11127 16 : }
11128 : else
11129 : {
11130 162019 : emit_move_insn (reg10, allocate_rtx);
11131 162019 : emit_move_insn (reg11, GEN_INT (args_size));
11132 162019 : use_reg (&call_fusage, reg11);
11133 : }
11134 :
11135 162035 : use_reg (&call_fusage, reg10);
11136 : }
11137 : else
11138 : {
11139 98001 : if (flag_force_indirect_call && flag_pic)
11140 : {
11141 0 : rtx x;
11142 :
11143 0 : gcc_assert (Pmode == SImode);
11144 :
11145 0 : scratch_reg = gen_rtx_REG (Pmode, scratch_regno);
11146 :
11147 0 : emit_insn (gen_set_got (scratch_reg));
11148 0 : x = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, split_stack_fn),
11149 : UNSPEC_GOT);
11150 0 : x = gen_rtx_CONST (Pmode, x);
11151 0 : x = gen_rtx_PLUS (Pmode, scratch_reg, x);
11152 0 : x = gen_const_mem (Pmode, x);
11153 0 : fn = copy_to_suggested_reg (x, scratch_reg, Pmode);
11154 : }
11155 :
11156 98001 : rtx_insn *insn = emit_insn (gen_push (GEN_INT (args_size)));
11157 196002 : add_reg_note (insn, REG_ARGS_SIZE, GEN_INT (UNITS_PER_WORD));
11158 98001 : insn = emit_insn (gen_push (allocate_rtx));
11159 196002 : add_reg_note (insn, REG_ARGS_SIZE, GEN_INT (2 * UNITS_PER_WORD));
11160 196002 : pop = GEN_INT (2 * UNITS_PER_WORD);
11161 : }
11162 :
11163 260036 : if (flag_force_indirect_call && !register_operand (fn, VOIDmode))
11164 : {
11165 12 : scratch_reg = gen_rtx_REG (word_mode, scratch_regno);
11166 :
11167 12 : if (GET_MODE (fn) != word_mode)
11168 0 : fn = gen_rtx_ZERO_EXTEND (word_mode, fn);
11169 :
11170 12 : fn = copy_to_suggested_reg (fn, scratch_reg, word_mode);
11171 : }
11172 :
11173 260036 : call_insn = ix86_expand_call (NULL_RTX, gen_rtx_MEM (QImode, fn),
11174 260036 : GEN_INT (UNITS_PER_WORD), constm1_rtx,
11175 : pop, false);
11176 260036 : add_function_usage_to (call_insn, call_fusage);
11177 260036 : if (!TARGET_64BIT)
11178 98001 : add_reg_note (call_insn, REG_ARGS_SIZE, GEN_INT (0));
11179 : /* Indicate that this function can't jump to non-local gotos. */
11180 260036 : make_reg_eh_region_note_nothrow_nononlocal (call_insn);
11181 :
11182 : /* In order to make call/return prediction work right, we now need
11183 : to execute a return instruction. See
11184 : libgcc/config/i386/morestack.S for the details on how this works.
11185 :
11186 : For flow purposes gcc must not see this as a return
11187 : instruction--we need control flow to continue at the subsequent
11188 : label. Therefore, we use an unspec. */
11189 260036 : gcc_assert (crtl->args.pops_args < 65536);
11190 260036 : rtx_insn *ret_insn
11191 260036 : = emit_insn (gen_split_stack_return (GEN_INT (crtl->args.pops_args)));
11192 :
11193 260036 : if ((flag_cf_protection & CF_BRANCH))
11194 : {
11195 : /* Insert ENDBR since __morestack will jump back here via indirect
11196 : call. */
11197 21 : rtx cet_eb = gen_nop_endbr ();
11198 21 : emit_insn_after (cet_eb, ret_insn);
11199 : }
11200 :
11201 : /* If we are in 64-bit mode and this function uses a static chain,
11202 : we saved %r10 in %rax before calling _morestack. */
11203 260036 : if (TARGET_64BIT && DECL_STATIC_CHAIN (cfun->decl))
11204 7505 : emit_move_insn (gen_rtx_REG (word_mode, R10_REG),
11205 : gen_rtx_REG (word_mode, AX_REG));
11206 :
11207 : /* If this function calls va_start, we need to store a pointer to
11208 : the arguments on the old stack, because they may not have been
11209 : all copied to the new stack. At this point the old stack can be
11210 : found at the frame pointer value used by __morestack, because
11211 : __morestack has set that up before calling back to us. Here we
11212 : store that pointer in a scratch register, and in
11213 : ix86_expand_prologue we store the scratch register in a stack
11214 : slot. */
11215 260036 : if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
11216 : {
11217 12 : rtx frame_reg;
11218 12 : int words;
11219 :
11220 12 : scratch_regno = split_stack_prologue_scratch_regno ();
11221 16 : scratch_reg = gen_rtx_REG (Pmode, scratch_regno);
11222 16 : frame_reg = gen_rtx_REG (Pmode, BP_REG);
11223 :
11224 : /* 64-bit:
11225 : fp -> old fp value
11226 : return address within this function
11227 : return address of caller of this function
11228 : stack arguments
11229 : So we add three words to get to the stack arguments.
11230 :
11231 : 32-bit:
11232 : fp -> old fp value
11233 : return address within this function
11234 : first argument to __morestack
11235 : second argument to __morestack
11236 : return address of caller of this function
11237 : stack arguments
11238 : So we add five words to get to the stack arguments.
11239 : */
11240 12 : words = TARGET_64BIT ? 3 : 5;
11241 20 : emit_insn (gen_rtx_SET (scratch_reg,
11242 : plus_constant (Pmode, frame_reg,
11243 : words * UNITS_PER_WORD)));
11244 :
11245 12 : varargs_label = gen_label_rtx ();
11246 12 : emit_jump_insn (gen_jump (varargs_label));
11247 12 : JUMP_LABEL (get_last_insn ()) = varargs_label;
11248 :
11249 12 : emit_barrier ();
11250 : }
11251 :
11252 260036 : emit_label (label);
11253 260036 : LABEL_NUSES (label) = 1;
11254 :
11255 : /* If this function calls va_start, we now have to set the scratch
11256 : register for the case where we do not call __morestack. In this
11257 : case we need to set it based on the stack pointer. */
11258 260036 : if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
11259 : {
11260 20 : emit_insn (gen_rtx_SET (scratch_reg,
11261 : plus_constant (Pmode, stack_pointer_rtx,
11262 : UNITS_PER_WORD)));
11263 :
11264 12 : emit_label (varargs_label);
11265 12 : LABEL_NUSES (varargs_label) = 1;
11266 : }
11267 : }
11268 :
11269 : /* We may have to tell the dataflow pass that the split stack prologue
11270 : is initializing a scratch register. */
11271 :
11272 : static void
11273 15992731 : ix86_live_on_entry (bitmap regs)
11274 : {
11275 15992731 : if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
11276 : {
11277 124 : gcc_assert (flag_split_stack);
11278 124 : bitmap_set_bit (regs, split_stack_prologue_scratch_regno ());
11279 : }
11280 15992731 : }
11281 :
11282 : /* Extract the parts of an RTL expression that is a valid memory address
11283 : for an instruction. Return false if the structure of the address is
11284 : grossly off. */
11285 :
11286 : bool
11287 4291060027 : ix86_decompose_address (rtx addr, struct ix86_address *out)
11288 : {
11289 4291060027 : rtx base = NULL_RTX, index = NULL_RTX, disp = NULL_RTX;
11290 4291060027 : rtx base_reg, index_reg;
11291 4291060027 : HOST_WIDE_INT scale = 1;
11292 4291060027 : rtx scale_rtx = NULL_RTX;
11293 4291060027 : rtx tmp;
11294 4291060027 : addr_space_t seg = ADDR_SPACE_GENERIC;
11295 :
11296 : /* Allow zero-extended SImode addresses,
11297 : they will be emitted with addr32 prefix. */
11298 4291060027 : if (TARGET_64BIT && GET_MODE (addr) == DImode)
11299 : {
11300 2268276949 : if (GET_CODE (addr) == ZERO_EXTEND
11301 2034053 : && GET_MODE (XEXP (addr, 0)) == SImode)
11302 : {
11303 1930183 : addr = XEXP (addr, 0);
11304 1930183 : if (CONST_INT_P (addr))
11305 : return false;
11306 : }
11307 2266346766 : else if (GET_CODE (addr) == AND)
11308 : {
11309 2867795 : rtx mask = XEXP (addr, 1);
11310 2867795 : rtx shift_val;
11311 :
11312 2867795 : if (const_32bit_mask (mask, DImode)
11313 : /* For ASHIFT inside AND, combine will not generate
11314 : canonical zero-extend. Merge mask for AND and shift_count
11315 : to check if it is canonical zero-extend. */
11316 2867795 : || (CONST_INT_P (mask)
11317 1879200 : && GET_CODE (XEXP (addr, 0)) == ASHIFT
11318 154118 : && CONST_INT_P (shift_val = XEXP (XEXP (addr, 0), 1))
11319 151015 : && ((UINTVAL (mask)
11320 151015 : | ((HOST_WIDE_INT_1U << INTVAL (shift_val)) - 1))
11321 : == HOST_WIDE_INT_UC (0xffffffff))))
11322 : {
11323 82597 : addr = lowpart_subreg (SImode, XEXP (addr, 0), DImode);
11324 82597 : if (addr == NULL_RTX)
11325 : return false;
11326 :
11327 82597 : if (CONST_INT_P (addr))
11328 : return false;
11329 : }
11330 : }
11331 : }
11332 :
11333 : /* Allow SImode subregs of DImode addresses,
11334 : they will be emitted with addr32 prefix. */
11335 4291060027 : if (TARGET_64BIT && GET_MODE (addr) == SImode)
11336 : {
11337 16599275 : if (SUBREG_P (addr)
11338 244503 : && GET_MODE (SUBREG_REG (addr)) == DImode)
11339 : {
11340 214263 : addr = SUBREG_REG (addr);
11341 214263 : if (CONST_INT_P (addr))
11342 : return false;
11343 : }
11344 : }
11345 :
11346 4291060027 : if (REG_P (addr))
11347 : base = addr;
11348 : else if (SUBREG_P (addr))
11349 : {
11350 466580 : if (REG_P (SUBREG_REG (addr)))
11351 : base = addr;
11352 : else
11353 : return false;
11354 : }
11355 : else if (GET_CODE (addr) == PLUS)
11356 : {
11357 : rtx addends[4], op;
11358 : int n = 0, i;
11359 :
11360 : op = addr;
11361 3118442007 : do
11362 : {
11363 3118442007 : if (n >= 4)
11364 641357740 : return false;
11365 3118436125 : addends[n++] = XEXP (op, 1);
11366 3118436125 : op = XEXP (op, 0);
11367 : }
11368 3118436125 : while (GET_CODE (op) == PLUS);
11369 3058335410 : if (n >= 4)
11370 : return false;
11371 3058327349 : addends[n] = op;
11372 :
11373 7955665846 : for (i = n; i >= 0; --i)
11374 : {
11375 5538682294 : op = addends[i];
11376 5538682294 : switch (GET_CODE (op))
11377 : {
11378 60584732 : case MULT:
11379 60584732 : if (index)
11380 : return false;
11381 60541646 : index = XEXP (op, 0);
11382 60541646 : scale_rtx = XEXP (op, 1);
11383 60541646 : break;
11384 :
11385 12806508 : case ASHIFT:
11386 12806508 : if (index)
11387 : return false;
11388 12733028 : index = XEXP (op, 0);
11389 12733028 : tmp = XEXP (op, 1);
11390 12733028 : if (!CONST_INT_P (tmp))
11391 : return false;
11392 12718347 : scale = INTVAL (tmp);
11393 12718347 : if ((unsigned HOST_WIDE_INT) scale > 3)
11394 : return false;
11395 12293373 : scale = 1 << scale;
11396 12293373 : break;
11397 :
11398 808578 : case ZERO_EXTEND:
11399 808578 : op = XEXP (op, 0);
11400 808578 : if (GET_CODE (op) != UNSPEC)
11401 : return false;
11402 : /* FALLTHRU */
11403 :
11404 701624 : case UNSPEC:
11405 701624 : if (XINT (op, 1) == UNSPEC_TP
11406 693281 : && TARGET_TLS_DIRECT_SEG_REFS
11407 693281 : && seg == ADDR_SPACE_GENERIC)
11408 693281 : seg = DEFAULT_TLS_SEG_REG;
11409 : else
11410 : return false;
11411 : break;
11412 :
11413 653019 : case SUBREG:
11414 653019 : if (!REG_P (SUBREG_REG (op)))
11415 : return false;
11416 : /* FALLTHRU */
11417 :
11418 2482932713 : case REG:
11419 2482932713 : if (!base)
11420 : base = op;
11421 77567433 : else if (!index)
11422 : index = op;
11423 : else
11424 : return false;
11425 : break;
11426 :
11427 2341728529 : case CONST:
11428 2341728529 : case CONST_INT:
11429 2341728529 : case SYMBOL_REF:
11430 2341728529 : case LABEL_REF:
11431 2341728529 : if (disp)
11432 : return false;
11433 : disp = op;
11434 : break;
11435 :
11436 : default:
11437 : return false;
11438 : }
11439 : }
11440 : }
11441 : else if (GET_CODE (addr) == MULT)
11442 : {
11443 3927843 : index = XEXP (addr, 0); /* index*scale */
11444 3927843 : scale_rtx = XEXP (addr, 1);
11445 : }
11446 : else if (GET_CODE (addr) == ASHIFT)
11447 : {
11448 : /* We're called for lea too, which implements ashift on occasion. */
11449 3326203 : index = XEXP (addr, 0);
11450 3326203 : tmp = XEXP (addr, 1);
11451 3326203 : if (!CONST_INT_P (tmp))
11452 : return false;
11453 2930494 : scale = INTVAL (tmp);
11454 2930494 : if ((unsigned HOST_WIDE_INT) scale > 3)
11455 : return false;
11456 2158566 : scale = 1 << scale;
11457 : }
11458 : else
11459 : disp = addr; /* displacement */
11460 :
11461 2423069961 : if (index)
11462 : {
11463 146763179 : if (REG_P (index))
11464 : ;
11465 4259930 : else if (SUBREG_P (index)
11466 448628 : && REG_P (SUBREG_REG (index)))
11467 : ;
11468 : else
11469 : return false;
11470 : }
11471 :
11472 : /* Extract the integral value of scale. */
11473 3644645405 : if (scale_rtx)
11474 : {
11475 55790133 : if (!CONST_INT_P (scale_rtx))
11476 : return false;
11477 55135434 : scale = INTVAL (scale_rtx);
11478 : }
11479 :
11480 3643990706 : base_reg = base && SUBREG_P (base) ? SUBREG_REG (base) : base;
11481 3643990706 : index_reg = index && SUBREG_P (index) ? SUBREG_REG (index) : index;
11482 :
11483 : /* Avoid useless 0 displacement. */
11484 3643990706 : if (disp == const0_rtx && (base || index))
11485 3643990706 : disp = NULL_RTX;
11486 :
11487 : /* Allow arg pointer and stack pointer as index if there is not scaling. */
11488 2636400931 : if (base_reg && index_reg && scale == 1
11489 3720028617 : && (REGNO (index_reg) == ARG_POINTER_REGNUM
11490 : || REGNO (index_reg) == FRAME_POINTER_REGNUM
11491 : || REGNO (index_reg) == SP_REG))
11492 : {
11493 : std::swap (base, index);
11494 : std::swap (base_reg, index_reg);
11495 : }
11496 :
11497 : /* Special case: rewrite index*1+disp into base+disp. */
11498 3643990706 : if (!base && index && scale == 1)
11499 4 : base = index, base_reg = index_reg, index = index_reg = NULL_RTX;
11500 :
11501 : /* Special case: %ebp cannot be encoded as a base without a displacement.
11502 : Similarly %r13. */
11503 313157762 : if (!disp && base_reg
11504 3952672433 : && (REGNO (base_reg) == ARG_POINTER_REGNUM
11505 : || REGNO (base_reg) == FRAME_POINTER_REGNUM
11506 : || REGNO (base_reg) == BP_REG
11507 : || REGNO (base_reg) == R13_REG))
11508 : disp = const0_rtx;
11509 :
11510 : /* Special case: on K6, [%esi] makes the instruction vector decoded.
11511 : Avoid this by transforming to [%esi+0].
11512 : Reload calls address legitimization without cfun defined, so we need
11513 : to test cfun for being non-NULL. */
11514 0 : if (TARGET_CPU_P (K6) && cfun && optimize_function_for_speed_p (cfun)
11515 0 : && base_reg && !index_reg && !disp
11516 3643990706 : && REGNO (base_reg) == SI_REG)
11517 0 : disp = const0_rtx;
11518 :
11519 : /* Special case: encode reg+reg instead of reg*2. */
11520 3643990706 : if (!base && index && scale == 2)
11521 1007589771 : base = index, base_reg = index_reg, scale = 1;
11522 :
11523 : /* Special case: scaling cannot be encoded without base or displacement. */
11524 1007589771 : if (!base && !disp && index && scale != 1)
11525 3567636 : disp = const0_rtx;
11526 :
11527 3643990706 : out->base = base;
11528 3643990706 : out->index = index;
11529 3643990706 : out->disp = disp;
11530 3643990706 : out->scale = scale;
11531 3643990706 : out->seg = seg;
11532 :
11533 3643990706 : return true;
11534 : }
11535 :
11536 : /* Return cost of the memory address x.
11537 : For i386, it is better to use a complex address than let gcc copy
11538 : the address into a reg and make a new pseudo. But not if the address
11539 : requires to two regs - that would mean more pseudos with longer
11540 : lifetimes. */
11541 : static int
11542 10661678 : ix86_address_cost (rtx x, machine_mode, addr_space_t, bool)
11543 : {
11544 10661678 : struct ix86_address parts;
11545 10661678 : int cost = 1;
11546 10661678 : int ok = ix86_decompose_address (x, &parts);
11547 :
11548 10661678 : gcc_assert (ok);
11549 :
11550 10661678 : if (parts.base && SUBREG_P (parts.base))
11551 243 : parts.base = SUBREG_REG (parts.base);
11552 10661678 : if (parts.index && SUBREG_P (parts.index))
11553 52 : parts.index = SUBREG_REG (parts.index);
11554 :
11555 : /* Attempt to minimize number of registers in the address by increasing
11556 : address cost for each used register. We don't increase address cost
11557 : for "pic_offset_table_rtx". When a memopt with "pic_offset_table_rtx"
11558 : is not invariant itself it most likely means that base or index is not
11559 : invariant. Therefore only "pic_offset_table_rtx" could be hoisted out,
11560 : which is not profitable for x86. */
11561 10661678 : if (parts.base
11562 9243052 : && (!REG_P (parts.base) || REGNO (parts.base) >= FIRST_PSEUDO_REGISTER)
11563 19595494 : && (current_pass->type == GIMPLE_PASS
11564 2653461 : || !pic_offset_table_rtx
11565 126987 : || !REG_P (parts.base)
11566 126987 : || REGNO (pic_offset_table_rtx) != REGNO (parts.base)))
11567 : cost++;
11568 :
11569 10661678 : if (parts.index
11570 5136232 : && (!REG_P (parts.index) || REGNO (parts.index) >= FIRST_PSEUDO_REGISTER)
11571 15786916 : && (current_pass->type == GIMPLE_PASS
11572 659937 : || !pic_offset_table_rtx
11573 55102 : || !REG_P (parts.index)
11574 55102 : || REGNO (pic_offset_table_rtx) != REGNO (parts.index)))
11575 5123954 : cost++;
11576 :
11577 : /* AMD-K6 don't like addresses with ModR/M set to 00_xxx_100b,
11578 : since it's predecode logic can't detect the length of instructions
11579 : and it degenerates to vector decoded. Increase cost of such
11580 : addresses here. The penalty is minimally 2 cycles. It may be worthwhile
11581 : to split such addresses or even refuse such addresses at all.
11582 :
11583 : Following addressing modes are affected:
11584 : [base+scale*index]
11585 : [scale*index+disp]
11586 : [base+index]
11587 :
11588 : The first and last case may be avoidable by explicitly coding the zero in
11589 : memory address, but I don't have AMD-K6 machine handy to check this
11590 : theory. */
11591 :
11592 10661678 : if (TARGET_CPU_P (K6)
11593 0 : && ((!parts.disp && parts.base && parts.index && parts.scale != 1)
11594 0 : || (parts.disp && !parts.base && parts.index && parts.scale != 1)
11595 0 : || (!parts.disp && parts.base && parts.index && parts.scale == 1)))
11596 0 : cost += 10;
11597 :
11598 10661678 : return cost;
11599 : }
11600 :
11601 : /* Implement TARGET_USE_BY_PIECES_INFRASTRUCTURE_P. */
11602 :
11603 : bool
11604 1199549 : ix86_use_by_pieces_infrastructure_p (unsigned HOST_WIDE_INT size,
11605 : unsigned int align,
11606 : enum by_pieces_operation op,
11607 : bool speed_p)
11608 : {
11609 : /* Return true when we are currently expanding memcpy/memset epilogue
11610 : with move_by_pieces or store_by_pieces. */
11611 1199549 : if (cfun->machine->by_pieces_in_use)
11612 : return true;
11613 :
11614 1197464 : return default_use_by_pieces_infrastructure_p (size, align, op,
11615 1197464 : speed_p);
11616 : }
11617 :
11618 : /* Allow {LABEL | SYMBOL}_REF - SYMBOL_REF-FOR-PICBASE for Mach-O as
11619 : this is used for to form addresses to local data when -fPIC is in
11620 : use. */
11621 :
11622 : static bool
11623 0 : darwin_local_data_pic (rtx disp)
11624 : {
11625 0 : return (GET_CODE (disp) == UNSPEC
11626 0 : && XINT (disp, 1) == UNSPEC_MACHOPIC_OFFSET);
11627 : }
11628 :
11629 : /* True if the function symbol operand X should be loaded from GOT.
11630 : If CALL_P is true, X is a call operand.
11631 :
11632 : NB: -mno-direct-extern-access doesn't force load from GOT for
11633 : call.
11634 :
11635 : NB: In 32-bit mode, only non-PIC is allowed in inline assembly
11636 : statements, since a PIC register could not be available at the
11637 : call site. */
11638 :
11639 : bool
11640 1867791034 : ix86_force_load_from_GOT_p (rtx x, bool call_p)
11641 : {
11642 95856895 : return ((TARGET_64BIT || (!flag_pic && HAVE_AS_IX86_GOT32X))
11643 : && !TARGET_PECOFF && !TARGET_MACHO
11644 1864929383 : && (!flag_pic || this_is_asm_operands)
11645 1844486501 : && ix86_cmodel != CM_LARGE
11646 1844480502 : && ix86_cmodel != CM_LARGE_PIC
11647 1844480501 : && SYMBOL_REF_P (x)
11648 1844480499 : && ((!call_p
11649 1838976418 : && (!ix86_direct_extern_access
11650 1838974148 : || (SYMBOL_REF_DECL (x)
11651 1658096085 : && lookup_attribute ("nodirect_extern_access",
11652 1658096085 : DECL_ATTRIBUTES (SYMBOL_REF_DECL (x))))))
11653 1844477775 : || (SYMBOL_REF_FUNCTION_P (x)
11654 698951531 : && (!flag_plt
11655 698947120 : || (SYMBOL_REF_DECL (x)
11656 698947120 : && lookup_attribute ("noplt",
11657 698947120 : DECL_ATTRIBUTES (SYMBOL_REF_DECL (x)))))))
11658 1867798568 : && !SYMBOL_REF_LOCAL_P (x));
11659 : }
11660 :
11661 : /* Determine if a given RTX is a valid constant. We already know this
11662 : satisfies CONSTANT_P. */
11663 :
11664 : static bool
11665 1565773507 : ix86_legitimate_constant_p (machine_mode mode, rtx x)
11666 : {
11667 1565773507 : switch (GET_CODE (x))
11668 : {
11669 137039131 : case CONST:
11670 137039131 : x = XEXP (x, 0);
11671 :
11672 137039131 : if (GET_CODE (x) == PLUS)
11673 : {
11674 136921063 : if (!CONST_INT_P (XEXP (x, 1)))
11675 : return false;
11676 136921063 : x = XEXP (x, 0);
11677 : }
11678 :
11679 137039131 : if (TARGET_MACHO && darwin_local_data_pic (x))
11680 : return true;
11681 :
11682 : /* Only some unspecs are valid as "constants". */
11683 137039131 : if (GET_CODE (x) == UNSPEC)
11684 501692 : switch (XINT (x, 1))
11685 : {
11686 21131 : case UNSPEC_GOT:
11687 21131 : case UNSPEC_GOTOFF:
11688 21131 : case UNSPEC_PLTOFF:
11689 21131 : return TARGET_64BIT;
11690 480198 : case UNSPEC_TPOFF:
11691 480198 : case UNSPEC_NTPOFF:
11692 480198 : x = XVECEXP (x, 0, 0);
11693 480198 : return (SYMBOL_REF_P (x)
11694 480198 : && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
11695 275 : case UNSPEC_DTPOFF:
11696 275 : x = XVECEXP (x, 0, 0);
11697 275 : return (SYMBOL_REF_P (x)
11698 275 : && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_DYNAMIC);
11699 0 : case UNSPEC_SECREL32:
11700 0 : x = XVECEXP (x, 0, 0);
11701 0 : return SYMBOL_REF_P (x);
11702 : default:
11703 : return false;
11704 : }
11705 :
11706 : /* We must have drilled down to a symbol. */
11707 136537439 : if (LABEL_REF_P (x))
11708 : return true;
11709 136531883 : if (!SYMBOL_REF_P (x))
11710 : return false;
11711 : /* FALLTHRU */
11712 :
11713 935482624 : case SYMBOL_REF:
11714 : /* TLS symbols are never valid. */
11715 935482624 : if (SYMBOL_REF_TLS_MODEL (x))
11716 : return false;
11717 :
11718 : /* DLLIMPORT symbols are never valid. */
11719 935377784 : if (TARGET_DLLIMPORT_DECL_ATTRIBUTES
11720 : && SYMBOL_REF_DLLIMPORT_P (x))
11721 : return false;
11722 :
11723 : #if TARGET_MACHO
11724 : /* mdynamic-no-pic */
11725 : if (MACHO_DYNAMIC_NO_PIC_P)
11726 : return machopic_symbol_defined_p (x);
11727 : #endif
11728 :
11729 : /* External function address should be loaded
11730 : via the GOT slot to avoid PLT. */
11731 935377784 : if (ix86_force_load_from_GOT_p (x))
11732 : return false;
11733 :
11734 : break;
11735 :
11736 608476026 : CASE_CONST_SCALAR_INT:
11737 608476026 : if (ix86_endbr_immediate_operand (x, VOIDmode))
11738 : return false;
11739 :
11740 608475825 : switch (mode)
11741 : {
11742 1479745 : case E_TImode:
11743 1479745 : if (TARGET_64BIT)
11744 : return true;
11745 : /* FALLTHRU */
11746 26049 : case E_OImode:
11747 26049 : case E_XImode:
11748 26049 : if (!standard_sse_constant_p (x, mode)
11749 43326 : && GET_MODE_SIZE (TARGET_AVX512F
11750 : ? XImode
11751 : : (TARGET_AVX
11752 : ? OImode
11753 : : (TARGET_SSE2
11754 17277 : ? TImode : DImode))) < GET_MODE_SIZE (mode))
11755 : return false;
11756 : default:
11757 : break;
11758 : }
11759 : break;
11760 :
11761 8926891 : case CONST_VECTOR:
11762 8926891 : if (!standard_sse_constant_p (x, mode))
11763 : return false;
11764 : break;
11765 :
11766 7689180 : case CONST_DOUBLE:
11767 7689180 : if (mode == E_BFmode)
11768 : return false;
11769 :
11770 : default:
11771 : break;
11772 : }
11773 :
11774 : /* Otherwise we handle everything else in the move patterns. */
11775 : return true;
11776 : }
11777 :
11778 : /* Determine if it's legal to put X into the constant pool. This
11779 : is not possible for the address of thread-local symbols, which
11780 : is checked above. */
11781 :
11782 : static bool
11783 62270956 : ix86_cannot_force_const_mem (machine_mode mode, rtx x)
11784 : {
11785 : /* We can put any immediate constant in memory. */
11786 62270956 : switch (GET_CODE (x))
11787 : {
11788 : CASE_CONST_ANY:
11789 : return false;
11790 :
11791 1812676 : default:
11792 1812676 : break;
11793 : }
11794 :
11795 1812676 : return !ix86_legitimate_constant_p (mode, x);
11796 : }
11797 :
11798 : /* Return a unique alias set for the GOT. */
11799 :
11800 : alias_set_type
11801 189424 : ix86_GOT_alias_set (void)
11802 : {
11803 189424 : static alias_set_type set = -1;
11804 189424 : if (set == -1)
11805 2957 : set = new_alias_set ();
11806 189424 : return set;
11807 : }
11808 :
11809 : /* Nonzero if the constant value X is a legitimate general operand
11810 : when generating PIC code. It is given that flag_pic is on and
11811 : that X satisfies CONSTANT_P. */
11812 :
11813 : bool
11814 125876945 : legitimate_pic_operand_p (rtx x)
11815 : {
11816 125876945 : rtx inner;
11817 :
11818 125876945 : switch (GET_CODE (x))
11819 : {
11820 2505530 : case CONST:
11821 2505530 : inner = XEXP (x, 0);
11822 2505530 : if (GET_CODE (inner) == PLUS
11823 357946 : && CONST_INT_P (XEXP (inner, 1)))
11824 357946 : inner = XEXP (inner, 0);
11825 :
11826 : /* Only some unspecs are valid as "constants". */
11827 2505530 : if (GET_CODE (inner) == UNSPEC)
11828 2255545 : switch (XINT (inner, 1))
11829 : {
11830 2194664 : case UNSPEC_GOT:
11831 2194664 : case UNSPEC_GOTOFF:
11832 2194664 : case UNSPEC_PLTOFF:
11833 2194664 : return TARGET_64BIT;
11834 0 : case UNSPEC_TPOFF:
11835 0 : x = XVECEXP (inner, 0, 0);
11836 0 : return (SYMBOL_REF_P (x)
11837 0 : && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
11838 0 : case UNSPEC_SECREL32:
11839 0 : x = XVECEXP (inner, 0, 0);
11840 0 : return SYMBOL_REF_P (x);
11841 0 : case UNSPEC_MACHOPIC_OFFSET:
11842 0 : return legitimate_pic_address_disp_p (x);
11843 : default:
11844 : return false;
11845 : }
11846 : /* FALLTHRU */
11847 :
11848 7052339 : case SYMBOL_REF:
11849 7052339 : case LABEL_REF:
11850 7052339 : return legitimate_pic_address_disp_p (x);
11851 :
11852 : default:
11853 : return true;
11854 : }
11855 : }
11856 :
11857 : /* Determine if a given CONST RTX is a valid memory displacement
11858 : in PIC mode. */
11859 :
11860 : bool
11861 65025224 : legitimate_pic_address_disp_p (rtx disp)
11862 : {
11863 65025224 : bool saw_plus;
11864 :
11865 : /* In 64bit mode we can allow direct addresses of symbols and labels
11866 : when they are not dynamic symbols. */
11867 65025224 : if (TARGET_64BIT)
11868 : {
11869 39831706 : rtx op0 = disp, op1;
11870 :
11871 39831706 : switch (GET_CODE (disp))
11872 : {
11873 : case LABEL_REF:
11874 : return true;
11875 :
11876 10955833 : case CONST:
11877 10955833 : if (GET_CODE (XEXP (disp, 0)) != PLUS)
11878 : break;
11879 1176685 : op0 = XEXP (XEXP (disp, 0), 0);
11880 1176685 : op1 = XEXP (XEXP (disp, 0), 1);
11881 1176685 : if (!CONST_INT_P (op1))
11882 : break;
11883 1176685 : if (GET_CODE (op0) == UNSPEC
11884 296 : && (XINT (op0, 1) == UNSPEC_DTPOFF
11885 296 : || XINT (op0, 1) == UNSPEC_NTPOFF)
11886 1176981 : && trunc_int_for_mode (INTVAL (op1), SImode) == INTVAL (op1))
11887 : return true;
11888 1176389 : if (INTVAL (op1) >= 16*1024*1024
11889 1176389 : || INTVAL (op1) < -16*1024*1024)
11890 : break;
11891 1176301 : if (LABEL_REF_P (op0))
11892 : return true;
11893 1176301 : if (GET_CODE (op0) == CONST
11894 0 : && GET_CODE (XEXP (op0, 0)) == UNSPEC
11895 0 : && XINT (XEXP (op0, 0), 1) == UNSPEC_PCREL)
11896 : return true;
11897 1176301 : if (GET_CODE (op0) == UNSPEC
11898 0 : && XINT (op0, 1) == UNSPEC_PCREL)
11899 : return true;
11900 1176301 : if (!SYMBOL_REF_P (op0))
11901 : break;
11902 : /* FALLTHRU */
11903 :
11904 29829932 : case SYMBOL_REF:
11905 : /* TLS references should always be enclosed in UNSPEC.
11906 : The dllimported symbol needs always to be resolved. */
11907 29829932 : if (SYMBOL_REF_TLS_MODEL (op0)
11908 : || (TARGET_DLLIMPORT_DECL_ATTRIBUTES && SYMBOL_REF_DLLIMPORT_P (op0)))
11909 : return false;
11910 :
11911 29673910 : if (TARGET_PECOFF)
11912 : {
11913 : #if TARGET_PECOFF
11914 : if (is_imported_p (op0))
11915 : return true;
11916 : #endif
11917 :
11918 : if (SYMBOL_REF_FAR_ADDR_P (op0) || !SYMBOL_REF_LOCAL_P (op0))
11919 : break;
11920 :
11921 : /* Non-external-weak function symbols need to be resolved only
11922 : for the large model. Non-external symbols don't need to be
11923 : resolved for large and medium models. For the small model,
11924 : we don't need to resolve anything here. */
11925 : if ((ix86_cmodel != CM_LARGE_PIC
11926 : && SYMBOL_REF_FUNCTION_P (op0)
11927 : && !(SYMBOL_REF_EXTERNAL_P (op0) && SYMBOL_REF_WEAK (op0)))
11928 : || !SYMBOL_REF_EXTERNAL_P (op0)
11929 : || ix86_cmodel == CM_SMALL_PIC)
11930 : return true;
11931 : }
11932 29673910 : else if (!SYMBOL_REF_FAR_ADDR_P (op0)
11933 29673906 : && (SYMBOL_REF_LOCAL_P (op0)
11934 17958203 : || ((ix86_direct_extern_access
11935 35744360 : && !(SYMBOL_REF_DECL (op0)
11936 17786320 : && lookup_attribute ("nodirect_extern_access",
11937 17786320 : DECL_ATTRIBUTES (SYMBOL_REF_DECL (op0)))))
11938 : && HAVE_LD_PIE_COPYRELOC
11939 17957877 : && flag_pie
11940 34868 : && !SYMBOL_REF_WEAK (op0)
11941 34480 : && !SYMBOL_REF_FUNCTION_P (op0)))
11942 41394166 : && ix86_cmodel != CM_LARGE_PIC)
11943 : return true;
11944 : break;
11945 :
11946 : default:
11947 : break;
11948 : }
11949 : }
11950 52930288 : if (GET_CODE (disp) != CONST)
11951 : return false;
11952 14979355 : disp = XEXP (disp, 0);
11953 :
11954 14979355 : if (TARGET_64BIT)
11955 : {
11956 : /* We are unsafe to allow PLUS expressions. This limit allowed distance
11957 : of GOT tables. We should not need these anyway. */
11958 9831668 : if (GET_CODE (disp) != UNSPEC
11959 9779148 : || (XINT (disp, 1) != UNSPEC_GOTPCREL
11960 9779148 : && XINT (disp, 1) != UNSPEC_GOTOFF
11961 : && XINT (disp, 1) != UNSPEC_PCREL
11962 : && XINT (disp, 1) != UNSPEC_PLTOFF))
11963 : return false;
11964 :
11965 9779148 : if (!SYMBOL_REF_P (XVECEXP (disp, 0, 0))
11966 9779148 : && !LABEL_REF_P (XVECEXP (disp, 0, 0)))
11967 : return false;
11968 : return true;
11969 : }
11970 :
11971 5147687 : saw_plus = false;
11972 5147687 : if (GET_CODE (disp) == PLUS)
11973 : {
11974 589622 : if (!CONST_INT_P (XEXP (disp, 1)))
11975 : return false;
11976 589622 : disp = XEXP (disp, 0);
11977 589622 : saw_plus = true;
11978 : }
11979 :
11980 5147687 : if (TARGET_MACHO && darwin_local_data_pic (disp))
11981 : return true;
11982 :
11983 5147687 : if (GET_CODE (disp) != UNSPEC)
11984 : return false;
11985 :
11986 4981983 : switch (XINT (disp, 1))
11987 : {
11988 2265168 : case UNSPEC_GOT:
11989 2265168 : if (saw_plus)
11990 : return false;
11991 : /* We need to check for both symbols and labels because VxWorks loads
11992 : text labels with @GOT rather than @GOTOFF. See gotoff_operand for
11993 : details. */
11994 2265167 : return (SYMBOL_REF_P (XVECEXP (disp, 0, 0))
11995 2265167 : || LABEL_REF_P (XVECEXP (disp, 0, 0)));
11996 2716815 : case UNSPEC_GOTOFF:
11997 : /* Refuse GOTOFF in 64bit mode since it is always 64bit when used.
11998 : While ABI specify also 32bit relocation but we don't produce it in
11999 : small PIC model at all. */
12000 2716815 : if ((SYMBOL_REF_P (XVECEXP (disp, 0, 0))
12001 2716815 : || LABEL_REF_P (XVECEXP (disp, 0, 0)))
12002 : && !TARGET_64BIT)
12003 5433630 : return !TARGET_PECOFF && gotoff_operand (XVECEXP (disp, 0, 0), Pmode);
12004 : return false;
12005 0 : case UNSPEC_GOTTPOFF:
12006 0 : case UNSPEC_GOTNTPOFF:
12007 0 : case UNSPEC_INDNTPOFF:
12008 0 : if (saw_plus)
12009 : return false;
12010 0 : disp = XVECEXP (disp, 0, 0);
12011 0 : return (SYMBOL_REF_P (disp)
12012 0 : && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_INITIAL_EXEC);
12013 0 : case UNSPEC_NTPOFF:
12014 0 : disp = XVECEXP (disp, 0, 0);
12015 0 : return (SYMBOL_REF_P (disp)
12016 0 : && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_EXEC);
12017 0 : case UNSPEC_DTPOFF:
12018 0 : disp = XVECEXP (disp, 0, 0);
12019 0 : return (SYMBOL_REF_P (disp)
12020 0 : && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_DYNAMIC);
12021 0 : case UNSPEC_SECREL32:
12022 0 : disp = XVECEXP (disp, 0, 0);
12023 0 : return SYMBOL_REF_P (disp);
12024 : }
12025 :
12026 : return false;
12027 : }
12028 :
12029 : /* Determine if op is suitable RTX for an address register.
12030 : Return naked register if a register or a register subreg is
12031 : found, otherwise return NULL_RTX. */
12032 :
12033 : static rtx
12034 1312818750 : ix86_validate_address_register (rtx op)
12035 : {
12036 1312818750 : machine_mode mode = GET_MODE (op);
12037 :
12038 : /* Only SImode or DImode registers can form the address. */
12039 1312818750 : if (mode != SImode && mode != DImode)
12040 : return NULL_RTX;
12041 :
12042 1312811579 : if (REG_P (op))
12043 : return op;
12044 864253 : else if (SUBREG_P (op))
12045 : {
12046 864253 : rtx reg = SUBREG_REG (op);
12047 :
12048 864253 : if (!REG_P (reg))
12049 : return NULL_RTX;
12050 :
12051 864253 : mode = GET_MODE (reg);
12052 :
12053 : /* Don't allow SUBREGs that span more than a word. It can
12054 : lead to spill failures when the register is one word out
12055 : of a two word structure. */
12056 1773913 : if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
12057 : return NULL_RTX;
12058 :
12059 : /* Allow only SUBREGs of non-eliminable hard registers. */
12060 396832 : if (register_no_elim_operand (reg, mode))
12061 : return reg;
12062 : }
12063 :
12064 : /* Op is not a register. */
12065 : return NULL_RTX;
12066 : }
12067 :
12068 : /* Determine which memory address register set insn can use. */
12069 :
12070 : static enum attr_addr
12071 255851228 : ix86_memory_address_reg_class (rtx_insn* insn)
12072 : {
12073 : /* LRA can do some initialization with NULL insn,
12074 : return maximum register class in this case. */
12075 255851228 : enum attr_addr addr_rclass = ADDR_GPR32;
12076 :
12077 255851228 : if (!insn)
12078 : return addr_rclass;
12079 :
12080 73215603 : if (asm_noperands (PATTERN (insn)) >= 0
12081 73215603 : || GET_CODE (PATTERN (insn)) == ASM_INPUT)
12082 87652 : return ix86_apx_inline_asm_use_gpr32 ? ADDR_GPR32 : ADDR_GPR16;
12083 :
12084 : /* Return maximum register class for unrecognized instructions. */
12085 73171777 : if (INSN_CODE (insn) < 0)
12086 : return addr_rclass;
12087 :
12088 : /* Try to recognize the insn before calling get_attr_addr.
12089 : Save current recog_data and current alternative. */
12090 73171777 : struct recog_data_d saved_recog_data = recog_data;
12091 73171777 : int saved_alternative = which_alternative;
12092 :
12093 : /* Update recog_data for processing of alternatives. */
12094 73171777 : extract_insn_cached (insn);
12095 :
12096 : /* If current alternative is not set, loop through enabled
12097 : alternatives and get the most limited register class. */
12098 73171777 : if (saved_alternative == -1)
12099 : {
12100 73171777 : alternative_mask enabled = get_enabled_alternatives (insn);
12101 :
12102 1266198772 : for (int i = 0; i < recog_data.n_alternatives; i++)
12103 : {
12104 1193026995 : if (!TEST_BIT (enabled, i))
12105 354664342 : continue;
12106 :
12107 838362653 : which_alternative = i;
12108 838362653 : addr_rclass = MIN (addr_rclass, get_attr_addr (insn));
12109 : }
12110 : }
12111 : else
12112 : {
12113 0 : which_alternative = saved_alternative;
12114 0 : addr_rclass = get_attr_addr (insn);
12115 : }
12116 :
12117 73171777 : recog_data = saved_recog_data;
12118 73171777 : which_alternative = saved_alternative;
12119 :
12120 73171777 : return addr_rclass;
12121 : }
12122 :
12123 : /* Return memory address register class insn can use. */
12124 :
12125 : enum reg_class
12126 214955284 : ix86_insn_base_reg_class (rtx_insn* insn)
12127 : {
12128 214955284 : switch (ix86_memory_address_reg_class (insn))
12129 : {
12130 : case ADDR_GPR8:
12131 : return LEGACY_GENERAL_REGS;
12132 : case ADDR_GPR16:
12133 : return GENERAL_GPR16;
12134 : case ADDR_GPR32:
12135 : break;
12136 0 : default:
12137 0 : gcc_unreachable ();
12138 : }
12139 :
12140 : return BASE_REG_CLASS;
12141 : }
12142 :
12143 : bool
12144 1186011 : ix86_regno_ok_for_insn_base_p (int regno, rtx_insn* insn)
12145 : {
12146 1186011 : switch (ix86_memory_address_reg_class (insn))
12147 : {
12148 0 : case ADDR_GPR8:
12149 0 : return LEGACY_INT_REGNO_P (regno);
12150 0 : case ADDR_GPR16:
12151 0 : return GENERAL_GPR16_REGNO_P (regno);
12152 1186011 : case ADDR_GPR32:
12153 1186011 : break;
12154 0 : default:
12155 0 : gcc_unreachable ();
12156 : }
12157 :
12158 1186011 : return GENERAL_REGNO_P (regno);
12159 : }
12160 :
12161 : enum reg_class
12162 39709933 : ix86_insn_index_reg_class (rtx_insn* insn)
12163 : {
12164 39709933 : switch (ix86_memory_address_reg_class (insn))
12165 : {
12166 : case ADDR_GPR8:
12167 : return LEGACY_INDEX_REGS;
12168 : case ADDR_GPR16:
12169 : return INDEX_GPR16;
12170 : case ADDR_GPR32:
12171 : break;
12172 0 : default:
12173 0 : gcc_unreachable ();
12174 : }
12175 :
12176 : return INDEX_REG_CLASS;
12177 : }
12178 :
12179 : /* Recognizes RTL expressions that are valid memory addresses for an
12180 : instruction. The MODE argument is the machine mode for the MEM
12181 : expression that wants to use this address.
12182 :
12183 : It only recognizes address in canonical form. LEGITIMIZE_ADDRESS should
12184 : convert common non-canonical forms to canonical form so that they will
12185 : be recognized. */
12186 :
12187 : static bool
12188 2198992018 : ix86_legitimate_address_p (machine_mode, rtx addr, bool strict,
12189 : code_helper = ERROR_MARK)
12190 : {
12191 2198992018 : struct ix86_address parts;
12192 2198992018 : rtx base, index, disp;
12193 2198992018 : HOST_WIDE_INT scale;
12194 2198992018 : addr_space_t seg;
12195 :
12196 2198992018 : if (ix86_decompose_address (addr, &parts) == 0)
12197 : /* Decomposition failed. */
12198 : return false;
12199 :
12200 2187319342 : base = parts.base;
12201 2187319342 : index = parts.index;
12202 2187319342 : disp = parts.disp;
12203 2187319342 : scale = parts.scale;
12204 2187319342 : seg = parts.seg;
12205 :
12206 : /* Validate base register. */
12207 2187319342 : if (base)
12208 : {
12209 1229396739 : rtx reg = ix86_validate_address_register (base);
12210 :
12211 1229396739 : if (reg == NULL_RTX)
12212 : return false;
12213 :
12214 1228960797 : unsigned int regno = REGNO (reg);
12215 1228960797 : if ((strict && !REGNO_OK_FOR_BASE_P (regno))
12216 1224520378 : || (!strict && !REGNO_OK_FOR_BASE_NONSTRICT_P (regno)))
12217 : /* Base is not valid. */
12218 : return false;
12219 : }
12220 :
12221 : /* Validate index register. */
12222 2185545204 : if (index)
12223 : {
12224 83422011 : rtx reg = ix86_validate_address_register (index);
12225 :
12226 83422011 : if (reg == NULL_RTX)
12227 : return false;
12228 :
12229 83383191 : unsigned int regno = REGNO (reg);
12230 83383191 : if ((strict && !REGNO_OK_FOR_INDEX_P (regno))
12231 83374565 : || (!strict && !REGNO_OK_FOR_INDEX_NONSTRICT_P (regno)))
12232 : /* Index is not valid. */
12233 : return false;
12234 : }
12235 :
12236 : /* Index and base should have the same mode. */
12237 2185504200 : if (base && index
12238 73770411 : && GET_MODE (base) != GET_MODE (index))
12239 : return false;
12240 :
12241 : /* Address override works only on the (%reg) part of %fs:(%reg). */
12242 2185307558 : if (seg != ADDR_SPACE_GENERIC
12243 2185307558 : && ((base && GET_MODE (base) != word_mode)
12244 341309 : || (index && GET_MODE (index) != word_mode)))
12245 : return false;
12246 :
12247 : /* Validate scale factor. */
12248 2185307529 : if (scale != 1)
12249 : {
12250 39582424 : if (!index)
12251 : /* Scale without index. */
12252 : return false;
12253 :
12254 39582424 : if (scale != 2 && scale != 4 && scale != 8)
12255 : /* Scale is not a valid multiplier. */
12256 : return false;
12257 : }
12258 :
12259 : /* Validate displacement. */
12260 2182177002 : if (disp)
12261 : {
12262 1965485352 : if (ix86_endbr_immediate_operand (disp, VOIDmode))
12263 : return false;
12264 :
12265 1965485309 : if (GET_CODE (disp) == CONST
12266 147554573 : && GET_CODE (XEXP (disp, 0)) == UNSPEC
12267 15416760 : && XINT (XEXP (disp, 0), 1) != UNSPEC_MACHOPIC_OFFSET)
12268 15416760 : switch (XINT (XEXP (disp, 0), 1))
12269 : {
12270 : /* Refuse GOTOFF and GOT in 64bit mode since it is always 64bit
12271 : when used. While ABI specify also 32bit relocations, we
12272 : don't produce them at all and use IP relative instead.
12273 : Allow GOT in 32bit mode for both PIC and non-PIC if symbol
12274 : should be loaded via GOT. */
12275 2265226 : case UNSPEC_GOT:
12276 2265226 : if (!TARGET_64BIT
12277 2265226 : && ix86_force_load_from_GOT_p (XVECEXP (XEXP (disp, 0), 0, 0)))
12278 0 : goto is_legitimate_pic;
12279 : /* FALLTHRU */
12280 4558268 : case UNSPEC_GOTOFF:
12281 4558268 : gcc_assert (flag_pic);
12282 4558268 : if (!TARGET_64BIT)
12283 4558065 : goto is_legitimate_pic;
12284 :
12285 : /* 64bit address unspec. */
12286 : return false;
12287 :
12288 9779120 : case UNSPEC_GOTPCREL:
12289 9779120 : if (ix86_force_load_from_GOT_p (XVECEXP (XEXP (disp, 0), 0, 0)))
12290 2534 : goto is_legitimate_pic;
12291 : /* FALLTHRU */
12292 9776586 : case UNSPEC_PCREL:
12293 9776586 : gcc_assert (flag_pic);
12294 9776586 : goto is_legitimate_pic;
12295 :
12296 : case UNSPEC_GOTTPOFF:
12297 : case UNSPEC_GOTNTPOFF:
12298 : case UNSPEC_INDNTPOFF:
12299 : case UNSPEC_NTPOFF:
12300 : case UNSPEC_DTPOFF:
12301 : case UNSPEC_SECREL32:
12302 : break;
12303 :
12304 : default:
12305 : /* Invalid address unspec. */
12306 : return false;
12307 : }
12308 :
12309 1216922416 : else if (SYMBOLIC_CONST (disp)
12310 2082206362 : && (flag_pic
12311 : #if TARGET_MACHO
12312 : || (MACHOPIC_INDIRECT
12313 : && !machopic_operand_p (disp))
12314 : #endif
12315 : ))
12316 : {
12317 :
12318 57812274 : is_legitimate_pic:
12319 57812274 : if (TARGET_64BIT && (index || base))
12320 : {
12321 : /* foo@dtpoff(%rX) is ok. */
12322 37045 : if (GET_CODE (disp) != CONST
12323 7003 : || GET_CODE (XEXP (disp, 0)) != PLUS
12324 7003 : || GET_CODE (XEXP (XEXP (disp, 0), 0)) != UNSPEC
12325 4637 : || !CONST_INT_P (XEXP (XEXP (disp, 0), 1))
12326 4637 : || (XINT (XEXP (XEXP (disp, 0), 0), 1) != UNSPEC_DTPOFF
12327 4637 : && XINT (XEXP (XEXP (disp, 0), 0), 1) != UNSPEC_NTPOFF
12328 6 : && XINT (XEXP (XEXP (disp, 0), 0), 1) != UNSPEC_SECREL32))
12329 : /* Non-constant pic memory reference. */
12330 : return false;
12331 : }
12332 57775229 : else if ((!TARGET_MACHO || flag_pic)
12333 57775229 : && ! legitimate_pic_address_disp_p (disp))
12334 : /* Displacement is an invalid pic construct. */
12335 : return false;
12336 : #if TARGET_MACHO
12337 : else if (MACHO_DYNAMIC_NO_PIC_P
12338 : && !ix86_legitimate_constant_p (Pmode, disp))
12339 : /* displacement must be referenced via non_lazy_pointer */
12340 : return false;
12341 : #endif
12342 :
12343 : /* This code used to verify that a symbolic pic displacement
12344 : includes the pic_offset_table_rtx register.
12345 :
12346 : While this is good idea, unfortunately these constructs may
12347 : be created by "adds using lea" optimization for incorrect
12348 : code like:
12349 :
12350 : int a;
12351 : int foo(int i)
12352 : {
12353 : return *(&a+i);
12354 : }
12355 :
12356 : This code is nonsensical, but results in addressing
12357 : GOT table with pic_offset_table_rtx base. We can't
12358 : just refuse it easily, since it gets matched by
12359 : "addsi3" pattern, that later gets split to lea in the
12360 : case output register differs from input. While this
12361 : can be handled by separate addsi pattern for this case
12362 : that never results in lea, this seems to be easier and
12363 : correct fix for crash to disable this test. */
12364 : }
12365 1906593460 : else if (!LABEL_REF_P (disp)
12366 1906434649 : && !CONST_INT_P (disp)
12367 879320721 : && (GET_CODE (disp) != CONST
12368 133578159 : || !ix86_legitimate_constant_p (Pmode, disp))
12369 2655303960 : && (!SYMBOL_REF_P (disp)
12370 756178319 : || !ix86_legitimate_constant_p (Pmode, disp)))
12371 : /* Displacement is not constant. */
12372 57749788 : return false;
12373 1848843672 : else if (TARGET_64BIT
12374 1848843672 : && !x86_64_immediate_operand (disp, VOIDmode))
12375 : /* Displacement is out of range. */
12376 : return false;
12377 : /* In x32 mode, constant addresses are sign extended to 64bit, so
12378 : we have to prevent addresses from 0x80000000 to 0xffffffff. */
12379 45961 : else if (TARGET_X32 && !(index || base)
12380 17399 : && CONST_INT_P (disp)
12381 1848328050 : && val_signbit_known_set_p (SImode, INTVAL (disp)))
12382 : return false;
12383 : }
12384 :
12385 : /* Everything looks valid. */
12386 : return true;
12387 : }
12388 :
12389 : /* Determine if a given RTX is a valid constant address. */
12390 :
12391 : bool
12392 2807421028 : constant_address_p (rtx x)
12393 : {
12394 2887829532 : return CONSTANT_P (x) && ix86_legitimate_address_p (Pmode, x, 1);
12395 : }
12396 :
12397 :
12398 : /* Return a legitimate reference for ORIG (an address) using the
12399 : register REG. If REG is 0, a new pseudo is generated.
12400 :
12401 : There are two types of references that must be handled:
12402 :
12403 : 1. Global data references must load the address from the GOT, via
12404 : the PIC reg. An insn is emitted to do this load, and the reg is
12405 : returned.
12406 :
12407 : 2. Static data references, constant pool addresses, and code labels
12408 : compute the address as an offset from the GOT, whose base is in
12409 : the PIC reg. Static data objects have SYMBOL_FLAG_LOCAL set to
12410 : differentiate them from global data objects. The returned
12411 : address is the PIC reg + an unspec constant.
12412 :
12413 : TARGET_LEGITIMATE_ADDRESS_P rejects symbolic references unless the PIC
12414 : reg also appears in the address. */
12415 :
12416 : rtx
12417 397610 : legitimize_pic_address (rtx orig, rtx reg)
12418 : {
12419 397610 : rtx addr = orig;
12420 397610 : rtx new_rtx = orig;
12421 :
12422 : #if TARGET_MACHO
12423 : if (TARGET_MACHO && !TARGET_64BIT)
12424 : {
12425 : if (reg == 0)
12426 : reg = gen_reg_rtx (Pmode);
12427 : /* Use the generic Mach-O PIC machinery. */
12428 : return machopic_legitimize_pic_address (orig, GET_MODE (orig), reg);
12429 : }
12430 : #endif
12431 :
12432 397610 : if (TARGET_64BIT && TARGET_DLLIMPORT_DECL_ATTRIBUTES)
12433 : {
12434 : #if TARGET_PECOFF
12435 : rtx tmp = legitimize_pe_coff_symbol (addr, true);
12436 : if (tmp)
12437 : return tmp;
12438 : #endif
12439 : }
12440 :
12441 397610 : if (TARGET_64BIT && legitimate_pic_address_disp_p (addr))
12442 : new_rtx = addr;
12443 302405 : else if ((!TARGET_64BIT
12444 102451 : || /* TARGET_64BIT && */ ix86_cmodel != CM_SMALL_PIC)
12445 : && !TARGET_PECOFF
12446 502456 : && gotoff_operand (addr, Pmode))
12447 : {
12448 : /* This symbol may be referenced via a displacement
12449 : from the PIC base address (@GOTOFF). */
12450 96585 : if (GET_CODE (addr) == CONST)
12451 3048 : addr = XEXP (addr, 0);
12452 :
12453 96585 : if (GET_CODE (addr) == PLUS)
12454 : {
12455 6096 : new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, XEXP (addr, 0)),
12456 : UNSPEC_GOTOFF);
12457 6096 : new_rtx = gen_rtx_PLUS (Pmode, new_rtx, XEXP (addr, 1));
12458 : }
12459 : else
12460 187045 : new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOTOFF);
12461 :
12462 193141 : new_rtx = gen_rtx_CONST (Pmode, new_rtx);
12463 :
12464 96585 : if (TARGET_64BIT)
12465 29 : new_rtx = copy_to_suggested_reg (new_rtx, reg, Pmode);
12466 :
12467 96585 : if (reg != 0)
12468 : {
12469 3 : gcc_assert (REG_P (reg));
12470 3 : new_rtx = expand_simple_binop (Pmode, PLUS, pic_offset_table_rtx,
12471 : new_rtx, reg, 1, OPTAB_DIRECT);
12472 : }
12473 : else
12474 193138 : new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
12475 : }
12476 384365 : else if ((SYMBOL_REF_P (addr) && SYMBOL_REF_TLS_MODEL (addr) == 0)
12477 : /* We can't always use @GOTOFF for text labels
12478 : on VxWorks, see gotoff_operand. */
12479 205820 : || (TARGET_VXWORKS_VAROFF && LABEL_REF_P (addr)))
12480 : {
12481 : #if TARGET_PECOFF
12482 : rtx tmp = legitimize_pe_coff_symbol (addr, true);
12483 : if (tmp)
12484 : return tmp;
12485 : #endif
12486 :
12487 : /* For x64 PE-COFF there is no GOT table,
12488 : so we use address directly. */
12489 178542 : if (TARGET_64BIT && TARGET_PECOFF)
12490 : {
12491 : new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_PCREL);
12492 : new_rtx = gen_rtx_CONST (Pmode, new_rtx);
12493 : }
12494 178542 : else if (TARGET_64BIT && ix86_cmodel != CM_LARGE_PIC)
12495 : {
12496 95171 : new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr),
12497 : UNSPEC_GOTPCREL);
12498 95171 : new_rtx = gen_rtx_CONST (Pmode, new_rtx);
12499 95171 : new_rtx = gen_const_mem (Pmode, new_rtx);
12500 95168 : set_mem_alias_set (new_rtx, GOT_ALIAS_SET);
12501 : }
12502 : else
12503 : {
12504 : /* This symbol must be referenced via a load
12505 : from the Global Offset Table (@GOT). */
12506 166725 : new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOT);
12507 166725 : new_rtx = gen_rtx_CONST (Pmode, new_rtx);
12508 :
12509 83374 : if (TARGET_64BIT)
12510 23 : new_rtx = copy_to_suggested_reg (new_rtx, reg, Pmode);
12511 :
12512 83374 : if (reg != 0)
12513 : {
12514 0 : gcc_assert (REG_P (reg));
12515 0 : new_rtx = expand_simple_binop (Pmode, PLUS, pic_offset_table_rtx,
12516 : new_rtx, reg, 1, OPTAB_DIRECT);
12517 : }
12518 : else
12519 166725 : new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
12520 :
12521 166725 : new_rtx = gen_const_mem (Pmode, new_rtx);
12522 83374 : set_mem_alias_set (new_rtx, GOT_ALIAS_SET);
12523 : }
12524 :
12525 261896 : new_rtx = copy_to_suggested_reg (new_rtx, reg, Pmode);
12526 : }
12527 : else
12528 : {
12529 27278 : if (CONST_INT_P (addr)
12530 27278 : && !x86_64_immediate_operand (addr, VOIDmode))
12531 8 : new_rtx = copy_to_suggested_reg (addr, reg, Pmode);
12532 27270 : else if (GET_CODE (addr) == CONST)
12533 : {
12534 16755 : addr = XEXP (addr, 0);
12535 :
12536 : /* We must match stuff we generate before. Assume the only
12537 : unspecs that can get here are ours. Not that we could do
12538 : anything with them anyway.... */
12539 16755 : if (GET_CODE (addr) == UNSPEC
12540 8978 : || (GET_CODE (addr) == PLUS
12541 8978 : && GET_CODE (XEXP (addr, 0)) == UNSPEC))
12542 : return orig;
12543 6850 : gcc_assert (GET_CODE (addr) == PLUS);
12544 : }
12545 :
12546 17373 : if (GET_CODE (addr) == PLUS)
12547 : {
12548 8687 : rtx op0 = XEXP (addr, 0), op1 = XEXP (addr, 1);
12549 :
12550 : /* Check first to see if this is a constant
12551 : offset from a @GOTOFF symbol reference. */
12552 8687 : if (!TARGET_PECOFF
12553 13758 : && gotoff_operand (op0, Pmode)
12554 8687 : && CONST_INT_P (op1))
12555 : {
12556 4 : if (!TARGET_64BIT)
12557 : {
12558 0 : new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, op0),
12559 : UNSPEC_GOTOFF);
12560 0 : new_rtx = gen_rtx_PLUS (Pmode, new_rtx, op1);
12561 0 : new_rtx = gen_rtx_CONST (Pmode, new_rtx);
12562 :
12563 0 : if (reg != 0)
12564 : {
12565 0 : gcc_assert (REG_P (reg));
12566 0 : new_rtx = expand_simple_binop (Pmode, PLUS,
12567 : pic_offset_table_rtx,
12568 : new_rtx, reg, 1,
12569 : OPTAB_DIRECT);
12570 : }
12571 : else
12572 0 : new_rtx
12573 0 : = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
12574 : }
12575 : else
12576 : {
12577 4 : if (INTVAL (op1) < -16*1024*1024
12578 4 : || INTVAL (op1) >= 16*1024*1024)
12579 : {
12580 4 : if (!x86_64_immediate_operand (op1, Pmode))
12581 4 : op1 = force_reg (Pmode, op1);
12582 :
12583 4 : new_rtx
12584 4 : = gen_rtx_PLUS (Pmode, force_reg (Pmode, op0), op1);
12585 : }
12586 : }
12587 : }
12588 : else
12589 : {
12590 8683 : rtx base = legitimize_pic_address (op0, reg);
12591 8683 : machine_mode mode = GET_MODE (base);
12592 8683 : new_rtx
12593 8683 : = legitimize_pic_address (op1, base == reg ? NULL_RTX : reg);
12594 :
12595 8683 : if (CONST_INT_P (new_rtx))
12596 : {
12597 6838 : if (INTVAL (new_rtx) < -16*1024*1024
12598 6838 : || INTVAL (new_rtx) >= 16*1024*1024)
12599 : {
12600 0 : if (!x86_64_immediate_operand (new_rtx, mode))
12601 0 : new_rtx = force_reg (mode, new_rtx);
12602 :
12603 0 : new_rtx
12604 0 : = gen_rtx_PLUS (mode, force_reg (mode, base), new_rtx);
12605 : }
12606 : else
12607 6838 : new_rtx = plus_constant (mode, base, INTVAL (new_rtx));
12608 : }
12609 : else
12610 : {
12611 : /* For %rip addressing, we have to use
12612 : just disp32, not base nor index. */
12613 1845 : if (TARGET_64BIT
12614 101 : && (SYMBOL_REF_P (base)
12615 101 : || LABEL_REF_P (base)))
12616 7 : base = force_reg (mode, base);
12617 1845 : if (GET_CODE (new_rtx) == PLUS
12618 1723 : && CONSTANT_P (XEXP (new_rtx, 1)))
12619 : {
12620 1719 : base = gen_rtx_PLUS (mode, base, XEXP (new_rtx, 0));
12621 1719 : new_rtx = XEXP (new_rtx, 1);
12622 : }
12623 1845 : new_rtx = gen_rtx_PLUS (mode, base, new_rtx);
12624 : }
12625 : }
12626 : }
12627 : }
12628 : return new_rtx;
12629 : }
12630 :
12631 : /* Load the thread pointer. If TO_REG is true, force it into a register. */
12632 :
12633 : static rtx
12634 24588 : get_thread_pointer (machine_mode tp_mode, bool to_reg)
12635 : {
12636 24588 : rtx tp = gen_rtx_UNSPEC (ptr_mode, gen_rtvec (1, const0_rtx), UNSPEC_TP);
12637 :
12638 24588 : if (GET_MODE (tp) != tp_mode)
12639 : {
12640 11 : gcc_assert (GET_MODE (tp) == SImode);
12641 11 : gcc_assert (tp_mode == DImode);
12642 :
12643 11 : tp = gen_rtx_ZERO_EXTEND (tp_mode, tp);
12644 : }
12645 :
12646 24588 : if (to_reg)
12647 8163 : tp = copy_to_mode_reg (tp_mode, tp);
12648 :
12649 24588 : return tp;
12650 : }
12651 :
12652 : /* Construct the SYMBOL_REF for the _tls_index symbol. */
12653 :
12654 : static GTY(()) rtx ix86_tls_index_symbol;
12655 :
12656 : static rtx
12657 0 : ix86_tls_index (void)
12658 : {
12659 0 : if (!ix86_tls_index_symbol)
12660 0 : ix86_tls_index_symbol = gen_rtx_SYMBOL_REF (SImode, "_tls_index");
12661 :
12662 0 : if (flag_pic)
12663 0 : return gen_rtx_CONST (Pmode,
12664 : gen_rtx_UNSPEC (Pmode,
12665 : gen_rtvec (1, ix86_tls_index_symbol),
12666 : UNSPEC_PCREL));
12667 : else
12668 0 : return ix86_tls_index_symbol;
12669 : }
12670 :
12671 : /* Construct the SYMBOL_REF for the tls_get_addr function. */
12672 :
12673 : static GTY(()) rtx ix86_tls_symbol;
12674 :
12675 : rtx
12676 6726 : ix86_tls_get_addr (void)
12677 : {
12678 6726 : if (cfun->machine->call_saved_registers
12679 6726 : == TYPE_NO_CALLER_SAVED_REGISTERS)
12680 : {
12681 : /* __tls_get_addr doesn't preserve vector registers. When a
12682 : function with no_caller_saved_registers attribute calls
12683 : __tls_get_addr, YMM and ZMM registers will be clobbered.
12684 : Issue an error and suggest -mtls-dialect=gnu2 in this case. */
12685 3 : if (cfun->machine->func_type == TYPE_NORMAL)
12686 1 : error (G_("%<-mtls-dialect=gnu2%> must be used with a function"
12687 : " with the %<no_caller_saved_registers%> attribute"));
12688 : else
12689 3 : error (cfun->machine->func_type == TYPE_EXCEPTION
12690 : ? G_("%<-mtls-dialect=gnu2%> must be used with an"
12691 : " exception service routine")
12692 : : G_("%<-mtls-dialect=gnu2%> must be used with an"
12693 : " interrupt service routine"));
12694 : /* Don't issue the same error twice. */
12695 3 : cfun->machine->func_type = TYPE_NORMAL;
12696 3 : cfun->machine->call_saved_registers
12697 3 : = TYPE_DEFAULT_CALL_SAVED_REGISTERS;
12698 : }
12699 :
12700 6726 : if (!ix86_tls_symbol)
12701 : {
12702 216 : const char *sym
12703 253 : = ((TARGET_ANY_GNU_TLS && !TARGET_64BIT)
12704 253 : ? "___tls_get_addr" : "__tls_get_addr");
12705 :
12706 290 : ix86_tls_symbol = gen_rtx_SYMBOL_REF (Pmode, sym);
12707 : }
12708 :
12709 6726 : if (ix86_cmodel == CM_LARGE_PIC && !TARGET_PECOFF)
12710 : {
12711 2 : rtx unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, ix86_tls_symbol),
12712 : UNSPEC_PLTOFF);
12713 2 : return gen_rtx_PLUS (Pmode, pic_offset_table_rtx,
12714 : gen_rtx_CONST (Pmode, unspec));
12715 : }
12716 :
12717 6724 : return ix86_tls_symbol;
12718 : }
12719 :
12720 : /* Return the descriptor of the function ABI type for the tls_get_addr
12721 : function. */
12722 :
12723 : const predefined_function_abi &
12724 5012 : ix86_tls_get_addr_abi (void)
12725 : {
12726 5012 : if (ix86_abi == SYSV_ABI)
12727 5000 : return default_function_abi;
12728 : else
12729 12 : return ix86_alternate_abi ();
12730 : }
12731 :
12732 : /* Construct the SYMBOL_REF for the _TLS_MODULE_BASE_ symbol. */
12733 :
12734 : static GTY(()) rtx ix86_tls_module_base_symbol;
12735 :
12736 : rtx
12737 98 : ix86_tls_module_base (void)
12738 : {
12739 98 : if (!ix86_tls_module_base_symbol)
12740 : {
12741 11 : ix86_tls_module_base_symbol
12742 11 : = gen_rtx_SYMBOL_REF (ptr_mode, "_TLS_MODULE_BASE_");
12743 :
12744 11 : SYMBOL_REF_FLAGS (ix86_tls_module_base_symbol)
12745 11 : |= TLS_MODEL_GLOBAL_DYNAMIC << SYMBOL_FLAG_TLS_SHIFT;
12746 : }
12747 :
12748 98 : return ix86_tls_module_base_symbol;
12749 : }
12750 :
12751 : /* A subroutine of ix86_legitimize_address and ix86_expand_move. FOR_MOV is
12752 : false if we expect this to be used for a memory address and true if
12753 : we expect to load the address into a register. */
12754 :
12755 : rtx
12756 31015 : legitimize_tls_address (rtx x, enum tls_model model, bool for_mov)
12757 : {
12758 31015 : rtx dest, base, off;
12759 31015 : rtx pic = NULL_RTX, tp = NULL_RTX;
12760 31015 : machine_mode tp_mode = Pmode;
12761 31015 : int type;
12762 :
12763 : /* Windows implements a single form of TLS. */
12764 31015 : if (TARGET_WIN32_TLS)
12765 : {
12766 : /* Load the 32-bit index. */
12767 : rtx ind = gen_const_mem (SImode, ix86_tls_index ());
12768 : set_mem_alias_set (ind, GOT_ALIAS_SET);
12769 : if (TARGET_64BIT)
12770 : ind = convert_to_mode (Pmode, ind, 1);
12771 : ind = force_reg (Pmode, ind);
12772 :
12773 : /* Add it to the thread pointer and load the base. */
12774 : tp = get_thread_pointer (Pmode, true);
12775 : rtx addr = gen_rtx_PLUS (Pmode, tp,
12776 : gen_rtx_MULT (Pmode, ind,
12777 : GEN_INT (UNITS_PER_WORD)));
12778 : base = gen_const_mem (Pmode, addr);
12779 : set_mem_alias_set (base, GOT_ALIAS_SET);
12780 :
12781 : /* Add the 32-bit section-relative offset to the base. */
12782 : base = force_reg (Pmode, base);
12783 : off = gen_rtx_CONST (Pmode,
12784 : gen_rtx_UNSPEC (SImode,
12785 : gen_rtvec (1, x),
12786 : UNSPEC_SECREL32));
12787 : return gen_rtx_PLUS (Pmode, base, off);
12788 : }
12789 :
12790 : /* Fall back to global dynamic model if tool chain cannot support local
12791 : dynamic. */
12792 31015 : if (TARGET_SUN_TLS && !TARGET_64BIT
12793 : && !HAVE_AS_IX86_TLSLDMPLT && !HAVE_AS_IX86_TLSLDM
12794 : && model == TLS_MODEL_LOCAL_DYNAMIC)
12795 : model = TLS_MODEL_GLOBAL_DYNAMIC;
12796 :
12797 31015 : switch (model)
12798 : {
12799 6112 : case TLS_MODEL_GLOBAL_DYNAMIC:
12800 6112 : if (!TARGET_64BIT)
12801 : {
12802 1922 : if (flag_pic && !TARGET_PECOFF)
12803 1922 : pic = pic_offset_table_rtx;
12804 : else
12805 : {
12806 0 : pic = gen_reg_rtx (Pmode);
12807 0 : emit_insn (gen_set_got (pic));
12808 : }
12809 : }
12810 :
12811 6112 : if (TARGET_GNU2_TLS)
12812 : {
12813 53 : dest = gen_reg_rtx (ptr_mode);
12814 53 : if (TARGET_64BIT)
12815 53 : emit_insn (gen_tls_dynamic_gnu2_64 (ptr_mode, dest, x));
12816 : else
12817 0 : emit_insn (gen_tls_dynamic_gnu2_32 (dest, x, pic));
12818 :
12819 53 : tp = get_thread_pointer (ptr_mode, true);
12820 53 : dest = gen_rtx_PLUS (ptr_mode, tp, dest);
12821 61 : if (GET_MODE (dest) != Pmode)
12822 6 : dest = gen_rtx_ZERO_EXTEND (Pmode, dest);
12823 61 : dest = force_reg (Pmode, dest);
12824 :
12825 61 : if (GET_MODE (x) != Pmode)
12826 3 : x = gen_rtx_ZERO_EXTEND (Pmode, x);
12827 :
12828 53 : set_unique_reg_note (get_last_insn (), REG_EQUAL, x);
12829 : }
12830 : else
12831 : {
12832 6059 : rtx caddr = ix86_tls_get_addr ();
12833 :
12834 7981 : dest = gen_reg_rtx (Pmode);
12835 6059 : if (TARGET_64BIT)
12836 : {
12837 4137 : rtx rax = gen_rtx_REG (Pmode, AX_REG);
12838 4137 : rtx rdi = gen_rtx_REG (Pmode, DI_REG);
12839 4137 : rtx_insn *insns;
12840 :
12841 4137 : start_sequence ();
12842 4137 : rtx_insn *call_insn = emit_call_insn
12843 4137 : (gen_tls_global_dynamic_64 (Pmode, rax, x, caddr, rdi));
12844 4137 : CALL_INSN_ABI_ID (call_insn)
12845 4137 : = ix86_tls_get_addr_abi ().id ();
12846 4137 : insns = end_sequence ();
12847 :
12848 4137 : if (GET_MODE (x) != Pmode)
12849 1 : x = gen_rtx_ZERO_EXTEND (Pmode, x);
12850 :
12851 4137 : RTL_CONST_CALL_P (insns) = 1;
12852 4137 : emit_libcall_block (insns, dest, rax, x);
12853 : }
12854 : else
12855 1922 : emit_insn (gen_tls_global_dynamic_32 (dest, x, pic, caddr));
12856 : }
12857 : break;
12858 :
12859 394 : case TLS_MODEL_LOCAL_DYNAMIC:
12860 394 : if (!TARGET_64BIT)
12861 : {
12862 92 : if (flag_pic)
12863 92 : pic = pic_offset_table_rtx;
12864 : else
12865 : {
12866 0 : pic = gen_reg_rtx (Pmode);
12867 0 : emit_insn (gen_set_got (pic));
12868 : }
12869 : }
12870 :
12871 394 : if (TARGET_GNU2_TLS)
12872 : {
12873 26 : rtx tmp = ix86_tls_module_base ();
12874 :
12875 26 : base = gen_reg_rtx (ptr_mode);
12876 26 : if (TARGET_64BIT)
12877 26 : emit_insn (gen_tls_dynamic_gnu2_64 (ptr_mode, base, tmp));
12878 : else
12879 0 : emit_insn (gen_tls_dynamic_gnu2_32 (base, tmp, pic));
12880 :
12881 26 : tp = get_thread_pointer (ptr_mode, true);
12882 32 : if (GET_MODE (base) != Pmode)
12883 2 : base = gen_rtx_ZERO_EXTEND (Pmode, base);
12884 32 : base = force_reg (Pmode, base);
12885 : }
12886 : else
12887 : {
12888 368 : rtx caddr = ix86_tls_get_addr ();
12889 :
12890 460 : base = gen_reg_rtx (Pmode);
12891 368 : if (TARGET_64BIT)
12892 : {
12893 276 : rtx rax = gen_rtx_REG (Pmode, AX_REG);
12894 276 : rtx rdi = gen_rtx_REG (Pmode, DI_REG);
12895 276 : rtx_insn *insns;
12896 276 : rtx eqv;
12897 :
12898 276 : start_sequence ();
12899 276 : rtx_insn *call_insn = emit_call_insn
12900 276 : (gen_tls_local_dynamic_base_64 (Pmode, rax, caddr, rdi));
12901 276 : CALL_INSN_ABI_ID (call_insn)
12902 276 : = ix86_tls_get_addr_abi ().id ();
12903 276 : insns = end_sequence ();
12904 :
12905 : /* Attach a unique REG_EQUAL, to allow the RTL optimizers to
12906 : share the LD_BASE result with other LD model accesses. */
12907 276 : eqv = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx),
12908 : UNSPEC_TLS_LD_BASE);
12909 :
12910 276 : RTL_CONST_CALL_P (insns) = 1;
12911 276 : emit_libcall_block (insns, base, rax, eqv);
12912 : }
12913 : else
12914 92 : emit_insn (gen_tls_local_dynamic_base_32 (base, pic, caddr));
12915 : }
12916 :
12917 492 : off = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x), UNSPEC_DTPOFF);
12918 492 : off = gen_rtx_CONST (Pmode, off);
12919 :
12920 590 : dest = force_reg (Pmode, gen_rtx_PLUS (Pmode, base, off));
12921 :
12922 394 : if (TARGET_GNU2_TLS)
12923 : {
12924 32 : if (GET_MODE (tp) != Pmode)
12925 : {
12926 2 : dest = lowpart_subreg (ptr_mode, dest, Pmode);
12927 2 : dest = gen_rtx_PLUS (ptr_mode, tp, dest);
12928 2 : dest = gen_rtx_ZERO_EXTEND (Pmode, dest);
12929 : }
12930 : else
12931 24 : dest = gen_rtx_PLUS (Pmode, tp, dest);
12932 32 : dest = force_reg (Pmode, dest);
12933 :
12934 32 : if (GET_MODE (x) != Pmode)
12935 1 : x = gen_rtx_ZERO_EXTEND (Pmode, x);
12936 :
12937 26 : set_unique_reg_note (get_last_insn (), REG_EQUAL, x);
12938 : }
12939 : break;
12940 :
12941 10862 : case TLS_MODEL_INITIAL_EXEC:
12942 10862 : if (TARGET_64BIT)
12943 : {
12944 : /* Generate DImode references to avoid %fs:(%reg32)
12945 : problems and linker IE->LE relaxation bug. */
12946 : tp_mode = DImode;
12947 : pic = NULL;
12948 : type = UNSPEC_GOTNTPOFF;
12949 : }
12950 786 : else if (flag_pic)
12951 : {
12952 785 : pic = pic_offset_table_rtx;
12953 785 : type = TARGET_ANY_GNU_TLS ? UNSPEC_GOTNTPOFF : UNSPEC_GOTTPOFF;
12954 : }
12955 1 : else if (!TARGET_ANY_GNU_TLS)
12956 : {
12957 0 : pic = gen_reg_rtx (Pmode);
12958 0 : emit_insn (gen_set_got (pic));
12959 0 : type = UNSPEC_GOTTPOFF;
12960 : }
12961 : else
12962 : {
12963 : pic = NULL;
12964 : type = UNSPEC_INDNTPOFF;
12965 : }
12966 :
12967 10862 : off = gen_rtx_UNSPEC (tp_mode, gen_rtvec (1, x), type);
12968 10862 : off = gen_rtx_CONST (tp_mode, off);
12969 10862 : if (pic)
12970 785 : off = gen_rtx_PLUS (tp_mode, pic, off);
12971 10862 : off = gen_const_mem (tp_mode, off);
12972 10862 : set_mem_alias_set (off, GOT_ALIAS_SET);
12973 :
12974 10862 : if (TARGET_64BIT || TARGET_ANY_GNU_TLS)
12975 : {
12976 10862 : base = get_thread_pointer (tp_mode,
12977 10862 : for_mov || !TARGET_TLS_DIRECT_SEG_REFS);
12978 10862 : off = force_reg (tp_mode, off);
12979 10862 : dest = gen_rtx_PLUS (tp_mode, base, off);
12980 11652 : if (tp_mode != Pmode)
12981 4 : dest = convert_to_mode (Pmode, dest, 1);
12982 : }
12983 : else
12984 : {
12985 0 : base = get_thread_pointer (Pmode, true);
12986 0 : dest = gen_reg_rtx (Pmode);
12987 0 : emit_insn (gen_sub3_insn (dest, base, off));
12988 : }
12989 : break;
12990 :
12991 13647 : case TLS_MODEL_LOCAL_EXEC:
12992 28092 : off = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x),
12993 : (TARGET_64BIT || TARGET_ANY_GNU_TLS)
12994 : ? UNSPEC_NTPOFF : UNSPEC_TPOFF);
12995 14445 : off = gen_rtx_CONST (Pmode, off);
12996 :
12997 13647 : if (TARGET_64BIT || TARGET_ANY_GNU_TLS)
12998 : {
12999 14445 : base = get_thread_pointer (Pmode,
13000 13647 : for_mov || !TARGET_TLS_DIRECT_SEG_REFS);
13001 14445 : return gen_rtx_PLUS (Pmode, base, off);
13002 : }
13003 : else
13004 : {
13005 0 : base = get_thread_pointer (Pmode, true);
13006 0 : dest = gen_reg_rtx (Pmode);
13007 0 : emit_insn (gen_sub3_insn (dest, base, off));
13008 : }
13009 0 : break;
13010 :
13011 0 : default:
13012 0 : gcc_unreachable ();
13013 : }
13014 :
13015 : return dest;
13016 : }
13017 :
13018 : /* Return true if the TLS address requires insn using integer registers.
13019 : It's used to prevent KMOV/VMOV in TLS code sequences which require integer
13020 : MOV instructions, refer to PR103275. */
13021 : bool
13022 15144710 : ix86_gpr_tls_address_pattern_p (rtx mem)
13023 : {
13024 15144710 : gcc_assert (MEM_P (mem));
13025 :
13026 15144710 : rtx addr = XEXP (mem, 0);
13027 15144710 : subrtx_var_iterator::array_type array;
13028 52633843 : FOR_EACH_SUBRTX_VAR (iter, array, addr, ALL)
13029 : {
13030 37496774 : rtx op = *iter;
13031 37496774 : if (GET_CODE (op) == UNSPEC)
13032 201468 : switch (XINT (op, 1))
13033 : {
13034 : case UNSPEC_GOTNTPOFF:
13035 7641 : return true;
13036 0 : case UNSPEC_TPOFF:
13037 0 : if (!TARGET_64BIT)
13038 : return true;
13039 : break;
13040 : default:
13041 : break;
13042 : }
13043 : }
13044 :
13045 15137069 : return false;
13046 15144710 : }
13047 :
13048 : /* Return true if OP refers to a TLS address. */
13049 : bool
13050 235631974 : ix86_tls_address_pattern_p (rtx op)
13051 : {
13052 235631974 : subrtx_var_iterator::array_type array;
13053 1405304840 : FOR_EACH_SUBRTX_VAR (iter, array, op, ALL)
13054 : {
13055 1169691129 : rtx op = *iter;
13056 1169691129 : if (MEM_P (op))
13057 : {
13058 105853403 : rtx *x = &XEXP (op, 0);
13059 167339924 : while (GET_CODE (*x) == PLUS)
13060 : {
13061 : int i;
13062 184477849 : for (i = 0; i < 2; i++)
13063 : {
13064 122991328 : rtx u = XEXP (*x, i);
13065 122991328 : if (GET_CODE (u) == ZERO_EXTEND)
13066 30000 : u = XEXP (u, 0);
13067 122991328 : if (GET_CODE (u) == UNSPEC
13068 18295 : && XINT (u, 1) == UNSPEC_TP)
13069 18263 : return true;
13070 : }
13071 61486521 : x = &XEXP (*x, 0);
13072 : }
13073 :
13074 105835140 : iter.skip_subrtxes ();
13075 : }
13076 : }
13077 :
13078 235613711 : return false;
13079 235631974 : }
13080 :
13081 : /* Rewrite *LOC so that it refers to a default TLS address space. */
13082 : static void
13083 18263 : ix86_rewrite_tls_address_1 (rtx *loc)
13084 : {
13085 18263 : subrtx_ptr_iterator::array_type array;
13086 54255 : FOR_EACH_SUBRTX_PTR (iter, array, loc, ALL)
13087 : {
13088 54255 : rtx *loc = *iter;
13089 54255 : if (MEM_P (*loc))
13090 : {
13091 18452 : rtx addr = XEXP (*loc, 0);
13092 18452 : rtx *x = &addr;
13093 23363 : while (GET_CODE (*x) == PLUS)
13094 : {
13095 : int i;
13096 33019 : for (i = 0; i < 2; i++)
13097 : {
13098 28108 : rtx u = XEXP (*x, i);
13099 28108 : if (GET_CODE (u) == ZERO_EXTEND)
13100 19 : u = XEXP (u, 0);
13101 28108 : if (GET_CODE (u) == UNSPEC
13102 18263 : && XINT (u, 1) == UNSPEC_TP)
13103 : {
13104 : /* NB: Since address override only applies to the
13105 : (reg32) part in fs:(reg32), return if address
13106 : override is used. */
13107 19962 : if (Pmode != word_mode
13108 18263 : && REG_P (XEXP (*x, 1 - i)))
13109 18263 : return;
13110 :
13111 18261 : addr_space_t as = DEFAULT_TLS_SEG_REG;
13112 :
13113 18261 : *x = XEXP (*x, 1 - i);
13114 :
13115 18261 : *loc = replace_equiv_address_nv (*loc, addr, true);
13116 18261 : set_mem_addr_space (*loc, as);
13117 18261 : return;
13118 : }
13119 : }
13120 4911 : x = &XEXP (*x, 0);
13121 : }
13122 :
13123 189 : iter.skip_subrtxes ();
13124 : }
13125 : }
13126 18263 : }
13127 :
13128 : /* Rewrite instruction pattern involvning TLS address
13129 : so that it refers to a default TLS address space. */
13130 : rtx
13131 18263 : ix86_rewrite_tls_address (rtx pattern)
13132 : {
13133 18263 : pattern = copy_insn (pattern);
13134 18263 : ix86_rewrite_tls_address_1 (&pattern);
13135 18263 : return pattern;
13136 : }
13137 :
13138 : /* Try machine-dependent ways of modifying an illegitimate address
13139 : to be legitimate. If we find one, return the new, valid address.
13140 : This macro is used in only one place: `memory_address' in explow.cc.
13141 :
13142 : OLDX is the address as it was before break_out_memory_refs was called.
13143 : In some cases it is useful to look at this to decide what needs to be done.
13144 :
13145 : It is always safe for this macro to do nothing. It exists to recognize
13146 : opportunities to optimize the output.
13147 :
13148 : For the 80386, we handle X+REG by loading X into a register R and
13149 : using R+REG. R will go in a general reg and indexing will be used.
13150 : However, if REG is a broken-out memory address or multiplication,
13151 : nothing needs to be done because REG can certainly go in a general reg.
13152 :
13153 : When -fpic is used, special handling is needed for symbolic references.
13154 : See comments by legitimize_pic_address in i386.cc for details. */
13155 :
13156 : static rtx
13157 661562 : ix86_legitimize_address (rtx x, rtx, machine_mode mode)
13158 : {
13159 661562 : bool changed = false;
13160 661562 : unsigned log;
13161 :
13162 661562 : log = SYMBOL_REF_P (x) ? SYMBOL_REF_TLS_MODEL (x) : 0;
13163 152022 : if (log)
13164 20845 : return legitimize_tls_address (x, (enum tls_model) log, false);
13165 640717 : if (GET_CODE (x) == CONST
13166 508 : && GET_CODE (XEXP (x, 0)) == PLUS
13167 508 : && SYMBOL_REF_P (XEXP (XEXP (x, 0), 0))
13168 641225 : && (log = SYMBOL_REF_TLS_MODEL (XEXP (XEXP (x, 0), 0))))
13169 : {
13170 4 : rtx t = legitimize_tls_address (XEXP (XEXP (x, 0), 0),
13171 : (enum tls_model) log, false);
13172 5 : return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (x, 0), 1));
13173 : }
13174 :
13175 640713 : if (TARGET_DLLIMPORT_DECL_ATTRIBUTES)
13176 : {
13177 : #if TARGET_PECOFF
13178 : rtx tmp = legitimize_pe_coff_symbol (x, true);
13179 : if (tmp)
13180 : return tmp;
13181 : #endif
13182 : }
13183 :
13184 640713 : if (flag_pic && SYMBOLIC_CONST (x))
13185 131560 : return legitimize_pic_address (x, 0);
13186 :
13187 : #if TARGET_MACHO
13188 : if (MACHO_DYNAMIC_NO_PIC_P && SYMBOLIC_CONST (x))
13189 : return machopic_indirect_data_reference (x, 0);
13190 : #endif
13191 :
13192 : /* Canonicalize shifts by 0, 1, 2, 3 into multiply */
13193 509153 : if (GET_CODE (x) == ASHIFT
13194 0 : && CONST_INT_P (XEXP (x, 1))
13195 0 : && (unsigned HOST_WIDE_INT) INTVAL (XEXP (x, 1)) < 4)
13196 : {
13197 0 : changed = true;
13198 0 : log = INTVAL (XEXP (x, 1));
13199 0 : x = gen_rtx_MULT (Pmode, force_reg (Pmode, XEXP (x, 0)),
13200 : GEN_INT (1 << log));
13201 : }
13202 :
13203 509153 : if (GET_CODE (x) == PLUS)
13204 : {
13205 : /* Canonicalize shifts by 0, 1, 2, 3 into multiply. */
13206 :
13207 165493 : if (GET_CODE (XEXP (x, 0)) == ASHIFT
13208 602 : && CONST_INT_P (XEXP (XEXP (x, 0), 1))
13209 602 : && (unsigned HOST_WIDE_INT) INTVAL (XEXP (XEXP (x, 0), 1)) < 4)
13210 : {
13211 602 : changed = true;
13212 602 : log = INTVAL (XEXP (XEXP (x, 0), 1));
13213 1762 : XEXP (x, 0) = gen_rtx_MULT (Pmode,
13214 : force_reg (Pmode, XEXP (XEXP (x, 0), 0)),
13215 : GEN_INT (1 << log));
13216 : }
13217 :
13218 165493 : if (GET_CODE (XEXP (x, 1)) == ASHIFT
13219 0 : && CONST_INT_P (XEXP (XEXP (x, 1), 1))
13220 0 : && (unsigned HOST_WIDE_INT) INTVAL (XEXP (XEXP (x, 1), 1)) < 4)
13221 : {
13222 0 : changed = true;
13223 0 : log = INTVAL (XEXP (XEXP (x, 1), 1));
13224 0 : XEXP (x, 1) = gen_rtx_MULT (Pmode,
13225 : force_reg (Pmode, XEXP (XEXP (x, 1), 0)),
13226 : GEN_INT (1 << log));
13227 : }
13228 :
13229 : /* Put multiply first if it isn't already. */
13230 165493 : if (GET_CODE (XEXP (x, 1)) == MULT)
13231 : {
13232 0 : std::swap (XEXP (x, 0), XEXP (x, 1));
13233 0 : changed = true;
13234 : }
13235 :
13236 : /* Canonicalize (plus (mult (reg) (const)) (plus (reg) (const)))
13237 : into (plus (plus (mult (reg) (const)) (reg)) (const)). This can be
13238 : created by virtual register instantiation, register elimination, and
13239 : similar optimizations. */
13240 165493 : if (GET_CODE (XEXP (x, 0)) == MULT && GET_CODE (XEXP (x, 1)) == PLUS)
13241 : {
13242 9536 : changed = true;
13243 15140 : x = gen_rtx_PLUS (Pmode,
13244 : gen_rtx_PLUS (Pmode, XEXP (x, 0),
13245 : XEXP (XEXP (x, 1), 0)),
13246 : XEXP (XEXP (x, 1), 1));
13247 : }
13248 :
13249 : /* Canonicalize
13250 : (plus (plus (mult (reg) (const)) (plus (reg) (const))) const)
13251 : into (plus (plus (mult (reg) (const)) (reg)) (const)). */
13252 155957 : else if (GET_CODE (x) == PLUS && GET_CODE (XEXP (x, 0)) == PLUS
13253 60484 : && GET_CODE (XEXP (XEXP (x, 0), 0)) == MULT
13254 51643 : && GET_CODE (XEXP (XEXP (x, 0), 1)) == PLUS
13255 0 : && CONSTANT_P (XEXP (x, 1)))
13256 : {
13257 0 : rtx constant;
13258 0 : rtx other = NULL_RTX;
13259 :
13260 0 : if (CONST_INT_P (XEXP (x, 1)))
13261 : {
13262 0 : constant = XEXP (x, 1);
13263 0 : other = XEXP (XEXP (XEXP (x, 0), 1), 1);
13264 : }
13265 0 : else if (CONST_INT_P (XEXP (XEXP (XEXP (x, 0), 1), 1)))
13266 : {
13267 : constant = XEXP (XEXP (XEXP (x, 0), 1), 1);
13268 : other = XEXP (x, 1);
13269 : }
13270 : else
13271 : constant = 0;
13272 :
13273 0 : if (constant)
13274 : {
13275 0 : changed = true;
13276 0 : x = gen_rtx_PLUS (Pmode,
13277 : gen_rtx_PLUS (Pmode, XEXP (XEXP (x, 0), 0),
13278 : XEXP (XEXP (XEXP (x, 0), 1), 0)),
13279 : plus_constant (Pmode, other,
13280 : INTVAL (constant)));
13281 : }
13282 : }
13283 :
13284 165493 : if (changed && ix86_legitimate_address_p (mode, x, false))
13285 9572 : return x;
13286 :
13287 155921 : if (GET_CODE (XEXP (x, 0)) == MULT)
13288 : {
13289 19849 : changed = true;
13290 19849 : XEXP (x, 0) = copy_addr_to_reg (XEXP (x, 0));
13291 : }
13292 :
13293 155921 : if (GET_CODE (XEXP (x, 1)) == MULT)
13294 : {
13295 0 : changed = true;
13296 0 : XEXP (x, 1) = copy_addr_to_reg (XEXP (x, 1));
13297 : }
13298 :
13299 155921 : if (changed
13300 19856 : && REG_P (XEXP (x, 1))
13301 16282 : && REG_P (XEXP (x, 0)))
13302 : return x;
13303 :
13304 139639 : if (flag_pic && SYMBOLIC_CONST (XEXP (x, 1)))
13305 : {
13306 1837 : changed = true;
13307 1837 : x = legitimize_pic_address (x, 0);
13308 : }
13309 :
13310 139639 : if (changed && ix86_legitimate_address_p (mode, x, false))
13311 3848 : return x;
13312 :
13313 135791 : if (REG_P (XEXP (x, 0)))
13314 : {
13315 77603 : rtx temp = gen_reg_rtx (Pmode);
13316 74847 : rtx val = force_operand (XEXP (x, 1), temp);
13317 74847 : if (val != temp)
13318 : {
13319 66927 : val = convert_to_mode (Pmode, val, 1);
13320 66636 : emit_move_insn (temp, val);
13321 : }
13322 :
13323 74847 : XEXP (x, 1) = temp;
13324 74847 : return x;
13325 : }
13326 :
13327 60944 : else if (REG_P (XEXP (x, 1)))
13328 : {
13329 4217 : rtx temp = gen_reg_rtx (Pmode);
13330 3393 : rtx val = force_operand (XEXP (x, 0), temp);
13331 3393 : if (val != temp)
13332 : {
13333 0 : val = convert_to_mode (Pmode, val, 1);
13334 0 : emit_move_insn (temp, val);
13335 : }
13336 :
13337 3393 : XEXP (x, 0) = temp;
13338 3393 : return x;
13339 : }
13340 : }
13341 :
13342 : return x;
13343 : }
13344 :
13345 : /* Print an integer constant expression in assembler syntax. Addition
13346 : and subtraction are the only arithmetic that may appear in these
13347 : expressions. FILE is the stdio stream to write to, X is the rtx, and
13348 : CODE is the operand print code from the output string. */
13349 :
13350 : static void
13351 3690207 : output_pic_addr_const (FILE *file, rtx x, int code)
13352 : {
13353 3920362 : char buf[256];
13354 :
13355 3920362 : switch (GET_CODE (x))
13356 : {
13357 0 : case PC:
13358 0 : gcc_assert (flag_pic);
13359 0 : putc ('.', file);
13360 0 : break;
13361 :
13362 865442 : case SYMBOL_REF:
13363 865442 : if (TARGET_64BIT || ! TARGET_MACHO_SYMBOL_STUBS)
13364 865442 : output_addr_const (file, x);
13365 : else
13366 : {
13367 : const char *name = XSTR (x, 0);
13368 :
13369 : /* Mark the decl as referenced so that cgraph will
13370 : output the function. */
13371 : if (SYMBOL_REF_DECL (x))
13372 : mark_decl_referenced (SYMBOL_REF_DECL (x));
13373 :
13374 : #if TARGET_MACHO
13375 : if (MACHOPIC_INDIRECT
13376 : && machopic_classify_symbol (x) == MACHOPIC_UNDEFINED_FUNCTION)
13377 : name = machopic_indirection_name (x, /*stub_p=*/true);
13378 : #endif
13379 : assemble_name (file, name);
13380 : }
13381 865442 : if (!TARGET_MACHO && !(TARGET_64BIT && TARGET_PECOFF)
13382 865442 : && code == 'P' && ix86_call_use_plt_p (x))
13383 392660 : fputs ("@PLT", file);
13384 : break;
13385 :
13386 2656 : case LABEL_REF:
13387 2656 : x = XEXP (x, 0);
13388 : /* FALLTHRU */
13389 2656 : case CODE_LABEL:
13390 2656 : ASM_GENERATE_INTERNAL_LABEL (buf, "L", CODE_LABEL_NUMBER (x));
13391 2656 : assemble_name (asm_out_file, buf);
13392 2656 : break;
13393 :
13394 2620712 : CASE_CONST_SCALAR_INT:
13395 2620712 : output_addr_const (file, x);
13396 2620712 : break;
13397 :
13398 211042 : case CONST:
13399 : /* This used to output parentheses around the expression,
13400 : but that does not work on the 386 (either ATT or BSD assembler). */
13401 211042 : output_pic_addr_const (file, XEXP (x, 0), code);
13402 211042 : break;
13403 :
13404 0 : case CONST_DOUBLE:
13405 : /* We can't handle floating point constants;
13406 : TARGET_PRINT_OPERAND must handle them. */
13407 0 : output_operand_lossage ("floating constant misused");
13408 0 : break;
13409 :
13410 19113 : case PLUS:
13411 : /* Some assemblers need integer constants to appear first. */
13412 19113 : if (CONST_INT_P (XEXP (x, 0)))
13413 : {
13414 0 : output_pic_addr_const (file, XEXP (x, 0), code);
13415 0 : putc ('+', file);
13416 0 : output_pic_addr_const (file, XEXP (x, 1), code);
13417 : }
13418 : else
13419 : {
13420 19113 : gcc_assert (CONST_INT_P (XEXP (x, 1)));
13421 19113 : output_pic_addr_const (file, XEXP (x, 1), code);
13422 19113 : putc ('+', file);
13423 19113 : output_pic_addr_const (file, XEXP (x, 0), code);
13424 : }
13425 : break;
13426 :
13427 0 : case MINUS:
13428 0 : if (!TARGET_MACHO)
13429 0 : putc (ASSEMBLER_DIALECT == ASM_INTEL ? '(' : '[', file);
13430 0 : output_pic_addr_const (file, XEXP (x, 0), code);
13431 0 : putc ('-', file);
13432 0 : output_pic_addr_const (file, XEXP (x, 1), code);
13433 0 : if (!TARGET_MACHO)
13434 0 : putc (ASSEMBLER_DIALECT == ASM_INTEL ? ')' : ']', file);
13435 0 : break;
13436 :
13437 201397 : case UNSPEC:
13438 201397 : gcc_assert (XVECLEN (x, 0) == 1);
13439 201397 : output_pic_addr_const (file, XVECEXP (x, 0, 0), code);
13440 201397 : switch (XINT (x, 1))
13441 : {
13442 43295 : case UNSPEC_GOT:
13443 43295 : fputs ("@GOT", file);
13444 43295 : break;
13445 78055 : case UNSPEC_GOTOFF:
13446 78055 : fputs ("@GOTOFF", file);
13447 78055 : break;
13448 36 : case UNSPEC_PLTOFF:
13449 36 : fputs ("@PLTOFF", file);
13450 36 : break;
13451 0 : case UNSPEC_PCREL:
13452 0 : fputs (ASSEMBLER_DIALECT == ASM_ATT ?
13453 : "(%rip)" : "[rip]", file);
13454 0 : break;
13455 75792 : case UNSPEC_GOTPCREL:
13456 75792 : fputs (ASSEMBLER_DIALECT == ASM_ATT ?
13457 : "@GOTPCREL(%rip)" : "@GOTPCREL[rip]", file);
13458 75792 : break;
13459 0 : case UNSPEC_GOTTPOFF:
13460 : /* FIXME: This might be @TPOFF in Sun ld too. */
13461 0 : fputs ("@gottpoff", file);
13462 0 : break;
13463 0 : case UNSPEC_TPOFF:
13464 0 : fputs ("@tpoff", file);
13465 0 : break;
13466 1459 : case UNSPEC_NTPOFF:
13467 1459 : if (TARGET_64BIT)
13468 1459 : fputs ("@tpoff", file);
13469 : else
13470 0 : fputs ("@ntpoff", file);
13471 : break;
13472 319 : case UNSPEC_DTPOFF:
13473 319 : fputs ("@dtpoff", file);
13474 319 : break;
13475 2441 : case UNSPEC_GOTNTPOFF:
13476 2441 : if (TARGET_64BIT)
13477 2164 : fputs (ASSEMBLER_DIALECT == ASM_ATT ?
13478 : "@gottpoff(%rip)": "@gottpoff[rip]", file);
13479 : else
13480 277 : fputs ("@gotntpoff", file);
13481 : break;
13482 0 : case UNSPEC_INDNTPOFF:
13483 0 : fputs ("@indntpoff", file);
13484 0 : break;
13485 0 : case UNSPEC_SECREL32:
13486 0 : fputs ("@secrel32", file);
13487 0 : break;
13488 : #if TARGET_MACHO
13489 : case UNSPEC_MACHOPIC_OFFSET:
13490 : putc ('-', file);
13491 : machopic_output_function_base_name (file);
13492 : break;
13493 : #endif
13494 0 : default:
13495 0 : output_operand_lossage ("invalid UNSPEC as operand");
13496 0 : break;
13497 : }
13498 : break;
13499 :
13500 0 : default:
13501 0 : output_operand_lossage ("invalid expression as operand");
13502 : }
13503 3690207 : }
13504 :
13505 : /* This is called from dwarf2out.cc via TARGET_ASM_OUTPUT_DWARF_DTPREL.
13506 : We need to emit DTP-relative relocations. */
13507 :
13508 : static void ATTRIBUTE_UNUSED
13509 689 : i386_output_dwarf_dtprel (FILE *file, int size, rtx x)
13510 : {
13511 689 : fputs (ASM_LONG, file);
13512 689 : output_addr_const (file, x);
13513 : #if TARGET_WIN32_TLS
13514 : fputs ("@secrel32", file);
13515 : #else
13516 689 : fputs ("@dtpoff", file);
13517 : #endif
13518 689 : switch (size)
13519 : {
13520 : case 4:
13521 : break;
13522 554 : case 8:
13523 554 : fputs (", 0", file);
13524 554 : break;
13525 0 : default:
13526 0 : gcc_unreachable ();
13527 : }
13528 689 : }
13529 :
13530 : /* Return true if X is a representation of the PIC register. This copes
13531 : with calls from ix86_find_base_term, where the register might have
13532 : been replaced by a cselib value. */
13533 :
13534 : static bool
13535 26860716 : ix86_pic_register_p (rtx x)
13536 : {
13537 26860716 : if (GET_CODE (x) == VALUE && CSELIB_VAL_PTR (x))
13538 753288 : return (pic_offset_table_rtx
13539 753288 : && rtx_equal_for_cselib_p (x, pic_offset_table_rtx));
13540 26107428 : else if (GET_CODE (x) == UNSPEC && XINT (x, 1) == UNSPEC_SET_GOT)
13541 : return true;
13542 26104337 : else if (!REG_P (x))
13543 : return false;
13544 25497246 : else if (pic_offset_table_rtx)
13545 : {
13546 25477523 : if (REGNO (x) == REGNO (pic_offset_table_rtx))
13547 : return true;
13548 403826 : if (HARD_REGISTER_P (x)
13549 382437 : && !HARD_REGISTER_P (pic_offset_table_rtx)
13550 786263 : && ORIGINAL_REGNO (x) == REGNO (pic_offset_table_rtx))
13551 : return true;
13552 : return false;
13553 : }
13554 : else
13555 19723 : return REGNO (x) == PIC_OFFSET_TABLE_REGNUM;
13556 : }
13557 :
13558 : /* Helper function for ix86_delegitimize_address.
13559 : Attempt to delegitimize TLS local-exec accesses. */
13560 :
13561 : static rtx
13562 3505994139 : ix86_delegitimize_tls_address (rtx orig_x)
13563 : {
13564 3505994139 : rtx x = orig_x, unspec;
13565 3505994139 : struct ix86_address addr;
13566 :
13567 3505994139 : if (!TARGET_TLS_DIRECT_SEG_REFS)
13568 : return orig_x;
13569 3505994139 : if (MEM_P (x))
13570 43252726 : x = XEXP (x, 0);
13571 5034093433 : if (GET_CODE (x) != PLUS || GET_MODE (x) != Pmode)
13572 : return orig_x;
13573 1687442216 : if (ix86_decompose_address (x, &addr) == 0
13574 1950428090 : || addr.seg != DEFAULT_TLS_SEG_REG
13575 270087 : || addr.disp == NULL_RTX
13576 1687660491 : || GET_CODE (addr.disp) != CONST)
13577 : return orig_x;
13578 112395 : unspec = XEXP (addr.disp, 0);
13579 112395 : if (GET_CODE (unspec) == PLUS && CONST_INT_P (XEXP (unspec, 1)))
13580 65649 : unspec = XEXP (unspec, 0);
13581 112395 : if (GET_CODE (unspec) != UNSPEC || XINT (unspec, 1) != UNSPEC_NTPOFF)
13582 : return orig_x;
13583 112332 : x = XVECEXP (unspec, 0, 0);
13584 112332 : gcc_assert (SYMBOL_REF_P (x));
13585 112332 : if (unspec != XEXP (addr.disp, 0))
13586 87152 : x = gen_rtx_PLUS (Pmode, x, XEXP (XEXP (addr.disp, 0), 1));
13587 112332 : if (addr.index)
13588 : {
13589 185 : rtx idx = addr.index;
13590 185 : if (addr.scale != 1)
13591 185 : idx = gen_rtx_MULT (Pmode, idx, GEN_INT (addr.scale));
13592 185 : x = gen_rtx_PLUS (Pmode, idx, x);
13593 : }
13594 112332 : if (addr.base)
13595 2 : x = gen_rtx_PLUS (Pmode, addr.base, x);
13596 112332 : if (MEM_P (orig_x))
13597 198 : x = replace_equiv_address_nv (orig_x, x);
13598 : return x;
13599 : }
13600 :
13601 : /* In the name of slightly smaller debug output, and to cater to
13602 : general assembler lossage, recognize PIC+GOTOFF and turn it back
13603 : into a direct symbol reference.
13604 :
13605 : On Darwin, this is necessary to avoid a crash, because Darwin
13606 : has a different PIC label for each routine but the DWARF debugging
13607 : information is not associated with any particular routine, so it's
13608 : necessary to remove references to the PIC label from RTL stored by
13609 : the DWARF output code.
13610 :
13611 : This helper is used in the normal ix86_delegitimize_address
13612 : entrypoint (e.g. used in the target delegitimization hook) and
13613 : in ix86_find_base_term. As compile time memory optimization, we
13614 : avoid allocating rtxes that will not change anything on the outcome
13615 : of the callers (find_base_value and find_base_term). */
13616 :
13617 : static inline rtx
13618 3530829073 : ix86_delegitimize_address_1 (rtx x, bool base_term_p)
13619 : {
13620 3530829073 : rtx orig_x = delegitimize_mem_from_attrs (x);
13621 : /* addend is NULL or some rtx if x is something+GOTOFF where
13622 : something doesn't include the PIC register. */
13623 3530829073 : rtx addend = NULL_RTX;
13624 : /* reg_addend is NULL or a multiple of some register. */
13625 3530829073 : rtx reg_addend = NULL_RTX;
13626 : /* const_addend is NULL or a const_int. */
13627 3530829073 : rtx const_addend = NULL_RTX;
13628 : /* This is the result, or NULL. */
13629 3530829073 : rtx result = NULL_RTX;
13630 :
13631 3530829073 : x = orig_x;
13632 :
13633 3530829073 : if (MEM_P (x))
13634 62453487 : x = XEXP (x, 0);
13635 :
13636 3530829073 : if (TARGET_64BIT)
13637 : {
13638 258723464 : if (GET_CODE (x) == CONST
13639 8692020 : && GET_CODE (XEXP (x, 0)) == PLUS
13640 6748546 : && GET_MODE (XEXP (x, 0)) == Pmode
13641 6748497 : && CONST_INT_P (XEXP (XEXP (x, 0), 1))
13642 6748497 : && GET_CODE (XEXP (XEXP (x, 0), 0)) == UNSPEC
13643 258727724 : && XINT (XEXP (XEXP (x, 0), 0), 1) == UNSPEC_PCREL)
13644 : {
13645 : /* find_base_{value,term} only care about MEMs with arg_pointer_rtx
13646 : base. A CONST can't be arg_pointer_rtx based. */
13647 0 : if (base_term_p && MEM_P (orig_x))
13648 : return orig_x;
13649 0 : rtx x2 = XVECEXP (XEXP (XEXP (x, 0), 0), 0, 0);
13650 0 : x = gen_rtx_PLUS (Pmode, XEXP (XEXP (x, 0), 1), x2);
13651 0 : if (MEM_P (orig_x))
13652 0 : x = replace_equiv_address_nv (orig_x, x);
13653 0 : return x;
13654 : }
13655 :
13656 258723464 : if (GET_CODE (x) == CONST
13657 8692020 : && GET_CODE (XEXP (x, 0)) == UNSPEC
13658 1943523 : && (XINT (XEXP (x, 0), 1) == UNSPEC_GOTPCREL
13659 656978 : || XINT (XEXP (x, 0), 1) == UNSPEC_PCREL)
13660 1286545 : && (MEM_P (orig_x) || XINT (XEXP (x, 0), 1) == UNSPEC_PCREL))
13661 : {
13662 294957 : x = XVECEXP (XEXP (x, 0), 0, 0);
13663 294957 : if (GET_MODE (orig_x) != GET_MODE (x) && MEM_P (orig_x))
13664 : {
13665 9 : x = lowpart_subreg (GET_MODE (orig_x), x, GET_MODE (x));
13666 9 : if (x == NULL_RTX)
13667 : return orig_x;
13668 : }
13669 294957 : return x;
13670 : }
13671 :
13672 258428507 : if (ix86_cmodel != CM_MEDIUM_PIC && ix86_cmodel != CM_LARGE_PIC)
13673 258426845 : return ix86_delegitimize_tls_address (orig_x);
13674 :
13675 : /* Fall thru into the code shared with -m32 for -mcmodel=large -fpic
13676 : and -mcmodel=medium -fpic. */
13677 : }
13678 :
13679 3272107271 : if (GET_CODE (x) != PLUS
13680 1552638847 : || GET_CODE (XEXP (x, 1)) != CONST)
13681 3245794960 : return ix86_delegitimize_tls_address (orig_x);
13682 :
13683 26312311 : if (ix86_pic_register_p (XEXP (x, 0)))
13684 : /* %ebx + GOT/GOTOFF */
13685 : ;
13686 1282201 : else if (GET_CODE (XEXP (x, 0)) == PLUS)
13687 : {
13688 : /* %ebx + %reg * scale + GOT/GOTOFF */
13689 472103 : reg_addend = XEXP (x, 0);
13690 472103 : if (ix86_pic_register_p (XEXP (reg_addend, 0)))
13691 395801 : reg_addend = XEXP (reg_addend, 1);
13692 76302 : else if (ix86_pic_register_p (XEXP (reg_addend, 1)))
13693 44749 : reg_addend = XEXP (reg_addend, 0);
13694 : else
13695 : {
13696 31553 : reg_addend = NULL_RTX;
13697 31553 : addend = XEXP (x, 0);
13698 : }
13699 : }
13700 : else
13701 : addend = XEXP (x, 0);
13702 :
13703 26312311 : x = XEXP (XEXP (x, 1), 0);
13704 26312311 : if (GET_CODE (x) == PLUS
13705 1450257 : && CONST_INT_P (XEXP (x, 1)))
13706 : {
13707 1450257 : const_addend = XEXP (x, 1);
13708 1450257 : x = XEXP (x, 0);
13709 : }
13710 :
13711 26312311 : if (GET_CODE (x) == UNSPEC
13712 25635928 : && ((XINT (x, 1) == UNSPEC_GOT && MEM_P (orig_x) && !addend)
13713 6730124 : || (XINT (x, 1) == UNSPEC_GOTOFF && !MEM_P (orig_x))
13714 1095955 : || (XINT (x, 1) == UNSPEC_PLTOFF && ix86_cmodel == CM_LARGE_PIC
13715 4 : && !MEM_P (orig_x) && !addend)))
13716 24539977 : result = XVECEXP (x, 0, 0);
13717 :
13718 24539977 : if (!TARGET_64BIT && TARGET_MACHO && darwin_local_data_pic (x)
13719 : && !MEM_P (orig_x))
13720 : result = XVECEXP (x, 0, 0);
13721 :
13722 24539977 : if (! result)
13723 1772334 : return ix86_delegitimize_tls_address (orig_x);
13724 :
13725 : /* For (PLUS something CONST_INT) both find_base_{value,term} just
13726 : recurse on the first operand. */
13727 24539977 : if (const_addend && !base_term_p)
13728 356190 : result = gen_rtx_CONST (Pmode, gen_rtx_PLUS (Pmode, result, const_addend));
13729 24539977 : if (reg_addend)
13730 856908 : result = gen_rtx_PLUS (Pmode, reg_addend, result);
13731 24539977 : if (addend)
13732 : {
13733 : /* If the rest of original X doesn't involve the PIC register, add
13734 : addend and subtract pic_offset_table_rtx. This can happen e.g.
13735 : for code like:
13736 : leal (%ebx, %ecx, 4), %ecx
13737 : ...
13738 : movl foo@GOTOFF(%ecx), %edx
13739 : in which case we return (%ecx - %ebx) + foo
13740 : or (%ecx - _GLOBAL_OFFSET_TABLE_) + foo if pseudo_pic_reg
13741 : and reload has completed. Don't do the latter for debug,
13742 : as _GLOBAL_OFFSET_TABLE_ can't be expressed in the assembly. */
13743 137444 : if (pic_offset_table_rtx
13744 137444 : && (!reload_completed || !ix86_use_pseudo_pic_reg ()))
13745 2364 : result = gen_rtx_PLUS (Pmode, gen_rtx_MINUS (Pmode, copy_rtx (addend),
13746 : pic_offset_table_rtx),
13747 : result);
13748 136656 : else if (base_term_p
13749 130384 : && pic_offset_table_rtx
13750 : && !TARGET_MACHO
13751 : && !TARGET_VXWORKS_VAROFF)
13752 : {
13753 260768 : rtx tmp = gen_rtx_SYMBOL_REF (Pmode, GOT_SYMBOL_NAME);
13754 260768 : tmp = gen_rtx_MINUS (Pmode, copy_rtx (addend), tmp);
13755 260768 : result = gen_rtx_PLUS (Pmode, tmp, result);
13756 130384 : }
13757 : else
13758 : return orig_x;
13759 : }
13760 49067325 : if (GET_MODE (orig_x) != Pmode && MEM_P (orig_x))
13761 : {
13762 0 : result = lowpart_subreg (GET_MODE (orig_x), result, Pmode);
13763 0 : if (result == NULL_RTX)
13764 : return orig_x;
13765 : }
13766 : return result;
13767 : }
13768 :
13769 : /* The normal instantiation of the above template. */
13770 :
13771 : static rtx
13772 329556950 : ix86_delegitimize_address (rtx x)
13773 : {
13774 329556950 : return ix86_delegitimize_address_1 (x, false);
13775 : }
13776 :
13777 : /* If X is a machine specific address (i.e. a symbol or label being
13778 : referenced as a displacement from the GOT implemented using an
13779 : UNSPEC), then return the base term. Otherwise return X. */
13780 :
13781 : rtx
13782 6721052652 : ix86_find_base_term (rtx x)
13783 : {
13784 6721052652 : rtx term;
13785 :
13786 6721052652 : if (TARGET_64BIT)
13787 : {
13788 3519780529 : if (GET_CODE (x) != CONST)
13789 : return x;
13790 42321833 : term = XEXP (x, 0);
13791 42321833 : if (GET_CODE (term) == PLUS
13792 42306982 : && CONST_INT_P (XEXP (term, 1)))
13793 42306982 : term = XEXP (term, 0);
13794 42321833 : if (GET_CODE (term) != UNSPEC
13795 39853 : || (XINT (term, 1) != UNSPEC_GOTPCREL
13796 39821 : && XINT (term, 1) != UNSPEC_PCREL))
13797 : return x;
13798 :
13799 32 : return XVECEXP (term, 0, 0);
13800 : }
13801 :
13802 3201272123 : return ix86_delegitimize_address_1 (x, true);
13803 : }
13804 :
13805 : /* Return true if X shouldn't be emitted into the debug info.
13806 : Disallow UNSPECs other than @gotoff - we can't emit _GLOBAL_OFFSET_TABLE_
13807 : symbol easily into the .debug_info section, so we need not to
13808 : delegitimize, but instead assemble as @gotoff.
13809 : Disallow _GLOBAL_OFFSET_TABLE_ SYMBOL_REF - the assembler magically
13810 : assembles that as _GLOBAL_OFFSET_TABLE_-. expression. */
13811 :
13812 : static bool
13813 1860781 : ix86_const_not_ok_for_debug_p (rtx x)
13814 : {
13815 1860781 : if (GET_CODE (x) == UNSPEC && XINT (x, 1) != UNSPEC_GOTOFF)
13816 : return true;
13817 :
13818 1860761 : if (SYMBOL_REF_P (x) && strcmp (XSTR (x, 0), GOT_SYMBOL_NAME) == 0)
13819 0 : return true;
13820 :
13821 : return false;
13822 : }
13823 :
13824 : static void
13825 7226679 : put_condition_code (enum rtx_code code, machine_mode mode, bool reverse,
13826 : bool fp, FILE *file)
13827 : {
13828 7226679 : const char *suffix;
13829 :
13830 7226679 : if (mode == CCFPmode)
13831 : {
13832 570087 : code = ix86_fp_compare_code_to_integer (code);
13833 570087 : mode = CCmode;
13834 : }
13835 7226679 : if (reverse)
13836 225321 : code = reverse_condition (code);
13837 :
13838 7226679 : switch (code)
13839 : {
13840 2788472 : case EQ:
13841 2788472 : gcc_assert (mode != CCGZmode);
13842 2788472 : switch (mode)
13843 : {
13844 : case E_CCAmode:
13845 : suffix = "a";
13846 : break;
13847 : case E_CCCmode:
13848 26541 : suffix = "c";
13849 : break;
13850 : case E_CCOmode:
13851 7226679 : suffix = "o";
13852 : break;
13853 : case E_CCPmode:
13854 235017 : suffix = "p";
13855 : break;
13856 : case E_CCSmode:
13857 133079 : suffix = "s";
13858 : break;
13859 2768681 : default:
13860 2768681 : suffix = "e";
13861 2768681 : break;
13862 : }
13863 : break;
13864 2343086 : case NE:
13865 2343086 : gcc_assert (mode != CCGZmode);
13866 2343086 : switch (mode)
13867 : {
13868 : case E_CCAmode:
13869 : suffix = "na";
13870 : break;
13871 : case E_CCCmode:
13872 12001 : suffix = "nc";
13873 : break;
13874 11037 : case E_CCOmode:
13875 11037 : suffix = "no";
13876 11037 : break;
13877 : case E_CCPmode:
13878 5006 : suffix = "np";
13879 : break;
13880 : case E_CCSmode:
13881 53455 : suffix = "ns";
13882 : break;
13883 2330163 : default:
13884 2330163 : suffix = "ne";
13885 2330163 : break;
13886 : }
13887 : break;
13888 264255 : case GT:
13889 264255 : gcc_assert (mode == CCmode || mode == CCNOmode || mode == CCGCmode);
13890 : suffix = "g";
13891 : break;
13892 185240 : case GTU:
13893 : /* ??? Use "nbe" instead of "a" for fcmov lossage on some assemblers.
13894 : Those same assemblers have the same but opposite lossage on cmov. */
13895 185240 : if (mode == CCmode)
13896 185302 : suffix = fp ? "nbe" : "a";
13897 : else
13898 0 : gcc_unreachable ();
13899 : break;
13900 243921 : case LT:
13901 243921 : switch (mode)
13902 : {
13903 : case E_CCNOmode:
13904 : case E_CCGOCmode:
13905 : suffix = "s";
13906 : break;
13907 :
13908 : case E_CCmode:
13909 : case E_CCGCmode:
13910 : case E_CCGZmode:
13911 7226679 : suffix = "l";
13912 : break;
13913 :
13914 0 : default:
13915 0 : gcc_unreachable ();
13916 : }
13917 : break;
13918 439494 : case LTU:
13919 439494 : if (mode == CCmode || mode == CCGZmode)
13920 : suffix = "b";
13921 25210 : else if (mode == CCCmode)
13922 26541 : suffix = fp ? "b" : "c";
13923 : else
13924 0 : gcc_unreachable ();
13925 : break;
13926 147306 : case GE:
13927 147306 : switch (mode)
13928 : {
13929 : case E_CCNOmode:
13930 : case E_CCGOCmode:
13931 : suffix = "ns";
13932 : break;
13933 :
13934 : case E_CCmode:
13935 : case E_CCGCmode:
13936 : case E_CCGZmode:
13937 7226679 : suffix = "ge";
13938 : break;
13939 :
13940 0 : default:
13941 0 : gcc_unreachable ();
13942 : }
13943 : break;
13944 197734 : case GEU:
13945 197734 : if (mode == CCmode || mode == CCGZmode)
13946 : suffix = "nb";
13947 10135 : else if (mode == CCCmode)
13948 12001 : suffix = fp ? "nb" : "nc";
13949 : else
13950 0 : gcc_unreachable ();
13951 : break;
13952 253340 : case LE:
13953 253340 : gcc_assert (mode == CCmode || mode == CCGCmode || mode == CCNOmode);
13954 : suffix = "le";
13955 : break;
13956 123806 : case LEU:
13957 123806 : if (mode == CCmode)
13958 : suffix = "be";
13959 : else
13960 0 : gcc_unreachable ();
13961 : break;
13962 235017 : case UNORDERED:
13963 235024 : suffix = fp ? "u" : "p";
13964 : break;
13965 5008 : case ORDERED:
13966 5013 : suffix = fp ? "nu" : "np";
13967 : break;
13968 0 : default:
13969 0 : gcc_unreachable ();
13970 : }
13971 7226679 : fputs (suffix, file);
13972 7226679 : }
13973 :
13974 : /* Print the name of register X to FILE based on its machine mode and number.
13975 : If CODE is 'w', pretend the mode is HImode.
13976 : If CODE is 'b', pretend the mode is QImode.
13977 : If CODE is 'k', pretend the mode is SImode.
13978 : If CODE is 'q', pretend the mode is DImode.
13979 : If CODE is 'x', pretend the mode is V4SFmode.
13980 : If CODE is 't', pretend the mode is V8SFmode.
13981 : If CODE is 'g', pretend the mode is V16SFmode.
13982 : If CODE is 'h', pretend the reg is the 'high' byte register.
13983 : If CODE is 'y', print "st(0)" instead of "st", if the reg is stack op.
13984 : If CODE is 'd', duplicate the operand for AVX instruction.
13985 : If CODE is 'V', print naked full integer register name without %.
13986 : */
13987 :
13988 : void
13989 124647948 : print_reg (rtx x, int code, FILE *file)
13990 : {
13991 124647948 : const char *reg;
13992 124647948 : int msize;
13993 124647948 : unsigned int regno;
13994 124647948 : bool duplicated;
13995 :
13996 124647948 : if (ASSEMBLER_DIALECT == ASM_ATT && code != 'V')
13997 124645541 : putc ('%', file);
13998 :
13999 124647948 : if (x == pc_rtx)
14000 : {
14001 5768168 : gcc_assert (TARGET_64BIT);
14002 5768168 : fputs ("rip", file);
14003 5768168 : return;
14004 : }
14005 :
14006 118879780 : if (code == 'y' && STACK_TOP_P (x))
14007 : {
14008 290465 : fputs ("st(0)", file);
14009 290465 : return;
14010 : }
14011 :
14012 118589315 : if (code == 'w')
14013 : msize = 2;
14014 : else if (code == 'b')
14015 : msize = 1;
14016 : else if (code == 'k')
14017 : msize = 4;
14018 : else if (code == 'q')
14019 : msize = 8;
14020 : else if (code == 'h')
14021 : msize = 0;
14022 : else if (code == 'x')
14023 : msize = 16;
14024 : else if (code == 't')
14025 : msize = 32;
14026 : else if (code == 'g')
14027 : msize = 64;
14028 : else
14029 202726574 : msize = GET_MODE_SIZE (GET_MODE (x));
14030 :
14031 118589315 : regno = REGNO (x);
14032 :
14033 118589315 : if (regno == ARG_POINTER_REGNUM
14034 118589315 : || regno == FRAME_POINTER_REGNUM
14035 118589315 : || regno == FPSR_REG)
14036 : {
14037 0 : output_operand_lossage
14038 0 : ("invalid use of register '%s'", reg_names[regno]);
14039 0 : return;
14040 : }
14041 118589315 : else if (regno == FLAGS_REG)
14042 : {
14043 1 : output_operand_lossage ("invalid use of asm flag output");
14044 1 : return;
14045 : }
14046 :
14047 118589314 : if (code == 'V')
14048 : {
14049 1 : if (GENERAL_REGNO_P (regno))
14050 2 : msize = GET_MODE_SIZE (word_mode);
14051 : else
14052 0 : error ("%<V%> modifier on non-integer register");
14053 : }
14054 :
14055 118589314 : duplicated = code == 'd' && TARGET_AVX;
14056 :
14057 118589314 : switch (msize)
14058 : {
14059 78903316 : case 16:
14060 78903316 : case 12:
14061 78903316 : case 8:
14062 147715619 : if (GENERAL_REGNO_P (regno) && msize > GET_MODE_SIZE (word_mode))
14063 5 : warning (0, "unsupported size for integer register");
14064 : /* FALLTHRU */
14065 115096622 : case 4:
14066 115096622 : if (LEGACY_INT_REGNO_P (regno))
14067 124420040 : putc (msize > 4 && TARGET_64BIT ? 'r' : 'e', file);
14068 : /* FALLTHRU */
14069 115991725 : case 2:
14070 22389549 : normal:
14071 115991725 : reg = hi_reg_name[regno];
14072 115991725 : break;
14073 2332865 : case 1:
14074 2332865 : if (regno >= ARRAY_SIZE (qi_reg_name))
14075 273327 : goto normal;
14076 2059538 : if (!ANY_QI_REGNO_P (regno))
14077 0 : error ("unsupported size for integer register");
14078 2059538 : reg = qi_reg_name[regno];
14079 2059538 : break;
14080 26407 : case 0:
14081 26407 : if (regno >= ARRAY_SIZE (qi_high_reg_name))
14082 0 : goto normal;
14083 26407 : reg = qi_high_reg_name[regno];
14084 26407 : break;
14085 511644 : case 32:
14086 511644 : case 64:
14087 511644 : if (SSE_REGNO_P (regno))
14088 : {
14089 511644 : gcc_assert (!duplicated);
14090 716737 : putc (msize == 32 ? 'y' : 'z', file);
14091 511644 : reg = hi_reg_name[regno] + 1;
14092 511644 : break;
14093 : }
14094 0 : goto normal;
14095 0 : default:
14096 0 : gcc_unreachable ();
14097 : }
14098 :
14099 118589314 : fputs (reg, file);
14100 :
14101 : /* Irritatingly, AMD extended registers use
14102 : different naming convention: "r%d[bwd]" */
14103 118589314 : if (REX_INT_REGNO_P (regno) || REX2_INT_REGNO_P (regno))
14104 : {
14105 10561082 : gcc_assert (TARGET_64BIT);
14106 10561082 : switch (msize)
14107 : {
14108 0 : case 0:
14109 0 : error ("extended registers have no high halves");
14110 0 : break;
14111 181373 : case 1:
14112 181373 : putc ('b', file);
14113 181373 : break;
14114 30749 : case 2:
14115 30749 : putc ('w', file);
14116 30749 : break;
14117 2530763 : case 4:
14118 2530763 : putc ('d', file);
14119 2530763 : break;
14120 : case 8:
14121 : /* no suffix */
14122 : break;
14123 0 : default:
14124 0 : error ("unsupported operand size for extended register");
14125 0 : break;
14126 : }
14127 10561082 : return;
14128 : }
14129 :
14130 108028232 : if (duplicated)
14131 : {
14132 16864 : if (ASSEMBLER_DIALECT == ASM_ATT)
14133 16847 : fprintf (file, ", %%%s", reg);
14134 : else
14135 17 : fprintf (file, ", %s", reg);
14136 : }
14137 : }
14138 :
14139 : /* Meaning of CODE:
14140 : L,W,B,Q,S,T -- print the opcode suffix for specified size of operand.
14141 : C -- print opcode suffix for set/cmov insn.
14142 : c -- like C, but print reversed condition
14143 : F,f -- likewise, but for floating-point.
14144 : O -- if HAVE_AS_IX86_CMOV_SUN_SYNTAX, expand to "w.", "l." or "q.",
14145 : otherwise nothing
14146 : R -- print embedded rounding and sae.
14147 : r -- print only sae.
14148 : z -- print the opcode suffix for the size of the current operand.
14149 : Z -- likewise, with special suffixes for x87 instructions.
14150 : * -- print a star (in certain assembler syntax)
14151 : A -- print an absolute memory reference.
14152 : E -- print address with DImode register names if TARGET_64BIT.
14153 : w -- print the operand as if it's a "word" (HImode) even if it isn't.
14154 : s -- print a shift double count, followed by the assemblers argument
14155 : delimiter.
14156 : b -- print the QImode name of the register for the indicated operand.
14157 : %b0 would print %al if operands[0] is reg 0.
14158 : w -- likewise, print the HImode name of the register.
14159 : k -- likewise, print the SImode name of the register.
14160 : q -- likewise, print the DImode name of the register.
14161 : x -- likewise, print the V4SFmode name of the register.
14162 : t -- likewise, print the V8SFmode name of the register.
14163 : g -- likewise, print the V16SFmode name of the register.
14164 : h -- print the QImode name for a "high" register, either ah, bh, ch or dh.
14165 : y -- print "st(0)" instead of "st" as a register.
14166 : d -- print duplicated register operand for AVX instruction.
14167 : D -- print condition for SSE cmp instruction.
14168 : P -- if PIC, print an @PLT suffix. For -fno-plt, load function
14169 : address from GOT.
14170 : p -- print raw symbol name.
14171 : X -- don't print any sort of PIC '@' suffix for a symbol.
14172 : & -- print some in-use local-dynamic symbol name.
14173 : H -- print a memory address offset by 8; used for sse high-parts
14174 : Y -- print condition for XOP pcom* instruction.
14175 : V -- print naked full integer register name without %.
14176 : v -- print segment override prefix
14177 : + -- print a branch hint as 'cs' or 'ds' prefix
14178 : ; -- print a semicolon (after prefixes due to bug in older gas).
14179 : ~ -- print "i" if TARGET_AVX2, "f" otherwise.
14180 : ^ -- print addr32 prefix if Pmode != word_mode
14181 : M -- print addr32 prefix for TARGET_X32 with VSIB address.
14182 : ! -- print NOTRACK prefix for jxx/call/ret instructions if required.
14183 : N -- print maskz if it's constant 0 operand.
14184 : G -- print embedded flag for ccmp/ctest.
14185 : */
14186 :
14187 : void
14188 178745949 : ix86_print_operand (FILE *file, rtx x, int code)
14189 : {
14190 178936891 : if (code)
14191 : {
14192 62678639 : switch (code)
14193 : {
14194 190938 : case 'A':
14195 190938 : switch (ASSEMBLER_DIALECT)
14196 : {
14197 190938 : case ASM_ATT:
14198 190938 : putc ('*', file);
14199 190938 : break;
14200 :
14201 0 : case ASM_INTEL:
14202 : /* Intel syntax. For absolute addresses, registers should not
14203 : be surrounded by braces. */
14204 0 : if (!REG_P (x))
14205 : {
14206 0 : putc ('[', file);
14207 0 : ix86_print_operand (file, x, 0);
14208 0 : putc (']', file);
14209 0 : return;
14210 : }
14211 : break;
14212 :
14213 0 : default:
14214 0 : gcc_unreachable ();
14215 : }
14216 :
14217 190938 : ix86_print_operand (file, x, 0);
14218 190938 : return;
14219 :
14220 3599845 : case 'E':
14221 : /* Wrap address in an UNSPEC to declare special handling. */
14222 3599845 : if (TARGET_64BIT)
14223 3114154 : x = gen_rtx_UNSPEC (DImode, gen_rtvec (1, x), UNSPEC_LEA_ADDR);
14224 :
14225 3599845 : output_address (VOIDmode, x);
14226 3599845 : return;
14227 :
14228 0 : case 'L':
14229 0 : if (ASSEMBLER_DIALECT == ASM_ATT)
14230 0 : putc ('l', file);
14231 0 : return;
14232 :
14233 0 : case 'W':
14234 0 : if (ASSEMBLER_DIALECT == ASM_ATT)
14235 0 : putc ('w', file);
14236 0 : return;
14237 :
14238 0 : case 'B':
14239 0 : if (ASSEMBLER_DIALECT == ASM_ATT)
14240 0 : putc ('b', file);
14241 0 : return;
14242 :
14243 0 : case 'Q':
14244 0 : if (ASSEMBLER_DIALECT == ASM_ATT)
14245 0 : putc ('l', file);
14246 0 : return;
14247 :
14248 0 : case 'S':
14249 0 : if (ASSEMBLER_DIALECT == ASM_ATT)
14250 0 : putc ('s', file);
14251 0 : return;
14252 :
14253 0 : case 'T':
14254 0 : if (ASSEMBLER_DIALECT == ASM_ATT)
14255 0 : putc ('t', file);
14256 0 : return;
14257 :
14258 : case 'O':
14259 : #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
14260 : if (ASSEMBLER_DIALECT != ASM_ATT)
14261 : return;
14262 :
14263 : switch (GET_MODE_SIZE (GET_MODE (x)))
14264 : {
14265 : case 2:
14266 : putc ('w', file);
14267 : break;
14268 :
14269 : case 4:
14270 : putc ('l', file);
14271 : break;
14272 :
14273 : case 8:
14274 : putc ('q', file);
14275 : break;
14276 :
14277 : default:
14278 : output_operand_lossage ("invalid operand size for operand "
14279 : "code 'O'");
14280 : return;
14281 : }
14282 :
14283 : putc ('.', file);
14284 : #endif
14285 : return;
14286 :
14287 38097 : case 'z':
14288 38097 : if (GET_MODE_CLASS (GET_MODE (x)) == MODE_INT)
14289 : {
14290 : /* Opcodes don't get size suffixes if using Intel opcodes. */
14291 38095 : if (ASSEMBLER_DIALECT == ASM_INTEL)
14292 : return;
14293 :
14294 76190 : switch (GET_MODE_SIZE (GET_MODE (x)))
14295 : {
14296 6 : case 1:
14297 6 : putc ('b', file);
14298 6 : return;
14299 :
14300 6 : case 2:
14301 6 : putc ('w', file);
14302 6 : return;
14303 :
14304 37598 : case 4:
14305 37598 : putc ('l', file);
14306 37598 : return;
14307 :
14308 485 : case 8:
14309 485 : putc ('q', file);
14310 485 : return;
14311 :
14312 0 : default:
14313 0 : output_operand_lossage ("invalid operand size for operand "
14314 : "code 'z'");
14315 0 : return;
14316 : }
14317 : }
14318 :
14319 2 : if (GET_MODE_CLASS (GET_MODE (x)) == MODE_FLOAT)
14320 : {
14321 1 : if (this_is_asm_operands)
14322 1 : warning_for_asm (this_is_asm_operands,
14323 : "non-integer operand used with operand code %<z%>");
14324 : else
14325 0 : warning (0, "non-integer operand used with operand code %<z%>");
14326 : }
14327 : /* FALLTHRU */
14328 :
14329 386077 : case 'Z':
14330 : /* 387 opcodes don't get size suffixes if using Intel opcodes. */
14331 386077 : if (ASSEMBLER_DIALECT == ASM_INTEL)
14332 : return;
14333 :
14334 386077 : if (GET_MODE_CLASS (GET_MODE (x)) == MODE_INT)
14335 : {
14336 29434 : switch (GET_MODE_SIZE (GET_MODE (x)))
14337 : {
14338 3525 : case 2:
14339 : #ifdef HAVE_AS_IX86_FILDS
14340 3525 : putc ('s', file);
14341 : #endif
14342 3525 : return;
14343 :
14344 3989 : case 4:
14345 3989 : putc ('l', file);
14346 3989 : return;
14347 :
14348 7203 : case 8:
14349 : #ifdef HAVE_AS_IX86_FILDQ
14350 7203 : putc ('q', file);
14351 : #else
14352 : fputs ("ll", file);
14353 : #endif
14354 7203 : return;
14355 :
14356 : default:
14357 : break;
14358 : }
14359 : }
14360 371360 : else if (GET_MODE_CLASS (GET_MODE (x)) == MODE_FLOAT)
14361 : {
14362 : /* 387 opcodes don't get size suffixes
14363 : if the operands are registers. */
14364 371358 : if (STACK_REG_P (x))
14365 : return;
14366 :
14367 691864 : switch (GET_MODE_SIZE (GET_MODE (x)))
14368 : {
14369 23577 : case 4:
14370 23577 : putc ('s', file);
14371 23577 : return;
14372 :
14373 32884 : case 8:
14374 32884 : putc ('l', file);
14375 32884 : return;
14376 :
14377 289469 : case 12:
14378 289469 : case 16:
14379 289469 : putc ('t', file);
14380 289469 : return;
14381 :
14382 : default:
14383 : break;
14384 : }
14385 : }
14386 : else
14387 : {
14388 2 : output_operand_lossage ("invalid operand type used with "
14389 : "operand code '%c'", code);
14390 2 : return;
14391 : }
14392 :
14393 2 : output_operand_lossage ("invalid operand size for operand code '%c'",
14394 : code);
14395 2 : return;
14396 :
14397 : case 'd':
14398 : case 'b':
14399 : case 'w':
14400 : case 'k':
14401 : case 'q':
14402 : case 'h':
14403 : case 't':
14404 : case 'g':
14405 : case 'y':
14406 : case 'x':
14407 : case 'X':
14408 : case 'P':
14409 : case 'p':
14410 : case 'V':
14411 : break;
14412 :
14413 0 : case 's':
14414 0 : if (CONST_INT_P (x) || ! SHIFT_DOUBLE_OMITS_COUNT)
14415 : {
14416 0 : ix86_print_operand (file, x, 0);
14417 0 : fputs (", ", file);
14418 : }
14419 0 : return;
14420 :
14421 494 : case 'Y':
14422 494 : switch (GET_CODE (x))
14423 : {
14424 182 : case NE:
14425 182 : fputs ("neq", file);
14426 182 : break;
14427 32 : case EQ:
14428 32 : fputs ("eq", file);
14429 32 : break;
14430 64 : case GE:
14431 64 : case GEU:
14432 64 : fputs (INTEGRAL_MODE_P (GET_MODE (x)) ? "ge" : "unlt", file);
14433 64 : break;
14434 40 : case GT:
14435 40 : case GTU:
14436 40 : fputs (INTEGRAL_MODE_P (GET_MODE (x)) ? "gt" : "unle", file);
14437 40 : break;
14438 64 : case LE:
14439 64 : case LEU:
14440 64 : fputs ("le", file);
14441 64 : break;
14442 112 : case LT:
14443 112 : case LTU:
14444 112 : fputs ("lt", file);
14445 112 : break;
14446 0 : case UNORDERED:
14447 0 : fputs ("unord", file);
14448 0 : break;
14449 0 : case ORDERED:
14450 0 : fputs ("ord", file);
14451 0 : break;
14452 0 : case UNEQ:
14453 0 : fputs ("ueq", file);
14454 0 : break;
14455 0 : case UNGE:
14456 0 : fputs ("nlt", file);
14457 0 : break;
14458 0 : case UNGT:
14459 0 : fputs ("nle", file);
14460 0 : break;
14461 0 : case UNLE:
14462 0 : fputs ("ule", file);
14463 0 : break;
14464 0 : case UNLT:
14465 0 : fputs ("ult", file);
14466 0 : break;
14467 0 : case LTGT:
14468 0 : fputs ("une", file);
14469 0 : break;
14470 0 : default:
14471 0 : output_operand_lossage ("operand is not a condition code, "
14472 : "invalid operand code 'Y'");
14473 0 : return;
14474 : }
14475 494 : return;
14476 :
14477 10193 : case 'D':
14478 : /* Little bit of braindamage here. The SSE compare instructions
14479 : does use completely different names for the comparisons that the
14480 : fp conditional moves. */
14481 10193 : switch (GET_CODE (x))
14482 : {
14483 15 : case UNEQ:
14484 15 : if (TARGET_AVX)
14485 : {
14486 15 : fputs ("eq_us", file);
14487 15 : break;
14488 : }
14489 : /* FALLTHRU */
14490 5249 : case EQ:
14491 5249 : fputs ("eq", file);
14492 5249 : break;
14493 0 : case UNLT:
14494 0 : if (TARGET_AVX)
14495 : {
14496 0 : fputs ("nge", file);
14497 0 : break;
14498 : }
14499 : /* FALLTHRU */
14500 1728 : case LT:
14501 1728 : fputs ("lt", file);
14502 1728 : break;
14503 0 : case UNLE:
14504 0 : if (TARGET_AVX)
14505 : {
14506 0 : fputs ("ngt", file);
14507 0 : break;
14508 : }
14509 : /* FALLTHRU */
14510 839 : case LE:
14511 839 : fputs ("le", file);
14512 839 : break;
14513 100 : case UNORDERED:
14514 100 : fputs ("unord", file);
14515 100 : break;
14516 12 : case LTGT:
14517 12 : if (TARGET_AVX)
14518 : {
14519 12 : fputs ("neq_oq", file);
14520 12 : break;
14521 : }
14522 : /* FALLTHRU */
14523 915 : case NE:
14524 915 : fputs ("neq", file);
14525 915 : break;
14526 0 : case GE:
14527 0 : if (TARGET_AVX)
14528 : {
14529 0 : fputs ("ge", file);
14530 0 : break;
14531 : }
14532 : /* FALLTHRU */
14533 453 : case UNGE:
14534 453 : fputs ("nlt", file);
14535 453 : break;
14536 0 : case GT:
14537 0 : if (TARGET_AVX)
14538 : {
14539 0 : fputs ("gt", file);
14540 0 : break;
14541 : }
14542 : /* FALLTHRU */
14543 799 : case UNGT:
14544 799 : fputs ("nle", file);
14545 799 : break;
14546 83 : case ORDERED:
14547 83 : fputs ("ord", file);
14548 83 : break;
14549 0 : default:
14550 0 : output_operand_lossage ("operand is not a condition code, "
14551 : "invalid operand code 'D'");
14552 0 : return;
14553 : }
14554 10193 : return;
14555 :
14556 7226679 : case 'F':
14557 7226679 : case 'f':
14558 : #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
14559 : if (ASSEMBLER_DIALECT == ASM_ATT)
14560 : putc ('.', file);
14561 : gcc_fallthrough ();
14562 : #endif
14563 :
14564 7226679 : case 'C':
14565 7226679 : case 'c':
14566 7226679 : if (!COMPARISON_P (x))
14567 : {
14568 0 : output_operand_lossage ("operand is not a condition code, "
14569 : "invalid operand code '%c'", code);
14570 0 : return;
14571 : }
14572 7226679 : put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)),
14573 7226679 : code == 'c' || code == 'f',
14574 7226679 : code == 'F' || code == 'f',
14575 : file);
14576 7226679 : return;
14577 :
14578 21 : case 'G':
14579 21 : {
14580 21 : int dfv = INTVAL (x);
14581 21 : const char *dfv_suffix = ix86_ccmp_dfv_mapping[dfv];
14582 21 : fputs (dfv_suffix, file);
14583 : }
14584 21 : return;
14585 :
14586 1292 : case 'H':
14587 1292 : if (!offsettable_memref_p (x))
14588 : {
14589 1 : output_operand_lossage ("operand is not an offsettable memory "
14590 : "reference, invalid operand code 'H'");
14591 1 : return;
14592 : }
14593 : /* It doesn't actually matter what mode we use here, as we're
14594 : only going to use this for printing. */
14595 1291 : x = adjust_address_nv (x, DImode, 8);
14596 : /* Output 'qword ptr' for intel assembler dialect. */
14597 1291 : if (ASSEMBLER_DIALECT == ASM_INTEL)
14598 0 : code = 'q';
14599 : break;
14600 :
14601 75841 : case 'K':
14602 75841 : if (!CONST_INT_P (x))
14603 : {
14604 1 : output_operand_lossage ("operand is not an integer, invalid "
14605 : "operand code 'K'");
14606 1 : return;
14607 : }
14608 :
14609 75840 : if (INTVAL (x) & IX86_HLE_ACQUIRE)
14610 : #ifdef HAVE_AS_IX86_HLE
14611 22 : fputs ("xacquire ", file);
14612 : #else
14613 : fputs ("\n" ASM_BYTE "0xf2\n\t", file);
14614 : #endif
14615 75818 : else if (INTVAL (x) & IX86_HLE_RELEASE)
14616 : #ifdef HAVE_AS_IX86_HLE
14617 24 : fputs ("xrelease ", file);
14618 : #else
14619 : fputs ("\n" ASM_BYTE "0xf3\n\t", file);
14620 : #endif
14621 : /* We do not want to print value of the operand. */
14622 75840 : return;
14623 :
14624 43023 : case 'N':
14625 43023 : if (x == const0_rtx || x == CONST0_RTX (GET_MODE (x)))
14626 15508 : fputs ("{z}", file);
14627 43023 : return;
14628 :
14629 4054 : case 'r':
14630 4054 : if (!CONST_INT_P (x) || INTVAL (x) != ROUND_SAE)
14631 : {
14632 2 : output_operand_lossage ("operand is not a specific integer, "
14633 : "invalid operand code 'r'");
14634 2 : return;
14635 : }
14636 :
14637 4052 : if (ASSEMBLER_DIALECT == ASM_INTEL)
14638 1 : fputs (", ", file);
14639 :
14640 4052 : fputs ("{sae}", file);
14641 :
14642 4052 : if (ASSEMBLER_DIALECT == ASM_ATT)
14643 4051 : fputs (", ", file);
14644 :
14645 4052 : return;
14646 :
14647 5969 : case 'R':
14648 5969 : if (!CONST_INT_P (x))
14649 : {
14650 1 : output_operand_lossage ("operand is not an integer, invalid "
14651 : "operand code 'R'");
14652 1 : return;
14653 : }
14654 :
14655 5968 : if (ASSEMBLER_DIALECT == ASM_INTEL)
14656 6 : fputs (", ", file);
14657 :
14658 5968 : switch (INTVAL (x))
14659 : {
14660 5153 : case ROUND_NEAREST_INT | ROUND_SAE:
14661 5153 : fputs ("{rn-sae}", file);
14662 5153 : break;
14663 637 : case ROUND_NEG_INF | ROUND_SAE:
14664 637 : fputs ("{rd-sae}", file);
14665 637 : break;
14666 56 : case ROUND_POS_INF | ROUND_SAE:
14667 56 : fputs ("{ru-sae}", file);
14668 56 : break;
14669 121 : case ROUND_ZERO | ROUND_SAE:
14670 121 : fputs ("{rz-sae}", file);
14671 121 : break;
14672 1 : default:
14673 1 : output_operand_lossage ("operand is not a specific integer, "
14674 : "invalid operand code 'R'");
14675 : }
14676 :
14677 5968 : if (ASSEMBLER_DIALECT == ASM_ATT)
14678 5962 : fputs (", ", file);
14679 :
14680 5968 : return;
14681 :
14682 10522 : case 'v':
14683 10522 : if (MEM_P (x))
14684 : {
14685 10650 : switch (MEM_ADDR_SPACE (x))
14686 : {
14687 : case ADDR_SPACE_GENERIC:
14688 : break;
14689 0 : case ADDR_SPACE_SEG_FS:
14690 0 : fputs ("fs ", file);
14691 0 : break;
14692 0 : case ADDR_SPACE_SEG_GS:
14693 0 : fputs ("gs ", file);
14694 0 : break;
14695 0 : default:
14696 0 : gcc_unreachable ();
14697 : }
14698 : }
14699 : else
14700 0 : output_operand_lossage ("operand is not a memory reference, "
14701 : "invalid operand code 'v'");
14702 10522 : return;
14703 :
14704 0 : case '*':
14705 0 : if (ASSEMBLER_DIALECT == ASM_ATT)
14706 0 : putc ('*', file);
14707 0 : return;
14708 :
14709 206 : case '&':
14710 206 : {
14711 206 : const char *name = get_some_local_dynamic_name ();
14712 206 : if (name == NULL)
14713 1 : output_operand_lossage ("'%%&' used without any "
14714 : "local dynamic TLS references");
14715 : else
14716 205 : assemble_name (file, name);
14717 206 : return;
14718 : }
14719 :
14720 6541565 : case '+':
14721 6541565 : {
14722 6541565 : rtx x;
14723 :
14724 6541565 : if (!optimize
14725 5110043 : || optimize_function_for_size_p (cfun)
14726 11462631 : || (!TARGET_BRANCH_PREDICTION_HINTS_NOT_TAKEN
14727 4921066 : && !TARGET_BRANCH_PREDICTION_HINTS_TAKEN))
14728 6541565 : return;
14729 :
14730 0 : x = find_reg_note (current_output_insn, REG_BR_PROB, 0);
14731 0 : if (x)
14732 : {
14733 0 : int pred_val = profile_probability::from_reg_br_prob_note
14734 0 : (XINT (x, 0)).to_reg_br_prob_base ();
14735 :
14736 0 : bool taken = pred_val > REG_BR_PROB_BASE / 2;
14737 : /* We use 3e (DS) prefix for taken branches and
14738 : 2e (CS) prefix for not taken branches. */
14739 0 : if (taken && TARGET_BRANCH_PREDICTION_HINTS_TAKEN)
14740 0 : fputs ("ds ; ", file);
14741 0 : else if (!taken && TARGET_BRANCH_PREDICTION_HINTS_NOT_TAKEN)
14742 0 : fputs ("cs ; ", file);
14743 : }
14744 0 : return;
14745 : }
14746 :
14747 : case ';':
14748 : #ifndef HAVE_AS_IX86_REP_LOCK_PREFIX
14749 : putc (';', file);
14750 : #endif
14751 : return;
14752 :
14753 3264 : case '~':
14754 3264 : putc (TARGET_AVX2 ? 'i' : 'f', file);
14755 3264 : return;
14756 :
14757 1678 : case 'M':
14758 1678 : if (TARGET_X32)
14759 : {
14760 : /* NB: 32-bit indices in VSIB address are sign-extended
14761 : to 64 bits. In x32, if 32-bit address 0xf7fa3010 is
14762 : sign-extended to 0xfffffffff7fa3010 which is invalid
14763 : address. Add addr32 prefix if there is no base
14764 : register nor symbol. */
14765 40 : bool ok;
14766 40 : struct ix86_address parts;
14767 40 : ok = ix86_decompose_address (x, &parts);
14768 40 : gcc_assert (ok && parts.index == NULL_RTX);
14769 40 : if (parts.base == NULL_RTX
14770 40 : && (parts.disp == NULL_RTX
14771 34 : || !symbolic_operand (parts.disp,
14772 34 : GET_MODE (parts.disp))))
14773 34 : fputs ("addr32 ", file);
14774 : }
14775 1678 : return;
14776 :
14777 24172 : case '^':
14778 27363 : if (Pmode != word_mode)
14779 0 : fputs ("addr32 ", file);
14780 24172 : return;
14781 :
14782 14992351 : case '!':
14783 14992351 : if (ix86_notrack_prefixed_insn_p (current_output_insn))
14784 3891 : fputs ("notrack ", file);
14785 14992351 : return;
14786 :
14787 1 : default:
14788 1 : output_operand_lossage ("invalid operand code '%c'", code);
14789 : }
14790 : }
14791 :
14792 145360520 : if (REG_P (x))
14793 86656667 : print_reg (x, code, file);
14794 :
14795 58703853 : else if (MEM_P (x))
14796 : {
14797 33523108 : rtx addr = XEXP (x, 0);
14798 :
14799 : /* No `byte ptr' prefix for call instructions ... */
14800 33523108 : if (ASSEMBLER_DIALECT == ASM_INTEL && code != 'X' && code != 'P')
14801 : {
14802 313 : machine_mode mode = GET_MODE (x);
14803 313 : const char *size;
14804 :
14805 : /* Check for explicit size override codes. */
14806 313 : if (code == 'b')
14807 : size = "BYTE";
14808 : else if (code == 'w')
14809 : size = "WORD";
14810 : else if (code == 'k')
14811 : size = "DWORD";
14812 : else if (code == 'q')
14813 : size = "QWORD";
14814 : else if (code == 'x')
14815 : size = "XMMWORD";
14816 : else if (code == 't')
14817 : size = "YMMWORD";
14818 : else if (code == 'g')
14819 : size = "ZMMWORD";
14820 227 : else if (mode == BLKmode)
14821 : /* ... or BLKmode operands, when not overridden. */
14822 : size = NULL;
14823 : else
14824 450 : switch (GET_MODE_SIZE (mode))
14825 : {
14826 : case 1: size = "BYTE"; break;
14827 : case 2: size = "WORD"; break;
14828 : case 4: size = "DWORD"; break;
14829 : case 8: size = "QWORD"; break;
14830 : case 12: size = "TBYTE"; break;
14831 7 : case 16:
14832 7 : if (mode == XFmode)
14833 : size = "TBYTE";
14834 : else
14835 : size = "XMMWORD";
14836 : break;
14837 : case 32: size = "YMMWORD"; break;
14838 : case 64: size = "ZMMWORD"; break;
14839 0 : default:
14840 0 : gcc_unreachable ();
14841 : }
14842 : if (size)
14843 : {
14844 311 : fputs (size, file);
14845 311 : fputs (" PTR ", file);
14846 : }
14847 : }
14848 :
14849 33523108 : if (this_is_asm_operands && ! address_operand (addr, VOIDmode))
14850 0 : output_operand_lossage ("invalid constraints for operand");
14851 : else
14852 33523108 : ix86_print_operand_address_as
14853 34073571 : (file, addr, MEM_ADDR_SPACE (x), code == 'p' || code == 'P');
14854 : }
14855 :
14856 25180745 : else if (CONST_DOUBLE_P (x) && GET_MODE (x) == HFmode)
14857 : {
14858 975 : long l = real_to_target (NULL, CONST_DOUBLE_REAL_VALUE (x),
14859 975 : REAL_MODE_FORMAT (HFmode));
14860 975 : if (ASSEMBLER_DIALECT == ASM_ATT)
14861 975 : putc ('$', file);
14862 975 : fprintf (file, "0x%04x", (unsigned int) l);
14863 975 : }
14864 :
14865 25179770 : else if (CONST_DOUBLE_P (x) && GET_MODE (x) == SFmode)
14866 : {
14867 21254 : long l;
14868 :
14869 21254 : REAL_VALUE_TO_TARGET_SINGLE (*CONST_DOUBLE_REAL_VALUE (x), l);
14870 :
14871 21254 : if (ASSEMBLER_DIALECT == ASM_ATT)
14872 21254 : putc ('$', file);
14873 : /* Sign extend 32bit SFmode immediate to 8 bytes. */
14874 21254 : if (code == 'q')
14875 327 : fprintf (file, "0x%08" HOST_LONG_LONG_FORMAT "x",
14876 : (unsigned long long) (int) l);
14877 : else
14878 20927 : fprintf (file, "0x%08x", (unsigned int) l);
14879 : }
14880 :
14881 25158516 : else if (CONST_DOUBLE_P (x) && GET_MODE (x) == DFmode)
14882 : {
14883 3386 : long l[2];
14884 :
14885 3386 : REAL_VALUE_TO_TARGET_DOUBLE (*CONST_DOUBLE_REAL_VALUE (x), l);
14886 :
14887 3386 : if (ASSEMBLER_DIALECT == ASM_ATT)
14888 3386 : putc ('$', file);
14889 3386 : fprintf (file, "0x%lx%08lx", l[1] & 0xffffffff, l[0] & 0xffffffff);
14890 3386 : }
14891 :
14892 : /* These float cases don't actually occur as immediate operands. */
14893 25155130 : else if (CONST_DOUBLE_P (x) && GET_MODE (x) == XFmode)
14894 : {
14895 0 : char dstr[30];
14896 :
14897 0 : real_to_decimal (dstr, CONST_DOUBLE_REAL_VALUE (x), sizeof (dstr), 0, 1);
14898 0 : fputs (dstr, file);
14899 0 : }
14900 :
14901 : /* Print bcst_mem_operand. */
14902 25155130 : else if (GET_CODE (x) == VEC_DUPLICATE)
14903 : {
14904 314 : machine_mode vmode = GET_MODE (x);
14905 : /* Must be bcst_memory_operand. */
14906 314 : gcc_assert (bcst_mem_operand (x, vmode));
14907 :
14908 314 : rtx mem = XEXP (x,0);
14909 314 : ix86_print_operand (file, mem, 0);
14910 :
14911 314 : switch (vmode)
14912 : {
14913 28 : case E_V2DImode:
14914 28 : case E_V2DFmode:
14915 28 : fputs ("{1to2}", file);
14916 28 : break;
14917 74 : case E_V4SImode:
14918 74 : case E_V4SFmode:
14919 74 : case E_V4DImode:
14920 74 : case E_V4DFmode:
14921 74 : fputs ("{1to4}", file);
14922 74 : break;
14923 94 : case E_V8SImode:
14924 94 : case E_V8SFmode:
14925 94 : case E_V8DFmode:
14926 94 : case E_V8DImode:
14927 94 : case E_V8HFmode:
14928 94 : fputs ("{1to8}", file);
14929 94 : break;
14930 110 : case E_V16SFmode:
14931 110 : case E_V16SImode:
14932 110 : case E_V16HFmode:
14933 110 : fputs ("{1to16}", file);
14934 110 : break;
14935 8 : case E_V32HFmode:
14936 8 : fputs ("{1to32}", file);
14937 8 : break;
14938 0 : default:
14939 0 : gcc_unreachable ();
14940 : }
14941 : }
14942 :
14943 : else
14944 : {
14945 : /* We have patterns that allow zero sets of memory, for instance.
14946 : In 64-bit mode, we should probably support all 8-byte vectors,
14947 : since we can in fact encode that into an immediate. */
14948 25154816 : if (CONST_VECTOR_P (x))
14949 : {
14950 3327 : if (x != CONST0_RTX (GET_MODE (x)))
14951 2 : output_operand_lossage ("invalid vector immediate");
14952 3327 : x = const0_rtx;
14953 : }
14954 :
14955 25154816 : if (code == 'P')
14956 : {
14957 5994106 : if (ix86_force_load_from_GOT_p (x, true))
14958 : {
14959 : /* For inline assembly statement, load function address
14960 : from GOT with 'P' operand modifier to avoid PLT. */
14961 4 : x = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x),
14962 : (TARGET_64BIT
14963 : ? UNSPEC_GOTPCREL
14964 : : UNSPEC_GOT));
14965 4 : x = gen_rtx_CONST (Pmode, x);
14966 4 : x = gen_const_mem (Pmode, x);
14967 4 : ix86_print_operand (file, x, 'A');
14968 4 : return;
14969 : }
14970 : }
14971 19160710 : else if (code != 'p')
14972 : {
14973 19160601 : if (CONST_INT_P (x))
14974 : {
14975 15816648 : if (ASSEMBLER_DIALECT == ASM_ATT)
14976 15816436 : putc ('$', file);
14977 : }
14978 3343953 : else if (GET_CODE (x) == CONST || SYMBOL_REF_P (x)
14979 9409 : || LABEL_REF_P (x))
14980 : {
14981 3343951 : if (ASSEMBLER_DIALECT == ASM_ATT)
14982 3343927 : putc ('$', file);
14983 : else
14984 24 : fputs ("OFFSET FLAT:", file);
14985 : }
14986 : }
14987 25154812 : if (CONST_INT_P (x))
14988 15816734 : fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (x));
14989 9338078 : else if (flag_pic || MACHOPIC_INDIRECT)
14990 524773 : output_pic_addr_const (file, x, code);
14991 : else
14992 8813305 : output_addr_const (file, x);
14993 : }
14994 : }
14995 :
14996 : static bool
14997 21644270 : ix86_print_operand_punct_valid_p (unsigned char code)
14998 : {
14999 21644270 : return (code == '*' || code == '+' || code == '&' || code == ';'
15000 15016523 : || code == '~' || code == '^' || code == '!');
15001 : }
15002 :
15003 : /* Print a memory operand whose address is ADDR. */
15004 :
15005 : static void
15006 37125224 : ix86_print_operand_address_as (FILE *file, rtx addr,
15007 : addr_space_t as, bool raw)
15008 : {
15009 37125224 : struct ix86_address parts;
15010 37125224 : rtx base, index, disp;
15011 37125224 : int scale;
15012 37125224 : int ok;
15013 37125224 : bool vsib = false;
15014 37125224 : int code = 0;
15015 :
15016 37125224 : if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_VSIBADDR)
15017 : {
15018 1678 : ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts);
15019 1678 : gcc_assert (parts.index == NULL_RTX);
15020 1678 : parts.index = XVECEXP (addr, 0, 1);
15021 1678 : parts.scale = INTVAL (XVECEXP (addr, 0, 2));
15022 1678 : addr = XVECEXP (addr, 0, 0);
15023 1678 : vsib = true;
15024 : }
15025 37123546 : else if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_LEA_ADDR)
15026 : {
15027 3114154 : gcc_assert (TARGET_64BIT);
15028 3114154 : ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts);
15029 3114154 : code = 'q';
15030 : }
15031 : else
15032 34009392 : ok = ix86_decompose_address (addr, &parts);
15033 :
15034 37125224 : gcc_assert (ok);
15035 :
15036 37125224 : base = parts.base;
15037 37125224 : index = parts.index;
15038 37125224 : disp = parts.disp;
15039 37125224 : scale = parts.scale;
15040 :
15041 37125224 : if (ADDR_SPACE_GENERIC_P (as))
15042 36843173 : as = parts.seg;
15043 : else
15044 282051 : gcc_assert (ADDR_SPACE_GENERIC_P (parts.seg));
15045 :
15046 37125224 : if (!ADDR_SPACE_GENERIC_P (as) && !raw)
15047 : {
15048 282066 : if (ASSEMBLER_DIALECT == ASM_ATT)
15049 282064 : putc ('%', file);
15050 :
15051 282066 : switch (as)
15052 : {
15053 182327 : case ADDR_SPACE_SEG_FS:
15054 182327 : fputs ("fs:", file);
15055 182327 : break;
15056 99739 : case ADDR_SPACE_SEG_GS:
15057 99739 : fputs ("gs:", file);
15058 99739 : break;
15059 0 : default:
15060 0 : gcc_unreachable ();
15061 : }
15062 : }
15063 :
15064 : /* Use one byte shorter RIP relative addressing for 64bit mode. */
15065 37125224 : if (TARGET_64BIT && !base && !index && !raw)
15066 : {
15067 6029922 : rtx symbol = disp;
15068 :
15069 6029922 : if (GET_CODE (disp) == CONST
15070 2183948 : && GET_CODE (XEXP (disp, 0)) == PLUS
15071 2098767 : && CONST_INT_P (XEXP (XEXP (disp, 0), 1)))
15072 2098767 : symbol = XEXP (XEXP (disp, 0), 0);
15073 :
15074 6029922 : if (LABEL_REF_P (symbol)
15075 6029922 : || (SYMBOL_REF_P (symbol)
15076 5768298 : && SYMBOL_REF_TLS_MODEL (symbol) == 0))
15077 5768168 : base = pc_rtx;
15078 : }
15079 :
15080 37125224 : if (!base && !index)
15081 : {
15082 : /* Displacement only requires special attention. */
15083 601430 : if (CONST_INT_P (disp))
15084 : {
15085 269435 : if (ASSEMBLER_DIALECT == ASM_INTEL && ADDR_SPACE_GENERIC_P (as))
15086 0 : fputs ("ds:", file);
15087 269435 : fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (disp));
15088 : }
15089 : /* Load the external function address via the GOT slot to avoid PLT. */
15090 331995 : else if (GET_CODE (disp) == CONST
15091 113615 : && GET_CODE (XEXP (disp, 0)) == UNSPEC
15092 85418 : && (XINT (XEXP (disp, 0), 1) == UNSPEC_GOTPCREL
15093 9626 : || XINT (XEXP (disp, 0), 1) == UNSPEC_GOT)
15094 407787 : && ix86_force_load_from_GOT_p (XVECEXP (XEXP (disp, 0), 0, 0)))
15095 24 : output_pic_addr_const (file, disp, 0);
15096 331971 : else if (flag_pic)
15097 114897 : output_pic_addr_const (file, disp, 0);
15098 : else
15099 217074 : output_addr_const (file, disp);
15100 : }
15101 : else
15102 : {
15103 : /* Print SImode register names to force addr32 prefix. */
15104 36523794 : if (SImode_address_operand (addr, VOIDmode))
15105 : {
15106 51 : if (flag_checking)
15107 : {
15108 51 : gcc_assert (TARGET_64BIT);
15109 51 : switch (GET_CODE (addr))
15110 : {
15111 0 : case SUBREG:
15112 0 : gcc_assert (GET_MODE (addr) == SImode);
15113 0 : gcc_assert (GET_MODE (SUBREG_REG (addr)) == DImode);
15114 : break;
15115 51 : case ZERO_EXTEND:
15116 51 : case AND:
15117 51 : gcc_assert (GET_MODE (addr) == DImode);
15118 : break;
15119 0 : default:
15120 0 : gcc_unreachable ();
15121 : }
15122 : }
15123 51 : gcc_assert (!code);
15124 : code = 'k';
15125 : }
15126 36523743 : else if (code == 0
15127 33411234 : && TARGET_X32
15128 539 : && disp
15129 448 : && CONST_INT_P (disp)
15130 349 : && INTVAL (disp) < -16*1024*1024)
15131 : {
15132 : /* X32 runs in 64-bit mode, where displacement, DISP, in
15133 : address DISP(%r64), is encoded as 32-bit immediate sign-
15134 : extended from 32-bit to 64-bit. For -0x40000300(%r64),
15135 : address is %r64 + 0xffffffffbffffd00. When %r64 <
15136 : 0x40000300, like 0x37ffe064, address is 0xfffffffff7ffdd64,
15137 : which is invalid for x32. The correct address is %r64
15138 : - 0x40000300 == 0xf7ffdd64. To properly encode
15139 : -0x40000300(%r64) for x32, we zero-extend negative
15140 : displacement by forcing addr32 prefix which truncates
15141 : 0xfffffffff7ffdd64 to 0xf7ffdd64. In theory, we should
15142 : zero-extend all negative displacements, including -1(%rsp).
15143 : However, for small negative displacements, sign-extension
15144 : won't cause overflow. We only zero-extend negative
15145 : displacements if they < -16*1024*1024, which is also used
15146 : to check legitimate address displacements for PIC. */
15147 51 : code = 'k';
15148 : }
15149 :
15150 : /* Since the upper 32 bits of RSP are always zero for x32,
15151 : we can encode %esp as %rsp to avoid 0x67 prefix if
15152 : there is no index register. */
15153 1089 : if (TARGET_X32 && Pmode == SImode
15154 36524255 : && !index && base && REG_P (base) && REGNO (base) == SP_REG)
15155 : code = 'q';
15156 :
15157 36523794 : if (ASSEMBLER_DIALECT == ASM_ATT)
15158 : {
15159 36523432 : if (disp)
15160 : {
15161 32450420 : if (flag_pic)
15162 2830003 : output_pic_addr_const (file, disp, 0);
15163 29620417 : else if (LABEL_REF_P (disp))
15164 5324 : output_asm_label (disp);
15165 : else
15166 29615093 : output_addr_const (file, disp);
15167 : }
15168 :
15169 36523432 : putc ('(', file);
15170 36523432 : if (base)
15171 36103299 : print_reg (base, code, file);
15172 36523432 : if (index)
15173 : {
15174 1887567 : putc (',', file);
15175 3773504 : print_reg (index, vsib ? 0 : code, file);
15176 1887567 : if (scale != 1 || vsib)
15177 1026066 : fprintf (file, ",%d", scale);
15178 : }
15179 36523432 : putc (')', file);
15180 : }
15181 : else
15182 : {
15183 362 : rtx offset = NULL_RTX;
15184 :
15185 362 : if (disp)
15186 : {
15187 : /* Pull out the offset of a symbol; print any symbol itself. */
15188 282 : if (GET_CODE (disp) == CONST
15189 14 : && GET_CODE (XEXP (disp, 0)) == PLUS
15190 14 : && CONST_INT_P (XEXP (XEXP (disp, 0), 1)))
15191 : {
15192 14 : offset = XEXP (XEXP (disp, 0), 1);
15193 14 : disp = gen_rtx_CONST (VOIDmode,
15194 : XEXP (XEXP (disp, 0), 0));
15195 : }
15196 :
15197 282 : if (flag_pic)
15198 0 : output_pic_addr_const (file, disp, 0);
15199 282 : else if (LABEL_REF_P (disp))
15200 0 : output_asm_label (disp);
15201 282 : else if (CONST_INT_P (disp))
15202 : offset = disp;
15203 : else
15204 115 : output_addr_const (file, disp);
15205 : }
15206 :
15207 362 : putc ('[', file);
15208 362 : if (base)
15209 : {
15210 325 : print_reg (base, code, file);
15211 325 : if (offset)
15212 : {
15213 181 : if (INTVAL (offset) >= 0)
15214 20 : putc ('+', file);
15215 181 : fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (offset));
15216 : }
15217 : }
15218 37 : else if (offset)
15219 0 : fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (offset));
15220 : else
15221 37 : putc ('0', file);
15222 :
15223 362 : if (index)
15224 : {
15225 90 : putc ('+', file);
15226 132 : print_reg (index, vsib ? 0 : code, file);
15227 90 : if (scale != 1 || vsib)
15228 88 : fprintf (file, "*%d", scale);
15229 : }
15230 362 : putc (']', file);
15231 : }
15232 : }
15233 37125224 : }
15234 :
15235 : static void
15236 3602117 : ix86_print_operand_address (FILE *file, machine_mode /*mode*/, rtx addr)
15237 : {
15238 3602117 : if (this_is_asm_operands && ! address_operand (addr, VOIDmode))
15239 1 : output_operand_lossage ("invalid constraints for operand");
15240 : else
15241 3602116 : ix86_print_operand_address_as (file, addr, ADDR_SPACE_GENERIC, false);
15242 3602117 : }
15243 :
15244 : /* Implementation of TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA. */
15245 :
15246 : static bool
15247 15503 : i386_asm_output_addr_const_extra (FILE *file, rtx x)
15248 : {
15249 15503 : rtx op;
15250 :
15251 15503 : if (GET_CODE (x) != UNSPEC)
15252 : return false;
15253 :
15254 15503 : op = XVECEXP (x, 0, 0);
15255 15503 : switch (XINT (x, 1))
15256 : {
15257 1293 : case UNSPEC_GOTOFF:
15258 1293 : output_addr_const (file, op);
15259 1293 : fputs ("@gotoff", file);
15260 1293 : break;
15261 0 : case UNSPEC_GOTTPOFF:
15262 0 : output_addr_const (file, op);
15263 : /* FIXME: This might be @TPOFF in Sun ld. */
15264 0 : fputs ("@gottpoff", file);
15265 0 : break;
15266 0 : case UNSPEC_TPOFF:
15267 0 : output_addr_const (file, op);
15268 0 : fputs ("@tpoff", file);
15269 0 : break;
15270 11113 : case UNSPEC_NTPOFF:
15271 11113 : output_addr_const (file, op);
15272 11113 : if (TARGET_64BIT)
15273 10337 : fputs ("@tpoff", file);
15274 : else
15275 776 : fputs ("@ntpoff", file);
15276 : break;
15277 0 : case UNSPEC_DTPOFF:
15278 0 : output_addr_const (file, op);
15279 0 : fputs ("@dtpoff", file);
15280 0 : break;
15281 3096 : case UNSPEC_GOTNTPOFF:
15282 3096 : output_addr_const (file, op);
15283 3096 : if (TARGET_64BIT)
15284 3096 : fputs (ASSEMBLER_DIALECT == ASM_ATT ?
15285 : "@gottpoff(%rip)" : "@gottpoff[rip]", file);
15286 : else
15287 0 : fputs ("@gotntpoff", file);
15288 : break;
15289 1 : case UNSPEC_INDNTPOFF:
15290 1 : output_addr_const (file, op);
15291 1 : fputs ("@indntpoff", file);
15292 1 : break;
15293 0 : case UNSPEC_SECREL32:
15294 0 : output_addr_const (file, op);
15295 0 : fputs ("@secrel32", file);
15296 0 : break;
15297 : #if TARGET_MACHO
15298 : case UNSPEC_MACHOPIC_OFFSET:
15299 : output_addr_const (file, op);
15300 : putc ('-', file);
15301 : machopic_output_function_base_name (file);
15302 : break;
15303 : #endif
15304 :
15305 : default:
15306 : return false;
15307 : }
15308 :
15309 : return true;
15310 : }
15311 :
15312 :
15313 : /* Output code to perform a 387 binary operation in INSN, one of PLUS,
15314 : MINUS, MULT or DIV. OPERANDS are the insn operands, where operands[3]
15315 : is the expression of the binary operation. The output may either be
15316 : emitted here, or returned to the caller, like all output_* functions.
15317 :
15318 : There is no guarantee that the operands are the same mode, as they
15319 : might be within FLOAT or FLOAT_EXTEND expressions. */
15320 :
15321 : #ifndef SYSV386_COMPAT
15322 : /* Set to 1 for compatibility with brain-damaged assemblers. No-one
15323 : wants to fix the assemblers because that causes incompatibility
15324 : with gcc. No-one wants to fix gcc because that causes
15325 : incompatibility with assemblers... You can use the option of
15326 : -DSYSV386_COMPAT=0 if you recompile both gcc and gas this way. */
15327 : #define SYSV386_COMPAT 1
15328 : #endif
15329 :
15330 : const char *
15331 606955 : output_387_binary_op (rtx_insn *insn, rtx *operands)
15332 : {
15333 606955 : static char buf[40];
15334 606955 : const char *p;
15335 606955 : bool is_sse
15336 606955 : = (SSE_REG_P (operands[0])
15337 663098 : || SSE_REG_P (operands[1]) || SSE_REG_P (operands[2]));
15338 :
15339 56143 : if (is_sse)
15340 : p = "%v";
15341 56143 : else if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
15342 56136 : || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
15343 : p = "fi";
15344 : else
15345 606955 : p = "f";
15346 :
15347 606955 : strcpy (buf, p);
15348 :
15349 606955 : switch (GET_CODE (operands[3]))
15350 : {
15351 : case PLUS:
15352 : p = "add"; break;
15353 : case MINUS:
15354 : p = "sub"; break;
15355 94572 : case MULT:
15356 94572 : p = "mul"; break;
15357 27472 : case DIV:
15358 27472 : p = "div"; break;
15359 0 : default:
15360 0 : gcc_unreachable ();
15361 : }
15362 :
15363 606955 : strcat (buf, p);
15364 :
15365 606955 : if (is_sse)
15366 : {
15367 550812 : p = GET_MODE (operands[0]) == SFmode ? "ss" : "sd";
15368 550812 : strcat (buf, p);
15369 :
15370 550812 : if (TARGET_AVX)
15371 : p = "\t{%2, %1, %0|%0, %1, %2}";
15372 : else
15373 535400 : p = "\t{%2, %0|%0, %2}";
15374 :
15375 550812 : strcat (buf, p);
15376 550812 : return buf;
15377 : }
15378 :
15379 : /* Even if we do not want to check the inputs, this documents input
15380 : constraints. Which helps in understanding the following code. */
15381 56143 : if (flag_checking)
15382 : {
15383 56142 : if (STACK_REG_P (operands[0])
15384 56142 : && ((REG_P (operands[1])
15385 54489 : && REGNO (operands[0]) == REGNO (operands[1])
15386 50376 : && (STACK_REG_P (operands[2]) || MEM_P (operands[2])))
15387 5766 : || (REG_P (operands[2])
15388 5766 : && REGNO (operands[0]) == REGNO (operands[2])
15389 5766 : && (STACK_REG_P (operands[1]) || MEM_P (operands[1]))))
15390 112284 : && (STACK_TOP_P (operands[1]) || STACK_TOP_P (operands[2])))
15391 : ; /* ok */
15392 : else
15393 0 : gcc_unreachable ();
15394 : }
15395 :
15396 56143 : switch (GET_CODE (operands[3]))
15397 : {
15398 41041 : case MULT:
15399 41041 : case PLUS:
15400 41041 : if (REG_P (operands[2]) && REGNO (operands[0]) == REGNO (operands[2]))
15401 2059 : std::swap (operands[1], operands[2]);
15402 :
15403 : /* know operands[0] == operands[1]. */
15404 :
15405 41041 : if (MEM_P (operands[2]))
15406 : {
15407 : p = "%Z2\t%2";
15408 : break;
15409 : }
15410 :
15411 36604 : if (find_regno_note (insn, REG_DEAD, REGNO (operands[2])))
15412 : {
15413 21355 : if (STACK_TOP_P (operands[0]))
15414 : /* How is it that we are storing to a dead operand[2]?
15415 : Well, presumably operands[1] is dead too. We can't
15416 : store the result to st(0) as st(0) gets popped on this
15417 : instruction. Instead store to operands[2] (which I
15418 : think has to be st(1)). st(1) will be popped later.
15419 : gcc <= 2.8.1 didn't have this check and generated
15420 : assembly code that the Unixware assembler rejected. */
15421 : p = "p\t{%0, %2|%2, %0}"; /* st(1) = st(0) op st(1); pop */
15422 : else
15423 : p = "p\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0); pop */
15424 : break;
15425 : }
15426 :
15427 15249 : if (STACK_TOP_P (operands[0]))
15428 : p = "\t{%y2, %0|%0, %y2}"; /* st(0) = st(0) op st(r2) */
15429 : else
15430 : p = "\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0) */
15431 : break;
15432 :
15433 15102 : case MINUS:
15434 15102 : case DIV:
15435 15102 : if (MEM_P (operands[1]))
15436 : {
15437 : p = "r%Z1\t%1";
15438 : break;
15439 : }
15440 :
15441 14640 : if (MEM_P (operands[2]))
15442 : {
15443 : p = "%Z2\t%2";
15444 : break;
15445 : }
15446 :
15447 13030 : if (find_regno_note (insn, REG_DEAD, REGNO (operands[2])))
15448 : {
15449 : #if SYSV386_COMPAT
15450 : /* The SystemV/386 SVR3.2 assembler, and probably all AT&T
15451 : derived assemblers, confusingly reverse the direction of
15452 : the operation for fsub{r} and fdiv{r} when the
15453 : destination register is not st(0). The Intel assembler
15454 : doesn't have this brain damage. Read !SYSV386_COMPAT to
15455 : figure out what the hardware really does. */
15456 6269 : if (STACK_TOP_P (operands[0]))
15457 : p = "{p\t%0, %2|rp\t%2, %0}";
15458 : else
15459 : p = "{rp\t%2, %0|p\t%0, %2}";
15460 : #else
15461 : if (STACK_TOP_P (operands[0]))
15462 : /* As above for fmul/fadd, we can't store to st(0). */
15463 : p = "rp\t{%0, %2|%2, %0}"; /* st(1) = st(0) op st(1); pop */
15464 : else
15465 : p = "p\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0); pop */
15466 : #endif
15467 : break;
15468 : }
15469 :
15470 6761 : if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
15471 : {
15472 : #if SYSV386_COMPAT
15473 3146 : if (STACK_TOP_P (operands[0]))
15474 : p = "{rp\t%0, %1|p\t%1, %0}";
15475 : else
15476 : p = "{p\t%1, %0|rp\t%0, %1}";
15477 : #else
15478 : if (STACK_TOP_P (operands[0]))
15479 : p = "p\t{%0, %1|%1, %0}"; /* st(1) = st(1) op st(0); pop */
15480 : else
15481 : p = "rp\t{%1, %0|%0, %1}"; /* st(r2) = st(0) op st(r2); pop */
15482 : #endif
15483 : break;
15484 : }
15485 :
15486 3615 : if (STACK_TOP_P (operands[0]))
15487 : {
15488 2737 : if (STACK_TOP_P (operands[1]))
15489 : p = "\t{%y2, %0|%0, %y2}"; /* st(0) = st(0) op st(r2) */
15490 : else
15491 : p = "r\t{%y1, %0|%0, %y1}"; /* st(0) = st(r1) op st(0) */
15492 : break;
15493 : }
15494 878 : else if (STACK_TOP_P (operands[1]))
15495 : {
15496 : #if SYSV386_COMPAT
15497 : p = "{\t%1, %0|r\t%0, %1}";
15498 : #else
15499 : p = "r\t{%1, %0|%0, %1}"; /* st(r2) = st(0) op st(r2) */
15500 : #endif
15501 : }
15502 : else
15503 : {
15504 : #if SYSV386_COMPAT
15505 : p = "{r\t%2, %0|\t%0, %2}";
15506 : #else
15507 : p = "\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0) */
15508 : #endif
15509 : }
15510 : break;
15511 :
15512 0 : default:
15513 0 : gcc_unreachable ();
15514 : }
15515 :
15516 56143 : strcat (buf, p);
15517 56143 : return buf;
15518 : }
15519 :
15520 : /* Return needed mode for entity in optimize_mode_switching pass. */
15521 :
15522 : static int
15523 1656 : ix86_dirflag_mode_needed (rtx_insn *insn)
15524 : {
15525 1656 : if (CALL_P (insn))
15526 : {
15527 339 : if (cfun->machine->func_type == TYPE_NORMAL)
15528 : return X86_DIRFLAG_ANY;
15529 : else
15530 : /* No need to emit CLD in interrupt handler for TARGET_CLD. */
15531 339 : return TARGET_CLD ? X86_DIRFLAG_ANY : X86_DIRFLAG_RESET;
15532 : }
15533 :
15534 1317 : if (recog_memoized (insn) < 0)
15535 : return X86_DIRFLAG_ANY;
15536 :
15537 1315 : if (get_attr_type (insn) == TYPE_STR)
15538 : {
15539 : /* Emit cld instruction if stringops are used in the function. */
15540 1 : if (cfun->machine->func_type == TYPE_NORMAL)
15541 0 : return TARGET_CLD ? X86_DIRFLAG_RESET : X86_DIRFLAG_ANY;
15542 : else
15543 : return X86_DIRFLAG_RESET;
15544 : }
15545 :
15546 : return X86_DIRFLAG_ANY;
15547 : }
15548 :
15549 : /* Check if a 256bit or 512 bit AVX register is referenced inside of EXP. */
15550 :
15551 : static bool
15552 2219274 : ix86_check_avx_upper_register (const_rtx exp)
15553 : {
15554 : /* construct_container may return a parallel with expr_list
15555 : which contains the real reg and mode */
15556 2219274 : subrtx_iterator::array_type array;
15557 8497087 : FOR_EACH_SUBRTX (iter, array, exp, NONCONST)
15558 : {
15559 6438494 : const_rtx x = *iter;
15560 2585314 : if (SSE_REG_P (x)
15561 834735 : && !EXT_REX_SSE_REG_P (x)
15562 8095816 : && GET_MODE_BITSIZE (GET_MODE (x)) > 128)
15563 160681 : return true;
15564 : }
15565 :
15566 2058593 : return false;
15567 2219274 : }
15568 :
15569 : /* Check if a 256bit or 512bit AVX register is referenced in stores. */
15570 :
15571 : static void
15572 51283 : ix86_check_avx_upper_stores (rtx dest, const_rtx, void *data)
15573 : {
15574 51283 : if (SSE_REG_P (dest)
15575 12741 : && !EXT_REX_SSE_REG_P (dest)
15576 76765 : && GET_MODE_BITSIZE (GET_MODE (dest)) > 128)
15577 : {
15578 754 : bool *used = (bool *) data;
15579 754 : *used = true;
15580 : }
15581 51283 : }
15582 :
15583 : /* Return needed mode for entity in optimize_mode_switching pass. */
15584 :
15585 : static int
15586 2071935 : ix86_avx_u128_mode_needed (rtx_insn *insn)
15587 : {
15588 2071935 : if (DEBUG_INSN_P (insn))
15589 : return AVX_U128_ANY;
15590 :
15591 2071935 : if (CALL_P (insn))
15592 : {
15593 49777 : rtx link;
15594 :
15595 : /* Needed mode is set to AVX_U128_CLEAN if there are
15596 : no 256bit or 512bit modes used in function arguments. */
15597 49777 : for (link = CALL_INSN_FUNCTION_USAGE (insn);
15598 136378 : link;
15599 86601 : link = XEXP (link, 1))
15600 : {
15601 87647 : if (GET_CODE (XEXP (link, 0)) == USE)
15602 : {
15603 86308 : rtx arg = XEXP (XEXP (link, 0), 0);
15604 :
15605 86308 : if (ix86_check_avx_upper_register (arg))
15606 : return AVX_U128_DIRTY;
15607 : }
15608 : }
15609 :
15610 : /* Needed mode is set to AVX_U128_CLEAN if there are no 256bit
15611 : nor 512bit registers used in the function return register. */
15612 48731 : bool avx_upper_reg_found = false;
15613 48731 : note_stores (insn, ix86_check_avx_upper_stores,
15614 : &avx_upper_reg_found);
15615 48731 : if (avx_upper_reg_found)
15616 : return AVX_U128_DIRTY;
15617 :
15618 : /* If the function is known to preserve some SSE registers,
15619 : RA and previous passes can legitimately rely on that for
15620 : modes wider than 256 bits. It's only safe to issue a
15621 : vzeroupper if all SSE registers are clobbered. */
15622 48547 : const function_abi &abi = insn_callee_abi (insn);
15623 48547 : if (vzeroupper_pattern (PATTERN (insn), VOIDmode)
15624 : /* Should be safe to issue an vzeroupper before sibling_call_p.
15625 : Also there not mode_exit for sibling_call, so there could be
15626 : missing vzeroupper for that. */
15627 48547 : || !(SIBLING_CALL_P (insn)
15628 47261 : || hard_reg_set_subset_p (reg_class_contents[SSE_REGS],
15629 47261 : abi.mode_clobbers (V4DImode))))
15630 8425 : return AVX_U128_ANY;
15631 :
15632 40122 : return AVX_U128_CLEAN;
15633 : }
15634 :
15635 2022158 : rtx set = single_set (insn);
15636 2022158 : if (set)
15637 : {
15638 1949242 : rtx dest = SET_DEST (set);
15639 1949242 : rtx src = SET_SRC (set);
15640 1467931 : if (SSE_REG_P (dest)
15641 553443 : && !EXT_REX_SSE_REG_P (dest)
15642 3044556 : && GET_MODE_BITSIZE (GET_MODE (dest)) > 128)
15643 : {
15644 : /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the
15645 : source isn't zero. */
15646 169055 : if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
15647 : return AVX_U128_DIRTY;
15648 : else
15649 : return AVX_U128_ANY;
15650 : }
15651 : else
15652 : {
15653 1780187 : if (ix86_check_avx_upper_register (src))
15654 : return AVX_U128_DIRTY;
15655 : }
15656 :
15657 : /* This isn't YMM/ZMM load/store. */
15658 : return AVX_U128_ANY;
15659 : }
15660 :
15661 : /* Require DIRTY mode if a 256bit or 512bit AVX register is referenced.
15662 : Hardware changes state only when a 256bit register is written to,
15663 : but we need to prevent the compiler from moving optimal insertion
15664 : point above eventual read from 256bit or 512 bit register. */
15665 72916 : if (ix86_check_avx_upper_register (PATTERN (insn)))
15666 : return AVX_U128_DIRTY;
15667 :
15668 : return AVX_U128_ANY;
15669 : }
15670 :
15671 : /* Return mode that i387 must be switched into
15672 : prior to the execution of insn. */
15673 :
15674 : static int
15675 422541 : ix86_i387_mode_needed (int entity, rtx_insn *insn)
15676 : {
15677 422541 : enum attr_i387_cw mode;
15678 :
15679 : /* The mode UNINITIALIZED is used to store control word after a
15680 : function call or ASM pattern. The mode ANY specify that function
15681 : has no requirements on the control word and make no changes in the
15682 : bits we are interested in. */
15683 :
15684 422541 : if (CALL_P (insn)
15685 422541 : || (NONJUMP_INSN_P (insn)
15686 345848 : && (asm_noperands (PATTERN (insn)) >= 0
15687 345795 : || GET_CODE (PATTERN (insn)) == ASM_INPUT)))
15688 15025 : return I387_CW_UNINITIALIZED;
15689 :
15690 407516 : if (recog_memoized (insn) < 0)
15691 : return I387_CW_ANY;
15692 :
15693 406574 : mode = get_attr_i387_cw (insn);
15694 :
15695 406574 : switch (entity)
15696 : {
15697 0 : case I387_ROUNDEVEN:
15698 0 : if (mode == I387_CW_ROUNDEVEN)
15699 : return mode;
15700 : break;
15701 :
15702 401911 : case I387_TRUNC:
15703 401911 : if (mode == I387_CW_TRUNC)
15704 : return mode;
15705 : break;
15706 :
15707 3628 : case I387_FLOOR:
15708 3628 : if (mode == I387_CW_FLOOR)
15709 : return mode;
15710 : break;
15711 :
15712 1035 : case I387_CEIL:
15713 1035 : if (mode == I387_CW_CEIL)
15714 : return mode;
15715 : break;
15716 :
15717 0 : default:
15718 0 : gcc_unreachable ();
15719 : }
15720 :
15721 : return I387_CW_ANY;
15722 : }
15723 :
15724 : /* Return mode that entity must be switched into
15725 : prior to the execution of insn. */
15726 :
15727 : static int
15728 2496132 : ix86_mode_needed (int entity, rtx_insn *insn, HARD_REG_SET)
15729 : {
15730 2496132 : switch (entity)
15731 : {
15732 1656 : case X86_DIRFLAG:
15733 1656 : return ix86_dirflag_mode_needed (insn);
15734 2071935 : case AVX_U128:
15735 2071935 : return ix86_avx_u128_mode_needed (insn);
15736 422541 : case I387_ROUNDEVEN:
15737 422541 : case I387_TRUNC:
15738 422541 : case I387_FLOOR:
15739 422541 : case I387_CEIL:
15740 422541 : return ix86_i387_mode_needed (entity, insn);
15741 0 : default:
15742 0 : gcc_unreachable ();
15743 : }
15744 : return 0;
15745 : }
15746 :
15747 : /* Calculate mode of upper 128bit AVX registers after the insn. */
15748 :
15749 : static int
15750 2071935 : ix86_avx_u128_mode_after (int mode, rtx_insn *insn)
15751 : {
15752 2071935 : rtx pat = PATTERN (insn);
15753 :
15754 2071935 : if (vzeroupper_pattern (pat, VOIDmode)
15755 2071935 : || vzeroall_pattern (pat, VOIDmode))
15756 161 : return AVX_U128_CLEAN;
15757 :
15758 : /* We know that state is clean after CALL insn if there are no
15759 : 256bit or 512bit registers used in the function return register. */
15760 2071774 : if (CALL_P (insn))
15761 : {
15762 49731 : bool avx_upper_reg_found = false;
15763 49731 : note_stores (insn, ix86_check_avx_upper_stores, &avx_upper_reg_found);
15764 :
15765 49731 : if (avx_upper_reg_found)
15766 : return AVX_U128_DIRTY;
15767 :
15768 : /* If the function doesn't clobber any sse registers or only clobber
15769 : 128-bit part, Then vzeroupper isn't issued before the function exit.
15770 : the status not CLEAN but ANY after the function. */
15771 49161 : const function_abi &abi = insn_callee_abi (insn);
15772 49161 : if (!(SIBLING_CALL_P (insn)
15773 47880 : || hard_reg_set_subset_p (reg_class_contents[SSE_REGS],
15774 47880 : abi.mode_clobbers (V4DImode))))
15775 8721 : return AVX_U128_ANY;
15776 :
15777 40440 : return AVX_U128_CLEAN;
15778 : }
15779 :
15780 : /* Otherwise, return current mode. Remember that if insn
15781 : references AVX 256bit or 512bit registers, the mode was already
15782 : changed to DIRTY from MODE_NEEDED. */
15783 : return mode;
15784 : }
15785 :
15786 : /* Return the mode that an insn results in. */
15787 :
15788 : static int
15789 2495287 : ix86_mode_after (int entity, int mode, rtx_insn *insn, HARD_REG_SET)
15790 : {
15791 2495287 : switch (entity)
15792 : {
15793 : case X86_DIRFLAG:
15794 : return mode;
15795 2071935 : case AVX_U128:
15796 2071935 : return ix86_avx_u128_mode_after (mode, insn);
15797 : case I387_ROUNDEVEN:
15798 : case I387_TRUNC:
15799 : case I387_FLOOR:
15800 : case I387_CEIL:
15801 : return mode;
15802 0 : default:
15803 0 : gcc_unreachable ();
15804 : }
15805 : }
15806 :
15807 : static int
15808 120 : ix86_dirflag_mode_entry (void)
15809 : {
15810 : /* For TARGET_CLD or in the interrupt handler we can't assume
15811 : direction flag state at function entry. */
15812 120 : if (TARGET_CLD
15813 118 : || cfun->machine->func_type != TYPE_NORMAL)
15814 120 : return X86_DIRFLAG_ANY;
15815 :
15816 : return X86_DIRFLAG_RESET;
15817 : }
15818 :
15819 : static int
15820 123119 : ix86_avx_u128_mode_entry (void)
15821 : {
15822 123119 : tree arg;
15823 :
15824 : /* Entry mode is set to AVX_U128_DIRTY if there are
15825 : 256bit or 512bit modes used in function arguments. */
15826 310992 : for (arg = DECL_ARGUMENTS (current_function_decl); arg;
15827 187873 : arg = TREE_CHAIN (arg))
15828 : {
15829 221822 : rtx incoming = DECL_INCOMING_RTL (arg);
15830 :
15831 221822 : if (incoming && ix86_check_avx_upper_register (incoming))
15832 : return AVX_U128_DIRTY;
15833 : }
15834 :
15835 : return AVX_U128_CLEAN;
15836 : }
15837 :
15838 : /* Return a mode that ENTITY is assumed to be
15839 : switched to at function entry. */
15840 :
15841 : static int
15842 75894 : ix86_mode_entry (int entity)
15843 : {
15844 75894 : switch (entity)
15845 : {
15846 120 : case X86_DIRFLAG:
15847 120 : return ix86_dirflag_mode_entry ();
15848 74629 : case AVX_U128:
15849 74629 : return ix86_avx_u128_mode_entry ();
15850 : case I387_ROUNDEVEN:
15851 : case I387_TRUNC:
15852 : case I387_FLOOR:
15853 : case I387_CEIL:
15854 : return I387_CW_ANY;
15855 0 : default:
15856 0 : gcc_unreachable ();
15857 : }
15858 : }
15859 :
15860 : static int
15861 73379 : ix86_avx_u128_mode_exit (void)
15862 : {
15863 73379 : rtx reg = crtl->return_rtx;
15864 :
15865 : /* Exit mode is set to AVX_U128_DIRTY if there are 256bit
15866 : or 512 bit modes used in the function return register. */
15867 73379 : if (reg && ix86_check_avx_upper_register (reg))
15868 : return AVX_U128_DIRTY;
15869 :
15870 : /* Exit mode is set to AVX_U128_DIRTY if there are 256bit or 512bit
15871 : modes used in function arguments, otherwise return AVX_U128_CLEAN.
15872 : */
15873 48490 : return ix86_avx_u128_mode_entry ();
15874 : }
15875 :
15876 : /* Return a mode that ENTITY is assumed to be
15877 : switched to at function exit. */
15878 :
15879 : static int
15880 74499 : ix86_mode_exit (int entity)
15881 : {
15882 74499 : switch (entity)
15883 : {
15884 : case X86_DIRFLAG:
15885 : return X86_DIRFLAG_ANY;
15886 73379 : case AVX_U128:
15887 73379 : return ix86_avx_u128_mode_exit ();
15888 1086 : case I387_ROUNDEVEN:
15889 1086 : case I387_TRUNC:
15890 1086 : case I387_FLOOR:
15891 1086 : case I387_CEIL:
15892 1086 : return I387_CW_ANY;
15893 0 : default:
15894 0 : gcc_unreachable ();
15895 : }
15896 : }
15897 :
15898 : static int
15899 2179091 : ix86_mode_priority (int, int n)
15900 : {
15901 2179091 : return n;
15902 : }
15903 :
15904 : /* Output code to initialize control word copies used by trunc?f?i and
15905 : rounding patterns. CURRENT_MODE is set to current control word,
15906 : while NEW_MODE is set to new control word. */
15907 :
15908 : static void
15909 3312 : emit_i387_cw_initialization (int mode)
15910 : {
15911 3312 : rtx stored_mode = assign_386_stack_local (HImode, SLOT_CW_STORED);
15912 3312 : rtx new_mode;
15913 :
15914 3312 : enum ix86_stack_slot slot;
15915 :
15916 3312 : rtx reg = gen_reg_rtx (HImode);
15917 :
15918 3312 : emit_insn (gen_x86_fnstcw_1 (stored_mode));
15919 3312 : emit_move_insn (reg, copy_rtx (stored_mode));
15920 :
15921 3312 : switch (mode)
15922 : {
15923 0 : case I387_CW_ROUNDEVEN:
15924 : /* round to nearest */
15925 0 : emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00)));
15926 0 : slot = SLOT_CW_ROUNDEVEN;
15927 0 : break;
15928 :
15929 3116 : case I387_CW_TRUNC:
15930 : /* round toward zero (truncate) */
15931 3116 : emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0c00)));
15932 3116 : slot = SLOT_CW_TRUNC;
15933 3116 : break;
15934 :
15935 137 : case I387_CW_FLOOR:
15936 : /* round down toward -oo */
15937 137 : emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00)));
15938 137 : emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0400)));
15939 137 : slot = SLOT_CW_FLOOR;
15940 137 : break;
15941 :
15942 59 : case I387_CW_CEIL:
15943 : /* round up toward +oo */
15944 59 : emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00)));
15945 59 : emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0800)));
15946 59 : slot = SLOT_CW_CEIL;
15947 59 : break;
15948 :
15949 0 : default:
15950 0 : gcc_unreachable ();
15951 : }
15952 :
15953 3312 : gcc_assert (slot < MAX_386_STACK_LOCALS);
15954 :
15955 3312 : new_mode = assign_386_stack_local (HImode, slot);
15956 3312 : emit_move_insn (new_mode, reg);
15957 3312 : }
15958 :
15959 : /* Generate one or more insns to set ENTITY to MODE. */
15960 :
15961 : static void
15962 52125 : ix86_emit_mode_set (int entity, int mode, int prev_mode ATTRIBUTE_UNUSED,
15963 : HARD_REG_SET regs_live ATTRIBUTE_UNUSED)
15964 : {
15965 52125 : switch (entity)
15966 : {
15967 265 : case X86_DIRFLAG:
15968 265 : if (mode == X86_DIRFLAG_RESET)
15969 265 : emit_insn (gen_cld ());
15970 : break;
15971 43558 : case AVX_U128:
15972 43558 : if (mode == AVX_U128_CLEAN)
15973 22252 : ix86_expand_avx_vzeroupper ();
15974 : break;
15975 8302 : case I387_ROUNDEVEN:
15976 8302 : case I387_TRUNC:
15977 8302 : case I387_FLOOR:
15978 8302 : case I387_CEIL:
15979 8302 : if (mode != I387_CW_ANY
15980 8302 : && mode != I387_CW_UNINITIALIZED)
15981 3312 : emit_i387_cw_initialization (mode);
15982 : break;
15983 0 : default:
15984 0 : gcc_unreachable ();
15985 : }
15986 52125 : }
15987 :
15988 : /* Output code for INSN to convert a float to a signed int. OPERANDS
15989 : are the insn operands. The output may be [HSD]Imode and the input
15990 : operand may be [SDX]Fmode. */
15991 :
15992 : const char *
15993 7456 : output_fix_trunc (rtx_insn *insn, rtx *operands, bool fisttp)
15994 : {
15995 7456 : bool stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG);
15996 7456 : bool dimode_p = GET_MODE (operands[0]) == DImode;
15997 7456 : int round_mode = get_attr_i387_cw (insn);
15998 :
15999 7456 : static char buf[40];
16000 7456 : const char *p;
16001 :
16002 : /* Jump through a hoop or two for DImode, since the hardware has no
16003 : non-popping instruction. We used to do this a different way, but
16004 : that was somewhat fragile and broke with post-reload splitters. */
16005 7456 : if ((dimode_p || fisttp) && !stack_top_dies)
16006 25 : output_asm_insn ("fld\t%y1", operands);
16007 :
16008 7456 : gcc_assert (STACK_TOP_P (operands[1]));
16009 7456 : gcc_assert (MEM_P (operands[0]));
16010 7456 : gcc_assert (GET_MODE (operands[1]) != TFmode);
16011 :
16012 7456 : if (fisttp)
16013 : return "fisttp%Z0\t%0";
16014 :
16015 7455 : strcpy (buf, "fist");
16016 :
16017 7455 : if (round_mode != I387_CW_ANY)
16018 7411 : output_asm_insn ("fldcw\t%3", operands);
16019 :
16020 7455 : p = "p%Z0\t%0";
16021 7455 : strcat (buf, p + !(stack_top_dies || dimode_p));
16022 :
16023 7455 : output_asm_insn (buf, operands);
16024 :
16025 7455 : if (round_mode != I387_CW_ANY)
16026 7411 : output_asm_insn ("fldcw\t%2", operands);
16027 :
16028 : return "";
16029 : }
16030 :
16031 : /* Output code for x87 ffreep insn. The OPNO argument, which may only
16032 : have the values zero or one, indicates the ffreep insn's operand
16033 : from the OPERANDS array. */
16034 :
16035 : static const char *
16036 274567 : output_387_ffreep (rtx *operands ATTRIBUTE_UNUSED, int opno)
16037 : {
16038 0 : if (TARGET_USE_FFREEP)
16039 : #ifdef HAVE_AS_IX86_FFREEP
16040 0 : return opno ? "ffreep\t%y1" : "ffreep\t%y0";
16041 : #else
16042 : {
16043 : static char retval[32];
16044 : int regno = REGNO (operands[opno]);
16045 :
16046 : gcc_assert (STACK_REGNO_P (regno));
16047 :
16048 : regno -= FIRST_STACK_REG;
16049 :
16050 : snprintf (retval, sizeof (retval), ASM_SHORT "0xc%ddf", regno);
16051 : return retval;
16052 : }
16053 : #endif
16054 :
16055 0 : return opno ? "fstp\t%y1" : "fstp\t%y0";
16056 : }
16057 :
16058 :
16059 : /* Output code for INSN to compare OPERANDS. EFLAGS_P is 1 when fcomi
16060 : should be used. UNORDERED_P is true when fucom should be used. */
16061 :
16062 : const char *
16063 108193 : output_fp_compare (rtx_insn *insn, rtx *operands,
16064 : bool eflags_p, bool unordered_p)
16065 : {
16066 108193 : rtx *xops = eflags_p ? &operands[0] : &operands[1];
16067 108193 : bool stack_top_dies;
16068 :
16069 108193 : static char buf[40];
16070 108193 : const char *p;
16071 :
16072 108193 : gcc_assert (STACK_TOP_P (xops[0]));
16073 :
16074 108193 : stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG);
16075 :
16076 108193 : if (eflags_p)
16077 : {
16078 108193 : p = unordered_p ? "fucomi" : "fcomi";
16079 108193 : strcpy (buf, p);
16080 :
16081 108193 : p = "p\t{%y1, %0|%0, %y1}";
16082 108193 : strcat (buf, p + !stack_top_dies);
16083 :
16084 108193 : return buf;
16085 : }
16086 :
16087 0 : if (STACK_REG_P (xops[1])
16088 0 : && stack_top_dies
16089 0 : && find_regno_note (insn, REG_DEAD, FIRST_STACK_REG + 1))
16090 : {
16091 0 : gcc_assert (REGNO (xops[1]) == FIRST_STACK_REG + 1);
16092 :
16093 : /* If both the top of the 387 stack die, and the other operand
16094 : is also a stack register that dies, then this must be a
16095 : `fcompp' float compare. */
16096 0 : p = unordered_p ? "fucompp" : "fcompp";
16097 0 : strcpy (buf, p);
16098 : }
16099 0 : else if (const0_operand (xops[1], VOIDmode))
16100 : {
16101 0 : gcc_assert (!unordered_p);
16102 0 : strcpy (buf, "ftst");
16103 : }
16104 : else
16105 : {
16106 0 : if (GET_MODE_CLASS (GET_MODE (xops[1])) == MODE_INT)
16107 : {
16108 0 : gcc_assert (!unordered_p);
16109 : p = "ficom";
16110 : }
16111 : else
16112 0 : p = unordered_p ? "fucom" : "fcom";
16113 :
16114 0 : strcpy (buf, p);
16115 :
16116 0 : p = "p%Z2\t%y2";
16117 0 : strcat (buf, p + !stack_top_dies);
16118 : }
16119 :
16120 0 : output_asm_insn (buf, operands);
16121 0 : return "fnstsw\t%0";
16122 : }
16123 :
16124 : void
16125 128769 : ix86_output_addr_vec_elt (FILE *file, int value)
16126 : {
16127 128769 : const char *directive = ASM_LONG;
16128 :
16129 : #ifdef ASM_QUAD
16130 128769 : if (TARGET_LP64)
16131 117030 : directive = ASM_QUAD;
16132 : #else
16133 : gcc_assert (!TARGET_64BIT);
16134 : #endif
16135 :
16136 128769 : fprintf (file, "%s%s%d\n", directive, LPREFIX, value);
16137 128769 : }
16138 :
16139 : void
16140 27054 : ix86_output_addr_diff_elt (FILE *file, int value, int rel)
16141 : {
16142 27054 : const char *directive = ASM_LONG;
16143 :
16144 : #ifdef ASM_QUAD
16145 40527 : if (TARGET_64BIT && CASE_VECTOR_MODE == DImode)
16146 : directive = ASM_QUAD;
16147 : #else
16148 : gcc_assert (!TARGET_64BIT);
16149 : #endif
16150 : /* We can't use @GOTOFF for text labels on VxWorks; see gotoff_operand. */
16151 27054 : if (TARGET_64BIT || TARGET_VXWORKS_VAROFF)
16152 13473 : fprintf (file, "%s%s%d-%s%d\n",
16153 : directive, LPREFIX, value, LPREFIX, rel);
16154 : #if TARGET_MACHO
16155 : else if (TARGET_MACHO)
16156 : {
16157 : fprintf (file, ASM_LONG "%s%d-", LPREFIX, value);
16158 : machopic_output_function_base_name (file);
16159 : putc ('\n', file);
16160 : }
16161 : #endif
16162 13581 : else if (HAVE_AS_GOTOFF_IN_DATA)
16163 13581 : fprintf (file, ASM_LONG "%s%d@GOTOFF\n", LPREFIX, value);
16164 : else
16165 : asm_fprintf (file, ASM_LONG "%U%s+[.-%s%d]\n",
16166 : GOT_SYMBOL_NAME, LPREFIX, value);
16167 27054 : }
16168 :
16169 : #define LEA_MAX_STALL (3)
16170 : #define LEA_SEARCH_THRESHOLD (LEA_MAX_STALL << 1)
16171 :
16172 : /* Increase given DISTANCE in half-cycles according to
16173 : dependencies between PREV and NEXT instructions.
16174 : Add 1 half-cycle if there is no dependency and
16175 : go to next cycle if there is some dependency. */
16176 :
16177 : static unsigned int
16178 2138 : increase_distance (rtx_insn *prev, rtx_insn *next, unsigned int distance)
16179 : {
16180 2138 : df_ref def, use;
16181 :
16182 2138 : if (!prev || !next)
16183 752 : return distance + (distance & 1) + 2;
16184 :
16185 1386 : if (!DF_INSN_USES (next) || !DF_INSN_DEFS (prev))
16186 228 : return distance + 1;
16187 :
16188 1920 : FOR_EACH_INSN_USE (use, next)
16189 2449 : FOR_EACH_INSN_DEF (def, prev)
16190 1687 : if (!DF_REF_IS_ARTIFICIAL (def)
16191 1687 : && DF_REF_REGNO (use) == DF_REF_REGNO (def))
16192 741 : return distance + (distance & 1) + 2;
16193 :
16194 417 : return distance + 1;
16195 : }
16196 :
16197 : /* Function checks if instruction INSN defines register number
16198 : REGNO1 or REGNO2. */
16199 :
16200 : bool
16201 2081 : insn_defines_reg (unsigned int regno1, unsigned int regno2,
16202 : rtx_insn *insn)
16203 : {
16204 2081 : df_ref def;
16205 :
16206 3757 : FOR_EACH_INSN_DEF (def, insn)
16207 2082 : if (DF_REF_REG_DEF_P (def)
16208 2082 : && !DF_REF_IS_ARTIFICIAL (def)
16209 2082 : && (regno1 == DF_REF_REGNO (def)
16210 1692 : || regno2 == DF_REF_REGNO (def)))
16211 : return true;
16212 :
16213 : return false;
16214 : }
16215 :
16216 : /* Function checks if instruction INSN uses register number
16217 : REGNO as a part of address expression. */
16218 :
16219 : static bool
16220 1189 : insn_uses_reg_mem (unsigned int regno, rtx insn)
16221 : {
16222 1189 : df_ref use;
16223 :
16224 2489 : FOR_EACH_INSN_USE (use, insn)
16225 1392 : if (DF_REF_REG_MEM_P (use) && regno == DF_REF_REGNO (use))
16226 : return true;
16227 :
16228 : return false;
16229 : }
16230 :
16231 : /* Search backward for non-agu definition of register number REGNO1
16232 : or register number REGNO2 in basic block starting from instruction
16233 : START up to head of basic block or instruction INSN.
16234 :
16235 : Function puts true value into *FOUND var if definition was found
16236 : and false otherwise.
16237 :
16238 : Distance in half-cycles between START and found instruction or head
16239 : of BB is added to DISTANCE and returned. */
16240 :
16241 : static int
16242 627 : distance_non_agu_define_in_bb (unsigned int regno1, unsigned int regno2,
16243 : rtx_insn *insn, int distance,
16244 : rtx_insn *start, bool *found)
16245 : {
16246 627 : basic_block bb = start ? BLOCK_FOR_INSN (start) : NULL;
16247 627 : rtx_insn *prev = start;
16248 627 : rtx_insn *next = NULL;
16249 :
16250 627 : *found = false;
16251 :
16252 627 : while (prev
16253 1865 : && prev != insn
16254 1865 : && distance < LEA_SEARCH_THRESHOLD)
16255 : {
16256 1663 : if (NONDEBUG_INSN_P (prev) && NONJUMP_INSN_P (prev))
16257 : {
16258 949 : distance = increase_distance (prev, next, distance);
16259 949 : if (insn_defines_reg (regno1, regno2, prev))
16260 : {
16261 245 : if (recog_memoized (prev) < 0
16262 245 : || get_attr_type (prev) != TYPE_LEA)
16263 : {
16264 201 : *found = true;
16265 201 : return distance;
16266 : }
16267 : }
16268 :
16269 : next = prev;
16270 : }
16271 1462 : if (prev == BB_HEAD (bb))
16272 : break;
16273 :
16274 1238 : prev = PREV_INSN (prev);
16275 : }
16276 :
16277 : return distance;
16278 : }
16279 :
16280 : /* Search backward for non-agu definition of register number REGNO1
16281 : or register number REGNO2 in INSN's basic block until
16282 : 1. Pass LEA_SEARCH_THRESHOLD instructions, or
16283 : 2. Reach neighbor BBs boundary, or
16284 : 3. Reach agu definition.
16285 : Returns the distance between the non-agu definition point and INSN.
16286 : If no definition point, returns -1. */
16287 :
16288 : static int
16289 431 : distance_non_agu_define (unsigned int regno1, unsigned int regno2,
16290 : rtx_insn *insn)
16291 : {
16292 431 : basic_block bb = BLOCK_FOR_INSN (insn);
16293 431 : int distance = 0;
16294 431 : bool found = false;
16295 :
16296 431 : if (insn != BB_HEAD (bb))
16297 431 : distance = distance_non_agu_define_in_bb (regno1, regno2, insn,
16298 : distance, PREV_INSN (insn),
16299 : &found);
16300 :
16301 431 : if (!found && distance < LEA_SEARCH_THRESHOLD)
16302 : {
16303 168 : edge e;
16304 168 : edge_iterator ei;
16305 168 : bool simple_loop = false;
16306 :
16307 338 : FOR_EACH_EDGE (e, ei, bb->preds)
16308 207 : if (e->src == bb)
16309 : {
16310 : simple_loop = true;
16311 : break;
16312 : }
16313 :
16314 168 : if (simple_loop)
16315 37 : distance = distance_non_agu_define_in_bb (regno1, regno2,
16316 : insn, distance,
16317 37 : BB_END (bb), &found);
16318 : else
16319 : {
16320 131 : int shortest_dist = -1;
16321 131 : bool found_in_bb = false;
16322 :
16323 290 : FOR_EACH_EDGE (e, ei, bb->preds)
16324 : {
16325 159 : int bb_dist
16326 318 : = distance_non_agu_define_in_bb (regno1, regno2,
16327 : insn, distance,
16328 159 : BB_END (e->src),
16329 : &found_in_bb);
16330 159 : if (found_in_bb)
16331 : {
16332 24 : if (shortest_dist < 0)
16333 : shortest_dist = bb_dist;
16334 0 : else if (bb_dist > 0)
16335 0 : shortest_dist = MIN (bb_dist, shortest_dist);
16336 :
16337 24 : found = true;
16338 : }
16339 : }
16340 :
16341 131 : distance = shortest_dist;
16342 : }
16343 : }
16344 :
16345 431 : if (!found)
16346 : return -1;
16347 :
16348 201 : return distance >> 1;
16349 : }
16350 :
16351 : /* Return the distance in half-cycles between INSN and the next
16352 : insn that uses register number REGNO in memory address added
16353 : to DISTANCE. Return -1 if REGNO0 is set.
16354 :
16355 : Put true value into *FOUND if register usage was found and
16356 : false otherwise.
16357 : Put true value into *REDEFINED if register redefinition was
16358 : found and false otherwise. */
16359 :
16360 : static int
16361 770 : distance_agu_use_in_bb (unsigned int regno,
16362 : rtx_insn *insn, int distance, rtx_insn *start,
16363 : bool *found, bool *redefined)
16364 : {
16365 770 : basic_block bb = NULL;
16366 770 : rtx_insn *next = start;
16367 770 : rtx_insn *prev = NULL;
16368 :
16369 770 : *found = false;
16370 770 : *redefined = false;
16371 :
16372 770 : if (start != NULL_RTX)
16373 : {
16374 753 : bb = BLOCK_FOR_INSN (start);
16375 753 : if (start != BB_HEAD (bb))
16376 : /* If insn and start belong to the same bb, set prev to insn,
16377 : so the call to increase_distance will increase the distance
16378 : between insns by 1. */
16379 414 : prev = insn;
16380 : }
16381 :
16382 2576 : while (next
16383 2576 : && next != insn
16384 2576 : && distance < LEA_SEARCH_THRESHOLD)
16385 : {
16386 2387 : if (NONDEBUG_INSN_P (next) && NONJUMP_INSN_P (next))
16387 : {
16388 1189 : distance = increase_distance(prev, next, distance);
16389 1189 : if (insn_uses_reg_mem (regno, next))
16390 : {
16391 : /* Return DISTANCE if OP0 is used in memory
16392 : address in NEXT. */
16393 92 : *found = true;
16394 92 : return distance;
16395 : }
16396 :
16397 1097 : if (insn_defines_reg (regno, INVALID_REGNUM, next))
16398 : {
16399 : /* Return -1 if OP0 is set in NEXT. */
16400 156 : *redefined = true;
16401 156 : return -1;
16402 : }
16403 :
16404 : prev = next;
16405 : }
16406 :
16407 2139 : if (next == BB_END (bb))
16408 : break;
16409 :
16410 1806 : next = NEXT_INSN (next);
16411 : }
16412 :
16413 : return distance;
16414 : }
16415 :
16416 : /* Return the distance between INSN and the next insn that uses
16417 : register number REGNO0 in memory address. Return -1 if no such
16418 : a use is found within LEA_SEARCH_THRESHOLD or REGNO0 is set. */
16419 :
16420 : static int
16421 431 : distance_agu_use (unsigned int regno0, rtx_insn *insn)
16422 : {
16423 431 : basic_block bb = BLOCK_FOR_INSN (insn);
16424 431 : int distance = 0;
16425 431 : bool found = false;
16426 431 : bool redefined = false;
16427 :
16428 431 : if (insn != BB_END (bb))
16429 414 : distance = distance_agu_use_in_bb (regno0, insn, distance,
16430 : NEXT_INSN (insn),
16431 : &found, &redefined);
16432 :
16433 431 : if (!found && !redefined && distance < LEA_SEARCH_THRESHOLD)
16434 : {
16435 251 : edge e;
16436 251 : edge_iterator ei;
16437 251 : bool simple_loop = false;
16438 :
16439 537 : FOR_EACH_EDGE (e, ei, bb->succs)
16440 356 : if (e->dest == bb)
16441 : {
16442 : simple_loop = true;
16443 : break;
16444 : }
16445 :
16446 251 : if (simple_loop)
16447 70 : distance = distance_agu_use_in_bb (regno0, insn,
16448 : distance, BB_HEAD (bb),
16449 : &found, &redefined);
16450 : else
16451 : {
16452 181 : int shortest_dist = -1;
16453 181 : bool found_in_bb = false;
16454 181 : bool redefined_in_bb = false;
16455 :
16456 467 : FOR_EACH_EDGE (e, ei, bb->succs)
16457 : {
16458 286 : int bb_dist
16459 572 : = distance_agu_use_in_bb (regno0, insn,
16460 286 : distance, BB_HEAD (e->dest),
16461 : &found_in_bb, &redefined_in_bb);
16462 286 : if (found_in_bb)
16463 : {
16464 18 : if (shortest_dist < 0)
16465 : shortest_dist = bb_dist;
16466 2 : else if (bb_dist > 0)
16467 2 : shortest_dist = MIN (bb_dist, shortest_dist);
16468 :
16469 18 : found = true;
16470 : }
16471 : }
16472 :
16473 181 : distance = shortest_dist;
16474 : }
16475 : }
16476 :
16477 431 : if (!found || redefined)
16478 : return -1;
16479 :
16480 90 : return distance >> 1;
16481 : }
16482 :
16483 : /* Define this macro to tune LEA priority vs ADD, it take effect when
16484 : there is a dilemma of choosing LEA or ADD
16485 : Negative value: ADD is more preferred than LEA
16486 : Zero: Neutral
16487 : Positive value: LEA is more preferred than ADD. */
16488 : #define IX86_LEA_PRIORITY 0
16489 :
16490 : /* Return true if usage of lea INSN has performance advantage
16491 : over a sequence of instructions. Instructions sequence has
16492 : SPLIT_COST cycles higher latency than lea latency. */
16493 :
16494 : static bool
16495 1601 : ix86_lea_outperforms (rtx_insn *insn, unsigned int regno0, unsigned int regno1,
16496 : unsigned int regno2, int split_cost, bool has_scale)
16497 : {
16498 1601 : int dist_define, dist_use;
16499 :
16500 : /* For Atom processors newer than Bonnell, if using a 2-source or
16501 : 3-source LEA for non-destructive destination purposes, or due to
16502 : wanting ability to use SCALE, the use of LEA is justified. */
16503 1601 : if (!TARGET_CPU_P (BONNELL))
16504 : {
16505 1170 : if (has_scale)
16506 : return true;
16507 1151 : if (split_cost < 1)
16508 : return false;
16509 406 : if (regno0 == regno1 || regno0 == regno2)
16510 : return false;
16511 : return true;
16512 : }
16513 :
16514 : /* Remember recog_data content. */
16515 431 : struct recog_data_d recog_data_save = recog_data;
16516 :
16517 431 : dist_define = distance_non_agu_define (regno1, regno2, insn);
16518 431 : dist_use = distance_agu_use (regno0, insn);
16519 :
16520 : /* distance_non_agu_define can call get_attr_type which can call
16521 : recog_memoized, restore recog_data back to previous content. */
16522 431 : recog_data = recog_data_save;
16523 :
16524 431 : if (dist_define < 0 || dist_define >= LEA_MAX_STALL)
16525 : {
16526 : /* If there is no non AGU operand definition, no AGU
16527 : operand usage and split cost is 0 then both lea
16528 : and non lea variants have same priority. Currently
16529 : we prefer lea for 64 bit code and non lea on 32 bit
16530 : code. */
16531 233 : if (dist_use < 0 && split_cost == 0)
16532 97 : return TARGET_64BIT || IX86_LEA_PRIORITY;
16533 : else
16534 : return true;
16535 : }
16536 :
16537 : /* With longer definitions distance lea is more preferable.
16538 : Here we change it to take into account splitting cost and
16539 : lea priority. */
16540 198 : dist_define += split_cost + IX86_LEA_PRIORITY;
16541 :
16542 : /* If there is no use in memory address then we just check
16543 : that split cost exceeds AGU stall. */
16544 198 : if (dist_use < 0)
16545 194 : return dist_define > LEA_MAX_STALL;
16546 :
16547 : /* If this insn has both backward non-agu dependence and forward
16548 : agu dependence, the one with short distance takes effect. */
16549 4 : return dist_define >= dist_use;
16550 : }
16551 :
16552 : /* Return true if we need to split op0 = op1 + op2 into a sequence of
16553 : move and add to avoid AGU stalls. */
16554 :
16555 : bool
16556 9159884 : ix86_avoid_lea_for_add (rtx_insn *insn, rtx operands[])
16557 : {
16558 9159884 : unsigned int regno0, regno1, regno2;
16559 :
16560 : /* Check if we need to optimize. */
16561 9159884 : if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
16562 9159100 : return false;
16563 :
16564 784 : regno0 = true_regnum (operands[0]);
16565 784 : regno1 = true_regnum (operands[1]);
16566 784 : regno2 = true_regnum (operands[2]);
16567 :
16568 : /* We need to split only adds with non destructive
16569 : destination operand. */
16570 784 : if (regno0 == regno1 || regno0 == regno2)
16571 : return false;
16572 : else
16573 246 : return !ix86_lea_outperforms (insn, regno0, regno1, regno2, 1, false);
16574 : }
16575 :
16576 : /* Return true if we should emit lea instruction instead of mov
16577 : instruction. */
16578 :
16579 : bool
16580 29850094 : ix86_use_lea_for_mov (rtx_insn *insn, rtx operands[])
16581 : {
16582 29850094 : unsigned int regno0, regno1;
16583 :
16584 : /* Check if we need to optimize. */
16585 29850094 : if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
16586 29847825 : return false;
16587 :
16588 : /* Use lea for reg to reg moves only. */
16589 2269 : if (!REG_P (operands[0]) || !REG_P (operands[1]))
16590 : return false;
16591 :
16592 465 : regno0 = true_regnum (operands[0]);
16593 465 : regno1 = true_regnum (operands[1]);
16594 :
16595 465 : return ix86_lea_outperforms (insn, regno0, regno1, INVALID_REGNUM, 0, false);
16596 : }
16597 :
16598 : /* Return true if we need to split lea into a sequence of
16599 : instructions to avoid AGU stalls during peephole2. */
16600 :
16601 : bool
16602 11345331 : ix86_avoid_lea_for_addr (rtx_insn *insn, rtx operands[])
16603 : {
16604 11345331 : unsigned int regno0, regno1, regno2;
16605 11345331 : int split_cost;
16606 11345331 : struct ix86_address parts;
16607 11345331 : int ok;
16608 :
16609 : /* The "at least two components" test below might not catch simple
16610 : move or zero extension insns if parts.base is non-NULL and parts.disp
16611 : is const0_rtx as the only components in the address, e.g. if the
16612 : register is %rbp or %r13. As this test is much cheaper and moves or
16613 : zero extensions are the common case, do this check first. */
16614 11345331 : if (REG_P (operands[1])
16615 11345331 : || (SImode_address_operand (operands[1], VOIDmode)
16616 138900 : && REG_P (XEXP (operands[1], 0))))
16617 4153771 : return false;
16618 :
16619 7191560 : ok = ix86_decompose_address (operands[1], &parts);
16620 7191560 : gcc_assert (ok);
16621 :
16622 : /* There should be at least two components in the address. */
16623 7191560 : if ((parts.base != NULL_RTX) + (parts.index != NULL_RTX)
16624 7191560 : + (parts.disp != NULL_RTX) + (parts.scale > 1) < 2)
16625 : return false;
16626 :
16627 : /* We should not split into add if non legitimate pic
16628 : operand is used as displacement. */
16629 2715350 : if (parts.disp && flag_pic && !LEGITIMATE_PIC_OPERAND_P (parts.disp))
16630 : return false;
16631 :
16632 2665298 : regno0 = true_regnum (operands[0]) ;
16633 2665298 : regno1 = INVALID_REGNUM;
16634 2665298 : regno2 = INVALID_REGNUM;
16635 :
16636 2665298 : if (parts.base)
16637 2587878 : regno1 = true_regnum (parts.base);
16638 2665298 : if (parts.index)
16639 481757 : regno2 = true_regnum (parts.index);
16640 :
16641 : /* Use add for a = a + b and a = b + a since it is faster and shorter
16642 : than lea for most processors. For the processors like BONNELL, if
16643 : the destination register of LEA holds an actual address which will
16644 : be used soon, LEA is better and otherwise ADD is better. */
16645 2665298 : if (!TARGET_CPU_P (BONNELL)
16646 2665167 : && parts.scale == 1
16647 2418238 : && (!parts.disp || parts.disp == const0_rtx)
16648 176845 : && (regno0 == regno1 || regno0 == regno2))
16649 : return true;
16650 :
16651 : /* Split with -Oz if the encoding requires fewer bytes. */
16652 2659067 : if (optimize_size > 1
16653 27 : && parts.scale > 1
16654 4 : && !parts.base
16655 4 : && (!parts.disp || parts.disp == const0_rtx))
16656 : return true;
16657 :
16658 : /* Check we need to optimize. */
16659 2659063 : if (!TARGET_AVOID_LEA_FOR_ADDR || optimize_function_for_size_p (cfun))
16660 2658720 : return false;
16661 :
16662 343 : split_cost = 0;
16663 :
16664 : /* Compute how many cycles we will add to execution time
16665 : if split lea into a sequence of instructions. */
16666 343 : if (parts.base || parts.index)
16667 : {
16668 : /* Have to use mov instruction if non destructive
16669 : destination form is used. */
16670 343 : if (regno1 != regno0 && regno2 != regno0)
16671 268 : split_cost += 1;
16672 :
16673 : /* Have to add index to base if both exist. */
16674 343 : if (parts.base && parts.index)
16675 55 : split_cost += 1;
16676 :
16677 : /* Have to use shift and adds if scale is 2 or greater. */
16678 343 : if (parts.scale > 1)
16679 : {
16680 30 : if (regno0 != regno1)
16681 24 : split_cost += 1;
16682 6 : else if (regno2 == regno0)
16683 0 : split_cost += 4;
16684 : else
16685 6 : split_cost += parts.scale;
16686 : }
16687 :
16688 : /* Have to use add instruction with immediate if
16689 : disp is non zero. */
16690 343 : if (parts.disp && parts.disp != const0_rtx)
16691 280 : split_cost += 1;
16692 :
16693 : /* Subtract the price of lea. */
16694 343 : split_cost -= 1;
16695 : }
16696 :
16697 343 : return !ix86_lea_outperforms (insn, regno0, regno1, regno2, split_cost,
16698 343 : parts.scale > 1);
16699 : }
16700 :
16701 : /* Return true if it is ok to optimize an ADD operation to LEA
16702 : operation to avoid flag register consumation. For most processors,
16703 : ADD is faster than LEA. For the processors like BONNELL, if the
16704 : destination register of LEA holds an actual address which will be
16705 : used soon, LEA is better and otherwise ADD is better. */
16706 :
16707 : bool
16708 9221132 : ix86_lea_for_add_ok (rtx_insn *insn, rtx operands[])
16709 : {
16710 9221132 : unsigned int regno0 = true_regnum (operands[0]);
16711 9221132 : unsigned int regno1 = true_regnum (operands[1]);
16712 9221132 : unsigned int regno2 = true_regnum (operands[2]);
16713 :
16714 : /* If a = b + c, (a!=b && a!=c), must use lea form. */
16715 9221132 : if (regno0 != regno1 && regno0 != regno2)
16716 : return true;
16717 :
16718 7163677 : if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
16719 7163130 : return false;
16720 :
16721 547 : return ix86_lea_outperforms (insn, regno0, regno1, regno2, 0, false);
16722 : }
16723 :
16724 : /* Return true if destination reg of SET_BODY is shift count of
16725 : USE_BODY. */
16726 :
16727 : static bool
16728 83 : ix86_dep_by_shift_count_body (const_rtx set_body, const_rtx use_body)
16729 : {
16730 83 : rtx set_dest;
16731 83 : rtx shift_rtx;
16732 83 : int i;
16733 :
16734 : /* Retrieve destination of SET_BODY. */
16735 83 : switch (GET_CODE (set_body))
16736 : {
16737 67 : case SET:
16738 67 : set_dest = SET_DEST (set_body);
16739 67 : if (!set_dest || !REG_P (set_dest))
16740 : return false;
16741 66 : break;
16742 8 : case PARALLEL:
16743 24 : for (i = XVECLEN (set_body, 0) - 1; i >= 0; i--)
16744 16 : if (ix86_dep_by_shift_count_body (XVECEXP (set_body, 0, i),
16745 : use_body))
16746 : return true;
16747 : /* FALLTHROUGH */
16748 : default:
16749 : return false;
16750 : }
16751 :
16752 : /* Retrieve shift count of USE_BODY. */
16753 66 : switch (GET_CODE (use_body))
16754 : {
16755 22 : case SET:
16756 22 : shift_rtx = XEXP (use_body, 1);
16757 22 : break;
16758 22 : case PARALLEL:
16759 66 : for (i = XVECLEN (use_body, 0) - 1; i >= 0; i--)
16760 44 : if (ix86_dep_by_shift_count_body (set_body,
16761 44 : XVECEXP (use_body, 0, i)))
16762 : return true;
16763 : /* FALLTHROUGH */
16764 : default:
16765 : return false;
16766 : }
16767 :
16768 22 : if (shift_rtx
16769 22 : && (GET_CODE (shift_rtx) == ASHIFT
16770 21 : || GET_CODE (shift_rtx) == LSHIFTRT
16771 5 : || GET_CODE (shift_rtx) == ASHIFTRT
16772 0 : || GET_CODE (shift_rtx) == ROTATE
16773 0 : || GET_CODE (shift_rtx) == ROTATERT))
16774 : {
16775 22 : rtx shift_count = XEXP (shift_rtx, 1);
16776 :
16777 : /* Return true if shift count is dest of SET_BODY. */
16778 22 : if (REG_P (shift_count))
16779 : {
16780 : /* Add check since it can be invoked before register
16781 : allocation in pre-reload schedule. */
16782 0 : if (reload_completed
16783 0 : && true_regnum (set_dest) == true_regnum (shift_count))
16784 : return true;
16785 0 : else if (REGNO(set_dest) == REGNO(shift_count))
16786 : return true;
16787 : }
16788 : }
16789 :
16790 : return false;
16791 : }
16792 :
16793 : /* Return true if destination reg of SET_INSN is shift count of
16794 : USE_INSN. */
16795 :
16796 : bool
16797 23 : ix86_dep_by_shift_count (const_rtx set_insn, const_rtx use_insn)
16798 : {
16799 23 : return ix86_dep_by_shift_count_body (PATTERN (set_insn),
16800 23 : PATTERN (use_insn));
16801 : }
16802 :
16803 : /* Return TRUE if the operands to a vec_interleave_{high,low}v2df
16804 : are ok, keeping in mind the possible movddup alternative. */
16805 :
16806 : bool
16807 94590 : ix86_vec_interleave_v2df_operator_ok (rtx operands[3], bool high)
16808 : {
16809 94590 : if (MEM_P (operands[0]))
16810 2055 : return rtx_equal_p (operands[0], operands[1 + high]);
16811 92535 : if (MEM_P (operands[1]) && MEM_P (operands[2]))
16812 1050 : return false;
16813 : return true;
16814 : }
16815 :
16816 : /* A subroutine of ix86_build_signbit_mask. If VECT is true,
16817 : then replicate the value for all elements of the vector
16818 : register. */
16819 :
16820 : rtx
16821 76063 : ix86_build_const_vector (machine_mode mode, bool vect, rtx value)
16822 : {
16823 76063 : int i, n_elt;
16824 76063 : rtvec v;
16825 76063 : machine_mode scalar_mode;
16826 :
16827 76063 : switch (mode)
16828 : {
16829 1323 : case E_V64QImode:
16830 1323 : case E_V32QImode:
16831 1323 : case E_V16QImode:
16832 1323 : case E_V32HImode:
16833 1323 : case E_V16HImode:
16834 1323 : case E_V8HImode:
16835 1323 : case E_V16SImode:
16836 1323 : case E_V8SImode:
16837 1323 : case E_V4SImode:
16838 1323 : case E_V2SImode:
16839 1323 : case E_V8DImode:
16840 1323 : case E_V4DImode:
16841 1323 : case E_V2DImode:
16842 1323 : gcc_assert (vect);
16843 : /* FALLTHRU */
16844 76063 : case E_V2HFmode:
16845 76063 : case E_V4HFmode:
16846 76063 : case E_V8HFmode:
16847 76063 : case E_V16HFmode:
16848 76063 : case E_V32HFmode:
16849 76063 : case E_V16SFmode:
16850 76063 : case E_V8SFmode:
16851 76063 : case E_V4SFmode:
16852 76063 : case E_V2SFmode:
16853 76063 : case E_V8DFmode:
16854 76063 : case E_V4DFmode:
16855 76063 : case E_V2DFmode:
16856 76063 : case E_V32BFmode:
16857 76063 : case E_V16BFmode:
16858 76063 : case E_V8BFmode:
16859 76063 : case E_V4BFmode:
16860 76063 : case E_V2BFmode:
16861 76063 : n_elt = GET_MODE_NUNITS (mode);
16862 76063 : v = rtvec_alloc (n_elt);
16863 76063 : scalar_mode = GET_MODE_INNER (mode);
16864 :
16865 76063 : RTVEC_ELT (v, 0) = value;
16866 :
16867 235600 : for (i = 1; i < n_elt; ++i)
16868 159537 : RTVEC_ELT (v, i) = vect ? value : CONST0_RTX (scalar_mode);
16869 :
16870 76063 : return gen_rtx_CONST_VECTOR (mode, v);
16871 :
16872 0 : default:
16873 0 : gcc_unreachable ();
16874 : }
16875 : }
16876 :
16877 : /* A subroutine of ix86_expand_fp_absneg_operator, copysign expanders
16878 : and ix86_expand_int_vcond. Create a mask for the sign bit in MODE
16879 : for an SSE register. If VECT is true, then replicate the mask for
16880 : all elements of the vector register. If INVERT is true, then create
16881 : a mask excluding the sign bit. */
16882 :
16883 : rtx
16884 77809 : ix86_build_signbit_mask (machine_mode mode, bool vect, bool invert)
16885 : {
16886 77809 : machine_mode vec_mode, imode;
16887 77809 : wide_int w;
16888 77809 : rtx mask, v;
16889 :
16890 77809 : switch (mode)
16891 : {
16892 : case E_V2HFmode:
16893 : case E_V4HFmode:
16894 : case E_V8HFmode:
16895 : case E_V16HFmode:
16896 : case E_V32HFmode:
16897 : case E_V32BFmode:
16898 : case E_V16BFmode:
16899 : case E_V8BFmode:
16900 : case E_V4BFmode:
16901 : case E_V2BFmode:
16902 : vec_mode = mode;
16903 : imode = HImode;
16904 : break;
16905 :
16906 34713 : case E_V16SImode:
16907 34713 : case E_V16SFmode:
16908 34713 : case E_V8SImode:
16909 34713 : case E_V4SImode:
16910 34713 : case E_V8SFmode:
16911 34713 : case E_V4SFmode:
16912 34713 : case E_V2SFmode:
16913 34713 : case E_V2SImode:
16914 34713 : vec_mode = mode;
16915 34713 : imode = SImode;
16916 34713 : break;
16917 :
16918 39830 : case E_V8DImode:
16919 39830 : case E_V4DImode:
16920 39830 : case E_V2DImode:
16921 39830 : case E_V8DFmode:
16922 39830 : case E_V4DFmode:
16923 39830 : case E_V2DFmode:
16924 39830 : vec_mode = mode;
16925 39830 : imode = DImode;
16926 39830 : break;
16927 :
16928 2765 : case E_TImode:
16929 2765 : case E_TFmode:
16930 2765 : vec_mode = VOIDmode;
16931 2765 : imode = TImode;
16932 2765 : break;
16933 :
16934 0 : default:
16935 0 : gcc_unreachable ();
16936 : }
16937 :
16938 77809 : machine_mode inner_mode = GET_MODE_INNER (mode);
16939 155618 : w = wi::set_bit_in_zero (GET_MODE_BITSIZE (inner_mode) - 1,
16940 155618 : GET_MODE_BITSIZE (inner_mode));
16941 77809 : if (invert)
16942 41310 : w = wi::bit_not (w);
16943 :
16944 : /* Force this value into the low part of a fp vector constant. */
16945 77809 : mask = immed_wide_int_const (w, imode);
16946 77809 : mask = gen_lowpart (inner_mode, mask);
16947 :
16948 77809 : if (vec_mode == VOIDmode)
16949 2765 : return force_reg (inner_mode, mask);
16950 :
16951 75044 : v = ix86_build_const_vector (vec_mode, vect, mask);
16952 75044 : return force_reg (vec_mode, v);
16953 77809 : }
16954 :
16955 : /* Return HOST_WIDE_INT for const vector OP in MODE. */
16956 :
16957 : HOST_WIDE_INT
16958 137254 : ix86_convert_const_vector_to_integer (rtx op, machine_mode mode)
16959 : {
16960 290737 : if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
16961 0 : gcc_unreachable ();
16962 :
16963 137254 : int nunits = GET_MODE_NUNITS (mode);
16964 274508 : wide_int val = wi::zero (GET_MODE_BITSIZE (mode));
16965 137254 : machine_mode innermode = GET_MODE_INNER (mode);
16966 137254 : unsigned int innermode_bits = GET_MODE_BITSIZE (innermode);
16967 :
16968 137254 : switch (mode)
16969 : {
16970 : case E_V2QImode:
16971 : case E_V4QImode:
16972 : case E_V2HImode:
16973 : case E_V8QImode:
16974 : case E_V4HImode:
16975 : case E_V2SImode:
16976 469709 : for (int i = 0; i < nunits; ++i)
16977 : {
16978 337652 : int v = INTVAL (XVECEXP (op, 0, i));
16979 337652 : wide_int wv = wi::shwi (v, innermode_bits);
16980 337652 : val = wi::insert (val, wv, innermode_bits * i, innermode_bits);
16981 337652 : }
16982 : break;
16983 88 : case E_V1SImode:
16984 88 : case E_V1DImode:
16985 88 : op = CONST_VECTOR_ELT (op, 0);
16986 88 : return INTVAL (op);
16987 : case E_V2HFmode:
16988 : case E_V2BFmode:
16989 : case E_V4HFmode:
16990 : case E_V4BFmode:
16991 : case E_V2SFmode:
16992 15373 : for (int i = 0; i < nunits; ++i)
16993 : {
16994 10264 : rtx x = XVECEXP (op, 0, i);
16995 10264 : int v = real_to_target (NULL, CONST_DOUBLE_REAL_VALUE (x),
16996 10264 : REAL_MODE_FORMAT (innermode));
16997 10264 : wide_int wv = wi::shwi (v, innermode_bits);
16998 10264 : val = wi::insert (val, wv, innermode_bits * i, innermode_bits);
16999 10264 : }
17000 : break;
17001 0 : default:
17002 0 : gcc_unreachable ();
17003 : }
17004 :
17005 137166 : return val.to_shwi ();
17006 137254 : }
17007 :
17008 32 : int ix86_get_flags_cc (rtx_code code)
17009 : {
17010 32 : switch (code)
17011 : {
17012 : case NE: return X86_CCNE;
17013 : case EQ: return X86_CCE;
17014 : case GE: return X86_CCNL;
17015 : case GT: return X86_CCNLE;
17016 : case LE: return X86_CCLE;
17017 : case LT: return X86_CCL;
17018 : case GEU: return X86_CCNB;
17019 : case GTU: return X86_CCNBE;
17020 : case LEU: return X86_CCBE;
17021 : case LTU: return X86_CCB;
17022 : default: return -1;
17023 : }
17024 : }
17025 :
17026 : /* Return TRUE or FALSE depending on whether the first SET in INSN
17027 : has source and destination with matching CC modes, and that the
17028 : CC mode is at least as constrained as REQ_MODE. */
17029 :
17030 : bool
17031 54468672 : ix86_match_ccmode (rtx insn, machine_mode req_mode)
17032 : {
17033 54468672 : rtx set;
17034 54468672 : machine_mode set_mode;
17035 :
17036 54468672 : set = PATTERN (insn);
17037 54468672 : if (GET_CODE (set) == PARALLEL)
17038 527432 : set = XVECEXP (set, 0, 0);
17039 54468672 : gcc_assert (GET_CODE (set) == SET);
17040 54468672 : gcc_assert (GET_CODE (SET_SRC (set)) == COMPARE);
17041 :
17042 54468672 : set_mode = GET_MODE (SET_DEST (set));
17043 54468672 : switch (set_mode)
17044 : {
17045 1425154 : case E_CCNOmode:
17046 1425154 : if (req_mode != CCNOmode
17047 102077 : && (req_mode != CCmode
17048 0 : || XEXP (SET_SRC (set), 1) != const0_rtx))
17049 : return false;
17050 : break;
17051 5859211 : case E_CCmode:
17052 5859211 : if (req_mode == CCGCmode)
17053 : return false;
17054 : /* FALLTHRU */
17055 9579167 : case E_CCGCmode:
17056 9579167 : if (req_mode == CCGOCmode || req_mode == CCNOmode)
17057 : return false;
17058 : /* FALLTHRU */
17059 10692182 : case E_CCGOCmode:
17060 10692182 : if (req_mode == CCZmode)
17061 : return false;
17062 : /* FALLTHRU */
17063 : case E_CCZmode:
17064 : break;
17065 :
17066 0 : case E_CCGZmode:
17067 :
17068 0 : case E_CCAmode:
17069 0 : case E_CCCmode:
17070 0 : case E_CCOmode:
17071 0 : case E_CCPmode:
17072 0 : case E_CCSmode:
17073 0 : if (set_mode != req_mode)
17074 : return false;
17075 : break;
17076 :
17077 0 : default:
17078 0 : gcc_unreachable ();
17079 : }
17080 :
17081 54357751 : return GET_MODE (SET_SRC (set)) == set_mode;
17082 : }
17083 :
17084 : machine_mode
17085 13799755 : ix86_cc_mode (enum rtx_code code, rtx op0, rtx op1)
17086 : {
17087 13799755 : machine_mode mode = GET_MODE (op0);
17088 :
17089 13799755 : if (SCALAR_FLOAT_MODE_P (mode))
17090 : {
17091 145574 : gcc_assert (!DECIMAL_FLOAT_MODE_P (mode));
17092 : return CCFPmode;
17093 : }
17094 :
17095 13654181 : switch (code)
17096 : {
17097 : /* Only zero flag is needed. */
17098 : case EQ: /* ZF=0 */
17099 : case NE: /* ZF!=0 */
17100 : return CCZmode;
17101 : /* Codes needing carry flag. */
17102 968770 : case GEU: /* CF=0 */
17103 968770 : case LTU: /* CF=1 */
17104 968770 : rtx geu;
17105 : /* Detect overflow checks. They need just the carry flag. */
17106 968770 : if (GET_CODE (op0) == PLUS
17107 968770 : && (rtx_equal_p (op1, XEXP (op0, 0))
17108 106402 : || rtx_equal_p (op1, XEXP (op0, 1))))
17109 17593 : return CCCmode;
17110 : /* Similarly for *setcc_qi_addqi3_cconly_overflow_1_* patterns.
17111 : Match LTU of op0
17112 : (neg:QI (geu:QI (reg:CC_CCC FLAGS_REG) (const_int 0)))
17113 : and op1
17114 : (ltu:QI (reg:CC_CCC FLAGS_REG) (const_int 0))
17115 : where CC_CCC is either CC or CCC. */
17116 951177 : else if (code == LTU
17117 345568 : && GET_CODE (op0) == NEG
17118 18 : && GET_CODE (geu = XEXP (op0, 0)) == GEU
17119 0 : && REG_P (XEXP (geu, 0))
17120 0 : && (GET_MODE (XEXP (geu, 0)) == CCCmode
17121 0 : || GET_MODE (XEXP (geu, 0)) == CCmode)
17122 0 : && REGNO (XEXP (geu, 0)) == FLAGS_REG
17123 0 : && XEXP (geu, 1) == const0_rtx
17124 0 : && GET_CODE (op1) == LTU
17125 0 : && REG_P (XEXP (op1, 0))
17126 0 : && GET_MODE (XEXP (op1, 0)) == GET_MODE (XEXP (geu, 0))
17127 0 : && REGNO (XEXP (op1, 0)) == FLAGS_REG
17128 951177 : && XEXP (op1, 1) == const0_rtx)
17129 : return CCCmode;
17130 : /* Similarly for *x86_cmc pattern.
17131 : Match LTU of op0 (neg:QI (ltu:QI (reg:CCC FLAGS_REG) (const_int 0)))
17132 : and op1 (geu:QI (reg:CCC FLAGS_REG) (const_int 0)).
17133 : It is sufficient to test that the operand modes are CCCmode. */
17134 951177 : else if (code == LTU
17135 345568 : && GET_CODE (op0) == NEG
17136 18 : && GET_CODE (XEXP (op0, 0)) == LTU
17137 0 : && GET_MODE (XEXP (XEXP (op0, 0), 0)) == CCCmode
17138 0 : && GET_CODE (op1) == GEU
17139 0 : && GET_MODE (XEXP (op1, 0)) == CCCmode)
17140 : return CCCmode;
17141 : /* Similarly for the comparison of addcarry/subborrow pattern. */
17142 345568 : else if (code == LTU
17143 345568 : && GET_CODE (op0) == ZERO_EXTEND
17144 15583 : && GET_CODE (op1) == PLUS
17145 11179 : && ix86_carry_flag_operator (XEXP (op1, 0), VOIDmode)
17146 11179 : && GET_CODE (XEXP (op1, 1)) == ZERO_EXTEND)
17147 : return CCCmode;
17148 : else
17149 939998 : return CCmode;
17150 : case GTU: /* CF=0 & ZF=0 */
17151 : case LEU: /* CF=1 | ZF=1 */
17152 : return CCmode;
17153 : /* Codes possibly doable only with sign flag when
17154 : comparing against zero. */
17155 827741 : case GE: /* SF=OF or SF=0 */
17156 827741 : case LT: /* SF<>OF or SF=1 */
17157 827741 : if (op1 == const0_rtx)
17158 : return CCGOCmode;
17159 : else
17160 : /* For other cases Carry flag is not required. */
17161 450363 : return CCGCmode;
17162 : /* Codes doable only with sign flag when comparing
17163 : against zero, but we miss jump instruction for it
17164 : so we need to use relational tests against overflow
17165 : that thus needs to be zero. */
17166 926318 : case GT: /* ZF=0 & SF=OF */
17167 926318 : case LE: /* ZF=1 | SF<>OF */
17168 926318 : if (op1 == const0_rtx)
17169 : return CCNOmode;
17170 : else
17171 620320 : return CCGCmode;
17172 : default:
17173 : /* CCmode should be used in all other cases. */
17174 : return CCmode;
17175 : }
17176 : }
17177 :
17178 : /* Return TRUE or FALSE depending on whether the ptest instruction
17179 : INSN has source and destination with suitable matching CC modes. */
17180 :
17181 : bool
17182 91273 : ix86_match_ptest_ccmode (rtx insn)
17183 : {
17184 91273 : rtx set, src;
17185 91273 : machine_mode set_mode;
17186 :
17187 91273 : set = PATTERN (insn);
17188 91273 : gcc_assert (GET_CODE (set) == SET);
17189 91273 : src = SET_SRC (set);
17190 91273 : gcc_assert (GET_CODE (src) == UNSPEC
17191 : && XINT (src, 1) == UNSPEC_PTEST);
17192 :
17193 91273 : set_mode = GET_MODE (src);
17194 91273 : if (set_mode != CCZmode
17195 : && set_mode != CCCmode
17196 : && set_mode != CCmode)
17197 : return false;
17198 91273 : return GET_MODE (SET_DEST (set)) == set_mode;
17199 : }
17200 :
17201 : /* Return the fixed registers used for condition codes. */
17202 :
17203 : static bool
17204 18854128 : ix86_fixed_condition_code_regs (unsigned int *p1, unsigned int *p2)
17205 : {
17206 18854128 : *p1 = FLAGS_REG;
17207 18854128 : *p2 = INVALID_REGNUM;
17208 18854128 : return true;
17209 : }
17210 :
17211 : /* If two condition code modes are compatible, return a condition code
17212 : mode which is compatible with both. Otherwise, return
17213 : VOIDmode. */
17214 :
17215 : static machine_mode
17216 31565 : ix86_cc_modes_compatible (machine_mode m1, machine_mode m2)
17217 : {
17218 31565 : if (m1 == m2)
17219 : return m1;
17220 :
17221 30418 : if (GET_MODE_CLASS (m1) != MODE_CC || GET_MODE_CLASS (m2) != MODE_CC)
17222 : return VOIDmode;
17223 :
17224 30418 : if ((m1 == CCGCmode && m2 == CCGOCmode)
17225 30418 : || (m1 == CCGOCmode && m2 == CCGCmode))
17226 : return CCGCmode;
17227 :
17228 30418 : if ((m1 == CCNOmode && m2 == CCGOCmode)
17229 30238 : || (m1 == CCGOCmode && m2 == CCNOmode))
17230 : return CCNOmode;
17231 :
17232 30126 : if (m1 == CCZmode
17233 16457 : && (m2 == CCGCmode || m2 == CCGOCmode || m2 == CCNOmode))
17234 : return m2;
17235 17581 : else if (m2 == CCZmode
17236 13413 : && (m1 == CCGCmode || m1 == CCGOCmode || m1 == CCNOmode))
17237 : return m1;
17238 :
17239 7407 : switch (m1)
17240 : {
17241 0 : default:
17242 0 : gcc_unreachable ();
17243 :
17244 7407 : case E_CCmode:
17245 7407 : case E_CCGCmode:
17246 7407 : case E_CCGOCmode:
17247 7407 : case E_CCNOmode:
17248 7407 : case E_CCAmode:
17249 7407 : case E_CCCmode:
17250 7407 : case E_CCOmode:
17251 7407 : case E_CCPmode:
17252 7407 : case E_CCSmode:
17253 7407 : case E_CCZmode:
17254 7407 : switch (m2)
17255 : {
17256 : default:
17257 : return VOIDmode;
17258 :
17259 : case E_CCmode:
17260 : case E_CCGCmode:
17261 : case E_CCGOCmode:
17262 : case E_CCNOmode:
17263 : case E_CCAmode:
17264 : case E_CCCmode:
17265 : case E_CCOmode:
17266 : case E_CCPmode:
17267 : case E_CCSmode:
17268 : case E_CCZmode:
17269 : return CCmode;
17270 : }
17271 :
17272 : case E_CCFPmode:
17273 : /* These are only compatible with themselves, which we already
17274 : checked above. */
17275 : return VOIDmode;
17276 : }
17277 : }
17278 :
17279 : /* Return strategy to use for floating-point. We assume that fcomi is always
17280 : preferable where available, since that is also true when looking at size
17281 : (2 bytes, vs. 3 for fnstsw+sahf and at least 5 for fnstsw+test). */
17282 :
17283 : enum ix86_fpcmp_strategy
17284 5595185 : ix86_fp_comparison_strategy (enum rtx_code)
17285 : {
17286 : /* Do fcomi/sahf based test when profitable. */
17287 :
17288 5595185 : if (TARGET_CMOVE)
17289 : return IX86_FPCMP_COMI;
17290 :
17291 0 : if (TARGET_SAHF && (TARGET_USE_SAHF || optimize_insn_for_size_p ()))
17292 0 : return IX86_FPCMP_SAHF;
17293 :
17294 : return IX86_FPCMP_ARITH;
17295 : }
17296 :
17297 : /* Convert comparison codes we use to represent FP comparison to integer
17298 : code that will result in proper branch. Return UNKNOWN if no such code
17299 : is available. */
17300 :
17301 : enum rtx_code
17302 592764 : ix86_fp_compare_code_to_integer (enum rtx_code code)
17303 : {
17304 592764 : switch (code)
17305 : {
17306 : case GT:
17307 : return GTU;
17308 18315 : case GE:
17309 18315 : return GEU;
17310 : case ORDERED:
17311 : case UNORDERED:
17312 : return code;
17313 119746 : case UNEQ:
17314 119746 : return EQ;
17315 21423 : case UNLT:
17316 21423 : return LTU;
17317 32268 : case UNLE:
17318 32268 : return LEU;
17319 114157 : case LTGT:
17320 114157 : return NE;
17321 695 : case EQ:
17322 695 : case NE:
17323 695 : if (TARGET_AVX10_2)
17324 : return code;
17325 : /* FALLTHRU. */
17326 265 : default:
17327 265 : return UNKNOWN;
17328 : }
17329 : }
17330 :
17331 : /* Zero extend possibly SImode EXP to Pmode register. */
17332 : rtx
17333 37873 : ix86_zero_extend_to_Pmode (rtx exp)
17334 : {
17335 49719 : return force_reg (Pmode, convert_to_mode (Pmode, exp, 1));
17336 : }
17337 :
17338 : /* Return true if the function is called via PLT. */
17339 :
17340 : bool
17341 985975 : ix86_call_use_plt_p (rtx call_op)
17342 : {
17343 985975 : if (SYMBOL_REF_LOCAL_P (call_op))
17344 : {
17345 196682 : if (SYMBOL_REF_DECL (call_op)
17346 196682 : && TREE_CODE (SYMBOL_REF_DECL (call_op)) == FUNCTION_DECL)
17347 : {
17348 : /* NB: All ifunc functions must be called via PLT. */
17349 113454 : cgraph_node *node
17350 113454 : = cgraph_node::get (SYMBOL_REF_DECL (call_op));
17351 113454 : if (node && node->ifunc_resolver)
17352 : return true;
17353 : }
17354 196662 : return false;
17355 : }
17356 : return true;
17357 : }
17358 :
17359 : /* Implement TARGET_IFUNC_REF_LOCAL_OK. If this hook returns true,
17360 : the PLT entry will be used as the function address for local IFUNC
17361 : functions. When the PIC register is needed for PLT call, indirect
17362 : call via the PLT entry will fail since the PIC register may not be
17363 : set up properly for indirect call. In this case, we should return
17364 : false. */
17365 :
17366 : static bool
17367 788129723 : ix86_ifunc_ref_local_ok (void)
17368 : {
17369 788129723 : return !flag_pic || (TARGET_64BIT && ix86_cmodel != CM_LARGE_PIC);
17370 : }
17371 :
17372 : /* Return true if the function being called was marked with attribute
17373 : "noplt" or using -fno-plt and we are compiling for non-PIC. We need
17374 : to handle the non-PIC case in the backend because there is no easy
17375 : interface for the front-end to force non-PLT calls to use the GOT.
17376 : This is currently used only with 64-bit or 32-bit GOT32X ELF targets
17377 : to call the function marked "noplt" indirectly. */
17378 :
17379 : bool
17380 5978296 : ix86_nopic_noplt_attribute_p (rtx call_op)
17381 : {
17382 5491019 : if (flag_pic || ix86_cmodel == CM_LARGE
17383 : || !(TARGET_64BIT || HAVE_AS_IX86_GOT32X)
17384 : || TARGET_MACHO || TARGET_SEH || TARGET_PECOFF
17385 11469315 : || SYMBOL_REF_LOCAL_P (call_op))
17386 : return false;
17387 :
17388 3830845 : tree symbol_decl = SYMBOL_REF_DECL (call_op);
17389 :
17390 3830845 : if (!flag_plt
17391 3830845 : || (symbol_decl != NULL_TREE
17392 3830813 : && lookup_attribute ("noplt", DECL_ATTRIBUTES (symbol_decl))))
17393 34 : return true;
17394 :
17395 : return false;
17396 : }
17397 :
17398 : /* Helper to output the jmp/call. */
17399 : static void
17400 33 : ix86_output_jmp_thunk_or_indirect (const char *thunk_name, const int regno)
17401 : {
17402 33 : if (thunk_name != NULL)
17403 : {
17404 22 : if ((REX_INT_REGNO_P (regno) || REX2_INT_REGNO_P (regno))
17405 1 : && ix86_indirect_branch_cs_prefix)
17406 1 : fprintf (asm_out_file, "\tcs\n");
17407 22 : fprintf (asm_out_file, "\tjmp\t");
17408 22 : assemble_name (asm_out_file, thunk_name);
17409 22 : putc ('\n', asm_out_file);
17410 22 : if ((ix86_harden_sls & harden_sls_indirect_jmp))
17411 2 : fputs ("\tint3\n", asm_out_file);
17412 : }
17413 : else
17414 11 : output_indirect_thunk (regno);
17415 33 : }
17416 :
17417 : /* Output indirect branch via a call and return thunk. CALL_OP is a
17418 : register which contains the branch target. XASM is the assembly
17419 : template for CALL_OP. Branch is a tail call if SIBCALL_P is true.
17420 : A normal call is converted to:
17421 :
17422 : call __x86_indirect_thunk_reg
17423 :
17424 : and a tail call is converted to:
17425 :
17426 : jmp __x86_indirect_thunk_reg
17427 : */
17428 :
17429 : static void
17430 50 : ix86_output_indirect_branch_via_reg (rtx call_op, bool sibcall_p)
17431 : {
17432 50 : char thunk_name_buf[32];
17433 50 : char *thunk_name;
17434 50 : enum indirect_thunk_prefix need_prefix
17435 50 : = indirect_thunk_need_prefix (current_output_insn);
17436 50 : int regno = REGNO (call_op);
17437 :
17438 50 : if (cfun->machine->indirect_branch_type
17439 50 : != indirect_branch_thunk_inline)
17440 : {
17441 39 : if (cfun->machine->indirect_branch_type == indirect_branch_thunk)
17442 16 : SET_HARD_REG_BIT (indirect_thunks_used, regno);
17443 :
17444 39 : indirect_thunk_name (thunk_name_buf, regno, need_prefix, false);
17445 39 : thunk_name = thunk_name_buf;
17446 : }
17447 : else
17448 : thunk_name = NULL;
17449 :
17450 50 : if (sibcall_p)
17451 27 : ix86_output_jmp_thunk_or_indirect (thunk_name, regno);
17452 : else
17453 : {
17454 23 : if (thunk_name != NULL)
17455 : {
17456 17 : if ((REX_INT_REGNO_P (regno) || REX_INT_REGNO_P (regno))
17457 1 : && ix86_indirect_branch_cs_prefix)
17458 1 : fprintf (asm_out_file, "\tcs\n");
17459 17 : fprintf (asm_out_file, "\tcall\t");
17460 17 : assemble_name (asm_out_file, thunk_name);
17461 17 : putc ('\n', asm_out_file);
17462 17 : return;
17463 : }
17464 :
17465 6 : char indirectlabel1[32];
17466 6 : char indirectlabel2[32];
17467 :
17468 6 : ASM_GENERATE_INTERNAL_LABEL (indirectlabel1,
17469 : INDIRECT_LABEL,
17470 : indirectlabelno++);
17471 6 : ASM_GENERATE_INTERNAL_LABEL (indirectlabel2,
17472 : INDIRECT_LABEL,
17473 : indirectlabelno++);
17474 :
17475 : /* Jump. */
17476 6 : fputs ("\tjmp\t", asm_out_file);
17477 6 : assemble_name_raw (asm_out_file, indirectlabel2);
17478 6 : fputc ('\n', asm_out_file);
17479 :
17480 6 : ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel1);
17481 :
17482 6 : ix86_output_jmp_thunk_or_indirect (thunk_name, regno);
17483 :
17484 6 : ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel2);
17485 :
17486 : /* Call. */
17487 6 : fputs ("\tcall\t", asm_out_file);
17488 6 : assemble_name_raw (asm_out_file, indirectlabel1);
17489 6 : fputc ('\n', asm_out_file);
17490 : }
17491 : }
17492 :
17493 : /* Output indirect branch via a call and return thunk. CALL_OP is
17494 : the branch target. XASM is the assembly template for CALL_OP.
17495 : Branch is a tail call if SIBCALL_P is true. A normal call is
17496 : converted to:
17497 :
17498 : jmp L2
17499 : L1:
17500 : push CALL_OP
17501 : jmp __x86_indirect_thunk
17502 : L2:
17503 : call L1
17504 :
17505 : and a tail call is converted to:
17506 :
17507 : push CALL_OP
17508 : jmp __x86_indirect_thunk
17509 : */
17510 :
17511 : static void
17512 0 : ix86_output_indirect_branch_via_push (rtx call_op, const char *xasm,
17513 : bool sibcall_p)
17514 : {
17515 0 : char thunk_name_buf[32];
17516 0 : char *thunk_name;
17517 0 : char push_buf[64];
17518 0 : enum indirect_thunk_prefix need_prefix
17519 0 : = indirect_thunk_need_prefix (current_output_insn);
17520 0 : int regno = -1;
17521 :
17522 0 : if (cfun->machine->indirect_branch_type
17523 0 : != indirect_branch_thunk_inline)
17524 : {
17525 0 : if (cfun->machine->indirect_branch_type == indirect_branch_thunk)
17526 0 : indirect_thunk_needed = true;
17527 0 : indirect_thunk_name (thunk_name_buf, regno, need_prefix, false);
17528 0 : thunk_name = thunk_name_buf;
17529 : }
17530 : else
17531 : thunk_name = NULL;
17532 :
17533 0 : snprintf (push_buf, sizeof (push_buf), "push{%c}\t%s",
17534 0 : TARGET_64BIT ? 'q' : 'l', xasm);
17535 :
17536 0 : if (sibcall_p)
17537 : {
17538 0 : output_asm_insn (push_buf, &call_op);
17539 0 : ix86_output_jmp_thunk_or_indirect (thunk_name, regno);
17540 : }
17541 : else
17542 : {
17543 0 : char indirectlabel1[32];
17544 0 : char indirectlabel2[32];
17545 :
17546 0 : ASM_GENERATE_INTERNAL_LABEL (indirectlabel1,
17547 : INDIRECT_LABEL,
17548 : indirectlabelno++);
17549 0 : ASM_GENERATE_INTERNAL_LABEL (indirectlabel2,
17550 : INDIRECT_LABEL,
17551 : indirectlabelno++);
17552 :
17553 : /* Jump. */
17554 0 : fputs ("\tjmp\t", asm_out_file);
17555 0 : assemble_name_raw (asm_out_file, indirectlabel2);
17556 0 : fputc ('\n', asm_out_file);
17557 :
17558 0 : ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel1);
17559 :
17560 : /* An external function may be called via GOT, instead of PLT. */
17561 0 : if (MEM_P (call_op))
17562 : {
17563 0 : struct ix86_address parts;
17564 0 : rtx addr = XEXP (call_op, 0);
17565 0 : if (ix86_decompose_address (addr, &parts)
17566 0 : && parts.base == stack_pointer_rtx)
17567 : {
17568 : /* Since call will adjust stack by -UNITS_PER_WORD,
17569 : we must convert "disp(stack, index, scale)" to
17570 : "disp+UNITS_PER_WORD(stack, index, scale)". */
17571 0 : if (parts.index)
17572 : {
17573 0 : addr = gen_rtx_MULT (Pmode, parts.index,
17574 : GEN_INT (parts.scale));
17575 0 : addr = gen_rtx_PLUS (Pmode, stack_pointer_rtx,
17576 : addr);
17577 : }
17578 : else
17579 : addr = stack_pointer_rtx;
17580 :
17581 0 : rtx disp;
17582 0 : if (parts.disp != NULL_RTX)
17583 0 : disp = plus_constant (Pmode, parts.disp,
17584 0 : UNITS_PER_WORD);
17585 : else
17586 0 : disp = GEN_INT (UNITS_PER_WORD);
17587 :
17588 0 : addr = gen_rtx_PLUS (Pmode, addr, disp);
17589 0 : call_op = gen_rtx_MEM (GET_MODE (call_op), addr);
17590 : }
17591 : }
17592 :
17593 0 : output_asm_insn (push_buf, &call_op);
17594 :
17595 0 : ix86_output_jmp_thunk_or_indirect (thunk_name, regno);
17596 :
17597 0 : ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel2);
17598 :
17599 : /* Call. */
17600 0 : fputs ("\tcall\t", asm_out_file);
17601 0 : assemble_name_raw (asm_out_file, indirectlabel1);
17602 0 : fputc ('\n', asm_out_file);
17603 : }
17604 0 : }
17605 :
17606 : /* Output indirect branch via a call and return thunk. CALL_OP is
17607 : the branch target. XASM is the assembly template for CALL_OP.
17608 : Branch is a tail call if SIBCALL_P is true. */
17609 :
17610 : static void
17611 50 : ix86_output_indirect_branch (rtx call_op, const char *xasm,
17612 : bool sibcall_p)
17613 : {
17614 50 : if (REG_P (call_op))
17615 50 : ix86_output_indirect_branch_via_reg (call_op, sibcall_p);
17616 : else
17617 0 : ix86_output_indirect_branch_via_push (call_op, xasm, sibcall_p);
17618 50 : }
17619 :
17620 : /* Output indirect jump. CALL_OP is the jump target. */
17621 :
17622 : const char *
17623 7836 : ix86_output_indirect_jmp (rtx call_op)
17624 : {
17625 7836 : if (cfun->machine->indirect_branch_type != indirect_branch_keep)
17626 : {
17627 : /* We can't have red-zone since "call" in the indirect thunk
17628 : pushes the return address onto stack, destroying red-zone. */
17629 4 : if (ix86_red_zone_used)
17630 0 : gcc_unreachable ();
17631 :
17632 4 : ix86_output_indirect_branch (call_op, "%0", true);
17633 : }
17634 : else
17635 7832 : output_asm_insn ("%!jmp\t%A0", &call_op);
17636 7836 : return (ix86_harden_sls & harden_sls_indirect_jmp) ? "int3" : "";
17637 : }
17638 :
17639 : /* Output return instrumentation for current function if needed. */
17640 :
17641 : static void
17642 1723420 : output_return_instrumentation (void)
17643 : {
17644 1723420 : if (ix86_instrument_return != instrument_return_none
17645 6 : && flag_fentry
17646 1723426 : && !DECL_NO_INSTRUMENT_FUNCTION_ENTRY_EXIT (cfun->decl))
17647 : {
17648 5 : if (ix86_flag_record_return)
17649 5 : fprintf (asm_out_file, "1:\n");
17650 5 : switch (ix86_instrument_return)
17651 : {
17652 2 : case instrument_return_call:
17653 2 : fprintf (asm_out_file, "\tcall\t__return__\n");
17654 2 : break;
17655 3 : case instrument_return_nop5:
17656 : /* 5 byte nop: nopl 0(%[re]ax,%[re]ax,1) */
17657 3 : fprintf (asm_out_file, ASM_BYTE "0x0f, 0x1f, 0x44, 0x00, 0x00\n");
17658 3 : break;
17659 : case instrument_return_none:
17660 : break;
17661 : }
17662 :
17663 5 : if (ix86_flag_record_return)
17664 : {
17665 5 : fprintf (asm_out_file, "\t.section __return_loc, \"a\",@progbits\n");
17666 5 : fprintf (asm_out_file, "\t.%s 1b\n", TARGET_64BIT ? "quad" : "long");
17667 5 : fprintf (asm_out_file, "\t.previous\n");
17668 : }
17669 : }
17670 1723420 : }
17671 :
17672 : /* Output function return. CALL_OP is the jump target. Add a REP
17673 : prefix to RET if LONG_P is true and function return is kept. */
17674 :
17675 : const char *
17676 1594702 : ix86_output_function_return (bool long_p)
17677 : {
17678 1594702 : output_return_instrumentation ();
17679 :
17680 1594702 : if (cfun->machine->function_return_type != indirect_branch_keep)
17681 : {
17682 17 : char thunk_name[32];
17683 17 : enum indirect_thunk_prefix need_prefix
17684 17 : = indirect_thunk_need_prefix (current_output_insn);
17685 :
17686 17 : if (cfun->machine->function_return_type
17687 17 : != indirect_branch_thunk_inline)
17688 : {
17689 12 : bool need_thunk = (cfun->machine->function_return_type
17690 : == indirect_branch_thunk);
17691 12 : indirect_thunk_name (thunk_name, INVALID_REGNUM, need_prefix,
17692 : true);
17693 12 : indirect_return_needed |= need_thunk;
17694 12 : fprintf (asm_out_file, "\tjmp\t");
17695 12 : assemble_name (asm_out_file, thunk_name);
17696 12 : putc ('\n', asm_out_file);
17697 : }
17698 : else
17699 5 : output_indirect_thunk (INVALID_REGNUM);
17700 :
17701 17 : return "";
17702 : }
17703 :
17704 3188882 : output_asm_insn (long_p ? "rep%; ret" : "ret", nullptr);
17705 1594685 : return (ix86_harden_sls & harden_sls_return) ? "int3" : "";
17706 : }
17707 :
17708 : /* Output indirect function return. RET_OP is the function return
17709 : target. */
17710 :
17711 : const char *
17712 17 : ix86_output_indirect_function_return (rtx ret_op)
17713 : {
17714 17 : if (cfun->machine->function_return_type != indirect_branch_keep)
17715 : {
17716 0 : char thunk_name[32];
17717 0 : enum indirect_thunk_prefix need_prefix
17718 0 : = indirect_thunk_need_prefix (current_output_insn);
17719 0 : unsigned int regno = REGNO (ret_op);
17720 0 : gcc_assert (regno == CX_REG);
17721 :
17722 0 : if (cfun->machine->function_return_type
17723 0 : != indirect_branch_thunk_inline)
17724 : {
17725 0 : bool need_thunk = (cfun->machine->function_return_type
17726 : == indirect_branch_thunk);
17727 0 : indirect_thunk_name (thunk_name, regno, need_prefix, true);
17728 :
17729 0 : if (need_thunk)
17730 : {
17731 0 : indirect_return_via_cx = true;
17732 0 : SET_HARD_REG_BIT (indirect_thunks_used, CX_REG);
17733 : }
17734 0 : fprintf (asm_out_file, "\tjmp\t");
17735 0 : assemble_name (asm_out_file, thunk_name);
17736 0 : putc ('\n', asm_out_file);
17737 : }
17738 : else
17739 0 : output_indirect_thunk (regno);
17740 : }
17741 : else
17742 : {
17743 17 : output_asm_insn ("%!jmp\t%A0", &ret_op);
17744 17 : if (ix86_harden_sls & harden_sls_indirect_jmp)
17745 1 : fputs ("\tint3\n", asm_out_file);
17746 : }
17747 17 : return "";
17748 : }
17749 :
17750 : /* Output the assembly for a call instruction. */
17751 :
17752 : const char *
17753 6160405 : ix86_output_call_insn (rtx_insn *insn, rtx call_op)
17754 : {
17755 6160405 : bool direct_p = constant_call_address_operand (call_op, VOIDmode);
17756 6160405 : bool output_indirect_p
17757 : = (!TARGET_SEH
17758 6160405 : && cfun->machine->indirect_branch_type != indirect_branch_keep);
17759 6160405 : bool seh_nop_p = false;
17760 6160405 : const char *xasm;
17761 :
17762 6160405 : if (SIBLING_CALL_P (insn))
17763 : {
17764 128718 : output_return_instrumentation ();
17765 128718 : if (direct_p)
17766 : {
17767 119045 : if (ix86_nopic_noplt_attribute_p (call_op))
17768 : {
17769 4 : direct_p = false;
17770 4 : if (TARGET_64BIT)
17771 : {
17772 4 : if (output_indirect_p)
17773 : xasm = "{%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}";
17774 : else
17775 4 : xasm = "%!jmp\t{*%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}";
17776 : }
17777 : else
17778 : {
17779 0 : if (output_indirect_p)
17780 : xasm = "{%p0@GOT|[DWORD PTR %p0@GOT]}";
17781 : else
17782 0 : xasm = "%!jmp\t{*%p0@GOT|[DWORD PTR %p0@GOT]}";
17783 : }
17784 : }
17785 : else
17786 : xasm = "%!jmp\t%P0";
17787 : }
17788 : /* SEH epilogue detection requires the indirect branch case
17789 : to include REX.W. */
17790 9673 : else if (TARGET_SEH)
17791 : xasm = "%!rex.W jmp\t%A0";
17792 : else
17793 : {
17794 9673 : if (output_indirect_p)
17795 : xasm = "%0";
17796 : else
17797 9650 : xasm = "%!jmp\t%A0";
17798 : }
17799 :
17800 128718 : if (output_indirect_p && !direct_p)
17801 23 : ix86_output_indirect_branch (call_op, xasm, true);
17802 : else
17803 : {
17804 128695 : output_asm_insn (xasm, &call_op);
17805 128695 : if (!direct_p
17806 9654 : && (ix86_harden_sls & harden_sls_indirect_jmp))
17807 : return "int3";
17808 : }
17809 128717 : return "";
17810 : }
17811 :
17812 : /* SEH unwinding can require an extra nop to be emitted in several
17813 : circumstances. Determine if we have one of those. */
17814 6031687 : if (TARGET_SEH)
17815 : {
17816 : rtx_insn *i;
17817 :
17818 : for (i = NEXT_INSN (insn); i ; i = NEXT_INSN (i))
17819 : {
17820 : /* Prevent a catch region from being adjacent to a jump that would
17821 : be interpreted as an epilogue sequence by the unwinder. */
17822 : if (JUMP_P(i) && CROSSING_JUMP_P (i))
17823 : {
17824 : seh_nop_p = true;
17825 : break;
17826 : }
17827 :
17828 : /* If we get to another real insn, we don't need the nop. */
17829 : if (INSN_P (i))
17830 : break;
17831 :
17832 : /* If we get to the epilogue note, prevent a catch region from
17833 : being adjacent to the standard epilogue sequence. Note that,
17834 : if non-call exceptions are enabled, we already did it during
17835 : epilogue expansion, or else, if the insn can throw internally,
17836 : we already did it during the reorg pass. */
17837 : if (NOTE_P (i) && NOTE_KIND (i) == NOTE_INSN_EPILOGUE_BEG
17838 : && !flag_non_call_exceptions
17839 : && !can_throw_internal (insn))
17840 : {
17841 : seh_nop_p = true;
17842 : break;
17843 : }
17844 : }
17845 :
17846 : /* If we didn't find a real insn following the call, prevent the
17847 : unwinder from looking into the next function. */
17848 : if (i == NULL)
17849 : seh_nop_p = true;
17850 : }
17851 :
17852 6031687 : if (direct_p)
17853 : {
17854 5858229 : if (ix86_nopic_noplt_attribute_p (call_op))
17855 : {
17856 6 : direct_p = false;
17857 6 : if (TARGET_64BIT)
17858 : {
17859 6 : if (output_indirect_p)
17860 : xasm = "{%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}";
17861 : else
17862 6 : xasm = "%!call\t{*%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}";
17863 : }
17864 : else
17865 : {
17866 0 : if (output_indirect_p)
17867 : xasm = "{%p0@GOT|[DWORD PTR %p0@GOT]}";
17868 : else
17869 0 : xasm = "%!call\t{*%p0@GOT|[DWORD PTR %p0@GOT]}";
17870 : }
17871 : }
17872 : else
17873 : xasm = "%!call\t%P0";
17874 : }
17875 : else
17876 : {
17877 173458 : if (output_indirect_p)
17878 : xasm = "%0";
17879 : else
17880 173435 : xasm = "%!call\t%A0";
17881 : }
17882 :
17883 6031687 : if (output_indirect_p && !direct_p)
17884 23 : ix86_output_indirect_branch (call_op, xasm, false);
17885 : else
17886 6031664 : output_asm_insn (xasm, &call_op);
17887 :
17888 : if (seh_nop_p)
17889 : return "nop";
17890 :
17891 : return "";
17892 : }
17893 :
17894 : /* Return a MEM corresponding to a stack slot with mode MODE.
17895 : Allocate a new slot if necessary.
17896 :
17897 : The RTL for a function can have several slots available: N is
17898 : which slot to use. */
17899 :
17900 : rtx
17901 22440 : assign_386_stack_local (machine_mode mode, enum ix86_stack_slot n)
17902 : {
17903 22440 : struct stack_local_entry *s;
17904 :
17905 22440 : gcc_assert (n < MAX_386_STACK_LOCALS);
17906 :
17907 33838 : for (s = ix86_stack_locals; s; s = s->next)
17908 31235 : if (s->mode == mode && s->n == n)
17909 19837 : return validize_mem (copy_rtx (s->rtl));
17910 :
17911 2603 : int align = 0;
17912 : /* For DImode with SLOT_FLOATxFDI_387 use 32-bit
17913 : alignment with -m32 -mpreferred-stack-boundary=2. */
17914 2603 : if (mode == DImode
17915 329 : && !TARGET_64BIT
17916 329 : && n == SLOT_FLOATxFDI_387
17917 2932 : && ix86_preferred_stack_boundary < GET_MODE_ALIGNMENT (DImode))
17918 : align = 32;
17919 2603 : s = ggc_alloc<stack_local_entry> ();
17920 2603 : s->n = n;
17921 2603 : s->mode = mode;
17922 5206 : s->rtl = assign_stack_local (mode, GET_MODE_SIZE (mode), align);
17923 :
17924 2603 : s->next = ix86_stack_locals;
17925 2603 : ix86_stack_locals = s;
17926 2603 : return validize_mem (copy_rtx (s->rtl));
17927 : }
17928 :
17929 : static void
17930 1493935 : ix86_instantiate_decls (void)
17931 : {
17932 1493935 : struct stack_local_entry *s;
17933 :
17934 1493935 : for (s = ix86_stack_locals; s; s = s->next)
17935 0 : if (s->rtl != NULL_RTX)
17936 0 : instantiate_decl_rtl (s->rtl);
17937 1493935 : }
17938 :
17939 : /* Check whether x86 address PARTS is a pc-relative address. */
17940 :
17941 : bool
17942 26869346 : ix86_rip_relative_addr_p (struct ix86_address *parts)
17943 : {
17944 26869346 : rtx base, index, disp;
17945 :
17946 26869346 : base = parts->base;
17947 26869346 : index = parts->index;
17948 26869346 : disp = parts->disp;
17949 :
17950 26869346 : if (disp && !base && !index)
17951 : {
17952 25082763 : if (TARGET_64BIT)
17953 : {
17954 23417842 : rtx symbol = disp;
17955 :
17956 23417842 : if (GET_CODE (disp) == CONST)
17957 7063009 : symbol = XEXP (disp, 0);
17958 23417842 : if (GET_CODE (symbol) == PLUS
17959 6547925 : && CONST_INT_P (XEXP (symbol, 1)))
17960 6547925 : symbol = XEXP (symbol, 0);
17961 :
17962 23417842 : if (LABEL_REF_P (symbol)
17963 23410604 : || (SYMBOL_REF_P (symbol)
17964 22144168 : && SYMBOL_REF_TLS_MODEL (symbol) == 0)
17965 24684278 : || (GET_CODE (symbol) == UNSPEC
17966 533943 : && (XINT (symbol, 1) == UNSPEC_GOTPCREL
17967 : || XINT (symbol, 1) == UNSPEC_PCREL
17968 : || XINT (symbol, 1) == UNSPEC_GOTNTPOFF)))
17969 22657836 : return true;
17970 : }
17971 : }
17972 : return false;
17973 : }
17974 :
17975 : /* Calculate the length of the memory address in the instruction encoding.
17976 : Includes addr32 prefix, does not include the one-byte modrm, opcode,
17977 : or other prefixes. We never generate addr32 prefix for LEA insn. */
17978 :
17979 : int
17980 275088234 : memory_address_length (rtx addr, bool lea)
17981 : {
17982 275088234 : struct ix86_address parts;
17983 275088234 : rtx base, index, disp;
17984 275088234 : int len;
17985 275088234 : int ok;
17986 :
17987 275088234 : if (GET_CODE (addr) == PRE_DEC
17988 266492826 : || GET_CODE (addr) == POST_INC
17989 261953029 : || GET_CODE (addr) == PRE_MODIFY
17990 261953029 : || GET_CODE (addr) == POST_MODIFY)
17991 : return 0;
17992 :
17993 261953029 : ok = ix86_decompose_address (addr, &parts);
17994 261953029 : gcc_assert (ok);
17995 :
17996 261953029 : len = (parts.seg == ADDR_SPACE_GENERIC) ? 0 : 1;
17997 :
17998 : /* If this is not LEA instruction, add the length of addr32 prefix. */
17999 223397926 : if (TARGET_64BIT && !lea
18000 459284371 : && (SImode_address_operand (addr, VOIDmode)
18001 197331105 : || (parts.base && GET_MODE (parts.base) == SImode)
18002 197320511 : || (parts.index && GET_MODE (parts.index) == SImode)))
18003 10831 : len++;
18004 :
18005 261953029 : base = parts.base;
18006 261953029 : index = parts.index;
18007 261953029 : disp = parts.disp;
18008 :
18009 261953029 : if (base && SUBREG_P (base))
18010 2 : base = SUBREG_REG (base);
18011 261953029 : if (index && SUBREG_P (index))
18012 0 : index = SUBREG_REG (index);
18013 :
18014 261953029 : gcc_assert (base == NULL_RTX || REG_P (base));
18015 261953029 : gcc_assert (index == NULL_RTX || REG_P (index));
18016 :
18017 : /* Rule of thumb:
18018 : - esp as the base always wants an index,
18019 : - ebp as the base always wants a displacement,
18020 : - r12 as the base always wants an index,
18021 : - r13 as the base always wants a displacement. */
18022 :
18023 : /* Register Indirect. */
18024 261953029 : if (base && !index && !disp)
18025 : {
18026 : /* esp (for its index) and ebp (for its displacement) need
18027 : the two-byte modrm form. Similarly for r12 and r13 in 64-bit
18028 : code. */
18029 17081861 : if (base == arg_pointer_rtx
18030 17081861 : || base == frame_pointer_rtx
18031 17081861 : || REGNO (base) == SP_REG
18032 10186972 : || REGNO (base) == BP_REG
18033 10186972 : || REGNO (base) == R12_REG
18034 26794598 : || REGNO (base) == R13_REG)
18035 7369124 : len++;
18036 : }
18037 :
18038 : /* Direct Addressing. In 64-bit mode mod 00 r/m 5
18039 : is not disp32, but disp32(%rip), so for disp32
18040 : SIB byte is needed, unless print_operand_address
18041 : optimizes it into disp32(%rip) or (%rip) is implied
18042 : by UNSPEC. */
18043 244871168 : else if (disp && !base && !index)
18044 : {
18045 24104027 : len += 4;
18046 24104027 : if (!ix86_rip_relative_addr_p (&parts))
18047 1852241 : len++;
18048 : }
18049 : else
18050 : {
18051 : /* Find the length of the displacement constant. */
18052 220767141 : if (disp)
18053 : {
18054 216790774 : if (base && satisfies_constraint_K (disp))
18055 125572970 : len += 1;
18056 : else
18057 91217804 : len += 4;
18058 : }
18059 : /* ebp always wants a displacement. Similarly r13. */
18060 3976367 : else if (base && (REGNO (base) == BP_REG || REGNO (base) == R13_REG))
18061 10643 : len++;
18062 :
18063 : /* An index requires the two-byte modrm form.... */
18064 220767141 : if (index
18065 : /* ...like esp (or r12), which always wants an index. */
18066 210504917 : || base == arg_pointer_rtx
18067 210504917 : || base == frame_pointer_rtx
18068 431272058 : || (base && (REGNO (base) == SP_REG || REGNO (base) == R12_REG)))
18069 158616521 : len++;
18070 : }
18071 :
18072 : return len;
18073 : }
18074 :
18075 : /* Compute default value for "length_immediate" attribute. When SHORTFORM
18076 : is set, expect that insn have 8bit immediate alternative. */
18077 : int
18078 322328635 : ix86_attr_length_immediate_default (rtx_insn *insn, bool shortform)
18079 : {
18080 322328635 : int len = 0;
18081 322328635 : int i;
18082 322328635 : extract_insn_cached (insn);
18083 1005789436 : for (i = recog_data.n_operands - 1; i >= 0; --i)
18084 683460801 : if (CONSTANT_P (recog_data.operand[i]))
18085 : {
18086 142020586 : enum attr_mode mode = get_attr_mode (insn);
18087 :
18088 142020586 : gcc_assert (!len);
18089 142020586 : if (shortform && CONST_INT_P (recog_data.operand[i]))
18090 : {
18091 38241076 : HOST_WIDE_INT ival = INTVAL (recog_data.operand[i]);
18092 38241076 : switch (mode)
18093 : {
18094 1431590 : case MODE_QI:
18095 1431590 : len = 1;
18096 1431590 : continue;
18097 435459 : case MODE_HI:
18098 435459 : ival = trunc_int_for_mode (ival, HImode);
18099 435459 : break;
18100 16053418 : case MODE_SI:
18101 16053418 : ival = trunc_int_for_mode (ival, SImode);
18102 16053418 : break;
18103 : default:
18104 : break;
18105 : }
18106 36809486 : if (IN_RANGE (ival, -128, 127))
18107 : {
18108 32669499 : len = 1;
18109 32669499 : continue;
18110 : }
18111 : }
18112 107919497 : switch (mode)
18113 : {
18114 : case MODE_QI:
18115 : len = 1;
18116 : break;
18117 : case MODE_HI:
18118 683460801 : len = 2;
18119 : break;
18120 : case MODE_SI:
18121 102547381 : len = 4;
18122 : break;
18123 : /* Immediates for DImode instructions are encoded
18124 : as 32bit sign extended values. */
18125 : case MODE_DI:
18126 102547381 : len = 4;
18127 : break;
18128 0 : default:
18129 0 : fatal_insn ("unknown insn mode", insn);
18130 : }
18131 : }
18132 322328635 : return len;
18133 : }
18134 :
18135 : /* Compute default value for "length_address" attribute. */
18136 : int
18137 452380851 : ix86_attr_length_address_default (rtx_insn *insn)
18138 : {
18139 452380851 : int i;
18140 :
18141 452380851 : if (get_attr_type (insn) == TYPE_LEA)
18142 : {
18143 28815552 : rtx set = PATTERN (insn), addr;
18144 :
18145 28815552 : if (GET_CODE (set) == PARALLEL)
18146 86839 : set = XVECEXP (set, 0, 0);
18147 :
18148 28815552 : gcc_assert (GET_CODE (set) == SET);
18149 :
18150 28815552 : addr = SET_SRC (set);
18151 :
18152 28815552 : return memory_address_length (addr, true);
18153 : }
18154 :
18155 423565299 : extract_insn_cached (insn);
18156 974985997 : for (i = recog_data.n_operands - 1; i >= 0; --i)
18157 : {
18158 797407653 : rtx op = recog_data.operand[i];
18159 797407653 : if (MEM_P (op))
18160 : {
18161 246267471 : constrain_operands_cached (insn, reload_completed);
18162 246267471 : if (which_alternative != -1)
18163 : {
18164 246267471 : const char *constraints = recog_data.constraints[i];
18165 246267471 : int alt = which_alternative;
18166 :
18167 390656443 : while (*constraints == '=' || *constraints == '+')
18168 144388972 : constraints++;
18169 1119615919 : while (alt-- > 0)
18170 2142574383 : while (*constraints++ != ',')
18171 : ;
18172 : /* Skip ignored operands. */
18173 246267471 : if (*constraints == 'X')
18174 280516 : continue;
18175 : }
18176 :
18177 245986955 : int len = memory_address_length (XEXP (op, 0), false);
18178 :
18179 : /* Account for segment prefix for non-default addr spaces. */
18180 258779622 : if (!ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (op)))
18181 784294 : len++;
18182 :
18183 245986955 : return len;
18184 : }
18185 : }
18186 : return 0;
18187 : }
18188 :
18189 : /* Compute default value for "length_vex" attribute. It includes
18190 : 2 or 3 byte VEX prefix and 1 opcode byte. */
18191 :
18192 : int
18193 5064385 : ix86_attr_length_vex_default (rtx_insn *insn, bool has_0f_opcode,
18194 : bool has_vex_w)
18195 : {
18196 5064385 : int i, reg_only = 2 + 1;
18197 5064385 : bool has_mem = false;
18198 :
18199 : /* Only 0f opcode can use 2 byte VEX prefix and VEX W bit uses 3
18200 : byte VEX prefix. */
18201 5064385 : if (!has_0f_opcode || has_vex_w)
18202 : return 3 + 1;
18203 :
18204 : /* We can always use 2 byte VEX prefix in 32bit. */
18205 4632407 : if (!TARGET_64BIT)
18206 : return 2 + 1;
18207 :
18208 3562139 : extract_insn_cached (insn);
18209 :
18210 11194997 : for (i = recog_data.n_operands - 1; i >= 0; --i)
18211 7924399 : if (REG_P (recog_data.operand[i]))
18212 : {
18213 : /* REX.W bit uses 3 byte VEX prefix.
18214 : REX2 with vex use extended EVEX prefix length is 4-byte. */
18215 5225777 : if (GET_MODE (recog_data.operand[i]) == DImode
18216 5225777 : && GENERAL_REG_P (recog_data.operand[i]))
18217 : return 3 + 1;
18218 :
18219 : /* REX.B bit requires 3-byte VEX. Right here we don't know which
18220 : operand will be encoded using VEX.B, so be conservative.
18221 : REX2 with vex use extended EVEX prefix length is 4-byte. */
18222 5213394 : if (REX_INT_REGNO_P (recog_data.operand[i])
18223 5213394 : || REX2_INT_REGNO_P (recog_data.operand[i])
18224 5213394 : || REX_SSE_REGNO_P (recog_data.operand[i]))
18225 0 : reg_only = 3 + 1;
18226 : }
18227 2698622 : else if (MEM_P (recog_data.operand[i]))
18228 : {
18229 : /* REX2.X or REX2.B bits use 3 byte VEX prefix. */
18230 2061195 : if (x86_extended_rex2reg_mentioned_p (recog_data.operand[i]))
18231 : return 4;
18232 :
18233 : /* REX.X or REX.B bits use 3 byte VEX prefix. */
18234 2060954 : if (x86_extended_reg_mentioned_p (recog_data.operand[i]))
18235 : return 3 + 1;
18236 :
18237 : has_mem = true;
18238 : }
18239 :
18240 3270598 : return has_mem ? 2 + 1 : reg_only;
18241 : }
18242 :
18243 :
18244 : static bool
18245 : ix86_class_likely_spilled_p (reg_class_t);
18246 :
18247 : /* Returns true if lhs of insn is HW function argument register and set up
18248 : is_spilled to true if it is likely spilled HW register. */
18249 : static bool
18250 1149 : insn_is_function_arg (rtx insn, bool* is_spilled)
18251 : {
18252 1149 : rtx dst;
18253 :
18254 1149 : if (!NONDEBUG_INSN_P (insn))
18255 : return false;
18256 : /* Call instructions are not movable, ignore it. */
18257 1149 : if (CALL_P (insn))
18258 : return false;
18259 1075 : insn = PATTERN (insn);
18260 1075 : if (GET_CODE (insn) == PARALLEL)
18261 73 : insn = XVECEXP (insn, 0, 0);
18262 1075 : if (GET_CODE (insn) != SET)
18263 : return false;
18264 1075 : dst = SET_DEST (insn);
18265 979 : if (REG_P (dst) && HARD_REGISTER_P (dst)
18266 1948 : && ix86_function_arg_regno_p (REGNO (dst)))
18267 : {
18268 : /* Is it likely spilled HW register? */
18269 873 : if (!TEST_HARD_REG_BIT (fixed_reg_set, REGNO (dst))
18270 873 : && ix86_class_likely_spilled_p (REGNO_REG_CLASS (REGNO (dst))))
18271 829 : *is_spilled = true;
18272 873 : return true;
18273 : }
18274 : return false;
18275 : }
18276 :
18277 : /* Add output dependencies for chain of function adjacent arguments if only
18278 : there is a move to likely spilled HW register. Return first argument
18279 : if at least one dependence was added or NULL otherwise. */
18280 : static rtx_insn *
18281 415 : add_parameter_dependencies (rtx_insn *call, rtx_insn *head)
18282 : {
18283 415 : rtx_insn *insn;
18284 415 : rtx_insn *last = call;
18285 415 : rtx_insn *first_arg = NULL;
18286 415 : bool is_spilled = false;
18287 :
18288 415 : head = PREV_INSN (head);
18289 :
18290 : /* Find nearest to call argument passing instruction. */
18291 415 : while (true)
18292 : {
18293 415 : last = PREV_INSN (last);
18294 415 : if (last == head)
18295 : return NULL;
18296 415 : if (!NONDEBUG_INSN_P (last))
18297 0 : continue;
18298 415 : if (insn_is_function_arg (last, &is_spilled))
18299 : break;
18300 : return NULL;
18301 : }
18302 :
18303 : first_arg = last;
18304 1054 : while (true)
18305 : {
18306 1054 : insn = PREV_INSN (last);
18307 1054 : if (!INSN_P (insn))
18308 : break;
18309 956 : if (insn == head)
18310 : break;
18311 915 : if (!NONDEBUG_INSN_P (insn))
18312 : {
18313 181 : last = insn;
18314 181 : continue;
18315 : }
18316 734 : if (insn_is_function_arg (insn, &is_spilled))
18317 : {
18318 : /* Add output dependence between two function arguments if chain
18319 : of output arguments contains likely spilled HW registers. */
18320 466 : if (is_spilled)
18321 466 : add_dependence (first_arg, insn, REG_DEP_OUTPUT);
18322 : first_arg = last = insn;
18323 : }
18324 : else
18325 : break;
18326 : }
18327 407 : if (!is_spilled)
18328 : return NULL;
18329 : return first_arg;
18330 : }
18331 :
18332 : /* Add output or anti dependency from insn to first_arg to restrict its code
18333 : motion. */
18334 : static void
18335 2335 : avoid_func_arg_motion (rtx_insn *first_arg, rtx_insn *insn)
18336 : {
18337 2335 : rtx set;
18338 2335 : rtx tmp;
18339 :
18340 2335 : set = single_set (insn);
18341 2335 : if (!set)
18342 : return;
18343 1451 : tmp = SET_DEST (set);
18344 1451 : if (REG_P (tmp))
18345 : {
18346 : /* Add output dependency to the first function argument. */
18347 1256 : add_dependence (first_arg, insn, REG_DEP_OUTPUT);
18348 1256 : return;
18349 : }
18350 : /* Add anti dependency. */
18351 195 : add_dependence (first_arg, insn, REG_DEP_ANTI);
18352 : }
18353 :
18354 : /* Avoid cross block motion of function argument through adding dependency
18355 : from the first non-jump instruction in bb. */
18356 : static void
18357 68 : add_dependee_for_func_arg (rtx_insn *arg, basic_block bb)
18358 : {
18359 68 : rtx_insn *insn = BB_END (bb);
18360 :
18361 134 : while (insn)
18362 : {
18363 134 : if (NONDEBUG_INSN_P (insn) && NONJUMP_INSN_P (insn))
18364 : {
18365 67 : rtx set = single_set (insn);
18366 67 : if (set)
18367 : {
18368 67 : avoid_func_arg_motion (arg, insn);
18369 67 : return;
18370 : }
18371 : }
18372 67 : if (insn == BB_HEAD (bb))
18373 : return;
18374 66 : insn = PREV_INSN (insn);
18375 : }
18376 : }
18377 :
18378 : /* Hook for pre-reload schedule - avoid motion of function arguments
18379 : passed in likely spilled HW registers. */
18380 : static void
18381 10350569 : ix86_dependencies_evaluation_hook (rtx_insn *head, rtx_insn *tail)
18382 : {
18383 10350569 : rtx_insn *insn;
18384 10350569 : rtx_insn *first_arg = NULL;
18385 10350569 : if (reload_completed)
18386 : return;
18387 2262 : while (head != tail && DEBUG_INSN_P (head))
18388 766 : head = NEXT_INSN (head);
18389 10999 : for (insn = tail; insn != head; insn = PREV_INSN (insn))
18390 9639 : if (INSN_P (insn) && CALL_P (insn))
18391 : {
18392 415 : first_arg = add_parameter_dependencies (insn, head);
18393 415 : if (first_arg)
18394 : {
18395 : /* Add dependee for first argument to predecessors if only
18396 : region contains more than one block. */
18397 407 : basic_block bb = BLOCK_FOR_INSN (insn);
18398 407 : int rgn = CONTAINING_RGN (bb->index);
18399 407 : int nr_blks = RGN_NR_BLOCKS (rgn);
18400 : /* Skip trivial regions and region head blocks that can have
18401 : predecessors outside of region. */
18402 407 : if (nr_blks > 1 && BLOCK_TO_BB (bb->index) != 0)
18403 : {
18404 67 : edge e;
18405 67 : edge_iterator ei;
18406 :
18407 : /* Regions are SCCs with the exception of selective
18408 : scheduling with pipelining of outer blocks enabled.
18409 : So also check that immediate predecessors of a non-head
18410 : block are in the same region. */
18411 137 : FOR_EACH_EDGE (e, ei, bb->preds)
18412 : {
18413 : /* Avoid creating of loop-carried dependencies through
18414 : using topological ordering in the region. */
18415 70 : if (rgn == CONTAINING_RGN (e->src->index)
18416 69 : && BLOCK_TO_BB (bb->index) > BLOCK_TO_BB (e->src->index))
18417 68 : add_dependee_for_func_arg (first_arg, e->src);
18418 : }
18419 : }
18420 407 : insn = first_arg;
18421 407 : if (insn == head)
18422 : break;
18423 : }
18424 : }
18425 9224 : else if (first_arg)
18426 2268 : avoid_func_arg_motion (first_arg, insn);
18427 : }
18428 :
18429 : /* Hook for pre-reload schedule - set priority of moves from likely spilled
18430 : HW registers to maximum, to schedule them at soon as possible. These are
18431 : moves from function argument registers at the top of the function entry
18432 : and moves from function return value registers after call. */
18433 : static int
18434 110871692 : ix86_adjust_priority (rtx_insn *insn, int priority)
18435 : {
18436 110871692 : rtx set;
18437 :
18438 110871692 : if (reload_completed)
18439 : return priority;
18440 :
18441 14711 : if (!NONDEBUG_INSN_P (insn))
18442 : return priority;
18443 :
18444 12653 : set = single_set (insn);
18445 12653 : if (set)
18446 : {
18447 12047 : rtx tmp = SET_SRC (set);
18448 12047 : if (REG_P (tmp)
18449 2537 : && HARD_REGISTER_P (tmp)
18450 501 : && !TEST_HARD_REG_BIT (fixed_reg_set, REGNO (tmp))
18451 12047 : && ix86_class_likely_spilled_p (REGNO_REG_CLASS (REGNO (tmp))))
18452 450 : return current_sched_info->sched_max_insns_priority;
18453 : }
18454 :
18455 : return priority;
18456 : }
18457 :
18458 : /* Prepare for scheduling pass. */
18459 : static void
18460 970494 : ix86_sched_init_global (FILE *, int, int)
18461 : {
18462 : /* Install scheduling hooks for current CPU. Some of these hooks are used
18463 : in time-critical parts of the scheduler, so we only set them up when
18464 : they are actually used. */
18465 970494 : switch (ix86_tune)
18466 : {
18467 923886 : case PROCESSOR_CORE2:
18468 923886 : case PROCESSOR_NEHALEM:
18469 923886 : case PROCESSOR_SANDYBRIDGE:
18470 923886 : case PROCESSOR_HASWELL:
18471 923886 : case PROCESSOR_TREMONT:
18472 923886 : case PROCESSOR_ALDERLAKE:
18473 923886 : case PROCESSOR_GENERIC:
18474 : /* Do not perform multipass scheduling for pre-reload schedule
18475 : to save compile time. */
18476 923886 : if (reload_completed)
18477 : {
18478 923399 : ix86_core2i7_init_hooks ();
18479 923399 : break;
18480 : }
18481 : /* Fall through. */
18482 47095 : default:
18483 47095 : targetm.sched.dfa_post_advance_cycle = NULL;
18484 47095 : targetm.sched.first_cycle_multipass_init = NULL;
18485 47095 : targetm.sched.first_cycle_multipass_begin = NULL;
18486 47095 : targetm.sched.first_cycle_multipass_issue = NULL;
18487 47095 : targetm.sched.first_cycle_multipass_backtrack = NULL;
18488 47095 : targetm.sched.first_cycle_multipass_end = NULL;
18489 47095 : targetm.sched.first_cycle_multipass_fini = NULL;
18490 47095 : break;
18491 : }
18492 970494 : }
18493 :
18494 :
18495 : /* Implement TARGET_STATIC_RTX_ALIGNMENT. */
18496 :
18497 : static HOST_WIDE_INT
18498 730904 : ix86_static_rtx_alignment (machine_mode mode)
18499 : {
18500 730904 : if (mode == DFmode)
18501 : return 64;
18502 : if (ALIGN_MODE_128 (mode))
18503 158242 : return MAX (128, GET_MODE_ALIGNMENT (mode));
18504 486758 : return GET_MODE_ALIGNMENT (mode);
18505 : }
18506 :
18507 : /* Implement TARGET_CONSTANT_ALIGNMENT. */
18508 :
18509 : static HOST_WIDE_INT
18510 6811209 : ix86_constant_alignment (const_tree exp, HOST_WIDE_INT align)
18511 : {
18512 6811209 : if (TREE_CODE (exp) == REAL_CST || TREE_CODE (exp) == VECTOR_CST
18513 : || TREE_CODE (exp) == INTEGER_CST)
18514 : {
18515 371861 : machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
18516 371861 : HOST_WIDE_INT mode_align = ix86_static_rtx_alignment (mode);
18517 371861 : return MAX (mode_align, align);
18518 : }
18519 6296803 : else if (!optimize_size && TREE_CODE (exp) == STRING_CST
18520 9456130 : && TREE_STRING_LENGTH (exp) >= 31 && align < BITS_PER_WORD)
18521 : return BITS_PER_WORD;
18522 :
18523 : return align;
18524 : }
18525 :
18526 : /* Implement TARGET_EMPTY_RECORD_P. */
18527 :
18528 : static bool
18529 1497983039 : ix86_is_empty_record (const_tree type)
18530 : {
18531 1497983039 : if (!TARGET_64BIT)
18532 : return false;
18533 1466729504 : return default_is_empty_record (type);
18534 : }
18535 :
18536 : /* Implement TARGET_WARN_PARAMETER_PASSING_ABI. */
18537 :
18538 : static void
18539 15435898 : ix86_warn_parameter_passing_abi (cumulative_args_t cum_v, tree type)
18540 : {
18541 15435898 : CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
18542 :
18543 15435898 : if (!cum->warn_empty)
18544 : return;
18545 :
18546 13174863 : if (!TYPE_EMPTY_P (type))
18547 : return;
18548 :
18549 : /* Don't warn if the function isn't visible outside of the TU. */
18550 15224 : if (cum->decl && !TREE_PUBLIC (cum->decl))
18551 : return;
18552 :
18553 13740 : tree decl = cum->decl;
18554 13740 : if (!decl)
18555 : /* If we don't know the target, look at the current TU. */
18556 39 : decl = current_function_decl;
18557 :
18558 13740 : const_tree ctx = get_ultimate_context (decl);
18559 13740 : if (ctx == NULL_TREE
18560 27446 : || !TRANSLATION_UNIT_WARN_EMPTY_P (ctx))
18561 : return;
18562 :
18563 : /* If the actual size of the type is zero, then there is no change
18564 : in how objects of this size are passed. */
18565 72 : if (int_size_in_bytes (type) == 0)
18566 : return;
18567 :
18568 66 : warning (OPT_Wabi, "empty class %qT parameter passing ABI "
18569 : "changes in %<-fabi-version=12%> (GCC 8)", type);
18570 :
18571 : /* Only warn once. */
18572 66 : cum->warn_empty = false;
18573 : }
18574 :
18575 : /* This hook returns name of multilib ABI. */
18576 :
18577 : static const char *
18578 3451359 : ix86_get_multilib_abi_name (void)
18579 : {
18580 3451359 : if (!(TARGET_64BIT_P (ix86_isa_flags)))
18581 : return "i386";
18582 3407187 : else if (TARGET_X32_P (ix86_isa_flags))
18583 : return "x32";
18584 : else
18585 3407187 : return "x86_64";
18586 : }
18587 :
18588 : /* Compute the alignment for a variable for Intel MCU psABI. TYPE is
18589 : the data type, and ALIGN is the alignment that the object would
18590 : ordinarily have. */
18591 :
18592 : static int
18593 0 : iamcu_alignment (tree type, int align)
18594 : {
18595 0 : machine_mode mode;
18596 :
18597 0 : if (align < 32 || TYPE_USER_ALIGN (type))
18598 : return align;
18599 :
18600 : /* Intel MCU psABI specifies scalar types > 4 bytes aligned to 4
18601 : bytes. */
18602 0 : type = strip_array_types (type);
18603 0 : if (TYPE_ATOMIC (type))
18604 : return align;
18605 :
18606 0 : mode = TYPE_MODE (type);
18607 0 : switch (GET_MODE_CLASS (mode))
18608 : {
18609 : case MODE_INT:
18610 : case MODE_COMPLEX_INT:
18611 : case MODE_COMPLEX_FLOAT:
18612 : case MODE_FLOAT:
18613 : case MODE_DECIMAL_FLOAT:
18614 : return 32;
18615 : default:
18616 : return align;
18617 : }
18618 : }
18619 :
18620 : /* Compute the alignment for a static variable.
18621 : TYPE is the data type, and ALIGN is the alignment that
18622 : the object would ordinarily have. The value of this function is used
18623 : instead of that alignment to align the object. */
18624 :
18625 : int
18626 12100865 : ix86_data_alignment (tree type, unsigned int align, bool opt)
18627 : {
18628 : /* GCC 4.8 and earlier used to incorrectly assume this alignment even
18629 : for symbols from other compilation units or symbols that don't need
18630 : to bind locally. In order to preserve some ABI compatibility with
18631 : those compilers, ensure we don't decrease alignment from what we
18632 : used to assume. */
18633 :
18634 12100865 : unsigned int max_align_compat = MIN (256, MAX_OFILE_ALIGNMENT);
18635 :
18636 : /* A data structure, equal or greater than the size of a cache line
18637 : (64 bytes in the Pentium 4 and other recent Intel processors, including
18638 : processors based on Intel Core microarchitecture) should be aligned
18639 : so that its base address is a multiple of a cache line size. */
18640 :
18641 24201730 : unsigned int max_align
18642 12100865 : = MIN ((unsigned) ix86_tune_cost->prefetch_block * 8, MAX_OFILE_ALIGNMENT);
18643 :
18644 14690849 : if (max_align < BITS_PER_WORD)
18645 0 : max_align = BITS_PER_WORD;
18646 :
18647 12100865 : switch (ix86_align_data_type)
18648 : {
18649 12100865 : case ix86_align_data_type_abi: opt = false; break;
18650 12100845 : case ix86_align_data_type_compat: max_align = BITS_PER_WORD; break;
18651 : case ix86_align_data_type_cacheline: break;
18652 : }
18653 :
18654 12100865 : if (TARGET_IAMCU)
18655 0 : align = iamcu_alignment (type, align);
18656 :
18657 12100865 : if (opt
18658 5816988 : && AGGREGATE_TYPE_P (type)
18659 3731042 : && TYPE_SIZE (type)
18660 15831855 : && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST)
18661 : {
18662 6760439 : if (wi::geu_p (wi::to_wide (TYPE_SIZE (type)), max_align_compat)
18663 3730990 : && align < max_align_compat)
18664 701541 : align = max_align_compat;
18665 7398616 : if (wi::geu_p (wi::to_wide (TYPE_SIZE (type)), max_align)
18666 3730990 : && align < max_align)
18667 63364 : align = max_align;
18668 : }
18669 :
18670 : /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
18671 : to 16byte boundary. */
18672 12100865 : if (TARGET_64BIT)
18673 : {
18674 4988126 : if ((opt ? AGGREGATE_TYPE_P (type) : TREE_CODE (type) == ARRAY_TYPE)
18675 3284669 : && TYPE_SIZE (type)
18676 3284607 : && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST
18677 10953866 : && wi::geu_p (wi::to_wide (TYPE_SIZE (type)), 128)
18678 11575900 : && align < 128)
18679 622034 : return 128;
18680 : }
18681 :
18682 11478831 : if (!opt)
18683 6087151 : return align;
18684 :
18685 5391680 : if (TREE_CODE (type) == ARRAY_TYPE)
18686 : {
18687 1107554 : if (TYPE_MODE (TREE_TYPE (type)) == DFmode && align < 64)
18688 : return 64;
18689 1107554 : if (ALIGN_MODE_128 (TYPE_MODE (TREE_TYPE (type))) && align < 128)
18690 : return 128;
18691 : }
18692 4284126 : else if (TREE_CODE (type) == COMPLEX_TYPE)
18693 : {
18694 :
18695 13003 : if (TYPE_MODE (type) == DCmode && align < 64)
18696 : return 64;
18697 13003 : if ((TYPE_MODE (type) == XCmode
18698 13003 : || TYPE_MODE (type) == TCmode) && align < 128)
18699 : return 128;
18700 : }
18701 4271123 : else if (RECORD_OR_UNION_TYPE_P (type)
18702 4271123 : && TYPE_FIELDS (type))
18703 : {
18704 2196421 : if (DECL_MODE (TYPE_FIELDS (type)) == DFmode && align < 64)
18705 : return 64;
18706 2196421 : if (ALIGN_MODE_128 (DECL_MODE (TYPE_FIELDS (type))) && align < 128)
18707 : return 128;
18708 : }
18709 2074702 : else if (SCALAR_FLOAT_TYPE_P (type) || VECTOR_TYPE_P (type)
18710 : || TREE_CODE (type) == INTEGER_TYPE)
18711 : {
18712 1923401 : if (TYPE_MODE (type) == DFmode && align < 64)
18713 : return 64;
18714 1923401 : if (ALIGN_MODE_128 (TYPE_MODE (type)) && align < 128)
18715 : return 128;
18716 : }
18717 :
18718 5391565 : return align;
18719 : }
18720 :
18721 : /* Implement TARGET_LOWER_LOCAL_DECL_ALIGNMENT. */
18722 : static void
18723 32648666 : ix86_lower_local_decl_alignment (tree decl)
18724 : {
18725 32648666 : unsigned int new_align = ix86_local_alignment (decl, VOIDmode,
18726 32648666 : DECL_ALIGN (decl), true);
18727 32648666 : if (new_align < DECL_ALIGN (decl))
18728 0 : SET_DECL_ALIGN (decl, new_align);
18729 32648666 : }
18730 :
18731 : /* Compute the alignment for a local variable or a stack slot. EXP is
18732 : the data type or decl itself, MODE is the widest mode available and
18733 : ALIGN is the alignment that the object would ordinarily have. The
18734 : value of this macro is used instead of that alignment to align the
18735 : object. */
18736 :
18737 : unsigned int
18738 50449332 : ix86_local_alignment (tree exp, machine_mode mode,
18739 : unsigned int align, bool may_lower)
18740 : {
18741 50449332 : tree type, decl;
18742 :
18743 50449332 : if (exp && DECL_P (exp))
18744 : {
18745 48264171 : type = TREE_TYPE (exp);
18746 48264171 : decl = exp;
18747 : }
18748 : else
18749 : {
18750 : type = exp;
18751 : decl = NULL;
18752 : }
18753 :
18754 : /* Don't do dynamic stack realignment for long long objects with
18755 : -mpreferred-stack-boundary=2. */
18756 50449332 : if (may_lower
18757 32648666 : && !TARGET_64BIT
18758 249703 : && align == 64
18759 38968 : && ix86_preferred_stack_boundary < 64
18760 0 : && (mode == DImode || (type && TYPE_MODE (type) == DImode))
18761 0 : && (!type || (!TYPE_USER_ALIGN (type)
18762 0 : && !TYPE_ATOMIC (strip_array_types (type))))
18763 50449332 : && (!decl || !DECL_USER_ALIGN (decl)))
18764 : align = 32;
18765 :
18766 : /* If TYPE is NULL, we are allocating a stack slot for caller-save
18767 : register in MODE. We will return the largest alignment of XF
18768 : and DF. */
18769 50449332 : if (!type)
18770 : {
18771 1430565 : if (mode == XFmode && align < GET_MODE_ALIGNMENT (DFmode))
18772 1513 : align = GET_MODE_ALIGNMENT (DFmode);
18773 1430565 : return align;
18774 : }
18775 :
18776 : /* Don't increase alignment for Intel MCU psABI. */
18777 49018767 : if (TARGET_IAMCU)
18778 : return align;
18779 :
18780 : /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
18781 : to 16byte boundary. Exact wording is:
18782 :
18783 : An array uses the same alignment as its elements, except that a local or
18784 : global array variable of length at least 16 bytes or
18785 : a C99 variable-length array variable always has alignment of at least 16 bytes.
18786 :
18787 : This was added to allow use of aligned SSE instructions at arrays. This
18788 : rule is meant for static storage (where compiler cannot do the analysis
18789 : by itself). We follow it for automatic variables only when convenient.
18790 : We fully control everything in the function compiled and functions from
18791 : other unit cannot rely on the alignment.
18792 :
18793 : Exclude va_list type. It is the common case of local array where
18794 : we cannot benefit from the alignment.
18795 :
18796 : TODO: Probably one should optimize for size only when var is not escaping. */
18797 46182898 : if (TARGET_64BIT && optimize_function_for_speed_p (cfun)
18798 94843898 : && TARGET_SSE)
18799 : {
18800 45785407 : if (AGGREGATE_TYPE_P (type)
18801 9797276 : && (va_list_type_node == NULL_TREE
18802 9797276 : || (TYPE_MAIN_VARIANT (type)
18803 9797276 : != TYPE_MAIN_VARIANT (va_list_type_node)))
18804 9697827 : && TYPE_SIZE (type)
18805 9697827 : && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST
18806 46898566 : && wi::geu_p (wi::to_wide (TYPE_SIZE (type)), 128)
18807 52917634 : && align < 128)
18808 6019068 : return 128;
18809 : }
18810 42999699 : if (TREE_CODE (type) == ARRAY_TYPE)
18811 : {
18812 807144 : if (TYPE_MODE (TREE_TYPE (type)) == DFmode && align < 64)
18813 : return 64;
18814 807144 : if (ALIGN_MODE_128 (TYPE_MODE (TREE_TYPE (type))) && align < 128)
18815 : return 128;
18816 : }
18817 42192555 : else if (TREE_CODE (type) == COMPLEX_TYPE)
18818 : {
18819 159882 : if (TYPE_MODE (type) == DCmode && align < 64)
18820 : return 64;
18821 159882 : if ((TYPE_MODE (type) == XCmode
18822 159882 : || TYPE_MODE (type) == TCmode) && align < 128)
18823 : return 128;
18824 : }
18825 42032673 : else if (RECORD_OR_UNION_TYPE_P (type)
18826 42032673 : && TYPE_FIELDS (type))
18827 : {
18828 4947636 : if (DECL_MODE (TYPE_FIELDS (type)) == DFmode && align < 64)
18829 : return 64;
18830 4944531 : if (ALIGN_MODE_128 (DECL_MODE (TYPE_FIELDS (type))) && align < 128)
18831 : return 128;
18832 : }
18833 37085037 : else if (SCALAR_FLOAT_TYPE_P (type) || VECTOR_TYPE_P (type)
18834 : || TREE_CODE (type) == INTEGER_TYPE)
18835 : {
18836 :
18837 30520899 : if (TYPE_MODE (type) == DFmode && align < 64)
18838 : return 64;
18839 30520899 : if (ALIGN_MODE_128 (TYPE_MODE (type)) && align < 128)
18840 : return 128;
18841 : }
18842 : return align;
18843 : }
18844 :
18845 : /* Compute the minimum required alignment for dynamic stack realignment
18846 : purposes for a local variable, parameter or a stack slot. EXP is
18847 : the data type or decl itself, MODE is its mode and ALIGN is the
18848 : alignment that the object would ordinarily have. */
18849 :
18850 : unsigned int
18851 48687269 : ix86_minimum_alignment (tree exp, machine_mode mode,
18852 : unsigned int align)
18853 : {
18854 48687269 : tree type, decl;
18855 :
18856 48687269 : if (exp && DECL_P (exp))
18857 : {
18858 15485770 : type = TREE_TYPE (exp);
18859 15485770 : decl = exp;
18860 : }
18861 : else
18862 : {
18863 : type = exp;
18864 : decl = NULL;
18865 : }
18866 :
18867 48687269 : if (TARGET_64BIT || align != 64 || ix86_preferred_stack_boundary >= 64)
18868 : return align;
18869 :
18870 : /* Don't do dynamic stack realignment for long long objects with
18871 : -mpreferred-stack-boundary=2. */
18872 0 : if ((mode == DImode || (type && TYPE_MODE (type) == DImode))
18873 0 : && (!type || (!TYPE_USER_ALIGN (type)
18874 0 : && !TYPE_ATOMIC (strip_array_types (type))))
18875 0 : && (!decl || !DECL_USER_ALIGN (decl)))
18876 : {
18877 0 : gcc_checking_assert (!TARGET_STV);
18878 : return 32;
18879 : }
18880 :
18881 : return align;
18882 : }
18883 :
18884 : /* Find a location for the static chain incoming to a nested function.
18885 : This is a register, unless all free registers are used by arguments. */
18886 :
18887 : static rtx
18888 270260 : ix86_static_chain (const_tree fndecl_or_type, bool incoming_p)
18889 : {
18890 270260 : unsigned regno;
18891 :
18892 270260 : if (TARGET_64BIT)
18893 : {
18894 : /* We always use R10 in 64-bit mode. */
18895 : regno = R10_REG;
18896 : }
18897 : else
18898 : {
18899 88353 : const_tree fntype, fndecl;
18900 88353 : unsigned int ccvt;
18901 :
18902 : /* By default in 32-bit mode we use ECX to pass the static chain. */
18903 88353 : regno = CX_REG;
18904 :
18905 88353 : if (TREE_CODE (fndecl_or_type) == FUNCTION_DECL)
18906 : {
18907 78379 : fntype = TREE_TYPE (fndecl_or_type);
18908 78379 : fndecl = fndecl_or_type;
18909 : }
18910 : else
18911 : {
18912 : fntype = fndecl_or_type;
18913 : fndecl = NULL;
18914 : }
18915 :
18916 88353 : ccvt = ix86_get_callcvt (fntype);
18917 88353 : if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
18918 : {
18919 : /* Fastcall functions use ecx/edx for arguments, which leaves
18920 : us with EAX for the static chain.
18921 : Thiscall functions use ecx for arguments, which also
18922 : leaves us with EAX for the static chain. */
18923 : regno = AX_REG;
18924 : }
18925 88353 : else if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
18926 : {
18927 : /* Thiscall functions use ecx for arguments, which leaves
18928 : us with EAX and EDX for the static chain.
18929 : We are using for abi-compatibility EAX. */
18930 : regno = AX_REG;
18931 : }
18932 88353 : else if (ix86_function_regparm (fntype, fndecl) == 3)
18933 : {
18934 : /* For regparm 3, we have no free call-clobbered registers in
18935 : which to store the static chain. In order to implement this,
18936 : we have the trampoline push the static chain to the stack.
18937 : However, we can't push a value below the return address when
18938 : we call the nested function directly, so we have to use an
18939 : alternate entry point. For this we use ESI, and have the
18940 : alternate entry point push ESI, so that things appear the
18941 : same once we're executing the nested function. */
18942 0 : if (incoming_p)
18943 : {
18944 0 : if (fndecl == current_function_decl
18945 0 : && !ix86_static_chain_on_stack)
18946 : {
18947 0 : gcc_assert (!reload_completed);
18948 0 : ix86_static_chain_on_stack = true;
18949 : }
18950 0 : return gen_frame_mem (SImode,
18951 0 : plus_constant (Pmode,
18952 : arg_pointer_rtx, -8));
18953 : }
18954 : regno = SI_REG;
18955 : }
18956 : }
18957 :
18958 358626 : return gen_rtx_REG (Pmode, regno);
18959 : }
18960 :
18961 : /* Emit RTL insns to initialize the variable parts of a trampoline.
18962 : FNDECL is the decl of the target address; M_TRAMP is a MEM for
18963 : the trampoline, and CHAIN_VALUE is an RTX for the static chain
18964 : to be passed to the target function. */
18965 :
18966 : static void
18967 306 : ix86_trampoline_init (rtx m_tramp, tree fndecl, rtx chain_value)
18968 : {
18969 306 : rtx mem, fnaddr;
18970 306 : int opcode;
18971 306 : int offset = 0;
18972 306 : bool need_endbr = (flag_cf_protection & CF_BRANCH);
18973 :
18974 306 : fnaddr = XEXP (DECL_RTL (fndecl), 0);
18975 :
18976 306 : if (TARGET_64BIT)
18977 : {
18978 306 : int size;
18979 :
18980 306 : if (need_endbr)
18981 : {
18982 : /* Insert ENDBR64. */
18983 1 : mem = adjust_address (m_tramp, SImode, offset);
18984 1 : emit_move_insn (mem, gen_int_mode (0xfa1e0ff3, SImode));
18985 1 : offset += 4;
18986 : }
18987 :
18988 : /* Load the function address to r11. Try to load address using
18989 : the shorter movl instead of movabs. We may want to support
18990 : movq for kernel mode, but kernel does not use trampolines at
18991 : the moment. FNADDR is a 32bit address and may not be in
18992 : DImode when ptr_mode == SImode. Always use movl in this
18993 : case. */
18994 306 : if (ptr_mode == SImode
18995 306 : || x86_64_zext_immediate_operand (fnaddr, VOIDmode))
18996 : {
18997 274 : fnaddr = copy_addr_to_reg (fnaddr);
18998 :
18999 274 : mem = adjust_address (m_tramp, HImode, offset);
19000 274 : emit_move_insn (mem, gen_int_mode (0xbb41, HImode));
19001 :
19002 274 : mem = adjust_address (m_tramp, SImode, offset + 2);
19003 274 : emit_move_insn (mem, gen_lowpart (SImode, fnaddr));
19004 274 : offset += 6;
19005 : }
19006 : else
19007 : {
19008 32 : mem = adjust_address (m_tramp, HImode, offset);
19009 32 : emit_move_insn (mem, gen_int_mode (0xbb49, HImode));
19010 :
19011 32 : mem = adjust_address (m_tramp, DImode, offset + 2);
19012 32 : emit_move_insn (mem, fnaddr);
19013 32 : offset += 10;
19014 : }
19015 :
19016 : /* Load static chain using movabs to r10. Use the shorter movl
19017 : instead of movabs when ptr_mode == SImode. */
19018 306 : if (ptr_mode == SImode)
19019 : {
19020 : opcode = 0xba41;
19021 : size = 6;
19022 : }
19023 : else
19024 : {
19025 306 : opcode = 0xba49;
19026 306 : size = 10;
19027 : }
19028 :
19029 306 : mem = adjust_address (m_tramp, HImode, offset);
19030 306 : emit_move_insn (mem, gen_int_mode (opcode, HImode));
19031 :
19032 306 : mem = adjust_address (m_tramp, ptr_mode, offset + 2);
19033 306 : emit_move_insn (mem, chain_value);
19034 306 : offset += size;
19035 :
19036 : /* Jump to r11; the last (unused) byte is a nop, only there to
19037 : pad the write out to a single 32-bit store. */
19038 306 : mem = adjust_address (m_tramp, SImode, offset);
19039 306 : emit_move_insn (mem, gen_int_mode (0x90e3ff49, SImode));
19040 306 : offset += 4;
19041 : }
19042 : else
19043 : {
19044 0 : rtx disp, chain;
19045 :
19046 : /* Depending on the static chain location, either load a register
19047 : with a constant, or push the constant to the stack. All of the
19048 : instructions are the same size. */
19049 0 : chain = ix86_static_chain (fndecl, true);
19050 0 : if (REG_P (chain))
19051 : {
19052 0 : switch (REGNO (chain))
19053 : {
19054 : case AX_REG:
19055 : opcode = 0xb8; break;
19056 0 : case CX_REG:
19057 0 : opcode = 0xb9; break;
19058 0 : default:
19059 0 : gcc_unreachable ();
19060 : }
19061 : }
19062 : else
19063 : opcode = 0x68;
19064 :
19065 0 : if (need_endbr)
19066 : {
19067 : /* Insert ENDBR32. */
19068 0 : mem = adjust_address (m_tramp, SImode, offset);
19069 0 : emit_move_insn (mem, gen_int_mode (0xfb1e0ff3, SImode));
19070 0 : offset += 4;
19071 : }
19072 :
19073 0 : mem = adjust_address (m_tramp, QImode, offset);
19074 0 : emit_move_insn (mem, gen_int_mode (opcode, QImode));
19075 :
19076 0 : mem = adjust_address (m_tramp, SImode, offset + 1);
19077 0 : emit_move_insn (mem, chain_value);
19078 0 : offset += 5;
19079 :
19080 0 : mem = adjust_address (m_tramp, QImode, offset);
19081 0 : emit_move_insn (mem, gen_int_mode (0xe9, QImode));
19082 :
19083 0 : mem = adjust_address (m_tramp, SImode, offset + 1);
19084 :
19085 : /* Compute offset from the end of the jmp to the target function.
19086 : In the case in which the trampoline stores the static chain on
19087 : the stack, we need to skip the first insn which pushes the
19088 : (call-saved) register static chain; this push is 1 byte. */
19089 0 : offset += 5;
19090 0 : int skip = MEM_P (chain) ? 1 : 0;
19091 : /* Skip ENDBR32 at the entry of the target function. */
19092 0 : if (need_endbr
19093 0 : && !cgraph_node::get (fndecl)->only_called_directly_p ())
19094 0 : skip += 4;
19095 0 : disp = expand_binop (SImode, sub_optab, fnaddr,
19096 0 : plus_constant (Pmode, XEXP (m_tramp, 0),
19097 0 : offset - skip),
19098 : NULL_RTX, 1, OPTAB_DIRECT);
19099 0 : emit_move_insn (mem, disp);
19100 : }
19101 :
19102 306 : gcc_assert (offset <= TRAMPOLINE_SIZE);
19103 :
19104 : #ifdef HAVE_ENABLE_EXECUTE_STACK
19105 : #ifdef CHECK_EXECUTE_STACK_ENABLED
19106 : if (CHECK_EXECUTE_STACK_ENABLED)
19107 : #endif
19108 : emit_library_call (gen_rtx_SYMBOL_REF (Pmode, "__enable_execute_stack"),
19109 : LCT_NORMAL, VOIDmode, XEXP (m_tramp, 0), Pmode);
19110 : #endif
19111 306 : }
19112 :
19113 : static bool
19114 61617551 : ix86_allocate_stack_slots_for_args (void)
19115 : {
19116 : /* Naked functions should not allocate stack slots for arguments. */
19117 61617551 : return !ix86_function_naked (current_function_decl);
19118 : }
19119 :
19120 : static bool
19121 39602034 : ix86_warn_func_return (tree decl)
19122 : {
19123 : /* Naked functions are implemented entirely in assembly, including the
19124 : return sequence, so suppress warnings about this. */
19125 39602034 : return !ix86_function_naked (decl);
19126 : }
19127 :
19128 : /* Return the shift count of a vector by scalar shift builtin second argument
19129 : ARG1. */
19130 : static tree
19131 14142 : ix86_vector_shift_count (tree arg1)
19132 : {
19133 14142 : if (tree_fits_uhwi_p (arg1))
19134 : return arg1;
19135 8316 : else if (TREE_CODE (arg1) == VECTOR_CST && CHAR_BIT == 8)
19136 : {
19137 : /* The count argument is weird, passed in as various 128-bit
19138 : (or 64-bit) vectors, the low 64 bits from it are the count. */
19139 162 : unsigned char buf[16];
19140 162 : int len = native_encode_expr (arg1, buf, 16);
19141 162 : if (len == 0)
19142 162 : return NULL_TREE;
19143 162 : tree t = native_interpret_expr (uint64_type_node, buf, len);
19144 162 : if (t && tree_fits_uhwi_p (t))
19145 : return t;
19146 : }
19147 : return NULL_TREE;
19148 : }
19149 :
19150 : /* Return true if arg_mask is all ones, ELEMS is elements number of
19151 : corresponding vector. */
19152 : static bool
19153 25042 : ix86_masked_all_ones (unsigned HOST_WIDE_INT elems, tree arg_mask)
19154 : {
19155 25042 : if (TREE_CODE (arg_mask) != INTEGER_CST)
19156 : return false;
19157 :
19158 7462 : unsigned HOST_WIDE_INT mask = TREE_INT_CST_LOW (arg_mask);
19159 7462 : if (elems == HOST_BITS_PER_WIDE_INT)
19160 33 : return mask == HOST_WIDE_INT_M1U;
19161 7429 : if ((mask | (HOST_WIDE_INT_M1U << elems)) != HOST_WIDE_INT_M1U)
19162 2681 : return false;
19163 :
19164 : return true;
19165 : }
19166 :
19167 : static tree
19168 68331744 : ix86_fold_builtin (tree fndecl, int n_args,
19169 : tree *args, bool ignore ATTRIBUTE_UNUSED)
19170 : {
19171 68331744 : if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD)
19172 : {
19173 68331744 : enum ix86_builtins fn_code
19174 68331744 : = (enum ix86_builtins) DECL_MD_FUNCTION_CODE (fndecl);
19175 68331744 : enum rtx_code rcode;
19176 68331744 : bool is_vshift;
19177 68331744 : enum tree_code tcode;
19178 68331744 : bool is_scalar;
19179 68331744 : unsigned HOST_WIDE_INT mask;
19180 :
19181 68331744 : switch (fn_code)
19182 : {
19183 8966 : case IX86_BUILTIN_CPU_IS:
19184 8966 : case IX86_BUILTIN_CPU_SUPPORTS:
19185 8966 : gcc_assert (n_args == 1);
19186 8966 : return fold_builtin_cpu (fndecl, args);
19187 :
19188 26005 : case IX86_BUILTIN_NANQ:
19189 26005 : case IX86_BUILTIN_NANSQ:
19190 26005 : {
19191 26005 : tree type = TREE_TYPE (TREE_TYPE (fndecl));
19192 26005 : const char *str = c_getstr (*args);
19193 26005 : int quiet = fn_code == IX86_BUILTIN_NANQ;
19194 26005 : REAL_VALUE_TYPE real;
19195 :
19196 26005 : if (str && real_nan (&real, str, quiet, TYPE_MODE (type)))
19197 26005 : return build_real (type, real);
19198 0 : return NULL_TREE;
19199 : }
19200 :
19201 108 : case IX86_BUILTIN_INFQ:
19202 108 : case IX86_BUILTIN_HUGE_VALQ:
19203 108 : {
19204 108 : tree type = TREE_TYPE (TREE_TYPE (fndecl));
19205 108 : REAL_VALUE_TYPE inf;
19206 108 : real_inf (&inf);
19207 108 : return build_real (type, inf);
19208 : }
19209 :
19210 62501 : case IX86_BUILTIN_TZCNT16:
19211 62501 : case IX86_BUILTIN_CTZS:
19212 62501 : case IX86_BUILTIN_TZCNT32:
19213 62501 : case IX86_BUILTIN_TZCNT64:
19214 62501 : gcc_assert (n_args == 1);
19215 62501 : if (TREE_CODE (args[0]) == INTEGER_CST)
19216 : {
19217 45 : tree type = TREE_TYPE (TREE_TYPE (fndecl));
19218 45 : tree arg = args[0];
19219 45 : if (fn_code == IX86_BUILTIN_TZCNT16
19220 45 : || fn_code == IX86_BUILTIN_CTZS)
19221 3 : arg = fold_convert (short_unsigned_type_node, arg);
19222 45 : if (integer_zerop (arg))
19223 6 : return build_int_cst (type, TYPE_PRECISION (TREE_TYPE (arg)));
19224 : else
19225 39 : return fold_const_call (CFN_CTZ, type, arg);
19226 : }
19227 : break;
19228 :
19229 52043 : case IX86_BUILTIN_LZCNT16:
19230 52043 : case IX86_BUILTIN_CLZS:
19231 52043 : case IX86_BUILTIN_LZCNT32:
19232 52043 : case IX86_BUILTIN_LZCNT64:
19233 52043 : gcc_assert (n_args == 1);
19234 52043 : if (TREE_CODE (args[0]) == INTEGER_CST)
19235 : {
19236 54 : tree type = TREE_TYPE (TREE_TYPE (fndecl));
19237 54 : tree arg = args[0];
19238 54 : if (fn_code == IX86_BUILTIN_LZCNT16
19239 54 : || fn_code == IX86_BUILTIN_CLZS)
19240 18 : arg = fold_convert (short_unsigned_type_node, arg);
19241 54 : if (integer_zerop (arg))
19242 3 : return build_int_cst (type, TYPE_PRECISION (TREE_TYPE (arg)));
19243 : else
19244 51 : return fold_const_call (CFN_CLZ, type, arg);
19245 : }
19246 : break;
19247 :
19248 61289 : case IX86_BUILTIN_BEXTR32:
19249 61289 : case IX86_BUILTIN_BEXTR64:
19250 61289 : case IX86_BUILTIN_BEXTRI32:
19251 61289 : case IX86_BUILTIN_BEXTRI64:
19252 61289 : gcc_assert (n_args == 2);
19253 61289 : if (tree_fits_uhwi_p (args[1]))
19254 : {
19255 152 : unsigned HOST_WIDE_INT res = 0;
19256 152 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (args[0]));
19257 152 : unsigned int start = tree_to_uhwi (args[1]);
19258 152 : unsigned int len = (start & 0xff00) >> 8;
19259 152 : tree lhs_type = TREE_TYPE (TREE_TYPE (fndecl));
19260 152 : start &= 0xff;
19261 152 : if (start >= prec || len == 0)
19262 111 : return omit_one_operand (lhs_type, build_zero_cst (lhs_type),
19263 : args[0]);
19264 41 : else if (!tree_fits_uhwi_p (args[0]))
19265 : break;
19266 : else
19267 24 : res = tree_to_uhwi (args[0]) >> start;
19268 24 : if (len > prec)
19269 : len = prec;
19270 24 : if (len < HOST_BITS_PER_WIDE_INT)
19271 15 : res &= (HOST_WIDE_INT_1U << len) - 1;
19272 24 : return build_int_cstu (lhs_type, res);
19273 : }
19274 : break;
19275 :
19276 21044 : case IX86_BUILTIN_BZHI32:
19277 21044 : case IX86_BUILTIN_BZHI64:
19278 21044 : gcc_assert (n_args == 2);
19279 21044 : if (tree_fits_uhwi_p (args[1]))
19280 : {
19281 190 : unsigned int idx = tree_to_uhwi (args[1]) & 0xff;
19282 190 : tree lhs_type = TREE_TYPE (TREE_TYPE (fndecl));
19283 190 : if (idx >= TYPE_PRECISION (TREE_TYPE (args[0])))
19284 : return args[0];
19285 190 : if (idx == 0)
19286 52 : return omit_one_operand (lhs_type, build_zero_cst (lhs_type),
19287 : args[0]);
19288 138 : if (!tree_fits_uhwi_p (args[0]))
19289 : break;
19290 12 : unsigned HOST_WIDE_INT res = tree_to_uhwi (args[0]);
19291 12 : res &= ~(HOST_WIDE_INT_M1U << idx);
19292 12 : return build_int_cstu (lhs_type, res);
19293 : }
19294 : break;
19295 :
19296 20810 : case IX86_BUILTIN_PDEP32:
19297 20810 : case IX86_BUILTIN_PDEP64:
19298 20810 : gcc_assert (n_args == 2);
19299 20810 : if (tree_fits_uhwi_p (args[0]) && tree_fits_uhwi_p (args[1]))
19300 : {
19301 46 : unsigned HOST_WIDE_INT src = tree_to_uhwi (args[0]);
19302 46 : unsigned HOST_WIDE_INT mask = tree_to_uhwi (args[1]);
19303 46 : unsigned HOST_WIDE_INT res = 0;
19304 46 : unsigned HOST_WIDE_INT m, k = 1;
19305 2990 : for (m = 1; m; m <<= 1)
19306 2944 : if ((mask & m) != 0)
19307 : {
19308 1440 : if ((src & k) != 0)
19309 789 : res |= m;
19310 1440 : k <<= 1;
19311 : }
19312 46 : return build_int_cstu (TREE_TYPE (TREE_TYPE (fndecl)), res);
19313 : }
19314 : break;
19315 :
19316 20812 : case IX86_BUILTIN_PEXT32:
19317 20812 : case IX86_BUILTIN_PEXT64:
19318 20812 : gcc_assert (n_args == 2);
19319 20812 : if (tree_fits_uhwi_p (args[0]) && tree_fits_uhwi_p (args[1]))
19320 : {
19321 46 : unsigned HOST_WIDE_INT src = tree_to_uhwi (args[0]);
19322 46 : unsigned HOST_WIDE_INT mask = tree_to_uhwi (args[1]);
19323 46 : unsigned HOST_WIDE_INT res = 0;
19324 46 : unsigned HOST_WIDE_INT m, k = 1;
19325 2990 : for (m = 1; m; m <<= 1)
19326 2944 : if ((mask & m) != 0)
19327 : {
19328 2016 : if ((src & m) != 0)
19329 1063 : res |= k;
19330 2016 : k <<= 1;
19331 : }
19332 46 : return build_int_cstu (TREE_TYPE (TREE_TYPE (fndecl)), res);
19333 : }
19334 : break;
19335 :
19336 127062 : case IX86_BUILTIN_MOVMSKPS:
19337 127062 : case IX86_BUILTIN_PMOVMSKB:
19338 127062 : case IX86_BUILTIN_MOVMSKPD:
19339 127062 : case IX86_BUILTIN_PMOVMSKB128:
19340 127062 : case IX86_BUILTIN_MOVMSKPD256:
19341 127062 : case IX86_BUILTIN_MOVMSKPS256:
19342 127062 : case IX86_BUILTIN_PMOVMSKB256:
19343 127062 : gcc_assert (n_args == 1);
19344 127062 : if (TREE_CODE (args[0]) == VECTOR_CST)
19345 : {
19346 : HOST_WIDE_INT res = 0;
19347 3213 : for (unsigned i = 0; i < VECTOR_CST_NELTS (args[0]); ++i)
19348 : {
19349 2620 : tree e = VECTOR_CST_ELT (args[0], i);
19350 2620 : if (TREE_CODE (e) == INTEGER_CST && !TREE_OVERFLOW (e))
19351 : {
19352 1168 : if (wi::neg_p (wi::to_wide (e)))
19353 1119 : res |= HOST_WIDE_INT_1 << i;
19354 : }
19355 1452 : else if (TREE_CODE (e) == REAL_CST && !TREE_OVERFLOW (e))
19356 : {
19357 1452 : if (TREE_REAL_CST (e).sign)
19358 1215 : res |= HOST_WIDE_INT_1 << i;
19359 : }
19360 : else
19361 : return NULL_TREE;
19362 : }
19363 593 : return build_int_cst (TREE_TYPE (TREE_TYPE (fndecl)), res);
19364 : }
19365 : break;
19366 :
19367 660338 : case IX86_BUILTIN_PSLLD:
19368 660338 : case IX86_BUILTIN_PSLLD128:
19369 660338 : case IX86_BUILTIN_PSLLD128_MASK:
19370 660338 : case IX86_BUILTIN_PSLLD256:
19371 660338 : case IX86_BUILTIN_PSLLD256_MASK:
19372 660338 : case IX86_BUILTIN_PSLLD512:
19373 660338 : case IX86_BUILTIN_PSLLDI:
19374 660338 : case IX86_BUILTIN_PSLLDI128:
19375 660338 : case IX86_BUILTIN_PSLLDI128_MASK:
19376 660338 : case IX86_BUILTIN_PSLLDI256:
19377 660338 : case IX86_BUILTIN_PSLLDI256_MASK:
19378 660338 : case IX86_BUILTIN_PSLLDI512:
19379 660338 : case IX86_BUILTIN_PSLLQ:
19380 660338 : case IX86_BUILTIN_PSLLQ128:
19381 660338 : case IX86_BUILTIN_PSLLQ128_MASK:
19382 660338 : case IX86_BUILTIN_PSLLQ256:
19383 660338 : case IX86_BUILTIN_PSLLQ256_MASK:
19384 660338 : case IX86_BUILTIN_PSLLQ512:
19385 660338 : case IX86_BUILTIN_PSLLQI:
19386 660338 : case IX86_BUILTIN_PSLLQI128:
19387 660338 : case IX86_BUILTIN_PSLLQI128_MASK:
19388 660338 : case IX86_BUILTIN_PSLLQI256:
19389 660338 : case IX86_BUILTIN_PSLLQI256_MASK:
19390 660338 : case IX86_BUILTIN_PSLLQI512:
19391 660338 : case IX86_BUILTIN_PSLLW:
19392 660338 : case IX86_BUILTIN_PSLLW128:
19393 660338 : case IX86_BUILTIN_PSLLW128_MASK:
19394 660338 : case IX86_BUILTIN_PSLLW256:
19395 660338 : case IX86_BUILTIN_PSLLW256_MASK:
19396 660338 : case IX86_BUILTIN_PSLLW512_MASK:
19397 660338 : case IX86_BUILTIN_PSLLWI:
19398 660338 : case IX86_BUILTIN_PSLLWI128:
19399 660338 : case IX86_BUILTIN_PSLLWI128_MASK:
19400 660338 : case IX86_BUILTIN_PSLLWI256:
19401 660338 : case IX86_BUILTIN_PSLLWI256_MASK:
19402 660338 : case IX86_BUILTIN_PSLLWI512_MASK:
19403 660338 : rcode = ASHIFT;
19404 660338 : is_vshift = false;
19405 660338 : goto do_shift;
19406 601973 : case IX86_BUILTIN_PSRAD:
19407 601973 : case IX86_BUILTIN_PSRAD128:
19408 601973 : case IX86_BUILTIN_PSRAD128_MASK:
19409 601973 : case IX86_BUILTIN_PSRAD256:
19410 601973 : case IX86_BUILTIN_PSRAD256_MASK:
19411 601973 : case IX86_BUILTIN_PSRAD512:
19412 601973 : case IX86_BUILTIN_PSRADI:
19413 601973 : case IX86_BUILTIN_PSRADI128:
19414 601973 : case IX86_BUILTIN_PSRADI128_MASK:
19415 601973 : case IX86_BUILTIN_PSRADI256:
19416 601973 : case IX86_BUILTIN_PSRADI256_MASK:
19417 601973 : case IX86_BUILTIN_PSRADI512:
19418 601973 : case IX86_BUILTIN_PSRAQ128_MASK:
19419 601973 : case IX86_BUILTIN_PSRAQ256_MASK:
19420 601973 : case IX86_BUILTIN_PSRAQ512:
19421 601973 : case IX86_BUILTIN_PSRAQI128_MASK:
19422 601973 : case IX86_BUILTIN_PSRAQI256_MASK:
19423 601973 : case IX86_BUILTIN_PSRAQI512:
19424 601973 : case IX86_BUILTIN_PSRAW:
19425 601973 : case IX86_BUILTIN_PSRAW128:
19426 601973 : case IX86_BUILTIN_PSRAW128_MASK:
19427 601973 : case IX86_BUILTIN_PSRAW256:
19428 601973 : case IX86_BUILTIN_PSRAW256_MASK:
19429 601973 : case IX86_BUILTIN_PSRAW512:
19430 601973 : case IX86_BUILTIN_PSRAWI:
19431 601973 : case IX86_BUILTIN_PSRAWI128:
19432 601973 : case IX86_BUILTIN_PSRAWI128_MASK:
19433 601973 : case IX86_BUILTIN_PSRAWI256:
19434 601973 : case IX86_BUILTIN_PSRAWI256_MASK:
19435 601973 : case IX86_BUILTIN_PSRAWI512:
19436 601973 : rcode = ASHIFTRT;
19437 601973 : is_vshift = false;
19438 601973 : goto do_shift;
19439 634257 : case IX86_BUILTIN_PSRLD:
19440 634257 : case IX86_BUILTIN_PSRLD128:
19441 634257 : case IX86_BUILTIN_PSRLD128_MASK:
19442 634257 : case IX86_BUILTIN_PSRLD256:
19443 634257 : case IX86_BUILTIN_PSRLD256_MASK:
19444 634257 : case IX86_BUILTIN_PSRLD512:
19445 634257 : case IX86_BUILTIN_PSRLDI:
19446 634257 : case IX86_BUILTIN_PSRLDI128:
19447 634257 : case IX86_BUILTIN_PSRLDI128_MASK:
19448 634257 : case IX86_BUILTIN_PSRLDI256:
19449 634257 : case IX86_BUILTIN_PSRLDI256_MASK:
19450 634257 : case IX86_BUILTIN_PSRLDI512:
19451 634257 : case IX86_BUILTIN_PSRLQ:
19452 634257 : case IX86_BUILTIN_PSRLQ128:
19453 634257 : case IX86_BUILTIN_PSRLQ128_MASK:
19454 634257 : case IX86_BUILTIN_PSRLQ256:
19455 634257 : case IX86_BUILTIN_PSRLQ256_MASK:
19456 634257 : case IX86_BUILTIN_PSRLQ512:
19457 634257 : case IX86_BUILTIN_PSRLQI:
19458 634257 : case IX86_BUILTIN_PSRLQI128:
19459 634257 : case IX86_BUILTIN_PSRLQI128_MASK:
19460 634257 : case IX86_BUILTIN_PSRLQI256:
19461 634257 : case IX86_BUILTIN_PSRLQI256_MASK:
19462 634257 : case IX86_BUILTIN_PSRLQI512:
19463 634257 : case IX86_BUILTIN_PSRLW:
19464 634257 : case IX86_BUILTIN_PSRLW128:
19465 634257 : case IX86_BUILTIN_PSRLW128_MASK:
19466 634257 : case IX86_BUILTIN_PSRLW256:
19467 634257 : case IX86_BUILTIN_PSRLW256_MASK:
19468 634257 : case IX86_BUILTIN_PSRLW512:
19469 634257 : case IX86_BUILTIN_PSRLWI:
19470 634257 : case IX86_BUILTIN_PSRLWI128:
19471 634257 : case IX86_BUILTIN_PSRLWI128_MASK:
19472 634257 : case IX86_BUILTIN_PSRLWI256:
19473 634257 : case IX86_BUILTIN_PSRLWI256_MASK:
19474 634257 : case IX86_BUILTIN_PSRLWI512:
19475 634257 : rcode = LSHIFTRT;
19476 634257 : is_vshift = false;
19477 634257 : goto do_shift;
19478 276266 : case IX86_BUILTIN_PSLLVV16HI:
19479 276266 : case IX86_BUILTIN_PSLLVV16SI:
19480 276266 : case IX86_BUILTIN_PSLLVV2DI:
19481 276266 : case IX86_BUILTIN_PSLLVV2DI_MASK:
19482 276266 : case IX86_BUILTIN_PSLLVV32HI:
19483 276266 : case IX86_BUILTIN_PSLLVV4DI:
19484 276266 : case IX86_BUILTIN_PSLLVV4DI_MASK:
19485 276266 : case IX86_BUILTIN_PSLLVV4SI:
19486 276266 : case IX86_BUILTIN_PSLLVV4SI_MASK:
19487 276266 : case IX86_BUILTIN_PSLLVV8DI:
19488 276266 : case IX86_BUILTIN_PSLLVV8HI:
19489 276266 : case IX86_BUILTIN_PSLLVV8SI:
19490 276266 : case IX86_BUILTIN_PSLLVV8SI_MASK:
19491 276266 : rcode = ASHIFT;
19492 276266 : is_vshift = true;
19493 276266 : goto do_shift;
19494 275847 : case IX86_BUILTIN_PSRAVQ128:
19495 275847 : case IX86_BUILTIN_PSRAVQ256:
19496 275847 : case IX86_BUILTIN_PSRAVV16HI:
19497 275847 : case IX86_BUILTIN_PSRAVV16SI:
19498 275847 : case IX86_BUILTIN_PSRAVV32HI:
19499 275847 : case IX86_BUILTIN_PSRAVV4SI:
19500 275847 : case IX86_BUILTIN_PSRAVV4SI_MASK:
19501 275847 : case IX86_BUILTIN_PSRAVV8DI:
19502 275847 : case IX86_BUILTIN_PSRAVV8HI:
19503 275847 : case IX86_BUILTIN_PSRAVV8SI:
19504 275847 : case IX86_BUILTIN_PSRAVV8SI_MASK:
19505 275847 : rcode = ASHIFTRT;
19506 275847 : is_vshift = true;
19507 275847 : goto do_shift;
19508 276257 : case IX86_BUILTIN_PSRLVV16HI:
19509 276257 : case IX86_BUILTIN_PSRLVV16SI:
19510 276257 : case IX86_BUILTIN_PSRLVV2DI:
19511 276257 : case IX86_BUILTIN_PSRLVV2DI_MASK:
19512 276257 : case IX86_BUILTIN_PSRLVV32HI:
19513 276257 : case IX86_BUILTIN_PSRLVV4DI:
19514 276257 : case IX86_BUILTIN_PSRLVV4DI_MASK:
19515 276257 : case IX86_BUILTIN_PSRLVV4SI:
19516 276257 : case IX86_BUILTIN_PSRLVV4SI_MASK:
19517 276257 : case IX86_BUILTIN_PSRLVV8DI:
19518 276257 : case IX86_BUILTIN_PSRLVV8HI:
19519 276257 : case IX86_BUILTIN_PSRLVV8SI:
19520 276257 : case IX86_BUILTIN_PSRLVV8SI_MASK:
19521 276257 : rcode = LSHIFTRT;
19522 276257 : is_vshift = true;
19523 276257 : goto do_shift;
19524 :
19525 2724938 : do_shift:
19526 2724938 : gcc_assert (n_args >= 2);
19527 2724938 : if (TREE_CODE (args[0]) != VECTOR_CST)
19528 : break;
19529 927 : mask = HOST_WIDE_INT_M1U;
19530 927 : if (n_args > 2)
19531 : {
19532 : /* This is masked shift. */
19533 678 : if (!tree_fits_uhwi_p (args[n_args - 1])
19534 678 : || TREE_SIDE_EFFECTS (args[n_args - 2]))
19535 : break;
19536 678 : mask = tree_to_uhwi (args[n_args - 1]);
19537 678 : unsigned elems = TYPE_VECTOR_SUBPARTS (TREE_TYPE (args[0]));
19538 678 : mask |= HOST_WIDE_INT_M1U << elems;
19539 678 : if (mask != HOST_WIDE_INT_M1U
19540 567 : && TREE_CODE (args[n_args - 2]) != VECTOR_CST)
19541 : break;
19542 633 : if (mask == (HOST_WIDE_INT_M1U << elems))
19543 : return args[n_args - 2];
19544 : }
19545 879 : if (is_vshift && TREE_CODE (args[1]) != VECTOR_CST)
19546 : break;
19547 879 : if (tree tem = (is_vshift ? integer_one_node
19548 879 : : ix86_vector_shift_count (args[1])))
19549 : {
19550 558 : unsigned HOST_WIDE_INT count = tree_to_uhwi (tem);
19551 558 : unsigned HOST_WIDE_INT prec
19552 558 : = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (args[0])));
19553 558 : if (count == 0 && mask == HOST_WIDE_INT_M1U)
19554 : return args[0];
19555 558 : if (count >= prec)
19556 : {
19557 72 : if (rcode == ASHIFTRT)
19558 27 : count = prec - 1;
19559 45 : else if (mask == HOST_WIDE_INT_M1U)
19560 3 : return build_zero_cst (TREE_TYPE (args[0]));
19561 : }
19562 555 : tree countt = NULL_TREE;
19563 555 : if (!is_vshift)
19564 : {
19565 377 : if (count >= prec)
19566 42 : countt = integer_zero_node;
19567 : else
19568 335 : countt = build_int_cst (integer_type_node, count);
19569 : }
19570 555 : tree_vector_builder builder;
19571 555 : if (mask != HOST_WIDE_INT_M1U || is_vshift)
19572 392 : builder.new_vector (TREE_TYPE (args[0]),
19573 784 : TYPE_VECTOR_SUBPARTS (TREE_TYPE (args[0])),
19574 : 1);
19575 : else
19576 163 : builder.new_unary_operation (TREE_TYPE (args[0]), args[0],
19577 : false);
19578 555 : unsigned int cnt = builder.encoded_nelts ();
19579 5967 : for (unsigned int i = 0; i < cnt; ++i)
19580 : {
19581 5412 : tree elt = VECTOR_CST_ELT (args[0], i);
19582 5412 : if (TREE_CODE (elt) != INTEGER_CST || TREE_OVERFLOW (elt))
19583 0 : return NULL_TREE;
19584 5412 : tree type = TREE_TYPE (elt);
19585 5412 : if (rcode == LSHIFTRT)
19586 2040 : elt = fold_convert (unsigned_type_for (type), elt);
19587 5412 : if (is_vshift)
19588 : {
19589 1846 : countt = VECTOR_CST_ELT (args[1], i);
19590 1846 : if (TREE_CODE (countt) != INTEGER_CST
19591 1846 : || TREE_OVERFLOW (countt))
19592 : return NULL_TREE;
19593 1846 : if (wi::neg_p (wi::to_wide (countt))
19594 3610 : || wi::to_widest (countt) >= prec)
19595 : {
19596 325 : if (rcode == ASHIFTRT)
19597 108 : countt = build_int_cst (TREE_TYPE (countt),
19598 108 : prec - 1);
19599 : else
19600 : {
19601 217 : elt = build_zero_cst (TREE_TYPE (elt));
19602 217 : countt = build_zero_cst (TREE_TYPE (countt));
19603 : }
19604 : }
19605 : }
19606 3566 : else if (count >= prec)
19607 504 : elt = build_zero_cst (TREE_TYPE (elt));
19608 8950 : elt = const_binop (rcode == ASHIFT
19609 : ? LSHIFT_EXPR : RSHIFT_EXPR,
19610 5412 : TREE_TYPE (elt), elt, countt);
19611 5412 : if (!elt || TREE_CODE (elt) != INTEGER_CST)
19612 : return NULL_TREE;
19613 5412 : if (rcode == LSHIFTRT)
19614 2040 : elt = fold_convert (type, elt);
19615 5412 : if ((mask & (HOST_WIDE_INT_1U << i)) == 0)
19616 : {
19617 1566 : elt = VECTOR_CST_ELT (args[n_args - 2], i);
19618 1566 : if (TREE_CODE (elt) != INTEGER_CST
19619 1566 : || TREE_OVERFLOW (elt))
19620 : return NULL_TREE;
19621 : }
19622 5412 : builder.quick_push (elt);
19623 : }
19624 555 : return builder.build ();
19625 555 : }
19626 : break;
19627 :
19628 32748 : case IX86_BUILTIN_MINSS:
19629 32748 : case IX86_BUILTIN_MINSH_MASK:
19630 32748 : tcode = LT_EXPR;
19631 32748 : is_scalar = true;
19632 32748 : goto do_minmax;
19633 :
19634 32748 : case IX86_BUILTIN_MAXSS:
19635 32748 : case IX86_BUILTIN_MAXSH_MASK:
19636 32748 : tcode = GT_EXPR;
19637 32748 : is_scalar = true;
19638 32748 : goto do_minmax;
19639 :
19640 350995 : case IX86_BUILTIN_MINPS:
19641 350995 : case IX86_BUILTIN_MINPD:
19642 350995 : case IX86_BUILTIN_MINPS256:
19643 350995 : case IX86_BUILTIN_MINPD256:
19644 350995 : case IX86_BUILTIN_MINPS512:
19645 350995 : case IX86_BUILTIN_MINPD512:
19646 350995 : case IX86_BUILTIN_MINPS128_MASK:
19647 350995 : case IX86_BUILTIN_MINPD128_MASK:
19648 350995 : case IX86_BUILTIN_MINPS256_MASK:
19649 350995 : case IX86_BUILTIN_MINPD256_MASK:
19650 350995 : case IX86_BUILTIN_MINPH128_MASK:
19651 350995 : case IX86_BUILTIN_MINPH256_MASK:
19652 350995 : case IX86_BUILTIN_MINPH512_MASK:
19653 350995 : tcode = LT_EXPR;
19654 350995 : is_scalar = false;
19655 350995 : goto do_minmax;
19656 :
19657 : case IX86_BUILTIN_MAXPS:
19658 : case IX86_BUILTIN_MAXPD:
19659 : case IX86_BUILTIN_MAXPS256:
19660 : case IX86_BUILTIN_MAXPD256:
19661 : case IX86_BUILTIN_MAXPS512:
19662 : case IX86_BUILTIN_MAXPD512:
19663 : case IX86_BUILTIN_MAXPS128_MASK:
19664 : case IX86_BUILTIN_MAXPD128_MASK:
19665 : case IX86_BUILTIN_MAXPS256_MASK:
19666 : case IX86_BUILTIN_MAXPD256_MASK:
19667 : case IX86_BUILTIN_MAXPH128_MASK:
19668 : case IX86_BUILTIN_MAXPH256_MASK:
19669 : case IX86_BUILTIN_MAXPH512_MASK:
19670 : tcode = GT_EXPR;
19671 : is_scalar = false;
19672 767506 : do_minmax:
19673 767506 : gcc_assert (n_args >= 2);
19674 767506 : if (TREE_CODE (args[0]) != VECTOR_CST
19675 76 : || TREE_CODE (args[1]) != VECTOR_CST)
19676 : break;
19677 76 : mask = HOST_WIDE_INT_M1U;
19678 76 : if (n_args > 2)
19679 : {
19680 36 : gcc_assert (n_args >= 4);
19681 : /* This is masked minmax. */
19682 36 : if (TREE_CODE (args[3]) != INTEGER_CST
19683 36 : || TREE_SIDE_EFFECTS (args[2]))
19684 : break;
19685 36 : mask = TREE_INT_CST_LOW (args[3]);
19686 36 : unsigned elems = TYPE_VECTOR_SUBPARTS (TREE_TYPE (args[0]));
19687 36 : mask |= HOST_WIDE_INT_M1U << elems;
19688 36 : if (mask != HOST_WIDE_INT_M1U
19689 32 : && TREE_CODE (args[2]) != VECTOR_CST)
19690 : break;
19691 36 : if (n_args >= 5)
19692 : {
19693 20 : if (!tree_fits_uhwi_p (args[4]))
19694 : break;
19695 20 : if (tree_to_uhwi (args[4]) != 4
19696 0 : && tree_to_uhwi (args[4]) != 8)
19697 : break;
19698 : }
19699 36 : if (mask == (HOST_WIDE_INT_M1U << elems))
19700 : return args[2];
19701 : }
19702 : /* Punt on NaNs, unless exceptions are disabled. */
19703 76 : if (HONOR_NANS (args[0])
19704 76 : && (n_args < 5 || tree_to_uhwi (args[4]) != 8))
19705 184 : for (int i = 0; i < 2; ++i)
19706 : {
19707 134 : unsigned count = vector_cst_encoded_nelts (args[i]);
19708 957 : for (unsigned j = 0; j < count; ++j)
19709 849 : if (tree_expr_nan_p (VECTOR_CST_ENCODED_ELT (args[i], j)))
19710 : return NULL_TREE;
19711 : }
19712 50 : {
19713 50 : tree res = const_binop (tcode,
19714 50 : truth_type_for (TREE_TYPE (args[0])),
19715 : args[0], args[1]);
19716 50 : if (res == NULL_TREE || TREE_CODE (res) != VECTOR_CST)
19717 : break;
19718 50 : res = fold_ternary (VEC_COND_EXPR, TREE_TYPE (args[0]), res,
19719 : args[0], args[1]);
19720 50 : if (res == NULL_TREE || TREE_CODE (res) != VECTOR_CST)
19721 : break;
19722 50 : if (mask != HOST_WIDE_INT_M1U)
19723 : {
19724 32 : unsigned nelts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (args[0]));
19725 32 : vec_perm_builder sel (nelts, nelts, 1);
19726 328 : for (unsigned int i = 0; i < nelts; i++)
19727 296 : if (mask & (HOST_WIDE_INT_1U << i))
19728 160 : sel.quick_push (i);
19729 : else
19730 136 : sel.quick_push (nelts + i);
19731 32 : vec_perm_indices indices (sel, 2, nelts);
19732 32 : res = fold_vec_perm (TREE_TYPE (args[0]), res, args[2],
19733 : indices);
19734 32 : if (res == NULL_TREE || TREE_CODE (res) != VECTOR_CST)
19735 : break;
19736 32 : }
19737 50 : if (is_scalar)
19738 : {
19739 10 : unsigned nelts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (args[0]));
19740 10 : vec_perm_builder sel (nelts, nelts, 1);
19741 10 : sel.quick_push (0);
19742 40 : for (unsigned int i = 1; i < nelts; i++)
19743 30 : sel.quick_push (nelts + i);
19744 10 : vec_perm_indices indices (sel, 2, nelts);
19745 10 : res = fold_vec_perm (TREE_TYPE (args[0]), res, args[0],
19746 : indices);
19747 10 : if (res == NULL_TREE || TREE_CODE (res) != VECTOR_CST)
19748 : break;
19749 10 : }
19750 50 : return res;
19751 : }
19752 :
19753 : default:
19754 : break;
19755 : }
19756 : }
19757 :
19758 : #ifdef SUBTARGET_FOLD_BUILTIN
19759 : return SUBTARGET_FOLD_BUILTIN (fndecl, n_args, args, ignore);
19760 : #endif
19761 :
19762 : return NULL_TREE;
19763 : }
19764 :
19765 : /* Fold a MD builtin (use ix86_fold_builtin for folding into
19766 : constant) in GIMPLE. */
19767 :
19768 : bool
19769 1154467 : ix86_gimple_fold_builtin (gimple_stmt_iterator *gsi)
19770 : {
19771 1154467 : gimple *stmt = gsi_stmt (*gsi), *g;
19772 1154467 : gimple_seq stmts = NULL;
19773 1154467 : tree fndecl = gimple_call_fndecl (stmt);
19774 1154467 : gcc_checking_assert (fndecl && fndecl_built_in_p (fndecl, BUILT_IN_MD));
19775 1154467 : int n_args = gimple_call_num_args (stmt);
19776 1154467 : enum ix86_builtins fn_code
19777 1154467 : = (enum ix86_builtins) DECL_MD_FUNCTION_CODE (fndecl);
19778 1154467 : tree decl = NULL_TREE;
19779 1154467 : tree arg0, arg1, arg2;
19780 1154467 : enum rtx_code rcode;
19781 1154467 : enum tree_code tcode;
19782 1154467 : unsigned HOST_WIDE_INT count;
19783 1154467 : bool is_vshift;
19784 1154467 : unsigned HOST_WIDE_INT elems;
19785 1154467 : location_t loc;
19786 :
19787 : /* Don't fold when there's isa mismatch. */
19788 1154467 : if (!ix86_check_builtin_isa_match (fn_code, NULL, NULL))
19789 : return false;
19790 :
19791 1154340 : switch (fn_code)
19792 : {
19793 288 : case IX86_BUILTIN_TZCNT32:
19794 288 : decl = builtin_decl_implicit (BUILT_IN_CTZ);
19795 288 : goto fold_tzcnt_lzcnt;
19796 :
19797 237 : case IX86_BUILTIN_TZCNT64:
19798 237 : decl = builtin_decl_implicit (BUILT_IN_CTZLL);
19799 237 : goto fold_tzcnt_lzcnt;
19800 :
19801 215 : case IX86_BUILTIN_LZCNT32:
19802 215 : decl = builtin_decl_implicit (BUILT_IN_CLZ);
19803 215 : goto fold_tzcnt_lzcnt;
19804 :
19805 224 : case IX86_BUILTIN_LZCNT64:
19806 224 : decl = builtin_decl_implicit (BUILT_IN_CLZLL);
19807 224 : goto fold_tzcnt_lzcnt;
19808 :
19809 964 : fold_tzcnt_lzcnt:
19810 964 : gcc_assert (n_args == 1);
19811 964 : arg0 = gimple_call_arg (stmt, 0);
19812 964 : if (TREE_CODE (arg0) == SSA_NAME && decl && gimple_call_lhs (stmt))
19813 : {
19814 799 : int prec = TYPE_PRECISION (TREE_TYPE (arg0));
19815 : /* If arg0 is provably non-zero, optimize into generic
19816 : __builtin_c[tl]z{,ll} function the middle-end handles
19817 : better. */
19818 799 : if (!expr_not_equal_to (arg0, wi::zero (prec)))
19819 : return false;
19820 :
19821 9 : loc = gimple_location (stmt);
19822 9 : g = gimple_build_call (decl, 1, arg0);
19823 9 : gimple_set_location (g, loc);
19824 9 : tree lhs = make_ssa_name (integer_type_node);
19825 9 : gimple_call_set_lhs (g, lhs);
19826 9 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
19827 9 : g = gimple_build_assign (gimple_call_lhs (stmt), NOP_EXPR, lhs);
19828 9 : gimple_set_location (g, loc);
19829 9 : gsi_replace (gsi, g, false);
19830 9 : return true;
19831 : }
19832 : break;
19833 :
19834 489 : case IX86_BUILTIN_BZHI32:
19835 489 : case IX86_BUILTIN_BZHI64:
19836 489 : gcc_assert (n_args == 2);
19837 489 : arg1 = gimple_call_arg (stmt, 1);
19838 489 : if (tree_fits_uhwi_p (arg1) && gimple_call_lhs (stmt))
19839 : {
19840 195 : unsigned int idx = tree_to_uhwi (arg1) & 0xff;
19841 195 : arg0 = gimple_call_arg (stmt, 0);
19842 195 : if (idx < TYPE_PRECISION (TREE_TYPE (arg0)))
19843 : break;
19844 31 : loc = gimple_location (stmt);
19845 31 : g = gimple_build_assign (gimple_call_lhs (stmt), arg0);
19846 31 : gimple_set_location (g, loc);
19847 31 : gsi_replace (gsi, g, false);
19848 31 : return true;
19849 : }
19850 : break;
19851 :
19852 502 : case IX86_BUILTIN_PDEP32:
19853 502 : case IX86_BUILTIN_PDEP64:
19854 502 : case IX86_BUILTIN_PEXT32:
19855 502 : case IX86_BUILTIN_PEXT64:
19856 502 : gcc_assert (n_args == 2);
19857 502 : arg1 = gimple_call_arg (stmt, 1);
19858 502 : if (integer_all_onesp (arg1) && gimple_call_lhs (stmt))
19859 : {
19860 4 : loc = gimple_location (stmt);
19861 4 : arg0 = gimple_call_arg (stmt, 0);
19862 4 : g = gimple_build_assign (gimple_call_lhs (stmt), arg0);
19863 4 : gimple_set_location (g, loc);
19864 4 : gsi_replace (gsi, g, false);
19865 4 : return true;
19866 : }
19867 : break;
19868 :
19869 145 : case IX86_BUILTIN_PBLENDVB256:
19870 145 : case IX86_BUILTIN_BLENDVPS256:
19871 145 : case IX86_BUILTIN_BLENDVPD256:
19872 : /* pcmpeqb/d/q is under avx2, w/o avx2, it's veclower
19873 : to scalar operations and not combined back. */
19874 145 : if (!TARGET_AVX2)
19875 : break;
19876 :
19877 : /* FALLTHRU. */
19878 112 : case IX86_BUILTIN_BLENDVPD:
19879 : /* blendvpd is under sse4.1 but pcmpgtq is under sse4.2,
19880 : w/o sse4.2, it's veclowered to scalar operations and
19881 : not combined back. */
19882 112 : if (!TARGET_SSE4_2)
19883 : break;
19884 : /* FALLTHRU. */
19885 166 : case IX86_BUILTIN_PBLENDVB128:
19886 166 : case IX86_BUILTIN_BLENDVPS:
19887 166 : gcc_assert (n_args == 3);
19888 166 : arg0 = gimple_call_arg (stmt, 0);
19889 166 : arg1 = gimple_call_arg (stmt, 1);
19890 166 : arg2 = gimple_call_arg (stmt, 2);
19891 166 : if (gimple_call_lhs (stmt))
19892 : {
19893 166 : loc = gimple_location (stmt);
19894 166 : tree type = TREE_TYPE (arg2);
19895 166 : if (VECTOR_FLOAT_TYPE_P (type))
19896 : {
19897 73 : tree itype = GET_MODE_INNER (TYPE_MODE (type)) == E_SFmode
19898 73 : ? intSI_type_node : intDI_type_node;
19899 73 : type = get_same_sized_vectype (itype, type);
19900 : }
19901 : else
19902 93 : type = signed_type_for (type);
19903 166 : arg2 = gimple_build (&stmts, VIEW_CONVERT_EXPR, type, arg2);
19904 166 : tree zero_vec = build_zero_cst (type);
19905 166 : tree cmp_type = truth_type_for (type);
19906 166 : tree cmp = gimple_build (&stmts, LT_EXPR, cmp_type, arg2, zero_vec);
19907 166 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
19908 166 : g = gimple_build_assign (gimple_call_lhs (stmt),
19909 : VEC_COND_EXPR, cmp,
19910 : arg1, arg0);
19911 166 : gimple_set_location (g, loc);
19912 166 : gsi_replace (gsi, g, false);
19913 : }
19914 : else
19915 0 : gsi_replace (gsi, gimple_build_nop (), false);
19916 : return true;
19917 :
19918 :
19919 16 : case IX86_BUILTIN_PCMPEQB128:
19920 16 : case IX86_BUILTIN_PCMPEQW128:
19921 16 : case IX86_BUILTIN_PCMPEQD128:
19922 16 : case IX86_BUILTIN_PCMPEQQ:
19923 16 : case IX86_BUILTIN_PCMPEQB256:
19924 16 : case IX86_BUILTIN_PCMPEQW256:
19925 16 : case IX86_BUILTIN_PCMPEQD256:
19926 16 : case IX86_BUILTIN_PCMPEQQ256:
19927 16 : tcode = EQ_EXPR;
19928 16 : goto do_cmp;
19929 :
19930 : case IX86_BUILTIN_PCMPGTB128:
19931 : case IX86_BUILTIN_PCMPGTW128:
19932 : case IX86_BUILTIN_PCMPGTD128:
19933 : case IX86_BUILTIN_PCMPGTQ:
19934 : case IX86_BUILTIN_PCMPGTB256:
19935 : case IX86_BUILTIN_PCMPGTW256:
19936 : case IX86_BUILTIN_PCMPGTD256:
19937 : case IX86_BUILTIN_PCMPGTQ256:
19938 : tcode = GT_EXPR;
19939 :
19940 33 : do_cmp:
19941 33 : gcc_assert (n_args == 2);
19942 33 : arg0 = gimple_call_arg (stmt, 0);
19943 33 : arg1 = gimple_call_arg (stmt, 1);
19944 33 : if (gimple_call_lhs (stmt))
19945 : {
19946 32 : loc = gimple_location (stmt);
19947 32 : tree type = TREE_TYPE (arg0);
19948 32 : tree zero_vec = build_zero_cst (type);
19949 32 : tree minus_one_vec = build_minus_one_cst (type);
19950 32 : tree cmp_type = truth_type_for (type);
19951 32 : tree cmp = gimple_build (&stmts, tcode, cmp_type, arg0, arg1);
19952 32 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
19953 32 : g = gimple_build_assign (gimple_call_lhs (stmt),
19954 : VEC_COND_EXPR, cmp,
19955 : minus_one_vec, zero_vec);
19956 32 : gimple_set_location (g, loc);
19957 32 : gsi_replace (gsi, g, false);
19958 : }
19959 : else
19960 1 : gsi_replace (gsi, gimple_build_nop (), false);
19961 : return true;
19962 :
19963 9297 : case IX86_BUILTIN_PSLLD:
19964 9297 : case IX86_BUILTIN_PSLLD128:
19965 9297 : case IX86_BUILTIN_PSLLD128_MASK:
19966 9297 : case IX86_BUILTIN_PSLLD256:
19967 9297 : case IX86_BUILTIN_PSLLD256_MASK:
19968 9297 : case IX86_BUILTIN_PSLLD512:
19969 9297 : case IX86_BUILTIN_PSLLDI:
19970 9297 : case IX86_BUILTIN_PSLLDI128:
19971 9297 : case IX86_BUILTIN_PSLLDI128_MASK:
19972 9297 : case IX86_BUILTIN_PSLLDI256:
19973 9297 : case IX86_BUILTIN_PSLLDI256_MASK:
19974 9297 : case IX86_BUILTIN_PSLLDI512:
19975 9297 : case IX86_BUILTIN_PSLLQ:
19976 9297 : case IX86_BUILTIN_PSLLQ128:
19977 9297 : case IX86_BUILTIN_PSLLQ128_MASK:
19978 9297 : case IX86_BUILTIN_PSLLQ256:
19979 9297 : case IX86_BUILTIN_PSLLQ256_MASK:
19980 9297 : case IX86_BUILTIN_PSLLQ512:
19981 9297 : case IX86_BUILTIN_PSLLQI:
19982 9297 : case IX86_BUILTIN_PSLLQI128:
19983 9297 : case IX86_BUILTIN_PSLLQI128_MASK:
19984 9297 : case IX86_BUILTIN_PSLLQI256:
19985 9297 : case IX86_BUILTIN_PSLLQI256_MASK:
19986 9297 : case IX86_BUILTIN_PSLLQI512:
19987 9297 : case IX86_BUILTIN_PSLLW:
19988 9297 : case IX86_BUILTIN_PSLLW128:
19989 9297 : case IX86_BUILTIN_PSLLW128_MASK:
19990 9297 : case IX86_BUILTIN_PSLLW256:
19991 9297 : case IX86_BUILTIN_PSLLW256_MASK:
19992 9297 : case IX86_BUILTIN_PSLLW512_MASK:
19993 9297 : case IX86_BUILTIN_PSLLWI:
19994 9297 : case IX86_BUILTIN_PSLLWI128:
19995 9297 : case IX86_BUILTIN_PSLLWI128_MASK:
19996 9297 : case IX86_BUILTIN_PSLLWI256:
19997 9297 : case IX86_BUILTIN_PSLLWI256_MASK:
19998 9297 : case IX86_BUILTIN_PSLLWI512_MASK:
19999 9297 : rcode = ASHIFT;
20000 9297 : is_vshift = false;
20001 9297 : goto do_shift;
20002 6495 : case IX86_BUILTIN_PSRAD:
20003 6495 : case IX86_BUILTIN_PSRAD128:
20004 6495 : case IX86_BUILTIN_PSRAD128_MASK:
20005 6495 : case IX86_BUILTIN_PSRAD256:
20006 6495 : case IX86_BUILTIN_PSRAD256_MASK:
20007 6495 : case IX86_BUILTIN_PSRAD512:
20008 6495 : case IX86_BUILTIN_PSRADI:
20009 6495 : case IX86_BUILTIN_PSRADI128:
20010 6495 : case IX86_BUILTIN_PSRADI128_MASK:
20011 6495 : case IX86_BUILTIN_PSRADI256:
20012 6495 : case IX86_BUILTIN_PSRADI256_MASK:
20013 6495 : case IX86_BUILTIN_PSRADI512:
20014 6495 : case IX86_BUILTIN_PSRAQ128_MASK:
20015 6495 : case IX86_BUILTIN_PSRAQ256_MASK:
20016 6495 : case IX86_BUILTIN_PSRAQ512:
20017 6495 : case IX86_BUILTIN_PSRAQI128_MASK:
20018 6495 : case IX86_BUILTIN_PSRAQI256_MASK:
20019 6495 : case IX86_BUILTIN_PSRAQI512:
20020 6495 : case IX86_BUILTIN_PSRAW:
20021 6495 : case IX86_BUILTIN_PSRAW128:
20022 6495 : case IX86_BUILTIN_PSRAW128_MASK:
20023 6495 : case IX86_BUILTIN_PSRAW256:
20024 6495 : case IX86_BUILTIN_PSRAW256_MASK:
20025 6495 : case IX86_BUILTIN_PSRAW512:
20026 6495 : case IX86_BUILTIN_PSRAWI:
20027 6495 : case IX86_BUILTIN_PSRAWI128:
20028 6495 : case IX86_BUILTIN_PSRAWI128_MASK:
20029 6495 : case IX86_BUILTIN_PSRAWI256:
20030 6495 : case IX86_BUILTIN_PSRAWI256_MASK:
20031 6495 : case IX86_BUILTIN_PSRAWI512:
20032 6495 : rcode = ASHIFTRT;
20033 6495 : is_vshift = false;
20034 6495 : goto do_shift;
20035 7960 : case IX86_BUILTIN_PSRLD:
20036 7960 : case IX86_BUILTIN_PSRLD128:
20037 7960 : case IX86_BUILTIN_PSRLD128_MASK:
20038 7960 : case IX86_BUILTIN_PSRLD256:
20039 7960 : case IX86_BUILTIN_PSRLD256_MASK:
20040 7960 : case IX86_BUILTIN_PSRLD512:
20041 7960 : case IX86_BUILTIN_PSRLDI:
20042 7960 : case IX86_BUILTIN_PSRLDI128:
20043 7960 : case IX86_BUILTIN_PSRLDI128_MASK:
20044 7960 : case IX86_BUILTIN_PSRLDI256:
20045 7960 : case IX86_BUILTIN_PSRLDI256_MASK:
20046 7960 : case IX86_BUILTIN_PSRLDI512:
20047 7960 : case IX86_BUILTIN_PSRLQ:
20048 7960 : case IX86_BUILTIN_PSRLQ128:
20049 7960 : case IX86_BUILTIN_PSRLQ128_MASK:
20050 7960 : case IX86_BUILTIN_PSRLQ256:
20051 7960 : case IX86_BUILTIN_PSRLQ256_MASK:
20052 7960 : case IX86_BUILTIN_PSRLQ512:
20053 7960 : case IX86_BUILTIN_PSRLQI:
20054 7960 : case IX86_BUILTIN_PSRLQI128:
20055 7960 : case IX86_BUILTIN_PSRLQI128_MASK:
20056 7960 : case IX86_BUILTIN_PSRLQI256:
20057 7960 : case IX86_BUILTIN_PSRLQI256_MASK:
20058 7960 : case IX86_BUILTIN_PSRLQI512:
20059 7960 : case IX86_BUILTIN_PSRLW:
20060 7960 : case IX86_BUILTIN_PSRLW128:
20061 7960 : case IX86_BUILTIN_PSRLW128_MASK:
20062 7960 : case IX86_BUILTIN_PSRLW256:
20063 7960 : case IX86_BUILTIN_PSRLW256_MASK:
20064 7960 : case IX86_BUILTIN_PSRLW512:
20065 7960 : case IX86_BUILTIN_PSRLWI:
20066 7960 : case IX86_BUILTIN_PSRLWI128:
20067 7960 : case IX86_BUILTIN_PSRLWI128_MASK:
20068 7960 : case IX86_BUILTIN_PSRLWI256:
20069 7960 : case IX86_BUILTIN_PSRLWI256_MASK:
20070 7960 : case IX86_BUILTIN_PSRLWI512:
20071 7960 : rcode = LSHIFTRT;
20072 7960 : is_vshift = false;
20073 7960 : goto do_shift;
20074 2384 : case IX86_BUILTIN_PSLLVV16HI:
20075 2384 : case IX86_BUILTIN_PSLLVV16SI:
20076 2384 : case IX86_BUILTIN_PSLLVV2DI:
20077 2384 : case IX86_BUILTIN_PSLLVV2DI_MASK:
20078 2384 : case IX86_BUILTIN_PSLLVV32HI:
20079 2384 : case IX86_BUILTIN_PSLLVV4DI:
20080 2384 : case IX86_BUILTIN_PSLLVV4DI_MASK:
20081 2384 : case IX86_BUILTIN_PSLLVV4SI:
20082 2384 : case IX86_BUILTIN_PSLLVV4SI_MASK:
20083 2384 : case IX86_BUILTIN_PSLLVV8DI:
20084 2384 : case IX86_BUILTIN_PSLLVV8HI:
20085 2384 : case IX86_BUILTIN_PSLLVV8SI:
20086 2384 : case IX86_BUILTIN_PSLLVV8SI_MASK:
20087 2384 : rcode = ASHIFT;
20088 2384 : is_vshift = true;
20089 2384 : goto do_shift;
20090 2341 : case IX86_BUILTIN_PSRAVQ128:
20091 2341 : case IX86_BUILTIN_PSRAVQ256:
20092 2341 : case IX86_BUILTIN_PSRAVV16HI:
20093 2341 : case IX86_BUILTIN_PSRAVV16SI:
20094 2341 : case IX86_BUILTIN_PSRAVV32HI:
20095 2341 : case IX86_BUILTIN_PSRAVV4SI:
20096 2341 : case IX86_BUILTIN_PSRAVV4SI_MASK:
20097 2341 : case IX86_BUILTIN_PSRAVV8DI:
20098 2341 : case IX86_BUILTIN_PSRAVV8HI:
20099 2341 : case IX86_BUILTIN_PSRAVV8SI:
20100 2341 : case IX86_BUILTIN_PSRAVV8SI_MASK:
20101 2341 : rcode = ASHIFTRT;
20102 2341 : is_vshift = true;
20103 2341 : goto do_shift;
20104 2380 : case IX86_BUILTIN_PSRLVV16HI:
20105 2380 : case IX86_BUILTIN_PSRLVV16SI:
20106 2380 : case IX86_BUILTIN_PSRLVV2DI:
20107 2380 : case IX86_BUILTIN_PSRLVV2DI_MASK:
20108 2380 : case IX86_BUILTIN_PSRLVV32HI:
20109 2380 : case IX86_BUILTIN_PSRLVV4DI:
20110 2380 : case IX86_BUILTIN_PSRLVV4DI_MASK:
20111 2380 : case IX86_BUILTIN_PSRLVV4SI:
20112 2380 : case IX86_BUILTIN_PSRLVV4SI_MASK:
20113 2380 : case IX86_BUILTIN_PSRLVV8DI:
20114 2380 : case IX86_BUILTIN_PSRLVV8HI:
20115 2380 : case IX86_BUILTIN_PSRLVV8SI:
20116 2380 : case IX86_BUILTIN_PSRLVV8SI_MASK:
20117 2380 : rcode = LSHIFTRT;
20118 2380 : is_vshift = true;
20119 2380 : goto do_shift;
20120 :
20121 30857 : do_shift:
20122 30857 : gcc_assert (n_args >= 2);
20123 30857 : if (!gimple_call_lhs (stmt))
20124 : {
20125 1 : gsi_replace (gsi, gimple_build_nop (), false);
20126 1 : return true;
20127 : }
20128 30856 : arg0 = gimple_call_arg (stmt, 0);
20129 30856 : arg1 = gimple_call_arg (stmt, 1);
20130 30856 : elems = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
20131 : /* For masked shift, only optimize if the mask is all ones. */
20132 30856 : if (n_args > 2
20133 30856 : && !ix86_masked_all_ones (elems, gimple_call_arg (stmt, n_args - 1)))
20134 : break;
20135 16081 : if (is_vshift)
20136 : {
20137 2640 : if (TREE_CODE (arg1) != VECTOR_CST)
20138 : break;
20139 69 : count = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (arg0)));
20140 69 : if (integer_zerop (arg1))
20141 27 : count = 0;
20142 42 : else if (rcode == ASHIFTRT)
20143 : break;
20144 : else
20145 230 : for (unsigned int i = 0; i < VECTOR_CST_NELTS (arg1); ++i)
20146 : {
20147 212 : tree elt = VECTOR_CST_ELT (arg1, i);
20148 212 : if (!wi::neg_p (wi::to_wide (elt))
20149 375 : && wi::to_widest (elt) < count)
20150 16 : return false;
20151 : }
20152 : }
20153 : else
20154 : {
20155 13441 : arg1 = ix86_vector_shift_count (arg1);
20156 13441 : if (!arg1)
20157 : break;
20158 5608 : count = tree_to_uhwi (arg1);
20159 : }
20160 5653 : if (count == 0)
20161 : {
20162 : /* Just return the first argument for shift by 0. */
20163 93 : loc = gimple_location (stmt);
20164 93 : g = gimple_build_assign (gimple_call_lhs (stmt), arg0);
20165 93 : gimple_set_location (g, loc);
20166 93 : gsi_replace (gsi, g, false);
20167 93 : return true;
20168 : }
20169 5560 : if (rcode != ASHIFTRT
20170 5560 : && count >= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (arg0))))
20171 : {
20172 : /* For shift counts equal or greater than precision, except for
20173 : arithmetic right shift the result is zero. */
20174 78 : loc = gimple_location (stmt);
20175 78 : g = gimple_build_assign (gimple_call_lhs (stmt),
20176 78 : build_zero_cst (TREE_TYPE (arg0)));
20177 78 : gimple_set_location (g, loc);
20178 78 : gsi_replace (gsi, g, false);
20179 78 : return true;
20180 : }
20181 : break;
20182 :
20183 531 : case IX86_BUILTIN_SHUFPD512:
20184 531 : case IX86_BUILTIN_SHUFPS512:
20185 531 : case IX86_BUILTIN_SHUFPD:
20186 531 : case IX86_BUILTIN_SHUFPD256:
20187 531 : case IX86_BUILTIN_SHUFPS:
20188 531 : case IX86_BUILTIN_SHUFPS256:
20189 531 : arg0 = gimple_call_arg (stmt, 0);
20190 531 : elems = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
20191 : /* This is masked shuffle. Only optimize if the mask is all ones. */
20192 531 : if (n_args > 3
20193 895 : && !ix86_masked_all_ones (elems,
20194 364 : gimple_call_arg (stmt, n_args - 1)))
20195 : break;
20196 203 : arg2 = gimple_call_arg (stmt, 2);
20197 203 : if (TREE_CODE (arg2) == INTEGER_CST && gimple_call_lhs (stmt))
20198 : {
20199 146 : unsigned HOST_WIDE_INT shuffle_mask = TREE_INT_CST_LOW (arg2);
20200 : /* Check valid imm, refer to gcc.target/i386/testimm-10.c. */
20201 146 : if (shuffle_mask > 255)
20202 : return false;
20203 :
20204 144 : machine_mode imode = GET_MODE_INNER (TYPE_MODE (TREE_TYPE (arg0)));
20205 144 : loc = gimple_location (stmt);
20206 144 : tree itype = (imode == E_DFmode
20207 144 : ? long_long_integer_type_node : integer_type_node);
20208 144 : tree vtype = build_vector_type (itype, elems);
20209 144 : tree_vector_builder elts (vtype, elems, 1);
20210 :
20211 :
20212 : /* Transform integer shuffle_mask to vector perm_mask which
20213 : is used by vec_perm_expr, refer to shuflp[sd]256/512 in sse.md. */
20214 840 : for (unsigned i = 0; i != elems; i++)
20215 : {
20216 696 : unsigned sel_idx;
20217 : /* Imm[1:0](if VL > 128, then use Imm[3:2],Imm[5:4],Imm[7:6])
20218 : provide 2 select controls for each element of the
20219 : destination. */
20220 696 : if (imode == E_DFmode)
20221 240 : sel_idx = (i & 1) * elems + (i & ~1)
20222 240 : + ((shuffle_mask >> i) & 1);
20223 : else
20224 : {
20225 : /* Imm[7:0](if VL > 128, also use Imm[7:0]) provide 4 select
20226 : controls for each element of the destination. */
20227 456 : unsigned j = i % 4;
20228 456 : sel_idx = ((i >> 1) & 1) * elems + (i & ~3)
20229 456 : + ((shuffle_mask >> 2 * j) & 3);
20230 : }
20231 696 : elts.quick_push (build_int_cst (itype, sel_idx));
20232 : }
20233 :
20234 144 : tree perm_mask = elts.build ();
20235 144 : arg1 = gimple_call_arg (stmt, 1);
20236 144 : g = gimple_build_assign (gimple_call_lhs (stmt),
20237 : VEC_PERM_EXPR,
20238 : arg0, arg1, perm_mask);
20239 144 : gimple_set_location (g, loc);
20240 144 : gsi_replace (gsi, g, false);
20241 144 : return true;
20242 144 : }
20243 : // Do not error yet, the constant could be propagated later?
20244 : break;
20245 :
20246 48 : case IX86_BUILTIN_PABSB:
20247 48 : case IX86_BUILTIN_PABSW:
20248 48 : case IX86_BUILTIN_PABSD:
20249 : /* 64-bit vector abs<mode>2 is only supported under TARGET_MMX_WITH_SSE. */
20250 48 : if (!TARGET_MMX_WITH_SSE)
20251 : break;
20252 : /* FALLTHRU. */
20253 2190 : case IX86_BUILTIN_PABSB128:
20254 2190 : case IX86_BUILTIN_PABSB256:
20255 2190 : case IX86_BUILTIN_PABSB512:
20256 2190 : case IX86_BUILTIN_PABSW128:
20257 2190 : case IX86_BUILTIN_PABSW256:
20258 2190 : case IX86_BUILTIN_PABSW512:
20259 2190 : case IX86_BUILTIN_PABSD128:
20260 2190 : case IX86_BUILTIN_PABSD256:
20261 2190 : case IX86_BUILTIN_PABSD512:
20262 2190 : case IX86_BUILTIN_PABSQ128:
20263 2190 : case IX86_BUILTIN_PABSQ256:
20264 2190 : case IX86_BUILTIN_PABSQ512:
20265 2190 : case IX86_BUILTIN_PABSB128_MASK:
20266 2190 : case IX86_BUILTIN_PABSB256_MASK:
20267 2190 : case IX86_BUILTIN_PABSW128_MASK:
20268 2190 : case IX86_BUILTIN_PABSW256_MASK:
20269 2190 : case IX86_BUILTIN_PABSD128_MASK:
20270 2190 : case IX86_BUILTIN_PABSD256_MASK:
20271 2190 : gcc_assert (n_args >= 1);
20272 2190 : if (!gimple_call_lhs (stmt))
20273 : {
20274 1 : gsi_replace (gsi, gimple_build_nop (), false);
20275 1 : return true;
20276 : }
20277 2189 : arg0 = gimple_call_arg (stmt, 0);
20278 2189 : elems = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
20279 : /* For masked ABS, only optimize if the mask is all ones. */
20280 2189 : if (n_args > 1
20281 2189 : && !ix86_masked_all_ones (elems, gimple_call_arg (stmt, n_args - 1)))
20282 : break;
20283 229 : {
20284 229 : tree utype, ures, vce;
20285 229 : utype = unsigned_type_for (TREE_TYPE (arg0));
20286 : /* PABSB/W/D/Q store the unsigned result in dst, use ABSU_EXPR
20287 : instead of ABS_EXPR to handle overflow case(TYPE_MIN). */
20288 229 : ures = gimple_build (&stmts, ABSU_EXPR, utype, arg0);
20289 229 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
20290 229 : loc = gimple_location (stmt);
20291 229 : vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (arg0), ures);
20292 229 : g = gimple_build_assign (gimple_call_lhs (stmt),
20293 : VIEW_CONVERT_EXPR, vce);
20294 229 : gsi_replace (gsi, g, false);
20295 : }
20296 229 : return true;
20297 :
20298 2225 : case IX86_BUILTIN_MINPS:
20299 2225 : case IX86_BUILTIN_MINPD:
20300 2225 : case IX86_BUILTIN_MINPS256:
20301 2225 : case IX86_BUILTIN_MINPD256:
20302 2225 : case IX86_BUILTIN_MINPS512:
20303 2225 : case IX86_BUILTIN_MINPD512:
20304 2225 : case IX86_BUILTIN_MINPS128_MASK:
20305 2225 : case IX86_BUILTIN_MINPD128_MASK:
20306 2225 : case IX86_BUILTIN_MINPS256_MASK:
20307 2225 : case IX86_BUILTIN_MINPD256_MASK:
20308 2225 : case IX86_BUILTIN_MINPH128_MASK:
20309 2225 : case IX86_BUILTIN_MINPH256_MASK:
20310 2225 : case IX86_BUILTIN_MINPH512_MASK:
20311 2225 : tcode = LT_EXPR;
20312 2225 : goto do_minmax;
20313 :
20314 : case IX86_BUILTIN_MAXPS:
20315 : case IX86_BUILTIN_MAXPD:
20316 : case IX86_BUILTIN_MAXPS256:
20317 : case IX86_BUILTIN_MAXPD256:
20318 : case IX86_BUILTIN_MAXPS512:
20319 : case IX86_BUILTIN_MAXPD512:
20320 : case IX86_BUILTIN_MAXPS128_MASK:
20321 : case IX86_BUILTIN_MAXPD128_MASK:
20322 : case IX86_BUILTIN_MAXPS256_MASK:
20323 : case IX86_BUILTIN_MAXPD256_MASK:
20324 : case IX86_BUILTIN_MAXPH128_MASK:
20325 : case IX86_BUILTIN_MAXPH256_MASK:
20326 : case IX86_BUILTIN_MAXPH512_MASK:
20327 : tcode = GT_EXPR;
20328 4435 : do_minmax:
20329 4435 : gcc_assert (n_args >= 2);
20330 : /* Without SSE4.1 we often aren't able to pattern match it back to the
20331 : desired instruction. */
20332 4435 : if (!gimple_call_lhs (stmt) || !optimize || !TARGET_SSE4_1)
20333 : break;
20334 3865 : arg0 = gimple_call_arg (stmt, 0);
20335 3865 : arg1 = gimple_call_arg (stmt, 1);
20336 3865 : elems = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
20337 : /* For masked minmax, only optimize if the mask is all ones. */
20338 3865 : if (n_args > 2
20339 3865 : && !ix86_masked_all_ones (elems, gimple_call_arg (stmt, 3)))
20340 : break;
20341 647 : if (n_args >= 5)
20342 : {
20343 436 : tree arg4 = gimple_call_arg (stmt, 4);
20344 436 : if (!tree_fits_uhwi_p (arg4))
20345 : break;
20346 424 : if (tree_to_uhwi (arg4) == 4)
20347 : /* Ok. */;
20348 416 : else if (tree_to_uhwi (arg4) != 8)
20349 : /* Invalid round argument. */
20350 : break;
20351 416 : else if (HONOR_NANS (arg0))
20352 : /* Lowering to comparison would raise exceptions which
20353 : shouldn't be raised. */
20354 : break;
20355 : }
20356 219 : {
20357 219 : tree type = truth_type_for (TREE_TYPE (arg0));
20358 219 : tree cmpres = gimple_build (&stmts, tcode, type, arg0, arg1);
20359 219 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
20360 219 : g = gimple_build_assign (gimple_call_lhs (stmt),
20361 : VEC_COND_EXPR, cmpres, arg0, arg1);
20362 219 : gsi_replace (gsi, g, false);
20363 : }
20364 219 : return true;
20365 :
20366 : default:
20367 : break;
20368 : }
20369 :
20370 : return false;
20371 : }
20372 :
20373 : /* Handler for an SVML-style interface to
20374 : a library with vectorized intrinsics. */
20375 :
20376 : tree
20377 10 : ix86_veclibabi_svml (combined_fn fn, tree type_out, tree type_in)
20378 : {
20379 10 : char name[20];
20380 10 : tree fntype, new_fndecl, args;
20381 10 : unsigned arity;
20382 10 : const char *bname;
20383 10 : machine_mode el_mode, in_mode;
20384 10 : int n, in_n;
20385 :
20386 : /* The SVML is suitable for unsafe math only. */
20387 10 : if (!flag_unsafe_math_optimizations)
20388 : return NULL_TREE;
20389 :
20390 10 : el_mode = TYPE_MODE (TREE_TYPE (type_out));
20391 10 : n = TYPE_VECTOR_SUBPARTS (type_out);
20392 10 : in_mode = TYPE_MODE (TREE_TYPE (type_in));
20393 10 : in_n = TYPE_VECTOR_SUBPARTS (type_in);
20394 10 : if (el_mode != in_mode
20395 10 : || n != in_n)
20396 : return NULL_TREE;
20397 :
20398 10 : switch (fn)
20399 : {
20400 10 : CASE_CFN_EXP:
20401 10 : CASE_CFN_LOG:
20402 10 : CASE_CFN_LOG10:
20403 10 : CASE_CFN_POW:
20404 10 : CASE_CFN_TANH:
20405 10 : CASE_CFN_TAN:
20406 10 : CASE_CFN_ATAN:
20407 10 : CASE_CFN_ATAN2:
20408 10 : CASE_CFN_ATANH:
20409 10 : CASE_CFN_CBRT:
20410 10 : CASE_CFN_SINH:
20411 10 : CASE_CFN_SIN:
20412 10 : CASE_CFN_ASINH:
20413 10 : CASE_CFN_ASIN:
20414 10 : CASE_CFN_COSH:
20415 10 : CASE_CFN_COS:
20416 10 : CASE_CFN_ACOSH:
20417 10 : CASE_CFN_ACOS:
20418 10 : if ((el_mode != DFmode || n != 2)
20419 8 : && (el_mode != SFmode || n != 4))
20420 : return NULL_TREE;
20421 6 : break;
20422 :
20423 : default:
20424 : return NULL_TREE;
20425 : }
20426 :
20427 6 : tree fndecl = mathfn_built_in (el_mode == DFmode
20428 : ? double_type_node : float_type_node, fn);
20429 6 : bname = IDENTIFIER_POINTER (DECL_NAME (fndecl));
20430 :
20431 6 : if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_LOGF)
20432 2 : strcpy (name, "vmlsLn4");
20433 4 : else if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_LOG)
20434 0 : strcpy (name, "vmldLn2");
20435 4 : else if (n == 4)
20436 : {
20437 2 : sprintf (name, "vmls%s", bname+10);
20438 2 : name[strlen (name)-1] = '4';
20439 : }
20440 : else
20441 2 : sprintf (name, "vmld%s2", bname+10);
20442 :
20443 : /* Convert to uppercase. */
20444 6 : name[4] &= ~0x20;
20445 :
20446 6 : arity = 0;
20447 6 : for (args = DECL_ARGUMENTS (fndecl); args; args = TREE_CHAIN (args))
20448 0 : arity++;
20449 :
20450 6 : if (arity == 1)
20451 0 : fntype = build_function_type_list (type_out, type_in, NULL);
20452 : else
20453 6 : fntype = build_function_type_list (type_out, type_in, type_in, NULL);
20454 :
20455 : /* Build a function declaration for the vectorized function. */
20456 6 : new_fndecl = build_decl (BUILTINS_LOCATION,
20457 : FUNCTION_DECL, get_identifier (name), fntype);
20458 6 : TREE_PUBLIC (new_fndecl) = 1;
20459 6 : DECL_EXTERNAL (new_fndecl) = 1;
20460 6 : DECL_IS_NOVOPS (new_fndecl) = 1;
20461 6 : TREE_READONLY (new_fndecl) = 1;
20462 :
20463 6 : return new_fndecl;
20464 : }
20465 :
20466 : /* Handler for an ACML-style interface to
20467 : a library with vectorized intrinsics. */
20468 :
20469 : tree
20470 3 : ix86_veclibabi_acml (combined_fn fn, tree type_out, tree type_in)
20471 : {
20472 3 : char name[20] = "__vr.._";
20473 3 : tree fntype, new_fndecl, args;
20474 3 : unsigned arity;
20475 3 : const char *bname;
20476 3 : machine_mode el_mode, in_mode;
20477 3 : int n, in_n;
20478 :
20479 : /* The ACML is 64bits only and suitable for unsafe math only as
20480 : it does not correctly support parts of IEEE with the required
20481 : precision such as denormals. */
20482 3 : if (!TARGET_64BIT
20483 3 : || !flag_unsafe_math_optimizations)
20484 : return NULL_TREE;
20485 :
20486 3 : el_mode = TYPE_MODE (TREE_TYPE (type_out));
20487 3 : n = TYPE_VECTOR_SUBPARTS (type_out);
20488 3 : in_mode = TYPE_MODE (TREE_TYPE (type_in));
20489 3 : in_n = TYPE_VECTOR_SUBPARTS (type_in);
20490 3 : if (el_mode != in_mode
20491 3 : || n != in_n)
20492 : return NULL_TREE;
20493 :
20494 3 : switch (fn)
20495 : {
20496 3 : CASE_CFN_SIN:
20497 3 : CASE_CFN_COS:
20498 3 : CASE_CFN_EXP:
20499 3 : CASE_CFN_LOG:
20500 3 : CASE_CFN_LOG2:
20501 3 : CASE_CFN_LOG10:
20502 3 : if (el_mode == DFmode && n == 2)
20503 : {
20504 3 : name[4] = 'd';
20505 3 : name[5] = '2';
20506 : }
20507 0 : else if (el_mode == SFmode && n == 4)
20508 : {
20509 0 : name[4] = 's';
20510 0 : name[5] = '4';
20511 : }
20512 : else
20513 : return NULL_TREE;
20514 3 : break;
20515 :
20516 : default:
20517 : return NULL_TREE;
20518 : }
20519 :
20520 3 : tree fndecl = mathfn_built_in (el_mode == DFmode
20521 : ? double_type_node : float_type_node, fn);
20522 3 : bname = IDENTIFIER_POINTER (DECL_NAME (fndecl));
20523 3 : sprintf (name + 7, "%s", bname+10);
20524 :
20525 3 : arity = 0;
20526 3 : for (args = DECL_ARGUMENTS (fndecl); args; args = TREE_CHAIN (args))
20527 0 : arity++;
20528 :
20529 3 : if (arity == 1)
20530 0 : fntype = build_function_type_list (type_out, type_in, NULL);
20531 : else
20532 3 : fntype = build_function_type_list (type_out, type_in, type_in, NULL);
20533 :
20534 : /* Build a function declaration for the vectorized function. */
20535 3 : new_fndecl = build_decl (BUILTINS_LOCATION,
20536 : FUNCTION_DECL, get_identifier (name), fntype);
20537 3 : TREE_PUBLIC (new_fndecl) = 1;
20538 3 : DECL_EXTERNAL (new_fndecl) = 1;
20539 3 : DECL_IS_NOVOPS (new_fndecl) = 1;
20540 3 : TREE_READONLY (new_fndecl) = 1;
20541 :
20542 3 : return new_fndecl;
20543 : }
20544 :
20545 : /* Handler for an AOCL-LibM-style interface to
20546 : a library with vectorized intrinsics. */
20547 :
20548 : tree
20549 386 : ix86_veclibabi_aocl (combined_fn fn, tree type_out, tree type_in)
20550 : {
20551 386 : char name[20] = "amd_vr";
20552 386 : int name_len = 6;
20553 386 : tree fntype, new_fndecl, args;
20554 386 : unsigned arity;
20555 386 : const char *bname;
20556 386 : machine_mode el_mode, in_mode;
20557 386 : int n, in_n;
20558 :
20559 : /* AOCL-LibM is 64bits only. It is also only suitable for unsafe math only
20560 : as it trades off some accuracy for increased performance. */
20561 386 : if (!TARGET_64BIT
20562 386 : || !flag_unsafe_math_optimizations)
20563 : return NULL_TREE;
20564 :
20565 386 : el_mode = TYPE_MODE (TREE_TYPE (type_out));
20566 386 : n = TYPE_VECTOR_SUBPARTS (type_out);
20567 386 : in_mode = TYPE_MODE (TREE_TYPE (type_in));
20568 386 : in_n = TYPE_VECTOR_SUBPARTS (type_in);
20569 386 : if (el_mode != in_mode
20570 386 : || n != in_n)
20571 : return NULL_TREE;
20572 :
20573 386 : gcc_checking_assert (n > 0);
20574 :
20575 : /* Decide whether there exists a function for the combination of FN, the mode
20576 : and the vector width. Return early if it doesn't. */
20577 :
20578 386 : if (el_mode != DFmode && el_mode != SFmode)
20579 : return NULL_TREE;
20580 :
20581 : /* Supported vector widths for given FN and single/double precision. Zeros
20582 : are used to fill out unused positions in the arrays. */
20583 386 : static const int supported_n[][2][3] = {
20584 : /* Single prec. , Double prec. */
20585 : { { 16, 0, 0 }, { 2, 4, 8 } }, /* TAN. */
20586 : { { 4, 8, 16 }, { 2, 4, 8 } }, /* EXP. */
20587 : { { 4, 8, 16 }, { 2, 4, 8 } }, /* EXP2. */
20588 : { { 4, 8, 16 }, { 2, 4, 8 } }, /* LOG. */
20589 : { { 4, 8, 16 }, { 2, 4, 8 } }, /* LOG2. */
20590 : { { 4, 8, 16 }, { 2, 4, 8 } }, /* COS. */
20591 : { { 4, 8, 16 }, { 2, 4, 8 } }, /* SIN. */
20592 : { { 4, 8, 16 }, { 2, 4, 8 } }, /* POW. */
20593 : { { 4, 8, 16 }, { 2, 4, 8 } }, /* ERF. */
20594 : { { 4, 8, 16 }, { 2, 8, 0 } }, /* ATAN. */
20595 : { { 4, 8, 16 }, { 2, 0, 0 } }, /* LOG10. */
20596 : { { 4, 0, 0 }, { 2, 0, 0 } }, /* EXP10. */
20597 : { { 4, 0, 0 }, { 2, 0, 0 } }, /* LOG1P. */
20598 : { { 4, 8, 16 }, { 8, 0, 0 } }, /* ASIN. */
20599 : { { 4, 16, 0 }, { 0, 0, 0 } }, /* ACOS. */
20600 : { { 4, 8, 16 }, { 0, 0, 0 } }, /* TANH. */
20601 : { { 4, 0, 0 }, { 0, 0, 0 } }, /* EXPM1. */
20602 : { { 4, 8, 0 }, { 0, 0, 0 } }, /* COSH. */
20603 : };
20604 :
20605 : /* We cannot simply index the supported_n array with FN since multiple FNs
20606 : may correspond to a single operation (see the definitions of these
20607 : CASE_CFN_* macros). */
20608 386 : int i;
20609 386 : switch (fn)
20610 : {
20611 : CASE_CFN_TAN : i = 0; break;
20612 28 : CASE_CFN_EXP : i = 1; break;
20613 28 : CASE_CFN_EXP2 : i = 2; break;
20614 28 : CASE_CFN_LOG : i = 3; break;
20615 28 : CASE_CFN_LOG2 : i = 4; break;
20616 28 : CASE_CFN_COS : i = 5; break;
20617 28 : CASE_CFN_SIN : i = 6; break;
20618 28 : CASE_CFN_POW : i = 7; break;
20619 28 : CASE_CFN_ERF : i = 8; break;
20620 25 : CASE_CFN_ATAN : i = 9; break;
20621 20 : CASE_CFN_LOG10 : i = 10; break;
20622 10 : CASE_CFN_EXP10 : i = 11; break;
20623 10 : CASE_CFN_LOG1P : i = 12; break;
20624 24 : CASE_CFN_ASIN : i = 13; break;
20625 14 : CASE_CFN_ACOS : i = 14; break;
20626 18 : CASE_CFN_TANH : i = 15; break;
20627 9 : CASE_CFN_EXPM1 : i = 16; break;
20628 14 : CASE_CFN_COSH : i = 17; break;
20629 : default: return NULL_TREE;
20630 : }
20631 :
20632 386 : int j = el_mode == DFmode;
20633 386 : bool n_is_supported = false;
20634 976 : for (unsigned k = 0; k < 3; k++)
20635 857 : if (supported_n[i][j][k] == n)
20636 : {
20637 : n_is_supported = true;
20638 : break;
20639 : }
20640 386 : if (!n_is_supported)
20641 : return NULL_TREE;
20642 :
20643 : /* Append the precision and the vector width to the function name we are
20644 : constructing. */
20645 267 : name[name_len++] = el_mode == DFmode ? 'd' : 's';
20646 267 : switch (n)
20647 : {
20648 214 : case 2:
20649 214 : case 4:
20650 214 : case 8:
20651 214 : name[name_len++] = '0' + n;
20652 214 : break;
20653 53 : case 16:
20654 53 : name[name_len++] = '1';
20655 53 : name[name_len++] = '6';
20656 53 : break;
20657 0 : default:
20658 0 : gcc_unreachable ();
20659 : }
20660 267 : name[name_len++] = '_';
20661 :
20662 : /* Append the operation name (steal it from the name of a builtin). */
20663 267 : tree fndecl = mathfn_built_in (el_mode == DFmode
20664 : ? double_type_node : float_type_node, fn);
20665 267 : bname = IDENTIFIER_POINTER (DECL_NAME (fndecl));
20666 267 : sprintf (name + name_len, "%s", bname + 10);
20667 :
20668 267 : arity = 0;
20669 267 : for (args = DECL_ARGUMENTS (fndecl); args; args = TREE_CHAIN (args))
20670 0 : arity++;
20671 :
20672 267 : if (arity == 1)
20673 0 : fntype = build_function_type_list (type_out, type_in, NULL);
20674 : else
20675 267 : fntype = build_function_type_list (type_out, type_in, type_in, NULL);
20676 :
20677 : /* Build a function declaration for the vectorized function. */
20678 267 : new_fndecl = build_decl (BUILTINS_LOCATION,
20679 : FUNCTION_DECL, get_identifier (name), fntype);
20680 267 : TREE_PUBLIC (new_fndecl) = 1;
20681 267 : DECL_EXTERNAL (new_fndecl) = 1;
20682 267 : TREE_READONLY (new_fndecl) = 1;
20683 :
20684 267 : return new_fndecl;
20685 : }
20686 :
20687 : /* Returns a decl of a function that implements scatter store with
20688 : register type VECTYPE and index type INDEX_TYPE and SCALE.
20689 : Return NULL_TREE if it is not available. */
20690 :
20691 : static tree
20692 133037 : ix86_vectorize_builtin_scatter (const_tree vectype,
20693 : const_tree index_type, int scale)
20694 : {
20695 133037 : bool si;
20696 133037 : enum ix86_builtins code;
20697 :
20698 133037 : if (!TARGET_AVX512F)
20699 : return NULL_TREE;
20700 :
20701 4193 : if (known_eq (TYPE_VECTOR_SUBPARTS (vectype), 2u)
20702 7367 : ? !TARGET_USE_SCATTER_2PARTS
20703 7367 : : (known_eq (TYPE_VECTOR_SUBPARTS (vectype), 4u)
20704 3174 : ? !TARGET_USE_SCATTER_4PARTS
20705 2057 : : !TARGET_USE_SCATTER_8PARTS))
20706 : return NULL_TREE;
20707 :
20708 4193 : if ((TREE_CODE (index_type) != INTEGER_TYPE
20709 463 : && !POINTER_TYPE_P (index_type))
20710 4656 : || (TYPE_MODE (index_type) != SImode
20711 1777 : && TYPE_MODE (index_type) != DImode))
20712 0 : return NULL_TREE;
20713 :
20714 4423 : if (TYPE_PRECISION (index_type) > POINTER_SIZE)
20715 : return NULL_TREE;
20716 :
20717 : /* v*scatter* insn sign extends index to pointer mode. */
20718 4193 : if (TYPE_PRECISION (index_type) < POINTER_SIZE
20719 4193 : && TYPE_UNSIGNED (index_type))
20720 : return NULL_TREE;
20721 :
20722 : /* Scale can be 1, 2, 4 or 8. */
20723 4193 : if (scale <= 0
20724 4193 : || scale > 8
20725 4177 : || (scale & (scale - 1)) != 0)
20726 : return NULL_TREE;
20727 :
20728 4177 : si = TYPE_MODE (index_type) == SImode;
20729 4177 : switch (TYPE_MODE (vectype))
20730 : {
20731 169 : case E_V8DFmode:
20732 169 : code = si ? IX86_BUILTIN_SCATTERALTSIV8DF : IX86_BUILTIN_SCATTERDIV8DF;
20733 : break;
20734 104 : case E_V8DImode:
20735 104 : code = si ? IX86_BUILTIN_SCATTERALTSIV8DI : IX86_BUILTIN_SCATTERDIV8DI;
20736 : break;
20737 177 : case E_V16SFmode:
20738 177 : code = si ? IX86_BUILTIN_SCATTERSIV16SF : IX86_BUILTIN_SCATTERALTDIV16SF;
20739 : break;
20740 257 : case E_V16SImode:
20741 257 : code = si ? IX86_BUILTIN_SCATTERSIV16SI : IX86_BUILTIN_SCATTERALTDIV16SI;
20742 : break;
20743 206 : case E_V4DFmode:
20744 206 : if (TARGET_AVX512VL)
20745 34 : code = si ? IX86_BUILTIN_SCATTERALTSIV4DF : IX86_BUILTIN_SCATTERDIV4DF;
20746 : else
20747 : return NULL_TREE;
20748 : break;
20749 142 : case E_V4DImode:
20750 142 : if (TARGET_AVX512VL)
20751 34 : code = si ? IX86_BUILTIN_SCATTERALTSIV4DI : IX86_BUILTIN_SCATTERDIV4DI;
20752 : else
20753 : return NULL_TREE;
20754 : break;
20755 248 : case E_V8SFmode:
20756 248 : if (TARGET_AVX512VL)
20757 40 : code = si ? IX86_BUILTIN_SCATTERSIV8SF : IX86_BUILTIN_SCATTERALTDIV8SF;
20758 : else
20759 : return NULL_TREE;
20760 : break;
20761 268 : case E_V8SImode:
20762 268 : if (TARGET_AVX512VL)
20763 82 : code = si ? IX86_BUILTIN_SCATTERSIV8SI : IX86_BUILTIN_SCATTERALTDIV8SI;
20764 : else
20765 : return NULL_TREE;
20766 : break;
20767 251 : case E_V2DFmode:
20768 251 : if (TARGET_AVX512VL)
20769 94 : code = si ? IX86_BUILTIN_SCATTERALTSIV2DF : IX86_BUILTIN_SCATTERDIV2DF;
20770 : else
20771 : return NULL_TREE;
20772 : break;
20773 196 : case E_V2DImode:
20774 196 : if (TARGET_AVX512VL)
20775 94 : code = si ? IX86_BUILTIN_SCATTERALTSIV2DI : IX86_BUILTIN_SCATTERDIV2DI;
20776 : else
20777 : return NULL_TREE;
20778 : break;
20779 301 : case E_V4SFmode:
20780 301 : if (TARGET_AVX512VL)
20781 96 : code = si ? IX86_BUILTIN_SCATTERSIV4SF : IX86_BUILTIN_SCATTERALTDIV4SF;
20782 : else
20783 : return NULL_TREE;
20784 : break;
20785 324 : case E_V4SImode:
20786 324 : if (TARGET_AVX512VL)
20787 138 : code = si ? IX86_BUILTIN_SCATTERSIV4SI : IX86_BUILTIN_SCATTERALTDIV4SI;
20788 : else
20789 : return NULL_TREE;
20790 : break;
20791 : default:
20792 : return NULL_TREE;
20793 : }
20794 :
20795 1319 : return get_ix86_builtin (code);
20796 : }
20797 :
20798 : /* Return true if it is safe to use the rsqrt optabs to optimize
20799 : 1.0/sqrt. */
20800 :
20801 : static bool
20802 66 : use_rsqrt_p (machine_mode mode)
20803 : {
20804 66 : return ((mode == HFmode
20805 42 : || (TARGET_SSE && TARGET_SSE_MATH))
20806 66 : && flag_finite_math_only
20807 65 : && !flag_trapping_math
20808 119 : && flag_unsafe_math_optimizations);
20809 : }
20810 :
20811 : /* Helper for avx_vpermilps256_operand et al. This is also used by
20812 : the expansion functions to turn the parallel back into a mask.
20813 : The return value is 0 for no match and the imm8+1 for a match. */
20814 :
20815 : int
20816 64196 : avx_vpermilp_parallel (rtx par, machine_mode mode)
20817 : {
20818 64196 : unsigned i, nelt = GET_MODE_NUNITS (mode);
20819 64196 : unsigned mask = 0;
20820 64196 : unsigned char ipar[16] = {}; /* Silence -Wuninitialized warning. */
20821 :
20822 64196 : if (XVECLEN (par, 0) != (int) nelt)
20823 : return 0;
20824 :
20825 : /* Validate that all of the elements are constants, and not totally
20826 : out of range. Copy the data into an integral array to make the
20827 : subsequent checks easier. */
20828 312544 : for (i = 0; i < nelt; ++i)
20829 : {
20830 248348 : rtx er = XVECEXP (par, 0, i);
20831 248348 : unsigned HOST_WIDE_INT ei;
20832 :
20833 248348 : if (!CONST_INT_P (er))
20834 : return 0;
20835 248348 : ei = INTVAL (er);
20836 248348 : if (ei >= nelt)
20837 : return 0;
20838 248348 : ipar[i] = ei;
20839 : }
20840 :
20841 64196 : switch (mode)
20842 : {
20843 : case E_V8DFmode:
20844 : case E_V8DImode:
20845 : /* In the 512-bit DFmode case, we can only move elements within
20846 : a 128-bit lane. First fill the second part of the mask,
20847 : then fallthru. */
20848 4762 : for (i = 4; i < 6; ++i)
20849 : {
20850 3305 : if (!IN_RANGE (ipar[i], 4, 5))
20851 : return 0;
20852 3080 : mask |= (ipar[i] - 4) << i;
20853 : }
20854 3519 : for (i = 6; i < 8; ++i)
20855 : {
20856 2488 : if (!IN_RANGE (ipar[i], 6, 7))
20857 : return 0;
20858 2062 : mask |= (ipar[i] - 6) << i;
20859 : }
20860 : /* FALLTHRU */
20861 :
20862 : case E_V4DFmode:
20863 : case E_V4DImode:
20864 : /* In the 256-bit DFmode case, we can only move elements within
20865 : a 128-bit lane. */
20866 46320 : for (i = 0; i < 2; ++i)
20867 : {
20868 38943 : if (!IN_RANGE (ipar[i], 0, 1))
20869 : return 0;
20870 25966 : mask |= ipar[i] << i;
20871 : }
20872 19441 : for (i = 2; i < 4; ++i)
20873 : {
20874 13414 : if (!IN_RANGE (ipar[i], 2, 3))
20875 : return 0;
20876 12064 : mask |= (ipar[i] - 2) << i;
20877 : }
20878 : break;
20879 :
20880 : case E_V16SFmode:
20881 : case E_V16SImode:
20882 : /* In 512 bit SFmode case, permutation in the upper 256 bits
20883 : must mirror the permutation in the lower 256-bits. */
20884 3652 : for (i = 0; i < 8; ++i)
20885 3256 : if (ipar[i] + 8 != ipar[i + 8])
20886 : return 0;
20887 : /* FALLTHRU */
20888 :
20889 : case E_V8SFmode:
20890 : case E_V8SImode:
20891 : /* In 256 bit SFmode case, we have full freedom of
20892 : movement within the low 128-bit lane, but the high 128-bit
20893 : lane must mirror the exact same pattern. */
20894 33640 : for (i = 0; i < 4; ++i)
20895 28529 : if (ipar[i] + 4 != ipar[i + 4])
20896 : return 0;
20897 : nelt = 4;
20898 : /* FALLTHRU */
20899 :
20900 38550 : case E_V2DFmode:
20901 38550 : case E_V2DImode:
20902 38550 : case E_V4SFmode:
20903 38550 : case E_V4SImode:
20904 : /* In the 128-bit case, we've full freedom in the placement of
20905 : the elements from the source operand. */
20906 134722 : for (i = 0; i < nelt; ++i)
20907 96172 : mask |= ipar[i] << (i * (nelt / 2));
20908 : break;
20909 :
20910 0 : default:
20911 0 : gcc_unreachable ();
20912 : }
20913 :
20914 : /* Make sure success has a non-zero value by adding one. */
20915 44577 : return mask + 1;
20916 : }
20917 :
20918 : /* Helper for avx_vperm2f128_v4df_operand et al. This is also used by
20919 : the expansion functions to turn the parallel back into a mask.
20920 : The return value is 0 for no match and the imm8+1 for a match. */
20921 :
20922 : int
20923 42818 : avx_vperm2f128_parallel (rtx par, machine_mode mode)
20924 : {
20925 42818 : unsigned i, nelt = GET_MODE_NUNITS (mode), nelt2 = nelt / 2;
20926 42818 : unsigned mask = 0;
20927 42818 : unsigned char ipar[8] = {}; /* Silence -Wuninitialized warning. */
20928 :
20929 42818 : if (XVECLEN (par, 0) != (int) nelt)
20930 : return 0;
20931 :
20932 : /* Validate that all of the elements are constants, and not totally
20933 : out of range. Copy the data into an integral array to make the
20934 : subsequent checks easier. */
20935 344490 : for (i = 0; i < nelt; ++i)
20936 : {
20937 301672 : rtx er = XVECEXP (par, 0, i);
20938 301672 : unsigned HOST_WIDE_INT ei;
20939 :
20940 301672 : if (!CONST_INT_P (er))
20941 : return 0;
20942 301672 : ei = INTVAL (er);
20943 301672 : if (ei >= 2 * nelt)
20944 : return 0;
20945 301672 : ipar[i] = ei;
20946 : }
20947 :
20948 : /* Validate that the halves of the permute are halves. */
20949 82729 : for (i = 0; i < nelt2 - 1; ++i)
20950 66550 : if (ipar[i] + 1 != ipar[i + 1])
20951 : return 0;
20952 51152 : for (i = nelt2; i < nelt - 1; ++i)
20953 35625 : if (ipar[i] + 1 != ipar[i + 1])
20954 : return 0;
20955 :
20956 : /* Reconstruct the mask. */
20957 46485 : for (i = 0; i < 2; ++i)
20958 : {
20959 31008 : unsigned e = ipar[i * nelt2];
20960 31008 : if (e % nelt2)
20961 : return 0;
20962 30958 : e /= nelt2;
20963 30958 : mask |= e << (i * 4);
20964 : }
20965 :
20966 : /* Make sure success has a non-zero value by adding one. */
20967 15477 : return mask + 1;
20968 : }
20969 :
20970 : /* Return a mask of VPTERNLOG operands that do not affect output. */
20971 :
20972 : int
20973 2431 : vpternlog_redundant_operand_mask (rtx pternlog_imm)
20974 : {
20975 2431 : int mask = 0;
20976 2431 : int imm8 = INTVAL (pternlog_imm);
20977 :
20978 2431 : if (((imm8 >> 4) & 0x0F) == (imm8 & 0x0F))
20979 6 : mask |= 1;
20980 2431 : if (((imm8 >> 2) & 0x33) == (imm8 & 0x33))
20981 6 : mask |= 2;
20982 2431 : if (((imm8 >> 1) & 0x55) == (imm8 & 0x55))
20983 155 : mask |= 4;
20984 :
20985 2431 : return mask;
20986 : }
20987 :
20988 : /* Eliminate false dependencies on operands that do not affect output
20989 : by substituting other operands of a VPTERNLOG. */
20990 :
20991 : void
20992 81 : substitute_vpternlog_operands (rtx *operands)
20993 : {
20994 81 : int mask = vpternlog_redundant_operand_mask (operands[4]);
20995 :
20996 81 : if (mask & 1) /* The first operand is redundant. */
20997 2 : operands[1] = operands[2];
20998 :
20999 81 : if (mask & 2) /* The second operand is redundant. */
21000 2 : operands[2] = operands[1];
21001 :
21002 81 : if (mask & 4) /* The third operand is redundant. */
21003 77 : operands[3] = operands[1];
21004 4 : else if (REG_P (operands[3]))
21005 : {
21006 0 : if (mask & 1)
21007 0 : operands[1] = operands[3];
21008 0 : if (mask & 2)
21009 0 : operands[2] = operands[3];
21010 : }
21011 81 : }
21012 :
21013 : /* Return a register priority for hard reg REGNO. */
21014 : static int
21015 59519954 : ix86_register_priority (int hard_regno)
21016 : {
21017 : /* ebp and r13 as the base always wants a displacement, r12 as the
21018 : base always wants an index. So discourage their usage in an
21019 : address. */
21020 59519954 : if (hard_regno == R12_REG || hard_regno == R13_REG)
21021 : return 0;
21022 54917803 : if (hard_regno == BP_REG)
21023 : return 1;
21024 : /* New x86-64 int registers result in bigger code size. Discourage them. */
21025 52984800 : if (REX_INT_REGNO_P (hard_regno))
21026 : return 2;
21027 36065613 : if (REX2_INT_REGNO_P (hard_regno))
21028 : return 2;
21029 : /* New x86-64 SSE registers result in bigger code size. Discourage them. */
21030 36063091 : if (REX_SSE_REGNO_P (hard_regno))
21031 : return 2;
21032 29776884 : if (EXT_REX_SSE_REGNO_P (hard_regno))
21033 : return 1;
21034 : /* Usage of AX register results in smaller code. Prefer it. */
21035 29499828 : if (hard_regno == AX_REG)
21036 3917602 : return 4;
21037 : return 3;
21038 : }
21039 :
21040 : /* Implement TARGET_PREFERRED_RELOAD_CLASS.
21041 :
21042 : Put float CONST_DOUBLE in the constant pool instead of fp regs.
21043 : QImode must go into class Q_REGS.
21044 : Narrow ALL_REGS to GENERAL_REGS. This supports allowing movsf and
21045 : movdf to do mem-to-mem moves through integer regs. */
21046 :
21047 : static reg_class_t
21048 553889752 : ix86_preferred_reload_class (rtx x, reg_class_t regclass)
21049 : {
21050 553889752 : machine_mode mode = GET_MODE (x);
21051 :
21052 : /* We're only allowed to return a subclass of CLASS. Many of the
21053 : following checks fail for NO_REGS, so eliminate that early. */
21054 553889752 : if (regclass == NO_REGS)
21055 : return NO_REGS;
21056 :
21057 : /* All classes can load zeros. */
21058 553015067 : if (x == CONST0_RTX (mode))
21059 : return regclass;
21060 :
21061 : /* Force constants into memory if we are loading a (nonzero) constant into
21062 : an MMX, SSE or MASK register. This is because there are no MMX/SSE/MASK
21063 : instructions to load from a constant. */
21064 527839508 : if (CONSTANT_P (x)
21065 527839508 : && (MAYBE_MMX_CLASS_P (regclass)
21066 153975018 : || MAYBE_SSE_CLASS_P (regclass)
21067 123317332 : || MAYBE_MASK_CLASS_P (regclass)))
21068 30791199 : return NO_REGS;
21069 :
21070 : /* Floating-point constants need more complex checks. */
21071 497048309 : if (CONST_DOUBLE_P (x))
21072 : {
21073 : /* General regs can load everything. */
21074 311108 : if (INTEGER_CLASS_P (regclass))
21075 : return regclass;
21076 :
21077 : /* Floats can load 0 and 1 plus some others. Note that we eliminated
21078 : zero above. We only want to wind up preferring 80387 registers if
21079 : we plan on doing computation with them. */
21080 183343 : if (IS_STACK_MODE (mode)
21081 241831 : && standard_80387_constant_p (x) > 0)
21082 : {
21083 : /* Limit class to FP regs. */
21084 40476 : if (FLOAT_CLASS_P (regclass))
21085 : return FLOAT_REGS;
21086 : }
21087 :
21088 142867 : return NO_REGS;
21089 : }
21090 :
21091 : /* Prefer SSE if we can use them for math. Also allow integer regs
21092 : when moves between register units are cheap. */
21093 496737201 : if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
21094 : {
21095 31176825 : if (TARGET_INTER_UNIT_MOVES_FROM_VEC
21096 31161912 : && TARGET_INTER_UNIT_MOVES_TO_VEC
21097 93491383 : && GET_MODE_SIZE (mode) <= GET_MODE_SIZE (word_mode))
21098 31019796 : return INT_SSE_CLASS_P (regclass) ? regclass : NO_REGS;
21099 : else
21100 157029 : return SSE_CLASS_P (regclass) ? regclass : NO_REGS;
21101 : }
21102 :
21103 : /* Generally when we see PLUS here, it's the function invariant
21104 : (plus soft-fp const_int). Which can only be computed into general
21105 : regs. */
21106 465560376 : if (GET_CODE (x) == PLUS)
21107 2024181 : return INTEGER_CLASS_P (regclass) ? regclass : NO_REGS;
21108 :
21109 : /* QImode constants are easy to load, but non-constant QImode data
21110 : must go into Q_REGS or ALL_MASK_REGS. */
21111 463536195 : if (GET_MODE (x) == QImode && !CONSTANT_P (x))
21112 : {
21113 24786422 : if (Q_CLASS_P (regclass))
21114 : return regclass;
21115 20021387 : else if (reg_class_subset_p (Q_REGS, regclass))
21116 : return Q_REGS;
21117 55309 : else if (MASK_CLASS_P (regclass))
21118 : return regclass;
21119 : else
21120 : return NO_REGS;
21121 : }
21122 :
21123 : return regclass;
21124 : }
21125 :
21126 : /* Discourage putting floating-point values in SSE registers unless
21127 : SSE math is being used, and likewise for the 387 registers. */
21128 : static reg_class_t
21129 75524541 : ix86_preferred_output_reload_class (rtx x, reg_class_t regclass)
21130 : {
21131 : /* Restrict the output reload class to the register bank that we are doing
21132 : math on. If we would like not to return a subset of CLASS, reject this
21133 : alternative: if reload cannot do this, it will still use its choice. */
21134 75524541 : machine_mode mode = GET_MODE (x);
21135 75524541 : if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
21136 7248581 : return MAYBE_SSE_CLASS_P (regclass) ? ALL_SSE_REGS : NO_REGS;
21137 :
21138 68275960 : if (IS_STACK_MODE (mode))
21139 212178 : return FLOAT_CLASS_P (regclass) ? regclass : NO_REGS;
21140 :
21141 : return regclass;
21142 : }
21143 :
21144 : static reg_class_t
21145 390538725 : ix86_secondary_reload (bool in_p, rtx x, reg_class_t rclass,
21146 : machine_mode mode, secondary_reload_info *sri)
21147 : {
21148 : /* Double-word spills from general registers to non-offsettable memory
21149 : references (zero-extended addresses) require special handling. */
21150 390538725 : if (TARGET_64BIT
21151 337546292 : && MEM_P (x)
21152 182434198 : && GET_MODE_SIZE (mode) > UNITS_PER_WORD
21153 18518743 : && INTEGER_CLASS_P (rclass)
21154 393250519 : && !offsettable_memref_p (x))
21155 : {
21156 2465252 : sri->icode = (in_p
21157 1232626 : ? CODE_FOR_reload_noff_load
21158 : : CODE_FOR_reload_noff_store);
21159 : /* Add the cost of moving address to a temporary. */
21160 1232626 : sri->extra_cost = 1;
21161 :
21162 1232626 : return NO_REGS;
21163 : }
21164 :
21165 : /* QImode spills from non-QI registers require
21166 : intermediate register on 32bit targets. */
21167 389306099 : if (mode == QImode
21168 389306099 : && ((!TARGET_64BIT && !in_p
21169 587374 : && INTEGER_CLASS_P (rclass)
21170 587334 : && MAYBE_NON_Q_CLASS_P (rclass))
21171 22437186 : || (!TARGET_AVX512DQ
21172 22228754 : && MAYBE_MASK_CLASS_P (rclass))))
21173 : {
21174 6555 : int regno = true_regnum (x);
21175 :
21176 : /* Return Q_REGS if the operand is in memory. */
21177 6555 : if (regno == -1)
21178 : return Q_REGS;
21179 :
21180 : return NO_REGS;
21181 : }
21182 :
21183 : /* Require movement to gpr, and then store to memory. */
21184 389299544 : if ((mode == HFmode || mode == HImode || mode == V2QImode
21185 : || mode == BFmode)
21186 4140911 : && !TARGET_SSE4_1
21187 3549200 : && SSE_CLASS_P (rclass)
21188 284722 : && !in_p && MEM_P (x))
21189 : {
21190 173852 : sri->extra_cost = 1;
21191 173852 : return GENERAL_REGS;
21192 : }
21193 :
21194 : /* This condition handles corner case where an expression involving
21195 : pointers gets vectorized. We're trying to use the address of a
21196 : stack slot as a vector initializer.
21197 :
21198 : (set (reg:V2DI 74 [ vect_cst_.2 ])
21199 : (vec_duplicate:V2DI (reg/f:DI 20 frame)))
21200 :
21201 : Eventually frame gets turned into sp+offset like this:
21202 :
21203 : (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
21204 : (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
21205 : (const_int 392 [0x188]))))
21206 :
21207 : That later gets turned into:
21208 :
21209 : (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
21210 : (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
21211 : (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))))
21212 :
21213 : We'll have the following reload recorded:
21214 :
21215 : Reload 0: reload_in (DI) =
21216 : (plus:DI (reg/f:DI 7 sp)
21217 : (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))
21218 : reload_out (V2DI) = (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
21219 : SSE_REGS, RELOAD_OTHER (opnum = 0), can't combine
21220 : reload_in_reg: (plus:DI (reg/f:DI 7 sp) (const_int 392 [0x188]))
21221 : reload_out_reg: (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
21222 : reload_reg_rtx: (reg:V2DI 22 xmm1)
21223 :
21224 : Which isn't going to work since SSE instructions can't handle scalar
21225 : additions. Returning GENERAL_REGS forces the addition into integer
21226 : register and reload can handle subsequent reloads without problems. */
21227 :
21228 223730059 : if (in_p && GET_CODE (x) == PLUS
21229 2 : && SSE_CLASS_P (rclass)
21230 389125692 : && SCALAR_INT_MODE_P (mode))
21231 : return GENERAL_REGS;
21232 :
21233 : return NO_REGS;
21234 : }
21235 :
21236 : /* Implement TARGET_CLASS_LIKELY_SPILLED_P. */
21237 :
21238 : static bool
21239 725194690 : ix86_class_likely_spilled_p (reg_class_t rclass)
21240 : {
21241 715020763 : switch (rclass)
21242 : {
21243 : case AREG:
21244 : case DREG:
21245 : case CREG:
21246 : case BREG:
21247 : case AD_REGS:
21248 : case SIREG:
21249 : case DIREG:
21250 : case SSE_FIRST_REG:
21251 : case FP_TOP_REG:
21252 : case FP_SECOND_REG:
21253 : return true;
21254 :
21255 693022554 : default:
21256 693022554 : break;
21257 : }
21258 :
21259 693022554 : return false;
21260 : }
21261 :
21262 : /* Implement TARGET_CALLEE_SAVE_COST. */
21263 :
21264 : static int
21265 82410772 : ix86_callee_save_cost (spill_cost_type, unsigned int hard_regno, machine_mode,
21266 : unsigned int, int mem_cost, const HARD_REG_SET &, bool)
21267 : {
21268 : /* Account for the fact that push and pop are shorter and do their
21269 : own allocation and deallocation. */
21270 82410772 : if (GENERAL_REGNO_P (hard_regno))
21271 : {
21272 : /* push is 1 byte while typical spill is 4-5 bytes.
21273 : ??? We probably should adjust size costs accordingly.
21274 : Costs are relative to reg-reg move that has 2 bytes for 32bit
21275 : and 3 bytes otherwise. Be sure that no cost table sets cost
21276 : to 2, so we end up with 0. */
21277 82400582 : if (mem_cost <= 2 || optimize_function_for_size_p (cfun))
21278 3585956 : return 1;
21279 78814626 : return mem_cost - 2;
21280 : }
21281 : return mem_cost;
21282 : }
21283 :
21284 : /* Return true if a set of DST by the expression SRC should be allowed.
21285 : This prevents complex sets of likely_spilled hard regs before split1. */
21286 :
21287 : bool
21288 614282848 : ix86_hardreg_mov_ok (rtx dst, rtx src)
21289 : {
21290 : /* Avoid complex sets of likely_spilled hard registers before reload. */
21291 508739774 : if (REG_P (dst) && HARD_REGISTER_P (dst)
21292 297395609 : && !REG_P (src) && !MEM_P (src)
21293 97554981 : && !(VECTOR_MODE_P (GET_MODE (dst))
21294 97554981 : ? standard_sse_constant_p (src, GET_MODE (dst))
21295 48538562 : : x86_64_immediate_operand (src, GET_MODE (dst)))
21296 10173927 : && ix86_class_likely_spilled_p (REGNO_REG_CLASS (REGNO (dst)))
21297 623215110 : && ix86_pre_reload_split ())
21298 : return false;
21299 : return true;
21300 : }
21301 :
21302 : /* If we are copying between registers from different register sets
21303 : (e.g. FP and integer), we may need a memory location.
21304 :
21305 : The function can't work reliably when one of the CLASSES is a class
21306 : containing registers from multiple sets. We avoid this by never combining
21307 : different sets in a single alternative in the machine description.
21308 : Ensure that this constraint holds to avoid unexpected surprises.
21309 :
21310 : When STRICT is false, we are being called from REGISTER_MOVE_COST,
21311 : so do not enforce these sanity checks.
21312 :
21313 : To optimize register_move_cost performance, define inline variant. */
21314 :
21315 : static inline bool
21316 5735133493 : inline_secondary_memory_needed (machine_mode mode, reg_class_t class1,
21317 : reg_class_t class2, int strict)
21318 : {
21319 5735133493 : if (lra_in_progress && (class1 == NO_REGS || class2 == NO_REGS))
21320 : return false;
21321 :
21322 5702668891 : if (MAYBE_FLOAT_CLASS_P (class1) != FLOAT_CLASS_P (class1)
21323 4859755271 : || MAYBE_FLOAT_CLASS_P (class2) != FLOAT_CLASS_P (class2)
21324 4150061630 : || MAYBE_SSE_CLASS_P (class1) != SSE_CLASS_P (class1)
21325 3959620155 : || MAYBE_SSE_CLASS_P (class2) != SSE_CLASS_P (class2)
21326 3779394272 : || MAYBE_MMX_CLASS_P (class1) != MMX_CLASS_P (class1)
21327 3779394272 : || MAYBE_MMX_CLASS_P (class2) != MMX_CLASS_P (class2)
21328 3779394272 : || MAYBE_MASK_CLASS_P (class1) != MASK_CLASS_P (class1)
21329 9310320143 : || MAYBE_MASK_CLASS_P (class2) != MASK_CLASS_P (class2))
21330 : {
21331 2258669682 : gcc_assert (!strict || lra_in_progress);
21332 : return true;
21333 : }
21334 :
21335 3443999209 : if (FLOAT_CLASS_P (class1) != FLOAT_CLASS_P (class2))
21336 : return true;
21337 :
21338 : /* ??? This is a lie. We do have moves between mmx/general, and for
21339 : mmx/sse2. But by saying we need secondary memory we discourage the
21340 : register allocator from using the mmx registers unless needed. */
21341 3293472879 : if (MMX_CLASS_P (class1) != MMX_CLASS_P (class2))
21342 : return true;
21343 :
21344 : /* Between mask and general, we have moves no larger than word size. */
21345 3196059354 : if (MASK_CLASS_P (class1) != MASK_CLASS_P (class2))
21346 : {
21347 2612214 : if (!(INTEGER_CLASS_P (class1) || INTEGER_CLASS_P (class2))
21348 3411939 : || GET_MODE_SIZE (mode) > UNITS_PER_WORD)
21349 193123 : return true;
21350 : }
21351 :
21352 3195866231 : if (SSE_CLASS_P (class1) != SSE_CLASS_P (class2))
21353 : {
21354 : /* SSE1 doesn't have any direct moves from other classes. */
21355 694495400 : if (!TARGET_SSE2)
21356 : return true;
21357 :
21358 692303996 : if (!(INTEGER_CLASS_P (class1) || INTEGER_CLASS_P (class2)))
21359 : return true;
21360 :
21361 : /* If the target says that inter-unit moves are more expensive
21362 : than moving through memory, then don't generate them. */
21363 1038050562 : if ((SSE_CLASS_P (class1) && !TARGET_INTER_UNIT_MOVES_FROM_VEC)
21364 1037564718 : || (SSE_CLASS_P (class2) && !TARGET_INTER_UNIT_MOVES_TO_VEC))
21365 1321114 : return true;
21366 :
21367 : /* With SSE4.1, *mov{ti,di}_internal supports moves between
21368 : SSE_REGS and GENERAL_REGS using pinsr{q,d} or pextr{q,d}. */
21369 690982882 : if (TARGET_SSE4_1
21370 36644205 : && (TARGET_64BIT ? mode == TImode : mode == DImode))
21371 : return false;
21372 :
21373 689394537 : int msize = GET_MODE_SIZE (mode);
21374 :
21375 : /* Between SSE and general, we have moves no larger than word size. */
21376 705775816 : if (msize > UNITS_PER_WORD)
21377 : return true;
21378 :
21379 : /* In addition to SImode moves, HImode moves are supported for SSE2 and above,
21380 : Use vmovw with AVX512FP16, or pinsrw/pextrw without AVX512FP16. */
21381 596408068 : int minsize = GET_MODE_SIZE (TARGET_SSE2 ? HImode : SImode);
21382 :
21383 596408068 : if (msize < minsize)
21384 : return true;
21385 : }
21386 :
21387 : return false;
21388 : }
21389 :
21390 : /* Implement TARGET_SECONDARY_MEMORY_NEEDED. */
21391 :
21392 : static bool
21393 72170173 : ix86_secondary_memory_needed (machine_mode mode, reg_class_t class1,
21394 : reg_class_t class2)
21395 : {
21396 72170173 : return inline_secondary_memory_needed (mode, class1, class2, true);
21397 : }
21398 :
21399 : /* Implement TARGET_SECONDARY_MEMORY_NEEDED_MODE.
21400 :
21401 : get_secondary_mem widens integral modes to BITS_PER_WORD.
21402 : There is no need to emit full 64 bit move on 64 bit targets
21403 : for integral modes that can be moved using 32 bit move. */
21404 :
21405 : static machine_mode
21406 13059 : ix86_secondary_memory_needed_mode (machine_mode mode)
21407 : {
21408 26118 : if (GET_MODE_BITSIZE (mode) < 32 && INTEGRAL_MODE_P (mode))
21409 19 : return mode_for_size (32, GET_MODE_CLASS (mode), 0).require ();
21410 : return mode;
21411 : }
21412 :
21413 : /* Implement the TARGET_CLASS_MAX_NREGS hook.
21414 :
21415 : On the 80386, this is the size of MODE in words,
21416 : except in the FP regs, where a single reg is always enough. */
21417 :
21418 : static unsigned char
21419 5766384688 : ix86_class_max_nregs (reg_class_t rclass, machine_mode mode)
21420 : {
21421 5766384688 : if (MAYBE_INTEGER_CLASS_P (rclass))
21422 : {
21423 3885289280 : if (mode == XFmode)
21424 145997999 : return (TARGET_64BIT ? 2 : 3);
21425 3739291281 : else if (mode == XCmode)
21426 145997624 : return (TARGET_64BIT ? 4 : 6);
21427 : else
21428 7292357270 : return CEIL (GET_MODE_SIZE (mode), UNITS_PER_WORD);
21429 : }
21430 : else
21431 : {
21432 1881095408 : if (COMPLEX_MODE_P (mode))
21433 : return 2;
21434 : else
21435 1605141309 : return 1;
21436 : }
21437 : }
21438 :
21439 : /* Implement TARGET_CAN_CHANGE_MODE_CLASS. */
21440 :
21441 : static bool
21442 41332483 : ix86_can_change_mode_class (machine_mode from, machine_mode to,
21443 : reg_class_t regclass)
21444 : {
21445 41332483 : if (from == to)
21446 : return true;
21447 :
21448 : /* x87 registers can't do subreg at all, as all values are reformatted
21449 : to extended precision.
21450 :
21451 : ??? middle-end queries mode changes for ALL_REGS and this makes
21452 : vec_series_lowpart_p to always return false. We probably should
21453 : restrict this to modes supported by i387 and check if it is enabled. */
21454 39848583 : if (MAYBE_FLOAT_CLASS_P (regclass))
21455 : return false;
21456 :
21457 35062085 : if (MAYBE_SSE_CLASS_P (regclass) || MAYBE_MMX_CLASS_P (regclass))
21458 : {
21459 : /* Vector registers do not support QI or HImode loads. If we don't
21460 : disallow a change to these modes, reload will assume it's ok to
21461 : drop the subreg from (subreg:SI (reg:HI 100) 0). This affects
21462 : the vec_dupv4hi pattern.
21463 : NB: SSE2 can load 16bit data to sse register via pinsrw. */
21464 16793165 : int mov_size = MAYBE_SSE_CLASS_P (regclass) && TARGET_SSE2 ? 2 : 4;
21465 16793165 : if (GET_MODE_SIZE (from) < mov_size
21466 33586016 : || GET_MODE_SIZE (to) < mov_size)
21467 : return false;
21468 : }
21469 :
21470 : return true;
21471 : }
21472 :
21473 : /* Return index of MODE in the sse load/store tables. */
21474 :
21475 : static inline int
21476 780689862 : sse_store_index (machine_mode mode)
21477 : {
21478 : /* NB: Use SFmode cost for HFmode instead of adding HFmode load/store
21479 : costs to processor_costs, which requires changes to all entries in
21480 : processor cost table. */
21481 780689862 : if (mode == E_HFmode)
21482 138359338 : mode = E_SFmode;
21483 :
21484 1561379724 : switch (GET_MODE_SIZE (mode))
21485 : {
21486 : case 4:
21487 : return 0;
21488 : case 8:
21489 : return 1;
21490 : case 16:
21491 : return 2;
21492 : case 32:
21493 : return 3;
21494 : case 64:
21495 : return 4;
21496 : default:
21497 : return -1;
21498 : }
21499 : }
21500 :
21501 : /* Return the cost of moving data of mode M between a
21502 : register and memory. A value of 2 is the default; this cost is
21503 : relative to those in `REGISTER_MOVE_COST'.
21504 :
21505 : This function is used extensively by register_move_cost that is used to
21506 : build tables at startup. Make it inline in this case.
21507 : When IN is 2, return maximum of in and out move cost.
21508 :
21509 : If moving between registers and memory is more expensive than
21510 : between two registers, you should define this macro to express the
21511 : relative cost.
21512 :
21513 : Model also increased moving costs of QImode registers in non
21514 : Q_REGS classes.
21515 : */
21516 : static inline int
21517 6978568990 : inline_memory_move_cost (machine_mode mode, enum reg_class regclass, int in)
21518 : {
21519 6978568990 : int cost;
21520 :
21521 6978568990 : if (FLOAT_CLASS_P (regclass))
21522 : {
21523 356439293 : int index;
21524 356439293 : switch (mode)
21525 : {
21526 : case E_SFmode:
21527 : index = 0;
21528 : break;
21529 : case E_DFmode:
21530 : index = 1;
21531 : break;
21532 : case E_XFmode:
21533 : index = 2;
21534 : break;
21535 : default:
21536 : return 100;
21537 : }
21538 106548725 : if (in == 2)
21539 102568157 : return MAX (ix86_cost->hard_register.fp_load [index],
21540 : ix86_cost->hard_register.fp_store [index]);
21541 3980568 : return in ? ix86_cost->hard_register.fp_load [index]
21542 3980568 : : ix86_cost->hard_register.fp_store [index];
21543 : }
21544 6622129697 : if (SSE_CLASS_P (regclass))
21545 : {
21546 650372704 : int index = sse_store_index (mode);
21547 650372704 : if (index == -1)
21548 : return 100;
21549 565610953 : if (in == 2)
21550 400705524 : return MAX (ix86_cost->hard_register.sse_load [index],
21551 : ix86_cost->hard_register.sse_store [index]);
21552 164905429 : return in ? ix86_cost->hard_register.sse_load [index]
21553 164905429 : : ix86_cost->hard_register.sse_store [index];
21554 : }
21555 5971756993 : if (MASK_CLASS_P (regclass))
21556 : {
21557 109287603 : int index;
21558 218575206 : switch (GET_MODE_SIZE (mode))
21559 : {
21560 : case 1:
21561 : index = 0;
21562 : break;
21563 9014229 : case 2:
21564 9014229 : index = 1;
21565 9014229 : break;
21566 : /* DImode loads and stores assumed to cost the same as SImode. */
21567 40559118 : case 4:
21568 40559118 : case 8:
21569 40559118 : index = 2;
21570 40559118 : break;
21571 : default:
21572 : return 100;
21573 : }
21574 :
21575 53183155 : if (in == 2)
21576 585187 : return MAX (ix86_cost->hard_register.mask_load[index],
21577 : ix86_cost->hard_register.mask_store[index]);
21578 52597968 : return in ? ix86_cost->hard_register.mask_load[2]
21579 52597968 : : ix86_cost->hard_register.mask_store[2];
21580 : }
21581 5862469390 : if (MMX_CLASS_P (regclass))
21582 : {
21583 174029132 : int index;
21584 348058264 : switch (GET_MODE_SIZE (mode))
21585 : {
21586 : case 4:
21587 : index = 0;
21588 : break;
21589 102136548 : case 8:
21590 102136548 : index = 1;
21591 102136548 : break;
21592 : default:
21593 : return 100;
21594 : }
21595 139840484 : if (in == 2)
21596 119677926 : return MAX (ix86_cost->hard_register.mmx_load [index],
21597 : ix86_cost->hard_register.mmx_store [index]);
21598 20162558 : return in ? ix86_cost->hard_register.mmx_load [index]
21599 20162558 : : ix86_cost->hard_register.mmx_store [index];
21600 : }
21601 11376880516 : switch (GET_MODE_SIZE (mode))
21602 : {
21603 125928078 : case 1:
21604 125928078 : if (Q_CLASS_P (regclass) || TARGET_64BIT)
21605 : {
21606 123297829 : if (!in)
21607 19778812 : return ix86_cost->hard_register.int_store[0];
21608 103519017 : if (TARGET_PARTIAL_REG_DEPENDENCY
21609 103519017 : && optimize_function_for_speed_p (cfun))
21610 96549220 : cost = ix86_cost->hard_register.movzbl_load;
21611 : else
21612 6969797 : cost = ix86_cost->hard_register.int_load[0];
21613 103519017 : if (in == 2)
21614 83712210 : return MAX (cost, ix86_cost->hard_register.int_store[0]);
21615 : return cost;
21616 : }
21617 : else
21618 : {
21619 2630249 : if (in == 2)
21620 1862690 : return MAX (ix86_cost->hard_register.movzbl_load,
21621 : ix86_cost->hard_register.int_store[0] + 4);
21622 767559 : if (in)
21623 383835 : return ix86_cost->hard_register.movzbl_load;
21624 : else
21625 383724 : return ix86_cost->hard_register.int_store[0] + 4;
21626 : }
21627 649948098 : break;
21628 649948098 : case 2:
21629 649948098 : {
21630 649948098 : int cost;
21631 649948098 : if (in == 2)
21632 549114901 : cost = MAX (ix86_cost->hard_register.int_load[1],
21633 : ix86_cost->hard_register.int_store[1]);
21634 : else
21635 100833197 : cost = in ? ix86_cost->hard_register.int_load[1]
21636 : : ix86_cost->hard_register.int_store[1];
21637 :
21638 649948098 : if (mode == E_HFmode)
21639 : {
21640 : /* Prefer SSE over GPR for HFmode. */
21641 125940554 : int sse_cost;
21642 125940554 : int index = sse_store_index (mode);
21643 125940554 : if (in == 2)
21644 115859176 : sse_cost = MAX (ix86_cost->hard_register.sse_load[index],
21645 : ix86_cost->hard_register.sse_store[index]);
21646 : else
21647 20162756 : sse_cost = (in
21648 10081378 : ? ix86_cost->hard_register.sse_load [index]
21649 : : ix86_cost->hard_register.sse_store [index]);
21650 125940554 : if (sse_cost >= cost)
21651 125940554 : cost = sse_cost + 1;
21652 : }
21653 : return cost;
21654 : }
21655 4912564082 : default:
21656 4912564082 : if (in == 2)
21657 3803245284 : cost = MAX (ix86_cost->hard_register.int_load[2],
21658 : ix86_cost->hard_register.int_store[2]);
21659 1109318798 : else if (in)
21660 554849058 : cost = ix86_cost->hard_register.int_load[2];
21661 : else
21662 554469740 : cost = ix86_cost->hard_register.int_store[2];
21663 : /* Multiply with the number of GPR moves needed. */
21664 9944149825 : return cost * CEIL ((int) GET_MODE_SIZE (mode), UNITS_PER_WORD);
21665 : }
21666 : }
21667 :
21668 : static int
21669 1794242602 : ix86_memory_move_cost (machine_mode mode, reg_class_t regclass, bool in)
21670 : {
21671 2691037138 : return inline_memory_move_cost (mode, (enum reg_class) regclass, in ? 1 : 0);
21672 : }
21673 :
21674 :
21675 : /* Return the cost of moving data from a register in class CLASS1 to
21676 : one in class CLASS2.
21677 :
21678 : It is not required that the cost always equal 2 when FROM is the same as TO;
21679 : on some machines it is expensive to move between registers if they are not
21680 : general registers. */
21681 :
21682 : static int
21683 5662963320 : ix86_register_move_cost (machine_mode mode, reg_class_t class1_i,
21684 : reg_class_t class2_i)
21685 : {
21686 5662963320 : enum reg_class class1 = (enum reg_class) class1_i;
21687 5662963320 : enum reg_class class2 = (enum reg_class) class2_i;
21688 :
21689 : /* In case we require secondary memory, compute cost of the store followed
21690 : by load. In order to avoid bad register allocation choices, we need
21691 : for this to be *at least* as high as the symmetric MEMORY_MOVE_COST. */
21692 :
21693 5662963320 : if (inline_secondary_memory_needed (mode, class1, class2, false))
21694 : {
21695 2592163194 : int cost = 1;
21696 :
21697 2592163194 : cost += inline_memory_move_cost (mode, class1, 2);
21698 2592163194 : cost += inline_memory_move_cost (mode, class2, 2);
21699 :
21700 : /* In case of copying from general_purpose_register we may emit multiple
21701 : stores followed by single load causing memory size mismatch stall.
21702 : Count this as arbitrarily high cost of 20. */
21703 5184326388 : if (GET_MODE_BITSIZE (mode) > BITS_PER_WORD
21704 775940591 : && TARGET_MEMORY_MISMATCH_STALL
21705 4144044376 : && targetm.class_max_nregs (class1, mode)
21706 775940591 : > targetm.class_max_nregs (class2, mode))
21707 147627669 : cost += 20;
21708 :
21709 : /* In the case of FP/MMX moves, the registers actually overlap, and we
21710 : have to switch modes in order to treat them differently. */
21711 59838999 : if ((MMX_CLASS_P (class1) && MAYBE_FLOAT_CLASS_P (class2))
21712 2642578399 : || (MMX_CLASS_P (class2) && MAYBE_FLOAT_CLASS_P (class1)))
21713 18847588 : cost += 20;
21714 :
21715 2592163194 : return cost;
21716 : }
21717 :
21718 : /* Moves between MMX and non-MMX units require secondary memory. */
21719 3070800126 : if (MMX_CLASS_P (class1) != MMX_CLASS_P (class2))
21720 0 : gcc_unreachable ();
21721 :
21722 3070800126 : if (SSE_CLASS_P (class1) != SSE_CLASS_P (class2))
21723 588192716 : return (SSE_CLASS_P (class1)
21724 588192716 : ? ix86_cost->hard_register.sse_to_integer
21725 588192716 : : ix86_cost->hard_register.integer_to_sse);
21726 :
21727 : /* Moves between mask register and GPR. */
21728 2482607410 : if (MASK_CLASS_P (class1) != MASK_CLASS_P (class2))
21729 : {
21730 1056274 : return (MASK_CLASS_P (class1)
21731 1056274 : ? ix86_cost->hard_register.mask_to_integer
21732 1056274 : : ix86_cost->hard_register.integer_to_mask);
21733 : }
21734 : /* Moving between mask registers. */
21735 2481551136 : if (MASK_CLASS_P (class1) && MASK_CLASS_P (class2))
21736 101327 : return ix86_cost->hard_register.mask_move;
21737 :
21738 2481449809 : if (MAYBE_FLOAT_CLASS_P (class1))
21739 11913696 : return ix86_cost->hard_register.fp_move;
21740 2469536113 : if (MAYBE_SSE_CLASS_P (class1))
21741 : {
21742 231792092 : if (GET_MODE_BITSIZE (mode) <= 128)
21743 113427659 : return ix86_cost->hard_register.xmm_move;
21744 4936774 : if (GET_MODE_BITSIZE (mode) <= 256)
21745 1568427 : return ix86_cost->hard_register.ymm_move;
21746 899960 : return ix86_cost->hard_register.zmm_move;
21747 : }
21748 2353640067 : if (MAYBE_MMX_CLASS_P (class1))
21749 2191635 : return ix86_cost->hard_register.mmx_move;
21750 : return 2;
21751 : }
21752 :
21753 : /* Implement TARGET_HARD_REGNO_NREGS. This is ordinarily the length in
21754 : words of a value of mode MODE but can be less for certain modes in
21755 : special long registers.
21756 :
21757 : Actually there are no two word move instructions for consecutive
21758 : registers. And only registers 0-3 may have mov byte instructions
21759 : applied to them. */
21760 :
21761 : static unsigned int
21762 8883261296 : ix86_hard_regno_nregs (unsigned int regno, machine_mode mode)
21763 : {
21764 8883261296 : if (GENERAL_REGNO_P (regno))
21765 : {
21766 3089830016 : if (mode == XFmode)
21767 25393824 : return TARGET_64BIT ? 2 : 3;
21768 3064912032 : if (mode == XCmode)
21769 25393824 : return TARGET_64BIT ? 4 : 6;
21770 6138040576 : return CEIL (GET_MODE_SIZE (mode), UNITS_PER_WORD);
21771 : }
21772 5793431280 : if (COMPLEX_MODE_P (mode))
21773 : return 2;
21774 : /* Register pair for mask registers. */
21775 5045891760 : if (mode == P2QImode || mode == P2HImode)
21776 93442440 : return 2;
21777 :
21778 : return 1;
21779 : }
21780 :
21781 : /* Implement REGMODE_NATURAL_SIZE(MODE). */
21782 : unsigned int
21783 114826619 : ix86_regmode_natural_size (machine_mode mode)
21784 : {
21785 114826619 : if (mode == P2HImode || mode == P2QImode)
21786 2462 : return GET_MODE_SIZE (mode) / 2;
21787 114825388 : return UNITS_PER_WORD;
21788 : }
21789 :
21790 : /* Implement TARGET_HARD_REGNO_MODE_OK. */
21791 :
21792 : static bool
21793 52441177903 : ix86_hard_regno_mode_ok (unsigned int regno, machine_mode mode)
21794 : {
21795 : /* Flags and only flags can only hold CCmode values. */
21796 52441177903 : if (CC_REGNO_P (regno))
21797 414277204 : return GET_MODE_CLASS (mode) == MODE_CC;
21798 52026900699 : if (GET_MODE_CLASS (mode) == MODE_CC
21799 : || GET_MODE_CLASS (mode) == MODE_RANDOM)
21800 : return false;
21801 46696101832 : if (STACK_REGNO_P (regno))
21802 4545917514 : return VALID_FP_MODE_P (mode);
21803 42150184318 : if (MASK_REGNO_P (regno))
21804 : {
21805 : /* Register pair only starts at even register number. */
21806 3542787009 : if ((mode == P2QImode || mode == P2HImode))
21807 48836558 : return MASK_PAIR_REGNO_P(regno);
21808 :
21809 987908435 : return ((TARGET_AVX512F && VALID_MASK_REG_MODE (mode))
21810 4461755861 : || (TARGET_AVX512BW && VALID_MASK_AVX512BW_MODE (mode)));
21811 : }
21812 :
21813 38607397309 : if (GET_MODE_CLASS (mode) == MODE_PARTIAL_INT)
21814 : return false;
21815 :
21816 37664262791 : if (SSE_REGNO_P (regno))
21817 : {
21818 : /* We implement the move patterns for all vector modes into and
21819 : out of SSE registers, even when no operation instructions
21820 : are available. */
21821 :
21822 : /* For AVX-512 we allow, regardless of regno:
21823 : - XI mode
21824 : - any of 512-bit wide vector mode
21825 : - any scalar mode. */
21826 16297107504 : if (TARGET_AVX512F
21827 : && ((VALID_AVX512F_REG_OR_XI_MODE (mode))
21828 : || VALID_AVX512F_SCALAR_MODE (mode)))
21829 : return true;
21830 :
21831 : /* TODO check for QI/HI scalars. */
21832 : /* AVX512VL allows sse regs16+ for 128/256 bit modes. */
21833 15615793997 : if (TARGET_AVX512VL
21834 1738101979 : && (VALID_AVX256_REG_OR_OI_MODE (mode)
21835 1527444541 : || VALID_AVX512VL_128_REG_MODE (mode)))
21836 : return true;
21837 :
21838 : /* xmm16-xmm31 are only available for AVX-512. */
21839 15172628089 : if (EXT_REX_SSE_REGNO_P (regno))
21840 : return false;
21841 :
21842 : /* OImode and AVX modes are available only when AVX is enabled. */
21843 8795830017 : return ((TARGET_AVX
21844 1892505800 : && VALID_AVX256_REG_OR_OI_MODE (mode))
21845 : || VALID_SSE_REG_MODE (mode)
21846 : || VALID_SSE2_REG_MODE (mode)
21847 : || VALID_MMX_REG_MODE (mode)
21848 8795830017 : || VALID_MMX_REG_MODE_3DNOW (mode));
21849 : }
21850 21367155287 : if (MMX_REGNO_P (regno))
21851 : {
21852 : /* We implement the move patterns for 3DNOW modes even in MMX mode,
21853 : so if the register is available at all, then we can move data of
21854 : the given mode into or out of it. */
21855 3784699516 : return (VALID_MMX_REG_MODE (mode)
21856 : || VALID_MMX_REG_MODE_3DNOW (mode));
21857 : }
21858 :
21859 17582455771 : if (mode == QImode)
21860 : {
21861 : /* Take care for QImode values - they can be in non-QI regs,
21862 : but then they do cause partial register stalls. */
21863 200810735 : if (ANY_QI_REGNO_P (regno))
21864 : return true;
21865 13847564 : if (!TARGET_PARTIAL_REG_STALL)
21866 : return true;
21867 : /* LRA checks if the hard register is OK for the given mode.
21868 : QImode values can live in non-QI regs, so we allow all
21869 : registers here. */
21870 19 : if (lra_in_progress)
21871 : return true;
21872 19 : return !can_create_pseudo_p ();
21873 : }
21874 : /* We handle both integer and floats in the general purpose registers. */
21875 17381645036 : else if (VALID_INT_MODE_P (mode)
21876 12696194401 : || VALID_FP_MODE_P (mode))
21877 : return true;
21878 : /* Lots of MMX code casts 8 byte vector modes to DImode. If we then go
21879 : on to use that value in smaller contexts, this can easily force a
21880 : pseudo to be allocated to GENERAL_REGS. Since this is no worse than
21881 : supporting DImode, allow it. */
21882 11670799719 : else if (VALID_MMX_REG_MODE_3DNOW (mode) || VALID_MMX_REG_MODE (mode))
21883 : return true;
21884 :
21885 : return false;
21886 : }
21887 :
21888 : /* Initialize function_abis[ABI_ID] with the set of register clobbers
21889 : in FULL_REG_CLOBBERS, adjusting for rules that apply to all ABIs. */
21890 :
21891 : static void
21892 762 : ix86_initialize_abi (unsigned int abi_id, HARD_REG_SET full_reg_clobbers)
21893 : {
21894 : /* The general rule is that fixed registers should be marked as
21895 : call-clobbered. This includes global registers, inaccessible
21896 : registers, the flags register, and the FPSR.
21897 :
21898 : Handle the exceptions below. */
21899 762 : full_reg_clobbers |= fixed_reg_set;
21900 :
21901 : /* Every ABI (even preserve_none) preserves EBP/RBP. */
21902 762 : CLEAR_HARD_REG_BIT (full_reg_clobbers, HARD_FRAME_POINTER_REGNUM);
21903 :
21904 : /* Treat GCC's internal frame-related registers as call-preserved. */
21905 762 : CLEAR_HARD_REG_BIT (full_reg_clobbers, FRAME_POINTER_REGNUM);
21906 762 : CLEAR_HARD_REG_BIT (full_reg_clobbers, ARG_POINTER_REGNUM);
21907 :
21908 : /* MMX registers aren't preserved. */
21909 762 : if (TARGET_MMX)
21910 762 : full_reg_clobbers |= reg_class_contents[MMX_REGS];
21911 :
21912 : /* X87 registers aren't preserved. */
21913 762 : if (TARGET_80387 || TARGET_FLOAT_RETURNS_IN_80387)
21914 762 : full_reg_clobbers |= reg_class_contents[FLOAT_REGS];
21915 :
21916 762 : function_abis[abi_id].initialize (abi_id, full_reg_clobbers);
21917 762 : }
21918 :
21919 : /* Return the descriptor of no_callee_saved_registers function type.
21920 : None of the enabled registers are preserved, except for the common
21921 : rules applied by ix86_initialize_abi. */
21922 :
21923 : static const predefined_function_abi &
21924 3088 : ix86_no_callee_saved_abi (void)
21925 : {
21926 3088 : auto &no_callee_saved_abi = function_abis[ABI_NO_CALLEE_SAVED];
21927 3088 : if (!no_callee_saved_abi.initialized_p ())
21928 98 : ix86_initialize_abi (ABI_NO_CALLEE_SAVED, accessible_reg_set);
21929 3088 : return no_callee_saved_abi;
21930 : }
21931 :
21932 : /* Return the descriptor of the no_caller_saved_registers function type
21933 : with ABI identifier ABI_ID. All registers are preserved, except for:
21934 :
21935 : - the return registers, which are enumerated in ABI_ID.
21936 :
21937 : - the common rules applied by ix86_initialize_abi. */
21938 :
21939 : static const predefined_function_abi &
21940 3705 : ix86_no_caller_saved_abi (unsigned int abi_id)
21941 : {
21942 3705 : auto &abi = function_abis[abi_id];
21943 3705 : if (!abi.initialized_p ())
21944 : {
21945 172 : HARD_REG_SET full_reg_clobbers = {};
21946 :
21947 172 : switch (abi_id)
21948 : {
21949 : case ABI_NO_CALLER_SAVED_RETURN_VOID:
21950 : break;
21951 :
21952 1 : case ABI_NO_CALLER_SAVED_RETURN_AX_DX:
21953 1 : SET_HARD_REG_BIT (full_reg_clobbers, DX_REG);
21954 : /* Fall through. */
21955 9 : case ABI_NO_CALLER_SAVED_RETURN_AX:
21956 9 : SET_HARD_REG_BIT (full_reg_clobbers, AX_REG);
21957 9 : break;
21958 :
21959 2 : case ABI_NO_CALLER_SAVED_RETURN_AX_XMM0:
21960 2 : SET_HARD_REG_BIT (full_reg_clobbers, AX_REG);
21961 2 : SET_HARD_REG_BIT (full_reg_clobbers, XMM0_REG);
21962 2 : break;
21963 :
21964 1 : case ABI_NO_CALLER_SAVED_RETURN_XMM0_XMM1:
21965 1 : SET_HARD_REG_BIT (full_reg_clobbers, XMM1_REG);
21966 : /* Fall through. */
21967 2 : case ABI_NO_CALLER_SAVED_RETURN_XMM0:
21968 2 : SET_HARD_REG_BIT (full_reg_clobbers, XMM0_REG);
21969 2 : break;
21970 :
21971 0 : default:
21972 0 : gcc_unreachable ();
21973 : }
21974 :
21975 172 : ix86_initialize_abi (abi_id, full_reg_clobbers);
21976 : }
21977 3705 : return abi;
21978 : }
21979 :
21980 : /* Return the descriptor of the standard function ABI type. If
21981 : ABI_TYPE == ABI_ALTERNATE, return the function alternate ABI type. */
21982 :
21983 : static const predefined_function_abi &
21984 1597203 : ix86_standard_abi (int abi_type)
21985 : {
21986 1597203 : static const char ix86_call_used_regs[] = CALL_USED_REGISTERS;
21987 1597203 : auto &standard_abi = function_abis[abi_type];
21988 1597203 : if (!standard_abi.initialized_p ())
21989 : {
21990 492 : HARD_REG_SET full_reg_clobbers = {};
21991 :
21992 : /* Add all registers that are clobbered by the call. NB: If the
21993 : current ABI is SYSV_ABI, the alternate ABI is MS_ABI. */
21994 1007 : bool is_64bit_ms_abi = (TARGET_64BIT
21995 492 : && ix86_abi == (abi_type == ABI_ALTERNATE
21996 492 : ? SYSV_ABI : MS_ABI));
21997 23 : char c_mask = CALL_USED_REGISTERS_MASK (is_64bit_ms_abi);
21998 45756 : for (int i = 0; i < FIRST_PSEUDO_REGISTER; i++)
21999 45264 : if (ix86_call_used_regs[i] == 1
22000 8856 : || (ix86_call_used_regs[i] & c_mask))
22001 36684 : SET_HARD_REG_BIT (full_reg_clobbers, i);
22002 :
22003 492 : ix86_initialize_abi (abi_type, full_reg_clobbers);
22004 : }
22005 1597203 : return standard_abi;
22006 : }
22007 :
22008 : /* Return the descriptor of the function alternate ABI type. */
22009 :
22010 : static const predefined_function_abi &
22011 1597203 : ix86_alternate_abi (void)
22012 : {
22013 12 : return ix86_standard_abi (ABI_ALTERNATE);
22014 : }
22015 :
22016 : /* Return the function ABI ID based on FNTYPE. */
22017 :
22018 : static int
22019 410293238 : ix86_function_abi_id (const_tree fntype)
22020 : {
22021 410293238 : auto call_saved_registers = ix86_fntype_call_saved_registers (fntype);
22022 410293238 : if (call_saved_registers == TYPE_PRESERVE_NONE
22023 410293238 : || call_saved_registers == TYPE_NO_CALLEE_SAVED_REGISTERS)
22024 : return ABI_NO_CALLEE_SAVED;
22025 :
22026 410290150 : if (call_saved_registers == TYPE_NO_CALLER_SAVED_REGISTERS)
22027 : {
22028 3705 : tree type = TREE_TYPE (fntype);
22029 3705 : if (VOID_TYPE_P (type))
22030 : return ABI_NO_CALLER_SAVED_RETURN_VOID;
22031 : /* AX register contains the address of the return value location
22032 : passed in by the caller. */
22033 2256 : else if (ix86_return_in_memory (type, fntype))
22034 : return ABI_NO_CALLER_SAVED_RETURN_AX;
22035 2197 : rtx ret = ix86_function_value (type, fntype, false);
22036 2197 : unsigned int nregs;
22037 2197 : if (REG_P (ret))
22038 : {
22039 2016 : unsigned int regno = REGNO (ret);
22040 2016 : if (STACK_REGNO_P (regno) || MMX_REGNO_P (regno))
22041 : return ABI_NO_CALLER_SAVED_RETURN_VOID;
22042 : else
22043 2016 : switch (regno)
22044 : {
22045 1947 : case AX_REG:
22046 1947 : nregs = REG_NREGS (ret);
22047 1947 : if (nregs == 1)
22048 : return ABI_NO_CALLER_SAVED_RETURN_AX;
22049 71 : else if (nregs == 2)
22050 : return ABI_NO_CALLER_SAVED_RETURN_AX_DX;
22051 0 : gcc_unreachable ();
22052 : case XMM0_REG:
22053 : return ABI_NO_CALLER_SAVED_RETURN_XMM0;
22054 0 : default:
22055 0 : gcc_unreachable ();
22056 : }
22057 : }
22058 181 : else if (GET_CODE (ret) == PARALLEL && XVECLEN (ret, 0) == 2)
22059 : {
22060 181 : rtx x0 = XVECEXP (ret, 0, 0);
22061 181 : rtx x1 = XVECEXP (ret, 0, 1);
22062 181 : if (GET_CODE (x0) == EXPR_LIST
22063 181 : && GET_CODE (x1) == EXPR_LIST)
22064 : {
22065 181 : x0 = XEXP (x0, 0);
22066 181 : x1 = XEXP (x1, 0);
22067 181 : if (REG_P (x0) && REGNO (x0) == AX_REG)
22068 : {
22069 59 : if (REG_P (x1) && REGNO (x1) == XMM0_REG)
22070 : return ABI_NO_CALLER_SAVED_RETURN_AX_XMM0;
22071 : }
22072 122 : if (REG_P (x0) && REGNO (x0) == XMM0_REG && REG_P (x1))
22073 : {
22074 122 : if (REGNO (x1) == AX_REG)
22075 : return ABI_NO_CALLER_SAVED_RETURN_AX_XMM0;
22076 64 : else if (REGNO (x1) == XMM1_REG)
22077 : return ABI_NO_CALLER_SAVED_RETURN_XMM0_XMM1;
22078 : }
22079 : }
22080 :
22081 0 : gcc_unreachable ();
22082 : }
22083 : }
22084 :
22085 : /* NB: This must be the last since other attributes change the
22086 : function ABI. */
22087 410286445 : if (ix86_function_type_abi (fntype) != ix86_abi)
22088 1597191 : return ABI_ALTERNATE;
22089 :
22090 : return ABI_DEFAULT;
22091 : }
22092 :
22093 : /* Implement TARGET_FNTYPE_ABI. */
22094 :
22095 : static const predefined_function_abi &
22096 410293238 : ix86_fntype_abi (const_tree fntype)
22097 : {
22098 410293238 : unsigned int abi_id = ix86_function_abi_id (fntype);
22099 410293238 : switch (abi_id)
22100 : {
22101 408689254 : case ABI_DEFAULT:
22102 408689254 : return default_function_abi;
22103 :
22104 1597191 : case ABI_ALTERNATE:
22105 1597191 : return ix86_alternate_abi ();
22106 :
22107 3088 : case ABI_NO_CALLEE_SAVED:
22108 3088 : return ix86_no_callee_saved_abi ();
22109 :
22110 3705 : case ABI_NO_CALLER_SAVED_RETURN_VOID:
22111 3705 : case ABI_NO_CALLER_SAVED_RETURN_AX:
22112 3705 : case ABI_NO_CALLER_SAVED_RETURN_AX_DX:
22113 3705 : case ABI_NO_CALLER_SAVED_RETURN_AX_XMM0:
22114 3705 : case ABI_NO_CALLER_SAVED_RETURN_XMM0:
22115 3705 : case ABI_NO_CALLER_SAVED_RETURN_XMM0_XMM1:
22116 3705 : return ix86_no_caller_saved_abi (abi_id);
22117 :
22118 0 : default:
22119 0 : gcc_unreachable ();
22120 : }
22121 :
22122 : return default_function_abi;
22123 : }
22124 :
22125 : /* Initialize function_abis with corresponding abi_id,
22126 : currently only handle vzeroupper. */
22127 : void
22128 22304 : ix86_initialize_callee_abi (unsigned int abi_id)
22129 : {
22130 22304 : gcc_assert (abi_id == ABI_VZEROUPPER);
22131 22304 : predefined_function_abi &vzeroupper_abi = function_abis[abi_id];
22132 22304 : if (!vzeroupper_abi.initialized_p ())
22133 : {
22134 : HARD_REG_SET full_reg_clobbers;
22135 4317 : CLEAR_HARD_REG_SET (full_reg_clobbers);
22136 4317 : vzeroupper_abi.initialize (ABI_VZEROUPPER, full_reg_clobbers);
22137 : }
22138 22304 : }
22139 :
22140 : void
22141 22304 : ix86_expand_avx_vzeroupper (void)
22142 : {
22143 : /* Initialize vzeroupper_abi here. */
22144 22304 : ix86_initialize_callee_abi (ABI_VZEROUPPER);
22145 22304 : rtx_insn *insn = emit_call_insn (gen_avx_vzeroupper_callee_abi ());
22146 22304 : CALL_INSN_ABI_ID (insn) = ABI_VZEROUPPER;
22147 : /* Return false for non-local goto in can_nonlocal_goto. */
22148 22304 : make_reg_eh_region_note (insn, 0, INT_MIN);
22149 : /* Flag used for call_insn indicates it's a fake call. */
22150 22304 : RTX_FLAG (insn, used) = 1;
22151 22304 : }
22152 :
22153 :
22154 : /* Implement TARGET_HARD_REGNO_CALL_PART_CLOBBERED. The only ABI that
22155 : saves SSE registers across calls is Win64 (thus no need to check the
22156 : current ABI here), and with AVX enabled Win64 only guarantees that
22157 : the low 16 bytes are saved. */
22158 :
22159 : static bool
22160 1903337341 : ix86_hard_regno_call_part_clobbered (unsigned int abi_id, unsigned int regno,
22161 : machine_mode mode)
22162 : {
22163 1903337341 : switch (abi_id)
22164 : {
22165 31230681 : case ABI_VZEROUPPER:
22166 : /* Special ABI for vzeroupper which only clobbers higher part of
22167 : SSE registers. */
22168 31230681 : return (GET_MODE_SIZE (mode) > 16
22169 31230681 : && ((TARGET_64BIT && REX_SSE_REGNO_P (regno))
22170 4784802 : || LEGACY_SSE_REGNO_P (regno)));
22171 :
22172 1871602090 : case ABI_DEFAULT:
22173 1871602090 : case ABI_ALTERNATE:
22174 1871602090 : case ABI_NO_CALLEE_SAVED:
22175 1871602090 : break;
22176 :
22177 : case ABI_NO_CALLER_SAVED_RETURN_VOID:
22178 : case ABI_NO_CALLER_SAVED_RETURN_AX:
22179 : case ABI_NO_CALLER_SAVED_RETURN_AX_DX:
22180 : /* These ABIs don't clobber SSE registers. */
22181 : return false;
22182 :
22183 : case ABI_NO_CALLER_SAVED_RETURN_AX_XMM0:
22184 : case ABI_NO_CALLER_SAVED_RETURN_XMM0:
22185 : case ABI_NO_CALLER_SAVED_RETURN_XMM0_XMM1:
22186 : /* These ABIs return some values in SSE registers and preserve
22187 : the rest. The return value registers (XMM0 and possibly XMM1)
22188 : are fully rather than partially call-clobbered. */
22189 : return false;
22190 :
22191 0 : default:
22192 0 : gcc_unreachable ();
22193 : }
22194 :
22195 2462278610 : return SSE_REGNO_P (regno) && GET_MODE_SIZE (mode) > 16;
22196 : }
22197 :
22198 : /* A subroutine of ix86_modes_tieable_p. Return true if MODE is a
22199 : tieable integer mode. */
22200 :
22201 : static bool
22202 53952716 : ix86_tieable_integer_mode_p (machine_mode mode)
22203 : {
22204 53952716 : switch (mode)
22205 : {
22206 : case E_HImode:
22207 : case E_SImode:
22208 : return true;
22209 :
22210 5374939 : case E_QImode:
22211 5374939 : return TARGET_64BIT || !TARGET_PARTIAL_REG_STALL;
22212 :
22213 10713224 : case E_DImode:
22214 10713224 : return TARGET_64BIT;
22215 :
22216 : default:
22217 : return false;
22218 : }
22219 : }
22220 :
22221 : /* Implement TARGET_MODES_TIEABLE_P.
22222 :
22223 : Return true if MODE1 is accessible in a register that can hold MODE2
22224 : without copying. That is, all register classes that can hold MODE2
22225 : can also hold MODE1. */
22226 :
22227 : static bool
22228 34840210 : ix86_modes_tieable_p (machine_mode mode1, machine_mode mode2)
22229 : {
22230 34840210 : if (mode1 == mode2)
22231 : return true;
22232 :
22233 34752847 : if (ix86_tieable_integer_mode_p (mode1)
22234 34752847 : && ix86_tieable_integer_mode_p (mode2))
22235 : return true;
22236 :
22237 : /* MODE2 being XFmode implies fp stack or general regs, which means we
22238 : can tie any smaller floating point modes to it. Note that we do not
22239 : tie this with TFmode. */
22240 25360455 : if (mode2 == XFmode)
22241 4314 : return mode1 == SFmode || mode1 == DFmode;
22242 :
22243 : /* MODE2 being DFmode implies fp stack, general or sse regs, which means
22244 : that we can tie it with SFmode. */
22245 25356141 : if (mode2 == DFmode)
22246 250259 : return mode1 == SFmode;
22247 :
22248 : /* If MODE2 is only appropriate for an SSE register, then tie with
22249 : any vector modes or scalar floating point modes acceptable to SSE
22250 : registers, excluding scalar integer modes with SUBREG:
22251 : (subreg:QI (reg:TI 99) 0))
22252 : (subreg:HI (reg:TI 99) 0))
22253 : (subreg:SI (reg:TI 99) 0))
22254 : (subreg:DI (reg:TI 99) 0))
22255 : to avoid unnecessary move from SSE register to integer register.
22256 : */
22257 25105882 : if (GET_MODE_SIZE (mode2) >= 16
22258 38987410 : && (GET_MODE_SIZE (mode1) == GET_MODE_SIZE (mode2)
22259 13701493 : || ((VECTOR_MODE_P (mode1) || SCALAR_FLOAT_MODE_P (mode1))
22260 512530 : && GET_MODE_SIZE (mode1) <= GET_MODE_SIZE (mode2)))
22261 31153269 : && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode2))
22262 5607284 : return ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode1);
22263 :
22264 : /* If MODE2 is appropriate for an MMX register, then tie
22265 : with any other mode acceptable to MMX registers. */
22266 19498598 : if (GET_MODE_SIZE (mode2) == 8
22267 19498598 : && ix86_hard_regno_mode_ok (FIRST_MMX_REG, mode2))
22268 3327403 : return (GET_MODE_SIZE (mode1) == 8
22269 3327403 : && ix86_hard_regno_mode_ok (FIRST_MMX_REG, mode1));
22270 :
22271 : /* SCmode and DImode can be tied. */
22272 16171195 : if ((mode1 == E_SCmode && mode2 == E_DImode)
22273 16171195 : || (mode1 == E_DImode && mode2 == E_SCmode))
22274 108 : return TARGET_64BIT;
22275 :
22276 : /* [SD]Cmode and V2[SD]Fmode modes can be tied. */
22277 16171087 : if ((mode1 == E_SCmode && mode2 == E_V2SFmode)
22278 16171087 : || (mode1 == E_V2SFmode && mode2 == E_SCmode)
22279 16171087 : || (mode1 == E_DCmode && mode2 == E_V2DFmode)
22280 16171087 : || (mode1 == E_V2DFmode && mode2 == E_DCmode))
22281 0 : return true;
22282 :
22283 : return false;
22284 : }
22285 :
22286 : /* Return the cost of moving between two registers of mode MODE. */
22287 :
22288 : static int
22289 29690254 : ix86_set_reg_reg_cost (machine_mode mode)
22290 : {
22291 29690254 : unsigned int units = UNITS_PER_WORD;
22292 :
22293 29690254 : switch (GET_MODE_CLASS (mode))
22294 : {
22295 : default:
22296 : break;
22297 :
22298 : case MODE_CC:
22299 29690254 : units = GET_MODE_SIZE (CCmode);
22300 : break;
22301 :
22302 1166133 : case MODE_FLOAT:
22303 1166133 : if ((TARGET_SSE && mode == TFmode)
22304 675566 : || (TARGET_80387 && mode == XFmode)
22305 213010 : || ((TARGET_80387 || TARGET_SSE2) && mode == DFmode)
22306 143734 : || ((TARGET_80387 || TARGET_SSE) && mode == SFmode))
22307 2300734 : units = GET_MODE_SIZE (mode);
22308 : break;
22309 :
22310 1326216 : case MODE_COMPLEX_FLOAT:
22311 1326216 : if ((TARGET_SSE && mode == TCmode)
22312 889090 : || (TARGET_80387 && mode == XCmode)
22313 451842 : || ((TARGET_80387 || TARGET_SSE2) && mode == DCmode)
22314 14560 : || ((TARGET_80387 || TARGET_SSE) && mode == SCmode))
22315 2645884 : units = GET_MODE_SIZE (mode);
22316 : break;
22317 :
22318 18979377 : case MODE_VECTOR_INT:
22319 18979377 : case MODE_VECTOR_FLOAT:
22320 18979377 : if ((TARGET_AVX512F && VALID_AVX512F_REG_MODE (mode))
22321 18883667 : || (TARGET_AVX && VALID_AVX256_REG_MODE (mode))
22322 18712247 : || (TARGET_SSE2 && VALID_SSE2_REG_MODE (mode))
22323 16045611 : || (TARGET_SSE && VALID_SSE_REG_MODE (mode))
22324 14720980 : || ((TARGET_MMX || TARGET_MMX_WITH_SSE)
22325 14675212 : && VALID_MMX_REG_MODE (mode)))
22326 8624688 : units = GET_MODE_SIZE (mode);
22327 : }
22328 :
22329 : /* Return the cost of moving between two registers of mode MODE,
22330 : assuming that the move will be in pieces of at most UNITS bytes. */
22331 29690254 : return COSTS_N_INSNS (CEIL (GET_MODE_SIZE (mode), units));
22332 : }
22333 :
22334 : /* Return cost of vector operation in MODE given that scalar version has
22335 : COST. */
22336 :
22337 : static int
22338 2873411586 : ix86_vec_cost (machine_mode mode, int cost)
22339 : {
22340 2873411586 : if (!VECTOR_MODE_P (mode))
22341 : return cost;
22342 :
22343 2873171211 : if (GET_MODE_BITSIZE (mode) == 128
22344 2873171211 : && TARGET_SSE_SPLIT_REGS)
22345 2862350 : return cost * GET_MODE_BITSIZE (mode) / 64;
22346 2871740036 : else if (GET_MODE_BITSIZE (mode) > 128
22347 2871740036 : && TARGET_AVX256_SPLIT_REGS)
22348 1674608 : return cost * GET_MODE_BITSIZE (mode) / 128;
22349 2870902732 : else if (GET_MODE_BITSIZE (mode) > 256
22350 2870902732 : && TARGET_AVX512_SPLIT_REGS)
22351 305596 : return cost * GET_MODE_BITSIZE (mode) / 256;
22352 : return cost;
22353 : }
22354 :
22355 : /* Return cost of vec_widen_<s>mult_hi/lo_<mode>,
22356 : vec_widen_<s>mul_hi/lo_<mode> is only available for VI124_AVX2. */
22357 : static int
22358 1221 : ix86_widen_mult_cost (const struct processor_costs *cost,
22359 : enum machine_mode mode, bool uns_p)
22360 : {
22361 1221 : gcc_assert (GET_MODE_CLASS (mode) == MODE_VECTOR_INT);
22362 1221 : int extra_cost = 0;
22363 1221 : int basic_cost = 0;
22364 1221 : switch (mode)
22365 : {
22366 130 : case V8HImode:
22367 130 : case V16HImode:
22368 130 : if (!uns_p || mode == V16HImode)
22369 55 : extra_cost = cost->sse_op * 2;
22370 130 : basic_cost = cost->mulss * 2 + cost->sse_op * 4;
22371 130 : break;
22372 219 : case V4SImode:
22373 219 : case V8SImode:
22374 : /* pmulhw/pmullw can be used. */
22375 219 : basic_cost = cost->mulss * 2 + cost->sse_op * 2;
22376 219 : break;
22377 804 : case V2DImode:
22378 : /* pmuludq under sse2, pmuldq under sse4.1, for sign_extend,
22379 : require extra 4 mul, 4 add, 4 cmp and 2 shift. */
22380 804 : if (!TARGET_SSE4_1 && !uns_p)
22381 488 : extra_cost = (cost->mulss + cost->sse_op + cost->sse_op) * 4
22382 488 : + cost->sse_op * 2;
22383 : /* Fallthru. */
22384 860 : case V4DImode:
22385 860 : basic_cost = cost->mulss * 2 + cost->sse_op * 4;
22386 860 : break;
22387 : default:
22388 : /* Not implemented. */
22389 : return 100;
22390 : }
22391 1209 : return ix86_vec_cost (mode, basic_cost + extra_cost);
22392 : }
22393 :
22394 : /* Return cost of multiplication in MODE. */
22395 :
22396 : static int
22397 1226179246 : ix86_multiplication_cost (const struct processor_costs *cost,
22398 : enum machine_mode mode)
22399 : {
22400 1226179246 : machine_mode inner_mode = mode;
22401 1226179246 : if (VECTOR_MODE_P (mode))
22402 1225145607 : inner_mode = GET_MODE_INNER (mode);
22403 :
22404 1226179246 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
22405 755145 : return inner_mode == DFmode ? cost->mulsd : cost->mulss;
22406 1225424101 : else if (X87_FLOAT_MODE_P (mode))
22407 166485 : return cost->fmul;
22408 1225257616 : else if (FLOAT_MODE_P (mode))
22409 241756 : return ix86_vec_cost (mode,
22410 241756 : inner_mode == DFmode ? cost->mulsd : cost->mulss);
22411 1225015860 : else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
22412 : {
22413 1224928025 : int nmults, nops;
22414 : /* Cost of reading the memory. */
22415 1224928025 : int extra;
22416 :
22417 1224928025 : switch (mode)
22418 : {
22419 19289754 : case V4QImode:
22420 19289754 : case V8QImode:
22421 : /* Partial V*QImode is emulated with 4-6 insns. */
22422 19289754 : nmults = 1;
22423 19289754 : nops = 3;
22424 19289754 : extra = 0;
22425 :
22426 19289754 : if (TARGET_AVX512BW && TARGET_AVX512VL)
22427 : ;
22428 19176928 : else if (TARGET_AVX2)
22429 : nops += 2;
22430 18672836 : else if (TARGET_XOP)
22431 10040 : extra += COSTS_N_INSNS (cost->sse_load[2]) / 2;
22432 : else
22433 : {
22434 18662796 : nops += 1;
22435 18662796 : extra += COSTS_N_INSNS (cost->sse_load[2]) / 2;
22436 : }
22437 19289754 : goto do_qimode;
22438 :
22439 9645360 : case V16QImode:
22440 : /* V*QImode is emulated with 4-11 insns. */
22441 9645360 : nmults = 1;
22442 9645360 : nops = 3;
22443 9645360 : extra = 0;
22444 :
22445 9645360 : if (TARGET_AVX2 && !TARGET_PREFER_AVX128)
22446 : {
22447 306247 : if (!(TARGET_AVX512BW && TARGET_AVX512VL))
22448 250120 : nops += 3;
22449 : }
22450 9339113 : else if (TARGET_XOP)
22451 : {
22452 5464 : nmults += 1;
22453 5464 : nops += 2;
22454 5464 : extra += COSTS_N_INSNS (cost->sse_load[2]) / 2;
22455 : }
22456 : else
22457 : {
22458 9333649 : nmults += 1;
22459 9333649 : nops += 4;
22460 9333649 : extra += COSTS_N_INSNS (cost->sse_load[2]) / 2;
22461 : }
22462 9645360 : goto do_qimode;
22463 :
22464 9643732 : case V32QImode:
22465 9643732 : nmults = 1;
22466 9643732 : nops = 3;
22467 9643732 : extra = 0;
22468 :
22469 9643732 : if (!TARGET_AVX512BW || TARGET_PREFER_AVX256)
22470 : {
22471 9560208 : nmults += 1;
22472 9560208 : nops += 4;
22473 : /* 2 loads, so no division by 2. */
22474 9560208 : extra += COSTS_N_INSNS (cost->sse_load[3]);
22475 : }
22476 9643732 : goto do_qimode;
22477 :
22478 9643253 : case V64QImode:
22479 9643253 : nmults = 2;
22480 9643253 : nops = 9;
22481 : /* 2 loads of each size, so no division by 2. */
22482 9643253 : extra = COSTS_N_INSNS (cost->sse_load[3] + cost->sse_load[4]);
22483 :
22484 48222099 : do_qimode:
22485 48222099 : return ix86_vec_cost (mode, cost->mulss * nmults
22486 48222099 : + cost->sse_op * nops) + extra;
22487 :
22488 41241292 : case V4SImode:
22489 : /* pmulld is used in this case. No emulation is needed. */
22490 41241292 : if (TARGET_SSE4_1)
22491 2242404 : goto do_native;
22492 : /* V4SImode is emulated with 7 insns. */
22493 : else
22494 38998888 : return ix86_vec_cost (mode, cost->mulss * 2 + cost->sse_op * 5);
22495 :
22496 166693886 : case V2DImode:
22497 166693886 : case V4DImode:
22498 : /* vpmullq is used in this case. No emulation is needed. */
22499 166693886 : if (TARGET_AVX512DQ && TARGET_AVX512VL)
22500 628707 : goto do_native;
22501 : /* V*DImode is emulated with 6-8 insns. */
22502 166065179 : else if (TARGET_XOP && mode == V2DImode)
22503 55100 : return ix86_vec_cost (mode, cost->mulss * 2 + cost->sse_op * 4);
22504 : /* FALLTHRU */
22505 249293653 : case V8DImode:
22506 : /* vpmullq is used in this case. No emulation is needed. */
22507 249293653 : if (TARGET_AVX512DQ && mode == V8DImode)
22508 405930 : goto do_native;
22509 : else
22510 248887723 : return ix86_vec_cost (mode, cost->mulss * 3 + cost->sse_op * 5);
22511 :
22512 888764215 : default:
22513 888764215 : do_native:
22514 888764215 : return ix86_vec_cost (mode, cost->mulss);
22515 : }
22516 : }
22517 : else
22518 175662 : return (cost->mult_init[MODE_INDEX (mode)] + cost->mult_bit * 7);
22519 : }
22520 :
22521 : /* Return cost of multiplication in MODE. */
22522 :
22523 : static int
22524 74209814 : ix86_division_cost (const struct processor_costs *cost,
22525 : enum machine_mode mode)
22526 : {
22527 74209814 : machine_mode inner_mode = mode;
22528 74209814 : if (VECTOR_MODE_P (mode))
22529 54371057 : inner_mode = GET_MODE_INNER (mode);
22530 :
22531 74209814 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
22532 242794 : return inner_mode == DFmode ? cost->divsd : cost->divss;
22533 73967020 : else if (X87_FLOAT_MODE_P (mode))
22534 46719 : return cost->fdiv;
22535 73920301 : else if (FLOAT_MODE_P (mode))
22536 22226 : return ix86_vec_cost (mode,
22537 22226 : inner_mode == DFmode ? cost->divsd : cost->divss);
22538 : else
22539 82926004 : return cost->divide[MODE_INDEX (mode)];
22540 : }
22541 :
22542 : /* Return cost of shift in MODE.
22543 : If CONSTANT_OP1 is true, the op1 value is known and set in OP1_VAL.
22544 : AND_IN_OP1 specify in op1 is result of AND and SHIFT_AND_TRUNCATE
22545 : if op1 is a result of subreg.
22546 :
22547 : SKIP_OP0/1 is set to true if cost of OP0/1 should be ignored. */
22548 :
22549 : static int
22550 787382654 : ix86_shift_rotate_cost (const struct processor_costs *cost,
22551 : enum rtx_code code,
22552 : enum machine_mode mode, bool constant_op1,
22553 : HOST_WIDE_INT op1_val,
22554 : bool and_in_op1,
22555 : bool shift_and_truncate,
22556 : bool *skip_op0, bool *skip_op1)
22557 : {
22558 787382654 : if (skip_op0)
22559 787295944 : *skip_op0 = *skip_op1 = false;
22560 :
22561 787382654 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
22562 : {
22563 404360045 : int count;
22564 : /* Cost of reading the memory. */
22565 404360045 : int extra;
22566 :
22567 404360045 : switch (mode)
22568 : {
22569 6137657 : case V4QImode:
22570 6137657 : case V8QImode:
22571 6137657 : if (TARGET_AVX2)
22572 : /* Use vpbroadcast. */
22573 196355 : extra = cost->sse_op;
22574 : else
22575 5941302 : extra = COSTS_N_INSNS (cost->sse_load[2]) / 2;
22576 :
22577 6137657 : if (constant_op1)
22578 : {
22579 6137627 : if (code == ASHIFTRT)
22580 : {
22581 190 : count = 4;
22582 190 : extra *= 2;
22583 : }
22584 : else
22585 : count = 2;
22586 : }
22587 30 : else if (TARGET_AVX512BW && TARGET_AVX512VL)
22588 30 : return ix86_vec_cost (mode, cost->sse_op * 4);
22589 0 : else if (TARGET_SSE4_1)
22590 : count = 5;
22591 0 : else if (code == ASHIFTRT)
22592 : count = 6;
22593 : else
22594 0 : count = 5;
22595 6137627 : return ix86_vec_cost (mode, cost->sse_op * count) + extra;
22596 :
22597 3072004 : case V16QImode:
22598 3072004 : if (TARGET_XOP)
22599 : {
22600 : /* For XOP we use vpshab, which requires a broadcast of the
22601 : value to the variable shift insn. For constants this
22602 : means a V16Q const in mem; even when we can perform the
22603 : shift with one insn set the cost to prefer paddb. */
22604 3573 : if (constant_op1)
22605 : {
22606 2614 : extra = COSTS_N_INSNS (cost->sse_load[2]) / 2;
22607 2614 : return ix86_vec_cost (mode, cost->sse_op) + extra;
22608 : }
22609 : else
22610 : {
22611 959 : count = (code == ASHIFT) ? 3 : 4;
22612 959 : return ix86_vec_cost (mode, cost->sse_op * count);
22613 : }
22614 : }
22615 : /* FALLTHRU */
22616 6136954 : case V32QImode:
22617 6136954 : if (TARGET_GFNI && constant_op1)
22618 : {
22619 : /* Use vgf2p8affine. One extra load for the mask, but in a loop
22620 : with enough registers it will be moved out. So for now don't
22621 : account the constant mask load. This is not quite right
22622 : for non loop vectorization. */
22623 12074 : extra = 0;
22624 12074 : return ix86_vec_cost (mode, cost->sse_op) + extra;
22625 : }
22626 6124880 : if (TARGET_AVX2)
22627 : /* Use vpbroadcast. */
22628 189078 : extra = cost->sse_op;
22629 : else
22630 5935802 : extra = COSTS_N_INSNS (mode == V16QImode
22631 : ? cost->sse_load[2]
22632 5935802 : : cost->sse_load[3]) / 2;
22633 :
22634 6124880 : if (constant_op1)
22635 : {
22636 6124656 : if (code == ASHIFTRT)
22637 : {
22638 269 : count = 4;
22639 269 : extra *= 2;
22640 : }
22641 : else
22642 : count = 2;
22643 : }
22644 224 : else if (TARGET_AVX512BW
22645 76 : && ((mode == V32QImode && !TARGET_PREFER_AVX256)
22646 38 : || (mode == V16QImode && TARGET_AVX512VL
22647 38 : && !TARGET_PREFER_AVX128)))
22648 76 : return ix86_vec_cost (mode, cost->sse_op * 4);
22649 148 : else if (TARGET_AVX2
22650 0 : && mode == V16QImode && !TARGET_PREFER_AVX128)
22651 : count = 6;
22652 148 : else if (TARGET_SSE4_1)
22653 : count = 9;
22654 148 : else if (code == ASHIFTRT)
22655 : count = 10;
22656 : else
22657 76 : count = 9;
22658 6124804 : return ix86_vec_cost (mode, cost->sse_op * count) + extra;
22659 :
22660 3068772 : case V64QImode:
22661 : /* Ignore the mask load for GF2P8AFFINEQB. */
22662 3068772 : extra = 0;
22663 3068772 : return ix86_vec_cost (mode, cost->sse_op) + extra;
22664 :
22665 55322234 : case V2DImode:
22666 55322234 : case V4DImode:
22667 : /* V*DImode arithmetic right shift is emulated. */
22668 55322234 : if (code == ASHIFTRT && !TARGET_AVX512VL)
22669 : {
22670 1532 : if (constant_op1)
22671 : {
22672 771 : if (op1_val == 63)
22673 500 : count = TARGET_SSE4_2 ? 1 : 2;
22674 570 : else if (TARGET_XOP)
22675 : count = 2;
22676 271 : else if (TARGET_SSE4_1)
22677 : count = 3;
22678 : else
22679 291 : count = 4;
22680 : }
22681 761 : else if (TARGET_XOP)
22682 : count = 3;
22683 96 : else if (TARGET_SSE4_2)
22684 : count = 4;
22685 : else
22686 1532 : count = 5;
22687 :
22688 1532 : return ix86_vec_cost (mode, cost->sse_op * count);
22689 : }
22690 : /* FALLTHRU */
22691 389011557 : default:
22692 389011557 : return ix86_vec_cost (mode, cost->sse_op);
22693 : }
22694 : }
22695 :
22696 774749071 : if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
22697 : {
22698 195895266 : if (constant_op1)
22699 : {
22700 195860504 : if (op1_val > 32)
22701 139162454 : return cost->shift_const + COSTS_N_INSNS (2);
22702 : else
22703 56698050 : return cost->shift_const * 2;
22704 : }
22705 : else
22706 : {
22707 34762 : if (and_in_op1)
22708 63 : return cost->shift_var * 2;
22709 : else
22710 34699 : return cost->shift_var * 6 + COSTS_N_INSNS (2);
22711 : }
22712 : }
22713 : else
22714 : {
22715 187127343 : if (constant_op1)
22716 186360639 : return cost->shift_const;
22717 766704 : else if (shift_and_truncate)
22718 : {
22719 23554 : if (skip_op0)
22720 23554 : *skip_op0 = *skip_op1 = true;
22721 : /* Return the cost after shift-and truncation. */
22722 23554 : return cost->shift_var;
22723 : }
22724 : else
22725 743150 : return cost->shift_var;
22726 : }
22727 : }
22728 :
22729 : static int
22730 150960124 : ix86_insn_cost (rtx_insn *insn, bool speed)
22731 : {
22732 150960124 : int insn_cost = 0;
22733 : /* Add extra cost to avoid post_reload late_combine revert
22734 : the optimization did in pass_rpad. */
22735 150960124 : if (reload_completed
22736 4585319 : && ix86_rpad_gate ()
22737 251500 : && recog_memoized (insn) >= 0
22738 151211353 : && get_attr_avx_partial_xmm_update (insn)
22739 : == AVX_PARTIAL_XMM_UPDATE_TRUE)
22740 : insn_cost += COSTS_N_INSNS (3);
22741 :
22742 150960124 : rtx pat = PATTERN (insn);
22743 : /* A USE of a memory is more expensive than a use of a REG.
22744 : For example *<absneg>mode2_1's use of a signbit mask. */
22745 150960124 : if (GET_CODE (pat) == PARALLEL)
22746 : {
22747 47215490 : for (int i = 0; i < XVECLEN (pat, 0); i++)
22748 : {
22749 31716122 : rtx x = XVECEXP (pat, 0, i);
22750 31716122 : if (GET_CODE (x) == USE && MEM_P (XEXP (x, 0)))
22751 63694 : insn_cost += !speed ? COSTS_N_BYTES (4)
22752 29878 : : TARGET_64BIT ? COSTS_N_INSNS (1) + 1
22753 : : COSTS_N_INSNS (3) + 1;
22754 : }
22755 : }
22756 : /* Cost *h{add,sub}<mode>[_low] directly as pattern cost for the
22757 : variants with outer vec_concat are artificially low. */
22758 150960124 : if (INSN_CODE (insn) >= 0
22759 150960124 : && get_attr_cost_special (insn) == COST_SPECIAL_HADDSUB)
22760 632 : return insn_cost + ix86_vec_cost (GET_MODE (pat),
22761 316 : (speed ? ix86_tune_cost
22762 316 : : &ix86_size_cost)->addss);
22763 :
22764 150959808 : return insn_cost + pattern_cost (pat, speed);
22765 : }
22766 :
22767 : /* Return cost of SSE/AVX FP->FP conversion (extensions and truncates). */
22768 :
22769 : static int
22770 765629 : vec_fp_conversion_cost (const struct processor_costs *cost, int size)
22771 : {
22772 765629 : if (size < 128)
22773 760399 : return cost->cvtss2sd;
22774 5230 : else if (size < 256)
22775 : {
22776 2445 : if (TARGET_SSE_SPLIT_REGS)
22777 0 : return cost->cvtss2sd * size / 64;
22778 2445 : return cost->cvtss2sd;
22779 : }
22780 2785 : if (size < 512)
22781 1483 : return cost->vcvtps2pd256;
22782 : else
22783 1302 : return cost->vcvtps2pd512;
22784 : }
22785 :
22786 : /* Return true of X is UNSPEC with UNSPEC_PCMP or UNSPEC_UNSIGNED_PCMP. */
22787 :
22788 : static bool
22789 274991 : unspec_pcmp_p (rtx x)
22790 : {
22791 274991 : return GET_CODE (x) == UNSPEC
22792 274991 : && (XINT (x, 1) == UNSPEC_PCMP || XINT (x, 1) == UNSPEC_UNSIGNED_PCMP);
22793 : }
22794 :
22795 : /* Compute a (partial) cost for rtx X. Return true if the complete
22796 : cost has been computed, and false if subexpressions should be
22797 : scanned. In either case, *TOTAL contains the cost result. */
22798 :
22799 : static bool
22800 7816587976 : ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
22801 : int *total, bool speed)
22802 : {
22803 7816587976 : rtx mask;
22804 7816587976 : enum rtx_code code = GET_CODE (x);
22805 7816587976 : enum rtx_code outer_code = (enum rtx_code) outer_code_i;
22806 4183563142 : const struct processor_costs *cost
22807 7816587976 : = speed ? ix86_tune_cost : &ix86_size_cost;
22808 7816587976 : int src_cost;
22809 :
22810 : /* Handling different vternlog variants. */
22811 7816587976 : if ((GET_MODE_SIZE (mode) == 64
22812 7816587976 : ? TARGET_AVX512F
22813 6614420634 : : (TARGET_AVX512VL
22814 6551494870 : || (TARGET_AVX512F && !TARGET_PREFER_AVX256)))
22815 177900506 : && GET_MODE_SIZE (mode) >= 16
22816 120798714 : && outer_code_i == SET
22817 7863257044 : && ternlog_operand (x, mode))
22818 : {
22819 33644 : rtx args[3];
22820 :
22821 33644 : args[0] = NULL_RTX;
22822 33644 : args[1] = NULL_RTX;
22823 33644 : args[2] = NULL_RTX;
22824 33644 : int idx = ix86_ternlog_idx (x, args);
22825 33644 : gcc_assert (idx >= 0);
22826 :
22827 33644 : *total = cost->sse_op;
22828 134576 : for (int i = 0; i != 3; i++)
22829 100932 : if (args[i])
22830 71162 : *total += rtx_cost (args[i], GET_MODE (args[i]), UNSPEC, i, speed);
22831 33644 : return true;
22832 : }
22833 :
22834 :
22835 7816554332 : switch (code)
22836 : {
22837 48233762 : case SET:
22838 48233762 : if (register_operand (SET_DEST (x), VOIDmode)
22839 48233762 : && register_operand (SET_SRC (x), VOIDmode))
22840 : {
22841 29690254 : *total = ix86_set_reg_reg_cost (GET_MODE (SET_DEST (x)));
22842 29690254 : return true;
22843 : }
22844 :
22845 18543508 : if (register_operand (SET_SRC (x), VOIDmode))
22846 : /* Avoid potentially incorrect high cost from rtx_costs
22847 : for non-tieable SUBREGs. */
22848 : src_cost = 0;
22849 : else
22850 : {
22851 15724041 : src_cost = rtx_cost (SET_SRC (x), mode, SET, 1, speed);
22852 :
22853 15724041 : if (CONSTANT_P (SET_SRC (x)))
22854 : /* Constant costs assume a base value of COSTS_N_INSNS (1) and add
22855 : a small value, possibly zero for cheap constants. */
22856 7117765 : src_cost += COSTS_N_INSNS (1);
22857 : }
22858 :
22859 18543508 : *total = src_cost + rtx_cost (SET_DEST (x), mode, SET, 0, speed);
22860 18543508 : return true;
22861 :
22862 2878086508 : case CONST_INT:
22863 2878086508 : case CONST:
22864 2878086508 : case LABEL_REF:
22865 2878086508 : case SYMBOL_REF:
22866 2878086508 : if (x86_64_immediate_operand (x, VOIDmode))
22867 2263286580 : *total = 0;
22868 614799928 : else if (TARGET_64BIT && x86_64_zext_immediate_operand (x, VOIDmode))
22869 : /* Consider the zext constants slightly more expensive, as they
22870 : can't appear in most instructions. */
22871 28464446 : *total = 1;
22872 : else
22873 : /* movabsq is slightly more expensive than a simple instruction. */
22874 586335482 : *total = COSTS_N_INSNS (1) + 1;
22875 : return true;
22876 :
22877 7612868 : case CONST_DOUBLE:
22878 7612868 : if (IS_STACK_MODE (mode))
22879 1295581 : switch (standard_80387_constant_p (x))
22880 : {
22881 : case -1:
22882 : case 0:
22883 : break;
22884 277231 : case 1: /* 0.0 */
22885 277231 : *total = 1;
22886 277231 : return true;
22887 468133 : default: /* Other constants */
22888 468133 : *total = 2;
22889 468133 : return true;
22890 : }
22891 : /* FALLTHRU */
22892 :
22893 14850013 : case CONST_VECTOR:
22894 14850013 : switch (standard_sse_constant_p (x, mode))
22895 : {
22896 : case 0:
22897 : break;
22898 4316784 : case 1: /* 0: xor eliminates false dependency */
22899 4316784 : *total = 0;
22900 4316784 : return true;
22901 211334 : default: /* -1: cmp contains false dependency */
22902 211334 : *total = 1;
22903 211334 : return true;
22904 : }
22905 : /* FALLTHRU */
22906 :
22907 11332741 : case CONST_WIDE_INT:
22908 : /* Fall back to (MEM (SYMBOL_REF)), since that's where
22909 : it'll probably end up. Add a penalty for size. */
22910 22665482 : *total = (COSTS_N_INSNS (1)
22911 22437664 : + (!TARGET_64BIT && flag_pic)
22912 22665482 : + (GET_MODE_SIZE (mode) <= 4
22913 19859009 : ? 0 : GET_MODE_SIZE (mode) <= 8 ? 1 : 2));
22914 11332741 : return true;
22915 :
22916 22854985 : case ZERO_EXTEND:
22917 : /* The zero extensions is often completely free on x86_64, so make
22918 : it as cheap as possible. */
22919 22854985 : if (TARGET_64BIT && mode == DImode
22920 4940232 : && GET_MODE (XEXP (x, 0)) == SImode)
22921 3009539 : *total = 1;
22922 19845446 : else if (TARGET_ZERO_EXTEND_WITH_AND)
22923 0 : *total = cost->add;
22924 : else
22925 19845446 : *total = cost->movzx;
22926 : return false;
22927 :
22928 2798073 : case SIGN_EXTEND:
22929 2798073 : *total = cost->movsx;
22930 2798073 : return false;
22931 :
22932 647632573 : case ASHIFT:
22933 647632573 : if (SCALAR_INT_MODE_P (mode)
22934 250238364 : && GET_MODE_SIZE (mode) < UNITS_PER_WORD
22935 691425896 : && CONST_INT_P (XEXP (x, 1)))
22936 : {
22937 43604779 : HOST_WIDE_INT value = INTVAL (XEXP (x, 1));
22938 43604779 : if (value == 1)
22939 : {
22940 2515787 : *total = cost->add;
22941 2515787 : return false;
22942 : }
22943 41088992 : if ((value == 2 || value == 3)
22944 4618388 : && cost->lea <= cost->shift_const)
22945 : {
22946 2180400 : *total = cost->lea;
22947 2180400 : return false;
22948 : }
22949 : }
22950 : /* FALLTHRU */
22951 :
22952 787295944 : case ROTATE:
22953 787295944 : case ASHIFTRT:
22954 787295944 : case LSHIFTRT:
22955 787295944 : case ROTATERT:
22956 787295944 : bool skip_op0, skip_op1;
22957 787295944 : *total = ix86_shift_rotate_cost (cost, code, mode,
22958 787295944 : CONSTANT_P (XEXP (x, 1)),
22959 : CONST_INT_P (XEXP (x, 1))
22960 : ? INTVAL (XEXP (x, 1)) : -1,
22961 : GET_CODE (XEXP (x, 1)) == AND,
22962 787295944 : SUBREG_P (XEXP (x, 1))
22963 787295944 : && GET_CODE (XEXP (XEXP (x, 1),
22964 : 0)) == AND,
22965 : &skip_op0, &skip_op1);
22966 787295944 : if (skip_op0 || skip_op1)
22967 : {
22968 23554 : if (!skip_op0)
22969 0 : *total += rtx_cost (XEXP (x, 0), mode, code, 0, speed);
22970 23554 : if (!skip_op1)
22971 0 : *total += rtx_cost (XEXP (x, 1), mode, code, 0, speed);
22972 23554 : return true;
22973 : }
22974 : return false;
22975 :
22976 229793 : case FMA:
22977 229793 : {
22978 229793 : rtx sub;
22979 :
22980 229793 : gcc_assert (FLOAT_MODE_P (mode));
22981 229793 : gcc_assert (TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F);
22982 :
22983 459586 : *total = ix86_vec_cost (mode,
22984 229793 : GET_MODE_INNER (mode) == SFmode
22985 : ? cost->fmass : cost->fmasd);
22986 229793 : *total += rtx_cost (XEXP (x, 1), mode, FMA, 1, speed);
22987 :
22988 : /* Negate in op0 or op2 is free: FMS, FNMA, FNMS. */
22989 229793 : sub = XEXP (x, 0);
22990 229793 : if (GET_CODE (sub) == NEG)
22991 51481 : sub = XEXP (sub, 0);
22992 229793 : *total += rtx_cost (sub, mode, FMA, 0, speed);
22993 :
22994 229793 : sub = XEXP (x, 2);
22995 229793 : if (GET_CODE (sub) == NEG)
22996 40537 : sub = XEXP (sub, 0);
22997 229793 : *total += rtx_cost (sub, mode, FMA, 2, speed);
22998 229793 : return true;
22999 : }
23000 :
23001 1786058958 : case MULT:
23002 1786058958 : if (!FLOAT_MODE_P (mode) && !VECTOR_MODE_P (mode))
23003 : {
23004 560143616 : rtx op0 = XEXP (x, 0);
23005 560143616 : rtx op1 = XEXP (x, 1);
23006 560143616 : int nbits;
23007 560143616 : if (CONST_INT_P (XEXP (x, 1)))
23008 : {
23009 541581165 : unsigned HOST_WIDE_INT value = INTVAL (XEXP (x, 1));
23010 1098976384 : for (nbits = 0; value != 0; value &= value - 1)
23011 557395219 : nbits++;
23012 : }
23013 : else
23014 : /* This is arbitrary. */
23015 : nbits = 7;
23016 :
23017 : /* Compute costs correctly for widening multiplication. */
23018 560143616 : if ((GET_CODE (op0) == SIGN_EXTEND || GET_CODE (op0) == ZERO_EXTEND)
23019 565757701 : && GET_MODE_SIZE (GET_MODE (XEXP (op0, 0))) * 2
23020 5614085 : == GET_MODE_SIZE (mode))
23021 : {
23022 5600624 : int is_mulwiden = 0;
23023 5600624 : machine_mode inner_mode = GET_MODE (op0);
23024 :
23025 5600624 : if (GET_CODE (op0) == GET_CODE (op1))
23026 5499808 : is_mulwiden = 1, op1 = XEXP (op1, 0);
23027 100816 : else if (CONST_INT_P (op1))
23028 : {
23029 90755 : if (GET_CODE (op0) == SIGN_EXTEND)
23030 40383 : is_mulwiden = trunc_int_for_mode (INTVAL (op1), inner_mode)
23031 40383 : == INTVAL (op1);
23032 : else
23033 50372 : is_mulwiden = !(INTVAL (op1) & ~GET_MODE_MASK (inner_mode));
23034 : }
23035 :
23036 5590563 : if (is_mulwiden)
23037 5590563 : op0 = XEXP (op0, 0), mode = GET_MODE (op0);
23038 : }
23039 :
23040 560143616 : int mult_init;
23041 : // Double word multiplication requires 3 mults and 2 adds.
23042 1135985362 : if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
23043 : {
23044 337074597 : mult_init = 3 * cost->mult_init[MODE_INDEX (word_mode)]
23045 337074597 : + 2 * cost->add;
23046 337074597 : nbits *= 3;
23047 : }
23048 384774638 : else mult_init = cost->mult_init[MODE_INDEX (mode)];
23049 :
23050 1120287232 : *total = (mult_init
23051 560143616 : + nbits * cost->mult_bit
23052 560143616 : + rtx_cost (op0, mode, outer_code, opno, speed)
23053 560143616 : + rtx_cost (op1, mode, outer_code, opno, speed));
23054 :
23055 560143616 : return true;
23056 : }
23057 1225915342 : *total = ix86_multiplication_cost (cost, mode);
23058 1225915342 : return false;
23059 :
23060 74194408 : case DIV:
23061 74194408 : case UDIV:
23062 74194408 : case MOD:
23063 74194408 : case UMOD:
23064 74194408 : *total = ix86_division_cost (cost, mode);
23065 74194408 : return false;
23066 :
23067 698426759 : case PLUS:
23068 698426759 : if (GET_MODE_CLASS (mode) == MODE_INT
23069 956414884 : && GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
23070 : {
23071 143141347 : if (GET_CODE (XEXP (x, 0)) == PLUS
23072 3434093 : && GET_CODE (XEXP (XEXP (x, 0), 0)) == MULT
23073 816308 : && CONST_INT_P (XEXP (XEXP (XEXP (x, 0), 0), 1))
23074 816283 : && CONSTANT_P (XEXP (x, 1)))
23075 : {
23076 816226 : HOST_WIDE_INT val = INTVAL (XEXP (XEXP (XEXP (x, 0), 0), 1));
23077 816226 : if (val == 2 || val == 4 || val == 8)
23078 : {
23079 816122 : *total = cost->lea;
23080 816122 : *total += rtx_cost (XEXP (XEXP (x, 0), 1), mode,
23081 : outer_code, opno, speed);
23082 816122 : *total += rtx_cost (XEXP (XEXP (XEXP (x, 0), 0), 0), mode,
23083 : outer_code, opno, speed);
23084 816122 : *total += rtx_cost (XEXP (x, 1), mode,
23085 : outer_code, opno, speed);
23086 816122 : return true;
23087 : }
23088 : }
23089 142325121 : else if (GET_CODE (XEXP (x, 0)) == MULT
23090 53132569 : && CONST_INT_P (XEXP (XEXP (x, 0), 1)))
23091 : {
23092 53068063 : HOST_WIDE_INT val = INTVAL (XEXP (XEXP (x, 0), 1));
23093 53068063 : if (val == 2 || val == 4 || val == 8)
23094 : {
23095 8064631 : *total = cost->lea;
23096 8064631 : *total += rtx_cost (XEXP (XEXP (x, 0), 0), mode,
23097 : outer_code, opno, speed);
23098 8064631 : *total += rtx_cost (XEXP (x, 1), mode,
23099 : outer_code, opno, speed);
23100 8064631 : return true;
23101 : }
23102 : }
23103 89257058 : else if (GET_CODE (XEXP (x, 0)) == PLUS)
23104 : {
23105 2617867 : rtx op = XEXP (XEXP (x, 0), 0);
23106 :
23107 : /* Add with carry, ignore the cost of adding a carry flag. */
23108 2617867 : if (ix86_carry_flag_operator (op, mode)
23109 2617867 : || ix86_carry_flag_unset_operator (op, mode))
23110 69649 : *total = cost->add;
23111 : else
23112 : {
23113 2548218 : *total = cost->lea;
23114 2548218 : *total += rtx_cost (op, mode,
23115 : outer_code, opno, speed);
23116 : }
23117 :
23118 2617867 : *total += rtx_cost (XEXP (XEXP (x, 0), 1), mode,
23119 : outer_code, opno, speed);
23120 2617867 : *total += rtx_cost (XEXP (x, 1), mode,
23121 : outer_code, opno, speed);
23122 2617867 : return true;
23123 : }
23124 : }
23125 : /* FALLTHRU */
23126 :
23127 1863704692 : case MINUS:
23128 : /* Subtract with borrow, ignore the cost of subtracting a carry flag. */
23129 1863704692 : if (GET_MODE_CLASS (mode) == MODE_INT
23130 525849171 : && GET_MODE_SIZE (mode) <= UNITS_PER_WORD
23131 237462929 : && GET_CODE (XEXP (x, 0)) == MINUS
23132 1863743677 : && (ix86_carry_flag_operator (XEXP (XEXP (x, 0), 1), mode)
23133 15621 : || ix86_carry_flag_unset_operator (XEXP (XEXP (x, 0), 1), mode)))
23134 : {
23135 23364 : *total = cost->add;
23136 23364 : *total += rtx_cost (XEXP (XEXP (x, 0), 0), mode,
23137 : outer_code, opno, speed);
23138 23364 : *total += rtx_cost (XEXP (x, 1), mode,
23139 : outer_code, opno, speed);
23140 23364 : return true;
23141 : }
23142 :
23143 1863681328 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
23144 2416367 : *total = cost->addss;
23145 1861264961 : else if (X87_FLOAT_MODE_P (mode))
23146 224315 : *total = cost->fadd;
23147 1861040646 : else if (FLOAT_MODE_P (mode))
23148 453678 : *total = ix86_vec_cost (mode, cost->addss);
23149 1860586968 : else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
23150 1225730410 : *total = ix86_vec_cost (mode, cost->sse_op);
23151 1309067141 : else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
23152 336053593 : *total = cost->add * 2;
23153 : else
23154 298802965 : *total = cost->add;
23155 : return false;
23156 :
23157 4049996 : case IOR:
23158 4049996 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
23159 3794955 : || SSE_FLOAT_MODE_P (mode))
23160 : {
23161 : /* (ior (not ...) ...) can be a single insn in AVX512. */
23162 2935 : if (GET_CODE (XEXP (x, 0)) == NOT && TARGET_AVX512F
23163 264861 : && (GET_MODE_SIZE (mode) == 64
23164 0 : || (TARGET_AVX512VL
23165 0 : && (GET_MODE_SIZE (mode) == 32
23166 0 : || GET_MODE_SIZE (mode) == 16))))
23167 : {
23168 0 : rtx right = GET_CODE (XEXP (x, 1)) != NOT
23169 0 : ? XEXP (x, 1) : XEXP (XEXP (x, 1), 0);
23170 :
23171 0 : *total = ix86_vec_cost (mode, cost->sse_op)
23172 0 : + rtx_cost (XEXP (XEXP (x, 0), 0), mode,
23173 : outer_code, opno, speed)
23174 0 : + rtx_cost (right, mode, outer_code, opno, speed);
23175 0 : return true;
23176 : }
23177 264861 : *total = ix86_vec_cost (mode, cost->sse_op);
23178 264861 : }
23179 3785135 : else if (TARGET_64BIT
23180 3491117 : && mode == TImode
23181 1718765 : && GET_CODE (XEXP (x, 0)) == ASHIFT
23182 259251 : && GET_CODE (XEXP (XEXP (x, 0), 0)) == ZERO_EXTEND
23183 257252 : && GET_MODE (XEXP (XEXP (XEXP (x, 0), 0), 0)) == DImode
23184 257252 : && CONST_INT_P (XEXP (XEXP (x, 0), 1))
23185 257252 : && INTVAL (XEXP (XEXP (x, 0), 1)) == 64
23186 257252 : && GET_CODE (XEXP (x, 1)) == ZERO_EXTEND
23187 234758 : && GET_MODE (XEXP (XEXP (x, 1), 0)) == DImode)
23188 : {
23189 : /* *concatditi3 is cheap. */
23190 234758 : rtx op0 = XEXP (XEXP (XEXP (x, 0), 0), 0);
23191 234758 : rtx op1 = XEXP (XEXP (x, 1), 0);
23192 1576 : *total = (SUBREG_P (op0) && GET_MODE (SUBREG_REG (op0)) == DFmode)
23193 234758 : ? COSTS_N_INSNS (1) /* movq. */
23194 233182 : : set_src_cost (op0, DImode, speed);
23195 2538 : *total += (SUBREG_P (op1) && GET_MODE (SUBREG_REG (op1)) == DFmode)
23196 234758 : ? COSTS_N_INSNS (1) /* movq. */
23197 232233 : : set_src_cost (op1, DImode, speed);
23198 234758 : return true;
23199 : }
23200 3550377 : else if (TARGET_64BIT
23201 3256359 : && mode == TImode
23202 1484007 : && GET_CODE (XEXP (x, 0)) == AND
23203 1423323 : && REG_P (XEXP (XEXP (x, 0), 0))
23204 1418026 : && CONST_WIDE_INT_P (XEXP (XEXP (x, 0), 1))
23205 1415283 : && CONST_WIDE_INT_NUNITS (XEXP (XEXP (x, 0), 1)) == 2
23206 1415283 : && CONST_WIDE_INT_ELT (XEXP (XEXP (x, 0), 1), 0) == -1
23207 923337 : && CONST_WIDE_INT_ELT (XEXP (XEXP (x, 0), 1), 1) == 0
23208 923337 : && GET_CODE (XEXP (x, 1)) == ASHIFT
23209 921088 : && GET_CODE (XEXP (XEXP (x, 1), 0)) == ZERO_EXTEND
23210 921088 : && GET_MODE (XEXP (XEXP (XEXP (x, 1), 0), 0)) == DImode
23211 921088 : && CONST_INT_P (XEXP (XEXP (x, 1), 1))
23212 4471465 : && INTVAL (XEXP (XEXP (x, 1), 1)) == 64)
23213 : {
23214 : /* *insvti_highpart is cheap. */
23215 921088 : rtx op = XEXP (XEXP (XEXP (x, 1), 0), 0);
23216 921088 : *total = COSTS_N_INSNS (1) + 1;
23217 1538 : *total += (SUBREG_P (op) && GET_MODE (SUBREG_REG (op)) == DFmode)
23218 921088 : ? COSTS_N_INSNS (1) /* movq. */
23219 920048 : : set_src_cost (op, DImode, speed);
23220 921088 : return true;
23221 : }
23222 5552596 : else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
23223 756068 : *total = cost->add * 2;
23224 : else
23225 1873221 : *total = cost->add;
23226 : return false;
23227 :
23228 601254 : case XOR:
23229 601254 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
23230 444289 : || SSE_FLOAT_MODE_P (mode))
23231 156965 : *total = ix86_vec_cost (mode, cost->sse_op);
23232 949005 : else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
23233 17339 : *total = cost->add * 2;
23234 : else
23235 426950 : *total = cost->add;
23236 : return false;
23237 :
23238 6511604 : case AND:
23239 6511604 : if (address_no_seg_operand (x, mode))
23240 : {
23241 13915 : *total = cost->lea;
23242 13915 : return true;
23243 : }
23244 6497689 : else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
23245 6057690 : || SSE_FLOAT_MODE_P (mode))
23246 : {
23247 : /* pandn is a single instruction. */
23248 473893 : if (GET_CODE (XEXP (x, 0)) == NOT)
23249 : {
23250 59972 : rtx right = XEXP (x, 1);
23251 :
23252 : /* (and (not ...) (not ...)) can be a single insn in AVX512. */
23253 158 : if (GET_CODE (right) == NOT && TARGET_AVX512F
23254 59972 : && (GET_MODE_SIZE (mode) == 64
23255 0 : || (TARGET_AVX512VL
23256 0 : && (GET_MODE_SIZE (mode) == 32
23257 0 : || GET_MODE_SIZE (mode) == 16))))
23258 0 : right = XEXP (right, 0);
23259 :
23260 59972 : *total = ix86_vec_cost (mode, cost->sse_op)
23261 59972 : + rtx_cost (XEXP (XEXP (x, 0), 0), mode,
23262 : outer_code, opno, speed)
23263 59972 : + rtx_cost (right, mode, outer_code, opno, speed);
23264 59972 : return true;
23265 : }
23266 413921 : else if (GET_CODE (XEXP (x, 1)) == NOT)
23267 : {
23268 2678 : *total = ix86_vec_cost (mode, cost->sse_op)
23269 2678 : + rtx_cost (XEXP (x, 0), mode,
23270 : outer_code, opno, speed)
23271 2678 : + rtx_cost (XEXP (XEXP (x, 1), 0), mode,
23272 : outer_code, opno, speed);
23273 2678 : return true;
23274 : }
23275 411243 : *total = ix86_vec_cost (mode, cost->sse_op);
23276 411243 : }
23277 12758483 : else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
23278 : {
23279 1144500 : if (TARGET_BMI && GET_CODE (XEXP (x,0)) == NOT)
23280 : {
23281 1670 : *total = cost->add * 2
23282 835 : + rtx_cost (XEXP (XEXP (x, 0), 0), mode,
23283 : outer_code, opno, speed)
23284 835 : + rtx_cost (XEXP (x, 1), mode,
23285 : outer_code, opno, speed);
23286 835 : return true;
23287 : }
23288 1143665 : else if (TARGET_BMI && GET_CODE (XEXP (x, 1)) == NOT)
23289 : {
23290 0 : *total = cost->add * 2
23291 0 : + rtx_cost (XEXP (x, 0), mode,
23292 : outer_code, opno, speed)
23293 0 : + rtx_cost (XEXP (XEXP (x, 1), 0), mode,
23294 : outer_code, opno, speed);
23295 0 : return true;
23296 : }
23297 1143665 : *total = cost->add * 2;
23298 : }
23299 4879296 : else if (TARGET_BMI && GET_CODE (XEXP (x,0)) == NOT)
23300 : {
23301 7410 : *total = cost->add
23302 3705 : + rtx_cost (XEXP (XEXP (x, 0), 0), mode,
23303 : outer_code, opno, speed)
23304 3705 : + rtx_cost (XEXP (x, 1), mode, outer_code, opno, speed);
23305 3705 : return true;
23306 : }
23307 4875591 : else if (TARGET_BMI && GET_CODE (XEXP (x,1)) == NOT)
23308 : {
23309 112 : *total = cost->add
23310 56 : + rtx_cost (XEXP (x, 0), mode, outer_code, opno, speed)
23311 56 : + rtx_cost (XEXP (XEXP (x, 1), 0), mode,
23312 : outer_code, opno, speed);
23313 56 : return true;
23314 : }
23315 : else
23316 4875535 : *total = cost->add;
23317 : return false;
23318 :
23319 547618 : case NOT:
23320 547618 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
23321 : {
23322 : /* (not (xor ...)) can be a single insn in AVX512. */
23323 0 : if (GET_CODE (XEXP (x, 0)) == XOR && TARGET_AVX512F
23324 22231 : && (GET_MODE_SIZE (mode) == 64
23325 0 : || (TARGET_AVX512VL
23326 0 : && (GET_MODE_SIZE (mode) == 32
23327 0 : || GET_MODE_SIZE (mode) == 16))))
23328 : {
23329 0 : *total = ix86_vec_cost (mode, cost->sse_op)
23330 0 : + rtx_cost (XEXP (XEXP (x, 0), 0), mode,
23331 : outer_code, opno, speed)
23332 0 : + rtx_cost (XEXP (XEXP (x, 0), 1), mode,
23333 : outer_code, opno, speed);
23334 0 : return true;
23335 : }
23336 :
23337 : // vnot is pxor -1.
23338 22231 : *total = ix86_vec_cost (mode, cost->sse_op) + 1;
23339 : }
23340 1196803 : else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
23341 46214 : *total = cost->add * 2;
23342 : else
23343 479173 : *total = cost->add;
23344 : return false;
23345 :
23346 18574124 : case NEG:
23347 18574124 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
23348 53614 : *total = cost->sse_op;
23349 18520510 : else if (X87_FLOAT_MODE_P (mode))
23350 15420 : *total = cost->fchs;
23351 18505090 : else if (FLOAT_MODE_P (mode))
23352 27424 : *total = ix86_vec_cost (mode, cost->sse_op);
23353 18477666 : else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
23354 13601367 : *total = ix86_vec_cost (mode, cost->sse_op);
23355 9904649 : else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
23356 1791190 : *total = cost->add * 3;
23357 : else
23358 3085109 : *total = cost->add;
23359 : return false;
23360 :
23361 54642510 : case COMPARE:
23362 54642510 : rtx op0, op1;
23363 54642510 : op0 = XEXP (x, 0);
23364 54642510 : op1 = XEXP (x, 1);
23365 54642510 : if (GET_CODE (op0) == ZERO_EXTRACT
23366 108312 : && XEXP (op0, 1) == const1_rtx
23367 92153 : && CONST_INT_P (XEXP (op0, 2))
23368 92117 : && op1 == const0_rtx)
23369 : {
23370 : /* This kind of construct is implemented using test[bwl].
23371 : Treat it as if we had an AND. */
23372 92117 : mode = GET_MODE (XEXP (op0, 0));
23373 184234 : *total = (cost->add
23374 92117 : + rtx_cost (XEXP (op0, 0), mode, outer_code,
23375 : opno, speed)
23376 92117 : + rtx_cost (const1_rtx, mode, outer_code, opno, speed));
23377 92117 : return true;
23378 : }
23379 :
23380 54550393 : if (GET_CODE (op0) == PLUS && rtx_equal_p (XEXP (op0, 0), op1))
23381 : {
23382 : /* This is an overflow detection, count it as a normal compare. */
23383 145087 : *total = rtx_cost (op0, GET_MODE (op0), COMPARE, 0, speed);
23384 145087 : return true;
23385 : }
23386 :
23387 54405306 : rtx geu;
23388 : /* Match x
23389 : (compare:CCC (neg:QI (geu:QI (reg:CC_CCC FLAGS_REG) (const_int 0)))
23390 : (ltu:QI (reg:CC_CCC FLAGS_REG) (const_int 0))) */
23391 54405306 : if (mode == CCCmode
23392 296822 : && GET_CODE (op0) == NEG
23393 8129 : && GET_CODE (geu = XEXP (op0, 0)) == GEU
23394 8126 : && REG_P (XEXP (geu, 0))
23395 8126 : && (GET_MODE (XEXP (geu, 0)) == CCCmode
23396 761 : || GET_MODE (XEXP (geu, 0)) == CCmode)
23397 8126 : && REGNO (XEXP (geu, 0)) == FLAGS_REG
23398 8126 : && XEXP (geu, 1) == const0_rtx
23399 8126 : && GET_CODE (op1) == LTU
23400 8126 : && REG_P (XEXP (op1, 0))
23401 8126 : && GET_MODE (XEXP (op1, 0)) == GET_MODE (XEXP (geu, 0))
23402 8126 : && REGNO (XEXP (op1, 0)) == FLAGS_REG
23403 54413432 : && XEXP (op1, 1) == const0_rtx)
23404 : {
23405 : /* This is *setcc_qi_addqi3_cconly_overflow_1_* patterns, a nop. */
23406 8126 : *total = 0;
23407 8126 : return true;
23408 : }
23409 : /* Match x
23410 : (compare:CCC (neg:QI (ltu:QI (reg:CCC FLAGS_REG) (const_int 0)))
23411 : (geu:QI (reg:CCC FLAGS_REG) (const_int 0))) */
23412 54397180 : if (mode == CCCmode
23413 288696 : && GET_CODE (op0) == NEG
23414 3 : && GET_CODE (XEXP (op0, 0)) == LTU
23415 3 : && REG_P (XEXP (XEXP (op0, 0), 0))
23416 3 : && GET_MODE (XEXP (XEXP (op0, 0), 0)) == CCCmode
23417 3 : && REGNO (XEXP (XEXP (op0, 0), 0)) == FLAGS_REG
23418 3 : && XEXP (XEXP (op0, 0), 1) == const0_rtx
23419 3 : && GET_CODE (op1) == GEU
23420 3 : && REG_P (XEXP (op1, 0))
23421 3 : && GET_MODE (XEXP (op1, 0)) == CCCmode
23422 3 : && REGNO (XEXP (op1, 0)) == FLAGS_REG
23423 54397183 : && XEXP (op1, 1) == const0_rtx)
23424 : {
23425 : /* This is *x86_cmc. */
23426 3 : if (!speed)
23427 0 : *total = COSTS_N_BYTES (1);
23428 3 : else if (TARGET_SLOW_STC)
23429 0 : *total = COSTS_N_INSNS (2);
23430 : else
23431 3 : *total = COSTS_N_INSNS (1);
23432 3 : return true;
23433 : }
23434 :
23435 54397177 : if (SCALAR_INT_MODE_P (GET_MODE (op0))
23436 113223723 : && GET_MODE_SIZE (GET_MODE (op0)) > UNITS_PER_WORD)
23437 : {
23438 761506 : if (op1 == const0_rtx)
23439 219018 : *total = cost->add
23440 109509 : + rtx_cost (op0, GET_MODE (op0), outer_code, opno, speed);
23441 : else
23442 1303994 : *total = 3*cost->add
23443 651997 : + rtx_cost (op0, GET_MODE (op0), outer_code, opno, speed)
23444 651997 : + rtx_cost (op1, GET_MODE (op0), outer_code, opno, speed);
23445 761506 : return true;
23446 : }
23447 :
23448 : /* The embedded comparison operand is completely free. */
23449 53635671 : if (!general_operand (op0, GET_MODE (op0)) && op1 == const0_rtx)
23450 364994 : *total = 0;
23451 :
23452 : return false;
23453 :
23454 1380242 : case FLOAT_EXTEND:
23455 : /* x87 represents all values extended to 80bit. */
23456 1380242 : if (!SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
23457 669776 : *total = 0;
23458 : else
23459 1420932 : *total = vec_fp_conversion_cost (cost, GET_MODE_BITSIZE (mode));
23460 : return false;
23461 :
23462 86024 : case FLOAT_TRUNCATE:
23463 86024 : if (!SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
23464 58189 : *total = cost->fadd;
23465 : else
23466 55670 : *total = vec_fp_conversion_cost (cost, GET_MODE_BITSIZE (mode));
23467 : return false;
23468 687654 : case FLOAT:
23469 687654 : case UNSIGNED_FLOAT:
23470 687654 : if (!SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
23471 : /* TODO: We do not have cost tables for x87. */
23472 95262 : *total = cost->fadd;
23473 592392 : else if (VECTOR_MODE_P (mode))
23474 0 : *total = ix86_vec_cost (mode, cost->cvtpi2ps);
23475 : else
23476 592392 : *total = cost->cvtsi2ss;
23477 : return false;
23478 :
23479 285233 : case FIX:
23480 285233 : case UNSIGNED_FIX:
23481 285233 : if (!SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
23482 : /* TODO: We do not have cost tables for x87. */
23483 285233 : *total = cost->fadd;
23484 0 : else if (VECTOR_MODE_P (mode))
23485 0 : *total = ix86_vec_cost (mode, cost->cvtps2pi);
23486 : else
23487 0 : *total = cost->cvtss2si;
23488 : return false;
23489 :
23490 419026 : case ABS:
23491 : /* SSE requires memory load for the constant operand. It may make
23492 : sense to account for this. Of course the constant operand may or
23493 : may not be reused. */
23494 419026 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
23495 297858 : *total = cost->sse_op;
23496 121168 : else if (X87_FLOAT_MODE_P (mode))
23497 36901 : *total = cost->fabs;
23498 84267 : else if (FLOAT_MODE_P (mode))
23499 33418 : *total = ix86_vec_cost (mode, cost->sse_op);
23500 50849 : else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
23501 6657 : *total = cost->sse_op;
23502 : return false;
23503 :
23504 28906 : case SQRT:
23505 28906 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
23506 18479 : *total = mode == SFmode ? cost->sqrtss : cost->sqrtsd;
23507 10427 : else if (X87_FLOAT_MODE_P (mode))
23508 4385 : *total = cost->fsqrt;
23509 6042 : else if (FLOAT_MODE_P (mode))
23510 6042 : *total = ix86_vec_cost (mode,
23511 : mode == SFmode ? cost->sqrtss : cost->sqrtsd);
23512 : return false;
23513 :
23514 4024982 : case UNSPEC:
23515 4024982 : switch (XINT (x, 1))
23516 : {
23517 126567 : case UNSPEC_TP:
23518 126567 : *total = 0;
23519 126567 : break;
23520 :
23521 5210 : case UNSPEC_VTERNLOG:
23522 5210 : *total = cost->sse_op;
23523 5210 : if (!REG_P (XVECEXP (x, 0, 0)))
23524 720 : *total += rtx_cost (XVECEXP (x, 0, 0), mode, code, 0, speed);
23525 5210 : if (!REG_P (XVECEXP (x, 0, 1)))
23526 694 : *total += rtx_cost (XVECEXP (x, 0, 1), mode, code, 1, speed);
23527 5210 : if (!REG_P (XVECEXP (x, 0, 2)))
23528 733 : *total += rtx_cost (XVECEXP (x, 0, 2), mode, code, 2, speed);
23529 : return true;
23530 :
23531 94814 : case UNSPEC_PTEST:
23532 94814 : {
23533 94814 : *total = cost->sse_op;
23534 94814 : rtx test_op0 = XVECEXP (x, 0, 0);
23535 94814 : if (!rtx_equal_p (test_op0, XVECEXP (x, 0, 1)))
23536 : return false;
23537 94166 : if (GET_CODE (test_op0) == AND)
23538 : {
23539 23 : rtx and_op0 = XEXP (test_op0, 0);
23540 23 : if (GET_CODE (and_op0) == NOT)
23541 0 : and_op0 = XEXP (and_op0, 0);
23542 23 : *total += rtx_cost (and_op0, GET_MODE (and_op0),
23543 : AND, 0, speed)
23544 23 : + rtx_cost (XEXP (test_op0, 1), GET_MODE (and_op0),
23545 : AND, 1, speed);
23546 : }
23547 : else
23548 94143 : *total = rtx_cost (test_op0, GET_MODE (test_op0),
23549 : UNSPEC, 0, speed);
23550 : }
23551 : return true;
23552 :
23553 20663 : case UNSPEC_BLENDV:
23554 20663 : *total = cost->sse_op;
23555 20663 : if (!REG_P (XVECEXP (x, 0, 0)))
23556 8658 : *total += rtx_cost (XVECEXP (x, 0, 0), mode, code, 0, speed);
23557 20663 : if (!REG_P (XVECEXP (x, 0, 1)))
23558 10127 : *total += rtx_cost (XVECEXP (x, 0, 1), mode, code, 1, speed);
23559 20663 : if (!REG_P (XVECEXP (x, 0, 2)))
23560 : {
23561 13123 : rtx cond = XVECEXP (x, 0, 2);
23562 13123 : if ((GET_CODE (cond) == LT || GET_CODE (cond) == GT)
23563 773 : && CONST_VECTOR_P (XEXP (cond, 1)))
23564 : {
23565 : /* avx2_blendvpd256_gt and friends. */
23566 153 : if (!REG_P (XEXP (cond, 0)))
23567 70 : *total += rtx_cost (XEXP (cond, 0), mode, code, 2, speed);
23568 : }
23569 : else
23570 12970 : *total += rtx_cost (cond, mode, code, 2, speed);
23571 : }
23572 : return true;
23573 :
23574 57925 : case UNSPEC_MOVMSK:
23575 57925 : *total = cost->sse_op;
23576 57925 : return true;
23577 :
23578 : default:
23579 : break;
23580 : }
23581 : return false;
23582 :
23583 2093506 : case VEC_CONCAT:
23584 : /* ??? Assume all of these vector manipulation patterns are
23585 : recognizable. In which case they all pretty much have the
23586 : same cost.
23587 : ??? We should still recruse when computing cost. */
23588 2093506 : *total = cost->sse_op;
23589 2093506 : return true;
23590 :
23591 2573319 : case VEC_SELECT:
23592 : /* Special case extracting lower part from the vector.
23593 : This by itself needs to code and most of SSE/AVX instructions have
23594 : packed and single forms where the single form may be represented
23595 : by such VEC_SELECT.
23596 :
23597 : Use cost 1 (despite the fact that functionally equivalent SUBREG has
23598 : cost 0). Making VEC_SELECT completely free, for example instructs CSE
23599 : to forward propagate VEC_SELECT into
23600 :
23601 : (set (reg eax) (reg src))
23602 :
23603 : which then prevents fwprop and combining. See i.e.
23604 : gcc.target/i386/pr91103-1.c.
23605 :
23606 : ??? rtvec_series_p test should be, for valid patterns, equivalent to
23607 : vec_series_lowpart_p but is not, since the latter calls
23608 : can_cange_mode_class on ALL_REGS and this return false since x87 does
23609 : not support subregs at all. */
23610 2573319 : if (rtvec_series_p (XVEC (XEXP (x, 1), 0), 0))
23611 810321 : *total = rtx_cost (XEXP (x, 0), GET_MODE (XEXP (x, 0)),
23612 810321 : outer_code, opno, speed) + 1;
23613 : else
23614 : /* ??? We should still recruse when computing cost. */
23615 1762998 : *total = cost->sse_op;
23616 : return true;
23617 :
23618 1251169 : case VEC_DUPLICATE:
23619 2502338 : *total = rtx_cost (XEXP (x, 0),
23620 1251169 : GET_MODE (XEXP (x, 0)),
23621 : VEC_DUPLICATE, 0, speed);
23622 : /* It's broadcast instruction, not embedded broadcasting. */
23623 1251169 : if (outer_code == SET)
23624 1199256 : *total += cost->sse_op;
23625 :
23626 : return true;
23627 :
23628 730418 : case VEC_MERGE:
23629 730418 : mask = XEXP (x, 2);
23630 : /* Scalar versions of SSE instructions may be represented as:
23631 :
23632 : (vec_merge (vec_duplicate (operation ....))
23633 : (register or memory)
23634 : (const_int 1))
23635 :
23636 : In this case vec_merge and vec_duplicate is for free.
23637 : Just recurse into operation and second operand. */
23638 730418 : if (mask == const1_rtx
23639 217230 : && GET_CODE (XEXP (x, 0)) == VEC_DUPLICATE)
23640 : {
23641 79406 : *total = rtx_cost (XEXP (XEXP (x, 0), 0), mode,
23642 : outer_code, opno, speed)
23643 79406 : + rtx_cost (XEXP (x, 1), mode, outer_code, opno, speed);
23644 79406 : return true;
23645 : }
23646 : /* This is masked instruction, assume the same cost,
23647 : as nonmasked variant. */
23648 651012 : else if (TARGET_AVX512F
23649 651012 : && (register_operand (mask, GET_MODE (mask))
23650 : /* Redunduant clean up of high bits for kmask with VL=2/4
23651 : .i.e (vec_merge op0, op1, (and op3 15)). */
23652 118917 : || (GET_CODE (mask) == AND
23653 411 : && register_operand (XEXP (mask, 0), GET_MODE (mask))
23654 411 : && CONST_INT_P (XEXP (mask, 1))
23655 411 : && ((INTVAL (XEXP (mask, 1)) == 3
23656 131 : && GET_MODE_NUNITS (mode) == 2)
23657 280 : || (INTVAL (XEXP (mask, 1)) == 15
23658 280 : && GET_MODE_NUNITS (mode) == 4)))))
23659 : {
23660 373771 : *total = rtx_cost (XEXP (x, 0), mode, outer_code, opno, speed)
23661 373771 : + rtx_cost (XEXP (x, 1), mode, outer_code, opno, speed);
23662 373771 : return true;
23663 : }
23664 : /* Combination of the two above:
23665 :
23666 : (vec_merge (vec_merge (vec_duplicate (operation ...))
23667 : (register or memory)
23668 : (reg:QI mask))
23669 : (register or memory)
23670 : (const_int 1))
23671 :
23672 : i.e. avx512fp16_vcvtss2sh_mask. */
23673 277241 : else if (TARGET_AVX512F
23674 118506 : && mask == const1_rtx
23675 45931 : && GET_CODE (XEXP (x, 0)) == VEC_MERGE
23676 27126 : && GET_CODE (XEXP (XEXP (x, 0), 0)) == VEC_DUPLICATE
23677 279503 : && register_operand (XEXP (XEXP (x, 0), 2),
23678 2262 : GET_MODE (XEXP (XEXP (x, 0), 2))))
23679 : {
23680 2250 : *total = rtx_cost (XEXP (XEXP (XEXP (x, 0), 0), 0),
23681 : mode, outer_code, opno, speed)
23682 2250 : + rtx_cost (XEXP (XEXP (x, 0), 1),
23683 : mode, outer_code, opno, speed)
23684 2250 : + rtx_cost (XEXP (x, 1), mode, outer_code, opno, speed);
23685 2250 : return true;
23686 : }
23687 : /* vcmp. */
23688 274991 : else if (unspec_pcmp_p (mask)
23689 274991 : || (GET_CODE (mask) == NOT
23690 0 : && unspec_pcmp_p (XEXP (mask, 0))))
23691 : {
23692 1950 : rtx uns = GET_CODE (mask) == NOT ? XEXP (mask, 0) : mask;
23693 1950 : rtx unsop0 = XVECEXP (uns, 0, 0);
23694 : /* Make (subreg:V4SI (not:V16QI (reg:V16QI ..)) 0)
23695 : cost the same as register.
23696 : This is used by avx_cmp<mode>3_ltint_not. */
23697 1950 : if (SUBREG_P (unsop0))
23698 417 : unsop0 = XEXP (unsop0, 0);
23699 1950 : if (GET_CODE (unsop0) == NOT)
23700 18 : unsop0 = XEXP (unsop0, 0);
23701 1950 : *total = rtx_cost (XEXP (x, 0), mode, outer_code, opno, speed)
23702 1950 : + rtx_cost (XEXP (x, 1), mode, outer_code, opno, speed)
23703 1950 : + rtx_cost (unsop0, mode, UNSPEC, opno, speed)
23704 1950 : + rtx_cost (XVECEXP (uns, 0, 1), mode, UNSPEC, opno, speed)
23705 1950 : + cost->sse_op;
23706 1950 : return true;
23707 : }
23708 : else
23709 273041 : *total = cost->sse_op;
23710 273041 : return false;
23711 :
23712 107955585 : case MEM:
23713 : /* CONST_VECTOR_DUPLICATE_P in constant_pool is just broadcast.
23714 : or variants in ix86_vector_duplicate_simode_const. */
23715 :
23716 107955585 : if (GET_MODE_SIZE (mode) >= 16
23717 17737969 : && VECTOR_MODE_P (mode)
23718 11603616 : && SYMBOL_REF_P (XEXP (x, 0))
23719 2284821 : && CONSTANT_POOL_ADDRESS_P (XEXP (x, 0))
23720 110012648 : && ix86_broadcast_from_constant (mode, x))
23721 : {
23722 524979 : *total = COSTS_N_INSNS (2) + speed;
23723 524979 : return true;
23724 : }
23725 :
23726 : /* An insn that accesses memory is slightly more expensive
23727 : than one that does not. */
23728 107430606 : if (speed)
23729 : {
23730 96131955 : *total += 1;
23731 96131955 : rtx addr = XEXP (x, 0);
23732 : /* For MEM, rtx_cost iterates each subrtx, and adds up the costs,
23733 : so for MEM (reg) and MEM (reg + 4), the former costs 5,
23734 : the latter costs 9, it is not accurate for x86. Ideally
23735 : address_cost should be used, but it reduce cost too much.
23736 : So current solution is make constant disp as cheap as possible. */
23737 96131955 : if (GET_CODE (addr) == PLUS
23738 78192435 : && x86_64_immediate_operand (XEXP (addr, 1), Pmode)
23739 : /* Only handle (reg + disp) since other forms of addr are mostly LEA,
23740 : there's no additional cost for the plus of disp. */
23741 168871920 : && register_operand (XEXP (addr, 0), Pmode))
23742 : {
23743 56879672 : *total += 1;
23744 69742323 : *total += rtx_cost (XEXP (addr, 0), Pmode, PLUS, 0, speed);
23745 56879672 : return true;
23746 : }
23747 : }
23748 :
23749 : return false;
23750 :
23751 51667 : case ZERO_EXTRACT:
23752 51667 : if (XEXP (x, 1) == const1_rtx
23753 11387 : && GET_CODE (XEXP (x, 2)) == ZERO_EXTEND
23754 0 : && GET_MODE (XEXP (x, 2)) == SImode
23755 0 : && GET_MODE (XEXP (XEXP (x, 2), 0)) == QImode)
23756 : {
23757 : /* Ignore cost of zero extension and masking of last argument. */
23758 0 : *total += rtx_cost (XEXP (x, 0), mode, code, 0, speed);
23759 0 : *total += rtx_cost (XEXP (x, 1), mode, code, 1, speed);
23760 0 : *total += rtx_cost (XEXP (XEXP (x, 2), 0), mode, code, 2, speed);
23761 0 : return true;
23762 : }
23763 : return false;
23764 :
23765 29416058 : case IF_THEN_ELSE:
23766 29416058 : if (TARGET_XOP
23767 25035 : && VECTOR_MODE_P (mode)
23768 29421449 : && (GET_MODE_SIZE (mode) == 16 || GET_MODE_SIZE (mode) == 32))
23769 : {
23770 : /* vpcmov. */
23771 4823 : *total = speed ? COSTS_N_INSNS (2) : COSTS_N_BYTES (6);
23772 4823 : if (!REG_P (XEXP (x, 0)))
23773 4663 : *total += rtx_cost (XEXP (x, 0), mode, code, 0, speed);
23774 4823 : if (!REG_P (XEXP (x, 1)))
23775 4630 : *total += rtx_cost (XEXP (x, 1), mode, code, 1, speed);
23776 4823 : if (!REG_P (XEXP (x, 2)))
23777 4632 : *total += rtx_cost (XEXP (x, 2), mode, code, 2, speed);
23778 4823 : return true;
23779 : }
23780 0 : else if (TARGET_CMOVE
23781 29411235 : && SCALAR_INT_MODE_P (mode)
23782 31816272 : && GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
23783 : {
23784 : /* cmov. */
23785 2208087 : *total = COSTS_N_INSNS (1);
23786 2208087 : if (!COMPARISON_P (XEXP (x, 0)) && !REG_P (XEXP (x, 0)))
23787 0 : *total += rtx_cost (XEXP (x, 0), mode, code, 0, speed);
23788 2208087 : if (!REG_P (XEXP (x, 1)))
23789 119472 : *total += rtx_cost (XEXP (x, 1), mode, code, 1, speed);
23790 2208087 : if (!REG_P (XEXP (x, 2)))
23791 719467 : *total += rtx_cost (XEXP (x, 2), mode, code, 2, speed);
23792 2208087 : return true;
23793 : }
23794 : return false;
23795 :
23796 18331050 : case EQ:
23797 18331050 : case GT:
23798 18331050 : case GTU:
23799 18331050 : case LT:
23800 18331050 : case LTU:
23801 18331050 : if (TARGET_SSE2
23802 18327850 : && GET_MODE_CLASS (mode) == MODE_VECTOR_INT
23803 18712250 : && GET_MODE_SIZE (mode) >= 8)
23804 : {
23805 : /* vpcmpeq */
23806 373548 : *total = speed ? COSTS_N_INSNS (1) : COSTS_N_BYTES (4);
23807 373548 : if (!REG_P (XEXP (x, 0)))
23808 74846 : *total += rtx_cost (XEXP (x, 0), mode, code, 0, speed);
23809 373548 : if (!REG_P (XEXP (x, 1)))
23810 141906 : *total += rtx_cost (XEXP (x, 1), mode, code, 1, speed);
23811 373548 : return true;
23812 : }
23813 17957502 : if (TARGET_XOP
23814 12168 : && GET_MODE_CLASS (mode) == MODE_VECTOR_INT
23815 17957610 : && GET_MODE_SIZE (mode) <= 16)
23816 : {
23817 : /* vpcomeq */
23818 108 : *total = speed ? COSTS_N_INSNS (1) : COSTS_N_BYTES (6);
23819 108 : if (!REG_P (XEXP (x, 0)))
23820 0 : *total += rtx_cost (XEXP (x, 0), mode, code, 0, speed);
23821 108 : if (!REG_P (XEXP (x, 1)))
23822 0 : *total += rtx_cost (XEXP (x, 1), mode, code, 1, speed);
23823 108 : return true;
23824 : }
23825 : return false;
23826 :
23827 16369459 : case NE:
23828 16369459 : case GE:
23829 16369459 : case GEU:
23830 16369459 : if (TARGET_XOP
23831 21924 : && GET_MODE_CLASS (mode) == MODE_VECTOR_INT
23832 16376241 : && GET_MODE_SIZE (mode) <= 16)
23833 : {
23834 : /* vpcomneq */
23835 6782 : *total = speed ? COSTS_N_INSNS (1) : COSTS_N_BYTES (6);
23836 6782 : if (!REG_P (XEXP (x, 0)))
23837 1405 : *total += rtx_cost (XEXP (x, 0), mode, code, 0, speed);
23838 6782 : if (!REG_P (XEXP (x, 1)))
23839 5734 : *total += rtx_cost (XEXP (x, 1), mode, code, 1, speed);
23840 6782 : return true;
23841 : }
23842 16362677 : if (TARGET_SSE2
23843 16360541 : && GET_MODE_CLASS (mode) == MODE_VECTOR_INT
23844 16365989 : && GET_MODE_SIZE (mode) >= 8)
23845 : {
23846 3347 : if (TARGET_AVX512F && GET_MODE_SIZE (mode) >= 16)
23847 : /* vpcmpeq + vpternlog */
23848 51 : *total = speed ? COSTS_N_INSNS (2) : COSTS_N_BYTES (11);
23849 : else
23850 : /* vpcmpeq + pxor + vpcmpeq */
23851 3392 : *total = speed ? COSTS_N_INSNS (3) : COSTS_N_BYTES (12);
23852 3288 : if (!REG_P (XEXP (x, 0)))
23853 342 : *total += rtx_cost (XEXP (x, 0), mode, code, 0, speed);
23854 3288 : if (!REG_P (XEXP (x, 1)))
23855 45 : *total += rtx_cost (XEXP (x, 1), mode, code, 1, speed);
23856 3288 : return true;
23857 : }
23858 : return false;
23859 :
23860 : default:
23861 : return false;
23862 : }
23863 : }
23864 :
23865 : #if TARGET_MACHO
23866 :
23867 : static int current_machopic_label_num;
23868 :
23869 : /* Given a symbol name and its associated stub, write out the
23870 : definition of the stub. */
23871 :
23872 : void
23873 : machopic_output_stub (FILE *file, const char *symb, const char *stub)
23874 : {
23875 : unsigned int length;
23876 : char *binder_name, *symbol_name, lazy_ptr_name[32];
23877 : int label = ++current_machopic_label_num;
23878 :
23879 : /* For 64-bit we shouldn't get here. */
23880 : gcc_assert (!TARGET_64BIT);
23881 :
23882 : /* Lose our funky encoding stuff so it doesn't contaminate the stub. */
23883 : symb = targetm.strip_name_encoding (symb);
23884 :
23885 : length = strlen (stub);
23886 : binder_name = XALLOCAVEC (char, length + 32);
23887 : GEN_BINDER_NAME_FOR_STUB (binder_name, stub, length);
23888 :
23889 : length = strlen (symb);
23890 : symbol_name = XALLOCAVEC (char, length + 32);
23891 : GEN_SYMBOL_NAME_FOR_SYMBOL (symbol_name, symb, length);
23892 :
23893 : sprintf (lazy_ptr_name, "L%d$lz", label);
23894 :
23895 : if (MACHOPIC_ATT_STUB)
23896 : switch_to_section (darwin_sections[machopic_picsymbol_stub3_section]);
23897 : else if (MACHOPIC_PURE)
23898 : switch_to_section (darwin_sections[machopic_picsymbol_stub2_section]);
23899 : else
23900 : switch_to_section (darwin_sections[machopic_symbol_stub_section]);
23901 :
23902 : fprintf (file, "%s:\n", stub);
23903 : fprintf (file, "\t.indirect_symbol %s\n", symbol_name);
23904 :
23905 : if (MACHOPIC_ATT_STUB)
23906 : {
23907 : fprintf (file, "\thlt ; hlt ; hlt ; hlt ; hlt\n");
23908 : }
23909 : else if (MACHOPIC_PURE)
23910 : {
23911 : /* PIC stub. */
23912 : /* 25-byte PIC stub using "CALL get_pc_thunk". */
23913 : rtx tmp = gen_rtx_REG (SImode, 2 /* ECX */);
23914 : output_set_got (tmp, NULL_RTX); /* "CALL ___<cpu>.get_pc_thunk.cx". */
23915 : fprintf (file, "LPC$%d:\tmovl\t%s-LPC$%d(%%ecx),%%ecx\n",
23916 : label, lazy_ptr_name, label);
23917 : fprintf (file, "\tjmp\t*%%ecx\n");
23918 : }
23919 : else
23920 : fprintf (file, "\tjmp\t*%s\n", lazy_ptr_name);
23921 :
23922 : /* The AT&T-style ("self-modifying") stub is not lazily bound, thus
23923 : it needs no stub-binding-helper. */
23924 : if (MACHOPIC_ATT_STUB)
23925 : return;
23926 :
23927 : fprintf (file, "%s:\n", binder_name);
23928 :
23929 : if (MACHOPIC_PURE)
23930 : {
23931 : fprintf (file, "\tlea\t%s-%s(%%ecx),%%ecx\n", lazy_ptr_name, binder_name);
23932 : fprintf (file, "\tpushl\t%%ecx\n");
23933 : }
23934 : else
23935 : fprintf (file, "\tpushl\t$%s\n", lazy_ptr_name);
23936 :
23937 : fputs ("\tjmp\tdyld_stub_binding_helper\n", file);
23938 :
23939 : /* N.B. Keep the correspondence of these
23940 : 'symbol_ptr/symbol_ptr2/symbol_ptr3' sections consistent with the
23941 : old-pic/new-pic/non-pic stubs; altering this will break
23942 : compatibility with existing dylibs. */
23943 : if (MACHOPIC_PURE)
23944 : {
23945 : /* 25-byte PIC stub using "CALL get_pc_thunk". */
23946 : switch_to_section (darwin_sections[machopic_lazy_symbol_ptr2_section]);
23947 : }
23948 : else
23949 : /* 16-byte -mdynamic-no-pic stub. */
23950 : switch_to_section(darwin_sections[machopic_lazy_symbol_ptr3_section]);
23951 :
23952 : fprintf (file, "%s:\n", lazy_ptr_name);
23953 : fprintf (file, "\t.indirect_symbol %s\n", symbol_name);
23954 : fprintf (file, ASM_LONG "%s\n", binder_name);
23955 : }
23956 : #endif /* TARGET_MACHO */
23957 :
23958 : /* Order the registers for register allocator. */
23959 :
23960 : void
23961 219158 : x86_order_regs_for_local_alloc (void)
23962 : {
23963 219158 : int pos = 0;
23964 219158 : int i;
23965 :
23966 : /* First allocate the local general purpose registers. */
23967 20381694 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
23968 20162536 : if (GENERAL_REGNO_P (i) && default_function_abi.clobbers_full_reg_p (i))
23969 5490190 : reg_alloc_order [pos++] = i;
23970 :
23971 : /* Global general purpose registers. */
23972 20381694 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
23973 20162536 : if (GENERAL_REGNO_P (i) && !default_function_abi.clobbers_full_reg_p (i))
23974 1522866 : reg_alloc_order [pos++] = i;
23975 :
23976 : /* x87 registers come first in case we are doing FP math
23977 : using them. */
23978 219158 : if (!TARGET_SSE_MATH)
23979 56367 : for (i = FIRST_STACK_REG; i <= LAST_STACK_REG; i++)
23980 50104 : reg_alloc_order [pos++] = i;
23981 :
23982 : /* SSE registers. */
23983 1972422 : for (i = FIRST_SSE_REG; i <= LAST_SSE_REG; i++)
23984 1753264 : reg_alloc_order [pos++] = i;
23985 1972422 : for (i = FIRST_REX_SSE_REG; i <= LAST_REX_SSE_REG; i++)
23986 1753264 : reg_alloc_order [pos++] = i;
23987 :
23988 : /* Extended REX SSE registers. */
23989 3725686 : for (i = FIRST_EXT_REX_SSE_REG; i <= LAST_EXT_REX_SSE_REG; i++)
23990 3506528 : reg_alloc_order [pos++] = i;
23991 :
23992 : /* Mask register. */
23993 1972422 : for (i = FIRST_MASK_REG; i <= LAST_MASK_REG; i++)
23994 1753264 : reg_alloc_order [pos++] = i;
23995 :
23996 : /* x87 registers. */
23997 219158 : if (TARGET_SSE_MATH)
23998 1916055 : for (i = FIRST_STACK_REG; i <= LAST_STACK_REG; i++)
23999 1703160 : reg_alloc_order [pos++] = i;
24000 :
24001 1972422 : for (i = FIRST_MMX_REG; i <= LAST_MMX_REG; i++)
24002 1753264 : reg_alloc_order [pos++] = i;
24003 :
24004 : /* Initialize the rest of array as we do not allocate some registers
24005 : at all. */
24006 1095790 : while (pos < FIRST_PSEUDO_REGISTER)
24007 876632 : reg_alloc_order [pos++] = 0;
24008 219158 : }
24009 :
24010 : static bool
24011 254083935 : ix86_ms_bitfield_layout_p (const_tree record_type)
24012 : {
24013 254083935 : return ((TARGET_MS_BITFIELD_LAYOUT
24014 215 : && !lookup_attribute ("gcc_struct", TYPE_ATTRIBUTES (record_type)))
24015 254083935 : || lookup_attribute ("ms_struct", TYPE_ATTRIBUTES (record_type)));
24016 : }
24017 :
24018 : /* Returns an expression indicating where the this parameter is
24019 : located on entry to the FUNCTION. */
24020 :
24021 : static rtx
24022 1770 : x86_this_parameter (tree function)
24023 : {
24024 1770 : tree type = TREE_TYPE (function);
24025 1770 : bool aggr = aggregate_value_p (TREE_TYPE (type), type) != 0;
24026 1770 : int nregs;
24027 :
24028 1770 : if (TARGET_64BIT)
24029 : {
24030 1768 : const int *parm_regs;
24031 :
24032 1768 : if (lookup_attribute ("preserve_none", TYPE_ATTRIBUTES (type)))
24033 : parm_regs = x86_64_preserve_none_int_parameter_registers;
24034 1768 : else if (ix86_function_type_abi (type) == MS_ABI)
24035 : parm_regs = x86_64_ms_abi_int_parameter_registers;
24036 : else
24037 1768 : parm_regs = x86_64_int_parameter_registers;
24038 1768 : return gen_rtx_REG (Pmode, parm_regs[aggr]);
24039 : }
24040 :
24041 2 : nregs = ix86_function_regparm (type, function);
24042 :
24043 2 : if (nregs > 0 && !stdarg_p (type))
24044 : {
24045 0 : int regno;
24046 0 : unsigned int ccvt = ix86_get_callcvt (type);
24047 :
24048 0 : if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
24049 0 : regno = aggr ? DX_REG : CX_REG;
24050 0 : else if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
24051 : {
24052 0 : regno = CX_REG;
24053 0 : if (aggr)
24054 0 : return gen_rtx_MEM (SImode,
24055 0 : plus_constant (Pmode, stack_pointer_rtx, 4));
24056 : }
24057 : else
24058 : {
24059 0 : regno = AX_REG;
24060 0 : if (aggr)
24061 : {
24062 0 : regno = DX_REG;
24063 0 : if (nregs == 1)
24064 0 : return gen_rtx_MEM (SImode,
24065 0 : plus_constant (Pmode,
24066 : stack_pointer_rtx, 4));
24067 : }
24068 : }
24069 0 : return gen_rtx_REG (SImode, regno);
24070 : }
24071 :
24072 4 : return gen_rtx_MEM (SImode, plus_constant (Pmode, stack_pointer_rtx,
24073 4 : aggr ? 8 : 4));
24074 : }
24075 :
24076 : /* Determine whether x86_output_mi_thunk can succeed. */
24077 :
24078 : static bool
24079 4926 : x86_can_output_mi_thunk (const_tree, HOST_WIDE_INT, HOST_WIDE_INT vcall_offset,
24080 : const_tree function)
24081 : {
24082 : /* 64-bit can handle anything. */
24083 4926 : if (TARGET_64BIT)
24084 : return true;
24085 :
24086 : /* For 32-bit, everything's fine if we have one free register. */
24087 76 : if (ix86_function_regparm (TREE_TYPE (function), function) < 3)
24088 : return true;
24089 :
24090 : /* Need a free register for vcall_offset. */
24091 0 : if (vcall_offset)
24092 : return false;
24093 :
24094 : /* Need a free register for GOT references. */
24095 0 : if (flag_pic && !targetm.binds_local_p (function))
24096 : return false;
24097 :
24098 : /* Otherwise ok. */
24099 : return true;
24100 : }
24101 :
24102 : /* Output the assembler code for a thunk function. THUNK_DECL is the
24103 : declaration for the thunk function itself, FUNCTION is the decl for
24104 : the target function. DELTA is an immediate constant offset to be
24105 : added to THIS. If VCALL_OFFSET is nonzero, the word at
24106 : *(*this + vcall_offset) should be added to THIS. */
24107 :
24108 : static void
24109 1770 : x86_output_mi_thunk (FILE *file, tree thunk_fndecl, HOST_WIDE_INT delta,
24110 : HOST_WIDE_INT vcall_offset, tree function)
24111 : {
24112 1770 : const char *fnname = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (thunk_fndecl));
24113 1770 : rtx this_param = x86_this_parameter (function);
24114 1770 : rtx this_reg, tmp, fnaddr;
24115 1770 : unsigned int tmp_regno;
24116 1770 : rtx_insn *insn;
24117 1770 : int saved_flag_force_indirect_call = flag_force_indirect_call;
24118 :
24119 1770 : if (TARGET_64BIT)
24120 : tmp_regno = R10_REG;
24121 : else
24122 : {
24123 2 : unsigned int ccvt = ix86_get_callcvt (TREE_TYPE (function));
24124 2 : if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
24125 : tmp_regno = AX_REG;
24126 2 : else if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
24127 : tmp_regno = DX_REG;
24128 : else
24129 2 : tmp_regno = CX_REG;
24130 :
24131 2 : if (flag_pic)
24132 2 : flag_force_indirect_call = 0;
24133 : }
24134 :
24135 1770 : emit_note (NOTE_INSN_PROLOGUE_END);
24136 :
24137 : /* CET is enabled, insert EB instruction. */
24138 1770 : if ((flag_cf_protection & CF_BRANCH))
24139 20 : emit_insn (gen_nop_endbr ());
24140 :
24141 : /* If VCALL_OFFSET, we'll need THIS in a register. Might as well
24142 : pull it in now and let DELTA benefit. */
24143 1770 : if (REG_P (this_param))
24144 : this_reg = this_param;
24145 2 : else if (vcall_offset)
24146 : {
24147 : /* Put the this parameter into %eax. */
24148 2 : this_reg = gen_rtx_REG (Pmode, AX_REG);
24149 1 : emit_move_insn (this_reg, this_param);
24150 : }
24151 : else
24152 : this_reg = NULL_RTX;
24153 :
24154 : /* Adjust the this parameter by a fixed constant. */
24155 1770 : if (delta)
24156 : {
24157 831 : rtx delta_rtx = GEN_INT (delta);
24158 831 : rtx delta_dst = this_reg ? this_reg : this_param;
24159 :
24160 831 : if (TARGET_64BIT)
24161 : {
24162 830 : if (!x86_64_general_operand (delta_rtx, Pmode))
24163 : {
24164 0 : tmp = gen_rtx_REG (Pmode, tmp_regno);
24165 0 : emit_move_insn (tmp, delta_rtx);
24166 0 : delta_rtx = tmp;
24167 : }
24168 : }
24169 :
24170 832 : ix86_emit_binop (PLUS, Pmode, delta_dst, delta_rtx);
24171 : }
24172 :
24173 : /* Adjust the this parameter by a value stored in the vtable. */
24174 1770 : if (vcall_offset)
24175 : {
24176 990 : rtx vcall_addr, vcall_mem, this_mem;
24177 :
24178 991 : tmp = gen_rtx_REG (Pmode, tmp_regno);
24179 :
24180 990 : this_mem = gen_rtx_MEM (ptr_mode, this_reg);
24181 991 : if (Pmode != ptr_mode)
24182 0 : this_mem = gen_rtx_ZERO_EXTEND (Pmode, this_mem);
24183 990 : emit_move_insn (tmp, this_mem);
24184 :
24185 : /* Adjust the this parameter. */
24186 991 : vcall_addr = plus_constant (Pmode, tmp, vcall_offset);
24187 990 : if (TARGET_64BIT
24188 990 : && !ix86_legitimate_address_p (ptr_mode, vcall_addr, true))
24189 : {
24190 0 : rtx tmp2 = gen_rtx_REG (Pmode, R11_REG);
24191 0 : emit_move_insn (tmp2, GEN_INT (vcall_offset));
24192 0 : vcall_addr = gen_rtx_PLUS (Pmode, tmp, tmp2);
24193 : }
24194 :
24195 990 : vcall_mem = gen_rtx_MEM (ptr_mode, vcall_addr);
24196 991 : if (Pmode != ptr_mode)
24197 0 : emit_insn (gen_addsi_1_zext (this_reg,
24198 : gen_rtx_REG (ptr_mode,
24199 : REGNO (this_reg)),
24200 : vcall_mem));
24201 : else
24202 990 : ix86_emit_binop (PLUS, Pmode, this_reg, vcall_mem);
24203 : }
24204 :
24205 : /* If necessary, drop THIS back to its stack slot. */
24206 1770 : if (this_reg && this_reg != this_param)
24207 1 : emit_move_insn (this_param, this_reg);
24208 :
24209 1770 : fnaddr = XEXP (DECL_RTL (function), 0);
24210 1770 : if (TARGET_64BIT)
24211 : {
24212 25 : if (!flag_pic || targetm.binds_local_p (function)
24213 1793 : || TARGET_PECOFF)
24214 : ;
24215 : else
24216 : {
24217 0 : tmp = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, fnaddr), UNSPEC_GOTPCREL);
24218 0 : tmp = gen_rtx_CONST (Pmode, tmp);
24219 0 : fnaddr = gen_const_mem (Pmode, tmp);
24220 : }
24221 : }
24222 : else
24223 : {
24224 2 : if (!flag_pic || targetm.binds_local_p (function))
24225 : ;
24226 : #if TARGET_MACHO
24227 : else if (TARGET_MACHO)
24228 : {
24229 : fnaddr = machopic_indirect_call_target (DECL_RTL (function));
24230 : fnaddr = XEXP (fnaddr, 0);
24231 : }
24232 : #endif /* TARGET_MACHO */
24233 : else
24234 : {
24235 0 : tmp = gen_rtx_REG (Pmode, CX_REG);
24236 0 : output_set_got (tmp, NULL_RTX);
24237 :
24238 0 : fnaddr = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, fnaddr), UNSPEC_GOT);
24239 0 : fnaddr = gen_rtx_CONST (Pmode, fnaddr);
24240 0 : fnaddr = gen_rtx_PLUS (Pmode, tmp, fnaddr);
24241 0 : fnaddr = gen_const_mem (Pmode, fnaddr);
24242 : }
24243 : }
24244 :
24245 : /* Our sibling call patterns do not allow memories, because we have no
24246 : predicate that can distinguish between frame and non-frame memory.
24247 : For our purposes here, we can get away with (ab)using a jump pattern,
24248 : because we're going to do no optimization. */
24249 1770 : if (MEM_P (fnaddr))
24250 : {
24251 0 : if (sibcall_insn_operand (fnaddr, word_mode))
24252 : {
24253 0 : fnaddr = XEXP (DECL_RTL (function), 0);
24254 0 : tmp = gen_rtx_MEM (QImode, fnaddr);
24255 0 : tmp = gen_rtx_CALL (VOIDmode, tmp, const0_rtx);
24256 0 : tmp = emit_call_insn (tmp);
24257 0 : SIBLING_CALL_P (tmp) = 1;
24258 : }
24259 : else
24260 0 : emit_jump_insn (gen_indirect_jump (fnaddr));
24261 : }
24262 : else
24263 : {
24264 1770 : if (ix86_cmodel == CM_LARGE_PIC && SYMBOLIC_CONST (fnaddr))
24265 : {
24266 : // CM_LARGE_PIC always uses pseudo PIC register which is
24267 : // uninitialized. Since FUNCTION is local and calling it
24268 : // doesn't go through PLT, we use scratch register %r11 as
24269 : // PIC register and initialize it here.
24270 3 : pic_offset_table_rtx = gen_rtx_REG (Pmode, R11_REG);
24271 3 : ix86_init_large_pic_reg (tmp_regno);
24272 3 : fnaddr = legitimize_pic_address (fnaddr,
24273 3 : gen_rtx_REG (Pmode, tmp_regno));
24274 : }
24275 :
24276 1770 : if (!sibcall_insn_operand (fnaddr, word_mode))
24277 : {
24278 9 : tmp = gen_rtx_REG (word_mode, tmp_regno);
24279 9 : if (GET_MODE (fnaddr) != word_mode)
24280 0 : fnaddr = gen_rtx_ZERO_EXTEND (word_mode, fnaddr);
24281 9 : emit_move_insn (tmp, fnaddr);
24282 9 : fnaddr = tmp;
24283 : }
24284 :
24285 1770 : tmp = gen_rtx_MEM (QImode, fnaddr);
24286 1770 : tmp = gen_rtx_CALL (VOIDmode, tmp, const0_rtx);
24287 1770 : tmp = emit_call_insn (tmp);
24288 1770 : SIBLING_CALL_P (tmp) = 1;
24289 : }
24290 1770 : emit_barrier ();
24291 :
24292 : /* Emit just enough of rest_of_compilation to get the insns emitted. */
24293 1770 : insn = get_insns ();
24294 1770 : shorten_branches (insn);
24295 1770 : assemble_start_function (thunk_fndecl, fnname);
24296 1770 : final_start_function (insn, file, 1);
24297 1770 : final (insn, file, 1);
24298 1770 : final_end_function ();
24299 1770 : assemble_end_function (thunk_fndecl, fnname);
24300 :
24301 1770 : flag_force_indirect_call = saved_flag_force_indirect_call;
24302 1770 : }
24303 :
24304 : static void
24305 277837 : x86_file_start (void)
24306 : {
24307 277837 : default_file_start ();
24308 277837 : if (TARGET_16BIT)
24309 6 : fputs ("\t.code16gcc\n", asm_out_file);
24310 : #if TARGET_MACHO
24311 : darwin_file_start ();
24312 : #endif
24313 277837 : if (X86_FILE_START_VERSION_DIRECTIVE)
24314 : fputs ("\t.version\t\"01.01\"\n", asm_out_file);
24315 277837 : if (X86_FILE_START_FLTUSED)
24316 : fputs ("\t.global\t__fltused\n", asm_out_file);
24317 277837 : if (ix86_asm_dialect == ASM_INTEL)
24318 66 : fputs ("\t.intel_syntax noprefix\n", asm_out_file);
24319 277837 : }
24320 :
24321 : int
24322 105553418 : x86_field_alignment (tree type, int computed)
24323 : {
24324 105553418 : machine_mode mode;
24325 :
24326 105553418 : if (TARGET_64BIT || TARGET_ALIGN_DOUBLE)
24327 : return computed;
24328 9127689 : if (TARGET_IAMCU)
24329 0 : return iamcu_alignment (type, computed);
24330 9127689 : type = strip_array_types (type);
24331 9127689 : mode = TYPE_MODE (type);
24332 9127689 : if (mode == DFmode || mode == DCmode
24333 9021723 : || GET_MODE_CLASS (mode) == MODE_INT
24334 3019945 : || GET_MODE_CLASS (mode) == MODE_COMPLEX_INT)
24335 : {
24336 6107744 : if (TYPE_ATOMIC (type) && computed > 32)
24337 : {
24338 0 : static bool warned;
24339 :
24340 0 : if (!warned && warn_psabi)
24341 : {
24342 0 : const char *url
24343 : = CHANGES_ROOT_URL "gcc-11/changes.html#ia32_atomic";
24344 :
24345 0 : warned = true;
24346 0 : inform (input_location, "the alignment of %<_Atomic %T%> "
24347 : "fields changed in %{GCC 11.1%}",
24348 0 : TYPE_MAIN_VARIANT (type), url);
24349 : }
24350 : }
24351 : else
24352 6107744 : return MIN (32, computed);
24353 : }
24354 : return computed;
24355 : }
24356 :
24357 : /* Print call to TARGET to FILE. */
24358 :
24359 : static void
24360 375 : x86_print_call_or_nop (FILE *file, const char *target,
24361 : const char *label)
24362 : {
24363 375 : if (flag_nop_mcount || !strcmp (target, "nop"))
24364 : {
24365 9 : if (TARGET_16BIT)
24366 : /* 3 byte no-op: lea 0(%si), %si */
24367 1 : fprintf (file, "%s" ASM_BYTE "0x8d, 0x74, 0x00\n", label);
24368 : else
24369 : /* 5 byte nop: nopl 0(%[re]ax,%[re]ax,1) */
24370 8 : fprintf (file, "%s" ASM_BYTE "0x0f, 0x1f, 0x44, 0x00, 0x00\n",
24371 : label);
24372 : }
24373 366 : else if (!TARGET_PECOFF && flag_pic)
24374 : {
24375 8 : gcc_assert (flag_plt);
24376 :
24377 8 : fprintf (file, "%s\tcall\t%s@PLT\n", label, target);
24378 : }
24379 : else
24380 358 : fprintf (file, "%s\tcall\t%s\n", label, target);
24381 375 : }
24382 :
24383 : static bool
24384 395 : current_fentry_name (const char **name)
24385 : {
24386 395 : tree attr = lookup_attribute ("fentry_name",
24387 395 : DECL_ATTRIBUTES (current_function_decl));
24388 395 : if (!attr)
24389 : return false;
24390 2 : *name = TREE_STRING_POINTER (TREE_VALUE (TREE_VALUE (attr)));
24391 2 : return true;
24392 : }
24393 :
24394 : static bool
24395 16 : current_fentry_section (const char **name)
24396 : {
24397 16 : tree attr = lookup_attribute ("fentry_section",
24398 16 : DECL_ATTRIBUTES (current_function_decl));
24399 16 : if (!attr)
24400 : return false;
24401 2 : *name = TREE_STRING_POINTER (TREE_VALUE (TREE_VALUE (attr)));
24402 2 : return true;
24403 : }
24404 :
24405 : /* Return a caller-saved register which isn't live or a callee-saved
24406 : register which has been saved on stack in the prologue at entry for
24407 : profile. */
24408 :
24409 : static int
24410 17 : x86_64_select_profile_regnum (bool r11_ok ATTRIBUTE_UNUSED)
24411 : {
24412 : /* Use %r10 if the profiler is emitted before the prologue or it isn't
24413 : used by DRAP. */
24414 17 : if (ix86_profile_before_prologue ()
24415 4 : || !crtl->drap_reg
24416 17 : || REGNO (crtl->drap_reg) != R10_REG)
24417 : return R10_REG;
24418 :
24419 : /* The profiler is emitted after the prologue. If there is a
24420 : caller-saved register which isn't live or a callee-saved
24421 : register saved on stack in the prologue, use it. */
24422 :
24423 0 : bitmap reg_live = df_get_live_out (ENTRY_BLOCK_PTR_FOR_FN (cfun));
24424 :
24425 0 : int i;
24426 0 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
24427 0 : if (GENERAL_REGNO_P (i)
24428 0 : && i != R10_REG
24429 : #ifdef NO_PROFILE_COUNTERS
24430 0 : && (r11_ok || i != R11_REG)
24431 : #else
24432 : && i != R11_REG
24433 : #endif
24434 0 : && TEST_HARD_REG_BIT (accessible_reg_set, i)
24435 0 : && (ix86_save_reg (i, true, true)
24436 0 : || (crtl->abi->clobbers_full_reg_p (i)
24437 0 : && !fixed_regs[i]
24438 0 : && !REGNO_REG_SET_P (reg_live, i))))
24439 0 : return i;
24440 :
24441 0 : sorry ("no register available for profiling %<-mcmodel=large%s%>",
24442 0 : ix86_cmodel == CM_LARGE_PIC ? " -fPIC" : "");
24443 :
24444 0 : return R10_REG;
24445 : }
24446 :
24447 : /* Output assembler code to FILE to increment profiler label # LABELNO
24448 : for profiling a function entry. */
24449 : void
24450 395 : x86_function_profiler (FILE *file, int labelno ATTRIBUTE_UNUSED)
24451 : {
24452 395 : if (cfun->machine->insn_queued_at_entrance)
24453 : {
24454 7 : if (cfun->machine->insn_queued_at_entrance == TYPE_ENDBR)
24455 6 : fprintf (file, "\t%s\n", TARGET_64BIT ? "endbr64" : "endbr32");
24456 7 : unsigned int patch_area_size
24457 7 : = crtl->patch_area_size - crtl->patch_area_entry;
24458 7 : if (patch_area_size)
24459 2 : ix86_output_patchable_area (patch_area_size,
24460 : crtl->patch_area_entry == 0);
24461 : }
24462 :
24463 395 : const char *mcount_name = MCOUNT_NAME;
24464 :
24465 395 : bool fentry_section_p
24466 395 : = (flag_record_mcount
24467 775 : || lookup_attribute ("fentry_section",
24468 380 : DECL_ATTRIBUTES (current_function_decl)));
24469 :
24470 : const char *label = fentry_section_p ? "1:" : "";
24471 :
24472 395 : if (current_fentry_name (&mcount_name))
24473 : ;
24474 393 : else if (fentry_name)
24475 1 : mcount_name = fentry_name;
24476 392 : else if (flag_fentry)
24477 380 : mcount_name = MCOUNT_NAME_BEFORE_PROLOGUE;
24478 :
24479 395 : if (TARGET_64BIT)
24480 : {
24481 : #ifndef NO_PROFILE_COUNTERS
24482 : if (ASSEMBLER_DIALECT == ASM_INTEL)
24483 : fprintf (file, "\tlea\tr11, %sP%d[rip]\n", LPREFIX, labelno);
24484 : else
24485 : fprintf (file, "\tleaq\t%sP%d(%%rip), %%r11\n", LPREFIX, labelno);
24486 : #endif
24487 :
24488 394 : int scratch;
24489 394 : const char *reg;
24490 394 : char legacy_reg[4] = { 0 };
24491 :
24492 394 : if (!TARGET_PECOFF)
24493 : {
24494 394 : switch (ix86_cmodel)
24495 : {
24496 7 : case CM_LARGE:
24497 7 : scratch = x86_64_select_profile_regnum (true);
24498 7 : reg = hi_reg_name[scratch];
24499 7 : if (LEGACY_INT_REGNO_P (scratch))
24500 : {
24501 0 : legacy_reg[0] = 'r';
24502 0 : legacy_reg[1] = reg[0];
24503 0 : legacy_reg[2] = reg[1];
24504 0 : reg = legacy_reg;
24505 : }
24506 7 : if (ASSEMBLER_DIALECT == ASM_INTEL)
24507 1 : fprintf (file, "%s\tmovabs\t%s, OFFSET FLAT:%s\n"
24508 : "\tcall\t%s\n", label, reg, mcount_name,
24509 : reg);
24510 : else
24511 6 : fprintf (file, "%s\tmovabsq\t$%s, %%%s\n\tcall\t*%%%s\n",
24512 : label, mcount_name, reg, reg);
24513 : break;
24514 10 : case CM_LARGE_PIC:
24515 : #ifdef NO_PROFILE_COUNTERS
24516 10 : scratch = x86_64_select_profile_regnum (false);
24517 10 : reg = hi_reg_name[scratch];
24518 10 : if (LEGACY_INT_REGNO_P (scratch))
24519 : {
24520 0 : legacy_reg[0] = 'r';
24521 0 : legacy_reg[1] = reg[0];
24522 0 : legacy_reg[2] = reg[1];
24523 0 : reg = legacy_reg;
24524 : }
24525 10 : if (ASSEMBLER_DIALECT == ASM_INTEL)
24526 : {
24527 1 : fprintf (file, "1:movabs\tr11, "
24528 : "OFFSET FLAT:_GLOBAL_OFFSET_TABLE_-1b\n");
24529 1 : fprintf (file, "\tlea\t%s, 1b[rip]\n", reg);
24530 1 : fprintf (file, "\tadd\t%s, r11\n", reg);
24531 1 : fprintf (file, "\tmovabs\tr11, OFFSET FLAT:%s@PLTOFF\n",
24532 : mcount_name);
24533 1 : fprintf (file, "\tadd\t%s, r11\n", reg);
24534 1 : fprintf (file, "\tcall\t%s\n", reg);
24535 1 : break;
24536 : }
24537 9 : fprintf (file,
24538 : "1:\tmovabsq\t$_GLOBAL_OFFSET_TABLE_-1b, %%r11\n");
24539 9 : fprintf (file, "\tleaq\t1b(%%rip), %%%s\n", reg);
24540 9 : fprintf (file, "\taddq\t%%r11, %%%s\n", reg);
24541 9 : fprintf (file, "\tmovabsq\t$%s@PLTOFF, %%r11\n", mcount_name);
24542 9 : fprintf (file, "\taddq\t%%r11, %%%s\n", reg);
24543 9 : fprintf (file, "\tcall\t*%%%s\n", reg);
24544 : #else
24545 : sorry ("profiling %<-mcmodel=large%> with PIC is not supported");
24546 : #endif
24547 9 : break;
24548 12 : case CM_SMALL_PIC:
24549 12 : case CM_MEDIUM_PIC:
24550 12 : if (!flag_plt)
24551 : {
24552 3 : if (ASSEMBLER_DIALECT == ASM_INTEL)
24553 0 : fprintf (file, "%s\tcall\t[QWORD PTR %s@GOTPCREL[rip]]\n",
24554 : label, mcount_name);
24555 : else
24556 3 : fprintf (file, "%s\tcall\t*%s@GOTPCREL(%%rip)\n",
24557 : label, mcount_name);
24558 : break;
24559 : }
24560 : /* fall through */
24561 374 : default:
24562 374 : x86_print_call_or_nop (file, mcount_name, label);
24563 374 : break;
24564 : }
24565 : }
24566 : else
24567 : x86_print_call_or_nop (file, mcount_name, label);
24568 : }
24569 1 : else if (flag_pic)
24570 : {
24571 : #ifndef NO_PROFILE_COUNTERS
24572 : if (ASSEMBLER_DIALECT == ASM_INTEL)
24573 : fprintf (file,
24574 : "\tlea\t" PROFILE_COUNT_REGISTER ", %sP%d@GOTOFF[ebx]\n",
24575 : LPREFIX, labelno);
24576 : else
24577 : fprintf (file,
24578 : "\tleal\t%sP%d@GOTOFF(%%ebx), %%" PROFILE_COUNT_REGISTER "\n",
24579 : LPREFIX, labelno);
24580 : #endif
24581 0 : if (flag_plt)
24582 0 : x86_print_call_or_nop (file, mcount_name, label);
24583 0 : else if (ASSEMBLER_DIALECT == ASM_INTEL)
24584 0 : fprintf (file, "%s\tcall\t[DWORD PTR %s@GOT[ebx]]\n",
24585 : label, mcount_name);
24586 : else
24587 0 : fprintf (file, "%s\tcall\t*%s@GOT(%%ebx)\n",
24588 : label, mcount_name);
24589 : }
24590 : else
24591 : {
24592 : #ifndef NO_PROFILE_COUNTERS
24593 : if (ASSEMBLER_DIALECT == ASM_INTEL)
24594 : fprintf (file,
24595 : "\tmov\t" PROFILE_COUNT_REGISTER ", OFFSET FLAT:%sP%d\n",
24596 : LPREFIX, labelno);
24597 : else
24598 : fprintf (file, "\tmovl\t$%sP%d, %%" PROFILE_COUNT_REGISTER "\n",
24599 : LPREFIX, labelno);
24600 : #endif
24601 1 : x86_print_call_or_nop (file, mcount_name, label);
24602 : }
24603 :
24604 395 : if (fentry_section_p)
24605 : {
24606 16 : const char *sname = "__mcount_loc";
24607 :
24608 16 : if (current_fentry_section (&sname))
24609 : ;
24610 14 : else if (fentry_section)
24611 1 : sname = fentry_section;
24612 :
24613 16 : fprintf (file, "\t.section %s, \"a\",@progbits\n", sname);
24614 16 : fprintf (file, "\t.%s 1b\n", TARGET_64BIT ? "quad" : "long");
24615 16 : fprintf (file, "\t.previous\n");
24616 : }
24617 395 : }
24618 :
24619 : /* We don't have exact information about the insn sizes, but we may assume
24620 : quite safely that we are informed about all 1 byte insns and memory
24621 : address sizes. This is enough to eliminate unnecessary padding in
24622 : 99% of cases. */
24623 :
24624 : int
24625 390837486 : ix86_min_insn_size (rtx_insn *insn)
24626 : {
24627 390837486 : int l = 0, len;
24628 :
24629 390837486 : if (!INSN_P (insn) || !active_insn_p (insn))
24630 500713 : return 0;
24631 :
24632 : /* Discard alignments we've emit and jump instructions. */
24633 390336773 : if (GET_CODE (PATTERN (insn)) == UNSPEC_VOLATILE
24634 390336773 : && XINT (PATTERN (insn), 1) == UNSPECV_ALIGN)
24635 : return 0;
24636 :
24637 : /* Important case - calls are always 5 bytes.
24638 : It is common to have many calls in the row. */
24639 390336769 : if (CALL_P (insn)
24640 9219655 : && symbolic_reference_mentioned_p (PATTERN (insn))
24641 399224170 : && !SIBLING_CALL_P (insn))
24642 : return 5;
24643 381685411 : len = get_attr_length (insn);
24644 381685411 : if (len <= 1)
24645 : return 1;
24646 :
24647 : /* For normal instructions we rely on get_attr_length being exact,
24648 : with a few exceptions. */
24649 373034817 : if (!JUMP_P (insn))
24650 : {
24651 367595839 : enum attr_type type = get_attr_type (insn);
24652 :
24653 367595839 : switch (type)
24654 : {
24655 96077 : case TYPE_MULTI:
24656 96077 : if (GET_CODE (PATTERN (insn)) == ASM_INPUT
24657 96077 : || asm_noperands (PATTERN (insn)) >= 0)
24658 527 : return 0;
24659 : break;
24660 : case TYPE_OTHER:
24661 : case TYPE_FCMP:
24662 : break;
24663 : default:
24664 : /* Otherwise trust get_attr_length. */
24665 : return len;
24666 : }
24667 :
24668 476158 : l = get_attr_length_address (insn);
24669 476158 : if (l < 4 && symbolic_reference_mentioned_p (PATTERN (insn)))
24670 : l = 4;
24671 : }
24672 385654 : if (l)
24673 90504 : return 1+l;
24674 : else
24675 5824632 : return 2;
24676 : }
24677 :
24678 : #ifdef ASM_OUTPUT_MAX_SKIP_ALIGN
24679 :
24680 : /* AMD K8 core mispredicts jumps when there are more than 3 jumps in 16 byte
24681 : window. */
24682 :
24683 : static void
24684 45472 : ix86_avoid_jump_mispredicts (void)
24685 : {
24686 45472 : rtx_insn *insn, *start = get_insns ();
24687 45472 : int nbytes = 0, njumps = 0;
24688 45472 : bool isjump = false;
24689 :
24690 : /* Look for all minimal intervals of instructions containing 4 jumps.
24691 : The intervals are bounded by START and INSN. NBYTES is the total
24692 : size of instructions in the interval including INSN and not including
24693 : START. When the NBYTES is smaller than 16 bytes, it is possible
24694 : that the end of START and INSN ends up in the same 16byte page.
24695 :
24696 : The smallest offset in the page INSN can start is the case where START
24697 : ends on the offset 0. Offset of INSN is then NBYTES - sizeof (INSN).
24698 : We add p2align to 16byte window with maxskip 15 - NBYTES + sizeof (INSN).
24699 :
24700 : Don't consider asm goto as jump, while it can contain a jump, it doesn't
24701 : have to, control transfer to label(s) can be performed through other
24702 : means, and also we estimate minimum length of all asm stmts as 0. */
24703 701057 : for (insn = start; insn; insn = NEXT_INSN (insn))
24704 : {
24705 655585 : int min_size;
24706 :
24707 655585 : if (LABEL_P (insn))
24708 : {
24709 956 : align_flags alignment = label_to_alignment (insn);
24710 956 : int align = alignment.levels[0].log;
24711 956 : int max_skip = alignment.levels[0].maxskip;
24712 :
24713 956 : if (max_skip > 15)
24714 : max_skip = 15;
24715 : /* If align > 3, only up to 16 - max_skip - 1 bytes can be
24716 : already in the current 16 byte page, because otherwise
24717 : ASM_OUTPUT_MAX_SKIP_ALIGN could skip max_skip or fewer
24718 : bytes to reach 16 byte boundary. */
24719 956 : if (align <= 0
24720 313 : || (align <= 3 && max_skip != (1 << align) - 1))
24721 956 : max_skip = 0;
24722 956 : if (dump_file)
24723 0 : fprintf (dump_file, "Label %i with max_skip %i\n",
24724 0 : INSN_UID (insn), max_skip);
24725 956 : if (max_skip)
24726 : {
24727 6120 : while (nbytes + max_skip >= 16)
24728 : {
24729 5807 : start = NEXT_INSN (start);
24730 295 : if ((JUMP_P (start) && asm_noperands (PATTERN (start)) < 0)
24731 5824 : || CALL_P (start))
24732 335 : njumps--, isjump = true;
24733 : else
24734 : isjump = false;
24735 5807 : nbytes -= ix86_min_insn_size (start);
24736 : }
24737 : }
24738 956 : continue;
24739 956 : }
24740 :
24741 654629 : min_size = ix86_min_insn_size (insn);
24742 654629 : nbytes += min_size;
24743 654629 : if (dump_file)
24744 0 : fprintf (dump_file, "Insn %i estimated to %i bytes\n",
24745 0 : INSN_UID (insn), min_size);
24746 46596 : if ((JUMP_P (insn) && asm_noperands (PATTERN (insn)) < 0)
24747 654649 : || CALL_P (insn))
24748 47609 : njumps++;
24749 : else
24750 607020 : continue;
24751 :
24752 55969 : while (njumps > 3)
24753 : {
24754 8360 : start = NEXT_INSN (start);
24755 527 : if ((JUMP_P (start) && asm_noperands (PATTERN (start)) < 0)
24756 8360 : || CALL_P (start))
24757 1227 : njumps--, isjump = true;
24758 : else
24759 : isjump = false;
24760 8360 : nbytes -= ix86_min_insn_size (start);
24761 : }
24762 47609 : gcc_assert (njumps >= 0);
24763 47609 : if (dump_file)
24764 0 : fprintf (dump_file, "Interval %i to %i has %i bytes\n",
24765 0 : INSN_UID (start), INSN_UID (insn), nbytes);
24766 :
24767 47609 : if (njumps == 3 && isjump && nbytes < 16)
24768 : {
24769 36 : int padsize = 15 - nbytes + ix86_min_insn_size (insn);
24770 :
24771 36 : if (dump_file)
24772 0 : fprintf (dump_file, "Padding insn %i by %i bytes!\n",
24773 0 : INSN_UID (insn), padsize);
24774 36 : emit_insn_before (gen_max_skip_align (GEN_INT (4), GEN_INT (padsize)), insn);
24775 : }
24776 : }
24777 45472 : }
24778 : #endif
24779 :
24780 : /* AMD Athlon works faster
24781 : when RET is not destination of conditional jump or directly preceded
24782 : by other jump instruction. We avoid the penalty by inserting NOP just
24783 : before the RET instructions in such cases. */
24784 : static void
24785 45192 : ix86_pad_returns (void)
24786 : {
24787 45192 : edge e;
24788 45192 : edge_iterator ei;
24789 :
24790 90409 : FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
24791 : {
24792 45217 : basic_block bb = e->src;
24793 45217 : rtx_insn *ret = BB_END (bb);
24794 45217 : rtx_insn *prev;
24795 45217 : bool replace = false;
24796 :
24797 45207 : if (!JUMP_P (ret) || !ANY_RETURN_P (PATTERN (ret))
24798 90424 : || optimize_bb_for_size_p (bb))
24799 23 : continue;
24800 179989 : for (prev = PREV_INSN (ret); prev; prev = PREV_INSN (prev))
24801 134377 : if (active_insn_p (prev) || LABEL_P (prev))
24802 : break;
24803 45194 : if (prev && LABEL_P (prev))
24804 : {
24805 43 : edge e;
24806 43 : edge_iterator ei;
24807 :
24808 56 : FOR_EACH_EDGE (e, ei, bb->preds)
24809 146 : if (EDGE_FREQUENCY (e) && e->src->index >= 0
24810 97 : && !(e->flags & EDGE_FALLTHRU))
24811 : {
24812 : replace = true;
24813 : break;
24814 : }
24815 : }
24816 43 : if (!replace)
24817 : {
24818 45158 : prev = prev_active_insn (ret);
24819 45158 : if (prev
24820 45158 : && ((JUMP_P (prev) && any_condjump_p (prev))
24821 44723 : || CALL_P (prev)))
24822 : replace = true;
24823 : /* Empty functions get branch mispredict even when
24824 : the jump destination is not visible to us. */
24825 45158 : if (!prev && !optimize_function_for_size_p (cfun))
24826 : replace = true;
24827 : }
24828 44740 : if (replace)
24829 : {
24830 488 : emit_jump_insn_before (gen_simple_return_internal_long (), ret);
24831 488 : delete_insn (ret);
24832 : }
24833 : }
24834 45192 : }
24835 :
24836 : /* Count the minimum number of instructions in BB. Return 4 if the
24837 : number of instructions >= 4. */
24838 :
24839 : static int
24840 42 : ix86_count_insn_bb (basic_block bb)
24841 : {
24842 42 : rtx_insn *insn;
24843 42 : int insn_count = 0;
24844 :
24845 : /* Count number of instructions in this block. Return 4 if the number
24846 : of instructions >= 4. */
24847 296 : FOR_BB_INSNS (bb, insn)
24848 : {
24849 : /* Only happen in exit blocks. */
24850 290 : if (JUMP_P (insn)
24851 290 : && ANY_RETURN_P (PATTERN (insn)))
24852 : break;
24853 :
24854 266 : if (NONDEBUG_INSN_P (insn)
24855 102 : && GET_CODE (PATTERN (insn)) != USE
24856 350 : && GET_CODE (PATTERN (insn)) != CLOBBER)
24857 : {
24858 84 : insn_count++;
24859 84 : if (insn_count >= 4)
24860 : return insn_count;
24861 : }
24862 : }
24863 :
24864 : return insn_count;
24865 : }
24866 :
24867 :
24868 : /* Count the minimum number of instructions in code path in BB.
24869 : Return 4 if the number of instructions >= 4. */
24870 :
24871 : static int
24872 62 : ix86_count_insn (basic_block bb)
24873 : {
24874 62 : edge e;
24875 62 : edge_iterator ei;
24876 62 : int min_prev_count;
24877 :
24878 : /* Only bother counting instructions along paths with no
24879 : more than 2 basic blocks between entry and exit. Given
24880 : that BB has an edge to exit, determine if a predecessor
24881 : of BB has an edge from entry. If so, compute the number
24882 : of instructions in the predecessor block. If there
24883 : happen to be multiple such blocks, compute the minimum. */
24884 62 : min_prev_count = 4;
24885 145 : FOR_EACH_EDGE (e, ei, bb->preds)
24886 : {
24887 109 : edge prev_e;
24888 109 : edge_iterator prev_ei;
24889 :
24890 109 : if (e->src == ENTRY_BLOCK_PTR_FOR_FN (cfun))
24891 : {
24892 26 : min_prev_count = 0;
24893 26 : break;
24894 : }
24895 182 : FOR_EACH_EDGE (prev_e, prev_ei, e->src->preds)
24896 : {
24897 109 : if (prev_e->src == ENTRY_BLOCK_PTR_FOR_FN (cfun))
24898 : {
24899 10 : int count = ix86_count_insn_bb (e->src);
24900 10 : if (count < min_prev_count)
24901 83 : min_prev_count = count;
24902 : break;
24903 : }
24904 : }
24905 : }
24906 :
24907 62 : if (min_prev_count < 4)
24908 32 : min_prev_count += ix86_count_insn_bb (bb);
24909 :
24910 62 : return min_prev_count;
24911 : }
24912 :
24913 : /* Pad short function to 4 instructions. */
24914 :
24915 : static void
24916 63 : ix86_pad_short_function (void)
24917 : {
24918 63 : edge e;
24919 63 : edge_iterator ei;
24920 :
24921 128 : FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
24922 : {
24923 65 : rtx_insn *ret = BB_END (e->src);
24924 65 : if (JUMP_P (ret) && ANY_RETURN_P (PATTERN (ret)))
24925 : {
24926 62 : int insn_count = ix86_count_insn (e->src);
24927 :
24928 : /* Pad short function. */
24929 62 : if (insn_count < 4)
24930 : {
24931 : rtx_insn *insn = ret;
24932 :
24933 : /* Find epilogue. */
24934 : while (insn
24935 60 : && (!NOTE_P (insn)
24936 26 : || NOTE_KIND (insn) != NOTE_INSN_EPILOGUE_BEG))
24937 37 : insn = PREV_INSN (insn);
24938 :
24939 23 : if (!insn)
24940 0 : insn = ret;
24941 :
24942 : /* Two NOPs count as one instruction. */
24943 23 : insn_count = 2 * (4 - insn_count);
24944 23 : emit_insn_before (gen_nops (GEN_INT (insn_count)), insn);
24945 : }
24946 : }
24947 : }
24948 63 : }
24949 :
24950 : /* Fix up a Windows system unwinder issue. If an EH region falls through into
24951 : the epilogue, the Windows system unwinder will apply epilogue logic and
24952 : produce incorrect offsets. This can be avoided by adding a nop between
24953 : the last insn that can throw and the first insn of the epilogue. */
24954 :
24955 : static void
24956 0 : ix86_seh_fixup_eh_fallthru (void)
24957 : {
24958 0 : edge e;
24959 0 : edge_iterator ei;
24960 :
24961 0 : FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
24962 : {
24963 0 : rtx_insn *insn, *next;
24964 :
24965 : /* Find the beginning of the epilogue. */
24966 0 : for (insn = BB_END (e->src); insn != NULL; insn = PREV_INSN (insn))
24967 0 : if (NOTE_P (insn) && NOTE_KIND (insn) == NOTE_INSN_EPILOGUE_BEG)
24968 : break;
24969 0 : if (insn == NULL)
24970 0 : continue;
24971 :
24972 : /* We only care about preceding insns that can throw. */
24973 0 : insn = prev_active_insn (insn);
24974 0 : if (insn == NULL || !can_throw_internal (insn))
24975 0 : continue;
24976 :
24977 : /* Do not separate calls from their debug information. */
24978 0 : for (next = NEXT_INSN (insn); next != NULL; next = NEXT_INSN (next))
24979 0 : if (NOTE_P (next) && NOTE_KIND (next) == NOTE_INSN_VAR_LOCATION)
24980 0 : insn = next;
24981 : else
24982 : break;
24983 :
24984 0 : emit_insn_after (gen_nops (const1_rtx), insn);
24985 : }
24986 0 : }
24987 : /* Split vector load from parm_decl to elemental loads to avoid STLF
24988 : stalls. */
24989 : static void
24990 983238 : ix86_split_stlf_stall_load ()
24991 : {
24992 983238 : rtx_insn* insn, *start = get_insns ();
24993 983238 : unsigned window = 0;
24994 :
24995 27138202 : for (insn = start; insn; insn = NEXT_INSN (insn))
24996 : {
24997 27137361 : if (!NONDEBUG_INSN_P (insn))
24998 15366847 : continue;
24999 11770514 : window++;
25000 : /* Insert 64 vaddps %xmm18, %xmm19, %xmm20(no dependence between each
25001 : other, just emulate for pipeline) before stalled load, stlf stall
25002 : case is as fast as no stall cases on CLX.
25003 : Since CFG is freed before machine_reorg, just do a rough
25004 : calculation of the window according to the layout. */
25005 11770514 : if (window > (unsigned) x86_stlf_window_ninsns)
25006 : return;
25007 :
25008 11752283 : if (any_uncondjump_p (insn)
25009 11716252 : || ANY_RETURN_P (PATTERN (insn))
25010 23089649 : || CALL_P (insn))
25011 : return;
25012 :
25013 10788117 : rtx set = single_set (insn);
25014 10788117 : if (!set)
25015 439316 : continue;
25016 10348801 : rtx src = SET_SRC (set);
25017 20697244 : if (!MEM_P (src)
25018 : /* Only handle V2DFmode load since it doesn't need any scratch
25019 : register. */
25020 1472794 : || GET_MODE (src) != E_V2DFmode
25021 5662 : || !MEM_EXPR (src)
25022 10352907 : || TREE_CODE (get_base_address (MEM_EXPR (src))) != PARM_DECL)
25023 10348443 : continue;
25024 :
25025 358 : rtx zero = CONST0_RTX (V2DFmode);
25026 358 : rtx dest = SET_DEST (set);
25027 358 : rtx m = adjust_address (src, DFmode, 0);
25028 358 : rtx loadlpd = gen_sse2_loadlpd (dest, zero, m);
25029 358 : emit_insn_before (loadlpd, insn);
25030 358 : m = adjust_address (src, DFmode, 8);
25031 358 : rtx loadhpd = gen_sse2_loadhpd (dest, dest, m);
25032 358 : if (dump_file && (dump_flags & TDF_DETAILS))
25033 : {
25034 0 : fputs ("Due to potential STLF stall, split instruction:\n",
25035 : dump_file);
25036 0 : print_rtl_single (dump_file, insn);
25037 0 : fputs ("To:\n", dump_file);
25038 0 : print_rtl_single (dump_file, loadlpd);
25039 0 : print_rtl_single (dump_file, loadhpd);
25040 : }
25041 358 : PATTERN (insn) = loadhpd;
25042 358 : INSN_CODE (insn) = -1;
25043 358 : gcc_assert (recog_memoized (insn) != -1);
25044 : }
25045 : }
25046 :
25047 : /* Implement machine specific optimizations. We implement padding of returns
25048 : for K8 CPUs and pass to avoid 4 jumps in the single 16 byte window. */
25049 : static void
25050 1493941 : ix86_reorg (void)
25051 : {
25052 : /* We are freeing block_for_insn in the toplev to keep compatibility
25053 : with old MDEP_REORGS that are not CFG based. Recompute it now. */
25054 1493941 : compute_bb_for_insn ();
25055 :
25056 1493941 : if (TARGET_SEH && current_function_has_exception_handlers ())
25057 : ix86_seh_fixup_eh_fallthru ();
25058 :
25059 1493941 : if (optimize && optimize_function_for_speed_p (cfun))
25060 : {
25061 985561 : if (TARGET_SSE2)
25062 983238 : ix86_split_stlf_stall_load ();
25063 985561 : if (TARGET_PAD_SHORT_FUNCTION)
25064 63 : ix86_pad_short_function ();
25065 985498 : else if (TARGET_PAD_RETURNS)
25066 45192 : ix86_pad_returns ();
25067 : #ifdef ASM_OUTPUT_MAX_SKIP_ALIGN
25068 985561 : if (TARGET_FOUR_JUMP_LIMIT)
25069 45472 : ix86_avoid_jump_mispredicts ();
25070 : #endif
25071 : }
25072 1493941 : }
25073 :
25074 : /* Return nonzero when QImode register that must be represented via REX prefix
25075 : is used. */
25076 : bool
25077 8720149 : x86_extended_QIreg_mentioned_p (rtx_insn *insn)
25078 : {
25079 8720149 : int i;
25080 8720149 : extract_insn_cached (insn);
25081 33047129 : for (i = 0; i < recog_data.n_operands; i++)
25082 4696962 : if (GENERAL_REG_P (recog_data.operand[i])
25083 21834818 : && !QI_REGNO_P (REGNO (recog_data.operand[i])))
25084 : return true;
25085 : return false;
25086 : }
25087 :
25088 : /* Return true when INSN mentions register that must be encoded using REX
25089 : prefix. */
25090 : bool
25091 199774093 : x86_extended_reg_mentioned_p (rtx insn)
25092 : {
25093 199774093 : subrtx_iterator::array_type array;
25094 1044690843 : FOR_EACH_SUBRTX (iter, array, INSN_P (insn) ? PATTERN (insn) : insn, NONCONST)
25095 : {
25096 893549284 : const_rtx x = *iter;
25097 893549284 : if (REG_P (x)
25098 893549284 : && (REX_INT_REGNO_P (REGNO (x)) || REX_SSE_REGNO_P (REGNO (x))
25099 257544522 : || REX2_INT_REGNO_P (REGNO (x))))
25100 48632534 : return true;
25101 : }
25102 151141559 : return false;
25103 199774093 : }
25104 :
25105 : /* Return true when INSN mentions register that must be encoded using REX2
25106 : prefix. */
25107 : bool
25108 2064731 : x86_extended_rex2reg_mentioned_p (rtx insn)
25109 : {
25110 2064731 : subrtx_iterator::array_type array;
25111 9590436 : FOR_EACH_SUBRTX (iter, array, INSN_P (insn) ? PATTERN (insn) : insn, NONCONST)
25112 : {
25113 7526362 : const_rtx x = *iter;
25114 7526362 : if (REG_P (x) && REX2_INT_REGNO_P (REGNO (x)))
25115 657 : return true;
25116 : }
25117 2064074 : return false;
25118 2064731 : }
25119 :
25120 : /* Return true when rtx operands mentions register that must be encoded using
25121 : evex prefix. */
25122 : bool
25123 10 : x86_evex_reg_mentioned_p (rtx operands[], int nops)
25124 : {
25125 10 : int i;
25126 28 : for (i = 0; i < nops; i++)
25127 22 : if (EXT_REX_SSE_REG_P (operands[i])
25128 40 : || x86_extended_rex2reg_mentioned_p (operands[i]))
25129 4 : return true;
25130 : return false;
25131 : }
25132 :
25133 : /* If profitable, negate (without causing overflow) integer constant
25134 : of mode MODE at location LOC. Return true in this case. */
25135 : bool
25136 5931978 : x86_maybe_negate_const_int (rtx *loc, machine_mode mode)
25137 : {
25138 5931978 : HOST_WIDE_INT val;
25139 :
25140 5931978 : if (!CONST_INT_P (*loc))
25141 : return false;
25142 :
25143 5005332 : switch (mode)
25144 : {
25145 2862974 : case E_DImode:
25146 : /* DImode x86_64 constants must fit in 32 bits. */
25147 2862974 : gcc_assert (x86_64_immediate_operand (*loc, mode));
25148 :
25149 : mode = SImode;
25150 : break;
25151 :
25152 : case E_SImode:
25153 : case E_HImode:
25154 : case E_QImode:
25155 : break;
25156 :
25157 0 : default:
25158 0 : gcc_unreachable ();
25159 : }
25160 :
25161 : /* Avoid overflows. */
25162 5005332 : if (mode_signbit_p (mode, *loc))
25163 : return false;
25164 :
25165 5004858 : val = INTVAL (*loc);
25166 :
25167 : /* Make things pretty and `subl $4,%eax' rather than `addl $-4,%eax'.
25168 : Exceptions: -128 encodes smaller than 128, so swap sign and op. */
25169 5004858 : if ((val < 0 && val != -128)
25170 3291272 : || val == 128)
25171 : {
25172 1725097 : *loc = GEN_INT (-val);
25173 1725097 : return true;
25174 : }
25175 :
25176 : return false;
25177 : }
25178 :
25179 : /* Generate an unsigned DImode/SImode to FP conversion. This is the same code
25180 : optabs would emit if we didn't have TFmode patterns. */
25181 :
25182 : void
25183 4585 : x86_emit_floatuns (rtx operands[2])
25184 : {
25185 4585 : rtx_code_label *neglab, *donelab;
25186 4585 : rtx i0, i1, f0, in, out;
25187 4585 : machine_mode mode, inmode;
25188 :
25189 4585 : inmode = GET_MODE (operands[1]);
25190 4585 : gcc_assert (inmode == SImode || inmode == DImode);
25191 :
25192 4585 : out = operands[0];
25193 4585 : in = force_reg (inmode, operands[1]);
25194 4585 : mode = GET_MODE (out);
25195 4585 : neglab = gen_label_rtx ();
25196 4585 : donelab = gen_label_rtx ();
25197 4585 : f0 = gen_reg_rtx (mode);
25198 :
25199 4585 : emit_cmp_and_jump_insns (in, const0_rtx, LT, const0_rtx, inmode, 0, neglab);
25200 :
25201 4585 : expand_float (out, in, 0);
25202 :
25203 4585 : emit_jump_insn (gen_jump (donelab));
25204 4585 : emit_barrier ();
25205 :
25206 4585 : emit_label (neglab);
25207 :
25208 4585 : i0 = expand_simple_binop (inmode, LSHIFTRT, in, const1_rtx, NULL,
25209 : 1, OPTAB_DIRECT);
25210 4585 : i1 = expand_simple_binop (inmode, AND, in, const1_rtx, NULL,
25211 : 1, OPTAB_DIRECT);
25212 4585 : i0 = expand_simple_binop (inmode, IOR, i0, i1, i0, 1, OPTAB_DIRECT);
25213 :
25214 4585 : expand_float (f0, i0, 0);
25215 :
25216 4585 : emit_insn (gen_rtx_SET (out, gen_rtx_PLUS (mode, f0, f0)));
25217 :
25218 4585 : emit_label (donelab);
25219 4585 : }
25220 :
25221 : /* Return the diagnostic message string if conversion from FROMTYPE to
25222 : TOTYPE is not allowed, NULL otherwise. */
25223 :
25224 : static const char *
25225 1110940074 : ix86_invalid_conversion (const_tree fromtype, const_tree totype)
25226 : {
25227 1110940074 : machine_mode from_mode = element_mode (fromtype);
25228 1110940074 : machine_mode to_mode = element_mode (totype);
25229 :
25230 1110940074 : if (!TARGET_SSE2 && from_mode != to_mode)
25231 : {
25232 : /* Do no allow conversions to/from BFmode/HFmode scalar types
25233 : when TARGET_SSE2 is not available. */
25234 468442 : if (from_mode == BFmode)
25235 : return N_("invalid conversion from type %<__bf16%> "
25236 : "without option %<-msse2%>");
25237 468441 : if (from_mode == HFmode)
25238 : return N_("invalid conversion from type %<_Float16%> "
25239 : "without option %<-msse2%>");
25240 468441 : if (to_mode == BFmode)
25241 : return N_("invalid conversion to type %<__bf16%> "
25242 : "without option %<-msse2%>");
25243 468441 : if (to_mode == HFmode)
25244 : return N_("invalid conversion to type %<_Float16%> "
25245 : "without option %<-msse2%>");
25246 : }
25247 :
25248 : /* Warn for silent implicit conversion between __bf16 and short,
25249 : since __bfloat16 is refined as real __bf16 instead of short
25250 : since GCC13. */
25251 1110940072 : if (element_mode (fromtype) != element_mode (totype)
25252 1110940072 : && (TARGET_AVX512BF16 || TARGET_AVXNECONVERT))
25253 : {
25254 : /* Warn for silent implicit conversion where user may expect
25255 : a bitcast. */
25256 7794044 : if ((TYPE_MODE (fromtype) == BFmode
25257 279 : && TYPE_MODE (totype) == HImode)
25258 7794322 : || (TYPE_MODE (totype) == BFmode
25259 423 : && TYPE_MODE (fromtype) == HImode))
25260 1 : warning (0, "%<__bfloat16%> is redefined from typedef %<short%> "
25261 : "to real %<__bf16%> since GCC 13.1, be careful of "
25262 : "implicit conversion between %<__bf16%> and %<short%>; "
25263 : "an explicit bitcast may be needed here");
25264 : }
25265 :
25266 : /* Conversion allowed. */
25267 : return NULL;
25268 : }
25269 :
25270 : /* Return the diagnostic message string if the unary operation OP is
25271 : not permitted on TYPE, NULL otherwise. */
25272 :
25273 : static const char *
25274 92367153 : ix86_invalid_unary_op (int op, const_tree type)
25275 : {
25276 92367153 : machine_mode mmode = element_mode (type);
25277 : /* Reject all single-operand operations on BFmode/HFmode except for &
25278 : when TARGET_SSE2 is not available. */
25279 92367153 : if (!TARGET_SSE2 && op != ADDR_EXPR)
25280 : {
25281 111212 : if (mmode == BFmode)
25282 : return N_("operation not permitted on type %<__bf16%> "
25283 : "without option %<-msse2%>");
25284 111212 : if (mmode == HFmode)
25285 0 : return N_("operation not permitted on type %<_Float16%> "
25286 : "without option %<-msse2%>");
25287 : }
25288 :
25289 : /* Operation allowed. */
25290 : return NULL;
25291 : }
25292 :
25293 : /* Return the diagnostic message string if the binary operation OP is
25294 : not permitted on TYPE1 and TYPE2, NULL otherwise. */
25295 :
25296 : static const char *
25297 167576333 : ix86_invalid_binary_op (int op ATTRIBUTE_UNUSED, const_tree type1,
25298 : const_tree type2)
25299 : {
25300 167576333 : machine_mode type1_mode = element_mode (type1);
25301 167576333 : machine_mode type2_mode = element_mode (type2);
25302 : /* Reject all 2-operand operations on BFmode or HFmode
25303 : when TARGET_SSE2 is not available. */
25304 167576333 : if (!TARGET_SSE2)
25305 : {
25306 1009932 : if (type1_mode == BFmode || type2_mode == BFmode)
25307 : return N_("operation not permitted on type %<__bf16%> "
25308 : "without option %<-msse2%>");
25309 :
25310 1009932 : if (type1_mode == HFmode || type2_mode == HFmode)
25311 0 : return N_("operation not permitted on type %<_Float16%> "
25312 : "without option %<-msse2%>");
25313 : }
25314 :
25315 : /* Operation allowed. */
25316 : return NULL;
25317 : }
25318 :
25319 :
25320 : /* Target hook for scalar_mode_supported_p. */
25321 : static bool
25322 4659157 : ix86_scalar_mode_supported_p (scalar_mode mode)
25323 : {
25324 4659157 : if (DECIMAL_FLOAT_MODE_P (mode))
25325 642450 : return default_decimal_float_supported_p ();
25326 4016707 : else if (mode == TFmode)
25327 : return true;
25328 3688747 : else if (mode == HFmode || mode == BFmode)
25329 : return true;
25330 : else
25331 3034810 : return default_scalar_mode_supported_p (mode);
25332 : }
25333 :
25334 : /* Implement TARGET_LIBGCC_FLOATING_POINT_MODE_SUPPORTED_P - return TRUE
25335 : if MODE is HFmode, and punt to the generic implementation otherwise. */
25336 :
25337 : static bool
25338 2252486 : ix86_libgcc_floating_mode_supported_p (scalar_float_mode mode)
25339 : {
25340 : /* NB: Always return TRUE for HFmode so that the _Float16 type will
25341 : be defined by the C front-end for AVX512FP16 intrinsics. We will
25342 : issue an error in ix86_expand_move for HFmode if AVX512FP16 isn't
25343 : enabled. */
25344 1925993 : return ((mode == HFmode || mode == BFmode)
25345 3851986 : ? true
25346 1599500 : : default_libgcc_floating_mode_supported_p (mode));
25347 : }
25348 :
25349 : /* Implements target hook vector_mode_supported_p. */
25350 : static bool
25351 1382422119 : ix86_vector_mode_supported_p (machine_mode mode)
25352 : {
25353 : /* For ia32, scalar TImode isn't supported and so V1TImode shouldn't be
25354 : either. */
25355 1520827807 : if (!TARGET_64BIT && GET_MODE_INNER (mode) == TImode)
25356 : return false;
25357 1382421915 : if (TARGET_SSE && VALID_SSE_REG_MODE (mode))
25358 : return true;
25359 1162784582 : if (TARGET_SSE2 && VALID_SSE2_REG_MODE (mode))
25360 : return true;
25361 517584274 : if (TARGET_AVX && VALID_AVX256_REG_MODE (mode))
25362 : return true;
25363 376527591 : if (TARGET_AVX512F && VALID_AVX512F_REG_MODE (mode))
25364 : return true;
25365 242000568 : if ((TARGET_MMX || TARGET_MMX_WITH_SSE)
25366 241938696 : && VALID_MMX_REG_MODE (mode))
25367 : return true;
25368 35980419 : if ((TARGET_3DNOW || TARGET_MMX_WITH_SSE)
25369 35335699 : && VALID_MMX_REG_MODE_3DNOW (mode))
25370 : return true;
25371 22835301 : if (mode == V2QImode)
25372 24855 : return true;
25373 : return false;
25374 : }
25375 :
25376 : /* Target hook for c_mode_for_suffix. */
25377 : static machine_mode
25378 198365 : ix86_c_mode_for_suffix (char suffix)
25379 : {
25380 198365 : if (suffix == 'q')
25381 : return TFmode;
25382 37 : if (suffix == 'w')
25383 : return XFmode;
25384 :
25385 0 : return VOIDmode;
25386 : }
25387 :
25388 : /* Helper function to map common constraints to non-EGPR ones.
25389 : All related constraints have h prefix, and h plus Upper letter
25390 : means the constraint is strictly EGPR enabled, while h plus
25391 : lower letter indicates the constraint is strictly gpr16 only.
25392 :
25393 : Specially for "g" constraint, split it to rmi as there is
25394 : no corresponding general constraint define for backend.
25395 :
25396 : Here is the full list to map constraints that may involve
25397 : gpr to h prefixed.
25398 :
25399 : "g" -> "jrjmi"
25400 : "r" -> "jr"
25401 : "m" -> "jm"
25402 : "<" -> "j<"
25403 : ">" -> "j>"
25404 : "o" -> "jo"
25405 : "V" -> "jV"
25406 : "p" -> "jp"
25407 : "Bm" -> "ja"
25408 : */
25409 :
25410 64 : static void map_egpr_constraints (vec<const char *> &constraints)
25411 : {
25412 74 : for (size_t i = 0; i < constraints.length(); i++)
25413 : {
25414 10 : const char *cur = constraints[i];
25415 :
25416 10 : if (startswith (cur, "=@cc"))
25417 0 : continue;
25418 :
25419 10 : int len = strlen (cur);
25420 10 : auto_vec<char> buf;
25421 :
25422 24 : for (int j = 0; j < len; j++)
25423 : {
25424 14 : switch (cur[j])
25425 : {
25426 2 : case 'g':
25427 2 : buf.safe_push ('j');
25428 2 : buf.safe_push ('r');
25429 2 : buf.safe_push ('j');
25430 2 : buf.safe_push ('m');
25431 2 : buf.safe_push ('i');
25432 2 : break;
25433 8 : case 'r':
25434 8 : case 'm':
25435 8 : case '<':
25436 8 : case '>':
25437 8 : case 'o':
25438 8 : case 'V':
25439 8 : case 'p':
25440 8 : buf.safe_push ('j');
25441 8 : buf.safe_push (cur[j]);
25442 8 : break;
25443 0 : case 'B':
25444 0 : if (cur[j + 1] == 'm')
25445 : {
25446 0 : buf.safe_push ('j');
25447 0 : buf.safe_push ('a');
25448 0 : j++;
25449 : }
25450 : else
25451 : {
25452 0 : buf.safe_push (cur[j]);
25453 0 : buf.safe_push (cur[j + 1]);
25454 0 : j++;
25455 : }
25456 : break;
25457 0 : case 'T':
25458 0 : case 'Y':
25459 0 : case 'W':
25460 0 : case 'j':
25461 0 : buf.safe_push (cur[j]);
25462 0 : buf.safe_push (cur[j + 1]);
25463 0 : j++;
25464 0 : break;
25465 0 : case '{':
25466 0 : do
25467 : {
25468 0 : buf.safe_push (cur[j]);
25469 0 : } while (cur[j++] != '}');
25470 : break;
25471 4 : default:
25472 4 : buf.safe_push (cur[j]);
25473 4 : break;
25474 : }
25475 : }
25476 10 : buf.safe_push ('\0');
25477 20 : constraints[i] = xstrdup (buf.address ());
25478 10 : }
25479 64 : }
25480 :
25481 : /* Worker function for TARGET_MD_ASM_ADJUST.
25482 :
25483 : We implement asm flag outputs, and maintain source compatibility
25484 : with the old cc0-based compiler. */
25485 :
25486 : static rtx_insn *
25487 109904 : ix86_md_asm_adjust (vec<rtx> &outputs, vec<rtx> & /*inputs*/,
25488 : vec<machine_mode> & /*input_modes*/,
25489 : vec<const char *> &constraints, vec<rtx> &/*uses*/,
25490 : vec<rtx> &clobbers, HARD_REG_SET &clobbered_regs,
25491 : location_t loc)
25492 : {
25493 109904 : bool saw_asm_flag = false;
25494 :
25495 109904 : start_sequence ();
25496 :
25497 109904 : if (TARGET_APX_EGPR && !ix86_apx_inline_asm_use_gpr32)
25498 64 : map_egpr_constraints (constraints);
25499 :
25500 298760 : for (unsigned i = 0, n = outputs.length (); i < n; ++i)
25501 : {
25502 79779 : const char *con = constraints[i];
25503 79779 : if (!startswith (con, "=@cc"))
25504 79691 : continue;
25505 88 : con += 4;
25506 88 : if (strchr (con, ',') != NULL)
25507 : {
25508 1 : error_at (loc, "alternatives not allowed in %<asm%> flag output");
25509 1 : continue;
25510 : }
25511 :
25512 87 : bool invert = false;
25513 87 : if (con[0] == 'n')
25514 19 : invert = true, con++;
25515 :
25516 87 : machine_mode mode = CCmode;
25517 87 : rtx_code code = UNKNOWN;
25518 :
25519 87 : switch (con[0])
25520 : {
25521 15 : case 'a':
25522 15 : if (con[1] == 0)
25523 : mode = CCAmode, code = EQ;
25524 4 : else if (con[1] == 'e' && con[2] == 0)
25525 : mode = CCCmode, code = NE;
25526 : break;
25527 11 : case 'b':
25528 11 : if (con[1] == 0)
25529 : mode = CCCmode, code = EQ;
25530 6 : else if (con[1] == 'e' && con[2] == 0)
25531 : mode = CCAmode, code = NE;
25532 : break;
25533 14 : case 'c':
25534 14 : if (con[1] == 0)
25535 : mode = CCCmode, code = EQ;
25536 : break;
25537 8 : case 'e':
25538 8 : if (con[1] == 0)
25539 : mode = CCZmode, code = EQ;
25540 : break;
25541 11 : case 'g':
25542 11 : if (con[1] == 0)
25543 : mode = CCGCmode, code = GT;
25544 5 : else if (con[1] == 'e' && con[2] == 0)
25545 : mode = CCGCmode, code = GE;
25546 : break;
25547 10 : case 'l':
25548 10 : if (con[1] == 0)
25549 : mode = CCGCmode, code = LT;
25550 5 : else if (con[1] == 'e' && con[2] == 0)
25551 : mode = CCGCmode, code = LE;
25552 : break;
25553 4 : case 'o':
25554 4 : if (con[1] == 0)
25555 : mode = CCOmode, code = EQ;
25556 : break;
25557 4 : case 'p':
25558 4 : if (con[1] == 0)
25559 : mode = CCPmode, code = EQ;
25560 : break;
25561 4 : case 's':
25562 4 : if (con[1] == 0)
25563 : mode = CCSmode, code = EQ;
25564 : break;
25565 6 : case 'z':
25566 6 : if (con[1] == 0)
25567 : mode = CCZmode, code = EQ;
25568 : break;
25569 : }
25570 1 : if (code == UNKNOWN)
25571 : {
25572 1 : error_at (loc, "unknown %<asm%> flag output %qs", constraints[i]);
25573 1 : continue;
25574 : }
25575 86 : if (invert)
25576 19 : code = reverse_condition (code);
25577 :
25578 86 : rtx dest = outputs[i];
25579 86 : if (!saw_asm_flag)
25580 : {
25581 : /* This is the first asm flag output. Here we put the flags
25582 : register in as the real output and adjust the condition to
25583 : allow it. */
25584 75 : constraints[i] = "=Bf";
25585 75 : outputs[i] = gen_rtx_REG (CCmode, FLAGS_REG);
25586 75 : saw_asm_flag = true;
25587 : }
25588 : else
25589 : {
25590 : /* We don't need the flags register as output twice. */
25591 11 : constraints[i] = "=X";
25592 11 : outputs[i] = gen_rtx_SCRATCH (SImode);
25593 : }
25594 :
25595 86 : rtx x = gen_rtx_REG (mode, FLAGS_REG);
25596 86 : x = gen_rtx_fmt_ee (code, QImode, x, const0_rtx);
25597 :
25598 86 : machine_mode dest_mode = GET_MODE (dest);
25599 86 : if (!SCALAR_INT_MODE_P (dest_mode))
25600 : {
25601 3 : error_at (loc, "invalid type for %<asm%> flag output");
25602 3 : continue;
25603 : }
25604 :
25605 83 : if (dest_mode == QImode)
25606 73 : emit_insn (gen_rtx_SET (dest, x));
25607 : else
25608 : {
25609 10 : rtx reg = gen_reg_rtx (QImode);
25610 10 : emit_insn (gen_rtx_SET (reg, x));
25611 :
25612 10 : reg = convert_to_mode (dest_mode, reg, 1);
25613 10 : emit_move_insn (dest, reg);
25614 : }
25615 : }
25616 :
25617 109904 : rtx_insn *seq = end_sequence ();
25618 :
25619 109904 : if (saw_asm_flag)
25620 : return seq;
25621 : else
25622 : {
25623 : /* If we had no asm flag outputs, clobber the flags. */
25624 109829 : clobbers.safe_push (gen_rtx_REG (CCmode, FLAGS_REG));
25625 109829 : SET_HARD_REG_BIT (clobbered_regs, FLAGS_REG);
25626 109829 : return NULL;
25627 : }
25628 : }
25629 :
25630 : /* Implements target vector targetm.asm.encode_section_info. */
25631 :
25632 : static void ATTRIBUTE_UNUSED
25633 10000453 : ix86_encode_section_info (tree decl, rtx rtl, int first)
25634 : {
25635 10000453 : default_encode_section_info (decl, rtl, first);
25636 :
25637 10000453 : if (ix86_in_large_data_p (decl))
25638 32 : SYMBOL_REF_FLAGS (XEXP (rtl, 0)) |= SYMBOL_FLAG_FAR_ADDR;
25639 10000453 : }
25640 :
25641 : /* Worker function for REVERSE_CONDITION. */
25642 :
25643 : enum rtx_code
25644 32028255 : ix86_reverse_condition (enum rtx_code code, machine_mode mode)
25645 : {
25646 32028255 : return (mode == CCFPmode
25647 32028255 : ? reverse_condition_maybe_unordered (code)
25648 27599963 : : reverse_condition (code));
25649 : }
25650 :
25651 : /* Output code to perform an x87 FP register move, from OPERANDS[1]
25652 : to OPERANDS[0]. */
25653 :
25654 : const char *
25655 653003 : output_387_reg_move (rtx_insn *insn, rtx *operands)
25656 : {
25657 653003 : if (REG_P (operands[0]))
25658 : {
25659 547000 : if (REG_P (operands[1])
25660 547000 : && find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
25661 : {
25662 292486 : if (REGNO (operands[0]) == FIRST_STACK_REG)
25663 274567 : return output_387_ffreep (operands, 0);
25664 : return "fstp\t%y0";
25665 : }
25666 254514 : if (STACK_TOP_P (operands[0]))
25667 254514 : return "fld%Z1\t%y1";
25668 : return "fst\t%y0";
25669 : }
25670 106003 : else if (MEM_P (operands[0]))
25671 : {
25672 106003 : gcc_assert (REG_P (operands[1]));
25673 106003 : if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
25674 : return "fstp%Z0\t%y0";
25675 : else
25676 : {
25677 : /* There is no non-popping store to memory for XFmode.
25678 : So if we need one, follow the store with a load. */
25679 6258 : if (GET_MODE (operands[0]) == XFmode)
25680 : return "fstp%Z0\t%y0\n\tfld%Z0\t%y0";
25681 : else
25682 1921 : return "fst%Z0\t%y0";
25683 : }
25684 : }
25685 : else
25686 0 : gcc_unreachable();
25687 : }
25688 : #ifdef TARGET_SOLARIS
25689 : /* Solaris implementation of TARGET_ASM_NAMED_SECTION. */
25690 :
25691 : static void
25692 : i386_solaris_elf_named_section (const char *name, unsigned int flags,
25693 : tree decl)
25694 : {
25695 : /* With Binutils 2.15, the "@unwind" marker must be specified on
25696 : every occurrence of the ".eh_frame" section, not just the first
25697 : one. */
25698 : if (TARGET_64BIT
25699 : && strcmp (name, ".eh_frame") == 0)
25700 : {
25701 : fprintf (asm_out_file, "\t.section\t%s,\"%s\",@unwind\n", name,
25702 : flags & SECTION_WRITE ? "aw" : "a");
25703 : return;
25704 : }
25705 :
25706 : #if HAVE_SOLARIS_AS
25707 : if (HAVE_COMDAT_GROUP && flags & SECTION_LINKONCE)
25708 : {
25709 : solaris_elf_asm_comdat_section (name, flags, decl);
25710 : return;
25711 : }
25712 :
25713 : /* Solaris/x86 as uses the same syntax for the SHF_EXCLUDE flags as the
25714 : SPARC assembler. One cannot mix single-letter flags and #exclude, so
25715 : only emit the latter here. */
25716 : if (flags & SECTION_EXCLUDE)
25717 : {
25718 : fprintf (asm_out_file, "\t.section\t%s,#exclude\n", name);
25719 : return;
25720 : }
25721 : #endif
25722 :
25723 : default_elf_asm_named_section (name, flags, decl);
25724 : }
25725 : #endif /* TARGET_SOLARIS */
25726 :
25727 : /* Return the mangling of TYPE if it is an extended fundamental type. */
25728 :
25729 : static const char *
25730 1084487097 : ix86_mangle_type (const_tree type)
25731 : {
25732 1084487097 : type = TYPE_MAIN_VARIANT (type);
25733 :
25734 1084487097 : if (TREE_CODE (type) != VOID_TYPE && TREE_CODE (type) != BOOLEAN_TYPE
25735 : && TREE_CODE (type) != INTEGER_TYPE && TREE_CODE (type) != REAL_TYPE)
25736 : return NULL;
25737 :
25738 588390499 : if (type == float128_type_node || type == float64x_type_node)
25739 : return NULL;
25740 :
25741 587603200 : switch (TYPE_MODE (type))
25742 : {
25743 : case E_BFmode:
25744 : return "DF16b";
25745 459409 : case E_HFmode:
25746 : /* _Float16 is "DF16_".
25747 : Align with clang's decision in https://reviews.llvm.org/D33719. */
25748 459409 : return "DF16_";
25749 681659 : case E_TFmode:
25750 : /* __float128 is "g". */
25751 681659 : return "g";
25752 8081689 : case E_XFmode:
25753 : /* "long double" or __float80 is "e". */
25754 8081689 : return "e";
25755 : default:
25756 : return NULL;
25757 : }
25758 : }
25759 :
25760 : /* Create C++ tinfo symbols for only conditionally available fundamental
25761 : types. */
25762 :
25763 : static void
25764 5 : ix86_emit_support_tinfos (emit_support_tinfos_callback callback)
25765 : {
25766 5 : extern tree ix86_float16_type_node;
25767 5 : extern tree ix86_bf16_type_node;
25768 :
25769 5 : if (!TARGET_SSE2)
25770 : {
25771 0 : if (!float16_type_node)
25772 0 : float16_type_node = ix86_float16_type_node;
25773 0 : if (!bfloat16_type_node)
25774 0 : bfloat16_type_node = ix86_bf16_type_node;
25775 0 : callback (float16_type_node);
25776 0 : callback (bfloat16_type_node);
25777 0 : float16_type_node = NULL_TREE;
25778 0 : bfloat16_type_node = NULL_TREE;
25779 : }
25780 5 : }
25781 :
25782 : static GTY(()) tree ix86_tls_stack_chk_guard_decl;
25783 :
25784 : static tree
25785 338 : ix86_stack_protect_guard (void)
25786 : {
25787 338 : if (TARGET_SSP_TLS_GUARD)
25788 : {
25789 263 : tree type_node = lang_hooks.types.type_for_mode (ptr_mode, 1);
25790 263 : int qual = ENCODE_QUAL_ADDR_SPACE (ix86_stack_protector_guard_reg);
25791 263 : tree type = build_qualified_type (type_node, qual);
25792 263 : tree t;
25793 :
25794 263 : if (OPTION_SET_P (ix86_stack_protector_guard_symbol_str))
25795 : {
25796 1 : t = ix86_tls_stack_chk_guard_decl;
25797 :
25798 1 : if (t == NULL)
25799 : {
25800 1 : rtx x;
25801 :
25802 1 : t = build_decl
25803 1 : (UNKNOWN_LOCATION, VAR_DECL,
25804 : get_identifier (ix86_stack_protector_guard_symbol_str),
25805 : type);
25806 1 : TREE_STATIC (t) = 1;
25807 1 : TREE_PUBLIC (t) = 1;
25808 1 : DECL_EXTERNAL (t) = 1;
25809 1 : TREE_USED (t) = 1;
25810 1 : TREE_THIS_VOLATILE (t) = 1;
25811 1 : DECL_ARTIFICIAL (t) = 1;
25812 1 : DECL_IGNORED_P (t) = 1;
25813 :
25814 : /* Do not share RTL as the declaration is visible outside of
25815 : current function. */
25816 1 : x = DECL_RTL (t);
25817 1 : RTX_FLAG (x, used) = 1;
25818 :
25819 1 : ix86_tls_stack_chk_guard_decl = t;
25820 : }
25821 : }
25822 : else
25823 : {
25824 262 : tree asptrtype = build_pointer_type (type);
25825 :
25826 262 : t = build_int_cst (asptrtype, ix86_stack_protector_guard_offset);
25827 262 : t = build2 (MEM_REF, asptrtype, t,
25828 : build_int_cst (asptrtype, 0));
25829 262 : TREE_THIS_VOLATILE (t) = 1;
25830 : }
25831 :
25832 263 : return t;
25833 : }
25834 :
25835 75 : return default_stack_protect_guard ();
25836 : }
25837 :
25838 : /* Implement TARGET_STACK_PROTECT_GUARD_SYMBOL_P. */
25839 :
25840 : static bool
25841 207986 : ix86_stack_protect_guard_symbol_p (void)
25842 : {
25843 207986 : return TARGET_SSP_GLOBAL_GUARD;
25844 : }
25845 :
25846 : static bool
25847 930 : ix86_stack_protect_runtime_enabled_p (void)
25848 : {
25849 : /* Naked functions should not enable stack protector. */
25850 930 : return !ix86_function_naked (current_function_decl);
25851 : }
25852 :
25853 : /* For 32-bit code we can save PIC register setup by using
25854 : __stack_chk_fail_local hidden function instead of calling
25855 : __stack_chk_fail directly. 64-bit code doesn't need to setup any PIC
25856 : register, so it is better to call __stack_chk_fail directly. */
25857 :
25858 : static tree ATTRIBUTE_UNUSED
25859 331 : ix86_stack_protect_fail (void)
25860 : {
25861 331 : return TARGET_64BIT
25862 331 : ? default_external_stack_protect_fail ()
25863 1 : : default_hidden_stack_protect_fail ();
25864 : }
25865 :
25866 : /* Select a format to encode pointers in exception handling data. CODE
25867 : is 0 for data, 1 for code labels, 2 for function pointers. GLOBAL is
25868 : true if the symbol may be affected by dynamic relocations.
25869 :
25870 : ??? All x86 object file formats are capable of representing this.
25871 : After all, the relocation needed is the same as for the call insn.
25872 : Whether or not a particular assembler allows us to enter such, I
25873 : guess we'll have to see. */
25874 :
25875 : int
25876 800244 : asm_preferred_eh_data_format (int code, int global)
25877 : {
25878 : /* PE-COFF is effectively always -fPIC because of the .reloc section. */
25879 800244 : if (flag_pic || TARGET_PECOFF || !ix86_direct_extern_access)
25880 : {
25881 38421 : int type = DW_EH_PE_sdata8;
25882 38421 : if (ptr_mode == SImode
25883 24442 : || ix86_cmodel == CM_SMALL_PIC
25884 38507 : || (ix86_cmodel == CM_MEDIUM_PIC && (global || code)))
25885 : type = DW_EH_PE_sdata4;
25886 53858 : return (global ? DW_EH_PE_indirect : 0) | DW_EH_PE_pcrel | type;
25887 : }
25888 :
25889 761823 : if (ix86_cmodel == CM_SMALL
25890 18670 : || (ix86_cmodel == CM_MEDIUM && code))
25891 743166 : return DW_EH_PE_udata4;
25892 :
25893 : return DW_EH_PE_absptr;
25894 : }
25895 :
25896 : /* Cost of constructing or destructing a vector in VECMODE from/to elements
25897 : of ELMODE. */
25898 : static int
25899 783658 : ix86_vector_cd_cost (machine_mode vecmode, machine_mode elmode)
25900 : {
25901 1567316 : if (GET_MODE_BITSIZE (vecmode) < 128)
25902 543084 : return ((GET_MODE_BITSIZE (vecmode) / GET_MODE_BITSIZE (elmode) - 1)
25903 271542 : * ix86_cost->sse_op);
25904 :
25905 512116 : int n = GET_MODE_BITSIZE (vecmode) / 128;
25906 512116 : int cost = 0;
25907 : /* Element inserts/extracts into/from N SSE vectors, the possible
25908 : GPR <-> XMM moves have to be accounted for elsewhere. */
25909 1024232 : if (GET_MODE_BITSIZE (elmode) < 128)
25910 1022986 : cost += n * (128 / GET_MODE_BITSIZE (elmode) - 1) * ix86_cost->sse_op;
25911 512116 : if (GET_MODE_BITSIZE (vecmode) >= 256
25912 521858 : && GET_MODE_BITSIZE (elmode) < 256)
25913 : /* N/2 vinserti128/vextracti128 for SSE <-> AVX256. */
25914 9742 : cost += n * ix86_vec_cost (V32QImode, ix86_cost->sse_op) / 2;
25915 1024232 : if (GET_MODE_BITSIZE (vecmode) == 512)
25916 : /* One vinserti64x4/vextracti64x4 for AVX256 <-> AVX512. */
25917 1968 : cost += ix86_vec_cost (vecmode, ix86_cost->sse_op);
25918 : return cost;
25919 : }
25920 :
25921 : /* Worker for ix86_builtin_vectorization_cost and the fallback calls
25922 : from ix86_vector_costs::add_stmt_cost. */
25923 : static int
25924 15608553 : ix86_default_vector_cost (enum vect_cost_for_stmt type_of_cost,
25925 : machine_mode mode)
25926 : {
25927 15608553 : bool fp = FLOAT_MODE_P (mode);
25928 15608553 : int index;
25929 15608553 : switch (type_of_cost)
25930 : {
25931 1858382 : case scalar_stmt:
25932 1858382 : return fp ? ix86_cost->addss : COSTS_N_INSNS (1);
25933 :
25934 1859087 : case scalar_load:
25935 : /* load/store costs are relative to register move which is 2. Recompute
25936 : it to COSTS_N_INSNS so everything have same base. */
25937 3718174 : return COSTS_N_INSNS (fp ? ix86_cost->sse_load[0]
25938 1859087 : : ix86_cost->int_load [2]) / 2;
25939 :
25940 4007184 : case scalar_store:
25941 8014368 : return COSTS_N_INSNS (fp ? ix86_cost->sse_store[0]
25942 4007184 : : ix86_cost->int_store [2]) / 2;
25943 :
25944 1119843 : case vector_stmt:
25945 2239686 : return ix86_vec_cost (mode,
25946 2239686 : fp ? ix86_cost->addss : ix86_cost->sse_op);
25947 :
25948 2000676 : case vector_load:
25949 2000676 : index = sse_store_index (mode);
25950 : /* See PR82713 - we may end up being called on non-vector type. */
25951 2000676 : if (index < 0)
25952 102116 : index = 2;
25953 2000676 : return COSTS_N_INSNS (ix86_cost->sse_load[index]) / 2;
25954 :
25955 997284 : case vector_store:
25956 997284 : index = sse_store_index (mode);
25957 : /* See PR82713 - we may end up being called on non-vector type. */
25958 997284 : if (index < 0)
25959 92657 : index = 2;
25960 997284 : return COSTS_N_INSNS (ix86_cost->sse_store[index]) / 2;
25961 :
25962 874628 : case vec_to_scalar:
25963 874628 : case scalar_to_vec:
25964 874628 : return ix86_vec_cost (mode, ix86_cost->sse_op);
25965 :
25966 : /* We should have separate costs for unaligned loads and gather/scatter.
25967 : Do that incrementally. */
25968 526886 : case unaligned_load:
25969 526886 : index = sse_store_index (mode);
25970 : /* See PR82713 - we may end up being called on non-vector type. */
25971 526886 : if (index < 0)
25972 3337 : index = 2;
25973 526886 : return COSTS_N_INSNS (ix86_cost->sse_unaligned_load[index]) / 2;
25974 :
25975 850787 : case unaligned_store:
25976 850787 : index = sse_store_index (mode);
25977 : /* See PR82713 - we may end up being called on non-vector type. */
25978 850787 : if (index < 0)
25979 17250 : index = 2;
25980 850787 : return COSTS_N_INSNS (ix86_cost->sse_unaligned_store[index]) / 2;
25981 :
25982 0 : case vector_gather_load:
25983 0 : return ix86_vec_cost (mode,
25984 0 : COSTS_N_INSNS
25985 : (ix86_cost->gather_static
25986 : + ix86_cost->gather_per_elt
25987 0 : * GET_MODE_NUNITS (mode)) / 2);
25988 :
25989 0 : case vector_scatter_store:
25990 0 : return ix86_vec_cost (mode,
25991 0 : COSTS_N_INSNS
25992 : (ix86_cost->scatter_static
25993 : + ix86_cost->scatter_per_elt
25994 0 : * GET_MODE_NUNITS (mode)) / 2);
25995 :
25996 364109 : case cond_branch_taken:
25997 364109 : return ix86_cost->cond_taken_branch_cost;
25998 :
25999 8614 : case cond_branch_not_taken:
26000 8614 : return ix86_cost->cond_not_taken_branch_cost;
26001 :
26002 299585 : case vec_perm:
26003 299585 : return ix86_vec_cost (mode, ix86_cost->sse_op);
26004 :
26005 90305 : case vec_promote_demote:
26006 90305 : if (fp)
26007 11739 : return vec_fp_conversion_cost (ix86_tune_cost, mode);
26008 78566 : return ix86_vec_cost (mode, ix86_cost->sse_op);
26009 :
26010 751183 : case vec_construct:
26011 751183 : case vec_deconstruct:
26012 1502366 : return ix86_vector_cd_cost (mode, GET_MODE_INNER (mode));
26013 :
26014 0 : default:
26015 0 : gcc_unreachable ();
26016 : }
26017 : }
26018 :
26019 : /* Implement targetm.vectorize.builtin_vectorization_cost. */
26020 : static int
26021 9314298 : ix86_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
26022 : tree vectype, int)
26023 : {
26024 9314298 : machine_mode mode = TImode;
26025 9314298 : if (vectype != NULL)
26026 5846576 : mode = TYPE_MODE (vectype);
26027 9314298 : return ix86_default_vector_cost (type_of_cost, mode);
26028 : }
26029 :
26030 :
26031 : /* This function returns the calling abi specific va_list type node.
26032 : It returns the FNDECL specific va_list type. */
26033 :
26034 : static tree
26035 47729 : ix86_fn_abi_va_list (tree fndecl)
26036 : {
26037 47729 : if (!TARGET_64BIT)
26038 726 : return va_list_type_node;
26039 47003 : gcc_assert (fndecl != NULL_TREE);
26040 :
26041 47003 : if (ix86_function_abi ((const_tree) fndecl) == MS_ABI)
26042 12880 : return ms_va_list_type_node;
26043 : else
26044 34123 : return sysv_va_list_type_node;
26045 : }
26046 :
26047 : /* Returns the canonical va_list type specified by TYPE. If there
26048 : is no valid TYPE provided, it return NULL_TREE. */
26049 :
26050 : static tree
26051 247454 : ix86_canonical_va_list_type (tree type)
26052 : {
26053 247454 : if (TARGET_64BIT)
26054 : {
26055 246952 : if (lookup_attribute ("ms_abi va_list", TYPE_ATTRIBUTES (type)))
26056 5948 : return ms_va_list_type_node;
26057 :
26058 241004 : if ((TREE_CODE (type) == ARRAY_TYPE
26059 50138 : && integer_zerop (array_type_nelts_minus_one (type)))
26060 241004 : || POINTER_TYPE_P (type))
26061 : {
26062 188964 : tree elem_type = TREE_TYPE (type);
26063 188964 : if (TREE_CODE (elem_type) == RECORD_TYPE
26064 340996 : && lookup_attribute ("sysv_abi va_list",
26065 152032 : TYPE_ATTRIBUTES (elem_type)))
26066 152032 : return sysv_va_list_type_node;
26067 : }
26068 :
26069 88972 : return NULL_TREE;
26070 : }
26071 :
26072 502 : return std_canonical_va_list_type (type);
26073 : }
26074 :
26075 : /* Iterate through the target-specific builtin types for va_list.
26076 : IDX denotes the iterator, *PTREE is set to the result type of
26077 : the va_list builtin, and *PNAME to its internal type.
26078 : Returns zero if there is no element for this index, otherwise
26079 : IDX should be increased upon the next call.
26080 : Note, do not iterate a base builtin's name like __builtin_va_list.
26081 : Used from c_common_nodes_and_builtins. */
26082 :
26083 : static int
26084 630576 : ix86_enum_va_list (int idx, const char **pname, tree *ptree)
26085 : {
26086 630576 : if (TARGET_64BIT)
26087 : {
26088 625200 : switch (idx)
26089 : {
26090 : default:
26091 : break;
26092 :
26093 208400 : case 0:
26094 208400 : *ptree = ms_va_list_type_node;
26095 208400 : *pname = "__builtin_ms_va_list";
26096 208400 : return 1;
26097 :
26098 208400 : case 1:
26099 208400 : *ptree = sysv_va_list_type_node;
26100 208400 : *pname = "__builtin_sysv_va_list";
26101 208400 : return 1;
26102 : }
26103 : }
26104 :
26105 : return 0;
26106 : }
26107 :
26108 : #undef TARGET_SCHED_DISPATCH
26109 : #define TARGET_SCHED_DISPATCH ix86_bd_has_dispatch
26110 : #undef TARGET_SCHED_DISPATCH_DO
26111 : #define TARGET_SCHED_DISPATCH_DO ix86_bd_do_dispatch
26112 : #undef TARGET_SCHED_REASSOCIATION_WIDTH
26113 : #define TARGET_SCHED_REASSOCIATION_WIDTH ix86_reassociation_width
26114 : #undef TARGET_SCHED_REORDER
26115 : #define TARGET_SCHED_REORDER ix86_atom_sched_reorder
26116 : #undef TARGET_SCHED_ADJUST_PRIORITY
26117 : #define TARGET_SCHED_ADJUST_PRIORITY ix86_adjust_priority
26118 : #undef TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK
26119 : #define TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK \
26120 : ix86_dependencies_evaluation_hook
26121 :
26122 :
26123 : /* Implementation of reassociation_width target hook used by
26124 : reassoc phase to identify parallelism level in reassociated
26125 : tree. Statements tree_code is passed in OPC. Arguments type
26126 : is passed in MODE. */
26127 :
26128 : static int
26129 28951 : ix86_reassociation_width (tree_code op, machine_mode mode)
26130 : {
26131 28951 : int width = 1;
26132 : /* Vector part. */
26133 28951 : if (VECTOR_MODE_P (mode))
26134 : {
26135 8745 : int div = 1;
26136 8745 : if (INTEGRAL_MODE_P (mode))
26137 2824 : width = ix86_cost->reassoc_vec_int;
26138 5921 : else if (FLOAT_MODE_P (mode))
26139 5921 : width = ix86_cost->reassoc_vec_fp;
26140 :
26141 8745 : if (width == 1)
26142 : return 1;
26143 :
26144 : /* Znver1-4 Integer vector instructions execute in FP unit
26145 : and can execute 3 additions and one multiplication per cycle. */
26146 8740 : if ((ix86_tune == PROCESSOR_ZNVER1 || ix86_tune == PROCESSOR_ZNVER2
26147 8740 : || ix86_tune == PROCESSOR_ZNVER3 || ix86_tune == PROCESSOR_ZNVER4
26148 8740 : || ix86_tune == PROCESSOR_C86_4G_M4
26149 8740 : || ix86_tune == PROCESSOR_C86_4G_M6
26150 8740 : || ix86_tune == PROCESSOR_C86_4G_M7
26151 8738 : || ix86_tune == PROCESSOR_C86_4G_M8)
26152 2 : && INTEGRAL_MODE_P (mode) && op != PLUS_EXPR && op != MINUS_EXPR)
26153 : return 1;
26154 : /* Znver5 can do 2 integer multiplications per cycle with latency
26155 : of 3. */
26156 8740 : if ((ix86_tune == PROCESSOR_ZNVER5 || ix86_tune == PROCESSOR_ZNVER6)
26157 0 : && INTEGRAL_MODE_P (mode) && op != PLUS_EXPR && op != MINUS_EXPR)
26158 8740 : width = 6;
26159 :
26160 : /* Account for targets that splits wide vectors into multiple parts. */
26161 8742 : if (TARGET_AVX512_SPLIT_REGS && GET_MODE_BITSIZE (mode) > 256)
26162 0 : div = GET_MODE_BITSIZE (mode) / 256;
26163 8740 : else if (TARGET_AVX256_SPLIT_REGS && GET_MODE_BITSIZE (mode) > 128)
26164 0 : div = GET_MODE_BITSIZE (mode) / 128;
26165 8740 : else if (TARGET_SSE_SPLIT_REGS && GET_MODE_BITSIZE (mode) > 64)
26166 0 : div = GET_MODE_BITSIZE (mode) / 64;
26167 8740 : width = (width + div - 1) / div;
26168 8740 : }
26169 : /* Scalar part. */
26170 : else if (INTEGRAL_MODE_P (mode))
26171 14602 : width = ix86_cost->reassoc_int;
26172 : else if (FLOAT_MODE_P (mode))
26173 5604 : width = ix86_cost->reassoc_fp;
26174 :
26175 : /* Avoid using too many registers in 32bit mode. */
26176 28946 : if (!TARGET_64BIT && width > 2)
26177 28951 : width = 2;
26178 : return width;
26179 : }
26180 :
26181 : /* ??? No autovectorization into MMX or 3DNOW until we can reliably
26182 : place emms and femms instructions. */
26183 :
26184 : static machine_mode
26185 5334994 : ix86_preferred_simd_mode (scalar_mode mode)
26186 : {
26187 5334994 : if (!TARGET_SSE)
26188 873 : return word_mode;
26189 :
26190 5334121 : switch (mode)
26191 : {
26192 432748 : case E_QImode:
26193 432748 : if (TARGET_AVX512BW && !TARGET_PREFER_AVX256)
26194 : return V64QImode;
26195 420202 : else if (TARGET_AVX && !TARGET_PREFER_AVX128)
26196 : return V32QImode;
26197 : else
26198 400527 : return V16QImode;
26199 :
26200 210029 : case E_HImode:
26201 210029 : if (TARGET_AVX512BW && !TARGET_PREFER_AVX256)
26202 : return V32HImode;
26203 199505 : else if (TARGET_AVX && !TARGET_PREFER_AVX128)
26204 : return V16HImode;
26205 : else
26206 183501 : return V8HImode;
26207 :
26208 1584712 : case E_SImode:
26209 1584712 : if (TARGET_AVX512F && !TARGET_PREFER_AVX256)
26210 : return V16SImode;
26211 1517065 : else if (TARGET_AVX && !TARGET_PREFER_AVX128)
26212 : return V8SImode;
26213 : else
26214 1363976 : return V4SImode;
26215 :
26216 1930400 : case E_DImode:
26217 1930400 : if (TARGET_AVX512F && !TARGET_PREFER_AVX256)
26218 : return V8DImode;
26219 1526247 : else if (TARGET_AVX && !TARGET_PREFER_AVX128)
26220 : return V4DImode;
26221 : else
26222 1464194 : return V2DImode;
26223 :
26224 144909 : case E_HFmode:
26225 144909 : if (TARGET_AVX512FP16)
26226 : {
26227 141978 : if (TARGET_AVX512VL)
26228 : {
26229 69102 : if (TARGET_PREFER_AVX128)
26230 : return V8HFmode;
26231 68880 : else if (TARGET_PREFER_AVX256)
26232 : return V16HFmode;
26233 : }
26234 139646 : return V32HFmode;
26235 : }
26236 2931 : return word_mode;
26237 :
26238 62952 : case E_BFmode:
26239 62952 : if (TARGET_AVX512F && !TARGET_PREFER_AVX256)
26240 : return V32BFmode;
26241 26492 : else if (TARGET_AVX && !TARGET_PREFER_AVX128)
26242 : return V16BFmode;
26243 : else
26244 13489 : return V8BFmode;
26245 :
26246 629769 : case E_SFmode:
26247 629769 : if (TARGET_AVX512F && !TARGET_PREFER_AVX256)
26248 : return V16SFmode;
26249 429091 : else if (TARGET_AVX && !TARGET_PREFER_AVX128)
26250 : return V8SFmode;
26251 : else
26252 362483 : return V4SFmode;
26253 :
26254 299696 : case E_DFmode:
26255 299696 : if (TARGET_AVX512F && !TARGET_PREFER_AVX256)
26256 : return V8DFmode;
26257 178442 : else if (TARGET_AVX && !TARGET_PREFER_AVX128)
26258 : return V4DFmode;
26259 124921 : else if (TARGET_SSE2)
26260 : return V2DFmode;
26261 : /* FALLTHRU */
26262 :
26263 38962 : default:
26264 38962 : return word_mode;
26265 : }
26266 : }
26267 :
26268 : /* If AVX is enabled then try vectorizing with both 256bit and 128bit
26269 : vectors. If AVX512F is enabled then try vectorizing with 512bit,
26270 : 256bit and 128bit vectors. */
26271 :
26272 : static unsigned int
26273 2215508 : ix86_autovectorize_vector_modes (vector_modes *modes, bool all)
26274 : {
26275 2215508 : if (TARGET_AVX512F && !TARGET_PREFER_AVX256)
26276 : {
26277 74942 : modes->safe_push (V64QImode);
26278 74942 : modes->safe_push (V32QImode);
26279 74942 : modes->safe_push (V16QImode);
26280 : }
26281 2140566 : else if (TARGET_AVX512F && all)
26282 : {
26283 574 : modes->safe_push (V32QImode);
26284 574 : modes->safe_push (V16QImode);
26285 574 : modes->safe_push (V64QImode);
26286 : }
26287 2139992 : else if (TARGET_AVX && !TARGET_PREFER_AVX128)
26288 : {
26289 28734 : modes->safe_push (V32QImode);
26290 28734 : modes->safe_push (V16QImode);
26291 : }
26292 2111258 : else if (TARGET_AVX && all)
26293 : {
26294 24 : modes->safe_push (V16QImode);
26295 24 : modes->safe_push (V32QImode);
26296 : }
26297 2111234 : else if (TARGET_SSE2)
26298 2108946 : modes->safe_push (V16QImode);
26299 :
26300 2215508 : if (TARGET_MMX_WITH_SSE)
26301 1818722 : modes->safe_push (V8QImode);
26302 :
26303 2215508 : if (TARGET_SSE2)
26304 2213220 : modes->safe_push (V4QImode);
26305 :
26306 2215508 : return ix86_vect_compare_costs ? VECT_COMPARE_COSTS : 0;
26307 : }
26308 :
26309 : /* Implementation of targetm.vectorize.get_mask_mode. */
26310 :
26311 : static opt_machine_mode
26312 3593869 : ix86_get_mask_mode (machine_mode data_mode)
26313 : {
26314 3593869 : unsigned vector_size = GET_MODE_SIZE (data_mode);
26315 3593869 : unsigned nunits = GET_MODE_NUNITS (data_mode);
26316 3593869 : unsigned elem_size = vector_size / nunits;
26317 :
26318 : /* Scalar mask case. */
26319 484748 : if ((TARGET_AVX512F && vector_size == 64)
26320 3477280 : || (TARGET_AVX512VL && (vector_size == 32 || vector_size == 16))
26321 : /* AVX512FP16 only supports vector comparison
26322 : to kmask for _Float16. */
26323 3301229 : || (TARGET_AVX512VL && TARGET_AVX512FP16
26324 18432 : && GET_MODE_INNER (data_mode) == E_HFmode)
26325 6899858 : || (TARGET_AVX10_2 && GET_MODE_INNER (data_mode) == E_BFmode))
26326 : {
26327 295664 : if (elem_size == 4
26328 295664 : || elem_size == 8
26329 136171 : || (TARGET_AVX512BW && (elem_size == 1 || elem_size == 2)))
26330 264418 : return smallest_int_mode_for_size (nunits).require ();
26331 : }
26332 :
26333 3329451 : scalar_int_mode elem_mode
26334 3329451 : = smallest_int_mode_for_size (elem_size * BITS_PER_UNIT).require ();
26335 :
26336 3329451 : gcc_assert (elem_size * nunits == vector_size);
26337 :
26338 3329451 : return mode_for_vector (elem_mode, nunits);
26339 : }
26340 :
26341 :
26342 :
26343 : /* Return class of registers which could be used for pseudo of MODE
26344 : and of class RCLASS for spilling instead of memory. Return NO_REGS
26345 : if it is not possible or non-profitable. */
26346 :
26347 : /* Disabled due to PRs 70902, 71453, 71555, 71596 and 71657. */
26348 :
26349 : static reg_class_t
26350 6298877477 : ix86_spill_class (reg_class_t rclass, machine_mode mode)
26351 : {
26352 6298877477 : if (0 && TARGET_GENERAL_REGS_SSE_SPILL
26353 : && TARGET_SSE2
26354 : && TARGET_INTER_UNIT_MOVES_TO_VEC
26355 : && TARGET_INTER_UNIT_MOVES_FROM_VEC
26356 : && (mode == SImode || (TARGET_64BIT && mode == DImode))
26357 : && INTEGER_CLASS_P (rclass))
26358 : return ALL_SSE_REGS;
26359 6298877477 : return NO_REGS;
26360 : }
26361 :
26362 : /* Implement TARGET_MAX_NOCE_IFCVT_SEQ_COST. Like the default implementation,
26363 : but returns a lower bound. */
26364 :
26365 : static unsigned int
26366 1873971 : ix86_max_noce_ifcvt_seq_cost (edge e)
26367 : {
26368 1873971 : bool predictable_p = predictable_edge_p (e);
26369 1873971 : if (predictable_p)
26370 : {
26371 147766 : if (OPTION_SET_P (param_max_rtl_if_conversion_predictable_cost))
26372 8 : return param_max_rtl_if_conversion_predictable_cost;
26373 : }
26374 : else
26375 : {
26376 1726205 : if (OPTION_SET_P (param_max_rtl_if_conversion_unpredictable_cost))
26377 73 : return param_max_rtl_if_conversion_unpredictable_cost;
26378 : }
26379 :
26380 : /* For modern machines with deeper pipeline, the penalty for branch
26381 : misprediction could be higher than before to reset the pipeline
26382 : slots. Add parameter br_mispredict_scale as a factor to describe
26383 : the impact of resetting the pipeline. */
26384 :
26385 1873890 : return BRANCH_COST (true, predictable_p)
26386 1873890 : * ix86_tune_cost->br_mispredict_scale;
26387 : }
26388 :
26389 : /* Return true if SEQ is a good candidate as a replacement for the
26390 : if-convertible sequence described in IF_INFO. */
26391 :
26392 : static bool
26393 178332 : ix86_noce_conversion_profitable_p (rtx_insn *seq, struct noce_if_info *if_info)
26394 : {
26395 178332 : if (TARGET_ONE_IF_CONV_INSN && if_info->speed_p)
26396 : {
26397 : int cmov_cnt = 0;
26398 : /* Punt if SEQ contains more than one CMOV or FCMOV instruction.
26399 : Maybe we should allow even more conditional moves as long as they
26400 : are used far enough not to stall the CPU, or also consider
26401 : IF_INFO->TEST_BB succ edge probabilities. */
26402 336 : for (rtx_insn *insn = seq; insn; insn = NEXT_INSN (insn))
26403 : {
26404 280 : rtx set = single_set (insn);
26405 280 : if (!set)
26406 0 : continue;
26407 280 : if (GET_CODE (SET_SRC (set)) != IF_THEN_ELSE)
26408 238 : continue;
26409 42 : rtx src = SET_SRC (set);
26410 42 : machine_mode mode = GET_MODE (src);
26411 42 : if (GET_MODE_CLASS (mode) != MODE_INT
26412 0 : && GET_MODE_CLASS (mode) != MODE_FLOAT)
26413 0 : continue;
26414 42 : if ((!REG_P (XEXP (src, 1)) && !MEM_P (XEXP (src, 1)))
26415 41 : || (!REG_P (XEXP (src, 2)) && !MEM_P (XEXP (src, 2))))
26416 1 : continue;
26417 : /* insn is CMOV or FCMOV. */
26418 41 : if (++cmov_cnt > 1)
26419 : return false;
26420 : }
26421 : }
26422 :
26423 : /* W/o TARGET_SSE4_1, it takes 3 instructions (pand, pandn and por)
26424 : for movdfcc/movsfcc, and could possibly fail cost comparison.
26425 : Increase branch cost will hurt performance for other modes, so
26426 : specially add some preference for floating point ifcvt. */
26427 178324 : if (!TARGET_SSE4_1 && if_info->x
26428 141540 : && GET_MODE_CLASS (GET_MODE (if_info->x)) == MODE_FLOAT
26429 17976 : && if_info->speed_p)
26430 : {
26431 10938 : unsigned cost = seq_cost (seq, true);
26432 :
26433 10938 : if (cost <= if_info->original_cost)
26434 : return true;
26435 :
26436 9734 : return cost <= (if_info->max_seq_cost + COSTS_N_INSNS (2));
26437 : }
26438 :
26439 167386 : return default_noce_conversion_profitable_p (seq, if_info);
26440 : }
26441 :
26442 : /* x86-specific vector costs. */
26443 : class ix86_vector_costs : public vector_costs
26444 : {
26445 : public:
26446 : ix86_vector_costs (vec_info *, bool);
26447 :
26448 : unsigned int add_stmt_cost (int count, vect_cost_for_stmt kind,
26449 : stmt_vec_info stmt_info, slp_tree node,
26450 : tree vectype, int misalign,
26451 : vect_cost_model_location where) override;
26452 : void finish_cost (const vector_costs *) override;
26453 : bool better_main_loop_than_p (const vector_costs *) const override;
26454 : bool better_epilogue_loop_than_p (const vector_costs *other,
26455 : loop_vec_info main_loop) const override;
26456 :
26457 : private:
26458 :
26459 : bool better_fold_left_reduc_than_p (const vector_costs *) const;
26460 : /* Estimate register pressure of the vectorized code. */
26461 : void ix86_vect_estimate_reg_pressure ();
26462 : /* Number of GENERAL_REGS/SSE_REGS used in the vectorizer, it's used for
26463 : estimation of register pressure.
26464 : ??? Currently it's only used by vec_construct/scalar_to_vec
26465 : where we know it's not loaded from memory. */
26466 : unsigned m_num_gpr_needed[3];
26467 : unsigned m_num_sse_needed[3];
26468 : /* Number of 256-bit vector permutation. */
26469 : unsigned m_num_avx256_vec_perm[3];
26470 : /* Number of 512-bit vector permutation. */
26471 : unsigned m_num_avx512_vec_perm[3];
26472 : /* Number of reductions for FMA/DOT_PROD_EXPR/SAD_EXPR */
26473 : unsigned m_num_reduc[X86_REDUC_LAST];
26474 : /* Don't do unroll if m_prefer_unroll is false, default is true. */
26475 : bool m_prefer_unroll;
26476 : /* Scalar lanes in fold-left reductions. */
26477 : unsigned int m_num_fold_left_reduc_lanes;
26478 : };
26479 :
26480 2193092 : ix86_vector_costs::ix86_vector_costs (vec_info* vinfo, bool costing_for_scalar)
26481 : : vector_costs (vinfo, costing_for_scalar),
26482 2193092 : m_num_gpr_needed (),
26483 2193092 : m_num_sse_needed (),
26484 2193092 : m_num_avx256_vec_perm (),
26485 2193092 : m_num_avx512_vec_perm (),
26486 2193092 : m_num_reduc (),
26487 2193092 : m_prefer_unroll (true),
26488 2193092 : m_num_fold_left_reduc_lanes (0)
26489 2193092 : {}
26490 :
26491 : /* Implement targetm.vectorize.create_costs. */
26492 :
26493 : static vector_costs *
26494 2193092 : ix86_vectorize_create_costs (vec_info *vinfo, bool costing_for_scalar)
26495 : {
26496 2193092 : return new ix86_vector_costs (vinfo, costing_for_scalar);
26497 : }
26498 :
26499 : /* Return true if a vec_perm should be counted as a cross-lane vector
26500 : permutation for a vector with NUNITS elements. */
26501 : static bool
26502 5336 : ix86_count_cross_lane_perm_p (vec_info *vinfo, slp_tree node, unsigned nunits)
26503 : {
26504 : /* TODO: For loop vectorization with no SLP load-permutation
26505 : information, conservatively treat these perms as cross-lane.
26506 : Repeated-index cases such as {0, 0, 0, 0} are emitted as
26507 : separate vec_perm_exprs for each index, so we cannot reliably
26508 : separate false positives from real cross-lane shuffles yet. */
26509 5336 : if (!node
26510 5327 : || !SLP_TREE_LOAD_PERMUTATION (node).exists ()
26511 9616 : || !is_a<bb_vec_info> (vinfo))
26512 : return true;
26513 :
26514 41 : unsigned half = nunits / 2;
26515 41 : bool allsame = true;
26516 41 : unsigned first = SLP_TREE_LOAD_PERMUTATION (node)[0];
26517 41 : bool cross_lane_p = false;
26518 :
26519 217 : for (unsigned i = 0; i != SLP_TREE_LANES (node); i++)
26520 : {
26521 215 : unsigned tmp = SLP_TREE_LOAD_PERMUTATION (node)[i];
26522 : /* allsame is just a broadcast. */
26523 215 : if (tmp != first)
26524 106 : allsame = false;
26525 :
26526 : /* The load permutation can cover multiple vectors, so compare
26527 : source and destination lanes modulo NUNITS. */
26528 215 : tmp = tmp & (nunits - 1);
26529 215 : unsigned index = i & (nunits - 1);
26530 215 : if ((index < half && tmp >= half) || (index >= half && tmp < half))
26531 67 : cross_lane_p = true;
26532 :
26533 215 : if (!allsame && cross_lane_p)
26534 : return true;
26535 : }
26536 :
26537 : return false;
26538 : }
26539 :
26540 : unsigned
26541 7845192 : ix86_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
26542 : stmt_vec_info stmt_info, slp_tree node,
26543 : tree vectype, int,
26544 : vect_cost_model_location where)
26545 : {
26546 7845192 : unsigned retval = 0;
26547 7845192 : bool scalar_p
26548 : = (kind == scalar_stmt || kind == scalar_load || kind == scalar_store);
26549 7845192 : int stmt_cost = - 1;
26550 :
26551 7845192 : bool fp = false;
26552 7845192 : machine_mode mode = scalar_p ? SImode : TImode;
26553 :
26554 7845192 : if (vectype != NULL)
26555 : {
26556 3576507 : fp = FLOAT_TYPE_P (vectype);
26557 3576507 : mode = TYPE_MODE (vectype);
26558 3576507 : if (scalar_p)
26559 283929 : mode = TYPE_MODE (TREE_TYPE (vectype));
26560 : }
26561 : /* When we are costing a scalar stmt use the scalar stmt to get at the
26562 : type of the operation. */
26563 4268685 : else if (scalar_p && stmt_info)
26564 4185081 : if (tree lhs = gimple_get_lhs (stmt_info->stmt))
26565 : {
26566 4003492 : fp = FLOAT_TYPE_P (TREE_TYPE (lhs));
26567 4003492 : mode = TYPE_MODE (TREE_TYPE (lhs));
26568 : }
26569 :
26570 7845192 : if ((kind == vector_stmt || kind == scalar_stmt)
26571 2134546 : && stmt_info
26572 9970499 : && stmt_info->stmt && gimple_code (stmt_info->stmt) == GIMPLE_ASSIGN)
26573 : {
26574 1706458 : tree_code subcode = gimple_assign_rhs_code (stmt_info->stmt);
26575 : /*machine_mode inner_mode = mode;
26576 : if (VECTOR_MODE_P (mode))
26577 : inner_mode = GET_MODE_INNER (mode);*/
26578 :
26579 1706458 : switch (subcode)
26580 : {
26581 679608 : case PLUS_EXPR:
26582 679608 : case POINTER_PLUS_EXPR:
26583 679608 : case MINUS_EXPR:
26584 679608 : if (kind == scalar_stmt)
26585 : {
26586 424548 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
26587 99039 : stmt_cost = ix86_cost->addss;
26588 325509 : else if (X87_FLOAT_MODE_P (mode))
26589 139 : stmt_cost = ix86_cost->fadd;
26590 : else
26591 325370 : stmt_cost = ix86_cost->add;
26592 : }
26593 : else
26594 255060 : stmt_cost = ix86_vec_cost (mode, fp ? ix86_cost->addss
26595 : : ix86_cost->sse_op);
26596 : break;
26597 :
26598 263842 : case MULT_EXPR:
26599 : /* For MULT_HIGHPART_EXPR, x86 only supports pmulhw,
26600 : take it as MULT_EXPR. */
26601 263842 : case MULT_HIGHPART_EXPR:
26602 263842 : stmt_cost = ix86_multiplication_cost (ix86_cost, mode);
26603 263842 : break;
26604 : /* There's no direct instruction for WIDEN_MULT_EXPR,
26605 : take emulation into account. */
26606 1221 : case WIDEN_MULT_EXPR:
26607 2442 : stmt_cost = ix86_widen_mult_cost (ix86_cost, mode,
26608 1221 : TYPE_UNSIGNED (vectype));
26609 1221 : break;
26610 :
26611 11305 : case NEGATE_EXPR:
26612 11305 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
26613 3564 : stmt_cost = ix86_cost->sse_op;
26614 7741 : else if (X87_FLOAT_MODE_P (mode))
26615 0 : stmt_cost = ix86_cost->fchs;
26616 7741 : else if (VECTOR_MODE_P (mode))
26617 3839 : stmt_cost = ix86_vec_cost (mode, ix86_cost->sse_op);
26618 : else
26619 3902 : stmt_cost = ix86_cost->add;
26620 : break;
26621 15406 : case TRUNC_DIV_EXPR:
26622 15406 : case CEIL_DIV_EXPR:
26623 15406 : case FLOOR_DIV_EXPR:
26624 15406 : case ROUND_DIV_EXPR:
26625 15406 : case TRUNC_MOD_EXPR:
26626 15406 : case CEIL_MOD_EXPR:
26627 15406 : case FLOOR_MOD_EXPR:
26628 15406 : case RDIV_EXPR:
26629 15406 : case ROUND_MOD_EXPR:
26630 15406 : case EXACT_DIV_EXPR:
26631 15406 : stmt_cost = ix86_division_cost (ix86_cost, mode);
26632 15406 : break;
26633 :
26634 86710 : case RSHIFT_EXPR:
26635 86710 : case LSHIFT_EXPR:
26636 86710 : case LROTATE_EXPR:
26637 86710 : case RROTATE_EXPR:
26638 86710 : {
26639 86710 : tree op1 = gimple_assign_rhs1 (stmt_info->stmt);
26640 86710 : tree op2 = gimple_assign_rhs2 (stmt_info->stmt);
26641 86710 : stmt_cost = ix86_shift_rotate_cost
26642 86710 : (ix86_cost,
26643 : (subcode == RSHIFT_EXPR
26644 46117 : && !TYPE_UNSIGNED (TREE_TYPE (op1)))
26645 : ? ASHIFTRT : LSHIFTRT, mode,
26646 86710 : TREE_CODE (op2) == INTEGER_CST,
26647 86710 : cst_and_fits_in_hwi (op2)
26648 55202 : ? int_cst_value (op2) : -1,
26649 : false, false, NULL, NULL);
26650 : }
26651 86710 : break;
26652 104461 : case NOP_EXPR:
26653 : /* Only sign-conversions are free. */
26654 104461 : if (tree_nop_conversion_p
26655 104461 : (TREE_TYPE (gimple_assign_lhs (stmt_info->stmt)),
26656 104461 : TREE_TYPE (gimple_assign_rhs1 (stmt_info->stmt))))
26657 : stmt_cost = 0;
26658 104461 : else if (fp)
26659 10461 : stmt_cost = vec_fp_conversion_cost
26660 10461 : (ix86_tune_cost, GET_MODE_BITSIZE (mode));
26661 : break;
26662 :
26663 23369 : case FLOAT_EXPR:
26664 23369 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
26665 17552 : stmt_cost = ix86_cost->cvtsi2ss;
26666 5817 : else if (X87_FLOAT_MODE_P (mode))
26667 : /* TODO: We do not have cost tables for x87. */
26668 50 : stmt_cost = ix86_cost->fadd;
26669 : else
26670 5767 : stmt_cost = ix86_vec_cost (mode, ix86_cost->cvtpi2ps);
26671 : break;
26672 :
26673 2409 : case FIX_TRUNC_EXPR:
26674 2409 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
26675 0 : stmt_cost = ix86_cost->cvtss2si;
26676 2409 : else if (X87_FLOAT_MODE_P (mode))
26677 : /* TODO: We do not have cost tables for x87. */
26678 0 : stmt_cost = ix86_cost->fadd;
26679 : else
26680 2409 : stmt_cost = ix86_vec_cost (mode, ix86_cost->cvtps2pi);
26681 : break;
26682 :
26683 56497 : case COND_EXPR:
26684 56497 : {
26685 : /* SSE2 conditinal move sequence is:
26686 : pcmpgtd %xmm5, %xmm0 (accounted separately)
26687 : pand %xmm0, %xmm2
26688 : pandn %xmm1, %xmm0
26689 : por %xmm2, %xmm0
26690 : while SSE4 uses cmp + blend
26691 : and AVX512 masked moves.
26692 :
26693 : The condition is accounted separately since we usually have
26694 : p = a < b
26695 : c = p ? x : y
26696 : and we will account first statement as setcc. Exception is when
26697 : p is loaded from memory as bool and then we will not account
26698 : the compare, but there is no way to check for this. */
26699 :
26700 56497 : int ninsns = TARGET_SSE4_1 ? 1 : 3;
26701 :
26702 : /* If one of parameters is 0 or -1 the sequence will be simplified:
26703 : (if_true & mask) | (if_false & ~mask) -> if_true & mask */
26704 23700 : if (ninsns > 1
26705 23700 : && (zerop (gimple_assign_rhs2 (stmt_info->stmt))
26706 23344 : || zerop (gimple_assign_rhs3 (stmt_info->stmt))
26707 13223 : || integer_minus_onep
26708 13223 : (gimple_assign_rhs2 (stmt_info->stmt))
26709 12930 : || integer_minus_onep
26710 12930 : (gimple_assign_rhs3 (stmt_info->stmt))))
26711 : ninsns = 1;
26712 :
26713 56497 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
26714 5062 : stmt_cost = ninsns * ix86_cost->sse_op;
26715 51435 : else if (X87_FLOAT_MODE_P (mode))
26716 : /* x87 requires conditional branch. We don't have cost for
26717 : that. */
26718 : ;
26719 51426 : else if (VECTOR_MODE_P (mode))
26720 21631 : stmt_cost = ix86_vec_cost (mode, ninsns * ix86_cost->sse_op);
26721 : else
26722 : /* compare (accounted separately) + cmov. */
26723 29795 : stmt_cost = ix86_cost->add;
26724 : }
26725 : break;
26726 :
26727 37427 : case MIN_EXPR:
26728 37427 : case MAX_EXPR:
26729 37427 : if (fp)
26730 : {
26731 1507 : if (X87_FLOAT_MODE_P (mode)
26732 533 : && !SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
26733 : /* x87 requires conditional branch. We don't have cost for
26734 : that. */
26735 : ;
26736 : else
26737 : /* minss */
26738 1507 : stmt_cost = ix86_vec_cost (mode, ix86_cost->sse_op);
26739 : }
26740 : else
26741 : {
26742 35920 : if (VECTOR_MODE_P (mode))
26743 : {
26744 10493 : stmt_cost = ix86_vec_cost (mode, ix86_cost->sse_op);
26745 : /* vpmin was introduced in SSE3.
26746 : SSE2 needs pcmpgtd + pand + pandn + pxor.
26747 : If one of parameters is 0 or -1 the sequence is simplified
26748 : to pcmpgtd + pand. */
26749 10493 : if (!TARGET_SSSE3)
26750 : {
26751 8534 : if (zerop (gimple_assign_rhs2 (stmt_info->stmt))
26752 15297 : || integer_minus_onep
26753 6763 : (gimple_assign_rhs2 (stmt_info->stmt)))
26754 1771 : stmt_cost *= 2;
26755 : else
26756 6763 : stmt_cost *= 4;
26757 : }
26758 : }
26759 : else
26760 : /* cmp + cmov. */
26761 25427 : stmt_cost = ix86_cost->add * 2;
26762 : }
26763 : break;
26764 :
26765 1951 : case ABS_EXPR:
26766 1951 : case ABSU_EXPR:
26767 1951 : if (fp)
26768 : {
26769 1090 : if (X87_FLOAT_MODE_P (mode)
26770 583 : && !SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
26771 : /* fabs. */
26772 0 : stmt_cost = ix86_cost->fabs;
26773 : else
26774 : /* andss of sign bit. */
26775 1090 : stmt_cost = ix86_vec_cost (mode, ix86_cost->sse_op);
26776 : }
26777 : else
26778 : {
26779 861 : if (VECTOR_MODE_P (mode))
26780 : {
26781 114 : stmt_cost = ix86_vec_cost (mode, ix86_cost->sse_op);
26782 : /* vabs was introduced in SSE3.
26783 : SSE3 uses psrat + pxor + psub. */
26784 114 : if (!TARGET_SSSE3)
26785 84 : stmt_cost *= 3;
26786 : }
26787 : else
26788 : /* neg + cmov. */
26789 747 : stmt_cost = ix86_cost->add * 2;
26790 : }
26791 : break;
26792 :
26793 164032 : case BIT_IOR_EXPR:
26794 164032 : case BIT_XOR_EXPR:
26795 164032 : case BIT_AND_EXPR:
26796 164032 : case BIT_NOT_EXPR:
26797 164032 : gcc_assert (!SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode)
26798 : && !X87_FLOAT_MODE_P (mode));
26799 164032 : if (VECTOR_MODE_P (mode))
26800 56478 : stmt_cost = ix86_vec_cost (mode, ix86_cost->sse_op);
26801 : else
26802 107554 : stmt_cost = ix86_cost->add;
26803 : break;
26804 :
26805 258220 : default:
26806 258220 : if (TREE_CODE_CLASS (subcode) == tcc_comparison)
26807 : {
26808 100295 : tree op_type;
26809 100295 : if (kind == vector_stmt)
26810 32718 : op_type = SLP_TREE_VECTYPE (SLP_TREE_CHILDREN (node)[0]);
26811 : else
26812 67577 : op_type = vect_comparison_type (stmt_info);
26813 100295 : mode = TYPE_MODE (op_type);
26814 :
26815 100295 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
26816 : /* CMPccS? insructions are cheap, so use sse_op. While they
26817 : produce a mask which may need to be turned to 0/1 by and,
26818 : expect that this will be optimized away in a common case. */
26819 8439 : stmt_cost = ix86_cost->sse_op;
26820 91856 : else if (X87_FLOAT_MODE_P (mode))
26821 : /* fcmp + setcc. */
26822 32 : stmt_cost = ix86_cost->fadd + ix86_cost->add;
26823 91824 : else if (VECTOR_MODE_P (mode))
26824 32661 : stmt_cost = ix86_vec_cost (mode, ix86_cost->sse_op);
26825 : else
26826 : /* setcc. */
26827 59163 : stmt_cost = ix86_cost->add;
26828 : break;
26829 : }
26830 : break;
26831 : }
26832 : }
26833 :
26834 : /* Record number of load/store/gather/scatter in vectorized body. */
26835 7845192 : if (where == vect_body && !m_costing_for_scalar)
26836 : {
26837 2082481 : int scale = 1;
26838 2082481 : if (vectype
26839 4150191 : && ((GET_MODE_SIZE (TYPE_MODE (vectype)) == 64
26840 59962 : && TARGET_AVX512_SPLIT_REGS)
26841 4135118 : || (GET_MODE_SIZE (TYPE_MODE (vectype)) == 32
26842 118743 : && TARGET_AVX256_SPLIT_REGS)))
26843 : scale = 2;
26844 :
26845 2082481 : switch (kind)
26846 : {
26847 : /* Emulated gather/scatter or any scalarization. */
26848 117855 : case scalar_load:
26849 117855 : case scalar_stmt:
26850 117855 : case scalar_store:
26851 117855 : case vector_gather_load:
26852 117855 : case vector_scatter_store:
26853 117855 : m_prefer_unroll = false;
26854 117855 : break;
26855 :
26856 648771 : case vector_stmt:
26857 648771 : case vec_to_scalar:
26858 : /* Count number of reduction FMA and "real" DOT_PROD_EXPR,
26859 : unroll in the vectorizer will enable partial sum. */
26860 648771 : if (stmt_info
26861 648742 : && vect_is_reduction (stmt_info)
26862 718274 : && stmt_info->stmt)
26863 : {
26864 : /* Handle __builtin_fma. */
26865 69503 : if (gimple_call_combined_fn (stmt_info->stmt) == CFN_FMA)
26866 : {
26867 11 : m_num_reduc[X86_REDUC_FMA] += count * scale;
26868 11 : break;
26869 : }
26870 :
26871 69492 : if (!is_gimple_assign (stmt_info->stmt))
26872 : break;
26873 :
26874 66726 : tree_code subcode = gimple_assign_rhs_code (stmt_info->stmt);
26875 66726 : machine_mode inner_mode = GET_MODE_INNER (mode);
26876 66726 : tree rhs1, rhs2;
26877 66726 : bool native_vnni_p = true;
26878 66726 : gimple* def;
26879 66726 : machine_mode mode_rhs;
26880 66726 : switch (subcode)
26881 : {
26882 49816 : case PLUS_EXPR:
26883 49816 : case MINUS_EXPR:
26884 49816 : if (!fp || !flag_associative_math
26885 26417 : || flag_fp_contract_mode != FP_CONTRACT_FAST)
26886 : break;
26887 :
26888 : /* FMA condition for different modes. */
26889 26417 : if (((inner_mode == DFmode || inner_mode == SFmode)
26890 26387 : && !TARGET_FMA && !TARGET_AVX512VL)
26891 8624 : || (inner_mode == HFmode && !TARGET_AVX512FP16)
26892 8624 : || (inner_mode == BFmode && !TARGET_AVX10_2))
26893 : break;
26894 :
26895 : /* MULT_EXPR + PLUS_EXPR/MINUS_EXPR is transformed
26896 : to FMA/FNMA after vectorization. */
26897 8624 : rhs1 = gimple_assign_rhs1 (stmt_info->stmt);
26898 8624 : rhs2 = gimple_assign_rhs2 (stmt_info->stmt);
26899 8624 : if (subcode == PLUS_EXPR
26900 6771 : && TREE_CODE (rhs1) == SSA_NAME
26901 6771 : && (def = SSA_NAME_DEF_STMT (rhs1), true)
26902 6771 : && is_gimple_assign (def)
26903 12029 : && gimple_assign_rhs_code (def) == MULT_EXPR)
26904 1992 : m_num_reduc[X86_REDUC_FMA] += count * scale;
26905 6632 : else if (TREE_CODE (rhs2) == SSA_NAME
26906 6632 : && (def = SSA_NAME_DEF_STMT (rhs2), true)
26907 6632 : && is_gimple_assign (def)
26908 13175 : && gimple_assign_rhs_code (def) == MULT_EXPR)
26909 6537 : m_num_reduc[X86_REDUC_FMA] += count * scale;
26910 : break;
26911 :
26912 : /* Vectorizer lane_reducing_op_p supports DOT_PROX_EXPR,
26913 : WIDEN_SUM_EXPR and SAD_EXPR, x86 backend only supports
26914 : SAD_EXPR (usad{v16qi,v32qi,v64qi}) and DOT_PROD_EXPR. */
26915 616 : case DOT_PROD_EXPR:
26916 616 : rhs1 = gimple_assign_rhs1 (stmt_info->stmt);
26917 616 : mode_rhs = TYPE_MODE (TREE_TYPE (rhs1));
26918 616 : if (mode_rhs == QImode)
26919 : {
26920 343 : rhs2 = gimple_assign_rhs2 (stmt_info->stmt);
26921 343 : signop signop1_p = TYPE_SIGN (TREE_TYPE (rhs1));
26922 343 : signop signop2_p = TYPE_SIGN (TREE_TYPE (rhs2));
26923 :
26924 : /* vpdpbusd. */
26925 343 : if (signop1_p != signop2_p)
26926 85 : native_vnni_p
26927 85 : = (GET_MODE_SIZE (mode) == 64
26928 85 : ? TARGET_AVX512VNNI
26929 28 : : ((TARGET_AVX512VNNI && TARGET_AVX512VL)
26930 85 : || TARGET_AVXVNNI));
26931 : else
26932 : /* vpdpbssd. */
26933 258 : native_vnni_p
26934 274 : = (GET_MODE_SIZE (mode) == 64
26935 258 : ? TARGET_AVX10_2
26936 242 : : (TARGET_AVXVNNIINT8 || TARGET_AVX10_2));
26937 : }
26938 616 : m_num_reduc[X86_REDUC_DOT_PROD] += count * scale;
26939 :
26940 : /* Dislike to do unroll and partial sum for
26941 : emulated DOT_PROD_EXPR. */
26942 616 : if (!native_vnni_p)
26943 159 : m_num_reduc[X86_REDUC_DOT_PROD] += 3 * count;
26944 : break;
26945 :
26946 106 : case SAD_EXPR:
26947 106 : m_num_reduc[X86_REDUC_SAD] += count * scale;
26948 106 : break;
26949 :
26950 : default:
26951 : break;
26952 : }
26953 : }
26954 :
26955 : default:
26956 : break;
26957 : }
26958 : }
26959 :
26960 :
26961 7845192 : combined_fn cfn;
26962 7845192 : if ((kind == vector_stmt || kind == scalar_stmt)
26963 2134546 : && stmt_info
26964 2125307 : && stmt_info->stmt
26965 9970499 : && is_gimple_call (stmt_info->stmt))
26966 : {
26967 26479 : tree fndecl = gimple_call_fndecl (stmt_info->stmt);
26968 26479 : cgraph_node *node;
26969 26479 : if ((fndecl
26970 5465 : && (node = cgraph_node::get (fndecl))
26971 5432 : && node->simd_clones)
26972 30939 : || gimple_call_internal_p (stmt_info->stmt, IFN_MASK_CALL))
26973 2460 : stmt_cost = 10 * ix86_vec_cost (mode,
26974 1230 : mode == SFmode ? ix86_cost->fmass
26975 : : ix86_cost->fmasd);
26976 25249 : else if ((cfn = gimple_call_combined_fn (stmt_info->stmt)) != CFN_LAST)
26977 23879 : switch (cfn)
26978 : {
26979 107 : case CFN_FMA:
26980 107 : stmt_cost = ix86_vec_cost (mode,
26981 107 : mode == SFmode ? ix86_cost->fmass
26982 : : ix86_cost->fmasd);
26983 107 : break;
26984 62 : case CFN_MULH:
26985 62 : stmt_cost = ix86_multiplication_cost (ix86_cost, mode);
26986 62 : break;
26987 : default:
26988 : break;
26989 : }
26990 : }
26991 :
26992 7845192 : if (kind == vec_promote_demote)
26993 : {
26994 62539 : int outer_size
26995 : = tree_to_uhwi
26996 62539 : (TYPE_SIZE
26997 62539 : (TREE_TYPE (gimple_assign_lhs (stmt_info->stmt))));
26998 62539 : int inner_size
26999 : = tree_to_uhwi
27000 62539 : (TYPE_SIZE
27001 62539 : (TREE_TYPE (gimple_assign_rhs1 (stmt_info->stmt))));
27002 62539 : bool inner_fp = FLOAT_TYPE_P
27003 : (TREE_TYPE (gimple_assign_rhs1 (stmt_info->stmt)));
27004 :
27005 5638 : if (fp && inner_fp)
27006 5128 : stmt_cost = vec_fp_conversion_cost
27007 5128 : (ix86_tune_cost, GET_MODE_BITSIZE (mode));
27008 57411 : else if (fp && !inner_fp)
27009 6153 : stmt_cost = ix86_vec_cost (mode, ix86_cost->cvtpi2ps);
27010 51258 : else if (!fp && inner_fp)
27011 510 : stmt_cost = ix86_vec_cost (mode, ix86_cost->cvtps2pi);
27012 : else
27013 50748 : stmt_cost = ix86_vec_cost (mode, ix86_cost->sse_op);
27014 : /* VEC_PACK_TRUNC_EXPR and similar demote operations: If outer size is
27015 : greater than inner size we will end up doing two conversions and
27016 : packing them. We always pack pairs; if the size difference is greater
27017 : it is split into multiple demote operations. */
27018 62539 : if (inner_size > outer_size)
27019 23798 : stmt_cost = stmt_cost * 2
27020 23798 : + ix86_vec_cost (mode, ix86_cost->sse_op);
27021 : }
27022 :
27023 : /* If we do elementwise loads into a vector then we are bound by
27024 : latency and execution resources for the many scalar loads
27025 : (AGU and load ports). Try to account for this by scaling the
27026 : construction cost by the number of elements involved. */
27027 7845192 : if ((kind == vec_construct || kind == vec_deconstruct)
27028 7845192 : && ((node
27029 334785 : && (((SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_ELEMENTWISE
27030 346056 : || SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_STRIDED_SLP)
27031 42508 : && (TREE_CODE (DR_STEP (STMT_VINFO_DATA_REF
27032 : (SLP_TREE_REPRESENTATIVE (node))))
27033 : != INTEGER_CST))
27034 22805 : || mat_gather_scatter_p (SLP_TREE_MEMORY_ACCESS_TYPE (node))))))
27035 : {
27036 32475 : auto lsdata = static_cast<vect_load_store_data *> (node->data);
27037 32475 : tree ls_type = lsdata->ls_type ? lsdata->ls_type : vectype;
27038 32475 : tree ls_eltype
27039 32475 : = lsdata->ls_eltype ? lsdata->ls_eltype : TREE_TYPE (ls_type);
27040 32475 : stmt_cost = ix86_vector_cd_cost (TYPE_MODE (ls_type),
27041 32475 : TYPE_MODE (ls_eltype));
27042 32475 : stmt_cost *= (GET_MODE_BITSIZE (TYPE_MODE (ls_type))
27043 64950 : / GET_MODE_BITSIZE (TYPE_MODE (ls_eltype)) + 1);
27044 : }
27045 7812717 : else if ((kind == vec_construct || kind == scalar_to_vec)
27046 517886 : && node
27047 484681 : && SLP_TREE_DEF_TYPE (node) == vect_external_def)
27048 : {
27049 340188 : stmt_cost = ix86_default_vector_cost (kind, mode);
27050 340188 : unsigned i;
27051 340188 : tree op;
27052 1447301 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
27053 766925 : if (TREE_CODE (op) == SSA_NAME)
27054 539694 : TREE_VISITED (op) = 0;
27055 1107113 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
27056 : {
27057 766925 : if (TREE_CODE (op) != SSA_NAME
27058 539694 : || TREE_VISITED (op))
27059 265283 : continue;
27060 501642 : TREE_VISITED (op) = 1;
27061 501642 : gimple *def = SSA_NAME_DEF_STMT (op);
27062 501642 : tree tem;
27063 : /* Look through a conversion. */
27064 501642 : if (is_gimple_assign (def)
27065 281377 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def))
27066 32446 : && ((tem = gimple_assign_rhs1 (def)), true)
27067 534088 : && TREE_CODE (tem) == SSA_NAME)
27068 32233 : def = SSA_NAME_DEF_STMT (tem);
27069 : /* When the component is loaded from memory without sign-
27070 : or zero-extension we can move it to a vector register and/or
27071 : insert it via vpinsr with a memory operand. */
27072 501642 : if (gimple_assign_load_p (def)
27073 142994 : && tree_nop_conversion_p (TREE_TYPE (op),
27074 142994 : TREE_TYPE (gimple_assign_lhs (def)))
27075 776788 : && (GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (op))) > 1
27076 6327 : || TARGET_SSE4_1))
27077 : ;
27078 : /* When the component is extracted from a vector it is already
27079 : in a vector register. */
27080 369305 : else if (is_gimple_assign (def)
27081 143782 : && gimple_assign_rhs_code (def) == BIT_FIELD_REF
27082 372166 : && VECTOR_TYPE_P (TREE_TYPE
27083 : (TREE_OPERAND (gimple_assign_rhs1 (def), 0))))
27084 : ;
27085 : else
27086 : {
27087 366876 : if (fp)
27088 : {
27089 : /* Scalar FP values residing in x87 registers need to be
27090 : spilled and reloaded. */
27091 16818 : auto mode2 = TYPE_MODE (TREE_TYPE (op));
27092 16818 : if (IS_STACK_MODE (mode2))
27093 : {
27094 971 : int cost
27095 : = (ix86_cost->hard_register.fp_store[mode2 == SFmode
27096 971 : ? 0 : 1]
27097 971 : + ix86_cost->sse_load[sse_store_index (mode2)]);
27098 971 : stmt_cost += COSTS_N_INSNS (cost) / 2;
27099 : }
27100 16818 : m_num_sse_needed[where]++;
27101 : }
27102 : else
27103 : {
27104 350058 : m_num_gpr_needed[where]++;
27105 :
27106 350058 : stmt_cost += COSTS_N_INSNS (ix86_cost->integer_to_sse) / 2;
27107 : }
27108 : }
27109 : }
27110 1107113 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
27111 766925 : if (TREE_CODE (op) == SSA_NAME)
27112 539694 : TREE_VISITED (op) = 0;
27113 : }
27114 7845192 : if (stmt_cost == -1)
27115 5954067 : stmt_cost = ix86_default_vector_cost (kind, mode);
27116 :
27117 : /* BIT_FIELD_REF <vect_**, 64, 0> with count 0 costs 0 in body. */
27118 7845192 : if (kind == vec_perm && vectype && count != 0)
27119 : {
27120 123353 : unsigned vec_size = GET_MODE_SIZE (TYPE_MODE (vectype));
27121 123353 : unsigned nunits = TYPE_VECTOR_SUBPARTS (vectype);
27122 123353 : unsigned *num_vec_perm = NULL;
27123 :
27124 123353 : if (vec_size == 32)
27125 4138 : num_vec_perm = m_num_avx256_vec_perm;
27126 119215 : else if (vec_size == 64)
27127 1198 : num_vec_perm = m_num_avx512_vec_perm;
27128 :
27129 5336 : if (num_vec_perm && ix86_count_cross_lane_perm_p (m_vinfo, node, nunits))
27130 : {
27131 5334 : num_vec_perm[where] += count;
27132 5334 : if (dump_file && (dump_flags & TDF_DETAILS))
27133 : {
27134 346 : fprintf (dump_file,
27135 : "Detected avx%u cross-lane permutation: ", vec_size * 8);
27136 346 : if (stmt_info)
27137 343 : print_gimple_expr (dump_file, stmt_info->stmt, 0, TDF_SLIM);
27138 346 : fprintf (dump_file, " \n");
27139 : }
27140 : }
27141 : }
27142 :
27143 : /* Penalize DFmode vector operations for Bonnell. */
27144 7845192 : if (TARGET_CPU_P (BONNELL) && kind == vector_stmt
27145 7845277 : && vectype && GET_MODE_INNER (TYPE_MODE (vectype)) == DFmode)
27146 12 : stmt_cost *= 5; /* FIXME: The value here is arbitrary. */
27147 :
27148 : /* Statements in an inner loop relative to the loop being
27149 : vectorized are weighted more heavily. The value here is
27150 : arbitrary and could potentially be improved with analysis. */
27151 7845192 : retval = adjust_cost_for_freq (stmt_info, where, count * stmt_cost);
27152 :
27153 : /* We need to multiply all vector stmt cost by 1.7 (estimated cost)
27154 : for Silvermont as it has out of order integer pipeline and can execute
27155 : 2 scalar instruction per tick, but has in order SIMD pipeline. */
27156 7845192 : if ((TARGET_CPU_P (SILVERMONT) || TARGET_CPU_P (GOLDMONT)
27157 7845192 : || TARGET_CPU_P (GOLDMONT_PLUS) || TARGET_CPU_P (INTEL))
27158 2551 : && stmt_info && stmt_info->stmt)
27159 : {
27160 2189 : tree lhs_op = gimple_get_lhs (stmt_info->stmt);
27161 2189 : if (lhs_op && TREE_CODE (TREE_TYPE (lhs_op)) == INTEGER_TYPE)
27162 1657 : retval = (retval * 17) / 10;
27163 : }
27164 :
27165 7845192 : m_costs[where] += retval;
27166 :
27167 7845192 : return retval;
27168 : }
27169 :
27170 : void
27171 1887649 : ix86_vector_costs::ix86_vect_estimate_reg_pressure ()
27172 : {
27173 1887649 : unsigned gpr_spill_cost = COSTS_N_INSNS (ix86_cost->int_store [2]) / 2;
27174 1887649 : unsigned sse_spill_cost = COSTS_N_INSNS (ix86_cost->sse_store[0]) / 2;
27175 :
27176 : /* Any better way to have target available fp registers, currently use SSE_REGS. */
27177 1887649 : unsigned target_avail_sse = TARGET_64BIT ? (TARGET_AVX512F ? 32 : 16) : 8;
27178 7550596 : for (unsigned i = 0; i != 3; i++)
27179 : {
27180 5662947 : if (m_num_gpr_needed[i] > target_avail_regs)
27181 924 : m_costs[i] += gpr_spill_cost * (m_num_gpr_needed[i] - target_avail_regs);
27182 : /* Only measure sse registers pressure. */
27183 5662947 : if (TARGET_SSE && (m_num_sse_needed[i] > target_avail_sse))
27184 100 : m_costs[i] += sse_spill_cost * (m_num_sse_needed[i] - target_avail_sse);
27185 : }
27186 1887649 : }
27187 :
27188 : void
27189 1887649 : ix86_vector_costs::finish_cost (const vector_costs *scalar_costs)
27190 : {
27191 1887649 : loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo);
27192 497017 : if (loop_vinfo && !m_costing_for_scalar)
27193 : {
27194 125492 : unsigned int vf = vect_vf_for_cost (loop_vinfo);
27195 547371 : for (auto inst : LOOP_VINFO_SLP_INSTANCES (loop_vinfo))
27196 170895 : if ((SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_group
27197 170895 : || SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_chain)
27198 170895 : && (vect_reduc_type (loop_vinfo, SLP_INSTANCE_TREE (inst))
27199 : == FOLD_LEFT_REDUCTION))
27200 3901 : m_num_fold_left_reduc_lanes
27201 3901 : += vf * SLP_TREE_LANES (SLP_INSTANCE_TREE (inst));
27202 :
27203 125492 : if (m_num_fold_left_reduc_lanes && dump_enabled_p ())
27204 19 : dump_printf_loc (MSG_NOTE, vect_location,
27205 : "in-order FP reduction lanes: %u\n",
27206 : m_num_fold_left_reduc_lanes);
27207 :
27208 : /* We are currently not asking the vectorizer to compare costs
27209 : between different vector mode sizes. When using predication
27210 : that will end up always choosing the preferred mode size even
27211 : if there's a smaller mode covering all lanes. Test for this
27212 : situation and artificially reject the larger mode attempt.
27213 : ??? We currently lack masked ops for sub-SSE sized modes,
27214 : so we could restrict this rejection to AVX and AVX512 modes
27215 : but error on the safe side for now. */
27216 125492 : if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
27217 29 : && !LOOP_VINFO_EPILOGUE_P (loop_vinfo)
27218 17 : && LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
27219 125503 : && (exact_log2 (LOOP_VINFO_VECT_FACTOR (loop_vinfo).to_constant ())
27220 22 : > ceil_log2 (LOOP_VINFO_INT_NITERS (loop_vinfo))))
27221 8 : m_costs[vect_body] = INT_MAX;
27222 :
27223 : /* We'd like to avoid using masking if there's an in-order reduction
27224 : to vectorize because that will also perform in-order adds of
27225 : masked elements (as neutral value, of course) here, but there
27226 : is currently no way to indicate to try un-masked with the same
27227 : mode. */
27228 :
27229 125492 : bool any_reduc_p = false;
27230 498501 : for (int i = 0; i != X86_REDUC_LAST; i++)
27231 374310 : if (m_num_reduc[i])
27232 : {
27233 : any_reduc_p = true;
27234 : break;
27235 : }
27236 :
27237 125492 : if (any_reduc_p
27238 : /* Not much gain for loop with gather and scatter. */
27239 1301 : && m_prefer_unroll
27240 1144 : && !LOOP_VINFO_EPILOGUE_P (loop_vinfo))
27241 : {
27242 1826 : unsigned unroll_factor
27243 913 : = OPTION_SET_P (ix86_vect_unroll_limit)
27244 913 : ? ix86_vect_unroll_limit
27245 913 : : ix86_cost->vect_unroll_limit;
27246 :
27247 913 : if (unroll_factor > 1)
27248 : {
27249 3652 : for (int i = 0 ; i != X86_REDUC_LAST; i++)
27250 : {
27251 2739 : if (m_num_reduc[i])
27252 : {
27253 913 : unsigned tmp = CEIL (ix86_cost->reduc_lat_mult_thr[i],
27254 : m_num_reduc[i]);
27255 2739 : unroll_factor = MIN (unroll_factor, tmp);
27256 : }
27257 : }
27258 :
27259 1826 : m_suggested_unroll_factor = 1 << ceil_log2 (unroll_factor);
27260 : }
27261 : }
27262 :
27263 : }
27264 :
27265 1887649 : ix86_vect_estimate_reg_pressure ();
27266 :
27267 7550596 : for (int i = 0; i != 3; i++)
27268 5662947 : if (m_num_avx256_vec_perm[i]
27269 522 : && TARGET_AVX256_AVOID_VEC_PERM)
27270 7 : m_costs[i] = INT_MAX;
27271 :
27272 7550596 : for (int i = 0; i != 3; i++)
27273 5662947 : if (m_num_avx512_vec_perm[i] && TARGET_AVX512_AVOID_VEC_PERM)
27274 5 : m_costs[i] = INT_MAX;
27275 :
27276 : /* When X86_TUNE_AVX512_TWO_EPILOGUES is enabled arrange for both
27277 : a AVX2 and a SSE epilogue for AVX512 vectorized loops. */
27278 1887649 : if (loop_vinfo
27279 497017 : && LOOP_VINFO_EPILOGUE_P (loop_vinfo)
27280 43386 : && GET_MODE_SIZE (loop_vinfo->vector_mode) == 32
27281 1888403 : && ix86_tune_features[X86_TUNE_AVX512_TWO_EPILOGUES])
27282 25 : m_suggested_epilogue_mode = V16QImode;
27283 : /* When a 128bit SSE vectorized epilogue still has a VF of 16 or larger
27284 : enable a 64bit SSE epilogue. */
27285 1887649 : if (loop_vinfo
27286 497017 : && LOOP_VINFO_EPILOGUE_P (loop_vinfo)
27287 43386 : && GET_MODE_SIZE (loop_vinfo->vector_mode) == 16
27288 1890154 : && LOOP_VINFO_VECT_FACTOR (loop_vinfo).to_constant () >= 16)
27289 95 : m_suggested_epilogue_mode = V8QImode;
27290 :
27291 : /* When X86_TUNE_AVX512_MASKED_EPILOGUES is enabled try to use
27292 : a masked epilogue if that doesn't seem detrimental. */
27293 1887649 : if (loop_vinfo
27294 497017 : && !LOOP_VINFO_EPILOGUE_P (loop_vinfo)
27295 475324 : && LOOP_VINFO_VECT_FACTOR (loop_vinfo).to_constant () > 2
27296 : /* Avoid a masked epilog if cascaded epilogues eventually get us
27297 : to one with VF 1 as that means no scalar epilog at all. */
27298 76312 : && !((GET_MODE_SIZE (loop_vinfo->vector_mode)
27299 76312 : / LOOP_VINFO_VECT_FACTOR (loop_vinfo).to_constant () == 16)
27300 34 : && ix86_tune_features[X86_TUNE_AVX512_TWO_EPILOGUES])
27301 76311 : && ix86_tune_features[X86_TUNE_AVX512_MASKED_EPILOGUES]
27302 1887833 : && !OPTION_SET_P (param_vect_partial_vector_usage))
27303 : {
27304 166 : bool avoid = false;
27305 166 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
27306 129 : && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) >= 0)
27307 : {
27308 129 : unsigned int peel_niter
27309 : = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
27310 129 : if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
27311 0 : peel_niter += 1;
27312 : /* When we know the number of scalar iterations of the epilogue,
27313 : avoid masking when a single vector epilog iteration handles
27314 : it in full. */
27315 129 : if (pow2p_hwi ((LOOP_VINFO_INT_NITERS (loop_vinfo) - peel_niter)
27316 129 : % LOOP_VINFO_VECT_FACTOR (loop_vinfo).to_constant ()))
27317 : avoid = true;
27318 : }
27319 164 : if (!avoid && loop_outer (loop_outer (LOOP_VINFO_LOOP (loop_vinfo))))
27320 14 : for (auto ddr : LOOP_VINFO_DDRS (loop_vinfo))
27321 : {
27322 4 : if (DDR_ARE_DEPENDENT (ddr) == chrec_known)
27323 : ;
27324 4 : else if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know)
27325 : ;
27326 : else
27327 : {
27328 2 : int loop_depth
27329 4 : = index_in_loop_nest (LOOP_VINFO_LOOP (loop_vinfo)->num,
27330 2 : DDR_LOOP_NEST (ddr));
27331 4 : if (DDR_NUM_DIST_VECTS (ddr) == 1
27332 2 : && DDR_DIST_VECTS (ddr)[0][loop_depth] == 0)
27333 : {
27334 : /* Avoid the case when there's an outer loop that might
27335 : traverse a multi-dimensional array with the inner
27336 : loop just executing the masked epilogue with a
27337 : read-write where the next outer iteration might
27338 : read from the masked part of the previous write,
27339 : 'n' filling half a vector.
27340 : for (j = 0; j < m; ++j)
27341 : for (i = 0; i < n; ++i)
27342 : a[j][i] = c * a[j][i]; */
27343 : avoid = true;
27344 : break;
27345 : }
27346 : }
27347 : }
27348 : /* Avoid using masking if there's an in-order reduction
27349 : to vectorize because that will also perform in-order adds of
27350 : masked elements (as neutral value, of course). */
27351 166 : if (!avoid)
27352 : {
27353 655 : for (auto inst : LOOP_VINFO_SLP_INSTANCES (loop_vinfo))
27354 173 : if (SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_group
27355 173 : && (vect_reduc_type (loop_vinfo, SLP_INSTANCE_TREE (inst))
27356 : == FOLD_LEFT_REDUCTION))
27357 : {
27358 : avoid = true;
27359 : break;
27360 : }
27361 : }
27362 162 : if (!avoid)
27363 : {
27364 158 : m_suggested_epilogue_mode = loop_vinfo->vector_mode;
27365 158 : m_masked_epilogue = 1;
27366 : }
27367 : }
27368 :
27369 1887649 : vector_costs::finish_cost (scalar_costs);
27370 1887649 : }
27371 :
27372 : /* Return true if THIS has a shorter fold-left reduction chain than OTHER. */
27373 :
27374 : bool
27375 32570 : ix86_vector_costs::better_fold_left_reduc_than_p
27376 : (const vector_costs *other) const
27377 : {
27378 32570 : auto other_costs = static_cast<const ix86_vector_costs *> (other);
27379 32570 : if (m_num_fold_left_reduc_lanes >= other_costs->m_num_fold_left_reduc_lanes)
27380 : return false;
27381 :
27382 : /* Do not let emulated sub-SSE modes win on this alone. */
27383 316 : loop_vec_info loop_vinfo = as_a<loop_vec_info> (m_vinfo);
27384 632 : return known_ge (GET_MODE_SIZE (loop_vinfo->vector_mode), 16);
27385 : }
27386 :
27387 : /* Return true if THIS should be preferred over OTHER as main vector loop. */
27388 :
27389 : bool
27390 31088 : ix86_vector_costs::better_main_loop_than_p (const vector_costs *other) const
27391 : {
27392 31088 : loop_vec_info this_loop_vinfo = as_a<loop_vec_info> (this->vinfo ());
27393 31088 : loop_vec_info other_loop_vinfo = as_a<loop_vec_info> (other->vinfo ());
27394 :
27395 31088 : if (better_fold_left_reduc_than_p (other))
27396 : return true;
27397 :
27398 : /* If the other loop is masked it does not need an epilog. Prefer that
27399 : if the current loop cannot be vectorized fully with a vector
27400 : epilogs with at most one scalar iteration left. */
27401 21119 : if (LOOP_VINFO_NITERS_KNOWN_P (this_loop_vinfo)
27402 21119 : && LOOP_VINFO_USING_PARTIAL_VECTORS_P (other_loop_vinfo)
27403 4 : && known_gt (LOOP_VINFO_VECT_FACTOR (other_loop_vinfo),
27404 : LOOP_VINFO_INT_NITERS (this_loop_vinfo))
27405 31057 : && (popcount_hwi (LOOP_VINFO_INT_NITERS (this_loop_vinfo) & ~1)
27406 4 : > (param_vect_epilogues_nomask != 0)))
27407 : return false;
27408 :
27409 31049 : return vector_costs::better_main_loop_than_p (other);
27410 : }
27411 :
27412 : /* Return true if THIS should be preferred over OTHER as epilog vector
27413 : loop when vectorizing MAIN_LOOP. */
27414 :
27415 : bool
27416 1482 : ix86_vector_costs::better_epilogue_loop_than_p (const vector_costs *other,
27417 : loop_vec_info main_loop) const
27418 : {
27419 1482 : loop_vec_info this_loop_info = as_a <loop_vec_info> (this->vinfo ());
27420 :
27421 1482 : if (better_fold_left_reduc_than_p (other))
27422 : return true;
27423 :
27424 : /* The x86 target allows for multiple vector epilogues, if THIS is
27425 : the suggested epilog mode of OTHER then keep the latter unless
27426 : THIS has a VF of one which means no further epilog needed. */
27427 1482 : int tem;
27428 1482 : if (known_gt (LOOP_VINFO_VECT_FACTOR (this_loop_info), 1U)
27429 1482 : && (GET_MODE_SIZE (other->suggested_epilogue_mode (tem))
27430 2940 : == GET_MODE_SIZE (this_loop_info->vector_mode)))
27431 : return false;
27432 1409 : return vector_costs::better_epilogue_loop_than_p (other, main_loop);
27433 : }
27434 :
27435 : /* Validate target specific memory model bits in VAL. */
27436 :
27437 : static unsigned HOST_WIDE_INT
27438 412280 : ix86_memmodel_check (unsigned HOST_WIDE_INT val)
27439 : {
27440 412280 : enum memmodel model = memmodel_from_int (val);
27441 412280 : bool strong;
27442 :
27443 412280 : if (val & ~(unsigned HOST_WIDE_INT)(IX86_HLE_ACQUIRE|IX86_HLE_RELEASE
27444 : |MEMMODEL_MASK)
27445 412276 : || ((val & IX86_HLE_ACQUIRE) && (val & IX86_HLE_RELEASE)))
27446 : {
27447 4 : warning (OPT_Winvalid_memory_model,
27448 : "unknown architecture specific memory model");
27449 4 : return MEMMODEL_SEQ_CST;
27450 : }
27451 412276 : strong = (is_mm_acq_rel (model) || is_mm_seq_cst (model));
27452 412276 : if (val & IX86_HLE_ACQUIRE && !(is_mm_acquire (model) || strong))
27453 : {
27454 0 : warning (OPT_Winvalid_memory_model,
27455 : "%<HLE_ACQUIRE%> not used with %<ACQUIRE%> or stronger "
27456 : "memory model");
27457 0 : return MEMMODEL_SEQ_CST | IX86_HLE_ACQUIRE;
27458 : }
27459 412276 : if (val & IX86_HLE_RELEASE && !(is_mm_release (model) || strong))
27460 : {
27461 0 : warning (OPT_Winvalid_memory_model,
27462 : "%<HLE_RELEASE%> not used with %<RELEASE%> or stronger "
27463 : "memory model");
27464 0 : return MEMMODEL_SEQ_CST | IX86_HLE_RELEASE;
27465 : }
27466 : return val;
27467 : }
27468 :
27469 : /* Set CLONEI->vecsize_mangle, CLONEI->mask_mode, CLONEI->vecsize_int,
27470 : CLONEI->vecsize_float and if CLONEI->simdlen is 0, also
27471 : CLONEI->simdlen. Return 0 if SIMD clones shouldn't be emitted,
27472 : or number of vecsize_mangle variants that should be emitted. */
27473 :
27474 : static int
27475 7689 : ix86_simd_clone_compute_vecsize_and_simdlen (struct cgraph_node *node,
27476 : struct cgraph_simd_clone *clonei,
27477 : tree base_type, int num,
27478 : bool explicit_p)
27479 : {
27480 7689 : int ret = 1;
27481 :
27482 7689 : if (clonei->simdlen
27483 7689 : && (clonei->simdlen < 2
27484 1321 : || clonei->simdlen > 1024
27485 1321 : || (clonei->simdlen & (clonei->simdlen - 1)) != 0))
27486 : {
27487 0 : if (explicit_p)
27488 0 : warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
27489 : "unsupported simdlen %wd", clonei->simdlen.to_constant ());
27490 0 : return 0;
27491 : }
27492 :
27493 7689 : tree ret_type = TREE_TYPE (TREE_TYPE (node->decl));
27494 7689 : if (TREE_CODE (ret_type) != VOID_TYPE)
27495 6897 : switch (TYPE_MODE (ret_type))
27496 : {
27497 6897 : case E_QImode:
27498 6897 : case E_HImode:
27499 6897 : case E_SImode:
27500 6897 : case E_DImode:
27501 6897 : case E_SFmode:
27502 6897 : case E_DFmode:
27503 : /* case E_SCmode: */
27504 : /* case E_DCmode: */
27505 6897 : if (!AGGREGATE_TYPE_P (ret_type))
27506 : break;
27507 : /* FALLTHRU */
27508 2 : default:
27509 2 : if (explicit_p)
27510 2 : warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
27511 : "unsupported return type %qT for simd", ret_type);
27512 2 : return 0;
27513 : }
27514 :
27515 7687 : tree t;
27516 7687 : int i;
27517 7687 : tree type_arg_types = TYPE_ARG_TYPES (TREE_TYPE (node->decl));
27518 7687 : bool decl_arg_p = (node->definition || type_arg_types == NULL_TREE);
27519 :
27520 7687 : for (t = (decl_arg_p ? DECL_ARGUMENTS (node->decl) : type_arg_types), i = 0;
27521 20654 : t && t != void_list_node; t = TREE_CHAIN (t), i++)
27522 : {
27523 16918 : tree arg_type = decl_arg_p ? TREE_TYPE (t) : TREE_VALUE (t);
27524 12972 : switch (TYPE_MODE (arg_type))
27525 : {
27526 12953 : case E_QImode:
27527 12953 : case E_HImode:
27528 12953 : case E_SImode:
27529 12953 : case E_DImode:
27530 12953 : case E_SFmode:
27531 12953 : case E_DFmode:
27532 : /* case E_SCmode: */
27533 : /* case E_DCmode: */
27534 12953 : if (!AGGREGATE_TYPE_P (arg_type))
27535 : break;
27536 : /* FALLTHRU */
27537 41 : default:
27538 41 : if (clonei->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM)
27539 : break;
27540 5 : if (explicit_p)
27541 5 : warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
27542 : "unsupported argument type %qT for simd", arg_type);
27543 : return 0;
27544 : }
27545 : }
27546 :
27547 7682 : if (!TREE_PUBLIC (node->decl) || !explicit_p)
27548 : {
27549 : /* If the function isn't exported, we can pick up just one ISA
27550 : for the clones. */
27551 114 : if (TARGET_AVX512F)
27552 0 : clonei->vecsize_mangle = 'e';
27553 114 : else if (TARGET_AVX2)
27554 1 : clonei->vecsize_mangle = 'd';
27555 113 : else if (TARGET_AVX)
27556 88 : clonei->vecsize_mangle = 'c';
27557 : else
27558 25 : clonei->vecsize_mangle = 'b';
27559 : ret = 1;
27560 : }
27561 : else
27562 : {
27563 7568 : clonei->vecsize_mangle = "bcde"[num];
27564 7568 : ret = 4;
27565 : }
27566 7682 : clonei->mask_mode = VOIDmode;
27567 7682 : switch (clonei->vecsize_mangle)
27568 : {
27569 1917 : case 'b':
27570 1917 : clonei->vecsize_int = 128;
27571 1917 : clonei->vecsize_float = 128;
27572 1917 : break;
27573 1980 : case 'c':
27574 1980 : clonei->vecsize_int = 128;
27575 1980 : clonei->vecsize_float = 256;
27576 1980 : break;
27577 1893 : case 'd':
27578 1893 : clonei->vecsize_int = 256;
27579 1893 : clonei->vecsize_float = 256;
27580 1893 : break;
27581 1892 : case 'e':
27582 1892 : clonei->vecsize_int = 512;
27583 1892 : clonei->vecsize_float = 512;
27584 1892 : if (TYPE_MODE (base_type) == QImode)
27585 19 : clonei->mask_mode = DImode;
27586 : else
27587 1873 : clonei->mask_mode = SImode;
27588 : break;
27589 : }
27590 7682 : if (clonei->simdlen == 0)
27591 : {
27592 6361 : if (SCALAR_INT_MODE_P (TYPE_MODE (base_type)))
27593 3297 : clonei->simdlen = clonei->vecsize_int;
27594 : else
27595 3064 : clonei->simdlen = clonei->vecsize_float;
27596 6361 : clonei->simdlen = clonei->simdlen
27597 12722 : / GET_MODE_BITSIZE (TYPE_MODE (base_type));
27598 : }
27599 1321 : else if (clonei->simdlen > 16)
27600 : {
27601 : /* For compatibility with ICC, use the same upper bounds
27602 : for simdlen. In particular, for CTYPE below, use the return type,
27603 : unless the function returns void, in that case use the characteristic
27604 : type. If it is possible for given SIMDLEN to pass CTYPE value
27605 : in registers (8 [XYZ]MM* regs for 32-bit code, 16 [XYZ]MM* regs
27606 : for 64-bit code), accept that SIMDLEN, otherwise warn and don't
27607 : emit corresponding clone. */
27608 12 : tree ctype = ret_type;
27609 12 : if (VOID_TYPE_P (ret_type))
27610 0 : ctype = base_type;
27611 24 : int cnt = GET_MODE_BITSIZE (TYPE_MODE (ctype)) * clonei->simdlen;
27612 12 : if (SCALAR_INT_MODE_P (TYPE_MODE (ctype)))
27613 8 : cnt /= clonei->vecsize_int;
27614 : else
27615 4 : cnt /= clonei->vecsize_float;
27616 12 : if (cnt > (TARGET_64BIT ? 16 : 8))
27617 : {
27618 0 : if (explicit_p)
27619 0 : warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
27620 : "unsupported simdlen %wd",
27621 : clonei->simdlen.to_constant ());
27622 0 : return 0;
27623 : }
27624 : }
27625 : return ret;
27626 : }
27627 :
27628 : /* If SIMD clone NODE can't be used in a vectorized loop
27629 : in current function, return -1, otherwise return a badness of using it
27630 : (0 if it is most desirable from vecsize_mangle point of view, 1
27631 : slightly less desirable, etc.). */
27632 :
27633 : static int
27634 1764 : ix86_simd_clone_usable (struct cgraph_node *node, machine_mode)
27635 : {
27636 1764 : switch (node->simdclone->vecsize_mangle)
27637 : {
27638 626 : case 'b':
27639 626 : if (!TARGET_SSE2)
27640 : return -1;
27641 626 : if (!TARGET_AVX)
27642 : return 0;
27643 525 : return TARGET_AVX512F ? 3 : TARGET_AVX2 ? 2 : 1;
27644 628 : case 'c':
27645 628 : if (!TARGET_AVX)
27646 : return -1;
27647 583 : return TARGET_AVX512F ? 2 : TARGET_AVX2 ? 1 : 0;
27648 322 : case 'd':
27649 322 : if (!TARGET_AVX2)
27650 : return -1;
27651 117 : return TARGET_AVX512F ? 1 : 0;
27652 188 : case 'e':
27653 188 : if (!TARGET_AVX512F)
27654 130 : return -1;
27655 : return 0;
27656 0 : default:
27657 0 : gcc_unreachable ();
27658 : }
27659 : }
27660 :
27661 : /* This function adjusts the unroll factor based on
27662 : the hardware capabilities. For ex, bdver3 has
27663 : a loop buffer which makes unrolling of smaller
27664 : loops less important. This function decides the
27665 : unroll factor using number of memory references
27666 : (value 32 is used) as a heuristic. */
27667 :
27668 : static unsigned
27669 795490 : ix86_loop_unroll_adjust (unsigned nunroll, class loop *loop)
27670 : {
27671 795490 : basic_block *bbs;
27672 795490 : rtx_insn *insn;
27673 795490 : unsigned i;
27674 795490 : unsigned mem_count = 0;
27675 :
27676 : /* Unroll small size loop when unroll factor is not explicitly
27677 : specified. */
27678 795490 : if (ix86_unroll_only_small_loops && !loop->unroll)
27679 : {
27680 751658 : if (loop->ninsns <= ix86_cost->small_unroll_ninsns)
27681 72684 : return MIN (nunroll, ix86_cost->small_unroll_factor);
27682 : else
27683 : return 1;
27684 : }
27685 :
27686 43832 : if (!TARGET_ADJUST_UNROLL)
27687 : return nunroll;
27688 :
27689 : /* Count the number of memory references within the loop body.
27690 : This value determines the unrolling factor for bdver3 and bdver4
27691 : architectures. */
27692 8 : subrtx_iterator::array_type array;
27693 8 : bbs = get_loop_body (loop);
27694 24 : for (i = 0; i < loop->num_nodes; i++)
27695 120 : FOR_BB_INSNS (bbs[i], insn)
27696 104 : if (NONDEBUG_INSN_P (insn))
27697 588 : FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
27698 516 : if (const_rtx x = *iter)
27699 516 : if (MEM_P (x))
27700 : {
27701 28 : machine_mode mode = GET_MODE (x);
27702 56 : unsigned int n_words = GET_MODE_SIZE (mode) / UNITS_PER_WORD;
27703 28 : if (n_words > 4)
27704 0 : mem_count += 2;
27705 : else
27706 28 : mem_count += 1;
27707 : }
27708 8 : free (bbs);
27709 :
27710 8 : if (mem_count && mem_count <=32)
27711 8 : return MIN (nunroll, 32 / mem_count);
27712 :
27713 : return nunroll;
27714 8 : }
27715 :
27716 :
27717 : /* Implement TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P. */
27718 :
27719 : static bool
27720 426476 : ix86_float_exceptions_rounding_supported_p (void)
27721 : {
27722 : /* For x87 floating point with standard excess precision handling,
27723 : there is no adddf3 pattern (since x87 floating point only has
27724 : XFmode operations) so the default hook implementation gets this
27725 : wrong. */
27726 426476 : return TARGET_80387 || (TARGET_SSE && TARGET_SSE_MATH);
27727 : }
27728 :
27729 : /* Implement TARGET_ATOMIC_ASSIGN_EXPAND_FENV. */
27730 :
27731 : static void
27732 7054 : ix86_atomic_assign_expand_fenv (tree *hold, tree *clear, tree *update)
27733 : {
27734 7054 : if (!TARGET_80387 && !(TARGET_SSE && TARGET_SSE_MATH))
27735 : return;
27736 7054 : tree exceptions_var = create_tmp_var_raw (integer_type_node);
27737 7054 : if (TARGET_80387)
27738 : {
27739 7054 : tree fenv_index_type = build_index_type (size_int (6));
27740 7054 : tree fenv_type = build_array_type (unsigned_type_node, fenv_index_type);
27741 7054 : tree fenv_var = create_tmp_var_raw (fenv_type);
27742 7054 : TREE_ADDRESSABLE (fenv_var) = 1;
27743 7054 : tree fenv_ptr = build_pointer_type (fenv_type);
27744 7054 : tree fenv_addr = build1 (ADDR_EXPR, fenv_ptr, fenv_var);
27745 7054 : fenv_addr = fold_convert (ptr_type_node, fenv_addr);
27746 7054 : tree fnstenv = get_ix86_builtin (IX86_BUILTIN_FNSTENV);
27747 7054 : tree fldenv = get_ix86_builtin (IX86_BUILTIN_FLDENV);
27748 7054 : tree fnstsw = get_ix86_builtin (IX86_BUILTIN_FNSTSW);
27749 7054 : tree fnclex = get_ix86_builtin (IX86_BUILTIN_FNCLEX);
27750 7054 : tree hold_fnstenv = build_call_expr (fnstenv, 1, fenv_addr);
27751 7054 : tree hold_fnclex = build_call_expr (fnclex, 0);
27752 7054 : fenv_var = build4 (TARGET_EXPR, fenv_type, fenv_var, hold_fnstenv,
27753 : NULL_TREE, NULL_TREE);
27754 7054 : *hold = build2 (COMPOUND_EXPR, void_type_node, fenv_var,
27755 : hold_fnclex);
27756 7054 : *clear = build_call_expr (fnclex, 0);
27757 7054 : tree sw_var = create_tmp_var_raw (short_unsigned_type_node);
27758 7054 : tree fnstsw_call = build_call_expr (fnstsw, 0);
27759 7054 : tree sw_mod = build4 (TARGET_EXPR, short_unsigned_type_node, sw_var,
27760 : fnstsw_call, NULL_TREE, NULL_TREE);
27761 7054 : tree exceptions_x87 = fold_convert (integer_type_node, sw_var);
27762 7054 : tree update_mod = build4 (TARGET_EXPR, integer_type_node,
27763 : exceptions_var, exceptions_x87,
27764 : NULL_TREE, NULL_TREE);
27765 7054 : *update = build2 (COMPOUND_EXPR, integer_type_node,
27766 : sw_mod, update_mod);
27767 7054 : tree update_fldenv = build_call_expr (fldenv, 1, fenv_addr);
27768 7054 : *update = build2 (COMPOUND_EXPR, void_type_node, *update, update_fldenv);
27769 : }
27770 7054 : if (TARGET_SSE && TARGET_SSE_MATH)
27771 : {
27772 7054 : tree mxcsr_orig_var = create_tmp_var_raw (unsigned_type_node);
27773 7054 : tree mxcsr_mod_var = create_tmp_var_raw (unsigned_type_node);
27774 7054 : tree stmxcsr = get_ix86_builtin (IX86_BUILTIN_STMXCSR);
27775 7054 : tree ldmxcsr = get_ix86_builtin (IX86_BUILTIN_LDMXCSR);
27776 7054 : tree stmxcsr_hold_call = build_call_expr (stmxcsr, 0);
27777 7054 : tree hold_assign_orig = build4 (TARGET_EXPR, unsigned_type_node,
27778 : mxcsr_orig_var, stmxcsr_hold_call,
27779 : NULL_TREE, NULL_TREE);
27780 7054 : tree hold_mod_val = build2 (BIT_IOR_EXPR, unsigned_type_node,
27781 : mxcsr_orig_var,
27782 : build_int_cst (unsigned_type_node, 0x1f80));
27783 7054 : hold_mod_val = build2 (BIT_AND_EXPR, unsigned_type_node, hold_mod_val,
27784 : build_int_cst (unsigned_type_node, 0xffffffc0));
27785 7054 : tree hold_assign_mod = build4 (TARGET_EXPR, unsigned_type_node,
27786 : mxcsr_mod_var, hold_mod_val,
27787 : NULL_TREE, NULL_TREE);
27788 7054 : tree ldmxcsr_hold_call = build_call_expr (ldmxcsr, 1, mxcsr_mod_var);
27789 7054 : tree hold_all = build2 (COMPOUND_EXPR, unsigned_type_node,
27790 : hold_assign_orig, hold_assign_mod);
27791 7054 : hold_all = build2 (COMPOUND_EXPR, void_type_node, hold_all,
27792 : ldmxcsr_hold_call);
27793 7054 : if (*hold)
27794 7054 : *hold = build2 (COMPOUND_EXPR, void_type_node, *hold, hold_all);
27795 : else
27796 0 : *hold = hold_all;
27797 7054 : tree ldmxcsr_clear_call = build_call_expr (ldmxcsr, 1, mxcsr_mod_var);
27798 7054 : if (*clear)
27799 7054 : *clear = build2 (COMPOUND_EXPR, void_type_node, *clear,
27800 : ldmxcsr_clear_call);
27801 : else
27802 0 : *clear = ldmxcsr_clear_call;
27803 7054 : tree stxmcsr_update_call = build_call_expr (stmxcsr, 0);
27804 7054 : tree exceptions_sse = fold_convert (integer_type_node,
27805 : stxmcsr_update_call);
27806 7054 : if (*update)
27807 : {
27808 7054 : tree exceptions_mod = build2 (BIT_IOR_EXPR, integer_type_node,
27809 : exceptions_var, exceptions_sse);
27810 7054 : tree exceptions_assign = build2 (MODIFY_EXPR, integer_type_node,
27811 : exceptions_var, exceptions_mod);
27812 7054 : *update = build2 (COMPOUND_EXPR, integer_type_node, *update,
27813 : exceptions_assign);
27814 : }
27815 : else
27816 0 : *update = build4 (TARGET_EXPR, integer_type_node, exceptions_var,
27817 : exceptions_sse, NULL_TREE, NULL_TREE);
27818 7054 : tree ldmxcsr_update_call = build_call_expr (ldmxcsr, 1, mxcsr_orig_var);
27819 7054 : *update = build2 (COMPOUND_EXPR, void_type_node, *update,
27820 : ldmxcsr_update_call);
27821 : }
27822 7054 : tree atomic_feraiseexcept
27823 7054 : = builtin_decl_implicit (BUILT_IN_ATOMIC_FERAISEEXCEPT);
27824 7054 : tree atomic_feraiseexcept_call = build_call_expr (atomic_feraiseexcept,
27825 : 1, exceptions_var);
27826 7054 : *update = build2 (COMPOUND_EXPR, void_type_node, *update,
27827 : atomic_feraiseexcept_call);
27828 : }
27829 :
27830 : #if !TARGET_MACHO && !TARGET_DLLIMPORT_DECL_ATTRIBUTES
27831 : /* For i386, common symbol is local only for non-PIE binaries. For
27832 : x86-64, common symbol is local only for non-PIE binaries or linker
27833 : supports copy reloc in PIE binaries. */
27834 :
27835 : static bool
27836 791667468 : ix86_binds_local_p (const_tree exp)
27837 : {
27838 791667468 : bool direct_extern_access
27839 791667468 : = (ix86_direct_extern_access
27840 1579803270 : && !(VAR_OR_FUNCTION_DECL_P (exp)
27841 788135802 : && lookup_attribute ("nodirect_extern_access",
27842 788135802 : DECL_ATTRIBUTES (exp))));
27843 791667468 : if (!direct_extern_access)
27844 1233 : ix86_has_no_direct_extern_access = true;
27845 791667468 : return default_binds_local_p_3 (exp, flag_shlib != 0, true,
27846 : direct_extern_access,
27847 : (direct_extern_access
27848 791666235 : && (!flag_pic
27849 132924541 : || (TARGET_64BIT
27850 791667468 : && HAVE_LD_PIE_COPYRELOC != 0))));
27851 : }
27852 :
27853 : /* If flag_pic or ix86_direct_extern_access is false, then neither
27854 : local nor global relocs should be placed in readonly memory. */
27855 :
27856 : static int
27857 5158374 : ix86_reloc_rw_mask (void)
27858 : {
27859 5158374 : return (flag_pic || !ix86_direct_extern_access) ? 3 : 0;
27860 : }
27861 : #endif
27862 :
27863 : /* Return true iff ADDR can be used as a symbolic base address. */
27864 :
27865 : static bool
27866 3011 : symbolic_base_address_p (rtx addr)
27867 : {
27868 0 : if (SYMBOL_REF_P (addr))
27869 : return true;
27870 :
27871 2987 : if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_GOTOFF)
27872 0 : return true;
27873 :
27874 : return false;
27875 : }
27876 :
27877 : /* Return true iff ADDR can be used as a base address. */
27878 :
27879 : static bool
27880 4590 : base_address_p (rtx addr)
27881 : {
27882 0 : if (REG_P (addr))
27883 : return true;
27884 :
27885 2857 : if (symbolic_base_address_p (addr))
27886 0 : return true;
27887 :
27888 : return false;
27889 : }
27890 :
27891 : /* If MEM is in the form of [(base+symbase)+offset], extract the three
27892 : parts of address and set to BASE, SYMBASE and OFFSET, otherwise
27893 : return false. */
27894 :
27895 : static bool
27896 2957 : extract_base_offset_in_addr (rtx mem, rtx *base, rtx *symbase, rtx *offset)
27897 : {
27898 2957 : rtx addr;
27899 :
27900 2957 : gcc_assert (MEM_P (mem));
27901 :
27902 2957 : addr = XEXP (mem, 0);
27903 :
27904 2957 : if (GET_CODE (addr) == CONST)
27905 10 : addr = XEXP (addr, 0);
27906 :
27907 2957 : if (base_address_p (addr))
27908 : {
27909 1324 : *base = addr;
27910 1324 : *symbase = const0_rtx;
27911 1324 : *offset = const0_rtx;
27912 1324 : return true;
27913 : }
27914 :
27915 1633 : if (GET_CODE (addr) == PLUS
27916 1633 : && base_address_p (XEXP (addr, 0)))
27917 : {
27918 433 : rtx addend = XEXP (addr, 1);
27919 :
27920 433 : if (GET_CODE (addend) == CONST)
27921 0 : addend = XEXP (addend, 0);
27922 :
27923 433 : if (CONST_INT_P (addend))
27924 : {
27925 279 : *base = XEXP (addr, 0);
27926 279 : *symbase = const0_rtx;
27927 279 : *offset = addend;
27928 279 : return true;
27929 : }
27930 :
27931 : /* Also accept REG + symbolic ref, with or without a CONST_INT
27932 : offset. */
27933 154 : if (REG_P (XEXP (addr, 0)))
27934 : {
27935 154 : if (symbolic_base_address_p (addend))
27936 : {
27937 0 : *base = XEXP (addr, 0);
27938 0 : *symbase = addend;
27939 0 : *offset = const0_rtx;
27940 0 : return true;
27941 : }
27942 :
27943 154 : if (GET_CODE (addend) == PLUS
27944 0 : && symbolic_base_address_p (XEXP (addend, 0))
27945 154 : && CONST_INT_P (XEXP (addend, 1)))
27946 : {
27947 0 : *base = XEXP (addr, 0);
27948 0 : *symbase = XEXP (addend, 0);
27949 0 : *offset = XEXP (addend, 1);
27950 0 : return true;
27951 : }
27952 : }
27953 : }
27954 :
27955 : return false;
27956 : }
27957 :
27958 : /* Given OPERANDS of consecutive load/store, check if we can merge
27959 : them into move multiple. LOAD is true if they are load instructions.
27960 : MODE is the mode of memory operands. */
27961 :
27962 : bool
27963 1626 : ix86_operands_ok_for_move_multiple (rtx *operands, bool load,
27964 : machine_mode mode)
27965 : {
27966 1626 : HOST_WIDE_INT offval_1, offval_2, msize;
27967 1626 : rtx mem_1, mem_2, reg_1, reg_2, base_1, base_2,
27968 : symbase_1, symbase_2, offset_1, offset_2;
27969 :
27970 1626 : if (load)
27971 : {
27972 1328 : mem_1 = operands[1];
27973 1328 : mem_2 = operands[3];
27974 1328 : reg_1 = operands[0];
27975 1328 : reg_2 = operands[2];
27976 : }
27977 : else
27978 : {
27979 298 : mem_1 = operands[0];
27980 298 : mem_2 = operands[2];
27981 298 : reg_1 = operands[1];
27982 298 : reg_2 = operands[3];
27983 : }
27984 :
27985 1626 : gcc_assert (REG_P (reg_1) && REG_P (reg_2));
27986 :
27987 1626 : if (REGNO (reg_1) != REGNO (reg_2))
27988 : return false;
27989 :
27990 : /* Check if the addresses are in the form of [base+offset]. */
27991 1622 : if (!extract_base_offset_in_addr (mem_1, &base_1, &symbase_1, &offset_1))
27992 : return false;
27993 1335 : if (!extract_base_offset_in_addr (mem_2, &base_2, &symbase_2, &offset_2))
27994 : return false;
27995 :
27996 : /* Check if the bases are the same. */
27997 268 : if (!rtx_equal_p (base_1, base_2) || !rtx_equal_p (symbase_1, symbase_2))
27998 121 : return false;
27999 :
28000 147 : offval_1 = INTVAL (offset_1);
28001 147 : offval_2 = INTVAL (offset_2);
28002 147 : msize = GET_MODE_SIZE (mode);
28003 : /* Check if mem_1 is adjacent to mem_2 and mem_1 has lower address. */
28004 147 : if (offval_1 + msize != offval_2)
28005 : return false;
28006 :
28007 : return true;
28008 : }
28009 :
28010 : /* Implement the TARGET_OPTAB_SUPPORTED_P hook. */
28011 :
28012 : static bool
28013 348661 : ix86_optab_supported_p (int op, machine_mode mode1, machine_mode,
28014 : optimization_type opt_type)
28015 : {
28016 348661 : switch (op)
28017 : {
28018 231 : case asin_optab:
28019 231 : case acos_optab:
28020 231 : case log1p_optab:
28021 231 : case exp_optab:
28022 231 : case exp10_optab:
28023 231 : case exp2_optab:
28024 231 : case expm1_optab:
28025 231 : case ldexp_optab:
28026 231 : case scalb_optab:
28027 231 : case round_optab:
28028 231 : case lround_optab:
28029 231 : return opt_type == OPTIMIZE_FOR_SPEED;
28030 :
28031 286 : case rint_optab:
28032 286 : if (SSE_FLOAT_MODE_P (mode1)
28033 139 : && TARGET_SSE_MATH
28034 127 : && !flag_trapping_math
28035 21 : && !TARGET_SSE4_1
28036 : && mode1 != HFmode)
28037 21 : return opt_type == OPTIMIZE_FOR_SPEED;
28038 : return true;
28039 :
28040 1971 : case floor_optab:
28041 1971 : case ceil_optab:
28042 1971 : case btrunc_optab:
28043 1971 : if ((SSE_FLOAT_MODE_P (mode1)
28044 1581 : && TARGET_SSE_MATH
28045 1514 : && TARGET_SSE4_1)
28046 1904 : || mode1 == HFmode)
28047 : return true;
28048 1835 : return opt_type == OPTIMIZE_FOR_SPEED;
28049 :
28050 66 : case rsqrt_optab:
28051 66 : return opt_type == OPTIMIZE_FOR_SPEED && use_rsqrt_p (mode1);
28052 :
28053 : default:
28054 : return true;
28055 : }
28056 : }
28057 :
28058 : /* Address space support.
28059 :
28060 : This is not "far pointers" in the 16-bit sense, but an easy way
28061 : to use %fs and %gs segment prefixes. Therefore:
28062 :
28063 : (a) All address spaces have the same modes,
28064 : (b) All address spaces have the same address forms,
28065 : (c) While %fs and %gs are technically subsets of the generic
28066 : address space, they are probably not subsets of each other.
28067 : (d) Since we have no access to the segment base register values
28068 : without resorting to a system call, we cannot convert a
28069 : non-default address space to a default address space.
28070 : Therefore we do not claim %fs or %gs are subsets of generic.
28071 :
28072 : Therefore we can (mostly) use the default hooks. */
28073 :
28074 : /* All use of segmentation is assumed to make address 0 valid. */
28075 :
28076 : static bool
28077 69380922 : ix86_addr_space_zero_address_valid (addr_space_t as)
28078 : {
28079 69380922 : return as != ADDR_SPACE_GENERIC;
28080 : }
28081 :
28082 : static void
28083 778684 : ix86_init_libfuncs (void)
28084 : {
28085 778684 : if (TARGET_64BIT)
28086 : {
28087 763814 : set_optab_libfunc (sdivmod_optab, TImode, "__divmodti4");
28088 763814 : set_optab_libfunc (udivmod_optab, TImode, "__udivmodti4");
28089 : }
28090 : else
28091 : {
28092 14870 : set_optab_libfunc (sdivmod_optab, DImode, "__divmoddi4");
28093 14870 : set_optab_libfunc (udivmod_optab, DImode, "__udivmoddi4");
28094 : }
28095 :
28096 : #if TARGET_MACHO
28097 : darwin_rename_builtins ();
28098 : #endif
28099 778684 : }
28100 :
28101 : /* Set the value of FLT_EVAL_METHOD in float.h. When using only the
28102 : FPU, assume that the fpcw is set to extended precision; when using
28103 : only SSE, rounding is correct; when using both SSE and the FPU,
28104 : the rounding precision is indeterminate, since either may be chosen
28105 : apparently at random. */
28106 :
28107 : static enum flt_eval_method
28108 91092438 : ix86_get_excess_precision (enum excess_precision_type type)
28109 : {
28110 91092438 : switch (type)
28111 : {
28112 86999065 : case EXCESS_PRECISION_TYPE_FAST:
28113 : /* The fastest type to promote to will always be the native type,
28114 : whether that occurs with implicit excess precision or
28115 : otherwise. */
28116 86999065 : return TARGET_AVX512FP16
28117 86999065 : ? FLT_EVAL_METHOD_PROMOTE_TO_FLOAT16
28118 86999065 : : FLT_EVAL_METHOD_PROMOTE_TO_FLOAT;
28119 4092629 : case EXCESS_PRECISION_TYPE_STANDARD:
28120 4092629 : case EXCESS_PRECISION_TYPE_IMPLICIT:
28121 : /* Otherwise, the excess precision we want when we are
28122 : in a standards compliant mode, and the implicit precision we
28123 : provide would be identical were it not for the unpredictable
28124 : cases. */
28125 4092629 : if (TARGET_AVX512FP16 && TARGET_SSE_MATH)
28126 : return FLT_EVAL_METHOD_PROMOTE_TO_FLOAT16;
28127 4086851 : else if (!TARGET_80387)
28128 : return FLT_EVAL_METHOD_PROMOTE_TO_FLOAT;
28129 4080719 : else if (!TARGET_MIX_SSE_I387)
28130 : {
28131 4080547 : if (!(TARGET_SSE && TARGET_SSE_MATH))
28132 : return FLT_EVAL_METHOD_PROMOTE_TO_LONG_DOUBLE;
28133 3091915 : else if (TARGET_SSE2)
28134 : return FLT_EVAL_METHOD_PROMOTE_TO_FLOAT;
28135 : }
28136 :
28137 : /* If we are in standards compliant mode, but we know we will
28138 : calculate in unpredictable precision, return
28139 : FLT_EVAL_METHOD_FLOAT. There is no reason to introduce explicit
28140 : excess precision if the target can't guarantee it will honor
28141 : it. */
28142 320 : return (type == EXCESS_PRECISION_TYPE_STANDARD
28143 320 : ? FLT_EVAL_METHOD_PROMOTE_TO_FLOAT
28144 : : FLT_EVAL_METHOD_UNPREDICTABLE);
28145 744 : case EXCESS_PRECISION_TYPE_FLOAT16:
28146 744 : if (TARGET_80387
28147 738 : && !(TARGET_SSE_MATH && TARGET_SSE))
28148 4 : error ("%<-fexcess-precision=16%> is not compatible with %<-mfpmath=387%>");
28149 : return FLT_EVAL_METHOD_PROMOTE_TO_FLOAT16;
28150 0 : default:
28151 0 : gcc_unreachable ();
28152 : }
28153 :
28154 : return FLT_EVAL_METHOD_UNPREDICTABLE;
28155 : }
28156 :
28157 : /* Return true if _BitInt(N) is supported and fill its details into *INFO. */
28158 : bool
28159 362558 : ix86_bitint_type_info (int n, struct bitint_info *info)
28160 : {
28161 362558 : if (n <= 8)
28162 11256 : info->limb_mode = QImode;
28163 351302 : else if (n <= 16)
28164 2558 : info->limb_mode = HImode;
28165 348744 : else if (n <= 32 || (!TARGET_64BIT && n > 64))
28166 47257 : info->limb_mode = SImode;
28167 : else
28168 301487 : info->limb_mode = DImode;
28169 362558 : info->abi_limb_mode = info->limb_mode;
28170 362558 : info->big_endian = false;
28171 362558 : info->extended = bitint_ext_undef;
28172 362558 : return true;
28173 : }
28174 :
28175 : /* Implement TARGET_C_MODE_FOR_FLOATING_TYPE. Return DFmode, TFmode
28176 : or XFmode for TI_LONG_DOUBLE_TYPE which is for long double type,
28177 : based on long double bits, go with the default one for the others. */
28178 :
28179 : static machine_mode
28180 3722960 : ix86_c_mode_for_floating_type (enum tree_index ti)
28181 : {
28182 3722960 : if (ti == TI_LONG_DOUBLE_TYPE)
28183 620986 : return (TARGET_LONG_DOUBLE_64 ? DFmode
28184 620954 : : (TARGET_LONG_DOUBLE_128 ? TFmode : XFmode));
28185 3101974 : return default_mode_for_floating_type (ti);
28186 : }
28187 :
28188 : /* Returns modified FUNCTION_TYPE for cdtor callabi. */
28189 : tree
28190 14713 : ix86_cxx_adjust_cdtor_callabi_fntype (tree fntype)
28191 : {
28192 14713 : if (TARGET_64BIT
28193 71 : || TARGET_RTD
28194 14784 : || ix86_function_type_abi (fntype) != MS_ABI)
28195 14713 : return fntype;
28196 : /* For 32-bit MS ABI add thiscall attribute. */
28197 0 : tree attribs = tree_cons (get_identifier ("thiscall"), NULL_TREE,
28198 0 : TYPE_ATTRIBUTES (fntype));
28199 0 : return build_type_attribute_variant (fntype, attribs);
28200 : }
28201 :
28202 : /* Implement PUSH_ROUNDING. On 386, we have pushw instruction that
28203 : decrements by exactly 2 no matter what the position was, there is no pushb.
28204 :
28205 : But as CIE data alignment factor on this arch is -4 for 32bit targets
28206 : and -8 for 64bit targets, we need to make sure all stack pointer adjustments
28207 : are in multiple of 4 for 32bit targets and 8 for 64bit targets. */
28208 :
28209 : poly_int64
28210 263040658 : ix86_push_rounding (poly_int64 bytes)
28211 : {
28212 338597931 : return ROUND_UP (bytes, UNITS_PER_WORD);
28213 : }
28214 :
28215 : /* Use 8 bits metadata start from bit48 for LAM_U48,
28216 : 6 bits metadata start from bit57 for LAM_U57. */
28217 : #define IX86_HWASAN_SHIFT (ix86_lam_type == lam_u48 \
28218 : ? 48 \
28219 : : (ix86_lam_type == lam_u57 ? 57 : 0))
28220 : #define IX86_HWASAN_TAG_SIZE (ix86_lam_type == lam_u48 \
28221 : ? 8 \
28222 : : (ix86_lam_type == lam_u57 ? 6 : 0))
28223 :
28224 : /* Implement TARGET_MEMTAG_CAN_TAG_ADDRESSES. */
28225 : bool
28226 6303938 : ix86_memtag_can_tag_addresses ()
28227 : {
28228 6303938 : return ix86_lam_type != lam_none && TARGET_LP64;
28229 : }
28230 :
28231 : /* Implement TARGET_MEMTAG_TAG_BITSIZE. */
28232 : unsigned char
28233 450 : ix86_memtag_tag_bitsize ()
28234 : {
28235 450 : return IX86_HWASAN_TAG_SIZE;
28236 : }
28237 :
28238 : /* Implement TARGET_MEMTAG_SET_TAG. */
28239 : rtx
28240 106 : ix86_memtag_set_tag (rtx untagged, rtx tag, rtx target)
28241 : {
28242 : /* default_memtag_insert_random_tag may
28243 : generate tag with value more than 6 bits. */
28244 106 : if (ix86_lam_type == lam_u57)
28245 : {
28246 106 : unsigned HOST_WIDE_INT and_imm
28247 : = (HOST_WIDE_INT_1U << IX86_HWASAN_TAG_SIZE) - 1;
28248 :
28249 106 : emit_insn (gen_andqi3 (tag, tag, GEN_INT (and_imm)));
28250 : }
28251 106 : tag = expand_simple_binop (Pmode, ASHIFT, tag,
28252 106 : GEN_INT (IX86_HWASAN_SHIFT), NULL_RTX,
28253 : /* unsignedp = */1, OPTAB_WIDEN);
28254 106 : rtx ret = expand_simple_binop (Pmode, IOR, untagged, tag, target,
28255 : /* unsignedp = */1, OPTAB_DIRECT);
28256 106 : return ret;
28257 : }
28258 :
28259 : /* Implement TARGET_MEMTAG_EXTRACT_TAG. */
28260 : rtx
28261 180 : ix86_memtag_extract_tag (rtx tagged_pointer, rtx target)
28262 : {
28263 180 : rtx tag = expand_simple_binop (Pmode, LSHIFTRT, tagged_pointer,
28264 180 : GEN_INT (IX86_HWASAN_SHIFT), target,
28265 : /* unsignedp = */0,
28266 : OPTAB_DIRECT);
28267 180 : rtx ret = gen_reg_rtx (QImode);
28268 : /* Mask off bit63 when LAM_U57. */
28269 180 : if (ix86_lam_type == lam_u57)
28270 : {
28271 180 : unsigned HOST_WIDE_INT and_imm
28272 : = (HOST_WIDE_INT_1U << IX86_HWASAN_TAG_SIZE) - 1;
28273 180 : emit_insn (gen_andqi3 (ret, gen_lowpart (QImode, tag),
28274 180 : gen_int_mode (and_imm, QImode)));
28275 : }
28276 : else
28277 0 : emit_move_insn (ret, gen_lowpart (QImode, tag));
28278 180 : return ret;
28279 : }
28280 :
28281 : /* The default implementation of TARGET_MEMTAG_UNTAGGED_POINTER. */
28282 : rtx
28283 114 : ix86_memtag_untagged_pointer (rtx tagged_pointer, rtx target)
28284 : {
28285 : /* Leave bit63 alone. */
28286 114 : rtx tag_mask = gen_int_mode (((HOST_WIDE_INT_1U << IX86_HWASAN_SHIFT)
28287 114 : + (HOST_WIDE_INT_1U << 63) - 1),
28288 114 : Pmode);
28289 114 : rtx untagged_base = expand_simple_binop (Pmode, AND, tagged_pointer,
28290 : tag_mask, target, true,
28291 : OPTAB_DIRECT);
28292 114 : gcc_assert (untagged_base);
28293 114 : return untagged_base;
28294 : }
28295 :
28296 : /* Implement TARGET_MEMTAG_ADD_TAG. */
28297 : rtx
28298 90 : ix86_memtag_add_tag (rtx base, poly_int64 offset, unsigned char tag_offset)
28299 : {
28300 90 : rtx base_tag = gen_reg_rtx (QImode);
28301 90 : rtx base_addr = gen_reg_rtx (Pmode);
28302 90 : rtx tagged_addr = gen_reg_rtx (Pmode);
28303 90 : rtx new_tag = gen_reg_rtx (QImode);
28304 180 : unsigned HOST_WIDE_INT and_imm
28305 90 : = (HOST_WIDE_INT_1U << IX86_HWASAN_SHIFT) - 1;
28306 :
28307 : /* When there's "overflow" in tag adding,
28308 : need to mask the most significant bit off. */
28309 90 : emit_move_insn (base_tag, ix86_memtag_extract_tag (base, NULL_RTX));
28310 90 : emit_move_insn (base_addr,
28311 : ix86_memtag_untagged_pointer (base, NULL_RTX));
28312 90 : emit_insn (gen_add2_insn (base_tag, gen_int_mode (tag_offset, QImode)));
28313 90 : emit_move_insn (new_tag, base_tag);
28314 90 : emit_insn (gen_andqi3 (new_tag, new_tag, gen_int_mode (and_imm, QImode)));
28315 90 : emit_move_insn (tagged_addr,
28316 : ix86_memtag_set_tag (base_addr, new_tag, NULL_RTX));
28317 90 : return plus_constant (Pmode, tagged_addr, offset);
28318 : }
28319 :
28320 : /* Implement TARGET_HAVE_CCMP. */
28321 : static bool
28322 8293988 : ix86_have_ccmp ()
28323 : {
28324 8293988 : return (bool) TARGET_APX_CCMP;
28325 : }
28326 :
28327 : /* Implement TARGET_MODE_CAN_TRANSFER_BITS. */
28328 : static bool
28329 4774932 : ix86_mode_can_transfer_bits (machine_mode mode)
28330 : {
28331 4774932 : if (GET_MODE_CLASS (mode) == MODE_FLOAT
28332 4723634 : || GET_MODE_CLASS (mode) == MODE_COMPLEX_FLOAT)
28333 124162 : switch (GET_MODE_INNER (mode))
28334 : {
28335 56411 : case E_SFmode:
28336 56411 : case E_DFmode:
28337 : /* These suffer from normalization upon load when not using SSE. */
28338 56411 : return !(ix86_fpmath & FPMATH_387);
28339 : default:
28340 : return true;
28341 : }
28342 :
28343 : return true;
28344 : }
28345 :
28346 : /* Implement TARGET_REDZONE_CLOBBER. */
28347 : static rtx
28348 2 : ix86_redzone_clobber ()
28349 : {
28350 2 : cfun->machine->asm_redzone_clobber_seen = true;
28351 2 : if (ix86_using_red_zone ())
28352 : {
28353 2 : rtx base = plus_constant (Pmode, stack_pointer_rtx, -RED_ZONE_SIZE);
28354 2 : rtx mem = gen_rtx_MEM (BLKmode, base);
28355 2 : set_mem_size (mem, RED_ZONE_SIZE);
28356 2 : return mem;
28357 : }
28358 : return NULL_RTX;
28359 : }
28360 :
28361 : /* Target-specific selftests. */
28362 :
28363 : #if CHECKING_P
28364 :
28365 : namespace selftest {
28366 :
28367 : /* Verify that hard regs are dumped as expected (in compact mode). */
28368 :
28369 : static void
28370 4 : ix86_test_dumping_hard_regs ()
28371 : {
28372 4 : ASSERT_RTL_DUMP_EQ ("(reg:SI ax)", gen_raw_REG (SImode, 0));
28373 4 : ASSERT_RTL_DUMP_EQ ("(reg:SI dx)", gen_raw_REG (SImode, 1));
28374 4 : }
28375 :
28376 : /* Test dumping an insn with repeated references to the same SCRATCH,
28377 : to verify the rtx_reuse code. */
28378 :
28379 : static void
28380 4 : ix86_test_dumping_memory_blockage ()
28381 : {
28382 4 : set_new_first_and_last_insn (NULL, NULL);
28383 :
28384 4 : rtx pat = gen_memory_blockage ();
28385 4 : rtx_reuse_manager r;
28386 4 : r.preprocess (pat);
28387 :
28388 : /* Verify that the repeated references to the SCRATCH show use
28389 : reuse IDS. The first should be prefixed with a reuse ID,
28390 : and the second should be dumped as a "reuse_rtx" of that ID.
28391 : The expected string assumes Pmode == DImode. */
28392 4 : if (Pmode == DImode)
28393 4 : ASSERT_RTL_DUMP_EQ_WITH_REUSE
28394 : ("(cinsn 1 (set (mem/v:BLK (0|scratch:DI) [0 A8])\n"
28395 : " (unspec:BLK [\n"
28396 : " (mem/v:BLK (reuse_rtx 0) [0 A8])\n"
28397 : " ] UNSPEC_MEMORY_BLOCKAGE)))\n", pat, &r);
28398 4 : }
28399 :
28400 : /* Verify loading an RTL dump; specifically a dump of copying
28401 : a param on x86_64 from a hard reg into the frame.
28402 : This test is target-specific since the dump contains target-specific
28403 : hard reg names. */
28404 :
28405 : static void
28406 4 : ix86_test_loading_dump_fragment_1 ()
28407 : {
28408 4 : rtl_dump_test t (SELFTEST_LOCATION,
28409 4 : locate_file ("x86_64/copy-hard-reg-into-frame.rtl"));
28410 :
28411 4 : rtx_insn *insn = get_insn_by_uid (1);
28412 :
28413 : /* The block structure and indentation here is purely for
28414 : readability; it mirrors the structure of the rtx. */
28415 4 : tree mem_expr;
28416 4 : {
28417 4 : rtx pat = PATTERN (insn);
28418 4 : ASSERT_EQ (SET, GET_CODE (pat));
28419 4 : {
28420 4 : rtx dest = SET_DEST (pat);
28421 4 : ASSERT_EQ (MEM, GET_CODE (dest));
28422 : /* Verify the "/c" was parsed. */
28423 4 : ASSERT_TRUE (RTX_FLAG (dest, call));
28424 4 : ASSERT_EQ (SImode, GET_MODE (dest));
28425 4 : {
28426 4 : rtx addr = XEXP (dest, 0);
28427 4 : ASSERT_EQ (PLUS, GET_CODE (addr));
28428 4 : ASSERT_EQ (DImode, GET_MODE (addr));
28429 4 : {
28430 4 : rtx lhs = XEXP (addr, 0);
28431 : /* Verify that the "frame" REG was consolidated. */
28432 4 : ASSERT_RTX_PTR_EQ (frame_pointer_rtx, lhs);
28433 : }
28434 4 : {
28435 4 : rtx rhs = XEXP (addr, 1);
28436 4 : ASSERT_EQ (CONST_INT, GET_CODE (rhs));
28437 4 : ASSERT_EQ (-4, INTVAL (rhs));
28438 : }
28439 : }
28440 : /* Verify the "[1 i+0 S4 A32]" was parsed. */
28441 4 : ASSERT_EQ (1, MEM_ALIAS_SET (dest));
28442 : /* "i" should have been handled by synthesizing a global int
28443 : variable named "i". */
28444 4 : mem_expr = MEM_EXPR (dest);
28445 4 : ASSERT_NE (mem_expr, NULL);
28446 4 : ASSERT_EQ (VAR_DECL, TREE_CODE (mem_expr));
28447 4 : ASSERT_EQ (integer_type_node, TREE_TYPE (mem_expr));
28448 4 : ASSERT_EQ (IDENTIFIER_NODE, TREE_CODE (DECL_NAME (mem_expr)));
28449 4 : ASSERT_STREQ ("i", IDENTIFIER_POINTER (DECL_NAME (mem_expr)));
28450 : /* "+0". */
28451 4 : ASSERT_TRUE (MEM_OFFSET_KNOWN_P (dest));
28452 4 : ASSERT_EQ (0, MEM_OFFSET (dest));
28453 : /* "S4". */
28454 4 : ASSERT_EQ (4, MEM_SIZE (dest));
28455 : /* "A32. */
28456 4 : ASSERT_EQ (32, MEM_ALIGN (dest));
28457 : }
28458 4 : {
28459 4 : rtx src = SET_SRC (pat);
28460 4 : ASSERT_EQ (REG, GET_CODE (src));
28461 4 : ASSERT_EQ (SImode, GET_MODE (src));
28462 4 : ASSERT_EQ (5, REGNO (src));
28463 4 : tree reg_expr = REG_EXPR (src);
28464 : /* "i" here should point to the same var as for the MEM_EXPR. */
28465 4 : ASSERT_EQ (reg_expr, mem_expr);
28466 : }
28467 : }
28468 4 : }
28469 :
28470 : /* Verify that the RTL loader copes with a call_insn dump.
28471 : This test is target-specific since the dump contains a target-specific
28472 : hard reg name. */
28473 :
28474 : static void
28475 4 : ix86_test_loading_call_insn ()
28476 : {
28477 : /* The test dump includes register "xmm0", where requires TARGET_SSE
28478 : to exist. */
28479 4 : if (!TARGET_SSE)
28480 0 : return;
28481 :
28482 4 : rtl_dump_test t (SELFTEST_LOCATION, locate_file ("x86_64/call-insn.rtl"));
28483 :
28484 4 : rtx_insn *insn = get_insns ();
28485 4 : ASSERT_EQ (CALL_INSN, GET_CODE (insn));
28486 :
28487 : /* "/j". */
28488 4 : ASSERT_TRUE (RTX_FLAG (insn, jump));
28489 :
28490 4 : rtx pat = PATTERN (insn);
28491 4 : ASSERT_EQ (CALL, GET_CODE (SET_SRC (pat)));
28492 :
28493 : /* Verify REG_NOTES. */
28494 4 : {
28495 : /* "(expr_list:REG_CALL_DECL". */
28496 4 : ASSERT_EQ (EXPR_LIST, GET_CODE (REG_NOTES (insn)));
28497 4 : rtx_expr_list *note0 = as_a <rtx_expr_list *> (REG_NOTES (insn));
28498 4 : ASSERT_EQ (REG_CALL_DECL, REG_NOTE_KIND (note0));
28499 :
28500 : /* "(expr_list:REG_EH_REGION (const_int 0 [0])". */
28501 4 : rtx_expr_list *note1 = note0->next ();
28502 4 : ASSERT_EQ (REG_EH_REGION, REG_NOTE_KIND (note1));
28503 :
28504 4 : ASSERT_EQ (NULL, note1->next ());
28505 : }
28506 :
28507 : /* Verify CALL_INSN_FUNCTION_USAGE. */
28508 4 : {
28509 : /* "(expr_list:DF (use (reg:DF 21 xmm0))". */
28510 4 : rtx_expr_list *usage
28511 4 : = as_a <rtx_expr_list *> (CALL_INSN_FUNCTION_USAGE (insn));
28512 4 : ASSERT_EQ (EXPR_LIST, GET_CODE (usage));
28513 4 : ASSERT_EQ (DFmode, GET_MODE (usage));
28514 4 : ASSERT_EQ (USE, GET_CODE (usage->element ()));
28515 4 : ASSERT_EQ (NULL, usage->next ());
28516 : }
28517 4 : }
28518 :
28519 : /* Verify that the RTL loader copes a dump from print_rtx_function.
28520 : This test is target-specific since the dump contains target-specific
28521 : hard reg names. */
28522 :
28523 : static void
28524 4 : ix86_test_loading_full_dump ()
28525 : {
28526 4 : rtl_dump_test t (SELFTEST_LOCATION, locate_file ("x86_64/times-two.rtl"));
28527 :
28528 4 : ASSERT_STREQ ("times_two", IDENTIFIER_POINTER (DECL_NAME (cfun->decl)));
28529 :
28530 4 : rtx_insn *insn_1 = get_insn_by_uid (1);
28531 4 : ASSERT_EQ (NOTE, GET_CODE (insn_1));
28532 :
28533 4 : rtx_insn *insn_7 = get_insn_by_uid (7);
28534 4 : ASSERT_EQ (INSN, GET_CODE (insn_7));
28535 4 : ASSERT_EQ (PARALLEL, GET_CODE (PATTERN (insn_7)));
28536 :
28537 4 : rtx_insn *insn_15 = get_insn_by_uid (15);
28538 4 : ASSERT_EQ (INSN, GET_CODE (insn_15));
28539 4 : ASSERT_EQ (USE, GET_CODE (PATTERN (insn_15)));
28540 :
28541 : /* Verify crtl->return_rtx. */
28542 4 : ASSERT_EQ (REG, GET_CODE (crtl->return_rtx));
28543 4 : ASSERT_EQ (0, REGNO (crtl->return_rtx));
28544 4 : ASSERT_EQ (SImode, GET_MODE (crtl->return_rtx));
28545 4 : }
28546 :
28547 : /* Verify that the RTL loader copes with UNSPEC and UNSPEC_VOLATILE insns.
28548 : In particular, verify that it correctly loads the 2nd operand.
28549 : This test is target-specific since these are machine-specific
28550 : operands (and enums). */
28551 :
28552 : static void
28553 4 : ix86_test_loading_unspec ()
28554 : {
28555 4 : rtl_dump_test t (SELFTEST_LOCATION, locate_file ("x86_64/unspec.rtl"));
28556 :
28557 4 : ASSERT_STREQ ("test_unspec", IDENTIFIER_POINTER (DECL_NAME (cfun->decl)));
28558 :
28559 4 : ASSERT_TRUE (cfun);
28560 :
28561 : /* Test of an UNSPEC. */
28562 4 : rtx_insn *insn = get_insns ();
28563 4 : ASSERT_EQ (INSN, GET_CODE (insn));
28564 4 : rtx set = single_set (insn);
28565 4 : ASSERT_NE (NULL, set);
28566 4 : rtx dst = SET_DEST (set);
28567 4 : ASSERT_EQ (MEM, GET_CODE (dst));
28568 4 : rtx src = SET_SRC (set);
28569 4 : ASSERT_EQ (UNSPEC, GET_CODE (src));
28570 4 : ASSERT_EQ (BLKmode, GET_MODE (src));
28571 4 : ASSERT_EQ (UNSPEC_MEMORY_BLOCKAGE, XINT (src, 1));
28572 :
28573 4 : rtx v0 = XVECEXP (src, 0, 0);
28574 :
28575 : /* Verify that the two uses of the first SCRATCH have pointer
28576 : equality. */
28577 4 : rtx scratch_a = XEXP (dst, 0);
28578 4 : ASSERT_EQ (SCRATCH, GET_CODE (scratch_a));
28579 :
28580 4 : rtx scratch_b = XEXP (v0, 0);
28581 4 : ASSERT_EQ (SCRATCH, GET_CODE (scratch_b));
28582 :
28583 4 : ASSERT_EQ (scratch_a, scratch_b);
28584 :
28585 : /* Verify that the two mems are thus treated as equal. */
28586 4 : ASSERT_TRUE (rtx_equal_p (dst, v0));
28587 :
28588 : /* Verify that the insn is recognized. */
28589 4 : ASSERT_NE(-1, recog_memoized (insn));
28590 :
28591 : /* Test of an UNSPEC_VOLATILE, which has its own enum values. */
28592 4 : insn = NEXT_INSN (insn);
28593 4 : ASSERT_EQ (INSN, GET_CODE (insn));
28594 :
28595 4 : set = single_set (insn);
28596 4 : ASSERT_NE (NULL, set);
28597 :
28598 4 : src = SET_SRC (set);
28599 4 : ASSERT_EQ (UNSPEC_VOLATILE, GET_CODE (src));
28600 4 : ASSERT_EQ (UNSPECV_RDTSCP, XINT (src, 1));
28601 4 : }
28602 :
28603 : /* Run all target-specific selftests. */
28604 :
28605 : static void
28606 4 : ix86_run_selftests (void)
28607 : {
28608 4 : ix86_test_dumping_hard_regs ();
28609 4 : ix86_test_dumping_memory_blockage ();
28610 :
28611 : /* Various tests of loading RTL dumps, here because they contain
28612 : ix86-isms (e.g. names of hard regs). */
28613 4 : ix86_test_loading_dump_fragment_1 ();
28614 4 : ix86_test_loading_call_insn ();
28615 4 : ix86_test_loading_full_dump ();
28616 4 : ix86_test_loading_unspec ();
28617 4 : }
28618 :
28619 : } // namespace selftest
28620 :
28621 : #endif /* CHECKING_P */
28622 :
28623 : static const scoped_attribute_specs *const ix86_attribute_table[] =
28624 : {
28625 : &ix86_gnu_attribute_table
28626 : };
28627 :
28628 : /* Initialize the GCC target structure. */
28629 : #undef TARGET_RETURN_IN_MEMORY
28630 : #define TARGET_RETURN_IN_MEMORY ix86_return_in_memory
28631 :
28632 : #undef TARGET_LEGITIMIZE_ADDRESS
28633 : #define TARGET_LEGITIMIZE_ADDRESS ix86_legitimize_address
28634 :
28635 : #undef TARGET_ATTRIBUTE_TABLE
28636 : #define TARGET_ATTRIBUTE_TABLE ix86_attribute_table
28637 : #undef TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P
28638 : #define TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P hook_bool_const_tree_true
28639 : #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
28640 : # undef TARGET_MERGE_DECL_ATTRIBUTES
28641 : # define TARGET_MERGE_DECL_ATTRIBUTES merge_dllimport_decl_attributes
28642 : #endif
28643 :
28644 : #undef TARGET_INVALID_CONVERSION
28645 : #define TARGET_INVALID_CONVERSION ix86_invalid_conversion
28646 :
28647 : #undef TARGET_INVALID_UNARY_OP
28648 : #define TARGET_INVALID_UNARY_OP ix86_invalid_unary_op
28649 :
28650 : #undef TARGET_INVALID_BINARY_OP
28651 : #define TARGET_INVALID_BINARY_OP ix86_invalid_binary_op
28652 :
28653 : #undef TARGET_COMP_TYPE_ATTRIBUTES
28654 : #define TARGET_COMP_TYPE_ATTRIBUTES ix86_comp_type_attributes
28655 :
28656 : #undef TARGET_INIT_BUILTINS
28657 : #define TARGET_INIT_BUILTINS ix86_init_builtins
28658 : #undef TARGET_BUILTIN_DECL
28659 : #define TARGET_BUILTIN_DECL ix86_builtin_decl
28660 : #undef TARGET_EXPAND_BUILTIN
28661 : #define TARGET_EXPAND_BUILTIN ix86_expand_builtin
28662 :
28663 : #undef TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION
28664 : #define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
28665 : ix86_builtin_vectorized_function
28666 :
28667 : #undef TARGET_VECTORIZE_BUILTIN_GATHER
28668 : #define TARGET_VECTORIZE_BUILTIN_GATHER ix86_vectorize_builtin_gather
28669 :
28670 : #undef TARGET_VECTORIZE_BUILTIN_SCATTER
28671 : #define TARGET_VECTORIZE_BUILTIN_SCATTER ix86_vectorize_builtin_scatter
28672 :
28673 : #undef TARGET_BUILTIN_RECIPROCAL
28674 : #define TARGET_BUILTIN_RECIPROCAL ix86_builtin_reciprocal
28675 :
28676 : #undef TARGET_ASM_FUNCTION_EPILOGUE
28677 : #define TARGET_ASM_FUNCTION_EPILOGUE ix86_output_function_epilogue
28678 :
28679 : #undef TARGET_ASM_PRINT_PATCHABLE_FUNCTION_ENTRY
28680 : #define TARGET_ASM_PRINT_PATCHABLE_FUNCTION_ENTRY \
28681 : ix86_print_patchable_function_entry
28682 :
28683 : #undef TARGET_ENCODE_SECTION_INFO
28684 : #ifndef SUBTARGET_ENCODE_SECTION_INFO
28685 : #define TARGET_ENCODE_SECTION_INFO ix86_encode_section_info
28686 : #else
28687 : #define TARGET_ENCODE_SECTION_INFO SUBTARGET_ENCODE_SECTION_INFO
28688 : #endif
28689 :
28690 : #undef TARGET_ASM_OPEN_PAREN
28691 : #define TARGET_ASM_OPEN_PAREN ""
28692 : #undef TARGET_ASM_CLOSE_PAREN
28693 : #define TARGET_ASM_CLOSE_PAREN ""
28694 :
28695 : #undef TARGET_ASM_BYTE_OP
28696 : #define TARGET_ASM_BYTE_OP ASM_BYTE
28697 :
28698 : #undef TARGET_ASM_ALIGNED_HI_OP
28699 : #define TARGET_ASM_ALIGNED_HI_OP ASM_SHORT
28700 : #undef TARGET_ASM_ALIGNED_SI_OP
28701 : #define TARGET_ASM_ALIGNED_SI_OP ASM_LONG
28702 : #ifdef ASM_QUAD
28703 : #undef TARGET_ASM_ALIGNED_DI_OP
28704 : #define TARGET_ASM_ALIGNED_DI_OP ASM_QUAD
28705 : #endif
28706 :
28707 : #undef TARGET_PROFILE_BEFORE_PROLOGUE
28708 : #define TARGET_PROFILE_BEFORE_PROLOGUE ix86_profile_before_prologue
28709 :
28710 : #undef TARGET_MANGLE_DECL_ASSEMBLER_NAME
28711 : #define TARGET_MANGLE_DECL_ASSEMBLER_NAME ix86_mangle_decl_assembler_name
28712 :
28713 : #undef TARGET_ASM_UNALIGNED_HI_OP
28714 : #define TARGET_ASM_UNALIGNED_HI_OP TARGET_ASM_ALIGNED_HI_OP
28715 : #undef TARGET_ASM_UNALIGNED_SI_OP
28716 : #define TARGET_ASM_UNALIGNED_SI_OP TARGET_ASM_ALIGNED_SI_OP
28717 : #undef TARGET_ASM_UNALIGNED_DI_OP
28718 : #define TARGET_ASM_UNALIGNED_DI_OP TARGET_ASM_ALIGNED_DI_OP
28719 :
28720 : #undef TARGET_PRINT_OPERAND
28721 : #define TARGET_PRINT_OPERAND ix86_print_operand
28722 : #undef TARGET_PRINT_OPERAND_ADDRESS
28723 : #define TARGET_PRINT_OPERAND_ADDRESS ix86_print_operand_address
28724 : #undef TARGET_PRINT_OPERAND_PUNCT_VALID_P
28725 : #define TARGET_PRINT_OPERAND_PUNCT_VALID_P ix86_print_operand_punct_valid_p
28726 : #undef TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA
28727 : #define TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA i386_asm_output_addr_const_extra
28728 :
28729 : #undef TARGET_SCHED_INIT_GLOBAL
28730 : #define TARGET_SCHED_INIT_GLOBAL ix86_sched_init_global
28731 : #undef TARGET_SCHED_ADJUST_COST
28732 : #define TARGET_SCHED_ADJUST_COST ix86_adjust_cost
28733 : #undef TARGET_SCHED_ISSUE_RATE
28734 : #define TARGET_SCHED_ISSUE_RATE ix86_issue_rate
28735 : #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
28736 : #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
28737 : ia32_multipass_dfa_lookahead
28738 : #undef TARGET_SCHED_MACRO_FUSION_P
28739 : #define TARGET_SCHED_MACRO_FUSION_P ix86_macro_fusion_p
28740 : #undef TARGET_SCHED_MACRO_FUSION_PAIR_P
28741 : #define TARGET_SCHED_MACRO_FUSION_PAIR_P ix86_macro_fusion_pair_p
28742 :
28743 : #undef TARGET_FUNCTION_OK_FOR_SIBCALL
28744 : #define TARGET_FUNCTION_OK_FOR_SIBCALL ix86_function_ok_for_sibcall
28745 :
28746 : #undef TARGET_MEMMODEL_CHECK
28747 : #define TARGET_MEMMODEL_CHECK ix86_memmodel_check
28748 :
28749 : #undef TARGET_ATOMIC_ASSIGN_EXPAND_FENV
28750 : #define TARGET_ATOMIC_ASSIGN_EXPAND_FENV ix86_atomic_assign_expand_fenv
28751 :
28752 : #ifdef HAVE_AS_TLS
28753 : #undef TARGET_HAVE_TLS
28754 : #define TARGET_HAVE_TLS true
28755 : #endif
28756 : #undef TARGET_CANNOT_FORCE_CONST_MEM
28757 : #define TARGET_CANNOT_FORCE_CONST_MEM ix86_cannot_force_const_mem
28758 : #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
28759 : #define TARGET_USE_BLOCKS_FOR_CONSTANT_P hook_bool_mode_const_rtx_true
28760 :
28761 : #undef TARGET_DELEGITIMIZE_ADDRESS
28762 : #define TARGET_DELEGITIMIZE_ADDRESS ix86_delegitimize_address
28763 :
28764 : #undef TARGET_CONST_NOT_OK_FOR_DEBUG_P
28765 : #define TARGET_CONST_NOT_OK_FOR_DEBUG_P ix86_const_not_ok_for_debug_p
28766 :
28767 : #undef TARGET_MS_BITFIELD_LAYOUT_P
28768 : #define TARGET_MS_BITFIELD_LAYOUT_P ix86_ms_bitfield_layout_p
28769 :
28770 : #if TARGET_MACHO
28771 : #undef TARGET_BINDS_LOCAL_P
28772 : #define TARGET_BINDS_LOCAL_P darwin_binds_local_p
28773 : #else
28774 : #undef TARGET_BINDS_LOCAL_P
28775 : #define TARGET_BINDS_LOCAL_P ix86_binds_local_p
28776 : #endif
28777 : #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
28778 : #undef TARGET_BINDS_LOCAL_P
28779 : #define TARGET_BINDS_LOCAL_P i386_pe_binds_local_p
28780 : #endif
28781 :
28782 : #undef TARGET_ASM_OUTPUT_MI_THUNK
28783 : #define TARGET_ASM_OUTPUT_MI_THUNK x86_output_mi_thunk
28784 : #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
28785 : #define TARGET_ASM_CAN_OUTPUT_MI_THUNK x86_can_output_mi_thunk
28786 :
28787 : #undef TARGET_ASM_FILE_START
28788 : #define TARGET_ASM_FILE_START x86_file_start
28789 :
28790 : #undef TARGET_OPTION_OVERRIDE
28791 : #define TARGET_OPTION_OVERRIDE ix86_option_override
28792 :
28793 : #undef TARGET_REGISTER_MOVE_COST
28794 : #define TARGET_REGISTER_MOVE_COST ix86_register_move_cost
28795 : #undef TARGET_MEMORY_MOVE_COST
28796 : #define TARGET_MEMORY_MOVE_COST ix86_memory_move_cost
28797 : #undef TARGET_RTX_COSTS
28798 : #define TARGET_RTX_COSTS ix86_rtx_costs
28799 : #undef TARGET_INSN_COST
28800 : #define TARGET_INSN_COST ix86_insn_cost
28801 : #undef TARGET_ADDRESS_COST
28802 : #define TARGET_ADDRESS_COST ix86_address_cost
28803 :
28804 : #undef TARGET_USE_BY_PIECES_INFRASTRUCTURE_P
28805 : #define TARGET_USE_BY_PIECES_INFRASTRUCTURE_P \
28806 : ix86_use_by_pieces_infrastructure_p
28807 :
28808 : #undef TARGET_OVERLAP_OP_BY_PIECES_P
28809 : #define TARGET_OVERLAP_OP_BY_PIECES_P hook_bool_void_true
28810 :
28811 : #undef TARGET_FLAGS_REGNUM
28812 : #define TARGET_FLAGS_REGNUM FLAGS_REG
28813 : #undef TARGET_FIXED_CONDITION_CODE_REGS
28814 : #define TARGET_FIXED_CONDITION_CODE_REGS ix86_fixed_condition_code_regs
28815 : #undef TARGET_CC_MODES_COMPATIBLE
28816 : #define TARGET_CC_MODES_COMPATIBLE ix86_cc_modes_compatible
28817 :
28818 : #undef TARGET_MACHINE_DEPENDENT_REORG
28819 : #define TARGET_MACHINE_DEPENDENT_REORG ix86_reorg
28820 :
28821 : #undef TARGET_BUILD_BUILTIN_VA_LIST
28822 : #define TARGET_BUILD_BUILTIN_VA_LIST ix86_build_builtin_va_list
28823 :
28824 : #undef TARGET_FOLD_BUILTIN
28825 : #define TARGET_FOLD_BUILTIN ix86_fold_builtin
28826 :
28827 : #undef TARGET_GIMPLE_FOLD_BUILTIN
28828 : #define TARGET_GIMPLE_FOLD_BUILTIN ix86_gimple_fold_builtin
28829 :
28830 : #undef TARGET_COMPARE_VERSION_PRIORITY
28831 : #define TARGET_COMPARE_VERSION_PRIORITY ix86_compare_version_priority
28832 :
28833 : #undef TARGET_GENERATE_VERSION_DISPATCHER_BODY
28834 : #define TARGET_GENERATE_VERSION_DISPATCHER_BODY \
28835 : ix86_generate_version_dispatcher_body
28836 :
28837 : #undef TARGET_GET_FUNCTION_VERSIONS_DISPATCHER
28838 : #define TARGET_GET_FUNCTION_VERSIONS_DISPATCHER \
28839 : ix86_get_function_versions_dispatcher
28840 :
28841 : #undef TARGET_ENUM_VA_LIST_P
28842 : #define TARGET_ENUM_VA_LIST_P ix86_enum_va_list
28843 :
28844 : #undef TARGET_FN_ABI_VA_LIST
28845 : #define TARGET_FN_ABI_VA_LIST ix86_fn_abi_va_list
28846 :
28847 : #undef TARGET_CANONICAL_VA_LIST_TYPE
28848 : #define TARGET_CANONICAL_VA_LIST_TYPE ix86_canonical_va_list_type
28849 :
28850 : #undef TARGET_EXPAND_BUILTIN_VA_START
28851 : #define TARGET_EXPAND_BUILTIN_VA_START ix86_va_start
28852 :
28853 : #undef TARGET_MD_ASM_ADJUST
28854 : #define TARGET_MD_ASM_ADJUST ix86_md_asm_adjust
28855 :
28856 : #undef TARGET_C_EXCESS_PRECISION
28857 : #define TARGET_C_EXCESS_PRECISION ix86_get_excess_precision
28858 : #undef TARGET_C_BITINT_TYPE_INFO
28859 : #define TARGET_C_BITINT_TYPE_INFO ix86_bitint_type_info
28860 : #undef TARGET_C_MODE_FOR_FLOATING_TYPE
28861 : #define TARGET_C_MODE_FOR_FLOATING_TYPE ix86_c_mode_for_floating_type
28862 : #undef TARGET_CXX_ADJUST_CDTOR_CALLABI_FNTYPE
28863 : #define TARGET_CXX_ADJUST_CDTOR_CALLABI_FNTYPE ix86_cxx_adjust_cdtor_callabi_fntype
28864 : #undef TARGET_PROMOTE_PROTOTYPES
28865 : #define TARGET_PROMOTE_PROTOTYPES hook_bool_const_tree_true
28866 : #undef TARGET_PUSH_ARGUMENT
28867 : #define TARGET_PUSH_ARGUMENT ix86_push_argument
28868 : #undef TARGET_SETUP_INCOMING_VARARGS
28869 : #define TARGET_SETUP_INCOMING_VARARGS ix86_setup_incoming_varargs
28870 : #undef TARGET_MUST_PASS_IN_STACK
28871 : #define TARGET_MUST_PASS_IN_STACK ix86_must_pass_in_stack
28872 : #undef TARGET_ALLOCATE_STACK_SLOTS_FOR_ARGS
28873 : #define TARGET_ALLOCATE_STACK_SLOTS_FOR_ARGS ix86_allocate_stack_slots_for_args
28874 : #undef TARGET_FUNCTION_ARG_ADVANCE
28875 : #define TARGET_FUNCTION_ARG_ADVANCE ix86_function_arg_advance
28876 : #undef TARGET_FUNCTION_ARG
28877 : #define TARGET_FUNCTION_ARG ix86_function_arg
28878 : #undef TARGET_INIT_PIC_REG
28879 : #define TARGET_INIT_PIC_REG ix86_init_pic_reg
28880 : #undef TARGET_USE_PSEUDO_PIC_REG
28881 : #define TARGET_USE_PSEUDO_PIC_REG ix86_use_pseudo_pic_reg
28882 : #undef TARGET_FUNCTION_ARG_BOUNDARY
28883 : #define TARGET_FUNCTION_ARG_BOUNDARY ix86_function_arg_boundary
28884 : #undef TARGET_PASS_BY_REFERENCE
28885 : #define TARGET_PASS_BY_REFERENCE ix86_pass_by_reference
28886 : #undef TARGET_INTERNAL_ARG_POINTER
28887 : #define TARGET_INTERNAL_ARG_POINTER ix86_internal_arg_pointer
28888 : #undef TARGET_UPDATE_STACK_BOUNDARY
28889 : #define TARGET_UPDATE_STACK_BOUNDARY ix86_update_stack_boundary
28890 : #undef TARGET_GET_DRAP_RTX
28891 : #define TARGET_GET_DRAP_RTX ix86_get_drap_rtx
28892 : #undef TARGET_STRICT_ARGUMENT_NAMING
28893 : #define TARGET_STRICT_ARGUMENT_NAMING hook_bool_CUMULATIVE_ARGS_true
28894 : #undef TARGET_STATIC_CHAIN
28895 : #define TARGET_STATIC_CHAIN ix86_static_chain
28896 : #undef TARGET_TRAMPOLINE_INIT
28897 : #define TARGET_TRAMPOLINE_INIT ix86_trampoline_init
28898 : #undef TARGET_RETURN_POPS_ARGS
28899 : #define TARGET_RETURN_POPS_ARGS ix86_return_pops_args
28900 :
28901 : #undef TARGET_WARN_FUNC_RETURN
28902 : #define TARGET_WARN_FUNC_RETURN ix86_warn_func_return
28903 :
28904 : #undef TARGET_LEGITIMATE_COMBINED_INSN
28905 : #define TARGET_LEGITIMATE_COMBINED_INSN ix86_legitimate_combined_insn
28906 :
28907 : #undef TARGET_ASAN_SHADOW_OFFSET
28908 : #define TARGET_ASAN_SHADOW_OFFSET ix86_asan_shadow_offset
28909 :
28910 : #undef TARGET_GIMPLIFY_VA_ARG_EXPR
28911 : #define TARGET_GIMPLIFY_VA_ARG_EXPR ix86_gimplify_va_arg
28912 :
28913 : #undef TARGET_SCALAR_MODE_SUPPORTED_P
28914 : #define TARGET_SCALAR_MODE_SUPPORTED_P ix86_scalar_mode_supported_p
28915 :
28916 : #undef TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P
28917 : #define TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P \
28918 : ix86_libgcc_floating_mode_supported_p
28919 :
28920 : #undef TARGET_VECTOR_MODE_SUPPORTED_P
28921 : #define TARGET_VECTOR_MODE_SUPPORTED_P ix86_vector_mode_supported_p
28922 :
28923 : #undef TARGET_C_MODE_FOR_SUFFIX
28924 : #define TARGET_C_MODE_FOR_SUFFIX ix86_c_mode_for_suffix
28925 :
28926 : #ifdef HAVE_AS_TLS
28927 : #undef TARGET_ASM_OUTPUT_DWARF_DTPREL
28928 : #define TARGET_ASM_OUTPUT_DWARF_DTPREL i386_output_dwarf_dtprel
28929 : #endif
28930 :
28931 : #ifdef SUBTARGET_INSERT_ATTRIBUTES
28932 : #undef TARGET_INSERT_ATTRIBUTES
28933 : #define TARGET_INSERT_ATTRIBUTES SUBTARGET_INSERT_ATTRIBUTES
28934 : #endif
28935 :
28936 : #undef TARGET_MANGLE_TYPE
28937 : #define TARGET_MANGLE_TYPE ix86_mangle_type
28938 :
28939 : #undef TARGET_EMIT_SUPPORT_TINFOS
28940 : #define TARGET_EMIT_SUPPORT_TINFOS ix86_emit_support_tinfos
28941 :
28942 : #undef TARGET_STACK_PROTECT_GUARD
28943 : #define TARGET_STACK_PROTECT_GUARD ix86_stack_protect_guard
28944 :
28945 : #undef TARGET_STACK_PROTECT_GUARD_SYMBOL_P
28946 : #define TARGET_STACK_PROTECT_GUARD_SYMBOL_P \
28947 : ix86_stack_protect_guard_symbol_p
28948 :
28949 : #undef TARGET_STACK_PROTECT_RUNTIME_ENABLED_P
28950 : #define TARGET_STACK_PROTECT_RUNTIME_ENABLED_P \
28951 : ix86_stack_protect_runtime_enabled_p
28952 :
28953 : #if !TARGET_MACHO
28954 : #undef TARGET_STACK_PROTECT_FAIL
28955 : #define TARGET_STACK_PROTECT_FAIL ix86_stack_protect_fail
28956 : #endif
28957 :
28958 : #undef TARGET_FUNCTION_VALUE
28959 : #define TARGET_FUNCTION_VALUE ix86_function_value
28960 :
28961 : #undef TARGET_FUNCTION_VALUE_REGNO_P
28962 : #define TARGET_FUNCTION_VALUE_REGNO_P ix86_function_value_regno_p
28963 :
28964 : #undef TARGET_ZERO_CALL_USED_REGS
28965 : #define TARGET_ZERO_CALL_USED_REGS ix86_zero_call_used_regs
28966 :
28967 : #undef TARGET_PROMOTE_FUNCTION_MODE
28968 : #define TARGET_PROMOTE_FUNCTION_MODE ix86_promote_function_mode
28969 :
28970 : #undef TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE
28971 : #define TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE ix86_override_options_after_change
28972 :
28973 : #undef TARGET_MEMBER_TYPE_FORCES_BLK
28974 : #define TARGET_MEMBER_TYPE_FORCES_BLK ix86_member_type_forces_blk
28975 :
28976 : #undef TARGET_INSTANTIATE_DECLS
28977 : #define TARGET_INSTANTIATE_DECLS ix86_instantiate_decls
28978 :
28979 : #undef TARGET_SECONDARY_RELOAD
28980 : #define TARGET_SECONDARY_RELOAD ix86_secondary_reload
28981 : #undef TARGET_SECONDARY_MEMORY_NEEDED
28982 : #define TARGET_SECONDARY_MEMORY_NEEDED ix86_secondary_memory_needed
28983 : #undef TARGET_SECONDARY_MEMORY_NEEDED_MODE
28984 : #define TARGET_SECONDARY_MEMORY_NEEDED_MODE ix86_secondary_memory_needed_mode
28985 :
28986 : #undef TARGET_CLASS_MAX_NREGS
28987 : #define TARGET_CLASS_MAX_NREGS ix86_class_max_nregs
28988 :
28989 : #undef TARGET_PREFERRED_RELOAD_CLASS
28990 : #define TARGET_PREFERRED_RELOAD_CLASS ix86_preferred_reload_class
28991 : #undef TARGET_PREFERRED_OUTPUT_RELOAD_CLASS
28992 : #define TARGET_PREFERRED_OUTPUT_RELOAD_CLASS ix86_preferred_output_reload_class
28993 : /* When this hook returns true for MODE, the compiler allows
28994 : registers explicitly used in the rtl to be used as spill registers
28995 : but prevents the compiler from extending the lifetime of these
28996 : registers. */
28997 : #undef TARGET_SMALL_REGISTER_CLASSES_FOR_MODE_P
28998 : #define TARGET_SMALL_REGISTER_CLASSES_FOR_MODE_P hook_bool_mode_true
28999 : #undef TARGET_CLASS_LIKELY_SPILLED_P
29000 : #define TARGET_CLASS_LIKELY_SPILLED_P ix86_class_likely_spilled_p
29001 : #undef TARGET_CALLEE_SAVE_COST
29002 : #define TARGET_CALLEE_SAVE_COST ix86_callee_save_cost
29003 :
29004 : #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
29005 : #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
29006 : ix86_builtin_vectorization_cost
29007 : #undef TARGET_VECTORIZE_VEC_PERM_CONST
29008 : #define TARGET_VECTORIZE_VEC_PERM_CONST ix86_vectorize_vec_perm_const
29009 : #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
29010 : #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE \
29011 : ix86_preferred_simd_mode
29012 : #undef TARGET_VECTORIZE_SPLIT_REDUCTION
29013 : #define TARGET_VECTORIZE_SPLIT_REDUCTION \
29014 : ix86_split_reduction
29015 : #undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_MODES
29016 : #define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_MODES \
29017 : ix86_autovectorize_vector_modes
29018 : #undef TARGET_VECTORIZE_GET_MASK_MODE
29019 : #define TARGET_VECTORIZE_GET_MASK_MODE ix86_get_mask_mode
29020 : #undef TARGET_VECTORIZE_CREATE_COSTS
29021 : #define TARGET_VECTORIZE_CREATE_COSTS ix86_vectorize_create_costs
29022 :
29023 : #undef TARGET_SET_CURRENT_FUNCTION
29024 : #define TARGET_SET_CURRENT_FUNCTION ix86_set_current_function
29025 :
29026 : #undef TARGET_OPTION_VALID_ATTRIBUTE_P
29027 : #define TARGET_OPTION_VALID_ATTRIBUTE_P ix86_valid_target_attribute_p
29028 :
29029 : #undef TARGET_OPTION_SAVE
29030 : #define TARGET_OPTION_SAVE ix86_function_specific_save
29031 :
29032 : #undef TARGET_OPTION_RESTORE
29033 : #define TARGET_OPTION_RESTORE ix86_function_specific_restore
29034 :
29035 : #undef TARGET_OPTION_POST_STREAM_IN
29036 : #define TARGET_OPTION_POST_STREAM_IN ix86_function_specific_post_stream_in
29037 :
29038 : #undef TARGET_OPTION_PRINT
29039 : #define TARGET_OPTION_PRINT ix86_function_specific_print
29040 :
29041 : #undef TARGET_CAN_INLINE_P
29042 : #define TARGET_CAN_INLINE_P ix86_can_inline_p
29043 :
29044 : #undef TARGET_LEGITIMATE_ADDRESS_P
29045 : #define TARGET_LEGITIMATE_ADDRESS_P ix86_legitimate_address_p
29046 :
29047 : #undef TARGET_REGISTER_PRIORITY
29048 : #define TARGET_REGISTER_PRIORITY ix86_register_priority
29049 :
29050 : #undef TARGET_REGISTER_USAGE_LEVELING_P
29051 : #define TARGET_REGISTER_USAGE_LEVELING_P hook_bool_void_true
29052 :
29053 : #undef TARGET_LEGITIMATE_CONSTANT_P
29054 : #define TARGET_LEGITIMATE_CONSTANT_P ix86_legitimate_constant_p
29055 :
29056 : #undef TARGET_COMPUTE_FRAME_LAYOUT
29057 : #define TARGET_COMPUTE_FRAME_LAYOUT ix86_compute_frame_layout
29058 :
29059 : #undef TARGET_FRAME_POINTER_REQUIRED
29060 : #define TARGET_FRAME_POINTER_REQUIRED ix86_frame_pointer_required
29061 :
29062 : #undef TARGET_CAN_ELIMINATE
29063 : #define TARGET_CAN_ELIMINATE ix86_can_eliminate
29064 :
29065 : #undef TARGET_EXTRA_LIVE_ON_ENTRY
29066 : #define TARGET_EXTRA_LIVE_ON_ENTRY ix86_live_on_entry
29067 :
29068 : #undef TARGET_ASM_CODE_END
29069 : #define TARGET_ASM_CODE_END ix86_code_end
29070 :
29071 : #undef TARGET_CONDITIONAL_REGISTER_USAGE
29072 : #define TARGET_CONDITIONAL_REGISTER_USAGE ix86_conditional_register_usage
29073 :
29074 : #undef TARGET_CANONICALIZE_COMPARISON
29075 : #define TARGET_CANONICALIZE_COMPARISON ix86_canonicalize_comparison
29076 :
29077 : #undef TARGET_LOOP_UNROLL_ADJUST
29078 : #define TARGET_LOOP_UNROLL_ADJUST ix86_loop_unroll_adjust
29079 :
29080 : /* Disabled due to PRs 70902, 71453, 71555, 71596 and 71657. */
29081 : #undef TARGET_SPILL_CLASS
29082 : #define TARGET_SPILL_CLASS ix86_spill_class
29083 :
29084 : #undef TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN
29085 : #define TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN \
29086 : ix86_simd_clone_compute_vecsize_and_simdlen
29087 :
29088 : #undef TARGET_SIMD_CLONE_ADJUST
29089 : #define TARGET_SIMD_CLONE_ADJUST ix86_simd_clone_adjust
29090 :
29091 : #undef TARGET_SIMD_CLONE_USABLE
29092 : #define TARGET_SIMD_CLONE_USABLE ix86_simd_clone_usable
29093 :
29094 : #undef TARGET_OMP_DEVICE_KIND_ARCH_ISA
29095 : #define TARGET_OMP_DEVICE_KIND_ARCH_ISA ix86_omp_device_kind_arch_isa
29096 :
29097 : #undef TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P
29098 : #define TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P \
29099 : ix86_float_exceptions_rounding_supported_p
29100 :
29101 : #undef TARGET_MODE_EMIT
29102 : #define TARGET_MODE_EMIT ix86_emit_mode_set
29103 :
29104 : #undef TARGET_MODE_NEEDED
29105 : #define TARGET_MODE_NEEDED ix86_mode_needed
29106 :
29107 : #undef TARGET_MODE_AFTER
29108 : #define TARGET_MODE_AFTER ix86_mode_after
29109 :
29110 : #undef TARGET_MODE_ENTRY
29111 : #define TARGET_MODE_ENTRY ix86_mode_entry
29112 :
29113 : #undef TARGET_MODE_EXIT
29114 : #define TARGET_MODE_EXIT ix86_mode_exit
29115 :
29116 : #undef TARGET_MODE_PRIORITY
29117 : #define TARGET_MODE_PRIORITY ix86_mode_priority
29118 :
29119 : #undef TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS
29120 : #define TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS true
29121 :
29122 : #undef TARGET_OFFLOAD_OPTIONS
29123 : #define TARGET_OFFLOAD_OPTIONS \
29124 : ix86_offload_options
29125 :
29126 : #undef TARGET_ABSOLUTE_BIGGEST_ALIGNMENT
29127 : #define TARGET_ABSOLUTE_BIGGEST_ALIGNMENT 512
29128 :
29129 : #undef TARGET_OPTAB_SUPPORTED_P
29130 : #define TARGET_OPTAB_SUPPORTED_P ix86_optab_supported_p
29131 :
29132 : #undef TARGET_HARD_REGNO_SCRATCH_OK
29133 : #define TARGET_HARD_REGNO_SCRATCH_OK ix86_hard_regno_scratch_ok
29134 :
29135 : #undef TARGET_CUSTOM_FUNCTION_DESCRIPTORS
29136 : #define TARGET_CUSTOM_FUNCTION_DESCRIPTORS X86_CUSTOM_FUNCTION_TEST
29137 :
29138 : #undef TARGET_ADDR_SPACE_ZERO_ADDRESS_VALID
29139 : #define TARGET_ADDR_SPACE_ZERO_ADDRESS_VALID ix86_addr_space_zero_address_valid
29140 :
29141 : #undef TARGET_INIT_LIBFUNCS
29142 : #define TARGET_INIT_LIBFUNCS ix86_init_libfuncs
29143 :
29144 : #undef TARGET_EXPAND_DIVMOD_LIBFUNC
29145 : #define TARGET_EXPAND_DIVMOD_LIBFUNC ix86_expand_divmod_libfunc
29146 :
29147 : #undef TARGET_MAX_NOCE_IFCVT_SEQ_COST
29148 : #define TARGET_MAX_NOCE_IFCVT_SEQ_COST ix86_max_noce_ifcvt_seq_cost
29149 :
29150 : #undef TARGET_NOCE_CONVERSION_PROFITABLE_P
29151 : #define TARGET_NOCE_CONVERSION_PROFITABLE_P ix86_noce_conversion_profitable_p
29152 :
29153 : #undef TARGET_HARD_REGNO_NREGS
29154 : #define TARGET_HARD_REGNO_NREGS ix86_hard_regno_nregs
29155 : #undef TARGET_HARD_REGNO_MODE_OK
29156 : #define TARGET_HARD_REGNO_MODE_OK ix86_hard_regno_mode_ok
29157 :
29158 : #undef TARGET_MODES_TIEABLE_P
29159 : #define TARGET_MODES_TIEABLE_P ix86_modes_tieable_p
29160 :
29161 : #undef TARGET_HARD_REGNO_CALL_PART_CLOBBERED
29162 : #define TARGET_HARD_REGNO_CALL_PART_CLOBBERED \
29163 : ix86_hard_regno_call_part_clobbered
29164 :
29165 : #undef TARGET_FNTYPE_ABI
29166 : #define TARGET_FNTYPE_ABI ix86_fntype_abi
29167 :
29168 : #undef TARGET_CAN_CHANGE_MODE_CLASS
29169 : #define TARGET_CAN_CHANGE_MODE_CLASS ix86_can_change_mode_class
29170 :
29171 : #undef TARGET_LOWER_LOCAL_DECL_ALIGNMENT
29172 : #define TARGET_LOWER_LOCAL_DECL_ALIGNMENT ix86_lower_local_decl_alignment
29173 :
29174 : #undef TARGET_STATIC_RTX_ALIGNMENT
29175 : #define TARGET_STATIC_RTX_ALIGNMENT ix86_static_rtx_alignment
29176 : #undef TARGET_CONSTANT_ALIGNMENT
29177 : #define TARGET_CONSTANT_ALIGNMENT ix86_constant_alignment
29178 :
29179 : #undef TARGET_EMPTY_RECORD_P
29180 : #define TARGET_EMPTY_RECORD_P ix86_is_empty_record
29181 :
29182 : #undef TARGET_WARN_PARAMETER_PASSING_ABI
29183 : #define TARGET_WARN_PARAMETER_PASSING_ABI ix86_warn_parameter_passing_abi
29184 :
29185 : #undef TARGET_GET_MULTILIB_ABI_NAME
29186 : #define TARGET_GET_MULTILIB_ABI_NAME \
29187 : ix86_get_multilib_abi_name
29188 :
29189 : #undef TARGET_IFUNC_REF_LOCAL_OK
29190 : #define TARGET_IFUNC_REF_LOCAL_OK ix86_ifunc_ref_local_ok
29191 :
29192 : #if !TARGET_MACHO && !TARGET_DLLIMPORT_DECL_ATTRIBUTES
29193 : # undef TARGET_ASM_RELOC_RW_MASK
29194 : # define TARGET_ASM_RELOC_RW_MASK ix86_reloc_rw_mask
29195 : #endif
29196 :
29197 : #undef TARGET_MEMTAG_CAN_TAG_ADDRESSES
29198 : #define TARGET_MEMTAG_CAN_TAG_ADDRESSES ix86_memtag_can_tag_addresses
29199 :
29200 : #undef TARGET_MEMTAG_ADD_TAG
29201 : #define TARGET_MEMTAG_ADD_TAG ix86_memtag_add_tag
29202 :
29203 : #undef TARGET_MEMTAG_SET_TAG
29204 : #define TARGET_MEMTAG_SET_TAG ix86_memtag_set_tag
29205 :
29206 : #undef TARGET_MEMTAG_EXTRACT_TAG
29207 : #define TARGET_MEMTAG_EXTRACT_TAG ix86_memtag_extract_tag
29208 :
29209 : #undef TARGET_MEMTAG_UNTAGGED_POINTER
29210 : #define TARGET_MEMTAG_UNTAGGED_POINTER ix86_memtag_untagged_pointer
29211 :
29212 : #undef TARGET_MEMTAG_TAG_BITSIZE
29213 : #define TARGET_MEMTAG_TAG_BITSIZE ix86_memtag_tag_bitsize
29214 :
29215 : #undef TARGET_GEN_CCMP_FIRST
29216 : #define TARGET_GEN_CCMP_FIRST ix86_gen_ccmp_first
29217 :
29218 : #undef TARGET_GEN_CCMP_NEXT
29219 : #define TARGET_GEN_CCMP_NEXT ix86_gen_ccmp_next
29220 :
29221 : #undef TARGET_HAVE_CCMP
29222 : #define TARGET_HAVE_CCMP ix86_have_ccmp
29223 :
29224 : #undef TARGET_MODE_CAN_TRANSFER_BITS
29225 : #define TARGET_MODE_CAN_TRANSFER_BITS ix86_mode_can_transfer_bits
29226 :
29227 : #undef TARGET_REDZONE_CLOBBER
29228 : #define TARGET_REDZONE_CLOBBER ix86_redzone_clobber
29229 :
29230 : static bool
29231 95857 : ix86_libc_has_fast_function (int fcode ATTRIBUTE_UNUSED)
29232 : {
29233 : #ifdef OPTION_GLIBC
29234 95857 : if (OPTION_GLIBC)
29235 95857 : return (built_in_function)fcode == BUILT_IN_MEMPCPY;
29236 : else
29237 : return false;
29238 : #else
29239 : return false;
29240 : #endif
29241 : }
29242 :
29243 : #undef TARGET_LIBC_HAS_FAST_FUNCTION
29244 : #define TARGET_LIBC_HAS_FAST_FUNCTION ix86_libc_has_fast_function
29245 :
29246 : static unsigned
29247 76397 : ix86_libm_function_max_error (unsigned cfn, machine_mode mode,
29248 : bool boundary_p)
29249 : {
29250 : #ifdef OPTION_GLIBC
29251 76397 : bool glibc_p = OPTION_GLIBC;
29252 : #else
29253 : bool glibc_p = false;
29254 : #endif
29255 76397 : if (glibc_p)
29256 : {
29257 : /* If __FAST_MATH__ is defined, glibc provides libmvec. */
29258 76397 : unsigned int libmvec_ret = 0;
29259 76397 : if (!flag_trapping_math
29260 8274 : && flag_unsafe_math_optimizations
29261 3364 : && flag_finite_math_only
29262 3338 : && !flag_signed_zeros
29263 3338 : && !flag_errno_math)
29264 3338 : switch (cfn)
29265 : {
29266 1388 : CASE_CFN_COS:
29267 1388 : CASE_CFN_COS_FN:
29268 1388 : CASE_CFN_SIN:
29269 1388 : CASE_CFN_SIN_FN:
29270 1388 : if (!boundary_p)
29271 : {
29272 : /* With non-default rounding modes, libmvec provides
29273 : complete garbage in results. E.g.
29274 : _ZGVcN8v_sinf for 1.40129846e-45f in FE_UPWARD
29275 : returns 0.00333309174f rather than 1.40129846e-45f. */
29276 583 : if (flag_rounding_math)
29277 : return ~0U;
29278 : /* https://www.gnu.org/software/libc/manual/html_node/Errors-in-Math-Functions.html
29279 : claims libmvec maximum error is 4ulps.
29280 : My own random testing indicates 2ulps for SFmode and
29281 : 0.5ulps for DFmode, but let's go with the 4ulps. */
29282 : libmvec_ret = 4;
29283 : }
29284 : break;
29285 : default:
29286 : break;
29287 : }
29288 76397 : unsigned int ret = glibc_linux_libm_function_max_error (cfn, mode,
29289 : boundary_p);
29290 76397 : return MAX (ret, libmvec_ret);
29291 : }
29292 0 : return default_libm_function_max_error (cfn, mode, boundary_p);
29293 : }
29294 :
29295 : #undef TARGET_LIBM_FUNCTION_MAX_ERROR
29296 : #define TARGET_LIBM_FUNCTION_MAX_ERROR ix86_libm_function_max_error
29297 :
29298 : #if TARGET_MACHO
29299 : static bool
29300 : ix86_cannot_copy_insn_p (rtx_insn *insn)
29301 : {
29302 : if (TARGET_64BIT)
29303 : return false;
29304 :
29305 : rtx set = single_set (insn);
29306 : if (set)
29307 : {
29308 : rtx src = SET_SRC (set);
29309 : if (GET_CODE (src) == UNSPEC
29310 : && XINT (src, 1) == UNSPEC_SET_GOT)
29311 : return true;
29312 : }
29313 : return false;
29314 : }
29315 :
29316 : #undef TARGET_CANNOT_COPY_INSN_P
29317 : #define TARGET_CANNOT_COPY_INSN_P ix86_cannot_copy_insn_p
29318 :
29319 : #endif
29320 :
29321 : #if CHECKING_P
29322 : #undef TARGET_RUN_TARGET_SELFTESTS
29323 : #define TARGET_RUN_TARGET_SELFTESTS selftest::ix86_run_selftests
29324 : #endif /* #if CHECKING_P */
29325 :
29326 : #undef TARGET_DOCUMENTATION_NAME
29327 : #define TARGET_DOCUMENTATION_NAME "x86"
29328 :
29329 : /* Implement TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS. */
29330 : sbitmap
29331 742256 : ix86_get_separate_components (void)
29332 : {
29333 742256 : HOST_WIDE_INT offset, to_allocate;
29334 742256 : sbitmap components = sbitmap_alloc (FIRST_PSEUDO_REGISTER);
29335 742256 : bitmap_clear (components);
29336 742256 : struct machine_function *m = cfun->machine;
29337 :
29338 742256 : offset = m->frame.stack_pointer_offset;
29339 742256 : to_allocate = offset - m->frame.sse_reg_save_offset;
29340 :
29341 : /* Shrink wrap separate uses MOV, which means APX PPX cannot be used.
29342 : Experiments show that APX PPX can speed up the prologue. If the function
29343 : does not exit early during actual execution, then using APX PPX is faster.
29344 : If the function always exits early during actual execution, then shrink
29345 : wrap separate reduces the number of MOV (PUSH/POP) instructions actually
29346 : executed, thus speeding up execution.
29347 : foo:
29348 : movl $1, %eax
29349 : testq %rdi, %rdi
29350 : jne.L60
29351 : ret ---> early return.
29352 : .L60:
29353 : subq $88, %rsp ---> belong to prologue.
29354 : xorl %eax, %eax
29355 : movq %rbx, 40 (%rsp) ---> belong to prologue.
29356 : movq 8 (%rdi), %rbx
29357 : movq %rbp, 48 (%rsp) ---> belong to prologue.
29358 : movq %rdi, %rbp
29359 : testq %rbx, %rbx
29360 : jne.L61
29361 : movq 40 (%rsp), %rbx
29362 : movq 48 (%rsp), %rbp
29363 : addq $88, %rsp
29364 : ret
29365 : .L61:
29366 : movq %r12, 56 (%rsp) ---> belong to prologue.
29367 : movq %r13, 64 (%rsp) ---> belong to prologue.
29368 : movq %r14, 72 (%rsp) ---> belong to prologue.
29369 : ... ...
29370 :
29371 : Disable shrink wrap separate when PPX is enabled. */
29372 742256 : if ((TARGET_APX_PPX && !crtl->calls_eh_return)
29373 741782 : || cfun->machine->func_type != TYPE_NORMAL
29374 : || TARGET_SEH
29375 741684 : || crtl->stack_realign_needed
29376 732518 : || m->call_ms2sysv)
29377 : return components;
29378 :
29379 : /* Since shrink wrapping separate uses MOV instead of PUSH/POP.
29380 : Disable shrink wrap separate when MOV is prohibited. */
29381 730596 : if (save_regs_using_push_pop (to_allocate))
29382 : return components;
29383 :
29384 33041226 : for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
29385 32685944 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true, true))
29386 : {
29387 : /* Skip registers with large offsets, where a pseudo may be needed. */
29388 612291 : if (IN_RANGE (offset, -0x8000, 0x7fff))
29389 611169 : bitmap_set_bit (components, regno);
29390 658355 : offset += UNITS_PER_WORD;
29391 : }
29392 :
29393 : /* Don't mess with the following registers. */
29394 355282 : if (frame_pointer_needed)
29395 6399 : bitmap_clear_bit (components, HARD_FRAME_POINTER_REGNUM);
29396 :
29397 355282 : if (crtl->drap_reg)
29398 138 : bitmap_clear_bit (components, REGNO (crtl->drap_reg));
29399 :
29400 355282 : if (pic_offset_table_rtx)
29401 29982 : bitmap_clear_bit (components, REAL_PIC_OFFSET_TABLE_REGNUM);
29402 :
29403 : return components;
29404 : }
29405 :
29406 : /* Implement TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB. */
29407 : sbitmap
29408 9479271 : ix86_components_for_bb (basic_block bb)
29409 : {
29410 9479271 : bitmap in = DF_LIVE_IN (bb);
29411 9479271 : bitmap gen = &DF_LIVE_BB_INFO (bb)->gen;
29412 9479271 : bitmap kill = &DF_LIVE_BB_INFO (bb)->kill;
29413 :
29414 9479271 : sbitmap components = sbitmap_alloc (FIRST_PSEUDO_REGISTER);
29415 9479271 : bitmap_clear (components);
29416 :
29417 9479271 : function_abi_aggregator callee_abis;
29418 9479271 : rtx_insn *insn;
29419 112732620 : FOR_BB_INSNS (bb, insn)
29420 103253349 : if (CALL_P (insn))
29421 3120844 : callee_abis.note_callee_abi (insn_callee_abi (insn));
29422 9479271 : HARD_REG_SET extra_caller_saves = callee_abis.caller_save_regs (*crtl->abi);
29423 :
29424 : /* GPRs are used in a bb if they are in the IN, GEN, or KILL sets. */
29425 881572203 : for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
29426 872092932 : if (!fixed_regs[regno]
29427 872092932 : && (TEST_HARD_REG_BIT (extra_caller_saves, regno)
29428 443004650 : || bitmap_bit_p (in, regno)
29429 417113116 : || bitmap_bit_p (gen, regno)
29430 404383183 : || bitmap_bit_p (kill, regno)))
29431 38938230 : bitmap_set_bit (components, regno);
29432 :
29433 9479271 : return components;
29434 : }
29435 :
29436 : /* Implement TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS. */
29437 : void
29438 474883 : ix86_disqualify_components (sbitmap, edge, sbitmap, bool)
29439 : {
29440 : /* Nothing to do for x86. */
29441 474883 : }
29442 :
29443 : /* Implement TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS. */
29444 : void
29445 162682 : ix86_emit_prologue_components (sbitmap components)
29446 : {
29447 162682 : HOST_WIDE_INT cfa_offset;
29448 162682 : struct machine_function *m = cfun->machine;
29449 :
29450 162682 : cfa_offset = m->frame.reg_save_offset + m->fs.sp_offset
29451 162682 : - m->frame.stack_pointer_offset;
29452 15129426 : for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
29453 14966744 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true, true))
29454 : {
29455 741034 : if (bitmap_bit_p (components, regno))
29456 193114 : ix86_emit_save_reg_using_mov (word_mode, regno, cfa_offset);
29457 790593 : cfa_offset -= UNITS_PER_WORD;
29458 : }
29459 162682 : }
29460 :
29461 : /* Implement TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS. */
29462 : void
29463 146569 : ix86_emit_epilogue_components (sbitmap components)
29464 : {
29465 146569 : HOST_WIDE_INT cfa_offset;
29466 146569 : struct machine_function *m = cfun->machine;
29467 146569 : cfa_offset = m->frame.reg_save_offset + m->fs.sp_offset
29468 146569 : - m->frame.stack_pointer_offset;
29469 :
29470 13630917 : for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
29471 13484348 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true, true))
29472 : {
29473 669388 : if (bitmap_bit_p (components, regno))
29474 : {
29475 258243 : rtx reg = gen_rtx_REG (word_mode, regno);
29476 258243 : rtx mem;
29477 258243 : rtx_insn *insn;
29478 :
29479 258243 : mem = choose_baseaddr (cfa_offset, NULL);
29480 258243 : mem = gen_frame_mem (word_mode, mem);
29481 258243 : insn = emit_move_insn (reg, mem);
29482 :
29483 258243 : RTX_FRAME_RELATED_P (insn) = 1;
29484 258243 : add_reg_note (insn, REG_CFA_RESTORE, reg);
29485 : }
29486 724885 : cfa_offset -= UNITS_PER_WORD;
29487 : }
29488 146569 : }
29489 :
29490 : /* Implement TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS. */
29491 : void
29492 44244 : ix86_set_handled_components (sbitmap components)
29493 : {
29494 4114692 : for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
29495 4070448 : if (bitmap_bit_p (components, regno))
29496 : {
29497 105752 : cfun->machine->reg_is_wrapped_separately[regno] = true;
29498 105752 : cfun->machine->use_fast_prologue_epilogue = true;
29499 105752 : cfun->machine->frame.save_regs_using_mov = true;
29500 : }
29501 44244 : }
29502 :
29503 : #undef TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS
29504 : #define TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS ix86_get_separate_components
29505 : #undef TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB
29506 : #define TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB ix86_components_for_bb
29507 : #undef TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS
29508 : #define TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS ix86_disqualify_components
29509 : #undef TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS
29510 : #define TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS \
29511 : ix86_emit_prologue_components
29512 : #undef TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS
29513 : #define TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS \
29514 : ix86_emit_epilogue_components
29515 : #undef TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS
29516 : #define TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS ix86_set_handled_components
29517 :
29518 : struct gcc_target targetm = TARGET_INITIALIZER;
29519 :
29520 : #include "gt-i386.h"
|
