Line data Source code
1 : /* Subroutines used for code generation on IA-32.
2 : Copyright (C) 1988-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify
7 : it under the terms of the GNU General Public License as published by
8 : the Free Software Foundation; either version 3, or (at your option)
9 : any later version.
10 :
11 : GCC is distributed in the hope that it will be useful,
12 : but WITHOUT ANY WARRANTY; without even the implied warranty of
13 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 : GNU General Public License for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #define INCLUDE_STRING
21 : #define IN_TARGET_CODE 1
22 :
23 : #include "config.h"
24 : #include "system.h"
25 : #include "coretypes.h"
26 : #include "backend.h"
27 : #include "rtl.h"
28 : #include "tree.h"
29 : #include "memmodel.h"
30 : #include "gimple.h"
31 : #include "cfghooks.h"
32 : #include "cfgloop.h"
33 : #include "df.h"
34 : #include "tm_p.h"
35 : #include "stringpool.h"
36 : #include "expmed.h"
37 : #include "optabs.h"
38 : #include "regs.h"
39 : #include "emit-rtl.h"
40 : #include "recog.h"
41 : #include "cgraph.h"
42 : #include "diagnostic.h"
43 : #include "cfgbuild.h"
44 : #include "alias.h"
45 : #include "fold-const.h"
46 : #include "attribs.h"
47 : #include "calls.h"
48 : #include "stor-layout.h"
49 : #include "varasm.h"
50 : #include "output.h"
51 : #include "insn-attr.h"
52 : #include "flags.h"
53 : #include "except.h"
54 : #include "explow.h"
55 : #include "expr.h"
56 : #include "cfgrtl.h"
57 : #include "common/common-target.h"
58 : #include "langhooks.h"
59 : #include "reload.h"
60 : #include "gimplify.h"
61 : #include "dwarf2.h"
62 : #include "tm-constrs.h"
63 : #include "cselib.h"
64 : #include "sched-int.h"
65 : #include "opts.h"
66 : #include "tree-pass.h"
67 : #include "context.h"
68 : #include "pass_manager.h"
69 : #include "target-globals.h"
70 : #include "gimple-iterator.h"
71 : #include "gimple-fold.h"
72 : #include "internal-fn.h"
73 : #include "tree-vectorizer.h"
74 : #include "shrink-wrap.h"
75 : #include "builtins.h"
76 : #include "rtl-iter.h"
77 : #include "tree-iterator.h"
78 : #include "dbgcnt.h"
79 : #include "case-cfn-macros.h"
80 : #include "dojump.h"
81 : #include "fold-const-call.h"
82 : #include "tree-vrp.h"
83 : #include "tree-ssanames.h"
84 : #include "selftest.h"
85 : #include "selftest-rtl.h"
86 : #include "print-rtl.h"
87 : #include "intl.h"
88 : #include "ifcvt.h"
89 : #include "symbol-summary.h"
90 : #include "sreal.h"
91 : #include "ipa-cp.h"
92 : #include "ipa-prop.h"
93 : #include "ipa-fnsummary.h"
94 : #include "wide-int-bitmask.h"
95 : #include "tree-vector-builder.h"
96 : #include "debug.h"
97 : #include "dwarf2out.h"
98 : #include "i386-options.h"
99 : #include "i386-builtins.h"
100 : #include "i386-expand.h"
101 : #include "i386-features.h"
102 : #include "function-abi.h"
103 : #include "rtl-error.h"
104 : #include "gimple-pretty-print.h"
105 :
106 : /* This file should be included last. */
107 : #include "target-def.h"
108 :
109 : static void ix86_print_operand_address_as (FILE *, rtx, addr_space_t, bool);
110 : static void ix86_emit_restore_reg_using_pop (rtx, bool = false);
111 : static const predefined_function_abi & ix86_alternate_abi (void);
112 :
113 :
114 : #ifndef CHECK_STACK_LIMIT
115 : #define CHECK_STACK_LIMIT (-1)
116 : #endif
117 :
118 : /* Return index of given mode in mult and division cost tables. */
119 : #define MODE_INDEX(mode) \
120 : ((mode) == QImode ? 0 \
121 : : (mode) == HImode ? 1 \
122 : : (mode) == SImode ? 2 \
123 : : (mode) == DImode ? 3 \
124 : : 4)
125 :
126 :
127 : /* Set by -mtune. */
128 : const struct processor_costs *ix86_tune_cost = NULL;
129 :
130 : /* Set by -mtune or -Os. */
131 : const struct processor_costs *ix86_cost = NULL;
132 :
133 : /* In case the average insn count for single function invocation is
134 : lower than this constant, emit fast (but longer) prologue and
135 : epilogue code. */
136 : #define FAST_PROLOGUE_INSN_COUNT 20
137 :
138 : /* Names for 8 (low), 8 (high), and 16-bit registers, respectively. */
139 : static const char *const qi_reg_name[] = QI_REGISTER_NAMES;
140 : static const char *const qi_high_reg_name[] = QI_HIGH_REGISTER_NAMES;
141 : static const char *const hi_reg_name[] = HI_REGISTER_NAMES;
142 :
143 : /* Array of the smallest class containing reg number REGNO, indexed by
144 : REGNO. Used by REGNO_REG_CLASS in i386.h. */
145 :
146 : enum reg_class const regclass_map[FIRST_PSEUDO_REGISTER] =
147 : {
148 : /* ax, dx, cx, bx */
149 : AREG, DREG, CREG, BREG,
150 : /* si, di, bp, sp */
151 : SIREG, DIREG, NON_Q_REGS, NON_Q_REGS,
152 : /* FP registers */
153 : FP_TOP_REG, FP_SECOND_REG, FLOAT_REGS, FLOAT_REGS,
154 : FLOAT_REGS, FLOAT_REGS, FLOAT_REGS, FLOAT_REGS,
155 : /* arg pointer, flags, fpsr, frame */
156 : NON_Q_REGS, NO_REGS, NO_REGS, NON_Q_REGS,
157 : /* SSE registers */
158 : SSE_FIRST_REG, SSE_REGS, SSE_REGS, SSE_REGS,
159 : SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS,
160 : /* MMX registers */
161 : MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS,
162 : MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS,
163 : /* REX registers */
164 : GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS,
165 : GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS,
166 : /* SSE REX registers */
167 : SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS,
168 : SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS,
169 : /* AVX-512 SSE registers */
170 : ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS,
171 : ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS,
172 : ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS,
173 : ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS,
174 : /* Mask registers. */
175 : ALL_MASK_REGS, MASK_REGS, MASK_REGS, MASK_REGS,
176 : MASK_REGS, MASK_REGS, MASK_REGS, MASK_REGS,
177 : /* REX2 registers */
178 : GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS,
179 : GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS,
180 : GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS,
181 : GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS,
182 : /* TMM fake register placeholder */
183 : NO_REGS,
184 : /* Block Scale register */
185 : NO_REGS,
186 : };
187 :
188 : /* The "default" register map used in 32bit mode. */
189 :
190 : unsigned int const debugger_register_map[FIRST_PSEUDO_REGISTER] =
191 : {
192 : /* general regs */
193 : 0, 2, 1, 3, 6, 7, 4, 5,
194 : /* fp regs */
195 : 12, 13, 14, 15, 16, 17, 18, 19,
196 : /* arg, flags, fpsr, frame */
197 : IGNORED_DWARF_REGNUM, IGNORED_DWARF_REGNUM,
198 : IGNORED_DWARF_REGNUM, IGNORED_DWARF_REGNUM,
199 : /* SSE */
200 : 21, 22, 23, 24, 25, 26, 27, 28,
201 : /* MMX */
202 : 29, 30, 31, 32, 33, 34, 35, 36,
203 : /* extended integer registers */
204 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
205 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
206 : /* extended sse registers */
207 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
208 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
209 : /* AVX-512 registers 16-23 */
210 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
211 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
212 : /* AVX-512 registers 24-31 */
213 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
214 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
215 : /* Mask registers */
216 : 93, 94, 95, 96, 97, 98, 99, 100,
217 : /* APX r16-r31 */
218 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
219 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
220 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
221 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
222 : /* TMM fake register placeholder */
223 : INVALID_REGNUM,
224 : /* Block Scale register */
225 : INVALID_REGNUM,
226 : };
227 :
228 : /* The "default" register map used in 64bit mode. */
229 :
230 : unsigned int const debugger64_register_map[FIRST_PSEUDO_REGISTER] =
231 : {
232 : /* general regs */
233 : 0, 1, 2, 3, 4, 5, 6, 7,
234 : /* fp regs */
235 : 33, 34, 35, 36, 37, 38, 39, 40,
236 : /* arg, flags, fpsr, frame */
237 : IGNORED_DWARF_REGNUM, IGNORED_DWARF_REGNUM,
238 : IGNORED_DWARF_REGNUM, IGNORED_DWARF_REGNUM,
239 : /* SSE */
240 : 17, 18, 19, 20, 21, 22, 23, 24,
241 : /* MMX */
242 : 41, 42, 43, 44, 45, 46, 47, 48,
243 : /* extended integer registers */
244 : 8, 9, 10, 11, 12, 13, 14, 15,
245 : /* extended SSE registers */
246 : 25, 26, 27, 28, 29, 30, 31, 32,
247 : /* AVX-512 registers 16-23 */
248 : 67, 68, 69, 70, 71, 72, 73, 74,
249 : /* AVX-512 registers 24-31 */
250 : 75, 76, 77, 78, 79, 80, 81, 82,
251 : /* Mask registers */
252 : 118, 119, 120, 121, 122, 123, 124, 125,
253 : /* rex2 extend integer registers */
254 : 130, 131, 132, 133, 134, 135, 136, 137,
255 : 138, 139, 140, 141, 142, 143, 144, 145,
256 : /* tmm fake register placeholder */
257 : IGNORED_DWARF_REGNUM,
258 : /* block scale register */
259 : IGNORED_DWARF_REGNUM,
260 : };
261 :
262 : /* Define the register numbers to be used in Dwarf debugging information.
263 : The SVR4 reference port C compiler uses the following register numbers
264 : in its Dwarf output code:
265 : 0 for %eax (gcc regno = 0)
266 : 1 for %ecx (gcc regno = 2)
267 : 2 for %edx (gcc regno = 1)
268 : 3 for %ebx (gcc regno = 3)
269 : 4 for %esp (gcc regno = 7)
270 : 5 for %ebp (gcc regno = 6)
271 : 6 for %esi (gcc regno = 4)
272 : 7 for %edi (gcc regno = 5)
273 : The following three DWARF register numbers are never generated by
274 : the SVR4 C compiler or by the GNU compilers, but SDB on x86/svr4
275 : believed these numbers have these meanings.
276 : 8 for %eip (no gcc equivalent)
277 : 9 for %eflags (gcc regno = 17)
278 : 10 for %trapno (no gcc equivalent)
279 : It is not at all clear how we should number the FP stack registers
280 : for the x86 architecture. If the version of SDB on x86/svr4 were
281 : a bit less brain dead with respect to floating-point then we would
282 : have a precedent to follow with respect to DWARF register numbers
283 : for x86 FP registers, but the SDB on x86/svr4 was so completely
284 : broken with respect to FP registers that it is hardly worth thinking
285 : of it as something to strive for compatibility with.
286 : The version of x86/svr4 SDB I had does (partially)
287 : seem to believe that DWARF register number 11 is associated with
288 : the x86 register %st(0), but that's about all. Higher DWARF
289 : register numbers don't seem to be associated with anything in
290 : particular, and even for DWARF regno 11, SDB only seemed to under-
291 : stand that it should say that a variable lives in %st(0) (when
292 : asked via an `=' command) if we said it was in DWARF regno 11,
293 : but SDB still printed garbage when asked for the value of the
294 : variable in question (via a `/' command).
295 : (Also note that the labels SDB printed for various FP stack regs
296 : when doing an `x' command were all wrong.)
297 : Note that these problems generally don't affect the native SVR4
298 : C compiler because it doesn't allow the use of -O with -g and
299 : because when it is *not* optimizing, it allocates a memory
300 : location for each floating-point variable, and the memory
301 : location is what gets described in the DWARF AT_location
302 : attribute for the variable in question.
303 : Regardless of the severe mental illness of the x86/svr4 SDB, we
304 : do something sensible here and we use the following DWARF
305 : register numbers. Note that these are all stack-top-relative
306 : numbers.
307 : 11 for %st(0) (gcc regno = 8)
308 : 12 for %st(1) (gcc regno = 9)
309 : 13 for %st(2) (gcc regno = 10)
310 : 14 for %st(3) (gcc regno = 11)
311 : 15 for %st(4) (gcc regno = 12)
312 : 16 for %st(5) (gcc regno = 13)
313 : 17 for %st(6) (gcc regno = 14)
314 : 18 for %st(7) (gcc regno = 15)
315 : */
316 : unsigned int const svr4_debugger_register_map[FIRST_PSEUDO_REGISTER] =
317 : {
318 : /* general regs */
319 : 0, 2, 1, 3, 6, 7, 5, 4,
320 : /* fp regs */
321 : 11, 12, 13, 14, 15, 16, 17, 18,
322 : /* arg, flags, fpsr, frame */
323 : IGNORED_DWARF_REGNUM, 9,
324 : IGNORED_DWARF_REGNUM, IGNORED_DWARF_REGNUM,
325 : /* SSE registers */
326 : 21, 22, 23, 24, 25, 26, 27, 28,
327 : /* MMX registers */
328 : 29, 30, 31, 32, 33, 34, 35, 36,
329 : /* extended integer registers */
330 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
331 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
332 : /* extended sse registers */
333 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
334 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
335 : /* AVX-512 registers 16-23 */
336 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
337 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
338 : /* AVX-512 registers 24-31 */
339 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
340 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
341 : /* Mask registers */
342 : 93, 94, 95, 96, 97, 98, 99, 100,
343 : /* APX r16-r31 */
344 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
345 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
346 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
347 : INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM, INVALID_REGNUM,
348 : /* TMM fake register placeholder */
349 : INVALID_REGNUM,
350 : /* Block Scale register */
351 : INVALID_REGNUM,
352 : };
353 :
354 : /* Define parameter passing and return registers. */
355 :
356 : static int const x86_64_int_parameter_registers[6] =
357 : {
358 : DI_REG, SI_REG, DX_REG, CX_REG, R8_REG, R9_REG
359 : };
360 :
361 : static int const x86_64_ms_abi_int_parameter_registers[4] =
362 : {
363 : CX_REG, DX_REG, R8_REG, R9_REG
364 : };
365 :
366 : /* Similar as Clang's preserve_none function parameter passing.
367 : NB: Use DI_REG and SI_REG, see ix86_function_arg_regno_p. */
368 :
369 : static int const x86_64_preserve_none_int_parameter_registers[6] =
370 : {
371 : R12_REG, R13_REG, R14_REG, R15_REG, DI_REG, SI_REG
372 : };
373 :
374 : static int const x86_64_int_return_registers[2] =
375 : {
376 : AX_REG, DX_REG
377 : };
378 :
379 : /* Define the structure for the machine field in struct function. */
380 :
381 : struct GTY(()) stack_local_entry {
382 : unsigned short mode;
383 : unsigned short n;
384 : rtx rtl;
385 : struct stack_local_entry *next;
386 : };
387 :
388 : /* Which cpu are we scheduling for. */
389 : enum attr_cpu ix86_schedule;
390 :
391 : /* Which cpu are we optimizing for. */
392 : enum processor_type ix86_tune;
393 :
394 : /* Which instruction set architecture to use. */
395 : enum processor_type ix86_arch;
396 :
397 : /* True if processor has SSE prefetch instruction. */
398 : unsigned char ix86_prefetch_sse;
399 :
400 : /* Preferred alignment for stack boundary in bits. */
401 : unsigned int ix86_preferred_stack_boundary;
402 :
403 : /* Alignment for incoming stack boundary in bits specified at
404 : command line. */
405 : unsigned int ix86_user_incoming_stack_boundary;
406 :
407 : /* Default alignment for incoming stack boundary in bits. */
408 : unsigned int ix86_default_incoming_stack_boundary;
409 :
410 : /* Alignment for incoming stack boundary in bits. */
411 : unsigned int ix86_incoming_stack_boundary;
412 :
413 : /* True if there is no direct access to extern symbols. */
414 : bool ix86_has_no_direct_extern_access;
415 :
416 : /* Calling abi specific va_list type nodes. */
417 : tree sysv_va_list_type_node;
418 : tree ms_va_list_type_node;
419 :
420 : /* Prefix built by ASM_GENERATE_INTERNAL_LABEL. */
421 : char internal_label_prefix[16];
422 : int internal_label_prefix_len;
423 :
424 : /* Fence to use after loop using movnt. */
425 : tree x86_mfence;
426 :
427 : /* Register class used for passing given 64bit part of the argument.
428 : These represent classes as documented by the PS ABI, with the exception
429 : of SSESF, SSEDF classes, that are basically SSE class, just gcc will
430 : use SF or DFmode move instead of DImode to avoid reformatting penalties.
431 :
432 : Similarly we play games with INTEGERSI_CLASS to use cheaper SImode moves
433 : whenever possible (upper half does contain padding). */
434 : enum x86_64_reg_class
435 : {
436 : X86_64_NO_CLASS,
437 : X86_64_INTEGER_CLASS,
438 : X86_64_INTEGERSI_CLASS,
439 : X86_64_SSE_CLASS,
440 : X86_64_SSEHF_CLASS,
441 : X86_64_SSESF_CLASS,
442 : X86_64_SSEDF_CLASS,
443 : X86_64_SSEUP_CLASS,
444 : X86_64_X87_CLASS,
445 : X86_64_X87UP_CLASS,
446 : X86_64_COMPLEX_X87_CLASS,
447 : X86_64_MEMORY_CLASS
448 : };
449 :
450 : #define MAX_CLASSES 8
451 :
452 : /* Table of constants used by fldpi, fldln2, etc.... */
453 : static REAL_VALUE_TYPE ext_80387_constants_table [5];
454 : static bool ext_80387_constants_init;
455 :
456 :
457 : static rtx ix86_function_value (const_tree, const_tree, bool);
458 : static bool ix86_function_value_regno_p (const unsigned int);
459 : static unsigned int ix86_function_arg_boundary (machine_mode,
460 : const_tree);
461 : static rtx ix86_static_chain (const_tree, bool);
462 : static int ix86_function_regparm (const_tree, const_tree);
463 : static void ix86_compute_frame_layout (void);
464 : static tree ix86_canonical_va_list_type (tree);
465 : static unsigned int split_stack_prologue_scratch_regno (void);
466 : static bool i386_asm_output_addr_const_extra (FILE *, rtx);
467 :
468 : static bool ix86_can_inline_p (tree, tree);
469 : static unsigned int ix86_minimum_incoming_stack_boundary (bool);
470 :
471 : typedef enum ix86_flags_cc
472 : {
473 : X86_CCO = 0, X86_CCNO, X86_CCB, X86_CCNB,
474 : X86_CCE, X86_CCNE, X86_CCBE, X86_CCNBE,
475 : X86_CCS, X86_CCNS, X86_CCP, X86_CCNP,
476 : X86_CCL, X86_CCNL, X86_CCLE, X86_CCNLE
477 : } ix86_cc;
478 :
479 : static const char *ix86_ccmp_dfv_mapping[] =
480 : {
481 : "{dfv=of}", "{dfv=}", "{dfv=cf}", "{dfv=}",
482 : "{dfv=zf}", "{dfv=}", "{dfv=cf, zf}", "{dfv=}",
483 : "{dfv=sf}", "{dfv=}", "{dfv=cf}", "{dfv=}",
484 : "{dfv=sf}", "{dfv=sf, of}", "{dfv=sf, of, zf}", "{dfv=sf, of}"
485 : };
486 :
487 :
488 : /* Whether -mtune= or -march= were specified */
489 : int ix86_tune_defaulted;
490 : int ix86_arch_specified;
491 :
492 : /* Return true if a red-zone is in use. We can't use red-zone when
493 : there are local indirect jumps, like "indirect_jump" or "tablejump",
494 : which jumps to another place in the function, since "call" in the
495 : indirect thunk pushes the return address onto stack, destroying
496 : red-zone.
497 :
498 : NB: Don't use red-zone for functions with no_caller_saved_registers
499 : and 32 GPRs or 16 XMM registers since 128-byte red-zone is too small
500 : for 31 GPRs or 15 GPRs + 16 XMM registers.
501 :
502 : TODO: If we can reserve the first 2 WORDs, for PUSH and, another
503 : for CALL, in red-zone, we can allow local indirect jumps with
504 : indirect thunk. */
505 :
506 : bool
507 10147348 : ix86_using_red_zone (void)
508 : {
509 10147348 : return (TARGET_RED_ZONE
510 9204329 : && !TARGET_64BIT_MS_ABI
511 8904783 : && ((!TARGET_APX_EGPR && !TARGET_SSE)
512 8881632 : || (cfun->machine->call_saved_registers
513 8881632 : != TYPE_NO_CALLER_SAVED_REGISTERS))
514 19052036 : && (!cfun->machine->has_local_indirect_jump
515 52989 : || cfun->machine->indirect_branch_type == indirect_branch_keep));
516 : }
517 :
518 : /* Return true, if profiling code should be emitted before
519 : prologue. Otherwise it returns false.
520 : Note: For x86 with "hotfix" it is sorried. */
521 : static bool
522 4615031 : ix86_profile_before_prologue (void)
523 : {
524 4615031 : return flag_fentry != 0;
525 : }
526 :
527 : /* Update register usage after having seen the compiler flags. */
528 :
529 : static void
530 791585 : ix86_conditional_register_usage (void)
531 : {
532 791585 : int i, c_mask;
533 :
534 : /* For 32-bit targets, disable the REX registers. */
535 791585 : if (! TARGET_64BIT)
536 : {
537 135378 : for (i = FIRST_REX_INT_REG; i <= LAST_REX_INT_REG; i++)
538 120336 : CLEAR_HARD_REG_BIT (accessible_reg_set, i);
539 135378 : for (i = FIRST_REX_SSE_REG; i <= LAST_REX_SSE_REG; i++)
540 120336 : CLEAR_HARD_REG_BIT (accessible_reg_set, i);
541 255714 : for (i = FIRST_EXT_REX_SSE_REG; i <= LAST_EXT_REX_SSE_REG; i++)
542 240672 : CLEAR_HARD_REG_BIT (accessible_reg_set, i);
543 : }
544 :
545 : /* Set up the call-used registers based on the system ABI (ix86_abi).
546 :
547 : See the definition of CALL_USED_REGISTERS in i386.h. */
548 791585 : c_mask = CALL_USED_REGISTERS_MASK (TARGET_64BIT && ix86_abi == MS_ABI);
549 :
550 791585 : CLEAR_HARD_REG_SET (reg_class_contents[(int)CLOBBERED_REGS]);
551 :
552 75200575 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
553 : {
554 : /* Set/reset conditionally defined registers from
555 : CALL_USED_REGISTERS initializer. */
556 74408990 : if (call_used_regs[i] > 1)
557 12665305 : call_used_regs[i] = !!(call_used_regs[i] & c_mask);
558 :
559 : /* Calculate registers of CLOBBERED_REGS register set
560 : as call used registers from GENERAL_REGS register set. */
561 74408990 : if (TEST_HARD_REG_BIT (reg_class_contents[(int)GENERAL_REGS], i)
562 74408990 : && call_used_regs[i])
563 22193170 : SET_HARD_REG_BIT (reg_class_contents[(int)CLOBBERED_REGS], i);
564 : }
565 :
566 : /* If MMX is disabled, disable the registers. */
567 791585 : if (! TARGET_MMX)
568 383632 : accessible_reg_set &= ~reg_class_contents[MMX_REGS];
569 :
570 : /* If SSE is disabled, disable the registers. */
571 791585 : if (! TARGET_SSE)
572 378432 : accessible_reg_set &= ~reg_class_contents[ALL_SSE_REGS];
573 :
574 : /* If the FPU is disabled, disable the registers. */
575 791585 : if (! (TARGET_80387 || TARGET_FLOAT_RETURNS_IN_80387))
576 378872 : accessible_reg_set &= ~reg_class_contents[FLOAT_REGS];
577 :
578 : /* If AVX512F is disabled, disable the registers. */
579 791585 : if (! TARGET_AVX512F)
580 : {
581 9236831 : for (i = FIRST_EXT_REX_SSE_REG; i <= LAST_EXT_REX_SSE_REG; i++)
582 8693488 : CLEAR_HARD_REG_BIT (accessible_reg_set, i);
583 :
584 1086686 : accessible_reg_set &= ~reg_class_contents[ALL_MASK_REGS];
585 : }
586 :
587 : /* If APX is disabled, disable the registers. */
588 791585 : if (! (TARGET_APX_EGPR && TARGET_64BIT))
589 : {
590 13448751 : for (i = FIRST_REX2_INT_REG; i <= LAST_REX2_INT_REG; i++)
591 12657648 : CLEAR_HARD_REG_BIT (accessible_reg_set, i);
592 : }
593 :
594 : /* If AMX-TILE or ACEV1 is disabled, disable tmm registers. */
595 791585 : if (! (TARGET_AMX_TILE || TARGET_ACEV1))
596 : {
597 746630 : CLEAR_HARD_REG_BIT (accessible_reg_set, TMM_REGNUM);
598 : }
599 :
600 : /* If ACEV1 is disabled, disable bsr0. */
601 791585 : if (! TARGET_ACEV1)
602 : {
603 786234 : CLEAR_HARD_REG_BIT (accessible_reg_set, BSR0_REG);
604 : }
605 791585 : }
606 :
607 : /* Canonicalize a comparison from one we don't have to one we do have. */
608 :
609 : static void
610 25343270 : ix86_canonicalize_comparison (int *code, rtx *op0, rtx *op1,
611 : bool op0_preserve_value)
612 : {
613 : /* The order of operands in x87 ficom compare is forced by combine in
614 : simplify_comparison () function. Float operator is treated as RTX_OBJ
615 : with a precedence over other operators and is always put in the first
616 : place. Swap condition and operands to match ficom instruction. */
617 25343270 : if (!op0_preserve_value
618 24521935 : && GET_CODE (*op0) == FLOAT && MEM_P (XEXP (*op0, 0)) && REG_P (*op1))
619 : {
620 26 : enum rtx_code scode = swap_condition ((enum rtx_code) *code);
621 :
622 : /* We are called only for compares that are split to SAHF instruction.
623 : Ensure that we have setcc/jcc insn for the swapped condition. */
624 26 : if (ix86_fp_compare_code_to_integer (scode) != UNKNOWN)
625 : {
626 10 : std::swap (*op0, *op1);
627 10 : *code = (int) scode;
628 10 : return;
629 : }
630 : }
631 :
632 : /* SUB (a, b) underflows precisely when a < b. Convert
633 : (compare (minus (a b)) a) to (compare (a b))
634 : to match *sub<mode>_3 pattern. */
635 24521925 : if (!op0_preserve_value
636 24521925 : && (*code == GTU || *code == LEU)
637 1936622 : && GET_CODE (*op0) == MINUS
638 86642 : && rtx_equal_p (XEXP (*op0, 0), *op1))
639 : {
640 487 : *op1 = XEXP (*op0, 1);
641 487 : *op0 = XEXP (*op0, 0);
642 487 : *code = (int) swap_condition ((enum rtx_code) *code);
643 487 : return;
644 : }
645 :
646 : /* Swap operands of GTU comparison to canonicalize
647 : addcarry/subborrow comparison. */
648 25342773 : if (!op0_preserve_value
649 24521438 : && *code == GTU
650 924117 : && GET_CODE (*op0) == PLUS
651 338011 : && ix86_carry_flag_operator (XEXP (*op0, 0), VOIDmode)
652 52423 : && GET_CODE (XEXP (*op0, 1)) == ZERO_EXTEND
653 25391141 : && GET_CODE (*op1) == ZERO_EXTEND)
654 : {
655 45497 : std::swap (*op0, *op1);
656 45497 : *code = (int) swap_condition ((enum rtx_code) *code);
657 45497 : return;
658 : }
659 : }
660 :
661 : /* Hook to determine if one function can safely inline another. */
662 :
663 : static bool
664 10802712 : ix86_can_inline_p (tree caller, tree callee)
665 : {
666 10802712 : tree caller_tree = DECL_FUNCTION_SPECIFIC_TARGET (caller);
667 10802712 : tree callee_tree = DECL_FUNCTION_SPECIFIC_TARGET (callee);
668 :
669 : /* Changes of those flags can be tolerated for always inlines. Lets hope
670 : user knows what he is doing. */
671 10802712 : unsigned HOST_WIDE_INT always_inline_safe_mask
672 : = (MASK_USE_8BIT_IDIV | MASK_ACCUMULATE_OUTGOING_ARGS
673 : | MASK_NO_ALIGN_STRINGOPS | MASK_AVX256_SPLIT_UNALIGNED_LOAD
674 : | MASK_AVX256_SPLIT_UNALIGNED_STORE | MASK_CLD
675 : | MASK_NO_FANCY_MATH_387 | MASK_IEEE_FP | MASK_INLINE_ALL_STRINGOPS
676 : | MASK_INLINE_STRINGOPS_DYNAMICALLY | MASK_RECIP | MASK_STACK_PROBE
677 : | MASK_STV | MASK_TLS_DIRECT_SEG_REFS | MASK_VZEROUPPER
678 : | MASK_NO_PUSH_ARGS | MASK_OMIT_LEAF_FRAME_POINTER);
679 :
680 :
681 10802712 : if (!callee_tree)
682 10202511 : callee_tree = target_option_default_node;
683 10802712 : if (!caller_tree)
684 10202342 : caller_tree = target_option_default_node;
685 10802712 : if (callee_tree == caller_tree)
686 : return true;
687 :
688 5582 : struct cl_target_option *caller_opts = TREE_TARGET_OPTION (caller_tree);
689 5582 : struct cl_target_option *callee_opts = TREE_TARGET_OPTION (callee_tree);
690 5582 : bool ret = false;
691 5582 : bool always_inline
692 5582 : = (DECL_DISREGARD_INLINE_LIMITS (callee)
693 10509 : && lookup_attribute ("always_inline",
694 4927 : DECL_ATTRIBUTES (callee)));
695 :
696 : /* If callee only uses GPRs, ignore MASK_80387. */
697 5582 : if (TARGET_GENERAL_REGS_ONLY_P (callee_opts->x_ix86_target_flags))
698 1030 : always_inline_safe_mask |= MASK_80387;
699 :
700 5582 : cgraph_node *callee_node = cgraph_node::get (callee);
701 : /* Callee's isa options should be a subset of the caller's, i.e. a SSE4
702 : function can inline a SSE2 function but a SSE2 function can't inline
703 : a SSE4 function. */
704 5582 : if (((caller_opts->x_ix86_isa_flags & callee_opts->x_ix86_isa_flags)
705 : != callee_opts->x_ix86_isa_flags)
706 5346 : || ((caller_opts->x_ix86_isa_flags2 & callee_opts->x_ix86_isa_flags2)
707 : != callee_opts->x_ix86_isa_flags2))
708 : ret = false;
709 :
710 : /* See if we have the same non-isa options. */
711 5309 : else if ((!always_inline
712 398 : && caller_opts->x_target_flags != callee_opts->x_target_flags)
713 5256 : || (caller_opts->x_target_flags & ~always_inline_safe_mask)
714 5256 : != (callee_opts->x_target_flags & ~always_inline_safe_mask))
715 : ret = false;
716 :
717 5256 : else if (caller_opts->x_ix86_fpmath != callee_opts->x_ix86_fpmath
718 : /* If the callee doesn't use FP expressions differences in
719 : ix86_fpmath can be ignored. We are called from FEs
720 : for multi-versioning call optimization, so beware of
721 : ipa_fn_summaries not available. */
722 1247 : && (! ipa_fn_summaries
723 1247 : || ipa_fn_summaries->get (callee_node) == NULL
724 1247 : || ipa_fn_summaries->get (callee_node)->fp_expressions))
725 : ret = false;
726 :
727 : /* At this point we cannot identify whether arch or tune setting
728 : comes from target attribute or not. So the most conservative way
729 : is to allow the callee that uses default arch and tune string to
730 : be inlined. */
731 4982 : else if (!strcmp (callee_opts->x_ix86_arch_string, "x86-64")
732 1431 : && !strcmp (callee_opts->x_ix86_tune_string, "generic"))
733 : ret = true;
734 :
735 : /* See if arch, tune, etc. are the same. As previous ISA flags already
736 : checks if callee's ISA is subset of caller's, do not block
737 : always_inline attribute for callee even it has different arch. */
738 3559 : else if (!always_inline && caller_opts->arch != callee_opts->arch)
739 : ret = false;
740 :
741 15 : else if (!always_inline && caller_opts->tune != callee_opts->tune)
742 : ret = false;
743 :
744 3559 : else if (!always_inline
745 15 : && caller_opts->branch_cost != callee_opts->branch_cost)
746 : ret = false;
747 :
748 : else
749 10802112 : ret = true;
750 :
751 : return ret;
752 : }
753 :
754 : /* Return true if this goes in large data/bss. */
755 :
756 : static bool
757 88006771 : ix86_in_large_data_p (tree exp)
758 : {
759 88006771 : if (ix86_cmodel != CM_MEDIUM && ix86_cmodel != CM_MEDIUM_PIC
760 88006533 : && ix86_cmodel != CM_LARGE && ix86_cmodel != CM_LARGE_PIC)
761 : return false;
762 :
763 1147 : if (exp == NULL_TREE)
764 : return false;
765 :
766 : /* Functions are never large data. */
767 1147 : if (TREE_CODE (exp) == FUNCTION_DECL)
768 : return false;
769 :
770 : /* Automatic variables are never large data. */
771 279 : if (VAR_P (exp) && !is_global_var (exp))
772 : return false;
773 :
774 279 : if (VAR_P (exp) && DECL_SECTION_NAME (exp))
775 : {
776 51 : const char *section = DECL_SECTION_NAME (exp);
777 51 : if (strcmp (section, ".ldata") == 0
778 51 : || startswith (section, ".ldata.")
779 51 : || strcmp (section, ".lbss") == 0
780 51 : || startswith (section, ".lbss.")
781 99 : || startswith (section, ".gnu.linkonce.lb."))
782 3 : return true;
783 : return false;
784 : }
785 : else
786 : {
787 228 : HOST_WIDE_INT size = int_size_in_bytes (TREE_TYPE (exp));
788 :
789 : /* If this is an incomplete type with size 0, then we can't put it
790 : in data because it might be too big when completed. Also,
791 : int_size_in_bytes returns -1 if size can vary or is larger than
792 : an integer in which case also it is safer to assume that it goes in
793 : large data. */
794 228 : if (size <= 0 || size > ix86_section_threshold)
795 48 : return true;
796 : }
797 :
798 : return false;
799 : }
800 :
801 : /* i386-specific section flag to mark large sections. */
802 : #define SECTION_LARGE SECTION_MACH_DEP
803 :
804 : /* Switch to the appropriate section for output of DECL.
805 : DECL is either a `VAR_DECL' node or a constant of some sort.
806 : RELOC indicates whether forming the initial value of DECL requires
807 : link-time relocations. */
808 :
809 : ATTRIBUTE_UNUSED static section *
810 1687102 : x86_64_elf_select_section (tree decl, int reloc,
811 : unsigned HOST_WIDE_INT align)
812 : {
813 1687102 : if (ix86_in_large_data_p (decl))
814 : {
815 6 : const char *sname = NULL;
816 6 : unsigned int flags = SECTION_WRITE | SECTION_LARGE;
817 6 : switch (categorize_decl_for_section (decl, reloc))
818 : {
819 1 : case SECCAT_DATA:
820 1 : sname = ".ldata";
821 1 : break;
822 0 : case SECCAT_DATA_REL:
823 0 : sname = ".ldata.rel";
824 0 : break;
825 0 : case SECCAT_DATA_REL_LOCAL:
826 0 : sname = ".ldata.rel.local";
827 0 : break;
828 0 : case SECCAT_DATA_REL_RO:
829 0 : sname = ".ldata.rel.ro";
830 0 : break;
831 0 : case SECCAT_DATA_REL_RO_LOCAL:
832 0 : sname = ".ldata.rel.ro.local";
833 0 : break;
834 0 : case SECCAT_BSS:
835 0 : sname = ".lbss";
836 0 : flags |= SECTION_BSS;
837 0 : break;
838 : case SECCAT_RODATA:
839 : case SECCAT_RODATA_MERGE_STR:
840 : case SECCAT_RODATA_MERGE_STR_INIT:
841 : case SECCAT_RODATA_MERGE_CONST:
842 : sname = ".lrodata";
843 : flags &= ~SECTION_WRITE;
844 : break;
845 0 : case SECCAT_SRODATA:
846 0 : case SECCAT_SDATA:
847 0 : case SECCAT_SBSS:
848 0 : gcc_unreachable ();
849 : case SECCAT_TEXT:
850 : case SECCAT_TDATA:
851 : case SECCAT_TBSS:
852 : /* We don't split these for medium model. Place them into
853 : default sections and hope for best. */
854 : break;
855 : }
856 1 : if (sname)
857 : {
858 : /* We might get called with string constants, but get_named_section
859 : doesn't like them as they are not DECLs. Also, we need to set
860 : flags in that case. */
861 6 : if (!DECL_P (decl))
862 3 : return get_section (sname, flags, NULL);
863 3 : return get_named_section (decl, sname, reloc);
864 : }
865 : }
866 1687096 : return default_elf_select_section (decl, reloc, align);
867 : }
868 :
869 : /* Select a set of attributes for section NAME based on the properties
870 : of DECL and whether or not RELOC indicates that DECL's initializer
871 : might contain runtime relocations. */
872 :
873 : static unsigned int ATTRIBUTE_UNUSED
874 74395655 : x86_64_elf_section_type_flags (tree decl, const char *name, int reloc)
875 : {
876 74395655 : unsigned int flags = default_section_type_flags (decl, name, reloc);
877 :
878 74395655 : if (ix86_in_large_data_p (decl))
879 10 : flags |= SECTION_LARGE;
880 :
881 74395655 : if (decl == NULL_TREE
882 370 : && (strcmp (name, ".ldata.rel.ro") == 0
883 370 : || strcmp (name, ".ldata.rel.ro.local") == 0))
884 0 : flags |= SECTION_RELRO;
885 :
886 74395655 : if (strcmp (name, ".lbss") == 0
887 74395651 : || startswith (name, ".lbss.")
888 148791303 : || startswith (name, ".gnu.linkonce.lb."))
889 : {
890 7 : flags |= SECTION_BSS;
891 : /* Clear SECTION_NOTYPE so .lbss etc. are marked @nobits in
892 : default_elf_asm_named_section. */
893 7 : flags &= ~SECTION_NOTYPE;
894 : }
895 :
896 74395655 : return flags;
897 : }
898 :
899 : /* Build up a unique section name, expressed as a
900 : STRING_CST node, and assign it to DECL_SECTION_NAME (decl).
901 : RELOC indicates whether the initial value of EXP requires
902 : link-time relocations. */
903 :
904 : static void ATTRIBUTE_UNUSED
905 1833900 : x86_64_elf_unique_section (tree decl, int reloc)
906 : {
907 1833900 : if (ix86_in_large_data_p (decl))
908 : {
909 3 : const char *prefix = NULL;
910 : /* We only need to use .gnu.linkonce if we don't have COMDAT groups. */
911 3 : bool one_only = DECL_COMDAT_GROUP (decl) && !HAVE_COMDAT_GROUP;
912 :
913 3 : switch (categorize_decl_for_section (decl, reloc))
914 : {
915 0 : case SECCAT_DATA:
916 0 : case SECCAT_DATA_REL:
917 0 : case SECCAT_DATA_REL_LOCAL:
918 0 : case SECCAT_DATA_REL_RO:
919 0 : case SECCAT_DATA_REL_RO_LOCAL:
920 0 : prefix = one_only ? ".ld" : ".ldata";
921 : break;
922 3 : case SECCAT_BSS:
923 3 : prefix = one_only ? ".lb" : ".lbss";
924 : break;
925 : case SECCAT_RODATA:
926 : case SECCAT_RODATA_MERGE_STR:
927 : case SECCAT_RODATA_MERGE_STR_INIT:
928 : case SECCAT_RODATA_MERGE_CONST:
929 : prefix = one_only ? ".lr" : ".lrodata";
930 : break;
931 0 : case SECCAT_SRODATA:
932 0 : case SECCAT_SDATA:
933 0 : case SECCAT_SBSS:
934 0 : gcc_unreachable ();
935 : case SECCAT_TEXT:
936 : case SECCAT_TDATA:
937 : case SECCAT_TBSS:
938 : /* We don't split these for medium model. Place them into
939 : default sections and hope for best. */
940 : break;
941 : }
942 3 : if (prefix)
943 : {
944 3 : const char *name, *linkonce;
945 3 : char *string;
946 :
947 3 : name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
948 3 : name = targetm.strip_name_encoding (name);
949 :
950 : /* If we're using one_only, then there needs to be a .gnu.linkonce
951 : prefix to the section name. */
952 3 : linkonce = one_only ? ".gnu.linkonce" : "";
953 :
954 3 : string = ACONCAT ((linkonce, prefix, ".", name, NULL));
955 :
956 3 : set_decl_section_name (decl, string);
957 3 : return;
958 : }
959 : }
960 1833897 : default_unique_section (decl, reloc);
961 : }
962 :
963 : /* Return true if TYPE has no_callee_saved_registers or preserve_none
964 : attribute. */
965 :
966 : static bool
967 1625535 : ix86_type_no_callee_saved_registers_p (const_tree fntype)
968 : {
969 136435 : auto type = ix86_fntype_call_saved_registers (fntype);
970 1625535 : return type == TYPE_PRESERVE_NONE || type == TYPE_NO_CALLEE_SAVED_REGISTERS;
971 : }
972 :
973 : #ifdef COMMON_ASM_OP
974 :
975 : #ifndef LARGECOMM_SECTION_ASM_OP
976 : #define LARGECOMM_SECTION_ASM_OP "\t.largecomm\t"
977 : #endif
978 :
979 : /* This says how to output assembler code to declare an
980 : uninitialized external linkage data object.
981 :
982 : For medium model x86-64 we need to use LARGECOMM_SECTION_ASM_OP opcode for
983 : large objects. */
984 : void
985 171872 : x86_elf_aligned_decl_common (FILE *file, tree decl,
986 : const char *name, unsigned HOST_WIDE_INT size,
987 : unsigned align)
988 : {
989 171872 : if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC
990 171866 : || ix86_cmodel == CM_LARGE || ix86_cmodel == CM_LARGE_PIC)
991 7 : && size > (unsigned int)ix86_section_threshold)
992 : {
993 1 : switch_to_section (get_named_section (decl, ".lbss", 0));
994 1 : fputs (LARGECOMM_SECTION_ASM_OP, file);
995 : }
996 : else
997 171871 : fputs (COMMON_ASM_OP, file);
998 171872 : assemble_name (file, name);
999 171872 : fprintf (file, "," HOST_WIDE_INT_PRINT_UNSIGNED ",%u\n",
1000 : size, align / BITS_PER_UNIT);
1001 171872 : }
1002 : #endif
1003 :
1004 : /* Utility function for targets to use in implementing
1005 : ASM_OUTPUT_ALIGNED_BSS. */
1006 :
1007 : void
1008 771304 : x86_output_aligned_bss (FILE *file, tree decl, const char *name,
1009 : unsigned HOST_WIDE_INT size, unsigned align)
1010 : {
1011 771304 : if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC
1012 771294 : || ix86_cmodel == CM_LARGE || ix86_cmodel == CM_LARGE_PIC)
1013 42 : && size > (unsigned int)ix86_section_threshold)
1014 3 : switch_to_section (get_named_section (decl, ".lbss", 0));
1015 : else
1016 771301 : switch_to_section (bss_section);
1017 928236 : ASM_OUTPUT_ALIGN (file, floor_log2 (align / BITS_PER_UNIT));
1018 : #ifdef ASM_DECLARE_OBJECT_NAME
1019 771304 : last_assemble_variable_decl = decl;
1020 771304 : ASM_DECLARE_OBJECT_NAME (file, name, decl);
1021 : #else
1022 : /* Standard thing is just output label for the object. */
1023 : ASM_OUTPUT_LABEL (file, name);
1024 : #endif /* ASM_DECLARE_OBJECT_NAME */
1025 771304 : ASM_OUTPUT_SKIP (file, size ? size : 1);
1026 771304 : }
1027 :
1028 : /* Decide whether we must probe the stack before any space allocation
1029 : on this target. It's essentially TARGET_STACK_PROBE except when
1030 : -fstack-check causes the stack to be already probed differently. */
1031 :
1032 : bool
1033 887694 : ix86_target_stack_probe (void)
1034 : {
1035 : /* Do not probe the stack twice if static stack checking is enabled. */
1036 887694 : if (flag_stack_check == STATIC_BUILTIN_STACK_CHECK)
1037 : return false;
1038 :
1039 887694 : return TARGET_STACK_PROBE;
1040 : }
1041 :
1042 : /* Decide whether we can make a sibling call to a function. DECL is the
1043 : declaration of the function being targeted by the call and EXP is the
1044 : CALL_EXPR representing the call. */
1045 :
1046 : static bool
1047 141766 : ix86_function_ok_for_sibcall (tree decl, tree exp)
1048 : {
1049 141766 : tree type, decl_or_type;
1050 141766 : rtx a, b;
1051 141766 : bool bind_global = decl && !targetm.binds_local_p (decl);
1052 :
1053 141766 : if (ix86_function_naked (current_function_decl))
1054 : return false;
1055 :
1056 : /* Sibling call isn't OK if there are no caller-saved registers
1057 : since all registers must be preserved before return. */
1058 141764 : if (cfun->machine->call_saved_registers
1059 141764 : == TYPE_NO_CALLER_SAVED_REGISTERS)
1060 : return false;
1061 :
1062 : /* If we are generating position-independent code, we cannot sibcall
1063 : optimize direct calls to global functions, as the PLT requires
1064 : %ebx be live. (Darwin does not have a PLT.) */
1065 141726 : if (!TARGET_MACHO
1066 141726 : && !TARGET_64BIT
1067 11353 : && flag_pic
1068 8411 : && flag_plt
1069 8411 : && bind_global)
1070 : return false;
1071 :
1072 : /* If we need to align the outgoing stack, then sibcalling would
1073 : unalign the stack, which may break the called function. */
1074 137069 : if (ix86_minimum_incoming_stack_boundary (true)
1075 137069 : < PREFERRED_STACK_BOUNDARY)
1076 : return false;
1077 :
1078 136488 : if (decl)
1079 : {
1080 125533 : decl_or_type = decl;
1081 125533 : type = TREE_TYPE (decl);
1082 : }
1083 : else
1084 : {
1085 : /* We're looking at the CALL_EXPR, we need the type of the function. */
1086 10955 : type = CALL_EXPR_FN (exp); /* pointer expression */
1087 10955 : type = TREE_TYPE (type); /* pointer type */
1088 10955 : type = TREE_TYPE (type); /* function type */
1089 10955 : decl_or_type = type;
1090 : }
1091 :
1092 : /* Sibling call isn't OK if callee has no callee-saved registers
1093 : and the calling function has callee-saved registers. */
1094 136488 : if ((cfun->machine->call_saved_registers
1095 136488 : != TYPE_NO_CALLEE_SAVED_REGISTERS)
1096 136488 : && cfun->machine->call_saved_registers != TYPE_PRESERVE_NONE
1097 272923 : && ix86_type_no_callee_saved_registers_p (type))
1098 : return false;
1099 :
1100 : /* If outgoing reg parm stack space changes, we cannot do sibcall. */
1101 136468 : if ((OUTGOING_REG_PARM_STACK_SPACE (type)
1102 136468 : != OUTGOING_REG_PARM_STACK_SPACE (TREE_TYPE (current_function_decl)))
1103 272178 : || (REG_PARM_STACK_SPACE (decl_or_type)
1104 135710 : != REG_PARM_STACK_SPACE (current_function_decl)))
1105 : {
1106 758 : maybe_complain_about_tail_call (exp,
1107 : "inconsistent size of stack space"
1108 : " allocated for arguments which are"
1109 : " passed in registers");
1110 758 : return false;
1111 : }
1112 :
1113 : /* Check that the return value locations are the same. Like
1114 : if we are returning floats on the 80387 register stack, we cannot
1115 : make a sibcall from a function that doesn't return a float to a
1116 : function that does or, conversely, from a function that does return
1117 : a float to a function that doesn't; the necessary stack adjustment
1118 : would not be executed. This is also the place we notice
1119 : differences in the return value ABI. Note that it is ok for one
1120 : of the functions to have void return type as long as the return
1121 : value of the other is passed in a register. */
1122 135710 : a = ix86_function_value (TREE_TYPE (exp), decl_or_type, false);
1123 135710 : b = ix86_function_value (TREE_TYPE (DECL_RESULT (cfun->decl)),
1124 135710 : cfun->decl, false);
1125 135710 : if (STACK_REG_P (a) || STACK_REG_P (b))
1126 : {
1127 989 : if (!rtx_equal_p (a, b))
1128 : return false;
1129 : }
1130 134721 : else if (VOID_TYPE_P (TREE_TYPE (DECL_RESULT (cfun->decl))))
1131 : ;
1132 24891 : else if (!rtx_equal_p (a, b))
1133 : return false;
1134 :
1135 135321 : if (TARGET_64BIT)
1136 : {
1137 : /* The SYSV ABI has more call-clobbered registers;
1138 : disallow sibcalls from MS to SYSV. */
1139 128625 : if (cfun->machine->call_abi == MS_ABI
1140 128625 : && ix86_function_type_abi (type) == SYSV_ABI)
1141 : return false;
1142 : }
1143 : else
1144 : {
1145 : /* If this call is indirect, we'll need to be able to use a
1146 : call-clobbered register for the address of the target function.
1147 : Make sure that all such registers are not used for passing
1148 : parameters. Note that DLLIMPORT functions and call to global
1149 : function via GOT slot are indirect. */
1150 6696 : if (!decl
1151 4799 : || (bind_global && flag_pic && !flag_plt)
1152 : || (TARGET_DLLIMPORT_DECL_ATTRIBUTES && DECL_DLLIMPORT_P (decl))
1153 4799 : || flag_force_indirect_call)
1154 : {
1155 : /* Check if regparm >= 3 since arg_reg_available is set to
1156 : false if regparm == 0. If regparm is 1 or 2, there is
1157 : always a call-clobbered register available.
1158 :
1159 : ??? The symbol indirect call doesn't need a call-clobbered
1160 : register. But we don't know if this is a symbol indirect
1161 : call or not here. */
1162 1897 : if (ix86_function_regparm (type, decl) >= 3
1163 1897 : && !cfun->machine->arg_reg_available)
1164 : return false;
1165 : }
1166 : }
1167 :
1168 135321 : if (decl && ix86_use_pseudo_pic_reg ())
1169 : {
1170 : /* When PIC register is used, it must be restored after ifunc
1171 : function returns. */
1172 2067 : cgraph_node *node = cgraph_node::get (decl);
1173 2067 : if (node && node->ifunc_resolver)
1174 : return false;
1175 : }
1176 :
1177 : /* Disable sibcall if callee has indirect_return attribute and
1178 : caller doesn't since callee will return to the caller's caller
1179 : via an indirect jump. */
1180 135321 : if (((flag_cf_protection & (CF_RETURN | CF_BRANCH))
1181 : == (CF_RETURN | CF_BRANCH))
1182 55979 : && lookup_attribute ("indirect_return", TYPE_ATTRIBUTES (type))
1183 135325 : && !lookup_attribute ("indirect_return",
1184 4 : TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl))))
1185 2 : return false;
1186 :
1187 : /* Otherwise okay. That also includes certain types of indirect calls. */
1188 : return true;
1189 : }
1190 :
1191 : /* This function determines from TYPE the calling-convention. */
1192 :
1193 : unsigned int
1194 6324869 : ix86_get_callcvt (const_tree type)
1195 : {
1196 6324869 : unsigned int ret = 0;
1197 6324869 : bool is_stdarg;
1198 6324869 : tree attrs;
1199 :
1200 6324869 : if (TARGET_64BIT)
1201 : return IX86_CALLCVT_CDECL;
1202 :
1203 3271165 : attrs = TYPE_ATTRIBUTES (type);
1204 3271165 : if (attrs != NULL_TREE)
1205 : {
1206 68391 : if (lookup_attribute ("cdecl", attrs))
1207 : ret |= IX86_CALLCVT_CDECL;
1208 68391 : else if (lookup_attribute ("stdcall", attrs))
1209 : ret |= IX86_CALLCVT_STDCALL;
1210 68391 : else if (lookup_attribute ("fastcall", attrs))
1211 : ret |= IX86_CALLCVT_FASTCALL;
1212 68382 : else if (lookup_attribute ("thiscall", attrs))
1213 : ret |= IX86_CALLCVT_THISCALL;
1214 :
1215 : /* Regparm isn't allowed for thiscall and fastcall. */
1216 : if ((ret & (IX86_CALLCVT_THISCALL | IX86_CALLCVT_FASTCALL)) == 0)
1217 : {
1218 68382 : if (lookup_attribute ("regparm", attrs))
1219 15844 : ret |= IX86_CALLCVT_REGPARM;
1220 68382 : if (lookup_attribute ("sseregparm", attrs))
1221 0 : ret |= IX86_CALLCVT_SSEREGPARM;
1222 : }
1223 :
1224 68382 : if (IX86_BASE_CALLCVT(ret) != 0)
1225 : return ret;
1226 : }
1227 :
1228 3271156 : is_stdarg = stdarg_p (type);
1229 3271156 : if (TARGET_RTD && !is_stdarg)
1230 0 : return IX86_CALLCVT_STDCALL | ret;
1231 :
1232 3271156 : if (ret != 0
1233 3271156 : || is_stdarg
1234 3246214 : || TREE_CODE (type) != METHOD_TYPE
1235 3408022 : || ix86_function_type_abi (type) != MS_ABI)
1236 3271156 : return IX86_CALLCVT_CDECL | ret;
1237 :
1238 : return IX86_CALLCVT_THISCALL;
1239 : }
1240 :
1241 : /* Return 0 if the attributes for two types are incompatible, 1 if they
1242 : are compatible, and 2 if they are nearly compatible (which causes a
1243 : warning to be generated). */
1244 :
1245 : static int
1246 1544841 : ix86_comp_type_attributes (const_tree type1, const_tree type2)
1247 : {
1248 1544841 : unsigned int ccvt1, ccvt2;
1249 :
1250 1544841 : if (TREE_CODE (type1) != FUNCTION_TYPE
1251 1544841 : && TREE_CODE (type1) != METHOD_TYPE)
1252 : return 1;
1253 :
1254 1537992 : ccvt1 = ix86_get_callcvt (type1);
1255 1537992 : ccvt2 = ix86_get_callcvt (type2);
1256 1537992 : if (ccvt1 != ccvt2)
1257 : return 0;
1258 3053824 : if (ix86_function_regparm (type1, NULL)
1259 1526912 : != ix86_function_regparm (type2, NULL))
1260 : return 0;
1261 :
1262 744550 : if (ix86_type_no_callee_saved_registers_p (type1)
1263 744550 : != ix86_type_no_callee_saved_registers_p (type2))
1264 : return 0;
1265 :
1266 : /* preserve_none attribute uses a different calling convention is
1267 : only for 64-bit. */
1268 744369 : if (TARGET_64BIT
1269 1488678 : && (lookup_attribute ("preserve_none", TYPE_ATTRIBUTES (type1))
1270 744309 : != lookup_attribute ("preserve_none",
1271 744309 : TYPE_ATTRIBUTES (type2))))
1272 2 : return 0;
1273 :
1274 : return 1;
1275 : }
1276 :
1277 : /* Return the regparm value for a function with the indicated TYPE and DECL.
1278 : DECL may be NULL when calling function indirectly
1279 : or considering a libcall. */
1280 :
1281 : static int
1282 4325444 : ix86_function_regparm (const_tree type, const_tree decl)
1283 : {
1284 4325444 : tree attr;
1285 4325444 : int regparm;
1286 4325444 : unsigned int ccvt;
1287 :
1288 4325444 : if (TARGET_64BIT)
1289 3053704 : return (ix86_function_type_abi (type) == SYSV_ABI
1290 3053704 : ? X86_64_REGPARM_MAX : X86_64_MS_REGPARM_MAX);
1291 1271740 : ccvt = ix86_get_callcvt (type);
1292 1271740 : regparm = ix86_regparm;
1293 :
1294 1271740 : if ((ccvt & IX86_CALLCVT_REGPARM) != 0)
1295 : {
1296 2020 : attr = lookup_attribute ("regparm", TYPE_ATTRIBUTES (type));
1297 2020 : if (attr)
1298 : {
1299 2020 : regparm = TREE_INT_CST_LOW (TREE_VALUE (TREE_VALUE (attr)));
1300 2020 : return regparm;
1301 : }
1302 : }
1303 1269720 : else if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
1304 : return 2;
1305 1269720 : else if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
1306 : return 1;
1307 :
1308 : /* Use register calling convention for local functions when possible. */
1309 1269720 : if (decl
1310 1205350 : && TREE_CODE (decl) == FUNCTION_DECL)
1311 : {
1312 1195259 : cgraph_node *target = cgraph_node::get (decl);
1313 1195259 : if (target)
1314 1187749 : target = target->function_symbol ();
1315 :
1316 : /* Caller and callee must agree on the calling convention, so
1317 : checking here just optimize means that with
1318 : __attribute__((optimize (...))) caller could use regparm convention
1319 : and callee not, or vice versa. Instead look at whether the callee
1320 : is optimized or not. */
1321 1187749 : if (target && opt_for_fn (target->decl, optimize)
1322 2374606 : && !(profile_flag && !flag_fentry))
1323 : {
1324 1186857 : if (target->local && target->can_change_signature)
1325 : {
1326 138252 : int local_regparm, globals = 0, regno;
1327 :
1328 : /* Make sure no regparm register is taken by a
1329 : fixed register variable. */
1330 138252 : for (local_regparm = 0; local_regparm < REGPARM_MAX;
1331 : local_regparm++)
1332 103689 : if (fixed_regs[local_regparm])
1333 : break;
1334 :
1335 : /* We don't want to use regparm(3) for nested functions as
1336 : these use a static chain pointer in the third argument. */
1337 34563 : if (local_regparm == 3 && DECL_STATIC_CHAIN (target->decl))
1338 : local_regparm = 2;
1339 :
1340 : /* Save a register for the split stack. */
1341 34563 : if (flag_split_stack)
1342 : {
1343 20152 : if (local_regparm == 3)
1344 : local_regparm = 2;
1345 707 : else if (local_regparm == 2
1346 707 : && DECL_STATIC_CHAIN (target->decl))
1347 : local_regparm = 1;
1348 : }
1349 :
1350 : /* Each fixed register usage increases register pressure,
1351 : so less registers should be used for argument passing.
1352 : This functionality can be overridden by an explicit
1353 : regparm value. */
1354 241941 : for (regno = AX_REG; regno <= DI_REG; regno++)
1355 207378 : if (fixed_regs[regno])
1356 0 : globals++;
1357 :
1358 34563 : local_regparm
1359 34563 : = globals < local_regparm ? local_regparm - globals : 0;
1360 :
1361 34563 : if (local_regparm > regparm)
1362 4325444 : regparm = local_regparm;
1363 : }
1364 : }
1365 : }
1366 :
1367 : return regparm;
1368 : }
1369 :
1370 : /* Return 1 or 2, if we can pass up to SSE_REGPARM_MAX SFmode (1) and
1371 : DFmode (2) arguments in SSE registers for a function with the
1372 : indicated TYPE and DECL. DECL may be NULL when calling function
1373 : indirectly or considering a libcall. Return -1 if any FP parameter
1374 : should be rejected by error. This is used in siutation we imply SSE
1375 : calling convention but the function is called from another function with
1376 : SSE disabled. Otherwise return 0. */
1377 :
1378 : static int
1379 1079083 : ix86_function_sseregparm (const_tree type, const_tree decl, bool warn)
1380 : {
1381 1079083 : gcc_assert (!TARGET_64BIT);
1382 :
1383 : /* Use SSE registers to pass SFmode and DFmode arguments if requested
1384 : by the sseregparm attribute. */
1385 1079083 : if (TARGET_SSEREGPARM
1386 1079083 : || (type && lookup_attribute ("sseregparm", TYPE_ATTRIBUTES (type))))
1387 : {
1388 0 : if (!TARGET_SSE)
1389 : {
1390 0 : if (warn)
1391 : {
1392 0 : if (decl)
1393 0 : error ("calling %qD with attribute sseregparm without "
1394 : "SSE/SSE2 enabled", decl);
1395 : else
1396 0 : error ("calling %qT with attribute sseregparm without "
1397 : "SSE/SSE2 enabled", type);
1398 : }
1399 : return 0;
1400 : }
1401 :
1402 : return 2;
1403 : }
1404 :
1405 1079083 : if (!decl)
1406 : return 0;
1407 :
1408 979496 : cgraph_node *target = cgraph_node::get (decl);
1409 979496 : if (target)
1410 971993 : target = target->function_symbol ();
1411 :
1412 : /* For local functions, pass up to SSE_REGPARM_MAX SFmode
1413 : (and DFmode for SSE2) arguments in SSE registers. */
1414 971993 : if (target
1415 : /* TARGET_SSE_MATH */
1416 971993 : && (target_opts_for_fn (target->decl)->x_ix86_fpmath & FPMATH_SSE)
1417 1296 : && opt_for_fn (target->decl, optimize)
1418 973289 : && !(profile_flag && !flag_fentry))
1419 : {
1420 1296 : if (target->local && target->can_change_signature)
1421 : {
1422 : /* Refuse to produce wrong code when local function with SSE enabled
1423 : is called from SSE disabled function.
1424 : FIXME: We need a way to detect these cases cross-ltrans partition
1425 : and avoid using SSE calling conventions on local functions called
1426 : from function with SSE disabled. For now at least delay the
1427 : warning until we know we are going to produce wrong code.
1428 : See PR66047 */
1429 0 : if (!TARGET_SSE && warn)
1430 : return -1;
1431 0 : return TARGET_SSE2_P (target_opts_for_fn (target->decl)
1432 0 : ->x_ix86_isa_flags) ? 2 : 1;
1433 : }
1434 : }
1435 :
1436 : return 0;
1437 : }
1438 :
1439 : /* Return true if EAX is live at the start of the function. Used by
1440 : ix86_expand_prologue to determine if we need special help before
1441 : calling allocate_stack_worker. */
1442 :
1443 : static bool
1444 7090 : ix86_eax_live_at_start_p (void)
1445 : {
1446 : /* Cheat. Don't bother working forward from ix86_function_regparm
1447 : to the function type to whether an actual argument is located in
1448 : eax. Instead just look at cfg info, which is still close enough
1449 : to correct at this point. This gives false positives for broken
1450 : functions that might use uninitialized data that happens to be
1451 : allocated in eax, but who cares? */
1452 7090 : return REGNO_REG_SET_P (df_get_live_out (ENTRY_BLOCK_PTR_FOR_FN (cfun)), 0);
1453 : }
1454 :
1455 : static bool
1456 159594 : ix86_keep_aggregate_return_pointer (tree fntype)
1457 : {
1458 159594 : tree attr;
1459 :
1460 159594 : if (!TARGET_64BIT)
1461 : {
1462 159594 : attr = lookup_attribute ("callee_pop_aggregate_return",
1463 159594 : TYPE_ATTRIBUTES (fntype));
1464 159594 : if (attr)
1465 0 : return (TREE_INT_CST_LOW (TREE_VALUE (TREE_VALUE (attr))) == 0);
1466 :
1467 : /* For 32-bit MS-ABI the default is to keep aggregate
1468 : return pointer. */
1469 159594 : if (ix86_function_type_abi (fntype) == MS_ABI)
1470 0 : return true;
1471 : }
1472 : return KEEP_AGGREGATE_RETURN_POINTER != 0;
1473 : }
1474 :
1475 : /* Value is the number of bytes of arguments automatically
1476 : popped when returning from a subroutine call.
1477 : FUNDECL is the declaration node of the function (as a tree),
1478 : FUNTYPE is the data type of the function (as a tree),
1479 : or for a library call it is an identifier node for the subroutine name.
1480 : SIZE is the number of bytes of arguments passed on the stack.
1481 :
1482 : On the 80386, the RTD insn may be used to pop them if the number
1483 : of args is fixed, but if the number is variable then the caller
1484 : must pop them all. RTD can't be used for library calls now
1485 : because the library is compiled with the Unix compiler.
1486 : Use of RTD is a selectable option, since it is incompatible with
1487 : standard Unix calling sequences. If the option is not selected,
1488 : the caller must always pop the args.
1489 :
1490 : The attribute stdcall is equivalent to RTD on a per module basis. */
1491 :
1492 : static poly_int64
1493 7863061 : ix86_return_pops_args (tree fundecl, tree funtype, poly_int64 size)
1494 : {
1495 7863061 : unsigned int ccvt;
1496 :
1497 : /* None of the 64-bit ABIs pop arguments. */
1498 7863061 : if (TARGET_64BIT)
1499 6989111 : return 0;
1500 :
1501 873950 : ccvt = ix86_get_callcvt (funtype);
1502 :
1503 873950 : if ((ccvt & (IX86_CALLCVT_STDCALL | IX86_CALLCVT_FASTCALL
1504 : | IX86_CALLCVT_THISCALL)) != 0
1505 873950 : && ! stdarg_p (funtype))
1506 3 : return size;
1507 :
1508 : /* Lose any fake structure return argument if it is passed on the stack. */
1509 873947 : if (aggregate_value_p (TREE_TYPE (funtype), fundecl)
1510 873947 : && !ix86_keep_aggregate_return_pointer (funtype))
1511 : {
1512 159594 : int nregs = ix86_function_regparm (funtype, fundecl);
1513 159594 : if (nregs == 0)
1514 458556 : return GET_MODE_SIZE (Pmode);
1515 : }
1516 :
1517 721095 : return 0;
1518 : }
1519 :
1520 : /* Implement the TARGET_LEGITIMATE_COMBINED_INSN hook. */
1521 :
1522 : static bool
1523 10320480 : ix86_legitimate_combined_insn (rtx_insn *insn)
1524 : {
1525 10320480 : int i;
1526 :
1527 : /* Check operand constraints in case hard registers were propagated
1528 : into insn pattern. This check prevents combine pass from
1529 : generating insn patterns with invalid hard register operands.
1530 : These invalid insns can eventually confuse reload to error out
1531 : with a spill failure. See also PRs 46829 and 46843. */
1532 :
1533 10320480 : gcc_assert (INSN_CODE (insn) >= 0);
1534 :
1535 10320480 : extract_insn (insn);
1536 10320480 : preprocess_constraints (insn);
1537 :
1538 10320480 : int n_operands = recog_data.n_operands;
1539 10320480 : int n_alternatives = recog_data.n_alternatives;
1540 35171814 : for (i = 0; i < n_operands; i++)
1541 : {
1542 24854990 : rtx op = recog_data.operand[i];
1543 24854990 : machine_mode mode = GET_MODE (op);
1544 24854990 : const operand_alternative *op_alt;
1545 24854990 : int offset = 0;
1546 24854990 : bool win;
1547 24854990 : int j;
1548 :
1549 : /* A unary operator may be accepted by the predicate, but it
1550 : is irrelevant for matching constraints. */
1551 24854990 : if (UNARY_P (op))
1552 47043 : op = XEXP (op, 0);
1553 :
1554 24854990 : if (SUBREG_P (op))
1555 : {
1556 883071 : if (REG_P (SUBREG_REG (op))
1557 883071 : && REGNO (SUBREG_REG (op)) < FIRST_PSEUDO_REGISTER)
1558 54 : offset = subreg_regno_offset (REGNO (SUBREG_REG (op)),
1559 54 : GET_MODE (SUBREG_REG (op)),
1560 54 : SUBREG_BYTE (op),
1561 54 : GET_MODE (op));
1562 883071 : op = SUBREG_REG (op);
1563 : }
1564 :
1565 24854990 : if (!(REG_P (op) && HARD_REGISTER_P (op)))
1566 24596595 : continue;
1567 :
1568 258395 : op_alt = recog_op_alt;
1569 :
1570 : /* Operand has no constraints, anything is OK. */
1571 258395 : win = !n_alternatives;
1572 :
1573 258395 : alternative_mask preferred = get_preferred_alternatives (insn);
1574 754176 : for (j = 0; j < n_alternatives; j++, op_alt += n_operands)
1575 : {
1576 492076 : if (!TEST_BIT (preferred, j))
1577 153647 : continue;
1578 338429 : if (op_alt[i].anything_ok
1579 207255 : || (op_alt[i].matches != -1
1580 35172 : && operands_match_p
1581 35172 : (recog_data.operand[i],
1582 35172 : recog_data.operand[op_alt[i].matches]))
1583 541532 : || reg_fits_class_p (op, op_alt[i].cl, offset, mode))
1584 : {
1585 : win = true;
1586 : break;
1587 : }
1588 : }
1589 :
1590 258395 : if (!win)
1591 : return false;
1592 : }
1593 :
1594 : return true;
1595 : }
1596 :
1597 : /* Implement the TARGET_ASAN_SHADOW_OFFSET hook. */
1598 :
1599 : static unsigned HOST_WIDE_INT
1600 4683 : ix86_asan_shadow_offset (void)
1601 : {
1602 4683 : return SUBTARGET_SHADOW_OFFSET;
1603 : }
1604 :
1605 : /* Argument support functions. */
1606 :
1607 : /* Return true when register may be used to pass function parameters. */
1608 : bool
1609 1552405554 : ix86_function_arg_regno_p (int regno)
1610 : {
1611 1552405554 : int i;
1612 1552405554 : enum calling_abi call_abi;
1613 1552405554 : const int *parm_regs;
1614 :
1615 1548848591 : if (TARGET_SSE && SSE_REGNO_P (regno)
1616 2546705508 : && regno < FIRST_SSE_REG + SSE_REGPARM_MAX)
1617 : return true;
1618 :
1619 1429569697 : if (!TARGET_64BIT)
1620 132128871 : return (regno < REGPARM_MAX
1621 132128871 : || (TARGET_MMX && MMX_REGNO_P (regno)
1622 11614360 : && regno < FIRST_MMX_REG + MMX_REGPARM_MAX));
1623 :
1624 : /* TODO: The function should depend on current function ABI but
1625 : builtins.cc would need updating then. Therefore we use the
1626 : default ABI. */
1627 1297440826 : call_abi = ix86_cfun_abi ();
1628 :
1629 : /* RAX is used as hidden argument to va_arg functions. */
1630 1297440826 : if (call_abi == SYSV_ABI && regno == AX_REG)
1631 : return true;
1632 :
1633 1282805060 : if (cfun
1634 1282804720 : && cfun->machine->call_saved_registers == TYPE_PRESERVE_NONE)
1635 : parm_regs = x86_64_preserve_none_int_parameter_registers;
1636 1282764160 : else if (call_abi == MS_ABI)
1637 : parm_regs = x86_64_ms_abi_int_parameter_registers;
1638 : else
1639 1244148086 : parm_regs = x86_64_int_parameter_registers;
1640 :
1641 17180970946 : for (i = 0; i < (call_abi == MS_ABI
1642 8590485473 : ? X86_64_MS_REGPARM_MAX : X86_64_REGPARM_MAX); i++)
1643 7397262199 : if (regno == parm_regs[i])
1644 : return true;
1645 : return false;
1646 : }
1647 :
1648 : /* Return if we do not know how to pass ARG solely in registers. */
1649 :
1650 : static bool
1651 425929848 : ix86_must_pass_in_stack (const function_arg_info &arg)
1652 : {
1653 425929848 : if (must_pass_in_stack_var_size_or_pad (arg))
1654 : return true;
1655 :
1656 : /* For 32-bit, we want TImode aggregates to go on the stack. But watch out!
1657 : The layout_type routine is crafty and tries to trick us into passing
1658 : currently unsupported vector types on the stack by using TImode. */
1659 1773181 : return (!TARGET_64BIT && arg.mode == TImode
1660 425929811 : && arg.type && TREE_CODE (arg.type) != VECTOR_TYPE);
1661 : }
1662 :
1663 : /* It returns the size, in bytes, of the area reserved for arguments passed
1664 : in registers for the function represented by fndecl dependent to the used
1665 : abi format. */
1666 : int
1667 11127206 : ix86_reg_parm_stack_space (const_tree fndecl)
1668 : {
1669 11127206 : enum calling_abi call_abi = SYSV_ABI;
1670 11127206 : if (fndecl != NULL_TREE && TREE_CODE (fndecl) == FUNCTION_DECL)
1671 10804641 : call_abi = ix86_function_abi (fndecl);
1672 : else
1673 322565 : call_abi = ix86_function_type_abi (fndecl);
1674 11127206 : if (TARGET_64BIT && call_abi == MS_ABI)
1675 119430 : return 32;
1676 : return 0;
1677 : }
1678 :
1679 : /* We add this as a workaround in order to use libc_has_function
1680 : hook in i386.md. */
1681 : bool
1682 0 : ix86_libc_has_function (enum function_class fn_class)
1683 : {
1684 0 : return targetm.libc_has_function (fn_class, NULL_TREE);
1685 : }
1686 :
1687 : /* Returns value SYSV_ABI, MS_ABI dependent on fntype,
1688 : specifying the call abi used. */
1689 : enum calling_abi
1690 887199794 : ix86_function_type_abi (const_tree fntype)
1691 : {
1692 887199794 : enum calling_abi abi = ix86_abi;
1693 :
1694 887199794 : if (fntype == NULL_TREE || TYPE_ATTRIBUTES (fntype) == NULL_TREE)
1695 : return abi;
1696 :
1697 90203491 : if (abi == SYSV_ABI
1698 90203491 : && lookup_attribute ("ms_abi", TYPE_ATTRIBUTES (fntype)))
1699 : {
1700 4265624 : static int warned;
1701 4265624 : if (TARGET_X32 && !warned)
1702 : {
1703 1 : error ("X32 does not support %<ms_abi%> attribute");
1704 1 : warned = 1;
1705 : }
1706 :
1707 : abi = MS_ABI;
1708 : }
1709 85937867 : else if (abi == MS_ABI
1710 85937867 : && lookup_attribute ("sysv_abi", TYPE_ATTRIBUTES (fntype)))
1711 : abi = SYSV_ABI;
1712 :
1713 : return abi;
1714 : }
1715 :
1716 : enum calling_abi
1717 229592212 : ix86_function_abi (const_tree fndecl)
1718 : {
1719 229592212 : return fndecl ? ix86_function_type_abi (TREE_TYPE (fndecl)) : ix86_abi;
1720 : }
1721 :
1722 : /* Returns value SYSV_ABI, MS_ABI dependent on cfun,
1723 : specifying the call abi used. */
1724 : enum calling_abi
1725 2175196605 : ix86_cfun_abi (void)
1726 : {
1727 2175196605 : return cfun ? cfun->machine->call_abi : ix86_abi;
1728 : }
1729 :
1730 : bool
1731 5135432 : ix86_function_ms_hook_prologue (const_tree fn)
1732 : {
1733 5135432 : if (fn && lookup_attribute ("ms_hook_prologue", DECL_ATTRIBUTES (fn)))
1734 : {
1735 8 : if (decl_function_context (fn) != NULL_TREE)
1736 0 : error_at (DECL_SOURCE_LOCATION (fn),
1737 : "%<ms_hook_prologue%> attribute is not compatible "
1738 : "with nested function");
1739 : else
1740 : return true;
1741 : }
1742 : return false;
1743 : }
1744 :
1745 : bool
1746 127099859 : ix86_function_naked (const_tree fn)
1747 : {
1748 127099859 : if (fn && lookup_attribute ("naked", DECL_ATTRIBUTES (fn)))
1749 663 : return true;
1750 :
1751 : return false;
1752 : }
1753 :
1754 : /* Write the extra assembler code needed to declare a function properly. */
1755 :
1756 : void
1757 1592525 : ix86_asm_output_function_label (FILE *out_file, const char *fname,
1758 : tree decl)
1759 : {
1760 1592525 : bool is_ms_hook = ix86_function_ms_hook_prologue (decl);
1761 :
1762 1592525 : if (cfun)
1763 1588863 : cfun->machine->function_label_emitted = true;
1764 :
1765 1592525 : if (is_ms_hook)
1766 : {
1767 2 : int i, filler_count = (TARGET_64BIT ? 32 : 16);
1768 2 : unsigned int filler_cc = 0xcccccccc;
1769 :
1770 18 : for (i = 0; i < filler_count; i += 4)
1771 16 : fprintf (out_file, ASM_LONG " %#x\n", filler_cc);
1772 : }
1773 :
1774 : #ifdef SUBTARGET_ASM_UNWIND_INIT
1775 : SUBTARGET_ASM_UNWIND_INIT (out_file);
1776 : #endif
1777 :
1778 1592525 : assemble_function_label_raw (out_file, fname);
1779 :
1780 : /* Output magic byte marker, if hot-patch attribute is set. */
1781 1592525 : if (is_ms_hook)
1782 : {
1783 2 : if (TARGET_64BIT)
1784 : {
1785 : /* leaq [%rsp + 0], %rsp */
1786 2 : fputs (ASM_BYTE "0x48, 0x8d, 0xa4, 0x24, 0x00, 0x00, 0x00, 0x00\n",
1787 : out_file);
1788 : }
1789 : else
1790 : {
1791 : /* movl.s %edi, %edi
1792 : push %ebp
1793 : movl.s %esp, %ebp */
1794 0 : fputs (ASM_BYTE "0x8b, 0xff, 0x55, 0x8b, 0xec\n", out_file);
1795 : }
1796 : }
1797 1592525 : }
1798 :
1799 : /* Output a user-defined label. In AT&T syntax, registers are prefixed
1800 : with %, so labels require no punctuation. In Intel syntax, registers
1801 : are unprefixed, so labels may clash with registers or other operators,
1802 : and require quoting. */
1803 : void
1804 35677982 : ix86_asm_output_labelref (FILE *file, const char *prefix, const char *label)
1805 : {
1806 35677982 : if (ASSEMBLER_DIALECT == ASM_ATT)
1807 35676888 : fprintf (file, "%s%s", prefix, label);
1808 : else
1809 1094 : fprintf (file, "\"%s%s\"", prefix, label);
1810 35677982 : }
1811 :
1812 : /* Implementation of call abi switching target hook. Specific to FNDECL
1813 : the specific call register sets are set. See also
1814 : ix86_conditional_register_usage for more details. */
1815 : void
1816 208253892 : ix86_call_abi_override (const_tree fndecl)
1817 : {
1818 208253892 : cfun->machine->call_abi = ix86_function_abi (fndecl);
1819 208253892 : }
1820 :
1821 : /* Return 1 if pseudo register should be created and used to hold
1822 : GOT address for PIC code. */
1823 : bool
1824 171584772 : ix86_use_pseudo_pic_reg (void)
1825 : {
1826 171584772 : if ((TARGET_64BIT
1827 160480249 : && (ix86_cmodel == CM_SMALL_PIC
1828 : || TARGET_PECOFF))
1829 160743172 : || !flag_pic)
1830 166753784 : return false;
1831 : return true;
1832 : }
1833 :
1834 : /* Initialize large model PIC register. */
1835 :
1836 : static void
1837 56 : ix86_init_large_pic_reg (unsigned int tmp_regno)
1838 : {
1839 56 : rtx_code_label *label;
1840 56 : rtx tmp_reg;
1841 :
1842 56 : gcc_assert (Pmode == DImode);
1843 56 : label = gen_label_rtx ();
1844 56 : emit_label (label);
1845 56 : LABEL_PRESERVE_P (label) = 1;
1846 56 : tmp_reg = gen_rtx_REG (Pmode, tmp_regno);
1847 56 : gcc_assert (REGNO (pic_offset_table_rtx) != tmp_regno);
1848 56 : emit_insn (gen_set_rip_rex64 (pic_offset_table_rtx,
1849 : label));
1850 56 : emit_insn (gen_set_got_offset_rex64 (tmp_reg, label));
1851 56 : emit_insn (gen_add2_insn (pic_offset_table_rtx, tmp_reg));
1852 56 : const char *name = LABEL_NAME (label);
1853 56 : PUT_CODE (label, NOTE);
1854 56 : NOTE_KIND (label) = NOTE_INSN_DELETED_LABEL;
1855 56 : NOTE_DELETED_LABEL_NAME (label) = name;
1856 56 : }
1857 :
1858 : /* Create and initialize PIC register if required. */
1859 : static void
1860 1520561 : ix86_init_pic_reg (void)
1861 : {
1862 1520561 : edge entry_edge;
1863 1520561 : rtx_insn *seq;
1864 :
1865 1520561 : if (!ix86_use_pseudo_pic_reg ())
1866 : return;
1867 :
1868 40512 : start_sequence ();
1869 :
1870 40512 : if (TARGET_64BIT)
1871 : {
1872 69 : if (ix86_cmodel == CM_LARGE_PIC)
1873 53 : ix86_init_large_pic_reg (R11_REG);
1874 : else
1875 16 : emit_insn (gen_set_got_rex64 (pic_offset_table_rtx));
1876 : }
1877 : else
1878 : {
1879 : /* If there is future mcount call in the function it is more profitable
1880 : to emit SET_GOT into ABI defined REAL_PIC_OFFSET_TABLE_REGNUM. */
1881 40443 : rtx reg = crtl->profile
1882 40443 : ? gen_rtx_REG (Pmode, REAL_PIC_OFFSET_TABLE_REGNUM)
1883 40443 : : pic_offset_table_rtx;
1884 40443 : rtx_insn *insn = emit_insn (gen_set_got (reg));
1885 40443 : RTX_FRAME_RELATED_P (insn) = 1;
1886 40443 : if (crtl->profile)
1887 0 : emit_move_insn (pic_offset_table_rtx, reg);
1888 40443 : add_reg_note (insn, REG_CFA_FLUSH_QUEUE, NULL_RTX);
1889 : }
1890 :
1891 40512 : seq = end_sequence ();
1892 :
1893 40512 : entry_edge = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1894 40512 : insert_insn_on_edge (seq, entry_edge);
1895 40512 : commit_one_edge_insertion (entry_edge);
1896 : }
1897 :
1898 : /* Initialize a variable CUM of type CUMULATIVE_ARGS
1899 : for a call to a function whose data type is FNTYPE.
1900 : For a library call, FNTYPE is 0. */
1901 :
1902 : void
1903 10839594 : init_cumulative_args (CUMULATIVE_ARGS *cum, /* Argument info to initialize */
1904 : tree fntype, /* tree ptr for function decl */
1905 : rtx libname, /* SYMBOL_REF of library name or 0 */
1906 : tree fndecl,
1907 : int caller)
1908 : {
1909 10839594 : struct cgraph_node *local_info_node = NULL;
1910 10839594 : struct cgraph_node *target = NULL;
1911 :
1912 : /* Set silent_p to false to raise an error for invalid calls when
1913 : expanding function body. */
1914 10839594 : cfun->machine->silent_p = false;
1915 :
1916 10839594 : memset (cum, 0, sizeof (*cum));
1917 :
1918 10839594 : tree preserve_none_type;
1919 10839594 : if (fndecl)
1920 : {
1921 10486736 : target = cgraph_node::get (fndecl);
1922 10486736 : if (target)
1923 : {
1924 10328707 : target = target->function_symbol ();
1925 10328707 : local_info_node = cgraph_node::local_info_node (target->decl);
1926 10328707 : cum->call_abi = ix86_function_abi (target->decl);
1927 10328707 : preserve_none_type = TREE_TYPE (target->decl);
1928 : }
1929 : else
1930 : {
1931 158029 : cum->call_abi = ix86_function_abi (fndecl);
1932 158029 : preserve_none_type = TREE_TYPE (fndecl);
1933 : }
1934 : }
1935 : else
1936 : {
1937 352858 : cum->call_abi = ix86_function_type_abi (fntype);
1938 352858 : preserve_none_type = fntype;
1939 : }
1940 :
1941 : /* For MS ABI functions, parameter passing scheme is unchanged. */
1942 10839594 : cum->preserve_none_abi
1943 10839594 : = (preserve_none_type
1944 10715484 : && cum->call_abi != MS_ABI
1945 21437968 : && (lookup_attribute ("preserve_none",
1946 10598374 : TYPE_ATTRIBUTES (preserve_none_type))
1947 : != nullptr));
1948 :
1949 10839594 : cum->caller = caller;
1950 :
1951 : /* Set up the number of registers to use for passing arguments. */
1952 10839594 : cum->nregs = ix86_regparm;
1953 10839594 : if (TARGET_64BIT)
1954 : {
1955 9802623 : cum->nregs = (cum->call_abi == SYSV_ABI
1956 9802623 : ? X86_64_REGPARM_MAX
1957 : : X86_64_MS_REGPARM_MAX);
1958 : }
1959 10839594 : if (TARGET_SSE)
1960 : {
1961 10830417 : cum->sse_nregs = SSE_REGPARM_MAX;
1962 10830417 : if (TARGET_64BIT)
1963 : {
1964 9793566 : cum->sse_nregs = (cum->call_abi == SYSV_ABI
1965 9793566 : ? X86_64_SSE_REGPARM_MAX
1966 : : X86_64_MS_SSE_REGPARM_MAX);
1967 : }
1968 : }
1969 10839594 : if (TARGET_MMX)
1970 11663681 : cum->mmx_nregs = MMX_REGPARM_MAX;
1971 10839594 : cum->warn_avx512f = true;
1972 10839594 : cum->warn_avx = true;
1973 10839594 : cum->warn_sse = true;
1974 10839594 : cum->warn_mmx = true;
1975 :
1976 : /* Because type might mismatch in between caller and callee, we need to
1977 : use actual type of function for local calls.
1978 : FIXME: cgraph_analyze can be told to actually record if function uses
1979 : va_start so for local functions maybe_vaarg can be made aggressive
1980 : helping K&R code.
1981 : FIXME: once typesytem is fixed, we won't need this code anymore. */
1982 10839594 : if (local_info_node && local_info_node->local
1983 440994 : && local_info_node->can_change_signature)
1984 417645 : fntype = TREE_TYPE (target->decl);
1985 10839594 : cum->stdarg = stdarg_p (fntype);
1986 21679188 : cum->maybe_vaarg = (fntype
1987 11317560 : ? (!prototype_p (fntype) || stdarg_p (fntype))
1988 124110 : : !libname);
1989 :
1990 10839594 : cum->decl = fndecl;
1991 :
1992 10839594 : cum->warn_empty = !warn_abi || cum->stdarg;
1993 10839594 : if (!cum->warn_empty && fntype)
1994 : {
1995 2994731 : function_args_iterator iter;
1996 2994731 : tree argtype;
1997 2994731 : bool seen_empty_type = false;
1998 8336865 : FOREACH_FUNCTION_ARGS (fntype, argtype, iter)
1999 : {
2000 8336802 : if (argtype == error_mark_node || VOID_TYPE_P (argtype))
2001 : break;
2002 5366283 : if (TYPE_EMPTY_P (argtype))
2003 : seen_empty_type = true;
2004 5281290 : else if (seen_empty_type)
2005 : {
2006 24149 : cum->warn_empty = true;
2007 24149 : break;
2008 : }
2009 : }
2010 : }
2011 :
2012 10839594 : if (!TARGET_64BIT)
2013 : {
2014 : /* If there are variable arguments, then we won't pass anything
2015 : in registers in 32-bit mode. */
2016 1036971 : if (stdarg_p (fntype))
2017 : {
2018 9155 : cum->nregs = 0;
2019 : /* Since in 32-bit, variable arguments are always passed on
2020 : stack, there is scratch register available for indirect
2021 : sibcall. */
2022 9155 : cfun->machine->arg_reg_available = true;
2023 9155 : cum->sse_nregs = 0;
2024 9155 : cum->mmx_nregs = 0;
2025 9155 : cum->warn_avx512f = false;
2026 9155 : cum->warn_avx = false;
2027 9155 : cum->warn_sse = false;
2028 9155 : cum->warn_mmx = false;
2029 9155 : return;
2030 : }
2031 :
2032 : /* Use ecx and edx registers if function has fastcall attribute,
2033 : else look for regparm information. */
2034 1027816 : if (fntype)
2035 : {
2036 1014575 : unsigned int ccvt = ix86_get_callcvt (fntype);
2037 1014575 : if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
2038 : {
2039 0 : cum->nregs = 1;
2040 0 : cum->fastcall = 1; /* Same first register as in fastcall. */
2041 : }
2042 1014575 : else if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
2043 : {
2044 4 : cum->nregs = 2;
2045 4 : cum->fastcall = 1;
2046 : }
2047 : else
2048 1014571 : cum->nregs = ix86_function_regparm (fntype, fndecl);
2049 : }
2050 :
2051 : /* Set up the number of SSE registers used for passing SFmode
2052 : and DFmode arguments. Warn for mismatching ABI. */
2053 1027816 : cum->float_in_sse = ix86_function_sseregparm (fntype, fndecl, true);
2054 : }
2055 :
2056 10830439 : cfun->machine->arg_reg_available = (cum->nregs > 0);
2057 : }
2058 :
2059 : /* Return the "natural" mode for TYPE. In most cases, this is just TYPE_MODE.
2060 : But in the case of vector types, it is some vector mode.
2061 :
2062 : When we have only some of our vector isa extensions enabled, then there
2063 : are some modes for which vector_mode_supported_p is false. For these
2064 : modes, the generic vector support in gcc will choose some non-vector mode
2065 : in order to implement the type. By computing the natural mode, we'll
2066 : select the proper ABI location for the operand and not depend on whatever
2067 : the middle-end decides to do with these vector types.
2068 :
2069 : The midde-end can't deal with the vector types > 16 bytes. In this
2070 : case, we return the original mode and warn ABI change if CUM isn't
2071 : NULL.
2072 :
2073 : If INT_RETURN is true, warn ABI change if the vector mode isn't
2074 : available for function return value. */
2075 :
2076 : static machine_mode
2077 239757087 : type_natural_mode (const_tree type, const CUMULATIVE_ARGS *cum,
2078 : bool in_return)
2079 : {
2080 239757087 : machine_mode mode = TYPE_MODE (type);
2081 :
2082 239757087 : if (VECTOR_TYPE_P (type) && !VECTOR_MODE_P (mode))
2083 : {
2084 499688 : HOST_WIDE_INT size = int_size_in_bytes (type);
2085 499688 : if ((size == 8 || size == 16 || size == 32 || size == 64)
2086 : /* ??? Generic code allows us to create width 1 vectors. Ignore. */
2087 499688 : && TYPE_VECTOR_SUBPARTS (type) > 1)
2088 : {
2089 453635 : machine_mode innermode = TYPE_MODE (TREE_TYPE (type));
2090 :
2091 : /* There are no XFmode vector modes ... */
2092 453635 : if (innermode == XFmode)
2093 : return mode;
2094 :
2095 : /* ... and no decimal float vector modes. */
2096 453082 : if (DECIMAL_FLOAT_MODE_P (innermode))
2097 : return mode;
2098 :
2099 452789 : if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (type)))
2100 : mode = MIN_MODE_VECTOR_FLOAT;
2101 : else
2102 374193 : mode = MIN_MODE_VECTOR_INT;
2103 :
2104 : /* Get the mode which has this inner mode and number of units. */
2105 9495938 : FOR_EACH_MODE_FROM (mode, mode)
2106 19769518 : if (GET_MODE_NUNITS (mode) == TYPE_VECTOR_SUBPARTS (type)
2107 10726369 : && GET_MODE_INNER (mode) == innermode)
2108 : {
2109 452789 : if (size == 64 && !TARGET_AVX512F && !TARGET_IAMCU)
2110 : {
2111 300665 : static bool warnedavx512f;
2112 300665 : static bool warnedavx512f_ret;
2113 :
2114 300665 : if (cum && cum->warn_avx512f && !warnedavx512f)
2115 : {
2116 1361 : if (warning (OPT_Wpsabi, "AVX512F vector argument "
2117 : "without AVX512F enabled changes the ABI"))
2118 2 : warnedavx512f = true;
2119 : }
2120 299304 : else if (in_return && !warnedavx512f_ret)
2121 : {
2122 283951 : if (warning (OPT_Wpsabi, "AVX512F vector return "
2123 : "without AVX512F enabled changes the ABI"))
2124 8 : warnedavx512f_ret = true;
2125 : }
2126 :
2127 300665 : return TYPE_MODE (type);
2128 : }
2129 152124 : else if (size == 32 && !TARGET_AVX && !TARGET_IAMCU)
2130 : {
2131 150198 : static bool warnedavx;
2132 150198 : static bool warnedavx_ret;
2133 :
2134 150198 : if (cum && cum->warn_avx && !warnedavx)
2135 : {
2136 770 : if (warning (OPT_Wpsabi, "AVX vector argument "
2137 : "without AVX enabled changes the ABI"))
2138 5 : warnedavx = true;
2139 : }
2140 149428 : else if (in_return && !warnedavx_ret)
2141 : {
2142 121906 : if (warning (OPT_Wpsabi, "AVX vector return "
2143 : "without AVX enabled changes the ABI"))
2144 13 : warnedavx_ret = true;
2145 : }
2146 :
2147 150198 : return TYPE_MODE (type);
2148 : }
2149 1926 : else if (((size == 8 && TARGET_64BIT) || size == 16)
2150 1923 : && !TARGET_SSE
2151 140 : && !TARGET_IAMCU)
2152 : {
2153 140 : static bool warnedsse;
2154 140 : static bool warnedsse_ret;
2155 :
2156 140 : if (cum && cum->warn_sse && !warnedsse)
2157 : {
2158 19 : if (warning (OPT_Wpsabi, "SSE vector argument "
2159 : "without SSE enabled changes the ABI"))
2160 6 : warnedsse = true;
2161 : }
2162 121 : else if (!TARGET_64BIT && in_return && !warnedsse_ret)
2163 : {
2164 0 : if (warning (OPT_Wpsabi, "SSE vector return "
2165 : "without SSE enabled changes the ABI"))
2166 0 : warnedsse_ret = true;
2167 : }
2168 : }
2169 1786 : else if ((size == 8 && !TARGET_64BIT)
2170 0 : && (!cfun
2171 0 : || cfun->machine->func_type == TYPE_NORMAL)
2172 0 : && !TARGET_MMX
2173 0 : && !TARGET_IAMCU)
2174 : {
2175 0 : static bool warnedmmx;
2176 0 : static bool warnedmmx_ret;
2177 :
2178 0 : if (cum && cum->warn_mmx && !warnedmmx)
2179 : {
2180 0 : if (warning (OPT_Wpsabi, "MMX vector argument "
2181 : "without MMX enabled changes the ABI"))
2182 0 : warnedmmx = true;
2183 : }
2184 0 : else if (in_return && !warnedmmx_ret)
2185 : {
2186 0 : if (warning (OPT_Wpsabi, "MMX vector return "
2187 : "without MMX enabled changes the ABI"))
2188 0 : warnedmmx_ret = true;
2189 : }
2190 : }
2191 : return mode;
2192 : }
2193 :
2194 0 : gcc_unreachable ();
2195 : }
2196 : }
2197 :
2198 : return mode;
2199 : }
2200 :
2201 : /* We want to pass a value in REGNO whose "natural" mode is MODE. However,
2202 : this may not agree with the mode that the type system has chosen for the
2203 : register, which is ORIG_MODE. If ORIG_MODE is not BLKmode, then we can
2204 : go ahead and use it. Otherwise we have to build a PARALLEL instead. */
2205 :
2206 : static rtx
2207 37863878 : gen_reg_or_parallel (machine_mode mode, machine_mode orig_mode,
2208 : unsigned int regno)
2209 : {
2210 37863878 : rtx tmp;
2211 :
2212 37863878 : if (orig_mode != BLKmode)
2213 37863715 : tmp = gen_rtx_REG (orig_mode, regno);
2214 : else
2215 : {
2216 163 : tmp = gen_rtx_REG (mode, regno);
2217 163 : tmp = gen_rtx_EXPR_LIST (VOIDmode, tmp, const0_rtx);
2218 163 : tmp = gen_rtx_PARALLEL (orig_mode, gen_rtvec (1, tmp));
2219 : }
2220 :
2221 37863878 : return tmp;
2222 : }
2223 :
2224 : /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
2225 : of this code is to classify each 8bytes of incoming argument by the register
2226 : class and assign registers accordingly. */
2227 :
2228 : /* Return the union class of CLASS1 and CLASS2.
2229 : See the x86-64 PS ABI for details. */
2230 :
2231 : static enum x86_64_reg_class
2232 59261133 : merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
2233 : {
2234 : /* Rule #1: If both classes are equal, this is the resulting class. */
2235 57957988 : if (class1 == class2)
2236 : return class1;
2237 :
2238 : /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
2239 : the other class. */
2240 51133134 : if (class1 == X86_64_NO_CLASS)
2241 : return class2;
2242 52000427 : if (class2 == X86_64_NO_CLASS)
2243 : return class1;
2244 :
2245 : /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
2246 1748470 : if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
2247 : return X86_64_MEMORY_CLASS;
2248 :
2249 : /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
2250 1589612 : if ((class1 == X86_64_INTEGERSI_CLASS
2251 192170 : && (class2 == X86_64_SSESF_CLASS || class2 == X86_64_SSEHF_CLASS))
2252 1588406 : || (class2 == X86_64_INTEGERSI_CLASS
2253 960079 : && (class1 == X86_64_SSESF_CLASS || class1 == X86_64_SSEHF_CLASS)))
2254 : return X86_64_INTEGERSI_CLASS;
2255 1584469 : if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
2256 418156 : || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
2257 : return X86_64_INTEGER_CLASS;
2258 :
2259 : /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
2260 : MEMORY is used. */
2261 101671 : if (class1 == X86_64_X87_CLASS
2262 : || class1 == X86_64_X87UP_CLASS
2263 101671 : || class1 == X86_64_COMPLEX_X87_CLASS
2264 : || class2 == X86_64_X87_CLASS
2265 100766 : || class2 == X86_64_X87UP_CLASS
2266 67264 : || class2 == X86_64_COMPLEX_X87_CLASS)
2267 34407 : return X86_64_MEMORY_CLASS;
2268 :
2269 : /* Rule #6: Otherwise class SSE is used. */
2270 : return X86_64_SSE_CLASS;
2271 : }
2272 :
2273 : /* Classify the argument of type TYPE and mode MODE.
2274 : CLASSES will be filled by the register class used to pass each word
2275 : of the operand. The number of words is returned. In case the parameter
2276 : should be passed in memory, 0 is returned. As a special case for zero
2277 : sized containers, classes[0] will be NO_CLASS and 1 is returned.
2278 :
2279 : BIT_OFFSET is used internally for handling records and specifies offset
2280 : of the offset in bits modulo 512 to avoid overflow cases.
2281 :
2282 : See the x86-64 PS ABI for details.
2283 : */
2284 :
2285 : static int
2286 411417800 : classify_argument (machine_mode mode, const_tree type,
2287 : enum x86_64_reg_class classes[MAX_CLASSES], int bit_offset,
2288 : int &zero_width_bitfields)
2289 : {
2290 411417800 : HOST_WIDE_INT bytes
2291 816413471 : = mode == BLKmode ? int_size_in_bytes (type) : (int) GET_MODE_SIZE (mode);
2292 411417800 : int words = CEIL (bytes + (bit_offset % 64) / 8, UNITS_PER_WORD);
2293 :
2294 : /* Variable sized entities are always passed/returned in memory. */
2295 411417800 : if (bytes < 0)
2296 : return 0;
2297 :
2298 411416601 : if (mode != VOIDmode)
2299 : {
2300 : /* The value of "named" doesn't matter. */
2301 410157041 : function_arg_info arg (const_cast<tree> (type), mode, /*named=*/true);
2302 410157041 : if (targetm.calls.must_pass_in_stack (arg))
2303 37 : return 0;
2304 : }
2305 :
2306 411416564 : if (type && (AGGREGATE_TYPE_P (type)
2307 372598176 : || (BITINT_TYPE_P (type) && words > 1)))
2308 : {
2309 39936729 : int i;
2310 39936729 : tree field;
2311 39936729 : enum x86_64_reg_class subclasses[MAX_CLASSES];
2312 :
2313 : /* On x86-64 we pass structures larger than 64 bytes on the stack. */
2314 39936729 : if (bytes > 64)
2315 : return 0;
2316 :
2317 100169527 : for (i = 0; i < words; i++)
2318 61076283 : classes[i] = X86_64_NO_CLASS;
2319 :
2320 : /* Zero sized arrays or structures are NO_CLASS. We return 0 to
2321 : signalize memory class, so handle it as special case. */
2322 39093244 : if (!words)
2323 : {
2324 85032 : classes[0] = X86_64_NO_CLASS;
2325 85032 : return 1;
2326 : }
2327 :
2328 : /* Classify each field of record and merge classes. */
2329 39008212 : switch (TREE_CODE (type))
2330 : {
2331 36883765 : case RECORD_TYPE:
2332 : /* And now merge the fields of structure. */
2333 1023778375 : for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
2334 : {
2335 987454376 : if (TREE_CODE (field) == FIELD_DECL)
2336 : {
2337 54129502 : int num;
2338 :
2339 54129502 : if (TREE_TYPE (field) == error_mark_node)
2340 4 : continue;
2341 :
2342 : /* Bitfields are always classified as integer. Handle them
2343 : early, since later code would consider them to be
2344 : misaligned integers. */
2345 54129498 : if (DECL_BIT_FIELD (field))
2346 : {
2347 1312364 : if (integer_zerop (DECL_SIZE (field)))
2348 : {
2349 12839 : if (DECL_FIELD_CXX_ZERO_WIDTH_BIT_FIELD (field))
2350 8021 : continue;
2351 4818 : if (zero_width_bitfields != 2)
2352 : {
2353 4284 : zero_width_bitfields = 1;
2354 4284 : continue;
2355 : }
2356 : }
2357 1300059 : for (i = (int_bit_position (field)
2358 1300059 : + (bit_offset % 64)) / 8 / 8;
2359 2603204 : i < ((int_bit_position (field) + (bit_offset % 64))
2360 2603204 : + tree_to_shwi (DECL_SIZE (field))
2361 2603204 : + 63) / 8 / 8; i++)
2362 1303145 : classes[i]
2363 2606290 : = merge_classes (X86_64_INTEGER_CLASS, classes[i]);
2364 : }
2365 : else
2366 : {
2367 52817134 : int pos;
2368 :
2369 52817134 : type = TREE_TYPE (field);
2370 :
2371 : /* Flexible array member is ignored. */
2372 52817134 : if (TYPE_MODE (type) == BLKmode
2373 709007 : && TREE_CODE (type) == ARRAY_TYPE
2374 178408 : && TYPE_SIZE (type) == NULL_TREE
2375 2007 : && TYPE_DOMAIN (type) != NULL_TREE
2376 52818376 : && (TYPE_MAX_VALUE (TYPE_DOMAIN (type))
2377 : == NULL_TREE))
2378 : {
2379 1242 : static bool warned;
2380 :
2381 1242 : if (!warned && warn_psabi)
2382 : {
2383 3 : warned = true;
2384 3 : inform (input_location,
2385 : "the ABI of passing struct with"
2386 : " a flexible array member has"
2387 : " changed in GCC 4.4");
2388 : }
2389 1242 : continue;
2390 1242 : }
2391 52815892 : num = classify_argument (TYPE_MODE (type), type,
2392 : subclasses,
2393 52815892 : (int_bit_position (field)
2394 52815892 : + bit_offset) % 512,
2395 : zero_width_bitfields);
2396 52815892 : if (!num)
2397 : return 0;
2398 52256126 : pos = (int_bit_position (field)
2399 52256126 : + (bit_offset % 64)) / 8 / 8;
2400 108150971 : for (i = 0; i < num && (i + pos) < words; i++)
2401 55894845 : classes[i + pos]
2402 55894845 : = merge_classes (subclasses[i], classes[i + pos]);
2403 : }
2404 : }
2405 : }
2406 : break;
2407 :
2408 494338 : case ARRAY_TYPE:
2409 : /* Arrays are handled as small records. */
2410 494338 : {
2411 494338 : int num;
2412 494338 : num = classify_argument (TYPE_MODE (TREE_TYPE (type)),
2413 494338 : TREE_TYPE (type), subclasses, bit_offset,
2414 : zero_width_bitfields);
2415 494338 : if (!num)
2416 : return 0;
2417 :
2418 : /* The partial classes are now full classes. */
2419 477818 : if (subclasses[0] == X86_64_SSESF_CLASS && bytes != 4)
2420 13959 : subclasses[0] = X86_64_SSE_CLASS;
2421 477818 : if (subclasses[0] == X86_64_SSEHF_CLASS && bytes != 2)
2422 5292 : subclasses[0] = X86_64_SSE_CLASS;
2423 477818 : if (subclasses[0] == X86_64_INTEGERSI_CLASS
2424 175988 : && !((bit_offset % 64) == 0 && bytes == 4))
2425 138185 : subclasses[0] = X86_64_INTEGER_CLASS;
2426 :
2427 1445120 : for (i = 0; i < words; i++)
2428 967302 : classes[i] = subclasses[i % num];
2429 :
2430 : break;
2431 : }
2432 287775 : case UNION_TYPE:
2433 287775 : case QUAL_UNION_TYPE:
2434 : /* Unions are similar to RECORD_TYPE but offset is always 0.
2435 : */
2436 3304839 : for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
2437 : {
2438 3052322 : if (TREE_CODE (field) == FIELD_DECL)
2439 : {
2440 1318882 : int num;
2441 :
2442 1318882 : if (TREE_TYPE (field) == error_mark_node)
2443 10 : continue;
2444 :
2445 1318872 : num = classify_argument (TYPE_MODE (TREE_TYPE (field)),
2446 1318872 : TREE_TYPE (field), subclasses,
2447 : bit_offset, zero_width_bitfields);
2448 1318872 : if (!num)
2449 : return 0;
2450 3346757 : for (i = 0; i < num && i < words; i++)
2451 2063143 : classes[i] = merge_classes (subclasses[i], classes[i]);
2452 : }
2453 : }
2454 : break;
2455 :
2456 1342334 : case BITINT_TYPE:
2457 1342334 : case ENUMERAL_TYPE:
2458 : /* _BitInt(N) for N > 64 is passed as structure containing
2459 : (N + 63) / 64 64-bit elements. */
2460 1342334 : if (words > 2)
2461 : return 0;
2462 77427 : classes[0] = classes[1] = X86_64_INTEGER_CLASS;
2463 77427 : return 2;
2464 :
2465 0 : default:
2466 0 : gcc_unreachable ();
2467 : }
2468 :
2469 37054334 : if (words > 2)
2470 : {
2471 : /* When size > 16 bytes, if the first one isn't
2472 : X86_64_SSE_CLASS or any other ones aren't
2473 : X86_64_SSEUP_CLASS, everything should be passed in
2474 : memory. */
2475 1734044 : if (classes[0] != X86_64_SSE_CLASS)
2476 : return 0;
2477 :
2478 203643 : for (i = 1; i < words; i++)
2479 185456 : if (classes[i] != X86_64_SSEUP_CLASS)
2480 : return 0;
2481 : }
2482 :
2483 : /* Final merger cleanup. */
2484 82955352 : for (i = 0; i < words; i++)
2485 : {
2486 : /* If one class is MEMORY, everything should be passed in
2487 : memory. */
2488 47652939 : if (classes[i] == X86_64_MEMORY_CLASS)
2489 : return 0;
2490 :
2491 : /* The X86_64_SSEUP_CLASS should be always preceded by
2492 : X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
2493 47619245 : if (classes[i] == X86_64_SSEUP_CLASS
2494 297204 : && classes[i - 1] != X86_64_SSE_CLASS
2495 76546 : && classes[i - 1] != X86_64_SSEUP_CLASS)
2496 : {
2497 : /* The first one should never be X86_64_SSEUP_CLASS. */
2498 1916 : gcc_assert (i != 0);
2499 1916 : classes[i] = X86_64_SSE_CLASS;
2500 : }
2501 :
2502 : /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
2503 : everything should be passed in memory. */
2504 47619245 : if (classes[i] == X86_64_X87UP_CLASS
2505 186831 : && (classes[i - 1] != X86_64_X87_CLASS))
2506 : {
2507 2370 : static bool warned;
2508 :
2509 : /* The first one should never be X86_64_X87UP_CLASS. */
2510 2370 : gcc_assert (i != 0);
2511 2370 : if (!warned && warn_psabi)
2512 : {
2513 1 : warned = true;
2514 1 : inform (input_location,
2515 : "the ABI of passing union with %<long double%>"
2516 : " has changed in GCC 4.4");
2517 : }
2518 : return 0;
2519 : }
2520 : }
2521 : return words;
2522 : }
2523 :
2524 : /* Compute alignment needed. We align all types to natural boundaries with
2525 : exception of XFmode that is aligned to 64bits. */
2526 371479835 : if (mode != VOIDmode && mode != BLKmode)
2527 : {
2528 369725471 : int mode_alignment = GET_MODE_BITSIZE (mode);
2529 :
2530 369725471 : if (mode == XFmode)
2531 : mode_alignment = 128;
2532 362529064 : else if (mode == XCmode)
2533 572503 : mode_alignment = 256;
2534 369725471 : if (COMPLEX_MODE_P (mode))
2535 2745872 : mode_alignment /= 2;
2536 : /* Misaligned fields are always returned in memory. */
2537 369725471 : if (bit_offset % mode_alignment)
2538 : return 0;
2539 : }
2540 :
2541 : /* for V1xx modes, just use the base mode */
2542 371472202 : if (VECTOR_MODE_P (mode) && mode != V1DImode && mode != V1TImode
2543 468239891 : && GET_MODE_UNIT_SIZE (mode) == bytes)
2544 6651 : mode = GET_MODE_INNER (mode);
2545 :
2546 : /* Classification of atomic types. */
2547 371472202 : switch (mode)
2548 : {
2549 206005 : case E_SDmode:
2550 206005 : case E_DDmode:
2551 206005 : classes[0] = X86_64_SSE_CLASS;
2552 206005 : return 1;
2553 98417 : case E_TDmode:
2554 98417 : classes[0] = X86_64_SSE_CLASS;
2555 98417 : classes[1] = X86_64_SSEUP_CLASS;
2556 98417 : return 2;
2557 243522399 : case E_DImode:
2558 243522399 : case E_SImode:
2559 243522399 : case E_HImode:
2560 243522399 : case E_QImode:
2561 243522399 : case E_CSImode:
2562 243522399 : case E_CHImode:
2563 243522399 : case E_CQImode:
2564 243522399 : {
2565 243522399 : int size = bit_offset + (int) GET_MODE_BITSIZE (mode);
2566 :
2567 : /* Analyze last 128 bits only. */
2568 243522399 : size = (size - 1) & 0x7f;
2569 :
2570 243522399 : if (size < 32)
2571 : {
2572 106647625 : classes[0] = X86_64_INTEGERSI_CLASS;
2573 106647625 : return 1;
2574 : }
2575 136874774 : else if (size < 64)
2576 : {
2577 125472412 : classes[0] = X86_64_INTEGER_CLASS;
2578 125472412 : return 1;
2579 : }
2580 11402362 : else if (size < 64+32)
2581 : {
2582 4256064 : classes[0] = X86_64_INTEGER_CLASS;
2583 4256064 : classes[1] = X86_64_INTEGERSI_CLASS;
2584 4256064 : return 2;
2585 : }
2586 7146298 : else if (size < 64+64)
2587 : {
2588 7146298 : classes[0] = classes[1] = X86_64_INTEGER_CLASS;
2589 7146298 : return 2;
2590 : }
2591 : else
2592 : gcc_unreachable ();
2593 : }
2594 2582217 : case E_CDImode:
2595 2582217 : case E_TImode:
2596 2582217 : classes[0] = classes[1] = X86_64_INTEGER_CLASS;
2597 2582217 : return 2;
2598 0 : case E_COImode:
2599 0 : case E_OImode:
2600 : /* OImode shouldn't be used directly. */
2601 0 : gcc_unreachable ();
2602 : case E_CTImode:
2603 : return 0;
2604 1033328 : case E_HFmode:
2605 1033328 : case E_BFmode:
2606 1033328 : if (!(bit_offset % 64))
2607 1020114 : classes[0] = X86_64_SSEHF_CLASS;
2608 : else
2609 13214 : classes[0] = X86_64_SSE_CLASS;
2610 : return 1;
2611 10151690 : case E_SFmode:
2612 10151690 : if (!(bit_offset % 64))
2613 10097752 : classes[0] = X86_64_SSESF_CLASS;
2614 : else
2615 53938 : classes[0] = X86_64_SSE_CLASS;
2616 : return 1;
2617 4490125 : case E_DFmode:
2618 4490125 : classes[0] = X86_64_SSEDF_CLASS;
2619 4490125 : return 1;
2620 7195691 : case E_XFmode:
2621 7195691 : classes[0] = X86_64_X87_CLASS;
2622 7195691 : classes[1] = X86_64_X87UP_CLASS;
2623 7195691 : return 2;
2624 1409775 : case E_TFmode:
2625 1409775 : classes[0] = X86_64_SSE_CLASS;
2626 1409775 : classes[1] = X86_64_SSEUP_CLASS;
2627 1409775 : return 2;
2628 106067 : case E_HCmode:
2629 106067 : case E_BCmode:
2630 106067 : classes[0] = X86_64_SSE_CLASS;
2631 106067 : if (!(bit_offset % 64))
2632 : return 1;
2633 : else
2634 : {
2635 98 : classes[1] = X86_64_SSEHF_CLASS;
2636 98 : return 2;
2637 : }
2638 734531 : case E_SCmode:
2639 734531 : classes[0] = X86_64_SSE_CLASS;
2640 734531 : if (!(bit_offset % 64))
2641 : return 1;
2642 : else
2643 : {
2644 1119 : static bool warned;
2645 :
2646 1119 : if (!warned && warn_psabi)
2647 : {
2648 2 : warned = true;
2649 2 : inform (input_location,
2650 : "the ABI of passing structure with %<complex float%>"
2651 : " member has changed in GCC 4.4");
2652 : }
2653 1119 : classes[1] = X86_64_SSESF_CLASS;
2654 1119 : return 2;
2655 : }
2656 742808 : case E_DCmode:
2657 742808 : classes[0] = X86_64_SSEDF_CLASS;
2658 742808 : classes[1] = X86_64_SSEDF_CLASS;
2659 742808 : return 2;
2660 572503 : case E_XCmode:
2661 572503 : classes[0] = X86_64_COMPLEX_X87_CLASS;
2662 572503 : return 1;
2663 : case E_TCmode:
2664 : /* This modes is larger than 16 bytes. */
2665 : return 0;
2666 25806029 : case E_V8SFmode:
2667 25806029 : case E_V8SImode:
2668 25806029 : case E_V32QImode:
2669 25806029 : case E_V16HFmode:
2670 25806029 : case E_V16BFmode:
2671 25806029 : case E_V16HImode:
2672 25806029 : case E_V4DFmode:
2673 25806029 : case E_V4DImode:
2674 25806029 : classes[0] = X86_64_SSE_CLASS;
2675 25806029 : classes[1] = X86_64_SSEUP_CLASS;
2676 25806029 : classes[2] = X86_64_SSEUP_CLASS;
2677 25806029 : classes[3] = X86_64_SSEUP_CLASS;
2678 25806029 : return 4;
2679 28040386 : case E_V8DFmode:
2680 28040386 : case E_V16SFmode:
2681 28040386 : case E_V32HFmode:
2682 28040386 : case E_V32BFmode:
2683 28040386 : case E_V8DImode:
2684 28040386 : case E_V16SImode:
2685 28040386 : case E_V32HImode:
2686 28040386 : case E_V64QImode:
2687 28040386 : classes[0] = X86_64_SSE_CLASS;
2688 28040386 : classes[1] = X86_64_SSEUP_CLASS;
2689 28040386 : classes[2] = X86_64_SSEUP_CLASS;
2690 28040386 : classes[3] = X86_64_SSEUP_CLASS;
2691 28040386 : classes[4] = X86_64_SSEUP_CLASS;
2692 28040386 : classes[5] = X86_64_SSEUP_CLASS;
2693 28040386 : classes[6] = X86_64_SSEUP_CLASS;
2694 28040386 : classes[7] = X86_64_SSEUP_CLASS;
2695 28040386 : return 8;
2696 39583269 : case E_V4SFmode:
2697 39583269 : case E_V4SImode:
2698 39583269 : case E_V16QImode:
2699 39583269 : case E_V8HImode:
2700 39583269 : case E_V8HFmode:
2701 39583269 : case E_V8BFmode:
2702 39583269 : case E_V2DFmode:
2703 39583269 : case E_V2DImode:
2704 39583269 : classes[0] = X86_64_SSE_CLASS;
2705 39583269 : classes[1] = X86_64_SSEUP_CLASS;
2706 39583269 : return 2;
2707 3299017 : case E_V1TImode:
2708 3299017 : case E_V1DImode:
2709 3299017 : case E_V2SFmode:
2710 3299017 : case E_V2SImode:
2711 3299017 : case E_V4HImode:
2712 3299017 : case E_V4HFmode:
2713 3299017 : case E_V4BFmode:
2714 3299017 : case E_V2HFmode:
2715 3299017 : case E_V2BFmode:
2716 3299017 : case E_V8QImode:
2717 3299017 : classes[0] = X86_64_SSE_CLASS;
2718 3299017 : return 1;
2719 : case E_BLKmode:
2720 : case E_VOIDmode:
2721 : return 0;
2722 52801 : default:
2723 52801 : gcc_assert (VECTOR_MODE_P (mode));
2724 :
2725 52801 : if (bytes > 16)
2726 : return 0;
2727 :
2728 75872 : gcc_assert (GET_MODE_CLASS (GET_MODE_INNER (mode)) == MODE_INT);
2729 :
2730 75872 : if (bit_offset + GET_MODE_BITSIZE (mode) <= 32)
2731 : classes[0] = X86_64_INTEGERSI_CLASS;
2732 : else
2733 434 : classes[0] = X86_64_INTEGER_CLASS;
2734 37936 : classes[1] = X86_64_INTEGER_CLASS;
2735 37936 : return 1 + (bytes > 8);
2736 : }
2737 : }
2738 :
2739 : /* Wrapper around classify_argument with the extra zero_width_bitfields
2740 : argument, to diagnose GCC 12.1 ABI differences for C. */
2741 :
2742 : static int
2743 356788164 : classify_argument (machine_mode mode, const_tree type,
2744 : enum x86_64_reg_class classes[MAX_CLASSES], int bit_offset)
2745 : {
2746 356788164 : int zero_width_bitfields = 0;
2747 356788164 : static bool warned = false;
2748 356788164 : int n = classify_argument (mode, type, classes, bit_offset,
2749 : zero_width_bitfields);
2750 356788164 : if (!zero_width_bitfields || warned || !warn_psabi)
2751 : return n;
2752 534 : enum x86_64_reg_class alt_classes[MAX_CLASSES];
2753 534 : zero_width_bitfields = 2;
2754 534 : if (classify_argument (mode, type, alt_classes, bit_offset,
2755 : zero_width_bitfields) != n)
2756 0 : zero_width_bitfields = 3;
2757 : else
2758 1286 : for (int i = 0; i < n; i++)
2759 760 : if (classes[i] != alt_classes[i])
2760 : {
2761 8 : zero_width_bitfields = 3;
2762 8 : break;
2763 : }
2764 534 : if (zero_width_bitfields == 3)
2765 : {
2766 8 : warned = true;
2767 8 : const char *url
2768 : = CHANGES_ROOT_URL "gcc-12/changes.html#zero_width_bitfields";
2769 :
2770 8 : inform (input_location,
2771 : "the ABI of passing C structures with zero-width bit-fields"
2772 : " has changed in GCC %{12.1%}", url);
2773 : }
2774 : return n;
2775 : }
2776 :
2777 : /* Examine the argument and return set number of register required in each
2778 : class. Return true iff parameter should be passed in memory. */
2779 :
2780 : static bool
2781 241408518 : examine_argument (machine_mode mode, const_tree type, bool in_return,
2782 : int *int_nregs, int *sse_nregs)
2783 : {
2784 241408518 : enum x86_64_reg_class regclass[MAX_CLASSES];
2785 241408518 : int n = classify_argument (mode, type, regclass, 0);
2786 :
2787 241408518 : *int_nregs = 0;
2788 241408518 : *sse_nregs = 0;
2789 :
2790 241408518 : if (!n)
2791 : return true;
2792 693889766 : for (n--; n >= 0; n--)
2793 458314377 : switch (regclass[n])
2794 : {
2795 161403411 : case X86_64_INTEGER_CLASS:
2796 161403411 : case X86_64_INTEGERSI_CLASS:
2797 161403411 : (*int_nregs)++;
2798 161403411 : break;
2799 77339757 : case X86_64_SSE_CLASS:
2800 77339757 : case X86_64_SSEHF_CLASS:
2801 77339757 : case X86_64_SSESF_CLASS:
2802 77339757 : case X86_64_SSEDF_CLASS:
2803 77339757 : (*sse_nregs)++;
2804 77339757 : break;
2805 : case X86_64_NO_CLASS:
2806 : case X86_64_SSEUP_CLASS:
2807 : break;
2808 9825982 : case X86_64_X87_CLASS:
2809 9825982 : case X86_64_X87UP_CLASS:
2810 9825982 : case X86_64_COMPLEX_X87_CLASS:
2811 9825982 : if (!in_return)
2812 : return true;
2813 : break;
2814 0 : case X86_64_MEMORY_CLASS:
2815 0 : gcc_unreachable ();
2816 : }
2817 :
2818 : return false;
2819 : }
2820 :
2821 : /* Construct container for the argument used by GCC interface. See
2822 : FUNCTION_ARG for the detailed description. */
2823 :
2824 : static rtx
2825 117006715 : construct_container (machine_mode mode, machine_mode orig_mode,
2826 : const_tree type, bool in_return, int nintregs,
2827 : int nsseregs, const int *intreg, int sse_regno)
2828 : {
2829 : /* The following variables hold the static issued_error state. */
2830 117006715 : static bool issued_sse_arg_error;
2831 117006715 : static bool issued_sse_ret_error;
2832 117006715 : static bool issued_x87_ret_error;
2833 :
2834 117006715 : machine_mode tmpmode;
2835 117006715 : int bytes
2836 233311949 : = mode == BLKmode ? int_size_in_bytes (type) : (int) GET_MODE_SIZE (mode);
2837 117006715 : enum x86_64_reg_class regclass[MAX_CLASSES];
2838 117006715 : int n;
2839 117006715 : int i;
2840 117006715 : int nexps = 0;
2841 117006715 : int needed_sseregs, needed_intregs;
2842 117006715 : rtx exp[MAX_CLASSES];
2843 117006715 : rtx ret;
2844 :
2845 117006715 : if (examine_argument (mode, type, in_return, &needed_intregs,
2846 : &needed_sseregs))
2847 : return NULL;
2848 :
2849 116486777 : if (needed_intregs > nintregs || needed_sseregs > nsseregs)
2850 : return NULL;
2851 :
2852 : /* We allowed the user to turn off SSE for kernel mode. Don't crash if
2853 : some less clueful developer tries to use floating-point anyway. */
2854 115379714 : if (needed_sseregs
2855 38218596 : && (!TARGET_SSE || (VALID_SSE2_TYPE_MODE (mode) && !TARGET_SSE2)))
2856 : {
2857 : /* Return early if we shouldn't raise an error for invalid
2858 : calls. */
2859 68 : if (cfun != NULL && cfun->machine->silent_p)
2860 : return NULL;
2861 36 : if (in_return)
2862 : {
2863 31 : if (!issued_sse_ret_error)
2864 : {
2865 13 : if (VALID_SSE2_TYPE_MODE (mode))
2866 5 : error ("SSE register return with SSE2 disabled");
2867 : else
2868 8 : error ("SSE register return with SSE disabled");
2869 13 : issued_sse_ret_error = true;
2870 : }
2871 : }
2872 5 : else if (!issued_sse_arg_error)
2873 : {
2874 5 : if (VALID_SSE2_TYPE_MODE (mode))
2875 0 : error ("SSE register argument with SSE2 disabled");
2876 : else
2877 5 : error ("SSE register argument with SSE disabled");
2878 5 : issued_sse_arg_error = true;
2879 : }
2880 : return NULL;
2881 : }
2882 :
2883 115379646 : n = classify_argument (mode, type, regclass, 0);
2884 115379646 : gcc_assert (n);
2885 :
2886 : /* Likewise, error if the ABI requires us to return values in the
2887 : x87 registers and the user specified -mno-80387. */
2888 115379646 : if (!TARGET_FLOAT_RETURNS_IN_80387 && in_return)
2889 1435853 : for (i = 0; i < n; i++)
2890 758306 : if (regclass[i] == X86_64_X87_CLASS
2891 : || regclass[i] == X86_64_X87UP_CLASS
2892 758306 : || regclass[i] == X86_64_COMPLEX_X87_CLASS)
2893 : {
2894 : /* Return early if we shouldn't raise an error for invalid
2895 : calls. */
2896 13 : if (cfun != NULL && cfun->machine->silent_p)
2897 : return NULL;
2898 10 : if (!issued_x87_ret_error)
2899 : {
2900 5 : error ("x87 register return with x87 disabled");
2901 5 : issued_x87_ret_error = true;
2902 : }
2903 : return NULL;
2904 : }
2905 :
2906 : /* First construct simple cases. Avoid SCmode, since we want to use
2907 : single register to pass this type. */
2908 115379633 : if (n == 1 && mode != SCmode && mode != HCmode)
2909 77006883 : switch (regclass[0])
2910 : {
2911 70669318 : case X86_64_INTEGER_CLASS:
2912 70669318 : case X86_64_INTEGERSI_CLASS:
2913 70669318 : return gen_rtx_REG (mode, intreg[0]);
2914 6129348 : case X86_64_SSE_CLASS:
2915 6129348 : case X86_64_SSEHF_CLASS:
2916 6129348 : case X86_64_SSESF_CLASS:
2917 6129348 : case X86_64_SSEDF_CLASS:
2918 6129348 : if (mode != BLKmode)
2919 12257836 : return gen_reg_or_parallel (mode, orig_mode,
2920 12257836 : GET_SSE_REGNO (sse_regno));
2921 : break;
2922 178402 : case X86_64_X87_CLASS:
2923 178402 : case X86_64_COMPLEX_X87_CLASS:
2924 178402 : return gen_rtx_REG (mode, FIRST_STACK_REG);
2925 : case X86_64_NO_CLASS:
2926 : /* Zero sized array, struct or class. */
2927 : return NULL;
2928 0 : default:
2929 0 : gcc_unreachable ();
2930 : }
2931 38373180 : if (n == 2
2932 20213456 : && regclass[0] == X86_64_SSE_CLASS
2933 13665507 : && regclass[1] == X86_64_SSEUP_CLASS
2934 13659896 : && mode != BLKmode)
2935 27319792 : return gen_reg_or_parallel (mode, orig_mode,
2936 27319792 : GET_SSE_REGNO (sse_regno));
2937 24713284 : if (n == 4
2938 8585843 : && regclass[0] == X86_64_SSE_CLASS
2939 8585843 : && regclass[1] == X86_64_SSEUP_CLASS
2940 8585843 : && regclass[2] == X86_64_SSEUP_CLASS
2941 8585843 : && regclass[3] == X86_64_SSEUP_CLASS
2942 8585843 : && mode != BLKmode)
2943 17168308 : return gen_reg_or_parallel (mode, orig_mode,
2944 17168308 : GET_SSE_REGNO (sse_regno));
2945 16129130 : if (n == 8
2946 9316509 : && regclass[0] == X86_64_SSE_CLASS
2947 9316509 : && regclass[1] == X86_64_SSEUP_CLASS
2948 9316509 : && regclass[2] == X86_64_SSEUP_CLASS
2949 9316509 : && regclass[3] == X86_64_SSEUP_CLASS
2950 9316509 : && regclass[4] == X86_64_SSEUP_CLASS
2951 9316509 : && regclass[5] == X86_64_SSEUP_CLASS
2952 9316509 : && regclass[6] == X86_64_SSEUP_CLASS
2953 9316509 : && regclass[7] == X86_64_SSEUP_CLASS
2954 9316509 : && mode != BLKmode)
2955 18628746 : return gen_reg_or_parallel (mode, orig_mode,
2956 18628746 : GET_SSE_REGNO (sse_regno));
2957 6814757 : if (n == 2
2958 6553560 : && regclass[0] == X86_64_X87_CLASS
2959 2347808 : && regclass[1] == X86_64_X87UP_CLASS)
2960 2347808 : return gen_rtx_REG (XFmode, FIRST_STACK_REG);
2961 :
2962 4466949 : if (n == 2
2963 4205752 : && regclass[0] == X86_64_INTEGER_CLASS
2964 3758566 : && regclass[1] == X86_64_INTEGER_CLASS
2965 3750125 : && (mode == CDImode || mode == TImode || mode == BLKmode)
2966 3750125 : && intreg[0] + 1 == intreg[1])
2967 : {
2968 3426630 : if (mode == BLKmode)
2969 : {
2970 : /* Use TImode for BLKmode values in 2 integer registers. */
2971 541884 : exp[0] = gen_rtx_EXPR_LIST (VOIDmode,
2972 270942 : gen_rtx_REG (TImode, intreg[0]),
2973 : GEN_INT (0));
2974 270942 : ret = gen_rtx_PARALLEL (mode, rtvec_alloc (1));
2975 270942 : XVECEXP (ret, 0, 0) = exp[0];
2976 270942 : return ret;
2977 : }
2978 : else
2979 3155688 : return gen_rtx_REG (mode, intreg[0]);
2980 : }
2981 :
2982 : /* Otherwise figure out the entries of the PARALLEL. */
2983 2859760 : for (i = 0; i < n; i++)
2984 : {
2985 1819441 : int pos;
2986 :
2987 1819441 : switch (regclass[i])
2988 : {
2989 : case X86_64_NO_CLASS:
2990 : break;
2991 1021671 : case X86_64_INTEGER_CLASS:
2992 1021671 : case X86_64_INTEGERSI_CLASS:
2993 : /* Merge TImodes on aligned occasions here too. */
2994 1021671 : if (i * 8 + 8 > bytes)
2995 : {
2996 3268 : unsigned int tmpbits = (bytes - i * 8) * BITS_PER_UNIT;
2997 3268 : if (!int_mode_for_size (tmpbits, 0).exists (&tmpmode))
2998 : /* We've requested 24 bytes we
2999 : don't have mode for. Use DImode. */
3000 357 : tmpmode = DImode;
3001 : }
3002 1018403 : else if (regclass[i] == X86_64_INTEGERSI_CLASS)
3003 : tmpmode = SImode;
3004 : else
3005 836854 : tmpmode = DImode;
3006 2043342 : exp [nexps++]
3007 1021671 : = gen_rtx_EXPR_LIST (VOIDmode,
3008 1021671 : gen_rtx_REG (tmpmode, *intreg),
3009 1021671 : GEN_INT (i*8));
3010 1021671 : intreg++;
3011 1021671 : break;
3012 600 : case X86_64_SSEHF_CLASS:
3013 600 : tmpmode = (mode == BFmode ? BFmode : HFmode);
3014 1200 : exp [nexps++]
3015 1200 : = gen_rtx_EXPR_LIST (VOIDmode,
3016 : gen_rtx_REG (tmpmode,
3017 600 : GET_SSE_REGNO (sse_regno)),
3018 600 : GEN_INT (i*8));
3019 600 : sse_regno++;
3020 600 : break;
3021 3060 : case X86_64_SSESF_CLASS:
3022 6120 : exp [nexps++]
3023 6120 : = gen_rtx_EXPR_LIST (VOIDmode,
3024 : gen_rtx_REG (SFmode,
3025 3060 : GET_SSE_REGNO (sse_regno)),
3026 3060 : GEN_INT (i*8));
3027 3060 : sse_regno++;
3028 3060 : break;
3029 513692 : case X86_64_SSEDF_CLASS:
3030 1027384 : exp [nexps++]
3031 1027384 : = gen_rtx_EXPR_LIST (VOIDmode,
3032 : gen_rtx_REG (DFmode,
3033 513692 : GET_SSE_REGNO (sse_regno)),
3034 513692 : GEN_INT (i*8));
3035 513692 : sse_regno++;
3036 513692 : break;
3037 272015 : case X86_64_SSE_CLASS:
3038 272015 : pos = i;
3039 272015 : switch (n)
3040 : {
3041 : case 1:
3042 : tmpmode = DImode;
3043 : break;
3044 11032 : case 2:
3045 11032 : if (i == 0 && regclass[1] == X86_64_SSEUP_CLASS)
3046 : {
3047 0 : tmpmode = TImode;
3048 0 : i++;
3049 : }
3050 : else
3051 : tmpmode = DImode;
3052 : break;
3053 1689 : case 4:
3054 1689 : gcc_assert (i == 0
3055 : && regclass[1] == X86_64_SSEUP_CLASS
3056 : && regclass[2] == X86_64_SSEUP_CLASS
3057 : && regclass[3] == X86_64_SSEUP_CLASS);
3058 : tmpmode = OImode;
3059 : i += 3;
3060 : break;
3061 2136 : case 8:
3062 2136 : gcc_assert (i == 0
3063 : && regclass[1] == X86_64_SSEUP_CLASS
3064 : && regclass[2] == X86_64_SSEUP_CLASS
3065 : && regclass[3] == X86_64_SSEUP_CLASS
3066 : && regclass[4] == X86_64_SSEUP_CLASS
3067 : && regclass[5] == X86_64_SSEUP_CLASS
3068 : && regclass[6] == X86_64_SSEUP_CLASS
3069 : && regclass[7] == X86_64_SSEUP_CLASS);
3070 : tmpmode = XImode;
3071 : i += 7;
3072 : break;
3073 0 : default:
3074 0 : gcc_unreachable ();
3075 : }
3076 544030 : exp [nexps++]
3077 544030 : = gen_rtx_EXPR_LIST (VOIDmode,
3078 : gen_rtx_REG (tmpmode,
3079 272015 : GET_SSE_REGNO (sse_regno)),
3080 272015 : GEN_INT (pos*8));
3081 272015 : sse_regno++;
3082 272015 : break;
3083 0 : default:
3084 0 : gcc_unreachable ();
3085 : }
3086 : }
3087 :
3088 : /* Empty aligned struct, union or class. */
3089 1040319 : if (nexps == 0)
3090 : return NULL;
3091 :
3092 1040064 : ret = gen_rtx_PARALLEL (mode, rtvec_alloc (nexps));
3093 2851102 : for (i = 0; i < nexps; i++)
3094 1811038 : XVECEXP (ret, 0, i) = exp [i];
3095 : return ret;
3096 : }
3097 :
3098 : /* Update the data in CUM to advance over an argument of mode MODE
3099 : and data type TYPE. (TYPE is null for libcalls where that information
3100 : may not be available.)
3101 :
3102 : Return a number of integer registers advanced over. */
3103 :
3104 : static int
3105 2131561 : function_arg_advance_32 (CUMULATIVE_ARGS *cum, machine_mode mode,
3106 : const_tree type, HOST_WIDE_INT bytes,
3107 : HOST_WIDE_INT words)
3108 : {
3109 2131561 : int res = 0;
3110 2131561 : bool error_p = false;
3111 :
3112 2131561 : if (TARGET_IAMCU)
3113 : {
3114 : /* Intel MCU psABI passes scalars and aggregates no larger than 8
3115 : bytes in registers. */
3116 0 : if (!VECTOR_MODE_P (mode) && bytes <= 8)
3117 0 : goto pass_in_reg;
3118 : return res;
3119 : }
3120 :
3121 2131561 : switch (mode)
3122 : {
3123 : default:
3124 : break;
3125 :
3126 93695 : case E_BLKmode:
3127 93695 : if (bytes < 0)
3128 : break;
3129 : /* FALLTHRU */
3130 :
3131 2094696 : case E_DImode:
3132 2094696 : case E_SImode:
3133 2094696 : case E_HImode:
3134 2094696 : case E_QImode:
3135 93695 : pass_in_reg:
3136 2094696 : cum->words += words;
3137 2094696 : cum->nregs -= words;
3138 2094696 : cum->regno += words;
3139 2094696 : if (cum->nregs >= 0)
3140 47160 : res = words;
3141 2094696 : if (cum->nregs <= 0)
3142 : {
3143 2060761 : cum->nregs = 0;
3144 2060761 : cfun->machine->arg_reg_available = false;
3145 2060761 : cum->regno = 0;
3146 : }
3147 : break;
3148 :
3149 0 : case E_OImode:
3150 : /* OImode shouldn't be used directly. */
3151 0 : gcc_unreachable ();
3152 :
3153 4771 : case E_DFmode:
3154 4771 : if (cum->float_in_sse == -1)
3155 0 : error_p = true;
3156 4771 : if (cum->float_in_sse < 2)
3157 : break;
3158 : /* FALLTHRU */
3159 1384 : case E_SFmode:
3160 1384 : if (cum->float_in_sse == -1)
3161 0 : error_p = true;
3162 1384 : if (cum->float_in_sse < 1)
3163 : break;
3164 : /* FALLTHRU */
3165 :
3166 52 : case E_V16HFmode:
3167 52 : case E_V16BFmode:
3168 52 : case E_V8SFmode:
3169 52 : case E_V8SImode:
3170 52 : case E_V64QImode:
3171 52 : case E_V32HImode:
3172 52 : case E_V16SImode:
3173 52 : case E_V8DImode:
3174 52 : case E_V32HFmode:
3175 52 : case E_V32BFmode:
3176 52 : case E_V16SFmode:
3177 52 : case E_V8DFmode:
3178 52 : case E_V32QImode:
3179 52 : case E_V16HImode:
3180 52 : case E_V4DFmode:
3181 52 : case E_V4DImode:
3182 52 : case E_TImode:
3183 52 : case E_V16QImode:
3184 52 : case E_V8HImode:
3185 52 : case E_V4SImode:
3186 52 : case E_V2DImode:
3187 52 : case E_V8HFmode:
3188 52 : case E_V8BFmode:
3189 52 : case E_V4SFmode:
3190 52 : case E_V2DFmode:
3191 52 : if (!type || !AGGREGATE_TYPE_P (type))
3192 : {
3193 52 : cum->sse_words += words;
3194 52 : cum->sse_nregs -= 1;
3195 52 : cum->sse_regno += 1;
3196 52 : if (cum->sse_nregs <= 0)
3197 : {
3198 4 : cum->sse_nregs = 0;
3199 4 : cum->sse_regno = 0;
3200 : }
3201 : }
3202 : break;
3203 :
3204 16 : case E_V8QImode:
3205 16 : case E_V4HImode:
3206 16 : case E_V4HFmode:
3207 16 : case E_V4BFmode:
3208 16 : case E_V2SImode:
3209 16 : case E_V2SFmode:
3210 16 : case E_V1TImode:
3211 16 : case E_V1DImode:
3212 16 : if (!type || !AGGREGATE_TYPE_P (type))
3213 : {
3214 16 : cum->mmx_words += words;
3215 16 : cum->mmx_nregs -= 1;
3216 16 : cum->mmx_regno += 1;
3217 16 : if (cum->mmx_nregs <= 0)
3218 : {
3219 0 : cum->mmx_nregs = 0;
3220 0 : cum->mmx_regno = 0;
3221 : }
3222 : }
3223 : break;
3224 : }
3225 2066968 : if (error_p)
3226 : {
3227 0 : cum->float_in_sse = 0;
3228 0 : error ("calling %qD with SSE calling convention without "
3229 : "SSE/SSE2 enabled", cum->decl);
3230 0 : sorry ("this is a GCC bug that can be worked around by adding "
3231 : "attribute used to function called");
3232 : }
3233 :
3234 : return res;
3235 : }
3236 :
3237 : static int
3238 19889137 : function_arg_advance_64 (CUMULATIVE_ARGS *cum, machine_mode mode,
3239 : const_tree type, HOST_WIDE_INT words, bool named)
3240 : {
3241 19889137 : int int_nregs, sse_nregs;
3242 :
3243 : /* Unnamed 512 and 256bit vector mode parameters are passed on stack. */
3244 19889137 : if (!named && (VALID_AVX512F_REG_MODE (mode)
3245 : || VALID_AVX256_REG_MODE (mode)))
3246 : return 0;
3247 :
3248 19888773 : if (!examine_argument (mode, type, false, &int_nregs, &sse_nregs)
3249 19888773 : && sse_nregs <= cum->sse_nregs && int_nregs <= cum->nregs)
3250 : {
3251 17603020 : cum->nregs -= int_nregs;
3252 17603020 : cum->sse_nregs -= sse_nregs;
3253 17603020 : cum->regno += int_nregs;
3254 17603020 : cum->sse_regno += sse_nregs;
3255 17603020 : return int_nregs;
3256 : }
3257 : else
3258 : {
3259 2285753 : int align = ix86_function_arg_boundary (mode, type) / BITS_PER_WORD;
3260 2285753 : cum->words = ROUND_UP (cum->words, align);
3261 2285753 : cum->words += words;
3262 2285753 : return 0;
3263 : }
3264 : }
3265 :
3266 : static int
3267 447474 : function_arg_advance_ms_64 (CUMULATIVE_ARGS *cum, HOST_WIDE_INT bytes,
3268 : HOST_WIDE_INT words)
3269 : {
3270 : /* Otherwise, this should be passed indirect. */
3271 447474 : gcc_assert (bytes == 1 || bytes == 2 || bytes == 4 || bytes == 8);
3272 :
3273 447474 : cum->words += words;
3274 447474 : if (cum->nregs > 0)
3275 : {
3276 289715 : cum->nregs -= 1;
3277 289715 : cum->regno += 1;
3278 289715 : return 1;
3279 : }
3280 : return 0;
3281 : }
3282 :
3283 : /* Update the data in CUM to advance over argument ARG. */
3284 :
3285 : static void
3286 22468539 : ix86_function_arg_advance (cumulative_args_t cum_v,
3287 : const function_arg_info &arg)
3288 : {
3289 22468539 : CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
3290 22468539 : machine_mode mode = arg.mode;
3291 22468539 : HOST_WIDE_INT bytes, words;
3292 22468539 : int nregs;
3293 :
3294 : /* The argument of interrupt handler is a special case and is
3295 : handled in ix86_function_arg. */
3296 22468539 : if (!cum->caller && cfun->machine->func_type != TYPE_NORMAL)
3297 : return;
3298 :
3299 22468172 : bytes = arg.promoted_size_in_bytes ();
3300 22468172 : words = CEIL (bytes, UNITS_PER_WORD);
3301 :
3302 22468172 : if (arg.type)
3303 22142058 : mode = type_natural_mode (arg.type, NULL, false);
3304 :
3305 22468172 : if (TARGET_64BIT)
3306 : {
3307 20336611 : enum calling_abi call_abi = cum ? cum->call_abi : ix86_abi;
3308 :
3309 20336611 : if (call_abi == MS_ABI)
3310 447474 : nregs = function_arg_advance_ms_64 (cum, bytes, words);
3311 : else
3312 19889137 : nregs = function_arg_advance_64 (cum, mode, arg.type, words,
3313 19889137 : arg.named);
3314 : }
3315 : else
3316 2131561 : nregs = function_arg_advance_32 (cum, mode, arg.type, bytes, words);
3317 :
3318 22468172 : if (!nregs)
3319 : {
3320 : /* Track if there are outgoing arguments on stack. */
3321 5760005 : if (cum->caller)
3322 2735007 : cfun->machine->outgoing_args_on_stack = true;
3323 : }
3324 : }
3325 :
3326 : /* Define where to put the arguments to a function.
3327 : Value is zero to push the argument on the stack,
3328 : or a hard register in which to store the argument.
3329 :
3330 : MODE is the argument's machine mode.
3331 : TYPE is the data type of the argument (as a tree).
3332 : This is null for libcalls where that information may
3333 : not be available.
3334 : CUM is a variable of type CUMULATIVE_ARGS which gives info about
3335 : the preceding args and about the function being called.
3336 : NAMED is nonzero if this argument is a named parameter
3337 : (otherwise it is an extra parameter matching an ellipsis). */
3338 :
3339 : static rtx
3340 2558286 : function_arg_32 (CUMULATIVE_ARGS *cum, machine_mode mode,
3341 : machine_mode orig_mode, const_tree type,
3342 : HOST_WIDE_INT bytes, HOST_WIDE_INT words)
3343 : {
3344 2558286 : bool error_p = false;
3345 :
3346 : /* Avoid the AL settings for the Unix64 ABI. */
3347 2558286 : if (mode == VOIDmode)
3348 742396 : return constm1_rtx;
3349 :
3350 1815890 : if (TARGET_IAMCU)
3351 : {
3352 : /* Intel MCU psABI passes scalars and aggregates no larger than 8
3353 : bytes in registers. */
3354 0 : if (!VECTOR_MODE_P (mode) && bytes <= 8)
3355 0 : goto pass_in_reg;
3356 : return NULL_RTX;
3357 : }
3358 :
3359 1815890 : switch (mode)
3360 : {
3361 : default:
3362 : break;
3363 :
3364 77649 : case E_BLKmode:
3365 77649 : if (bytes < 0)
3366 : break;
3367 : /* FALLTHRU */
3368 1782282 : case E_DImode:
3369 1782282 : case E_SImode:
3370 1782282 : case E_HImode:
3371 1782282 : case E_QImode:
3372 77649 : pass_in_reg:
3373 1782282 : if (words <= cum->nregs)
3374 : {
3375 45328 : int regno = cum->regno;
3376 :
3377 : /* Fastcall allocates the first two DWORD (SImode) or
3378 : smaller arguments to ECX and EDX if it isn't an
3379 : aggregate type . */
3380 45328 : if (cum->fastcall)
3381 : {
3382 6 : if (mode == BLKmode
3383 6 : || mode == DImode
3384 6 : || (type && AGGREGATE_TYPE_P (type)))
3385 : break;
3386 :
3387 : /* ECX not EAX is the first allocated register. */
3388 6 : if (regno == AX_REG)
3389 45328 : regno = CX_REG;
3390 : }
3391 45328 : return gen_rtx_REG (mode, regno);
3392 : }
3393 : break;
3394 :
3395 3381 : case E_DFmode:
3396 3381 : if (cum->float_in_sse == -1)
3397 0 : error_p = true;
3398 3381 : if (cum->float_in_sse < 2)
3399 : break;
3400 : /* FALLTHRU */
3401 984 : case E_SFmode:
3402 984 : if (cum->float_in_sse == -1)
3403 0 : error_p = true;
3404 984 : if (cum->float_in_sse < 1)
3405 : break;
3406 : /* FALLTHRU */
3407 12 : case E_TImode:
3408 : /* In 32bit, we pass TImode in xmm registers. */
3409 12 : case E_V16QImode:
3410 12 : case E_V8HImode:
3411 12 : case E_V4SImode:
3412 12 : case E_V2DImode:
3413 12 : case E_V8HFmode:
3414 12 : case E_V8BFmode:
3415 12 : case E_V4SFmode:
3416 12 : case E_V2DFmode:
3417 12 : if (!type || !AGGREGATE_TYPE_P (type))
3418 : {
3419 12 : if (cum->sse_nregs)
3420 12 : return gen_reg_or_parallel (mode, orig_mode,
3421 12 : cum->sse_regno + FIRST_SSE_REG);
3422 : }
3423 : break;
3424 :
3425 0 : case E_OImode:
3426 0 : case E_XImode:
3427 : /* OImode and XImode shouldn't be used directly. */
3428 0 : gcc_unreachable ();
3429 :
3430 9 : case E_V64QImode:
3431 9 : case E_V32HImode:
3432 9 : case E_V16SImode:
3433 9 : case E_V8DImode:
3434 9 : case E_V32HFmode:
3435 9 : case E_V32BFmode:
3436 9 : case E_V16SFmode:
3437 9 : case E_V8DFmode:
3438 9 : case E_V16HFmode:
3439 9 : case E_V16BFmode:
3440 9 : case E_V8SFmode:
3441 9 : case E_V8SImode:
3442 9 : case E_V32QImode:
3443 9 : case E_V16HImode:
3444 9 : case E_V4DFmode:
3445 9 : case E_V4DImode:
3446 9 : if (!type || !AGGREGATE_TYPE_P (type))
3447 : {
3448 9 : if (cum->sse_nregs)
3449 9 : return gen_reg_or_parallel (mode, orig_mode,
3450 9 : cum->sse_regno + FIRST_SSE_REG);
3451 : }
3452 : break;
3453 :
3454 8 : case E_V8QImode:
3455 8 : case E_V4HImode:
3456 8 : case E_V4HFmode:
3457 8 : case E_V4BFmode:
3458 8 : case E_V2SImode:
3459 8 : case E_V2SFmode:
3460 8 : case E_V1TImode:
3461 8 : case E_V1DImode:
3462 8 : if (!type || !AGGREGATE_TYPE_P (type))
3463 : {
3464 8 : if (cum->mmx_nregs)
3465 8 : return gen_reg_or_parallel (mode, orig_mode,
3466 8 : cum->mmx_regno + FIRST_MMX_REG);
3467 : }
3468 : break;
3469 : }
3470 4365 : if (error_p)
3471 : {
3472 0 : cum->float_in_sse = 0;
3473 0 : error ("calling %qD with SSE calling convention without "
3474 : "SSE/SSE2 enabled", cum->decl);
3475 0 : sorry ("this is a GCC bug that can be worked around by adding "
3476 : "attribute used to function called");
3477 : }
3478 :
3479 : return NULL_RTX;
3480 : }
3481 :
3482 : static rtx
3483 19525296 : function_arg_64 (const CUMULATIVE_ARGS *cum, machine_mode mode,
3484 : machine_mode orig_mode, const_tree type, bool named)
3485 : {
3486 : /* Handle a hidden AL argument containing number of registers
3487 : for varargs x86-64 functions. */
3488 19525296 : if (mode == VOIDmode)
3489 5431141 : return GEN_INT (cum->maybe_vaarg
3490 : ? (cum->sse_nregs < 0
3491 : ? X86_64_SSE_REGPARM_MAX
3492 : : cum->sse_regno)
3493 : : -1);
3494 :
3495 14094155 : switch (mode)
3496 : {
3497 : default:
3498 : break;
3499 :
3500 91670 : case E_V16HFmode:
3501 91670 : case E_V16BFmode:
3502 91670 : case E_V8SFmode:
3503 91670 : case E_V8SImode:
3504 91670 : case E_V32QImode:
3505 91670 : case E_V16HImode:
3506 91670 : case E_V4DFmode:
3507 91670 : case E_V4DImode:
3508 91670 : case E_V32HFmode:
3509 91670 : case E_V32BFmode:
3510 91670 : case E_V16SFmode:
3511 91670 : case E_V16SImode:
3512 91670 : case E_V64QImode:
3513 91670 : case E_V32HImode:
3514 91670 : case E_V8DFmode:
3515 91670 : case E_V8DImode:
3516 : /* Unnamed 256 and 512bit vector mode parameters are passed on stack. */
3517 91670 : if (!named)
3518 : return NULL;
3519 : break;
3520 : }
3521 :
3522 14093791 : const int *parm_regs;
3523 14093791 : if (cum->preserve_none_abi)
3524 : parm_regs = x86_64_preserve_none_int_parameter_registers;
3525 : else
3526 14093591 : parm_regs = x86_64_int_parameter_registers;
3527 :
3528 14093791 : return construct_container (mode, orig_mode, type, false,
3529 14093791 : cum->nregs, cum->sse_nregs,
3530 14093791 : &parm_regs[cum->regno],
3531 14093791 : cum->sse_regno);
3532 : }
3533 :
3534 : static rtx
3535 296679 : function_arg_ms_64 (const CUMULATIVE_ARGS *cum, machine_mode mode,
3536 : machine_mode orig_mode, bool named, const_tree type,
3537 : HOST_WIDE_INT bytes)
3538 : {
3539 296679 : unsigned int regno;
3540 :
3541 : /* We need to add clobber for MS_ABI->SYSV ABI calls in expand_call.
3542 : We use value of -2 to specify that current function call is MSABI. */
3543 296679 : if (mode == VOIDmode)
3544 36329 : return GEN_INT (-2);
3545 :
3546 : /* If we've run out of registers, it goes on the stack. */
3547 260350 : if (cum->nregs == 0)
3548 : return NULL_RTX;
3549 :
3550 176508 : regno = x86_64_ms_abi_int_parameter_registers[cum->regno];
3551 :
3552 : /* Only floating point modes less than 64 bits are passed in anything but
3553 : integer regs. Larger floating point types are excluded as the Windows
3554 : ABI requires vreg args can be shadowed in GPRs (for red zone / varargs). */
3555 176508 : if (TARGET_SSE && (mode == HFmode || mode == SFmode || mode == DFmode))
3556 : {
3557 38309 : if (named)
3558 : {
3559 38309 : if (type == NULL_TREE || !AGGREGATE_TYPE_P (type))
3560 37312 : regno = cum->regno + FIRST_SSE_REG;
3561 : }
3562 : else
3563 : {
3564 0 : rtx t1, t2;
3565 :
3566 : /* Unnamed floating parameters are passed in both the
3567 : SSE and integer registers. */
3568 0 : t1 = gen_rtx_REG (mode, cum->regno + FIRST_SSE_REG);
3569 0 : t2 = gen_rtx_REG (mode, regno);
3570 0 : t1 = gen_rtx_EXPR_LIST (VOIDmode, t1, const0_rtx);
3571 0 : t2 = gen_rtx_EXPR_LIST (VOIDmode, t2, const0_rtx);
3572 0 : return gen_rtx_PARALLEL (mode, gen_rtvec (2, t1, t2));
3573 : }
3574 : }
3575 : /* Handle aggregated types passed in register. */
3576 176508 : if (orig_mode == BLKmode)
3577 : {
3578 0 : if (bytes > 0 && bytes <= 8)
3579 0 : mode = (bytes > 4 ? DImode : SImode);
3580 0 : if (mode == BLKmode)
3581 0 : mode = DImode;
3582 : }
3583 :
3584 176508 : return gen_reg_or_parallel (mode, orig_mode, regno);
3585 : }
3586 :
3587 : /* Return where to put the arguments to a function.
3588 : Return zero to push the argument on the stack, or a hard register in which to store the argument.
3589 :
3590 : ARG describes the argument while CUM gives information about the
3591 : preceding args and about the function being called. */
3592 :
3593 : static rtx
3594 22380448 : ix86_function_arg (cumulative_args_t cum_v, const function_arg_info &arg)
3595 : {
3596 22380448 : CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
3597 22380448 : machine_mode mode = arg.mode;
3598 22380448 : HOST_WIDE_INT bytes, words;
3599 22380448 : rtx reg;
3600 :
3601 22380448 : if (!cum->caller && cfun->machine->func_type != TYPE_NORMAL)
3602 : {
3603 187 : gcc_assert (arg.type != NULL_TREE);
3604 187 : if (POINTER_TYPE_P (arg.type))
3605 : {
3606 : /* This is the pointer argument. */
3607 122 : gcc_assert (TYPE_MODE (arg.type) == ptr_mode);
3608 : /* It is at -WORD(AP) in the current frame in interrupt and
3609 : exception handlers. */
3610 122 : reg = plus_constant (Pmode, arg_pointer_rtx, -UNITS_PER_WORD);
3611 : }
3612 : else
3613 : {
3614 65 : gcc_assert (cfun->machine->func_type == TYPE_EXCEPTION
3615 : && TREE_CODE (arg.type) == INTEGER_TYPE
3616 : && TYPE_MODE (arg.type) == word_mode);
3617 : /* The error code is the word-mode integer argument at
3618 : -2 * WORD(AP) in the current frame of the exception
3619 : handler. */
3620 65 : reg = gen_rtx_MEM (word_mode,
3621 65 : plus_constant (Pmode,
3622 : arg_pointer_rtx,
3623 65 : -2 * UNITS_PER_WORD));
3624 : }
3625 : return reg;
3626 : }
3627 :
3628 22380261 : bytes = arg.promoted_size_in_bytes ();
3629 22380261 : words = CEIL (bytes, UNITS_PER_WORD);
3630 :
3631 : /* To simplify the code below, represent vector types with a vector mode
3632 : even if MMX/SSE are not active. */
3633 22380261 : if (arg.type && VECTOR_TYPE_P (arg.type))
3634 175104 : mode = type_natural_mode (arg.type, cum, false);
3635 :
3636 22380261 : if (TARGET_64BIT)
3637 : {
3638 19821975 : if (cum->call_abi == MS_ABI)
3639 296679 : reg = function_arg_ms_64 (cum, mode, arg.mode, arg.named,
3640 296679 : arg.type, bytes);
3641 : else
3642 19525296 : reg = function_arg_64 (cum, mode, arg.mode, arg.type, arg.named);
3643 : }
3644 : else
3645 2558286 : reg = function_arg_32 (cum, mode, arg.mode, arg.type, bytes, words);
3646 :
3647 : /* Track if there are outgoing arguments on stack. */
3648 22380261 : if (reg == NULL_RTX && cum->caller)
3649 2204696 : cfun->machine->outgoing_args_on_stack = true;
3650 :
3651 : return reg;
3652 : }
3653 :
3654 : /* A C expression that indicates when an argument must be passed by
3655 : reference. If nonzero for an argument, a copy of that argument is
3656 : made in memory and a pointer to the argument is passed instead of
3657 : the argument itself. The pointer is passed in whatever way is
3658 : appropriate for passing a pointer to that type. */
3659 :
3660 : static bool
3661 22384638 : ix86_pass_by_reference (cumulative_args_t cum_v, const function_arg_info &arg)
3662 : {
3663 22384638 : CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
3664 :
3665 22384638 : if (TARGET_64BIT)
3666 : {
3667 20263325 : enum calling_abi call_abi = cum ? cum->call_abi : ix86_abi;
3668 :
3669 : /* See Windows x64 Software Convention. */
3670 20263325 : if (call_abi == MS_ABI)
3671 : {
3672 441871 : HOST_WIDE_INT msize = GET_MODE_SIZE (arg.mode);
3673 :
3674 441871 : if (tree type = arg.type)
3675 : {
3676 : /* Arrays are passed by reference. */
3677 441871 : if (TREE_CODE (type) == ARRAY_TYPE)
3678 : return true;
3679 :
3680 441871 : if (RECORD_OR_UNION_TYPE_P (type))
3681 : {
3682 : /* Structs/unions of sizes other than 8, 16, 32, or 64 bits
3683 : are passed by reference. */
3684 15103 : msize = int_size_in_bytes (type);
3685 : }
3686 : }
3687 :
3688 : /* __m128 is passed by reference. */
3689 441871 : return msize != 1 && msize != 2 && msize != 4 && msize != 8;
3690 : }
3691 19821454 : else if (arg.type && int_size_in_bytes (arg.type) == -1)
3692 0 : return true;
3693 : }
3694 :
3695 : return false;
3696 : }
3697 :
3698 : /* Return true when TYPE should be 128bit aligned for 32bit argument
3699 : passing ABI. XXX: This function is obsolete and is only used for
3700 : checking psABI compatibility with previous versions of GCC. */
3701 :
3702 : static bool
3703 1976485 : ix86_compat_aligned_value_p (const_tree type)
3704 : {
3705 1976485 : machine_mode mode = TYPE_MODE (type);
3706 1976485 : if (((TARGET_SSE && SSE_REG_MODE_P (mode))
3707 1976443 : || mode == TDmode
3708 1976443 : || mode == TFmode
3709 : || mode == TCmode)
3710 1976697 : && (!TYPE_USER_ALIGN (type) || TYPE_ALIGN (type) > 128))
3711 : return true;
3712 1976273 : if (TYPE_ALIGN (type) < 128)
3713 : return false;
3714 :
3715 0 : if (AGGREGATE_TYPE_P (type))
3716 : {
3717 : /* Walk the aggregates recursively. */
3718 0 : switch (TREE_CODE (type))
3719 : {
3720 0 : case RECORD_TYPE:
3721 0 : case UNION_TYPE:
3722 0 : case QUAL_UNION_TYPE:
3723 0 : {
3724 0 : tree field;
3725 :
3726 : /* Walk all the structure fields. */
3727 0 : for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
3728 : {
3729 0 : if (TREE_CODE (field) == FIELD_DECL
3730 0 : && ix86_compat_aligned_value_p (TREE_TYPE (field)))
3731 : return true;
3732 : }
3733 : break;
3734 : }
3735 :
3736 0 : case ARRAY_TYPE:
3737 : /* Just for use if some languages passes arrays by value. */
3738 0 : if (ix86_compat_aligned_value_p (TREE_TYPE (type)))
3739 : return true;
3740 : break;
3741 :
3742 : default:
3743 : gcc_unreachable ();
3744 : }
3745 : }
3746 : return false;
3747 : }
3748 :
3749 : /* Return the alignment boundary for MODE and TYPE with alignment ALIGN.
3750 : XXX: This function is obsolete and is only used for checking psABI
3751 : compatibility with previous versions of GCC. */
3752 :
3753 : static unsigned int
3754 5653608 : ix86_compat_function_arg_boundary (machine_mode mode,
3755 : const_tree type, unsigned int align)
3756 : {
3757 : /* In 32bit, only _Decimal128 and __float128 are aligned to their
3758 : natural boundaries. */
3759 5653608 : if (!TARGET_64BIT && mode != TDmode && mode != TFmode)
3760 : {
3761 : /* i386 ABI defines all arguments to be 4 byte aligned. We have to
3762 : make an exception for SSE modes since these require 128bit
3763 : alignment.
3764 :
3765 : The handling here differs from field_alignment. ICC aligns MMX
3766 : arguments to 4 byte boundaries, while structure fields are aligned
3767 : to 8 byte boundaries. */
3768 1988431 : if (!type)
3769 : {
3770 11946 : if (!(TARGET_SSE && SSE_REG_MODE_P (mode)))
3771 1988219 : align = PARM_BOUNDARY;
3772 : }
3773 : else
3774 : {
3775 1976485 : if (!ix86_compat_aligned_value_p (type))
3776 1988219 : align = PARM_BOUNDARY;
3777 : }
3778 : }
3779 10900835 : if (align > BIGGEST_ALIGNMENT)
3780 105 : align = BIGGEST_ALIGNMENT;
3781 5653608 : return align;
3782 : }
3783 :
3784 : /* Return true when TYPE should be 128bit aligned for 32bit argument
3785 : passing ABI. */
3786 :
3787 : static bool
3788 1979182 : ix86_contains_aligned_value_p (const_tree type)
3789 : {
3790 1979182 : machine_mode mode = TYPE_MODE (type);
3791 :
3792 1979182 : if (mode == XFmode || mode == XCmode)
3793 : return false;
3794 :
3795 1977063 : if (TYPE_ALIGN (type) < 128)
3796 : return false;
3797 :
3798 2909 : if (AGGREGATE_TYPE_P (type))
3799 : {
3800 : /* Walk the aggregates recursively. */
3801 0 : switch (TREE_CODE (type))
3802 : {
3803 0 : case RECORD_TYPE:
3804 0 : case UNION_TYPE:
3805 0 : case QUAL_UNION_TYPE:
3806 0 : {
3807 0 : tree field;
3808 :
3809 : /* Walk all the structure fields. */
3810 0 : for (field = TYPE_FIELDS (type);
3811 0 : field;
3812 0 : field = DECL_CHAIN (field))
3813 : {
3814 0 : if (TREE_CODE (field) == FIELD_DECL
3815 0 : && ix86_contains_aligned_value_p (TREE_TYPE (field)))
3816 : return true;
3817 : }
3818 : break;
3819 : }
3820 :
3821 0 : case ARRAY_TYPE:
3822 : /* Just for use if some languages passes arrays by value. */
3823 0 : if (ix86_contains_aligned_value_p (TREE_TYPE (type)))
3824 : return true;
3825 : break;
3826 :
3827 : default:
3828 : gcc_unreachable ();
3829 : }
3830 : }
3831 : else
3832 2909 : return TYPE_ALIGN (type) >= 128;
3833 :
3834 : return false;
3835 : }
3836 :
3837 : /* Gives the alignment boundary, in bits, of an argument with the
3838 : specified mode and type. */
3839 :
3840 : static unsigned int
3841 11091069 : ix86_function_arg_boundary (machine_mode mode, const_tree type)
3842 : {
3843 11091069 : unsigned int align;
3844 11091069 : if (type)
3845 : {
3846 : /* Since the main variant type is used for call, we convert it to
3847 : the main variant type. */
3848 11051146 : type = TYPE_MAIN_VARIANT (type);
3849 11051146 : align = TYPE_ALIGN (type);
3850 11051146 : if (TYPE_EMPTY_P (type))
3851 25563 : return PARM_BOUNDARY;
3852 : }
3853 : else
3854 39923 : align = GET_MODE_ALIGNMENT (mode);
3855 13091928 : if (align < PARM_BOUNDARY)
3856 4145363 : align = PARM_BOUNDARY;
3857 : else
3858 : {
3859 6920143 : static bool warned;
3860 6920143 : unsigned int saved_align = align;
3861 :
3862 6920143 : if (!TARGET_64BIT)
3863 : {
3864 : /* i386 ABI defines XFmode arguments to be 4 byte aligned. */
3865 2015030 : if (!type)
3866 : {
3867 35848 : if (mode == XFmode || mode == XCmode)
3868 : align = PARM_BOUNDARY;
3869 : }
3870 1979182 : else if (!ix86_contains_aligned_value_p (type))
3871 : align = PARM_BOUNDARY;
3872 :
3873 38757 : if (align < 128)
3874 1988219 : align = PARM_BOUNDARY;
3875 : }
3876 :
3877 6920143 : if (warn_psabi
3878 5662126 : && !warned
3879 12573751 : && align != ix86_compat_function_arg_boundary (mode, type,
3880 : saved_align))
3881 : {
3882 105 : warned = true;
3883 105 : inform (input_location,
3884 : "the ABI for passing parameters with %d-byte"
3885 : " alignment has changed in GCC 4.6",
3886 : align / BITS_PER_UNIT);
3887 : }
3888 : }
3889 :
3890 : return align;
3891 : }
3892 :
3893 : /* Return true if N is a possible register number of function value. */
3894 :
3895 : static bool
3896 4524812 : ix86_function_value_regno_p (const unsigned int regno)
3897 : {
3898 4524812 : switch (regno)
3899 : {
3900 : case AX_REG:
3901 : return true;
3902 109432 : case DX_REG:
3903 109432 : return (!TARGET_64BIT || ix86_cfun_abi () != MS_ABI);
3904 :
3905 : /* Complex values are returned in %st(0)/%st(1) pair. */
3906 16252 : case ST0_REG:
3907 16252 : case ST1_REG:
3908 : /* TODO: An ABI identifier should be passed as a parameter.
3909 : For now, most callers, including those in builtins.cc,
3910 : expect us to use the default ABI. */
3911 16252 : if (TARGET_64BIT && ix86_abi == MS_ABI)
3912 : return false;
3913 16252 : return TARGET_FLOAT_RETURNS_IN_80387;
3914 :
3915 : /* Complex values are returned in %xmm0/%xmm1 pair. */
3916 1290676 : case XMM0_REG:
3917 1290676 : case XMM1_REG:
3918 1290676 : return TARGET_SSE;
3919 :
3920 5469 : case MM0_REG:
3921 5469 : if (TARGET_MACHO || TARGET_64BIT)
3922 : return false;
3923 2499 : return TARGET_MMX;
3924 : }
3925 :
3926 : return false;
3927 : }
3928 :
3929 : /* Check whether the register REGNO should be zeroed on X86.
3930 : When ALL_SSE_ZEROED is true, all SSE registers have been zeroed
3931 : together, no need to zero it again.
3932 : When NEED_ZERO_MMX is true, MMX registers should be cleared. */
3933 :
3934 : static bool
3935 1377 : zero_call_used_regno_p (const unsigned int regno,
3936 : bool all_sse_zeroed,
3937 : bool need_zero_mmx)
3938 : {
3939 835 : return GENERAL_REGNO_P (regno)
3940 819 : || (!all_sse_zeroed && SSE_REGNO_P (regno))
3941 439 : || MASK_REGNO_P (regno)
3942 1800 : || (need_zero_mmx && MMX_REGNO_P (regno));
3943 : }
3944 :
3945 : /* Return the machine_mode that is used to zero register REGNO. */
3946 :
3947 : static machine_mode
3948 954 : zero_call_used_regno_mode (const unsigned int regno)
3949 : {
3950 : /* NB: We only need to zero the lower 32 bits for integer registers
3951 : and the lower 128 bits for vector registers since destination are
3952 : zero-extended to the full register width. */
3953 954 : if (GENERAL_REGNO_P (regno))
3954 : return SImode;
3955 : else if (SSE_REGNO_P (regno))
3956 380 : return V4SFmode;
3957 : else if (MASK_REGNO_P (regno))
3958 : return HImode;
3959 : else if (MMX_REGNO_P (regno))
3960 0 : return V2SImode;
3961 : else
3962 0 : gcc_unreachable ();
3963 : }
3964 :
3965 : /* Generate a rtx to zero all vector registers together if possible,
3966 : otherwise, return NULL. */
3967 :
3968 : static rtx
3969 131 : zero_all_vector_registers (HARD_REG_SET need_zeroed_hardregs)
3970 : {
3971 131 : if (!TARGET_AVX)
3972 : return NULL;
3973 :
3974 380 : for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
3975 376 : if ((LEGACY_SSE_REGNO_P (regno)
3976 344 : || (TARGET_64BIT
3977 344 : && (REX_SSE_REGNO_P (regno)
3978 312 : || (TARGET_AVX512F && EXT_REX_SSE_REGNO_P (regno)))))
3979 440 : && !TEST_HARD_REG_BIT (need_zeroed_hardregs, regno))
3980 : return NULL;
3981 :
3982 4 : return gen_avx_vzeroall ();
3983 : }
3984 :
3985 : /* Generate insns to zero all st registers together.
3986 : Return true when zeroing instructions are generated.
3987 : Assume the number of st registers that are zeroed is num_of_st,
3988 : we will emit the following sequence to zero them together:
3989 : fldz; \
3990 : fldz; \
3991 : ...
3992 : fldz; \
3993 : fstp %%st(0); \
3994 : fstp %%st(0); \
3995 : ...
3996 : fstp %%st(0);
3997 : i.e., num_of_st fldz followed by num_of_st fstp to clear the stack
3998 : mark stack slots empty.
3999 :
4000 : How to compute the num_of_st:
4001 : There is no direct mapping from stack registers to hard register
4002 : numbers. If one stack register needs to be cleared, we don't know
4003 : where in the stack the value remains. So, if any stack register
4004 : needs to be cleared, the whole stack should be cleared. However,
4005 : x87 stack registers that hold the return value should be excluded.
4006 : x87 returns in the top (two for complex values) register, so
4007 : num_of_st should be 7/6 when x87 returns, otherwise it will be 8.
4008 : return the value of num_of_st. */
4009 :
4010 :
4011 : static int
4012 131 : zero_all_st_registers (HARD_REG_SET need_zeroed_hardregs)
4013 : {
4014 :
4015 : /* If the FPU is disabled, no need to zero all st registers. */
4016 131 : if (! (TARGET_80387 || TARGET_FLOAT_RETURNS_IN_80387))
4017 : return 0;
4018 :
4019 10547 : unsigned int num_of_st = 0;
4020 10547 : for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
4021 10438 : if ((STACK_REGNO_P (regno) || MMX_REGNO_P (regno))
4022 10438 : && TEST_HARD_REG_BIT (need_zeroed_hardregs, regno))
4023 : {
4024 : num_of_st++;
4025 : break;
4026 : }
4027 :
4028 130 : if (num_of_st == 0)
4029 : return 0;
4030 :
4031 21 : bool return_with_x87 = false;
4032 42 : return_with_x87 = (crtl->return_rtx
4033 21 : && (STACK_REG_P (crtl->return_rtx)));
4034 :
4035 21 : bool complex_return = false;
4036 42 : complex_return = (crtl->return_rtx
4037 21 : && COMPLEX_MODE_P (GET_MODE (crtl->return_rtx)));
4038 :
4039 21 : if (return_with_x87)
4040 2 : if (complex_return)
4041 : num_of_st = 6;
4042 : else
4043 1 : num_of_st = 7;
4044 : else
4045 : num_of_st = 8;
4046 :
4047 21 : rtx st_reg = gen_rtx_REG (XFmode, FIRST_STACK_REG);
4048 207 : for (unsigned int i = 0; i < num_of_st; i++)
4049 165 : emit_insn (gen_rtx_SET (st_reg, CONST0_RTX (XFmode)));
4050 :
4051 186 : for (unsigned int i = 0; i < num_of_st; i++)
4052 : {
4053 165 : rtx insn;
4054 165 : insn = emit_insn (gen_rtx_SET (st_reg, st_reg));
4055 165 : add_reg_note (insn, REG_DEAD, st_reg);
4056 : }
4057 21 : return num_of_st;
4058 : }
4059 :
4060 :
4061 : /* When the routine exit in MMX mode, if any ST register needs
4062 : to be zeroed, we should clear all MMX registers except the
4063 : RET_MMX_REGNO that holds the return value. */
4064 : static bool
4065 0 : zero_all_mm_registers (HARD_REG_SET need_zeroed_hardregs,
4066 : unsigned int ret_mmx_regno)
4067 : {
4068 0 : bool need_zero_all_mm = false;
4069 0 : for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
4070 0 : if (STACK_REGNO_P (regno)
4071 0 : && TEST_HARD_REG_BIT (need_zeroed_hardregs, regno))
4072 : {
4073 : need_zero_all_mm = true;
4074 : break;
4075 : }
4076 :
4077 0 : if (!need_zero_all_mm)
4078 : return false;
4079 :
4080 : machine_mode mode = V2SImode;
4081 0 : for (unsigned int regno = FIRST_MMX_REG; regno <= LAST_MMX_REG; regno++)
4082 0 : if (regno != ret_mmx_regno)
4083 : {
4084 0 : rtx reg = gen_rtx_REG (mode, regno);
4085 0 : emit_insn (gen_rtx_SET (reg, CONST0_RTX (mode)));
4086 : }
4087 : return true;
4088 : }
4089 :
4090 : /* TARGET_ZERO_CALL_USED_REGS. */
4091 : /* Generate a sequence of instructions that zero registers specified by
4092 : NEED_ZEROED_HARDREGS. Return the ZEROED_HARDREGS that are actually
4093 : zeroed. */
4094 : static HARD_REG_SET
4095 131 : ix86_zero_call_used_regs (HARD_REG_SET need_zeroed_hardregs)
4096 : {
4097 131 : HARD_REG_SET zeroed_hardregs;
4098 131 : bool all_sse_zeroed = false;
4099 131 : int all_st_zeroed_num = 0;
4100 131 : bool all_mm_zeroed = false;
4101 :
4102 131 : CLEAR_HARD_REG_SET (zeroed_hardregs);
4103 :
4104 : /* first, let's see whether we can zero all vector registers together. */
4105 131 : rtx zero_all_vec_insn = zero_all_vector_registers (need_zeroed_hardregs);
4106 131 : if (zero_all_vec_insn)
4107 : {
4108 4 : emit_insn (zero_all_vec_insn);
4109 4 : all_sse_zeroed = true;
4110 4 : if (TARGET_64BIT && TARGET_AVX512F)
4111 : {
4112 2 : rtx zero = CONST0_RTX (V4SFmode);
4113 34 : for (unsigned int regno = XMM16_REG;
4114 34 : regno <= XMM31_REG;
4115 : regno++)
4116 : {
4117 32 : rtx reg = gen_rtx_REG (V4SFmode, regno);
4118 32 : emit_move_insn (reg, zero);
4119 : }
4120 : }
4121 : }
4122 :
4123 : /* mm/st registers are shared registers set, we should follow the following
4124 : rules to clear them:
4125 : MMX exit mode x87 exit mode
4126 : -------------|----------------------|---------------
4127 : uses x87 reg | clear all MMX | clear all x87
4128 : uses MMX reg | clear individual MMX | clear all x87
4129 : x87 + MMX | clear all MMX | clear all x87
4130 :
4131 : first, we should decide which mode (MMX mode or x87 mode) the function
4132 : exit with. */
4133 :
4134 131 : bool exit_with_mmx_mode = (crtl->return_rtx
4135 131 : && (MMX_REG_P (crtl->return_rtx)));
4136 :
4137 131 : if (!exit_with_mmx_mode)
4138 : /* x87 exit mode, we should zero all st registers together. */
4139 : {
4140 131 : all_st_zeroed_num = zero_all_st_registers (need_zeroed_hardregs);
4141 :
4142 131 : if (all_st_zeroed_num > 0)
4143 189 : for (unsigned int regno = FIRST_STACK_REG; regno <= LAST_STACK_REG; regno++)
4144 : /* x87 stack registers that hold the return value should be excluded.
4145 : x87 returns in the top (two for complex values) register. */
4146 168 : if (all_st_zeroed_num == 8
4147 168 : || !((all_st_zeroed_num >= 6 && regno == REGNO (crtl->return_rtx))
4148 : || (all_st_zeroed_num == 6
4149 7 : && (regno == (REGNO (crtl->return_rtx) + 1)))))
4150 165 : SET_HARD_REG_BIT (zeroed_hardregs, regno);
4151 : }
4152 : else
4153 : /* MMX exit mode, check whether we can zero all mm registers. */
4154 : {
4155 0 : unsigned int exit_mmx_regno = REGNO (crtl->return_rtx);
4156 0 : all_mm_zeroed = zero_all_mm_registers (need_zeroed_hardregs,
4157 : exit_mmx_regno);
4158 0 : if (all_mm_zeroed)
4159 0 : for (unsigned int regno = FIRST_MMX_REG; regno <= LAST_MMX_REG; regno++)
4160 0 : if (regno != exit_mmx_regno)
4161 0 : SET_HARD_REG_BIT (zeroed_hardregs, regno);
4162 : }
4163 :
4164 : /* Now, generate instructions to zero all the other registers. */
4165 :
4166 12445 : for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
4167 : {
4168 12314 : if (!TEST_HARD_REG_BIT (need_zeroed_hardregs, regno))
4169 10937 : continue;
4170 1800 : if (!zero_call_used_regno_p (regno, all_sse_zeroed,
4171 1377 : exit_with_mmx_mode && !all_mm_zeroed))
4172 423 : continue;
4173 :
4174 954 : SET_HARD_REG_BIT (zeroed_hardregs, regno);
4175 :
4176 954 : machine_mode mode = zero_call_used_regno_mode (regno);
4177 :
4178 954 : rtx reg = gen_rtx_REG (mode, regno);
4179 954 : rtx tmp = gen_rtx_SET (reg, CONST0_RTX (mode));
4180 :
4181 954 : switch (mode)
4182 : {
4183 558 : case E_SImode:
4184 558 : if (!TARGET_USE_MOV0 || optimize_insn_for_size_p ())
4185 : {
4186 558 : rtx clob = gen_rtx_CLOBBER (VOIDmode,
4187 : gen_rtx_REG (CCmode,
4188 : FLAGS_REG));
4189 558 : tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2,
4190 : tmp,
4191 : clob));
4192 : }
4193 : /* FALLTHRU. */
4194 :
4195 954 : case E_V4SFmode:
4196 954 : case E_HImode:
4197 954 : case E_V2SImode:
4198 954 : emit_insn (tmp);
4199 954 : break;
4200 :
4201 : default:
4202 : gcc_unreachable ();
4203 : }
4204 : }
4205 131 : return zeroed_hardregs;
4206 : }
4207 :
4208 : /* Define how to find the value returned by a function.
4209 : VALTYPE is the data type of the value (as a tree).
4210 : If the precise function being called is known, FUNC is its FUNCTION_DECL;
4211 : otherwise, FUNC is 0. */
4212 :
4213 : static rtx
4214 3958055 : function_value_32 (machine_mode orig_mode, machine_mode mode,
4215 : const_tree fntype, const_tree fn)
4216 : {
4217 3958055 : unsigned int regno;
4218 :
4219 : /* 8-byte vector modes in %mm0. See ix86_return_in_memory for where
4220 : we normally prevent this case when mmx is not available. However
4221 : some ABIs may require the result to be returned like DImode. */
4222 4233400 : if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 8)
4223 : regno = FIRST_MMX_REG;
4224 :
4225 : /* 16-byte vector modes in %xmm0. See ix86_return_in_memory for where
4226 : we prevent this case when sse is not available. However some ABIs
4227 : may require the result to be returned like integer TImode. */
4228 3948779 : else if (mode == TImode
4229 4214848 : || (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 16))
4230 : regno = FIRST_SSE_REG;
4231 :
4232 : /* 32-byte vector modes in %ymm0. */
4233 3986126 : else if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 32)
4234 : regno = FIRST_SSE_REG;
4235 :
4236 : /* 64-byte vector modes in %zmm0. */
4237 3839992 : else if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 64)
4238 : regno = FIRST_SSE_REG;
4239 :
4240 : /* Floating point return values in %st(0) (unless -mno-fp-ret-in-387). */
4241 3682710 : else if (X87_FLOAT_MODE_P (mode) && TARGET_FLOAT_RETURNS_IN_80387)
4242 : regno = FIRST_FLOAT_REG;
4243 : else
4244 : /* Most things go in %eax. */
4245 3616906 : regno = AX_REG;
4246 :
4247 : /* Return __bf16/ _Float16/_Complex _Foat16 by sse register. */
4248 3958055 : if (mode == HFmode || mode == BFmode)
4249 : {
4250 1917 : if (!TARGET_SSE2)
4251 : {
4252 0 : error ("SSE register return with SSE2 disabled");
4253 0 : regno = AX_REG;
4254 : }
4255 : else
4256 : regno = FIRST_SSE_REG;
4257 : }
4258 :
4259 3958055 : if (mode == HCmode)
4260 : {
4261 129 : if (!TARGET_SSE2)
4262 0 : error ("SSE register return with SSE2 disabled");
4263 :
4264 129 : rtx ret = gen_rtx_PARALLEL (mode, rtvec_alloc(1));
4265 258 : XVECEXP (ret, 0, 0)
4266 258 : = gen_rtx_EXPR_LIST (VOIDmode,
4267 : gen_rtx_REG (SImode,
4268 129 : TARGET_SSE2 ? FIRST_SSE_REG : AX_REG),
4269 : GEN_INT (0));
4270 129 : return ret;
4271 : }
4272 :
4273 : /* Override FP return register with %xmm0 for local functions when
4274 : SSE math is enabled or for functions with sseregparm attribute. */
4275 3957926 : if ((fn || fntype) && (mode == SFmode || mode == DFmode))
4276 : {
4277 51267 : int sse_level = ix86_function_sseregparm (fntype, fn, false);
4278 51267 : if (sse_level == -1)
4279 : {
4280 0 : error ("calling %qD with SSE calling convention without "
4281 : "SSE/SSE2 enabled", fn);
4282 0 : sorry ("this is a GCC bug that can be worked around by adding "
4283 : "attribute used to function called");
4284 : }
4285 51267 : else if ((sse_level >= 1 && mode == SFmode)
4286 51267 : || (sse_level == 2 && mode == DFmode))
4287 : regno = FIRST_SSE_REG;
4288 : }
4289 :
4290 : /* OImode shouldn't be used directly. */
4291 3957926 : gcc_assert (mode != OImode);
4292 :
4293 3957926 : return gen_rtx_REG (orig_mode, regno);
4294 : }
4295 :
4296 : static rtx
4297 102970004 : function_value_64 (machine_mode orig_mode, machine_mode mode,
4298 : const_tree valtype)
4299 : {
4300 102970004 : rtx ret;
4301 :
4302 : /* Handle libcalls, which don't provide a type node. */
4303 102970004 : if (valtype == NULL)
4304 : {
4305 109110 : unsigned int regno;
4306 :
4307 109110 : switch (mode)
4308 : {
4309 : case E_BFmode:
4310 : case E_HFmode:
4311 : case E_HCmode:
4312 : case E_SFmode:
4313 : case E_SCmode:
4314 : case E_DFmode:
4315 : case E_DCmode:
4316 : case E_TFmode:
4317 : case E_SDmode:
4318 : case E_DDmode:
4319 : case E_TDmode:
4320 : regno = FIRST_SSE_REG;
4321 : break;
4322 1026 : case E_XFmode:
4323 1026 : case E_XCmode:
4324 1026 : regno = FIRST_FLOAT_REG;
4325 1026 : break;
4326 : case E_TCmode:
4327 : return NULL;
4328 57293 : default:
4329 57293 : regno = AX_REG;
4330 : }
4331 :
4332 109110 : return gen_rtx_REG (mode, regno);
4333 : }
4334 102860894 : else if (POINTER_TYPE_P (valtype))
4335 : {
4336 : /* Pointers are always returned in word_mode. */
4337 17369692 : mode = word_mode;
4338 : }
4339 :
4340 102860894 : ret = construct_container (mode, orig_mode, valtype, true,
4341 : X86_64_MAX_RETURN_NREGS,
4342 : X86_64_MAX_SSE_RETURN_NREGS,
4343 : x86_64_int_return_registers, 0);
4344 :
4345 : /* For zero sized structures, construct_container returns NULL, but we
4346 : need to keep rest of compiler happy by returning meaningful value. */
4347 102860894 : if (!ret)
4348 210976 : ret = gen_rtx_REG (orig_mode, AX_REG);
4349 :
4350 : return ret;
4351 : }
4352 :
4353 : static rtx
4354 0 : function_value_ms_32 (machine_mode orig_mode, machine_mode mode,
4355 : const_tree fntype, const_tree fn, const_tree valtype)
4356 : {
4357 0 : unsigned int regno;
4358 :
4359 : /* Floating point return values in %st(0)
4360 : (unless -mno-fp-ret-in-387 or aggregate type of up to 8 bytes). */
4361 0 : if (X87_FLOAT_MODE_P (mode) && TARGET_FLOAT_RETURNS_IN_80387
4362 0 : && (GET_MODE_SIZE (mode) > 8
4363 0 : || valtype == NULL_TREE || !AGGREGATE_TYPE_P (valtype)))
4364 : {
4365 0 : regno = FIRST_FLOAT_REG;
4366 0 : return gen_rtx_REG (orig_mode, regno);
4367 : }
4368 : else
4369 0 : return function_value_32(orig_mode, mode, fntype,fn);
4370 : }
4371 :
4372 : static rtx
4373 804507 : function_value_ms_64 (machine_mode orig_mode, machine_mode mode,
4374 : const_tree valtype)
4375 : {
4376 804507 : unsigned int regno = AX_REG;
4377 :
4378 804507 : if (TARGET_SSE)
4379 : {
4380 803778 : unsigned int mode_size = GET_MODE_SIZE (mode);
4381 :
4382 803778 : switch (mode_size)
4383 : {
4384 35435 : case 16:
4385 35435 : case 32:
4386 35435 : case 64:
4387 35435 : if (mode_size == 32 && !TARGET_AVX)
4388 : break;
4389 35435 : if (mode_size == 64 && !TARGET_AVX512F)
4390 : break;
4391 35435 : if (valtype != NULL_TREE
4392 35435 : && !VECTOR_INTEGER_TYPE_P (valtype)
4393 51401 : && !VECTOR_FLOAT_TYPE_P (valtype))
4394 : break;
4395 35435 : if (VECTOR_MODE_P (mode)
4396 : && !COMPLEX_MODE_P (mode))
4397 225402 : regno = FIRST_SSE_REG;
4398 : break;
4399 756704 : case 8:
4400 756704 : case 4:
4401 756704 : case 2:
4402 756704 : if (valtype != NULL_TREE && AGGREGATE_TYPE_P (valtype))
4403 : break;
4404 739410 : if (mode == HFmode || mode == SFmode || mode == DFmode)
4405 225402 : regno = FIRST_SSE_REG;
4406 : break;
4407 : default:
4408 : break;
4409 : }
4410 : }
4411 804507 : return gen_rtx_REG (orig_mode, regno);
4412 : }
4413 :
4414 : static rtx
4415 107732566 : ix86_function_value_1 (const_tree valtype, const_tree fntype_or_decl,
4416 : machine_mode orig_mode, machine_mode mode)
4417 : {
4418 107732566 : const_tree fn, fntype;
4419 :
4420 107732566 : fn = NULL_TREE;
4421 107732566 : if (fntype_or_decl && DECL_P (fntype_or_decl))
4422 3664362 : fn = fntype_or_decl;
4423 3664362 : fntype = fn ? TREE_TYPE (fn) : fntype_or_decl;
4424 :
4425 107732566 : if (ix86_function_type_abi (fntype) == MS_ABI)
4426 : {
4427 804507 : if (TARGET_64BIT)
4428 804507 : return function_value_ms_64 (orig_mode, mode, valtype);
4429 : else
4430 0 : return function_value_ms_32 (orig_mode, mode, fntype, fn, valtype);
4431 : }
4432 106928059 : else if (TARGET_64BIT)
4433 102970004 : return function_value_64 (orig_mode, mode, valtype);
4434 : else
4435 3958055 : return function_value_32 (orig_mode, mode, fntype, fn);
4436 : }
4437 :
4438 : static rtx
4439 107620271 : ix86_function_value (const_tree valtype, const_tree fntype_or_decl, bool)
4440 : {
4441 107620271 : machine_mode mode, orig_mode;
4442 :
4443 107620271 : orig_mode = TYPE_MODE (valtype);
4444 107620271 : mode = type_natural_mode (valtype, NULL, true);
4445 107620271 : return ix86_function_value_1 (valtype, fntype_or_decl, orig_mode, mode);
4446 : }
4447 :
4448 : /* Pointer function arguments and return values are promoted to
4449 : word_mode for normal functions. */
4450 :
4451 : static machine_mode
4452 33120221 : ix86_promote_function_mode (const_tree type, machine_mode mode,
4453 : int *punsignedp, const_tree fntype,
4454 : int for_return)
4455 : {
4456 33120221 : if (cfun->machine->func_type == TYPE_NORMAL
4457 33119198 : && type != NULL_TREE
4458 33084732 : && POINTER_TYPE_P (type))
4459 : {
4460 16741759 : *punsignedp = POINTERS_EXTEND_UNSIGNED;
4461 16741759 : return word_mode;
4462 : }
4463 16378462 : return default_promote_function_mode (type, mode, punsignedp, fntype,
4464 16378462 : for_return);
4465 : }
4466 :
4467 : /* Return true if a structure, union or array with MODE containing FIELD
4468 : should be accessed using BLKmode. */
4469 :
4470 : static bool
4471 151978658 : ix86_member_type_forces_blk (const_tree field, machine_mode mode)
4472 : {
4473 : /* Union with XFmode must be in BLKmode. */
4474 151978658 : return (mode == XFmode
4475 152156355 : && (TREE_CODE (DECL_FIELD_CONTEXT (field)) == UNION_TYPE
4476 135931 : || TREE_CODE (DECL_FIELD_CONTEXT (field)) == QUAL_UNION_TYPE));
4477 : }
4478 :
4479 : rtx
4480 112295 : ix86_libcall_value (machine_mode mode)
4481 : {
4482 112295 : return ix86_function_value_1 (NULL, NULL, mode, mode);
4483 : }
4484 :
4485 : /* Return true iff type is returned in memory. */
4486 :
4487 : static bool
4488 109767596 : ix86_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED)
4489 : {
4490 109767596 : const machine_mode mode = type_natural_mode (type, NULL, true);
4491 109767596 : HOST_WIDE_INT size;
4492 :
4493 109767596 : if (TARGET_64BIT)
4494 : {
4495 105207978 : if (ix86_function_type_abi (fntype) == MS_ABI)
4496 : {
4497 723912 : size = int_size_in_bytes (type);
4498 :
4499 : /* __m128 is returned in xmm0. 256/512-bit vector values are
4500 : returned in ymm0/zmm0 when AVX/AVX512 is enabled. */
4501 723912 : if ((!type || VECTOR_INTEGER_TYPE_P (type)
4502 703441 : || VECTOR_FLOAT_TYPE_P (type))
4503 36437 : && VECTOR_MODE_P (mode)
4504 : && !COMPLEX_MODE_P (mode)
4505 760349 : && ((GET_MODE_SIZE (mode) == 16 || size == 16)
4506 43404 : || (TARGET_AVX && (GET_MODE_SIZE (mode) == 32 || size == 32))
4507 11826 : || (TARGET_AVX512F
4508 11322 : && (GET_MODE_SIZE (mode) == 64 || size == 64))))
4509 : return false;
4510 :
4511 : /* Otherwise, the size must be exactly in [1248]. */
4512 688483 : return size != 1 && size != 2 && size != 4 && size != 8;
4513 : }
4514 : else
4515 : {
4516 104484066 : int needed_intregs, needed_sseregs;
4517 :
4518 104484066 : return examine_argument (mode, type, true,
4519 : &needed_intregs, &needed_sseregs);
4520 : }
4521 : }
4522 : else
4523 : {
4524 4559618 : size = int_size_in_bytes (type);
4525 :
4526 : /* Intel MCU psABI returns scalars and aggregates no larger than 8
4527 : bytes in registers. */
4528 4559618 : if (TARGET_IAMCU)
4529 0 : return VECTOR_MODE_P (mode) || size < 0 || size > 8;
4530 :
4531 4559618 : if (mode == BLKmode)
4532 : return true;
4533 :
4534 4559618 : if (MS_AGGREGATE_RETURN && AGGREGATE_TYPE_P (type) && size <= 8)
4535 : return false;
4536 :
4537 4559618 : if (VECTOR_MODE_P (mode) || mode == TImode)
4538 : {
4539 : /* User-created vectors small enough to fit in EAX. */
4540 275315 : if (size < 8)
4541 : return false;
4542 :
4543 : /* Unless ABI prescribes otherwise,
4544 : MMX/3dNow values are returned in MM0 if available. */
4545 :
4546 275315 : if (size == 8)
4547 9266 : return TARGET_VECT8_RETURNS || !TARGET_MMX;
4548 :
4549 : /* SSE values are returned in XMM0 if available. */
4550 266049 : if (size == 16)
4551 114351 : return !TARGET_SSE;
4552 :
4553 : /* AVX values are returned in YMM0 if available. */
4554 151698 : if (size == 32)
4555 73058 : return !TARGET_AVX;
4556 :
4557 : /* AVX512F values are returned in ZMM0 if available. */
4558 78640 : if (size == 64)
4559 78640 : return !TARGET_AVX512F;
4560 : }
4561 :
4562 4284303 : if (mode == XFmode)
4563 : return false;
4564 :
4565 4272221 : if (size > 12)
4566 : return true;
4567 :
4568 : /* OImode shouldn't be used directly. */
4569 3296623 : gcc_assert (mode != OImode);
4570 :
4571 : return false;
4572 : }
4573 : }
4574 :
4575 : /* Implement TARGET_PUSH_ARGUMENT. */
4576 :
4577 : static bool
4578 9558945 : ix86_push_argument (unsigned int npush)
4579 : {
4580 : /* If SSE2 is available, use vector move to put large argument onto
4581 : stack. NB: In 32-bit mode, use 8-byte vector move. */
4582 11985581 : return ((!TARGET_SSE2 || npush < (TARGET_64BIT ? 16 : 8))
4583 9293621 : && TARGET_PUSH_ARGS
4584 18852408 : && !ACCUMULATE_OUTGOING_ARGS);
4585 : }
4586 :
4587 :
4588 : /* Create the va_list data type. */
4589 :
4590 : static tree
4591 287896 : ix86_build_builtin_va_list_64 (void)
4592 : {
4593 287896 : tree f_gpr, f_fpr, f_ovf, f_sav, record, type_decl;
4594 :
4595 287896 : record = lang_hooks.types.make_type (RECORD_TYPE);
4596 287896 : type_decl = build_decl (BUILTINS_LOCATION,
4597 : TYPE_DECL, get_identifier ("__va_list_tag"), record);
4598 :
4599 287896 : f_gpr = build_decl (BUILTINS_LOCATION,
4600 : FIELD_DECL, get_identifier ("gp_offset"),
4601 : unsigned_type_node);
4602 287896 : f_fpr = build_decl (BUILTINS_LOCATION,
4603 : FIELD_DECL, get_identifier ("fp_offset"),
4604 : unsigned_type_node);
4605 287896 : f_ovf = build_decl (BUILTINS_LOCATION,
4606 : FIELD_DECL, get_identifier ("overflow_arg_area"),
4607 : ptr_type_node);
4608 287896 : f_sav = build_decl (BUILTINS_LOCATION,
4609 : FIELD_DECL, get_identifier ("reg_save_area"),
4610 : ptr_type_node);
4611 :
4612 287896 : va_list_gpr_counter_field = f_gpr;
4613 287896 : va_list_fpr_counter_field = f_fpr;
4614 :
4615 287896 : DECL_FIELD_CONTEXT (f_gpr) = record;
4616 287896 : DECL_FIELD_CONTEXT (f_fpr) = record;
4617 287896 : DECL_FIELD_CONTEXT (f_ovf) = record;
4618 287896 : DECL_FIELD_CONTEXT (f_sav) = record;
4619 :
4620 287896 : TYPE_STUB_DECL (record) = type_decl;
4621 287896 : TYPE_NAME (record) = type_decl;
4622 287896 : TYPE_FIELDS (record) = f_gpr;
4623 287896 : DECL_CHAIN (f_gpr) = f_fpr;
4624 287896 : DECL_CHAIN (f_fpr) = f_ovf;
4625 287896 : DECL_CHAIN (f_ovf) = f_sav;
4626 287896 : TREE_PUBLIC (type_decl) = 1;
4627 :
4628 287896 : layout_type (record);
4629 :
4630 287896 : TYPE_ATTRIBUTES (record) = tree_cons (get_identifier ("sysv_abi va_list"),
4631 287896 : NULL_TREE, TYPE_ATTRIBUTES (record));
4632 :
4633 : /* The correct type is an array type of one element. */
4634 287896 : return build_array_type (record, build_index_type (size_zero_node));
4635 : }
4636 :
4637 : /* Setup the builtin va_list data type and for 64-bit the additional
4638 : calling convention specific va_list data types. */
4639 :
4640 : static tree
4641 295058 : ix86_build_builtin_va_list (void)
4642 : {
4643 295058 : if (TARGET_64BIT)
4644 : {
4645 : /* Initialize ABI specific va_list builtin types.
4646 :
4647 : In lto1, we can encounter two va_list types:
4648 : - one as a result of the type-merge across TUs, and
4649 : - the one constructed here.
4650 : These two types will not have the same TYPE_MAIN_VARIANT, and therefore
4651 : a type identity check in canonical_va_list_type based on
4652 : TYPE_MAIN_VARIANT (which we used to have) will not work.
4653 : Instead, we tag each va_list_type_node with its unique attribute, and
4654 : look for the attribute in the type identity check in
4655 : canonical_va_list_type.
4656 :
4657 : Tagging sysv_va_list_type_node directly with the attribute is
4658 : problematic since it's a array of one record, which will degrade into a
4659 : pointer to record when used as parameter (see build_va_arg comments for
4660 : an example), dropping the attribute in the process. So we tag the
4661 : record instead. */
4662 :
4663 : /* For SYSV_ABI we use an array of one record. */
4664 287896 : sysv_va_list_type_node = ix86_build_builtin_va_list_64 ();
4665 :
4666 : /* For MS_ABI we use plain pointer to argument area. */
4667 287896 : tree char_ptr_type = build_pointer_type (char_type_node);
4668 287896 : tree attr = tree_cons (get_identifier ("ms_abi va_list"), NULL_TREE,
4669 287896 : TYPE_ATTRIBUTES (char_ptr_type));
4670 287896 : ms_va_list_type_node = build_type_attribute_variant (char_ptr_type, attr);
4671 :
4672 287896 : return ((ix86_abi == MS_ABI)
4673 287896 : ? ms_va_list_type_node
4674 287846 : : sysv_va_list_type_node);
4675 : }
4676 : else
4677 : {
4678 : /* For i386 we use plain pointer to argument area. */
4679 7162 : return build_pointer_type (char_type_node);
4680 : }
4681 : }
4682 :
4683 : /* Worker function for TARGET_SETUP_INCOMING_VARARGS. */
4684 :
4685 : static void
4686 15794 : setup_incoming_varargs_64 (CUMULATIVE_ARGS *cum)
4687 : {
4688 15794 : rtx save_area, mem;
4689 15794 : alias_set_type set;
4690 15794 : int i, max;
4691 :
4692 : /* GPR size of varargs save area. */
4693 15794 : if (cfun->va_list_gpr_size)
4694 15331 : ix86_varargs_gpr_size = X86_64_REGPARM_MAX * UNITS_PER_WORD;
4695 : else
4696 463 : ix86_varargs_gpr_size = 0;
4697 :
4698 : /* FPR size of varargs save area. We don't need it if we don't pass
4699 : anything in SSE registers. */
4700 15794 : if (TARGET_SSE && cfun->va_list_fpr_size)
4701 14724 : ix86_varargs_fpr_size = X86_64_SSE_REGPARM_MAX * 16;
4702 : else
4703 1070 : ix86_varargs_fpr_size = 0;
4704 :
4705 15794 : if (! ix86_varargs_gpr_size && ! ix86_varargs_fpr_size)
4706 : return;
4707 :
4708 15500 : save_area = frame_pointer_rtx;
4709 15500 : set = get_varargs_alias_set ();
4710 :
4711 15500 : max = cum->regno + cfun->va_list_gpr_size / UNITS_PER_WORD;
4712 15500 : if (max > X86_64_REGPARM_MAX)
4713 : max = X86_64_REGPARM_MAX;
4714 :
4715 15500 : const int *parm_regs;
4716 15500 : if (cum->preserve_none_abi)
4717 : parm_regs = x86_64_preserve_none_int_parameter_registers;
4718 : else
4719 15498 : parm_regs = x86_64_int_parameter_registers;
4720 :
4721 86087 : for (i = cum->regno; i < max; i++)
4722 : {
4723 70587 : mem = gen_rtx_MEM (word_mode,
4724 70587 : plus_constant (Pmode, save_area, i * UNITS_PER_WORD));
4725 70587 : MEM_NOTRAP_P (mem) = 1;
4726 70587 : set_mem_alias_set (mem, set);
4727 70587 : emit_move_insn (mem,
4728 70587 : gen_rtx_REG (word_mode, parm_regs[i]));
4729 : }
4730 :
4731 15500 : if (ix86_varargs_fpr_size)
4732 : {
4733 14724 : machine_mode smode;
4734 14724 : rtx_code_label *label;
4735 14724 : rtx test;
4736 :
4737 : /* Now emit code to save SSE registers. The AX parameter contains number
4738 : of SSE parameter registers used to call this function, though all we
4739 : actually check here is the zero/non-zero status. */
4740 :
4741 14724 : label = gen_label_rtx ();
4742 14724 : test = gen_rtx_EQ (VOIDmode, gen_rtx_REG (QImode, AX_REG), const0_rtx);
4743 14724 : emit_jump_insn (gen_cbranchqi4 (test, XEXP (test, 0), XEXP (test, 1),
4744 : label));
4745 :
4746 : /* ??? If !TARGET_SSE_TYPELESS_STORES, would we perform better if
4747 : we used movdqa (i.e. TImode) instead? Perhaps even better would
4748 : be if we could determine the real mode of the data, via a hook
4749 : into pass_stdarg. Ignore all that for now. */
4750 14724 : smode = V4SFmode;
4751 14724 : if (crtl->stack_alignment_needed < GET_MODE_ALIGNMENT (smode))
4752 4217 : crtl->stack_alignment_needed = GET_MODE_ALIGNMENT (smode);
4753 :
4754 14724 : max = cum->sse_regno + cfun->va_list_fpr_size / 16;
4755 14724 : if (max > X86_64_SSE_REGPARM_MAX)
4756 : max = X86_64_SSE_REGPARM_MAX;
4757 :
4758 130892 : for (i = cum->sse_regno; i < max; ++i)
4759 : {
4760 116168 : mem = plus_constant (Pmode, save_area,
4761 116168 : i * 16 + ix86_varargs_gpr_size);
4762 116168 : mem = gen_rtx_MEM (smode, mem);
4763 116168 : MEM_NOTRAP_P (mem) = 1;
4764 116168 : set_mem_alias_set (mem, set);
4765 116168 : set_mem_align (mem, GET_MODE_ALIGNMENT (smode));
4766 :
4767 116168 : emit_move_insn (mem, gen_rtx_REG (smode, GET_SSE_REGNO (i)));
4768 : }
4769 :
4770 14724 : emit_label (label);
4771 : }
4772 : }
4773 :
4774 : static void
4775 5656 : setup_incoming_varargs_ms_64 (CUMULATIVE_ARGS *cum)
4776 : {
4777 5656 : alias_set_type set = get_varargs_alias_set ();
4778 5656 : int i;
4779 :
4780 : /* Reset to zero, as there might be a sysv vaarg used
4781 : before. */
4782 5656 : ix86_varargs_gpr_size = 0;
4783 5656 : ix86_varargs_fpr_size = 0;
4784 :
4785 14170 : for (i = cum->regno; i < X86_64_MS_REGPARM_MAX; i++)
4786 : {
4787 8514 : rtx reg, mem;
4788 :
4789 8514 : mem = gen_rtx_MEM (Pmode,
4790 8514 : plus_constant (Pmode, virtual_incoming_args_rtx,
4791 8514 : i * UNITS_PER_WORD));
4792 8514 : MEM_NOTRAP_P (mem) = 1;
4793 8514 : set_mem_alias_set (mem, set);
4794 :
4795 8514 : reg = gen_rtx_REG (Pmode, x86_64_ms_abi_int_parameter_registers[i]);
4796 8514 : emit_move_insn (mem, reg);
4797 : }
4798 5656 : }
4799 :
4800 : static void
4801 21604 : ix86_setup_incoming_varargs (cumulative_args_t cum_v,
4802 : const function_arg_info &arg,
4803 : int *, int no_rtl)
4804 : {
4805 21604 : CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
4806 21604 : CUMULATIVE_ARGS next_cum;
4807 21604 : tree fntype;
4808 :
4809 : /* This argument doesn't appear to be used anymore. Which is good,
4810 : because the old code here didn't suppress rtl generation. */
4811 21604 : gcc_assert (!no_rtl);
4812 :
4813 21604 : if (!TARGET_64BIT)
4814 154 : return;
4815 :
4816 21450 : fntype = TREE_TYPE (current_function_decl);
4817 :
4818 : /* For varargs, we do not want to skip the dummy va_dcl argument.
4819 : For stdargs, we do want to skip the last named argument. */
4820 21450 : next_cum = *cum;
4821 21450 : if ((!TYPE_NO_NAMED_ARGS_STDARG_P (TREE_TYPE (current_function_decl))
4822 151 : || arg.type != NULL_TREE)
4823 21487 : && stdarg_p (fntype))
4824 21336 : ix86_function_arg_advance (pack_cumulative_args (&next_cum), arg);
4825 :
4826 21450 : if (cum->call_abi == MS_ABI)
4827 5656 : setup_incoming_varargs_ms_64 (&next_cum);
4828 : else
4829 15794 : setup_incoming_varargs_64 (&next_cum);
4830 : }
4831 :
4832 : /* Checks if TYPE is of kind va_list char *. */
4833 :
4834 : static bool
4835 73410 : is_va_list_char_pointer (tree type)
4836 : {
4837 73410 : tree canonic;
4838 :
4839 : /* For 32-bit it is always true. */
4840 73410 : if (!TARGET_64BIT)
4841 : return true;
4842 73248 : canonic = ix86_canonical_va_list_type (type);
4843 73248 : return (canonic == ms_va_list_type_node
4844 73248 : || (ix86_abi == MS_ABI && canonic == va_list_type_node));
4845 : }
4846 :
4847 : /* Implement va_start. */
4848 :
4849 : static void
4850 21092 : ix86_va_start (tree valist, rtx nextarg)
4851 : {
4852 21092 : HOST_WIDE_INT words, n_gpr, n_fpr;
4853 21092 : tree f_gpr, f_fpr, f_ovf, f_sav;
4854 21092 : tree gpr, fpr, ovf, sav, t;
4855 21092 : tree type;
4856 21092 : rtx ovf_rtx;
4857 :
4858 21092 : if (flag_split_stack
4859 12 : && cfun->machine->split_stack_varargs_pointer == NULL_RTX)
4860 : {
4861 12 : unsigned int scratch_regno;
4862 :
4863 : /* When we are splitting the stack, we can't refer to the stack
4864 : arguments using internal_arg_pointer, because they may be on
4865 : the old stack. The split stack prologue will arrange to
4866 : leave a pointer to the old stack arguments in a scratch
4867 : register, which we here copy to a pseudo-register. The split
4868 : stack prologue can't set the pseudo-register directly because
4869 : it (the prologue) runs before any registers have been saved. */
4870 :
4871 12 : scratch_regno = split_stack_prologue_scratch_regno ();
4872 12 : if (scratch_regno != INVALID_REGNUM)
4873 : {
4874 12 : rtx reg;
4875 12 : rtx_insn *seq;
4876 :
4877 16 : reg = gen_reg_rtx (Pmode);
4878 12 : cfun->machine->split_stack_varargs_pointer = reg;
4879 :
4880 12 : start_sequence ();
4881 16 : emit_move_insn (reg, gen_rtx_REG (Pmode, scratch_regno));
4882 12 : seq = end_sequence ();
4883 :
4884 12 : push_topmost_sequence ();
4885 12 : emit_insn_after (seq, entry_of_function ());
4886 12 : pop_topmost_sequence ();
4887 : }
4888 : }
4889 :
4890 : /* Only 64bit target needs something special. */
4891 21092 : if (is_va_list_char_pointer (TREE_TYPE (valist)))
4892 : {
4893 5660 : if (cfun->machine->split_stack_varargs_pointer == NULL_RTX)
4894 5656 : std_expand_builtin_va_start (valist, nextarg);
4895 : else
4896 : {
4897 4 : rtx va_r, next;
4898 :
4899 4 : va_r = expand_expr (valist, NULL_RTX, VOIDmode, EXPAND_WRITE);
4900 8 : next = expand_binop (ptr_mode, add_optab,
4901 4 : cfun->machine->split_stack_varargs_pointer,
4902 : crtl->args.arg_offset_rtx,
4903 : NULL_RTX, 0, OPTAB_LIB_WIDEN);
4904 4 : convert_move (va_r, next, 0);
4905 : }
4906 : return;
4907 : }
4908 :
4909 15432 : f_gpr = TYPE_FIELDS (TREE_TYPE (sysv_va_list_type_node));
4910 15432 : f_fpr = DECL_CHAIN (f_gpr);
4911 15432 : f_ovf = DECL_CHAIN (f_fpr);
4912 15432 : f_sav = DECL_CHAIN (f_ovf);
4913 :
4914 15432 : valist = build_simple_mem_ref (valist);
4915 15432 : TREE_TYPE (valist) = TREE_TYPE (sysv_va_list_type_node);
4916 : /* The following should be folded into the MEM_REF offset. */
4917 15432 : gpr = build3 (COMPONENT_REF, TREE_TYPE (f_gpr), unshare_expr (valist),
4918 : f_gpr, NULL_TREE);
4919 15432 : fpr = build3 (COMPONENT_REF, TREE_TYPE (f_fpr), unshare_expr (valist),
4920 : f_fpr, NULL_TREE);
4921 15432 : ovf = build3 (COMPONENT_REF, TREE_TYPE (f_ovf), unshare_expr (valist),
4922 : f_ovf, NULL_TREE);
4923 15432 : sav = build3 (COMPONENT_REF, TREE_TYPE (f_sav), unshare_expr (valist),
4924 : f_sav, NULL_TREE);
4925 :
4926 : /* Count number of gp and fp argument registers used. */
4927 15432 : words = crtl->args.info.words;
4928 15432 : n_gpr = crtl->args.info.regno;
4929 15432 : n_fpr = crtl->args.info.sse_regno;
4930 :
4931 15432 : if (cfun->va_list_gpr_size)
4932 : {
4933 15185 : type = TREE_TYPE (gpr);
4934 15185 : t = build2 (MODIFY_EXPR, type,
4935 15185 : gpr, build_int_cst (type, n_gpr * 8));
4936 15185 : TREE_SIDE_EFFECTS (t) = 1;
4937 15185 : expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
4938 : }
4939 :
4940 15432 : if (TARGET_SSE && cfun->va_list_fpr_size)
4941 : {
4942 14566 : type = TREE_TYPE (fpr);
4943 14566 : t = build2 (MODIFY_EXPR, type, fpr,
4944 14566 : build_int_cst (type, n_fpr * 16 + 8*X86_64_REGPARM_MAX));
4945 14566 : TREE_SIDE_EFFECTS (t) = 1;
4946 14566 : expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
4947 : }
4948 :
4949 : /* Find the overflow area. */
4950 15432 : type = TREE_TYPE (ovf);
4951 15432 : if (cfun->machine->split_stack_varargs_pointer == NULL_RTX)
4952 15424 : ovf_rtx = crtl->args.internal_arg_pointer;
4953 : else
4954 : ovf_rtx = cfun->machine->split_stack_varargs_pointer;
4955 15432 : t = make_tree (type, ovf_rtx);
4956 15432 : if (words != 0)
4957 492 : t = fold_build_pointer_plus_hwi (t, words * UNITS_PER_WORD);
4958 :
4959 15432 : t = build2 (MODIFY_EXPR, type, ovf, t);
4960 15432 : TREE_SIDE_EFFECTS (t) = 1;
4961 15432 : expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
4962 :
4963 15432 : if (ix86_varargs_gpr_size || ix86_varargs_fpr_size)
4964 : {
4965 : /* Find the register save area.
4966 : Prologue of the function save it right above stack frame. */
4967 15354 : type = TREE_TYPE (sav);
4968 15354 : t = make_tree (type, frame_pointer_rtx);
4969 15354 : if (!ix86_varargs_gpr_size)
4970 169 : t = fold_build_pointer_plus_hwi (t, -8 * X86_64_REGPARM_MAX);
4971 :
4972 15354 : t = build2 (MODIFY_EXPR, type, sav, t);
4973 15354 : TREE_SIDE_EFFECTS (t) = 1;
4974 15354 : expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
4975 : }
4976 : }
4977 :
4978 : /* Implement va_arg. */
4979 :
4980 : static tree
4981 52318 : ix86_gimplify_va_arg (tree valist, tree type, gimple_seq *pre_p,
4982 : gimple_seq *post_p)
4983 : {
4984 52318 : static const int intreg[6] = { 0, 1, 2, 3, 4, 5 };
4985 52318 : tree f_gpr, f_fpr, f_ovf, f_sav;
4986 52318 : tree gpr, fpr, ovf, sav, t;
4987 52318 : int size, rsize;
4988 52318 : tree lab_false, lab_over = NULL_TREE;
4989 52318 : tree addr, t2;
4990 52318 : rtx container;
4991 52318 : int indirect_p = 0;
4992 52318 : tree ptrtype;
4993 52318 : machine_mode nat_mode;
4994 52318 : unsigned int arg_boundary;
4995 52318 : unsigned int type_align;
4996 :
4997 : /* Only 64bit target needs something special. */
4998 52318 : if (is_va_list_char_pointer (TREE_TYPE (valist)))
4999 260 : return std_gimplify_va_arg_expr (valist, type, pre_p, post_p);
5000 :
5001 52058 : f_gpr = TYPE_FIELDS (TREE_TYPE (sysv_va_list_type_node));
5002 52058 : f_fpr = DECL_CHAIN (f_gpr);
5003 52058 : f_ovf = DECL_CHAIN (f_fpr);
5004 52058 : f_sav = DECL_CHAIN (f_ovf);
5005 :
5006 52058 : gpr = build3 (COMPONENT_REF, TREE_TYPE (f_gpr),
5007 : valist, f_gpr, NULL_TREE);
5008 :
5009 52058 : fpr = build3 (COMPONENT_REF, TREE_TYPE (f_fpr), valist, f_fpr, NULL_TREE);
5010 52058 : ovf = build3 (COMPONENT_REF, TREE_TYPE (f_ovf), valist, f_ovf, NULL_TREE);
5011 52058 : sav = build3 (COMPONENT_REF, TREE_TYPE (f_sav), valist, f_sav, NULL_TREE);
5012 :
5013 52058 : indirect_p = pass_va_arg_by_reference (type);
5014 52058 : if (indirect_p)
5015 103 : type = build_pointer_type (type);
5016 52058 : size = arg_int_size_in_bytes (type);
5017 52058 : rsize = CEIL (size, UNITS_PER_WORD);
5018 :
5019 52058 : nat_mode = type_natural_mode (type, NULL, false);
5020 52058 : switch (nat_mode)
5021 : {
5022 28 : case E_V16HFmode:
5023 28 : case E_V16BFmode:
5024 28 : case E_V8SFmode:
5025 28 : case E_V8SImode:
5026 28 : case E_V32QImode:
5027 28 : case E_V16HImode:
5028 28 : case E_V4DFmode:
5029 28 : case E_V4DImode:
5030 28 : case E_V32HFmode:
5031 28 : case E_V32BFmode:
5032 28 : case E_V16SFmode:
5033 28 : case E_V16SImode:
5034 28 : case E_V64QImode:
5035 28 : case E_V32HImode:
5036 28 : case E_V8DFmode:
5037 28 : case E_V8DImode:
5038 : /* Unnamed 256 and 512bit vector mode parameters are passed on stack. */
5039 28 : if (!TARGET_64BIT_MS_ABI)
5040 : {
5041 : container = NULL;
5042 : break;
5043 : }
5044 : /* FALLTHRU */
5045 :
5046 52030 : default:
5047 52030 : container = construct_container (nat_mode, TYPE_MODE (type),
5048 : type, false, X86_64_REGPARM_MAX,
5049 : X86_64_SSE_REGPARM_MAX, intreg, 0);
5050 52030 : break;
5051 : }
5052 :
5053 : /* Pull the value out of the saved registers. */
5054 :
5055 52058 : addr = create_tmp_var (ptr_type_node, "addr");
5056 52058 : type_align = TYPE_ALIGN (type);
5057 :
5058 52058 : if (container)
5059 : {
5060 28964 : int needed_intregs, needed_sseregs;
5061 28964 : bool need_temp;
5062 28964 : tree int_addr, sse_addr;
5063 :
5064 28964 : lab_false = create_artificial_label (UNKNOWN_LOCATION);
5065 28964 : lab_over = create_artificial_label (UNKNOWN_LOCATION);
5066 :
5067 28964 : examine_argument (nat_mode, type, false,
5068 : &needed_intregs, &needed_sseregs);
5069 :
5070 28964 : bool container_in_reg = false;
5071 28964 : if (REG_P (container))
5072 : container_in_reg = true;
5073 1641 : else if (GET_CODE (container) == PARALLEL
5074 1641 : && GET_MODE (container) == BLKmode
5075 580 : && XVECLEN (container, 0) == 1)
5076 : {
5077 : /* Check if it is a PARALLEL BLKmode container of an EXPR_LIST
5078 : expression in a TImode register. In this case, temp isn't
5079 : needed. Otherwise, the TImode variable will be put in the
5080 : GPR save area which guarantees only 8-byte alignment. */
5081 509 : rtx x = XVECEXP (container, 0, 0);
5082 509 : if (GET_CODE (x) == EXPR_LIST
5083 509 : && REG_P (XEXP (x, 0))
5084 509 : && XEXP (x, 1) == const0_rtx)
5085 : container_in_reg = true;
5086 : }
5087 :
5088 680 : need_temp = (!container_in_reg
5089 1150 : && ((needed_intregs && TYPE_ALIGN (type) > 64)
5090 680 : || TYPE_ALIGN (type) > 128));
5091 :
5092 : /* In case we are passing structure, verify that it is consecutive block
5093 : on the register save area. If not we need to do moves. */
5094 680 : if (!need_temp && !container_in_reg)
5095 : {
5096 : /* Verify that all registers are strictly consecutive */
5097 966 : if (SSE_REGNO_P (REGNO (XEXP (XVECEXP (container, 0, 0), 0))))
5098 : {
5099 : int i;
5100 :
5101 815 : for (i = 0; i < XVECLEN (container, 0) && !need_temp; i++)
5102 : {
5103 529 : rtx slot = XVECEXP (container, 0, i);
5104 529 : if (REGNO (XEXP (slot, 0)) != FIRST_SSE_REG + (unsigned int) i
5105 529 : || INTVAL (XEXP (slot, 1)) != i * 16)
5106 : need_temp = true;
5107 : }
5108 : }
5109 : else
5110 : {
5111 : int i;
5112 :
5113 1120 : for (i = 0; i < XVECLEN (container, 0) && !need_temp; i++)
5114 : {
5115 726 : rtx slot = XVECEXP (container, 0, i);
5116 726 : if (REGNO (XEXP (slot, 0)) != (unsigned int) i
5117 726 : || INTVAL (XEXP (slot, 1)) != i * 8)
5118 : need_temp = true;
5119 : }
5120 : }
5121 : }
5122 28964 : if (!need_temp)
5123 : {
5124 : int_addr = addr;
5125 : sse_addr = addr;
5126 : }
5127 : else
5128 : {
5129 877 : int_addr = create_tmp_var (ptr_type_node, "int_addr");
5130 877 : sse_addr = create_tmp_var (ptr_type_node, "sse_addr");
5131 : }
5132 :
5133 : /* First ensure that we fit completely in registers. */
5134 28964 : if (needed_intregs)
5135 : {
5136 18198 : t = build_int_cst (TREE_TYPE (gpr),
5137 18198 : (X86_64_REGPARM_MAX - needed_intregs + 1) * 8);
5138 18198 : t = build2 (GE_EXPR, boolean_type_node, gpr, t);
5139 18198 : t2 = build1 (GOTO_EXPR, void_type_node, lab_false);
5140 18198 : t = build3 (COND_EXPR, void_type_node, t, t2, NULL_TREE);
5141 18198 : gimplify_and_add (t, pre_p);
5142 : }
5143 28964 : if (needed_sseregs)
5144 : {
5145 11158 : t = build_int_cst (TREE_TYPE (fpr),
5146 : (X86_64_SSE_REGPARM_MAX - needed_sseregs + 1) * 16
5147 11158 : + X86_64_REGPARM_MAX * 8);
5148 11158 : t = build2 (GE_EXPR, boolean_type_node, fpr, t);
5149 11158 : t2 = build1 (GOTO_EXPR, void_type_node, lab_false);
5150 11158 : t = build3 (COND_EXPR, void_type_node, t, t2, NULL_TREE);
5151 11158 : gimplify_and_add (t, pre_p);
5152 : }
5153 :
5154 : /* Compute index to start of area used for integer regs. */
5155 28964 : if (needed_intregs)
5156 : {
5157 : /* int_addr = gpr + sav; */
5158 18198 : t = fold_build_pointer_plus (sav, gpr);
5159 18198 : gimplify_assign (int_addr, t, pre_p);
5160 : }
5161 28964 : if (needed_sseregs)
5162 : {
5163 : /* sse_addr = fpr + sav; */
5164 11158 : t = fold_build_pointer_plus (sav, fpr);
5165 11158 : gimplify_assign (sse_addr, t, pre_p);
5166 : }
5167 28964 : if (need_temp)
5168 : {
5169 877 : int i, prev_size = 0;
5170 877 : tree temp = create_tmp_var (type, "va_arg_tmp");
5171 877 : TREE_ADDRESSABLE (temp) = 1;
5172 :
5173 : /* addr = &temp; */
5174 877 : t = build1 (ADDR_EXPR, build_pointer_type (type), temp);
5175 877 : gimplify_assign (addr, t, pre_p);
5176 :
5177 3118 : for (i = 0; i < XVECLEN (container, 0); i++)
5178 : {
5179 1364 : rtx slot = XVECEXP (container, 0, i);
5180 1364 : rtx reg = XEXP (slot, 0);
5181 1364 : machine_mode mode = GET_MODE (reg);
5182 1364 : tree piece_type;
5183 1364 : tree addr_type;
5184 1364 : tree daddr_type;
5185 1364 : tree src_addr, src;
5186 1364 : int src_offset;
5187 1364 : tree dest_addr, dest;
5188 1364 : int cur_size = GET_MODE_SIZE (mode);
5189 :
5190 1364 : gcc_assert (prev_size <= INTVAL (XEXP (slot, 1)));
5191 1364 : prev_size = INTVAL (XEXP (slot, 1));
5192 1364 : if (prev_size + cur_size > size)
5193 : {
5194 30 : cur_size = size - prev_size;
5195 30 : unsigned int nbits = cur_size * BITS_PER_UNIT;
5196 30 : if (!int_mode_for_size (nbits, 1).exists (&mode))
5197 10 : mode = QImode;
5198 : }
5199 1364 : piece_type = lang_hooks.types.type_for_mode (mode, 1);
5200 1364 : if (mode == GET_MODE (reg))
5201 1334 : addr_type = build_pointer_type (piece_type);
5202 : else
5203 30 : addr_type = build_pointer_type_for_mode (piece_type, ptr_mode,
5204 : true);
5205 1364 : daddr_type = build_pointer_type_for_mode (piece_type, ptr_mode,
5206 : true);
5207 :
5208 1364 : if (SSE_REGNO_P (REGNO (reg)))
5209 : {
5210 534 : src_addr = sse_addr;
5211 534 : src_offset = (REGNO (reg) - FIRST_SSE_REG) * 16;
5212 : }
5213 : else
5214 : {
5215 830 : src_addr = int_addr;
5216 830 : src_offset = REGNO (reg) * 8;
5217 : }
5218 1364 : src_addr = fold_convert (addr_type, src_addr);
5219 1364 : src_addr = fold_build_pointer_plus_hwi (src_addr, src_offset);
5220 :
5221 1364 : dest_addr = fold_convert (daddr_type, addr);
5222 1364 : dest_addr = fold_build_pointer_plus_hwi (dest_addr, prev_size);
5223 2728 : if (cur_size == GET_MODE_SIZE (mode))
5224 : {
5225 1354 : src = build_va_arg_indirect_ref (src_addr);
5226 1354 : dest = build_va_arg_indirect_ref (dest_addr);
5227 :
5228 1354 : gimplify_assign (dest, src, pre_p);
5229 : }
5230 : else
5231 : {
5232 10 : tree copy
5233 20 : = build_call_expr (builtin_decl_implicit (BUILT_IN_MEMCPY),
5234 : 3, dest_addr, src_addr,
5235 10 : size_int (cur_size));
5236 10 : gimplify_and_add (copy, pre_p);
5237 : }
5238 1364 : prev_size += cur_size;
5239 : }
5240 : }
5241 :
5242 28964 : if (needed_intregs)
5243 : {
5244 18198 : t = build2 (PLUS_EXPR, TREE_TYPE (gpr), gpr,
5245 18198 : build_int_cst (TREE_TYPE (gpr), needed_intregs * 8));
5246 18198 : gimplify_assign (gpr, t, pre_p);
5247 : /* The GPR save area guarantees only 8-byte alignment. */
5248 18198 : if (!need_temp)
5249 17394 : type_align = MIN (type_align, 64);
5250 : }
5251 :
5252 28964 : if (needed_sseregs)
5253 : {
5254 11158 : t = build2 (PLUS_EXPR, TREE_TYPE (fpr), fpr,
5255 11158 : build_int_cst (TREE_TYPE (fpr), needed_sseregs * 16));
5256 11158 : gimplify_assign (unshare_expr (fpr), t, pre_p);
5257 : }
5258 :
5259 28964 : gimple_seq_add_stmt (pre_p, gimple_build_goto (lab_over));
5260 :
5261 28964 : gimple_seq_add_stmt (pre_p, gimple_build_label (lab_false));
5262 : }
5263 :
5264 : /* ... otherwise out of the overflow area. */
5265 :
5266 : /* When we align parameter on stack for caller, if the parameter
5267 : alignment is beyond MAX_SUPPORTED_STACK_ALIGNMENT, it will be
5268 : aligned at MAX_SUPPORTED_STACK_ALIGNMENT. We will match callee
5269 : here with caller. */
5270 52058 : arg_boundary = ix86_function_arg_boundary (VOIDmode, type);
5271 52058 : if ((unsigned int) arg_boundary > MAX_SUPPORTED_STACK_ALIGNMENT)
5272 : arg_boundary = MAX_SUPPORTED_STACK_ALIGNMENT;
5273 :
5274 : /* ARG_BOUNDARY is computed from TYPE_MAIN_VARIANT, so it can be smaller
5275 : than the alignment of a variant type that carries a user-specified
5276 : alignment (e.g. a typedef with attribute aligned). Do not let the
5277 : access claim more alignment than the argument slot is placed at. */
5278 52058 : type_align = MIN (type_align, arg_boundary);
5279 :
5280 : /* Care for on-stack alignment if needed. */
5281 52058 : if (arg_boundary <= 64 || size == 0)
5282 35018 : t = ovf;
5283 : else
5284 : {
5285 17040 : HOST_WIDE_INT align = arg_boundary / 8;
5286 17040 : t = fold_build_pointer_plus_hwi (ovf, align - 1);
5287 17040 : t = build2 (BIT_AND_EXPR, TREE_TYPE (t), t,
5288 17040 : build_int_cst (TREE_TYPE (t), -align));
5289 : }
5290 :
5291 52058 : gimplify_expr (&t, pre_p, NULL, is_gimple_val, fb_rvalue);
5292 52058 : gimplify_assign (addr, t, pre_p);
5293 :
5294 52058 : t = fold_build_pointer_plus_hwi (t, rsize * UNITS_PER_WORD);
5295 52058 : gimplify_assign (unshare_expr (ovf), t, pre_p);
5296 :
5297 52058 : if (container)
5298 28964 : gimple_seq_add_stmt (pre_p, gimple_build_label (lab_over));
5299 :
5300 52058 : type = build_aligned_type (type, type_align);
5301 52058 : ptrtype = build_pointer_type_for_mode (type, ptr_mode, true);
5302 52058 : addr = fold_convert (ptrtype, addr);
5303 :
5304 52058 : if (indirect_p)
5305 103 : addr = build_va_arg_indirect_ref (addr);
5306 52058 : return build_va_arg_indirect_ref (addr);
5307 : }
5308 :
5309 : /* Return true if OPNUM's MEM should be matched
5310 : in movabs* patterns. */
5311 :
5312 : bool
5313 421 : ix86_check_movabs (rtx insn, int opnum)
5314 : {
5315 421 : rtx set, mem;
5316 :
5317 421 : set = PATTERN (insn);
5318 421 : if (GET_CODE (set) == PARALLEL)
5319 0 : set = XVECEXP (set, 0, 0);
5320 421 : gcc_assert (GET_CODE (set) == SET);
5321 421 : mem = XEXP (set, opnum);
5322 421 : while (SUBREG_P (mem))
5323 0 : mem = SUBREG_REG (mem);
5324 421 : gcc_assert (MEM_P (mem));
5325 421 : return volatile_ok || !MEM_VOLATILE_P (mem);
5326 : }
5327 :
5328 : /* Return true if XVECEXP idx of INSN satisfies MOVS arguments. */
5329 : bool
5330 235925 : ix86_check_movs (rtx insn, int idx)
5331 : {
5332 235925 : rtx pat = PATTERN (insn);
5333 235925 : gcc_assert (GET_CODE (pat) == PARALLEL);
5334 :
5335 235925 : rtx set = XVECEXP (pat, 0, idx);
5336 235925 : gcc_assert (GET_CODE (set) == SET);
5337 :
5338 235925 : rtx dst = SET_DEST (set);
5339 235925 : gcc_assert (MEM_P (dst));
5340 :
5341 235925 : rtx src = SET_SRC (set);
5342 235925 : gcc_assert (MEM_P (src));
5343 :
5344 235925 : return (ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (dst))
5345 471850 : && (ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (src))
5346 0 : || Pmode == word_mode));
5347 : }
5348 :
5349 : /* Return false if INSN contains a MEM with a non-default address space. */
5350 : bool
5351 65499 : ix86_check_no_addr_space (rtx insn)
5352 : {
5353 65499 : subrtx_var_iterator::array_type array;
5354 1441431 : FOR_EACH_SUBRTX_VAR (iter, array, PATTERN (insn), ALL)
5355 : {
5356 1375932 : rtx x = *iter;
5357 1506930 : if (MEM_P (x) && !ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (x)))
5358 0 : return false;
5359 : }
5360 65499 : return true;
5361 65499 : }
5362 :
5363 : /* Initialize the table of extra 80387 mathematical constants. */
5364 :
5365 : static void
5366 2403 : init_ext_80387_constants (void)
5367 : {
5368 2403 : static const char * cst[5] =
5369 : {
5370 : "0.3010299956639811952256464283594894482", /* 0: fldlg2 */
5371 : "0.6931471805599453094286904741849753009", /* 1: fldln2 */
5372 : "1.4426950408889634073876517827983434472", /* 2: fldl2e */
5373 : "3.3219280948873623478083405569094566090", /* 3: fldl2t */
5374 : "3.1415926535897932385128089594061862044", /* 4: fldpi */
5375 : };
5376 2403 : int i;
5377 :
5378 14418 : for (i = 0; i < 5; i++)
5379 : {
5380 12015 : real_from_string (&ext_80387_constants_table[i], cst[i]);
5381 : /* Ensure each constant is rounded to XFmode precision. */
5382 12015 : real_convert (&ext_80387_constants_table[i],
5383 24030 : XFmode, &ext_80387_constants_table[i]);
5384 : }
5385 :
5386 2403 : ext_80387_constants_init = 1;
5387 2403 : }
5388 :
5389 : /* Return non-zero if the constant is something that
5390 : can be loaded with a special instruction. */
5391 :
5392 : int
5393 4958406 : standard_80387_constant_p (rtx x)
5394 : {
5395 4958406 : machine_mode mode = GET_MODE (x);
5396 :
5397 4958406 : const REAL_VALUE_TYPE *r;
5398 :
5399 4958406 : if (!(CONST_DOUBLE_P (x) && X87_FLOAT_MODE_P (mode)))
5400 : return -1;
5401 :
5402 4484360 : if (x == CONST0_RTX (mode))
5403 : return 1;
5404 2030273 : if (x == CONST1_RTX (mode))
5405 : return 2;
5406 :
5407 1229544 : r = CONST_DOUBLE_REAL_VALUE (x);
5408 :
5409 : /* For XFmode constants, try to find a special 80387 instruction when
5410 : optimizing for size or on those CPUs that benefit from them. */
5411 1229544 : if (mode == XFmode
5412 791721 : && (optimize_function_for_size_p (cfun) || TARGET_EXT_80387_CONSTANTS)
5413 2021265 : && !flag_rounding_math)
5414 : {
5415 783671 : int i;
5416 :
5417 783671 : if (! ext_80387_constants_init)
5418 2396 : init_ext_80387_constants ();
5419 :
5420 4690651 : for (i = 0; i < 5; i++)
5421 3916638 : if (real_identical (r, &ext_80387_constants_table[i]))
5422 9658 : return i + 3;
5423 : }
5424 :
5425 : /* Load of the constant -0.0 or -1.0 will be split as
5426 : fldz;fchs or fld1;fchs sequence. */
5427 1219886 : if (real_isnegzero (r))
5428 : return 8;
5429 1203461 : if (real_identical (r, &dconstm1))
5430 280312 : return 9;
5431 :
5432 : return 0;
5433 : }
5434 :
5435 : /* Return the opcode of the special instruction to be used to load
5436 : the constant X. */
5437 :
5438 : const char *
5439 44737 : standard_80387_constant_opcode (rtx x)
5440 : {
5441 44737 : switch (standard_80387_constant_p (x))
5442 : {
5443 : case 1:
5444 : return "fldz";
5445 25038 : case 2:
5446 25038 : return "fld1";
5447 1 : case 3:
5448 1 : return "fldlg2";
5449 10 : case 4:
5450 10 : return "fldln2";
5451 12 : case 5:
5452 12 : return "fldl2e";
5453 2 : case 6:
5454 2 : return "fldl2t";
5455 210 : case 7:
5456 210 : return "fldpi";
5457 0 : case 8:
5458 0 : case 9:
5459 0 : return "#";
5460 0 : default:
5461 0 : gcc_unreachable ();
5462 : }
5463 : }
5464 :
5465 : /* Return the CONST_DOUBLE representing the 80387 constant that is
5466 : loaded by the specified special instruction. The argument IDX
5467 : matches the return value from standard_80387_constant_p. */
5468 :
5469 : rtx
5470 24 : standard_80387_constant_rtx (int idx)
5471 : {
5472 24 : int i;
5473 :
5474 24 : if (! ext_80387_constants_init)
5475 7 : init_ext_80387_constants ();
5476 :
5477 24 : switch (idx)
5478 : {
5479 24 : case 3:
5480 24 : case 4:
5481 24 : case 5:
5482 24 : case 6:
5483 24 : case 7:
5484 24 : i = idx - 3;
5485 24 : break;
5486 :
5487 0 : default:
5488 0 : gcc_unreachable ();
5489 : }
5490 :
5491 24 : return const_double_from_real_value (ext_80387_constants_table[i],
5492 24 : XFmode);
5493 : }
5494 :
5495 : /* Return 1 if X is all bits 0, 2 if X is all bits 1
5496 : and 3 if X is all bits 1 with zero extend
5497 : in supported SSE/AVX vector mode. */
5498 :
5499 : int
5500 71216379 : standard_sse_constant_p (rtx x, machine_mode pred_mode)
5501 : {
5502 71216379 : machine_mode mode;
5503 :
5504 71216379 : if (!TARGET_SSE)
5505 : return 0;
5506 :
5507 71045032 : mode = GET_MODE (x);
5508 :
5509 71045032 : if (x == const0_rtx || const0_operand (x, mode))
5510 : return 1;
5511 :
5512 56975865 : if (x == constm1_rtx
5513 56832783 : || vector_all_ones_operand (x, mode)
5514 113138295 : || ((GET_MODE_CLASS (mode) == MODE_VECTOR_FLOAT
5515 49162471 : || GET_MODE_CLASS (pred_mode) == MODE_VECTOR_FLOAT)
5516 7001005 : && float_vector_all_ones_operand (x, mode)))
5517 : {
5518 : /* VOIDmode integer constant, get mode from the predicate. */
5519 815423 : if (mode == VOIDmode)
5520 143082 : mode = pred_mode;
5521 :
5522 1630846 : switch (GET_MODE_SIZE (mode))
5523 : {
5524 33445 : case 64:
5525 33445 : if (TARGET_AVX512F)
5526 : return 2;
5527 : break;
5528 43527 : case 32:
5529 43527 : if (TARGET_AVX2)
5530 : return 2;
5531 : break;
5532 724996 : case 16:
5533 724996 : if (TARGET_SSE2)
5534 : return 2;
5535 : break;
5536 0 : case 0:
5537 : /* VOIDmode */
5538 0 : gcc_unreachable ();
5539 : default:
5540 : break;
5541 : }
5542 : }
5543 :
5544 56174828 : if (vector_all_ones_zero_extend_half_operand (x, mode)
5545 56174828 : || vector_all_ones_zero_extend_quarter_operand (x, mode))
5546 768 : return 3;
5547 :
5548 : return 0;
5549 : }
5550 :
5551 : /* Return the opcode of the special instruction to be used to load
5552 : the constant operands[1] into operands[0]. */
5553 :
5554 : const char *
5555 469287 : standard_sse_constant_opcode (rtx_insn *insn, rtx *operands)
5556 : {
5557 469287 : machine_mode mode;
5558 469287 : rtx x = operands[1];
5559 :
5560 469287 : gcc_assert (TARGET_SSE);
5561 :
5562 469287 : mode = GET_MODE (x);
5563 :
5564 469287 : if (x == const0_rtx || const0_operand (x, mode))
5565 : {
5566 456587 : switch (get_attr_mode (insn))
5567 : {
5568 438580 : case MODE_TI:
5569 438580 : if (!EXT_REX_SSE_REG_P (operands[0]))
5570 : return "%vpxor\t%0, %d0";
5571 : /* FALLTHRU */
5572 6235 : case MODE_XI:
5573 6235 : case MODE_OI:
5574 6235 : if (EXT_REX_SSE_REG_P (operands[0]))
5575 : {
5576 65 : if (TARGET_AVX512VL)
5577 : return "vpxord\t%x0, %x0, %x0";
5578 : else
5579 28 : return "vpxord\t%g0, %g0, %g0";
5580 : }
5581 : return "vpxor\t%x0, %x0, %x0";
5582 :
5583 2130 : case MODE_V2DF:
5584 2130 : if (!EXT_REX_SSE_REG_P (operands[0]))
5585 : return "%vxorpd\t%0, %d0";
5586 : /* FALLTHRU */
5587 856 : case MODE_V8DF:
5588 856 : case MODE_V4DF:
5589 856 : if (EXT_REX_SSE_REG_P (operands[0]))
5590 : {
5591 6 : if (TARGET_AVX512DQ)
5592 : {
5593 0 : if (TARGET_AVX512VL)
5594 : return "vxorpd\t%x0, %x0, %x0";
5595 : else
5596 0 : return "vxorpd\t%g0, %g0, %g0";
5597 : }
5598 : else
5599 : {
5600 6 : if (TARGET_AVX512VL)
5601 : return "vpxorq\t%x0, %x0, %x0";
5602 : else
5603 6 : return "vpxorq\t%g0, %g0, %g0";
5604 : }
5605 : }
5606 : return "vxorpd\t%x0, %x0, %x0";
5607 :
5608 6835 : case MODE_V4SF:
5609 6835 : if (!EXT_REX_SSE_REG_P (operands[0]))
5610 : return "%vxorps\t%0, %d0";
5611 : /* FALLTHRU */
5612 2013 : case MODE_V16SF:
5613 2013 : case MODE_V8SF:
5614 2013 : if (EXT_REX_SSE_REG_P (operands[0]))
5615 : {
5616 68 : if (TARGET_AVX512DQ)
5617 : {
5618 26 : if (TARGET_AVX512VL)
5619 : return "vxorps\t%x0, %x0, %x0";
5620 : else
5621 0 : return "vxorps\t%g0, %g0, %g0";
5622 : }
5623 : else
5624 : {
5625 42 : if (TARGET_AVX512VL)
5626 : return "vpxord\t%x0, %x0, %x0";
5627 : else
5628 40 : return "vpxord\t%g0, %g0, %g0";
5629 : }
5630 : }
5631 : return "vxorps\t%x0, %x0, %x0";
5632 :
5633 0 : default:
5634 0 : gcc_unreachable ();
5635 : }
5636 : }
5637 12700 : else if (x == constm1_rtx
5638 12689 : || vector_all_ones_operand (x, mode)
5639 12767 : || (GET_MODE_CLASS (mode) == MODE_VECTOR_FLOAT
5640 45 : && float_vector_all_ones_operand (x, mode)))
5641 : {
5642 12678 : enum attr_mode insn_mode = get_attr_mode (insn);
5643 :
5644 12678 : switch (insn_mode)
5645 : {
5646 3 : case MODE_XI:
5647 3 : case MODE_V8DF:
5648 3 : case MODE_V16SF:
5649 3 : gcc_assert (TARGET_AVX512F);
5650 : return "vpternlogd\t{$0xFF, %g0, %g0, %g0|%g0, %g0, %g0, 0xFF}";
5651 :
5652 958 : case MODE_OI:
5653 958 : case MODE_V4DF:
5654 958 : case MODE_V8SF:
5655 958 : gcc_assert (TARGET_AVX2);
5656 : /* FALLTHRU */
5657 12675 : case MODE_TI:
5658 12675 : case MODE_V2DF:
5659 12675 : case MODE_V4SF:
5660 12675 : gcc_assert (TARGET_SSE2);
5661 12675 : if (EXT_REX_SSE_REG_P (operands[0]))
5662 : {
5663 2 : if (TARGET_AVX512VL)
5664 : return "vpternlogd\t{$0xFF, %0, %0, %0|%0, %0, %0, 0xFF}";
5665 : else
5666 0 : return "vpternlogd\t{$0xFF, %g0, %g0, %g0|%g0, %g0, %g0, 0xFF}";
5667 : }
5668 12673 : return (TARGET_AVX
5669 12673 : ? "vpcmpeqd\t%0, %0, %0"
5670 : : "pcmpeqd\t%0, %0");
5671 :
5672 0 : default:
5673 0 : gcc_unreachable ();
5674 : }
5675 : }
5676 22 : else if (vector_all_ones_zero_extend_half_operand (x, mode))
5677 : {
5678 40 : if (GET_MODE_SIZE (mode) == 64)
5679 : {
5680 5 : gcc_assert (TARGET_AVX512F);
5681 : return "vpcmpeqd\t%t0, %t0, %t0";
5682 : }
5683 30 : else if (GET_MODE_SIZE (mode) == 32)
5684 : {
5685 15 : gcc_assert (TARGET_AVX);
5686 : return "vpcmpeqd\t%x0, %x0, %x0";
5687 : }
5688 0 : gcc_unreachable ();
5689 : }
5690 2 : else if (vector_all_ones_zero_extend_quarter_operand (x, mode))
5691 : {
5692 2 : gcc_assert (TARGET_AVX512F);
5693 : return "vpcmpeqd\t%x0, %x0, %x0";
5694 : }
5695 :
5696 0 : gcc_unreachable ();
5697 : }
5698 :
5699 : /* Returns true if INSN can be transformed from a memory load
5700 : to a supported FP constant load. */
5701 :
5702 : bool
5703 2155647 : ix86_standard_x87sse_constant_load_p (const rtx_insn *insn, rtx dst)
5704 : {
5705 2155647 : rtx src = find_constant_src (insn);
5706 :
5707 2155647 : gcc_assert (REG_P (dst));
5708 :
5709 2155647 : if (src == NULL
5710 594997 : || (SSE_REGNO_P (REGNO (dst))
5711 470588 : && standard_sse_constant_p (src, GET_MODE (dst)) != 1)
5712 158467 : || (!TARGET_AVX512VL
5713 158406 : && EXT_REX_SSE_REGNO_P (REGNO (dst))
5714 0 : && standard_sse_constant_p (src, GET_MODE (dst)) == 1)
5715 2314114 : || (STACK_REGNO_P (REGNO (dst))
5716 124409 : && standard_80387_constant_p (src) < 1))
5717 2087970 : return false;
5718 :
5719 : return true;
5720 : }
5721 :
5722 : /* Predicate for pre-reload splitters with associated instructions,
5723 : which can match any time before the split1 pass (usually combine),
5724 : then are unconditionally split in that pass and should not be
5725 : matched again afterwards. */
5726 :
5727 : bool
5728 19356516 : ix86_pre_reload_split (void)
5729 : {
5730 19356516 : return (can_create_pseudo_p ()
5731 29742438 : && !(cfun->curr_properties & PROP_rtl_split_insns));
5732 : }
5733 :
5734 : /* Return the opcode of the TYPE_SSEMOV instruction. To move from
5735 : or to xmm16-xmm31/ymm16-ymm31 registers, we either require
5736 : TARGET_AVX512VL or it is a register to register move which can
5737 : be done with zmm register move. */
5738 :
5739 : static const char *
5740 4096417 : ix86_get_ssemov (rtx *operands, unsigned size,
5741 : enum attr_mode insn_mode, machine_mode mode)
5742 : {
5743 4096417 : char buf[128];
5744 4096417 : bool misaligned_p = (misaligned_operand (operands[0], mode)
5745 4096417 : || misaligned_operand (operands[1], mode));
5746 4096417 : bool evex_reg_p = (size == 64
5747 4008979 : || EXT_REX_SSE_REG_P (operands[0])
5748 8104658 : || EXT_REX_SSE_REG_P (operands[1]));
5749 :
5750 4096417 : bool egpr_p = (TARGET_APX_EGPR
5751 4096417 : && (x86_extended_rex2reg_mentioned_p (operands[0])
5752 198 : || x86_extended_rex2reg_mentioned_p (operands[1])));
5753 204 : bool egpr_vl = egpr_p && TARGET_AVX512VL;
5754 :
5755 4096417 : machine_mode scalar_mode;
5756 :
5757 4096417 : const char *opcode = NULL;
5758 4096417 : enum
5759 : {
5760 : opcode_int,
5761 : opcode_float,
5762 : opcode_double
5763 4096417 : } type = opcode_int;
5764 :
5765 4096417 : switch (insn_mode)
5766 : {
5767 : case MODE_V16SF:
5768 : case MODE_V8SF:
5769 : case MODE_V4SF:
5770 : scalar_mode = E_SFmode;
5771 : type = opcode_float;
5772 : break;
5773 214633 : case MODE_V8DF:
5774 214633 : case MODE_V4DF:
5775 214633 : case MODE_V2DF:
5776 214633 : scalar_mode = E_DFmode;
5777 214633 : type = opcode_double;
5778 214633 : break;
5779 1462021 : case MODE_XI:
5780 1462021 : case MODE_OI:
5781 1462021 : case MODE_TI:
5782 1462021 : scalar_mode = GET_MODE_INNER (mode);
5783 : break;
5784 0 : default:
5785 0 : gcc_unreachable ();
5786 : }
5787 :
5788 : /* NB: To move xmm16-xmm31/ymm16-ymm31 registers without AVX512VL,
5789 : we can only use zmm register move without memory operand. */
5790 4096417 : if (evex_reg_p
5791 89497 : && !TARGET_AVX512VL
5792 4146472 : && GET_MODE_SIZE (mode) < 64)
5793 : {
5794 : /* NB: Even though ix86_hard_regno_mode_ok doesn't allow
5795 : xmm16-xmm31 nor ymm16-ymm31 in 128/256 bit modes when
5796 : AVX512VL is disabled, LRA can still generate reg to
5797 : reg moves with xmm16-xmm31 and ymm16-ymm31 in 128/256 bit
5798 : modes. */
5799 0 : if (memory_operand (operands[0], mode)
5800 0 : || memory_operand (operands[1], mode))
5801 0 : gcc_unreachable ();
5802 0 : size = 64;
5803 0 : switch (type)
5804 : {
5805 0 : case opcode_int:
5806 0 : if (scalar_mode == E_HFmode || scalar_mode == E_BFmode)
5807 0 : opcode = (misaligned_p
5808 0 : ? (TARGET_AVX512BW ? "vmovdqu16" : "vmovdqu64")
5809 : : "vmovdqa64");
5810 : else
5811 0 : opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32";
5812 : break;
5813 0 : case opcode_float:
5814 0 : opcode = misaligned_p ? "vmovups" : "vmovaps";
5815 : break;
5816 0 : case opcode_double:
5817 0 : opcode = misaligned_p ? "vmovupd" : "vmovapd";
5818 : break;
5819 : }
5820 : }
5821 4096417 : else if (SCALAR_FLOAT_MODE_P (scalar_mode))
5822 : {
5823 2811615 : switch (scalar_mode)
5824 : {
5825 36826 : case E_HFmode:
5826 36826 : case E_BFmode:
5827 36826 : if (evex_reg_p || egpr_vl)
5828 11622 : opcode = (misaligned_p
5829 175 : ? (TARGET_AVX512BW
5830 : ? "vmovdqu16"
5831 : : "vmovdqu64")
5832 : : "vmovdqa64");
5833 25204 : else if (egpr_p)
5834 739336 : opcode = (misaligned_p
5835 0 : ? (TARGET_AVX512BW
5836 0 : ? "vmovdqu16"
5837 : : "%vmovups")
5838 : : "%vmovaps");
5839 : else
5840 357986 : opcode = (misaligned_p
5841 25204 : ? (TARGET_AVX512BW && evex_reg_p
5842 : ? "vmovdqu16"
5843 : : "%vmovdqu")
5844 : : "%vmovdqa");
5845 : break;
5846 2419763 : case E_SFmode:
5847 2419763 : opcode = misaligned_p ? "%vmovups" : "%vmovaps";
5848 : break;
5849 214633 : case E_DFmode:
5850 214633 : opcode = misaligned_p ? "%vmovupd" : "%vmovapd";
5851 : break;
5852 140393 : case E_TFmode:
5853 140393 : if (evex_reg_p || egpr_vl)
5854 14 : opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64";
5855 140379 : else if (egpr_p)
5856 0 : opcode = misaligned_p ? "%vmovups" : "%vmovaps";
5857 : else
5858 140379 : opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa";
5859 : break;
5860 0 : default:
5861 0 : gcc_unreachable ();
5862 : }
5863 : }
5864 1284802 : else if (SCALAR_INT_MODE_P (scalar_mode))
5865 : {
5866 1284802 : switch (scalar_mode)
5867 : {
5868 74707 : case E_QImode:
5869 74707 : if (evex_reg_p || egpr_vl)
5870 4107116 : opcode = (misaligned_p
5871 10699 : ? (TARGET_AVX512BW
5872 5264 : ? "vmovdqu8"
5873 : : "vmovdqu64")
5874 : : "vmovdqa64");
5875 64008 : else if (egpr_p)
5876 30 : opcode = (misaligned_p
5877 0 : ? (TARGET_AVX512BW
5878 : ? "vmovdqu8"
5879 : : "%vmovups")
5880 : : "%vmovaps");
5881 : else
5882 63978 : opcode = (misaligned_p
5883 : ? (TARGET_AVX512BW && evex_reg_p
5884 : ? "vmovdqu8"
5885 : : "%vmovdqu")
5886 : : "%vmovdqa");
5887 : break;
5888 44604 : case E_HImode:
5889 44604 : if (evex_reg_p || egpr_vl)
5890 3693 : opcode = (misaligned_p
5891 310 : ? (TARGET_AVX512BW
5892 : ? "vmovdqu16"
5893 : : "vmovdqu64")
5894 : : "vmovdqa64");
5895 40911 : else if (egpr_p)
5896 739336 : opcode = (misaligned_p
5897 27 : ? (TARGET_AVX512BW
5898 0 : ? "vmovdqu16"
5899 : : "%vmovups")
5900 : : "%vmovaps");
5901 : else
5902 332782 : opcode = (misaligned_p
5903 40884 : ? (TARGET_AVX512BW && evex_reg_p
5904 : ? "vmovdqu16"
5905 : : "%vmovdqu")
5906 : : "%vmovdqa");
5907 : break;
5908 189015 : case E_SImode:
5909 189015 : if (evex_reg_p || egpr_vl)
5910 8346 : opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32";
5911 180669 : else if (egpr_p)
5912 14 : opcode = misaligned_p ? "%vmovups" : "%vmovaps";
5913 : else
5914 180655 : opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa";
5915 : break;
5916 964632 : case E_DImode:
5917 964632 : case E_TImode:
5918 964632 : case E_OImode:
5919 964632 : if (evex_reg_p || egpr_vl)
5920 18787 : opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64";
5921 945845 : else if (egpr_p)
5922 26 : opcode = misaligned_p ? "%vmovups" : "%vmovaps";
5923 : else
5924 945819 : opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa";
5925 : break;
5926 11844 : case E_XImode:
5927 50005 : opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64";
5928 : break;
5929 0 : default:
5930 0 : gcc_unreachable ();
5931 : }
5932 : }
5933 : else
5934 0 : gcc_unreachable ();
5935 :
5936 4096417 : switch (size)
5937 : {
5938 87438 : case 64:
5939 87438 : snprintf (buf, sizeof (buf), "%s\t{%%g1, %%g0|%%g0, %%g1}",
5940 : opcode);
5941 87438 : break;
5942 92361 : case 32:
5943 92361 : snprintf (buf, sizeof (buf), "%s\t{%%t1, %%t0|%%t0, %%t1}",
5944 : opcode);
5945 92361 : break;
5946 3916618 : case 16:
5947 3916618 : snprintf (buf, sizeof (buf), "%s\t{%%x1, %%x0|%%x0, %%x1}",
5948 : opcode);
5949 3916618 : break;
5950 0 : default:
5951 0 : gcc_unreachable ();
5952 : }
5953 4096417 : output_asm_insn (buf, operands);
5954 4096417 : return "";
5955 : }
5956 :
5957 : /* Return the template of the TYPE_SSEMOV instruction to move
5958 : operands[1] into operands[0]. */
5959 :
5960 : const char *
5961 6458566 : ix86_output_ssemov (rtx_insn *insn, rtx *operands)
5962 : {
5963 6458566 : machine_mode mode = GET_MODE (operands[0]);
5964 6458566 : if (get_attr_type (insn) != TYPE_SSEMOV
5965 6458566 : || mode != GET_MODE (operands[1]))
5966 0 : gcc_unreachable ();
5967 :
5968 6458566 : enum attr_mode insn_mode = get_attr_mode (insn);
5969 :
5970 6458566 : switch (insn_mode)
5971 : {
5972 87438 : case MODE_XI:
5973 87438 : case MODE_V8DF:
5974 87438 : case MODE_V16SF:
5975 87438 : return ix86_get_ssemov (operands, 64, insn_mode, mode);
5976 :
5977 92361 : case MODE_OI:
5978 92361 : case MODE_V4DF:
5979 92361 : case MODE_V8SF:
5980 92361 : return ix86_get_ssemov (operands, 32, insn_mode, mode);
5981 :
5982 3916618 : case MODE_TI:
5983 3916618 : case MODE_V2DF:
5984 3916618 : case MODE_V4SF:
5985 3916618 : return ix86_get_ssemov (operands, 16, insn_mode, mode);
5986 :
5987 646431 : case MODE_DI:
5988 : /* Handle broken assemblers that require movd instead of movq. */
5989 646431 : if (GENERAL_REG_P (operands[0]))
5990 : {
5991 : if (HAVE_AS_IX86_INTERUNIT_MOVQ)
5992 : return "%vmovq\t{%1, %q0|%q0, %1}";
5993 : else
5994 : return "%vmovd\t{%1, %q0|%q0, %1}";
5995 : }
5996 578229 : else if (GENERAL_REG_P (operands[1]))
5997 : {
5998 168515 : if (HAVE_AS_IX86_INTERUNIT_MOVQ)
5999 168515 : return "%vmovq\t{%q1, %0|%0, %q1}";
6000 : else
6001 : return "%vmovd\t{%q1, %0|%0, %q1}";
6002 : }
6003 : else
6004 : return "%vmovq\t{%1, %0|%0, %1}";
6005 :
6006 194706 : case MODE_SI:
6007 194706 : if (GENERAL_REG_P (operands[0]))
6008 : return "%vmovd\t{%1, %k0|%k0, %1}";
6009 133576 : else if (GENERAL_REG_P (operands[1]))
6010 82541 : return "%vmovd\t{%k1, %0|%0, %k1}";
6011 : else
6012 : return "%vmovd\t{%1, %0|%0, %1}";
6013 :
6014 54062 : case MODE_HI:
6015 54062 : if (GENERAL_REG_P (operands[0]))
6016 : return "vmovw\t{%1, %k0|%k0, %1}";
6017 53898 : else if (GENERAL_REG_P (operands[1]))
6018 233 : return "vmovw\t{%k1, %0|%0, %k1}";
6019 : else
6020 : return "vmovw\t{%1, %0|%0, %1}";
6021 :
6022 791639 : case MODE_DF:
6023 791639 : if (TARGET_AVX && REG_P (operands[0]) && REG_P (operands[1]))
6024 : return "vmovsd\t{%d1, %0|%0, %d1}";
6025 : else
6026 790800 : return "%vmovsd\t{%1, %0|%0, %1}";
6027 :
6028 671318 : case MODE_SF:
6029 671318 : if (TARGET_AVX && REG_P (operands[0]) && REG_P (operands[1]))
6030 : return "vmovss\t{%d1, %0|%0, %d1}";
6031 : else
6032 670727 : return "%vmovss\t{%1, %0|%0, %1}";
6033 :
6034 96 : case MODE_HF:
6035 96 : case MODE_BF:
6036 96 : if (REG_P (operands[0]) && REG_P (operands[1]))
6037 : return "vmovsh\t{%d1, %0|%0, %d1}";
6038 : else
6039 0 : return "vmovsh\t{%1, %0|%0, %1}";
6040 :
6041 35 : case MODE_V1DF:
6042 35 : gcc_assert (!TARGET_AVX);
6043 : return "movlpd\t{%1, %0|%0, %1}";
6044 :
6045 3862 : case MODE_V2SF:
6046 3862 : if (TARGET_AVX && REG_P (operands[0]))
6047 : return "vmovlps\t{%1, %d0|%d0, %1}";
6048 : else
6049 3789 : return "%vmovlps\t{%1, %0|%0, %1}";
6050 :
6051 0 : default:
6052 0 : gcc_unreachable ();
6053 : }
6054 : }
6055 :
6056 : /* Returns true if OP contains a symbol reference */
6057 :
6058 : bool
6059 582057944 : symbolic_reference_mentioned_p (const_rtx op)
6060 : {
6061 582057944 : const char *fmt;
6062 582057944 : int i;
6063 :
6064 582057944 : if (SYMBOL_REF_P (op) || LABEL_REF_P (op))
6065 : return true;
6066 :
6067 440047404 : fmt = GET_RTX_FORMAT (GET_CODE (op));
6068 747016110 : for (i = GET_RTX_LENGTH (GET_CODE (op)) - 1; i >= 0; i--)
6069 : {
6070 596068641 : if (fmt[i] == 'E')
6071 : {
6072 2032036 : int j;
6073 :
6074 4077530 : for (j = XVECLEN (op, i) - 1; j >= 0; j--)
6075 3355394 : if (symbolic_reference_mentioned_p (XVECEXP (op, i, j)))
6076 : return true;
6077 : }
6078 :
6079 594036605 : else if (fmt[i] == 'e' && symbolic_reference_mentioned_p (XEXP (op, i)))
6080 : return true;
6081 : }
6082 :
6083 : return false;
6084 : }
6085 :
6086 : /* Return true if it is appropriate to emit `ret' instructions in the
6087 : body of a function. Do this only if the epilogue is simple, needing a
6088 : couple of insns. Prior to reloading, we can't tell how many registers
6089 : must be saved, so return false then. Return false if there is no frame
6090 : marker to de-allocate. */
6091 :
6092 : bool
6093 0 : ix86_can_use_return_insn_p (void)
6094 : {
6095 0 : if (ix86_function_ms_hook_prologue (current_function_decl))
6096 : return false;
6097 :
6098 0 : if (ix86_function_naked (current_function_decl))
6099 : return false;
6100 :
6101 : /* Don't use `ret' instruction in interrupt handler. */
6102 0 : if (! reload_completed
6103 0 : || frame_pointer_needed
6104 0 : || cfun->machine->func_type != TYPE_NORMAL)
6105 : return 0;
6106 :
6107 : /* Don't allow more than 32k pop, since that's all we can do
6108 : with one instruction. */
6109 0 : if (crtl->args.pops_args && crtl->args.size >= 32768)
6110 : return 0;
6111 :
6112 0 : struct ix86_frame &frame = cfun->machine->frame;
6113 0 : return (frame.stack_pointer_offset == UNITS_PER_WORD
6114 0 : && (frame.nregs + frame.nsseregs) == 0);
6115 : }
6116 :
6117 : /* Return stack frame size. get_frame_size () returns used stack slots
6118 : during compilation, which may be optimized out later. If stack frame
6119 : is needed, stack_frame_required should be true. */
6120 :
6121 : static HOST_WIDE_INT
6122 8447974 : ix86_get_frame_size (void)
6123 : {
6124 8447974 : if (cfun->machine->stack_frame_required)
6125 8376747 : return get_frame_size ();
6126 : else
6127 : return 0;
6128 : }
6129 :
6130 : /* Value should be nonzero if functions must have frame pointers.
6131 : Zero means the frame pointer need not be set up (and parms may
6132 : be accessed via the stack pointer) in functions that seem suitable. */
6133 :
6134 : static bool
6135 1253249 : ix86_frame_pointer_required (void)
6136 : {
6137 : /* If we accessed previous frames, then the generated code expects
6138 : to be able to access the saved ebp value in our frame. */
6139 1253249 : if (cfun->machine->accesses_prev_frame)
6140 : return true;
6141 :
6142 : /* Several x86 os'es need a frame pointer for other reasons,
6143 : usually pertaining to setjmp. */
6144 1253216 : if (SUBTARGET_FRAME_POINTER_REQUIRED)
6145 : return true;
6146 :
6147 : /* For older 32-bit runtimes setjmp requires valid frame-pointer. */
6148 1253216 : if (TARGET_32BIT_MS_ABI && cfun->calls_setjmp)
6149 : return true;
6150 :
6151 : /* Win64 SEH, very large frames need a frame-pointer as maximum stack
6152 : allocation is 4GB. */
6153 1253216 : if (TARGET_64BIT_MS_ABI && ix86_get_frame_size () > SEH_MAX_FRAME_SIZE)
6154 : return true;
6155 :
6156 : /* SSE saves require frame-pointer when stack is misaligned. */
6157 1253216 : if (TARGET_64BIT_MS_ABI && ix86_incoming_stack_boundary < 128)
6158 : return true;
6159 :
6160 : /* In ix86_option_override_internal, TARGET_OMIT_LEAF_FRAME_POINTER
6161 : turns off the frame pointer by default. Turn it back on now if
6162 : we've not got a leaf function. */
6163 1253215 : if (TARGET_OMIT_LEAF_FRAME_POINTER
6164 1253215 : && (!crtl->is_leaf
6165 0 : || ix86_current_function_calls_tls_descriptor))
6166 : return true;
6167 :
6168 : /* Several versions of mcount for the x86 assumes that there is a
6169 : frame, so we cannot allow profiling without a frame pointer. */
6170 1253215 : if (crtl->profile && !flag_fentry)
6171 12 : return true;
6172 :
6173 : return false;
6174 : }
6175 :
6176 : /* Record that the current function accesses previous call frames. */
6177 :
6178 : void
6179 972 : ix86_setup_frame_addresses (void)
6180 : {
6181 972 : cfun->machine->accesses_prev_frame = 1;
6182 972 : }
6183 :
6184 : #if defined(HAVE_GAS_HIDDEN) && (SUPPORTS_ONE_ONLY - 0)
6185 : # define USE_HIDDEN_LINKONCE 1
6186 : #else
6187 : # define USE_HIDDEN_LINKONCE 0
6188 : #endif
6189 :
6190 : /* Label count for call and return thunks. It is used to make unique
6191 : labels in call and return thunks. */
6192 : static int indirectlabelno;
6193 :
6194 : /* True if call thunk function is needed. */
6195 : static bool indirect_thunk_needed = false;
6196 :
6197 : /* Bit masks of integer registers, which contain branch target, used
6198 : by call thunk functions. */
6199 : static HARD_REG_SET indirect_thunks_used;
6200 :
6201 : /* True if return thunk function is needed. */
6202 : static bool indirect_return_needed = false;
6203 :
6204 : /* True if return thunk function via CX is needed. */
6205 : static bool indirect_return_via_cx;
6206 :
6207 : #ifndef INDIRECT_LABEL
6208 : # define INDIRECT_LABEL "LIND"
6209 : #endif
6210 :
6211 : /* Indicate what prefix is needed for an indirect branch. */
6212 : enum indirect_thunk_prefix
6213 : {
6214 : indirect_thunk_prefix_none,
6215 : indirect_thunk_prefix_nt
6216 : };
6217 :
6218 : /* Return the prefix needed for an indirect branch INSN. */
6219 :
6220 : enum indirect_thunk_prefix
6221 67 : indirect_thunk_need_prefix (rtx_insn *insn)
6222 : {
6223 67 : enum indirect_thunk_prefix need_prefix;
6224 67 : if ((cfun->machine->indirect_branch_type
6225 67 : == indirect_branch_thunk_extern)
6226 67 : && ix86_notrack_prefixed_insn_p (insn))
6227 : {
6228 : /* NOTRACK prefix is only used with external thunk so that it
6229 : can be properly updated to support CET at run-time. */
6230 : need_prefix = indirect_thunk_prefix_nt;
6231 : }
6232 : else
6233 : need_prefix = indirect_thunk_prefix_none;
6234 67 : return need_prefix;
6235 : }
6236 :
6237 : /* Fills in the label name that should be used for the indirect thunk. */
6238 :
6239 : static void
6240 73 : indirect_thunk_name (char name[32], unsigned int regno,
6241 : enum indirect_thunk_prefix need_prefix,
6242 : bool ret_p)
6243 : {
6244 73 : if (regno != INVALID_REGNUM && regno != CX_REG && ret_p)
6245 0 : gcc_unreachable ();
6246 :
6247 73 : if (USE_HIDDEN_LINKONCE)
6248 : {
6249 73 : const char *prefix;
6250 :
6251 73 : if (need_prefix == indirect_thunk_prefix_nt
6252 73 : && regno != INVALID_REGNUM)
6253 : {
6254 : /* NOTRACK prefix is only used with external thunk via
6255 : register so that NOTRACK prefix can be added to indirect
6256 : branch via register to support CET at run-time. */
6257 : prefix = "_nt";
6258 : }
6259 : else
6260 71 : prefix = "";
6261 :
6262 73 : const char *ret = ret_p ? "return" : "indirect";
6263 :
6264 73 : if (regno != INVALID_REGNUM)
6265 : {
6266 55 : const char *reg_prefix;
6267 55 : if (LEGACY_INT_REGNO_P (regno))
6268 53 : reg_prefix = TARGET_64BIT ? "r" : "e";
6269 : else
6270 : reg_prefix = "";
6271 55 : sprintf (name, "__x86_%s_thunk%s_%s%s",
6272 : ret, prefix, reg_prefix, reg_names[regno]);
6273 : }
6274 : else
6275 18 : sprintf (name, "__x86_%s_thunk%s", ret, prefix);
6276 : }
6277 : else
6278 : {
6279 : if (regno != INVALID_REGNUM)
6280 : ASM_GENERATE_INTERNAL_LABEL (name, "LITR", regno);
6281 : else
6282 : {
6283 : if (ret_p)
6284 : ASM_GENERATE_INTERNAL_LABEL (name, "LRT", 0);
6285 : else
6286 73 : ASM_GENERATE_INTERNAL_LABEL (name, "LIT", 0);
6287 : }
6288 : }
6289 73 : }
6290 :
6291 : /* Output a call and return thunk for indirect branch. If REGNO != -1,
6292 : the function address is in REGNO and the call and return thunk looks like:
6293 :
6294 : call L2
6295 : L1:
6296 : pause
6297 : lfence
6298 : jmp L1
6299 : L2:
6300 : mov %REG, (%sp)
6301 : ret
6302 :
6303 : Otherwise, the function address is on the top of stack and the
6304 : call and return thunk looks like:
6305 :
6306 : call L2
6307 : L1:
6308 : pause
6309 : lfence
6310 : jmp L1
6311 : L2:
6312 : lea WORD_SIZE(%sp), %sp
6313 : ret
6314 : */
6315 :
6316 : static void
6317 38 : output_indirect_thunk (unsigned int regno)
6318 : {
6319 38 : char indirectlabel1[32];
6320 38 : char indirectlabel2[32];
6321 :
6322 38 : ASM_GENERATE_INTERNAL_LABEL (indirectlabel1, INDIRECT_LABEL,
6323 : indirectlabelno++);
6324 38 : ASM_GENERATE_INTERNAL_LABEL (indirectlabel2, INDIRECT_LABEL,
6325 : indirectlabelno++);
6326 :
6327 : /* Call */
6328 38 : fputs ("\tcall\t", asm_out_file);
6329 38 : assemble_name_raw (asm_out_file, indirectlabel2);
6330 38 : fputc ('\n', asm_out_file);
6331 :
6332 38 : ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel1);
6333 :
6334 : /* AMD and Intel CPUs prefer each a different instruction as loop filler.
6335 : Usage of both pause + lfence is compromise solution. */
6336 38 : fprintf (asm_out_file, "\tpause\n\tlfence\n");
6337 :
6338 : /* Jump. */
6339 38 : fputs ("\tjmp\t", asm_out_file);
6340 38 : assemble_name_raw (asm_out_file, indirectlabel1);
6341 38 : fputc ('\n', asm_out_file);
6342 :
6343 38 : ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel2);
6344 :
6345 : /* The above call insn pushed a word to stack. Adjust CFI info. */
6346 38 : if (flag_asynchronous_unwind_tables && dwarf2out_do_frame ())
6347 : {
6348 38 : if (! dwarf2out_do_cfi_asm ())
6349 : {
6350 0 : dw_cfi_ref xcfi = ggc_cleared_alloc<dw_cfi_node> ();
6351 0 : xcfi->dw_cfi_opc = DW_CFA_advance_loc4;
6352 0 : xcfi->dw_cfi_oprnd1.dw_cfi_addr = ggc_strdup (indirectlabel2);
6353 0 : vec_safe_push (cfun->fde->dw_fde_cfi, xcfi);
6354 : }
6355 38 : dw_cfi_ref xcfi = ggc_cleared_alloc<dw_cfi_node> ();
6356 38 : xcfi->dw_cfi_opc = DW_CFA_def_cfa_offset;
6357 38 : xcfi->dw_cfi_oprnd1.dw_cfi_offset = 2 * UNITS_PER_WORD;
6358 38 : vec_safe_push (cfun->fde->dw_fde_cfi, xcfi);
6359 38 : dwarf2out_emit_cfi (xcfi);
6360 : }
6361 :
6362 38 : if (regno != INVALID_REGNUM)
6363 : {
6364 : /* MOV. */
6365 27 : rtx xops[2];
6366 27 : xops[0] = gen_rtx_MEM (word_mode, stack_pointer_rtx);
6367 27 : xops[1] = gen_rtx_REG (word_mode, regno);
6368 27 : output_asm_insn ("mov\t{%1, %0|%0, %1}", xops);
6369 : }
6370 : else
6371 : {
6372 : /* LEA. */
6373 11 : rtx xops[2];
6374 11 : xops[0] = stack_pointer_rtx;
6375 11 : xops[1] = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
6376 11 : output_asm_insn ("lea\t{%E1, %0|%0, %E1}", xops);
6377 : }
6378 :
6379 38 : fputs ("\tret\n", asm_out_file);
6380 38 : if ((ix86_harden_sls & harden_sls_return))
6381 1 : fputs ("\tint3\n", asm_out_file);
6382 38 : }
6383 :
6384 : /* Output a function with a call and return thunk for indirect branch.
6385 : If REGNO != INVALID_REGNUM, the function address is in REGNO.
6386 : Otherwise, the function address is on the top of stack. Thunk is
6387 : used for function return if RET_P is true. */
6388 :
6389 : static void
6390 22 : output_indirect_thunk_function (enum indirect_thunk_prefix need_prefix,
6391 : unsigned int regno, bool ret_p)
6392 : {
6393 22 : char name[32];
6394 22 : tree decl;
6395 :
6396 : /* Create __x86_indirect_thunk. */
6397 22 : indirect_thunk_name (name, regno, need_prefix, ret_p);
6398 22 : decl = build_decl (BUILTINS_LOCATION, FUNCTION_DECL,
6399 : get_identifier (name),
6400 : build_function_type_list (void_type_node, NULL_TREE));
6401 22 : DECL_RESULT (decl) = build_decl (BUILTINS_LOCATION, RESULT_DECL,
6402 : NULL_TREE, void_type_node);
6403 22 : TREE_PUBLIC (decl) = 1;
6404 22 : TREE_STATIC (decl) = 1;
6405 22 : DECL_IGNORED_P (decl) = 1;
6406 :
6407 : #if TARGET_MACHO
6408 : if (TARGET_MACHO)
6409 : {
6410 : switch_to_section (darwin_sections[picbase_thunk_section]);
6411 : fputs ("\t.weak_definition\t", asm_out_file);
6412 : assemble_name (asm_out_file, name);
6413 : fputs ("\n\t.private_extern\t", asm_out_file);
6414 : assemble_name (asm_out_file, name);
6415 : putc ('\n', asm_out_file);
6416 : ASM_OUTPUT_LABEL (asm_out_file, name);
6417 : DECL_WEAK (decl) = 1;
6418 : }
6419 : else
6420 : #endif
6421 22 : if (USE_HIDDEN_LINKONCE)
6422 : {
6423 22 : cgraph_node::create (decl)->set_comdat_group (DECL_ASSEMBLER_NAME (decl));
6424 :
6425 22 : targetm.asm_out.unique_section (decl, 0);
6426 22 : switch_to_section (get_named_section (decl, NULL, 0));
6427 :
6428 22 : targetm.asm_out.globalize_label (asm_out_file, name);
6429 22 : fputs ("\t.hidden\t", asm_out_file);
6430 22 : assemble_name (asm_out_file, name);
6431 22 : putc ('\n', asm_out_file);
6432 22 : ASM_DECLARE_FUNCTION_NAME (asm_out_file, name, decl);
6433 : }
6434 : else
6435 : {
6436 : switch_to_section (text_section);
6437 22 : ASM_OUTPUT_LABEL (asm_out_file, name);
6438 : }
6439 :
6440 22 : DECL_INITIAL (decl) = make_node (BLOCK);
6441 22 : current_function_decl = decl;
6442 22 : allocate_struct_function (decl, false);
6443 22 : init_function_start (decl);
6444 : /* We're about to hide the function body from callees of final_* by
6445 : emitting it directly; tell them we're a thunk, if they care. */
6446 22 : cfun->is_thunk = true;
6447 22 : first_function_block_is_cold = false;
6448 : /* Make sure unwind info is emitted for the thunk if needed. */
6449 22 : final_start_function (emit_barrier (), asm_out_file, 1);
6450 :
6451 22 : output_indirect_thunk (regno);
6452 :
6453 22 : final_end_function ();
6454 22 : init_insn_lengths ();
6455 22 : free_after_compilation (cfun);
6456 22 : set_cfun (NULL);
6457 22 : current_function_decl = NULL;
6458 22 : }
6459 :
6460 : static int pic_labels_used;
6461 :
6462 : /* Fills in the label name that should be used for a pc thunk for
6463 : the given register. */
6464 :
6465 : static void
6466 37558 : get_pc_thunk_name (char name[32], unsigned int regno)
6467 : {
6468 37558 : gcc_assert (!TARGET_64BIT);
6469 :
6470 37558 : if (USE_HIDDEN_LINKONCE)
6471 37558 : sprintf (name, "__x86.get_pc_thunk.%s", reg_names[regno]);
6472 : else
6473 37558 : ASM_GENERATE_INTERNAL_LABEL (name, "LPR", regno);
6474 37558 : }
6475 :
6476 :
6477 : /* This function generates code for -fpic that loads %ebx with
6478 : the return address of the caller and then returns. */
6479 :
6480 : static void
6481 238343 : ix86_code_end (void)
6482 : {
6483 238343 : rtx xops[2];
6484 238343 : unsigned int regno;
6485 :
6486 238343 : if (indirect_return_needed)
6487 6 : output_indirect_thunk_function (indirect_thunk_prefix_none,
6488 : INVALID_REGNUM, true);
6489 238343 : if (indirect_return_via_cx)
6490 0 : output_indirect_thunk_function (indirect_thunk_prefix_none,
6491 : CX_REG, true);
6492 238343 : if (indirect_thunk_needed)
6493 0 : output_indirect_thunk_function (indirect_thunk_prefix_none,
6494 : INVALID_REGNUM, false);
6495 :
6496 2145087 : for (regno = FIRST_REX_INT_REG; regno <= LAST_REX_INT_REG; regno++)
6497 : {
6498 1906744 : if (TEST_HARD_REG_BIT (indirect_thunks_used, regno))
6499 0 : output_indirect_thunk_function (indirect_thunk_prefix_none,
6500 : regno, false);
6501 : }
6502 :
6503 4051831 : for (regno = FIRST_REX2_INT_REG; regno <= LAST_REX2_INT_REG; regno++)
6504 : {
6505 3813488 : if (TEST_HARD_REG_BIT (indirect_thunks_used, regno))
6506 0 : output_indirect_thunk_function (indirect_thunk_prefix_none,
6507 : regno, false);
6508 : }
6509 :
6510 2145087 : for (regno = FIRST_INT_REG; regno <= LAST_INT_REG; regno++)
6511 : {
6512 1906744 : char name[32];
6513 1906744 : tree decl;
6514 :
6515 1906744 : if (TEST_HARD_REG_BIT (indirect_thunks_used, regno))
6516 16 : output_indirect_thunk_function (indirect_thunk_prefix_none,
6517 : regno, false);
6518 :
6519 1906744 : if (!(pic_labels_used & (1 << regno)))
6520 1903104 : continue;
6521 :
6522 3640 : get_pc_thunk_name (name, regno);
6523 :
6524 3640 : decl = build_decl (BUILTINS_LOCATION, FUNCTION_DECL,
6525 : get_identifier (name),
6526 : build_function_type_list (void_type_node, NULL_TREE));
6527 3640 : DECL_RESULT (decl) = build_decl (BUILTINS_LOCATION, RESULT_DECL,
6528 : NULL_TREE, void_type_node);
6529 3640 : TREE_PUBLIC (decl) = 1;
6530 3640 : TREE_STATIC (decl) = 1;
6531 3640 : DECL_IGNORED_P (decl) = 1;
6532 :
6533 : #if TARGET_MACHO
6534 : if (TARGET_MACHO)
6535 : {
6536 : switch_to_section (darwin_sections[picbase_thunk_section]);
6537 : fputs ("\t.weak_definition\t", asm_out_file);
6538 : assemble_name (asm_out_file, name);
6539 : fputs ("\n\t.private_extern\t", asm_out_file);
6540 : assemble_name (asm_out_file, name);
6541 : putc ('\n', asm_out_file);
6542 : ASM_OUTPUT_LABEL (asm_out_file, name);
6543 : DECL_WEAK (decl) = 1;
6544 : }
6545 : else
6546 : #endif
6547 3640 : if (USE_HIDDEN_LINKONCE)
6548 : {
6549 3640 : cgraph_node::create (decl)->set_comdat_group (DECL_ASSEMBLER_NAME (decl));
6550 :
6551 3640 : targetm.asm_out.unique_section (decl, 0);
6552 3640 : switch_to_section (get_named_section (decl, NULL, 0));
6553 :
6554 3640 : targetm.asm_out.globalize_label (asm_out_file, name);
6555 3640 : fputs ("\t.hidden\t", asm_out_file);
6556 3640 : assemble_name (asm_out_file, name);
6557 3640 : putc ('\n', asm_out_file);
6558 3640 : ASM_DECLARE_FUNCTION_NAME (asm_out_file, name, decl);
6559 : }
6560 : else
6561 : {
6562 : switch_to_section (text_section);
6563 3640 : ASM_OUTPUT_LABEL (asm_out_file, name);
6564 : }
6565 :
6566 3640 : DECL_INITIAL (decl) = make_node (BLOCK);
6567 3640 : current_function_decl = decl;
6568 3640 : allocate_struct_function (decl, false);
6569 3640 : init_function_start (decl);
6570 : /* We're about to hide the function body from callees of final_* by
6571 : emitting it directly; tell them we're a thunk, if they care. */
6572 3640 : cfun->is_thunk = true;
6573 3640 : first_function_block_is_cold = false;
6574 : /* Make sure unwind info is emitted for the thunk if needed. */
6575 3640 : final_start_function (emit_barrier (), asm_out_file, 1);
6576 :
6577 : /* Pad stack IP move with 4 instructions (two NOPs count
6578 : as one instruction). */
6579 3640 : if (TARGET_PAD_SHORT_FUNCTION)
6580 : {
6581 : int i = 8;
6582 :
6583 0 : while (i--)
6584 0 : fputs ("\tnop\n", asm_out_file);
6585 : }
6586 :
6587 7280 : xops[0] = gen_rtx_REG (Pmode, regno);
6588 7280 : xops[1] = gen_rtx_MEM (Pmode, stack_pointer_rtx);
6589 3640 : output_asm_insn ("mov%z0\t{%1, %0|%0, %1}", xops);
6590 3640 : fputs ("\tret\n", asm_out_file);
6591 3640 : final_end_function ();
6592 3640 : init_insn_lengths ();
6593 3640 : free_after_compilation (cfun);
6594 3640 : set_cfun (NULL);
6595 3640 : current_function_decl = NULL;
6596 : }
6597 :
6598 238343 : if (flag_split_stack)
6599 4712 : file_end_indicate_split_stack ();
6600 238343 : }
6601 :
6602 : /* Emit code for the SET_GOT patterns. */
6603 :
6604 : const char *
6605 33918 : output_set_got (rtx dest, rtx label)
6606 : {
6607 33918 : rtx xops[3];
6608 :
6609 33918 : xops[0] = dest;
6610 :
6611 33918 : if (TARGET_VXWORKS_GOTTPIC && TARGET_VXWORKS_RTP && flag_pic)
6612 : {
6613 : /* Load (*VXWORKS_GOTT_BASE) into the PIC register. */
6614 : xops[2] = gen_rtx_MEM (Pmode,
6615 : gen_rtx_SYMBOL_REF (Pmode, VXWORKS_GOTT_BASE));
6616 : output_asm_insn ("mov{l}\t{%2, %0|%0, %2}", xops);
6617 :
6618 : /* Load (*VXWORKS_GOTT_BASE)[VXWORKS_GOTT_INDEX] into the PIC register.
6619 : Use %P and a local symbol in order to print VXWORKS_GOTT_INDEX as
6620 : an unadorned address. */
6621 : xops[2] = gen_rtx_SYMBOL_REF (Pmode, VXWORKS_GOTT_INDEX);
6622 : SYMBOL_REF_FLAGS (xops[2]) |= SYMBOL_FLAG_LOCAL;
6623 : output_asm_insn ("mov{l}\t{%P2(%0), %0|%0, DWORD PTR %P2[%0]}", xops);
6624 : return "";
6625 : }
6626 :
6627 67836 : xops[1] = gen_rtx_SYMBOL_REF (Pmode, GOT_SYMBOL_NAME);
6628 :
6629 33918 : if (flag_pic)
6630 : {
6631 33918 : char name[32];
6632 33918 : get_pc_thunk_name (name, REGNO (dest));
6633 33918 : pic_labels_used |= 1 << REGNO (dest);
6634 :
6635 67836 : xops[2] = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (name));
6636 33918 : xops[2] = gen_rtx_MEM (QImode, xops[2]);
6637 33918 : output_asm_insn ("%!call\t%X2", xops);
6638 :
6639 : #if TARGET_MACHO
6640 : /* Output the Mach-O "canonical" pic base label name ("Lxx$pb") here.
6641 : This is what will be referenced by the Mach-O PIC subsystem. */
6642 : if (machopic_should_output_picbase_label () || !label)
6643 : ASM_OUTPUT_LABEL (asm_out_file, MACHOPIC_FUNCTION_BASE_NAME);
6644 :
6645 : /* When we are restoring the pic base at the site of a nonlocal label,
6646 : and we decided to emit the pic base above, we will still output a
6647 : local label used for calculating the correction offset (even though
6648 : the offset will be 0 in that case). */
6649 : if (label)
6650 : targetm.asm_out.internal_label (asm_out_file, "L",
6651 : CODE_LABEL_NUMBER (label));
6652 : #endif
6653 : }
6654 : else
6655 : {
6656 0 : if (TARGET_MACHO)
6657 : /* We don't need a pic base, we're not producing pic. */
6658 : gcc_unreachable ();
6659 :
6660 0 : xops[2] = gen_rtx_LABEL_REF (Pmode, label ? label : gen_label_rtx ());
6661 0 : output_asm_insn ("mov%z0\t{%2, %0|%0, %2}", xops);
6662 0 : targetm.asm_out.internal_label (asm_out_file, "L",
6663 0 : CODE_LABEL_NUMBER (XEXP (xops[2], 0)));
6664 : }
6665 :
6666 33918 : if (!TARGET_MACHO)
6667 33918 : output_asm_insn ("add%z0\t{%1, %0|%0, %1}", xops);
6668 :
6669 33918 : return "";
6670 : }
6671 :
6672 : /* Generate an "push" pattern for input ARG. */
6673 :
6674 : rtx
6675 1932249 : gen_push (rtx arg, bool ppx_p)
6676 : {
6677 1932249 : struct machine_function *m = cfun->machine;
6678 :
6679 1932249 : if (m->fs.cfa_reg == stack_pointer_rtx)
6680 1656406 : m->fs.cfa_offset += UNITS_PER_WORD;
6681 1932249 : m->fs.sp_offset += UNITS_PER_WORD;
6682 :
6683 1932249 : if (REG_P (arg) && GET_MODE (arg) != word_mode)
6684 26 : arg = gen_rtx_REG (word_mode, REGNO (arg));
6685 :
6686 1932249 : rtx stack = gen_rtx_MEM (word_mode,
6687 1932249 : gen_rtx_PRE_DEC (Pmode,
6688 : stack_pointer_rtx));
6689 3864409 : return ppx_p ? gen_pushp_di (stack, arg) : gen_rtx_SET (stack, arg);
6690 : }
6691 :
6692 : rtx
6693 21 : gen_pushfl (void)
6694 : {
6695 21 : struct machine_function *m = cfun->machine;
6696 21 : rtx flags, mem;
6697 :
6698 21 : if (m->fs.cfa_reg == stack_pointer_rtx)
6699 0 : m->fs.cfa_offset += UNITS_PER_WORD;
6700 21 : m->fs.sp_offset += UNITS_PER_WORD;
6701 :
6702 21 : flags = gen_rtx_REG (CCmode, FLAGS_REG);
6703 :
6704 21 : mem = gen_rtx_MEM (word_mode,
6705 21 : gen_rtx_PRE_DEC (Pmode, stack_pointer_rtx));
6706 :
6707 21 : return gen_pushfl2 (word_mode, mem, flags);
6708 : }
6709 :
6710 : /* Generate an "pop" pattern for input ARG. */
6711 :
6712 : rtx
6713 1509902 : gen_pop (rtx arg, bool ppx_p)
6714 : {
6715 1509902 : if (REG_P (arg) && GET_MODE (arg) != word_mode)
6716 22 : arg = gen_rtx_REG (word_mode, REGNO (arg));
6717 :
6718 1509902 : rtx stack = gen_rtx_MEM (word_mode,
6719 1509902 : gen_rtx_POST_INC (Pmode,
6720 : stack_pointer_rtx));
6721 :
6722 3019715 : return ppx_p ? gen_popp_di (arg, stack) : gen_rtx_SET (arg, stack);
6723 : }
6724 :
6725 : rtx
6726 21 : gen_popfl (void)
6727 : {
6728 21 : rtx flags, mem;
6729 :
6730 21 : flags = gen_rtx_REG (CCmode, FLAGS_REG);
6731 :
6732 21 : mem = gen_rtx_MEM (word_mode,
6733 21 : gen_rtx_POST_INC (Pmode, stack_pointer_rtx));
6734 :
6735 21 : return gen_popfl1 (word_mode, flags, mem);
6736 : }
6737 :
6738 : /* Generate a "push2" pattern for input ARG. */
6739 : rtx
6740 19 : gen_push2 (rtx mem, rtx reg1, rtx reg2, bool ppx_p = false)
6741 : {
6742 19 : struct machine_function *m = cfun->machine;
6743 19 : const int offset = UNITS_PER_WORD * 2;
6744 :
6745 19 : if (m->fs.cfa_reg == stack_pointer_rtx)
6746 14 : m->fs.cfa_offset += offset;
6747 19 : m->fs.sp_offset += offset;
6748 :
6749 19 : if (REG_P (reg1) && GET_MODE (reg1) != word_mode)
6750 0 : reg1 = gen_rtx_REG (word_mode, REGNO (reg1));
6751 :
6752 19 : if (REG_P (reg2) && GET_MODE (reg2) != word_mode)
6753 0 : reg2 = gen_rtx_REG (word_mode, REGNO (reg2));
6754 :
6755 19 : return ppx_p ? gen_push2p_di (mem, reg1, reg2)
6756 4 : : gen_push2_di (mem, reg1, reg2);
6757 : }
6758 :
6759 : /* Return >= 0 if there is an unused call-clobbered register available
6760 : for the entire function. */
6761 :
6762 : static unsigned int
6763 0 : ix86_select_alt_pic_regnum (void)
6764 : {
6765 0 : if (ix86_use_pseudo_pic_reg ())
6766 : return INVALID_REGNUM;
6767 :
6768 0 : if (crtl->is_leaf
6769 0 : && !crtl->profile
6770 0 : && !ix86_current_function_calls_tls_descriptor)
6771 : {
6772 0 : int i, drap;
6773 : /* Can't use the same register for both PIC and DRAP. */
6774 0 : if (crtl->drap_reg)
6775 0 : drap = REGNO (crtl->drap_reg);
6776 : else
6777 : drap = -1;
6778 0 : for (i = 2; i >= 0; --i)
6779 0 : if (i != drap && !df_regs_ever_live_p (i))
6780 : return i;
6781 : }
6782 :
6783 : return INVALID_REGNUM;
6784 : }
6785 :
6786 : /* Return true if REGNO is used by the epilogue. */
6787 :
6788 : bool
6789 1746145460 : ix86_epilogue_uses (int regno)
6790 : {
6791 : /* If there are no caller-saved registers, we preserve all registers,
6792 : except for MMX and x87 registers which aren't supported when saving
6793 : and restoring registers. Don't explicitly save SP register since
6794 : it is always preserved. */
6795 1746145460 : return (epilogue_completed
6796 275120084 : && (cfun->machine->call_saved_registers
6797 275120084 : == TYPE_NO_CALLER_SAVED_REGISTERS)
6798 29986 : && !fixed_regs[regno]
6799 5441 : && !STACK_REGNO_P (regno)
6800 1746150901 : && !MMX_REGNO_P (regno));
6801 : }
6802 :
6803 : /* Return nonzero if register REGNO can be used as a scratch register
6804 : in peephole2. */
6805 :
6806 : static bool
6807 1329901 : ix86_hard_regno_scratch_ok (unsigned int regno)
6808 : {
6809 : /* If there are no caller-saved registers, we can't use any register
6810 : as a scratch register after epilogue and use REGNO as scratch
6811 : register only if it has been used before to avoid saving and
6812 : restoring it. */
6813 1329901 : return ((cfun->machine->call_saved_registers
6814 1329901 : != TYPE_NO_CALLER_SAVED_REGISTERS)
6815 1329901 : || (!epilogue_completed
6816 0 : && df_regs_ever_live_p (regno)));
6817 : }
6818 :
6819 : /* Return TRUE if we need to save REGNO. */
6820 :
6821 : bool
6822 361983702 : ix86_save_reg (unsigned int regno, bool maybe_eh_return, bool ignore_outlined)
6823 : {
6824 : /* Save and restore DRAP register between prologue and epilogue so
6825 : that stack pointer can be restored. */
6826 361983702 : if (crtl->drap_reg
6827 2267366 : && regno == REGNO (crtl->drap_reg)
6828 362038760 : && !cfun->machine->no_drap_save_restore)
6829 : return true;
6830 :
6831 : /* ??? Treat no_callee_saved_registers as a special case in order
6832 : to cope with -mnoreturn-no-callee-saved-registers, which is not
6833 : reflected in crtl->abi. */
6834 361928644 : if (cfun->machine->call_saved_registers == TYPE_NO_CALLEE_SAVED_REGISTERS
6835 16312 : && regno != HARD_FRAME_POINTER_REGNUM)
6836 : return false;
6837 :
6838 395915439 : if (regno == REAL_PIC_OFFSET_TABLE_REGNUM
6839 11021027 : && pic_offset_table_rtx)
6840 : {
6841 386091 : if (ix86_use_pseudo_pic_reg ())
6842 : {
6843 : /* REAL_PIC_OFFSET_TABLE_REGNUM used by call to
6844 : _mcount in prologue. */
6845 386091 : if (!TARGET_64BIT && flag_pic && crtl->profile)
6846 : return true;
6847 : }
6848 0 : else if (df_regs_ever_live_p (REAL_PIC_OFFSET_TABLE_REGNUM)
6849 0 : || crtl->profile
6850 0 : || crtl->calls_eh_return
6851 0 : || crtl->uses_const_pool
6852 0 : || cfun->has_nonlocal_label)
6853 0 : return ix86_select_alt_pic_regnum () == INVALID_REGNUM;
6854 : }
6855 :
6856 361912822 : if (crtl->calls_eh_return && maybe_eh_return)
6857 : {
6858 : unsigned i;
6859 13237 : for (i = 0; ; i++)
6860 : {
6861 20181 : unsigned test = EH_RETURN_DATA_REGNO (i);
6862 13671 : if (test == INVALID_REGNUM)
6863 : break;
6864 13671 : if (test == regno)
6865 : return true;
6866 13237 : }
6867 : }
6868 :
6869 361912388 : if (ignore_outlined && cfun->machine->call_ms2sysv)
6870 : {
6871 2642240 : unsigned count = cfun->machine->call_ms2sysv_extra_regs
6872 : + xlogue_layout::MIN_REGS;
6873 2642240 : if (xlogue_layout::is_stub_managed_reg (regno, count))
6874 : return false;
6875 : }
6876 :
6877 361413295 : return (df_regs_ever_live_p (regno)
6878 68776292 : && !fixed_regs[regno]
6879 60731806 : && !crtl->abi->clobbers_full_reg_p (regno)
6880 380512483 : && (regno != HARD_FRAME_POINTER_REGNUM || !frame_pointer_needed));
6881 : }
6882 :
6883 : /* Return number of saved general prupose registers. */
6884 :
6885 : static int
6886 8371998 : ix86_nsaved_regs (void)
6887 : {
6888 8371998 : int nregs = 0;
6889 8371998 : int regno;
6890 :
6891 795339810 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
6892 786967812 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true, true))
6893 8403132 : nregs ++;
6894 8371998 : return nregs;
6895 : }
6896 :
6897 : /* Return number of saved SSE registers. */
6898 :
6899 : static int
6900 8407894 : ix86_nsaved_sseregs (void)
6901 : {
6902 8407894 : int nregs = 0;
6903 8407894 : int regno;
6904 :
6905 7605039 : if (!TARGET_64BIT_MS_ABI
6906 8407894 : && (cfun->machine->call_saved_registers
6907 8185497 : != TYPE_NO_CALLER_SAVED_REGISTERS))
6908 : return 0;
6909 21204570 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
6910 20981364 : if (SSE_REGNO_P (regno) && ix86_save_reg (regno, true, true))
6911 1863843 : nregs ++;
6912 : return nregs;
6913 : }
6914 :
6915 : /* Given FROM and TO register numbers, say whether this elimination is
6916 : allowed. If stack alignment is needed, we can only replace argument
6917 : pointer with hard frame pointer, or replace frame pointer with stack
6918 : pointer. Otherwise, frame pointer elimination is automatically
6919 : handled and all other eliminations are valid. */
6920 :
6921 : static bool
6922 49585587 : ix86_can_eliminate (const int from, const int to)
6923 : {
6924 49585587 : if (stack_realign_fp)
6925 1732184 : return ((from == ARG_POINTER_REGNUM
6926 1732184 : && to == HARD_FRAME_POINTER_REGNUM)
6927 1732184 : || (from == FRAME_POINTER_REGNUM
6928 1299138 : && to == STACK_POINTER_REGNUM));
6929 : else
6930 47853403 : return to == STACK_POINTER_REGNUM ? !frame_pointer_needed : true;
6931 : }
6932 :
6933 : /* Return the offset between two registers, one to be eliminated, and the other
6934 : its replacement, at the start of a routine. */
6935 :
6936 : HOST_WIDE_INT
6937 142167359 : ix86_initial_elimination_offset (int from, int to)
6938 : {
6939 142167359 : struct ix86_frame &frame = cfun->machine->frame;
6940 :
6941 142167359 : if (from == ARG_POINTER_REGNUM && to == HARD_FRAME_POINTER_REGNUM)
6942 10626858 : return frame.hard_frame_pointer_offset;
6943 131540501 : else if (from == FRAME_POINTER_REGNUM
6944 131540501 : && to == HARD_FRAME_POINTER_REGNUM)
6945 8335724 : return frame.hard_frame_pointer_offset - frame.frame_pointer_offset;
6946 : else
6947 : {
6948 123204777 : gcc_assert (to == STACK_POINTER_REGNUM);
6949 :
6950 123204777 : if (from == ARG_POINTER_REGNUM)
6951 114869053 : return frame.stack_pointer_offset;
6952 :
6953 8335724 : gcc_assert (from == FRAME_POINTER_REGNUM);
6954 8335724 : return frame.stack_pointer_offset - frame.frame_pointer_offset;
6955 : }
6956 : }
6957 :
6958 : /* Emits a warning for unsupported msabi to sysv pro/epilogues. */
6959 : void
6960 0 : warn_once_call_ms2sysv_xlogues (const char *feature)
6961 : {
6962 0 : static bool warned_once = false;
6963 0 : if (!warned_once)
6964 : {
6965 0 : warning (0, "%<-mcall-ms2sysv-xlogues%> is not compatible with %s",
6966 : feature);
6967 0 : warned_once = true;
6968 : }
6969 0 : }
6970 :
6971 : /* Return the probing interval for -fstack-clash-protection. */
6972 :
6973 : static HOST_WIDE_INT
6974 494 : get_probe_interval (void)
6975 : {
6976 341 : if (flag_stack_clash_protection)
6977 412 : return (HOST_WIDE_INT_1U
6978 412 : << param_stack_clash_protection_probe_interval);
6979 : else
6980 : return (HOST_WIDE_INT_1U << STACK_CHECK_PROBE_INTERVAL_EXP);
6981 : }
6982 :
6983 : /* When using -fsplit-stack, the allocation routines set a field in
6984 : the TCB to the bottom of the stack plus this much space, measured
6985 : in bytes. */
6986 :
6987 : #define SPLIT_STACK_AVAILABLE 256
6988 :
6989 : /* Return true if push2/pop2 can be generated. */
6990 :
6991 : static bool
6992 8372667 : ix86_can_use_push2pop2 (void)
6993 : {
6994 : /* Use push2/pop2 only if the incoming stack is 16-byte aligned. */
6995 8372667 : unsigned int incoming_stack_boundary
6996 8372667 : = (crtl->parm_stack_boundary > ix86_incoming_stack_boundary
6997 8372667 : ? crtl->parm_stack_boundary : ix86_incoming_stack_boundary);
6998 8372667 : return incoming_stack_boundary % 128 == 0;
6999 : }
7000 :
7001 : /* Helper function to determine whether push2/pop2 can be used in prologue or
7002 : epilogue for register save/restore. */
7003 : static bool
7004 8371998 : ix86_pro_and_epilogue_can_use_push2pop2 (int nregs)
7005 : {
7006 8371998 : if (!ix86_can_use_push2pop2 ())
7007 : return false;
7008 8336030 : int aligned = cfun->machine->fs.sp_offset % 16 == 0;
7009 8336030 : return TARGET_APX_PUSH2POP2
7010 2912 : && !cfun->machine->frame.save_regs_using_mov
7011 2900 : && cfun->machine->func_type == TYPE_NORMAL
7012 8338922 : && (nregs + aligned) >= 3;
7013 : }
7014 :
7015 : /* Check if push/pop should be used to save/restore registers. */
7016 : static bool
7017 9119339 : save_regs_using_push_pop (HOST_WIDE_INT to_allocate)
7018 : {
7019 3288100 : return ((!to_allocate && cfun->machine->frame.nregs <= 1)
7020 6082099 : || (TARGET_64BIT && to_allocate >= HOST_WIDE_INT_C (0x80000000))
7021 : /* If static stack checking is enabled and done with probes,
7022 : the registers need to be saved before allocating the frame. */
7023 6081438 : || flag_stack_check == STATIC_BUILTIN_STACK_CHECK
7024 : /* If stack clash probing needs a loop, then it needs a
7025 : scratch register. But the returned register is only guaranteed
7026 : to be safe to use after register saves are complete. So if
7027 : stack clash protections are enabled and the allocated frame is
7028 : larger than the probe interval, then use pushes to save
7029 : callee saved registers. */
7030 15200703 : || (flag_stack_clash_protection
7031 341 : && !ix86_target_stack_probe ()
7032 341 : && to_allocate > get_probe_interval ()));
7033 : }
7034 :
7035 : /* Fill structure ix86_frame about frame of currently computed function. */
7036 :
7037 : static void
7038 8371998 : ix86_compute_frame_layout (void)
7039 : {
7040 8371998 : struct ix86_frame *frame = &cfun->machine->frame;
7041 8371998 : struct machine_function *m = cfun->machine;
7042 8371998 : unsigned HOST_WIDE_INT stack_alignment_needed;
7043 8371998 : HOST_WIDE_INT offset;
7044 8371998 : unsigned HOST_WIDE_INT preferred_alignment;
7045 8371998 : HOST_WIDE_INT size = ix86_get_frame_size ();
7046 8371998 : HOST_WIDE_INT to_allocate;
7047 :
7048 : /* m->call_ms2sysv is initially enabled in ix86_expand_call for all 64-bit
7049 : * ms_abi functions that call a sysv function. We now need to prune away
7050 : * cases where it should be disabled. */
7051 8371998 : if (TARGET_64BIT && m->call_ms2sysv)
7052 : {
7053 35225 : gcc_assert (TARGET_64BIT_MS_ABI);
7054 35225 : gcc_assert (TARGET_CALL_MS2SYSV_XLOGUES);
7055 35225 : gcc_assert (!TARGET_SEH);
7056 35225 : gcc_assert (TARGET_SSE);
7057 35225 : gcc_assert (!ix86_using_red_zone ());
7058 :
7059 35225 : if (crtl->calls_eh_return)
7060 : {
7061 0 : gcc_assert (!reload_completed);
7062 0 : m->call_ms2sysv = false;
7063 0 : warn_once_call_ms2sysv_xlogues ("__builtin_eh_return");
7064 : }
7065 :
7066 35225 : else if (ix86_static_chain_on_stack)
7067 : {
7068 0 : gcc_assert (!reload_completed);
7069 0 : m->call_ms2sysv = false;
7070 0 : warn_once_call_ms2sysv_xlogues ("static call chains");
7071 : }
7072 :
7073 : /* Finally, compute which registers the stub will manage. */
7074 : else
7075 : {
7076 35225 : unsigned count = xlogue_layout::count_stub_managed_regs ();
7077 35225 : m->call_ms2sysv_extra_regs = count - xlogue_layout::MIN_REGS;
7078 35225 : m->call_ms2sysv_pad_in = 0;
7079 : }
7080 : }
7081 :
7082 8371998 : frame->nregs = ix86_nsaved_regs ();
7083 8371998 : frame->nsseregs = ix86_nsaved_sseregs ();
7084 :
7085 : /* 64-bit MS ABI seem to require stack alignment to be always 16,
7086 : except for function prologues, leaf functions and when the default
7087 : incoming stack boundary is overridden at command line or via
7088 : force_align_arg_pointer attribute.
7089 :
7090 : Darwin's ABI specifies 128b alignment for both 32 and 64 bit variants
7091 : at call sites, including profile function calls.
7092 :
7093 : For APX push2/pop2, the stack also requires 128b alignment. */
7094 8371998 : if ((ix86_pro_and_epilogue_can_use_push2pop2 (frame->nregs)
7095 65 : && crtl->preferred_stack_boundary < 128)
7096 8372061 : || (((TARGET_64BIT_MS_ABI || TARGET_MACHO)
7097 222395 : && crtl->preferred_stack_boundary < 128)
7098 0 : && (!crtl->is_leaf || cfun->calls_alloca != 0
7099 0 : || ix86_current_function_calls_tls_descriptor
7100 0 : || (TARGET_MACHO && crtl->profile)
7101 0 : || ix86_incoming_stack_boundary < 128)))
7102 : {
7103 2 : crtl->preferred_stack_boundary = 128;
7104 2 : if (crtl->stack_alignment_needed < 128)
7105 1 : crtl->stack_alignment_needed = 128;
7106 : }
7107 :
7108 8371998 : stack_alignment_needed = crtl->stack_alignment_needed / BITS_PER_UNIT;
7109 8371998 : preferred_alignment = crtl->preferred_stack_boundary / BITS_PER_UNIT;
7110 :
7111 8371998 : gcc_assert (!size || stack_alignment_needed);
7112 9174824 : gcc_assert (preferred_alignment >= STACK_BOUNDARY / BITS_PER_UNIT);
7113 8371998 : gcc_assert (preferred_alignment <= stack_alignment_needed);
7114 :
7115 : /* The only ABI saving SSE regs should be 64-bit ms_abi or with
7116 : no_caller_saved_registers attribute. */
7117 8371998 : gcc_assert (TARGET_64BIT
7118 : || (cfun->machine->call_saved_registers
7119 : == TYPE_NO_CALLER_SAVED_REGISTERS)
7120 : || !frame->nsseregs);
7121 8371998 : if (TARGET_64BIT && m->call_ms2sysv)
7122 : {
7123 35225 : gcc_assert (stack_alignment_needed >= 16);
7124 35225 : gcc_assert ((cfun->machine->call_saved_registers
7125 : == TYPE_NO_CALLER_SAVED_REGISTERS)
7126 : || !frame->nsseregs);
7127 : }
7128 :
7129 : /* For SEH we have to limit the amount of code movement into the prologue.
7130 : At present we do this via a BLOCKAGE, at which point there's very little
7131 : scheduling that can be done, which means that there's very little point
7132 : in doing anything except PUSHs. */
7133 8371998 : if (TARGET_SEH)
7134 : m->use_fast_prologue_epilogue = false;
7135 8371998 : else if (!optimize_bb_for_size_p (ENTRY_BLOCK_PTR_FOR_FN (cfun)))
7136 : {
7137 8031663 : int count = frame->nregs;
7138 8031663 : struct cgraph_node *node = cgraph_node::get (current_function_decl);
7139 :
7140 : /* The fast prologue uses move instead of push to save registers. This
7141 : is significantly longer, but also executes faster as modern hardware
7142 : can execute the moves in parallel, but can't do that for push/pop.
7143 :
7144 : Be careful about choosing what prologue to emit: When function takes
7145 : many instructions to execute we may use slow version as well as in
7146 : case function is known to be outside hot spot (this is known with
7147 : feedback only). Weight the size of function by number of registers
7148 : to save as it is cheap to use one or two push instructions but very
7149 : slow to use many of them.
7150 :
7151 : Calling this hook multiple times with the same frame requirements
7152 : must produce the same layout, since the RA might otherwise be
7153 : unable to reach a fixed point or might fail its final sanity checks.
7154 : This means that once we've assumed that a function does or doesn't
7155 : have a particular size, we have to stick to that assumption
7156 : regardless of how the function has changed since. */
7157 8031663 : if (count)
7158 2670022 : count = (count - 1) * FAST_PROLOGUE_INSN_COUNT;
7159 8031663 : if (node->frequency < NODE_FREQUENCY_NORMAL
7160 7325108 : || (flag_branch_probabilities
7161 971 : && node->frequency < NODE_FREQUENCY_HOT))
7162 706880 : m->use_fast_prologue_epilogue = false;
7163 : else
7164 : {
7165 7324783 : if (count != frame->expensive_count)
7166 : {
7167 293072 : frame->expensive_count = count;
7168 293072 : frame->expensive_p = expensive_function_p (count);
7169 : }
7170 7324783 : m->use_fast_prologue_epilogue = !frame->expensive_p;
7171 : }
7172 : }
7173 :
7174 8371998 : frame->save_regs_using_mov
7175 8371998 : = TARGET_PROLOGUE_USING_MOVE && m->use_fast_prologue_epilogue;
7176 :
7177 : /* Skip return address and error code in exception handler. */
7178 8371998 : offset = INCOMING_FRAME_SP_OFFSET;
7179 :
7180 : /* Skip pushed static chain. */
7181 8371998 : if (ix86_static_chain_on_stack)
7182 0 : offset += UNITS_PER_WORD;
7183 :
7184 : /* Skip saved base pointer. */
7185 8371998 : if (frame_pointer_needed)
7186 2858241 : offset += UNITS_PER_WORD;
7187 8371998 : frame->hfp_save_offset = offset;
7188 :
7189 : /* The traditional frame pointer location is at the top of the frame. */
7190 8371998 : frame->hard_frame_pointer_offset = offset;
7191 :
7192 : /* Register save area */
7193 8371998 : offset += frame->nregs * UNITS_PER_WORD;
7194 8371998 : frame->reg_save_offset = offset;
7195 :
7196 : /* Calculate the size of the va-arg area (not including padding, if any). */
7197 8371998 : frame->va_arg_size = ix86_varargs_gpr_size + ix86_varargs_fpr_size;
7198 :
7199 : /* Also adjust stack_realign_offset for the largest alignment of
7200 : stack slot actually used. */
7201 8371998 : if (stack_realign_fp
7202 8060711 : || (cfun->machine->max_used_stack_alignment != 0
7203 188 : && (offset % cfun->machine->max_used_stack_alignment) != 0))
7204 : {
7205 : /* We may need a 16-byte aligned stack for the remainder of the
7206 : register save area, but the stack frame for the local function
7207 : may require a greater alignment if using AVX/2/512. In order
7208 : to avoid wasting space, we first calculate the space needed for
7209 : the rest of the register saves, add that to the stack pointer,
7210 : and then realign the stack to the boundary of the start of the
7211 : frame for the local function. */
7212 311373 : HOST_WIDE_INT space_needed = 0;
7213 311373 : HOST_WIDE_INT sse_reg_space_needed = 0;
7214 :
7215 311373 : if (TARGET_64BIT)
7216 : {
7217 309546 : if (m->call_ms2sysv)
7218 : {
7219 6415 : m->call_ms2sysv_pad_in = 0;
7220 6415 : space_needed = xlogue_layout::get_instance ().get_stack_space_used ();
7221 : }
7222 :
7223 303131 : else if (frame->nsseregs)
7224 : /* The only ABI that has saved SSE registers (Win64) also has a
7225 : 16-byte aligned default stack. However, many programs violate
7226 : the ABI, and Wine64 forces stack realignment to compensate. */
7227 6447 : space_needed = frame->nsseregs * 16;
7228 :
7229 309546 : sse_reg_space_needed = space_needed = ROUND_UP (space_needed, 16);
7230 :
7231 : /* 64-bit frame->va_arg_size should always be a multiple of 16, but
7232 : rounding to be pedantic. */
7233 309546 : space_needed = ROUND_UP (space_needed + frame->va_arg_size, 16);
7234 : }
7235 : else
7236 1827 : space_needed = frame->va_arg_size;
7237 :
7238 : /* Record the allocation size required prior to the realignment AND. */
7239 311373 : frame->stack_realign_allocate = space_needed;
7240 :
7241 : /* The re-aligned stack starts at frame->stack_realign_offset. Values
7242 : before this point are not directly comparable with values below
7243 : this point. Use sp_valid_at to determine if the stack pointer is
7244 : valid for a given offset, fp_valid_at for the frame pointer, or
7245 : choose_baseaddr to have a base register chosen for you.
7246 :
7247 : Note that the result of (frame->stack_realign_offset
7248 : & (stack_alignment_needed - 1)) may not equal zero. */
7249 311373 : offset = ROUND_UP (offset + space_needed, stack_alignment_needed);
7250 311373 : frame->stack_realign_offset = offset - space_needed;
7251 311373 : frame->sse_reg_save_offset = frame->stack_realign_offset
7252 311373 : + sse_reg_space_needed;
7253 311373 : }
7254 : else
7255 : {
7256 8060625 : frame->stack_realign_offset = offset;
7257 :
7258 8060625 : if (TARGET_64BIT && m->call_ms2sysv)
7259 : {
7260 28810 : m->call_ms2sysv_pad_in = !!(offset & UNITS_PER_WORD);
7261 28810 : offset += xlogue_layout::get_instance ().get_stack_space_used ();
7262 : }
7263 :
7264 : /* Align and set SSE register save area. */
7265 8031815 : else if (frame->nsseregs)
7266 : {
7267 : /* If the incoming stack boundary is at least 16 bytes, or DRAP is
7268 : required and the DRAP re-alignment boundary is at least 16 bytes,
7269 : then we want the SSE register save area properly aligned. */
7270 179934 : if (ix86_incoming_stack_boundary >= 128
7271 6400 : || (stack_realign_drap && stack_alignment_needed >= 16))
7272 179934 : offset = ROUND_UP (offset, 16);
7273 179934 : offset += frame->nsseregs * 16;
7274 : }
7275 8060625 : frame->sse_reg_save_offset = offset;
7276 8060625 : offset += frame->va_arg_size;
7277 : }
7278 :
7279 : /* Align start of frame for local function. When a function call
7280 : is removed, it may become a leaf function. But if argument may
7281 : be passed on stack, we need to align the stack when there is no
7282 : tail call. */
7283 8371998 : if (m->call_ms2sysv
7284 8336773 : || frame->va_arg_size != 0
7285 8257295 : || size != 0
7286 4489129 : || !crtl->is_leaf
7287 2104694 : || (!crtl->tail_call_emit
7288 1778088 : && cfun->machine->outgoing_args_on_stack)
7289 2104644 : || cfun->calls_alloca
7290 10474918 : || ix86_current_function_calls_tls_descriptor)
7291 6269498 : offset = ROUND_UP (offset, stack_alignment_needed);
7292 :
7293 : /* Frame pointer points here. */
7294 8371998 : frame->frame_pointer_offset = offset;
7295 :
7296 8371998 : offset += size;
7297 :
7298 : /* Add outgoing arguments area. Can be skipped if we eliminated
7299 : all the function calls as dead code.
7300 : Skipping is however impossible when function calls alloca. Alloca
7301 : expander assumes that last crtl->outgoing_args_size
7302 : of stack frame are unused. */
7303 8371998 : if (ACCUMULATE_OUTGOING_ARGS
7304 8997561 : && (!crtl->is_leaf || cfun->calls_alloca
7305 402598 : || ix86_current_function_calls_tls_descriptor))
7306 : {
7307 222965 : offset += crtl->outgoing_args_size;
7308 222965 : frame->outgoing_arguments_size = crtl->outgoing_args_size;
7309 : }
7310 : else
7311 8149033 : frame->outgoing_arguments_size = 0;
7312 :
7313 : /* Align stack boundary. Only needed if we're calling another function
7314 : or using alloca. */
7315 2847840 : if (!crtl->is_leaf || cfun->calls_alloca
7316 11216440 : || ix86_current_function_calls_tls_descriptor)
7317 5529372 : offset = ROUND_UP (offset, preferred_alignment);
7318 :
7319 : /* We've reached end of stack frame. */
7320 8371998 : frame->stack_pointer_offset = offset;
7321 :
7322 : /* Size prologue needs to allocate. */
7323 8371998 : to_allocate = offset - frame->sse_reg_save_offset;
7324 :
7325 8371998 : if (save_regs_using_push_pop (to_allocate))
7326 2652640 : frame->save_regs_using_mov = false;
7327 :
7328 8371998 : if (ix86_using_red_zone ()
7329 7346558 : && crtl->sp_is_unchanging
7330 6701810 : && crtl->is_leaf
7331 2748570 : && !cfun->machine->asm_redzone_clobber_seen
7332 2748557 : && !ix86_pc_thunk_call_expanded
7333 11120555 : && !ix86_current_function_calls_tls_descriptor)
7334 : {
7335 2748542 : frame->red_zone_size = to_allocate;
7336 2748542 : if (frame->save_regs_using_mov)
7337 143541 : frame->red_zone_size += frame->nregs * UNITS_PER_WORD;
7338 2748542 : if (frame->red_zone_size > RED_ZONE_SIZE - RED_ZONE_RESERVE)
7339 103845 : frame->red_zone_size = RED_ZONE_SIZE - RED_ZONE_RESERVE;
7340 : }
7341 : else
7342 5623456 : frame->red_zone_size = 0;
7343 8371998 : frame->stack_pointer_offset -= frame->red_zone_size;
7344 :
7345 : /* The SEH frame pointer location is near the bottom of the frame.
7346 : This is enforced by the fact that the difference between the
7347 : stack pointer and the frame pointer is limited to 240 bytes in
7348 : the unwind data structure. */
7349 8371998 : if (TARGET_SEH)
7350 : {
7351 : /* Force the frame pointer to point at or below the lowest register save
7352 : area, see the SEH code in config/i386/winnt.cc for the rationale. */
7353 : frame->hard_frame_pointer_offset = frame->sse_reg_save_offset;
7354 :
7355 : /* If we can leave the frame pointer where it is, do so; however return
7356 : the establisher frame for __builtin_frame_address (0) or else if the
7357 : frame overflows the SEH maximum frame size.
7358 :
7359 : Note that the value returned by __builtin_frame_address (0) is quite
7360 : constrained, because setjmp is piggybacked on the SEH machinery with
7361 : recent versions of MinGW:
7362 :
7363 : # elif defined(__SEH__)
7364 : # if defined(__aarch64__) || defined(_ARM64_)
7365 : # define setjmp(BUF) _setjmp((BUF), __builtin_sponentry())
7366 : # elif (__MINGW_GCC_VERSION < 40702)
7367 : # define setjmp(BUF) _setjmp((BUF), mingw_getsp())
7368 : # else
7369 : # define setjmp(BUF) _setjmp((BUF), __builtin_frame_address (0))
7370 : # endif
7371 :
7372 : and the second argument passed to _setjmp, if not null, is forwarded
7373 : to the TargetFrame parameter of RtlUnwindEx by longjmp (after it has
7374 : built an ExceptionRecord on the fly describing the setjmp buffer). */
7375 : const HOST_WIDE_INT diff
7376 : = frame->stack_pointer_offset - frame->hard_frame_pointer_offset;
7377 : if (diff <= 255 && !crtl->accesses_prior_frames)
7378 : {
7379 : /* The resulting diff will be a multiple of 16 lower than 255,
7380 : i.e. at most 240 as required by the unwind data structure. */
7381 : frame->hard_frame_pointer_offset += (diff & 15);
7382 : }
7383 : else if (diff <= SEH_MAX_FRAME_SIZE && !crtl->accesses_prior_frames)
7384 : {
7385 : /* Ideally we'd determine what portion of the local stack frame
7386 : (within the constraint of the lowest 240) is most heavily used.
7387 : But without that complication, simply bias the frame pointer
7388 : by 128 bytes so as to maximize the amount of the local stack
7389 : frame that is addressable with 8-bit offsets. */
7390 : frame->hard_frame_pointer_offset = frame->stack_pointer_offset - 128;
7391 : }
7392 : else
7393 : frame->hard_frame_pointer_offset = frame->hfp_save_offset;
7394 : }
7395 8371998 : }
7396 :
7397 : /* This is semi-inlined memory_address_length, but simplified
7398 : since we know that we're always dealing with reg+offset, and
7399 : to avoid having to create and discard all that rtl. */
7400 :
7401 : static inline int
7402 1018047 : choose_baseaddr_len (unsigned int regno, HOST_WIDE_INT offset)
7403 : {
7404 1018047 : int len = 4;
7405 :
7406 0 : if (offset == 0)
7407 : {
7408 : /* EBP and R13 cannot be encoded without an offset. */
7409 0 : len = (regno == BP_REG || regno == R13_REG);
7410 : }
7411 1009660 : else if (IN_RANGE (offset, -128, 127))
7412 649693 : len = 1;
7413 :
7414 : /* ESP and R12 must be encoded with a SIB byte. */
7415 0 : if (regno == SP_REG || regno == R12_REG)
7416 0 : len++;
7417 :
7418 1018047 : return len;
7419 : }
7420 :
7421 : /* Determine if the stack pointer is valid for accessing the CFA_OFFSET in
7422 : the frame save area. The register is saved at CFA - CFA_OFFSET. */
7423 :
7424 : static bool
7425 3546140 : sp_valid_at (HOST_WIDE_INT cfa_offset)
7426 : {
7427 3546140 : const struct machine_frame_state &fs = cfun->machine->fs;
7428 3546140 : if (fs.sp_realigned && cfa_offset <= fs.sp_realigned_offset)
7429 : {
7430 : /* Validate that the cfa_offset isn't in a "no-man's land". */
7431 46788 : gcc_assert (cfa_offset <= fs.sp_realigned_fp_last);
7432 : return false;
7433 : }
7434 3499352 : return fs.sp_valid;
7435 : }
7436 :
7437 : /* Determine if the frame pointer is valid for accessing the CFA_OFFSET in
7438 : the frame save area. The register is saved at CFA - CFA_OFFSET. */
7439 :
7440 : static inline bool
7441 1354375 : fp_valid_at (HOST_WIDE_INT cfa_offset)
7442 : {
7443 1354375 : const struct machine_frame_state &fs = cfun->machine->fs;
7444 1354375 : if (fs.sp_realigned && cfa_offset > fs.sp_realigned_fp_last)
7445 : {
7446 : /* Validate that the cfa_offset isn't in a "no-man's land". */
7447 28328 : gcc_assert (cfa_offset >= fs.sp_realigned_offset);
7448 : return false;
7449 : }
7450 1326047 : return fs.fp_valid;
7451 : }
7452 :
7453 : /* Choose a base register based upon alignment requested, speed and/or
7454 : size. */
7455 :
7456 : static void
7457 1354375 : choose_basereg (HOST_WIDE_INT cfa_offset, rtx &base_reg,
7458 : HOST_WIDE_INT &base_offset,
7459 : unsigned int align_reqested, unsigned int *align)
7460 : {
7461 1354375 : const struct machine_function *m = cfun->machine;
7462 1354375 : unsigned int hfp_align;
7463 1354375 : unsigned int drap_align;
7464 1354375 : unsigned int sp_align;
7465 1354375 : bool hfp_ok = fp_valid_at (cfa_offset);
7466 1354375 : bool drap_ok = m->fs.drap_valid;
7467 1354375 : bool sp_ok = sp_valid_at (cfa_offset);
7468 :
7469 1354375 : hfp_align = drap_align = sp_align = INCOMING_STACK_BOUNDARY;
7470 :
7471 : /* Filter out any registers that don't meet the requested alignment
7472 : criteria. */
7473 1354375 : if (align_reqested)
7474 : {
7475 973633 : if (m->fs.realigned)
7476 28160 : hfp_align = drap_align = sp_align = crtl->stack_alignment_needed;
7477 : /* SEH unwind code does do not currently support REG_CFA_EXPRESSION
7478 : notes (which we would need to use a realigned stack pointer),
7479 : so disable on SEH targets. */
7480 945473 : else if (m->fs.sp_realigned)
7481 28328 : sp_align = crtl->stack_alignment_needed;
7482 :
7483 973633 : hfp_ok = hfp_ok && hfp_align >= align_reqested;
7484 973633 : drap_ok = drap_ok && drap_align >= align_reqested;
7485 973633 : sp_ok = sp_ok && sp_align >= align_reqested;
7486 : }
7487 :
7488 1354375 : if (m->use_fast_prologue_epilogue)
7489 : {
7490 : /* Choose the base register most likely to allow the most scheduling
7491 : opportunities. Generally FP is valid throughout the function,
7492 : while DRAP must be reloaded within the epilogue. But choose either
7493 : over the SP due to increased encoding size. */
7494 :
7495 631882 : if (hfp_ok)
7496 : {
7497 119965 : base_reg = hard_frame_pointer_rtx;
7498 119965 : base_offset = m->fs.fp_offset - cfa_offset;
7499 : }
7500 511917 : else if (drap_ok)
7501 : {
7502 0 : base_reg = crtl->drap_reg;
7503 0 : base_offset = 0 - cfa_offset;
7504 : }
7505 511917 : else if (sp_ok)
7506 : {
7507 511917 : base_reg = stack_pointer_rtx;
7508 511917 : base_offset = m->fs.sp_offset - cfa_offset;
7509 : }
7510 : }
7511 : else
7512 : {
7513 722493 : HOST_WIDE_INT toffset;
7514 722493 : int len = 16, tlen;
7515 :
7516 : /* Choose the base register with the smallest address encoding.
7517 : With a tie, choose FP > DRAP > SP. */
7518 722493 : if (sp_ok)
7519 : {
7520 705175 : base_reg = stack_pointer_rtx;
7521 705175 : base_offset = m->fs.sp_offset - cfa_offset;
7522 1401963 : len = choose_baseaddr_len (STACK_POINTER_REGNUM, base_offset);
7523 : }
7524 722493 : if (drap_ok)
7525 : {
7526 0 : toffset = 0 - cfa_offset;
7527 0 : tlen = choose_baseaddr_len (REGNO (crtl->drap_reg), toffset);
7528 0 : if (tlen <= len)
7529 : {
7530 0 : base_reg = crtl->drap_reg;
7531 0 : base_offset = toffset;
7532 0 : len = tlen;
7533 : }
7534 : }
7535 722493 : if (hfp_ok)
7536 : {
7537 312872 : toffset = m->fs.fp_offset - cfa_offset;
7538 312872 : tlen = choose_baseaddr_len (HARD_FRAME_POINTER_REGNUM, toffset);
7539 312872 : if (tlen <= len)
7540 : {
7541 225412 : base_reg = hard_frame_pointer_rtx;
7542 225412 : base_offset = toffset;
7543 : }
7544 : }
7545 : }
7546 :
7547 : /* Set the align return value. */
7548 1354375 : if (align)
7549 : {
7550 973633 : if (base_reg == stack_pointer_rtx)
7551 688596 : *align = sp_align;
7552 285037 : else if (base_reg == crtl->drap_reg)
7553 0 : *align = drap_align;
7554 285037 : else if (base_reg == hard_frame_pointer_rtx)
7555 285037 : *align = hfp_align;
7556 : }
7557 1354375 : }
7558 :
7559 : /* Return an RTX that points to CFA_OFFSET within the stack frame and
7560 : the alignment of address. If ALIGN is non-null, it should point to
7561 : an alignment value (in bits) that is preferred or zero and will
7562 : receive the alignment of the base register that was selected,
7563 : irrespective of rather or not CFA_OFFSET is a multiple of that
7564 : alignment value. If it is possible for the base register offset to be
7565 : non-immediate then SCRATCH_REGNO should specify a scratch register to
7566 : use.
7567 :
7568 : The valid base registers are taken from CFUN->MACHINE->FS. */
7569 :
7570 : static rtx
7571 1354375 : choose_baseaddr (HOST_WIDE_INT cfa_offset, unsigned int *align,
7572 : unsigned int scratch_regno = INVALID_REGNUM)
7573 : {
7574 1354375 : rtx base_reg = NULL;
7575 1354375 : HOST_WIDE_INT base_offset = 0;
7576 :
7577 : /* If a specific alignment is requested, try to get a base register
7578 : with that alignment first. */
7579 1354375 : if (align && *align)
7580 973633 : choose_basereg (cfa_offset, base_reg, base_offset, *align, align);
7581 :
7582 1354375 : if (!base_reg)
7583 380742 : choose_basereg (cfa_offset, base_reg, base_offset, 0, align);
7584 :
7585 1354375 : gcc_assert (base_reg != NULL);
7586 :
7587 1354375 : rtx base_offset_rtx = GEN_INT (base_offset);
7588 :
7589 1405136 : if (!x86_64_immediate_operand (base_offset_rtx, Pmode))
7590 : {
7591 1 : gcc_assert (scratch_regno != INVALID_REGNUM);
7592 :
7593 1 : rtx scratch_reg = gen_rtx_REG (Pmode, scratch_regno);
7594 1 : emit_move_insn (scratch_reg, base_offset_rtx);
7595 :
7596 1 : return gen_rtx_PLUS (Pmode, base_reg, scratch_reg);
7597 : }
7598 :
7599 1405135 : return plus_constant (Pmode, base_reg, base_offset);
7600 : }
7601 :
7602 : /* Emit code to save registers in the prologue. */
7603 :
7604 : static void
7605 440414 : ix86_emit_save_regs (void)
7606 : {
7607 440414 : int regno;
7608 440414 : rtx_insn *insn;
7609 440414 : bool use_ppx = TARGET_APX_PPX && !crtl->calls_eh_return;
7610 :
7611 440414 : if (!TARGET_APX_PUSH2POP2
7612 91 : || !ix86_can_use_push2pop2 ()
7613 440503 : || cfun->machine->func_type != TYPE_NORMAL)
7614 : {
7615 41830970 : for (regno = FIRST_PSEUDO_REGISTER - 1; regno >= 0; regno--)
7616 41390644 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true, true))
7617 : {
7618 1231709 : insn = emit_insn (gen_push (gen_rtx_REG (word_mode, regno),
7619 : use_ppx));
7620 1231709 : RTX_FRAME_RELATED_P (insn) = 1;
7621 : }
7622 : }
7623 : else
7624 : {
7625 88 : int regno_list[2];
7626 88 : regno_list[0] = regno_list[1] = -1;
7627 88 : int loaded_regnum = 0;
7628 88 : bool aligned = cfun->machine->fs.sp_offset % 16 == 0;
7629 :
7630 8360 : for (regno = FIRST_PSEUDO_REGISTER - 1; regno >= 0; regno--)
7631 8272 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true, true))
7632 : {
7633 128 : if (aligned)
7634 : {
7635 45 : regno_list[loaded_regnum++] = regno;
7636 45 : if (loaded_regnum == 2)
7637 : {
7638 19 : gcc_assert (regno_list[0] != -1
7639 : && regno_list[1] != -1
7640 : && regno_list[0] != regno_list[1]);
7641 19 : const int offset = UNITS_PER_WORD * 2;
7642 19 : rtx mem = gen_rtx_MEM (TImode,
7643 19 : gen_rtx_PRE_DEC (Pmode,
7644 : stack_pointer_rtx));
7645 19 : insn = emit_insn (gen_push2 (mem,
7646 : gen_rtx_REG (word_mode,
7647 : regno_list[0]),
7648 : gen_rtx_REG (word_mode,
7649 : regno_list[1]),
7650 : use_ppx));
7651 19 : RTX_FRAME_RELATED_P (insn) = 1;
7652 19 : rtx dwarf = gen_rtx_SEQUENCE (VOIDmode, rtvec_alloc (3));
7653 :
7654 57 : for (int i = 0; i < 2; i++)
7655 : {
7656 76 : rtx dwarf_reg = gen_rtx_REG (word_mode,
7657 38 : regno_list[i]);
7658 38 : rtx sp_offset = plus_constant (Pmode,
7659 : stack_pointer_rtx,
7660 38 : + UNITS_PER_WORD
7661 38 : * (1 - i));
7662 38 : rtx tmp = gen_rtx_SET (gen_frame_mem (DImode,
7663 : sp_offset),
7664 : dwarf_reg);
7665 38 : RTX_FRAME_RELATED_P (tmp) = 1;
7666 38 : XVECEXP (dwarf, 0, i + 1) = tmp;
7667 : }
7668 19 : rtx sp_tmp = gen_rtx_SET (stack_pointer_rtx,
7669 : plus_constant (Pmode,
7670 : stack_pointer_rtx,
7671 : -offset));
7672 19 : RTX_FRAME_RELATED_P (sp_tmp) = 1;
7673 19 : XVECEXP (dwarf, 0, 0) = sp_tmp;
7674 19 : add_reg_note (insn, REG_FRAME_RELATED_EXPR, dwarf);
7675 :
7676 19 : loaded_regnum = 0;
7677 19 : regno_list[0] = regno_list[1] = -1;
7678 : }
7679 : }
7680 : else
7681 : {
7682 83 : insn = emit_insn (gen_push (gen_rtx_REG (word_mode, regno),
7683 : use_ppx));
7684 83 : RTX_FRAME_RELATED_P (insn) = 1;
7685 83 : aligned = true;
7686 : }
7687 : }
7688 88 : if (loaded_regnum == 1)
7689 : {
7690 7 : insn = emit_insn (gen_push (gen_rtx_REG (word_mode,
7691 7 : regno_list[0]),
7692 : use_ppx));
7693 7 : RTX_FRAME_RELATED_P (insn) = 1;
7694 : }
7695 : }
7696 440414 : }
7697 :
7698 : /* Emit a single register save at CFA - CFA_OFFSET. */
7699 :
7700 : static void
7701 619438 : ix86_emit_save_reg_using_mov (machine_mode mode, unsigned int regno,
7702 : HOST_WIDE_INT cfa_offset)
7703 : {
7704 619438 : struct machine_function *m = cfun->machine;
7705 619438 : rtx reg = gen_rtx_REG (mode, regno);
7706 619438 : rtx mem, addr, base, insn;
7707 619438 : unsigned int align = GET_MODE_ALIGNMENT (mode);
7708 :
7709 619438 : addr = choose_baseaddr (cfa_offset, &align);
7710 619438 : mem = gen_frame_mem (mode, addr);
7711 :
7712 : /* The location alignment depends upon the base register. */
7713 619438 : align = MIN (GET_MODE_ALIGNMENT (mode), align);
7714 619438 : gcc_assert (! (cfa_offset & (align / BITS_PER_UNIT - 1)));
7715 619438 : set_mem_align (mem, align);
7716 :
7717 619438 : insn = emit_insn (gen_rtx_SET (mem, reg));
7718 619438 : RTX_FRAME_RELATED_P (insn) = 1;
7719 :
7720 619438 : base = addr;
7721 619438 : if (GET_CODE (base) == PLUS)
7722 607352 : base = XEXP (base, 0);
7723 619438 : gcc_checking_assert (REG_P (base));
7724 :
7725 : /* When saving registers into a re-aligned local stack frame, avoid
7726 : any tricky guessing by dwarf2out. */
7727 619438 : if (m->fs.realigned)
7728 : {
7729 12800 : gcc_checking_assert (stack_realign_drap);
7730 :
7731 12800 : if (regno == REGNO (crtl->drap_reg))
7732 : {
7733 : /* A bit of a hack. We force the DRAP register to be saved in
7734 : the re-aligned stack frame, which provides us with a copy
7735 : of the CFA that will last past the prologue. Install it. */
7736 0 : gcc_checking_assert (cfun->machine->fs.fp_valid);
7737 0 : addr = plus_constant (Pmode, hard_frame_pointer_rtx,
7738 0 : cfun->machine->fs.fp_offset - cfa_offset);
7739 0 : mem = gen_rtx_MEM (mode, addr);
7740 0 : add_reg_note (insn, REG_CFA_DEF_CFA, mem);
7741 : }
7742 : else
7743 : {
7744 : /* The frame pointer is a stable reference within the
7745 : aligned frame. Use it. */
7746 12800 : gcc_checking_assert (cfun->machine->fs.fp_valid);
7747 12800 : addr = plus_constant (Pmode, hard_frame_pointer_rtx,
7748 12800 : cfun->machine->fs.fp_offset - cfa_offset);
7749 12800 : mem = gen_rtx_MEM (mode, addr);
7750 12800 : add_reg_note (insn, REG_CFA_EXPRESSION, gen_rtx_SET (mem, reg));
7751 : }
7752 : }
7753 :
7754 606638 : else if (base == stack_pointer_rtx && m->fs.sp_realigned
7755 12881 : && cfa_offset >= m->fs.sp_realigned_offset)
7756 : {
7757 12881 : gcc_checking_assert (stack_realign_fp);
7758 12881 : add_reg_note (insn, REG_CFA_EXPRESSION, gen_rtx_SET (mem, reg));
7759 : }
7760 :
7761 : /* The memory may not be relative to the current CFA register,
7762 : which means that we may need to generate a new pattern for
7763 : use by the unwind info. */
7764 593757 : else if (base != m->fs.cfa_reg)
7765 : {
7766 43473 : addr = plus_constant (Pmode, m->fs.cfa_reg,
7767 43473 : m->fs.cfa_offset - cfa_offset);
7768 43473 : mem = gen_rtx_MEM (mode, addr);
7769 43473 : add_reg_note (insn, REG_CFA_OFFSET, gen_rtx_SET (mem, reg));
7770 : }
7771 619438 : }
7772 :
7773 : /* Emit code to save registers using MOV insns.
7774 : First register is stored at CFA - CFA_OFFSET. */
7775 : static void
7776 45895 : ix86_emit_save_regs_using_mov (HOST_WIDE_INT cfa_offset)
7777 : {
7778 45895 : unsigned int regno;
7779 :
7780 4360025 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
7781 4314130 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true, true))
7782 : {
7783 : /* Skip registers, already processed by shrink wrap separate. */
7784 195223 : if (!cfun->machine->reg_is_wrapped_separately[regno])
7785 86480 : ix86_emit_save_reg_using_mov (word_mode, regno, cfa_offset);
7786 209979 : cfa_offset -= UNITS_PER_WORD;
7787 : }
7788 45895 : }
7789 :
7790 : /* Emit code to save SSE registers using MOV insns.
7791 : First register is stored at CFA - CFA_OFFSET. */
7792 : static void
7793 33376 : ix86_emit_save_sse_regs_using_mov (HOST_WIDE_INT cfa_offset)
7794 : {
7795 33376 : unsigned int regno;
7796 :
7797 3170720 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
7798 3137344 : if (SSE_REGNO_P (regno) && ix86_save_reg (regno, true, true))
7799 : {
7800 333769 : ix86_emit_save_reg_using_mov (V4SFmode, regno, cfa_offset);
7801 333769 : cfa_offset -= GET_MODE_SIZE (V4SFmode);
7802 : }
7803 33376 : }
7804 :
7805 : static GTY(()) rtx queued_cfa_restores;
7806 :
7807 : /* Add a REG_CFA_RESTORE REG note to INSN or queue them until next stack
7808 : manipulation insn. The value is on the stack at CFA - CFA_OFFSET.
7809 : Don't add the note if the previously saved value will be left untouched
7810 : within stack red-zone till return, as unwinders can find the same value
7811 : in the register and on the stack. */
7812 :
7813 : static void
7814 2342819 : ix86_add_cfa_restore_note (rtx_insn *insn, rtx reg, HOST_WIDE_INT cfa_offset)
7815 : {
7816 2342819 : if (!crtl->shrink_wrapped
7817 2323614 : && cfa_offset <= cfun->machine->fs.red_zone_offset)
7818 : return;
7819 :
7820 769101 : if (insn)
7821 : {
7822 358255 : add_reg_note (insn, REG_CFA_RESTORE, reg);
7823 358255 : RTX_FRAME_RELATED_P (insn) = 1;
7824 : }
7825 : else
7826 410846 : queued_cfa_restores
7827 410846 : = alloc_reg_note (REG_CFA_RESTORE, reg, queued_cfa_restores);
7828 : }
7829 :
7830 : /* Add queued REG_CFA_RESTORE notes if any to INSN. */
7831 :
7832 : static void
7833 2613921 : ix86_add_queued_cfa_restore_notes (rtx insn)
7834 : {
7835 2613921 : rtx last;
7836 2613921 : if (!queued_cfa_restores)
7837 : return;
7838 410846 : for (last = queued_cfa_restores; XEXP (last, 1); last = XEXP (last, 1))
7839 : ;
7840 52965 : XEXP (last, 1) = REG_NOTES (insn);
7841 52965 : REG_NOTES (insn) = queued_cfa_restores;
7842 52965 : queued_cfa_restores = NULL_RTX;
7843 52965 : RTX_FRAME_RELATED_P (insn) = 1;
7844 : }
7845 :
7846 : /* Expand prologue or epilogue stack adjustment.
7847 : The pattern exist to put a dependency on all ebp-based memory accesses.
7848 : STYLE should be negative if instructions should be marked as frame related,
7849 : zero if %r11 register is live and cannot be freely used and positive
7850 : otherwise. */
7851 :
7852 : static rtx
7853 1614041 : pro_epilogue_adjust_stack (rtx dest, rtx src, rtx offset,
7854 : int style, bool set_cfa)
7855 : {
7856 1614041 : struct machine_function *m = cfun->machine;
7857 1614041 : rtx addend = offset;
7858 1614041 : rtx insn;
7859 1614041 : bool add_frame_related_expr = false;
7860 :
7861 1832563 : if (!x86_64_immediate_operand (offset, Pmode))
7862 : {
7863 : /* r11 is used by indirect sibcall return as well, set before the
7864 : epilogue and used after the epilogue. */
7865 199 : if (style)
7866 174 : addend = gen_rtx_REG (Pmode, R11_REG);
7867 : else
7868 : {
7869 25 : gcc_assert (src != hard_frame_pointer_rtx
7870 : && dest != hard_frame_pointer_rtx);
7871 : addend = hard_frame_pointer_rtx;
7872 : }
7873 199 : emit_insn (gen_rtx_SET (addend, offset));
7874 199 : if (style < 0)
7875 88 : add_frame_related_expr = true;
7876 : }
7877 :
7878 : /* Shrink wrap separate may insert prologue between TEST and JMP. In order
7879 : not to affect EFlags, emit add without reg clobbering. */
7880 1614041 : if (crtl->shrink_wrapped_separate)
7881 96979 : insn = emit_insn (gen_pro_epilogue_adjust_stack_add_nocc
7882 96979 : (Pmode, dest, src, addend));
7883 : else
7884 1517062 : insn = emit_insn (gen_pro_epilogue_adjust_stack_add
7885 1517062 : (Pmode, dest, src, addend));
7886 :
7887 1614041 : if (style >= 0)
7888 710900 : ix86_add_queued_cfa_restore_notes (insn);
7889 :
7890 1614041 : if (set_cfa)
7891 : {
7892 1244840 : rtx r;
7893 :
7894 1244840 : gcc_assert (m->fs.cfa_reg == src);
7895 1244840 : m->fs.cfa_offset += INTVAL (offset);
7896 1244840 : m->fs.cfa_reg = dest;
7897 :
7898 1440950 : r = gen_rtx_PLUS (Pmode, src, offset);
7899 1244840 : r = gen_rtx_SET (dest, r);
7900 1244840 : add_reg_note (insn, REG_CFA_ADJUST_CFA, r);
7901 1244840 : RTX_FRAME_RELATED_P (insn) = 1;
7902 : }
7903 369201 : else if (style < 0)
7904 : {
7905 302500 : RTX_FRAME_RELATED_P (insn) = 1;
7906 302500 : if (add_frame_related_expr)
7907 : {
7908 20 : rtx r = gen_rtx_PLUS (Pmode, src, offset);
7909 20 : r = gen_rtx_SET (dest, r);
7910 20 : add_reg_note (insn, REG_FRAME_RELATED_EXPR, r);
7911 : }
7912 : }
7913 :
7914 1614041 : if (dest == stack_pointer_rtx)
7915 : {
7916 1614041 : HOST_WIDE_INT ooffset = m->fs.sp_offset;
7917 1614041 : bool valid = m->fs.sp_valid;
7918 1614041 : bool realigned = m->fs.sp_realigned;
7919 :
7920 1614041 : if (src == hard_frame_pointer_rtx)
7921 : {
7922 29704 : valid = m->fs.fp_valid;
7923 29704 : realigned = false;
7924 29704 : ooffset = m->fs.fp_offset;
7925 : }
7926 1584337 : else if (src == crtl->drap_reg)
7927 : {
7928 0 : valid = m->fs.drap_valid;
7929 0 : realigned = false;
7930 0 : ooffset = 0;
7931 : }
7932 : else
7933 : {
7934 : /* Else there are two possibilities: SP itself, which we set
7935 : up as the default above. Or EH_RETURN_STACKADJ_RTX, which is
7936 : taken care of this by hand along the eh_return path. */
7937 1584337 : gcc_checking_assert (src == stack_pointer_rtx
7938 : || offset == const0_rtx);
7939 : }
7940 :
7941 1614041 : m->fs.sp_offset = ooffset - INTVAL (offset);
7942 1614041 : m->fs.sp_valid = valid;
7943 1614041 : m->fs.sp_realigned = realigned;
7944 : }
7945 1614041 : return insn;
7946 : }
7947 :
7948 : /* Find an available register to be used as dynamic realign argument
7949 : pointer register. Such a register will be written in prologue and
7950 : used in begin of body, so it must not be
7951 : 1. parameter passing register.
7952 : 2. GOT pointer.
7953 : We reuse static-chain register if it is available. Otherwise, we
7954 : use DI for i386 and R13 for x86-64. We chose R13 since it has
7955 : shorter encoding.
7956 :
7957 : Return: the regno of chosen register. */
7958 :
7959 : static unsigned int
7960 7243 : find_drap_reg (void)
7961 : {
7962 7243 : tree decl = cfun->decl;
7963 :
7964 : /* Always use callee-saved register if there are no caller-saved
7965 : registers. */
7966 7243 : if (TARGET_64BIT)
7967 : {
7968 : /* In preserve_none functions, any register can be used for DRAP,
7969 : except AX, R12–R15, DI, SI (argument registers), SP, and BP. */
7970 6958 : if (cfun->machine->call_saved_registers == TYPE_PRESERVE_NONE)
7971 : return R11_REG;
7972 :
7973 : /* Use R13 for nested function or function need static chain.
7974 : Since function with tail call may use any caller-saved
7975 : registers in epilogue, DRAP must not use caller-saved
7976 : register in such case. */
7977 6957 : if (DECL_STATIC_CHAIN (decl)
7978 6915 : || (cfun->machine->call_saved_registers
7979 : == TYPE_NO_CALLER_SAVED_REGISTERS)
7980 13872 : || crtl->tail_call_emit)
7981 191 : return R13_REG;
7982 :
7983 : return R10_REG;
7984 : }
7985 : else
7986 : {
7987 : /* Use DI for nested function or function need static chain.
7988 : Since function with tail call may use any caller-saved
7989 : registers in epilogue, DRAP must not use caller-saved
7990 : register in such case. */
7991 285 : if (DECL_STATIC_CHAIN (decl)
7992 285 : || (cfun->machine->call_saved_registers
7993 285 : == TYPE_NO_CALLER_SAVED_REGISTERS)
7994 285 : || crtl->tail_call_emit
7995 550 : || crtl->calls_eh_return)
7996 : return DI_REG;
7997 :
7998 : /* Reuse static chain register if it isn't used for parameter
7999 : passing. */
8000 265 : if (ix86_function_regparm (TREE_TYPE (decl), decl) <= 2)
8001 : {
8002 265 : unsigned int ccvt = ix86_get_callcvt (TREE_TYPE (decl));
8003 265 : if ((ccvt & (IX86_CALLCVT_FASTCALL | IX86_CALLCVT_THISCALL)) == 0)
8004 : return CX_REG;
8005 : }
8006 0 : return DI_REG;
8007 : }
8008 : }
8009 :
8010 : /* Return minimum incoming stack alignment. */
8011 :
8012 : static unsigned int
8013 1658409 : ix86_minimum_incoming_stack_boundary (bool sibcall)
8014 : {
8015 1658409 : unsigned int incoming_stack_boundary;
8016 :
8017 : /* Stack of interrupt handler is aligned to 128 bits in 64bit mode. */
8018 1658409 : if (cfun->machine->func_type != TYPE_NORMAL)
8019 120 : incoming_stack_boundary = TARGET_64BIT ? 128 : MIN_STACK_BOUNDARY;
8020 : /* Prefer the one specified at command line. */
8021 1658289 : else if (ix86_user_incoming_stack_boundary)
8022 : incoming_stack_boundary = ix86_user_incoming_stack_boundary;
8023 : /* In 32bit, use MIN_STACK_BOUNDARY for incoming stack boundary
8024 : if -mstackrealign is used, it isn't used for sibcall check and
8025 : estimated stack alignment is 128bit. */
8026 1658267 : else if (!sibcall
8027 1521203 : && ix86_force_align_arg_pointer
8028 4572 : && crtl->stack_alignment_estimated == 128)
8029 596 : incoming_stack_boundary = MIN_STACK_BOUNDARY;
8030 : else
8031 1657671 : incoming_stack_boundary = ix86_default_incoming_stack_boundary;
8032 :
8033 : /* Incoming stack alignment can be changed on individual functions
8034 : via force_align_arg_pointer attribute. We use the smallest
8035 : incoming stack boundary. */
8036 1658409 : if (incoming_stack_boundary > MIN_STACK_BOUNDARY
8037 3316212 : && lookup_attribute ("force_align_arg_pointer",
8038 1657803 : TYPE_ATTRIBUTES (TREE_TYPE (current_function_decl))))
8039 5708 : incoming_stack_boundary = MIN_STACK_BOUNDARY;
8040 :
8041 : /* The incoming stack frame has to be aligned at least at
8042 : parm_stack_boundary. */
8043 1658409 : if (incoming_stack_boundary < crtl->parm_stack_boundary)
8044 : incoming_stack_boundary = crtl->parm_stack_boundary;
8045 :
8046 : /* Stack at entrance of main is aligned by runtime. We use the
8047 : smallest incoming stack boundary. */
8048 1658409 : if (incoming_stack_boundary > MAIN_STACK_BOUNDARY
8049 140919 : && DECL_NAME (current_function_decl)
8050 140919 : && MAIN_NAME_P (DECL_NAME (current_function_decl))
8051 1660886 : && DECL_FILE_SCOPE_P (current_function_decl))
8052 2477 : incoming_stack_boundary = MAIN_STACK_BOUNDARY;
8053 :
8054 1658409 : return incoming_stack_boundary;
8055 : }
8056 :
8057 : /* Update incoming stack boundary and estimated stack alignment. */
8058 :
8059 : static void
8060 1521340 : ix86_update_stack_boundary (void)
8061 : {
8062 1521340 : ix86_incoming_stack_boundary
8063 1521340 : = ix86_minimum_incoming_stack_boundary (false);
8064 :
8065 : /* x86_64 vararg needs 16byte stack alignment for register save area. */
8066 1521340 : if (TARGET_64BIT
8067 1394685 : && cfun->stdarg
8068 21451 : && crtl->stack_alignment_estimated < 128)
8069 10270 : crtl->stack_alignment_estimated = 128;
8070 :
8071 : /* __tls_get_addr needs to be called with 16-byte aligned stack. */
8072 1521340 : if (ix86_tls_descriptor_calls_expanded_in_cfun
8073 1085 : && crtl->preferred_stack_boundary < 128)
8074 750 : crtl->preferred_stack_boundary = 128;
8075 :
8076 : /* For 32-bit MS ABI, both the incoming and preferred stack boundaries
8077 : are 32 bits, but if force_align_arg_pointer is specified, it should
8078 : prefer 128 bits for a backward-compatibility reason, which is also
8079 : what the doc suggests. */
8080 1521340 : if (lookup_attribute ("force_align_arg_pointer",
8081 1521340 : TYPE_ATTRIBUTES (TREE_TYPE (current_function_decl)))
8082 1521340 : && crtl->preferred_stack_boundary < 128)
8083 4 : crtl->preferred_stack_boundary = 128;
8084 1521340 : }
8085 :
8086 : /* Handle the TARGET_GET_DRAP_RTX hook. Return NULL if no DRAP is
8087 : needed or an rtx for DRAP otherwise. */
8088 :
8089 : static rtx
8090 1628529 : ix86_get_drap_rtx (void)
8091 : {
8092 : /* We must use DRAP if there are outgoing arguments on stack or
8093 : the stack pointer register is clobbered by asm statement and
8094 : ACCUMULATE_OUTGOING_ARGS is false. */
8095 1628529 : if (ix86_force_drap
8096 1628529 : || ((cfun->machine->outgoing_args_on_stack
8097 1293543 : || crtl->sp_is_clobbered_by_asm)
8098 333041 : && !ACCUMULATE_OUTGOING_ARGS))
8099 312831 : crtl->need_drap = true;
8100 :
8101 1628529 : if (stack_realign_drap)
8102 : {
8103 : /* Assign DRAP to vDRAP and returns vDRAP */
8104 7243 : unsigned int regno = find_drap_reg ();
8105 7243 : rtx drap_vreg;
8106 7243 : rtx arg_ptr;
8107 7243 : rtx_insn *seq, *insn;
8108 :
8109 7528 : arg_ptr = gen_rtx_REG (Pmode, regno);
8110 7243 : crtl->drap_reg = arg_ptr;
8111 :
8112 7243 : start_sequence ();
8113 7243 : drap_vreg = copy_to_reg (arg_ptr);
8114 7243 : seq = end_sequence ();
8115 :
8116 7243 : insn = emit_insn_before (seq, NEXT_INSN (entry_of_function ()));
8117 7243 : if (!optimize)
8118 : {
8119 1897 : add_reg_note (insn, REG_CFA_SET_VDRAP, drap_vreg);
8120 1897 : RTX_FRAME_RELATED_P (insn) = 1;
8121 : }
8122 : return drap_vreg;
8123 : }
8124 : else
8125 : return NULL;
8126 : }
8127 :
8128 : /* Handle the TARGET_INTERNAL_ARG_POINTER hook. */
8129 :
8130 : static rtx
8131 1521342 : ix86_internal_arg_pointer (void)
8132 : {
8133 1521342 : return virtual_incoming_args_rtx;
8134 : }
8135 :
8136 : struct scratch_reg {
8137 : rtx reg;
8138 : bool saved;
8139 : };
8140 :
8141 : /* Return a short-lived scratch register for use on function entry.
8142 : In 32-bit mode, it is valid only after the registers are saved
8143 : in the prologue. This register must be released by means of
8144 : release_scratch_register_on_entry once it is dead. */
8145 :
8146 : static void
8147 25 : get_scratch_register_on_entry (struct scratch_reg *sr)
8148 : {
8149 25 : int regno;
8150 :
8151 25 : sr->saved = false;
8152 :
8153 25 : if (TARGET_64BIT)
8154 : {
8155 : /* We always use R11 in 64-bit mode. */
8156 : regno = R11_REG;
8157 : }
8158 : else
8159 : {
8160 0 : tree decl = current_function_decl, fntype = TREE_TYPE (decl);
8161 0 : bool fastcall_p
8162 0 : = lookup_attribute ("fastcall", TYPE_ATTRIBUTES (fntype)) != NULL_TREE;
8163 0 : bool thiscall_p
8164 0 : = lookup_attribute ("thiscall", TYPE_ATTRIBUTES (fntype)) != NULL_TREE;
8165 0 : bool static_chain_p = DECL_STATIC_CHAIN (decl);
8166 0 : int regparm = ix86_function_regparm (fntype, decl);
8167 0 : int drap_regno
8168 0 : = crtl->drap_reg ? REGNO (crtl->drap_reg) : INVALID_REGNUM;
8169 :
8170 : /* 'fastcall' sets regparm to 2, uses ecx/edx for arguments and eax
8171 : for the static chain register. */
8172 0 : if ((regparm < 1 || (fastcall_p && !static_chain_p))
8173 0 : && drap_regno != AX_REG)
8174 : regno = AX_REG;
8175 : /* 'thiscall' sets regparm to 1, uses ecx for arguments and edx
8176 : for the static chain register. */
8177 0 : else if (thiscall_p && !static_chain_p && drap_regno != AX_REG)
8178 : regno = AX_REG;
8179 0 : else if (regparm < 2 && !thiscall_p && drap_regno != DX_REG)
8180 : regno = DX_REG;
8181 : /* ecx is the static chain register. */
8182 0 : else if (regparm < 3 && !fastcall_p && !thiscall_p
8183 0 : && !static_chain_p
8184 0 : && drap_regno != CX_REG)
8185 : regno = CX_REG;
8186 0 : else if (ix86_save_reg (BX_REG, true, false))
8187 : regno = BX_REG;
8188 : /* esi is the static chain register. */
8189 0 : else if (!(regparm == 3 && static_chain_p)
8190 0 : && ix86_save_reg (SI_REG, true, false))
8191 : regno = SI_REG;
8192 0 : else if (ix86_save_reg (DI_REG, true, false))
8193 : regno = DI_REG;
8194 : else
8195 : {
8196 0 : regno = (drap_regno == AX_REG ? DX_REG : AX_REG);
8197 0 : sr->saved = true;
8198 : }
8199 : }
8200 :
8201 25 : sr->reg = gen_rtx_REG (Pmode, regno);
8202 25 : if (sr->saved)
8203 : {
8204 0 : rtx_insn *insn = emit_insn (gen_push (sr->reg));
8205 0 : RTX_FRAME_RELATED_P (insn) = 1;
8206 : }
8207 25 : }
8208 :
8209 : /* Release a scratch register obtained from the preceding function.
8210 :
8211 : If RELEASE_VIA_POP is true, we just pop the register off the stack
8212 : to release it. This is what non-Linux systems use with -fstack-check.
8213 :
8214 : Otherwise we use OFFSET to locate the saved register and the
8215 : allocated stack space becomes part of the local frame and is
8216 : deallocated by the epilogue. */
8217 :
8218 : static void
8219 25 : release_scratch_register_on_entry (struct scratch_reg *sr, HOST_WIDE_INT offset,
8220 : bool release_via_pop)
8221 : {
8222 25 : if (sr->saved)
8223 : {
8224 0 : if (release_via_pop)
8225 : {
8226 0 : struct machine_function *m = cfun->machine;
8227 0 : rtx x, insn = emit_insn (gen_pop (sr->reg));
8228 :
8229 : /* The RX FRAME_RELATED_P mechanism doesn't know about pop. */
8230 0 : RTX_FRAME_RELATED_P (insn) = 1;
8231 0 : x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
8232 0 : x = gen_rtx_SET (stack_pointer_rtx, x);
8233 0 : add_reg_note (insn, REG_FRAME_RELATED_EXPR, x);
8234 0 : m->fs.sp_offset -= UNITS_PER_WORD;
8235 : }
8236 : else
8237 : {
8238 0 : rtx x = plus_constant (Pmode, stack_pointer_rtx, offset);
8239 0 : x = gen_rtx_SET (sr->reg, gen_rtx_MEM (word_mode, x));
8240 0 : emit_insn (x);
8241 : }
8242 : }
8243 25 : }
8244 :
8245 : /* Emit code to adjust the stack pointer by SIZE bytes while probing it.
8246 :
8247 : If INT_REGISTERS_SAVED is true, then integer registers have already been
8248 : pushed on the stack.
8249 :
8250 : If PROTECTION AREA is true, then probe PROBE_INTERVAL plus a small dope
8251 : beyond SIZE bytes.
8252 :
8253 : This assumes no knowledge of the current probing state, i.e. it is never
8254 : allowed to allocate more than PROBE_INTERVAL bytes of stack space without
8255 : a suitable probe. */
8256 :
8257 : static void
8258 126 : ix86_adjust_stack_and_probe (HOST_WIDE_INT size,
8259 : const bool int_registers_saved,
8260 : const bool protection_area)
8261 : {
8262 126 : struct machine_function *m = cfun->machine;
8263 :
8264 : /* If this function does not statically allocate stack space, then
8265 : no probes are needed. */
8266 126 : if (!size)
8267 : {
8268 : /* However, the allocation of space via pushes for register
8269 : saves could be viewed as allocating space, but without the
8270 : need to probe. */
8271 43 : if (m->frame.nregs || m->frame.nsseregs || frame_pointer_needed)
8272 23 : dump_stack_clash_frame_info (NO_PROBE_SMALL_FRAME, true);
8273 : else
8274 20 : dump_stack_clash_frame_info (NO_PROBE_NO_FRAME, false);
8275 : return;
8276 : }
8277 :
8278 : /* If we are a noreturn function, then we have to consider the
8279 : possibility that we're called via a jump rather than a call.
8280 :
8281 : Thus we don't have the implicit probe generated by saving the
8282 : return address into the stack at the call. Thus, the stack
8283 : pointer could be anywhere in the guard page. The safe thing
8284 : to do is emit a probe now.
8285 :
8286 : The probe can be avoided if we have already emitted any callee
8287 : register saves into the stack or have a frame pointer (which will
8288 : have been saved as well). Those saves will function as implicit
8289 : probes.
8290 :
8291 : ?!? This should be revamped to work like aarch64 and s390 where
8292 : we track the offset from the most recent probe. Normally that
8293 : offset would be zero. For a noreturn function we would reset
8294 : it to PROBE_INTERVAL - (STACK_BOUNDARY / BITS_PER_UNIT). Then
8295 : we just probe when we cross PROBE_INTERVAL. */
8296 83 : if (TREE_THIS_VOLATILE (cfun->decl)
8297 15 : && !(m->frame.nregs || m->frame.nsseregs || frame_pointer_needed))
8298 : {
8299 : /* We can safely use any register here since we're just going to push
8300 : its value and immediately pop it back. But we do try and avoid
8301 : argument passing registers so as not to introduce dependencies in
8302 : the pipeline. For 32 bit we use %esi and for 64 bit we use %rax. */
8303 15 : rtx dummy_reg = gen_rtx_REG (word_mode, TARGET_64BIT ? AX_REG : SI_REG);
8304 15 : rtx_insn *insn_push = emit_insn (gen_push (dummy_reg));
8305 15 : rtx_insn *insn_pop = emit_insn (gen_pop (dummy_reg));
8306 15 : m->fs.sp_offset -= UNITS_PER_WORD;
8307 15 : if (m->fs.cfa_reg == stack_pointer_rtx)
8308 : {
8309 15 : m->fs.cfa_offset -= UNITS_PER_WORD;
8310 15 : rtx x = plus_constant (Pmode, stack_pointer_rtx, -UNITS_PER_WORD);
8311 15 : x = gen_rtx_SET (stack_pointer_rtx, x);
8312 15 : add_reg_note (insn_push, REG_CFA_ADJUST_CFA, x);
8313 15 : RTX_FRAME_RELATED_P (insn_push) = 1;
8314 15 : x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
8315 15 : x = gen_rtx_SET (stack_pointer_rtx, x);
8316 15 : add_reg_note (insn_pop, REG_CFA_ADJUST_CFA, x);
8317 15 : RTX_FRAME_RELATED_P (insn_pop) = 1;
8318 : }
8319 15 : emit_insn (gen_blockage ());
8320 : }
8321 :
8322 83 : const HOST_WIDE_INT probe_interval = get_probe_interval ();
8323 83 : const int dope = 4 * UNITS_PER_WORD;
8324 :
8325 : /* If there is protection area, take it into account in the size. */
8326 83 : if (protection_area)
8327 24 : size += probe_interval + dope;
8328 :
8329 : /* If we allocate less than the size of the guard statically,
8330 : then no probing is necessary, but we do need to allocate
8331 : the stack. */
8332 59 : else if (size < (1 << param_stack_clash_protection_guard_size))
8333 : {
8334 38 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
8335 : GEN_INT (-size), -1,
8336 38 : m->fs.cfa_reg == stack_pointer_rtx);
8337 38 : dump_stack_clash_frame_info (NO_PROBE_SMALL_FRAME, true);
8338 38 : return;
8339 : }
8340 :
8341 : /* We're allocating a large enough stack frame that we need to
8342 : emit probes. Either emit them inline or in a loop depending
8343 : on the size. */
8344 45 : if (size <= 4 * probe_interval)
8345 : {
8346 : HOST_WIDE_INT i;
8347 47 : for (i = probe_interval; i <= size; i += probe_interval)
8348 : {
8349 : /* Allocate PROBE_INTERVAL bytes. */
8350 27 : rtx insn
8351 27 : = pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
8352 : GEN_INT (-probe_interval), -1,
8353 27 : m->fs.cfa_reg == stack_pointer_rtx);
8354 27 : add_reg_note (insn, REG_STACK_CHECK, const0_rtx);
8355 :
8356 : /* And probe at *sp. */
8357 27 : emit_stack_probe (stack_pointer_rtx);
8358 27 : emit_insn (gen_blockage ());
8359 : }
8360 :
8361 : /* We need to allocate space for the residual, but we do not need
8362 : to probe the residual... */
8363 20 : HOST_WIDE_INT residual = (i - probe_interval - size);
8364 20 : if (residual)
8365 : {
8366 20 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
8367 : GEN_INT (residual), -1,
8368 20 : m->fs.cfa_reg == stack_pointer_rtx);
8369 :
8370 : /* ...except if there is a protection area to maintain. */
8371 20 : if (protection_area)
8372 11 : emit_stack_probe (stack_pointer_rtx);
8373 : }
8374 :
8375 20 : dump_stack_clash_frame_info (PROBE_INLINE, residual != 0);
8376 : }
8377 : else
8378 : {
8379 : /* We expect the GP registers to be saved when probes are used
8380 : as the probing sequences might need a scratch register and
8381 : the routine to allocate one assumes the integer registers
8382 : have already been saved. */
8383 25 : gcc_assert (int_registers_saved);
8384 :
8385 25 : struct scratch_reg sr;
8386 25 : get_scratch_register_on_entry (&sr);
8387 :
8388 : /* If we needed to save a register, then account for any space
8389 : that was pushed (we are not going to pop the register when
8390 : we do the restore). */
8391 25 : if (sr.saved)
8392 0 : size -= UNITS_PER_WORD;
8393 :
8394 : /* Step 1: round SIZE down to a multiple of the interval. */
8395 25 : HOST_WIDE_INT rounded_size = size & -probe_interval;
8396 :
8397 : /* Step 2: compute final value of the loop counter. Use lea if
8398 : possible. */
8399 25 : rtx addr = plus_constant (Pmode, stack_pointer_rtx, -rounded_size);
8400 25 : rtx insn;
8401 25 : if (address_no_seg_operand (addr, Pmode))
8402 13 : insn = emit_insn (gen_rtx_SET (sr.reg, addr));
8403 : else
8404 : {
8405 12 : emit_move_insn (sr.reg, GEN_INT (-rounded_size));
8406 12 : insn = emit_insn (gen_rtx_SET (sr.reg,
8407 : gen_rtx_PLUS (Pmode, sr.reg,
8408 : stack_pointer_rtx)));
8409 : }
8410 25 : if (m->fs.cfa_reg == stack_pointer_rtx)
8411 : {
8412 22 : add_reg_note (insn, REG_CFA_DEF_CFA,
8413 22 : plus_constant (Pmode, sr.reg,
8414 22 : m->fs.cfa_offset + rounded_size));
8415 22 : RTX_FRAME_RELATED_P (insn) = 1;
8416 : }
8417 :
8418 : /* Step 3: the loop. */
8419 25 : rtx size_rtx = GEN_INT (rounded_size);
8420 25 : insn = emit_insn (gen_adjust_stack_and_probe (Pmode, sr.reg, sr.reg,
8421 : size_rtx));
8422 25 : if (m->fs.cfa_reg == stack_pointer_rtx)
8423 : {
8424 22 : m->fs.cfa_offset += rounded_size;
8425 22 : add_reg_note (insn, REG_CFA_DEF_CFA,
8426 22 : plus_constant (Pmode, stack_pointer_rtx,
8427 22 : m->fs.cfa_offset));
8428 22 : RTX_FRAME_RELATED_P (insn) = 1;
8429 : }
8430 25 : m->fs.sp_offset += rounded_size;
8431 25 : emit_insn (gen_blockage ());
8432 :
8433 : /* Step 4: adjust SP if we cannot assert at compile-time that SIZE
8434 : is equal to ROUNDED_SIZE. */
8435 :
8436 25 : if (size != rounded_size)
8437 : {
8438 25 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
8439 : GEN_INT (rounded_size - size), -1,
8440 25 : m->fs.cfa_reg == stack_pointer_rtx);
8441 :
8442 25 : if (protection_area)
8443 13 : emit_stack_probe (stack_pointer_rtx);
8444 : }
8445 :
8446 25 : dump_stack_clash_frame_info (PROBE_LOOP, size != rounded_size);
8447 :
8448 : /* This does not deallocate the space reserved for the scratch
8449 : register. That will be deallocated in the epilogue. */
8450 25 : release_scratch_register_on_entry (&sr, size, false);
8451 : }
8452 :
8453 : /* Adjust back to account for the protection area. */
8454 45 : if (protection_area)
8455 24 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
8456 24 : GEN_INT (probe_interval + dope), -1,
8457 24 : m->fs.cfa_reg == stack_pointer_rtx);
8458 :
8459 : /* Make sure nothing is scheduled before we are done. */
8460 45 : emit_insn (gen_blockage ());
8461 : }
8462 :
8463 : /* Adjust the stack pointer up to REG while probing it. */
8464 :
8465 : const char *
8466 25 : output_adjust_stack_and_probe (rtx reg)
8467 : {
8468 25 : static int labelno = 0;
8469 25 : char loop_lab[32];
8470 25 : rtx xops[2];
8471 :
8472 25 : ASM_GENERATE_INTERNAL_LABEL (loop_lab, "LPSRL", labelno++);
8473 :
8474 : /* Loop. */
8475 25 : ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, loop_lab);
8476 :
8477 : /* SP = SP + PROBE_INTERVAL. */
8478 25 : xops[0] = stack_pointer_rtx;
8479 37 : xops[1] = GEN_INT (get_probe_interval ());
8480 25 : output_asm_insn ("sub%z0\t{%1, %0|%0, %1}", xops);
8481 :
8482 : /* Probe at SP. */
8483 25 : xops[1] = const0_rtx;
8484 25 : output_asm_insn ("or{b}\t{%1, (%0)|BYTE PTR [%0], %1}", xops);
8485 :
8486 : /* Test if SP == LAST_ADDR. */
8487 25 : xops[0] = stack_pointer_rtx;
8488 25 : xops[1] = reg;
8489 25 : output_asm_insn ("cmp%z0\t{%1, %0|%0, %1}", xops);
8490 :
8491 : /* Branch. */
8492 25 : fputs ("\tjne\t", asm_out_file);
8493 25 : assemble_name_raw (asm_out_file, loop_lab);
8494 25 : fputc ('\n', asm_out_file);
8495 :
8496 25 : return "";
8497 : }
8498 :
8499 : /* Emit code to probe a range of stack addresses from FIRST to FIRST+SIZE,
8500 : inclusive. These are offsets from the current stack pointer.
8501 :
8502 : INT_REGISTERS_SAVED is true if integer registers have already been
8503 : pushed on the stack. */
8504 :
8505 : static void
8506 0 : ix86_emit_probe_stack_range (HOST_WIDE_INT first, HOST_WIDE_INT size,
8507 : const bool int_registers_saved)
8508 : {
8509 0 : const HOST_WIDE_INT probe_interval = get_probe_interval ();
8510 :
8511 : /* See if we have a constant small number of probes to generate. If so,
8512 : that's the easy case. The run-time loop is made up of 6 insns in the
8513 : generic case while the compile-time loop is made up of n insns for n #
8514 : of intervals. */
8515 0 : if (size <= 6 * probe_interval)
8516 : {
8517 : HOST_WIDE_INT i;
8518 :
8519 : /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 1 until
8520 : it exceeds SIZE. If only one probe is needed, this will not
8521 : generate any code. Then probe at FIRST + SIZE. */
8522 0 : for (i = probe_interval; i < size; i += probe_interval)
8523 0 : emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx,
8524 0 : -(first + i)));
8525 :
8526 0 : emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx,
8527 0 : -(first + size)));
8528 : }
8529 :
8530 : /* Otherwise, do the same as above, but in a loop. Note that we must be
8531 : extra careful with variables wrapping around because we might be at
8532 : the very top (or the very bottom) of the address space and we have
8533 : to be able to handle this case properly; in particular, we use an
8534 : equality test for the loop condition. */
8535 : else
8536 : {
8537 : /* We expect the GP registers to be saved when probes are used
8538 : as the probing sequences might need a scratch register and
8539 : the routine to allocate one assumes the integer registers
8540 : have already been saved. */
8541 0 : gcc_assert (int_registers_saved);
8542 :
8543 0 : HOST_WIDE_INT rounded_size, last;
8544 0 : struct scratch_reg sr;
8545 :
8546 0 : get_scratch_register_on_entry (&sr);
8547 :
8548 :
8549 : /* Step 1: round SIZE to the previous multiple of the interval. */
8550 :
8551 0 : rounded_size = ROUND_DOWN (size, probe_interval);
8552 :
8553 :
8554 : /* Step 2: compute initial and final value of the loop counter. */
8555 :
8556 : /* TEST_OFFSET = FIRST. */
8557 0 : emit_move_insn (sr.reg, GEN_INT (-first));
8558 :
8559 : /* LAST_OFFSET = FIRST + ROUNDED_SIZE. */
8560 0 : last = first + rounded_size;
8561 :
8562 :
8563 : /* Step 3: the loop
8564 :
8565 : do
8566 : {
8567 : TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
8568 : probe at TEST_ADDR
8569 : }
8570 : while (TEST_ADDR != LAST_ADDR)
8571 :
8572 : probes at FIRST + N * PROBE_INTERVAL for values of N from 1
8573 : until it is equal to ROUNDED_SIZE. */
8574 :
8575 0 : emit_insn
8576 0 : (gen_probe_stack_range (Pmode, sr.reg, sr.reg, GEN_INT (-last)));
8577 :
8578 :
8579 : /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
8580 : that SIZE is equal to ROUNDED_SIZE. */
8581 :
8582 0 : if (size != rounded_size)
8583 0 : emit_stack_probe (plus_constant (Pmode,
8584 0 : gen_rtx_PLUS (Pmode,
8585 : stack_pointer_rtx,
8586 : sr.reg),
8587 0 : rounded_size - size));
8588 :
8589 0 : release_scratch_register_on_entry (&sr, size, true);
8590 : }
8591 :
8592 : /* Make sure nothing is scheduled before we are done. */
8593 0 : emit_insn (gen_blockage ());
8594 0 : }
8595 :
8596 : /* Probe a range of stack addresses from REG to END, inclusive. These are
8597 : offsets from the current stack pointer. */
8598 :
8599 : const char *
8600 0 : output_probe_stack_range (rtx reg, rtx end)
8601 : {
8602 0 : static int labelno = 0;
8603 0 : char loop_lab[32];
8604 0 : rtx xops[3];
8605 :
8606 0 : ASM_GENERATE_INTERNAL_LABEL (loop_lab, "LPSRL", labelno++);
8607 :
8608 : /* Loop. */
8609 0 : ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, loop_lab);
8610 :
8611 : /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL. */
8612 0 : xops[0] = reg;
8613 0 : xops[1] = GEN_INT (get_probe_interval ());
8614 0 : output_asm_insn ("sub%z0\t{%1, %0|%0, %1}", xops);
8615 :
8616 : /* Probe at TEST_ADDR. */
8617 0 : xops[0] = stack_pointer_rtx;
8618 0 : xops[1] = reg;
8619 0 : xops[2] = const0_rtx;
8620 0 : output_asm_insn ("or{b}\t{%2, (%0,%1)|BYTE PTR [%0+%1], %2}", xops);
8621 :
8622 : /* Test if TEST_ADDR == LAST_ADDR. */
8623 0 : xops[0] = reg;
8624 0 : xops[1] = end;
8625 0 : output_asm_insn ("cmp%z0\t{%1, %0|%0, %1}", xops);
8626 :
8627 : /* Branch. */
8628 0 : fputs ("\tjne\t", asm_out_file);
8629 0 : assemble_name_raw (asm_out_file, loop_lab);
8630 0 : fputc ('\n', asm_out_file);
8631 :
8632 0 : return "";
8633 : }
8634 :
8635 : /* Data passed to ix86_update_stack_alignment. */
8636 : struct stack_access_data
8637 : {
8638 : /* The stack access register. */
8639 : const_rtx reg;
8640 : /* Pointer to stack alignment. */
8641 : unsigned int *stack_alignment;
8642 : };
8643 :
8644 : /* Return true if OP is a stack argument set up by the caller. */
8645 :
8646 : static bool
8647 61842 : ix86_argument_passed_on_stack_p (const_rtx op)
8648 : {
8649 61842 : tree mem_expr = MEM_EXPR (op);
8650 61842 : if (!mem_expr)
8651 : return false;
8652 :
8653 59866 : tree var = get_base_address (mem_expr);
8654 59866 : if (TREE_CODE (var) != PARM_DECL)
8655 : return false;
8656 :
8657 : /* For PARM_DECL, DECL_INCOMING_RTL holds an RTL for the stack slot
8658 : or register where the data was actually passed. Return true if
8659 : OP is passed in memory. */
8660 36706 : return DECL_INCOMING_RTL (var) && MEM_P (DECL_INCOMING_RTL (var));
8661 : }
8662 :
8663 : /* Update the maximum stack slot alignment from memory alignment in PAT. */
8664 :
8665 : static void
8666 98885 : ix86_update_stack_alignment (rtx, const_rtx pat, void *data)
8667 : {
8668 : /* This insn may reference stack slot. Update the maximum stack slot
8669 : alignment if the memory is referenced by the stack access register. */
8670 98885 : stack_access_data *p = (stack_access_data *) data;
8671 :
8672 98885 : subrtx_iterator::array_type array;
8673 370777 : FOR_EACH_SUBRTX (iter, array, pat, ALL)
8674 : {
8675 324158 : auto op = *iter;
8676 324158 : if (MEM_P (op))
8677 : {
8678 : /* NB: Ignore arguments passed on stack since caller is
8679 : responsible to align the outgoing stack for arguments
8680 : passed on stack. */
8681 94933 : if (reg_mentioned_p (p->reg, XEXP (op, 0))
8682 94933 : && !ix86_argument_passed_on_stack_p (op))
8683 : {
8684 52266 : unsigned int alignment = MEM_ALIGN (op);
8685 :
8686 52266 : if (alignment > *p->stack_alignment)
8687 52161 : *p->stack_alignment = alignment;
8688 : break;
8689 : }
8690 : else
8691 42667 : iter.skip_subrtxes ();
8692 : }
8693 : }
8694 98885 : }
8695 :
8696 : /* Helper function for ix86_find_all_reg_uses. */
8697 :
8698 : static void
8699 46600467 : ix86_find_all_reg_uses_1 (HARD_REG_SET ®set,
8700 : rtx set, unsigned int regno,
8701 : auto_bitmap &worklist)
8702 : {
8703 46600467 : rtx dest = SET_DEST (set);
8704 :
8705 46600467 : if (!REG_P (dest))
8706 42182546 : return;
8707 :
8708 : /* Reject non-Pmode modes. */
8709 35333135 : if (GET_MODE (dest) != Pmode)
8710 : return;
8711 :
8712 18918307 : unsigned int dst_regno = REGNO (dest);
8713 :
8714 18918307 : if (TEST_HARD_REG_BIT (regset, dst_regno))
8715 : return;
8716 :
8717 4417921 : const_rtx src = SET_SRC (set);
8718 :
8719 4417921 : subrtx_iterator::array_type array;
8720 8820888 : FOR_EACH_SUBRTX (iter, array, src, ALL)
8721 : {
8722 5725045 : auto op = *iter;
8723 :
8724 5725045 : if (MEM_P (op))
8725 3072803 : iter.skip_subrtxes ();
8726 :
8727 5725045 : if (REG_P (op) && REGNO (op) == regno)
8728 : {
8729 : /* Add this register to register set. */
8730 1489471 : add_to_hard_reg_set (®set, Pmode, dst_regno);
8731 1322078 : bitmap_set_bit (worklist, dst_regno);
8732 1322078 : break;
8733 : }
8734 : }
8735 4417921 : }
8736 :
8737 : /* Find all registers defined with register REGNO. */
8738 :
8739 : static void
8740 2367068 : ix86_find_all_reg_uses (HARD_REG_SET ®set,
8741 : unsigned int regno, auto_bitmap &worklist)
8742 : {
8743 2367068 : for (df_ref ref = DF_REG_USE_CHAIN (regno);
8744 84658965 : ref != NULL;
8745 82291897 : ref = DF_REF_NEXT_REG (ref))
8746 : {
8747 82291897 : if (DF_REF_IS_ARTIFICIAL (ref))
8748 17036490 : continue;
8749 :
8750 65255407 : rtx_insn *insn = DF_REF_INSN (ref);
8751 :
8752 65255407 : if (!NONJUMP_INSN_P (insn))
8753 19389950 : continue;
8754 :
8755 45865457 : unsigned int ref_regno = DF_REF_REGNO (ref);
8756 :
8757 45865457 : rtx set = single_set (insn);
8758 45865457 : if (set)
8759 : {
8760 45027643 : ix86_find_all_reg_uses_1 (regset, set,
8761 : ref_regno, worklist);
8762 45027643 : continue;
8763 : }
8764 :
8765 837814 : rtx pat = PATTERN (insn);
8766 837814 : if (GET_CODE (pat) != PARALLEL)
8767 126815 : continue;
8768 :
8769 2764234 : for (int i = 0; i < XVECLEN (pat, 0); i++)
8770 : {
8771 2053235 : rtx exp = XVECEXP (pat, 0, i);
8772 :
8773 2053235 : if (GET_CODE (exp) == SET)
8774 1572824 : ix86_find_all_reg_uses_1 (regset, exp,
8775 : ref_regno, worklist);
8776 : }
8777 : }
8778 2367068 : }
8779 :
8780 : /* Return true if the hard register REGNO used for a stack access is
8781 : defined in a basic block that dominates the block where it is used. */
8782 :
8783 : static bool
8784 41890 : ix86_access_stack_p (unsigned int regno, basic_block bb,
8785 : HARD_REG_SET &set_up_by_prologue,
8786 : HARD_REG_SET &prologue_used,
8787 : auto_bitmap reg_dominate_bbs_known[],
8788 : auto_bitmap reg_dominate_bbs[])
8789 : {
8790 41890 : if (bitmap_bit_p (reg_dominate_bbs_known[regno], bb->index))
8791 12135 : return bitmap_bit_p (reg_dominate_bbs[regno], bb->index);
8792 :
8793 29755 : bitmap_set_bit (reg_dominate_bbs_known[regno], bb->index);
8794 :
8795 : /* Get all BBs which set REGNO and dominate the current BB from all
8796 : DEFs of REGNO. */
8797 29755 : for (df_ref def = DF_REG_DEF_CHAIN (regno);
8798 1558943 : def;
8799 1529188 : def = DF_REF_NEXT_REG (def))
8800 1557474 : if (!DF_REF_IS_ARTIFICIAL (def)
8801 1555762 : && !DF_REF_FLAGS_IS_SET (def, DF_REF_MAY_CLOBBER)
8802 1530120 : && !DF_REF_FLAGS_IS_SET (def, DF_REF_MUST_CLOBBER))
8803 : {
8804 1528305 : basic_block set_bb = DF_REF_BB (def);
8805 1528305 : if (dominated_by_p (CDI_DOMINATORS, bb, set_bb))
8806 : {
8807 87830 : rtx_insn *insn = DF_REF_INSN (def);
8808 : /* Return true if INSN requires stack. */
8809 87830 : if (requires_stack_frame_p (insn, prologue_used,
8810 : set_up_by_prologue))
8811 : {
8812 28286 : bitmap_set_bit (reg_dominate_bbs[regno], bb->index);
8813 28286 : return true;
8814 : }
8815 : }
8816 : }
8817 :
8818 : /* When we get here, REGNO used in the current BB doesn't access
8819 : stack. */
8820 : return false;
8821 : }
8822 :
8823 : /* Return true if OP isn't a memory operand with SYMBOLIC_CONST and
8824 : needs alignment > ALIGNMENT. */
8825 :
8826 : static bool
8827 28284903 : ix86_need_alignment_p_2 (const_rtx op, unsigned int alignment)
8828 : {
8829 28284903 : bool need_alignment = MEM_ALIGN (op) > alignment;
8830 28284903 : tree mem_expr = MEM_EXPR (op);
8831 28284903 : if (!mem_expr)
8832 : return need_alignment;
8833 :
8834 23407796 : tree var = get_base_address (mem_expr);
8835 23407796 : if (!VAR_P (var) || !DECL_RTL_SET_P (var))
8836 : return need_alignment;
8837 :
8838 14787326 : rtx x = DECL_RTL (var);
8839 14787326 : if (!MEM_P (x))
8840 : return need_alignment;
8841 :
8842 14787323 : x = XEXP (x, 0);
8843 14787323 : return !SYMBOLIC_CONST (x) && need_alignment;
8844 : }
8845 :
8846 : /* Return true if SET needs alignment > ALIGNMENT. */
8847 :
8848 : static bool
8849 46599651 : ix86_need_alignment_p_1 (rtx set, unsigned int alignment)
8850 : {
8851 46599651 : rtx dest = SET_DEST (set);
8852 :
8853 46599651 : if (MEM_P (dest))
8854 17490262 : return ix86_need_alignment_p_2 (dest, alignment);
8855 :
8856 29109389 : const_rtx src = SET_SRC (set);
8857 :
8858 29109389 : subrtx_iterator::array_type array;
8859 84507922 : FOR_EACH_SUBRTX (iter, array, src, ALL)
8860 : {
8861 66193174 : auto op = *iter;
8862 :
8863 66193174 : if (MEM_P (op))
8864 10794641 : return ix86_need_alignment_p_2 (op, alignment);
8865 : }
8866 :
8867 18314748 : return false;
8868 29109389 : }
8869 :
8870 : /* Return true if INSN needs alignment > ALIGNMENT. */
8871 :
8872 : static bool
8873 45865457 : ix86_need_alignment_p (rtx_insn *insn, unsigned int alignment)
8874 : {
8875 45865457 : rtx set = single_set (insn);
8876 45865457 : if (set)
8877 45027643 : return ix86_need_alignment_p_1 (set, alignment);
8878 :
8879 837814 : rtx pat = PATTERN (insn);
8880 837814 : if (GET_CODE (pat) != PARALLEL)
8881 : return false;
8882 :
8883 2762151 : for (int i = 0; i < XVECLEN (pat, 0); i++)
8884 : {
8885 2052103 : rtx exp = XVECEXP (pat, 0, i);
8886 :
8887 2052103 : if (GET_CODE (exp) == SET
8888 2052103 : && ix86_need_alignment_p_1 (exp, alignment))
8889 : return true;
8890 : }
8891 :
8892 : return false;
8893 : }
8894 :
8895 : /* Set stack_frame_required to false if stack frame isn't required.
8896 : Update STACK_ALIGNMENT to the largest alignment, in bits, of stack
8897 : slot used if stack frame is required and CHECK_STACK_SLOT is true. */
8898 :
8899 : static void
8900 1520486 : ix86_find_max_used_stack_alignment (unsigned int &stack_alignment,
8901 : bool check_stack_slot)
8902 : {
8903 1520486 : HARD_REG_SET set_up_by_prologue, prologue_used;
8904 1520486 : basic_block bb;
8905 :
8906 6081944 : CLEAR_HARD_REG_SET (prologue_used);
8907 1520486 : CLEAR_HARD_REG_SET (set_up_by_prologue);
8908 1647248 : add_to_hard_reg_set (&set_up_by_prologue, Pmode, STACK_POINTER_REGNUM);
8909 1520486 : add_to_hard_reg_set (&set_up_by_prologue, Pmode, ARG_POINTER_REGNUM);
8910 1520486 : add_to_hard_reg_set (&set_up_by_prologue, Pmode,
8911 : HARD_FRAME_POINTER_REGNUM);
8912 :
8913 1520486 : bool require_stack_frame = false;
8914 :
8915 16318688 : FOR_EACH_BB_FN (bb, cfun)
8916 : {
8917 14798202 : rtx_insn *insn;
8918 94452168 : FOR_BB_INSNS (bb, insn)
8919 87820504 : if (NONDEBUG_INSN_P (insn)
8920 87820504 : && requires_stack_frame_p (insn, prologue_used,
8921 : set_up_by_prologue))
8922 : {
8923 : require_stack_frame = true;
8924 : break;
8925 : }
8926 : }
8927 :
8928 1520486 : cfun->machine->stack_frame_required = require_stack_frame;
8929 :
8930 : /* Stop if we don't need to check stack slot. */
8931 1520486 : if (!check_stack_slot)
8932 810038 : return;
8933 :
8934 : /* The preferred stack alignment is the minimum stack alignment. */
8935 710448 : if (stack_alignment > crtl->preferred_stack_boundary)
8936 145749 : stack_alignment = crtl->preferred_stack_boundary;
8937 :
8938 710448 : HARD_REG_SET stack_slot_access;
8939 710448 : CLEAR_HARD_REG_SET (stack_slot_access);
8940 :
8941 : /* Stack slot can be accessed by stack pointer, frame pointer or
8942 : registers defined by stack pointer or frame pointer. */
8943 710448 : auto_bitmap worklist;
8944 :
8945 770022 : add_to_hard_reg_set (&stack_slot_access, Pmode, STACK_POINTER_REGNUM);
8946 710448 : bitmap_set_bit (worklist, STACK_POINTER_REGNUM);
8947 :
8948 710448 : if (frame_pointer_needed)
8949 : {
8950 343593 : add_to_hard_reg_set (&stack_slot_access, Pmode,
8951 : HARD_FRAME_POINTER_REGNUM);
8952 334542 : bitmap_set_bit (worklist, HARD_FRAME_POINTER_REGNUM);
8953 : }
8954 :
8955 : /* Registers on HARD_STACK_SLOT_ACCESS always access stack. */
8956 710448 : HARD_REG_SET hard_stack_slot_access = stack_slot_access;
8957 :
8958 710448 : calculate_dominance_info (CDI_DOMINATORS);
8959 :
8960 2367068 : unsigned int regno;
8961 :
8962 2367068 : do
8963 : {
8964 2367068 : regno = bitmap_clear_first_set_bit (worklist);
8965 2367068 : ix86_find_all_reg_uses (stack_slot_access, regno, worklist);
8966 : }
8967 2367068 : while (!bitmap_empty_p (worklist));
8968 :
8969 710448 : hard_reg_set_iterator hrsi;
8970 710448 : stack_access_data data;
8971 :
8972 134274672 : auto_bitmap reg_dominate_bbs_known[FIRST_PSEUDO_REGISTER];
8973 134274672 : auto_bitmap reg_dominate_bbs[FIRST_PSEUDO_REGISTER];
8974 :
8975 710448 : data.stack_alignment = &stack_alignment;
8976 :
8977 3077516 : EXECUTE_IF_SET_IN_HARD_REG_SET (stack_slot_access, 0, regno, hrsi)
8978 : {
8979 2367068 : for (df_ref ref = DF_REG_USE_CHAIN (regno);
8980 84658965 : ref != NULL;
8981 82291897 : ref = DF_REF_NEXT_REG (ref))
8982 : {
8983 82291897 : if (DF_REF_IS_ARTIFICIAL (ref))
8984 17036490 : continue;
8985 :
8986 65255407 : rtx_insn *insn = DF_REF_INSN (ref);
8987 :
8988 65255407 : if (!NONJUMP_INSN_P (insn))
8989 19389950 : continue;
8990 :
8991 : /* Call ix86_access_stack_p only if INSN needs alignment >
8992 : STACK_ALIGNMENT. */
8993 45865457 : if (ix86_need_alignment_p (insn, stack_alignment)
8994 45865457 : && (TEST_HARD_REG_BIT (hard_stack_slot_access, regno)
8995 41890 : || ix86_access_stack_p (regno, BLOCK_FOR_INSN (insn),
8996 : set_up_by_prologue,
8997 : prologue_used,
8998 : reg_dominate_bbs_known,
8999 : reg_dominate_bbs)))
9000 : {
9001 : /* Update stack alignment if REGNO is used for stack
9002 : access. */
9003 92007 : data.reg = DF_REF_REG (ref);
9004 92007 : note_stores (insn, ix86_update_stack_alignment, &data);
9005 : }
9006 : }
9007 : }
9008 :
9009 710448 : free_dominance_info (CDI_DOMINATORS);
9010 135695568 : }
9011 :
9012 : /* Finalize stack_realign_needed and frame_pointer_needed flags, which
9013 : will guide prologue/epilogue to be generated in correct form. */
9014 :
9015 : static void
9016 3520517 : ix86_finalize_stack_frame_flags (void)
9017 : {
9018 : /* Check if stack realign is really needed after reload, and
9019 : stores result in cfun */
9020 3520517 : unsigned int incoming_stack_boundary
9021 3520517 : = (crtl->parm_stack_boundary > ix86_incoming_stack_boundary
9022 3520517 : ? crtl->parm_stack_boundary : ix86_incoming_stack_boundary);
9023 3520517 : unsigned int stack_alignment
9024 1215539 : = (crtl->is_leaf && !ix86_current_function_calls_tls_descriptor
9025 4736056 : ? crtl->max_used_stack_slot_alignment
9026 3520517 : : crtl->stack_alignment_needed);
9027 3520517 : unsigned int stack_realign
9028 3520517 : = (incoming_stack_boundary < stack_alignment);
9029 3520517 : bool recompute_frame_layout_p = false;
9030 :
9031 3520517 : if (crtl->stack_realign_finalized)
9032 : {
9033 : /* After stack_realign_needed is finalized, we can't no longer
9034 : change it. */
9035 2000031 : gcc_assert (crtl->stack_realign_needed == stack_realign);
9036 2000031 : return;
9037 : }
9038 :
9039 : /* It is always safe to compute max_used_stack_alignment. We
9040 : compute it only if 128-bit aligned load/store may be generated
9041 : on misaligned stack slot which will lead to segfault. */
9042 3040972 : bool check_stack_slot
9043 1520486 : = (stack_realign || crtl->max_used_stack_slot_alignment >= 128);
9044 1520486 : ix86_find_max_used_stack_alignment (stack_alignment,
9045 : check_stack_slot);
9046 :
9047 : /* If the only reason for frame_pointer_needed is that we conservatively
9048 : assumed stack realignment might be needed or -fno-omit-frame-pointer
9049 : is used, but in the end nothing that needed the stack alignment had
9050 : been spilled nor stack access, clear frame_pointer_needed and say we
9051 : don't need stack realignment.
9052 :
9053 : When vector register is used for piecewise move and store, we don't
9054 : increase stack_alignment_needed as there is no register spill for
9055 : piecewise move and store. Since stack_realign_needed is set to true
9056 : by checking stack_alignment_estimated which is updated by pseudo
9057 : vector register usage, we also need to check stack_realign_needed to
9058 : eliminate frame pointer. */
9059 1520486 : if ((stack_realign
9060 1453275 : || (!flag_omit_frame_pointer && optimize)
9061 1443025 : || crtl->stack_realign_needed)
9062 78188 : && frame_pointer_needed
9063 78188 : && crtl->is_leaf
9064 54001 : && crtl->sp_is_unchanging
9065 53952 : && !ix86_current_function_calls_tls_descriptor
9066 53952 : && !crtl->accesses_prior_frames
9067 53952 : && !cfun->calls_alloca
9068 53952 : && !crtl->calls_eh_return
9069 : /* See ira_setup_eliminable_regset for the rationale. */
9070 53952 : && !(STACK_CHECK_MOVING_SP
9071 53952 : && flag_stack_check
9072 0 : && flag_exceptions
9073 0 : && cfun->can_throw_non_call_exceptions)
9074 53952 : && !ix86_frame_pointer_required ()
9075 53951 : && ix86_get_frame_size () == 0
9076 35896 : && ix86_nsaved_sseregs () == 0
9077 1556382 : && ix86_varargs_gpr_size + ix86_varargs_fpr_size == 0)
9078 : {
9079 35896 : if (cfun->machine->stack_frame_required)
9080 : {
9081 : /* Stack frame is required. If stack alignment needed is less
9082 : than incoming stack boundary, don't realign stack. */
9083 284 : stack_realign = incoming_stack_boundary < stack_alignment;
9084 284 : if (!stack_realign)
9085 : {
9086 284 : crtl->max_used_stack_slot_alignment
9087 284 : = incoming_stack_boundary;
9088 284 : crtl->stack_alignment_needed
9089 284 : = incoming_stack_boundary;
9090 : /* Also update preferred_stack_boundary for leaf
9091 : functions. */
9092 284 : crtl->preferred_stack_boundary
9093 284 : = incoming_stack_boundary;
9094 : }
9095 : }
9096 : else
9097 : {
9098 : /* If drap has been set, but it actually isn't live at the
9099 : start of the function, there is no reason to set it up. */
9100 35612 : if (crtl->drap_reg)
9101 : {
9102 35 : basic_block bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
9103 70 : if (! REGNO_REG_SET_P (DF_LR_IN (bb),
9104 : REGNO (crtl->drap_reg)))
9105 : {
9106 35 : crtl->drap_reg = NULL_RTX;
9107 35 : crtl->need_drap = false;
9108 : }
9109 : }
9110 : else
9111 35577 : cfun->machine->no_drap_save_restore = true;
9112 :
9113 35612 : frame_pointer_needed = false;
9114 35612 : stack_realign = false;
9115 35612 : crtl->max_used_stack_slot_alignment = incoming_stack_boundary;
9116 35612 : crtl->stack_alignment_needed = incoming_stack_boundary;
9117 35612 : crtl->stack_alignment_estimated = incoming_stack_boundary;
9118 35612 : if (crtl->preferred_stack_boundary > incoming_stack_boundary)
9119 1 : crtl->preferred_stack_boundary = incoming_stack_boundary;
9120 35612 : df_finish_pass (true);
9121 35612 : df_scan_alloc (NULL);
9122 35612 : df_scan_blocks ();
9123 35612 : df_compute_regs_ever_live (true);
9124 35612 : df_analyze ();
9125 :
9126 35612 : if (flag_var_tracking)
9127 : {
9128 : /* Since frame pointer is no longer available, replace it with
9129 : stack pointer - UNITS_PER_WORD in debug insns. */
9130 136 : df_ref ref, next;
9131 136 : for (ref = DF_REG_USE_CHAIN (HARD_FRAME_POINTER_REGNUM);
9132 136 : ref; ref = next)
9133 : {
9134 0 : next = DF_REF_NEXT_REG (ref);
9135 0 : if (!DF_REF_INSN_INFO (ref))
9136 0 : continue;
9137 :
9138 : /* Make sure the next ref is for a different instruction,
9139 : so that we're not affected by the rescan. */
9140 0 : rtx_insn *insn = DF_REF_INSN (ref);
9141 0 : while (next && DF_REF_INSN (next) == insn)
9142 0 : next = DF_REF_NEXT_REG (next);
9143 :
9144 0 : if (DEBUG_INSN_P (insn))
9145 : {
9146 : bool changed = false;
9147 0 : for (; ref != next; ref = DF_REF_NEXT_REG (ref))
9148 : {
9149 0 : rtx *loc = DF_REF_LOC (ref);
9150 0 : if (*loc == hard_frame_pointer_rtx)
9151 : {
9152 0 : *loc = plus_constant (Pmode,
9153 : stack_pointer_rtx,
9154 0 : -UNITS_PER_WORD);
9155 0 : changed = true;
9156 : }
9157 : }
9158 0 : if (changed)
9159 0 : df_insn_rescan (insn);
9160 : }
9161 : }
9162 : }
9163 :
9164 : recompute_frame_layout_p = true;
9165 : }
9166 : }
9167 1484590 : else if (crtl->max_used_stack_slot_alignment >= 128
9168 673556 : && cfun->machine->stack_frame_required)
9169 : {
9170 : /* We don't need to realign stack. max_used_stack_alignment is
9171 : used to decide how stack frame should be aligned. This is
9172 : independent of any psABIs nor 32-bit vs 64-bit. */
9173 626937 : cfun->machine->max_used_stack_alignment
9174 626937 : = stack_alignment / BITS_PER_UNIT;
9175 : }
9176 :
9177 1520486 : if (crtl->stack_realign_needed != stack_realign)
9178 36180 : recompute_frame_layout_p = true;
9179 1520486 : crtl->stack_realign_needed = stack_realign;
9180 1520486 : crtl->stack_realign_finalized = true;
9181 1520486 : if (recompute_frame_layout_p)
9182 36274 : ix86_compute_frame_layout ();
9183 : }
9184 :
9185 : /* Delete SET_GOT right after entry block if it is allocated to reg. */
9186 :
9187 : static void
9188 0 : ix86_elim_entry_set_got (rtx reg)
9189 : {
9190 0 : basic_block bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
9191 0 : rtx_insn *c_insn = BB_HEAD (bb);
9192 0 : if (!NONDEBUG_INSN_P (c_insn))
9193 0 : c_insn = next_nonnote_nondebug_insn (c_insn);
9194 0 : if (c_insn && NONJUMP_INSN_P (c_insn))
9195 : {
9196 0 : rtx pat = PATTERN (c_insn);
9197 0 : if (GET_CODE (pat) == PARALLEL)
9198 : {
9199 0 : rtx set = XVECEXP (pat, 0, 0);
9200 0 : if (GET_CODE (set) == SET
9201 0 : && GET_CODE (SET_SRC (set)) == UNSPEC
9202 0 : && XINT (SET_SRC (set), 1) == UNSPEC_SET_GOT
9203 0 : && REGNO (SET_DEST (set)) == REGNO (reg))
9204 0 : delete_insn (c_insn);
9205 : }
9206 : }
9207 0 : }
9208 :
9209 : static rtx
9210 192918 : gen_frame_set (rtx reg, rtx frame_reg, int offset, bool store)
9211 : {
9212 192918 : rtx addr, mem;
9213 :
9214 192918 : if (offset)
9215 184232 : addr = plus_constant (Pmode, frame_reg, offset);
9216 192918 : mem = gen_frame_mem (GET_MODE (reg), offset ? addr : frame_reg);
9217 192918 : return gen_rtx_SET (store ? mem : reg, store ? reg : mem);
9218 : }
9219 :
9220 : static inline rtx
9221 100209 : gen_frame_load (rtx reg, rtx frame_reg, int offset)
9222 : {
9223 100209 : return gen_frame_set (reg, frame_reg, offset, false);
9224 : }
9225 :
9226 : static inline rtx
9227 92709 : gen_frame_store (rtx reg, rtx frame_reg, int offset)
9228 : {
9229 92709 : return gen_frame_set (reg, frame_reg, offset, true);
9230 : }
9231 :
9232 : static void
9233 7045 : ix86_emit_outlined_ms2sysv_save (const struct ix86_frame &frame)
9234 : {
9235 7045 : struct machine_function *m = cfun->machine;
9236 7045 : const unsigned ncregs = NUM_X86_64_MS_CLOBBERED_REGS
9237 7045 : + m->call_ms2sysv_extra_regs;
9238 7045 : rtvec v = rtvec_alloc (ncregs + 1);
9239 7045 : unsigned int align, i, vi = 0;
9240 7045 : rtx_insn *insn;
9241 7045 : rtx sym, addr;
9242 7045 : rtx rax = gen_rtx_REG (word_mode, AX_REG);
9243 7045 : const class xlogue_layout &xlogue = xlogue_layout::get_instance ();
9244 :
9245 : /* AL should only be live with sysv_abi. */
9246 7045 : gcc_assert (!ix86_eax_live_at_start_p ());
9247 7045 : gcc_assert (m->fs.sp_offset >= frame.sse_reg_save_offset);
9248 :
9249 : /* Setup RAX as the stub's base pointer. We use stack_realign_offset rather
9250 : we've actually realigned the stack or not. */
9251 7045 : align = GET_MODE_ALIGNMENT (V4SFmode);
9252 7045 : addr = choose_baseaddr (frame.stack_realign_offset
9253 7045 : + xlogue.get_stub_ptr_offset (), &align, AX_REG);
9254 7045 : gcc_assert (align >= GET_MODE_ALIGNMENT (V4SFmode));
9255 :
9256 7045 : emit_insn (gen_rtx_SET (rax, addr));
9257 :
9258 : /* Get the stub symbol. */
9259 8327 : sym = xlogue.get_stub_rtx (frame_pointer_needed ? XLOGUE_STUB_SAVE_HFP
9260 : : XLOGUE_STUB_SAVE);
9261 7045 : RTVEC_ELT (v, vi++) = gen_rtx_USE (VOIDmode, sym);
9262 :
9263 99754 : for (i = 0; i < ncregs; ++i)
9264 : {
9265 92709 : const xlogue_layout::reginfo &r = xlogue.get_reginfo (i);
9266 92709 : rtx reg = gen_rtx_REG ((SSE_REGNO_P (r.regno) ? V4SFmode : word_mode),
9267 92709 : r.regno);
9268 92709 : RTVEC_ELT (v, vi++) = gen_frame_store (reg, rax, -r.offset);
9269 : }
9270 :
9271 7045 : gcc_assert (vi == (unsigned)GET_NUM_ELEM (v));
9272 :
9273 7045 : insn = emit_insn (gen_rtx_PARALLEL (VOIDmode, v));
9274 7045 : RTX_FRAME_RELATED_P (insn) = true;
9275 7045 : }
9276 :
9277 : /* Generate and return an insn body to AND X with Y. */
9278 :
9279 : static rtx_insn *
9280 31758 : gen_and2_insn (rtx x, rtx y)
9281 : {
9282 31758 : enum insn_code icode = optab_handler (and_optab, GET_MODE (x));
9283 :
9284 31758 : gcc_assert (insn_operand_matches (icode, 0, x));
9285 31758 : gcc_assert (insn_operand_matches (icode, 1, x));
9286 31758 : gcc_assert (insn_operand_matches (icode, 2, y));
9287 :
9288 31758 : return GEN_FCN (icode) (x, x, y);
9289 : }
9290 :
9291 : /* Expand the prologue into a bunch of separate insns. */
9292 :
9293 : void
9294 1566395 : ix86_expand_prologue (void)
9295 : {
9296 1566395 : struct machine_function *m = cfun->machine;
9297 1566395 : rtx insn, t;
9298 1566395 : HOST_WIDE_INT allocate;
9299 1566395 : bool int_registers_saved;
9300 1566395 : bool sse_registers_saved;
9301 1566395 : bool save_stub_call_needed;
9302 1566395 : rtx static_chain = NULL_RTX;
9303 :
9304 1566395 : ix86_last_zero_store_uid = 0;
9305 1566395 : if (ix86_function_naked (current_function_decl))
9306 : {
9307 76 : if (flag_stack_usage_info)
9308 0 : current_function_static_stack_size = 0;
9309 : return;
9310 : }
9311 :
9312 1566319 : ix86_finalize_stack_frame_flags ();
9313 :
9314 : /* DRAP should not coexist with stack_realign_fp */
9315 1566319 : gcc_assert (!(crtl->drap_reg && stack_realign_fp));
9316 :
9317 1566319 : memset (&m->fs, 0, sizeof (m->fs));
9318 :
9319 : /* Initialize CFA state for before the prologue. */
9320 1566319 : m->fs.cfa_reg = stack_pointer_rtx;
9321 1566319 : m->fs.cfa_offset = INCOMING_FRAME_SP_OFFSET;
9322 :
9323 : /* Track SP offset to the CFA. We continue tracking this after we've
9324 : swapped the CFA register away from SP. In the case of re-alignment
9325 : this is fudged; we're interested to offsets within the local frame. */
9326 1566319 : m->fs.sp_offset = INCOMING_FRAME_SP_OFFSET;
9327 1566319 : m->fs.sp_valid = true;
9328 1566319 : m->fs.sp_realigned = false;
9329 :
9330 1566319 : const struct ix86_frame &frame = cfun->machine->frame;
9331 :
9332 1566319 : if (!TARGET_64BIT && ix86_function_ms_hook_prologue (current_function_decl))
9333 : {
9334 : /* We should have already generated an error for any use of
9335 : ms_hook on a nested function. */
9336 0 : gcc_checking_assert (!ix86_static_chain_on_stack);
9337 :
9338 : /* Check if profiling is active and we shall use profiling before
9339 : prologue variant. If so sorry. */
9340 0 : if (crtl->profile && flag_fentry != 0)
9341 0 : sorry ("%<ms_hook_prologue%> attribute is not compatible "
9342 : "with %<-mfentry%> for 32-bit");
9343 :
9344 : /* In ix86_asm_output_function_label we emitted:
9345 : 8b ff movl.s %edi,%edi
9346 : 55 push %ebp
9347 : 8b ec movl.s %esp,%ebp
9348 :
9349 : This matches the hookable function prologue in Win32 API
9350 : functions in Microsoft Windows XP Service Pack 2 and newer.
9351 : Wine uses this to enable Windows apps to hook the Win32 API
9352 : functions provided by Wine.
9353 :
9354 : What that means is that we've already set up the frame pointer. */
9355 :
9356 0 : if (frame_pointer_needed
9357 0 : && !(crtl->drap_reg && crtl->stack_realign_needed))
9358 : {
9359 0 : rtx push, mov;
9360 :
9361 : /* We've decided to use the frame pointer already set up.
9362 : Describe this to the unwinder by pretending that both
9363 : push and mov insns happen right here.
9364 :
9365 : Putting the unwind info here at the end of the ms_hook
9366 : is done so that we can make absolutely certain we get
9367 : the required byte sequence at the start of the function,
9368 : rather than relying on an assembler that can produce
9369 : the exact encoding required.
9370 :
9371 : However it does mean (in the unpatched case) that we have
9372 : a 1 insn window where the asynchronous unwind info is
9373 : incorrect. However, if we placed the unwind info at
9374 : its correct location we would have incorrect unwind info
9375 : in the patched case. Which is probably all moot since
9376 : I don't expect Wine generates dwarf2 unwind info for the
9377 : system libraries that use this feature. */
9378 :
9379 0 : insn = emit_insn (gen_blockage ());
9380 :
9381 0 : push = gen_push (hard_frame_pointer_rtx);
9382 0 : mov = gen_rtx_SET (hard_frame_pointer_rtx,
9383 : stack_pointer_rtx);
9384 0 : RTX_FRAME_RELATED_P (push) = 1;
9385 0 : RTX_FRAME_RELATED_P (mov) = 1;
9386 :
9387 0 : RTX_FRAME_RELATED_P (insn) = 1;
9388 0 : add_reg_note (insn, REG_FRAME_RELATED_EXPR,
9389 : gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, push, mov)));
9390 :
9391 : /* Note that gen_push incremented m->fs.cfa_offset, even
9392 : though we didn't emit the push insn here. */
9393 0 : m->fs.cfa_reg = hard_frame_pointer_rtx;
9394 0 : m->fs.fp_offset = m->fs.cfa_offset;
9395 0 : m->fs.fp_valid = true;
9396 0 : }
9397 : else
9398 : {
9399 : /* The frame pointer is not needed so pop %ebp again.
9400 : This leaves us with a pristine state. */
9401 0 : emit_insn (gen_pop (hard_frame_pointer_rtx));
9402 : }
9403 : }
9404 :
9405 : /* The first insn of a function that accepts its static chain on the
9406 : stack is to push the register that would be filled in by a direct
9407 : call. This insn will be skipped by the trampoline. */
9408 1566319 : else if (ix86_static_chain_on_stack)
9409 : {
9410 0 : static_chain = ix86_static_chain (cfun->decl, false);
9411 0 : insn = emit_insn (gen_push (static_chain));
9412 0 : emit_insn (gen_blockage ());
9413 :
9414 : /* We don't want to interpret this push insn as a register save,
9415 : only as a stack adjustment. The real copy of the register as
9416 : a save will be done later, if needed. */
9417 0 : t = plus_constant (Pmode, stack_pointer_rtx, -UNITS_PER_WORD);
9418 0 : t = gen_rtx_SET (stack_pointer_rtx, t);
9419 0 : add_reg_note (insn, REG_CFA_ADJUST_CFA, t);
9420 0 : RTX_FRAME_RELATED_P (insn) = 1;
9421 : }
9422 :
9423 : /* Emit prologue code to adjust stack alignment and setup DRAP, in case
9424 : of DRAP is needed and stack realignment is really needed after reload */
9425 1566319 : if (stack_realign_drap)
9426 : {
9427 7012 : int align_bytes = crtl->stack_alignment_needed / BITS_PER_UNIT;
9428 :
9429 : /* Can't use DRAP in interrupt function. */
9430 7012 : if (cfun->machine->func_type != TYPE_NORMAL)
9431 0 : sorry ("Dynamic Realign Argument Pointer (DRAP) not supported "
9432 : "in interrupt service routine. This may be worked "
9433 : "around by avoiding functions with aggregate return.");
9434 :
9435 : /* Only need to push parameter pointer reg if it is caller saved. */
9436 7012 : if (!call_used_or_fixed_reg_p (REGNO (crtl->drap_reg)))
9437 : {
9438 : /* Push arg pointer reg */
9439 137 : insn = emit_insn (gen_push (crtl->drap_reg));
9440 137 : RTX_FRAME_RELATED_P (insn) = 1;
9441 : }
9442 :
9443 : /* Grab the argument pointer. */
9444 7297 : t = plus_constant (Pmode, stack_pointer_rtx, m->fs.sp_offset);
9445 7012 : insn = emit_insn (gen_rtx_SET (crtl->drap_reg, t));
9446 7012 : RTX_FRAME_RELATED_P (insn) = 1;
9447 7012 : m->fs.cfa_reg = crtl->drap_reg;
9448 7012 : m->fs.cfa_offset = 0;
9449 :
9450 : /* Align the stack. */
9451 7012 : insn = emit_insn (gen_and2_insn (stack_pointer_rtx,
9452 7012 : GEN_INT (-align_bytes)));
9453 7012 : RTX_FRAME_RELATED_P (insn) = 1;
9454 :
9455 : /* Replicate the return address on the stack so that return
9456 : address can be reached via (argp - 1) slot. This is needed
9457 : to implement macro RETURN_ADDR_RTX and intrinsic function
9458 : expand_builtin_return_addr etc. */
9459 7582 : t = plus_constant (Pmode, crtl->drap_reg, -UNITS_PER_WORD);
9460 7012 : t = gen_frame_mem (word_mode, t);
9461 7012 : insn = emit_insn (gen_push (t));
9462 7012 : RTX_FRAME_RELATED_P (insn) = 1;
9463 :
9464 : /* For the purposes of frame and register save area addressing,
9465 : we've started over with a new frame. */
9466 7012 : m->fs.sp_offset = INCOMING_FRAME_SP_OFFSET;
9467 7012 : m->fs.realigned = true;
9468 :
9469 7012 : if (static_chain)
9470 : {
9471 : /* Replicate static chain on the stack so that static chain
9472 : can be reached via (argp - 2) slot. This is needed for
9473 : nested function with stack realignment. */
9474 0 : insn = emit_insn (gen_push (static_chain));
9475 0 : RTX_FRAME_RELATED_P (insn) = 1;
9476 : }
9477 : }
9478 :
9479 1566319 : int_registers_saved = (frame.nregs == 0);
9480 1566319 : sse_registers_saved = (frame.nsseregs == 0);
9481 1566319 : save_stub_call_needed = (m->call_ms2sysv);
9482 1566319 : gcc_assert (sse_registers_saved || !save_stub_call_needed);
9483 :
9484 1566319 : if (frame_pointer_needed && !m->fs.fp_valid)
9485 : {
9486 : /* Note: AT&T enter does NOT have reversed args. Enter is probably
9487 : slower on all targets. Also sdb didn't like it. */
9488 497319 : insn = emit_insn (gen_push (hard_frame_pointer_rtx));
9489 497319 : RTX_FRAME_RELATED_P (insn) = 1;
9490 :
9491 497319 : if (m->fs.sp_offset == frame.hard_frame_pointer_offset)
9492 : {
9493 497319 : insn = emit_move_insn (hard_frame_pointer_rtx, stack_pointer_rtx);
9494 497319 : RTX_FRAME_RELATED_P (insn) = 1;
9495 :
9496 497319 : if (m->fs.cfa_reg == stack_pointer_rtx)
9497 490307 : m->fs.cfa_reg = hard_frame_pointer_rtx;
9498 497319 : m->fs.fp_offset = m->fs.sp_offset;
9499 497319 : m->fs.fp_valid = true;
9500 : }
9501 : }
9502 :
9503 1566319 : if (!int_registers_saved)
9504 : {
9505 : /* If saving registers via PUSH, do so now. */
9506 486309 : if (!frame.save_regs_using_mov)
9507 : {
9508 440414 : ix86_emit_save_regs ();
9509 440414 : m->fs.apx_ppx_used = TARGET_APX_PPX && !crtl->calls_eh_return;
9510 440414 : int_registers_saved = true;
9511 440414 : gcc_assert (m->fs.sp_offset == frame.reg_save_offset);
9512 : }
9513 :
9514 : /* When using red zone we may start register saving before allocating
9515 : the stack frame saving one cycle of the prologue. However, avoid
9516 : doing this if we have to probe the stack; at least on x86_64 the
9517 : stack probe can turn into a call that clobbers a red zone location. */
9518 45895 : else if (ix86_using_red_zone ()
9519 45895 : && (! TARGET_STACK_PROBE
9520 0 : || frame.stack_pointer_offset < CHECK_STACK_LIMIT))
9521 : {
9522 41443 : HOST_WIDE_INT allocate_offset;
9523 41443 : if (crtl->shrink_wrapped_separate)
9524 : {
9525 41387 : allocate_offset = m->fs.sp_offset - frame.stack_pointer_offset;
9526 :
9527 : /* Adjust the total offset at the beginning of the function. */
9528 41387 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
9529 : GEN_INT (allocate_offset), -1,
9530 41387 : m->fs.cfa_reg == stack_pointer_rtx);
9531 41387 : m->fs.sp_offset = cfun->machine->frame.stack_pointer_offset;
9532 : }
9533 :
9534 41443 : ix86_emit_save_regs_using_mov (frame.reg_save_offset);
9535 41443 : int_registers_saved = true;
9536 : }
9537 : }
9538 :
9539 1566319 : if (frame.red_zone_size != 0)
9540 146770 : cfun->machine->red_zone_used = true;
9541 :
9542 1566319 : if (stack_realign_fp)
9543 : {
9544 24746 : int align_bytes = crtl->stack_alignment_needed / BITS_PER_UNIT;
9545 25095 : gcc_assert (align_bytes > MIN_STACK_BOUNDARY / BITS_PER_UNIT);
9546 :
9547 : /* Record last valid frame pointer offset. */
9548 24746 : m->fs.sp_realigned_fp_last = frame.reg_save_offset;
9549 :
9550 : /* The computation of the size of the re-aligned stack frame means
9551 : that we must allocate the size of the register save area before
9552 : performing the actual alignment. Otherwise we cannot guarantee
9553 : that there's enough storage above the realignment point. */
9554 24746 : allocate = frame.reg_save_offset - m->fs.sp_offset
9555 24746 : + frame.stack_realign_allocate;
9556 24746 : if (allocate)
9557 2691 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
9558 : GEN_INT (-allocate), -1, false);
9559 :
9560 : /* Align the stack. */
9561 24746 : emit_insn (gen_and2_insn (stack_pointer_rtx, GEN_INT (-align_bytes)));
9562 24746 : m->fs.sp_offset = ROUND_UP (m->fs.sp_offset, align_bytes);
9563 24746 : m->fs.sp_realigned_offset = m->fs.sp_offset
9564 24746 : - frame.stack_realign_allocate;
9565 : /* The stack pointer may no longer be equal to CFA - m->fs.sp_offset.
9566 : Beyond this point, stack access should be done via choose_baseaddr or
9567 : by using sp_valid_at and fp_valid_at to determine the correct base
9568 : register. Henceforth, any CFA offset should be thought of as logical
9569 : and not physical. */
9570 24746 : gcc_assert (m->fs.sp_realigned_offset >= m->fs.sp_realigned_fp_last);
9571 24746 : gcc_assert (m->fs.sp_realigned_offset == frame.stack_realign_offset);
9572 24746 : m->fs.sp_realigned = true;
9573 :
9574 : /* SEH unwind emit doesn't currently support REG_CFA_EXPRESSION, which
9575 : is needed to describe where a register is saved using a realigned
9576 : stack pointer, so we need to invalidate the stack pointer for that
9577 : target. */
9578 24746 : if (TARGET_SEH)
9579 : m->fs.sp_valid = false;
9580 :
9581 : /* If SP offset is non-immediate after allocation of the stack frame,
9582 : then emit SSE saves or stub call prior to allocating the rest of the
9583 : stack frame. This is less efficient for the out-of-line stub because
9584 : we can't combine allocations across the call barrier, but it's better
9585 : than using a scratch register. */
9586 24746 : else if (!x86_64_immediate_operand (GEN_INT (frame.stack_pointer_offset
9587 : - m->fs.sp_realigned_offset),
9588 24746 : Pmode))
9589 : {
9590 3 : if (!sse_registers_saved)
9591 : {
9592 1 : ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset);
9593 1 : sse_registers_saved = true;
9594 : }
9595 2 : else if (save_stub_call_needed)
9596 : {
9597 1 : ix86_emit_outlined_ms2sysv_save (frame);
9598 1 : save_stub_call_needed = false;
9599 : }
9600 : }
9601 : }
9602 :
9603 1566319 : allocate = frame.stack_pointer_offset - m->fs.sp_offset;
9604 :
9605 1566319 : if (flag_stack_usage_info)
9606 : {
9607 : /* We start to count from ARG_POINTER. */
9608 355 : HOST_WIDE_INT stack_size = frame.stack_pointer_offset;
9609 :
9610 : /* If it was realigned, take into account the fake frame. */
9611 355 : if (stack_realign_drap)
9612 : {
9613 1 : if (ix86_static_chain_on_stack)
9614 0 : stack_size += UNITS_PER_WORD;
9615 :
9616 1 : if (!call_used_or_fixed_reg_p (REGNO (crtl->drap_reg)))
9617 0 : stack_size += UNITS_PER_WORD;
9618 :
9619 : /* This over-estimates by 1 minimal-stack-alignment-unit but
9620 : mitigates that by counting in the new return address slot. */
9621 1 : current_function_dynamic_stack_size
9622 1 : += crtl->stack_alignment_needed / BITS_PER_UNIT;
9623 : }
9624 :
9625 355 : current_function_static_stack_size = stack_size;
9626 : }
9627 :
9628 : /* On SEH target with very large frame size, allocate an area to save
9629 : SSE registers (as the very large allocation won't be described). */
9630 1566319 : if (TARGET_SEH
9631 : && frame.stack_pointer_offset > SEH_MAX_FRAME_SIZE
9632 : && !sse_registers_saved)
9633 : {
9634 : HOST_WIDE_INT sse_size
9635 : = frame.sse_reg_save_offset - frame.reg_save_offset;
9636 :
9637 : gcc_assert (int_registers_saved);
9638 :
9639 : /* No need to do stack checking as the area will be immediately
9640 : written. */
9641 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
9642 : GEN_INT (-sse_size), -1,
9643 : m->fs.cfa_reg == stack_pointer_rtx);
9644 : allocate -= sse_size;
9645 : ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset);
9646 : sse_registers_saved = true;
9647 : }
9648 :
9649 : /* If stack clash protection is requested, then probe the stack, unless it
9650 : is already probed on the target. */
9651 1566319 : if (allocate >= 0
9652 1566315 : && flag_stack_clash_protection
9653 1566417 : && !ix86_target_stack_probe ())
9654 : {
9655 98 : ix86_adjust_stack_and_probe (allocate, int_registers_saved, false);
9656 98 : allocate = 0;
9657 : }
9658 :
9659 : /* The stack has already been decremented by the instruction calling us
9660 : so probe if the size is non-negative to preserve the protection area. */
9661 1566221 : else if (allocate >= 0 && flag_stack_check == STATIC_BUILTIN_STACK_CHECK)
9662 : {
9663 45 : const HOST_WIDE_INT probe_interval = get_probe_interval ();
9664 :
9665 45 : if (STACK_CHECK_MOVING_SP)
9666 : {
9667 45 : if (crtl->is_leaf
9668 18 : && !cfun->calls_alloca
9669 18 : && allocate <= probe_interval)
9670 : ;
9671 :
9672 : else
9673 : {
9674 28 : ix86_adjust_stack_and_probe (allocate, int_registers_saved, true);
9675 28 : allocate = 0;
9676 : }
9677 : }
9678 :
9679 : else
9680 : {
9681 : HOST_WIDE_INT size = allocate;
9682 :
9683 : if (TARGET_64BIT && size >= HOST_WIDE_INT_C (0x80000000))
9684 : size = 0x80000000 - get_stack_check_protect () - 1;
9685 :
9686 : if (TARGET_STACK_PROBE)
9687 : {
9688 : if (crtl->is_leaf && !cfun->calls_alloca)
9689 : {
9690 : if (size > probe_interval)
9691 : ix86_emit_probe_stack_range (0, size, int_registers_saved);
9692 : }
9693 : else
9694 : ix86_emit_probe_stack_range (0,
9695 : size + get_stack_check_protect (),
9696 : int_registers_saved);
9697 : }
9698 : else
9699 : {
9700 : if (crtl->is_leaf && !cfun->calls_alloca)
9701 : {
9702 : if (size > probe_interval
9703 : && size > get_stack_check_protect ())
9704 : ix86_emit_probe_stack_range (get_stack_check_protect (),
9705 : (size
9706 : - get_stack_check_protect ()),
9707 : int_registers_saved);
9708 : }
9709 : else
9710 : ix86_emit_probe_stack_range (get_stack_check_protect (), size,
9711 : int_registers_saved);
9712 : }
9713 : }
9714 : }
9715 :
9716 1566315 : if (allocate == 0)
9717 : ;
9718 858974 : else if (!ix86_target_stack_probe ()
9719 858974 : || frame.stack_pointer_offset < CHECK_STACK_LIMIT)
9720 : {
9721 858929 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
9722 : GEN_INT (-allocate), -1,
9723 858929 : m->fs.cfa_reg == stack_pointer_rtx);
9724 : }
9725 : else
9726 : {
9727 45 : rtx eax = gen_rtx_REG (Pmode, AX_REG);
9728 45 : rtx r10 = NULL;
9729 45 : const bool sp_is_cfa_reg = (m->fs.cfa_reg == stack_pointer_rtx);
9730 45 : bool eax_live = ix86_eax_live_at_start_p ();
9731 45 : bool r10_live = false;
9732 :
9733 45 : if (TARGET_64BIT)
9734 45 : r10_live = (DECL_STATIC_CHAIN (current_function_decl) != 0);
9735 :
9736 45 : if (eax_live)
9737 : {
9738 0 : insn = emit_insn (gen_push (eax));
9739 0 : allocate -= UNITS_PER_WORD;
9740 : /* Note that SEH directives need to continue tracking the stack
9741 : pointer even after the frame pointer has been set up. */
9742 0 : if (sp_is_cfa_reg || TARGET_SEH)
9743 : {
9744 0 : if (sp_is_cfa_reg)
9745 0 : m->fs.cfa_offset += UNITS_PER_WORD;
9746 0 : RTX_FRAME_RELATED_P (insn) = 1;
9747 0 : add_reg_note (insn, REG_FRAME_RELATED_EXPR,
9748 0 : gen_rtx_SET (stack_pointer_rtx,
9749 : plus_constant (Pmode,
9750 : stack_pointer_rtx,
9751 : -UNITS_PER_WORD)));
9752 : }
9753 : }
9754 :
9755 45 : if (r10_live)
9756 : {
9757 0 : r10 = gen_rtx_REG (Pmode, R10_REG);
9758 0 : insn = emit_insn (gen_push (r10));
9759 0 : allocate -= UNITS_PER_WORD;
9760 0 : if (sp_is_cfa_reg || TARGET_SEH)
9761 : {
9762 0 : if (sp_is_cfa_reg)
9763 0 : m->fs.cfa_offset += UNITS_PER_WORD;
9764 0 : RTX_FRAME_RELATED_P (insn) = 1;
9765 0 : add_reg_note (insn, REG_FRAME_RELATED_EXPR,
9766 0 : gen_rtx_SET (stack_pointer_rtx,
9767 : plus_constant (Pmode,
9768 : stack_pointer_rtx,
9769 : -UNITS_PER_WORD)));
9770 : }
9771 : }
9772 :
9773 45 : emit_move_insn (eax, GEN_INT (allocate));
9774 45 : emit_insn (gen_allocate_stack_worker_probe (Pmode, eax, eax));
9775 :
9776 : /* Use the fact that AX still contains ALLOCATE. */
9777 45 : insn = emit_insn (gen_pro_epilogue_adjust_stack_sub
9778 45 : (Pmode, stack_pointer_rtx, stack_pointer_rtx, eax));
9779 :
9780 45 : if (sp_is_cfa_reg || TARGET_SEH)
9781 : {
9782 37 : if (sp_is_cfa_reg)
9783 37 : m->fs.cfa_offset += allocate;
9784 37 : RTX_FRAME_RELATED_P (insn) = 1;
9785 37 : add_reg_note (insn, REG_FRAME_RELATED_EXPR,
9786 37 : gen_rtx_SET (stack_pointer_rtx,
9787 : plus_constant (Pmode, stack_pointer_rtx,
9788 : -allocate)));
9789 : }
9790 45 : m->fs.sp_offset += allocate;
9791 :
9792 : /* Use stack_pointer_rtx for relative addressing so that code works for
9793 : realigned stack. But this means that we need a blockage to prevent
9794 : stores based on the frame pointer from being scheduled before. */
9795 45 : if (r10_live && eax_live)
9796 : {
9797 0 : t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, eax);
9798 0 : emit_move_insn (gen_rtx_REG (word_mode, R10_REG),
9799 : gen_frame_mem (word_mode, t));
9800 0 : t = plus_constant (Pmode, t, UNITS_PER_WORD);
9801 0 : emit_move_insn (gen_rtx_REG (word_mode, AX_REG),
9802 : gen_frame_mem (word_mode, t));
9803 0 : emit_insn (gen_memory_blockage ());
9804 : }
9805 45 : else if (eax_live || r10_live)
9806 : {
9807 0 : t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, eax);
9808 0 : emit_move_insn (gen_rtx_REG (word_mode,
9809 : (eax_live ? AX_REG : R10_REG)),
9810 : gen_frame_mem (word_mode, t));
9811 0 : emit_insn (gen_memory_blockage ());
9812 : }
9813 : }
9814 1566319 : gcc_assert (m->fs.sp_offset == frame.stack_pointer_offset);
9815 :
9816 : /* If we haven't already set up the frame pointer, do so now. */
9817 1566319 : if (frame_pointer_needed && !m->fs.fp_valid)
9818 : {
9819 0 : insn = gen_add3_insn (hard_frame_pointer_rtx, stack_pointer_rtx,
9820 0 : GEN_INT (frame.stack_pointer_offset
9821 : - frame.hard_frame_pointer_offset));
9822 0 : insn = emit_insn (insn);
9823 0 : RTX_FRAME_RELATED_P (insn) = 1;
9824 0 : add_reg_note (insn, REG_CFA_ADJUST_CFA, NULL);
9825 :
9826 0 : if (m->fs.cfa_reg == stack_pointer_rtx)
9827 0 : m->fs.cfa_reg = hard_frame_pointer_rtx;
9828 0 : m->fs.fp_offset = frame.hard_frame_pointer_offset;
9829 0 : m->fs.fp_valid = true;
9830 : }
9831 :
9832 1566319 : if (!int_registers_saved)
9833 4452 : ix86_emit_save_regs_using_mov (frame.reg_save_offset);
9834 1566319 : if (!sse_registers_saved)
9835 33375 : ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset);
9836 1532944 : else if (save_stub_call_needed)
9837 7044 : ix86_emit_outlined_ms2sysv_save (frame);
9838 :
9839 : /* For the mcount profiling on 32 bit PIC mode we need to emit SET_GOT
9840 : in PROLOGUE. */
9841 1566319 : if (!TARGET_64BIT && pic_offset_table_rtx && crtl->profile && !flag_fentry)
9842 : {
9843 0 : rtx pic = gen_rtx_REG (Pmode, REAL_PIC_OFFSET_TABLE_REGNUM);
9844 0 : insn = emit_insn (gen_set_got (pic));
9845 0 : RTX_FRAME_RELATED_P (insn) = 1;
9846 0 : add_reg_note (insn, REG_CFA_FLUSH_QUEUE, NULL_RTX);
9847 0 : emit_insn (gen_prologue_use (pic));
9848 : /* Deleting already emitted SET_GOT if exist and allocated to
9849 : REAL_PIC_OFFSET_TABLE_REGNUM. */
9850 0 : ix86_elim_entry_set_got (pic);
9851 : }
9852 :
9853 1566319 : if (crtl->drap_reg && !crtl->stack_realign_needed)
9854 : {
9855 : /* vDRAP is setup but after reload it turns out stack realign
9856 : isn't necessary, here we will emit prologue to setup DRAP
9857 : without stack realign adjustment */
9858 195 : t = choose_baseaddr (0, NULL);
9859 195 : emit_insn (gen_rtx_SET (crtl->drap_reg, t));
9860 : }
9861 :
9862 : /* Prevent instructions from being scheduled into register save push
9863 : sequence when access to the redzone area is done through frame pointer.
9864 : The offset between the frame pointer and the stack pointer is calculated
9865 : relative to the value of the stack pointer at the end of the function
9866 : prologue, and moving instructions that access redzone area via frame
9867 : pointer inside push sequence violates this assumption. */
9868 1566319 : if (frame_pointer_needed && frame.red_zone_size)
9869 135716 : emit_insn (gen_memory_blockage ());
9870 :
9871 : /* SEH requires that the prologue end within 256 bytes of the start of
9872 : the function. Prevent instruction schedules that would extend that.
9873 : Further, prevent alloca modifications to the stack pointer from being
9874 : combined with prologue modifications. */
9875 : if (TARGET_SEH)
9876 : emit_insn (gen_prologue_use (stack_pointer_rtx));
9877 : }
9878 :
9879 : /* Emit code to restore REG using a POP or POPP insn. */
9880 :
9881 : static void
9882 1509849 : ix86_emit_restore_reg_using_pop (rtx reg, bool ppx_p)
9883 : {
9884 1509849 : struct machine_function *m = cfun->machine;
9885 1509849 : rtx_insn *insn = emit_insn (gen_pop (reg, ppx_p));
9886 :
9887 1509849 : ix86_add_cfa_restore_note (insn, reg, m->fs.sp_offset);
9888 1509849 : m->fs.sp_offset -= UNITS_PER_WORD;
9889 :
9890 1509849 : if (m->fs.cfa_reg == crtl->drap_reg
9891 1509849 : && REGNO (reg) == REGNO (crtl->drap_reg))
9892 : {
9893 : /* Previously we'd represented the CFA as an expression
9894 : like *(%ebp - 8). We've just popped that value from
9895 : the stack, which means we need to reset the CFA to
9896 : the drap register. This will remain until we restore
9897 : the stack pointer. */
9898 3975 : add_reg_note (insn, REG_CFA_DEF_CFA, reg);
9899 3975 : RTX_FRAME_RELATED_P (insn) = 1;
9900 :
9901 : /* This means that the DRAP register is valid for addressing too. */
9902 3975 : m->fs.drap_valid = true;
9903 3975 : return;
9904 : }
9905 :
9906 1505874 : if (m->fs.cfa_reg == stack_pointer_rtx)
9907 : {
9908 1415716 : rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
9909 1051617 : x = gen_rtx_SET (stack_pointer_rtx, x);
9910 1051617 : add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
9911 1051617 : RTX_FRAME_RELATED_P (insn) = 1;
9912 :
9913 1233659 : m->fs.cfa_offset -= UNITS_PER_WORD;
9914 : }
9915 :
9916 : /* When the frame pointer is the CFA, and we pop it, we are
9917 : swapping back to the stack pointer as the CFA. This happens
9918 : for stack frames that don't allocate other data, so we assume
9919 : the stack pointer is now pointing at the return address, i.e.
9920 : the function entry state, which makes the offset be 1 word. */
9921 1505874 : if (reg == hard_frame_pointer_rtx)
9922 : {
9923 246225 : m->fs.fp_valid = false;
9924 246225 : if (m->fs.cfa_reg == hard_frame_pointer_rtx)
9925 : {
9926 242237 : m->fs.cfa_reg = stack_pointer_rtx;
9927 242237 : m->fs.cfa_offset -= UNITS_PER_WORD;
9928 :
9929 242237 : add_reg_note (insn, REG_CFA_DEF_CFA,
9930 242237 : plus_constant (Pmode, stack_pointer_rtx,
9931 242237 : m->fs.cfa_offset));
9932 242237 : RTX_FRAME_RELATED_P (insn) = 1;
9933 : }
9934 : }
9935 : }
9936 :
9937 : /* Emit code to restore REG using a POP2 insn. */
9938 : static void
9939 19 : ix86_emit_restore_reg_using_pop2 (rtx reg1, rtx reg2, bool ppx_p = false)
9940 : {
9941 19 : struct machine_function *m = cfun->machine;
9942 19 : const int offset = UNITS_PER_WORD * 2;
9943 19 : rtx_insn *insn;
9944 :
9945 19 : rtx mem = gen_rtx_MEM (TImode, gen_rtx_POST_INC (Pmode,
9946 : stack_pointer_rtx));
9947 :
9948 19 : if (ppx_p)
9949 15 : insn = emit_insn (gen_pop2p_di (reg1, mem, reg2));
9950 : else
9951 4 : insn = emit_insn (gen_pop2_di (reg1, mem, reg2));
9952 :
9953 19 : RTX_FRAME_RELATED_P (insn) = 1;
9954 :
9955 19 : rtx dwarf = NULL_RTX;
9956 19 : dwarf = alloc_reg_note (REG_CFA_RESTORE, reg1, dwarf);
9957 19 : dwarf = alloc_reg_note (REG_CFA_RESTORE, reg2, dwarf);
9958 19 : REG_NOTES (insn) = dwarf;
9959 19 : m->fs.sp_offset -= offset;
9960 :
9961 19 : if (m->fs.cfa_reg == crtl->drap_reg
9962 19 : && (REGNO (reg1) == REGNO (crtl->drap_reg)
9963 3 : || REGNO (reg2) == REGNO (crtl->drap_reg)))
9964 : {
9965 : /* Previously we'd represented the CFA as an expression
9966 : like *(%ebp - 8). We've just popped that value from
9967 : the stack, which means we need to reset the CFA to
9968 : the drap register. This will remain until we restore
9969 : the stack pointer. */
9970 1 : add_reg_note (insn, REG_CFA_DEF_CFA,
9971 1 : REGNO (reg1) == REGNO (crtl->drap_reg) ? reg1 : reg2);
9972 1 : RTX_FRAME_RELATED_P (insn) = 1;
9973 :
9974 : /* This means that the DRAP register is valid for addressing too. */
9975 1 : m->fs.drap_valid = true;
9976 1 : return;
9977 : }
9978 :
9979 18 : if (m->fs.cfa_reg == stack_pointer_rtx)
9980 : {
9981 14 : rtx x = plus_constant (Pmode, stack_pointer_rtx, offset);
9982 14 : x = gen_rtx_SET (stack_pointer_rtx, x);
9983 14 : add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
9984 14 : RTX_FRAME_RELATED_P (insn) = 1;
9985 :
9986 14 : m->fs.cfa_offset -= offset;
9987 : }
9988 :
9989 : /* When the frame pointer is the CFA, and we pop it, we are
9990 : swapping back to the stack pointer as the CFA. This happens
9991 : for stack frames that don't allocate other data, so we assume
9992 : the stack pointer is now pointing at the return address, i.e.
9993 : the function entry state, which makes the offset be 1 word. */
9994 18 : if (reg1 == hard_frame_pointer_rtx || reg2 == hard_frame_pointer_rtx)
9995 : {
9996 0 : m->fs.fp_valid = false;
9997 0 : if (m->fs.cfa_reg == hard_frame_pointer_rtx)
9998 : {
9999 0 : m->fs.cfa_reg = stack_pointer_rtx;
10000 0 : m->fs.cfa_offset -= offset;
10001 :
10002 0 : add_reg_note (insn, REG_CFA_DEF_CFA,
10003 0 : plus_constant (Pmode, stack_pointer_rtx,
10004 0 : m->fs.cfa_offset));
10005 0 : RTX_FRAME_RELATED_P (insn) = 1;
10006 : }
10007 : }
10008 : }
10009 :
10010 : /* Emit code to restore saved registers using POP insns. */
10011 :
10012 : static void
10013 1389793 : ix86_emit_restore_regs_using_pop (bool ppx_p)
10014 : {
10015 1389793 : unsigned int regno;
10016 :
10017 132030335 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
10018 130640542 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, false, true))
10019 1263302 : ix86_emit_restore_reg_using_pop (gen_rtx_REG (word_mode, regno), ppx_p);
10020 1389793 : }
10021 :
10022 : /* Emit code to restore saved registers using POP2 insns. */
10023 :
10024 : static void
10025 575 : ix86_emit_restore_regs_using_pop2 (void)
10026 : {
10027 575 : int regno;
10028 575 : int regno_list[2];
10029 575 : regno_list[0] = regno_list[1] = -1;
10030 575 : int loaded_regnum = 0;
10031 575 : bool aligned = cfun->machine->fs.sp_offset % 16 == 0;
10032 :
10033 54625 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
10034 54050 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, false, true))
10035 : {
10036 128 : if (aligned)
10037 : {
10038 121 : regno_list[loaded_regnum++] = regno;
10039 121 : if (loaded_regnum == 2)
10040 : {
10041 19 : gcc_assert (regno_list[0] != -1
10042 : && regno_list[1] != -1
10043 : && regno_list[0] != regno_list[1]);
10044 :
10045 19 : ix86_emit_restore_reg_using_pop2 (gen_rtx_REG (word_mode,
10046 : regno_list[0]),
10047 : gen_rtx_REG (word_mode,
10048 : regno_list[1]),
10049 19 : TARGET_APX_PPX);
10050 19 : loaded_regnum = 0;
10051 19 : regno_list[0] = regno_list[1] = -1;
10052 : }
10053 : }
10054 : else
10055 : {
10056 14 : ix86_emit_restore_reg_using_pop (gen_rtx_REG (word_mode, regno),
10057 7 : TARGET_APX_PPX);
10058 7 : aligned = true;
10059 : }
10060 : }
10061 :
10062 575 : if (loaded_regnum == 1)
10063 83 : ix86_emit_restore_reg_using_pop (gen_rtx_REG (word_mode, regno_list[0]),
10064 83 : TARGET_APX_PPX);
10065 575 : }
10066 :
10067 : /* Emit code and notes for the LEAVE instruction. If insn is non-null,
10068 : omits the emit and only attaches the notes. */
10069 :
10070 : static void
10071 252398 : ix86_emit_leave (rtx_insn *insn)
10072 : {
10073 252398 : struct machine_function *m = cfun->machine;
10074 :
10075 252398 : if (!insn)
10076 251435 : insn = emit_insn (gen_leave (word_mode));
10077 :
10078 252398 : ix86_add_queued_cfa_restore_notes (insn);
10079 :
10080 252398 : gcc_assert (m->fs.fp_valid);
10081 252398 : m->fs.sp_valid = true;
10082 252398 : m->fs.sp_realigned = false;
10083 252398 : m->fs.sp_offset = m->fs.fp_offset - UNITS_PER_WORD;
10084 252398 : m->fs.fp_valid = false;
10085 :
10086 252398 : if (m->fs.cfa_reg == hard_frame_pointer_rtx)
10087 : {
10088 249266 : m->fs.cfa_reg = stack_pointer_rtx;
10089 249266 : m->fs.cfa_offset = m->fs.sp_offset;
10090 :
10091 249266 : add_reg_note (insn, REG_CFA_DEF_CFA,
10092 249266 : plus_constant (Pmode, stack_pointer_rtx,
10093 249266 : m->fs.sp_offset));
10094 249266 : RTX_FRAME_RELATED_P (insn) = 1;
10095 : }
10096 252398 : ix86_add_cfa_restore_note (insn, hard_frame_pointer_rtx,
10097 : m->fs.fp_offset);
10098 252398 : }
10099 :
10100 : /* Emit code to restore saved registers using MOV insns.
10101 : First register is restored from CFA - CFA_OFFSET. */
10102 : static void
10103 99159 : ix86_emit_restore_regs_using_mov (HOST_WIDE_INT cfa_offset,
10104 : bool maybe_eh_return)
10105 : {
10106 99159 : struct machine_function *m = cfun->machine;
10107 99159 : unsigned int regno;
10108 :
10109 9420105 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
10110 9320946 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, maybe_eh_return, true))
10111 : {
10112 :
10113 : /* Skip registers, already processed by shrink wrap separate. */
10114 270971 : if (!cfun->machine->reg_is_wrapped_separately[regno])
10115 : {
10116 143966 : rtx reg = gen_rtx_REG (word_mode, regno);
10117 143966 : rtx mem;
10118 143966 : rtx_insn *insn;
10119 :
10120 143966 : mem = choose_baseaddr (cfa_offset, NULL);
10121 143966 : mem = gen_frame_mem (word_mode, mem);
10122 143966 : insn = emit_move_insn (reg, mem);
10123 :
10124 143966 : if (m->fs.cfa_reg == crtl->drap_reg
10125 143966 : && regno == REGNO (crtl->drap_reg))
10126 : {
10127 : /* Previously we'd represented the CFA as an expression
10128 : like *(%ebp - 8). We've just popped that value from
10129 : the stack, which means we need to reset the CFA to
10130 : the drap register. This will remain until we restore
10131 : the stack pointer. */
10132 3132 : add_reg_note (insn, REG_CFA_DEF_CFA, reg);
10133 3132 : RTX_FRAME_RELATED_P (insn) = 1;
10134 :
10135 : /* DRAP register is valid for addressing. */
10136 3132 : m->fs.drap_valid = true;
10137 : }
10138 : else
10139 140834 : ix86_add_cfa_restore_note (NULL, reg, cfa_offset);
10140 : }
10141 291393 : cfa_offset -= UNITS_PER_WORD;
10142 : }
10143 99159 : }
10144 :
10145 : /* Emit code to restore saved registers using MOV insns.
10146 : First register is restored from CFA - CFA_OFFSET. */
10147 : static void
10148 33952 : ix86_emit_restore_sse_regs_using_mov (HOST_WIDE_INT cfa_offset,
10149 : bool maybe_eh_return)
10150 : {
10151 33952 : unsigned int regno;
10152 :
10153 3225440 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
10154 3191488 : if (SSE_REGNO_P (regno) && ix86_save_reg (regno, maybe_eh_return, true))
10155 : {
10156 339529 : rtx reg = gen_rtx_REG (V4SFmode, regno);
10157 339529 : rtx mem;
10158 339529 : unsigned int align = GET_MODE_ALIGNMENT (V4SFmode);
10159 :
10160 339529 : mem = choose_baseaddr (cfa_offset, &align);
10161 339529 : mem = gen_rtx_MEM (V4SFmode, mem);
10162 :
10163 : /* The location alignment depends upon the base register. */
10164 339529 : align = MIN (GET_MODE_ALIGNMENT (V4SFmode), align);
10165 339529 : gcc_assert (! (cfa_offset & (align / BITS_PER_UNIT - 1)));
10166 339529 : set_mem_align (mem, align);
10167 339529 : emit_insn (gen_rtx_SET (reg, mem));
10168 :
10169 339529 : ix86_add_cfa_restore_note (NULL, reg, cfa_offset);
10170 :
10171 339529 : cfa_offset -= GET_MODE_SIZE (V4SFmode);
10172 : }
10173 33952 : }
10174 :
10175 : static void
10176 7621 : ix86_emit_outlined_ms2sysv_restore (const struct ix86_frame &frame,
10177 : bool use_call, int style)
10178 : {
10179 7621 : struct machine_function *m = cfun->machine;
10180 7621 : const unsigned ncregs = NUM_X86_64_MS_CLOBBERED_REGS
10181 7621 : + m->call_ms2sysv_extra_regs;
10182 7621 : rtvec v;
10183 7621 : unsigned int elems_needed, align, i, vi = 0;
10184 7621 : rtx_insn *insn;
10185 7621 : rtx sym, tmp;
10186 7621 : rtx rsi = gen_rtx_REG (word_mode, SI_REG);
10187 7621 : rtx r10 = NULL_RTX;
10188 7621 : const class xlogue_layout &xlogue = xlogue_layout::get_instance ();
10189 7621 : HOST_WIDE_INT stub_ptr_offset = xlogue.get_stub_ptr_offset ();
10190 7621 : HOST_WIDE_INT rsi_offset = frame.stack_realign_offset + stub_ptr_offset;
10191 7621 : rtx rsi_frame_load = NULL_RTX;
10192 7621 : HOST_WIDE_INT rsi_restore_offset = (HOST_WIDE_INT)-1;
10193 7621 : enum xlogue_stub stub;
10194 :
10195 7621 : gcc_assert (!m->fs.fp_valid || frame_pointer_needed);
10196 :
10197 : /* If using a realigned stack, we should never start with padding. */
10198 7621 : gcc_assert (!stack_realign_fp || !xlogue.get_stack_align_off_in ());
10199 :
10200 : /* Setup RSI as the stub's base pointer. */
10201 7621 : align = GET_MODE_ALIGNMENT (V4SFmode);
10202 7621 : tmp = choose_baseaddr (rsi_offset, &align, SI_REG);
10203 7621 : gcc_assert (align >= GET_MODE_ALIGNMENT (V4SFmode));
10204 :
10205 7621 : emit_insn (gen_rtx_SET (rsi, tmp));
10206 :
10207 : /* Get a symbol for the stub. */
10208 7621 : if (frame_pointer_needed)
10209 5955 : stub = use_call ? XLOGUE_STUB_RESTORE_HFP
10210 : : XLOGUE_STUB_RESTORE_HFP_TAIL;
10211 : else
10212 1666 : stub = use_call ? XLOGUE_STUB_RESTORE
10213 : : XLOGUE_STUB_RESTORE_TAIL;
10214 7621 : sym = xlogue.get_stub_rtx (stub);
10215 :
10216 7621 : elems_needed = ncregs;
10217 7621 : if (use_call)
10218 6506 : elems_needed += 1;
10219 : else
10220 1267 : elems_needed += frame_pointer_needed ? 5 : 3;
10221 7621 : v = rtvec_alloc (elems_needed);
10222 :
10223 : /* We call the epilogue stub when we need to pop incoming args or we are
10224 : doing a sibling call as the tail. Otherwise, we will emit a jmp to the
10225 : epilogue stub and it is the tail-call. */
10226 7621 : if (use_call)
10227 6506 : RTVEC_ELT (v, vi++) = gen_rtx_USE (VOIDmode, sym);
10228 : else
10229 : {
10230 1115 : RTVEC_ELT (v, vi++) = ret_rtx;
10231 1115 : RTVEC_ELT (v, vi++) = gen_rtx_USE (VOIDmode, sym);
10232 1115 : if (frame_pointer_needed)
10233 : {
10234 963 : rtx rbp = gen_rtx_REG (DImode, BP_REG);
10235 963 : gcc_assert (m->fs.fp_valid);
10236 963 : gcc_assert (m->fs.cfa_reg == hard_frame_pointer_rtx);
10237 :
10238 963 : tmp = plus_constant (DImode, rbp, 8);
10239 963 : RTVEC_ELT (v, vi++) = gen_rtx_SET (stack_pointer_rtx, tmp);
10240 963 : RTVEC_ELT (v, vi++) = gen_rtx_SET (rbp, gen_rtx_MEM (DImode, rbp));
10241 963 : tmp = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode));
10242 963 : RTVEC_ELT (v, vi++) = gen_rtx_CLOBBER (VOIDmode, tmp);
10243 : }
10244 : else
10245 : {
10246 : /* If no hard frame pointer, we set R10 to the SP restore value. */
10247 152 : gcc_assert (!m->fs.fp_valid);
10248 152 : gcc_assert (m->fs.cfa_reg == stack_pointer_rtx);
10249 152 : gcc_assert (m->fs.sp_valid);
10250 :
10251 152 : r10 = gen_rtx_REG (DImode, R10_REG);
10252 152 : tmp = plus_constant (Pmode, rsi, stub_ptr_offset);
10253 152 : emit_insn (gen_rtx_SET (r10, tmp));
10254 :
10255 152 : RTVEC_ELT (v, vi++) = gen_rtx_SET (stack_pointer_rtx, r10);
10256 : }
10257 : }
10258 :
10259 : /* Generate frame load insns and restore notes. */
10260 107830 : for (i = 0; i < ncregs; ++i)
10261 : {
10262 100209 : const xlogue_layout::reginfo &r = xlogue.get_reginfo (i);
10263 100209 : machine_mode mode = SSE_REGNO_P (r.regno) ? V4SFmode : word_mode;
10264 100209 : rtx reg, frame_load;
10265 :
10266 100209 : reg = gen_rtx_REG (mode, r.regno);
10267 100209 : frame_load = gen_frame_load (reg, rsi, r.offset);
10268 :
10269 : /* Save RSI frame load insn & note to add last. */
10270 100209 : if (r.regno == SI_REG)
10271 : {
10272 7621 : gcc_assert (!rsi_frame_load);
10273 7621 : rsi_frame_load = frame_load;
10274 7621 : rsi_restore_offset = r.offset;
10275 : }
10276 : else
10277 : {
10278 92588 : RTVEC_ELT (v, vi++) = frame_load;
10279 92588 : ix86_add_cfa_restore_note (NULL, reg, r.offset);
10280 : }
10281 : }
10282 :
10283 : /* Add RSI frame load & restore note at the end. */
10284 7621 : gcc_assert (rsi_frame_load);
10285 7621 : gcc_assert (rsi_restore_offset != (HOST_WIDE_INT)-1);
10286 7621 : RTVEC_ELT (v, vi++) = rsi_frame_load;
10287 7621 : ix86_add_cfa_restore_note (NULL, gen_rtx_REG (DImode, SI_REG),
10288 : rsi_restore_offset);
10289 :
10290 : /* Finally, for tail-call w/o a hard frame pointer, set SP to R10. */
10291 7621 : if (!use_call && !frame_pointer_needed)
10292 : {
10293 152 : gcc_assert (m->fs.sp_valid);
10294 152 : gcc_assert (!m->fs.sp_realigned);
10295 :
10296 : /* At this point, R10 should point to frame.stack_realign_offset. */
10297 152 : if (m->fs.cfa_reg == stack_pointer_rtx)
10298 152 : m->fs.cfa_offset += m->fs.sp_offset - frame.stack_realign_offset;
10299 152 : m->fs.sp_offset = frame.stack_realign_offset;
10300 : }
10301 :
10302 7621 : gcc_assert (vi == (unsigned int)GET_NUM_ELEM (v));
10303 7621 : tmp = gen_rtx_PARALLEL (VOIDmode, v);
10304 7621 : if (use_call)
10305 6506 : insn = emit_insn (tmp);
10306 : else
10307 : {
10308 1115 : insn = emit_jump_insn (tmp);
10309 1115 : JUMP_LABEL (insn) = ret_rtx;
10310 :
10311 1115 : if (frame_pointer_needed)
10312 963 : ix86_emit_leave (insn);
10313 : else
10314 : {
10315 : /* Need CFA adjust note. */
10316 152 : tmp = gen_rtx_SET (stack_pointer_rtx, r10);
10317 152 : add_reg_note (insn, REG_CFA_ADJUST_CFA, tmp);
10318 : }
10319 : }
10320 :
10321 7621 : RTX_FRAME_RELATED_P (insn) = true;
10322 7621 : ix86_add_queued_cfa_restore_notes (insn);
10323 :
10324 : /* If we're not doing a tail-call, we need to adjust the stack. */
10325 7621 : if (use_call && m->fs.sp_valid)
10326 : {
10327 3706 : HOST_WIDE_INT dealloc = m->fs.sp_offset - frame.stack_realign_offset;
10328 3706 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
10329 : GEN_INT (dealloc), style,
10330 3706 : m->fs.cfa_reg == stack_pointer_rtx);
10331 : }
10332 7621 : }
10333 :
10334 : /* Restore function stack, frame, and registers. */
10335 :
10336 : void
10337 1694304 : ix86_expand_epilogue (int style)
10338 : {
10339 1694304 : struct machine_function *m = cfun->machine;
10340 1694304 : struct machine_frame_state frame_state_save = m->fs;
10341 1694304 : bool restore_regs_via_mov;
10342 1694304 : bool using_drap;
10343 1694304 : bool restore_stub_is_tail = false;
10344 :
10345 1694304 : if (ix86_function_naked (current_function_decl))
10346 : {
10347 : /* The program should not reach this point. */
10348 76 : emit_insn (gen_ud2 ());
10349 128032 : return;
10350 : }
10351 :
10352 1694228 : ix86_finalize_stack_frame_flags ();
10353 1694228 : const struct ix86_frame &frame = cfun->machine->frame;
10354 :
10355 1694228 : m->fs.sp_realigned = stack_realign_fp;
10356 31893 : m->fs.sp_valid = stack_realign_fp
10357 1669443 : || !frame_pointer_needed
10358 2168093 : || crtl->sp_is_unchanging;
10359 1694228 : gcc_assert (!m->fs.sp_valid
10360 : || m->fs.sp_offset == frame.stack_pointer_offset);
10361 :
10362 : /* The FP must be valid if the frame pointer is present. */
10363 1694228 : gcc_assert (frame_pointer_needed == m->fs.fp_valid);
10364 1694228 : gcc_assert (!m->fs.fp_valid
10365 : || m->fs.fp_offset == frame.hard_frame_pointer_offset);
10366 :
10367 : /* We must have *some* valid pointer to the stack frame. */
10368 1694228 : gcc_assert (m->fs.sp_valid || m->fs.fp_valid);
10369 :
10370 : /* The DRAP is never valid at this point. */
10371 1694228 : gcc_assert (!m->fs.drap_valid);
10372 :
10373 : /* See the comment about red zone and frame
10374 : pointer usage in ix86_expand_prologue. */
10375 1694228 : if (frame_pointer_needed && frame.red_zone_size)
10376 135746 : emit_insn (gen_memory_blockage ());
10377 :
10378 1694228 : using_drap = crtl->drap_reg && crtl->stack_realign_needed;
10379 7108 : gcc_assert (!using_drap || m->fs.cfa_reg == crtl->drap_reg);
10380 :
10381 : /* Determine the CFA offset of the end of the red-zone. */
10382 1694228 : m->fs.red_zone_offset = 0;
10383 1694228 : if (ix86_using_red_zone () && crtl->args.pops_args < 65536)
10384 : {
10385 : /* The red-zone begins below return address and error code in
10386 : exception handler. */
10387 1516637 : m->fs.red_zone_offset = RED_ZONE_SIZE + INCOMING_FRAME_SP_OFFSET;
10388 :
10389 : /* When the register save area is in the aligned portion of
10390 : the stack, determine the maximum runtime displacement that
10391 : matches up with the aligned frame. */
10392 1516637 : if (stack_realign_drap)
10393 8486 : m->fs.red_zone_offset -= (crtl->stack_alignment_needed / BITS_PER_UNIT
10394 4243 : + UNITS_PER_WORD);
10395 : }
10396 :
10397 1694228 : HOST_WIDE_INT reg_save_offset = frame.reg_save_offset;
10398 :
10399 : /* Special care must be taken for the normal return case of a function
10400 : using eh_return: the eax and edx registers are marked as saved, but
10401 : not restored along this path. Adjust the save location to match. */
10402 1694228 : if (crtl->calls_eh_return && style != 2)
10403 37 : reg_save_offset -= 2 * UNITS_PER_WORD;
10404 :
10405 : /* EH_RETURN requires the use of moves to function properly. */
10406 1694228 : if (crtl->calls_eh_return)
10407 : restore_regs_via_mov = true;
10408 : /* SEH requires the use of pops to identify the epilogue. */
10409 1694170 : else if (TARGET_SEH)
10410 : restore_regs_via_mov = false;
10411 : /* If we already save reg with pushp, don't use move at epilogue. */
10412 1694170 : else if (m->fs.apx_ppx_used)
10413 : restore_regs_via_mov = false;
10414 : /* If we're only restoring one register and sp cannot be used then
10415 : using a move instruction to restore the register since it's
10416 : less work than reloading sp and popping the register. */
10417 1694082 : else if (!sp_valid_at (frame.hfp_save_offset) && frame.nregs <= 1)
10418 : restore_regs_via_mov = true;
10419 1632298 : else if (crtl->shrink_wrapped_separate
10420 1578776 : || (TARGET_EPILOGUE_USING_MOVE
10421 58252 : && cfun->machine->use_fast_prologue_epilogue
10422 58196 : && (frame.nregs > 1
10423 58182 : || m->fs.sp_offset != reg_save_offset)))
10424 : restore_regs_via_mov = true;
10425 1578541 : else if (frame_pointer_needed
10426 434238 : && !frame.nregs
10427 337156 : && m->fs.sp_offset != reg_save_offset)
10428 : restore_regs_via_mov = true;
10429 1421813 : else if (frame_pointer_needed
10430 277510 : && TARGET_USE_LEAVE
10431 277435 : && cfun->machine->use_fast_prologue_epilogue
10432 220335 : && frame.nregs == 1)
10433 : restore_regs_via_mov = true;
10434 : else
10435 1694228 : restore_regs_via_mov = false;
10436 :
10437 1694228 : if (crtl->shrink_wrapped_separate)
10438 53553 : gcc_assert (restore_regs_via_mov);
10439 :
10440 1640675 : if (restore_regs_via_mov || frame.nsseregs)
10441 : {
10442 : /* Ensure that the entire register save area is addressable via
10443 : the stack pointer, if we will restore SSE regs via sp. */
10444 337457 : if (TARGET_64BIT
10445 324876 : && m->fs.sp_offset > 0x7fffffff
10446 23 : && sp_valid_at (frame.stack_realign_offset + 1)
10447 337479 : && (frame.nsseregs + frame.nregs) != 0)
10448 : {
10449 6 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
10450 6 : GEN_INT (m->fs.sp_offset
10451 : - frame.sse_reg_save_offset),
10452 : style,
10453 6 : m->fs.cfa_reg == stack_pointer_rtx);
10454 : }
10455 : }
10456 :
10457 : /* If there are any SSE registers to restore, then we have to do it
10458 : via moves, since there's obviously no pop for SSE regs. */
10459 1694228 : if (frame.nsseregs)
10460 33952 : ix86_emit_restore_sse_regs_using_mov (frame.sse_reg_save_offset,
10461 : style == 2);
10462 :
10463 1694228 : if (m->call_ms2sysv)
10464 : {
10465 7621 : int pop_incoming_args = crtl->args.pops_args && crtl->args.size;
10466 :
10467 : /* We cannot use a tail-call for the stub if:
10468 : 1. We have to pop incoming args,
10469 : 2. We have additional int regs to restore, or
10470 : 3. A sibling call will be the tail-call, or
10471 : 4. We are emitting an eh_return_internal epilogue.
10472 :
10473 : TODO: Item 4 has not yet tested!
10474 :
10475 : If any of the above are true, we will call the stub rather than
10476 : jump to it. */
10477 7621 : restore_stub_is_tail = !(pop_incoming_args || frame.nregs || style != 1);
10478 7621 : ix86_emit_outlined_ms2sysv_restore (frame, !restore_stub_is_tail, style);
10479 : }
10480 :
10481 : /* If using out-of-line stub that is a tail-call, then...*/
10482 1694228 : if (m->call_ms2sysv && restore_stub_is_tail)
10483 : {
10484 : /* TODO: parinoid tests. (remove eventually) */
10485 1115 : gcc_assert (m->fs.sp_valid);
10486 1115 : gcc_assert (!m->fs.sp_realigned);
10487 1115 : gcc_assert (!m->fs.fp_valid);
10488 1115 : gcc_assert (!m->fs.realigned);
10489 1115 : gcc_assert (m->fs.sp_offset == UNITS_PER_WORD);
10490 1115 : gcc_assert (!crtl->drap_reg);
10491 1115 : gcc_assert (!frame.nregs);
10492 1115 : gcc_assert (!crtl->shrink_wrapped_separate);
10493 : }
10494 1693113 : else if (restore_regs_via_mov)
10495 : {
10496 302745 : rtx t;
10497 :
10498 302745 : if (frame.nregs)
10499 99159 : ix86_emit_restore_regs_using_mov (reg_save_offset, style == 2);
10500 :
10501 : /* eh_return epilogues need %ecx added to the stack pointer. */
10502 302745 : if (style == 2)
10503 : {
10504 37 : rtx sa = EH_RETURN_STACKADJ_RTX;
10505 29 : rtx_insn *insn;
10506 :
10507 29 : gcc_assert (!crtl->shrink_wrapped_separate);
10508 :
10509 : /* Stack realignment doesn't work with eh_return. */
10510 29 : if (crtl->stack_realign_needed)
10511 0 : sorry ("Stack realignment not supported with "
10512 : "%<__builtin_eh_return%>");
10513 :
10514 : /* regparm nested functions don't work with eh_return. */
10515 29 : if (ix86_static_chain_on_stack)
10516 0 : sorry ("regparm nested function not supported with "
10517 : "%<__builtin_eh_return%>");
10518 :
10519 29 : if (frame_pointer_needed)
10520 : {
10521 35 : t = gen_rtx_PLUS (Pmode, hard_frame_pointer_rtx, sa);
10522 43 : t = plus_constant (Pmode, t, m->fs.fp_offset - UNITS_PER_WORD);
10523 27 : emit_insn (gen_rtx_SET (sa, t));
10524 :
10525 : /* NB: eh_return epilogues must restore the frame pointer
10526 : in word_mode since the upper 32 bits of RBP register
10527 : can have any values. */
10528 27 : t = gen_frame_mem (word_mode, hard_frame_pointer_rtx);
10529 27 : rtx frame_reg = gen_rtx_REG (word_mode,
10530 : HARD_FRAME_POINTER_REGNUM);
10531 27 : insn = emit_move_insn (frame_reg, t);
10532 :
10533 : /* Note that we use SA as a temporary CFA, as the return
10534 : address is at the proper place relative to it. We
10535 : pretend this happens at the FP restore insn because
10536 : prior to this insn the FP would be stored at the wrong
10537 : offset relative to SA, and after this insn we have no
10538 : other reasonable register to use for the CFA. We don't
10539 : bother resetting the CFA to the SP for the duration of
10540 : the return insn, unless the control flow instrumentation
10541 : is done. In this case the SP is used later and we have
10542 : to reset CFA to SP. */
10543 27 : add_reg_note (insn, REG_CFA_DEF_CFA,
10544 35 : plus_constant (Pmode, sa, UNITS_PER_WORD));
10545 27 : ix86_add_queued_cfa_restore_notes (insn);
10546 27 : add_reg_note (insn, REG_CFA_RESTORE, frame_reg);
10547 27 : RTX_FRAME_RELATED_P (insn) = 1;
10548 :
10549 27 : m->fs.cfa_reg = sa;
10550 27 : m->fs.cfa_offset = UNITS_PER_WORD;
10551 27 : m->fs.fp_valid = false;
10552 :
10553 27 : pro_epilogue_adjust_stack (stack_pointer_rtx, sa,
10554 : const0_rtx, style,
10555 27 : flag_cf_protection);
10556 : }
10557 : else
10558 : {
10559 2 : t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, sa);
10560 2 : t = plus_constant (Pmode, t, m->fs.sp_offset - UNITS_PER_WORD);
10561 2 : insn = emit_insn (gen_rtx_SET (stack_pointer_rtx, t));
10562 2 : ix86_add_queued_cfa_restore_notes (insn);
10563 :
10564 2 : gcc_assert (m->fs.cfa_reg == stack_pointer_rtx);
10565 2 : if (m->fs.cfa_offset != UNITS_PER_WORD)
10566 : {
10567 2 : m->fs.cfa_offset = UNITS_PER_WORD;
10568 2 : add_reg_note (insn, REG_CFA_DEF_CFA,
10569 2 : plus_constant (Pmode, stack_pointer_rtx,
10570 2 : UNITS_PER_WORD));
10571 2 : RTX_FRAME_RELATED_P (insn) = 1;
10572 : }
10573 : }
10574 29 : m->fs.sp_offset = UNITS_PER_WORD;
10575 29 : m->fs.sp_valid = true;
10576 29 : m->fs.sp_realigned = false;
10577 : }
10578 : }
10579 : else
10580 : {
10581 : /* SEH requires that the function end with (1) a stack adjustment
10582 : if necessary, (2) a sequence of pops, and (3) a return or
10583 : jump instruction. Prevent insns from the function body from
10584 : being scheduled into this sequence. */
10585 1390368 : if (TARGET_SEH)
10586 : {
10587 : /* Prevent a catch region from being adjacent to the standard
10588 : epilogue sequence. Unfortunately neither crtl->uses_eh_lsda
10589 : nor several other flags that would be interesting to test are
10590 : set up yet. */
10591 : if (flag_non_call_exceptions)
10592 : emit_insn (gen_nops (const1_rtx));
10593 : else
10594 : emit_insn (gen_blockage ());
10595 : }
10596 :
10597 : /* First step is to deallocate the stack frame so that we can
10598 : pop the registers. If the stack pointer was realigned, it needs
10599 : to be restored now. Also do it on SEH target for very large
10600 : frame as the emitted instructions aren't allowed by the ABI
10601 : in epilogues. */
10602 1390368 : if (!m->fs.sp_valid || m->fs.sp_realigned
10603 : || (TARGET_SEH
10604 : && (m->fs.sp_offset - reg_save_offset
10605 : >= SEH_MAX_FRAME_SIZE)))
10606 : {
10607 29692 : pro_epilogue_adjust_stack (stack_pointer_rtx, hard_frame_pointer_rtx,
10608 29692 : GEN_INT (m->fs.fp_offset
10609 : - reg_save_offset),
10610 : style, false);
10611 : }
10612 1360676 : else if (m->fs.sp_offset != reg_save_offset)
10613 : {
10614 626202 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
10615 : GEN_INT (m->fs.sp_offset
10616 : - reg_save_offset),
10617 : style,
10618 626202 : m->fs.cfa_reg == stack_pointer_rtx);
10619 : }
10620 :
10621 1390368 : if (TARGET_APX_PUSH2POP2
10622 578 : && ix86_can_use_push2pop2 ()
10623 1390944 : && m->func_type == TYPE_NORMAL)
10624 575 : ix86_emit_restore_regs_using_pop2 ();
10625 : else
10626 1389793 : ix86_emit_restore_regs_using_pop (TARGET_APX_PPX);
10627 : }
10628 :
10629 : /* If we used a stack pointer and haven't already got rid of it,
10630 : then do so now. */
10631 1694228 : if (m->fs.fp_valid)
10632 : {
10633 : /* If the stack pointer is valid and pointing at the frame
10634 : pointer store address, then we only need a pop. */
10635 497660 : if (sp_valid_at (frame.hfp_save_offset)
10636 497660 : && m->fs.sp_offset == frame.hfp_save_offset)
10637 246213 : ix86_emit_restore_reg_using_pop (hard_frame_pointer_rtx);
10638 : /* Leave results in shorter dependency chains on CPUs that are
10639 : able to grok it fast. */
10640 251447 : else if (TARGET_USE_LEAVE
10641 12 : || optimize_bb_for_size_p (EXIT_BLOCK_PTR_FOR_FN (cfun))
10642 251459 : || !cfun->machine->use_fast_prologue_epilogue)
10643 251435 : ix86_emit_leave (NULL);
10644 : else
10645 : {
10646 12 : pro_epilogue_adjust_stack (stack_pointer_rtx,
10647 : hard_frame_pointer_rtx,
10648 : const0_rtx, style, !using_drap);
10649 12 : ix86_emit_restore_reg_using_pop (hard_frame_pointer_rtx);
10650 : }
10651 : }
10652 :
10653 1694228 : if (using_drap)
10654 : {
10655 7108 : int param_ptr_offset = UNITS_PER_WORD;
10656 7108 : rtx_insn *insn;
10657 :
10658 7108 : gcc_assert (stack_realign_drap);
10659 :
10660 7108 : if (ix86_static_chain_on_stack)
10661 0 : param_ptr_offset += UNITS_PER_WORD;
10662 7108 : if (!call_used_or_fixed_reg_p (REGNO (crtl->drap_reg)))
10663 232 : param_ptr_offset += UNITS_PER_WORD;
10664 :
10665 7413 : insn = emit_insn (gen_rtx_SET
10666 : (stack_pointer_rtx,
10667 : plus_constant (Pmode, crtl->drap_reg,
10668 : -param_ptr_offset)));
10669 7108 : m->fs.cfa_reg = stack_pointer_rtx;
10670 7108 : m->fs.cfa_offset = param_ptr_offset;
10671 7108 : m->fs.sp_offset = param_ptr_offset;
10672 7108 : m->fs.realigned = false;
10673 :
10674 7413 : add_reg_note (insn, REG_CFA_DEF_CFA,
10675 7108 : plus_constant (Pmode, stack_pointer_rtx,
10676 7108 : param_ptr_offset));
10677 7108 : RTX_FRAME_RELATED_P (insn) = 1;
10678 :
10679 7108 : if (!call_used_or_fixed_reg_p (REGNO (crtl->drap_reg)))
10680 232 : ix86_emit_restore_reg_using_pop (crtl->drap_reg);
10681 : }
10682 :
10683 : /* At this point the stack pointer must be valid, and we must have
10684 : restored all of the registers. We may not have deallocated the
10685 : entire stack frame. We've delayed this until now because it may
10686 : be possible to merge the local stack deallocation with the
10687 : deallocation forced by ix86_static_chain_on_stack. */
10688 1694228 : gcc_assert (m->fs.sp_valid);
10689 1694228 : gcc_assert (!m->fs.sp_realigned);
10690 1694228 : gcc_assert (!m->fs.fp_valid);
10691 1694228 : gcc_assert (!m->fs.realigned);
10692 1830021 : if (m->fs.sp_offset != UNITS_PER_WORD)
10693 : {
10694 51255 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
10695 : GEN_INT (m->fs.sp_offset - UNITS_PER_WORD),
10696 : style, true);
10697 : }
10698 : else
10699 1642973 : ix86_add_queued_cfa_restore_notes (get_last_insn ());
10700 :
10701 : /* Sibcall epilogues don't want a return instruction. */
10702 1694228 : if (style == 0)
10703 : {
10704 127880 : m->fs = frame_state_save;
10705 127880 : return;
10706 : }
10707 :
10708 1566348 : if (cfun->machine->func_type != TYPE_NORMAL)
10709 120 : emit_jump_insn (gen_interrupt_return ());
10710 1566228 : else if (crtl->args.pops_args && crtl->args.size)
10711 : {
10712 25976 : rtx popc = GEN_INT (crtl->args.pops_args);
10713 :
10714 : /* i386 can only pop 64K bytes. If asked to pop more, pop return
10715 : address, do explicit add, and jump indirectly to the caller. */
10716 :
10717 25976 : if (crtl->args.pops_args >= 65536)
10718 : {
10719 0 : rtx ecx = gen_rtx_REG (SImode, CX_REG);
10720 0 : rtx_insn *insn;
10721 :
10722 : /* There is no "pascal" calling convention in any 64bit ABI. */
10723 0 : gcc_assert (!TARGET_64BIT);
10724 :
10725 0 : insn = emit_insn (gen_pop (ecx));
10726 0 : m->fs.cfa_offset -= UNITS_PER_WORD;
10727 0 : m->fs.sp_offset -= UNITS_PER_WORD;
10728 :
10729 0 : rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
10730 0 : x = gen_rtx_SET (stack_pointer_rtx, x);
10731 0 : add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
10732 0 : add_reg_note (insn, REG_CFA_REGISTER, gen_rtx_SET (ecx, pc_rtx));
10733 0 : RTX_FRAME_RELATED_P (insn) = 1;
10734 :
10735 0 : pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
10736 : popc, -1, true);
10737 0 : emit_jump_insn (gen_simple_return_indirect_internal (ecx));
10738 : }
10739 : else
10740 25976 : emit_jump_insn (gen_simple_return_pop_internal (popc));
10741 : }
10742 1540252 : else if (!m->call_ms2sysv || !restore_stub_is_tail)
10743 : {
10744 : /* In case of return from EH a simple return cannot be used
10745 : as a return address will be compared with a shadow stack
10746 : return address. Use indirect jump instead. */
10747 1539137 : if (style == 2 && flag_cf_protection)
10748 : {
10749 : /* Register used in indirect jump must be in word_mode. But
10750 : Pmode may not be the same as word_mode for x32. */
10751 17 : rtx ecx = gen_rtx_REG (word_mode, CX_REG);
10752 17 : rtx_insn *insn;
10753 :
10754 17 : insn = emit_insn (gen_pop (ecx));
10755 17 : m->fs.cfa_offset -= UNITS_PER_WORD;
10756 17 : m->fs.sp_offset -= UNITS_PER_WORD;
10757 :
10758 33 : rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
10759 17 : x = gen_rtx_SET (stack_pointer_rtx, x);
10760 17 : add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
10761 17 : add_reg_note (insn, REG_CFA_REGISTER, gen_rtx_SET (ecx, pc_rtx));
10762 17 : RTX_FRAME_RELATED_P (insn) = 1;
10763 :
10764 17 : emit_jump_insn (gen_simple_return_indirect_internal (ecx));
10765 17 : }
10766 : else
10767 1539120 : emit_jump_insn (gen_simple_return_internal ());
10768 : }
10769 :
10770 : /* Restore the state back to the state from the prologue,
10771 : so that it's correct for the next epilogue. */
10772 1566348 : m->fs = frame_state_save;
10773 : }
10774 :
10775 : /* Reset from the function's potential modifications. */
10776 :
10777 : static void
10778 1526121 : ix86_output_function_epilogue (FILE *file ATTRIBUTE_UNUSED)
10779 : {
10780 1526121 : if (pic_offset_table_rtx
10781 1526121 : && !ix86_use_pseudo_pic_reg ())
10782 0 : SET_REGNO (pic_offset_table_rtx, REAL_PIC_OFFSET_TABLE_REGNUM);
10783 :
10784 1526121 : if (TARGET_MACHO)
10785 : {
10786 : rtx_insn *insn = get_last_insn ();
10787 : rtx_insn *deleted_debug_label = NULL;
10788 :
10789 : /* Mach-O doesn't support labels at the end of objects, so if
10790 : it looks like we might want one, take special action.
10791 : First, collect any sequence of deleted debug labels. */
10792 : while (insn
10793 : && NOTE_P (insn)
10794 : && NOTE_KIND (insn) != NOTE_INSN_DELETED_LABEL)
10795 : {
10796 : /* Don't insert a nop for NOTE_INSN_DELETED_DEBUG_LABEL
10797 : notes only, instead set their CODE_LABEL_NUMBER to -1,
10798 : otherwise there would be code generation differences
10799 : in between -g and -g0. */
10800 : if (NOTE_P (insn) && NOTE_KIND (insn)
10801 : == NOTE_INSN_DELETED_DEBUG_LABEL)
10802 : deleted_debug_label = insn;
10803 : insn = PREV_INSN (insn);
10804 : }
10805 :
10806 : /* If we have:
10807 : label:
10808 : barrier
10809 : then this needs to be detected, so skip past the barrier. */
10810 :
10811 : if (insn && BARRIER_P (insn))
10812 : insn = PREV_INSN (insn);
10813 :
10814 : /* Up to now we've only seen notes or barriers. */
10815 : if (insn)
10816 : {
10817 : if (LABEL_P (insn)
10818 : || (NOTE_P (insn)
10819 : && NOTE_KIND (insn) == NOTE_INSN_DELETED_LABEL))
10820 : /* Trailing label. */
10821 : fputs ("\tnop\n", file);
10822 : else if (cfun && ! cfun->is_thunk)
10823 : {
10824 : /* See if we have a completely empty function body, skipping
10825 : the special case of the picbase thunk emitted as asm. */
10826 : while (insn && ! INSN_P (insn))
10827 : insn = PREV_INSN (insn);
10828 : /* If we don't find any insns, we've got an empty function body;
10829 : I.e. completely empty - without a return or branch. This is
10830 : taken as the case where a function body has been removed
10831 : because it contains an inline __builtin_unreachable(). GCC
10832 : declares that reaching __builtin_unreachable() means UB so
10833 : we're not obliged to do anything special; however, we want
10834 : non-zero-sized function bodies. To meet this, and help the
10835 : user out, let's trap the case. */
10836 : if (insn == NULL)
10837 : fputs ("\tud2\n", file);
10838 : }
10839 : }
10840 : else if (deleted_debug_label)
10841 : for (insn = deleted_debug_label; insn; insn = NEXT_INSN (insn))
10842 : if (NOTE_KIND (insn) == NOTE_INSN_DELETED_DEBUG_LABEL)
10843 : CODE_LABEL_NUMBER (insn) = -1;
10844 : }
10845 1526121 : }
10846 :
10847 : /* Implement TARGET_ASM_PRINT_PATCHABLE_FUNCTION_ENTRY. */
10848 :
10849 : void
10850 59 : ix86_print_patchable_function_entry (FILE *file,
10851 : unsigned HOST_WIDE_INT patch_area_size,
10852 : bool record_p)
10853 : {
10854 59 : if (cfun->machine->function_label_emitted)
10855 : {
10856 : /* NB: When ix86_print_patchable_function_entry is called after
10857 : function table has been emitted, we have inserted or queued
10858 : a pseudo UNSPECV_PATCHABLE_AREA instruction at the proper
10859 : place. There is nothing to do here. */
10860 : return;
10861 : }
10862 :
10863 8 : default_print_patchable_function_entry (file, patch_area_size,
10864 : record_p);
10865 : }
10866 :
10867 : /* Output patchable area. NB: default_print_patchable_function_entry
10868 : isn't available in i386.md. */
10869 :
10870 : void
10871 51 : ix86_output_patchable_area (unsigned int patch_area_size,
10872 : bool record_p)
10873 : {
10874 51 : default_print_patchable_function_entry (asm_out_file,
10875 : patch_area_size,
10876 : record_p);
10877 51 : }
10878 :
10879 : /* Return a scratch register to use in the split stack prologue. The
10880 : split stack prologue is used for -fsplit-stack. It is the first
10881 : instructions in the function, even before the regular prologue.
10882 : The scratch register can be any caller-saved register which is not
10883 : used for parameters or for the static chain. */
10884 :
10885 : static unsigned int
10886 24529 : split_stack_prologue_scratch_regno (void)
10887 : {
10888 24529 : if (TARGET_64BIT)
10889 : return R11_REG;
10890 : else
10891 : {
10892 6862 : bool is_fastcall, is_thiscall;
10893 6862 : int regparm;
10894 :
10895 6862 : is_fastcall = (lookup_attribute ("fastcall",
10896 6862 : TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
10897 : != NULL);
10898 6862 : is_thiscall = (lookup_attribute ("thiscall",
10899 6862 : TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
10900 : != NULL);
10901 6862 : regparm = ix86_function_regparm (TREE_TYPE (cfun->decl), cfun->decl);
10902 :
10903 6862 : if (is_fastcall)
10904 : {
10905 0 : if (DECL_STATIC_CHAIN (cfun->decl))
10906 : {
10907 0 : sorry ("%<-fsplit-stack%> does not support fastcall with "
10908 : "nested function");
10909 0 : return INVALID_REGNUM;
10910 : }
10911 : return AX_REG;
10912 : }
10913 6862 : else if (is_thiscall)
10914 : {
10915 0 : if (!DECL_STATIC_CHAIN (cfun->decl))
10916 : return DX_REG;
10917 0 : return AX_REG;
10918 : }
10919 6862 : else if (regparm < 3)
10920 : {
10921 6862 : if (!DECL_STATIC_CHAIN (cfun->decl))
10922 : return CX_REG;
10923 : else
10924 : {
10925 449 : if (regparm >= 2)
10926 : {
10927 0 : sorry ("%<-fsplit-stack%> does not support 2 register "
10928 : "parameters for a nested function");
10929 0 : return INVALID_REGNUM;
10930 : }
10931 : return DX_REG;
10932 : }
10933 : }
10934 : else
10935 : {
10936 : /* FIXME: We could make this work by pushing a register
10937 : around the addition and comparison. */
10938 0 : sorry ("%<-fsplit-stack%> does not support 3 register parameters");
10939 0 : return INVALID_REGNUM;
10940 : }
10941 : }
10942 : }
10943 :
10944 : /* A SYMBOL_REF for the function which allocates new stackspace for
10945 : -fsplit-stack. */
10946 :
10947 : static GTY(()) rtx split_stack_fn;
10948 :
10949 : /* A SYMBOL_REF for the more stack function when using the large model. */
10950 :
10951 : static GTY(()) rtx split_stack_fn_large;
10952 :
10953 : /* Return location of the stack guard value in the TLS block. */
10954 :
10955 : rtx
10956 259980 : ix86_split_stack_guard (void)
10957 : {
10958 259980 : int offset;
10959 259980 : addr_space_t as = DEFAULT_TLS_SEG_REG;
10960 259980 : rtx r;
10961 :
10962 259980 : gcc_assert (flag_split_stack);
10963 :
10964 : #ifdef TARGET_THREAD_SPLIT_STACK_OFFSET
10965 259980 : offset = TARGET_THREAD_SPLIT_STACK_OFFSET;
10966 : #else
10967 : gcc_unreachable ();
10968 : #endif
10969 :
10970 259980 : r = GEN_INT (offset);
10971 357953 : r = gen_const_mem (Pmode, r);
10972 259980 : set_mem_addr_space (r, as);
10973 :
10974 259980 : return r;
10975 : }
10976 :
10977 : /* Handle -fsplit-stack. These are the first instructions in the
10978 : function, even before the regular prologue. */
10979 :
10980 : void
10981 259970 : ix86_expand_split_stack_prologue (void)
10982 : {
10983 259970 : HOST_WIDE_INT allocate;
10984 259970 : unsigned HOST_WIDE_INT args_size;
10985 259970 : rtx_code_label *label;
10986 259970 : rtx limit, current, allocate_rtx, call_fusage;
10987 259970 : rtx_insn *call_insn;
10988 259970 : unsigned int scratch_regno = INVALID_REGNUM;
10989 259970 : rtx scratch_reg = NULL_RTX;
10990 259970 : rtx_code_label *varargs_label = NULL;
10991 259970 : rtx fn;
10992 :
10993 259970 : gcc_assert (flag_split_stack && reload_completed);
10994 :
10995 259970 : ix86_finalize_stack_frame_flags ();
10996 259970 : struct ix86_frame &frame = cfun->machine->frame;
10997 259970 : allocate = frame.stack_pointer_offset - INCOMING_FRAME_SP_OFFSET;
10998 :
10999 : /* This is the label we will branch to if we have enough stack
11000 : space. We expect the basic block reordering pass to reverse this
11001 : branch if optimizing, so that we branch in the unlikely case. */
11002 259970 : label = gen_label_rtx ();
11003 :
11004 : /* We need to compare the stack pointer minus the frame size with
11005 : the stack boundary in the TCB. The stack boundary always gives
11006 : us SPLIT_STACK_AVAILABLE bytes, so if we need less than that we
11007 : can compare directly. Otherwise we need to do an addition. */
11008 :
11009 259970 : limit = ix86_split_stack_guard ();
11010 :
11011 259970 : if (allocate >= SPLIT_STACK_AVAILABLE
11012 235604 : || flag_force_indirect_call)
11013 : {
11014 24381 : scratch_regno = split_stack_prologue_scratch_regno ();
11015 24381 : if (scratch_regno == INVALID_REGNUM)
11016 0 : return;
11017 : }
11018 :
11019 259970 : if (allocate >= SPLIT_STACK_AVAILABLE)
11020 : {
11021 24366 : rtx offset;
11022 :
11023 : /* We need a scratch register to hold the stack pointer minus
11024 : the required frame size. Since this is the very start of the
11025 : function, the scratch register can be any caller-saved
11026 : register which is not used for parameters. */
11027 24366 : offset = GEN_INT (- allocate);
11028 :
11029 31174 : scratch_reg = gen_rtx_REG (Pmode, scratch_regno);
11030 24366 : if (!TARGET_64BIT || x86_64_immediate_operand (offset, Pmode))
11031 : {
11032 : /* We don't use gen_add in this case because it will
11033 : want to split to lea, but when not optimizing the insn
11034 : will not be split after this point. */
11035 31174 : emit_insn (gen_rtx_SET (scratch_reg,
11036 : gen_rtx_PLUS (Pmode, stack_pointer_rtx,
11037 : offset)));
11038 : }
11039 : else
11040 : {
11041 0 : emit_move_insn (scratch_reg, offset);
11042 0 : emit_insn (gen_add2_insn (scratch_reg, stack_pointer_rtx));
11043 : }
11044 : current = scratch_reg;
11045 : }
11046 : else
11047 235604 : current = stack_pointer_rtx;
11048 :
11049 259970 : ix86_expand_branch (GEU, current, limit, label);
11050 259970 : rtx_insn *jump_insn = get_last_insn ();
11051 259970 : JUMP_LABEL (jump_insn) = label;
11052 :
11053 : /* Mark the jump as very likely to be taken. */
11054 259970 : add_reg_br_prob_note (jump_insn, profile_probability::very_likely ());
11055 :
11056 259970 : if (split_stack_fn == NULL_RTX)
11057 : {
11058 5451 : split_stack_fn = gen_rtx_SYMBOL_REF (Pmode, "__morestack");
11059 4347 : SYMBOL_REF_FLAGS (split_stack_fn) |= SYMBOL_FLAG_LOCAL;
11060 : }
11061 259970 : fn = split_stack_fn;
11062 :
11063 : /* Get more stack space. We pass in the desired stack space and the
11064 : size of the arguments to copy to the new stack. In 32-bit mode
11065 : we push the parameters; __morestack will return on a new stack
11066 : anyhow. In 64-bit mode we pass the parameters in r10 and
11067 : r11. */
11068 259970 : allocate_rtx = GEN_INT (allocate);
11069 259970 : args_size = crtl->args.size >= 0 ? (HOST_WIDE_INT) crtl->args.size : 0;
11070 259970 : call_fusage = NULL_RTX;
11071 259970 : rtx pop = NULL_RTX;
11072 259970 : if (TARGET_64BIT)
11073 : {
11074 161997 : rtx reg10, reg11;
11075 :
11076 161997 : reg10 = gen_rtx_REG (DImode, R10_REG);
11077 161997 : reg11 = gen_rtx_REG (DImode, R11_REG);
11078 :
11079 : /* If this function uses a static chain, it will be in %r10.
11080 : Preserve it across the call to __morestack. */
11081 161997 : if (DECL_STATIC_CHAIN (cfun->decl))
11082 : {
11083 7505 : rtx rax;
11084 :
11085 7505 : rax = gen_rtx_REG (word_mode, AX_REG);
11086 7505 : emit_move_insn (rax, gen_rtx_REG (word_mode, R10_REG));
11087 7505 : use_reg (&call_fusage, rax);
11088 : }
11089 :
11090 161997 : if (flag_force_indirect_call
11091 161982 : || ix86_cmodel == CM_LARGE || ix86_cmodel == CM_LARGE_PIC)
11092 : {
11093 16 : HOST_WIDE_INT argval;
11094 :
11095 16 : if (split_stack_fn_large == NULL_RTX)
11096 : {
11097 7 : split_stack_fn_large
11098 7 : = gen_rtx_SYMBOL_REF (Pmode, "__morestack_large_model");
11099 7 : SYMBOL_REF_FLAGS (split_stack_fn_large) |= SYMBOL_FLAG_LOCAL;
11100 : }
11101 :
11102 16 : fn = split_stack_fn_large;
11103 :
11104 16 : if (ix86_cmodel == CM_LARGE_PIC)
11105 : {
11106 3 : rtx_code_label *label;
11107 3 : rtx x;
11108 :
11109 3 : gcc_assert (Pmode == DImode);
11110 :
11111 3 : label = gen_label_rtx ();
11112 3 : emit_label (label);
11113 3 : LABEL_PRESERVE_P (label) = 1;
11114 3 : emit_insn (gen_set_rip_rex64 (reg10, label));
11115 3 : emit_insn (gen_set_got_offset_rex64 (reg11, label));
11116 3 : emit_insn (gen_add2_insn (reg10, reg11));
11117 3 : x = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, fn), UNSPEC_GOT);
11118 3 : x = gen_rtx_CONST (Pmode, x);
11119 3 : emit_move_insn (reg11, x);
11120 3 : x = gen_rtx_PLUS (Pmode, reg10, reg11);
11121 3 : x = gen_const_mem (Pmode, x);
11122 3 : fn = copy_to_suggested_reg (x, reg11, Pmode);
11123 : }
11124 13 : else if (ix86_cmodel == CM_LARGE)
11125 1 : fn = copy_to_suggested_reg (fn, reg11, Pmode);
11126 :
11127 : /* When using the large model we need to load the address
11128 : into a register, and we've run out of registers. So we
11129 : switch to a different calling convention, and we call a
11130 : different function: __morestack_large. We pass the
11131 : argument size in the upper 32 bits of r10 and pass the
11132 : frame size in the lower 32 bits. */
11133 16 : gcc_assert ((allocate & HOST_WIDE_INT_C (0xffffffff)) == allocate);
11134 16 : gcc_assert ((args_size & 0xffffffff) == args_size);
11135 :
11136 16 : argval = ((args_size << 16) << 16) + allocate;
11137 16 : emit_move_insn (reg10, GEN_INT (argval));
11138 16 : }
11139 : else
11140 : {
11141 161981 : emit_move_insn (reg10, allocate_rtx);
11142 161981 : emit_move_insn (reg11, GEN_INT (args_size));
11143 161981 : use_reg (&call_fusage, reg11);
11144 : }
11145 :
11146 161997 : use_reg (&call_fusage, reg10);
11147 : }
11148 : else
11149 : {
11150 97973 : if (flag_force_indirect_call && flag_pic)
11151 : {
11152 0 : rtx x;
11153 :
11154 0 : gcc_assert (Pmode == SImode);
11155 :
11156 0 : scratch_reg = gen_rtx_REG (Pmode, scratch_regno);
11157 :
11158 0 : emit_insn (gen_set_got (scratch_reg));
11159 0 : x = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, split_stack_fn),
11160 : UNSPEC_GOT);
11161 0 : x = gen_rtx_CONST (Pmode, x);
11162 0 : x = gen_rtx_PLUS (Pmode, scratch_reg, x);
11163 0 : x = gen_const_mem (Pmode, x);
11164 0 : fn = copy_to_suggested_reg (x, scratch_reg, Pmode);
11165 : }
11166 :
11167 97973 : rtx_insn *insn = emit_insn (gen_push (GEN_INT (args_size)));
11168 195946 : add_reg_note (insn, REG_ARGS_SIZE, GEN_INT (UNITS_PER_WORD));
11169 97973 : insn = emit_insn (gen_push (allocate_rtx));
11170 195946 : add_reg_note (insn, REG_ARGS_SIZE, GEN_INT (2 * UNITS_PER_WORD));
11171 195946 : pop = GEN_INT (2 * UNITS_PER_WORD);
11172 : }
11173 :
11174 259970 : if (flag_force_indirect_call && !register_operand (fn, VOIDmode))
11175 : {
11176 12 : scratch_reg = gen_rtx_REG (word_mode, scratch_regno);
11177 :
11178 12 : if (GET_MODE (fn) != word_mode)
11179 0 : fn = gen_rtx_ZERO_EXTEND (word_mode, fn);
11180 :
11181 12 : fn = copy_to_suggested_reg (fn, scratch_reg, word_mode);
11182 : }
11183 :
11184 259970 : call_insn = ix86_expand_call (NULL_RTX, gen_rtx_MEM (QImode, fn),
11185 259970 : GEN_INT (UNITS_PER_WORD), constm1_rtx,
11186 : pop, false);
11187 259970 : add_function_usage_to (call_insn, call_fusage);
11188 259970 : if (!TARGET_64BIT)
11189 97973 : add_reg_note (call_insn, REG_ARGS_SIZE, GEN_INT (0));
11190 : /* Indicate that this function can't jump to non-local gotos. */
11191 259970 : make_reg_eh_region_note_nothrow_nononlocal (call_insn);
11192 :
11193 : /* In order to make call/return prediction work right, we now need
11194 : to execute a return instruction. See
11195 : libgcc/config/i386/morestack.S for the details on how this works.
11196 :
11197 : For flow purposes gcc must not see this as a return
11198 : instruction--we need control flow to continue at the subsequent
11199 : label. Therefore, we use an unspec. */
11200 259970 : gcc_assert (crtl->args.pops_args < 65536);
11201 259970 : rtx_insn *ret_insn
11202 259970 : = emit_insn (gen_split_stack_return (GEN_INT (crtl->args.pops_args)));
11203 :
11204 259970 : if ((flag_cf_protection & CF_BRANCH))
11205 : {
11206 : /* Insert ENDBR since __morestack will jump back here via indirect
11207 : call. */
11208 21 : rtx cet_eb = gen_nop_endbr ();
11209 21 : emit_insn_after (cet_eb, ret_insn);
11210 : }
11211 :
11212 : /* If we are in 64-bit mode and this function uses a static chain,
11213 : we saved %r10 in %rax before calling _morestack. */
11214 259970 : if (TARGET_64BIT && DECL_STATIC_CHAIN (cfun->decl))
11215 7505 : emit_move_insn (gen_rtx_REG (word_mode, R10_REG),
11216 : gen_rtx_REG (word_mode, AX_REG));
11217 :
11218 : /* If this function calls va_start, we need to store a pointer to
11219 : the arguments on the old stack, because they may not have been
11220 : all copied to the new stack. At this point the old stack can be
11221 : found at the frame pointer value used by __morestack, because
11222 : __morestack has set that up before calling back to us. Here we
11223 : store that pointer in a scratch register, and in
11224 : ix86_expand_prologue we store the scratch register in a stack
11225 : slot. */
11226 259970 : if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
11227 : {
11228 12 : rtx frame_reg;
11229 12 : int words;
11230 :
11231 12 : scratch_regno = split_stack_prologue_scratch_regno ();
11232 16 : scratch_reg = gen_rtx_REG (Pmode, scratch_regno);
11233 16 : frame_reg = gen_rtx_REG (Pmode, BP_REG);
11234 :
11235 : /* 64-bit:
11236 : fp -> old fp value
11237 : return address within this function
11238 : return address of caller of this function
11239 : stack arguments
11240 : So we add three words to get to the stack arguments.
11241 :
11242 : 32-bit:
11243 : fp -> old fp value
11244 : return address within this function
11245 : first argument to __morestack
11246 : second argument to __morestack
11247 : return address of caller of this function
11248 : stack arguments
11249 : So we add five words to get to the stack arguments.
11250 : */
11251 12 : words = TARGET_64BIT ? 3 : 5;
11252 20 : emit_insn (gen_rtx_SET (scratch_reg,
11253 : plus_constant (Pmode, frame_reg,
11254 : words * UNITS_PER_WORD)));
11255 :
11256 12 : varargs_label = gen_label_rtx ();
11257 12 : emit_jump_insn (gen_jump (varargs_label));
11258 12 : JUMP_LABEL (get_last_insn ()) = varargs_label;
11259 :
11260 12 : emit_barrier ();
11261 : }
11262 :
11263 259970 : emit_label (label);
11264 259970 : LABEL_NUSES (label) = 1;
11265 :
11266 : /* If this function calls va_start, we now have to set the scratch
11267 : register for the case where we do not call __morestack. In this
11268 : case we need to set it based on the stack pointer. */
11269 259970 : if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
11270 : {
11271 20 : emit_insn (gen_rtx_SET (scratch_reg,
11272 : plus_constant (Pmode, stack_pointer_rtx,
11273 : UNITS_PER_WORD)));
11274 :
11275 12 : emit_label (varargs_label);
11276 12 : LABEL_NUSES (varargs_label) = 1;
11277 : }
11278 : }
11279 :
11280 : /* We may have to tell the dataflow pass that the split stack prologue
11281 : is initializing a scratch register. */
11282 :
11283 : static void
11284 16279703 : ix86_live_on_entry (bitmap regs)
11285 : {
11286 16279703 : if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
11287 : {
11288 124 : gcc_assert (flag_split_stack);
11289 124 : bitmap_set_bit (regs, split_stack_prologue_scratch_regno ());
11290 : }
11291 16279703 : }
11292 :
11293 : /* Extract the parts of an RTL expression that is a valid memory address
11294 : for an instruction. Return false if the structure of the address is
11295 : grossly off. */
11296 :
11297 : bool
11298 4366235458 : ix86_decompose_address (rtx addr, struct ix86_address *out)
11299 : {
11300 4366235458 : rtx base = NULL_RTX, index = NULL_RTX, disp = NULL_RTX;
11301 4366235458 : rtx base_reg, index_reg;
11302 4366235458 : HOST_WIDE_INT scale = 1;
11303 4366235458 : rtx scale_rtx = NULL_RTX;
11304 4366235458 : rtx tmp;
11305 4366235458 : addr_space_t seg = ADDR_SPACE_GENERIC;
11306 :
11307 : /* Allow zero-extended SImode addresses,
11308 : they will be emitted with addr32 prefix. */
11309 4366235458 : if (TARGET_64BIT && GET_MODE (addr) == DImode)
11310 : {
11311 2324731084 : if (GET_CODE (addr) == ZERO_EXTEND
11312 2076601 : && GET_MODE (XEXP (addr, 0)) == SImode)
11313 : {
11314 1970798 : addr = XEXP (addr, 0);
11315 1970798 : if (CONST_INT_P (addr))
11316 : return false;
11317 : }
11318 2322760286 : else if (GET_CODE (addr) == AND)
11319 : {
11320 2927190 : rtx mask = XEXP (addr, 1);
11321 2927190 : rtx shift_val;
11322 :
11323 2927190 : if (const_32bit_mask (mask, DImode)
11324 : /* For ASHIFT inside AND, combine will not generate
11325 : canonical zero-extend. Merge mask for AND and shift_count
11326 : to check if it is canonical zero-extend. */
11327 2927190 : || (CONST_INT_P (mask)
11328 1914552 : && GET_CODE (XEXP (addr, 0)) == ASHIFT
11329 156490 : && CONST_INT_P (shift_val = XEXP (XEXP (addr, 0), 1))
11330 153384 : && ((UINTVAL (mask)
11331 153384 : | ((HOST_WIDE_INT_1U << INTVAL (shift_val)) - 1))
11332 : == HOST_WIDE_INT_UC (0xffffffff))))
11333 : {
11334 83417 : addr = lowpart_subreg (SImode, XEXP (addr, 0), DImode);
11335 83417 : if (addr == NULL_RTX)
11336 : return false;
11337 :
11338 83417 : if (CONST_INT_P (addr))
11339 : return false;
11340 : }
11341 : }
11342 : }
11343 :
11344 : /* Allow SImode subregs of DImode addresses,
11345 : they will be emitted with addr32 prefix. */
11346 4366235458 : if (TARGET_64BIT && GET_MODE (addr) == SImode)
11347 : {
11348 17072860 : if (SUBREG_P (addr)
11349 245132 : && GET_MODE (SUBREG_REG (addr)) == DImode)
11350 : {
11351 217513 : addr = SUBREG_REG (addr);
11352 217513 : if (CONST_INT_P (addr))
11353 : return false;
11354 : }
11355 : }
11356 :
11357 4366235458 : if (REG_P (addr))
11358 : base = addr;
11359 : else if (SUBREG_P (addr))
11360 : {
11361 471921 : if (REG_P (SUBREG_REG (addr)))
11362 : base = addr;
11363 : else
11364 : return false;
11365 : }
11366 : else if (GET_CODE (addr) == PLUS)
11367 : {
11368 : rtx addends[4], op;
11369 : int n = 0, i;
11370 :
11371 : op = addr;
11372 3170490638 : do
11373 : {
11374 3170490638 : if (n >= 4)
11375 646788469 : return false;
11376 3170468795 : addends[n++] = XEXP (op, 1);
11377 3170468795 : op = XEXP (op, 0);
11378 : }
11379 3170468795 : while (GET_CODE (op) == PLUS);
11380 3108950988 : if (n >= 4)
11381 : return false;
11382 3108938751 : addends[n] = op;
11383 :
11384 8097573546 : for (i = n; i >= 0; --i)
11385 : {
11386 5635389184 : op = addends[i];
11387 5635389184 : switch (GET_CODE (op))
11388 : {
11389 60179930 : case MULT:
11390 60179930 : if (index)
11391 : return false;
11392 60139783 : index = XEXP (op, 0);
11393 60139783 : scale_rtx = XEXP (op, 1);
11394 60139783 : break;
11395 :
11396 12879487 : case ASHIFT:
11397 12879487 : if (index)
11398 : return false;
11399 12806996 : index = XEXP (op, 0);
11400 12806996 : tmp = XEXP (op, 1);
11401 12806996 : if (!CONST_INT_P (tmp))
11402 : return false;
11403 12792268 : scale = INTVAL (tmp);
11404 12792268 : if ((unsigned HOST_WIDE_INT) scale > 3)
11405 : return false;
11406 12334782 : scale = 1 << scale;
11407 12334782 : break;
11408 :
11409 827369 : case ZERO_EXTEND:
11410 827369 : op = XEXP (op, 0);
11411 827369 : if (GET_CODE (op) != UNSPEC)
11412 : return false;
11413 : /* FALLTHRU */
11414 :
11415 702229 : case UNSPEC:
11416 702229 : if (XINT (op, 1) == UNSPEC_TP
11417 694033 : && TARGET_TLS_DIRECT_SEG_REFS
11418 694033 : && seg == ADDR_SPACE_GENERIC)
11419 694033 : seg = DEFAULT_TLS_SEG_REG;
11420 : else
11421 : return false;
11422 : break;
11423 :
11424 692993 : case SUBREG:
11425 692993 : if (!REG_P (SUBREG_REG (op)))
11426 : return false;
11427 : /* FALLTHRU */
11428 :
11429 2529676493 : case REG:
11430 2529676493 : if (!base)
11431 : base = op;
11432 78844477 : else if (!index)
11433 : index = op;
11434 : else
11435 : return false;
11436 : break;
11437 :
11438 2386688205 : case CONST:
11439 2386688205 : case CONST_INT:
11440 2386688205 : case SYMBOL_REF:
11441 2386688205 : case LABEL_REF:
11442 2386688205 : if (disp)
11443 : return false;
11444 : disp = op;
11445 : break;
11446 :
11447 : default:
11448 : return false;
11449 : }
11450 : }
11451 : }
11452 : else if (GET_CODE (addr) == MULT)
11453 : {
11454 4107380 : index = XEXP (addr, 0); /* index*scale */
11455 4107380 : scale_rtx = XEXP (addr, 1);
11456 : }
11457 : else if (GET_CODE (addr) == ASHIFT)
11458 : {
11459 : /* We're called for lea too, which implements ashift on occasion. */
11460 3358440 : index = XEXP (addr, 0);
11461 3358440 : tmp = XEXP (addr, 1);
11462 3358440 : if (!CONST_INT_P (tmp))
11463 : return false;
11464 2966741 : scale = INTVAL (tmp);
11465 2966741 : if ((unsigned HOST_WIDE_INT) scale > 3)
11466 : return false;
11467 2201765 : scale = 1 << scale;
11468 : }
11469 : else
11470 : disp = addr; /* displacement */
11471 :
11472 2468493507 : if (index)
11473 : {
11474 147625119 : if (REG_P (index))
11475 : ;
11476 4428560 : else if (SUBREG_P (index)
11477 476601 : && REG_P (SUBREG_REG (index)))
11478 : ;
11479 : else
11480 : return false;
11481 : }
11482 :
11483 : /* Extract the integral value of scale. */
11484 3714260848 : if (scale_rtx)
11485 : {
11486 55316744 : if (!CONST_INT_P (scale_rtx))
11487 : return false;
11488 54653805 : scale = INTVAL (scale_rtx);
11489 : }
11490 :
11491 3713597909 : base_reg = base && SUBREG_P (base) ? SUBREG_REG (base) : base;
11492 3713597909 : index_reg = index && SUBREG_P (index) ? SUBREG_REG (index) : index;
11493 :
11494 : /* Avoid useless 0 displacement. */
11495 3713597909 : if (disp == const0_rtx && (base || index))
11496 3713597909 : disp = NULL_RTX;
11497 :
11498 : /* Allow arg pointer and stack pointer as index if there is not scaling. */
11499 2685723873 : if (base_reg && index_reg && scale == 1
11500 3790812603 : && (REGNO (index_reg) == ARG_POINTER_REGNUM
11501 : || REGNO (index_reg) == FRAME_POINTER_REGNUM
11502 : || REGNO (index_reg) == SP_REG))
11503 : {
11504 : std::swap (base, index);
11505 : std::swap (base_reg, index_reg);
11506 : }
11507 :
11508 : /* Special case: rewrite index*1+disp into base+disp. */
11509 3713597909 : if (!base && index && scale == 1)
11510 4 : base = index, base_reg = index_reg, index = index_reg = NULL_RTX;
11511 :
11512 : /* Special case: %ebp cannot be encoded as a base without a displacement.
11513 : Similarly %r13. */
11514 317573947 : if (!disp && base_reg
11515 4026530536 : && (REGNO (base_reg) == ARG_POINTER_REGNUM
11516 : || REGNO (base_reg) == FRAME_POINTER_REGNUM
11517 : || REGNO (base_reg) == BP_REG
11518 : || REGNO (base_reg) == R13_REG))
11519 : disp = const0_rtx;
11520 :
11521 : /* Special case: on K6, [%esi] makes the instruction vector decoded.
11522 : Avoid this by transforming to [%esi+0].
11523 : Reload calls address legitimization without cfun defined, so we need
11524 : to test cfun for being non-NULL. */
11525 0 : if (TARGET_CPU_P (K6) && cfun && optimize_function_for_speed_p (cfun)
11526 0 : && base_reg && !index_reg && !disp
11527 3713597909 : && REGNO (base_reg) == SI_REG)
11528 0 : disp = const0_rtx;
11529 :
11530 : /* Special case: encode reg+reg instead of reg*2. */
11531 3713597909 : if (!base && index && scale == 2)
11532 1027874032 : base = index, base_reg = index_reg, scale = 1;
11533 :
11534 : /* Special case: scaling cannot be encoded without base or displacement. */
11535 1027874032 : if (!base && !disp && index && scale != 1)
11536 3652541 : disp = const0_rtx;
11537 :
11538 3713597909 : out->base = base;
11539 3713597909 : out->index = index;
11540 3713597909 : out->disp = disp;
11541 3713597909 : out->scale = scale;
11542 3713597909 : out->seg = seg;
11543 :
11544 3713597909 : return true;
11545 : }
11546 :
11547 : /* Return cost of the memory address x.
11548 : For i386, it is better to use a complex address than let gcc copy
11549 : the address into a reg and make a new pseudo. But not if the address
11550 : requires to two regs - that would mean more pseudos with longer
11551 : lifetimes. */
11552 : static int
11553 10879034 : ix86_address_cost (rtx x, machine_mode, addr_space_t, bool)
11554 : {
11555 10879034 : struct ix86_address parts;
11556 10879034 : int cost = 1;
11557 10879034 : int ok = ix86_decompose_address (x, &parts);
11558 :
11559 10879034 : gcc_assert (ok);
11560 :
11561 10879034 : if (parts.base && SUBREG_P (parts.base))
11562 245 : parts.base = SUBREG_REG (parts.base);
11563 10879034 : if (parts.index && SUBREG_P (parts.index))
11564 66 : parts.index = SUBREG_REG (parts.index);
11565 :
11566 : /* Attempt to minimize number of registers in the address by increasing
11567 : address cost for each used register. We don't increase address cost
11568 : for "pic_offset_table_rtx". When a memopt with "pic_offset_table_rtx"
11569 : is not invariant itself it most likely means that base or index is not
11570 : invariant. Therefore only "pic_offset_table_rtx" could be hoisted out,
11571 : which is not profitable for x86. */
11572 10879034 : if (parts.base
11573 9450835 : && (!REG_P (parts.base) || REGNO (parts.base) >= FIRST_PSEUDO_REGISTER)
11574 20014049 : && (current_pass->type == GIMPLE_PASS
11575 2706232 : || !pic_offset_table_rtx
11576 124882 : || !REG_P (parts.base)
11577 124882 : || REGNO (pic_offset_table_rtx) != REGNO (parts.base)))
11578 : cost++;
11579 :
11580 10879034 : if (parts.index
11581 5187454 : && (!REG_P (parts.index) || REGNO (parts.index) >= FIRST_PSEUDO_REGISTER)
11582 16054787 : && (current_pass->type == GIMPLE_PASS
11583 668167 : || !pic_offset_table_rtx
11584 55926 : || !REG_P (parts.index)
11585 55926 : || REGNO (pic_offset_table_rtx) != REGNO (parts.index)))
11586 5174457 : cost++;
11587 :
11588 : /* AMD-K6 don't like addresses with ModR/M set to 00_xxx_100b,
11589 : since it's predecode logic can't detect the length of instructions
11590 : and it degenerates to vector decoded. Increase cost of such
11591 : addresses here. The penalty is minimally 2 cycles. It may be worthwhile
11592 : to split such addresses or even refuse such addresses at all.
11593 :
11594 : Following addressing modes are affected:
11595 : [base+scale*index]
11596 : [scale*index+disp]
11597 : [base+index]
11598 :
11599 : The first and last case may be avoidable by explicitly coding the zero in
11600 : memory address, but I don't have AMD-K6 machine handy to check this
11601 : theory. */
11602 :
11603 10879034 : if (TARGET_CPU_P (K6)
11604 0 : && ((!parts.disp && parts.base && parts.index && parts.scale != 1)
11605 0 : || (parts.disp && !parts.base && parts.index && parts.scale != 1)
11606 0 : || (!parts.disp && parts.base && parts.index && parts.scale == 1)))
11607 0 : cost += 10;
11608 :
11609 10879034 : return cost;
11610 : }
11611 :
11612 : /* Implement TARGET_USE_BY_PIECES_INFRASTRUCTURE_P. */
11613 :
11614 : bool
11615 1207958 : ix86_use_by_pieces_infrastructure_p (unsigned HOST_WIDE_INT size,
11616 : unsigned int align,
11617 : enum by_pieces_operation op,
11618 : bool speed_p)
11619 : {
11620 : /* Return true when we are currently expanding memcpy/memset epilogue
11621 : with move_by_pieces or store_by_pieces. */
11622 1207958 : if (cfun->machine->by_pieces_in_use)
11623 : return true;
11624 :
11625 1205872 : return default_use_by_pieces_infrastructure_p (size, align, op,
11626 1205872 : speed_p);
11627 : }
11628 :
11629 : /* Allow {LABEL | SYMBOL}_REF - SYMBOL_REF-FOR-PICBASE for Mach-O as
11630 : this is used for to form addresses to local data when -fPIC is in
11631 : use. */
11632 :
11633 : static bool
11634 0 : darwin_local_data_pic (rtx disp)
11635 : {
11636 0 : return (GET_CODE (disp) == UNSPEC
11637 0 : && XINT (disp, 1) == UNSPEC_MACHOPIC_OFFSET);
11638 : }
11639 :
11640 : /* True if the function symbol operand X should be loaded from GOT.
11641 : If CALL_P is true, X is a call operand.
11642 :
11643 : NB: -mno-direct-extern-access doesn't force load from GOT for
11644 : call.
11645 :
11646 : NB: In 32-bit mode, only non-PIC is allowed in inline assembly
11647 : statements, since a PIC register could not be available at the
11648 : call site. */
11649 :
11650 : bool
11651 1862224176 : ix86_force_load_from_GOT_p (rtx x, bool call_p)
11652 : {
11653 95703976 : return ((TARGET_64BIT || (!flag_pic && HAVE_AS_IX86_GOT32X))
11654 : && !TARGET_PECOFF && !TARGET_MACHO
11655 1859359770 : && (!flag_pic || this_is_asm_operands)
11656 1822646841 : && ix86_cmodel != CM_LARGE
11657 1822640842 : && ix86_cmodel != CM_LARGE_PIC
11658 1822640841 : && SYMBOL_REF_P (x)
11659 1822640839 : && ((!call_p
11660 1817195172 : && (!ix86_direct_extern_access
11661 1817192902 : || (SYMBOL_REF_DECL (x)
11662 1635677040 : && lookup_attribute ("nodirect_extern_access",
11663 1635677040 : DECL_ATTRIBUTES (SYMBOL_REF_DECL (x))))))
11664 1822638115 : || (SYMBOL_REF_FUNCTION_P (x)
11665 699272020 : && (!flag_plt
11666 699267609 : || (SYMBOL_REF_DECL (x)
11667 699267609 : && lookup_attribute ("noplt",
11668 699267609 : DECL_ATTRIBUTES (SYMBOL_REF_DECL (x)))))))
11669 1862231710 : && !SYMBOL_REF_LOCAL_P (x));
11670 : }
11671 :
11672 : /* Determine if a given RTX is a valid constant. We already know this
11673 : satisfies CONSTANT_P. */
11674 :
11675 : static bool
11676 1571227985 : ix86_legitimate_constant_p (machine_mode mode, rtx x)
11677 : {
11678 1571227985 : switch (GET_CODE (x))
11679 : {
11680 134965884 : case CONST:
11681 134965884 : x = XEXP (x, 0);
11682 :
11683 134965884 : if (GET_CODE (x) == PLUS)
11684 : {
11685 134848065 : if (!CONST_INT_P (XEXP (x, 1)))
11686 : return false;
11687 134848065 : x = XEXP (x, 0);
11688 : }
11689 :
11690 134965884 : if (TARGET_MACHO && darwin_local_data_pic (x))
11691 : return true;
11692 :
11693 : /* Only some unspecs are valid as "constants". */
11694 134965884 : if (GET_CODE (x) == UNSPEC)
11695 501796 : switch (XINT (x, 1))
11696 : {
11697 20883 : case UNSPEC_GOT:
11698 20883 : case UNSPEC_GOTOFF:
11699 20883 : case UNSPEC_PLTOFF:
11700 20883 : return TARGET_64BIT;
11701 480550 : case UNSPEC_TPOFF:
11702 480550 : case UNSPEC_NTPOFF:
11703 480550 : x = XVECEXP (x, 0, 0);
11704 480550 : return (SYMBOL_REF_P (x)
11705 480550 : && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
11706 275 : case UNSPEC_DTPOFF:
11707 275 : x = XVECEXP (x, 0, 0);
11708 275 : return (SYMBOL_REF_P (x)
11709 275 : && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_DYNAMIC);
11710 0 : case UNSPEC_SECREL32:
11711 0 : x = XVECEXP (x, 0, 0);
11712 0 : return SYMBOL_REF_P (x);
11713 : default:
11714 : return false;
11715 : }
11716 :
11717 : /* We must have drilled down to a symbol. */
11718 134464088 : if (LABEL_REF_P (x))
11719 : return true;
11720 134458156 : if (!SYMBOL_REF_P (x))
11721 : return false;
11722 : /* FALLTHRU */
11723 :
11724 929344373 : case SYMBOL_REF:
11725 : /* TLS symbols are never valid. */
11726 929344373 : if (SYMBOL_REF_TLS_MODEL (x))
11727 : return false;
11728 :
11729 : /* DLLIMPORT symbols are never valid. */
11730 929239732 : if (TARGET_DLLIMPORT_DECL_ATTRIBUTES
11731 : && SYMBOL_REF_DLLIMPORT_P (x))
11732 : return false;
11733 :
11734 : #if TARGET_MACHO
11735 : /* mdynamic-no-pic */
11736 : if (MACHO_DYNAMIC_NO_PIC_P)
11737 : return machopic_symbol_defined_p (x);
11738 : #endif
11739 :
11740 : /* External function address should be loaded
11741 : via the GOT slot to avoid PLT. */
11742 929239732 : if (ix86_force_load_from_GOT_p (x))
11743 : return false;
11744 :
11745 : break;
11746 :
11747 619628470 : CASE_CONST_SCALAR_INT:
11748 619628470 : if (ix86_endbr_immediate_operand (x, VOIDmode))
11749 : return false;
11750 :
11751 619628269 : switch (mode)
11752 : {
11753 1494403 : case E_TImode:
11754 1494403 : if (TARGET_64BIT)
11755 : return true;
11756 : /* FALLTHRU */
11757 25328 : case E_OImode:
11758 25328 : case E_XImode:
11759 25328 : if (!standard_sse_constant_p (x, mode)
11760 41383 : && GET_MODE_SIZE (TARGET_AVX512F
11761 : ? XImode
11762 : : (TARGET_AVX
11763 : ? OImode
11764 : : (TARGET_SSE2
11765 16055 : ? TImode : DImode))) < GET_MODE_SIZE (mode))
11766 : return false;
11767 : default:
11768 : break;
11769 : }
11770 : break;
11771 :
11772 9245223 : case CONST_VECTOR:
11773 9245223 : if (!standard_sse_constant_p (x, mode))
11774 : return false;
11775 : break;
11776 :
11777 7753712 : case CONST_DOUBLE:
11778 7753712 : if (mode == E_BFmode)
11779 : return false;
11780 :
11781 : default:
11782 : break;
11783 : }
11784 :
11785 : /* Otherwise we handle everything else in the move patterns. */
11786 : return true;
11787 : }
11788 :
11789 : /* Determine if it's legal to put X into the constant pool. This
11790 : is not possible for the address of thread-local symbols, which
11791 : is checked above. */
11792 :
11793 : static bool
11794 64045759 : ix86_cannot_force_const_mem (machine_mode mode, rtx x)
11795 : {
11796 : /* We can put any immediate constant in memory. */
11797 64045759 : switch (GET_CODE (x))
11798 : {
11799 : CASE_CONST_ANY:
11800 : return false;
11801 :
11802 2448414 : default:
11803 2448414 : break;
11804 : }
11805 :
11806 2448414 : return !ix86_legitimate_constant_p (mode, x);
11807 : }
11808 :
11809 : /* Return a unique alias set for the GOT. */
11810 :
11811 : alias_set_type
11812 219176 : ix86_GOT_alias_set (void)
11813 : {
11814 219176 : static alias_set_type set = -1;
11815 219176 : if (set == -1)
11816 4750 : set = new_alias_set ();
11817 219176 : return set;
11818 : }
11819 :
11820 : /* Nonzero if the constant value X is a legitimate general operand
11821 : when generating PIC code. It is given that flag_pic is on and
11822 : that X satisfies CONSTANT_P. */
11823 :
11824 : bool
11825 143794758 : legitimate_pic_operand_p (rtx x)
11826 : {
11827 143794758 : rtx inner;
11828 :
11829 143794758 : switch (GET_CODE (x))
11830 : {
11831 2556539 : case CONST:
11832 2556539 : inner = XEXP (x, 0);
11833 2556539 : if (GET_CODE (inner) == PLUS
11834 405064 : && CONST_INT_P (XEXP (inner, 1)))
11835 405064 : inner = XEXP (inner, 0);
11836 :
11837 : /* Only some unspecs are valid as "constants". */
11838 2556539 : if (GET_CODE (inner) == UNSPEC)
11839 2258848 : switch (XINT (inner, 1))
11840 : {
11841 2197900 : case UNSPEC_GOT:
11842 2197900 : case UNSPEC_GOTOFF:
11843 2197900 : case UNSPEC_PLTOFF:
11844 2197900 : return TARGET_64BIT;
11845 0 : case UNSPEC_TPOFF:
11846 0 : x = XVECEXP (inner, 0, 0);
11847 0 : return (SYMBOL_REF_P (x)
11848 0 : && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
11849 0 : case UNSPEC_SECREL32:
11850 0 : x = XVECEXP (inner, 0, 0);
11851 0 : return SYMBOL_REF_P (x);
11852 0 : case UNSPEC_MACHOPIC_OFFSET:
11853 0 : return legitimate_pic_address_disp_p (x);
11854 : default:
11855 : return false;
11856 : }
11857 : /* FALLTHRU */
11858 :
11859 13286301 : case SYMBOL_REF:
11860 13286301 : case LABEL_REF:
11861 13286301 : return legitimate_pic_address_disp_p (x);
11862 :
11863 : default:
11864 : return true;
11865 : }
11866 : }
11867 :
11868 : /* Determine if a given CONST RTX is a valid memory displacement
11869 : in PIC mode. */
11870 :
11871 : bool
11872 93874371 : legitimate_pic_address_disp_p (rtx disp)
11873 : {
11874 93874371 : bool saw_plus;
11875 :
11876 : /* In 64bit mode we can allow direct addresses of symbols and labels
11877 : when they are not dynamic symbols. */
11878 93874371 : if (TARGET_64BIT)
11879 : {
11880 68674813 : rtx op0 = disp, op1;
11881 :
11882 68674813 : switch (GET_CODE (disp))
11883 : {
11884 : case LABEL_REF:
11885 : return true;
11886 :
11887 15082671 : case CONST:
11888 15082671 : if (GET_CODE (XEXP (disp, 0)) != PLUS)
11889 : break;
11890 4045062 : op0 = XEXP (XEXP (disp, 0), 0);
11891 4045062 : op1 = XEXP (XEXP (disp, 0), 1);
11892 4045062 : if (!CONST_INT_P (op1))
11893 : break;
11894 4045062 : if (GET_CODE (op0) == UNSPEC
11895 296 : && (XINT (op0, 1) == UNSPEC_DTPOFF
11896 296 : || XINT (op0, 1) == UNSPEC_NTPOFF)
11897 4045358 : && trunc_int_for_mode (INTVAL (op1), SImode) == INTVAL (op1))
11898 : return true;
11899 4044766 : if (INTVAL (op1) >= 16*1024*1024
11900 4044766 : || INTVAL (op1) < -16*1024*1024)
11901 : break;
11902 4044678 : if (LABEL_REF_P (op0))
11903 : return true;
11904 4044678 : if (GET_CODE (op0) == CONST
11905 0 : && GET_CODE (XEXP (op0, 0)) == UNSPEC
11906 0 : && XINT (XEXP (op0, 0), 1) == UNSPEC_PCREL)
11907 : return true;
11908 4044678 : if (GET_CODE (op0) == UNSPEC
11909 0 : && XINT (op0, 1) == UNSPEC_PCREL)
11910 : return true;
11911 4044678 : if (!SYMBOL_REF_P (op0))
11912 : break;
11913 : /* FALLTHRU */
11914 :
11915 57401992 : case SYMBOL_REF:
11916 : /* TLS references should always be enclosed in UNSPEC.
11917 : The dllimported symbol needs always to be resolved. */
11918 57401992 : if (SYMBOL_REF_TLS_MODEL (op0)
11919 : || (TARGET_DLLIMPORT_DECL_ATTRIBUTES && SYMBOL_REF_DLLIMPORT_P (op0)))
11920 : return false;
11921 :
11922 57246003 : if (TARGET_PECOFF)
11923 : {
11924 : #if TARGET_PECOFF
11925 : if (is_imported_p (op0))
11926 : return true;
11927 : #endif
11928 :
11929 : if (SYMBOL_REF_FAR_ADDR_P (op0) || !SYMBOL_REF_LOCAL_P (op0))
11930 : break;
11931 :
11932 : /* Non-external-weak function symbols need to be resolved only
11933 : for the large model. Non-external symbols don't need to be
11934 : resolved for large and medium models. For the small model,
11935 : we don't need to resolve anything here. */
11936 : if ((ix86_cmodel != CM_LARGE_PIC
11937 : && SYMBOL_REF_FUNCTION_P (op0)
11938 : && !(SYMBOL_REF_EXTERNAL_P (op0) && SYMBOL_REF_WEAK (op0)))
11939 : || !SYMBOL_REF_EXTERNAL_P (op0)
11940 : || ix86_cmodel == CM_SMALL_PIC)
11941 : return true;
11942 : }
11943 57246003 : else if (!SYMBOL_REF_FAR_ADDR_P (op0)
11944 57245999 : && (SYMBOL_REF_LOCAL_P (op0)
11945 25353615 : || ((ix86_direct_extern_access
11946 50535188 : && !(SYMBOL_REF_DECL (op0)
11947 25181736 : && lookup_attribute ("nodirect_extern_access",
11948 25181736 : DECL_ATTRIBUTES (SYMBOL_REF_DECL (op0)))))
11949 : && HAVE_LD_PIE_COPYRELOC
11950 25353289 : && flag_pie
11951 6946128 : && !SYMBOL_REF_WEAK (op0)
11952 6945740 : && !SYMBOL_REF_FUNCTION_P (op0)))
11953 89262959 : && ix86_cmodel != CM_LARGE_PIC)
11954 : return true;
11955 : break;
11956 :
11957 : default:
11958 : break;
11959 : }
11960 : }
11961 61470171 : if (GET_CODE (disp) != CONST)
11962 : return false;
11963 16233699 : disp = XEXP (disp, 0);
11964 :
11965 16233699 : if (TARGET_64BIT)
11966 : {
11967 : /* We are unsafe to allow PLUS expressions. This limit allowed distance
11968 : of GOT tables. We should not need these anyway. */
11969 11088898 : if (GET_CODE (disp) != UNSPEC
11970 11037609 : || (XINT (disp, 1) != UNSPEC_GOTPCREL
11971 11037609 : && XINT (disp, 1) != UNSPEC_GOTOFF
11972 : && XINT (disp, 1) != UNSPEC_PCREL
11973 : && XINT (disp, 1) != UNSPEC_PLTOFF))
11974 : return false;
11975 :
11976 11037609 : if (!SYMBOL_REF_P (XVECEXP (disp, 0, 0))
11977 11037609 : && !LABEL_REF_P (XVECEXP (disp, 0, 0)))
11978 : return false;
11979 : return true;
11980 : }
11981 :
11982 5144801 : saw_plus = false;
11983 5144801 : if (GET_CODE (disp) == PLUS)
11984 : {
11985 586082 : if (!CONST_INT_P (XEXP (disp, 1)))
11986 : return false;
11987 586082 : disp = XEXP (disp, 0);
11988 586082 : saw_plus = true;
11989 : }
11990 :
11991 5144801 : if (TARGET_MACHO && darwin_local_data_pic (disp))
11992 : return true;
11993 :
11994 5144801 : if (GET_CODE (disp) != UNSPEC)
11995 : return false;
11996 :
11997 4979727 : switch (XINT (disp, 1))
11998 : {
11999 2267569 : case UNSPEC_GOT:
12000 2267569 : if (saw_plus)
12001 : return false;
12002 : /* We need to check for both symbols and labels because VxWorks loads
12003 : text labels with @GOT rather than @GOTOFF. See gotoff_operand for
12004 : details. */
12005 2267568 : return (SYMBOL_REF_P (XVECEXP (disp, 0, 0))
12006 2267568 : || LABEL_REF_P (XVECEXP (disp, 0, 0)));
12007 2712158 : case UNSPEC_GOTOFF:
12008 : /* Refuse GOTOFF in 64bit mode since it is always 64bit when used.
12009 : While ABI specify also 32bit relocation but we don't produce it in
12010 : small PIC model at all. */
12011 2712158 : if ((SYMBOL_REF_P (XVECEXP (disp, 0, 0))
12012 2712158 : || LABEL_REF_P (XVECEXP (disp, 0, 0)))
12013 : && !TARGET_64BIT)
12014 5424316 : return !TARGET_PECOFF && gotoff_operand (XVECEXP (disp, 0, 0), Pmode);
12015 : return false;
12016 0 : case UNSPEC_GOTTPOFF:
12017 0 : case UNSPEC_GOTNTPOFF:
12018 0 : case UNSPEC_INDNTPOFF:
12019 0 : if (saw_plus)
12020 : return false;
12021 0 : disp = XVECEXP (disp, 0, 0);
12022 0 : return (SYMBOL_REF_P (disp)
12023 0 : && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_INITIAL_EXEC);
12024 0 : case UNSPEC_NTPOFF:
12025 0 : disp = XVECEXP (disp, 0, 0);
12026 0 : return (SYMBOL_REF_P (disp)
12027 0 : && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_EXEC);
12028 0 : case UNSPEC_DTPOFF:
12029 0 : disp = XVECEXP (disp, 0, 0);
12030 0 : return (SYMBOL_REF_P (disp)
12031 0 : && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_DYNAMIC);
12032 0 : case UNSPEC_SECREL32:
12033 0 : disp = XVECEXP (disp, 0, 0);
12034 0 : return SYMBOL_REF_P (disp);
12035 : }
12036 :
12037 : return false;
12038 : }
12039 :
12040 : /* Determine if op is suitable RTX for an address register.
12041 : Return naked register if a register or a register subreg is
12042 : found, otherwise return NULL_RTX. */
12043 :
12044 : static rtx
12045 1335877774 : ix86_validate_address_register (rtx op)
12046 : {
12047 1335877774 : machine_mode mode = GET_MODE (op);
12048 :
12049 : /* Only SImode or DImode registers can form the address. */
12050 1335877774 : if (mode != SImode && mode != DImode)
12051 : return NULL_RTX;
12052 :
12053 1335869914 : if (REG_P (op))
12054 : return op;
12055 894941 : else if (SUBREG_P (op))
12056 : {
12057 894941 : rtx reg = SUBREG_REG (op);
12058 :
12059 894941 : if (!REG_P (reg))
12060 : return NULL_RTX;
12061 :
12062 894941 : mode = GET_MODE (reg);
12063 :
12064 : /* Don't allow SUBREGs that span more than a word. It can
12065 : lead to spill failures when the register is one word out
12066 : of a two word structure. */
12067 1836125 : if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
12068 : return NULL_RTX;
12069 :
12070 : /* Allow only SUBREGs of non-eliminable hard registers. */
12071 418127 : if (register_no_elim_operand (reg, mode))
12072 417957 : return reg;
12073 : }
12074 :
12075 : /* Op is not a register. */
12076 : return NULL_RTX;
12077 : }
12078 :
12079 : /* Determine which memory address register set insn can use. */
12080 :
12081 : static enum attr_addr
12082 260292476 : ix86_memory_address_reg_class (rtx_insn* insn)
12083 : {
12084 : /* LRA can do some initialization with NULL insn,
12085 : return maximum register class in this case. */
12086 260292476 : enum attr_addr addr_rclass = ADDR_GPR32;
12087 :
12088 260292476 : if (!insn)
12089 : return addr_rclass;
12090 :
12091 134530249 : if (asm_noperands (PATTERN (insn)) >= 0
12092 134530249 : || GET_CODE (PATTERN (insn)) == ASM_INPUT)
12093 75542 : return ix86_apx_inline_asm_use_gpr32 ? ADDR_GPR32 : ADDR_GPR16;
12094 :
12095 : /* Return maximum register class for unrecognized instructions. */
12096 134454707 : if (INSN_CODE (insn) < 0)
12097 : return addr_rclass;
12098 :
12099 : /* Try to recognize the insn before calling get_attr_addr.
12100 : Save current recog_data and current alternative. */
12101 134437407 : recog_state_saver recog_save;
12102 :
12103 : /* Update recog_data for processing of alternatives. */
12104 134437407 : extract_insn_cached (insn);
12105 :
12106 : /* If current alternative is not set, loop through enabled
12107 : alternatives and get the most limited register class. */
12108 134437407 : if (recog_save.saved_alternative == -1)
12109 : {
12110 76230141 : alternative_mask enabled = get_enabled_alternatives (insn);
12111 :
12112 1375187791 : for (int i = 0; i < recog_data.n_alternatives; i++)
12113 : {
12114 1222727509 : if (!TEST_BIT (enabled, i))
12115 364218134 : continue;
12116 :
12117 858509375 : which_alternative = i;
12118 858509375 : addr_rclass = MIN (addr_rclass, get_attr_addr (insn));
12119 : }
12120 : }
12121 : else
12122 : {
12123 58207266 : which_alternative = recog_save.saved_alternative;
12124 58207266 : addr_rclass = get_attr_addr (insn);
12125 : }
12126 :
12127 134437407 : return addr_rclass;
12128 134437407 : }
12129 :
12130 : /* Implement TARGET_BASE_REG_CLASS.
12131 :
12132 : Return memory address register class INSN can use. MODE, AS, OUTER_CODE,
12133 : INDEX_CODE and MEM are unused: the EGPR-encoding restriction this
12134 : implements depends only on the instruction, not on the particular address
12135 : being formed. */
12136 :
12137 : static reg_class_t
12138 218745163 : ix86_base_reg_class (machine_mode, addr_space_t, enum rtx_code, enum rtx_code,
12139 : rtx, rtx_insn *insn)
12140 : {
12141 218745163 : switch (ix86_memory_address_reg_class (insn))
12142 : {
12143 : case ADDR_GPR8:
12144 : return LEGACY_GENERAL_REGS;
12145 : case ADDR_GPR16:
12146 : return GENERAL_GPR16;
12147 : case ADDR_GPR32:
12148 : break;
12149 0 : default:
12150 0 : gcc_unreachable ();
12151 : }
12152 :
12153 : return BASE_REG_CLASS;
12154 : }
12155 :
12156 : bool
12157 1194534 : ix86_regno_ok_for_insn_base_p (int regno, rtx_insn* insn)
12158 : {
12159 1194534 : switch (ix86_memory_address_reg_class (insn))
12160 : {
12161 0 : case ADDR_GPR8:
12162 0 : return LEGACY_INT_REGNO_P (regno);
12163 0 : case ADDR_GPR16:
12164 0 : return GENERAL_GPR16_REGNO_P (regno);
12165 1194534 : case ADDR_GPR32:
12166 1194534 : break;
12167 0 : default:
12168 0 : gcc_unreachable ();
12169 : }
12170 :
12171 1194534 : return GENERAL_REGNO_P (regno);
12172 : }
12173 :
12174 : enum reg_class
12175 40352779 : ix86_insn_index_reg_class (rtx_insn* insn)
12176 : {
12177 40352779 : switch (ix86_memory_address_reg_class (insn))
12178 : {
12179 : case ADDR_GPR8:
12180 : return LEGACY_INDEX_REGS;
12181 : case ADDR_GPR16:
12182 : return INDEX_GPR16;
12183 : case ADDR_GPR32:
12184 : break;
12185 0 : default:
12186 0 : gcc_unreachable ();
12187 : }
12188 :
12189 : return INDEX_REG_CLASS;
12190 : }
12191 :
12192 : /* Recognizes RTL expressions that are valid memory addresses for an
12193 : instruction. The MODE argument is the machine mode for the MEM
12194 : expression that wants to use this address.
12195 :
12196 : It only recognizes address in canonical form. LEGITIMIZE_ADDRESS should
12197 : convert common non-canonical forms to canonical form so that they will
12198 : be recognized. */
12199 :
12200 : static bool
12201 2240065908 : ix86_legitimate_address_p (machine_mode, rtx addr, bool strict,
12202 : code_helper = ERROR_MARK)
12203 : {
12204 2240065908 : struct ix86_address parts;
12205 2240065908 : rtx base, index, disp;
12206 2240065908 : HOST_WIDE_INT scale;
12207 2240065908 : addr_space_t seg;
12208 :
12209 2240065908 : if (ix86_decompose_address (addr, &parts) == 0)
12210 : /* Decomposition failed. */
12211 : return false;
12212 :
12213 2228006010 : base = parts.base;
12214 2228006010 : index = parts.index;
12215 2228006010 : disp = parts.disp;
12216 2228006010 : scale = parts.scale;
12217 2228006010 : seg = parts.seg;
12218 :
12219 : /* Validate base register. */
12220 2228006010 : if (base)
12221 : {
12222 1252406791 : rtx reg = ix86_validate_address_register (base);
12223 :
12224 1252406791 : if (reg == NULL_RTX)
12225 : return false;
12226 :
12227 1251962912 : unsigned int regno = REGNO (reg);
12228 1251962912 : if ((strict && !REGNO_OK_FOR_BASE_P (regno))
12229 1247510745 : || (!strict && !REGNO_OK_FOR_BASE_NONSTRICT_P (regno)))
12230 : /* Base is not valid. */
12231 : return false;
12232 : }
12233 :
12234 : /* Validate index register. */
12235 2226238762 : if (index)
12236 : {
12237 83470983 : rtx reg = ix86_validate_address_register (index);
12238 :
12239 83470983 : if (reg == NULL_RTX)
12240 : return false;
12241 :
12242 83430018 : unsigned int regno = REGNO (reg);
12243 83430018 : if ((strict && !REGNO_OK_FOR_INDEX_P (regno))
12244 83421846 : || (!strict && !REGNO_OK_FOR_INDEX_NONSTRICT_P (regno)))
12245 : /* Index is not valid. */
12246 : return false;
12247 : }
12248 :
12249 : /* Index and base should have the same mode. */
12250 2226195587 : if (base && index
12251 73824270 : && GET_MODE (base) != GET_MODE (index))
12252 : return false;
12253 :
12254 : /* Address override works only on the (%reg) part of %fs:(%reg). */
12255 2225985136 : if (seg != ADDR_SPACE_GENERIC
12256 2225985136 : && ((base && GET_MODE (base) != word_mode)
12257 341327 : || (index && GET_MODE (index) != word_mode)))
12258 : return false;
12259 :
12260 : /* Validate scale factor. */
12261 2225985107 : if (scale != 1)
12262 : {
12263 39127595 : if (!index)
12264 : /* Scale without index. */
12265 : return false;
12266 :
12267 39127595 : if (scale != 2 && scale != 4 && scale != 8)
12268 : /* Scale is not a valid multiplier. */
12269 : return false;
12270 : }
12271 :
12272 : /* Validate displacement. */
12273 2222897829 : if (disp)
12274 : {
12275 2003350748 : if (ix86_endbr_immediate_operand (disp, VOIDmode))
12276 : return false;
12277 :
12278 2003350705 : if (GET_CODE (disp) == CONST
12279 149409584 : && GET_CODE (XEXP (disp, 0)) == UNSPEC
12280 16674847 : && XINT (XEXP (disp, 0), 1) != UNSPEC_MACHOPIC_OFFSET)
12281 16674847 : switch (XINT (XEXP (disp, 0), 1))
12282 : {
12283 : /* Refuse GOTOFF and GOT in 64bit mode since it is always 64bit
12284 : when used. While ABI specify also 32bit relocations, we
12285 : don't produce them at all and use IP relative instead.
12286 : Allow GOT in 32bit mode for both PIC and non-PIC if symbol
12287 : should be loaded via GOT. */
12288 2267627 : case UNSPEC_GOT:
12289 2267627 : if (!TARGET_64BIT
12290 2267627 : && ix86_force_load_from_GOT_p (XVECEXP (XEXP (disp, 0), 0, 0)))
12291 0 : goto is_legitimate_pic;
12292 : /* FALLTHRU */
12293 4558922 : case UNSPEC_GOTOFF:
12294 4558922 : gcc_assert (flag_pic);
12295 4558922 : if (!TARGET_64BIT)
12296 4558719 : goto is_legitimate_pic;
12297 :
12298 : /* 64bit address unspec. */
12299 : return false;
12300 :
12301 11037581 : case UNSPEC_GOTPCREL:
12302 11037581 : if (ix86_force_load_from_GOT_p (XVECEXP (XEXP (disp, 0), 0, 0)))
12303 2534 : goto is_legitimate_pic;
12304 : /* FALLTHRU */
12305 11035047 : case UNSPEC_PCREL:
12306 11035047 : gcc_assert (flag_pic);
12307 11035047 : goto is_legitimate_pic;
12308 :
12309 : case UNSPEC_GOTTPOFF:
12310 : case UNSPEC_GOTNTPOFF:
12311 : case UNSPEC_INDNTPOFF:
12312 : case UNSPEC_NTPOFF:
12313 : case UNSPEC_DTPOFF:
12314 : case UNSPEC_SECREL32:
12315 : break;
12316 :
12317 70 : default:
12318 : /* Invalid address unspec. */
12319 70 : return false;
12320 : }
12321 :
12322 1239010284 : else if (SYMBOLIC_CONST (disp)
12323 2119410595 : && (flag_pic
12324 : #if TARGET_MACHO
12325 : || (MACHOPIC_INDIRECT
12326 : && !machopic_operand_p (disp))
12327 : #endif
12328 : ))
12329 : {
12330 :
12331 80067662 : is_legitimate_pic:
12332 80067662 : if (TARGET_64BIT && (index || base))
12333 : {
12334 : /* foo@dtpoff(%rX) is ok. */
12335 50388 : if (GET_CODE (disp) != CONST
12336 8734 : || GET_CODE (XEXP (disp, 0)) != PLUS
12337 8734 : || GET_CODE (XEXP (XEXP (disp, 0), 0)) != UNSPEC
12338 4637 : || !CONST_INT_P (XEXP (XEXP (disp, 0), 1))
12339 4637 : || (XINT (XEXP (XEXP (disp, 0), 0), 1) != UNSPEC_DTPOFF
12340 4637 : && XINT (XEXP (XEXP (disp, 0), 0), 1) != UNSPEC_NTPOFF
12341 6 : && XINT (XEXP (XEXP (disp, 0), 0), 1) != UNSPEC_SECREL32))
12342 : /* Non-constant pic memory reference. */
12343 45757 : return false;
12344 : }
12345 80017274 : else if ((!TARGET_MACHO || flag_pic)
12346 80017274 : && ! legitimate_pic_address_disp_p (disp))
12347 : /* Displacement is an invalid pic construct. */
12348 : return false;
12349 : #if TARGET_MACHO
12350 : else if (MACHO_DYNAMIC_NO_PIC_P
12351 : && !ix86_legitimate_constant_p (Pmode, disp))
12352 : /* displacement must be referenced via non_lazy_pointer */
12353 : return false;
12354 : #endif
12355 :
12356 : /* This code used to verify that a symbolic pic displacement
12357 : includes the pic_offset_table_rtx register.
12358 :
12359 : While this is good idea, unfortunately these constructs may
12360 : be created by "adds using lea" optimization for incorrect
12361 : code like:
12362 :
12363 : int a;
12364 : int foo(int i)
12365 : {
12366 : return *(&a+i);
12367 : }
12368 :
12369 : This code is nonsensical, but results in addressing
12370 : GOT table with pic_offset_table_rtx base. We can't
12371 : just refuse it easily, since it gets matched by
12372 : "addsi3" pattern, that later gets split to lea in the
12373 : case output register differs from input. While this
12374 : can be handled by separate addsi pattern for this case
12375 : that never results in lea, this seems to be easier and
12376 : correct fix for crash to disable this test. */
12377 : }
12378 1922204496 : else if (!LABEL_REF_P (disp)
12379 1922022524 : && !CONST_INT_P (disp)
12380 874281369 : && (GET_CODE (disp) != CONST
12381 131365501 : || !ix86_legitimate_constant_p (Pmode, disp))
12382 2668093007 : && (!SYMBOL_REF_P (disp)
12383 752310696 : || !ix86_legitimate_constant_p (Pmode, disp)))
12384 : /* Displacement is not constant. */
12385 : return false;
12386 1863591161 : else if (TARGET_64BIT
12387 1863591161 : && !x86_64_immediate_operand (disp, VOIDmode))
12388 : /* Displacement is out of range. */
12389 : return false;
12390 : /* In x32 mode, constant addresses are sign extended to 64bit, so
12391 : we have to prevent addresses from 0x80000000 to 0xffffffff. */
12392 44405 : else if (TARGET_X32 && !(index || base)
12393 15241 : && CONST_INT_P (disp)
12394 1863052310 : && val_signbit_known_set_p (SImode, INTVAL (disp)))
12395 88 : return false;
12396 : }
12397 :
12398 : /* Everything looks valid. */
12399 : return true;
12400 : }
12401 :
12402 : /* Determine if a given RTX is a valid constant address. */
12403 :
12404 : bool
12405 2860503090 : constant_address_p (rtx x)
12406 : {
12407 2940842247 : return CONSTANT_P (x) && ix86_legitimate_address_p (Pmode, x, 1);
12408 : }
12409 :
12410 :
12411 : /* Return a legitimate reference for ORIG (an address) using the
12412 : register REG. If REG is 0, a new pseudo is generated.
12413 :
12414 : There are two types of references that must be handled:
12415 :
12416 : 1. Global data references must load the address from the GOT, via
12417 : the PIC reg. An insn is emitted to do this load, and the reg is
12418 : returned.
12419 :
12420 : 2. Static data references, constant pool addresses, and code labels
12421 : compute the address as an offset from the GOT, whose base is in
12422 : the PIC reg. Static data objects have SYMBOL_FLAG_LOCAL set to
12423 : differentiate them from global data objects. The returned
12424 : address is the PIC reg + an unspec constant.
12425 :
12426 : TARGET_LEGITIMATE_ADDRESS_P rejects symbolic references unless the PIC
12427 : reg also appears in the address. */
12428 :
12429 : rtx
12430 770587 : legitimize_pic_address (rtx orig, rtx reg)
12431 : {
12432 770587 : rtx addr = orig;
12433 770587 : rtx new_rtx = orig;
12434 :
12435 : #if TARGET_MACHO
12436 : if (TARGET_MACHO && !TARGET_64BIT)
12437 : {
12438 : if (reg == 0)
12439 : reg = gen_reg_rtx (Pmode);
12440 : /* Use the generic Mach-O PIC machinery. */
12441 : return machopic_legitimize_pic_address (orig, GET_MODE (orig), reg);
12442 : }
12443 : #endif
12444 :
12445 770587 : if (TARGET_64BIT && TARGET_DLLIMPORT_DECL_ATTRIBUTES)
12446 : {
12447 : #if TARGET_PECOFF
12448 : rtx tmp = legitimize_pe_coff_symbol (addr, true);
12449 : if (tmp)
12450 : return tmp;
12451 : #endif
12452 : }
12453 :
12454 770587 : if (TARGET_64BIT && legitimate_pic_address_disp_p (addr))
12455 : new_rtx = addr;
12456 331910 : else if ((!TARGET_64BIT
12457 132119 : || /* TARGET_64BIT && */ ix86_cmodel != CM_SMALL_PIC)
12458 : && !TARGET_PECOFF
12459 531798 : && gotoff_operand (addr, Pmode))
12460 : {
12461 : /* This symbol may be referenced via a displacement
12462 : from the PIC base address (@GOTOFF). */
12463 96446 : if (GET_CODE (addr) == CONST)
12464 3045 : addr = XEXP (addr, 0);
12465 :
12466 96446 : if (GET_CODE (addr) == PLUS)
12467 : {
12468 6090 : new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, XEXP (addr, 0)),
12469 : UNSPEC_GOTOFF);
12470 6090 : new_rtx = gen_rtx_PLUS (Pmode, new_rtx, XEXP (addr, 1));
12471 : }
12472 : else
12473 186773 : new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOTOFF);
12474 :
12475 192863 : new_rtx = gen_rtx_CONST (Pmode, new_rtx);
12476 :
12477 96446 : if (TARGET_64BIT)
12478 29 : new_rtx = copy_to_suggested_reg (new_rtx, reg, Pmode);
12479 :
12480 96446 : if (reg != 0)
12481 : {
12482 3 : gcc_assert (REG_P (reg));
12483 3 : new_rtx = expand_simple_binop (Pmode, PLUS, pic_offset_table_rtx,
12484 : new_rtx, reg, 1, OPTAB_DIRECT);
12485 : }
12486 : else
12487 192860 : new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
12488 : }
12489 443757 : else if ((SYMBOL_REF_P (addr) && SYMBOL_REF_TLS_MODEL (addr) == 0)
12490 : /* We can't always use @GOTOFF for text labels
12491 : on VxWorks, see gotoff_operand. */
12492 235464 : || (TARGET_VXWORKS_VAROFF && LABEL_REF_P (addr)))
12493 : {
12494 : #if TARGET_PECOFF
12495 : rtx tmp = legitimize_pe_coff_symbol (addr, true);
12496 : if (tmp)
12497 : return tmp;
12498 : #endif
12499 :
12500 : /* For x64 PE-COFF there is no GOT table,
12501 : so we use address directly. */
12502 208290 : if (TARGET_64BIT && TARGET_PECOFF)
12503 : {
12504 : new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_PCREL);
12505 : new_rtx = gen_rtx_CONST (Pmode, new_rtx);
12506 : }
12507 208290 : else if (TARGET_64BIT && ix86_cmodel != CM_LARGE_PIC)
12508 : {
12509 124859 : new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr),
12510 : UNSPEC_GOTPCREL);
12511 124859 : new_rtx = gen_rtx_CONST (Pmode, new_rtx);
12512 124859 : new_rtx = gen_const_mem (Pmode, new_rtx);
12513 124856 : set_mem_alias_set (new_rtx, GOT_ALIAS_SET);
12514 : }
12515 : else
12516 : {
12517 : /* This symbol must be referenced via a load
12518 : from the Global Offset Table (@GOT). */
12519 166845 : new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOT);
12520 166845 : new_rtx = gen_rtx_CONST (Pmode, new_rtx);
12521 :
12522 83434 : if (TARGET_64BIT)
12523 23 : new_rtx = copy_to_suggested_reg (new_rtx, reg, Pmode);
12524 :
12525 83434 : if (reg != 0)
12526 : {
12527 0 : gcc_assert (REG_P (reg));
12528 0 : new_rtx = expand_simple_binop (Pmode, PLUS, pic_offset_table_rtx,
12529 : new_rtx, reg, 1, OPTAB_DIRECT);
12530 : }
12531 : else
12532 166845 : new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
12533 :
12534 166845 : new_rtx = gen_const_mem (Pmode, new_rtx);
12535 83434 : set_mem_alias_set (new_rtx, GOT_ALIAS_SET);
12536 : }
12537 :
12538 291704 : new_rtx = copy_to_suggested_reg (new_rtx, reg, Pmode);
12539 : }
12540 : else
12541 : {
12542 27174 : if (CONST_INT_P (addr)
12543 27174 : && !x86_64_immediate_operand (addr, VOIDmode))
12544 8 : new_rtx = copy_to_suggested_reg (addr, reg, Pmode);
12545 27166 : else if (GET_CODE (addr) == CONST)
12546 : {
12547 16626 : addr = XEXP (addr, 0);
12548 :
12549 : /* We must match stuff we generate before. Assume the only
12550 : unspecs that can get here are ours. Not that we could do
12551 : anything with them anyway.... */
12552 16626 : if (GET_CODE (addr) == UNSPEC
12553 8981 : || (GET_CODE (addr) == PLUS
12554 8981 : && GET_CODE (XEXP (addr, 0)) == UNSPEC))
12555 : return orig;
12556 6845 : gcc_assert (GET_CODE (addr) == PLUS);
12557 : }
12558 :
12559 17393 : if (GET_CODE (addr) == PLUS)
12560 : {
12561 8697 : rtx op0 = XEXP (addr, 0), op1 = XEXP (addr, 1);
12562 :
12563 : /* Check first to see if this is a constant
12564 : offset from a @GOTOFF symbol reference. */
12565 8697 : if (!TARGET_PECOFF
12566 13788 : && gotoff_operand (op0, Pmode)
12567 8697 : && CONST_INT_P (op1))
12568 : {
12569 4 : if (!TARGET_64BIT)
12570 : {
12571 0 : new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, op0),
12572 : UNSPEC_GOTOFF);
12573 0 : new_rtx = gen_rtx_PLUS (Pmode, new_rtx, op1);
12574 0 : new_rtx = gen_rtx_CONST (Pmode, new_rtx);
12575 :
12576 0 : if (reg != 0)
12577 : {
12578 0 : gcc_assert (REG_P (reg));
12579 0 : new_rtx = expand_simple_binop (Pmode, PLUS,
12580 : pic_offset_table_rtx,
12581 : new_rtx, reg, 1,
12582 : OPTAB_DIRECT);
12583 : }
12584 : else
12585 0 : new_rtx
12586 0 : = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
12587 : }
12588 : else
12589 : {
12590 4 : if (INTVAL (op1) < -16*1024*1024
12591 4 : || INTVAL (op1) >= 16*1024*1024)
12592 : {
12593 4 : if (!x86_64_immediate_operand (op1, Pmode))
12594 4 : op1 = force_reg (Pmode, op1);
12595 :
12596 4 : new_rtx
12597 4 : = gen_rtx_PLUS (Pmode, force_reg (Pmode, op0), op1);
12598 : }
12599 : }
12600 : }
12601 : else
12602 : {
12603 8693 : rtx base = legitimize_pic_address (op0, reg);
12604 8693 : machine_mode mode = GET_MODE (base);
12605 8693 : new_rtx
12606 8693 : = legitimize_pic_address (op1, base == reg ? NULL_RTX : reg);
12607 :
12608 8693 : if (CONST_INT_P (new_rtx))
12609 : {
12610 6833 : if (INTVAL (new_rtx) < -16*1024*1024
12611 6833 : || INTVAL (new_rtx) >= 16*1024*1024)
12612 : {
12613 0 : if (!x86_64_immediate_operand (new_rtx, mode))
12614 0 : new_rtx = force_reg (mode, new_rtx);
12615 :
12616 0 : new_rtx
12617 0 : = gen_rtx_PLUS (mode, force_reg (mode, base), new_rtx);
12618 : }
12619 : else
12620 6833 : new_rtx = plus_constant (mode, base, INTVAL (new_rtx));
12621 : }
12622 : else
12623 : {
12624 : /* For %rip addressing, we have to use
12625 : just disp32, not base nor index. */
12626 1860 : if (TARGET_64BIT
12627 100 : && (SYMBOL_REF_P (base)
12628 100 : || LABEL_REF_P (base)))
12629 7 : base = force_reg (mode, base);
12630 1860 : if (GET_CODE (new_rtx) == PLUS
12631 1740 : && CONSTANT_P (XEXP (new_rtx, 1)))
12632 : {
12633 1736 : base = gen_rtx_PLUS (mode, base, XEXP (new_rtx, 0));
12634 1736 : new_rtx = XEXP (new_rtx, 1);
12635 : }
12636 1860 : new_rtx = gen_rtx_PLUS (mode, base, new_rtx);
12637 : }
12638 : }
12639 : }
12640 : }
12641 : return new_rtx;
12642 : }
12643 :
12644 : /* Load the thread pointer. If TO_REG is true, force it into a register. */
12645 :
12646 : static rtx
12647 24546 : get_thread_pointer (machine_mode tp_mode, bool to_reg)
12648 : {
12649 24546 : rtx tp = gen_rtx_UNSPEC (ptr_mode, gen_rtvec (1, const0_rtx), UNSPEC_TP);
12650 :
12651 24546 : if (GET_MODE (tp) != tp_mode)
12652 : {
12653 11 : gcc_assert (GET_MODE (tp) == SImode);
12654 11 : gcc_assert (tp_mode == DImode);
12655 :
12656 11 : tp = gen_rtx_ZERO_EXTEND (tp_mode, tp);
12657 : }
12658 :
12659 24546 : if (to_reg)
12660 8163 : tp = copy_to_mode_reg (tp_mode, tp);
12661 :
12662 24546 : return tp;
12663 : }
12664 :
12665 : /* Construct the SYMBOL_REF for the _tls_index symbol. */
12666 :
12667 : static GTY(()) rtx ix86_tls_index_symbol;
12668 :
12669 : static rtx
12670 0 : ix86_tls_index (void)
12671 : {
12672 0 : if (!ix86_tls_index_symbol)
12673 0 : ix86_tls_index_symbol = gen_rtx_SYMBOL_REF (SImode, "_tls_index");
12674 :
12675 0 : if (flag_pic)
12676 0 : return gen_rtx_CONST (Pmode,
12677 : gen_rtx_UNSPEC (Pmode,
12678 : gen_rtvec (1, ix86_tls_index_symbol),
12679 : UNSPEC_PCREL));
12680 : else
12681 0 : return ix86_tls_index_symbol;
12682 : }
12683 :
12684 : /* Construct the SYMBOL_REF for the tls_get_addr function. */
12685 :
12686 : static GTY(()) rtx ix86_tls_symbol;
12687 :
12688 : rtx
12689 6726 : ix86_tls_get_addr (void)
12690 : {
12691 6726 : if (cfun->machine->call_saved_registers
12692 6726 : == TYPE_NO_CALLER_SAVED_REGISTERS)
12693 : {
12694 : /* __tls_get_addr doesn't preserve vector registers. When a
12695 : function with no_caller_saved_registers attribute calls
12696 : __tls_get_addr, YMM and ZMM registers will be clobbered.
12697 : Issue an error and suggest -mtls-dialect=gnu2 in this case. */
12698 3 : if (cfun->machine->func_type == TYPE_NORMAL)
12699 1 : error (G_("%<-mtls-dialect=gnu2%> must be used with a function"
12700 : " with the %<no_caller_saved_registers%> attribute"));
12701 : else
12702 3 : error (cfun->machine->func_type == TYPE_EXCEPTION
12703 : ? G_("%<-mtls-dialect=gnu2%> must be used with an"
12704 : " exception service routine")
12705 : : G_("%<-mtls-dialect=gnu2%> must be used with an"
12706 : " interrupt service routine"));
12707 : /* Don't issue the same error twice. */
12708 3 : cfun->machine->func_type = TYPE_NORMAL;
12709 3 : cfun->machine->call_saved_registers
12710 3 : = TYPE_DEFAULT_CALL_SAVED_REGISTERS;
12711 : }
12712 :
12713 6726 : if (!ix86_tls_symbol)
12714 : {
12715 216 : const char *sym
12716 253 : = ((TARGET_ANY_GNU_TLS && !TARGET_64BIT)
12717 253 : ? "___tls_get_addr" : "__tls_get_addr");
12718 :
12719 290 : ix86_tls_symbol = gen_rtx_SYMBOL_REF (Pmode, sym);
12720 : }
12721 :
12722 6726 : if (ix86_cmodel == CM_LARGE_PIC && !TARGET_PECOFF)
12723 : {
12724 2 : rtx unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, ix86_tls_symbol),
12725 : UNSPEC_PLTOFF);
12726 2 : return gen_rtx_PLUS (Pmode, pic_offset_table_rtx,
12727 : gen_rtx_CONST (Pmode, unspec));
12728 : }
12729 :
12730 6724 : return ix86_tls_symbol;
12731 : }
12732 :
12733 : /* Return the descriptor of the function ABI type for the tls_get_addr
12734 : function. */
12735 :
12736 : const predefined_function_abi &
12737 5012 : ix86_tls_get_addr_abi (void)
12738 : {
12739 5012 : if (ix86_abi == SYSV_ABI)
12740 5000 : return default_function_abi;
12741 : else
12742 12 : return ix86_alternate_abi ();
12743 : }
12744 :
12745 : /* Construct the SYMBOL_REF for the _TLS_MODULE_BASE_ symbol. */
12746 :
12747 : static GTY(()) rtx ix86_tls_module_base_symbol;
12748 :
12749 : rtx
12750 98 : ix86_tls_module_base (void)
12751 : {
12752 98 : if (!ix86_tls_module_base_symbol)
12753 : {
12754 11 : ix86_tls_module_base_symbol
12755 11 : = gen_rtx_SYMBOL_REF (ptr_mode, "_TLS_MODULE_BASE_");
12756 :
12757 11 : SYMBOL_REF_FLAGS (ix86_tls_module_base_symbol)
12758 11 : |= TLS_MODEL_GLOBAL_DYNAMIC << SYMBOL_FLAG_TLS_SHIFT;
12759 : }
12760 :
12761 98 : return ix86_tls_module_base_symbol;
12762 : }
12763 :
12764 : /* A subroutine of ix86_legitimize_address and ix86_expand_move. FOR_MOV is
12765 : false if we expect this to be used for a memory address and true if
12766 : we expect to load the address into a register. */
12767 :
12768 : rtx
12769 30973 : legitimize_tls_address (rtx x, enum tls_model model, bool for_mov)
12770 : {
12771 30973 : rtx dest, base, off;
12772 30973 : rtx pic = NULL_RTX, tp = NULL_RTX;
12773 30973 : machine_mode tp_mode = Pmode;
12774 30973 : int type;
12775 :
12776 : /* Windows implements a single form of TLS. */
12777 30973 : if (TARGET_WIN32_TLS)
12778 : {
12779 : /* Load the 32-bit index. */
12780 : rtx ind = gen_const_mem (SImode, ix86_tls_index ());
12781 : set_mem_alias_set (ind, GOT_ALIAS_SET);
12782 : if (TARGET_64BIT)
12783 : ind = convert_to_mode (Pmode, ind, 1);
12784 : ind = force_reg (Pmode, ind);
12785 :
12786 : /* Add it to the thread pointer and load the base. */
12787 : tp = get_thread_pointer (Pmode, true);
12788 : rtx addr = gen_rtx_PLUS (Pmode, tp,
12789 : gen_rtx_MULT (Pmode, ind,
12790 : GEN_INT (UNITS_PER_WORD)));
12791 : base = gen_const_mem (Pmode, addr);
12792 : set_mem_alias_set (base, GOT_ALIAS_SET);
12793 :
12794 : /* Add the 32-bit section-relative offset to the base. */
12795 : base = force_reg (Pmode, base);
12796 : off = gen_rtx_CONST (Pmode,
12797 : gen_rtx_UNSPEC (SImode,
12798 : gen_rtvec (1, x),
12799 : UNSPEC_SECREL32));
12800 : return gen_rtx_PLUS (Pmode, base, off);
12801 : }
12802 :
12803 : /* Fall back to global dynamic model if tool chain cannot support local
12804 : dynamic. */
12805 30973 : if (TARGET_SUN_TLS && !TARGET_64BIT
12806 : && !HAVE_AS_IX86_TLSLDMPLT && !HAVE_AS_IX86_TLSLDM
12807 : && model == TLS_MODEL_LOCAL_DYNAMIC)
12808 : model = TLS_MODEL_GLOBAL_DYNAMIC;
12809 :
12810 30973 : switch (model)
12811 : {
12812 6112 : case TLS_MODEL_GLOBAL_DYNAMIC:
12813 6112 : if (!TARGET_64BIT)
12814 : {
12815 1922 : if (flag_pic && !TARGET_PECOFF)
12816 1922 : pic = pic_offset_table_rtx;
12817 : else
12818 : {
12819 0 : pic = gen_reg_rtx (Pmode);
12820 0 : emit_insn (gen_set_got (pic));
12821 : }
12822 : }
12823 :
12824 6112 : if (TARGET_GNU2_TLS)
12825 : {
12826 53 : dest = gen_reg_rtx (ptr_mode);
12827 53 : if (TARGET_64BIT)
12828 53 : emit_insn (gen_tls_dynamic_gnu2_64 (ptr_mode, dest, x));
12829 : else
12830 0 : emit_insn (gen_tls_dynamic_gnu2_32 (dest, x, pic));
12831 :
12832 53 : tp = get_thread_pointer (ptr_mode, true);
12833 53 : dest = gen_rtx_PLUS (ptr_mode, tp, dest);
12834 61 : if (GET_MODE (dest) != Pmode)
12835 6 : dest = gen_rtx_ZERO_EXTEND (Pmode, dest);
12836 61 : dest = force_reg (Pmode, dest);
12837 :
12838 61 : if (GET_MODE (x) != Pmode)
12839 3 : x = gen_rtx_ZERO_EXTEND (Pmode, x);
12840 :
12841 53 : set_unique_reg_note (get_last_insn (), REG_EQUAL, x);
12842 : }
12843 : else
12844 : {
12845 6059 : rtx caddr = ix86_tls_get_addr ();
12846 :
12847 7981 : dest = gen_reg_rtx (Pmode);
12848 6059 : if (TARGET_64BIT)
12849 : {
12850 4137 : rtx rax = gen_rtx_REG (Pmode, AX_REG);
12851 4137 : rtx rdi = gen_rtx_REG (Pmode, DI_REG);
12852 4137 : rtx_insn *insns;
12853 :
12854 4137 : start_sequence ();
12855 4137 : rtx_insn *call_insn = emit_call_insn
12856 4137 : (gen_tls_global_dynamic_64 (Pmode, rax, x, caddr, rdi));
12857 4137 : CALL_INSN_ABI_ID (call_insn)
12858 4137 : = ix86_tls_get_addr_abi ().id ();
12859 4137 : insns = end_sequence ();
12860 :
12861 4137 : if (GET_MODE (x) != Pmode)
12862 1 : x = gen_rtx_ZERO_EXTEND (Pmode, x);
12863 :
12864 4137 : RTL_CONST_CALL_P (insns) = 1;
12865 4137 : emit_libcall_block (insns, dest, rax, x);
12866 : }
12867 : else
12868 1922 : emit_insn (gen_tls_global_dynamic_32 (dest, x, pic, caddr));
12869 : }
12870 : break;
12871 :
12872 394 : case TLS_MODEL_LOCAL_DYNAMIC:
12873 394 : if (!TARGET_64BIT)
12874 : {
12875 92 : if (flag_pic)
12876 92 : pic = pic_offset_table_rtx;
12877 : else
12878 : {
12879 0 : pic = gen_reg_rtx (Pmode);
12880 0 : emit_insn (gen_set_got (pic));
12881 : }
12882 : }
12883 :
12884 394 : if (TARGET_GNU2_TLS)
12885 : {
12886 26 : rtx tmp = ix86_tls_module_base ();
12887 :
12888 26 : base = gen_reg_rtx (ptr_mode);
12889 26 : if (TARGET_64BIT)
12890 26 : emit_insn (gen_tls_dynamic_gnu2_64 (ptr_mode, base, tmp));
12891 : else
12892 0 : emit_insn (gen_tls_dynamic_gnu2_32 (base, tmp, pic));
12893 :
12894 26 : tp = get_thread_pointer (ptr_mode, true);
12895 32 : if (GET_MODE (base) != Pmode)
12896 2 : base = gen_rtx_ZERO_EXTEND (Pmode, base);
12897 32 : base = force_reg (Pmode, base);
12898 : }
12899 : else
12900 : {
12901 368 : rtx caddr = ix86_tls_get_addr ();
12902 :
12903 460 : base = gen_reg_rtx (Pmode);
12904 368 : if (TARGET_64BIT)
12905 : {
12906 276 : rtx rax = gen_rtx_REG (Pmode, AX_REG);
12907 276 : rtx rdi = gen_rtx_REG (Pmode, DI_REG);
12908 276 : rtx_insn *insns;
12909 276 : rtx eqv;
12910 :
12911 276 : start_sequence ();
12912 276 : rtx_insn *call_insn = emit_call_insn
12913 276 : (gen_tls_local_dynamic_base_64 (Pmode, rax, caddr, rdi));
12914 276 : CALL_INSN_ABI_ID (call_insn)
12915 276 : = ix86_tls_get_addr_abi ().id ();
12916 276 : insns = end_sequence ();
12917 :
12918 : /* Attach a unique REG_EQUAL, to allow the RTL optimizers to
12919 : share the LD_BASE result with other LD model accesses. */
12920 276 : eqv = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx),
12921 : UNSPEC_TLS_LD_BASE);
12922 :
12923 276 : RTL_CONST_CALL_P (insns) = 1;
12924 276 : emit_libcall_block (insns, base, rax, eqv);
12925 : }
12926 : else
12927 92 : emit_insn (gen_tls_local_dynamic_base_32 (base, pic, caddr));
12928 : }
12929 :
12930 492 : off = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x), UNSPEC_DTPOFF);
12931 492 : off = gen_rtx_CONST (Pmode, off);
12932 :
12933 590 : dest = force_reg (Pmode, gen_rtx_PLUS (Pmode, base, off));
12934 :
12935 394 : if (TARGET_GNU2_TLS)
12936 : {
12937 32 : if (GET_MODE (tp) != Pmode)
12938 : {
12939 2 : dest = lowpart_subreg (ptr_mode, dest, Pmode);
12940 2 : dest = gen_rtx_PLUS (ptr_mode, tp, dest);
12941 2 : dest = gen_rtx_ZERO_EXTEND (Pmode, dest);
12942 : }
12943 : else
12944 24 : dest = gen_rtx_PLUS (Pmode, tp, dest);
12945 32 : dest = force_reg (Pmode, dest);
12946 :
12947 32 : if (GET_MODE (x) != Pmode)
12948 1 : x = gen_rtx_ZERO_EXTEND (Pmode, x);
12949 :
12950 26 : set_unique_reg_note (get_last_insn (), REG_EQUAL, x);
12951 : }
12952 : break;
12953 :
12954 10866 : case TLS_MODEL_INITIAL_EXEC:
12955 10866 : if (TARGET_64BIT)
12956 : {
12957 : /* Generate DImode references to avoid %fs:(%reg32)
12958 : problems and linker IE->LE relaxation bug. */
12959 : tp_mode = DImode;
12960 : pic = NULL;
12961 : type = UNSPEC_GOTNTPOFF;
12962 : }
12963 786 : else if (flag_pic)
12964 : {
12965 785 : pic = pic_offset_table_rtx;
12966 785 : type = TARGET_ANY_GNU_TLS ? UNSPEC_GOTNTPOFF : UNSPEC_GOTTPOFF;
12967 : }
12968 1 : else if (!TARGET_ANY_GNU_TLS)
12969 : {
12970 0 : pic = gen_reg_rtx (Pmode);
12971 0 : emit_insn (gen_set_got (pic));
12972 0 : type = UNSPEC_GOTTPOFF;
12973 : }
12974 : else
12975 : {
12976 : pic = NULL;
12977 : type = UNSPEC_INDNTPOFF;
12978 : }
12979 :
12980 10866 : off = gen_rtx_UNSPEC (tp_mode, gen_rtvec (1, x), type);
12981 10866 : off = gen_rtx_CONST (tp_mode, off);
12982 10866 : if (pic)
12983 785 : off = gen_rtx_PLUS (tp_mode, pic, off);
12984 10866 : off = gen_const_mem (tp_mode, off);
12985 10866 : set_mem_alias_set (off, GOT_ALIAS_SET);
12986 :
12987 10866 : if (TARGET_64BIT || TARGET_ANY_GNU_TLS)
12988 : {
12989 21732 : base = get_thread_pointer (tp_mode,
12990 8165 : for_mov || !TARGET_TLS_DIRECT_SEG_REFS);
12991 10866 : off = force_reg (tp_mode, off);
12992 10866 : dest = gen_rtx_PLUS (tp_mode, base, off);
12993 11656 : if (tp_mode != Pmode)
12994 4 : dest = convert_to_mode (Pmode, dest, 1);
12995 : }
12996 : else
12997 : {
12998 0 : base = get_thread_pointer (Pmode, true);
12999 0 : dest = gen_reg_rtx (Pmode);
13000 0 : emit_insn (gen_sub3_insn (dest, base, off));
13001 : }
13002 : break;
13003 :
13004 13601 : case TLS_MODEL_LOCAL_EXEC:
13005 28000 : off = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x),
13006 : (TARGET_64BIT || TARGET_ANY_GNU_TLS)
13007 : ? UNSPEC_NTPOFF : UNSPEC_TPOFF);
13008 14399 : off = gen_rtx_CONST (Pmode, off);
13009 :
13010 13601 : if (TARGET_64BIT || TARGET_ANY_GNU_TLS)
13011 : {
13012 28000 : base = get_thread_pointer (Pmode,
13013 8218 : for_mov || !TARGET_TLS_DIRECT_SEG_REFS);
13014 14399 : return gen_rtx_PLUS (Pmode, base, off);
13015 : }
13016 : else
13017 : {
13018 0 : base = get_thread_pointer (Pmode, true);
13019 0 : dest = gen_reg_rtx (Pmode);
13020 0 : emit_insn (gen_sub3_insn (dest, base, off));
13021 : }
13022 0 : break;
13023 :
13024 0 : default:
13025 0 : gcc_unreachable ();
13026 : }
13027 :
13028 : return dest;
13029 : }
13030 :
13031 : /* Return true if the TLS address requires insn using integer registers.
13032 : It's used to prevent KMOV/VMOV in TLS code sequences which require integer
13033 : MOV instructions, refer to PR103275. */
13034 : bool
13035 14900873 : ix86_gpr_tls_address_pattern_p (rtx mem)
13036 : {
13037 14900873 : gcc_assert (MEM_P (mem));
13038 :
13039 14900873 : rtx addr = XEXP (mem, 0);
13040 14900873 : subrtx_var_iterator::array_type array;
13041 51667031 : FOR_EACH_SUBRTX_VAR (iter, array, addr, ALL)
13042 : {
13043 36773807 : rtx op = *iter;
13044 36773807 : if (GET_CODE (op) == UNSPEC)
13045 230691 : switch (XINT (op, 1))
13046 : {
13047 : case UNSPEC_GOTNTPOFF:
13048 7649 : return true;
13049 0 : case UNSPEC_TPOFF:
13050 0 : if (!TARGET_64BIT)
13051 : return true;
13052 : break;
13053 : default:
13054 : break;
13055 : }
13056 : }
13057 :
13058 14893224 : return false;
13059 14900873 : }
13060 :
13061 : /* Return true if OP refers to a TLS address. */
13062 : bool
13063 239947773 : ix86_tls_address_pattern_p (rtx op)
13064 : {
13065 239947773 : subrtx_var_iterator::array_type array;
13066 1430928261 : FOR_EACH_SUBRTX_VAR (iter, array, op, ALL)
13067 : {
13068 1190998721 : rtx op = *iter;
13069 1190998721 : if (MEM_P (op))
13070 : {
13071 107404916 : rtx *x = &XEXP (op, 0);
13072 169870418 : while (GET_CODE (*x) == PLUS)
13073 : {
13074 : int i;
13075 187414762 : for (i = 0; i < 2; i++)
13076 : {
13077 124949260 : rtx u = XEXP (*x, i);
13078 124949260 : if (GET_CODE (u) == ZERO_EXTEND)
13079 31593 : u = XEXP (u, 0);
13080 124949260 : if (GET_CODE (u) == UNSPEC
13081 18265 : && XINT (u, 1) == UNSPEC_TP)
13082 18233 : return true;
13083 : }
13084 62465502 : x = &XEXP (*x, 0);
13085 : }
13086 :
13087 107386683 : iter.skip_subrtxes ();
13088 : }
13089 : }
13090 :
13091 239929540 : return false;
13092 239947773 : }
13093 :
13094 : /* Rewrite *LOC so that it refers to a default TLS address space. */
13095 : static void
13096 18233 : ix86_rewrite_tls_address_1 (rtx *loc)
13097 : {
13098 18233 : subrtx_ptr_iterator::array_type array;
13099 54277 : FOR_EACH_SUBRTX_PTR (iter, array, loc, ALL)
13100 : {
13101 54277 : rtx *loc = *iter;
13102 54277 : if (MEM_P (*loc))
13103 : {
13104 18422 : rtx addr = XEXP (*loc, 0);
13105 18422 : rtx *x = &addr;
13106 23333 : while (GET_CODE (*x) == PLUS)
13107 : {
13108 : int i;
13109 32989 : for (i = 0; i < 2; i++)
13110 : {
13111 28078 : rtx u = XEXP (*x, i);
13112 28078 : if (GET_CODE (u) == ZERO_EXTEND)
13113 19 : u = XEXP (u, 0);
13114 28078 : if (GET_CODE (u) == UNSPEC
13115 18233 : && XINT (u, 1) == UNSPEC_TP)
13116 : {
13117 : /* NB: Since address override only applies to the
13118 : (reg32) part in fs:(reg32), return if address
13119 : override is used. */
13120 19932 : if (Pmode != word_mode
13121 18233 : && REG_P (XEXP (*x, 1 - i)))
13122 18233 : return;
13123 :
13124 18231 : addr_space_t as = DEFAULT_TLS_SEG_REG;
13125 :
13126 18231 : *x = XEXP (*x, 1 - i);
13127 :
13128 18231 : *loc = replace_equiv_address_nv (*loc, addr, true);
13129 18231 : set_mem_addr_space (*loc, as);
13130 18231 : return;
13131 : }
13132 : }
13133 4911 : x = &XEXP (*x, 0);
13134 : }
13135 :
13136 189 : iter.skip_subrtxes ();
13137 : }
13138 : }
13139 18233 : }
13140 :
13141 : /* Rewrite instruction pattern involvning TLS address
13142 : so that it refers to a default TLS address space. */
13143 : rtx
13144 18233 : ix86_rewrite_tls_address (rtx pattern)
13145 : {
13146 18233 : pattern = copy_insn (pattern);
13147 18233 : ix86_rewrite_tls_address_1 (&pattern);
13148 18233 : return pattern;
13149 : }
13150 :
13151 : /* Try machine-dependent ways of modifying an illegitimate address
13152 : to be legitimate. If we find one, return the new, valid address.
13153 : This macro is used in only one place: `memory_address' in explow.cc.
13154 :
13155 : OLDX is the address as it was before break_out_memory_refs was called.
13156 : In some cases it is useful to look at this to decide what needs to be done.
13157 :
13158 : It is always safe for this macro to do nothing. It exists to recognize
13159 : opportunities to optimize the output.
13160 :
13161 : For the 80386, we handle X+REG by loading X into a register R and
13162 : using R+REG. R will go in a general reg and indexing will be used.
13163 : However, if REG is a broken-out memory address or multiplication,
13164 : nothing needs to be done because REG can certainly go in a general reg.
13165 :
13166 : When -fpic is used, special handling is needed for symbolic references.
13167 : See comments by legitimize_pic_address in i386.cc for details. */
13168 :
13169 : static rtx
13170 669398 : ix86_legitimize_address (rtx x, rtx, machine_mode mode)
13171 : {
13172 669398 : bool changed = false;
13173 669398 : unsigned log;
13174 :
13175 669398 : log = SYMBOL_REF_P (x) ? SYMBOL_REF_TLS_MODEL (x) : 0;
13176 152241 : if (log)
13177 20802 : return legitimize_tls_address (x, (enum tls_model) log, false);
13178 648596 : if (GET_CODE (x) == CONST
13179 508 : && GET_CODE (XEXP (x, 0)) == PLUS
13180 508 : && SYMBOL_REF_P (XEXP (XEXP (x, 0), 0))
13181 649104 : && (log = SYMBOL_REF_TLS_MODEL (XEXP (XEXP (x, 0), 0))))
13182 : {
13183 5 : rtx t = legitimize_tls_address (XEXP (XEXP (x, 0), 0),
13184 : (enum tls_model) log, false);
13185 6 : return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (x, 0), 1));
13186 : }
13187 :
13188 648591 : if (TARGET_DLLIMPORT_DECL_ATTRIBUTES)
13189 : {
13190 : #if TARGET_PECOFF
13191 : rtx tmp = legitimize_pe_coff_symbol (x, true);
13192 : if (tmp)
13193 : return tmp;
13194 : #endif
13195 : }
13196 :
13197 648591 : if (flag_pic && SYMBOLIC_CONST (x))
13198 131821 : return legitimize_pic_address (x, 0);
13199 :
13200 : #if TARGET_MACHO
13201 : if (MACHO_DYNAMIC_NO_PIC_P && SYMBOLIC_CONST (x))
13202 : return machopic_indirect_data_reference (x, 0);
13203 : #endif
13204 :
13205 : /* Canonicalize shifts by 0, 1, 2, 3 into multiply */
13206 516770 : if (GET_CODE (x) == ASHIFT
13207 0 : && CONST_INT_P (XEXP (x, 1))
13208 0 : && (unsigned HOST_WIDE_INT) INTVAL (XEXP (x, 1)) < 4)
13209 : {
13210 0 : changed = true;
13211 0 : log = INTVAL (XEXP (x, 1));
13212 0 : x = gen_rtx_MULT (Pmode, force_reg (Pmode, XEXP (x, 0)),
13213 : GEN_INT (1 << log));
13214 : }
13215 :
13216 516770 : if (GET_CODE (x) == PLUS)
13217 : {
13218 : /* Canonicalize shifts by 0, 1, 2, 3 into multiply. */
13219 :
13220 169112 : if (GET_CODE (XEXP (x, 0)) == ASHIFT
13221 609 : && CONST_INT_P (XEXP (XEXP (x, 0), 1))
13222 609 : && (unsigned HOST_WIDE_INT) INTVAL (XEXP (XEXP (x, 0), 1)) < 4)
13223 : {
13224 609 : changed = true;
13225 609 : log = INTVAL (XEXP (XEXP (x, 0), 1));
13226 1783 : XEXP (x, 0) = gen_rtx_MULT (Pmode,
13227 : force_reg (Pmode, XEXP (XEXP (x, 0), 0)),
13228 : GEN_INT (1 << log));
13229 : }
13230 :
13231 169112 : if (GET_CODE (XEXP (x, 1)) == ASHIFT
13232 0 : && CONST_INT_P (XEXP (XEXP (x, 1), 1))
13233 0 : && (unsigned HOST_WIDE_INT) INTVAL (XEXP (XEXP (x, 1), 1)) < 4)
13234 : {
13235 0 : changed = true;
13236 0 : log = INTVAL (XEXP (XEXP (x, 1), 1));
13237 0 : XEXP (x, 1) = gen_rtx_MULT (Pmode,
13238 : force_reg (Pmode, XEXP (XEXP (x, 1), 0)),
13239 : GEN_INT (1 << log));
13240 : }
13241 :
13242 : /* Put multiply first if it isn't already. */
13243 169112 : if (GET_CODE (XEXP (x, 1)) == MULT)
13244 : {
13245 0 : std::swap (XEXP (x, 0), XEXP (x, 1));
13246 0 : changed = true;
13247 : }
13248 :
13249 : /* Canonicalize (plus (mult (reg) (const)) (plus (reg) (const)))
13250 : into (plus (plus (mult (reg) (const)) (reg)) (const)). This can be
13251 : created by virtual register instantiation, register elimination, and
13252 : similar optimizations. */
13253 169112 : if (GET_CODE (XEXP (x, 0)) == MULT && GET_CODE (XEXP (x, 1)) == PLUS)
13254 : {
13255 8952 : changed = true;
13256 13878 : x = gen_rtx_PLUS (Pmode,
13257 : gen_rtx_PLUS (Pmode, XEXP (x, 0),
13258 : XEXP (XEXP (x, 1), 0)),
13259 : XEXP (XEXP (x, 1), 1));
13260 : }
13261 :
13262 : /* Canonicalize
13263 : (plus (plus (mult (reg) (const)) (plus (reg) (const))) const)
13264 : into (plus (plus (mult (reg) (const)) (reg)) (const)). */
13265 160160 : else if (GET_CODE (x) == PLUS && GET_CODE (XEXP (x, 0)) == PLUS
13266 61044 : && GET_CODE (XEXP (XEXP (x, 0), 0)) == MULT
13267 52076 : && GET_CODE (XEXP (XEXP (x, 0), 1)) == PLUS
13268 0 : && CONSTANT_P (XEXP (x, 1)))
13269 : {
13270 0 : rtx constant;
13271 0 : rtx other = NULL_RTX;
13272 :
13273 0 : if (CONST_INT_P (XEXP (x, 1)))
13274 : {
13275 0 : constant = XEXP (x, 1);
13276 0 : other = XEXP (XEXP (XEXP (x, 0), 1), 1);
13277 : }
13278 0 : else if (CONST_INT_P (XEXP (XEXP (XEXP (x, 0), 1), 1)))
13279 : {
13280 : constant = XEXP (XEXP (XEXP (x, 0), 1), 1);
13281 : other = XEXP (x, 1);
13282 : }
13283 : else
13284 : constant = 0;
13285 :
13286 0 : if (constant)
13287 : {
13288 0 : changed = true;
13289 0 : x = gen_rtx_PLUS (Pmode,
13290 : gen_rtx_PLUS (Pmode, XEXP (XEXP (x, 0), 0),
13291 : XEXP (XEXP (XEXP (x, 0), 1), 0)),
13292 : plus_constant (Pmode, other,
13293 : INTVAL (constant)));
13294 : }
13295 : }
13296 :
13297 169112 : if (changed && ix86_legitimate_address_p (mode, x, false))
13298 8989 : return x;
13299 :
13300 160123 : if (GET_CODE (XEXP (x, 0)) == MULT)
13301 : {
13302 19824 : changed = true;
13303 19824 : XEXP (x, 0) = copy_addr_to_reg (XEXP (x, 0));
13304 : }
13305 :
13306 160123 : if (GET_CODE (XEXP (x, 1)) == MULT)
13307 : {
13308 0 : changed = true;
13309 0 : XEXP (x, 1) = copy_addr_to_reg (XEXP (x, 1));
13310 : }
13311 :
13312 160123 : if (changed
13313 19830 : && REG_P (XEXP (x, 1))
13314 16135 : && REG_P (XEXP (x, 0)))
13315 : return x;
13316 :
13317 143988 : if (flag_pic && SYMBOLIC_CONST (XEXP (x, 1)))
13318 : {
13319 1852 : changed = true;
13320 1852 : x = legitimize_pic_address (x, 0);
13321 : }
13322 :
13323 143988 : if (changed && ix86_legitimate_address_p (mode, x, false))
13324 3972 : return x;
13325 :
13326 140016 : if (REG_P (XEXP (x, 0)))
13327 : {
13328 81273 : rtx temp = gen_reg_rtx (Pmode);
13329 78513 : rtx val = force_operand (XEXP (x, 1), temp);
13330 78513 : if (val != temp)
13331 : {
13332 71597 : val = convert_to_mode (Pmode, val, 1);
13333 71273 : emit_move_insn (temp, val);
13334 : }
13335 :
13336 78513 : XEXP (x, 1) = temp;
13337 78513 : return x;
13338 : }
13339 :
13340 61503 : else if (REG_P (XEXP (x, 1)))
13341 : {
13342 4246 : rtx temp = gen_reg_rtx (Pmode);
13343 3383 : rtx val = force_operand (XEXP (x, 0), temp);
13344 3383 : if (val != temp)
13345 : {
13346 0 : val = convert_to_mode (Pmode, val, 1);
13347 0 : emit_move_insn (temp, val);
13348 : }
13349 :
13350 3383 : XEXP (x, 0) = temp;
13351 3383 : return x;
13352 : }
13353 : }
13354 :
13355 : return x;
13356 : }
13357 :
13358 : /* Print an integer constant expression in assembler syntax. Addition
13359 : and subtraction are the only arithmetic that may appear in these
13360 : expressions. FILE is the stdio stream to write to, X is the rtx, and
13361 : CODE is the operand print code from the output string. */
13362 :
13363 : static void
13364 5063392 : output_pic_addr_const (FILE *file, rtx x, int code)
13365 : {
13366 5416376 : char buf[256];
13367 :
13368 5416376 : switch (GET_CODE (x))
13369 : {
13370 0 : case PC:
13371 0 : gcc_assert (flag_pic);
13372 0 : putc ('.', file);
13373 0 : break;
13374 :
13375 1572545 : case SYMBOL_REF:
13376 1572545 : if (TARGET_64BIT || ! TARGET_MACHO_SYMBOL_STUBS)
13377 1572545 : output_addr_const (file, x);
13378 : else
13379 : {
13380 : const char *name = XSTR (x, 0);
13381 :
13382 : /* Mark the decl as referenced so that cgraph will
13383 : output the function. */
13384 : if (SYMBOL_REF_DECL (x))
13385 : mark_decl_referenced (SYMBOL_REF_DECL (x));
13386 :
13387 : #if TARGET_MACHO
13388 : if (MACHOPIC_INDIRECT
13389 : && machopic_classify_symbol (x) == MACHOPIC_UNDEFINED_FUNCTION)
13390 : name = machopic_indirection_name (x, /*stub_p=*/true);
13391 : #endif
13392 : assemble_name (file, name);
13393 : }
13394 1572545 : if (!TARGET_MACHO && !(TARGET_64BIT && TARGET_PECOFF)
13395 1572545 : && code == 'P' && ix86_call_use_plt_p (x))
13396 548181 : fputs ("@PLT", file);
13397 : break;
13398 :
13399 2717 : case LABEL_REF:
13400 2717 : x = XEXP (x, 0);
13401 : /* FALLTHRU */
13402 2717 : case CODE_LABEL:
13403 2717 : ASM_GENERATE_INTERNAL_LABEL (buf, "L", CODE_LABEL_NUMBER (x));
13404 2717 : assemble_name (asm_out_file, buf);
13405 2717 : break;
13406 :
13407 3257282 : CASE_CONST_SCALAR_INT:
13408 3257282 : output_addr_const (file, x);
13409 3257282 : break;
13410 :
13411 287240 : case CONST:
13412 : /* This used to output parentheses around the expression,
13413 : but that does not work on the 386 (either ATT or BSD assembler). */
13414 287240 : output_pic_addr_const (file, XEXP (x, 0), code);
13415 287240 : break;
13416 :
13417 0 : case CONST_DOUBLE:
13418 : /* We can't handle floating point constants;
13419 : TARGET_PRINT_OPERAND must handle them. */
13420 0 : output_operand_lossage ("floating constant misused");
13421 0 : break;
13422 :
13423 65744 : case PLUS:
13424 : /* Some assemblers need integer constants to appear first. */
13425 65744 : if (CONST_INT_P (XEXP (x, 0)))
13426 : {
13427 0 : output_pic_addr_const (file, XEXP (x, 0), code);
13428 0 : putc ('+', file);
13429 0 : output_pic_addr_const (file, XEXP (x, 1), code);
13430 : }
13431 : else
13432 : {
13433 65744 : gcc_assert (CONST_INT_P (XEXP (x, 1)));
13434 65744 : output_pic_addr_const (file, XEXP (x, 1), code);
13435 65744 : putc ('+', file);
13436 65744 : output_pic_addr_const (file, XEXP (x, 0), code);
13437 : }
13438 : break;
13439 :
13440 0 : case MINUS:
13441 0 : if (!TARGET_MACHO)
13442 0 : putc (ASSEMBLER_DIALECT == ASM_INTEL ? '(' : '[', file);
13443 0 : output_pic_addr_const (file, XEXP (x, 0), code);
13444 0 : putc ('-', file);
13445 0 : output_pic_addr_const (file, XEXP (x, 1), code);
13446 0 : if (!TARGET_MACHO)
13447 0 : putc (ASSEMBLER_DIALECT == ASM_INTEL ? ')' : ']', file);
13448 0 : break;
13449 :
13450 230848 : case UNSPEC:
13451 230848 : gcc_assert (XVECLEN (x, 0) == 1);
13452 230848 : output_pic_addr_const (file, XVECEXP (x, 0, 0), code);
13453 230848 : switch (XINT (x, 1))
13454 : {
13455 43354 : case UNSPEC_GOT:
13456 43354 : fputs ("@GOT", file);
13457 43354 : break;
13458 78006 : case UNSPEC_GOTOFF:
13459 78006 : fputs ("@GOTOFF", file);
13460 78006 : break;
13461 36 : case UNSPEC_PLTOFF:
13462 36 : fputs ("@PLTOFF", file);
13463 36 : break;
13464 0 : case UNSPEC_PCREL:
13465 0 : fputs (ASSEMBLER_DIALECT == ASM_ATT ?
13466 : "(%rip)" : "[rip]", file);
13467 0 : break;
13468 105233 : case UNSPEC_GOTPCREL:
13469 105233 : fputs (ASSEMBLER_DIALECT == ASM_ATT ?
13470 : "@GOTPCREL(%rip)" : "@GOTPCREL[rip]", file);
13471 105233 : break;
13472 0 : case UNSPEC_GOTTPOFF:
13473 : /* FIXME: This might be @TPOFF in Sun ld too. */
13474 0 : fputs ("@gottpoff", file);
13475 0 : break;
13476 0 : case UNSPEC_TPOFF:
13477 0 : fputs ("@tpoff", file);
13478 0 : break;
13479 1459 : case UNSPEC_NTPOFF:
13480 1459 : if (TARGET_64BIT)
13481 1459 : fputs ("@tpoff", file);
13482 : else
13483 0 : fputs ("@ntpoff", file);
13484 : break;
13485 319 : case UNSPEC_DTPOFF:
13486 319 : fputs ("@dtpoff", file);
13487 319 : break;
13488 2441 : case UNSPEC_GOTNTPOFF:
13489 2441 : if (TARGET_64BIT)
13490 2164 : fputs (ASSEMBLER_DIALECT == ASM_ATT ?
13491 : "@gottpoff(%rip)": "@gottpoff[rip]", file);
13492 : else
13493 277 : fputs ("@gotntpoff", file);
13494 : break;
13495 0 : case UNSPEC_INDNTPOFF:
13496 0 : fputs ("@indntpoff", file);
13497 0 : break;
13498 0 : case UNSPEC_SECREL32:
13499 0 : fputs ("@secrel32", file);
13500 0 : break;
13501 : #if TARGET_MACHO
13502 : case UNSPEC_MACHOPIC_OFFSET:
13503 : putc ('-', file);
13504 : machopic_output_function_base_name (file);
13505 : break;
13506 : #endif
13507 0 : default:
13508 0 : output_operand_lossage ("invalid UNSPEC as operand");
13509 0 : break;
13510 : }
13511 : break;
13512 :
13513 0 : default:
13514 0 : output_operand_lossage ("invalid expression as operand");
13515 : }
13516 5063392 : }
13517 :
13518 : /* This is called from dwarf2out.cc via TARGET_ASM_OUTPUT_DWARF_DTPREL.
13519 : We need to emit DTP-relative relocations. */
13520 :
13521 : static void ATTRIBUTE_UNUSED
13522 685 : i386_output_dwarf_dtprel (FILE *file, int size, rtx x)
13523 : {
13524 685 : fputs (ASM_LONG, file);
13525 685 : output_addr_const (file, x);
13526 : #if TARGET_WIN32_TLS
13527 : fputs ("@secrel32", file);
13528 : #else
13529 685 : fputs ("@dtpoff", file);
13530 : #endif
13531 685 : switch (size)
13532 : {
13533 : case 4:
13534 : break;
13535 553 : case 8:
13536 553 : fputs (", 0", file);
13537 553 : break;
13538 0 : default:
13539 0 : gcc_unreachable ();
13540 : }
13541 685 : }
13542 :
13543 : /* Return true if X is a representation of the PIC register. This copes
13544 : with calls from ix86_find_base_term, where the register might have
13545 : been replaced by a cselib value. */
13546 :
13547 : static bool
13548 26871888 : ix86_pic_register_p (rtx x)
13549 : {
13550 26871888 : if (GET_CODE (x) == VALUE && CSELIB_VAL_PTR (x))
13551 768179 : return (pic_offset_table_rtx
13552 768179 : && rtx_equal_for_cselib_p (x, pic_offset_table_rtx));
13553 26103709 : else if (GET_CODE (x) == UNSPEC && XINT (x, 1) == UNSPEC_SET_GOT)
13554 : return true;
13555 26100880 : else if (!REG_P (x))
13556 : return false;
13557 25495210 : else if (pic_offset_table_rtx)
13558 : {
13559 25475575 : if (REGNO (x) == REGNO (pic_offset_table_rtx))
13560 : return true;
13561 402795 : if (HARD_REGISTER_P (x)
13562 381066 : && !HARD_REGISTER_P (pic_offset_table_rtx)
13563 783861 : && ORIGINAL_REGNO (x) == REGNO (pic_offset_table_rtx))
13564 : return true;
13565 : return false;
13566 : }
13567 : else
13568 19635 : return REGNO (x) == PIC_OFFSET_TABLE_REGNUM;
13569 : }
13570 :
13571 : /* Helper function for ix86_delegitimize_address.
13572 : Attempt to delegitimize TLS local-exec accesses. */
13573 :
13574 : static rtx
13575 3548095329 : ix86_delegitimize_tls_address (rtx orig_x)
13576 : {
13577 3548095329 : rtx x = orig_x, unspec;
13578 3548095329 : struct ix86_address addr;
13579 :
13580 3548095329 : if (!TARGET_TLS_DIRECT_SEG_REFS)
13581 : return orig_x;
13582 3548095329 : if (MEM_P (x))
13583 43963565 : x = XEXP (x, 0);
13584 5092592377 : if (GET_CODE (x) != PLUS || GET_MODE (x) != Pmode)
13585 : return orig_x;
13586 1709772150 : if (ix86_decompose_address (x, &addr) == 0
13587 1978934883 : || addr.seg != DEFAULT_TLS_SEG_REG
13588 270763 : || addr.disp == NULL_RTX
13589 1709990951 : || GET_CODE (addr.disp) != CONST)
13590 : return orig_x;
13591 112893 : unspec = XEXP (addr.disp, 0);
13592 112893 : if (GET_CODE (unspec) == PLUS && CONST_INT_P (XEXP (unspec, 1)))
13593 66056 : unspec = XEXP (unspec, 0);
13594 112893 : if (GET_CODE (unspec) != UNSPEC || XINT (unspec, 1) != UNSPEC_NTPOFF)
13595 : return orig_x;
13596 112830 : x = XVECEXP (unspec, 0, 0);
13597 112830 : gcc_assert (SYMBOL_REF_P (x));
13598 112830 : if (unspec != XEXP (addr.disp, 0))
13599 87545 : x = gen_rtx_PLUS (Pmode, x, XEXP (XEXP (addr.disp, 0), 1));
13600 112830 : if (addr.index)
13601 : {
13602 185 : rtx idx = addr.index;
13603 185 : if (addr.scale != 1)
13604 185 : idx = gen_rtx_MULT (Pmode, idx, GEN_INT (addr.scale));
13605 185 : x = gen_rtx_PLUS (Pmode, idx, x);
13606 : }
13607 112830 : if (addr.base)
13608 2 : x = gen_rtx_PLUS (Pmode, addr.base, x);
13609 112830 : if (MEM_P (orig_x))
13610 198 : x = replace_equiv_address_nv (orig_x, x);
13611 : return x;
13612 : }
13613 :
13614 : /* In the name of slightly smaller debug output, and to cater to
13615 : general assembler lossage, recognize PIC+GOTOFF and turn it back
13616 : into a direct symbol reference.
13617 :
13618 : On Darwin, this is necessary to avoid a crash, because Darwin
13619 : has a different PIC label for each routine but the DWARF debugging
13620 : information is not associated with any particular routine, so it's
13621 : necessary to remove references to the PIC label from RTL stored by
13622 : the DWARF output code.
13623 :
13624 : This helper is used in the normal ix86_delegitimize_address
13625 : entrypoint (e.g. used in the target delegitimization hook) and
13626 : in ix86_find_base_term. As compile time memory optimization, we
13627 : avoid allocating rtxes that will not change anything on the outcome
13628 : of the callers (find_base_value and find_base_term). */
13629 :
13630 : static inline rtx
13631 3572932462 : ix86_delegitimize_address_1 (rtx x, bool base_term_p)
13632 : {
13633 3572932462 : rtx orig_x = delegitimize_mem_from_attrs (x);
13634 : /* addend is NULL or some rtx if x is something+GOTOFF where
13635 : something doesn't include the PIC register. */
13636 3572932462 : rtx addend = NULL_RTX;
13637 : /* reg_addend is NULL or a multiple of some register. */
13638 3572932462 : rtx reg_addend = NULL_RTX;
13639 : /* const_addend is NULL or a const_int. */
13640 3572932462 : rtx const_addend = NULL_RTX;
13641 : /* This is the result, or NULL. */
13642 3572932462 : rtx result = NULL_RTX;
13643 :
13644 3572932462 : x = orig_x;
13645 :
13646 3572932462 : if (MEM_P (x))
13647 63167823 : x = XEXP (x, 0);
13648 :
13649 3572932462 : if (TARGET_64BIT)
13650 : {
13651 266354754 : if (GET_CODE (x) == CONST
13652 8805197 : && GET_CODE (XEXP (x, 0)) == PLUS
13653 6855518 : && GET_MODE (XEXP (x, 0)) == Pmode
13654 6855469 : && CONST_INT_P (XEXP (XEXP (x, 0), 1))
13655 6855469 : && GET_CODE (XEXP (XEXP (x, 0), 0)) == UNSPEC
13656 266359113 : && XINT (XEXP (XEXP (x, 0), 0), 1) == UNSPEC_PCREL)
13657 : {
13658 : /* find_base_{value,term} only care about MEMs with arg_pointer_rtx
13659 : base. A CONST can't be arg_pointer_rtx based. */
13660 0 : if (base_term_p && MEM_P (orig_x))
13661 : return orig_x;
13662 0 : rtx x2 = XVECEXP (XEXP (XEXP (x, 0), 0), 0, 0);
13663 0 : x = gen_rtx_PLUS (Pmode, XEXP (XEXP (x, 0), 1), x2);
13664 0 : if (MEM_P (orig_x))
13665 0 : x = replace_equiv_address_nv (orig_x, x);
13666 : return x;
13667 : }
13668 :
13669 266354754 : if (GET_CODE (x) == CONST
13670 8805197 : && GET_CODE (XEXP (x, 0)) == UNSPEC
13671 1949728 : && (XINT (XEXP (x, 0), 1) == UNSPEC_GOTPCREL
13672 657126 : || XINT (XEXP (x, 0), 1) == UNSPEC_PCREL)
13673 1292602 : && (MEM_P (orig_x) || XINT (XEXP (x, 0), 1) == UNSPEC_PCREL))
13674 : {
13675 296637 : x = XVECEXP (XEXP (x, 0), 0, 0);
13676 296637 : if (GET_MODE (orig_x) != GET_MODE (x) && MEM_P (orig_x))
13677 : {
13678 9 : x = lowpart_subreg (GET_MODE (orig_x), x, GET_MODE (x));
13679 9 : if (x == NULL_RTX)
13680 : return orig_x;
13681 : }
13682 296637 : return x;
13683 : }
13684 :
13685 266058117 : if (ix86_cmodel != CM_MEDIUM_PIC && ix86_cmodel != CM_LARGE_PIC)
13686 266056434 : return ix86_delegitimize_tls_address (orig_x);
13687 :
13688 : /* Fall thru into the code shared with -m32 for -mcmodel=large -fpic
13689 : and -mcmodel=medium -fpic. */
13690 : }
13691 :
13692 3306579391 : if (GET_CODE (x) != PLUS
13693 1569036414 : || GET_CODE (XEXP (x, 1)) != CONST)
13694 3280258032 : return ix86_delegitimize_tls_address (orig_x);
13695 :
13696 26321359 : if (ix86_pic_register_p (XEXP (x, 0)))
13697 : /* %ebx + GOT/GOTOFF */
13698 : ;
13699 1294120 : else if (GET_CODE (XEXP (x, 0)) == PLUS)
13700 : {
13701 : /* %ebx + %reg * scale + GOT/GOTOFF */
13702 473132 : reg_addend = XEXP (x, 0);
13703 473132 : if (ix86_pic_register_p (XEXP (reg_addend, 0)))
13704 395735 : reg_addend = XEXP (reg_addend, 1);
13705 77397 : else if (ix86_pic_register_p (XEXP (reg_addend, 1)))
13706 45121 : reg_addend = XEXP (reg_addend, 0);
13707 : else
13708 : {
13709 32276 : reg_addend = NULL_RTX;
13710 32276 : addend = XEXP (x, 0);
13711 : }
13712 : }
13713 : else
13714 : addend = XEXP (x, 0);
13715 :
13716 26321359 : x = XEXP (XEXP (x, 1), 0);
13717 26321359 : if (GET_CODE (x) == PLUS
13718 1457328 : && CONST_INT_P (XEXP (x, 1)))
13719 : {
13720 1457328 : const_addend = XEXP (x, 1);
13721 1457328 : x = XEXP (x, 0);
13722 : }
13723 :
13724 26321359 : if (GET_CODE (x) == UNSPEC
13725 25634413 : && ((XINT (x, 1) == UNSPEC_GOT && MEM_P (orig_x) && !addend)
13726 6726792 : || (XINT (x, 1) == UNSPEC_GOTOFF && !MEM_P (orig_x))
13727 1093921 : || (XINT (x, 1) == UNSPEC_PLTOFF && ix86_cmodel == CM_LARGE_PIC
13728 4 : && !MEM_P (orig_x) && !addend)))
13729 24540496 : result = XVECEXP (x, 0, 0);
13730 :
13731 24540496 : if (!TARGET_64BIT && TARGET_MACHO && darwin_local_data_pic (x)
13732 : && !MEM_P (orig_x))
13733 : result = XVECEXP (x, 0, 0);
13734 :
13735 24540496 : if (! result)
13736 1780863 : return ix86_delegitimize_tls_address (orig_x);
13737 :
13738 : /* For (PLUS something CONST_INT) both find_base_{value,term} just
13739 : recurse on the first operand. */
13740 24540496 : if (const_addend && !base_term_p)
13741 341022 : result = gen_rtx_CONST (Pmode, gen_rtx_PLUS (Pmode, result, const_addend));
13742 24540496 : if (reg_addend)
13743 857246 : result = gen_rtx_PLUS (Pmode, reg_addend, result);
13744 24540496 : if (addend)
13745 : {
13746 : /* If the rest of original X doesn't involve the PIC register, add
13747 : addend and subtract pic_offset_table_rtx. This can happen e.g.
13748 : for code like:
13749 : leal (%ebx, %ecx, 4), %ecx
13750 : ...
13751 : movl foo@GOTOFF(%ecx), %edx
13752 : in which case we return (%ecx - %ebx) + foo
13753 : or (%ecx - _GLOBAL_OFFSET_TABLE_) + foo if pseudo_pic_reg
13754 : and reload has completed. Don't do the latter for debug,
13755 : as _GLOBAL_OFFSET_TABLE_ can't be expressed in the assembly. */
13756 139208 : if (pic_offset_table_rtx
13757 139208 : && (!reload_completed || !ix86_use_pseudo_pic_reg ()))
13758 2328 : result = gen_rtx_PLUS (Pmode, gen_rtx_MINUS (Pmode, copy_rtx (addend),
13759 : pic_offset_table_rtx),
13760 : result);
13761 138432 : else if (base_term_p
13762 133705 : && pic_offset_table_rtx
13763 : && !TARGET_MACHO
13764 : && !TARGET_VXWORKS_VAROFF)
13765 : {
13766 267410 : rtx tmp = gen_rtx_SYMBOL_REF (Pmode, GOT_SYMBOL_NAME);
13767 267410 : tmp = gen_rtx_MINUS (Pmode, copy_rtx (addend), tmp);
13768 267410 : result = gen_rtx_PLUS (Pmode, tmp, result);
13769 133705 : }
13770 : else
13771 : return orig_x;
13772 : }
13773 49071453 : if (GET_MODE (orig_x) != Pmode && MEM_P (orig_x))
13774 : {
13775 0 : result = lowpart_subreg (GET_MODE (orig_x), result, Pmode);
13776 0 : if (result == NULL_RTX)
13777 0 : return orig_x;
13778 : }
13779 : return result;
13780 : }
13781 :
13782 : /* The normal instantiation of the above template. */
13783 :
13784 : static rtx
13785 338851296 : ix86_delegitimize_address (rtx x)
13786 : {
13787 338851296 : return ix86_delegitimize_address_1 (x, false);
13788 : }
13789 :
13790 : /* If X is a machine specific address (i.e. a symbol or label being
13791 : referenced as a displacement from the GOT implemented using an
13792 : UNSPEC), then return the base term. Otherwise return X. */
13793 :
13794 : rtx
13795 6803784030 : ix86_find_base_term (rtx x)
13796 : {
13797 6803784030 : rtx term;
13798 :
13799 6803784030 : if (TARGET_64BIT)
13800 : {
13801 3569702864 : if (GET_CODE (x) != CONST)
13802 : return x;
13803 42242475 : term = XEXP (x, 0);
13804 42242475 : if (GET_CODE (term) == PLUS
13805 42227595 : && CONST_INT_P (XEXP (term, 1)))
13806 42227595 : term = XEXP (term, 0);
13807 42242475 : if (GET_CODE (term) != UNSPEC
13808 39826 : || (XINT (term, 1) != UNSPEC_GOTPCREL
13809 39794 : && XINT (term, 1) != UNSPEC_PCREL))
13810 : return x;
13811 :
13812 32 : return XVECEXP (term, 0, 0);
13813 : }
13814 :
13815 3234081166 : return ix86_delegitimize_address_1 (x, true);
13816 : }
13817 :
13818 : /* Return true if X shouldn't be emitted into the debug info.
13819 : Disallow UNSPECs other than @gotoff - we can't emit _GLOBAL_OFFSET_TABLE_
13820 : symbol easily into the .debug_info section, so we need not to
13821 : delegitimize, but instead assemble as @gotoff.
13822 : Disallow _GLOBAL_OFFSET_TABLE_ SYMBOL_REF - the assembler magically
13823 : assembles that as _GLOBAL_OFFSET_TABLE_-. expression. */
13824 :
13825 : static bool
13826 1961452 : ix86_const_not_ok_for_debug_p (rtx x)
13827 : {
13828 1961452 : if (GET_CODE (x) == UNSPEC && XINT (x, 1) != UNSPEC_GOTOFF)
13829 : return true;
13830 :
13831 1961432 : if (SYMBOL_REF_P (x) && strcmp (XSTR (x, 0), GOT_SYMBOL_NAME) == 0)
13832 0 : return true;
13833 :
13834 : return false;
13835 : }
13836 :
13837 : static void
13838 7337517 : put_condition_code (enum rtx_code code, machine_mode mode, bool reverse,
13839 : bool fp, FILE *file)
13840 : {
13841 7337517 : const char *suffix;
13842 :
13843 7337517 : if (mode == CCFPmode)
13844 : {
13845 573370 : code = ix86_fp_compare_code_to_integer (code);
13846 573370 : mode = CCmode;
13847 : }
13848 7337517 : if (reverse)
13849 243908 : code = reverse_condition (code);
13850 :
13851 7337517 : switch (code)
13852 : {
13853 2859108 : case EQ:
13854 2859108 : gcc_assert (mode != CCGZmode);
13855 2859108 : switch (mode)
13856 : {
13857 : case E_CCAmode:
13858 : suffix = "a";
13859 : break;
13860 : case E_CCCmode:
13861 28888 : suffix = "c";
13862 : break;
13863 : case E_CCOmode:
13864 7337517 : suffix = "o";
13865 : break;
13866 : case E_CCPmode:
13867 235217 : suffix = "p";
13868 : break;
13869 : case E_CCSmode:
13870 133375 : suffix = "s";
13871 : break;
13872 2839123 : default:
13873 2839123 : suffix = "e";
13874 2839123 : break;
13875 : }
13876 : break;
13877 2379635 : case NE:
13878 2379635 : gcc_assert (mode != CCGZmode);
13879 2379635 : switch (mode)
13880 : {
13881 : case E_CCAmode:
13882 : suffix = "na";
13883 : break;
13884 : case E_CCCmode:
13885 12073 : suffix = "nc";
13886 : break;
13887 11025 : case E_CCOmode:
13888 11025 : suffix = "no";
13889 11025 : break;
13890 : case E_CCPmode:
13891 5011 : suffix = "np";
13892 : break;
13893 : case E_CCSmode:
13894 53752 : suffix = "ns";
13895 : break;
13896 2366723 : default:
13897 2366723 : suffix = "ne";
13898 2366723 : break;
13899 : }
13900 : break;
13901 258603 : case GT:
13902 258603 : gcc_assert (mode == CCmode || mode == CCNOmode || mode == CCGCmode);
13903 : suffix = "g";
13904 : break;
13905 191163 : case GTU:
13906 : /* ??? Use "nbe" instead of "a" for fcmov lossage on some assemblers.
13907 : Those same assemblers have the same but opposite lossage on cmov. */
13908 191163 : if (mode == CCmode)
13909 191225 : suffix = fp ? "nbe" : "a";
13910 : else
13911 0 : gcc_unreachable ();
13912 : break;
13913 232907 : case LT:
13914 232907 : switch (mode)
13915 : {
13916 : case E_CCNOmode:
13917 : case E_CCGOCmode:
13918 : suffix = "s";
13919 : break;
13920 :
13921 : case E_CCmode:
13922 : case E_CCGCmode:
13923 : case E_CCGZmode:
13924 7337517 : suffix = "l";
13925 : break;
13926 :
13927 0 : default:
13928 0 : gcc_unreachable ();
13929 : }
13930 : break;
13931 441681 : case LTU:
13932 441681 : if (mode == CCmode || mode == CCGZmode)
13933 : suffix = "b";
13934 27558 : else if (mode == CCCmode)
13935 28888 : suffix = fp ? "b" : "c";
13936 : else
13937 0 : gcc_unreachable ();
13938 : break;
13939 146880 : case GE:
13940 146880 : switch (mode)
13941 : {
13942 : case E_CCNOmode:
13943 : case E_CCGOCmode:
13944 : suffix = "ns";
13945 : break;
13946 :
13947 : case E_CCmode:
13948 : case E_CCGCmode:
13949 : case E_CCGZmode:
13950 7337517 : suffix = "ge";
13951 : break;
13952 :
13953 0 : default:
13954 0 : gcc_unreachable ();
13955 : }
13956 : break;
13957 202539 : case GEU:
13958 202539 : if (mode == CCmode || mode == CCGZmode)
13959 : suffix = "nb";
13960 10206 : else if (mode == CCCmode)
13961 12073 : suffix = fp ? "nb" : "nc";
13962 : else
13963 0 : gcc_unreachable ();
13964 : break;
13965 256432 : case LE:
13966 256432 : gcc_assert (mode == CCmode || mode == CCGCmode || mode == CCNOmode);
13967 : suffix = "le";
13968 : break;
13969 128339 : case LEU:
13970 128339 : if (mode == CCmode)
13971 : suffix = "be";
13972 : else
13973 0 : gcc_unreachable ();
13974 : break;
13975 235217 : case UNORDERED:
13976 235224 : suffix = fp ? "u" : "p";
13977 : break;
13978 5013 : case ORDERED:
13979 5018 : suffix = fp ? "nu" : "np";
13980 : break;
13981 0 : default:
13982 0 : gcc_unreachable ();
13983 : }
13984 7337517 : fputs (suffix, file);
13985 7337517 : }
13986 :
13987 : /* Print the name of register X to FILE based on its machine mode and number.
13988 : If CODE is 'w', pretend the mode is HImode.
13989 : If CODE is 'b', pretend the mode is QImode.
13990 : If CODE is 'k', pretend the mode is SImode.
13991 : If CODE is 'q', pretend the mode is DImode.
13992 : If CODE is 'x', pretend the mode is V4SFmode.
13993 : If CODE is 't', pretend the mode is V8SFmode.
13994 : If CODE is 'g', pretend the mode is V16SFmode.
13995 : If CODE is 'h', pretend the reg is the 'high' byte register.
13996 : If CODE is 'y', print "st(0)" instead of "st", if the reg is stack op.
13997 : If CODE is 'd', duplicate the operand for AVX instruction.
13998 : If CODE is 'V', print naked full integer register name without %.
13999 : */
14000 :
14001 : void
14002 127060576 : print_reg (rtx x, int code, FILE *file)
14003 : {
14004 127060576 : const char *reg;
14005 127060576 : int msize;
14006 127060576 : unsigned int regno;
14007 127060576 : bool duplicated;
14008 :
14009 127060576 : if (ASSEMBLER_DIALECT == ASM_ATT && code != 'V')
14010 127058129 : putc ('%', file);
14011 :
14012 127060576 : if (x == pc_rtx)
14013 : {
14014 6079856 : gcc_assert (TARGET_64BIT);
14015 6079856 : fputs ("rip", file);
14016 6079856 : return;
14017 : }
14018 :
14019 120980720 : if (code == 'y' && STACK_TOP_P (x))
14020 : {
14021 290488 : fputs ("st(0)", file);
14022 290488 : return;
14023 : }
14024 :
14025 120690232 : if (code == 'w')
14026 : msize = 2;
14027 : else if (code == 'b')
14028 : msize = 1;
14029 : else if (code == 'k')
14030 : msize = 4;
14031 : else if (code == 'q')
14032 : msize = 8;
14033 : else if (code == 'h')
14034 : msize = 0;
14035 : else if (code == 'x')
14036 : msize = 16;
14037 : else if (code == 't')
14038 : msize = 32;
14039 : else if (code == 'g')
14040 : msize = 64;
14041 : else
14042 206958892 : msize = GET_MODE_SIZE (GET_MODE (x));
14043 :
14044 120690232 : regno = REGNO (x);
14045 :
14046 120690232 : if (regno == ARG_POINTER_REGNUM
14047 120690232 : || regno == FRAME_POINTER_REGNUM
14048 120690232 : || regno == FPSR_REG)
14049 : {
14050 0 : output_operand_lossage
14051 0 : ("invalid use of register '%s'", reg_names[regno]);
14052 0 : return;
14053 : }
14054 120690232 : else if (regno == FLAGS_REG)
14055 : {
14056 1 : output_operand_lossage ("invalid use of asm flag output");
14057 1 : return;
14058 : }
14059 :
14060 120690231 : if (code == 'V')
14061 : {
14062 1 : if (GENERAL_REGNO_P (regno))
14063 2 : msize = GET_MODE_SIZE (word_mode);
14064 : else
14065 0 : error ("%<V%> modifier on non-integer register");
14066 : }
14067 :
14068 120690231 : duplicated = code == 'd' && TARGET_AVX;
14069 :
14070 120690231 : switch (msize)
14071 : {
14072 80867893 : case 16:
14073 80867893 : case 12:
14074 80867893 : case 8:
14075 151647864 : if (GENERAL_REGNO_P (regno) && msize > GET_MODE_SIZE (word_mode))
14076 5 : warning (0, "unsupported size for integer register");
14077 : /* FALLTHRU */
14078 117167837 : case 4:
14079 117167837 : if (LEGACY_INT_REGNO_P (regno))
14080 126462370 : putc (msize > 4 && TARGET_64BIT ? 'r' : 'e', file);
14081 : /* FALLTHRU */
14082 118074002 : case 2:
14083 118074002 : case 128:
14084 22561643 : normal:
14085 118074002 : reg = hi_reg_name[regno];
14086 118074002 : break;
14087 2353120 : case 1:
14088 2353120 : if (regno >= ARRAY_SIZE (qi_reg_name))
14089 277705 : goto normal;
14090 2075415 : if (!ANY_QI_REGNO_P (regno))
14091 0 : error ("unsupported size for integer register");
14092 2075415 : reg = qi_reg_name[regno];
14093 2075415 : break;
14094 26303 : case 0:
14095 26303 : if (regno >= ARRAY_SIZE (qi_high_reg_name))
14096 0 : goto normal;
14097 26303 : reg = qi_high_reg_name[regno];
14098 26303 : break;
14099 514511 : case 32:
14100 514511 : case 64:
14101 514511 : if (SSE_REGNO_P (regno))
14102 : {
14103 514511 : gcc_assert (!duplicated);
14104 721606 : putc (msize == 32 ? 'y' : 'z', file);
14105 514511 : reg = hi_reg_name[regno] + 1;
14106 514511 : break;
14107 : }
14108 0 : goto normal;
14109 0 : default:
14110 0 : gcc_unreachable ();
14111 : }
14112 :
14113 120690231 : fputs (reg, file);
14114 :
14115 : /* Irritatingly, AMD extended registers use
14116 : different naming convention: "r%d[bwd]" */
14117 120690231 : if (REX_INT_REGNO_P (regno) || REX2_INT_REGNO_P (regno))
14118 : {
14119 10763978 : gcc_assert (TARGET_64BIT);
14120 10763978 : switch (msize)
14121 : {
14122 0 : case 0:
14123 0 : error ("extended registers have no high halves");
14124 0 : break;
14125 184122 : case 1:
14126 184122 : putc ('b', file);
14127 184122 : break;
14128 32838 : case 2:
14129 32838 : putc ('w', file);
14130 32838 : break;
14131 2517065 : case 4:
14132 2517065 : putc ('d', file);
14133 2517065 : break;
14134 : case 8:
14135 : /* no suffix */
14136 : break;
14137 0 : default:
14138 0 : error ("unsupported operand size for extended register");
14139 0 : break;
14140 : }
14141 : return;
14142 : }
14143 :
14144 109926253 : if (duplicated)
14145 : {
14146 17087 : if (ASSEMBLER_DIALECT == ASM_ATT)
14147 17072 : fprintf (file, ", %%%s", reg);
14148 : else
14149 15 : fprintf (file, ", %s", reg);
14150 : }
14151 : }
14152 :
14153 : /* Meaning of CODE:
14154 : L,W,B,Q,S,T -- print the opcode suffix for specified size of operand.
14155 : C -- print opcode suffix for set/cmov insn.
14156 : c -- like C, but print reversed condition
14157 : F,f -- likewise, but for floating-point.
14158 : O -- if HAVE_AS_IX86_CMOV_SUN_SYNTAX, expand to "w.", "l." or "q.",
14159 : otherwise nothing
14160 : R -- print embedded rounding and sae.
14161 : r -- print only sae.
14162 : z -- print the opcode suffix for the size of the current operand.
14163 : Z -- likewise, with special suffixes for x87 instructions.
14164 : * -- print a star (in certain assembler syntax)
14165 : A -- print an absolute memory reference.
14166 : E -- print address with DImode register names if TARGET_64BIT.
14167 : w -- print the operand as if it's a "word" (HImode) even if it isn't.
14168 : s -- print a shift double count, followed by the assemblers argument
14169 : delimiter.
14170 : b -- print the QImode name of the register for the indicated operand.
14171 : %b0 would print %al if operands[0] is reg 0.
14172 : w -- likewise, print the HImode name of the register.
14173 : k -- likewise, print the SImode name of the register.
14174 : q -- likewise, print the DImode name of the register.
14175 : x -- likewise, print the V4SFmode name of the register.
14176 : t -- likewise, print the V8SFmode name of the register.
14177 : g -- likewise, print the V16SFmode name of the register.
14178 : h -- print the QImode name for a "high" register, either ah, bh, ch or dh.
14179 : y -- print "st(0)" instead of "st" as a register.
14180 : d -- print duplicated register operand for AVX instruction.
14181 : D -- print condition for SSE cmp instruction.
14182 : P -- if PIC, print an @PLT suffix. For -fno-plt, load function
14183 : address from GOT.
14184 : p -- print raw symbol name.
14185 : X -- don't print any sort of PIC '@' suffix for a symbol.
14186 : & -- print some in-use local-dynamic symbol name.
14187 : H -- print a memory address offset by 8; used for sse high-parts
14188 : Y -- print condition for XOP pcom* instruction.
14189 : V -- print naked full integer register name without %.
14190 : v -- print segment override prefix
14191 : + -- print a branch hint as 'cs' or 'ds' prefix
14192 : ; -- print a semicolon (after prefixes due to bug in older gas).
14193 : ~ -- print "i" if TARGET_AVX2, "f" otherwise.
14194 : ^ -- print addr32 prefix if Pmode != word_mode
14195 : M -- print addr32 prefix for TARGET_X32 with VSIB address.
14196 : ! -- print NOTRACK prefix for jxx/call/ret instructions if required.
14197 : N -- print maskz if it's constant 0 operand.
14198 : G -- print embedded flag for ccmp/ctest.
14199 : */
14200 :
14201 : void
14202 181830644 : ix86_print_operand (FILE *file, rtx x, int code)
14203 : {
14204 182026418 : if (code)
14205 : {
14206 63468596 : switch (code)
14207 : {
14208 195770 : case 'A':
14209 195770 : switch (ASSEMBLER_DIALECT)
14210 : {
14211 195770 : case ASM_ATT:
14212 195770 : putc ('*', file);
14213 195770 : break;
14214 :
14215 0 : case ASM_INTEL:
14216 : /* Intel syntax. For absolute addresses, registers should not
14217 : be surrounded by braces. */
14218 0 : if (!REG_P (x))
14219 : {
14220 0 : putc ('[', file);
14221 0 : ix86_print_operand (file, x, 0);
14222 0 : putc (']', file);
14223 0 : return;
14224 : }
14225 : break;
14226 :
14227 0 : default:
14228 0 : gcc_unreachable ();
14229 : }
14230 :
14231 : ix86_print_operand (file, x, 0);
14232 : return;
14233 :
14234 4019627 : case 'E':
14235 : /* Wrap address in an UNSPEC to declare special handling. */
14236 4019627 : if (TARGET_64BIT)
14237 3534224 : x = gen_rtx_UNSPEC (DImode, gen_rtvec (1, x), UNSPEC_LEA_ADDR);
14238 :
14239 4019627 : output_address (VOIDmode, x);
14240 4019627 : return;
14241 :
14242 0 : case 'L':
14243 0 : if (ASSEMBLER_DIALECT == ASM_ATT)
14244 0 : putc ('l', file);
14245 : return;
14246 :
14247 0 : case 'W':
14248 0 : if (ASSEMBLER_DIALECT == ASM_ATT)
14249 0 : putc ('w', file);
14250 : return;
14251 :
14252 0 : case 'B':
14253 0 : if (ASSEMBLER_DIALECT == ASM_ATT)
14254 0 : putc ('b', file);
14255 : return;
14256 :
14257 0 : case 'Q':
14258 0 : if (ASSEMBLER_DIALECT == ASM_ATT)
14259 0 : putc ('l', file);
14260 : return;
14261 :
14262 0 : case 'S':
14263 0 : if (ASSEMBLER_DIALECT == ASM_ATT)
14264 0 : putc ('s', file);
14265 : return;
14266 :
14267 0 : case 'T':
14268 0 : if (ASSEMBLER_DIALECT == ASM_ATT)
14269 0 : putc ('t', file);
14270 : return;
14271 :
14272 : case 'O':
14273 : #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
14274 : if (ASSEMBLER_DIALECT != ASM_ATT)
14275 : return;
14276 :
14277 : switch (GET_MODE_SIZE (GET_MODE (x)))
14278 : {
14279 : case 2:
14280 : putc ('w', file);
14281 : break;
14282 :
14283 : case 4:
14284 : putc ('l', file);
14285 : break;
14286 :
14287 : case 8:
14288 : putc ('q', file);
14289 : break;
14290 :
14291 : default:
14292 : output_operand_lossage ("invalid operand size for operand "
14293 : "code 'O'");
14294 : return;
14295 : }
14296 :
14297 : putc ('.', file);
14298 : #endif
14299 : return;
14300 :
14301 38160 : case 'z':
14302 38160 : if (GET_MODE_CLASS (GET_MODE (x)) == MODE_INT)
14303 : {
14304 : /* Opcodes don't get size suffixes if using Intel opcodes. */
14305 38158 : if (ASSEMBLER_DIALECT == ASM_INTEL)
14306 : return;
14307 :
14308 76316 : switch (GET_MODE_SIZE (GET_MODE (x)))
14309 : {
14310 6 : case 1:
14311 6 : putc ('b', file);
14312 6 : return;
14313 :
14314 6 : case 2:
14315 6 : putc ('w', file);
14316 6 : return;
14317 :
14318 37640 : case 4:
14319 37640 : putc ('l', file);
14320 37640 : return;
14321 :
14322 506 : case 8:
14323 506 : putc ('q', file);
14324 506 : return;
14325 :
14326 0 : default:
14327 0 : output_operand_lossage ("invalid operand size for operand "
14328 : "code 'z'");
14329 0 : return;
14330 : }
14331 : }
14332 :
14333 2 : if (GET_MODE_CLASS (GET_MODE (x)) == MODE_FLOAT)
14334 : {
14335 1 : if (this_is_asm_operands)
14336 1 : warning_for_asm (this_is_asm_operands,
14337 : "non-integer operand used with operand code %<z%>");
14338 : else
14339 0 : warning (0, "non-integer operand used with operand code %<z%>");
14340 : }
14341 : /* FALLTHRU */
14342 :
14343 385497 : case 'Z':
14344 : /* 387 opcodes don't get size suffixes if using Intel opcodes. */
14345 385497 : if (ASSEMBLER_DIALECT == ASM_INTEL)
14346 : return;
14347 :
14348 385497 : if (GET_MODE_CLASS (GET_MODE (x)) == MODE_INT)
14349 : {
14350 29454 : switch (GET_MODE_SIZE (GET_MODE (x)))
14351 : {
14352 3523 : case 2:
14353 : #ifdef HAVE_AS_IX86_FILDS
14354 3523 : putc ('s', file);
14355 : #endif
14356 3523 : return;
14357 :
14358 3991 : case 4:
14359 3991 : putc ('l', file);
14360 3991 : return;
14361 :
14362 7213 : case 8:
14363 : #ifdef HAVE_AS_IX86_FILDQ
14364 7213 : putc ('q', file);
14365 : #else
14366 : fputs ("ll", file);
14367 : #endif
14368 7213 : return;
14369 :
14370 : default:
14371 : break;
14372 : }
14373 : }
14374 370770 : else if (GET_MODE_CLASS (GET_MODE (x)) == MODE_FLOAT)
14375 : {
14376 : /* 387 opcodes don't get size suffixes
14377 : if the operands are registers. */
14378 370768 : if (STACK_REG_P (x))
14379 : return;
14380 :
14381 690976 : switch (GET_MODE_SIZE (GET_MODE (x)))
14382 : {
14383 24059 : case 4:
14384 24059 : putc ('s', file);
14385 24059 : return;
14386 :
14387 32940 : case 8:
14388 32940 : putc ('l', file);
14389 32940 : return;
14390 :
14391 288487 : case 12:
14392 288487 : case 16:
14393 288487 : putc ('t', file);
14394 288487 : return;
14395 :
14396 : default:
14397 : break;
14398 : }
14399 : }
14400 : else
14401 : {
14402 2 : output_operand_lossage ("invalid operand type used with "
14403 : "operand code '%c'", code);
14404 2 : return;
14405 : }
14406 :
14407 2 : output_operand_lossage ("invalid operand size for operand code '%c'",
14408 : code);
14409 2 : return;
14410 :
14411 : case 'd':
14412 : case 'b':
14413 : case 'w':
14414 : case 'k':
14415 : case 'q':
14416 : case 'h':
14417 : case 't':
14418 : case 'g':
14419 : case 'y':
14420 : case 'x':
14421 : case 'X':
14422 : case 'P':
14423 : case 'p':
14424 : case 'V':
14425 : break;
14426 :
14427 0 : case 's':
14428 0 : if (CONST_INT_P (x) || ! SHIFT_DOUBLE_OMITS_COUNT)
14429 : {
14430 0 : ix86_print_operand (file, x, 0);
14431 0 : fputs (", ", file);
14432 : }
14433 : return;
14434 :
14435 488 : case 'Y':
14436 488 : switch (GET_CODE (x))
14437 : {
14438 176 : case NE:
14439 176 : fputs ("neq", file);
14440 176 : break;
14441 32 : case EQ:
14442 32 : fputs ("eq", file);
14443 32 : break;
14444 64 : case GE:
14445 64 : case GEU:
14446 64 : fputs (INTEGRAL_MODE_P (GET_MODE (x)) ? "ge" : "unlt", file);
14447 64 : break;
14448 40 : case GT:
14449 40 : case GTU:
14450 40 : fputs (INTEGRAL_MODE_P (GET_MODE (x)) ? "gt" : "unle", file);
14451 40 : break;
14452 64 : case LE:
14453 64 : case LEU:
14454 64 : fputs ("le", file);
14455 64 : break;
14456 112 : case LT:
14457 112 : case LTU:
14458 112 : fputs ("lt", file);
14459 112 : break;
14460 0 : case UNORDERED:
14461 0 : fputs ("unord", file);
14462 0 : break;
14463 0 : case ORDERED:
14464 0 : fputs ("ord", file);
14465 0 : break;
14466 0 : case UNEQ:
14467 0 : fputs ("ueq", file);
14468 0 : break;
14469 0 : case UNGE:
14470 0 : fputs ("nlt", file);
14471 0 : break;
14472 0 : case UNGT:
14473 0 : fputs ("nle", file);
14474 0 : break;
14475 0 : case UNLE:
14476 0 : fputs ("ule", file);
14477 0 : break;
14478 0 : case UNLT:
14479 0 : fputs ("ult", file);
14480 0 : break;
14481 0 : case LTGT:
14482 0 : fputs ("une", file);
14483 0 : break;
14484 0 : default:
14485 0 : output_operand_lossage ("operand is not a condition code, "
14486 : "invalid operand code 'Y'");
14487 0 : return;
14488 : }
14489 : return;
14490 :
14491 10130 : case 'D':
14492 : /* Little bit of braindamage here. The SSE compare instructions
14493 : does use completely different names for the comparisons that the
14494 : fp conditional moves. */
14495 10130 : switch (GET_CODE (x))
14496 : {
14497 16 : case UNEQ:
14498 16 : if (TARGET_AVX)
14499 : {
14500 16 : fputs ("eq_uq", file);
14501 16 : break;
14502 : }
14503 : /* FALLTHRU */
14504 5226 : case EQ:
14505 5226 : fputs ("eq", file);
14506 5226 : break;
14507 0 : case UNLT:
14508 0 : if (TARGET_AVX)
14509 : {
14510 0 : fputs ("nge_uq", file);
14511 0 : break;
14512 : }
14513 : /* FALLTHRU */
14514 1746 : case LT:
14515 1746 : fputs ("lt", file);
14516 1746 : break;
14517 0 : case UNLE:
14518 0 : if (TARGET_AVX)
14519 : {
14520 0 : fputs ("ngt_uq", file);
14521 0 : break;
14522 : }
14523 : /* FALLTHRU */
14524 820 : case LE:
14525 820 : fputs ("le", file);
14526 820 : break;
14527 103 : case UNORDERED:
14528 103 : fputs ("unord", file);
14529 103 : break;
14530 12 : case LTGT:
14531 12 : if (TARGET_AVX)
14532 : {
14533 12 : fputs ("neq_oq", file);
14534 12 : break;
14535 : }
14536 : /* FALLTHRU */
14537 905 : case NE:
14538 905 : fputs ("neq", file);
14539 905 : break;
14540 0 : case GE:
14541 0 : if (TARGET_AVX)
14542 : {
14543 0 : fputs ("ge", file);
14544 0 : break;
14545 : }
14546 : /* FALLTHRU */
14547 446 : case UNGE:
14548 : /* Only AVX has the quiet form. */
14549 446 : fputs (TARGET_AVX ? "nlt_uq" : "nlt", file);
14550 446 : break;
14551 0 : case GT:
14552 0 : if (TARGET_AVX)
14553 : {
14554 0 : fputs ("gt", file);
14555 0 : break;
14556 : }
14557 : /* FALLTHRU */
14558 773 : case UNGT:
14559 773 : fputs (TARGET_AVX ? "nle_uq" : "nle", file);
14560 773 : break;
14561 83 : case ORDERED:
14562 83 : fputs ("ord", file);
14563 83 : break;
14564 0 : default:
14565 0 : output_operand_lossage ("operand is not a condition code, "
14566 : "invalid operand code 'D'");
14567 0 : return;
14568 : }
14569 : return;
14570 :
14571 7337517 : case 'F':
14572 7337517 : case 'f':
14573 : #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
14574 : if (ASSEMBLER_DIALECT == ASM_ATT)
14575 : putc ('.', file);
14576 : gcc_fallthrough ();
14577 : #endif
14578 :
14579 7337517 : case 'C':
14580 7337517 : case 'c':
14581 7337517 : if (!COMPARISON_P (x))
14582 : {
14583 0 : output_operand_lossage ("operand is not a condition code, "
14584 : "invalid operand code '%c'", code);
14585 0 : return;
14586 : }
14587 7337517 : put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)),
14588 7337517 : code == 'c' || code == 'f',
14589 7337517 : code == 'F' || code == 'f',
14590 : file);
14591 7337517 : return;
14592 :
14593 23 : case 'G':
14594 23 : {
14595 23 : int dfv = INTVAL (x);
14596 23 : const char *dfv_suffix = ix86_ccmp_dfv_mapping[dfv];
14597 23 : fputs (dfv_suffix, file);
14598 : }
14599 23 : return;
14600 :
14601 1306 : case 'H':
14602 1306 : if (!offsettable_memref_p (x))
14603 : {
14604 1 : output_operand_lossage ("operand is not an offsettable memory "
14605 : "reference, invalid operand code 'H'");
14606 1 : return;
14607 : }
14608 : /* It doesn't actually matter what mode we use here, as we're
14609 : only going to use this for printing. */
14610 1305 : x = adjust_address_nv (x, DImode, 8);
14611 : /* Output 'qword ptr' for intel assembler dialect. */
14612 1305 : if (ASSEMBLER_DIALECT == ASM_INTEL)
14613 0 : code = 'q';
14614 : break;
14615 :
14616 75994 : case 'K':
14617 75994 : if (!CONST_INT_P (x))
14618 : {
14619 1 : output_operand_lossage ("operand is not an integer, invalid "
14620 : "operand code 'K'");
14621 1 : return;
14622 : }
14623 :
14624 75993 : if (INTVAL (x) & IX86_HLE_ACQUIRE)
14625 : #ifdef HAVE_AS_IX86_HLE
14626 22 : fputs ("xacquire ", file);
14627 : #else
14628 : fputs ("\n" ASM_BYTE "0xf2\n\t", file);
14629 : #endif
14630 75971 : else if (INTVAL (x) & IX86_HLE_RELEASE)
14631 : #ifdef HAVE_AS_IX86_HLE
14632 24 : fputs ("xrelease ", file);
14633 : #else
14634 : fputs ("\n" ASM_BYTE "0xf3\n\t", file);
14635 : #endif
14636 : /* We do not want to print value of the operand. */
14637 : return;
14638 :
14639 44326 : case 'N':
14640 44326 : if (x == const0_rtx || x == CONST0_RTX (GET_MODE (x)))
14641 15862 : fputs ("{z}", file);
14642 : return;
14643 :
14644 4060 : case 'r':
14645 4060 : if (!CONST_INT_P (x) || INTVAL (x) != ROUND_SAE)
14646 : {
14647 2 : output_operand_lossage ("operand is not a specific integer, "
14648 : "invalid operand code 'r'");
14649 2 : return;
14650 : }
14651 :
14652 4058 : if (ASSEMBLER_DIALECT == ASM_INTEL)
14653 1 : fputs (", ", file);
14654 :
14655 4058 : fputs ("{sae}", file);
14656 :
14657 4058 : if (ASSEMBLER_DIALECT == ASM_ATT)
14658 4057 : fputs (", ", file);
14659 :
14660 : return;
14661 :
14662 5975 : case 'R':
14663 5975 : if (!CONST_INT_P (x))
14664 : {
14665 1 : output_operand_lossage ("operand is not an integer, invalid "
14666 : "operand code 'R'");
14667 1 : return;
14668 : }
14669 :
14670 5974 : if (ASSEMBLER_DIALECT == ASM_INTEL)
14671 6 : fputs (", ", file);
14672 :
14673 5974 : switch (INTVAL (x))
14674 : {
14675 5159 : case ROUND_NEAREST_INT | ROUND_SAE:
14676 5159 : fputs ("{rn-sae}", file);
14677 5159 : break;
14678 637 : case ROUND_NEG_INF | ROUND_SAE:
14679 637 : fputs ("{rd-sae}", file);
14680 637 : break;
14681 56 : case ROUND_POS_INF | ROUND_SAE:
14682 56 : fputs ("{ru-sae}", file);
14683 56 : break;
14684 121 : case ROUND_ZERO | ROUND_SAE:
14685 121 : fputs ("{rz-sae}", file);
14686 121 : break;
14687 1 : default:
14688 1 : output_operand_lossage ("operand is not a specific integer, "
14689 : "invalid operand code 'R'");
14690 : }
14691 :
14692 5974 : if (ASSEMBLER_DIALECT == ASM_ATT)
14693 5968 : fputs (", ", file);
14694 :
14695 : return;
14696 :
14697 11020 : case 'v':
14698 11020 : if (MEM_P (x))
14699 : {
14700 11148 : switch (MEM_ADDR_SPACE (x))
14701 : {
14702 : case ADDR_SPACE_GENERIC:
14703 : break;
14704 0 : case ADDR_SPACE_SEG_FS:
14705 0 : fputs ("fs ", file);
14706 0 : break;
14707 0 : case ADDR_SPACE_SEG_GS:
14708 0 : fputs ("gs ", file);
14709 0 : break;
14710 0 : default:
14711 0 : gcc_unreachable ();
14712 : }
14713 : }
14714 : else
14715 0 : output_operand_lossage ("operand is not a memory reference, "
14716 : "invalid operand code 'v'");
14717 : return;
14718 :
14719 0 : case '*':
14720 0 : if (ASSEMBLER_DIALECT == ASM_ATT)
14721 0 : putc ('*', file);
14722 : return;
14723 :
14724 206 : case '&':
14725 206 : {
14726 206 : const char *name = get_some_local_dynamic_name ();
14727 206 : if (name == NULL)
14728 1 : output_operand_lossage ("'%%&' used without any "
14729 : "local dynamic TLS references");
14730 : else
14731 205 : assemble_name (file, name);
14732 : return;
14733 : }
14734 :
14735 6637126 : case '+':
14736 6637126 : {
14737 6637126 : rtx x;
14738 :
14739 6637126 : if (!optimize
14740 5197345 : || optimize_function_for_size_p (cfun)
14741 11645309 : || (!TARGET_BRANCH_PREDICTION_HINTS_NOT_TAKEN
14742 5008183 : && !TARGET_BRANCH_PREDICTION_HINTS_TAKEN))
14743 : return;
14744 :
14745 0 : x = find_reg_note (current_output_insn, REG_BR_PROB, 0);
14746 0 : if (x)
14747 : {
14748 0 : int pred_val = profile_probability::from_reg_br_prob_note
14749 0 : (XINT (x, 0)).to_reg_br_prob_base ();
14750 :
14751 0 : bool taken = pred_val > REG_BR_PROB_BASE / 2;
14752 : /* We use 3e (DS) prefix for taken branches and
14753 : 2e (CS) prefix for not taken branches. */
14754 0 : if (taken && TARGET_BRANCH_PREDICTION_HINTS_TAKEN)
14755 0 : fputs ("ds ; ", file);
14756 0 : else if (!taken && TARGET_BRANCH_PREDICTION_HINTS_NOT_TAKEN)
14757 0 : fputs ("cs ; ", file);
14758 : }
14759 : return;
14760 : }
14761 :
14762 : case ';':
14763 : #ifndef HAVE_AS_IX86_REP_LOCK_PREFIX
14764 : putc (';', file);
14765 : #endif
14766 : return;
14767 :
14768 3258 : case '~':
14769 3258 : putc (TARGET_AVX2 ? 'i' : 'f', file);
14770 3258 : return;
14771 :
14772 1646 : case 'M':
14773 1646 : if (TARGET_X32)
14774 : {
14775 : /* NB: 32-bit indices in VSIB address are sign-extended
14776 : to 64 bits. In x32, if 32-bit address 0xf7fa3010 is
14777 : sign-extended to 0xfffffffff7fa3010 which is invalid
14778 : address. Add addr32 prefix if there is no base
14779 : register nor symbol. */
14780 8 : bool ok;
14781 8 : struct ix86_address parts;
14782 8 : ok = ix86_decompose_address (x, &parts);
14783 8 : gcc_assert (ok && parts.index == NULL_RTX);
14784 8 : if (parts.base == NULL_RTX
14785 8 : && (parts.disp == NULL_RTX
14786 2 : || !symbolic_operand (parts.disp,
14787 2 : GET_MODE (parts.disp))))
14788 2 : fputs ("addr32 ", file);
14789 : }
14790 : return;
14791 :
14792 24708 : case '^':
14793 27913 : if (Pmode != word_mode)
14794 0 : fputs ("addr32 ", file);
14795 : return;
14796 :
14797 15311784 : case '!':
14798 15311784 : if (ix86_notrack_prefixed_insn_p (current_output_insn))
14799 4650 : fputs ("notrack ", file);
14800 : return;
14801 :
14802 1 : default:
14803 1 : output_operand_lossage ("invalid operand code '%c'", code);
14804 : }
14805 : }
14806 :
14807 147474438 : if (REG_P (x))
14808 88204863 : print_reg (x, code, file);
14809 :
14810 59269575 : else if (MEM_P (x))
14811 : {
14812 33985730 : rtx addr = XEXP (x, 0);
14813 :
14814 : /* No `byte ptr' prefix for call instructions ... */
14815 33985730 : if (ASSEMBLER_DIALECT == ASM_INTEL && code != 'X' && code != 'P')
14816 : {
14817 316 : machine_mode mode = GET_MODE (x);
14818 316 : const char *size;
14819 :
14820 : /* Check for explicit size override codes. */
14821 316 : if (code == 'b')
14822 : size = "BYTE";
14823 : else if (code == 'w')
14824 : size = "WORD";
14825 : else if (code == 'k')
14826 : size = "DWORD";
14827 : else if (code == 'q')
14828 : size = "QWORD";
14829 : else if (code == 'x')
14830 : size = "XMMWORD";
14831 : else if (code == 't')
14832 : size = "YMMWORD";
14833 : else if (code == 'g')
14834 : size = "ZMMWORD";
14835 230 : else if (mode == BLKmode)
14836 : /* ... or BLKmode operands, when not overridden. */
14837 : size = NULL;
14838 : else
14839 456 : switch (GET_MODE_SIZE (mode))
14840 : {
14841 : case 1: size = "BYTE"; break;
14842 : case 2: size = "WORD"; break;
14843 : case 4: size = "DWORD"; break;
14844 : case 8: size = "QWORD"; break;
14845 : case 12: size = "TBYTE"; break;
14846 8 : case 16:
14847 8 : if (mode == XFmode)
14848 : size = "TBYTE";
14849 : else
14850 : size = "XMMWORD";
14851 : break;
14852 : case 32: size = "YMMWORD"; break;
14853 : case 64: size = "ZMMWORD"; break;
14854 0 : default:
14855 0 : gcc_unreachable ();
14856 : }
14857 : if (size)
14858 : {
14859 314 : fputs (size, file);
14860 314 : fputs (" PTR ", file);
14861 : }
14862 : }
14863 :
14864 33985730 : if (this_is_asm_operands && ! address_operand (addr, VOIDmode))
14865 0 : output_operand_lossage ("invalid constraints for operand");
14866 : else
14867 33985730 : ix86_print_operand_address_as
14868 34547106 : (file, addr, MEM_ADDR_SPACE (x), code == 'p' || code == 'P');
14869 : }
14870 :
14871 25283845 : else if (CONST_DOUBLE_P (x) && GET_MODE (x) == HFmode)
14872 : {
14873 971 : long l = real_to_target (NULL, CONST_DOUBLE_REAL_VALUE (x),
14874 971 : REAL_MODE_FORMAT (HFmode));
14875 971 : if (ASSEMBLER_DIALECT == ASM_ATT)
14876 971 : putc ('$', file);
14877 971 : fprintf (file, "0x%04x", (unsigned int) l);
14878 971 : }
14879 :
14880 25282874 : else if (CONST_DOUBLE_P (x) && GET_MODE (x) == SFmode)
14881 : {
14882 20909 : long l;
14883 :
14884 20909 : REAL_VALUE_TO_TARGET_SINGLE (*CONST_DOUBLE_REAL_VALUE (x), l);
14885 :
14886 20909 : if (ASSEMBLER_DIALECT == ASM_ATT)
14887 20909 : putc ('$', file);
14888 : /* Sign extend 32bit SFmode immediate to 8 bytes. */
14889 20909 : if (code == 'q')
14890 327 : fprintf (file, "0x%08" HOST_LONG_LONG_FORMAT "x",
14891 : (unsigned long long) (int) l);
14892 : else
14893 20582 : fprintf (file, "0x%08x", (unsigned int) l);
14894 : }
14895 :
14896 25261965 : else if (CONST_DOUBLE_P (x) && GET_MODE (x) == DFmode)
14897 : {
14898 3909 : long l[2];
14899 :
14900 3909 : REAL_VALUE_TO_TARGET_DOUBLE (*CONST_DOUBLE_REAL_VALUE (x), l);
14901 :
14902 3909 : if (ASSEMBLER_DIALECT == ASM_ATT)
14903 3909 : putc ('$', file);
14904 3909 : fprintf (file, "0x%lx%08lx", l[1] & 0xffffffff, l[0] & 0xffffffff);
14905 3909 : }
14906 :
14907 : /* These float cases don't actually occur as immediate operands. */
14908 25258056 : else if (CONST_DOUBLE_P (x) && GET_MODE (x) == XFmode)
14909 : {
14910 0 : char dstr[30];
14911 :
14912 0 : real_to_decimal (dstr, CONST_DOUBLE_REAL_VALUE (x), sizeof (dstr), 0, 1);
14913 0 : fputs (dstr, file);
14914 0 : }
14915 :
14916 : /* Print bcst_mem_operand. */
14917 25258056 : else if (GET_CODE (x) == VEC_DUPLICATE)
14918 : {
14919 314 : machine_mode vmode = GET_MODE (x);
14920 : /* Must be bcst_memory_operand. */
14921 314 : gcc_assert (bcst_mem_operand (x, vmode));
14922 :
14923 314 : rtx mem = XEXP (x,0);
14924 314 : ix86_print_operand (file, mem, 0);
14925 :
14926 314 : switch (vmode)
14927 : {
14928 28 : case E_V2DImode:
14929 28 : case E_V2DFmode:
14930 28 : fputs ("{1to2}", file);
14931 28 : break;
14932 74 : case E_V4SImode:
14933 74 : case E_V4SFmode:
14934 74 : case E_V4DImode:
14935 74 : case E_V4DFmode:
14936 74 : fputs ("{1to4}", file);
14937 74 : break;
14938 94 : case E_V8SImode:
14939 94 : case E_V8SFmode:
14940 94 : case E_V8DFmode:
14941 94 : case E_V8DImode:
14942 94 : case E_V8HFmode:
14943 94 : fputs ("{1to8}", file);
14944 94 : break;
14945 110 : case E_V16SFmode:
14946 110 : case E_V16SImode:
14947 110 : case E_V16HFmode:
14948 110 : fputs ("{1to16}", file);
14949 110 : break;
14950 8 : case E_V32HFmode:
14951 8 : fputs ("{1to32}", file);
14952 8 : break;
14953 0 : default:
14954 0 : gcc_unreachable ();
14955 : }
14956 : }
14957 :
14958 : else
14959 : {
14960 : /* We have patterns that allow zero sets of memory, for instance.
14961 : In 64-bit mode, we should probably support all 8-byte vectors,
14962 : since we can in fact encode that into an immediate. */
14963 25257742 : if (CONST_VECTOR_P (x))
14964 : {
14965 3204 : if (x != CONST0_RTX (GET_MODE (x)))
14966 2 : output_operand_lossage ("invalid vector immediate");
14967 3204 : x = const0_rtx;
14968 : }
14969 :
14970 25257742 : if (code == 'P')
14971 : {
14972 6156682 : if (ix86_force_load_from_GOT_p (x, true))
14973 : {
14974 : /* For inline assembly statement, load function address
14975 : from GOT with 'P' operand modifier to avoid PLT. */
14976 4 : x = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x),
14977 : (TARGET_64BIT
14978 : ? UNSPEC_GOTPCREL
14979 : : UNSPEC_GOT));
14980 4 : x = gen_rtx_CONST (Pmode, x);
14981 4 : x = gen_const_mem (Pmode, x);
14982 4 : ix86_print_operand (file, x, 'A');
14983 4 : return;
14984 : }
14985 : }
14986 19101060 : else if (code != 'p')
14987 : {
14988 19100951 : if (CONST_INT_P (x))
14989 : {
14990 16015490 : if (ASSEMBLER_DIALECT == ASM_ATT)
14991 16015264 : putc ('$', file);
14992 : }
14993 3085461 : else if (GET_CODE (x) == CONST || SYMBOL_REF_P (x)
14994 9355 : || LABEL_REF_P (x))
14995 : {
14996 3085459 : if (ASSEMBLER_DIALECT == ASM_ATT)
14997 3085453 : putc ('$', file);
14998 : else
14999 6 : fputs ("OFFSET FLAT:", file);
15000 : }
15001 : }
15002 25257738 : if (CONST_INT_P (x))
15003 16015576 : fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (x));
15004 9242162 : else if (flag_pic || MACHOPIC_INDIRECT)
15005 745751 : output_pic_addr_const (file, x, code);
15006 : else
15007 8496411 : output_addr_const (file, x);
15008 : }
15009 : }
15010 :
15011 : static bool
15012 22060531 : ix86_print_operand_punct_valid_p (unsigned char code)
15013 : {
15014 22060531 : return (code == '*' || code == '+' || code == '&' || code == ';'
15015 15336492 : || code == '~' || code == '^' || code == '!');
15016 : }
15017 :
15018 : /* Print a memory operand whose address is ADDR. */
15019 :
15020 : static void
15021 38007636 : ix86_print_operand_address_as (FILE *file, rtx addr,
15022 : addr_space_t as, bool raw)
15023 : {
15024 38007636 : struct ix86_address parts;
15025 38007636 : rtx base, index, disp;
15026 38007636 : int scale;
15027 38007636 : int ok;
15028 38007636 : bool vsib = false;
15029 38007636 : int code = 0;
15030 :
15031 38007636 : if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_VSIBADDR)
15032 : {
15033 1646 : ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts);
15034 1646 : gcc_assert (parts.index == NULL_RTX);
15035 1646 : parts.index = XVECEXP (addr, 0, 1);
15036 1646 : parts.scale = INTVAL (XVECEXP (addr, 0, 2));
15037 1646 : addr = XVECEXP (addr, 0, 0);
15038 1646 : vsib = true;
15039 : }
15040 38005990 : else if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_LEA_ADDR)
15041 : {
15042 3534224 : gcc_assert (TARGET_64BIT);
15043 3534224 : ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts);
15044 3534224 : code = 'q';
15045 : }
15046 : else
15047 34471766 : ok = ix86_decompose_address (addr, &parts);
15048 :
15049 38007636 : gcc_assert (ok);
15050 :
15051 38007636 : base = parts.base;
15052 38007636 : index = parts.index;
15053 38007636 : disp = parts.disp;
15054 38007636 : scale = parts.scale;
15055 :
15056 38007636 : if (ADDR_SPACE_GENERIC_P (as))
15057 37725693 : as = parts.seg;
15058 : else
15059 281943 : gcc_assert (ADDR_SPACE_GENERIC_P (parts.seg));
15060 :
15061 38007636 : if (!ADDR_SPACE_GENERIC_P (as) && !raw)
15062 : {
15063 281958 : if (ASSEMBLER_DIALECT == ASM_ATT)
15064 281956 : putc ('%', file);
15065 :
15066 281958 : switch (as)
15067 : {
15068 182247 : case ADDR_SPACE_SEG_FS:
15069 182247 : fputs ("fs:", file);
15070 182247 : break;
15071 99711 : case ADDR_SPACE_SEG_GS:
15072 99711 : fputs ("gs:", file);
15073 99711 : break;
15074 0 : default:
15075 0 : gcc_unreachable ();
15076 : }
15077 : }
15078 :
15079 : /* Use one byte shorter RIP relative addressing for 64bit mode. */
15080 38007636 : if (TARGET_64BIT && !base && !index && !raw)
15081 : {
15082 6371027 : rtx symbol = disp;
15083 :
15084 6371027 : if (GET_CODE (disp) == CONST
15085 2217211 : && GET_CODE (XEXP (disp, 0)) == PLUS
15086 2102626 : && CONST_INT_P (XEXP (XEXP (disp, 0), 1)))
15087 2102626 : symbol = XEXP (XEXP (disp, 0), 0);
15088 :
15089 6371027 : if (LABEL_REF_P (symbol)
15090 6371027 : || (SYMBOL_REF_P (symbol)
15091 6079928 : && SYMBOL_REF_TLS_MODEL (symbol) == 0))
15092 6079856 : base = pc_rtx;
15093 : }
15094 :
15095 38007636 : if (!base && !index)
15096 : {
15097 : /* Displacement only requires special attention. */
15098 629011 : if (CONST_INT_P (disp))
15099 : {
15100 269422 : if (ASSEMBLER_DIALECT == ASM_INTEL && ADDR_SPACE_GENERIC_P (as))
15101 0 : fputs ("ds:", file);
15102 269422 : fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (disp));
15103 : }
15104 : /* Load the external function address via the GOT slot to avoid PLT. */
15105 359589 : else if (GET_CODE (disp) == CONST
15106 143046 : && GET_CODE (XEXP (disp, 0)) == UNSPEC
15107 114826 : && (XINT (XEXP (disp, 0), 1) == UNSPEC_GOTPCREL
15108 9593 : || XINT (XEXP (disp, 0), 1) == UNSPEC_GOT)
15109 464822 : && ix86_force_load_from_GOT_p (XVECEXP (XEXP (disp, 0), 0, 0)))
15110 24 : output_pic_addr_const (file, disp, 0);
15111 359565 : else if (flag_pic)
15112 144332 : output_pic_addr_const (file, disp, 0);
15113 : else
15114 215233 : output_addr_const (file, disp);
15115 : }
15116 : else
15117 : {
15118 : /* Print SImode register names to force addr32 prefix. */
15119 37378625 : if (SImode_address_operand (addr, VOIDmode))
15120 : {
15121 67 : if (flag_checking)
15122 : {
15123 67 : gcc_assert (TARGET_64BIT);
15124 67 : switch (GET_CODE (addr))
15125 : {
15126 0 : case SUBREG:
15127 0 : gcc_assert (GET_MODE (addr) == SImode);
15128 0 : gcc_assert (GET_MODE (SUBREG_REG (addr)) == DImode);
15129 : break;
15130 67 : case ZERO_EXTEND:
15131 67 : case AND:
15132 67 : gcc_assert (GET_MODE (addr) == DImode);
15133 : break;
15134 0 : default:
15135 0 : gcc_unreachable ();
15136 : }
15137 : }
15138 67 : gcc_assert (!code);
15139 : code = 'k';
15140 : }
15141 37378558 : else if (code == 0
15142 33845979 : && TARGET_X32
15143 516 : && disp
15144 416 : && CONST_INT_P (disp)
15145 324 : && INTVAL (disp) < -16*1024*1024)
15146 : {
15147 : /* X32 runs in 64-bit mode, where displacement, DISP, in
15148 : address DISP(%r64), is encoded as 32-bit immediate sign-
15149 : extended from 32-bit to 64-bit. For -0x40000300(%r64),
15150 : address is %r64 + 0xffffffffbffffd00. When %r64 <
15151 : 0x40000300, like 0x37ffe064, address is 0xfffffffff7ffdd64,
15152 : which is invalid for x32. The correct address is %r64
15153 : - 0x40000300 == 0xf7ffdd64. To properly encode
15154 : -0x40000300(%r64) for x32, we zero-extend negative
15155 : displacement by forcing addr32 prefix which truncates
15156 : 0xfffffffff7ffdd64 to 0xf7ffdd64. In theory, we should
15157 : zero-extend all negative displacements, including -1(%rsp).
15158 : However, for small negative displacements, sign-extension
15159 : won't cause overflow. We only zero-extend negative
15160 : displacements if they < -16*1024*1024, which is also used
15161 : to check legitimate address displacements for PIC. */
15162 67 : code = 'k';
15163 : }
15164 :
15165 : /* Since the upper 32 bits of RSP are always zero for x32,
15166 : we can encode %esp as %rsp to avoid 0x67 prefix if
15167 : there is no index register. */
15168 1083 : if (TARGET_X32 && Pmode == SImode
15169 37379085 : && !index && base && REG_P (base) && REGNO (base) == SP_REG)
15170 : code = 'q';
15171 :
15172 37378625 : if (ASSEMBLER_DIALECT == ASM_ATT)
15173 : {
15174 37378241 : if (disp)
15175 : {
15176 33251985 : if (flag_pic)
15177 3876625 : output_pic_addr_const (file, disp, 0);
15178 29375360 : else if (LABEL_REF_P (disp))
15179 6073 : output_asm_label (disp);
15180 : else
15181 29369287 : output_addr_const (file, disp);
15182 : }
15183 :
15184 37378241 : putc ('(', file);
15185 37378241 : if (base)
15186 36927671 : print_reg (base, code, file);
15187 37378241 : if (index)
15188 : {
15189 1927605 : putc (',', file);
15190 3853612 : print_reg (index, vsib ? 0 : code, file);
15191 1927605 : if (scale != 1 || vsib)
15192 1052125 : fprintf (file, ",%d", scale);
15193 : }
15194 37378241 : putc (')', file);
15195 : }
15196 : else
15197 : {
15198 384 : rtx offset = NULL_RTX;
15199 :
15200 384 : if (disp)
15201 : {
15202 : /* Pull out the offset of a symbol; print any symbol itself. */
15203 304 : if (GET_CODE (disp) == CONST
15204 14 : && GET_CODE (XEXP (disp, 0)) == PLUS
15205 14 : && CONST_INT_P (XEXP (XEXP (disp, 0), 1)))
15206 : {
15207 14 : offset = XEXP (XEXP (disp, 0), 1);
15208 14 : disp = gen_rtx_CONST (VOIDmode,
15209 : XEXP (XEXP (disp, 0), 0));
15210 : }
15211 :
15212 304 : if (flag_pic)
15213 68 : output_pic_addr_const (file, disp, 0);
15214 236 : else if (LABEL_REF_P (disp))
15215 0 : output_asm_label (disp);
15216 236 : else if (CONST_INT_P (disp))
15217 : offset = disp;
15218 : else
15219 99 : output_addr_const (file, disp);
15220 : }
15221 :
15222 384 : putc ('[', file);
15223 384 : if (base)
15224 : {
15225 347 : print_reg (base, code, file);
15226 347 : if (offset)
15227 : {
15228 151 : if (INTVAL (offset) >= 0)
15229 20 : putc ('+', file);
15230 151 : fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (offset));
15231 : }
15232 : }
15233 37 : else if (offset)
15234 0 : fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (offset));
15235 : else
15236 37 : putc ('0', file);
15237 :
15238 384 : if (index)
15239 : {
15240 90 : putc ('+', file);
15241 132 : print_reg (index, vsib ? 0 : code, file);
15242 90 : if (scale != 1 || vsib)
15243 88 : fprintf (file, "*%d", scale);
15244 : }
15245 384 : putc (']', file);
15246 : }
15247 : }
15248 38007636 : }
15249 :
15250 : static void
15251 4021907 : ix86_print_operand_address (FILE *file, machine_mode /*mode*/, rtx addr)
15252 : {
15253 4021907 : if (this_is_asm_operands && ! address_operand (addr, VOIDmode))
15254 1 : output_operand_lossage ("invalid constraints for operand");
15255 : else
15256 4021906 : ix86_print_operand_address_as (file, addr, ADDR_SPACE_GENERIC, false);
15257 4021907 : }
15258 :
15259 : /* Implementation of TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA. */
15260 :
15261 : static bool
15262 14774 : i386_asm_output_addr_const_extra (FILE *file, rtx x)
15263 : {
15264 14774 : rtx op;
15265 :
15266 14774 : if (GET_CODE (x) != UNSPEC)
15267 : return false;
15268 :
15269 14774 : op = XVECEXP (x, 0, 0);
15270 14774 : switch (XINT (x, 1))
15271 : {
15272 599 : case UNSPEC_GOTOFF:
15273 599 : output_addr_const (file, op);
15274 599 : fputs ("@gotoff", file);
15275 599 : break;
15276 0 : case UNSPEC_GOTTPOFF:
15277 0 : output_addr_const (file, op);
15278 : /* FIXME: This might be @TPOFF in Sun ld. */
15279 0 : fputs ("@gottpoff", file);
15280 0 : break;
15281 0 : case UNSPEC_TPOFF:
15282 0 : output_addr_const (file, op);
15283 0 : fputs ("@tpoff", file);
15284 0 : break;
15285 11070 : case UNSPEC_NTPOFF:
15286 11070 : output_addr_const (file, op);
15287 11070 : if (TARGET_64BIT)
15288 10294 : fputs ("@tpoff", file);
15289 : else
15290 776 : fputs ("@ntpoff", file);
15291 : break;
15292 0 : case UNSPEC_DTPOFF:
15293 0 : output_addr_const (file, op);
15294 0 : fputs ("@dtpoff", file);
15295 0 : break;
15296 3104 : case UNSPEC_GOTNTPOFF:
15297 3104 : output_addr_const (file, op);
15298 3104 : if (TARGET_64BIT)
15299 3104 : fputs (ASSEMBLER_DIALECT == ASM_ATT ?
15300 : "@gottpoff(%rip)" : "@gottpoff[rip]", file);
15301 : else
15302 0 : fputs ("@gotntpoff", file);
15303 : break;
15304 1 : case UNSPEC_INDNTPOFF:
15305 1 : output_addr_const (file, op);
15306 1 : fputs ("@indntpoff", file);
15307 1 : break;
15308 0 : case UNSPEC_SECREL32:
15309 0 : output_addr_const (file, op);
15310 0 : fputs ("@secrel32", file);
15311 0 : break;
15312 : #if TARGET_MACHO
15313 : case UNSPEC_MACHOPIC_OFFSET:
15314 : output_addr_const (file, op);
15315 : putc ('-', file);
15316 : machopic_output_function_base_name (file);
15317 : break;
15318 : #endif
15319 :
15320 : default:
15321 : return false;
15322 : }
15323 :
15324 : return true;
15325 : }
15326 :
15327 :
15328 : /* Output code to perform a 387 binary operation in INSN, one of PLUS,
15329 : MINUS, MULT or DIV. OPERANDS are the insn operands, where operands[3]
15330 : is the expression of the binary operation. The output may either be
15331 : emitted here, or returned to the caller, like all output_* functions.
15332 :
15333 : There is no guarantee that the operands are the same mode, as they
15334 : might be within FLOAT or FLOAT_EXTEND expressions. */
15335 :
15336 : #ifndef SYSV386_COMPAT
15337 : /* Set to 1 for compatibility with brain-damaged assemblers. No-one
15338 : wants to fix the assemblers because that causes incompatibility
15339 : with gcc. No-one wants to fix gcc because that causes
15340 : incompatibility with assemblers... You can use the option of
15341 : -DSYSV386_COMPAT=0 if you recompile both gcc and gas this way. */
15342 : #define SYSV386_COMPAT 1
15343 : #endif
15344 :
15345 : const char *
15346 610697 : output_387_binary_op (rtx_insn *insn, rtx *operands)
15347 : {
15348 610697 : static char buf[40];
15349 610697 : const char *p;
15350 610697 : bool is_sse
15351 610697 : = (SSE_REG_P (operands[0])
15352 666841 : || SSE_REG_P (operands[1]) || SSE_REG_P (operands[2]));
15353 :
15354 56144 : if (is_sse)
15355 : p = "%v";
15356 56144 : else if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
15357 56137 : || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
15358 : p = "fi";
15359 : else
15360 610697 : p = "f";
15361 :
15362 610697 : strcpy (buf, p);
15363 :
15364 610697 : switch (GET_CODE (operands[3]))
15365 : {
15366 : case PLUS:
15367 : p = "add"; break;
15368 : case MINUS:
15369 : p = "sub"; break;
15370 95268 : case MULT:
15371 95268 : p = "mul"; break;
15372 28432 : case DIV:
15373 28432 : p = "div"; break;
15374 0 : default:
15375 0 : gcc_unreachable ();
15376 : }
15377 :
15378 610697 : strcat (buf, p);
15379 :
15380 610697 : if (is_sse)
15381 : {
15382 554553 : p = GET_MODE (operands[0]) == SFmode ? "ss" : "sd";
15383 554553 : strcat (buf, p);
15384 :
15385 554553 : if (TARGET_AVX)
15386 : p = "\t{%2, %1, %0|%0, %1, %2}";
15387 : else
15388 539177 : p = "\t{%2, %0|%0, %2}";
15389 :
15390 554553 : strcat (buf, p);
15391 554553 : return buf;
15392 : }
15393 :
15394 : /* Even if we do not want to check the inputs, this documents input
15395 : constraints. Which helps in understanding the following code. */
15396 56144 : if (flag_checking)
15397 : {
15398 56143 : if (STACK_REG_P (operands[0])
15399 56143 : && ((REG_P (operands[1])
15400 54476 : && REGNO (operands[0]) == REGNO (operands[1])
15401 50367 : && (STACK_REG_P (operands[2]) || MEM_P (operands[2])))
15402 5776 : || (REG_P (operands[2])
15403 5776 : && REGNO (operands[0]) == REGNO (operands[2])
15404 5776 : && (STACK_REG_P (operands[1]) || MEM_P (operands[1]))))
15405 112286 : && (STACK_TOP_P (operands[1]) || STACK_TOP_P (operands[2])))
15406 : ; /* ok */
15407 : else
15408 0 : gcc_unreachable ();
15409 : }
15410 :
15411 56144 : switch (GET_CODE (operands[3]))
15412 : {
15413 41067 : case MULT:
15414 41067 : case PLUS:
15415 41067 : if (REG_P (operands[2]) && REGNO (operands[0]) == REGNO (operands[2]))
15416 2073 : std::swap (operands[1], operands[2]);
15417 :
15418 : /* know operands[0] == operands[1]. */
15419 :
15420 41067 : if (MEM_P (operands[2]))
15421 : {
15422 : p = "%Z2\t%2";
15423 : break;
15424 : }
15425 :
15426 36622 : if (find_regno_note (insn, REG_DEAD, REGNO (operands[2])))
15427 : {
15428 21394 : if (STACK_TOP_P (operands[0]))
15429 : /* How is it that we are storing to a dead operand[2]?
15430 : Well, presumably operands[1] is dead too. We can't
15431 : store the result to st(0) as st(0) gets popped on this
15432 : instruction. Instead store to operands[2] (which I
15433 : think has to be st(1)). st(1) will be popped later.
15434 : gcc <= 2.8.1 didn't have this check and generated
15435 : assembly code that the Unixware assembler rejected. */
15436 : p = "p\t{%0, %2|%2, %0}"; /* st(1) = st(0) op st(1); pop */
15437 : else
15438 : p = "p\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0); pop */
15439 : break;
15440 : }
15441 :
15442 15228 : if (STACK_TOP_P (operands[0]))
15443 : p = "\t{%y2, %0|%0, %y2}"; /* st(0) = st(0) op st(r2) */
15444 : else
15445 : p = "\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0) */
15446 : break;
15447 :
15448 15077 : case MINUS:
15449 15077 : case DIV:
15450 15077 : if (MEM_P (operands[1]))
15451 : {
15452 : p = "r%Z1\t%1";
15453 : break;
15454 : }
15455 :
15456 14614 : if (MEM_P (operands[2]))
15457 : {
15458 : p = "%Z2\t%2";
15459 : break;
15460 : }
15461 :
15462 13006 : if (find_regno_note (insn, REG_DEAD, REGNO (operands[2])))
15463 : {
15464 : #if SYSV386_COMPAT
15465 : /* The SystemV/386 SVR3.2 assembler, and probably all AT&T
15466 : derived assemblers, confusingly reverse the direction of
15467 : the operation for fsub{r} and fdiv{r} when the
15468 : destination register is not st(0). The Intel assembler
15469 : doesn't have this brain damage. Read !SYSV386_COMPAT to
15470 : figure out what the hardware really does. */
15471 6260 : if (STACK_TOP_P (operands[0]))
15472 : p = "{p\t%0, %2|rp\t%2, %0}";
15473 : else
15474 : p = "{rp\t%2, %0|p\t%0, %2}";
15475 : #else
15476 : if (STACK_TOP_P (operands[0]))
15477 : /* As above for fmul/fadd, we can't store to st(0). */
15478 : p = "rp\t{%0, %2|%2, %0}"; /* st(1) = st(0) op st(1); pop */
15479 : else
15480 : p = "p\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0); pop */
15481 : #endif
15482 : break;
15483 : }
15484 :
15485 6746 : if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
15486 : {
15487 : #if SYSV386_COMPAT
15488 3144 : if (STACK_TOP_P (operands[0]))
15489 : p = "{rp\t%0, %1|p\t%1, %0}";
15490 : else
15491 : p = "{p\t%1, %0|rp\t%0, %1}";
15492 : #else
15493 : if (STACK_TOP_P (operands[0]))
15494 : p = "p\t{%0, %1|%1, %0}"; /* st(1) = st(1) op st(0); pop */
15495 : else
15496 : p = "rp\t{%1, %0|%0, %1}"; /* st(r2) = st(0) op st(r2); pop */
15497 : #endif
15498 : break;
15499 : }
15500 :
15501 3602 : if (STACK_TOP_P (operands[0]))
15502 : {
15503 2719 : if (STACK_TOP_P (operands[1]))
15504 : p = "\t{%y2, %0|%0, %y2}"; /* st(0) = st(0) op st(r2) */
15505 : else
15506 : p = "r\t{%y1, %0|%0, %y1}"; /* st(0) = st(r1) op st(0) */
15507 : break;
15508 : }
15509 883 : else if (STACK_TOP_P (operands[1]))
15510 : {
15511 : #if SYSV386_COMPAT
15512 : p = "{\t%1, %0|r\t%0, %1}";
15513 : #else
15514 : p = "r\t{%1, %0|%0, %1}"; /* st(r2) = st(0) op st(r2) */
15515 : #endif
15516 : }
15517 : else
15518 : {
15519 : #if SYSV386_COMPAT
15520 : p = "{r\t%2, %0|\t%0, %2}";
15521 : #else
15522 : p = "\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0) */
15523 : #endif
15524 : }
15525 : break;
15526 :
15527 0 : default:
15528 0 : gcc_unreachable ();
15529 : }
15530 :
15531 56144 : strcat (buf, p);
15532 56144 : return buf;
15533 : }
15534 :
15535 : /* Return needed mode for entity in optimize_mode_switching pass. */
15536 :
15537 : static int
15538 1656 : ix86_dirflag_mode_needed (rtx_insn *insn)
15539 : {
15540 1656 : if (CALL_P (insn))
15541 : {
15542 339 : if (cfun->machine->func_type == TYPE_NORMAL)
15543 : return X86_DIRFLAG_ANY;
15544 : else
15545 : /* No need to emit CLD in interrupt handler for TARGET_CLD. */
15546 339 : return TARGET_CLD ? X86_DIRFLAG_ANY : X86_DIRFLAG_RESET;
15547 : }
15548 :
15549 1317 : if (recog_memoized (insn) < 0)
15550 : return X86_DIRFLAG_ANY;
15551 :
15552 1315 : if (get_attr_type (insn) == TYPE_STR)
15553 : {
15554 : /* Emit cld instruction if stringops are used in the function. */
15555 1 : if (cfun->machine->func_type == TYPE_NORMAL)
15556 0 : return TARGET_CLD ? X86_DIRFLAG_RESET : X86_DIRFLAG_ANY;
15557 : else
15558 : return X86_DIRFLAG_RESET;
15559 : }
15560 :
15561 : return X86_DIRFLAG_ANY;
15562 : }
15563 :
15564 : /* Check if a 256bit or 512 bit AVX register is referenced inside of EXP. */
15565 :
15566 : static bool
15567 2238254 : ix86_check_avx_upper_register (const_rtx exp)
15568 : {
15569 : /* construct_container may return a parallel with expr_list
15570 : which contains the real reg and mode */
15571 2238254 : subrtx_iterator::array_type array;
15572 8559360 : FOR_EACH_SUBRTX (iter, array, exp, NONCONST)
15573 : {
15574 6484516 : const_rtx x = *iter;
15575 2608670 : if (SSE_REG_P (x)
15576 840132 : && !EXT_REX_SSE_REG_P (x)
15577 8151936 : && GET_MODE_BITSIZE (GET_MODE (x)) > 128)
15578 163410 : return true;
15579 : }
15580 :
15581 2074844 : return false;
15582 2238254 : }
15583 :
15584 : /* Check if a 256bit or 512bit AVX register is referenced in stores. */
15585 :
15586 : static void
15587 51451 : ix86_check_avx_upper_stores (rtx dest, const_rtx, void *data)
15588 : {
15589 51451 : if (SSE_REG_P (dest)
15590 12721 : && !EXT_REX_SSE_REG_P (dest)
15591 76893 : && GET_MODE_BITSIZE (GET_MODE (dest)) > 128)
15592 : {
15593 754 : bool *used = (bool *) data;
15594 754 : *used = true;
15595 : }
15596 51451 : }
15597 :
15598 : /* Return needed mode for entity in optimize_mode_switching pass. */
15599 :
15600 : static int
15601 2085343 : ix86_avx_u128_mode_needed (rtx_insn *insn)
15602 : {
15603 2085343 : if (DEBUG_INSN_P (insn))
15604 : return AVX_U128_ANY;
15605 :
15606 2085343 : if (CALL_P (insn))
15607 : {
15608 49860 : rtx link;
15609 :
15610 : /* Needed mode is set to AVX_U128_CLEAN if there are
15611 : no 256bit or 512bit modes used in function arguments. */
15612 49860 : for (link = CALL_INSN_FUNCTION_USAGE (insn);
15613 136560 : link;
15614 86700 : link = XEXP (link, 1))
15615 : {
15616 87746 : if (GET_CODE (XEXP (link, 0)) == USE)
15617 : {
15618 86407 : rtx arg = XEXP (XEXP (link, 0), 0);
15619 :
15620 86407 : if (ix86_check_avx_upper_register (arg))
15621 : return AVX_U128_DIRTY;
15622 : }
15623 : }
15624 :
15625 : /* Needed mode is set to AVX_U128_CLEAN if there are no 256bit
15626 : nor 512bit registers used in the function return register. */
15627 48814 : bool avx_upper_reg_found = false;
15628 48814 : note_stores (insn, ix86_check_avx_upper_stores,
15629 : &avx_upper_reg_found);
15630 48814 : if (avx_upper_reg_found)
15631 : return AVX_U128_DIRTY;
15632 :
15633 : /* If the function is known to preserve some SSE registers,
15634 : RA and previous passes can legitimately rely on that for
15635 : modes wider than 256 bits. It's only safe to issue a
15636 : vzeroupper if all SSE registers are clobbered. */
15637 48630 : const function_abi &abi = insn_callee_abi (insn);
15638 48630 : if (vzeroupper_pattern (PATTERN (insn), VOIDmode)
15639 : /* Should be safe to issue an vzeroupper before sibling_call_p.
15640 : Also there not mode_exit for sibling_call, so there could be
15641 : missing vzeroupper for that. */
15642 48630 : || !(SIBLING_CALL_P (insn)
15643 47342 : || hard_reg_set_subset_p (reg_class_contents[SSE_REGS],
15644 55776 : abi.mode_clobbers (V4DImode))))
15645 : return AVX_U128_ANY;
15646 :
15647 40196 : return AVX_U128_CLEAN;
15648 : }
15649 :
15650 2035483 : rtx set = single_set (insn);
15651 2035483 : if (set)
15652 : {
15653 1961314 : rtx dest = SET_DEST (set);
15654 1961314 : rtx src = SET_SRC (set);
15655 1476597 : if (SSE_REG_P (dest)
15656 554735 : && !EXT_REX_SSE_REG_P (dest)
15657 3058646 : && GET_MODE_BITSIZE (GET_MODE (dest)) > 128)
15658 : {
15659 : /* This is an YMM/ZMM load. Return AVX_U128_DIRTY if the
15660 : source isn't zero. */
15661 169226 : if (standard_sse_constant_p (src, GET_MODE (dest)) != 1)
15662 : return AVX_U128_DIRTY;
15663 : else
15664 3006 : return AVX_U128_ANY;
15665 : }
15666 : else
15667 : {
15668 1792088 : if (ix86_check_avx_upper_register (src))
15669 : return AVX_U128_DIRTY;
15670 : }
15671 :
15672 : /* This isn't YMM/ZMM load/store. */
15673 1715494 : return AVX_U128_ANY;
15674 : }
15675 :
15676 : /* Require DIRTY mode if a 256bit or 512bit AVX register is referenced.
15677 : Hardware changes state only when a 256bit register is written to,
15678 : but we need to prevent the compiler from moving optimal insertion
15679 : point above eventual read from 256bit or 512 bit register. */
15680 74169 : if (ix86_check_avx_upper_register (PATTERN (insn)))
15681 25390 : return AVX_U128_DIRTY;
15682 :
15683 : return AVX_U128_ANY;
15684 : }
15685 :
15686 : /* Return mode that i387 must be switched into
15687 : prior to the execution of insn. */
15688 :
15689 : static int
15690 423239 : ix86_i387_mode_needed (int entity, rtx_insn *insn)
15691 : {
15692 423239 : enum attr_i387_cw mode;
15693 :
15694 : /* The mode UNINITIALIZED is used to store control word after a
15695 : function call or ASM pattern. The mode ANY specify that function
15696 : has no requirements on the control word and make no changes in the
15697 : bits we are interested in. */
15698 :
15699 423239 : if (CALL_P (insn)
15700 423239 : || (NONJUMP_INSN_P (insn)
15701 346987 : && (asm_noperands (PATTERN (insn)) >= 0
15702 346934 : || GET_CODE (PATTERN (insn)) == ASM_INPUT)))
15703 : return I387_CW_UNINITIALIZED;
15704 :
15705 408189 : if (recog_memoized (insn) < 0)
15706 : return I387_CW_ANY;
15707 :
15708 407247 : mode = get_attr_i387_cw (insn);
15709 :
15710 407247 : switch (entity)
15711 : {
15712 0 : case I387_ROUNDEVEN:
15713 0 : if (mode == I387_CW_ROUNDEVEN)
15714 0 : return mode;
15715 : break;
15716 :
15717 402584 : case I387_TRUNC:
15718 402584 : if (mode == I387_CW_TRUNC)
15719 7377 : return mode;
15720 : break;
15721 :
15722 3628 : case I387_FLOOR:
15723 3628 : if (mode == I387_CW_FLOOR)
15724 139 : return mode;
15725 : break;
15726 :
15727 1035 : case I387_CEIL:
15728 1035 : if (mode == I387_CW_CEIL)
15729 61 : return mode;
15730 : break;
15731 :
15732 0 : default:
15733 0 : gcc_unreachable ();
15734 : }
15735 :
15736 : return I387_CW_ANY;
15737 : }
15738 :
15739 : /* Return mode that entity must be switched into
15740 : prior to the execution of insn. */
15741 :
15742 : static int
15743 2510238 : ix86_mode_needed (int entity, rtx_insn *insn, HARD_REG_SET)
15744 : {
15745 2510238 : switch (entity)
15746 : {
15747 1656 : case X86_DIRFLAG:
15748 1656 : return ix86_dirflag_mode_needed (insn);
15749 2085343 : case AVX_U128:
15750 2085343 : return ix86_avx_u128_mode_needed (insn);
15751 423239 : case I387_ROUNDEVEN:
15752 423239 : case I387_TRUNC:
15753 423239 : case I387_FLOOR:
15754 423239 : case I387_CEIL:
15755 423239 : return ix86_i387_mode_needed (entity, insn);
15756 0 : default:
15757 0 : gcc_unreachable ();
15758 : }
15759 : return 0;
15760 : }
15761 :
15762 : /* Calculate mode of upper 128bit AVX registers after the insn. */
15763 :
15764 : static int
15765 2085343 : ix86_avx_u128_mode_after (int mode, rtx_insn *insn)
15766 : {
15767 2085343 : rtx pat = PATTERN (insn);
15768 :
15769 2085343 : if (vzeroupper_pattern (pat, VOIDmode)
15770 2085343 : || vzeroall_pattern (pat, VOIDmode))
15771 : return AVX_U128_CLEAN;
15772 :
15773 : /* We know that state is clean after CALL insn if there are no
15774 : 256bit or 512bit registers used in the function return register. */
15775 2085184 : if (CALL_P (insn))
15776 : {
15777 49814 : bool avx_upper_reg_found = false;
15778 49814 : note_stores (insn, ix86_check_avx_upper_stores, &avx_upper_reg_found);
15779 :
15780 49814 : if (avx_upper_reg_found)
15781 : return AVX_U128_DIRTY;
15782 :
15783 : /* If the function doesn't clobber any sse registers or only clobber
15784 : 128-bit part, Then vzeroupper isn't issued before the function exit.
15785 : the status not CLEAN but ANY after the function. */
15786 49244 : const function_abi &abi = insn_callee_abi (insn);
15787 49244 : if (!(SIBLING_CALL_P (insn)
15788 47961 : || hard_reg_set_subset_p (reg_class_contents[SSE_REGS],
15789 47961 : abi.mode_clobbers (V4DImode))))
15790 8730 : return AVX_U128_ANY;
15791 :
15792 40514 : return AVX_U128_CLEAN;
15793 : }
15794 :
15795 : /* Otherwise, return current mode. Remember that if insn
15796 : references AVX 256bit or 512bit registers, the mode was already
15797 : changed to DIRTY from MODE_NEEDED. */
15798 : return mode;
15799 : }
15800 :
15801 : /* Return the mode that an insn results in. */
15802 :
15803 : static int
15804 2509393 : ix86_mode_after (int entity, int mode, rtx_insn *insn, HARD_REG_SET)
15805 : {
15806 2509393 : switch (entity)
15807 : {
15808 : case X86_DIRFLAG:
15809 : return mode;
15810 2085343 : case AVX_U128:
15811 2085343 : return ix86_avx_u128_mode_after (mode, insn);
15812 : case I387_ROUNDEVEN:
15813 : case I387_TRUNC:
15814 : case I387_FLOOR:
15815 : case I387_CEIL:
15816 : return mode;
15817 0 : default:
15818 0 : gcc_unreachable ();
15819 : }
15820 : }
15821 :
15822 : static int
15823 120 : ix86_dirflag_mode_entry (void)
15824 : {
15825 : /* For TARGET_CLD or in the interrupt handler we can't assume
15826 : direction flag state at function entry. */
15827 120 : if (TARGET_CLD
15828 118 : || cfun->machine->func_type != TYPE_NORMAL)
15829 120 : return X86_DIRFLAG_ANY;
15830 :
15831 : return X86_DIRFLAG_RESET;
15832 : }
15833 :
15834 : static int
15835 125656 : ix86_avx_u128_mode_entry (void)
15836 : {
15837 125656 : tree arg;
15838 :
15839 : /* Entry mode is set to AVX_U128_DIRTY if there are
15840 : 256bit or 512bit modes used in function arguments. */
15841 316818 : for (arg = DECL_ARGUMENTS (current_function_decl); arg;
15842 191162 : arg = TREE_CHAIN (arg))
15843 : {
15844 226339 : rtx incoming = DECL_INCOMING_RTL (arg);
15845 :
15846 226339 : if (incoming && ix86_check_avx_upper_register (incoming))
15847 : return AVX_U128_DIRTY;
15848 : }
15849 :
15850 : return AVX_U128_CLEAN;
15851 : }
15852 :
15853 : /* Return a mode that ENTITY is assumed to be
15854 : switched to at function entry. */
15855 :
15856 : static int
15857 77322 : ix86_mode_entry (int entity)
15858 : {
15859 77322 : switch (entity)
15860 : {
15861 120 : case X86_DIRFLAG:
15862 120 : return ix86_dirflag_mode_entry ();
15863 76055 : case AVX_U128:
15864 76055 : return ix86_avx_u128_mode_entry ();
15865 : case I387_ROUNDEVEN:
15866 : case I387_TRUNC:
15867 : case I387_FLOOR:
15868 : case I387_CEIL:
15869 : return I387_CW_ANY;
15870 0 : default:
15871 0 : gcc_unreachable ();
15872 : }
15873 : }
15874 :
15875 : static int
15876 74804 : ix86_avx_u128_mode_exit (void)
15877 : {
15878 74804 : rtx reg = crtl->return_rtx;
15879 :
15880 : /* Exit mode is set to AVX_U128_DIRTY if there are 256bit
15881 : or 512 bit modes used in the function return register. */
15882 74804 : if (reg && ix86_check_avx_upper_register (reg))
15883 : return AVX_U128_DIRTY;
15884 :
15885 : /* Exit mode is set to AVX_U128_DIRTY if there are 256bit or 512bit
15886 : modes used in function arguments, otherwise return AVX_U128_CLEAN.
15887 : */
15888 49601 : return ix86_avx_u128_mode_entry ();
15889 : }
15890 :
15891 : /* Return a mode that ENTITY is assumed to be
15892 : switched to at function exit. */
15893 :
15894 : static int
15895 75926 : ix86_mode_exit (int entity)
15896 : {
15897 75926 : switch (entity)
15898 : {
15899 : case X86_DIRFLAG:
15900 : return X86_DIRFLAG_ANY;
15901 74804 : case AVX_U128:
15902 74804 : return ix86_avx_u128_mode_exit ();
15903 1088 : case I387_ROUNDEVEN:
15904 1088 : case I387_TRUNC:
15905 1088 : case I387_FLOOR:
15906 1088 : case I387_CEIL:
15907 1088 : return I387_CW_ANY;
15908 0 : default:
15909 0 : gcc_unreachable ();
15910 : }
15911 : }
15912 :
15913 : static int
15914 2184781 : ix86_mode_priority (int, int n)
15915 : {
15916 2184781 : return n;
15917 : }
15918 :
15919 : /* Output code to initialize control word copies used by trunc?f?i and
15920 : rounding patterns. CURRENT_MODE is set to current control word,
15921 : while NEW_MODE is set to new control word. */
15922 :
15923 : static void
15924 3320 : emit_i387_cw_initialization (int mode)
15925 : {
15926 3320 : rtx stored_mode = assign_386_stack_local (HImode, SLOT_CW_STORED);
15927 3320 : rtx new_mode;
15928 :
15929 3320 : enum ix86_stack_slot slot;
15930 :
15931 3320 : rtx reg = gen_reg_rtx (HImode);
15932 :
15933 3320 : emit_insn (gen_x86_fnstcw_1 (stored_mode));
15934 3320 : emit_move_insn (reg, copy_rtx (stored_mode));
15935 :
15936 3320 : switch (mode)
15937 : {
15938 0 : case I387_CW_ROUNDEVEN:
15939 : /* round to nearest */
15940 0 : emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00)));
15941 0 : slot = SLOT_CW_ROUNDEVEN;
15942 0 : break;
15943 :
15944 3124 : case I387_CW_TRUNC:
15945 : /* round toward zero (truncate) */
15946 3124 : emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0c00)));
15947 3124 : slot = SLOT_CW_TRUNC;
15948 3124 : break;
15949 :
15950 137 : case I387_CW_FLOOR:
15951 : /* round down toward -oo */
15952 137 : emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00)));
15953 137 : emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0400)));
15954 137 : slot = SLOT_CW_FLOOR;
15955 137 : break;
15956 :
15957 59 : case I387_CW_CEIL:
15958 : /* round up toward +oo */
15959 59 : emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00)));
15960 59 : emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0800)));
15961 59 : slot = SLOT_CW_CEIL;
15962 59 : break;
15963 :
15964 0 : default:
15965 0 : gcc_unreachable ();
15966 : }
15967 :
15968 3320 : gcc_assert (slot < MAX_386_STACK_LOCALS);
15969 :
15970 3320 : new_mode = assign_386_stack_local (HImode, slot);
15971 3320 : emit_move_insn (new_mode, reg);
15972 3320 : }
15973 :
15974 : /* Generate one or more insns to set ENTITY to MODE. */
15975 :
15976 : static void
15977 52441 : ix86_emit_mode_set (int entity, int mode, int prev_mode ATTRIBUTE_UNUSED,
15978 : HARD_REG_SET regs_live ATTRIBUTE_UNUSED)
15979 : {
15980 52441 : switch (entity)
15981 : {
15982 265 : case X86_DIRFLAG:
15983 265 : if (mode == X86_DIRFLAG_RESET)
15984 265 : emit_insn (gen_cld ());
15985 : break;
15986 43876 : case AVX_U128:
15987 43876 : if (mode == AVX_U128_CLEAN)
15988 22323 : ix86_expand_avx_vzeroupper ();
15989 : break;
15990 8300 : case I387_ROUNDEVEN:
15991 8300 : case I387_TRUNC:
15992 8300 : case I387_FLOOR:
15993 8300 : case I387_CEIL:
15994 8300 : if (mode != I387_CW_ANY
15995 8300 : && mode != I387_CW_UNINITIALIZED)
15996 3320 : emit_i387_cw_initialization (mode);
15997 : break;
15998 0 : default:
15999 0 : gcc_unreachable ();
16000 : }
16001 52441 : }
16002 :
16003 : /* Output code for INSN to convert a float to a signed int. OPERANDS
16004 : are the insn operands. The output may be [HSD]Imode and the input
16005 : operand may be [SDX]Fmode. */
16006 :
16007 : const char *
16008 7465 : output_fix_trunc (rtx_insn *insn, rtx *operands, bool fisttp)
16009 : {
16010 7465 : bool stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG);
16011 7465 : bool dimode_p = GET_MODE (operands[0]) == DImode;
16012 7465 : int round_mode = get_attr_i387_cw (insn);
16013 :
16014 7465 : static char buf[40];
16015 7465 : const char *p;
16016 :
16017 : /* Jump through a hoop or two for DImode, since the hardware has no
16018 : non-popping instruction. We used to do this a different way, but
16019 : that was somewhat fragile and broke with post-reload splitters. */
16020 7465 : if ((dimode_p || fisttp) && !stack_top_dies)
16021 25 : output_asm_insn ("fld\t%y1", operands);
16022 :
16023 7465 : gcc_assert (STACK_TOP_P (operands[1]));
16024 7465 : gcc_assert (MEM_P (operands[0]));
16025 7465 : gcc_assert (GET_MODE (operands[1]) != TFmode);
16026 :
16027 7465 : if (fisttp)
16028 : return "fisttp%Z0\t%0";
16029 :
16030 7464 : strcpy (buf, "fist");
16031 :
16032 7464 : if (round_mode != I387_CW_ANY)
16033 7420 : output_asm_insn ("fldcw\t%3", operands);
16034 :
16035 7464 : p = "p%Z0\t%0";
16036 7464 : strcat (buf, p + !(stack_top_dies || dimode_p));
16037 :
16038 7464 : output_asm_insn (buf, operands);
16039 :
16040 7464 : if (round_mode != I387_CW_ANY)
16041 7420 : output_asm_insn ("fldcw\t%2", operands);
16042 :
16043 : return "";
16044 : }
16045 :
16046 : /* Output code for x87 ffreep insn. The OPNO argument, which may only
16047 : have the values zero or one, indicates the ffreep insn's operand
16048 : from the OPERANDS array. */
16049 :
16050 : static const char *
16051 274718 : output_387_ffreep (rtx *operands ATTRIBUTE_UNUSED, int opno)
16052 : {
16053 0 : if (TARGET_USE_FFREEP)
16054 : #ifdef HAVE_AS_IX86_FFREEP
16055 0 : return opno ? "ffreep\t%y1" : "ffreep\t%y0";
16056 : #else
16057 : {
16058 : static char retval[32];
16059 : int regno = REGNO (operands[opno]);
16060 :
16061 : gcc_assert (STACK_REGNO_P (regno));
16062 :
16063 : regno -= FIRST_STACK_REG;
16064 :
16065 : snprintf (retval, sizeof (retval), ASM_SHORT "0xc%ddf", regno);
16066 : return retval;
16067 : }
16068 : #endif
16069 :
16070 0 : return opno ? "fstp\t%y1" : "fstp\t%y0";
16071 : }
16072 :
16073 :
16074 : /* Output code for INSN to compare OPERANDS. EFLAGS_P is 1 when fcomi
16075 : should be used. UNORDERED_P is true when fucom should be used. */
16076 :
16077 : const char *
16078 108254 : output_fp_compare (rtx_insn *insn, rtx *operands,
16079 : bool eflags_p, bool unordered_p)
16080 : {
16081 108254 : rtx *xops = eflags_p ? &operands[0] : &operands[1];
16082 108254 : bool stack_top_dies;
16083 :
16084 108254 : static char buf[40];
16085 108254 : const char *p;
16086 :
16087 108254 : gcc_assert (STACK_TOP_P (xops[0]));
16088 :
16089 108254 : stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG);
16090 :
16091 108254 : if (eflags_p)
16092 : {
16093 108254 : p = unordered_p ? "fucomi" : "fcomi";
16094 108254 : strcpy (buf, p);
16095 :
16096 108254 : p = "p\t{%y1, %0|%0, %y1}";
16097 108254 : strcat (buf, p + !stack_top_dies);
16098 :
16099 108254 : return buf;
16100 : }
16101 :
16102 0 : if (STACK_REG_P (xops[1])
16103 0 : && stack_top_dies
16104 0 : && find_regno_note (insn, REG_DEAD, FIRST_STACK_REG + 1))
16105 : {
16106 0 : gcc_assert (REGNO (xops[1]) == FIRST_STACK_REG + 1);
16107 :
16108 : /* If both the top of the 387 stack die, and the other operand
16109 : is also a stack register that dies, then this must be a
16110 : `fcompp' float compare. */
16111 0 : p = unordered_p ? "fucompp" : "fcompp";
16112 0 : strcpy (buf, p);
16113 : }
16114 0 : else if (const0_operand (xops[1], VOIDmode))
16115 : {
16116 0 : gcc_assert (!unordered_p);
16117 0 : strcpy (buf, "ftst");
16118 : }
16119 : else
16120 : {
16121 0 : if (GET_MODE_CLASS (GET_MODE (xops[1])) == MODE_INT)
16122 : {
16123 0 : gcc_assert (!unordered_p);
16124 : p = "ficom";
16125 : }
16126 : else
16127 0 : p = unordered_p ? "fucom" : "fcom";
16128 :
16129 0 : strcpy (buf, p);
16130 :
16131 0 : p = "p%Z2\t%y2";
16132 0 : strcat (buf, p + !stack_top_dies);
16133 : }
16134 :
16135 0 : output_asm_insn (buf, operands);
16136 0 : return "fnstsw\t%0";
16137 : }
16138 :
16139 : void
16140 140037 : ix86_output_addr_vec_elt (FILE *file, int value)
16141 : {
16142 140037 : const char *directive = ASM_LONG;
16143 :
16144 : #ifdef ASM_QUAD
16145 140037 : if (TARGET_LP64)
16146 128305 : directive = ASM_QUAD;
16147 : #else
16148 : gcc_assert (!TARGET_64BIT);
16149 : #endif
16150 :
16151 140037 : fprintf (file, "%s%s%d\n", directive, LPREFIX, value);
16152 140037 : }
16153 :
16154 : void
16155 27471 : ix86_output_addr_diff_elt (FILE *file, int value, int rel)
16156 : {
16157 27471 : const char *directive = ASM_LONG;
16158 :
16159 : #ifdef ASM_QUAD
16160 41129 : if (TARGET_64BIT && CASE_VECTOR_MODE == DImode)
16161 : directive = ASM_QUAD;
16162 : #else
16163 : gcc_assert (!TARGET_64BIT);
16164 : #endif
16165 : /* We can't use @GOTOFF for text labels on VxWorks; see gotoff_operand. */
16166 27471 : if (TARGET_64BIT || TARGET_VXWORKS_VAROFF)
16167 13658 : fprintf (file, "%s%s%d-%s%d\n",
16168 : directive, LPREFIX, value, LPREFIX, rel);
16169 : #if TARGET_MACHO
16170 : else if (TARGET_MACHO)
16171 : {
16172 : fprintf (file, ASM_LONG "%s%d-", LPREFIX, value);
16173 : machopic_output_function_base_name (file);
16174 : putc ('\n', file);
16175 : }
16176 : #endif
16177 13813 : else if (HAVE_AS_GOTOFF_IN_DATA)
16178 13813 : fprintf (file, ASM_LONG "%s%d@GOTOFF\n", LPREFIX, value);
16179 : else
16180 : asm_fprintf (file, ASM_LONG "%U%s+[.-%s%d]\n",
16181 : GOT_SYMBOL_NAME, LPREFIX, value);
16182 27471 : }
16183 :
16184 : #define LEA_MAX_STALL (3)
16185 : #define LEA_SEARCH_THRESHOLD (LEA_MAX_STALL << 1)
16186 :
16187 : /* Increase given DISTANCE in half-cycles according to
16188 : dependencies between PREV and NEXT instructions.
16189 : Add 1 half-cycle if there is no dependency and
16190 : go to next cycle if there is some dependency. */
16191 :
16192 : static unsigned int
16193 2115 : increase_distance (rtx_insn *prev, rtx_insn *next, unsigned int distance)
16194 : {
16195 2115 : df_ref def, use;
16196 :
16197 2115 : if (!prev || !next)
16198 758 : return distance + (distance & 1) + 2;
16199 :
16200 1357 : if (!DF_INSN_USES (next) || !DF_INSN_DEFS (prev))
16201 228 : return distance + 1;
16202 :
16203 1855 : FOR_EACH_INSN_USE (use, next)
16204 2343 : FOR_EACH_INSN_DEF (def, prev)
16205 1617 : if (!DF_REF_IS_ARTIFICIAL (def)
16206 1617 : && DF_REF_REGNO (use) == DF_REF_REGNO (def))
16207 721 : return distance + (distance & 1) + 2;
16208 :
16209 408 : return distance + 1;
16210 : }
16211 :
16212 : /* Function checks if instruction INSN defines register number
16213 : REGNO1 or REGNO2. */
16214 :
16215 : bool
16216 2063 : insn_defines_reg (unsigned int regno1, unsigned int regno2,
16217 : rtx_insn *insn)
16218 : {
16219 2063 : df_ref def;
16220 :
16221 3739 : FOR_EACH_INSN_DEF (def, insn)
16222 2076 : if (DF_REF_REG_DEF_P (def)
16223 2076 : && !DF_REF_IS_ARTIFICIAL (def)
16224 2076 : && (regno1 == DF_REF_REGNO (def)
16225 1693 : || regno2 == DF_REF_REGNO (def)))
16226 : return true;
16227 :
16228 : return false;
16229 : }
16230 :
16231 : /* Function checks if instruction INSN uses register number
16232 : REGNO as a part of address expression. */
16233 :
16234 : static bool
16235 1176 : insn_uses_reg_mem (unsigned int regno, rtx insn)
16236 : {
16237 1176 : df_ref use;
16238 :
16239 2445 : FOR_EACH_INSN_USE (use, insn)
16240 1356 : if (DF_REF_REG_MEM_P (use) && regno == DF_REF_REGNO (use))
16241 : return true;
16242 :
16243 : return false;
16244 : }
16245 :
16246 : /* Search backward for non-agu definition of register number REGNO1
16247 : or register number REGNO2 in basic block starting from instruction
16248 : START up to head of basic block or instruction INSN.
16249 :
16250 : Function puts true value into *FOUND var if definition was found
16251 : and false otherwise.
16252 :
16253 : Distance in half-cycles between START and found instruction or head
16254 : of BB is added to DISTANCE and returned. */
16255 :
16256 : static int
16257 625 : distance_non_agu_define_in_bb (unsigned int regno1, unsigned int regno2,
16258 : rtx_insn *insn, int distance,
16259 : rtx_insn *start, bool *found)
16260 : {
16261 625 : basic_block bb = start ? BLOCK_FOR_INSN (start) : NULL;
16262 625 : rtx_insn *prev = start;
16263 625 : rtx_insn *next = NULL;
16264 :
16265 625 : *found = false;
16266 :
16267 625 : while (prev
16268 1861 : && prev != insn
16269 1861 : && distance < LEA_SEARCH_THRESHOLD)
16270 : {
16271 1660 : if (NONDEBUG_INSN_P (prev) && NONJUMP_INSN_P (prev))
16272 : {
16273 939 : distance = increase_distance (prev, next, distance);
16274 939 : if (insn_defines_reg (regno1, regno2, prev))
16275 : {
16276 238 : if (recog_memoized (prev) < 0
16277 238 : || get_attr_type (prev) != TYPE_LEA)
16278 : {
16279 202 : *found = true;
16280 202 : return distance;
16281 : }
16282 : }
16283 :
16284 : next = prev;
16285 : }
16286 1458 : if (prev == BB_HEAD (bb))
16287 : break;
16288 :
16289 1236 : prev = PREV_INSN (prev);
16290 : }
16291 :
16292 : return distance;
16293 : }
16294 :
16295 : /* Search backward for non-agu definition of register number REGNO1
16296 : or register number REGNO2 in INSN's basic block until
16297 : 1. Pass LEA_SEARCH_THRESHOLD instructions, or
16298 : 2. Reach neighbor BBs boundary, or
16299 : 3. Reach agu definition.
16300 : Returns the distance between the non-agu definition point and INSN.
16301 : If no definition point, returns -1. */
16302 :
16303 : static int
16304 428 : distance_non_agu_define (unsigned int regno1, unsigned int regno2,
16305 : rtx_insn *insn)
16306 : {
16307 428 : basic_block bb = BLOCK_FOR_INSN (insn);
16308 428 : int distance = 0;
16309 428 : bool found = false;
16310 :
16311 428 : if (insn != BB_HEAD (bb))
16312 428 : distance = distance_non_agu_define_in_bb (regno1, regno2, insn,
16313 : distance, PREV_INSN (insn),
16314 : &found);
16315 :
16316 428 : if (!found && distance < LEA_SEARCH_THRESHOLD)
16317 : {
16318 166 : edge e;
16319 166 : edge_iterator ei;
16320 166 : bool simple_loop = false;
16321 :
16322 337 : FOR_EACH_EDGE (e, ei, bb->preds)
16323 208 : if (e->src == bb)
16324 : {
16325 : simple_loop = true;
16326 : break;
16327 : }
16328 :
16329 166 : if (simple_loop)
16330 37 : distance = distance_non_agu_define_in_bb (regno1, regno2,
16331 : insn, distance,
16332 37 : BB_END (bb), &found);
16333 : else
16334 : {
16335 129 : int shortest_dist = -1;
16336 129 : bool found_in_bb = false;
16337 :
16338 289 : FOR_EACH_EDGE (e, ei, bb->preds)
16339 : {
16340 160 : int bb_dist
16341 320 : = distance_non_agu_define_in_bb (regno1, regno2,
16342 : insn, distance,
16343 160 : BB_END (e->src),
16344 : &found_in_bb);
16345 160 : if (found_in_bb)
16346 : {
16347 24 : if (shortest_dist < 0)
16348 : shortest_dist = bb_dist;
16349 0 : else if (bb_dist > 0)
16350 0 : shortest_dist = MIN (bb_dist, shortest_dist);
16351 :
16352 24 : found = true;
16353 : }
16354 : }
16355 :
16356 129 : distance = shortest_dist;
16357 : }
16358 : }
16359 :
16360 428 : if (!found)
16361 : return -1;
16362 :
16363 202 : return distance >> 1;
16364 : }
16365 :
16366 : /* Return the distance in half-cycles between INSN and the next
16367 : insn that uses register number REGNO in memory address added
16368 : to DISTANCE. Return -1 if REGNO0 is set.
16369 :
16370 : Put true value into *FOUND if register usage was found and
16371 : false otherwise.
16372 : Put true value into *REDEFINED if register redefinition was
16373 : found and false otherwise. */
16374 :
16375 : static int
16376 770 : distance_agu_use_in_bb (unsigned int regno,
16377 : rtx_insn *insn, int distance, rtx_insn *start,
16378 : bool *found, bool *redefined)
16379 : {
16380 770 : basic_block bb = NULL;
16381 770 : rtx_insn *next = start;
16382 770 : rtx_insn *prev = NULL;
16383 :
16384 770 : *found = false;
16385 770 : *redefined = false;
16386 :
16387 770 : if (start != NULL_RTX)
16388 : {
16389 753 : bb = BLOCK_FOR_INSN (start);
16390 753 : if (start != BB_HEAD (bb))
16391 : /* If insn and start belong to the same bb, set prev to insn,
16392 : so the call to increase_distance will increase the distance
16393 : between insns by 1. */
16394 411 : prev = insn;
16395 : }
16396 :
16397 2559 : while (next
16398 2559 : && next != insn
16399 2559 : && distance < LEA_SEARCH_THRESHOLD)
16400 : {
16401 2374 : if (NONDEBUG_INSN_P (next) && NONJUMP_INSN_P (next))
16402 : {
16403 1176 : distance = increase_distance(prev, next, distance);
16404 1176 : if (insn_uses_reg_mem (regno, next))
16405 : {
16406 : /* Return DISTANCE if OP0 is used in memory
16407 : address in NEXT. */
16408 87 : *found = true;
16409 87 : return distance;
16410 : }
16411 :
16412 1089 : if (insn_defines_reg (regno, INVALID_REGNUM, next))
16413 : {
16414 : /* Return -1 if OP0 is set in NEXT. */
16415 157 : *redefined = true;
16416 157 : return -1;
16417 : }
16418 :
16419 : prev = next;
16420 : }
16421 :
16422 2130 : if (next == BB_END (bb))
16423 : break;
16424 :
16425 1789 : next = NEXT_INSN (next);
16426 : }
16427 :
16428 : return distance;
16429 : }
16430 :
16431 : /* Return the distance between INSN and the next insn that uses
16432 : register number REGNO0 in memory address. Return -1 if no such
16433 : a use is found within LEA_SEARCH_THRESHOLD or REGNO0 is set. */
16434 :
16435 : static int
16436 428 : distance_agu_use (unsigned int regno0, rtx_insn *insn)
16437 : {
16438 428 : basic_block bb = BLOCK_FOR_INSN (insn);
16439 428 : int distance = 0;
16440 428 : bool found = false;
16441 428 : bool redefined = false;
16442 :
16443 428 : if (insn != BB_END (bb))
16444 411 : distance = distance_agu_use_in_bb (regno0, insn, distance,
16445 : NEXT_INSN (insn),
16446 : &found, &redefined);
16447 :
16448 428 : if (!found && !redefined && distance < LEA_SEARCH_THRESHOLD)
16449 : {
16450 255 : edge e;
16451 255 : edge_iterator ei;
16452 255 : bool simple_loop = false;
16453 :
16454 544 : FOR_EACH_EDGE (e, ei, bb->succs)
16455 359 : if (e->dest == bb)
16456 : {
16457 : simple_loop = true;
16458 : break;
16459 : }
16460 :
16461 255 : if (simple_loop)
16462 70 : distance = distance_agu_use_in_bb (regno0, insn,
16463 : distance, BB_HEAD (bb),
16464 : &found, &redefined);
16465 : else
16466 : {
16467 185 : int shortest_dist = -1;
16468 185 : bool found_in_bb = false;
16469 185 : bool redefined_in_bb = false;
16470 :
16471 474 : FOR_EACH_EDGE (e, ei, bb->succs)
16472 : {
16473 289 : int bb_dist
16474 578 : = distance_agu_use_in_bb (regno0, insn,
16475 289 : distance, BB_HEAD (e->dest),
16476 : &found_in_bb, &redefined_in_bb);
16477 289 : if (found_in_bb)
16478 : {
16479 15 : if (shortest_dist < 0)
16480 : shortest_dist = bb_dist;
16481 0 : else if (bb_dist > 0)
16482 0 : shortest_dist = MIN (bb_dist, shortest_dist);
16483 :
16484 15 : found = true;
16485 : }
16486 : }
16487 :
16488 185 : distance = shortest_dist;
16489 : }
16490 : }
16491 :
16492 428 : if (!found || redefined)
16493 : return -1;
16494 :
16495 87 : return distance >> 1;
16496 : }
16497 :
16498 : /* Define this macro to tune LEA priority vs ADD, it take effect when
16499 : there is a dilemma of choosing LEA or ADD
16500 : Negative value: ADD is more preferred than LEA
16501 : Zero: Neutral
16502 : Positive value: LEA is more preferred than ADD. */
16503 : #define IX86_LEA_PRIORITY 0
16504 :
16505 : /* Return true if usage of lea INSN has performance advantage
16506 : over a sequence of instructions. Instructions sequence has
16507 : SPLIT_COST cycles higher latency than lea latency. */
16508 :
16509 : static bool
16510 1596 : ix86_lea_outperforms (rtx_insn *insn, unsigned int regno0, unsigned int regno1,
16511 : unsigned int regno2, int split_cost, bool has_scale)
16512 : {
16513 1596 : int dist_define, dist_use;
16514 :
16515 : /* For Atom processors newer than Bonnell, if using a 2-source or
16516 : 3-source LEA for non-destructive destination purposes, or due to
16517 : wanting ability to use SCALE, the use of LEA is justified. */
16518 1596 : if (!TARGET_CPU_P (BONNELL))
16519 : {
16520 1168 : if (has_scale)
16521 : return true;
16522 1149 : if (split_cost < 1)
16523 : return false;
16524 406 : if (regno0 == regno1 || regno0 == regno2)
16525 : return false;
16526 406 : return true;
16527 : }
16528 :
16529 : /* distance_non_agu_define can call get_attr_type which can call
16530 : recog_memoized, restore recog_data back to previous content. */
16531 428 : {
16532 428 : recog_state_saver recog_save;
16533 428 : dist_define = distance_non_agu_define (regno1, regno2, insn);
16534 428 : dist_use = distance_agu_use (regno0, insn);
16535 428 : }
16536 :
16537 428 : if (dist_define < 0 || dist_define >= LEA_MAX_STALL)
16538 : {
16539 : /* If there is no non AGU operand definition, no AGU
16540 : operand usage and split cost is 0 then both lea
16541 : and non lea variants have same priority. Currently
16542 : we prefer lea for 64 bit code and non lea on 32 bit
16543 : code. */
16544 228 : if (dist_use < 0 && split_cost == 0)
16545 103 : return TARGET_64BIT || IX86_LEA_PRIORITY;
16546 : else
16547 : return true;
16548 : }
16549 :
16550 : /* With longer definitions distance lea is more preferable.
16551 : Here we change it to take into account splitting cost and
16552 : lea priority. */
16553 200 : dist_define += split_cost + IX86_LEA_PRIORITY;
16554 :
16555 : /* If there is no use in memory address then we just check
16556 : that split cost exceeds AGU stall. */
16557 200 : if (dist_use < 0)
16558 196 : return dist_define > LEA_MAX_STALL;
16559 :
16560 : /* If this insn has both backward non-agu dependence and forward
16561 : agu dependence, the one with short distance takes effect. */
16562 4 : return dist_define >= dist_use;
16563 : }
16564 :
16565 : /* Return true if we need to split op0 = op1 + op2 into a sequence of
16566 : move and add to avoid AGU stalls. */
16567 :
16568 : bool
16569 9389704 : ix86_avoid_lea_for_add (rtx_insn *insn, rtx operands[])
16570 : {
16571 9389704 : unsigned int regno0, regno1, regno2;
16572 :
16573 : /* Check if we need to optimize. */
16574 9389704 : if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
16575 : return false;
16576 :
16577 779 : regno0 = true_regnum (operands[0]);
16578 779 : regno1 = true_regnum (operands[1]);
16579 779 : regno2 = true_regnum (operands[2]);
16580 :
16581 : /* We need to split only adds with non destructive
16582 : destination operand. */
16583 779 : if (regno0 == regno1 || regno0 == regno2)
16584 : return false;
16585 : else
16586 244 : return !ix86_lea_outperforms (insn, regno0, regno1, regno2, 1, false);
16587 : }
16588 :
16589 : /* Return true if we should emit lea instruction instead of mov
16590 : instruction. */
16591 :
16592 : bool
16593 30517993 : ix86_use_lea_for_mov (rtx_insn *insn, rtx operands[])
16594 : {
16595 30517993 : unsigned int regno0, regno1;
16596 :
16597 : /* Check if we need to optimize. */
16598 30517993 : if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
16599 : return false;
16600 :
16601 : /* Use lea for reg to reg moves only. */
16602 2274 : if (!REG_P (operands[0]) || !REG_P (operands[1]))
16603 : return false;
16604 :
16605 469 : regno0 = true_regnum (operands[0]);
16606 469 : regno1 = true_regnum (operands[1]);
16607 :
16608 469 : return ix86_lea_outperforms (insn, regno0, regno1, INVALID_REGNUM, 0, false);
16609 : }
16610 :
16611 : /* Return true if we need to split lea into a sequence of
16612 : instructions to avoid AGU stalls during peephole2. */
16613 :
16614 : bool
16615 11683231 : ix86_avoid_lea_for_addr (rtx_insn *insn, rtx operands[])
16616 : {
16617 11683231 : unsigned int regno0, regno1, regno2;
16618 11683231 : int split_cost;
16619 11683231 : struct ix86_address parts;
16620 11683231 : int ok;
16621 :
16622 : /* The "at least two components" test below might not catch simple
16623 : move or zero extension insns if parts.base is non-NULL and parts.disp
16624 : is const0_rtx as the only components in the address, e.g. if the
16625 : register is %rbp or %r13. As this test is much cheaper and moves or
16626 : zero extensions are the common case, do this check first. */
16627 11683231 : if (REG_P (operands[1])
16628 11683231 : || (SImode_address_operand (operands[1], VOIDmode)
16629 141498 : && REG_P (XEXP (operands[1], 0))))
16630 : return false;
16631 :
16632 7372398 : ok = ix86_decompose_address (operands[1], &parts);
16633 7372398 : gcc_assert (ok);
16634 :
16635 : /* There should be at least two components in the address. */
16636 7372398 : if ((parts.base != NULL_RTX) + (parts.index != NULL_RTX)
16637 7372398 : + (parts.disp != NULL_RTX) + (parts.scale > 1) < 2)
16638 : return false;
16639 :
16640 : /* We should not split into add if non legitimate pic
16641 : operand is used as displacement. */
16642 2810261 : if (parts.disp && flag_pic && !LEGITIMATE_PIC_OPERAND_P (parts.disp))
16643 : return false;
16644 :
16645 2760174 : regno0 = true_regnum (operands[0]) ;
16646 2760174 : regno1 = INVALID_REGNUM;
16647 2760174 : regno2 = INVALID_REGNUM;
16648 :
16649 2760174 : if (parts.base)
16650 2685177 : regno1 = true_regnum (parts.base);
16651 2760174 : if (parts.index)
16652 486839 : regno2 = true_regnum (parts.index);
16653 :
16654 : /* Use add for a = a + b and a = b + a since it is faster and shorter
16655 : than lea for most processors. For the processors like BONNELL, if
16656 : the destination register of LEA holds an actual address which will
16657 : be used soon, LEA is better and otherwise ADD is better. */
16658 2760174 : if (!TARGET_CPU_P (BONNELL)
16659 2760047 : && parts.scale == 1
16660 2515628 : && (!parts.disp || parts.disp == const0_rtx)
16661 186112 : && (regno0 == regno1 || regno0 == regno2))
16662 : return true;
16663 :
16664 : /* Split with -Oz if the encoding requires fewer bytes. */
16665 2753427 : if (optimize_size > 1
16666 27 : && parts.scale > 1
16667 4 : && !parts.base
16668 4 : && (!parts.disp || parts.disp == const0_rtx))
16669 : return true;
16670 :
16671 : /* Check we need to optimize. */
16672 2753423 : if (!TARGET_AVOID_LEA_FOR_ADDR || optimize_function_for_size_p (cfun))
16673 : return false;
16674 :
16675 339 : split_cost = 0;
16676 :
16677 : /* Compute how many cycles we will add to execution time
16678 : if split lea into a sequence of instructions. */
16679 339 : if (parts.base || parts.index)
16680 : {
16681 : /* Have to use mov instruction if non destructive
16682 : destination form is used. */
16683 339 : if (regno1 != regno0 && regno2 != regno0)
16684 262 : split_cost += 1;
16685 :
16686 : /* Have to add index to base if both exist. */
16687 339 : if (parts.base && parts.index)
16688 54 : split_cost += 1;
16689 :
16690 : /* Have to use shift and adds if scale is 2 or greater. */
16691 339 : if (parts.scale > 1)
16692 : {
16693 30 : if (regno0 != regno1)
16694 24 : split_cost += 1;
16695 6 : else if (regno2 == regno0)
16696 0 : split_cost += 4;
16697 : else
16698 6 : split_cost += parts.scale;
16699 : }
16700 :
16701 : /* Have to use add instruction with immediate if
16702 : disp is non zero. */
16703 339 : if (parts.disp && parts.disp != const0_rtx)
16704 278 : split_cost += 1;
16705 :
16706 : /* Subtract the price of lea. */
16707 339 : split_cost -= 1;
16708 : }
16709 :
16710 339 : return !ix86_lea_outperforms (insn, regno0, regno1, regno2, split_cost,
16711 339 : parts.scale > 1);
16712 : }
16713 :
16714 : /* Return true if it is ok to optimize an ADD operation to LEA
16715 : operation to avoid flag register consumation. For most processors,
16716 : ADD is faster than LEA. For the processors like BONNELL, if the
16717 : destination register of LEA holds an actual address which will be
16718 : used soon, LEA is better and otherwise ADD is better. */
16719 :
16720 : bool
16721 9451692 : ix86_lea_for_add_ok (rtx_insn *insn, rtx operands[])
16722 : {
16723 9451692 : unsigned int regno0 = true_regnum (operands[0]);
16724 9451692 : unsigned int regno1 = true_regnum (operands[1]);
16725 9451692 : unsigned int regno2 = true_regnum (operands[2]);
16726 :
16727 : /* If a = b + c, (a!=b && a!=c), must use lea form. */
16728 9451692 : if (regno0 != regno1 && regno0 != regno2)
16729 : return true;
16730 :
16731 7331614 : if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
16732 : return false;
16733 :
16734 544 : return ix86_lea_outperforms (insn, regno0, regno1, regno2, 0, false);
16735 : }
16736 :
16737 : /* Return true if destination reg of SET_BODY is shift count of
16738 : USE_BODY. */
16739 :
16740 : static bool
16741 85 : ix86_dep_by_shift_count_body (const_rtx set_body, const_rtx use_body)
16742 : {
16743 85 : rtx set_dest;
16744 85 : rtx shift_rtx;
16745 85 : int i;
16746 :
16747 : /* Retrieve destination of SET_BODY. */
16748 85 : switch (GET_CODE (set_body))
16749 : {
16750 67 : case SET:
16751 67 : set_dest = SET_DEST (set_body);
16752 67 : if (!set_dest || !REG_P (set_dest))
16753 : return false;
16754 66 : break;
16755 9 : case PARALLEL:
16756 27 : for (i = XVECLEN (set_body, 0) - 1; i >= 0; i--)
16757 18 : if (ix86_dep_by_shift_count_body (XVECEXP (set_body, 0, i),
16758 : use_body))
16759 : return true;
16760 : /* FALLTHROUGH */
16761 : default:
16762 : return false;
16763 : }
16764 :
16765 : /* Retrieve shift count of USE_BODY. */
16766 66 : switch (GET_CODE (use_body))
16767 : {
16768 22 : case SET:
16769 22 : shift_rtx = XEXP (use_body, 1);
16770 22 : break;
16771 22 : case PARALLEL:
16772 66 : for (i = XVECLEN (use_body, 0) - 1; i >= 0; i--)
16773 44 : if (ix86_dep_by_shift_count_body (set_body,
16774 44 : XVECEXP (use_body, 0, i)))
16775 : return true;
16776 : /* FALLTHROUGH */
16777 : default:
16778 : return false;
16779 : }
16780 :
16781 22 : if (shift_rtx
16782 22 : && (GET_CODE (shift_rtx) == ASHIFT
16783 21 : || GET_CODE (shift_rtx) == LSHIFTRT
16784 5 : || GET_CODE (shift_rtx) == ASHIFTRT
16785 0 : || GET_CODE (shift_rtx) == ROTATE
16786 0 : || GET_CODE (shift_rtx) == ROTATERT))
16787 : {
16788 22 : rtx shift_count = XEXP (shift_rtx, 1);
16789 :
16790 : /* Return true if shift count is dest of SET_BODY. */
16791 22 : if (REG_P (shift_count))
16792 : {
16793 : /* Add check since it can be invoked before register
16794 : allocation in pre-reload schedule. */
16795 0 : if (reload_completed
16796 0 : && true_regnum (set_dest) == true_regnum (shift_count))
16797 : return true;
16798 0 : else if (REGNO(set_dest) == REGNO(shift_count))
16799 0 : return true;
16800 : }
16801 : }
16802 :
16803 : return false;
16804 : }
16805 :
16806 : /* Return true if destination reg of SET_INSN is shift count of
16807 : USE_INSN. */
16808 :
16809 : bool
16810 23 : ix86_dep_by_shift_count (const_rtx set_insn, const_rtx use_insn)
16811 : {
16812 23 : return ix86_dep_by_shift_count_body (PATTERN (set_insn),
16813 23 : PATTERN (use_insn));
16814 : }
16815 :
16816 : /* Return TRUE if the operands to a vec_interleave_{high,low}v2df
16817 : are ok, keeping in mind the possible movddup alternative. */
16818 :
16819 : bool
16820 93713 : ix86_vec_interleave_v2df_operator_ok (rtx operands[3], bool high)
16821 : {
16822 93713 : if (MEM_P (operands[0]))
16823 2114 : return rtx_equal_p (operands[0], operands[1 + high]);
16824 91599 : if (MEM_P (operands[1]) && MEM_P (operands[2]))
16825 1129 : return false;
16826 : return true;
16827 : }
16828 :
16829 : /* A subroutine of ix86_build_signbit_mask. If VECT is true,
16830 : then replicate the value for all elements of the vector
16831 : register. */
16832 :
16833 : rtx
16834 76373 : ix86_build_const_vector (machine_mode mode, bool vect, rtx value)
16835 : {
16836 76373 : int i, n_elt;
16837 76373 : rtvec v;
16838 76373 : machine_mode scalar_mode;
16839 :
16840 76373 : switch (mode)
16841 : {
16842 1507 : case E_V64QImode:
16843 1507 : case E_V32QImode:
16844 1507 : case E_V16QImode:
16845 1507 : case E_V32HImode:
16846 1507 : case E_V16HImode:
16847 1507 : case E_V8HImode:
16848 1507 : case E_V16SImode:
16849 1507 : case E_V8SImode:
16850 1507 : case E_V4SImode:
16851 1507 : case E_V2SImode:
16852 1507 : case E_V8DImode:
16853 1507 : case E_V4DImode:
16854 1507 : case E_V2DImode:
16855 1507 : gcc_assert (vect);
16856 : /* FALLTHRU */
16857 76373 : case E_V2HFmode:
16858 76373 : case E_V4HFmode:
16859 76373 : case E_V8HFmode:
16860 76373 : case E_V16HFmode:
16861 76373 : case E_V32HFmode:
16862 76373 : case E_V16SFmode:
16863 76373 : case E_V8SFmode:
16864 76373 : case E_V4SFmode:
16865 76373 : case E_V2SFmode:
16866 76373 : case E_V8DFmode:
16867 76373 : case E_V4DFmode:
16868 76373 : case E_V2DFmode:
16869 76373 : case E_V32BFmode:
16870 76373 : case E_V16BFmode:
16871 76373 : case E_V8BFmode:
16872 76373 : case E_V4BFmode:
16873 76373 : case E_V2BFmode:
16874 76373 : n_elt = GET_MODE_NUNITS (mode);
16875 76373 : v = rtvec_alloc (n_elt);
16876 76373 : scalar_mode = GET_MODE_INNER (mode);
16877 :
16878 76373 : RTVEC_ELT (v, 0) = value;
16879 :
16880 237490 : for (i = 1; i < n_elt; ++i)
16881 161117 : RTVEC_ELT (v, i) = vect ? value : CONST0_RTX (scalar_mode);
16882 :
16883 76373 : return gen_rtx_CONST_VECTOR (mode, v);
16884 :
16885 0 : default:
16886 0 : gcc_unreachable ();
16887 : }
16888 : }
16889 :
16890 : /* A subroutine of ix86_expand_fp_absneg_operator, copysign expanders
16891 : and ix86_expand_int_vcond. Create a mask for the sign bit in MODE
16892 : for an SSE register. If VECT is true, then replicate the mask for
16893 : all elements of the vector register. If INVERT is true, then create
16894 : a mask excluding the sign bit. */
16895 :
16896 : rtx
16897 78057 : ix86_build_signbit_mask (machine_mode mode, bool vect, bool invert)
16898 : {
16899 78057 : machine_mode vec_mode, imode;
16900 78057 : wide_int w;
16901 78057 : rtx mask, v;
16902 :
16903 78057 : switch (mode)
16904 : {
16905 : case E_V2HFmode:
16906 : case E_V4HFmode:
16907 : case E_V8HFmode:
16908 : case E_V16HFmode:
16909 : case E_V32HFmode:
16910 : case E_V32BFmode:
16911 : case E_V16BFmode:
16912 : case E_V8BFmode:
16913 : case E_V4BFmode:
16914 : case E_V2BFmode:
16915 : vec_mode = mode;
16916 : imode = HImode;
16917 : break;
16918 :
16919 34836 : case E_V16SImode:
16920 34836 : case E_V16SFmode:
16921 34836 : case E_V8SImode:
16922 34836 : case E_V4SImode:
16923 34836 : case E_V8SFmode:
16924 34836 : case E_V4SFmode:
16925 34836 : case E_V2SFmode:
16926 34836 : case E_V2SImode:
16927 34836 : vec_mode = mode;
16928 34836 : imode = SImode;
16929 34836 : break;
16930 :
16931 39935 : case E_V8DImode:
16932 39935 : case E_V4DImode:
16933 39935 : case E_V2DImode:
16934 39935 : case E_V8DFmode:
16935 39935 : case E_V4DFmode:
16936 39935 : case E_V2DFmode:
16937 39935 : vec_mode = mode;
16938 39935 : imode = DImode;
16939 39935 : break;
16940 :
16941 2781 : case E_TImode:
16942 2781 : case E_TFmode:
16943 2781 : vec_mode = VOIDmode;
16944 2781 : imode = TImode;
16945 2781 : break;
16946 :
16947 0 : default:
16948 0 : gcc_unreachable ();
16949 : }
16950 :
16951 78057 : machine_mode inner_mode = GET_MODE_INNER (mode);
16952 156114 : w = wi::set_bit_in_zero (GET_MODE_BITSIZE (inner_mode) - 1,
16953 156114 : GET_MODE_BITSIZE (inner_mode));
16954 78057 : if (invert)
16955 41392 : w = wi::bit_not (w);
16956 :
16957 : /* Force this value into the low part of a fp vector constant. */
16958 78057 : mask = immed_wide_int_const (w, imode);
16959 78057 : mask = gen_lowpart (inner_mode, mask);
16960 :
16961 78057 : if (vec_mode == VOIDmode)
16962 2781 : return force_reg (inner_mode, mask);
16963 :
16964 75276 : v = ix86_build_const_vector (vec_mode, vect, mask);
16965 75276 : return force_reg (vec_mode, v);
16966 78057 : }
16967 :
16968 : /* Return HOST_WIDE_INT for const vector OP in MODE. */
16969 :
16970 : HOST_WIDE_INT
16971 137725 : ix86_convert_const_vector_to_integer (rtx op, machine_mode mode)
16972 : {
16973 291673 : if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
16974 0 : gcc_unreachable ();
16975 :
16976 137725 : int nunits = GET_MODE_NUNITS (mode);
16977 275450 : wide_int val = wi::zero (GET_MODE_BITSIZE (mode));
16978 137725 : machine_mode innermode = GET_MODE_INNER (mode);
16979 137725 : unsigned int innermode_bits = GET_MODE_BITSIZE (innermode);
16980 :
16981 137725 : switch (mode)
16982 : {
16983 : case E_V2QImode:
16984 : case E_V4QImode:
16985 : case E_V2HImode:
16986 : case E_V8QImode:
16987 : case E_V4HImode:
16988 : case E_V2SImode:
16989 471041 : for (int i = 0; i < nunits; ++i)
16990 : {
16991 338696 : int v = INTVAL (XVECEXP (op, 0, i));
16992 338696 : wide_int wv = wi::shwi (v, innermode_bits);
16993 338696 : val = wi::insert (val, wv, innermode_bits * i, innermode_bits);
16994 338696 : }
16995 : break;
16996 145 : case E_V1SImode:
16997 145 : case E_V1DImode:
16998 145 : op = CONST_VECTOR_ELT (op, 0);
16999 145 : return INTVAL (op);
17000 : case E_V2HFmode:
17001 : case E_V2BFmode:
17002 : case E_V4HFmode:
17003 : case E_V4BFmode:
17004 : case E_V2SFmode:
17005 15751 : for (int i = 0; i < nunits; ++i)
17006 : {
17007 10516 : rtx x = XVECEXP (op, 0, i);
17008 10516 : int v = real_to_target (NULL, CONST_DOUBLE_REAL_VALUE (x),
17009 10516 : REAL_MODE_FORMAT (innermode));
17010 10516 : wide_int wv = wi::shwi (v, innermode_bits);
17011 10516 : val = wi::insert (val, wv, innermode_bits * i, innermode_bits);
17012 10516 : }
17013 : break;
17014 0 : default:
17015 0 : gcc_unreachable ();
17016 : }
17017 :
17018 137580 : return val.to_shwi ();
17019 137725 : }
17020 :
17021 34 : int ix86_get_flags_cc (rtx_code code)
17022 : {
17023 34 : switch (code)
17024 : {
17025 : case NE: return X86_CCNE;
17026 : case EQ: return X86_CCE;
17027 : case GE: return X86_CCNL;
17028 : case GT: return X86_CCNLE;
17029 : case LE: return X86_CCLE;
17030 : case LT: return X86_CCL;
17031 : case GEU: return X86_CCNB;
17032 : case GTU: return X86_CCNBE;
17033 : case LEU: return X86_CCBE;
17034 : case LTU: return X86_CCB;
17035 : default: return -1;
17036 : }
17037 : }
17038 :
17039 : /* Return TRUE or FALSE depending on whether the first SET in INSN
17040 : has source and destination with matching CC modes, and that the
17041 : CC mode is at least as constrained as REQ_MODE. */
17042 :
17043 : bool
17044 55453034 : ix86_match_ccmode (rtx insn, machine_mode req_mode)
17045 : {
17046 55453034 : rtx set;
17047 55453034 : machine_mode set_mode;
17048 :
17049 55453034 : set = PATTERN (insn);
17050 55453034 : if (GET_CODE (set) == PARALLEL)
17051 545075 : set = XVECEXP (set, 0, 0);
17052 55453034 : gcc_assert (GET_CODE (set) == SET);
17053 55453034 : gcc_assert (GET_CODE (SET_SRC (set)) == COMPARE);
17054 :
17055 55453034 : set_mode = GET_MODE (SET_DEST (set));
17056 55453034 : switch (set_mode)
17057 : {
17058 1448604 : case E_CCNOmode:
17059 1448604 : if (req_mode != CCNOmode
17060 103602 : && (req_mode != CCmode
17061 0 : || XEXP (SET_SRC (set), 1) != const0_rtx))
17062 : return false;
17063 : break;
17064 5980265 : case E_CCmode:
17065 5980265 : if (req_mode == CCGCmode)
17066 : return false;
17067 : /* FALLTHRU */
17068 9602861 : case E_CCGCmode:
17069 9602861 : if (req_mode == CCGOCmode || req_mode == CCNOmode)
17070 : return false;
17071 : /* FALLTHRU */
17072 10708276 : case E_CCGOCmode:
17073 10708276 : if (req_mode == CCZmode)
17074 : return false;
17075 : /* FALLTHRU */
17076 : case E_CCZmode:
17077 : break;
17078 :
17079 0 : case E_CCGZmode:
17080 :
17081 0 : case E_CCAmode:
17082 0 : case E_CCCmode:
17083 0 : case E_CCOmode:
17084 0 : case E_CCPmode:
17085 0 : case E_CCSmode:
17086 0 : if (set_mode != req_mode)
17087 : return false;
17088 : break;
17089 :
17090 0 : default:
17091 0 : gcc_unreachable ();
17092 : }
17093 :
17094 55340373 : return GET_MODE (SET_SRC (set)) == set_mode;
17095 : }
17096 :
17097 : machine_mode
17098 14129789 : ix86_cc_mode (enum rtx_code code, rtx op0, rtx op1)
17099 : {
17100 14129789 : machine_mode mode = GET_MODE (op0);
17101 :
17102 14129789 : if (SCALAR_FLOAT_MODE_P (mode))
17103 : {
17104 151422 : gcc_assert (!DECIMAL_FLOAT_MODE_P (mode));
17105 : return CCFPmode;
17106 : }
17107 :
17108 13978367 : switch (code)
17109 : {
17110 : /* Only zero flag is needed. */
17111 : case EQ: /* ZF=0 */
17112 : case NE: /* ZF!=0 */
17113 : return CCZmode;
17114 : /* Codes needing carry flag. */
17115 1000766 : case GEU: /* CF=0 */
17116 1000766 : case LTU: /* CF=1 */
17117 1000766 : rtx geu;
17118 : /* Detect overflow checks. They need just the carry flag. */
17119 1000766 : if (GET_CODE (op0) == PLUS
17120 1000766 : && (rtx_equal_p (op1, XEXP (op0, 0))
17121 102149 : || rtx_equal_p (op1, XEXP (op0, 1))))
17122 : return CCCmode;
17123 : /* Similarly for *setcc_qi_addqi3_cconly_overflow_1_* patterns.
17124 : Match LTU of op0
17125 : (neg:QI (geu:QI (reg:CC_CCC FLAGS_REG) (const_int 0)))
17126 : and op1
17127 : (ltu:QI (reg:CC_CCC FLAGS_REG) (const_int 0))
17128 : where CC_CCC is either CC or CCC. */
17129 985617 : else if (code == LTU
17130 359218 : && GET_CODE (op0) == NEG
17131 19 : && GET_CODE (geu = XEXP (op0, 0)) == GEU
17132 0 : && REG_P (XEXP (geu, 0))
17133 0 : && (GET_MODE (XEXP (geu, 0)) == CCCmode
17134 0 : || GET_MODE (XEXP (geu, 0)) == CCmode)
17135 0 : && REGNO (XEXP (geu, 0)) == FLAGS_REG
17136 0 : && XEXP (geu, 1) == const0_rtx
17137 0 : && GET_CODE (op1) == LTU
17138 0 : && REG_P (XEXP (op1, 0))
17139 0 : && GET_MODE (XEXP (op1, 0)) == GET_MODE (XEXP (geu, 0))
17140 0 : && REGNO (XEXP (op1, 0)) == FLAGS_REG
17141 985617 : && XEXP (op1, 1) == const0_rtx)
17142 : return CCCmode;
17143 : /* Similarly for *x86_cmc pattern.
17144 : Match LTU of op0 (neg:QI (ltu:QI (reg:CCC FLAGS_REG) (const_int 0)))
17145 : and op1 (geu:QI (reg:CCC FLAGS_REG) (const_int 0)).
17146 : It is sufficient to test that the operand modes are CCCmode. */
17147 985617 : else if (code == LTU
17148 359218 : && GET_CODE (op0) == NEG
17149 19 : && GET_CODE (XEXP (op0, 0)) == LTU
17150 0 : && GET_MODE (XEXP (XEXP (op0, 0), 0)) == CCCmode
17151 0 : && GET_CODE (op1) == GEU
17152 0 : && GET_MODE (XEXP (op1, 0)) == CCCmode)
17153 : return CCCmode;
17154 : /* Similarly for the comparison of addcarry/subborrow pattern. */
17155 359218 : else if (code == LTU
17156 359218 : && GET_CODE (op0) == ZERO_EXTEND
17157 15659 : && GET_CODE (op1) == PLUS
17158 11152 : && ix86_carry_flag_operator (XEXP (op1, 0), VOIDmode)
17159 11152 : && GET_CODE (XEXP (op1, 1)) == ZERO_EXTEND)
17160 : return CCCmode;
17161 : else
17162 : return CCmode;
17163 : case GTU: /* CF=0 & ZF=0 */
17164 : case LEU: /* CF=1 | ZF=1 */
17165 : return CCmode;
17166 : /* Codes possibly doable only with sign flag when
17167 : comparing against zero. */
17168 813290 : case GE: /* SF=OF or SF=0 */
17169 813290 : case LT: /* SF<>OF or SF=1 */
17170 813290 : if (op1 == const0_rtx)
17171 : return CCGOCmode;
17172 : else
17173 : /* For other cases Carry flag is not required. */
17174 437929 : return CCGCmode;
17175 : /* Codes doable only with sign flag when comparing
17176 : against zero, but we miss jump instruction for it
17177 : so we need to use relational tests against overflow
17178 : that thus needs to be zero. */
17179 933285 : case GT: /* ZF=0 & SF=OF */
17180 933285 : case LE: /* ZF=1 | SF<>OF */
17181 933285 : if (op1 == const0_rtx)
17182 : return CCNOmode;
17183 : else
17184 621438 : return CCGCmode;
17185 : default:
17186 : /* CCmode should be used in all other cases. */
17187 : return CCmode;
17188 : }
17189 : }
17190 :
17191 : /* Return TRUE or FALSE depending on whether the ptest instruction
17192 : INSN has source and destination with suitable matching CC modes. */
17193 :
17194 : bool
17195 91408 : ix86_match_ptest_ccmode (rtx insn)
17196 : {
17197 91408 : rtx set, src;
17198 91408 : machine_mode set_mode;
17199 :
17200 91408 : set = PATTERN (insn);
17201 91408 : gcc_assert (GET_CODE (set) == SET);
17202 91408 : src = SET_SRC (set);
17203 91408 : gcc_assert (GET_CODE (src) == UNSPEC
17204 : && XINT (src, 1) == UNSPEC_PTEST);
17205 :
17206 91408 : set_mode = GET_MODE (src);
17207 91408 : if (set_mode != CCZmode
17208 : && set_mode != CCCmode
17209 : && set_mode != CCmode)
17210 : return false;
17211 91408 : return GET_MODE (SET_DEST (set)) == set_mode;
17212 : }
17213 :
17214 : /* Return the fixed registers used for condition codes. */
17215 :
17216 : static bool
17217 19449303 : ix86_fixed_condition_code_regs (unsigned int *p1, unsigned int *p2)
17218 : {
17219 19449303 : *p1 = FLAGS_REG;
17220 19449303 : *p2 = INVALID_REGNUM;
17221 19449303 : return true;
17222 : }
17223 :
17224 : /* If two condition code modes are compatible, return a condition code
17225 : mode which is compatible with both. Otherwise, return
17226 : VOIDmode. */
17227 :
17228 : static machine_mode
17229 32680 : ix86_cc_modes_compatible (machine_mode m1, machine_mode m2)
17230 : {
17231 32680 : if (m1 == m2)
17232 : return m1;
17233 :
17234 31075 : if (GET_MODE_CLASS (m1) != MODE_CC || GET_MODE_CLASS (m2) != MODE_CC)
17235 : return VOIDmode;
17236 :
17237 31075 : if ((m1 == CCGCmode && m2 == CCGOCmode)
17238 31075 : || (m1 == CCGOCmode && m2 == CCGCmode))
17239 : return CCGCmode;
17240 :
17241 31075 : if ((m1 == CCNOmode && m2 == CCGOCmode)
17242 30897 : || (m1 == CCGOCmode && m2 == CCNOmode))
17243 : return CCNOmode;
17244 :
17245 30781 : if (m1 == CCZmode
17246 16591 : && (m2 == CCGCmode || m2 == CCGOCmode || m2 == CCNOmode))
17247 : return m2;
17248 18278 : else if (m2 == CCZmode
17249 13943 : && (m1 == CCGCmode || m1 == CCGOCmode || m1 == CCNOmode))
17250 : return m1;
17251 :
17252 7636 : switch (m1)
17253 : {
17254 0 : default:
17255 0 : gcc_unreachable ();
17256 :
17257 7636 : case E_CCmode:
17258 7636 : case E_CCGCmode:
17259 7636 : case E_CCGOCmode:
17260 7636 : case E_CCNOmode:
17261 7636 : case E_CCAmode:
17262 7636 : case E_CCCmode:
17263 7636 : case E_CCOmode:
17264 7636 : case E_CCPmode:
17265 7636 : case E_CCSmode:
17266 7636 : case E_CCZmode:
17267 7636 : switch (m2)
17268 : {
17269 : default:
17270 : return VOIDmode;
17271 :
17272 : case E_CCmode:
17273 : case E_CCGCmode:
17274 : case E_CCGOCmode:
17275 : case E_CCNOmode:
17276 : case E_CCAmode:
17277 : case E_CCCmode:
17278 : case E_CCOmode:
17279 : case E_CCPmode:
17280 : case E_CCSmode:
17281 : case E_CCZmode:
17282 : return CCmode;
17283 : }
17284 :
17285 : case E_CCFPmode:
17286 : /* These are only compatible with themselves, which we already
17287 : checked above. */
17288 : return VOIDmode;
17289 : }
17290 : }
17291 :
17292 : /* Return strategy to use for floating-point. We assume that fcomi is always
17293 : preferable where available, since that is also true when looking at size
17294 : (2 bytes, vs. 3 for fnstsw+sahf and at least 5 for fnstsw+test). */
17295 :
17296 : enum ix86_fpcmp_strategy
17297 5651238 : ix86_fp_comparison_strategy (enum rtx_code)
17298 : {
17299 : /* Do fcomi/sahf based test when profitable. */
17300 :
17301 5651238 : if (TARGET_CMOVE)
17302 : return IX86_FPCMP_COMI;
17303 :
17304 0 : if (TARGET_SAHF && (TARGET_USE_SAHF || optimize_insn_for_size_p ()))
17305 0 : return IX86_FPCMP_SAHF;
17306 :
17307 : return IX86_FPCMP_ARITH;
17308 : }
17309 :
17310 : /* Convert comparison codes we use to represent FP comparison to integer
17311 : code that will result in proper branch. Return UNKNOWN if no such code
17312 : is available. */
17313 :
17314 : enum rtx_code
17315 600160 : ix86_fp_compare_code_to_integer (enum rtx_code code)
17316 : {
17317 600160 : switch (code)
17318 : {
17319 : case GT:
17320 : return GTU;
17321 19390 : case GE:
17322 19390 : return GEU;
17323 : case ORDERED:
17324 : case UNORDERED:
17325 : return code;
17326 119758 : case UNEQ:
17327 119758 : return EQ;
17328 25142 : case UNLT:
17329 25142 : return LTU;
17330 32794 : case UNLE:
17331 32794 : return LEU;
17332 114361 : case LTGT:
17333 114361 : return NE;
17334 639 : case EQ:
17335 639 : case NE:
17336 639 : if (TARGET_AVX10_2)
17337 : return code;
17338 : /* FALLTHRU. */
17339 254 : default:
17340 254 : return UNKNOWN;
17341 : }
17342 : }
17343 :
17344 : /* Zero extend possibly SImode EXP to Pmode register. */
17345 : rtx
17346 38415 : ix86_zero_extend_to_Pmode (rtx exp)
17347 : {
17348 50307 : return force_reg (Pmode, convert_to_mode (Pmode, exp, 1));
17349 : }
17350 :
17351 : /* Return true if the function is called via PLT. */
17352 :
17353 : bool
17354 1429417 : ix86_call_use_plt_p (rtx call_op)
17355 : {
17356 1429417 : if (SYMBOL_REF_LOCAL_P (call_op))
17357 : {
17358 328827 : if (SYMBOL_REF_DECL (call_op)
17359 328827 : && TREE_CODE (SYMBOL_REF_DECL (call_op)) == FUNCTION_DECL)
17360 : {
17361 : /* NB: All ifunc functions must be called via PLT. */
17362 245647 : cgraph_node *node
17363 245647 : = cgraph_node::get (SYMBOL_REF_DECL (call_op));
17364 245647 : if (node && node->ifunc_resolver)
17365 : return true;
17366 : }
17367 328807 : return false;
17368 : }
17369 : return true;
17370 : }
17371 :
17372 : /* Implement TARGET_IFUNC_REF_LOCAL_OK. If this hook returns true,
17373 : the PLT entry will be used as the function address for local IFUNC
17374 : functions. When the PIC register is needed for PLT call, indirect
17375 : call via the PLT entry will fail since the PIC register may not be
17376 : set up properly for indirect call. In this case, we should return
17377 : false. */
17378 :
17379 : static bool
17380 803812557 : ix86_ifunc_ref_local_ok (void)
17381 : {
17382 803812557 : return !flag_pic || (TARGET_64BIT && ix86_cmodel != CM_LARGE_PIC);
17383 : }
17384 :
17385 : /* Return true if the function being called was marked with attribute
17386 : "noplt" or using -fno-plt and we are compiling for non-PIC. We need
17387 : to handle the non-PIC case in the backend because there is no easy
17388 : interface for the front-end to force non-PLT calls to use the GOT.
17389 : This is currently used only with 64-bit or 32-bit GOT32X ELF targets
17390 : to call the function marked "noplt" indirectly. */
17391 :
17392 : bool
17393 6140872 : ix86_nopic_noplt_attribute_p (rtx call_op)
17394 : {
17395 5432605 : if (flag_pic || ix86_cmodel == CM_LARGE
17396 : || !(TARGET_64BIT || HAVE_AS_IX86_GOT32X)
17397 : || TARGET_MACHO || TARGET_SEH || TARGET_PECOFF
17398 11573477 : || SYMBOL_REF_LOCAL_P (call_op))
17399 : return false;
17400 :
17401 3791732 : tree symbol_decl = SYMBOL_REF_DECL (call_op);
17402 :
17403 3791732 : if (!flag_plt
17404 3791732 : || (symbol_decl != NULL_TREE
17405 3791700 : && lookup_attribute ("noplt", DECL_ATTRIBUTES (symbol_decl))))
17406 34 : return true;
17407 :
17408 : return false;
17409 : }
17410 :
17411 : /* Helper to output the jmp/call. */
17412 : static void
17413 33 : ix86_output_jmp_thunk_or_indirect (const char *thunk_name, const int regno)
17414 : {
17415 33 : if (thunk_name != NULL)
17416 : {
17417 22 : if ((REX_INT_REGNO_P (regno) || REX2_INT_REGNO_P (regno))
17418 1 : && ix86_indirect_branch_cs_prefix)
17419 1 : fprintf (asm_out_file, "\tcs\n");
17420 22 : fprintf (asm_out_file, "\tjmp\t");
17421 22 : assemble_name (asm_out_file, thunk_name);
17422 22 : putc ('\n', asm_out_file);
17423 22 : if ((ix86_harden_sls & harden_sls_indirect_jmp))
17424 2 : fputs ("\tint3\n", asm_out_file);
17425 : }
17426 : else
17427 11 : output_indirect_thunk (regno);
17428 33 : }
17429 :
17430 : /* Output indirect branch via a call and return thunk. CALL_OP is a
17431 : register which contains the branch target. XASM is the assembly
17432 : template for CALL_OP. Branch is a tail call if SIBCALL_P is true.
17433 : A normal call is converted to:
17434 :
17435 : call __x86_indirect_thunk_reg
17436 :
17437 : and a tail call is converted to:
17438 :
17439 : jmp __x86_indirect_thunk_reg
17440 : */
17441 :
17442 : static void
17443 50 : ix86_output_indirect_branch_via_reg (rtx call_op, bool sibcall_p)
17444 : {
17445 50 : char thunk_name_buf[32];
17446 50 : char *thunk_name;
17447 50 : enum indirect_thunk_prefix need_prefix
17448 50 : = indirect_thunk_need_prefix (current_output_insn);
17449 50 : int regno = REGNO (call_op);
17450 :
17451 50 : if (cfun->machine->indirect_branch_type
17452 50 : != indirect_branch_thunk_inline)
17453 : {
17454 39 : if (cfun->machine->indirect_branch_type == indirect_branch_thunk)
17455 16 : SET_HARD_REG_BIT (indirect_thunks_used, regno);
17456 :
17457 39 : indirect_thunk_name (thunk_name_buf, regno, need_prefix, false);
17458 39 : thunk_name = thunk_name_buf;
17459 : }
17460 : else
17461 : thunk_name = NULL;
17462 :
17463 50 : if (sibcall_p)
17464 27 : ix86_output_jmp_thunk_or_indirect (thunk_name, regno);
17465 : else
17466 : {
17467 23 : if (thunk_name != NULL)
17468 : {
17469 17 : if ((REX_INT_REGNO_P (regno) || REX_INT_REGNO_P (regno))
17470 1 : && ix86_indirect_branch_cs_prefix)
17471 1 : fprintf (asm_out_file, "\tcs\n");
17472 17 : fprintf (asm_out_file, "\tcall\t");
17473 17 : assemble_name (asm_out_file, thunk_name);
17474 17 : putc ('\n', asm_out_file);
17475 17 : return;
17476 : }
17477 :
17478 6 : char indirectlabel1[32];
17479 6 : char indirectlabel2[32];
17480 :
17481 6 : ASM_GENERATE_INTERNAL_LABEL (indirectlabel1,
17482 : INDIRECT_LABEL,
17483 : indirectlabelno++);
17484 6 : ASM_GENERATE_INTERNAL_LABEL (indirectlabel2,
17485 : INDIRECT_LABEL,
17486 : indirectlabelno++);
17487 :
17488 : /* Jump. */
17489 6 : fputs ("\tjmp\t", asm_out_file);
17490 6 : assemble_name_raw (asm_out_file, indirectlabel2);
17491 6 : fputc ('\n', asm_out_file);
17492 :
17493 6 : ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel1);
17494 :
17495 6 : ix86_output_jmp_thunk_or_indirect (thunk_name, regno);
17496 :
17497 6 : ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel2);
17498 :
17499 : /* Call. */
17500 6 : fputs ("\tcall\t", asm_out_file);
17501 6 : assemble_name_raw (asm_out_file, indirectlabel1);
17502 6 : fputc ('\n', asm_out_file);
17503 : }
17504 : }
17505 :
17506 : /* Output indirect branch via a call and return thunk. CALL_OP is
17507 : the branch target. XASM is the assembly template for CALL_OP.
17508 : Branch is a tail call if SIBCALL_P is true. A normal call is
17509 : converted to:
17510 :
17511 : jmp L2
17512 : L1:
17513 : push CALL_OP
17514 : jmp __x86_indirect_thunk
17515 : L2:
17516 : call L1
17517 :
17518 : and a tail call is converted to:
17519 :
17520 : push CALL_OP
17521 : jmp __x86_indirect_thunk
17522 : */
17523 :
17524 : static void
17525 0 : ix86_output_indirect_branch_via_push (rtx call_op, const char *xasm,
17526 : bool sibcall_p)
17527 : {
17528 0 : char thunk_name_buf[32];
17529 0 : char *thunk_name;
17530 0 : char push_buf[64];
17531 0 : enum indirect_thunk_prefix need_prefix
17532 0 : = indirect_thunk_need_prefix (current_output_insn);
17533 0 : int regno = -1;
17534 :
17535 0 : if (cfun->machine->indirect_branch_type
17536 0 : != indirect_branch_thunk_inline)
17537 : {
17538 0 : if (cfun->machine->indirect_branch_type == indirect_branch_thunk)
17539 0 : indirect_thunk_needed = true;
17540 0 : indirect_thunk_name (thunk_name_buf, regno, need_prefix, false);
17541 0 : thunk_name = thunk_name_buf;
17542 : }
17543 : else
17544 : thunk_name = NULL;
17545 :
17546 0 : snprintf (push_buf, sizeof (push_buf), "push{%c}\t%s",
17547 0 : TARGET_64BIT ? 'q' : 'l', xasm);
17548 :
17549 0 : if (sibcall_p)
17550 : {
17551 0 : output_asm_insn (push_buf, &call_op);
17552 0 : ix86_output_jmp_thunk_or_indirect (thunk_name, regno);
17553 : }
17554 : else
17555 : {
17556 0 : char indirectlabel1[32];
17557 0 : char indirectlabel2[32];
17558 :
17559 0 : ASM_GENERATE_INTERNAL_LABEL (indirectlabel1,
17560 : INDIRECT_LABEL,
17561 : indirectlabelno++);
17562 0 : ASM_GENERATE_INTERNAL_LABEL (indirectlabel2,
17563 : INDIRECT_LABEL,
17564 : indirectlabelno++);
17565 :
17566 : /* Jump. */
17567 0 : fputs ("\tjmp\t", asm_out_file);
17568 0 : assemble_name_raw (asm_out_file, indirectlabel2);
17569 0 : fputc ('\n', asm_out_file);
17570 :
17571 0 : ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel1);
17572 :
17573 : /* An external function may be called via GOT, instead of PLT. */
17574 0 : if (MEM_P (call_op))
17575 : {
17576 0 : struct ix86_address parts;
17577 0 : rtx addr = XEXP (call_op, 0);
17578 0 : if (ix86_decompose_address (addr, &parts)
17579 0 : && parts.base == stack_pointer_rtx)
17580 : {
17581 : /* Since call will adjust stack by -UNITS_PER_WORD,
17582 : we must convert "disp(stack, index, scale)" to
17583 : "disp+UNITS_PER_WORD(stack, index, scale)". */
17584 0 : if (parts.index)
17585 : {
17586 0 : addr = gen_rtx_MULT (Pmode, parts.index,
17587 : GEN_INT (parts.scale));
17588 0 : addr = gen_rtx_PLUS (Pmode, stack_pointer_rtx,
17589 : addr);
17590 : }
17591 : else
17592 : addr = stack_pointer_rtx;
17593 :
17594 0 : rtx disp;
17595 0 : if (parts.disp != NULL_RTX)
17596 0 : disp = plus_constant (Pmode, parts.disp,
17597 0 : UNITS_PER_WORD);
17598 : else
17599 0 : disp = GEN_INT (UNITS_PER_WORD);
17600 :
17601 0 : addr = gen_rtx_PLUS (Pmode, addr, disp);
17602 0 : call_op = gen_rtx_MEM (GET_MODE (call_op), addr);
17603 : }
17604 : }
17605 :
17606 0 : output_asm_insn (push_buf, &call_op);
17607 :
17608 0 : ix86_output_jmp_thunk_or_indirect (thunk_name, regno);
17609 :
17610 0 : ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel2);
17611 :
17612 : /* Call. */
17613 0 : fputs ("\tcall\t", asm_out_file);
17614 0 : assemble_name_raw (asm_out_file, indirectlabel1);
17615 0 : fputc ('\n', asm_out_file);
17616 : }
17617 0 : }
17618 :
17619 : /* Output indirect branch via a call and return thunk. CALL_OP is
17620 : the branch target. XASM is the assembly template for CALL_OP.
17621 : Branch is a tail call if SIBCALL_P is true. */
17622 :
17623 : static void
17624 50 : ix86_output_indirect_branch (rtx call_op, const char *xasm,
17625 : bool sibcall_p)
17626 : {
17627 50 : if (REG_P (call_op))
17628 50 : ix86_output_indirect_branch_via_reg (call_op, sibcall_p);
17629 : else
17630 0 : ix86_output_indirect_branch_via_push (call_op, xasm, sibcall_p);
17631 50 : }
17632 :
17633 : /* Output indirect jump. CALL_OP is the jump target. */
17634 :
17635 : const char *
17636 8588 : ix86_output_indirect_jmp (rtx call_op)
17637 : {
17638 8588 : if (cfun->machine->indirect_branch_type != indirect_branch_keep)
17639 : {
17640 : /* We can't have red-zone since "call" in the indirect thunk
17641 : pushes the return address onto stack, destroying red-zone. */
17642 4 : if (ix86_red_zone_used)
17643 0 : gcc_unreachable ();
17644 :
17645 4 : ix86_output_indirect_branch (call_op, "%0", true);
17646 : }
17647 : else
17648 8584 : output_asm_insn ("%!jmp\t%A0", &call_op);
17649 8588 : return (ix86_harden_sls & harden_sls_indirect_jmp) ? "int3" : "";
17650 : }
17651 :
17652 : /* Output return instrumentation for current function if needed. */
17653 :
17654 : static void
17655 1751774 : output_return_instrumentation (void)
17656 : {
17657 1751774 : if (ix86_instrument_return != instrument_return_none
17658 6 : && flag_fentry
17659 1751780 : && !DECL_NO_INSTRUMENT_FUNCTION_ENTRY_EXIT (cfun->decl))
17660 : {
17661 5 : if (ix86_flag_record_return)
17662 5 : fprintf (asm_out_file, "1:\n");
17663 5 : switch (ix86_instrument_return)
17664 : {
17665 2 : case instrument_return_call:
17666 2 : fprintf (asm_out_file, "\tcall\t__return__\n");
17667 2 : break;
17668 3 : case instrument_return_nop5:
17669 : /* 5 byte nop: nopl 0(%[re]ax,%[re]ax,1) */
17670 3 : fprintf (asm_out_file, ASM_BYTE "0x0f, 0x1f, 0x44, 0x00, 0x00\n");
17671 3 : break;
17672 : case instrument_return_none:
17673 : break;
17674 : }
17675 :
17676 5 : if (ix86_flag_record_return)
17677 : {
17678 5 : fprintf (asm_out_file, "\t.section __return_loc, \"a\",@progbits\n");
17679 5 : fprintf (asm_out_file, "\t.%s 1b\n", TARGET_64BIT ? "quad" : "long");
17680 5 : fprintf (asm_out_file, "\t.previous\n");
17681 : }
17682 : }
17683 1751774 : }
17684 :
17685 : /* Output function return. CALL_OP is the jump target. Add a REP
17686 : prefix to RET if LONG_P is true and function return is kept. */
17687 :
17688 : const char *
17689 1618803 : ix86_output_function_return (bool long_p)
17690 : {
17691 1618803 : output_return_instrumentation ();
17692 :
17693 1618803 : if (cfun->machine->function_return_type != indirect_branch_keep)
17694 : {
17695 17 : char thunk_name[32];
17696 17 : enum indirect_thunk_prefix need_prefix
17697 17 : = indirect_thunk_need_prefix (current_output_insn);
17698 :
17699 17 : if (cfun->machine->function_return_type
17700 17 : != indirect_branch_thunk_inline)
17701 : {
17702 12 : bool need_thunk = (cfun->machine->function_return_type
17703 : == indirect_branch_thunk);
17704 12 : indirect_thunk_name (thunk_name, INVALID_REGNUM, need_prefix,
17705 : true);
17706 12 : indirect_return_needed |= need_thunk;
17707 12 : fprintf (asm_out_file, "\tjmp\t");
17708 12 : assemble_name (asm_out_file, thunk_name);
17709 12 : putc ('\n', asm_out_file);
17710 : }
17711 : else
17712 5 : output_indirect_thunk (INVALID_REGNUM);
17713 :
17714 17 : return "";
17715 : }
17716 :
17717 3237084 : output_asm_insn (long_p ? "rep%; ret" : "ret", nullptr);
17718 1618786 : return (ix86_harden_sls & harden_sls_return) ? "int3" : "";
17719 : }
17720 :
17721 : /* Output indirect function return. RET_OP is the function return
17722 : target. */
17723 :
17724 : const char *
17725 17 : ix86_output_indirect_function_return (rtx ret_op)
17726 : {
17727 17 : if (cfun->machine->function_return_type != indirect_branch_keep)
17728 : {
17729 0 : char thunk_name[32];
17730 0 : enum indirect_thunk_prefix need_prefix
17731 0 : = indirect_thunk_need_prefix (current_output_insn);
17732 0 : unsigned int regno = REGNO (ret_op);
17733 0 : gcc_assert (regno == CX_REG);
17734 :
17735 0 : if (cfun->machine->function_return_type
17736 0 : != indirect_branch_thunk_inline)
17737 : {
17738 0 : bool need_thunk = (cfun->machine->function_return_type
17739 : == indirect_branch_thunk);
17740 0 : indirect_thunk_name (thunk_name, regno, need_prefix, true);
17741 :
17742 0 : if (need_thunk)
17743 : {
17744 0 : indirect_return_via_cx = true;
17745 0 : SET_HARD_REG_BIT (indirect_thunks_used, CX_REG);
17746 : }
17747 0 : fprintf (asm_out_file, "\tjmp\t");
17748 0 : assemble_name (asm_out_file, thunk_name);
17749 0 : putc ('\n', asm_out_file);
17750 : }
17751 : else
17752 0 : output_indirect_thunk (regno);
17753 : }
17754 : else
17755 : {
17756 17 : output_asm_insn ("%!jmp\t%A0", &ret_op);
17757 17 : if (ix86_harden_sls & harden_sls_indirect_jmp)
17758 1 : fputs ("\tint3\n", asm_out_file);
17759 : }
17760 17 : return "";
17761 : }
17762 :
17763 : /* Output the assembly for a call instruction. */
17764 :
17765 : const char *
17766 6327061 : ix86_output_call_insn (rtx_insn *insn, rtx call_op)
17767 : {
17768 6327061 : bool direct_p = constant_call_address_operand (call_op, VOIDmode);
17769 6327061 : bool output_indirect_p
17770 : = (!TARGET_SEH
17771 6327061 : && cfun->machine->indirect_branch_type != indirect_branch_keep);
17772 6327061 : bool seh_nop_p = false;
17773 6327061 : const char *xasm;
17774 :
17775 6327061 : if (SIBLING_CALL_P (insn))
17776 : {
17777 132971 : output_return_instrumentation ();
17778 132971 : if (direct_p)
17779 : {
17780 123253 : if (ix86_nopic_noplt_attribute_p (call_op))
17781 : {
17782 4 : direct_p = false;
17783 4 : if (TARGET_64BIT)
17784 : {
17785 4 : if (output_indirect_p)
17786 : xasm = "{%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}";
17787 : else
17788 4 : xasm = "%!jmp\t{*%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}";
17789 : }
17790 : else
17791 : {
17792 0 : if (output_indirect_p)
17793 : xasm = "{%p0@GOT|[DWORD PTR %p0@GOT]}";
17794 : else
17795 0 : xasm = "%!jmp\t{*%p0@GOT|[DWORD PTR %p0@GOT]}";
17796 : }
17797 : }
17798 : else
17799 : xasm = "%!jmp\t%P0";
17800 : }
17801 : /* SEH epilogue detection requires the indirect branch case
17802 : to include REX.W. */
17803 9718 : else if (TARGET_SEH)
17804 : xasm = "%!rex.W jmp\t%A0";
17805 : else
17806 : {
17807 9718 : if (output_indirect_p)
17808 : xasm = "%0";
17809 : else
17810 9695 : xasm = "%!jmp\t%A0";
17811 : }
17812 :
17813 132971 : if (output_indirect_p && !direct_p)
17814 23 : ix86_output_indirect_branch (call_op, xasm, true);
17815 : else
17816 : {
17817 132948 : output_asm_insn (xasm, &call_op);
17818 132948 : if (!direct_p
17819 9699 : && (ix86_harden_sls & harden_sls_indirect_jmp))
17820 : return "int3";
17821 : }
17822 132970 : return "";
17823 : }
17824 :
17825 : /* SEH unwinding can require an extra nop to be emitted in several
17826 : circumstances. Determine if we have one of those. */
17827 6194090 : if (TARGET_SEH)
17828 : {
17829 : rtx_insn *i;
17830 :
17831 : for (i = NEXT_INSN (insn); i ; i = NEXT_INSN (i))
17832 : {
17833 : /* Prevent a catch region from being adjacent to a jump that would
17834 : be interpreted as an epilogue sequence by the unwinder. */
17835 : if (JUMP_P(i) && CROSSING_JUMP_P (i))
17836 : {
17837 : seh_nop_p = true;
17838 : break;
17839 : }
17840 :
17841 : /* If we get to another real insn, we don't need the nop. */
17842 : if (INSN_P (i))
17843 : break;
17844 :
17845 : /* If we get to the epilogue note, prevent a catch region from
17846 : being adjacent to the standard epilogue sequence. Note that,
17847 : if non-call exceptions are enabled, we already did it during
17848 : epilogue expansion, or else, if the insn can throw internally,
17849 : we already did it during the reorg pass. */
17850 : if (NOTE_P (i) && NOTE_KIND (i) == NOTE_INSN_EPILOGUE_BEG
17851 : && !flag_non_call_exceptions
17852 : && !can_throw_internal (insn))
17853 : {
17854 : seh_nop_p = true;
17855 : break;
17856 : }
17857 : }
17858 :
17859 : /* If we didn't find a real insn following the call, prevent the
17860 : unwinder from looking into the next function. */
17861 : if (i == NULL)
17862 : seh_nop_p = true;
17863 : }
17864 :
17865 6194090 : if (direct_p)
17866 : {
17867 6016597 : if (ix86_nopic_noplt_attribute_p (call_op))
17868 : {
17869 6 : direct_p = false;
17870 6 : if (TARGET_64BIT)
17871 : {
17872 6 : if (output_indirect_p)
17873 : xasm = "{%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}";
17874 : else
17875 6 : xasm = "%!call\t{*%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}";
17876 : }
17877 : else
17878 : {
17879 0 : if (output_indirect_p)
17880 : xasm = "{%p0@GOT|[DWORD PTR %p0@GOT]}";
17881 : else
17882 0 : xasm = "%!call\t{*%p0@GOT|[DWORD PTR %p0@GOT]}";
17883 : }
17884 : }
17885 : else
17886 : xasm = "%!call\t%P0";
17887 : }
17888 : else
17889 : {
17890 177493 : if (output_indirect_p)
17891 : xasm = "%0";
17892 : else
17893 177470 : xasm = "%!call\t%A0";
17894 : }
17895 :
17896 6194090 : if (output_indirect_p && !direct_p)
17897 23 : ix86_output_indirect_branch (call_op, xasm, false);
17898 : else
17899 6194067 : output_asm_insn (xasm, &call_op);
17900 :
17901 : if (seh_nop_p)
17902 : return "nop";
17903 :
17904 : return "";
17905 : }
17906 :
17907 : /* Return a MEM corresponding to a stack slot with mode MODE.
17908 : Allocate a new slot if necessary.
17909 :
17910 : The RTL for a function can have several slots available: N is
17911 : which slot to use. */
17912 :
17913 : rtx
17914 22476 : assign_386_stack_local (machine_mode mode, enum ix86_stack_slot n)
17915 : {
17916 22476 : struct stack_local_entry *s;
17917 :
17918 22476 : gcc_assert (n < MAX_386_STACK_LOCALS);
17919 :
17920 33892 : for (s = ix86_stack_locals; s; s = s->next)
17921 31285 : if (s->mode == mode && s->n == n)
17922 19869 : return validize_mem (copy_rtx (s->rtl));
17923 :
17924 2607 : int align = 0;
17925 : /* For DImode with SLOT_FLOATxFDI_387 use 32-bit
17926 : alignment with -m32 -mpreferred-stack-boundary=2. */
17927 2607 : if (mode == DImode
17928 329 : && !TARGET_64BIT
17929 329 : && n == SLOT_FLOATxFDI_387
17930 2936 : && ix86_preferred_stack_boundary < GET_MODE_ALIGNMENT (DImode))
17931 : align = 32;
17932 2607 : s = ggc_alloc<stack_local_entry> ();
17933 2607 : s->n = n;
17934 2607 : s->mode = mode;
17935 5214 : s->rtl = assign_stack_local (mode, GET_MODE_SIZE (mode), align);
17936 :
17937 2607 : s->next = ix86_stack_locals;
17938 2607 : ix86_stack_locals = s;
17939 2607 : return validize_mem (copy_rtx (s->rtl));
17940 : }
17941 :
17942 : static void
17943 1520556 : ix86_instantiate_decls (void)
17944 : {
17945 1520556 : struct stack_local_entry *s;
17946 :
17947 1520556 : for (s = ix86_stack_locals; s; s = s->next)
17948 0 : if (s->rtl != NULL_RTX)
17949 0 : instantiate_decl_rtl (s->rtl);
17950 1520556 : }
17951 :
17952 : /* Check whether x86 address PARTS is a pc-relative address. */
17953 :
17954 : bool
17955 28828598 : ix86_rip_relative_addr_p (struct ix86_address *parts)
17956 : {
17957 28828598 : rtx base, index, disp;
17958 :
17959 28828598 : base = parts->base;
17960 28828598 : index = parts->index;
17961 28828598 : disp = parts->disp;
17962 :
17963 28828598 : if (disp && !base && !index)
17964 : {
17965 26994907 : if (TARGET_64BIT)
17966 : {
17967 25341455 : rtx symbol = disp;
17968 :
17969 25341455 : if (GET_CODE (disp) == CONST)
17970 7168206 : symbol = XEXP (disp, 0);
17971 25341455 : if (GET_CODE (symbol) == PLUS
17972 6623830 : && CONST_INT_P (XEXP (symbol, 1)))
17973 6623830 : symbol = XEXP (symbol, 0);
17974 :
17975 25341455 : if (LABEL_REF_P (symbol)
17976 25334094 : || (SYMBOL_REF_P (symbol)
17977 24039035 : && SYMBOL_REF_TLS_MODEL (symbol) == 0)
17978 26636514 : || (GET_CODE (symbol) == UNSPEC
17979 563119 : && (XINT (symbol, 1) == UNSPEC_GOTPCREL
17980 : || XINT (symbol, 1) == UNSPEC_PCREL
17981 : || XINT (symbol, 1) == UNSPEC_GOTNTPOFF)))
17982 24582166 : return true;
17983 : }
17984 : }
17985 : return false;
17986 : }
17987 :
17988 : /* Calculate the length of the memory address in the instruction encoding.
17989 : Includes addr32 prefix, does not include the one-byte modrm, opcode,
17990 : or other prefixes. We never generate addr32 prefix for LEA insn. */
17991 :
17992 : int
17993 283158989 : memory_address_length (rtx addr, bool lea)
17994 : {
17995 283158989 : struct ix86_address parts;
17996 283158989 : rtx base, index, disp;
17997 283158989 : int len;
17998 283158989 : int ok;
17999 :
18000 283158989 : if (GET_CODE (addr) == PRE_DEC
18001 274468710 : || GET_CODE (addr) == POST_INC
18002 269884809 : || GET_CODE (addr) == PRE_MODIFY
18003 269884809 : || GET_CODE (addr) == POST_MODIFY)
18004 : return 0;
18005 :
18006 269884809 : ok = ix86_decompose_address (addr, &parts);
18007 269884809 : gcc_assert (ok);
18008 :
18009 269884809 : len = (parts.seg == ADDR_SPACE_GENERIC) ? 0 : 1;
18010 :
18011 : /* If this is not LEA instruction, add the length of addr32 prefix. */
18012 231039225 : if (TARGET_64BIT && !lea
18013 472255391 : && (SImode_address_operand (addr, VOIDmode)
18014 202370283 : || (parts.base && GET_MODE (parts.base) == SImode)
18015 202359624 : || (parts.index && GET_MODE (parts.index) == SImode)))
18016 10958 : len++;
18017 :
18018 269884809 : base = parts.base;
18019 269884809 : index = parts.index;
18020 269884809 : disp = parts.disp;
18021 :
18022 269884809 : if (base && SUBREG_P (base))
18023 2 : base = SUBREG_REG (base);
18024 269884809 : if (index && SUBREG_P (index))
18025 0 : index = SUBREG_REG (index);
18026 :
18027 269884809 : gcc_assert (base == NULL_RTX || REG_P (base));
18028 269884809 : gcc_assert (index == NULL_RTX || REG_P (index));
18029 :
18030 : /* Rule of thumb:
18031 : - esp as the base always wants an index,
18032 : - ebp as the base always wants a displacement,
18033 : - r12 as the base always wants an index,
18034 : - r13 as the base always wants a displacement. */
18035 :
18036 : /* Register Indirect. */
18037 269884809 : if (base && !index && !disp)
18038 : {
18039 : /* esp (for its index) and ebp (for its displacement) need
18040 : the two-byte modrm form. Similarly for r12 and r13 in 64-bit
18041 : code. */
18042 17364732 : if (base == arg_pointer_rtx
18043 17364732 : || base == frame_pointer_rtx
18044 17364732 : || REGNO (base) == SP_REG
18045 10436614 : || REGNO (base) == BP_REG
18046 10436614 : || REGNO (base) == R12_REG
18047 27331855 : || REGNO (base) == R13_REG)
18048 7397609 : len++;
18049 : }
18050 :
18051 : /* Direct Addressing. In 64-bit mode mod 00 r/m 5
18052 : is not disp32, but disp32(%rip), so for disp32
18053 : SIB byte is needed, unless print_operand_address
18054 : optimizes it into disp32(%rip) or (%rip) is implied
18055 : by UNSPEC. */
18056 252520077 : else if (disp && !base && !index)
18057 : {
18058 26012124 : len += 4;
18059 26012124 : if (!ix86_rip_relative_addr_p (&parts))
18060 1839976 : len++;
18061 : }
18062 : else
18063 : {
18064 : /* Find the length of the displacement constant. */
18065 226507953 : if (disp)
18066 : {
18067 222479335 : if (base && satisfies_constraint_K (disp))
18068 130149122 : len += 1;
18069 : else
18070 92330213 : len += 4;
18071 : }
18072 : /* ebp always wants a displacement. Similarly r13. */
18073 4028618 : else if (base && (REGNO (base) == BP_REG || REGNO (base) == R13_REG))
18074 8604 : len++;
18075 :
18076 : /* An index requires the two-byte modrm form.... */
18077 226507953 : if (index
18078 : /* ...like esp (or r12), which always wants an index. */
18079 216168070 : || base == arg_pointer_rtx
18080 216168070 : || base == frame_pointer_rtx
18081 442676023 : || (base && (REGNO (base) == SP_REG || REGNO (base) == R12_REG)))
18082 162831645 : len++;
18083 : }
18084 :
18085 : return len;
18086 : }
18087 :
18088 : /* Compute default value for "length_immediate" attribute. When SHORTFORM
18089 : is set, expect that insn have 8bit immediate alternative. */
18090 : int
18091 328068575 : ix86_attr_length_immediate_default (rtx_insn *insn, bool shortform)
18092 : {
18093 328068575 : int len = 0;
18094 328068575 : int i;
18095 328068575 : extract_insn_cached (insn);
18096 1023757138 : for (i = recog_data.n_operands - 1; i >= 0; --i)
18097 695688563 : if (CONSTANT_P (recog_data.operand[i]))
18098 : {
18099 141733199 : enum attr_mode mode = get_attr_mode (insn);
18100 :
18101 141733199 : gcc_assert (!len);
18102 141733199 : if (shortform && CONST_INT_P (recog_data.operand[i]))
18103 : {
18104 38886029 : HOST_WIDE_INT ival = INTVAL (recog_data.operand[i]);
18105 38886029 : switch (mode)
18106 : {
18107 1461144 : case MODE_QI:
18108 1461144 : len = 1;
18109 1461144 : continue;
18110 449312 : case MODE_HI:
18111 449312 : ival = trunc_int_for_mode (ival, HImode);
18112 449312 : break;
18113 16260604 : case MODE_SI:
18114 16260604 : ival = trunc_int_for_mode (ival, SImode);
18115 16260604 : break;
18116 : default:
18117 : break;
18118 : }
18119 37424885 : if (IN_RANGE (ival, -128, 127))
18120 : {
18121 33234032 : len = 1;
18122 33234032 : continue;
18123 : }
18124 : }
18125 107038023 : switch (mode)
18126 : {
18127 : case MODE_QI:
18128 : len = 1;
18129 : break;
18130 : case MODE_HI:
18131 695688563 : len = 2;
18132 : break;
18133 : case MODE_SI:
18134 101409773 : len = 4;
18135 : break;
18136 : /* Immediates for DImode instructions are encoded
18137 : as 32bit sign extended values. */
18138 : case MODE_DI:
18139 101409773 : len = 4;
18140 : break;
18141 0 : default:
18142 0 : fatal_insn ("unknown insn mode", insn);
18143 : }
18144 : }
18145 328068575 : return len;
18146 : }
18147 :
18148 : /* Compute default value for "length_address" attribute. */
18149 : int
18150 461590338 : ix86_attr_length_address_default (rtx_insn *insn)
18151 : {
18152 461590338 : int i;
18153 :
18154 461590338 : if (get_attr_type (insn) == TYPE_LEA)
18155 : {
18156 31432403 : rtx set = PATTERN (insn), addr;
18157 :
18158 31432403 : if (GET_CODE (set) == PARALLEL)
18159 87040 : set = XVECEXP (set, 0, 0);
18160 :
18161 31432403 : gcc_assert (GET_CODE (set) == SET);
18162 :
18163 31432403 : addr = SET_SRC (set);
18164 :
18165 31432403 : return memory_address_length (addr, true);
18166 : }
18167 :
18168 430157935 : extract_insn_cached (insn);
18169 989188582 : for (i = recog_data.n_operands - 1; i >= 0; --i)
18170 : {
18171 810471485 : rtx op = recog_data.operand[i];
18172 810471485 : if (MEM_P (op))
18173 : {
18174 251723665 : constrain_operands_cached (insn, reload_completed);
18175 251723665 : if (which_alternative != -1)
18176 : {
18177 251723665 : const char *constraints = recog_data.constraints[i];
18178 251723665 : int alt = which_alternative;
18179 :
18180 399788910 : while (*constraints == '=' || *constraints == '+')
18181 148065245 : constraints++;
18182 1144542635 : while (alt-- > 0)
18183 2186599341 : while (*constraints++ != ',')
18184 : ;
18185 : /* Skip ignored operands. */
18186 251723665 : if (*constraints == 'X')
18187 282827 : continue;
18188 : }
18189 :
18190 251440838 : int len = memory_address_length (XEXP (op, 0), false);
18191 :
18192 : /* Account for segment prefix for non-default addr spaces. */
18193 264402624 : if (!ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (op)))
18194 784144 : len++;
18195 :
18196 : return len;
18197 : }
18198 : }
18199 : return 0;
18200 : }
18201 :
18202 : /* Compute default value for "length_vex" attribute. It includes
18203 : 2 or 3 byte VEX prefix and 1 opcode byte. */
18204 :
18205 : int
18206 5078561 : ix86_attr_length_vex_default (rtx_insn *insn, bool has_0f_opcode,
18207 : bool has_vex_w)
18208 : {
18209 5078561 : int i, reg_only = 2 + 1;
18210 5078561 : bool has_mem = false;
18211 :
18212 : /* Only 0f opcode can use 2 byte VEX prefix and VEX W bit uses 3
18213 : byte VEX prefix. */
18214 5078561 : if (!has_0f_opcode || has_vex_w)
18215 : return 3 + 1;
18216 :
18217 : /* We can always use 2 byte VEX prefix in 32bit. */
18218 4634662 : if (!TARGET_64BIT)
18219 : return 2 + 1;
18220 :
18221 3567683 : extract_insn_cached (insn);
18222 :
18223 11215827 : for (i = recog_data.n_operands - 1; i >= 0; --i)
18224 7963393 : if (REG_P (recog_data.operand[i]))
18225 : {
18226 : /* REX.W bit uses 3 byte VEX prefix.
18227 : REX2 with vex use extended EVEX prefix length is 4-byte. */
18228 5246635 : if (GET_MODE (recog_data.operand[i]) == DImode
18229 5246635 : && GENERAL_REG_P (recog_data.operand[i]))
18230 : return 3 + 1;
18231 :
18232 : /* REX.B bit requires 3-byte VEX. Right here we don't know which
18233 : operand will be encoded using VEX.B, so be conservative.
18234 : REX2 with vex use extended EVEX prefix length is 4-byte. */
18235 5234206 : if (REX_INT_REGNO_P (recog_data.operand[i])
18236 5234206 : || REX2_INT_REGNO_P (recog_data.operand[i])
18237 5234206 : || REX_SSE_REGNO_P (recog_data.operand[i]))
18238 0 : reg_only = 3 + 1;
18239 : }
18240 2716758 : else if (MEM_P (recog_data.operand[i]))
18241 : {
18242 : /* REX2.X or REX2.B bits use 3 byte VEX prefix. */
18243 2045434 : if (x86_extended_rex2reg_mentioned_p (recog_data.operand[i]))
18244 : return 4;
18245 :
18246 : /* REX.X or REX.B bits use 3 byte VEX prefix. */
18247 2045039 : if (x86_extended_reg_mentioned_p (recog_data.operand[i]))
18248 : return 3 + 1;
18249 :
18250 : has_mem = true;
18251 : }
18252 :
18253 3252434 : return has_mem ? 2 + 1 : reg_only;
18254 : }
18255 :
18256 :
18257 : static bool
18258 : ix86_class_likely_spilled_p (reg_class_t);
18259 :
18260 : /* Returns true if lhs of insn is HW function argument register and set up
18261 : is_spilled to true if it is likely spilled HW register. */
18262 : static bool
18263 1154 : insn_is_function_arg (rtx insn, bool* is_spilled)
18264 : {
18265 1154 : rtx dst;
18266 :
18267 1154 : if (!NONDEBUG_INSN_P (insn))
18268 : return false;
18269 : /* Call instructions are not movable, ignore it. */
18270 1154 : if (CALL_P (insn))
18271 : return false;
18272 1080 : insn = PATTERN (insn);
18273 1080 : if (GET_CODE (insn) == PARALLEL)
18274 73 : insn = XVECEXP (insn, 0, 0);
18275 1080 : if (GET_CODE (insn) != SET)
18276 : return false;
18277 1080 : dst = SET_DEST (insn);
18278 984 : if (REG_P (dst) && HARD_REGISTER_P (dst)
18279 1957 : && ix86_function_arg_regno_p (REGNO (dst)))
18280 : {
18281 : /* Is it likely spilled HW register? */
18282 877 : if (!TEST_HARD_REG_BIT (fixed_reg_set, REGNO (dst))
18283 877 : && ix86_class_likely_spilled_p (REGNO_REG_CLASS (REGNO (dst))))
18284 833 : *is_spilled = true;
18285 : return true;
18286 : }
18287 : return false;
18288 : }
18289 :
18290 : /* Add output dependencies for chain of function adjacent arguments if only
18291 : there is a move to likely spilled HW register. Return first argument
18292 : if at least one dependence was added or NULL otherwise. */
18293 : static rtx_insn *
18294 419 : add_parameter_dependencies (rtx_insn *call, rtx_insn *head)
18295 : {
18296 419 : rtx_insn *insn;
18297 419 : rtx_insn *last = call;
18298 419 : rtx_insn *first_arg = NULL;
18299 419 : bool is_spilled = false;
18300 :
18301 419 : head = PREV_INSN (head);
18302 :
18303 : /* Find nearest to call argument passing instruction. */
18304 419 : while (true)
18305 : {
18306 419 : last = PREV_INSN (last);
18307 419 : if (last == head)
18308 : return NULL;
18309 419 : if (!NONDEBUG_INSN_P (last))
18310 0 : continue;
18311 419 : if (insn_is_function_arg (last, &is_spilled))
18312 : break;
18313 : return NULL;
18314 : }
18315 :
18316 : first_arg = last;
18317 1058 : while (true)
18318 : {
18319 1058 : insn = PREV_INSN (last);
18320 1058 : if (!INSN_P (insn))
18321 : break;
18322 960 : if (insn == head)
18323 : break;
18324 916 : if (!NONDEBUG_INSN_P (insn))
18325 : {
18326 181 : last = insn;
18327 181 : continue;
18328 : }
18329 735 : if (insn_is_function_arg (insn, &is_spilled))
18330 : {
18331 : /* Add output dependence between two function arguments if chain
18332 : of output arguments contains likely spilled HW registers. */
18333 466 : if (is_spilled)
18334 466 : add_dependence (first_arg, insn, REG_DEP_OUTPUT);
18335 647 : first_arg = last = insn;
18336 : }
18337 : else
18338 : break;
18339 : }
18340 411 : if (!is_spilled)
18341 0 : return NULL;
18342 : return first_arg;
18343 : }
18344 :
18345 : /* Add output or anti dependency from insn to first_arg to restrict its code
18346 : motion. */
18347 : static void
18348 2336 : avoid_func_arg_motion (rtx_insn *first_arg, rtx_insn *insn)
18349 : {
18350 2336 : rtx set;
18351 2336 : rtx tmp;
18352 :
18353 2336 : set = single_set (insn);
18354 2336 : if (!set)
18355 : return;
18356 1452 : tmp = SET_DEST (set);
18357 1452 : if (REG_P (tmp))
18358 : {
18359 : /* Add output dependency to the first function argument. */
18360 1257 : add_dependence (first_arg, insn, REG_DEP_OUTPUT);
18361 1257 : return;
18362 : }
18363 : /* Add anti dependency. */
18364 195 : add_dependence (first_arg, insn, REG_DEP_ANTI);
18365 : }
18366 :
18367 : /* Avoid cross block motion of function argument through adding dependency
18368 : from the first non-jump instruction in bb. */
18369 : static void
18370 68 : add_dependee_for_func_arg (rtx_insn *arg, basic_block bb)
18371 : {
18372 68 : rtx_insn *insn = BB_END (bb);
18373 :
18374 134 : while (insn)
18375 : {
18376 134 : if (NONDEBUG_INSN_P (insn) && NONJUMP_INSN_P (insn))
18377 : {
18378 67 : rtx set = single_set (insn);
18379 67 : if (set)
18380 : {
18381 67 : avoid_func_arg_motion (arg, insn);
18382 67 : return;
18383 : }
18384 : }
18385 67 : if (insn == BB_HEAD (bb))
18386 : return;
18387 66 : insn = PREV_INSN (insn);
18388 : }
18389 : }
18390 :
18391 : /* Hook for pre-reload schedule - avoid motion of function arguments
18392 : passed in likely spilled HW registers. */
18393 : static void
18394 10616742 : ix86_dependencies_evaluation_hook (rtx_insn *head, rtx_insn *tail)
18395 : {
18396 10616742 : rtx_insn *insn;
18397 10616742 : rtx_insn *first_arg = NULL;
18398 10616742 : if (reload_completed)
18399 : return;
18400 2276 : while (head != tail && DEBUG_INSN_P (head))
18401 786 : head = NEXT_INSN (head);
18402 10860 : for (insn = tail; insn != head; insn = PREV_INSN (insn))
18403 9509 : if (INSN_P (insn) && CALL_P (insn))
18404 : {
18405 419 : first_arg = add_parameter_dependencies (insn, head);
18406 419 : if (first_arg)
18407 : {
18408 : /* Add dependee for first argument to predecessors if only
18409 : region contains more than one block. */
18410 411 : basic_block bb = BLOCK_FOR_INSN (insn);
18411 411 : int rgn = CONTAINING_RGN (bb->index);
18412 411 : int nr_blks = RGN_NR_BLOCKS (rgn);
18413 : /* Skip trivial regions and region head blocks that can have
18414 : predecessors outside of region. */
18415 411 : if (nr_blks > 1 && BLOCK_TO_BB (bb->index) != 0)
18416 : {
18417 67 : edge e;
18418 67 : edge_iterator ei;
18419 :
18420 : /* Regions are SCCs with the exception of selective
18421 : scheduling with pipelining of outer blocks enabled.
18422 : So also check that immediate predecessors of a non-head
18423 : block are in the same region. */
18424 137 : FOR_EACH_EDGE (e, ei, bb->preds)
18425 : {
18426 : /* Avoid creating of loop-carried dependencies through
18427 : using topological ordering in the region. */
18428 70 : if (rgn == CONTAINING_RGN (e->src->index)
18429 69 : && BLOCK_TO_BB (bb->index) > BLOCK_TO_BB (e->src->index))
18430 68 : add_dependee_for_func_arg (first_arg, e->src);
18431 : }
18432 : }
18433 411 : insn = first_arg;
18434 411 : if (insn == head)
18435 : break;
18436 : }
18437 : }
18438 9090 : else if (first_arg)
18439 2269 : avoid_func_arg_motion (first_arg, insn);
18440 : }
18441 :
18442 : /* Hook for pre-reload schedule - set priority of moves from likely spilled
18443 : HW registers to maximum, to schedule them at soon as possible. These are
18444 : moves from function argument registers at the top of the function entry
18445 : and moves from function return value registers after call. */
18446 : static int
18447 115934411 : ix86_adjust_priority (rtx_insn *insn, int priority)
18448 : {
18449 115934411 : rtx set;
18450 :
18451 115934411 : if (reload_completed)
18452 : return priority;
18453 :
18454 14941 : if (!NONDEBUG_INSN_P (insn))
18455 : return priority;
18456 :
18457 12856 : set = single_set (insn);
18458 12856 : if (set)
18459 : {
18460 12227 : rtx tmp = SET_SRC (set);
18461 12227 : if (REG_P (tmp)
18462 2548 : && HARD_REGISTER_P (tmp)
18463 504 : && !TEST_HARD_REG_BIT (fixed_reg_set, REGNO (tmp))
18464 12227 : && ix86_class_likely_spilled_p (REGNO_REG_CLASS (REGNO (tmp))))
18465 453 : return current_sched_info->sched_max_insns_priority;
18466 : }
18467 :
18468 : return priority;
18469 : }
18470 :
18471 : /* Prepare for scheduling pass. */
18472 : static void
18473 987265 : ix86_sched_init_global (FILE *, int, int)
18474 : {
18475 : /* Install scheduling hooks for current CPU. Some of these hooks are used
18476 : in time-critical parts of the scheduler, so we only set them up when
18477 : they are actually used. */
18478 987265 : switch (ix86_tune)
18479 : {
18480 939476 : case PROCESSOR_CORE2:
18481 939476 : case PROCESSOR_NEHALEM:
18482 939476 : case PROCESSOR_SANDYBRIDGE:
18483 939476 : case PROCESSOR_HASWELL:
18484 939476 : case PROCESSOR_TREMONT:
18485 939476 : case PROCESSOR_ALDERLAKE:
18486 939476 : case PROCESSOR_GENERIC:
18487 : /* Do not perform multipass scheduling for pre-reload schedule
18488 : to save compile time. */
18489 939476 : if (reload_completed)
18490 : {
18491 938987 : ix86_core2i7_init_hooks ();
18492 938987 : break;
18493 : }
18494 : /* Fall through. */
18495 48278 : default:
18496 48278 : targetm.sched.dfa_post_advance_cycle = NULL;
18497 48278 : targetm.sched.first_cycle_multipass_init = NULL;
18498 48278 : targetm.sched.first_cycle_multipass_begin = NULL;
18499 48278 : targetm.sched.first_cycle_multipass_issue = NULL;
18500 48278 : targetm.sched.first_cycle_multipass_backtrack = NULL;
18501 48278 : targetm.sched.first_cycle_multipass_end = NULL;
18502 48278 : targetm.sched.first_cycle_multipass_fini = NULL;
18503 48278 : break;
18504 : }
18505 987265 : }
18506 :
18507 :
18508 : /* Implement TARGET_STATIC_RTX_ALIGNMENT. */
18509 :
18510 : static HOST_WIDE_INT
18511 734346 : ix86_static_rtx_alignment (machine_mode mode)
18512 : {
18513 734346 : if (mode == DFmode)
18514 : return 64;
18515 : if (ALIGN_MODE_128 (mode))
18516 160555 : return MAX (128, GET_MODE_ALIGNMENT (mode));
18517 487431 : return GET_MODE_ALIGNMENT (mode);
18518 : }
18519 :
18520 : /* Implement TARGET_CONSTANT_ALIGNMENT. */
18521 :
18522 : static HOST_WIDE_INT
18523 6912364 : ix86_constant_alignment (const_tree exp, HOST_WIDE_INT align)
18524 : {
18525 6912364 : if (TREE_CODE (exp) == REAL_CST || TREE_CODE (exp) == VECTOR_CST
18526 : || TREE_CODE (exp) == INTEGER_CST)
18527 : {
18528 374728 : machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
18529 374728 : HOST_WIDE_INT mode_align = ix86_static_rtx_alignment (mode);
18530 374728 : return MAX (mode_align, align);
18531 : }
18532 6393490 : else if (!optimize_size && TREE_CODE (exp) == STRING_CST
18533 9642489 : && TREE_STRING_LENGTH (exp) >= 31 && align < BITS_PER_WORD)
18534 : return BITS_PER_WORD;
18535 :
18536 : return align;
18537 : }
18538 :
18539 : /* Implement TARGET_EMPTY_RECORD_P. */
18540 :
18541 : static bool
18542 1528922169 : ix86_is_empty_record (const_tree type)
18543 : {
18544 1528922169 : if (!TARGET_64BIT)
18545 : return false;
18546 1498338509 : return default_is_empty_record (type);
18547 : }
18548 :
18549 : /* Implement TARGET_WARN_PARAMETER_PASSING_ABI. */
18550 :
18551 : static void
18552 15772807 : ix86_warn_parameter_passing_abi (cumulative_args_t cum_v, tree type)
18553 : {
18554 15772807 : CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
18555 :
18556 15772807 : if (!cum->warn_empty)
18557 : return;
18558 :
18559 13254601 : if (!TYPE_EMPTY_P (type))
18560 : return;
18561 :
18562 : /* Don't warn if the function isn't visible outside of the TU. */
18563 15423 : if (cum->decl && !TREE_PUBLIC (cum->decl))
18564 : return;
18565 :
18566 13935 : tree decl = cum->decl;
18567 13935 : if (!decl)
18568 : /* If we don't know the target, look at the current TU. */
18569 39 : decl = current_function_decl;
18570 :
18571 13935 : const_tree ctx = get_ultimate_context (decl);
18572 13935 : if (ctx == NULL_TREE
18573 27836 : || !TRANSLATION_UNIT_WARN_EMPTY_P (ctx))
18574 : return;
18575 :
18576 : /* If the actual size of the type is zero, then there is no change
18577 : in how objects of this size are passed. */
18578 72 : if (int_size_in_bytes (type) == 0)
18579 : return;
18580 :
18581 66 : warning (OPT_Wabi, "empty class %qT parameter passing ABI "
18582 : "changes in %<-fabi-version=12%> (GCC 8)", type);
18583 :
18584 : /* Only warn once. */
18585 66 : cum->warn_empty = false;
18586 : }
18587 :
18588 : /* This hook returns name of multilib ABI. */
18589 :
18590 : static const char *
18591 3487215 : ix86_get_multilib_abi_name (void)
18592 : {
18593 3487215 : if (!(TARGET_64BIT_P (ix86_isa_flags)))
18594 : return "i386";
18595 3443043 : else if (TARGET_X32_P (ix86_isa_flags))
18596 : return "x32";
18597 : else
18598 3443043 : return "x86_64";
18599 : }
18600 :
18601 : /* Compute the alignment for a variable for Intel MCU psABI. TYPE is
18602 : the data type, and ALIGN is the alignment that the object would
18603 : ordinarily have. */
18604 :
18605 : static int
18606 0 : iamcu_alignment (tree type, int align)
18607 : {
18608 0 : machine_mode mode;
18609 :
18610 0 : if (align < 32 || TYPE_USER_ALIGN (type))
18611 : return align;
18612 :
18613 : /* Intel MCU psABI specifies scalar types > 4 bytes aligned to 4
18614 : bytes. */
18615 0 : type = strip_array_types (type);
18616 0 : if (TYPE_ATOMIC (type))
18617 : return align;
18618 :
18619 0 : mode = TYPE_MODE (type);
18620 0 : switch (GET_MODE_CLASS (mode))
18621 : {
18622 : case MODE_INT:
18623 : case MODE_COMPLEX_INT:
18624 : case MODE_COMPLEX_FLOAT:
18625 : case MODE_FLOAT:
18626 : case MODE_DECIMAL_FLOAT:
18627 : return 32;
18628 : default:
18629 : return align;
18630 : }
18631 : }
18632 :
18633 : /* Compute the alignment for a static variable.
18634 : TYPE is the data type, and ALIGN is the alignment that
18635 : the object would ordinarily have. The value of this function is used
18636 : instead of that alignment to align the object. */
18637 :
18638 : int
18639 12146474 : ix86_data_alignment (tree type, unsigned int align, bool opt)
18640 : {
18641 : /* GCC 4.8 and earlier used to incorrectly assume this alignment even
18642 : for symbols from other compilation units or symbols that don't need
18643 : to bind locally. In order to preserve some ABI compatibility with
18644 : those compilers, ensure we don't decrease alignment from what we
18645 : used to assume. */
18646 :
18647 12146474 : unsigned int max_align_compat = MIN (256, MAX_OFILE_ALIGNMENT);
18648 :
18649 : /* A data structure, equal or greater than the size of a cache line
18650 : (64 bytes in the Pentium 4 and other recent Intel processors, including
18651 : processors based on Intel Core microarchitecture) should be aligned
18652 : so that its base address is a multiple of a cache line size. */
18653 :
18654 12146474 : unsigned int max_align
18655 12146474 : = MIN ((unsigned) ix86_tune_cost->prefetch_block * 8, MAX_OFILE_ALIGNMENT);
18656 :
18657 14736818 : if (max_align < BITS_PER_WORD)
18658 0 : max_align = BITS_PER_WORD;
18659 :
18660 12146474 : switch (ix86_align_data_type)
18661 : {
18662 12146474 : case ix86_align_data_type_abi: opt = false; break;
18663 12146454 : case ix86_align_data_type_compat: max_align = BITS_PER_WORD; break;
18664 : case ix86_align_data_type_cacheline: break;
18665 : }
18666 :
18667 12146474 : if (TARGET_IAMCU)
18668 0 : align = iamcu_alignment (type, align);
18669 :
18670 12146474 : if (opt
18671 5835763 : && AGGREGATE_TYPE_P (type)
18672 3741616 : && TYPE_SIZE (type)
18673 15888038 : && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST)
18674 : {
18675 6778218 : if (wi::geu_p (wi::to_wide (TYPE_SIZE (type)), max_align_compat)
18676 3741564 : && align < max_align_compat)
18677 704910 : align = max_align_compat;
18678 7418975 : if (wi::geu_p (wi::to_wide (TYPE_SIZE (type)), max_align)
18679 3741564 : && align < max_align)
18680 64153 : align = max_align;
18681 : }
18682 :
18683 : /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
18684 : to 16byte boundary. */
18685 12146474 : if (TARGET_64BIT)
18686 : {
18687 5014780 : if ((opt ? AGGREGATE_TYPE_P (type) : TREE_CODE (type) == ARRAY_TYPE)
18688 3301013 : && TYPE_SIZE (type)
18689 3300951 : && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST
18690 11008176 : && wi::geu_p (wi::to_wide (TYPE_SIZE (type)), 128)
18691 11633225 : && align < 128)
18692 625049 : return 128;
18693 : }
18694 :
18695 11521425 : if (!opt)
18696 6111585 : return align;
18697 :
18698 5409840 : if (TREE_CODE (type) == ARRAY_TYPE)
18699 : {
18700 1113461 : if (TYPE_MODE (TREE_TYPE (type)) == DFmode && align < 64)
18701 : return 64;
18702 1113461 : if (ALIGN_MODE_128 (TYPE_MODE (TREE_TYPE (type))) && align < 128)
18703 : return 128;
18704 : }
18705 4296379 : else if (TREE_CODE (type) == COMPLEX_TYPE)
18706 : {
18707 :
18708 13017 : if (TYPE_MODE (type) == DCmode && align < 64)
18709 : return 64;
18710 13017 : if ((TYPE_MODE (type) == XCmode
18711 13017 : || TYPE_MODE (type) == TCmode) && align < 128)
18712 : return 128;
18713 : }
18714 4283362 : else if (RECORD_OR_UNION_TYPE_P (type)
18715 4283362 : && TYPE_FIELDS (type))
18716 : {
18717 2200461 : if (DECL_MODE (TYPE_FIELDS (type)) == DFmode && align < 64)
18718 : return 64;
18719 2200461 : if (ALIGN_MODE_128 (DECL_MODE (TYPE_FIELDS (type))) && align < 128)
18720 : return 128;
18721 : }
18722 2082901 : else if (SCALAR_FLOAT_TYPE_P (type) || VECTOR_TYPE_P (type)
18723 : || TREE_CODE (type) == INTEGER_TYPE)
18724 : {
18725 1927026 : if (TYPE_MODE (type) == DFmode && align < 64)
18726 : return 64;
18727 1927026 : if (ALIGN_MODE_128 (TYPE_MODE (type)) && align < 128)
18728 : return 128;
18729 : }
18730 :
18731 5409725 : return align;
18732 : }
18733 :
18734 : /* Implement TARGET_LOWER_LOCAL_DECL_ALIGNMENT. */
18735 : static void
18736 33118024 : ix86_lower_local_decl_alignment (tree decl)
18737 : {
18738 33118024 : unsigned int new_align = ix86_local_alignment (decl, VOIDmode,
18739 33118024 : DECL_ALIGN (decl), true);
18740 33118024 : if (new_align < DECL_ALIGN (decl))
18741 0 : SET_DECL_ALIGN (decl, new_align);
18742 33118024 : }
18743 :
18744 : /* Compute the alignment for a local variable or a stack slot. EXP is
18745 : the data type or decl itself, MODE is the widest mode available and
18746 : ALIGN is the alignment that the object would ordinarily have. The
18747 : value of this macro is used instead of that alignment to align the
18748 : object. */
18749 :
18750 : unsigned int
18751 57659948 : ix86_local_alignment (tree exp, machine_mode mode,
18752 : unsigned int align, bool may_lower)
18753 : {
18754 57659948 : tree type, decl;
18755 :
18756 57659948 : if (exp && DECL_P (exp))
18757 : {
18758 55434952 : type = TREE_TYPE (exp);
18759 55434952 : decl = exp;
18760 : }
18761 : else
18762 : {
18763 : type = exp;
18764 : decl = NULL;
18765 : }
18766 :
18767 : /* Don't do dynamic stack realignment for long long objects with
18768 : -mpreferred-stack-boundary=2. */
18769 57659948 : if (may_lower
18770 33118024 : && !TARGET_64BIT
18771 250492 : && align == 64
18772 38970 : && ix86_preferred_stack_boundary < 64
18773 0 : && (mode == DImode || (type && TYPE_MODE (type) == DImode))
18774 0 : && (!type || (!TYPE_USER_ALIGN (type)
18775 0 : && !TYPE_ATOMIC (strip_array_types (type))))
18776 57659948 : && (!decl || !DECL_USER_ALIGN (decl)))
18777 : align = 32;
18778 :
18779 : /* If TYPE is NULL, we are allocating a stack slot for caller-save
18780 : register in MODE. We will return the largest alignment of XF
18781 : and DF. */
18782 57659948 : if (!type)
18783 : {
18784 1458163 : if (mode == XFmode && align < GET_MODE_ALIGNMENT (DFmode))
18785 1452 : align = GET_MODE_ALIGNMENT (DFmode);
18786 : return align;
18787 : }
18788 :
18789 : /* Don't increase alignment for Intel MCU psABI. */
18790 56201785 : if (TARGET_IAMCU)
18791 : return align;
18792 :
18793 : /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
18794 : to 16byte boundary. Exact wording is:
18795 :
18796 : An array uses the same alignment as its elements, except that a local or
18797 : global array variable of length at least 16 bytes or
18798 : a C99 variable-length array variable always has alignment of at least 16 bytes.
18799 :
18800 : This was added to allow use of aligned SSE instructions at arrays. This
18801 : rule is meant for static storage (where compiler cannot do the analysis
18802 : by itself). We follow it for automatic variables only when convenient.
18803 : We fully control everything in the function compiled and functions from
18804 : other unit cannot rely on the alignment.
18805 :
18806 : Exclude va_list type. It is the common case of local array where
18807 : we cannot benefit from the alignment.
18808 :
18809 : TODO: Probably one should optimize for size only when var is not escaping. */
18810 52157057 : if (TARGET_64BIT && optimize_function_for_speed_p (cfun)
18811 107757520 : && TARGET_SSE)
18812 : {
18813 51515652 : if (AGGREGATE_TYPE_P (type)
18814 14244570 : && (va_list_type_node == NULL_TREE
18815 14244570 : || (TYPE_MAIN_VARIANT (type)
18816 14244570 : != TYPE_MAIN_VARIANT (va_list_type_node)))
18817 14107639 : && TYPE_SIZE (type)
18818 14107639 : && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST
18819 53533210 : && wi::geu_p (wi::to_wide (TYPE_SIZE (type)), 128)
18820 62111737 : && align < 128)
18821 8578527 : return 128;
18822 : }
18823 47623258 : if (TREE_CODE (type) == ARRAY_TYPE)
18824 : {
18825 1392256 : if (TYPE_MODE (TREE_TYPE (type)) == DFmode && align < 64)
18826 : return 64;
18827 1392256 : if (ALIGN_MODE_128 (TYPE_MODE (TREE_TYPE (type))) && align < 128)
18828 : return 128;
18829 : }
18830 46231002 : else if (TREE_CODE (type) == COMPLEX_TYPE)
18831 : {
18832 232048 : if (TYPE_MODE (type) == DCmode && align < 64)
18833 : return 64;
18834 232048 : if ((TYPE_MODE (type) == XCmode
18835 232048 : || TYPE_MODE (type) == TCmode) && align < 128)
18836 : return 128;
18837 : }
18838 45998954 : else if (RECORD_OR_UNION_TYPE_P (type)
18839 45998954 : && TYPE_FIELDS (type))
18840 : {
18841 7579486 : if (DECL_MODE (TYPE_FIELDS (type)) == DFmode && align < 64)
18842 : return 64;
18843 7574053 : if (ALIGN_MODE_128 (DECL_MODE (TYPE_FIELDS (type))) && align < 128)
18844 : return 128;
18845 : }
18846 38419468 : else if (SCALAR_FLOAT_TYPE_P (type) || VECTOR_TYPE_P (type)
18847 : || TREE_CODE (type) == INTEGER_TYPE)
18848 : {
18849 :
18850 31372542 : if (TYPE_MODE (type) == DFmode && align < 64)
18851 : return 64;
18852 31372542 : if (ALIGN_MODE_128 (TYPE_MODE (type)) && align < 128)
18853 : return 128;
18854 : }
18855 : return align;
18856 : }
18857 :
18858 : /* Compute the minimum required alignment for dynamic stack realignment
18859 : purposes for a local variable, parameter or a stack slot. EXP is
18860 : the data type or decl itself, MODE is its mode and ALIGN is the
18861 : alignment that the object would ordinarily have. */
18862 :
18863 : unsigned int
18864 49584323 : ix86_minimum_alignment (tree exp, machine_mode mode,
18865 : unsigned int align)
18866 : {
18867 49584323 : tree type, decl;
18868 :
18869 49584323 : if (exp && DECL_P (exp))
18870 : {
18871 15719502 : type = TREE_TYPE (exp);
18872 15719502 : decl = exp;
18873 : }
18874 : else
18875 : {
18876 : type = exp;
18877 : decl = NULL;
18878 : }
18879 :
18880 49584323 : if (TARGET_64BIT || align != 64 || ix86_preferred_stack_boundary >= 64)
18881 : return align;
18882 :
18883 : /* Don't do dynamic stack realignment for long long objects with
18884 : -mpreferred-stack-boundary=2. */
18885 0 : if ((mode == DImode || (type && TYPE_MODE (type) == DImode))
18886 0 : && (!type || (!TYPE_USER_ALIGN (type)
18887 0 : && !TYPE_ATOMIC (strip_array_types (type))))
18888 0 : && (!decl || !DECL_USER_ALIGN (decl)))
18889 : {
18890 0 : gcc_checking_assert (!TARGET_STV);
18891 : return 32;
18892 : }
18893 :
18894 : return align;
18895 : }
18896 :
18897 : /* Find a location for the static chain incoming to a nested function.
18898 : This is a register, unless all free registers are used by arguments. */
18899 :
18900 : static rtx
18901 270880 : ix86_static_chain (const_tree fndecl_or_type, bool incoming_p)
18902 : {
18903 270880 : unsigned regno;
18904 :
18905 270880 : if (TARGET_64BIT)
18906 : {
18907 : /* We always use R10 in 64-bit mode. */
18908 : regno = R10_REG;
18909 : }
18910 : else
18911 : {
18912 88353 : const_tree fntype, fndecl;
18913 88353 : unsigned int ccvt;
18914 :
18915 : /* By default in 32-bit mode we use ECX to pass the static chain. */
18916 88353 : regno = CX_REG;
18917 :
18918 88353 : if (TREE_CODE (fndecl_or_type) == FUNCTION_DECL)
18919 : {
18920 78381 : fntype = TREE_TYPE (fndecl_or_type);
18921 78381 : fndecl = fndecl_or_type;
18922 : }
18923 : else
18924 : {
18925 : fntype = fndecl_or_type;
18926 : fndecl = NULL;
18927 : }
18928 :
18929 88353 : ccvt = ix86_get_callcvt (fntype);
18930 88353 : if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
18931 : {
18932 : /* Fastcall functions use ecx/edx for arguments, which leaves
18933 : us with EAX for the static chain.
18934 : Thiscall functions use ecx for arguments, which also
18935 : leaves us with EAX for the static chain. */
18936 : regno = AX_REG;
18937 : }
18938 88353 : else if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
18939 : {
18940 : /* Thiscall functions use ecx for arguments, which leaves
18941 : us with EAX and EDX for the static chain.
18942 : We are using for abi-compatibility EAX. */
18943 : regno = AX_REG;
18944 : }
18945 88353 : else if (ix86_function_regparm (fntype, fndecl) == 3)
18946 : {
18947 : /* For regparm 3, we have no free call-clobbered registers in
18948 : which to store the static chain. In order to implement this,
18949 : we have the trampoline push the static chain to the stack.
18950 : However, we can't push a value below the return address when
18951 : we call the nested function directly, so we have to use an
18952 : alternate entry point. For this we use ESI, and have the
18953 : alternate entry point push ESI, so that things appear the
18954 : same once we're executing the nested function. */
18955 0 : if (incoming_p)
18956 : {
18957 0 : if (fndecl == current_function_decl
18958 0 : && !ix86_static_chain_on_stack)
18959 : {
18960 0 : gcc_assert (!reload_completed);
18961 0 : ix86_static_chain_on_stack = true;
18962 : }
18963 0 : return gen_frame_mem (SImode,
18964 0 : plus_constant (Pmode,
18965 : arg_pointer_rtx, -8));
18966 : }
18967 : regno = SI_REG;
18968 : }
18969 : }
18970 :
18971 359246 : return gen_rtx_REG (Pmode, regno);
18972 : }
18973 :
18974 : /* Emit RTL insns to initialize the variable parts of a trampoline.
18975 : FNDECL is the decl of the target address; M_TRAMP is a MEM for
18976 : the trampoline, and CHAIN_VALUE is an RTX for the static chain
18977 : to be passed to the target function. */
18978 :
18979 : static void
18980 306 : ix86_trampoline_init (rtx m_tramp, tree fndecl, rtx chain_value)
18981 : {
18982 306 : rtx mem, fnaddr;
18983 306 : int opcode;
18984 306 : int offset = 0;
18985 306 : bool need_endbr = (flag_cf_protection & CF_BRANCH);
18986 :
18987 306 : fnaddr = XEXP (DECL_RTL (fndecl), 0);
18988 :
18989 306 : if (TARGET_64BIT)
18990 : {
18991 306 : int size;
18992 :
18993 306 : if (need_endbr)
18994 : {
18995 : /* Insert ENDBR64. */
18996 1 : mem = adjust_address (m_tramp, SImode, offset);
18997 1 : emit_move_insn (mem, gen_int_mode (0xfa1e0ff3, SImode));
18998 1 : offset += 4;
18999 : }
19000 :
19001 : /* Load the function address to r11. Try to load address using
19002 : the shorter movl instead of movabs. We may want to support
19003 : movq for kernel mode, but kernel does not use trampolines at
19004 : the moment. FNADDR is a 32bit address and may not be in
19005 : DImode when ptr_mode == SImode. Always use movl in this
19006 : case. */
19007 306 : if (ptr_mode == SImode
19008 306 : || x86_64_zext_immediate_operand (fnaddr, VOIDmode))
19009 : {
19010 274 : fnaddr = copy_addr_to_reg (fnaddr);
19011 :
19012 274 : mem = adjust_address (m_tramp, HImode, offset);
19013 274 : emit_move_insn (mem, gen_int_mode (0xbb41, HImode));
19014 :
19015 274 : mem = adjust_address (m_tramp, SImode, offset + 2);
19016 274 : emit_move_insn (mem, gen_lowpart (SImode, fnaddr));
19017 274 : offset += 6;
19018 : }
19019 : else
19020 : {
19021 32 : mem = adjust_address (m_tramp, HImode, offset);
19022 32 : emit_move_insn (mem, gen_int_mode (0xbb49, HImode));
19023 :
19024 32 : mem = adjust_address (m_tramp, DImode, offset + 2);
19025 32 : emit_move_insn (mem, fnaddr);
19026 32 : offset += 10;
19027 : }
19028 :
19029 : /* Load static chain using movabs to r10. Use the shorter movl
19030 : instead of movabs when ptr_mode == SImode. */
19031 306 : if (ptr_mode == SImode)
19032 : {
19033 : opcode = 0xba41;
19034 : size = 6;
19035 : }
19036 : else
19037 : {
19038 306 : opcode = 0xba49;
19039 306 : size = 10;
19040 : }
19041 :
19042 306 : mem = adjust_address (m_tramp, HImode, offset);
19043 306 : emit_move_insn (mem, gen_int_mode (opcode, HImode));
19044 :
19045 306 : mem = adjust_address (m_tramp, ptr_mode, offset + 2);
19046 306 : emit_move_insn (mem, chain_value);
19047 306 : offset += size;
19048 :
19049 : /* Jump to r11; the last (unused) byte is a nop, only there to
19050 : pad the write out to a single 32-bit store. */
19051 306 : mem = adjust_address (m_tramp, SImode, offset);
19052 306 : emit_move_insn (mem, gen_int_mode (0x90e3ff49, SImode));
19053 306 : offset += 4;
19054 : }
19055 : else
19056 : {
19057 0 : rtx disp, chain;
19058 :
19059 : /* Depending on the static chain location, either load a register
19060 : with a constant, or push the constant to the stack. All of the
19061 : instructions are the same size. */
19062 0 : chain = ix86_static_chain (fndecl, true);
19063 0 : if (REG_P (chain))
19064 : {
19065 0 : switch (REGNO (chain))
19066 : {
19067 : case AX_REG:
19068 : opcode = 0xb8; break;
19069 0 : case CX_REG:
19070 0 : opcode = 0xb9; break;
19071 0 : default:
19072 0 : gcc_unreachable ();
19073 : }
19074 : }
19075 : else
19076 : opcode = 0x68;
19077 :
19078 0 : if (need_endbr)
19079 : {
19080 : /* Insert ENDBR32. */
19081 0 : mem = adjust_address (m_tramp, SImode, offset);
19082 0 : emit_move_insn (mem, gen_int_mode (0xfb1e0ff3, SImode));
19083 0 : offset += 4;
19084 : }
19085 :
19086 0 : mem = adjust_address (m_tramp, QImode, offset);
19087 0 : emit_move_insn (mem, gen_int_mode (opcode, QImode));
19088 :
19089 0 : mem = adjust_address (m_tramp, SImode, offset + 1);
19090 0 : emit_move_insn (mem, chain_value);
19091 0 : offset += 5;
19092 :
19093 0 : mem = adjust_address (m_tramp, QImode, offset);
19094 0 : emit_move_insn (mem, gen_int_mode (0xe9, QImode));
19095 :
19096 0 : mem = adjust_address (m_tramp, SImode, offset + 1);
19097 :
19098 : /* Compute offset from the end of the jmp to the target function.
19099 : In the case in which the trampoline stores the static chain on
19100 : the stack, we need to skip the first insn which pushes the
19101 : (call-saved) register static chain; this push is 1 byte. */
19102 0 : offset += 5;
19103 0 : int skip = MEM_P (chain) ? 1 : 0;
19104 : /* Skip ENDBR32 at the entry of the target function. */
19105 0 : if (need_endbr
19106 0 : && !cgraph_node::get (fndecl)->only_called_directly_p ())
19107 0 : skip += 4;
19108 0 : disp = expand_binop (SImode, sub_optab, fnaddr,
19109 0 : plus_constant (Pmode, XEXP (m_tramp, 0),
19110 0 : offset - skip),
19111 : NULL_RTX, 1, OPTAB_DIRECT);
19112 0 : emit_move_insn (mem, disp);
19113 : }
19114 :
19115 306 : gcc_assert (offset <= TRAMPOLINE_SIZE);
19116 :
19117 : #ifdef HAVE_ENABLE_EXECUTE_STACK
19118 : #ifdef CHECK_EXECUTE_STACK_ENABLED
19119 : if (CHECK_EXECUTE_STACK_ENABLED)
19120 : #endif
19121 : emit_library_call (gen_rtx_SYMBOL_REF (Pmode, "__enable_execute_stack"),
19122 : LCT_NORMAL, VOIDmode, XEXP (m_tramp, 0), Pmode);
19123 : #endif
19124 306 : }
19125 :
19126 : static bool
19127 62811297 : ix86_allocate_stack_slots_for_args (void)
19128 : {
19129 : /* Naked functions should not allocate stack slots for arguments. */
19130 62811297 : return !ix86_function_naked (current_function_decl);
19131 : }
19132 :
19133 : static bool
19134 40440632 : ix86_warn_func_return (tree decl)
19135 : {
19136 : /* Naked functions are implemented entirely in assembly, including the
19137 : return sequence, so suppress warnings about this. */
19138 40440632 : return !ix86_function_naked (decl);
19139 : }
19140 :
19141 : /* Return the shift count of a vector by scalar shift builtin second argument
19142 : ARG1. */
19143 : static tree
19144 14142 : ix86_vector_shift_count (tree arg1)
19145 : {
19146 14142 : if (tree_fits_uhwi_p (arg1))
19147 : return arg1;
19148 8316 : else if (TREE_CODE (arg1) == VECTOR_CST && CHAR_BIT == 8)
19149 : {
19150 : /* The count argument is weird, passed in as various 128-bit
19151 : (or 64-bit) vectors, the low 64 bits from it are the count. */
19152 162 : unsigned char buf[16];
19153 162 : int len = native_encode_expr (arg1, buf, 16);
19154 162 : if (len == 0)
19155 162 : return NULL_TREE;
19156 162 : tree t = native_interpret_expr (uint64_type_node, buf, len);
19157 162 : if (t && tree_fits_uhwi_p (t))
19158 : return t;
19159 : }
19160 : return NULL_TREE;
19161 : }
19162 :
19163 : /* Return true if arg_mask is all ones, ELEMS is elements number of
19164 : corresponding vector. */
19165 : static bool
19166 26083 : ix86_masked_all_ones (unsigned HOST_WIDE_INT elems, tree arg_mask)
19167 : {
19168 26083 : if (TREE_CODE (arg_mask) != INTEGER_CST)
19169 : return false;
19170 :
19171 7615 : unsigned HOST_WIDE_INT mask = TREE_INT_CST_LOW (arg_mask);
19172 7615 : if (elems == HOST_BITS_PER_WIDE_INT)
19173 65 : return mask == HOST_WIDE_INT_M1U;
19174 7550 : if ((mask | (HOST_WIDE_INT_M1U << elems)) != HOST_WIDE_INT_M1U)
19175 2784 : return false;
19176 :
19177 : return true;
19178 : }
19179 :
19180 : static tree
19181 71123431 : ix86_fold_builtin (tree fndecl, int n_args,
19182 : tree *args, bool ignore ATTRIBUTE_UNUSED)
19183 : {
19184 71123431 : if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD)
19185 : {
19186 71123431 : enum ix86_builtins fn_code
19187 71123431 : = (enum ix86_builtins) DECL_MD_FUNCTION_CODE (fndecl);
19188 71123431 : enum rtx_code rcode;
19189 71123431 : bool is_vshift;
19190 71123431 : enum tree_code tcode;
19191 71123431 : bool is_scalar;
19192 71123431 : unsigned HOST_WIDE_INT mask;
19193 :
19194 71123431 : switch (fn_code)
19195 : {
19196 8992 : case IX86_BUILTIN_CPU_IS:
19197 8992 : case IX86_BUILTIN_CPU_SUPPORTS:
19198 8992 : gcc_assert (n_args == 1);
19199 8992 : return fold_builtin_cpu (fndecl, args);
19200 :
19201 26757 : case IX86_BUILTIN_NANQ:
19202 26757 : case IX86_BUILTIN_NANSQ:
19203 26757 : {
19204 26757 : tree type = TREE_TYPE (TREE_TYPE (fndecl));
19205 26757 : const char *str = c_getstr (*args);
19206 26757 : int quiet = fn_code == IX86_BUILTIN_NANQ;
19207 26757 : REAL_VALUE_TYPE real;
19208 :
19209 26757 : if (str && real_nan (&real, str, quiet, TYPE_MODE (type)))
19210 26757 : return build_real (type, real);
19211 0 : return NULL_TREE;
19212 : }
19213 :
19214 108 : case IX86_BUILTIN_INFQ:
19215 108 : case IX86_BUILTIN_HUGE_VALQ:
19216 108 : {
19217 108 : tree type = TREE_TYPE (TREE_TYPE (fndecl));
19218 108 : REAL_VALUE_TYPE inf;
19219 108 : real_inf (&inf);
19220 108 : return build_real (type, inf);
19221 : }
19222 :
19223 63461 : case IX86_BUILTIN_TZCNT16:
19224 63461 : case IX86_BUILTIN_CTZS:
19225 63461 : case IX86_BUILTIN_TZCNT32:
19226 63461 : case IX86_BUILTIN_TZCNT64:
19227 63461 : gcc_assert (n_args == 1);
19228 63461 : if (TREE_CODE (args[0]) == INTEGER_CST)
19229 : {
19230 45 : tree type = TREE_TYPE (TREE_TYPE (fndecl));
19231 45 : tree arg = args[0];
19232 45 : if (fn_code == IX86_BUILTIN_TZCNT16
19233 45 : || fn_code == IX86_BUILTIN_CTZS)
19234 3 : arg = fold_convert (short_unsigned_type_node, arg);
19235 45 : if (integer_zerop (arg))
19236 6 : return build_int_cst (type, TYPE_PRECISION (TREE_TYPE (arg)));
19237 : else
19238 39 : return fold_const_call (CFN_CTZ, type, arg);
19239 : }
19240 : break;
19241 :
19242 52822 : case IX86_BUILTIN_LZCNT16:
19243 52822 : case IX86_BUILTIN_CLZS:
19244 52822 : case IX86_BUILTIN_LZCNT32:
19245 52822 : case IX86_BUILTIN_LZCNT64:
19246 52822 : gcc_assert (n_args == 1);
19247 52822 : if (TREE_CODE (args[0]) == INTEGER_CST)
19248 : {
19249 54 : tree type = TREE_TYPE (TREE_TYPE (fndecl));
19250 54 : tree arg = args[0];
19251 54 : if (fn_code == IX86_BUILTIN_LZCNT16
19252 54 : || fn_code == IX86_BUILTIN_CLZS)
19253 18 : arg = fold_convert (short_unsigned_type_node, arg);
19254 54 : if (integer_zerop (arg))
19255 3 : return build_int_cst (type, TYPE_PRECISION (TREE_TYPE (arg)));
19256 : else
19257 51 : return fold_const_call (CFN_CLZ, type, arg);
19258 : }
19259 : break;
19260 :
19261 62199 : case IX86_BUILTIN_BEXTR32:
19262 62199 : case IX86_BUILTIN_BEXTR64:
19263 62199 : case IX86_BUILTIN_BEXTRI32:
19264 62199 : case IX86_BUILTIN_BEXTRI64:
19265 62199 : gcc_assert (n_args == 2);
19266 62199 : if (tree_fits_uhwi_p (args[1]))
19267 : {
19268 152 : unsigned HOST_WIDE_INT res = 0;
19269 152 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (args[0]));
19270 152 : unsigned int start = tree_to_uhwi (args[1]);
19271 152 : unsigned int len = (start & 0xff00) >> 8;
19272 152 : tree lhs_type = TREE_TYPE (TREE_TYPE (fndecl));
19273 152 : start &= 0xff;
19274 152 : if (start >= prec || len == 0)
19275 111 : return omit_one_operand (lhs_type, build_zero_cst (lhs_type),
19276 : args[0]);
19277 41 : else if (!tree_fits_uhwi_p (args[0]))
19278 : break;
19279 : else
19280 24 : res = tree_to_uhwi (args[0]) >> start;
19281 24 : if (len > prec)
19282 : len = prec;
19283 24 : if (len < HOST_BITS_PER_WIDE_INT)
19284 15 : res &= (HOST_WIDE_INT_1U << len) - 1;
19285 24 : return build_int_cstu (lhs_type, res);
19286 : }
19287 : break;
19288 :
19289 21360 : case IX86_BUILTIN_BZHI32:
19290 21360 : case IX86_BUILTIN_BZHI64:
19291 21360 : gcc_assert (n_args == 2);
19292 21360 : if (tree_fits_uhwi_p (args[1]))
19293 : {
19294 192 : unsigned int idx = tree_to_uhwi (args[1]) & 0xff;
19295 192 : tree lhs_type = TREE_TYPE (TREE_TYPE (fndecl));
19296 192 : if (idx >= TYPE_PRECISION (TREE_TYPE (args[0])))
19297 71123431 : return args[0];
19298 192 : if (idx == 0)
19299 52 : return omit_one_operand (lhs_type, build_zero_cst (lhs_type),
19300 : args[0]);
19301 140 : if (!tree_fits_uhwi_p (args[0]))
19302 : break;
19303 12 : unsigned HOST_WIDE_INT res = tree_to_uhwi (args[0]);
19304 12 : res &= ~(HOST_WIDE_INT_M1U << idx);
19305 12 : return build_int_cstu (lhs_type, res);
19306 : }
19307 : break;
19308 :
19309 21116 : case IX86_BUILTIN_PDEP32:
19310 21116 : case IX86_BUILTIN_PDEP64:
19311 21116 : gcc_assert (n_args == 2);
19312 21116 : if (tree_fits_uhwi_p (args[0]) && tree_fits_uhwi_p (args[1]))
19313 : {
19314 46 : unsigned HOST_WIDE_INT src = tree_to_uhwi (args[0]);
19315 46 : unsigned HOST_WIDE_INT mask = tree_to_uhwi (args[1]);
19316 46 : unsigned HOST_WIDE_INT res = 0;
19317 46 : unsigned HOST_WIDE_INT m, k = 1;
19318 2990 : for (m = 1; m; m <<= 1)
19319 2944 : if ((mask & m) != 0)
19320 : {
19321 1440 : if ((src & k) != 0)
19322 789 : res |= m;
19323 1440 : k <<= 1;
19324 : }
19325 46 : return build_int_cstu (TREE_TYPE (TREE_TYPE (fndecl)), res);
19326 : }
19327 : break;
19328 :
19329 21118 : case IX86_BUILTIN_PEXT32:
19330 21118 : case IX86_BUILTIN_PEXT64:
19331 21118 : gcc_assert (n_args == 2);
19332 21118 : if (tree_fits_uhwi_p (args[0]) && tree_fits_uhwi_p (args[1]))
19333 : {
19334 46 : unsigned HOST_WIDE_INT src = tree_to_uhwi (args[0]);
19335 46 : unsigned HOST_WIDE_INT mask = tree_to_uhwi (args[1]);
19336 46 : unsigned HOST_WIDE_INT res = 0;
19337 46 : unsigned HOST_WIDE_INT m, k = 1;
19338 2990 : for (m = 1; m; m <<= 1)
19339 2944 : if ((mask & m) != 0)
19340 : {
19341 2016 : if ((src & m) != 0)
19342 1063 : res |= k;
19343 2016 : k <<= 1;
19344 : }
19345 46 : return build_int_cstu (TREE_TYPE (TREE_TYPE (fndecl)), res);
19346 : }
19347 : break;
19348 :
19349 127814 : case IX86_BUILTIN_MOVMSKPS:
19350 127814 : case IX86_BUILTIN_PMOVMSKB:
19351 127814 : case IX86_BUILTIN_MOVMSKPD:
19352 127814 : case IX86_BUILTIN_PMOVMSKB128:
19353 127814 : case IX86_BUILTIN_MOVMSKPD256:
19354 127814 : case IX86_BUILTIN_MOVMSKPS256:
19355 127814 : case IX86_BUILTIN_PMOVMSKB256:
19356 127814 : gcc_assert (n_args == 1);
19357 127814 : if (TREE_CODE (args[0]) == VECTOR_CST)
19358 : {
19359 : HOST_WIDE_INT res = 0;
19360 2969 : for (unsigned i = 0; i < VECTOR_CST_NELTS (args[0]); ++i)
19361 : {
19362 2448 : tree e = VECTOR_CST_ELT (args[0], i);
19363 2448 : if (TREE_CODE (e) == INTEGER_CST && !TREE_OVERFLOW (e))
19364 : {
19365 1168 : if (wi::neg_p (wi::to_wide (e)))
19366 1119 : res |= HOST_WIDE_INT_1 << i;
19367 : }
19368 1280 : else if (TREE_CODE (e) == REAL_CST && !TREE_OVERFLOW (e))
19369 : {
19370 1280 : if (TREE_REAL_CST (e).sign)
19371 1031 : res |= HOST_WIDE_INT_1 << i;
19372 : }
19373 : else
19374 : return NULL_TREE;
19375 : }
19376 521 : return build_int_cst (TREE_TYPE (TREE_TYPE (fndecl)), res);
19377 : }
19378 : break;
19379 :
19380 669360 : case IX86_BUILTIN_PSLLD:
19381 669360 : case IX86_BUILTIN_PSLLD128:
19382 669360 : case IX86_BUILTIN_PSLLD128_MASK:
19383 669360 : case IX86_BUILTIN_PSLLD256:
19384 669360 : case IX86_BUILTIN_PSLLD256_MASK:
19385 669360 : case IX86_BUILTIN_PSLLD512:
19386 669360 : case IX86_BUILTIN_PSLLDI:
19387 669360 : case IX86_BUILTIN_PSLLDI128:
19388 669360 : case IX86_BUILTIN_PSLLDI128_MASK:
19389 669360 : case IX86_BUILTIN_PSLLDI256:
19390 669360 : case IX86_BUILTIN_PSLLDI256_MASK:
19391 669360 : case IX86_BUILTIN_PSLLDI512:
19392 669360 : case IX86_BUILTIN_PSLLQ:
19393 669360 : case IX86_BUILTIN_PSLLQ128:
19394 669360 : case IX86_BUILTIN_PSLLQ128_MASK:
19395 669360 : case IX86_BUILTIN_PSLLQ256:
19396 669360 : case IX86_BUILTIN_PSLLQ256_MASK:
19397 669360 : case IX86_BUILTIN_PSLLQ512:
19398 669360 : case IX86_BUILTIN_PSLLQI:
19399 669360 : case IX86_BUILTIN_PSLLQI128:
19400 669360 : case IX86_BUILTIN_PSLLQI128_MASK:
19401 669360 : case IX86_BUILTIN_PSLLQI256:
19402 669360 : case IX86_BUILTIN_PSLLQI256_MASK:
19403 669360 : case IX86_BUILTIN_PSLLQI512:
19404 669360 : case IX86_BUILTIN_PSLLW:
19405 669360 : case IX86_BUILTIN_PSLLW128:
19406 669360 : case IX86_BUILTIN_PSLLW128_MASK:
19407 669360 : case IX86_BUILTIN_PSLLW256:
19408 669360 : case IX86_BUILTIN_PSLLW256_MASK:
19409 669360 : case IX86_BUILTIN_PSLLW512_MASK:
19410 669360 : case IX86_BUILTIN_PSLLWI:
19411 669360 : case IX86_BUILTIN_PSLLWI128:
19412 669360 : case IX86_BUILTIN_PSLLWI128_MASK:
19413 669360 : case IX86_BUILTIN_PSLLWI256:
19414 669360 : case IX86_BUILTIN_PSLLWI256_MASK:
19415 669360 : case IX86_BUILTIN_PSLLWI512_MASK:
19416 669360 : rcode = ASHIFT;
19417 669360 : is_vshift = false;
19418 669360 : goto do_shift;
19419 609145 : case IX86_BUILTIN_PSRAD:
19420 609145 : case IX86_BUILTIN_PSRAD128:
19421 609145 : case IX86_BUILTIN_PSRAD128_MASK:
19422 609145 : case IX86_BUILTIN_PSRAD256:
19423 609145 : case IX86_BUILTIN_PSRAD256_MASK:
19424 609145 : case IX86_BUILTIN_PSRAD512:
19425 609145 : case IX86_BUILTIN_PSRADI:
19426 609145 : case IX86_BUILTIN_PSRADI128:
19427 609145 : case IX86_BUILTIN_PSRADI128_MASK:
19428 609145 : case IX86_BUILTIN_PSRADI256:
19429 609145 : case IX86_BUILTIN_PSRADI256_MASK:
19430 609145 : case IX86_BUILTIN_PSRADI512:
19431 609145 : case IX86_BUILTIN_PSRAQ128_MASK:
19432 609145 : case IX86_BUILTIN_PSRAQ256_MASK:
19433 609145 : case IX86_BUILTIN_PSRAQ512:
19434 609145 : case IX86_BUILTIN_PSRAQI128_MASK:
19435 609145 : case IX86_BUILTIN_PSRAQI256_MASK:
19436 609145 : case IX86_BUILTIN_PSRAQI512:
19437 609145 : case IX86_BUILTIN_PSRAW:
19438 609145 : case IX86_BUILTIN_PSRAW128:
19439 609145 : case IX86_BUILTIN_PSRAW128_MASK:
19440 609145 : case IX86_BUILTIN_PSRAW256:
19441 609145 : case IX86_BUILTIN_PSRAW256_MASK:
19442 609145 : case IX86_BUILTIN_PSRAW512:
19443 609145 : case IX86_BUILTIN_PSRAWI:
19444 609145 : case IX86_BUILTIN_PSRAWI128:
19445 609145 : case IX86_BUILTIN_PSRAWI128_MASK:
19446 609145 : case IX86_BUILTIN_PSRAWI256:
19447 609145 : case IX86_BUILTIN_PSRAWI256_MASK:
19448 609145 : case IX86_BUILTIN_PSRAWI512:
19449 609145 : rcode = ASHIFTRT;
19450 609145 : is_vshift = false;
19451 609145 : goto do_shift;
19452 642891 : case IX86_BUILTIN_PSRLD:
19453 642891 : case IX86_BUILTIN_PSRLD128:
19454 642891 : case IX86_BUILTIN_PSRLD128_MASK:
19455 642891 : case IX86_BUILTIN_PSRLD256:
19456 642891 : case IX86_BUILTIN_PSRLD256_MASK:
19457 642891 : case IX86_BUILTIN_PSRLD512:
19458 642891 : case IX86_BUILTIN_PSRLDI:
19459 642891 : case IX86_BUILTIN_PSRLDI128:
19460 642891 : case IX86_BUILTIN_PSRLDI128_MASK:
19461 642891 : case IX86_BUILTIN_PSRLDI256:
19462 642891 : case IX86_BUILTIN_PSRLDI256_MASK:
19463 642891 : case IX86_BUILTIN_PSRLDI512:
19464 642891 : case IX86_BUILTIN_PSRLQ:
19465 642891 : case IX86_BUILTIN_PSRLQ128:
19466 642891 : case IX86_BUILTIN_PSRLQ128_MASK:
19467 642891 : case IX86_BUILTIN_PSRLQ256:
19468 642891 : case IX86_BUILTIN_PSRLQ256_MASK:
19469 642891 : case IX86_BUILTIN_PSRLQ512:
19470 642891 : case IX86_BUILTIN_PSRLQI:
19471 642891 : case IX86_BUILTIN_PSRLQI128:
19472 642891 : case IX86_BUILTIN_PSRLQI128_MASK:
19473 642891 : case IX86_BUILTIN_PSRLQI256:
19474 642891 : case IX86_BUILTIN_PSRLQI256_MASK:
19475 642891 : case IX86_BUILTIN_PSRLQI512:
19476 642891 : case IX86_BUILTIN_PSRLW:
19477 642891 : case IX86_BUILTIN_PSRLW128:
19478 642891 : case IX86_BUILTIN_PSRLW128_MASK:
19479 642891 : case IX86_BUILTIN_PSRLW256:
19480 642891 : case IX86_BUILTIN_PSRLW256_MASK:
19481 642891 : case IX86_BUILTIN_PSRLW512:
19482 642891 : case IX86_BUILTIN_PSRLWI:
19483 642891 : case IX86_BUILTIN_PSRLWI128:
19484 642891 : case IX86_BUILTIN_PSRLWI128_MASK:
19485 642891 : case IX86_BUILTIN_PSRLWI256:
19486 642891 : case IX86_BUILTIN_PSRLWI256_MASK:
19487 642891 : case IX86_BUILTIN_PSRLWI512:
19488 642891 : rcode = LSHIFTRT;
19489 642891 : is_vshift = false;
19490 642891 : goto do_shift;
19491 279005 : case IX86_BUILTIN_PSLLVV16HI:
19492 279005 : case IX86_BUILTIN_PSLLVV16SI:
19493 279005 : case IX86_BUILTIN_PSLLVV2DI:
19494 279005 : case IX86_BUILTIN_PSLLVV2DI_MASK:
19495 279005 : case IX86_BUILTIN_PSLLVV32HI:
19496 279005 : case IX86_BUILTIN_PSLLVV4DI:
19497 279005 : case IX86_BUILTIN_PSLLVV4DI_MASK:
19498 279005 : case IX86_BUILTIN_PSLLVV4SI:
19499 279005 : case IX86_BUILTIN_PSLLVV4SI_MASK:
19500 279005 : case IX86_BUILTIN_PSLLVV8DI:
19501 279005 : case IX86_BUILTIN_PSLLVV8HI:
19502 279005 : case IX86_BUILTIN_PSLLVV8SI:
19503 279005 : case IX86_BUILTIN_PSLLVV8SI_MASK:
19504 279005 : rcode = ASHIFT;
19505 279005 : is_vshift = true;
19506 279005 : goto do_shift;
19507 278305 : case IX86_BUILTIN_PSRAVQ128:
19508 278305 : case IX86_BUILTIN_PSRAVQ256:
19509 278305 : case IX86_BUILTIN_PSRAVV16HI:
19510 278305 : case IX86_BUILTIN_PSRAVV16SI:
19511 278305 : case IX86_BUILTIN_PSRAVV32HI:
19512 278305 : case IX86_BUILTIN_PSRAVV4SI:
19513 278305 : case IX86_BUILTIN_PSRAVV4SI_MASK:
19514 278305 : case IX86_BUILTIN_PSRAVV8DI:
19515 278305 : case IX86_BUILTIN_PSRAVV8HI:
19516 278305 : case IX86_BUILTIN_PSRAVV8SI:
19517 278305 : case IX86_BUILTIN_PSRAVV8SI_MASK:
19518 278305 : rcode = ASHIFTRT;
19519 278305 : is_vshift = true;
19520 278305 : goto do_shift;
19521 278996 : case IX86_BUILTIN_PSRLVV16HI:
19522 278996 : case IX86_BUILTIN_PSRLVV16SI:
19523 278996 : case IX86_BUILTIN_PSRLVV2DI:
19524 278996 : case IX86_BUILTIN_PSRLVV2DI_MASK:
19525 278996 : case IX86_BUILTIN_PSRLVV32HI:
19526 278996 : case IX86_BUILTIN_PSRLVV4DI:
19527 278996 : case IX86_BUILTIN_PSRLVV4DI_MASK:
19528 278996 : case IX86_BUILTIN_PSRLVV4SI:
19529 278996 : case IX86_BUILTIN_PSRLVV4SI_MASK:
19530 278996 : case IX86_BUILTIN_PSRLVV8DI:
19531 278996 : case IX86_BUILTIN_PSRLVV8HI:
19532 278996 : case IX86_BUILTIN_PSRLVV8SI:
19533 278996 : case IX86_BUILTIN_PSRLVV8SI_MASK:
19534 278996 : rcode = LSHIFTRT;
19535 278996 : is_vshift = true;
19536 278996 : goto do_shift;
19537 :
19538 2757702 : do_shift:
19539 2757702 : gcc_assert (n_args >= 2);
19540 2757702 : if (TREE_CODE (args[0]) != VECTOR_CST)
19541 : break;
19542 927 : mask = HOST_WIDE_INT_M1U;
19543 927 : if (n_args > 2)
19544 : {
19545 : /* This is masked shift. */
19546 678 : if (!tree_fits_uhwi_p (args[n_args - 1])
19547 678 : || TREE_SIDE_EFFECTS (args[n_args - 2]))
19548 : break;
19549 678 : mask = tree_to_uhwi (args[n_args - 1]);
19550 678 : unsigned elems = TYPE_VECTOR_SUBPARTS (TREE_TYPE (args[0]));
19551 678 : mask |= HOST_WIDE_INT_M1U << elems;
19552 678 : if (mask != HOST_WIDE_INT_M1U
19553 567 : && TREE_CODE (args[n_args - 2]) != VECTOR_CST)
19554 : break;
19555 633 : if (mask == (HOST_WIDE_INT_M1U << elems))
19556 : return args[n_args - 2];
19557 : }
19558 879 : if (is_vshift && TREE_CODE (args[1]) != VECTOR_CST)
19559 : break;
19560 879 : if (tree tem = (is_vshift ? integer_one_node
19561 879 : : ix86_vector_shift_count (args[1])))
19562 : {
19563 558 : unsigned HOST_WIDE_INT count = tree_to_uhwi (tem);
19564 558 : unsigned HOST_WIDE_INT prec
19565 558 : = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (args[0])));
19566 558 : if (count == 0 && mask == HOST_WIDE_INT_M1U)
19567 : return args[0];
19568 558 : if (count >= prec)
19569 : {
19570 72 : if (rcode == ASHIFTRT)
19571 27 : count = prec - 1;
19572 45 : else if (mask == HOST_WIDE_INT_M1U)
19573 3 : return build_zero_cst (TREE_TYPE (args[0]));
19574 : }
19575 555 : tree countt = NULL_TREE;
19576 555 : if (!is_vshift)
19577 : {
19578 377 : if (count >= prec)
19579 42 : countt = integer_zero_node;
19580 : else
19581 335 : countt = build_int_cst (integer_type_node, count);
19582 : }
19583 555 : tree_vector_builder builder;
19584 555 : if (mask != HOST_WIDE_INT_M1U || is_vshift)
19585 392 : builder.new_vector (TREE_TYPE (args[0]),
19586 784 : TYPE_VECTOR_SUBPARTS (TREE_TYPE (args[0])),
19587 : 1);
19588 : else
19589 163 : builder.new_unary_operation (TREE_TYPE (args[0]), args[0],
19590 : false);
19591 555 : unsigned int cnt = builder.encoded_nelts ();
19592 5967 : for (unsigned int i = 0; i < cnt; ++i)
19593 : {
19594 5412 : tree elt = VECTOR_CST_ELT (args[0], i);
19595 5412 : if (TREE_CODE (elt) != INTEGER_CST || TREE_OVERFLOW (elt))
19596 0 : return NULL_TREE;
19597 5412 : tree type = TREE_TYPE (elt);
19598 5412 : if (rcode == LSHIFTRT)
19599 2040 : elt = fold_convert (unsigned_type_for (type), elt);
19600 5412 : if (is_vshift)
19601 : {
19602 1846 : countt = VECTOR_CST_ELT (args[1], i);
19603 1846 : if (TREE_CODE (countt) != INTEGER_CST
19604 1846 : || TREE_OVERFLOW (countt))
19605 : return NULL_TREE;
19606 1846 : if (wi::neg_p (wi::to_wide (countt))
19607 3610 : || wi::to_widest (countt) >= prec)
19608 : {
19609 325 : if (rcode == ASHIFTRT)
19610 108 : countt = build_int_cst (TREE_TYPE (countt),
19611 108 : prec - 1);
19612 : else
19613 : {
19614 217 : elt = build_zero_cst (TREE_TYPE (elt));
19615 217 : countt = build_zero_cst (TREE_TYPE (countt));
19616 : }
19617 : }
19618 : }
19619 3566 : else if (count >= prec)
19620 504 : elt = build_zero_cst (TREE_TYPE (elt));
19621 8950 : elt = const_binop (rcode == ASHIFT
19622 : ? LSHIFT_EXPR : RSHIFT_EXPR,
19623 5412 : TREE_TYPE (elt), elt, countt);
19624 5412 : if (!elt || TREE_CODE (elt) != INTEGER_CST)
19625 : return NULL_TREE;
19626 5412 : if (rcode == LSHIFTRT)
19627 2040 : elt = fold_convert (type, elt);
19628 5412 : if ((mask & (HOST_WIDE_INT_1U << i)) == 0)
19629 : {
19630 1566 : elt = VECTOR_CST_ELT (args[n_args - 2], i);
19631 1566 : if (TREE_CODE (elt) != INTEGER_CST
19632 1566 : || TREE_OVERFLOW (elt))
19633 : return NULL_TREE;
19634 : }
19635 5412 : builder.quick_push (elt);
19636 : }
19637 555 : return builder.build ();
19638 555 : }
19639 : break;
19640 :
19641 33265 : case IX86_BUILTIN_MINSS:
19642 33265 : case IX86_BUILTIN_MINSH_MASK:
19643 33265 : tcode = LT_EXPR;
19644 33265 : is_scalar = true;
19645 33265 : goto do_minmax;
19646 :
19647 33265 : case IX86_BUILTIN_MAXSS:
19648 33265 : case IX86_BUILTIN_MAXSH_MASK:
19649 33265 : tcode = GT_EXPR;
19650 33265 : is_scalar = true;
19651 33265 : goto do_minmax;
19652 :
19653 354623 : case IX86_BUILTIN_MINPS:
19654 354623 : case IX86_BUILTIN_MINPD:
19655 354623 : case IX86_BUILTIN_MINPS256:
19656 354623 : case IX86_BUILTIN_MINPD256:
19657 354623 : case IX86_BUILTIN_MINPS512:
19658 354623 : case IX86_BUILTIN_MINPD512:
19659 354623 : case IX86_BUILTIN_MINPS128_MASK:
19660 354623 : case IX86_BUILTIN_MINPD128_MASK:
19661 354623 : case IX86_BUILTIN_MINPS256_MASK:
19662 354623 : case IX86_BUILTIN_MINPD256_MASK:
19663 354623 : case IX86_BUILTIN_MINPH128_MASK:
19664 354623 : case IX86_BUILTIN_MINPH256_MASK:
19665 354623 : case IX86_BUILTIN_MINPH512_MASK:
19666 354623 : tcode = LT_EXPR;
19667 354623 : is_scalar = false;
19668 354623 : goto do_minmax;
19669 :
19670 : case IX86_BUILTIN_MAXPS:
19671 : case IX86_BUILTIN_MAXPD:
19672 : case IX86_BUILTIN_MAXPS256:
19673 : case IX86_BUILTIN_MAXPD256:
19674 : case IX86_BUILTIN_MAXPS512:
19675 : case IX86_BUILTIN_MAXPD512:
19676 : case IX86_BUILTIN_MAXPS128_MASK:
19677 : case IX86_BUILTIN_MAXPD128_MASK:
19678 : case IX86_BUILTIN_MAXPS256_MASK:
19679 : case IX86_BUILTIN_MAXPD256_MASK:
19680 : case IX86_BUILTIN_MAXPH128_MASK:
19681 : case IX86_BUILTIN_MAXPH256_MASK:
19682 : case IX86_BUILTIN_MAXPH512_MASK:
19683 : tcode = GT_EXPR;
19684 : is_scalar = false;
19685 775803 : do_minmax:
19686 775803 : gcc_assert (n_args >= 2);
19687 775803 : if (TREE_CODE (args[0]) != VECTOR_CST
19688 76 : || TREE_CODE (args[1]) != VECTOR_CST)
19689 : break;
19690 76 : mask = HOST_WIDE_INT_M1U;
19691 76 : if (n_args > 2)
19692 : {
19693 36 : gcc_assert (n_args >= 4);
19694 : /* This is masked minmax. */
19695 36 : if (TREE_CODE (args[3]) != INTEGER_CST
19696 36 : || TREE_SIDE_EFFECTS (args[2]))
19697 : break;
19698 36 : mask = TREE_INT_CST_LOW (args[3]);
19699 36 : unsigned elems = TYPE_VECTOR_SUBPARTS (TREE_TYPE (args[0]));
19700 36 : mask |= HOST_WIDE_INT_M1U << elems;
19701 36 : if (mask != HOST_WIDE_INT_M1U
19702 32 : && TREE_CODE (args[2]) != VECTOR_CST)
19703 : break;
19704 36 : if (n_args >= 5)
19705 : {
19706 20 : if (!tree_fits_uhwi_p (args[4]))
19707 : break;
19708 20 : if (tree_to_uhwi (args[4]) != 4
19709 0 : && tree_to_uhwi (args[4]) != 8)
19710 : break;
19711 : }
19712 36 : if (mask == (HOST_WIDE_INT_M1U << elems))
19713 : return args[2];
19714 : }
19715 : /* Punt on NaNs, unless exceptions are disabled. */
19716 76 : if (HONOR_NANS (args[0])
19717 76 : && (n_args < 5 || tree_to_uhwi (args[4]) != 8))
19718 184 : for (int i = 0; i < 2; ++i)
19719 : {
19720 134 : unsigned count = vector_cst_encoded_nelts (args[i]);
19721 1091 : for (unsigned j = 0; j < count; ++j)
19722 849 : if (tree_expr_nan_p (VECTOR_CST_ENCODED_ELT (args[i], j)))
19723 : return NULL_TREE;
19724 : }
19725 50 : {
19726 50 : tree res = const_binop (tcode,
19727 50 : truth_type_for (TREE_TYPE (args[0])),
19728 : args[0], args[1]);
19729 50 : if (res == NULL_TREE || TREE_CODE (res) != VECTOR_CST)
19730 : break;
19731 50 : res = fold_ternary (VEC_COND_EXPR, TREE_TYPE (args[0]), res,
19732 : args[0], args[1]);
19733 50 : if (res == NULL_TREE || TREE_CODE (res) != VECTOR_CST)
19734 : break;
19735 50 : if (mask != HOST_WIDE_INT_M1U)
19736 : {
19737 32 : unsigned nelts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (args[0]));
19738 32 : vec_perm_builder sel (nelts, nelts, 1);
19739 328 : for (unsigned int i = 0; i < nelts; i++)
19740 296 : if (mask & (HOST_WIDE_INT_1U << i))
19741 160 : sel.quick_push (i);
19742 : else
19743 136 : sel.quick_push (nelts + i);
19744 32 : vec_perm_indices indices (sel, 2, nelts);
19745 32 : res = fold_vec_perm (TREE_TYPE (args[0]), res, args[2],
19746 : indices);
19747 32 : if (res == NULL_TREE || TREE_CODE (res) != VECTOR_CST)
19748 : break;
19749 32 : }
19750 50 : if (is_scalar)
19751 : {
19752 10 : unsigned nelts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (args[0]));
19753 10 : vec_perm_builder sel (nelts, nelts, 1);
19754 10 : sel.quick_push (0);
19755 40 : for (unsigned int i = 1; i < nelts; i++)
19756 30 : sel.quick_push (nelts + i);
19757 10 : vec_perm_indices indices (sel, 2, nelts);
19758 10 : res = fold_vec_perm (TREE_TYPE (args[0]), res, args[0],
19759 : indices);
19760 10 : if (res == NULL_TREE || TREE_CODE (res) != VECTOR_CST)
19761 : break;
19762 10 : }
19763 : return res;
19764 : }
19765 :
19766 : default:
19767 : break;
19768 : }
19769 : }
19770 :
19771 : #ifdef SUBTARGET_FOLD_BUILTIN
19772 : return SUBTARGET_FOLD_BUILTIN (fndecl, n_args, args, ignore);
19773 : #endif
19774 :
19775 : return NULL_TREE;
19776 : }
19777 :
19778 : /* Fold a MD builtin (use ix86_fold_builtin for folding into
19779 : constant) in GIMPLE. */
19780 :
19781 : bool
19782 1157723 : ix86_gimple_fold_builtin (gimple_stmt_iterator *gsi)
19783 : {
19784 1157723 : gimple *stmt = gsi_stmt (*gsi), *g;
19785 1157723 : gimple_seq stmts = NULL;
19786 1157723 : tree fndecl = gimple_call_fndecl (stmt);
19787 1157723 : gcc_checking_assert (fndecl && fndecl_built_in_p (fndecl, BUILT_IN_MD));
19788 1157723 : int n_args = gimple_call_num_args (stmt);
19789 1157723 : enum ix86_builtins fn_code
19790 1157723 : = (enum ix86_builtins) DECL_MD_FUNCTION_CODE (fndecl);
19791 1157723 : tree decl = NULL_TREE;
19792 1157723 : tree arg0, arg1, arg2;
19793 1157723 : enum rtx_code rcode;
19794 1157723 : enum tree_code tcode;
19795 1157723 : unsigned HOST_WIDE_INT count;
19796 1157723 : bool is_vshift;
19797 1157723 : unsigned HOST_WIDE_INT elems;
19798 1157723 : location_t loc;
19799 :
19800 : /* Don't fold when there's isa mismatch. */
19801 1157723 : if (!ix86_check_builtin_isa_match (fn_code, NULL, NULL))
19802 : return false;
19803 :
19804 1157596 : switch (fn_code)
19805 : {
19806 288 : case IX86_BUILTIN_TZCNT32:
19807 288 : decl = builtin_decl_implicit (BUILT_IN_CTZ);
19808 288 : goto fold_tzcnt_lzcnt;
19809 :
19810 237 : case IX86_BUILTIN_TZCNT64:
19811 237 : decl = builtin_decl_implicit (BUILT_IN_CTZLL);
19812 237 : goto fold_tzcnt_lzcnt;
19813 :
19814 215 : case IX86_BUILTIN_LZCNT32:
19815 215 : decl = builtin_decl_implicit (BUILT_IN_CLZ);
19816 215 : goto fold_tzcnt_lzcnt;
19817 :
19818 224 : case IX86_BUILTIN_LZCNT64:
19819 224 : decl = builtin_decl_implicit (BUILT_IN_CLZLL);
19820 224 : goto fold_tzcnt_lzcnt;
19821 :
19822 964 : fold_tzcnt_lzcnt:
19823 964 : gcc_assert (n_args == 1);
19824 964 : arg0 = gimple_call_arg (stmt, 0);
19825 964 : if (TREE_CODE (arg0) == SSA_NAME && decl && gimple_call_lhs (stmt))
19826 : {
19827 799 : int prec = TYPE_PRECISION (TREE_TYPE (arg0));
19828 : /* If arg0 is provably non-zero, optimize into generic
19829 : __builtin_c[tl]z{,ll} function the middle-end handles
19830 : better. */
19831 799 : if (!expr_not_equal_to (arg0, wi::zero (prec)))
19832 : return false;
19833 :
19834 9 : loc = gimple_location (stmt);
19835 9 : g = gimple_build_call (decl, 1, arg0);
19836 9 : gimple_set_location (g, loc);
19837 9 : tree lhs = make_ssa_name (integer_type_node);
19838 9 : gimple_call_set_lhs (g, lhs);
19839 9 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
19840 9 : g = gimple_build_assign (gimple_call_lhs (stmt), NOP_EXPR, lhs);
19841 9 : gimple_set_location (g, loc);
19842 9 : gsi_replace (gsi, g, false);
19843 9 : return true;
19844 : }
19845 : break;
19846 :
19847 491 : case IX86_BUILTIN_BZHI32:
19848 491 : case IX86_BUILTIN_BZHI64:
19849 491 : gcc_assert (n_args == 2);
19850 491 : arg1 = gimple_call_arg (stmt, 1);
19851 491 : if (tree_fits_uhwi_p (arg1) && gimple_call_lhs (stmt))
19852 : {
19853 195 : unsigned int idx = tree_to_uhwi (arg1) & 0xff;
19854 195 : arg0 = gimple_call_arg (stmt, 0);
19855 195 : if (idx < TYPE_PRECISION (TREE_TYPE (arg0)))
19856 : break;
19857 31 : loc = gimple_location (stmt);
19858 31 : g = gimple_build_assign (gimple_call_lhs (stmt), arg0);
19859 31 : gimple_set_location (g, loc);
19860 31 : gsi_replace (gsi, g, false);
19861 31 : return true;
19862 : }
19863 : break;
19864 :
19865 502 : case IX86_BUILTIN_PDEP32:
19866 502 : case IX86_BUILTIN_PDEP64:
19867 502 : case IX86_BUILTIN_PEXT32:
19868 502 : case IX86_BUILTIN_PEXT64:
19869 502 : gcc_assert (n_args == 2);
19870 502 : arg1 = gimple_call_arg (stmt, 1);
19871 502 : if (integer_all_onesp (arg1) && gimple_call_lhs (stmt))
19872 : {
19873 4 : loc = gimple_location (stmt);
19874 4 : arg0 = gimple_call_arg (stmt, 0);
19875 4 : g = gimple_build_assign (gimple_call_lhs (stmt), arg0);
19876 4 : gimple_set_location (g, loc);
19877 4 : gsi_replace (gsi, g, false);
19878 4 : return true;
19879 : }
19880 : break;
19881 :
19882 1148 : case IX86_BUILTIN_PAVGUSB:
19883 1148 : case IX86_BUILTIN_PAVGB:
19884 1148 : case IX86_BUILTIN_PAVGW:
19885 1148 : case IX86_BUILTIN_PAVGB128:
19886 1148 : case IX86_BUILTIN_PAVGW128:
19887 1148 : case IX86_BUILTIN_PAVGB256:
19888 1148 : case IX86_BUILTIN_PAVGW256:
19889 1148 : case IX86_BUILTIN_PAVGB128_MASK:
19890 1148 : case IX86_BUILTIN_PAVGW128_MASK:
19891 1148 : case IX86_BUILTIN_PAVGB256_MASK:
19892 1148 : case IX86_BUILTIN_PAVGW256_MASK:
19893 1148 : case IX86_BUILTIN_PAVGB512:
19894 1148 : case IX86_BUILTIN_PAVGW512:
19895 1148 : gcc_assert (n_args == 2 || n_args == 4);
19896 1148 : if (!gimple_call_lhs (stmt))
19897 : break;
19898 1148 : arg0 = gimple_call_arg (stmt, 0);
19899 1148 : arg1 = gimple_call_arg (stmt, 1);
19900 : /* For masked PAVG, only canonicalize if the mask is all ones. */
19901 1148 : if (n_args == 4)
19902 : {
19903 1041 : elems = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
19904 1041 : if (!ix86_masked_all_ones (elems, gimple_call_arg (stmt, 3)))
19905 : break;
19906 : }
19907 138 : {
19908 : /* PAVG computes an unsigned average rounded towards positive
19909 : infinity. The IFN selects signedness from its operand type. */
19910 138 : tree utype = unsigned_type_for (TREE_TYPE (arg0));
19911 138 : bool equal_p = operand_equal_p (arg0, arg1, 0);
19912 138 : if (!equal_p
19913 138 : && !direct_internal_fn_supported_p (IFN_AVG_CEIL, utype,
19914 : OPTIMIZE_FOR_BOTH))
19915 : break;
19916 :
19917 138 : loc = gimple_location (stmt);
19918 138 : tree uarg0 = gimple_build (&stmts, loc, VIEW_CONVERT_EXPR,
19919 : utype, arg0);
19920 138 : tree uarg1 = equal_p
19921 138 : ? uarg0
19922 138 : : gimple_build (&stmts, loc, VIEW_CONVERT_EXPR, utype, arg1);
19923 138 : tree res = gimple_build (&stmts, loc, IFN_AVG_CEIL, utype,
19924 : uarg0, uarg1);
19925 138 : res = gimple_build (&stmts, loc, VIEW_CONVERT_EXPR,
19926 138 : TREE_TYPE (gimple_call_lhs (stmt)), res);
19927 138 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
19928 138 : g = gimple_build_assign (gimple_call_lhs (stmt), res);
19929 138 : gimple_set_location (g, loc);
19930 138 : gsi_replace (gsi, g, false);
19931 138 : return true;
19932 : }
19933 :
19934 145 : case IX86_BUILTIN_PBLENDVB256:
19935 145 : case IX86_BUILTIN_BLENDVPS256:
19936 145 : case IX86_BUILTIN_BLENDVPD256:
19937 : /* pcmpeqb/d/q is under avx2, w/o avx2, it's veclower
19938 : to scalar operations and not combined back. */
19939 145 : if (!TARGET_AVX2)
19940 : break;
19941 :
19942 : /* FALLTHRU. */
19943 112 : case IX86_BUILTIN_BLENDVPD:
19944 : /* blendvpd is under sse4.1 but pcmpgtq is under sse4.2,
19945 : w/o sse4.2, it's veclowered to scalar operations and
19946 : not combined back. */
19947 112 : if (!TARGET_SSE4_2)
19948 : break;
19949 : /* FALLTHRU. */
19950 166 : case IX86_BUILTIN_PBLENDVB128:
19951 166 : case IX86_BUILTIN_BLENDVPS:
19952 166 : gcc_assert (n_args == 3);
19953 166 : arg0 = gimple_call_arg (stmt, 0);
19954 166 : arg1 = gimple_call_arg (stmt, 1);
19955 166 : arg2 = gimple_call_arg (stmt, 2);
19956 166 : if (gimple_call_lhs (stmt))
19957 : {
19958 166 : loc = gimple_location (stmt);
19959 166 : tree type = TREE_TYPE (arg2);
19960 166 : if (VECTOR_FLOAT_TYPE_P (type))
19961 : {
19962 73 : tree itype = GET_MODE_INNER (TYPE_MODE (type)) == E_SFmode
19963 73 : ? intSI_type_node : intDI_type_node;
19964 73 : type = get_same_sized_vectype (itype, type);
19965 : }
19966 : else
19967 93 : type = signed_type_for (type);
19968 166 : arg2 = gimple_build (&stmts, VIEW_CONVERT_EXPR, type, arg2);
19969 166 : tree zero_vec = build_zero_cst (type);
19970 166 : tree cmp_type = truth_type_for (type);
19971 166 : tree cmp = gimple_build (&stmts, LT_EXPR, cmp_type, arg2, zero_vec);
19972 166 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
19973 166 : g = gimple_build_assign (gimple_call_lhs (stmt),
19974 : VEC_COND_EXPR, cmp,
19975 : arg1, arg0);
19976 166 : gimple_set_location (g, loc);
19977 166 : gsi_replace (gsi, g, false);
19978 : }
19979 : else
19980 0 : gsi_replace (gsi, gimple_build_nop (), false);
19981 : return true;
19982 :
19983 :
19984 16 : case IX86_BUILTIN_PCMPEQB128:
19985 16 : case IX86_BUILTIN_PCMPEQW128:
19986 16 : case IX86_BUILTIN_PCMPEQD128:
19987 16 : case IX86_BUILTIN_PCMPEQQ:
19988 16 : case IX86_BUILTIN_PCMPEQB256:
19989 16 : case IX86_BUILTIN_PCMPEQW256:
19990 16 : case IX86_BUILTIN_PCMPEQD256:
19991 16 : case IX86_BUILTIN_PCMPEQQ256:
19992 16 : tcode = EQ_EXPR;
19993 16 : goto do_cmp;
19994 :
19995 : case IX86_BUILTIN_PCMPGTB128:
19996 : case IX86_BUILTIN_PCMPGTW128:
19997 : case IX86_BUILTIN_PCMPGTD128:
19998 : case IX86_BUILTIN_PCMPGTQ:
19999 : case IX86_BUILTIN_PCMPGTB256:
20000 : case IX86_BUILTIN_PCMPGTW256:
20001 : case IX86_BUILTIN_PCMPGTD256:
20002 : case IX86_BUILTIN_PCMPGTQ256:
20003 : tcode = GT_EXPR;
20004 :
20005 33 : do_cmp:
20006 33 : gcc_assert (n_args == 2);
20007 33 : arg0 = gimple_call_arg (stmt, 0);
20008 33 : arg1 = gimple_call_arg (stmt, 1);
20009 33 : if (gimple_call_lhs (stmt))
20010 : {
20011 32 : loc = gimple_location (stmt);
20012 32 : tree type = TREE_TYPE (arg0);
20013 32 : tree zero_vec = build_zero_cst (type);
20014 32 : tree minus_one_vec = build_minus_one_cst (type);
20015 32 : tree cmp_type = truth_type_for (type);
20016 32 : tree cmp = gimple_build (&stmts, tcode, cmp_type, arg0, arg1);
20017 32 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
20018 32 : g = gimple_build_assign (gimple_call_lhs (stmt),
20019 : VEC_COND_EXPR, cmp,
20020 : minus_one_vec, zero_vec);
20021 32 : gimple_set_location (g, loc);
20022 32 : gsi_replace (gsi, g, false);
20023 : }
20024 : else
20025 1 : gsi_replace (gsi, gimple_build_nop (), false);
20026 : return true;
20027 :
20028 9297 : case IX86_BUILTIN_PSLLD:
20029 9297 : case IX86_BUILTIN_PSLLD128:
20030 9297 : case IX86_BUILTIN_PSLLD128_MASK:
20031 9297 : case IX86_BUILTIN_PSLLD256:
20032 9297 : case IX86_BUILTIN_PSLLD256_MASK:
20033 9297 : case IX86_BUILTIN_PSLLD512:
20034 9297 : case IX86_BUILTIN_PSLLDI:
20035 9297 : case IX86_BUILTIN_PSLLDI128:
20036 9297 : case IX86_BUILTIN_PSLLDI128_MASK:
20037 9297 : case IX86_BUILTIN_PSLLDI256:
20038 9297 : case IX86_BUILTIN_PSLLDI256_MASK:
20039 9297 : case IX86_BUILTIN_PSLLDI512:
20040 9297 : case IX86_BUILTIN_PSLLQ:
20041 9297 : case IX86_BUILTIN_PSLLQ128:
20042 9297 : case IX86_BUILTIN_PSLLQ128_MASK:
20043 9297 : case IX86_BUILTIN_PSLLQ256:
20044 9297 : case IX86_BUILTIN_PSLLQ256_MASK:
20045 9297 : case IX86_BUILTIN_PSLLQ512:
20046 9297 : case IX86_BUILTIN_PSLLQI:
20047 9297 : case IX86_BUILTIN_PSLLQI128:
20048 9297 : case IX86_BUILTIN_PSLLQI128_MASK:
20049 9297 : case IX86_BUILTIN_PSLLQI256:
20050 9297 : case IX86_BUILTIN_PSLLQI256_MASK:
20051 9297 : case IX86_BUILTIN_PSLLQI512:
20052 9297 : case IX86_BUILTIN_PSLLW:
20053 9297 : case IX86_BUILTIN_PSLLW128:
20054 9297 : case IX86_BUILTIN_PSLLW128_MASK:
20055 9297 : case IX86_BUILTIN_PSLLW256:
20056 9297 : case IX86_BUILTIN_PSLLW256_MASK:
20057 9297 : case IX86_BUILTIN_PSLLW512_MASK:
20058 9297 : case IX86_BUILTIN_PSLLWI:
20059 9297 : case IX86_BUILTIN_PSLLWI128:
20060 9297 : case IX86_BUILTIN_PSLLWI128_MASK:
20061 9297 : case IX86_BUILTIN_PSLLWI256:
20062 9297 : case IX86_BUILTIN_PSLLWI256_MASK:
20063 9297 : case IX86_BUILTIN_PSLLWI512_MASK:
20064 9297 : rcode = ASHIFT;
20065 9297 : is_vshift = false;
20066 9297 : goto do_shift;
20067 6495 : case IX86_BUILTIN_PSRAD:
20068 6495 : case IX86_BUILTIN_PSRAD128:
20069 6495 : case IX86_BUILTIN_PSRAD128_MASK:
20070 6495 : case IX86_BUILTIN_PSRAD256:
20071 6495 : case IX86_BUILTIN_PSRAD256_MASK:
20072 6495 : case IX86_BUILTIN_PSRAD512:
20073 6495 : case IX86_BUILTIN_PSRADI:
20074 6495 : case IX86_BUILTIN_PSRADI128:
20075 6495 : case IX86_BUILTIN_PSRADI128_MASK:
20076 6495 : case IX86_BUILTIN_PSRADI256:
20077 6495 : case IX86_BUILTIN_PSRADI256_MASK:
20078 6495 : case IX86_BUILTIN_PSRADI512:
20079 6495 : case IX86_BUILTIN_PSRAQ128_MASK:
20080 6495 : case IX86_BUILTIN_PSRAQ256_MASK:
20081 6495 : case IX86_BUILTIN_PSRAQ512:
20082 6495 : case IX86_BUILTIN_PSRAQI128_MASK:
20083 6495 : case IX86_BUILTIN_PSRAQI256_MASK:
20084 6495 : case IX86_BUILTIN_PSRAQI512:
20085 6495 : case IX86_BUILTIN_PSRAW:
20086 6495 : case IX86_BUILTIN_PSRAW128:
20087 6495 : case IX86_BUILTIN_PSRAW128_MASK:
20088 6495 : case IX86_BUILTIN_PSRAW256:
20089 6495 : case IX86_BUILTIN_PSRAW256_MASK:
20090 6495 : case IX86_BUILTIN_PSRAW512:
20091 6495 : case IX86_BUILTIN_PSRAWI:
20092 6495 : case IX86_BUILTIN_PSRAWI128:
20093 6495 : case IX86_BUILTIN_PSRAWI128_MASK:
20094 6495 : case IX86_BUILTIN_PSRAWI256:
20095 6495 : case IX86_BUILTIN_PSRAWI256_MASK:
20096 6495 : case IX86_BUILTIN_PSRAWI512:
20097 6495 : rcode = ASHIFTRT;
20098 6495 : is_vshift = false;
20099 6495 : goto do_shift;
20100 7960 : case IX86_BUILTIN_PSRLD:
20101 7960 : case IX86_BUILTIN_PSRLD128:
20102 7960 : case IX86_BUILTIN_PSRLD128_MASK:
20103 7960 : case IX86_BUILTIN_PSRLD256:
20104 7960 : case IX86_BUILTIN_PSRLD256_MASK:
20105 7960 : case IX86_BUILTIN_PSRLD512:
20106 7960 : case IX86_BUILTIN_PSRLDI:
20107 7960 : case IX86_BUILTIN_PSRLDI128:
20108 7960 : case IX86_BUILTIN_PSRLDI128_MASK:
20109 7960 : case IX86_BUILTIN_PSRLDI256:
20110 7960 : case IX86_BUILTIN_PSRLDI256_MASK:
20111 7960 : case IX86_BUILTIN_PSRLDI512:
20112 7960 : case IX86_BUILTIN_PSRLQ:
20113 7960 : case IX86_BUILTIN_PSRLQ128:
20114 7960 : case IX86_BUILTIN_PSRLQ128_MASK:
20115 7960 : case IX86_BUILTIN_PSRLQ256:
20116 7960 : case IX86_BUILTIN_PSRLQ256_MASK:
20117 7960 : case IX86_BUILTIN_PSRLQ512:
20118 7960 : case IX86_BUILTIN_PSRLQI:
20119 7960 : case IX86_BUILTIN_PSRLQI128:
20120 7960 : case IX86_BUILTIN_PSRLQI128_MASK:
20121 7960 : case IX86_BUILTIN_PSRLQI256:
20122 7960 : case IX86_BUILTIN_PSRLQI256_MASK:
20123 7960 : case IX86_BUILTIN_PSRLQI512:
20124 7960 : case IX86_BUILTIN_PSRLW:
20125 7960 : case IX86_BUILTIN_PSRLW128:
20126 7960 : case IX86_BUILTIN_PSRLW128_MASK:
20127 7960 : case IX86_BUILTIN_PSRLW256:
20128 7960 : case IX86_BUILTIN_PSRLW256_MASK:
20129 7960 : case IX86_BUILTIN_PSRLW512:
20130 7960 : case IX86_BUILTIN_PSRLWI:
20131 7960 : case IX86_BUILTIN_PSRLWI128:
20132 7960 : case IX86_BUILTIN_PSRLWI128_MASK:
20133 7960 : case IX86_BUILTIN_PSRLWI256:
20134 7960 : case IX86_BUILTIN_PSRLWI256_MASK:
20135 7960 : case IX86_BUILTIN_PSRLWI512:
20136 7960 : rcode = LSHIFTRT;
20137 7960 : is_vshift = false;
20138 7960 : goto do_shift;
20139 2384 : case IX86_BUILTIN_PSLLVV16HI:
20140 2384 : case IX86_BUILTIN_PSLLVV16SI:
20141 2384 : case IX86_BUILTIN_PSLLVV2DI:
20142 2384 : case IX86_BUILTIN_PSLLVV2DI_MASK:
20143 2384 : case IX86_BUILTIN_PSLLVV32HI:
20144 2384 : case IX86_BUILTIN_PSLLVV4DI:
20145 2384 : case IX86_BUILTIN_PSLLVV4DI_MASK:
20146 2384 : case IX86_BUILTIN_PSLLVV4SI:
20147 2384 : case IX86_BUILTIN_PSLLVV4SI_MASK:
20148 2384 : case IX86_BUILTIN_PSLLVV8DI:
20149 2384 : case IX86_BUILTIN_PSLLVV8HI:
20150 2384 : case IX86_BUILTIN_PSLLVV8SI:
20151 2384 : case IX86_BUILTIN_PSLLVV8SI_MASK:
20152 2384 : rcode = ASHIFT;
20153 2384 : is_vshift = true;
20154 2384 : goto do_shift;
20155 2341 : case IX86_BUILTIN_PSRAVQ128:
20156 2341 : case IX86_BUILTIN_PSRAVQ256:
20157 2341 : case IX86_BUILTIN_PSRAVV16HI:
20158 2341 : case IX86_BUILTIN_PSRAVV16SI:
20159 2341 : case IX86_BUILTIN_PSRAVV32HI:
20160 2341 : case IX86_BUILTIN_PSRAVV4SI:
20161 2341 : case IX86_BUILTIN_PSRAVV4SI_MASK:
20162 2341 : case IX86_BUILTIN_PSRAVV8DI:
20163 2341 : case IX86_BUILTIN_PSRAVV8HI:
20164 2341 : case IX86_BUILTIN_PSRAVV8SI:
20165 2341 : case IX86_BUILTIN_PSRAVV8SI_MASK:
20166 2341 : rcode = ASHIFTRT;
20167 2341 : is_vshift = true;
20168 2341 : goto do_shift;
20169 2380 : case IX86_BUILTIN_PSRLVV16HI:
20170 2380 : case IX86_BUILTIN_PSRLVV16SI:
20171 2380 : case IX86_BUILTIN_PSRLVV2DI:
20172 2380 : case IX86_BUILTIN_PSRLVV2DI_MASK:
20173 2380 : case IX86_BUILTIN_PSRLVV32HI:
20174 2380 : case IX86_BUILTIN_PSRLVV4DI:
20175 2380 : case IX86_BUILTIN_PSRLVV4DI_MASK:
20176 2380 : case IX86_BUILTIN_PSRLVV4SI:
20177 2380 : case IX86_BUILTIN_PSRLVV4SI_MASK:
20178 2380 : case IX86_BUILTIN_PSRLVV8DI:
20179 2380 : case IX86_BUILTIN_PSRLVV8HI:
20180 2380 : case IX86_BUILTIN_PSRLVV8SI:
20181 2380 : case IX86_BUILTIN_PSRLVV8SI_MASK:
20182 2380 : rcode = LSHIFTRT;
20183 2380 : is_vshift = true;
20184 2380 : goto do_shift;
20185 :
20186 30857 : do_shift:
20187 30857 : gcc_assert (n_args >= 2);
20188 30857 : if (!gimple_call_lhs (stmt))
20189 : {
20190 1 : gsi_replace (gsi, gimple_build_nop (), false);
20191 1 : return true;
20192 : }
20193 30856 : arg0 = gimple_call_arg (stmt, 0);
20194 30856 : arg1 = gimple_call_arg (stmt, 1);
20195 30856 : elems = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
20196 : /* For masked shift, only optimize if the mask is all ones. */
20197 30856 : if (n_args > 2
20198 30856 : && !ix86_masked_all_ones (elems, gimple_call_arg (stmt, n_args - 1)))
20199 : break;
20200 16081 : if (is_vshift)
20201 : {
20202 2640 : if (TREE_CODE (arg1) != VECTOR_CST)
20203 : break;
20204 69 : count = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (arg0)));
20205 69 : if (integer_zerop (arg1))
20206 27 : count = 0;
20207 42 : else if (rcode == ASHIFTRT)
20208 : break;
20209 : else
20210 230 : for (unsigned int i = 0; i < VECTOR_CST_NELTS (arg1); ++i)
20211 : {
20212 212 : tree elt = VECTOR_CST_ELT (arg1, i);
20213 212 : if (!wi::neg_p (wi::to_wide (elt))
20214 375 : && wi::to_widest (elt) < count)
20215 16 : return false;
20216 : }
20217 : }
20218 : else
20219 : {
20220 13441 : arg1 = ix86_vector_shift_count (arg1);
20221 13441 : if (!arg1)
20222 : break;
20223 5608 : count = tree_to_uhwi (arg1);
20224 : }
20225 5653 : if (count == 0)
20226 : {
20227 : /* Just return the first argument for shift by 0. */
20228 93 : loc = gimple_location (stmt);
20229 93 : g = gimple_build_assign (gimple_call_lhs (stmt), arg0);
20230 93 : gimple_set_location (g, loc);
20231 93 : gsi_replace (gsi, g, false);
20232 93 : return true;
20233 : }
20234 5560 : if (rcode != ASHIFTRT
20235 5560 : && count >= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (arg0))))
20236 : {
20237 : /* For shift counts equal or greater than precision, except for
20238 : arithmetic right shift the result is zero. */
20239 78 : loc = gimple_location (stmt);
20240 78 : g = gimple_build_assign (gimple_call_lhs (stmt),
20241 78 : build_zero_cst (TREE_TYPE (arg0)));
20242 78 : gimple_set_location (g, loc);
20243 78 : gsi_replace (gsi, g, false);
20244 78 : return true;
20245 : }
20246 : break;
20247 :
20248 531 : case IX86_BUILTIN_SHUFPD512:
20249 531 : case IX86_BUILTIN_SHUFPS512:
20250 531 : case IX86_BUILTIN_SHUFPD:
20251 531 : case IX86_BUILTIN_SHUFPD256:
20252 531 : case IX86_BUILTIN_SHUFPS:
20253 531 : case IX86_BUILTIN_SHUFPS256:
20254 531 : arg0 = gimple_call_arg (stmt, 0);
20255 531 : elems = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
20256 : /* This is masked shuffle. Only optimize if the mask is all ones. */
20257 531 : if (n_args > 3
20258 895 : && !ix86_masked_all_ones (elems,
20259 364 : gimple_call_arg (stmt, n_args - 1)))
20260 : break;
20261 203 : arg2 = gimple_call_arg (stmt, 2);
20262 203 : if (TREE_CODE (arg2) == INTEGER_CST && gimple_call_lhs (stmt))
20263 : {
20264 146 : unsigned HOST_WIDE_INT shuffle_mask = TREE_INT_CST_LOW (arg2);
20265 : /* Check valid imm, refer to gcc.target/i386/testimm-10.c. */
20266 146 : if (shuffle_mask > 255)
20267 : return false;
20268 :
20269 144 : machine_mode imode = GET_MODE_INNER (TYPE_MODE (TREE_TYPE (arg0)));
20270 144 : loc = gimple_location (stmt);
20271 144 : tree itype = (imode == E_DFmode
20272 144 : ? long_long_integer_type_node : integer_type_node);
20273 144 : tree vtype = build_vector_type (itype, elems);
20274 144 : tree_vector_builder elts (vtype, elems, 1);
20275 :
20276 :
20277 : /* Transform integer shuffle_mask to vector perm_mask which
20278 : is used by vec_perm_expr, refer to shuflp[sd]256/512 in sse.md. */
20279 984 : for (unsigned i = 0; i != elems; i++)
20280 : {
20281 696 : unsigned sel_idx;
20282 : /* Imm[1:0](if VL > 128, then use Imm[3:2],Imm[5:4],Imm[7:6])
20283 : provide 2 select controls for each element of the
20284 : destination. */
20285 696 : if (imode == E_DFmode)
20286 240 : sel_idx = (i & 1) * elems + (i & ~1)
20287 240 : + ((shuffle_mask >> i) & 1);
20288 : else
20289 : {
20290 : /* Imm[7:0](if VL > 128, also use Imm[7:0]) provide 4 select
20291 : controls for each element of the destination. */
20292 456 : unsigned j = i % 4;
20293 456 : sel_idx = ((i >> 1) & 1) * elems + (i & ~3)
20294 456 : + ((shuffle_mask >> 2 * j) & 3);
20295 : }
20296 696 : elts.quick_push (build_int_cst (itype, sel_idx));
20297 : }
20298 :
20299 144 : tree perm_mask = elts.build ();
20300 144 : arg1 = gimple_call_arg (stmt, 1);
20301 144 : g = gimple_build_assign (gimple_call_lhs (stmt),
20302 : VEC_PERM_EXPR,
20303 : arg0, arg1, perm_mask);
20304 144 : gimple_set_location (g, loc);
20305 144 : gsi_replace (gsi, g, false);
20306 144 : return true;
20307 144 : }
20308 : // Do not error yet, the constant could be propagated later?
20309 : break;
20310 :
20311 48 : case IX86_BUILTIN_PABSB:
20312 48 : case IX86_BUILTIN_PABSW:
20313 48 : case IX86_BUILTIN_PABSD:
20314 : /* 64-bit vector abs<mode>2 is only supported under TARGET_MMX_WITH_SSE. */
20315 48 : if (!TARGET_MMX_WITH_SSE)
20316 : break;
20317 : /* FALLTHRU. */
20318 2190 : case IX86_BUILTIN_PABSB128:
20319 2190 : case IX86_BUILTIN_PABSB256:
20320 2190 : case IX86_BUILTIN_PABSB512:
20321 2190 : case IX86_BUILTIN_PABSW128:
20322 2190 : case IX86_BUILTIN_PABSW256:
20323 2190 : case IX86_BUILTIN_PABSW512:
20324 2190 : case IX86_BUILTIN_PABSD128:
20325 2190 : case IX86_BUILTIN_PABSD256:
20326 2190 : case IX86_BUILTIN_PABSD512:
20327 2190 : case IX86_BUILTIN_PABSQ128:
20328 2190 : case IX86_BUILTIN_PABSQ256:
20329 2190 : case IX86_BUILTIN_PABSQ512:
20330 2190 : case IX86_BUILTIN_PABSB128_MASK:
20331 2190 : case IX86_BUILTIN_PABSB256_MASK:
20332 2190 : case IX86_BUILTIN_PABSW128_MASK:
20333 2190 : case IX86_BUILTIN_PABSW256_MASK:
20334 2190 : case IX86_BUILTIN_PABSD128_MASK:
20335 2190 : case IX86_BUILTIN_PABSD256_MASK:
20336 2190 : gcc_assert (n_args >= 1);
20337 2190 : if (!gimple_call_lhs (stmt))
20338 : {
20339 1 : gsi_replace (gsi, gimple_build_nop (), false);
20340 1 : return true;
20341 : }
20342 2189 : arg0 = gimple_call_arg (stmt, 0);
20343 2189 : elems = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
20344 : /* For masked ABS, only optimize if the mask is all ones. */
20345 2189 : if (n_args > 1
20346 2189 : && !ix86_masked_all_ones (elems, gimple_call_arg (stmt, n_args - 1)))
20347 : break;
20348 229 : {
20349 229 : tree utype, ures, vce;
20350 229 : utype = unsigned_type_for (TREE_TYPE (arg0));
20351 : /* PABSB/W/D/Q store the unsigned result in dst, use ABSU_EXPR
20352 : instead of ABS_EXPR to handle overflow case(TYPE_MIN). */
20353 229 : ures = gimple_build (&stmts, ABSU_EXPR, utype, arg0);
20354 229 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
20355 229 : loc = gimple_location (stmt);
20356 229 : vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (arg0), ures);
20357 229 : g = gimple_build_assign (gimple_call_lhs (stmt),
20358 : VIEW_CONVERT_EXPR, vce);
20359 229 : gsi_replace (gsi, g, false);
20360 : }
20361 229 : return true;
20362 :
20363 2225 : case IX86_BUILTIN_MINPS:
20364 2225 : case IX86_BUILTIN_MINPD:
20365 2225 : case IX86_BUILTIN_MINPS256:
20366 2225 : case IX86_BUILTIN_MINPD256:
20367 2225 : case IX86_BUILTIN_MINPS512:
20368 2225 : case IX86_BUILTIN_MINPD512:
20369 2225 : case IX86_BUILTIN_MINPS128_MASK:
20370 2225 : case IX86_BUILTIN_MINPD128_MASK:
20371 2225 : case IX86_BUILTIN_MINPS256_MASK:
20372 2225 : case IX86_BUILTIN_MINPD256_MASK:
20373 2225 : case IX86_BUILTIN_MINPH128_MASK:
20374 2225 : case IX86_BUILTIN_MINPH256_MASK:
20375 2225 : case IX86_BUILTIN_MINPH512_MASK:
20376 2225 : tcode = LT_EXPR;
20377 2225 : goto do_minmax;
20378 :
20379 : case IX86_BUILTIN_MAXPS:
20380 : case IX86_BUILTIN_MAXPD:
20381 : case IX86_BUILTIN_MAXPS256:
20382 : case IX86_BUILTIN_MAXPD256:
20383 : case IX86_BUILTIN_MAXPS512:
20384 : case IX86_BUILTIN_MAXPD512:
20385 : case IX86_BUILTIN_MAXPS128_MASK:
20386 : case IX86_BUILTIN_MAXPD128_MASK:
20387 : case IX86_BUILTIN_MAXPS256_MASK:
20388 : case IX86_BUILTIN_MAXPD256_MASK:
20389 : case IX86_BUILTIN_MAXPH128_MASK:
20390 : case IX86_BUILTIN_MAXPH256_MASK:
20391 : case IX86_BUILTIN_MAXPH512_MASK:
20392 : tcode = GT_EXPR;
20393 4435 : do_minmax:
20394 4435 : gcc_assert (n_args >= 2);
20395 : /* Without SSE4.1 we often aren't able to pattern match it back to the
20396 : desired instruction. */
20397 4435 : if (!gimple_call_lhs (stmt) || !optimize || !TARGET_SSE4_1)
20398 : break;
20399 3865 : arg0 = gimple_call_arg (stmt, 0);
20400 3865 : arg1 = gimple_call_arg (stmt, 1);
20401 3865 : elems = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
20402 : /* For masked minmax, only optimize if the mask is all ones. */
20403 3865 : if (n_args > 2
20404 3865 : && !ix86_masked_all_ones (elems, gimple_call_arg (stmt, 3)))
20405 : break;
20406 647 : if (n_args >= 5)
20407 : {
20408 436 : tree arg4 = gimple_call_arg (stmt, 4);
20409 436 : if (!tree_fits_uhwi_p (arg4))
20410 : break;
20411 424 : if (tree_to_uhwi (arg4) == 4)
20412 : /* Ok. */;
20413 416 : else if (tree_to_uhwi (arg4) != 8)
20414 : /* Invalid round argument. */
20415 : break;
20416 416 : else if (HONOR_NANS (arg0))
20417 : /* Lowering to comparison would raise exceptions which
20418 : shouldn't be raised. */
20419 : break;
20420 : }
20421 219 : {
20422 219 : tree type = truth_type_for (TREE_TYPE (arg0));
20423 219 : tree cmpres = gimple_build (&stmts, tcode, type, arg0, arg1);
20424 219 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
20425 219 : g = gimple_build_assign (gimple_call_lhs (stmt),
20426 : VEC_COND_EXPR, cmpres, arg0, arg1);
20427 219 : gsi_replace (gsi, g, false);
20428 : }
20429 219 : return true;
20430 :
20431 : default:
20432 : break;
20433 : }
20434 :
20435 : return false;
20436 : }
20437 :
20438 : /* Handler for an SVML-style interface to
20439 : a library with vectorized intrinsics. */
20440 :
20441 : tree
20442 10 : ix86_veclibabi_svml (combined_fn fn, tree type_out, tree type_in)
20443 : {
20444 10 : char name[20];
20445 10 : tree fntype, new_fndecl, args;
20446 10 : unsigned arity;
20447 10 : const char *bname;
20448 10 : machine_mode el_mode, in_mode;
20449 10 : int n, in_n;
20450 :
20451 : /* The SVML is suitable for unsafe math only. */
20452 10 : if (!flag_unsafe_math_optimizations)
20453 : return NULL_TREE;
20454 :
20455 10 : el_mode = TYPE_MODE (TREE_TYPE (type_out));
20456 10 : n = TYPE_VECTOR_SUBPARTS (type_out);
20457 10 : in_mode = TYPE_MODE (TREE_TYPE (type_in));
20458 10 : in_n = TYPE_VECTOR_SUBPARTS (type_in);
20459 10 : if (el_mode != in_mode
20460 10 : || n != in_n)
20461 : return NULL_TREE;
20462 :
20463 10 : switch (fn)
20464 : {
20465 10 : CASE_CFN_EXP:
20466 10 : CASE_CFN_LOG:
20467 10 : CASE_CFN_LOG10:
20468 10 : CASE_CFN_POW:
20469 10 : CASE_CFN_TANH:
20470 10 : CASE_CFN_TAN:
20471 10 : CASE_CFN_ATAN:
20472 10 : CASE_CFN_ATAN2:
20473 10 : CASE_CFN_ATANH:
20474 10 : CASE_CFN_CBRT:
20475 10 : CASE_CFN_SINH:
20476 10 : CASE_CFN_SIN:
20477 10 : CASE_CFN_ASINH:
20478 10 : CASE_CFN_ASIN:
20479 10 : CASE_CFN_COSH:
20480 10 : CASE_CFN_COS:
20481 10 : CASE_CFN_ACOSH:
20482 10 : CASE_CFN_ACOS:
20483 10 : if ((el_mode != DFmode || n != 2)
20484 8 : && (el_mode != SFmode || n != 4))
20485 : return NULL_TREE;
20486 6 : break;
20487 :
20488 : default:
20489 : return NULL_TREE;
20490 : }
20491 :
20492 6 : tree fndecl = mathfn_built_in (el_mode == DFmode
20493 : ? double_type_node : float_type_node, fn);
20494 6 : bname = IDENTIFIER_POINTER (DECL_NAME (fndecl));
20495 :
20496 6 : if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_LOGF)
20497 2 : strcpy (name, "vmlsLn4");
20498 4 : else if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_LOG)
20499 0 : strcpy (name, "vmldLn2");
20500 4 : else if (n == 4)
20501 : {
20502 2 : sprintf (name, "vmls%s", bname+10);
20503 2 : name[strlen (name)-1] = '4';
20504 : }
20505 : else
20506 2 : sprintf (name, "vmld%s2", bname+10);
20507 :
20508 : /* Convert to uppercase. */
20509 6 : name[4] &= ~0x20;
20510 :
20511 6 : arity = 0;
20512 6 : for (args = DECL_ARGUMENTS (fndecl); args; args = TREE_CHAIN (args))
20513 0 : arity++;
20514 :
20515 6 : if (arity == 1)
20516 0 : fntype = build_function_type_list (type_out, type_in, NULL);
20517 : else
20518 6 : fntype = build_function_type_list (type_out, type_in, type_in, NULL);
20519 :
20520 : /* Build a function declaration for the vectorized function. */
20521 6 : new_fndecl = build_decl (BUILTINS_LOCATION,
20522 : FUNCTION_DECL, get_identifier (name), fntype);
20523 6 : TREE_PUBLIC (new_fndecl) = 1;
20524 6 : DECL_EXTERNAL (new_fndecl) = 1;
20525 6 : DECL_IS_NOVOPS (new_fndecl) = 1;
20526 6 : TREE_READONLY (new_fndecl) = 1;
20527 :
20528 6 : return new_fndecl;
20529 : }
20530 :
20531 : /* Handler for an ACML-style interface to
20532 : a library with vectorized intrinsics. */
20533 :
20534 : tree
20535 3 : ix86_veclibabi_acml (combined_fn fn, tree type_out, tree type_in)
20536 : {
20537 3 : char name[20] = "__vr.._";
20538 3 : tree fntype, new_fndecl, args;
20539 3 : unsigned arity;
20540 3 : const char *bname;
20541 3 : machine_mode el_mode, in_mode;
20542 3 : int n, in_n;
20543 :
20544 : /* The ACML is 64bits only and suitable for unsafe math only as
20545 : it does not correctly support parts of IEEE with the required
20546 : precision such as denormals. */
20547 3 : if (!TARGET_64BIT
20548 3 : || !flag_unsafe_math_optimizations)
20549 : return NULL_TREE;
20550 :
20551 3 : el_mode = TYPE_MODE (TREE_TYPE (type_out));
20552 3 : n = TYPE_VECTOR_SUBPARTS (type_out);
20553 3 : in_mode = TYPE_MODE (TREE_TYPE (type_in));
20554 3 : in_n = TYPE_VECTOR_SUBPARTS (type_in);
20555 3 : if (el_mode != in_mode
20556 3 : || n != in_n)
20557 : return NULL_TREE;
20558 :
20559 3 : switch (fn)
20560 : {
20561 3 : CASE_CFN_SIN:
20562 3 : CASE_CFN_COS:
20563 3 : CASE_CFN_EXP:
20564 3 : CASE_CFN_LOG:
20565 3 : CASE_CFN_LOG2:
20566 3 : CASE_CFN_LOG10:
20567 3 : if (el_mode == DFmode && n == 2)
20568 : {
20569 3 : name[4] = 'd';
20570 3 : name[5] = '2';
20571 : }
20572 0 : else if (el_mode == SFmode && n == 4)
20573 : {
20574 0 : name[4] = 's';
20575 0 : name[5] = '4';
20576 : }
20577 : else
20578 : return NULL_TREE;
20579 3 : break;
20580 :
20581 : default:
20582 : return NULL_TREE;
20583 : }
20584 :
20585 3 : tree fndecl = mathfn_built_in (el_mode == DFmode
20586 : ? double_type_node : float_type_node, fn);
20587 3 : bname = IDENTIFIER_POINTER (DECL_NAME (fndecl));
20588 3 : sprintf (name + 7, "%s", bname+10);
20589 :
20590 3 : arity = 0;
20591 3 : for (args = DECL_ARGUMENTS (fndecl); args; args = TREE_CHAIN (args))
20592 0 : arity++;
20593 :
20594 3 : if (arity == 1)
20595 0 : fntype = build_function_type_list (type_out, type_in, NULL);
20596 : else
20597 3 : fntype = build_function_type_list (type_out, type_in, type_in, NULL);
20598 :
20599 : /* Build a function declaration for the vectorized function. */
20600 3 : new_fndecl = build_decl (BUILTINS_LOCATION,
20601 : FUNCTION_DECL, get_identifier (name), fntype);
20602 3 : TREE_PUBLIC (new_fndecl) = 1;
20603 3 : DECL_EXTERNAL (new_fndecl) = 1;
20604 3 : DECL_IS_NOVOPS (new_fndecl) = 1;
20605 3 : TREE_READONLY (new_fndecl) = 1;
20606 :
20607 3 : return new_fndecl;
20608 : }
20609 :
20610 : /* Handler for an AOCL-LibM-style interface to
20611 : a library with vectorized intrinsics. */
20612 :
20613 : tree
20614 386 : ix86_veclibabi_aocl (combined_fn fn, tree type_out, tree type_in)
20615 : {
20616 386 : char name[20] = "amd_vr";
20617 386 : int name_len = 6;
20618 386 : tree fntype, new_fndecl, args;
20619 386 : unsigned arity;
20620 386 : const char *bname;
20621 386 : machine_mode el_mode, in_mode;
20622 386 : int n, in_n;
20623 :
20624 : /* AOCL-LibM is 64bits only. It is also only suitable for unsafe math only
20625 : as it trades off some accuracy for increased performance. */
20626 386 : if (!TARGET_64BIT
20627 386 : || !flag_unsafe_math_optimizations)
20628 : return NULL_TREE;
20629 :
20630 386 : el_mode = TYPE_MODE (TREE_TYPE (type_out));
20631 386 : n = TYPE_VECTOR_SUBPARTS (type_out);
20632 386 : in_mode = TYPE_MODE (TREE_TYPE (type_in));
20633 386 : in_n = TYPE_VECTOR_SUBPARTS (type_in);
20634 386 : if (el_mode != in_mode
20635 386 : || n != in_n)
20636 : return NULL_TREE;
20637 :
20638 386 : gcc_checking_assert (n > 0);
20639 :
20640 : /* Decide whether there exists a function for the combination of FN, the mode
20641 : and the vector width. Return early if it doesn't. */
20642 :
20643 386 : if (el_mode != DFmode && el_mode != SFmode)
20644 : return NULL_TREE;
20645 :
20646 : /* Supported vector widths for given FN and single/double precision. Zeros
20647 : are used to fill out unused positions in the arrays. */
20648 386 : static const int supported_n[][2][3] = {
20649 : /* Single prec. , Double prec. */
20650 : { { 16, 0, 0 }, { 2, 4, 8 } }, /* TAN. */
20651 : { { 4, 8, 16 }, { 2, 4, 8 } }, /* EXP. */
20652 : { { 4, 8, 16 }, { 2, 4, 8 } }, /* EXP2. */
20653 : { { 4, 8, 16 }, { 2, 4, 8 } }, /* LOG. */
20654 : { { 4, 8, 16 }, { 2, 4, 8 } }, /* LOG2. */
20655 : { { 4, 8, 16 }, { 2, 4, 8 } }, /* COS. */
20656 : { { 4, 8, 16 }, { 2, 4, 8 } }, /* SIN. */
20657 : { { 4, 8, 16 }, { 2, 4, 8 } }, /* POW. */
20658 : { { 4, 8, 16 }, { 2, 4, 8 } }, /* ERF. */
20659 : { { 4, 8, 16 }, { 2, 8, 0 } }, /* ATAN. */
20660 : { { 4, 8, 16 }, { 2, 0, 0 } }, /* LOG10. */
20661 : { { 4, 0, 0 }, { 2, 0, 0 } }, /* EXP10. */
20662 : { { 4, 0, 0 }, { 2, 0, 0 } }, /* LOG1P. */
20663 : { { 4, 8, 16 }, { 8, 0, 0 } }, /* ASIN. */
20664 : { { 4, 16, 0 }, { 0, 0, 0 } }, /* ACOS. */
20665 : { { 4, 8, 16 }, { 0, 0, 0 } }, /* TANH. */
20666 : { { 4, 0, 0 }, { 0, 0, 0 } }, /* EXPM1. */
20667 : { { 4, 8, 0 }, { 0, 0, 0 } }, /* COSH. */
20668 : };
20669 :
20670 : /* We cannot simply index the supported_n array with FN since multiple FNs
20671 : may correspond to a single operation (see the definitions of these
20672 : CASE_CFN_* macros). */
20673 386 : int i;
20674 386 : switch (fn)
20675 : {
20676 : CASE_CFN_TAN : i = 0; break;
20677 28 : CASE_CFN_EXP : i = 1; break;
20678 28 : CASE_CFN_EXP2 : i = 2; break;
20679 28 : CASE_CFN_LOG : i = 3; break;
20680 28 : CASE_CFN_LOG2 : i = 4; break;
20681 28 : CASE_CFN_COS : i = 5; break;
20682 28 : CASE_CFN_SIN : i = 6; break;
20683 28 : CASE_CFN_POW : i = 7; break;
20684 28 : CASE_CFN_ERF : i = 8; break;
20685 25 : CASE_CFN_ATAN : i = 9; break;
20686 20 : CASE_CFN_LOG10 : i = 10; break;
20687 10 : CASE_CFN_EXP10 : i = 11; break;
20688 10 : CASE_CFN_LOG1P : i = 12; break;
20689 24 : CASE_CFN_ASIN : i = 13; break;
20690 14 : CASE_CFN_ACOS : i = 14; break;
20691 18 : CASE_CFN_TANH : i = 15; break;
20692 9 : CASE_CFN_EXPM1 : i = 16; break;
20693 14 : CASE_CFN_COSH : i = 17; break;
20694 : default: return NULL_TREE;
20695 : }
20696 :
20697 386 : int j = el_mode == DFmode;
20698 386 : bool n_is_supported = false;
20699 976 : for (unsigned k = 0; k < 3; k++)
20700 857 : if (supported_n[i][j][k] == n)
20701 : {
20702 : n_is_supported = true;
20703 : break;
20704 : }
20705 386 : if (!n_is_supported)
20706 : return NULL_TREE;
20707 :
20708 : /* Append the precision and the vector width to the function name we are
20709 : constructing. */
20710 267 : name[name_len++] = el_mode == DFmode ? 'd' : 's';
20711 267 : switch (n)
20712 : {
20713 214 : case 2:
20714 214 : case 4:
20715 214 : case 8:
20716 214 : name[name_len++] = '0' + n;
20717 214 : break;
20718 53 : case 16:
20719 53 : name[name_len++] = '1';
20720 53 : name[name_len++] = '6';
20721 53 : break;
20722 0 : default:
20723 0 : gcc_unreachable ();
20724 : }
20725 267 : name[name_len++] = '_';
20726 :
20727 : /* Append the operation name (steal it from the name of a builtin). */
20728 267 : tree fndecl = mathfn_built_in (el_mode == DFmode
20729 : ? double_type_node : float_type_node, fn);
20730 267 : bname = IDENTIFIER_POINTER (DECL_NAME (fndecl));
20731 267 : sprintf (name + name_len, "%s", bname + 10);
20732 :
20733 267 : arity = 0;
20734 267 : for (args = DECL_ARGUMENTS (fndecl); args; args = TREE_CHAIN (args))
20735 0 : arity++;
20736 :
20737 267 : if (arity == 1)
20738 0 : fntype = build_function_type_list (type_out, type_in, NULL);
20739 : else
20740 267 : fntype = build_function_type_list (type_out, type_in, type_in, NULL);
20741 :
20742 : /* Build a function declaration for the vectorized function. */
20743 267 : new_fndecl = build_decl (BUILTINS_LOCATION,
20744 : FUNCTION_DECL, get_identifier (name), fntype);
20745 267 : TREE_PUBLIC (new_fndecl) = 1;
20746 267 : DECL_EXTERNAL (new_fndecl) = 1;
20747 267 : TREE_READONLY (new_fndecl) = 1;
20748 :
20749 267 : return new_fndecl;
20750 : }
20751 :
20752 : /* Returns a decl of a function that implements scatter store with
20753 : register type VECTYPE and index type INDEX_TYPE and SCALE.
20754 : Return NULL_TREE if it is not available. */
20755 :
20756 : static tree
20757 142198 : ix86_vectorize_builtin_scatter (const_tree vectype,
20758 : const_tree index_type, int scale)
20759 : {
20760 142198 : bool si;
20761 142198 : enum ix86_builtins code;
20762 :
20763 142198 : if (!TARGET_AVX512F)
20764 : return NULL_TREE;
20765 :
20766 4222 : if (known_eq (TYPE_VECTOR_SUBPARTS (vectype), 2u)
20767 7425 : ? !TARGET_USE_SCATTER_2PARTS
20768 7425 : : (known_eq (TYPE_VECTOR_SUBPARTS (vectype), 4u)
20769 3203 : ? !TARGET_USE_SCATTER_4PARTS
20770 2076 : : !TARGET_USE_SCATTER_8PARTS))
20771 : return NULL_TREE;
20772 :
20773 4222 : if ((TREE_CODE (index_type) != INTEGER_TYPE
20774 465 : && !POINTER_TYPE_P (index_type))
20775 4687 : || (TYPE_MODE (index_type) != SImode
20776 1779 : && TYPE_MODE (index_type) != DImode))
20777 : return NULL_TREE;
20778 :
20779 4452 : if (TYPE_PRECISION (index_type) > POINTER_SIZE)
20780 : return NULL_TREE;
20781 :
20782 : /* v*scatter* insn sign extends index to pointer mode. */
20783 4222 : if (TYPE_PRECISION (index_type) < POINTER_SIZE
20784 4222 : && TYPE_UNSIGNED (index_type))
20785 : return NULL_TREE;
20786 :
20787 : /* Scale can be 1, 2, 4 or 8. */
20788 4222 : if (scale <= 0
20789 4222 : || scale > 8
20790 4206 : || (scale & (scale - 1)) != 0)
20791 : return NULL_TREE;
20792 :
20793 4206 : si = TYPE_MODE (index_type) == SImode;
20794 4206 : switch (TYPE_MODE (vectype))
20795 : {
20796 170 : case E_V8DFmode:
20797 170 : code = si ? IX86_BUILTIN_SCATTERALTSIV8DF : IX86_BUILTIN_SCATTERDIV8DF;
20798 : break;
20799 104 : case E_V8DImode:
20800 104 : code = si ? IX86_BUILTIN_SCATTERALTSIV8DI : IX86_BUILTIN_SCATTERDIV8DI;
20801 : break;
20802 177 : case E_V16SFmode:
20803 177 : code = si ? IX86_BUILTIN_SCATTERSIV16SF : IX86_BUILTIN_SCATTERALTDIV16SF;
20804 : break;
20805 266 : case E_V16SImode:
20806 266 : code = si ? IX86_BUILTIN_SCATTERSIV16SI : IX86_BUILTIN_SCATTERALTDIV16SI;
20807 : break;
20808 207 : case E_V4DFmode:
20809 207 : if (TARGET_AVX512VL)
20810 34 : code = si ? IX86_BUILTIN_SCATTERALTSIV4DF : IX86_BUILTIN_SCATTERDIV4DF;
20811 : else
20812 : return NULL_TREE;
20813 : break;
20814 142 : case E_V4DImode:
20815 142 : if (TARGET_AVX512VL)
20816 34 : code = si ? IX86_BUILTIN_SCATTERALTSIV4DI : IX86_BUILTIN_SCATTERDIV4DI;
20817 : else
20818 : return NULL_TREE;
20819 : break;
20820 248 : case E_V8SFmode:
20821 248 : if (TARGET_AVX512VL)
20822 40 : code = si ? IX86_BUILTIN_SCATTERSIV8SF : IX86_BUILTIN_SCATTERALTDIV8SF;
20823 : else
20824 : return NULL_TREE;
20825 : break;
20826 277 : case E_V8SImode:
20827 277 : if (TARGET_AVX512VL)
20828 82 : code = si ? IX86_BUILTIN_SCATTERSIV8SI : IX86_BUILTIN_SCATTERALTDIV8SI;
20829 : else
20830 : return NULL_TREE;
20831 : break;
20832 251 : case E_V2DFmode:
20833 251 : if (TARGET_AVX512VL)
20834 94 : code = si ? IX86_BUILTIN_SCATTERALTSIV2DF : IX86_BUILTIN_SCATTERDIV2DF;
20835 : else
20836 : return NULL_TREE;
20837 : break;
20838 196 : case E_V2DImode:
20839 196 : if (TARGET_AVX512VL)
20840 94 : code = si ? IX86_BUILTIN_SCATTERALTSIV2DI : IX86_BUILTIN_SCATTERDIV2DI;
20841 : else
20842 : return NULL_TREE;
20843 : break;
20844 301 : case E_V4SFmode:
20845 301 : if (TARGET_AVX512VL)
20846 96 : code = si ? IX86_BUILTIN_SCATTERSIV4SF : IX86_BUILTIN_SCATTERALTDIV4SF;
20847 : else
20848 : return NULL_TREE;
20849 : break;
20850 333 : case E_V4SImode:
20851 333 : if (TARGET_AVX512VL)
20852 138 : code = si ? IX86_BUILTIN_SCATTERSIV4SI : IX86_BUILTIN_SCATTERALTDIV4SI;
20853 : else
20854 : return NULL_TREE;
20855 : break;
20856 : default:
20857 : return NULL_TREE;
20858 : }
20859 :
20860 1329 : return get_ix86_builtin (code);
20861 : }
20862 :
20863 : /* Return true if it is safe to use the rsqrt optabs to optimize
20864 : 1.0/sqrt. */
20865 :
20866 : static bool
20867 66 : use_rsqrt_p (machine_mode mode)
20868 : {
20869 66 : return ((mode == HFmode
20870 42 : || (TARGET_SSE && TARGET_SSE_MATH))
20871 66 : && flag_finite_math_only
20872 65 : && !flag_trapping_math
20873 119 : && flag_unsafe_math_optimizations);
20874 : }
20875 :
20876 : /* Helper for avx_vpermilps256_operand et al. This is also used by
20877 : the expansion functions to turn the parallel back into a mask.
20878 : The return value is 0 for no match and the imm8+1 for a match. */
20879 :
20880 : int
20881 65476 : avx_vpermilp_parallel (rtx par, machine_mode mode)
20882 : {
20883 65476 : unsigned i, nelt = GET_MODE_NUNITS (mode);
20884 65476 : unsigned mask = 0;
20885 65476 : unsigned char ipar[16] = {}; /* Silence -Wuninitialized warning. */
20886 :
20887 65476 : if (XVECLEN (par, 0) != (int) nelt)
20888 : return 0;
20889 :
20890 : /* Validate that all of the elements are constants, and not totally
20891 : out of range. Copy the data into an integral array to make the
20892 : subsequent checks easier. */
20893 316886 : for (i = 0; i < nelt; ++i)
20894 : {
20895 251410 : rtx er = XVECEXP (par, 0, i);
20896 251410 : unsigned HOST_WIDE_INT ei;
20897 :
20898 251410 : if (!CONST_INT_P (er))
20899 : return 0;
20900 251410 : ei = INTVAL (er);
20901 251410 : if (ei >= nelt)
20902 : return 0;
20903 251410 : ipar[i] = ei;
20904 : }
20905 :
20906 65476 : switch (mode)
20907 : {
20908 : case E_V8DFmode:
20909 : case E_V8DImode:
20910 : /* In the 512-bit DFmode case, we can only move elements within
20911 : a 128-bit lane. First fill the second part of the mask,
20912 : then fallthru. */
20913 4735 : for (i = 4; i < 6; ++i)
20914 : {
20915 3287 : if (!IN_RANGE (ipar[i], 4, 5))
20916 : return 0;
20917 3062 : mask |= (ipar[i] - 4) << i;
20918 : }
20919 3492 : for (i = 6; i < 8; ++i)
20920 : {
20921 2470 : if (!IN_RANGE (ipar[i], 6, 7))
20922 : return 0;
20923 2044 : mask |= (ipar[i] - 6) << i;
20924 : }
20925 : /* FALLTHRU */
20926 :
20927 20303 : case E_V4DFmode:
20928 20303 : case E_V4DImode:
20929 : /* In the 256-bit DFmode case, we can only move elements within
20930 : a 128-bit lane. */
20931 46224 : for (i = 0; i < 2; ++i)
20932 : {
20933 38851 : if (!IN_RANGE (ipar[i], 0, 1))
20934 : return 0;
20935 25921 : mask |= ipar[i] << i;
20936 : }
20937 19421 : for (i = 2; i < 4; ++i)
20938 : {
20939 13402 : if (!IN_RANGE (ipar[i], 2, 3))
20940 : return 0;
20941 12048 : mask |= (ipar[i] - 2) << i;
20942 : }
20943 : break;
20944 :
20945 : case E_V16SFmode:
20946 : case E_V16SImode:
20947 : /* In 512 bit SFmode case, permutation in the upper 256 bits
20948 : must mirror the permutation in the lower 256-bits. */
20949 3652 : for (i = 0; i < 8; ++i)
20950 3256 : if (ipar[i] + 8 != ipar[i + 8])
20951 : return 0;
20952 : /* FALLTHRU */
20953 :
20954 : case E_V8SFmode:
20955 : case E_V8SImode:
20956 : /* In 256 bit SFmode case, we have full freedom of
20957 : movement within the low 128-bit lane, but the high 128-bit
20958 : lane must mirror the exact same pattern. */
20959 34080 : for (i = 0; i < 4; ++i)
20960 28881 : if (ipar[i] + 4 != ipar[i + 4])
20961 : return 0;
20962 : nelt = 4;
20963 : /* FALLTHRU */
20964 :
20965 39881 : case E_V2DFmode:
20966 39881 : case E_V2DImode:
20967 39881 : case E_V4SFmode:
20968 39881 : case E_V4SImode:
20969 : /* In the 128-bit case, we've full freedom in the placement of
20970 : the elements from the source operand. */
20971 139003 : for (i = 0; i < nelt; ++i)
20972 99122 : mask |= ipar[i] << (i * (nelt / 2));
20973 : break;
20974 :
20975 0 : default:
20976 0 : gcc_unreachable ();
20977 : }
20978 :
20979 : /* Make sure success has a non-zero value by adding one. */
20980 45900 : return mask + 1;
20981 : }
20982 :
20983 : /* Helper for avx_vperm2f128_v4df_operand et al. This is also used by
20984 : the expansion functions to turn the parallel back into a mask.
20985 : The return value is 0 for no match and the imm8+1 for a match. */
20986 :
20987 : int
20988 42800 : avx_vperm2f128_parallel (rtx par, machine_mode mode)
20989 : {
20990 42800 : unsigned i, nelt = GET_MODE_NUNITS (mode), nelt2 = nelt / 2;
20991 42800 : unsigned mask = 0;
20992 42800 : unsigned char ipar[8] = {}; /* Silence -Wuninitialized warning. */
20993 :
20994 42800 : if (XVECLEN (par, 0) != (int) nelt)
20995 : return 0;
20996 :
20997 : /* Validate that all of the elements are constants, and not totally
20998 : out of range. Copy the data into an integral array to make the
20999 : subsequent checks easier. */
21000 344520 : for (i = 0; i < nelt; ++i)
21001 : {
21002 301720 : rtx er = XVECEXP (par, 0, i);
21003 301720 : unsigned HOST_WIDE_INT ei;
21004 :
21005 301720 : if (!CONST_INT_P (er))
21006 : return 0;
21007 301720 : ei = INTVAL (er);
21008 301720 : if (ei >= 2 * nelt)
21009 : return 0;
21010 301720 : ipar[i] = ei;
21011 : }
21012 :
21013 : /* Validate that the halves of the permute are halves. */
21014 82747 : for (i = 0; i < nelt2 - 1; ++i)
21015 66596 : if (ipar[i] + 1 != ipar[i + 1])
21016 : return 0;
21017 51160 : for (i = nelt2; i < nelt - 1; ++i)
21018 35661 : if (ipar[i] + 1 != ipar[i + 1])
21019 : return 0;
21020 :
21021 : /* Reconstruct the mask. */
21022 46401 : for (i = 0; i < 2; ++i)
21023 : {
21024 30952 : unsigned e = ipar[i * nelt2];
21025 30952 : if (e % nelt2)
21026 : return 0;
21027 30902 : e /= nelt2;
21028 30902 : mask |= e << (i * 4);
21029 : }
21030 :
21031 : /* Make sure success has a non-zero value by adding one. */
21032 15449 : return mask + 1;
21033 : }
21034 :
21035 : /* Return a mask of VPTERNLOG operands that do not affect output. */
21036 :
21037 : int
21038 2434 : vpternlog_redundant_operand_mask (rtx pternlog_imm)
21039 : {
21040 2434 : int mask = 0;
21041 2434 : int imm8 = INTVAL (pternlog_imm);
21042 :
21043 2434 : if (((imm8 >> 4) & 0x0F) == (imm8 & 0x0F))
21044 6 : mask |= 1;
21045 2434 : if (((imm8 >> 2) & 0x33) == (imm8 & 0x33))
21046 6 : mask |= 2;
21047 2434 : if (((imm8 >> 1) & 0x55) == (imm8 & 0x55))
21048 151 : mask |= 4;
21049 :
21050 2434 : return mask;
21051 : }
21052 :
21053 : /* Eliminate false dependencies on operands that do not affect output
21054 : by substituting other operands of a VPTERNLOG. */
21055 :
21056 : void
21057 79 : substitute_vpternlog_operands (rtx *operands)
21058 : {
21059 79 : int mask = vpternlog_redundant_operand_mask (operands[4]);
21060 :
21061 79 : if (mask & 1) /* The first operand is redundant. */
21062 2 : operands[1] = operands[2];
21063 :
21064 79 : if (mask & 2) /* The second operand is redundant. */
21065 2 : operands[2] = operands[1];
21066 :
21067 79 : if (mask & 4) /* The third operand is redundant. */
21068 75 : operands[3] = operands[1];
21069 4 : else if (REG_P (operands[3]))
21070 : {
21071 0 : if (mask & 1)
21072 0 : operands[1] = operands[3];
21073 0 : if (mask & 2)
21074 0 : operands[2] = operands[3];
21075 : }
21076 79 : }
21077 :
21078 : /* Return a register priority for hard reg REGNO. */
21079 : static int
21080 60199083 : ix86_register_priority (int hard_regno)
21081 : {
21082 : /* ebp and r13 as the base always wants a displacement, r12 as the
21083 : base always wants an index. So discourage their usage in an
21084 : address. */
21085 60199083 : if (hard_regno == R12_REG || hard_regno == R13_REG)
21086 : return 0;
21087 55670076 : if (hard_regno == BP_REG)
21088 : return 1;
21089 : /* New x86-64 int registers result in bigger code size. Discourage them. */
21090 53680501 : if (REX_INT_REGNO_P (hard_regno))
21091 : return 2;
21092 36387440 : if (REX2_INT_REGNO_P (hard_regno))
21093 : return 2;
21094 : /* New x86-64 SSE registers result in bigger code size. Discourage them. */
21095 36384918 : if (REX_SSE_REGNO_P (hard_regno))
21096 : return 2;
21097 30330180 : if (EXT_REX_SSE_REGNO_P (hard_regno))
21098 : return 1;
21099 : /* Usage of AX register results in smaller code. Prefer it. */
21100 30050779 : if (hard_regno == AX_REG)
21101 4076297 : return 4;
21102 : return 3;
21103 : }
21104 :
21105 : /* Implement TARGET_PREFERRED_RELOAD_CLASS.
21106 :
21107 : Put float CONST_DOUBLE in the constant pool instead of fp regs.
21108 : QImode must go into class Q_REGS.
21109 : Narrow ALL_REGS to GENERAL_REGS. This supports allowing movsf and
21110 : movdf to do mem-to-mem moves through integer regs. */
21111 :
21112 : static reg_class_t
21113 565768905 : ix86_preferred_reload_class (rtx x, reg_class_t regclass)
21114 : {
21115 565768905 : machine_mode mode = GET_MODE (x);
21116 :
21117 : /* We're only allowed to return a subclass of CLASS. Many of the
21118 : following checks fail for NO_REGS, so eliminate that early. */
21119 565768905 : if (regclass == NO_REGS)
21120 : return NO_REGS;
21121 :
21122 : /* All classes can load zeros. */
21123 564858146 : if (x == CONST0_RTX (mode))
21124 : return regclass;
21125 :
21126 : /* Force constants into memory if we are loading a non-zero constant
21127 : into an MMX, SSE or MASK register. This is because there are no
21128 : MMX/SSE/MASK instructions to load from a constant. Exceptions are
21129 : minus ones for MASK register and standard SSE constants for SSE
21130 : register. */
21131 539194277 : if (CONSTANT_P (x))
21132 : {
21133 157973174 : if (MAYBE_MMX_CLASS_P (regclass))
21134 : return NO_REGS;
21135 157849107 : if (MAYBE_MASK_CLASS_P (regclass))
21136 18947 : return x == constm1_rtx ? regclass : NO_REGS;
21137 157830160 : if (MAYBE_SSE_CLASS_P (regclass))
21138 13310360 : return (mode != VOIDmode && standard_sse_constant_p (x, mode)
21139 31354146 : ? regclass : NO_REGS);
21140 : }
21141 :
21142 : /* Floating-point constants need more complex checks. */
21143 507697117 : if (CONST_DOUBLE_P (x))
21144 : {
21145 : /* General regs can load everything. */
21146 313858 : if (INTEGER_CLASS_P (regclass))
21147 : return regclass;
21148 :
21149 : /* Floats can load 0 and 1 plus some others. Note that we eliminated
21150 : zero above. We only want to wind up preferring 80387 registers if
21151 : we plan on doing computation with them. */
21152 184204 : if (IS_STACK_MODE (mode)
21153 242707 : && standard_80387_constant_p (x) > 0)
21154 : {
21155 : /* Limit class to FP regs. */
21156 40511 : if (FLOAT_CLASS_P (regclass))
21157 : return FLOAT_REGS;
21158 : }
21159 :
21160 143693 : return NO_REGS;
21161 : }
21162 :
21163 : /* Prefer SSE if we can use them for math. Also allow integer regs
21164 : when moves between register units are cheap. */
21165 507383259 : if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
21166 : {
21167 31518560 : if (TARGET_INTER_UNIT_MOVES_FROM_VEC
21168 31505077 : && TARGET_INTER_UNIT_MOVES_TO_VEC
21169 94519486 : && GET_MODE_SIZE (mode) <= GET_MODE_SIZE (word_mode))
21170 31363162 : return INT_SSE_CLASS_P (regclass) ? regclass : NO_REGS;
21171 : else
21172 155398 : return SSE_CLASS_P (regclass) ? regclass : NO_REGS;
21173 : }
21174 :
21175 : /* Generally when we see PLUS here, it's the function invariant
21176 : (plus soft-fp const_int). Which can only be computed into general
21177 : regs. */
21178 475864699 : if (GET_CODE (x) == PLUS)
21179 2121144 : return INTEGER_CLASS_P (regclass) ? regclass : NO_REGS;
21180 :
21181 : /* QImode constants are easy to load, but non-constant QImode data
21182 : must go into Q_REGS or ALL_MASK_REGS. */
21183 473743555 : if (GET_MODE (x) == QImode && !CONSTANT_P (x))
21184 : {
21185 25463699 : if (Q_CLASS_P (regclass))
21186 : return regclass;
21187 20583853 : else if (reg_class_subset_p (Q_REGS, regclass))
21188 : return Q_REGS;
21189 55038 : else if (MASK_CLASS_P (regclass))
21190 : return regclass;
21191 : else
21192 : return NO_REGS;
21193 : }
21194 :
21195 : return regclass;
21196 : }
21197 :
21198 : /* Discourage putting floating-point values in SSE registers unless
21199 : SSE math is being used, and likewise for the 387 registers. */
21200 : static reg_class_t
21201 76784406 : ix86_preferred_output_reload_class (rtx x, reg_class_t regclass)
21202 : {
21203 : /* Restrict the output reload class to the register bank that we are doing
21204 : math on. If we would like not to return a subset of CLASS, reject this
21205 : alternative: if reload cannot do this, it will still use its choice. */
21206 76784406 : machine_mode mode = GET_MODE (x);
21207 76784406 : if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
21208 7357099 : return MAYBE_SSE_CLASS_P (regclass) ? ALL_SSE_REGS : NO_REGS;
21209 :
21210 69427307 : if (IS_STACK_MODE (mode))
21211 211578 : return FLOAT_CLASS_P (regclass) ? regclass : NO_REGS;
21212 :
21213 : return regclass;
21214 : }
21215 :
21216 : static reg_class_t
21217 399077844 : ix86_secondary_reload (bool in_p, rtx x, reg_class_t rclass,
21218 : machine_mode mode, secondary_reload_info *sri)
21219 : {
21220 : /* Double-word spills from general registers to non-offsettable memory
21221 : references (zero-extended addresses) require special handling. */
21222 399077844 : if (TARGET_64BIT
21223 345994093 : && MEM_P (x)
21224 187313656 : && GET_MODE_SIZE (mode) > UNITS_PER_WORD
21225 19329181 : && INTEGER_CLASS_P (rclass)
21226 401796204 : && !offsettable_memref_p (x))
21227 : {
21228 2467678 : sri->icode = (in_p
21229 1233839 : ? CODE_FOR_reload_noff_load
21230 : : CODE_FOR_reload_noff_store);
21231 : /* Add the cost of moving address to a temporary. */
21232 1233839 : sri->extra_cost = 1;
21233 :
21234 1233839 : return NO_REGS;
21235 : }
21236 :
21237 : /* QImode spills from non-QI registers require
21238 : intermediate register on 32bit targets. */
21239 397844005 : if (mode == QImode
21240 397844005 : && ((!TARGET_64BIT && !in_p
21241 589653 : && INTEGER_CLASS_P (rclass)
21242 589605 : && MAYBE_NON_Q_CLASS_P (rclass))
21243 23122987 : || (!TARGET_AVX512DQ
21244 22908963 : && MAYBE_MASK_CLASS_P (rclass))))
21245 : {
21246 6640 : int regno = true_regnum (x);
21247 :
21248 : /* Return Q_REGS if the operand is in memory. */
21249 6640 : if (regno == -1)
21250 : return Q_REGS;
21251 :
21252 5142 : return NO_REGS;
21253 : }
21254 :
21255 : /* Require movement to gpr, and then store to memory. */
21256 397837365 : if ((mode == HFmode || mode == HImode || mode == V2QImode
21257 : || mode == BFmode)
21258 4210891 : && !TARGET_SSE4_1
21259 3616138 : && SSE_CLASS_P (rclass)
21260 294887 : && !in_p && MEM_P (x))
21261 : {
21262 178844 : sri->extra_cost = 1;
21263 178844 : return GENERAL_REGS;
21264 : }
21265 :
21266 : /* This condition handles corner case where an expression involving
21267 : pointers gets vectorized. We're trying to use the address of a
21268 : stack slot as a vector initializer.
21269 :
21270 : (set (reg:V2DI 74 [ vect_cst_.2 ])
21271 : (vec_duplicate:V2DI (reg/f:DI 20 frame)))
21272 :
21273 : Eventually frame gets turned into sp+offset like this:
21274 :
21275 : (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
21276 : (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
21277 : (const_int 392 [0x188]))))
21278 :
21279 : That later gets turned into:
21280 :
21281 : (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
21282 : (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
21283 : (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))))
21284 :
21285 : We'll have the following reload recorded:
21286 :
21287 : Reload 0: reload_in (DI) =
21288 : (plus:DI (reg/f:DI 7 sp)
21289 : (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))
21290 : reload_out (V2DI) = (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
21291 : SSE_REGS, RELOAD_OTHER (opnum = 0), can't combine
21292 : reload_in_reg: (plus:DI (reg/f:DI 7 sp) (const_int 392 [0x188]))
21293 : reload_out_reg: (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
21294 : reload_reg_rtx: (reg:V2DI 22 xmm1)
21295 :
21296 : Which isn't going to work since SSE instructions can't handle scalar
21297 : additions. Returning GENERAL_REGS forces the addition into integer
21298 : register and reload can handle subsequent reloads without problems. */
21299 :
21300 229813810 : if (in_p && GET_CODE (x) == PLUS
21301 2 : && SSE_CLASS_P (rclass)
21302 397658521 : && SCALAR_INT_MODE_P (mode))
21303 0 : return GENERAL_REGS;
21304 :
21305 : return NO_REGS;
21306 : }
21307 :
21308 : /* Implement TARGET_CLASS_LIKELY_SPILLED_P. */
21309 :
21310 : static bool
21311 740578787 : ix86_class_likely_spilled_p (reg_class_t rclass)
21312 : {
21313 729606327 : switch (rclass)
21314 : {
21315 : case AREG:
21316 : case DREG:
21317 : case CREG:
21318 : case BREG:
21319 : case AD_REGS:
21320 : case SIREG:
21321 : case DIREG:
21322 : case SSE_FIRST_REG:
21323 : case FP_TOP_REG:
21324 : case FP_SECOND_REG:
21325 : return true;
21326 :
21327 706408455 : default:
21328 706408455 : break;
21329 : }
21330 :
21331 706408455 : return false;
21332 : }
21333 :
21334 : /* Implement TARGET_CALLEE_SAVE_COST. */
21335 :
21336 : static int
21337 83264120 : ix86_callee_save_cost (spill_cost_type, unsigned int hard_regno, machine_mode,
21338 : unsigned int, int mem_cost, const HARD_REG_SET &, bool)
21339 : {
21340 : /* Account for the fact that push and pop are shorter and do their
21341 : own allocation and deallocation. */
21342 83264120 : if (GENERAL_REGNO_P (hard_regno))
21343 : {
21344 : /* push is 1 byte while typical spill is 4-5 bytes.
21345 : ??? We probably should adjust size costs accordingly.
21346 : Costs are relative to reg-reg move that has 2 bytes for 32bit
21347 : and 3 bytes otherwise. Be sure that no cost table sets cost
21348 : to 2, so we end up with 0. */
21349 83253930 : if (mem_cost <= 2 || optimize_function_for_size_p (cfun))
21350 : return 1;
21351 79640634 : return mem_cost - 2;
21352 : }
21353 : return mem_cost;
21354 : }
21355 :
21356 : /* Return true if a set of DST by the expression SRC should be allowed.
21357 : This prevents complex sets of likely_spilled hard regs before split1. */
21358 :
21359 : bool
21360 626794732 : ix86_hardreg_mov_ok (rtx dst, rtx src)
21361 : {
21362 : /* Avoid complex sets of likely_spilled hard registers before reload. */
21363 519630584 : if (REG_P (dst) && HARD_REGISTER_P (dst)
21364 302690397 : && !REG_P (src) && !MEM_P (src)
21365 99223874 : && !(VECTOR_MODE_P (GET_MODE (dst))
21366 99223874 : ? standard_sse_constant_p (src, GET_MODE (dst))
21367 49369501 : : x86_64_immediate_operand (src, GET_MODE (dst)))
21368 10972460 : && ix86_class_likely_spilled_p (REGNO_REG_CLASS (REGNO (dst)))
21369 636471026 : && ix86_pre_reload_split ())
21370 3306889 : return false;
21371 : return true;
21372 : }
21373 :
21374 : /* If we are copying between registers from different register sets
21375 : (e.g. FP and integer), we may need a memory location.
21376 :
21377 : The function can't work reliably when one of the CLASSES is a class
21378 : containing registers from multiple sets. We avoid this by never combining
21379 : different sets in a single alternative in the machine description.
21380 : Ensure that this constraint holds to avoid unexpected surprises.
21381 :
21382 : When STRICT is false, we are being called from REGISTER_MOVE_COST,
21383 : so do not enforce these sanity checks.
21384 :
21385 : To optimize register_move_cost performance, define inline variant. */
21386 :
21387 : static inline bool
21388 5786112263 : inline_secondary_memory_needed (machine_mode mode, reg_class_t class1,
21389 : reg_class_t class2, int strict)
21390 : {
21391 5786112263 : if (lra_in_progress && (class1 == NO_REGS || class2 == NO_REGS))
21392 : return false;
21393 :
21394 5753099311 : if (MAYBE_FLOAT_CLASS_P (class1) != FLOAT_CLASS_P (class1)
21395 4902818225 : || MAYBE_FLOAT_CLASS_P (class2) != FLOAT_CLASS_P (class2)
21396 4186920514 : || MAYBE_SSE_CLASS_P (class1) != SSE_CLASS_P (class1)
21397 3994798190 : || MAYBE_SSE_CLASS_P (class2) != SSE_CLASS_P (class2)
21398 3812982733 : || MAYBE_MMX_CLASS_P (class1) != MMX_CLASS_P (class1)
21399 3812982733 : || MAYBE_MMX_CLASS_P (class2) != MMX_CLASS_P (class2)
21400 3812982733 : || MAYBE_MASK_CLASS_P (class1) != MASK_CLASS_P (class1)
21401 9392825000 : || MAYBE_MASK_CLASS_P (class2) != MASK_CLASS_P (class2))
21402 : {
21403 2278467812 : gcc_assert (!strict || lra_in_progress);
21404 : return true;
21405 : }
21406 :
21407 3474631499 : if (FLOAT_CLASS_P (class1) != FLOAT_CLASS_P (class2))
21408 : return true;
21409 :
21410 : /* ??? This is a lie. We do have moves between mmx/general, and for
21411 : mmx/sse2. But by saying we need secondary memory we discourage the
21412 : register allocator from using the mmx registers unless needed. */
21413 3322762369 : if (MMX_CLASS_P (class1) != MMX_CLASS_P (class2))
21414 : return true;
21415 :
21416 : /* Between mask and general, we have moves no larger than word size. */
21417 3224596779 : if (MASK_CLASS_P (class1) != MASK_CLASS_P (class2))
21418 : {
21419 2642893 : if (!(INTEGER_CLASS_P (class1) || INTEGER_CLASS_P (class2))
21420 3456984 : || GET_MODE_SIZE (mode) > UNITS_PER_WORD)
21421 : return true;
21422 : }
21423 :
21424 3224401209 : if (SSE_CLASS_P (class1) != SSE_CLASS_P (class2))
21425 : {
21426 : /* SSE1 doesn't have any direct moves from other classes. */
21427 700156373 : if (!TARGET_SSE2)
21428 : return true;
21429 :
21430 698320309 : if (!(INTEGER_CLASS_P (class1) || INTEGER_CLASS_P (class2)))
21431 : return true;
21432 :
21433 : /* If the target says that inter-unit moves are more expensive
21434 : than moving through memory, then don't generate them. */
21435 1047157317 : if ((SSE_CLASS_P (class1) && !TARGET_INTER_UNIT_MOVES_FROM_VEC)
21436 1046665208 : || (SSE_CLASS_P (class2) && !TARGET_INTER_UNIT_MOVES_TO_VEC))
21437 : return true;
21438 :
21439 : /* With SSE4.1, *mov{ti,di}_internal supports moves between
21440 : SSE_REGS and GENERAL_REGS using pinsr{q,d} or pextr{q,d}. */
21441 696988595 : if (TARGET_SSE4_1
21442 37061515 : && (TARGET_64BIT ? mode == TImode : mode == DImode))
21443 : return false;
21444 :
21445 : /* Moves from SSE_REGS to GENERAL_REGS need only SSE2:
21446 : *movti_internal splits them into movq + shufpd + movq. */
21447 695382260 : if (TARGET_64BIT && mode == TImode && SSE_CLASS_P (class1))
21448 : return false;
21449 :
21450 681492340 : int msize = GET_MODE_SIZE (mode);
21451 :
21452 : /* Between SSE and general, we have moves no larger than word size. */
21453 697807510 : if (msize > UNITS_PER_WORD)
21454 : return true;
21455 :
21456 : /* In addition to SImode moves, HImode moves are supported for SSE2 and above,
21457 : Use vmovw with AVX512FP16, or pinsrw/pextrw without AVX512FP16. */
21458 601641180 : int minsize = GET_MODE_SIZE (TARGET_SSE2 ? HImode : SImode);
21459 :
21460 601641180 : if (msize < minsize)
21461 30 : return true;
21462 : }
21463 :
21464 : return false;
21465 : }
21466 :
21467 : /* Implement TARGET_SECONDARY_MEMORY_NEEDED. */
21468 :
21469 : static bool
21470 73192061 : ix86_secondary_memory_needed (machine_mode mode, reg_class_t class1,
21471 : reg_class_t class2)
21472 : {
21473 73192061 : return inline_secondary_memory_needed (mode, class1, class2, true);
21474 : }
21475 :
21476 : /* Implement TARGET_SECONDARY_MEMORY_NEEDED_MODE.
21477 :
21478 : get_secondary_mem widens integral modes to BITS_PER_WORD.
21479 : There is no need to emit full 64 bit move on 64 bit targets
21480 : for integral modes that can be moved using 32 bit move. */
21481 :
21482 : static machine_mode
21483 12742 : ix86_secondary_memory_needed_mode (machine_mode mode)
21484 : {
21485 25484 : if (GET_MODE_BITSIZE (mode) < 32 && INTEGRAL_MODE_P (mode))
21486 17 : return mode_for_size (32, GET_MODE_CLASS (mode), 0).require ();
21487 : return mode;
21488 : }
21489 :
21490 : /* Implement the TARGET_CLASS_MAX_NREGS hook.
21491 :
21492 : On the 80386, this is the size of MODE in words,
21493 : except in the FP regs, where a single reg is always enough. */
21494 :
21495 : static unsigned char
21496 5814599306 : ix86_class_max_nregs (reg_class_t rclass, machine_mode mode)
21497 : {
21498 5814599306 : if (MAYBE_INTEGER_CLASS_P (rclass))
21499 : {
21500 3921933688 : if (mode == XFmode)
21501 147420851 : return (TARGET_64BIT ? 2 : 3);
21502 3774512837 : else if (mode == XCmode)
21503 147420494 : return (TARGET_64BIT ? 4 : 6);
21504 : else
21505 7360433205 : return CEIL (GET_MODE_SIZE (mode), UNITS_PER_WORD);
21506 : }
21507 : else
21508 : {
21509 1892665618 : if (COMPLEX_MODE_P (mode))
21510 : return 2;
21511 : else
21512 1613139261 : return 1;
21513 : }
21514 : }
21515 :
21516 : /* Implement TARGET_CAN_CHANGE_MODE_CLASS. */
21517 :
21518 : static bool
21519 42082193 : ix86_can_change_mode_class (machine_mode from, machine_mode to,
21520 : reg_class_t regclass)
21521 : {
21522 42082193 : if (from == to)
21523 : return true;
21524 :
21525 : /* x87 registers can't do subreg at all, as all values are reformatted
21526 : to extended precision.
21527 :
21528 : ??? middle-end queries mode changes for ALL_REGS and this makes
21529 : vec_series_lowpart_p to always return false. We probably should
21530 : restrict this to modes supported by i387 and check if it is enabled. */
21531 40564046 : if (MAYBE_FLOAT_CLASS_P (regclass))
21532 : return false;
21533 :
21534 35399183 : if (MAYBE_SSE_CLASS_P (regclass) || MAYBE_MMX_CLASS_P (regclass))
21535 : {
21536 : /* Vector registers do not support QI or HImode loads. If we don't
21537 : disallow a change to these modes, reload will assume it's ok to
21538 : drop the subreg from (subreg:SI (reg:HI 100) 0). This affects
21539 : the vec_dupv4hi pattern.
21540 : NB: SSE2 can load 16bit data to sse register via pinsrw. */
21541 16581651 : int mov_size = MAYBE_SSE_CLASS_P (regclass) && TARGET_SSE2 ? 2 : 4;
21542 16581651 : if (GET_MODE_SIZE (from) < mov_size
21543 33163272 : || GET_MODE_SIZE (to) < mov_size)
21544 2090263 : return false;
21545 : }
21546 :
21547 : return true;
21548 : }
21549 :
21550 : /* Return index of MODE in the sse load/store tables. */
21551 :
21552 : static inline int
21553 773070146 : sse_store_index (machine_mode mode)
21554 : {
21555 : /* NB: Use SFmode cost for HFmode instead of adding HFmode load/store
21556 : costs to processor_costs, which requires changes to all entries in
21557 : processor cost table. */
21558 773070146 : if (mode == E_HFmode)
21559 139259684 : mode = E_SFmode;
21560 :
21561 1546140292 : switch (GET_MODE_SIZE (mode))
21562 : {
21563 : case 4:
21564 : return 0;
21565 : case 8:
21566 : return 1;
21567 : case 16:
21568 : return 2;
21569 : case 32:
21570 : return 3;
21571 : case 64:
21572 : return 4;
21573 : default:
21574 : return -1;
21575 : }
21576 : }
21577 :
21578 : /* Return the cost of moving data of mode M between a
21579 : register and memory. A value of 2 is the default; this cost is
21580 : relative to those in `REGISTER_MOVE_COST'.
21581 :
21582 : This function is used extensively by register_move_cost that is used to
21583 : build tables at startup. Make it inline in this case.
21584 : When IN is 2, return maximum of in and out move cost.
21585 :
21586 : If moving between registers and memory is more expensive than
21587 : between two registers, you should define this macro to express the
21588 : relative cost.
21589 :
21590 : Model also increased moving costs of QImode registers in non
21591 : Q_REGS classes.
21592 : */
21593 : static inline int
21594 7011510301 : inline_memory_move_cost (machine_mode mode, enum reg_class regclass, int in)
21595 : {
21596 7011510301 : int cost;
21597 :
21598 7011510301 : if (FLOAT_CLASS_P (regclass))
21599 : {
21600 359593417 : int index;
21601 359593417 : switch (mode)
21602 : {
21603 : case E_SFmode:
21604 : index = 0;
21605 : break;
21606 : case E_DFmode:
21607 : index = 1;
21608 : break;
21609 : case E_XFmode:
21610 : index = 2;
21611 : break;
21612 : default:
21613 : return 100;
21614 : }
21615 107491433 : if (in == 2)
21616 103476054 : return MAX (ix86_cost->hard_register.fp_load [index],
21617 : ix86_cost->hard_register.fp_store [index]);
21618 4015379 : return in ? ix86_cost->hard_register.fp_load [index]
21619 4015379 : : ix86_cost->hard_register.fp_store [index];
21620 : }
21621 6651916884 : if (SSE_CLASS_P (regclass))
21622 : {
21623 641787289 : int index = sse_store_index (mode);
21624 641787289 : if (index == -1)
21625 : return 100;
21626 556666507 : if (in == 2)
21627 390306241 : return MAX (ix86_cost->hard_register.sse_load [index],
21628 : ix86_cost->hard_register.sse_store [index]);
21629 166360266 : return in ? ix86_cost->hard_register.sse_load [index]
21630 166360266 : : ix86_cost->hard_register.sse_store [index];
21631 : }
21632 6010129595 : if (MASK_CLASS_P (regclass))
21633 : {
21634 110257063 : int index;
21635 220514126 : switch (GET_MODE_SIZE (mode))
21636 : {
21637 : case 1:
21638 : index = 0;
21639 : break;
21640 9094277 : case 2:
21641 9094277 : index = 1;
21642 9094277 : break;
21643 : /* DImode loads and stores assumed to cost the same as SImode. */
21644 40919830 : case 4:
21645 40919830 : case 8:
21646 40919830 : index = 2;
21647 40919830 : break;
21648 : default:
21649 : return 100;
21650 : }
21651 :
21652 53655975 : if (in == 2)
21653 592407 : return MAX (ix86_cost->hard_register.mask_load[index],
21654 : ix86_cost->hard_register.mask_store[index]);
21655 53063568 : return in ? ix86_cost->hard_register.mask_load[2]
21656 53063568 : : ix86_cost->hard_register.mask_store[2];
21657 : }
21658 5899872532 : if (MMX_CLASS_P (regclass))
21659 : {
21660 175569796 : int index;
21661 351139592 : switch (GET_MODE_SIZE (mode))
21662 : {
21663 : case 4:
21664 : index = 0;
21665 : break;
21666 103040740 : case 8:
21667 103040740 : index = 1;
21668 103040740 : break;
21669 : default:
21670 : return 100;
21671 : }
21672 141078508 : if (in == 2)
21673 120737470 : return MAX (ix86_cost->hard_register.mmx_load [index],
21674 : ix86_cost->hard_register.mmx_store [index]);
21675 20341038 : return in ? ix86_cost->hard_register.mmx_load [index]
21676 20341038 : : ix86_cost->hard_register.mmx_store [index];
21677 : }
21678 11448605472 : switch (GET_MODE_SIZE (mode))
21679 : {
21680 127039297 : case 1:
21681 127039297 : if (Q_CLASS_P (regclass) || TARGET_64BIT)
21682 : {
21683 124408574 : if (!in)
21684 19957224 : return ix86_cost->hard_register.int_store[0];
21685 104451350 : if (TARGET_PARTIAL_REG_DEPENDENCY
21686 104451350 : && optimize_function_for_speed_p (cfun))
21687 97423714 : cost = ix86_cost->hard_register.movzbl_load;
21688 : else
21689 7027636 : cost = ix86_cost->hard_register.int_load[0];
21690 104451350 : if (in == 2)
21691 84469500 : return MAX (cost, ix86_cost->hard_register.int_store[0]);
21692 : return cost;
21693 : }
21694 : else
21695 : {
21696 2630723 : if (in == 2)
21697 1863020 : return MAX (ix86_cost->hard_register.movzbl_load,
21698 : ix86_cost->hard_register.int_store[0] + 4);
21699 767703 : if (in)
21700 383911 : return ix86_cost->hard_register.movzbl_load;
21701 : else
21702 383792 : return ix86_cost->hard_register.int_store[0] + 4;
21703 : }
21704 654925068 : break;
21705 654925068 : case 2:
21706 654925068 : {
21707 654925068 : int cost;
21708 654925068 : if (in == 2)
21709 553199180 : cost = MAX (ix86_cost->hard_register.int_load[1],
21710 : ix86_cost->hard_register.int_store[1]);
21711 : else
21712 101725888 : cost = in ? ix86_cost->hard_register.int_load[1]
21713 : : ix86_cost->hard_register.int_store[1];
21714 :
21715 654925068 : if (mode == E_HFmode)
21716 : {
21717 : /* Prefer SSE over GPR for HFmode. */
21718 126861192 : int sse_cost;
21719 126861192 : int index = sse_store_index (mode);
21720 126861192 : if (in == 2)
21721 116690586 : sse_cost = MAX (ix86_cost->hard_register.sse_load[index],
21722 : ix86_cost->hard_register.sse_store[index]);
21723 : else
21724 20341212 : sse_cost = (in
21725 10170606 : ? ix86_cost->hard_register.sse_load [index]
21726 : : ix86_cost->hard_register.sse_store [index]);
21727 126861192 : if (sse_cost >= cost)
21728 126861192 : cost = sse_cost + 1;
21729 : }
21730 : return cost;
21731 : }
21732 4942338371 : default:
21733 4942338371 : if (in == 2)
21734 3823208696 : cost = MAX (ix86_cost->hard_register.int_load[2],
21735 : ix86_cost->hard_register.int_store[2]);
21736 1119129675 : else if (in)
21737 559751735 : cost = ix86_cost->hard_register.int_load[2];
21738 : else
21739 559377940 : cost = ix86_cost->hard_register.int_store[2];
21740 : /* Multiply with the number of GPR moves needed. */
21741 10003730266 : return cost * CEIL ((int) GET_MODE_SIZE (mode), UNITS_PER_WORD);
21742 : }
21743 : }
21744 :
21745 : static int
21746 1810106833 : ix86_memory_move_cost (machine_mode mode, reg_class_t regclass, bool in)
21747 : {
21748 2714839849 : return inline_memory_move_cost (mode, (enum reg_class) regclass, in ? 1 : 0);
21749 : }
21750 :
21751 :
21752 : /* Return the cost of moving data from a register in class CLASS1 to
21753 : one in class CLASS2.
21754 :
21755 : It is not required that the cost always equal 2 when FROM is the same as TO;
21756 : on some machines it is expensive to move between registers if they are not
21757 : general registers. */
21758 :
21759 : static int
21760 5712920202 : ix86_register_move_cost (machine_mode mode, reg_class_t class1_i,
21761 : reg_class_t class2_i)
21762 : {
21763 5712920202 : enum reg_class class1 = (enum reg_class) class1_i;
21764 5712920202 : enum reg_class class2 = (enum reg_class) class2_i;
21765 :
21766 : /* In case we require secondary memory, compute cost of the store followed
21767 : by load. In order to avoid bad register allocation choices, we need
21768 : for this to be *at least* as high as the symmetric MEMORY_MOVE_COST. */
21769 :
21770 5712920202 : if (inline_secondary_memory_needed (mode, class1, class2, false))
21771 : {
21772 2600701734 : int cost = 1;
21773 :
21774 2600701734 : cost += inline_memory_move_cost (mode, class1, 2);
21775 2600701734 : cost += inline_memory_move_cost (mode, class2, 2);
21776 :
21777 : /* In case of copying from general_purpose_register we may emit multiple
21778 : stores followed by single load causing memory size mismatch stall.
21779 : Count this as arbitrarily high cost of 20. */
21780 5201403468 : if (GET_MODE_BITSIZE (mode) > BITS_PER_WORD
21781 768845363 : && TARGET_MEMORY_MISMATCH_STALL
21782 4138392460 : && targetm.class_max_nregs (class1, mode)
21783 768845363 : > targetm.class_max_nregs (class2, mode))
21784 148885528 : cost += 20;
21785 :
21786 : /* In the case of FP/MMX moves, the registers actually overlap, and we
21787 : have to switch modes in order to treat them differently. */
21788 60368771 : if ((MMX_CLASS_P (class1) && MAYBE_FLOAT_CLASS_P (class2))
21789 2651563291 : || (MMX_CLASS_P (class2) && MAYBE_FLOAT_CLASS_P (class1)))
21790 19014428 : cost += 20;
21791 :
21792 : return cost;
21793 : }
21794 :
21795 : /* Moves between MMX and non-MMX units require secondary memory. */
21796 3112218468 : if (MMX_CLASS_P (class1) != MMX_CLASS_P (class2))
21797 0 : gcc_unreachable ();
21798 :
21799 3112218468 : if (SSE_CLASS_P (class1) != SSE_CLASS_P (class2))
21800 607228338 : return (SSE_CLASS_P (class1)
21801 607228338 : ? ix86_cost->hard_register.sse_to_integer
21802 607228338 : : ix86_cost->hard_register.integer_to_sse);
21803 :
21804 : /* Moves between mask register and GPR. */
21805 2504990130 : if (MASK_CLASS_P (class1) != MASK_CLASS_P (class2))
21806 : {
21807 1069541 : return (MASK_CLASS_P (class1)
21808 1069541 : ? ix86_cost->hard_register.mask_to_integer
21809 1069541 : : ix86_cost->hard_register.integer_to_mask);
21810 : }
21811 : /* Moving between mask registers. */
21812 2503920589 : if (MASK_CLASS_P (class1) && MASK_CLASS_P (class2))
21813 102447 : return ix86_cost->hard_register.mask_move;
21814 :
21815 2503818142 : if (MAYBE_FLOAT_CLASS_P (class1))
21816 12019900 : return ix86_cost->hard_register.fp_move;
21817 2491798242 : if (MAYBE_SSE_CLASS_P (class1))
21818 : {
21819 233742060 : if (GET_MODE_BITSIZE (mode) <= 128)
21820 114376578 : return ix86_cost->hard_register.xmm_move;
21821 4988904 : if (GET_MODE_BITSIZE (mode) <= 256)
21822 1585230 : return ix86_cost->hard_register.ymm_move;
21823 909222 : return ix86_cost->hard_register.zmm_move;
21824 : }
21825 2374927212 : if (MAYBE_MMX_CLASS_P (class1))
21826 2211035 : return ix86_cost->hard_register.mmx_move;
21827 : return 2;
21828 : }
21829 :
21830 : /* Implement TARGET_HARD_REGNO_NREGS. This is ordinarily the length in
21831 : words of a value of mode MODE but can be less for certain modes in
21832 : special long registers.
21833 :
21834 : Actually there are no two word move instructions for consecutive
21835 : registers. And only registers 0-3 may have mov byte instructions
21836 : applied to them. */
21837 :
21838 : static unsigned int
21839 9225863872 : ix86_hard_regno_nregs (unsigned int regno, machine_mode mode)
21840 : {
21841 9225863872 : if (GENERAL_REGNO_P (regno))
21842 : {
21843 3140719616 : if (mode == XFmode)
21844 25328384 : return TARGET_64BIT ? 2 : 3;
21845 3115391232 : if (mode == XCmode)
21846 25328384 : return TARGET_64BIT ? 4 : 6;
21847 6238849664 : return CEIL (GET_MODE_SIZE (mode), UNITS_PER_WORD);
21848 : }
21849 6085144256 : if (COMPLEX_MODE_P (mode))
21850 : return 2;
21851 : /* Register pair for mask registers. */
21852 5299964352 : if (mode == P2QImode || mode == P2HImode)
21853 98147488 : return 2;
21854 :
21855 : return 1;
21856 : }
21857 :
21858 : /* Implement REGMODE_NATURAL_SIZE(MODE). */
21859 : unsigned int
21860 238176920 : ix86_regmode_natural_size (machine_mode mode)
21861 : {
21862 238176920 : if (mode == P2HImode || mode == P2QImode)
21863 1771702 : return GET_MODE_SIZE (mode) / 2;
21864 237291069 : return UNITS_PER_WORD;
21865 : }
21866 :
21867 : /* Implement TARGET_HARD_REGNO_MODE_OK. */
21868 :
21869 : static bool
21870 54022302582 : ix86_hard_regno_mode_ok (unsigned int regno, machine_mode mode)
21871 : {
21872 : /* Flags and only flags can only hold CCmode values. */
21873 54022302582 : if (CC_REGNO_P (regno))
21874 420663901 : return GET_MODE_CLASS (mode) == MODE_CC;
21875 53601638681 : if (GET_MODE_CLASS (mode) == MODE_CC
21876 : || GET_MODE_CLASS (mode) == MODE_RANDOM)
21877 : return false;
21878 48115949386 : if (STACK_REGNO_P (regno))
21879 4611180220 : return VALID_FP_MODE_P (mode);
21880 43504769166 : if (MASK_REGNO_P (regno))
21881 : {
21882 : /* Register pair only starts at even register number. */
21883 3598702775 : if ((mode == P2QImode || mode == P2HImode))
21884 49694822 : return MASK_PAIR_REGNO_P(regno);
21885 :
21886 1032084257 : return ((TARGET_AVX512F && VALID_MASK_REG_MODE (mode))
21887 4560101467 : || (TARGET_AVX512BW && VALID_MASK_AVX512BW_MODE (mode)));
21888 : }
21889 :
21890 39906066391 : if (GET_MODE_CLASS (mode) == MODE_PARTIAL_INT)
21891 : return false;
21892 :
21893 38932393332 : if (SSE_REGNO_P (regno))
21894 : {
21895 : /* We implement the move patterns for all vector modes into and
21896 : out of SSE registers, even when no operation instructions
21897 : are available. */
21898 :
21899 : /* For AVX-512 we allow, regardless of regno:
21900 : - XI mode
21901 : - any of 512-bit wide vector mode
21902 : - any scalar mode. */
21903 16537280835 : if (TARGET_AVX512F
21904 : && ((VALID_AVX512F_REG_OR_XI_MODE (mode))
21905 : || VALID_AVX512F_SCALAR_MODE (mode)))
21906 : return true;
21907 :
21908 : /* TODO check for QI/HI scalars. */
21909 : /* AVX512VL allows sse regs16+ for 128/256 bit modes. */
21910 15825931293 : if (TARGET_AVX512VL
21911 1882664568 : && (VALID_AVX256_REG_OR_OI_MODE (mode)
21912 1654467199 : || VALID_AVX512VL_128_REG_MODE (mode)))
21913 : return true;
21914 :
21915 : /* xmm16-xmm31 are only available for AVX-512. */
21916 15345915300 : if (EXT_REX_SSE_REGNO_P (regno))
21917 : return false;
21918 :
21919 : /* Without SSE2, 16-bit moves are not supported. */
21920 10080138342 : if (!TARGET_SSE2 && GET_MODE_SIZE (mode) == GET_MODE_SIZE (HImode))
21921 : return false;
21922 :
21923 : /* OImode and AVX modes are available only when AVX is enabled. */
21924 8799108308 : return ((TARGET_AVX
21925 1948227581 : && VALID_AVX256_REG_OR_OI_MODE (mode))
21926 : || VALID_SSE_REG_MODE (mode)
21927 : || VALID_SSE2_REG_MODE (mode)
21928 : || VALID_MMX_REG_MODE (mode)
21929 8799108308 : || VALID_MMX_REG_MODE_3DNOW (mode));
21930 : }
21931 22395112497 : if (MMX_REGNO_P (regno))
21932 : {
21933 : /* We implement the move patterns for 3DNOW modes even in MMX mode,
21934 : so if the register is available at all, then we can move data of
21935 : the given mode into or out of it. */
21936 3842629013 : return (VALID_MMX_REG_MODE (mode)
21937 : || VALID_MMX_REG_MODE_3DNOW (mode));
21938 : }
21939 :
21940 18552483484 : if (mode == QImode)
21941 : {
21942 : /* Take care for QImode values - they can be in non-QI regs,
21943 : but then they do cause partial register stalls. */
21944 211308278 : if (ANY_QI_REGNO_P (regno))
21945 : return true;
21946 21584413 : if (!TARGET_PARTIAL_REG_STALL)
21947 : return true;
21948 : /* LRA checks if the hard register is OK for the given mode.
21949 : QImode values can live in non-QI regs, so we allow all
21950 : registers here. */
21951 33 : if (lra_in_progress)
21952 : return true;
21953 33 : return !can_create_pseudo_p ();
21954 : }
21955 : /* We handle both integer and floats in the general purpose registers. */
21956 18341175206 : else if (VALID_INT_MODE_P (mode)
21957 13407293577 : || VALID_FP_MODE_P (mode))
21958 : return true;
21959 : /* Lots of MMX code casts 8 byte vector modes to DImode. If we then go
21960 : on to use that value in smaller contexts, this can easily force a
21961 : pseudo to be allocated to GENERAL_REGS. Since this is no worse than
21962 : supporting DImode, allow it. */
21963 12327437331 : else if (VALID_MMX_REG_MODE_3DNOW (mode) || VALID_MMX_REG_MODE (mode))
21964 : return true;
21965 :
21966 : return false;
21967 : }
21968 :
21969 : /* Initialize function_abis[ABI_ID] with the set of register clobbers
21970 : in FULL_REG_CLOBBERS, adjusting for rules that apply to all ABIs. */
21971 :
21972 : static void
21973 762 : ix86_initialize_abi (unsigned int abi_id, HARD_REG_SET full_reg_clobbers)
21974 : {
21975 : /* The general rule is that fixed registers should be marked as
21976 : call-clobbered. This includes global registers, inaccessible
21977 : registers, the flags register, and the FPSR.
21978 :
21979 : Handle the exceptions below. */
21980 762 : full_reg_clobbers |= fixed_reg_set;
21981 :
21982 : /* Every ABI (even preserve_none) preserves EBP/RBP. */
21983 762 : CLEAR_HARD_REG_BIT (full_reg_clobbers, HARD_FRAME_POINTER_REGNUM);
21984 :
21985 : /* Treat GCC's internal frame-related registers as call-preserved. */
21986 762 : CLEAR_HARD_REG_BIT (full_reg_clobbers, FRAME_POINTER_REGNUM);
21987 762 : CLEAR_HARD_REG_BIT (full_reg_clobbers, ARG_POINTER_REGNUM);
21988 :
21989 : /* MMX registers aren't preserved. */
21990 762 : if (TARGET_MMX)
21991 762 : full_reg_clobbers |= reg_class_contents[MMX_REGS];
21992 :
21993 : /* X87 registers aren't preserved. */
21994 762 : if (TARGET_80387 || TARGET_FLOAT_RETURNS_IN_80387)
21995 762 : full_reg_clobbers |= reg_class_contents[FLOAT_REGS];
21996 :
21997 762 : function_abis[abi_id].initialize (abi_id, full_reg_clobbers);
21998 762 : }
21999 :
22000 : /* Return the descriptor of no_callee_saved_registers function type.
22001 : None of the enabled registers are preserved, except for the common
22002 : rules applied by ix86_initialize_abi. */
22003 :
22004 : static const predefined_function_abi &
22005 3088 : ix86_no_callee_saved_abi (void)
22006 : {
22007 3088 : auto &no_callee_saved_abi = function_abis[ABI_NO_CALLEE_SAVED];
22008 3088 : if (!no_callee_saved_abi.initialized_p ())
22009 98 : ix86_initialize_abi (ABI_NO_CALLEE_SAVED, accessible_reg_set);
22010 3088 : return no_callee_saved_abi;
22011 : }
22012 :
22013 : /* Return the descriptor of the no_caller_saved_registers function type
22014 : with ABI identifier ABI_ID. All registers are preserved, except for:
22015 :
22016 : - the return registers, which are enumerated in ABI_ID.
22017 :
22018 : - the common rules applied by ix86_initialize_abi. */
22019 :
22020 : static const predefined_function_abi &
22021 3705 : ix86_no_caller_saved_abi (unsigned int abi_id)
22022 : {
22023 3705 : auto &abi = function_abis[abi_id];
22024 3705 : if (!abi.initialized_p ())
22025 : {
22026 172 : HARD_REG_SET full_reg_clobbers = {};
22027 :
22028 172 : switch (abi_id)
22029 : {
22030 : case ABI_NO_CALLER_SAVED_RETURN_VOID:
22031 : break;
22032 :
22033 1 : case ABI_NO_CALLER_SAVED_RETURN_AX_DX:
22034 1 : SET_HARD_REG_BIT (full_reg_clobbers, DX_REG);
22035 : /* Fall through. */
22036 9 : case ABI_NO_CALLER_SAVED_RETURN_AX:
22037 9 : SET_HARD_REG_BIT (full_reg_clobbers, AX_REG);
22038 9 : break;
22039 :
22040 2 : case ABI_NO_CALLER_SAVED_RETURN_AX_XMM0:
22041 2 : SET_HARD_REG_BIT (full_reg_clobbers, AX_REG);
22042 2 : SET_HARD_REG_BIT (full_reg_clobbers, XMM0_REG);
22043 2 : break;
22044 :
22045 1 : case ABI_NO_CALLER_SAVED_RETURN_XMM0_XMM1:
22046 1 : SET_HARD_REG_BIT (full_reg_clobbers, XMM1_REG);
22047 : /* Fall through. */
22048 2 : case ABI_NO_CALLER_SAVED_RETURN_XMM0:
22049 2 : SET_HARD_REG_BIT (full_reg_clobbers, XMM0_REG);
22050 2 : break;
22051 :
22052 0 : default:
22053 0 : gcc_unreachable ();
22054 : }
22055 :
22056 172 : ix86_initialize_abi (abi_id, full_reg_clobbers);
22057 : }
22058 3705 : return abi;
22059 : }
22060 :
22061 : /* Return the descriptor of the standard function ABI type. If
22062 : ABI_TYPE == ABI_ALTERNATE, return the function alternate ABI type. */
22063 :
22064 : static const predefined_function_abi &
22065 1596088 : ix86_standard_abi (int abi_type)
22066 : {
22067 1596088 : static const char ix86_call_used_regs[] = CALL_USED_REGISTERS;
22068 1596088 : auto &standard_abi = function_abis[abi_type];
22069 1596088 : if (!standard_abi.initialized_p ())
22070 : {
22071 492 : HARD_REG_SET full_reg_clobbers = {};
22072 :
22073 : /* Add all registers that are clobbered by the call. NB: If the
22074 : current ABI is SYSV_ABI, the alternate ABI is MS_ABI. */
22075 1007 : bool is_64bit_ms_abi = (TARGET_64BIT
22076 492 : && ix86_abi == (abi_type == ABI_ALTERNATE
22077 492 : ? SYSV_ABI : MS_ABI));
22078 23 : char c_mask = CALL_USED_REGISTERS_MASK (is_64bit_ms_abi);
22079 46740 : for (int i = 0; i < FIRST_PSEUDO_REGISTER; i++)
22080 46248 : if (ix86_call_used_regs[i] == 1
22081 8856 : || (ix86_call_used_regs[i] & c_mask))
22082 37668 : SET_HARD_REG_BIT (full_reg_clobbers, i);
22083 :
22084 492 : ix86_initialize_abi (abi_type, full_reg_clobbers);
22085 : }
22086 1596088 : return standard_abi;
22087 : }
22088 :
22089 : /* Return the descriptor of the function alternate ABI type. */
22090 :
22091 : static const predefined_function_abi &
22092 1596088 : ix86_alternate_abi (void)
22093 : {
22094 12 : return ix86_standard_abi (ABI_ALTERNATE);
22095 : }
22096 :
22097 : /* Return the function ABI ID based on FNTYPE. */
22098 :
22099 : static int
22100 424237153 : ix86_function_abi_id (const_tree fntype)
22101 : {
22102 424237153 : auto call_saved_registers = ix86_fntype_call_saved_registers (fntype);
22103 424237153 : if (call_saved_registers == TYPE_PRESERVE_NONE
22104 424237153 : || call_saved_registers == TYPE_NO_CALLEE_SAVED_REGISTERS)
22105 : return ABI_NO_CALLEE_SAVED;
22106 :
22107 424234065 : if (call_saved_registers == TYPE_NO_CALLER_SAVED_REGISTERS)
22108 : {
22109 3705 : tree type = TREE_TYPE (fntype);
22110 3705 : if (VOID_TYPE_P (type))
22111 : return ABI_NO_CALLER_SAVED_RETURN_VOID;
22112 : /* AX register contains the address of the return value location
22113 : passed in by the caller. */
22114 2256 : else if (ix86_return_in_memory (type, fntype))
22115 : return ABI_NO_CALLER_SAVED_RETURN_AX;
22116 2197 : rtx ret = ix86_function_value (type, fntype, false);
22117 2197 : unsigned int nregs;
22118 2197 : if (REG_P (ret))
22119 : {
22120 2016 : unsigned int regno = REGNO (ret);
22121 2016 : if (STACK_REGNO_P (regno) || MMX_REGNO_P (regno))
22122 : return ABI_NO_CALLER_SAVED_RETURN_VOID;
22123 : else
22124 2016 : switch (regno)
22125 : {
22126 1947 : case AX_REG:
22127 1947 : nregs = REG_NREGS (ret);
22128 1947 : if (nregs == 1)
22129 : return ABI_NO_CALLER_SAVED_RETURN_AX;
22130 71 : else if (nregs == 2)
22131 : return ABI_NO_CALLER_SAVED_RETURN_AX_DX;
22132 0 : gcc_unreachable ();
22133 : case XMM0_REG:
22134 : return ABI_NO_CALLER_SAVED_RETURN_XMM0;
22135 0 : default:
22136 0 : gcc_unreachable ();
22137 : }
22138 : }
22139 181 : else if (GET_CODE (ret) == PARALLEL && XVECLEN (ret, 0) == 2)
22140 : {
22141 181 : rtx x0 = XVECEXP (ret, 0, 0);
22142 181 : rtx x1 = XVECEXP (ret, 0, 1);
22143 181 : if (GET_CODE (x0) == EXPR_LIST
22144 181 : && GET_CODE (x1) == EXPR_LIST)
22145 : {
22146 181 : x0 = XEXP (x0, 0);
22147 181 : x1 = XEXP (x1, 0);
22148 181 : if (REG_P (x0) && REGNO (x0) == AX_REG)
22149 : {
22150 59 : if (REG_P (x1) && REGNO (x1) == XMM0_REG)
22151 : return ABI_NO_CALLER_SAVED_RETURN_AX_XMM0;
22152 : }
22153 122 : if (REG_P (x0) && REGNO (x0) == XMM0_REG && REG_P (x1))
22154 : {
22155 122 : if (REGNO (x1) == AX_REG)
22156 : return ABI_NO_CALLER_SAVED_RETURN_AX_XMM0;
22157 64 : else if (REGNO (x1) == XMM1_REG)
22158 : return ABI_NO_CALLER_SAVED_RETURN_XMM0_XMM1;
22159 : }
22160 : }
22161 :
22162 0 : gcc_unreachable ();
22163 : }
22164 : }
22165 :
22166 : /* NB: This must be the last since other attributes change the
22167 : function ABI. */
22168 424230360 : if (ix86_function_type_abi (fntype) != ix86_abi)
22169 1596076 : return ABI_ALTERNATE;
22170 :
22171 : return ABI_DEFAULT;
22172 : }
22173 :
22174 : /* Implement TARGET_FNTYPE_ABI. */
22175 :
22176 : static const predefined_function_abi &
22177 424237153 : ix86_fntype_abi (const_tree fntype)
22178 : {
22179 424237153 : unsigned int abi_id = ix86_function_abi_id (fntype);
22180 424237153 : switch (abi_id)
22181 : {
22182 422634284 : case ABI_DEFAULT:
22183 422634284 : return default_function_abi;
22184 :
22185 1596076 : case ABI_ALTERNATE:
22186 1596076 : return ix86_alternate_abi ();
22187 :
22188 3088 : case ABI_NO_CALLEE_SAVED:
22189 3088 : return ix86_no_callee_saved_abi ();
22190 :
22191 3705 : case ABI_NO_CALLER_SAVED_RETURN_VOID:
22192 3705 : case ABI_NO_CALLER_SAVED_RETURN_AX:
22193 3705 : case ABI_NO_CALLER_SAVED_RETURN_AX_DX:
22194 3705 : case ABI_NO_CALLER_SAVED_RETURN_AX_XMM0:
22195 3705 : case ABI_NO_CALLER_SAVED_RETURN_XMM0:
22196 3705 : case ABI_NO_CALLER_SAVED_RETURN_XMM0_XMM1:
22197 3705 : return ix86_no_caller_saved_abi (abi_id);
22198 :
22199 : default:
22200 : gcc_unreachable ();
22201 : }
22202 :
22203 : return default_function_abi;
22204 : }
22205 :
22206 : /* Initialize function_abis with corresponding abi_id,
22207 : currently only handle vzeroupper. */
22208 : void
22209 22375 : ix86_initialize_callee_abi (unsigned int abi_id)
22210 : {
22211 22375 : gcc_assert (abi_id == ABI_VZEROUPPER);
22212 22375 : predefined_function_abi &vzeroupper_abi = function_abis[abi_id];
22213 22375 : if (!vzeroupper_abi.initialized_p ())
22214 : {
22215 : HARD_REG_SET full_reg_clobbers;
22216 4347 : CLEAR_HARD_REG_SET (full_reg_clobbers);
22217 4347 : vzeroupper_abi.initialize (ABI_VZEROUPPER, full_reg_clobbers);
22218 : }
22219 22375 : }
22220 :
22221 : void
22222 22375 : ix86_expand_avx_vzeroupper (void)
22223 : {
22224 : /* Initialize vzeroupper_abi here. */
22225 22375 : ix86_initialize_callee_abi (ABI_VZEROUPPER);
22226 22375 : rtx_insn *insn = emit_call_insn (gen_avx_vzeroupper_callee_abi ());
22227 22375 : CALL_INSN_ABI_ID (insn) = ABI_VZEROUPPER;
22228 : /* Return false for non-local goto in can_nonlocal_goto. */
22229 22375 : make_reg_eh_region_note (insn, 0, INT_MIN);
22230 : /* Flag used for call_insn indicates it's a fake call. */
22231 22375 : RTX_FLAG (insn, used) = 1;
22232 22375 : }
22233 :
22234 :
22235 : /* Implement TARGET_HARD_REGNO_CALL_PART_CLOBBERED. The only ABI that
22236 : saves SSE registers across calls is Win64 (thus no need to check the
22237 : current ABI here), and with AVX enabled Win64 only guarantees that
22238 : the low 16 bytes are saved. */
22239 :
22240 : static bool
22241 1973086162 : ix86_hard_regno_call_part_clobbered (unsigned int abi_id, unsigned int regno,
22242 : machine_mode mode)
22243 : {
22244 1973086162 : switch (abi_id)
22245 : {
22246 32323101 : case ABI_VZEROUPPER:
22247 : /* Special ABI for vzeroupper which only clobbers higher part of
22248 : SSE registers. */
22249 32323101 : return (GET_MODE_SIZE (mode) > 16
22250 32323101 : && ((TARGET_64BIT && REX_SSE_REGNO_P (regno))
22251 4868610 : || LEGACY_SSE_REGNO_P (regno)));
22252 :
22253 1940258889 : case ABI_DEFAULT:
22254 1940258889 : case ABI_ALTERNATE:
22255 1940258889 : case ABI_NO_CALLEE_SAVED:
22256 1940258889 : break;
22257 :
22258 : case ABI_NO_CALLER_SAVED_RETURN_VOID:
22259 : case ABI_NO_CALLER_SAVED_RETURN_AX:
22260 : case ABI_NO_CALLER_SAVED_RETURN_AX_DX:
22261 : /* These ABIs don't clobber SSE registers. */
22262 : return false;
22263 :
22264 : case ABI_NO_CALLER_SAVED_RETURN_AX_XMM0:
22265 : case ABI_NO_CALLER_SAVED_RETURN_XMM0:
22266 : case ABI_NO_CALLER_SAVED_RETURN_XMM0_XMM1:
22267 : /* These ABIs return some values in SSE registers and preserve
22268 : the rest. The return value registers (XMM0 and possibly XMM1)
22269 : are fully rather than partially call-clobbered. */
22270 : return false;
22271 :
22272 0 : default:
22273 0 : gcc_unreachable ();
22274 : }
22275 :
22276 2535530305 : return SSE_REGNO_P (regno) && GET_MODE_SIZE (mode) > 16;
22277 : }
22278 :
22279 : /* A subroutine of ix86_modes_tieable_p. Return true if MODE is a
22280 : tieable integer mode. */
22281 :
22282 : static bool
22283 54807634 : ix86_tieable_integer_mode_p (machine_mode mode)
22284 : {
22285 54807634 : switch (mode)
22286 : {
22287 : case E_HImode:
22288 : case E_SImode:
22289 : return true;
22290 :
22291 5442598 : case E_QImode:
22292 5442598 : return TARGET_64BIT || !TARGET_PARTIAL_REG_STALL;
22293 :
22294 10966715 : case E_DImode:
22295 10966715 : return TARGET_64BIT;
22296 :
22297 : default:
22298 : return false;
22299 : }
22300 : }
22301 :
22302 : /* Implement TARGET_MODES_TIEABLE_P.
22303 :
22304 : Return true if MODE1 is accessible in a register that can hold MODE2
22305 : without copying. That is, all register classes that can hold MODE2
22306 : can also hold MODE1. */
22307 :
22308 : static bool
22309 35335184 : ix86_modes_tieable_p (machine_mode mode1, machine_mode mode2)
22310 : {
22311 35335184 : if (mode1 == mode2)
22312 : return true;
22313 :
22314 35249366 : if (ix86_tieable_integer_mode_p (mode1)
22315 35249366 : && ix86_tieable_integer_mode_p (mode2))
22316 : return true;
22317 :
22318 : /* MODE2 being XFmode implies fp stack or general regs, which means we
22319 : can tie any smaller floating point modes to it. Note that we do not
22320 : tie this with TFmode. */
22321 25691944 : if (mode2 == XFmode)
22322 4315 : return mode1 == SFmode || mode1 == DFmode;
22323 :
22324 : /* MODE2 being DFmode implies fp stack, general or sse regs, which means
22325 : that we can tie it with SFmode. */
22326 25687629 : if (mode2 == DFmode)
22327 250412 : return mode1 == SFmode;
22328 :
22329 : /* If MODE2 is only appropriate for an SSE register, then tie with
22330 : any vector modes or scalar floating point modes acceptable to SSE
22331 : registers, excluding scalar integer modes with SUBREG:
22332 : (subreg:QI (reg:TI 99) 0))
22333 : (subreg:HI (reg:TI 99) 0))
22334 : (subreg:SI (reg:TI 99) 0))
22335 : (subreg:DI (reg:TI 99) 0))
22336 : to avoid unnecessary move from SSE register to integer register.
22337 : */
22338 25437217 : if (GET_MODE_SIZE (mode2) >= 16
22339 39395382 : && (GET_MODE_SIZE (mode1) == GET_MODE_SIZE (mode2)
22340 13820525 : || ((VECTOR_MODE_P (mode1) || SCALAR_FLOAT_MODE_P (mode1))
22341 508018 : && GET_MODE_SIZE (mode1) <= GET_MODE_SIZE (mode2)))
22342 31567302 : && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode2))
22343 5686404 : return ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode1);
22344 :
22345 : /* If MODE2 is appropriate for an MMX register, then tie
22346 : with any other mode acceptable to MMX registers. */
22347 19750813 : if (GET_MODE_SIZE (mode2) == 8
22348 19750813 : && ix86_hard_regno_mode_ok (FIRST_MMX_REG, mode2))
22349 3383654 : return (GET_MODE_SIZE (mode1) == 8
22350 3383654 : && ix86_hard_regno_mode_ok (FIRST_MMX_REG, mode1));
22351 :
22352 : /* SCmode and DImode can be tied. */
22353 16367159 : if ((mode1 == E_SCmode && mode2 == E_DImode)
22354 16367159 : || (mode1 == E_DImode && mode2 == E_SCmode))
22355 108 : return TARGET_64BIT;
22356 :
22357 : /* [SD]Cmode and V2[SD]Fmode modes can be tied. */
22358 16367051 : if ((mode1 == E_SCmode && mode2 == E_V2SFmode)
22359 16367051 : || (mode1 == E_V2SFmode && mode2 == E_SCmode)
22360 16367051 : || (mode1 == E_DCmode && mode2 == E_V2DFmode)
22361 16367051 : || (mode1 == E_V2DFmode && mode2 == E_DCmode))
22362 0 : return true;
22363 :
22364 : return false;
22365 : }
22366 :
22367 : /* Return the cost of moving between two registers of mode MODE. */
22368 :
22369 : static int
22370 30090342 : ix86_set_reg_reg_cost (machine_mode mode)
22371 : {
22372 30090342 : unsigned int units = UNITS_PER_WORD;
22373 :
22374 30090342 : switch (GET_MODE_CLASS (mode))
22375 : {
22376 : default:
22377 : break;
22378 :
22379 : case MODE_CC:
22380 30090342 : units = GET_MODE_SIZE (CCmode);
22381 : break;
22382 :
22383 1180146 : case MODE_FLOAT:
22384 1180146 : if ((TARGET_SSE && mode == TFmode)
22385 685591 : || (TARGET_80387 && mode == XFmode)
22386 219182 : || ((TARGET_80387 || TARGET_SSE2) && mode == DFmode)
22387 148381 : || ((TARGET_80387 || TARGET_SSE) && mode == SFmode))
22388 2325112 : units = GET_MODE_SIZE (mode);
22389 : break;
22390 :
22391 1337858 : case MODE_COMPLEX_FLOAT:
22392 1337858 : if ((TARGET_SSE && mode == TCmode)
22393 896856 : || (TARGET_80387 && mode == XCmode)
22394 455730 : || ((TARGET_80387 || TARGET_SSE2) && mode == DCmode)
22395 14570 : || ((TARGET_80387 || TARGET_SSE) && mode == SCmode))
22396 2669152 : units = GET_MODE_SIZE (mode);
22397 : break;
22398 :
22399 19151589 : case MODE_VECTOR_INT:
22400 19151589 : case MODE_VECTOR_FLOAT:
22401 19151589 : if ((TARGET_AVX512F && VALID_AVX512F_REG_MODE (mode))
22402 19054265 : || (TARGET_AVX && VALID_AVX256_REG_MODE (mode))
22403 18880376 : || (TARGET_SSE2 && VALID_SSE2_REG_MODE (mode))
22404 16189099 : || (TARGET_SSE && VALID_SSE_REG_MODE (mode))
22405 14850908 : || ((TARGET_MMX || TARGET_MMX_WITH_SSE)
22406 14805060 : && VALID_MMX_REG_MODE (mode)))
22407 8710406 : units = GET_MODE_SIZE (mode);
22408 : }
22409 :
22410 : /* Return the cost of moving between two registers of mode MODE,
22411 : assuming that the move will be in pieces of at most UNITS bytes. */
22412 30090342 : return COSTS_N_INSNS (CEIL (GET_MODE_SIZE (mode), units));
22413 : }
22414 :
22415 : /* Return cost of vector operation in MODE given that scalar version has
22416 : COST. */
22417 :
22418 : static int
22419 2899036211 : ix86_vec_cost (machine_mode mode, int cost)
22420 : {
22421 2899036211 : if (!VECTOR_MODE_P (mode))
22422 : return cost;
22423 :
22424 2898783079 : if (GET_MODE_BITSIZE (mode) == 128
22425 2898783079 : && TARGET_SSE_SPLIT_REGS)
22426 2882536 : return cost * GET_MODE_BITSIZE (mode) / 64;
22427 2897341811 : else if (GET_MODE_BITSIZE (mode) > 128
22428 2897341811 : && TARGET_AVX256_SPLIT_REGS)
22429 1720252 : return cost * GET_MODE_BITSIZE (mode) / 128;
22430 2896481685 : else if (GET_MODE_BITSIZE (mode) > 256
22431 2896481685 : && TARGET_AVX512_SPLIT_REGS)
22432 335640 : return cost * GET_MODE_BITSIZE (mode) / 256;
22433 : return cost;
22434 : }
22435 :
22436 : /* Return cost of vec_widen_<s>mult_hi/lo_<mode>,
22437 : vec_widen_<s>mul_hi/lo_<mode> is only available for VI124_AVX2. */
22438 : static int
22439 1217 : ix86_widen_mult_cost (const struct processor_costs *cost,
22440 : enum machine_mode mode, bool uns_p)
22441 : {
22442 1217 : gcc_assert (GET_MODE_CLASS (mode) == MODE_VECTOR_INT);
22443 1217 : int extra_cost = 0;
22444 1217 : int basic_cost = 0;
22445 1217 : switch (mode)
22446 : {
22447 136 : case V8HImode:
22448 136 : case V16HImode:
22449 136 : if (!uns_p || mode == V16HImode)
22450 55 : extra_cost = cost->sse_op * 2;
22451 136 : basic_cost = cost->mulss * 2 + cost->sse_op * 4;
22452 136 : break;
22453 209 : case V4SImode:
22454 209 : case V8SImode:
22455 : /* pmulhw/pmullw can be used. */
22456 209 : basic_cost = cost->mulss * 2 + cost->sse_op * 2;
22457 209 : break;
22458 804 : case V2DImode:
22459 : /* pmuludq under sse2, pmuldq under sse4.1, for sign_extend,
22460 : require extra 4 mul, 4 add, 4 cmp and 2 shift. */
22461 804 : if (!TARGET_SSE4_1 && !uns_p)
22462 484 : extra_cost = (cost->mulss + cost->sse_op + cost->sse_op) * 4
22463 484 : + cost->sse_op * 2;
22464 : /* Fallthru. */
22465 860 : case V4DImode:
22466 860 : basic_cost = cost->mulss * 2 + cost->sse_op * 4;
22467 860 : break;
22468 : default:
22469 : /* Not implemented. */
22470 : return 100;
22471 : }
22472 1205 : return ix86_vec_cost (mode, basic_cost + extra_cost);
22473 : }
22474 :
22475 : /* Return cost of multiplication in MODE. */
22476 :
22477 : static int
22478 1237032216 : ix86_multiplication_cost (const struct processor_costs *cost,
22479 : enum machine_mode mode)
22480 : {
22481 1237032216 : machine_mode inner_mode = mode;
22482 1237032216 : if (VECTOR_MODE_P (mode))
22483 1235989393 : inner_mode = GET_MODE_INNER (mode);
22484 :
22485 1237032216 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
22486 760785 : return inner_mode == DFmode ? cost->mulsd : cost->mulss;
22487 1236271431 : else if (X87_FLOAT_MODE_P (mode))
22488 166707 : return cost->fmul;
22489 1236104724 : else if (FLOAT_MODE_P (mode))
22490 241868 : return ix86_vec_cost (mode,
22491 241868 : inner_mode == DFmode ? cost->mulsd : cost->mulss);
22492 1235862856 : else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
22493 : {
22494 1235771687 : int nmults, nops;
22495 : /* Cost of reading the memory. */
22496 1235771687 : int extra;
22497 :
22498 1235771687 : switch (mode)
22499 : {
22500 19460482 : case V4QImode:
22501 19460482 : case V8QImode:
22502 : /* Partial V*QImode is emulated with 4-6 insns. */
22503 19460482 : nmults = 1;
22504 19460482 : nops = 3;
22505 19460482 : extra = 0;
22506 :
22507 19460482 : if (TARGET_AVX512BW && TARGET_AVX512VL)
22508 : ;
22509 19344312 : else if (TARGET_AVX2)
22510 : nops += 2;
22511 18836172 : else if (TARGET_XOP)
22512 10040 : extra += COSTS_N_INSNS (cost->sse_load[2]) / 2;
22513 : else
22514 : {
22515 18826132 : nops += 1;
22516 18826132 : extra += COSTS_N_INSNS (cost->sse_load[2]) / 2;
22517 : }
22518 19460482 : goto do_qimode;
22519 :
22520 9730726 : case V16QImode:
22521 : /* V*QImode is emulated with 4-11 insns. */
22522 9730726 : nmults = 1;
22523 9730726 : nops = 3;
22524 9730726 : extra = 0;
22525 :
22526 9730726 : if (TARGET_AVX2 && !TARGET_PREFER_AVX128)
22527 : {
22528 309899 : if (!(TARGET_AVX512BW && TARGET_AVX512VL))
22529 252100 : nops += 3;
22530 : }
22531 9420827 : else if (TARGET_XOP)
22532 : {
22533 5464 : nmults += 1;
22534 5464 : nops += 2;
22535 5464 : extra += COSTS_N_INSNS (cost->sse_load[2]) / 2;
22536 : }
22537 : else
22538 : {
22539 9415363 : nmults += 1;
22540 9415363 : nops += 4;
22541 9415363 : extra += COSTS_N_INSNS (cost->sse_load[2]) / 2;
22542 : }
22543 9730726 : goto do_qimode;
22544 :
22545 9729092 : case V32QImode:
22546 9729092 : nmults = 1;
22547 9729092 : nops = 3;
22548 9729092 : extra = 0;
22549 :
22550 9729092 : if (!TARGET_AVX512BW || TARGET_PREFER_AVX256)
22551 : {
22552 9643940 : nmults += 1;
22553 9643940 : nops += 4;
22554 : /* 2 loads, so no division by 2. */
22555 9643940 : extra += COSTS_N_INSNS (cost->sse_load[3]);
22556 : }
22557 9729092 : goto do_qimode;
22558 :
22559 9728613 : case V64QImode:
22560 9728613 : nmults = 2;
22561 9728613 : nops = 9;
22562 : /* 2 loads of each size, so no division by 2. */
22563 9728613 : extra = COSTS_N_INSNS (cost->sse_load[3] + cost->sse_load[4]);
22564 :
22565 48648913 : do_qimode:
22566 48648913 : return ix86_vec_cost (mode, cost->mulss * nmults
22567 48648913 : + cost->sse_op * nops) + extra;
22568 :
22569 41606634 : case V4SImode:
22570 : /* pmulld is used in this case. No emulation is needed. */
22571 41606634 : if (TARGET_SSE4_1)
22572 2268645 : goto do_native;
22573 : /* V4SImode is emulated with 7 insns. */
22574 : else
22575 39337989 : return ix86_vec_cost (mode, cost->mulss * 2 + cost->sse_op * 5);
22576 :
22577 168171263 : case V2DImode:
22578 168171263 : case V4DImode:
22579 : /* vpmullq is used in this case. No emulation is needed. */
22580 168171263 : if (TARGET_AVX512DQ && TARGET_AVX512VL)
22581 650747 : goto do_native;
22582 : /* V*DImode is emulated with 6-8 insns. */
22583 167520516 : else if (TARGET_XOP && mode == V2DImode)
22584 55164 : return ix86_vec_cost (mode, cost->mulss * 2 + cost->sse_op * 4);
22585 : /* FALLTHRU */
22586 251486032 : case V8DImode:
22587 : /* vpmullq is used in this case. No emulation is needed. */
22588 251486032 : if (TARGET_AVX512DQ && mode == V8DImode)
22589 417628 : goto do_native;
22590 : else
22591 251068404 : return ix86_vec_cost (mode, cost->mulss * 3 + cost->sse_op * 5);
22592 :
22593 896661217 : default:
22594 896661217 : do_native:
22595 896661217 : return ix86_vec_cost (mode, cost->mulss);
22596 : }
22597 : }
22598 : else
22599 182262 : return (cost->mult_init[MODE_INDEX (mode)] + cost->mult_bit * 7);
22600 : }
22601 :
22602 : /* Return cost of multiplication in MODE. */
22603 :
22604 : static int
22605 74843840 : ix86_division_cost (const struct processor_costs *cost,
22606 : enum machine_mode mode)
22607 : {
22608 74843840 : machine_mode inner_mode = mode;
22609 74843840 : if (VECTOR_MODE_P (mode))
22610 54852312 : inner_mode = GET_MODE_INNER (mode);
22611 :
22612 74843840 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
22613 256392 : return inner_mode == DFmode ? cost->divsd : cost->divss;
22614 74587448 : else if (X87_FLOAT_MODE_P (mode))
22615 46863 : return cost->fdiv;
22616 74540585 : else if (FLOAT_MODE_P (mode))
22617 22361 : return ix86_vec_cost (mode,
22618 22361 : inner_mode == DFmode ? cost->divsd : cost->divss);
22619 : else
22620 83592294 : return cost->divide[MODE_INDEX (mode)];
22621 : }
22622 :
22623 : /* Return cost of shift in MODE.
22624 : If CONSTANT_OP1 is true, the op1 value is known and set in OP1_VAL.
22625 : AND_IN_OP1 specify in op1 is result of AND and SHIFT_AND_TRUNCATE
22626 : if op1 is a result of subreg.
22627 :
22628 : SKIP_OP0/1 is set to true if cost of OP0/1 should be ignored. */
22629 :
22630 : static int
22631 794436966 : ix86_shift_rotate_cost (const struct processor_costs *cost,
22632 : enum rtx_code code,
22633 : enum machine_mode mode, bool constant_op1,
22634 : HOST_WIDE_INT op1_val,
22635 : bool and_in_op1,
22636 : bool shift_and_truncate,
22637 : bool *skip_op0, bool *skip_op1)
22638 : {
22639 794436966 : if (skip_op0)
22640 794352945 : *skip_op0 = *skip_op1 = false;
22641 :
22642 794436966 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
22643 : {
22644 407988114 : int count;
22645 : /* Cost of reading the memory. */
22646 407988114 : int extra;
22647 :
22648 407988114 : switch (mode)
22649 : {
22650 6192129 : case V4QImode:
22651 6192129 : case V8QImode:
22652 6192129 : if (TARGET_AVX2)
22653 : /* Use vpbroadcast. */
22654 199007 : extra = cost->sse_op;
22655 : else
22656 5993122 : extra = COSTS_N_INSNS (cost->sse_load[2]) / 2;
22657 :
22658 6192129 : if (constant_op1)
22659 : {
22660 6192099 : if (code == ASHIFTRT)
22661 : {
22662 308 : count = 4;
22663 308 : extra *= 2;
22664 : }
22665 : else
22666 : count = 2;
22667 : }
22668 30 : else if (TARGET_AVX512BW && TARGET_AVX512VL)
22669 30 : return ix86_vec_cost (mode, cost->sse_op * 4);
22670 0 : else if (TARGET_SSE4_1)
22671 : count = 5;
22672 0 : else if (code == ASHIFTRT)
22673 : count = 6;
22674 : else
22675 0 : count = 5;
22676 6192099 : return ix86_vec_cost (mode, cost->sse_op * count) + extra;
22677 :
22678 3099146 : case V16QImode:
22679 3099146 : if (TARGET_XOP)
22680 : {
22681 : /* For XOP we use vpshab, which requires a broadcast of the
22682 : value to the variable shift insn. For constants this
22683 : means a V16Q const in mem; even when we can perform the
22684 : shift with one insn set the cost to prefer paddb. */
22685 3573 : if (constant_op1)
22686 : {
22687 2614 : extra = COSTS_N_INSNS (cost->sse_load[2]) / 2;
22688 2614 : return ix86_vec_cost (mode, cost->sse_op) + extra;
22689 : }
22690 : else
22691 : {
22692 959 : count = (code == ASHIFT) ? 3 : 4;
22693 959 : return ix86_vec_cost (mode, cost->sse_op * count);
22694 : }
22695 : }
22696 : /* FALLTHRU */
22697 6191378 : case V32QImode:
22698 6191378 : if (TARGET_GFNI && constant_op1)
22699 : {
22700 : /* Use vgf2p8affine. One extra load for the mask, but in a loop
22701 : with enough registers it will be moved out. So for now don't
22702 : account the constant mask load. This is not quite right
22703 : for non loop vectorization. */
22704 12368 : extra = 0;
22705 12368 : return ix86_vec_cost (mode, cost->sse_op) + extra;
22706 : }
22707 6179010 : if (TARGET_AVX2)
22708 : /* Use vpbroadcast. */
22709 191318 : extra = cost->sse_op;
22710 : else
22711 5987692 : extra = COSTS_N_INSNS (mode == V16QImode
22712 : ? cost->sse_load[2]
22713 5987692 : : cost->sse_load[3]) / 2;
22714 :
22715 6179010 : if (constant_op1)
22716 : {
22717 6178778 : if (code == ASHIFTRT)
22718 : {
22719 381 : count = 4;
22720 381 : extra *= 2;
22721 : }
22722 : else
22723 : count = 2;
22724 : }
22725 232 : else if (TARGET_AVX512BW
22726 76 : && ((mode == V32QImode && !TARGET_PREFER_AVX256)
22727 38 : || (mode == V16QImode && TARGET_AVX512VL
22728 38 : && !TARGET_PREFER_AVX128)))
22729 76 : return ix86_vec_cost (mode, cost->sse_op * 4);
22730 156 : else if (TARGET_AVX2
22731 0 : && mode == V16QImode && !TARGET_PREFER_AVX128)
22732 : count = 6;
22733 156 : else if (TARGET_SSE4_1)
22734 : count = 9;
22735 148 : else if (code == ASHIFTRT)
22736 : count = 10;
22737 : else
22738 84 : count = 9;
22739 6178934 : return ix86_vec_cost (mode, cost->sse_op * count) + extra;
22740 :
22741 3095932 : case V64QImode:
22742 : /* Ignore the mask load for GF2P8AFFINEQB. */
22743 3095932 : extra = 0;
22744 3095932 : return ix86_vec_cost (mode, cost->sse_op) + extra;
22745 :
22746 55822937 : case V2DImode:
22747 55822937 : case V4DImode:
22748 : /* V*DImode arithmetic right shift is emulated. */
22749 55822937 : if (code == ASHIFTRT && !TARGET_AVX512VL)
22750 : {
22751 1522 : if (constant_op1)
22752 : {
22753 761 : if (op1_val == 63)
22754 490 : count = TARGET_SSE4_2 ? 1 : 2;
22755 570 : else if (TARGET_XOP)
22756 : count = 2;
22757 271 : else if (TARGET_SSE4_1)
22758 : count = 3;
22759 : else
22760 291 : count = 4;
22761 : }
22762 761 : else if (TARGET_XOP)
22763 : count = 3;
22764 96 : else if (TARGET_SSE4_2)
22765 : count = 4;
22766 : else
22767 1522 : count = 5;
22768 :
22769 1522 : return ix86_vec_cost (mode, cost->sse_op * count);
22770 : }
22771 : /* FALLTHRU */
22772 392503580 : default:
22773 392503580 : return ix86_vec_cost (mode, cost->sse_op);
22774 : }
22775 : }
22776 :
22777 781628900 : if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
22778 : {
22779 197654266 : if (constant_op1)
22780 : {
22781 197619325 : if (op1_val > 32)
22782 140407473 : return cost->shift_const + COSTS_N_INSNS (2);
22783 : else
22784 57211852 : return cost->shift_const * 2;
22785 : }
22786 : else
22787 : {
22788 34941 : if (and_in_op1)
22789 63 : return cost->shift_var * 2;
22790 : else
22791 34878 : return cost->shift_var * 6 + COSTS_N_INSNS (2);
22792 : }
22793 : }
22794 : else
22795 : {
22796 188794586 : if (constant_op1)
22797 188019804 : return cost->shift_const;
22798 774782 : else if (shift_and_truncate)
22799 : {
22800 24056 : if (skip_op0)
22801 24056 : *skip_op0 = *skip_op1 = true;
22802 : /* Return the cost after shift-and truncation. */
22803 24056 : return cost->shift_var;
22804 : }
22805 : else
22806 750726 : return cost->shift_var;
22807 : }
22808 : }
22809 :
22810 : static int
22811 157985231 : ix86_insn_cost (rtx_insn *insn, bool speed)
22812 : {
22813 157985231 : int insn_cost = 0;
22814 : /* Add extra cost to avoid post_reload late_combine revert
22815 : the optimization did in pass_rpad. */
22816 157985231 : if (reload_completed
22817 4613156 : && ix86_rpad_gate ()
22818 249350 : && recog_memoized (insn) >= 0
22819 158234260 : && get_attr_avx_partial_xmm_update (insn)
22820 : == AVX_PARTIAL_XMM_UPDATE_TRUE)
22821 : insn_cost += COSTS_N_INSNS (3);
22822 :
22823 157985231 : rtx pat = PATTERN (insn);
22824 : /* A USE of a memory is more expensive than a use of a REG.
22825 : For example *<absneg>mode2_1's use of a signbit mask. */
22826 157985231 : if (GET_CODE (pat) == PARALLEL)
22827 : {
22828 48645382 : for (int i = 0; i < XVECLEN (pat, 0); i++)
22829 : {
22830 32673351 : rtx x = XVECEXP (pat, 0, i);
22831 32673351 : if (GET_CODE (x) == USE && MEM_P (XEXP (x, 0)))
22832 63529 : insn_cost += !speed ? COSTS_N_BYTES (4)
22833 29796 : : TARGET_64BIT ? COSTS_N_INSNS (1) + 1
22834 : : COSTS_N_INSNS (3) + 1;
22835 : }
22836 : }
22837 : /* Cost *h{add,sub}<mode>[_low] directly as pattern cost for the
22838 : variants with outer vec_concat are artificially low. */
22839 157985231 : if (INSN_CODE (insn) >= 0
22840 157985231 : && get_attr_cost_special (insn) == COST_SPECIAL_HADDSUB)
22841 632 : return insn_cost + ix86_vec_cost (GET_MODE (pat),
22842 316 : (speed ? ix86_tune_cost
22843 316 : : &ix86_size_cost)->addss);
22844 :
22845 157984915 : return insn_cost + pattern_cost (pat, speed);
22846 : }
22847 :
22848 : /* Return cost of SSE/AVX FP->FP conversion (extensions and truncates). */
22849 :
22850 : static int
22851 778880 : vec_fp_conversion_cost (const struct processor_costs *cost, int size)
22852 : {
22853 778880 : if (size < 128)
22854 773650 : return cost->cvtss2sd;
22855 5230 : else if (size < 256)
22856 : {
22857 2445 : if (TARGET_SSE_SPLIT_REGS)
22858 0 : return cost->cvtss2sd * size / 64;
22859 2445 : return cost->cvtss2sd;
22860 : }
22861 2785 : if (size < 512)
22862 1483 : return cost->vcvtps2pd256;
22863 : else
22864 1302 : return cost->vcvtps2pd512;
22865 : }
22866 :
22867 : /* Return true of X is UNSPEC with UNSPEC_PCMP or UNSPEC_UNSIGNED_PCMP. */
22868 :
22869 : static bool
22870 299971 : unspec_pcmp_p (rtx x)
22871 : {
22872 299971 : return GET_CODE (x) == UNSPEC
22873 299971 : && (XINT (x, 1) == UNSPEC_PCMP || XINT (x, 1) == UNSPEC_UNSIGNED_PCMP);
22874 : }
22875 :
22876 : /* Compute a (partial) cost for rtx X. Return true if the complete
22877 : cost has been computed, and false if subexpressions should be
22878 : scanned. In either case, *TOTAL contains the cost result. */
22879 :
22880 : static bool
22881 7896605739 : ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
22882 : int *total, bool speed)
22883 : {
22884 7896605739 : rtx mask;
22885 7896605739 : enum rtx_code code = GET_CODE (x);
22886 7896605739 : enum rtx_code outer_code = (enum rtx_code) outer_code_i;
22887 4230429591 : const struct processor_costs *cost
22888 7896605739 : = speed ? ix86_tune_cost : &ix86_size_cost;
22889 7896605739 : int src_cost;
22890 :
22891 : /* Handling different vternlog variants. */
22892 7896605739 : if ((GET_MODE_SIZE (mode) == 64
22893 7896605739 : ? TARGET_AVX512F
22894 6683801830 : : (TARGET_AVX512VL
22895 6619761916 : || (TARGET_AVX512F && !TARGET_PREFER_AVX256)))
22896 179572817 : && GET_MODE_SIZE (mode) >= 16
22897 121993840 : && outer_code_i == SET
22898 7943734178 : && ternlog_operand (x, mode))
22899 : {
22900 33622 : rtx args[3];
22901 :
22902 33622 : args[0] = NULL_RTX;
22903 33622 : args[1] = NULL_RTX;
22904 33622 : args[2] = NULL_RTX;
22905 33622 : int idx = ix86_ternlog_idx (x, args);
22906 33622 : gcc_assert (idx >= 0);
22907 :
22908 33622 : *total = cost->sse_op;
22909 134488 : for (int i = 0; i != 3; i++)
22910 100866 : if (args[i])
22911 71132 : *total += rtx_cost (args[i], GET_MODE (args[i]), UNSPEC, i, speed);
22912 33622 : return true;
22913 : }
22914 :
22915 :
22916 7896572117 : switch (code)
22917 : {
22918 49696740 : case SET:
22919 49696740 : if (register_operand (SET_DEST (x), VOIDmode)
22920 49696740 : && register_operand (SET_SRC (x), VOIDmode))
22921 : {
22922 30090342 : *total = ix86_set_reg_reg_cost (GET_MODE (SET_DEST (x)));
22923 30090342 : return true;
22924 : }
22925 :
22926 19606398 : if (register_operand (SET_SRC (x), VOIDmode))
22927 : /* Avoid potentially incorrect high cost from rtx_costs
22928 : for non-tieable SUBREGs. */
22929 : src_cost = 0;
22930 : else
22931 : {
22932 16740735 : src_cost = rtx_cost (SET_SRC (x), mode, SET, 1, speed);
22933 :
22934 16740735 : if (CONSTANT_P (SET_SRC (x)))
22935 : /* Constant costs assume a base value of COSTS_N_INSNS (1) and add
22936 : a small value, possibly zero for cheap constants. */
22937 7368155 : src_cost += COSTS_N_INSNS (1);
22938 : }
22939 :
22940 19606398 : *total = src_cost + rtx_cost (SET_DEST (x), mode, SET, 0, speed);
22941 19606398 : return true;
22942 :
22943 2908097480 : case CONST_INT:
22944 2908097480 : case CONST:
22945 2908097480 : case LABEL_REF:
22946 2908097480 : case SYMBOL_REF:
22947 2908097480 : if (x86_64_immediate_operand (x, VOIDmode))
22948 2286251538 : *total = 0;
22949 621845942 : else if (TARGET_64BIT && x86_64_zext_immediate_operand (x, VOIDmode))
22950 : /* Consider the zext constants slightly more expensive, as they
22951 : can't appear in most instructions. */
22952 28720679 : *total = 1;
22953 : else
22954 : /* movabsq is slightly more expensive than a simple instruction. */
22955 593125263 : *total = COSTS_N_INSNS (1) + 1;
22956 : return true;
22957 :
22958 7700238 : case CONST_DOUBLE:
22959 7700238 : if (IS_STACK_MODE (mode))
22960 1295768 : switch (standard_80387_constant_p (x))
22961 : {
22962 : case -1:
22963 : case 0:
22964 : break;
22965 274842 : case 1: /* 0.0 */
22966 274842 : *total = 1;
22967 274842 : return true;
22968 469112 : default: /* Other constants */
22969 469112 : *total = 2;
22970 469112 : return true;
22971 : }
22972 : /* FALLTHRU */
22973 :
22974 15196511 : case CONST_VECTOR:
22975 15196511 : switch (standard_sse_constant_p (x, mode))
22976 : {
22977 : case 0:
22978 : break;
22979 4504137 : case 1: /* 0: xor eliminates false dependency */
22980 4504137 : *total = 0;
22981 4504137 : return true;
22982 211476 : default: /* -1: cmp contains false dependency */
22983 211476 : *total = 1;
22984 211476 : return true;
22985 : }
22986 : /* FALLTHRU */
22987 :
22988 11513622 : case CONST_WIDE_INT:
22989 : /* Fall back to (MEM (SYMBOL_REF)), since that's where
22990 : it'll probably end up. Add a penalty for size. */
22991 23027244 : *total = (COSTS_N_INSNS (1)
22992 22798371 : + (!TARGET_64BIT && flag_pic)
22993 23027244 : + (GET_MODE_SIZE (mode) <= 4
22994 20170227 : ? 0 : GET_MODE_SIZE (mode) <= 8 ? 1 : 2));
22995 11513622 : return true;
22996 :
22997 23175366 : case ZERO_EXTEND:
22998 : /* The zero extensions is often completely free on x86_64, so make
22999 : it as cheap as possible. */
23000 23175366 : if (TARGET_64BIT && mode == DImode
23001 5040292 : && GET_MODE (XEXP (x, 0)) == SImode)
23002 3081017 : *total = 1;
23003 20094349 : else if (TARGET_ZERO_EXTEND_WITH_AND)
23004 0 : *total = cost->add;
23005 : else
23006 20094349 : *total = cost->movzx;
23007 : return false;
23008 :
23009 2872517 : case SIGN_EXTEND:
23010 2872517 : *total = cost->movsx;
23011 2872517 : return false;
23012 :
23013 653479454 : case ASHIFT:
23014 653479454 : if (SCALAR_INT_MODE_P (mode)
23015 252445898 : && GET_MODE_SIZE (mode) < UNITS_PER_WORD
23016 697674857 : && CONST_INT_P (XEXP (x, 1)))
23017 : {
23018 44006556 : HOST_WIDE_INT value = INTVAL (XEXP (x, 1));
23019 44006556 : if (value == 1)
23020 : {
23021 2540211 : *total = cost->add;
23022 2540211 : return false;
23023 : }
23024 41466345 : if ((value == 2 || value == 3)
23025 4657776 : && cost->lea <= cost->shift_const)
23026 : {
23027 2200039 : *total = cost->lea;
23028 2200039 : return false;
23029 : }
23030 : }
23031 : /* FALLTHRU */
23032 :
23033 794352945 : case ROTATE:
23034 794352945 : case ASHIFTRT:
23035 794352945 : case LSHIFTRT:
23036 794352945 : case ROTATERT:
23037 794352945 : bool skip_op0, skip_op1;
23038 794352945 : *total = ix86_shift_rotate_cost (cost, code, mode,
23039 794352945 : CONSTANT_P (XEXP (x, 1)),
23040 : CONST_INT_P (XEXP (x, 1))
23041 : ? INTVAL (XEXP (x, 1)) : -1,
23042 : GET_CODE (XEXP (x, 1)) == AND,
23043 794352945 : SUBREG_P (XEXP (x, 1))
23044 675351 : && GET_CODE (XEXP (XEXP (x, 1),
23045 : 0)) == AND,
23046 : &skip_op0, &skip_op1);
23047 794352945 : if (skip_op0 || skip_op1)
23048 : {
23049 24056 : if (!skip_op0)
23050 0 : *total += rtx_cost (XEXP (x, 0), mode, code, 0, speed);
23051 24056 : if (!skip_op1)
23052 0 : *total += rtx_cost (XEXP (x, 1), mode, code, 0, speed);
23053 : return true;
23054 : }
23055 : return false;
23056 :
23057 228384 : case FMA:
23058 228384 : {
23059 228384 : rtx sub;
23060 :
23061 228384 : gcc_assert (FLOAT_MODE_P (mode));
23062 228384 : gcc_assert (TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F);
23063 :
23064 456768 : *total = ix86_vec_cost (mode,
23065 228384 : GET_MODE_INNER (mode) == SFmode
23066 : ? cost->fmass : cost->fmasd);
23067 228384 : *total += rtx_cost (XEXP (x, 1), mode, FMA, 1, speed);
23068 :
23069 : /* Negate in op0 or op2 is free: FMS, FNMA, FNMS. */
23070 228384 : sub = XEXP (x, 0);
23071 228384 : if (GET_CODE (sub) == NEG)
23072 51259 : sub = XEXP (sub, 0);
23073 228384 : *total += rtx_cost (sub, mode, FMA, 0, speed);
23074 :
23075 228384 : sub = XEXP (x, 2);
23076 228384 : if (GET_CODE (sub) == NEG)
23077 40590 : sub = XEXP (sub, 0);
23078 228384 : *total += rtx_cost (sub, mode, FMA, 2, speed);
23079 228384 : return true;
23080 : }
23081 :
23082 1801843990 : case MULT:
23083 1801843990 : if (!FLOAT_MODE_P (mode) && !VECTOR_MODE_P (mode))
23084 : {
23085 565079678 : rtx op0 = XEXP (x, 0);
23086 565079678 : rtx op1 = XEXP (x, 1);
23087 565079678 : int nbits;
23088 565079678 : if (CONST_INT_P (XEXP (x, 1)))
23089 : {
23090 546393877 : unsigned HOST_WIDE_INT value = INTVAL (XEXP (x, 1));
23091 1108588077 : for (nbits = 0; value != 0; value &= value - 1)
23092 562194200 : nbits++;
23093 : }
23094 : else
23095 : /* This is arbitrary. */
23096 : nbits = 7;
23097 :
23098 : /* Compute costs correctly for widening multiplication. */
23099 565079678 : if ((GET_CODE (op0) == SIGN_EXTEND || GET_CODE (op0) == ZERO_EXTEND)
23100 570787639 : && GET_MODE_SIZE (GET_MODE (XEXP (op0, 0))) * 2
23101 5707961 : == GET_MODE_SIZE (mode))
23102 : {
23103 5692553 : int is_mulwiden = 0;
23104 5692553 : machine_mode inner_mode = GET_MODE (op0);
23105 :
23106 5692553 : if (GET_CODE (op0) == GET_CODE (op1))
23107 5589191 : is_mulwiden = 1, op1 = XEXP (op1, 0);
23108 103362 : else if (CONST_INT_P (op1))
23109 : {
23110 93224 : if (GET_CODE (op0) == SIGN_EXTEND)
23111 40115 : is_mulwiden = trunc_int_for_mode (INTVAL (op1), inner_mode)
23112 40115 : == INTVAL (op1);
23113 : else
23114 53109 : is_mulwiden = !(INTVAL (op1) & ~GET_MODE_MASK (inner_mode));
23115 : }
23116 :
23117 5682415 : if (is_mulwiden)
23118 5682415 : op0 = XEXP (op0, 0), mode = GET_MODE (op0);
23119 : }
23120 :
23121 565079678 : int mult_init;
23122 : // Double word multiplication requires 3 mults and 2 adds.
23123 1145895947 : if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
23124 : {
23125 340041769 : mult_init = 3 * cost->mult_init[MODE_INDEX (word_mode)]
23126 340041769 : + 2 * cost->add;
23127 340041769 : nbits *= 3;
23128 : }
23129 388169218 : else mult_init = cost->mult_init[MODE_INDEX (mode)];
23130 :
23131 1130159356 : *total = (mult_init
23132 565079678 : + nbits * cost->mult_bit
23133 565079678 : + rtx_cost (op0, mode, outer_code, opno, speed)
23134 565079678 : + rtx_cost (op1, mode, outer_code, opno, speed));
23135 :
23136 565079678 : return true;
23137 : }
23138 1236764312 : *total = ix86_multiplication_cost (cost, mode);
23139 1236764312 : return false;
23140 :
23141 74828408 : case DIV:
23142 74828408 : case UDIV:
23143 74828408 : case MOD:
23144 74828408 : case UMOD:
23145 74828408 : *total = ix86_division_cost (cost, mode);
23146 74828408 : return false;
23147 :
23148 706008760 : case PLUS:
23149 706008760 : if (GET_MODE_CLASS (mode) == MODE_INT
23150 967626473 : && GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
23151 : {
23152 145806871 : if (GET_CODE (XEXP (x, 0)) == PLUS
23153 3483464 : && GET_CODE (XEXP (XEXP (x, 0), 0)) == MULT
23154 811484 : && CONST_INT_P (XEXP (XEXP (XEXP (x, 0), 0), 1))
23155 811475 : && CONSTANT_P (XEXP (x, 1)))
23156 : {
23157 811443 : HOST_WIDE_INT val = INTVAL (XEXP (XEXP (XEXP (x, 0), 0), 1));
23158 811443 : if (val == 2 || val == 4 || val == 8)
23159 : {
23160 811341 : *total = cost->lea;
23161 811341 : *total += rtx_cost (XEXP (XEXP (x, 0), 1), mode,
23162 : outer_code, opno, speed);
23163 811341 : *total += rtx_cost (XEXP (XEXP (XEXP (x, 0), 0), 0), mode,
23164 : outer_code, opno, speed);
23165 811341 : *total += rtx_cost (XEXP (x, 1), mode,
23166 : outer_code, opno, speed);
23167 811341 : return true;
23168 : }
23169 : }
23170 144995428 : else if (GET_CODE (XEXP (x, 0)) == MULT
23171 53561103 : && CONST_INT_P (XEXP (XEXP (x, 0), 1)))
23172 : {
23173 53497945 : HOST_WIDE_INT val = INTVAL (XEXP (XEXP (x, 0), 1));
23174 53497945 : if (val == 2 || val == 4 || val == 8)
23175 : {
23176 8088438 : *total = cost->lea;
23177 8088438 : *total += rtx_cost (XEXP (XEXP (x, 0), 0), mode,
23178 : outer_code, opno, speed);
23179 8088438 : *total += rtx_cost (XEXP (x, 1), mode,
23180 : outer_code, opno, speed);
23181 8088438 : return true;
23182 : }
23183 : }
23184 91497483 : else if (GET_CODE (XEXP (x, 0)) == PLUS)
23185 : {
23186 2672021 : rtx op = XEXP (XEXP (x, 0), 0);
23187 :
23188 : /* Add with carry, ignore the cost of adding a carry flag. */
23189 2672021 : if (ix86_carry_flag_operator (op, mode)
23190 2672021 : || ix86_carry_flag_unset_operator (op, mode))
23191 69046 : *total = cost->add;
23192 : else
23193 : {
23194 2602975 : *total = cost->lea;
23195 2602975 : *total += rtx_cost (op, mode,
23196 : outer_code, opno, speed);
23197 : }
23198 :
23199 2672021 : *total += rtx_cost (XEXP (XEXP (x, 0), 1), mode,
23200 : outer_code, opno, speed);
23201 2672021 : *total += rtx_cost (XEXP (x, 1), mode,
23202 : outer_code, opno, speed);
23203 2672021 : return true;
23204 : }
23205 : }
23206 : /* FALLTHRU */
23207 :
23208 1881625043 : case MINUS:
23209 : /* Subtract with borrow, ignore the cost of subtracting a carry flag. */
23210 1881625043 : if (GET_MODE_CLASS (mode) == MODE_INT
23211 531807722 : && GET_MODE_SIZE (mode) <= UNITS_PER_WORD
23212 240989937 : && GET_CODE (XEXP (x, 0)) == MINUS
23213 1881664151 : && (ix86_carry_flag_operator (XEXP (XEXP (x, 0), 1), mode)
23214 15447 : || ix86_carry_flag_unset_operator (XEXP (XEXP (x, 0), 1), mode)))
23215 : {
23216 23661 : *total = cost->add;
23217 23661 : *total += rtx_cost (XEXP (XEXP (x, 0), 0), mode,
23218 : outer_code, opno, speed);
23219 23661 : *total += rtx_cost (XEXP (x, 1), mode,
23220 : outer_code, opno, speed);
23221 23661 : return true;
23222 : }
23223 :
23224 1881601382 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
23225 2434936 : *total = cost->addss;
23226 1879166446 : else if (X87_FLOAT_MODE_P (mode))
23227 224016 : *total = cost->fadd;
23228 1878942430 : else if (FLOAT_MODE_P (mode))
23229 455148 : *total = ix86_vec_cost (mode, cost->addss);
23230 1878487282 : else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
23231 1236542768 : *total = ix86_vec_cost (mode, cost->sse_op);
23232 1323394375 : else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
23233 339071638 : *total = cost->add * 2;
23234 : else
23235 302872876 : *total = cost->add;
23236 : return false;
23237 :
23238 3933007 : case IOR:
23239 3933007 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
23240 3680338 : || SSE_FLOAT_MODE_P (mode))
23241 : {
23242 : /* (ior (not ...) ...) can be a single insn in AVX512. */
23243 3091 : if (GET_CODE (XEXP (x, 0)) == NOT && TARGET_AVX512F
23244 266632 : && (GET_MODE_SIZE (mode) == 64
23245 0 : || (TARGET_AVX512VL
23246 0 : && (GET_MODE_SIZE (mode) == 32
23247 0 : || GET_MODE_SIZE (mode) == 16))))
23248 : {
23249 0 : rtx right = GET_CODE (XEXP (x, 1)) != NOT
23250 0 : ? XEXP (x, 1) : XEXP (XEXP (x, 1), 0);
23251 :
23252 0 : *total = ix86_vec_cost (mode, cost->sse_op)
23253 0 : + rtx_cost (XEXP (XEXP (x, 0), 0), mode,
23254 : outer_code, opno, speed)
23255 0 : + rtx_cost (right, mode, outer_code, opno, speed);
23256 0 : return true;
23257 : }
23258 266632 : *total = ix86_vec_cost (mode, cost->sse_op);
23259 266632 : }
23260 3666375 : else if (TARGET_64BIT
23261 3367020 : && mode == TImode
23262 1755784 : && GET_CODE (XEXP (x, 0)) == ASHIFT
23263 262157 : && GET_CODE (XEXP (XEXP (x, 0), 0)) == ZERO_EXTEND
23264 260144 : && GET_MODE (XEXP (XEXP (XEXP (x, 0), 0), 0)) == DImode
23265 260144 : && CONST_INT_P (XEXP (XEXP (x, 0), 1))
23266 260144 : && INTVAL (XEXP (XEXP (x, 0), 1)) == 64
23267 260144 : && GET_CODE (XEXP (x, 1)) == ZERO_EXTEND
23268 236779 : && GET_MODE (XEXP (XEXP (x, 1), 0)) == DImode)
23269 : {
23270 : /* *concatditi3 is cheap. */
23271 236779 : rtx op0 = XEXP (XEXP (XEXP (x, 0), 0), 0);
23272 236779 : rtx op1 = XEXP (XEXP (x, 1), 0);
23273 1574 : *total = (SUBREG_P (op0) && GET_MODE (SUBREG_REG (op0)) == DFmode)
23274 236779 : ? COSTS_N_INSNS (1) /* movq. */
23275 235205 : : set_src_cost (op0, DImode, speed);
23276 2506 : *total += (SUBREG_P (op1) && GET_MODE (SUBREG_REG (op1)) == DFmode)
23277 236779 : ? COSTS_N_INSNS (1) /* movq. */
23278 234286 : : set_src_cost (op1, DImode, speed);
23279 236779 : return true;
23280 : }
23281 3429596 : else if (TARGET_64BIT
23282 3130241 : && mode == TImode
23283 1519005 : && GET_CODE (XEXP (x, 0)) == AND
23284 1457030 : && REG_P (XEXP (XEXP (x, 0), 0))
23285 1451703 : && CONST_WIDE_INT_P (XEXP (XEXP (x, 0), 1))
23286 1448960 : && CONST_WIDE_INT_NUNITS (XEXP (XEXP (x, 0), 1)) == 2
23287 1448960 : && CONST_WIDE_INT_ELT (XEXP (XEXP (x, 0), 1), 0) == -1
23288 941449 : && CONST_WIDE_INT_ELT (XEXP (XEXP (x, 0), 1), 1) == 0
23289 941449 : && GET_CODE (XEXP (x, 1)) == ASHIFT
23290 938991 : && GET_CODE (XEXP (XEXP (x, 1), 0)) == ZERO_EXTEND
23291 938991 : && GET_MODE (XEXP (XEXP (XEXP (x, 1), 0), 0)) == DImode
23292 938991 : && CONST_INT_P (XEXP (XEXP (x, 1), 1))
23293 4368587 : && INTVAL (XEXP (XEXP (x, 1), 1)) == 64)
23294 : {
23295 : /* *insvti_highpart is cheap. */
23296 938991 : rtx op = XEXP (XEXP (XEXP (x, 1), 0), 0);
23297 938991 : *total = COSTS_N_INSNS (1) + 1;
23298 1564 : *total += (SUBREG_P (op) && GET_MODE (SUBREG_REG (op)) == DFmode)
23299 938991 : ? COSTS_N_INSNS (1) /* movq. */
23300 937951 : : set_src_cost (op, DImode, speed);
23301 938991 : return true;
23302 : }
23303 5280565 : else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
23304 778627 : *total = cost->add * 2;
23305 : else
23306 1711978 : *total = cost->add;
23307 : return false;
23308 :
23309 618940 : case XOR:
23310 618940 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
23311 461686 : || SSE_FLOAT_MODE_P (mode))
23312 157254 : *total = ix86_vec_cost (mode, cost->sse_op);
23313 984789 : else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
23314 17837 : *total = cost->add * 2;
23315 : else
23316 443849 : *total = cost->add;
23317 : return false;
23318 :
23319 6741839 : case AND:
23320 6741839 : if (address_no_seg_operand (x, mode))
23321 : {
23322 13895 : *total = cost->lea;
23323 13895 : return true;
23324 : }
23325 6727944 : else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
23326 6277858 : || SSE_FLOAT_MODE_P (mode))
23327 : {
23328 : /* pandn is a single instruction. */
23329 492820 : if (GET_CODE (XEXP (x, 0)) == NOT)
23330 : {
23331 64666 : rtx right = XEXP (x, 1);
23332 :
23333 : /* (and (not ...) (not ...)) can be a single insn in AVX512. */
23334 180 : if (GET_CODE (right) == NOT && TARGET_AVX512F
23335 64666 : && (GET_MODE_SIZE (mode) == 64
23336 0 : || (TARGET_AVX512VL
23337 0 : && (GET_MODE_SIZE (mode) == 32
23338 0 : || GET_MODE_SIZE (mode) == 16))))
23339 0 : right = XEXP (right, 0);
23340 :
23341 64666 : *total = ix86_vec_cost (mode, cost->sse_op)
23342 64666 : + rtx_cost (XEXP (XEXP (x, 0), 0), mode,
23343 : outer_code, opno, speed)
23344 64666 : + rtx_cost (right, mode, outer_code, opno, speed);
23345 64666 : return true;
23346 : }
23347 428154 : else if (GET_CODE (XEXP (x, 1)) == NOT)
23348 : {
23349 2770 : *total = ix86_vec_cost (mode, cost->sse_op)
23350 2770 : + rtx_cost (XEXP (x, 0), mode,
23351 : outer_code, opno, speed)
23352 2770 : + rtx_cost (XEXP (XEXP (x, 1), 0), mode,
23353 : outer_code, opno, speed);
23354 2770 : return true;
23355 : }
23356 425384 : *total = ix86_vec_cost (mode, cost->sse_op);
23357 425384 : }
23358 13205845 : else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
23359 : {
23360 1178334 : if (TARGET_BMI && GET_CODE (XEXP (x,0)) == NOT)
23361 : {
23362 1670 : *total = cost->add * 2
23363 835 : + rtx_cost (XEXP (XEXP (x, 0), 0), mode,
23364 : outer_code, opno, speed)
23365 835 : + rtx_cost (XEXP (x, 1), mode,
23366 : outer_code, opno, speed);
23367 835 : return true;
23368 : }
23369 1177499 : else if (TARGET_BMI && GET_CODE (XEXP (x, 1)) == NOT)
23370 : {
23371 0 : *total = cost->add * 2
23372 0 : + rtx_cost (XEXP (x, 0), mode,
23373 : outer_code, opno, speed)
23374 0 : + rtx_cost (XEXP (XEXP (x, 1), 0), mode,
23375 : outer_code, opno, speed);
23376 0 : return true;
23377 : }
23378 1177499 : *total = cost->add * 2;
23379 : }
23380 5056790 : else if (TARGET_BMI && GET_CODE (XEXP (x,0)) == NOT)
23381 : {
23382 7410 : *total = cost->add
23383 3705 : + rtx_cost (XEXP (XEXP (x, 0), 0), mode,
23384 : outer_code, opno, speed)
23385 3705 : + rtx_cost (XEXP (x, 1), mode, outer_code, opno, speed);
23386 3705 : return true;
23387 : }
23388 5053085 : else if (TARGET_BMI && GET_CODE (XEXP (x,1)) == NOT)
23389 : {
23390 112 : *total = cost->add
23391 56 : + rtx_cost (XEXP (x, 0), mode, outer_code, opno, speed)
23392 56 : + rtx_cost (XEXP (XEXP (x, 1), 0), mode,
23393 : outer_code, opno, speed);
23394 56 : return true;
23395 : }
23396 : else
23397 5053029 : *total = cost->add;
23398 : return false;
23399 :
23400 575413 : case NOT:
23401 575413 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
23402 : {
23403 : /* (not (xor ...)) can be a single insn in AVX512. */
23404 0 : if (GET_CODE (XEXP (x, 0)) == XOR && TARGET_AVX512F
23405 21796 : && (GET_MODE_SIZE (mode) == 64
23406 0 : || (TARGET_AVX512VL
23407 0 : && (GET_MODE_SIZE (mode) == 32
23408 0 : || GET_MODE_SIZE (mode) == 16))))
23409 : {
23410 0 : *total = ix86_vec_cost (mode, cost->sse_op)
23411 0 : + rtx_cost (XEXP (XEXP (x, 0), 0), mode,
23412 : outer_code, opno, speed)
23413 0 : + rtx_cost (XEXP (XEXP (x, 0), 1), mode,
23414 : outer_code, opno, speed);
23415 0 : return true;
23416 : }
23417 :
23418 : // vnot is pxor -1.
23419 21796 : *total = ix86_vec_cost (mode, cost->sse_op) + 1;
23420 : }
23421 1257396 : else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
23422 50237 : *total = cost->add * 2;
23423 : else
23424 503380 : *total = cost->add;
23425 : return false;
23426 :
23427 18729076 : case NEG:
23428 18729076 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
23429 54089 : *total = cost->sse_op;
23430 18674987 : else if (X87_FLOAT_MODE_P (mode))
23431 15497 : *total = cost->fchs;
23432 18659490 : else if (FLOAT_MODE_P (mode))
23433 27409 : *total = ix86_vec_cost (mode, cost->sse_op);
23434 18632081 : else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
23435 13721816 : *total = ix86_vec_cost (mode, cost->sse_op);
23436 9972221 : else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
23437 1806966 : *total = cost->add * 3;
23438 : else
23439 3103299 : *total = cost->add;
23440 : return false;
23441 :
23442 55826699 : case COMPARE:
23443 55826699 : rtx op0, op1;
23444 55826699 : op0 = XEXP (x, 0);
23445 55826699 : op1 = XEXP (x, 1);
23446 55826699 : if (GET_CODE (op0) == ZERO_EXTRACT
23447 99488 : && XEXP (op0, 1) == const1_rtx
23448 88319 : && CONST_INT_P (XEXP (op0, 2))
23449 88287 : && op1 == const0_rtx)
23450 : {
23451 : /* This kind of construct is implemented using test[bwl].
23452 : Treat it as if we had an AND. */
23453 88287 : mode = GET_MODE (XEXP (op0, 0));
23454 176574 : *total = (cost->add
23455 88287 : + rtx_cost (XEXP (op0, 0), mode, outer_code,
23456 : opno, speed)
23457 88287 : + rtx_cost (const1_rtx, mode, outer_code, opno, speed));
23458 88287 : return true;
23459 : }
23460 :
23461 55738412 : if (GET_CODE (op0) == PLUS && rtx_equal_p (XEXP (op0, 0), op1))
23462 : {
23463 : /* This is an overflow detection, count it as a normal compare. */
23464 146291 : *total = rtx_cost (op0, GET_MODE (op0), COMPARE, 0, speed);
23465 146291 : return true;
23466 : }
23467 :
23468 55592121 : rtx geu;
23469 : /* Match x
23470 : (compare:CCC (neg:QI (geu:QI (reg:CC_CCC FLAGS_REG) (const_int 0)))
23471 : (ltu:QI (reg:CC_CCC FLAGS_REG) (const_int 0))) */
23472 55592121 : if (mode == CCCmode
23473 312913 : && GET_CODE (op0) == NEG
23474 8174 : && GET_CODE (geu = XEXP (op0, 0)) == GEU
23475 8171 : && REG_P (XEXP (geu, 0))
23476 8171 : && (GET_MODE (XEXP (geu, 0)) == CCCmode
23477 769 : || GET_MODE (XEXP (geu, 0)) == CCmode)
23478 8171 : && REGNO (XEXP (geu, 0)) == FLAGS_REG
23479 8171 : && XEXP (geu, 1) == const0_rtx
23480 8171 : && GET_CODE (op1) == LTU
23481 8171 : && REG_P (XEXP (op1, 0))
23482 8171 : && GET_MODE (XEXP (op1, 0)) == GET_MODE (XEXP (geu, 0))
23483 8171 : && REGNO (XEXP (op1, 0)) == FLAGS_REG
23484 55600292 : && XEXP (op1, 1) == const0_rtx)
23485 : {
23486 : /* This is *setcc_qi_addqi3_cconly_overflow_1_* patterns, a nop. */
23487 8171 : *total = 0;
23488 8171 : return true;
23489 : }
23490 : /* Match x
23491 : (compare:CCC (neg:QI (ltu:QI (reg:CCC FLAGS_REG) (const_int 0)))
23492 : (geu:QI (reg:CCC FLAGS_REG) (const_int 0))) */
23493 55583950 : if (mode == CCCmode
23494 304742 : && GET_CODE (op0) == NEG
23495 3 : && GET_CODE (XEXP (op0, 0)) == LTU
23496 3 : && REG_P (XEXP (XEXP (op0, 0), 0))
23497 3 : && GET_MODE (XEXP (XEXP (op0, 0), 0)) == CCCmode
23498 3 : && REGNO (XEXP (XEXP (op0, 0), 0)) == FLAGS_REG
23499 3 : && XEXP (XEXP (op0, 0), 1) == const0_rtx
23500 3 : && GET_CODE (op1) == GEU
23501 3 : && REG_P (XEXP (op1, 0))
23502 3 : && GET_MODE (XEXP (op1, 0)) == CCCmode
23503 3 : && REGNO (XEXP (op1, 0)) == FLAGS_REG
23504 55583953 : && XEXP (op1, 1) == const0_rtx)
23505 : {
23506 : /* This is *x86_cmc. */
23507 3 : if (!speed)
23508 0 : *total = COSTS_N_BYTES (1);
23509 3 : else if (TARGET_SLOW_STC)
23510 0 : *total = COSTS_N_INSNS (2);
23511 : else
23512 3 : *total = COSTS_N_INSNS (1);
23513 : return true;
23514 : }
23515 :
23516 55583947 : if (SCALAR_INT_MODE_P (GET_MODE (op0))
23517 115629152 : && GET_MODE_SIZE (GET_MODE (op0)) > UNITS_PER_WORD)
23518 : {
23519 782969 : if (op1 == const0_rtx)
23520 109654 : *total = cost->add
23521 109654 : + rtx_cost (op0, GET_MODE (op0), outer_code, opno, speed);
23522 : else
23523 673315 : *total = 3*cost->add
23524 673315 : + rtx_cost (op0, GET_MODE (op0), outer_code, opno, speed)
23525 673315 : + rtx_cost (op1, GET_MODE (op0), outer_code, opno, speed);
23526 782969 : return true;
23527 : }
23528 :
23529 : /* The embedded comparison operand is completely free. */
23530 54800978 : if (!general_operand (op0, GET_MODE (op0)) && op1 == const0_rtx)
23531 379283 : *total = 0;
23532 :
23533 : return false;
23534 :
23535 1392289 : case FLOAT_EXTEND:
23536 : /* x87 represents all values extended to 80bit. */
23537 1392289 : if (!SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
23538 667824 : *total = 0;
23539 : else
23540 1448930 : *total = vec_fp_conversion_cost (cost, GET_MODE_BITSIZE (mode));
23541 : return false;
23542 :
23543 85765 : case FLOAT_TRUNCATE:
23544 85765 : if (!SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
23545 58691 : *total = cost->fadd;
23546 : else
23547 54148 : *total = vec_fp_conversion_cost (cost, GET_MODE_BITSIZE (mode));
23548 : return false;
23549 703456 : case FLOAT:
23550 703456 : case UNSIGNED_FLOAT:
23551 703456 : if (!SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
23552 : /* TODO: We do not have cost tables for x87. */
23553 95760 : *total = cost->fadd;
23554 607696 : else if (VECTOR_MODE_P (mode))
23555 0 : *total = ix86_vec_cost (mode, cost->cvtpi2ps);
23556 : else
23557 607696 : *total = cost->cvtsi2ss;
23558 : return false;
23559 :
23560 290251 : case FIX:
23561 290251 : case UNSIGNED_FIX:
23562 290251 : if (!SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
23563 : /* TODO: We do not have cost tables for x87. */
23564 290251 : *total = cost->fadd;
23565 0 : else if (VECTOR_MODE_P (mode))
23566 0 : *total = ix86_vec_cost (mode, cost->cvtps2pi);
23567 : else
23568 0 : *total = cost->cvtss2si;
23569 : return false;
23570 :
23571 417450 : case ABS:
23572 : /* SSE requires memory load for the constant operand. It may make
23573 : sense to account for this. Of course the constant operand may or
23574 : may not be reused. */
23575 417450 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
23576 296741 : *total = cost->sse_op;
23577 120709 : else if (X87_FLOAT_MODE_P (mode))
23578 36906 : *total = cost->fabs;
23579 83803 : else if (FLOAT_MODE_P (mode))
23580 33864 : *total = ix86_vec_cost (mode, cost->sse_op);
23581 49939 : else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
23582 6657 : *total = cost->sse_op;
23583 : return false;
23584 :
23585 29325 : case SQRT:
23586 29325 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
23587 18880 : *total = mode == SFmode ? cost->sqrtss : cost->sqrtsd;
23588 10445 : else if (X87_FLOAT_MODE_P (mode))
23589 4402 : *total = cost->fsqrt;
23590 6043 : else if (FLOAT_MODE_P (mode))
23591 6043 : *total = ix86_vec_cost (mode,
23592 : mode == SFmode ? cost->sqrtss : cost->sqrtsd);
23593 : return false;
23594 :
23595 4068946 : case UNSPEC:
23596 4068946 : switch (XINT (x, 1))
23597 : {
23598 126637 : case UNSPEC_TP:
23599 126637 : *total = 0;
23600 126637 : break;
23601 :
23602 5210 : case UNSPEC_VTERNLOG:
23603 5210 : *total = cost->sse_op;
23604 5210 : if (!REG_P (XVECEXP (x, 0, 0)))
23605 720 : *total += rtx_cost (XVECEXP (x, 0, 0), mode, code, 0, speed);
23606 5210 : if (!REG_P (XVECEXP (x, 0, 1)))
23607 694 : *total += rtx_cost (XVECEXP (x, 0, 1), mode, code, 1, speed);
23608 5210 : if (!REG_P (XVECEXP (x, 0, 2)))
23609 733 : *total += rtx_cost (XVECEXP (x, 0, 2), mode, code, 2, speed);
23610 : return true;
23611 :
23612 94907 : case UNSPEC_PTEST:
23613 94907 : {
23614 94907 : *total = cost->sse_op;
23615 94907 : rtx test_op0 = XVECEXP (x, 0, 0);
23616 94907 : if (!rtx_equal_p (test_op0, XVECEXP (x, 0, 1)))
23617 : return false;
23618 94259 : if (GET_CODE (test_op0) == AND)
23619 : {
23620 23 : rtx and_op0 = XEXP (test_op0, 0);
23621 23 : if (GET_CODE (and_op0) == NOT)
23622 0 : and_op0 = XEXP (and_op0, 0);
23623 23 : *total += rtx_cost (and_op0, GET_MODE (and_op0),
23624 : AND, 0, speed)
23625 23 : + rtx_cost (XEXP (test_op0, 1), GET_MODE (and_op0),
23626 : AND, 1, speed);
23627 : }
23628 : else
23629 94236 : *total = rtx_cost (test_op0, GET_MODE (test_op0),
23630 : UNSPEC, 0, speed);
23631 : }
23632 : return true;
23633 :
23634 20388 : case UNSPEC_BLENDV:
23635 20388 : *total = cost->sse_op;
23636 20388 : if (!REG_P (XVECEXP (x, 0, 0)))
23637 8517 : *total += rtx_cost (XVECEXP (x, 0, 0), mode, code, 0, speed);
23638 20388 : if (!REG_P (XVECEXP (x, 0, 1)))
23639 9982 : *total += rtx_cost (XVECEXP (x, 0, 1), mode, code, 1, speed);
23640 20388 : if (!REG_P (XVECEXP (x, 0, 2)))
23641 : {
23642 12933 : rtx cond = XVECEXP (x, 0, 2);
23643 12933 : if ((GET_CODE (cond) == LT || GET_CODE (cond) == GT)
23644 773 : && CONST_VECTOR_P (XEXP (cond, 1)))
23645 : {
23646 : /* avx2_blendvpd256_gt and friends. */
23647 153 : if (!REG_P (XEXP (cond, 0)))
23648 70 : *total += rtx_cost (XEXP (cond, 0), mode, code, 2, speed);
23649 : }
23650 : else
23651 12780 : *total += rtx_cost (cond, mode, code, 2, speed);
23652 : }
23653 : return true;
23654 :
23655 57469 : case UNSPEC_MOVMSK:
23656 57469 : *total = cost->sse_op;
23657 57469 : return true;
23658 :
23659 : default:
23660 : break;
23661 : }
23662 : return false;
23663 :
23664 1824608 : case VEC_CONCAT:
23665 : /* ??? Assume all of these vector manipulation patterns are
23666 : recognizable. In which case they all pretty much have the
23667 : same cost.
23668 : ??? We should still recruse when computing cost. */
23669 1824608 : *total = cost->sse_op;
23670 1824608 : return true;
23671 :
23672 2495160 : case VEC_SELECT:
23673 : /* Special case extracting lower part from the vector.
23674 : This by itself needs to code and most of SSE/AVX instructions have
23675 : packed and single forms where the single form may be represented
23676 : by such VEC_SELECT.
23677 :
23678 : Use cost 1 (despite the fact that functionally equivalent SUBREG has
23679 : cost 0). Making VEC_SELECT completely free, for example instructs CSE
23680 : to forward propagate VEC_SELECT into
23681 :
23682 : (set (reg eax) (reg src))
23683 :
23684 : which then prevents fwprop and combining. See i.e.
23685 : gcc.target/i386/pr91103-1.c.
23686 :
23687 : ??? rtvec_series_p test should be, for valid patterns, equivalent to
23688 : vec_series_lowpart_p but is not, since the latter calls
23689 : can_cange_mode_class on ALL_REGS and this return false since x87 does
23690 : not support subregs at all. */
23691 2495160 : if (rtvec_series_p (XVEC (XEXP (x, 1), 0), 0))
23692 685173 : *total = rtx_cost (XEXP (x, 0), GET_MODE (XEXP (x, 0)),
23693 685173 : outer_code, opno, speed) + 1;
23694 : else
23695 : /* ??? We should still recruse when computing cost. */
23696 1809987 : *total = cost->sse_op;
23697 : return true;
23698 :
23699 1287656 : case VEC_DUPLICATE:
23700 2575312 : *total = rtx_cost (XEXP (x, 0),
23701 1287656 : GET_MODE (XEXP (x, 0)),
23702 : VEC_DUPLICATE, 0, speed);
23703 : /* It's broadcast instruction, not embedded broadcasting. */
23704 1287656 : if (outer_code == SET)
23705 1213172 : *total += cost->sse_op;
23706 :
23707 : return true;
23708 :
23709 935035 : case VEC_MERGE:
23710 935035 : mask = XEXP (x, 2);
23711 : /* Scalar versions of SSE instructions may be represented as:
23712 :
23713 : (vec_merge (vec_duplicate (operation ....))
23714 : (register or memory)
23715 : (const_int 1))
23716 :
23717 : In this case vec_merge and vec_duplicate is for free.
23718 : Just recurse into operation and second operand. */
23719 935035 : if (mask == const1_rtx
23720 394512 : && GET_CODE (XEXP (x, 0)) == VEC_DUPLICATE)
23721 : {
23722 254462 : *total = rtx_cost (XEXP (XEXP (x, 0), 0), mode,
23723 : outer_code, opno, speed)
23724 254462 : + rtx_cost (XEXP (x, 1), mode, outer_code, opno, speed);
23725 254462 : return true;
23726 : }
23727 : /* This is masked instruction, assume the same cost,
23728 : as nonmasked variant. */
23729 680573 : else if (TARGET_AVX512F
23730 680573 : && (register_operand (mask, GET_MODE (mask))
23731 : /* Redunduant clean up of high bits for kmask with VL=2/4
23732 : .i.e (vec_merge op0, op1, (and op3 15)). */
23733 124633 : || (GET_CODE (mask) == AND
23734 403 : && register_operand (XEXP (mask, 0), GET_MODE (mask))
23735 403 : && CONST_INT_P (XEXP (mask, 1))
23736 403 : && ((INTVAL (XEXP (mask, 1)) == 3
23737 131 : && GET_MODE_NUNITS (mode) == 2)
23738 272 : || (INTVAL (XEXP (mask, 1)) == 15
23739 272 : && GET_MODE_NUNITS (mode) == 4)))))
23740 : {
23741 378352 : *total = rtx_cost (XEXP (x, 0), mode, outer_code, opno, speed)
23742 378352 : + rtx_cost (XEXP (x, 1), mode, outer_code, opno, speed);
23743 378352 : return true;
23744 : }
23745 : /* Combination of the two above:
23746 :
23747 : (vec_merge (vec_merge (vec_duplicate (operation ...))
23748 : (register or memory)
23749 : (reg:QI mask))
23750 : (register or memory)
23751 : (const_int 1))
23752 :
23753 : i.e. avx512fp16_vcvtss2sh_mask. */
23754 302221 : else if (TARGET_AVX512F
23755 124230 : && mask == const1_rtx
23756 46033 : && GET_CODE (XEXP (x, 0)) == VEC_MERGE
23757 27126 : && GET_CODE (XEXP (XEXP (x, 0), 0)) == VEC_DUPLICATE
23758 304483 : && register_operand (XEXP (XEXP (x, 0), 2),
23759 2262 : GET_MODE (XEXP (XEXP (x, 0), 2))))
23760 : {
23761 2250 : *total = rtx_cost (XEXP (XEXP (XEXP (x, 0), 0), 0),
23762 : mode, outer_code, opno, speed)
23763 2250 : + rtx_cost (XEXP (XEXP (x, 0), 1),
23764 : mode, outer_code, opno, speed)
23765 2250 : + rtx_cost (XEXP (x, 1), mode, outer_code, opno, speed);
23766 2250 : return true;
23767 : }
23768 : /* vcmp. */
23769 299971 : else if (unspec_pcmp_p (mask)
23770 299971 : || (GET_CODE (mask) == NOT
23771 0 : && unspec_pcmp_p (XEXP (mask, 0))))
23772 : {
23773 1950 : rtx uns = GET_CODE (mask) == NOT ? XEXP (mask, 0) : mask;
23774 1950 : rtx unsop0 = XVECEXP (uns, 0, 0);
23775 : /* Make (subreg:V4SI (not:V16QI (reg:V16QI ..)) 0)
23776 : cost the same as register.
23777 : This is used by avx_cmp<mode>3_ltint_not. */
23778 1950 : if (SUBREG_P (unsop0))
23779 417 : unsop0 = XEXP (unsop0, 0);
23780 1950 : if (GET_CODE (unsop0) == NOT)
23781 18 : unsop0 = XEXP (unsop0, 0);
23782 1950 : *total = rtx_cost (XEXP (x, 0), mode, outer_code, opno, speed)
23783 1950 : + rtx_cost (XEXP (x, 1), mode, outer_code, opno, speed)
23784 1950 : + rtx_cost (unsop0, mode, UNSPEC, opno, speed)
23785 1950 : + rtx_cost (XVECEXP (uns, 0, 1), mode, UNSPEC, opno, speed)
23786 1950 : + cost->sse_op;
23787 1950 : return true;
23788 : }
23789 : else
23790 298021 : *total = cost->sse_op;
23791 298021 : return false;
23792 :
23793 110391589 : case MEM:
23794 : /* CONST_VECTOR_DUPLICATE_P in constant_pool is just broadcast.
23795 : or variants in ix86_vector_duplicate_simode_const. */
23796 :
23797 110391589 : if (GET_MODE_SIZE (mode) >= 16
23798 17715761 : && VECTOR_MODE_P (mode)
23799 11568076 : && SYMBOL_REF_P (XEXP (x, 0))
23800 2324656 : && CONSTANT_POOL_ADDRESS_P (XEXP (x, 0))
23801 112486597 : && ix86_broadcast_from_constant (mode, x))
23802 : {
23803 535723 : *total = COSTS_N_INSNS (2) + speed;
23804 535723 : return true;
23805 : }
23806 :
23807 : /* An insn that accesses memory is slightly more expensive
23808 : than one that does not. */
23809 109855866 : if (speed)
23810 : {
23811 98344098 : *total += 1;
23812 98344098 : rtx addr = XEXP (x, 0);
23813 : /* For MEM, rtx_cost iterates each subrtx, and adds up the costs,
23814 : so for MEM (reg) and MEM (reg + 4), the former costs 5,
23815 : the latter costs 9, it is not accurate for x86. Ideally
23816 : address_cost should be used, but it reduce cost too much.
23817 : So current solution is make constant disp as cheap as possible. */
23818 98344098 : if (GET_CODE (addr) == PLUS
23819 79725246 : && x86_64_immediate_operand (XEXP (addr, 1), Pmode)
23820 : /* Only handle (reg + disp) since other forms of addr are mostly LEA,
23821 : there's no additional cost for the plus of disp. */
23822 172622703 : && register_operand (XEXP (addr, 0), Pmode))
23823 : {
23824 58412274 : *total += 1;
23825 71214481 : *total += rtx_cost (XEXP (addr, 0), Pmode, PLUS, 0, speed);
23826 58412274 : return true;
23827 : }
23828 : }
23829 :
23830 : return false;
23831 :
23832 46558 : case ZERO_EXTRACT:
23833 46558 : if (XEXP (x, 1) == const1_rtx
23834 11440 : && GET_CODE (XEXP (x, 2)) == ZERO_EXTEND
23835 0 : && GET_MODE (XEXP (x, 2)) == SImode
23836 0 : && GET_MODE (XEXP (XEXP (x, 2), 0)) == QImode)
23837 : {
23838 : /* Ignore cost of zero extension and masking of last argument. */
23839 0 : *total += rtx_cost (XEXP (x, 0), mode, code, 0, speed);
23840 0 : *total += rtx_cost (XEXP (x, 1), mode, code, 1, speed);
23841 0 : *total += rtx_cost (XEXP (XEXP (x, 2), 0), mode, code, 2, speed);
23842 0 : return true;
23843 : }
23844 : return false;
23845 :
23846 30328786 : case IF_THEN_ELSE:
23847 30328786 : if (TARGET_XOP
23848 25149 : && VECTOR_MODE_P (mode)
23849 30334177 : && (GET_MODE_SIZE (mode) == 16 || GET_MODE_SIZE (mode) == 32))
23850 : {
23851 : /* vpcmov. */
23852 4823 : *total = speed ? COSTS_N_INSNS (2) : COSTS_N_BYTES (6);
23853 4823 : if (!REG_P (XEXP (x, 0)))
23854 4663 : *total += rtx_cost (XEXP (x, 0), mode, code, 0, speed);
23855 4823 : if (!REG_P (XEXP (x, 1)))
23856 4630 : *total += rtx_cost (XEXP (x, 1), mode, code, 1, speed);
23857 4823 : if (!REG_P (XEXP (x, 2)))
23858 4632 : *total += rtx_cost (XEXP (x, 2), mode, code, 2, speed);
23859 : return true;
23860 : }
23861 0 : else if (TARGET_CMOVE
23862 30323963 : && SCALAR_INT_MODE_P (mode)
23863 33130813 : && GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
23864 : {
23865 : /* cmov. */
23866 2577564 : *total = COSTS_N_INSNS (1);
23867 2577564 : if (!COMPARISON_P (XEXP (x, 0)) && !REG_P (XEXP (x, 0)))
23868 0 : *total += rtx_cost (XEXP (x, 0), mode, code, 0, speed);
23869 2577564 : if (!REG_P (XEXP (x, 1)))
23870 145681 : *total += rtx_cost (XEXP (x, 1), mode, code, 1, speed);
23871 2577564 : if (!REG_P (XEXP (x, 2)))
23872 747040 : *total += rtx_cost (XEXP (x, 2), mode, code, 2, speed);
23873 : return true;
23874 : }
23875 : return false;
23876 :
23877 18677954 : case EQ:
23878 18677954 : case GT:
23879 18677954 : case GTU:
23880 18677954 : case LT:
23881 18677954 : case LTU:
23882 18677954 : if (TARGET_SSE2
23883 18674754 : && GET_MODE_CLASS (mode) == MODE_VECTOR_INT
23884 19053858 : && GET_MODE_SIZE (mode) >= 8)
23885 : {
23886 : /* vpcmpeq */
23887 368200 : *total = speed ? COSTS_N_INSNS (1) : COSTS_N_BYTES (4);
23888 368200 : if (!REG_P (XEXP (x, 0)))
23889 71340 : *total += rtx_cost (XEXP (x, 0), mode, code, 0, speed);
23890 368200 : if (!REG_P (XEXP (x, 1)))
23891 139713 : *total += rtx_cost (XEXP (x, 1), mode, code, 1, speed);
23892 : return true;
23893 : }
23894 18309754 : if (TARGET_XOP
23895 12236 : && GET_MODE_CLASS (mode) == MODE_VECTOR_INT
23896 18309862 : && GET_MODE_SIZE (mode) <= 16)
23897 : {
23898 : /* vpcomeq */
23899 108 : *total = speed ? COSTS_N_INSNS (1) : COSTS_N_BYTES (6);
23900 108 : if (!REG_P (XEXP (x, 0)))
23901 0 : *total += rtx_cost (XEXP (x, 0), mode, code, 0, speed);
23902 108 : if (!REG_P (XEXP (x, 1)))
23903 0 : *total += rtx_cost (XEXP (x, 1), mode, code, 1, speed);
23904 : return true;
23905 : }
23906 : return false;
23907 :
23908 16621386 : case NE:
23909 16621386 : case GE:
23910 16621386 : case GEU:
23911 16621386 : if (TARGET_XOP
23912 21827 : && GET_MODE_CLASS (mode) == MODE_VECTOR_INT
23913 16628053 : && GET_MODE_SIZE (mode) <= 16)
23914 : {
23915 : /* vpcomneq */
23916 6667 : *total = speed ? COSTS_N_INSNS (1) : COSTS_N_BYTES (6);
23917 6667 : if (!REG_P (XEXP (x, 0)))
23918 1405 : *total += rtx_cost (XEXP (x, 0), mode, code, 0, speed);
23919 6667 : if (!REG_P (XEXP (x, 1)))
23920 5683 : *total += rtx_cost (XEXP (x, 1), mode, code, 1, speed);
23921 : return true;
23922 : }
23923 16614719 : if (TARGET_SSE2
23924 16612567 : && GET_MODE_CLASS (mode) == MODE_VECTOR_INT
23925 16618027 : && GET_MODE_SIZE (mode) >= 8)
23926 : {
23927 3343 : if (TARGET_AVX512F && GET_MODE_SIZE (mode) >= 16)
23928 : /* vpcmpeq + vpternlog */
23929 51 : *total = speed ? COSTS_N_INSNS (2) : COSTS_N_BYTES (11);
23930 : else
23931 : /* vpcmpeq + pxor + vpcmpeq */
23932 3394 : *total = speed ? COSTS_N_INSNS (3) : COSTS_N_BYTES (12);
23933 3284 : if (!REG_P (XEXP (x, 0)))
23934 342 : *total += rtx_cost (XEXP (x, 0), mode, code, 0, speed);
23935 3284 : if (!REG_P (XEXP (x, 1)))
23936 45 : *total += rtx_cost (XEXP (x, 1), mode, code, 1, speed);
23937 : return true;
23938 : }
23939 : return false;
23940 :
23941 : default:
23942 : return false;
23943 : }
23944 : }
23945 :
23946 : #if TARGET_MACHO
23947 :
23948 : static int current_machopic_label_num;
23949 :
23950 : /* Given a symbol name and its associated stub, write out the
23951 : definition of the stub. */
23952 :
23953 : void
23954 : machopic_output_stub (FILE *file, const char *symb, const char *stub)
23955 : {
23956 : unsigned int length;
23957 : char *binder_name, *symbol_name, lazy_ptr_name[32];
23958 : int label = ++current_machopic_label_num;
23959 :
23960 : /* For 64-bit we shouldn't get here. */
23961 : gcc_assert (!TARGET_64BIT);
23962 :
23963 : /* Lose our funky encoding stuff so it doesn't contaminate the stub. */
23964 : symb = targetm.strip_name_encoding (symb);
23965 :
23966 : length = strlen (stub);
23967 : binder_name = XALLOCAVEC (char, length + 32);
23968 : GEN_BINDER_NAME_FOR_STUB (binder_name, stub, length);
23969 :
23970 : length = strlen (symb);
23971 : symbol_name = XALLOCAVEC (char, length + 32);
23972 : GEN_SYMBOL_NAME_FOR_SYMBOL (symbol_name, symb, length);
23973 :
23974 : sprintf (lazy_ptr_name, "L%d$lz", label);
23975 :
23976 : if (MACHOPIC_ATT_STUB)
23977 : switch_to_section (darwin_sections[machopic_picsymbol_stub3_section]);
23978 : else if (MACHOPIC_PURE)
23979 : switch_to_section (darwin_sections[machopic_picsymbol_stub2_section]);
23980 : else
23981 : switch_to_section (darwin_sections[machopic_symbol_stub_section]);
23982 :
23983 : fprintf (file, "%s:\n", stub);
23984 : fprintf (file, "\t.indirect_symbol %s\n", symbol_name);
23985 :
23986 : if (MACHOPIC_ATT_STUB)
23987 : {
23988 : fprintf (file, "\thlt ; hlt ; hlt ; hlt ; hlt\n");
23989 : }
23990 : else if (MACHOPIC_PURE)
23991 : {
23992 : /* PIC stub. */
23993 : /* 25-byte PIC stub using "CALL get_pc_thunk". */
23994 : rtx tmp = gen_rtx_REG (SImode, 2 /* ECX */);
23995 : output_set_got (tmp, NULL_RTX); /* "CALL ___<cpu>.get_pc_thunk.cx". */
23996 : fprintf (file, "LPC$%d:\tmovl\t%s-LPC$%d(%%ecx),%%ecx\n",
23997 : label, lazy_ptr_name, label);
23998 : fprintf (file, "\tjmp\t*%%ecx\n");
23999 : }
24000 : else
24001 : fprintf (file, "\tjmp\t*%s\n", lazy_ptr_name);
24002 :
24003 : /* The AT&T-style ("self-modifying") stub is not lazily bound, thus
24004 : it needs no stub-binding-helper. */
24005 : if (MACHOPIC_ATT_STUB)
24006 : return;
24007 :
24008 : fprintf (file, "%s:\n", binder_name);
24009 :
24010 : if (MACHOPIC_PURE)
24011 : {
24012 : fprintf (file, "\tlea\t%s-%s(%%ecx),%%ecx\n", lazy_ptr_name, binder_name);
24013 : fprintf (file, "\tpushl\t%%ecx\n");
24014 : }
24015 : else
24016 : fprintf (file, "\tpushl\t$%s\n", lazy_ptr_name);
24017 :
24018 : fputs ("\tjmp\tdyld_stub_binding_helper\n", file);
24019 :
24020 : /* N.B. Keep the correspondence of these
24021 : 'symbol_ptr/symbol_ptr2/symbol_ptr3' sections consistent with the
24022 : old-pic/new-pic/non-pic stubs; altering this will break
24023 : compatibility with existing dylibs. */
24024 : if (MACHOPIC_PURE)
24025 : {
24026 : /* 25-byte PIC stub using "CALL get_pc_thunk". */
24027 : switch_to_section (darwin_sections[machopic_lazy_symbol_ptr2_section]);
24028 : }
24029 : else
24030 : /* 16-byte -mdynamic-no-pic stub. */
24031 : switch_to_section(darwin_sections[machopic_lazy_symbol_ptr3_section]);
24032 :
24033 : fprintf (file, "%s:\n", lazy_ptr_name);
24034 : fprintf (file, "\t.indirect_symbol %s\n", symbol_name);
24035 : fprintf (file, ASM_LONG "%s\n", binder_name);
24036 : }
24037 : #endif /* TARGET_MACHO */
24038 :
24039 : /* Order the registers for register allocator. */
24040 :
24041 : void
24042 221098 : x86_order_regs_for_local_alloc (void)
24043 : {
24044 221098 : int pos = 0;
24045 221098 : int i;
24046 :
24047 : /* First allocate the local general purpose registers. */
24048 21004310 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
24049 20783212 : if (GENERAL_REGNO_P (i) && default_function_abi.clobbers_full_reg_p (i))
24050 5538692 : reg_alloc_order [pos++] = i;
24051 :
24052 : /* Global general purpose registers. */
24053 21004310 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
24054 20783212 : if (GENERAL_REGNO_P (i) && !default_function_abi.clobbers_full_reg_p (i))
24055 1536444 : reg_alloc_order [pos++] = i;
24056 :
24057 : /* x87 registers come first in case we are doing FP math
24058 : using them. */
24059 221098 : if (!TARGET_SSE_MATH)
24060 56385 : for (i = FIRST_STACK_REG; i <= LAST_STACK_REG; i++)
24061 50120 : reg_alloc_order [pos++] = i;
24062 :
24063 : /* SSE registers. */
24064 1989882 : for (i = FIRST_SSE_REG; i <= LAST_SSE_REG; i++)
24065 1768784 : reg_alloc_order [pos++] = i;
24066 1989882 : for (i = FIRST_REX_SSE_REG; i <= LAST_REX_SSE_REG; i++)
24067 1768784 : reg_alloc_order [pos++] = i;
24068 :
24069 : /* Extended REX SSE registers. */
24070 3758666 : for (i = FIRST_EXT_REX_SSE_REG; i <= LAST_EXT_REX_SSE_REG; i++)
24071 3537568 : reg_alloc_order [pos++] = i;
24072 :
24073 : /* Mask register. */
24074 1989882 : for (i = FIRST_MASK_REG; i <= LAST_MASK_REG; i++)
24075 1768784 : reg_alloc_order [pos++] = i;
24076 :
24077 : /* x87 registers. */
24078 221098 : if (TARGET_SSE_MATH)
24079 1933497 : for (i = FIRST_STACK_REG; i <= LAST_STACK_REG; i++)
24080 1718664 : reg_alloc_order [pos++] = i;
24081 :
24082 1989882 : for (i = FIRST_MMX_REG; i <= LAST_MMX_REG; i++)
24083 1768784 : reg_alloc_order [pos++] = i;
24084 :
24085 : /* Initialize the rest of array as we do not allocate some registers
24086 : at all. */
24087 1547686 : while (pos < FIRST_PSEUDO_REGISTER)
24088 1326588 : reg_alloc_order [pos++] = 0;
24089 221098 : }
24090 :
24091 : static bool
24092 258497266 : ix86_ms_bitfield_layout_p (const_tree record_type)
24093 : {
24094 258497266 : return ((TARGET_MS_BITFIELD_LAYOUT
24095 215 : && !lookup_attribute ("gcc_struct", TYPE_ATTRIBUTES (record_type)))
24096 258497266 : || lookup_attribute ("ms_struct", TYPE_ATTRIBUTES (record_type)));
24097 : }
24098 :
24099 : /* Returns an expression indicating where the this parameter is
24100 : located on entry to the FUNCTION. */
24101 :
24102 : static rtx
24103 1896 : x86_this_parameter (tree function)
24104 : {
24105 1896 : tree type = TREE_TYPE (function);
24106 1896 : bool aggr = aggregate_value_p (TREE_TYPE (type), type) != 0;
24107 1896 : int nregs;
24108 :
24109 1896 : if (TARGET_64BIT)
24110 : {
24111 1894 : const int *parm_regs;
24112 :
24113 1894 : if (lookup_attribute ("preserve_none", TYPE_ATTRIBUTES (type)))
24114 : parm_regs = x86_64_preserve_none_int_parameter_registers;
24115 1894 : else if (ix86_function_type_abi (type) == MS_ABI)
24116 : parm_regs = x86_64_ms_abi_int_parameter_registers;
24117 : else
24118 1894 : parm_regs = x86_64_int_parameter_registers;
24119 1894 : return gen_rtx_REG (Pmode, parm_regs[aggr]);
24120 : }
24121 :
24122 2 : nregs = ix86_function_regparm (type, function);
24123 :
24124 2 : if (nregs > 0 && !stdarg_p (type))
24125 : {
24126 0 : int regno;
24127 0 : unsigned int ccvt = ix86_get_callcvt (type);
24128 :
24129 0 : if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
24130 0 : regno = aggr ? DX_REG : CX_REG;
24131 0 : else if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
24132 : {
24133 0 : regno = CX_REG;
24134 0 : if (aggr)
24135 0 : return gen_rtx_MEM (SImode,
24136 0 : plus_constant (Pmode, stack_pointer_rtx, 4));
24137 : }
24138 : else
24139 : {
24140 0 : regno = AX_REG;
24141 0 : if (aggr)
24142 : {
24143 0 : regno = DX_REG;
24144 0 : if (nregs == 1)
24145 0 : return gen_rtx_MEM (SImode,
24146 0 : plus_constant (Pmode,
24147 : stack_pointer_rtx, 4));
24148 : }
24149 : }
24150 0 : return gen_rtx_REG (SImode, regno);
24151 : }
24152 :
24153 4 : return gen_rtx_MEM (SImode, plus_constant (Pmode, stack_pointer_rtx,
24154 4 : aggr ? 8 : 4));
24155 : }
24156 :
24157 : /* Determine whether x86_output_mi_thunk can succeed. */
24158 :
24159 : static bool
24160 5186 : x86_can_output_mi_thunk (const_tree, HOST_WIDE_INT, HOST_WIDE_INT vcall_offset,
24161 : const_tree function)
24162 : {
24163 : /* 64-bit can handle anything. */
24164 5186 : if (TARGET_64BIT)
24165 : return true;
24166 :
24167 : /* For 32-bit, everything's fine if we have one free register. */
24168 76 : if (ix86_function_regparm (TREE_TYPE (function), function) < 3)
24169 : return true;
24170 :
24171 : /* Need a free register for vcall_offset. */
24172 0 : if (vcall_offset)
24173 : return false;
24174 :
24175 : /* Need a free register for GOT references. */
24176 0 : if (flag_pic && !targetm.binds_local_p (function))
24177 : return false;
24178 :
24179 : /* Otherwise ok. */
24180 : return true;
24181 : }
24182 :
24183 : /* Output the assembler code for a thunk function. THUNK_DECL is the
24184 : declaration for the thunk function itself, FUNCTION is the decl for
24185 : the target function. DELTA is an immediate constant offset to be
24186 : added to THIS. If VCALL_OFFSET is nonzero, the word at
24187 : *(*this + vcall_offset) should be added to THIS. */
24188 :
24189 : static void
24190 1896 : x86_output_mi_thunk (FILE *file, tree thunk_fndecl, HOST_WIDE_INT delta,
24191 : HOST_WIDE_INT vcall_offset, tree function)
24192 : {
24193 1896 : const char *fnname = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (thunk_fndecl));
24194 1896 : rtx this_param = x86_this_parameter (function);
24195 1896 : rtx this_reg, tmp, fnaddr;
24196 1896 : unsigned int tmp_regno;
24197 1896 : rtx_insn *insn;
24198 1896 : int saved_flag_force_indirect_call = flag_force_indirect_call;
24199 :
24200 1896 : if (TARGET_64BIT)
24201 : tmp_regno = R10_REG;
24202 : else
24203 : {
24204 2 : unsigned int ccvt = ix86_get_callcvt (TREE_TYPE (function));
24205 2 : if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
24206 : tmp_regno = AX_REG;
24207 2 : else if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
24208 : tmp_regno = DX_REG;
24209 : else
24210 2 : tmp_regno = CX_REG;
24211 :
24212 2 : if (flag_pic)
24213 2 : flag_force_indirect_call = 0;
24214 : }
24215 :
24216 1896 : emit_note (NOTE_INSN_PROLOGUE_END);
24217 :
24218 : /* CET is enabled, insert EB instruction. */
24219 1896 : if ((flag_cf_protection & CF_BRANCH))
24220 20 : emit_insn (gen_nop_endbr ());
24221 :
24222 : /* If VCALL_OFFSET, we'll need THIS in a register. Might as well
24223 : pull it in now and let DELTA benefit. */
24224 1896 : if (REG_P (this_param))
24225 : this_reg = this_param;
24226 2 : else if (vcall_offset)
24227 : {
24228 : /* Put the this parameter into %eax. */
24229 2 : this_reg = gen_rtx_REG (Pmode, AX_REG);
24230 1 : emit_move_insn (this_reg, this_param);
24231 : }
24232 : else
24233 : this_reg = NULL_RTX;
24234 :
24235 : /* Adjust the this parameter by a fixed constant. */
24236 1896 : if (delta)
24237 : {
24238 921 : rtx delta_rtx = GEN_INT (delta);
24239 921 : rtx delta_dst = this_reg ? this_reg : this_param;
24240 :
24241 921 : if (TARGET_64BIT)
24242 : {
24243 920 : if (!x86_64_general_operand (delta_rtx, Pmode))
24244 : {
24245 0 : tmp = gen_rtx_REG (Pmode, tmp_regno);
24246 0 : emit_move_insn (tmp, delta_rtx);
24247 0 : delta_rtx = tmp;
24248 : }
24249 : }
24250 :
24251 922 : ix86_emit_binop (PLUS, Pmode, delta_dst, delta_rtx);
24252 : }
24253 :
24254 : /* Adjust the this parameter by a value stored in the vtable. */
24255 1896 : if (vcall_offset)
24256 : {
24257 1026 : rtx vcall_addr, vcall_mem, this_mem;
24258 :
24259 1027 : tmp = gen_rtx_REG (Pmode, tmp_regno);
24260 :
24261 1026 : this_mem = gen_rtx_MEM (ptr_mode, this_reg);
24262 1027 : if (Pmode != ptr_mode)
24263 0 : this_mem = gen_rtx_ZERO_EXTEND (Pmode, this_mem);
24264 1026 : emit_move_insn (tmp, this_mem);
24265 :
24266 : /* Adjust the this parameter. */
24267 1027 : vcall_addr = plus_constant (Pmode, tmp, vcall_offset);
24268 1026 : if (TARGET_64BIT
24269 1026 : && !ix86_legitimate_address_p (ptr_mode, vcall_addr, true))
24270 : {
24271 0 : rtx tmp2 = gen_rtx_REG (Pmode, R11_REG);
24272 0 : emit_move_insn (tmp2, GEN_INT (vcall_offset));
24273 0 : vcall_addr = gen_rtx_PLUS (Pmode, tmp, tmp2);
24274 : }
24275 :
24276 1026 : vcall_mem = gen_rtx_MEM (ptr_mode, vcall_addr);
24277 1027 : if (Pmode != ptr_mode)
24278 0 : emit_insn (gen_addsi_1_zext (this_reg,
24279 : gen_rtx_REG (ptr_mode,
24280 : REGNO (this_reg)),
24281 : vcall_mem));
24282 : else
24283 1026 : ix86_emit_binop (PLUS, Pmode, this_reg, vcall_mem);
24284 : }
24285 :
24286 : /* If necessary, drop THIS back to its stack slot. */
24287 1896 : if (this_reg && this_reg != this_param)
24288 1 : emit_move_insn (this_param, this_reg);
24289 :
24290 1896 : fnaddr = XEXP (DECL_RTL (function), 0);
24291 1896 : if (TARGET_64BIT)
24292 : {
24293 25 : if (!flag_pic || targetm.binds_local_p (function)
24294 1919 : || TARGET_PECOFF)
24295 : ;
24296 : else
24297 : {
24298 0 : tmp = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, fnaddr), UNSPEC_GOTPCREL);
24299 0 : tmp = gen_rtx_CONST (Pmode, tmp);
24300 0 : fnaddr = gen_const_mem (Pmode, tmp);
24301 : }
24302 : }
24303 : else
24304 : {
24305 2 : if (!flag_pic || targetm.binds_local_p (function))
24306 : ;
24307 : #if TARGET_MACHO
24308 : else if (TARGET_MACHO)
24309 : {
24310 : fnaddr = machopic_indirect_call_target (DECL_RTL (function));
24311 : fnaddr = XEXP (fnaddr, 0);
24312 : }
24313 : #endif /* TARGET_MACHO */
24314 : else
24315 : {
24316 0 : tmp = gen_rtx_REG (Pmode, CX_REG);
24317 0 : output_set_got (tmp, NULL_RTX);
24318 :
24319 0 : fnaddr = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, fnaddr), UNSPEC_GOT);
24320 0 : fnaddr = gen_rtx_CONST (Pmode, fnaddr);
24321 0 : fnaddr = gen_rtx_PLUS (Pmode, tmp, fnaddr);
24322 0 : fnaddr = gen_const_mem (Pmode, fnaddr);
24323 : }
24324 : }
24325 :
24326 : /* Our sibling call patterns do not allow memories, because we have no
24327 : predicate that can distinguish between frame and non-frame memory.
24328 : For our purposes here, we can get away with (ab)using a jump pattern,
24329 : because we're going to do no optimization. */
24330 1896 : if (MEM_P (fnaddr))
24331 : {
24332 0 : if (sibcall_insn_operand (fnaddr, word_mode))
24333 : {
24334 0 : fnaddr = XEXP (DECL_RTL (function), 0);
24335 0 : tmp = gen_rtx_MEM (QImode, fnaddr);
24336 0 : tmp = gen_rtx_CALL (VOIDmode, tmp, const0_rtx);
24337 0 : tmp = emit_call_insn (tmp);
24338 0 : SIBLING_CALL_P (tmp) = 1;
24339 : }
24340 : else
24341 0 : emit_jump_insn (gen_indirect_jump (fnaddr));
24342 : }
24343 : else
24344 : {
24345 1896 : if (ix86_cmodel == CM_LARGE_PIC && SYMBOLIC_CONST (fnaddr))
24346 : {
24347 : // CM_LARGE_PIC always uses pseudo PIC register which is
24348 : // uninitialized. Since FUNCTION is local and calling it
24349 : // doesn't go through PLT, we use scratch register %r11 as
24350 : // PIC register and initialize it here.
24351 3 : pic_offset_table_rtx = gen_rtx_REG (Pmode, R11_REG);
24352 3 : ix86_init_large_pic_reg (tmp_regno);
24353 3 : fnaddr = legitimize_pic_address (fnaddr,
24354 3 : gen_rtx_REG (Pmode, tmp_regno));
24355 : }
24356 :
24357 1896 : if (!sibcall_insn_operand (fnaddr, word_mode))
24358 : {
24359 9 : tmp = gen_rtx_REG (word_mode, tmp_regno);
24360 9 : if (GET_MODE (fnaddr) != word_mode)
24361 0 : fnaddr = gen_rtx_ZERO_EXTEND (word_mode, fnaddr);
24362 9 : emit_move_insn (tmp, fnaddr);
24363 9 : fnaddr = tmp;
24364 : }
24365 :
24366 1896 : tmp = gen_rtx_MEM (QImode, fnaddr);
24367 1896 : tmp = gen_rtx_CALL (VOIDmode, tmp, const0_rtx);
24368 1896 : tmp = emit_call_insn (tmp);
24369 1896 : SIBLING_CALL_P (tmp) = 1;
24370 : }
24371 1896 : emit_barrier ();
24372 :
24373 : /* Emit just enough of rest_of_compilation to get the insns emitted. */
24374 1896 : insn = get_insns ();
24375 1896 : shorten_branches (insn);
24376 1896 : assemble_start_function (thunk_fndecl, fnname);
24377 1896 : final_start_function (insn, file, 1);
24378 1896 : final (insn, file, 1);
24379 1896 : final_end_function ();
24380 1896 : assemble_end_function (thunk_fndecl, fnname);
24381 :
24382 1896 : flag_force_indirect_call = saved_flag_force_indirect_call;
24383 1896 : }
24384 :
24385 : static void
24386 279576 : x86_file_start (void)
24387 : {
24388 279576 : default_file_start ();
24389 279576 : if (TARGET_16BIT)
24390 6 : fputs ("\t.code16gcc\n", asm_out_file);
24391 : #if TARGET_MACHO
24392 : darwin_file_start ();
24393 : #endif
24394 279576 : if (X86_FILE_START_VERSION_DIRECTIVE)
24395 : fputs ("\t.version\t\"01.01\"\n", asm_out_file);
24396 279576 : if (X86_FILE_START_FLTUSED)
24397 : fputs ("\t.global\t__fltused\n", asm_out_file);
24398 279576 : if (ix86_asm_dialect == ASM_INTEL)
24399 65 : fputs ("\t.intel_syntax noprefix\n", asm_out_file);
24400 279576 : }
24401 :
24402 : int
24403 107874814 : x86_field_alignment (tree type, int computed)
24404 : {
24405 107874814 : machine_mode mode;
24406 :
24407 107874814 : if (TARGET_64BIT || TARGET_ALIGN_DOUBLE)
24408 : return computed;
24409 9135488 : if (TARGET_IAMCU)
24410 0 : return iamcu_alignment (type, computed);
24411 9135488 : type = strip_array_types (type);
24412 9135488 : mode = TYPE_MODE (type);
24413 9135488 : if (mode == DFmode || mode == DCmode
24414 9029707 : || GET_MODE_CLASS (mode) == MODE_INT
24415 3024154 : || GET_MODE_CLASS (mode) == MODE_COMPLEX_INT)
24416 : {
24417 6111334 : if (TYPE_ATOMIC (type) && computed > 32)
24418 : {
24419 0 : static bool warned;
24420 :
24421 0 : if (!warned && warn_psabi)
24422 : {
24423 0 : const char *url
24424 : = CHANGES_ROOT_URL "gcc-11/changes.html#ia32_atomic";
24425 :
24426 0 : warned = true;
24427 0 : inform (input_location, "the alignment of %<_Atomic %T%> "
24428 : "fields changed in %{GCC 11.1%}",
24429 0 : TYPE_MAIN_VARIANT (type), url);
24430 : }
24431 : }
24432 : else
24433 6111334 : return MIN (32, computed);
24434 : }
24435 : return computed;
24436 : }
24437 :
24438 : /* Print call to TARGET to FILE. */
24439 :
24440 : static void
24441 293 : x86_print_call_or_nop (FILE *file, const char *target,
24442 : const char *label)
24443 : {
24444 293 : if (flag_nop_mcount || !strcmp (target, "nop"))
24445 : {
24446 9 : if (TARGET_16BIT)
24447 : /* 3 byte no-op: lea 0(%si), %si */
24448 1 : fprintf (file, "%s" ASM_BYTE "0x8d, 0x74, 0x00\n", label);
24449 : else
24450 : /* 5 byte nop: nopl 0(%[re]ax,%[re]ax,1) */
24451 8 : fprintf (file, "%s" ASM_BYTE "0x0f, 0x1f, 0x44, 0x00, 0x00\n",
24452 : label);
24453 : }
24454 284 : else if (!TARGET_PECOFF && flag_pic)
24455 : {
24456 8 : gcc_assert (flag_plt);
24457 :
24458 8 : fprintf (file, "%s\tcall\t%s@PLT\n", label, target);
24459 : }
24460 : else
24461 276 : fprintf (file, "%s\tcall\t%s\n", label, target);
24462 293 : }
24463 :
24464 : static bool
24465 313 : current_fentry_name (const char **name)
24466 : {
24467 313 : tree attr = lookup_attribute ("fentry_name",
24468 313 : DECL_ATTRIBUTES (current_function_decl));
24469 313 : if (!attr)
24470 : return false;
24471 2 : *name = TREE_STRING_POINTER (TREE_VALUE (TREE_VALUE (attr)));
24472 2 : return true;
24473 : }
24474 :
24475 : static bool
24476 16 : current_fentry_section (const char **name)
24477 : {
24478 16 : tree attr = lookup_attribute ("fentry_section",
24479 16 : DECL_ATTRIBUTES (current_function_decl));
24480 16 : if (!attr)
24481 : return false;
24482 2 : *name = TREE_STRING_POINTER (TREE_VALUE (TREE_VALUE (attr)));
24483 2 : return true;
24484 : }
24485 :
24486 : /* Return a caller-saved register which isn't live or a callee-saved
24487 : register which has been saved on stack in the prologue at entry for
24488 : profile. */
24489 :
24490 : static int
24491 17 : x86_64_select_profile_regnum (bool r11_ok ATTRIBUTE_UNUSED)
24492 : {
24493 : /* Use %r10 if the profiler is emitted before the prologue or it isn't
24494 : used by DRAP. */
24495 17 : if (ix86_profile_before_prologue ()
24496 4 : || !crtl->drap_reg
24497 17 : || REGNO (crtl->drap_reg) != R10_REG)
24498 : return R10_REG;
24499 :
24500 : /* The profiler is emitted after the prologue. If there is a
24501 : caller-saved register which isn't live or a callee-saved
24502 : register saved on stack in the prologue, use it. */
24503 :
24504 0 : bitmap reg_live = df_get_live_out (ENTRY_BLOCK_PTR_FOR_FN (cfun));
24505 :
24506 0 : int i;
24507 0 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
24508 0 : if (GENERAL_REGNO_P (i)
24509 0 : && i != R10_REG
24510 : #ifdef NO_PROFILE_COUNTERS
24511 0 : && (r11_ok || i != R11_REG)
24512 : #else
24513 : && i != R11_REG
24514 : #endif
24515 0 : && TEST_HARD_REG_BIT (accessible_reg_set, i)
24516 0 : && (ix86_save_reg (i, true, true)
24517 0 : || (crtl->abi->clobbers_full_reg_p (i)
24518 0 : && !fixed_regs[i]
24519 0 : && !REGNO_REG_SET_P (reg_live, i))))
24520 : return i;
24521 :
24522 0 : sorry ("no register available for profiling %<-mcmodel=large%s%>",
24523 0 : ix86_cmodel == CM_LARGE_PIC ? " -fPIC" : "");
24524 :
24525 0 : return R10_REG;
24526 : }
24527 :
24528 : /* Output assembler code to FILE to increment profiler label # LABELNO
24529 : for profiling a function entry. */
24530 : void
24531 313 : x86_function_profiler (FILE *file, int labelno ATTRIBUTE_UNUSED)
24532 : {
24533 313 : if (cfun->machine->insn_queued_at_entrance)
24534 : {
24535 7 : if (cfun->machine->insn_queued_at_entrance == TYPE_ENDBR)
24536 6 : fprintf (file, "\t%s\n", TARGET_64BIT ? "endbr64" : "endbr32");
24537 7 : unsigned int patch_area_size
24538 7 : = crtl->patch_area_size - crtl->patch_area_entry;
24539 7 : if (patch_area_size)
24540 2 : ix86_output_patchable_area (patch_area_size,
24541 : crtl->patch_area_entry == 0);
24542 : }
24543 :
24544 313 : const char *mcount_name = MCOUNT_NAME;
24545 :
24546 313 : bool fentry_section_p
24547 313 : = (flag_record_mcount
24548 611 : || lookup_attribute ("fentry_section",
24549 298 : DECL_ATTRIBUTES (current_function_decl)));
24550 :
24551 : const char *label = fentry_section_p ? "1:" : "";
24552 :
24553 313 : if (current_fentry_name (&mcount_name))
24554 : ;
24555 311 : else if (fentry_name)
24556 1 : mcount_name = fentry_name;
24557 310 : else if (flag_fentry)
24558 298 : mcount_name = MCOUNT_NAME_BEFORE_PROLOGUE;
24559 :
24560 313 : if (TARGET_64BIT)
24561 : {
24562 : #ifndef NO_PROFILE_COUNTERS
24563 : if (ASSEMBLER_DIALECT == ASM_INTEL)
24564 : fprintf (file, "\tlea\tr11, %sP%d[rip]\n", LPREFIX, labelno);
24565 : else
24566 : fprintf (file, "\tleaq\t%sP%d(%%rip), %%r11\n", LPREFIX, labelno);
24567 : #endif
24568 :
24569 312 : int scratch;
24570 312 : const char *reg;
24571 312 : char legacy_reg[4] = { 0 };
24572 :
24573 312 : if (!TARGET_PECOFF)
24574 : {
24575 312 : switch (ix86_cmodel)
24576 : {
24577 7 : case CM_LARGE:
24578 7 : scratch = x86_64_select_profile_regnum (true);
24579 7 : reg = hi_reg_name[scratch];
24580 7 : if (LEGACY_INT_REGNO_P (scratch))
24581 : {
24582 0 : legacy_reg[0] = 'r';
24583 0 : legacy_reg[1] = reg[0];
24584 0 : legacy_reg[2] = reg[1];
24585 0 : reg = legacy_reg;
24586 : }
24587 7 : if (ASSEMBLER_DIALECT == ASM_INTEL)
24588 1 : fprintf (file, "%s\tmovabs\t%s, OFFSET FLAT:%s\n"
24589 : "\tcall\t%s\n", label, reg, mcount_name,
24590 : reg);
24591 : else
24592 6 : fprintf (file, "%s\tmovabsq\t$%s, %%%s\n\tcall\t*%%%s\n",
24593 : label, mcount_name, reg, reg);
24594 : break;
24595 10 : case CM_LARGE_PIC:
24596 : #ifdef NO_PROFILE_COUNTERS
24597 10 : scratch = x86_64_select_profile_regnum (false);
24598 10 : reg = hi_reg_name[scratch];
24599 10 : if (LEGACY_INT_REGNO_P (scratch))
24600 : {
24601 0 : legacy_reg[0] = 'r';
24602 0 : legacy_reg[1] = reg[0];
24603 0 : legacy_reg[2] = reg[1];
24604 0 : reg = legacy_reg;
24605 : }
24606 10 : if (ASSEMBLER_DIALECT == ASM_INTEL)
24607 : {
24608 1 : fprintf (file, "1:movabs\tr11, "
24609 : "OFFSET FLAT:_GLOBAL_OFFSET_TABLE_-1b\n");
24610 1 : fprintf (file, "\tlea\t%s, 1b[rip]\n", reg);
24611 1 : fprintf (file, "\tadd\t%s, r11\n", reg);
24612 1 : fprintf (file, "\tmovabs\tr11, OFFSET FLAT:%s@PLTOFF\n",
24613 : mcount_name);
24614 1 : fprintf (file, "\tadd\t%s, r11\n", reg);
24615 1 : fprintf (file, "\tcall\t%s\n", reg);
24616 1 : break;
24617 : }
24618 9 : fprintf (file,
24619 : "1:\tmovabsq\t$_GLOBAL_OFFSET_TABLE_-1b, %%r11\n");
24620 9 : fprintf (file, "\tleaq\t1b(%%rip), %%%s\n", reg);
24621 9 : fprintf (file, "\taddq\t%%r11, %%%s\n", reg);
24622 9 : fprintf (file, "\tmovabsq\t$%s@PLTOFF, %%r11\n", mcount_name);
24623 9 : fprintf (file, "\taddq\t%%r11, %%%s\n", reg);
24624 9 : fprintf (file, "\tcall\t*%%%s\n", reg);
24625 : #else
24626 : sorry ("profiling %<-mcmodel=large%> with PIC is not supported");
24627 : #endif
24628 9 : break;
24629 12 : case CM_SMALL_PIC:
24630 12 : case CM_MEDIUM_PIC:
24631 12 : if (!flag_plt)
24632 : {
24633 3 : if (ASSEMBLER_DIALECT == ASM_INTEL)
24634 0 : fprintf (file, "%s\tcall\t[QWORD PTR %s@GOTPCREL[rip]]\n",
24635 : label, mcount_name);
24636 : else
24637 3 : fprintf (file, "%s\tcall\t*%s@GOTPCREL(%%rip)\n",
24638 : label, mcount_name);
24639 : break;
24640 : }
24641 : /* fall through */
24642 292 : default:
24643 292 : x86_print_call_or_nop (file, mcount_name, label);
24644 292 : break;
24645 : }
24646 : }
24647 : else
24648 : x86_print_call_or_nop (file, mcount_name, label);
24649 : }
24650 1 : else if (flag_pic)
24651 : {
24652 : #ifndef NO_PROFILE_COUNTERS
24653 : if (ASSEMBLER_DIALECT == ASM_INTEL)
24654 : fprintf (file,
24655 : "\tlea\t" PROFILE_COUNT_REGISTER ", %sP%d@GOTOFF[ebx]\n",
24656 : LPREFIX, labelno);
24657 : else
24658 : fprintf (file,
24659 : "\tleal\t%sP%d@GOTOFF(%%ebx), %%" PROFILE_COUNT_REGISTER "\n",
24660 : LPREFIX, labelno);
24661 : #endif
24662 0 : if (flag_plt)
24663 0 : x86_print_call_or_nop (file, mcount_name, label);
24664 0 : else if (ASSEMBLER_DIALECT == ASM_INTEL)
24665 0 : fprintf (file, "%s\tcall\t[DWORD PTR %s@GOT[ebx]]\n",
24666 : label, mcount_name);
24667 : else
24668 0 : fprintf (file, "%s\tcall\t*%s@GOT(%%ebx)\n",
24669 : label, mcount_name);
24670 : }
24671 : else
24672 : {
24673 : #ifndef NO_PROFILE_COUNTERS
24674 : if (ASSEMBLER_DIALECT == ASM_INTEL)
24675 : fprintf (file,
24676 : "\tmov\t" PROFILE_COUNT_REGISTER ", OFFSET FLAT:%sP%d\n",
24677 : LPREFIX, labelno);
24678 : else
24679 : fprintf (file, "\tmovl\t$%sP%d, %%" PROFILE_COUNT_REGISTER "\n",
24680 : LPREFIX, labelno);
24681 : #endif
24682 1 : x86_print_call_or_nop (file, mcount_name, label);
24683 : }
24684 :
24685 313 : if (fentry_section_p)
24686 : {
24687 16 : const char *sname = "__mcount_loc";
24688 :
24689 16 : if (current_fentry_section (&sname))
24690 : ;
24691 14 : else if (fentry_section)
24692 1 : sname = fentry_section;
24693 :
24694 16 : fprintf (file, "\t.section %s, \"a\",@progbits\n", sname);
24695 16 : fprintf (file, "\t.%s 1b\n", TARGET_64BIT ? "quad" : "long");
24696 16 : fprintf (file, "\t.previous\n");
24697 : }
24698 313 : }
24699 :
24700 : /* We don't have exact information about the insn sizes, but we may assume
24701 : quite safely that we are informed about all 1 byte insns and memory
24702 : address sizes. This is enough to eliminate unnecessary padding in
24703 : 99% of cases. */
24704 :
24705 : int
24706 399254295 : ix86_min_insn_size (rtx_insn *insn)
24707 : {
24708 399254295 : int l = 0, len;
24709 :
24710 399254295 : if (!INSN_P (insn) || !active_insn_p (insn))
24711 : return 0;
24712 :
24713 : /* Discard alignments we've emit and jump instructions. */
24714 398741849 : if (GET_CODE (PATTERN (insn)) == UNSPEC_VOLATILE
24715 398741849 : && XINT (PATTERN (insn), 1) == UNSPECV_ALIGN)
24716 : return 0;
24717 :
24718 : /* Important case - calls are always 5 bytes.
24719 : It is common to have many calls in the row. */
24720 398741845 : if (CALL_P (insn)
24721 9525810 : && symbolic_reference_mentioned_p (PATTERN (insn))
24722 407926585 : && !SIBLING_CALL_P (insn))
24723 : return 5;
24724 389801266 : len = get_attr_length (insn);
24725 389801266 : if (len <= 1)
24726 : return 1;
24727 :
24728 : /* For normal instructions we rely on get_attr_length being exact,
24729 : with a few exceptions. */
24730 381061494 : if (!JUMP_P (insn))
24731 : {
24732 375492137 : enum attr_type type = get_attr_type (insn);
24733 :
24734 375492137 : switch (type)
24735 : {
24736 96117 : case TYPE_MULTI:
24737 96117 : if (GET_CODE (PATTERN (insn)) == ASM_INPUT
24738 96117 : || asm_noperands (PATTERN (insn)) >= 0)
24739 : return 0;
24740 : break;
24741 : case TYPE_OTHER:
24742 : case TYPE_FCMP:
24743 : break;
24744 : default:
24745 : /* Otherwise trust get_attr_length. */
24746 : return len;
24747 : }
24748 :
24749 478197 : l = get_attr_length_address (insn);
24750 478197 : if (l < 4 && symbolic_reference_mentioned_p (PATTERN (insn)))
24751 : l = 4;
24752 : }
24753 387673 : if (l)
24754 90524 : return 1+l;
24755 : else
24756 : return 2;
24757 : }
24758 :
24759 : #ifdef ASM_OUTPUT_MAX_SKIP_ALIGN
24760 :
24761 : /* AMD K8 core mispredicts jumps when there are more than 3 jumps in 16 byte
24762 : window. */
24763 :
24764 : static void
24765 46636 : ix86_avoid_jump_mispredicts (void)
24766 : {
24767 46636 : rtx_insn *insn, *start = get_insns ();
24768 46636 : int nbytes = 0, njumps = 0;
24769 46636 : bool isjump = false;
24770 :
24771 : /* Look for all minimal intervals of instructions containing 4 jumps.
24772 : The intervals are bounded by START and INSN. NBYTES is the total
24773 : size of instructions in the interval including INSN and not including
24774 : START. When the NBYTES is smaller than 16 bytes, it is possible
24775 : that the end of START and INSN ends up in the same 16byte page.
24776 :
24777 : The smallest offset in the page INSN can start is the case where START
24778 : ends on the offset 0. Offset of INSN is then NBYTES - sizeof (INSN).
24779 : We add p2align to 16byte window with maxskip 15 - NBYTES + sizeof (INSN).
24780 :
24781 : Don't consider asm goto as jump, while it can contain a jump, it doesn't
24782 : have to, control transfer to label(s) can be performed through other
24783 : means, and also we estimate minimum length of all asm stmts as 0. */
24784 716076 : for (insn = start; insn; insn = NEXT_INSN (insn))
24785 : {
24786 669440 : int min_size;
24787 :
24788 669440 : if (LABEL_P (insn))
24789 : {
24790 942 : align_flags alignment = label_to_alignment (insn);
24791 942 : int align = alignment.levels[0].log;
24792 942 : int max_skip = alignment.levels[0].maxskip;
24793 :
24794 942 : if (max_skip > 15)
24795 : max_skip = 15;
24796 : /* If align > 3, only up to 16 - max_skip - 1 bytes can be
24797 : already in the current 16 byte page, because otherwise
24798 : ASM_OUTPUT_MAX_SKIP_ALIGN could skip max_skip or fewer
24799 : bytes to reach 16 byte boundary. */
24800 942 : if (align <= 0
24801 307 : || (align <= 3 && max_skip != (1 << align) - 1))
24802 942 : max_skip = 0;
24803 942 : if (dump_file)
24804 0 : fprintf (dump_file, "Label %i with max_skip %i\n",
24805 0 : INSN_UID (insn), max_skip);
24806 942 : if (max_skip)
24807 : {
24808 6098 : while (nbytes + max_skip >= 16)
24809 : {
24810 5791 : start = NEXT_INSN (start);
24811 295 : if ((JUMP_P (start) && asm_noperands (PATTERN (start)) < 0)
24812 5808 : || CALL_P (start))
24813 335 : njumps--, isjump = true;
24814 : else
24815 : isjump = false;
24816 5791 : nbytes -= ix86_min_insn_size (start);
24817 : }
24818 : }
24819 942 : continue;
24820 942 : }
24821 :
24822 668498 : min_size = ix86_min_insn_size (insn);
24823 668498 : nbytes += min_size;
24824 668498 : if (dump_file)
24825 0 : fprintf (dump_file, "Insn %i estimated to %i bytes\n",
24826 0 : INSN_UID (insn), min_size);
24827 47741 : if ((JUMP_P (insn) && asm_noperands (PATTERN (insn)) < 0)
24828 668518 : || CALL_P (insn))
24829 48754 : njumps++;
24830 : else
24831 619744 : continue;
24832 :
24833 57011 : while (njumps > 3)
24834 : {
24835 8257 : start = NEXT_INSN (start);
24836 508 : if ((JUMP_P (start) && asm_noperands (PATTERN (start)) < 0)
24837 8257 : || CALL_P (start))
24838 1208 : njumps--, isjump = true;
24839 : else
24840 : isjump = false;
24841 8257 : nbytes -= ix86_min_insn_size (start);
24842 : }
24843 48754 : gcc_assert (njumps >= 0);
24844 48754 : if (dump_file)
24845 0 : fprintf (dump_file, "Interval %i to %i has %i bytes\n",
24846 0 : INSN_UID (start), INSN_UID (insn), nbytes);
24847 :
24848 48754 : if (njumps == 3 && isjump && nbytes < 16)
24849 : {
24850 30 : int padsize = 15 - nbytes + ix86_min_insn_size (insn);
24851 :
24852 30 : if (dump_file)
24853 0 : fprintf (dump_file, "Padding insn %i by %i bytes!\n",
24854 0 : INSN_UID (insn), padsize);
24855 30 : emit_insn_before (gen_max_skip_align (GEN_INT (4), GEN_INT (padsize)), insn);
24856 : }
24857 : }
24858 46636 : }
24859 : #endif
24860 :
24861 : /* AMD Athlon works faster
24862 : when RET is not destination of conditional jump or directly preceded
24863 : by other jump instruction. We avoid the penalty by inserting NOP just
24864 : before the RET instructions in such cases. */
24865 : static void
24866 46356 : ix86_pad_returns (void)
24867 : {
24868 46356 : edge e;
24869 46356 : edge_iterator ei;
24870 :
24871 92737 : FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
24872 : {
24873 46381 : basic_block bb = e->src;
24874 46381 : rtx_insn *ret = BB_END (bb);
24875 46381 : rtx_insn *prev;
24876 46381 : bool replace = false;
24877 :
24878 46371 : if (!JUMP_P (ret) || !ANY_RETURN_P (PATTERN (ret))
24879 92752 : || optimize_bb_for_size_p (bb))
24880 23 : continue;
24881 184525 : for (prev = PREV_INSN (ret); prev; prev = PREV_INSN (prev))
24882 137749 : if (active_insn_p (prev) || LABEL_P (prev))
24883 : break;
24884 46358 : if (prev && LABEL_P (prev))
24885 : {
24886 42 : edge e;
24887 42 : edge_iterator ei;
24888 :
24889 55 : FOR_EACH_EDGE (e, ei, bb->preds)
24890 143 : if (EDGE_FREQUENCY (e) && e->src->index >= 0
24891 95 : && !(e->flags & EDGE_FALLTHRU))
24892 : {
24893 : replace = true;
24894 : break;
24895 : }
24896 : }
24897 42 : if (!replace)
24898 : {
24899 46323 : prev = prev_active_insn (ret);
24900 46323 : if (prev
24901 46323 : && ((JUMP_P (prev) && any_condjump_p (prev))
24902 45887 : || CALL_P (prev)))
24903 : replace = true;
24904 : /* Empty functions get branch mispredict even when
24905 : the jump destination is not visible to us. */
24906 46323 : if (!prev && !optimize_function_for_size_p (cfun))
24907 : replace = true;
24908 : }
24909 45905 : if (replace)
24910 : {
24911 488 : emit_jump_insn_before (gen_simple_return_internal_long (), ret);
24912 488 : delete_insn (ret);
24913 : }
24914 : }
24915 46356 : }
24916 :
24917 : /* Count the minimum number of instructions in BB. Return 4 if the
24918 : number of instructions >= 4. */
24919 :
24920 : static int
24921 41 : ix86_count_insn_bb (basic_block bb)
24922 : {
24923 41 : rtx_insn *insn;
24924 41 : int insn_count = 0;
24925 :
24926 : /* Count number of instructions in this block. Return 4 if the number
24927 : of instructions >= 4. */
24928 295 : FOR_BB_INSNS (bb, insn)
24929 : {
24930 : /* Only happen in exit blocks. */
24931 290 : if (JUMP_P (insn)
24932 290 : && ANY_RETURN_P (PATTERN (insn)))
24933 : break;
24934 :
24935 267 : if (NONDEBUG_INSN_P (insn)
24936 104 : && GET_CODE (PATTERN (insn)) != USE
24937 353 : && GET_CODE (PATTERN (insn)) != CLOBBER)
24938 : {
24939 86 : insn_count++;
24940 86 : if (insn_count >= 4)
24941 : return insn_count;
24942 : }
24943 : }
24944 :
24945 : return insn_count;
24946 : }
24947 :
24948 :
24949 : /* Count the minimum number of instructions in code path in BB.
24950 : Return 4 if the number of instructions >= 4. */
24951 :
24952 : static int
24953 61 : ix86_count_insn (basic_block bb)
24954 : {
24955 61 : edge e;
24956 61 : edge_iterator ei;
24957 61 : int min_prev_count;
24958 :
24959 : /* Only bother counting instructions along paths with no
24960 : more than 2 basic blocks between entry and exit. Given
24961 : that BB has an edge to exit, determine if a predecessor
24962 : of BB has an edge from entry. If so, compute the number
24963 : of instructions in the predecessor block. If there
24964 : happen to be multiple such blocks, compute the minimum. */
24965 61 : min_prev_count = 4;
24966 144 : FOR_EACH_EDGE (e, ei, bb->preds)
24967 : {
24968 109 : edge prev_e;
24969 109 : edge_iterator prev_ei;
24970 :
24971 109 : if (e->src == ENTRY_BLOCK_PTR_FOR_FN (cfun))
24972 : {
24973 26 : min_prev_count = 0;
24974 26 : break;
24975 : }
24976 182 : FOR_EACH_EDGE (prev_e, prev_ei, e->src->preds)
24977 : {
24978 109 : if (prev_e->src == ENTRY_BLOCK_PTR_FOR_FN (cfun))
24979 : {
24980 10 : int count = ix86_count_insn_bb (e->src);
24981 10 : if (count < min_prev_count)
24982 83 : min_prev_count = count;
24983 : break;
24984 : }
24985 : }
24986 : }
24987 :
24988 61 : if (min_prev_count < 4)
24989 31 : min_prev_count += ix86_count_insn_bb (bb);
24990 :
24991 61 : return min_prev_count;
24992 : }
24993 :
24994 : /* Pad short function to 4 instructions. */
24995 :
24996 : static void
24997 63 : ix86_pad_short_function (void)
24998 : {
24999 63 : edge e;
25000 63 : edge_iterator ei;
25001 :
25002 127 : FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
25003 : {
25004 64 : rtx_insn *ret = BB_END (e->src);
25005 64 : if (JUMP_P (ret) && ANY_RETURN_P (PATTERN (ret)))
25006 : {
25007 61 : int insn_count = ix86_count_insn (e->src);
25008 :
25009 : /* Pad short function. */
25010 61 : if (insn_count < 4)
25011 : {
25012 : rtx_insn *insn = ret;
25013 :
25014 : /* Find epilogue. */
25015 : while (insn
25016 50 : && (!NOTE_P (insn)
25017 23 : || NOTE_KIND (insn) != NOTE_INSN_EPILOGUE_BEG))
25018 28 : insn = PREV_INSN (insn);
25019 :
25020 22 : if (!insn)
25021 0 : insn = ret;
25022 :
25023 : /* Two NOPs count as one instruction. */
25024 22 : insn_count = 2 * (4 - insn_count);
25025 22 : emit_insn_before (gen_nops (GEN_INT (insn_count)), insn);
25026 : }
25027 : }
25028 : }
25029 63 : }
25030 :
25031 : /* Fix up a Windows system unwinder issue. If an EH region falls through into
25032 : the epilogue, the Windows system unwinder will apply epilogue logic and
25033 : produce incorrect offsets. This can be avoided by adding a nop between
25034 : the last insn that can throw and the first insn of the epilogue. */
25035 :
25036 : static void
25037 0 : ix86_seh_fixup_eh_fallthru (void)
25038 : {
25039 0 : edge e;
25040 0 : edge_iterator ei;
25041 :
25042 0 : FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
25043 : {
25044 0 : rtx_insn *insn, *next;
25045 :
25046 : /* Find the beginning of the epilogue. */
25047 0 : for (insn = BB_END (e->src); insn != NULL; insn = PREV_INSN (insn))
25048 0 : if (NOTE_P (insn) && NOTE_KIND (insn) == NOTE_INSN_EPILOGUE_BEG)
25049 : break;
25050 0 : if (insn == NULL)
25051 0 : continue;
25052 :
25053 : /* We only care about preceding insns that can throw. */
25054 0 : insn = prev_active_insn (insn);
25055 0 : if (insn == NULL || !can_throw_internal (insn))
25056 0 : continue;
25057 :
25058 : /* Do not separate calls from their debug information. */
25059 0 : for (next = NEXT_INSN (insn); next != NULL; next = NEXT_INSN (next))
25060 0 : if (NOTE_P (next) && NOTE_KIND (next) == NOTE_INSN_VAR_LOCATION)
25061 0 : insn = next;
25062 : else
25063 : break;
25064 :
25065 0 : emit_insn_after (gen_nops (const1_rtx), insn);
25066 : }
25067 0 : }
25068 : /* Split vector load from parm_decl to elemental loads to avoid STLF
25069 : stalls. */
25070 : static void
25071 999980 : ix86_split_stlf_stall_load ()
25072 : {
25073 999980 : rtx_insn* insn, *start = get_insns ();
25074 999980 : unsigned window = 0;
25075 :
25076 27661095 : for (insn = start; insn; insn = NEXT_INSN (insn))
25077 : {
25078 27660245 : if (!NONDEBUG_INSN_P (insn))
25079 15720438 : continue;
25080 11939807 : window++;
25081 : /* Insert 64 vaddps %xmm18, %xmm19, %xmm20(no dependence between each
25082 : other, just emulate for pipeline) before stalled load, stlf stall
25083 : case is as fast as no stall cases on CLX.
25084 : Since CFG is freed before machine_reorg, just do a rough
25085 : calculation of the window according to the layout. */
25086 11939807 : if (window > (unsigned) x86_stlf_window_ninsns)
25087 : return;
25088 :
25089 11921407 : if (any_uncondjump_p (insn)
25090 11885752 : || ANY_RETURN_P (PATTERN (insn))
25091 23421448 : || CALL_P (insn))
25092 : return;
25093 :
25094 10940677 : rtx set = single_set (insn);
25095 10940677 : if (!set)
25096 446600 : continue;
25097 10494077 : rtx src = SET_SRC (set);
25098 20987796 : if (!MEM_P (src)
25099 : /* Only handle V2DFmode load since it doesn't need any scratch
25100 : register. */
25101 1492383 : || GET_MODE (src) != E_V2DFmode
25102 5748 : || !MEM_EXPR (src)
25103 4184 : || TREE_CODE (get_base_address (MEM_EXPR (src))) != PARM_DECL
25104 : /* Avoid invalid memory address.
25105 : i.e.
25106 : (mem/c:V2DF (plus:DI (reg/f:DI 7 sp)
25107 : (const_int 2147483640 [0x7ffffff8])))
25108 : Adjusting it by 8 puts the displacement at 0x80000000, out of
25109 : range for the signed 32-bit field an x86 address can encode. */
25110 10494795 : || !memory_address_addr_space_p (DFmode,
25111 10493719 : XEXP (adjust_address_nv (src, DFmode,
25112 : 8), 0),
25113 359 : MEM_ADDR_SPACE (src)))
25114 10493719 : continue;
25115 :
25116 358 : rtx zero = CONST0_RTX (V2DFmode);
25117 358 : rtx dest = SET_DEST (set);
25118 358 : rtx m = adjust_address (src, DFmode, 0);
25119 358 : rtx loadlpd = gen_sse2_loadlpd (dest, zero, m);
25120 358 : emit_insn_before (loadlpd, insn);
25121 358 : m = adjust_address (src, DFmode, 8);
25122 358 : rtx loadhpd = gen_sse2_loadhpd (dest, dest, m);
25123 358 : if (dump_file && (dump_flags & TDF_DETAILS))
25124 : {
25125 0 : fputs ("Due to potential STLF stall, split instruction:\n",
25126 : dump_file);
25127 0 : print_rtl_single (dump_file, insn);
25128 0 : fputs ("To:\n", dump_file);
25129 0 : print_rtl_single (dump_file, loadlpd);
25130 0 : print_rtl_single (dump_file, loadhpd);
25131 : }
25132 358 : PATTERN (insn) = loadhpd;
25133 358 : INSN_CODE (insn) = -1;
25134 358 : gcc_assert (recog_memoized (insn) != -1);
25135 : }
25136 : }
25137 :
25138 : /* Implement machine specific optimizations. We implement padding of returns
25139 : for K8 CPUs and pass to avoid 4 jumps in the single 16 byte window. */
25140 : static void
25141 1520562 : ix86_reorg (void)
25142 : {
25143 : /* We are freeing block_for_insn in the toplev to keep compatibility
25144 : with old MDEP_REORGS that are not CFG based. Recompute it now. */
25145 1520562 : compute_bb_for_insn ();
25146 :
25147 1520562 : if (TARGET_SEH && current_function_has_exception_handlers ())
25148 : ix86_seh_fixup_eh_fallthru ();
25149 :
25150 1520562 : if (optimize && optimize_function_for_speed_p (cfun))
25151 : {
25152 1002377 : if (TARGET_SSE2)
25153 999980 : ix86_split_stlf_stall_load ();
25154 1002377 : if (TARGET_PAD_SHORT_FUNCTION)
25155 63 : ix86_pad_short_function ();
25156 1002314 : else if (TARGET_PAD_RETURNS)
25157 46356 : ix86_pad_returns ();
25158 : #ifdef ASM_OUTPUT_MAX_SKIP_ALIGN
25159 1002377 : if (TARGET_FOUR_JUMP_LIMIT)
25160 46636 : ix86_avoid_jump_mispredicts ();
25161 : #endif
25162 : }
25163 1520562 : }
25164 :
25165 : /* Return nonzero when QImode register that must be represented via REX prefix
25166 : is used. */
25167 : bool
25168 9035951 : x86_extended_QIreg_mentioned_p (rtx_insn *insn)
25169 : {
25170 9035951 : int i;
25171 9035951 : extract_insn_cached (insn);
25172 34293370 : for (i = 0; i < recog_data.n_operands; i++)
25173 4758339 : if (GENERAL_REG_P (recog_data.operand[i])
25174 22535591 : && !QI_REGNO_P (REGNO (recog_data.operand[i])))
25175 : return true;
25176 : return false;
25177 : }
25178 :
25179 : /* Return true when INSN mentions register that must be encoded using REX
25180 : prefix. */
25181 : bool
25182 200571650 : x86_extended_reg_mentioned_p (rtx insn)
25183 : {
25184 200571650 : subrtx_iterator::array_type array;
25185 1054408171 : FOR_EACH_SUBRTX (iter, array, INSN_P (insn) ? PATTERN (insn) : insn, NONCONST)
25186 : {
25187 903378686 : const_rtx x = *iter;
25188 903378686 : if (REG_P (x)
25189 903378686 : && (REX_INT_REGNO_P (REGNO (x)) || REX_SSE_REGNO_P (REGNO (x))
25190 259168888 : || REX2_INT_REGNO_P (REGNO (x))))
25191 49542165 : return true;
25192 : }
25193 151029485 : return false;
25194 200571650 : }
25195 :
25196 : /* Return true when INSN mentions register that must be encoded using REX2
25197 : prefix. */
25198 : bool
25199 2048997 : x86_extended_rex2reg_mentioned_p (rtx insn)
25200 : {
25201 2048997 : subrtx_iterator::array_type array;
25202 9486667 : FOR_EACH_SUBRTX (iter, array, INSN_P (insn) ? PATTERN (insn) : insn, NONCONST)
25203 : {
25204 7438291 : const_rtx x = *iter;
25205 7438291 : if (REG_P (x) && REX2_INT_REGNO_P (REGNO (x)))
25206 621 : return true;
25207 : }
25208 2048376 : return false;
25209 2048997 : }
25210 :
25211 : /* Return true when rtx operands mentions register that must be encoded using
25212 : evex prefix. */
25213 : bool
25214 11 : x86_evex_reg_mentioned_p (rtx operands[], int nops)
25215 : {
25216 11 : int i;
25217 32 : for (i = 0; i < nops; i++)
25218 25 : if (EXT_REX_SSE_REG_P (operands[i])
25219 46 : || x86_extended_rex2reg_mentioned_p (operands[i]))
25220 : return true;
25221 : return false;
25222 : }
25223 :
25224 : /* If profitable, negate (without causing overflow) integer constant
25225 : of mode MODE at location LOC. Return true in this case. */
25226 : bool
25227 6052933 : x86_maybe_negate_const_int (rtx *loc, machine_mode mode)
25228 : {
25229 6052933 : HOST_WIDE_INT val;
25230 :
25231 6052933 : if (!CONST_INT_P (*loc))
25232 : return false;
25233 :
25234 5079138 : switch (mode)
25235 : {
25236 2922501 : case E_DImode:
25237 : /* DImode x86_64 constants must fit in 32 bits. */
25238 2922501 : gcc_assert (x86_64_immediate_operand (*loc, mode));
25239 :
25240 : mode = SImode;
25241 : break;
25242 :
25243 : case E_SImode:
25244 : case E_HImode:
25245 : case E_QImode:
25246 : break;
25247 :
25248 0 : default:
25249 0 : gcc_unreachable ();
25250 : }
25251 :
25252 : /* Avoid overflows. */
25253 5079138 : if (mode_signbit_p (mode, *loc))
25254 : return false;
25255 :
25256 5078391 : val = INTVAL (*loc);
25257 :
25258 : /* Make things pretty and `subl $4,%eax' rather than `addl $-4,%eax'.
25259 : Exceptions: -128 encodes smaller than 128, so swap sign and op. */
25260 5078391 : if ((val < 0 && val != -128)
25261 3348842 : || val == 128)
25262 : {
25263 1741497 : *loc = GEN_INT (-val);
25264 1741497 : return true;
25265 : }
25266 :
25267 : return false;
25268 : }
25269 :
25270 : /* Generate an unsigned DImode/SImode to FP conversion. This is the same code
25271 : optabs would emit if we didn't have TFmode patterns. */
25272 :
25273 : void
25274 5546 : x86_emit_floatuns (rtx operands[2])
25275 : {
25276 5546 : rtx_code_label *neglab, *donelab;
25277 5546 : rtx i0, i1, f0, in, out;
25278 5546 : machine_mode mode, inmode;
25279 :
25280 5546 : inmode = GET_MODE (operands[1]);
25281 5546 : gcc_assert (inmode == SImode || inmode == DImode);
25282 :
25283 5546 : out = operands[0];
25284 5546 : in = force_reg (inmode, operands[1]);
25285 5546 : mode = GET_MODE (out);
25286 5546 : neglab = gen_label_rtx ();
25287 5546 : donelab = gen_label_rtx ();
25288 5546 : f0 = gen_reg_rtx (mode);
25289 :
25290 5546 : emit_cmp_and_jump_insns (in, const0_rtx, LT, const0_rtx, inmode, 0, neglab);
25291 :
25292 5546 : expand_float (out, in, 0);
25293 :
25294 5546 : emit_jump_insn (gen_jump (donelab));
25295 5546 : emit_barrier ();
25296 :
25297 5546 : emit_label (neglab);
25298 :
25299 5546 : i0 = expand_simple_binop (inmode, LSHIFTRT, in, const1_rtx, NULL,
25300 : 1, OPTAB_DIRECT);
25301 5546 : i1 = expand_simple_binop (inmode, AND, in, const1_rtx, NULL,
25302 : 1, OPTAB_DIRECT);
25303 5546 : i0 = expand_simple_binop (inmode, IOR, i0, i1, i0, 1, OPTAB_DIRECT);
25304 :
25305 5546 : expand_float (f0, i0, 0);
25306 :
25307 5546 : emit_insn (gen_rtx_SET (out, gen_rtx_PLUS (mode, f0, f0)));
25308 :
25309 5546 : emit_label (donelab);
25310 5546 : }
25311 :
25312 : /* Return the diagnostic message string if conversion from FROMTYPE to
25313 : TOTYPE is not allowed, NULL otherwise. */
25314 :
25315 : static const char *
25316 1138633486 : ix86_invalid_conversion (const_tree fromtype, const_tree totype)
25317 : {
25318 1138633486 : machine_mode from_mode = element_mode (fromtype);
25319 1138633486 : machine_mode to_mode = element_mode (totype);
25320 :
25321 1138633486 : if (!TARGET_SSE2 && from_mode != to_mode)
25322 : {
25323 : /* Do no allow conversions to/from BFmode/HFmode scalar types
25324 : when TARGET_SSE2 is not available. */
25325 470309 : if (from_mode == BFmode)
25326 : return N_("invalid conversion from type %<__bf16%> "
25327 : "without option %<-msse2%>");
25328 470308 : if (from_mode == HFmode)
25329 : return N_("invalid conversion from type %<_Float16%> "
25330 : "without option %<-msse2%>");
25331 470308 : if (to_mode == BFmode)
25332 : return N_("invalid conversion to type %<__bf16%> "
25333 : "without option %<-msse2%>");
25334 470308 : if (to_mode == HFmode)
25335 : return N_("invalid conversion to type %<_Float16%> "
25336 : "without option %<-msse2%>");
25337 : }
25338 :
25339 : /* Warn for silent implicit conversion between __bf16 and short,
25340 : since __bfloat16 is refined as real __bf16 instead of short
25341 : since GCC13. */
25342 1138633484 : if (element_mode (fromtype) != element_mode (totype)
25343 1138633484 : && (TARGET_AVX512BF16 || TARGET_AVXNECONVERT))
25344 : {
25345 : /* Warn for silent implicit conversion where user may expect
25346 : a bitcast. */
25347 10462012 : if ((TYPE_MODE (fromtype) == BFmode
25348 279 : && TYPE_MODE (totype) == HImode)
25349 10462290 : || (TYPE_MODE (totype) == BFmode
25350 427 : && TYPE_MODE (fromtype) == HImode))
25351 1 : warning (0, "%<__bfloat16%> is redefined from typedef %<short%> "
25352 : "to real %<__bf16%> since GCC 13.1, be careful of "
25353 : "implicit conversion between %<__bf16%> and %<short%>; "
25354 : "an explicit bitcast may be needed here");
25355 : }
25356 :
25357 : /* Conversion allowed. */
25358 : return NULL;
25359 : }
25360 :
25361 : /* Return the diagnostic message string if the unary operation OP is
25362 : not permitted on TYPE, NULL otherwise. */
25363 :
25364 : static const char *
25365 94982360 : ix86_invalid_unary_op (int op, const_tree type)
25366 : {
25367 94982360 : machine_mode mmode = element_mode (type);
25368 : /* Reject all single-operand operations on BFmode/HFmode except for &
25369 : when TARGET_SSE2 is not available. */
25370 94982360 : if (!TARGET_SSE2 && op != ADDR_EXPR)
25371 : {
25372 111586 : if (mmode == BFmode)
25373 : return N_("operation not permitted on type %<__bf16%> "
25374 : "without option %<-msse2%>");
25375 111586 : if (mmode == HFmode)
25376 0 : return N_("operation not permitted on type %<_Float16%> "
25377 : "without option %<-msse2%>");
25378 : }
25379 :
25380 : /* Operation allowed. */
25381 : return NULL;
25382 : }
25383 :
25384 : /* Return the diagnostic message string if the binary operation OP is
25385 : not permitted on TYPE1 and TYPE2, NULL otherwise. */
25386 :
25387 : static const char *
25388 172348286 : ix86_invalid_binary_op (int op ATTRIBUTE_UNUSED, const_tree type1,
25389 : const_tree type2)
25390 : {
25391 172348286 : machine_mode type1_mode = element_mode (type1);
25392 172348286 : machine_mode type2_mode = element_mode (type2);
25393 : /* Reject all 2-operand operations on BFmode or HFmode
25394 : when TARGET_SSE2 is not available. */
25395 172348286 : if (!TARGET_SSE2)
25396 : {
25397 1013344 : if (type1_mode == BFmode || type2_mode == BFmode)
25398 : return N_("operation not permitted on type %<__bf16%> "
25399 : "without option %<-msse2%>");
25400 :
25401 1013344 : if (type1_mode == HFmode || type2_mode == HFmode)
25402 0 : return N_("operation not permitted on type %<_Float16%> "
25403 : "without option %<-msse2%>");
25404 : }
25405 :
25406 : /* Operation allowed. */
25407 : return NULL;
25408 : }
25409 :
25410 :
25411 : /* Target hook for scalar_mode_supported_p. */
25412 : static bool
25413 5593113 : ix86_scalar_mode_supported_p (scalar_mode mode)
25414 : {
25415 5593113 : if (DECIMAL_FLOAT_MODE_P (mode))
25416 643974 : return default_decimal_float_supported_p ();
25417 4949139 : else if (mode == TFmode)
25418 : return true;
25419 4619317 : else if (mode == HFmode || mode == BFmode)
25420 : return true;
25421 : else
25422 3961656 : return default_scalar_mode_supported_p (mode);
25423 : }
25424 :
25425 : /* Implement TARGET_LIBGCC_FLOATING_POINT_MODE_SUPPORTED_P - return TRUE
25426 : if MODE is HFmode, and punt to the generic implementation otherwise. */
25427 :
25428 : static bool
25429 2265188 : ix86_libgcc_floating_mode_supported_p (scalar_float_mode mode)
25430 : {
25431 : /* NB: Always return TRUE for HFmode so that the _Float16 type will
25432 : be defined by the C front-end for AVX512FP16 intrinsics. We will
25433 : issue an error in ix86_expand_move for HFmode if AVX512FP16 isn't
25434 : enabled. */
25435 1936833 : return ((mode == HFmode || mode == BFmode)
25436 3873666 : ? true
25437 1608478 : : default_libgcc_floating_mode_supported_p (mode));
25438 : }
25439 :
25440 : /* Implements target hook vector_mode_supported_p. */
25441 : static bool
25442 1371658039 : ix86_vector_mode_supported_p (machine_mode mode)
25443 : {
25444 : /* For ia32, scalar TImode isn't supported and so V1TImode shouldn't be
25445 : either. */
25446 1502641970 : if (!TARGET_64BIT && GET_MODE_INNER (mode) == TImode)
25447 : return false;
25448 1371551859 : if (TARGET_SSE && VALID_SSE_REG_MODE (mode))
25449 : return true;
25450 1156480454 : if (TARGET_SSE2 && VALID_SSE2_REG_MODE (mode))
25451 : return true;
25452 520254064 : if (TARGET_AVX && VALID_AVX256_REG_MODE (mode))
25453 : return true;
25454 376294256 : if (TARGET_AVX512F && VALID_AVX512F_REG_MODE (mode))
25455 : return true;
25456 238104169 : if ((TARGET_MMX || TARGET_MMX_WITH_SSE)
25457 238046612 : && VALID_MMX_REG_MODE (mode))
25458 : return true;
25459 37085880 : if ((TARGET_3DNOW || TARGET_MMX_WITH_SSE)
25460 36436505 : && VALID_MMX_REG_MODE_3DNOW (mode))
25461 : return true;
25462 23775022 : if (mode == V2QImode)
25463 24813 : return true;
25464 : return false;
25465 : }
25466 :
25467 : /* Target hook for c_mode_for_suffix. */
25468 : static machine_mode
25469 203917 : ix86_c_mode_for_suffix (char suffix)
25470 : {
25471 203917 : if (suffix == 'q')
25472 : return TFmode;
25473 37 : if (suffix == 'w')
25474 : return XFmode;
25475 :
25476 0 : return VOIDmode;
25477 : }
25478 :
25479 : /* Helper function to map common constraints to non-EGPR ones.
25480 : All related constraints have h prefix, and h plus Upper letter
25481 : means the constraint is strictly EGPR enabled, while h plus
25482 : lower letter indicates the constraint is strictly gpr16 only.
25483 :
25484 : Specially for "g" constraint, split it to rmi as there is
25485 : no corresponding general constraint define for backend.
25486 :
25487 : Here is the full list to map constraints that may involve
25488 : gpr to h prefixed.
25489 :
25490 : "g" -> "jrjmi"
25491 : "r" -> "jr"
25492 : "m" -> "jm"
25493 : "<" -> "j<"
25494 : ">" -> "j>"
25495 : "o" -> "jo"
25496 : "V" -> "jV"
25497 : "p" -> "jp"
25498 : "Bm" -> "ja"
25499 : */
25500 :
25501 64 : static void map_egpr_constraints (vec<const char *> &constraints)
25502 : {
25503 74 : for (size_t i = 0; i < constraints.length(); i++)
25504 : {
25505 10 : const char *cur = constraints[i];
25506 :
25507 10 : if (startswith (cur, "=@cc"))
25508 0 : continue;
25509 :
25510 10 : int len = strlen (cur);
25511 10 : auto_vec<char> buf;
25512 :
25513 24 : for (int j = 0; j < len; j++)
25514 : {
25515 14 : switch (cur[j])
25516 : {
25517 2 : case 'g':
25518 2 : buf.safe_push ('j');
25519 2 : buf.safe_push ('r');
25520 2 : buf.safe_push ('j');
25521 2 : buf.safe_push ('m');
25522 2 : buf.safe_push ('i');
25523 2 : break;
25524 8 : case 'r':
25525 8 : case 'm':
25526 8 : case '<':
25527 8 : case '>':
25528 8 : case 'o':
25529 8 : case 'V':
25530 8 : case 'p':
25531 8 : buf.safe_push ('j');
25532 8 : buf.safe_push (cur[j]);
25533 8 : break;
25534 0 : case 'B':
25535 0 : if (cur[j + 1] == 'm')
25536 : {
25537 0 : buf.safe_push ('j');
25538 0 : buf.safe_push ('a');
25539 0 : j++;
25540 : }
25541 : else
25542 : {
25543 0 : buf.safe_push (cur[j]);
25544 0 : buf.safe_push (cur[j + 1]);
25545 0 : j++;
25546 : }
25547 : break;
25548 0 : case 'T':
25549 0 : case 'Y':
25550 0 : case 'W':
25551 0 : case 'j':
25552 0 : buf.safe_push (cur[j]);
25553 0 : buf.safe_push (cur[j + 1]);
25554 0 : j++;
25555 0 : break;
25556 0 : case '{':
25557 0 : do
25558 : {
25559 0 : buf.safe_push (cur[j]);
25560 0 : } while (cur[j++] != '}');
25561 : break;
25562 4 : default:
25563 4 : buf.safe_push (cur[j]);
25564 4 : break;
25565 : }
25566 : }
25567 10 : buf.safe_push ('\0');
25568 20 : constraints[i] = xstrdup (buf.address ());
25569 10 : }
25570 64 : }
25571 :
25572 : /* Worker function for TARGET_MD_ASM_ADJUST.
25573 :
25574 : We implement asm flag outputs, and maintain source compatibility
25575 : with the old cc0-based compiler. */
25576 :
25577 : static rtx_insn *
25578 94293 : ix86_md_asm_adjust (vec<rtx> &outputs, vec<rtx> & /*inputs*/,
25579 : vec<machine_mode> & /*input_modes*/,
25580 : vec<const char *> &constraints, vec<rtx> &/*uses*/,
25581 : vec<rtx> &clobbers, HARD_REG_SET &clobbered_regs,
25582 : location_t loc)
25583 : {
25584 94293 : bool saw_asm_flag = false;
25585 :
25586 94293 : start_sequence ();
25587 :
25588 94293 : if (TARGET_APX_EGPR && !ix86_apx_inline_asm_use_gpr32)
25589 64 : map_egpr_constraints (constraints);
25590 :
25591 267148 : for (unsigned i = 0, n = outputs.length (); i < n; ++i)
25592 : {
25593 79380 : const char *con = constraints[i];
25594 79380 : if (!startswith (con, "=@cc"))
25595 79292 : continue;
25596 88 : con += 4;
25597 88 : if (strchr (con, ',') != NULL)
25598 : {
25599 1 : error_at (loc, "alternatives not allowed in %<asm%> flag output");
25600 1 : continue;
25601 : }
25602 :
25603 87 : bool invert = false;
25604 87 : if (con[0] == 'n')
25605 19 : invert = true, con++;
25606 :
25607 87 : machine_mode mode = CCmode;
25608 87 : rtx_code code = UNKNOWN;
25609 :
25610 87 : switch (con[0])
25611 : {
25612 15 : case 'a':
25613 15 : if (con[1] == 0)
25614 : mode = CCAmode, code = EQ;
25615 4 : else if (con[1] == 'e' && con[2] == 0)
25616 : mode = CCCmode, code = NE;
25617 : break;
25618 11 : case 'b':
25619 11 : if (con[1] == 0)
25620 : mode = CCCmode, code = EQ;
25621 6 : else if (con[1] == 'e' && con[2] == 0)
25622 : mode = CCAmode, code = NE;
25623 : break;
25624 14 : case 'c':
25625 14 : if (con[1] == 0)
25626 : mode = CCCmode, code = EQ;
25627 : break;
25628 8 : case 'e':
25629 8 : if (con[1] == 0)
25630 : mode = CCZmode, code = EQ;
25631 : break;
25632 11 : case 'g':
25633 11 : if (con[1] == 0)
25634 : mode = CCGCmode, code = GT;
25635 5 : else if (con[1] == 'e' && con[2] == 0)
25636 : mode = CCGCmode, code = GE;
25637 : break;
25638 10 : case 'l':
25639 10 : if (con[1] == 0)
25640 : mode = CCGCmode, code = LT;
25641 5 : else if (con[1] == 'e' && con[2] == 0)
25642 : mode = CCGCmode, code = LE;
25643 : break;
25644 4 : case 'o':
25645 4 : if (con[1] == 0)
25646 : mode = CCOmode, code = EQ;
25647 : break;
25648 4 : case 'p':
25649 4 : if (con[1] == 0)
25650 : mode = CCPmode, code = EQ;
25651 : break;
25652 4 : case 's':
25653 4 : if (con[1] == 0)
25654 : mode = CCSmode, code = EQ;
25655 : break;
25656 6 : case 'z':
25657 6 : if (con[1] == 0)
25658 : mode = CCZmode, code = EQ;
25659 : break;
25660 : }
25661 1 : if (code == UNKNOWN)
25662 : {
25663 1 : error_at (loc, "unknown %<asm%> flag output %qs", constraints[i]);
25664 1 : continue;
25665 : }
25666 86 : if (invert)
25667 19 : code = reverse_condition (code);
25668 :
25669 86 : rtx dest = outputs[i];
25670 86 : if (!saw_asm_flag)
25671 : {
25672 : /* This is the first asm flag output. Here we put the flags
25673 : register in as the real output and adjust the condition to
25674 : allow it. */
25675 75 : constraints[i] = "=Bf";
25676 75 : outputs[i] = gen_rtx_REG (CCmode, FLAGS_REG);
25677 75 : saw_asm_flag = true;
25678 : }
25679 : else
25680 : {
25681 : /* We don't need the flags register as output twice. */
25682 11 : constraints[i] = "=X";
25683 11 : outputs[i] = gen_rtx_SCRATCH (SImode);
25684 : }
25685 :
25686 86 : rtx x = gen_rtx_REG (mode, FLAGS_REG);
25687 86 : x = gen_rtx_fmt_ee (code, QImode, x, const0_rtx);
25688 :
25689 86 : machine_mode dest_mode = GET_MODE (dest);
25690 86 : if (!SCALAR_INT_MODE_P (dest_mode))
25691 : {
25692 3 : error_at (loc, "invalid type for %<asm%> flag output");
25693 3 : continue;
25694 : }
25695 :
25696 83 : if (dest_mode == QImode)
25697 73 : emit_insn (gen_rtx_SET (dest, x));
25698 : else
25699 : {
25700 10 : rtx reg = gen_reg_rtx (QImode);
25701 10 : emit_insn (gen_rtx_SET (reg, x));
25702 :
25703 10 : reg = convert_to_mode (dest_mode, reg, 1);
25704 10 : emit_move_insn (dest, reg);
25705 : }
25706 : }
25707 :
25708 94293 : rtx_insn *seq = end_sequence ();
25709 :
25710 94293 : if (saw_asm_flag)
25711 : return seq;
25712 : else
25713 : {
25714 : /* If we had no asm flag outputs, clobber the flags. */
25715 94218 : clobbers.safe_push (gen_rtx_REG (CCmode, FLAGS_REG));
25716 94218 : SET_HARD_REG_BIT (clobbered_regs, FLAGS_REG);
25717 94218 : return NULL;
25718 : }
25719 : }
25720 :
25721 : /* Implements target vector targetm.asm.encode_section_info. */
25722 :
25723 : static void ATTRIBUTE_UNUSED
25724 10090114 : ix86_encode_section_info (tree decl, rtx rtl, int first)
25725 : {
25726 10090114 : default_encode_section_info (decl, rtl, first);
25727 :
25728 10090114 : if (ix86_in_large_data_p (decl))
25729 32 : SYMBOL_REF_FLAGS (XEXP (rtl, 0)) |= SYMBOL_FLAG_FAR_ADDR;
25730 10090114 : }
25731 :
25732 : /* Worker function for REVERSE_CONDITION. */
25733 :
25734 : enum rtx_code
25735 32820279 : ix86_reverse_condition (enum rtx_code code, machine_mode mode)
25736 : {
25737 32820279 : return (mode == CCFPmode
25738 32820279 : ? reverse_condition_maybe_unordered (code)
25739 28353851 : : reverse_condition (code));
25740 : }
25741 :
25742 : /* Output code to perform an x87 FP register move, from OPERANDS[1]
25743 : to OPERANDS[0]. */
25744 :
25745 : const char *
25746 653236 : output_387_reg_move (rtx_insn *insn, rtx *operands)
25747 : {
25748 653236 : if (REG_P (operands[0]))
25749 : {
25750 547752 : if (REG_P (operands[1])
25751 547752 : && find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
25752 : {
25753 292637 : if (REGNO (operands[0]) == FIRST_STACK_REG)
25754 274718 : return output_387_ffreep (operands, 0);
25755 : return "fstp\t%y0";
25756 : }
25757 255115 : if (STACK_TOP_P (operands[0]))
25758 255115 : return "fld%Z1\t%y1";
25759 : return "fst\t%y0";
25760 : }
25761 105484 : else if (MEM_P (operands[0]))
25762 : {
25763 105484 : gcc_assert (REG_P (operands[1]));
25764 105484 : if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
25765 : return "fstp%Z0\t%y0";
25766 : else
25767 : {
25768 : /* There is no non-popping store to memory for XFmode.
25769 : So if we need one, follow the store with a load. */
25770 5593 : if (GET_MODE (operands[0]) == XFmode)
25771 : return "fstp%Z0\t%y0\n\tfld%Z0\t%y0";
25772 : else
25773 1935 : return "fst%Z0\t%y0";
25774 : }
25775 : }
25776 : else
25777 0 : gcc_unreachable();
25778 : }
25779 : #ifdef TARGET_SOLARIS
25780 : /* Solaris implementation of TARGET_ASM_NAMED_SECTION. */
25781 :
25782 : static void
25783 : i386_solaris_elf_named_section (const char *name, unsigned int flags,
25784 : tree decl)
25785 : {
25786 : /* With Binutils 2.15, the "@unwind" marker must be specified on
25787 : every occurrence of the ".eh_frame" section, not just the first
25788 : one. */
25789 : if (TARGET_64BIT
25790 : && strcmp (name, ".eh_frame") == 0)
25791 : {
25792 : fprintf (asm_out_file, "\t.section\t%s,\"%s\",@unwind\n", name,
25793 : flags & SECTION_WRITE ? "aw" : "a");
25794 : return;
25795 : }
25796 :
25797 : #if HAVE_SOLARIS_AS
25798 : if (HAVE_COMDAT_GROUP && flags & SECTION_LINKONCE)
25799 : {
25800 : solaris_elf_asm_comdat_section (name, flags, decl);
25801 : return;
25802 : }
25803 :
25804 : /* Solaris/x86 as uses the same syntax for the SHF_EXCLUDE flags as the
25805 : SPARC assembler. One cannot mix single-letter flags and #exclude, so
25806 : only emit the latter here. */
25807 : if (flags & SECTION_EXCLUDE)
25808 : {
25809 : fprintf (asm_out_file, "\t.section\t%s,#exclude\n", name);
25810 : return;
25811 : }
25812 : #endif
25813 :
25814 : default_elf_asm_named_section (name, flags, decl);
25815 : }
25816 : #endif /* TARGET_SOLARIS */
25817 :
25818 : /* Return the mangling of TYPE if it is an extended fundamental type. */
25819 :
25820 : static const char *
25821 1108305373 : ix86_mangle_type (const_tree type)
25822 : {
25823 1108305373 : type = TYPE_MAIN_VARIANT (type);
25824 :
25825 1108305373 : if (TREE_CODE (type) != VOID_TYPE && TREE_CODE (type) != BOOLEAN_TYPE
25826 : && TREE_CODE (type) != INTEGER_TYPE && TREE_CODE (type) != REAL_TYPE)
25827 : return NULL;
25828 :
25829 601137702 : if (type == float128_type_node || type == float64x_type_node)
25830 : return NULL;
25831 :
25832 600333058 : switch (TYPE_MODE (type))
25833 : {
25834 : case E_BFmode:
25835 : return "DF16b";
25836 504824 : case E_HFmode:
25837 : /* _Float16 is "DF16_".
25838 : Align with clang's decision in https://reviews.llvm.org/D33719. */
25839 504824 : return "DF16_";
25840 700248 : case E_TFmode:
25841 : /* __float128 is "g". */
25842 700248 : return "g";
25843 8210639 : case E_XFmode:
25844 : /* "long double" or __float80 is "e". */
25845 8210639 : return "e";
25846 : default:
25847 : return NULL;
25848 : }
25849 : }
25850 :
25851 : /* Create C++ tinfo symbols for only conditionally available fundamental
25852 : types. */
25853 :
25854 : static void
25855 5 : ix86_emit_support_tinfos (emit_support_tinfos_callback callback)
25856 : {
25857 5 : extern tree ix86_float16_type_node;
25858 5 : extern tree ix86_bf16_type_node;
25859 :
25860 5 : if (!TARGET_SSE2)
25861 : {
25862 0 : if (!float16_type_node)
25863 0 : float16_type_node = ix86_float16_type_node;
25864 0 : if (!bfloat16_type_node)
25865 0 : bfloat16_type_node = ix86_bf16_type_node;
25866 0 : callback (float16_type_node);
25867 0 : callback (bfloat16_type_node);
25868 0 : float16_type_node = NULL_TREE;
25869 0 : bfloat16_type_node = NULL_TREE;
25870 : }
25871 5 : }
25872 :
25873 : static GTY(()) tree ix86_tls_stack_chk_guard_decl;
25874 :
25875 : static tree
25876 336 : ix86_stack_protect_guard (void)
25877 : {
25878 336 : if (TARGET_SSP_TLS_GUARD)
25879 : {
25880 261 : tree type_node = lang_hooks.types.type_for_mode (ptr_mode, 1);
25881 261 : int qual = ENCODE_QUAL_ADDR_SPACE (ix86_stack_protector_guard_reg);
25882 261 : tree type = build_qualified_type (type_node, qual);
25883 261 : tree t;
25884 :
25885 261 : if (OPTION_SET_P (ix86_stack_protector_guard_symbol_str))
25886 : {
25887 1 : t = ix86_tls_stack_chk_guard_decl;
25888 :
25889 1 : if (t == NULL)
25890 : {
25891 1 : rtx x;
25892 :
25893 1 : t = build_decl
25894 1 : (UNKNOWN_LOCATION, VAR_DECL,
25895 : get_identifier (ix86_stack_protector_guard_symbol_str),
25896 : type);
25897 1 : TREE_STATIC (t) = 1;
25898 1 : TREE_PUBLIC (t) = 1;
25899 1 : DECL_EXTERNAL (t) = 1;
25900 1 : TREE_USED (t) = 1;
25901 1 : TREE_THIS_VOLATILE (t) = 1;
25902 1 : DECL_ARTIFICIAL (t) = 1;
25903 1 : DECL_IGNORED_P (t) = 1;
25904 :
25905 : /* Do not share RTL as the declaration is visible outside of
25906 : current function. */
25907 1 : x = DECL_RTL (t);
25908 1 : RTX_FLAG (x, used) = 1;
25909 :
25910 1 : ix86_tls_stack_chk_guard_decl = t;
25911 : }
25912 : }
25913 : else
25914 : {
25915 260 : tree asptrtype = build_pointer_type (type);
25916 :
25917 260 : t = build_int_cst (asptrtype, ix86_stack_protector_guard_offset);
25918 260 : t = build2 (MEM_REF, asptrtype, t,
25919 : build_int_cst (asptrtype, 0));
25920 260 : TREE_THIS_VOLATILE (t) = 1;
25921 : }
25922 :
25923 : return t;
25924 : }
25925 :
25926 75 : return default_stack_protect_guard ();
25927 : }
25928 :
25929 : /* Implement TARGET_STACK_PROTECT_GUARD_SYMBOL_P. */
25930 :
25931 : static bool
25932 208511 : ix86_stack_protect_guard_symbol_p (void)
25933 : {
25934 208511 : return TARGET_SSP_GLOBAL_GUARD;
25935 : }
25936 :
25937 : static bool
25938 924 : ix86_stack_protect_runtime_enabled_p (void)
25939 : {
25940 : /* Naked functions should not enable stack protector. */
25941 924 : return !ix86_function_naked (current_function_decl);
25942 : }
25943 :
25944 : /* For 32-bit code we can save PIC register setup by using
25945 : __stack_chk_fail_local hidden function instead of calling
25946 : __stack_chk_fail directly. 64-bit code doesn't need to setup any PIC
25947 : register, so it is better to call __stack_chk_fail directly. */
25948 :
25949 : static tree ATTRIBUTE_UNUSED
25950 329 : ix86_stack_protect_fail (void)
25951 : {
25952 329 : return TARGET_64BIT
25953 329 : ? default_external_stack_protect_fail ()
25954 1 : : default_hidden_stack_protect_fail ();
25955 : }
25956 :
25957 : /* Select a format to encode pointers in exception handling data. CODE
25958 : is 0 for data, 1 for code labels, 2 for function pointers. GLOBAL is
25959 : true if the symbol may be affected by dynamic relocations.
25960 :
25961 : ??? All x86 object file formats are capable of representing this.
25962 : After all, the relocation needed is the same as for the call insn.
25963 : Whether or not a particular assembler allows us to enter such, I
25964 : guess we'll have to see. */
25965 :
25966 : int
25967 815679 : asm_preferred_eh_data_format (int code, int global)
25968 : {
25969 : /* PE-COFF is effectively always -fPIC because of the .reloc section. */
25970 815679 : if (flag_pic || TARGET_PECOFF || !ix86_direct_extern_access)
25971 : {
25972 85479 : int type = DW_EH_PE_sdata8;
25973 85479 : if (ptr_mode == SImode
25974 71494 : || ix86_cmodel == CM_SMALL_PIC
25975 85565 : || (ix86_cmodel == CM_MEDIUM_PIC && (global || code)))
25976 : type = DW_EH_PE_sdata4;
25977 121396 : return (global ? DW_EH_PE_indirect : 0) | DW_EH_PE_pcrel | type;
25978 : }
25979 :
25980 730200 : if (ix86_cmodel == CM_SMALL
25981 18670 : || (ix86_cmodel == CM_MEDIUM && code))
25982 711543 : return DW_EH_PE_udata4;
25983 :
25984 : return DW_EH_PE_absptr;
25985 : }
25986 :
25987 : /* Cost of constructing or destructing a vector in VECMODE from/to elements
25988 : of ELMODE. */
25989 : static int
25990 789445 : ix86_vector_cd_cost (machine_mode vecmode, machine_mode elmode)
25991 : {
25992 1578890 : if (GET_MODE_BITSIZE (vecmode) < 128)
25993 545928 : return ((GET_MODE_BITSIZE (vecmode) / GET_MODE_BITSIZE (elmode) - 1)
25994 272964 : * ix86_cost->sse_op);
25995 :
25996 516481 : int n = GET_MODE_BITSIZE (vecmode) / 128;
25997 516481 : int cost = 0;
25998 : /* Element inserts/extracts into/from N SSE vectors, the possible
25999 : GPR <-> XMM moves have to be accounted for elsewhere. */
26000 1032962 : if (GET_MODE_BITSIZE (elmode) < 128)
26001 1031716 : cost += n * (128 / GET_MODE_BITSIZE (elmode) - 1) * ix86_cost->sse_op;
26002 516481 : if (GET_MODE_BITSIZE (vecmode) >= 256
26003 526436 : && GET_MODE_BITSIZE (elmode) < 256)
26004 : /* N/2 vinserti128/vextracti128 for SSE <-> AVX256. */
26005 9955 : cost += n * ix86_vec_cost (V32QImode, ix86_cost->sse_op) / 2;
26006 1032962 : if (GET_MODE_BITSIZE (vecmode) == 512)
26007 : /* One vinserti64x4/vextracti64x4 for AVX256 <-> AVX512. */
26008 2048 : cost += ix86_vec_cost (vecmode, ix86_cost->sse_op);
26009 : return cost;
26010 : }
26011 :
26012 : /* Worker for ix86_builtin_vectorization_cost and the fallback calls
26013 : from ix86_vector_costs::add_stmt_cost. */
26014 : static int
26015 15968454 : ix86_default_vector_cost (enum vect_cost_for_stmt type_of_cost,
26016 : machine_mode mode)
26017 : {
26018 15968454 : bool fp = FLOAT_MODE_P (mode);
26019 15968454 : int index;
26020 15968454 : switch (type_of_cost)
26021 : {
26022 1889240 : case scalar_stmt:
26023 1889240 : return fp ? ix86_cost->addss : COSTS_N_INSNS (1);
26024 :
26025 1897571 : case scalar_load:
26026 : /* load/store costs are relative to register move which is 2. Recompute
26027 : it to COSTS_N_INSNS so everything have same base. */
26028 3795142 : return COSTS_N_INSNS (fp ? ix86_cost->sse_load[0]
26029 1897571 : : ix86_cost->int_load [2]) / 2;
26030 :
26031 4048918 : case scalar_store:
26032 8097836 : return COSTS_N_INSNS (fp ? ix86_cost->sse_store[0]
26033 4048918 : : ix86_cost->int_store [2]) / 2;
26034 :
26035 1129063 : case vector_stmt:
26036 2258126 : return ix86_vec_cost (mode,
26037 2258126 : fp ? ix86_cost->addss : ix86_cost->sse_op);
26038 :
26039 2020521 : case vector_load:
26040 2020521 : index = sse_store_index (mode);
26041 : /* See PR82713 - we may end up being called on non-vector type. */
26042 2020521 : if (index < 0)
26043 104414 : index = 2;
26044 2020521 : return COSTS_N_INSNS (ix86_cost->sse_load[index]) / 2;
26045 :
26046 1007217 : case vector_store:
26047 1007217 : index = sse_store_index (mode);
26048 : /* See PR82713 - we may end up being called on non-vector type. */
26049 1007217 : if (index < 0)
26050 94765 : index = 2;
26051 1007217 : return COSTS_N_INSNS (ix86_cost->sse_store[index]) / 2;
26052 :
26053 564892 : case vec_to_scalar:
26054 : /* This is a lane extraction, possibly from lane zero. */
26055 564892 : return (ix86_vec_cost (mode, ix86_cost->sse_op)
26056 564892 : + (fp ? 0 : COSTS_N_INSNS (ix86_cost->sse_to_integer) / 2));
26057 :
26058 490324 : case scalar_to_vec:
26059 : /* This is always a full splat from scalar, not a lane insertion. */
26060 490324 : return ix86_vec_cost (mode, ix86_cost->sse_op);
26061 :
26062 : /* We should have separate costs for unaligned loads and gather/scatter.
26063 : Do that incrementally. */
26064 534834 : case unaligned_load:
26065 534834 : index = sse_store_index (mode);
26066 : /* See PR82713 - we may end up being called on non-vector type. */
26067 534834 : if (index < 0)
26068 3652 : index = 2;
26069 534834 : return COSTS_N_INSNS (ix86_cost->sse_unaligned_load[index]) / 2;
26070 :
26071 858066 : case unaligned_store:
26072 858066 : index = sse_store_index (mode);
26073 : /* See PR82713 - we may end up being called on non-vector type. */
26074 858066 : if (index < 0)
26075 17343 : index = 2;
26076 858066 : return COSTS_N_INSNS (ix86_cost->sse_unaligned_store[index]) / 2;
26077 :
26078 0 : case vector_gather_load:
26079 0 : return ix86_vec_cost (mode,
26080 0 : COSTS_N_INSNS
26081 : (ix86_cost->gather_static
26082 : + ix86_cost->gather_per_elt
26083 0 : * GET_MODE_NUNITS (mode)) / 2);
26084 :
26085 0 : case vector_scatter_store:
26086 0 : return ix86_vec_cost (mode,
26087 0 : COSTS_N_INSNS
26088 : (ix86_cost->scatter_static
26089 : + ix86_cost->scatter_per_elt
26090 0 : * GET_MODE_NUNITS (mode)) / 2);
26091 :
26092 369962 : case cond_branch_taken:
26093 369962 : return ix86_cost->cond_taken_branch_cost;
26094 :
26095 8696 : case cond_branch_not_taken:
26096 8696 : return ix86_cost->cond_not_taken_branch_cost;
26097 :
26098 302881 : case vec_perm:
26099 302881 : return ix86_vec_cost (mode, ix86_cost->sse_op);
26100 :
26101 89894 : case vec_promote_demote:
26102 89894 : if (fp)
26103 11734 : return vec_fp_conversion_cost (ix86_tune_cost, mode);
26104 78160 : return ix86_vec_cost (mode, ix86_cost->sse_op);
26105 :
26106 756375 : case vec_construct:
26107 756375 : case vec_deconstruct:
26108 1512750 : return ix86_vector_cd_cost (mode, GET_MODE_INNER (mode));
26109 :
26110 0 : default:
26111 0 : gcc_unreachable ();
26112 : }
26113 : }
26114 :
26115 : /* Implement targetm.vectorize.builtin_vectorization_cost. */
26116 : static int
26117 9511836 : ix86_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
26118 : tree vectype, int)
26119 : {
26120 9511836 : machine_mode mode = TImode;
26121 9511836 : if (vectype != NULL)
26122 5992692 : mode = TYPE_MODE (vectype);
26123 9511836 : return ix86_default_vector_cost (type_of_cost, mode);
26124 : }
26125 :
26126 :
26127 : /* This function returns the calling abi specific va_list type node.
26128 : It returns the FNDECL specific va_list type. */
26129 :
26130 : static tree
26131 47669 : ix86_fn_abi_va_list (tree fndecl)
26132 : {
26133 47669 : if (!TARGET_64BIT)
26134 726 : return va_list_type_node;
26135 46943 : gcc_assert (fndecl != NULL_TREE);
26136 :
26137 46943 : if (ix86_function_abi ((const_tree) fndecl) == MS_ABI)
26138 12880 : return ms_va_list_type_node;
26139 : else
26140 34063 : return sysv_va_list_type_node;
26141 : }
26142 :
26143 : /* Returns the canonical va_list type specified by TYPE. If there
26144 : is no valid TYPE provided, it return NULL_TREE. */
26145 :
26146 : static tree
26147 247575 : ix86_canonical_va_list_type (tree type)
26148 : {
26149 247575 : if (TARGET_64BIT)
26150 : {
26151 247073 : if (lookup_attribute ("ms_abi va_list", TYPE_ATTRIBUTES (type)))
26152 5948 : return ms_va_list_type_node;
26153 :
26154 241125 : if ((TREE_CODE (type) == ARRAY_TYPE
26155 50159 : && integer_zerop (array_type_nelts_minus_one (type)))
26156 241125 : || POINTER_TYPE_P (type))
26157 : {
26158 189063 : tree elem_type = TREE_TYPE (type);
26159 189063 : if (TREE_CODE (elem_type) == RECORD_TYPE
26160 341158 : && lookup_attribute ("sysv_abi va_list",
26161 152095 : TYPE_ATTRIBUTES (elem_type)))
26162 152095 : return sysv_va_list_type_node;
26163 : }
26164 :
26165 : return NULL_TREE;
26166 : }
26167 :
26168 502 : return std_canonical_va_list_type (type);
26169 : }
26170 :
26171 : /* Iterate through the target-specific builtin types for va_list.
26172 : IDX denotes the iterator, *PTREE is set to the result type of
26173 : the va_list builtin, and *PNAME to its internal type.
26174 : Returns zero if there is no element for this index, otherwise
26175 : IDX should be increased upon the next call.
26176 : Note, do not iterate a base builtin's name like __builtin_va_list.
26177 : Used from c_common_nodes_and_builtins. */
26178 :
26179 : static int
26180 632184 : ix86_enum_va_list (int idx, const char **pname, tree *ptree)
26181 : {
26182 632184 : if (TARGET_64BIT)
26183 : {
26184 626808 : switch (idx)
26185 : {
26186 : default:
26187 : break;
26188 :
26189 208936 : case 0:
26190 208936 : *ptree = ms_va_list_type_node;
26191 208936 : *pname = "__builtin_ms_va_list";
26192 208936 : return 1;
26193 :
26194 208936 : case 1:
26195 208936 : *ptree = sysv_va_list_type_node;
26196 208936 : *pname = "__builtin_sysv_va_list";
26197 208936 : return 1;
26198 : }
26199 : }
26200 :
26201 : return 0;
26202 : }
26203 :
26204 : #undef TARGET_SCHED_DISPATCH
26205 : #define TARGET_SCHED_DISPATCH ix86_bd_has_dispatch
26206 : #undef TARGET_SCHED_DISPATCH_DO
26207 : #define TARGET_SCHED_DISPATCH_DO ix86_bd_do_dispatch
26208 : #undef TARGET_SCHED_REASSOCIATION_WIDTH
26209 : #define TARGET_SCHED_REASSOCIATION_WIDTH ix86_reassociation_width
26210 : #undef TARGET_SCHED_REORDER
26211 : #define TARGET_SCHED_REORDER ix86_atom_sched_reorder
26212 : #undef TARGET_SCHED_ADJUST_PRIORITY
26213 : #define TARGET_SCHED_ADJUST_PRIORITY ix86_adjust_priority
26214 : #undef TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK
26215 : #define TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK \
26216 : ix86_dependencies_evaluation_hook
26217 :
26218 :
26219 : /* Implementation of reassociation_width target hook used by
26220 : reassoc phase to identify parallelism level in reassociated
26221 : tree. Statements tree_code is passed in OPC. Arguments type
26222 : is passed in MODE. */
26223 :
26224 : static int
26225 25994 : ix86_reassociation_width (tree_code op, machine_mode mode)
26226 : {
26227 25994 : int width = 1;
26228 : /* Vector part. */
26229 25994 : if (VECTOR_MODE_P (mode))
26230 : {
26231 8735 : int div = 1;
26232 8735 : if (INTEGRAL_MODE_P (mode))
26233 2814 : width = ix86_cost->reassoc_vec_int;
26234 5921 : else if (FLOAT_MODE_P (mode))
26235 5921 : width = ix86_cost->reassoc_vec_fp;
26236 :
26237 8735 : if (width == 1)
26238 : return 1;
26239 :
26240 : /* Znver1-4 Integer vector instructions execute in FP unit
26241 : and can execute 3 additions and one multiplication per cycle. */
26242 8730 : if ((ix86_tune == PROCESSOR_ZNVER1 || ix86_tune == PROCESSOR_ZNVER2
26243 8730 : || ix86_tune == PROCESSOR_ZNVER3 || ix86_tune == PROCESSOR_ZNVER4
26244 8730 : || ix86_tune == PROCESSOR_C86_4G_M4
26245 8730 : || ix86_tune == PROCESSOR_C86_4G_M6
26246 8730 : || ix86_tune == PROCESSOR_C86_4G_M7
26247 8728 : || ix86_tune == PROCESSOR_C86_4G_M8)
26248 2 : && INTEGRAL_MODE_P (mode) && op != PLUS_EXPR && op != MINUS_EXPR)
26249 : return 1;
26250 : /* Znver5 can do 2 integer multiplications per cycle with latency
26251 : of 3. */
26252 8730 : if ((ix86_tune == PROCESSOR_ZNVER5 || ix86_tune == PROCESSOR_ZNVER6)
26253 0 : && INTEGRAL_MODE_P (mode) && op != PLUS_EXPR && op != MINUS_EXPR)
26254 8730 : width = 6;
26255 :
26256 : /* Account for targets that splits wide vectors into multiple parts. */
26257 8732 : if (TARGET_AVX512_SPLIT_REGS && GET_MODE_BITSIZE (mode) > 256)
26258 0 : div = GET_MODE_BITSIZE (mode) / 256;
26259 8730 : else if (TARGET_AVX256_SPLIT_REGS && GET_MODE_BITSIZE (mode) > 128)
26260 0 : div = GET_MODE_BITSIZE (mode) / 128;
26261 8730 : else if (TARGET_SSE_SPLIT_REGS && GET_MODE_BITSIZE (mode) > 64)
26262 0 : div = GET_MODE_BITSIZE (mode) / 64;
26263 8730 : width = (width + div - 1) / div;
26264 8730 : }
26265 : /* Scalar part. */
26266 : else if (INTEGRAL_MODE_P (mode))
26267 11651 : width = ix86_cost->reassoc_int;
26268 : else if (FLOAT_MODE_P (mode))
26269 5608 : width = ix86_cost->reassoc_fp;
26270 :
26271 : /* Avoid using too many registers in 32bit mode. */
26272 25989 : if (!TARGET_64BIT && width > 2)
26273 25994 : width = 2;
26274 : return width;
26275 : }
26276 :
26277 : /* ??? No autovectorization into MMX or 3DNOW until we can reliably
26278 : place emms and femms instructions. */
26279 :
26280 : static machine_mode
26281 5502013 : ix86_preferred_simd_mode (scalar_mode mode)
26282 : {
26283 5502013 : if (!TARGET_SSE)
26284 873 : return word_mode;
26285 :
26286 5501140 : switch (mode)
26287 : {
26288 463525 : case E_QImode:
26289 463525 : if (TARGET_AVX512BW && !TARGET_PREFER_AVX256)
26290 : return V64QImode;
26291 450121 : else if (TARGET_AVX && !TARGET_PREFER_AVX128)
26292 : return V32QImode;
26293 : else
26294 428068 : return V16QImode;
26295 :
26296 223178 : case E_HImode:
26297 223178 : if (TARGET_AVX512BW && !TARGET_PREFER_AVX256)
26298 : return V32HImode;
26299 211927 : else if (TARGET_AVX && !TARGET_PREFER_AVX128)
26300 : return V16HImode;
26301 : else
26302 193105 : return V8HImode;
26303 :
26304 1641229 : case E_SImode:
26305 1641229 : if (TARGET_AVX512F && !TARGET_PREFER_AVX256)
26306 : return V16SImode;
26307 1571831 : else if (TARGET_AVX && !TARGET_PREFER_AVX128)
26308 : return V8SImode;
26309 : else
26310 1417470 : return V4SImode;
26311 :
26312 1936381 : case E_DImode:
26313 1936381 : if (TARGET_AVX512F && !TARGET_PREFER_AVX256)
26314 : return V8DImode;
26315 1521528 : else if (TARGET_AVX && !TARGET_PREFER_AVX128)
26316 : return V4DImode;
26317 : else
26318 1453744 : return V2DImode;
26319 :
26320 144983 : case E_HFmode:
26321 144983 : if (TARGET_AVX512FP16)
26322 : {
26323 142753 : if (TARGET_AVX512VL)
26324 : {
26325 69581 : if (TARGET_PREFER_AVX128)
26326 : return V8HFmode;
26327 69351 : else if (TARGET_PREFER_AVX256)
26328 : return V16HFmode;
26329 : }
26330 139971 : return V32HFmode;
26331 : }
26332 2230 : return word_mode;
26333 :
26334 63439 : case E_BFmode:
26335 63439 : if (TARGET_AVX512F && !TARGET_PREFER_AVX256)
26336 : return V32BFmode;
26337 26748 : else if (TARGET_AVX && !TARGET_PREFER_AVX128)
26338 : return V16BFmode;
26339 : else
26340 13617 : return V8BFmode;
26341 :
26342 669242 : case E_SFmode:
26343 669242 : if (TARGET_AVX512F && !TARGET_PREFER_AVX256)
26344 : return V16SFmode;
26345 464662 : else if (TARGET_AVX && !TARGET_PREFER_AVX128)
26346 : return V8SFmode;
26347 : else
26348 393143 : return V4SFmode;
26349 :
26350 319895 : case E_DFmode:
26351 319895 : if (TARGET_AVX512F && !TARGET_PREFER_AVX256)
26352 : return V8DFmode;
26353 198159 : else if (TARGET_AVX && !TARGET_PREFER_AVX128)
26354 : return V4DFmode;
26355 139589 : else if (TARGET_SSE2)
26356 : return V2DFmode;
26357 : /* FALLTHRU */
26358 :
26359 39330 : default:
26360 39330 : return word_mode;
26361 : }
26362 : }
26363 :
26364 : /* If AVX is enabled then try vectorizing with both 256bit and 128bit
26365 : vectors. If AVX512F is enabled then try vectorizing with 512bit,
26366 : 256bit and 128bit vectors. */
26367 :
26368 : static unsigned int
26369 2300391 : ix86_autovectorize_vector_modes (vector_modes *modes, bool all)
26370 : {
26371 2300391 : if (TARGET_AVX512F && !TARGET_PREFER_AVX256)
26372 : {
26373 76295 : modes->safe_push (V64QImode);
26374 76295 : modes->safe_push (V32QImode);
26375 76295 : modes->safe_push (V16QImode);
26376 : }
26377 2224096 : else if (TARGET_AVX512F && all)
26378 : {
26379 574 : modes->safe_push (V32QImode);
26380 574 : modes->safe_push (V16QImode);
26381 574 : modes->safe_push (V64QImode);
26382 : }
26383 2223522 : else if (TARGET_AVX && !TARGET_PREFER_AVX128)
26384 : {
26385 28741 : modes->safe_push (V32QImode);
26386 28741 : modes->safe_push (V16QImode);
26387 : }
26388 2194781 : else if (TARGET_AVX && all)
26389 : {
26390 24 : modes->safe_push (V16QImode);
26391 24 : modes->safe_push (V32QImode);
26392 : }
26393 2194757 : else if (TARGET_SSE2)
26394 2192395 : modes->safe_push (V16QImode);
26395 :
26396 2300391 : if (TARGET_MMX_WITH_SSE)
26397 1905908 : modes->safe_push (V8QImode);
26398 :
26399 2300391 : if (TARGET_SSE2)
26400 2298029 : modes->safe_push (V4QImode);
26401 :
26402 2300391 : return ix86_vect_compare_costs ? VECT_COMPARE_COSTS : 0;
26403 : }
26404 :
26405 : /* Implementation of targetm.vectorize.get_mask_mode. */
26406 :
26407 : static opt_machine_mode
26408 3583726 : ix86_get_mask_mode (machine_mode data_mode)
26409 : {
26410 3583726 : unsigned vector_size = GET_MODE_SIZE (data_mode);
26411 3583726 : unsigned nunits = GET_MODE_NUNITS (data_mode);
26412 3583726 : unsigned elem_size = vector_size / nunits;
26413 :
26414 : /* Scalar mask case. */
26415 483208 : if ((TARGET_AVX512F && vector_size == 64)
26416 3467557 : || (TARGET_AVX512VL && (vector_size == 32 || vector_size == 16))
26417 : /* AVX512FP16 only supports vector comparison
26418 : to kmask for _Float16. */
26419 3291471 : || (TARGET_AVX512VL && TARGET_AVX512FP16
26420 18413 : && GET_MODE_INNER (data_mode) == E_HFmode)
26421 6879928 : || (TARGET_AVX10_2 && GET_MODE_INNER (data_mode) == E_BFmode))
26422 : {
26423 295283 : if (elem_size == 4
26424 295283 : || elem_size == 8
26425 136103 : || (TARGET_AVX512BW && (elem_size == 1 || elem_size == 2)))
26426 264006 : return smallest_int_mode_for_size (nunits).require ();
26427 : }
26428 :
26429 3319720 : scalar_int_mode elem_mode
26430 3319720 : = smallest_int_mode_for_size (elem_size * BITS_PER_UNIT).require ();
26431 :
26432 3319720 : gcc_assert (elem_size * nunits == vector_size);
26433 :
26434 3319720 : return mode_for_vector (elem_mode, nunits);
26435 : }
26436 :
26437 :
26438 :
26439 : /* Return class of registers which could be used for pseudo of MODE
26440 : and of class RCLASS for spilling instead of memory. Return NO_REGS
26441 : if it is not possible or non-profitable. */
26442 :
26443 : /* Disabled due to PRs 70902, 71453, 71555, 71596 and 71657. */
26444 :
26445 : static reg_class_t
26446 6411135578 : ix86_spill_class (reg_class_t rclass, machine_mode mode)
26447 : {
26448 6411135578 : if (0 && TARGET_GENERAL_REGS_SSE_SPILL
26449 : && TARGET_SSE2
26450 : && TARGET_INTER_UNIT_MOVES_TO_VEC
26451 : && TARGET_INTER_UNIT_MOVES_FROM_VEC
26452 : && (mode == SImode || (TARGET_64BIT && mode == DImode))
26453 : && INTEGER_CLASS_P (rclass))
26454 : return ALL_SSE_REGS;
26455 6411135578 : return NO_REGS;
26456 : }
26457 :
26458 : /* Implement TARGET_MAX_NOCE_IFCVT_SEQ_COST. Like the default implementation,
26459 : but returns a lower bound. */
26460 :
26461 : static unsigned int
26462 1930139 : ix86_max_noce_ifcvt_seq_cost (edge e)
26463 : {
26464 1930139 : bool predictable_p = predictable_edge_p (e);
26465 1930139 : if (predictable_p)
26466 : {
26467 146340 : if (OPTION_SET_P (param_max_rtl_if_conversion_predictable_cost))
26468 8 : return param_max_rtl_if_conversion_predictable_cost;
26469 : }
26470 : else
26471 : {
26472 1783799 : if (OPTION_SET_P (param_max_rtl_if_conversion_unpredictable_cost))
26473 69 : return param_max_rtl_if_conversion_unpredictable_cost;
26474 : }
26475 :
26476 : /* For modern machines with deeper pipeline, the penalty for branch
26477 : misprediction could be higher than before to reset the pipeline
26478 : slots. Add parameter br_mispredict_scale as a factor to describe
26479 : the impact of resetting the pipeline. */
26480 :
26481 1930062 : return BRANCH_COST (true, predictable_p)
26482 1930062 : * ix86_tune_cost->br_mispredict_scale;
26483 : }
26484 :
26485 : /* Return true if SEQ is a good candidate as a replacement for the
26486 : if-convertible sequence described in IF_INFO. */
26487 :
26488 : static bool
26489 199062 : ix86_noce_conversion_profitable_p (rtx_insn *seq, struct noce_if_info *if_info)
26490 : {
26491 199062 : if (TARGET_ONE_IF_CONV_INSN && if_info->speed_p)
26492 : {
26493 : int cmov_cnt = 0;
26494 : /* Punt if SEQ contains more than one CMOV or FCMOV instruction.
26495 : Maybe we should allow even more conditional moves as long as they
26496 : are used far enough not to stall the CPU, or also consider
26497 : IF_INFO->TEST_BB succ edge probabilities. */
26498 411 : for (rtx_insn *insn = seq; insn; insn = NEXT_INSN (insn))
26499 : {
26500 342 : rtx set = single_set (insn);
26501 342 : if (!set)
26502 0 : continue;
26503 342 : if (GET_CODE (SET_SRC (set)) != IF_THEN_ELSE)
26504 299 : continue;
26505 43 : rtx src = SET_SRC (set);
26506 43 : machine_mode mode = GET_MODE (src);
26507 43 : if (GET_MODE_CLASS (mode) != MODE_INT
26508 0 : && GET_MODE_CLASS (mode) != MODE_FLOAT)
26509 0 : continue;
26510 43 : if ((!REG_P (XEXP (src, 1)) && !MEM_P (XEXP (src, 1)))
26511 42 : || (!REG_P (XEXP (src, 2)) && !MEM_P (XEXP (src, 2))))
26512 1 : continue;
26513 : /* insn is CMOV or FCMOV. */
26514 42 : if (++cmov_cnt > 1)
26515 : return false;
26516 : }
26517 : }
26518 :
26519 : /* W/o TARGET_SSE4_1, it takes 3 instructions (pand, pandn and por)
26520 : for movdfcc/movsfcc, and could possibly fail cost comparison.
26521 : Increase branch cost will hurt performance for other modes, so
26522 : specially add some preference for floating point ifcvt. */
26523 199053 : if (!TARGET_SSE4_1 && if_info->x
26524 143296 : && GET_MODE_CLASS (GET_MODE (if_info->x)) == MODE_FLOAT
26525 17958 : && if_info->speed_p)
26526 : {
26527 11002 : unsigned cost = seq_cost (seq, true);
26528 :
26529 11002 : if (cost <= if_info->original_cost)
26530 : return true;
26531 :
26532 9742 : return cost <= (if_info->max_seq_cost + COSTS_N_INSNS (2));
26533 : }
26534 :
26535 188051 : return default_noce_conversion_profitable_p (seq, if_info);
26536 : }
26537 :
26538 : /* x86-specific vector costs. */
26539 : class ix86_vector_costs : public vector_costs
26540 : {
26541 : public:
26542 : ix86_vector_costs (vec_info *, bool);
26543 :
26544 : unsigned int add_stmt_cost (int count, vect_cost_for_stmt kind,
26545 : stmt_vec_info stmt_info, slp_tree node,
26546 : tree vectype, int misalign,
26547 : vect_cost_model_location where) override;
26548 : unsigned int add_slp_cost (slp_tree, const array_slice<stmt_info_for_cost> &);
26549 : void finish_cost (const vector_costs *) override;
26550 : bool better_main_loop_than_p (const vector_costs *) const override;
26551 : bool better_epilogue_loop_than_p (const vector_costs *other,
26552 : loop_vec_info main_loop) const override;
26553 :
26554 : private:
26555 :
26556 : bool better_fold_left_reduc_than_p (const vector_costs *) const;
26557 : /* Estimate register pressure of the vectorized code. */
26558 : void ix86_vect_estimate_reg_pressure ();
26559 : /* Number of GENERAL_REGS/SSE_REGS used in the vectorizer, it's used for
26560 : estimation of register pressure.
26561 : ??? Currently it's only used by vec_construct/scalar_to_vec
26562 : where we know it's not loaded from memory. */
26563 : unsigned m_num_gpr_needed[3];
26564 : unsigned m_num_sse_needed[3];
26565 : /* Number of 256-bit vector permutation. */
26566 : unsigned m_num_avx256_vec_perm[3];
26567 : /* Number of 512-bit vector permutation. */
26568 : unsigned m_num_avx512_vec_perm[3];
26569 : /* Number of reductions for FMA/DOT_PROD_EXPR/SAD_EXPR */
26570 : unsigned m_num_reduc[X86_REDUC_LAST];
26571 : /* Don't do unroll if m_prefer_unroll is false, default is true. */
26572 : bool m_prefer_unroll;
26573 : /* Scalar lanes in fold-left reductions. */
26574 : unsigned int m_num_fold_left_reduc_lanes;
26575 : };
26576 :
26577 2269476 : ix86_vector_costs::ix86_vector_costs (vec_info* vinfo, bool costing_for_scalar)
26578 : : vector_costs (vinfo, costing_for_scalar),
26579 2269476 : m_num_gpr_needed (),
26580 2269476 : m_num_sse_needed (),
26581 2269476 : m_num_avx256_vec_perm (),
26582 2269476 : m_num_avx512_vec_perm (),
26583 2269476 : m_num_reduc (),
26584 2269476 : m_prefer_unroll (true),
26585 2269476 : m_num_fold_left_reduc_lanes (0)
26586 2269476 : {}
26587 :
26588 : /* Implement targetm.vectorize.create_costs. */
26589 :
26590 : static vector_costs *
26591 2269476 : ix86_vectorize_create_costs (vec_info *vinfo, bool costing_for_scalar)
26592 : {
26593 2269476 : return new ix86_vector_costs (vinfo, costing_for_scalar);
26594 : }
26595 :
26596 : /* Return true if a vec_perm should be counted as a cross-lane vector
26597 : permutation for a vector with NUNITS elements. */
26598 : static bool
26599 5356 : ix86_count_cross_lane_perm_p (vec_info *vinfo, slp_tree node, unsigned nunits)
26600 : {
26601 : /* TODO: For loop vectorization with no SLP load-permutation
26602 : information, conservatively treat these perms as cross-lane.
26603 : Repeated-index cases such as {0, 0, 0, 0} are emitted as
26604 : separate vec_perm_exprs for each index, so we cannot reliably
26605 : separate false positives from real cross-lane shuffles yet. */
26606 5356 : if (!node
26607 5329 : || !SLP_TREE_LOAD_PERMUTATION (node).exists ()
26608 9636 : || !is_a<bb_vec_info> (vinfo))
26609 : return true;
26610 :
26611 41 : unsigned half = nunits / 2;
26612 41 : bool allsame = true;
26613 41 : unsigned first = SLP_TREE_LOAD_PERMUTATION (node)[0];
26614 41 : bool cross_lane_p = false;
26615 :
26616 217 : for (unsigned i = 0; i != SLP_TREE_LANES (node); i++)
26617 : {
26618 215 : unsigned tmp = SLP_TREE_LOAD_PERMUTATION (node)[i];
26619 : /* allsame is just a broadcast. */
26620 215 : if (tmp != first)
26621 106 : allsame = false;
26622 :
26623 : /* The load permutation can cover multiple vectors, so compare
26624 : source and destination lanes modulo NUNITS. */
26625 215 : tmp = tmp & (nunits - 1);
26626 215 : unsigned index = i & (nunits - 1);
26627 215 : if ((index < half && tmp >= half) || (index >= half && tmp < half))
26628 67 : cross_lane_p = true;
26629 :
26630 215 : if (!allsame && cross_lane_p)
26631 : return true;
26632 : }
26633 :
26634 : return false;
26635 : }
26636 :
26637 : unsigned
26638 8023840 : ix86_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
26639 : stmt_vec_info stmt_info, slp_tree node,
26640 : tree vectype, int,
26641 : vect_cost_model_location where)
26642 : {
26643 8023840 : unsigned retval = 0;
26644 8023840 : bool scalar_p
26645 : = (kind == scalar_stmt || kind == scalar_load || kind == scalar_store);
26646 8023840 : int stmt_cost = - 1;
26647 :
26648 8023840 : bool fp = false;
26649 8023840 : machine_mode mode = scalar_p ? SImode : TImode;
26650 :
26651 8023840 : if (vectype != NULL)
26652 : {
26653 3694527 : fp = FLOAT_TYPE_P (vectype);
26654 3694527 : mode = TYPE_MODE (vectype);
26655 3694527 : if (scalar_p)
26656 287154 : mode = TYPE_MODE (TREE_TYPE (vectype));
26657 : }
26658 : /* When we are costing a scalar stmt use the scalar stmt to get at the
26659 : type of the operation. */
26660 4329313 : else if (scalar_p && stmt_info)
26661 4245114 : if (tree lhs = gimple_get_lhs (stmt_info->stmt))
26662 : {
26663 4059036 : fp = FLOAT_TYPE_P (TREE_TYPE (lhs));
26664 4059036 : mode = TYPE_MODE (TREE_TYPE (lhs));
26665 : }
26666 :
26667 8023840 : if ((kind == vector_stmt || kind == scalar_stmt)
26668 2166211 : && stmt_info
26669 10180723 : && stmt_info->stmt && gimple_code (stmt_info->stmt) == GIMPLE_ASSIGN)
26670 : {
26671 1724639 : tree_code subcode = gimple_assign_rhs_code (stmt_info->stmt);
26672 : /*machine_mode inner_mode = mode;
26673 : if (VECTOR_MODE_P (mode))
26674 : inner_mode = GET_MODE_INNER (mode);*/
26675 :
26676 1724639 : switch (subcode)
26677 : {
26678 688301 : case PLUS_EXPR:
26679 688301 : case POINTER_PLUS_EXPR:
26680 688301 : case MINUS_EXPR:
26681 688301 : if (kind == scalar_stmt)
26682 : {
26683 430163 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
26684 99806 : stmt_cost = ix86_cost->addss;
26685 330357 : else if (X87_FLOAT_MODE_P (mode))
26686 155 : stmt_cost = ix86_cost->fadd;
26687 : else
26688 330202 : stmt_cost = ix86_cost->add;
26689 : }
26690 : else
26691 258138 : stmt_cost = ix86_vec_cost (mode, fp ? ix86_cost->addss
26692 : : ix86_cost->sse_op);
26693 : break;
26694 :
26695 267842 : case MULT_EXPR:
26696 : /* For MULT_HIGHPART_EXPR, x86 only supports pmulhw,
26697 : take it as MULT_EXPR. */
26698 267842 : case MULT_HIGHPART_EXPR:
26699 267842 : stmt_cost = ix86_multiplication_cost (ix86_cost, mode);
26700 267842 : break;
26701 : /* There's no direct instruction for WIDEN_MULT_EXPR,
26702 : take emulation into account. */
26703 1217 : case WIDEN_MULT_EXPR:
26704 2434 : stmt_cost = ix86_widen_mult_cost (ix86_cost, mode,
26705 1217 : TYPE_UNSIGNED (vectype));
26706 1217 : break;
26707 :
26708 11370 : case NEGATE_EXPR:
26709 11370 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
26710 3563 : stmt_cost = ix86_cost->sse_op;
26711 7807 : else if (X87_FLOAT_MODE_P (mode))
26712 0 : stmt_cost = ix86_cost->fchs;
26713 7807 : else if (VECTOR_MODE_P (mode))
26714 3866 : stmt_cost = ix86_vec_cost (mode, ix86_cost->sse_op);
26715 : else
26716 3941 : stmt_cost = ix86_cost->add;
26717 : break;
26718 15432 : case TRUNC_DIV_EXPR:
26719 15432 : case CEIL_DIV_EXPR:
26720 15432 : case FLOOR_DIV_EXPR:
26721 15432 : case ROUND_DIV_EXPR:
26722 15432 : case TRUNC_MOD_EXPR:
26723 15432 : case CEIL_MOD_EXPR:
26724 15432 : case FLOOR_MOD_EXPR:
26725 15432 : case RDIV_EXPR:
26726 15432 : case ROUND_MOD_EXPR:
26727 15432 : case EXACT_DIV_EXPR:
26728 15432 : stmt_cost = ix86_division_cost (ix86_cost, mode);
26729 15432 : break;
26730 :
26731 84021 : case RSHIFT_EXPR:
26732 84021 : case LSHIFT_EXPR:
26733 84021 : case LROTATE_EXPR:
26734 84021 : case RROTATE_EXPR:
26735 84021 : {
26736 84021 : tree op1 = gimple_assign_rhs1 (stmt_info->stmt);
26737 84021 : tree op2 = gimple_assign_rhs2 (stmt_info->stmt);
26738 84021 : stmt_cost = ix86_shift_rotate_cost
26739 84021 : (ix86_cost,
26740 : (subcode == RSHIFT_EXPR
26741 42753 : && !TYPE_UNSIGNED (TREE_TYPE (op1)))
26742 : ? ASHIFTRT : LSHIFTRT, mode,
26743 84021 : TREE_CODE (op2) == INTEGER_CST,
26744 84021 : cst_and_fits_in_hwi (op2)
26745 52565 : ? int_cst_value (op2) : -1,
26746 : false, false, NULL, NULL);
26747 : }
26748 84021 : break;
26749 105722 : case NOP_EXPR:
26750 : /* Only sign-conversions are free. */
26751 105722 : if (tree_nop_conversion_p
26752 105722 : (TREE_TYPE (gimple_assign_lhs (stmt_info->stmt)),
26753 105722 : TREE_TYPE (gimple_assign_rhs1 (stmt_info->stmt))))
26754 : stmt_cost = 0;
26755 105722 : else if (fp)
26756 10479 : stmt_cost = vec_fp_conversion_cost
26757 10479 : (ix86_tune_cost, GET_MODE_BITSIZE (mode));
26758 : break;
26759 :
26760 23657 : case FLOAT_EXPR:
26761 23657 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
26762 17724 : stmt_cost = ix86_cost->cvtsi2ss;
26763 5933 : else if (X87_FLOAT_MODE_P (mode))
26764 : /* TODO: We do not have cost tables for x87. */
26765 50 : stmt_cost = ix86_cost->fadd;
26766 : else
26767 5883 : stmt_cost = ix86_vec_cost (mode, ix86_cost->cvtpi2ps);
26768 : break;
26769 :
26770 2459 : case FIX_TRUNC_EXPR:
26771 2459 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
26772 0 : stmt_cost = ix86_cost->cvtss2si;
26773 2459 : else if (X87_FLOAT_MODE_P (mode))
26774 : /* TODO: We do not have cost tables for x87. */
26775 0 : stmt_cost = ix86_cost->fadd;
26776 : else
26777 2459 : stmt_cost = ix86_vec_cost (mode, ix86_cost->cvtps2pi);
26778 : break;
26779 :
26780 56788 : case COND_EXPR:
26781 56788 : {
26782 : /* SSE2 conditinal move sequence is:
26783 : pcmpgtd %xmm5, %xmm0 (accounted separately)
26784 : pand %xmm0, %xmm2
26785 : pandn %xmm1, %xmm0
26786 : por %xmm2, %xmm0
26787 : while SSE4 uses cmp + blend
26788 : and AVX512 masked moves.
26789 :
26790 : The condition is accounted separately since we usually have
26791 : p = a < b
26792 : c = p ? x : y
26793 : and we will account first statement as setcc. Exception is when
26794 : p is loaded from memory as bool and then we will not account
26795 : the compare, but there is no way to check for this. */
26796 :
26797 56788 : int ninsns = TARGET_SSE4_1 ? 1 : 3;
26798 :
26799 : /* If one of parameters is 0 or -1 the sequence will be simplified:
26800 : (if_true & mask) | (if_false & ~mask) -> if_true & mask */
26801 24070 : if (ninsns > 1
26802 24070 : && (zerop (gimple_assign_rhs2 (stmt_info->stmt))
26803 23734 : || zerop (gimple_assign_rhs3 (stmt_info->stmt))
26804 13577 : || integer_minus_onep
26805 13577 : (gimple_assign_rhs2 (stmt_info->stmt))
26806 13290 : || integer_minus_onep
26807 13290 : (gimple_assign_rhs3 (stmt_info->stmt))))
26808 : ninsns = 1;
26809 :
26810 56788 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
26811 5072 : stmt_cost = ninsns * ix86_cost->sse_op;
26812 51716 : else if (X87_FLOAT_MODE_P (mode))
26813 : /* x87 requires conditional branch. We don't have cost for
26814 : that. */
26815 : ;
26816 51707 : else if (VECTOR_MODE_P (mode))
26817 21728 : stmt_cost = ix86_vec_cost (mode, ninsns * ix86_cost->sse_op);
26818 : else
26819 : /* compare (accounted separately) + cmov. */
26820 29979 : stmt_cost = ix86_cost->add;
26821 : }
26822 : break;
26823 :
26824 37436 : case MIN_EXPR:
26825 37436 : case MAX_EXPR:
26826 37436 : if (fp)
26827 : {
26828 1507 : if (X87_FLOAT_MODE_P (mode)
26829 533 : && !SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
26830 : /* x87 requires conditional branch. We don't have cost for
26831 : that. */
26832 : ;
26833 : else
26834 : /* minss */
26835 1507 : stmt_cost = ix86_vec_cost (mode, ix86_cost->sse_op);
26836 : }
26837 : else
26838 : {
26839 35929 : if (VECTOR_MODE_P (mode))
26840 : {
26841 10453 : stmt_cost = ix86_vec_cost (mode, ix86_cost->sse_op);
26842 : /* vpmin was introduced in SSE3.
26843 : SSE2 needs pcmpgtd + pand + pandn + pxor.
26844 : If one of parameters is 0 or -1 the sequence is simplified
26845 : to pcmpgtd + pand. */
26846 10453 : if (!TARGET_SSSE3)
26847 : {
26848 8597 : if (zerop (gimple_assign_rhs2 (stmt_info->stmt))
26849 15420 : || integer_minus_onep
26850 6823 : (gimple_assign_rhs2 (stmt_info->stmt)))
26851 1774 : stmt_cost *= 2;
26852 : else
26853 6823 : stmt_cost *= 4;
26854 : }
26855 : }
26856 : else
26857 : /* cmp + cmov. */
26858 25476 : stmt_cost = ix86_cost->add * 2;
26859 : }
26860 : break;
26861 :
26862 1839 : case ABS_EXPR:
26863 1839 : case ABSU_EXPR:
26864 1839 : if (fp)
26865 : {
26866 1116 : if (X87_FLOAT_MODE_P (mode)
26867 601 : && !SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
26868 : /* fabs. */
26869 0 : stmt_cost = ix86_cost->fabs;
26870 : else
26871 : /* andss of sign bit. */
26872 1116 : stmt_cost = ix86_vec_cost (mode, ix86_cost->sse_op);
26873 : }
26874 : else
26875 : {
26876 723 : if (VECTOR_MODE_P (mode))
26877 : {
26878 116 : stmt_cost = ix86_vec_cost (mode, ix86_cost->sse_op);
26879 : /* vabs was introduced in SSE3.
26880 : SSE3 uses psrat + pxor + psub. */
26881 116 : if (!TARGET_SSSE3)
26882 86 : stmt_cost *= 3;
26883 : }
26884 : else
26885 : /* neg + cmov. */
26886 607 : stmt_cost = ix86_cost->add * 2;
26887 : }
26888 : break;
26889 :
26890 169093 : case BIT_IOR_EXPR:
26891 169093 : case BIT_XOR_EXPR:
26892 169093 : case BIT_AND_EXPR:
26893 169093 : case BIT_NOT_EXPR:
26894 169093 : gcc_assert (!SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode)
26895 : && !X87_FLOAT_MODE_P (mode));
26896 169093 : if (VECTOR_MODE_P (mode))
26897 59140 : stmt_cost = ix86_vec_cost (mode, ix86_cost->sse_op);
26898 : else
26899 109953 : stmt_cost = ix86_cost->add;
26900 : break;
26901 :
26902 259462 : default:
26903 259462 : if (TREE_CODE_CLASS (subcode) == tcc_comparison)
26904 : {
26905 100637 : tree op_type;
26906 100637 : if (kind == vector_stmt)
26907 32841 : op_type = SLP_TREE_VECTYPE (SLP_TREE_CHILDREN (node)[0]);
26908 : else
26909 67796 : op_type = vect_comparison_type (stmt_info);
26910 100637 : mode = TYPE_MODE (op_type);
26911 :
26912 100637 : if (SSE_FLOAT_MODE_SSEMATH_OR_HFBF_P (mode))
26913 : /* CMPccS? insructions are cheap, so use sse_op. While they
26914 : produce a mask which may need to be turned to 0/1 by and,
26915 : expect that this will be optimized away in a common case. */
26916 8494 : stmt_cost = ix86_cost->sse_op;
26917 92143 : else if (X87_FLOAT_MODE_P (mode))
26918 : /* fcmp + setcc. */
26919 32 : stmt_cost = ix86_cost->fadd + ix86_cost->add;
26920 92111 : else if (VECTOR_MODE_P (mode))
26921 32784 : stmt_cost = ix86_vec_cost (mode, ix86_cost->sse_op);
26922 : else
26923 : /* setcc. */
26924 59327 : stmt_cost = ix86_cost->add;
26925 : break;
26926 : }
26927 : break;
26928 : }
26929 : }
26930 :
26931 : /* Record number of load/store/gather/scatter in vectorized body. */
26932 8023840 : if (where == vect_body && !m_costing_for_scalar)
26933 : {
26934 2213610 : int scale = 1;
26935 2213610 : if (vectype
26936 4413269 : && ((GET_MODE_SIZE (TYPE_MODE (vectype)) == 64
26937 60076 : && TARGET_AVX512_SPLIT_REGS)
26938 4399016 : || (GET_MODE_SIZE (TYPE_MODE (vectype)) == 32
26939 119168 : && TARGET_AVX256_SPLIT_REGS)))
26940 : scale = 2;
26941 :
26942 2213610 : switch (kind)
26943 : {
26944 : /* Emulated gather/scatter or any scalarization. */
26945 118315 : case scalar_load:
26946 118315 : case scalar_stmt:
26947 118315 : case scalar_store:
26948 118315 : case vector_gather_load:
26949 118315 : case vector_scatter_store:
26950 118315 : m_prefer_unroll = false;
26951 118315 : break;
26952 :
26953 765018 : case vector_stmt:
26954 765018 : case vec_to_scalar:
26955 : /* Count number of reduction FMA and "real" DOT_PROD_EXPR,
26956 : unroll in the vectorizer will enable partial sum. */
26957 765018 : if (stmt_info
26958 763056 : && vect_is_reduction (stmt_info)
26959 834765 : && stmt_info->stmt)
26960 : {
26961 : /* Handle __builtin_fma. */
26962 69747 : if (gimple_call_combined_fn (stmt_info->stmt) == CFN_FMA)
26963 : {
26964 11 : m_num_reduc[X86_REDUC_FMA] += count * scale;
26965 11 : break;
26966 : }
26967 :
26968 69736 : if (!is_gimple_assign (stmt_info->stmt))
26969 : break;
26970 :
26971 66970 : tree_code subcode = gimple_assign_rhs_code (stmt_info->stmt);
26972 66970 : machine_mode inner_mode = GET_MODE_INNER (mode);
26973 66970 : tree rhs1, rhs2;
26974 66970 : bool native_vnni_p = true;
26975 66970 : gimple* def;
26976 66970 : machine_mode mode_rhs;
26977 66970 : switch (subcode)
26978 : {
26979 50091 : case PLUS_EXPR:
26980 50091 : case MINUS_EXPR:
26981 50091 : if (!fp || !flag_associative_math
26982 26428 : || flag_fp_contract_mode != FP_CONTRACT_FAST)
26983 : break;
26984 :
26985 : /* FMA condition for different modes. */
26986 26428 : if (((inner_mode == DFmode || inner_mode == SFmode)
26987 26398 : && !TARGET_FMA && !TARGET_AVX512VL)
26988 8622 : || (inner_mode == HFmode && !TARGET_AVX512FP16)
26989 8622 : || (inner_mode == BFmode && !TARGET_AVX10_2))
26990 : break;
26991 :
26992 : /* MULT_EXPR + PLUS_EXPR/MINUS_EXPR is transformed
26993 : to FMA/FNMA after vectorization. */
26994 8622 : rhs1 = gimple_assign_rhs1 (stmt_info->stmt);
26995 8622 : rhs2 = gimple_assign_rhs2 (stmt_info->stmt);
26996 8622 : if (subcode == PLUS_EXPR
26997 6769 : && TREE_CODE (rhs1) == SSA_NAME
26998 6769 : && (def = SSA_NAME_DEF_STMT (rhs1), true)
26999 6769 : && is_gimple_assign (def)
27000 12025 : && gimple_assign_rhs_code (def) == MULT_EXPR)
27001 1992 : m_num_reduc[X86_REDUC_FMA] += count * scale;
27002 6630 : else if (TREE_CODE (rhs2) == SSA_NAME
27003 6630 : && (def = SSA_NAME_DEF_STMT (rhs2), true)
27004 6630 : && is_gimple_assign (def)
27005 13173 : && gimple_assign_rhs_code (def) == MULT_EXPR)
27006 6537 : m_num_reduc[X86_REDUC_FMA] += count * scale;
27007 : break;
27008 :
27009 : /* Vectorizer lane_reducing_op_p supports DOT_PROX_EXPR,
27010 : WIDEN_SUM_EXPR and SAD_EXPR, x86 backend only supports
27011 : SAD_EXPR (usad{v16qi,v32qi,v64qi}) and DOT_PROD_EXPR. */
27012 628 : case DOT_PROD_EXPR:
27013 628 : rhs1 = gimple_assign_rhs1 (stmt_info->stmt);
27014 628 : mode_rhs = TYPE_MODE (TREE_TYPE (rhs1));
27015 628 : if (mode_rhs == QImode)
27016 : {
27017 355 : rhs2 = gimple_assign_rhs2 (stmt_info->stmt);
27018 355 : signop signop1_p = TYPE_SIGN (TREE_TYPE (rhs1));
27019 355 : signop signop2_p = TYPE_SIGN (TREE_TYPE (rhs2));
27020 :
27021 : /* vpdpbusd. */
27022 355 : if (signop1_p != signop2_p)
27023 88 : native_vnni_p
27024 88 : = (GET_MODE_SIZE (mode) == 64
27025 88 : ? TARGET_AVX512VNNI
27026 28 : : ((TARGET_AVX512VNNI && TARGET_AVX512VL)
27027 88 : || TARGET_AVXVNNI));
27028 : else
27029 : /* vpdpbssd. */
27030 267 : native_vnni_p
27031 283 : = (GET_MODE_SIZE (mode) == 64
27032 267 : ? TARGET_AVX10_2
27033 251 : : (TARGET_AVXVNNIINT8 || TARGET_AVX10_2));
27034 : }
27035 628 : m_num_reduc[X86_REDUC_DOT_PROD] += count * scale;
27036 :
27037 : /* Dislike to do unroll and partial sum for
27038 : emulated DOT_PROD_EXPR. */
27039 628 : if (!native_vnni_p)
27040 171 : m_num_reduc[X86_REDUC_DOT_PROD] += 3 * count;
27041 : break;
27042 :
27043 112 : case SAD_EXPR:
27044 112 : m_num_reduc[X86_REDUC_SAD] += count * scale;
27045 112 : break;
27046 :
27047 : default:
27048 : break;
27049 : }
27050 : }
27051 :
27052 : default:
27053 : break;
27054 : }
27055 : }
27056 :
27057 :
27058 8023840 : combined_fn cfn;
27059 8023840 : if ((kind == vector_stmt || kind == scalar_stmt)
27060 2166211 : && stmt_info
27061 2156883 : && stmt_info->stmt
27062 10180723 : && is_gimple_call (stmt_info->stmt))
27063 : {
27064 26928 : tree fndecl = gimple_call_fndecl (stmt_info->stmt);
27065 26928 : cgraph_node *node;
27066 26928 : if ((fndecl
27067 5595 : && (node = cgraph_node::get (fndecl))
27068 5566 : && node->simd_clones)
27069 31518 : || gimple_call_internal_p (stmt_info->stmt, IFN_MASK_CALL))
27070 2460 : stmt_cost = 10 * ix86_vec_cost (mode,
27071 1230 : mode == SFmode ? ix86_cost->fmass
27072 : : ix86_cost->fmasd);
27073 25698 : else if ((cfn = gimple_call_combined_fn (stmt_info->stmt)) != CFN_LAST)
27074 24318 : switch (cfn)
27075 : {
27076 107 : case CFN_FMA:
27077 107 : stmt_cost = ix86_vec_cost (mode,
27078 107 : mode == SFmode ? ix86_cost->fmass
27079 : : ix86_cost->fmasd);
27080 107 : break;
27081 62 : case CFN_MULH:
27082 62 : stmt_cost = ix86_multiplication_cost (ix86_cost, mode);
27083 62 : break;
27084 : default:
27085 : break;
27086 : }
27087 : }
27088 :
27089 8023840 : if (kind == vec_promote_demote)
27090 : {
27091 62191 : int outer_size
27092 : = tree_to_uhwi
27093 62191 : (TYPE_SIZE
27094 62191 : (TREE_TYPE (gimple_assign_lhs (stmt_info->stmt))));
27095 62191 : int inner_size
27096 : = tree_to_uhwi
27097 62191 : (TYPE_SIZE
27098 62191 : (TREE_TYPE (gimple_assign_rhs1 (stmt_info->stmt))));
27099 62191 : bool inner_fp = FLOAT_TYPE_P
27100 : (TREE_TYPE (gimple_assign_rhs1 (stmt_info->stmt)));
27101 :
27102 5638 : if (fp && inner_fp)
27103 5128 : stmt_cost = vec_fp_conversion_cost
27104 5128 : (ix86_tune_cost, GET_MODE_BITSIZE (mode));
27105 57063 : else if (fp && !inner_fp)
27106 6151 : stmt_cost = ix86_vec_cost (mode, ix86_cost->cvtpi2ps);
27107 50912 : else if (!fp && inner_fp)
27108 510 : stmt_cost = ix86_vec_cost (mode, ix86_cost->cvtps2pi);
27109 : else
27110 50402 : stmt_cost = ix86_vec_cost (mode, ix86_cost->sse_op);
27111 : /* VEC_PACK_TRUNC_EXPR and similar demote operations: If outer size is
27112 : greater than inner size we will end up doing two conversions and
27113 : packing them. We always pack pairs; if the size difference is greater
27114 : it is split into multiple demote operations. */
27115 62191 : if (inner_size > outer_size)
27116 23813 : stmt_cost = stmt_cost * 2
27117 23813 : + ix86_vec_cost (mode, ix86_cost->sse_op);
27118 : }
27119 :
27120 : /* If we do elementwise loads into a vector then we are bound by
27121 : latency and execution resources for the many scalar loads
27122 : (AGU and load ports). Try to account for this by scaling the
27123 : construction cost by the number of elements involved. */
27124 8023840 : if ((kind == vec_construct || kind == vec_deconstruct)
27125 8023840 : && ((node
27126 368110 : && (((SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_ELEMENTWISE
27127 379446 : || SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_STRIDED_SLP)
27128 43062 : && (TREE_CODE (DR_STEP (STMT_VINFO_DATA_REF
27129 : (SLP_TREE_REPRESENTATIVE (node))))
27130 : != INTEGER_CST))
27131 23139 : || mat_gather_scatter_p (SLP_TREE_MEMORY_ACCESS_TYPE (node))))))
27132 : {
27133 33070 : auto lsdata = static_cast<vect_load_store_data *> (node->data);
27134 33070 : tree ls_type = lsdata->ls_type ? lsdata->ls_type : vectype;
27135 33070 : tree ls_eltype
27136 33070 : = lsdata->ls_eltype ? lsdata->ls_eltype : TREE_TYPE (ls_type);
27137 33070 : stmt_cost = ix86_vector_cd_cost (TYPE_MODE (ls_type),
27138 33070 : TYPE_MODE (ls_eltype));
27139 33070 : stmt_cost *= (GET_MODE_BITSIZE (TYPE_MODE (ls_type))
27140 66140 : / GET_MODE_BITSIZE (TYPE_MODE (ls_eltype)) + 1);
27141 : }
27142 8023840 : if (stmt_cost == -1)
27143 6456618 : stmt_cost = ix86_default_vector_cost (kind, mode);
27144 :
27145 : /* BIT_FIELD_REF <vect_**, 64, 0> with count 0 costs 0 in body. */
27146 8023840 : if (kind == vec_perm && vectype && count != 0)
27147 : {
27148 125253 : unsigned vec_size = GET_MODE_SIZE (TYPE_MODE (vectype));
27149 125253 : unsigned nunits = TYPE_VECTOR_SUBPARTS (vectype);
27150 125253 : unsigned *num_vec_perm = NULL;
27151 :
27152 125253 : if (vec_size == 32)
27153 4157 : num_vec_perm = m_num_avx256_vec_perm;
27154 121096 : else if (vec_size == 64)
27155 1199 : num_vec_perm = m_num_avx512_vec_perm;
27156 :
27157 5356 : if (num_vec_perm && ix86_count_cross_lane_perm_p (m_vinfo, node, nunits))
27158 : {
27159 5354 : num_vec_perm[where] += count;
27160 5354 : if (dump_file && (dump_flags & TDF_DETAILS))
27161 : {
27162 349 : fprintf (dump_file,
27163 : "Detected avx%u cross-lane permutation: ", vec_size * 8);
27164 349 : if (stmt_info)
27165 126 : print_gimple_expr (dump_file, stmt_info->stmt, 0, TDF_SLIM);
27166 349 : fprintf (dump_file, " \n");
27167 : }
27168 : }
27169 : }
27170 :
27171 : /* Penalize DFmode vector operations for Bonnell. */
27172 8023840 : if (TARGET_CPU_P (BONNELL) && kind == vector_stmt
27173 8023925 : && vectype && GET_MODE_INNER (TYPE_MODE (vectype)) == DFmode)
27174 12 : stmt_cost *= 5; /* FIXME: The value here is arbitrary. */
27175 :
27176 : /* Statements in an inner loop relative to the loop being
27177 : vectorized are weighted more heavily. The value here is
27178 : arbitrary and could potentially be improved with analysis. */
27179 8023840 : retval = adjust_cost_for_freq (stmt_info, where, count * stmt_cost);
27180 :
27181 : /* We need to multiply all vector stmt cost by 1.7 (estimated cost)
27182 : for Silvermont as it has out of order integer pipeline and can execute
27183 : 2 scalar instruction per tick, but has in order SIMD pipeline. */
27184 8023840 : if ((TARGET_CPU_P (SILVERMONT) || TARGET_CPU_P (GOLDMONT)
27185 8023840 : || TARGET_CPU_P (GOLDMONT_PLUS) || TARGET_CPU_P (INTEL))
27186 2551 : && stmt_info && stmt_info->stmt)
27187 : {
27188 2144 : tree lhs_op = gimple_get_lhs (stmt_info->stmt);
27189 2144 : if (lhs_op && TREE_CODE (TREE_TYPE (lhs_op)) == INTEGER_TYPE)
27190 1612 : retval = (retval * 17) / 10;
27191 : }
27192 :
27193 8023840 : m_costs[where] += retval;
27194 :
27195 8023840 : return retval;
27196 : }
27197 :
27198 : unsigned
27199 2771642 : ix86_vector_costs::add_slp_cost (slp_tree node,
27200 : const array_slice<stmt_info_for_cost> &parts)
27201 : {
27202 2771642 : int stmt_cost = 0;
27203 :
27204 : /* For vector construction account for the cost of moving data between
27205 : GRP and XMM. As we are looking at the SLP nodes elements, avoid
27206 : duplicate costs by doing this in add_slp_cost, leaving the actual
27207 : splat/ctor cost to add_stmt_cost. */
27208 2771642 : if (SLP_TREE_DEF_TYPE (node) == vect_external_def)
27209 : {
27210 : unsigned i;
27211 : tree op;
27212 1118894 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
27213 775476 : if (TREE_CODE (op) == SSA_NAME)
27214 546021 : TREE_VISITED (op) = 0;
27215 1118894 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
27216 : {
27217 775476 : if (TREE_CODE (op) != SSA_NAME
27218 546021 : || TREE_VISITED (op))
27219 268567 : continue;
27220 506909 : TREE_VISITED (op) = 1;
27221 506909 : gimple *def = SSA_NAME_DEF_STMT (op);
27222 506909 : tree tem;
27223 : /* Look through a conversion. */
27224 506909 : if (is_gimple_assign (def)
27225 282869 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def))
27226 32187 : && ((tem = gimple_assign_rhs1 (def)), true)
27227 539096 : && TREE_CODE (tem) == SSA_NAME)
27228 31974 : def = SSA_NAME_DEF_STMT (tem);
27229 : /* When the component is loaded from memory without sign-
27230 : or zero-extension we can move it to a vector register and/or
27231 : insert it via vpinsr with a memory operand. */
27232 506909 : if (gimple_assign_load_p (def)
27233 145831 : && tree_nop_conversion_p (TREE_TYPE (op),
27234 145831 : TREE_TYPE (gimple_assign_lhs (def)))
27235 788087 : && (GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (op))) > 1
27236 6393 : || TARGET_SSE4_1))
27237 : ;
27238 : /* When the component is extracted from a vector it is already
27239 : in a vector register. */
27240 371616 : else if (is_gimple_assign (def)
27241 142132 : && gimple_assign_rhs_code (def) == BIT_FIELD_REF
27242 372990 : && VECTOR_TYPE_P (TREE_TYPE
27243 : (TREE_OPERAND (gimple_assign_rhs1 (def), 0))))
27244 : ;
27245 : else
27246 : {
27247 370658 : if (FLOAT_TYPE_P (TREE_TYPE (op)))
27248 : {
27249 : /* Scalar FP values residing in x87 registers need to be
27250 : spilled and reloaded. */
27251 17144 : auto mode2 = TYPE_MODE (TREE_TYPE (op));
27252 17144 : if (IS_STACK_MODE (mode2))
27253 : {
27254 1027 : int cost
27255 : = (ix86_cost->hard_register.fp_store[mode2 == SFmode
27256 1027 : ? 0 : 1]
27257 1027 : + ix86_cost->sse_load[sse_store_index (mode2)]);
27258 1027 : stmt_cost += COSTS_N_INSNS (cost) / 2;
27259 : }
27260 17144 : m_num_sse_needed[vect_prologue]++;
27261 : }
27262 : else
27263 : {
27264 353514 : m_num_gpr_needed[vect_prologue]++;
27265 :
27266 353514 : stmt_cost += COSTS_N_INSNS (ix86_cost->integer_to_sse) / 2;
27267 : }
27268 : }
27269 : }
27270 1118894 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
27271 775476 : if (TREE_CODE (op) == SSA_NAME)
27272 546021 : TREE_VISITED (op) = 0;
27273 :
27274 343418 : if (stmt_cost > 0
27275 343418 : && dump_file && (dump_flags & TDF_DETAILS))
27276 2057 : fprintf (dump_file, "node %p gpr->xmm moves costs %d in prologue\n",
27277 : (void *)node, stmt_cost);
27278 343418 : m_costs[vect_prologue] += stmt_cost;
27279 : }
27280 :
27281 2771642 : return stmt_cost + vector_costs::add_slp_cost (node, parts);
27282 : }
27283 :
27284 : void
27285 1958021 : ix86_vector_costs::ix86_vect_estimate_reg_pressure ()
27286 : {
27287 1958021 : unsigned gpr_spill_cost = COSTS_N_INSNS (ix86_cost->int_store [2]) / 2;
27288 1958021 : unsigned sse_spill_cost = COSTS_N_INSNS (ix86_cost->sse_store[0]) / 2;
27289 :
27290 : /* Any better way to have target available fp registers, currently use SSE_REGS. */
27291 1958021 : unsigned target_avail_sse = TARGET_64BIT ? (TARGET_AVX512F ? 32 : 16) : 8;
27292 7832084 : for (unsigned i = 0; i != 3; i++)
27293 : {
27294 5874063 : if (m_num_gpr_needed[i] > target_avail_regs)
27295 865 : m_costs[i] += gpr_spill_cost * (m_num_gpr_needed[i] - target_avail_regs);
27296 : /* Only measure sse registers pressure. */
27297 5874063 : if (TARGET_SSE && (m_num_sse_needed[i] > target_avail_sse))
27298 100 : m_costs[i] += sse_spill_cost * (m_num_sse_needed[i] - target_avail_sse);
27299 : }
27300 1958021 : }
27301 :
27302 : void
27303 1958021 : ix86_vector_costs::finish_cost (const vector_costs *scalar_costs)
27304 : {
27305 1958021 : loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo);
27306 503755 : if (loop_vinfo && !m_costing_for_scalar)
27307 : {
27308 126474 : unsigned int vf = vect_vf_for_cost (loop_vinfo);
27309 551655 : for (auto inst : LOOP_VINFO_SLP_INSTANCES (loop_vinfo))
27310 172233 : if ((SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_group
27311 172233 : || SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_chain)
27312 172233 : && (vect_reduc_type (loop_vinfo, SLP_INSTANCE_TREE (inst))
27313 : == FOLD_LEFT_REDUCTION))
27314 3980 : m_num_fold_left_reduc_lanes
27315 3980 : += vf * SLP_TREE_LANES (SLP_INSTANCE_TREE (inst));
27316 :
27317 126474 : if (m_num_fold_left_reduc_lanes && dump_enabled_p ())
27318 19 : dump_printf_loc (MSG_NOTE, vect_location,
27319 : "in-order FP reduction lanes: %u\n",
27320 : m_num_fold_left_reduc_lanes);
27321 :
27322 : /* We are currently not asking the vectorizer to compare costs
27323 : between different vector mode sizes. When using predication
27324 : that will end up always choosing the preferred mode size even
27325 : if there's a smaller mode covering all lanes. Test for this
27326 : situation and artificially reject the larger mode attempt.
27327 : ??? We currently lack masked ops for sub-SSE sized modes,
27328 : so we could restrict this rejection to AVX and AVX512 modes
27329 : but error on the safe side for now. */
27330 126474 : if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
27331 29 : && !LOOP_VINFO_EPILOGUE_P (loop_vinfo)
27332 17 : && LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
27333 126485 : && (exact_log2 (LOOP_VINFO_VECT_FACTOR (loop_vinfo).to_constant ())
27334 22 : > ceil_log2 (LOOP_VINFO_INT_NITERS (loop_vinfo))))
27335 8 : m_costs[vect_body] = INT_MAX;
27336 :
27337 : /* We'd like to avoid using masking if there's an in-order reduction
27338 : to vectorize because that will also perform in-order adds of
27339 : masked elements (as neutral value, of course) here, but there
27340 : is currently no way to indicate to try un-masked with the same
27341 : mode. */
27342 :
27343 126474 : bool any_reduc_p = false;
27344 502429 : for (int i = 0; i != X86_REDUC_LAST; i++)
27345 377256 : if (m_num_reduc[i])
27346 : {
27347 : any_reduc_p = true;
27348 : break;
27349 : }
27350 :
27351 126474 : if (any_reduc_p
27352 : /* Not much gain for loop with gather and scatter. */
27353 1301 : && m_prefer_unroll
27354 1144 : && !LOOP_VINFO_EPILOGUE_P (loop_vinfo))
27355 : {
27356 1826 : unsigned unroll_factor
27357 913 : = OPTION_SET_P (ix86_vect_unroll_limit)
27358 913 : ? ix86_vect_unroll_limit
27359 913 : : ix86_cost->vect_unroll_limit;
27360 :
27361 913 : if (unroll_factor > 1)
27362 : {
27363 3652 : for (int i = 0 ; i != X86_REDUC_LAST; i++)
27364 : {
27365 2739 : if (m_num_reduc[i])
27366 : {
27367 913 : unsigned tmp = CEIL (ix86_cost->reduc_lat_mult_thr[i],
27368 : m_num_reduc[i]);
27369 2739 : unroll_factor = MIN (unroll_factor, tmp);
27370 : }
27371 : }
27372 :
27373 1826 : m_suggested_unroll_factor = 1 << ceil_log2 (unroll_factor);
27374 : }
27375 : }
27376 :
27377 : }
27378 :
27379 1958021 : ix86_vect_estimate_reg_pressure ();
27380 :
27381 9790105 : for (int i = 0; i != 3; i++)
27382 5874063 : if (m_num_avx256_vec_perm[i]
27383 539 : && TARGET_AVX256_AVOID_VEC_PERM)
27384 7 : m_costs[i] = INT_MAX;
27385 :
27386 7832084 : for (int i = 0; i != 3; i++)
27387 5874063 : if (m_num_avx512_vec_perm[i] && TARGET_AVX512_AVOID_VEC_PERM)
27388 5 : m_costs[i] = INT_MAX;
27389 :
27390 : /* When X86_TUNE_AVX512_TWO_EPILOGUES is enabled arrange for both
27391 : a AVX2 and a SSE epilogue for AVX512 vectorized loops. */
27392 1958021 : if (loop_vinfo
27393 503755 : && LOOP_VINFO_EPILOGUE_P (loop_vinfo)
27394 44400 : && GET_MODE_SIZE (loop_vinfo->vector_mode) == 32
27395 1958788 : && ix86_tune_features[X86_TUNE_AVX512_TWO_EPILOGUES])
27396 25 : m_suggested_epilogue_mode = V16QImode;
27397 : /* When a 128bit SSE vectorized epilogue still has a VF of 16 or larger
27398 : enable a 64bit SSE epilogue. */
27399 1958021 : if (loop_vinfo
27400 503755 : && LOOP_VINFO_EPILOGUE_P (loop_vinfo)
27401 44400 : && GET_MODE_SIZE (loop_vinfo->vector_mode) == 16
27402 1960575 : && LOOP_VINFO_VECT_FACTOR (loop_vinfo).to_constant () >= 16)
27403 110 : m_suggested_epilogue_mode = V8QImode;
27404 :
27405 : /* When X86_TUNE_AVX512_MASKED_EPILOGUES is enabled try to use
27406 : a masked epilogue if that doesn't seem detrimental. */
27407 1958021 : if (loop_vinfo
27408 503755 : && !LOOP_VINFO_EPILOGUE_P (loop_vinfo)
27409 481555 : && LOOP_VINFO_VECT_FACTOR (loop_vinfo).to_constant () > 2
27410 : /* Avoid a masked epilog if cascaded epilogues eventually get us
27411 : to one with VF 1 as that means no scalar epilog at all. */
27412 76750 : && !((GET_MODE_SIZE (loop_vinfo->vector_mode)
27413 76750 : / LOOP_VINFO_VECT_FACTOR (loop_vinfo).to_constant () == 16)
27414 34 : && ix86_tune_features[X86_TUNE_AVX512_TWO_EPILOGUES])
27415 76749 : && ix86_tune_features[X86_TUNE_AVX512_MASKED_EPILOGUES]
27416 1958205 : && !OPTION_SET_P (param_vect_partial_vector_usage))
27417 : {
27418 166 : bool avoid = false;
27419 166 : if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
27420 129 : && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) >= 0)
27421 : {
27422 129 : unsigned int peel_niter
27423 : = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
27424 129 : if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
27425 0 : peel_niter += 1;
27426 : /* When we know the number of scalar iterations of the epilogue,
27427 : avoid masking when a single vector epilog iteration handles
27428 : it in full. */
27429 129 : if (pow2p_hwi ((LOOP_VINFO_INT_NITERS (loop_vinfo) - peel_niter)
27430 129 : % LOOP_VINFO_VECT_FACTOR (loop_vinfo).to_constant ()))
27431 : avoid = true;
27432 : }
27433 164 : if (!avoid && loop_outer (loop_outer (LOOP_VINFO_LOOP (loop_vinfo))))
27434 14 : for (auto ddr : LOOP_VINFO_DDRS (loop_vinfo))
27435 : {
27436 4 : if (DDR_ARE_DEPENDENT (ddr) == chrec_known)
27437 : ;
27438 4 : else if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know)
27439 : ;
27440 : else
27441 : {
27442 2 : int loop_depth
27443 4 : = index_in_loop_nest (LOOP_VINFO_LOOP (loop_vinfo)->num,
27444 2 : DDR_LOOP_NEST (ddr));
27445 4 : if (DDR_NUM_DIST_VECTS (ddr) == 1
27446 2 : && DDR_DIST_VECTS (ddr)[0][loop_depth] == 0)
27447 : {
27448 : /* Avoid the case when there's an outer loop that might
27449 : traverse a multi-dimensional array with the inner
27450 : loop just executing the masked epilogue with a
27451 : read-write where the next outer iteration might
27452 : read from the masked part of the previous write,
27453 : 'n' filling half a vector.
27454 : for (j = 0; j < m; ++j)
27455 : for (i = 0; i < n; ++i)
27456 : a[j][i] = c * a[j][i]; */
27457 : avoid = true;
27458 : break;
27459 : }
27460 : }
27461 : }
27462 : /* Avoid using masking if there's an in-order reduction
27463 : to vectorize because that will also perform in-order adds of
27464 : masked elements (as neutral value, of course). */
27465 166 : if (!avoid)
27466 : {
27467 655 : for (auto inst : LOOP_VINFO_SLP_INSTANCES (loop_vinfo))
27468 173 : if (SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_group
27469 173 : && (vect_reduc_type (loop_vinfo, SLP_INSTANCE_TREE (inst))
27470 : == FOLD_LEFT_REDUCTION))
27471 : {
27472 : avoid = true;
27473 : break;
27474 : }
27475 : }
27476 162 : if (!avoid)
27477 : {
27478 158 : m_suggested_epilogue_mode = loop_vinfo->vector_mode;
27479 158 : m_masked_epilogue = 1;
27480 : }
27481 : }
27482 :
27483 1958021 : vector_costs::finish_cost (scalar_costs);
27484 1958021 : }
27485 :
27486 : /* Return true if THIS has a shorter fold-left reduction chain than OTHER. */
27487 :
27488 : bool
27489 35522 : ix86_vector_costs::better_fold_left_reduc_than_p
27490 : (const vector_costs *other) const
27491 : {
27492 35522 : auto other_costs = static_cast<const ix86_vector_costs *> (other);
27493 35522 : if (m_num_fold_left_reduc_lanes >= other_costs->m_num_fold_left_reduc_lanes)
27494 : return false;
27495 :
27496 : /* Do not let emulated sub-SSE modes win on this alone. */
27497 328 : loop_vec_info loop_vinfo = as_a<loop_vec_info> (m_vinfo);
27498 656 : return known_ge (GET_MODE_SIZE (loop_vinfo->vector_mode), 16);
27499 : }
27500 :
27501 : /* Return true if THIS should be preferred over OTHER as main vector loop. */
27502 :
27503 : bool
27504 33975 : ix86_vector_costs::better_main_loop_than_p (const vector_costs *other) const
27505 : {
27506 33975 : loop_vec_info this_loop_vinfo = as_a<loop_vec_info> (this->vinfo ());
27507 33975 : loop_vec_info other_loop_vinfo = as_a<loop_vec_info> (other->vinfo ());
27508 :
27509 33975 : if (better_fold_left_reduc_than_p (other))
27510 : return true;
27511 :
27512 : /* If the other loop is masked it does not need an epilog. Prefer that
27513 : if the current loop cannot be vectorized fully with a vector
27514 : epilogs with at most one scalar iteration left. */
27515 25682 : if (LOOP_VINFO_NITERS_KNOWN_P (this_loop_vinfo)
27516 25682 : && LOOP_VINFO_USING_PARTIAL_VECTORS_P (other_loop_vinfo)
27517 4 : && known_gt (LOOP_VINFO_VECT_FACTOR (other_loop_vinfo),
27518 : LOOP_VINFO_INT_NITERS (this_loop_vinfo))
27519 33942 : && (popcount_hwi (LOOP_VINFO_INT_NITERS (this_loop_vinfo) & ~1)
27520 4 : > (param_vect_epilogues_nomask != 0)))
27521 : return false;
27522 :
27523 33934 : return vector_costs::better_main_loop_than_p (other);
27524 : }
27525 :
27526 : /* Return true if THIS should be preferred over OTHER as epilog vector
27527 : loop when vectorizing MAIN_LOOP. */
27528 :
27529 : bool
27530 1547 : ix86_vector_costs::better_epilogue_loop_than_p (const vector_costs *other,
27531 : loop_vec_info main_loop) const
27532 : {
27533 1547 : loop_vec_info this_loop_info = as_a <loop_vec_info> (this->vinfo ());
27534 :
27535 1547 : if (better_fold_left_reduc_than_p (other))
27536 : return true;
27537 :
27538 : /* The x86 target allows for multiple vector epilogues, if THIS is
27539 : the suggested epilog mode of OTHER then keep the latter unless
27540 : THIS has a VF of one which means no further epilog needed. */
27541 1547 : int tem;
27542 1547 : if (known_gt (LOOP_VINFO_VECT_FACTOR (this_loop_info), 1U)
27543 1547 : && (GET_MODE_SIZE (other->suggested_epilogue_mode (tem))
27544 3070 : == GET_MODE_SIZE (this_loop_info->vector_mode)))
27545 : return false;
27546 1459 : return vector_costs::better_epilogue_loop_than_p (other, main_loop);
27547 : }
27548 :
27549 : /* Validate target specific memory model bits in VAL. */
27550 :
27551 : static unsigned HOST_WIDE_INT
27552 413746 : ix86_memmodel_check (unsigned HOST_WIDE_INT val)
27553 : {
27554 413746 : enum memmodel model = memmodel_from_int (val);
27555 413746 : bool strong;
27556 :
27557 413746 : if (val & ~(unsigned HOST_WIDE_INT)(IX86_HLE_ACQUIRE|IX86_HLE_RELEASE
27558 : |MEMMODEL_MASK)
27559 413742 : || ((val & IX86_HLE_ACQUIRE) && (val & IX86_HLE_RELEASE)))
27560 : {
27561 4 : warning (OPT_Winvalid_memory_model,
27562 : "unknown architecture specific memory model");
27563 4 : return MEMMODEL_SEQ_CST;
27564 : }
27565 413742 : strong = (is_mm_acq_rel (model) || is_mm_seq_cst (model));
27566 413742 : if (val & IX86_HLE_ACQUIRE && !(is_mm_acquire (model) || strong))
27567 : {
27568 0 : warning (OPT_Winvalid_memory_model,
27569 : "%<HLE_ACQUIRE%> not used with %<ACQUIRE%> or stronger "
27570 : "memory model");
27571 0 : return MEMMODEL_SEQ_CST | IX86_HLE_ACQUIRE;
27572 : }
27573 413742 : if (val & IX86_HLE_RELEASE && !(is_mm_release (model) || strong))
27574 : {
27575 0 : warning (OPT_Winvalid_memory_model,
27576 : "%<HLE_RELEASE%> not used with %<RELEASE%> or stronger "
27577 : "memory model");
27578 0 : return MEMMODEL_SEQ_CST | IX86_HLE_RELEASE;
27579 : }
27580 : return val;
27581 : }
27582 :
27583 : /* Set CLONEI->vecsize_mangle, CLONEI->mask_mode, CLONEI->vecsize_int,
27584 : CLONEI->vecsize_float and if CLONEI->simdlen is 0, also
27585 : CLONEI->simdlen. Return 0 if SIMD clones shouldn't be emitted,
27586 : or number of vecsize_mangle variants that should be emitted. */
27587 :
27588 : static int
27589 7725 : ix86_simd_clone_compute_vecsize_and_simdlen (struct cgraph_node *node,
27590 : struct cgraph_simd_clone *clonei,
27591 : tree base_type, int num,
27592 : bool explicit_p)
27593 : {
27594 7725 : int ret = 1;
27595 :
27596 7725 : if (clonei->simdlen
27597 7725 : && (clonei->simdlen < 2
27598 1325 : || clonei->simdlen > 1024
27599 1325 : || (clonei->simdlen & (clonei->simdlen - 1)) != 0))
27600 : {
27601 0 : if (explicit_p)
27602 0 : warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
27603 : "unsupported simdlen %wd", clonei->simdlen.to_constant ());
27604 : return 0;
27605 : }
27606 :
27607 7725 : tree ret_type = TREE_TYPE (TREE_TYPE (node->decl));
27608 7725 : if (TREE_CODE (ret_type) != VOID_TYPE)
27609 6905 : switch (TYPE_MODE (ret_type))
27610 : {
27611 6905 : case E_QImode:
27612 6905 : case E_HImode:
27613 6905 : case E_SImode:
27614 6905 : case E_DImode:
27615 6905 : case E_SFmode:
27616 6905 : case E_DFmode:
27617 : /* case E_SCmode: */
27618 : /* case E_DCmode: */
27619 6905 : if (!AGGREGATE_TYPE_P (ret_type))
27620 : break;
27621 : /* FALLTHRU */
27622 2 : default:
27623 2 : if (explicit_p)
27624 2 : warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
27625 : "unsupported return type %qT for simd", ret_type);
27626 : return 0;
27627 : }
27628 :
27629 7723 : tree t;
27630 7723 : int i;
27631 7723 : tree type_arg_types = TYPE_ARG_TYPES (TREE_TYPE (node->decl));
27632 7723 : bool decl_arg_p = (node->definition || type_arg_types == NULL_TREE);
27633 :
27634 7723 : for (t = (decl_arg_p ? DECL_ARGUMENTS (node->decl) : type_arg_types), i = 0;
27635 20710 : t && t != void_list_node; t = TREE_CHAIN (t), i++)
27636 : {
27637 16938 : tree arg_type = decl_arg_p ? TREE_TYPE (t) : TREE_VALUE (t);
27638 12992 : switch (TYPE_MODE (arg_type))
27639 : {
27640 12973 : case E_QImode:
27641 12973 : case E_HImode:
27642 12973 : case E_SImode:
27643 12973 : case E_DImode:
27644 12973 : case E_SFmode:
27645 12973 : case E_DFmode:
27646 : /* case E_SCmode: */
27647 : /* case E_DCmode: */
27648 12973 : if (!AGGREGATE_TYPE_P (arg_type))
27649 : break;
27650 : /* FALLTHRU */
27651 41 : default:
27652 41 : if (clonei->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM)
27653 : break;
27654 5 : if (explicit_p)
27655 5 : warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
27656 : "unsupported argument type %qT for simd", arg_type);
27657 : return 0;
27658 : }
27659 : }
27660 :
27661 7718 : if (!TREE_PUBLIC (node->decl) || !explicit_p)
27662 : {
27663 : /* If the function isn't exported, we can pick up just one ISA
27664 : for the clones. */
27665 114 : if (TARGET_AVX512F)
27666 0 : clonei->vecsize_mangle = 'e';
27667 114 : else if (TARGET_AVX2)
27668 1 : clonei->vecsize_mangle = 'd';
27669 113 : else if (TARGET_AVX)
27670 88 : clonei->vecsize_mangle = 'c';
27671 : else
27672 25 : clonei->vecsize_mangle = 'b';
27673 : ret = 1;
27674 : }
27675 : else
27676 : {
27677 7604 : clonei->vecsize_mangle = "bcde"[num];
27678 7604 : ret = 4;
27679 : }
27680 7718 : clonei->mask_mode = VOIDmode;
27681 7718 : switch (clonei->vecsize_mangle)
27682 : {
27683 1926 : case 'b':
27684 1926 : clonei->vecsize_int = 128;
27685 1926 : clonei->vecsize_float = 128;
27686 1926 : break;
27687 1989 : case 'c':
27688 1989 : clonei->vecsize_int = 128;
27689 1989 : clonei->vecsize_float = 256;
27690 1989 : break;
27691 1902 : case 'd':
27692 1902 : clonei->vecsize_int = 256;
27693 1902 : clonei->vecsize_float = 256;
27694 1902 : break;
27695 1901 : case 'e':
27696 1901 : clonei->vecsize_int = 512;
27697 1901 : clonei->vecsize_float = 512;
27698 1901 : if (TYPE_MODE (base_type) == QImode)
27699 19 : clonei->mask_mode = DImode;
27700 : else
27701 1882 : clonei->mask_mode = SImode;
27702 : break;
27703 : }
27704 7718 : if (clonei->simdlen == 0)
27705 : {
27706 6393 : if (SCALAR_INT_MODE_P (TYPE_MODE (base_type)))
27707 3329 : clonei->simdlen = clonei->vecsize_int;
27708 : else
27709 3064 : clonei->simdlen = clonei->vecsize_float;
27710 6393 : clonei->simdlen = clonei->simdlen
27711 12786 : / GET_MODE_BITSIZE (TYPE_MODE (base_type));
27712 : }
27713 1325 : else if (clonei->simdlen > 16)
27714 : {
27715 : /* For compatibility with ICC, use the same upper bounds
27716 : for simdlen. In particular, for CTYPE below, use the return type,
27717 : unless the function returns void, in that case use the characteristic
27718 : type. If it is possible for given SIMDLEN to pass CTYPE value
27719 : in registers (8 [XYZ]MM* regs for 32-bit code, 16 [XYZ]MM* regs
27720 : for 64-bit code), accept that SIMDLEN, otherwise warn and don't
27721 : emit corresponding clone. */
27722 12 : tree ctype = ret_type;
27723 12 : if (VOID_TYPE_P (ret_type))
27724 0 : ctype = base_type;
27725 24 : int cnt = GET_MODE_BITSIZE (TYPE_MODE (ctype)) * clonei->simdlen;
27726 12 : if (SCALAR_INT_MODE_P (TYPE_MODE (ctype)))
27727 8 : cnt /= clonei->vecsize_int;
27728 : else
27729 4 : cnt /= clonei->vecsize_float;
27730 12 : if (cnt > (TARGET_64BIT ? 16 : 8))
27731 : {
27732 0 : if (explicit_p)
27733 0 : warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
27734 : "unsupported simdlen %wd",
27735 : clonei->simdlen.to_constant ());
27736 : return 0;
27737 : }
27738 : }
27739 : return ret;
27740 : }
27741 :
27742 : /* If SIMD clone NODE can't be used in a vectorized loop
27743 : in current function, return -1, otherwise return a badness of using it
27744 : (0 if it is most desirable from vecsize_mangle point of view, 1
27745 : slightly less desirable, etc.). */
27746 :
27747 : static int
27748 1764 : ix86_simd_clone_usable (struct cgraph_node *node, machine_mode)
27749 : {
27750 1764 : switch (node->simdclone->vecsize_mangle)
27751 : {
27752 626 : case 'b':
27753 626 : if (!TARGET_SSE2)
27754 : return -1;
27755 626 : if (!TARGET_AVX)
27756 : return 0;
27757 525 : return TARGET_AVX512F ? 3 : TARGET_AVX2 ? 2 : 1;
27758 628 : case 'c':
27759 628 : if (!TARGET_AVX)
27760 : return -1;
27761 583 : return TARGET_AVX512F ? 2 : TARGET_AVX2 ? 1 : 0;
27762 322 : case 'd':
27763 322 : if (!TARGET_AVX2)
27764 : return -1;
27765 117 : return TARGET_AVX512F ? 1 : 0;
27766 188 : case 'e':
27767 188 : if (!TARGET_AVX512F)
27768 130 : return -1;
27769 : return 0;
27770 0 : default:
27771 0 : gcc_unreachable ();
27772 : }
27773 : }
27774 :
27775 : /* This function adjusts the unroll factor based on
27776 : the hardware capabilities. For ex, bdver3 has
27777 : a loop buffer which makes unrolling of smaller
27778 : loops less important. This function decides the
27779 : unroll factor using number of memory references
27780 : (value 32 is used) as a heuristic. */
27781 :
27782 : static unsigned
27783 811195 : ix86_loop_unroll_adjust (unsigned nunroll, class loop *loop)
27784 : {
27785 811195 : basic_block *bbs;
27786 811195 : rtx_insn *insn;
27787 811195 : unsigned i;
27788 811195 : unsigned mem_count = 0;
27789 :
27790 : /* Unroll small size loop when unroll factor is not explicitly
27791 : specified. */
27792 811195 : if (ix86_unroll_only_small_loops && !loop->unroll)
27793 : {
27794 767348 : if (loop->ninsns <= ix86_cost->small_unroll_ninsns)
27795 73792 : return MIN (nunroll, ix86_cost->small_unroll_factor);
27796 : else
27797 : return 1;
27798 : }
27799 :
27800 43847 : if (!TARGET_ADJUST_UNROLL)
27801 : return nunroll;
27802 :
27803 : /* Count the number of memory references within the loop body.
27804 : This value determines the unrolling factor for bdver3 and bdver4
27805 : architectures. */
27806 8 : subrtx_iterator::array_type array;
27807 8 : bbs = get_loop_body (loop);
27808 32 : for (i = 0; i < loop->num_nodes; i++)
27809 120 : FOR_BB_INSNS (bbs[i], insn)
27810 104 : if (NONDEBUG_INSN_P (insn))
27811 588 : FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
27812 516 : if (const_rtx x = *iter)
27813 516 : if (MEM_P (x))
27814 : {
27815 28 : machine_mode mode = GET_MODE (x);
27816 56 : unsigned int n_words = GET_MODE_SIZE (mode) / UNITS_PER_WORD;
27817 28 : if (n_words > 4)
27818 0 : mem_count += 2;
27819 : else
27820 28 : mem_count += 1;
27821 : }
27822 8 : free (bbs);
27823 :
27824 8 : if (mem_count && mem_count <=32)
27825 8 : return MIN (nunroll, 32 / mem_count);
27826 :
27827 : return nunroll;
27828 8 : }
27829 :
27830 :
27831 : /* Implement TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P. */
27832 :
27833 : static bool
27834 427528 : ix86_float_exceptions_rounding_supported_p (void)
27835 : {
27836 : /* For x87 floating point with standard excess precision handling,
27837 : there is no adddf3 pattern (since x87 floating point only has
27838 : XFmode operations) so the default hook implementation gets this
27839 : wrong. */
27840 427528 : return TARGET_80387 || (TARGET_SSE && TARGET_SSE_MATH);
27841 : }
27842 :
27843 : /* Implement TARGET_ATOMIC_ASSIGN_EXPAND_FENV. */
27844 :
27845 : static void
27846 7054 : ix86_atomic_assign_expand_fenv (tree *hold, tree *clear, tree *update)
27847 : {
27848 7054 : if (!TARGET_80387 && !(TARGET_SSE && TARGET_SSE_MATH))
27849 : return;
27850 7054 : tree exceptions_var = create_tmp_var_raw (integer_type_node);
27851 7054 : if (TARGET_80387)
27852 : {
27853 7054 : tree fenv_index_type = build_index_type (size_int (6));
27854 7054 : tree fenv_type = build_array_type (unsigned_type_node, fenv_index_type);
27855 7054 : tree fenv_var = create_tmp_var_raw (fenv_type);
27856 7054 : TREE_ADDRESSABLE (fenv_var) = 1;
27857 7054 : tree fenv_ptr = build_pointer_type (fenv_type);
27858 7054 : tree fenv_addr = build1 (ADDR_EXPR, fenv_ptr, fenv_var);
27859 7054 : fenv_addr = fold_convert (ptr_type_node, fenv_addr);
27860 7054 : tree fnstenv = get_ix86_builtin (IX86_BUILTIN_FNSTENV);
27861 7054 : tree fldenv = get_ix86_builtin (IX86_BUILTIN_FLDENV);
27862 7054 : tree fnstsw = get_ix86_builtin (IX86_BUILTIN_FNSTSW);
27863 7054 : tree fnclex = get_ix86_builtin (IX86_BUILTIN_FNCLEX);
27864 7054 : tree hold_fnstenv = build_call_expr (fnstenv, 1, fenv_addr);
27865 7054 : tree hold_fnclex = build_call_expr (fnclex, 0);
27866 7054 : fenv_var = build4 (TARGET_EXPR, fenv_type, fenv_var, hold_fnstenv,
27867 : NULL_TREE, NULL_TREE);
27868 7054 : *hold = build2 (COMPOUND_EXPR, void_type_node, fenv_var,
27869 : hold_fnclex);
27870 7054 : *clear = build_call_expr (fnclex, 0);
27871 7054 : tree sw_var = create_tmp_var_raw (short_unsigned_type_node);
27872 7054 : tree fnstsw_call = build_call_expr (fnstsw, 0);
27873 7054 : tree sw_mod = build4 (TARGET_EXPR, short_unsigned_type_node, sw_var,
27874 : fnstsw_call, NULL_TREE, NULL_TREE);
27875 7054 : tree exceptions_x87 = fold_convert (integer_type_node, sw_var);
27876 7054 : tree update_mod = build4 (TARGET_EXPR, integer_type_node,
27877 : exceptions_var, exceptions_x87,
27878 : NULL_TREE, NULL_TREE);
27879 7054 : *update = build2 (COMPOUND_EXPR, integer_type_node,
27880 : sw_mod, update_mod);
27881 7054 : tree update_fldenv = build_call_expr (fldenv, 1, fenv_addr);
27882 7054 : *update = build2 (COMPOUND_EXPR, void_type_node, *update, update_fldenv);
27883 : }
27884 7054 : if (TARGET_SSE && TARGET_SSE_MATH)
27885 : {
27886 7054 : tree mxcsr_orig_var = create_tmp_var_raw (unsigned_type_node);
27887 7054 : tree mxcsr_mod_var = create_tmp_var_raw (unsigned_type_node);
27888 7054 : tree stmxcsr = get_ix86_builtin (IX86_BUILTIN_STMXCSR);
27889 7054 : tree ldmxcsr = get_ix86_builtin (IX86_BUILTIN_LDMXCSR);
27890 7054 : tree stmxcsr_hold_call = build_call_expr (stmxcsr, 0);
27891 7054 : tree hold_assign_orig = build4 (TARGET_EXPR, unsigned_type_node,
27892 : mxcsr_orig_var, stmxcsr_hold_call,
27893 : NULL_TREE, NULL_TREE);
27894 7054 : tree hold_mod_val = build2 (BIT_IOR_EXPR, unsigned_type_node,
27895 : mxcsr_orig_var,
27896 : build_int_cst (unsigned_type_node, 0x1f80));
27897 7054 : hold_mod_val = build2 (BIT_AND_EXPR, unsigned_type_node, hold_mod_val,
27898 : build_int_cst (unsigned_type_node, 0xffffffc0));
27899 7054 : tree hold_assign_mod = build4 (TARGET_EXPR, unsigned_type_node,
27900 : mxcsr_mod_var, hold_mod_val,
27901 : NULL_TREE, NULL_TREE);
27902 7054 : tree ldmxcsr_hold_call = build_call_expr (ldmxcsr, 1, mxcsr_mod_var);
27903 7054 : tree hold_all = build2 (COMPOUND_EXPR, unsigned_type_node,
27904 : hold_assign_orig, hold_assign_mod);
27905 7054 : hold_all = build2 (COMPOUND_EXPR, void_type_node, hold_all,
27906 : ldmxcsr_hold_call);
27907 7054 : if (*hold)
27908 7054 : *hold = build2 (COMPOUND_EXPR, void_type_node, *hold, hold_all);
27909 : else
27910 : *hold = hold_all;
27911 7054 : tree ldmxcsr_clear_call = build_call_expr (ldmxcsr, 1, mxcsr_mod_var);
27912 7054 : if (*clear)
27913 7054 : *clear = build2 (COMPOUND_EXPR, void_type_node, *clear,
27914 : ldmxcsr_clear_call);
27915 : else
27916 : *clear = ldmxcsr_clear_call;
27917 7054 : tree stxmcsr_update_call = build_call_expr (stmxcsr, 0);
27918 7054 : tree exceptions_sse = fold_convert (integer_type_node,
27919 : stxmcsr_update_call);
27920 7054 : if (*update)
27921 : {
27922 7054 : tree exceptions_mod = build2 (BIT_IOR_EXPR, integer_type_node,
27923 : exceptions_var, exceptions_sse);
27924 7054 : tree exceptions_assign = build2 (MODIFY_EXPR, integer_type_node,
27925 : exceptions_var, exceptions_mod);
27926 7054 : *update = build2 (COMPOUND_EXPR, integer_type_node, *update,
27927 : exceptions_assign);
27928 : }
27929 : else
27930 0 : *update = build4 (TARGET_EXPR, integer_type_node, exceptions_var,
27931 : exceptions_sse, NULL_TREE, NULL_TREE);
27932 7054 : tree ldmxcsr_update_call = build_call_expr (ldmxcsr, 1, mxcsr_orig_var);
27933 7054 : *update = build2 (COMPOUND_EXPR, void_type_node, *update,
27934 : ldmxcsr_update_call);
27935 : }
27936 7054 : tree atomic_feraiseexcept
27937 7054 : = builtin_decl_implicit (BUILT_IN_ATOMIC_FERAISEEXCEPT);
27938 7054 : tree atomic_feraiseexcept_call = build_call_expr (atomic_feraiseexcept,
27939 : 1, exceptions_var);
27940 7054 : *update = build2 (COMPOUND_EXPR, void_type_node, *update,
27941 : atomic_feraiseexcept_call);
27942 : }
27943 :
27944 : #if !TARGET_MACHO && !TARGET_DLLIMPORT_DECL_ATTRIBUTES
27945 : /* For i386, common symbol is local only for non-PIE binaries. For
27946 : x86-64, common symbol is local only for non-PIE binaries or linker
27947 : supports copy reloc in PIE binaries. */
27948 :
27949 : static bool
27950 807363893 : ix86_binds_local_p (const_tree exp)
27951 : {
27952 807363893 : bool direct_extern_access
27953 807363893 : = (ix86_direct_extern_access
27954 1611182529 : && !(VAR_OR_FUNCTION_DECL_P (exp)
27955 803818636 : && lookup_attribute ("nodirect_extern_access",
27956 803818636 : DECL_ATTRIBUTES (exp))));
27957 1233 : if (!direct_extern_access)
27958 1233 : ix86_has_no_direct_extern_access = true;
27959 807363893 : return default_binds_local_p_3 (exp, flag_shlib != 0, true,
27960 : direct_extern_access,
27961 : (direct_extern_access
27962 807362660 : && (!flag_pic
27963 145802733 : || (TARGET_64BIT
27964 807363893 : && HAVE_LD_PIE_COPYRELOC != 0))));
27965 : }
27966 :
27967 : /* If flag_pic or ix86_direct_extern_access is false, then neither
27968 : local nor global relocs should be placed in readonly memory. */
27969 :
27970 : static int
27971 5173563 : ix86_reloc_rw_mask (void)
27972 : {
27973 5173563 : return (flag_pic || !ix86_direct_extern_access) ? 3 : 0;
27974 : }
27975 : #endif
27976 :
27977 : /* Return true iff ADDR can be used as a symbolic base address. */
27978 :
27979 : static bool
27980 3155 : symbolic_base_address_p (rtx addr)
27981 : {
27982 0 : if (SYMBOL_REF_P (addr))
27983 : return true;
27984 :
27985 3120 : if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_GOTOFF)
27986 0 : return true;
27987 :
27988 : return false;
27989 : }
27990 :
27991 : /* Return true iff ADDR can be used as a base address. */
27992 :
27993 : static bool
27994 4879 : base_address_p (rtx addr)
27995 : {
27996 0 : if (REG_P (addr))
27997 : return true;
27998 :
27999 2998 : if (symbolic_base_address_p (addr))
28000 0 : return true;
28001 :
28002 : return false;
28003 : }
28004 :
28005 : /* If MEM is in the form of [(base+symbase)+offset], extract the three
28006 : parts of address and set to BASE, SYMBASE and OFFSET, otherwise
28007 : return false. */
28008 :
28009 : static bool
28010 3129 : extract_base_offset_in_addr (rtx mem, rtx *base, rtx *symbase, rtx *offset)
28011 : {
28012 3129 : rtx addr;
28013 :
28014 3129 : gcc_assert (MEM_P (mem));
28015 :
28016 3129 : addr = XEXP (mem, 0);
28017 :
28018 3129 : if (GET_CODE (addr) == CONST)
28019 10 : addr = XEXP (addr, 0);
28020 :
28021 3129 : if (base_address_p (addr))
28022 : {
28023 1379 : *base = addr;
28024 1379 : *symbase = const0_rtx;
28025 1379 : *offset = const0_rtx;
28026 1379 : return true;
28027 : }
28028 :
28029 1750 : if (GET_CODE (addr) == PLUS
28030 1750 : && base_address_p (XEXP (addr, 0)))
28031 : {
28032 537 : rtx addend = XEXP (addr, 1);
28033 :
28034 537 : if (GET_CODE (addend) == CONST)
28035 0 : addend = XEXP (addend, 0);
28036 :
28037 537 : if (CONST_INT_P (addend))
28038 : {
28039 380 : *base = XEXP (addr, 0);
28040 380 : *symbase = const0_rtx;
28041 380 : *offset = addend;
28042 380 : return true;
28043 : }
28044 :
28045 : /* Also accept REG + symbolic ref, with or without a CONST_INT
28046 : offset. */
28047 157 : if (REG_P (XEXP (addr, 0)))
28048 : {
28049 157 : if (symbolic_base_address_p (addend))
28050 : {
28051 0 : *base = XEXP (addr, 0);
28052 0 : *symbase = addend;
28053 0 : *offset = const0_rtx;
28054 0 : return true;
28055 : }
28056 :
28057 157 : if (GET_CODE (addend) == PLUS
28058 0 : && symbolic_base_address_p (XEXP (addend, 0))
28059 157 : && CONST_INT_P (XEXP (addend, 1)))
28060 : {
28061 0 : *base = XEXP (addr, 0);
28062 0 : *symbase = XEXP (addend, 0);
28063 0 : *offset = XEXP (addend, 1);
28064 0 : return true;
28065 : }
28066 : }
28067 : }
28068 :
28069 : return false;
28070 : }
28071 :
28072 : /* Given OPERANDS of consecutive load/store, check if we can merge
28073 : them into move multiple. LOAD is true if they are load instructions.
28074 : MODE is the mode of memory operands. */
28075 :
28076 : bool
28077 1715 : ix86_operands_ok_for_move_multiple (rtx *operands, bool load,
28078 : machine_mode mode)
28079 : {
28080 1715 : HOST_WIDE_INT offval_1, offval_2, msize;
28081 1715 : rtx mem_1, mem_2, reg_1, reg_2, base_1, base_2,
28082 : symbase_1, symbase_2, offset_1, offset_2;
28083 :
28084 1715 : if (load)
28085 : {
28086 1354 : mem_1 = operands[1];
28087 1354 : mem_2 = operands[3];
28088 : /* Avoid fusing two loads from distinct parameters as that might
28089 : cause STLF failure. */
28090 1354 : tree base_1;
28091 1354 : if (MEM_EXPR (mem_1)
28092 1346 : && MEM_EXPR (mem_2)
28093 1343 : && (TREE_CODE (base_1 = get_base_address (MEM_EXPR (mem_1)))
28094 : == PARM_DECL)
28095 1354 : && get_base_address (MEM_EXPR (mem_2)) != base_1)
28096 : return false;
28097 1354 : reg_1 = operands[0];
28098 1354 : reg_2 = operands[2];
28099 : }
28100 : else
28101 : {
28102 361 : mem_1 = operands[0];
28103 361 : mem_2 = operands[2];
28104 361 : reg_1 = operands[1];
28105 361 : reg_2 = operands[3];
28106 : }
28107 :
28108 1715 : gcc_assert (REG_P (reg_1) && REG_P (reg_2));
28109 :
28110 1715 : if (REGNO (reg_1) != REGNO (reg_2))
28111 : return false;
28112 :
28113 : /* Check if the addresses are in the form of [base+offset]. */
28114 1710 : if (!extract_base_offset_in_addr (mem_1, &base_1, &symbase_1, &offset_1))
28115 : return false;
28116 1419 : if (!extract_base_offset_in_addr (mem_2, &base_2, &symbase_2, &offset_2))
28117 : return false;
28118 :
28119 : /* Check if the bases are the same. */
28120 340 : if (!rtx_equal_p (base_1, base_2) || !rtx_equal_p (symbase_1, symbase_2))
28121 : return false;
28122 :
28123 211 : offval_1 = INTVAL (offset_1);
28124 211 : offval_2 = INTVAL (offset_2);
28125 211 : msize = GET_MODE_SIZE (mode);
28126 : /* Check if mem_1 is adjacent to mem_2 and mem_1 has lower address. */
28127 211 : if (offval_1 + msize != offval_2)
28128 164 : return false;
28129 :
28130 : return true;
28131 : }
28132 :
28133 : /* Implement the TARGET_OPTAB_SUPPORTED_P hook. */
28134 :
28135 : static bool
28136 349230 : ix86_optab_supported_p (int op, machine_mode mode1, machine_mode,
28137 : optimization_type opt_type)
28138 : {
28139 349230 : switch (op)
28140 : {
28141 158 : case asin_optab:
28142 158 : case acos_optab:
28143 158 : case log1p_optab:
28144 158 : case exp_optab:
28145 158 : case exp10_optab:
28146 158 : case exp2_optab:
28147 158 : case expm1_optab:
28148 158 : case ldexp_optab:
28149 158 : case scalb_optab:
28150 158 : case lround_optab:
28151 158 : return opt_type == OPTIMIZE_FOR_SPEED;
28152 :
28153 82 : case round_optab:
28154 : /* Inlined sequence for round may takes 2 more insns
28155 : than current -Os path. */
28156 82 : if (mode1 == HFmode)
28157 : return true;
28158 78 : return opt_type == OPTIMIZE_FOR_SPEED;
28159 :
28160 286 : case rint_optab:
28161 286 : if (SSE_FLOAT_MODE_P (mode1)
28162 139 : && TARGET_SSE_MATH
28163 127 : && !flag_trapping_math
28164 21 : && !TARGET_SSE4_1
28165 : && mode1 != HFmode)
28166 21 : return opt_type == OPTIMIZE_FOR_SPEED;
28167 : return true;
28168 :
28169 2058 : case floor_optab:
28170 2058 : case ceil_optab:
28171 2058 : case btrunc_optab:
28172 2058 : if ((SSE_FLOAT_MODE_P (mode1)
28173 1597 : && TARGET_SSE_MATH
28174 1530 : && TARGET_SSE4_1)
28175 1991 : || mode1 == HFmode)
28176 : return true;
28177 1913 : return opt_type == OPTIMIZE_FOR_SPEED;
28178 :
28179 66 : case rsqrt_optab:
28180 66 : return opt_type == OPTIMIZE_FOR_SPEED && use_rsqrt_p (mode1);
28181 :
28182 : default:
28183 : return true;
28184 : }
28185 : }
28186 :
28187 : /* Address space support.
28188 :
28189 : This is not "far pointers" in the 16-bit sense, but an easy way
28190 : to use %fs and %gs segment prefixes. Therefore:
28191 :
28192 : (a) All address spaces have the same modes,
28193 : (b) All address spaces have the same address forms,
28194 : (c) While %fs and %gs are technically subsets of the generic
28195 : address space, they are probably not subsets of each other.
28196 : (d) Since we have no access to the segment base register values
28197 : without resorting to a system call, we cannot convert a
28198 : non-default address space to a default address space.
28199 : Therefore we do not claim %fs or %gs are subsets of generic.
28200 :
28201 : Therefore we can (mostly) use the default hooks. */
28202 :
28203 : /* All use of segmentation is assumed to make address 0 valid. */
28204 :
28205 : static bool
28206 70078986 : ix86_addr_space_zero_address_valid (addr_space_t as)
28207 : {
28208 70078986 : return as != ADDR_SPACE_GENERIC;
28209 : }
28210 :
28211 : static void
28212 791509 : ix86_init_libfuncs (void)
28213 : {
28214 791509 : if (TARGET_64BIT)
28215 : {
28216 776467 : set_optab_libfunc (sdivmod_optab, TImode, "__divmodti4");
28217 776467 : set_optab_libfunc (udivmod_optab, TImode, "__udivmodti4");
28218 : }
28219 : else
28220 : {
28221 15042 : set_optab_libfunc (sdivmod_optab, DImode, "__divmoddi4");
28222 15042 : set_optab_libfunc (udivmod_optab, DImode, "__udivmoddi4");
28223 : }
28224 :
28225 : #if TARGET_MACHO
28226 : darwin_rename_builtins ();
28227 : #endif
28228 791509 : }
28229 :
28230 : /* Set the value of FLT_EVAL_METHOD in float.h. When using only the
28231 : FPU, assume that the fpcw is set to extended precision; when using
28232 : only SSE, rounding is correct; when using both SSE and the FPU,
28233 : the rounding precision is indeterminate, since either may be chosen
28234 : apparently at random. */
28235 :
28236 : static enum flt_eval_method
28237 92106789 : ix86_get_excess_precision (enum excess_precision_type type)
28238 : {
28239 92106789 : switch (type)
28240 : {
28241 87974471 : case EXCESS_PRECISION_TYPE_FAST:
28242 : /* The fastest type to promote to will always be the native type,
28243 : whether that occurs with implicit excess precision or
28244 : otherwise. */
28245 87974471 : return TARGET_AVX512FP16
28246 87974471 : ? FLT_EVAL_METHOD_PROMOTE_TO_FLOAT16
28247 87974471 : : FLT_EVAL_METHOD_PROMOTE_TO_FLOAT;
28248 4131574 : case EXCESS_PRECISION_TYPE_STANDARD:
28249 4131574 : case EXCESS_PRECISION_TYPE_IMPLICIT:
28250 : /* Otherwise, the excess precision we want when we are
28251 : in a standards compliant mode, and the implicit precision we
28252 : provide would be identical were it not for the unpredictable
28253 : cases. */
28254 4131574 : if (TARGET_AVX512FP16 && TARGET_SSE_MATH)
28255 : return FLT_EVAL_METHOD_PROMOTE_TO_FLOAT16;
28256 4125647 : else if (!TARGET_80387)
28257 : return FLT_EVAL_METHOD_PROMOTE_TO_FLOAT;
28258 4119229 : else if (!TARGET_MIX_SSE_I387)
28259 : {
28260 4119057 : if (!(TARGET_SSE && TARGET_SSE_MATH))
28261 : return FLT_EVAL_METHOD_PROMOTE_TO_LONG_DOUBLE;
28262 3130435 : else if (TARGET_SSE2)
28263 : return FLT_EVAL_METHOD_PROMOTE_TO_FLOAT;
28264 : }
28265 :
28266 : /* If we are in standards compliant mode, but we know we will
28267 : calculate in unpredictable precision, return
28268 : FLT_EVAL_METHOD_FLOAT. There is no reason to introduce explicit
28269 : excess precision if the target can't guarantee it will honor
28270 : it. */
28271 336 : return (type == EXCESS_PRECISION_TYPE_STANDARD
28272 336 : ? FLT_EVAL_METHOD_PROMOTE_TO_FLOAT
28273 : : FLT_EVAL_METHOD_UNPREDICTABLE);
28274 744 : case EXCESS_PRECISION_TYPE_FLOAT16:
28275 744 : if (TARGET_80387
28276 738 : && !(TARGET_SSE_MATH && TARGET_SSE))
28277 4 : error ("%<-fexcess-precision=16%> is not compatible with %<-mfpmath=387%>");
28278 : return FLT_EVAL_METHOD_PROMOTE_TO_FLOAT16;
28279 0 : default:
28280 0 : gcc_unreachable ();
28281 : }
28282 :
28283 : return FLT_EVAL_METHOD_UNPREDICTABLE;
28284 : }
28285 :
28286 : /* Return true if _BitInt(N) is supported and fill its details into *INFO. */
28287 : bool
28288 366772 : ix86_bitint_type_info (int n, struct bitint_info *info)
28289 : {
28290 366772 : if (n <= 8)
28291 12003 : info->limb_mode = QImode;
28292 354769 : else if (n <= 16)
28293 3722 : info->limb_mode = HImode;
28294 351047 : else if (n <= 32 || (!TARGET_64BIT && n > 64))
28295 47515 : info->limb_mode = SImode;
28296 : else
28297 303532 : info->limb_mode = DImode;
28298 366772 : info->abi_limb_mode = info->limb_mode;
28299 366772 : info->big_endian = false;
28300 366772 : info->extended = bitint_ext_undef;
28301 366772 : return true;
28302 : }
28303 :
28304 : /* Implement TARGET_C_MODE_FOR_FLOATING_TYPE. Return DFmode, TFmode
28305 : or XFmode for TI_LONG_DOUBLE_TYPE which is for long double type,
28306 : based on long double bits, go with the default one for the others. */
28307 :
28308 : static machine_mode
28309 3743243 : ix86_c_mode_for_floating_type (enum tree_index ti)
28310 : {
28311 3743243 : if (ti == TI_LONG_DOUBLE_TYPE)
28312 624378 : return (TARGET_LONG_DOUBLE_64 ? DFmode
28313 624346 : : (TARGET_LONG_DOUBLE_128 ? TFmode : XFmode));
28314 3118865 : return default_mode_for_floating_type (ti);
28315 : }
28316 :
28317 : /* Returns modified FUNCTION_TYPE for cdtor callabi. */
28318 : tree
28319 14894 : ix86_cxx_adjust_cdtor_callabi_fntype (tree fntype)
28320 : {
28321 14894 : if (TARGET_64BIT
28322 71 : || TARGET_RTD
28323 14965 : || ix86_function_type_abi (fntype) != MS_ABI)
28324 : return fntype;
28325 : /* For 32-bit MS ABI add thiscall attribute. */
28326 0 : tree attribs = tree_cons (get_identifier ("thiscall"), NULL_TREE,
28327 0 : TYPE_ATTRIBUTES (fntype));
28328 0 : return build_type_attribute_variant (fntype, attribs);
28329 : }
28330 :
28331 : /* Implement PUSH_ROUNDING. On 386, we have pushw instruction that
28332 : decrements by exactly 2 no matter what the position was, there is no pushb.
28333 :
28334 : But as CIE data alignment factor on this arch is -4 for 32bit targets
28335 : and -8 for 64bit targets, we need to make sure all stack pointer adjustments
28336 : are in multiple of 4 for 32bit targets and 8 for 64bit targets. */
28337 :
28338 : poly_int64
28339 266453041 : ix86_push_rounding (poly_int64 bytes)
28340 : {
28341 342133456 : return ROUND_UP (bytes, UNITS_PER_WORD);
28342 : }
28343 :
28344 : /* Use 8 bits metadata start from bit48 for LAM_U48,
28345 : 6 bits metadata start from bit57 for LAM_U57. */
28346 : #define IX86_HWASAN_SHIFT (ix86_lam_type == lam_u48 \
28347 : ? 48 \
28348 : : (ix86_lam_type == lam_u57 ? 57 : 0))
28349 : #define IX86_HWASAN_TAG_SIZE (ix86_lam_type == lam_u48 \
28350 : ? 8 \
28351 : : (ix86_lam_type == lam_u57 ? 6 : 0))
28352 :
28353 : /* Implement TARGET_MEMTAG_CAN_TAG_ADDRESSES. */
28354 : bool
28355 6470718 : ix86_memtag_can_tag_addresses ()
28356 : {
28357 6470718 : return ix86_lam_type != lam_none && TARGET_LP64;
28358 : }
28359 :
28360 : /* Implement TARGET_MEMTAG_TAG_BITSIZE. */
28361 : unsigned char
28362 445 : ix86_memtag_tag_bitsize ()
28363 : {
28364 445 : return IX86_HWASAN_TAG_SIZE;
28365 : }
28366 :
28367 : /* Implement TARGET_MEMTAG_SET_TAG. */
28368 : rtx
28369 105 : ix86_memtag_set_tag (rtx untagged, rtx tag, rtx target)
28370 : {
28371 : /* default_memtag_insert_random_tag may
28372 : generate tag with value more than 6 bits. */
28373 105 : if (ix86_lam_type == lam_u57)
28374 : {
28375 105 : unsigned HOST_WIDE_INT and_imm
28376 : = (HOST_WIDE_INT_1U << IX86_HWASAN_TAG_SIZE) - 1;
28377 :
28378 105 : emit_insn (gen_andqi3 (tag, tag, GEN_INT (and_imm)));
28379 : }
28380 105 : tag = expand_simple_binop (Pmode, ASHIFT, tag,
28381 105 : GEN_INT (IX86_HWASAN_SHIFT), NULL_RTX,
28382 : /* unsignedp = */1, OPTAB_WIDEN);
28383 105 : rtx ret = expand_simple_binop (Pmode, IOR, untagged, tag, target,
28384 : /* unsignedp = */1, OPTAB_DIRECT);
28385 105 : return ret;
28386 : }
28387 :
28388 : /* Implement TARGET_MEMTAG_EXTRACT_TAG. */
28389 : rtx
28390 178 : ix86_memtag_extract_tag (rtx tagged_pointer, rtx target)
28391 : {
28392 178 : rtx tag = expand_simple_binop (Pmode, LSHIFTRT, tagged_pointer,
28393 178 : GEN_INT (IX86_HWASAN_SHIFT), target,
28394 : /* unsignedp = */0,
28395 : OPTAB_DIRECT);
28396 178 : rtx ret = gen_reg_rtx (QImode);
28397 : /* Mask off bit63 when LAM_U57. */
28398 178 : if (ix86_lam_type == lam_u57)
28399 : {
28400 178 : unsigned HOST_WIDE_INT and_imm
28401 : = (HOST_WIDE_INT_1U << IX86_HWASAN_TAG_SIZE) - 1;
28402 178 : emit_insn (gen_andqi3 (ret, gen_lowpart (QImode, tag),
28403 178 : gen_int_mode (and_imm, QImode)));
28404 : }
28405 : else
28406 0 : emit_move_insn (ret, gen_lowpart (QImode, tag));
28407 178 : return ret;
28408 : }
28409 :
28410 : /* The default implementation of TARGET_MEMTAG_UNTAGGED_POINTER. */
28411 : rtx
28412 113 : ix86_memtag_untagged_pointer (rtx tagged_pointer, rtx target)
28413 : {
28414 : /* Leave bit63 alone. */
28415 113 : rtx tag_mask = gen_int_mode (((HOST_WIDE_INT_1U << IX86_HWASAN_SHIFT)
28416 113 : + (HOST_WIDE_INT_1U << 63) - 1),
28417 113 : Pmode);
28418 113 : rtx untagged_base = expand_simple_binop (Pmode, AND, tagged_pointer,
28419 : tag_mask, target, true,
28420 : OPTAB_DIRECT);
28421 113 : gcc_assert (untagged_base);
28422 113 : return untagged_base;
28423 : }
28424 :
28425 : /* Implement TARGET_MEMTAG_ADD_TAG. */
28426 : rtx
28427 89 : ix86_memtag_add_tag (rtx base, poly_int64 offset, unsigned char tag_offset)
28428 : {
28429 89 : rtx base_tag = gen_reg_rtx (QImode);
28430 89 : rtx base_addr = gen_reg_rtx (Pmode);
28431 89 : rtx tagged_addr = gen_reg_rtx (Pmode);
28432 89 : rtx new_tag = gen_reg_rtx (QImode);
28433 178 : unsigned HOST_WIDE_INT and_imm
28434 89 : = (HOST_WIDE_INT_1U << IX86_HWASAN_SHIFT) - 1;
28435 :
28436 : /* When there's "overflow" in tag adding,
28437 : need to mask the most significant bit off. */
28438 89 : emit_move_insn (base_tag, ix86_memtag_extract_tag (base, NULL_RTX));
28439 89 : emit_move_insn (base_addr,
28440 : ix86_memtag_untagged_pointer (base, NULL_RTX));
28441 89 : emit_insn (gen_add2_insn (base_tag, gen_int_mode (tag_offset, QImode)));
28442 89 : emit_move_insn (new_tag, base_tag);
28443 89 : emit_insn (gen_andqi3 (new_tag, new_tag, gen_int_mode (and_imm, QImode)));
28444 89 : emit_move_insn (tagged_addr,
28445 : ix86_memtag_set_tag (base_addr, new_tag, NULL_RTX));
28446 89 : return plus_constant (Pmode, tagged_addr, offset);
28447 : }
28448 :
28449 : /* Implement TARGET_HAVE_CCMP. */
28450 : static bool
28451 8414576 : ix86_have_ccmp ()
28452 : {
28453 8414576 : return (bool) TARGET_APX_CCMP;
28454 : }
28455 :
28456 : /* Implement TARGET_MODE_CAN_TRANSFER_BITS. */
28457 : static bool
28458 4834480 : ix86_mode_can_transfer_bits (machine_mode mode)
28459 : {
28460 4834480 : if (GET_MODE_CLASS (mode) == MODE_FLOAT
28461 4783070 : || GET_MODE_CLASS (mode) == MODE_COMPLEX_FLOAT)
28462 124474 : switch (GET_MODE_INNER (mode))
28463 : {
28464 56558 : case E_SFmode:
28465 56558 : case E_DFmode:
28466 : /* These suffer from normalization upon load when not using SSE. */
28467 56558 : return !(ix86_fpmath & FPMATH_387);
28468 : default:
28469 : return true;
28470 : }
28471 :
28472 : return true;
28473 : }
28474 :
28475 : /* Implement TARGET_REDZONE_CLOBBER. */
28476 : static rtx
28477 2 : ix86_redzone_clobber ()
28478 : {
28479 2 : cfun->machine->asm_redzone_clobber_seen = true;
28480 2 : if (ix86_using_red_zone ())
28481 : {
28482 2 : rtx base = plus_constant (Pmode, stack_pointer_rtx, -RED_ZONE_SIZE);
28483 2 : rtx mem = gen_rtx_MEM (BLKmode, base);
28484 2 : set_mem_size (mem, RED_ZONE_SIZE);
28485 2 : return mem;
28486 : }
28487 : return NULL_RTX;
28488 : }
28489 :
28490 : /* Target-specific selftests. */
28491 :
28492 : #if CHECKING_P
28493 :
28494 : namespace selftest {
28495 :
28496 : /* Verify that hard regs are dumped as expected (in compact mode). */
28497 :
28498 : static void
28499 4 : ix86_test_dumping_hard_regs ()
28500 : {
28501 4 : ASSERT_RTL_DUMP_EQ ("(reg:SI ax)", gen_raw_REG (SImode, 0));
28502 4 : ASSERT_RTL_DUMP_EQ ("(reg:SI dx)", gen_raw_REG (SImode, 1));
28503 4 : }
28504 :
28505 : /* Test dumping an insn with repeated references to the same SCRATCH,
28506 : to verify the rtx_reuse code. */
28507 :
28508 : static void
28509 4 : ix86_test_dumping_memory_blockage ()
28510 : {
28511 4 : set_new_first_and_last_insn (NULL, NULL);
28512 :
28513 4 : rtx pat = gen_memory_blockage ();
28514 4 : rtx_reuse_manager r;
28515 4 : r.preprocess (pat);
28516 :
28517 : /* Verify that the repeated references to the SCRATCH show use
28518 : reuse IDS. The first should be prefixed with a reuse ID,
28519 : and the second should be dumped as a "reuse_rtx" of that ID.
28520 : The expected string assumes Pmode == DImode. */
28521 4 : if (Pmode == DImode)
28522 4 : ASSERT_RTL_DUMP_EQ_WITH_REUSE
28523 : ("(cinsn 1 (set (mem/v:BLK (0|scratch:DI) [0 A8])\n"
28524 : " (unspec:BLK [\n"
28525 : " (mem/v:BLK (reuse_rtx 0) [0 A8])\n"
28526 : " ] UNSPEC_MEMORY_BLOCKAGE)))\n", pat, &r);
28527 4 : }
28528 :
28529 : /* Verify loading an RTL dump; specifically a dump of copying
28530 : a param on x86_64 from a hard reg into the frame.
28531 : This test is target-specific since the dump contains target-specific
28532 : hard reg names. */
28533 :
28534 : static void
28535 4 : ix86_test_loading_dump_fragment_1 ()
28536 : {
28537 4 : rtl_dump_test t (SELFTEST_LOCATION,
28538 4 : locate_file ("x86_64/copy-hard-reg-into-frame.rtl"));
28539 :
28540 4 : rtx_insn *insn = get_insn_by_uid (1);
28541 :
28542 : /* The block structure and indentation here is purely for
28543 : readability; it mirrors the structure of the rtx. */
28544 4 : tree mem_expr;
28545 4 : {
28546 4 : rtx pat = PATTERN (insn);
28547 4 : ASSERT_EQ (SET, GET_CODE (pat));
28548 4 : {
28549 4 : rtx dest = SET_DEST (pat);
28550 4 : ASSERT_EQ (MEM, GET_CODE (dest));
28551 : /* Verify the "/c" was parsed. */
28552 4 : ASSERT_TRUE (RTX_FLAG (dest, call));
28553 4 : ASSERT_EQ (SImode, GET_MODE (dest));
28554 4 : {
28555 4 : rtx addr = XEXP (dest, 0);
28556 4 : ASSERT_EQ (PLUS, GET_CODE (addr));
28557 4 : ASSERT_EQ (DImode, GET_MODE (addr));
28558 4 : {
28559 4 : rtx lhs = XEXP (addr, 0);
28560 : /* Verify that the "frame" REG was consolidated. */
28561 4 : ASSERT_RTX_PTR_EQ (frame_pointer_rtx, lhs);
28562 : }
28563 4 : {
28564 4 : rtx rhs = XEXP (addr, 1);
28565 4 : ASSERT_EQ (CONST_INT, GET_CODE (rhs));
28566 4 : ASSERT_EQ (-4, INTVAL (rhs));
28567 : }
28568 : }
28569 : /* Verify the "[1 i+0 S4 A32]" was parsed. */
28570 4 : ASSERT_EQ (1, MEM_ALIAS_SET (dest));
28571 : /* "i" should have been handled by synthesizing a global int
28572 : variable named "i". */
28573 4 : mem_expr = MEM_EXPR (dest);
28574 4 : ASSERT_NE (mem_expr, NULL);
28575 4 : ASSERT_EQ (VAR_DECL, TREE_CODE (mem_expr));
28576 4 : ASSERT_EQ (integer_type_node, TREE_TYPE (mem_expr));
28577 4 : ASSERT_EQ (IDENTIFIER_NODE, TREE_CODE (DECL_NAME (mem_expr)));
28578 4 : ASSERT_STREQ ("i", IDENTIFIER_POINTER (DECL_NAME (mem_expr)));
28579 : /* "+0". */
28580 4 : ASSERT_TRUE (MEM_OFFSET_KNOWN_P (dest));
28581 4 : ASSERT_EQ (0, MEM_OFFSET (dest));
28582 : /* "S4". */
28583 4 : ASSERT_EQ (4, MEM_SIZE (dest));
28584 : /* "A32. */
28585 4 : ASSERT_EQ (32, MEM_ALIGN (dest));
28586 : }
28587 4 : {
28588 4 : rtx src = SET_SRC (pat);
28589 4 : ASSERT_EQ (REG, GET_CODE (src));
28590 4 : ASSERT_EQ (SImode, GET_MODE (src));
28591 4 : ASSERT_EQ (5, REGNO (src));
28592 4 : tree reg_expr = REG_EXPR (src);
28593 : /* "i" here should point to the same var as for the MEM_EXPR. */
28594 4 : ASSERT_EQ (reg_expr, mem_expr);
28595 : }
28596 : }
28597 4 : }
28598 :
28599 : /* Verify that the RTL loader copes with a call_insn dump.
28600 : This test is target-specific since the dump contains a target-specific
28601 : hard reg name. */
28602 :
28603 : static void
28604 4 : ix86_test_loading_call_insn ()
28605 : {
28606 : /* The test dump includes register "xmm0", where requires TARGET_SSE
28607 : to exist. */
28608 4 : if (!TARGET_SSE)
28609 0 : return;
28610 :
28611 4 : rtl_dump_test t (SELFTEST_LOCATION, locate_file ("x86_64/call-insn.rtl"));
28612 :
28613 4 : rtx_insn *insn = get_insns ();
28614 4 : ASSERT_EQ (CALL_INSN, GET_CODE (insn));
28615 :
28616 : /* "/j". */
28617 4 : ASSERT_TRUE (RTX_FLAG (insn, jump));
28618 :
28619 4 : rtx pat = PATTERN (insn);
28620 4 : ASSERT_EQ (CALL, GET_CODE (SET_SRC (pat)));
28621 :
28622 : /* Verify REG_NOTES. */
28623 4 : {
28624 : /* "(expr_list:REG_CALL_DECL". */
28625 4 : ASSERT_EQ (EXPR_LIST, GET_CODE (REG_NOTES (insn)));
28626 4 : rtx_expr_list *note0 = as_a <rtx_expr_list *> (REG_NOTES (insn));
28627 4 : ASSERT_EQ (REG_CALL_DECL, REG_NOTE_KIND (note0));
28628 :
28629 : /* "(expr_list:REG_EH_REGION (const_int 0 [0])". */
28630 4 : rtx_expr_list *note1 = note0->next ();
28631 4 : ASSERT_EQ (REG_EH_REGION, REG_NOTE_KIND (note1));
28632 :
28633 4 : ASSERT_EQ (NULL, note1->next ());
28634 : }
28635 :
28636 : /* Verify CALL_INSN_FUNCTION_USAGE. */
28637 4 : {
28638 : /* "(expr_list:DF (use (reg:DF 21 xmm0))". */
28639 4 : rtx_expr_list *usage
28640 4 : = as_a <rtx_expr_list *> (CALL_INSN_FUNCTION_USAGE (insn));
28641 4 : ASSERT_EQ (EXPR_LIST, GET_CODE (usage));
28642 4 : ASSERT_EQ (DFmode, GET_MODE (usage));
28643 4 : ASSERT_EQ (USE, GET_CODE (usage->element ()));
28644 4 : ASSERT_EQ (NULL, usage->next ());
28645 : }
28646 4 : }
28647 :
28648 : /* Verify that the RTL loader copes a dump from print_rtx_function.
28649 : This test is target-specific since the dump contains target-specific
28650 : hard reg names. */
28651 :
28652 : static void
28653 4 : ix86_test_loading_full_dump ()
28654 : {
28655 4 : rtl_dump_test t (SELFTEST_LOCATION, locate_file ("x86_64/times-two.rtl"));
28656 :
28657 4 : ASSERT_STREQ ("times_two", IDENTIFIER_POINTER (DECL_NAME (cfun->decl)));
28658 :
28659 4 : rtx_insn *insn_1 = get_insn_by_uid (1);
28660 4 : ASSERT_EQ (NOTE, GET_CODE (insn_1));
28661 :
28662 4 : rtx_insn *insn_7 = get_insn_by_uid (7);
28663 4 : ASSERT_EQ (INSN, GET_CODE (insn_7));
28664 4 : ASSERT_EQ (PARALLEL, GET_CODE (PATTERN (insn_7)));
28665 :
28666 4 : rtx_insn *insn_15 = get_insn_by_uid (15);
28667 4 : ASSERT_EQ (INSN, GET_CODE (insn_15));
28668 4 : ASSERT_EQ (USE, GET_CODE (PATTERN (insn_15)));
28669 :
28670 : /* Verify crtl->return_rtx. */
28671 4 : ASSERT_EQ (REG, GET_CODE (crtl->return_rtx));
28672 4 : ASSERT_EQ (0, REGNO (crtl->return_rtx));
28673 4 : ASSERT_EQ (SImode, GET_MODE (crtl->return_rtx));
28674 4 : }
28675 :
28676 : /* Verify that the RTL loader copes with UNSPEC and UNSPEC_VOLATILE insns.
28677 : In particular, verify that it correctly loads the 2nd operand.
28678 : This test is target-specific since these are machine-specific
28679 : operands (and enums). */
28680 :
28681 : static void
28682 4 : ix86_test_loading_unspec ()
28683 : {
28684 4 : rtl_dump_test t (SELFTEST_LOCATION, locate_file ("x86_64/unspec.rtl"));
28685 :
28686 4 : ASSERT_STREQ ("test_unspec", IDENTIFIER_POINTER (DECL_NAME (cfun->decl)));
28687 :
28688 4 : ASSERT_TRUE (cfun);
28689 :
28690 : /* Test of an UNSPEC. */
28691 4 : rtx_insn *insn = get_insns ();
28692 4 : ASSERT_EQ (INSN, GET_CODE (insn));
28693 4 : rtx set = single_set (insn);
28694 4 : ASSERT_NE (NULL, set);
28695 4 : rtx dst = SET_DEST (set);
28696 4 : ASSERT_EQ (MEM, GET_CODE (dst));
28697 4 : rtx src = SET_SRC (set);
28698 4 : ASSERT_EQ (UNSPEC, GET_CODE (src));
28699 4 : ASSERT_EQ (BLKmode, GET_MODE (src));
28700 4 : ASSERT_EQ (UNSPEC_MEMORY_BLOCKAGE, XINT (src, 1));
28701 :
28702 4 : rtx v0 = XVECEXP (src, 0, 0);
28703 :
28704 : /* Verify that the two uses of the first SCRATCH have pointer
28705 : equality. */
28706 4 : rtx scratch_a = XEXP (dst, 0);
28707 4 : ASSERT_EQ (SCRATCH, GET_CODE (scratch_a));
28708 :
28709 4 : rtx scratch_b = XEXP (v0, 0);
28710 4 : ASSERT_EQ (SCRATCH, GET_CODE (scratch_b));
28711 :
28712 4 : ASSERT_EQ (scratch_a, scratch_b);
28713 :
28714 : /* Verify that the two mems are thus treated as equal. */
28715 4 : ASSERT_TRUE (rtx_equal_p (dst, v0));
28716 :
28717 : /* Verify that the insn is recognized. */
28718 4 : ASSERT_NE(-1, recog_memoized (insn));
28719 :
28720 : /* Test of an UNSPEC_VOLATILE, which has its own enum values. */
28721 4 : insn = NEXT_INSN (insn);
28722 4 : ASSERT_EQ (INSN, GET_CODE (insn));
28723 :
28724 4 : set = single_set (insn);
28725 4 : ASSERT_NE (NULL, set);
28726 :
28727 4 : src = SET_SRC (set);
28728 4 : ASSERT_EQ (UNSPEC_VOLATILE, GET_CODE (src));
28729 4 : ASSERT_EQ (UNSPECV_RDTSCP, XINT (src, 1));
28730 4 : }
28731 :
28732 : /* Run all target-specific selftests. */
28733 :
28734 : static void
28735 4 : ix86_run_selftests (void)
28736 : {
28737 4 : ix86_test_dumping_hard_regs ();
28738 4 : ix86_test_dumping_memory_blockage ();
28739 :
28740 : /* Various tests of loading RTL dumps, here because they contain
28741 : ix86-isms (e.g. names of hard regs). */
28742 4 : ix86_test_loading_dump_fragment_1 ();
28743 4 : ix86_test_loading_call_insn ();
28744 4 : ix86_test_loading_full_dump ();
28745 4 : ix86_test_loading_unspec ();
28746 4 : }
28747 :
28748 : } // namespace selftest
28749 :
28750 : #endif /* CHECKING_P */
28751 :
28752 : static const scoped_attribute_specs *const ix86_attribute_table[] =
28753 : {
28754 : &ix86_gnu_attribute_table
28755 : };
28756 :
28757 : /* Initialize the GCC target structure. */
28758 : #undef TARGET_RETURN_IN_MEMORY
28759 : #define TARGET_RETURN_IN_MEMORY ix86_return_in_memory
28760 :
28761 : #undef TARGET_LEGITIMIZE_ADDRESS
28762 : #define TARGET_LEGITIMIZE_ADDRESS ix86_legitimize_address
28763 :
28764 : #undef TARGET_ATTRIBUTE_TABLE
28765 : #define TARGET_ATTRIBUTE_TABLE ix86_attribute_table
28766 : #undef TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P
28767 : #define TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P hook_bool_const_tree_true
28768 : #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
28769 : # undef TARGET_MERGE_DECL_ATTRIBUTES
28770 : # define TARGET_MERGE_DECL_ATTRIBUTES merge_dllimport_decl_attributes
28771 : #endif
28772 :
28773 : #undef TARGET_INVALID_CONVERSION
28774 : #define TARGET_INVALID_CONVERSION ix86_invalid_conversion
28775 :
28776 : #undef TARGET_INVALID_UNARY_OP
28777 : #define TARGET_INVALID_UNARY_OP ix86_invalid_unary_op
28778 :
28779 : #undef TARGET_INVALID_BINARY_OP
28780 : #define TARGET_INVALID_BINARY_OP ix86_invalid_binary_op
28781 :
28782 : #undef TARGET_COMP_TYPE_ATTRIBUTES
28783 : #define TARGET_COMP_TYPE_ATTRIBUTES ix86_comp_type_attributes
28784 :
28785 : #undef TARGET_INIT_BUILTINS
28786 : #define TARGET_INIT_BUILTINS ix86_init_builtins
28787 : #undef TARGET_BUILTIN_DECL
28788 : #define TARGET_BUILTIN_DECL ix86_builtin_decl
28789 : #undef TARGET_EXPAND_BUILTIN
28790 : #define TARGET_EXPAND_BUILTIN ix86_expand_builtin
28791 :
28792 : #undef TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION
28793 : #define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
28794 : ix86_builtin_vectorized_function
28795 :
28796 : #undef TARGET_VECTORIZE_BUILTIN_GATHER
28797 : #define TARGET_VECTORIZE_BUILTIN_GATHER ix86_vectorize_builtin_gather
28798 :
28799 : #undef TARGET_VECTORIZE_BUILTIN_SCATTER
28800 : #define TARGET_VECTORIZE_BUILTIN_SCATTER ix86_vectorize_builtin_scatter
28801 :
28802 : #undef TARGET_BUILTIN_RECIPROCAL
28803 : #define TARGET_BUILTIN_RECIPROCAL ix86_builtin_reciprocal
28804 :
28805 : #undef TARGET_ASM_FUNCTION_EPILOGUE
28806 : #define TARGET_ASM_FUNCTION_EPILOGUE ix86_output_function_epilogue
28807 :
28808 : #undef TARGET_ASM_PRINT_PATCHABLE_FUNCTION_ENTRY
28809 : #define TARGET_ASM_PRINT_PATCHABLE_FUNCTION_ENTRY \
28810 : ix86_print_patchable_function_entry
28811 :
28812 : #undef TARGET_ENCODE_SECTION_INFO
28813 : #ifndef SUBTARGET_ENCODE_SECTION_INFO
28814 : #define TARGET_ENCODE_SECTION_INFO ix86_encode_section_info
28815 : #else
28816 : #define TARGET_ENCODE_SECTION_INFO SUBTARGET_ENCODE_SECTION_INFO
28817 : #endif
28818 :
28819 : #undef TARGET_ASM_OPEN_PAREN
28820 : #define TARGET_ASM_OPEN_PAREN ""
28821 : #undef TARGET_ASM_CLOSE_PAREN
28822 : #define TARGET_ASM_CLOSE_PAREN ""
28823 :
28824 : #undef TARGET_ASM_BYTE_OP
28825 : #define TARGET_ASM_BYTE_OP ASM_BYTE
28826 :
28827 : #undef TARGET_ASM_ALIGNED_HI_OP
28828 : #define TARGET_ASM_ALIGNED_HI_OP ASM_SHORT
28829 : #undef TARGET_ASM_ALIGNED_SI_OP
28830 : #define TARGET_ASM_ALIGNED_SI_OP ASM_LONG
28831 : #ifdef ASM_QUAD
28832 : #undef TARGET_ASM_ALIGNED_DI_OP
28833 : #define TARGET_ASM_ALIGNED_DI_OP ASM_QUAD
28834 : #endif
28835 :
28836 : #undef TARGET_PROFILE_BEFORE_PROLOGUE
28837 : #define TARGET_PROFILE_BEFORE_PROLOGUE ix86_profile_before_prologue
28838 :
28839 : #undef TARGET_MANGLE_DECL_ASSEMBLER_NAME
28840 : #define TARGET_MANGLE_DECL_ASSEMBLER_NAME ix86_mangle_decl_assembler_name
28841 :
28842 : #undef TARGET_ASM_UNALIGNED_HI_OP
28843 : #define TARGET_ASM_UNALIGNED_HI_OP TARGET_ASM_ALIGNED_HI_OP
28844 : #undef TARGET_ASM_UNALIGNED_SI_OP
28845 : #define TARGET_ASM_UNALIGNED_SI_OP TARGET_ASM_ALIGNED_SI_OP
28846 : #undef TARGET_ASM_UNALIGNED_DI_OP
28847 : #define TARGET_ASM_UNALIGNED_DI_OP TARGET_ASM_ALIGNED_DI_OP
28848 :
28849 : #undef TARGET_PRINT_OPERAND
28850 : #define TARGET_PRINT_OPERAND ix86_print_operand
28851 : #undef TARGET_PRINT_OPERAND_ADDRESS
28852 : #define TARGET_PRINT_OPERAND_ADDRESS ix86_print_operand_address
28853 : #undef TARGET_PRINT_OPERAND_PUNCT_VALID_P
28854 : #define TARGET_PRINT_OPERAND_PUNCT_VALID_P ix86_print_operand_punct_valid_p
28855 : #undef TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA
28856 : #define TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA i386_asm_output_addr_const_extra
28857 :
28858 : #undef TARGET_SCHED_INIT_GLOBAL
28859 : #define TARGET_SCHED_INIT_GLOBAL ix86_sched_init_global
28860 : #undef TARGET_SCHED_ADJUST_COST
28861 : #define TARGET_SCHED_ADJUST_COST ix86_adjust_cost
28862 : #undef TARGET_SCHED_ISSUE_RATE
28863 : #define TARGET_SCHED_ISSUE_RATE ix86_issue_rate
28864 : #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
28865 : #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
28866 : ia32_multipass_dfa_lookahead
28867 : #undef TARGET_SCHED_MACRO_FUSION_P
28868 : #define TARGET_SCHED_MACRO_FUSION_P ix86_macro_fusion_p
28869 : #undef TARGET_SCHED_MACRO_FUSION_PAIR_P
28870 : #define TARGET_SCHED_MACRO_FUSION_PAIR_P ix86_macro_fusion_pair_p
28871 :
28872 : #undef TARGET_FUNCTION_OK_FOR_SIBCALL
28873 : #define TARGET_FUNCTION_OK_FOR_SIBCALL ix86_function_ok_for_sibcall
28874 :
28875 : #undef TARGET_MEMMODEL_CHECK
28876 : #define TARGET_MEMMODEL_CHECK ix86_memmodel_check
28877 :
28878 : #undef TARGET_ATOMIC_ASSIGN_EXPAND_FENV
28879 : #define TARGET_ATOMIC_ASSIGN_EXPAND_FENV ix86_atomic_assign_expand_fenv
28880 :
28881 : #ifdef HAVE_AS_TLS
28882 : #undef TARGET_HAVE_TLS
28883 : #define TARGET_HAVE_TLS true
28884 : #endif
28885 : #undef TARGET_CANNOT_FORCE_CONST_MEM
28886 : #define TARGET_CANNOT_FORCE_CONST_MEM ix86_cannot_force_const_mem
28887 : #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
28888 : #define TARGET_USE_BLOCKS_FOR_CONSTANT_P hook_bool_mode_const_rtx_true
28889 :
28890 : #undef TARGET_DELEGITIMIZE_ADDRESS
28891 : #define TARGET_DELEGITIMIZE_ADDRESS ix86_delegitimize_address
28892 :
28893 : #undef TARGET_CONST_NOT_OK_FOR_DEBUG_P
28894 : #define TARGET_CONST_NOT_OK_FOR_DEBUG_P ix86_const_not_ok_for_debug_p
28895 :
28896 : #undef TARGET_MS_BITFIELD_LAYOUT_P
28897 : #define TARGET_MS_BITFIELD_LAYOUT_P ix86_ms_bitfield_layout_p
28898 :
28899 : #if TARGET_MACHO
28900 : #undef TARGET_BINDS_LOCAL_P
28901 : #define TARGET_BINDS_LOCAL_P darwin_binds_local_p
28902 : #else
28903 : #undef TARGET_BINDS_LOCAL_P
28904 : #define TARGET_BINDS_LOCAL_P ix86_binds_local_p
28905 : #endif
28906 : #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
28907 : #undef TARGET_BINDS_LOCAL_P
28908 : #define TARGET_BINDS_LOCAL_P i386_pe_binds_local_p
28909 : #endif
28910 :
28911 : #undef TARGET_ASM_OUTPUT_MI_THUNK
28912 : #define TARGET_ASM_OUTPUT_MI_THUNK x86_output_mi_thunk
28913 : #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
28914 : #define TARGET_ASM_CAN_OUTPUT_MI_THUNK x86_can_output_mi_thunk
28915 :
28916 : #undef TARGET_ASM_FILE_START
28917 : #define TARGET_ASM_FILE_START x86_file_start
28918 :
28919 : #undef TARGET_OPTION_OVERRIDE
28920 : #define TARGET_OPTION_OVERRIDE ix86_option_override
28921 :
28922 : #undef TARGET_REGISTER_MOVE_COST
28923 : #define TARGET_REGISTER_MOVE_COST ix86_register_move_cost
28924 : #undef TARGET_MEMORY_MOVE_COST
28925 : #define TARGET_MEMORY_MOVE_COST ix86_memory_move_cost
28926 : #undef TARGET_RTX_COSTS
28927 : #define TARGET_RTX_COSTS ix86_rtx_costs
28928 : #undef TARGET_INSN_COST
28929 : #define TARGET_INSN_COST ix86_insn_cost
28930 : #undef TARGET_ADDRESS_COST
28931 : #define TARGET_ADDRESS_COST ix86_address_cost
28932 :
28933 : #undef TARGET_USE_BY_PIECES_INFRASTRUCTURE_P
28934 : #define TARGET_USE_BY_PIECES_INFRASTRUCTURE_P \
28935 : ix86_use_by_pieces_infrastructure_p
28936 :
28937 : #undef TARGET_OVERLAP_OP_BY_PIECES_P
28938 : #define TARGET_OVERLAP_OP_BY_PIECES_P hook_bool_void_true
28939 :
28940 : #undef TARGET_FLAGS_REGNUM
28941 : #define TARGET_FLAGS_REGNUM FLAGS_REG
28942 : #undef TARGET_FIXED_CONDITION_CODE_REGS
28943 : #define TARGET_FIXED_CONDITION_CODE_REGS ix86_fixed_condition_code_regs
28944 : #undef TARGET_CC_MODES_COMPATIBLE
28945 : #define TARGET_CC_MODES_COMPATIBLE ix86_cc_modes_compatible
28946 :
28947 : #undef TARGET_MACHINE_DEPENDENT_REORG
28948 : #define TARGET_MACHINE_DEPENDENT_REORG ix86_reorg
28949 :
28950 : #undef TARGET_BUILD_BUILTIN_VA_LIST
28951 : #define TARGET_BUILD_BUILTIN_VA_LIST ix86_build_builtin_va_list
28952 :
28953 : #undef TARGET_FOLD_BUILTIN
28954 : #define TARGET_FOLD_BUILTIN ix86_fold_builtin
28955 :
28956 : #undef TARGET_GIMPLE_FOLD_BUILTIN
28957 : #define TARGET_GIMPLE_FOLD_BUILTIN ix86_gimple_fold_builtin
28958 :
28959 : #undef TARGET_COMPARE_VERSION_PRIORITY
28960 : #define TARGET_COMPARE_VERSION_PRIORITY ix86_compare_version_priority
28961 :
28962 : #undef TARGET_GENERATE_VERSION_DISPATCHER_BODY
28963 : #define TARGET_GENERATE_VERSION_DISPATCHER_BODY \
28964 : ix86_generate_version_dispatcher_body
28965 :
28966 : #undef TARGET_GET_FUNCTION_VERSIONS_DISPATCHER
28967 : #define TARGET_GET_FUNCTION_VERSIONS_DISPATCHER \
28968 : ix86_get_function_versions_dispatcher
28969 :
28970 : #undef TARGET_ENUM_VA_LIST_P
28971 : #define TARGET_ENUM_VA_LIST_P ix86_enum_va_list
28972 :
28973 : #undef TARGET_FN_ABI_VA_LIST
28974 : #define TARGET_FN_ABI_VA_LIST ix86_fn_abi_va_list
28975 :
28976 : #undef TARGET_CANONICAL_VA_LIST_TYPE
28977 : #define TARGET_CANONICAL_VA_LIST_TYPE ix86_canonical_va_list_type
28978 :
28979 : #undef TARGET_EXPAND_BUILTIN_VA_START
28980 : #define TARGET_EXPAND_BUILTIN_VA_START ix86_va_start
28981 :
28982 : #undef TARGET_MD_ASM_ADJUST
28983 : #define TARGET_MD_ASM_ADJUST ix86_md_asm_adjust
28984 :
28985 : #undef TARGET_C_EXCESS_PRECISION
28986 : #define TARGET_C_EXCESS_PRECISION ix86_get_excess_precision
28987 : #undef TARGET_C_BITINT_TYPE_INFO
28988 : #define TARGET_C_BITINT_TYPE_INFO ix86_bitint_type_info
28989 : #undef TARGET_C_MODE_FOR_FLOATING_TYPE
28990 : #define TARGET_C_MODE_FOR_FLOATING_TYPE ix86_c_mode_for_floating_type
28991 : #undef TARGET_CXX_ADJUST_CDTOR_CALLABI_FNTYPE
28992 : #define TARGET_CXX_ADJUST_CDTOR_CALLABI_FNTYPE ix86_cxx_adjust_cdtor_callabi_fntype
28993 : #undef TARGET_PROMOTE_PROTOTYPES
28994 : #define TARGET_PROMOTE_PROTOTYPES hook_bool_const_tree_true
28995 : #undef TARGET_PUSH_ARGUMENT
28996 : #define TARGET_PUSH_ARGUMENT ix86_push_argument
28997 : #undef TARGET_SETUP_INCOMING_VARARGS
28998 : #define TARGET_SETUP_INCOMING_VARARGS ix86_setup_incoming_varargs
28999 : #undef TARGET_MUST_PASS_IN_STACK
29000 : #define TARGET_MUST_PASS_IN_STACK ix86_must_pass_in_stack
29001 : #undef TARGET_ALLOCATE_STACK_SLOTS_FOR_ARGS
29002 : #define TARGET_ALLOCATE_STACK_SLOTS_FOR_ARGS ix86_allocate_stack_slots_for_args
29003 : #undef TARGET_FUNCTION_ARG_ADVANCE
29004 : #define TARGET_FUNCTION_ARG_ADVANCE ix86_function_arg_advance
29005 : #undef TARGET_FUNCTION_ARG
29006 : #define TARGET_FUNCTION_ARG ix86_function_arg
29007 : #undef TARGET_INIT_PIC_REG
29008 : #define TARGET_INIT_PIC_REG ix86_init_pic_reg
29009 : #undef TARGET_USE_PSEUDO_PIC_REG
29010 : #define TARGET_USE_PSEUDO_PIC_REG ix86_use_pseudo_pic_reg
29011 : #undef TARGET_FUNCTION_ARG_BOUNDARY
29012 : #define TARGET_FUNCTION_ARG_BOUNDARY ix86_function_arg_boundary
29013 : #undef TARGET_PASS_BY_REFERENCE
29014 : #define TARGET_PASS_BY_REFERENCE ix86_pass_by_reference
29015 : #undef TARGET_INTERNAL_ARG_POINTER
29016 : #define TARGET_INTERNAL_ARG_POINTER ix86_internal_arg_pointer
29017 : #undef TARGET_UPDATE_STACK_BOUNDARY
29018 : #define TARGET_UPDATE_STACK_BOUNDARY ix86_update_stack_boundary
29019 : #undef TARGET_GET_DRAP_RTX
29020 : #define TARGET_GET_DRAP_RTX ix86_get_drap_rtx
29021 : #undef TARGET_STRICT_ARGUMENT_NAMING
29022 : #define TARGET_STRICT_ARGUMENT_NAMING hook_bool_CUMULATIVE_ARGS_true
29023 : #undef TARGET_STATIC_CHAIN
29024 : #define TARGET_STATIC_CHAIN ix86_static_chain
29025 : #undef TARGET_TRAMPOLINE_INIT
29026 : #define TARGET_TRAMPOLINE_INIT ix86_trampoline_init
29027 : #undef TARGET_RETURN_POPS_ARGS
29028 : #define TARGET_RETURN_POPS_ARGS ix86_return_pops_args
29029 :
29030 : #undef TARGET_WARN_FUNC_RETURN
29031 : #define TARGET_WARN_FUNC_RETURN ix86_warn_func_return
29032 :
29033 : #undef TARGET_LEGITIMATE_COMBINED_INSN
29034 : #define TARGET_LEGITIMATE_COMBINED_INSN ix86_legitimate_combined_insn
29035 :
29036 : #undef TARGET_ASAN_SHADOW_OFFSET
29037 : #define TARGET_ASAN_SHADOW_OFFSET ix86_asan_shadow_offset
29038 :
29039 : #undef TARGET_GIMPLIFY_VA_ARG_EXPR
29040 : #define TARGET_GIMPLIFY_VA_ARG_EXPR ix86_gimplify_va_arg
29041 :
29042 : #undef TARGET_SCALAR_MODE_SUPPORTED_P
29043 : #define TARGET_SCALAR_MODE_SUPPORTED_P ix86_scalar_mode_supported_p
29044 :
29045 : #undef TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P
29046 : #define TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P \
29047 : ix86_libgcc_floating_mode_supported_p
29048 :
29049 : #undef TARGET_VECTOR_MODE_SUPPORTED_P
29050 : #define TARGET_VECTOR_MODE_SUPPORTED_P ix86_vector_mode_supported_p
29051 :
29052 : #undef TARGET_C_MODE_FOR_SUFFIX
29053 : #define TARGET_C_MODE_FOR_SUFFIX ix86_c_mode_for_suffix
29054 :
29055 : #ifdef HAVE_AS_TLS
29056 : #undef TARGET_ASM_OUTPUT_DWARF_DTPREL
29057 : #define TARGET_ASM_OUTPUT_DWARF_DTPREL i386_output_dwarf_dtprel
29058 : #endif
29059 :
29060 : #ifdef SUBTARGET_INSERT_ATTRIBUTES
29061 : #undef TARGET_INSERT_ATTRIBUTES
29062 : #define TARGET_INSERT_ATTRIBUTES SUBTARGET_INSERT_ATTRIBUTES
29063 : #endif
29064 :
29065 : #undef TARGET_MANGLE_TYPE
29066 : #define TARGET_MANGLE_TYPE ix86_mangle_type
29067 :
29068 : #undef TARGET_EMIT_SUPPORT_TINFOS
29069 : #define TARGET_EMIT_SUPPORT_TINFOS ix86_emit_support_tinfos
29070 :
29071 : #undef TARGET_STACK_PROTECT_GUARD
29072 : #define TARGET_STACK_PROTECT_GUARD ix86_stack_protect_guard
29073 :
29074 : #undef TARGET_STACK_PROTECT_GUARD_SYMBOL_P
29075 : #define TARGET_STACK_PROTECT_GUARD_SYMBOL_P \
29076 : ix86_stack_protect_guard_symbol_p
29077 :
29078 : #undef TARGET_STACK_PROTECT_RUNTIME_ENABLED_P
29079 : #define TARGET_STACK_PROTECT_RUNTIME_ENABLED_P \
29080 : ix86_stack_protect_runtime_enabled_p
29081 :
29082 : #if !TARGET_MACHO
29083 : #undef TARGET_STACK_PROTECT_FAIL
29084 : #define TARGET_STACK_PROTECT_FAIL ix86_stack_protect_fail
29085 : #endif
29086 :
29087 : #undef TARGET_FUNCTION_VALUE
29088 : #define TARGET_FUNCTION_VALUE ix86_function_value
29089 :
29090 : #undef TARGET_FUNCTION_VALUE_REGNO_P
29091 : #define TARGET_FUNCTION_VALUE_REGNO_P ix86_function_value_regno_p
29092 :
29093 : #undef TARGET_ZERO_CALL_USED_REGS
29094 : #define TARGET_ZERO_CALL_USED_REGS ix86_zero_call_used_regs
29095 :
29096 : #undef TARGET_PROMOTE_FUNCTION_MODE
29097 : #define TARGET_PROMOTE_FUNCTION_MODE ix86_promote_function_mode
29098 :
29099 : #undef TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE
29100 : #define TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE ix86_override_options_after_change
29101 :
29102 : #undef TARGET_MEMBER_TYPE_FORCES_BLK
29103 : #define TARGET_MEMBER_TYPE_FORCES_BLK ix86_member_type_forces_blk
29104 :
29105 : #undef TARGET_INSTANTIATE_DECLS
29106 : #define TARGET_INSTANTIATE_DECLS ix86_instantiate_decls
29107 :
29108 : #undef TARGET_SECONDARY_RELOAD
29109 : #define TARGET_SECONDARY_RELOAD ix86_secondary_reload
29110 : #undef TARGET_SECONDARY_MEMORY_NEEDED
29111 : #define TARGET_SECONDARY_MEMORY_NEEDED ix86_secondary_memory_needed
29112 : #undef TARGET_SECONDARY_MEMORY_NEEDED_MODE
29113 : #define TARGET_SECONDARY_MEMORY_NEEDED_MODE ix86_secondary_memory_needed_mode
29114 :
29115 : #undef TARGET_CLASS_MAX_NREGS
29116 : #define TARGET_CLASS_MAX_NREGS ix86_class_max_nregs
29117 :
29118 : #undef TARGET_BASE_REG_CLASS
29119 : #define TARGET_BASE_REG_CLASS ix86_base_reg_class
29120 :
29121 : #undef TARGET_PREFERRED_RELOAD_CLASS
29122 : #define TARGET_PREFERRED_RELOAD_CLASS ix86_preferred_reload_class
29123 : #undef TARGET_PREFERRED_OUTPUT_RELOAD_CLASS
29124 : #define TARGET_PREFERRED_OUTPUT_RELOAD_CLASS ix86_preferred_output_reload_class
29125 : /* When this hook returns true for MODE, the compiler allows
29126 : registers explicitly used in the rtl to be used as spill registers
29127 : but prevents the compiler from extending the lifetime of these
29128 : registers. */
29129 : #undef TARGET_SMALL_REGISTER_CLASSES_FOR_MODE_P
29130 : #define TARGET_SMALL_REGISTER_CLASSES_FOR_MODE_P hook_bool_mode_true
29131 : #undef TARGET_CLASS_LIKELY_SPILLED_P
29132 : #define TARGET_CLASS_LIKELY_SPILLED_P ix86_class_likely_spilled_p
29133 : #undef TARGET_CALLEE_SAVE_COST
29134 : #define TARGET_CALLEE_SAVE_COST ix86_callee_save_cost
29135 :
29136 : #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
29137 : #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
29138 : ix86_builtin_vectorization_cost
29139 : #undef TARGET_VECTORIZE_VEC_PERM_CONST
29140 : #define TARGET_VECTORIZE_VEC_PERM_CONST ix86_vectorize_vec_perm_const
29141 : #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
29142 : #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE \
29143 : ix86_preferred_simd_mode
29144 : #undef TARGET_VECTORIZE_SPLIT_REDUCTION
29145 : #define TARGET_VECTORIZE_SPLIT_REDUCTION \
29146 : ix86_split_reduction
29147 : #undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_MODES
29148 : #define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_MODES \
29149 : ix86_autovectorize_vector_modes
29150 : #undef TARGET_VECTORIZE_GET_MASK_MODE
29151 : #define TARGET_VECTORIZE_GET_MASK_MODE ix86_get_mask_mode
29152 : #undef TARGET_VECTORIZE_CREATE_COSTS
29153 : #define TARGET_VECTORIZE_CREATE_COSTS ix86_vectorize_create_costs
29154 :
29155 : #undef TARGET_SET_CURRENT_FUNCTION
29156 : #define TARGET_SET_CURRENT_FUNCTION ix86_set_current_function
29157 :
29158 : #undef TARGET_OPTION_VALID_ATTRIBUTE_P
29159 : #define TARGET_OPTION_VALID_ATTRIBUTE_P ix86_valid_target_attribute_p
29160 :
29161 : #undef TARGET_OPTION_SAVE
29162 : #define TARGET_OPTION_SAVE ix86_function_specific_save
29163 :
29164 : #undef TARGET_OPTION_RESTORE
29165 : #define TARGET_OPTION_RESTORE ix86_function_specific_restore
29166 :
29167 : #undef TARGET_OPTION_POST_STREAM_IN
29168 : #define TARGET_OPTION_POST_STREAM_IN ix86_function_specific_post_stream_in
29169 :
29170 : #undef TARGET_OPTION_PRINT
29171 : #define TARGET_OPTION_PRINT ix86_function_specific_print
29172 :
29173 : #undef TARGET_CAN_INLINE_P
29174 : #define TARGET_CAN_INLINE_P ix86_can_inline_p
29175 :
29176 : #undef TARGET_LEGITIMATE_ADDRESS_P
29177 : #define TARGET_LEGITIMATE_ADDRESS_P ix86_legitimate_address_p
29178 :
29179 : #undef TARGET_REGISTER_PRIORITY
29180 : #define TARGET_REGISTER_PRIORITY ix86_register_priority
29181 :
29182 : #undef TARGET_REGISTER_USAGE_LEVELING_P
29183 : #define TARGET_REGISTER_USAGE_LEVELING_P hook_bool_void_true
29184 :
29185 : #undef TARGET_LEGITIMATE_CONSTANT_P
29186 : #define TARGET_LEGITIMATE_CONSTANT_P ix86_legitimate_constant_p
29187 :
29188 : #undef TARGET_COMPUTE_FRAME_LAYOUT
29189 : #define TARGET_COMPUTE_FRAME_LAYOUT ix86_compute_frame_layout
29190 :
29191 : #undef TARGET_FRAME_POINTER_REQUIRED
29192 : #define TARGET_FRAME_POINTER_REQUIRED ix86_frame_pointer_required
29193 :
29194 : #undef TARGET_CAN_ELIMINATE
29195 : #define TARGET_CAN_ELIMINATE ix86_can_eliminate
29196 :
29197 : #undef TARGET_EXTRA_LIVE_ON_ENTRY
29198 : #define TARGET_EXTRA_LIVE_ON_ENTRY ix86_live_on_entry
29199 :
29200 : #undef TARGET_ASM_CODE_END
29201 : #define TARGET_ASM_CODE_END ix86_code_end
29202 :
29203 : #undef TARGET_CONDITIONAL_REGISTER_USAGE
29204 : #define TARGET_CONDITIONAL_REGISTER_USAGE ix86_conditional_register_usage
29205 :
29206 : #undef TARGET_CANONICALIZE_COMPARISON
29207 : #define TARGET_CANONICALIZE_COMPARISON ix86_canonicalize_comparison
29208 :
29209 : #undef TARGET_LOOP_UNROLL_ADJUST
29210 : #define TARGET_LOOP_UNROLL_ADJUST ix86_loop_unroll_adjust
29211 :
29212 : /* Disabled due to PRs 70902, 71453, 71555, 71596 and 71657. */
29213 : #undef TARGET_SPILL_CLASS
29214 : #define TARGET_SPILL_CLASS ix86_spill_class
29215 :
29216 : #undef TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN
29217 : #define TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN \
29218 : ix86_simd_clone_compute_vecsize_and_simdlen
29219 :
29220 : #undef TARGET_SIMD_CLONE_ADJUST
29221 : #define TARGET_SIMD_CLONE_ADJUST ix86_simd_clone_adjust
29222 :
29223 : #undef TARGET_SIMD_CLONE_USABLE
29224 : #define TARGET_SIMD_CLONE_USABLE ix86_simd_clone_usable
29225 :
29226 : #undef TARGET_OMP_DEVICE_KIND_ARCH_ISA
29227 : #define TARGET_OMP_DEVICE_KIND_ARCH_ISA ix86_omp_device_kind_arch_isa
29228 :
29229 : #undef TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P
29230 : #define TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P \
29231 : ix86_float_exceptions_rounding_supported_p
29232 :
29233 : #undef TARGET_MODE_EMIT
29234 : #define TARGET_MODE_EMIT ix86_emit_mode_set
29235 :
29236 : #undef TARGET_MODE_NEEDED
29237 : #define TARGET_MODE_NEEDED ix86_mode_needed
29238 :
29239 : #undef TARGET_MODE_AFTER
29240 : #define TARGET_MODE_AFTER ix86_mode_after
29241 :
29242 : #undef TARGET_MODE_ENTRY
29243 : #define TARGET_MODE_ENTRY ix86_mode_entry
29244 :
29245 : #undef TARGET_MODE_EXIT
29246 : #define TARGET_MODE_EXIT ix86_mode_exit
29247 :
29248 : #undef TARGET_MODE_PRIORITY
29249 : #define TARGET_MODE_PRIORITY ix86_mode_priority
29250 :
29251 : #undef TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS
29252 : #define TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS true
29253 :
29254 : #undef TARGET_OFFLOAD_OPTIONS
29255 : #define TARGET_OFFLOAD_OPTIONS \
29256 : ix86_offload_options
29257 :
29258 : #undef TARGET_ABSOLUTE_BIGGEST_ALIGNMENT
29259 : #define TARGET_ABSOLUTE_BIGGEST_ALIGNMENT 512
29260 :
29261 : #undef TARGET_OPTAB_SUPPORTED_P
29262 : #define TARGET_OPTAB_SUPPORTED_P ix86_optab_supported_p
29263 :
29264 : #undef TARGET_HARD_REGNO_SCRATCH_OK
29265 : #define TARGET_HARD_REGNO_SCRATCH_OK ix86_hard_regno_scratch_ok
29266 :
29267 : #undef TARGET_CUSTOM_FUNCTION_DESCRIPTORS
29268 : #define TARGET_CUSTOM_FUNCTION_DESCRIPTORS X86_CUSTOM_FUNCTION_TEST
29269 :
29270 : #undef TARGET_ADDR_SPACE_ZERO_ADDRESS_VALID
29271 : #define TARGET_ADDR_SPACE_ZERO_ADDRESS_VALID ix86_addr_space_zero_address_valid
29272 :
29273 : #undef TARGET_INIT_LIBFUNCS
29274 : #define TARGET_INIT_LIBFUNCS ix86_init_libfuncs
29275 :
29276 : #undef TARGET_EXPAND_DIVMOD_LIBFUNC
29277 : #define TARGET_EXPAND_DIVMOD_LIBFUNC ix86_expand_divmod_libfunc
29278 :
29279 : #undef TARGET_MAX_NOCE_IFCVT_SEQ_COST
29280 : #define TARGET_MAX_NOCE_IFCVT_SEQ_COST ix86_max_noce_ifcvt_seq_cost
29281 :
29282 : #undef TARGET_NOCE_CONVERSION_PROFITABLE_P
29283 : #define TARGET_NOCE_CONVERSION_PROFITABLE_P ix86_noce_conversion_profitable_p
29284 :
29285 : #undef TARGET_HARD_REGNO_NREGS
29286 : #define TARGET_HARD_REGNO_NREGS ix86_hard_regno_nregs
29287 : #undef TARGET_HARD_REGNO_MODE_OK
29288 : #define TARGET_HARD_REGNO_MODE_OK ix86_hard_regno_mode_ok
29289 :
29290 : #undef TARGET_MODES_TIEABLE_P
29291 : #define TARGET_MODES_TIEABLE_P ix86_modes_tieable_p
29292 :
29293 : #undef TARGET_HARD_REGNO_CALL_PART_CLOBBERED
29294 : #define TARGET_HARD_REGNO_CALL_PART_CLOBBERED \
29295 : ix86_hard_regno_call_part_clobbered
29296 :
29297 : #undef TARGET_FNTYPE_ABI
29298 : #define TARGET_FNTYPE_ABI ix86_fntype_abi
29299 :
29300 : #undef TARGET_CAN_CHANGE_MODE_CLASS
29301 : #define TARGET_CAN_CHANGE_MODE_CLASS ix86_can_change_mode_class
29302 :
29303 : #undef TARGET_LOWER_LOCAL_DECL_ALIGNMENT
29304 : #define TARGET_LOWER_LOCAL_DECL_ALIGNMENT ix86_lower_local_decl_alignment
29305 :
29306 : #undef TARGET_STATIC_RTX_ALIGNMENT
29307 : #define TARGET_STATIC_RTX_ALIGNMENT ix86_static_rtx_alignment
29308 : #undef TARGET_CONSTANT_ALIGNMENT
29309 : #define TARGET_CONSTANT_ALIGNMENT ix86_constant_alignment
29310 :
29311 : #undef TARGET_EMPTY_RECORD_P
29312 : #define TARGET_EMPTY_RECORD_P ix86_is_empty_record
29313 :
29314 : #undef TARGET_WARN_PARAMETER_PASSING_ABI
29315 : #define TARGET_WARN_PARAMETER_PASSING_ABI ix86_warn_parameter_passing_abi
29316 :
29317 : #undef TARGET_GET_MULTILIB_ABI_NAME
29318 : #define TARGET_GET_MULTILIB_ABI_NAME \
29319 : ix86_get_multilib_abi_name
29320 :
29321 : #undef TARGET_IFUNC_REF_LOCAL_OK
29322 : #define TARGET_IFUNC_REF_LOCAL_OK ix86_ifunc_ref_local_ok
29323 :
29324 : #if !TARGET_MACHO && !TARGET_DLLIMPORT_DECL_ATTRIBUTES
29325 : # undef TARGET_ASM_RELOC_RW_MASK
29326 : # define TARGET_ASM_RELOC_RW_MASK ix86_reloc_rw_mask
29327 : #endif
29328 :
29329 : #undef TARGET_MEMTAG_CAN_TAG_ADDRESSES
29330 : #define TARGET_MEMTAG_CAN_TAG_ADDRESSES ix86_memtag_can_tag_addresses
29331 :
29332 : #undef TARGET_MEMTAG_ADD_TAG
29333 : #define TARGET_MEMTAG_ADD_TAG ix86_memtag_add_tag
29334 :
29335 : #undef TARGET_MEMTAG_SET_TAG
29336 : #define TARGET_MEMTAG_SET_TAG ix86_memtag_set_tag
29337 :
29338 : #undef TARGET_MEMTAG_EXTRACT_TAG
29339 : #define TARGET_MEMTAG_EXTRACT_TAG ix86_memtag_extract_tag
29340 :
29341 : #undef TARGET_MEMTAG_UNTAGGED_POINTER
29342 : #define TARGET_MEMTAG_UNTAGGED_POINTER ix86_memtag_untagged_pointer
29343 :
29344 : #undef TARGET_MEMTAG_TAG_BITSIZE
29345 : #define TARGET_MEMTAG_TAG_BITSIZE ix86_memtag_tag_bitsize
29346 :
29347 : #undef TARGET_GEN_CCMP_FIRST
29348 : #define TARGET_GEN_CCMP_FIRST ix86_gen_ccmp_first
29349 :
29350 : #undef TARGET_GEN_CCMP_NEXT
29351 : #define TARGET_GEN_CCMP_NEXT ix86_gen_ccmp_next
29352 :
29353 : #undef TARGET_HAVE_CCMP
29354 : #define TARGET_HAVE_CCMP ix86_have_ccmp
29355 :
29356 : #undef TARGET_MODE_CAN_TRANSFER_BITS
29357 : #define TARGET_MODE_CAN_TRANSFER_BITS ix86_mode_can_transfer_bits
29358 :
29359 : #undef TARGET_REDZONE_CLOBBER
29360 : #define TARGET_REDZONE_CLOBBER ix86_redzone_clobber
29361 :
29362 : static bool
29363 99968 : ix86_libc_has_fast_function (int fcode ATTRIBUTE_UNUSED)
29364 : {
29365 : #ifdef OPTION_GLIBC
29366 99968 : if (OPTION_GLIBC)
29367 99968 : return (built_in_function)fcode == BUILT_IN_MEMPCPY;
29368 : else
29369 : return false;
29370 : #else
29371 : return false;
29372 : #endif
29373 : }
29374 :
29375 : #undef TARGET_LIBC_HAS_FAST_FUNCTION
29376 : #define TARGET_LIBC_HAS_FAST_FUNCTION ix86_libc_has_fast_function
29377 :
29378 : static unsigned
29379 80763 : ix86_libm_function_max_error (unsigned cfn, machine_mode mode,
29380 : bool boundary_p)
29381 : {
29382 : #ifdef OPTION_GLIBC
29383 80763 : bool glibc_p = OPTION_GLIBC;
29384 : #else
29385 : bool glibc_p = false;
29386 : #endif
29387 80763 : if (glibc_p)
29388 : {
29389 : /* If __FAST_MATH__ is defined, glibc provides libmvec. */
29390 80763 : unsigned int libmvec_ret = 0;
29391 80763 : if (!flag_trapping_math
29392 8518 : && flag_unsafe_math_optimizations
29393 3552 : && flag_finite_math_only
29394 3340 : && !flag_signed_zeros
29395 3340 : && !flag_errno_math)
29396 3340 : switch (cfn)
29397 : {
29398 1388 : CASE_CFN_COS:
29399 1388 : CASE_CFN_COS_FN:
29400 1388 : CASE_CFN_SIN:
29401 1388 : CASE_CFN_SIN_FN:
29402 1388 : if (!boundary_p)
29403 : {
29404 : /* With non-default rounding modes, libmvec provides
29405 : complete garbage in results. E.g.
29406 : _ZGVcN8v_sinf for 1.40129846e-45f in FE_UPWARD
29407 : returns 0.00333309174f rather than 1.40129846e-45f. */
29408 583 : if (flag_rounding_math)
29409 : return ~0U;
29410 : /* https://www.gnu.org/software/libc/manual/html_node/Errors-in-Math-Functions.html
29411 : claims libmvec maximum error is 4ulps.
29412 : My own random testing indicates 2ulps for SFmode and
29413 : 0.5ulps for DFmode, but let's go with the 4ulps. */
29414 : libmvec_ret = 4;
29415 : }
29416 : break;
29417 : default:
29418 : break;
29419 : }
29420 80763 : unsigned int ret = glibc_linux_libm_function_max_error (cfn, mode,
29421 : boundary_p);
29422 80763 : return MAX (ret, libmvec_ret);
29423 : }
29424 0 : return default_libm_function_max_error (cfn, mode, boundary_p);
29425 : }
29426 :
29427 : #undef TARGET_LIBM_FUNCTION_MAX_ERROR
29428 : #define TARGET_LIBM_FUNCTION_MAX_ERROR ix86_libm_function_max_error
29429 :
29430 : #if TARGET_MACHO
29431 : static bool
29432 : ix86_cannot_copy_insn_p (rtx_insn *insn)
29433 : {
29434 : if (TARGET_64BIT)
29435 : return false;
29436 :
29437 : rtx set = single_set (insn);
29438 : if (set)
29439 : {
29440 : rtx src = SET_SRC (set);
29441 : if (GET_CODE (src) == UNSPEC
29442 : && XINT (src, 1) == UNSPEC_SET_GOT)
29443 : return true;
29444 : }
29445 : return false;
29446 : }
29447 :
29448 : #undef TARGET_CANNOT_COPY_INSN_P
29449 : #define TARGET_CANNOT_COPY_INSN_P ix86_cannot_copy_insn_p
29450 :
29451 : #endif
29452 :
29453 : #if CHECKING_P
29454 : #undef TARGET_RUN_TARGET_SELFTESTS
29455 : #define TARGET_RUN_TARGET_SELFTESTS selftest::ix86_run_selftests
29456 : #endif /* #if CHECKING_P */
29457 :
29458 : #undef TARGET_DOCUMENTATION_NAME
29459 : #define TARGET_DOCUMENTATION_NAME "x86"
29460 :
29461 : /* Implement TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS. */
29462 : sbitmap
29463 759033 : ix86_get_separate_components (void)
29464 : {
29465 759033 : HOST_WIDE_INT offset, to_allocate;
29466 759033 : sbitmap components = sbitmap_alloc (FIRST_PSEUDO_REGISTER);
29467 759033 : bitmap_clear (components);
29468 759033 : struct machine_function *m = cfun->machine;
29469 :
29470 759033 : offset = m->frame.stack_pointer_offset;
29471 759033 : to_allocate = offset - m->frame.sse_reg_save_offset;
29472 :
29473 : /* Shrink wrap separate uses MOV, which means APX PPX cannot be used.
29474 : Experiments show that APX PPX can speed up the prologue. If the function
29475 : does not exit early during actual execution, then using APX PPX is faster.
29476 : If the function always exits early during actual execution, then shrink
29477 : wrap separate reduces the number of MOV (PUSH/POP) instructions actually
29478 : executed, thus speeding up execution.
29479 : foo:
29480 : movl $1, %eax
29481 : testq %rdi, %rdi
29482 : jne.L60
29483 : ret ---> early return.
29484 : .L60:
29485 : subq $88, %rsp ---> belong to prologue.
29486 : xorl %eax, %eax
29487 : movq %rbx, 40 (%rsp) ---> belong to prologue.
29488 : movq 8 (%rdi), %rbx
29489 : movq %rbp, 48 (%rsp) ---> belong to prologue.
29490 : movq %rdi, %rbp
29491 : testq %rbx, %rbx
29492 : jne.L61
29493 : movq 40 (%rsp), %rbx
29494 : movq 48 (%rsp), %rbp
29495 : addq $88, %rsp
29496 : ret
29497 : .L61:
29498 : movq %r12, 56 (%rsp) ---> belong to prologue.
29499 : movq %r13, 64 (%rsp) ---> belong to prologue.
29500 : movq %r14, 72 (%rsp) ---> belong to prologue.
29501 : ... ...
29502 :
29503 : Disable shrink wrap separate when PPX is enabled. */
29504 759033 : if ((TARGET_APX_PPX && !crtl->calls_eh_return)
29505 758554 : || cfun->machine->func_type != TYPE_NORMAL
29506 : || TARGET_SEH
29507 758456 : || crtl->stack_realign_needed
29508 749263 : || m->call_ms2sysv)
29509 : return components;
29510 :
29511 : /* Since shrink wrapping separate uses MOV instead of PUSH/POP.
29512 : Disable shrink wrap separate when MOV is prohibited. */
29513 747341 : if (save_regs_using_push_pop (to_allocate))
29514 : return components;
29515 :
29516 34377270 : for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
29517 34015404 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true, true))
29518 : {
29519 : /* Skip registers with large offsets, where a pseudo may be needed. */
29520 633343 : if (IN_RANGE (offset, -0x8000, 0x7fff))
29521 632202 : bitmap_set_bit (components, regno);
29522 679467 : offset += UNITS_PER_WORD;
29523 : }
29524 :
29525 : /* Don't mess with the following registers. */
29526 361866 : if (frame_pointer_needed)
29527 6422 : bitmap_clear_bit (components, HARD_FRAME_POINTER_REGNUM);
29528 :
29529 361866 : if (crtl->drap_reg)
29530 138 : bitmap_clear_bit (components, REGNO (crtl->drap_reg));
29531 :
29532 361866 : if (pic_offset_table_rtx)
29533 29998 : bitmap_clear_bit (components, REAL_PIC_OFFSET_TABLE_REGNUM);
29534 :
29535 : return components;
29536 : }
29537 :
29538 : /* Implement TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB. */
29539 : sbitmap
29540 9797297 : ix86_components_for_bb (basic_block bb)
29541 : {
29542 9797297 : bitmap in = DF_LIVE_IN (bb);
29543 9797297 : bitmap gen = &DF_LIVE_BB_INFO (bb)->gen;
29544 9797297 : bitmap kill = &DF_LIVE_BB_INFO (bb)->kill;
29545 :
29546 9797297 : sbitmap components = sbitmap_alloc (FIRST_PSEUDO_REGISTER);
29547 9797297 : bitmap_clear (components);
29548 :
29549 9797297 : function_abi_aggregator callee_abis;
29550 9797297 : rtx_insn *insn;
29551 118754359 : FOR_BB_INSNS (bb, insn)
29552 108957062 : if (CALL_P (insn))
29553 3269518 : callee_abis.note_callee_abi (insn_callee_abi (insn));
29554 9797297 : HARD_REG_SET extra_caller_saves = callee_abis.caller_save_regs (*crtl->abi);
29555 :
29556 : /* GPRs are used in a bb if they are in the IN, GEN, or KILL sets. */
29557 930743215 : for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
29558 920945918 : if (!fixed_regs[regno]
29559 920945918 : && (TEST_HARD_REG_BIT (extra_caller_saves, regno)
29560 458063248 : || bitmap_bit_p (in, regno)
29561 431342751 : || bitmap_bit_p (gen, regno)
29562 418112619 : || bitmap_bit_p (kill, regno)))
29563 40267137 : bitmap_set_bit (components, regno);
29564 :
29565 9797297 : return components;
29566 : }
29567 :
29568 : /* Implement TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS. */
29569 : void
29570 484749 : ix86_disqualify_components (sbitmap, edge, sbitmap, bool)
29571 : {
29572 : /* Nothing to do for x86. */
29573 484749 : }
29574 :
29575 : /* Implement TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS. */
29576 : void
29577 163505 : ix86_emit_prologue_components (sbitmap components)
29578 : {
29579 163505 : HOST_WIDE_INT cfa_offset;
29580 163505 : struct machine_function *m = cfun->machine;
29581 :
29582 163505 : cfa_offset = m->frame.reg_save_offset + m->fs.sp_offset
29583 163505 : - m->frame.stack_pointer_offset;
29584 15532975 : for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
29585 15369470 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true, true))
29586 : {
29587 735253 : if (bitmap_bit_p (components, regno))
29588 199189 : ix86_emit_save_reg_using_mov (word_mode, regno, cfa_offset);
29589 784913 : cfa_offset -= UNITS_PER_WORD;
29590 : }
29591 163505 : }
29592 :
29593 : /* Implement TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS. */
29594 : void
29595 145912 : ix86_emit_epilogue_components (sbitmap components)
29596 : {
29597 145912 : HOST_WIDE_INT cfa_offset;
29598 145912 : struct machine_function *m = cfun->machine;
29599 145912 : cfa_offset = m->frame.reg_save_offset + m->fs.sp_offset
29600 145912 : - m->frame.stack_pointer_offset;
29601 :
29602 13861640 : for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
29603 13715728 : if (GENERAL_REGNO_P (regno) && ix86_save_reg (regno, true, true))
29604 : {
29605 657434 : if (bitmap_bit_p (components, regno))
29606 : {
29607 236581 : rtx reg = gen_rtx_REG (word_mode, regno);
29608 236581 : rtx mem;
29609 236581 : rtx_insn *insn;
29610 :
29611 236581 : mem = choose_baseaddr (cfa_offset, NULL);
29612 236581 : mem = gen_frame_mem (word_mode, mem);
29613 236581 : insn = emit_move_insn (reg, mem);
29614 :
29615 236581 : RTX_FRAME_RELATED_P (insn) = 1;
29616 236581 : add_reg_note (insn, REG_CFA_RESTORE, reg);
29617 : }
29618 713982 : cfa_offset -= UNITS_PER_WORD;
29619 : }
29620 145912 : }
29621 :
29622 : /* Implement TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS. */
29623 : void
29624 45833 : ix86_set_handled_components (sbitmap components)
29625 : {
29626 4354135 : for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
29627 4308302 : if (bitmap_bit_p (components, regno))
29628 : {
29629 108743 : cfun->machine->reg_is_wrapped_separately[regno] = true;
29630 108743 : cfun->machine->use_fast_prologue_epilogue = true;
29631 108743 : cfun->machine->frame.save_regs_using_mov = true;
29632 : }
29633 45833 : }
29634 :
29635 : #undef TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS
29636 : #define TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS ix86_get_separate_components
29637 : #undef TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB
29638 : #define TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB ix86_components_for_bb
29639 : #undef TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS
29640 : #define TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS ix86_disqualify_components
29641 : #undef TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS
29642 : #define TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS \
29643 : ix86_emit_prologue_components
29644 : #undef TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS
29645 : #define TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS \
29646 : ix86_emit_epilogue_components
29647 : #undef TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS
29648 : #define TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS ix86_set_handled_components
29649 :
29650 : struct gcc_target targetm = TARGET_INITIALIZER;
29651 :
29652 : #include "gt-i386.h"
|
