Line data Source code
1 : /* Emit RTL for the GCC expander.
2 : Copyright (C) 1987-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 :
21 : /* Middle-to-low level generation of rtx code and insns.
22 :
23 : This file contains support functions for creating rtl expressions
24 : and manipulating them in the doubly-linked chain of insns.
25 :
26 : The patterns of the insns are created by machine-dependent
27 : routines in insn-emit.cc, which is generated automatically from
28 : the machine description. These routines make the individual rtx's
29 : of the pattern with `gen_rtx_fmt_ee' and others in genrtl.[ch],
30 : which are automatically generated from rtl.def; what is machine
31 : dependent is the kind of rtx's they make and what arguments they
32 : use. */
33 :
34 : #include "config.h"
35 : #include "system.h"
36 : #include "coretypes.h"
37 : #include "memmodel.h"
38 : #include "backend.h"
39 : #include "target.h"
40 : #include "rtl.h"
41 : #include "tree.h"
42 : #include "df.h"
43 : #include "tm_p.h"
44 : #include "stringpool.h"
45 : #include "insn-config.h"
46 : #include "regs.h"
47 : #include "emit-rtl.h"
48 : #include "recog.h"
49 : #include "diagnostic-core.h"
50 : #include "alias.h"
51 : #include "fold-const.h"
52 : #include "varasm.h"
53 : #include "cfgrtl.h"
54 : #include "tree-eh.h"
55 : #include "explow.h"
56 : #include "expr.h"
57 : #include "builtins.h"
58 : #include "rtl-iter.h"
59 : #include "stor-layout.h"
60 : #include "opts.h"
61 : #include "optabs.h"
62 : #include "predict.h"
63 : #include "rtx-vector-builder.h"
64 : #include "gimple.h"
65 : #include "gimple-ssa.h"
66 : #include "bbitmap.h"
67 :
68 : struct target_rtl default_target_rtl;
69 : #if SWITCHABLE_TARGET
70 : struct target_rtl *this_target_rtl = &default_target_rtl;
71 : #endif
72 :
73 : #define initial_regno_reg_rtx (this_target_rtl->x_initial_regno_reg_rtx)
74 :
75 : /* Commonly used modes. */
76 :
77 : scalar_int_mode byte_mode; /* Mode whose width is BITS_PER_UNIT. */
78 : scalar_int_mode word_mode; /* Mode whose width is BITS_PER_WORD. */
79 : scalar_int_mode ptr_mode; /* Mode whose width is POINTER_SIZE. */
80 :
81 : /* Datastructures maintained for currently processed function in RTL form. */
82 :
83 : struct rtl_data x_rtl;
84 :
85 : /* Indexed by pseudo register number, gives the rtx for that pseudo.
86 : Allocated in parallel with regno_pointer_align.
87 : FIXME: We could put it into emit_status struct, but gengtype is not able to deal
88 : with length attribute nested in top level structures. */
89 :
90 : rtx * regno_reg_rtx;
91 :
92 : /* This is *not* reset after each function. It gives each CODE_LABEL
93 : in the entire compilation a unique label number. */
94 :
95 : static GTY(()) int label_num = 1;
96 :
97 : /* We record floating-point CONST_DOUBLEs in each floating-point mode for
98 : the values of 0, 1, and 2. For the integer entries and VOIDmode, we
99 : record a copy of const[012]_rtx and constm1_rtx. CONSTM1_RTX
100 : is set only for MODE_INT and MODE_VECTOR_INT modes. */
101 :
102 : rtx const_tiny_rtx[4][(int) MAX_MACHINE_MODE];
103 :
104 : rtx const_true_rtx;
105 :
106 : REAL_VALUE_TYPE dconst0;
107 : REAL_VALUE_TYPE dconst1;
108 : REAL_VALUE_TYPE dconst2;
109 : REAL_VALUE_TYPE dconstm0;
110 : REAL_VALUE_TYPE dconstm1;
111 : REAL_VALUE_TYPE dconsthalf;
112 : REAL_VALUE_TYPE dconstinf;
113 : REAL_VALUE_TYPE dconstninf;
114 :
115 : /* Record fixed-point constant 0 and 1. */
116 : FIXED_VALUE_TYPE fconst0[MAX_FCONST0];
117 : FIXED_VALUE_TYPE fconst1[MAX_FCONST1];
118 :
119 : /* We make one copy of (const_int C) where C is in
120 : [- MAX_SAVED_CONST_INT, MAX_SAVED_CONST_INT]
121 : to save space during the compilation and simplify comparisons of
122 : integers. */
123 :
124 : rtx const_int_rtx[MAX_SAVED_CONST_INT * 2 + 1];
125 :
126 : /* Standard pieces of rtx, to be substituted directly into things. */
127 : rtx pc_rtx;
128 : rtx ret_rtx;
129 : rtx simple_return_rtx;
130 :
131 : /* Marker used for denoting an INSN, which should never be accessed (i.e.,
132 : this pointer should normally never be dereferenced), but is required to be
133 : distinct from NULL_RTX. Currently used by peephole2 pass. */
134 : rtx_insn *invalid_insn_rtx;
135 :
136 : /* A hash table storing CONST_INTs whose absolute value is greater
137 : than MAX_SAVED_CONST_INT. */
138 :
139 : struct const_int_hasher : ggc_cache_ptr_hash<rtx_def>
140 : {
141 : typedef HOST_WIDE_INT compare_type;
142 :
143 : static hashval_t hash (rtx i);
144 : static bool equal (rtx i, HOST_WIDE_INT h);
145 : };
146 :
147 : static GTY ((cache)) hash_table<const_int_hasher> *const_int_htab;
148 :
149 : struct const_wide_int_hasher : ggc_cache_ptr_hash<rtx_def>
150 : {
151 : static hashval_t hash (rtx x);
152 : static bool equal (rtx x, rtx y);
153 : };
154 :
155 : static GTY ((cache)) hash_table<const_wide_int_hasher> *const_wide_int_htab;
156 :
157 : struct const_poly_int_hasher : ggc_cache_ptr_hash<rtx_def>
158 : {
159 : typedef std::pair<machine_mode, poly_wide_int_ref> compare_type;
160 :
161 : static hashval_t hash (rtx x);
162 : static bool equal (rtx x, const compare_type &y);
163 : };
164 :
165 : static GTY ((cache)) hash_table<const_poly_int_hasher> *const_poly_int_htab;
166 :
167 : /* A hash table storing register attribute structures. */
168 : struct reg_attr_hasher : ggc_cache_ptr_hash<reg_attrs>
169 : {
170 : static hashval_t hash (reg_attrs *x);
171 : static bool equal (reg_attrs *a, reg_attrs *b);
172 : };
173 :
174 : static GTY ((cache)) hash_table<reg_attr_hasher> *reg_attrs_htab;
175 :
176 : /* A hash table storing all CONST_DOUBLEs. */
177 : struct const_double_hasher : ggc_cache_ptr_hash<rtx_def>
178 : {
179 : static hashval_t hash (rtx x);
180 : static bool equal (rtx x, rtx y);
181 : };
182 :
183 : static GTY ((cache)) hash_table<const_double_hasher> *const_double_htab;
184 :
185 : /* A hash table storing all CONST_FIXEDs. */
186 : struct const_fixed_hasher : ggc_cache_ptr_hash<rtx_def>
187 : {
188 : static hashval_t hash (rtx x);
189 : static bool equal (rtx x, rtx y);
190 : };
191 :
192 : static GTY ((cache)) hash_table<const_fixed_hasher> *const_fixed_htab;
193 :
194 : #define cur_insn_uid (crtl->emit.x_cur_insn_uid)
195 : #define cur_debug_insn_uid (crtl->emit.x_cur_debug_insn_uid)
196 : #define first_label_num (crtl->emit.x_first_label_num)
197 :
198 : static void set_used_decls (tree);
199 : static void mark_label_nuses (rtx);
200 : #if TARGET_SUPPORTS_WIDE_INT
201 : static rtx lookup_const_wide_int (rtx);
202 : #endif
203 : static rtx lookup_const_double (rtx);
204 : static rtx lookup_const_fixed (rtx);
205 : static rtx gen_const_vector (machine_mode, int);
206 : static void copy_rtx_if_shared_1 (rtx *orig);
207 :
208 : /* Probability of the conditional branch currently proceeded by try_split. */
209 : profile_probability split_branch_probability;
210 :
211 : /* Returns a hash code for X (which is a really a CONST_INT). */
212 :
213 : hashval_t
214 3448211539 : const_int_hasher::hash (rtx x)
215 : {
216 3448211539 : return (hashval_t) INTVAL (x);
217 : }
218 :
219 : /* Returns true if the value represented by X (which is really a
220 : CONST_INT) is the same as that given by Y (which is really a
221 : HOST_WIDE_INT *). */
222 :
223 : bool
224 4359677659 : const_int_hasher::equal (rtx x, HOST_WIDE_INT y)
225 : {
226 4359677659 : return (INTVAL (x) == y);
227 : }
228 :
229 : #if TARGET_SUPPORTS_WIDE_INT
230 : /* Returns a hash code for X (which is a really a CONST_WIDE_INT). */
231 :
232 : hashval_t
233 1471787 : const_wide_int_hasher::hash (rtx x)
234 : {
235 1471787 : int i;
236 1471787 : unsigned HOST_WIDE_INT hash = 0;
237 1471787 : const_rtx xr = x;
238 :
239 4440840 : for (i = 0; i < CONST_WIDE_INT_NUNITS (xr); i++)
240 2969053 : hash += CONST_WIDE_INT_ELT (xr, i);
241 :
242 1471787 : return (hashval_t) hash;
243 : }
244 :
245 : /* Returns true if the value represented by X (which is really a
246 : CONST_WIDE_INT) is the same as that given by Y (which is really a
247 : CONST_WIDE_INT). */
248 :
249 : bool
250 1502821 : const_wide_int_hasher::equal (rtx x, rtx y)
251 : {
252 1502821 : int i;
253 1502821 : const_rtx xr = x;
254 1502821 : const_rtx yr = y;
255 1502821 : if (CONST_WIDE_INT_NUNITS (xr) != CONST_WIDE_INT_NUNITS (yr))
256 : return false;
257 :
258 2624200 : for (i = 0; i < CONST_WIDE_INT_NUNITS (xr); i++)
259 2093149 : if (CONST_WIDE_INT_ELT (xr, i) != CONST_WIDE_INT_ELT (yr, i))
260 : return false;
261 :
262 : return true;
263 : }
264 : #endif
265 :
266 : /* Returns a hash code for CONST_POLY_INT X. */
267 :
268 : hashval_t
269 0 : const_poly_int_hasher::hash (rtx x)
270 : {
271 0 : inchash::hash h;
272 0 : h.add_int (GET_MODE (x));
273 0 : for (unsigned int i = 0; i < NUM_POLY_INT_COEFFS; ++i)
274 0 : h.add_wide_int (CONST_POLY_INT_COEFFS (x)[i]);
275 0 : return h.end ();
276 : }
277 :
278 : /* Returns true if CONST_POLY_INT X is an rtx representation of Y. */
279 :
280 : bool
281 0 : const_poly_int_hasher::equal (rtx x, const compare_type &y)
282 : {
283 0 : if (GET_MODE (x) != y.first)
284 : return false;
285 0 : for (unsigned int i = 0; i < NUM_POLY_INT_COEFFS; ++i)
286 0 : if (CONST_POLY_INT_COEFFS (x)[i] != y.second.coeffs[i])
287 : return false;
288 : return true;
289 : }
290 :
291 : /* Returns a hash code for X (which is really a CONST_DOUBLE). */
292 : hashval_t
293 18879216 : const_double_hasher::hash (rtx x)
294 : {
295 18879216 : const_rtx const value = x;
296 18879216 : hashval_t h;
297 :
298 18879216 : if (TARGET_SUPPORTS_WIDE_INT == 0 && GET_MODE (value) == VOIDmode)
299 : h = CONST_DOUBLE_LOW (value) ^ CONST_DOUBLE_HIGH (value);
300 : else
301 : {
302 18879216 : h = real_hash (CONST_DOUBLE_REAL_VALUE (value));
303 : /* MODE is used in the comparison, so it should be in the hash. */
304 18879216 : h ^= GET_MODE (value);
305 : }
306 18879216 : return h;
307 : }
308 :
309 : /* Returns true if the value represented by X (really a ...)
310 : is the same as that represented by Y (really a ...) */
311 : bool
312 12992864 : const_double_hasher::equal (rtx x, rtx y)
313 : {
314 12992864 : const_rtx const a = x, b = y;
315 :
316 12992864 : if (GET_MODE (a) != GET_MODE (b))
317 : return false;
318 4676606 : if (TARGET_SUPPORTS_WIDE_INT == 0 && GET_MODE (a) == VOIDmode)
319 : return (CONST_DOUBLE_LOW (a) == CONST_DOUBLE_LOW (b)
320 : && CONST_DOUBLE_HIGH (a) == CONST_DOUBLE_HIGH (b));
321 : else
322 4676606 : return real_identical (CONST_DOUBLE_REAL_VALUE (a),
323 4676606 : CONST_DOUBLE_REAL_VALUE (b));
324 : }
325 :
326 : /* Returns a hash code for X (which is really a CONST_FIXED). */
327 :
328 : hashval_t
329 12172113 : const_fixed_hasher::hash (rtx x)
330 : {
331 12172113 : const_rtx const value = x;
332 12172113 : hashval_t h;
333 :
334 12172113 : h = fixed_hash (CONST_FIXED_VALUE (value));
335 : /* MODE is used in the comparison, so it should be in the hash. */
336 12172113 : h ^= GET_MODE (value);
337 12172113 : return h;
338 : }
339 :
340 : /* Returns true if the value represented by X is the same as that
341 : represented by Y. */
342 :
343 : bool
344 6089598 : const_fixed_hasher::equal (rtx x, rtx y)
345 : {
346 6089598 : const_rtx const a = x, b = y;
347 :
348 6089598 : if (GET_MODE (a) != GET_MODE (b))
349 : return false;
350 579444 : return fixed_identical (CONST_FIXED_VALUE (a), CONST_FIXED_VALUE (b));
351 : }
352 :
353 : /* Return true if the given memory attributes are equal. */
354 :
355 : bool
356 234822459 : mem_attrs_eq_p (const class mem_attrs *p, const class mem_attrs *q)
357 : {
358 234822459 : if (p == q)
359 : return true;
360 231795526 : if (!p || !q)
361 : return false;
362 231188602 : return (p->alias == q->alias
363 145257709 : && p->offset_known_p == q->offset_known_p
364 115504815 : && (!p->offset_known_p || known_eq (p->offset, q->offset))
365 99568707 : && p->size_known_p == q->size_known_p
366 96210664 : && (!p->size_known_p || known_eq (p->size, q->size))
367 82710886 : && p->align == q->align
368 68065353 : && p->addrspace == q->addrspace
369 298970995 : && (p->expr == q->expr
370 32144120 : || (p->expr != NULL_TREE && q->expr != NULL_TREE
371 24961135 : && operand_equal_p (p->expr, q->expr, 0))));
372 : }
373 :
374 : /* Set MEM's memory attributes so that they are the same as ATTRS. */
375 :
376 : static void
377 102696285 : set_mem_attrs (rtx mem, mem_attrs *attrs)
378 : {
379 : /* If everything is the default, we can just clear the attributes. */
380 102696285 : if (mem_attrs_eq_p (attrs, mode_mem_attrs[(int) GET_MODE (mem)]))
381 : {
382 3356048 : MEM_ATTRS (mem) = 0;
383 3356048 : return;
384 : }
385 :
386 99340237 : if (!MEM_ATTRS (mem)
387 99340237 : || !mem_attrs_eq_p (attrs, MEM_ATTRS (mem)))
388 : {
389 75835600 : MEM_ATTRS (mem) = ggc_alloc<mem_attrs> ();
390 75835600 : memcpy (MEM_ATTRS (mem), attrs, sizeof (mem_attrs));
391 : }
392 : }
393 :
394 : /* Returns a hash code for X (which is a really a reg_attrs *). */
395 :
396 : hashval_t
397 322111355 : reg_attr_hasher::hash (reg_attrs *x)
398 : {
399 322111355 : const reg_attrs *const p = x;
400 :
401 322111355 : inchash::hash h;
402 322111355 : h.add_ptr (p->decl);
403 322111355 : h.add_poly_hwi (p->offset);
404 322111355 : return h.end ();
405 : }
406 :
407 : /* Returns true if the value represented by X is the same as that given by
408 : Y. */
409 :
410 : bool
411 311133396 : reg_attr_hasher::equal (reg_attrs *x, reg_attrs *y)
412 : {
413 311133396 : const reg_attrs *const p = x;
414 311133396 : const reg_attrs *const q = y;
415 :
416 311133396 : return (p->decl == q->decl && known_eq (p->offset, q->offset));
417 : }
418 : /* Allocate a new reg_attrs structure and insert it into the hash table if
419 : one identical to it is not already in the table. We are doing this for
420 : MEM of mode MODE. */
421 :
422 : static reg_attrs *
423 66593083 : get_reg_attrs (tree decl, poly_int64 offset)
424 : {
425 66593083 : reg_attrs attrs;
426 :
427 : /* If everything is the default, we can just return zero. */
428 66593083 : if (decl == 0 && known_eq (offset, 0))
429 : return 0;
430 :
431 60872185 : attrs.decl = decl;
432 60872185 : attrs.offset = offset;
433 :
434 60872185 : reg_attrs **slot = reg_attrs_htab->find_slot (&attrs, INSERT);
435 60872185 : if (*slot == 0)
436 : {
437 30749154 : *slot = ggc_alloc<reg_attrs> ();
438 30749154 : memcpy (*slot, &attrs, sizeof (reg_attrs));
439 : }
440 :
441 60872185 : return *slot;
442 : }
443 :
444 :
445 : #if !HAVE_blockage
446 : /* Generate an empty ASM_INPUT, which is used to block attempts to schedule,
447 : and to block register equivalences to be seen across this insn. */
448 :
449 : rtx
450 : gen_blockage (void)
451 : {
452 : rtx x = gen_rtx_ASM_INPUT (VOIDmode, "");
453 : MEM_VOLATILE_P (x) = true;
454 : return x;
455 : }
456 : #endif
457 :
458 :
459 : /* Set the mode and register number of X to MODE and REGNO. */
460 :
461 : void
462 1775267869 : set_mode_and_regno (rtx x, machine_mode mode, unsigned int regno)
463 : {
464 1775267869 : unsigned int nregs = (HARD_REGISTER_NUM_P (regno)
465 1775267869 : ? hard_regno_nregs (regno, mode)
466 1339107683 : : 1);
467 1775267869 : PUT_MODE_RAW (x, mode);
468 1775267869 : set_regno_raw (x, regno, nregs);
469 1775267869 : }
470 :
471 : /* Initialize a fresh REG rtx with mode MODE and register REGNO. */
472 :
473 : rtx
474 359932105 : init_raw_REG (rtx x, machine_mode mode, unsigned int regno)
475 : {
476 359932105 : set_mode_and_regno (x, mode, regno);
477 359932105 : REG_ATTRS (x) = NULL;
478 359932105 : ORIGINAL_REGNO (x) = regno;
479 359932105 : return x;
480 : }
481 :
482 : /* Generate a new REG rtx. Make sure ORIGINAL_REGNO is set properly, and
483 : don't attempt to share with the various global pieces of rtl (such as
484 : frame_pointer_rtx). */
485 :
486 : rtx
487 357764614 : gen_raw_REG (machine_mode mode, unsigned int regno)
488 : {
489 357764614 : rtx x = rtx_alloc (REG MEM_STAT_INFO);
490 357764614 : init_raw_REG (x, mode, regno);
491 357764614 : return x;
492 : }
493 :
494 : /* There are some RTL codes that require special attention; the generation
495 : functions do the raw handling. If you add to this list, modify
496 : special_rtx in gengenrtl.cc as well. */
497 :
498 : rtx_expr_list *
499 209576912 : gen_rtx_EXPR_LIST (machine_mode mode, rtx expr, rtx expr_list)
500 : {
501 209576912 : return as_a <rtx_expr_list *> (gen_rtx_fmt_ee (EXPR_LIST, mode, expr,
502 209576912 : expr_list));
503 : }
504 :
505 : rtx_insn_list *
506 349038142 : gen_rtx_INSN_LIST (machine_mode mode, rtx insn, rtx insn_list)
507 : {
508 349038142 : return as_a <rtx_insn_list *> (gen_rtx_fmt_ue (INSN_LIST, mode, insn,
509 349038142 : insn_list));
510 : }
511 :
512 : rtx_insn *
513 847948 : gen_rtx_INSN (machine_mode mode, rtx_insn *prev_insn, rtx_insn *next_insn,
514 : basic_block bb, rtx pattern, location_t location, int code,
515 : rtx reg_notes)
516 : {
517 847948 : return as_a <rtx_insn *> (gen_rtx_fmt_uuBeLie (INSN, mode,
518 : prev_insn, next_insn,
519 : bb, pattern, location, code,
520 847948 : reg_notes));
521 : }
522 :
523 : rtx
524 1326144646 : gen_rtx_CONST_INT (machine_mode mode ATTRIBUTE_UNUSED, HOST_WIDE_INT arg)
525 : {
526 1326144646 : if (arg >= - MAX_SAVED_CONST_INT && arg <= MAX_SAVED_CONST_INT)
527 852526930 : return const_int_rtx[arg + MAX_SAVED_CONST_INT];
528 :
529 : #if STORE_FLAG_VALUE != 1 && STORE_FLAG_VALUE != -1
530 : if (const_true_rtx && arg == STORE_FLAG_VALUE)
531 : return const_true_rtx;
532 : #endif
533 :
534 : /* Look up the CONST_INT in the hash table. */
535 473617716 : rtx *slot = const_int_htab->find_slot_with_hash (arg, (hashval_t) arg,
536 : INSERT);
537 473617716 : if (*slot == 0)
538 36182592 : *slot = gen_rtx_raw_CONST_INT (VOIDmode, arg);
539 :
540 473617716 : return *slot;
541 : }
542 :
543 : /* Return a CONST_INT, CONST_POLY_INT, or CONST_VECTOR containing the given
544 : value. In the case of a vector, the value will be duplicated. */
545 :
546 : rtx
547 1239837570 : gen_int_mode (poly_int64 c, machine_mode mode)
548 : {
549 1239837570 : machine_mode s_mode = (VECTOR_MODE_P (mode) ? GET_MODE_INNER (mode) : mode);
550 1239837570 : rtx val;
551 :
552 1239837570 : c = trunc_int_for_mode (c, s_mode);
553 1239837570 : if (c.is_constant ())
554 1239837570 : val = GEN_INT (c.coeffs[0]);
555 : else
556 : {
557 : unsigned int prec = GET_MODE_PRECISION (as_a <scalar_mode> (s_mode));
558 : val = immed_wide_int_const (poly_wide_int::from (c, prec, SIGNED),
559 : s_mode);
560 : }
561 :
562 1239837570 : if (VECTOR_MODE_P (mode))
563 0 : val = gen_const_vec_duplicate (mode, val);
564 :
565 1239837570 : return val;
566 : }
567 :
568 : /* CONST_DOUBLEs might be created from pairs of integers, or from
569 : REAL_VALUE_TYPEs. Also, their length is known only at run time,
570 : so we cannot use gen_rtx_raw_CONST_DOUBLE. */
571 :
572 : /* Determine whether REAL, a CONST_DOUBLE, already exists in the
573 : hash table. If so, return its counterpart; otherwise add it
574 : to the hash table and return it. */
575 : static rtx
576 10047643 : lookup_const_double (rtx real)
577 : {
578 10047643 : rtx *slot = const_double_htab->find_slot (real, INSERT);
579 10047643 : if (*slot == 0)
580 8028268 : *slot = real;
581 :
582 10047643 : return *slot;
583 : }
584 :
585 : /* Return a CONST_DOUBLE rtx for a floating-point value specified by
586 : VALUE in mode MODE. */
587 : rtx
588 10047643 : const_double_from_real_value (REAL_VALUE_TYPE value, machine_mode mode)
589 : {
590 10047643 : rtx real = rtx_alloc (CONST_DOUBLE);
591 10047643 : PUT_MODE (real, mode);
592 :
593 10047643 : real->u.rv = value;
594 :
595 10047643 : return lookup_const_double (real);
596 : }
597 :
598 : /* Determine whether FIXED, a CONST_FIXED, already exists in the
599 : hash table. If so, return its counterpart; otherwise add it
600 : to the hash table and return it. */
601 :
602 : static rtx
603 7530666 : lookup_const_fixed (rtx fixed)
604 : {
605 7530666 : rtx *slot = const_fixed_htab->find_slot (fixed, INSERT);
606 7530666 : if (*slot == 0)
607 7530666 : *slot = fixed;
608 :
609 7530666 : return *slot;
610 : }
611 :
612 : /* Return a CONST_FIXED rtx for a fixed-point value specified by
613 : VALUE in mode MODE. */
614 :
615 : rtx
616 7530666 : const_fixed_from_fixed_value (FIXED_VALUE_TYPE value, machine_mode mode)
617 : {
618 7530666 : rtx fixed = rtx_alloc (CONST_FIXED);
619 7530666 : PUT_MODE (fixed, mode);
620 :
621 7530666 : fixed->u.fv = value;
622 :
623 7530666 : return lookup_const_fixed (fixed);
624 : }
625 :
626 : #if TARGET_SUPPORTS_WIDE_INT == 0
627 : /* Constructs double_int from rtx CST. */
628 :
629 : double_int
630 : rtx_to_double_int (const_rtx cst)
631 : {
632 : double_int r;
633 :
634 : if (CONST_INT_P (cst))
635 : r = double_int::from_shwi (INTVAL (cst));
636 : else if (CONST_DOUBLE_AS_INT_P (cst))
637 : {
638 : r.low = CONST_DOUBLE_LOW (cst);
639 : r.high = CONST_DOUBLE_HIGH (cst);
640 : }
641 : else
642 : gcc_unreachable ();
643 :
644 : return r;
645 : }
646 : #endif
647 :
648 : #if TARGET_SUPPORTS_WIDE_INT
649 : /* Determine whether CONST_WIDE_INT WINT already exists in the hash table.
650 : If so, return its counterpart; otherwise add it to the hash table and
651 : return it. */
652 :
653 : static rtx
654 591175 : lookup_const_wide_int (rtx wint)
655 : {
656 591175 : rtx *slot = const_wide_int_htab->find_slot (wint, INSERT);
657 591175 : if (*slot == 0)
658 60124 : *slot = wint;
659 :
660 591175 : return *slot;
661 : }
662 : #endif
663 :
664 : /* Return an rtx constant for V, given that the constant has mode MODE.
665 : The returned rtx will be a CONST_INT if V fits, otherwise it will be
666 : a CONST_DOUBLE (if !TARGET_SUPPORTS_WIDE_INT) or a CONST_WIDE_INT
667 : (if TARGET_SUPPORTS_WIDE_INT). */
668 :
669 : static rtx
670 639499693 : immed_wide_int_const_1 (const wide_int_ref &v, machine_mode mode)
671 : {
672 639499693 : unsigned int len = v.get_len ();
673 : /* Not scalar_int_mode because we also allow pointer bound modes. */
674 639499693 : unsigned int prec = GET_MODE_PRECISION (as_a <scalar_mode> (mode));
675 :
676 : /* Allow truncation but not extension since we do not know if the
677 : number is signed or unsigned. */
678 639499693 : gcc_assert (prec <= v.get_precision ());
679 :
680 639499693 : if (len < 2 || prec <= HOST_BITS_PER_WIDE_INT)
681 638908518 : return gen_int_mode (v.elt (0), mode);
682 :
683 : #if TARGET_SUPPORTS_WIDE_INT
684 591175 : {
685 591175 : unsigned int i;
686 591175 : rtx value;
687 591175 : unsigned int blocks_needed
688 591175 : = (prec + HOST_BITS_PER_WIDE_INT - 1) / HOST_BITS_PER_WIDE_INT;
689 :
690 591175 : if (len > blocks_needed)
691 : len = blocks_needed;
692 :
693 591175 : value = const_wide_int_alloc (len);
694 :
695 : /* It is so tempting to just put the mode in here. Must control
696 : myself ... */
697 591175 : PUT_MODE (value, VOIDmode);
698 591175 : CWI_PUT_NUM_ELEM (value, len);
699 :
700 1781224 : for (i = 0; i < len; i++)
701 1190049 : CONST_WIDE_INT_ELT (value, i) = v.elt (i);
702 :
703 591175 : return lookup_const_wide_int (value);
704 : }
705 : #else
706 : return immed_double_const (v.elt (0), v.elt (1), mode);
707 : #endif
708 : }
709 :
710 : #if TARGET_SUPPORTS_WIDE_INT == 0
711 : /* Return a CONST_DOUBLE or CONST_INT for a value specified as a pair
712 : of ints: I0 is the low-order word and I1 is the high-order word.
713 : For values that are larger than HOST_BITS_PER_DOUBLE_INT, the
714 : implied upper bits are copies of the high bit of i1. The value
715 : itself is neither signed nor unsigned. Do not use this routine for
716 : non-integer modes; convert to REAL_VALUE_TYPE and use
717 : const_double_from_real_value. */
718 :
719 : rtx
720 : immed_double_const (HOST_WIDE_INT i0, HOST_WIDE_INT i1, machine_mode mode)
721 : {
722 : rtx value;
723 : unsigned int i;
724 :
725 : /* There are the following cases (note that there are no modes with
726 : HOST_BITS_PER_WIDE_INT < GET_MODE_BITSIZE (mode) < HOST_BITS_PER_DOUBLE_INT):
727 :
728 : 1) If GET_MODE_BITSIZE (mode) <= HOST_BITS_PER_WIDE_INT, then we use
729 : gen_int_mode.
730 : 2) If the value of the integer fits into HOST_WIDE_INT anyway
731 : (i.e., i1 consists only from copies of the sign bit, and sign
732 : of i0 and i1 are the same), then we return a CONST_INT for i0.
733 : 3) Otherwise, we create a CONST_DOUBLE for i0 and i1. */
734 : scalar_mode smode;
735 : if (is_a <scalar_mode> (mode, &smode)
736 : && GET_MODE_BITSIZE (smode) <= HOST_BITS_PER_WIDE_INT)
737 : return gen_int_mode (i0, mode);
738 :
739 : /* If this integer fits in one word, return a CONST_INT. */
740 : if ((i1 == 0 && i0 >= 0) || (i1 == ~0 && i0 < 0))
741 : return GEN_INT (i0);
742 :
743 : /* We use VOIDmode for integers. */
744 : value = rtx_alloc (CONST_DOUBLE);
745 : PUT_MODE (value, VOIDmode);
746 :
747 : CONST_DOUBLE_LOW (value) = i0;
748 : CONST_DOUBLE_HIGH (value) = i1;
749 :
750 : for (i = 2; i < (sizeof CONST_DOUBLE_FORMAT - 1); i++)
751 : XWINT (value, i) = 0;
752 :
753 : return lookup_const_double (value);
754 : }
755 : #endif
756 :
757 : /* Return an rtx representation of C in mode MODE. */
758 :
759 : rtx
760 639499693 : immed_wide_int_const (const poly_wide_int_ref &c, machine_mode mode)
761 : {
762 639499693 : if (c.is_constant ())
763 639499693 : return immed_wide_int_const_1 (c.coeffs[0], mode);
764 :
765 : /* Not scalar_int_mode because we also allow pointer bound modes. */
766 : unsigned int prec = GET_MODE_PRECISION (as_a <scalar_mode> (mode));
767 :
768 : /* Allow truncation but not extension since we do not know if the
769 : number is signed or unsigned. */
770 : gcc_assert (prec <= c.coeffs[0].get_precision ());
771 : poly_wide_int newc = poly_wide_int::from (c, prec, SIGNED);
772 :
773 : /* See whether we already have an rtx for this constant. */
774 : inchash::hash h;
775 : h.add_int (mode);
776 : for (unsigned int i = 0; i < NUM_POLY_INT_COEFFS; ++i)
777 : h.add_wide_int (newc.coeffs[i]);
778 : const_poly_int_hasher::compare_type typed_value (mode, newc);
779 : rtx *slot = const_poly_int_htab->find_slot_with_hash (typed_value,
780 : h.end (), INSERT);
781 : rtx x = *slot;
782 : if (x)
783 : return x;
784 :
785 : /* Create a new rtx. There's a choice to be made here between installing
786 : the actual mode of the rtx or leaving it as VOIDmode (for consistency
787 : with CONST_INT). In practice the handling of the codes is different
788 : enough that we get no benefit from using VOIDmode, and various places
789 : assume that VOIDmode implies CONST_INT. Using the real mode seems like
790 : the right long-term direction anyway. */
791 : typedef trailing_wide_ints<NUM_POLY_INT_COEFFS> twi;
792 : size_t extra_size = twi::extra_size (prec);
793 : x = rtx_alloc_v (CONST_POLY_INT,
794 : sizeof (struct const_poly_int_def) + extra_size);
795 : PUT_MODE (x, mode);
796 : CONST_POLY_INT_COEFFS (x).set_precision (prec);
797 : for (unsigned int i = 0; i < NUM_POLY_INT_COEFFS; ++i)
798 : CONST_POLY_INT_COEFFS (x)[i] = newc.coeffs[i];
799 :
800 : *slot = x;
801 : return x;
802 : }
803 :
804 : rtx
805 196193859 : gen_rtx_REG (machine_mode mode, unsigned int regno)
806 : {
807 : /* In case the MD file explicitly references the frame pointer, have
808 : all such references point to the same frame pointer. This is
809 : used during frame pointer elimination to distinguish the explicit
810 : references to these registers from pseudos that happened to be
811 : assigned to them.
812 :
813 : If we have eliminated the frame pointer or arg pointer, we will
814 : be using it as a normal register, for example as a spill
815 : register. In such cases, we might be accessing it in a mode that
816 : is not Pmode and therefore cannot use the pre-allocated rtx.
817 :
818 : Also don't do this when we are making new REGs in reload, since
819 : we don't want to get confused with the real pointers. */
820 :
821 210799344 : if (mode == Pmode && !reload_in_progress && !lra_in_progress)
822 : {
823 63321546 : if (regno == FRAME_POINTER_REGNUM
824 3041024 : && (!reload_completed || frame_pointer_needed))
825 3041024 : return frame_pointer_rtx;
826 :
827 60280522 : if (!HARD_FRAME_POINTER_IS_FRAME_POINTER
828 : && regno == HARD_FRAME_POINTER_REGNUM
829 4354495 : && (!reload_completed || frame_pointer_needed))
830 3829973 : return hard_frame_pointer_rtx;
831 : #if !HARD_FRAME_POINTER_IS_ARG_POINTER
832 56450549 : if (FRAME_POINTER_REGNUM != ARG_POINTER_REGNUM
833 : && regno == ARG_POINTER_REGNUM)
834 3045493 : return arg_pointer_rtx;
835 : #endif
836 : #ifdef RETURN_ADDRESS_POINTER_REGNUM
837 : if (regno == RETURN_ADDRESS_POINTER_REGNUM)
838 : return return_address_pointer_rtx;
839 : #endif
840 53405056 : if (regno == (unsigned) PIC_OFFSET_TABLE_REGNUM
841 0 : && PIC_OFFSET_TABLE_REGNUM != INVALID_REGNUM
842 53405056 : && fixed_regs[PIC_OFFSET_TABLE_REGNUM])
843 0 : return pic_offset_table_rtx;
844 53405056 : if (regno == STACK_POINTER_REGNUM)
845 4074853 : return stack_pointer_rtx;
846 : }
847 :
848 : #if 0
849 : /* If the per-function register table has been set up, try to re-use
850 : an existing entry in that table to avoid useless generation of RTL.
851 :
852 : This code is disabled for now until we can fix the various backends
853 : which depend on having non-shared hard registers in some cases. Long
854 : term we want to re-enable this code as it can significantly cut down
855 : on the amount of useless RTL that gets generated.
856 :
857 : We'll also need to fix some code that runs after reload that wants to
858 : set ORIGINAL_REGNO. */
859 :
860 : if (cfun
861 : && cfun->emit
862 : && regno_reg_rtx
863 : && regno < FIRST_PSEUDO_REGISTER
864 : && reg_raw_mode[regno] == mode)
865 : return regno_reg_rtx[regno];
866 : #endif
867 :
868 182202516 : return gen_raw_REG (mode, regno);
869 : }
870 :
871 : rtx
872 254480401 : gen_rtx_MEM (machine_mode mode, rtx addr)
873 : {
874 254480401 : rtx rt = gen_rtx_raw_MEM (mode, addr);
875 :
876 : /* This field is not cleared by the mere allocation of the rtx, so
877 : we clear it here. */
878 254480401 : MEM_ATTRS (rt) = 0;
879 :
880 254480401 : return rt;
881 : }
882 :
883 : /* Generate a memory referring to non-trapping constant memory. */
884 :
885 : rtx
886 2173724 : gen_const_mem (machine_mode mode, rtx addr)
887 : {
888 2173724 : rtx mem = gen_rtx_MEM (mode, addr);
889 2173724 : MEM_READONLY_P (mem) = 1;
890 2173724 : MEM_NOTRAP_P (mem) = 1;
891 2173724 : return mem;
892 : }
893 :
894 : /* Generate a MEM referring to fixed portions of the frame, e.g., register
895 : save areas. */
896 :
897 : rtx
898 1226188 : gen_frame_mem (machine_mode mode, rtx addr)
899 : {
900 1226188 : rtx mem = gen_rtx_MEM (mode, addr);
901 1226188 : MEM_NOTRAP_P (mem) = 1;
902 1226188 : set_mem_alias_set (mem, get_frame_alias_set ());
903 1226188 : return mem;
904 : }
905 :
906 : /* Generate a MEM referring to a temporary use of the stack, not part
907 : of the fixed stack frame. For example, something which is pushed
908 : by a target splitter. */
909 : rtx
910 0 : gen_tmp_stack_mem (machine_mode mode, rtx addr)
911 : {
912 0 : rtx mem = gen_rtx_MEM (mode, addr);
913 0 : MEM_NOTRAP_P (mem) = 1;
914 0 : if (!cfun->calls_alloca)
915 0 : set_mem_alias_set (mem, get_frame_alias_set ());
916 0 : return mem;
917 : }
918 :
919 : /* We want to create (subreg:OMODE (obj:IMODE) OFFSET). Return true if
920 : this construct would be valid, and false otherwise. */
921 :
922 : bool
923 52488818 : validate_subreg (machine_mode omode, machine_mode imode,
924 : const_rtx reg, poly_uint64 offset)
925 : {
926 104977636 : poly_uint64 isize = GET_MODE_SIZE (imode);
927 104977636 : poly_uint64 osize = GET_MODE_SIZE (omode);
928 :
929 : /* The sizes must be ordered, so that we know whether the subreg
930 : is partial, paradoxical or complete. */
931 52488818 : if (!ordered_p (isize, osize))
932 : return false;
933 :
934 : /* All subregs must be aligned. */
935 52488818 : if (!multiple_p (offset, osize))
936 : return false;
937 :
938 : /* The subreg offset cannot be outside the inner object. */
939 52488798 : if (maybe_ge (offset, isize))
940 : return false;
941 :
942 52488798 : poly_uint64 regsize = REGMODE_NATURAL_SIZE (imode);
943 :
944 : /* ??? This should not be here. Temporarily continue to allow word_mode
945 : subregs of anything. The most common offender is (subreg:SI (reg:DF)).
946 : Generally, backends are doing something sketchy but it'll take time to
947 : fix them all. */
948 52488798 : if (omode == word_mode)
949 : ;
950 : /* ??? Similarly, e.g. with (subreg:DF (reg:TI)). Though store_bit_field
951 : is the culprit here, and not the backends. */
952 31941115 : else if (known_ge (osize, regsize) && known_ge (isize, osize))
953 : ;
954 : /* Allow component subregs of complex and vector. Though given the below
955 : extraction rules, it's not always clear what that means. */
956 25616590 : else if ((COMPLEX_MODE_P (imode) || VECTOR_MODE_P (imode))
957 26815869 : && GET_MODE_INNER (imode) == omode)
958 : ;
959 : /* ??? x86 sse code makes heavy use of *paradoxical* vector subregs,
960 : i.e. (subreg:V4SF (reg:SF) 0) or (subreg:V4SF (reg:V2SF) 0). This
961 : surely isn't the cleanest way to represent this. It's questionable
962 : if this ought to be represented at all -- why can't this all be hidden
963 : in post-reload splitters that make arbitrarily mode changes to the
964 : registers themselves. */
965 24167588 : else if (VECTOR_MODE_P (omode)
966 26449960 : && GET_MODE_UNIT_SIZE (omode) == GET_MODE_UNIT_SIZE (imode))
967 : ;
968 : /* Subregs involving floating point modes are not allowed to
969 : change size unless it's an insert into a complex mode.
970 : Therefore (subreg:DI (reg:DF) 0) and (subreg:CS (reg:SF) 0) are fine, but
971 : (subreg:SI (reg:DF) 0) isn't. */
972 24063523 : else if ((FLOAT_MODE_P (imode) || FLOAT_MODE_P (omode))
973 326423 : && !COMPLEX_MODE_P (omode))
974 : {
975 326261 : if (! (known_eq (isize, osize)
976 : /* LRA can use subreg to store a floating point value in
977 : an integer mode. Although the floating point and the
978 : integer modes need the same number of hard registers,
979 : the size of floating point mode can be less than the
980 : integer mode. LRA also uses subregs for a register
981 : should be used in different mode in on insn. */
982 73114 : || lra_in_progress))
983 : return false;
984 : }
985 :
986 : /* Paradoxical subregs must have offset zero. */
987 52417711 : if (maybe_gt (osize, isize) && !known_eq (offset, 0U))
988 : return false;
989 :
990 : /* Verify that the offset is representable. */
991 :
992 : /* Ensure that subregs of hard registers can be folded. In other words,
993 : the hardware register must be valid in the subreg's outer mode,
994 : and consequently the subreg can be replaced with a hardware register. */
995 52417711 : if (reg && REG_P (reg) && HARD_REGISTER_P (reg))
996 : {
997 316348 : unsigned int regno = REGNO (reg);
998 :
999 316348 : if ((COMPLEX_MODE_P (imode) || VECTOR_MODE_P (imode))
1000 611920 : && GET_MODE_INNER (imode) == omode)
1001 : ;
1002 116822 : else if (!REG_CAN_CHANGE_MODE_P (regno, imode, omode))
1003 : return false;
1004 :
1005 : /* Pass true to allow_stack_regs because targets like x86
1006 : expect to be able to take subregs of the stack pointer. */
1007 316251 : return simplify_subreg_regno (regno, imode, offset, omode, true) >= 0;
1008 : }
1009 : /* Do not allow normal SUBREG with stricter alignment than the inner MEM.
1010 :
1011 : PR120329: Combine can create paradoxical mem subregs even for
1012 : strict-alignment targets. Allow it until combine is fixed. */
1013 52101363 : else if (reg && MEM_P (reg) && STRICT_ALIGNMENT
1014 : && MEM_ALIGN (reg) < GET_MODE_ALIGNMENT (omode)
1015 : && known_le (osize, isize))
1016 : return false;
1017 :
1018 : /* If ISIZE is greater than REGSIZE, the inner value is split into blocks
1019 : of size REGSIZE. The outer size must then be ordered wrt REGSIZE,
1020 : otherwise we wouldn't know at compile time how many blocks the
1021 : outer mode occupies. */
1022 52101363 : if (maybe_gt (isize, regsize) && !ordered_p (osize, regsize))
1023 : return false;
1024 :
1025 : /* For normal pseudo registers, we want most of the same checks. Namely:
1026 :
1027 : Assume that the pseudo register will be allocated to hard registers
1028 : that can hold REGSIZE bytes each. If OSIZE is not a multiple of REGSIZE,
1029 : the remainder must correspond to the lowpart of the containing hard
1030 : register. If BYTES_BIG_ENDIAN, the lowpart is at the highest offset,
1031 : otherwise it is at the lowest offset.
1032 :
1033 : Given that we've already checked the mode and offset alignment,
1034 : we only have to check subblock subregs here.
1035 :
1036 : For paradoxical little-endian registers, this check is redundant. The
1037 : offset has already been validated to be zero.
1038 :
1039 : For paradoxical big-endian registers, this check is not valid
1040 : because the offset is zero. */
1041 52101363 : if (maybe_lt (osize, regsize)
1042 24726343 : && known_le (osize, isize)
1043 69333628 : && ! (lra_in_progress && (FLOAT_MODE_P (imode) || FLOAT_MODE_P (omode))))
1044 : {
1045 : /* It is invalid for the target to pick a register size for a mode
1046 : that isn't ordered wrt to the size of that mode. */
1047 17223839 : poly_uint64 block_size = ordered_min (isize, regsize);
1048 17223839 : unsigned int start_reg;
1049 17223839 : poly_uint64 offset_within_reg;
1050 17223839 : if (!can_div_trunc_p (offset, block_size, &start_reg, &offset_within_reg)
1051 17223839 : || (BYTES_BIG_ENDIAN
1052 : ? maybe_ne (offset_within_reg, block_size - osize)
1053 17223839 : : maybe_ne (offset_within_reg, 0U)))
1054 4437 : return false;
1055 : }
1056 : return true;
1057 : }
1058 :
1059 : rtx
1060 26035543 : gen_rtx_SUBREG (machine_mode mode, rtx reg, poly_uint64 offset)
1061 : {
1062 26035543 : gcc_assert (validate_subreg (mode, GET_MODE (reg), reg, offset));
1063 26035543 : return gen_rtx_raw_SUBREG (mode, reg, offset);
1064 : }
1065 :
1066 : /* Generate a SUBREG representing the least-significant part of REG if MODE
1067 : is smaller than mode of REG, otherwise paradoxical SUBREG. */
1068 :
1069 : rtx
1070 598796 : gen_lowpart_SUBREG (machine_mode mode, rtx reg)
1071 : {
1072 598796 : machine_mode inmode;
1073 :
1074 598796 : inmode = GET_MODE (reg);
1075 598796 : if (inmode == VOIDmode)
1076 0 : inmode = mode;
1077 598796 : return gen_rtx_SUBREG (mode, reg,
1078 598796 : subreg_lowpart_offset (mode, inmode));
1079 : }
1080 :
1081 : rtx
1082 107375816 : gen_rtx_VAR_LOCATION (machine_mode mode, tree decl, rtx loc,
1083 : enum var_init_status status)
1084 : {
1085 107375816 : rtx x = gen_rtx_fmt_te (VAR_LOCATION, mode, decl, loc);
1086 107375816 : PAT_VAR_LOCATION_STATUS (x) = status;
1087 107375816 : return x;
1088 : }
1089 :
1090 :
1091 : /* Create an rtvec and stores within it the RTXen passed in the arguments. */
1092 :
1093 : rtvec
1094 25156843 : gen_rtvec (int n, ...)
1095 : {
1096 25156843 : int i;
1097 25156843 : rtvec rt_val;
1098 25156843 : va_list p;
1099 :
1100 25156843 : va_start (p, n);
1101 :
1102 : /* Don't allocate an empty rtvec... */
1103 25156843 : if (n == 0)
1104 : {
1105 0 : va_end (p);
1106 0 : return NULL_RTVEC;
1107 : }
1108 :
1109 25156843 : rt_val = rtvec_alloc (n);
1110 :
1111 95754323 : for (i = 0; i < n; i++)
1112 45440637 : rt_val->elem[i] = va_arg (p, rtx);
1113 :
1114 25156843 : va_end (p);
1115 25156843 : return rt_val;
1116 : }
1117 :
1118 : rtvec
1119 472345 : gen_rtvec_v (int n, rtx *argp)
1120 : {
1121 472345 : int i;
1122 472345 : rtvec rt_val;
1123 :
1124 : /* Don't allocate an empty rtvec... */
1125 472345 : if (n == 0)
1126 : return NULL_RTVEC;
1127 :
1128 472345 : rt_val = rtvec_alloc (n);
1129 :
1130 2149051 : for (i = 0; i < n; i++)
1131 1204361 : rt_val->elem[i] = *argp++;
1132 :
1133 : return rt_val;
1134 : }
1135 :
1136 : rtvec
1137 0 : gen_rtvec_v (int n, rtx_insn **argp)
1138 : {
1139 0 : int i;
1140 0 : rtvec rt_val;
1141 :
1142 : /* Don't allocate an empty rtvec... */
1143 0 : if (n == 0)
1144 : return NULL_RTVEC;
1145 :
1146 0 : rt_val = rtvec_alloc (n);
1147 :
1148 0 : for (i = 0; i < n; i++)
1149 0 : rt_val->elem[i] = *argp++;
1150 :
1151 : return rt_val;
1152 : }
1153 :
1154 :
1155 : /* Return the number of bytes between the start of an OUTER_MODE
1156 : in-memory value and the start of an INNER_MODE in-memory value,
1157 : given that the former is a lowpart of the latter. It may be a
1158 : paradoxical lowpart, in which case the offset will be negative
1159 : on big-endian targets. */
1160 :
1161 : poly_int64
1162 75259848 : byte_lowpart_offset (machine_mode outer_mode,
1163 : machine_mode inner_mode)
1164 : {
1165 75259848 : if (paradoxical_subreg_p (outer_mode, inner_mode))
1166 3925209 : return -subreg_lowpart_offset (inner_mode, outer_mode);
1167 : else
1168 71334639 : return subreg_lowpart_offset (outer_mode, inner_mode);
1169 : }
1170 :
1171 : /* Return the offset of (subreg:OUTER_MODE (mem:INNER_MODE X) OFFSET)
1172 : from address X. For paradoxical big-endian subregs this is a
1173 : negative value, otherwise it's the same as OFFSET. */
1174 :
1175 : poly_int64
1176 12562971 : subreg_memory_offset (machine_mode outer_mode, machine_mode inner_mode,
1177 : poly_uint64 offset)
1178 : {
1179 12562971 : if (paradoxical_subreg_p (outer_mode, inner_mode))
1180 : {
1181 3461269 : gcc_assert (known_eq (offset, 0U));
1182 3461269 : return -subreg_lowpart_offset (inner_mode, outer_mode);
1183 : }
1184 9101702 : return offset;
1185 : }
1186 :
1187 : /* As above, but return the offset that existing subreg X would have
1188 : if SUBREG_REG (X) were stored in memory. The only significant thing
1189 : about the current SUBREG_REG is its mode. */
1190 :
1191 : poly_int64
1192 682354 : subreg_memory_offset (const_rtx x)
1193 : {
1194 1364708 : return subreg_memory_offset (GET_MODE (x), GET_MODE (SUBREG_REG (x)),
1195 682354 : SUBREG_BYTE (x));
1196 : }
1197 :
1198 : /* Generate a REG rtx for a new pseudo register of mode MODE.
1199 : This pseudo is assigned the next sequential register number. */
1200 :
1201 : rtx
1202 69368741 : gen_reg_rtx (machine_mode mode)
1203 : {
1204 69368741 : rtx val;
1205 69368741 : unsigned int align = GET_MODE_ALIGNMENT (mode);
1206 :
1207 69368741 : gcc_assert (can_create_pseudo_p ());
1208 :
1209 : /* If a virtual register with bigger mode alignment is generated,
1210 : increase stack alignment estimation because it might be spilled
1211 : to stack later. */
1212 69368741 : if (SUPPORTS_STACK_ALIGNMENT
1213 69368741 : && crtl->stack_alignment_estimated < align
1214 376532 : && !crtl->stack_realign_processed)
1215 : {
1216 375458 : unsigned int min_align = MINIMUM_ALIGNMENT (NULL, mode, align);
1217 375458 : if (crtl->stack_alignment_estimated < min_align)
1218 375458 : crtl->stack_alignment_estimated = min_align;
1219 : }
1220 :
1221 69368741 : if (generating_concat_p
1222 52320428 : && (GET_MODE_CLASS (mode) == MODE_COMPLEX_FLOAT
1223 52320428 : || GET_MODE_CLASS (mode) == MODE_COMPLEX_INT))
1224 : {
1225 : /* For complex modes, don't make a single pseudo.
1226 : Instead, make a CONCAT of two pseudos.
1227 : This allows noncontiguous allocation of the real and imaginary parts,
1228 : which makes much better code. Besides, allocating DCmode
1229 : pseudos overstrains reload on some machines like the 386. */
1230 226007 : rtx realpart, imagpart;
1231 226007 : machine_mode partmode = GET_MODE_INNER (mode);
1232 :
1233 226007 : realpart = gen_reg_rtx (partmode);
1234 226007 : imagpart = gen_reg_rtx (partmode);
1235 226007 : return gen_rtx_CONCAT (mode, realpart, imagpart);
1236 : }
1237 :
1238 : /* Do not call gen_reg_rtx with uninitialized crtl. */
1239 69142734 : gcc_assert (crtl->emit.regno_pointer_align_length);
1240 :
1241 69142734 : crtl->emit.ensure_regno_capacity ();
1242 69142734 : gcc_assert (reg_rtx_no < crtl->emit.regno_pointer_align_length);
1243 :
1244 69142734 : val = gen_raw_REG (mode, reg_rtx_no);
1245 69142734 : regno_reg_rtx[reg_rtx_no++] = val;
1246 69142734 : return val;
1247 : }
1248 :
1249 : /* Make sure m_regno_pointer_align, and regno_reg_rtx are large
1250 : enough to have elements in the range 0 <= idx <= reg_rtx_no. */
1251 :
1252 : void
1253 69143960 : emit_status::ensure_regno_capacity ()
1254 : {
1255 69143960 : int old_size = regno_pointer_align_length;
1256 :
1257 69143960 : if (reg_rtx_no < old_size)
1258 : return;
1259 :
1260 182447 : int new_size = old_size * 2;
1261 182447 : while (reg_rtx_no >= new_size)
1262 0 : new_size *= 2;
1263 :
1264 182447 : char *tmp = XRESIZEVEC (char, regno_pointer_align, new_size);
1265 182447 : memset (tmp + old_size, 0, new_size - old_size);
1266 182447 : regno_pointer_align = (unsigned char *) tmp;
1267 :
1268 182447 : rtx *new1 = GGC_RESIZEVEC (rtx, regno_reg_rtx, new_size);
1269 182447 : memset (new1 + old_size, 0, (new_size - old_size) * sizeof (rtx));
1270 182447 : regno_reg_rtx = new1;
1271 :
1272 182447 : crtl->emit.regno_pointer_align_length = new_size;
1273 : }
1274 :
1275 : /* Return TRUE if REG is a PARM_DECL, FALSE otherwise. */
1276 :
1277 : bool
1278 12103 : reg_is_parm_p (rtx reg)
1279 : {
1280 12103 : tree decl;
1281 :
1282 12103 : gcc_assert (REG_P (reg));
1283 12103 : decl = REG_EXPR (reg);
1284 9657 : return (decl && TREE_CODE (decl) == PARM_DECL);
1285 : }
1286 :
1287 : /* Update NEW with the same attributes as REG, but with OFFSET added
1288 : to the REG_OFFSET. */
1289 :
1290 : static void
1291 18675708 : update_reg_offset (rtx new_rtx, rtx reg, poly_int64 offset)
1292 : {
1293 18675708 : REG_ATTRS (new_rtx) = get_reg_attrs (REG_EXPR (reg),
1294 18675708 : REG_OFFSET (reg) + offset);
1295 18675708 : }
1296 :
1297 : /* Generate a register with same attributes as REG, but with OFFSET
1298 : added to the REG_OFFSET. */
1299 :
1300 : rtx
1301 11069598 : gen_rtx_REG_offset (rtx reg, machine_mode mode, unsigned int regno,
1302 : poly_int64 offset)
1303 : {
1304 : /* Use gen_raw_REG rather than gen_rtx_REG, because otherwise we'd
1305 : overwrite REG_ATTRS (and in the callers often ORIGINAL_REGNO too)
1306 : of the shared REG rtxes like stack_pointer_rtx etc. This should
1307 : happen only for SUBREGs from DEBUG_INSNs, RA should ensure
1308 : multi-word registers don't overlap the special registers like
1309 : stack pointer. */
1310 11069598 : rtx new_rtx = gen_raw_REG (mode, regno);
1311 :
1312 11069598 : update_reg_offset (new_rtx, reg, offset);
1313 11069598 : return new_rtx;
1314 : }
1315 :
1316 : /* Generate a new pseudo-register with the same attributes as REG, but
1317 : with OFFSET added to the REG_OFFSET. */
1318 :
1319 : rtx
1320 630634 : gen_reg_rtx_offset (rtx reg, machine_mode mode, int offset)
1321 : {
1322 630634 : rtx new_rtx = gen_reg_rtx (mode);
1323 :
1324 630634 : update_reg_offset (new_rtx, reg, offset);
1325 630634 : return new_rtx;
1326 : }
1327 :
1328 : /* Adjust REG in-place so that it has mode MODE. It is assumed that the
1329 : new register is a (possibly paradoxical) lowpart of the old one. */
1330 :
1331 : void
1332 2892516 : adjust_reg_mode (rtx reg, machine_mode mode)
1333 : {
1334 2892516 : update_reg_offset (reg, reg, byte_lowpart_offset (mode, GET_MODE (reg)));
1335 2892516 : PUT_MODE (reg, mode);
1336 2892516 : }
1337 :
1338 : /* Copy REG's attributes from X, if X has any attributes. If REG and X
1339 : have different modes, REG is a (possibly paradoxical) lowpart of X. */
1340 :
1341 : void
1342 76120689 : set_reg_attrs_from_value (rtx reg, rtx x)
1343 : {
1344 76120689 : poly_int64 offset;
1345 76120689 : bool can_be_reg_pointer = true;
1346 :
1347 : /* Don't call mark_reg_pointer for incompatible pointer sign
1348 : extension. */
1349 76120689 : while (GET_CODE (x) == SIGN_EXTEND
1350 : || GET_CODE (x) == ZERO_EXTEND
1351 77174359 : || GET_CODE (x) == TRUNCATE
1352 77174359 : || (GET_CODE (x) == SUBREG && subreg_lowpart_p (x)))
1353 : {
1354 : #if defined(POINTERS_EXTEND_UNSIGNED)
1355 1053670 : if (((GET_CODE (x) == SIGN_EXTEND && POINTERS_EXTEND_UNSIGNED)
1356 : || (GET_CODE (x) == ZERO_EXTEND && ! POINTERS_EXTEND_UNSIGNED)
1357 491227 : || (paradoxical_subreg_p (x)
1358 18075 : && ! (SUBREG_PROMOTED_VAR_P (x)
1359 0 : && SUBREG_CHECK_PROMOTED_SIGN (x,
1360 : POINTERS_EXTEND_UNSIGNED))))
1361 1071745 : && !targetm.have_ptr_extend ())
1362 : can_be_reg_pointer = false;
1363 : #endif
1364 1053670 : x = XEXP (x, 0);
1365 : }
1366 :
1367 : /* Hard registers can be reused for multiple purposes within the same
1368 : function, so setting REG_ATTRS, REG_POINTER and REG_POINTER_ALIGN
1369 : on them is wrong. */
1370 76120689 : if (HARD_REGISTER_P (reg))
1371 49368739 : return;
1372 :
1373 26751950 : offset = byte_lowpart_offset (GET_MODE (reg), GET_MODE (x));
1374 26751950 : if (MEM_P (x))
1375 : {
1376 5723121 : if (MEM_OFFSET_KNOWN_P (x))
1377 2686248 : REG_ATTRS (reg) = get_reg_attrs (MEM_EXPR (x),
1378 2686248 : MEM_OFFSET (x) + offset);
1379 5262785 : if (can_be_reg_pointer && MEM_POINTER (x))
1380 663982 : mark_reg_pointer (reg, 0);
1381 : }
1382 21489165 : else if (REG_P (x))
1383 : {
1384 6960597 : if (REG_ATTRS (x))
1385 4082960 : update_reg_offset (reg, x, offset);
1386 6960597 : if (can_be_reg_pointer && REG_POINTER (x))
1387 1929262 : mark_reg_pointer (reg, REGNO_POINTER_ALIGN (REGNO (x)));
1388 : }
1389 : }
1390 :
1391 : /* Generate a REG rtx for a new pseudo register, copying the mode
1392 : and attributes from X. */
1393 :
1394 : rtx
1395 572418 : gen_reg_rtx_and_attrs (rtx x)
1396 : {
1397 572418 : rtx reg = gen_reg_rtx (GET_MODE (x));
1398 572418 : set_reg_attrs_from_value (reg, x);
1399 572418 : return reg;
1400 : }
1401 :
1402 : /* Set the register attributes for registers contained in PARM_RTX.
1403 : Use needed values from memory attributes of MEM. */
1404 :
1405 : void
1406 110335 : set_reg_attrs_for_parm (rtx parm_rtx, rtx mem)
1407 : {
1408 110335 : if (REG_P (parm_rtx))
1409 110335 : set_reg_attrs_from_value (parm_rtx, mem);
1410 0 : else if (GET_CODE (parm_rtx) == PARALLEL)
1411 : {
1412 : /* Check for a NULL entry in the first slot, used to indicate that the
1413 : parameter goes both on the stack and in registers. */
1414 0 : int i = XEXP (XVECEXP (parm_rtx, 0, 0), 0) ? 0 : 1;
1415 0 : for (; i < XVECLEN (parm_rtx, 0); i++)
1416 : {
1417 0 : rtx x = XVECEXP (parm_rtx, 0, i);
1418 0 : if (REG_P (XEXP (x, 0)))
1419 0 : REG_ATTRS (XEXP (x, 0))
1420 0 : = get_reg_attrs (MEM_EXPR (mem),
1421 0 : INTVAL (XEXP (x, 1)));
1422 : }
1423 : }
1424 110335 : }
1425 :
1426 : /* Set the REG_ATTRS for registers in value X, given that X represents
1427 : decl T. */
1428 :
1429 : void
1430 67882303 : set_reg_attrs_for_decl_rtl (tree t, rtx x)
1431 : {
1432 67882303 : if (!t)
1433 : return;
1434 67882107 : tree tdecl = t;
1435 67882107 : if (GET_CODE (x) == SUBREG)
1436 : {
1437 621 : gcc_assert (subreg_lowpart_p (x));
1438 621 : x = SUBREG_REG (x);
1439 : }
1440 67882107 : if (REG_P (x))
1441 44656942 : REG_ATTRS (x)
1442 44656942 : = get_reg_attrs (t, byte_lowpart_offset (GET_MODE (x),
1443 44656942 : DECL_P (tdecl)
1444 21824535 : ? DECL_MODE (tdecl)
1445 22832407 : : TYPE_MODE (TREE_TYPE (tdecl))));
1446 67882107 : if (GET_CODE (x) == CONCAT)
1447 : {
1448 229410 : if (REG_P (XEXP (x, 0)))
1449 229410 : REG_ATTRS (XEXP (x, 0)) = get_reg_attrs (t, 0);
1450 229410 : if (REG_P (XEXP (x, 1)))
1451 229410 : REG_ATTRS (XEXP (x, 1))
1452 458820 : = get_reg_attrs (t, GET_MODE_UNIT_SIZE (GET_MODE (XEXP (x, 0))));
1453 : }
1454 67882107 : if (GET_CODE (x) == PARALLEL)
1455 : {
1456 61912 : int i, start;
1457 :
1458 : /* Check for a NULL entry, used to indicate that the parameter goes
1459 : both on the stack and in registers. */
1460 61912 : if (XEXP (XVECEXP (x, 0, 0), 0))
1461 : start = 0;
1462 : else
1463 0 : start = 1;
1464 :
1465 177277 : for (i = start; i < XVECLEN (x, 0); i++)
1466 : {
1467 115365 : rtx y = XVECEXP (x, 0, i);
1468 115365 : if (REG_P (XEXP (y, 0)))
1469 115365 : REG_ATTRS (XEXP (y, 0)) = get_reg_attrs (t, INTVAL (XEXP (y, 1)));
1470 : }
1471 : }
1472 : }
1473 :
1474 : /* Assign the RTX X to declaration T. */
1475 :
1476 : void
1477 643686011 : set_decl_rtl (tree t, rtx x)
1478 : {
1479 643686011 : DECL_WRTL_CHECK (t)->decl_with_rtl.rtl = x;
1480 643686011 : if (x)
1481 34206265 : set_reg_attrs_for_decl_rtl (t, x);
1482 643686011 : }
1483 :
1484 : /* Assign the RTX X to parameter declaration T. BY_REFERENCE_P is true
1485 : if the ABI requires the parameter to be passed by reference. */
1486 :
1487 : void
1488 3256885 : set_decl_incoming_rtl (tree t, rtx x, bool by_reference_p)
1489 : {
1490 3256885 : DECL_INCOMING_RTL (t) = x;
1491 3256885 : if (x && !by_reference_p)
1492 3251919 : set_reg_attrs_for_decl_rtl (t, x);
1493 3256885 : }
1494 :
1495 : /* Identify REG (which may be a CONCAT) as a user register. */
1496 :
1497 : void
1498 6397059 : mark_user_reg (rtx reg)
1499 : {
1500 6397059 : if (GET_CODE (reg) == CONCAT)
1501 : {
1502 3346 : REG_USERVAR_P (XEXP (reg, 0)) = 1;
1503 3346 : REG_USERVAR_P (XEXP (reg, 1)) = 1;
1504 : }
1505 : else
1506 : {
1507 6393713 : gcc_assert (REG_P (reg));
1508 6393713 : REG_USERVAR_P (reg) = 1;
1509 : }
1510 6397059 : }
1511 :
1512 : /* Identify REG as a probable pointer register and show its alignment
1513 : as ALIGN, if nonzero. */
1514 :
1515 : void
1516 17563660 : mark_reg_pointer (rtx reg, int align)
1517 : {
1518 17563660 : if (! REG_POINTER (reg))
1519 : {
1520 11123029 : REG_POINTER (reg) = 1;
1521 :
1522 11123029 : if (align)
1523 9656670 : REGNO_POINTER_ALIGN (REGNO (reg)) = align;
1524 : }
1525 6440631 : else if (align && align < REGNO_POINTER_ALIGN (REGNO (reg)))
1526 : /* We can no-longer be sure just how aligned this pointer is. */
1527 1325816 : REGNO_POINTER_ALIGN (REGNO (reg)) = align;
1528 17563660 : }
1529 :
1530 : /* Return 1 plus largest pseudo reg number used in the current function. */
1531 :
1532 : int
1533 7171036264 : max_reg_num (void)
1534 : {
1535 7171036264 : return reg_rtx_no;
1536 : }
1537 :
1538 : /* Return 1 + the largest label number used so far in the current function. */
1539 :
1540 : int
1541 4107363 : max_label_num (void)
1542 : {
1543 4107363 : return label_num;
1544 : }
1545 :
1546 : /* Return first label number used in this function (if any were used). */
1547 :
1548 : int
1549 2588324 : get_first_label_num (void)
1550 : {
1551 2588324 : return first_label_num;
1552 : }
1553 :
1554 : /* If the rtx for label was created during the expansion of a nested
1555 : function, then first_label_num won't include this label number.
1556 : Fix this now so that array indices work later. */
1557 :
1558 : void
1559 27192 : maybe_set_first_label_num (rtx_code_label *x)
1560 : {
1561 27192 : if (CODE_LABEL_NUMBER (x) < first_label_num)
1562 400 : first_label_num = CODE_LABEL_NUMBER (x);
1563 27192 : }
1564 :
1565 : /* For use by the RTL function loader, when mingling with normal
1566 : functions.
1567 : Ensure that label_num is greater than the label num of X, to avoid
1568 : duplicate labels in the generated assembler. */
1569 :
1570 : void
1571 28 : maybe_set_max_label_num (rtx_code_label *x)
1572 : {
1573 28 : if (CODE_LABEL_NUMBER (x) >= label_num)
1574 24 : label_num = CODE_LABEL_NUMBER (x) + 1;
1575 28 : }
1576 :
1577 :
1578 : /* Return a value representing some low-order bits of X, where the number
1579 : of low-order bits is given by MODE. Note that no conversion is done
1580 : between floating-point and fixed-point values, rather, the bit
1581 : representation is returned.
1582 :
1583 : This function handles the cases in common between gen_lowpart, below,
1584 : and two variants in cse.cc and combine.cc. These are the cases that can
1585 : be safely handled at all points in the compilation.
1586 :
1587 : If this is not a case we can handle, return 0. */
1588 :
1589 : rtx
1590 54314739 : gen_lowpart_common (machine_mode mode, rtx x)
1591 : {
1592 108655108 : poly_uint64 msize = GET_MODE_SIZE (mode);
1593 54327554 : machine_mode innermode;
1594 :
1595 : /* Unfortunately, this routine doesn't take a parameter for the mode of X,
1596 : so we have to make one up. Yuk. */
1597 54327554 : innermode = GET_MODE (x);
1598 54327554 : if (CONST_INT_P (x)
1599 54327554 : && known_le (msize * BITS_PER_UNIT,
1600 : (unsigned HOST_WIDE_INT) HOST_BITS_PER_WIDE_INT))
1601 16042795 : innermode = int_mode_for_size (HOST_BITS_PER_WIDE_INT, 0).require ();
1602 38284759 : else if (innermode == VOIDmode)
1603 197634 : innermode = int_mode_for_size (HOST_BITS_PER_DOUBLE_INT, 0).require ();
1604 :
1605 54327554 : gcc_assert (innermode != VOIDmode && innermode != BLKmode);
1606 :
1607 54327554 : if (innermode == mode)
1608 : return x;
1609 :
1610 : /* The size of the outer and inner modes must be ordered. */
1611 63009074 : poly_uint64 xsize = GET_MODE_SIZE (innermode);
1612 31504537 : if (!ordered_p (msize, xsize))
1613 : return 0;
1614 :
1615 31504537 : if (SCALAR_FLOAT_MODE_P (mode))
1616 : {
1617 : /* Don't allow paradoxical FLOAT_MODE subregs. */
1618 267436 : if (maybe_gt (msize, xsize))
1619 : return 0;
1620 : }
1621 : else
1622 : {
1623 : /* MODE must occupy no more of the underlying registers than X. */
1624 31237101 : poly_uint64 regsize = REGMODE_NATURAL_SIZE (innermode);
1625 31237101 : unsigned int mregs, xregs;
1626 31237101 : if (!can_div_away_from_zero_p (msize, regsize, &mregs)
1627 31237101 : || !can_div_away_from_zero_p (xsize, regsize, &xregs)
1628 31237101 : || mregs > xregs)
1629 1243 : return 0;
1630 : }
1631 :
1632 31503294 : scalar_int_mode int_mode, int_innermode, from_mode;
1633 31503294 : if ((GET_CODE (x) == ZERO_EXTEND || GET_CODE (x) == SIGN_EXTEND)
1634 171871 : && is_a <scalar_int_mode> (mode, &int_mode)
1635 31503294 : && is_a <scalar_int_mode> (innermode, &int_innermode)
1636 31675117 : && is_a <scalar_int_mode> (GET_MODE (XEXP (x, 0)), &from_mode))
1637 : {
1638 : /* If we are getting the low-order part of something that has been
1639 : sign- or zero-extended, we can either just use the object being
1640 : extended or make a narrower extension. If we want an even smaller
1641 : piece than the size of the object being extended, call ourselves
1642 : recursively.
1643 :
1644 : This case is used mostly by combine and cse. */
1645 :
1646 171823 : if (from_mode == int_mode)
1647 : return XEXP (x, 0);
1648 50262 : else if (GET_MODE_SIZE (int_mode) < GET_MODE_SIZE (from_mode))
1649 : return gen_lowpart_common (int_mode, XEXP (x, 0));
1650 11817 : else if (GET_MODE_SIZE (int_mode) < GET_MODE_SIZE (int_innermode))
1651 3624 : return gen_rtx_fmt_e (GET_CODE (x), int_mode, XEXP (x, 0));
1652 : }
1653 31331471 : else if (GET_CODE (x) == SUBREG || REG_P (x)
1654 : || GET_CODE (x) == CONCAT || GET_CODE (x) == CONST_VECTOR
1655 17608650 : || CONST_DOUBLE_AS_FLOAT_P (x) || CONST_SCALAR_INT_P (x)
1656 : || CONST_POLY_INT_P (x))
1657 20980445 : return lowpart_subreg (mode, x, innermode);
1658 :
1659 : /* Otherwise, we can't do this. */
1660 : return 0;
1661 : }
1662 :
1663 : rtx
1664 11743 : gen_highpart (machine_mode mode, rtx x)
1665 : {
1666 23486 : poly_uint64 msize = GET_MODE_SIZE (mode);
1667 11743 : rtx result;
1668 :
1669 : /* This case loses if X is a subreg. To catch bugs early,
1670 : complain if an invalid MODE is used even in other cases. */
1671 13481 : gcc_assert (known_le (msize, (unsigned int) UNITS_PER_WORD)
1672 : || known_eq (msize, GET_MODE_UNIT_SIZE (GET_MODE (x))));
1673 :
1674 : /* gen_lowpart_common handles a lot of special cases due to needing to handle
1675 : paradoxical subregs; it only calls simplify_gen_subreg when certain that
1676 : it will produce something meaningful. The only case we need to handle
1677 : specially here is MEM. */
1678 11743 : if (MEM_P (x))
1679 : {
1680 111 : poly_int64 offset = subreg_highpart_offset (mode, GET_MODE (x));
1681 111 : return adjust_address (x, mode, offset);
1682 : }
1683 :
1684 11632 : result = simplify_gen_subreg (mode, x, GET_MODE (x),
1685 11632 : subreg_highpart_offset (mode, GET_MODE (x)));
1686 : /* Since we handle MEM directly above, we should never get a MEM back
1687 : from simplify_gen_subreg. */
1688 11632 : gcc_assert (result && !MEM_P (result));
1689 :
1690 : return result;
1691 : }
1692 :
1693 : /* Like gen_highpart, but accept mode of EXP operand in case EXP can
1694 : be VOIDmode constant. */
1695 : rtx
1696 0 : gen_highpart_mode (machine_mode outermode, machine_mode innermode, rtx exp)
1697 : {
1698 0 : if (GET_MODE (exp) != VOIDmode)
1699 : {
1700 0 : gcc_assert (GET_MODE (exp) == innermode);
1701 0 : return gen_highpart (outermode, exp);
1702 : }
1703 0 : return simplify_gen_subreg (outermode, exp, innermode,
1704 0 : subreg_highpart_offset (outermode, innermode));
1705 : }
1706 :
1707 : /* Return the SUBREG_BYTE for a lowpart subreg whose outer mode has
1708 : OUTER_BYTES bytes and whose inner mode has INNER_BYTES bytes. */
1709 :
1710 : poly_uint64
1711 239345036 : subreg_size_lowpart_offset (poly_uint64 outer_bytes, poly_uint64 inner_bytes)
1712 : {
1713 239345036 : gcc_checking_assert (ordered_p (outer_bytes, inner_bytes));
1714 239345036 : if (maybe_gt (outer_bytes, inner_bytes))
1715 : /* Paradoxical subregs always have a SUBREG_BYTE of 0. */
1716 : return 0;
1717 :
1718 239345036 : if (BYTES_BIG_ENDIAN && WORDS_BIG_ENDIAN)
1719 : return inner_bytes - outer_bytes;
1720 239345036 : else if (!BYTES_BIG_ENDIAN && !WORDS_BIG_ENDIAN)
1721 239345036 : return 0;
1722 : else
1723 : return subreg_size_offset_from_lsb (outer_bytes, inner_bytes, 0);
1724 : }
1725 :
1726 : /* Return the SUBREG_BYTE for a highpart subreg whose outer mode has
1727 : OUTER_BYTES bytes and whose inner mode has INNER_BYTES bytes. */
1728 :
1729 : poly_uint64
1730 46603 : subreg_size_highpart_offset (poly_uint64 outer_bytes, poly_uint64 inner_bytes)
1731 : {
1732 46603 : gcc_assert (known_ge (inner_bytes, outer_bytes));
1733 :
1734 : if (BYTES_BIG_ENDIAN && WORDS_BIG_ENDIAN)
1735 : return 0;
1736 : else if (!BYTES_BIG_ENDIAN && !WORDS_BIG_ENDIAN)
1737 46603 : return inner_bytes - outer_bytes;
1738 : else
1739 : return subreg_size_offset_from_lsb (outer_bytes, inner_bytes,
1740 : (inner_bytes - outer_bytes)
1741 : * BITS_PER_UNIT);
1742 : }
1743 :
1744 : /* Return true iff X, assumed to be a SUBREG,
1745 : refers to the least significant part of its containing reg.
1746 : If X is not a SUBREG, always return true (it is its own low part!). */
1747 :
1748 : bool
1749 56062115 : subreg_lowpart_p (const_rtx x)
1750 : {
1751 56062115 : if (GET_CODE (x) != SUBREG)
1752 : return true;
1753 56062115 : else if (GET_MODE (SUBREG_REG (x)) == VOIDmode)
1754 : return false;
1755 :
1756 56062115 : return known_eq (subreg_lowpart_offset (GET_MODE (x),
1757 : GET_MODE (SUBREG_REG (x))),
1758 : SUBREG_BYTE (x));
1759 : }
1760 :
1761 : /* Return subword OFFSET of operand OP.
1762 : The word number, OFFSET, is interpreted as the word number starting
1763 : at the low-order address. OFFSET 0 is the low-order word if not
1764 : WORDS_BIG_ENDIAN, otherwise it is the high-order word.
1765 :
1766 : If we cannot extract the required word, we return zero. Otherwise,
1767 : an rtx corresponding to the requested word will be returned.
1768 :
1769 : VALIDATE_ADDRESS is nonzero if the address should be validated. Before
1770 : reload has completed, a valid address will always be returned. After
1771 : reload, if a valid address cannot be returned, we return zero.
1772 :
1773 : If VALIDATE_ADDRESS is zero, we simply form the required address; validating
1774 : it is the responsibility of the caller.
1775 :
1776 : MODE is the mode of OP in case it is a CONST_INT.
1777 :
1778 : ??? This is still rather broken for some cases. The problem for the
1779 : moment is that all callers of this thing provide no 'goal mode' to
1780 : tell us to work with. This exists because all callers were written
1781 : in a word based SUBREG world.
1782 : Now use of this function can be deprecated by simplify_subreg in most
1783 : cases.
1784 : */
1785 :
1786 : rtx
1787 89711 : operand_subword (rtx op, poly_uint64 offset, int validate_address,
1788 : machine_mode mode)
1789 : {
1790 89711 : if (mode == VOIDmode)
1791 3038 : mode = GET_MODE (op);
1792 :
1793 3038 : gcc_assert (mode != VOIDmode);
1794 :
1795 : /* If OP is narrower than a word, fail. */
1796 89711 : if (mode != BLKmode
1797 194559 : && maybe_lt (GET_MODE_SIZE (mode), UNITS_PER_WORD))
1798 : return 0;
1799 :
1800 : /* If we want a word outside OP, return zero. */
1801 89711 : if (mode != BLKmode
1802 194559 : && maybe_gt ((offset + 1) * UNITS_PER_WORD, GET_MODE_SIZE (mode)))
1803 0 : return const0_rtx;
1804 :
1805 : /* Form a new MEM at the requested address. */
1806 89711 : if (MEM_P (op))
1807 : {
1808 11285 : rtx new_rtx = adjust_address_nv (op, word_mode, offset * UNITS_PER_WORD);
1809 :
1810 9949 : if (! validate_address)
1811 : return new_rtx;
1812 :
1813 9949 : else if (reload_completed)
1814 : {
1815 0 : if (! strict_memory_address_addr_space_p (word_mode,
1816 : XEXP (new_rtx, 0),
1817 0 : MEM_ADDR_SPACE (op)))
1818 : return 0;
1819 : }
1820 : else
1821 9949 : return replace_equiv_address (new_rtx, XEXP (new_rtx, 0));
1822 : }
1823 :
1824 : /* Rest can be handled by simplify_subreg. */
1825 96674 : return simplify_gen_subreg (word_mode, op, mode, (offset * UNITS_PER_WORD));
1826 : }
1827 :
1828 : /* Similar to `operand_subword', but never return 0. If we can't
1829 : extract the required subword, put OP into a register and try again.
1830 : The second attempt must succeed. We always validate the address in
1831 : this case.
1832 :
1833 : MODE is the mode of OP, in case it is CONST_INT. */
1834 :
1835 : rtx
1836 67688 : operand_subword_force (rtx op, poly_uint64 offset, machine_mode mode)
1837 : {
1838 67688 : rtx result = operand_subword (op, offset, 1, mode);
1839 :
1840 67688 : if (result)
1841 : return result;
1842 :
1843 0 : if (mode != BLKmode && mode != VOIDmode)
1844 : {
1845 : /* If this is a register which cannot be accessed by words, copy it
1846 : to a pseudo register. */
1847 0 : if (REG_P (op))
1848 0 : op = copy_to_reg (op);
1849 : else
1850 0 : op = force_reg (mode, op);
1851 : }
1852 :
1853 0 : result = operand_subword (op, offset, 1, mode);
1854 0 : gcc_assert (result);
1855 :
1856 : return result;
1857 : }
1858 :
1859 32448615 : mem_attrs::mem_attrs ()
1860 32448615 : : expr (NULL_TREE),
1861 32448615 : offset (0),
1862 32448615 : size (0),
1863 32448615 : alias (0),
1864 32448615 : align (0),
1865 32448615 : addrspace (ADDR_SPACE_GENERIC),
1866 32448615 : offset_known_p (false),
1867 32448615 : size_known_p (false)
1868 32448615 : {}
1869 :
1870 : /* Returns true if both MEM_EXPR can be considered equal
1871 : and false otherwise. */
1872 :
1873 : bool
1874 45388 : mem_expr_equal_p (const_tree expr1, const_tree expr2)
1875 : {
1876 45388 : if (expr1 == expr2)
1877 : return true;
1878 :
1879 44783 : if (! expr1 || ! expr2)
1880 : return false;
1881 :
1882 34826 : if (TREE_CODE (expr1) != TREE_CODE (expr2))
1883 : return false;
1884 :
1885 33216 : return operand_equal_p (expr1, expr2, 0);
1886 : }
1887 :
1888 : /* Return OFFSET if XEXP (MEM, 0) - OFFSET is known to be ALIGN
1889 : bits aligned for 0 <= OFFSET < ALIGN / BITS_PER_UNIT, or
1890 : -1 if not known. */
1891 :
1892 : int
1893 22 : get_mem_align_offset (rtx mem, unsigned int align)
1894 : {
1895 22 : tree expr;
1896 22 : poly_uint64 offset;
1897 :
1898 : /* This function can't use
1899 : if (!MEM_EXPR (mem) || !MEM_OFFSET_KNOWN_P (mem)
1900 : || (MAX (MEM_ALIGN (mem),
1901 : MAX (align, get_object_alignment (MEM_EXPR (mem))))
1902 : < align))
1903 : return -1;
1904 : else
1905 : return (- MEM_OFFSET (mem)) & (align / BITS_PER_UNIT - 1);
1906 : for two reasons:
1907 : - COMPONENT_REFs in MEM_EXPR can have NULL first operand,
1908 : for <variable>. get_inner_reference doesn't handle it and
1909 : even if it did, the alignment in that case needs to be determined
1910 : from DECL_FIELD_CONTEXT's TYPE_ALIGN.
1911 : - it would do suboptimal job for COMPONENT_REFs, even if MEM_EXPR
1912 : isn't sufficiently aligned, the object it is in might be. */
1913 22 : gcc_assert (MEM_P (mem));
1914 22 : expr = MEM_EXPR (mem);
1915 22 : if (expr == NULL_TREE || !MEM_OFFSET_KNOWN_P (mem))
1916 : return -1;
1917 :
1918 21 : offset = MEM_OFFSET (mem);
1919 21 : if (DECL_P (expr))
1920 : {
1921 2 : if (DECL_ALIGN (expr) < align)
1922 : return -1;
1923 : }
1924 19 : else if (INDIRECT_REF_P (expr))
1925 : {
1926 0 : if (TYPE_ALIGN (TREE_TYPE (expr)) < (unsigned int) align)
1927 : return -1;
1928 : }
1929 19 : else if (TREE_CODE (expr) == COMPONENT_REF)
1930 : {
1931 0 : while (1)
1932 : {
1933 0 : tree inner = TREE_OPERAND (expr, 0);
1934 0 : tree field = TREE_OPERAND (expr, 1);
1935 0 : tree byte_offset = component_ref_field_offset (expr);
1936 0 : tree bit_offset = DECL_FIELD_BIT_OFFSET (field);
1937 :
1938 0 : poly_uint64 suboffset;
1939 0 : if (!byte_offset
1940 0 : || !poly_int_tree_p (byte_offset, &suboffset)
1941 0 : || !tree_fits_uhwi_p (bit_offset))
1942 0 : return -1;
1943 :
1944 0 : offset += suboffset;
1945 0 : offset += tree_to_uhwi (bit_offset) / BITS_PER_UNIT;
1946 :
1947 0 : if (inner == NULL_TREE)
1948 : {
1949 0 : if (TYPE_ALIGN (DECL_FIELD_CONTEXT (field))
1950 : < (unsigned int) align)
1951 : return -1;
1952 0 : break;
1953 : }
1954 0 : else if (DECL_P (inner))
1955 : {
1956 0 : if (DECL_ALIGN (inner) < align)
1957 : return -1;
1958 : break;
1959 : }
1960 0 : else if (TREE_CODE (inner) != COMPONENT_REF)
1961 : return -1;
1962 0 : expr = inner;
1963 0 : }
1964 : }
1965 : else
1966 : return -1;
1967 :
1968 0 : HOST_WIDE_INT misalign;
1969 0 : if (!known_misalignment (offset, align / BITS_PER_UNIT, &misalign))
1970 : return -1;
1971 0 : return misalign;
1972 : }
1973 :
1974 : /* Given REF (a MEM) and T, either the type of X or the expression
1975 : corresponding to REF, set the memory attributes. OBJECTP is nonzero
1976 : if we are making a new object of this type. BITPOS is nonzero if
1977 : there is an offset outstanding on T that will be applied later.
1978 : MAY_STORE_P is true when REF can be the destination of a store. */
1979 :
1980 : void
1981 32448615 : set_mem_attributes_minus_bitpos (rtx ref, tree t, int objectp,
1982 : poly_int64 bitpos, bool may_store_p)
1983 : {
1984 32448615 : poly_int64 apply_bitpos = 0;
1985 32448615 : tree type;
1986 32448615 : class mem_attrs attrs, *defattrs, *refattrs;
1987 32448615 : addr_space_t as;
1988 :
1989 : /* It can happen that type_for_mode was given a mode for which there
1990 : is no language-level type. In which case it returns NULL, which
1991 : we can see here. */
1992 32448615 : if (t == NULL_TREE)
1993 0 : return;
1994 :
1995 32448615 : type = TYPE_P (t) ? t : TREE_TYPE (t);
1996 32448615 : if (type == error_mark_node)
1997 : return;
1998 :
1999 : /* If we have already set DECL_RTL = ref, get_alias_set will get the
2000 : wrong answer, as it assumes that DECL_RTL already has the right alias
2001 : info. Callers should not set DECL_RTL until after the call to
2002 : set_mem_attributes. */
2003 32448615 : gcc_assert (!DECL_P (t) || ref != DECL_RTL_IF_SET (t));
2004 :
2005 : /* Get the alias set from the expression or type (perhaps using a
2006 : front-end routine) and use it. */
2007 32448615 : attrs.alias = get_alias_set (t);
2008 :
2009 32448615 : MEM_VOLATILE_P (ref) |= TYPE_VOLATILE (type);
2010 32448615 : MEM_POINTER (ref) = POINTER_TYPE_P (type);
2011 :
2012 : /* Default values from pre-existing memory attributes if present. */
2013 32448615 : refattrs = MEM_ATTRS (ref);
2014 32448615 : if (refattrs)
2015 : {
2016 : /* ??? Can this ever happen? Calling this routine on a MEM that
2017 : already carries memory attributes should probably be invalid. */
2018 13154327 : attrs.expr = refattrs->expr;
2019 13154327 : attrs.offset_known_p = refattrs->offset_known_p;
2020 13154327 : attrs.offset = refattrs->offset;
2021 13154327 : attrs.size_known_p = refattrs->size_known_p;
2022 13154327 : attrs.size = refattrs->size;
2023 13154327 : attrs.align = refattrs->align;
2024 : }
2025 :
2026 : /* Otherwise, default values from the mode of the MEM reference. */
2027 : else
2028 : {
2029 19294288 : defattrs = mode_mem_attrs[(int) GET_MODE (ref)];
2030 19294288 : gcc_assert (!defattrs->expr);
2031 19294288 : gcc_assert (!defattrs->offset_known_p);
2032 :
2033 : /* Respect mode size. */
2034 19294288 : attrs.size_known_p = defattrs->size_known_p;
2035 19294288 : attrs.size = defattrs->size;
2036 : /* ??? Is this really necessary? We probably should always get
2037 : the size from the type below. */
2038 :
2039 : /* Respect mode alignment for STRICT_ALIGNMENT targets if T is a type;
2040 : if T is an object, always compute the object alignment below. */
2041 19294288 : if (TYPE_P (t))
2042 2041520 : attrs.align = defattrs->align;
2043 : else
2044 17252768 : attrs.align = BITS_PER_UNIT;
2045 : /* ??? If T is a type, respecting mode alignment may *also* be wrong
2046 : e.g. if the type carries an alignment attribute. Should we be
2047 : able to simply always use TYPE_ALIGN? */
2048 : }
2049 :
2050 : /* We can set the alignment from the type if we are making an object or if
2051 : this is an INDIRECT_REF. */
2052 32448615 : if (objectp || TREE_CODE (t) == INDIRECT_REF)
2053 10000813 : attrs.align = MAX (attrs.align, TYPE_ALIGN (type));
2054 :
2055 : /* If the size is known, we can set that. */
2056 32448615 : tree new_size = TYPE_SIZE_UNIT (type);
2057 :
2058 : /* The address-space is that of the type. */
2059 32448615 : as = TYPE_ADDR_SPACE (type);
2060 :
2061 : /* If T is not a type, we may be able to deduce some more information about
2062 : the expression. */
2063 32448615 : if (! TYPE_P (t))
2064 : {
2065 30400773 : tree base;
2066 :
2067 30400773 : if (TREE_THIS_VOLATILE (t))
2068 246920 : MEM_VOLATILE_P (ref) = 1;
2069 :
2070 : /* Now remove any conversions: they don't change what the underlying
2071 : object is. Likewise for SAVE_EXPR. */
2072 30405596 : while (CONVERT_EXPR_P (t)
2073 : || TREE_CODE (t) == VIEW_CONVERT_EXPR
2074 30405596 : || TREE_CODE (t) == SAVE_EXPR)
2075 4823 : t = TREE_OPERAND (t, 0);
2076 :
2077 : /* Note whether this expression can trap. */
2078 30400773 : MEM_NOTRAP_P (ref)
2079 30400773 : = !(may_store_p ? lhs_could_trap_p (t) : tree_could_trap_p (t));
2080 :
2081 30400773 : base = get_base_address (t);
2082 30400773 : if (base)
2083 : {
2084 30400773 : if (DECL_P (base)
2085 17446761 : && TREE_READONLY (base)
2086 2247210 : && (TREE_STATIC (base) || DECL_EXTERNAL (base))
2087 32295733 : && !TREE_THIS_VOLATILE (base))
2088 1894581 : MEM_READONLY_P (ref) = 1;
2089 :
2090 : /* Mark static const strings readonly as well. */
2091 30400773 : if (TREE_CODE (base) == STRING_CST
2092 287620 : && TREE_READONLY (base)
2093 30663238 : && TREE_STATIC (base))
2094 262465 : MEM_READONLY_P (ref) = 1;
2095 :
2096 : /* Address-space information is on the base object. */
2097 30400773 : if (TREE_CODE (base) == MEM_REF
2098 30400773 : || TREE_CODE (base) == TARGET_MEM_REF)
2099 12666380 : as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (base,
2100 : 0))));
2101 : else
2102 17734393 : as = TYPE_ADDR_SPACE (TREE_TYPE (base));
2103 : }
2104 :
2105 : /* If this expression uses it's parent's alias set, mark it such
2106 : that we won't change it. */
2107 30400773 : if (component_uses_parent_alias_set_from (t) != NULL_TREE)
2108 1150829 : MEM_KEEP_ALIAS_SET_P (ref) = 1;
2109 :
2110 : /* If this is a decl, set the attributes of the MEM from it. */
2111 30400773 : if (DECL_P (t))
2112 : {
2113 7986062 : attrs.expr = t;
2114 7986062 : attrs.offset_known_p = true;
2115 7986062 : attrs.offset = 0;
2116 7986062 : apply_bitpos = bitpos;
2117 7986062 : new_size = DECL_SIZE_UNIT (t);
2118 : }
2119 :
2120 : /* ??? If we end up with a constant or a descriptor do not
2121 : record a MEM_EXPR. */
2122 22414711 : else if (CONSTANT_CLASS_P (t)
2123 22316684 : || TREE_CODE (t) == CONSTRUCTOR)
2124 : ;
2125 :
2126 : /* If this is a field reference, record it. */
2127 22316684 : else if (TREE_CODE (t) == COMPONENT_REF)
2128 : {
2129 10067449 : attrs.expr = t;
2130 10067449 : attrs.offset_known_p = true;
2131 10067449 : attrs.offset = 0;
2132 10067449 : apply_bitpos = bitpos;
2133 10067449 : if (DECL_BIT_FIELD (TREE_OPERAND (t, 1)))
2134 66388 : new_size = DECL_SIZE_UNIT (TREE_OPERAND (t, 1));
2135 : }
2136 :
2137 : /* Else record it. */
2138 : else
2139 : {
2140 12249235 : gcc_assert (handled_component_p (t)
2141 : || TREE_CODE (t) == MEM_REF
2142 : || TREE_CODE (t) == TARGET_MEM_REF);
2143 12249235 : attrs.expr = t;
2144 12249235 : attrs.offset_known_p = true;
2145 12249235 : attrs.offset = 0;
2146 12249235 : apply_bitpos = bitpos;
2147 : }
2148 :
2149 : /* If this is a reference based on a partitioned decl replace the
2150 : base with a MEM_REF of the pointer representative we created
2151 : during stack slot partitioning. */
2152 30400773 : if (attrs.expr
2153 30302746 : && VAR_P (base)
2154 14322553 : && ! is_global_var (base)
2155 39254939 : && cfun->gimple_df->decls_to_pointers != NULL)
2156 : {
2157 4097450 : tree *namep = cfun->gimple_df->decls_to_pointers->get (base);
2158 4097450 : if (namep)
2159 : {
2160 2393778 : attrs.expr = unshare_expr (attrs.expr);
2161 2393778 : tree *orig_base = &attrs.expr;
2162 5020577 : while (handled_component_p (*orig_base))
2163 2626799 : orig_base = &TREE_OPERAND (*orig_base, 0);
2164 2393778 : if (TREE_CODE (*orig_base) == MEM_REF
2165 2393778 : || TREE_CODE (*orig_base) == TARGET_MEM_REF)
2166 727460 : TREE_OPERAND (*orig_base, 0) = *namep;
2167 : else
2168 : {
2169 1666318 : tree aptrt = reference_alias_ptr_type (*orig_base);
2170 1666318 : *orig_base = build2 (MEM_REF, TREE_TYPE (*orig_base),
2171 : *namep, build_int_cst (aptrt, 0));
2172 : }
2173 : }
2174 : }
2175 :
2176 : /* Compute the alignment. */
2177 30400773 : unsigned int obj_align;
2178 30400773 : unsigned HOST_WIDE_INT obj_bitpos;
2179 30400773 : get_object_alignment_1 (t, &obj_align, &obj_bitpos);
2180 30400773 : unsigned int diff_align = known_alignment (obj_bitpos - bitpos);
2181 30400773 : if (diff_align != 0)
2182 3996399 : obj_align = MIN (obj_align, diff_align);
2183 30400773 : attrs.align = MAX (attrs.align, obj_align);
2184 : }
2185 :
2186 32448615 : poly_uint64 const_size;
2187 32448615 : if (poly_int_tree_p (new_size, &const_size))
2188 : {
2189 31805003 : attrs.size_known_p = true;
2190 31805003 : attrs.size = const_size;
2191 : }
2192 :
2193 : /* If we modified OFFSET based on T, then subtract the outstanding
2194 : bit position offset. Similarly, increase the size of the accessed
2195 : object to contain the negative offset. */
2196 32448615 : if (maybe_ne (apply_bitpos, 0))
2197 : {
2198 2888752 : gcc_assert (attrs.offset_known_p);
2199 2888752 : poly_int64 bytepos = bits_to_bytes_round_down (apply_bitpos);
2200 2888752 : attrs.offset -= bytepos;
2201 2888752 : if (attrs.size_known_p)
2202 32448615 : attrs.size += bytepos;
2203 : }
2204 :
2205 : /* Now set the attributes we computed above. */
2206 32448615 : attrs.addrspace = as;
2207 32448615 : set_mem_attrs (ref, &attrs);
2208 : }
2209 :
2210 : void
2211 28239550 : set_mem_attributes (rtx ref, tree t, int objectp, bool may_store_p)
2212 : {
2213 28239550 : set_mem_attributes_minus_bitpos (ref, t, objectp, 0, may_store_p);
2214 28239550 : }
2215 :
2216 : /* Set the alias set of MEM to SET. */
2217 :
2218 : void
2219 7929287 : set_mem_alias_set (rtx mem, alias_set_type set)
2220 : {
2221 : /* If the new and old alias sets don't conflict, something is wrong. */
2222 11071570 : gcc_checking_assert (alias_sets_conflict_p (set, MEM_ALIAS_SET (mem)));
2223 7929287 : mem_attrs attrs (*get_mem_attrs (mem));
2224 7929287 : attrs.alias = set;
2225 7929287 : set_mem_attrs (mem, &attrs);
2226 7929287 : }
2227 :
2228 : /* Set the address space of MEM to ADDRSPACE (target-defined). */
2229 :
2230 : void
2231 10341562 : set_mem_addr_space (rtx mem, addr_space_t addrspace)
2232 : {
2233 10341562 : mem_attrs attrs (*get_mem_attrs (mem));
2234 10341562 : attrs.addrspace = addrspace;
2235 10341562 : set_mem_attrs (mem, &attrs);
2236 10341562 : }
2237 :
2238 : /* Set the alignment of MEM to ALIGN bits. */
2239 :
2240 : void
2241 12829456 : set_mem_align (rtx mem, unsigned int align)
2242 : {
2243 12829456 : mem_attrs attrs (*get_mem_attrs (mem));
2244 12829456 : attrs.align = align;
2245 12829456 : set_mem_attrs (mem, &attrs);
2246 12829456 : }
2247 :
2248 : /* Set the expr for MEM to EXPR. */
2249 :
2250 : void
2251 7126041 : set_mem_expr (rtx mem, tree expr)
2252 : {
2253 7126041 : mem_attrs attrs (*get_mem_attrs (mem));
2254 7126041 : attrs.expr = expr;
2255 7126041 : set_mem_attrs (mem, &attrs);
2256 7126041 : }
2257 :
2258 : /* Set the offset of MEM to OFFSET. */
2259 :
2260 : void
2261 134 : set_mem_offset (rtx mem, poly_int64 offset)
2262 : {
2263 134 : mem_attrs attrs (*get_mem_attrs (mem));
2264 134 : attrs.offset_known_p = true;
2265 134 : attrs.offset = offset;
2266 134 : set_mem_attrs (mem, &attrs);
2267 134 : }
2268 :
2269 : /* Clear the offset of MEM. */
2270 :
2271 : void
2272 91381 : clear_mem_offset (rtx mem)
2273 : {
2274 91381 : mem_attrs attrs (*get_mem_attrs (mem));
2275 91381 : attrs.offset_known_p = false;
2276 91381 : set_mem_attrs (mem, &attrs);
2277 91381 : }
2278 :
2279 : /* Set the size of MEM to SIZE. */
2280 :
2281 : void
2282 2503489 : set_mem_size (rtx mem, poly_int64 size)
2283 : {
2284 2503489 : mem_attrs attrs (*get_mem_attrs (mem));
2285 2503489 : attrs.size_known_p = true;
2286 2503489 : attrs.size = size;
2287 2503489 : set_mem_attrs (mem, &attrs);
2288 2503489 : }
2289 :
2290 : /* Clear the size of MEM. */
2291 :
2292 : void
2293 615 : clear_mem_size (rtx mem)
2294 : {
2295 615 : mem_attrs attrs (*get_mem_attrs (mem));
2296 615 : attrs.size_known_p = false;
2297 615 : set_mem_attrs (mem, &attrs);
2298 615 : }
2299 :
2300 : /* Return a memory reference like MEMREF, but with its mode changed to MODE
2301 : and its address changed to ADDR. (VOIDmode means don't change the mode.
2302 : NULL for ADDR means don't change the address.) VALIDATE is nonzero if the
2303 : returned memory location is required to be valid. INPLACE is true if any
2304 : changes can be made directly to MEMREF or false if MEMREF must be treated
2305 : as immutable.
2306 :
2307 : The memory attributes are not changed. */
2308 :
2309 : static rtx
2310 507442941 : change_address_1 (rtx memref, machine_mode mode, rtx addr, int validate,
2311 : bool inplace)
2312 : {
2313 507442941 : addr_space_t as;
2314 507442941 : rtx new_rtx;
2315 :
2316 507442941 : gcc_assert (MEM_P (memref));
2317 507442941 : as = MEM_ADDR_SPACE (memref);
2318 507442941 : if (mode == VOIDmode)
2319 480274519 : mode = GET_MODE (memref);
2320 507442941 : if (addr == 0)
2321 64 : addr = XEXP (memref, 0);
2322 486853137 : if (mode == GET_MODE (memref) && addr == XEXP (memref, 0)
2323 853231137 : && (!validate || memory_address_addr_space_p (mode, addr, as)))
2324 : return memref;
2325 :
2326 : /* Don't validate address for LRA. LRA can make the address valid
2327 : by itself in most efficient way. */
2328 161808474 : if (validate && !lra_in_progress)
2329 : {
2330 23496591 : if (reload_in_progress || reload_completed)
2331 3870853 : gcc_assert (memory_address_addr_space_p (mode, addr, as));
2332 : else
2333 19625738 : addr = memory_address_addr_space (mode, addr, as);
2334 : }
2335 :
2336 161808474 : if (rtx_equal_p (addr, XEXP (memref, 0)) && mode == GET_MODE (memref))
2337 : return memref;
2338 :
2339 157653543 : if (inplace)
2340 : {
2341 261256 : XEXP (memref, 0) = addr;
2342 261256 : return memref;
2343 : }
2344 :
2345 157392287 : new_rtx = gen_rtx_MEM (mode, addr);
2346 157392287 : MEM_COPY_ATTRIBUTES (new_rtx, memref);
2347 157392287 : return new_rtx;
2348 : }
2349 :
2350 : /* Version of change_address that takes a validate parameter. */
2351 :
2352 : static rtx
2353 254825 : change_address (rtx memref, machine_mode mode, rtx addr, int validate)
2354 : {
2355 254825 : rtx new_rtx = change_address_1 (memref, mode, addr, validate, false);
2356 254825 : machine_mode mmode = GET_MODE (new_rtx);
2357 254825 : class mem_attrs *defattrs;
2358 :
2359 254825 : mem_attrs attrs (*get_mem_attrs (memref));
2360 254825 : defattrs = mode_mem_attrs[(int) mmode];
2361 254825 : attrs.expr = NULL_TREE;
2362 254825 : attrs.offset_known_p = false;
2363 254825 : attrs.size_known_p = defattrs->size_known_p;
2364 254825 : attrs.size = defattrs->size;
2365 254825 : attrs.align = defattrs->align;
2366 :
2367 : /* If there are no changes, just return the original memory reference. */
2368 254825 : if (new_rtx == memref)
2369 : {
2370 57235 : if (mem_attrs_eq_p (get_mem_attrs (memref), &attrs))
2371 : return new_rtx;
2372 :
2373 56137 : new_rtx = gen_rtx_MEM (mmode, XEXP (memref, 0));
2374 56137 : MEM_COPY_ATTRIBUTES (new_rtx, memref);
2375 : }
2376 :
2377 253727 : set_mem_attrs (new_rtx, &attrs);
2378 253727 : return new_rtx;
2379 : }
2380 :
2381 : /* Like change_address_1 with VALIDATE nonzero, but we are not saying in what
2382 : way we are changing MEMREF, so we only preserve the alias set. */
2383 :
2384 : rtx
2385 254825 : change_address (rtx memref, machine_mode mode, rtx addr)
2386 : {
2387 254825 : return change_address (memref, mode, addr, true);
2388 : }
2389 :
2390 : /* Like change_address_1 with VALIDATE zero, but we are not saying in what
2391 : way we are changing MEMREF, so we only preserve the alias set. */
2392 :
2393 : rtx
2394 0 : change_address_nv (rtx memref, machine_mode mode, rtx addr)
2395 : {
2396 0 : return change_address (memref, mode, addr, false);
2397 : }
2398 :
2399 : /* Return a memory reference like MEMREF, but with its mode changed
2400 : to MODE and its address offset by OFFSET bytes. If VALIDATE is
2401 : nonzero, the memory address is forced to be valid.
2402 : If ADJUST_ADDRESS is zero, OFFSET is only used to update MEM_ATTRS
2403 : and the caller is responsible for adjusting MEMREF base register.
2404 : If ADJUST_OBJECT is zero, the underlying object associated with the
2405 : memory reference is left unchanged and the caller is responsible for
2406 : dealing with it. Otherwise, if the new memory reference is outside
2407 : the underlying object, even partially, then the object is dropped.
2408 : SIZE, if nonzero, is the size of an access in cases where MODE
2409 : has no inherent size. */
2410 :
2411 : rtx
2412 36066682 : adjust_address_1 (rtx memref, machine_mode mode, poly_int64 offset,
2413 : int validate, int adjust_address, int adjust_object,
2414 : poly_int64 size)
2415 : {
2416 36066682 : rtx addr = XEXP (memref, 0);
2417 36066682 : rtx new_rtx;
2418 36066682 : machine_mode address_mode;
2419 36066682 : class mem_attrs attrs (*get_mem_attrs (memref)), *defattrs;
2420 36066682 : unsigned HOST_WIDE_INT max_align;
2421 : #ifdef POINTERS_EXTEND_UNSIGNED
2422 36066682 : scalar_int_mode pointer_mode
2423 36066682 : = targetm.addr_space.pointer_mode (attrs.addrspace);
2424 : #endif
2425 :
2426 : /* VOIDmode means no mode change for change_address_1. */
2427 36066682 : if (mode == VOIDmode)
2428 17715 : mode = GET_MODE (memref);
2429 :
2430 : /* Take the size of non-BLKmode accesses from the mode. */
2431 36066682 : defattrs = mode_mem_attrs[(int) mode];
2432 36066682 : if (defattrs->size_known_p)
2433 29855875 : size = defattrs->size;
2434 :
2435 : /* If there are no changes, just return the original memory reference. */
2436 36066682 : if (mode == GET_MODE (memref)
2437 15602946 : && known_eq (offset, 0)
2438 9153743 : && (known_eq (size, 0)
2439 3818754 : || (attrs.size_known_p && known_eq (attrs.size, size)))
2440 50184103 : && (!validate || memory_address_addr_space_p (mode, addr,
2441 4964330 : attrs.addrspace)))
2442 : return memref;
2443 :
2444 : /* ??? Prefer to create garbage instead of creating shared rtl.
2445 : This may happen even if offset is nonzero -- consider
2446 : (plus (plus reg reg) const_int) -- so do this always. */
2447 26913597 : addr = copy_rtx (addr);
2448 :
2449 : /* Convert a possibly large offset to a signed value within the
2450 : range of the target address space. */
2451 26913597 : address_mode = get_address_mode (memref);
2452 26913597 : if (!VECTOR_MODE_P (address_mode))
2453 26913597 : offset = trunc_int_for_mode (offset, address_mode);
2454 : else
2455 0 : offset = trunc_int_for_mode (offset, GET_MODE_INNER (address_mode));
2456 :
2457 26913597 : if (adjust_address)
2458 : {
2459 : /* If MEMREF is a LO_SUM and the offset is within the alignment of the
2460 : object, we can merge it into the LO_SUM. */
2461 26882233 : if (GET_MODE (memref) != BLKmode
2462 12313451 : && GET_CODE (addr) == LO_SUM
2463 26882233 : && known_in_range_p (offset,
2464 0 : 0, (GET_MODE_ALIGNMENT (GET_MODE (memref))
2465 0 : / BITS_PER_UNIT)))
2466 0 : addr = gen_rtx_LO_SUM (address_mode, XEXP (addr, 0),
2467 : plus_constant (address_mode,
2468 : XEXP (addr, 1), offset));
2469 : #ifdef POINTERS_EXTEND_UNSIGNED
2470 : /* If MEMREF is a ZERO_EXTEND from pointer_mode and the offset is valid
2471 : in that mode, we merge it into the ZERO_EXTEND. We take advantage of
2472 : the fact that pointers are not allowed to overflow. */
2473 26882233 : else if (POINTERS_EXTEND_UNSIGNED > 0
2474 26882233 : && GET_CODE (addr) == ZERO_EXTEND
2475 12 : && GET_MODE (XEXP (addr, 0)) == pointer_mode
2476 26882235 : && known_eq (trunc_int_for_mode (offset, pointer_mode), offset))
2477 2 : addr = gen_rtx_ZERO_EXTEND (address_mode,
2478 : plus_constant (pointer_mode,
2479 : XEXP (addr, 0), offset));
2480 : #endif
2481 : else
2482 26882231 : addr = plus_constant (address_mode, addr, offset);
2483 : }
2484 :
2485 26913597 : new_rtx = change_address_1 (memref, mode, addr, validate, false);
2486 :
2487 : /* If the address is a REG, change_address_1 rightfully returns memref,
2488 : but this would destroy memref's MEM_ATTRS. */
2489 26913597 : if (new_rtx == memref && maybe_ne (offset, 0))
2490 31319 : new_rtx = copy_rtx (new_rtx);
2491 :
2492 : /* Conservatively drop the object if we don't know where we start from. */
2493 26913597 : if (adjust_object && (!attrs.offset_known_p || !attrs.size_known_p))
2494 : {
2495 1663 : attrs.expr = NULL_TREE;
2496 1663 : attrs.alias = 0;
2497 : }
2498 :
2499 : /* Compute the new values of the memory attributes due to this adjustment.
2500 : We add the offsets and update the alignment. */
2501 26913597 : if (attrs.offset_known_p)
2502 : {
2503 23068170 : attrs.offset += offset;
2504 :
2505 : /* Drop the object if the new left end is not within its bounds. */
2506 23068170 : if (adjust_object && maybe_lt (attrs.offset, 0))
2507 : {
2508 25715 : attrs.expr = NULL_TREE;
2509 25715 : attrs.alias = 0;
2510 : }
2511 : }
2512 :
2513 : /* Compute the new alignment by taking the MIN of the alignment and the
2514 : lowest-order set bit in OFFSET, but don't change the alignment if OFFSET
2515 : if zero. */
2516 26913597 : if (maybe_ne (offset, 0))
2517 : {
2518 12319390 : max_align = known_alignment (offset) * BITS_PER_UNIT;
2519 12319390 : attrs.align = MIN (attrs.align, max_align);
2520 : }
2521 :
2522 26913597 : if (maybe_ne (size, 0))
2523 : {
2524 : /* Drop the object if the new right end is not within its bounds. */
2525 26199283 : if (adjust_object && maybe_gt (offset + size, attrs.size))
2526 : {
2527 64969 : attrs.expr = NULL_TREE;
2528 64969 : attrs.alias = 0;
2529 : }
2530 26199283 : attrs.size_known_p = true;
2531 26199283 : attrs.size = size;
2532 : }
2533 714314 : else if (attrs.size_known_p)
2534 : {
2535 707866 : gcc_assert (!adjust_object);
2536 26913597 : attrs.size -= offset;
2537 : /* ??? The store_by_pieces machinery generates negative sizes,
2538 : so don't assert for that here. */
2539 : }
2540 :
2541 26913597 : set_mem_attrs (new_rtx, &attrs);
2542 :
2543 26913597 : return new_rtx;
2544 : }
2545 :
2546 : /* Return a memory reference like MEMREF, but with its mode changed
2547 : to MODE and its address changed to ADDR, which is assumed to be
2548 : MEMREF offset by OFFSET bytes. If VALIDATE is
2549 : nonzero, the memory address is forced to be valid. */
2550 :
2551 : rtx
2552 61370 : adjust_automodify_address_1 (rtx memref, machine_mode mode, rtx addr,
2553 : poly_int64 offset, int validate)
2554 : {
2555 61370 : memref = change_address_1 (memref, VOIDmode, addr, validate, false);
2556 61370 : return adjust_address_1 (memref, mode, offset, validate, 0, 0, 0);
2557 : }
2558 :
2559 : /* Return a memory reference like MEMREF, but whose address is changed by
2560 : adding OFFSET, an RTX, to it. POW2 is the highest power of two factor
2561 : known to be in OFFSET (possibly 1). */
2562 :
2563 : rtx
2564 774607 : offset_address (rtx memref, rtx offset, unsigned HOST_WIDE_INT pow2)
2565 : {
2566 774607 : rtx new_rtx, addr = XEXP (memref, 0);
2567 774607 : machine_mode address_mode;
2568 774607 : class mem_attrs *defattrs;
2569 :
2570 774607 : mem_attrs attrs (*get_mem_attrs (memref));
2571 774607 : address_mode = get_address_mode (memref);
2572 774607 : new_rtx = simplify_gen_binary (PLUS, address_mode, addr, offset);
2573 :
2574 : /* At this point we don't know _why_ the address is invalid. It
2575 : could have secondary memory references, multiplies or anything.
2576 :
2577 : However, if we did go and rearrange things, we can wind up not
2578 : being able to recognize the magic around pic_offset_table_rtx.
2579 : This stuff is fragile, and is yet another example of why it is
2580 : bad to expose PIC machinery too early. */
2581 1547477 : if (! memory_address_addr_space_p (GET_MODE (memref), new_rtx,
2582 774607 : attrs.addrspace)
2583 136799 : && GET_CODE (addr) == PLUS
2584 825500 : && XEXP (addr, 0) == pic_offset_table_rtx)
2585 : {
2586 1737 : addr = force_reg (GET_MODE (addr), addr);
2587 1737 : new_rtx = simplify_gen_binary (PLUS, address_mode, addr, offset);
2588 : }
2589 :
2590 774607 : update_temp_slot_address (XEXP (memref, 0), new_rtx);
2591 774607 : new_rtx = change_address_1 (memref, VOIDmode, new_rtx, 1, false);
2592 :
2593 : /* If there are no changes, just return the original memory reference. */
2594 774607 : if (new_rtx == memref)
2595 : return new_rtx;
2596 :
2597 : /* Update the alignment to reflect the offset. Reset the offset, which
2598 : we don't know. */
2599 774607 : defattrs = mode_mem_attrs[(int) GET_MODE (new_rtx)];
2600 774607 : attrs.offset_known_p = false;
2601 774607 : attrs.size_known_p = defattrs->size_known_p;
2602 774607 : attrs.size = defattrs->size;
2603 774607 : attrs.align = MIN (attrs.align, pow2 * BITS_PER_UNIT);
2604 774607 : set_mem_attrs (new_rtx, &attrs);
2605 774607 : return new_rtx;
2606 : }
2607 :
2608 : /* Return a memory reference like MEMREF, but with its address changed to
2609 : ADDR. The caller is asserting that the actual piece of memory pointed
2610 : to is the same, just the form of the address is being changed, such as
2611 : by putting something into a register. INPLACE is true if any changes
2612 : can be made directly to MEMREF or false if MEMREF must be treated as
2613 : immutable. */
2614 :
2615 : rtx
2616 13061676 : replace_equiv_address (rtx memref, rtx addr, bool inplace)
2617 : {
2618 : /* change_address_1 copies the memory attribute structure without change
2619 : and that's exactly what we want here. */
2620 13061676 : update_temp_slot_address (XEXP (memref, 0), addr);
2621 13061676 : return change_address_1 (memref, VOIDmode, addr, 1, inplace);
2622 : }
2623 :
2624 : /* Likewise, but the reference is not required to be valid. */
2625 :
2626 : rtx
2627 466376866 : replace_equiv_address_nv (rtx memref, rtx addr, bool inplace)
2628 : {
2629 466376866 : return change_address_1 (memref, VOIDmode, addr, 0, inplace);
2630 : }
2631 :
2632 :
2633 : /* Emit insns to reload VALUE into a new register. VALUE is an
2634 : auto-increment or auto-decrement RTX whose operand is a register or
2635 : memory location; so reloading involves incrementing that location.
2636 :
2637 : INC_AMOUNT is the number to increment or decrement by (always
2638 : positive and ignored for POST_MODIFY/PRE_MODIFY).
2639 :
2640 : Return a pseudo containing the result. */
2641 : rtx
2642 0 : address_reload_context::emit_autoinc (rtx value, poly_int64 inc_amount)
2643 : {
2644 : /* Since we're going to call recog, and might be called within recog,
2645 : we need to ensure we save and restore recog_data. */
2646 0 : recog_state_saver recog_save;
2647 :
2648 : /* REG or MEM to be copied and incremented. */
2649 0 : rtx incloc = XEXP (value, 0);
2650 :
2651 0 : const rtx_code code = GET_CODE (value);
2652 0 : const bool post_p
2653 0 : = code == POST_DEC || code == POST_INC || code == POST_MODIFY;
2654 :
2655 0 : bool plus_p = true;
2656 0 : rtx inc;
2657 0 : if (code == PRE_MODIFY || code == POST_MODIFY)
2658 : {
2659 0 : gcc_assert (GET_CODE (XEXP (value, 1)) == PLUS
2660 : || GET_CODE (XEXP (value, 1)) == MINUS);
2661 0 : gcc_assert (rtx_equal_p (XEXP (XEXP (value, 1), 0), XEXP (value, 0)));
2662 0 : plus_p = GET_CODE (XEXP (value, 1)) == PLUS;
2663 0 : inc = XEXP (XEXP (value, 1), 1);
2664 : }
2665 : else
2666 : {
2667 0 : if (code == PRE_DEC || code == POST_DEC)
2668 0 : inc_amount = -inc_amount;
2669 :
2670 0 : inc = gen_int_mode (inc_amount, GET_MODE (value));
2671 : }
2672 :
2673 0 : rtx result;
2674 0 : if (!post_p && REG_P (incloc))
2675 : result = incloc;
2676 : else
2677 : {
2678 0 : result = get_reload_reg ();
2679 : /* First copy the location to the result register. */
2680 0 : emit_insn (gen_move_insn (result, incloc));
2681 : }
2682 :
2683 : /* See if we can directly increment INCLOC. */
2684 0 : rtx_insn *last = get_last_insn ();
2685 0 : rtx_insn *add_insn = emit_insn (plus_p
2686 0 : ? gen_add2_insn (incloc, inc)
2687 0 : : gen_sub2_insn (incloc, inc));
2688 0 : const int icode = recog_memoized (add_insn);
2689 0 : if (icode >= 0)
2690 : {
2691 0 : if (!post_p && result != incloc)
2692 0 : emit_insn (gen_move_insn (result, incloc));
2693 : return result;
2694 : }
2695 0 : delete_insns_since (last);
2696 :
2697 : /* If couldn't do the increment directly, must increment in RESULT.
2698 : The way we do this depends on whether this is pre- or
2699 : post-increment. For pre-increment, copy INCLOC to the reload
2700 : register, increment it there, then save back. */
2701 0 : if (!post_p)
2702 : {
2703 0 : if (incloc != result)
2704 0 : emit_insn (gen_move_insn (result, incloc));
2705 0 : if (plus_p)
2706 0 : emit_insn (gen_add2_insn (result, inc));
2707 : else
2708 0 : emit_insn (gen_sub2_insn (result, inc));
2709 0 : if (incloc != result)
2710 0 : emit_insn (gen_move_insn (incloc, result));
2711 : }
2712 : else
2713 : {
2714 : /* Post-increment.
2715 :
2716 : Because this might be a jump insn or a compare, and because
2717 : RESULT may not be available after the insn in an input
2718 : reload, we must do the incrementing before the insn being
2719 : reloaded for.
2720 :
2721 : We have already copied INCLOC to RESULT. Increment the copy in
2722 : RESULT, save that back, then decrement RESULT so it has
2723 : the original value. */
2724 0 : if (plus_p)
2725 0 : emit_insn (gen_add2_insn (result, inc));
2726 : else
2727 0 : emit_insn (gen_sub2_insn (result, inc));
2728 0 : emit_insn (gen_move_insn (incloc, result));
2729 : /* Restore non-modified value for the result. We prefer this
2730 : way because it does not require an additional hard
2731 : register. */
2732 0 : if (plus_p)
2733 : {
2734 0 : poly_int64 offset;
2735 0 : if (poly_int_rtx_p (inc, &offset))
2736 0 : emit_insn (gen_add2_insn (result,
2737 : gen_int_mode (-offset,
2738 0 : GET_MODE (result))));
2739 : else
2740 0 : emit_insn (gen_sub2_insn (result, inc));
2741 : }
2742 : else
2743 0 : emit_insn (gen_add2_insn (result, inc));
2744 : }
2745 : return result;
2746 0 : }
2747 :
2748 : /* Return a memory reference like MEM, but with the address reloaded into a
2749 : pseudo register. */
2750 :
2751 : rtx
2752 0 : force_reload_address (rtx mem)
2753 : {
2754 0 : rtx addr = XEXP (mem, 0);
2755 0 : if (GET_RTX_CLASS (GET_CODE (addr)) == RTX_AUTOINC)
2756 : {
2757 0 : const auto size = GET_MODE_SIZE (GET_MODE (mem));
2758 0 : addr = address_reload_context ().emit_autoinc (addr, size);
2759 : }
2760 : else
2761 0 : addr = force_reg (Pmode, addr);
2762 :
2763 0 : return replace_equiv_address (mem, addr);
2764 : }
2765 :
2766 : /* Return a memory reference like MEMREF, but with its mode widened to
2767 : MODE and offset by OFFSET. This would be used by targets that e.g.
2768 : cannot issue QImode memory operations and have to use SImode memory
2769 : operations plus masking logic. */
2770 :
2771 : rtx
2772 47 : widen_memory_access (rtx memref, machine_mode mode, poly_int64 offset)
2773 : {
2774 47 : rtx new_rtx = adjust_address_1 (memref, mode, offset, 1, 1, 0, 0);
2775 94 : poly_uint64 size = GET_MODE_SIZE (mode);
2776 :
2777 : /* If there are no changes, just return the original memory reference. */
2778 47 : if (new_rtx == memref)
2779 : return new_rtx;
2780 :
2781 47 : mem_attrs attrs (*get_mem_attrs (new_rtx));
2782 :
2783 : /* If we don't know what offset we were at within the expression, then
2784 : we can't know if we've overstepped the bounds. */
2785 47 : if (! attrs.offset_known_p)
2786 5 : attrs.expr = NULL_TREE;
2787 :
2788 47 : while (attrs.expr)
2789 : {
2790 42 : if (TREE_CODE (attrs.expr) == COMPONENT_REF)
2791 : {
2792 0 : tree field = TREE_OPERAND (attrs.expr, 1);
2793 0 : tree offset = component_ref_field_offset (attrs.expr);
2794 :
2795 0 : if (! DECL_SIZE_UNIT (field))
2796 : {
2797 0 : attrs.expr = NULL_TREE;
2798 0 : break;
2799 : }
2800 :
2801 : /* Is the field at least as large as the access? If so, ok,
2802 : otherwise strip back to the containing structure. */
2803 0 : if (poly_int_tree_p (DECL_SIZE_UNIT (field))
2804 0 : && known_ge (wi::to_poly_offset (DECL_SIZE_UNIT (field)), size)
2805 0 : && known_ge (attrs.offset, 0))
2806 : break;
2807 :
2808 0 : poly_uint64 suboffset;
2809 0 : if (!poly_int_tree_p (offset, &suboffset))
2810 : {
2811 0 : attrs.expr = NULL_TREE;
2812 0 : break;
2813 : }
2814 :
2815 0 : attrs.expr = TREE_OPERAND (attrs.expr, 0);
2816 0 : attrs.offset += suboffset;
2817 0 : attrs.offset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (field))
2818 0 : / BITS_PER_UNIT);
2819 : }
2820 : /* Similarly for the decl. */
2821 42 : else if (DECL_P (attrs.expr)
2822 12 : && DECL_SIZE_UNIT (attrs.expr)
2823 7 : && poly_int_tree_p (DECL_SIZE_UNIT (attrs.expr))
2824 7 : && known_ge (wi::to_poly_offset (DECL_SIZE_UNIT (attrs.expr)),
2825 : size)
2826 42 : && known_ge (attrs.offset, 0))
2827 : break;
2828 : else
2829 : {
2830 : /* The widened memory access overflows the expression, which means
2831 : that it could alias another expression. Zap it. */
2832 42 : attrs.expr = NULL_TREE;
2833 42 : break;
2834 : }
2835 : }
2836 :
2837 47 : if (! attrs.expr)
2838 47 : attrs.offset_known_p = false;
2839 :
2840 : /* The widened memory may alias other stuff, so zap the alias set. */
2841 : /* ??? Maybe use get_alias_set on any remaining expression. */
2842 47 : attrs.alias = 0;
2843 47 : attrs.size_known_p = true;
2844 47 : attrs.size = size;
2845 47 : set_mem_attrs (new_rtx, &attrs);
2846 47 : return new_rtx;
2847 : }
2848 :
2849 : /* A fake decl that is used as the MEM_EXPR of spill slots. */
2850 : static GTY(()) tree spill_slot_decl;
2851 :
2852 : tree
2853 842101699 : get_spill_slot_decl (bool force_build_p)
2854 : {
2855 842101699 : tree d = spill_slot_decl;
2856 842101699 : rtx rd;
2857 :
2858 842101699 : if (d || !force_build_p)
2859 : return d;
2860 :
2861 31003 : d = build_decl (DECL_SOURCE_LOCATION (current_function_decl),
2862 : VAR_DECL, get_identifier ("%sfp"), void_type_node);
2863 31003 : DECL_ARTIFICIAL (d) = 1;
2864 31003 : DECL_IGNORED_P (d) = 1;
2865 31003 : TREE_USED (d) = 1;
2866 31003 : spill_slot_decl = d;
2867 :
2868 31003 : rd = gen_rtx_MEM (BLKmode, frame_pointer_rtx);
2869 31003 : MEM_NOTRAP_P (rd) = 1;
2870 31003 : mem_attrs attrs (*mode_mem_attrs[(int) BLKmode]);
2871 31003 : attrs.alias = new_alias_set ();
2872 31003 : attrs.expr = d;
2873 31003 : set_mem_attrs (rd, &attrs);
2874 31003 : SET_DECL_RTL (d, rd);
2875 :
2876 31003 : return d;
2877 : }
2878 :
2879 : /* Given MEM, a result from assign_stack_local, fill in the memory
2880 : attributes as appropriate for a register allocator spill slot.
2881 : These slots are not aliasable by other memory. We arrange for
2882 : them all to use a single MEM_EXPR, so that the aliasing code can
2883 : work properly in the case of shared spill slots. */
2884 :
2885 : void
2886 1452724 : set_mem_attrs_for_spill (rtx mem)
2887 : {
2888 1452724 : rtx addr;
2889 :
2890 1452724 : mem_attrs attrs (*get_mem_attrs (mem));
2891 1452724 : attrs.expr = get_spill_slot_decl (true);
2892 1452724 : attrs.alias = MEM_ALIAS_SET (DECL_RTL (attrs.expr));
2893 1452724 : attrs.addrspace = ADDR_SPACE_GENERIC;
2894 :
2895 : /* We expect the incoming memory to be of the form:
2896 : (mem:MODE (plus (reg sfp) (const_int offset)))
2897 : with perhaps the plus missing for offset = 0. */
2898 1452724 : addr = XEXP (mem, 0);
2899 1452724 : attrs.offset_known_p = true;
2900 1452724 : strip_offset (addr, &attrs.offset);
2901 :
2902 1452724 : set_mem_attrs (mem, &attrs);
2903 1452724 : MEM_NOTRAP_P (mem) = 1;
2904 1452724 : }
2905 :
2906 : /* Return a newly created CODE_LABEL rtx with a unique label number. */
2907 :
2908 : rtx_code_label *
2909 15646181 : gen_label_rtx (void)
2910 : {
2911 15646181 : return as_a <rtx_code_label *> (
2912 : gen_rtx_CODE_LABEL (VOIDmode, NULL_RTX, NULL_RTX,
2913 15646181 : NULL, label_num++, NULL));
2914 : }
2915 :
2916 : /* For procedure integration. */
2917 :
2918 : /* Install new pointers to the first and last insns in the chain.
2919 : Also, set cur_insn_uid to one higher than the last in use.
2920 : Used for an inline-procedure after copying the insn chain. */
2921 :
2922 : void
2923 8 : set_new_first_and_last_insn (rtx_insn *first, rtx_insn *last)
2924 : {
2925 8 : rtx_insn *insn;
2926 :
2927 8 : set_first_insn (first);
2928 8 : set_last_insn (last);
2929 8 : cur_insn_uid = 0;
2930 :
2931 8 : if (param_min_nondebug_insn_uid || MAY_HAVE_DEBUG_INSNS)
2932 : {
2933 0 : int debug_count = 0;
2934 :
2935 0 : cur_insn_uid = param_min_nondebug_insn_uid - 1;
2936 0 : cur_debug_insn_uid = 0;
2937 :
2938 0 : for (insn = first; insn; insn = NEXT_INSN (insn))
2939 0 : if (INSN_UID (insn) < param_min_nondebug_insn_uid)
2940 0 : cur_debug_insn_uid = MAX (cur_debug_insn_uid, INSN_UID (insn));
2941 : else
2942 : {
2943 0 : cur_insn_uid = MAX (cur_insn_uid, INSN_UID (insn));
2944 0 : if (DEBUG_INSN_P (insn))
2945 0 : debug_count++;
2946 : }
2947 :
2948 0 : if (debug_count)
2949 0 : cur_debug_insn_uid = param_min_nondebug_insn_uid + debug_count;
2950 : else
2951 0 : cur_debug_insn_uid++;
2952 : }
2953 : else
2954 8 : for (insn = first; insn; insn = NEXT_INSN (insn))
2955 0 : cur_insn_uid = MAX (cur_insn_uid, INSN_UID (insn));
2956 :
2957 8 : cur_insn_uid++;
2958 8 : }
2959 :
2960 : /* Go through all the RTL insn bodies and copy any invalid shared
2961 : structure. This routine should only be called once. */
2962 :
2963 : static void
2964 3035033 : unshare_all_rtl_1 (rtx_insn *insn)
2965 : {
2966 : /* Unshare just about everything else. */
2967 3035033 : unshare_all_rtl_in_chain (insn);
2968 :
2969 : /* Make sure the addresses of stack slots found outside the insn chain
2970 : (such as, in DECL_RTL of a variable) are not shared
2971 : with the insn chain.
2972 :
2973 : This special care is necessary when the stack slot MEM does not
2974 : actually appear in the insn chain. If it does appear, its address
2975 : is unshared from all else at that point. */
2976 3035033 : unsigned int i;
2977 3035033 : rtx temp;
2978 9358579 : FOR_EACH_VEC_SAFE_ELT (stack_slot_list, i, temp)
2979 3288513 : (*stack_slot_list)[i] = copy_rtx_if_shared (temp);
2980 3035033 : }
2981 :
2982 : /* Go through all the RTL insn bodies and copy any invalid shared
2983 : structure, again. This is a fairly expensive thing to do so it
2984 : should be done sparingly. */
2985 :
2986 : void
2987 1517127 : unshare_all_rtl_again (rtx_insn *insn)
2988 : {
2989 1517127 : rtx_insn *p;
2990 1517127 : tree decl;
2991 :
2992 174512168 : for (p = insn; p; p = NEXT_INSN (p))
2993 172995041 : if (INSN_P (p))
2994 : {
2995 137122512 : reset_used_flags (PATTERN (p));
2996 137122512 : reset_used_flags (REG_NOTES (p));
2997 179168339 : if (auto call = dyn_cast<rtx_call_insn *> (p))
2998 6173298 : reset_used_flags (CALL_INSN_FUNCTION_USAGE (call));
2999 : }
3000 :
3001 : /* Make sure that virtual stack slots are not shared. */
3002 1517127 : set_used_decls (DECL_INITIAL (cfun->decl));
3003 :
3004 : /* Make sure that virtual parameters are not shared. */
3005 4683770 : for (decl = DECL_ARGUMENTS (cfun->decl); decl; decl = DECL_CHAIN (decl))
3006 3166643 : set_used_flags (DECL_RTL (decl));
3007 :
3008 : rtx temp;
3009 : unsigned int i;
3010 3956998 : FOR_EACH_VEC_SAFE_ELT (stack_slot_list, i, temp)
3011 2439871 : reset_used_flags (temp);
3012 :
3013 1517127 : unshare_all_rtl_1 (insn);
3014 1517127 : }
3015 :
3016 : void
3017 1517906 : unshare_all_rtl (void)
3018 : {
3019 1517906 : unshare_all_rtl_1 (get_insns ());
3020 :
3021 4685106 : for (tree decl = DECL_ARGUMENTS (cfun->decl); decl; decl = DECL_CHAIN (decl))
3022 : {
3023 3167200 : if (DECL_RTL_SET_P (decl))
3024 3167200 : SET_DECL_RTL (decl, copy_rtx_if_shared (DECL_RTL (decl)));
3025 3167200 : DECL_INCOMING_RTL (decl) = copy_rtx_if_shared (DECL_INCOMING_RTL (decl));
3026 : }
3027 1517906 : }
3028 :
3029 :
3030 : /* Check that ORIG is not marked when it should not be and mark ORIG as in use,
3031 : Recursively does the same for subexpressions. */
3032 :
3033 : static void
3034 57609826888 : verify_rtx_sharing (rtx orig, rtx_insn *insn)
3035 : {
3036 57609826888 : rtx x = orig;
3037 57609826888 : int i;
3038 57609826888 : enum rtx_code code;
3039 57609826888 : const char *format_ptr;
3040 :
3041 57609826888 : if (x == 0)
3042 : return;
3043 :
3044 47725773256 : code = GET_CODE (x);
3045 :
3046 : /* These types may be freely shared. */
3047 :
3048 47725773256 : switch (code)
3049 : {
3050 : case REG:
3051 : case DEBUG_EXPR:
3052 : case VALUE:
3053 : CASE_CONST_ANY:
3054 : case SYMBOL_REF:
3055 : case LABEL_REF:
3056 : case CODE_LABEL:
3057 : case PC:
3058 : case RETURN:
3059 : case SIMPLE_RETURN:
3060 : case SCRATCH:
3061 : /* SCRATCH must be shared because they represent distinct values. */
3062 : return;
3063 2046238132 : case CLOBBER:
3064 : /* Share clobbers of hard registers, but do not share pseudo reg
3065 : clobbers or clobbers of hard registers that originated as pseudos.
3066 : This is needed to allow safe register renaming. */
3067 2046238132 : if (REG_P (XEXP (x, 0))
3068 740741167 : && HARD_REGISTER_NUM_P (REGNO (XEXP (x, 0)))
3069 2783839361 : && HARD_REGISTER_NUM_P (ORIGINAL_REGNO (XEXP (x, 0))))
3070 : return;
3071 : break;
3072 :
3073 36705766 : case CONST:
3074 36705766 : if (shared_const_p (orig))
3075 : return;
3076 : break;
3077 :
3078 2816687674 : case MEM:
3079 : /* A MEM is allowed to be shared if its address is constant. */
3080 2816687674 : if (CONSTANT_ADDRESS_P (XEXP (x, 0))
3081 2816687674 : || reload_completed || reload_in_progress)
3082 : return;
3083 :
3084 : break;
3085 :
3086 : default:
3087 : break;
3088 : }
3089 :
3090 : /* This rtx may not be shared. If it has already been seen,
3091 : replace it with a copy of itself. */
3092 23164982267 : if (flag_checking && RTX_FLAG (x, used))
3093 : {
3094 0 : error ("invalid rtl sharing found in the insn");
3095 0 : debug_rtx (insn);
3096 0 : error ("shared rtx");
3097 0 : debug_rtx (x);
3098 0 : internal_error ("internal consistency failure");
3099 : }
3100 23164982267 : gcc_assert (!RTX_FLAG (x, used));
3101 :
3102 23164982267 : RTX_FLAG (x, used) = 1;
3103 :
3104 : /* Now scan the subexpressions recursively. */
3105 :
3106 23164982267 : format_ptr = GET_RTX_FORMAT (code);
3107 :
3108 65004334632 : for (i = 0; i < GET_RTX_LENGTH (code); i++)
3109 : {
3110 41839352365 : switch (*format_ptr++)
3111 : {
3112 36495096425 : case 'e':
3113 36495096425 : verify_rtx_sharing (XEXP (x, i), insn);
3114 36495096425 : break;
3115 :
3116 921744775 : case 'E':
3117 921744775 : if (XVEC (x, i) != NULL)
3118 : {
3119 921744775 : int j;
3120 921744775 : int len = XVECLEN (x, i);
3121 :
3122 2791030904 : for (j = 0; j < len; j++)
3123 : {
3124 : /* We allow sharing of ASM_OPERANDS inside single
3125 : instruction. */
3126 1869286129 : if (j && GET_CODE (XVECEXP (x, i, j)) == SET
3127 57047708 : && (GET_CODE (SET_SRC (XVECEXP (x, i, j)))
3128 : == ASM_OPERANDS))
3129 3260268 : verify_rtx_sharing (SET_DEST (XVECEXP (x, i, j)), insn);
3130 : else
3131 1866025861 : verify_rtx_sharing (XVECEXP (x, i, j), insn);
3132 : }
3133 : }
3134 : break;
3135 : }
3136 : }
3137 : }
3138 :
3139 : /* Reset used-flags for INSN. */
3140 :
3141 : static void
3142 18823174386 : reset_insn_used_flags (rtx_insn *insn)
3143 : {
3144 18823174386 : gcc_assert (INSN_P (insn));
3145 18823174386 : reset_used_flags (PATTERN (insn));
3146 18823174386 : reset_used_flags (REG_NOTES (insn));
3147 18823174386 : if (auto call = dyn_cast<rtx_call_insn *> (insn))
3148 844539896 : reset_used_flags (CALL_INSN_FUNCTION_USAGE (call));
3149 18823174386 : }
3150 :
3151 : /* Go through all the RTL insn bodies and clear all the USED bits. */
3152 :
3153 : static void
3154 192614712 : reset_all_used_flags (void)
3155 : {
3156 192614712 : rtx_insn *p;
3157 :
3158 25438973004 : for (p = get_insns (); p; p = NEXT_INSN (p))
3159 25246358292 : if (INSN_P (p))
3160 : {
3161 18823174386 : rtx pat = PATTERN (p);
3162 18823174386 : if (rtx_sequence *seq = dyn_cast<rtx_sequence *> (pat))
3163 : {
3164 0 : gcc_assert (REG_NOTES (p) == NULL);
3165 0 : for (int i = 0; i < seq->len (); i++)
3166 : {
3167 0 : rtx_insn *insn = seq->insn (i);
3168 0 : if (INSN_P (insn))
3169 0 : reset_insn_used_flags (insn);
3170 : }
3171 : }
3172 : else
3173 18823174386 : reset_insn_used_flags (p);
3174 : }
3175 192614712 : }
3176 :
3177 : /* Verify sharing in INSN. */
3178 :
3179 : static void
3180 9411587193 : verify_insn_sharing (rtx_insn *insn)
3181 : {
3182 9411587193 : gcc_assert (INSN_P (insn));
3183 9411587193 : verify_rtx_sharing (PATTERN (insn), insn);
3184 9411587193 : verify_rtx_sharing (REG_NOTES (insn), insn);
3185 9411587193 : if (auto call = dyn_cast<rtx_call_insn *> (insn))
3186 422269948 : verify_rtx_sharing (CALL_INSN_FUNCTION_USAGE (call), call);
3187 9411587193 : }
3188 :
3189 : /* Go through all the RTL insn bodies and check that there is no unexpected
3190 : sharing in between the subexpressions. */
3191 :
3192 : DEBUG_FUNCTION void
3193 96307356 : verify_rtl_sharing (void)
3194 : {
3195 96307356 : rtx_insn *p;
3196 :
3197 96307356 : timevar_push (TV_VERIFY_RTL_SHARING);
3198 :
3199 96307356 : reset_all_used_flags ();
3200 :
3201 12719486502 : for (p = get_insns (); p; p = NEXT_INSN (p))
3202 12623179146 : if (INSN_P (p))
3203 : {
3204 9411587193 : rtx pat = PATTERN (p);
3205 9411587193 : if (rtx_sequence *seq = dyn_cast<rtx_sequence *> (pat))
3206 0 : for (int i = 0; i < seq->len (); i++)
3207 : {
3208 0 : rtx_insn *insn = seq->insn (i);
3209 0 : if (INSN_P (insn))
3210 0 : verify_insn_sharing (insn);
3211 : }
3212 : else
3213 9411587193 : verify_insn_sharing (p);
3214 : }
3215 :
3216 96307356 : reset_all_used_flags ();
3217 :
3218 96307356 : timevar_pop (TV_VERIFY_RTL_SHARING);
3219 96307356 : }
3220 :
3221 : /* Go through all the RTL insn bodies and copy any invalid shared structure.
3222 : Assumes the mark bits are cleared at entry. */
3223 :
3224 : void
3225 9604035 : unshare_all_rtl_in_chain (rtx_insn *insn)
3226 : {
3227 372257464 : for (; insn; insn = NEXT_INSN (insn))
3228 362653429 : if (INSN_P (insn))
3229 : {
3230 296940761 : PATTERN (insn) = copy_rtx_if_shared (PATTERN (insn));
3231 296940761 : REG_NOTES (insn) = copy_rtx_if_shared (REG_NOTES (insn));
3232 374994024 : if (auto call = dyn_cast<rtx_call_insn *> (insn))
3233 12340595 : CALL_INSN_FUNCTION_USAGE (call)
3234 12340595 : = copy_rtx_if_shared (CALL_INSN_FUNCTION_USAGE (call));
3235 : }
3236 9604035 : }
3237 :
3238 : /* Go through all virtual stack slots of a function and mark them as
3239 : shared. We never replace the DECL_RTLs themselves with a copy,
3240 : but expressions mentioned into a DECL_RTL cannot be shared with
3241 : expressions in the instruction stream.
3242 :
3243 : Note that reload may convert pseudo registers into memories in-place.
3244 : Pseudo registers are always shared, but MEMs never are. Thus if we
3245 : reset the used flags on MEMs in the instruction stream, we must set
3246 : them again on MEMs that appear in DECL_RTLs. */
3247 :
3248 : static void
3249 17236788 : set_used_decls (tree blk)
3250 : {
3251 17236788 : tree t;
3252 :
3253 : /* Mark decls. */
3254 37554203 : for (t = BLOCK_VARS (blk); t; t = DECL_CHAIN (t))
3255 20317415 : if (DECL_RTL_SET_P (t))
3256 2343318 : set_used_flags (DECL_RTL (t));
3257 :
3258 : /* Now process sub-blocks. */
3259 32956449 : for (t = BLOCK_SUBBLOCKS (blk); t; t = BLOCK_CHAIN (t))
3260 15719661 : set_used_decls (t);
3261 17236788 : }
3262 :
3263 : /* Mark ORIG as in use, and return a copy of it if it was already in use.
3264 : Recursively does the same for subexpressions. Uses
3265 : copy_rtx_if_shared_1 to reduce stack space. */
3266 :
3267 : rtx
3268 643979948 : copy_rtx_if_shared (rtx orig)
3269 : {
3270 643979948 : copy_rtx_if_shared_1 (&orig);
3271 643979948 : return orig;
3272 : }
3273 :
3274 : /* Mark *ORIG1 as in use, and set it to a copy of it if it was already in
3275 : use. Recursively does the same for subexpressions. */
3276 :
3277 : static void
3278 1122833395 : copy_rtx_if_shared_1 (rtx *orig1)
3279 : {
3280 1122833395 : rtx x;
3281 1122833395 : int i;
3282 1122833395 : enum rtx_code code;
3283 1122833395 : rtx *last_ptr;
3284 1122833395 : const char *format_ptr;
3285 1122833395 : int copied = 0;
3286 1854823846 : int length;
3287 :
3288 : /* Repeat is used to turn tail-recursion into iteration. */
3289 1854823846 : repeat:
3290 1854823846 : x = *orig1;
3291 :
3292 1854823846 : if (x == 0)
3293 : return;
3294 :
3295 1524629854 : code = GET_CODE (x);
3296 :
3297 : /* These types may be freely shared. */
3298 :
3299 1524629854 : switch (code)
3300 : {
3301 : case REG:
3302 : case DEBUG_EXPR:
3303 : case VALUE:
3304 : CASE_CONST_ANY:
3305 : case SYMBOL_REF:
3306 : case LABEL_REF:
3307 : case CODE_LABEL:
3308 : case PC:
3309 : case RETURN:
3310 : case SIMPLE_RETURN:
3311 : case SCRATCH:
3312 : /* SCRATCH must be shared because they represent distinct values. */
3313 : return;
3314 63145096 : case CLOBBER:
3315 : /* Share clobbers of hard registers, but do not share pseudo reg
3316 : clobbers or clobbers of hard registers that originated as pseudos.
3317 : This is needed to allow safe register renaming. */
3318 63145096 : if (REG_P (XEXP (x, 0))
3319 26455232 : && HARD_REGISTER_NUM_P (REGNO (XEXP (x, 0)))
3320 89420034 : && HARD_REGISTER_NUM_P (ORIGINAL_REGNO (XEXP (x, 0))))
3321 : return;
3322 : break;
3323 :
3324 5782776 : case CONST:
3325 5782776 : if (shared_const_p (x))
3326 : return;
3327 : break;
3328 :
3329 : case DEBUG_INSN:
3330 : case INSN:
3331 : case JUMP_INSN:
3332 : case CALL_INSN:
3333 : case NOTE:
3334 : case BARRIER:
3335 : /* The chain of insns is not being copied. */
3336 : return;
3337 :
3338 : default:
3339 : break;
3340 : }
3341 :
3342 : /* This rtx may not be shared. If it has already been seen,
3343 : replace it with a copy of itself. */
3344 :
3345 759517086 : if (RTX_FLAG (x, used))
3346 : {
3347 4963348 : x = shallow_copy_rtx (x);
3348 4963348 : copied = 1;
3349 : }
3350 759517086 : RTX_FLAG (x, used) = 1;
3351 :
3352 : /* Now scan the subexpressions recursively.
3353 : We can store any replaced subexpressions directly into X
3354 : since we know X is not shared! Any vectors in X
3355 : must be copied if X was copied. */
3356 :
3357 759517086 : format_ptr = GET_RTX_FORMAT (code);
3358 759517086 : length = GET_RTX_LENGTH (code);
3359 759517086 : last_ptr = NULL;
3360 :
3361 2140739142 : for (i = 0; i < length; i++)
3362 : {
3363 1381222056 : switch (*format_ptr++)
3364 : {
3365 1147363805 : case 'e':
3366 1147363805 : if (last_ptr)
3367 446960251 : copy_rtx_if_shared_1 (last_ptr);
3368 1147363805 : last_ptr = &XEXP (x, i);
3369 1147363805 : break;
3370 :
3371 32250100 : case 'E':
3372 32250100 : if (XVEC (x, i) != NULL)
3373 : {
3374 32250100 : int j;
3375 32250100 : int len = XVECLEN (x, i);
3376 :
3377 : /* Copy the vector iff I copied the rtx and the length
3378 : is nonzero. */
3379 32250100 : if (copied && len > 0)
3380 1366 : XVEC (x, i) = gen_rtvec_v (len, XVEC (x, i)->elem);
3381 :
3382 : /* Call recursively on all inside the vector. */
3383 95730193 : for (j = 0; j < len; j++)
3384 : {
3385 63480093 : if (last_ptr)
3386 31893196 : copy_rtx_if_shared_1 (last_ptr);
3387 63480093 : last_ptr = &XVECEXP (x, i, j);
3388 : }
3389 : }
3390 : break;
3391 : }
3392 : }
3393 759517086 : *orig1 = x;
3394 759517086 : if (last_ptr)
3395 : {
3396 731990451 : orig1 = last_ptr;
3397 731990451 : goto repeat;
3398 : }
3399 : }
3400 :
3401 : /* Set the USED bit in X and its non-shareable subparts to FLAG. */
3402 :
3403 : static void
3404 81161901879 : mark_used_flags (rtx x, int flag)
3405 : {
3406 >12623*10^7 : int i, j;
3407 >12623*10^7 : enum rtx_code code;
3408 >12623*10^7 : const char *format_ptr;
3409 >12623*10^7 : int length;
3410 :
3411 : /* Repeat is used to turn tail-recursion into iteration. */
3412 >12623*10^7 : repeat:
3413 >12623*10^7 : if (x == 0)
3414 : return;
3415 :
3416 >10630*10^7 : code = GET_CODE (x);
3417 :
3418 : /* These types may be freely shared so we needn't do any resetting
3419 : for them. */
3420 :
3421 >10630*10^7 : switch (code)
3422 : {
3423 : case REG:
3424 : case DEBUG_EXPR:
3425 : case VALUE:
3426 : CASE_CONST_ANY:
3427 : case SYMBOL_REF:
3428 : case CODE_LABEL:
3429 : case PC:
3430 : case RETURN:
3431 : case SIMPLE_RETURN:
3432 : return;
3433 :
3434 : case DEBUG_INSN:
3435 : case INSN:
3436 : case JUMP_INSN:
3437 : case CALL_INSN:
3438 : case NOTE:
3439 : case LABEL_REF:
3440 : case BARRIER:
3441 : /* The chain of insns is not being copied. */
3442 : return;
3443 :
3444 54910901119 : default:
3445 54910901119 : break;
3446 : }
3447 :
3448 54910901119 : RTX_FLAG (x, used) = flag;
3449 :
3450 54910901119 : format_ptr = GET_RTX_FORMAT (code);
3451 54910901119 : length = GET_RTX_LENGTH (code);
3452 :
3453 >10828*10^7 : for (i = 0; i < length; i++)
3454 : {
3455 98450647502 : switch (*format_ptr++)
3456 : {
3457 83619560292 : case 'e':
3458 83619560292 : if (i == length-1)
3459 : {
3460 45077057504 : x = XEXP (x, i);
3461 45077057504 : goto repeat;
3462 : }
3463 38542502788 : mark_used_flags (XEXP (x, i), flag);
3464 38542502788 : break;
3465 :
3466 : case 'E':
3467 5715724556 : for (j = 0; j < XVECLEN (x, i); j++)
3468 3813038040 : mark_used_flags (XVECEXP (x, i, j), flag);
3469 : break;
3470 : }
3471 : }
3472 : }
3473 :
3474 : /* Clear all the USED bits in X to allow copy_rtx_if_shared to be used
3475 : to look for shared sub-parts. */
3476 :
3477 : void
3478 38798827511 : reset_used_flags (rtx x)
3479 : {
3480 38798827511 : mark_used_flags (x, 0);
3481 38798827511 : }
3482 :
3483 : /* Set all the USED bits in X to allow copy_rtx_if_shared to be used
3484 : to look for shared sub-parts. */
3485 :
3486 : void
3487 7533540 : set_used_flags (rtx x)
3488 : {
3489 7533540 : mark_used_flags (x, 1);
3490 7533540 : }
3491 :
3492 : /* Copy X if necessary so that it won't be altered by changes in OTHER.
3493 : Return X or the rtx for the pseudo reg the value of X was copied into.
3494 : OTHER must be valid as a SET_DEST. */
3495 :
3496 : rtx
3497 0 : make_safe_from (rtx x, rtx other)
3498 : {
3499 0 : while (1)
3500 0 : switch (GET_CODE (other))
3501 : {
3502 0 : case SUBREG:
3503 0 : other = SUBREG_REG (other);
3504 0 : break;
3505 0 : case STRICT_LOW_PART:
3506 0 : case SIGN_EXTEND:
3507 0 : case ZERO_EXTEND:
3508 0 : other = XEXP (other, 0);
3509 0 : break;
3510 0 : default:
3511 0 : goto done;
3512 : }
3513 0 : done:
3514 0 : if ((MEM_P (other)
3515 0 : && ! CONSTANT_P (x)
3516 0 : && !REG_P (x)
3517 0 : && GET_CODE (x) != SUBREG)
3518 0 : || (REG_P (other)
3519 0 : && (REGNO (other) < FIRST_PSEUDO_REGISTER
3520 0 : || reg_mentioned_p (other, x))))
3521 : {
3522 0 : rtx temp = gen_reg_rtx (GET_MODE (x));
3523 0 : emit_move_insn (temp, x);
3524 0 : return temp;
3525 : }
3526 : return x;
3527 : }
3528 :
3529 : /* Emission of insns (adding them to the doubly-linked list). */
3530 :
3531 : /* Return the last insn emitted, even if it is in a sequence now pushed. */
3532 :
3533 : rtx_insn *
3534 0 : get_last_insn_anywhere (void)
3535 : {
3536 0 : struct sequence_stack *seq;
3537 0 : for (seq = get_current_sequence (); seq; seq = seq->next)
3538 0 : if (seq->last != 0)
3539 : return seq->last;
3540 : return 0;
3541 : }
3542 :
3543 : /* Return the first nonnote insn emitted in current sequence or current
3544 : function. This routine looks inside SEQUENCEs. */
3545 :
3546 : rtx_insn *
3547 0 : get_first_nonnote_insn (void)
3548 : {
3549 0 : rtx_insn *insn = get_insns ();
3550 :
3551 0 : if (insn)
3552 : {
3553 0 : if (NOTE_P (insn))
3554 0 : for (insn = next_insn (insn);
3555 0 : insn && NOTE_P (insn);
3556 0 : insn = next_insn (insn))
3557 0 : continue;
3558 : else
3559 : {
3560 0 : rtx_sequence *seq;
3561 0 : if (NONJUMP_INSN_P (insn)
3562 0 : && (seq = dyn_cast<rtx_sequence *> (PATTERN (insn))))
3563 0 : insn = seq->insn (0);
3564 : }
3565 : }
3566 :
3567 0 : return insn;
3568 : }
3569 :
3570 : /* Return the first nonnote, nondebug insn emitted in current sequence or
3571 : current function. This routine follows get_first_nonnote_insn into a
3572 : leading SEQUENCE. */
3573 :
3574 : rtx_insn *
3575 0 : get_first_nonnote_nondebug_insn (void)
3576 : {
3577 0 : rtx_insn *insn = get_first_nonnote_insn ();
3578 0 : while (insn && (NOTE_P (insn) || DEBUG_INSN_P (insn)))
3579 0 : insn = next_insn (insn);
3580 0 : return insn;
3581 : }
3582 :
3583 : /* Return the last nonnote insn emitted in current sequence or current
3584 : function. This routine looks inside SEQUENCEs. */
3585 :
3586 : rtx_insn *
3587 0 : get_last_nonnote_insn (void)
3588 : {
3589 0 : rtx_insn *insn = get_last_insn ();
3590 :
3591 0 : if (insn)
3592 : {
3593 0 : if (NOTE_P (insn))
3594 0 : for (insn = previous_insn (insn);
3595 0 : insn && NOTE_P (insn);
3596 0 : insn = previous_insn (insn))
3597 0 : continue;
3598 : else
3599 : {
3600 0 : if (NONJUMP_INSN_P (insn))
3601 0 : if (rtx_sequence *seq = dyn_cast <rtx_sequence *> (PATTERN (insn)))
3602 0 : insn = seq->insn (seq->len () - 1);
3603 : }
3604 : }
3605 :
3606 0 : return insn;
3607 : }
3608 :
3609 : /* Return the number of actual (non-debug) insns emitted in this
3610 : function. */
3611 :
3612 : int
3613 3577947 : get_max_insn_count (void)
3614 : {
3615 3577947 : int n = cur_insn_uid;
3616 :
3617 : /* The table size must be stable across -g, to avoid codegen
3618 : differences due to debug insns, and not be affected by
3619 : -fmin-insn-uid, to avoid excessive table size and to simplify
3620 : debugging of -fcompare-debug failures. */
3621 3577947 : if (cur_debug_insn_uid > param_min_nondebug_insn_uid)
3622 3577947 : n -= cur_debug_insn_uid;
3623 : else
3624 0 : n -= param_min_nondebug_insn_uid;
3625 :
3626 3577947 : return n;
3627 : }
3628 :
3629 :
3630 : /* Return the next insn. If it is a SEQUENCE, return the first insn
3631 : of the sequence. */
3632 :
3633 : rtx_insn *
3634 1159725232 : next_insn (rtx_insn *insn)
3635 : {
3636 1159725232 : if (insn)
3637 : {
3638 1159725232 : insn = NEXT_INSN (insn);
3639 1159725232 : if (insn && NONJUMP_INSN_P (insn))
3640 14383213 : if (rtx_sequence *seq = dyn_cast <rtx_sequence *> (PATTERN (insn)))
3641 0 : insn = seq->insn (0);
3642 : }
3643 :
3644 1159725232 : return insn;
3645 : }
3646 :
3647 : /* Return the previous insn. If it is a SEQUENCE, return the last insn
3648 : of the sequence. */
3649 :
3650 : rtx_insn *
3651 72462 : previous_insn (rtx_insn *insn)
3652 : {
3653 72462 : if (insn)
3654 : {
3655 72462 : insn = PREV_INSN (insn);
3656 72462 : if (insn && NONJUMP_INSN_P (insn))
3657 8737 : if (rtx_sequence *seq = dyn_cast <rtx_sequence *> (PATTERN (insn)))
3658 0 : insn = seq->insn (seq->len () - 1);
3659 : }
3660 :
3661 72462 : return insn;
3662 : }
3663 :
3664 : /* Return the next insn after INSN that is not a NOTE. This routine does not
3665 : look inside SEQUENCEs. */
3666 :
3667 : rtx_insn *
3668 551040 : next_nonnote_insn (rtx_insn *insn)
3669 : {
3670 574634 : while (insn)
3671 : {
3672 574634 : insn = NEXT_INSN (insn);
3673 574634 : if (insn == 0 || !NOTE_P (insn))
3674 : break;
3675 : }
3676 :
3677 551040 : return insn;
3678 : }
3679 :
3680 : /* Return the next insn after INSN that is not a DEBUG_INSN. This
3681 : routine does not look inside SEQUENCEs. */
3682 :
3683 : rtx_insn *
3684 6206106 : next_nondebug_insn (rtx_insn *insn)
3685 : {
3686 8581019 : while (insn)
3687 : {
3688 8581019 : insn = NEXT_INSN (insn);
3689 8581019 : if (insn == 0 || !DEBUG_INSN_P (insn))
3690 : break;
3691 : }
3692 :
3693 6206106 : return insn;
3694 : }
3695 :
3696 : /* Return the previous insn before INSN that is not a NOTE. This routine does
3697 : not look inside SEQUENCEs. */
3698 :
3699 : rtx_insn *
3700 94517682 : prev_nonnote_insn (rtx_insn *insn)
3701 : {
3702 99494884 : while (insn)
3703 : {
3704 99494884 : insn = PREV_INSN (insn);
3705 99494884 : if (insn == 0 || !NOTE_P (insn))
3706 : break;
3707 : }
3708 :
3709 94517682 : return insn;
3710 : }
3711 :
3712 : /* Return the previous insn before INSN that is not a DEBUG_INSN.
3713 : This routine does not look inside SEQUENCEs. */
3714 :
3715 : rtx_insn *
3716 3176265 : prev_nondebug_insn (rtx_insn *insn)
3717 : {
3718 10935773 : while (insn)
3719 : {
3720 10935773 : insn = PREV_INSN (insn);
3721 10935773 : if (insn == 0 || !DEBUG_INSN_P (insn))
3722 : break;
3723 : }
3724 :
3725 3176265 : return insn;
3726 : }
3727 :
3728 : /* Return the next insn after INSN that is not a NOTE nor DEBUG_INSN.
3729 : This routine does not look inside SEQUENCEs. */
3730 :
3731 : rtx_insn *
3732 52589418 : next_nonnote_nondebug_insn (rtx_insn *insn)
3733 : {
3734 71512805 : while (insn)
3735 : {
3736 71512805 : insn = NEXT_INSN (insn);
3737 71512805 : if (insn == 0 || (!NOTE_P (insn) && !DEBUG_INSN_P (insn)))
3738 : break;
3739 : }
3740 :
3741 52589418 : return insn;
3742 : }
3743 :
3744 : /* Return the next insn after INSN that is not a NOTE nor DEBUG_INSN,
3745 : but stop the search before we enter another basic block. This
3746 : routine does not look inside SEQUENCEs.
3747 : NOTE: This can potentially bleed into next BB. If current insn is
3748 : last insn of BB, followed by a code_label before the start of
3749 : the next BB, code_label will be returned. But this is the
3750 : behavior rest of gcc assumes/relies on e.g. get_last_bb_insn. */
3751 :
3752 : rtx_insn *
3753 7608296 : next_nonnote_nondebug_insn_bb (rtx_insn *insn)
3754 : {
3755 7608296 : while (insn)
3756 : {
3757 7641826 : insn = NEXT_INSN (insn);
3758 7641826 : if (insn == 0)
3759 : break;
3760 7413971 : if (DEBUG_INSN_P (insn))
3761 0 : continue;
3762 7413971 : if (!NOTE_P (insn))
3763 : break;
3764 9820328 : if (NOTE_INSN_BASIC_BLOCK_P (insn))
3765 : return NULL;
3766 : }
3767 :
3768 : return insn;
3769 : }
3770 :
3771 : /* Return the previous insn before INSN that is not a NOTE nor DEBUG_INSN.
3772 : This routine does not look inside SEQUENCEs. */
3773 :
3774 : rtx_insn *
3775 145585522 : prev_nonnote_nondebug_insn (rtx_insn *insn)
3776 : {
3777 271707960 : while (insn)
3778 : {
3779 271707960 : insn = PREV_INSN (insn);
3780 271707960 : if (insn == 0 || (!NOTE_P (insn) && !DEBUG_INSN_P (insn)))
3781 : break;
3782 : }
3783 :
3784 145585522 : return insn;
3785 : }
3786 :
3787 : /* Return the previous insn before INSN that is not a NOTE nor
3788 : DEBUG_INSN, but stop the search before we enter another basic
3789 : block. This routine does not look inside SEQUENCEs. */
3790 :
3791 : rtx_insn *
3792 71214224 : prev_nonnote_nondebug_insn_bb (rtx_insn *insn)
3793 : {
3794 71214224 : while (insn)
3795 : {
3796 140542847 : insn = PREV_INSN (insn);
3797 140542847 : if (insn == 0)
3798 : break;
3799 140542843 : if (DEBUG_INSN_P (insn))
3800 59733577 : continue;
3801 80809266 : if (!NOTE_P (insn))
3802 : break;
3803 162112992 : if (NOTE_INSN_BASIC_BLOCK_P (insn))
3804 : return NULL;
3805 : }
3806 :
3807 : return insn;
3808 : }
3809 :
3810 : /* Return the next INSN, CALL_INSN, JUMP_INSN or DEBUG_INSN after INSN;
3811 : or 0, if there is none. This routine does not look inside
3812 : SEQUENCEs. */
3813 :
3814 : rtx_insn *
3815 4411951 : next_real_insn (rtx_insn *insn)
3816 : {
3817 4478118 : while (insn)
3818 : {
3819 4478118 : insn = NEXT_INSN (insn);
3820 4478118 : if (insn == 0 || INSN_P (insn))
3821 : break;
3822 : }
3823 :
3824 4411951 : return insn;
3825 : }
3826 :
3827 : /* Return the last INSN, CALL_INSN, JUMP_INSN or DEBUG_INSN before INSN;
3828 : or 0, if there is none. This routine does not look inside
3829 : SEQUENCEs. */
3830 :
3831 : rtx_insn *
3832 988342 : prev_real_insn (rtx_insn *insn)
3833 : {
3834 2923261 : while (insn)
3835 : {
3836 2923261 : insn = PREV_INSN (insn);
3837 2923261 : if (insn == 0 || INSN_P (insn))
3838 : break;
3839 : }
3840 :
3841 988342 : return insn;
3842 : }
3843 :
3844 : /* Return the next INSN, CALL_INSN or JUMP_INSN after INSN;
3845 : or 0, if there is none. This routine does not look inside
3846 : SEQUENCEs. */
3847 :
3848 : rtx_insn *
3849 0 : next_real_nondebug_insn (rtx_insn *insn)
3850 : {
3851 0 : while (insn)
3852 : {
3853 0 : insn = NEXT_INSN (insn);
3854 0 : if (insn == 0 || NONDEBUG_INSN_P (insn))
3855 : break;
3856 : }
3857 :
3858 0 : return insn;
3859 : }
3860 :
3861 : /* Return the last INSN, CALL_INSN or JUMP_INSN before INSN;
3862 : or 0, if there is none. This routine does not look inside
3863 : SEQUENCEs. */
3864 :
3865 : rtx_insn *
3866 5134812 : prev_real_nondebug_insn (rtx_insn *insn)
3867 : {
3868 15815901 : while (insn)
3869 : {
3870 15815901 : insn = PREV_INSN (insn);
3871 15815901 : if (insn == 0 || NONDEBUG_INSN_P (insn))
3872 : break;
3873 : }
3874 :
3875 5134812 : return insn;
3876 : }
3877 :
3878 : /* Return the last CALL_INSN in the current list, or 0 if there is none.
3879 : This routine does not look inside SEQUENCEs. */
3880 :
3881 : rtx_call_insn *
3882 12330731 : last_call_insn (void)
3883 : {
3884 12330731 : rtx_insn *insn;
3885 :
3886 12458208 : for (insn = get_last_insn ();
3887 12458208 : insn && !CALL_P (insn);
3888 127477 : insn = PREV_INSN (insn))
3889 : ;
3890 :
3891 12330731 : return safe_as_a <rtx_call_insn *> (insn);
3892 : }
3893 :
3894 : bool
3895 1187911227 : active_insn_p (const rtx_insn *insn)
3896 : {
3897 1187911227 : return (CALL_P (insn) || JUMP_P (insn)
3898 1187911227 : || JUMP_TABLE_DATA_P (insn) /* FIXME */
3899 1187911227 : || (NONJUMP_INSN_P (insn)
3900 702679318 : && (! reload_completed
3901 499441379 : || (GET_CODE (PATTERN (insn)) != USE
3902 497823831 : && GET_CODE (PATTERN (insn)) != CLOBBER))));
3903 : }
3904 :
3905 : /* Find the next insn after INSN that really does something. This routine
3906 : does not look inside SEQUENCEs. After reload this also skips over
3907 : standalone USE and CLOBBER insn. */
3908 :
3909 : rtx_insn *
3910 115679757 : next_active_insn (rtx_insn *insn)
3911 : {
3912 258346527 : while (insn)
3913 : {
3914 258346527 : insn = NEXT_INSN (insn);
3915 374026284 : if (insn == 0 || active_insn_p (insn))
3916 : break;
3917 : }
3918 :
3919 115679757 : return insn;
3920 : }
3921 :
3922 : /* Find the last insn before INSN that really does something. This routine
3923 : does not look inside SEQUENCEs. After reload this also skips over
3924 : standalone USE and CLOBBER insn. */
3925 :
3926 : rtx_insn *
3927 46323 : prev_active_insn (rtx_insn *insn)
3928 : {
3929 138046 : while (insn)
3930 : {
3931 138046 : insn = PREV_INSN (insn);
3932 184369 : if (insn == 0 || active_insn_p (insn))
3933 : break;
3934 : }
3935 :
3936 46323 : return insn;
3937 : }
3938 :
3939 : /* Find a RTX_AUTOINC class rtx which matches DATA. */
3940 :
3941 : static int
3942 0 : find_auto_inc (const_rtx x, const_rtx reg)
3943 : {
3944 0 : subrtx_iterator::array_type array;
3945 0 : FOR_EACH_SUBRTX (iter, array, x, NONCONST)
3946 : {
3947 0 : const_rtx x = *iter;
3948 0 : if (GET_RTX_CLASS (GET_CODE (x)) == RTX_AUTOINC
3949 0 : && rtx_equal_p (reg, XEXP (x, 0)))
3950 0 : return true;
3951 : }
3952 0 : return false;
3953 0 : }
3954 :
3955 : /* Increment the label uses for all labels present in rtx. */
3956 :
3957 : static void
3958 57166057 : mark_label_nuses (rtx x)
3959 : {
3960 57166057 : enum rtx_code code;
3961 57166057 : int i, j;
3962 57166057 : const char *fmt;
3963 :
3964 57166057 : code = GET_CODE (x);
3965 57166062 : if (code == LABEL_REF && LABEL_P (label_ref_label (x)))
3966 1 : LABEL_NUSES (label_ref_label (x))++;
3967 :
3968 57166057 : fmt = GET_RTX_FORMAT (code);
3969 140999719 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
3970 : {
3971 83833662 : if (fmt[i] == 'e')
3972 44884827 : mark_label_nuses (XEXP (x, i));
3973 38948835 : else if (fmt[i] == 'E')
3974 4013113 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
3975 2780303 : mark_label_nuses (XVECEXP (x, i, j));
3976 : }
3977 57166057 : }
3978 :
3979 :
3980 : /* Try splitting insns that can be split for better scheduling.
3981 : PAT is the pattern which might split.
3982 : TRIAL is the insn providing PAT.
3983 : LAST is nonzero if we should return the last insn of the sequence produced.
3984 :
3985 : If this routine succeeds in splitting, it returns the first or last
3986 : replacement insn depending on the value of LAST. Otherwise, it
3987 : returns TRIAL. If the insn to be returned can be split, it will be. */
3988 :
3989 : rtx_insn *
3990 404497076 : try_split (rtx pat, rtx_insn *trial, int last)
3991 : {
3992 404497076 : rtx_insn *before, *after;
3993 404497076 : rtx note;
3994 404497076 : rtx_insn *seq, *tem;
3995 404497076 : profile_probability probability;
3996 404497076 : rtx_insn *insn_last, *insn;
3997 404497076 : int njumps = 0;
3998 404497076 : rtx_insn *call_insn = NULL;
3999 :
4000 404497076 : if (any_condjump_p (trial)
4001 404497076 : && (note = find_reg_note (trial, REG_BR_PROB, 0)))
4002 14965041 : split_branch_probability
4003 14965041 : = profile_probability::from_reg_br_prob_note (XINT (note, 0));
4004 : else
4005 389532035 : split_branch_probability = profile_probability::uninitialized ();
4006 :
4007 404497076 : probability = split_branch_probability;
4008 :
4009 404497076 : seq = split_insns (pat, trial);
4010 :
4011 404497076 : split_branch_probability = profile_probability::uninitialized ();
4012 :
4013 404497076 : if (!seq)
4014 : return trial;
4015 :
4016 : int split_insn_count = 0;
4017 : /* Avoid infinite loop if any insn of the result matches
4018 : the original pattern. */
4019 : insn_last = seq;
4020 9516808 : while (1)
4021 : {
4022 9516808 : if (INSN_P (insn_last)
4023 9516808 : && rtx_equal_p (PATTERN (insn_last), pat))
4024 : return trial;
4025 9515289 : split_insn_count++;
4026 9515289 : if (!NEXT_INSN (insn_last))
4027 : break;
4028 : insn_last = NEXT_INSN (insn_last);
4029 : }
4030 :
4031 : /* We're not good at redistributing frame information if
4032 : the split occurs before reload or if it results in more
4033 : than one insn. */
4034 6287432 : if (RTX_FRAME_RELATED_P (trial))
4035 : {
4036 37 : if (!reload_completed || split_insn_count != 1)
4037 : return trial;
4038 :
4039 37 : rtx_insn *new_insn = seq;
4040 37 : rtx_insn *old_insn = trial;
4041 37 : copy_frame_info_to_split_insn (old_insn, new_insn);
4042 : }
4043 :
4044 : /* We will be adding the new sequence to the function. The splitters
4045 : may have introduced invalid RTL sharing, so unshare the sequence now. */
4046 6287432 : unshare_all_rtl_in_chain (seq);
4047 :
4048 : /* Mark labels and copy flags. */
4049 22090153 : for (insn = insn_last; insn ; insn = PREV_INSN (insn))
4050 : {
4051 9515289 : if (JUMP_P (insn))
4052 : {
4053 6198 : if (JUMP_P (trial))
4054 5358 : set_crossing_jump (insn, CROSSING_JUMP_P (trial));
4055 6198 : mark_jump_label (PATTERN (insn), insn, 0);
4056 6198 : njumps++;
4057 6198 : if (probability.initialized_p ()
4058 5326 : && any_condjump_p (insn)
4059 11524 : && !find_reg_note (insn, REG_BR_PROB, 0))
4060 : {
4061 : /* We can preserve the REG_BR_PROB notes only if exactly
4062 : one jump is created, otherwise the machine description
4063 : is responsible for this step using
4064 : split_branch_probability variable. */
4065 5326 : gcc_assert (njumps == 1);
4066 5326 : add_reg_br_prob_note (insn, probability);
4067 : }
4068 : }
4069 : }
4070 :
4071 : /* If we are splitting a CALL_INSN, look for the CALL_INSN
4072 : in SEQ and copy any additional information across. */
4073 6287432 : if (auto call_trial = dyn_cast<rtx_call_insn *> (trial))
4074 : {
4075 0 : for (insn = insn_last; insn ; insn = PREV_INSN (insn))
4076 0 : if (auto call = dyn_cast<rtx_call_insn *> (insn))
4077 : {
4078 0 : gcc_assert (call_insn == NULL_RTX);
4079 0 : call_insn = insn;
4080 :
4081 : /* Add the old CALL_INSN_FUNCTION_USAGE to whatever the
4082 : target may have explicitly specified. */
4083 0 : rtx *p = &CALL_INSN_FUNCTION_USAGE (call);
4084 0 : while (*p)
4085 0 : p = &XEXP (*p, 1);
4086 0 : *p = CALL_INSN_FUNCTION_USAGE (call_trial);
4087 :
4088 : /* Preserve the ABI information from the original call. */
4089 0 : CALL_INSN_ABI_ID (call) = CALL_INSN_ABI_ID (call_trial);
4090 :
4091 : /* If the old call was a sibling call, the new one must
4092 : be too. */
4093 0 : SIBLING_CALL_P (insn) = SIBLING_CALL_P (trial);
4094 : }
4095 : }
4096 :
4097 : /* Copy notes, particularly those related to the CFG. */
4098 8149981 : for (note = REG_NOTES (trial); note; note = XEXP (note, 1))
4099 : {
4100 1862549 : switch (REG_NOTE_KIND (note))
4101 : {
4102 2790 : case REG_EH_REGION:
4103 2790 : copy_reg_eh_region_note_backward (note, insn_last, NULL);
4104 2790 : break;
4105 :
4106 : case REG_NORETURN:
4107 : case REG_SETJMP:
4108 : case REG_TM:
4109 : case REG_CALL_NOCF_CHECK:
4110 : case REG_CALL_ARG_LOCATION:
4111 0 : for (insn = insn_last; insn != NULL_RTX; insn = PREV_INSN (insn))
4112 : {
4113 0 : if (CALL_P (insn))
4114 0 : add_reg_note (insn, REG_NOTE_KIND (note), XEXP (note, 0));
4115 : }
4116 : break;
4117 :
4118 : case REG_NON_LOCAL_GOTO:
4119 : case REG_LABEL_TARGET:
4120 0 : for (insn = insn_last; insn != NULL_RTX; insn = PREV_INSN (insn))
4121 : {
4122 0 : if (JUMP_P (insn))
4123 0 : add_reg_note (insn, REG_NOTE_KIND (note), XEXP (note, 0));
4124 : }
4125 : break;
4126 :
4127 : case REG_INC:
4128 : if (!AUTO_INC_DEC)
4129 : break;
4130 :
4131 : for (insn = insn_last; insn != NULL_RTX; insn = PREV_INSN (insn))
4132 : {
4133 : rtx reg = XEXP (note, 0);
4134 : if (!FIND_REG_INC_NOTE (insn, reg)
4135 : && find_auto_inc (PATTERN (insn), reg))
4136 : add_reg_note (insn, REG_INC, reg);
4137 : }
4138 : break;
4139 :
4140 306100 : case REG_ARGS_SIZE:
4141 306100 : fixup_args_size_notes (NULL, insn_last, get_args_size (note));
4142 306100 : break;
4143 :
4144 0 : case REG_CALL_DECL:
4145 0 : case REG_UNTYPED_CALL:
4146 0 : gcc_assert (call_insn != NULL_RTX);
4147 0 : add_reg_note (call_insn, REG_NOTE_KIND (note), XEXP (note, 0));
4148 0 : break;
4149 :
4150 : default:
4151 : break;
4152 : }
4153 : }
4154 :
4155 : /* If there are LABELS inside the split insns increment the
4156 : usage count so we don't delete the label. */
4157 6287432 : if (INSN_P (trial))
4158 : {
4159 : insn = insn_last;
4160 15802721 : while (insn != NULL_RTX)
4161 : {
4162 : /* JUMP_P insns have already been "marked" above. */
4163 9515289 : if (NONJUMP_INSN_P (insn))
4164 9500927 : mark_label_nuses (PATTERN (insn));
4165 :
4166 9515289 : insn = PREV_INSN (insn);
4167 : }
4168 : }
4169 :
4170 6287432 : before = PREV_INSN (trial);
4171 6287432 : after = NEXT_INSN (trial);
4172 :
4173 6287432 : emit_insn_after_setloc (seq, trial, INSN_LOCATION (trial));
4174 :
4175 6287432 : delete_insn (trial);
4176 :
4177 : /* Recursively call try_split for each new insn created; by the
4178 : time control returns here that insn will be fully split, so
4179 : set LAST and continue from the insn after the one returned.
4180 : We can't use next_active_insn here since AFTER may be a note.
4181 : Ignore deleted insns, which can be occur if not optimizing. */
4182 22090153 : for (tem = NEXT_INSN (before); tem != after; tem = NEXT_INSN (tem))
4183 9515289 : if (! tem->deleted () && INSN_P (tem))
4184 9507125 : tem = try_split (PATTERN (tem), tem, 1);
4185 :
4186 : /* Return either the first or the last insn, depending on which was
4187 : requested. */
4188 6287432 : return last
4189 6287432 : ? (after ? PREV_INSN (after) : get_last_insn ())
4190 0 : : NEXT_INSN (before);
4191 : }
4192 :
4193 : /* Make and return an INSN rtx, initializing all its slots.
4194 : Store PATTERN in the pattern slots. */
4195 :
4196 : rtx_insn *
4197 129320420 : make_insn_raw (rtx pattern)
4198 : {
4199 129320420 : rtx_insn *insn;
4200 :
4201 129320420 : insn = as_a <rtx_insn *> (rtx_alloc (INSN));
4202 :
4203 129320420 : INSN_UID (insn) = cur_insn_uid++;
4204 129320420 : PATTERN (insn) = pattern;
4205 129320420 : INSN_CODE (insn) = -1;
4206 129320420 : REG_NOTES (insn) = NULL;
4207 129320420 : INSN_LOCATION (insn) = curr_insn_location ();
4208 129320420 : BLOCK_FOR_INSN (insn) = NULL;
4209 :
4210 : #ifdef ENABLE_RTL_CHECKING
4211 : if (insn
4212 : && INSN_P (insn)
4213 : && (returnjump_p (insn)
4214 : || (GET_CODE (insn) == SET
4215 : && SET_DEST (insn) == pc_rtx)))
4216 : {
4217 : warning (0, "ICE: %<emit_insn%> used where %<emit_jump_insn%> needed:");
4218 : debug_rtx (insn);
4219 : }
4220 : #endif
4221 :
4222 129320420 : return insn;
4223 : }
4224 :
4225 : /* Like `make_insn_raw' but make a DEBUG_INSN instead of an insn. */
4226 :
4227 : static rtx_insn *
4228 54478789 : make_debug_insn_raw (rtx pattern)
4229 : {
4230 54478789 : rtx_debug_insn *insn;
4231 :
4232 54478789 : insn = as_a <rtx_debug_insn *> (rtx_alloc (DEBUG_INSN));
4233 54478789 : INSN_UID (insn) = cur_debug_insn_uid++;
4234 54478789 : if (cur_debug_insn_uid > param_min_nondebug_insn_uid)
4235 54478789 : INSN_UID (insn) = cur_insn_uid++;
4236 :
4237 54478789 : PATTERN (insn) = pattern;
4238 54478789 : INSN_CODE (insn) = -1;
4239 54478789 : REG_NOTES (insn) = NULL;
4240 54478789 : INSN_LOCATION (insn) = curr_insn_location ();
4241 54478789 : BLOCK_FOR_INSN (insn) = NULL;
4242 :
4243 54478789 : return insn;
4244 : }
4245 :
4246 : /* Like `make_insn_raw' but make a JUMP_INSN instead of an insn. */
4247 :
4248 : static rtx_insn *
4249 18485139 : make_jump_insn_raw (rtx pattern)
4250 : {
4251 18485139 : rtx_jump_insn *insn;
4252 :
4253 18485139 : insn = as_a <rtx_jump_insn *> (rtx_alloc (JUMP_INSN));
4254 18485139 : INSN_UID (insn) = cur_insn_uid++;
4255 :
4256 18485139 : PATTERN (insn) = pattern;
4257 18485139 : INSN_CODE (insn) = -1;
4258 18485139 : REG_NOTES (insn) = NULL;
4259 18485139 : JUMP_LABEL (insn) = NULL;
4260 18485139 : INSN_LOCATION (insn) = curr_insn_location ();
4261 18485139 : BLOCK_FOR_INSN (insn) = NULL;
4262 :
4263 18485139 : return insn;
4264 : }
4265 :
4266 : /* Like `make_insn_raw' but make a CALL_INSN instead of an insn. */
4267 :
4268 : static rtx_insn *
4269 6479861 : make_call_insn_raw (rtx pattern)
4270 : {
4271 6479861 : rtx_call_insn *insn;
4272 :
4273 6479861 : insn = as_a <rtx_call_insn *> (rtx_alloc (CALL_INSN));
4274 6479861 : INSN_UID (insn) = cur_insn_uid++;
4275 :
4276 6479861 : PATTERN (insn) = pattern;
4277 6479861 : INSN_CODE (insn) = -1;
4278 6479861 : REG_NOTES (insn) = NULL;
4279 6479861 : CALL_INSN_FUNCTION_USAGE (insn) = NULL;
4280 6479861 : CALL_INSN_ABI_ID (insn) = 0;
4281 6479861 : INSN_LOCATION (insn) = curr_insn_location ();
4282 6479861 : BLOCK_FOR_INSN (insn) = NULL;
4283 :
4284 6479861 : return insn;
4285 : }
4286 :
4287 : /* Like `make_insn_raw' but make a NOTE instead of an insn. */
4288 :
4289 : static rtx_note *
4290 176332756 : make_note_raw (enum insn_note subtype)
4291 : {
4292 : /* Some notes are never created this way at all. These notes are
4293 : only created by patching out insns. */
4294 176332756 : gcc_assert (subtype != NOTE_INSN_DELETED_LABEL
4295 : && subtype != NOTE_INSN_DELETED_DEBUG_LABEL);
4296 :
4297 176332756 : rtx_note *note = as_a <rtx_note *> (rtx_alloc (NOTE));
4298 176332756 : INSN_UID (note) = cur_insn_uid++;
4299 176332756 : NOTE_KIND (note) = subtype;
4300 176332756 : BLOCK_FOR_INSN (note) = NULL;
4301 176332756 : memset (&NOTE_DATA (note), 0, sizeof (NOTE_DATA (note)));
4302 176332756 : return note;
4303 : }
4304 :
4305 : /* Add INSN to the end of the doubly-linked list, between PREV and NEXT.
4306 : INSN may be any object that can appear in the chain: INSN_P and NOTE_P objects,
4307 : but also BARRIERs and JUMP_TABLE_DATAs. PREV and NEXT may be NULL. */
4308 :
4309 : static inline void
4310 616051534 : link_insn_into_chain (rtx_insn *insn, rtx_insn *prev, rtx_insn *next)
4311 : {
4312 616051534 : SET_PREV_INSN (insn) = prev;
4313 616051534 : SET_NEXT_INSN (insn) = next;
4314 616051534 : if (prev != NULL)
4315 : {
4316 443573384 : SET_NEXT_INSN (prev) = insn;
4317 443573384 : if (NONJUMP_INSN_P (prev))
4318 177568425 : if (rtx_sequence *sequence
4319 177568425 : = dyn_cast <rtx_sequence *> (PATTERN (prev)))
4320 0 : SET_NEXT_INSN (sequence->insn (sequence->len () - 1)) = insn;
4321 : }
4322 616051534 : if (next != NULL)
4323 : {
4324 181391118 : SET_PREV_INSN (next) = insn;
4325 181391118 : if (NONJUMP_INSN_P (next))
4326 49288362 : if (rtx_sequence *sequence
4327 49288362 : = dyn_cast <rtx_sequence *> (PATTERN (next)))
4328 0 : SET_PREV_INSN (sequence->insn (0)) = insn;
4329 : }
4330 :
4331 616051534 : if (NONJUMP_INSN_P (insn))
4332 301126026 : if (rtx_sequence *sequence
4333 301126026 : = dyn_cast <rtx_sequence *> (PATTERN (insn)))
4334 : {
4335 0 : SET_PREV_INSN (sequence->insn (0)) = prev;
4336 0 : SET_NEXT_INSN (sequence->insn (sequence->len () - 1)) = next;
4337 : }
4338 616051534 : }
4339 :
4340 : /* Add INSN to the end of the doubly-linked list.
4341 : INSN may be an INSN, JUMP_INSN, CALL_INSN, CODE_LABEL, BARRIER or NOTE. */
4342 :
4343 : void
4344 423334214 : add_insn (rtx_insn *insn)
4345 : {
4346 423334214 : rtx_insn *prev = get_last_insn ();
4347 423334214 : link_insn_into_chain (insn, prev, NULL);
4348 423334214 : if (get_insns () == NULL)
4349 172288516 : set_first_insn (insn);
4350 423334214 : set_last_insn (insn);
4351 423334214 : }
4352 :
4353 : /* Add INSN into the doubly-linked list after insn AFTER. */
4354 :
4355 : static void
4356 71418036 : add_insn_after_nobb (rtx_insn *insn, rtx_insn *after)
4357 : {
4358 71418036 : rtx_insn *next = NEXT_INSN (after);
4359 :
4360 71418036 : gcc_assert (!optimize || !after->deleted ());
4361 :
4362 71418036 : link_insn_into_chain (insn, after, next);
4363 :
4364 71418036 : if (next == NULL)
4365 : {
4366 : struct sequence_stack *seq;
4367 :
4368 11326202 : for (seq = get_current_sequence (); seq; seq = seq->next)
4369 11326202 : if (after == seq->last)
4370 : {
4371 11326202 : seq->last = insn;
4372 11326202 : break;
4373 : }
4374 : }
4375 71418036 : }
4376 :
4377 : /* Add INSN into the doubly-linked list before insn BEFORE. */
4378 :
4379 : static void
4380 121299284 : add_insn_before_nobb (rtx_insn *insn, rtx_insn *before)
4381 : {
4382 121299284 : rtx_insn *prev = PREV_INSN (before);
4383 :
4384 121299284 : gcc_assert (!optimize || !before->deleted ());
4385 :
4386 121299284 : link_insn_into_chain (insn, prev, before);
4387 :
4388 121299284 : if (prev == NULL)
4389 : {
4390 : struct sequence_stack *seq;
4391 :
4392 189634 : for (seq = get_current_sequence (); seq; seq = seq->next)
4393 189634 : if (before == seq->first)
4394 : {
4395 189634 : seq->first = insn;
4396 189634 : break;
4397 : }
4398 :
4399 189634 : gcc_assert (seq);
4400 : }
4401 121299284 : }
4402 :
4403 : /* Like add_insn_after_nobb, but try to set BLOCK_FOR_INSN.
4404 : If BB is NULL, an attempt is made to infer the bb from before.
4405 :
4406 : This and the next function should be the only functions called
4407 : to insert an insn once delay slots have been filled since only
4408 : they know how to update a SEQUENCE. */
4409 :
4410 : void
4411 69965800 : add_insn_after (rtx_insn *insn, rtx_insn *after, basic_block bb)
4412 : {
4413 69965800 : add_insn_after_nobb (insn, after);
4414 69965800 : if (!BARRIER_P (after)
4415 69934775 : && !BARRIER_P (insn)
4416 132080047 : && (bb = BLOCK_FOR_INSN (after)))
4417 : {
4418 51484385 : set_block_for_insn (insn, bb);
4419 51484385 : if (INSN_P (insn))
4420 842535 : df_insn_rescan (insn);
4421 : /* Should not happen as first in the BB is always
4422 : either NOTE or LABEL. */
4423 51484385 : if (BB_END (bb) == after
4424 : /* Avoid clobbering of structure when creating new BB. */
4425 1025066 : && !BARRIER_P (insn)
4426 51484385 : && !NOTE_INSN_BASIC_BLOCK_P (insn))
4427 334004 : BB_END (bb) = insn;
4428 : }
4429 69965800 : }
4430 :
4431 : /* Like add_insn_before_nobb, but try to set BLOCK_FOR_INSN.
4432 : If BB is NULL, an attempt is made to infer the bb from before.
4433 :
4434 : This and the previous function should be the only functions called
4435 : to insert an insn once delay slots have been filled since only
4436 : they know how to update a SEQUENCE. */
4437 :
4438 : void
4439 97812595 : add_insn_before (rtx_insn *insn, rtx_insn *before, basic_block bb)
4440 : {
4441 97812595 : add_insn_before_nobb (insn, before);
4442 :
4443 97812595 : if (BARRIER_P (insn))
4444 : return;
4445 :
4446 97812478 : if (!bb
4447 23254396 : && !BARRIER_P (before))
4448 23254395 : bb = BLOCK_FOR_INSN (before);
4449 :
4450 23254396 : if (bb)
4451 : {
4452 94086585 : set_block_for_insn (insn, bb);
4453 94086585 : if (INSN_P (insn))
4454 13764508 : df_insn_rescan (insn);
4455 : /* Should not happen as first in the BB is always either NOTE or
4456 : LABEL. */
4457 94086585 : gcc_assert (BB_HEAD (bb) != insn
4458 : /* Avoid clobbering of structure when creating new BB. */
4459 : || BARRIER_P (insn)
4460 : || NOTE_INSN_BASIC_BLOCK_P (insn));
4461 : }
4462 : }
4463 :
4464 : /* Replace insn with an deleted instruction note. */
4465 :
4466 : void
4467 14456498 : set_insn_deleted (rtx_insn *insn)
4468 : {
4469 14456498 : if (INSN_P (insn))
4470 14456498 : df_insn_delete (insn);
4471 14456498 : PUT_CODE (insn, NOTE);
4472 14456498 : NOTE_KIND (insn) = NOTE_INSN_DELETED;
4473 14456498 : }
4474 :
4475 :
4476 : /* Unlink INSN from the insn chain.
4477 :
4478 : This function knows how to handle sequences.
4479 :
4480 : This function does not invalidate data flow information associated with
4481 : INSN (i.e. does not call df_insn_delete). That makes this function
4482 : usable for only disconnecting an insn from the chain, and re-emit it
4483 : elsewhere later.
4484 :
4485 : To later insert INSN elsewhere in the insn chain via add_insn and
4486 : similar functions, PREV_INSN and NEXT_INSN must be nullified by
4487 : the caller. Nullifying them here breaks many insn chain walks.
4488 :
4489 : To really delete an insn and related DF information, use delete_insn. */
4490 :
4491 : void
4492 141962512 : remove_insn (rtx_insn *insn)
4493 : {
4494 141962512 : rtx_insn *next = NEXT_INSN (insn);
4495 141962512 : rtx_insn *prev = PREV_INSN (insn);
4496 141962512 : basic_block bb;
4497 :
4498 141962512 : if (prev)
4499 : {
4500 141962253 : SET_NEXT_INSN (prev) = next;
4501 141962253 : if (NONJUMP_INSN_P (prev))
4502 52317896 : if (rtx_sequence *sequence
4503 52317896 : = dyn_cast <rtx_sequence *> (PATTERN (prev)))
4504 0 : SET_NEXT_INSN (sequence->insn (sequence->len () - 1)) = next;
4505 : }
4506 : else
4507 : {
4508 : struct sequence_stack *seq;
4509 :
4510 259 : for (seq = get_current_sequence (); seq; seq = seq->next)
4511 259 : if (insn == seq->first)
4512 : {
4513 259 : seq->first = next;
4514 259 : break;
4515 : }
4516 :
4517 259 : gcc_assert (seq);
4518 : }
4519 :
4520 141962512 : if (next)
4521 : {
4522 139831287 : SET_PREV_INSN (next) = prev;
4523 139831287 : if (NONJUMP_INSN_P (next))
4524 47142166 : if (rtx_sequence *sequence
4525 47142166 : = dyn_cast <rtx_sequence *> (PATTERN (next)))
4526 0 : SET_PREV_INSN (sequence->insn (0)) = prev;
4527 : }
4528 : else
4529 : {
4530 : struct sequence_stack *seq;
4531 :
4532 2131225 : for (seq = get_current_sequence (); seq; seq = seq->next)
4533 2131225 : if (insn == seq->last)
4534 : {
4535 2131225 : seq->last = prev;
4536 2131225 : break;
4537 : }
4538 :
4539 2131225 : gcc_assert (seq);
4540 : }
4541 :
4542 : /* Fix up basic block boundaries, if necessary. */
4543 141962512 : if (!BARRIER_P (insn)
4544 141962512 : && (bb = BLOCK_FOR_INSN (insn)))
4545 : {
4546 138892617 : if (BB_HEAD (bb) == insn)
4547 : {
4548 : /* Never ever delete the basic block note without deleting whole
4549 : basic block. */
4550 2481381 : gcc_assert (!NOTE_P (insn));
4551 2481381 : BB_HEAD (bb) = next;
4552 : }
4553 138892617 : if (BB_END (bb) == insn)
4554 23873755 : BB_END (bb) = prev;
4555 : }
4556 141962512 : }
4557 :
4558 : /* Append CALL_FUSAGE to the CALL_INSN_FUNCTION_USAGE for CALL_INSN. */
4559 :
4560 : void
4561 6425601 : add_function_usage_to (rtx_insn *call_insn, rtx call_fusage)
4562 : {
4563 6425601 : gcc_assert (call_insn && CALL_P (call_insn));
4564 :
4565 : /* Put the register usage information on the CALL. If there is already
4566 : some usage information, put ours at the end. */
4567 6425601 : if (CALL_INSN_FUNCTION_USAGE (call_insn))
4568 : {
4569 : rtx link;
4570 :
4571 572385 : for (link = CALL_INSN_FUNCTION_USAGE (call_insn); XEXP (link, 1) != 0;
4572 : link = XEXP (link, 1))
4573 : ;
4574 :
4575 572384 : XEXP (link, 1) = call_fusage;
4576 : }
4577 : else
4578 5853217 : CALL_INSN_FUNCTION_USAGE (call_insn) = call_fusage;
4579 6425601 : }
4580 :
4581 : /* Delete all insns made since FROM.
4582 : FROM becomes the new last instruction. */
4583 :
4584 : void
4585 1332083 : delete_insns_since (rtx_insn *from)
4586 : {
4587 1332083 : if (from == 0)
4588 143483 : set_first_insn (0);
4589 : else
4590 1188600 : SET_NEXT_INSN (from) = 0;
4591 1332083 : set_last_insn (from);
4592 1332083 : }
4593 :
4594 : /* This function is deprecated, please use sequences instead.
4595 :
4596 : Move a consecutive bunch of insns to a different place in the chain.
4597 : The insns to be moved are those between FROM and TO.
4598 : They are moved to a new position after the insn AFTER.
4599 : AFTER must not be FROM or TO or any insn in between.
4600 :
4601 : This function does not know about SEQUENCEs and hence should not be
4602 : called after delay-slot filling has been done. */
4603 :
4604 : void
4605 1836303 : reorder_insns_nobb (rtx_insn *from, rtx_insn *to, rtx_insn *after)
4606 : {
4607 1836303 : if (flag_checking)
4608 : {
4609 2471295 : for (rtx_insn *x = from; x != to; x = NEXT_INSN (x))
4610 635015 : gcc_assert (after != x);
4611 1836280 : gcc_assert (after != to);
4612 : }
4613 :
4614 : /* Splice this bunch out of where it is now. */
4615 1836303 : if (PREV_INSN (from))
4616 1836303 : SET_NEXT_INSN (PREV_INSN (from)) = NEXT_INSN (to);
4617 1836303 : if (NEXT_INSN (to))
4618 1826494 : SET_PREV_INSN (NEXT_INSN (to)) = PREV_INSN (from);
4619 1836303 : if (get_last_insn () == to)
4620 9809 : set_last_insn (PREV_INSN (from));
4621 1836303 : if (get_insns () == from)
4622 0 : set_first_insn (NEXT_INSN (to));
4623 :
4624 : /* Make the new neighbors point to it and it to them. */
4625 1836303 : if (NEXT_INSN (after))
4626 1828396 : SET_PREV_INSN (NEXT_INSN (after)) = to;
4627 :
4628 1836303 : SET_NEXT_INSN (to) = NEXT_INSN (after);
4629 1836303 : SET_PREV_INSN (from) = after;
4630 1836303 : SET_NEXT_INSN (after) = from;
4631 1836303 : if (after == get_last_insn ())
4632 7907 : set_last_insn (to);
4633 1836303 : }
4634 :
4635 : /* Same as function above, but take care to update BB boundaries. */
4636 : void
4637 1192734 : reorder_insns (rtx_insn *from, rtx_insn *to, rtx_insn *after)
4638 : {
4639 1192734 : rtx_insn *prev = PREV_INSN (from);
4640 1192734 : basic_block bb, bb2;
4641 :
4642 1192734 : reorder_insns_nobb (from, to, after);
4643 :
4644 1192734 : if (!BARRIER_P (after)
4645 1192734 : && (bb = BLOCK_FOR_INSN (after)))
4646 : {
4647 1185075 : rtx_insn *x;
4648 1185075 : df_set_bb_dirty (bb);
4649 :
4650 1185075 : if (!BARRIER_P (from)
4651 1185075 : && (bb2 = BLOCK_FOR_INSN (from)))
4652 : {
4653 1185075 : if (BB_END (bb2) == to)
4654 19481 : BB_END (bb2) = prev;
4655 1185075 : df_set_bb_dirty (bb2);
4656 : }
4657 :
4658 1185075 : if (BB_END (bb) == after)
4659 339028 : BB_END (bb) = to;
4660 :
4661 2425575 : for (x = from; x != NEXT_INSN (to); x = NEXT_INSN (x))
4662 1240500 : if (!BARRIER_P (x))
4663 1240500 : df_insn_change_bb (x, bb);
4664 : }
4665 1192734 : }
4666 :
4667 :
4668 : /* Emit insn(s) of given code and pattern
4669 : at a specified place within the doubly-linked list.
4670 :
4671 : All of the emit_foo global entry points accept an object
4672 : X which is either an insn list or a PATTERN of a single
4673 : instruction.
4674 :
4675 : There are thus a few canonical ways to generate code and
4676 : emit it at a specific place in the instruction stream. For
4677 : example, consider the instruction named SPOT and the fact that
4678 : we would like to emit some instructions before SPOT. We might
4679 : do it like this:
4680 :
4681 : start_sequence ();
4682 : ... emit the new instructions ...
4683 : insns_head = end_sequence ();
4684 :
4685 : emit_insn_before (insns_head, SPOT);
4686 :
4687 : It used to be common to generate SEQUENCE rtl instead, but that
4688 : is a relic of the past which no longer occurs. The reason is that
4689 : SEQUENCE rtl results in much fragmented RTL memory since the SEQUENCE
4690 : generated would almost certainly die right after it was created. */
4691 :
4692 : static rtx_insn *
4693 12008851 : emit_pattern_before_noloc (rtx x, rtx_insn *before, rtx_insn *last,
4694 : basic_block bb,
4695 : rtx_insn *(*make_raw) (rtx))
4696 : {
4697 12008851 : rtx_insn *insn;
4698 :
4699 12008851 : gcc_assert (before);
4700 :
4701 12008851 : if (x == NULL_RTX)
4702 : return last;
4703 :
4704 11993326 : switch (GET_CODE (x))
4705 : {
4706 10469251 : case DEBUG_INSN:
4707 10469251 : case INSN:
4708 10469251 : case JUMP_INSN:
4709 10469251 : case CALL_INSN:
4710 10469251 : case CODE_LABEL:
4711 10469251 : case BARRIER:
4712 10469251 : case NOTE:
4713 10469251 : insn = as_a <rtx_insn *> (x);
4714 33854264 : while (insn)
4715 : {
4716 12915762 : rtx_insn *next = NEXT_INSN (insn);
4717 12915762 : add_insn_before (insn, before, bb);
4718 12915762 : last = insn;
4719 12915762 : insn = next;
4720 : }
4721 : break;
4722 :
4723 : #ifdef ENABLE_RTL_CHECKING
4724 : case SEQUENCE:
4725 : gcc_unreachable ();
4726 : break;
4727 : #endif
4728 :
4729 1524075 : default:
4730 1524075 : last = (*make_raw) (x);
4731 1524075 : add_insn_before (last, before, bb);
4732 1524075 : break;
4733 : }
4734 :
4735 : return last;
4736 : }
4737 :
4738 : /* Make X be output before the instruction BEFORE. */
4739 :
4740 : rtx_insn *
4741 483938 : emit_insn_before_noloc (rtx x, rtx_insn *before, basic_block bb)
4742 : {
4743 483938 : return emit_pattern_before_noloc (x, before, before, bb, make_insn_raw);
4744 : }
4745 :
4746 : /* Make an instruction with body X and code JUMP_INSN
4747 : and output it before the instruction BEFORE. */
4748 :
4749 : rtx_jump_insn *
4750 0 : emit_jump_insn_before_noloc (rtx x, rtx_insn *before)
4751 : {
4752 0 : return as_a <rtx_jump_insn *> (
4753 : emit_pattern_before_noloc (x, before, NULL, NULL,
4754 0 : make_jump_insn_raw));
4755 : }
4756 :
4757 : /* Make an instruction with body X and code CALL_INSN
4758 : and output it before the instruction BEFORE. */
4759 :
4760 : rtx_insn *
4761 0 : emit_call_insn_before_noloc (rtx x, rtx_insn *before)
4762 : {
4763 0 : return emit_pattern_before_noloc (x, before, NULL, NULL,
4764 0 : make_call_insn_raw);
4765 : }
4766 :
4767 : /* Make an instruction with body X and code DEBUG_INSN
4768 : and output it before the instruction BEFORE. */
4769 :
4770 : rtx_insn *
4771 0 : emit_debug_insn_before_noloc (rtx x, rtx_insn *before)
4772 : {
4773 0 : return emit_pattern_before_noloc (x, before, NULL, NULL,
4774 0 : make_debug_insn_raw);
4775 : }
4776 :
4777 : /* Make an insn of code BARRIER
4778 : and output it before the insn BEFORE. */
4779 :
4780 : rtx_barrier *
4781 0 : emit_barrier_before (rtx_insn *before)
4782 : {
4783 0 : rtx_barrier *insn = as_a <rtx_barrier *> (rtx_alloc (BARRIER));
4784 :
4785 0 : INSN_UID (insn) = cur_insn_uid++;
4786 :
4787 0 : add_insn_before (insn, before, NULL);
4788 0 : return insn;
4789 : }
4790 :
4791 : /* Emit the label LABEL before the insn BEFORE. */
4792 :
4793 : rtx_code_label *
4794 6714161 : emit_label_before (rtx_code_label *label, rtx_insn *before)
4795 : {
4796 6714161 : gcc_checking_assert (INSN_UID (label) == 0);
4797 6714161 : INSN_UID (label) = cur_insn_uid++;
4798 6714161 : add_insn_before (label, before, NULL);
4799 6714161 : return label;
4800 : }
4801 :
4802 : /* Helper for emit_insn_after, handles lists of instructions
4803 : efficiently. */
4804 :
4805 : static rtx_insn *
4806 24959210 : emit_insn_after_1 (rtx_insn *first, rtx_insn *after, basic_block bb)
4807 : {
4808 24959210 : rtx_insn *last;
4809 24959210 : rtx_insn *after_after;
4810 24959210 : if (!bb && !BARRIER_P (after))
4811 20282827 : bb = BLOCK_FOR_INSN (after);
4812 :
4813 20282827 : if (bb)
4814 : {
4815 24959115 : df_set_bb_dirty (bb);
4816 68912616 : for (last = first; NEXT_INSN (last); last = NEXT_INSN (last))
4817 18994386 : if (!BARRIER_P (last))
4818 : {
4819 18992674 : set_block_for_insn (last, bb);
4820 18992674 : df_insn_rescan (last);
4821 : }
4822 24959115 : if (!BARRIER_P (last))
4823 : {
4824 24959115 : set_block_for_insn (last, bb);
4825 24959115 : df_insn_rescan (last);
4826 : }
4827 24959115 : if (BB_END (bb) == after)
4828 9234171 : BB_END (bb) = last;
4829 : }
4830 : else
4831 95 : for (last = first; NEXT_INSN (last); last = NEXT_INSN (last))
4832 0 : continue;
4833 :
4834 24959210 : after_after = NEXT_INSN (after);
4835 :
4836 24959210 : SET_NEXT_INSN (after) = first;
4837 24959210 : SET_PREV_INSN (first) = after;
4838 24959210 : SET_NEXT_INSN (last) = after_after;
4839 24959210 : if (after_after)
4840 24846455 : SET_PREV_INSN (after_after) = last;
4841 :
4842 24959210 : if (after == get_last_insn ())
4843 112755 : set_last_insn (last);
4844 :
4845 24959210 : return last;
4846 : }
4847 :
4848 : static rtx_insn *
4849 29989381 : emit_pattern_after_noloc (rtx x, rtx_insn *after, basic_block bb,
4850 : rtx_insn *(*make_raw)(rtx))
4851 : {
4852 29989381 : rtx_insn *last = after;
4853 :
4854 29989381 : gcc_assert (after);
4855 :
4856 29989381 : if (x == NULL_RTX)
4857 : return last;
4858 :
4859 29423604 : switch (GET_CODE (x))
4860 : {
4861 24959210 : case DEBUG_INSN:
4862 24959210 : case INSN:
4863 24959210 : case JUMP_INSN:
4864 24959210 : case CALL_INSN:
4865 24959210 : case CODE_LABEL:
4866 24959210 : case BARRIER:
4867 24959210 : case NOTE:
4868 24959210 : last = emit_insn_after_1 (as_a <rtx_insn *> (x), after, bb);
4869 24959210 : break;
4870 :
4871 : #ifdef ENABLE_RTL_CHECKING
4872 : case SEQUENCE:
4873 : gcc_unreachable ();
4874 : break;
4875 : #endif
4876 :
4877 4464394 : default:
4878 4464394 : last = (*make_raw) (x);
4879 4464394 : add_insn_after (last, after, bb);
4880 4464394 : break;
4881 : }
4882 :
4883 : return last;
4884 : }
4885 :
4886 : /* Make X be output after the insn AFTER and set the BB of insn. If
4887 : BB is NULL, an attempt is made to infer the BB from AFTER. */
4888 :
4889 : rtx_insn *
4890 4918061 : emit_insn_after_noloc (rtx x, rtx_insn *after, basic_block bb)
4891 : {
4892 4918061 : return emit_pattern_after_noloc (x, after, bb, make_insn_raw);
4893 : }
4894 :
4895 :
4896 : /* Make an insn of code JUMP_INSN with body X
4897 : and output it after the insn AFTER. */
4898 :
4899 : rtx_jump_insn *
4900 2313 : emit_jump_insn_after_noloc (rtx x, rtx_insn *after)
4901 : {
4902 2313 : return as_a <rtx_jump_insn *> (
4903 2313 : emit_pattern_after_noloc (x, after, NULL, make_jump_insn_raw));
4904 : }
4905 :
4906 : /* Make an instruction with body X and code CALL_INSN
4907 : and output it after the instruction AFTER. */
4908 :
4909 : rtx_insn *
4910 0 : emit_call_insn_after_noloc (rtx x, rtx_insn *after)
4911 : {
4912 0 : return emit_pattern_after_noloc (x, after, NULL, make_call_insn_raw);
4913 : }
4914 :
4915 : /* Make an instruction with body X and code CALL_INSN
4916 : and output it after the instruction AFTER. */
4917 :
4918 : rtx_insn *
4919 0 : emit_debug_insn_after_noloc (rtx x, rtx_insn *after)
4920 : {
4921 0 : return emit_pattern_after_noloc (x, after, NULL, make_debug_insn_raw);
4922 : }
4923 :
4924 : /* Make an insn of code BARRIER
4925 : and output it after the insn AFTER. */
4926 :
4927 : rtx_barrier *
4928 7820528 : emit_barrier_after (rtx_insn *after)
4929 : {
4930 7820528 : rtx_barrier *insn = as_a <rtx_barrier *> (rtx_alloc (BARRIER));
4931 :
4932 7820528 : INSN_UID (insn) = cur_insn_uid++;
4933 :
4934 7820528 : add_insn_after (insn, after, NULL);
4935 7820528 : return insn;
4936 : }
4937 :
4938 : /* Emit the label LABEL after the insn AFTER. */
4939 :
4940 : rtx_insn *
4941 0 : emit_label_after (rtx_insn *label, rtx_insn *after)
4942 : {
4943 0 : gcc_checking_assert (INSN_UID (label) == 0);
4944 0 : INSN_UID (label) = cur_insn_uid++;
4945 0 : add_insn_after (label, after, NULL);
4946 0 : return label;
4947 : }
4948 :
4949 : /* Notes require a bit of special handling: Some notes need to have their
4950 : BLOCK_FOR_INSN set, others should never have it set, and some should
4951 : have it set or clear depending on the context. */
4952 :
4953 : /* Return true iff a note of kind SUBTYPE should be emitted with routines
4954 : that never set BLOCK_FOR_INSN on NOTE. BB_BOUNDARY is true if the
4955 : caller is asked to emit a note before BB_HEAD, or after BB_END. */
4956 :
4957 : static bool
4958 159253382 : note_outside_basic_block_p (enum insn_note subtype, bool on_bb_boundary_p)
4959 : {
4960 0 : switch (subtype)
4961 : {
4962 : /* NOTE_INSN_SWITCH_TEXT_SECTIONS only appears between basic blocks. */
4963 : case NOTE_INSN_SWITCH_TEXT_SECTIONS:
4964 : return true;
4965 :
4966 : /* Notes for var tracking and EH region markers can appear between or
4967 : inside basic blocks. If the caller is emitting on the basic block
4968 : boundary, do not set BLOCK_FOR_INSN on the new note. */
4969 66960888 : case NOTE_INSN_VAR_LOCATION:
4970 66960888 : case NOTE_INSN_EH_REGION_BEG:
4971 66960888 : case NOTE_INSN_EH_REGION_END:
4972 0 : return on_bb_boundary_p;
4973 :
4974 : /* Otherwise, BLOCK_FOR_INSN must be set. */
4975 0 : default:
4976 0 : return false;
4977 : }
4978 : }
4979 :
4980 : /* Emit a note of subtype SUBTYPE after the insn AFTER. */
4981 :
4982 : rtx_note *
4983 59108096 : emit_note_after (enum insn_note subtype, rtx_insn *after)
4984 : {
4985 59108096 : rtx_note *note = make_note_raw (subtype);
4986 59108096 : basic_block bb = BARRIER_P (after) ? NULL : BLOCK_FOR_INSN (after);
4987 59077071 : bool on_bb_boundary_p = (bb != NULL && BB_END (bb) == after);
4988 :
4989 59108096 : if (note_outside_basic_block_p (subtype, on_bb_boundary_p))
4990 1452236 : add_insn_after_nobb (note, after);
4991 : else
4992 57655860 : add_insn_after (note, after, bb);
4993 59108096 : return note;
4994 : }
4995 :
4996 : /* Emit a note of subtype SUBTYPE before the insn BEFORE. */
4997 :
4998 : rtx_note *
4999 100145286 : emit_note_before (enum insn_note subtype, rtx_insn *before)
5000 : {
5001 100145286 : rtx_note *note = make_note_raw (subtype);
5002 100145286 : basic_block bb = BARRIER_P (before) ? NULL : BLOCK_FOR_INSN (before);
5003 100145285 : bool on_bb_boundary_p = (bb != NULL && BB_HEAD (bb) == before);
5004 :
5005 100145286 : if (note_outside_basic_block_p (subtype, on_bb_boundary_p))
5006 23486689 : add_insn_before_nobb (note, before);
5007 : else
5008 76658597 : add_insn_before (note, before, bb);
5009 100145286 : return note;
5010 : }
5011 :
5012 : /* Insert PATTERN after AFTER, setting its INSN_LOCATION to LOC.
5013 : MAKE_RAW indicates how to turn PATTERN into a real insn. */
5014 :
5015 : static rtx_insn *
5016 22869349 : emit_pattern_after_setloc (rtx pattern, rtx_insn *after, location_t loc,
5017 : rtx_insn *(*make_raw) (rtx))
5018 : {
5019 22869349 : rtx_insn *last = emit_pattern_after_noloc (pattern, after, NULL, make_raw);
5020 :
5021 22869349 : if (pattern == NULL_RTX || !loc)
5022 : return last;
5023 :
5024 16144285 : after = NEXT_INSN (after);
5025 4422025 : while (1)
5026 : {
5027 20566310 : if (active_insn_p (after)
5028 19420333 : && !JUMP_TABLE_DATA_P (after) /* FIXME */
5029 39986643 : && !INSN_LOCATION (after))
5030 19414208 : INSN_LOCATION (after) = loc;
5031 20566310 : if (after == last)
5032 : break;
5033 4422025 : after = NEXT_INSN (after);
5034 : }
5035 : return last;
5036 : }
5037 :
5038 : /* Insert PATTERN after AFTER. MAKE_RAW indicates how to turn PATTERN
5039 : into a real insn. SKIP_DEBUG_INSNS indicates whether to insert after
5040 : any DEBUG_INSNs. */
5041 :
5042 : static rtx_insn *
5043 11125350 : emit_pattern_after (rtx pattern, rtx_insn *after, bool skip_debug_insns,
5044 : rtx_insn *(*make_raw) (rtx))
5045 : {
5046 11125350 : rtx_insn *prev = after;
5047 :
5048 11125350 : if (skip_debug_insns)
5049 11712370 : while (DEBUG_INSN_P (prev))
5050 2073175 : prev = PREV_INSN (prev);
5051 :
5052 11125350 : if (INSN_P (prev))
5053 8925692 : return emit_pattern_after_setloc (pattern, after, INSN_LOCATION (prev),
5054 8925692 : make_raw);
5055 : else
5056 2199658 : return emit_pattern_after_noloc (pattern, after, NULL, make_raw);
5057 : }
5058 :
5059 : /* Like emit_insn_after_noloc, but set INSN_LOCATION according to LOC. */
5060 : rtx_insn *
5061 8577765 : emit_insn_after_setloc (rtx pattern, rtx_insn *after, location_t loc)
5062 : {
5063 8577765 : return emit_pattern_after_setloc (pattern, after, loc, make_insn_raw);
5064 : }
5065 :
5066 : /* Like emit_insn_after_noloc, but set INSN_LOCATION according to AFTER. */
5067 : rtx_insn *
5068 8996612 : emit_insn_after (rtx pattern, rtx_insn *after)
5069 : {
5070 8996612 : return emit_pattern_after (pattern, after, true, make_insn_raw);
5071 : }
5072 :
5073 : /* Like emit_jump_insn_after_noloc, but set INSN_LOCATION according to LOC. */
5074 : rtx_jump_insn *
5075 5365892 : emit_jump_insn_after_setloc (rtx pattern, rtx_insn *after, location_t loc)
5076 : {
5077 5365892 : return as_a <rtx_jump_insn *> (
5078 5365892 : emit_pattern_after_setloc (pattern, after, loc, make_jump_insn_raw));
5079 : }
5080 :
5081 : /* Like emit_jump_insn_after_noloc, but set INSN_LOCATION according to AFTER. */
5082 : rtx_jump_insn *
5083 617873 : emit_jump_insn_after (rtx pattern, rtx_insn *after)
5084 : {
5085 617873 : return as_a <rtx_jump_insn *> (
5086 617873 : emit_pattern_after (pattern, after, true, make_jump_insn_raw));
5087 : }
5088 :
5089 : /* Like emit_call_insn_after_noloc, but set INSN_LOCATION according to LOC. */
5090 : rtx_insn *
5091 0 : emit_call_insn_after_setloc (rtx pattern, rtx_insn *after, location_t loc)
5092 : {
5093 0 : return emit_pattern_after_setloc (pattern, after, loc, make_call_insn_raw);
5094 : }
5095 :
5096 : /* Like emit_call_insn_after_noloc, but set INSN_LOCATION according to AFTER. */
5097 : rtx_insn *
5098 24710 : emit_call_insn_after (rtx pattern, rtx_insn *after)
5099 : {
5100 24710 : return emit_pattern_after (pattern, after, true, make_call_insn_raw);
5101 : }
5102 :
5103 : /* Like emit_debug_insn_after_noloc, but set INSN_LOCATION according to LOC. */
5104 : rtx_insn *
5105 0 : emit_debug_insn_after_setloc (rtx pattern, rtx_insn *after, location_t loc)
5106 : {
5107 0 : return emit_pattern_after_setloc (pattern, after, loc, make_debug_insn_raw);
5108 : }
5109 :
5110 : /* Like emit_debug_insn_after_noloc, but set INSN_LOCATION according to AFTER. */
5111 : rtx_insn *
5112 1486155 : emit_debug_insn_after (rtx pattern, rtx_insn *after)
5113 : {
5114 1486155 : return emit_pattern_after (pattern, after, false, make_debug_insn_raw);
5115 : }
5116 :
5117 : /* Insert PATTERN before BEFORE, setting its INSN_LOCATION to LOC.
5118 : MAKE_RAW indicates how to turn PATTERN into a real insn. INSNP
5119 : indicates if PATTERN is meant for an INSN as opposed to a JUMP_INSN,
5120 : CALL_INSN, etc. */
5121 :
5122 : static rtx_insn *
5123 10933415 : emit_pattern_before_setloc (rtx pattern, rtx_insn *before, location_t loc,
5124 : bool insnp, rtx_insn *(*make_raw) (rtx))
5125 : {
5126 10933415 : rtx_insn *first = PREV_INSN (before);
5127 11278680 : rtx_insn *last = emit_pattern_before_noloc (pattern, before,
5128 : insnp ? before : NULL,
5129 : NULL, make_raw);
5130 :
5131 10933415 : if (pattern == NULL_RTX || !loc)
5132 : return last;
5133 :
5134 9917432 : if (!first)
5135 91356 : first = get_insns ();
5136 : else
5137 9826076 : first = NEXT_INSN (first);
5138 1693777 : while (1)
5139 : {
5140 11611209 : if (active_insn_p (first)
5141 11028039 : && !JUMP_TABLE_DATA_P (first) /* FIXME */
5142 22639248 : && !INSN_LOCATION (first))
5143 10779807 : INSN_LOCATION (first) = loc;
5144 11611209 : if (first == last)
5145 : break;
5146 1693777 : first = NEXT_INSN (first);
5147 : }
5148 : return last;
5149 : }
5150 :
5151 : /* Insert PATTERN before BEFORE. MAKE_RAW indicates how to turn PATTERN
5152 : into a real insn. SKIP_DEBUG_INSNS indicates whether to insert
5153 : before any DEBUG_INSNs. INSNP indicates if PATTERN is meant for an
5154 : INSN as opposed to a JUMP_INSN, CALL_INSN, etc. */
5155 :
5156 : static rtx_insn *
5157 11124363 : emit_pattern_before (rtx pattern, rtx_insn *before, bool skip_debug_insns,
5158 : bool insnp, rtx_insn *(*make_raw) (rtx))
5159 : {
5160 11124363 : rtx_insn *next = before;
5161 :
5162 11124363 : if (skip_debug_insns)
5163 10788072 : while (DEBUG_INSN_P (next))
5164 34 : next = PREV_INSN (next);
5165 :
5166 11124363 : if (INSN_P (next))
5167 10532865 : return emit_pattern_before_setloc (pattern, before, INSN_LOCATION (next),
5168 10532865 : insnp, make_raw);
5169 : else
5170 592664 : return emit_pattern_before_noloc (pattern, before,
5171 : insnp ? before : NULL,
5172 591498 : NULL, make_raw);
5173 : }
5174 :
5175 : /* Like emit_insn_before_noloc, but set INSN_LOCATION according to LOC. */
5176 : rtx_insn *
5177 400550 : emit_insn_before_setloc (rtx pattern, rtx_insn *before, location_t loc)
5178 : {
5179 400550 : return emit_pattern_before_setloc (pattern, before, loc, true,
5180 400550 : make_insn_raw);
5181 : }
5182 :
5183 : /* Like emit_insn_before_noloc, but set INSN_LOCATION according to BEFORE. */
5184 : rtx_insn *
5185 10777932 : emit_insn_before (rtx pattern, rtx_insn *before)
5186 : {
5187 10777932 : return emit_pattern_before (pattern, before, true, true, make_insn_raw);
5188 : }
5189 :
5190 : /* like emit_insn_before_noloc, but set INSN_LOCATION according to LOC. */
5191 : rtx_jump_insn *
5192 0 : emit_jump_insn_before_setloc (rtx pattern, rtx_insn *before, location_t loc)
5193 : {
5194 0 : return as_a <rtx_jump_insn *> (
5195 : emit_pattern_before_setloc (pattern, before, loc, false,
5196 0 : make_jump_insn_raw));
5197 : }
5198 :
5199 : /* Like emit_jump_insn_before_noloc, but set INSN_LOCATION according to BEFORE. */
5200 : rtx_jump_insn *
5201 10106 : emit_jump_insn_before (rtx pattern, rtx_insn *before)
5202 : {
5203 10106 : return as_a <rtx_jump_insn *> (
5204 : emit_pattern_before (pattern, before, true, false,
5205 10106 : make_jump_insn_raw));
5206 : }
5207 :
5208 : /* Like emit_insn_before_noloc, but set INSN_LOCATION according to LOC. */
5209 : rtx_insn *
5210 0 : emit_call_insn_before_setloc (rtx pattern, rtx_insn *before, location_t loc)
5211 : {
5212 0 : return emit_pattern_before_setloc (pattern, before, loc, false,
5213 0 : make_call_insn_raw);
5214 : }
5215 :
5216 : /* Like emit_call_insn_before_noloc,
5217 : but set insn_location according to BEFORE. */
5218 : rtx_insn *
5219 0 : emit_call_insn_before (rtx pattern, rtx_insn *before)
5220 : {
5221 0 : return emit_pattern_before (pattern, before, true, false,
5222 0 : make_call_insn_raw);
5223 : }
5224 :
5225 : /* Like emit_insn_before_noloc, but set INSN_LOCATION according to LOC. */
5226 : rtx_insn *
5227 0 : emit_debug_insn_before_setloc (rtx pattern, rtx_insn *before, location_t loc)
5228 : {
5229 0 : return emit_pattern_before_setloc (pattern, before, loc, false,
5230 0 : make_debug_insn_raw);
5231 : }
5232 :
5233 : /* Like emit_debug_insn_before_noloc,
5234 : but set insn_location according to BEFORE. */
5235 : rtx_insn *
5236 336325 : emit_debug_insn_before (rtx pattern, rtx_insn *before)
5237 : {
5238 336325 : return emit_pattern_before (pattern, before, false, false,
5239 336325 : make_debug_insn_raw);
5240 : }
5241 :
5242 : /* Take X and emit it at the end of the doubly-linked
5243 : INSN list.
5244 :
5245 : Returns the last insn emitted. */
5246 :
5247 : rtx_insn *
5248 251106802 : emit_insn (rtx x)
5249 : {
5250 251106802 : rtx_insn *last = get_last_insn ();
5251 251106802 : rtx_insn *insn;
5252 :
5253 251106802 : if (x == NULL_RTX)
5254 : return last;
5255 :
5256 246372918 : switch (GET_CODE (x))
5257 : {
5258 120816983 : case DEBUG_INSN:
5259 120816983 : case INSN:
5260 120816983 : case JUMP_INSN:
5261 120816983 : case CALL_INSN:
5262 120816983 : case CODE_LABEL:
5263 120816983 : case BARRIER:
5264 120816983 : case NOTE:
5265 120816983 : insn = as_a <rtx_insn *> (x);
5266 410438312 : while (insn)
5267 : {
5268 168804346 : rtx_insn *next = NEXT_INSN (insn);
5269 168804346 : add_insn (insn);
5270 168804346 : last = insn;
5271 168804346 : insn = next;
5272 : }
5273 : break;
5274 :
5275 : #ifdef ENABLE_RTL_CHECKING
5276 : case JUMP_TABLE_DATA:
5277 : case SEQUENCE:
5278 : gcc_unreachable ();
5279 : break;
5280 : #endif
5281 :
5282 125555935 : default:
5283 125555935 : last = make_insn_raw (x);
5284 125555935 : add_insn (last);
5285 125555935 : break;
5286 : }
5287 :
5288 : return last;
5289 : }
5290 :
5291 : /* Make an insn of code DEBUG_INSN with pattern X
5292 : and add it to the end of the doubly-linked list. */
5293 :
5294 : rtx_insn *
5295 52656309 : emit_debug_insn (rtx x)
5296 : {
5297 52656309 : rtx_insn *last = get_last_insn ();
5298 52656309 : rtx_insn *insn;
5299 :
5300 52656309 : if (x == NULL_RTX)
5301 : return last;
5302 :
5303 52656309 : switch (GET_CODE (x))
5304 : {
5305 0 : case DEBUG_INSN:
5306 0 : case INSN:
5307 0 : case JUMP_INSN:
5308 0 : case CALL_INSN:
5309 0 : case CODE_LABEL:
5310 0 : case BARRIER:
5311 0 : case NOTE:
5312 0 : insn = as_a <rtx_insn *> (x);
5313 0 : while (insn)
5314 : {
5315 0 : rtx_insn *next = NEXT_INSN (insn);
5316 0 : add_insn (insn);
5317 0 : last = insn;
5318 0 : insn = next;
5319 : }
5320 : break;
5321 :
5322 : #ifdef ENABLE_RTL_CHECKING
5323 : case JUMP_TABLE_DATA:
5324 : case SEQUENCE:
5325 : gcc_unreachable ();
5326 : break;
5327 : #endif
5328 :
5329 52656309 : default:
5330 52656309 : last = make_debug_insn_raw (x);
5331 52656309 : add_insn (last);
5332 52656309 : break;
5333 : }
5334 :
5335 : return last;
5336 : }
5337 :
5338 : /* Make an insn of code JUMP_INSN with pattern X
5339 : and add it to the end of the doubly-linked list. */
5340 :
5341 : rtx_insn *
5342 30074868 : emit_jump_insn (rtx x)
5343 : {
5344 30074868 : rtx_insn *last = NULL;
5345 30074868 : rtx_insn *insn;
5346 :
5347 30074868 : switch (GET_CODE (x))
5348 : {
5349 12091069 : case DEBUG_INSN:
5350 12091069 : case INSN:
5351 12091069 : case JUMP_INSN:
5352 12091069 : case CALL_INSN:
5353 12091069 : case CODE_LABEL:
5354 12091069 : case BARRIER:
5355 12091069 : case NOTE:
5356 12091069 : insn = as_a <rtx_insn *> (x);
5357 45994435 : while (insn)
5358 : {
5359 21812297 : rtx_insn *next = NEXT_INSN (insn);
5360 21812297 : add_insn (insn);
5361 21812297 : last = insn;
5362 21812297 : insn = next;
5363 : }
5364 : break;
5365 :
5366 : #ifdef ENABLE_RTL_CHECKING
5367 : case JUMP_TABLE_DATA:
5368 : case SEQUENCE:
5369 : gcc_unreachable ();
5370 : break;
5371 : #endif
5372 :
5373 17983799 : default:
5374 17983799 : last = make_jump_insn_raw (x);
5375 17983799 : add_insn (last);
5376 17983799 : break;
5377 : }
5378 :
5379 30074868 : return last;
5380 : }
5381 :
5382 : /* Make an insn of code JUMP_INSN with pattern X,
5383 : add a REG_BR_PROB note that indicates very likely probability,
5384 : and add it to the end of the doubly-linked list. */
5385 :
5386 : rtx_insn *
5387 0 : emit_likely_jump_insn (rtx x)
5388 : {
5389 0 : rtx_insn *jump = emit_jump_insn (x);
5390 0 : add_reg_br_prob_note (jump, profile_probability::very_likely ());
5391 0 : return jump;
5392 : }
5393 :
5394 : /* Make an insn of code JUMP_INSN with pattern X,
5395 : add a REG_BR_PROB note that indicates very unlikely probability,
5396 : and add it to the end of the doubly-linked list. */
5397 :
5398 : rtx_insn *
5399 0 : emit_unlikely_jump_insn (rtx x)
5400 : {
5401 0 : rtx_insn *jump = emit_jump_insn (x);
5402 0 : add_reg_br_prob_note (jump, profile_probability::very_unlikely ());
5403 0 : return jump;
5404 : }
5405 :
5406 : /* Make an insn of code CALL_INSN with pattern X
5407 : and add it to the end of the doubly-linked list. */
5408 :
5409 : rtx_insn *
5410 6459576 : emit_call_insn (rtx x)
5411 : {
5412 6459576 : rtx_insn *insn;
5413 :
5414 6459576 : switch (GET_CODE (x))
5415 : {
5416 4425 : case DEBUG_INSN:
5417 4425 : case INSN:
5418 4425 : case JUMP_INSN:
5419 4425 : case CALL_INSN:
5420 4425 : case CODE_LABEL:
5421 4425 : case BARRIER:
5422 4425 : case NOTE:
5423 4425 : insn = emit_insn (x);
5424 4425 : break;
5425 :
5426 : #ifdef ENABLE_RTL_CHECKING
5427 : case SEQUENCE:
5428 : case JUMP_TABLE_DATA:
5429 : gcc_unreachable ();
5430 : break;
5431 : #endif
5432 :
5433 6455151 : default:
5434 6455151 : insn = make_call_insn_raw (x);
5435 6455151 : add_insn (insn);
5436 6455151 : break;
5437 : }
5438 :
5439 6459576 : return insn;
5440 : }
5441 :
5442 : /* Add the label LABEL to the end of the doubly-linked list. */
5443 :
5444 : rtx_code_label *
5445 8929409 : emit_label (rtx uncast_label)
5446 : {
5447 8929409 : rtx_code_label *label = as_a <rtx_code_label *> (uncast_label);
5448 :
5449 8929409 : gcc_checking_assert (INSN_UID (label) == 0);
5450 8929409 : INSN_UID (label) = cur_insn_uid++;
5451 8929409 : add_insn (label);
5452 8929409 : return label;
5453 : }
5454 :
5455 : /* Make an insn of code JUMP_TABLE_DATA
5456 : and add it to the end of the doubly-linked list. */
5457 :
5458 : rtx_jump_table_data *
5459 7659 : emit_jump_table_data (rtx table)
5460 : {
5461 7659 : rtx_jump_table_data *jump_table_data =
5462 7659 : as_a <rtx_jump_table_data *> (rtx_alloc (JUMP_TABLE_DATA));
5463 7659 : INSN_UID (jump_table_data) = cur_insn_uid++;
5464 7659 : PATTERN (jump_table_data) = table;
5465 7659 : BLOCK_FOR_INSN (jump_table_data) = NULL;
5466 7659 : add_insn (jump_table_data);
5467 7659 : return jump_table_data;
5468 : }
5469 :
5470 : /* Make an insn of code BARRIER
5471 : and add it to the end of the doubly-linked list. */
5472 :
5473 : rtx_barrier *
5474 3636539 : emit_barrier (void)
5475 : {
5476 3636539 : rtx_barrier *barrier = as_a <rtx_barrier *> (rtx_alloc (BARRIER));
5477 3636539 : INSN_UID (barrier) = cur_insn_uid++;
5478 3636539 : add_insn (barrier);
5479 3636539 : return barrier;
5480 : }
5481 :
5482 : /* Emit a copy of note ORIG. */
5483 :
5484 : rtx_note *
5485 176817 : emit_note_copy (rtx_note *orig)
5486 : {
5487 176817 : enum insn_note kind = (enum insn_note) NOTE_KIND (orig);
5488 176817 : rtx_note *note = make_note_raw (kind);
5489 176817 : NOTE_DATA (note) = NOTE_DATA (orig);
5490 176817 : add_insn (note);
5491 176817 : return note;
5492 : }
5493 :
5494 : /* Make an insn of code NOTE or type NOTE_NO
5495 : and add it to the end of the doubly-linked list. */
5496 :
5497 : rtx_note *
5498 16902557 : emit_note (enum insn_note kind)
5499 : {
5500 16902557 : rtx_note *note = make_note_raw (kind);
5501 16902557 : add_insn (note);
5502 16902557 : return note;
5503 : }
5504 :
5505 : /* Emit a clobber of lvalue X. */
5506 :
5507 : rtx_insn *
5508 552654 : emit_clobber (rtx x)
5509 : {
5510 : /* CONCATs should not appear in the insn stream. */
5511 552654 : if (GET_CODE (x) == CONCAT)
5512 : {
5513 0 : emit_clobber (XEXP (x, 0));
5514 0 : return emit_clobber (XEXP (x, 1));
5515 : }
5516 552654 : return emit_insn (gen_rtx_CLOBBER (VOIDmode, x));
5517 : }
5518 :
5519 : /* Return a sequence of insns to clobber lvalue X. */
5520 :
5521 : rtx_insn *
5522 0 : gen_clobber (rtx x)
5523 : {
5524 0 : rtx_insn *seq;
5525 :
5526 0 : start_sequence ();
5527 0 : emit_clobber (x);
5528 0 : seq = end_sequence ();
5529 0 : return seq;
5530 : }
5531 :
5532 : /* Emit a use of rvalue X. */
5533 :
5534 : rtx_insn *
5535 811460 : emit_use (rtx x)
5536 : {
5537 : /* CONCATs should not appear in the insn stream. */
5538 811460 : if (GET_CODE (x) == CONCAT)
5539 : {
5540 0 : emit_use (XEXP (x, 0));
5541 0 : return emit_use (XEXP (x, 1));
5542 : }
5543 811460 : return emit_insn (gen_rtx_USE (VOIDmode, x));
5544 : }
5545 :
5546 : /* Return a sequence of insns to use rvalue X. */
5547 :
5548 : rtx_insn *
5549 0 : gen_use (rtx x)
5550 : {
5551 0 : rtx_insn *seq;
5552 :
5553 0 : start_sequence ();
5554 0 : emit_use (x);
5555 0 : seq = end_sequence ();
5556 0 : return seq;
5557 : }
5558 :
5559 : /* Notes like REG_EQUAL and REG_EQUIV refer to a set in an instruction.
5560 : Return the set in INSN that such notes describe, or NULL if the notes
5561 : have no meaning for INSN. */
5562 :
5563 : rtx
5564 240038716 : set_for_reg_notes (rtx_insn *insn)
5565 : {
5566 240038716 : rtx pat, reg;
5567 :
5568 240038716 : if (!INSN_P (insn))
5569 : return NULL_RTX;
5570 :
5571 240033191 : pat = PATTERN (insn);
5572 240033191 : if (GET_CODE (pat) == PARALLEL)
5573 : {
5574 : /* We do not use single_set because that ignores SETs of unused
5575 : registers. REG_EQUAL and REG_EQUIV notes really do require the
5576 : PARALLEL to have a single SET. */
5577 19015698 : if (multiple_sets (insn))
5578 : return NULL_RTX;
5579 18292684 : pat = XVECEXP (pat, 0, 0);
5580 : }
5581 :
5582 239310177 : if (GET_CODE (pat) != SET)
5583 : return NULL_RTX;
5584 :
5585 132548293 : reg = SET_DEST (pat);
5586 :
5587 : /* Notes apply to the contents of a STRICT_LOW_PART. */
5588 132548293 : if (GET_CODE (reg) == STRICT_LOW_PART
5589 132541858 : || GET_CODE (reg) == ZERO_EXTRACT)
5590 8564 : reg = XEXP (reg, 0);
5591 :
5592 : /* Check that we have a register. */
5593 132548293 : if (!(REG_P (reg) || GET_CODE (reg) == SUBREG))
5594 35302957 : return NULL_RTX;
5595 :
5596 : return pat;
5597 : }
5598 :
5599 : /* Place a note of KIND on insn INSN with DATUM as the datum. If a
5600 : note of this type already exists, remove it first. */
5601 :
5602 : rtx
5603 23973855 : set_unique_reg_note (rtx_insn *insn, enum reg_note kind, rtx datum)
5604 : {
5605 23973855 : rtx note = find_reg_note (insn, kind, NULL_RTX);
5606 :
5607 23973855 : switch (kind)
5608 : {
5609 23973855 : case REG_EQUAL:
5610 23973855 : case REG_EQUIV:
5611 : /* We need to support the REG_EQUAL on USE trick of find_reloads. */
5612 23973855 : if (!set_for_reg_notes (insn) && GET_CODE (PATTERN (insn)) != USE)
5613 : return NULL_RTX;
5614 :
5615 : /* Don't add ASM_OPERAND REG_EQUAL/REG_EQUIV notes.
5616 : It serves no useful purpose and breaks eliminate_regs. */
5617 23939584 : if (GET_CODE (datum) == ASM_OPERANDS)
5618 : return NULL_RTX;
5619 :
5620 : /* Notes with side effects are dangerous. Even if the side-effect
5621 : initially mirrors one in PATTERN (INSN), later optimizations
5622 : might alter the way that the final register value is calculated
5623 : and so move or alter the side-effect in some way. The note would
5624 : then no longer be a valid substitution for SET_SRC. */
5625 23929267 : if (side_effects_p (datum))
5626 : return NULL_RTX;
5627 : break;
5628 :
5629 : default:
5630 : break;
5631 : }
5632 :
5633 23928801 : if (note)
5634 8109476 : XEXP (note, 0) = datum;
5635 : else
5636 : {
5637 15819325 : add_reg_note (insn, kind, datum);
5638 15819325 : note = REG_NOTES (insn);
5639 : }
5640 :
5641 23928801 : switch (kind)
5642 : {
5643 23928801 : case REG_EQUAL:
5644 23928801 : case REG_EQUIV:
5645 23928801 : df_notes_rescan (insn);
5646 23928801 : break;
5647 : default:
5648 : break;
5649 : }
5650 :
5651 : return note;
5652 : }
5653 :
5654 : /* Like set_unique_reg_note, but don't do anything unless INSN sets DST. */
5655 : rtx
5656 1408109 : set_dst_reg_note (rtx_insn *insn, enum reg_note kind, rtx datum, rtx dst)
5657 : {
5658 1408109 : rtx set = set_for_reg_notes (insn);
5659 :
5660 1408109 : if (set && SET_DEST (set) == dst)
5661 1391222 : return set_unique_reg_note (insn, kind, datum);
5662 : return NULL_RTX;
5663 : }
5664 :
5665 : /* Emit the rtl pattern X as an appropriate kind of insn. Also emit a
5666 : following barrier if the instruction needs one and if ALLOW_BARRIER_P
5667 : is true.
5668 :
5669 : If X is a label, it is simply added into the insn chain. */
5670 :
5671 : rtx_insn *
5672 19016275 : emit (rtx x, bool allow_barrier_p)
5673 : {
5674 19016275 : enum rtx_code code = classify_insn (x);
5675 :
5676 19016275 : switch (code)
5677 : {
5678 0 : case CODE_LABEL:
5679 0 : return emit_label (x);
5680 9402658 : case INSN:
5681 9402658 : return emit_insn (x);
5682 9607355 : case JUMP_INSN:
5683 9607355 : {
5684 9607355 : rtx_insn *insn = emit_jump_insn (x);
5685 9607355 : if (allow_barrier_p
5686 9607355 : && (any_uncondjump_p (insn) || GET_CODE (x) == RETURN))
5687 0 : return emit_barrier ();
5688 : return insn;
5689 : }
5690 6262 : case CALL_INSN:
5691 6262 : return emit_call_insn (x);
5692 0 : case DEBUG_INSN:
5693 0 : return emit_debug_insn (x);
5694 0 : default:
5695 0 : gcc_unreachable ();
5696 : }
5697 : }
5698 :
5699 : /* Space for free sequence stack entries. */
5700 : static GTY ((deletable)) struct sequence_stack *free_sequence_stack;
5701 :
5702 : /* Begin emitting insns to a sequence. If this sequence will contain
5703 : something that might cause the compiler to pop arguments to function
5704 : calls (because those pops have previously been deferred; see
5705 : INHIBIT_DEFER_POP for more details), use do_pending_stack_adjust
5706 : before calling this function. That will ensure that the deferred
5707 : pops are not accidentally emitted in the middle of this sequence. */
5708 :
5709 : void
5710 241225603 : start_sequence (void)
5711 : {
5712 241225603 : struct sequence_stack *tem;
5713 :
5714 241225603 : if (free_sequence_stack != NULL)
5715 : {
5716 240539070 : tem = free_sequence_stack;
5717 240539070 : free_sequence_stack = tem->next;
5718 : }
5719 : else
5720 686533 : tem = ggc_alloc<sequence_stack> ();
5721 :
5722 241225603 : tem->next = get_current_sequence ()->next;
5723 241225603 : tem->first = get_insns ();
5724 241225603 : tem->last = get_last_insn ();
5725 241225603 : get_current_sequence ()->next = tem;
5726 :
5727 241225603 : set_first_insn (0);
5728 241225603 : set_last_insn (0);
5729 241225603 : }
5730 :
5731 : /* Set up the insn chain starting with FIRST as the current sequence,
5732 : saving the previously current one. See the documentation for
5733 : start_sequence for more information about how to use this function. */
5734 :
5735 : void
5736 5623604 : push_to_sequence (rtx_insn *first)
5737 : {
5738 5623604 : rtx_insn *last;
5739 :
5740 5623604 : start_sequence ();
5741 :
5742 14839759 : for (last = first; last && NEXT_INSN (last); last = NEXT_INSN (last))
5743 : ;
5744 :
5745 5623604 : set_first_insn (first);
5746 5623604 : set_last_insn (last);
5747 5623604 : }
5748 :
5749 : /* Like push_to_sequence, but take the last insn as an argument to avoid
5750 : looping through the list. */
5751 :
5752 : void
5753 79455 : push_to_sequence2 (rtx_insn *first, rtx_insn *last)
5754 : {
5755 79455 : start_sequence ();
5756 :
5757 79455 : set_first_insn (first);
5758 79455 : set_last_insn (last);
5759 79455 : }
5760 :
5761 : /* Set up the outer-level insn chain
5762 : as the current sequence, saving the previously current one. */
5763 :
5764 : void
5765 476 : push_topmost_sequence (void)
5766 : {
5767 476 : struct sequence_stack *top;
5768 :
5769 476 : start_sequence ();
5770 :
5771 476 : top = get_topmost_sequence ();
5772 476 : set_first_insn (top->first);
5773 476 : set_last_insn (top->last);
5774 476 : }
5775 :
5776 : /* After emitting to the outer-level insn chain, update the outer-level
5777 : insn chain, and restore the previous saved state. */
5778 :
5779 : void
5780 476 : pop_topmost_sequence (void)
5781 : {
5782 476 : struct sequence_stack *top;
5783 :
5784 476 : top = get_topmost_sequence ();
5785 476 : top->first = get_insns ();
5786 476 : top->last = get_last_insn ();
5787 :
5788 476 : end_sequence ();
5789 476 : }
5790 :
5791 : /* After emitting to a sequence, restore the previous saved state and return
5792 : the start of the completed sequence.
5793 :
5794 : If the compiler might have deferred popping arguments while
5795 : generating this sequence, and this sequence will not be immediately
5796 : inserted into the instruction stream, use do_pending_stack_adjust
5797 : before calling this function. That will ensure that the deferred
5798 : pops are inserted into this sequence, and not into some random
5799 : location in the instruction stream. See INHIBIT_DEFER_POP for more
5800 : information about deferred popping of arguments. */
5801 :
5802 : rtx_insn *
5803 241225601 : end_sequence (void)
5804 : {
5805 241225601 : rtx_insn *insns = get_insns ();
5806 :
5807 241225601 : struct sequence_stack *tem = get_current_sequence ()->next;
5808 :
5809 241225601 : set_first_insn (tem->first);
5810 241225601 : set_last_insn (tem->last);
5811 241225601 : get_current_sequence ()->next = tem->next;
5812 :
5813 241225601 : memset (tem, 0, sizeof (*tem));
5814 241225601 : tem->next = free_sequence_stack;
5815 241225601 : free_sequence_stack = tem;
5816 :
5817 241225601 : return insns;
5818 : }
5819 :
5820 : /* Return true if currently emitting into a sequence. */
5821 :
5822 : bool
5823 5060669 : in_sequence_p (void)
5824 : {
5825 5060669 : return get_current_sequence ()->next != 0;
5826 : }
5827 :
5828 : /* Put the various virtual registers into REGNO_REG_RTX. */
5829 :
5830 : static void
5831 1745944 : init_virtual_regs (void)
5832 : {
5833 1745944 : regno_reg_rtx[VIRTUAL_INCOMING_ARGS_REGNUM] = virtual_incoming_args_rtx;
5834 1745944 : regno_reg_rtx[VIRTUAL_STACK_VARS_REGNUM] = virtual_stack_vars_rtx;
5835 1745944 : regno_reg_rtx[VIRTUAL_STACK_DYNAMIC_REGNUM] = virtual_stack_dynamic_rtx;
5836 1745944 : regno_reg_rtx[VIRTUAL_OUTGOING_ARGS_REGNUM] = virtual_outgoing_args_rtx;
5837 1745944 : regno_reg_rtx[VIRTUAL_CFA_REGNUM] = virtual_cfa_rtx;
5838 1745944 : regno_reg_rtx[VIRTUAL_PREFERRED_STACK_BOUNDARY_REGNUM]
5839 1745944 : = virtual_preferred_stack_boundary_rtx;
5840 1745944 : }
5841 :
5842 :
5843 : /* Used by copy_insn_1 to avoid copying SCRATCHes more than once. */
5844 : static rtx copy_insn_scratch_in[MAX_RECOG_OPERANDS];
5845 : static rtx copy_insn_scratch_out[MAX_RECOG_OPERANDS];
5846 : static int copy_insn_n_scratches;
5847 :
5848 : /* When an insn is being copied by copy_insn_1, this is nonzero if we have
5849 : copied an ASM_OPERANDS.
5850 : In that case, it is the original input-operand vector. */
5851 : static rtvec orig_asm_operands_vector;
5852 :
5853 : /* When an insn is being copied by copy_insn_1, this is nonzero if we have
5854 : copied an ASM_OPERANDS.
5855 : In that case, it is the copied input-operand vector. */
5856 : static rtvec copy_asm_operands_vector;
5857 :
5858 : /* Likewise for the constraints vector. */
5859 : static rtvec orig_asm_constraints_vector;
5860 : static rtvec copy_asm_constraints_vector;
5861 :
5862 : /* Recursively create a new copy of an rtx for copy_insn.
5863 : This function differs from copy_rtx in that it handles SCRATCHes and
5864 : ASM_OPERANDs properly.
5865 : Normally, this function is not used directly; use copy_insn as front end.
5866 : However, you could first copy an insn pattern with copy_insn and then use
5867 : this function afterwards to properly copy any REG_NOTEs containing
5868 : SCRATCHes. */
5869 :
5870 : rtx
5871 21676893 : copy_insn_1 (rtx orig)
5872 : {
5873 21676893 : rtx copy;
5874 21676893 : int i, j;
5875 21676893 : RTX_CODE code;
5876 21676893 : const char *format_ptr;
5877 :
5878 21676893 : if (orig == NULL)
5879 : return NULL;
5880 :
5881 21675270 : code = GET_CODE (orig);
5882 :
5883 21675270 : switch (code)
5884 : {
5885 : case REG:
5886 : case DEBUG_EXPR:
5887 : CASE_CONST_ANY:
5888 : case SYMBOL_REF:
5889 : case CODE_LABEL:
5890 : case PC:
5891 : case RETURN:
5892 : case SIMPLE_RETURN:
5893 : return orig;
5894 1052775 : case CLOBBER:
5895 : /* Share clobbers of hard registers, but do not share pseudo reg
5896 : clobbers or clobbers of hard registers that originated as pseudos.
5897 : This is needed to allow safe register renaming. */
5898 1052775 : if (REG_P (XEXP (orig, 0))
5899 452804 : && HARD_REGISTER_NUM_P (REGNO (XEXP (orig, 0)))
5900 1505573 : && HARD_REGISTER_NUM_P (ORIGINAL_REGNO (XEXP (orig, 0))))
5901 : return orig;
5902 : break;
5903 :
5904 : case SCRATCH:
5905 73243 : for (i = 0; i < copy_insn_n_scratches; i++)
5906 1688 : if (copy_insn_scratch_in[i] == orig)
5907 1660 : return copy_insn_scratch_out[i];
5908 : break;
5909 :
5910 103897 : case CONST:
5911 103897 : if (shared_const_p (orig))
5912 : return orig;
5913 : break;
5914 :
5915 : /* A MEM with a constant address is not sharable. The problem is that
5916 : the constant address may need to be reloaded. If the mem is shared,
5917 : then reloading one copy of this mem will cause all copies to appear
5918 : to have been reloaded. */
5919 :
5920 : default:
5921 : break;
5922 : }
5923 :
5924 : /* Copy the various flags, fields, and other information. We assume
5925 : that all fields need copying, and then clear the fields that should
5926 : not be copied. That is the sensible default behavior, and forces
5927 : us to explicitly document why we are *not* copying a flag. */
5928 9665748 : copy = shallow_copy_rtx (orig);
5929 :
5930 : /* We do not copy JUMP, CALL, or FRAME_RELATED for INSNs. */
5931 9665748 : if (INSN_P (orig))
5932 : {
5933 0 : RTX_FLAG (copy, jump) = 0;
5934 0 : RTX_FLAG (copy, call) = 0;
5935 0 : RTX_FLAG (copy, frame_related) = 0;
5936 : }
5937 :
5938 9665748 : format_ptr = GET_RTX_FORMAT (GET_CODE (copy));
5939 :
5940 26486732 : for (i = 0; i < GET_RTX_LENGTH (GET_CODE (copy)); i++)
5941 16820984 : switch (*format_ptr++)
5942 : {
5943 13731743 : case 'e':
5944 13731743 : if (XEXP (orig, i) != NULL)
5945 13709397 : XEXP (copy, i) = copy_insn_1 (XEXP (orig, i));
5946 : break;
5947 :
5948 640072 : case 'E':
5949 640072 : case 'V':
5950 640072 : if (XVEC (orig, i) == orig_asm_constraints_vector)
5951 12 : XVEC (copy, i) = copy_asm_constraints_vector;
5952 640060 : else if (XVEC (orig, i) == orig_asm_operands_vector)
5953 12 : XVEC (copy, i) = copy_asm_operands_vector;
5954 640048 : else if (XVEC (orig, i) != NULL)
5955 : {
5956 640048 : XVEC (copy, i) = rtvec_alloc (XVECLEN (orig, i));
5957 1889733 : for (j = 0; j < XVECLEN (copy, i); j++)
5958 1249685 : XVECEXP (copy, i, j) = copy_insn_1 (XVECEXP (orig, i, j));
5959 : }
5960 : break;
5961 :
5962 : case 't':
5963 : case 'w':
5964 : case 'i':
5965 : case 'L':
5966 : case 'p':
5967 : case 's':
5968 : case 'S':
5969 : case 'u':
5970 : case '0':
5971 : /* These are left unchanged. */
5972 : break;
5973 :
5974 0 : default:
5975 0 : gcc_unreachable ();
5976 : }
5977 :
5978 9665748 : if (code == SCRATCH)
5979 : {
5980 71555 : i = copy_insn_n_scratches++;
5981 71555 : gcc_assert (i < MAX_RECOG_OPERANDS);
5982 71555 : copy_insn_scratch_in[i] = orig;
5983 71555 : copy_insn_scratch_out[i] = copy;
5984 : }
5985 9594193 : else if (code == ASM_OPERANDS)
5986 : {
5987 226 : orig_asm_operands_vector = ASM_OPERANDS_INPUT_VEC (orig);
5988 226 : copy_asm_operands_vector = ASM_OPERANDS_INPUT_VEC (copy);
5989 226 : orig_asm_constraints_vector = ASM_OPERANDS_INPUT_CONSTRAINT_VEC (orig);
5990 226 : copy_asm_constraints_vector = ASM_OPERANDS_INPUT_CONSTRAINT_VEC (copy);
5991 : }
5992 :
5993 : return copy;
5994 : }
5995 :
5996 : /* Create a new copy of an rtx.
5997 : This function differs from copy_rtx in that it handles SCRATCHes and
5998 : ASM_OPERANDs properly.
5999 : INSN doesn't really have to be a full INSN; it could be just the
6000 : pattern. */
6001 : rtx
6002 3714332 : copy_insn (rtx insn)
6003 : {
6004 3714332 : copy_insn_n_scratches = 0;
6005 3714332 : orig_asm_operands_vector = 0;
6006 3714332 : orig_asm_constraints_vector = 0;
6007 3714332 : copy_asm_operands_vector = 0;
6008 3714332 : copy_asm_constraints_vector = 0;
6009 3714332 : return copy_insn_1 (insn);
6010 : }
6011 :
6012 : /* Return a copy of INSN that can be used in a SEQUENCE delay slot,
6013 : on that assumption that INSN itself remains in its original place. */
6014 :
6015 : rtx_insn *
6016 0 : copy_delay_slot_insn (rtx_insn *insn)
6017 : {
6018 : /* Copy INSN with its rtx_code, all its notes, location etc. */
6019 0 : insn = as_a <rtx_insn *> (copy_rtx (insn));
6020 0 : INSN_UID (insn) = cur_insn_uid++;
6021 0 : return insn;
6022 : }
6023 :
6024 : /* Initialize data structures and variables in this file
6025 : before generating rtl for each function. */
6026 :
6027 : void
6028 1745944 : init_emit (void)
6029 : {
6030 1745944 : set_first_insn (NULL);
6031 1745944 : set_last_insn (NULL);
6032 1745944 : if (param_min_nondebug_insn_uid)
6033 : cur_insn_uid = param_min_nondebug_insn_uid;
6034 : else
6035 1745944 : cur_insn_uid = 1;
6036 1745944 : cur_debug_insn_uid = 1;
6037 1745944 : reg_rtx_no = LAST_VIRTUAL_REGISTER + 1;
6038 1745944 : first_label_num = label_num;
6039 1745944 : get_current_sequence ()->next = NULL;
6040 :
6041 : /* Init the tables that describe all the pseudo regs. */
6042 :
6043 1745944 : crtl->emit.regno_pointer_align_length = LAST_VIRTUAL_REGISTER + 101;
6044 :
6045 1745944 : crtl->emit.regno_pointer_align
6046 1745944 : = XCNEWVEC (unsigned char, crtl->emit.regno_pointer_align_length);
6047 :
6048 1745944 : regno_reg_rtx
6049 1745944 : = ggc_cleared_vec_alloc<rtx> (crtl->emit.regno_pointer_align_length);
6050 :
6051 : /* Put copies of all the hard registers into regno_reg_rtx. */
6052 1745944 : memcpy (regno_reg_rtx,
6053 1745944 : initial_regno_reg_rtx,
6054 : FIRST_PSEUDO_REGISTER * sizeof (rtx));
6055 :
6056 : /* Put copies of all the virtual register rtx into regno_reg_rtx. */
6057 1745944 : init_virtual_regs ();
6058 :
6059 : /* Indicate that the virtual registers and stack locations are
6060 : all pointers. */
6061 1745944 : REG_POINTER (stack_pointer_rtx) = 1;
6062 1745944 : REG_POINTER (frame_pointer_rtx) = 1;
6063 1745944 : REG_POINTER (hard_frame_pointer_rtx) = 1;
6064 1745944 : REG_POINTER (arg_pointer_rtx) = 1;
6065 :
6066 1745944 : REG_POINTER (virtual_incoming_args_rtx) = 1;
6067 1745944 : REG_POINTER (virtual_stack_vars_rtx) = 1;
6068 1745944 : REG_POINTER (virtual_stack_dynamic_rtx) = 1;
6069 1745944 : REG_POINTER (virtual_outgoing_args_rtx) = 1;
6070 1745944 : REG_POINTER (virtual_cfa_rtx) = 1;
6071 :
6072 : #ifdef STACK_BOUNDARY
6073 1745944 : REGNO_POINTER_ALIGN (STACK_POINTER_REGNUM) = STACK_BOUNDARY;
6074 1745944 : REGNO_POINTER_ALIGN (FRAME_POINTER_REGNUM) = STACK_BOUNDARY;
6075 1745944 : REGNO_POINTER_ALIGN (HARD_FRAME_POINTER_REGNUM) = STACK_BOUNDARY;
6076 1745944 : REGNO_POINTER_ALIGN (ARG_POINTER_REGNUM) = STACK_BOUNDARY;
6077 :
6078 1745944 : REGNO_POINTER_ALIGN (VIRTUAL_INCOMING_ARGS_REGNUM) = STACK_BOUNDARY;
6079 1745944 : REGNO_POINTER_ALIGN (VIRTUAL_STACK_VARS_REGNUM) = STACK_BOUNDARY;
6080 1745944 : REGNO_POINTER_ALIGN (VIRTUAL_STACK_DYNAMIC_REGNUM) = STACK_BOUNDARY;
6081 1745944 : REGNO_POINTER_ALIGN (VIRTUAL_OUTGOING_ARGS_REGNUM) = STACK_BOUNDARY;
6082 :
6083 1745944 : REGNO_POINTER_ALIGN (VIRTUAL_CFA_REGNUM) = BITS_PER_WORD;
6084 : #endif
6085 :
6086 : #ifdef INIT_EXPANDERS
6087 : INIT_EXPANDERS;
6088 : #endif
6089 1745944 : }
6090 :
6091 : /* Return the value of element I of CONST_VECTOR X as a wide_int. */
6092 :
6093 : wide_int
6094 1068 : const_vector_int_elt (const_rtx x, unsigned int i)
6095 : {
6096 : /* First handle elements that are directly encoded. */
6097 1068 : machine_mode elt_mode = GET_MODE_INNER (GET_MODE (x));
6098 1068 : if (i < (unsigned int) XVECLEN (x, 0))
6099 0 : return rtx_mode_t (CONST_VECTOR_ENCODED_ELT (x, i), elt_mode);
6100 :
6101 : /* Identify the pattern that contains element I and work out the index of
6102 : the last encoded element for that pattern. */
6103 1068 : unsigned int encoded_nelts = const_vector_encoded_nelts (x);
6104 1068 : unsigned int npatterns = CONST_VECTOR_NPATTERNS (x);
6105 1068 : unsigned int count = i / npatterns;
6106 1068 : unsigned int pattern = i % npatterns;
6107 1068 : unsigned int final_i = encoded_nelts - npatterns + pattern;
6108 :
6109 : /* If there are no steps, the final encoded value is the right one. */
6110 1068 : if (!CONST_VECTOR_STEPPED_P (x))
6111 0 : return rtx_mode_t (CONST_VECTOR_ENCODED_ELT (x, final_i), elt_mode);
6112 :
6113 : /* Otherwise work out the value from the last two encoded elements. */
6114 1068 : rtx v1 = CONST_VECTOR_ENCODED_ELT (x, final_i - npatterns);
6115 1068 : rtx v2 = CONST_VECTOR_ENCODED_ELT (x, final_i);
6116 1068 : wide_int diff = wi::sub (rtx_mode_t (v2, elt_mode),
6117 1068 : rtx_mode_t (v1, elt_mode));
6118 1068 : return wi::add (rtx_mode_t (v2, elt_mode), (count - 2) * diff);
6119 1068 : }
6120 :
6121 : /* Return the value of element I of CONST_VECTOR X. */
6122 :
6123 : rtx
6124 4904583 : const_vector_elt (const_rtx x, unsigned int i)
6125 : {
6126 : /* First handle elements that are directly encoded. */
6127 4904583 : if (i < (unsigned int) XVECLEN (x, 0))
6128 4902447 : return CONST_VECTOR_ENCODED_ELT (x, i);
6129 :
6130 : /* If there are no steps, the final encoded value is the right one. */
6131 2136 : if (!CONST_VECTOR_STEPPED_P (x))
6132 : {
6133 : /* Identify the pattern that contains element I and work out the index of
6134 : the last encoded element for that pattern. */
6135 1068 : unsigned int encoded_nelts = const_vector_encoded_nelts (x);
6136 1068 : unsigned int npatterns = CONST_VECTOR_NPATTERNS (x);
6137 1068 : unsigned int pattern = i % npatterns;
6138 1068 : unsigned int final_i = encoded_nelts - npatterns + pattern;
6139 1068 : return CONST_VECTOR_ENCODED_ELT (x, final_i);
6140 : }
6141 :
6142 : /* Otherwise work out the value from the last two encoded elements. */
6143 1068 : return immed_wide_int_const (const_vector_int_elt (x, i),
6144 2136 : GET_MODE_INNER (GET_MODE (x)));
6145 : }
6146 :
6147 : /* Return true if X is a valid element for a CONST_VECTOR of the given
6148 : mode. */
6149 :
6150 : bool
6151 563234 : valid_for_const_vector_p (machine_mode, rtx x)
6152 : {
6153 563234 : return (CONST_SCALAR_INT_P (x)
6154 : || CONST_POLY_INT_P (x)
6155 164726 : || CONST_DOUBLE_AS_FLOAT_P (x)
6156 708274 : || CONST_FIXED_P (x));
6157 : }
6158 :
6159 : /* Generate a vector constant of mode MODE in which every element has
6160 : value ELT. */
6161 :
6162 : rtx
6163 42812169 : gen_const_vec_duplicate (machine_mode mode, rtx elt)
6164 : {
6165 42812169 : rtx_vector_builder builder (mode, 1, 1);
6166 42812169 : builder.quick_push (elt);
6167 42812169 : return builder.build ();
6168 42812169 : }
6169 :
6170 : /* Return a vector rtx of mode MODE in which every element has value X.
6171 : The result will be a constant if X is constant. */
6172 :
6173 : rtx
6174 255068 : gen_vec_duplicate (machine_mode mode, rtx x)
6175 : {
6176 255068 : if (valid_for_const_vector_p (mode, x))
6177 115812 : return gen_const_vec_duplicate (mode, x);
6178 139256 : return gen_rtx_VEC_DUPLICATE (mode, x);
6179 : }
6180 :
6181 : /* A subroutine of const_vec_series_p that handles the case in which:
6182 :
6183 : (GET_CODE (X) == CONST_VECTOR
6184 : && CONST_VECTOR_NPATTERNS (X) == 1
6185 : && !CONST_VECTOR_DUPLICATE_P (X))
6186 :
6187 : is known to hold. */
6188 :
6189 : bool
6190 5911 : const_vec_series_p_1 (const_rtx x, rtx *base_out, rtx *step_out)
6191 : {
6192 : /* Stepped sequences are only defined for integers, to avoid specifying
6193 : rounding behavior. */
6194 5911 : if (GET_MODE_CLASS (GET_MODE (x)) != MODE_VECTOR_INT)
6195 : return false;
6196 :
6197 : /* A non-duplicated vector with two elements can always be seen as a
6198 : series with a nonzero step. Longer vectors must have a stepped
6199 : encoding. */
6200 5911 : if (maybe_ne (CONST_VECTOR_NUNITS (x), 2)
6201 5911 : && !CONST_VECTOR_STEPPED_P (x))
6202 : return false;
6203 :
6204 : /* Calculate the step between the first and second elements. */
6205 5907 : scalar_mode inner = GET_MODE_INNER (GET_MODE (x));
6206 5907 : rtx base = CONST_VECTOR_ELT (x, 0);
6207 11814 : rtx step = simplify_binary_operation (MINUS, inner,
6208 5907 : CONST_VECTOR_ENCODED_ELT (x, 1), base);
6209 5907 : if (rtx_equal_p (step, CONST0_RTX (inner)))
6210 : return false;
6211 :
6212 : /* If we have a stepped encoding, check that the step between the
6213 : second and third elements is the same as STEP. */
6214 5907 : if (CONST_VECTOR_STEPPED_P (x))
6215 : {
6216 9352 : rtx diff = simplify_binary_operation (MINUS, inner,
6217 : CONST_VECTOR_ENCODED_ELT (x, 2),
6218 4676 : CONST_VECTOR_ENCODED_ELT (x, 1));
6219 4676 : if (!rtx_equal_p (step, diff))
6220 : return false;
6221 : }
6222 :
6223 5907 : *base_out = base;
6224 5907 : *step_out = step;
6225 5907 : return true;
6226 : }
6227 :
6228 : /* Generate a vector constant of mode MODE in which element I has
6229 : the value BASE + I * STEP. */
6230 :
6231 : rtx
6232 645 : gen_const_vec_series (machine_mode mode, rtx base, rtx step)
6233 : {
6234 645 : gcc_assert (valid_for_const_vector_p (mode, base)
6235 : && valid_for_const_vector_p (mode, step));
6236 :
6237 645 : rtx_vector_builder builder (mode, 1, 3);
6238 645 : builder.quick_push (base);
6239 2580 : for (int i = 1; i < 3; ++i)
6240 1290 : builder.quick_push (simplify_gen_binary (PLUS, GET_MODE_INNER (mode),
6241 1290 : builder[i - 1], step));
6242 645 : return builder.build ();
6243 645 : }
6244 :
6245 : /* Generate a vector of mode MODE in which element I has the value
6246 : BASE + I * STEP. The result will be a constant if BASE and STEP
6247 : are both constants. */
6248 :
6249 : rtx
6250 5056 : gen_vec_series (machine_mode mode, rtx base, rtx step)
6251 : {
6252 5056 : if (step == const0_rtx)
6253 211 : return gen_vec_duplicate (mode, base);
6254 4845 : if (valid_for_const_vector_p (mode, base)
6255 4845 : && valid_for_const_vector_p (mode, step))
6256 0 : return gen_const_vec_series (mode, base, step);
6257 4845 : return gen_rtx_VEC_SERIES (mode, base, step);
6258 : }
6259 :
6260 : /* Generate a new vector constant for mode MODE and constant value
6261 : CONSTANT. */
6262 :
6263 : static rtx
6264 41418663 : gen_const_vector (machine_mode mode, int constant)
6265 : {
6266 41418663 : machine_mode inner = GET_MODE_INNER (mode);
6267 :
6268 41418663 : gcc_assert (!DECIMAL_FLOAT_MODE_P (inner));
6269 :
6270 41418663 : rtx el = const_tiny_rtx[constant][(int) inner];
6271 41418663 : gcc_assert (el);
6272 :
6273 41418663 : return gen_const_vec_duplicate (mode, el);
6274 : }
6275 :
6276 : /* Generate a vector like gen_rtx_raw_CONST_VEC, but use the zero vector when
6277 : all elements are zero, and the one vector when all elements are one. */
6278 : rtx
6279 167995 : gen_rtx_CONST_VECTOR (machine_mode mode, rtvec v)
6280 : {
6281 335990 : gcc_assert (known_eq (GET_MODE_NUNITS (mode), GET_NUM_ELEM (v)));
6282 :
6283 : /* If the values are all the same, check to see if we can use one of the
6284 : standard constant vectors. */
6285 167995 : if (rtvec_all_equal_p (v))
6286 48583 : return gen_const_vec_duplicate (mode, RTVEC_ELT (v, 0));
6287 :
6288 119412 : unsigned int nunits = GET_NUM_ELEM (v);
6289 119412 : rtx_vector_builder builder (mode, nunits, 1);
6290 815576 : for (unsigned int i = 0; i < nunits; ++i)
6291 576752 : builder.quick_push (RTVEC_ELT (v, i));
6292 119412 : return builder.build (v);
6293 119412 : }
6294 :
6295 : /* Initialise global register information required by all functions. */
6296 :
6297 : void
6298 801328 : init_emit_regs (void)
6299 : {
6300 801328 : int i;
6301 801328 : machine_mode mode;
6302 801328 : mem_attrs *attrs;
6303 :
6304 : /* Reset register attributes */
6305 801328 : reg_attrs_htab->empty ();
6306 :
6307 : /* We need reg_raw_mode, so initialize the modes now. */
6308 801328 : init_reg_modes_target ();
6309 :
6310 : /* Assign register numbers to the globally defined register rtx. */
6311 816523 : stack_pointer_rtx = gen_raw_REG (Pmode, STACK_POINTER_REGNUM);
6312 816523 : frame_pointer_rtx = gen_raw_REG (Pmode, FRAME_POINTER_REGNUM);
6313 816523 : hard_frame_pointer_rtx = gen_raw_REG (Pmode, HARD_FRAME_POINTER_REGNUM);
6314 816523 : arg_pointer_rtx = gen_raw_REG (Pmode, ARG_POINTER_REGNUM);
6315 2403984 : virtual_incoming_args_rtx =
6316 816523 : gen_raw_REG (Pmode, VIRTUAL_INCOMING_ARGS_REGNUM);
6317 2403984 : virtual_stack_vars_rtx =
6318 816523 : gen_raw_REG (Pmode, VIRTUAL_STACK_VARS_REGNUM);
6319 2403984 : virtual_stack_dynamic_rtx =
6320 816523 : gen_raw_REG (Pmode, VIRTUAL_STACK_DYNAMIC_REGNUM);
6321 2403984 : virtual_outgoing_args_rtx =
6322 816523 : gen_raw_REG (Pmode, VIRTUAL_OUTGOING_ARGS_REGNUM);
6323 816523 : virtual_cfa_rtx = gen_raw_REG (Pmode, VIRTUAL_CFA_REGNUM);
6324 2403984 : virtual_preferred_stack_boundary_rtx =
6325 816523 : gen_raw_REG (Pmode, VIRTUAL_PREFERRED_STACK_BOUNDARY_REGNUM);
6326 :
6327 : /* Initialize RTL for commonly used hard registers. These are
6328 : copied into regno_reg_rtx as we begin to compile each function. */
6329 76126160 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
6330 75324832 : initial_regno_reg_rtx[i] = gen_raw_REG (reg_raw_mode[i], i);
6331 :
6332 : #ifdef RETURN_ADDRESS_POINTER_REGNUM
6333 : return_address_pointer_rtx
6334 : = gen_raw_REG (Pmode, RETURN_ADDRESS_POINTER_REGNUM);
6335 : #endif
6336 :
6337 801328 : pic_offset_table_rtx = NULL_RTX;
6338 801328 : if ((unsigned) PIC_OFFSET_TABLE_REGNUM != INVALID_REGNUM)
6339 20661 : pic_offset_table_rtx = gen_raw_REG (Pmode, PIC_OFFSET_TABLE_REGNUM);
6340 :
6341 : /* Process stack-limiting command-line options. */
6342 801328 : if (opt_fstack_limit_symbol_arg != NULL)
6343 0 : stack_limit_rtx
6344 0 : = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (opt_fstack_limit_symbol_arg));
6345 801328 : if (opt_fstack_limit_register_no >= 0)
6346 0 : stack_limit_rtx = gen_rtx_REG (Pmode, opt_fstack_limit_register_no);
6347 :
6348 100166000 : for (i = 0; i < (int) MAX_MACHINE_MODE; i++)
6349 : {
6350 99364672 : mode = (machine_mode) i;
6351 99364672 : attrs = ggc_cleared_alloc<mem_attrs> ();
6352 99364672 : attrs->align = BITS_PER_UNIT;
6353 99364672 : attrs->addrspace = ADDR_SPACE_GENERIC;
6354 99364672 : if (mode != BLKmode && mode != VOIDmode)
6355 : {
6356 97762016 : attrs->size_known_p = true;
6357 195524032 : attrs->size = GET_MODE_SIZE (mode);
6358 97762016 : if (STRICT_ALIGNMENT)
6359 : attrs->align = GET_MODE_ALIGNMENT (mode);
6360 : }
6361 99364672 : mode_mem_attrs[i] = attrs;
6362 : }
6363 :
6364 801328 : split_branch_probability = profile_probability::uninitialized ();
6365 801328 : }
6366 :
6367 : /* Initialize global machine_mode variables. */
6368 :
6369 : void
6370 296879 : init_derived_machine_modes (void)
6371 : {
6372 296879 : opt_scalar_int_mode mode_iter, opt_byte_mode, opt_word_mode;
6373 2375032 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
6374 : {
6375 2078153 : scalar_int_mode mode = mode_iter.require ();
6376 :
6377 2375032 : if (GET_MODE_BITSIZE (mode) == BITS_PER_UNIT
6378 2078153 : && !opt_byte_mode.exists ())
6379 296879 : opt_byte_mode = mode;
6380 :
6381 4156306 : if (GET_MODE_BITSIZE (mode) == BITS_PER_WORD
6382 2078153 : && !opt_word_mode.exists ())
6383 296879 : opt_word_mode = mode;
6384 : }
6385 :
6386 296879 : byte_mode = opt_byte_mode.require ();
6387 296879 : word_mode = opt_word_mode.require ();
6388 296879 : ptr_mode = as_a <scalar_int_mode>
6389 311323 : (mode_for_size (POINTER_SIZE, GET_MODE_CLASS (Pmode), 0).require ());
6390 296879 : }
6391 :
6392 : /* Create some permanent unique rtl objects shared between all functions. */
6393 :
6394 : void
6395 289641 : init_emit_once (void)
6396 : {
6397 289641 : int i;
6398 289641 : machine_mode mode;
6399 289641 : scalar_float_mode double_mode;
6400 289641 : opt_scalar_mode smode_iter;
6401 :
6402 : /* Initialize the CONST_INT, CONST_WIDE_INT, CONST_DOUBLE,
6403 : CONST_FIXED, and memory attribute hash tables. */
6404 289641 : const_int_htab = hash_table<const_int_hasher>::create_ggc (37);
6405 :
6406 : #if TARGET_SUPPORTS_WIDE_INT
6407 289641 : const_wide_int_htab = hash_table<const_wide_int_hasher>::create_ggc (37);
6408 : #endif
6409 289641 : const_double_htab = hash_table<const_double_hasher>::create_ggc (37);
6410 :
6411 289641 : if (NUM_POLY_INT_COEFFS > 1)
6412 : const_poly_int_htab = hash_table<const_poly_int_hasher>::create_ggc (37);
6413 :
6414 289641 : const_fixed_htab = hash_table<const_fixed_hasher>::create_ggc (37);
6415 :
6416 289641 : reg_attrs_htab = hash_table<reg_attr_hasher>::create_ggc (37);
6417 :
6418 : #ifdef INIT_EXPANDERS
6419 : /* This is to initialize {init|mark|free}_machine_status before the first
6420 : call to push_function_context_to. This is needed by the Chill front
6421 : end which calls push_function_context_to before the first call to
6422 : init_function_start. */
6423 : INIT_EXPANDERS;
6424 : #endif
6425 :
6426 : /* Create the unique rtx's for certain rtx codes and operand values. */
6427 :
6428 : /* Don't use gen_rtx_CONST_INT here since gen_rtx_CONST_INT in this case
6429 : tries to use these variables. */
6430 37653330 : for (i = - MAX_SAVED_CONST_INT; i <= MAX_SAVED_CONST_INT; i++)
6431 74727378 : const_int_rtx[i + MAX_SAVED_CONST_INT] =
6432 37363689 : gen_rtx_raw_CONST_INT (VOIDmode, (HOST_WIDE_INT) i);
6433 :
6434 289641 : if (STORE_FLAG_VALUE >= - MAX_SAVED_CONST_INT
6435 : && STORE_FLAG_VALUE <= MAX_SAVED_CONST_INT)
6436 289641 : const_true_rtx = const_int_rtx[STORE_FLAG_VALUE + MAX_SAVED_CONST_INT];
6437 : else
6438 : const_true_rtx = gen_rtx_CONST_INT (VOIDmode, STORE_FLAG_VALUE);
6439 :
6440 289641 : mode = targetm.c.mode_for_floating_type (TI_DOUBLE_TYPE);
6441 289641 : double_mode = as_a<scalar_float_mode> (mode);
6442 :
6443 289641 : real_from_integer (&dconst0, double_mode, 0, SIGNED);
6444 289641 : real_from_integer (&dconst1, double_mode, 1, SIGNED);
6445 289641 : real_from_integer (&dconst2, double_mode, 2, SIGNED);
6446 :
6447 289641 : dconstm0 = dconst0;
6448 289641 : dconstm0.sign = 1;
6449 :
6450 289641 : dconstm1 = dconst1;
6451 289641 : dconstm1.sign = 1;
6452 :
6453 289641 : dconsthalf = dconst1;
6454 289641 : SET_REAL_EXP (&dconsthalf, REAL_EXP (&dconsthalf) - 1);
6455 :
6456 289641 : real_inf (&dconstinf);
6457 289641 : real_inf (&dconstninf, true);
6458 :
6459 1448205 : for (i = 0; i < 3; i++)
6460 : {
6461 868923 : const REAL_VALUE_TYPE *const r =
6462 : (i == 0 ? &dconst0 : i == 1 ? &dconst1 : &dconst2);
6463 :
6464 6082461 : FOR_EACH_MODE_IN_CLASS (mode, MODE_FLOAT)
6465 5213538 : const_tiny_rtx[i][(int) mode] =
6466 5213538 : const_double_from_real_value (*r, mode);
6467 :
6468 3475692 : FOR_EACH_MODE_IN_CLASS (mode, MODE_DECIMAL_FLOAT)
6469 2606769 : const_tiny_rtx[i][(int) mode] =
6470 2606769 : const_double_from_real_value (*r, mode);
6471 :
6472 868923 : const_tiny_rtx[i][(int) VOIDmode] = GEN_INT (i);
6473 :
6474 6951384 : FOR_EACH_MODE_IN_CLASS (mode, MODE_INT)
6475 6082461 : const_tiny_rtx[i][(int) mode] = GEN_INT (i);
6476 :
6477 2606769 : for (mode = MIN_MODE_PARTIAL_INT;
6478 3475692 : mode <= MAX_MODE_PARTIAL_INT;
6479 2606769 : mode = (machine_mode)((int)(mode) + 1))
6480 2606769 : const_tiny_rtx[i][(int) mode] = GEN_INT (i);
6481 : }
6482 :
6483 289641 : const_tiny_rtx[3][(int) VOIDmode] = constm1_rtx;
6484 :
6485 2317128 : FOR_EACH_MODE_IN_CLASS (mode, MODE_INT)
6486 2027487 : const_tiny_rtx[3][(int) mode] = constm1_rtx;
6487 :
6488 : /* For BImode, 1 and -1 are unsigned and signed interpretations
6489 : of the same value. */
6490 289641 : for (mode = MIN_MODE_BOOL;
6491 579282 : mode <= MAX_MODE_BOOL;
6492 289641 : mode = (machine_mode)((int)(mode) + 1))
6493 : {
6494 289641 : const_tiny_rtx[0][(int) mode] = const0_rtx;
6495 289641 : if (mode == BImode)
6496 : {
6497 289641 : const_tiny_rtx[1][(int) mode] = const_true_rtx;
6498 289641 : const_tiny_rtx[3][(int) mode] = const_true_rtx;
6499 : }
6500 : else
6501 : {
6502 : const_tiny_rtx[1][(int) mode] = const1_rtx;
6503 : const_tiny_rtx[3][(int) mode] = constm1_rtx;
6504 : }
6505 : }
6506 :
6507 868923 : for (mode = MIN_MODE_PARTIAL_INT;
6508 1158564 : mode <= MAX_MODE_PARTIAL_INT;
6509 868923 : mode = (machine_mode)((int)(mode) + 1))
6510 868923 : const_tiny_rtx[3][(int) mode] = constm1_rtx;
6511 :
6512 3186051 : FOR_EACH_MODE_IN_CLASS (mode, MODE_COMPLEX_INT)
6513 : {
6514 2896410 : rtx inner = const_tiny_rtx[0][(int)GET_MODE_INNER (mode)];
6515 2896410 : const_tiny_rtx[0][(int) mode] = gen_rtx_CONCAT (mode, inner, inner);
6516 : }
6517 :
6518 2027487 : FOR_EACH_MODE_IN_CLASS (mode, MODE_COMPLEX_FLOAT)
6519 : {
6520 1737846 : rtx inner = const_tiny_rtx[0][(int)GET_MODE_INNER (mode)];
6521 1737846 : const_tiny_rtx[0][(int) mode] = gen_rtx_CONCAT (mode, inner, inner);
6522 : }
6523 :
6524 289641 : FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_BOOL)
6525 : {
6526 0 : const_tiny_rtx[0][(int) mode] = gen_const_vector (mode, 0);
6527 0 : const_tiny_rtx[3][(int) mode] = gen_const_vector (mode, 3);
6528 0 : if (GET_MODE_INNER (mode) == BImode)
6529 : /* As for BImode, "all 1" and "all -1" are unsigned and signed
6530 : interpretations of the same value. */
6531 : const_tiny_rtx[1][(int) mode] = const_tiny_rtx[3][(int) mode];
6532 : else
6533 0 : const_tiny_rtx[1][(int) mode] = gen_const_vector (mode, 1);
6534 : }
6535 :
6536 9268512 : FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_INT)
6537 : {
6538 8978871 : const_tiny_rtx[0][(int) mode] = gen_const_vector (mode, 0);
6539 8978871 : const_tiny_rtx[1][(int) mode] = gen_const_vector (mode, 1);
6540 8978871 : const_tiny_rtx[3][(int) mode] = gen_const_vector (mode, 3);
6541 : }
6542 :
6543 7530666 : FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_FLOAT)
6544 : {
6545 7241025 : const_tiny_rtx[0][(int) mode] = gen_const_vector (mode, 0);
6546 7241025 : const_tiny_rtx[1][(int) mode] = gen_const_vector (mode, 1);
6547 : }
6548 :
6549 1737846 : FOR_EACH_MODE_IN_CLASS (smode_iter, MODE_FRACT)
6550 : {
6551 1448205 : scalar_mode smode = smode_iter.require ();
6552 1448205 : FCONST0 (smode).data.high = 0;
6553 1448205 : FCONST0 (smode).data.low = 0;
6554 1448205 : FCONST0 (smode).mode = smode;
6555 2896410 : const_tiny_rtx[0][(int) smode]
6556 1448205 : = CONST_FIXED_FROM_FIXED_VALUE (FCONST0 (smode), smode);
6557 : }
6558 :
6559 1737846 : FOR_EACH_MODE_IN_CLASS (smode_iter, MODE_UFRACT)
6560 : {
6561 1448205 : scalar_mode smode = smode_iter.require ();
6562 1448205 : FCONST0 (smode).data.high = 0;
6563 1448205 : FCONST0 (smode).data.low = 0;
6564 1448205 : FCONST0 (smode).mode = smode;
6565 2896410 : const_tiny_rtx[0][(int) smode]
6566 1448205 : = CONST_FIXED_FROM_FIXED_VALUE (FCONST0 (smode), smode);
6567 : }
6568 :
6569 1448205 : FOR_EACH_MODE_IN_CLASS (smode_iter, MODE_ACCUM)
6570 : {
6571 1158564 : scalar_mode smode = smode_iter.require ();
6572 1158564 : FCONST0 (smode).data.high = 0;
6573 1158564 : FCONST0 (smode).data.low = 0;
6574 1158564 : FCONST0 (smode).mode = smode;
6575 2317128 : const_tiny_rtx[0][(int) smode]
6576 1158564 : = CONST_FIXED_FROM_FIXED_VALUE (FCONST0 (smode), smode);
6577 :
6578 : /* We store the value 1. */
6579 1158564 : FCONST1 (smode).data.high = 0;
6580 1158564 : FCONST1 (smode).data.low = 0;
6581 1158564 : FCONST1 (smode).mode = smode;
6582 1158564 : FCONST1 (smode).data
6583 1158564 : = double_int_one.lshift (GET_MODE_FBIT (smode),
6584 : HOST_BITS_PER_DOUBLE_INT,
6585 1158564 : SIGNED_FIXED_POINT_MODE_P (smode));
6586 2317128 : const_tiny_rtx[1][(int) smode]
6587 1158564 : = CONST_FIXED_FROM_FIXED_VALUE (FCONST1 (smode), smode);
6588 : }
6589 :
6590 1448205 : FOR_EACH_MODE_IN_CLASS (smode_iter, MODE_UACCUM)
6591 : {
6592 1158564 : scalar_mode smode = smode_iter.require ();
6593 1158564 : FCONST0 (smode).data.high = 0;
6594 1158564 : FCONST0 (smode).data.low = 0;
6595 1158564 : FCONST0 (smode).mode = smode;
6596 2317128 : const_tiny_rtx[0][(int) smode]
6597 1158564 : = CONST_FIXED_FROM_FIXED_VALUE (FCONST0 (smode), smode);
6598 :
6599 : /* We store the value 1. */
6600 1158564 : FCONST1 (smode).data.high = 0;
6601 1158564 : FCONST1 (smode).data.low = 0;
6602 1158564 : FCONST1 (smode).mode = smode;
6603 1158564 : FCONST1 (smode).data
6604 1158564 : = double_int_one.lshift (GET_MODE_FBIT (smode),
6605 : HOST_BITS_PER_DOUBLE_INT,
6606 1158564 : SIGNED_FIXED_POINT_MODE_P (smode));
6607 2317128 : const_tiny_rtx[1][(int) smode]
6608 1158564 : = CONST_FIXED_FROM_FIXED_VALUE (FCONST1 (smode), smode);
6609 : }
6610 :
6611 289641 : FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_FRACT)
6612 : {
6613 0 : const_tiny_rtx[0][(int) mode] = gen_const_vector (mode, 0);
6614 : }
6615 :
6616 289641 : FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_UFRACT)
6617 : {
6618 0 : const_tiny_rtx[0][(int) mode] = gen_const_vector (mode, 0);
6619 : }
6620 :
6621 289641 : FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_ACCUM)
6622 : {
6623 0 : const_tiny_rtx[0][(int) mode] = gen_const_vector (mode, 0);
6624 0 : const_tiny_rtx[1][(int) mode] = gen_const_vector (mode, 1);
6625 : }
6626 :
6627 289641 : FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_UACCUM)
6628 : {
6629 0 : const_tiny_rtx[0][(int) mode] = gen_const_vector (mode, 0);
6630 0 : const_tiny_rtx[1][(int) mode] = gen_const_vector (mode, 1);
6631 : }
6632 :
6633 35625843 : for (i = (int) CCmode; i < (int) MAX_MACHINE_MODE; ++i)
6634 35336202 : if (GET_MODE_CLASS ((machine_mode) i) == MODE_CC)
6635 3475692 : const_tiny_rtx[0][i] = const0_rtx;
6636 :
6637 289641 : pc_rtx = gen_rtx_fmt_ (PC, VOIDmode);
6638 289641 : ret_rtx = gen_rtx_fmt_ (RETURN, VOIDmode);
6639 289641 : simple_return_rtx = gen_rtx_fmt_ (SIMPLE_RETURN, VOIDmode);
6640 289641 : invalid_insn_rtx = gen_rtx_INSN (VOIDmode,
6641 : /*prev_insn=*/NULL,
6642 : /*next_insn=*/NULL,
6643 : /*bb=*/NULL,
6644 : /*pattern=*/NULL_RTX,
6645 : /*location=*/-1,
6646 : CODE_FOR_nothing,
6647 : /*reg_notes=*/NULL_RTX);
6648 289641 : }
6649 :
6650 : /* Produce exact duplicate of insn INSN after AFTER.
6651 : Care updating of libcall regions if present. */
6652 :
6653 : rtx_insn *
6654 3646492 : emit_copy_of_insn_after (rtx_insn *insn, rtx_insn *after)
6655 : {
6656 3646492 : rtx_insn *new_rtx;
6657 3646492 : rtx link;
6658 :
6659 3646492 : switch (GET_CODE (insn))
6660 : {
6661 1712401 : case INSN:
6662 1712401 : new_rtx = emit_insn_after (copy_insn (PATTERN (insn)), after);
6663 1712401 : break;
6664 :
6665 500852 : case JUMP_INSN:
6666 500852 : new_rtx = emit_jump_insn_after (copy_insn (PATTERN (insn)), after);
6667 500852 : set_crossing_jump (new_rtx, CROSSING_JUMP_P (insn));
6668 500852 : break;
6669 :
6670 1408529 : case DEBUG_INSN:
6671 1408529 : new_rtx = emit_debug_insn_after (copy_insn (PATTERN (insn)), after);
6672 1408529 : break;
6673 :
6674 24710 : case CALL_INSN:
6675 24710 : new_rtx = emit_call_insn_after (copy_insn (PATTERN (insn)), after);
6676 24710 : if (CALL_INSN_FUNCTION_USAGE (insn))
6677 22346 : CALL_INSN_FUNCTION_USAGE (new_rtx)
6678 22346 : = copy_insn (CALL_INSN_FUNCTION_USAGE (insn));
6679 24710 : CALL_INSN_ABI_ID (new_rtx) = CALL_INSN_ABI_ID (insn);
6680 24710 : SIBLING_CALL_P (new_rtx) = SIBLING_CALL_P (insn);
6681 24710 : RTL_CONST_CALL_P (new_rtx) = RTL_CONST_CALL_P (insn);
6682 24710 : RTL_PURE_CALL_P (new_rtx) = RTL_PURE_CALL_P (insn);
6683 24710 : RTL_LOOPING_CONST_OR_PURE_CALL_P (new_rtx)
6684 24710 : = RTL_LOOPING_CONST_OR_PURE_CALL_P (insn);
6685 24710 : break;
6686 :
6687 0 : default:
6688 0 : gcc_unreachable ();
6689 : }
6690 :
6691 : /* Update LABEL_NUSES. */
6692 3646492 : if (NONDEBUG_INSN_P (insn))
6693 2237963 : mark_jump_label (PATTERN (new_rtx), new_rtx, 0);
6694 :
6695 3646492 : INSN_LOCATION (new_rtx) = INSN_LOCATION (insn);
6696 :
6697 : /* If the old insn is frame related, then so is the new one. This is
6698 : primarily needed for IA-64 unwind info which marks epilogue insns,
6699 : which may be duplicated by the basic block reordering code. */
6700 3646492 : RTX_FRAME_RELATED_P (new_rtx) = RTX_FRAME_RELATED_P (insn);
6701 :
6702 : /* Locate the end of existing REG_NOTES in NEW_RTX. */
6703 3646492 : rtx *ptail = ®_NOTES (new_rtx);
6704 3646501 : while (*ptail != NULL_RTX)
6705 9 : ptail = &XEXP (*ptail, 1);
6706 :
6707 : /* Copy all REG_NOTES except REG_LABEL_OPERAND since mark_jump_label
6708 : will make them. REG_LABEL_TARGETs are created there too, but are
6709 : supposed to be sticky, so we copy them. */
6710 5961608 : for (link = REG_NOTES (insn); link; link = XEXP (link, 1))
6711 2315116 : if (REG_NOTE_KIND (link) != REG_LABEL_OPERAND)
6712 : {
6713 2313991 : *ptail = duplicate_reg_note (link);
6714 2313991 : ptail = &XEXP (*ptail, 1);
6715 : }
6716 :
6717 3646492 : INSN_CODE (new_rtx) = INSN_CODE (insn);
6718 3646492 : return new_rtx;
6719 : }
6720 :
6721 : static GTY((deletable)) rtx hard_reg_clobbers [NUM_MACHINE_MODES][FIRST_PSEUDO_REGISTER];
6722 : rtx
6723 4653509 : gen_hard_reg_clobber (machine_mode mode, unsigned int regno)
6724 : {
6725 4653509 : if (hard_reg_clobbers[mode][regno])
6726 : return hard_reg_clobbers[mode][regno];
6727 : else
6728 196478 : return (hard_reg_clobbers[mode][regno] =
6729 392956 : gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (mode, regno)));
6730 : }
6731 :
6732 : location_t prologue_location;
6733 : location_t epilogue_location;
6734 :
6735 : /* Hold current location information and last location information, so the
6736 : datastructures are built lazily only when some instructions in given
6737 : place are needed. */
6738 : static location_t curr_location;
6739 :
6740 : /* Allocate insn location datastructure. */
6741 : void
6742 1759257 : insn_locations_init (void)
6743 : {
6744 1759257 : prologue_location = epilogue_location = 0;
6745 1759257 : curr_location = UNKNOWN_LOCATION;
6746 1759257 : }
6747 :
6748 : /* At the end of emit stage, clear current location. */
6749 : void
6750 1519810 : insn_locations_finalize (void)
6751 : {
6752 1519810 : epilogue_location = curr_location;
6753 1519810 : curr_location = UNKNOWN_LOCATION;
6754 1519810 : }
6755 :
6756 : /* Set current location. */
6757 : void
6758 157394548 : set_curr_insn_location (location_t location)
6759 : {
6760 157394548 : curr_location = location;
6761 157394548 : }
6762 :
6763 : /* Get current location. */
6764 : location_t
6765 272427266 : curr_insn_location (void)
6766 : {
6767 272427266 : return curr_location;
6768 : }
6769 :
6770 : /* Set the location of the insn chain starting at INSN to LOC. */
6771 : void
6772 3676601 : set_insn_locations (rtx_insn *insn, location_t loc)
6773 : {
6774 16496660 : while (insn)
6775 : {
6776 12820059 : if (INSN_P (insn))
6777 10847547 : INSN_LOCATION (insn) = loc;
6778 12820059 : insn = NEXT_INSN (insn);
6779 : }
6780 3676601 : }
6781 :
6782 : /* Return lexical scope block insn belongs to. */
6783 : tree
6784 50051560 : insn_scope (const rtx_insn *insn)
6785 : {
6786 50051560 : return LOCATION_BLOCK (INSN_LOCATION (insn));
6787 : }
6788 :
6789 : /* Return line number of the statement that produced this insn. */
6790 : int
6791 0 : insn_line (const rtx_insn *insn)
6792 : {
6793 0 : return LOCATION_LINE (INSN_LOCATION (insn));
6794 : }
6795 :
6796 : /* Return source file of the statement that produced this insn. */
6797 : const char *
6798 0 : insn_file (const rtx_insn *insn)
6799 : {
6800 0 : return LOCATION_FILE (INSN_LOCATION (insn));
6801 : }
6802 :
6803 : /* Return expanded location of the statement that produced this insn. */
6804 : expanded_location
6805 86030901 : insn_location (const rtx_insn *insn)
6806 : {
6807 86030901 : return expand_location (INSN_LOCATION (insn));
6808 : }
6809 :
6810 : /* Return true if memory model MODEL requires a pre-operation (release-style)
6811 : barrier or a post-operation (acquire-style) barrier. While not universal,
6812 : this function matches behavior of several targets. */
6813 :
6814 : bool
6815 0 : need_atomic_barrier_p (enum memmodel model, bool pre)
6816 : {
6817 0 : switch (model & MEMMODEL_BASE_MASK)
6818 : {
6819 : case MEMMODEL_RELAXED:
6820 : case MEMMODEL_CONSUME:
6821 : return false;
6822 0 : case MEMMODEL_RELEASE:
6823 0 : return pre;
6824 0 : case MEMMODEL_ACQUIRE:
6825 0 : return !pre;
6826 0 : case MEMMODEL_ACQ_REL:
6827 0 : case MEMMODEL_SEQ_CST:
6828 0 : return true;
6829 0 : default:
6830 0 : gcc_unreachable ();
6831 : }
6832 : }
6833 :
6834 : /* Return a constant shift amount for shifting a value of mode MODE
6835 : by VALUE bits. */
6836 :
6837 : rtx
6838 216260435 : gen_int_shift_amount (machine_mode, poly_int64 value)
6839 : {
6840 : /* Use a 64-bit mode, to avoid any truncation.
6841 :
6842 : ??? Perhaps this should be automatically derived from the .md files
6843 : instead, or perhaps have a target hook. */
6844 216260435 : scalar_int_mode shift_mode = (BITS_PER_UNIT == 8
6845 : ? DImode
6846 : : int_mode_for_size (64, 0).require ());
6847 216260435 : return gen_int_mode (value, shift_mode);
6848 : }
6849 :
6850 : namespace {
6851 : /* Helper class for expanding an rtx using the encoding generated by
6852 : genemit.cc. The code needs to be kept in sync with there. */
6853 :
6854 : class rtx_expander
6855 : {
6856 : public:
6857 : rtx_expander (const uint8_t *, rtx *);
6858 :
6859 : rtx get_rtx ();
6860 : rtvec get_rtvec ();
6861 : void expand_seq ();
6862 :
6863 : protected:
6864 : uint64_t get_uint ();
6865 86687596 : machine_mode get_mode () { return machine_mode (get_uint ()); }
6866 : char *get_string ();
6867 : rtx get_shared_operand ();
6868 : rtx get_unshared_operand ();
6869 :
6870 : rtx get_rtx (expand_opcode);
6871 : rtx get_rtx (rtx_code, machine_mode);
6872 :
6873 : /* Points to the first unread byte. */
6874 : const uint8_t *m_seq;
6875 :
6876 : /* The operands passed to the gen_* function. */
6877 : rtx *m_operands;
6878 :
6879 : /* A bitmap of operands that have already been used to replace a
6880 : MATCH_OPERAND or MATCH_DUP. In order to ensure correct sharing,
6881 : further replacements need to use a copy of the operand, rather than
6882 : the original rtx. */
6883 : bbitmap<MAX_RECOG_OPERANDS> m_used;
6884 : };
6885 : }
6886 :
6887 25639794 : rtx_expander::rtx_expander (const uint8_t *seq, rtx *operands)
6888 25639794 : : m_seq (seq), m_operands (operands), m_used ()
6889 0 : {}
6890 :
6891 : /* Read and return the next encoded "BEB128" integer. */
6892 :
6893 : inline uint64_t
6894 250524043 : rtx_expander::get_uint ()
6895 : {
6896 250524043 : const uint8_t *seq = m_seq;
6897 250524043 : uint64_t res = 0;
6898 252759739 : do
6899 252759739 : res = (res << 7) | (*seq & 127);
6900 252759739 : while (*seq++ >= 128);
6901 250524043 : m_seq = seq;
6902 250524043 : return res;
6903 : }
6904 :
6905 : /* Read an operand number and return the associated operand rtx,
6906 : without copying it. */
6907 :
6908 : rtx
6909 892912 : rtx_expander::get_shared_operand ()
6910 : {
6911 892912 : return m_operands[get_uint ()];
6912 : }
6913 :
6914 : /* Read an operand number and return a correctly-shared instance of
6915 : the associated operand rtx. This can be either the original rtx
6916 : or a copy. */
6917 :
6918 : rtx
6919 46012218 : rtx_expander::get_unshared_operand ()
6920 : {
6921 46012218 : auto opno = get_uint ();
6922 46012218 : auto mask = m_used.from_index (opno);
6923 46012218 : if (m_used & mask)
6924 2995351 : return copy_rtx (m_operands[opno]);
6925 :
6926 43016867 : m_used |= mask;
6927 43016867 : return m_operands[opno];
6928 : }
6929 :
6930 : /* Read an encoded rtx. */
6931 :
6932 : rtx
6933 130860247 : rtx_expander::get_rtx ()
6934 : {
6935 130860247 : auto FIRST_CODE = (unsigned) expand_opcode::FIRST_CODE;
6936 130860247 : auto opcode = get_uint ();
6937 130860247 : if (opcode < FIRST_CODE)
6938 51558639 : return get_rtx (expand_opcode (opcode));
6939 79301608 : return get_rtx (rtx_code (opcode - FIRST_CODE), NUM_MACHINE_MODES);
6940 : }
6941 :
6942 : /* Read an encoded rtx that starts with the given opcode. */
6943 :
6944 : rtx
6945 51558639 : rtx_expander::get_rtx (expand_opcode opcode)
6946 : {
6947 51558639 : switch (opcode)
6948 : {
6949 : case expand_opcode::NO_RTX:
6950 : return NULL_RTX;
6951 :
6952 46012218 : case expand_opcode::MATCH_OPERAND:
6953 46012218 : return get_unshared_operand ();
6954 :
6955 42856 : case expand_opcode::MATCH_OPERATOR_WITH_MODE:
6956 42856 : {
6957 42856 : auto mode = get_mode ();
6958 42856 : auto op = get_shared_operand ();
6959 42856 : return get_rtx (GET_CODE (op), mode);
6960 : }
6961 :
6962 848247 : case expand_opcode::MATCH_OPERATOR:
6963 848247 : {
6964 848247 : auto op = get_shared_operand ();
6965 848247 : return get_rtx (GET_CODE (op), GET_MODE (op));
6966 : }
6967 :
6968 1809 : case expand_opcode::MATCH_PARALLEL:
6969 1809 : return get_shared_operand ();
6970 :
6971 4653509 : case expand_opcode::CLOBBER_REG:
6972 4653509 : {
6973 4653509 : auto mode = get_mode ();
6974 4653509 : auto regno = get_uint ();
6975 4653509 : return gen_hard_reg_clobber (mode, regno);
6976 : }
6977 :
6978 : case expand_opcode::FIRST_CODE:
6979 : break;
6980 : }
6981 0 : gcc_unreachable ();
6982 : }
6983 :
6984 : /* Read the rest of an rtx of code CODE. If such rtxes are not always
6985 : VOIDmode, MODE is the mode that the rtx should have, or NUM_MACHINE_MODES
6986 : if the mode is encoded at the current iterator position. */
6987 :
6988 : rtx
6989 80192711 : rtx_expander::get_rtx (rtx_code code, machine_mode mode)
6990 : {
6991 80192711 : switch (code)
6992 : {
6993 : /* Please keep the cases below in sync with gengenrtl.cc:special_rtx. */
6994 :
6995 0 : case EXPR_LIST:
6996 0 : case INSN_LIST:
6997 0 : case INSN:
6998 0 : gcc_unreachable ();
6999 :
7000 4909320 : case CONST_INT:
7001 9818640 : return GEN_INT (get_uint ());
7002 :
7003 1943141 : case REG:
7004 1943141 : if (mode == NUM_MACHINE_MODES)
7005 3886282 : mode = get_mode ();
7006 3886282 : return gen_rtx_REG (mode, get_uint ());
7007 :
7008 52243 : case SUBREG:
7009 52243 : {
7010 52243 : if (mode == NUM_MACHINE_MODES)
7011 104486 : mode = get_mode ();
7012 52243 : auto reg = get_rtx ();
7013 52243 : auto byte = get_uint ();
7014 52243 : return gen_rtx_SUBREG (mode, reg, byte);
7015 : }
7016 :
7017 2515253 : case MEM:
7018 2515253 : if (mode == NUM_MACHINE_MODES)
7019 5030506 : mode = get_mode ();
7020 2515253 : return gen_rtx_MEM (mode, get_rtx ());
7021 :
7022 9683421 : case PC:
7023 9683421 : return pc_rtx;
7024 :
7025 0 : case RETURN:
7026 0 : return ret_rtx;
7027 :
7028 1617284 : case SIMPLE_RETURN:
7029 1617284 : return simple_return_rtx;
7030 :
7031 1926 : case CONST_VECTOR:
7032 1926 : if (mode == NUM_MACHINE_MODES)
7033 3852 : mode = get_mode ();
7034 1926 : return gen_rtx_CONST_VECTOR (mode, get_rtvec ());
7035 :
7036 : /* Please keep the cases below in sync with
7037 : gengenrtl.cc:excluded_rtx. */
7038 :
7039 0 : case VAR_LOCATION:
7040 0 : gcc_unreachable ();
7041 :
7042 0 : case CONST_DOUBLE:
7043 : /* genemit.cc only accepts zero const_doubles. */
7044 0 : if (mode == NUM_MACHINE_MODES)
7045 0 : mode = get_mode ();
7046 0 : return CONST0_RTX (mode);
7047 :
7048 0 : case CONST_WIDE_INT:
7049 0 : case CONST_POLY_INT:
7050 0 : case CONST_FIXED:
7051 0 : gcc_unreachable ();
7052 :
7053 59470123 : default:
7054 59470123 : break;
7055 : }
7056 :
7057 59470123 : rtx x = rtx_alloc (code);
7058 59470123 : if (!always_void_p (code))
7059 : {
7060 35025973 : if (mode == NUM_MACHINE_MODES)
7061 68269740 : mode = get_mode ();
7062 35025973 : PUT_MODE_RAW (x, mode);
7063 : }
7064 :
7065 59470123 : const char *fmt = GET_RTX_FORMAT (code);
7066 153695604 : for (unsigned int i = 0; fmt[i]; ++i)
7067 94225481 : switch (fmt[i])
7068 : {
7069 : /* Please keep these cases in sync with
7070 : gengenrtl.cc:type_from_format. */
7071 :
7072 1860266 : case 'i':
7073 1860266 : XINT (x, i) = get_uint ();
7074 1860266 : break;
7075 :
7076 0 : case 'L':
7077 0 : case 'w':
7078 0 : case 'p':
7079 0 : case 's':
7080 0 : gcc_unreachable ();
7081 :
7082 84807511 : case 'e': case 'u':
7083 84807511 : XEXP (x, i) = get_rtx ();
7084 84807511 : break;
7085 :
7086 7557704 : case 'E':
7087 7557704 : XVEC (x, i) = get_rtvec ();
7088 7557704 : break;
7089 :
7090 0 : case 't':
7091 0 : case 'B':
7092 0 : default:
7093 0 : gcc_unreachable ();
7094 : }
7095 :
7096 : return x;
7097 : }
7098 :
7099 : /* Read an encoded rtvec. */
7100 :
7101 : rtvec
7102 7559630 : rtx_expander::get_rtvec ()
7103 : {
7104 7559630 : unsigned int len = get_uint ();
7105 7559630 : rtvec v = rtvec_alloc (len);
7106 31911594 : for (unsigned int i = 0; i < len; ++i)
7107 16792334 : RTVEC_ELT (v, i) = get_rtx ();
7108 7559630 : return v;
7109 : }
7110 :
7111 : /* Read and emit an encoded sequence of instructions. */
7112 :
7113 : void
7114 8436759 : rtx_expander::expand_seq ()
7115 : {
7116 8436759 : unsigned int len = get_uint ();
7117 17926630 : for (unsigned int i = 0; i < len; ++i)
7118 9489871 : emit (get_rtx (), i < len - 1);
7119 8436759 : }
7120 :
7121 : /* Read an rtx from the bytecode in SEQ, which was generated by genemit.cc.
7122 : Replace operand placeholders with the values given in OPERANDS. */
7123 :
7124 : rtx
7125 17203035 : expand_rtx (const uint8_t *seq, rtx *operands)
7126 : {
7127 17203035 : return rtx_expander (seq, operands).get_rtx ();
7128 : }
7129 :
7130 : /* Read and emit a sequence of instructions from the bytecode in SEQ,
7131 : which was generated by genemit.cc. Replace operand placeholders with
7132 : the values given in OPERANDS. */
7133 :
7134 : rtx_insn *
7135 8436759 : complete_seq (const uint8_t *seq, rtx *operands)
7136 : {
7137 8436759 : rtx_expander (seq, operands).expand_seq ();
7138 8436759 : return end_sequence ();
7139 : }
7140 :
7141 : /* Note in the dump file that WHAT, which names a define_split or a
7142 : define_peephole2 and where it came from, is being applied. genemit.cc
7143 : emits a call to this rather than the test and the fprintf, so that the
7144 : dump is written out once instead of once per pattern. */
7145 :
7146 : void
7147 8594520 : note_split (const char *what)
7148 : {
7149 8594520 : if (dump_file)
7150 117 : fprintf (dump_file, "Splitting with %s\n", what);
7151 8594520 : }
7152 :
7153 : /* Initialize fields of rtl_data related to stack alignment. */
7154 :
7155 : void
7156 1518011 : rtl_data::init_stack_alignment ()
7157 : {
7158 1518011 : stack_alignment_needed = STACK_BOUNDARY;
7159 1518011 : max_used_stack_slot_alignment = STACK_BOUNDARY;
7160 1518011 : stack_alignment_estimated = 0;
7161 1518011 : preferred_stack_boundary = STACK_BOUNDARY;
7162 1518011 : }
7163 :
7164 :
7165 : #include "gt-emit-rtl.h"
|