Line data Source code
1 : /* Code for RTL register eliminations.
2 : Copyright (C) 2010-2026 Free Software Foundation, Inc.
3 : Contributed by Vladimir Makarov <vmakarov@redhat.com>.
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : /* Eliminable registers (like a soft argument or frame pointer) are
22 : widely used in RTL. These eliminable registers should be replaced
23 : by real hard registers (like the stack pointer or hard frame
24 : pointer) plus some offset. The offsets usually change whenever the
25 : stack is expanded. We know the final offsets only at the very end
26 : of LRA.
27 :
28 : Within LRA, we usually keep the RTL in such a state that the
29 : eliminable registers can be replaced by just the corresponding hard
30 : register (without any offset). To achieve this we should add the
31 : initial elimination offset at the beginning of LRA and update the
32 : offsets whenever the stack is expanded. We need to do this before
33 : every constraint pass because the choice of offset often affects
34 : whether a particular address or memory constraint is satisfied.
35 :
36 : We keep RTL code at most time in such state that the virtual
37 : registers can be changed by just the corresponding hard registers
38 : (with zero offsets) and we have the right RTL code. To achieve this
39 : we should add initial offset at the beginning of LRA work and update
40 : offsets after each stack expanding. But actually we update virtual
41 : registers to the same virtual registers + corresponding offsets
42 : before every constraint pass because it affects constraint
43 : satisfaction (e.g. an address displacement became too big for some
44 : target).
45 :
46 : The final change of eliminable registers to the corresponding hard
47 : registers are done at the very end of LRA when there were no change
48 : in offsets anymore:
49 :
50 : fp + 42 => sp + 42
51 :
52 : */
53 :
54 : #include "config.h"
55 : #include "system.h"
56 : #include "coretypes.h"
57 : #include "backend.h"
58 : #include "target.h"
59 : #include "rtl.h"
60 : #include "tree.h"
61 : #include "df.h"
62 : #include "memmodel.h"
63 : #include "tm_p.h"
64 : #include "optabs.h"
65 : #include "regs.h"
66 : #include "ira.h"
67 : #include "recog.h"
68 : #include "output.h"
69 : #include "rtl-error.h"
70 : #include "lra-int.h"
71 :
72 : /* This structure is used to record information about hard register
73 : eliminations. */
74 : class lra_elim_table
75 : {
76 : public:
77 : /* Hard register number to be eliminated. */
78 : int from;
79 : /* Hard register number used as replacement. */
80 : int to;
81 : /* Difference between values of the two hard registers above on
82 : previous iteration. */
83 : poly_int64 previous_offset;
84 : /* Difference between the values on the current iteration. */
85 : poly_int64 offset;
86 : /* Nonzero if this elimination can be done. */
87 : bool can_eliminate;
88 : /* CAN_ELIMINATE since the last check. */
89 : bool prev_can_eliminate;
90 : /* REG rtx for the register to be eliminated. We cannot simply
91 : compare the number since we might then spuriously replace a hard
92 : register corresponding to a pseudo assigned to the reg to be
93 : eliminated. */
94 : rtx from_rtx;
95 : /* REG rtx for the replacement. */
96 : rtx to_rtx;
97 : };
98 :
99 : /* The elimination table. Each array entry describes one possible way
100 : of eliminating a register in favor of another. If there is more
101 : than one way of eliminating a particular register, the most
102 : preferred should be specified first. */
103 : static class lra_elim_table *reg_eliminate = 0;
104 :
105 : /* This is an intermediate structure to initialize the table. It has
106 : exactly the members provided by ELIMINABLE_REGS. */
107 : static const struct elim_table_1
108 : {
109 : const int from;
110 : const int to;
111 : } reg_eliminate_1[] =
112 :
113 : ELIMINABLE_REGS;
114 :
115 : #define NUM_ELIMINABLE_REGS ARRAY_SIZE (reg_eliminate_1)
116 :
117 : /* Print info about elimination table to file F. */
118 : static void
119 194 : print_elim_table (FILE *f)
120 : {
121 194 : class lra_elim_table *ep;
122 :
123 970 : for (ep = reg_eliminate; ep < ®_eliminate[NUM_ELIMINABLE_REGS]; ep++)
124 : {
125 776 : fprintf (f, "%s eliminate %d to %d (offset=",
126 776 : ep->can_eliminate ? "Can" : "Can't", ep->from, ep->to);
127 776 : print_dec (ep->offset, f);
128 776 : fprintf (f, ", prev_offset=");
129 776 : print_dec (ep->previous_offset, f);
130 776 : fprintf (f, ")\n");
131 : }
132 194 : }
133 :
134 : /* Print info about elimination table to stderr. */
135 : void
136 0 : lra_debug_elim_table (void)
137 : {
138 0 : print_elim_table (stderr);
139 0 : }
140 :
141 : /* Setup possibility of elimination in elimination table element EP to
142 : VALUE. Setup FRAME_POINTER_NEEDED if elimination from frame
143 : pointer to stack pointer is not possible anymore. */
144 : static void
145 39132562 : setup_can_eliminate (class lra_elim_table *ep, bool value)
146 : {
147 39132562 : ep->can_eliminate = ep->prev_can_eliminate = value;
148 39132562 : if (! value
149 6570678 : && ep->from == FRAME_POINTER_REGNUM && ep->to == STACK_POINTER_REGNUM)
150 2919106 : frame_pointer_needed = 1;
151 39132562 : if (!frame_pointer_needed)
152 26076879 : REGNO_POINTER_ALIGN (HARD_FRAME_POINTER_REGNUM) = 0;
153 39132562 : }
154 :
155 : /* Map: eliminable "from" register -> its current elimination,
156 : or NULL if none. The elimination table may contain more than
157 : one elimination for the same hard register, but this map specifies
158 : the one that we are currently using. */
159 : static class lra_elim_table *elimination_map[FIRST_PSEUDO_REGISTER];
160 :
161 : /* When an eliminable hard register becomes not eliminable, we use the
162 : following special structure to restore original offsets for the
163 : register. */
164 : static class lra_elim_table self_elim_table;
165 :
166 : /* Offsets should be used to restore original offsets for eliminable
167 : hard register which just became not eliminable. Zero,
168 : otherwise. */
169 : static poly_int64 self_elim_offsets[FIRST_PSEUDO_REGISTER];
170 :
171 : /* Map: hard regno -> RTL presentation. RTL presentations of all
172 : potentially eliminable hard registers are stored in the map. */
173 : static rtx eliminable_reg_rtx[FIRST_PSEUDO_REGISTER];
174 :
175 : /* Set up ELIMINATION_MAP of the currently used eliminations. */
176 : static void
177 9764299 : setup_elimination_map (void)
178 : {
179 9764299 : int i;
180 9764299 : class lra_elim_table *ep;
181 :
182 908079807 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
183 898315508 : elimination_map[i] = NULL;
184 48821495 : for (ep = reg_eliminate; ep < ®_eliminate[NUM_ELIMINABLE_REGS]; ep++)
185 39057196 : if (ep->can_eliminate && elimination_map[ep->from] == NULL)
186 19528598 : elimination_map[ep->from] = ep;
187 9764299 : }
188 :
189 :
190 :
191 : /* Compute the sum of X and Y, making canonicalizations assumed in an
192 : address, namely: sum constant integers, surround the sum of two
193 : constants with a CONST, put the constant as the second operand, and
194 : group the constant on the outermost sum.
195 :
196 : This routine assumes both inputs are already in canonical form. */
197 : static rtx
198 1726862 : form_sum (rtx x, rtx y)
199 : {
200 2087323 : machine_mode mode = GET_MODE (x);
201 2087323 : poly_int64 offset;
202 :
203 2087323 : if (mode == VOIDmode)
204 116523 : mode = GET_MODE (y);
205 :
206 2087323 : if (mode == VOIDmode)
207 0 : mode = Pmode;
208 :
209 2087323 : if (poly_int_rtx_p (x, &offset))
210 116523 : return plus_constant (mode, y, offset);
211 1970800 : else if (poly_int_rtx_p (y, &offset))
212 1091669 : return plus_constant (mode, x, offset);
213 879131 : else if (CONSTANT_P (x))
214 14 : std::swap (x, y);
215 :
216 879131 : if (GET_CODE (x) == PLUS && CONSTANT_P (XEXP (x, 1)))
217 116537 : return form_sum (XEXP (x, 0), form_sum (XEXP (x, 1), y));
218 :
219 : /* Note that if the operands of Y are specified in the opposite
220 : order in the recursive calls below, infinite recursion will
221 : occur. */
222 762594 : if (GET_CODE (y) == PLUS && CONSTANT_P (XEXP (y, 1)))
223 243924 : return form_sum (form_sum (x, XEXP (y, 0)), XEXP (y, 1));
224 :
225 : /* If both constant, encapsulate sum. Otherwise, just form sum. A
226 : constant will have been placed second. */
227 518670 : if (CONSTANT_P (x) && CONSTANT_P (y))
228 : {
229 0 : if (GET_CODE (x) == CONST)
230 0 : x = XEXP (x, 0);
231 0 : if (GET_CODE (y) == CONST)
232 0 : y = XEXP (y, 0);
233 :
234 0 : return gen_rtx_CONST (VOIDmode, gen_rtx_PLUS (mode, x, y));
235 : }
236 :
237 518670 : return gen_rtx_PLUS (mode, x, y);
238 : }
239 :
240 : /* Return the current substitution hard register of the elimination of
241 : HARD_REGNO. If HARD_REGNO is not eliminable, return itself. */
242 : int
243 123618268 : lra_get_elimination_hard_regno (int hard_regno)
244 : {
245 123618268 : class lra_elim_table *ep;
246 :
247 123618268 : if (hard_regno < 0 || hard_regno >= FIRST_PSEUDO_REGISTER)
248 : return hard_regno;
249 123618268 : if ((ep = elimination_map[hard_regno]) == NULL)
250 : return hard_regno;
251 33003256 : return ep->to;
252 : }
253 :
254 : /* Return elimination which will be used for hard reg REG, NULL
255 : otherwise. */
256 : static class lra_elim_table *
257 168404876 : get_elimination (rtx reg)
258 : {
259 168404876 : int hard_regno;
260 168404876 : class lra_elim_table *ep;
261 :
262 168404876 : lra_assert (REG_P (reg));
263 168404876 : if ((hard_regno = REGNO (reg)) < 0 || hard_regno >= FIRST_PSEUDO_REGISTER)
264 : return NULL;
265 140127922 : if ((ep = elimination_map[hard_regno]) != NULL)
266 134247663 : return ep->from_rtx != reg ? NULL : ep;
267 5880259 : poly_int64 offset = self_elim_offsets[hard_regno];
268 5880259 : if (known_eq (offset, 0))
269 : return NULL;
270 : /* This is an iteration to restore offsets just after HARD_REGNO
271 : stopped to be eliminable. */
272 0 : self_elim_table.from = self_elim_table.to = hard_regno;
273 0 : self_elim_table.from_rtx
274 0 : = self_elim_table.to_rtx
275 0 : = eliminable_reg_rtx[hard_regno];
276 0 : lra_assert (self_elim_table.from_rtx != NULL);
277 0 : self_elim_table.offset = offset;
278 0 : return &self_elim_table;
279 : }
280 :
281 : /* Transform (subreg (plus reg const)) to (plus (subreg reg) const)
282 : when it is possible. Return X or the transformation result if the
283 : transformation is done. */
284 : static rtx
285 2914412 : move_plus_up (rtx x)
286 : {
287 2914412 : rtx subreg_reg;
288 2914412 : machine_mode x_mode, subreg_reg_mode;
289 :
290 2914412 : if (GET_CODE (x) != SUBREG || !subreg_lowpart_p (x))
291 2914185 : return x;
292 227 : subreg_reg = SUBREG_REG (x);
293 227 : x_mode = GET_MODE (x);
294 227 : subreg_reg_mode = GET_MODE (subreg_reg);
295 227 : if (!paradoxical_subreg_p (x)
296 227 : && GET_CODE (subreg_reg) == PLUS
297 3 : && CONSTANT_P (XEXP (subreg_reg, 1))
298 3 : && GET_MODE_CLASS (x_mode) == MODE_INT
299 230 : && GET_MODE_CLASS (subreg_reg_mode) == MODE_INT)
300 : {
301 1 : rtx cst = simplify_subreg (x_mode, XEXP (subreg_reg, 1), subreg_reg_mode,
302 : subreg_lowpart_offset (x_mode,
303 : subreg_reg_mode));
304 1 : if (cst && CONSTANT_P (cst))
305 : {
306 1 : rtx lowpart = lowpart_subreg (x_mode, XEXP (subreg_reg, 0),
307 : subreg_reg_mode);
308 1 : if (lowpart)
309 1 : return gen_rtx_PLUS (x_mode, lowpart, cst);
310 : }
311 : }
312 : return x;
313 : }
314 :
315 : /* Flag that we already applied nonzero stack pointer elimination
316 : offset; such sp updates cannot currently be undone. */
317 : static bool elimination_2sp_occurred_p = false;
318 :
319 : /* Take note of any nonzero sp-OFFSET used in eliminations to sp. */
320 : static inline poly_int64
321 13470086 : note_spoff (poly_int64 offset)
322 : {
323 13470086 : if (maybe_ne (offset, 0))
324 1424268 : elimination_2sp_occurred_p = true;
325 13022195 : return offset;
326 : }
327 :
328 : /* Scan X and replace any eliminable registers (such as fp) with a
329 : replacement (such as sp) if SUBST_P, plus an offset. The offset is
330 : a change in the offset between the eliminable register and its
331 : substitution if UPDATE_P, or the full offset if FULL_P, or
332 : otherwise zero. If FULL_P, we also use the SP offsets for
333 : elimination to SP. If UPDATE_P, use UPDATE_SP_OFFSET for updating
334 : offsets of register elimnable to SP. If UPDATE_SP_OFFSET is
335 : non-zero, don't use difference of the offset and the previous
336 : offset.
337 :
338 : MEM_MODE is the mode of an enclosing MEM. We need this to know how
339 : much to adjust a register for, e.g., PRE_DEC. Also, if we are
340 : inside a MEM, we are allowed to replace a sum of a hard register
341 : and the constant zero with the hard register, which we cannot do
342 : outside a MEM. In addition, we need to record the fact that a
343 : hard register is referenced outside a MEM.
344 :
345 : If we make full substitution to SP for non-null INSN, add the insn
346 : sp offset. */
347 : rtx
348 149529289 : lra_eliminate_regs_1 (rtx_insn *insn, rtx x, machine_mode mem_mode,
349 : bool subst_p, bool update_p,
350 : poly_int64 update_sp_offset, bool full_p)
351 : {
352 149529289 : enum rtx_code code = GET_CODE (x);
353 149529289 : class lra_elim_table *ep;
354 149529289 : rtx new_rtx;
355 149529289 : int i, j;
356 149529289 : const char *fmt;
357 149529289 : int copied = 0;
358 :
359 149529289 : lra_assert (!update_p || !full_p);
360 149529289 : lra_assert (known_eq (update_sp_offset, 0)
361 : || (!subst_p && update_p && !full_p));
362 149529289 : if (! current_function_decl)
363 : return x;
364 :
365 149529289 : switch (code)
366 : {
367 : CASE_CONST_ANY:
368 : case CONST:
369 : case SYMBOL_REF:
370 : case CODE_LABEL:
371 : case PC:
372 : case ASM_INPUT:
373 : case ADDR_VEC:
374 : case ADDR_DIFF_VEC:
375 : case RETURN:
376 : return x;
377 :
378 42495427 : case REG:
379 : /* First handle the case where we encounter a bare hard register
380 : that is eliminable. Replace it with a PLUS. */
381 42495427 : if ((ep = get_elimination (x)) != NULL)
382 : {
383 12107735 : rtx to = subst_p ? ep->to_rtx : ep->from_rtx;
384 :
385 12107735 : if (maybe_ne (update_sp_offset, 0))
386 : {
387 0 : if (ep->to_rtx == stack_pointer_rtx)
388 0 : return plus_constant (Pmode, to, note_spoff (update_sp_offset));
389 : return to;
390 : }
391 12107735 : else if (update_p)
392 702387 : return plus_constant (Pmode, to, ep->offset - ep->previous_offset);
393 11476407 : else if (full_p)
394 12482271 : return plus_constant (Pmode, to,
395 10498347 : ep->offset
396 : - (insn != NULL_RTX
397 662542 : && ep->to_rtx == stack_pointer_rtx
398 11160889 : ? note_spoff (lra_get_insn_recog_data
399 447891 : (insn)->sp_offset)
400 : : 0));
401 : else
402 : return to;
403 : }
404 30387692 : return x;
405 :
406 52328280 : case PLUS:
407 : /* If this is the sum of an eliminable register and a constant, rework
408 : the sum. */
409 52328280 : if (REG_P (XEXP (x, 0)) && CONSTANT_P (XEXP (x, 1)))
410 : {
411 50871074 : if ((ep = get_elimination (XEXP (x, 0))) != NULL)
412 : {
413 50764647 : poly_int64 offset, curr_offset;
414 50764647 : rtx to = subst_p ? ep->to_rtx : ep->from_rtx;
415 :
416 50764647 : if (! update_p && ! full_p)
417 23350318 : return simplify_gen_binary (PLUS, Pmode, to, XEXP (x, 1));
418 :
419 30792311 : if (maybe_ne (update_sp_offset, 0))
420 0 : offset = (ep->to_rtx == stack_pointer_rtx
421 0 : ? note_spoff (update_sp_offset)
422 : : 0);
423 : else
424 30792311 : offset = (update_p
425 30792311 : ? ep->offset - ep->previous_offset : ep->offset);
426 30792311 : if (full_p && insn != NULL_RTX && ep->to_rtx == stack_pointer_rtx)
427 26044390 : offset -= note_spoff (lra_get_insn_recog_data (insn)->sp_offset);
428 30792311 : if (poly_int_rtx_p (XEXP (x, 1), &curr_offset)
429 30792311 : && known_eq (curr_offset, -offset))
430 : return to;
431 : else
432 40423531 : return gen_rtx_PLUS (Pmode, to,
433 : plus_constant (Pmode,
434 : XEXP (x, 1), offset));
435 : }
436 :
437 : /* If the hard register is not eliminable, we are done since
438 : the other operand is a constant. */
439 106427 : return x;
440 : }
441 :
442 : /* If this is part of an address, we want to bring any constant
443 : to the outermost PLUS. We will do this by doing hard
444 : register replacement in our operands and seeing if a constant
445 : shows up in one of them.
446 :
447 : Note that there is no risk of modifying the structure of the
448 : insn, since we only get called for its operands, thus we are
449 : either modifying the address inside a MEM, or something like
450 : an address operand of a load-address insn. */
451 :
452 1457206 : {
453 1457206 : rtx new0 = lra_eliminate_regs_1 (insn, XEXP (x, 0), mem_mode,
454 : subst_p, update_p,
455 : update_sp_offset, full_p);
456 1457206 : rtx new1 = lra_eliminate_regs_1 (insn, XEXP (x, 1), mem_mode,
457 : subst_p, update_p,
458 : update_sp_offset, full_p);
459 :
460 1457206 : new0 = move_plus_up (new0);
461 1457206 : new1 = move_plus_up (new1);
462 1457206 : if (new0 != XEXP (x, 0) || new1 != XEXP (x, 1))
463 1366401 : return form_sum (new0, new1);
464 : }
465 : return x;
466 :
467 260062 : case MULT:
468 : /* If this is the product of an eliminable hard register and a
469 : constant, apply the distribute law and move the constant out
470 : so that we have (plus (mult ..) ..). This is needed in order
471 : to keep load-address insns valid. This case is pathological.
472 : We ignore the possibility of overflow here. */
473 241424 : if (REG_P (XEXP (x, 0)) && CONST_INT_P (XEXP (x, 1))
474 499314 : && (ep = get_elimination (XEXP (x, 0))) != NULL)
475 : {
476 0 : rtx to = subst_p ? ep->to_rtx : ep->from_rtx;
477 :
478 0 : if (maybe_ne (update_sp_offset, 0))
479 : {
480 0 : if (ep->to_rtx == stack_pointer_rtx)
481 0 : return plus_constant (Pmode,
482 0 : gen_rtx_MULT (Pmode, to, XEXP (x, 1)),
483 0 : note_spoff (update_sp_offset)
484 0 : * INTVAL (XEXP (x, 1)));
485 0 : return gen_rtx_MULT (Pmode, to, XEXP (x, 1));
486 : }
487 0 : else if (update_p)
488 0 : return plus_constant (Pmode,
489 0 : gen_rtx_MULT (Pmode, to, XEXP (x, 1)),
490 0 : (ep->offset - ep->previous_offset)
491 0 : * INTVAL (XEXP (x, 1)));
492 0 : else if (full_p)
493 : {
494 0 : poly_int64 offset = ep->offset;
495 :
496 0 : if (insn != NULL_RTX && ep->to_rtx == stack_pointer_rtx)
497 0 : offset -= note_spoff (lra_get_insn_recog_data (insn)->sp_offset);
498 0 : return
499 0 : plus_constant (Pmode,
500 0 : gen_rtx_MULT (Pmode, to, XEXP (x, 1)),
501 0 : offset * INTVAL (XEXP (x, 1)));
502 : }
503 : else
504 0 : return gen_rtx_MULT (Pmode, to, XEXP (x, 1));
505 : }
506 :
507 : /* fall through */
508 :
509 397773 : case CALL:
510 397773 : case COMPARE:
511 : /* See comments before PLUS about handling MINUS. */
512 397773 : case MINUS:
513 397773 : case DIV: case UDIV:
514 397773 : case MOD: case UMOD:
515 397773 : case AND: case IOR: case XOR:
516 397773 : case ROTATERT: case ROTATE:
517 397773 : case ASHIFTRT: case LSHIFTRT: case ASHIFT:
518 397773 : case NE: case EQ:
519 397773 : case GE: case GT: case GEU: case GTU:
520 397773 : case LE: case LT: case LEU: case LTU:
521 397773 : {
522 397773 : rtx new0 = lra_eliminate_regs_1 (insn, XEXP (x, 0), mem_mode,
523 : subst_p, update_p,
524 : update_sp_offset, full_p);
525 397773 : rtx new1 = XEXP (x, 1)
526 397773 : ? lra_eliminate_regs_1 (insn, XEXP (x, 1), mem_mode,
527 : subst_p, update_p,
528 : update_sp_offset, full_p) : 0;
529 :
530 397773 : if (new0 != XEXP (x, 0) || new1 != XEXP (x, 1))
531 116256 : return gen_rtx_fmt_ee (code, GET_MODE (x), new0, new1);
532 : }
533 : return x;
534 :
535 55 : case EXPR_LIST:
536 : /* If we have something in XEXP (x, 0), the usual case,
537 : eliminate it. */
538 55 : if (XEXP (x, 0))
539 : {
540 55 : new_rtx = lra_eliminate_regs_1 (insn, XEXP (x, 0), mem_mode,
541 : subst_p, update_p,
542 : update_sp_offset, full_p);
543 55 : if (new_rtx != XEXP (x, 0))
544 : {
545 : /* If this is a REG_DEAD note, it is not valid anymore.
546 : Using the eliminated version could result in creating a
547 : REG_DEAD note for the stack or frame pointer. */
548 0 : if (REG_NOTE_KIND (x) == REG_DEAD)
549 0 : return (XEXP (x, 1)
550 0 : ? lra_eliminate_regs_1 (insn, XEXP (x, 1), mem_mode,
551 : subst_p, update_p,
552 : update_sp_offset, full_p)
553 : : NULL_RTX);
554 :
555 0 : x = alloc_reg_note (REG_NOTE_KIND (x), new_rtx, XEXP (x, 1));
556 : }
557 : }
558 :
559 : /* fall through */
560 :
561 55 : case INSN_LIST:
562 55 : case INT_LIST:
563 : /* Now do eliminations in the rest of the chain. If this was
564 : an EXPR_LIST, this might result in allocating more memory than is
565 : strictly needed, but it simplifies the code. */
566 55 : if (XEXP (x, 1))
567 : {
568 55 : new_rtx = lra_eliminate_regs_1 (insn, XEXP (x, 1), mem_mode,
569 : subst_p, update_p,
570 : update_sp_offset, full_p);
571 55 : if (new_rtx != XEXP (x, 1))
572 0 : return
573 0 : gen_rtx_fmt_ee (GET_CODE (x), GET_MODE (x),
574 : XEXP (x, 0), new_rtx);
575 : }
576 55 : return x;
577 :
578 536032 : case PRE_INC:
579 536032 : case POST_INC:
580 536032 : case PRE_DEC:
581 536032 : case POST_DEC:
582 : /* Recurse to adjust elimination offsets in a spilled pseudo. */
583 536032 : if (GET_CODE (XEXP (x, 0)) == MEM)
584 : break;
585 : /* We do not support elimination of a register that is modified.
586 : elimination_effects has already make sure that this does not
587 : happen. */
588 : return x;
589 :
590 186 : case PRE_MODIFY:
591 186 : case POST_MODIFY:
592 : /* Recurse to adjust elimination offsets in a spilled pseudo. */
593 186 : if (GET_CODE (XEXP (x, 0)) == MEM)
594 : break;
595 : /* We do not support elimination of a hard register that is
596 : modified. LRA has already make sure that this does not
597 : happen. The only remaining case we need to consider here is
598 : that the increment value may be an eliminable register. */
599 186 : if (GET_CODE (XEXP (x, 1)) == PLUS
600 186 : && XEXP (XEXP (x, 1), 0) == XEXP (x, 0))
601 : {
602 186 : rtx new_rtx = lra_eliminate_regs_1 (insn, XEXP (XEXP (x, 1), 1),
603 : mem_mode, subst_p, update_p,
604 : update_sp_offset, full_p);
605 :
606 186 : if (new_rtx != XEXP (XEXP (x, 1), 1))
607 0 : return gen_rtx_fmt_ee (code, GET_MODE (x), XEXP (x, 0),
608 : gen_rtx_PLUS (GET_MODE (x),
609 : XEXP (x, 0), new_rtx));
610 : }
611 186 : return x;
612 :
613 36631 : case STRICT_LOW_PART:
614 36631 : case NEG: case NOT:
615 36631 : case SIGN_EXTEND: case ZERO_EXTEND:
616 36631 : case TRUNCATE: case FLOAT_EXTEND: case FLOAT_TRUNCATE:
617 36631 : case FLOAT: case FIX:
618 36631 : case UNSIGNED_FIX: case UNSIGNED_FLOAT:
619 36631 : case ABS:
620 36631 : case SQRT:
621 36631 : case FFS:
622 36631 : case CLZ:
623 36631 : case CTZ:
624 36631 : case POPCOUNT:
625 36631 : case PARITY:
626 36631 : case BSWAP:
627 36631 : new_rtx = lra_eliminate_regs_1 (insn, XEXP (x, 0), mem_mode,
628 : subst_p, update_p,
629 : update_sp_offset, full_p);
630 36631 : if (new_rtx != XEXP (x, 0))
631 26745 : return gen_rtx_fmt_e (code, GET_MODE (x), new_rtx);
632 : return x;
633 :
634 615079 : case SUBREG:
635 615079 : new_rtx = lra_eliminate_regs_1 (insn, SUBREG_REG (x), mem_mode,
636 : subst_p, update_p,
637 : update_sp_offset, full_p);
638 :
639 615079 : if (new_rtx != SUBREG_REG (x))
640 : {
641 4674 : if (MEM_P (new_rtx) && !paradoxical_subreg_p (x))
642 : {
643 11 : SUBREG_REG (x) = new_rtx;
644 11 : alter_subreg (&x, false);
645 11 : return x;
646 : }
647 4663 : else if (! subst_p)
648 : {
649 : /* LRA can transform subregs itself. So don't call
650 : simplify_gen_subreg until LRA transformations are
651 : finished. Function simplify_gen_subreg can do
652 : non-trivial transformations (like truncation) which
653 : might make LRA work to fail. */
654 2805 : SUBREG_REG (x) = new_rtx;
655 2805 : return x;
656 : }
657 : else
658 : {
659 3716 : rtx nx = simplify_gen_subreg (GET_MODE (x), new_rtx,
660 1858 : GET_MODE (new_rtx), SUBREG_BYTE (x));
661 : /* If inside a debug insn, then generate the subreg manually as it might
662 : be an invalid one for outside of a debug insn. */
663 1858 : if (DEBUG_INSN_P (insn) && !nx)
664 1 : nx = gen_rtx_raw_SUBREG (GET_MODE (x), new_rtx, SUBREG_BYTE (x));
665 7 : gcc_assert (nx);
666 1858 : return nx;
667 : }
668 : }
669 :
670 : return x;
671 :
672 45102561 : case MEM:
673 : /* Our only special processing is to pass the mode of the MEM to our
674 : recursive call and copy the flags. While we are here, handle this
675 : case more efficiently. */
676 45102561 : return
677 : replace_equiv_address_nv
678 45102561 : (x,
679 45102561 : lra_eliminate_regs_1 (insn, XEXP (x, 0), GET_MODE (x),
680 45102561 : subst_p, update_p, update_sp_offset, full_p));
681 :
682 0 : case USE:
683 : /* Handle insn_list USE that a call to a pure function may generate. */
684 0 : new_rtx = lra_eliminate_regs_1 (insn, XEXP (x, 0), VOIDmode,
685 : subst_p, update_p, update_sp_offset, full_p);
686 0 : if (new_rtx != XEXP (x, 0))
687 0 : return gen_rtx_USE (GET_MODE (x), new_rtx);
688 : return x;
689 :
690 0 : case CLOBBER:
691 0 : case ASM_OPERANDS:
692 0 : gcc_assert (insn && DEBUG_INSN_P (insn));
693 : break;
694 :
695 0 : case SET:
696 0 : gcc_unreachable ();
697 :
698 : default:
699 : break;
700 : }
701 :
702 : /* Process each of our operands recursively. If any have changed, make a
703 : copy of the rtx. */
704 76810 : fmt = GET_RTX_FORMAT (code);
705 168158 : for (i = 0; i < GET_RTX_LENGTH (code); i++, fmt++)
706 : {
707 91348 : if (*fmt == 'e')
708 : {
709 27730 : new_rtx = lra_eliminate_regs_1 (insn, XEXP (x, i), mem_mode,
710 : subst_p, update_p,
711 : update_sp_offset, full_p);
712 27730 : if (new_rtx != XEXP (x, i) && ! copied)
713 : {
714 12963 : x = shallow_copy_rtx (x);
715 12963 : copied = 1;
716 : }
717 27730 : XEXP (x, i) = new_rtx;
718 : }
719 63618 : else if (*fmt == 'E')
720 : {
721 : int copied_vec = 0;
722 41763 : for (j = 0; j < XVECLEN (x, i); j++)
723 : {
724 28920 : new_rtx = lra_eliminate_regs_1 (insn, XVECEXP (x, i, j), mem_mode,
725 : subst_p, update_p,
726 : update_sp_offset, full_p);
727 28920 : if (new_rtx != XVECEXP (x, i, j) && ! copied_vec)
728 : {
729 23196 : rtvec new_v = gen_rtvec_v (XVECLEN (x, i),
730 11598 : XVEC (x, i)->elem);
731 11598 : if (! copied)
732 : {
733 11598 : x = shallow_copy_rtx (x);
734 11598 : copied = 1;
735 : }
736 11598 : XVEC (x, i) = new_v;
737 11598 : copied_vec = 1;
738 : }
739 28920 : XVECEXP (x, i, j) = new_rtx;
740 : }
741 : }
742 : }
743 :
744 76810 : return x;
745 : }
746 :
747 : /* This function is used externally in subsequent passes of GCC. It
748 : always does a full elimination of X. */
749 : rtx
750 9835805 : lra_eliminate_regs (rtx x, machine_mode mem_mode,
751 : rtx insn ATTRIBUTE_UNUSED)
752 : {
753 9835805 : return lra_eliminate_regs_1 (NULL, x, mem_mode, true, false, 0, true);
754 : }
755 :
756 : /* Stack pointer offset before the current insn relative to one at the
757 : func start. RTL insns can change SP explicitly. We keep the
758 : changes from one insn to another through this variable. */
759 : static poly_int64 curr_sp_change;
760 :
761 : /* Scan rtx X for references to elimination source or target registers
762 : in contexts that would prevent the elimination from happening.
763 : Update the table of eliminables to reflect the changed state.
764 : MEM_MODE is the mode of an enclosing MEM rtx, or VOIDmode if not
765 : within a MEM. */
766 : static void
767 362601699 : mark_not_eliminable (rtx x, machine_mode mem_mode)
768 : {
769 498484233 : enum rtx_code code = GET_CODE (x);
770 498484233 : class lra_elim_table *ep;
771 498484233 : int i, j;
772 498484233 : const char *fmt;
773 498484233 : poly_int64 offset = 0;
774 :
775 498484233 : switch (code)
776 : {
777 1991917 : case PRE_INC:
778 1991917 : case POST_INC:
779 1991917 : case PRE_DEC:
780 1991917 : case POST_DEC:
781 1991917 : case POST_MODIFY:
782 1991917 : case PRE_MODIFY:
783 1991917 : if (XEXP (x, 0) == stack_pointer_rtx
784 1991917 : && ((code != PRE_MODIFY && code != POST_MODIFY)
785 108550 : || (GET_CODE (XEXP (x, 1)) == PLUS
786 108550 : && XEXP (x, 0) == XEXP (XEXP (x, 1), 0)
787 108550 : && poly_int_rtx_p (XEXP (XEXP (x, 1), 1), &offset))))
788 : {
789 3983834 : poly_int64 size = GET_MODE_SIZE (mem_mode);
790 :
791 : #ifdef PUSH_ROUNDING
792 : /* If more bytes than MEM_MODE are pushed, account for
793 : them. */
794 1991917 : size = PUSH_ROUNDING (size);
795 : #endif
796 1991917 : if (code == PRE_DEC || code == POST_DEC)
797 362601699 : curr_sp_change -= size;
798 108592 : else if (code == PRE_INC || code == POST_INC)
799 362601699 : curr_sp_change += size;
800 108550 : else if (code == PRE_MODIFY || code == POST_MODIFY)
801 362601699 : curr_sp_change += offset;
802 : }
803 0 : else if (REG_P (XEXP (x, 0))
804 0 : && REGNO (XEXP (x, 0)) < FIRST_PSEUDO_REGISTER)
805 : {
806 : /* If we modify the source of an elimination rule, disable
807 : it. Do the same if it is the destination and not the
808 : hard frame register. */
809 0 : for (ep = reg_eliminate;
810 0 : ep < ®_eliminate[NUM_ELIMINABLE_REGS];
811 : ep++)
812 0 : if (ep->from_rtx == XEXP (x, 0)
813 0 : || (ep->to_rtx == XEXP (x, 0)
814 0 : && ep->to_rtx != hard_frame_pointer_rtx))
815 0 : setup_can_eliminate (ep, false);
816 : }
817 : return;
818 :
819 886969 : case USE:
820 886969 : if (REG_P (XEXP (x, 0)) && REGNO (XEXP (x, 0)) < FIRST_PSEUDO_REGISTER)
821 : /* If using a hard register that is the source of an eliminate
822 : we still think can be performed, note it cannot be
823 : performed since we don't know how this hard register is
824 : used. */
825 3850400 : for (ep = reg_eliminate;
826 3850400 : ep < ®_eliminate[NUM_ELIMINABLE_REGS];
827 : ep++)
828 3080320 : if (ep->from_rtx == XEXP (x, 0)
829 0 : && ep->to_rtx != hard_frame_pointer_rtx)
830 0 : setup_can_eliminate (ep, false);
831 : return;
832 :
833 13130567 : case CLOBBER:
834 13130567 : if (REG_P (XEXP (x, 0)) && REGNO (XEXP (x, 0)) < FIRST_PSEUDO_REGISTER)
835 : /* If clobbering a hard register that is the replacement
836 : register for an elimination we still think can be
837 : performed, note that it cannot be performed. Otherwise, we
838 : need not be concerned about it. */
839 63928330 : for (ep = reg_eliminate;
840 63928330 : ep < ®_eliminate[NUM_ELIMINABLE_REGS];
841 : ep++)
842 51142664 : if (ep->to_rtx == XEXP (x, 0)
843 4390 : && ep->to_rtx != hard_frame_pointer_rtx)
844 2 : setup_can_eliminate (ep, false);
845 : return;
846 :
847 100233915 : case SET:
848 100233915 : if (SET_DEST (x) == stack_pointer_rtx
849 2196292 : && GET_CODE (SET_SRC (x)) == PLUS
850 2168059 : && XEXP (SET_SRC (x), 0) == SET_DEST (x)
851 102401906 : && poly_int_rtx_p (XEXP (SET_SRC (x), 1), &offset))
852 : {
853 2167991 : curr_sp_change += offset;
854 2167991 : return;
855 : }
856 98065924 : if (! REG_P (SET_DEST (x))
857 98065924 : || REGNO (SET_DEST (x)) >= FIRST_PSEUDO_REGISTER)
858 75854212 : mark_not_eliminable (SET_DEST (x), mem_mode);
859 : else
860 : {
861 : /* See if this is setting the replacement hard register for
862 : an elimination.
863 :
864 : If DEST is the hard frame pointer, we do nothing because
865 : we assume that all assignments to the frame pointer are
866 : for non-local gotos and are being done at a time when
867 : they are valid and do not disturb anything else. Some
868 : machines want to eliminate a fake argument pointer (or
869 : even a fake frame pointer) with either the real frame
870 : pointer or the stack pointer. Assignments to the hard
871 : frame pointer must not prevent this elimination. */
872 111058560 : for (ep = reg_eliminate;
873 111058560 : ep < ®_eliminate[NUM_ELIMINABLE_REGS];
874 : ep++)
875 88846848 : if (ep->to_rtx == SET_DEST (x)
876 58408 : && SET_DEST (x) != hard_frame_pointer_rtx)
877 56602 : setup_can_eliminate (ep, false);
878 : }
879 :
880 98065924 : mark_not_eliminable (SET_SRC (x), mem_mode);
881 98065924 : return;
882 :
883 37816610 : case MEM:
884 : /* Our only special processing is to pass the mode of the MEM to
885 : our recursive call. */
886 37816610 : mark_not_eliminable (XEXP (x, 0), GET_MODE (x));
887 37816610 : return;
888 :
889 344424255 : default:
890 344424255 : break;
891 : }
892 :
893 344424255 : fmt = GET_RTX_FORMAT (code);
894 770920079 : for (i = 0; i < GET_RTX_LENGTH (code); i++, fmt++)
895 : {
896 426495824 : if (*fmt == 'e')
897 149302083 : mark_not_eliminable (XEXP (x, i), mem_mode);
898 277193741 : else if (*fmt == 'E')
899 48482722 : for (j = 0; j < XVECLEN (x, i); j++)
900 32541120 : mark_not_eliminable (XVECEXP (x, i, j), mem_mode);
901 : }
902 : }
903 :
904 :
905 :
906 : /* Scan INSN and eliminate all eliminable hard registers in it.
907 :
908 : If REPLACE_P is true, do the replacement destructively. Also
909 : delete the insn as dead it if it is setting an eliminable register.
910 :
911 : If REPLACE_P is false, just update the offsets while keeping the
912 : base register the same. If FIRST_P, use the sp offset for
913 : elimination to sp. Otherwise, use UPDATE_SP_OFFSET for this. If
914 : UPDATE_SP_OFFSET is non-zero, don't use difference of the offset
915 : and the previous offset. Attach the note about used elimination
916 : for insns setting frame pointer to update elimination easy (without
917 : parsing already generated elimination insns to find offset
918 : previously used) in future. */
919 :
920 : void
921 50544912 : eliminate_regs_in_insn (rtx_insn *insn, bool replace_p, bool first_p,
922 : poly_int64 update_sp_offset)
923 : {
924 50544912 : int icode = recog_memoized (insn);
925 50544912 : rtx set, old_set = single_set (insn);
926 50544912 : bool validate_p;
927 50544912 : int i;
928 50544912 : rtx substed_operand[MAX_RECOG_OPERANDS];
929 50544912 : rtx orig_operand[MAX_RECOG_OPERANDS];
930 50544912 : class lra_elim_table *ep;
931 50544912 : rtx plus_src, plus_cst_src;
932 50544912 : lra_insn_recog_data_t id;
933 50544912 : struct lra_static_insn_data *static_id;
934 :
935 50544912 : if (icode < 0 && asm_noperands (PATTERN (insn)) < 0 && ! DEBUG_INSN_P (insn))
936 : {
937 11848 : lra_assert (GET_CODE (PATTERN (insn)) == USE
938 : || GET_CODE (PATTERN (insn)) == CLOBBER
939 : || GET_CODE (PATTERN (insn)) == ASM_INPUT);
940 7430527 : return;
941 : }
942 :
943 : /* We allow one special case which happens to work on all machines we
944 : currently support: a single set with the source or a REG_EQUAL
945 : note being a PLUS of an eliminable register and a constant. */
946 50533064 : plus_src = plus_cst_src = 0;
947 50533064 : poly_int64 offset = 0;
948 50533064 : if (old_set && REG_P (SET_DEST (old_set)))
949 : {
950 20879648 : if (GET_CODE (SET_SRC (old_set)) == PLUS)
951 4573154 : plus_src = SET_SRC (old_set);
952 : /* First see if the source is of the form (plus (...) CST). */
953 4573154 : if (plus_src && poly_int_rtx_p (XEXP (plus_src, 1), &offset))
954 : plus_cst_src = plus_src;
955 : /* If we are doing initial offset computation, then utilize
956 : eqivalences to discover a constant for the second term
957 : of PLUS_SRC. */
958 16626467 : else if (plus_src && REG_P (XEXP (plus_src, 1)))
959 : {
960 32401 : int regno = REGNO (XEXP (plus_src, 1));
961 32401 : if (regno < ira_reg_equiv_len
962 32401 : && ira_reg_equiv[regno].constant != NULL_RTX
963 2826 : && !replace_p
964 35191 : && poly_int_rtx_p (ira_reg_equiv[regno].constant, &offset))
965 98 : plus_cst_src = plus_src;
966 : }
967 : /* Check that the first operand of the PLUS is a hard reg or
968 : the lowpart subreg of one. */
969 4253279 : if (plus_cst_src)
970 : {
971 4253279 : rtx reg = XEXP (plus_cst_src, 0);
972 :
973 4253279 : if (GET_CODE (reg) == SUBREG && subreg_lowpart_p (reg))
974 83 : reg = SUBREG_REG (reg);
975 :
976 4253279 : if (!REG_P (reg) || REGNO (reg) >= FIRST_PSEUDO_REGISTER)
977 : plus_cst_src = 0;
978 : }
979 : }
980 4227826 : if (plus_cst_src)
981 : {
982 4227826 : rtx reg = XEXP (plus_cst_src, 0);
983 :
984 4227826 : if (GET_CODE (reg) == SUBREG)
985 83 : reg = SUBREG_REG (reg);
986 :
987 4227826 : if (REG_P (reg) && (ep = get_elimination (reg)) != NULL)
988 : {
989 4227826 : rtx to_rtx = replace_p ? ep->to_rtx : ep->from_rtx;
990 :
991 1917255 : if (! replace_p)
992 : {
993 2310571 : if (known_eq (update_sp_offset, 0))
994 2310571 : offset += (!first_p
995 2310571 : ? ep->offset - ep->previous_offset : ep->offset);
996 2310571 : if (ep->to_rtx == stack_pointer_rtx)
997 : {
998 1770515 : if (first_p)
999 921968 : offset -= lra_get_insn_recog_data (insn)->sp_offset;
1000 : else
1001 2310571 : offset += update_sp_offset;
1002 : }
1003 2310571 : offset = trunc_int_for_mode (offset, GET_MODE (plus_cst_src));
1004 : }
1005 :
1006 4227826 : if (GET_CODE (XEXP (plus_cst_src, 0)) == SUBREG)
1007 83 : to_rtx = gen_lowpart (GET_MODE (XEXP (plus_cst_src, 0)), to_rtx);
1008 : /* If we have a nonzero offset, and the source is already a
1009 : simple REG, the following transformation would increase
1010 : the cost of the insn by replacing a simple REG with (plus
1011 : (reg sp) CST). So try only when we already had a PLUS
1012 : before. */
1013 4227826 : if (known_eq (offset, 0) || plus_src)
1014 : {
1015 4227826 : rtx new_src = plus_constant (GET_MODE (to_rtx), to_rtx, offset);
1016 :
1017 4227826 : old_set = single_set (insn);
1018 :
1019 : /* First see if this insn remains valid when we make the
1020 : change. If not, try to replace the whole pattern
1021 : with a simple set (this may help if the original insn
1022 : was a PARALLEL that was only recognized as single_set
1023 : due to REG_UNUSED notes). If this isn't valid
1024 : either, keep the INSN_CODE the same and let the
1025 : constraint pass fix it up. */
1026 4227826 : if (! validate_change (insn, &SET_SRC (old_set), new_src, 0))
1027 : {
1028 11 : rtx new_pat = gen_rtx_SET (SET_DEST (old_set), new_src);
1029 :
1030 11 : if (! validate_change (insn, &PATTERN (insn), new_pat, 0))
1031 11 : SET_SRC (old_set) = new_src;
1032 : }
1033 4227826 : lra_update_insn_recog_data (insn);
1034 : /* This can't have an effect on elimination offsets, so skip
1035 : right to the end. */
1036 4227826 : return;
1037 : }
1038 : }
1039 : }
1040 :
1041 : /* Eliminate all eliminable registers occurring in operands that
1042 : can be handled by the constraint pass. */
1043 46305238 : id = lra_get_insn_recog_data (insn);
1044 46305238 : static_id = id->insn_static_data;
1045 46305238 : validate_p = false;
1046 130208433 : for (i = 0; i < static_id->n_operands; i++)
1047 : {
1048 83903195 : orig_operand[i] = *id->operand_loc[i];
1049 83903195 : substed_operand[i] = *id->operand_loc[i];
1050 :
1051 : /* For an asm statement, every operand is eliminable. */
1052 83903195 : if (icode < 0 || insn_data[icode].operand[i].eliminable)
1053 : {
1054 : /* Check for setting a hard register that we know about. */
1055 83604466 : if (static_id->operand[i].type != OP_IN
1056 34701727 : && REG_P (orig_operand[i]))
1057 : {
1058 : /* If we are assigning to a hard register that can be
1059 : eliminated, it must be as part of a PARALLEL, since
1060 : the code above handles single SETs. This reg cannot
1061 : be longer eliminated -- it is forced by
1062 : mark_not_eliminable. */
1063 78196655 : for (ep = reg_eliminate;
1064 78196655 : ep < ®_eliminate[NUM_ELIMINABLE_REGS];
1065 : ep++)
1066 62557324 : lra_assert (ep->from_rtx != orig_operand[i]
1067 : || ! ep->can_eliminate);
1068 : }
1069 :
1070 : /* Companion to the above plus substitution, we can allow
1071 : invariants as the source of a plain move. */
1072 83604466 : substed_operand[i]
1073 167208932 : = lra_eliminate_regs_1 (insn, *id->operand_loc[i], VOIDmode,
1074 83604466 : replace_p, ! replace_p && ! first_p,
1075 : update_sp_offset, first_p);
1076 83604466 : if (substed_operand[i] != orig_operand[i])
1077 83903195 : validate_p = true;
1078 : }
1079 : }
1080 :
1081 46305238 : if (! validate_p)
1082 : return;
1083 :
1084 : /* Substitute the operands; the new values are in the substed_operand
1085 : array. */
1086 120415836 : for (i = 0; i < static_id->n_operands; i++)
1087 77301451 : *id->operand_loc[i] = substed_operand[i];
1088 43213890 : for (i = 0; i < static_id->n_dups; i++)
1089 99505 : *id->dup_loc[i] = substed_operand[(int) static_id->dup_num[i]];
1090 :
1091 : /* Transform plus (plus (hard reg, const), pseudo) to plus (plus (pseudo,
1092 : const), hard reg) in order to keep insn containing eliminated register
1093 : after all reloads calculating its offset. This permits to keep register
1094 : pressure under control and helps to avoid LRA cycling in patalogical
1095 : cases. */
1096 22412696 : if (! replace_p && (set = single_set (insn)) != NULL
1097 15900491 : && GET_CODE (SET_SRC (set)) == PLUS
1098 43476542 : && GET_CODE (XEXP (SET_SRC (set), 0)) == PLUS)
1099 : {
1100 24400 : rtx reg1, reg2, op1, op2;
1101 :
1102 24400 : reg1 = op1 = XEXP (XEXP (SET_SRC (set), 0), 0);
1103 24400 : reg2 = op2 = XEXP (SET_SRC (set), 1);
1104 24400 : if (GET_CODE (reg1) == SUBREG)
1105 0 : reg1 = SUBREG_REG (reg1);
1106 24400 : if (GET_CODE (reg2) == SUBREG)
1107 0 : reg2 = SUBREG_REG (reg2);
1108 20171 : if (REG_P (reg1) && REG_P (reg2)
1109 8723 : && REGNO (reg1) < FIRST_PSEUDO_REGISTER
1110 8723 : && REGNO (reg2) >= FIRST_PSEUDO_REGISTER
1111 8760 : && GET_MODE (reg1) == Pmode
1112 33123 : && !have_addptr3_insn (lra_pmode_pseudo, reg1,
1113 : XEXP (XEXP (SET_SRC (set), 0), 1)))
1114 : {
1115 8723 : XEXP (XEXP (SET_SRC (set), 0), 0) = op2;
1116 8723 : XEXP (SET_SRC (set), 1) = op1;
1117 : }
1118 : }
1119 :
1120 : /* If we had a move insn but now we don't, re-recognize it.
1121 : This will cause spurious re-recognition if the old move had a
1122 : PARALLEL since the new one still will, but we can't call
1123 : single_set without having put new body into the insn and the
1124 : re-recognition won't hurt in this rare case. */
1125 43114385 : lra_update_insn_recog_data (insn);
1126 : }
1127 :
1128 : /* Spill pseudos which are assigned to hard registers in SET, record them in
1129 : SPILLED_PSEUDOS unless it is null, and return the recorded pseudos number.
1130 : Add affected insns for processing in the subsequent constraint pass. */
1131 : static int
1132 8259349 : spill_pseudos (HARD_REG_SET set, int *spilled_pseudos)
1133 : {
1134 8259349 : int i, n;
1135 8259349 : unsigned int j;
1136 8259349 : bitmap_head to_process;
1137 8259349 : rtx_insn *insn;
1138 :
1139 16518698 : if (hard_reg_set_empty_p (set))
1140 : return 0;
1141 8259349 : if (lra_dump_file != NULL)
1142 : {
1143 485 : fprintf (lra_dump_file, " Spilling non-eliminable hard regs:");
1144 485 : j = 0;
1145 485 : hard_reg_set_iterator hrsi;
1146 970 : EXECUTE_IF_SET_IN_HARD_REG_SET (set, 0, j, hrsi)
1147 485 : fprintf (lra_dump_file, " %d", j);
1148 485 : fprintf (lra_dump_file, "\n");
1149 : }
1150 8259349 : bitmap_initialize (&to_process, ®_obstack);
1151 8259349 : n = 0;
1152 541079217 : for (i = FIRST_PSEUDO_REGISTER; i < max_reg_num (); i++)
1153 254253847 : if (lra_reg_info[i].nrefs != 0 && reg_renumber[i] >= 0
1154 532819868 : && overlaps_hard_reg_set_p (set,
1155 241334147 : PSEUDO_REGNO_MODE (i), reg_renumber[i]))
1156 : {
1157 26494 : if (lra_dump_file != NULL)
1158 0 : fprintf (lra_dump_file, " Spilling r%d(%d)\n",
1159 : i, reg_renumber[i]);
1160 26494 : reg_renumber[i] = -1;
1161 26494 : if (spilled_pseudos != NULL)
1162 0 : spilled_pseudos[n++] = i;
1163 26494 : bitmap_ior_into (&to_process, &lra_reg_info[i].insn_bitmap);
1164 : }
1165 8259349 : lra_no_alloc_regs |= set;
1166 1346350971 : for (insn = get_insns (); insn != NULL_RTX; insn = NEXT_INSN (insn))
1167 1338091622 : if (bitmap_bit_p (&to_process, INSN_UID (insn)))
1168 : {
1169 173068 : lra_push_insn (insn);
1170 173068 : lra_set_used_insn_alternative (insn, LRA_UNKNOWN_ALT);
1171 : }
1172 8259349 : bitmap_clear (&to_process);
1173 8259349 : return n;
1174 : }
1175 :
1176 : /* Update all offsets and possibility for elimination on eliminable
1177 : registers. Spill pseudos assigned to registers which are
1178 : uneliminable, update LRA_NO_ALLOC_REGS and ELIMINABLE_REG_SET. Add
1179 : insns to INSNS_WITH_CHANGED_OFFSETS containing eliminable hard
1180 : registers whose offsets should be changed. Return true if any
1181 : elimination offset changed. */
1182 : static bool
1183 8259349 : update_reg_eliminate (bitmap insns_with_changed_offsets)
1184 : {
1185 8259349 : bool prev, result;
1186 8259349 : class lra_elim_table *ep, *ep1;
1187 8259349 : HARD_REG_SET temp_hard_reg_set;
1188 :
1189 8259349 : targetm.compute_frame_layout ();
1190 :
1191 : /* Clear self elimination offsets. */
1192 41296745 : for (ep = reg_eliminate; ep < ®_eliminate[NUM_ELIMINABLE_REGS]; ep++)
1193 33037396 : self_elim_offsets[ep->from] = 0;
1194 41296745 : for (ep = reg_eliminate; ep < ®_eliminate[NUM_ELIMINABLE_REGS]; ep++)
1195 : {
1196 : /* If it is a currently used elimination: update the previous
1197 : offset. */
1198 33037396 : if (elimination_map[ep->from] == ep)
1199 : {
1200 16518698 : ep->previous_offset = ep->offset;
1201 : /* Restore the stack_pointer_rtx into to_rtx, that
1202 : lra_update_fp2sp_elimination set to from_rtx, so that the assert
1203 : below still checks what it was supposed to check. */
1204 16518698 : if (ep->from_rtx == ep->to_rtx
1205 0 : && ep->from != ep->to
1206 0 : && ep->from == FRAME_POINTER_REGNUM)
1207 0 : ep->to_rtx = stack_pointer_rtx;
1208 : }
1209 :
1210 33037396 : prev = ep->prev_can_eliminate;
1211 33037396 : setup_can_eliminate (ep, targetm.can_eliminate (ep->from, ep->to));
1212 33037396 : if (ep->can_eliminate && ! prev)
1213 : {
1214 : /* It is possible that not eliminable register becomes
1215 : eliminable because we took other reasons into account to
1216 : set up eliminable regs in the initial set up. Just
1217 : ignore new eliminable registers. */
1218 0 : setup_can_eliminate (ep, false);
1219 0 : continue;
1220 : }
1221 33037396 : if (!ep->can_eliminate && elimination_map[ep->from] == ep)
1222 : {
1223 : /* We cannot use this elimination anymore -- find another
1224 : one. */
1225 0 : if (lra_dump_file != NULL)
1226 0 : fprintf (lra_dump_file,
1227 : " Elimination %d to %d is not possible anymore\n",
1228 : ep->from, ep->to);
1229 : /* If after processing RTL we decides that SP can be used as a result
1230 : of elimination, it cannot be changed. For frame pointer to stack
1231 : pointer elimination the condition is a bit relaxed and we just require
1232 : that actual elimination has not been done yet. */
1233 0 : gcc_assert (ep->to_rtx != stack_pointer_rtx
1234 : || !elimination_2sp_occurred_p
1235 : || (ep->from < FIRST_PSEUDO_REGISTER
1236 : && fixed_regs [ep->from]));
1237 :
1238 : /* Mark that is not eliminable anymore. */
1239 0 : elimination_map[ep->from] = NULL;
1240 0 : for (ep1 = ep + 1; ep1 < ®_eliminate[NUM_ELIMINABLE_REGS]; ep1++)
1241 0 : if (ep1->can_eliminate && ep1->from == ep->from)
1242 : break;
1243 0 : if (ep1 < ®_eliminate[NUM_ELIMINABLE_REGS])
1244 : {
1245 0 : if (lra_dump_file != NULL)
1246 0 : fprintf (lra_dump_file, " Using elimination %d to %d now\n",
1247 : ep1->from, ep1->to);
1248 0 : lra_assert (known_eq (ep1->previous_offset, -1));
1249 0 : ep1->previous_offset = ep->offset;
1250 : }
1251 : else
1252 : {
1253 : /* There is no elimination anymore just use the hard
1254 : register `from' itself. Setup self elimination
1255 : offset to restore the original offset values. */
1256 0 : if (lra_dump_file != NULL)
1257 0 : fprintf (lra_dump_file, " %d is not eliminable at all\n",
1258 : ep->from);
1259 0 : self_elim_offsets[ep->from] = -ep->offset;
1260 0 : if (maybe_ne (ep->offset, 0))
1261 0 : bitmap_ior_into (insns_with_changed_offsets,
1262 0 : &lra_reg_info[ep->from].insn_bitmap);
1263 : }
1264 : }
1265 :
1266 33037396 : INITIAL_ELIMINATION_OFFSET (ep->from, ep->to, ep->offset);
1267 : }
1268 8259349 : setup_elimination_map ();
1269 8259349 : result = false;
1270 8259349 : CLEAR_HARD_REG_SET (temp_hard_reg_set);
1271 41296745 : for (ep = reg_eliminate; ep < ®_eliminate[NUM_ELIMINABLE_REGS]; ep++)
1272 33037396 : if (elimination_map[ep->from] == NULL)
1273 0 : add_to_hard_reg_set (&temp_hard_reg_set, Pmode, ep->from);
1274 33037396 : else if (elimination_map[ep->from] == ep)
1275 : {
1276 : /* Prevent the hard register into which we eliminate from
1277 : the usage for pseudos. */
1278 16518698 : if (ep->from != ep->to)
1279 18126722 : add_to_hard_reg_set (&temp_hard_reg_set, Pmode, ep->to);
1280 16518698 : if (maybe_ne (ep->previous_offset, ep->offset))
1281 : {
1282 3375748 : bitmap_ior_into (insns_with_changed_offsets,
1283 3375748 : &lra_reg_info[ep->from].insn_bitmap);
1284 :
1285 : /* Update offset when the eliminate offset have been
1286 : changed. */
1287 3375748 : lra_update_reg_val_offset (lra_reg_info[ep->from].val,
1288 3375748 : ep->offset - ep->previous_offset);
1289 3375748 : result = true;
1290 : }
1291 : }
1292 8259349 : lra_no_alloc_regs |= temp_hard_reg_set;
1293 8259349 : eliminable_regset &= ~temp_hard_reg_set;
1294 8259349 : spill_pseudos (temp_hard_reg_set, NULL);
1295 8259349 : return result;
1296 : }
1297 :
1298 : /* Initialize the table of hard registers to eliminate.
1299 : Pre-condition: global flag frame_pointer_needed has been set before
1300 : calling this function. */
1301 : static void
1302 1504950 : init_elim_table (void)
1303 : {
1304 1504950 : class lra_elim_table *ep;
1305 1504950 : bool value_p;
1306 1504950 : const struct elim_table_1 *ep1;
1307 :
1308 1504950 : if (!reg_eliminate)
1309 213309 : reg_eliminate = XCNEWVEC (class lra_elim_table, NUM_ELIMINABLE_REGS);
1310 :
1311 1504950 : memset (self_elim_offsets, 0, sizeof (self_elim_offsets));
1312 : /* Initiate member values which will be never changed. */
1313 1504950 : self_elim_table.can_eliminate = self_elim_table.prev_can_eliminate = true;
1314 1504950 : self_elim_table.previous_offset = 0;
1315 :
1316 1504950 : for (ep = reg_eliminate, ep1 = reg_eliminate_1;
1317 7524750 : ep < ®_eliminate[NUM_ELIMINABLE_REGS]; ep++, ep1++)
1318 : {
1319 6019800 : ep->offset = ep->previous_offset = -1;
1320 6019800 : ep->from = ep1->from;
1321 6019800 : ep->to = ep1->to;
1322 6019800 : value_p = (targetm.can_eliminate (ep->from, ep->to)
1323 6019800 : && ! (ep->to == STACK_POINTER_REGNUM
1324 2038580 : && frame_pointer_needed
1325 59642 : && (! SUPPORTS_STACK_ALIGNMENT
1326 59642 : || ! stack_realign_fp)));
1327 6019800 : setup_can_eliminate (ep, value_p);
1328 : }
1329 :
1330 : /* Build the FROM and TO REG rtx's. Note that code in gen_rtx_REG
1331 : will cause, e.g., gen_rtx_REG (Pmode, STACK_POINTER_REGNUM) to
1332 : equal stack_pointer_rtx. We depend on this. Therefore we switch
1333 : off that we are in LRA temporarily. */
1334 1504950 : lra_in_progress = false;
1335 7524750 : for (ep = reg_eliminate; ep < ®_eliminate[NUM_ELIMINABLE_REGS]; ep++)
1336 : {
1337 6526924 : ep->from_rtx = gen_rtx_REG (Pmode, ep->from);
1338 6526924 : ep->to_rtx = gen_rtx_REG (Pmode, ep->to);
1339 6019800 : eliminable_reg_rtx[ep->from] = ep->from_rtx;
1340 : }
1341 1504950 : lra_in_progress = true;
1342 1504950 : }
1343 :
1344 : /* Function for initialization of elimination once per function. It
1345 : sets up sp offset for each insn. */
1346 : static void
1347 1504950 : init_elimination (void)
1348 : {
1349 1504950 : bool stop_to_sp_elimination_p;
1350 1504950 : basic_block bb;
1351 1504950 : rtx_insn *insn;
1352 1504950 : class lra_elim_table *ep;
1353 :
1354 1504950 : init_elim_table ();
1355 16238489 : FOR_EACH_BB_FN (bb, cfun)
1356 : {
1357 14733539 : curr_sp_change = 0;
1358 14733539 : stop_to_sp_elimination_p = false;
1359 180480316 : FOR_BB_INSNS (bb, insn)
1360 165746777 : if (INSN_P (insn))
1361 : {
1362 138601393 : lra_get_insn_recog_data (insn)->sp_offset = curr_sp_change;
1363 138601393 : if (NONDEBUG_INSN_P (insn))
1364 : {
1365 86820373 : mark_not_eliminable (PATTERN (insn), VOIDmode);
1366 86820373 : if (maybe_ne (curr_sp_change, 0)
1367 86820373 : && find_reg_note (insn, REG_LABEL_OPERAND, NULL_RTX))
1368 : stop_to_sp_elimination_p = true;
1369 : }
1370 : }
1371 14733539 : if (! frame_pointer_needed
1372 10034296 : && (maybe_ne (curr_sp_change, 0) || stop_to_sp_elimination_p)
1373 14787246 : && bb->succs && bb->succs->length () != 0)
1374 46905 : for (ep = reg_eliminate; ep < ®_eliminate[NUM_ELIMINABLE_REGS]; ep++)
1375 37524 : if (ep->to == STACK_POINTER_REGNUM)
1376 18762 : setup_can_eliminate (ep, false);
1377 : }
1378 1504950 : setup_elimination_map ();
1379 1504950 : }
1380 :
1381 : /* Update and return stack pointer OFFSET after processing X. */
1382 : poly_int64
1383 18083911 : lra_update_sp_offset (rtx x, poly_int64 offset)
1384 : {
1385 18083911 : curr_sp_change = offset;
1386 18083911 : mark_not_eliminable (x, VOIDmode);
1387 18083911 : return curr_sp_change;
1388 : }
1389 :
1390 :
1391 : /* Eliminate hard reg given by its location LOC. */
1392 : void
1393 95001157 : lra_eliminate_reg_if_possible (rtx *loc)
1394 : {
1395 95001157 : int regno;
1396 95001157 : class lra_elim_table *ep;
1397 :
1398 95001157 : lra_assert (REG_P (*loc));
1399 95001157 : if ((regno = REGNO (*loc)) >= FIRST_PSEUDO_REGISTER
1400 95001157 : || ! TEST_HARD_REG_BIT (lra_no_alloc_regs, regno))
1401 : return;
1402 70571297 : if ((ep = get_elimination (*loc)) != NULL)
1403 67147455 : *loc = ep->to_rtx;
1404 : }
1405 :
1406 : /* Do (final if FINAL_P or first if FIRST_P) elimination in INSN. Add
1407 : the insn for subsequent processing in the constraint pass, update
1408 : the insn info. */
1409 : static void
1410 50544802 : process_insn_for_elimination (rtx_insn *insn, bool final_p, bool first_p)
1411 : {
1412 50544802 : eliminate_regs_in_insn (insn, final_p, first_p, 0);
1413 50544802 : if (! final_p)
1414 : {
1415 : /* Check that insn changed its code. This is a case when a move
1416 : insn becomes an add insn and we do not want to process the
1417 : insn as a move anymore. */
1418 27911135 : int icode = recog (PATTERN (insn), insn, 0);
1419 :
1420 27911135 : if (icode >= 0 && icode != INSN_CODE (insn))
1421 : {
1422 94037 : if (INSN_CODE (insn) >= 0)
1423 : /* Insn code is changed. It may change its operand type
1424 : from IN to INOUT. Inform the subsequent assignment
1425 : subpass about this situation. */
1426 94037 : check_and_force_assignment_correctness_p = true;
1427 94037 : INSN_CODE (insn) = icode;
1428 94037 : lra_update_insn_recog_data (insn);
1429 : }
1430 27911135 : lra_update_insn_regno_info (insn);
1431 27911135 : lra_push_insn (insn);
1432 27911135 : lra_set_used_insn_alternative (insn, LRA_UNKNOWN_ALT);
1433 : }
1434 50544802 : }
1435 :
1436 : /* Update frame pointer to stack pointer elimination if we started with
1437 : permitted frame pointer elimination and now target reports that we can not
1438 : do this elimination anymore. Record spilled pseudos in SPILLED_PSEUDOS
1439 : unless it is null, and return the recorded pseudos number. */
1440 : int
1441 200663 : lra_update_fp2sp_elimination (int *spilled_pseudos)
1442 : {
1443 200663 : int n;
1444 200663 : HARD_REG_SET set;
1445 200663 : class lra_elim_table *ep;
1446 :
1447 200663 : if (frame_pointer_needed || !targetm.frame_pointer_required ())
1448 200663 : return 0;
1449 0 : gcc_assert (!elimination_2sp_occurred_p);
1450 0 : ep = elimination_map[FRAME_POINTER_REGNUM];
1451 0 : if (ep->to == STACK_POINTER_REGNUM)
1452 : {
1453 : /* Prevent any further uses of fp, say in spill addresses, from being
1454 : eliminated to sp and affected by sp offsets. Alas, deactivating the
1455 : elimination altogether causes the next chosen fp elimination to miss
1456 : the offset propagation, so it may keep -1 as its prev_offset, and that
1457 : will make subsequent offsets incorrect. */
1458 0 : ep->to_rtx = ep->from_rtx;
1459 0 : setup_can_eliminate (ep, false);
1460 : }
1461 : else
1462 0 : for (ep = reg_eliminate; ep < ®_eliminate[NUM_ELIMINABLE_REGS]; ep++)
1463 0 : if (ep->from == FRAME_POINTER_REGNUM && ep->to == STACK_POINTER_REGNUM)
1464 0 : setup_can_eliminate (ep, false);
1465 0 : if (lra_dump_file != NULL)
1466 0 : fprintf (lra_dump_file,
1467 : " Frame pointer can not be eliminated anymore\n");
1468 0 : frame_pointer_needed = true;
1469 : /* If !lra_reg_spill_p, we likely have incomplete range information
1470 : for pseudos assigned to the frame pointer that will have to be
1471 : spilled, and so we may end up incorrectly sharing them unless we
1472 : get live range information for them. */
1473 0 : if (lra_complete_live_ranges ())
1474 : /* If lives ranges changed, update the aggregate live ranges in
1475 : slots as well before spilling any further pseudos. */
1476 0 : lra_recompute_slots_live_ranges ();
1477 0 : CLEAR_HARD_REG_SET (set);
1478 0 : add_to_hard_reg_set (&set, Pmode, HARD_FRAME_POINTER_REGNUM);
1479 0 : n = spill_pseudos (set, spilled_pseudos);
1480 0 : return n;
1481 : }
1482 :
1483 : /* Return true if we have a pseudo assigned to hard frame pointer. */
1484 : bool
1485 1767 : lra_fp_pseudo_p (void)
1486 : {
1487 1767 : HARD_REG_SET set;
1488 :
1489 1767 : if (frame_pointer_needed)
1490 : /* At this stage it means we have no pseudos assigned to FP: */
1491 : return false;
1492 1484 : CLEAR_HARD_REG_SET (set);
1493 1484 : add_to_hard_reg_set (&set, Pmode, HARD_FRAME_POINTER_REGNUM);
1494 151975 : for (int i = FIRST_PSEUDO_REGISTER; i < max_reg_num (); i++)
1495 43939 : if (lra_reg_info[i].nrefs != 0 && reg_renumber[i] >= 0
1496 186312 : && overlaps_hard_reg_set_p (set, PSEUDO_REGNO_MODE (i),
1497 : reg_renumber[i]))
1498 : return true;
1499 : return false;
1500 : }
1501 :
1502 : /* Entry function to do final elimination if FINAL_P or to update
1503 : elimination register offsets (FIRST_P if we are doing it the first
1504 : time). */
1505 : void
1506 8259369 : lra_eliminate (bool final_p, bool first_p)
1507 : {
1508 8259369 : unsigned int uid;
1509 8259369 : bitmap_head insns_with_changed_offsets;
1510 8259369 : bitmap_iterator bi;
1511 8259369 : class lra_elim_table *ep;
1512 :
1513 8259369 : gcc_assert (! final_p || ! first_p);
1514 :
1515 8259369 : timevar_push (TV_LRA_ELIMINATE);
1516 :
1517 8259369 : if (first_p)
1518 : {
1519 1504950 : elimination_2sp_occurred_p = false;
1520 1504950 : init_elimination ();
1521 : }
1522 :
1523 8259369 : bitmap_initialize (&insns_with_changed_offsets, ®_obstack);
1524 8259369 : if (final_p)
1525 : {
1526 1504950 : if (flag_checking)
1527 : {
1528 1504930 : update_reg_eliminate (&insns_with_changed_offsets);
1529 1504930 : gcc_assert (bitmap_empty_p (&insns_with_changed_offsets));
1530 : }
1531 : /* We change eliminable hard registers in insns so we should do
1532 : this for all insns containing any eliminable hard
1533 : register. */
1534 7524750 : for (ep = reg_eliminate; ep < ®_eliminate[NUM_ELIMINABLE_REGS]; ep++)
1535 6019800 : if (elimination_map[ep->from] != NULL)
1536 6019800 : bitmap_ior_into (&insns_with_changed_offsets,
1537 6019800 : &lra_reg_info[ep->from].insn_bitmap);
1538 : }
1539 6754419 : else if (! update_reg_eliminate (&insns_with_changed_offsets))
1540 5051229 : goto lra_eliminate_done;
1541 3208140 : if (lra_dump_file != NULL)
1542 : {
1543 194 : fprintf (lra_dump_file, "New elimination table:\n");
1544 194 : print_elim_table (lra_dump_file);
1545 : }
1546 53752942 : EXECUTE_IF_SET_IN_BITMAP (&insns_with_changed_offsets, 0, uid, bi)
1547 : /* A dead insn can be deleted in process_insn_for_elimination. */
1548 50544802 : if (lra_insn_recog_data[uid] != NULL)
1549 : {
1550 50544802 : rtx_insn *insn = lra_insn_recog_data[uid]->insn;
1551 50544802 : start_sequence ();
1552 50544802 : process_insn_for_elimination (insn, final_p, first_p);
1553 50544802 : rtx_insn *first = get_insns ();
1554 50544802 : end_sequence ();
1555 50544802 : if (first != NULL)
1556 : {
1557 0 : lra_assert (!final_p);
1558 0 : lra_process_new_insns (insn, first, NULL,
1559 : "Inserting elimination insn", true);
1560 : }
1561 : }
1562 3208140 : bitmap_clear (&insns_with_changed_offsets);
1563 :
1564 8259369 : lra_eliminate_done:
1565 8259369 : timevar_pop (TV_LRA_ELIMINATE);
1566 8259369 : }
|
