Line data Source code
1 : /* Code for RTL transformations to satisfy insn constraints.
2 : Copyright (C) 2010-2026 Free Software Foundation, Inc.
3 : Contributed by Vladimir Makarov <vmakarov@redhat.com>.
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 :
22 : /* This file contains code for 3 passes: constraint pass,
23 : inheritance/split pass, and pass for undoing failed inheritance and
24 : split.
25 :
26 : The major goal of constraint pass is to transform RTL to satisfy
27 : insn and address constraints by:
28 : o choosing insn alternatives;
29 : o generating *reload insns* (or reloads in brief) and *reload
30 : pseudos* which will get necessary hard registers later;
31 : o substituting pseudos with equivalent values and removing the
32 : instructions that initialized those pseudos.
33 :
34 : The constraint pass has biggest and most complicated code in LRA.
35 : There are a lot of important details like:
36 : o reuse of input reload pseudos to simplify reload pseudo
37 : allocations;
38 : o some heuristics to choose insn alternative to improve the
39 : inheritance;
40 : o early clobbers etc.
41 :
42 : The pass is mimicking former reload pass in alternative choosing
43 : because the reload pass is oriented to current machine description
44 : model. It might be changed if the machine description model is
45 : changed.
46 :
47 : There is special code for preventing all LRA and this pass cycling
48 : in case of bugs.
49 :
50 : On the first iteration of the pass we process every instruction and
51 : choose an alternative for each one. On subsequent iterations we try
52 : to avoid reprocessing instructions if we can be sure that the old
53 : choice is still valid.
54 :
55 : The inheritance/spilt pass is to transform code to achieve
56 : ineheritance and live range splitting. It is done on backward
57 : traversal of EBBs.
58 :
59 : The inheritance optimization goal is to reuse values in hard
60 : registers. There is analogous optimization in old reload pass. The
61 : inheritance is achieved by following transformation:
62 :
63 : reload_p1 <- p reload_p1 <- p
64 : ... new_p <- reload_p1
65 : ... => ...
66 : reload_p2 <- p reload_p2 <- new_p
67 :
68 : where p is spilled and not changed between the insns. Reload_p1 is
69 : also called *original pseudo* and new_p is called *inheritance
70 : pseudo*.
71 :
72 : The subsequent assignment pass will try to assign the same (or
73 : another if it is not possible) hard register to new_p as to
74 : reload_p1 or reload_p2.
75 :
76 : If the assignment pass fails to assign a hard register to new_p,
77 : this file will undo the inheritance and restore the original code.
78 : This is because implementing the above sequence with a spilled
79 : new_p would make the code much worse. The inheritance is done in
80 : EBB scope. The above is just a simplified example to get an idea
81 : of the inheritance as the inheritance is also done for non-reload
82 : insns.
83 :
84 : Splitting (transformation) is also done in EBB scope on the same
85 : pass as the inheritance:
86 :
87 : r <- ... or ... <- r r <- ... or ... <- r
88 : ... s <- r (new insn -- save)
89 : ... =>
90 : ... r <- s (new insn -- restore)
91 : ... <- r ... <- r
92 :
93 : The *split pseudo* s is assigned to the hard register of the
94 : original pseudo or hard register r.
95 :
96 : Splitting is done:
97 : o In EBBs with high register pressure for global pseudos (living
98 : in at least 2 BBs) and assigned to hard registers when there
99 : are more one reloads needing the hard registers;
100 : o for pseudos needing save/restore code around calls.
101 :
102 : If the split pseudo still has the same hard register as the
103 : original pseudo after the subsequent assignment pass or the
104 : original pseudo was split, the opposite transformation is done on
105 : the same pass for undoing inheritance. */
106 :
107 : #undef REG_OK_STRICT
108 :
109 : #include "config.h"
110 : #include "system.h"
111 : #include "coretypes.h"
112 : #include "backend.h"
113 : #include "hooks.h"
114 : #include "target.h"
115 : #include "rtl.h"
116 : #include "tree.h"
117 : #include "stmt.h"
118 : #include "predict.h"
119 : #include "df.h"
120 : #include "memmodel.h"
121 : #include "tm_p.h"
122 : #include "expmed.h"
123 : #include "optabs.h"
124 : #include "regs.h"
125 : #include "ira.h"
126 : #include "ira-int.h"
127 : #include "recog.h"
128 : #include "output.h"
129 : #include "addresses.h"
130 : #include "expr.h"
131 : #include "cfgrtl.h"
132 : #include "rtl-error.h"
133 : #include "lra.h"
134 : #include "lra-int.h"
135 : #include "print-rtl.h"
136 : #include "function-abi.h"
137 : #include "rtl-iter.h"
138 : #include "hash-set.h"
139 :
140 : /* Value of LRA_CURR_RELOAD_NUM at the beginning of BB of the current
141 : insn. Remember that LRA_CURR_RELOAD_NUM is the number of emitted
142 : reload insns. */
143 : static int bb_reload_num;
144 :
145 : /* The current insn being processed and corresponding its single set
146 : (NULL otherwise), its data (basic block, the insn data, the insn
147 : static data, and the mode of each operand). */
148 : static rtx_insn *curr_insn;
149 : static rtx curr_insn_set;
150 : static basic_block curr_bb;
151 : static lra_insn_recog_data_t curr_id;
152 : static struct lra_static_insn_data *curr_static_id;
153 : static machine_mode curr_operand_mode[MAX_RECOG_OPERANDS];
154 : /* Mode of the register substituted by its equivalence with VOIDmode
155 : (e.g. constant) and whose subreg is given operand of the current
156 : insn. VOIDmode in all other cases. */
157 : static machine_mode original_subreg_reg_mode[MAX_RECOG_OPERANDS];
158 : /* The first call insn after curr_insn within the EBB during inherit_in_ebb
159 : or NULL outside of that function. */
160 : static rtx_insn *first_call_insn;
161 :
162 :
163 :
164 : /* Start numbers for new registers and insns at the current constraints
165 : pass start. */
166 : static int new_regno_start;
167 : static int new_insn_uid_start;
168 :
169 : /* If LOC is nonnull, strip any outer subreg from it. */
170 : static inline rtx *
171 233859686 : strip_subreg (rtx *loc)
172 : {
173 103262952 : return loc && GET_CODE (*loc) == SUBREG ? &SUBREG_REG (*loc) : loc;
174 : }
175 :
176 : /* Return hard regno of REGNO or if it is was not assigned to a hard
177 : register, use a hard register from its allocno class. */
178 : static int
179 93720 : get_try_hard_regno (int regno)
180 : {
181 93720 : int hard_regno;
182 93720 : enum reg_class rclass;
183 :
184 93720 : if ((hard_regno = regno) >= FIRST_PSEUDO_REGISTER)
185 93720 : hard_regno = lra_get_regno_hard_regno (regno);
186 93720 : if (hard_regno >= 0)
187 : return hard_regno;
188 59325 : rclass = lra_get_allocno_class (regno);
189 59325 : if (rclass == NO_REGS)
190 : return -1;
191 56972 : return ira_class_hard_regs[rclass][0];
192 : }
193 :
194 : /* Return the hard regno of X after removing its subreg. If X is not a
195 : register or a subreg of a register, return -1. If X is a pseudo, use its
196 : assignment. If X is a hard regno, return the final hard regno which will be
197 : after elimination. */
198 : static int
199 296438000 : get_hard_regno (rtx x)
200 : {
201 296438000 : rtx reg;
202 296438000 : int hard_regno;
203 :
204 296438000 : reg = x;
205 296438000 : if (SUBREG_P (x))
206 5385318 : reg = SUBREG_REG (x);
207 296438000 : if (! REG_P (reg))
208 : return -1;
209 204792333 : int regno = REGNO (reg);
210 204792333 : if (HARD_REGISTER_NUM_P (regno))
211 35767547 : hard_regno = lra_get_elimination_hard_regno (regno);
212 : else
213 169024786 : hard_regno = lra_get_regno_hard_regno (regno);
214 204792333 : if (hard_regno < 0)
215 : return -1;
216 186598867 : if (SUBREG_P (x))
217 4663764 : hard_regno += subreg_regno_offset (hard_regno, GET_MODE (reg),
218 4663764 : SUBREG_BYTE (x), GET_MODE (x));
219 : return hard_regno;
220 : }
221 :
222 : /* If REGNO is a hard register or has been allocated a hard register,
223 : return the class of that register. If REGNO is a reload pseudo
224 : created by the current constraints pass, return its allocno class.
225 : Return NO_REGS otherwise. */
226 : static enum reg_class
227 526055434 : get_reg_class (int regno)
228 : {
229 526055434 : int hard_regno;
230 :
231 526055434 : if (HARD_REGISTER_NUM_P (regno))
232 67314213 : hard_regno = lra_get_elimination_hard_regno (regno);
233 : else
234 458741221 : hard_regno = lra_get_regno_hard_regno (regno);
235 526055434 : if (hard_regno >= 0)
236 330391383 : return REGNO_REG_CLASS (hard_regno);
237 195664051 : if (regno >= new_regno_start)
238 62687117 : return lra_get_allocno_class (regno);
239 : return NO_REGS;
240 : }
241 :
242 : /* Return true if REG_CLASS has enough allocatable hard regs to keep value of
243 : REG_MODE. */
244 : static bool
245 18875674 : enough_allocatable_hard_regs_p (enum reg_class reg_class,
246 : enum machine_mode reg_mode)
247 : {
248 18875674 : int i, j, hard_regno, class_size, nregs;
249 :
250 37751348 : if (hard_reg_set_subset_p (reg_class_contents[reg_class], lra_no_alloc_regs))
251 : return false;
252 6571872 : class_size = ira_class_hard_regs_num[reg_class];
253 6571872 : for (i = 0; i < class_size; i++)
254 : {
255 6571872 : hard_regno = ira_class_hard_regs[reg_class][i];
256 6571872 : nregs = hard_regno_nregs (hard_regno, reg_mode);
257 6571872 : if (nregs == 1)
258 : return true;
259 288870 : for (j = 0; j < nregs; j++)
260 192580 : if (TEST_HARD_REG_BIT (lra_no_alloc_regs, hard_regno + j)
261 192580 : || ! TEST_HARD_REG_BIT (reg_class_contents[reg_class],
262 : hard_regno + j))
263 : break;
264 96290 : if (j >= nregs)
265 : return true;
266 : }
267 : return false;
268 : }
269 :
270 : /* True if C is a non-empty register class that has too few registers
271 : to be safely used as a reload target class. */
272 : #define SMALL_REGISTER_CLASS_P(C) \
273 : (ira_class_hard_regs_num [(C)] == 1 \
274 : || (ira_class_hard_regs_num [(C)] >= 1 \
275 : && targetm.class_likely_spilled_p (C)))
276 :
277 : /* Return true if REG satisfies (or will satisfy) reg class constraint
278 : CL. Use elimination first if REG is a hard register. If REG is a
279 : reload pseudo created by this constraints pass, assume that it will
280 : be allocated a hard register from its allocno class, but allow that
281 : class to be narrowed to CL if it is currently a superset of CL and
282 : if either:
283 :
284 : - ALLOW_ALL_RELOAD_CLASS_CHANGES_P is true or
285 : - the instruction we're processing is not a reload move.
286 :
287 : If NEW_CLASS is nonnull, set *NEW_CLASS to the new allocno class of
288 : REGNO (reg), or NO_REGS if no change in its class was needed. */
289 : static bool
290 223347492 : in_class_p (rtx reg, enum reg_class cl, enum reg_class *new_class,
291 : bool allow_all_reload_class_changes_p = false)
292 : {
293 223347492 : enum reg_class rclass, common_class;
294 223347492 : machine_mode reg_mode;
295 223347492 : rtx src;
296 223347492 : int regno = REGNO (reg);
297 :
298 223347492 : if (new_class != NULL)
299 115137025 : *new_class = NO_REGS;
300 223347492 : if (regno < FIRST_PSEUDO_REGISTER)
301 : {
302 28400654 : rtx final_reg = reg;
303 28400654 : rtx *final_loc = &final_reg;
304 :
305 28400654 : lra_eliminate_reg_if_possible (final_loc);
306 28400654 : return TEST_HARD_REG_BIT (reg_class_contents[cl], REGNO (*final_loc));
307 : }
308 194946838 : reg_mode = GET_MODE (reg);
309 194946838 : rclass = get_reg_class (regno);
310 194946838 : src = curr_insn_set != NULL ? SET_SRC (curr_insn_set) : NULL;
311 194946838 : if (regno < new_regno_start
312 : /* Do not allow the constraints for reload instructions to
313 : influence the classes of new pseudos. These reloads are
314 : typically moves that have many alternatives, and restricting
315 : reload pseudos for one alternative may lead to situations
316 : where other reload pseudos are no longer allocatable. */
317 194946838 : || (!allow_all_reload_class_changes_p
318 15101018 : && INSN_UID (curr_insn) >= new_insn_uid_start
319 14555038 : && src != NULL
320 14555038 : && ((REG_P (src) || MEM_P (src))
321 1491953 : || (GET_CODE (src) == SUBREG
322 687183 : && (REG_P (SUBREG_REG (src)) || MEM_P (SUBREG_REG (src)))))))
323 : /* When we don't know what class will be used finally for reload
324 : pseudos, we use ALL_REGS. */
325 13750268 : return ((regno >= new_regno_start && rclass == ALL_REGS)
326 189818897 : || (rclass != NO_REGS && ira_class_subset_p[rclass][cl]
327 205865544 : && ! hard_reg_set_subset_p (reg_class_contents[cl],
328 : lra_no_alloc_regs)));
329 : else
330 : {
331 18875674 : common_class = ira_reg_class_subset[rclass][cl];
332 18875674 : if (new_class != NULL)
333 5402432 : *new_class = common_class;
334 18875674 : return (enough_allocatable_hard_regs_p (common_class, reg_mode)
335 : /* Do not permit reload insn operand matching (new_class == NULL
336 : case) if the new class is too small. */
337 18875674 : && (new_class != NULL || common_class == rclass
338 1078514 : || !SMALL_REGISTER_CLASS_P (common_class)));
339 : }
340 : }
341 :
342 : /* Return true if REGNO satisfies a memory constraint. */
343 : static bool
344 65157930 : in_mem_p (int regno)
345 : {
346 0 : return get_reg_class (regno) == NO_REGS;
347 : }
348 :
349 : /* Return true if ADDR is a valid memory address for mode MODE in address
350 : space AS, and check that each pseudo has the proper kind of hard
351 : reg. */
352 : static bool
353 36378136 : valid_address_p (machine_mode mode ATTRIBUTE_UNUSED,
354 : rtx addr, addr_space_t as)
355 : {
356 : #ifdef GO_IF_LEGITIMATE_ADDRESS
357 : lra_assert (ADDR_SPACE_GENERIC_P (as));
358 : GO_IF_LEGITIMATE_ADDRESS (mode, addr, win);
359 : return false;
360 :
361 : win:
362 : return true;
363 : #else
364 0 : return targetm.addr_space.legitimate_address_p (mode, addr, 0, as,
365 36378136 : ERROR_MARK);
366 : #endif
367 : }
368 :
369 : namespace {
370 : /* Temporarily eliminates registers in an address (for the lifetime of
371 : the object). */
372 : class address_eliminator {
373 : public:
374 : address_eliminator (struct address_info *ad);
375 : ~address_eliminator ();
376 :
377 : private:
378 : struct address_info *m_ad;
379 : rtx *m_base_loc;
380 : rtx m_base_reg;
381 : rtx *m_index_loc;
382 : rtx m_index_reg;
383 : };
384 : }
385 :
386 76635228 : address_eliminator::address_eliminator (struct address_info *ad)
387 76635228 : : m_ad (ad),
388 76635228 : m_base_loc (strip_subreg (ad->base_term)),
389 76635228 : m_base_reg (NULL_RTX),
390 76635228 : m_index_loc (strip_subreg (ad->index_term)),
391 76635228 : m_index_reg (NULL_RTX)
392 : {
393 76635228 : if (m_base_loc != NULL)
394 : {
395 63893871 : m_base_reg = *m_base_loc;
396 : /* If we have non-legitimate address which is decomposed not in
397 : the way we expected, don't do elimination here. In such case
398 : the address will be reloaded and elimination will be done in
399 : reload insn finally. */
400 63893871 : if (REG_P (m_base_reg))
401 63893871 : lra_eliminate_reg_if_possible (m_base_loc);
402 63893871 : if (m_ad->base_term2 != NULL)
403 0 : *m_ad->base_term2 = *m_ad->base_term;
404 : }
405 76635228 : if (m_index_loc != NULL)
406 : {
407 3569656 : m_index_reg = *m_index_loc;
408 3569656 : if (REG_P (m_index_reg))
409 3569656 : lra_eliminate_reg_if_possible (m_index_loc);
410 : }
411 76635228 : }
412 :
413 76635228 : address_eliminator::~address_eliminator ()
414 : {
415 76635228 : if (m_base_loc && *m_base_loc != m_base_reg)
416 : {
417 45216862 : *m_base_loc = m_base_reg;
418 45216862 : if (m_ad->base_term2 != NULL)
419 0 : *m_ad->base_term2 = *m_ad->base_term;
420 : }
421 76635228 : if (m_index_loc && *m_index_loc != m_index_reg)
422 0 : *m_index_loc = m_index_reg;
423 76635228 : }
424 :
425 : /* Return true if the eliminated form of AD is a legitimate target address.
426 : If OP is a MEM, AD is the address within OP, otherwise OP should be
427 : ignored. CONSTRAINT is one constraint that the operand may need
428 : to meet. */
429 : static bool
430 36355344 : valid_address_p (rtx op, struct address_info *ad,
431 : enum constraint_num constraint)
432 : {
433 36355344 : address_eliminator eliminator (ad);
434 :
435 : /* Allow a memory OP if it matches CONSTRAINT, even if CONSTRAINT is more
436 : forgiving than "m".
437 : Need to extract memory from op for special memory constraint,
438 : i.e. bcst_mem_operand in i386 backend. */
439 36355344 : if (MEM_P (extract_mem_from_operand (op))
440 : && insn_extra_relaxed_memory_constraint (constraint)
441 : && constraint_satisfied_p (op, constraint))
442 : return true;
443 :
444 36355344 : return valid_address_p (ad->mode, *ad->outer, ad->as);
445 36355344 : }
446 :
447 : /* For special_memory_operand, it could be false for MEM_P (op),
448 : i.e. bcst_mem_operand in i386 backend.
449 : Extract and return real memory operand or op. */
450 : rtx
451 641748764 : extract_mem_from_operand (rtx op)
452 : {
453 643478229 : for (rtx x = op;; x = XEXP (x, 0))
454 : {
455 643478229 : if (MEM_P (x))
456 : return x;
457 458483606 : if (GET_RTX_LENGTH (GET_CODE (x)) != 1
458 375763661 : || GET_RTX_FORMAT (GET_CODE (x))[0] != 'e')
459 : break;
460 : }
461 : return op;
462 : }
463 :
464 : /* Return true if the eliminated form of memory reference OP satisfies
465 : extra (special) memory constraint CONSTRAINT. */
466 : static bool
467 37673871 : satisfies_memory_constraint_p (rtx op, enum constraint_num constraint)
468 : {
469 37673871 : struct address_info ad;
470 37673871 : rtx mem = extract_mem_from_operand (op);
471 37673871 : if (!MEM_P (mem))
472 : return false;
473 :
474 36647960 : decompose_mem_address (&ad, mem);
475 36647960 : address_eliminator eliminator (&ad);
476 36647960 : return constraint_satisfied_p (op, constraint);
477 36647960 : }
478 :
479 : /* Return true if the eliminated form of address AD satisfies extra
480 : address constraint CONSTRAINT. */
481 : static bool
482 3631924 : satisfies_address_constraint_p (struct address_info *ad,
483 : enum constraint_num constraint)
484 : {
485 3631924 : address_eliminator eliminator (ad);
486 3631924 : return constraint_satisfied_p (*ad->outer, constraint);
487 3631924 : }
488 :
489 : /* Return true if the eliminated form of address OP satisfies extra
490 : address constraint CONSTRAINT. */
491 : static bool
492 1785617 : satisfies_address_constraint_p (rtx op, enum constraint_num constraint)
493 : {
494 1785617 : struct address_info ad;
495 :
496 1785617 : decompose_lea_address (&ad, &op);
497 1785617 : return satisfies_address_constraint_p (&ad, constraint);
498 : }
499 :
500 : /* Set of equivalences whose original targets have set up pointer flag. */
501 : static hash_set <rtx> *pointer_equiv_set;
502 :
503 : /* Add x to pointer_equiv_set. */
504 : void
505 2029999 : lra_pointer_equiv_set_add (rtx x)
506 : {
507 2029999 : pointer_equiv_set->add (x);
508 2029999 : }
509 :
510 : /* Return true if x is in pointer_equiv_set. */
511 : bool
512 10143882 : lra_pointer_equiv_set_in (rtx x)
513 : {
514 10143882 : return pointer_equiv_set->contains (x);
515 : }
516 :
517 : /* Initiate equivalences for LRA. As we keep original equivalences
518 : before any elimination, we need to make copies otherwise any change
519 : in insns might change the equivalences. */
520 : void
521 1517127 : lra_init_equiv (void)
522 : {
523 1517127 : ira_expand_reg_equiv ();
524 73687836 : for (int i = FIRST_PSEUDO_REGISTER; i < max_reg_num (); i++)
525 : {
526 70653582 : rtx res;
527 :
528 70653582 : if ((res = ira_reg_equiv[i].memory) != NULL_RTX)
529 3015060 : ira_reg_equiv[i].memory = copy_rtx (res);
530 70653582 : if ((res = ira_reg_equiv[i].invariant) != NULL_RTX)
531 909498 : ira_reg_equiv[i].invariant = copy_rtx (res);
532 : }
533 1517127 : pointer_equiv_set = new hash_set <rtx>;
534 1517127 : }
535 :
536 : /* Finish equivalence data for LRA. */
537 : void
538 1517127 : lra_finish_equiv (void)
539 : {
540 3034254 : delete pointer_equiv_set;
541 1517127 : }
542 :
543 : static rtx loc_equivalence_callback (rtx, const_rtx, void *);
544 :
545 : /* Update equivalence for REGNO. We need to this as the equivalence
546 : might contain other pseudos which are changed by their
547 : equivalences. */
548 : static void
549 209419043 : update_equiv (int regno)
550 : {
551 209419043 : rtx x;
552 :
553 209419043 : if ((x = ira_reg_equiv[regno].memory) != NULL_RTX)
554 9127444 : ira_reg_equiv[regno].memory
555 9127444 : = simplify_replace_fn_rtx (x, NULL_RTX, loc_equivalence_callback,
556 : NULL_RTX);
557 209419043 : if ((x = ira_reg_equiv[regno].invariant) != NULL_RTX)
558 2823439 : ira_reg_equiv[regno].invariant
559 2823439 : = simplify_replace_fn_rtx (x, NULL_RTX, loc_equivalence_callback,
560 : NULL_RTX);
561 209419043 : }
562 :
563 : /* If we have decided to substitute X with another value, return that
564 : value, otherwise return X. */
565 : static rtx
566 447641770 : get_equiv (rtx x)
567 : {
568 447641770 : int regno;
569 447641770 : rtx res;
570 :
571 302754536 : if (! REG_P (x) || (regno = REGNO (x)) < FIRST_PSEUDO_REGISTER
572 200888634 : || regno >= ira_reg_equiv_len
573 200888634 : || ! ira_reg_equiv[regno].defined_p
574 26555174 : || ! ira_reg_equiv[regno].profitable_p
575 474156094 : || lra_get_regno_hard_regno (regno) >= 0)
576 : return x;
577 5221295 : if ((res = ira_reg_equiv[regno].memory) != NULL_RTX)
578 : {
579 2287059 : if (targetm.cannot_substitute_mem_equiv_p (res))
580 : return x;
581 2287059 : return res;
582 : }
583 2934236 : if ((res = ira_reg_equiv[regno].constant) != NULL_RTX)
584 : {
585 940929 : if (targetm.cannot_substitute_const_equiv_p (res))
586 : return x;
587 940929 : return res;
588 : }
589 1993307 : if ((res = ira_reg_equiv[regno].invariant) != NULL_RTX)
590 : return res;
591 0 : gcc_unreachable ();
592 : }
593 :
594 : /* If we have decided to substitute X with the equivalent value, return that
595 : value after elimination for INSN, otherwise return X. Add the result to
596 : pointer_equiv_set if X has set up pointer flag. */
597 : static rtx
598 251807320 : get_equiv_with_elimination (rtx x, rtx_insn *insn)
599 : {
600 251807320 : rtx res = get_equiv (x);
601 :
602 251807320 : if (x == res || CONSTANT_P (res))
603 : return res;
604 1572370 : res = lra_eliminate_regs_1 (insn, res, GET_MODE (res),
605 : false, false, 0, true);
606 1572370 : if (REG_POINTER (x))
607 1092689 : lra_pointer_equiv_set_add (res);
608 : return res;
609 : }
610 :
611 : /* Record invariant INVARIANT_RTX as an equivalence of REG. */
612 : static void
613 30185 : add_invariant (rtx reg, rtx invariant_rtx)
614 : {
615 30185 : int regno = REGNO (reg);
616 :
617 30185 : ira_expand_reg_equiv ();
618 30185 : ira_reg_equiv[regno].defined_p = true;
619 30185 : ira_reg_equiv[regno].profitable_p = true;
620 30185 : if (CONSTANT_P (invariant_rtx))
621 30185 : ira_reg_equiv[regno].constant = copy_rtx (invariant_rtx);
622 : else
623 0 : ira_reg_equiv[regno].invariant = copy_rtx (invariant_rtx);
624 30185 : }
625 :
626 : /* Set up curr_operand_mode. */
627 : static void
628 108817941 : init_curr_operand_mode (void)
629 : {
630 108817941 : int nop = curr_static_id->n_operands;
631 339400008 : for (int i = 0; i < nop; i++)
632 : {
633 230582067 : machine_mode mode = GET_MODE (*curr_id->operand_loc[i]);
634 230582067 : if (mode == VOIDmode)
635 : {
636 : /* The .md mode for address operands is the mode of the
637 : addressed value rather than the mode of the address itself. */
638 44646631 : if (curr_id->icode >= 0 && curr_static_id->operand[i].is_address)
639 63 : mode = Pmode;
640 : else
641 44646568 : mode = curr_static_id->operand[i].mode;
642 : }
643 230582067 : curr_operand_mode[i] = mode;
644 : }
645 108817941 : }
646 :
647 :
648 :
649 : /* The page contains code to reuse input reloads. */
650 :
651 : /* Structure describes input reload of the current insns. */
652 : struct input_reload
653 : {
654 : /* True for input reload of matched operands. */
655 : bool match_p;
656 : /* True for input reload of inout earlyclobber operand. */
657 : bool early_clobber_p;
658 : /* Reloaded value. */
659 : rtx input;
660 : /* Reload pseudo used. */
661 : rtx reg;
662 : };
663 :
664 : /* The number of elements in the following array. */
665 : static int curr_insn_input_reloads_num;
666 : /* Array containing info about input reloads. It is used to find the
667 : same input reload and reuse the reload pseudo in this case. */
668 : static struct input_reload curr_insn_input_reloads[LRA_MAX_INSN_RELOADS];
669 :
670 : /* Initiate data concerning reuse of input reloads for the current
671 : insn. */
672 : static void
673 108817941 : init_curr_insn_input_reloads (void)
674 : {
675 108817941 : curr_insn_input_reloads_num = 0;
676 0 : }
677 :
678 : /* The canonical form of an rtx inside a MEM is not necessarily the same as the
679 : canonical form of the rtx outside the MEM. Fix this up in the case that
680 : we're reloading an address (and therefore pulling it outside a MEM). */
681 : static rtx
682 125 : canonicalize_reload_addr (rtx addr)
683 : {
684 125 : subrtx_var_iterator::array_type array;
685 458 : FOR_EACH_SUBRTX_VAR (iter, array, addr, NONCONST)
686 : {
687 333 : rtx x = *iter;
688 333 : if (GET_CODE (x) == MULT && CONST_INT_P (XEXP (x, 1)))
689 : {
690 14 : const HOST_WIDE_INT ci = INTVAL (XEXP (x, 1));
691 347 : const int pwr2 = exact_log2 (ci);
692 14 : if (pwr2 > 0)
693 : {
694 : /* Rewrite this to use a shift instead, which is canonical when
695 : outside of a MEM. */
696 14 : PUT_CODE (x, ASHIFT);
697 14 : XEXP (x, 1) = GEN_INT (pwr2);
698 : }
699 : }
700 : }
701 :
702 125 : return addr;
703 125 : }
704 :
705 : /* Return rtx accessing reload REG of RCLASS matching another reload reg in
706 : MODE. */
707 : static rtx
708 114311 : get_matching_reload_reg_subreg (machine_mode mode, rtx reg,
709 : enum reg_class rclass)
710 : {
711 114311 : int hard_regno = ira_class_hard_regs[rclass][0];
712 114311 : if (subreg_regno_offset (hard_regno,
713 114311 : GET_MODE (reg),
714 114311 : subreg_lowpart_offset (mode, GET_MODE (reg)),
715 : mode) == 0)
716 : /* For matching scalar int modes generate the right subreg byte offset for
717 : BE targets -- see call of reload.cc:operands_match_p in
718 : recog.cc:constrain_operands. */
719 114311 : return lowpart_subreg (mode, reg, GET_MODE (reg));
720 0 : int offset = (lra_constraint_offset (hard_regno, GET_MODE (reg))
721 0 : - lra_constraint_offset (hard_regno, mode)) * UNITS_PER_WORD;
722 0 : lra_assert (offset >= 0);
723 0 : return gen_rtx_SUBREG (mode, reg, offset);
724 : }
725 :
726 : /* Create a new pseudo using MODE, RCLASS, EXCLUDE_START_HARD_REGS, ORIGINAL or
727 : reuse an existing reload pseudo. Don't reuse an existing reload pseudo if
728 : IN_SUBREG_P is true and the reused pseudo should be wrapped up in a SUBREG.
729 : EARLY_CLOBBER_P is true for input reload of inout early clobber operand.
730 : The result pseudo is returned through RESULT_REG. Return TRUE if we created
731 : a new pseudo, FALSE if we reused an existing reload pseudo. Use TITLE to
732 : describe new registers for debug purposes. */
733 : static bool
734 3911390 : get_reload_reg (enum op_type type, machine_mode mode, rtx original,
735 : enum reg_class rclass, HARD_REG_SET *exclude_start_hard_regs,
736 : bool in_subreg_p, bool early_clobber_p,
737 : const char *title, rtx *result_reg)
738 : {
739 3911390 : int i, regno;
740 3911390 : enum reg_class new_class;
741 :
742 3911390 : if (type == OP_OUT)
743 : {
744 : /* Output reload registers tend to start out with a conservative
745 : choice of register class. Usually this is ALL_REGS, although
746 : a target might narrow it (for performance reasons) through
747 : targetm.preferred_reload_class. It's therefore quite common
748 : for a reload instruction to require a more restrictive class
749 : than the class that was originally assigned to the reload register.
750 :
751 : In these situations, it's more efficient to refine the choice
752 : of register class rather than create a second reload register.
753 : This also helps to avoid cycling for registers that are only
754 : used by reload instructions. */
755 980882 : if (REG_P (original)
756 707677 : && (int) REGNO (original) >= new_regno_start
757 2772 : && (INSN_UID (curr_insn) >= new_insn_uid_start
758 250 : || ira_former_scratch_p (REGNO (original)))
759 2772 : && in_class_p (original, rclass, &new_class, true)
760 981132 : && (exclude_start_hard_regs == nullptr
761 250 : || hard_reg_set_intersect_p (
762 981132 : ~lra_reg_info[REGNO (original)].exclude_start_hard_regs,
763 500 : ~*exclude_start_hard_regs)))
764 : {
765 250 : unsigned int regno = REGNO (original);
766 250 : if (lra_dump_file != NULL)
767 : {
768 0 : fprintf (lra_dump_file, " Reuse r%d for output ", regno);
769 0 : dump_value_slim (lra_dump_file, original, 1);
770 : }
771 500 : if (new_class != lra_get_allocno_class (regno))
772 250 : lra_change_class (regno, new_class, ", change to", false);
773 250 : if (lra_dump_file != NULL)
774 0 : fprintf (lra_dump_file, "\n");
775 250 : if (exclude_start_hard_regs)
776 250 : lra_reg_info[regno].exclude_start_hard_regs
777 250 : |= *exclude_start_hard_regs;
778 250 : *result_reg = original;
779 250 : return false;
780 : }
781 980632 : *result_reg
782 980632 : = lra_create_new_reg_with_unique_value (mode, original, rclass,
783 : exclude_start_hard_regs, title);
784 980632 : return true;
785 : }
786 :
787 2930508 : bool unique_p = early_clobber_p;
788 : /* Prevent reuse value of expression with side effects,
789 : e.g. volatile memory. */
790 2930508 : if (! side_effects_p (original))
791 3160156 : for (i = 0; i < curr_insn_input_reloads_num; i++)
792 : {
793 239553 : if (! curr_insn_input_reloads[i].match_p
794 97717 : && ! curr_insn_input_reloads[i].early_clobber_p
795 97716 : && rtx_equal_p (curr_insn_input_reloads[i].input, original)
796 249209 : && in_class_p (curr_insn_input_reloads[i].reg, rclass, &new_class))
797 : {
798 9643 : rtx reg = curr_insn_input_reloads[i].reg;
799 9643 : regno = REGNO (reg);
800 : /* If input is equal to original and both are VOIDmode,
801 : GET_MODE (reg) might be still different from mode.
802 : Ensure we don't return *result_reg with wrong mode. */
803 9643 : if (GET_MODE (reg) != mode)
804 : {
805 0 : if (in_subreg_p)
806 0 : continue;
807 0 : if (maybe_lt (GET_MODE_SIZE (GET_MODE (reg)),
808 0 : GET_MODE_SIZE (mode)))
809 0 : continue;
810 0 : reg = get_matching_reload_reg_subreg (mode, reg, new_class);
811 0 : if (reg == NULL_RTX || GET_CODE (reg) != SUBREG)
812 0 : continue;
813 : }
814 : /* If the existing reload and this have no start hard register in
815 : common, then skip. Otherwise update exclude_start_hard_regs. */
816 9643 : if (exclude_start_hard_regs
817 11122 : && ! hard_reg_set_empty_p (*exclude_start_hard_regs))
818 : {
819 1 : HARD_REG_SET r = lra_reg_info[regno].exclude_start_hard_regs
820 1 : | *exclude_start_hard_regs;
821 2 : if (hard_reg_set_empty_p (~r))
822 0 : continue;
823 : else
824 1 : lra_reg_info[regno].exclude_start_hard_regs = r;
825 : }
826 9643 : *result_reg = reg;
827 9643 : if (lra_dump_file != NULL)
828 : {
829 0 : fprintf (lra_dump_file, " Reuse r%d for reload ", regno);
830 0 : dump_value_slim (lra_dump_file, original, 1);
831 : }
832 19286 : if (new_class != lra_get_allocno_class (regno))
833 4757 : lra_change_class (regno, new_class, ", change to", false);
834 9643 : if (lra_dump_file != NULL)
835 0 : fprintf (lra_dump_file, "\n");
836 : return false;
837 : }
838 : /* If we have an input reload with a different mode, make sure it
839 : will get a different hard reg. */
840 229910 : else if (REG_P (original)
841 179048 : && REG_P (curr_insn_input_reloads[i].input)
842 148310 : && REGNO (original) == REGNO (curr_insn_input_reloads[i].input)
843 229910 : && (GET_MODE (original)
844 2159 : != GET_MODE (curr_insn_input_reloads[i].input)))
845 : unique_p = true;
846 : }
847 5841730 : *result_reg = (unique_p
848 2920865 : ? lra_create_new_reg_with_unique_value
849 2920865 : : lra_create_new_reg) (mode, original, rclass,
850 : exclude_start_hard_regs, title);
851 2920865 : lra_assert (curr_insn_input_reloads_num < LRA_MAX_INSN_RELOADS);
852 2920865 : curr_insn_input_reloads[curr_insn_input_reloads_num].input = original;
853 2920865 : curr_insn_input_reloads[curr_insn_input_reloads_num].match_p = false;
854 2920865 : curr_insn_input_reloads[curr_insn_input_reloads_num].early_clobber_p
855 2920865 : = early_clobber_p;
856 2920865 : curr_insn_input_reloads[curr_insn_input_reloads_num++].reg = *result_reg;
857 2920865 : return true;
858 : }
859 :
860 :
861 : /* The page contains major code to choose the current insn alternative
862 : and generate reloads for it. */
863 :
864 : /* Return the offset from REGNO of the least significant register
865 : in (reg:MODE REGNO).
866 :
867 : This function is used to tell whether two registers satisfy
868 : a matching constraint. (reg:MODE1 REGNO1) matches (reg:MODE2 REGNO2) if:
869 :
870 : REGNO1 + lra_constraint_offset (REGNO1, MODE1)
871 : == REGNO2 + lra_constraint_offset (REGNO2, MODE2) */
872 : int
873 44991632 : lra_constraint_offset (int regno, machine_mode mode)
874 : {
875 44991632 : lra_assert (regno < FIRST_PSEUDO_REGISTER);
876 :
877 44991632 : scalar_int_mode int_mode;
878 44991632 : if (WORDS_BIG_ENDIAN
879 : && is_a <scalar_int_mode> (mode, &int_mode)
880 : && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD)
881 : return hard_regno_nregs (regno, mode) - 1;
882 44991632 : return 0;
883 : }
884 :
885 : /* Like rtx_equal_p except that it allows a REG and a SUBREG to match
886 : if they are the same hard reg, and has special hacks for
887 : auto-increment and auto-decrement. This is specifically intended for
888 : process_alt_operands to use in determining whether two operands
889 : match. X is the operand whose number is the lower of the two.
890 :
891 : It is supposed that X is the output operand and Y is the input
892 : operand. Y_HARD_REGNO is the final hard regno of register Y or
893 : register in subreg Y as we know it now. Otherwise, it is a
894 : negative value. */
895 : static bool
896 59101548 : operands_match_p (rtx x, rtx y, int y_hard_regno)
897 : {
898 59101548 : int i;
899 59101548 : RTX_CODE code = GET_CODE (x);
900 59101548 : const char *fmt;
901 :
902 59101548 : if (x == y)
903 : return true;
904 51325364 : if ((code == REG || (code == SUBREG && REG_P (SUBREG_REG (x))))
905 24591894 : && (REG_P (y) || (GET_CODE (y) == SUBREG && REG_P (SUBREG_REG (y)))))
906 : {
907 24496614 : int j;
908 :
909 24496614 : i = get_hard_regno (x);
910 24496614 : if (i < 0)
911 1304134 : goto slow;
912 :
913 23192480 : if ((j = y_hard_regno) < 0)
914 696664 : goto slow;
915 :
916 22495816 : i += lra_constraint_offset (i, GET_MODE (x));
917 22495816 : j += lra_constraint_offset (j, GET_MODE (y));
918 :
919 22495816 : return i == j;
920 : }
921 :
922 : /* If two operands must match, because they are really a single
923 : operand of an assembler insn, then two post-increments are invalid
924 : because the assembler insn would increment only once. On the
925 : other hand, a post-increment matches ordinary indexing if the
926 : post-increment is the output operand. */
927 26828750 : if (code == POST_DEC || code == POST_INC || code == POST_MODIFY)
928 0 : return operands_match_p (XEXP (x, 0), y, y_hard_regno);
929 :
930 : /* Two pre-increments are invalid because the assembler insn would
931 : increment only once. On the other hand, a pre-increment matches
932 : ordinary indexing if the pre-increment is the input operand. */
933 26828750 : if (GET_CODE (y) == PRE_DEC || GET_CODE (y) == PRE_INC
934 26828750 : || GET_CODE (y) == PRE_MODIFY)
935 0 : return operands_match_p (x, XEXP (y, 0), -1);
936 :
937 26828750 : slow:
938 :
939 28829548 : if (code == REG && REG_P (y))
940 1901108 : return REGNO (x) == REGNO (y);
941 :
942 97692 : if (code == REG && GET_CODE (y) == SUBREG && REG_P (SUBREG_REG (y))
943 9273 : && x == SUBREG_REG (y))
944 : return true;
945 26928440 : if (GET_CODE (y) == REG && code == SUBREG && REG_P (SUBREG_REG (x))
946 68047 : && SUBREG_REG (x) == y)
947 : return true;
948 :
949 : /* Now we have disposed of all the cases in which different rtx
950 : codes can match. */
951 26928311 : if (code != GET_CODE (y))
952 : return false;
953 :
954 : /* (MULT:SI x y) and (MULT:HI x y) are NOT equivalent. */
955 1064190 : if (GET_MODE (x) != GET_MODE (y))
956 : return false;
957 :
958 1063460 : switch (code)
959 : {
960 : CASE_CONST_UNIQUE:
961 : return false;
962 :
963 : case CONST_VECTOR:
964 : if (!same_vector_encodings_p (x, y))
965 : return false;
966 : break;
967 :
968 0 : case LABEL_REF:
969 0 : return label_ref_label (x) == label_ref_label (y);
970 25 : case SYMBOL_REF:
971 25 : return XSTR (x, 0) == XSTR (y, 0);
972 :
973 : default:
974 : break;
975 : }
976 :
977 : /* Compare the elements. If any pair of corresponding elements fail
978 : to match, return false for the whole things. */
979 :
980 1036552 : fmt = GET_RTX_FORMAT (code);
981 2993762 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
982 : {
983 2035659 : int val, j;
984 2035659 : switch (fmt[i])
985 : {
986 0 : case 'w':
987 0 : if (XWINT (x, i) != XWINT (y, i))
988 : return false;
989 : break;
990 :
991 524 : case 'i':
992 524 : if (XINT (x, i) != XINT (y, i))
993 : return false;
994 : break;
995 :
996 0 : case 'L':
997 0 : if (XLOC (x, i) != XLOC (y, i))
998 : return false;
999 : break;
1000 :
1001 22295 : case 'p':
1002 22295 : if (maybe_ne (SUBREG_BYTE (x), SUBREG_BYTE (y)))
1003 : return false;
1004 : break;
1005 :
1006 1479189 : case 'e':
1007 1479189 : val = operands_match_p (XEXP (x, i), XEXP (y, i), -1);
1008 1479189 : if (val == 0)
1009 : return false;
1010 : break;
1011 :
1012 : case '0':
1013 : break;
1014 :
1015 524 : case 'E':
1016 524 : if (XVECLEN (x, i) != XVECLEN (y, i))
1017 : return false;
1018 1048 : for (j = XVECLEN (x, i) - 1; j >= 0; --j)
1019 : {
1020 524 : val = operands_match_p (XVECEXP (x, i, j), XVECEXP (y, i, j), -1);
1021 524 : if (val == 0)
1022 : return false;
1023 : }
1024 : break;
1025 :
1026 : /* It is believed that rtx's at this level will never
1027 : contain anything but integers and other rtx's, except for
1028 : within LABEL_REFs and SYMBOL_REFs. */
1029 0 : default:
1030 0 : gcc_unreachable ();
1031 : }
1032 : }
1033 : return true;
1034 : }
1035 :
1036 : /* If REG is a reload pseudo, try to make its class satisfying CL. */
1037 : static void
1038 3563396 : narrow_reload_pseudo_class (rtx reg, enum reg_class cl)
1039 : {
1040 3563396 : enum reg_class rclass;
1041 :
1042 : /* Do not make more accurate class from reloads generated. They are
1043 : mostly moves with a lot of constraints. Making more accurate
1044 : class may results in very narrow class and impossibility of find
1045 : registers for several reloads of one insn. */
1046 3563396 : if (INSN_UID (curr_insn) >= new_insn_uid_start)
1047 3551060 : return;
1048 3563226 : if (GET_CODE (reg) == SUBREG)
1049 172423 : reg = SUBREG_REG (reg);
1050 3563226 : if (! REG_P (reg) || (int) REGNO (reg) < new_regno_start)
1051 : return;
1052 12336 : if (in_class_p (reg, cl, &rclass) && rclass != cl)
1053 13 : lra_change_class (REGNO (reg), rclass, " Change to", true);
1054 : }
1055 :
1056 : /* Searches X for any reference to a reg with the same value as REGNO,
1057 : returning the rtx of the reference found if any. Otherwise,
1058 : returns NULL_RTX. */
1059 : static rtx
1060 573226 : regno_val_use_in (unsigned int regno, rtx x)
1061 : {
1062 573226 : const char *fmt;
1063 573226 : int i, j;
1064 573226 : rtx tem;
1065 :
1066 573226 : if (REG_P (x) && lra_reg_info[REGNO (x)].val == lra_reg_info[regno].val)
1067 : return x;
1068 :
1069 572923 : fmt = GET_RTX_FORMAT (GET_CODE (x));
1070 1152102 : for (i = GET_RTX_LENGTH (GET_CODE (x)) - 1; i >= 0; i--)
1071 : {
1072 579179 : if (fmt[i] == 'e')
1073 : {
1074 7559 : if ((tem = regno_val_use_in (regno, XEXP (x, i))))
1075 : return tem;
1076 : }
1077 571620 : else if (fmt[i] == 'E')
1078 0 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
1079 0 : if ((tem = regno_val_use_in (regno , XVECEXP (x, i, j))))
1080 : return tem;
1081 : }
1082 :
1083 : return NULL_RTX;
1084 : }
1085 :
1086 : /* Return true if all current insn non-output operands except INS (it
1087 : has a negaitve end marker) do not use pseudos with the same value
1088 : as REGNO. */
1089 : static bool
1090 2 : check_conflict_input_operands (int regno, signed char *ins)
1091 : {
1092 2 : int in;
1093 2 : int n_operands = curr_static_id->n_operands;
1094 :
1095 8 : for (int nop = 0; nop < n_operands; nop++)
1096 7 : if (! curr_static_id->operand[nop].is_operator
1097 7 : && curr_static_id->operand[nop].type != OP_OUT)
1098 : {
1099 5 : for (int i = 0; (in = ins[i]) >= 0; i++)
1100 4 : if (in == nop)
1101 : break;
1102 3 : if (in < 0
1103 3 : && regno_val_use_in (regno, *curr_id->operand_loc[nop]) != NULL_RTX)
1104 : return false;
1105 : }
1106 : return true;
1107 : }
1108 :
1109 : /* Generate reloads for matching OUT and INS (array of input operand numbers
1110 : with end marker -1) with reg class GOAL_CLASS and EXCLUDE_START_HARD_REGS,
1111 : considering output operands OUTS (similar array to INS) needing to be in
1112 : different registers. Add input and output reloads correspondingly to the
1113 : lists *BEFORE and *AFTER. OUT might be negative. In this case we generate
1114 : input reloads for matched input operands INS. EARLY_CLOBBER_P is a flag
1115 : that the output operand is early clobbered for chosen alternative. */
1116 : static void
1117 1781698 : match_reload (signed char out, signed char *ins, signed char *outs,
1118 : enum reg_class goal_class, HARD_REG_SET *exclude_start_hard_regs,
1119 : rtx_insn **before, rtx_insn **after, bool early_clobber_p)
1120 : {
1121 1781698 : bool out_conflict;
1122 1781698 : int i, in;
1123 1781698 : rtx new_in_reg, new_out_reg, reg;
1124 1781698 : machine_mode inmode, outmode;
1125 1781698 : rtx in_rtx = *curr_id->operand_loc[ins[0]];
1126 1781698 : rtx out_rtx = out < 0 ? in_rtx : *curr_id->operand_loc[out];
1127 :
1128 1781698 : inmode = curr_operand_mode[ins[0]];
1129 1781698 : outmode = out < 0 ? inmode : curr_operand_mode[out];
1130 1781698 : push_to_sequence (*before);
1131 1781698 : if (inmode != outmode)
1132 : {
1133 : /* process_alt_operands has already checked that the mode sizes
1134 : are ordered. */
1135 114311 : if (partial_subreg_p (outmode, inmode))
1136 : {
1137 2354 : bool asm_p = asm_noperands (PATTERN (curr_insn)) >= 0;
1138 2354 : int hr;
1139 2354 : HARD_REG_SET temp_hard_reg_set;
1140 :
1141 19 : if (asm_p && (hr = get_hard_regno (out_rtx)) >= 0
1142 2357 : && hard_regno_nregs (hr, inmode) > 1)
1143 : {
1144 : /* See gcc.c-torture/execute/20030222-1.c.
1145 : Consider the code for 32-bit (e.g. BE) target:
1146 : int i, v; long x; x = v; asm ("" : "=r" (i) : "0" (x));
1147 : We generate the following RTL with reload insns:
1148 : 1. subreg:si(x:di, 0) = 0;
1149 : 2. subreg:si(x:di, 4) = v:si;
1150 : 3. t:di = x:di, dead x;
1151 : 4. asm ("" : "=r" (subreg:si(t:di,4)) : "0" (t:di))
1152 : 5. i:si = subreg:si(t:di,4);
1153 : If we assign hard reg of x to t, dead code elimination
1154 : will remove insn #2 and we will use uninitialized hard reg.
1155 : So exclude the hard reg of x for t. We could ignore this
1156 : problem for non-empty asm using all x value but it is hard to
1157 : check that the asm are expanded into insn really using x
1158 : and setting r. */
1159 0 : CLEAR_HARD_REG_SET (temp_hard_reg_set);
1160 0 : if (exclude_start_hard_regs != NULL)
1161 0 : temp_hard_reg_set = *exclude_start_hard_regs;
1162 0 : SET_HARD_REG_BIT (temp_hard_reg_set, hr);
1163 0 : exclude_start_hard_regs = &temp_hard_reg_set;
1164 : }
1165 4708 : reg = new_in_reg
1166 2354 : = lra_create_new_reg_with_unique_value (inmode, in_rtx, goal_class,
1167 : exclude_start_hard_regs,
1168 : "");
1169 2354 : new_out_reg = get_matching_reload_reg_subreg (outmode, reg, goal_class);
1170 2354 : LRA_SUBREG_P (new_out_reg) = 1;
1171 : /* If the input reg is dying here, we can use the same hard
1172 : register for REG and IN_RTX. We do it only for original
1173 : pseudos as reload pseudos can die although original
1174 : pseudos still live where reload pseudos dies. */
1175 1847 : if (REG_P (in_rtx) && (int) REGNO (in_rtx) < lra_new_regno_start
1176 1791 : && find_regno_note (curr_insn, REG_DEAD, REGNO (in_rtx))
1177 3213 : && (!early_clobber_p
1178 2 : || check_conflict_input_operands(REGNO (in_rtx), ins)))
1179 858 : lra_assign_reg_val (REGNO (in_rtx), REGNO (reg));
1180 : }
1181 : else
1182 : {
1183 223914 : reg = new_out_reg
1184 111957 : = lra_create_new_reg_with_unique_value (outmode, out_rtx,
1185 : goal_class,
1186 : exclude_start_hard_regs,
1187 : "");
1188 111957 : new_in_reg = get_matching_reload_reg_subreg (inmode, reg, goal_class);
1189 : /* NEW_IN_REG is non-paradoxical subreg. We don't want
1190 : NEW_OUT_REG living above. We add clobber clause for
1191 : this. This is just a temporary clobber. We can remove
1192 : it at the end of LRA work. */
1193 111957 : rtx_insn *clobber = emit_clobber (new_out_reg);
1194 111957 : LRA_TEMP_CLOBBER_P (PATTERN (clobber)) = 1;
1195 111957 : LRA_SUBREG_P (new_in_reg) = 1;
1196 111957 : if (GET_CODE (in_rtx) == SUBREG)
1197 : {
1198 2082 : rtx subreg_reg = SUBREG_REG (in_rtx);
1199 :
1200 : /* If SUBREG_REG is dying here and sub-registers IN_RTX
1201 : and NEW_IN_REG are similar, we can use the same hard
1202 : register for REG and SUBREG_REG. */
1203 2082 : if (REG_P (subreg_reg)
1204 2082 : && (int) REGNO (subreg_reg) < lra_new_regno_start
1205 2082 : && GET_MODE (subreg_reg) == outmode
1206 1734 : && known_eq (SUBREG_BYTE (in_rtx), SUBREG_BYTE (new_in_reg))
1207 1734 : && find_regno_note (curr_insn, REG_DEAD, REGNO (subreg_reg))
1208 2387 : && (! early_clobber_p
1209 0 : || check_conflict_input_operands (REGNO (subreg_reg),
1210 : ins)))
1211 305 : lra_assign_reg_val (REGNO (subreg_reg), REGNO (reg));
1212 : }
1213 : }
1214 : }
1215 : else
1216 : {
1217 : /* Pseudos have values -- see comments for lra_reg_info.
1218 : Different pseudos with the same value do not conflict even if
1219 : they live in the same place. When we create a pseudo we
1220 : assign value of original pseudo (if any) from which we
1221 : created the new pseudo. If we create the pseudo from the
1222 : input pseudo, the new pseudo will have no conflict with the
1223 : input pseudo which is wrong when the input pseudo lives after
1224 : the insn and as the new pseudo value is changed by the insn
1225 : output. Therefore we create the new pseudo from the output
1226 : except the case when we have single matched dying input
1227 : pseudo.
1228 :
1229 : We cannot reuse the current output register because we might
1230 : have a situation like "a <- a op b", where the constraints
1231 : force the second input operand ("b") to match the output
1232 : operand ("a"). "b" must then be copied into a new register
1233 : so that it doesn't clobber the current value of "a".
1234 :
1235 : We cannot use the same value if the output pseudo is
1236 : early clobbered or the input pseudo is mentioned in the
1237 : output, e.g. as an address part in memory, because
1238 : output reload will actually extend the pseudo liveness.
1239 : We don't care about eliminable hard regs here as we are
1240 : interesting only in pseudos. */
1241 :
1242 : /* Matching input's register value is the same as one of the other
1243 : output operand. Output operands in a parallel insn must be in
1244 : different registers. */
1245 1667387 : out_conflict = false;
1246 1667387 : if (REG_P (in_rtx))
1247 : {
1248 2868427 : for (i = 0; outs[i] >= 0; i++)
1249 : {
1250 1494208 : rtx other_out_rtx = *curr_id->operand_loc[outs[i]];
1251 119811 : if (outs[i] != out && REG_P (other_out_rtx)
1252 1613818 : && (regno_val_use_in (REGNO (in_rtx), other_out_rtx)
1253 : != NULL_RTX))
1254 : {
1255 : out_conflict = true;
1256 : break;
1257 : }
1258 : }
1259 : }
1260 :
1261 1667387 : new_in_reg = new_out_reg
1262 1632337 : = (! early_clobber_p && ins[1] < 0 && REG_P (in_rtx)
1263 1340193 : && (int) REGNO (in_rtx) < lra_new_regno_start
1264 1339875 : && find_regno_note (curr_insn, REG_DEAD, REGNO (in_rtx))
1265 446056 : && (out < 0
1266 446056 : || regno_val_use_in (REGNO (in_rtx), out_rtx) == NULL_RTX)
1267 445990 : && !out_conflict
1268 2113375 : ? lra_create_new_reg (inmode, in_rtx, goal_class,
1269 : exclude_start_hard_regs, "")
1270 1221399 : : lra_create_new_reg_with_unique_value (outmode, out_rtx, goal_class,
1271 : exclude_start_hard_regs,
1272 : ""));
1273 : }
1274 : /* In operand can be got from transformations before processing insn
1275 : constraints. One example of such transformations is subreg
1276 : reloading (see function simplify_operand_subreg). The new
1277 : pseudos created by the transformations might have inaccurate
1278 : class (ALL_REGS) and we should make their classes more
1279 : accurate. */
1280 1781698 : narrow_reload_pseudo_class (in_rtx, goal_class);
1281 1781698 : lra_emit_move (copy_rtx (new_in_reg), in_rtx);
1282 1781698 : *before = end_sequence ();
1283 : /* Add the new pseudo to consider values of subsequent input reload
1284 : pseudos. */
1285 1781698 : lra_assert (curr_insn_input_reloads_num < LRA_MAX_INSN_RELOADS);
1286 1781698 : curr_insn_input_reloads[curr_insn_input_reloads_num].input = in_rtx;
1287 1781698 : curr_insn_input_reloads[curr_insn_input_reloads_num].match_p = true;
1288 1781698 : curr_insn_input_reloads[curr_insn_input_reloads_num].early_clobber_p = false;
1289 1781698 : curr_insn_input_reloads[curr_insn_input_reloads_num++].reg = new_in_reg;
1290 3563397 : for (i = 0; (in = ins[i]) >= 0; i++)
1291 1781699 : if (GET_MODE (*curr_id->operand_loc[in]) == VOIDmode
1292 1757977 : || GET_MODE (new_in_reg) == GET_MODE (*curr_id->operand_loc[in]))
1293 1781698 : *curr_id->operand_loc[in] = new_in_reg;
1294 : else
1295 : {
1296 1 : lra_assert
1297 : (GET_MODE (new_out_reg) == GET_MODE (*curr_id->operand_loc[in]));
1298 1 : *curr_id->operand_loc[in] = new_out_reg;
1299 : }
1300 1781698 : lra_update_dups (curr_id, ins);
1301 1781698 : if (out < 0)
1302 : return;
1303 : /* See a comment for the input operand above. */
1304 1781698 : narrow_reload_pseudo_class (out_rtx, goal_class);
1305 1781698 : reg = SUBREG_P (out_rtx) ? SUBREG_REG (out_rtx) : out_rtx;
1306 1781698 : if (find_reg_note (curr_insn, REG_UNUSED, reg) == NULL_RTX
1307 1781698 : && (!REG_P (reg) || !ira_former_scratch_p (REGNO (reg))))
1308 : {
1309 1683437 : start_sequence ();
1310 : /* If we had strict_low_part, use it also in reload to keep other
1311 : parts unchanged but do it only for regs as strict_low_part
1312 : has no sense for memory and probably there is no insn pattern
1313 : to match the reload insn in memory case. */
1314 1683437 : if (out >= 0 && curr_static_id->operand[out].strict_low && REG_P (reg))
1315 0 : out_rtx = gen_rtx_STRICT_LOW_PART (VOIDmode, out_rtx);
1316 1683437 : lra_emit_move (out_rtx, copy_rtx (new_out_reg));
1317 1683437 : emit_insn (*after);
1318 1683437 : *after = end_sequence ();
1319 : }
1320 1781698 : *curr_id->operand_loc[out] = new_out_reg;
1321 1781698 : lra_update_dup (curr_id, out);
1322 : }
1323 :
1324 : /* Return register class which is union of all reg classes in insn
1325 : constraint alternative string starting with P. */
1326 : static enum reg_class
1327 0 : reg_class_from_constraints (const char *p)
1328 : {
1329 0 : int c, len;
1330 0 : enum reg_class op_class = NO_REGS;
1331 :
1332 0 : do
1333 0 : switch ((c = *p, len = CONSTRAINT_LEN (c, p)), c)
1334 : {
1335 : case '#':
1336 : case ',':
1337 : return op_class;
1338 :
1339 0 : case 'g':
1340 0 : op_class = reg_class_subunion[op_class][GENERAL_REGS];
1341 0 : break;
1342 :
1343 0 : default:
1344 0 : enum constraint_num cn = lookup_constraint (p);
1345 0 : enum reg_class cl = reg_class_for_constraint (cn);
1346 0 : if (cl == NO_REGS)
1347 : {
1348 0 : if (insn_extra_address_constraint (cn))
1349 0 : op_class
1350 0 : = (reg_class_subunion
1351 0 : [op_class][base_reg_class (VOIDmode, ADDR_SPACE_GENERIC,
1352 0 : ADDRESS, SCRATCH)]);
1353 : break;
1354 : }
1355 :
1356 0 : op_class = reg_class_subunion[op_class][cl];
1357 0 : break;
1358 : }
1359 0 : while ((p += len), c);
1360 : return op_class;
1361 : }
1362 :
1363 : /* If OP is a register, return the class of the register as per
1364 : get_reg_class, otherwise return NO_REGS. */
1365 : static inline enum reg_class
1366 167378490 : get_op_class (rtx op)
1367 : {
1368 139115256 : return REG_P (op) ? get_reg_class (REGNO (op)) : NO_REGS;
1369 : }
1370 :
1371 : /* Return generated insn mem_pseudo:=val if TO_P or val:=mem_pseudo
1372 : otherwise. If modes of MEM_PSEUDO and VAL are different, use
1373 : SUBREG for VAL to make them equal. */
1374 : static rtx_insn *
1375 1352482 : emit_spill_move (bool to_p, rtx mem_pseudo, rtx val)
1376 : {
1377 1352482 : if (GET_MODE (mem_pseudo) != GET_MODE (val))
1378 : {
1379 : /* Usually size of mem_pseudo is greater than val size but in
1380 : rare cases it can be less as it can be defined by target
1381 : dependent macro HARD_REGNO_CALLER_SAVE_MODE. */
1382 3352 : if (! MEM_P (val))
1383 : {
1384 3352 : val = gen_lowpart_SUBREG (GET_MODE (mem_pseudo),
1385 : GET_CODE (val) == SUBREG
1386 : ? SUBREG_REG (val) : val);
1387 3352 : LRA_SUBREG_P (val) = 1;
1388 : }
1389 : else
1390 : {
1391 0 : mem_pseudo = gen_lowpart_SUBREG (GET_MODE (val), mem_pseudo);
1392 0 : LRA_SUBREG_P (mem_pseudo) = 1;
1393 : }
1394 : }
1395 1352482 : return to_p ? gen_move_insn (mem_pseudo, val)
1396 680413 : : gen_move_insn (val, mem_pseudo);
1397 : }
1398 :
1399 : /* Process a special case insn (register move), return true if we
1400 : don't need to process it anymore. INSN should be a single set
1401 : insn. Set up that RTL was changed through CHANGE_P and that hook
1402 : TARGET_SECONDARY_MEMORY_NEEDED says to use secondary memory through
1403 : SEC_MEM_P. */
1404 : static bool
1405 77491451 : check_and_process_move (bool *change_p, bool *sec_mem_p ATTRIBUTE_UNUSED)
1406 : {
1407 77491451 : int sregno, dregno;
1408 77491451 : rtx dest, src, dreg, sreg, new_reg, scratch_reg;
1409 77491451 : rtx_insn *before;
1410 77491451 : enum reg_class dclass, sclass, secondary_class;
1411 77491451 : secondary_reload_info sri;
1412 :
1413 77491451 : lra_assert (curr_insn_set != NULL_RTX);
1414 77491451 : dreg = dest = SET_DEST (curr_insn_set);
1415 77491451 : sreg = src = SET_SRC (curr_insn_set);
1416 77491451 : if (GET_CODE (dest) == SUBREG)
1417 1207143 : dreg = SUBREG_REG (dest);
1418 77491451 : if (GET_CODE (src) == SUBREG)
1419 1293544 : sreg = SUBREG_REG (src);
1420 77491451 : if (! (REG_P (dreg) || MEM_P (dreg)) || ! (REG_P (sreg) || MEM_P (sreg)))
1421 : return false;
1422 36109003 : sclass = dclass = NO_REGS;
1423 36109003 : if (REG_P (dreg))
1424 23253066 : dclass = get_reg_class (REGNO (dreg));
1425 23253066 : gcc_assert (dclass < LIM_REG_CLASSES && dclass >= NO_REGS);
1426 36109003 : if (dclass == ALL_REGS)
1427 : /* ALL_REGS is used for new pseudos created by transformations
1428 : like reload of SUBREG_REG (see function
1429 : simplify_operand_subreg). We don't know their class yet. We
1430 : should figure out the class from processing the insn
1431 : constraints not in this fast path function. Even if ALL_REGS
1432 : were a right class for the pseudo, secondary_... hooks usually
1433 : are not define for ALL_REGS. */
1434 : return false;
1435 36106727 : if (REG_P (sreg))
1436 20187228 : sclass = get_reg_class (REGNO (sreg));
1437 20187228 : gcc_assert (sclass < LIM_REG_CLASSES && sclass >= NO_REGS);
1438 36106727 : if (sclass == ALL_REGS)
1439 : /* See comments above. */
1440 : return false;
1441 36106727 : if (sclass == NO_REGS && dclass == NO_REGS)
1442 : return false;
1443 34621754 : if (targetm.secondary_memory_needed (GET_MODE (src), sclass, dclass)
1444 34621754 : && ((sclass != NO_REGS && dclass != NO_REGS)
1445 0 : || (GET_MODE (src)
1446 0 : != targetm.secondary_memory_needed_mode (GET_MODE (src)))))
1447 : {
1448 12950 : *sec_mem_p = true;
1449 12950 : return false;
1450 : }
1451 34608804 : if (! REG_P (dreg) || ! REG_P (sreg))
1452 : return false;
1453 7843492 : sri.prev_sri = NULL;
1454 7843492 : sri.icode = CODE_FOR_nothing;
1455 7843492 : sri.extra_cost = 0;
1456 7843492 : secondary_class = NO_REGS;
1457 : /* Set up hard register for a reload pseudo for hook
1458 : secondary_reload because some targets just ignore unassigned
1459 : pseudos in the hook. */
1460 7843492 : if (dclass != NO_REGS && lra_get_regno_hard_regno (REGNO (dreg)) < 0)
1461 : {
1462 2918702 : dregno = REGNO (dreg);
1463 2918702 : reg_renumber[dregno] = ira_class_hard_regs[dclass][0];
1464 : }
1465 : else
1466 : dregno = -1;
1467 7843492 : if (sclass != NO_REGS && lra_get_regno_hard_regno (REGNO (sreg)) < 0)
1468 : {
1469 1294108 : sregno = REGNO (sreg);
1470 1294108 : reg_renumber[sregno] = ira_class_hard_regs[sclass][0];
1471 : }
1472 : else
1473 : sregno = -1;
1474 7843492 : if (sclass != NO_REGS)
1475 3979854 : secondary_class
1476 7959708 : = (enum reg_class) targetm.secondary_reload (false, dest,
1477 : (reg_class_t) sclass,
1478 3979854 : GET_MODE (src), &sri);
1479 3979854 : if (sclass == NO_REGS
1480 3979854 : || ((secondary_class != NO_REGS || sri.icode != CODE_FOR_nothing)
1481 1339 : && dclass != NO_REGS))
1482 : {
1483 3863638 : enum reg_class old_sclass = secondary_class;
1484 3863638 : secondary_reload_info old_sri = sri;
1485 :
1486 3863638 : sri.prev_sri = NULL;
1487 3863638 : sri.icode = CODE_FOR_nothing;
1488 3863638 : sri.extra_cost = 0;
1489 3863638 : secondary_class
1490 7727276 : = (enum reg_class) targetm.secondary_reload (true, src,
1491 : (reg_class_t) dclass,
1492 3863638 : GET_MODE (src), &sri);
1493 : /* Check the target hook consistency. */
1494 3863638 : lra_assert
1495 : ((secondary_class == NO_REGS && sri.icode == CODE_FOR_nothing)
1496 : || (old_sclass == NO_REGS && old_sri.icode == CODE_FOR_nothing)
1497 : || (secondary_class == old_sclass && sri.icode == old_sri.icode));
1498 : }
1499 7843492 : if (sregno >= 0)
1500 1294108 : reg_renumber [sregno] = -1;
1501 7843492 : if (dregno >= 0)
1502 2918702 : reg_renumber [dregno] = -1;
1503 7843492 : if (secondary_class == NO_REGS && sri.icode == CODE_FOR_nothing)
1504 : return false;
1505 1340 : *change_p = true;
1506 1340 : new_reg = NULL_RTX;
1507 0 : if (secondary_class != NO_REGS)
1508 1340 : new_reg = lra_create_new_reg_with_unique_value (GET_MODE (src), NULL_RTX,
1509 : secondary_class, NULL,
1510 : "secondary");
1511 1340 : start_sequence ();
1512 1340 : if (sri.icode == CODE_FOR_nothing)
1513 1340 : lra_emit_move (new_reg, src);
1514 : else
1515 : {
1516 0 : enum reg_class scratch_class;
1517 :
1518 0 : scratch_class = (reg_class_from_constraints
1519 0 : (insn_data[sri.icode].operand[2].constraint));
1520 0 : scratch_reg = (lra_create_new_reg_with_unique_value
1521 0 : (insn_data[sri.icode].operand[2].mode, NULL_RTX,
1522 : scratch_class, NULL, "scratch"));
1523 0 : emit_insn (GEN_FCN (sri.icode) (new_reg != NULL_RTX ? new_reg : dest,
1524 : src, scratch_reg));
1525 : }
1526 1340 : before = end_sequence ();
1527 1340 : lra_process_new_insns (curr_insn, before, NULL, "Inserting the move");
1528 1340 : if (new_reg != NULL_RTX)
1529 1340 : SET_SRC (curr_insn_set) = new_reg;
1530 : else
1531 : {
1532 0 : if (lra_dump_file != NULL)
1533 : {
1534 0 : fprintf (lra_dump_file, "Deleting move %u\n", INSN_UID (curr_insn));
1535 0 : dump_insn_slim (lra_dump_file, curr_insn);
1536 : }
1537 0 : lra_set_insn_deleted (curr_insn);
1538 0 : return true;
1539 : }
1540 1340 : return false;
1541 : }
1542 :
1543 : /* The following data describe the result of process_alt_operands.
1544 : The data are used in curr_insn_transform to generate reloads. */
1545 :
1546 : /* The chosen reg classes which should be used for the corresponding
1547 : operands. */
1548 : static enum reg_class goal_alt[MAX_RECOG_OPERANDS];
1549 : /* Hard registers which cannot be a start hard register for the corresponding
1550 : operands. */
1551 : static HARD_REG_SET goal_alt_exclude_start_hard_regs[MAX_RECOG_OPERANDS];
1552 : /* True if the operand should be the same as another operand and that
1553 : other operand does not need a reload. */
1554 : static bool goal_alt_match_win[MAX_RECOG_OPERANDS];
1555 : /* True if the operand does not need a reload. */
1556 : static bool goal_alt_win[MAX_RECOG_OPERANDS];
1557 : /* True if the operand can be offsetable memory. */
1558 : static bool goal_alt_offmemok[MAX_RECOG_OPERANDS];
1559 : /* The number of an operand to which given operand can be matched to. */
1560 : static int goal_alt_matches[MAX_RECOG_OPERANDS];
1561 : /* The number of elements in the following array. */
1562 : static int goal_alt_dont_inherit_ops_num;
1563 : /* Numbers of operands whose reload pseudos should not be inherited. */
1564 : static int goal_alt_dont_inherit_ops[MAX_RECOG_OPERANDS];
1565 : /* True if we should try only this alternative for the next constraint sub-pass
1566 : to speed up the sub-pass. */
1567 : static bool goal_reuse_alt_p;
1568 : /* True if the insn commutative operands should be swapped. */
1569 : static bool goal_alt_swapped;
1570 : /* The chosen insn alternative. */
1571 : static int goal_alt_number;
1572 : /* True if output reload of the stack pointer should be generated. */
1573 : static bool goal_alt_out_sp_reload_p;
1574 :
1575 : /* True if the corresponding operand is the result of an equivalence
1576 : substitution. */
1577 : static bool equiv_substition_p[MAX_RECOG_OPERANDS];
1578 :
1579 : /* The following five variables are used to choose the best insn
1580 : alternative. They reflect final characteristics of the best
1581 : alternative. */
1582 :
1583 : /* Number of necessary reloads and overall cost reflecting the
1584 : previous value and other unpleasantness of the best alternative. */
1585 : static int best_losers, best_overall;
1586 : /* Overall number hard registers used for reloads. For example, on
1587 : some targets we need 2 general registers to reload DFmode and only
1588 : one floating point register. */
1589 : static int best_reload_nregs;
1590 : /* Overall number reflecting distances of previous reloading the same
1591 : value. The distances are counted from the current BB start. It is
1592 : used to improve inheritance chances. */
1593 : static int best_reload_sum;
1594 :
1595 : /* True if the current insn should have no correspondingly input or
1596 : output reloads. */
1597 : static bool no_input_reloads_p, no_output_reloads_p;
1598 :
1599 : /* True if we swapped the commutative operands in the current
1600 : insn. */
1601 : static int curr_swapped;
1602 :
1603 : /* if CHECK_ONLY_P is false, arrange for address element *LOC to be a
1604 : register of class CL. Add any input reloads to list BEFORE. AFTER
1605 : is nonnull if *LOC is an automodified value; handle that case by
1606 : adding the required output reloads to list AFTER. Return true if
1607 : the RTL was changed.
1608 :
1609 : if CHECK_ONLY_P is true, check that the *LOC is a correct address
1610 : register. Return false if the address register is correct. */
1611 : static bool
1612 35807032 : process_addr_reg (rtx *loc, bool check_only_p, rtx_insn **before, rtx_insn **after,
1613 : enum reg_class cl)
1614 : {
1615 35807032 : int regno;
1616 35807032 : enum reg_class rclass, new_class;
1617 35807032 : rtx reg;
1618 35807032 : rtx new_reg;
1619 35807032 : machine_mode mode;
1620 35807032 : bool subreg_p, before_p = false;
1621 :
1622 35807032 : subreg_p = GET_CODE (*loc) == SUBREG;
1623 35807032 : if (subreg_p)
1624 : {
1625 16633 : reg = SUBREG_REG (*loc);
1626 16633 : mode = GET_MODE (reg);
1627 :
1628 : /* For mode with size bigger than ptr_mode, there unlikely to be "mov"
1629 : between two registers with different classes, but there normally will
1630 : be "mov" which transfers element of vector register into the general
1631 : register, and this normally will be a subreg which should be reloaded
1632 : as a whole. This is particularly likely to be triggered when
1633 : -fno-split-wide-types specified. */
1634 16633 : if (!REG_P (reg)
1635 16633 : || in_class_p (reg, cl, &new_class)
1636 18997 : || known_le (GET_MODE_SIZE (mode), GET_MODE_SIZE (ptr_mode)))
1637 16633 : loc = &SUBREG_REG (*loc);
1638 : }
1639 :
1640 35807032 : reg = *loc;
1641 35807032 : mode = GET_MODE (reg);
1642 35807032 : if (! REG_P (reg))
1643 : {
1644 0 : if (check_only_p)
1645 : return true;
1646 : /* Always reload memory in an address even if the target supports
1647 : such addresses. */
1648 0 : new_reg = lra_create_new_reg_with_unique_value (mode, reg, cl, NULL,
1649 : "address");
1650 0 : before_p = true;
1651 : }
1652 : else
1653 : {
1654 35807032 : regno = REGNO (reg);
1655 35807032 : rclass = get_reg_class (regno);
1656 35807032 : if (! check_only_p
1657 35807032 : && (*loc = get_equiv_with_elimination (reg, curr_insn)) != reg)
1658 : {
1659 113138 : if (lra_dump_file != NULL)
1660 : {
1661 0 : fprintf (lra_dump_file,
1662 : "Changing pseudo %d in address of insn %u on equiv ",
1663 0 : REGNO (reg), INSN_UID (curr_insn));
1664 0 : dump_value_slim (lra_dump_file, *loc, 1);
1665 0 : fprintf (lra_dump_file, "\n");
1666 : }
1667 113138 : rtx new_equiv = copy_rtx (*loc);
1668 113138 : if (lra_pointer_equiv_set_in (*loc))
1669 107793 : lra_pointer_equiv_set_add (new_equiv);
1670 113138 : *loc = new_equiv;
1671 : }
1672 35807032 : if (*loc != reg || ! in_class_p (reg, cl, &new_class))
1673 : {
1674 481989 : if (check_only_p)
1675 : return true;
1676 481989 : reg = *loc;
1677 481989 : if (get_reload_reg (after == NULL ? OP_IN : OP_INOUT,
1678 : mode, reg, cl, NULL,
1679 : subreg_p, false, "address", &new_reg))
1680 : before_p = true;
1681 : }
1682 35325043 : else if (new_class != NO_REGS && rclass != new_class)
1683 : {
1684 469769 : if (check_only_p)
1685 : return true;
1686 469769 : lra_change_class (regno, new_class, " Change to", true);
1687 469769 : return false;
1688 : }
1689 : else
1690 : return false;
1691 : }
1692 0 : if (before_p)
1693 : {
1694 473825 : push_to_sequence (*before);
1695 473825 : lra_emit_move (new_reg, reg);
1696 473825 : *before = end_sequence ();
1697 : }
1698 481989 : *loc = new_reg;
1699 481989 : if (after != NULL)
1700 : {
1701 0 : start_sequence ();
1702 0 : lra_emit_move (before_p ? copy_rtx (reg) : reg, new_reg);
1703 0 : emit_insn (*after);
1704 0 : *after = end_sequence ();
1705 : }
1706 : return true;
1707 : }
1708 :
1709 : /* Insert move insn in simplify_operand_subreg. BEFORE returns
1710 : the insn to be inserted before curr insn. AFTER returns the
1711 : the insn to be inserted after curr insn. ORIGREG and NEWREG
1712 : are the original reg and new reg for reload. */
1713 : static void
1714 458 : insert_move_for_subreg (rtx_insn **before, rtx_insn **after, rtx origreg,
1715 : rtx newreg)
1716 : {
1717 458 : if (before)
1718 : {
1719 458 : push_to_sequence (*before);
1720 458 : lra_emit_move (newreg, origreg);
1721 458 : *before = end_sequence ();
1722 : }
1723 458 : if (after)
1724 : {
1725 0 : start_sequence ();
1726 0 : lra_emit_move (origreg, newreg);
1727 0 : emit_insn (*after);
1728 0 : *after = end_sequence ();
1729 : }
1730 458 : }
1731 :
1732 : static bool valid_address_p (machine_mode mode, rtx addr, addr_space_t as);
1733 : static bool process_address (int, bool, rtx_insn **, rtx_insn **);
1734 :
1735 : /* Make reloads for subreg in operand NOP with internal subreg mode
1736 : REG_MODE, add new reloads for further processing. Return true if
1737 : any change was done. */
1738 : static bool
1739 180174206 : simplify_operand_subreg (int nop, machine_mode reg_mode)
1740 : {
1741 180174206 : int hard_regno, inner_hard_regno;
1742 180174206 : rtx_insn *before, *after;
1743 180174206 : machine_mode mode, innermode;
1744 180174206 : rtx reg, new_reg;
1745 180174206 : rtx operand = *curr_id->operand_loc[nop];
1746 180174206 : enum reg_class regclass;
1747 180174206 : enum op_type type;
1748 :
1749 180174206 : before = after = NULL;
1750 :
1751 180174206 : if (GET_CODE (operand) != SUBREG)
1752 : return false;
1753 :
1754 3845030 : mode = GET_MODE (operand);
1755 3845030 : reg = SUBREG_REG (operand);
1756 3845030 : innermode = GET_MODE (reg);
1757 3845030 : type = curr_static_id->operand[nop].type;
1758 3845030 : if (MEM_P (reg))
1759 : {
1760 11396 : const bool addr_was_valid
1761 11396 : = valid_address_p (innermode, XEXP (reg, 0), MEM_ADDR_SPACE (reg));
1762 11396 : alter_subreg (curr_id->operand_loc[nop], false);
1763 11396 : rtx subst = *curr_id->operand_loc[nop];
1764 11396 : lra_assert (MEM_P (subst));
1765 11396 : const bool addr_is_valid = valid_address_p (GET_MODE (subst),
1766 : XEXP (subst, 0),
1767 11396 : MEM_ADDR_SPACE (subst));
1768 11396 : if (!addr_was_valid
1769 11396 : || addr_is_valid
1770 11396 : || ((get_constraint_type (lookup_constraint
1771 0 : (curr_static_id->operand[nop].constraint))
1772 : != CT_SPECIAL_MEMORY)
1773 : /* We still can reload address and if the address is
1774 : valid, we can remove subreg without reloading its
1775 : inner memory. */
1776 0 : && valid_address_p (GET_MODE (subst),
1777 0 : regno_reg_rtx
1778 : [ira_class_hard_regs
1779 0 : [base_reg_class (GET_MODE (subst),
1780 0 : MEM_ADDR_SPACE (subst),
1781 : ADDRESS, SCRATCH,
1782 0 : subst, curr_insn)][0]],
1783 0 : MEM_ADDR_SPACE (subst))))
1784 : {
1785 : /* If we change the address for a paradoxical subreg of memory, the
1786 : new address might violate the necessary alignment or the access
1787 : might be slow; take this into consideration. We need not worry
1788 : about accesses beyond allocated memory for paradoxical memory
1789 : subregs as we don't substitute such equiv memory (see processing
1790 : equivalences in function lra_constraints) and because for spilled
1791 : pseudos we allocate stack memory enough for the biggest
1792 : corresponding paradoxical subreg.
1793 :
1794 : However, do not blindly simplify a (subreg (mem ...)) for
1795 : WORD_REGISTER_OPERATIONS targets as this may lead to loading junk
1796 : data into a register when the inner is narrower than outer or
1797 : missing important data from memory when the inner is wider than
1798 : outer. This rule only applies to modes that are no wider than
1799 : a word.
1800 :
1801 : If valid memory becomes invalid after subreg elimination
1802 : and address might be different we still have to reload
1803 : memory.
1804 : */
1805 11396 : if ((! addr_was_valid
1806 : || addr_is_valid
1807 0 : || known_eq (GET_MODE_SIZE (mode), GET_MODE_SIZE (innermode)))
1808 11396 : && !(maybe_ne (GET_MODE_PRECISION (mode),
1809 11396 : GET_MODE_PRECISION (innermode))
1810 13092 : && known_le (GET_MODE_SIZE (mode), UNITS_PER_WORD)
1811 19792 : && known_le (GET_MODE_SIZE (innermode), UNITS_PER_WORD)
1812 : && WORD_REGISTER_OPERATIONS)
1813 23887 : && (!(MEM_ALIGN (subst) < GET_MODE_ALIGNMENT (mode)
1814 1095 : && targetm.slow_unaligned_access (mode, MEM_ALIGN (subst)))
1815 0 : || (MEM_ALIGN (reg) < GET_MODE_ALIGNMENT (innermode)
1816 0 : && targetm.slow_unaligned_access (innermode,
1817 0 : MEM_ALIGN (reg)))))
1818 : return true;
1819 :
1820 0 : *curr_id->operand_loc[nop] = operand;
1821 :
1822 : /* But if the address was not valid, we cannot reload the MEM without
1823 : reloading the address first. */
1824 0 : if (!addr_was_valid)
1825 0 : process_address (nop, false, &before, &after);
1826 :
1827 : /* INNERMODE is fast, MODE slow. Reload the mem in INNERMODE. */
1828 0 : enum reg_class rclass
1829 0 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
1830 0 : if (get_reload_reg (curr_static_id->operand[nop].type, innermode,
1831 : reg, rclass, NULL,
1832 : true, false, "slow/invalid mem", &new_reg))
1833 : {
1834 0 : bool insert_before, insert_after;
1835 0 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
1836 :
1837 0 : insert_before = (type != OP_OUT
1838 0 : || partial_subreg_p (mode, innermode));
1839 0 : insert_after = type != OP_IN;
1840 0 : insert_move_for_subreg (insert_before ? &before : NULL,
1841 : insert_after ? &after : NULL,
1842 : reg, new_reg);
1843 : }
1844 0 : SUBREG_REG (operand) = new_reg;
1845 :
1846 : /* Convert to MODE. */
1847 0 : reg = operand;
1848 0 : rclass
1849 0 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
1850 0 : if (get_reload_reg (curr_static_id->operand[nop].type, mode, reg,
1851 : rclass, NULL,
1852 : true, false, "slow/invalid mem", &new_reg))
1853 : {
1854 0 : bool insert_before, insert_after;
1855 0 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
1856 :
1857 0 : insert_before = type != OP_OUT;
1858 0 : insert_after = type != OP_IN;
1859 0 : insert_move_for_subreg (insert_before ? &before : NULL,
1860 : insert_after ? &after : NULL,
1861 : reg, new_reg);
1862 : }
1863 0 : *curr_id->operand_loc[nop] = new_reg;
1864 0 : lra_process_new_insns (curr_insn, before, after,
1865 : "Inserting slow/invalid mem reload");
1866 0 : return true;
1867 : }
1868 :
1869 : /* If the address was valid and became invalid, prefer to reload
1870 : the memory. Typical case is when the index scale should
1871 : correspond the memory. */
1872 0 : *curr_id->operand_loc[nop] = operand;
1873 : /* Do not return false here as the MEM_P (reg) will be processed
1874 : later in this function. */
1875 : }
1876 3833634 : else if (REG_P (reg) && REGNO (reg) < FIRST_PSEUDO_REGISTER)
1877 : {
1878 : /* A narrowing subreg of a hard register that is not representable as a
1879 : hard register (its offset does not fall on a register boundary) cannot
1880 : be turned into one, and would otherwise be resolved to the wrong part
1881 : of the register. Reload it through memory so that the correct bytes
1882 : are accessed, as is done for pseudos below. Leave the frame, arg and
1883 : stack pointers alone: simplify_subreg_regno can reject them simply
1884 : because reload is not finished yet. */
1885 65 : if (partial_subreg_p (mode, innermode)
1886 65 : && REGNO (reg) != FRAME_POINTER_REGNUM
1887 : && REGNO (reg) != ARG_POINTER_REGNUM
1888 : && REGNO (reg) != STACK_POINTER_REGNUM
1889 70 : && simplify_subreg_regno (REGNO (reg), innermode,
1890 5 : SUBREG_BYTE (operand), mode) < 0)
1891 : {
1892 0 : if (get_reload_reg (type, innermode, reg, NO_REGS, NULL,
1893 : true, false, "non-representable subreg", &new_reg))
1894 : {
1895 0 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
1896 0 : bool insert_before = (type != OP_OUT
1897 0 : || read_modify_subreg_p (operand));
1898 0 : bool insert_after = (type != OP_IN);
1899 0 : insert_move_for_subreg (insert_before ? &before : NULL,
1900 : insert_after ? &after : NULL,
1901 : reg, new_reg);
1902 : }
1903 0 : SUBREG_REG (operand) = new_reg;
1904 0 : lra_process_new_insns (curr_insn, before, after,
1905 : "Inserting non-representable subreg reload");
1906 0 : return true;
1907 : }
1908 65 : alter_subreg (curr_id->operand_loc[nop], false);
1909 65 : return true;
1910 : }
1911 3833569 : else if (CONSTANT_P (reg))
1912 : {
1913 : /* Try to simplify subreg of constant. It is usually result of
1914 : equivalence substitution. */
1915 43942 : if (innermode == VOIDmode
1916 43942 : && (innermode = original_subreg_reg_mode[nop]) == VOIDmode)
1917 0 : innermode = curr_static_id->operand[nop].mode;
1918 43942 : if ((new_reg = simplify_subreg (mode, reg, innermode,
1919 43942 : SUBREG_BYTE (operand))) != NULL_RTX)
1920 : {
1921 43526 : *curr_id->operand_loc[nop] = new_reg;
1922 43526 : return true;
1923 : }
1924 : }
1925 : /* Put constant into memory when we have mixed modes. It generates
1926 : a better code in most cases as it does not need a secondary
1927 : reload memory. It also prevents LRA looping when LRA is using
1928 : secondary reload memory again and again. */
1929 832 : if (CONSTANT_P (reg) && CONST_POOL_OK_P (reg_mode, reg)
1930 3790459 : && SCALAR_INT_MODE_P (reg_mode) != SCALAR_INT_MODE_P (mode))
1931 : {
1932 8 : SUBREG_REG (operand) = force_const_mem (reg_mode, reg);
1933 8 : alter_subreg (curr_id->operand_loc[nop], false);
1934 8 : return true;
1935 : }
1936 3790035 : auto fp_subreg_can_be_simplified_after_reload_p = [] (machine_mode innermode,
1937 : poly_uint64 offset,
1938 : machine_mode mode) {
1939 0 : reload_completed = 1;
1940 0 : bool res = simplify_subreg_regno (FRAME_POINTER_REGNUM,
1941 : innermode,
1942 0 : offset, mode) >= 0;
1943 0 : reload_completed = 0;
1944 0 : return res;
1945 : };
1946 : /* Force a reload of the SUBREG_REG if this is a constant or PLUS or
1947 : if there may be a problem accessing OPERAND in the outer
1948 : mode. */
1949 3790035 : if ((REG_P (reg)
1950 3789577 : && REGNO (reg) >= FIRST_PSEUDO_REGISTER
1951 3789577 : && (hard_regno = lra_get_regno_hard_regno (REGNO (reg))) >= 0
1952 : /* Don't reload paradoxical subregs because we could be looping
1953 : having repeatedly final regno out of hard regs range. */
1954 3209032 : && (hard_regno_nregs (hard_regno, innermode)
1955 3209032 : >= hard_regno_nregs (hard_regno, mode))
1956 3203826 : && simplify_subreg_regno (hard_regno, innermode,
1957 3203826 : SUBREG_BYTE (operand), mode) < 0
1958 : /* Exclude reloading of frame pointer in subreg if frame pointer can not
1959 : be simplified here only because the reload is not finished yet. */
1960 846 : && (hard_regno != FRAME_POINTER_REGNUM
1961 458 : || !fp_subreg_can_be_simplified_after_reload_p (innermode,
1962 0 : SUBREG_BYTE (operand),
1963 : mode))
1964 : /* Don't reload subreg for matching reload. It is actually
1965 : valid subreg in LRA. */
1966 846 : && ! LRA_SUBREG_P (operand))
1967 7579612 : || CONSTANT_P (reg) || GET_CODE (reg) == PLUS || MEM_P (reg))
1968 : {
1969 458 : enum reg_class rclass;
1970 :
1971 458 : if (REG_P (reg))
1972 : /* There is a big probability that we will get the same class
1973 : for the new pseudo and we will get the same insn which
1974 : means infinite looping. So spill the new pseudo. */
1975 : rclass = NO_REGS;
1976 : else
1977 : /* The class will be defined later in curr_insn_transform. */
1978 458 : rclass
1979 458 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
1980 :
1981 458 : if (get_reload_reg (curr_static_id->operand[nop].type, reg_mode, reg,
1982 : rclass, NULL,
1983 : true, false, "subreg reg", &new_reg))
1984 : {
1985 458 : bool insert_before, insert_after;
1986 458 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
1987 :
1988 916 : insert_before = (type != OP_OUT
1989 458 : || read_modify_subreg_p (operand));
1990 458 : insert_after = (type != OP_IN);
1991 916 : insert_move_for_subreg (insert_before ? &before : NULL,
1992 : insert_after ? &after : NULL,
1993 : reg, new_reg);
1994 : }
1995 458 : SUBREG_REG (operand) = new_reg;
1996 458 : lra_process_new_insns (curr_insn, before, after,
1997 : "Inserting subreg reload");
1998 458 : return true;
1999 : }
2000 : /* Force a reload for a paradoxical subreg. For paradoxical subreg,
2001 : IRA allocates hardreg to the inner pseudo reg according to its mode
2002 : instead of the outermode, so the size of the hardreg may not be enough
2003 : to contain the outermode operand, in that case we may need to insert
2004 : reload for the reg. For the following two types of paradoxical subreg,
2005 : we need to insert reload:
2006 : 1. If the op_type is OP_IN, and the hardreg could not be paired with
2007 : other hardreg to contain the outermode operand
2008 : (checked by in_hard_reg_set_p), we need to insert the reload.
2009 : 2. If the op_type is OP_OUT or OP_INOUT.
2010 :
2011 : Here is a paradoxical subreg example showing how the reload is generated:
2012 :
2013 : (insn 5 4 7 2 (set (reg:TI 106 [ __comp ])
2014 : (subreg:TI (reg:DI 107 [ __comp ]) 0)) {*movti_internal_rex64}
2015 :
2016 : In IRA, reg107 is allocated to a DImode hardreg. We use x86-64 as example
2017 : here, if reg107 is assigned to hardreg R15, because R15 is the last
2018 : hardreg, compiler cannot find another hardreg to pair with R15 to
2019 : contain TImode data. So we insert a TImode reload reg180 for it.
2020 : After reload is inserted:
2021 :
2022 : (insn 283 0 0 (set (subreg:DI (reg:TI 180 [orig:107 __comp ] [107]) 0)
2023 : (reg:DI 107 [ __comp ])) -1
2024 : (insn 5 4 7 2 (set (reg:TI 106 [ __comp ])
2025 : (subreg:TI (reg:TI 180 [orig:107 __comp ] [107]) 0)) {*movti_internal_rex64}
2026 :
2027 : Two reload hard registers will be allocated to reg180 to save TImode data
2028 : in LRA_assign.
2029 :
2030 : For LRA pseudos this should normally be handled by the biggest_mode
2031 : mechanism. However, it's possible for new uses of an LRA pseudo
2032 : to be introduced after we've allocated it, such as when undoing
2033 : inheritance, and the allocated register might not then be appropriate
2034 : for the new uses. */
2035 3789577 : else if (REG_P (reg)
2036 3789577 : && REGNO (reg) >= FIRST_PSEUDO_REGISTER
2037 3789577 : && paradoxical_subreg_p (operand)
2038 1022387 : && (inner_hard_regno = lra_get_regno_hard_regno (REGNO (reg))) >= 0
2039 945648 : && hard_regno_nregs (inner_hard_regno, mode) > 1
2040 3789577 : && ((hard_regno
2041 3794783 : = simplify_subreg_regno (inner_hard_regno, innermode,
2042 5206 : SUBREG_BYTE (operand), mode)) < 0
2043 5206 : || ((hard_regno_nregs (inner_hard_regno, innermode)
2044 5206 : < hard_regno_nregs (hard_regno, mode))
2045 10412 : && (regclass = lra_get_allocno_class (REGNO (reg)))
2046 5206 : && (type != OP_IN
2047 5206 : || !in_hard_reg_set_p (reg_class_contents[regclass],
2048 : mode, hard_regno)
2049 5206 : || overlaps_hard_reg_set_p (lra_no_alloc_regs,
2050 : mode, hard_regno)))))
2051 : {
2052 : /* The class will be defined later in curr_insn_transform. */
2053 0 : enum reg_class rclass
2054 0 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
2055 :
2056 0 : if (get_reload_reg (curr_static_id->operand[nop].type, mode, reg,
2057 : rclass, NULL,
2058 : true, false, "paradoxical subreg", &new_reg))
2059 : {
2060 0 : rtx subreg;
2061 0 : bool insert_before, insert_after;
2062 :
2063 0 : PUT_MODE (new_reg, mode);
2064 0 : subreg = gen_lowpart_SUBREG (innermode, new_reg);
2065 0 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
2066 :
2067 0 : insert_before = (type != OP_OUT);
2068 0 : insert_after = (type != OP_IN);
2069 0 : insert_move_for_subreg (insert_before ? &before : NULL,
2070 : insert_after ? &after : NULL,
2071 : reg, subreg);
2072 : }
2073 0 : SUBREG_REG (operand) = new_reg;
2074 0 : lra_process_new_insns (curr_insn, before, after,
2075 : "Inserting paradoxical subreg reload");
2076 0 : return true;
2077 : }
2078 : return false;
2079 : }
2080 :
2081 : /* Return TRUE if X refers for a hard register from SET. */
2082 : static bool
2083 413252 : uses_hard_regs_p (rtx x, HARD_REG_SET set)
2084 : {
2085 413252 : int i, j, x_hard_regno;
2086 413252 : machine_mode mode;
2087 413252 : const char *fmt;
2088 413252 : enum rtx_code code;
2089 :
2090 413252 : if (x == NULL_RTX)
2091 : return false;
2092 413252 : code = GET_CODE (x);
2093 413252 : mode = GET_MODE (x);
2094 :
2095 413252 : if (code == SUBREG)
2096 : {
2097 : /* For all SUBREGs we want to check whether the full multi-register
2098 : overlaps the set. For normal SUBREGs this means 'get_hard_regno' of
2099 : the inner register, for paradoxical SUBREGs this means the
2100 : 'get_hard_regno' of the full SUBREG and for complete SUBREGs either is
2101 : fine. Use the wider mode for all cases. */
2102 10277 : rtx subreg = SUBREG_REG (x);
2103 10277 : mode = wider_subreg_mode (x);
2104 10277 : if (mode == GET_MODE (subreg))
2105 : {
2106 9244 : x = subreg;
2107 9244 : code = GET_CODE (x);
2108 : }
2109 : }
2110 :
2111 413252 : if (REG_P (x) || SUBREG_P (x))
2112 : {
2113 265908 : x_hard_regno = get_hard_regno (x);
2114 265908 : return (x_hard_regno >= 0
2115 265908 : && overlaps_hard_reg_set_p (set, mode, x_hard_regno));
2116 : }
2117 147344 : fmt = GET_RTX_FORMAT (code);
2118 381792 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
2119 : {
2120 236514 : if (fmt[i] == 'e')
2121 : {
2122 115221 : if (uses_hard_regs_p (XEXP (x, i), set))
2123 : return true;
2124 : }
2125 121293 : else if (fmt[i] == 'E')
2126 : {
2127 4398 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
2128 3980 : if (uses_hard_regs_p (XVECEXP (x, i, j), set))
2129 : return true;
2130 : }
2131 : }
2132 : return false;
2133 : }
2134 :
2135 : /* Return true if OP is a spilled pseudo. */
2136 : static inline bool
2137 82825483 : spilled_pseudo_p (rtx op)
2138 : {
2139 82825483 : return (REG_P (op)
2140 82825483 : && REGNO (op) >= FIRST_PSEUDO_REGISTER && in_mem_p (REGNO (op)));
2141 : }
2142 :
2143 : /* Return true if OP is (plus (reg) (const)). */
2144 : static inline bool
2145 57769 : reg_plus_const_p (rtx op)
2146 : {
2147 39444 : return (GET_CODE (op) == PLUS && REG_P (XEXP (op, 0))
2148 97213 : && CONSTANT_P (XEXP (op, 1)));
2149 : }
2150 :
2151 : /* Return true if X is a general constant. */
2152 : static inline bool
2153 8037038 : general_constant_p (rtx x)
2154 : {
2155 8037038 : return CONSTANT_P (x) && (! flag_pic || LEGITIMATE_PIC_OPERAND_P (x));
2156 : }
2157 :
2158 : static bool
2159 25676412 : reg_in_class_p (rtx reg, enum reg_class cl)
2160 : {
2161 25676412 : if (cl == NO_REGS)
2162 1141400 : return get_reg_class (REGNO (reg)) == NO_REGS;
2163 24535012 : return in_class_p (reg, cl, NULL);
2164 : }
2165 :
2166 : /* Return true if SET of RCLASS contains no hard regs which can be
2167 : used in MODE. */
2168 : static bool
2169 3870865 : prohibited_class_reg_set_mode_p (enum reg_class rclass,
2170 : HARD_REG_SET &set,
2171 : machine_mode mode)
2172 : {
2173 3870865 : HARD_REG_SET temp;
2174 :
2175 7741730 : lra_assert (hard_reg_set_subset_p (reg_class_contents[rclass], set));
2176 3870865 : temp = set & ~lra_no_alloc_regs;
2177 3870865 : return (hard_reg_set_subset_p
2178 3870865 : (temp, ira_prohibited_class_mode_regs[rclass][mode]));
2179 : }
2180 :
2181 :
2182 : /* Used to check validity info about small class input operands. It
2183 : should be incremented at start of processing an insn
2184 : alternative. */
2185 : static unsigned int curr_small_class_check = 0;
2186 :
2187 : /* Update number of used inputs of class OP_CLASS for operand NOP
2188 : of alternative NALT. Return true if we have more such class operands
2189 : than the number of available regs. */
2190 : static bool
2191 404460767 : update_and_check_small_class_inputs (int nop, int nalt,
2192 : enum reg_class op_class)
2193 : {
2194 404460767 : static unsigned int small_class_check[LIM_REG_CLASSES];
2195 404460767 : static int small_class_input_nums[LIM_REG_CLASSES];
2196 :
2197 401430919 : if (SMALL_REGISTER_CLASS_P (op_class)
2198 : /* We are interesting in classes became small because of fixing
2199 : some hard regs, e.g. by an user through GCC options. */
2200 3160014 : && hard_reg_set_intersect_p (reg_class_contents[op_class],
2201 3160014 : ira_no_alloc_regs)
2202 404460805 : && (curr_static_id->operand[nop].type != OP_OUT
2203 32 : || TEST_BIT (curr_static_id->operand[nop].early_clobber_alts, nalt)))
2204 : {
2205 6 : if (small_class_check[op_class] == curr_small_class_check)
2206 0 : small_class_input_nums[op_class]++;
2207 : else
2208 : {
2209 6 : small_class_check[op_class] = curr_small_class_check;
2210 6 : small_class_input_nums[op_class] = 1;
2211 : }
2212 6 : if (small_class_input_nums[op_class] > ira_class_hard_regs_num[op_class])
2213 0 : return true;
2214 : }
2215 : return false;
2216 : }
2217 :
2218 : /* Print operand constraints for alternative ALT_NUMBER of the current
2219 : insn. */
2220 : static void
2221 4573 : print_curr_insn_alt (int alt_number)
2222 : {
2223 15881 : for (int i = 0; i < curr_static_id->n_operands; i++)
2224 : {
2225 11308 : const char *p = (curr_static_id->operand_alternative
2226 11308 : [alt_number * curr_static_id->n_operands + i].constraint);
2227 11308 : if (*p == '\0')
2228 216 : continue;
2229 11092 : fprintf (lra_dump_file, " (%d) ", i);
2230 39370 : for (; *p != '\0' && *p != ',' && *p != '#'; p++)
2231 17186 : fputc (*p, lra_dump_file);
2232 : }
2233 4573 : }
2234 :
2235 : struct dependent_filter_cache_hasher
2236 : : free_ptr_hash <dependent_filter_entry>
2237 : {
2238 0 : static inline hashval_t hash (const dependent_filter_entry *e)
2239 : {
2240 0 : hashval_t h = (hashval_t) e->id;
2241 0 : h = iterative_hash_hashval_t ((hashval_t) e->mode, h);
2242 0 : h = iterative_hash_hashval_t ((hashval_t) e->partner_mode, h);
2243 0 : h = iterative_hash_hashval_t (e->partner_regno, h);
2244 0 : h = iterative_hash_hashval_t ((hashval_t) e->is_ref, h);
2245 0 : return h;
2246 : }
2247 0 : static inline bool equal (const dependent_filter_entry *a,
2248 : const dependent_filter_entry *b)
2249 : {
2250 0 : return (a->id == b->id && a->mode == b->mode
2251 0 : && a->partner_mode == b->partner_mode
2252 0 : && a->partner_regno == b->partner_regno
2253 0 : && a->is_ref == b->is_ref);
2254 : }
2255 : };
2256 :
2257 : static hash_table<dependent_filter_cache_hasher> *dependent_filter_htab;
2258 :
2259 : /* Allocate the dependent-filter cache. */
2260 :
2261 : void
2262 217848 : lra_init_dependent_filter_cache (void)
2263 : {
2264 217848 : gcc_assert (!dependent_filter_htab);
2265 217848 : dependent_filter_htab = new hash_table<dependent_filter_cache_hasher> (16);
2266 217848 : }
2267 :
2268 : /* Release the dependent-filter cache. */
2269 :
2270 : void
2271 287724 : lra_finish_dependent_filter_cache (void)
2272 : {
2273 287724 : delete dependent_filter_htab;
2274 287724 : dependent_filter_htab = nullptr;
2275 287724 : }
2276 :
2277 : /* Reset the dependent-filter cache. */
2278 :
2279 : void
2280 3034254 : lra_reset_dependent_filters (void)
2281 : {
2282 3034254 : if (dependent_filter_htab)
2283 3034254 : dependent_filter_htab->empty ();
2284 3034254 : }
2285 :
2286 : /* Return the set of hardregs allowed by dependent filter ID for an operand
2287 : of mode MODE when the referenced operand has PARTNER_REGNO in
2288 : PARTNER_MODE. That is when IS_REF is false. If it is true, the roles are
2289 : reversed and PARTNER_REGNO as well as PARTNER_MODE refer to the dependent
2290 : operand.
2291 :
2292 : We fill the set for all hardregs and add the resulting regset to
2293 : a hash table if it doesn't already exist in it. */
2294 :
2295 : const HARD_REG_SET *
2296 0 : lra_get_dependent_filter (int id, machine_mode mode,
2297 : unsigned int partner_regno,
2298 : machine_mode partner_mode, bool is_ref)
2299 : {
2300 0 : dependent_filter_entry key;
2301 0 : key.id = id;
2302 0 : key.mode = mode;
2303 0 : key.partner_mode = partner_mode;
2304 0 : key.partner_regno = partner_regno;
2305 0 : key.is_ref = is_ref;
2306 :
2307 0 : dependent_filter_entry **slot
2308 0 : = dependent_filter_htab->find_slot (&key, INSERT);
2309 0 : if (*slot)
2310 0 : return &(*slot)->allowed;
2311 :
2312 0 : auto *e = XCNEW (dependent_filter_entry);
2313 0 : e->id = id;
2314 0 : e->mode = mode;
2315 0 : e->partner_mode = partner_mode;
2316 0 : e->partner_regno = partner_regno;
2317 0 : e->is_ref = is_ref;
2318 :
2319 : /* ??? Should we restrict ourselves to a register class here? */
2320 0 : for (unsigned regno = 0; regno < FIRST_PSEUDO_REGISTER; ++regno)
2321 : {
2322 0 : bool ok;
2323 0 : if (is_ref)
2324 0 : ok = eval_dependent_filter (id, partner_regno, partner_mode,
2325 : regno, mode);
2326 : else
2327 0 : ok = eval_dependent_filter (id, regno, mode,
2328 : partner_regno, partner_mode);
2329 0 : if (ok)
2330 0 : SET_HARD_REG_BIT (e->allowed, regno);
2331 : }
2332 0 : *slot = e;
2333 0 : return &e->allowed;
2334 : }
2335 :
2336 : /* Add the dependent filter ID to REGNO's dependent filters with
2337 : referenced PARTNER_REGNO and PARTNER_MODE if IS_REF is false.
2338 : If IS_REF is true, the roles are reversed. We need both
2339 : directions in case the referenced op is assigned a hardreg first. */
2340 :
2341 : void
2342 0 : lra_add_dependent_filter (int regno, int id, machine_mode mode,
2343 : int partner_regno, machine_mode partner_mode,
2344 : bool is_ref)
2345 : {
2346 : /* Only meaningful for pseudo partners. */
2347 0 : if (partner_regno < FIRST_PSEUDO_REGISTER)
2348 0 : return;
2349 :
2350 0 : vec<dependent_filter> &filters = lra_reg_info[regno].dependent_filters;
2351 0 : unsigned int i;
2352 0 : dependent_filter *filter;
2353 :
2354 : /* If this filter already exists, do nothing. */
2355 0 : FOR_EACH_VEC_ELT (filters, i, filter)
2356 0 : if (filter->id == id
2357 0 : && filter->partner_regno == (unsigned int) partner_regno
2358 0 : && filter->mode == mode && filter->partner_mode == partner_mode
2359 0 : && filter->is_ref == is_ref)
2360 : return;
2361 :
2362 : /* Otherwise, create a new filter. */
2363 0 : dependent_filter new_filter;
2364 0 : new_filter.id = id;
2365 0 : new_filter.mode = mode;
2366 0 : new_filter.partner_mode = partner_mode;
2367 0 : new_filter.partner_regno = (unsigned int) partner_regno;
2368 0 : new_filter.is_ref = is_ref;
2369 0 : filters.safe_push (new_filter);
2370 : }
2371 :
2372 : /* Return the set of hardregs for the constraint CN in MODE,
2373 : allowed by its dependent filter.
2374 : If there is no dependent filter or the involved registers are no
2375 : hardregs, return NULL. */
2376 :
2377 : static const HARD_REG_SET *
2378 353695825 : get_dependent_filter (constraint_num cn, machine_mode mode)
2379 : {
2380 353695825 : int id = get_dependent_filter_id (cn);
2381 353695825 : if (id < 0)
2382 353695825 : return nullptr;
2383 :
2384 : gcc_assert (reg_class_for_constraint (cn) != NO_REGS);
2385 :
2386 : int ref_opno = get_dependent_filter_ref (id);
2387 : gcc_assert (ref_opno >= 0 && ref_opno < curr_static_id->n_operands);
2388 :
2389 : rtx ref_op = *curr_id->operand_loc[ref_opno];
2390 :
2391 : rtx inner = SUBREG_P (ref_op) ? SUBREG_REG (ref_op) : ref_op;
2392 :
2393 : if (!REG_P (inner))
2394 : return nullptr;
2395 :
2396 : int ref_regno = REGNO (inner);
2397 :
2398 : if (ref_regno >= FIRST_PSEUDO_REGISTER)
2399 : {
2400 : ref_regno = reg_renumber[ref_regno];
2401 : /* Even with a pseudo reference op, the filter can still reject
2402 : based on the partner. We call it with INVALID_REGNUM
2403 : to give it a chance to do so. Otherwise we'd introduce
2404 : an "all choices legal" filter that might later
2405 : "change its mind" once there is a fixed reference. */
2406 : if (ref_regno < 0)
2407 : return lra_get_dependent_filter (id, mode, INVALID_REGNUM,
2408 : GET_MODE (ref_op), false);
2409 : }
2410 :
2411 : /* For a subreg, get the actual regno. */
2412 : if (SUBREG_P (ref_op))
2413 : {
2414 : int subreg_regno = simplify_subreg_regno
2415 : (ref_regno, GET_MODE (inner), SUBREG_BYTE (ref_op),
2416 : GET_MODE (ref_op), true);
2417 : /* The subreg should always map to a hard regno here. */
2418 : gcc_assert (subreg_regno >= 0);
2419 :
2420 : ref_regno = subreg_regno;
2421 : }
2422 :
2423 : return lra_get_dependent_filter (id, mode, ref_regno, GET_MODE (ref_op),
2424 : false);
2425 : }
2426 :
2427 : /* Go through all operands and their constraints, looking for a dependent
2428 : filter. If we find one, add it to the respective reg_info's
2429 : dependent-filter list. After assigning a hardreg to either the dependent
2430 : or the referenced op, this allows us to filter the other's regset. */
2431 :
2432 : static void
2433 81404926 : process_dependent_filters (void)
2434 : {
2435 81404926 : int n_operands = curr_static_id->n_operands;
2436 263021125 : for (int i = 0; i < n_operands; i++)
2437 : {
2438 181616199 : rtx op = *curr_id->operand_loc[i];
2439 181616199 : if (!REG_P (op) || HARD_REGISTER_P (op))
2440 99602885 : continue;
2441 82013314 : const char *constraint
2442 82013314 : = curr_static_id->operand_alternative
2443 82013314 : [goal_alt_number * n_operands + i].constraint;
2444 82013314 : for (const char *p = constraint;
2445 215394192 : *p && *p != ',' && *p != '#';
2446 133380878 : p += CONSTRAINT_LEN (*p, p))
2447 : {
2448 133380878 : enum constraint_num cn = lookup_constraint (p);
2449 133380878 : int id = get_dependent_filter_id (cn);
2450 133380878 : if (id < 0)
2451 133380878 : continue;
2452 : if (reg_class_for_constraint (cn) == NO_REGS)
2453 : continue;
2454 : int ref_opno = get_dependent_filter_ref (id);
2455 : if (ref_opno < 0 || ref_opno >= n_operands)
2456 : continue;
2457 : rtx ref_op = *curr_id->operand_loc[ref_opno];
2458 : if (!REG_P (ref_op) || HARD_REGISTER_P (ref_op))
2459 : continue;
2460 : machine_mode mode = curr_operand_mode[i];
2461 : lra_add_dependent_filter (REGNO (op), id, mode,
2462 : REGNO (ref_op), GET_MODE (ref_op),
2463 : false);
2464 : lra_add_dependent_filter (REGNO (ref_op), id, GET_MODE (ref_op),
2465 : REGNO (op), mode, true);
2466 : }
2467 : }
2468 81404926 : }
2469 :
2470 : /* Major function to choose the current insn alternative and what
2471 : operands should be reloaded and how. If ONLY_ALTERNATIVE is not
2472 : negative we should consider only this alternative. Return false if
2473 : we cannot choose the alternative or find how to reload the
2474 : operands. */
2475 : static bool
2476 92043805 : process_alt_operands (int only_alternative)
2477 : {
2478 92043805 : bool ok_p = false;
2479 92043805 : int nop, overall, nalt;
2480 92043805 : int n_alternatives = curr_static_id->n_alternatives;
2481 92043805 : int n_operands = curr_static_id->n_operands;
2482 : /* LOSERS counts the operands that don't fit this alternative and
2483 : would require loading. */
2484 92043805 : int losers;
2485 92043805 : int addr_losers;
2486 : /* REJECT is a count of how undesirable this alternative says it is
2487 : if any reloading is required. If the alternative matches exactly
2488 : then REJECT is ignored, but otherwise it gets this much counted
2489 : against it in addition to the reloading needed. */
2490 92043805 : int reject;
2491 : /* This is defined by '!' or '?' alternative constraint and added to
2492 : reject. But in some cases it can be ignored. */
2493 92043805 : int static_reject;
2494 92043805 : int op_reject;
2495 : /* The number of elements in the following array. */
2496 92043805 : int early_clobbered_regs_num;
2497 : /* Numbers of operands which are early clobber registers. */
2498 92043805 : int early_clobbered_nops[MAX_RECOG_OPERANDS];
2499 92043805 : enum reg_class curr_alt[MAX_RECOG_OPERANDS];
2500 92043805 : enum reg_class all_this_alternative;
2501 92043805 : int all_used_nregs, all_reload_nregs;
2502 92043805 : HARD_REG_SET curr_alt_set[MAX_RECOG_OPERANDS];
2503 92043805 : HARD_REG_SET curr_alt_exclude_start_hard_regs[MAX_RECOG_OPERANDS];
2504 92043805 : bool curr_alt_match_win[MAX_RECOG_OPERANDS];
2505 92043805 : bool curr_alt_win[MAX_RECOG_OPERANDS];
2506 92043805 : bool curr_alt_offmemok[MAX_RECOG_OPERANDS];
2507 92043805 : int curr_alt_matches[MAX_RECOG_OPERANDS];
2508 : /* The number of elements in the following array. */
2509 92043805 : int curr_alt_dont_inherit_ops_num;
2510 : /* Numbers of operands whose reload pseudos should not be inherited. */
2511 92043805 : int curr_alt_dont_inherit_ops[MAX_RECOG_OPERANDS];
2512 92043805 : bool curr_reuse_alt_p;
2513 : /* True if output stack pointer reload should be generated for the current
2514 : alternative. */
2515 92043805 : bool curr_alt_out_sp_reload_p;
2516 92043805 : bool curr_alt_class_change_p;
2517 92043805 : rtx op;
2518 : /* The register when the operand is a subreg of register, otherwise the
2519 : operand itself. */
2520 92043805 : rtx no_subreg_reg_operand[MAX_RECOG_OPERANDS];
2521 : /* The register if the operand is a register or subreg of register,
2522 : otherwise NULL. */
2523 92043805 : rtx operand_reg[MAX_RECOG_OPERANDS];
2524 92043805 : int hard_regno[MAX_RECOG_OPERANDS];
2525 92043805 : machine_mode biggest_mode[MAX_RECOG_OPERANDS];
2526 92043805 : int reload_nregs, reload_sum;
2527 92043805 : bool costly_p;
2528 92043805 : enum reg_class cl;
2529 92043805 : const HARD_REG_SET *cl_filter;
2530 92043805 : HARD_REG_SET hard_reg_constraint;
2531 :
2532 : /* Calculate some data common for all alternatives to speed up the
2533 : function. */
2534 306097430 : for (nop = 0; nop < n_operands; nop++)
2535 : {
2536 214053625 : rtx reg;
2537 :
2538 214053625 : op = no_subreg_reg_operand[nop] = *curr_id->operand_loc[nop];
2539 : /* The real hard regno of the operand after the allocation. */
2540 214053625 : hard_regno[nop] = get_hard_regno (op);
2541 :
2542 214053625 : operand_reg[nop] = reg = op;
2543 214053625 : biggest_mode[nop] = GET_MODE (op);
2544 214053625 : if (GET_CODE (op) == SUBREG)
2545 : {
2546 4316642 : biggest_mode[nop] = wider_subreg_mode (op);
2547 4316642 : operand_reg[nop] = reg = SUBREG_REG (op);
2548 : }
2549 214053625 : if (! REG_P (reg))
2550 90577762 : operand_reg[nop] = NULL_RTX;
2551 123475863 : else if (REGNO (reg) >= FIRST_PSEUDO_REGISTER
2552 145007369 : || ((int) REGNO (reg)
2553 21531506 : == lra_get_elimination_hard_regno (REGNO (reg))))
2554 120397792 : no_subreg_reg_operand[nop] = reg;
2555 : else
2556 3078071 : operand_reg[nop] = no_subreg_reg_operand[nop]
2557 : /* Just use natural mode for elimination result. It should
2558 : be enough for extra constraints hooks. */
2559 3078071 : = regno_reg_rtx[hard_regno[nop]];
2560 : }
2561 :
2562 : /* The constraints are made of several alternatives. Each operand's
2563 : constraint looks like foo,bar,... with commas separating the
2564 : alternatives. The first alternatives for all operands go
2565 : together, the second alternatives go together, etc.
2566 :
2567 : First loop over alternatives. */
2568 92043805 : alternative_mask preferred = curr_id->preferred_alternatives;
2569 92043805 : if (only_alternative >= 0)
2570 982155 : preferred &= ALTERNATIVE_BIT (only_alternative);
2571 :
2572 92043805 : bool prefer_memory_p = false;
2573 92044366 : repeat:
2574 389440217 : for (nalt = 0; nalt < n_alternatives; nalt++)
2575 : {
2576 : /* Loop over operands for one constraint alternative. */
2577 375446591 : if (!TEST_BIT (preferred, nalt))
2578 111459741 : continue;
2579 :
2580 263986850 : if (lra_dump_file != NULL)
2581 : {
2582 3388 : fprintf (lra_dump_file, " Considering alt=%d of insn %d: ",
2583 3388 : nalt, INSN_UID (curr_insn));
2584 3388 : print_curr_insn_alt (nalt);
2585 3388 : fprintf (lra_dump_file, "\n");
2586 : }
2587 :
2588 263986850 : bool matching_early_clobber[MAX_RECOG_OPERANDS];
2589 263986850 : curr_small_class_check++;
2590 263986850 : overall = losers = addr_losers = 0;
2591 263986850 : static_reject = reject = reload_nregs = reload_sum = 0;
2592 875398777 : for (nop = 0; nop < n_operands; nop++)
2593 : {
2594 611411927 : int inc = (curr_static_id
2595 611411927 : ->operand_alternative[nalt * n_operands + nop].reject);
2596 611411927 : if (lra_dump_file != NULL && inc != 0)
2597 52 : fprintf (lra_dump_file,
2598 : " Statically defined alt reject+=%d\n", inc);
2599 611411927 : static_reject += inc;
2600 611411927 : matching_early_clobber[nop] = 0;
2601 : }
2602 : reject += static_reject;
2603 : early_clobbered_regs_num = 0;
2604 : curr_alt_out_sp_reload_p = false;
2605 : curr_reuse_alt_p = true;
2606 : curr_alt_class_change_p = false;
2607 : all_this_alternative = NO_REGS;
2608 : all_used_nregs = all_reload_nregs = 0;
2609 683646187 : for (nop = 0; nop < n_operands; nop++)
2610 : {
2611 545117047 : const char *p;
2612 545117047 : char *end;
2613 545117047 : int len, c, m, i, opalt_num, this_alternative_matches;
2614 545117047 : bool win, did_match, offmemok, early_clobber_p;
2615 : /* false => this operand can be reloaded somehow for this
2616 : alternative. */
2617 545117047 : bool badop;
2618 : /* true => this operand can be reloaded if the alternative
2619 : allows regs. */
2620 545117047 : bool winreg;
2621 : /* True if a constant forced into memory would be OK for
2622 : this operand. */
2623 545117047 : bool constmemok;
2624 545117047 : enum reg_class this_alternative, this_costly_alternative;
2625 545117047 : HARD_REG_SET this_alternative_set, this_costly_alternative_set;
2626 545117047 : HARD_REG_SET this_alternative_exclude_start_hard_regs;
2627 545117047 : bool this_alternative_match_win, this_alternative_win;
2628 545117047 : bool this_alternative_offmemok;
2629 545117047 : bool scratch_p;
2630 545117047 : machine_mode mode;
2631 545117047 : enum constraint_num cn;
2632 545117047 : bool class_change_p = false;
2633 :
2634 545117047 : opalt_num = nalt * n_operands + nop;
2635 545117047 : if (curr_static_id->operand_alternative[opalt_num].anything_ok)
2636 : {
2637 : /* Fast track for no constraints at all. */
2638 15198570 : curr_alt[nop] = NO_REGS;
2639 15198570 : CLEAR_HARD_REG_SET (curr_alt_set[nop]);
2640 15198570 : curr_alt_win[nop] = true;
2641 15198570 : curr_alt_match_win[nop] = false;
2642 15198570 : curr_alt_offmemok[nop] = false;
2643 15198570 : curr_alt_matches[nop] = -1;
2644 15198570 : continue;
2645 : }
2646 :
2647 529918477 : op = no_subreg_reg_operand[nop];
2648 529918477 : mode = curr_operand_mode[nop];
2649 :
2650 529918477 : win = did_match = winreg = offmemok = constmemok = false;
2651 529918477 : badop = true;
2652 529918477 : const HARD_REG_SET *cl_dep_filter = nullptr;
2653 :
2654 529918477 : early_clobber_p = false;
2655 529918477 : p = curr_static_id->operand_alternative[opalt_num].constraint;
2656 :
2657 529918477 : this_costly_alternative = this_alternative = NO_REGS;
2658 : /* We update set of possible hard regs besides its class
2659 : because reg class might be inaccurate. For example,
2660 : union of LO_REGS (l), HI_REGS(h), and STACK_REG(k) in ARM
2661 : is translated in HI_REGS because classes are merged by
2662 : pairs and there is no accurate intermediate class. */
2663 2119673908 : CLEAR_HARD_REG_SET (this_alternative_set);
2664 1589755431 : CLEAR_HARD_REG_SET (this_costly_alternative_set);
2665 529918477 : CLEAR_HARD_REG_SET (this_alternative_exclude_start_hard_regs);
2666 529918477 : this_alternative_win = false;
2667 529918477 : this_alternative_match_win = false;
2668 529918477 : this_alternative_offmemok = false;
2669 529918477 : this_alternative_matches = -1;
2670 :
2671 : /* An empty constraint should be excluded by the fast
2672 : track. */
2673 529918477 : lra_assert (*p != 0 && *p != ',');
2674 :
2675 : op_reject = 0;
2676 : /* Scan this alternative's specs for this operand; set WIN
2677 : if the operand fits any letter in this alternative.
2678 : Otherwise, clear BADOP if this operand could fit some
2679 : letter after reloads, or set WINREG if this operand could
2680 : fit after reloads provided the constraint allows some
2681 : registers. */
2682 : costly_p = false;
2683 1345695251 : do
2684 : {
2685 1345695251 : switch ((c = *p, len = CONSTRAINT_LEN (c, p)), c)
2686 : {
2687 : case '\0':
2688 : len = 0;
2689 : break;
2690 506640141 : case ',':
2691 506640141 : c = '\0';
2692 506640141 : break;
2693 :
2694 192170 : case '&':
2695 192170 : early_clobber_p = true;
2696 192170 : break;
2697 :
2698 19848 : case '$':
2699 19848 : op_reject += LRA_MAX_REJECT;
2700 19848 : break;
2701 0 : case '^':
2702 0 : op_reject += LRA_LOSER_COST_FACTOR;
2703 0 : break;
2704 :
2705 0 : case '#':
2706 : /* Ignore rest of this alternative. */
2707 0 : c = '\0';
2708 0 : break;
2709 :
2710 57621835 : case '0': case '1': case '2': case '3': case '4':
2711 57621835 : case '5': case '6': case '7': case '8': case '9':
2712 57621835 : {
2713 57621835 : int m_hregno;
2714 57621835 : bool match_p;
2715 :
2716 57621835 : m = strtoul (p, &end, 10);
2717 57621835 : p = end;
2718 57621835 : len = 0;
2719 57621835 : lra_assert (nop > m);
2720 :
2721 : /* Reject matches if we don't know which operand is
2722 : bigger. This situation would arguably be a bug in
2723 : an .md pattern, but could also occur in a user asm. */
2724 172865505 : if (!ordered_p (GET_MODE_SIZE (biggest_mode[m]),
2725 57621835 : GET_MODE_SIZE (biggest_mode[nop])))
2726 : break;
2727 :
2728 : /* Don't match wrong asm insn operands for proper
2729 : diagnostic later. */
2730 57621835 : if (INSN_CODE (curr_insn) < 0
2731 34583 : && (curr_operand_mode[m] == BLKmode
2732 34582 : || curr_operand_mode[nop] == BLKmode)
2733 1 : && curr_operand_mode[m] != curr_operand_mode[nop])
2734 : break;
2735 :
2736 57621834 : m_hregno = get_hard_regno (*curr_id->operand_loc[m]);
2737 : /* We are supposed to match a previous operand.
2738 : If we do, we win if that one did. If we do
2739 : not, count both of the operands as losers.
2740 : (This is too conservative, since most of the
2741 : time only a single reload insn will be needed
2742 : to make the two operands win. As a result,
2743 : this alternative may be rejected when it is
2744 : actually desirable.) */
2745 57621834 : match_p = false;
2746 57621834 : if (operands_match_p (*curr_id->operand_loc[nop],
2747 57621834 : *curr_id->operand_loc[m], m_hregno))
2748 : {
2749 : /* We should reject matching of an early
2750 : clobber operand if the matching operand is
2751 : not dying in the insn. */
2752 15173183 : if (!TEST_BIT (curr_static_id->operand[m]
2753 : .early_clobber_alts, nalt)
2754 20660 : || operand_reg[nop] == NULL_RTX
2755 15193843 : || (find_regno_note (curr_insn, REG_DEAD,
2756 : REGNO (op))
2757 4404 : || REGNO (op) == REGNO (operand_reg[m])))
2758 15173183 : match_p = true;
2759 : }
2760 15173183 : if (match_p)
2761 : {
2762 : /* If we are matching a non-offsettable
2763 : address where an offsettable address was
2764 : expected, then we must reject this
2765 : combination, because we can't reload
2766 : it. */
2767 15173183 : if (curr_alt_offmemok[m]
2768 1505 : && MEM_P (*curr_id->operand_loc[m])
2769 5 : && curr_alt[m] == NO_REGS && ! curr_alt_win[m])
2770 0 : continue;
2771 : }
2772 : else
2773 : {
2774 : /* If the operands do not match and one
2775 : operand is INOUT, we can not match them.
2776 : Try other possibilities, e.g. other
2777 : alternatives or commutative operand
2778 : exchange. */
2779 42448651 : if (curr_static_id->operand[nop].type == OP_INOUT
2780 42448651 : || curr_static_id->operand[m].type == OP_INOUT)
2781 : break;
2782 : /* Operands don't match. For asm if the operands
2783 : are different user defined explicit hard
2784 : registers, then we cannot make them match
2785 : when one is early clobber operand. */
2786 42448242 : if ((REG_P (*curr_id->operand_loc[nop])
2787 26317295 : || SUBREG_P (*curr_id->operand_loc[nop]))
2788 16728792 : && (REG_P (*curr_id->operand_loc[m])
2789 203432 : || SUBREG_P (*curr_id->operand_loc[m]))
2790 16633557 : && INSN_CODE (curr_insn) < 0)
2791 : {
2792 630 : rtx nop_reg = *curr_id->operand_loc[nop];
2793 630 : if (SUBREG_P (nop_reg))
2794 0 : nop_reg = SUBREG_REG (nop_reg);
2795 630 : rtx m_reg = *curr_id->operand_loc[m];
2796 630 : if (SUBREG_P (m_reg))
2797 0 : m_reg = SUBREG_REG (m_reg);
2798 :
2799 630 : if (REG_P (nop_reg)
2800 630 : && HARD_REGISTER_P (nop_reg)
2801 0 : && REG_USERVAR_P (nop_reg)
2802 0 : && REG_P (m_reg)
2803 0 : && HARD_REGISTER_P (m_reg)
2804 630 : && REG_USERVAR_P (m_reg))
2805 : {
2806 : int i;
2807 :
2808 0 : for (i = 0; i < early_clobbered_regs_num; i++)
2809 0 : if (m == early_clobbered_nops[i])
2810 : break;
2811 0 : if (i < early_clobbered_regs_num
2812 0 : || early_clobber_p)
2813 : break;
2814 : }
2815 : }
2816 : /* Both operands must allow a reload register or the
2817 : matched operand accepts memory (e.g. 'g' with a mode
2818 : that fits no hard reg), in which case the operands
2819 : can be matched in memory. */
2820 42448242 : if (curr_alt[m] == NO_REGS && !curr_alt_offmemok[m])
2821 : break;
2822 : /* Retroactively mark the operand we had to
2823 : match as a loser, if it wasn't already and
2824 : it wasn't matched to a register constraint
2825 : (e.g it might be matched by memory). */
2826 42424905 : if (curr_alt_win[m]
2827 41566941 : && (operand_reg[m] == NULL_RTX
2828 41052488 : || hard_regno[m] < 0))
2829 : {
2830 1254045 : if (lra_dump_file != NULL)
2831 9 : fprintf
2832 9 : (lra_dump_file,
2833 : " %d Matched operand reload: "
2834 : "losers++\n", m);
2835 1254045 : losers++;
2836 1254045 : reload_nregs
2837 1254045 : += (ira_reg_class_max_nregs[curr_alt[m]]
2838 1254045 : [GET_MODE (*curr_id->operand_loc[m])]);
2839 : }
2840 :
2841 : /* Prefer matching earlyclobber alternative as
2842 : it results in less hard regs required for
2843 : the insn than a non-matching earlyclobber
2844 : alternative. */
2845 42424905 : if (TEST_BIT (curr_static_id->operand[m]
2846 : .early_clobber_alts, nalt))
2847 : {
2848 19718 : if (lra_dump_file != NULL)
2849 0 : fprintf
2850 0 : (lra_dump_file,
2851 : " %d Matching earlyclobber alt:"
2852 : " reject--\n",
2853 : nop);
2854 19718 : if (!matching_early_clobber[m])
2855 : {
2856 19718 : reject--;
2857 19718 : matching_early_clobber[m] = 1;
2858 : }
2859 : }
2860 : /* Otherwise we prefer no matching
2861 : alternatives because it gives more freedom
2862 : in RA. */
2863 42405187 : else if (operand_reg[nop] == NULL_RTX
2864 42405187 : || (find_regno_note (curr_insn, REG_DEAD,
2865 16702987 : REGNO (operand_reg[nop]))
2866 : == NULL_RTX))
2867 : {
2868 37245038 : if (lra_dump_file != NULL)
2869 916 : fprintf
2870 916 : (lra_dump_file,
2871 : " %d Matching alt: reject+=2\n",
2872 : nop);
2873 37245038 : reject += 2;
2874 : }
2875 : }
2876 : /* If we have to reload this operand and some
2877 : previous operand also had to match the same
2878 : thing as this operand, we don't know how to do
2879 : that. */
2880 57598088 : if (!match_p || !curr_alt_win[m])
2881 : {
2882 89258938 : for (i = 0; i < nop; i++)
2883 46719493 : if (curr_alt_matches[i] == m)
2884 : break;
2885 42539446 : if (i < nop)
2886 : break;
2887 : }
2888 : else
2889 : did_match = true;
2890 :
2891 57598087 : this_alternative_matches = m;
2892 : /* This can be fixed with reloads if the operand
2893 : we are supposed to match can be fixed with
2894 : reloads. */
2895 57598087 : badop = false;
2896 57598087 : this_alternative = curr_alt[m];
2897 57598087 : this_alternative_set = curr_alt_set[m];
2898 57598087 : this_alternative_exclude_start_hard_regs
2899 57598087 : = curr_alt_exclude_start_hard_regs[m];
2900 : /* If the matched operand permits memory (e.g. 'g'),
2901 : so does this one -- they share the same location.
2902 : This lets a matching reload go to memory when no
2903 : hard reg of the class can hold the operand mode. */
2904 57598087 : offmemok = curr_alt_offmemok[m];
2905 57598087 : winreg = this_alternative != NO_REGS;
2906 57598087 : break;
2907 : }
2908 :
2909 11931164 : case 'g':
2910 11931164 : if (MEM_P (op)
2911 8037038 : || general_constant_p (op)
2912 16543977 : || spilled_pseudo_p (op))
2913 : win = true;
2914 11931164 : if (REG_P (op) && prefer_memory_p)
2915 : {
2916 11931164 : badop = false;
2917 11931164 : offmemok = true;
2918 : }
2919 11931164 : cl = GENERAL_REGS;
2920 11931164 : cl_filter = nullptr;
2921 11931164 : cl_dep_filter = nullptr;
2922 11931164 : goto reg;
2923 :
2924 685 : case '{':
2925 685 : {
2926 685 : int regno = decode_hard_reg_constraint (p);
2927 685 : gcc_assert (regno >= 0);
2928 685 : cl = NO_REGS;
2929 685 : int nregs = hard_regno_nregs (regno, mode);
2930 1370 : for (int i = 0; i < nregs; ++i)
2931 685 : cl = reg_class_superunion[cl][REGNO_REG_CLASS (regno + i)];
2932 685 : CLEAR_HARD_REG_SET (hard_reg_constraint);
2933 685 : SET_HARD_REG_BIT (hard_reg_constraint, regno);
2934 685 : cl_filter = &hard_reg_constraint;
2935 685 : cl_dep_filter = nullptr;
2936 685 : goto reg;
2937 : }
2938 :
2939 746011072 : default:
2940 746011072 : cn = lookup_constraint (p);
2941 746011072 : switch (get_constraint_type (cn))
2942 : {
2943 493244037 : case CT_REGISTER:
2944 493244037 : cl = reg_class_for_constraint (cn);
2945 363306698 : if (cl != NO_REGS)
2946 : {
2947 353695825 : cl_filter = get_register_filter (cn);
2948 353695825 : cl_dep_filter = get_dependent_filter (cn, mode);
2949 353695825 : goto reg;
2950 : }
2951 : break;
2952 :
2953 2227559 : case CT_CONST_INT:
2954 2227559 : if (CONST_INT_P (op)
2955 2227559 : && insn_const_int_ok_for_constraint (INTVAL (op), cn))
2956 : win = true;
2957 : break;
2958 :
2959 113513125 : case CT_MEMORY:
2960 113513125 : case CT_RELAXED_MEMORY:
2961 113513125 : if (MEM_P (op)
2962 113513125 : && satisfies_memory_constraint_p (op, cn))
2963 : win = true;
2964 77149745 : else if (spilled_pseudo_p (op))
2965 45969634 : win = true;
2966 :
2967 : /* If we didn't already win, we can reload constants
2968 : via force_const_mem or put the pseudo value into
2969 : memory, or make other memory by reloading the
2970 : address like for 'o'. */
2971 118992574 : if (CONST_POOL_OK_P (mode, op)
2972 108033476 : || MEM_P (op) || REG_P (op)
2973 : /* We can restore the equiv insn by a
2974 : reload. */
2975 632603 : || equiv_substition_p[nop]
2976 : /* It can be a sum produced by elimination of a
2977 : register. We can materialized into a register and
2978 : then spill into memory. */
2979 113548607 : || reg_plus_const_p (op))
2980 : badop = false;
2981 : constmemok = true;
2982 : offmemok = true;
2983 : break;
2984 :
2985 1785636 : case CT_ADDRESS:
2986 : /* An asm operand with an address constraint
2987 : that doesn't satisfy address_operand has
2988 : is_address cleared, so that we don't try to
2989 : make a non-address fit. */
2990 1785636 : if (!curr_static_id->operand[nop].is_address)
2991 : break;
2992 : /* If we didn't already win, we can reload the address
2993 : into a base register. */
2994 1785617 : if (satisfies_address_constraint_p (op, cn))
2995 1785617 : win = true;
2996 1785617 : cl = base_reg_class (VOIDmode, ADDR_SPACE_GENERIC,
2997 : ADDRESS, SCRATCH, NULL_RTX,
2998 : curr_insn);
2999 1785617 : cl_filter = nullptr;
3000 1785617 : cl_dep_filter = nullptr;
3001 1785617 : badop = false;
3002 1785617 : goto reg;
3003 :
3004 134027337 : case CT_FIXED_FORM:
3005 134027337 : if (constraint_satisfied_p (op, cn))
3006 1345695251 : win = true;
3007 : break;
3008 :
3009 1213378 : case CT_SPECIAL_MEMORY:
3010 1213378 : if (satisfies_memory_constraint_p (op, cn))
3011 : win = true;
3012 1062925 : else if (spilled_pseudo_p (op))
3013 : {
3014 1345695251 : curr_reuse_alt_p = false;
3015 1345695251 : win = true;
3016 : }
3017 : break;
3018 : }
3019 : break;
3020 :
3021 367413291 : reg:
3022 367413291 : if (mode == BLKmode)
3023 : break;
3024 367413273 : this_alternative = reg_class_subunion[this_alternative][cl];
3025 367413273 : if (hard_reg_set_subset_p (this_alternative_set,
3026 367413273 : reg_class_contents[cl]))
3027 367409624 : this_alternative_exclude_start_hard_regs
3028 367409624 : = ira_exclude_class_mode_regs[cl][mode];
3029 3649 : else if (!hard_reg_set_subset_p (reg_class_contents[cl],
3030 : this_alternative_set))
3031 3648 : this_alternative_exclude_start_hard_regs
3032 3648 : |= ira_exclude_class_mode_regs[cl][mode];
3033 367413273 : this_alternative_set |= reg_class_contents[cl];
3034 367413273 : if (cl_filter)
3035 1370 : this_alternative_exclude_start_hard_regs |= ~*cl_filter;
3036 367413273 : if (cl_dep_filter)
3037 : this_alternative_exclude_start_hard_regs |= ~*cl_dep_filter;
3038 367413273 : if (costly_p)
3039 : {
3040 21781913 : this_costly_alternative
3041 21781913 : = reg_class_subunion[this_costly_alternative][cl];
3042 21781913 : this_costly_alternative_set |= reg_class_contents[cl];
3043 : }
3044 367413273 : winreg = true;
3045 367413273 : if (REG_P (op))
3046 : {
3047 234023836 : rtx orig_op = *curr_id->operand_loc[nop];
3048 6883967 : if (GET_CODE (orig_op) == SUBREG && HARD_REGISTER_P (op)
3049 234023924 : && !targetm.hard_regno_mode_ok (REGNO (op),
3050 88 : GET_MODE(orig_op)))
3051 : break;
3052 :
3053 234023836 : tree decl;
3054 :
3055 234023836 : if (hard_regno[nop] >= 0
3056 197865521 : && in_hard_reg_set_p (this_alternative_set,
3057 : mode, hard_regno[nop])
3058 179052546 : && (!cl_filter
3059 410 : || TEST_HARD_REG_BIT (*cl_filter,
3060 : hard_regno[nop]))
3061 : && (!cl_dep_filter
3062 : || TEST_HARD_REG_BIT (*cl_dep_filter,
3063 : hard_regno[nop]))
3064 413076376 : && ((REG_ATTRS (op) && (decl = REG_EXPR (op)) != NULL
3065 99655192 : && VAR_P (decl) && DECL_HARD_REGISTER (decl))
3066 179049241 : || !(TEST_HARD_REG_BIT
3067 179049241 : (this_alternative_exclude_start_hard_regs,
3068 : hard_regno[nop]))))
3069 : win = true;
3070 54971297 : else if (hard_regno[nop] < 0 && !prefer_memory_p)
3071 : {
3072 36157568 : if (in_class_p (op, this_alternative, NULL))
3073 : win = true;
3074 27141010 : else if (in_class_p (op, this_alternative, NULL, true))
3075 : {
3076 1345695251 : class_change_p = true;
3077 1345695251 : win = true;
3078 : }
3079 : }
3080 : }
3081 : break;
3082 : }
3083 1345695251 : if (c != ' ' && c != '\t')
3084 1345695251 : costly_p = c == '*';
3085 : }
3086 1345695251 : while ((p += len), c);
3087 :
3088 1059836954 : scratch_p = (operand_reg[nop] != NULL_RTX
3089 529918477 : && ira_former_scratch_p (REGNO (operand_reg[nop])));
3090 : /* Record which operands fit this alternative. */
3091 529918477 : if (win)
3092 : {
3093 285663461 : if (early_clobber_p
3094 285510745 : || curr_static_id->operand[nop].type != OP_OUT)
3095 : {
3096 124922129 : if (winreg)
3097 103280406 : all_used_nregs
3098 103280406 : += ira_reg_class_min_nregs[this_alternative][mode];
3099 124922129 : all_this_alternative
3100 124922129 : = (reg_class_subunion
3101 124922129 : [all_this_alternative][this_alternative]);
3102 : }
3103 285663461 : this_alternative_win = true;
3104 : /* If reg and memory is acceptable here (e.g. 'g') but no hard
3105 : reg of the operand class can hold the operand mode, resolve
3106 : the operand to memory. Mark it so that an operand matching
3107 : this one can also fall back to memory, otherwise the matched
3108 : operand would try to reload into an impossible hard reg and
3109 : LRA would cycle (e.g. TDmode "+g" asm on 32-bit x86).
3110 : An operand which is already memory and won implies the
3111 : constraint accepts memory too, even if nothing set
3112 : OFFMEMOK for it. */
3113 285663461 : if ((offmemok || MEM_P (op)) && this_alternative != NO_REGS
3114 364342039 : && hard_reg_set_subset_p (reg_class_contents[this_alternative],
3115 : ira_prohibited_class_mode_regs
3116 78678578 : [this_alternative][mode]
3117 364341856 : | lra_no_alloc_regs))
3118 : {
3119 183 : this_alternative = NO_REGS;
3120 183 : this_alternative_offmemok = true;
3121 : }
3122 285663461 : if (class_change_p)
3123 : {
3124 264583 : curr_alt_class_change_p = true;
3125 264583 : if (lra_dump_file != NULL)
3126 8 : fprintf (lra_dump_file,
3127 : " %d Narrowing class: reject+=3\n",
3128 : nop);
3129 264583 : reject += 3;
3130 : }
3131 285663461 : if (operand_reg[nop] != NULL_RTX)
3132 : {
3133 199345948 : if (hard_regno[nop] >= 0)
3134 : {
3135 178995782 : if (in_hard_reg_set_p (this_costly_alternative_set,
3136 : mode, hard_regno[nop]))
3137 : {
3138 790676 : if (lra_dump_file != NULL)
3139 21 : fprintf (lra_dump_file,
3140 : " %d Costly set: reject++\n",
3141 : nop);
3142 790676 : reject++;
3143 : }
3144 : }
3145 : else
3146 : {
3147 : /* Prefer won reg to spilled pseudo under other
3148 : equal conditions for possible inheritance. */
3149 20350166 : if (! scratch_p)
3150 : {
3151 20345811 : if (lra_dump_file != NULL)
3152 52 : fprintf
3153 52 : (lra_dump_file,
3154 : " %d Non pseudo reload: reject++\n",
3155 : nop);
3156 20345811 : reject++;
3157 : }
3158 20350166 : if (in_class_p (operand_reg[nop],
3159 : this_costly_alternative, NULL, true))
3160 : {
3161 133876 : if (lra_dump_file != NULL)
3162 0 : fprintf
3163 0 : (lra_dump_file,
3164 : " %d Non pseudo costly reload:"
3165 : " reject++\n",
3166 : nop);
3167 133876 : reject++;
3168 : }
3169 : }
3170 : /* We simulate the behavior of old reload here.
3171 : Although scratches need hard registers and it
3172 : might result in spilling other pseudos, no reload
3173 : insns are generated for the scratches. So it
3174 : might cost something but probably less than old
3175 : reload pass believes. */
3176 199345948 : if (scratch_p)
3177 : {
3178 124481 : if (lra_dump_file != NULL)
3179 6 : fprintf (lra_dump_file,
3180 : " %d Scratch win: reject+=2\n",
3181 : nop);
3182 124481 : reject += 2;
3183 : }
3184 : }
3185 : }
3186 244255016 : else if (did_match)
3187 : this_alternative_match_win = true;
3188 : else
3189 : {
3190 229196374 : if (prefer_memory_p && offmemok)
3191 : {
3192 502 : winreg = false;
3193 502 : this_alternative = NO_REGS;
3194 : }
3195 :
3196 229196374 : int const_to_mem = 0;
3197 229196374 : bool no_regs_p;
3198 :
3199 229196374 : reject += op_reject;
3200 : /* Mark output reload of the stack pointer. */
3201 229196374 : if (op == stack_pointer_rtx
3202 56488 : && curr_static_id->operand[nop].type != OP_IN)
3203 229196374 : curr_alt_out_sp_reload_p = true;
3204 :
3205 : /* If this alternative asks for a specific reg class, see if there
3206 : is at least one allocatable register in that class. */
3207 229196374 : no_regs_p
3208 399654109 : = (this_alternative == NO_REGS
3209 229196374 : || (hard_reg_set_subset_p
3210 340915492 : (reg_class_contents[this_alternative],
3211 : lra_no_alloc_regs)));
3212 :
3213 : /* For asms, verify that the class for this alternative is possible
3214 : for the mode that is specified. */
3215 170457735 : if (!no_regs_p && INSN_CODE (curr_insn) < 0)
3216 : {
3217 : int i;
3218 70837 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
3219 70834 : if (targetm.hard_regno_mode_ok (i, mode)
3220 70834 : && in_hard_reg_set_p (reg_class_contents[this_alternative],
3221 : mode, i))
3222 : break;
3223 20211 : if (i == FIRST_PSEUDO_REGISTER)
3224 229196374 : winreg = false;
3225 : }
3226 :
3227 : /* If this operand accepts a register, and if the
3228 : register class has at least one allocatable register,
3229 : then this operand can be reloaded. */
3230 229196374 : if (winreg && !no_regs_p)
3231 : badop = false;
3232 :
3233 58738642 : if (badop)
3234 : {
3235 49143640 : if (lra_dump_file != NULL)
3236 607 : fprintf (lra_dump_file,
3237 : " Bad operand -- refuse\n");
3238 125457249 : goto fail;
3239 : }
3240 :
3241 180052734 : if (this_alternative != NO_REGS)
3242 : {
3243 170457734 : HARD_REG_SET available_regs
3244 170457734 : = (reg_class_contents[this_alternative]
3245 170457734 : & ~((ira_prohibited_class_mode_regs
3246 170457734 : [this_alternative][mode])
3247 170457734 : | lra_no_alloc_regs));
3248 340915468 : if (!hard_reg_set_empty_p (available_regs))
3249 : {
3250 170455785 : if (early_clobber_p
3251 170416331 : || curr_static_id->operand[nop].type != OP_OUT)
3252 : {
3253 89340685 : all_reload_nregs
3254 89340685 : += ira_reg_class_min_nregs[this_alternative][mode];
3255 89340685 : all_this_alternative
3256 89340685 : = (reg_class_subunion
3257 89340685 : [all_this_alternative][this_alternative]);
3258 : }
3259 : }
3260 : else
3261 : {
3262 : /* There are no hard regs holding a value of given
3263 : mode. */
3264 1949 : if (offmemok)
3265 : {
3266 144 : this_alternative = NO_REGS;
3267 144 : if (lra_dump_file != NULL)
3268 0 : fprintf (lra_dump_file,
3269 : " %d Using memory because of"
3270 : " a bad mode: reject+=2\n",
3271 : nop);
3272 144 : reject += 2;
3273 : }
3274 1805 : else if (!prefer_memory_p)
3275 : {
3276 : /* No hard reg of the operand class can hold the
3277 : operand mode, but the constraint may still permit
3278 : memory (e.g. 'g'). Prefer memory for the operands
3279 : before refusing. */
3280 461 : prefer_memory_p = true;
3281 461 : if (lra_dump_file != NULL)
3282 0 : fprintf (lra_dump_file,
3283 : " Trying now memory for"
3284 : " operands\n");
3285 461 : goto repeat;
3286 : }
3287 : else
3288 : {
3289 1344 : if (lra_dump_file != NULL)
3290 0 : fprintf (lra_dump_file,
3291 : " Wrong mode -- refuse\n");
3292 1344 : goto fail;
3293 : }
3294 : }
3295 : }
3296 :
3297 : /* If not assigned pseudo has a class which a subset of
3298 : required reg class, it is a less costly alternative
3299 : as the pseudo still can get a hard reg of necessary
3300 : class. */
3301 170455929 : if (! no_regs_p && REG_P (op) && hard_regno[nop] < 0
3302 21848003 : && (cl = get_reg_class (REGNO (op))) != NO_REGS
3303 183030791 : && ira_class_subset_p[this_alternative][cl])
3304 : {
3305 25720 : if (lra_dump_file != NULL)
3306 0 : fprintf
3307 0 : (lra_dump_file,
3308 : " %d Super set class reg: reject-=3\n", nop);
3309 25720 : reject -= 3;
3310 : }
3311 :
3312 180050929 : this_alternative_offmemok = offmemok;
3313 180050929 : if (this_costly_alternative != NO_REGS)
3314 : {
3315 19588007 : if (lra_dump_file != NULL)
3316 25 : fprintf (lra_dump_file,
3317 : " %d Costly loser: reject++\n", nop);
3318 19588007 : reject++;
3319 : }
3320 : /* If the operand is dying, has a matching constraint,
3321 : and satisfies constraints of the matched operand
3322 : which failed to satisfy the own constraints, most probably
3323 : the reload for this operand will be gone. */
3324 180050929 : if (this_alternative_matches >= 0
3325 42521571 : && !curr_alt_win[this_alternative_matches]
3326 971978 : && REG_P (op)
3327 718737 : && find_regno_note (curr_insn, REG_DEAD, REGNO (op))
3328 180785324 : && (hard_regno[nop] >= 0
3329 380553 : ? in_hard_reg_set_p (this_alternative_set,
3330 : mode, hard_regno[nop])
3331 26711 : : in_class_p (op, this_alternative, NULL)))
3332 : {
3333 238342 : if (lra_dump_file != NULL)
3334 1 : fprintf
3335 1 : (lra_dump_file,
3336 : " %d Dying matched operand reload: reject++\n",
3337 : nop);
3338 238342 : reject++;
3339 : }
3340 : else
3341 : {
3342 : /* Strict_low_part requires to reload the register
3343 : not the sub-register. In this case we should
3344 : check that a final reload hard reg can hold the
3345 : value mode. */
3346 179812587 : if (curr_static_id->operand[nop].strict_low
3347 86 : && REG_P (op)
3348 79 : && hard_regno[nop] < 0
3349 55 : && GET_CODE (*curr_id->operand_loc[nop]) == SUBREG
3350 55 : && ira_class_hard_regs_num[this_alternative] > 0
3351 179812642 : && (!targetm.hard_regno_mode_ok
3352 55 : (ira_class_hard_regs[this_alternative][0],
3353 55 : GET_MODE (*curr_id->operand_loc[nop]))))
3354 : {
3355 0 : if (lra_dump_file != NULL)
3356 0 : fprintf
3357 0 : (lra_dump_file,
3358 : " Strict low subreg reload -- refuse\n");
3359 0 : goto fail;
3360 : }
3361 179812587 : if (lra_dump_file != NULL)
3362 2162 : fprintf
3363 2162 : (lra_dump_file,
3364 : " %d Operand reload: losers++\n", nop);
3365 179812587 : losers++;
3366 : }
3367 180050929 : if (operand_reg[nop] != NULL_RTX
3368 : /* Output operands and matched input operands are
3369 : not inherited. The following conditions do not
3370 : exactly describe the previous statement but they
3371 : are pretty close. */
3372 64895240 : && curr_static_id->operand[nop].type != OP_OUT
3373 28670640 : && (this_alternative_matches < 0
3374 16797768 : || curr_static_id->operand[nop].type != OP_IN))
3375 : {
3376 11872872 : int last_reload = (lra_reg_info[ORIGINAL_REGNO
3377 11872872 : (operand_reg[nop])]
3378 11872872 : .last_reload);
3379 :
3380 : /* The value of reload_sum has sense only if we
3381 : process insns in their order. It happens only on
3382 : the first constraints sub-pass when we do most of
3383 : reload work. */
3384 11872872 : if (lra_constraint_iter == 1 && last_reload > bb_reload_num)
3385 2568755 : reload_sum += last_reload - bb_reload_num;
3386 : }
3387 : /* If this is a constant that is reloaded into the
3388 : desired class by copying it to memory first, count
3389 : that as another reload. This is consistent with
3390 : other code and is required to avoid choosing another
3391 : alternative when the constant is moved into memory.
3392 : Note that the test here is precisely the same as in
3393 : the code below that calls force_const_mem. */
3394 231236316 : if (CONST_POOL_OK_P (mode, op)
3395 231236373 : && ((targetm.preferred_reload_class
3396 51185444 : (op, this_alternative) == NO_REGS)
3397 49597413 : || no_input_reloads_p))
3398 : {
3399 1588031 : const_to_mem = 1;
3400 1588031 : if (! no_regs_p)
3401 : {
3402 729540 : if (lra_dump_file != NULL)
3403 0 : fprintf
3404 0 : (lra_dump_file,
3405 : " %d Constant reload through memory: "
3406 : "losers++\n", nop);
3407 729540 : losers++;
3408 : }
3409 : }
3410 :
3411 : /* Alternative loses if it requires a type of reload not
3412 : permitted for this insn. We can always reload
3413 : objects with a REG_UNUSED note. */
3414 180050929 : if ((curr_static_id->operand[nop].type != OP_IN
3415 87438641 : && no_output_reloads_p
3416 0 : && ! find_reg_note (curr_insn, REG_UNUSED, op)
3417 0 : && ! scratch_p)
3418 180050929 : || (curr_static_id->operand[nop].type != OP_OUT
3419 92612480 : && no_input_reloads_p && ! const_to_mem)
3420 360101858 : || (this_alternative_matches >= 0
3421 42521571 : && (no_input_reloads_p
3422 42521571 : || (no_output_reloads_p
3423 0 : && (curr_static_id->operand
3424 0 : [this_alternative_matches].type != OP_IN)
3425 0 : && ! find_reg_note (curr_insn, REG_UNUSED,
3426 : no_subreg_reg_operand
3427 0 : [this_alternative_matches])
3428 0 : && ! scratch_p))))
3429 : {
3430 0 : if (lra_dump_file != NULL)
3431 0 : fprintf
3432 0 : (lra_dump_file,
3433 : " No input/output reload -- refuse\n");
3434 0 : goto fail;
3435 : }
3436 :
3437 : /* Alternative loses if it required class pseudo cannot
3438 : hold value of required mode. Such insns can be
3439 : described by insn definitions with mode iterators. */
3440 180050929 : if (GET_MODE (*curr_id->operand_loc[nop]) != VOIDmode
3441 130366859 : && ! hard_reg_set_empty_p (this_alternative_set)
3442 : /* It is common practice for constraints to use a
3443 : class which does not have actually enough regs to
3444 : hold the value (e.g. x86 AREG for mode requiring
3445 : more one general reg). Therefore we have 2
3446 : conditions to check that the reload pseudo cannot
3447 : hold the mode value. */
3448 121527673 : && (!targetm.hard_regno_mode_ok
3449 121527673 : (ira_class_hard_regs[this_alternative][0],
3450 : GET_MODE (*curr_id->operand_loc[nop])))
3451 : /* The above condition is not enough as the first
3452 : reg in ira_class_hard_regs can be not aligned for
3453 : multi-words mode values. */
3454 180050929 : && (prohibited_class_reg_set_mode_p
3455 0 : (this_alternative, this_alternative_set,
3456 0 : GET_MODE (*curr_id->operand_loc[nop]))))
3457 : {
3458 0 : if (lra_dump_file != NULL)
3459 0 : fprintf (lra_dump_file,
3460 : " reload pseudo for op %d "
3461 : "cannot hold the mode value -- refuse\n",
3462 : nop);
3463 0 : goto fail;
3464 : }
3465 :
3466 : /* Check strong discouragement of reload of non-constant
3467 : into class THIS_ALTERNATIVE. */
3468 128865482 : if (! CONSTANT_P (op) && ! no_regs_p
3469 300179902 : && (targetm.preferred_reload_class
3470 120128973 : (op, this_alternative) == NO_REGS
3471 111403138 : || (curr_static_id->operand[nop].type == OP_OUT
3472 76625308 : && (targetm.preferred_output_reload_class
3473 76625308 : (op, this_alternative) == NO_REGS))))
3474 : {
3475 13362902 : if (offmemok && REG_P (op))
3476 : {
3477 811317 : if (lra_dump_file != NULL)
3478 0 : fprintf
3479 0 : (lra_dump_file,
3480 : " %d Spill pseudo into memory: reject+=3\n",
3481 : nop);
3482 811317 : reject += 3;
3483 : }
3484 : else
3485 : {
3486 12551585 : if (lra_dump_file != NULL)
3487 0 : fprintf
3488 0 : (lra_dump_file,
3489 : " %d Non-preferred reload: reject+=%d\n",
3490 : nop, LRA_MAX_REJECT);
3491 12551585 : reject += LRA_MAX_REJECT;
3492 : }
3493 : }
3494 :
3495 180050929 : if (! (MEM_P (op) && offmemok)
3496 180050052 : && ! (const_to_mem && constmemok))
3497 : {
3498 : /* We prefer to reload pseudos over reloading other
3499 : things, since such reloads may be able to be
3500 : eliminated later. So bump REJECT in other cases.
3501 : Don't do this in the case where we are forcing a
3502 : constant into memory and it will then win since
3503 : we don't want to have a different alternative
3504 : match then. */
3505 179070616 : if (! (REG_P (op) && REGNO (op) >= FIRST_PSEUDO_REGISTER))
3506 : {
3507 128251616 : if (lra_dump_file != NULL)
3508 1649 : fprintf
3509 1649 : (lra_dump_file,
3510 : " %d Non-pseudo reload: reject+=2\n",
3511 : nop);
3512 128251616 : reject += 2;
3513 : }
3514 :
3515 179070616 : if (! no_regs_p)
3516 170334337 : reload_nregs
3517 170334337 : += ira_reg_class_max_nregs[this_alternative][mode];
3518 :
3519 179070616 : if (SMALL_REGISTER_CLASS_P (this_alternative))
3520 : {
3521 934279 : if (lra_dump_file != NULL)
3522 45 : fprintf
3523 45 : (lra_dump_file,
3524 : " %d Small class reload: reject+=%d\n",
3525 : nop, LRA_LOSER_COST_FACTOR / 2);
3526 934279 : reject += LRA_LOSER_COST_FACTOR / 2;
3527 : }
3528 : }
3529 :
3530 : /* We are trying to spill pseudo into memory. It is
3531 : usually more costly than moving to a hard register
3532 : although it might takes the same number of
3533 : reloads.
3534 :
3535 : Non-pseudo spill may happen also. Suppose a target allows both
3536 : register and memory in the operand constraint alternatives,
3537 : then it's typical that an eliminable register has a substition
3538 : of "base + offset" which can either be reloaded by a simple
3539 : "new_reg <= base + offset" which will match the register
3540 : constraint, or a similar reg addition followed by further spill
3541 : to and reload from memory which will match the memory
3542 : constraint, but this memory spill will be much more costly
3543 : usually.
3544 :
3545 : Code below increases the reject for both pseudo and non-pseudo
3546 : spill. */
3547 180050929 : if (no_regs_p
3548 9595000 : && !(MEM_P (op) && offmemok)
3549 9594643 : && !(REG_P (op) && hard_regno[nop] < 0))
3550 : {
3551 8524622 : if (lra_dump_file != NULL)
3552 13 : fprintf
3553 20 : (lra_dump_file,
3554 : " %d Spill %spseudo into memory: reject+=3\n",
3555 : nop, REG_P (op) ? "" : "Non-");
3556 8524622 : reject += 3;
3557 8524622 : if (VECTOR_MODE_P (mode))
3558 : {
3559 : /* Spilling vectors into memory is usually more
3560 : costly as they contain big values. */
3561 386201 : if (lra_dump_file != NULL)
3562 0 : fprintf
3563 0 : (lra_dump_file,
3564 : " %d Spill vector pseudo: reject+=2\n",
3565 : nop);
3566 386201 : reject += 2;
3567 : }
3568 : }
3569 :
3570 : /* When we use an operand requiring memory in given
3571 : alternative, the insn should write *and* read the
3572 : value to/from memory it is costly in comparison with
3573 : an insn alternative which does not use memory
3574 : (e.g. register or immediate operand). We exclude
3575 : memory operand for such case as we can satisfy the
3576 : memory constraints by reloading address. */
3577 9595000 : if (no_regs_p && offmemok && !MEM_P (op))
3578 : {
3579 9594643 : if (lra_dump_file != NULL)
3580 26 : fprintf
3581 26 : (lra_dump_file,
3582 : " Using memory insn operand %d: reject+=3\n",
3583 : nop);
3584 9594643 : reject += 3;
3585 : }
3586 :
3587 : /* If reload requires moving value through secondary
3588 : memory, it will need one more insn at least. */
3589 180050929 : if (this_alternative != NO_REGS
3590 170455785 : && REG_P (op) && (cl = get_reg_class (REGNO (op))) != NO_REGS
3591 217348047 : && ((curr_static_id->operand[nop].type != OP_OUT
3592 21110025 : && targetm.secondary_memory_needed (mode, cl,
3593 : this_alternative))
3594 34122087 : || (curr_static_id->operand[nop].type != OP_IN
3595 16187211 : && (targetm.secondary_memory_needed
3596 16187211 : (mode, this_alternative, cl)))))
3597 : {
3598 10967908 : if (lra_dump_file != NULL)
3599 16 : fprintf
3600 16 : (lra_dump_file,
3601 : " %d Secondary memory reload needed: "
3602 : "losers++\n", nop);
3603 10967908 : losers++;
3604 : }
3605 :
3606 180050929 : if (MEM_P (op) && offmemok)
3607 877 : addr_losers++;
3608 : else
3609 : {
3610 : /* Input reloads can be inherited more often than
3611 : output reloads can be removed, so penalize output
3612 : reloads. */
3613 180050052 : if (!REG_P (op) || curr_static_id->operand[nop].type != OP_IN)
3614 : {
3615 151379597 : if (lra_dump_file != NULL)
3616 1709 : fprintf
3617 1709 : (lra_dump_file,
3618 : " %d Non input pseudo reload: reject++\n",
3619 : nop);
3620 151379597 : reject++;
3621 : }
3622 :
3623 180050052 : if (curr_static_id->operand[nop].type == OP_INOUT)
3624 : {
3625 192 : if (lra_dump_file != NULL)
3626 0 : fprintf
3627 0 : (lra_dump_file,
3628 : " %d Input/Output reload: reject+=%d\n",
3629 : nop, LRA_LOSER_COST_FACTOR);
3630 192 : reject += LRA_LOSER_COST_FACTOR;
3631 : }
3632 : }
3633 : }
3634 :
3635 480773032 : if (early_clobber_p && ! scratch_p)
3636 : {
3637 181363 : if (lra_dump_file != NULL)
3638 4 : fprintf (lra_dump_file,
3639 : " %d Early clobber: reject++\n", nop);
3640 181363 : reject++;
3641 : }
3642 : /* ??? We check early clobbers after processing all operands
3643 : (see loop below) and there we update the costs more.
3644 : Should we update the cost (may be approximately) here
3645 : because of early clobber register reloads or it is a rare
3646 : or non-important thing to be worth to do it. */
3647 961546064 : overall = (losers * LRA_LOSER_COST_FACTOR + reject
3648 480773032 : - (addr_losers == losers ? static_reject : 0));
3649 480773032 : if ((best_losers == 0 || losers != 0) && best_overall < overall)
3650 : {
3651 76312265 : if (lra_dump_file != NULL)
3652 1029 : fprintf (lra_dump_file,
3653 : " overall=%d,losers=%d -- refuse\n",
3654 : overall, losers);
3655 76312265 : goto fail;
3656 : }
3657 :
3658 404460767 : if (update_and_check_small_class_inputs (nop, nalt,
3659 : this_alternative))
3660 : {
3661 0 : if (lra_dump_file != NULL)
3662 0 : fprintf (lra_dump_file,
3663 : " not enough small class regs -- refuse\n");
3664 0 : goto fail;
3665 : }
3666 404460767 : curr_alt[nop] = this_alternative;
3667 404460767 : curr_alt_set[nop] = this_alternative_set;
3668 404460767 : curr_alt_exclude_start_hard_regs[nop]
3669 404460767 : = this_alternative_exclude_start_hard_regs;
3670 404460767 : curr_alt_win[nop] = this_alternative_win;
3671 404460767 : curr_alt_match_win[nop] = this_alternative_match_win;
3672 404460767 : curr_alt_offmemok[nop] = this_alternative_offmemok;
3673 404460767 : curr_alt_matches[nop] = this_alternative_matches;
3674 :
3675 404460767 : if (this_alternative_matches >= 0
3676 404460767 : && !did_match && !this_alternative_win)
3677 13780336 : curr_alt_win[this_alternative_matches] = false;
3678 :
3679 404460767 : if (early_clobber_p && operand_reg[nop] != NULL_RTX)
3680 185132 : early_clobbered_nops[early_clobbered_regs_num++] = nop;
3681 : }
3682 :
3683 138529140 : if (curr_insn_set != NULL_RTX
3684 : /* Allow just two operands or three operands where the third
3685 : is a clobber. */
3686 134541681 : && (n_operands == 2
3687 29665217 : || (n_operands == 3
3688 27392509 : && GET_CODE (PATTERN (curr_insn)) == PARALLEL
3689 23241962 : && XVECLEN (PATTERN (curr_insn), 0) == 2
3690 23186842 : && GET_CODE (XVECEXP (PATTERN (curr_insn), 0, 1))
3691 : == CLOBBER))
3692 : /* Prevent processing non-move insns. */
3693 127977240 : && (GET_CODE (SET_SRC (curr_insn_set)) == SUBREG
3694 126071953 : || SET_SRC (curr_insn_set) == no_subreg_reg_operand[1])
3695 232270040 : && ((! curr_alt_win[0] && ! curr_alt_win[1]
3696 6131501 : && REG_P (no_subreg_reg_operand[0])
3697 2988288 : && REG_P (no_subreg_reg_operand[1])
3698 1249325 : && (reg_in_class_p (no_subreg_reg_operand[0], curr_alt[1])
3699 1031686 : || reg_in_class_p (no_subreg_reg_operand[1], curr_alt[0])))
3700 93192803 : || (! curr_alt_win[0] && curr_alt_win[1]
3701 27709726 : && REG_P (no_subreg_reg_operand[1])
3702 : /* Check that we reload memory not the memory
3703 : address. */
3704 15861204 : && ! (curr_alt_offmemok[0]
3705 393882 : && MEM_P (no_subreg_reg_operand[0]))
3706 15861204 : && reg_in_class_p (no_subreg_reg_operand[1], curr_alt[0]))
3707 78750928 : || (curr_alt_win[0] && ! curr_alt_win[1]
3708 10056699 : && REG_P (no_subreg_reg_operand[0])
3709 : /* Check that we reload memory not the memory
3710 : address. */
3711 7534199 : && ! (curr_alt_offmemok[1]
3712 1388456 : && MEM_P (no_subreg_reg_operand[1]))
3713 7534197 : && reg_in_class_p (no_subreg_reg_operand[0], curr_alt[1])
3714 6638826 : && (! CONST_POOL_OK_P (curr_operand_mode[1],
3715 : no_subreg_reg_operand[1])
3716 2632895 : || (targetm.preferred_reload_class
3717 2632895 : (no_subreg_reg_operand[1],
3718 : (enum reg_class) curr_alt[1]) != NO_REGS))
3719 : /* If it is a result of recent elimination in move
3720 : insn we can transform it into an add still by
3721 : using this alternative. */
3722 6594929 : && GET_CODE (no_subreg_reg_operand[1]) != PLUS
3723 : /* Likewise if the source has been replaced with an
3724 : equivalent value. This only happens once -- the reload
3725 : will use the equivalent value instead of the register it
3726 : replaces -- so there should be no danger of cycling. */
3727 6059074 : && !equiv_substition_p[1])))
3728 : {
3729 : /* We have a move insn and a new reload insn will be similar
3730 : to the current insn. We should avoid such situation as
3731 : it results in LRA cycling. */
3732 20998331 : if (lra_dump_file != NULL)
3733 233 : fprintf (lra_dump_file,
3734 : " Cycle danger: overall += LRA_MAX_REJECT\n");
3735 20998331 : overall += LRA_MAX_REJECT;
3736 : }
3737 138529140 : if (all_this_alternative != NO_REGS
3738 118642711 : && !SMALL_REGISTER_CLASS_P (all_this_alternative)
3739 117792247 : && all_used_nregs != 0 && all_reload_nregs != 0
3740 138529140 : && (all_used_nregs + all_reload_nregs + 1
3741 4242887 : >= ira_class_hard_regs_num[all_this_alternative]))
3742 : {
3743 363 : if (lra_dump_file != NULL)
3744 0 : fprintf
3745 0 : (lra_dump_file,
3746 : " Register starvation: overall += LRA_MAX_REJECT"
3747 : "(class=%s,avail=%d,used=%d,reload=%d)\n",
3748 : reg_class_names[all_this_alternative],
3749 : ira_class_hard_regs_num[all_this_alternative],
3750 : all_used_nregs, all_reload_nregs);
3751 363 : overall += LRA_MAX_REJECT;
3752 363 : if (!prefer_memory_p && INSN_CODE (curr_insn) < 0)
3753 : {
3754 : /* asm can permit memory and reg and can be not enough regs for
3755 : asm -- try now memory: */
3756 100 : prefer_memory_p = true;
3757 100 : if (lra_dump_file != NULL)
3758 0 : fprintf
3759 0 : (lra_dump_file,
3760 : " Trying now memory for operands\n");
3761 100 : goto repeat;
3762 : }
3763 : }
3764 138529040 : ok_p = true;
3765 138529040 : curr_alt_dont_inherit_ops_num = 0;
3766 138709674 : for (nop = 0; nop < early_clobbered_regs_num; nop++)
3767 : {
3768 180635 : int i, j, clobbered_hard_regno, first_conflict_j, last_conflict_j;
3769 180635 : HARD_REG_SET temp_set;
3770 :
3771 180635 : i = early_clobbered_nops[nop];
3772 180635 : if ((! curr_alt_win[i] && ! curr_alt_match_win[i])
3773 134439 : || hard_regno[i] < 0)
3774 179930 : continue;
3775 132860 : lra_assert (operand_reg[i] != NULL_RTX);
3776 : clobbered_hard_regno = hard_regno[i];
3777 132860 : CLEAR_HARD_REG_SET (temp_set);
3778 132860 : add_to_hard_reg_set (&temp_set, GET_MODE (*curr_id->operand_loc[i]),
3779 : clobbered_hard_regno);
3780 132860 : first_conflict_j = last_conflict_j = -1;
3781 801777 : for (j = 0; j < n_operands; j++)
3782 536058 : if (j == i
3783 : /* We don't want process insides of match_operator and
3784 : match_parallel because otherwise we would process
3785 : their operands once again generating a wrong
3786 : code. */
3787 403198 : || curr_static_id->operand[j].is_operator)
3788 134974 : continue;
3789 401084 : else if ((curr_alt_matches[j] == i && curr_alt_match_win[j])
3790 380816 : || (curr_alt_matches[i] == j && curr_alt_match_win[i]))
3791 20268 : continue;
3792 : /* If we don't reload j-th operand, check conflicts. */
3793 143968 : else if ((curr_alt_win[j] || curr_alt_match_win[j])
3794 438019 : && uses_hard_regs_p (*curr_id->operand_loc[j], temp_set))
3795 : {
3796 1140 : if (first_conflict_j < 0)
3797 705 : first_conflict_j = j;
3798 1140 : last_conflict_j = j;
3799 : /* Both the earlyclobber operand and conflicting operand
3800 : cannot both be user defined hard registers for asm.
3801 : Let curr_insn_transform diagnose it. */
3802 1140 : if (HARD_REGISTER_P (operand_reg[i])
3803 1 : && REG_USERVAR_P (operand_reg[i])
3804 1 : && operand_reg[j] != NULL_RTX
3805 1 : && HARD_REGISTER_P (operand_reg[j])
3806 1 : && REG_USERVAR_P (operand_reg[j])
3807 1141 : && INSN_CODE (curr_insn) < 0)
3808 1 : return false;
3809 : }
3810 132859 : if (last_conflict_j < 0)
3811 132155 : continue;
3812 :
3813 : /* If an earlyclobber operand conflicts with another non-matching
3814 : operand (ie, they have been assigned the same hard register),
3815 : then it is better to reload the other operand, as there may
3816 : exist yet another operand with a matching constraint associated
3817 : with the earlyclobber operand. However, if one of the operands
3818 : is an explicit use of a hard register, then we must reload the
3819 : other non-hard register operand. */
3820 704 : if (HARD_REGISTER_P (operand_reg[i])
3821 704 : || (first_conflict_j == last_conflict_j
3822 269 : && operand_reg[last_conflict_j] != NULL_RTX
3823 63 : && !curr_alt_match_win[last_conflict_j]
3824 63 : && !HARD_REGISTER_P (operand_reg[last_conflict_j])))
3825 : {
3826 63 : curr_alt_win[last_conflict_j] = false;
3827 63 : curr_alt_dont_inherit_ops[curr_alt_dont_inherit_ops_num++]
3828 63 : = last_conflict_j;
3829 63 : losers++;
3830 63 : if (lra_dump_file != NULL)
3831 0 : fprintf
3832 0 : (lra_dump_file,
3833 : " %d Conflict early clobber reload: losers++\n",
3834 : i);
3835 : }
3836 : else
3837 : {
3838 : /* We need to reload early clobbered register and the
3839 : matched registers. */
3840 3199 : for (j = 0; j < n_operands; j++)
3841 2558 : if (curr_alt_matches[j] == i)
3842 : {
3843 2 : curr_alt_match_win[j] = false;
3844 2 : losers++;
3845 2 : if (lra_dump_file != NULL)
3846 0 : fprintf
3847 0 : (lra_dump_file,
3848 : " %d Matching conflict early clobber "
3849 : "reloads: losers++\n",
3850 : j);
3851 2 : overall += LRA_LOSER_COST_FACTOR;
3852 : }
3853 641 : if (! curr_alt_match_win[i])
3854 641 : curr_alt_dont_inherit_ops[curr_alt_dont_inherit_ops_num++] = i;
3855 : else
3856 : {
3857 : /* Remember pseudos used for match reloads are never
3858 : inherited. */
3859 0 : lra_assert (curr_alt_matches[i] >= 0);
3860 0 : curr_alt_win[curr_alt_matches[i]] = false;
3861 : }
3862 641 : curr_alt_win[i] = curr_alt_match_win[i] = false;
3863 641 : losers++;
3864 641 : if (lra_dump_file != NULL)
3865 0 : fprintf
3866 0 : (lra_dump_file,
3867 : " %d Matched conflict early clobber reloads: "
3868 : "losers++\n",
3869 : i);
3870 : }
3871 : /* Early clobber was already reflected in REJECT. */
3872 704 : if (!matching_early_clobber[i])
3873 : {
3874 704 : lra_assert (reject > 0);
3875 704 : reject--;
3876 704 : matching_early_clobber[i] = 1;
3877 : }
3878 704 : overall += LRA_LOSER_COST_FACTOR - 1;
3879 : }
3880 138529039 : if (lra_dump_file != NULL)
3881 1752 : fprintf (lra_dump_file, " overall=%d,losers=%d,rld_nregs=%d\n",
3882 : overall, losers, reload_nregs);
3883 :
3884 : /* If this alternative can be made to work by reloading, and it
3885 : needs less reloading than the others checked so far, record
3886 : it as the chosen goal for reloading. */
3887 138529039 : if ((best_losers != 0 && losers == 0)
3888 61560534 : || (((best_losers == 0 && losers == 0)
3889 60466138 : || (best_losers != 0 && losers != 0))
3890 61560534 : && (best_overall > overall
3891 15813233 : || (best_overall == overall
3892 : /* If the cost of the reloads is the same,
3893 : prefer alternative which requires minimal
3894 : number of reload regs. */
3895 11834146 : && (reload_nregs < best_reload_nregs
3896 11732792 : || (reload_nregs == best_reload_nregs
3897 11689376 : && (best_reload_sum < reload_sum
3898 11665394 : || (best_reload_sum == reload_sum
3899 11642096 : && nalt < goal_alt_number))))))))
3900 : {
3901 400001603 : for (nop = 0; nop < n_operands; nop++)
3902 : {
3903 276871902 : goal_alt_win[nop] = curr_alt_win[nop];
3904 276871902 : goal_alt_match_win[nop] = curr_alt_match_win[nop];
3905 276871902 : goal_alt_matches[nop] = curr_alt_matches[nop];
3906 276871902 : goal_alt[nop] = curr_alt[nop];
3907 276871902 : goal_alt_exclude_start_hard_regs[nop]
3908 276871902 : = curr_alt_exclude_start_hard_regs[nop];
3909 276871902 : goal_alt_offmemok[nop] = curr_alt_offmemok[nop];
3910 : }
3911 123129701 : goal_alt_dont_inherit_ops_num = curr_alt_dont_inherit_ops_num;
3912 123129701 : goal_reuse_alt_p = curr_reuse_alt_p;
3913 123130395 : for (nop = 0; nop < curr_alt_dont_inherit_ops_num; nop++)
3914 694 : goal_alt_dont_inherit_ops[nop] = curr_alt_dont_inherit_ops[nop];
3915 123129701 : goal_alt_swapped = curr_swapped;
3916 123129701 : goal_alt_out_sp_reload_p = curr_alt_out_sp_reload_p;
3917 123129701 : best_overall = overall;
3918 123129701 : best_losers = losers;
3919 123129701 : best_reload_nregs = reload_nregs;
3920 123129701 : best_reload_sum = reload_sum;
3921 123129701 : goal_alt_number = nalt;
3922 : }
3923 138529039 : if (losers == 0 && !curr_alt_class_change_p)
3924 : /* Everything is satisfied. Do not process alternatives
3925 : anymore. */
3926 : break;
3927 60478861 : fail:
3928 185936110 : ;
3929 : }
3930 : return ok_p;
3931 : }
3932 :
3933 : /* Make reload base reg from address AD. */
3934 : static rtx
3935 0 : base_to_reg (struct address_info *ad)
3936 : {
3937 0 : enum reg_class cl;
3938 0 : int code = -1;
3939 0 : rtx new_inner = NULL_RTX;
3940 0 : rtx new_reg = NULL_RTX;
3941 0 : rtx_insn *insn;
3942 0 : rtx_insn *last_insn = get_last_insn();
3943 :
3944 0 : lra_assert (ad->disp == ad->disp_term);
3945 0 : cl = base_reg_class (ad->mode, ad->as, ad->base_outer_code,
3946 : get_index_code (ad), ad->mem, curr_insn);
3947 0 : new_reg = lra_create_new_reg (GET_MODE (*ad->base), NULL_RTX, cl, NULL,
3948 : "base");
3949 0 : new_inner = simplify_gen_binary (PLUS, GET_MODE (new_reg), new_reg,
3950 0 : ad->disp_term == NULL
3951 : ? const0_rtx
3952 : : *ad->disp_term);
3953 0 : if (!valid_address_p (ad->mode, new_inner, ad->as))
3954 : return NULL_RTX;
3955 0 : insn = emit_insn (gen_rtx_SET (new_reg, *ad->base));
3956 0 : code = recog_memoized (insn);
3957 0 : if (code < 0)
3958 : {
3959 0 : delete_insns_since (last_insn);
3960 0 : return NULL_RTX;
3961 : }
3962 :
3963 : return new_inner;
3964 : }
3965 :
3966 : /* Make reload base reg + DISP from address AD. Return the new pseudo. */
3967 : static rtx
3968 47 : base_plus_disp_to_reg (struct address_info *ad, rtx disp)
3969 : {
3970 47 : enum reg_class cl;
3971 47 : rtx new_reg;
3972 :
3973 47 : lra_assert (ad->base == ad->base_term);
3974 47 : cl = base_reg_class (ad->mode, ad->as, ad->base_outer_code,
3975 : get_index_code (ad), ad->mem, curr_insn);
3976 47 : new_reg = lra_create_new_reg (GET_MODE (*ad->base_term), NULL_RTX, cl, NULL,
3977 : "base + disp");
3978 47 : lra_emit_add (new_reg, *ad->base_term, disp);
3979 47 : return new_reg;
3980 : }
3981 :
3982 : /* Make reload of index part of address AD. Return the new
3983 : pseudo. */
3984 : static rtx
3985 0 : index_part_to_reg (struct address_info *ad, enum reg_class index_class)
3986 : {
3987 0 : rtx new_reg;
3988 :
3989 0 : new_reg = lra_create_new_reg (GET_MODE (*ad->index), NULL_RTX,
3990 : index_class, NULL, "index term");
3991 0 : expand_mult (GET_MODE (*ad->index), *ad->index_term,
3992 : GEN_INT (get_index_scale (ad)), new_reg, 1);
3993 0 : return new_reg;
3994 : }
3995 :
3996 : /* Return true if we can add a displacement to address AD, even if that
3997 : makes the address invalid. The fix-up code requires any new address
3998 : to be the sum of the BASE_TERM, INDEX and DISP_TERM fields. */
3999 : static bool
4000 21964 : can_add_disp_p (struct address_info *ad)
4001 : {
4002 21964 : return (!ad->autoinc_p
4003 21964 : && ad->segment == NULL
4004 21964 : && ad->base == ad->base_term
4005 43928 : && ad->disp == ad->disp_term);
4006 : }
4007 :
4008 : /* Make equiv substitution in address AD. Return true if a substitution
4009 : was made. */
4010 : static bool
4011 40294615 : equiv_address_substitution (struct address_info *ad)
4012 : {
4013 40294615 : rtx base_reg, new_base_reg, index_reg, new_index_reg, *base_term, *index_term;
4014 40294615 : poly_int64 disp;
4015 40294615 : HOST_WIDE_INT scale;
4016 40294615 : bool change_p;
4017 :
4018 40294615 : base_term = strip_subreg (ad->base_term);
4019 10130 : if (base_term == NULL)
4020 : base_reg = new_base_reg = NULL_RTX;
4021 : else
4022 : {
4023 33980006 : base_reg = *base_term;
4024 33980006 : new_base_reg = get_equiv_with_elimination (base_reg, curr_insn);
4025 : }
4026 40294615 : index_term = strip_subreg (ad->index_term);
4027 6089 : if (index_term == NULL)
4028 : index_reg = new_index_reg = NULL_RTX;
4029 : else
4030 : {
4031 1819419 : index_reg = *index_term;
4032 1819419 : new_index_reg = get_equiv_with_elimination (index_reg, curr_insn);
4033 : }
4034 40294615 : if (base_reg == new_base_reg && index_reg == new_index_reg)
4035 : return false;
4036 143426 : disp = 0;
4037 143426 : change_p = false;
4038 143426 : if (lra_dump_file != NULL)
4039 : {
4040 0 : fprintf (lra_dump_file, "Changing address in insn %d ",
4041 0 : INSN_UID (curr_insn));
4042 0 : dump_value_slim (lra_dump_file, *ad->outer, 1);
4043 : }
4044 143426 : if (base_reg != new_base_reg)
4045 : {
4046 142971 : poly_int64 offset;
4047 142971 : if (REG_P (new_base_reg))
4048 : {
4049 8532 : *base_term = new_base_reg;
4050 8532 : change_p = true;
4051 : }
4052 134439 : else if (GET_CODE (new_base_reg) == PLUS
4053 21963 : && REG_P (XEXP (new_base_reg, 0))
4054 21963 : && poly_int_rtx_p (XEXP (new_base_reg, 1), &offset)
4055 156402 : && can_add_disp_p (ad))
4056 : {
4057 : disp += offset;
4058 21963 : *base_term = XEXP (new_base_reg, 0);
4059 21963 : change_p = true;
4060 : }
4061 142971 : if (ad->base_term2 != NULL)
4062 0 : *ad->base_term2 = *ad->base_term;
4063 : }
4064 143426 : if (index_reg != new_index_reg)
4065 : {
4066 663 : poly_int64 offset;
4067 663 : if (REG_P (new_index_reg))
4068 : {
4069 0 : *index_term = new_index_reg;
4070 0 : change_p = true;
4071 : }
4072 663 : else if (GET_CODE (new_index_reg) == PLUS
4073 1 : && REG_P (XEXP (new_index_reg, 0))
4074 1 : && poly_int_rtx_p (XEXP (new_index_reg, 1), &offset)
4075 1 : && can_add_disp_p (ad)
4076 664 : && (scale = get_index_scale (ad)))
4077 : {
4078 1 : disp += offset * scale;
4079 1 : *index_term = XEXP (new_index_reg, 0);
4080 1 : change_p = true;
4081 : }
4082 : }
4083 143426 : if (maybe_ne (disp, 0))
4084 : {
4085 21964 : if (ad->disp != NULL)
4086 8010 : *ad->disp = plus_constant (GET_MODE (*ad->inner), *ad->disp, disp);
4087 : else
4088 : {
4089 13954 : *ad->inner = plus_constant (GET_MODE (*ad->inner), *ad->inner, disp);
4090 13954 : update_address (ad);
4091 : }
4092 : change_p = true;
4093 : }
4094 143426 : if (lra_dump_file != NULL)
4095 : {
4096 0 : if (! change_p)
4097 0 : fprintf (lra_dump_file, " -- no change\n");
4098 : else
4099 : {
4100 0 : fprintf (lra_dump_file, " on equiv ");
4101 0 : dump_value_slim (lra_dump_file, *ad->outer, 1);
4102 0 : fprintf (lra_dump_file, "\n");
4103 : }
4104 : }
4105 : return change_p;
4106 : }
4107 :
4108 : /* Skip all modifiers and whitespaces in constraint STR and return the
4109 : result. */
4110 : static const char *
4111 525809313 : skip_constraint_modifiers (const char *str)
4112 : {
4113 739981429 : for (;;str++)
4114 632895371 : switch (*str)
4115 : {
4116 107086058 : case '+': case '&' : case '=': case '*': case ' ': case '\t':
4117 107086058 : case '$': case '^' : case '%': case '?': case '!':
4118 107086058 : break;
4119 525809313 : default: return str;
4120 : }
4121 : }
4122 :
4123 : /* Takes a string of 0 or more comma-separated constraints. When more
4124 : than one constraint is present, evaluate whether they all correspond
4125 : to a single, repeated constraint (e.g. "r,r") or whether we have
4126 : more than one distinct constraints (e.g. "r,m"). */
4127 : static bool
4128 166736499 : constraint_unique (const char *cstr)
4129 : {
4130 166736499 : enum constraint_num ca, cb;
4131 166736499 : ca = CONSTRAINT__UNKNOWN;
4132 330936641 : for (;;)
4133 : {
4134 330936641 : cstr = skip_constraint_modifiers (cstr);
4135 330936641 : if (*cstr == '\0' || *cstr == ',')
4136 : cb = CONSTRAINT_X;
4137 : else
4138 : {
4139 330936641 : cb = lookup_constraint (cstr);
4140 330936641 : if (cb == CONSTRAINT__UNKNOWN)
4141 : return false;
4142 318572697 : cstr += CONSTRAINT_LEN (cstr[0], cstr);
4143 : }
4144 : /* Handle the first iteration of the loop. */
4145 318572697 : if (ca == CONSTRAINT__UNKNOWN)
4146 : ca = cb;
4147 : /* Handle the general case of comparing ca with subsequent
4148 : constraints. */
4149 164022276 : else if (ca != cb)
4150 : return false;
4151 171943831 : if (*cstr == '\0')
4152 : return true;
4153 164200142 : if (*cstr == ',')
4154 90893757 : cstr += 1;
4155 : }
4156 : }
4157 :
4158 : /* Major function to make reloads for an address in operand NOP or
4159 : check its correctness (If CHECK_ONLY_P is true). The supported
4160 : cases are:
4161 :
4162 : 1) an address that existed before LRA started, at which point it
4163 : must have been valid. These addresses are subject to elimination
4164 : and may have become invalid due to the elimination offset being out
4165 : of range.
4166 :
4167 : 2) an address created by forcing a constant to memory
4168 : (force_const_to_mem). The initial form of these addresses might
4169 : not be valid, and it is this function's job to make them valid.
4170 :
4171 : 3) a frame address formed from a register and a (possibly zero)
4172 : constant offset. As above, these addresses might not be valid and
4173 : this function must make them so.
4174 :
4175 : Add reloads to the lists *BEFORE and *AFTER. We might need to add
4176 : reloads to *AFTER because of inc/dec, {pre, post} modify in the
4177 : address. Return true for any RTL change.
4178 :
4179 : The function is a helper function which does not produce all
4180 : transformations (when CHECK_ONLY_P is false) which can be
4181 : necessary. It does just basic steps. To do all necessary
4182 : transformations use function process_address. */
4183 : static bool
4184 180848181 : process_address_1 (int nop, bool check_only_p,
4185 : rtx_insn **before, rtx_insn **after)
4186 : {
4187 180848181 : struct address_info ad;
4188 180848181 : rtx new_reg;
4189 180848181 : HOST_WIDE_INT scale;
4190 180848181 : rtx op = *curr_id->operand_loc[nop];
4191 180848181 : rtx mem = extract_mem_from_operand (op);
4192 180848181 : const char *constraint;
4193 180848181 : enum constraint_num cn;
4194 180848181 : bool change_p = false;
4195 :
4196 180848181 : if (MEM_P (mem)
4197 38455514 : && GET_MODE (mem) == BLKmode
4198 24736 : && GET_CODE (XEXP (mem, 0)) == SCRATCH)
4199 : return false;
4200 :
4201 180848181 : constraint
4202 180848181 : = skip_constraint_modifiers (curr_static_id->operand[nop].constraint);
4203 180848181 : if (IN_RANGE (constraint[0], '0', '9'))
4204 : {
4205 14024491 : char *end;
4206 14024491 : unsigned long dup = strtoul (constraint, &end, 10);
4207 14024491 : constraint
4208 14024491 : = skip_constraint_modifiers (curr_static_id->operand[dup].constraint);
4209 : }
4210 193113538 : cn = lookup_constraint (*constraint == '\0' ? "X" : constraint);
4211 : /* If we have several alternatives or/and several constraints in an
4212 : alternative and we can not say at this stage what constraint will be used,
4213 : use unknown constraint. The exception is an address constraint. If
4214 : operand has one address constraint, probably all others constraints are
4215 : address ones. */
4216 168582824 : if (constraint[0] != '\0' && get_constraint_type (cn) != CT_ADDRESS
4217 347584680 : && !constraint_unique (constraint))
4218 : cn = CONSTRAINT__UNKNOWN;
4219 21855371 : if (insn_extra_address_constraint (cn)
4220 : /* When we find an asm operand with an address constraint that
4221 : doesn't satisfy address_operand to begin with, we clear
4222 : is_address, so that we don't try to make a non-address fit.
4223 : If the asm statement got this far, it's because other
4224 : constraints are available, and we'll use them, disregarding
4225 : the unsatisfiable address ones. */
4226 21855371 : && curr_static_id->operand[nop].is_address)
4227 1846306 : decompose_lea_address (&ad, curr_id->operand_loc[nop]);
4228 : /* Do not attempt to decompose arbitrary addresses generated by combine
4229 : for asm operands with loose constraints, e.g 'X'.
4230 : Need to extract memory from op for special memory constraint,
4231 : i.e. bcst_mem_operand in i386 backend. */
4232 179001875 : else if (MEM_P (mem)
4233 179002037 : && !(INSN_CODE (curr_insn) < 0
4234 20561 : && get_constraint_type (cn) == CT_FIXED_FORM
4235 162 : && constraint_satisfied_p (op, cn)))
4236 38455352 : decompose_mem_address (&ad, mem);
4237 140546523 : else if (GET_CODE (op) == SUBREG
4238 3789635 : && MEM_P (SUBREG_REG (op)))
4239 0 : decompose_mem_address (&ad, SUBREG_REG (op));
4240 : else
4241 : return false;
4242 : /* If INDEX_REG_CLASS is assigned to base_term already and isn't to
4243 : index_term, swap them so to avoid assigning INDEX_REG_CLASS to both
4244 : when INDEX_REG_CLASS is a single register class. */
4245 40301658 : enum reg_class index_cl = index_reg_class (curr_insn);
4246 40301658 : if (ad.base_term != NULL
4247 33986994 : && ad.index_term != NULL
4248 1450591 : && ira_class_hard_regs_num[index_cl] == 1
4249 0 : && REG_P (*ad.base_term)
4250 0 : && REG_P (*ad.index_term)
4251 0 : && in_class_p (*ad.base_term, index_cl, NULL)
4252 40301658 : && ! in_class_p (*ad.index_term, index_cl, NULL))
4253 : {
4254 0 : std::swap (ad.base, ad.index);
4255 0 : std::swap (ad.base_term, ad.index_term);
4256 : }
4257 40301658 : if (! check_only_p)
4258 40294615 : change_p = equiv_address_substitution (&ad);
4259 40301658 : if (ad.base_term != NULL
4260 74288652 : && (process_addr_reg
4261 67973988 : (ad.base_term, check_only_p, before,
4262 33986994 : (ad.autoinc_p
4263 4191560 : && !(REG_P (*ad.base_term)
4264 2095780 : && find_regno_note (curr_insn, REG_DEAD,
4265 : REGNO (*ad.base_term)) != NULL_RTX)
4266 : ? after : NULL),
4267 : base_reg_class (ad.mode, ad.as, ad.base_outer_code,
4268 : get_index_code (&ad), ad.mem, curr_insn))))
4269 : {
4270 438555 : change_p = true;
4271 438555 : if (ad.base_term2 != NULL)
4272 0 : *ad.base_term2 = *ad.base_term;
4273 : }
4274 40301658 : if (ad.index_term != NULL
4275 40301658 : && process_addr_reg (ad.index_term, check_only_p,
4276 : before, NULL, index_cl))
4277 : change_p = true;
4278 :
4279 : /* Target hooks sometimes don't treat extra-constraint addresses as
4280 : legitimate address_operands, so handle them specially. */
4281 40301658 : if (insn_extra_address_constraint (cn)
4282 40301658 : && satisfies_address_constraint_p (&ad, cn))
4283 : return change_p;
4284 :
4285 38455359 : if (check_only_p)
4286 : return change_p;
4287 :
4288 : /* There are three cases where the shape of *AD.INNER may now be invalid:
4289 :
4290 : 1) the original address was valid, but either elimination or
4291 : equiv_address_substitution was applied and that made
4292 : the address invalid.
4293 :
4294 : 2) the address is an invalid symbolic address created by
4295 : force_const_to_mem.
4296 :
4297 : 3) the address is a frame address with an invalid offset.
4298 :
4299 : 4) the address is a frame address with an invalid base.
4300 :
4301 : All these cases involve a non-autoinc address, so there is no
4302 : point revalidating other types. */
4303 38448962 : if (ad.autoinc_p || valid_address_p (op, &ad, cn))
4304 : return change_p;
4305 :
4306 : /* Any index existed before LRA started, so we can assume that the
4307 : presence and shape of the index is valid. */
4308 452 : push_to_sequence (*before);
4309 452 : lra_assert (ad.disp == ad.disp_term);
4310 452 : if (ad.base == NULL)
4311 : {
4312 352 : if (ad.index == NULL)
4313 : {
4314 352 : rtx_insn *insn;
4315 352 : rtx_insn *last = get_last_insn ();
4316 352 : int code = -1;
4317 352 : enum reg_class cl = base_reg_class (ad.mode, ad.as,
4318 : SCRATCH, SCRATCH,
4319 : ad.mem, curr_insn);
4320 352 : rtx addr = *ad.inner;
4321 :
4322 695 : new_reg = lra_create_new_reg (Pmode, NULL_RTX, cl, NULL, "addr");
4323 352 : if (HAVE_lo_sum)
4324 : {
4325 : /* addr => lo_sum (new_base, addr), case (2) above. */
4326 : insn = emit_insn (gen_rtx_SET
4327 : (new_reg,
4328 : gen_rtx_HIGH (Pmode, copy_rtx (addr))));
4329 : code = recog_memoized (insn);
4330 : if (code >= 0)
4331 : {
4332 : *ad.inner = gen_rtx_LO_SUM (Pmode, new_reg, addr);
4333 : if (!valid_address_p (op, &ad, cn))
4334 : {
4335 : /* Try to put lo_sum into register. */
4336 : insn = emit_insn (gen_rtx_SET
4337 : (new_reg,
4338 : gen_rtx_LO_SUM (Pmode, new_reg, addr)));
4339 : code = recog_memoized (insn);
4340 : if (code >= 0)
4341 : {
4342 : *ad.inner = new_reg;
4343 : if (!valid_address_p (op, &ad, cn))
4344 : {
4345 : *ad.inner = addr;
4346 : code = -1;
4347 : }
4348 : }
4349 :
4350 : }
4351 : }
4352 : if (code < 0)
4353 : delete_insns_since (last);
4354 : }
4355 :
4356 352 : if (code < 0)
4357 : {
4358 : /* addr => new_base, case (2) above. */
4359 352 : lra_emit_move (new_reg, addr);
4360 :
4361 704 : for (insn = last == NULL_RTX ? get_insns () : NEXT_INSN (last);
4362 704 : insn != NULL_RTX;
4363 352 : insn = NEXT_INSN (insn))
4364 352 : if (recog_memoized (insn) < 0)
4365 : break;
4366 352 : if (insn != NULL_RTX)
4367 : {
4368 : /* Do nothing if we cannot generate right insns.
4369 : This is analogous to reload pass behavior. */
4370 0 : delete_insns_since (last);
4371 0 : end_sequence ();
4372 0 : return false;
4373 : }
4374 352 : *ad.inner = new_reg;
4375 : }
4376 : }
4377 : else
4378 : {
4379 : /* index * scale + disp => new base + index * scale,
4380 : case (1) above. */
4381 0 : enum reg_class cl = base_reg_class (ad.mode, ad.as, PLUS,
4382 0 : GET_CODE (*ad.index),
4383 : ad.mem, curr_insn);
4384 :
4385 0 : lra_assert (index_cl != NO_REGS);
4386 0 : new_reg = lra_create_new_reg (Pmode, NULL_RTX, cl, NULL, "disp");
4387 0 : lra_emit_move (new_reg, *ad.disp);
4388 0 : *ad.inner = simplify_gen_binary (PLUS, GET_MODE (new_reg),
4389 0 : new_reg, *ad.index);
4390 : }
4391 : }
4392 100 : else if (ad.index == NULL)
4393 : {
4394 53 : int regno;
4395 53 : enum reg_class cl;
4396 53 : rtx set;
4397 53 : rtx_insn *insns, *last_insn;
4398 :
4399 53 : cl = base_reg_class (ad.mode, ad.as, ad.base_outer_code,
4400 : get_index_code (&ad), ad.mem, curr_insn);
4401 :
4402 53 : if (REG_P (*ad.base_term)
4403 53 : && ira_class_subset_p[get_reg_class (REGNO (*ad.base_term))][cl])
4404 : /* It seems base reg is already in the base reg class and changing it
4405 : does not make a progress. So reload the whole inner address. */
4406 53 : goto reload_inner_addr;
4407 :
4408 : /* Try to reload base into register only if the base is invalid
4409 : for the address but with valid offset, case (4) above. */
4410 0 : start_sequence ();
4411 0 : new_reg = base_to_reg (&ad);
4412 :
4413 : /* base + disp => new base, cases (1) and (3) above. */
4414 : /* Another option would be to reload the displacement into an
4415 : index register. However, postreload has code to optimize
4416 : address reloads that have the same base and different
4417 : displacements, so reloading into an index register would
4418 : not necessarily be a win. */
4419 0 : if (new_reg == NULL_RTX)
4420 : {
4421 : /* See if the target can split the displacement into a
4422 : legitimate new displacement from a local anchor. */
4423 0 : gcc_assert (ad.disp == ad.disp_term);
4424 0 : poly_int64 orig_offset;
4425 0 : rtx offset1, offset2;
4426 0 : if (poly_int_rtx_p (*ad.disp, &orig_offset)
4427 0 : && targetm.legitimize_address_displacement (&offset1, &offset2,
4428 : orig_offset,
4429 : ad.mode))
4430 : {
4431 0 : new_reg = base_plus_disp_to_reg (&ad, offset1);
4432 0 : new_reg = gen_rtx_PLUS (GET_MODE (new_reg), new_reg, offset2);
4433 : }
4434 : else
4435 0 : new_reg = base_plus_disp_to_reg (&ad, *ad.disp);
4436 : }
4437 0 : insns = get_insns ();
4438 0 : last_insn = get_last_insn ();
4439 : /* If we generated at least two insns, try last insn source as
4440 : an address. If we succeed, we generate one less insn. */
4441 0 : if (REG_P (new_reg)
4442 0 : && last_insn != insns
4443 0 : && (set = single_set (last_insn)) != NULL_RTX
4444 0 : && reg_plus_const_p (SET_SRC (set)))
4445 : {
4446 0 : *ad.inner = SET_SRC (set);
4447 0 : if (valid_address_p (op, &ad, cn))
4448 : {
4449 0 : *ad.base_term = XEXP (SET_SRC (set), 0);
4450 0 : *ad.disp_term = XEXP (SET_SRC (set), 1);
4451 0 : regno = REGNO (*ad.base_term);
4452 0 : if (regno >= FIRST_PSEUDO_REGISTER
4453 0 : && cl != lra_get_allocno_class (regno))
4454 0 : lra_change_class (regno, cl, " Change to", true);
4455 0 : new_reg = SET_SRC (set);
4456 0 : delete_insns_since (PREV_INSN (last_insn));
4457 : }
4458 : }
4459 0 : end_sequence ();
4460 0 : emit_insn (insns);
4461 0 : *ad.inner = new_reg;
4462 : }
4463 47 : else if (ad.disp_term != NULL)
4464 : {
4465 : /* base + scale * index + disp => new base + scale * index,
4466 : case (1) above. */
4467 47 : gcc_assert (ad.disp == ad.disp_term);
4468 47 : new_reg = base_plus_disp_to_reg (&ad, *ad.disp);
4469 47 : *ad.inner = simplify_gen_binary (PLUS, GET_MODE (new_reg),
4470 47 : new_reg, *ad.index);
4471 : }
4472 0 : else if ((scale = get_index_scale (&ad)) == 1)
4473 : {
4474 : /* The last transformation to one reg will be made in
4475 : curr_insn_transform function. */
4476 0 : end_sequence ();
4477 0 : return false;
4478 : }
4479 0 : else if (scale != 0)
4480 : {
4481 : /* base + scale * index => base + new_reg,
4482 : case (1) above.
4483 : Index part of address may become invalid. For example, we
4484 : changed pseudo on the equivalent memory and a subreg of the
4485 : pseudo onto the memory of different mode for which the scale is
4486 : prohibited. */
4487 0 : new_reg = index_part_to_reg (&ad, index_cl);
4488 0 : *ad.inner = simplify_gen_binary (PLUS, GET_MODE (new_reg),
4489 0 : *ad.base_term, new_reg);
4490 : }
4491 : else
4492 : {
4493 53 : enum reg_class cl;
4494 53 : rtx addr;
4495 0 : reload_inner_addr:
4496 53 : cl = base_reg_class (ad.mode, ad.as, SCRATCH, SCRATCH, ad.mem,
4497 : curr_insn);
4498 53 : addr = *ad.inner;
4499 53 : new_reg = lra_create_new_reg (Pmode, NULL_RTX, cl, NULL, "addr");
4500 : /* addr => new_base. */
4501 53 : lra_emit_move (new_reg, canonicalize_reload_addr (addr));
4502 53 : *ad.inner = new_reg;
4503 : }
4504 452 : *before = end_sequence ();
4505 452 : return true;
4506 : }
4507 :
4508 : /* If CHECK_ONLY_P is false, do address reloads until it is necessary.
4509 : Use process_address_1 as a helper function. Return true for any
4510 : RTL changes.
4511 :
4512 : If CHECK_ONLY_P is true, just check address correctness. Return
4513 : false if the address correct. */
4514 : static bool
4515 180352378 : process_address (int nop, bool check_only_p,
4516 : rtx_insn **before, rtx_insn **after)
4517 : {
4518 180352378 : bool res = false;
4519 : /* Use enough iterations to process all address parts: */
4520 180848181 : for (int i = 0; i < 10; i++)
4521 : {
4522 180848181 : if (!process_address_1 (nop, check_only_p, before, after))
4523 : {
4524 : return res;
4525 : }
4526 : else
4527 : {
4528 495803 : if (check_only_p)
4529 : return true;
4530 495803 : res = true;
4531 : }
4532 : }
4533 0 : fatal_insn ("unable to reload address in ", curr_insn);
4534 : }
4535 :
4536 : /* Override the generic address_reload_context in order to
4537 : control the creation of reload pseudos. */
4538 : class lra_autoinc_reload_context : public address_reload_context
4539 : {
4540 : machine_mode mode;
4541 : enum reg_class rclass;
4542 :
4543 : public:
4544 0 : lra_autoinc_reload_context (machine_mode mode, enum reg_class new_rclass)
4545 0 : : mode (mode), rclass (new_rclass) {}
4546 :
4547 0 : rtx get_reload_reg () const override final
4548 : {
4549 0 : return lra_create_new_reg (mode, NULL_RTX, rclass, NULL, "INC/DEC result");
4550 : }
4551 : };
4552 :
4553 : /* Emit insns to reload VALUE into a new register. VALUE is an
4554 : auto-increment or auto-decrement RTX whose operand is a register or
4555 : memory location; so reloading involves incrementing that location.
4556 :
4557 : INC_AMOUNT is the number to increment or decrement by (always
4558 : positive and ignored for POST_MODIFY/PRE_MODIFY).
4559 :
4560 : Return a pseudo containing the result. */
4561 : static rtx
4562 0 : emit_inc (enum reg_class new_rclass, rtx value, poly_int64 inc_amount)
4563 : {
4564 0 : lra_autoinc_reload_context context (GET_MODE (value), new_rclass);
4565 0 : return context.emit_autoinc (value, inc_amount);
4566 : }
4567 :
4568 : /* Return true if the current move insn does not need processing as we
4569 : already know that it satisfies its constraints. */
4570 : static bool
4571 103349950 : simple_move_p (void)
4572 : {
4573 103349950 : rtx dest, src;
4574 103349950 : enum reg_class dclass, sclass;
4575 :
4576 103349950 : lra_assert (curr_insn_set != NULL_RTX);
4577 103349950 : dest = SET_DEST (curr_insn_set);
4578 103349950 : src = SET_SRC (curr_insn_set);
4579 :
4580 : /* If the instruction has multiple sets we need to process it even if it
4581 : is single_set. This can happen if one or more of the SETs are dead.
4582 : See PR73650. */
4583 103349950 : if (multiple_sets (curr_insn))
4584 : return false;
4585 :
4586 103145015 : return ((dclass = get_op_class (dest)) != NO_REGS
4587 21804023 : && (sclass = get_op_class (src)) != NO_REGS
4588 : /* The backend guarantees that register moves of cost 2
4589 : never need reloads. */
4590 92203841 : && targetm.register_move_cost (GET_MODE (src), sclass, dclass) == 2);
4591 : }
4592 :
4593 : /* Swap operands NOP and NOP + 1. */
4594 : static inline void
4595 21872757 : swap_operands (int nop)
4596 : {
4597 21872757 : std::swap (curr_operand_mode[nop], curr_operand_mode[nop + 1]);
4598 21872757 : std::swap (original_subreg_reg_mode[nop], original_subreg_reg_mode[nop + 1]);
4599 21872757 : std::swap (*curr_id->operand_loc[nop], *curr_id->operand_loc[nop + 1]);
4600 21872757 : std::swap (equiv_substition_p[nop], equiv_substition_p[nop + 1]);
4601 : /* Swap the duplicates too. */
4602 21872757 : lra_update_dup (curr_id, nop);
4603 21872757 : lra_update_dup (curr_id, nop + 1);
4604 21872757 : }
4605 :
4606 : /* Return TRUE if X is a (subreg of) reg and there are no hard regs of X class
4607 : which can contain value of MODE. */
4608 32 : static bool invalid_mode_reg_p (enum machine_mode mode, rtx x)
4609 : {
4610 32 : if (SUBREG_P (x))
4611 1 : x = SUBREG_REG (x);
4612 32 : if (! REG_P (x))
4613 : return false;
4614 32 : enum reg_class rclass = get_reg_class (REGNO (x));
4615 32 : return (!hard_reg_set_empty_p (reg_class_contents[rclass])
4616 32 : && hard_reg_set_subset_p
4617 32 : (reg_class_contents[rclass],
4618 32 : ira_prohibited_class_mode_regs[rclass][mode]));
4619 : }
4620 :
4621 : /* Return TRUE if regno is referenced in more than one non-debug insn. */
4622 : static bool
4623 2958668 : multiple_insn_refs_p (int regno)
4624 : {
4625 2958668 : unsigned int uid;
4626 2958668 : bitmap_iterator bi;
4627 2958668 : int nrefs = 0;
4628 7240729 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi)
4629 : {
4630 7235824 : if (!NONDEBUG_INSN_P (lra_insn_recog_data[uid]->insn))
4631 1323393 : continue;
4632 5912431 : if (nrefs == 1)
4633 : return true;
4634 2958668 : nrefs++;
4635 : }
4636 : return false;
4637 : }
4638 :
4639 : /* Mark insns starting with FIRST as postponed for processing their
4640 : constraints. See comments for lra_postponed_insns. */
4641 : static void
4642 93986 : postpone_insns (rtx_insn *first)
4643 : {
4644 106290 : for (auto insn = first; insn != NULL_RTX; insn = NEXT_INSN (insn))
4645 : {
4646 12304 : bitmap_set_bit (&lra_postponed_insns, INSN_UID (insn));
4647 12304 : if (lra_dump_file != NULL)
4648 : {
4649 7 : fprintf (lra_dump_file, " Postponing constraint processing: ");
4650 7 : dump_insn_slim (lra_dump_file, insn);
4651 : }
4652 : }
4653 93986 : }
4654 :
4655 : /* Test whether the n-th operand is a MEM where the address is the sum of a
4656 : section anchor and a constant and return true in case of reloading the
4657 : section anchor only results in a satisfiable operand w.r.t. its
4658 : corresponding constraint. Otherwise return false. */
4659 :
4660 : static bool
4661 72 : reload_section_anchor_p (int nop)
4662 : {
4663 72 : rtx op = *curr_id->operand_loc[nop];
4664 72 : if (!MEM_P (op))
4665 : return false;
4666 72 : rtx addr = XEXP (op, 0);
4667 :
4668 72 : if (GET_CODE (addr) != CONST
4669 13 : || GET_CODE (XEXP (addr, 0)) != PLUS
4670 0 : || GET_CODE (XEXP (XEXP (addr, 0), 0)) != SYMBOL_REF
4671 0 : || !SYMBOL_REF_ANCHOR_P (XEXP (XEXP (addr, 0), 0))
4672 0 : || !CONST_INT_P (XEXP (XEXP (addr, 0), 1))
4673 : /* Some offsets are valid in conjunction with a symbol and
4674 : invalid in conjunction with a register. Thus, pull out
4675 : the anchor only in case the offset is a valid anchor
4676 : offset. */
4677 0 : || INTVAL (XEXP (XEXP (addr, 0), 1)) < targetm.min_anchor_offset
4678 72 : || INTVAL (XEXP (XEXP (addr, 0), 1)) > targetm.max_anchor_offset)
4679 : return false;
4680 :
4681 : /* Now test whether a new address of the form REG+DISPLACEMENT is valid for
4682 : the selected alternative. In order to do so, utilize lra_pmode_pseudo
4683 : instead of an actual reload register. */
4684 :
4685 0 : rtx offset = XEXP (XEXP (addr, 0), 1);
4686 0 : rtx new_addr = gen_rtx_PLUS (Pmode, lra_pmode_pseudo, offset);
4687 0 : rtx new_op = shallow_copy_rtx (op);
4688 0 : XEXP (new_op, 0) = new_addr;
4689 :
4690 : /* Get operand constraints for given alternative. */
4691 0 : const char *p = (curr_static_id->operand_alternative
4692 0 : [goal_alt_number * curr_static_id->n_operands + nop]
4693 : .constraint);
4694 0 : char c;
4695 0 : for (;
4696 0 : (c = *p) && c != ',' && c != '#';
4697 0 : p += CONSTRAINT_LEN (c, p))
4698 : {
4699 0 : enum constraint_num cn = lookup_constraint (p);
4700 0 : if (constraint_satisfied_p (new_op, cn))
4701 : return true;
4702 : }
4703 :
4704 : return false;
4705 : }
4706 :
4707 : /* Main entry point of the constraint code: search the body of the
4708 : current insn to choose the best alternative. It is mimicking insn
4709 : alternative cost calculation model of former reload pass. That is
4710 : because machine descriptions were written to use this model. This
4711 : model can be changed in future. Make commutative operand exchange
4712 : if it is chosen.
4713 :
4714 : if CHECK_ONLY_P is false, do RTL changes to satisfy the
4715 : constraints. Return true if any change happened during function
4716 : call.
4717 :
4718 : If CHECK_ONLY_P is true then don't do any transformation. Just
4719 : check that the insn satisfies all constraints. If the insn does
4720 : not satisfy any constraint, return true. */
4721 : static bool
4722 108836770 : curr_insn_transform (bool check_only_p)
4723 : {
4724 108836770 : int i, j, k;
4725 108836770 : int n_operands;
4726 108836770 : int n_alternatives;
4727 108836770 : int n_outputs;
4728 108836770 : int commutative;
4729 108836770 : signed char goal_alt_matched[MAX_RECOG_OPERANDS][MAX_RECOG_OPERANDS];
4730 108836770 : signed char match_inputs[MAX_RECOG_OPERANDS + 1];
4731 108836770 : signed char outputs[MAX_RECOG_OPERANDS + 1];
4732 108836770 : rtx_insn *before, *after;
4733 108836770 : bool alt_p = false;
4734 : /* Flag that the insn has been changed through a transformation. */
4735 108836770 : bool change_p;
4736 108836770 : bool sec_mem_p;
4737 108836770 : bool use_sec_mem_p;
4738 108836770 : int max_regno_before;
4739 108836770 : int reused_alternative_num;
4740 :
4741 108836770 : curr_insn_set = single_set (curr_insn);
4742 108836770 : if (curr_insn_set != NULL_RTX && simple_move_p ())
4743 : {
4744 : /* We assume that the corresponding insn alternative has no
4745 : earlier clobbers. If it is not the case, don't define move
4746 : cost equal to 2 for the corresponding register classes. */
4747 16776372 : lra_set_used_insn_alternative (curr_insn, LRA_NON_CLOBBERED_ALT);
4748 16776372 : return false;
4749 : }
4750 :
4751 92060398 : no_input_reloads_p = no_output_reloads_p = false;
4752 92060398 : goal_alt_number = -1;
4753 92060398 : change_p = sec_mem_p = false;
4754 :
4755 : /* CALL_INSNs are not allowed to have any output reloads. */
4756 92060398 : if (CALL_P (curr_insn))
4757 6178527 : no_output_reloads_p = true;
4758 :
4759 92060398 : n_operands = curr_static_id->n_operands;
4760 92060398 : n_alternatives = curr_static_id->n_alternatives;
4761 :
4762 : /* Just return "no reloads" if insn has no operands with
4763 : constraints. */
4764 92060398 : if (n_operands == 0 || n_alternatives == 0)
4765 : return false;
4766 :
4767 81442586 : max_regno_before = max_reg_num ();
4768 :
4769 344587556 : for (i = 0; i < n_operands; i++)
4770 : {
4771 181702384 : goal_alt_matched[i][0] = -1;
4772 181702384 : goal_alt_matches[i] = -1;
4773 : }
4774 :
4775 81442586 : commutative = curr_static_id->commutative;
4776 :
4777 : /* Now see what we need for pseudos that didn't get hard regs or got
4778 : the wrong kind of hard reg. For this, we must consider all the
4779 : operands together against the register constraints. */
4780 :
4781 81442586 : best_losers = best_overall = INT_MAX;
4782 81442586 : best_reload_sum = 0;
4783 :
4784 81442586 : curr_swapped = false;
4785 81442586 : goal_alt_swapped = false;
4786 :
4787 81442586 : if (! check_only_p)
4788 : /* Make equivalence substitution and memory subreg elimination
4789 : before address processing because an address legitimacy can
4790 : depend on memory mode. */
4791 263078322 : for (i = 0; i < n_operands; i++)
4792 : {
4793 181654331 : rtx op, subst, old;
4794 181654331 : bool op_change_p = false;
4795 :
4796 181654331 : if (curr_static_id->operand[i].is_operator)
4797 1480125 : continue;
4798 :
4799 180174206 : old = op = *curr_id->operand_loc[i];
4800 180174206 : if (GET_CODE (old) == SUBREG)
4801 3845030 : old = SUBREG_REG (old);
4802 180174206 : subst = get_equiv_with_elimination (old, curr_insn);
4803 180174206 : original_subreg_reg_mode[i] = VOIDmode;
4804 180174206 : equiv_substition_p[i] = false;
4805 :
4806 180174206 : if (subst != old
4807 : /* We don't want to change an out operand by constant or invariant
4808 : which will require additional reloads, e.g. by putting a constant
4809 : into memory. */
4810 1614036 : && (curr_static_id->operand[i].type == OP_IN || MEM_P (subst)
4811 0 : || (GET_CODE (subst) == SUBREG && MEM_P (SUBREG_REG (subst)))))
4812 : {
4813 1614036 : equiv_substition_p[i] = true;
4814 1614036 : rtx new_subst = copy_rtx (subst);
4815 1614036 : if (lra_pointer_equiv_set_in (subst))
4816 829517 : lra_pointer_equiv_set_add (new_subst);
4817 1614036 : subst = new_subst;
4818 1614036 : lra_assert (REG_P (old));
4819 1614036 : if (GET_CODE (op) != SUBREG)
4820 1559108 : *curr_id->operand_loc[i] = subst;
4821 : else
4822 : {
4823 54928 : SUBREG_REG (op) = subst;
4824 54928 : if (GET_MODE (subst) == VOIDmode)
4825 61 : original_subreg_reg_mode[i] = GET_MODE (old);
4826 : }
4827 1614036 : if (lra_dump_file != NULL)
4828 : {
4829 3 : fprintf (lra_dump_file,
4830 : "Changing pseudo %d in operand %i of insn %u on equiv ",
4831 3 : REGNO (old), i, INSN_UID (curr_insn));
4832 3 : dump_value_slim (lra_dump_file, subst, 1);
4833 3 : fprintf (lra_dump_file, "\n");
4834 : }
4835 1614036 : op_change_p = change_p = true;
4836 : }
4837 180174206 : if (simplify_operand_subreg (i, GET_MODE (old)) || op_change_p)
4838 : {
4839 1614561 : change_p = true;
4840 1614561 : lra_update_dup (curr_id, i);
4841 : }
4842 : }
4843 :
4844 : /* We process equivalences before ignoring postponed insns on the current
4845 : constraint sub-pass but before any reload insn generation for the
4846 : postponed insn. */
4847 81423991 : if (! check_only_p
4848 81423991 : && bitmap_bit_p (&lra_postponed_insns, INSN_UID (curr_insn)))
4849 : return true;
4850 :
4851 : /* Reload address registers and displacements. We do it before
4852 : finding an alternative because of memory constraints. */
4853 81436305 : before = after = NULL;
4854 263126127 : for (i = 0; i < n_operands; i++)
4855 181689822 : if (! curr_static_id->operand[i].is_operator
4856 181689822 : && process_address (i, check_only_p, &before, &after))
4857 : {
4858 495801 : if (check_only_p)
4859 : return true;
4860 495801 : change_p = true;
4861 495801 : lra_update_dup (curr_id, i);
4862 : }
4863 :
4864 81436305 : if (change_p)
4865 : /* If we've changed the instruction then any alternative that
4866 : we chose previously may no longer be valid. */
4867 2063725 : lra_set_used_insn_alternative (curr_insn, LRA_UNKNOWN_ALT);
4868 :
4869 81417710 : if (! check_only_p && curr_insn_set != NULL_RTX
4870 158927756 : && check_and_process_move (&change_p, &sec_mem_p))
4871 0 : return change_p;
4872 :
4873 81436305 : try_swapped:
4874 :
4875 92043805 : reused_alternative_num = check_only_p ? LRA_UNKNOWN_ALT : curr_id->used_insn_alternative;
4876 92043805 : if (lra_dump_file != NULL && reused_alternative_num >= 0)
4877 0 : fprintf (lra_dump_file, "Reusing alternative %d for insn #%u\n",
4878 0 : reused_alternative_num, INSN_UID (curr_insn));
4879 :
4880 92043805 : if (process_alt_operands (reused_alternative_num))
4881 83331995 : alt_p = true;
4882 :
4883 92043805 : if (check_only_p)
4884 18595 : return ! alt_p || best_losers != 0;
4885 :
4886 : /* If insn is commutative (it's safe to exchange a certain pair of
4887 : operands) then we need to try each alternative twice, the second
4888 : time matching those two operands as if we had exchanged them. To
4889 : do this, really exchange them in operands.
4890 :
4891 : If we have just tried the alternatives the second time, return
4892 : operands to normal and drop through. */
4893 :
4894 92025210 : if (reused_alternative_num < 0 && commutative >= 0)
4895 : {
4896 21215000 : curr_swapped = !curr_swapped;
4897 21215000 : if (curr_swapped)
4898 : {
4899 10607500 : swap_operands (commutative);
4900 10607500 : goto try_swapped;
4901 : }
4902 : else
4903 10607500 : swap_operands (commutative);
4904 : }
4905 :
4906 81417710 : if (! alt_p && ! sec_mem_p)
4907 : {
4908 : /* No alternative works with reloads?? */
4909 6 : if (INSN_CODE (curr_insn) >= 0)
4910 0 : fatal_insn ("unable to generate reloads for:", curr_insn);
4911 6 : error_for_asm (curr_insn,
4912 : "inconsistent operand constraints in an %<asm%>");
4913 6 : lra_asm_error_p = true;
4914 6 : if (! JUMP_P (curr_insn))
4915 : {
4916 : /* Avoid further trouble with this insn. Don't generate use
4917 : pattern here as we could use the insn SP offset. */
4918 6 : lra_set_insn_deleted (curr_insn);
4919 : }
4920 : else
4921 : {
4922 0 : lra_invalidate_insn_data (curr_insn);
4923 0 : ira_nullify_asm_goto (curr_insn);
4924 0 : lra_update_insn_regno_info (curr_insn);
4925 : }
4926 : return true;
4927 : }
4928 :
4929 : /* If the best alternative is with operands 1 and 2 swapped, swap
4930 : them. Update the operand numbers of any reloads already
4931 : pushed. */
4932 :
4933 81417704 : if (goal_alt_swapped)
4934 : {
4935 653367 : if (lra_dump_file != NULL)
4936 21 : fprintf (lra_dump_file, " Commutative operand exchange in insn %u\n",
4937 21 : INSN_UID (curr_insn));
4938 :
4939 : /* Swap the duplicates too. */
4940 653367 : swap_operands (commutative);
4941 653367 : change_p = true;
4942 : }
4943 :
4944 : /* Some targets' TARGET_SECONDARY_MEMORY_NEEDED (e.g. x86) are defined
4945 : too conservatively. So we use the secondary memory only if there
4946 : is no any alternative without reloads. */
4947 81417704 : use_sec_mem_p = false;
4948 81417704 : if (! alt_p)
4949 : use_sec_mem_p = true;
4950 81417704 : else if (sec_mem_p)
4951 : {
4952 14506 : for (i = 0; i < n_operands; i++)
4953 14334 : if (! goal_alt_win[i] && ! goal_alt_match_win[i])
4954 : break;
4955 12950 : use_sec_mem_p = i < n_operands;
4956 : }
4957 :
4958 12950 : if (use_sec_mem_p)
4959 : {
4960 12778 : int in = -1, out = -1;
4961 12778 : rtx new_reg, src, dest, rld;
4962 12778 : machine_mode sec_mode, rld_mode;
4963 :
4964 12778 : lra_assert (curr_insn_set != NULL_RTX && sec_mem_p);
4965 12778 : dest = SET_DEST (curr_insn_set);
4966 12778 : src = SET_SRC (curr_insn_set);
4967 38334 : for (i = 0; i < n_operands; i++)
4968 25556 : if (*curr_id->operand_loc[i] == dest)
4969 : out = i;
4970 12778 : else if (*curr_id->operand_loc[i] == src)
4971 12778 : in = i;
4972 12778 : for (i = 0; i < curr_static_id->n_dups; i++)
4973 0 : if (out < 0 && *curr_id->dup_loc[i] == dest)
4974 0 : out = curr_static_id->dup_num[i];
4975 0 : else if (in < 0 && *curr_id->dup_loc[i] == src)
4976 0 : in = curr_static_id->dup_num[i];
4977 12778 : lra_assert (out >= 0 && in >= 0
4978 : && curr_static_id->operand[out].type == OP_OUT
4979 : && curr_static_id->operand[in].type == OP_IN);
4980 12778 : rld = partial_subreg_p (GET_MODE (src), GET_MODE (dest)) ? src : dest;
4981 12778 : rld_mode = GET_MODE (rld);
4982 12778 : sec_mode = targetm.secondary_memory_needed_mode (rld_mode);
4983 12778 : if (rld_mode != sec_mode
4984 12778 : && (invalid_mode_reg_p (sec_mode, dest)
4985 16 : || invalid_mode_reg_p (sec_mode, src)))
4986 : sec_mode = rld_mode;
4987 12778 : new_reg = lra_create_new_reg (sec_mode, NULL_RTX, NO_REGS, NULL,
4988 : "secondary");
4989 : /* If the mode is changed, it should be wider. */
4990 12778 : lra_assert (!partial_subreg_p (sec_mode, rld_mode));
4991 12778 : if (sec_mode != rld_mode)
4992 : {
4993 : /* If the target says specifically to use another mode for
4994 : secondary memory moves we cannot reuse the original
4995 : insn. */
4996 16 : after = emit_spill_move (false, new_reg, dest);
4997 16 : lra_process_new_insns (curr_insn, NULL, after,
4998 : "Inserting the sec. move");
4999 : /* We may have non null BEFORE here (e.g. after address
5000 : processing. */
5001 16 : push_to_sequence (before);
5002 16 : before = emit_spill_move (true, new_reg, src);
5003 16 : emit_insn (before);
5004 16 : before = end_sequence ();
5005 16 : lra_process_new_insns (curr_insn, before, NULL, "Changing on");
5006 16 : lra_set_insn_deleted (curr_insn);
5007 : }
5008 12762 : else if (dest == rld)
5009 : {
5010 12762 : *curr_id->operand_loc[out] = new_reg;
5011 12762 : lra_update_dup (curr_id, out);
5012 12762 : after = emit_spill_move (false, new_reg, dest);
5013 12762 : lra_process_new_insns (curr_insn, NULL, after,
5014 : "Inserting the sec. move");
5015 : }
5016 : else
5017 : {
5018 0 : *curr_id->operand_loc[in] = new_reg;
5019 0 : lra_update_dup (curr_id, in);
5020 : /* See comments above. */
5021 0 : push_to_sequence (before);
5022 0 : before = emit_spill_move (true, new_reg, src);
5023 0 : emit_insn (before);
5024 0 : before = end_sequence ();
5025 0 : lra_process_new_insns (curr_insn, before, NULL,
5026 : "Inserting the sec. move");
5027 : }
5028 12778 : lra_update_insn_regno_info (curr_insn);
5029 12778 : return true;
5030 : }
5031 :
5032 81404926 : lra_assert (goal_alt_number >= 0);
5033 162719532 : lra_set_used_insn_alternative (curr_insn, goal_reuse_alt_p
5034 : ? goal_alt_number : LRA_UNKNOWN_ALT);
5035 :
5036 81404926 : if (lra_dump_file != NULL)
5037 : {
5038 1185 : const char *p;
5039 :
5040 1185 : fprintf (lra_dump_file, " Choosing alt %d in insn %u:",
5041 1185 : goal_alt_number, INSN_UID (curr_insn));
5042 1185 : print_curr_insn_alt (goal_alt_number);
5043 1185 : if (INSN_CODE (curr_insn) >= 0
5044 1185 : && (p = get_insn_name (INSN_CODE (curr_insn))) != NULL)
5045 1178 : fprintf (lra_dump_file, " {%s}", p);
5046 1185 : if (maybe_ne (curr_id->sp_offset, 0))
5047 : {
5048 0 : fprintf (lra_dump_file, " (sp_off=");
5049 0 : print_dec (curr_id->sp_offset, lra_dump_file);
5050 0 : fprintf (lra_dump_file, ")");
5051 : }
5052 1185 : fprintf (lra_dump_file, "\n");
5053 : }
5054 :
5055 : /* Right now, for any pair of operands I and J that are required to
5056 : match, with J < I, goal_alt_matches[I] is J. Add I to
5057 : goal_alt_matched[J]. */
5058 :
5059 263021125 : for (i = 0; i < n_operands; i++)
5060 181616199 : if ((j = goal_alt_matches[i]) >= 0)
5061 : {
5062 10755416 : for (k = 0; goal_alt_matched[j][k] >= 0; k++)
5063 : ;
5064 : /* We allow matching one output operand and several input
5065 : operands. */
5066 10755415 : lra_assert (k == 0
5067 : || (curr_static_id->operand[j].type == OP_OUT
5068 : && curr_static_id->operand[i].type == OP_IN
5069 : && (curr_static_id->operand
5070 : [goal_alt_matched[j][0]].type == OP_IN)));
5071 10755415 : goal_alt_matched[j][k] = i;
5072 10755415 : goal_alt_matched[j][k + 1] = -1;
5073 : }
5074 :
5075 263021125 : for (i = 0; i < n_operands; i++)
5076 181616199 : goal_alt_win[i] |= goal_alt_match_win[i];
5077 :
5078 : /* Any constants that aren't allowed and can't be reloaded into
5079 : registers are here changed into memory references. */
5080 263021125 : for (i = 0; i < n_operands; i++)
5081 181616199 : if (goal_alt_win[i])
5082 : {
5083 175281917 : int regno;
5084 175281917 : enum reg_class new_class;
5085 175281917 : rtx reg = *curr_id->operand_loc[i];
5086 :
5087 175281917 : if (GET_CODE (reg) == SUBREG)
5088 3511723 : reg = SUBREG_REG (reg);
5089 :
5090 175281917 : if (REG_P (reg) && (regno = REGNO (reg)) >= FIRST_PSEUDO_REGISTER)
5091 : {
5092 79401734 : bool ok_p = in_class_p (reg, goal_alt[i], &new_class, true);
5093 :
5094 79401734 : if (new_class != NO_REGS && get_reg_class (regno) != new_class)
5095 : {
5096 3603575 : lra_assert (ok_p);
5097 3603575 : lra_change_class (regno, new_class, " Change to", true);
5098 : }
5099 : }
5100 : }
5101 : else
5102 : {
5103 6334282 : const char *constraint;
5104 6334282 : char c;
5105 6334282 : rtx op = *curr_id->operand_loc[i];
5106 6334282 : rtx subreg = NULL_RTX;
5107 6334282 : machine_mode op_mode = curr_operand_mode[i], mode = op_mode;
5108 :
5109 6334282 : if (GET_CODE (op) == SUBREG)
5110 : {
5111 268546 : subreg = op;
5112 268546 : op = SUBREG_REG (op);
5113 268546 : mode = GET_MODE (op);
5114 : }
5115 :
5116 6554708 : if (CONST_POOL_OK_P (mode, op)
5117 6554708 : && ((targetm.preferred_reload_class
5118 220426 : (op, (enum reg_class) goal_alt[i]) == NO_REGS)
5119 77265 : || no_input_reloads_p))
5120 : {
5121 143161 : rtx tem = force_const_mem (mode, op);
5122 :
5123 143161 : change_p = true;
5124 143161 : if (subreg != NULL_RTX)
5125 0 : tem = gen_rtx_SUBREG (op_mode, tem, SUBREG_BYTE (subreg));
5126 :
5127 143161 : *curr_id->operand_loc[i] = tem;
5128 143161 : lra_update_dup (curr_id, i);
5129 143161 : process_address (i, false, &before, &after);
5130 :
5131 : /* If the alternative accepts constant pool refs directly
5132 : there will be no reload needed at all. */
5133 143161 : if (subreg != NULL_RTX)
5134 0 : continue;
5135 : /* Skip alternatives before the one requested. */
5136 143161 : constraint = (curr_static_id->operand_alternative
5137 143161 : [goal_alt_number * n_operands + i].constraint);
5138 143161 : for (;
5139 244822 : (c = *constraint) && c != ',' && c != '#';
5140 101661 : constraint += CONSTRAINT_LEN (c, constraint))
5141 : {
5142 198702 : enum constraint_num cn = lookup_constraint (constraint);
5143 198702 : if ((insn_extra_memory_constraint (cn)
5144 101915 : || insn_extra_special_memory_constraint (cn)
5145 : || insn_extra_relaxed_memory_constraint (cn))
5146 198956 : && satisfies_memory_constraint_p (tem, cn))
5147 : break;
5148 : }
5149 143161 : if (c == '\0' || c == ',' || c == '#')
5150 46120 : continue;
5151 :
5152 97041 : goal_alt_win[i] = true;
5153 : }
5154 : }
5155 :
5156 : n_outputs = 0;
5157 263021125 : for (i = 0; i < n_operands; i++)
5158 181616199 : if (curr_static_id->operand[i].type == OP_OUT)
5159 70706934 : outputs[n_outputs++] = i;
5160 81404926 : outputs[n_outputs] = -1;
5161 263021125 : for (i = 0; i < n_operands; i++)
5162 : {
5163 181616199 : int regno;
5164 181616199 : bool optional_p = false;
5165 181616199 : rtx old, new_reg;
5166 181616199 : rtx op = *curr_id->operand_loc[i];
5167 :
5168 181616199 : if (goal_alt_win[i])
5169 : {
5170 175378958 : if (goal_alt[i] == NO_REGS
5171 47920410 : && REG_P (op)
5172 5517846 : && (regno = REGNO (op)) >= FIRST_PSEUDO_REGISTER
5173 : /* We assigned a hard register to the pseudo in the past but now
5174 : decided to spill it for the insn. If the pseudo is used only
5175 : in this insn, it is better to spill it here as we free hard
5176 : registers for other pseudos referenced in the insn. The most
5177 : common case of this is a scratch register which will be
5178 : transformed to scratch back at the end of LRA. */
5179 178337626 : && !multiple_insn_refs_p (regno))
5180 : {
5181 9810 : if (lra_get_allocno_class (regno) != NO_REGS)
5182 4634 : lra_change_class (regno, NO_REGS, " Change to", true);
5183 4905 : reg_renumber[regno] = -1;
5184 : }
5185 : /* We can do an optional reload. If the pseudo got a hard
5186 : reg, we might improve the code through inheritance. If
5187 : it does not get a hard register we coalesce memory/memory
5188 : moves later. Ignore move insns to avoid cycling. */
5189 175378958 : if (! lra_simple_p
5190 174833340 : && lra_undo_inheritance_iter < LRA_MAX_INHERITANCE_PASSES
5191 162173205 : && goal_alt[i] != NO_REGS && REG_P (op)
5192 80352271 : && (regno = REGNO (op)) >= FIRST_PSEUDO_REGISTER
5193 67187197 : && regno < new_regno_start
5194 62335010 : && ! ira_former_scratch_p (regno)
5195 62276693 : && reg_renumber[regno] < 0
5196 : /* Check that the optional reload pseudo will be able to
5197 : hold given mode value. */
5198 3870795 : && ! (prohibited_class_reg_set_mode_p
5199 3870795 : (goal_alt[i], reg_class_contents[goal_alt[i]],
5200 3870795 : PSEUDO_REGNO_MODE (regno)))
5201 179249743 : && (curr_insn_set == NULL_RTX
5202 3864289 : || !((REG_P (SET_SRC (curr_insn_set))
5203 : || MEM_P (SET_SRC (curr_insn_set))
5204 : || GET_CODE (SET_SRC (curr_insn_set)) == SUBREG)
5205 3236743 : && (REG_P (SET_DEST (curr_insn_set))
5206 : || MEM_P (SET_DEST (curr_insn_set))
5207 : || GET_CODE (SET_DEST (curr_insn_set)) == SUBREG))))
5208 : optional_p = true;
5209 174744878 : else if (goal_alt_matched[i][0] != -1
5210 8989713 : && curr_static_id->operand[i].type == OP_OUT
5211 8988744 : && (curr_static_id->operand_alternative
5212 8988744 : [goal_alt_number * n_operands + i].earlyclobber)
5213 20850 : && REG_P (op))
5214 : {
5215 25597 : for (j = 0; goal_alt_matched[i][j] != -1; j++)
5216 : {
5217 20797 : rtx op2 = *curr_id->operand_loc[goal_alt_matched[i][j]];
5218 :
5219 20797 : if (REG_P (op2) && REGNO (op) != REGNO (op2))
5220 : break;
5221 : }
5222 20797 : if (goal_alt_matched[i][j] != -1)
5223 : {
5224 : /* Generate reloads for different output and matched
5225 : input registers. This is the easiest way to avoid
5226 : creation of non-existing register conflicts in
5227 : lra-lives.cc. */
5228 15997 : match_reload (i, goal_alt_matched[i], outputs, goal_alt[i],
5229 : &goal_alt_exclude_start_hard_regs[i], &before,
5230 : &after, true);
5231 : }
5232 176421555 : continue;
5233 20797 : }
5234 : else
5235 : {
5236 174724081 : enum reg_class rclass, common_class;
5237 :
5238 91388570 : if (REG_P (op) && goal_alt[i] != NO_REGS
5239 85870724 : && (regno = REGNO (op)) >= new_regno_start
5240 4861708 : && (rclass = get_reg_class (regno)) == ALL_REGS
5241 0 : && ((common_class = ira_reg_class_subset[rclass][goal_alt[i]])
5242 : != NO_REGS)
5243 0 : && common_class != ALL_REGS
5244 174724081 : && enough_allocatable_hard_regs_p (common_class,
5245 0 : GET_MODE (op)))
5246 : /* Refine reload pseudo class from chosen alternative
5247 : constraint. */
5248 0 : lra_change_class (regno, common_class, " Change to", true);
5249 174724081 : continue;
5250 174724081 : }
5251 : }
5252 :
5253 : /* Operands that match previous ones have already been handled. */
5254 6871321 : if (goal_alt_matches[i] >= 0)
5255 1676677 : continue;
5256 :
5257 : /* We should not have an operand with a non-offsettable address
5258 : appearing where an offsettable address will do. It also may
5259 : be a case when the address should be special in other words
5260 : not a general one (e.g. it needs no index reg). */
5261 5194644 : if (goal_alt_matched[i][0] == -1 && goal_alt_offmemok[i] && MEM_P (op))
5262 : {
5263 72 : enum reg_class rclass;
5264 72 : rtx *loc = &XEXP (op, 0);
5265 72 : enum rtx_code code = GET_CODE (*loc);
5266 :
5267 72 : push_to_sequence (before);
5268 72 : rclass = base_reg_class (GET_MODE (op), MEM_ADDR_SPACE (op),
5269 : MEM, SCRATCH, op, curr_insn);
5270 72 : if (GET_RTX_CLASS (code) == RTX_AUTOINC)
5271 0 : new_reg = emit_inc (rclass, *loc,
5272 : /* This value does not matter for MODIFY. */
5273 0 : GET_MODE_SIZE (GET_MODE (op)));
5274 : /* Try to pull out section anchors. For example, instead of
5275 : reloading an "entire" address like .LANCHOR42+offset only reload
5276 : .LANCHOR42 and use the new reload register as the base register.
5277 : This allows following optimizations to share section anchors and
5278 : remove redundant loads. */
5279 72 : else if (reload_section_anchor_p (i))
5280 : {
5281 0 : rtx anchor = XEXP (XEXP (*loc, 0), 0);
5282 0 : rtx offset = XEXP (XEXP (*loc, 0), 1);
5283 :
5284 0 : if (get_reload_reg (OP_IN, Pmode, anchor, rclass, NULL, false,
5285 : false, "offsetable address", &new_reg))
5286 0 : lra_emit_move (new_reg, anchor);
5287 :
5288 0 : rtx new_addr = gen_rtx_PLUS (Pmode, new_reg, offset);
5289 0 : rtx new_op = shallow_copy_rtx (op);
5290 0 : XEXP (new_op, 0) = new_addr;
5291 :
5292 0 : new_reg = new_op;
5293 0 : loc = curr_id->operand_loc[i];
5294 : }
5295 86 : else if (get_reload_reg (OP_IN, Pmode, *loc, rclass,
5296 : NULL, false, false,
5297 : "offsetable address", &new_reg))
5298 : {
5299 72 : rtx addr = *loc;
5300 72 : enum rtx_code code = GET_CODE (addr);
5301 72 : bool align_p = false;
5302 :
5303 72 : if (code == AND && CONST_INT_P (XEXP (addr, 1)))
5304 : {
5305 : /* (and ... (const_int -X)) is used to align to X bytes. */
5306 0 : align_p = true;
5307 0 : addr = XEXP (*loc, 0);
5308 : }
5309 : else
5310 72 : addr = canonicalize_reload_addr (addr);
5311 :
5312 72 : lra_emit_move (new_reg, addr);
5313 72 : if (align_p)
5314 0 : emit_move_insn (new_reg, gen_rtx_AND (GET_MODE (new_reg),
5315 : new_reg, XEXP (*loc, 1)));
5316 : }
5317 72 : before = end_sequence ();
5318 72 : *loc = new_reg;
5319 72 : lra_update_dup (curr_id, i);
5320 72 : }
5321 3428871 : else if (goal_alt_matched[i][0] == -1 && goal_alt[i] == NO_REGS
5322 5216859 : && goal_alt_offmemok[i] && reg_plus_const_p (op))
5323 : {
5324 : /* Materialize the sum probably produced by elimination of a register
5325 : and then spill it into memory to satisfy the memory constraint. */
5326 4418 : machine_mode mode = curr_operand_mode[i];
5327 4418 : enum reg_class rclass
5328 4418 : = base_reg_class (mode, ADDR_SPACE_GENERIC, PLUS, SCRATCH, op,
5329 : curr_insn);
5330 4418 : rtx mem_pseudo;
5331 :
5332 4418 : push_to_sequence (before);
5333 4418 : if (get_reload_reg (curr_static_id->operand[i].type, mode, op,
5334 : rclass, NULL, false, false, "elimination sum",
5335 : &new_reg))
5336 4418 : lra_emit_move (new_reg, op);
5337 4418 : mem_pseudo = lra_create_new_reg (mode, NULL_RTX, NO_REGS,
5338 : NULL, "elimination sum mem");
5339 4418 : emit_insn (emit_spill_move (true, mem_pseudo, new_reg));
5340 4418 : before = end_sequence ();
5341 4418 : *curr_id->operand_loc[i] = mem_pseudo;
5342 4418 : lra_update_dup (curr_id, i);
5343 : }
5344 5190154 : else if (goal_alt_matched[i][0] == -1)
5345 : {
5346 3424453 : machine_mode mode;
5347 3424453 : rtx reg, *loc;
5348 3424453 : int hard_regno;
5349 3424453 : enum op_type type = curr_static_id->operand[i].type;
5350 :
5351 3424453 : loc = curr_id->operand_loc[i];
5352 3424453 : mode = curr_operand_mode[i];
5353 3424453 : if (GET_CODE (*loc) == SUBREG)
5354 : {
5355 96123 : reg = SUBREG_REG (*loc);
5356 96123 : poly_int64 byte = SUBREG_BYTE (*loc);
5357 96123 : if (REG_P (reg)
5358 : /* Strict_low_part requires reloading the register and not
5359 : just the subreg. Likewise for a strict subreg no wider
5360 : than a word for WORD_REGISTER_OPERATIONS targets. */
5361 96123 : && (curr_static_id->operand[i].strict_low
5362 96053 : || (!paradoxical_subreg_p (mode, GET_MODE (reg))
5363 93720 : && (hard_regno
5364 93720 : = get_try_hard_regno (REGNO (reg))) >= 0
5365 91367 : && (simplify_subreg_regno
5366 187490 : (hard_regno,
5367 91367 : GET_MODE (reg), byte, mode) < 0)
5368 0 : && (goal_alt[i] == NO_REGS
5369 0 : || (simplify_subreg_regno
5370 70 : (ira_class_hard_regs[goal_alt[i]][0],
5371 0 : GET_MODE (reg), byte, mode) >= 0)))
5372 96053 : || (partial_subreg_p (mode, GET_MODE (reg))
5373 96053 : && known_le (GET_MODE_SIZE (GET_MODE (reg)),
5374 : UNITS_PER_WORD)
5375 : && WORD_REGISTER_OPERATIONS))
5376 : /* Avoid the situation when there are no available hard regs
5377 : for the pseudo mode but there are ones for the subreg
5378 : mode: */
5379 96193 : && !(goal_alt[i] != NO_REGS
5380 70 : && REGNO (reg) >= FIRST_PSEUDO_REGISTER
5381 70 : && (prohibited_class_reg_set_mode_p
5382 70 : (goal_alt[i], reg_class_contents[goal_alt[i]],
5383 70 : GET_MODE (reg)))
5384 : && !(prohibited_class_reg_set_mode_p
5385 0 : (goal_alt[i], reg_class_contents[goal_alt[i]],
5386 : mode))))
5387 : {
5388 : /* An OP_INOUT is required when reloading a subreg of a
5389 : mode wider than a word to ensure that data beyond the
5390 : word being reloaded is preserved. Also automatically
5391 : ensure that strict_low_part reloads are made into
5392 : OP_INOUT which should already be true from the backend
5393 : constraints. */
5394 70 : if (type == OP_OUT
5395 70 : && (curr_static_id->operand[i].strict_low
5396 0 : || read_modify_subreg_p (*loc)))
5397 : type = OP_INOUT;
5398 70 : loc = &SUBREG_REG (*loc);
5399 70 : mode = GET_MODE (*loc);
5400 : }
5401 : }
5402 3424453 : old = *loc;
5403 3424453 : if (get_reload_reg (type, mode, old, goal_alt[i],
5404 : &goal_alt_exclude_start_hard_regs[i],
5405 3424453 : loc != curr_id->operand_loc[i],
5406 3424453 : curr_static_id->operand_alternative
5407 3424453 : [goal_alt_number * n_operands + i].earlyclobber,
5408 : "", &new_reg)
5409 3424453 : && type != OP_OUT)
5410 : {
5411 2442092 : push_to_sequence (before);
5412 2442092 : lra_emit_move (new_reg, old);
5413 2442092 : before = end_sequence ();
5414 : }
5415 3424453 : *loc = new_reg;
5416 3424453 : if (type != OP_IN
5417 981063 : && find_reg_note (curr_insn, REG_UNUSED, old) == NULL_RTX
5418 : /* OLD can be an equivalent constant here. */
5419 955566 : && !CONSTANT_P (old)
5420 : /* No need to write back anything for a scratch. */
5421 955566 : && GET_CODE (old) != SCRATCH
5422 4380019 : && (!REG_P(old) || !ira_former_scratch_p (REGNO (old))))
5423 : {
5424 955566 : start_sequence ();
5425 955566 : lra_emit_move (type == OP_INOUT ? copy_rtx (old) : old, new_reg);
5426 955566 : emit_insn (after);
5427 955566 : after = end_sequence ();
5428 955566 : *loc = new_reg;
5429 : }
5430 3424453 : for (j = 0; j < goal_alt_dont_inherit_ops_num; j++)
5431 656 : if (goal_alt_dont_inherit_ops[j] == i)
5432 : {
5433 656 : lra_set_regno_unique_value (REGNO (new_reg));
5434 656 : break;
5435 : }
5436 3424453 : lra_update_dup (curr_id, i);
5437 : }
5438 1765701 : else if (curr_static_id->operand[i].type == OP_IN
5439 1765701 : && (curr_static_id->operand[goal_alt_matched[i][0]].type
5440 : == OP_OUT
5441 0 : || (curr_static_id->operand[goal_alt_matched[i][0]].type
5442 : == OP_INOUT
5443 0 : && (operands_match_p
5444 0 : (*curr_id->operand_loc[i],
5445 0 : *curr_id->operand_loc[goal_alt_matched[i][0]],
5446 : -1)))))
5447 : {
5448 : /* generate reloads for input and matched outputs. */
5449 17758 : match_inputs[0] = i;
5450 17758 : match_inputs[1] = -1;
5451 17758 : match_reload (goal_alt_matched[i][0], match_inputs, outputs,
5452 : goal_alt[i], &goal_alt_exclude_start_hard_regs[i],
5453 : &before, &after,
5454 17758 : curr_static_id->operand_alternative
5455 17758 : [goal_alt_number * n_operands + goal_alt_matched[i][0]]
5456 17758 : .earlyclobber);
5457 : }
5458 1747943 : else if ((curr_static_id->operand[i].type == OP_OUT
5459 1 : || (curr_static_id->operand[i].type == OP_INOUT
5460 1 : && (operands_match_p
5461 1 : (*curr_id->operand_loc[i],
5462 1 : *curr_id->operand_loc[goal_alt_matched[i][0]],
5463 : -1))))
5464 1747944 : && (curr_static_id->operand[goal_alt_matched[i][0]].type
5465 : == OP_IN))
5466 : /* Generate reloads for output and matched inputs. */
5467 1747943 : match_reload (i, goal_alt_matched[i], outputs, goal_alt[i],
5468 : &goal_alt_exclude_start_hard_regs[i], &before, &after,
5469 1747943 : curr_static_id->operand_alternative
5470 1747943 : [goal_alt_number * n_operands + i].earlyclobber);
5471 0 : else if (curr_static_id->operand[i].type == OP_IN
5472 0 : && (curr_static_id->operand[goal_alt_matched[i][0]].type
5473 : == OP_IN))
5474 : {
5475 : /* Generate reloads for matched inputs. */
5476 0 : match_inputs[0] = i;
5477 0 : for (j = 0; (k = goal_alt_matched[i][j]) >= 0; j++)
5478 0 : match_inputs[j + 1] = k;
5479 0 : match_inputs[j + 1] = -1;
5480 0 : match_reload (-1, match_inputs, outputs, goal_alt[i],
5481 : &goal_alt_exclude_start_hard_regs[i],
5482 : &before, &after, false);
5483 : }
5484 : else
5485 : /* We must generate code in any case when function
5486 : process_alt_operands decides that it is possible. */
5487 0 : gcc_unreachable ();
5488 :
5489 5194644 : if (optional_p)
5490 : {
5491 634080 : rtx reg = op;
5492 :
5493 634080 : lra_assert (REG_P (reg));
5494 634080 : regno = REGNO (reg);
5495 634080 : op = *curr_id->operand_loc[i]; /* Substitution. */
5496 634080 : if (GET_CODE (op) == SUBREG)
5497 0 : op = SUBREG_REG (op);
5498 634080 : gcc_assert (REG_P (op) && (int) REGNO (op) >= new_regno_start);
5499 634080 : bitmap_set_bit (&lra_optional_reload_pseudos, REGNO (op));
5500 634080 : lra_reg_info[REGNO (op)].restore_rtx = reg;
5501 634080 : if (lra_dump_file != NULL)
5502 2 : fprintf (lra_dump_file,
5503 : " Making reload reg %d for reg %d optional\n",
5504 : REGNO (op), regno);
5505 : }
5506 : }
5507 :
5508 : /* After the dust has settled, collect and attach dependent filters. */
5509 81404926 : process_dependent_filters ();
5510 :
5511 76947375 : if (before != NULL_RTX || after != NULL_RTX
5512 157539058 : || max_regno_before != max_reg_num ())
5513 5296623 : change_p = true;
5514 81404926 : if (change_p)
5515 : {
5516 6319466 : lra_update_operator_dups (curr_id);
5517 : /* Something changes -- process the insn. */
5518 6319466 : lra_update_insn_regno_info (curr_insn);
5519 6319466 : if (asm_noperands (PATTERN (curr_insn)) >= 0
5520 6319466 : && ++curr_id->asm_reloads_num >= FIRST_PSEUDO_REGISTER)
5521 : /* Most probably there are no enough registers to satisfy asm insn: */
5522 : {
5523 5 : lra_asm_insn_error (curr_insn);
5524 5 : return change_p;
5525 : }
5526 : }
5527 81404921 : if (goal_alt_out_sp_reload_p)
5528 : {
5529 : /* We have an output stack pointer reload -- update sp offset: */
5530 0 : rtx set;
5531 0 : bool done_p = false;
5532 0 : poly_int64 sp_offset = curr_id->sp_offset;
5533 0 : for (rtx_insn *insn = after; insn != NULL_RTX; insn = NEXT_INSN (insn))
5534 0 : if ((set = single_set (insn)) != NULL_RTX
5535 0 : && SET_DEST (set) == stack_pointer_rtx)
5536 : {
5537 0 : lra_assert (!done_p);
5538 0 : done_p = true;
5539 0 : curr_id->sp_offset = 0;
5540 0 : lra_insn_recog_data_t id = lra_get_insn_recog_data (insn);
5541 0 : id->sp_offset = sp_offset;
5542 0 : if (lra_dump_file != NULL)
5543 0 : fprintf (lra_dump_file,
5544 : " Moving sp offset from insn %u to %u\n",
5545 0 : INSN_UID (curr_insn), INSN_UID (insn));
5546 : }
5547 0 : lra_assert (done_p);
5548 : }
5549 81404921 : int const_regno = -1;
5550 81404921 : rtx set;
5551 81404921 : rtx_insn *prev, *const_insn = NULL;
5552 4457550 : if (before != NULL_RTX && (prev = PREV_INSN (curr_insn)) != NULL_RTX
5553 85862471 : && (set = single_set (prev)) != NULL_RTX && CONSTANT_P (SET_SRC (set)))
5554 : {
5555 340137 : rtx reg = SET_DEST (set);
5556 340137 : if (GET_CODE (reg) == SUBREG)
5557 10541 : reg = SUBREG_REG (reg);
5558 : /* Consider only reload insns as we don't want to change the order
5559 : created by previous optimizations. */
5560 247234 : if (REG_P (reg) && (int) REGNO (reg) >= lra_new_regno_start
5561 341758 : && bitmap_bit_p (&lra_reg_info[REGNO (reg)].insn_bitmap,
5562 1621 : INSN_UID (curr_insn)))
5563 : {
5564 1102 : const_regno = REGNO (reg);
5565 1102 : const_insn = prev;
5566 : }
5567 : }
5568 81404921 : if (asm_noperands (PATTERN (curr_insn)) >= 0)
5569 : {
5570 : /* Asm can have a lot of operands. To guarantee their assignment,
5571 : postpone processing the reload insns until the reload pseudos are
5572 : assigned. */
5573 46993 : postpone_insns (before);
5574 46993 : postpone_insns (after);
5575 : }
5576 81404921 : lra_process_new_insns (curr_insn, before, after,
5577 : "Inserting insn reload", true);
5578 81404921 : if (const_regno >= 0) {
5579 2204 : bool move_p = true;
5580 2204 : for (rtx_insn *insn = before; insn != curr_insn; insn = NEXT_INSN (insn))
5581 1102 : if (bitmap_bit_p (&lra_reg_info[const_regno].insn_bitmap,
5582 1102 : INSN_UID (insn)))
5583 : {
5584 : move_p = false;
5585 : break;
5586 : }
5587 1102 : if (move_p)
5588 : {
5589 1102 : reorder_insns_nobb (const_insn, const_insn, PREV_INSN (curr_insn));
5590 1102 : if (lra_dump_file != NULL)
5591 : {
5592 0 : dump_insn_slim (lra_dump_file, const_insn);
5593 0 : fprintf (lra_dump_file,
5594 : " to decrease reg pressure, it is moved before:\n");
5595 0 : dump_insn_slim (lra_dump_file, curr_insn);
5596 : }
5597 : }
5598 : }
5599 : return change_p;
5600 : }
5601 :
5602 : /* Return true if INSN satisfies all constraints. In other words, no
5603 : reload insns are needed. */
5604 : bool
5605 3434 : lra_constrain_insn (rtx_insn *insn)
5606 : {
5607 3434 : int saved_new_regno_start = new_regno_start;
5608 3434 : int saved_new_insn_uid_start = new_insn_uid_start;
5609 3434 : bool change_p;
5610 :
5611 3434 : curr_insn = insn;
5612 3434 : curr_id = lra_get_insn_recog_data (curr_insn);
5613 3434 : curr_static_id = curr_id->insn_static_data;
5614 3434 : new_insn_uid_start = get_max_uid ();
5615 3434 : new_regno_start = max_reg_num ();
5616 3434 : change_p = curr_insn_transform (true);
5617 3434 : new_regno_start = saved_new_regno_start;
5618 3434 : new_insn_uid_start = saved_new_insn_uid_start;
5619 3434 : return ! change_p;
5620 : }
5621 :
5622 : /* Return true if X is in LIST. */
5623 : static bool
5624 1412458 : in_list_p (rtx x, rtx list)
5625 : {
5626 2421624 : for (; list != NULL_RTX; list = XEXP (list, 1))
5627 1323332 : if (XEXP (list, 0) == x)
5628 : return true;
5629 : return false;
5630 : }
5631 :
5632 : /* Return true if X contains an allocatable hard register (if
5633 : HARD_REG_P) or a (spilled if SPILLED_P) pseudo. */
5634 : static bool
5635 7789455 : contains_reg_p (rtx x, bool hard_reg_p, bool spilled_p)
5636 : {
5637 7789455 : int i, j;
5638 7789455 : const char *fmt;
5639 7789455 : enum rtx_code code;
5640 :
5641 7789455 : code = GET_CODE (x);
5642 7789455 : if (REG_P (x))
5643 : {
5644 1550766 : int regno = REGNO (x);
5645 1550766 : HARD_REG_SET alloc_regs;
5646 :
5647 1550766 : if (hard_reg_p)
5648 : {
5649 488437 : if (regno >= FIRST_PSEUDO_REGISTER)
5650 143995 : regno = lra_get_regno_hard_regno (regno);
5651 488437 : if (regno < 0)
5652 : return false;
5653 488437 : alloc_regs = ~lra_no_alloc_regs;
5654 488437 : return overlaps_hard_reg_set_p (alloc_regs, GET_MODE (x), regno);
5655 : }
5656 : else
5657 : {
5658 1062329 : if (regno < FIRST_PSEUDO_REGISTER)
5659 : return false;
5660 338627 : if (! spilled_p)
5661 : return true;
5662 178860 : return lra_get_regno_hard_regno (regno) < 0;
5663 : }
5664 : }
5665 6238689 : fmt = GET_RTX_FORMAT (code);
5666 15369914 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
5667 : {
5668 9722146 : if (fmt[i] == 'e')
5669 : {
5670 4212390 : if (contains_reg_p (XEXP (x, i), hard_reg_p, spilled_p))
5671 : return true;
5672 : }
5673 5509756 : else if (fmt[i] == 'E')
5674 : {
5675 1462249 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
5676 1337223 : if (contains_reg_p (XVECEXP (x, i, j), hard_reg_p, spilled_p))
5677 : return true;
5678 : }
5679 : }
5680 : return false;
5681 : }
5682 :
5683 : /* Process all regs in location *LOC and change them on equivalent
5684 : substitution. Return true if any change was done. */
5685 : static bool
5686 3348 : loc_equivalence_change_p (rtx *loc)
5687 : {
5688 3348 : rtx subst, reg, x = *loc;
5689 3348 : bool result = false;
5690 3348 : enum rtx_code code = GET_CODE (x);
5691 3348 : const char *fmt;
5692 3348 : int i, j;
5693 :
5694 3348 : if (code == SUBREG)
5695 : {
5696 20 : reg = SUBREG_REG (x);
5697 20 : if ((subst = get_equiv_with_elimination (reg, curr_insn)) != reg
5698 20 : && GET_MODE (subst) == VOIDmode)
5699 : {
5700 : /* We cannot reload debug location. Simplify subreg here
5701 : while we know the inner mode. */
5702 0 : *loc = simplify_gen_subreg (GET_MODE (x), subst,
5703 0 : GET_MODE (reg), SUBREG_BYTE (x));
5704 0 : return true;
5705 : }
5706 : }
5707 3348 : if (code == REG && (subst = get_equiv_with_elimination (x, curr_insn)) != x)
5708 : {
5709 8 : *loc = subst;
5710 8 : return true;
5711 : }
5712 :
5713 : /* Scan all the operand sub-expressions. */
5714 3340 : fmt = GET_RTX_FORMAT (code);
5715 8168 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
5716 : {
5717 4828 : if (fmt[i] == 'e')
5718 2577 : result = loc_equivalence_change_p (&XEXP (x, i)) || result;
5719 2251 : else if (fmt[i] == 'E')
5720 270 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
5721 200 : result
5722 210 : = loc_equivalence_change_p (&XVECEXP (x, i, j)) || result;
5723 : }
5724 : return result;
5725 : }
5726 :
5727 : /* Similar to loc_equivalence_change_p, but for use as
5728 : simplify_replace_fn_rtx callback. DATA is insn for which the
5729 : elimination is done. If it null we don't do the elimination. */
5730 : static rtx
5731 41888073 : loc_equivalence_callback (rtx loc, const_rtx, void *data)
5732 : {
5733 41888073 : if (!REG_P (loc))
5734 : return NULL_RTX;
5735 :
5736 10764409 : rtx subst = (data == NULL
5737 10764409 : ? get_equiv (loc) : get_equiv_with_elimination (loc, (rtx_insn *) data));
5738 10764409 : if (subst != loc)
5739 38736 : return subst;
5740 :
5741 : return NULL_RTX;
5742 : }
5743 :
5744 : /* Maximum number of generated reload insns per an insn. It is for
5745 : preventing this pass cycling in a bug case. */
5746 : #define MAX_RELOAD_INSNS_NUMBER LRA_MAX_INSN_RELOADS
5747 :
5748 : /* The current iteration number of this LRA pass. */
5749 : int lra_constraint_iter;
5750 :
5751 : /* True if we should during assignment sub-pass check assignment
5752 : correctness for all pseudos and spill some of them to correct
5753 : conflicts. It can be necessary when we substitute equiv which
5754 : needs checking register allocation correctness because the
5755 : equivalent value contains allocatable hard registers, or when we
5756 : restore multi-register pseudo, or when we change the insn code and
5757 : its operand became INOUT operand when it was IN one before. */
5758 : bool check_and_force_assignment_correctness_p;
5759 :
5760 : /* Return true if REGNO is referenced in more than one block. */
5761 : static bool
5762 153649 : multi_block_pseudo_p (int regno)
5763 : {
5764 153649 : basic_block bb = NULL;
5765 153649 : unsigned int uid;
5766 153649 : bitmap_iterator bi;
5767 :
5768 153649 : if (regno < FIRST_PSEUDO_REGISTER)
5769 : return false;
5770 :
5771 471026 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi)
5772 322182 : if (bb == NULL)
5773 153649 : bb = BLOCK_FOR_INSN (lra_insn_recog_data[uid]->insn);
5774 168533 : else if (BLOCK_FOR_INSN (lra_insn_recog_data[uid]->insn) != bb)
5775 : return true;
5776 : return false;
5777 : }
5778 :
5779 : /* Return true if LIST contains a deleted insn. */
5780 : static bool
5781 749664 : contains_deleted_insn_p (rtx_insn_list *list)
5782 : {
5783 1427454 : for (; list != NULL_RTX; list = list->next ())
5784 677790 : if (NOTE_P (list->insn ())
5785 677790 : && NOTE_KIND (list->insn ()) == NOTE_INSN_DELETED)
5786 : return true;
5787 : return false;
5788 : }
5789 :
5790 : /* Return true if X contains a pseudo dying in INSN. */
5791 : static bool
5792 2311718 : dead_pseudo_p (rtx x, rtx_insn *insn)
5793 : {
5794 2311718 : int i, j;
5795 2311718 : const char *fmt;
5796 2311718 : enum rtx_code code;
5797 :
5798 2311718 : if (REG_P (x))
5799 518283 : return (insn != NULL_RTX
5800 518283 : && find_regno_note (insn, REG_DEAD, REGNO (x)) != NULL_RTX);
5801 1793435 : code = GET_CODE (x);
5802 1793435 : fmt = GET_RTX_FORMAT (code);
5803 4584388 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
5804 : {
5805 2796950 : if (fmt[i] == 'e')
5806 : {
5807 1366879 : if (dead_pseudo_p (XEXP (x, i), insn))
5808 : return true;
5809 : }
5810 1430071 : else if (fmt[i] == 'E')
5811 : {
5812 293352 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
5813 268435 : if (dead_pseudo_p (XVECEXP (x, i, j), insn))
5814 : return true;
5815 : }
5816 : }
5817 : return false;
5818 : }
5819 :
5820 : /* Return true if INSN contains a dying pseudo in INSN right hand
5821 : side. */
5822 : static bool
5823 676404 : insn_rhs_dead_pseudo_p (rtx_insn *insn)
5824 : {
5825 676404 : rtx set = single_set (insn);
5826 :
5827 676404 : gcc_assert (set != NULL);
5828 676404 : return dead_pseudo_p (SET_SRC (set), insn);
5829 : }
5830 :
5831 : /* Return true if any init insn of REGNO contains a dying pseudo in
5832 : insn right hand side. */
5833 : static bool
5834 748278 : init_insn_rhs_dead_pseudo_p (int regno)
5835 : {
5836 748278 : rtx_insn_list *insns = ira_reg_equiv[regno].init_insns;
5837 :
5838 748278 : if (insns == NULL)
5839 : return false;
5840 1345337 : for (; insns != NULL_RTX; insns = insns->next ())
5841 676404 : if (insn_rhs_dead_pseudo_p (insns->insn ()))
5842 : return true;
5843 : return false;
5844 : }
5845 :
5846 : /* Return TRUE if REGNO has a reverse equivalence. The equivalence is
5847 : reverse only if we have one init insn with given REGNO as a
5848 : source. */
5849 : static bool
5850 749664 : reverse_equiv_p (int regno)
5851 : {
5852 749664 : rtx_insn_list *insns = ira_reg_equiv[regno].init_insns;
5853 749664 : rtx set;
5854 :
5855 749664 : if (insns == NULL)
5856 : return false;
5857 677790 : if (! INSN_P (insns->insn ())
5858 1355580 : || insns->next () != NULL)
5859 : return false;
5860 677790 : if ((set = single_set (insns->insn ())) == NULL_RTX)
5861 : return false;
5862 677790 : return REG_P (SET_SRC (set)) && (int) REGNO (SET_SRC (set)) == regno;
5863 : }
5864 :
5865 : /* Return TRUE if REGNO was reloaded in an equivalence init insn. We
5866 : call this function only for non-reverse equivalence. */
5867 : static bool
5868 740807 : contains_reloaded_insn_p (int regno)
5869 : {
5870 740807 : rtx set;
5871 740807 : rtx_insn_list *list = ira_reg_equiv[regno].init_insns;
5872 :
5873 1409738 : for (; list != NULL; list = list->next ())
5874 668933 : if ((set = single_set (list->insn ())) == NULL_RTX
5875 668933 : || ! REG_P (SET_DEST (set))
5876 1337866 : || (int) REGNO (SET_DEST (set)) != regno)
5877 : return true;
5878 : return false;
5879 : }
5880 :
5881 : /* Try combine secondary memory reload insn FROM for insn TO into TO insn.
5882 : FROM should be a load insn (usually a secondary memory reload insn). Return
5883 : TRUE in case of success. */
5884 : static bool
5885 7478901 : combine_reload_insn (rtx_insn *from, rtx_insn *to)
5886 : {
5887 7478901 : bool ok_p;
5888 7478901 : rtx_insn *saved_insn;
5889 7478901 : rtx set, from_reg, to_reg, op;
5890 7478901 : enum reg_class to_class, from_class;
5891 7478901 : int n, nop;
5892 7478901 : signed char changed_nops[MAX_RECOG_OPERANDS + 1];
5893 :
5894 : /* Check conditions for second memory reload and original insn: */
5895 7478901 : if ((targetm.secondary_memory_needed
5896 : == hook_bool_mode_reg_class_t_reg_class_t_false)
5897 7478901 : || NEXT_INSN (from) != to
5898 4460722 : || !NONDEBUG_INSN_P (to)
5899 11939623 : || CALL_P (to))
5900 : return false;
5901 :
5902 4455357 : lra_insn_recog_data_t id = lra_get_insn_recog_data (to);
5903 4455357 : struct lra_static_insn_data *static_id = id->insn_static_data;
5904 :
5905 4455357 : if (id->used_insn_alternative == LRA_UNKNOWN_ALT
5906 4455357 : || (set = single_set (from)) == NULL_RTX)
5907 : return false;
5908 4423284 : from_reg = SET_DEST (set);
5909 4423284 : to_reg = SET_SRC (set);
5910 : /* Ignore optional reloads: */
5911 4346110 : if (! REG_P (from_reg) || ! REG_P (to_reg)
5912 7350208 : || bitmap_bit_p (&lra_optional_reload_pseudos, REGNO (from_reg)))
5913 : return false;
5914 2379620 : to_class = lra_get_allocno_class (REGNO (to_reg));
5915 2379620 : from_class = lra_get_allocno_class (REGNO (from_reg));
5916 : /* Check that reload insn is a load: */
5917 2379620 : if (to_class != NO_REGS || from_class == NO_REGS)
5918 : return false;
5919 47391 : for (n = nop = 0; nop < static_id->n_operands; nop++)
5920 : {
5921 34172 : if (static_id->operand[nop].type != OP_IN)
5922 12222 : continue;
5923 21950 : op = *id->operand_loc[nop];
5924 21950 : if (!REG_P (op) || REGNO (op) != REGNO (from_reg))
5925 8914 : continue;
5926 13036 : *id->operand_loc[nop] = to_reg;
5927 13036 : changed_nops[n++] = nop;
5928 : }
5929 13219 : changed_nops[n] = -1;
5930 13219 : lra_update_dups (id, changed_nops);
5931 13219 : lra_update_insn_regno_info (to);
5932 13219 : ok_p = recog_memoized (to) >= 0;
5933 13219 : if (ok_p)
5934 : {
5935 : /* Check that combined insn does not need any reloads: */
5936 13200 : saved_insn = curr_insn;
5937 13200 : curr_insn = to;
5938 13200 : curr_id = lra_get_insn_recog_data (curr_insn);
5939 13200 : curr_static_id = curr_id->insn_static_data;
5940 13200 : for (bool swapped_p = false;;)
5941 : {
5942 15395 : ok_p = !curr_insn_transform (true);
5943 15395 : if (ok_p || curr_static_id->commutative < 0)
5944 : break;
5945 4390 : swap_operands (curr_static_id->commutative);
5946 4390 : if (lra_dump_file != NULL)
5947 : {
5948 0 : fprintf (lra_dump_file,
5949 : " Swapping %scombined insn operands:\n",
5950 : swapped_p ? "back " : "");
5951 0 : dump_insn_slim (lra_dump_file, to);
5952 : }
5953 4390 : if (swapped_p)
5954 : break;
5955 : swapped_p = true;
5956 : }
5957 13200 : curr_insn = saved_insn;
5958 13200 : curr_id = lra_get_insn_recog_data (curr_insn);
5959 13200 : curr_static_id = curr_id->insn_static_data;
5960 : }
5961 13219 : if (ok_p)
5962 : {
5963 3622 : id->used_insn_alternative = -1;
5964 3622 : lra_push_insn_and_update_insn_regno_info (to);
5965 3622 : if (lra_dump_file != NULL)
5966 : {
5967 0 : fprintf (lra_dump_file, " Use combined insn:\n");
5968 0 : dump_insn_slim (lra_dump_file, to);
5969 : }
5970 : return true;
5971 : }
5972 9597 : if (lra_dump_file != NULL)
5973 : {
5974 0 : fprintf (lra_dump_file, " Failed combined insn:\n");
5975 0 : dump_insn_slim (lra_dump_file, to);
5976 : }
5977 19471 : for (int i = 0; i < n; i++)
5978 : {
5979 9874 : nop = changed_nops[i];
5980 9874 : *id->operand_loc[nop] = from_reg;
5981 : }
5982 9597 : lra_update_dups (id, changed_nops);
5983 9597 : lra_update_insn_regno_info (to);
5984 9597 : if (lra_dump_file != NULL)
5985 : {
5986 0 : fprintf (lra_dump_file, " Restoring insn after failed combining:\n");
5987 0 : dump_insn_slim (lra_dump_file, to);
5988 : }
5989 : return false;
5990 : }
5991 :
5992 : static bool
5993 527726 : virtual_reg_p (const_rtx reg)
5994 : {
5995 0 : return (VIRTUAL_REGISTER_P (reg)
5996 527726 : || (REGNO(reg) == ARG_POINTER_REGNUM
5997 : && !HARD_FRAME_POINTER_IS_ARG_POINTER)
5998 979900 : || (REGNO(reg) == FRAME_POINTER_REGNUM
5999 0 : && !HARD_FRAME_POINTER_IS_FRAME_POINTER));
6000 : }
6001 :
6002 : /* Return true if X contains a virtual register. */
6003 : static bool
6004 738844 : contains_virtual_reg_p (rtx x)
6005 : {
6006 738844 : subrtx_iterator::array_type array;
6007 2101682 : FOR_EACH_SUBRTX (iter, array, x, NONCONST)
6008 1890564 : if (REG_P (*iter) && virtual_reg_p (*iter))
6009 360280 : return true;
6010 378564 : return false;
6011 738844 : }
6012 :
6013 : /* Return true if pseudo REGNO is an operand of an incdec rtx in any of its
6014 : insns. */
6015 : static bool
6016 360280 : reg_has_incdec_p (int regno)
6017 : {
6018 360280 : unsigned int uid;
6019 360280 : bitmap_iterator bi;
6020 1830095 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi)
6021 : {
6022 1469815 : rtx_insn *insn = lra_insn_recog_data[uid]->insn;
6023 1469815 : if (!NONDEBUG_INSN_P (insn))
6024 26430 : continue;
6025 1443385 : subrtx_iterator::array_type array;
6026 8473406 : FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
6027 : {
6028 7030021 : const_rtx x = *iter;
6029 7030021 : if ((GET_CODE (x) == PRE_INC || GET_CODE (x) == PRE_DEC
6030 6925554 : || GET_CODE (x) == POST_INC || GET_CODE (x) == POST_DEC
6031 6925554 : || GET_CODE (x) == PRE_MODIFY || GET_CODE (x) == POST_MODIFY)
6032 105830 : && REG_P (XEXP (x, 0))
6033 7135851 : && REGNO (XEXP (x, 0)) == (unsigned int) regno)
6034 0 : return true;
6035 : }
6036 1443385 : }
6037 : return false;
6038 : }
6039 :
6040 : /* Entry function of LRA constraint pass. Return true if the
6041 : constraint pass did change the code. */
6042 : bool
6043 3300295 : lra_constraints (bool first_p)
6044 : {
6045 3300295 : bool changed_p;
6046 3300295 : int i, hard_regno, new_insns_num;
6047 3300295 : unsigned int min_len, new_min_len, uid;
6048 3300295 : rtx set, x, reg, nosubreg_dest;
6049 3300295 : rtx_insn *original_insn;
6050 3300295 : basic_block last_bb;
6051 3300295 : bitmap_iterator bi;
6052 :
6053 3300295 : lra_constraint_iter++;
6054 3300295 : if (lra_dump_file != NULL)
6055 193 : fprintf (lra_dump_file, "\n********** Local #%d: **********\n\n",
6056 : lra_constraint_iter);
6057 3300295 : changed_p = false;
6058 3300295 : if (pic_offset_table_rtx
6059 3300295 : && REGNO (pic_offset_table_rtx) >= FIRST_PSEUDO_REGISTER)
6060 105136 : check_and_force_assignment_correctness_p = true;
6061 3195159 : else if (first_p)
6062 : /* On the first iteration we should check IRA assignment
6063 : correctness. In rare cases, the assignments can be wrong as
6064 : early clobbers operands are ignored in IRA or usages of
6065 : paradoxical sub-registers are not taken into account by
6066 : IRA. */
6067 1476563 : check_and_force_assignment_correctness_p = true;
6068 3300295 : new_insn_uid_start = get_max_uid ();
6069 3300295 : new_regno_start = first_p ? lra_constraint_new_regno_start : max_reg_num ();
6070 : /* Mark used hard regs for target stack size calculations. */
6071 212719338 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
6072 209419043 : if (lra_reg_info[i].nrefs != 0
6073 308691620 : && (hard_regno = lra_get_regno_hard_regno (i)) >= 0)
6074 : {
6075 95270913 : int j, nregs;
6076 :
6077 95270913 : nregs = hard_regno_nregs (hard_regno, lra_reg_info[i].biggest_mode);
6078 193783419 : for (j = 0; j < nregs; j++)
6079 98512506 : df_set_regs_ever_live (hard_regno + j, true);
6080 : }
6081 : /* Do elimination before the equivalence processing as we can spill
6082 : some pseudos during elimination. */
6083 3300295 : lra_eliminate (false, first_p);
6084 3300295 : auto_bitmap equiv_insn_bitmap (®_obstack);
6085 :
6086 : /* Register elimination can create new pseudos via the addptr pattern,
6087 : so make sure the equivalency tables are resized appropriately. */
6088 3300295 : ira_expand_reg_equiv ();
6089 216019633 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
6090 209419043 : if (lra_reg_info[i].nrefs != 0)
6091 : {
6092 99272577 : ira_reg_equiv[i].profitable_p = true;
6093 99272577 : reg = regno_reg_rtx[i];
6094 99272577 : if (lra_get_regno_hard_regno (i) < 0 && (x = get_equiv (reg)) != reg)
6095 : {
6096 760587 : bool pseudo_p = contains_reg_p (x, false, false);
6097 :
6098 : /* After RTL transformation, we cannot guarantee that
6099 : pseudo in the substitution was not reloaded which might
6100 : make equivalence invalid. For example, in reverse
6101 : equiv of p0
6102 :
6103 : p0 <- ...
6104 : ...
6105 : equiv_mem <- p0
6106 :
6107 : the memory address register was reloaded before the 2nd
6108 : insn. */
6109 760587 : if ((! first_p && pseudo_p)
6110 : /* We don't use DF for compilation speed sake. So it
6111 : is problematic to update live info when we use an
6112 : equivalence containing pseudos in more than one
6113 : BB. */
6114 754469 : || (pseudo_p && multi_block_pseudo_p (i))
6115 : /* If an init insn was deleted for some reason, cancel
6116 : the equiv. We could update the equiv insns after
6117 : transformations including an equiv insn deletion
6118 : but it is not worthy as such cases are extremely
6119 : rare. */
6120 749664 : || contains_deleted_insn_p (ira_reg_equiv[i].init_insns)
6121 : /* If it is not a reverse equivalence, we check that a
6122 : pseudo in rhs of the init insn is not dying in the
6123 : insn. Otherwise, the live info at the beginning of
6124 : the corresponding BB might be wrong after we
6125 : removed the insn. When the equiv can be a
6126 : constant, the right hand side of the init insn can
6127 : be a pseudo. */
6128 749664 : || (! reverse_equiv_p (i)
6129 748278 : && (init_insn_rhs_dead_pseudo_p (i)
6130 : /* If we reloaded the pseudo in an equivalence
6131 : init insn, we cannot remove the equiv init
6132 : insns and the init insns might write into
6133 : const memory in this case. */
6134 740807 : || contains_reloaded_insn_p (i)))
6135 : /* Prevent access beyond equivalent memory for
6136 : paradoxical subregs. */
6137 742191 : || (MEM_P (x)
6138 1179960 : && maybe_gt (GET_MODE_SIZE (lra_reg_info[i].biggest_mode),
6139 : GET_MODE_SIZE (GET_MODE (x))))
6140 741457 : || (pic_offset_table_rtx
6141 54471 : && ((CONST_POOL_OK_P (PSEUDO_REGNO_MODE (i), x)
6142 9512 : && (targetm.preferred_reload_class
6143 4756 : (x, lra_get_allocno_class (i)) == NO_REGS))
6144 52670 : || contains_symbol_ref_p (x)))
6145 : /* PR120165: An inc/dec changes the reg value, so we cannot
6146 : substitute the equiv which might contain virtual regs
6147 : requiring elimination. An insn with inc/dec can also change
6148 : the value of another reg (e.g. sp) against which we
6149 : eliminate. As we update elimination once per insn and an
6150 : inc/dec operand might require input and output reloads, we
6151 : can generate a wrong offset for a reload insn. */
6152 1499431 : || (contains_virtual_reg_p (x) && reg_has_incdec_p (i)))
6153 21743 : ira_reg_equiv[i].defined_p
6154 21743 : = ira_reg_equiv[i].caller_save_p = false;
6155 760587 : if (contains_reg_p (x, false, true))
6156 9390 : ira_reg_equiv[i].profitable_p = false;
6157 760587 : if (get_equiv (reg) != reg)
6158 734196 : bitmap_ior_into (equiv_insn_bitmap, &lra_reg_info[i].insn_bitmap);
6159 : }
6160 : }
6161 212719338 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
6162 209419043 : update_equiv (i);
6163 : /* We should add all insns containing pseudos which should be
6164 : substituted by their equivalences. */
6165 5765358 : EXECUTE_IF_SET_IN_BITMAP (equiv_insn_bitmap, 0, uid, bi)
6166 2465063 : lra_push_insn_by_uid (uid);
6167 3300295 : min_len = lra_insn_stack_length ();
6168 3300295 : new_insns_num = 0;
6169 3300295 : last_bb = NULL;
6170 3300295 : changed_p = false;
6171 3300295 : original_insn = NULL;
6172 173207081 : while ((new_min_len = lra_insn_stack_length ()) != 0)
6173 : {
6174 166606491 : curr_insn = lra_pop_insn ();
6175 166606491 : --new_min_len;
6176 166606491 : curr_bb = BLOCK_FOR_INSN (curr_insn);
6177 166606491 : if (curr_bb != last_bb)
6178 : {
6179 20978365 : last_bb = curr_bb;
6180 20978365 : bb_reload_num = lra_curr_reload_num;
6181 : }
6182 166606491 : if (min_len > new_min_len)
6183 : {
6184 : min_len = new_min_len;
6185 : new_insns_num = 0;
6186 : original_insn = curr_insn;
6187 : }
6188 7478901 : else if (combine_reload_insn (curr_insn, original_insn))
6189 : {
6190 3622 : continue;
6191 : }
6192 7475279 : if (new_insns_num > MAX_RELOAD_INSNS_NUMBER)
6193 0 : internal_error
6194 0 : ("maximum number of generated reload insns per insn achieved (%d)",
6195 : MAX_RELOAD_INSNS_NUMBER);
6196 166602869 : new_insns_num++;
6197 166602869 : if (DEBUG_INSN_P (curr_insn))
6198 : {
6199 : /* We need to check equivalence in debug insn and change
6200 : pseudo to the equivalent value if necessary. */
6201 55998147 : curr_id = lra_get_insn_recog_data (curr_insn);
6202 55998147 : if (bitmap_bit_p (equiv_insn_bitmap, INSN_UID (curr_insn)))
6203 : {
6204 30019 : rtx old = *curr_id->operand_loc[0];
6205 30019 : *curr_id->operand_loc[0]
6206 30019 : = simplify_replace_fn_rtx (old, NULL_RTX,
6207 : loc_equivalence_callback, curr_insn);
6208 30019 : if (old != *curr_id->operand_loc[0])
6209 : {
6210 : /* If we substitute pseudo by shared equivalence, we can fail
6211 : to update LRA reg info and this can result in many
6212 : unexpected consequences. So keep rtl unshared: */
6213 30019 : *curr_id->operand_loc[0]
6214 30019 : = copy_rtx (*curr_id->operand_loc[0]);
6215 30019 : lra_update_insn_regno_info (curr_insn);
6216 30019 : changed_p = true;
6217 : }
6218 : }
6219 : }
6220 110604722 : else if (INSN_P (curr_insn))
6221 : {
6222 109536609 : if ((set = single_set (curr_insn)) != NULL_RTX)
6223 : {
6224 104050187 : nosubreg_dest = SET_DEST (set);
6225 : /* The equivalence pseudo could be set up as SUBREG in a
6226 : case when it is a call restore insn in a mode
6227 : different from the pseudo mode. */
6228 104050187 : if (GET_CODE (nosubreg_dest) == SUBREG)
6229 1208938 : nosubreg_dest = SUBREG_REG (nosubreg_dest);
6230 104768855 : if ((REG_P (nosubreg_dest)
6231 76980971 : && (x = get_equiv (nosubreg_dest)) != nosubreg_dest
6232 : /* Remove insns which set up a pseudo whose value
6233 : cannot be changed. Such insns might be not in
6234 : init_insns because we don't update equiv data
6235 : during insn transformations.
6236 :
6237 : As an example, let suppose that a pseudo got
6238 : hard register and on the 1st pass was not
6239 : changed to equivalent constant. We generate an
6240 : additional insn setting up the pseudo because of
6241 : secondary memory movement. Then the pseudo is
6242 : spilled and we use the equiv constant. In this
6243 : case we should remove the additional insn and
6244 : this insn is not init_insns list. */
6245 738533 : && (! MEM_P (x) || MEM_READONLY_P (x)
6246 : /* Check that this is actually an insn setting
6247 : up the equivalence. */
6248 334031 : || in_list_p (curr_insn,
6249 334031 : ira_reg_equiv
6250 334031 : [REGNO (nosubreg_dest)].init_insns)))
6251 180313808 : || (((x = get_equiv (SET_SRC (set))) != SET_SRC (set))
6252 2156854 : && in_list_p (curr_insn,
6253 1078427 : ira_reg_equiv
6254 1078427 : [REGNO (SET_SRC (set))].init_insns)
6255 : /* This is a reverse equivalence to memory (see ira.cc)
6256 : in store insn. We can reload all the destination and
6257 : have an output reload which is a store to memory. If
6258 : we just remove the insn, we will have the output
6259 : reload storing an undefined value to the memory.
6260 : Check that we did not reload the memory to prevent a
6261 : wrong code generation. We could implement using the
6262 : equivalence still in such case but doing this is not
6263 : worth the efforts as such case is very rare. */
6264 1318 : && MEM_P (nosubreg_dest)))
6265 : {
6266 : /* This is equiv init insn of pseudo which did not get a
6267 : hard register -- remove the insn. */
6268 718668 : if (lra_dump_file != NULL)
6269 : {
6270 9 : fprintf (lra_dump_file,
6271 : " Removing equiv init insn %i (freq=%d)\n",
6272 3 : INSN_UID (curr_insn),
6273 6 : REG_FREQ_FROM_BB (BLOCK_FOR_INSN (curr_insn)));
6274 3 : dump_insn_slim (lra_dump_file, curr_insn);
6275 : }
6276 718668 : if (contains_reg_p (x, true, false))
6277 143995 : check_and_force_assignment_correctness_p = true;
6278 718668 : lra_set_insn_deleted (curr_insn);
6279 718668 : continue;
6280 : }
6281 : }
6282 108817941 : curr_id = lra_get_insn_recog_data (curr_insn);
6283 108817941 : curr_static_id = curr_id->insn_static_data;
6284 108817941 : init_curr_insn_input_reloads ();
6285 108817941 : init_curr_operand_mode ();
6286 108817941 : if (curr_insn_transform (false))
6287 : changed_p = true;
6288 : /* Check non-transformed insns too for equiv change as USE
6289 : or CLOBBER don't need reloads but can contain pseudos
6290 : being changed on their equivalences. */
6291 102479410 : else if (bitmap_bit_p (equiv_insn_bitmap, INSN_UID (curr_insn))
6292 102479410 : && loc_equivalence_change_p (&PATTERN (curr_insn)))
6293 : {
6294 8 : lra_update_insn_regno_info (curr_insn);
6295 8 : lra_push_insn_by_uid (INSN_UID (curr_insn));
6296 8 : changed_p = true;
6297 : }
6298 : }
6299 : }
6300 :
6301 : /* If we used a new hard regno, changed_p should be true because the
6302 : hard reg is assigned to a new pseudo. */
6303 3300295 : if (flag_checking && !changed_p)
6304 : {
6305 136682567 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
6306 134013625 : if (lra_reg_info[i].nrefs != 0
6307 195499597 : && (hard_regno = lra_get_regno_hard_regno (i)) >= 0)
6308 : {
6309 60007893 : int j, nregs = hard_regno_nregs (hard_regno,
6310 60007893 : PSEUDO_REGNO_MODE (i));
6311 :
6312 122147954 : for (j = 0; j < nregs; j++)
6313 62140061 : lra_assert (df_regs_ever_live_p (hard_regno + j));
6314 : }
6315 : }
6316 2668982 : if (changed_p)
6317 631316 : lra_dump_insns_if_possible ("changed func after local");
6318 3300295 : return changed_p;
6319 3300295 : }
6320 :
6321 : static void initiate_invariants (void);
6322 : static void finish_invariants (void);
6323 :
6324 : /* Initiate the LRA constraint pass. It is done once per
6325 : function. */
6326 : void
6327 1517127 : lra_constraints_init (void)
6328 : {
6329 1517127 : initiate_invariants ();
6330 1517127 : }
6331 :
6332 : /* Finalize the LRA constraint pass. It is done once per
6333 : function. */
6334 : void
6335 1517127 : lra_constraints_finish (void)
6336 : {
6337 1517127 : finish_invariants ();
6338 1517127 : }
6339 :
6340 :
6341 :
6342 : /* Structure describes invariants for ineheritance. */
6343 : struct lra_invariant
6344 : {
6345 : /* The order number of the invariant. */
6346 : int num;
6347 : /* The invariant RTX. */
6348 : rtx invariant_rtx;
6349 : /* The origin insn of the invariant. */
6350 : rtx_insn *insn;
6351 : };
6352 :
6353 : typedef lra_invariant invariant_t;
6354 : typedef invariant_t *invariant_ptr_t;
6355 : typedef const invariant_t *const_invariant_ptr_t;
6356 :
6357 : /* Pointer to the inheritance invariants. */
6358 : static vec<invariant_ptr_t> invariants;
6359 :
6360 : /* Allocation pool for the invariants. */
6361 : static object_allocator<lra_invariant> *invariants_pool;
6362 :
6363 : /* Hash table for the invariants. */
6364 : static htab_t invariant_table;
6365 :
6366 : /* Hash function for INVARIANT. */
6367 : static hashval_t
6368 203488 : invariant_hash (const void *invariant)
6369 : {
6370 203488 : rtx inv = ((const_invariant_ptr_t) invariant)->invariant_rtx;
6371 203488 : return lra_rtx_hash (inv);
6372 : }
6373 :
6374 : /* Equal function for invariants INVARIANT1 and INVARIANT2. */
6375 : static int
6376 73248 : invariant_eq_p (const void *invariant1, const void *invariant2)
6377 : {
6378 73248 : rtx inv1 = ((const_invariant_ptr_t) invariant1)->invariant_rtx;
6379 73248 : rtx inv2 = ((const_invariant_ptr_t) invariant2)->invariant_rtx;
6380 :
6381 73248 : return rtx_equal_p (inv1, inv2);
6382 : }
6383 :
6384 : /* Insert INVARIANT_RTX into the table if it is not there yet. Return
6385 : invariant which is in the table. */
6386 : static invariant_ptr_t
6387 202248 : insert_invariant (rtx invariant_rtx)
6388 : {
6389 202248 : void **entry_ptr;
6390 202248 : invariant_t invariant;
6391 202248 : invariant_ptr_t invariant_ptr;
6392 :
6393 202248 : invariant.invariant_rtx = invariant_rtx;
6394 202248 : entry_ptr = htab_find_slot (invariant_table, &invariant, INSERT);
6395 202248 : if (*entry_ptr == NULL)
6396 : {
6397 170802 : invariant_ptr = invariants_pool->allocate ();
6398 170802 : invariant_ptr->invariant_rtx = invariant_rtx;
6399 170802 : invariant_ptr->insn = NULL;
6400 170802 : invariants.safe_push (invariant_ptr);
6401 170802 : *entry_ptr = (void *) invariant_ptr;
6402 : }
6403 202248 : return (invariant_ptr_t) *entry_ptr;
6404 : }
6405 :
6406 : /* Initiate the invariant table. */
6407 : static void
6408 1517127 : initiate_invariants (void)
6409 : {
6410 1517127 : invariants.create (100);
6411 1517127 : invariants_pool
6412 1517127 : = new object_allocator<lra_invariant> ("Inheritance invariants");
6413 1517127 : invariant_table = htab_create (100, invariant_hash, invariant_eq_p, NULL);
6414 1517127 : }
6415 :
6416 : /* Finish the invariant table. */
6417 : static void
6418 1517127 : finish_invariants (void)
6419 : {
6420 1517127 : htab_delete (invariant_table);
6421 3034254 : delete invariants_pool;
6422 1517127 : invariants.release ();
6423 1517127 : }
6424 :
6425 : /* Make the invariant table empty. */
6426 : static void
6427 12865729 : clear_invariants (void)
6428 : {
6429 12865729 : htab_empty (invariant_table);
6430 12865729 : invariants_pool->release ();
6431 12865729 : invariants.truncate (0);
6432 12865729 : }
6433 :
6434 :
6435 :
6436 : /* This page contains code to do inheritance/split
6437 : transformations. */
6438 :
6439 : /* Number of reloads passed so far in current EBB. */
6440 : static int reloads_num;
6441 :
6442 : /* Number of calls passed so far in current EBB. */
6443 : static int calls_num;
6444 :
6445 : /* Index ID is the CALLS_NUM associated the last call we saw with
6446 : ABI identifier ID. */
6447 : static int last_call_for_abi[NUM_ABI_IDS];
6448 :
6449 : /* Which registers have been fully or partially clobbered by a call
6450 : since they were last used. */
6451 : static HARD_REG_SET full_and_partial_call_clobbers;
6452 :
6453 : /* Current reload pseudo check for validity of elements in
6454 : USAGE_INSNS. */
6455 : static int curr_usage_insns_check;
6456 :
6457 : /* Info about last usage of registers in EBB to do inheritance/split
6458 : transformation. Inheritance transformation is done from a spilled
6459 : pseudo and split transformations from a hard register or a pseudo
6460 : assigned to a hard register. */
6461 : struct usage_insns
6462 : {
6463 : /* If the value is equal to CURR_USAGE_INSNS_CHECK, then the member
6464 : value INSNS is valid. The insns is chain of optional debug insns
6465 : and a finishing non-debug insn using the corresponding reg. The
6466 : value is also used to mark the registers which are set up in the
6467 : current insn. The negated insn uid is used for this. */
6468 : int check;
6469 : /* Value of global reloads_num at the last insn in INSNS. */
6470 : int reloads_num;
6471 : /* Value of global reloads_nums at the last insn in INSNS. */
6472 : int calls_num;
6473 : /* It can be true only for splitting. And it means that the restore
6474 : insn should be put after insn given by the following member. */
6475 : bool after_p;
6476 : /* Next insns in the current EBB which use the original reg and the
6477 : original reg value is not changed between the current insn and
6478 : the next insns. In order words, e.g. for inheritance, if we need
6479 : to use the original reg value again in the next insns we can try
6480 : to use the value in a hard register from a reload insn of the
6481 : current insn. */
6482 : rtx_insn_list *insns;
6483 : };
6484 :
6485 : /* Map: regno -> corresponding pseudo usage insns. */
6486 : static struct usage_insns *usage_insns;
6487 :
6488 : static void
6489 251282154 : setup_next_usage_insn (int regno, rtx_insn *insn, int reloads_num, bool after_p)
6490 : {
6491 251282154 : usage_insns[regno].check = curr_usage_insns_check;
6492 251282154 : usage_insns[regno].insns = gen_rtx_INSN_LIST (VOIDmode, insn, nullptr);
6493 251282154 : usage_insns[regno].reloads_num = reloads_num;
6494 251282154 : usage_insns[regno].calls_num = calls_num;
6495 251282154 : usage_insns[regno].after_p = after_p;
6496 251282154 : if (regno >= FIRST_PSEUDO_REGISTER && reg_renumber[regno] >= 0)
6497 113226928 : remove_from_hard_reg_set (&full_and_partial_call_clobbers,
6498 113226928 : PSEUDO_REGNO_MODE (regno),
6499 : reg_renumber[regno]);
6500 251282154 : }
6501 :
6502 : /* The function is used to form list REGNO usages which consists of
6503 : optional debug insns finished by a non-debug insn using REGNO.
6504 : RELOADS_NUM is current number of reload insns processed so far. */
6505 : static void
6506 143247947 : add_next_usage_insn (int regno, rtx_insn *insn, int reloads_num)
6507 : {
6508 143247947 : rtx_insn_list *next_usage_insns;
6509 :
6510 143247947 : if (usage_insns[regno].check == curr_usage_insns_check
6511 74776730 : && (next_usage_insns = usage_insns[regno].insns) != NULL_RTX
6512 218024677 : && DEBUG_INSN_P (insn))
6513 : {
6514 : /* Check that we did not add the debug insn yet. */
6515 14537001 : if (next_usage_insns->insn () != insn)
6516 14536987 : usage_insns[regno].insns = gen_rtx_INSN_LIST (VOIDmode, insn,
6517 : next_usage_insns);
6518 : }
6519 128710946 : else if (NONDEBUG_INSN_P (insn))
6520 128193559 : setup_next_usage_insn (regno, insn, reloads_num, false);
6521 : else
6522 517387 : usage_insns[regno].check = 0;
6523 143247947 : }
6524 :
6525 : /* Return first non-debug insn in list USAGE_INSNS. */
6526 : static rtx_insn *
6527 1186966 : skip_usage_debug_insns (rtx_insn_list *usage_insns)
6528 : {
6529 1186966 : rtx_insn_list *next;
6530 :
6531 : /* All but the last insn are debug insns. Skip them. */
6532 1502965 : while (usage_insns)
6533 : {
6534 1502965 : next = usage_insns->next ();
6535 1502965 : if (next)
6536 : usage_insns = next;
6537 : else
6538 1186966 : return usage_insns->insn ();
6539 : }
6540 :
6541 : return nullptr;
6542 : }
6543 :
6544 : /* Return true if we need secondary memory moves for insn in
6545 : USAGE_INSNS after inserting inherited pseudo of class INHER_CL
6546 : into the insn. */
6547 : static bool
6548 1186973 : check_secondary_memory_needed_p (enum reg_class inher_cl ATTRIBUTE_UNUSED,
6549 : rtx_insn_list *usage_insns)
6550 : {
6551 1186973 : rtx_insn *insn;
6552 1186973 : rtx set, dest;
6553 1186973 : enum reg_class cl;
6554 :
6555 1186973 : if (inher_cl == ALL_REGS
6556 1186973 : || (insn = skip_usage_debug_insns (usage_insns)) == NULL_RTX)
6557 : return false;
6558 1186966 : lra_assert (INSN_P (insn));
6559 1186966 : if ((set = single_set (insn)) == NULL_RTX || ! REG_P (SET_DEST (set)))
6560 : return false;
6561 1149883 : dest = SET_DEST (set);
6562 1149883 : if (! REG_P (dest))
6563 : return false;
6564 1149883 : lra_assert (inher_cl != NO_REGS);
6565 1149883 : cl = get_reg_class (REGNO (dest));
6566 1149883 : return (cl != NO_REGS && cl != ALL_REGS
6567 1149883 : && targetm.secondary_memory_needed (GET_MODE (dest), inher_cl, cl));
6568 : }
6569 :
6570 : /* Registers involved in inheritance/split in the current EBB
6571 : (inheritance/split pseudos and original registers). */
6572 : static bitmap_head check_only_regs;
6573 :
6574 : /* Reload pseudos cannot be involded in invariant inheritance in the
6575 : current EBB. */
6576 : static bitmap_head invalid_invariant_regs;
6577 :
6578 : /* Do inheritance transformations for insn INSN, which defines (if
6579 : DEF_P) or uses ORIGINAL_REGNO. NEXT_USAGE_INSNS specifies which
6580 : instruction in the EBB next uses ORIGINAL_REGNO; it has the same
6581 : form as the "insns" field of usage_insns. Return true if we
6582 : succeed in such transformation.
6583 :
6584 : The transformations look like:
6585 :
6586 : p <- ... i <- ...
6587 : ... p <- i (new insn)
6588 : ... =>
6589 : <- ... p ... <- ... i ...
6590 : or
6591 : ... i <- p (new insn)
6592 : <- ... p ... <- ... i ...
6593 : ... =>
6594 : <- ... p ... <- ... i ...
6595 : where p is a spilled original pseudo and i is a new inheritance pseudo.
6596 :
6597 :
6598 : The inheritance pseudo has the smallest class of two classes CL and
6599 : class of ORIGINAL REGNO. */
6600 : static bool
6601 1275203 : inherit_reload_reg (bool def_p, int original_regno,
6602 : enum reg_class cl, rtx_insn *insn,
6603 : rtx_insn_list *next_usage_insns)
6604 : {
6605 1275203 : if (optimize_function_for_size_p (cfun))
6606 : return false;
6607 :
6608 1248746 : enum reg_class rclass = lra_get_allocno_class (original_regno);
6609 1248746 : rtx original_reg = regno_reg_rtx[original_regno];
6610 1248746 : rtx new_reg;
6611 1248746 : rtx_insn *new_insns;
6612 :
6613 1248746 : lra_assert (! usage_insns[original_regno].after_p);
6614 1248746 : if (lra_dump_file != NULL)
6615 1 : fprintf (lra_dump_file,
6616 : " <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
6617 1248746 : if (! ira_reg_classes_intersect_p[cl][rclass])
6618 : {
6619 61773 : if (lra_dump_file != NULL)
6620 : {
6621 0 : fprintf (lra_dump_file,
6622 : " Rejecting inheritance for %d "
6623 : "because of disjoint classes %s and %s\n",
6624 : original_regno, reg_class_names[cl],
6625 : reg_class_names[rclass]);
6626 0 : fprintf (lra_dump_file,
6627 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6628 : }
6629 : return false;
6630 : }
6631 1186973 : if ((ira_class_subset_p[cl][rclass] && cl != rclass)
6632 : /* We don't use a subset of two classes because it can be
6633 : NO_REGS. This transformation is still profitable in most
6634 : cases even if the classes are not intersected as register
6635 : move is probably cheaper than a memory load. */
6636 448507 : || ira_class_hard_regs_num[cl] < ira_class_hard_regs_num[rclass])
6637 : {
6638 738466 : if (lra_dump_file != NULL)
6639 1 : fprintf (lra_dump_file, " Use smallest class of %s and %s\n",
6640 : reg_class_names[cl], reg_class_names[rclass]);
6641 :
6642 : rclass = cl;
6643 : }
6644 1186973 : if (check_secondary_memory_needed_p (rclass, next_usage_insns))
6645 : {
6646 : /* Reject inheritance resulting in secondary memory moves.
6647 : Otherwise, there is a danger in LRA cycling. Also such
6648 : transformation will be unprofitable. */
6649 15136 : if (lra_dump_file != NULL)
6650 : {
6651 0 : rtx_insn *insn = skip_usage_debug_insns (next_usage_insns);
6652 0 : rtx set = single_set (insn);
6653 :
6654 0 : lra_assert (set != NULL_RTX);
6655 :
6656 0 : rtx dest = SET_DEST (set);
6657 :
6658 0 : lra_assert (REG_P (dest));
6659 0 : fprintf (lra_dump_file,
6660 : " Rejecting inheritance for insn %d(%s)<-%d(%s) "
6661 : "as secondary mem is needed\n",
6662 0 : REGNO (dest), reg_class_names[get_reg_class (REGNO (dest))],
6663 0 : original_regno, reg_class_names[rclass]);
6664 0 : fprintf (lra_dump_file,
6665 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6666 : }
6667 : return false;
6668 : }
6669 1171837 : if (ira_reg_class_min_nregs[rclass][GET_MODE (original_reg)]
6670 1171837 : != ira_reg_class_max_nregs[rclass][GET_MODE (original_reg)])
6671 : {
6672 24 : if (lra_dump_file != NULL)
6673 : {
6674 0 : fprintf (lra_dump_file,
6675 : " Rejecting inheritance for %d "
6676 : "because of requiring non-uniform class %s\n",
6677 : original_regno, reg_class_names[rclass]);
6678 0 : fprintf (lra_dump_file,
6679 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6680 : }
6681 : return false;
6682 : }
6683 1171813 : new_reg = lra_create_new_reg (GET_MODE (original_reg), original_reg,
6684 : rclass, NULL, "inheritance");
6685 1171813 : start_sequence ();
6686 1171813 : if (def_p)
6687 547370 : lra_emit_move (original_reg, new_reg);
6688 : else
6689 624443 : lra_emit_move (new_reg, original_reg);
6690 1171813 : new_insns = end_sequence ();
6691 1171813 : if (NEXT_INSN (new_insns) != NULL_RTX)
6692 : {
6693 0 : if (lra_dump_file != NULL)
6694 : {
6695 0 : fprintf (lra_dump_file,
6696 : " Rejecting inheritance %d->%d "
6697 : "as it results in 2 or more insns:\n",
6698 : original_regno, REGNO (new_reg));
6699 0 : dump_rtl_slim (lra_dump_file, new_insns, NULL, -1, 0);
6700 0 : fprintf (lra_dump_file,
6701 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6702 : }
6703 : return false;
6704 : }
6705 1171813 : lra_substitute_pseudo_within_insn (insn, original_regno, new_reg, false);
6706 1171813 : lra_update_insn_regno_info (insn);
6707 1171813 : if (! def_p)
6708 : /* We now have a new usage insn for original regno. */
6709 624443 : setup_next_usage_insn (original_regno, new_insns, reloads_num, false);
6710 1171813 : if (lra_dump_file != NULL)
6711 1 : fprintf (lra_dump_file, " Original reg change %d->%d (bb%d):\n",
6712 1 : original_regno, REGNO (new_reg), BLOCK_FOR_INSN (insn)->index);
6713 1171813 : lra_reg_info[REGNO (new_reg)].restore_rtx = regno_reg_rtx[original_regno];
6714 1171813 : bitmap_set_bit (&check_only_regs, REGNO (new_reg));
6715 1171813 : bitmap_set_bit (&check_only_regs, original_regno);
6716 1171813 : bitmap_set_bit (&lra_inheritance_pseudos, REGNO (new_reg));
6717 1171813 : if (def_p)
6718 547370 : lra_process_new_insns (insn, NULL, new_insns,
6719 : "Add original<-inheritance");
6720 : else
6721 624443 : lra_process_new_insns (insn, new_insns, NULL,
6722 : "Add inheritance<-original");
6723 : rtx_insn *usage_insn;
6724 2656814 : while (next_usage_insns != NULL_RTX)
6725 : {
6726 1485001 : usage_insn = next_usage_insns->insn ();
6727 1485001 : if (!next_usage_insns->next ())
6728 : {
6729 1171813 : lra_assert (NONDEBUG_INSN_P (usage_insn));
6730 : next_usage_insns = NULL;
6731 : }
6732 : else
6733 : {
6734 313188 : lra_assert (DEBUG_INSN_P (usage_insn));
6735 : next_usage_insns = next_usage_insns->next ();
6736 : }
6737 1485001 : rtx temp = usage_insn;
6738 1485001 : lra_substitute_pseudo (&temp, original_regno, new_reg, false,
6739 1485001 : DEBUG_INSN_P (usage_insn));
6740 1485001 : lra_update_insn_regno_info (usage_insn);
6741 1485001 : if (lra_dump_file != NULL)
6742 : {
6743 1 : basic_block bb = BLOCK_FOR_INSN (usage_insn);
6744 1 : fprintf (lra_dump_file,
6745 : " Inheritance reuse change %d->%d (bb%d):\n",
6746 : original_regno, REGNO (new_reg),
6747 : bb ? bb->index : -1);
6748 1 : dump_insn_slim (lra_dump_file, usage_insn);
6749 : }
6750 : }
6751 1171813 : if (lra_dump_file != NULL)
6752 1 : fprintf (lra_dump_file,
6753 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6754 : return true;
6755 : }
6756 :
6757 : /* Return true if we need a caller save/restore for pseudo REGNO which
6758 : was assigned to a hard register. */
6759 : static inline bool
6760 115977555 : need_for_call_save_p (int regno)
6761 : {
6762 115977555 : lra_assert (regno >= FIRST_PSEUDO_REGISTER && reg_renumber[regno] >= 0);
6763 115977555 : if (usage_insns[regno].calls_num < calls_num)
6764 : {
6765 : unsigned int abis = 0;
6766 121029961 : for (unsigned int i = 0; i < NUM_ABI_IDS; ++i)
6767 111719964 : if (last_call_for_abi[i] > usage_insns[regno].calls_num)
6768 9359316 : abis |= 1 << i;
6769 9309997 : gcc_assert (abis);
6770 9309997 : if (call_clobbered_in_region_p (abis, full_and_partial_call_clobbers,
6771 9309997 : PSEUDO_REGNO_MODE (regno),
6772 : reg_renumber[regno]))
6773 : return true;
6774 : }
6775 : return false;
6776 : }
6777 :
6778 : /* Global registers occurring in the current EBB. */
6779 : static bitmap_head ebb_global_regs;
6780 :
6781 : /* Return true if we need a split for hard register REGNO or pseudo
6782 : REGNO which was assigned to a hard register.
6783 : POTENTIAL_RELOAD_HARD_REGS contains hard registers which might be
6784 : used for reloads since the EBB end. It is an approximation of the
6785 : used hard registers in the split range. The exact value would
6786 : require expensive calculations. If we were aggressive with
6787 : splitting because of the approximation, the split pseudo will save
6788 : the same hard register assignment and will be removed in the undo
6789 : pass. We still need the approximation because too aggressive
6790 : splitting would result in too inaccurate cost calculation in the
6791 : assignment pass because of too many generated moves which will be
6792 : probably removed in the undo pass. */
6793 : static inline bool
6794 246473716 : need_for_split_p (HARD_REG_SET potential_reload_hard_regs, int regno)
6795 : {
6796 246473716 : int hard_regno = regno < FIRST_PSEUDO_REGISTER ? regno : reg_renumber[regno];
6797 :
6798 246473716 : lra_assert (hard_regno >= 0);
6799 246473716 : return ((TEST_HARD_REG_BIT (potential_reload_hard_regs, hard_regno)
6800 : /* Don't split eliminable hard registers, otherwise we can
6801 : split hard registers like hard frame pointer, which
6802 : lives on BB start/end according to DF-infrastructure,
6803 : when there is a pseudo assigned to the register and
6804 : living in the same BB. */
6805 676758 : && (regno >= FIRST_PSEUDO_REGISTER
6806 44609 : || ! TEST_HARD_REG_BIT (eliminable_regset, hard_regno))
6807 647142 : && ! TEST_HARD_REG_BIT (lra_no_alloc_regs, hard_regno)
6808 : /* Don't split call clobbered hard regs living through
6809 : calls, otherwise we might have a check problem in the
6810 : assign sub-pass as in the most cases (exception is a
6811 : situation when check_and_force_assignment_correctness_p value is
6812 : true) the assign pass assumes that all pseudos living
6813 : through calls are assigned to call saved hard regs. */
6814 633193 : && (regno >= FIRST_PSEUDO_REGISTER
6815 1044 : || !TEST_HARD_REG_BIT (full_and_partial_call_clobbers, regno))
6816 : /* We need at least 2 reloads to make pseudo splitting
6817 : profitable. We should provide hard regno splitting in
6818 : any case to solve 1st insn scheduling problem when
6819 : moving hard register definition up might result in
6820 : impossibility to find hard register for reload pseudo of
6821 : small register class. */
6822 1266334 : && (usage_insns[regno].reloads_num
6823 1265316 : + (regno < FIRST_PSEUDO_REGISTER ? 0 : 3) < reloads_num)
6824 2933 : && (regno < FIRST_PSEUDO_REGISTER
6825 : /* For short living pseudos, spilling + inheritance can
6826 : be considered a substitution for splitting.
6827 : Therefore we do not splitting for local pseudos. It
6828 : decreases also aggressiveness of splitting. The
6829 : minimal number of references is chosen taking into
6830 : account that for 2 references splitting has no sense
6831 : as we can just spill the pseudo. */
6832 : || (regno >= FIRST_PSEUDO_REGISTER
6833 2886 : && lra_reg_info[regno].nrefs > 3
6834 2662 : && bitmap_bit_p (&ebb_global_regs, regno))))
6835 247148974 : || (regno >= FIRST_PSEUDO_REGISTER && need_for_call_save_p (regno)));
6836 : }
6837 :
6838 : /* Return class for the split pseudo created from original pseudo with
6839 : ALLOCNO_CLASS and MODE which got a hard register HARD_REGNO. We
6840 : choose subclass of ALLOCNO_CLASS which contains HARD_REGNO and
6841 : results in no secondary memory movements. */
6842 : static enum reg_class
6843 1660 : choose_split_class (enum reg_class allocno_class,
6844 : int hard_regno ATTRIBUTE_UNUSED,
6845 : machine_mode mode ATTRIBUTE_UNUSED)
6846 : {
6847 1660 : int i;
6848 1660 : enum reg_class cl, best_cl = NO_REGS;
6849 1660 : enum reg_class hard_reg_class ATTRIBUTE_UNUSED
6850 : = REGNO_REG_CLASS (hard_regno);
6851 :
6852 1660 : if (! targetm.secondary_memory_needed (mode, allocno_class, allocno_class)
6853 1660 : && TEST_HARD_REG_BIT (reg_class_contents[allocno_class], hard_regno))
6854 : return allocno_class;
6855 0 : for (i = 0;
6856 0 : (cl = reg_class_subclasses[allocno_class][i]) != LIM_REG_CLASSES;
6857 : i++)
6858 0 : if (! targetm.secondary_memory_needed (mode, cl, hard_reg_class)
6859 0 : && ! targetm.secondary_memory_needed (mode, hard_reg_class, cl)
6860 0 : && TEST_HARD_REG_BIT (reg_class_contents[cl], hard_regno)
6861 0 : && (best_cl == NO_REGS
6862 0 : || ira_class_hard_regs_num[best_cl] < ira_class_hard_regs_num[cl]))
6863 : best_cl = cl;
6864 : return best_cl;
6865 : }
6866 :
6867 : /* Copy any equivalence information from ORIGINAL_REGNO to NEW_REGNO. It only
6868 : makes sense to call this function if NEW_REGNO is always equal to
6869 : ORIGINAL_REGNO. Set up defined_p flag when caller_save_p flag is set up and
6870 : CALL_SAVE_P is true. */
6871 :
6872 : static void
6873 665767 : lra_copy_reg_equiv (unsigned int new_regno, unsigned int original_regno,
6874 : bool call_save_p)
6875 : {
6876 665767 : if (!ira_reg_equiv[original_regno].defined_p
6877 595879 : && !(call_save_p && ira_reg_equiv[original_regno].caller_save_p))
6878 : return;
6879 :
6880 70075 : ira_expand_reg_equiv ();
6881 70075 : ira_reg_equiv[new_regno].defined_p = true;
6882 70075 : if (ira_reg_equiv[original_regno].memory)
6883 33381 : ira_reg_equiv[new_regno].memory
6884 33381 : = copy_rtx (ira_reg_equiv[original_regno].memory);
6885 70075 : if (ira_reg_equiv[original_regno].constant)
6886 30111 : ira_reg_equiv[new_regno].constant
6887 30111 : = copy_rtx (ira_reg_equiv[original_regno].constant);
6888 70075 : if (ira_reg_equiv[original_regno].invariant)
6889 6583 : ira_reg_equiv[new_regno].invariant
6890 6583 : = copy_rtx (ira_reg_equiv[original_regno].invariant);
6891 : }
6892 :
6893 : /* Do split transformations for insn INSN, which defines or uses
6894 : ORIGINAL_REGNO. NEXT_USAGE_INSNS specifies which instruction in
6895 : the EBB next uses ORIGINAL_REGNO; it has the same form as the
6896 : "insns" field of usage_insns. If TO is not NULL, we don't use
6897 : usage_insns, we put restore insns after TO insn. It is a case when
6898 : we call it from lra_split_hard_reg_for, outside the inheritance
6899 : pass.
6900 :
6901 : The transformations look like:
6902 :
6903 : p <- ... p <- ...
6904 : ... s <- p (new insn -- save)
6905 : ... =>
6906 : ... p <- s (new insn -- restore)
6907 : <- ... p ... <- ... p ...
6908 : or
6909 : <- ... p ... <- ... p ...
6910 : ... s <- p (new insn -- save)
6911 : ... =>
6912 : ... p <- s (new insn -- restore)
6913 : <- ... p ... <- ... p ...
6914 :
6915 : where p is an original pseudo got a hard register or a hard
6916 : register and s is a new split pseudo. The save is put before INSN
6917 : if BEFORE_P is true. Return true if we succeed in such
6918 : transformation. */
6919 : static bool
6920 667635 : split_reg (bool before_p, int original_regno, rtx_insn *insn,
6921 : rtx_insn_list *next_usage_insns, rtx_insn *to)
6922 : {
6923 667635 : enum reg_class rclass;
6924 667635 : rtx original_reg;
6925 667635 : int hard_regno, nregs;
6926 667635 : rtx new_reg;
6927 667635 : rtx_insn *restore, *save;
6928 667635 : bool after_p;
6929 667635 : bool call_save_p;
6930 667635 : machine_mode mode;
6931 :
6932 667635 : if (original_regno < FIRST_PSEUDO_REGISTER)
6933 : {
6934 208 : rclass = ira_allocno_class_translate[REGNO_REG_CLASS (original_regno)];
6935 208 : hard_regno = original_regno;
6936 208 : call_save_p = false;
6937 208 : nregs = 1;
6938 208 : mode = lra_reg_info[hard_regno].biggest_mode;
6939 208 : machine_mode reg_rtx_mode = GET_MODE (regno_reg_rtx[hard_regno]);
6940 : /* A reg can have a biggest_mode of VOIDmode if it was only ever seen as
6941 : part of a multi-word register. In that case, just use the reg_rtx
6942 : mode. Do the same also if the biggest mode was larger than a register
6943 : or we can not compare the modes. Otherwise, limit the size to that of
6944 : the biggest access in the function or to the natural mode at least. */
6945 208 : if (mode == VOIDmode
6946 208 : || !ordered_p (GET_MODE_PRECISION (mode),
6947 208 : GET_MODE_PRECISION (reg_rtx_mode))
6948 208 : || paradoxical_subreg_p (mode, reg_rtx_mode)
6949 415 : || maybe_gt (GET_MODE_PRECISION (reg_rtx_mode), GET_MODE_PRECISION (mode)))
6950 : {
6951 : original_reg = regno_reg_rtx[hard_regno];
6952 : mode = reg_rtx_mode;
6953 : }
6954 : else
6955 191 : original_reg = gen_rtx_REG (mode, hard_regno);
6956 : }
6957 : else
6958 : {
6959 667427 : mode = PSEUDO_REGNO_MODE (original_regno);
6960 667427 : hard_regno = reg_renumber[original_regno];
6961 667427 : nregs = hard_regno_nregs (hard_regno, mode);
6962 667427 : rclass = lra_get_allocno_class (original_regno);
6963 667427 : original_reg = regno_reg_rtx[original_regno];
6964 667427 : call_save_p = need_for_call_save_p (original_regno);
6965 : }
6966 667635 : lra_assert (hard_regno >= 0);
6967 667635 : if (lra_dump_file != NULL)
6968 0 : fprintf (lra_dump_file,
6969 : " ((((((((((((((((((((((((((((((((((((((((((((((((\n");
6970 :
6971 667635 : if (call_save_p)
6972 : {
6973 665975 : mode = HARD_REGNO_CALLER_SAVE_MODE (hard_regno,
6974 : hard_regno_nregs (hard_regno, mode),
6975 : mode);
6976 665975 : new_reg = lra_create_new_reg (mode, NULL_RTX, NO_REGS, NULL, "save");
6977 : }
6978 : else
6979 : {
6980 1660 : rclass = choose_split_class (rclass, hard_regno, mode);
6981 1660 : if (rclass == NO_REGS)
6982 : {
6983 0 : if (lra_dump_file != NULL)
6984 : {
6985 0 : fprintf (lra_dump_file,
6986 : " Rejecting split of %d(%s): "
6987 : "no good reg class for %d(%s)\n",
6988 : original_regno,
6989 0 : reg_class_names[lra_get_allocno_class (original_regno)],
6990 : hard_regno,
6991 0 : reg_class_names[REGNO_REG_CLASS (hard_regno)]);
6992 0 : fprintf
6993 0 : (lra_dump_file,
6994 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
6995 : }
6996 : return false;
6997 : }
6998 : /* Split_if_necessary can split hard registers used as part of a
6999 : multi-register mode but splits each register individually. The
7000 : mode used for each independent register may not be supported
7001 : so reject the split. Splitting the wider mode should theoretically
7002 : be possible but is not implemented. */
7003 1660 : if (!targetm.hard_regno_mode_ok (hard_regno, mode))
7004 : {
7005 0 : if (lra_dump_file != NULL)
7006 : {
7007 0 : fprintf (lra_dump_file,
7008 : " Rejecting split of %d(%s): unsuitable mode %s\n",
7009 : original_regno,
7010 0 : reg_class_names[lra_get_allocno_class (original_regno)],
7011 0 : GET_MODE_NAME (mode));
7012 0 : fprintf
7013 0 : (lra_dump_file,
7014 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
7015 : }
7016 : return false;
7017 : }
7018 1660 : new_reg = lra_create_new_reg (mode, original_reg, rclass, NULL, "split");
7019 1660 : reg_renumber[REGNO (new_reg)] = hard_regno;
7020 : }
7021 667635 : int new_regno = REGNO (new_reg);
7022 667635 : save = emit_spill_move (true, new_reg, original_reg);
7023 667635 : if (NEXT_INSN (save) != NULL_RTX && !call_save_p)
7024 : {
7025 0 : if (lra_dump_file != NULL)
7026 : {
7027 0 : fprintf
7028 0 : (lra_dump_file,
7029 : " Rejecting split %d->%d resulting in > 2 save insns:\n",
7030 : original_regno, new_regno);
7031 0 : dump_rtl_slim (lra_dump_file, save, NULL, -1, 0);
7032 0 : fprintf (lra_dump_file,
7033 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
7034 : }
7035 : return false;
7036 : }
7037 667635 : restore = emit_spill_move (false, new_reg, original_reg);
7038 667635 : if (NEXT_INSN (restore) != NULL_RTX && !call_save_p)
7039 : {
7040 0 : if (lra_dump_file != NULL)
7041 : {
7042 0 : fprintf (lra_dump_file,
7043 : " Rejecting split %d->%d "
7044 : "resulting in > 2 restore insns:\n",
7045 : original_regno, new_regno);
7046 0 : dump_rtl_slim (lra_dump_file, restore, NULL, -1, 0);
7047 0 : fprintf (lra_dump_file,
7048 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
7049 : }
7050 : return false;
7051 : }
7052 : /* Transfer equivalence information to the spill register, so that
7053 : if we fail to allocate the spill register, we have the option of
7054 : rematerializing the original value instead of spilling to the stack. */
7055 667635 : if (!HARD_REGISTER_NUM_P (original_regno)
7056 667427 : && mode == PSEUDO_REGNO_MODE (original_regno))
7057 665767 : lra_copy_reg_equiv (new_regno, original_regno, call_save_p);
7058 667635 : lra_reg_info[new_regno].restore_rtx = regno_reg_rtx[original_regno];
7059 667635 : bitmap_set_bit (&lra_split_regs, new_regno);
7060 667635 : rtx_insn *usage_insn;
7061 667635 : if (to != NULL)
7062 : {
7063 161 : lra_assert (next_usage_insns == NULL);
7064 : usage_insn = to;
7065 : after_p = true;
7066 : }
7067 : else
7068 : {
7069 : /* We need check_only_regs only inside the inheritance pass. */
7070 667474 : bitmap_set_bit (&check_only_regs, new_regno);
7071 667474 : bitmap_set_bit (&check_only_regs, original_regno);
7072 667474 : after_p = usage_insns[original_regno].after_p;
7073 893078 : for (;;)
7074 : {
7075 780276 : usage_insn = next_usage_insns->insn ();
7076 780276 : if (!next_usage_insns->next ())
7077 : break;
7078 :
7079 112802 : lra_assert (DEBUG_INSN_P (usage_insn));
7080 112802 : next_usage_insns = next_usage_insns->next ();
7081 112802 : rtx temp = usage_insn;
7082 112802 : lra_substitute_pseudo (&temp, original_regno, new_reg, false,
7083 : true);
7084 112802 : lra_update_insn_regno_info (usage_insn);
7085 112802 : if (lra_dump_file != NULL)
7086 : {
7087 0 : fprintf (lra_dump_file, " Split reuse change %d->%d:\n",
7088 : original_regno, new_regno);
7089 0 : dump_insn_slim (lra_dump_file, usage_insn);
7090 : }
7091 112802 : }
7092 : }
7093 667635 : lra_assert (NOTE_P (usage_insn) || NONDEBUG_INSN_P (usage_insn));
7094 667635 : lra_assert (usage_insn != insn || (after_p && before_p));
7095 1126947 : lra_process_new_insns (usage_insn,
7096 : after_p ? NULL : restore,
7097 : after_p ? restore : NULL,
7098 : call_save_p ? "Add reg<-save" : "Add reg<-split");
7099 667635 : if (call_save_p
7100 665975 : && first_call_insn != NULL
7101 1333610 : && BLOCK_FOR_INSN (first_call_insn) != BLOCK_FOR_INSN (insn))
7102 : /* PR116028: If original_regno is a pseudo that has been assigned a
7103 : callee-saved hard register, then emit the spill insn before the call
7104 : insn 'first_call_insn' instead of adjacent to 'insn'. If 'insn'
7105 : and 'first_call_insn' belong to the same EBB but to two separate
7106 : BBs, and if 'insn' is present in the entry BB, then generating the
7107 : spill insn in the entry BB can prevent shrink wrap from happening.
7108 : This is because the spill insn references the stack pointer and
7109 : hence the prolog gets generated in the entry BB itself. It is
7110 : also more efficient to generate the spill before
7111 : 'first_call_insn' as the spill now occurs only in the path
7112 : containing the call. */
7113 25120 : lra_process_new_insns (first_call_insn, save, NULL, "Add save<-reg");
7114 : else
7115 1286163 : lra_process_new_insns (insn, before_p ? save : NULL,
7116 : before_p ? NULL : save,
7117 : call_save_p ? "Add save<-reg" : "Add split<-reg");
7118 667635 : if (nregs > 1 || original_regno < FIRST_PSEUDO_REGISTER)
7119 : /* If we are trying to split multi-register. We should check
7120 : conflicts on the next assignment sub-pass. IRA can allocate on
7121 : sub-register levels, LRA do this on pseudos level right now and
7122 : this discrepancy may create allocation conflicts after
7123 : splitting.
7124 :
7125 : If we are trying to split hard register we should also check conflicts
7126 : as such splitting can create artificial conflict of the hard register
7127 : with another pseudo because of simplified conflict calculation in
7128 : LRA. */
7129 9254 : check_and_force_assignment_correctness_p = true;
7130 667635 : if (lra_dump_file != NULL)
7131 0 : fprintf (lra_dump_file,
7132 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
7133 : return true;
7134 : }
7135 :
7136 : /* Split a hard reg for reload pseudo REGNO having RCLASS and living
7137 : in the range [FROM, TO]. Return true if did a split. Otherwise,
7138 : return false. */
7139 : bool
7140 991 : spill_hard_reg_in_range (int regno, enum reg_class rclass, rtx_insn *from, rtx_insn *to)
7141 : {
7142 991 : int i, hard_regno;
7143 991 : int rclass_size;
7144 991 : rtx_insn *insn;
7145 991 : unsigned int uid;
7146 991 : bitmap_iterator bi;
7147 991 : HARD_REG_SET ignore;
7148 :
7149 991 : lra_assert (from != NULL && to != NULL);
7150 991 : ignore = lra_no_alloc_regs;
7151 2768 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi)
7152 : {
7153 1777 : lra_insn_recog_data_t id = lra_insn_recog_data[uid];
7154 1777 : struct lra_static_insn_data *static_id = id->insn_static_data;
7155 1777 : struct lra_insn_reg *reg;
7156 :
7157 6357 : for (reg = id->regs; reg != NULL; reg = reg->next)
7158 4580 : if (reg->regno < FIRST_PSEUDO_REGISTER)
7159 295 : SET_HARD_REG_BIT (ignore, reg->regno);
7160 2850 : for (reg = static_id->hard_regs; reg != NULL; reg = reg->next)
7161 1073 : SET_HARD_REG_BIT (ignore, reg->regno);
7162 : }
7163 991 : rclass_size = ira_class_hard_regs_num[rclass];
7164 3165 : for (i = 0; i < rclass_size; i++)
7165 : {
7166 2335 : hard_regno = ira_class_hard_regs[rclass][i];
7167 2335 : if (! TEST_HARD_REG_BIT (lra_reg_info[regno].conflict_hard_regs, hard_regno)
7168 2335 : || TEST_HARD_REG_BIT (ignore, hard_regno))
7169 2168 : continue;
7170 482 : for (insn = from; insn != NEXT_INSN (to); insn = NEXT_INSN (insn))
7171 : {
7172 321 : struct lra_static_insn_data *static_id;
7173 321 : struct lra_insn_reg *reg;
7174 :
7175 321 : if (!INSN_P (insn))
7176 0 : continue;
7177 321 : if (bitmap_bit_p (&lra_reg_info[hard_regno].insn_bitmap,
7178 321 : INSN_UID (insn)))
7179 : break;
7180 315 : static_id = lra_get_insn_recog_data (insn)->insn_static_data;
7181 369 : for (reg = static_id->hard_regs; reg != NULL; reg = reg->next)
7182 54 : if (reg->regno == hard_regno)
7183 : break;
7184 : if (reg != NULL)
7185 : break;
7186 : }
7187 167 : if (insn != NEXT_INSN (to))
7188 6 : continue;
7189 161 : if (split_reg (true, hard_regno, from, NULL, to))
7190 : return true;
7191 : }
7192 : return false;
7193 : }
7194 :
7195 : /* Recognize that we need a split transformation for insn INSN, which
7196 : defines or uses REGNO in its insn biggest MODE (we use it only if
7197 : REGNO is a hard register). POTENTIAL_RELOAD_HARD_REGS contains
7198 : hard registers which might be used for reloads since the EBB end.
7199 : Put the save before INSN if BEFORE_P is true. MAX_UID is maximla
7200 : uid before starting INSN processing. Return true if we succeed in
7201 : such transformation. */
7202 : static bool
7203 203750327 : split_if_necessary (int regno, machine_mode mode,
7204 : HARD_REG_SET potential_reload_hard_regs,
7205 : bool before_p, rtx_insn *insn, int max_uid)
7206 : {
7207 203750327 : bool res = false;
7208 203750327 : int i, nregs = 1;
7209 203750327 : rtx_insn_list *next_usage_insns;
7210 :
7211 203750327 : if (regno < FIRST_PSEUDO_REGISTER)
7212 95784605 : nregs = hard_regno_nregs (regno, mode);
7213 407862260 : for (i = 0; i < nregs; i++)
7214 204111933 : if (usage_insns[regno + i].check == curr_usage_insns_check
7215 136140733 : && (next_usage_insns = usage_insns[regno + i].insns) != NULL_RTX
7216 : /* To avoid processing the register twice or more. */
7217 136140733 : && INSN_UID (next_usage_insns->insn ()) < max_uid
7218 136140733 : && need_for_split_p (potential_reload_hard_regs, regno + i)
7219 204393974 : && split_reg (before_p, regno + i, insn, next_usage_insns, NULL))
7220 : res = true;
7221 203750327 : return res;
7222 : }
7223 :
7224 : /* Return TRUE if rtx X is considered as an invariant for
7225 : inheritance. */
7226 : static bool
7227 12297541 : invariant_p (const_rtx x)
7228 : {
7229 12297541 : machine_mode mode;
7230 12297541 : const char *fmt;
7231 12297541 : enum rtx_code code;
7232 12297541 : int i, j;
7233 :
7234 12297541 : if (side_effects_p (x))
7235 : return false;
7236 :
7237 12282921 : code = GET_CODE (x);
7238 12282921 : mode = GET_MODE (x);
7239 12282921 : if (code == SUBREG)
7240 : {
7241 499846 : x = SUBREG_REG (x);
7242 499846 : code = GET_CODE (x);
7243 499846 : mode = wider_subreg_mode (mode, GET_MODE (x));
7244 : }
7245 :
7246 12282921 : if (MEM_P (x))
7247 : return false;
7248 :
7249 10510940 : if (REG_P (x))
7250 : {
7251 3713963 : int i, nregs, regno = REGNO (x);
7252 :
7253 3713963 : if (regno >= FIRST_PSEUDO_REGISTER || regno == STACK_POINTER_REGNUM
7254 1009932 : || TEST_HARD_REG_BIT (eliminable_regset, regno)
7255 3730067 : || GET_MODE_CLASS (GET_MODE (x)) == MODE_CC)
7256 : return false;
7257 2 : nregs = hard_regno_nregs (regno, mode);
7258 2 : for (i = 0; i < nregs; i++)
7259 2 : if (! fixed_regs[regno + i]
7260 : /* A hard register may be clobbered in the current insn
7261 : but we can ignore this case because if the hard
7262 : register is used it should be set somewhere after the
7263 : clobber. */
7264 2 : || bitmap_bit_p (&invalid_invariant_regs, regno + i))
7265 : return false;
7266 : }
7267 6796977 : fmt = GET_RTX_FORMAT (code);
7268 12010280 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
7269 : {
7270 9025865 : if (fmt[i] == 'e')
7271 : {
7272 5863337 : if (! invariant_p (XEXP (x, i)))
7273 : return false;
7274 : }
7275 3162528 : else if (fmt[i] == 'E')
7276 : {
7277 747471 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
7278 629542 : if (! invariant_p (XVECEXP (x, i, j)))
7279 : return false;
7280 : }
7281 : }
7282 : return true;
7283 : }
7284 :
7285 : /* We have 'dest_reg <- invariant'. Let us try to make an invariant
7286 : inheritance transformation (using dest_reg instead invariant in a
7287 : subsequent insn). */
7288 : static bool
7289 202248 : process_invariant_for_inheritance (rtx dst_reg, rtx invariant_rtx)
7290 : {
7291 202248 : invariant_ptr_t invariant_ptr;
7292 202248 : rtx_insn *insn, *new_insns;
7293 202248 : rtx insn_set, insn_reg, new_reg;
7294 202248 : int insn_regno;
7295 202248 : bool succ_p = false;
7296 202248 : int dst_regno = REGNO (dst_reg);
7297 202248 : machine_mode dst_mode = GET_MODE (dst_reg);
7298 202248 : enum reg_class cl = lra_get_allocno_class (dst_regno), insn_reg_cl;
7299 :
7300 202248 : invariant_ptr = insert_invariant (invariant_rtx);
7301 202248 : if ((insn = invariant_ptr->insn) != NULL_RTX)
7302 : {
7303 : /* We have a subsequent insn using the invariant. */
7304 31446 : insn_set = single_set (insn);
7305 31446 : lra_assert (insn_set != NULL);
7306 31446 : insn_reg = SET_DEST (insn_set);
7307 31446 : lra_assert (REG_P (insn_reg));
7308 31446 : insn_regno = REGNO (insn_reg);
7309 31446 : insn_reg_cl = lra_get_allocno_class (insn_regno);
7310 :
7311 31446 : if (dst_mode == GET_MODE (insn_reg)
7312 : /* We should consider only result move reg insns which are
7313 : cheap. */
7314 31375 : && targetm.register_move_cost (dst_mode, cl, insn_reg_cl) == 2
7315 61631 : && targetm.register_move_cost (dst_mode, cl, cl) == 2)
7316 : {
7317 30185 : if (lra_dump_file != NULL)
7318 0 : fprintf (lra_dump_file,
7319 : " [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[\n");
7320 30185 : new_reg = lra_create_new_reg (dst_mode, dst_reg, cl, NULL,
7321 : "invariant inheritance");
7322 30185 : bitmap_set_bit (&lra_inheritance_pseudos, REGNO (new_reg));
7323 30185 : bitmap_set_bit (&check_only_regs, REGNO (new_reg));
7324 30185 : add_invariant (new_reg, invariant_rtx);
7325 30185 : lra_reg_info[REGNO (new_reg)].restore_rtx = PATTERN (insn);
7326 30185 : start_sequence ();
7327 30185 : lra_emit_move (new_reg, dst_reg);
7328 30185 : new_insns = end_sequence ();
7329 30185 : lra_process_new_insns (curr_insn, NULL, new_insns,
7330 : "Add invariant inheritance<-original");
7331 30185 : start_sequence ();
7332 30185 : lra_emit_move (SET_DEST (insn_set), new_reg);
7333 30185 : new_insns = end_sequence ();
7334 30185 : lra_process_new_insns (insn, NULL, new_insns,
7335 : "Changing reload<-inheritance");
7336 30185 : lra_set_insn_deleted (insn);
7337 30185 : succ_p = true;
7338 30185 : if (lra_dump_file != NULL)
7339 : {
7340 0 : fprintf (lra_dump_file,
7341 : " Invariant inheritance reuse change %d (bb%d):\n",
7342 0 : REGNO (new_reg), BLOCK_FOR_INSN (insn)->index);
7343 0 : dump_insn_slim (lra_dump_file, insn);
7344 0 : fprintf (lra_dump_file,
7345 : " ]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]\n");
7346 : }
7347 : }
7348 : }
7349 202248 : invariant_ptr->insn = curr_insn;
7350 202248 : return succ_p;
7351 : }
7352 :
7353 : /* Check only registers living at the current program point in the
7354 : current EBB. */
7355 : static bitmap_head live_regs;
7356 :
7357 : /* Update live info in EBB given by its HEAD and TAIL insns after
7358 : inheritance/split transformation. The function removes dead moves
7359 : too. */
7360 : static void
7361 745513 : update_ebb_live_info (rtx_insn *head, rtx_insn *tail)
7362 : {
7363 745513 : unsigned int j;
7364 745513 : int i, regno;
7365 745513 : bool live_p;
7366 745513 : rtx_insn *prev_insn;
7367 745513 : rtx set;
7368 745513 : bool remove_p;
7369 745513 : basic_block last_bb, prev_bb, curr_bb;
7370 745513 : bitmap_iterator bi;
7371 745513 : struct lra_insn_reg *reg;
7372 745513 : edge e;
7373 745513 : edge_iterator ei;
7374 :
7375 745513 : last_bb = BLOCK_FOR_INSN (tail);
7376 745513 : prev_bb = NULL;
7377 745513 : for (curr_insn = tail;
7378 39016989 : curr_insn != PREV_INSN (head);
7379 38271476 : curr_insn = prev_insn)
7380 : {
7381 38271476 : prev_insn = PREV_INSN (curr_insn);
7382 : /* We need to process empty blocks too. They contain
7383 : NOTE_INSN_BASIC_BLOCK referring for the basic block. */
7384 38271476 : if (NOTE_P (curr_insn) && NOTE_KIND (curr_insn) != NOTE_INSN_BASIC_BLOCK)
7385 1435980 : continue;
7386 36835496 : curr_bb = BLOCK_FOR_INSN (curr_insn);
7387 36835496 : if (curr_bb != prev_bb)
7388 : {
7389 1490579 : if (prev_bb != NULL)
7390 : {
7391 : /* Update df_get_live_in (prev_bb): */
7392 9686146 : EXECUTE_IF_SET_IN_BITMAP (&check_only_regs, 0, j, bi)
7393 8941080 : if (bitmap_bit_p (&live_regs, j))
7394 1252902 : bitmap_set_bit (df_get_live_in (prev_bb), j);
7395 : else
7396 7688178 : bitmap_clear_bit (df_get_live_in (prev_bb), j);
7397 : }
7398 1490579 : if (curr_bb != last_bb)
7399 : {
7400 : /* Update df_get_live_out (curr_bb): */
7401 9686146 : EXECUTE_IF_SET_IN_BITMAP (&check_only_regs, 0, j, bi)
7402 : {
7403 8941080 : live_p = bitmap_bit_p (&live_regs, j);
7404 8941080 : if (! live_p)
7405 22966450 : FOR_EACH_EDGE (e, ei, curr_bb->succs)
7406 15334211 : if (bitmap_bit_p (df_get_live_in (e->dest), j))
7407 : {
7408 : live_p = true;
7409 : break;
7410 : }
7411 7688178 : if (live_p)
7412 1308841 : bitmap_set_bit (df_get_live_out (curr_bb), j);
7413 : else
7414 7632239 : bitmap_clear_bit (df_get_live_out (curr_bb), j);
7415 : }
7416 : }
7417 1490579 : prev_bb = curr_bb;
7418 1490579 : bitmap_and (&live_regs, &check_only_regs, df_get_live_out (curr_bb));
7419 : }
7420 36835496 : if (! NONDEBUG_INSN_P (curr_insn))
7421 14337277 : continue;
7422 22498219 : curr_id = lra_get_insn_recog_data (curr_insn);
7423 22498219 : curr_static_id = curr_id->insn_static_data;
7424 22498219 : remove_p = false;
7425 22498219 : if ((set = single_set (curr_insn)) != NULL_RTX
7426 21796080 : && REG_P (SET_DEST (set))
7427 17402406 : && (regno = REGNO (SET_DEST (set))) >= FIRST_PSEUDO_REGISTER
7428 12912472 : && SET_DEST (set) != pic_offset_table_rtx
7429 12905646 : && bitmap_bit_p (&check_only_regs, regno)
7430 25772148 : && ! bitmap_bit_p (&live_regs, regno))
7431 : remove_p = true;
7432 : /* See which defined values die here. */
7433 62094760 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7434 39596541 : if (reg->type == OP_OUT && ! reg->subreg_p)
7435 15498957 : bitmap_clear_bit (&live_regs, reg->regno);
7436 26741689 : for (reg = curr_static_id->hard_regs; reg != NULL; reg = reg->next)
7437 4243470 : if (reg->type == OP_OUT && ! reg->subreg_p)
7438 3222886 : bitmap_clear_bit (&live_regs, reg->regno);
7439 22498219 : if (curr_id->arg_hard_regs != NULL)
7440 : /* Make clobbered argument hard registers die. */
7441 3271481 : for (i = 0; (regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7442 2286202 : if (regno >= FIRST_PSEUDO_REGISTER)
7443 0 : bitmap_clear_bit (&live_regs, regno - FIRST_PSEUDO_REGISTER);
7444 : /* Mark each used value as live. */
7445 62094760 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7446 39596541 : if (reg->type != OP_OUT
7447 39596541 : && bitmap_bit_p (&check_only_regs, reg->regno))
7448 4600934 : bitmap_set_bit (&live_regs, reg->regno);
7449 26741689 : for (reg = curr_static_id->hard_regs; reg != NULL; reg = reg->next)
7450 4243470 : if (reg->type != OP_OUT
7451 4243470 : && bitmap_bit_p (&check_only_regs, reg->regno))
7452 0 : bitmap_set_bit (&live_regs, reg->regno);
7453 22498219 : if (curr_id->arg_hard_regs != NULL)
7454 : /* Make used argument hard registers live. */
7455 3271481 : for (i = 0; (regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7456 2286202 : if (regno < FIRST_PSEUDO_REGISTER
7457 2286202 : && bitmap_bit_p (&check_only_regs, regno))
7458 0 : bitmap_set_bit (&live_regs, regno);
7459 : /* It is quite important to remove dead move insns because it
7460 : means removing dead store. We don't need to process them for
7461 : constraints. */
7462 22498219 : if (remove_p)
7463 : {
7464 291634 : if (lra_dump_file != NULL)
7465 : {
7466 1 : fprintf (lra_dump_file, " Removing dead insn:\n ");
7467 1 : dump_insn_slim (lra_dump_file, curr_insn);
7468 : }
7469 291634 : lra_set_insn_deleted (curr_insn);
7470 : }
7471 : }
7472 745513 : }
7473 :
7474 : /* The structure describes info to do an inheritance for the current
7475 : insn. We need to collect such info first before doing the
7476 : transformations because the transformations change the insn
7477 : internal representation. */
7478 : struct to_inherit
7479 : {
7480 : /* Original regno. */
7481 : int regno;
7482 : /* Subsequent insns which can inherit original reg value. */
7483 : rtx_insn_list *insns;
7484 : };
7485 :
7486 : /* Array containing all info for doing inheritance from the current
7487 : insn. */
7488 : static struct to_inherit to_inherit[LRA_MAX_INSN_RELOADS];
7489 :
7490 : /* Number elements in the previous array. */
7491 : static int to_inherit_num;
7492 :
7493 : /* Add inheritance info REGNO and INSNS. Their meaning is described in
7494 : structure to_inherit. */
7495 : static void
7496 325921 : add_to_inherit (int regno, rtx_insn_list *insns)
7497 : {
7498 325921 : int i;
7499 :
7500 326025 : for (i = 0; i < to_inherit_num; i++)
7501 104 : if (to_inherit[i].regno == regno)
7502 : return;
7503 325921 : lra_assert (to_inherit_num < LRA_MAX_INSN_RELOADS);
7504 325921 : to_inherit[to_inherit_num].regno = regno;
7505 325921 : to_inherit[to_inherit_num++].insns = insns;
7506 : }
7507 :
7508 : /* Return the last non-debug insn in basic block BB, or the block begin
7509 : note if none. */
7510 : static rtx_insn *
7511 30567100 : get_last_insertion_point (basic_block bb)
7512 : {
7513 30567100 : rtx_insn *insn;
7514 :
7515 33033844 : FOR_BB_INSNS_REVERSE (bb, insn)
7516 2466744 : if (NONDEBUG_INSN_P (insn) || NOTE_INSN_BASIC_BLOCK_P (insn))
7517 30567100 : return insn;
7518 0 : gcc_unreachable ();
7519 : }
7520 :
7521 : /* Set up RES by registers living on edges FROM except the edge (FROM,
7522 : TO) or by registers set up in a jump insn in BB FROM. */
7523 : static void
7524 11801004 : get_live_on_other_edges (basic_block from, basic_block to, bitmap res)
7525 : {
7526 11801004 : rtx_insn *last;
7527 11801004 : struct lra_insn_reg *reg;
7528 11801004 : edge e;
7529 11801004 : edge_iterator ei;
7530 :
7531 11801004 : lra_assert (to != NULL);
7532 11801004 : bitmap_clear (res);
7533 35131302 : FOR_EACH_EDGE (e, ei, from->succs)
7534 23330298 : if (e->dest != to)
7535 11529294 : bitmap_ior_into (res, df_get_live_in (e->dest));
7536 11801004 : last = get_last_insertion_point (from);
7537 11801004 : if (! JUMP_P (last))
7538 1972625 : return;
7539 9828379 : curr_id = lra_get_insn_recog_data (last);
7540 19656454 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7541 9828075 : if (reg->type != OP_IN)
7542 78 : bitmap_set_bit (res, reg->regno);
7543 : }
7544 :
7545 : /* Used as a temporary results of some bitmap calculations. */
7546 : static bitmap_head temp_bitmap;
7547 :
7548 : /* We split for reloads of small class of hard regs. The following
7549 : defines how many hard regs the class should have to be qualified as
7550 : small. The code is mostly oriented to x86/x86-64 architecture
7551 : where some insns need to use only specific register or pair of
7552 : registers and these register can live in RTL explicitly, e.g. for
7553 : parameter passing. */
7554 : static const int max_small_class_regs_num = 2;
7555 :
7556 : /* Do inheritance/split transformations in EBB starting with HEAD and
7557 : finishing on TAIL. We process EBB insns in the reverse order.
7558 : Return true if we did any inheritance/split transformation in the
7559 : EBB.
7560 :
7561 : We should avoid excessive splitting which results in worse code
7562 : because of inaccurate cost calculations for spilling new split
7563 : pseudos in such case. To achieve this we do splitting only if
7564 : register pressure is high in given basic block and there are reload
7565 : pseudos requiring hard registers. We could do more register
7566 : pressure calculations at any given program point to avoid necessary
7567 : splitting even more but it is to expensive and the current approach
7568 : works well enough. */
7569 : static bool
7570 12865729 : inherit_in_ebb (rtx_insn *head, rtx_insn *tail)
7571 : {
7572 12865729 : int i, src_regno, dst_regno, nregs;
7573 12865729 : bool change_p, succ_p, update_reloads_num_p;
7574 12865729 : rtx_insn *prev_insn, *last_insn;
7575 12865729 : rtx_insn_list *next_usage_insns;
7576 12865729 : rtx curr_set;
7577 12865729 : enum reg_class cl;
7578 12865729 : struct lra_insn_reg *reg;
7579 12865729 : basic_block last_processed_bb, curr_bb = NULL;
7580 12865729 : HARD_REG_SET potential_reload_hard_regs, live_hard_regs;
7581 12865729 : bitmap to_process;
7582 12865729 : unsigned int j;
7583 12865729 : bitmap_iterator bi;
7584 12865729 : bool head_p, after_p;
7585 :
7586 12865729 : change_p = false;
7587 12865729 : curr_usage_insns_check++;
7588 12865729 : clear_invariants ();
7589 12865729 : reloads_num = calls_num = 0;
7590 167254477 : for (unsigned int i = 0; i < NUM_ABI_IDS; ++i)
7591 154388748 : last_call_for_abi[i] = 0;
7592 12865729 : CLEAR_HARD_REG_SET (full_and_partial_call_clobbers);
7593 12865729 : bitmap_clear (&check_only_regs);
7594 12865729 : bitmap_clear (&invalid_invariant_regs);
7595 12865729 : last_processed_bb = NULL;
7596 12865729 : CLEAR_HARD_REG_SET (potential_reload_hard_regs);
7597 12865729 : live_hard_regs = eliminable_regset | lra_no_alloc_regs;
7598 : /* We don't process new insns generated in the loop. */
7599 244458284 : for (curr_insn = tail; curr_insn != PREV_INSN (head); curr_insn = prev_insn)
7600 : {
7601 231592555 : prev_insn = PREV_INSN (curr_insn);
7602 231592555 : if (BLOCK_FOR_INSN (curr_insn) != NULL)
7603 231592285 : curr_bb = BLOCK_FOR_INSN (curr_insn);
7604 231592555 : if (last_processed_bb != curr_bb)
7605 : {
7606 : /* We are at the end of BB. Add qualified living
7607 : pseudos for potential splitting. */
7608 18766096 : to_process = df_get_live_out (curr_bb);
7609 18766096 : if (last_processed_bb != NULL)
7610 : {
7611 : /* We are somewhere in the middle of EBB. */
7612 5900367 : get_live_on_other_edges (curr_bb, last_processed_bb,
7613 : &temp_bitmap);
7614 5900367 : to_process = &temp_bitmap;
7615 : }
7616 18766096 : last_processed_bb = curr_bb;
7617 18766096 : last_insn = get_last_insertion_point (curr_bb);
7618 18766096 : auto call = dyn_cast<rtx_call_insn *> (last_insn);
7619 37532192 : after_p = (! JUMP_P (last_insn)
7620 18766096 : && (! call
7621 2392677 : || (find_reg_note (call,
7622 : REG_NORETURN, NULL_RTX) == NULL_RTX
7623 1425490 : && ! SIBLING_CALL_P (call))));
7624 18766096 : CLEAR_HARD_REG_SET (potential_reload_hard_regs);
7625 203204487 : EXECUTE_IF_SET_IN_BITMAP (to_process, 0, j, bi)
7626 : {
7627 184438398 : if ((int) j >= lra_constraint_new_regno_start)
7628 : break;
7629 184438391 : if (j < FIRST_PSEUDO_REGISTER || reg_renumber[j] >= 0)
7630 : {
7631 120258011 : if (j < FIRST_PSEUDO_REGISTER)
7632 70451783 : SET_HARD_REG_BIT (live_hard_regs, j);
7633 : else
7634 49806228 : add_to_hard_reg_set (&live_hard_regs,
7635 49806228 : PSEUDO_REGNO_MODE (j),
7636 49806228 : reg_renumber[j]);
7637 120258011 : setup_next_usage_insn (j, last_insn, reloads_num, after_p);
7638 : }
7639 : }
7640 : }
7641 231592555 : src_regno = dst_regno = -1;
7642 231592555 : curr_set = single_set (curr_insn);
7643 231592555 : if (curr_set != NULL_RTX && REG_P (SET_DEST (curr_set)))
7644 85721422 : dst_regno = REGNO (SET_DEST (curr_set));
7645 116070419 : if (curr_set != NULL_RTX && REG_P (SET_SRC (curr_set)))
7646 40237138 : src_regno = REGNO (SET_SRC (curr_set));
7647 231592555 : update_reloads_num_p = true;
7648 231592555 : if (src_regno < lra_constraint_new_regno_start
7649 224999337 : && src_regno >= FIRST_PSEUDO_REGISTER
7650 28315729 : && reg_renumber[src_regno] < 0
7651 3746128 : && dst_regno >= lra_constraint_new_regno_start
7652 234227874 : && (cl = lra_get_allocno_class (dst_regno)) != NO_REGS)
7653 : {
7654 : /* 'reload_pseudo <- original_pseudo'. */
7655 2635319 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7656 28739 : reloads_num++;
7657 2635319 : update_reloads_num_p = false;
7658 2635319 : succ_p = false;
7659 2635319 : if (usage_insns[src_regno].check == curr_usage_insns_check
7660 2635319 : && (next_usage_insns = usage_insns[src_regno].insns) != NULL_RTX)
7661 467688 : succ_p = inherit_reload_reg (false, src_regno, cl,
7662 : curr_insn, next_usage_insns);
7663 467688 : if (succ_p)
7664 : change_p = true;
7665 : else
7666 2184642 : setup_next_usage_insn (src_regno, curr_insn, reloads_num, false);
7667 5270638 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7668 652109004 : potential_reload_hard_regs |= reg_class_contents[cl];
7669 : }
7670 228957236 : else if (src_regno < 0
7671 191355417 : && dst_regno >= lra_constraint_new_regno_start
7672 5804662 : && invariant_p (SET_SRC (curr_set))
7673 304098 : && (cl = lra_get_allocno_class (dst_regno)) != NO_REGS
7674 303561 : && ! bitmap_bit_p (&invalid_invariant_regs, dst_regno)
7675 229219413 : && ! bitmap_bit_p (&invalid_invariant_regs,
7676 262177 : ORIGINAL_REGNO(regno_reg_rtx[dst_regno])))
7677 : {
7678 : /* 'reload_pseudo <- invariant'. */
7679 202248 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7680 11159 : reloads_num++;
7681 202248 : update_reloads_num_p = false;
7682 202248 : if (process_invariant_for_inheritance (SET_DEST (curr_set), SET_SRC (curr_set)))
7683 30185 : change_p = true;
7684 404496 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7685 652109004 : potential_reload_hard_regs |= reg_class_contents[cl];
7686 : }
7687 228754988 : else if (src_regno >= lra_constraint_new_regno_start
7688 6593218 : && dst_regno < lra_constraint_new_regno_start
7689 5787068 : && dst_regno >= FIRST_PSEUDO_REGISTER
7690 3829593 : && reg_renumber[dst_regno] < 0
7691 1468079 : && (cl = lra_get_allocno_class (src_regno)) != NO_REGS
7692 1468079 : && usage_insns[dst_regno].check == curr_usage_insns_check
7693 228754988 : && (next_usage_insns
7694 481594 : = usage_insns[dst_regno].insns) != NULL_RTX)
7695 : {
7696 481594 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7697 10089 : reloads_num++;
7698 481594 : update_reloads_num_p = false;
7699 : /* 'original_pseudo <- reload_pseudo'. */
7700 481594 : if (! JUMP_P (curr_insn)
7701 481594 : && inherit_reload_reg (true, dst_regno, cl,
7702 : curr_insn, next_usage_insns))
7703 : change_p = true;
7704 : /* Invalidate. */
7705 481594 : usage_insns[dst_regno].check = 0;
7706 963188 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7707 652109004 : potential_reload_hard_regs |= reg_class_contents[cl];
7708 : }
7709 228273394 : else if (INSN_P (curr_insn))
7710 : {
7711 192168201 : int iter;
7712 192168201 : int max_uid = get_max_uid ();
7713 :
7714 192168201 : curr_id = lra_get_insn_recog_data (curr_insn);
7715 192168201 : curr_static_id = curr_id->insn_static_data;
7716 192168201 : to_inherit_num = 0;
7717 : /* Process insn definitions. */
7718 576504603 : for (iter = 0; iter < 2; iter++)
7719 384336402 : for (reg = iter == 0 ? curr_id->regs : curr_static_id->hard_regs;
7720 616928724 : reg != NULL;
7721 232592322 : reg = reg->next)
7722 232592322 : if (reg->type != OP_IN
7723 232592322 : && (dst_regno = reg->regno) < lra_constraint_new_regno_start)
7724 : {
7725 46782761 : if (dst_regno >= FIRST_PSEUDO_REGISTER && reg->type == OP_OUT
7726 44550926 : && reg_renumber[dst_regno] < 0 && ! reg->subreg_p
7727 1811986 : && usage_insns[dst_regno].check == curr_usage_insns_check
7728 92914298 : && (next_usage_insns
7729 130656 : = usage_insns[dst_regno].insns) != NULL_RTX)
7730 : {
7731 130656 : struct lra_insn_reg *r;
7732 :
7733 389833 : for (r = curr_id->regs; r != NULL; r = r->next)
7734 259177 : if (r->type != OP_OUT && r->regno == dst_regno)
7735 : break;
7736 : /* Don't do inheritance if the pseudo is also
7737 : used in the insn. */
7738 130656 : if (r == NULL)
7739 : /* We cannot do inheritance right now
7740 : because the current insn reg info (chain
7741 : regs) can change after that. */
7742 130656 : add_to_inherit (dst_regno, next_usage_insns);
7743 : }
7744 : /* We cannot process one reg twice here because of
7745 : usage_insns invalidation. */
7746 92914298 : if ((dst_regno < FIRST_PSEUDO_REGISTER
7747 46782761 : || reg_renumber[dst_regno] >= 0)
7748 90964710 : && ! reg->subreg_p && reg->type != OP_IN)
7749 : {
7750 90676641 : HARD_REG_SET s;
7751 :
7752 90676641 : if (split_if_necessary (dst_regno, reg->biggest_mode,
7753 : potential_reload_hard_regs,
7754 : false, curr_insn, max_uid))
7755 58235 : change_p = true;
7756 90676641 : CLEAR_HARD_REG_SET (s);
7757 90676641 : if (dst_regno < FIRST_PSEUDO_REGISTER)
7758 46131537 : add_to_hard_reg_set (&s, reg->biggest_mode, dst_regno);
7759 : else
7760 44545104 : add_to_hard_reg_set (&s, PSEUDO_REGNO_MODE (dst_regno),
7761 44545104 : reg_renumber[dst_regno]);
7762 90676641 : live_hard_regs &= ~s;
7763 181353282 : potential_reload_hard_regs &= ~s;
7764 : }
7765 : /* We should invalidate potential inheritance or
7766 : splitting for the current insn usages to the next
7767 : usage insns (see code below) as the output pseudo
7768 : prevents this. */
7769 92914298 : if ((dst_regno >= FIRST_PSEUDO_REGISTER
7770 46782761 : && reg_renumber[dst_regno] < 0)
7771 90964710 : || (reg->type == OP_OUT && ! reg->subreg_p
7772 82850758 : && (dst_regno < FIRST_PSEUDO_REGISTER
7773 42321834 : || reg_renumber[dst_regno] >= 0)))
7774 : {
7775 : /* Invalidate and mark definitions. */
7776 44271422 : if (dst_regno >= FIRST_PSEUDO_REGISTER)
7777 44271422 : usage_insns[dst_regno].check = -(int) INSN_UID (curr_insn);
7778 : else
7779 : {
7780 40528924 : nregs = hard_regno_nregs (dst_regno,
7781 40528924 : reg->biggest_mode);
7782 81320483 : for (i = 0; i < nregs; i++)
7783 81583118 : usage_insns[dst_regno + i].check
7784 40791559 : = -(int) INSN_UID (curr_insn);
7785 : }
7786 : }
7787 : }
7788 : /* Process clobbered call regs. */
7789 192168201 : if (curr_id->arg_hard_regs != NULL)
7790 19734366 : for (i = 0; (dst_regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7791 13812375 : if (dst_regno >= FIRST_PSEUDO_REGISTER)
7792 0 : usage_insns[dst_regno - FIRST_PSEUDO_REGISTER].check
7793 0 : = -(int) INSN_UID (curr_insn);
7794 192168201 : if (! JUMP_P (curr_insn))
7795 180609405 : for (i = 0; i < to_inherit_num; i++)
7796 130656 : if (inherit_reload_reg (true, to_inherit[i].regno,
7797 : ALL_REGS, curr_insn,
7798 : to_inherit[i].insns))
7799 103719 : change_p = true;
7800 192168201 : if (auto call = dyn_cast<rtx_call_insn *> (curr_insn))
7801 : {
7802 7506722 : rtx cheap, pat, dest;
7803 7506722 : rtx_insn *restore;
7804 7506722 : int regno, hard_regno;
7805 :
7806 7506722 : calls_num++;
7807 7506722 : function_abi callee_abi = insn_callee_abi (call);
7808 7506722 : last_call_for_abi[callee_abi.id ()] = calls_num;
7809 7506722 : full_and_partial_call_clobbers
7810 7506722 : |= callee_abi.full_and_partial_reg_clobbers ();
7811 7506722 : first_call_insn = call;
7812 7506722 : if ((cheap = find_reg_note (call,
7813 : REG_RETURNED, NULL_RTX)) != NULL_RTX
7814 46174 : && ((cheap = XEXP (cheap, 0)), true)
7815 46174 : && (regno = REGNO (cheap)) >= FIRST_PSEUDO_REGISTER
7816 46174 : && (hard_regno = reg_renumber[regno]) >= 0
7817 34182 : && usage_insns[regno].check == curr_usage_insns_check
7818 : /* If there are pending saves/restores, the
7819 : optimization is not worth. */
7820 29240 : && usage_insns[regno].calls_num == calls_num - 1
7821 7534102 : && callee_abi.clobbers_reg_p (GET_MODE (cheap), hard_regno))
7822 : {
7823 : /* Restore the pseudo from the call result as
7824 : REG_RETURNED note says that the pseudo value is
7825 : in the call result and the pseudo is an argument
7826 : of the call. */
7827 12085 : pat = PATTERN (call);
7828 12085 : if (GET_CODE (pat) == PARALLEL)
7829 0 : pat = XVECEXP (pat, 0, 0);
7830 12085 : dest = SET_DEST (pat);
7831 : /* For multiple return values dest is PARALLEL.
7832 : Currently we handle only single return value case. */
7833 12085 : if (REG_P (dest))
7834 : {
7835 12085 : start_sequence ();
7836 12085 : emit_move_insn (cheap, copy_rtx (dest));
7837 12085 : restore = end_sequence ();
7838 12085 : lra_process_new_insns (call, NULL, restore,
7839 : "Inserting call parameter restore");
7840 : /* We don't need to save/restore of the pseudo from
7841 : this call. */
7842 12085 : usage_insns[regno].calls_num = calls_num;
7843 12085 : remove_from_hard_reg_set
7844 12085 : (&full_and_partial_call_clobbers,
7845 12085 : GET_MODE (cheap), hard_regno);
7846 12085 : bitmap_set_bit (&check_only_regs, regno);
7847 : }
7848 : }
7849 : }
7850 192168201 : to_inherit_num = 0;
7851 : /* Process insn usages. */
7852 576504603 : for (iter = 0; iter < 2; iter++)
7853 384336402 : for (reg = iter == 0 ? curr_id->regs : curr_static_id->hard_regs;
7854 616928724 : reg != NULL;
7855 232592322 : reg = reg->next)
7856 232592322 : if ((reg->type != OP_OUT
7857 91708606 : || (reg->type == OP_OUT && reg->subreg_p))
7858 233128559 : && (src_regno = reg->regno) < lra_constraint_new_regno_start)
7859 : {
7860 129668697 : if (src_regno >= FIRST_PSEUDO_REGISTER
7861 75071270 : && reg_renumber[src_regno] < 0 && reg->type == OP_IN)
7862 : {
7863 2562357 : if (usage_insns[src_regno].check == curr_usage_insns_check
7864 839385 : && (next_usage_insns
7865 839385 : = usage_insns[src_regno].insns) != NULL_RTX
7866 3401742 : && NONDEBUG_INSN_P (curr_insn))
7867 195265 : add_to_inherit (src_regno, next_usage_insns);
7868 4734184 : else if (usage_insns[src_regno].check
7869 2367092 : != -(int) INSN_UID (curr_insn))
7870 : /* Add usages but only if the reg is not set up
7871 : in the same insn. */
7872 2367092 : add_next_usage_insn (src_regno, curr_insn, reloads_num);
7873 : }
7874 72508913 : else if (src_regno < FIRST_PSEUDO_REGISTER
7875 72508913 : || reg_renumber[src_regno] >= 0)
7876 : {
7877 126968738 : bool before_p;
7878 126968738 : rtx_insn *use_insn = curr_insn;
7879 126968738 : rtx_insn *prev_insn = PREV_INSN (curr_insn);
7880 :
7881 253937476 : before_p = (JUMP_P (curr_insn)
7882 126968738 : || (CALL_P (curr_insn) && reg->type == OP_IN));
7883 126968738 : if (NONDEBUG_INSN_P (curr_insn)
7884 113073791 : && (! JUMP_P (curr_insn) || reg->type == OP_IN)
7885 240042424 : && split_if_necessary (src_regno, reg->biggest_mode,
7886 : potential_reload_hard_regs,
7887 : before_p, curr_insn, max_uid))
7888 : {
7889 223806 : if (reg->subreg_p)
7890 3124 : check_and_force_assignment_correctness_p = true;
7891 223806 : change_p = true;
7892 : /* Invalidate. */
7893 223806 : usage_insns[src_regno].check = 0;
7894 223806 : if (before_p && PREV_INSN (curr_insn) != prev_insn)
7895 : use_insn = PREV_INSN (curr_insn);
7896 : }
7897 126968738 : if (NONDEBUG_INSN_P (curr_insn))
7898 : {
7899 113073791 : if (src_regno < FIRST_PSEUDO_REGISTER)
7900 49653091 : add_to_hard_reg_set (&live_hard_regs,
7901 49653091 : reg->biggest_mode, src_regno);
7902 : else
7903 63420700 : add_to_hard_reg_set (&live_hard_regs,
7904 63420700 : PSEUDO_REGNO_MODE (src_regno),
7905 63420700 : reg_renumber[src_regno]);
7906 : }
7907 126968738 : if (src_regno >= FIRST_PSEUDO_REGISTER)
7908 72371311 : add_next_usage_insn (src_regno, use_insn, reloads_num);
7909 : else
7910 : {
7911 109294596 : for (i = 0; i < hard_regno_nregs (src_regno, reg->biggest_mode); i++)
7912 54697169 : add_next_usage_insn (src_regno + i, use_insn, reloads_num);
7913 : }
7914 : }
7915 : }
7916 : /* Process used call regs. */
7917 192168201 : if (curr_id->arg_hard_regs != NULL)
7918 19734366 : for (i = 0; (src_regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7919 13812375 : if (src_regno < FIRST_PSEUDO_REGISTER)
7920 : {
7921 13812375 : SET_HARD_REG_BIT (live_hard_regs, src_regno);
7922 13812375 : add_next_usage_insn (src_regno, curr_insn, reloads_num);
7923 : }
7924 192363466 : for (i = 0; i < to_inherit_num; i++)
7925 : {
7926 195265 : src_regno = to_inherit[i].regno;
7927 195265 : if (inherit_reload_reg (false, src_regno, ALL_REGS,
7928 : curr_insn, to_inherit[i].insns))
7929 : change_p = true;
7930 : else
7931 21499 : setup_next_usage_insn (src_regno, curr_insn, reloads_num, false);
7932 : }
7933 : }
7934 192243055 : if (update_reloads_num_p
7935 228273394 : && NONDEBUG_INSN_P (curr_insn) && curr_set != NULL_RTX)
7936 : {
7937 112751258 : int regno = -1;
7938 112751258 : if ((REG_P (SET_DEST (curr_set))
7939 82402261 : && (regno = REGNO (SET_DEST (curr_set))) >= lra_constraint_new_regno_start
7940 8482208 : && reg_renumber[regno] < 0
7941 5423362 : && (cl = lra_get_allocno_class (regno)) != NO_REGS)
7942 190006929 : || (REG_P (SET_SRC (curr_set))
7943 35793632 : && (regno = REGNO (SET_SRC (curr_set))) >= lra_constraint_new_regno_start
7944 6185038 : && reg_renumber[regno] < 0
7945 3651059 : && (cl = lra_get_allocno_class (regno)) != NO_REGS))
7946 : {
7947 8533453 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7948 260269 : reloads_num++;
7949 17066906 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7950 231592555 : potential_reload_hard_regs |= reg_class_contents[cl];
7951 : }
7952 : }
7953 231592555 : if (NONDEBUG_INSN_P (curr_insn))
7954 : {
7955 122530742 : int regno;
7956 :
7957 : /* Invalidate invariants with changed regs. */
7958 122530742 : curr_id = lra_get_insn_recog_data (curr_insn);
7959 314238742 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7960 191708000 : if (reg->type != OP_IN)
7961 : {
7962 82236831 : bitmap_set_bit (&invalid_invariant_regs, reg->regno);
7963 164473662 : bitmap_set_bit (&invalid_invariant_regs,
7964 82236831 : ORIGINAL_REGNO (regno_reg_rtx[reg->regno]));
7965 : }
7966 122530742 : curr_static_id = curr_id->insn_static_data;
7967 154661400 : for (reg = curr_static_id->hard_regs; reg != NULL; reg = reg->next)
7968 32130658 : if (reg->type != OP_IN)
7969 23010210 : bitmap_set_bit (&invalid_invariant_regs, reg->regno);
7970 122530742 : if (curr_id->arg_hard_regs != NULL)
7971 19734366 : for (i = 0; (regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7972 13812375 : if (regno >= FIRST_PSEUDO_REGISTER)
7973 0 : bitmap_set_bit (&invalid_invariant_regs,
7974 : regno - FIRST_PSEUDO_REGISTER);
7975 : }
7976 : /* We reached the start of the current basic block. */
7977 231592547 : if (prev_insn == NULL_RTX || prev_insn == PREV_INSN (head)
7978 450319381 : || BLOCK_FOR_INSN (prev_insn) != curr_bb)
7979 : {
7980 : /* We reached the beginning of the current block -- do
7981 : rest of splitting in the current BB. */
7982 18766366 : to_process = df_get_live_in (curr_bb);
7983 18766366 : if (BLOCK_FOR_INSN (head) != curr_bb)
7984 : {
7985 : /* We are somewhere in the middle of EBB. */
7986 5900637 : get_live_on_other_edges (EDGE_PRED (curr_bb, 0)->src,
7987 : curr_bb, &temp_bitmap);
7988 5900637 : to_process = &temp_bitmap;
7989 : }
7990 18766366 : head_p = true;
7991 196312103 : EXECUTE_IF_SET_IN_BITMAP (to_process, 0, j, bi)
7992 : {
7993 177545745 : if ((int) j >= lra_constraint_new_regno_start)
7994 : break;
7995 112979191 : if (((int) j < FIRST_PSEUDO_REGISTER || reg_renumber[j] >= 0)
7996 112263156 : && usage_insns[j].check == curr_usage_insns_check
7997 287878720 : && (next_usage_insns = usage_insns[j].insns) != NULL_RTX)
7998 : {
7999 110332983 : if (need_for_split_p (potential_reload_hard_regs, j))
8000 : {
8001 385433 : if (lra_dump_file != NULL && head_p)
8002 : {
8003 0 : fprintf (lra_dump_file,
8004 : " ----------------------------------\n");
8005 0 : head_p = false;
8006 : }
8007 385433 : if (split_reg (false, j, bb_note (curr_bb),
8008 : next_usage_insns, NULL))
8009 385433 : change_p = true;
8010 : }
8011 110332983 : usage_insns[j].check = 0;
8012 : }
8013 : }
8014 : }
8015 : }
8016 12865729 : first_call_insn = NULL;
8017 12865729 : return change_p;
8018 : }
8019 :
8020 : /* This value affects EBB forming. If probability of edge from EBB to
8021 : a BB is not greater than the following value, we don't add the BB
8022 : to EBB. */
8023 : #define EBB_PROBABILITY_CUTOFF \
8024 : ((REG_BR_PROB_BASE * param_lra_inheritance_ebb_probability_cutoff) / 100)
8025 :
8026 : /* Current number of inheritance/split iteration. */
8027 : int lra_inheritance_iter;
8028 :
8029 : /* Entry function for inheritance/split pass. */
8030 : void
8031 1583953 : lra_inheritance (void)
8032 : {
8033 1583953 : int i;
8034 1583953 : basic_block bb, start_bb;
8035 1583953 : edge e;
8036 :
8037 1583953 : lra_inheritance_iter++;
8038 1583953 : if (lra_inheritance_iter > LRA_MAX_INHERITANCE_PASSES)
8039 : return;
8040 1580625 : timevar_push (TV_LRA_INHERITANCE);
8041 1580625 : if (lra_dump_file != NULL)
8042 96 : fprintf (lra_dump_file, "\n********** Inheritance #%d: **********\n\n",
8043 : lra_inheritance_iter);
8044 1580625 : curr_usage_insns_check = 0;
8045 1580625 : usage_insns = XNEWVEC (struct usage_insns, lra_constraint_new_regno_start);
8046 238281195 : for (i = 0; i < lra_constraint_new_regno_start; i++)
8047 236700570 : usage_insns[i].check = 0;
8048 1580625 : bitmap_initialize (&check_only_regs, ®_obstack);
8049 1580625 : bitmap_initialize (&invalid_invariant_regs, ®_obstack);
8050 1580625 : bitmap_initialize (&live_regs, ®_obstack);
8051 1580625 : bitmap_initialize (&temp_bitmap, ®_obstack);
8052 1580625 : bitmap_initialize (&ebb_global_regs, ®_obstack);
8053 14446354 : FOR_EACH_BB_FN (bb, cfun)
8054 : {
8055 12865729 : start_bb = bb;
8056 12865729 : if (lra_dump_file != NULL)
8057 321 : fprintf (lra_dump_file, "EBB");
8058 : /* Form a EBB starting with BB. */
8059 12865729 : bitmap_clear (&ebb_global_regs);
8060 12865729 : bitmap_ior_into (&ebb_global_regs, df_get_live_in (bb));
8061 24666463 : for (;;)
8062 : {
8063 18766096 : if (lra_dump_file != NULL)
8064 435 : fprintf (lra_dump_file, " %d", bb->index);
8065 18766096 : if (bb->next_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
8066 17185471 : || LABEL_P (BB_HEAD (bb->next_bb)))
8067 : break;
8068 8304841 : e = find_fallthru_edge (bb->succs);
8069 8304841 : if (! e)
8070 : break;
8071 8304841 : if (e->probability.initialized_p ()
8072 8304841 : && e->probability.to_reg_br_prob_base () < EBB_PROBABILITY_CUTOFF)
8073 : break;
8074 5900367 : bb = bb->next_bb;
8075 : }
8076 12865729 : bitmap_ior_into (&ebb_global_regs, df_get_live_out (bb));
8077 12865729 : if (lra_dump_file != NULL)
8078 321 : fprintf (lra_dump_file, "\n");
8079 12865729 : if (inherit_in_ebb (BB_HEAD (start_bb), BB_END (bb)))
8080 : /* Remember that the EBB head and tail can change in
8081 : inherit_in_ebb. */
8082 745513 : update_ebb_live_info (BB_HEAD (start_bb), BB_END (bb));
8083 : }
8084 1580625 : bitmap_release (&ebb_global_regs);
8085 1580625 : bitmap_release (&temp_bitmap);
8086 1580625 : bitmap_release (&live_regs);
8087 1580625 : bitmap_release (&invalid_invariant_regs);
8088 1580625 : bitmap_release (&check_only_regs);
8089 1580625 : free (usage_insns);
8090 1580625 : lra_dump_insns_if_possible ("func after inheritance");
8091 1580625 : timevar_pop (TV_LRA_INHERITANCE);
8092 : }
8093 :
8094 :
8095 :
8096 : /* This page contains code to undo failed inheritance/split
8097 : transformations. */
8098 :
8099 : /* Current number of iteration undoing inheritance/split. */
8100 : int lra_undo_inheritance_iter;
8101 :
8102 : /* Fix BB live info LIVE after removing pseudos created on pass doing
8103 : inheritance/split which are REMOVED_PSEUDOS. */
8104 : static void
8105 37532192 : fix_bb_live_info (bitmap live, bitmap removed_pseudos)
8106 : {
8107 37532192 : unsigned int regno;
8108 37532192 : bitmap_iterator bi;
8109 :
8110 179882752 : EXECUTE_IF_SET_IN_BITMAP (removed_pseudos, 0, regno, bi)
8111 142350560 : if (bitmap_clear_bit (live, regno)
8112 142350560 : && REG_P (lra_reg_info[regno].restore_rtx))
8113 509870 : bitmap_set_bit (live, REGNO (lra_reg_info[regno].restore_rtx));
8114 37532192 : }
8115 :
8116 : /* Return regno of the (subreg of) REG. Otherwise, return a negative
8117 : number. */
8118 : static int
8119 70277539 : get_regno (rtx reg)
8120 : {
8121 1013033 : if (GET_CODE (reg) == SUBREG)
8122 1062119 : reg = SUBREG_REG (reg);
8123 70277539 : if (REG_P (reg))
8124 45462606 : return REGNO (reg);
8125 : return -1;
8126 : }
8127 :
8128 : /* Delete a move INSN with destination reg DREGNO and a previous
8129 : clobber insn with the same regno. The inheritance/split code can
8130 : generate moves with preceding clobber and when we delete such moves
8131 : we should delete the clobber insn too to keep the correct life
8132 : info. */
8133 : static void
8134 759627 : delete_move_and_clobber (rtx_insn *insn, int dregno)
8135 : {
8136 759627 : rtx_insn *prev_insn = PREV_INSN (insn);
8137 :
8138 759627 : lra_set_insn_deleted (insn);
8139 759627 : lra_assert (dregno >= 0);
8140 759627 : if (prev_insn != NULL && NONDEBUG_INSN_P (prev_insn)
8141 331791 : && GET_CODE (PATTERN (prev_insn)) == CLOBBER
8142 760131 : && dregno == get_regno (XEXP (PATTERN (prev_insn), 0)))
8143 0 : lra_set_insn_deleted (prev_insn);
8144 759627 : }
8145 :
8146 : /* Remove inheritance/split pseudos which are in REMOVE_PSEUDOS and
8147 : return true if we did any change. The undo transformations for
8148 : inheritance looks like
8149 : i <- i2
8150 : p <- i => p <- i2
8151 : or removing
8152 : p <- i, i <- p, and i <- i3
8153 : where p is original pseudo from which inheritance pseudo i was
8154 : created, i and i3 are removed inheritance pseudos, i2 is another
8155 : not removed inheritance pseudo. All split pseudos or other
8156 : occurrences of removed inheritance pseudos are changed on the
8157 : corresponding original pseudos.
8158 :
8159 : The function also schedules insns changed and created during
8160 : inheritance/split pass for processing by the subsequent constraint
8161 : pass. */
8162 : static bool
8163 1580625 : remove_inheritance_pseudos (bitmap remove_pseudos)
8164 : {
8165 1580625 : basic_block bb;
8166 1580625 : int regno, sregno, prev_sregno, dregno;
8167 1580625 : rtx restore_rtx;
8168 1580625 : rtx set, prev_set;
8169 1580625 : rtx_insn *prev_insn;
8170 1580625 : bool change_p, done_p;
8171 :
8172 1580625 : change_p = ! bitmap_empty_p (remove_pseudos);
8173 : /* We cannot finish the function right away if CHANGE_P is true
8174 : because we need to marks insns affected by previous
8175 : inheritance/split pass for processing by the subsequent
8176 : constraint pass. */
8177 20346721 : FOR_EACH_BB_FN (bb, cfun)
8178 : {
8179 18766096 : fix_bb_live_info (df_get_live_in (bb), remove_pseudos);
8180 18766096 : fix_bb_live_info (df_get_live_out (bb), remove_pseudos);
8181 252937597 : FOR_BB_INSNS_REVERSE (bb, curr_insn)
8182 : {
8183 234171501 : if (! INSN_P (curr_insn))
8184 36455483 : continue;
8185 197716018 : done_p = false;
8186 197716018 : sregno = dregno = -1;
8187 49363868 : if (change_p && NONDEBUG_INSN_P (curr_insn)
8188 230404570 : && (set = single_set (curr_insn)) != NULL_RTX)
8189 : {
8190 31637787 : dregno = get_regno (SET_DEST (set));
8191 63275574 : sregno = get_regno (SET_SRC (set));
8192 : }
8193 :
8194 197716018 : if (sregno >= 0 && dregno >= 0)
8195 : {
8196 11267601 : if (bitmap_bit_p (remove_pseudos, dregno)
8197 11267601 : && ! REG_P (lra_reg_info[dregno].restore_rtx))
8198 : {
8199 : /* invariant inheritance pseudo <- original pseudo */
8200 9370 : if (lra_dump_file != NULL)
8201 : {
8202 0 : fprintf (lra_dump_file, " Removing invariant inheritance:\n");
8203 0 : dump_insn_slim (lra_dump_file, curr_insn);
8204 0 : fprintf (lra_dump_file, "\n");
8205 : }
8206 9370 : delete_move_and_clobber (curr_insn, dregno);
8207 9370 : done_p = true;
8208 : }
8209 11258231 : else if (bitmap_bit_p (remove_pseudos, sregno)
8210 11258231 : && ! REG_P (lra_reg_info[sregno].restore_rtx))
8211 : {
8212 : /* reload pseudo <- invariant inheritance pseudo */
8213 9370 : start_sequence ();
8214 : /* We cannot just change the source. It might be
8215 : an insn different from the move. */
8216 9370 : emit_insn (lra_reg_info[sregno].restore_rtx);
8217 9370 : rtx_insn *new_insns = end_sequence ();
8218 9370 : lra_assert (single_set (new_insns) != NULL
8219 : && SET_DEST (set) == SET_DEST (single_set (new_insns)));
8220 9370 : lra_process_new_insns (curr_insn, NULL, new_insns,
8221 : "Changing reload<-invariant inheritance");
8222 9370 : delete_move_and_clobber (curr_insn, dregno);
8223 9370 : done_p = true;
8224 : }
8225 11248861 : else if ((bitmap_bit_p (remove_pseudos, sregno)
8226 1291372 : && (get_regno (lra_reg_info[sregno].restore_rtx) == dregno
8227 606519 : || (bitmap_bit_p (remove_pseudos, dregno)
8228 210618 : && get_regno (lra_reg_info[sregno].restore_rtx) >= 0
8229 210618 : && (get_regno (lra_reg_info[sregno].restore_rtx)
8230 210618 : == get_regno (lra_reg_info[dregno].restore_rtx)))))
8231 11750071 : || (bitmap_bit_p (remove_pseudos, dregno)
8232 681734 : && get_regno (lra_reg_info[dregno].restore_rtx) == sregno))
8233 : /* One of the following cases:
8234 : original <- removed inheritance pseudo
8235 : removed inherit pseudo <- another removed inherit pseudo
8236 : removed inherit pseudo <- original pseudo
8237 : Or
8238 : removed_split_pseudo <- original_reg
8239 : original_reg <- removed_split_pseudo */
8240 : {
8241 192781 : if (lra_dump_file != NULL)
8242 : {
8243 0 : fprintf (lra_dump_file, " Removing %s:\n",
8244 0 : bitmap_bit_p (&lra_split_regs, sregno)
8245 0 : || bitmap_bit_p (&lra_split_regs, dregno)
8246 : ? "split" : "inheritance");
8247 0 : dump_insn_slim (lra_dump_file, curr_insn);
8248 : }
8249 192781 : delete_move_and_clobber (curr_insn, dregno);
8250 192781 : done_p = true;
8251 : }
8252 11056080 : else if (bitmap_bit_p (remove_pseudos, sregno)
8253 11056080 : && bitmap_bit_p (&lra_inheritance_pseudos, sregno))
8254 : {
8255 : /* Search the following pattern:
8256 : inherit_or_split_pseudo1 <- inherit_or_split_pseudo2
8257 : original_pseudo <- inherit_or_split_pseudo1
8258 : where the 2nd insn is the current insn and
8259 : inherit_or_split_pseudo2 is not removed. If it is found,
8260 : change the current insn onto:
8261 : original_pseudo <- inherit_or_split_pseudo2. */
8262 781515 : for (prev_insn = PREV_INSN (curr_insn);
8263 781515 : prev_insn != NULL_RTX && ! NONDEBUG_INSN_P (prev_insn);
8264 280305 : prev_insn = PREV_INSN (prev_insn))
8265 : ;
8266 501210 : if (prev_insn != NULL_RTX && BLOCK_FOR_INSN (prev_insn) == bb
8267 488817 : && (prev_set = single_set (prev_insn)) != NULL_RTX
8268 : /* There should be no subregs in insn we are
8269 : searching because only the original reg might
8270 : be in subreg when we changed the mode of
8271 : load/store for splitting. */
8272 482682 : && REG_P (SET_DEST (prev_set))
8273 375093 : && REG_P (SET_SRC (prev_set))
8274 288625 : && (int) REGNO (SET_DEST (prev_set)) == sregno
8275 195181 : && ((prev_sregno = REGNO (SET_SRC (prev_set)))
8276 : >= FIRST_PSEUDO_REGISTER)
8277 195181 : && (lra_reg_info[prev_sregno].restore_rtx == NULL_RTX
8278 154209 : ||
8279 : /* As we consider chain of inheritance or
8280 : splitting described in above comment we should
8281 : check that sregno and prev_sregno were
8282 : inheritance/split pseudos created from the
8283 : same original regno. */
8284 308418 : (get_regno (lra_reg_info[sregno].restore_rtx) >= 0
8285 308418 : && (get_regno (lra_reg_info[sregno].restore_rtx)
8286 308418 : == get_regno (lra_reg_info[prev_sregno].restore_rtx))))
8287 696391 : && ! bitmap_bit_p (remove_pseudos, prev_sregno))
8288 : {
8289 104954 : int restore_regno = get_regno (lra_reg_info[sregno].restore_rtx);
8290 104954 : if (restore_regno < 0)
8291 0 : restore_regno = prev_sregno;
8292 104954 : lra_assert (GET_MODE (SET_SRC (prev_set))
8293 : == GET_MODE (regno_reg_rtx[restore_regno]));
8294 : /* Although we have a single set, the insn can
8295 : contain more one sregno register occurrence
8296 : as a source. Change all occurrences. */
8297 104954 : lra_substitute_pseudo_within_insn (curr_insn, sregno,
8298 : regno_reg_rtx[restore_regno],
8299 : false);
8300 : /* As we are finishing with processing the insn
8301 : here, check the destination too as it might
8302 : inheritance pseudo for another pseudo. */
8303 104954 : if (bitmap_bit_p (remove_pseudos, dregno)
8304 0 : && bitmap_bit_p (&lra_inheritance_pseudos, dregno)
8305 104954 : && (restore_rtx
8306 0 : = lra_reg_info[dregno].restore_rtx) != NULL_RTX)
8307 : {
8308 0 : if (GET_CODE (SET_DEST (set)) == SUBREG)
8309 0 : SUBREG_REG (SET_DEST (set)) = restore_rtx;
8310 : else
8311 0 : SET_DEST (set) = restore_rtx;
8312 : }
8313 104954 : lra_push_insn_and_update_insn_regno_info (curr_insn);
8314 104954 : lra_set_used_insn_alternative_by_uid
8315 104954 : (INSN_UID (curr_insn), LRA_UNKNOWN_ALT);
8316 104954 : done_p = true;
8317 104954 : if (lra_dump_file != NULL)
8318 : {
8319 0 : fprintf (lra_dump_file, " Change reload insn:\n");
8320 0 : dump_insn_slim (lra_dump_file, curr_insn);
8321 : }
8322 : }
8323 : }
8324 : }
8325 211521 : if (! done_p)
8326 : {
8327 197399543 : struct lra_insn_reg *reg;
8328 197399543 : bool restored_regs_p = false;
8329 197399543 : bool kept_regs_p = false;
8330 :
8331 197399543 : curr_id = lra_get_insn_recog_data (curr_insn);
8332 408171227 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
8333 : {
8334 210771684 : regno = reg->regno;
8335 210771684 : restore_rtx = lra_reg_info[regno].restore_rtx;
8336 210771684 : if (restore_rtx != NULL_RTX)
8337 : {
8338 5964398 : if (change_p && bitmap_bit_p (remove_pseudos, regno))
8339 : {
8340 888653 : lra_substitute_pseudo_within_insn
8341 888653 : (curr_insn, regno, restore_rtx, false);
8342 888653 : restored_regs_p = true;
8343 : }
8344 : else
8345 : kept_regs_p = true;
8346 : }
8347 : }
8348 197399543 : if (NONDEBUG_INSN_P (curr_insn) && kept_regs_p)
8349 : {
8350 : /* The instruction has changed since the previous
8351 : constraints pass. */
8352 4461865 : lra_push_insn_and_update_insn_regno_info (curr_insn);
8353 4461865 : lra_set_used_insn_alternative_by_uid
8354 4461865 : (INSN_UID (curr_insn), LRA_UNKNOWN_ALT);
8355 : }
8356 192937678 : else if (restored_regs_p)
8357 : /* The instruction has been restored to the form that
8358 : it had during the previous constraints pass. */
8359 712187 : lra_update_insn_regno_info (curr_insn);
8360 5174052 : if (restored_regs_p && lra_dump_file != NULL)
8361 : {
8362 0 : fprintf (lra_dump_file, " Insn after restoring regs:\n");
8363 0 : dump_insn_slim (lra_dump_file, curr_insn);
8364 : }
8365 : }
8366 : }
8367 : }
8368 1580625 : return change_p;
8369 : }
8370 :
8371 : /* Return true if REGNO is an inheritance pseudo. */
8372 : static bool
8373 130985 : inheritance_pseudo_p (int regno)
8374 : {
8375 130985 : return (regno >= new_regno_start
8376 130985 : && bitmap_bit_p (&lra_inheritance_pseudos, regno));
8377 : }
8378 :
8379 : /* If optional reload pseudos failed to get a hard register or was not
8380 : inherited, it is better to remove optional reloads. We do this
8381 : transformation after undoing inheritance to figure out necessity to
8382 : remove optional reloads easier. Return true if we do any
8383 : change. */
8384 : static bool
8385 1580625 : undo_optional_reloads (void)
8386 : {
8387 1580625 : bool change_p, keep_p;
8388 1580625 : unsigned int regno, uid;
8389 1580625 : bitmap_iterator bi, bi2;
8390 1580625 : rtx_insn *insn;
8391 1580625 : rtx set, src, dest;
8392 1580625 : auto_bitmap removed_optional_reload_pseudos (®_obstack);
8393 :
8394 1580625 : bitmap_copy (removed_optional_reload_pseudos, &lra_optional_reload_pseudos);
8395 2564044 : EXECUTE_IF_SET_IN_BITMAP (&lra_optional_reload_pseudos, 0, regno, bi)
8396 : {
8397 983419 : keep_p = false;
8398 : /* Keep optional reloads from previous subpasses. */
8399 983419 : if (lra_reg_info[regno].restore_rtx == NULL_RTX
8400 : /* If the original pseudo changed its allocation, just
8401 : removing the optional pseudo is dangerous as the original
8402 : pseudo will have longer live range. */
8403 983419 : || reg_renumber[REGNO (lra_reg_info[regno].restore_rtx)] >= 0)
8404 : keep_p = true;
8405 608421 : else if (reg_renumber[regno] >= 0)
8406 1732783 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi2)
8407 : {
8408 1268554 : insn = lra_insn_recog_data[uid]->insn;
8409 1268554 : if ((set = single_set (insn)) == NULL_RTX)
8410 6006 : continue;
8411 1262548 : int src_regno = get_regno (SET_SRC (set));
8412 1262548 : int dest_regno = get_regno (SET_DEST (set));
8413 1262548 : if (src_regno < 0 || dest_regno < 0)
8414 589208 : continue;
8415 673340 : int restore_regno = get_regno (lra_reg_info[regno].restore_rtx);
8416 673340 : if (restore_regno < 0)
8417 0 : continue;
8418 673340 : if (((dest_regno == (int) regno
8419 : /* Ignore insn for optional reloads itself. */
8420 577186 : && restore_regno != src_regno
8421 : /* Check that the optional reload was inherited. */
8422 115423 : && inheritance_pseudo_p (src_regno))
8423 1135103 : || (src_regno == (int) regno
8424 96154 : && restore_regno != dest_regno
8425 : /* The optional reload is an inheritance source. */
8426 15562 : && inheritance_pseudo_p (dest_regno))))
8427 : {
8428 : keep_p = true;
8429 : break;
8430 : }
8431 : }
8432 620873 : if (keep_p)
8433 : {
8434 505983 : bitmap_clear_bit (removed_optional_reload_pseudos, regno);
8435 505983 : if (lra_dump_file != NULL)
8436 2 : fprintf (lra_dump_file, "Keep optional reload reg %d\n", regno);
8437 : }
8438 : }
8439 1580625 : change_p = ! bitmap_empty_p (removed_optional_reload_pseudos);
8440 1580625 : auto_bitmap insn_bitmap (®_obstack);
8441 2058061 : EXECUTE_IF_SET_IN_BITMAP (removed_optional_reload_pseudos, 0, regno, bi)
8442 : {
8443 477436 : if (lra_dump_file != NULL)
8444 0 : fprintf (lra_dump_file, "Remove optional reload reg %d\n", regno);
8445 477436 : bitmap_copy (insn_bitmap, &lra_reg_info[regno].insn_bitmap);
8446 1503067 : EXECUTE_IF_SET_IN_BITMAP (insn_bitmap, 0, uid, bi2)
8447 : {
8448 : /* We may have already removed a clobber. */
8449 1025631 : if (!lra_insn_recog_data[uid])
8450 0 : continue;
8451 1025631 : insn = lra_insn_recog_data[uid]->insn;
8452 1025631 : if ((set = single_set (insn)) != NULL_RTX)
8453 : {
8454 1019835 : src = SET_SRC (set);
8455 1019835 : dest = SET_DEST (set);
8456 471621 : if ((REG_P (src) || SUBREG_P (src))
8457 548226 : && (REG_P (dest) || SUBREG_P (dest))
8458 1568030 : && ((get_regno (src) == (int) regno
8459 166800 : && (get_regno (lra_reg_info[regno].restore_rtx)
8460 83400 : == get_regno (dest)))
8461 464838 : || (get_regno (dest) == (int) regno
8462 464795 : && (get_regno (lra_reg_info[regno].restore_rtx)
8463 464795 : == get_regno (src)))))
8464 : {
8465 548106 : if (lra_dump_file != NULL)
8466 : {
8467 0 : fprintf (lra_dump_file, " Deleting move %u\n",
8468 0 : INSN_UID (insn));
8469 0 : dump_insn_slim (lra_dump_file, insn);
8470 : }
8471 1096212 : delete_move_and_clobber (insn, get_regno (dest));
8472 548106 : continue;
8473 : }
8474 : /* We should not worry about generation memory-memory
8475 : moves here as if the corresponding inheritance did
8476 : not work (inheritance pseudo did not get a hard reg),
8477 : we remove the inheritance pseudo and the optional
8478 : reload. */
8479 : }
8480 477525 : rtx pat = PATTERN (insn);
8481 0 : if (GET_CODE (pat) == CLOBBER && REG_P (SET_DEST (pat))
8482 477525 : && get_regno (SET_DEST (pat)) == (int) regno)
8483 : /* Refuse to remap clobbers to preexisting pseudos. */
8484 0 : gcc_unreachable ();
8485 477525 : lra_substitute_pseudo_within_insn
8486 477525 : (insn, regno, lra_reg_info[regno].restore_rtx, false);
8487 477525 : lra_update_insn_regno_info (insn);
8488 477525 : if (lra_dump_file != NULL)
8489 : {
8490 0 : fprintf (lra_dump_file,
8491 : " Restoring original insn:\n");
8492 0 : dump_insn_slim (lra_dump_file, insn);
8493 : }
8494 : }
8495 : }
8496 : /* Clear restore_regnos. */
8497 2564044 : EXECUTE_IF_SET_IN_BITMAP (&lra_optional_reload_pseudos, 0, regno, bi)
8498 983419 : lra_reg_info[regno].restore_rtx = NULL_RTX;
8499 1580625 : return change_p;
8500 1580625 : }
8501 :
8502 : /* Entry function for undoing inheritance/split transformation. Return true
8503 : if we did any RTL change in this pass. */
8504 : bool
8505 1583953 : lra_undo_inheritance (void)
8506 : {
8507 1583953 : unsigned int regno;
8508 1583953 : int hard_regno;
8509 1583953 : int n_all_inherit, n_inherit, n_all_split, n_split;
8510 1583953 : rtx restore_rtx;
8511 1583953 : bitmap_iterator bi;
8512 1583953 : bool change_p;
8513 :
8514 1583953 : lra_undo_inheritance_iter++;
8515 1583953 : if (lra_undo_inheritance_iter > LRA_MAX_INHERITANCE_PASSES)
8516 : return false;
8517 1580625 : if (lra_dump_file != NULL)
8518 96 : fprintf (lra_dump_file,
8519 : "\n********** Undoing inheritance #%d: **********\n\n",
8520 : lra_undo_inheritance_iter);
8521 1580625 : auto_bitmap remove_pseudos (®_obstack);
8522 1580625 : n_inherit = n_all_inherit = 0;
8523 3474979 : EXECUTE_IF_SET_IN_BITMAP (&lra_inheritance_pseudos, 0, regno, bi)
8524 1894354 : if (lra_reg_info[regno].restore_rtx != NULL_RTX)
8525 : {
8526 1201998 : n_all_inherit++;
8527 1201998 : if (reg_renumber[regno] < 0
8528 : /* If the original pseudo changed its allocation, just
8529 : removing inheritance is dangerous as for changing
8530 : allocation we used shorter live-ranges. */
8531 1201998 : && (! REG_P (lra_reg_info[regno].restore_rtx)
8532 447068 : || reg_renumber[REGNO (lra_reg_info[regno].restore_rtx)] < 0))
8533 456438 : bitmap_set_bit (remove_pseudos, regno);
8534 : else
8535 745560 : n_inherit++;
8536 : }
8537 1580625 : if (lra_dump_file != NULL && n_all_inherit != 0)
8538 1 : fprintf (lra_dump_file, "Inherit %d out of %d (%.2f%%)\n",
8539 : n_inherit, n_all_inherit,
8540 1 : (double) n_inherit / n_all_inherit * 100);
8541 1580625 : n_split = n_all_split = 0;
8542 2549749 : EXECUTE_IF_SET_IN_BITMAP (&lra_split_regs, 0, regno, bi)
8543 969124 : if ((restore_rtx = lra_reg_info[regno].restore_rtx) != NULL_RTX)
8544 : {
8545 667509 : int restore_regno = REGNO (restore_rtx);
8546 :
8547 667509 : n_all_split++;
8548 1334936 : hard_regno = (restore_regno >= FIRST_PSEUDO_REGISTER
8549 667509 : ? reg_renumber[restore_regno] : restore_regno);
8550 667509 : if (hard_regno < 0 || reg_renumber[regno] == hard_regno)
8551 2335 : bitmap_set_bit (remove_pseudos, regno);
8552 : else
8553 : {
8554 665174 : n_split++;
8555 665174 : if (lra_dump_file != NULL)
8556 0 : fprintf (lra_dump_file, " Keep split r%d (orig=r%d)\n",
8557 : regno, restore_regno);
8558 : }
8559 : }
8560 1580625 : if (lra_dump_file != NULL && n_all_split != 0)
8561 0 : fprintf (lra_dump_file, "Split %d out of %d (%.2f%%)\n",
8562 : n_split, n_all_split,
8563 0 : (double) n_split / n_all_split * 100);
8564 1580625 : change_p = remove_inheritance_pseudos (remove_pseudos);
8565 : /* Clear restore_regnos. */
8566 3474979 : EXECUTE_IF_SET_IN_BITMAP (&lra_inheritance_pseudos, 0, regno, bi)
8567 1894354 : lra_reg_info[regno].restore_rtx = NULL_RTX;
8568 2549749 : EXECUTE_IF_SET_IN_BITMAP (&lra_split_regs, 0, regno, bi)
8569 969124 : lra_reg_info[regno].restore_rtx = NULL_RTX;
8570 1580625 : change_p = undo_optional_reloads () || change_p;
8571 : if (change_p)
8572 111311 : lra_dump_insns_if_possible ("changed func after undoing inheritance");
8573 1580625 : return change_p;
8574 1580625 : }
|