Line data Source code
1 : /* Code for RTL transformations to satisfy insn constraints.
2 : Copyright (C) 2010-2026 Free Software Foundation, Inc.
3 : Contributed by Vladimir Makarov <vmakarov@redhat.com>.
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 :
22 : /* This file contains code for 3 passes: constraint pass,
23 : inheritance/split pass, and pass for undoing failed inheritance and
24 : split.
25 :
26 : The major goal of constraint pass is to transform RTL to satisfy
27 : insn and address constraints by:
28 : o choosing insn alternatives;
29 : o generating *reload insns* (or reloads in brief) and *reload
30 : pseudos* which will get necessary hard registers later;
31 : o substituting pseudos with equivalent values and removing the
32 : instructions that initialized those pseudos.
33 :
34 : The constraint pass has biggest and most complicated code in LRA.
35 : There are a lot of important details like:
36 : o reuse of input reload pseudos to simplify reload pseudo
37 : allocations;
38 : o some heuristics to choose insn alternative to improve the
39 : inheritance;
40 : o early clobbers etc.
41 :
42 : The pass is mimicking former reload pass in alternative choosing
43 : because the reload pass is oriented to current machine description
44 : model. It might be changed if the machine description model is
45 : changed.
46 :
47 : There is special code for preventing all LRA and this pass cycling
48 : in case of bugs.
49 :
50 : On the first iteration of the pass we process every instruction and
51 : choose an alternative for each one. On subsequent iterations we try
52 : to avoid reprocessing instructions if we can be sure that the old
53 : choice is still valid.
54 :
55 : The inheritance/spilt pass is to transform code to achieve
56 : ineheritance and live range splitting. It is done on backward
57 : traversal of EBBs.
58 :
59 : The inheritance optimization goal is to reuse values in hard
60 : registers. There is analogous optimization in old reload pass. The
61 : inheritance is achieved by following transformation:
62 :
63 : reload_p1 <- p reload_p1 <- p
64 : ... new_p <- reload_p1
65 : ... => ...
66 : reload_p2 <- p reload_p2 <- new_p
67 :
68 : where p is spilled and not changed between the insns. Reload_p1 is
69 : also called *original pseudo* and new_p is called *inheritance
70 : pseudo*.
71 :
72 : The subsequent assignment pass will try to assign the same (or
73 : another if it is not possible) hard register to new_p as to
74 : reload_p1 or reload_p2.
75 :
76 : If the assignment pass fails to assign a hard register to new_p,
77 : this file will undo the inheritance and restore the original code.
78 : This is because implementing the above sequence with a spilled
79 : new_p would make the code much worse. The inheritance is done in
80 : EBB scope. The above is just a simplified example to get an idea
81 : of the inheritance as the inheritance is also done for non-reload
82 : insns.
83 :
84 : Splitting (transformation) is also done in EBB scope on the same
85 : pass as the inheritance:
86 :
87 : r <- ... or ... <- r r <- ... or ... <- r
88 : ... s <- r (new insn -- save)
89 : ... =>
90 : ... r <- s (new insn -- restore)
91 : ... <- r ... <- r
92 :
93 : The *split pseudo* s is assigned to the hard register of the
94 : original pseudo or hard register r.
95 :
96 : Splitting is done:
97 : o In EBBs with high register pressure for global pseudos (living
98 : in at least 2 BBs) and assigned to hard registers when there
99 : are more one reloads needing the hard registers;
100 : o for pseudos needing save/restore code around calls.
101 :
102 : If the split pseudo still has the same hard register as the
103 : original pseudo after the subsequent assignment pass or the
104 : original pseudo was split, the opposite transformation is done on
105 : the same pass for undoing inheritance. */
106 :
107 : #undef REG_OK_STRICT
108 :
109 : #include "config.h"
110 : #include "system.h"
111 : #include "coretypes.h"
112 : #include "backend.h"
113 : #include "hooks.h"
114 : #include "target.h"
115 : #include "rtl.h"
116 : #include "tree.h"
117 : #include "stmt.h"
118 : #include "predict.h"
119 : #include "df.h"
120 : #include "memmodel.h"
121 : #include "tm_p.h"
122 : #include "expmed.h"
123 : #include "optabs.h"
124 : #include "regs.h"
125 : #include "ira.h"
126 : #include "ira-int.h"
127 : #include "recog.h"
128 : #include "output.h"
129 : #include "addresses.h"
130 : #include "expr.h"
131 : #include "cfgrtl.h"
132 : #include "rtl-error.h"
133 : #include "lra.h"
134 : #include "lra-int.h"
135 : #include "print-rtl.h"
136 : #include "function-abi.h"
137 : #include "rtl-iter.h"
138 : #include "hash-set.h"
139 :
140 : /* Value of LRA_CURR_RELOAD_NUM at the beginning of BB of the current
141 : insn. Remember that LRA_CURR_RELOAD_NUM is the number of emitted
142 : reload insns. */
143 : static int bb_reload_num;
144 :
145 : /* The current insn being processed and corresponding its single set
146 : (NULL otherwise), its data (basic block, the insn data, the insn
147 : static data, and the mode of each operand). */
148 : static rtx_insn *curr_insn;
149 : static rtx curr_insn_set;
150 : static basic_block curr_bb;
151 : static lra_insn_recog_data_t curr_id;
152 : static struct lra_static_insn_data *curr_static_id;
153 : static machine_mode curr_operand_mode[MAX_RECOG_OPERANDS];
154 : /* Mode of the register substituted by its equivalence with VOIDmode
155 : (e.g. constant) and whose subreg is given operand of the current
156 : insn. VOIDmode in all other cases. */
157 : static machine_mode original_subreg_reg_mode[MAX_RECOG_OPERANDS];
158 : /* The first call insn after curr_insn within the EBB during inherit_in_ebb
159 : or NULL outside of that function. */
160 : static rtx_insn *first_call_insn;
161 :
162 :
163 :
164 : /* Start numbers for new registers and insns at the current constraints
165 : pass start. */
166 : static int new_regno_start;
167 : static int new_insn_uid_start;
168 :
169 : /* If LOC is nonnull, strip any outer subreg from it. */
170 : static inline rtx *
171 233179872 : strip_subreg (rtx *loc)
172 : {
173 103135499 : return loc && GET_CODE (*loc) == SUBREG ? &SUBREG_REG (*loc) : loc;
174 : }
175 :
176 : /* Return hard regno of REGNO or if it is was not assigned to a hard
177 : register, use a hard register from its allocno class. */
178 : static int
179 91749 : get_try_hard_regno (int regno)
180 : {
181 91749 : int hard_regno;
182 91749 : enum reg_class rclass;
183 :
184 91749 : if ((hard_regno = regno) >= FIRST_PSEUDO_REGISTER)
185 91749 : hard_regno = lra_get_regno_hard_regno (regno);
186 91749 : if (hard_regno >= 0)
187 : return hard_regno;
188 57163 : rclass = lra_get_allocno_class (regno);
189 57163 : if (rclass == NO_REGS)
190 : return -1;
191 55875 : return ira_class_hard_regs[rclass][0];
192 : }
193 :
194 : /* Return the hard regno of X after removing its subreg. If X is not a
195 : register or a subreg of a register, return -1. If X is a pseudo, use its
196 : assignment. If X is a hard regno, return the final hard regno which will be
197 : after elimination. */
198 : static int
199 295830255 : get_hard_regno (rtx x)
200 : {
201 295830255 : rtx reg;
202 295830255 : int hard_regno;
203 :
204 295830255 : reg = x;
205 295830255 : if (SUBREG_P (x))
206 5375098 : reg = SUBREG_REG (x);
207 295830255 : if (! REG_P (reg))
208 : return -1;
209 204489723 : int regno = REGNO (reg);
210 204489723 : if (HARD_REGISTER_NUM_P (regno))
211 35629492 : hard_regno = lra_get_elimination_hard_regno (regno);
212 : else
213 168860231 : hard_regno = lra_get_regno_hard_regno (regno);
214 204489723 : if (hard_regno < 0)
215 : return -1;
216 186084232 : if (SUBREG_P (x))
217 4657640 : hard_regno += subreg_regno_offset (hard_regno, GET_MODE (reg),
218 4657640 : SUBREG_BYTE (x), GET_MODE (x));
219 : return hard_regno;
220 : }
221 :
222 : /* If REGNO is a hard register or has been allocated a hard register,
223 : return the class of that register. If REGNO is a reload pseudo
224 : created by the current constraints pass, return its allocno class.
225 : Return NO_REGS otherwise. */
226 : static enum reg_class
227 527120511 : get_reg_class (int regno)
228 : {
229 527120511 : int hard_regno;
230 :
231 527120511 : if (HARD_REGISTER_NUM_P (regno))
232 67234598 : hard_regno = lra_get_elimination_hard_regno (regno);
233 : else
234 459885913 : hard_regno = lra_get_regno_hard_regno (regno);
235 527120511 : if (hard_regno >= 0)
236 329845582 : return REGNO_REG_CLASS (hard_regno);
237 197274929 : if (regno >= new_regno_start)
238 63522025 : return lra_get_allocno_class (regno);
239 : return NO_REGS;
240 : }
241 :
242 : /* Return true if REG_CLASS has enough allocatable hard regs to keep value of
243 : REG_MODE. */
244 : static bool
245 19113131 : enough_allocatable_hard_regs_p (enum reg_class reg_class,
246 : enum machine_mode reg_mode)
247 : {
248 19113131 : int i, j, hard_regno, class_size, nregs;
249 :
250 38226262 : if (hard_reg_set_subset_p (reg_class_contents[reg_class], lra_no_alloc_regs))
251 : return false;
252 6596759 : class_size = ira_class_hard_regs_num[reg_class];
253 6596759 : for (i = 0; i < class_size; i++)
254 : {
255 6596759 : hard_regno = ira_class_hard_regs[reg_class][i];
256 6596759 : nregs = hard_regno_nregs (hard_regno, reg_mode);
257 6596759 : if (nregs == 1)
258 : return true;
259 288375 : for (j = 0; j < nregs; j++)
260 192250 : if (TEST_HARD_REG_BIT (lra_no_alloc_regs, hard_regno + j)
261 192250 : || ! TEST_HARD_REG_BIT (reg_class_contents[reg_class],
262 : hard_regno + j))
263 : break;
264 96125 : if (j >= nregs)
265 : return true;
266 : }
267 : return false;
268 : }
269 :
270 : /* True if C is a non-empty register class that has too few registers
271 : to be safely used as a reload target class. */
272 : #define SMALL_REGISTER_CLASS_P(C) \
273 : (ira_class_hard_regs_num [(C)] == 1 \
274 : || (ira_class_hard_regs_num [(C)] >= 1 \
275 : && targetm.class_likely_spilled_p (C)))
276 :
277 : /* Return true if REG satisfies (or will satisfy) reg class constraint
278 : CL. Use elimination first if REG is a hard register. If REG is a
279 : reload pseudo created by this constraints pass, assume that it will
280 : be allocated a hard register from its allocno class, but allow that
281 : class to be narrowed to CL if it is currently a superset of CL and
282 : if either:
283 :
284 : - ALLOW_ALL_RELOAD_CLASS_CHANGES_P is true or
285 : - the instruction we're processing is not a reload move.
286 :
287 : If NEW_CLASS is nonnull, set *NEW_CLASS to the new allocno class of
288 : REGNO (reg), or NO_REGS if no change in its class was needed. */
289 : static bool
290 224239341 : in_class_p (rtx reg, enum reg_class cl, enum reg_class *new_class,
291 : bool allow_all_reload_class_changes_p = false)
292 : {
293 224239341 : enum reg_class rclass, common_class;
294 224239341 : machine_mode reg_mode;
295 224239341 : rtx src;
296 224239341 : int regno = REGNO (reg);
297 :
298 224239341 : if (new_class != NULL)
299 115194654 : *new_class = NO_REGS;
300 224239341 : if (regno < FIRST_PSEUDO_REGISTER)
301 : {
302 28421495 : rtx final_reg = reg;
303 28421495 : rtx *final_loc = &final_reg;
304 :
305 28421495 : lra_eliminate_reg_if_possible (final_loc);
306 28421495 : return TEST_HARD_REG_BIT (reg_class_contents[cl], REGNO (*final_loc));
307 : }
308 195817846 : reg_mode = GET_MODE (reg);
309 195817846 : rclass = get_reg_class (regno);
310 195817846 : src = curr_insn_set != NULL ? SET_SRC (curr_insn_set) : NULL;
311 195817846 : if (regno < new_regno_start
312 : /* Do not allow the constraints for reload instructions to
313 : influence the classes of new pseudos. These reloads are
314 : typically moves that have many alternatives, and restricting
315 : reload pseudos for one alternative may lead to situations
316 : where other reload pseudos are no longer allocatable. */
317 195817846 : || (!allow_all_reload_class_changes_p
318 15339101 : && INSN_UID (curr_insn) >= new_insn_uid_start
319 14809734 : && src != NULL
320 14809734 : && ((REG_P (src) || MEM_P (src))
321 1484073 : || (GET_CODE (src) == SUBREG
322 686154 : && (REG_P (SUBREG_REG (src)) || MEM_P (SUBREG_REG (src)))))))
323 : /* When we don't know what class will be used finally for reload
324 : pseudos, we use ALL_REGS. */
325 14011815 : return ((regno >= new_regno_start && rclass == ALL_REGS)
326 190714123 : || (rclass != NO_REGS && ira_class_subset_p[rclass][cl]
327 205912478 : && ! hard_reg_set_subset_p (reg_class_contents[cl],
328 : lra_no_alloc_regs)));
329 : else
330 : {
331 19113131 : common_class = ira_reg_class_subset[rclass][cl];
332 19113131 : if (new_class != NULL)
333 5433115 : *new_class = common_class;
334 19113131 : return (enough_allocatable_hard_regs_p (common_class, reg_mode)
335 : /* Do not permit reload insn operand matching (new_class == NULL
336 : case) if the new class is too small. */
337 19113131 : && (new_class != NULL || common_class == rclass
338 1076165 : || !SMALL_REGISTER_CLASS_P (common_class)));
339 : }
340 : }
341 :
342 : /* Return true if REGNO satisfies a memory constraint. */
343 : static bool
344 65234840 : in_mem_p (int regno)
345 : {
346 0 : return get_reg_class (regno) == NO_REGS;
347 : }
348 :
349 : /* Return true if ADDR is a valid memory address for mode MODE in address
350 : space AS, and check that each pseudo has the proper kind of hard
351 : reg. */
352 : static bool
353 36278755 : valid_address_p (machine_mode mode ATTRIBUTE_UNUSED,
354 : rtx addr, addr_space_t as)
355 : {
356 : #ifdef GO_IF_LEGITIMATE_ADDRESS
357 : lra_assert (ADDR_SPACE_GENERIC_P (as));
358 : GO_IF_LEGITIMATE_ADDRESS (mode, addr, win);
359 : return false;
360 :
361 : win:
362 : return true;
363 : #else
364 0 : return targetm.addr_space.legitimate_address_p (mode, addr, 0, as,
365 36278755 : ERROR_MARK);
366 : #endif
367 : }
368 :
369 : namespace {
370 : /* Temporarily eliminates registers in an address (for the lifetime of
371 : the object). */
372 : class address_eliminator {
373 : public:
374 : address_eliminator (struct address_info *ad);
375 : ~address_eliminator ();
376 :
377 : private:
378 : struct address_info *m_ad;
379 : rtx *m_base_loc;
380 : rtx m_base_reg;
381 : rtx *m_index_loc;
382 : rtx m_index_reg;
383 : };
384 : }
385 :
386 76401109 : address_eliminator::address_eliminator (struct address_info *ad)
387 76401109 : : m_ad (ad),
388 76401109 : m_base_loc (strip_subreg (ad->base_term)),
389 76401109 : m_base_reg (NULL_RTX),
390 76401109 : m_index_loc (strip_subreg (ad->index_term)),
391 76401109 : m_index_reg (NULL_RTX)
392 : {
393 76401109 : if (m_base_loc != NULL)
394 : {
395 63809947 : m_base_reg = *m_base_loc;
396 : /* If we have non-legitimate address which is decomposed not in
397 : the way we expected, don't do elimination here. In such case
398 : the address will be reloaded and elimination will be done in
399 : reload insn finally. */
400 63809947 : if (REG_P (m_base_reg))
401 63809947 : lra_eliminate_reg_if_possible (m_base_loc);
402 63809947 : if (m_ad->base_term2 != NULL)
403 0 : *m_ad->base_term2 = *m_ad->base_term;
404 : }
405 76401109 : if (m_index_loc != NULL)
406 : {
407 3563739 : m_index_reg = *m_index_loc;
408 3563739 : if (REG_P (m_index_reg))
409 3563739 : lra_eliminate_reg_if_possible (m_index_loc);
410 : }
411 76401109 : }
412 :
413 76401109 : address_eliminator::~address_eliminator ()
414 : {
415 76401109 : if (m_base_loc && *m_base_loc != m_base_reg)
416 : {
417 45256859 : *m_base_loc = m_base_reg;
418 45256859 : if (m_ad->base_term2 != NULL)
419 0 : *m_ad->base_term2 = *m_ad->base_term;
420 : }
421 76401109 : if (m_index_loc && *m_index_loc != m_index_reg)
422 0 : *m_index_loc = m_index_reg;
423 76401109 : }
424 :
425 : /* Return true if the eliminated form of AD is a legitimate target address.
426 : If OP is a MEM, AD is the address within OP, otherwise OP should be
427 : ignored. CONSTRAINT is one constraint that the operand may need
428 : to meet. */
429 : static bool
430 36255771 : valid_address_p (rtx op, struct address_info *ad,
431 : enum constraint_num constraint)
432 : {
433 36255771 : address_eliminator eliminator (ad);
434 :
435 : /* Allow a memory OP if it matches CONSTRAINT, even if CONSTRAINT is more
436 : forgiving than "m".
437 : Need to extract memory from op for special memory constraint,
438 : i.e. bcst_mem_operand in i386 backend. */
439 36255771 : if (MEM_P (extract_mem_from_operand (op))
440 : && insn_extra_relaxed_memory_constraint (constraint)
441 : && constraint_satisfied_p (op, constraint))
442 : return true;
443 :
444 36255771 : return valid_address_p (ad->mode, *ad->outer, ad->as);
445 36255771 : }
446 :
447 : /* For special_memory_operand, it could be false for MEM_P (op),
448 : i.e. bcst_mem_operand in i386 backend.
449 : Extract and return real memory operand or op. */
450 : rtx
451 638959756 : extract_mem_from_operand (rtx op)
452 : {
453 640658381 : for (rtx x = op;; x = XEXP (x, 0))
454 : {
455 640658381 : if (MEM_P (x))
456 : return x;
457 456397441 : if (GET_RTX_LENGTH (GET_CODE (x)) != 1
458 373958548 : || GET_RTX_FORMAT (GET_CODE (x))[0] != 'e')
459 : break;
460 : }
461 : return op;
462 : }
463 :
464 : /* Return true if the eliminated form of memory reference OP satisfies
465 : extra (special) memory constraint CONSTRAINT. */
466 : static bool
467 37565590 : satisfies_memory_constraint_p (rtx op, enum constraint_num constraint)
468 : {
469 37565590 : struct address_info ad;
470 37565590 : rtx mem = extract_mem_from_operand (op);
471 37565590 : if (!MEM_P (mem))
472 : return false;
473 :
474 36522286 : decompose_mem_address (&ad, mem);
475 36522286 : address_eliminator eliminator (&ad);
476 36522286 : return constraint_satisfied_p (op, constraint);
477 36522286 : }
478 :
479 : /* Return true if the eliminated form of address AD satisfies extra
480 : address constraint CONSTRAINT. */
481 : static bool
482 3623052 : satisfies_address_constraint_p (struct address_info *ad,
483 : enum constraint_num constraint)
484 : {
485 3623052 : address_eliminator eliminator (ad);
486 3623052 : return constraint_satisfied_p (*ad->outer, constraint);
487 3623052 : }
488 :
489 : /* Return true if the eliminated form of address OP satisfies extra
490 : address constraint CONSTRAINT. */
491 : static bool
492 1780826 : satisfies_address_constraint_p (rtx op, enum constraint_num constraint)
493 : {
494 1780826 : struct address_info ad;
495 :
496 1780826 : decompose_lea_address (&ad, &op);
497 1780826 : return satisfies_address_constraint_p (&ad, constraint);
498 : }
499 :
500 : /* Set of equivalences whose original targets have set up pointer flag. */
501 : static hash_set <rtx> *pointer_equiv_set;
502 :
503 : /* Add x to pointer_equiv_set. */
504 : void
505 2027092 : lra_pointer_equiv_set_add (rtx x)
506 : {
507 2027092 : pointer_equiv_set->add (x);
508 2027092 : }
509 :
510 : /* Return true if x is in pointer_equiv_set. */
511 : bool
512 10199473 : lra_pointer_equiv_set_in (rtx x)
513 : {
514 10199473 : return pointer_equiv_set->contains (x);
515 : }
516 :
517 : /* Initiate equivalences for LRA. As we keep original equivalences
518 : before any elimination, we need to make copies otherwise any change
519 : in insns might change the equivalences. */
520 : void
521 1515121 : lra_init_equiv (void)
522 : {
523 1515121 : ira_expand_reg_equiv ();
524 73353019 : for (int i = FIRST_PSEUDO_REGISTER; i < max_reg_num (); i++)
525 : {
526 70322777 : rtx res;
527 :
528 70322777 : if ((res = ira_reg_equiv[i].memory) != NULL_RTX)
529 3043447 : ira_reg_equiv[i].memory = copy_rtx (res);
530 70322777 : if ((res = ira_reg_equiv[i].invariant) != NULL_RTX)
531 907729 : ira_reg_equiv[i].invariant = copy_rtx (res);
532 : }
533 1515121 : pointer_equiv_set = new hash_set <rtx>;
534 1515121 : }
535 :
536 : /* Finish equivalence data for LRA. */
537 : void
538 1515121 : lra_finish_equiv (void)
539 : {
540 3030242 : delete pointer_equiv_set;
541 1515121 : }
542 :
543 : static rtx loc_equivalence_callback (rtx, const_rtx, void *);
544 :
545 : /* Update equivalence for REGNO. We need to this as the equivalence
546 : might contain other pseudos which are changed by their
547 : equivalences. */
548 : static void
549 209083501 : update_equiv (int regno)
550 : {
551 209083501 : rtx x;
552 :
553 209083501 : if ((x = ira_reg_equiv[regno].memory) != NULL_RTX)
554 9242692 : ira_reg_equiv[regno].memory
555 9242692 : = simplify_replace_fn_rtx (x, NULL_RTX, loc_equivalence_callback,
556 : NULL_RTX);
557 209083501 : if ((x = ira_reg_equiv[regno].invariant) != NULL_RTX)
558 2821005 : ira_reg_equiv[regno].invariant
559 2821005 : = simplify_replace_fn_rtx (x, NULL_RTX, loc_equivalence_callback,
560 : NULL_RTX);
561 209083501 : }
562 :
563 : /* If we have decided to substitute X with another value, return that
564 : value, otherwise return X. */
565 : static rtx
566 447163674 : get_equiv (rtx x)
567 : {
568 447163674 : int regno;
569 447163674 : rtx res;
570 :
571 302714420 : if (! REG_P (x) || (regno = REGNO (x)) < FIRST_PSEUDO_REGISTER
572 200854796 : || regno >= ira_reg_equiv_len
573 200854796 : || ! ira_reg_equiv[regno].defined_p
574 26087130 : || ! ira_reg_equiv[regno].profitable_p
575 473208863 : || lra_get_regno_hard_regno (regno) >= 0)
576 : return x;
577 5204652 : if ((res = ira_reg_equiv[regno].memory) != NULL_RTX)
578 : {
579 2316058 : if (targetm.cannot_substitute_mem_equiv_p (res))
580 : return x;
581 2316058 : return res;
582 : }
583 2888594 : if ((res = ira_reg_equiv[regno].constant) != NULL_RTX)
584 : {
585 910947 : if (targetm.cannot_substitute_const_equiv_p (res))
586 : return x;
587 910947 : return res;
588 : }
589 1977647 : if ((res = ira_reg_equiv[regno].invariant) != NULL_RTX)
590 : return res;
591 0 : gcc_unreachable ();
592 : }
593 :
594 : /* If we have decided to substitute X with the equivalent value, return that
595 : value after elimination for INSN, otherwise return X. Add the result to
596 : pointer_equiv_set if X has set up pointer flag. */
597 : static rtx
598 251396335 : get_equiv_with_elimination (rtx x, rtx_insn *insn)
599 : {
600 251396335 : rtx res = get_equiv (x);
601 :
602 251396335 : if (x == res || CONSTANT_P (res))
603 : return res;
604 1576656 : res = lra_eliminate_regs_1 (insn, res, GET_MODE (res),
605 : false, false, 0, true);
606 1576656 : if (REG_POINTER (x))
607 1090707 : lra_pointer_equiv_set_add (res);
608 : return res;
609 : }
610 :
611 : /* Set up curr_operand_mode. */
612 : static void
613 108674477 : init_curr_operand_mode (void)
614 : {
615 108674477 : int nop = curr_static_id->n_operands;
616 338876679 : for (int i = 0; i < nop; i++)
617 : {
618 230202202 : machine_mode mode = GET_MODE (*curr_id->operand_loc[i]);
619 230202202 : if (mode == VOIDmode)
620 : {
621 : /* The .md mode for address operands is the mode of the
622 : addressed value rather than the mode of the address itself. */
623 44493940 : if (curr_id->icode >= 0 && curr_static_id->operand[i].is_address)
624 95 : mode = Pmode;
625 : else
626 44493845 : mode = curr_static_id->operand[i].mode;
627 : }
628 230202202 : curr_operand_mode[i] = mode;
629 : }
630 108674477 : }
631 :
632 :
633 :
634 : /* The page contains code to reuse input reloads. */
635 :
636 : /* Structure describes input reload of the current insns. */
637 : struct input_reload
638 : {
639 : /* True for input reload of matched operands. */
640 : bool match_p;
641 : /* True for input reload of inout earlyclobber operand. */
642 : bool early_clobber_p;
643 : /* Reloaded value. */
644 : rtx input;
645 : /* Reload pseudo used. */
646 : rtx reg;
647 : };
648 :
649 : /* The number of elements in the following array. */
650 : static int curr_insn_input_reloads_num;
651 : /* Array containing info about input reloads. It is used to find the
652 : same input reload and reuse the reload pseudo in this case. */
653 : static struct input_reload curr_insn_input_reloads[LRA_MAX_INSN_RELOADS];
654 :
655 : /* Initiate data concerning reuse of input reloads for the current
656 : insn. */
657 : static void
658 108674477 : init_curr_insn_input_reloads (void)
659 : {
660 108674477 : curr_insn_input_reloads_num = 0;
661 0 : }
662 :
663 : /* The canonical form of an rtx inside a MEM is not necessarily the same as the
664 : canonical form of the rtx outside the MEM. Fix this up in the case that
665 : we're reloading an address (and therefore pulling it outside a MEM). */
666 : static rtx
667 72 : canonicalize_reload_addr (rtx addr)
668 : {
669 72 : subrtx_var_iterator::array_type array;
670 246 : FOR_EACH_SUBRTX_VAR (iter, array, addr, NONCONST)
671 : {
672 174 : rtx x = *iter;
673 174 : if (GET_CODE (x) == MULT && CONST_INT_P (XEXP (x, 1)))
674 : {
675 14 : const HOST_WIDE_INT ci = INTVAL (XEXP (x, 1));
676 188 : const int pwr2 = exact_log2 (ci);
677 14 : if (pwr2 > 0)
678 : {
679 : /* Rewrite this to use a shift instead, which is canonical when
680 : outside of a MEM. */
681 14 : PUT_CODE (x, ASHIFT);
682 14 : XEXP (x, 1) = GEN_INT (pwr2);
683 : }
684 : }
685 : }
686 :
687 72 : return addr;
688 72 : }
689 :
690 : /* Return rtx accessing reload REG of RCLASS matching another reload reg in
691 : MODE. */
692 : static rtx
693 112576 : get_matching_reload_reg_subreg (machine_mode mode, rtx reg,
694 : enum reg_class rclass)
695 : {
696 112576 : int hard_regno = ira_class_hard_regs[rclass][0];
697 112576 : if (subreg_regno_offset (hard_regno,
698 112576 : GET_MODE (reg),
699 112576 : subreg_lowpart_offset (mode, GET_MODE (reg)),
700 : mode) == 0)
701 : /* For matching scalar int modes generate the right subreg byte offset for
702 : BE targets -- see call of reload.cc:operands_match_p in
703 : recog.cc:constrain_operands. */
704 112576 : return lowpart_subreg (mode, reg, GET_MODE (reg));
705 0 : int offset = (lra_constraint_offset (hard_regno, GET_MODE (reg))
706 0 : - lra_constraint_offset (hard_regno, mode)) * UNITS_PER_WORD;
707 0 : lra_assert (offset >= 0);
708 0 : return gen_rtx_SUBREG (mode, reg, offset);
709 : }
710 :
711 : /* Create a new pseudo using MODE, RCLASS, EXCLUDE_START_HARD_REGS, ORIGINAL or
712 : reuse an existing reload pseudo. Don't reuse an existing reload pseudo if
713 : IN_SUBREG_P is true and the reused pseudo should be wrapped up in a SUBREG.
714 : EARLY_CLOBBER_P is true for input reload of inout early clobber operand.
715 : The result pseudo is returned through RESULT_REG. Return TRUE if we created
716 : a new pseudo, FALSE if we reused an existing reload pseudo. Use TITLE to
717 : describe new registers for debug purposes. */
718 : static bool
719 3949259 : get_reload_reg (enum op_type type, machine_mode mode, rtx original,
720 : enum reg_class rclass, HARD_REG_SET *exclude_start_hard_regs,
721 : bool in_subreg_p, bool early_clobber_p,
722 : const char *title, rtx *result_reg)
723 : {
724 3949259 : int i, regno;
725 3949259 : enum reg_class new_class;
726 :
727 3949259 : if (type == OP_OUT)
728 : {
729 : /* Output reload registers tend to start out with a conservative
730 : choice of register class. Usually this is ALL_REGS, although
731 : a target might narrow it (for performance reasons) through
732 : targetm.preferred_reload_class. It's therefore quite common
733 : for a reload instruction to require a more restrictive class
734 : than the class that was originally assigned to the reload register.
735 :
736 : In these situations, it's more efficient to refine the choice
737 : of register class rather than create a second reload register.
738 : This also helps to avoid cycling for registers that are only
739 : used by reload instructions. */
740 991031 : if (REG_P (original)
741 720227 : && (int) REGNO (original) >= new_regno_start
742 7149 : && (INSN_UID (curr_insn) >= new_insn_uid_start
743 250 : || ira_former_scratch_p (REGNO (original)))
744 7149 : && in_class_p (original, rclass, &new_class, true)
745 991281 : && (exclude_start_hard_regs == nullptr
746 250 : || hard_reg_set_intersect_p (
747 991281 : ~lra_reg_info[REGNO (original)].exclude_start_hard_regs,
748 500 : ~*exclude_start_hard_regs)))
749 : {
750 250 : unsigned int regno = REGNO (original);
751 250 : if (lra_dump_file != NULL)
752 : {
753 0 : fprintf (lra_dump_file, " Reuse r%d for output ", regno);
754 0 : dump_value_slim (lra_dump_file, original, 1);
755 : }
756 500 : if (new_class != lra_get_allocno_class (regno))
757 250 : lra_change_class (regno, new_class, ", change to", false);
758 250 : if (lra_dump_file != NULL)
759 0 : fprintf (lra_dump_file, "\n");
760 250 : if (exclude_start_hard_regs)
761 250 : lra_reg_info[regno].exclude_start_hard_regs
762 250 : |= *exclude_start_hard_regs;
763 250 : *result_reg = original;
764 250 : return false;
765 : }
766 990781 : *result_reg
767 990781 : = lra_create_new_reg_with_unique_value (mode, original, rclass,
768 : exclude_start_hard_regs, title);
769 990781 : return true;
770 : }
771 :
772 2958228 : bool unique_p = early_clobber_p;
773 : /* Prevent reuse value of expression with side effects,
774 : e.g. volatile memory. */
775 2958228 : if (! side_effects_p (original))
776 3183770 : for (i = 0; i < curr_insn_input_reloads_num; i++)
777 : {
778 245004 : if (! curr_insn_input_reloads[i].match_p
779 105240 : && ! curr_insn_input_reloads[i].early_clobber_p
780 105239 : && rtx_equal_p (curr_insn_input_reloads[i].input, original)
781 254438 : && in_class_p (curr_insn_input_reloads[i].reg, rclass, &new_class))
782 : {
783 9421 : rtx reg = curr_insn_input_reloads[i].reg;
784 9421 : regno = REGNO (reg);
785 : /* If input is equal to original and both are VOIDmode,
786 : GET_MODE (reg) might be still different from mode.
787 : Ensure we don't return *result_reg with wrong mode. */
788 9421 : if (GET_MODE (reg) != mode)
789 : {
790 0 : if (in_subreg_p)
791 0 : continue;
792 0 : if (maybe_lt (GET_MODE_SIZE (GET_MODE (reg)),
793 0 : GET_MODE_SIZE (mode)))
794 0 : continue;
795 0 : reg = get_matching_reload_reg_subreg (mode, reg, new_class);
796 0 : if (reg == NULL_RTX || GET_CODE (reg) != SUBREG)
797 0 : continue;
798 : }
799 : /* If the existing reload and this have no start hard register in
800 : common, then skip. Otherwise update exclude_start_hard_regs. */
801 9421 : if (exclude_start_hard_regs
802 10873 : && ! hard_reg_set_empty_p (*exclude_start_hard_regs))
803 : {
804 1 : HARD_REG_SET r = lra_reg_info[regno].exclude_start_hard_regs
805 1 : | *exclude_start_hard_regs;
806 2 : if (hard_reg_set_empty_p (~r))
807 0 : continue;
808 : else
809 1 : lra_reg_info[regno].exclude_start_hard_regs = r;
810 : }
811 9421 : *result_reg = reg;
812 9421 : if (lra_dump_file != NULL)
813 : {
814 0 : fprintf (lra_dump_file, " Reuse r%d for reload ", regno);
815 0 : dump_value_slim (lra_dump_file, original, 1);
816 : }
817 18842 : if (new_class != lra_get_allocno_class (regno))
818 4556 : lra_change_class (regno, new_class, ", change to", false);
819 9421 : if (lra_dump_file != NULL)
820 0 : fprintf (lra_dump_file, "\n");
821 : return false;
822 : }
823 : /* If we have an input reload with a different mode, make sure it
824 : will get a different hard reg. */
825 235583 : else if (REG_P (original)
826 185846 : && REG_P (curr_insn_input_reloads[i].input)
827 155149 : && REGNO (original) == REGNO (curr_insn_input_reloads[i].input)
828 235583 : && (GET_MODE (original)
829 2306 : != GET_MODE (curr_insn_input_reloads[i].input)))
830 : unique_p = true;
831 : }
832 5897614 : *result_reg = (unique_p
833 2948807 : ? lra_create_new_reg_with_unique_value
834 2948807 : : lra_create_new_reg) (mode, original, rclass,
835 : exclude_start_hard_regs, title);
836 2948807 : lra_assert (curr_insn_input_reloads_num < LRA_MAX_INSN_RELOADS);
837 2948807 : curr_insn_input_reloads[curr_insn_input_reloads_num].input = original;
838 2948807 : curr_insn_input_reloads[curr_insn_input_reloads_num].match_p = false;
839 2948807 : curr_insn_input_reloads[curr_insn_input_reloads_num].early_clobber_p
840 2948807 : = early_clobber_p;
841 2948807 : curr_insn_input_reloads[curr_insn_input_reloads_num++].reg = *result_reg;
842 2948807 : return true;
843 : }
844 :
845 :
846 : /* The page contains major code to choose the current insn alternative
847 : and generate reloads for it. */
848 :
849 : /* Return the offset from REGNO of the least significant register
850 : in (reg:MODE REGNO).
851 :
852 : This function is used to tell whether two registers satisfy
853 : a matching constraint. (reg:MODE1 REGNO1) matches (reg:MODE2 REGNO2) if:
854 :
855 : REGNO1 + lra_constraint_offset (REGNO1, MODE1)
856 : == REGNO2 + lra_constraint_offset (REGNO2, MODE2) */
857 : int
858 44847012 : lra_constraint_offset (int regno, machine_mode mode)
859 : {
860 44847012 : lra_assert (regno < FIRST_PSEUDO_REGISTER);
861 :
862 44847012 : scalar_int_mode int_mode;
863 44847012 : if (WORDS_BIG_ENDIAN
864 : && is_a <scalar_int_mode> (mode, &int_mode)
865 : && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD)
866 : return hard_regno_nregs (regno, mode) - 1;
867 44847012 : return 0;
868 : }
869 :
870 : /* Like rtx_equal_p except that it allows a REG and a SUBREG to match
871 : if they are the same hard reg, and has special hacks for
872 : auto-increment and auto-decrement. This is specifically intended for
873 : process_alt_operands to use in determining whether two operands
874 : match. X is the operand whose number is the lower of the two.
875 :
876 : It is supposed that X is the output operand and Y is the input
877 : operand. Y_HARD_REGNO is the final hard regno of register Y or
878 : register in subreg Y as we know it now. Otherwise, it is a
879 : negative value. */
880 : static bool
881 59024220 : operands_match_p (rtx x, rtx y, int y_hard_regno)
882 : {
883 59024220 : int i;
884 59024220 : RTX_CODE code = GET_CODE (x);
885 59024220 : const char *fmt;
886 :
887 59024220 : if (x == y)
888 : return true;
889 51287736 : if ((code == REG || (code == SUBREG && REG_P (SUBREG_REG (x))))
890 24485872 : && (REG_P (y) || (GET_CODE (y) == SUBREG && REG_P (SUBREG_REG (y)))))
891 : {
892 24391943 : int j;
893 :
894 24391943 : i = get_hard_regno (x);
895 24391943 : if (i < 0)
896 1281056 : goto slow;
897 :
898 23110887 : if ((j = y_hard_regno) < 0)
899 687381 : goto slow;
900 :
901 22423506 : i += lra_constraint_offset (i, GET_MODE (x));
902 22423506 : j += lra_constraint_offset (j, GET_MODE (y));
903 :
904 22423506 : return i == j;
905 : }
906 :
907 : /* If two operands must match, because they are really a single
908 : operand of an assembler insn, then two post-increments are invalid
909 : because the assembler insn would increment only once. On the
910 : other hand, a post-increment matches ordinary indexing if the
911 : post-increment is the output operand. */
912 26895793 : if (code == POST_DEC || code == POST_INC || code == POST_MODIFY)
913 0 : return operands_match_p (XEXP (x, 0), y, y_hard_regno);
914 :
915 : /* Two pre-increments are invalid because the assembler insn would
916 : increment only once. On the other hand, a pre-increment matches
917 : ordinary indexing if the pre-increment is the input operand. */
918 26895793 : if (GET_CODE (y) == PRE_DEC || GET_CODE (y) == PRE_INC
919 26895793 : || GET_CODE (y) == PRE_MODIFY)
920 0 : return operands_match_p (x, XEXP (y, 0), -1);
921 :
922 26895793 : slow:
923 :
924 28864230 : if (code == REG && REG_P (y))
925 1871488 : return REGNO (x) == REGNO (y);
926 :
927 96224 : if (code == REG && GET_CODE (y) == SUBREG && REG_P (SUBREG_REG (y))
928 9122 : && x == SUBREG_REG (y))
929 : return true;
930 26992742 : if (GET_CODE (y) == REG && code == SUBREG && REG_P (SUBREG_REG (x))
931 64835 : && SUBREG_REG (x) == y)
932 : return true;
933 :
934 : /* Now we have disposed of all the cases in which different rtx
935 : codes can match. */
936 26992614 : if (code != GET_CODE (y))
937 : return false;
938 :
939 : /* (MULT:SI x y) and (MULT:HI x y) are NOT equivalent. */
940 1082091 : if (GET_MODE (x) != GET_MODE (y))
941 : return false;
942 :
943 1081344 : switch (code)
944 : {
945 : CASE_CONST_UNIQUE:
946 : return false;
947 :
948 : case CONST_VECTOR:
949 : if (!same_vector_encodings_p (x, y))
950 : return false;
951 : break;
952 :
953 0 : case LABEL_REF:
954 0 : return label_ref_label (x) == label_ref_label (y);
955 25 : case SYMBOL_REF:
956 25 : return XSTR (x, 0) == XSTR (y, 0);
957 :
958 : default:
959 : break;
960 : }
961 :
962 : /* Compare the elements. If any pair of corresponding elements fail
963 : to match, return false for the whole things. */
964 :
965 1055620 : fmt = GET_RTX_FORMAT (code);
966 3054581 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
967 : {
968 2075472 : int val, j;
969 2075472 : switch (fmt[i])
970 : {
971 0 : case 'w':
972 0 : if (XWINT (x, i) != XWINT (y, i))
973 : return false;
974 : break;
975 :
976 492 : case 'i':
977 492 : if (XINT (x, i) != XINT (y, i))
978 : return false;
979 : break;
980 :
981 0 : case 'L':
982 0 : if (XLOC (x, i) != XLOC (y, i))
983 : return false;
984 : break;
985 :
986 22904 : case 'p':
987 22904 : if (maybe_ne (SUBREG_BYTE (x), SUBREG_BYTE (y)))
988 : return false;
989 : break;
990 :
991 1508696 : case 'e':
992 1508696 : val = operands_match_p (XEXP (x, i), XEXP (y, i), -1);
993 1508696 : if (val == 0)
994 : return false;
995 : break;
996 :
997 : case '0':
998 : break;
999 :
1000 492 : case 'E':
1001 492 : if (XVECLEN (x, i) != XVECLEN (y, i))
1002 : return false;
1003 984 : for (j = XVECLEN (x, i) - 1; j >= 0; --j)
1004 : {
1005 492 : val = operands_match_p (XVECEXP (x, i, j), XVECEXP (y, i, j), -1);
1006 492 : if (val == 0)
1007 : return false;
1008 : }
1009 : break;
1010 :
1011 : /* It is believed that rtx's at this level will never
1012 : contain anything but integers and other rtx's, except for
1013 : within LABEL_REFs and SYMBOL_REFs. */
1014 0 : default:
1015 0 : gcc_unreachable ();
1016 : }
1017 : }
1018 : return true;
1019 : }
1020 :
1021 : /* If REG is a reload pseudo, try to make its class satisfying CL. */
1022 : static void
1023 3564336 : narrow_reload_pseudo_class (rtx reg, enum reg_class cl)
1024 : {
1025 3564336 : enum reg_class rclass;
1026 :
1027 : /* Do not make more accurate class from reloads generated. They are
1028 : mostly moves with a lot of constraints. Making more accurate
1029 : class may results in very narrow class and impossibility of find
1030 : registers for several reloads of one insn. */
1031 3564336 : if (INSN_UID (curr_insn) >= new_insn_uid_start)
1032 3564304 : return;
1033 3564218 : if (GET_CODE (reg) == SUBREG)
1034 169486 : reg = SUBREG_REG (reg);
1035 3564218 : if (! REG_P (reg) || (int) REGNO (reg) < new_regno_start)
1036 : return;
1037 32 : if (in_class_p (reg, cl, &rclass) && rclass != cl)
1038 13 : lra_change_class (REGNO (reg), rclass, " Change to", true);
1039 : }
1040 :
1041 : /* Searches X for any reference to a reg with the same value as REGNO,
1042 : returning the rtx of the reference found if any. Otherwise,
1043 : returns NULL_RTX. */
1044 : static rtx
1045 568113 : regno_val_use_in (unsigned int regno, rtx x)
1046 : {
1047 568113 : const char *fmt;
1048 568113 : int i, j;
1049 568113 : rtx tem;
1050 :
1051 568113 : if (REG_P (x) && lra_reg_info[REGNO (x)].val == lra_reg_info[regno].val)
1052 : return x;
1053 :
1054 567799 : fmt = GET_RTX_FORMAT (GET_CODE (x));
1055 1142003 : for (i = GET_RTX_LENGTH (GET_CODE (x)) - 1; i >= 0; i--)
1056 : {
1057 574204 : if (fmt[i] == 'e')
1058 : {
1059 7706 : if ((tem = regno_val_use_in (regno, XEXP (x, i))))
1060 : return tem;
1061 : }
1062 566498 : else if (fmt[i] == 'E')
1063 0 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
1064 0 : if ((tem = regno_val_use_in (regno , XVECEXP (x, i, j))))
1065 : return tem;
1066 : }
1067 :
1068 : return NULL_RTX;
1069 : }
1070 :
1071 : /* Return true if all current insn non-output operands except INS (it
1072 : has a negaitve end marker) do not use pseudos with the same value
1073 : as REGNO. */
1074 : static bool
1075 2 : check_conflict_input_operands (int regno, signed char *ins)
1076 : {
1077 2 : int in;
1078 2 : int n_operands = curr_static_id->n_operands;
1079 :
1080 8 : for (int nop = 0; nop < n_operands; nop++)
1081 7 : if (! curr_static_id->operand[nop].is_operator
1082 7 : && curr_static_id->operand[nop].type != OP_OUT)
1083 : {
1084 5 : for (int i = 0; (in = ins[i]) >= 0; i++)
1085 4 : if (in == nop)
1086 : break;
1087 3 : if (in < 0
1088 3 : && regno_val_use_in (regno, *curr_id->operand_loc[nop]) != NULL_RTX)
1089 : return false;
1090 : }
1091 : return true;
1092 : }
1093 :
1094 : /* Generate reloads for matching OUT and INS (array of input operand numbers
1095 : with end marker -1) with reg class GOAL_CLASS and EXCLUDE_START_HARD_REGS,
1096 : considering output operands OUTS (similar array to INS) needing to be in
1097 : different registers. Add input and output reloads correspondingly to the
1098 : lists *BEFORE and *AFTER. OUT might be negative. In this case we generate
1099 : input reloads for matched input operands INS. EARLY_CLOBBER_P is a flag
1100 : that the output operand is early clobbered for chosen alternative. */
1101 : static void
1102 1782168 : match_reload (signed char out, signed char *ins, signed char *outs,
1103 : enum reg_class goal_class, HARD_REG_SET *exclude_start_hard_regs,
1104 : rtx_insn **before, rtx_insn **after, bool early_clobber_p)
1105 : {
1106 1782168 : bool out_conflict;
1107 1782168 : int i, in;
1108 1782168 : rtx new_in_reg, new_out_reg, reg;
1109 1782168 : machine_mode inmode, outmode;
1110 1782168 : rtx in_rtx = *curr_id->operand_loc[ins[0]];
1111 1782168 : rtx out_rtx = out < 0 ? in_rtx : *curr_id->operand_loc[out];
1112 :
1113 1782168 : inmode = curr_operand_mode[ins[0]];
1114 1782168 : outmode = out < 0 ? inmode : curr_operand_mode[out];
1115 1782168 : push_to_sequence (*before);
1116 1782168 : if (inmode != outmode)
1117 : {
1118 : /* process_alt_operands has already checked that the mode sizes
1119 : are ordered. */
1120 112576 : if (partial_subreg_p (outmode, inmode))
1121 : {
1122 2346 : bool asm_p = asm_noperands (PATTERN (curr_insn)) >= 0;
1123 2346 : int hr;
1124 2346 : HARD_REG_SET temp_hard_reg_set;
1125 :
1126 19 : if (asm_p && (hr = get_hard_regno (out_rtx)) >= 0
1127 2349 : && hard_regno_nregs (hr, inmode) > 1)
1128 : {
1129 : /* See gcc.c-torture/execute/20030222-1.c.
1130 : Consider the code for 32-bit (e.g. BE) target:
1131 : int i, v; long x; x = v; asm ("" : "=r" (i) : "0" (x));
1132 : We generate the following RTL with reload insns:
1133 : 1. subreg:si(x:di, 0) = 0;
1134 : 2. subreg:si(x:di, 4) = v:si;
1135 : 3. t:di = x:di, dead x;
1136 : 4. asm ("" : "=r" (subreg:si(t:di,4)) : "0" (t:di))
1137 : 5. i:si = subreg:si(t:di,4);
1138 : If we assign hard reg of x to t, dead code elimination
1139 : will remove insn #2 and we will use uninitialized hard reg.
1140 : So exclude the hard reg of x for t. We could ignore this
1141 : problem for non-empty asm using all x value but it is hard to
1142 : check that the asm are expanded into insn really using x
1143 : and setting r. */
1144 0 : CLEAR_HARD_REG_SET (temp_hard_reg_set);
1145 0 : if (exclude_start_hard_regs != NULL)
1146 0 : temp_hard_reg_set = *exclude_start_hard_regs;
1147 0 : SET_HARD_REG_BIT (temp_hard_reg_set, hr);
1148 0 : exclude_start_hard_regs = &temp_hard_reg_set;
1149 : }
1150 4692 : reg = new_in_reg
1151 2346 : = lra_create_new_reg_with_unique_value (inmode, in_rtx, goal_class,
1152 : exclude_start_hard_regs,
1153 : "");
1154 2346 : new_out_reg = get_matching_reload_reg_subreg (outmode, reg, goal_class);
1155 2346 : LRA_SUBREG_P (new_out_reg) = 1;
1156 : /* If the input reg is dying here, we can use the same hard
1157 : register for REG and IN_RTX. We do it only for original
1158 : pseudos as reload pseudos can die although original
1159 : pseudos still live where reload pseudos dies. */
1160 1825 : if (REG_P (in_rtx) && (int) REGNO (in_rtx) < lra_new_regno_start
1161 1766 : && find_regno_note (curr_insn, REG_DEAD, REGNO (in_rtx))
1162 3192 : && (!early_clobber_p
1163 2 : || check_conflict_input_operands(REGNO (in_rtx), ins)))
1164 845 : lra_assign_reg_val (REGNO (in_rtx), REGNO (reg));
1165 : }
1166 : else
1167 : {
1168 220460 : reg = new_out_reg
1169 110230 : = lra_create_new_reg_with_unique_value (outmode, out_rtx,
1170 : goal_class,
1171 : exclude_start_hard_regs,
1172 : "");
1173 110230 : new_in_reg = get_matching_reload_reg_subreg (inmode, reg, goal_class);
1174 : /* NEW_IN_REG is non-paradoxical subreg. We don't want
1175 : NEW_OUT_REG living above. We add clobber clause for
1176 : this. This is just a temporary clobber. We can remove
1177 : it at the end of LRA work. */
1178 110230 : rtx_insn *clobber = emit_clobber (new_out_reg);
1179 110230 : LRA_TEMP_CLOBBER_P (PATTERN (clobber)) = 1;
1180 110230 : LRA_SUBREG_P (new_in_reg) = 1;
1181 110230 : if (GET_CODE (in_rtx) == SUBREG)
1182 : {
1183 2059 : rtx subreg_reg = SUBREG_REG (in_rtx);
1184 :
1185 : /* If SUBREG_REG is dying here and sub-registers IN_RTX
1186 : and NEW_IN_REG are similar, we can use the same hard
1187 : register for REG and SUBREG_REG. */
1188 2059 : if (REG_P (subreg_reg)
1189 2059 : && (int) REGNO (subreg_reg) < lra_new_regno_start
1190 2059 : && GET_MODE (subreg_reg) == outmode
1191 1435 : && known_eq (SUBREG_BYTE (in_rtx), SUBREG_BYTE (new_in_reg))
1192 1435 : && find_regno_note (curr_insn, REG_DEAD, REGNO (subreg_reg))
1193 2322 : && (! early_clobber_p
1194 0 : || check_conflict_input_operands (REGNO (subreg_reg),
1195 : ins)))
1196 263 : lra_assign_reg_val (REGNO (subreg_reg), REGNO (reg));
1197 : }
1198 : }
1199 : }
1200 : else
1201 : {
1202 : /* Pseudos have values -- see comments for lra_reg_info.
1203 : Different pseudos with the same value do not conflict even if
1204 : they live in the same place. When we create a pseudo we
1205 : assign value of original pseudo (if any) from which we
1206 : created the new pseudo. If we create the pseudo from the
1207 : input pseudo, the new pseudo will have no conflict with the
1208 : input pseudo which is wrong when the input pseudo lives after
1209 : the insn and as the new pseudo value is changed by the insn
1210 : output. Therefore we create the new pseudo from the output
1211 : except the case when we have single matched dying input
1212 : pseudo.
1213 :
1214 : We cannot reuse the current output register because we might
1215 : have a situation like "a <- a op b", where the constraints
1216 : force the second input operand ("b") to match the output
1217 : operand ("a"). "b" must then be copied into a new register
1218 : so that it doesn't clobber the current value of "a".
1219 :
1220 : We cannot use the same value if the output pseudo is
1221 : early clobbered or the input pseudo is mentioned in the
1222 : output, e.g. as an address part in memory, because
1223 : output reload will actually extend the pseudo liveness.
1224 : We don't care about eliminable hard regs here as we are
1225 : interesting only in pseudos. */
1226 :
1227 : /* Matching input's register value is the same as one of the other
1228 : output operand. Output operands in a parallel insn must be in
1229 : different registers. */
1230 1669592 : out_conflict = false;
1231 1669592 : if (REG_P (in_rtx))
1232 : {
1233 2881141 : for (i = 0; outs[i] >= 0; i++)
1234 : {
1235 1499295 : rtx other_out_rtx = *curr_id->operand_loc[outs[i]];
1236 117245 : if (outs[i] != out && REG_P (other_out_rtx)
1237 1616339 : && (regno_val_use_in (REGNO (in_rtx), other_out_rtx)
1238 : != NULL_RTX))
1239 : {
1240 : out_conflict = true;
1241 : break;
1242 : }
1243 : }
1244 : }
1245 :
1246 1669592 : new_in_reg = new_out_reg
1247 1634289 : = (! early_clobber_p && ins[1] < 0 && REG_P (in_rtx)
1248 1347588 : && (int) REGNO (in_rtx) < lra_new_regno_start
1249 1347297 : && find_regno_note (curr_insn, REG_DEAD, REGNO (in_rtx))
1250 443362 : && (out < 0
1251 443362 : || regno_val_use_in (REGNO (in_rtx), out_rtx) == NULL_RTX)
1252 443311 : && !out_conflict
1253 2112901 : ? lra_create_new_reg (inmode, in_rtx, goal_class,
1254 : exclude_start_hard_regs, "")
1255 1226283 : : lra_create_new_reg_with_unique_value (outmode, out_rtx, goal_class,
1256 : exclude_start_hard_regs,
1257 : ""));
1258 : }
1259 : /* In operand can be got from transformations before processing insn
1260 : constraints. One example of such transformations is subreg
1261 : reloading (see function simplify_operand_subreg). The new
1262 : pseudos created by the transformations might have inaccurate
1263 : class (ALL_REGS) and we should make their classes more
1264 : accurate. */
1265 1782168 : narrow_reload_pseudo_class (in_rtx, goal_class);
1266 1782168 : lra_emit_move (copy_rtx (new_in_reg), in_rtx);
1267 1782168 : *before = end_sequence ();
1268 : /* Add the new pseudo to consider values of subsequent input reload
1269 : pseudos. */
1270 1782168 : lra_assert (curr_insn_input_reloads_num < LRA_MAX_INSN_RELOADS);
1271 1782168 : curr_insn_input_reloads[curr_insn_input_reloads_num].input = in_rtx;
1272 1782168 : curr_insn_input_reloads[curr_insn_input_reloads_num].match_p = true;
1273 1782168 : curr_insn_input_reloads[curr_insn_input_reloads_num].early_clobber_p = false;
1274 1782168 : curr_insn_input_reloads[curr_insn_input_reloads_num++].reg = new_in_reg;
1275 3564337 : for (i = 0; (in = ins[i]) >= 0; i++)
1276 1782169 : if (GET_MODE (*curr_id->operand_loc[in]) == VOIDmode
1277 1758077 : || GET_MODE (new_in_reg) == GET_MODE (*curr_id->operand_loc[in]))
1278 1782168 : *curr_id->operand_loc[in] = new_in_reg;
1279 : else
1280 : {
1281 1 : lra_assert
1282 : (GET_MODE (new_out_reg) == GET_MODE (*curr_id->operand_loc[in]));
1283 1 : *curr_id->operand_loc[in] = new_out_reg;
1284 : }
1285 1782168 : lra_update_dups (curr_id, ins);
1286 1782168 : if (out < 0)
1287 : return;
1288 : /* See a comment for the input operand above. */
1289 1782168 : narrow_reload_pseudo_class (out_rtx, goal_class);
1290 1782168 : reg = SUBREG_P (out_rtx) ? SUBREG_REG (out_rtx) : out_rtx;
1291 1782168 : if (find_reg_note (curr_insn, REG_UNUSED, reg) == NULL_RTX
1292 1782168 : && (!REG_P (reg) || !ira_former_scratch_p (REGNO (reg))))
1293 : {
1294 1685463 : start_sequence ();
1295 : /* If we had strict_low_part, use it also in reload to keep other
1296 : parts unchanged but do it only for regs as strict_low_part
1297 : has no sense for memory and probably there is no insn pattern
1298 : to match the reload insn in memory case. */
1299 1685463 : if (out >= 0 && curr_static_id->operand[out].strict_low && REG_P (reg))
1300 0 : out_rtx = gen_rtx_STRICT_LOW_PART (VOIDmode, out_rtx);
1301 1685463 : lra_emit_move (out_rtx, copy_rtx (new_out_reg));
1302 1685463 : emit_insn (*after);
1303 1685463 : *after = end_sequence ();
1304 : }
1305 1782168 : *curr_id->operand_loc[out] = new_out_reg;
1306 1782168 : lra_update_dup (curr_id, out);
1307 : }
1308 :
1309 : /* Return register class which is union of all reg classes in insn
1310 : constraint alternative string starting with P. */
1311 : static enum reg_class
1312 0 : reg_class_from_constraints (const char *p)
1313 : {
1314 0 : int c, len;
1315 0 : enum reg_class op_class = NO_REGS;
1316 :
1317 0 : do
1318 0 : switch ((c = *p, len = CONSTRAINT_LEN (c, p)), c)
1319 : {
1320 : case '#':
1321 : case ',':
1322 : return op_class;
1323 :
1324 0 : case 'g':
1325 0 : op_class = reg_class_subunion[op_class][GENERAL_REGS];
1326 0 : break;
1327 :
1328 0 : default:
1329 0 : enum constraint_num cn = lookup_constraint (p);
1330 0 : enum reg_class cl = reg_class_for_constraint (cn);
1331 0 : if (cl == NO_REGS)
1332 : {
1333 0 : if (insn_extra_address_constraint (cn))
1334 0 : op_class
1335 0 : = (reg_class_subunion
1336 0 : [op_class][base_reg_class (VOIDmode, ADDR_SPACE_GENERIC,
1337 0 : ADDRESS, SCRATCH)]);
1338 : break;
1339 : }
1340 :
1341 0 : op_class = reg_class_subunion[op_class][cl];
1342 0 : break;
1343 : }
1344 0 : while ((p += len), c);
1345 : return op_class;
1346 : }
1347 :
1348 : /* If OP is a register, return the class of the register as per
1349 : get_reg_class, otherwise return NO_REGS. */
1350 : static inline enum reg_class
1351 167111849 : get_op_class (rtx op)
1352 : {
1353 138920256 : return REG_P (op) ? get_reg_class (REGNO (op)) : NO_REGS;
1354 : }
1355 :
1356 : /* Return generated insn mem_pseudo:=val if TO_P or val:=mem_pseudo
1357 : otherwise. If modes of MEM_PSEUDO and VAL are different, use
1358 : SUBREG for VAL to make them equal. */
1359 : static rtx_insn *
1360 1353258 : emit_spill_move (bool to_p, rtx mem_pseudo, rtx val)
1361 : {
1362 1353258 : if (GET_MODE (mem_pseudo) != GET_MODE (val))
1363 : {
1364 : /* Usually size of mem_pseudo is greater than val size but in
1365 : rare cases it can be less as it can be defined by target
1366 : dependent macro HARD_REGNO_CALLER_SAVE_MODE. */
1367 3028 : if (! MEM_P (val))
1368 : {
1369 3028 : val = gen_lowpart_SUBREG (GET_MODE (mem_pseudo),
1370 : GET_CODE (val) == SUBREG
1371 : ? SUBREG_REG (val) : val);
1372 3028 : LRA_SUBREG_P (val) = 1;
1373 : }
1374 : else
1375 : {
1376 0 : mem_pseudo = gen_lowpart_SUBREG (GET_MODE (val), mem_pseudo);
1377 0 : LRA_SUBREG_P (mem_pseudo) = 1;
1378 : }
1379 : }
1380 1353258 : return to_p ? gen_move_insn (mem_pseudo, val)
1381 683149 : : gen_move_insn (val, mem_pseudo);
1382 : }
1383 :
1384 : /* Process a special case insn (register move), return true if we
1385 : don't need to process it anymore. INSN should be a single set
1386 : insn. Set up that RTL was changed through CHANGE_P and that hook
1387 : TARGET_SECONDARY_MEMORY_NEEDED says to use secondary memory through
1388 : SEC_MEM_P. */
1389 : static bool
1390 77359777 : check_and_process_move (bool *change_p, bool *sec_mem_p ATTRIBUTE_UNUSED)
1391 : {
1392 77359777 : int sregno, dregno;
1393 77359777 : rtx dest, src, dreg, sreg, new_reg, scratch_reg;
1394 77359777 : rtx_insn *before;
1395 77359777 : enum reg_class dclass, sclass, secondary_class;
1396 77359777 : secondary_reload_info sri;
1397 :
1398 77359777 : lra_assert (curr_insn_set != NULL_RTX);
1399 77359777 : dreg = dest = SET_DEST (curr_insn_set);
1400 77359777 : sreg = src = SET_SRC (curr_insn_set);
1401 77359777 : if (GET_CODE (dest) == SUBREG)
1402 1219520 : dreg = SUBREG_REG (dest);
1403 77359777 : if (GET_CODE (src) == SUBREG)
1404 1293684 : sreg = SUBREG_REG (src);
1405 77359777 : if (! (REG_P (dreg) || MEM_P (dreg)) || ! (REG_P (sreg) || MEM_P (sreg)))
1406 : return false;
1407 36119855 : sclass = dclass = NO_REGS;
1408 36119855 : if (REG_P (dreg))
1409 23290848 : dclass = get_reg_class (REGNO (dreg));
1410 23290848 : gcc_assert (dclass < LIM_REG_CLASSES && dclass >= NO_REGS);
1411 36119855 : if (dclass == ALL_REGS)
1412 : /* ALL_REGS is used for new pseudos created by transformations
1413 : like reload of SUBREG_REG (see function
1414 : simplify_operand_subreg). We don't know their class yet. We
1415 : should figure out the class from processing the insn
1416 : constraints not in this fast path function. Even if ALL_REGS
1417 : were a right class for the pseudo, secondary_... hooks usually
1418 : are not define for ALL_REGS. */
1419 : return false;
1420 36117629 : if (REG_P (sreg))
1421 20205289 : sclass = get_reg_class (REGNO (sreg));
1422 20205289 : gcc_assert (sclass < LIM_REG_CLASSES && sclass >= NO_REGS);
1423 36117629 : if (sclass == ALL_REGS)
1424 : /* See comments above. */
1425 : return false;
1426 36117629 : if (sclass == NO_REGS && dclass == NO_REGS)
1427 : return false;
1428 34635021 : if (targetm.secondary_memory_needed (GET_MODE (src), sclass, dclass)
1429 34635021 : && ((sclass != NO_REGS && dclass != NO_REGS)
1430 0 : || (GET_MODE (src)
1431 0 : != targetm.secondary_memory_needed_mode (GET_MODE (src)))))
1432 : {
1433 13231 : *sec_mem_p = true;
1434 13231 : return false;
1435 : }
1436 34621790 : if (! REG_P (dreg) || ! REG_P (sreg))
1437 : return false;
1438 7889755 : sri.prev_sri = NULL;
1439 7889755 : sri.icode = CODE_FOR_nothing;
1440 7889755 : sri.extra_cost = 0;
1441 7889755 : secondary_class = NO_REGS;
1442 : /* Set up hard register for a reload pseudo for hook
1443 : secondary_reload because some targets just ignore unassigned
1444 : pseudos in the hook. */
1445 7889755 : if (dclass != NO_REGS && lra_get_regno_hard_regno (REGNO (dreg)) < 0)
1446 : {
1447 2929249 : dregno = REGNO (dreg);
1448 2929249 : reg_renumber[dregno] = ira_class_hard_regs[dclass][0];
1449 : }
1450 : else
1451 : dregno = -1;
1452 7889755 : if (sclass != NO_REGS && lra_get_regno_hard_regno (REGNO (sreg)) < 0)
1453 : {
1454 1305770 : sregno = REGNO (sreg);
1455 1305770 : reg_renumber[sregno] = ira_class_hard_regs[sclass][0];
1456 : }
1457 : else
1458 : sregno = -1;
1459 7889755 : if (sclass != NO_REGS)
1460 3988069 : secondary_class
1461 7976138 : = (enum reg_class) targetm.secondary_reload (false, dest,
1462 : (reg_class_t) sclass,
1463 3988069 : GET_MODE (src), &sri);
1464 3988069 : if (sclass == NO_REGS
1465 3988069 : || ((secondary_class != NO_REGS || sri.icode != CODE_FOR_nothing)
1466 1338 : && dclass != NO_REGS))
1467 : {
1468 3901686 : enum reg_class old_sclass = secondary_class;
1469 3901686 : secondary_reload_info old_sri = sri;
1470 :
1471 3901686 : sri.prev_sri = NULL;
1472 3901686 : sri.icode = CODE_FOR_nothing;
1473 3901686 : sri.extra_cost = 0;
1474 3901686 : secondary_class
1475 7803372 : = (enum reg_class) targetm.secondary_reload (true, src,
1476 : (reg_class_t) dclass,
1477 3901686 : GET_MODE (src), &sri);
1478 : /* Check the target hook consistency. */
1479 3901686 : lra_assert
1480 : ((secondary_class == NO_REGS && sri.icode == CODE_FOR_nothing)
1481 : || (old_sclass == NO_REGS && old_sri.icode == CODE_FOR_nothing)
1482 : || (secondary_class == old_sclass && sri.icode == old_sri.icode));
1483 : }
1484 7889755 : if (sregno >= 0)
1485 1305770 : reg_renumber [sregno] = -1;
1486 7889755 : if (dregno >= 0)
1487 2929249 : reg_renumber [dregno] = -1;
1488 7889755 : if (secondary_class == NO_REGS && sri.icode == CODE_FOR_nothing)
1489 : return false;
1490 1339 : *change_p = true;
1491 1339 : new_reg = NULL_RTX;
1492 0 : if (secondary_class != NO_REGS)
1493 1339 : new_reg = lra_create_new_reg_with_unique_value (GET_MODE (src), NULL_RTX,
1494 : secondary_class, NULL,
1495 : "secondary");
1496 1339 : start_sequence ();
1497 1339 : if (sri.icode == CODE_FOR_nothing)
1498 1339 : lra_emit_move (new_reg, src);
1499 : else
1500 : {
1501 0 : enum reg_class scratch_class;
1502 :
1503 0 : scratch_class = (reg_class_from_constraints
1504 0 : (insn_data[sri.icode].operand[2].constraint));
1505 0 : scratch_reg = (lra_create_new_reg_with_unique_value
1506 0 : (insn_data[sri.icode].operand[2].mode, NULL_RTX,
1507 : scratch_class, NULL, "scratch"));
1508 0 : emit_insn (GEN_FCN (sri.icode) (new_reg != NULL_RTX ? new_reg : dest,
1509 : src, scratch_reg));
1510 : }
1511 1339 : before = end_sequence ();
1512 1339 : lra_process_new_insns (curr_insn, before, NULL, "Inserting the move");
1513 1339 : if (new_reg != NULL_RTX)
1514 1339 : SET_SRC (curr_insn_set) = new_reg;
1515 : else
1516 : {
1517 0 : if (lra_dump_file != NULL)
1518 : {
1519 0 : fprintf (lra_dump_file, "Deleting move %u\n", INSN_UID (curr_insn));
1520 0 : dump_insn_slim (lra_dump_file, curr_insn);
1521 : }
1522 0 : lra_set_insn_deleted (curr_insn);
1523 0 : return true;
1524 : }
1525 1339 : return false;
1526 : }
1527 :
1528 : /* The following data describe the result of process_alt_operands.
1529 : The data are used in curr_insn_transform to generate reloads. */
1530 :
1531 : /* The chosen reg classes which should be used for the corresponding
1532 : operands. */
1533 : static enum reg_class goal_alt[MAX_RECOG_OPERANDS];
1534 : /* Hard registers which cannot be a start hard register for the corresponding
1535 : operands. */
1536 : static HARD_REG_SET goal_alt_exclude_start_hard_regs[MAX_RECOG_OPERANDS];
1537 : /* True if the operand should be the same as another operand and that
1538 : other operand does not need a reload. */
1539 : static bool goal_alt_match_win[MAX_RECOG_OPERANDS];
1540 : /* True if the operand does not need a reload. */
1541 : static bool goal_alt_win[MAX_RECOG_OPERANDS];
1542 : /* True if the operand can be offsetable memory. */
1543 : static bool goal_alt_offmemok[MAX_RECOG_OPERANDS];
1544 : /* The number of an operand to which given operand can be matched to. */
1545 : static int goal_alt_matches[MAX_RECOG_OPERANDS];
1546 : /* The number of elements in the following array. */
1547 : static int goal_alt_dont_inherit_ops_num;
1548 : /* Numbers of operands whose reload pseudos should not be inherited. */
1549 : static int goal_alt_dont_inherit_ops[MAX_RECOG_OPERANDS];
1550 : /* True if we should try only this alternative for the next constraint sub-pass
1551 : to speed up the sub-pass. */
1552 : static bool goal_reuse_alt_p;
1553 : /* True if the insn commutative operands should be swapped. */
1554 : static bool goal_alt_swapped;
1555 : /* The chosen insn alternative. */
1556 : static int goal_alt_number;
1557 : /* True if output reload of the stack pointer should be generated. */
1558 : static bool goal_alt_out_sp_reload_p;
1559 :
1560 : /* True if the corresponding operand is the result of an equivalence
1561 : substitution. */
1562 : static bool equiv_substition_p[MAX_RECOG_OPERANDS];
1563 :
1564 : /* The following five variables are used to choose the best insn
1565 : alternative. They reflect final characteristics of the best
1566 : alternative. */
1567 :
1568 : /* Number of necessary reloads and overall cost reflecting the
1569 : previous value and other unpleasantness of the best alternative. */
1570 : static int best_losers, best_overall;
1571 : /* Overall number hard registers used for reloads. For example, on
1572 : some targets we need 2 general registers to reload DFmode and only
1573 : one floating point register. */
1574 : static int best_reload_nregs;
1575 : /* Overall number reflecting distances of previous reloading the same
1576 : value. The distances are counted from the current BB start. It is
1577 : used to improve inheritance chances. */
1578 : static int best_reload_sum;
1579 :
1580 : /* True if the current insn should have no correspondingly input or
1581 : output reloads. */
1582 : static bool no_input_reloads_p, no_output_reloads_p;
1583 :
1584 : /* True if we swapped the commutative operands in the current
1585 : insn. */
1586 : static int curr_swapped;
1587 :
1588 : /* if CHECK_ONLY_P is false, arrange for address element *LOC to be a
1589 : register of class CL. Add any input reloads to list BEFORE. AFTER
1590 : is nonnull if *LOC is an automodified value; handle that case by
1591 : adding the required output reloads to list AFTER. Return true if
1592 : the RTL was changed.
1593 :
1594 : if CHECK_ONLY_P is true, check that the *LOC is a correct address
1595 : register. Return false if the address register is correct. */
1596 : static bool
1597 35769502 : process_addr_reg (rtx *loc, bool check_only_p, rtx_insn **before, rtx_insn **after,
1598 : enum reg_class cl)
1599 : {
1600 35769502 : int regno;
1601 35769502 : enum reg_class rclass, new_class;
1602 35769502 : rtx reg;
1603 35769502 : rtx new_reg;
1604 35769502 : machine_mode mode;
1605 35769502 : bool subreg_p, before_p = false;
1606 :
1607 35769502 : subreg_p = GET_CODE (*loc) == SUBREG;
1608 35769502 : if (subreg_p)
1609 : {
1610 15468 : reg = SUBREG_REG (*loc);
1611 15468 : mode = GET_MODE (reg);
1612 :
1613 : /* For mode with size bigger than ptr_mode, there unlikely to be "mov"
1614 : between two registers with different classes, but there normally will
1615 : be "mov" which transfers element of vector register into the general
1616 : register, and this normally will be a subreg which should be reloaded
1617 : as a whole. This is particularly likely to be triggered when
1618 : -fno-split-wide-types specified. */
1619 15468 : if (!REG_P (reg)
1620 15468 : || in_class_p (reg, cl, &new_class)
1621 17872 : || known_le (GET_MODE_SIZE (mode), GET_MODE_SIZE (ptr_mode)))
1622 15468 : loc = &SUBREG_REG (*loc);
1623 : }
1624 :
1625 35769502 : reg = *loc;
1626 35769502 : mode = GET_MODE (reg);
1627 35769502 : if (! REG_P (reg))
1628 : {
1629 0 : if (check_only_p)
1630 : return true;
1631 : /* Always reload memory in an address even if the target supports
1632 : such addresses. */
1633 0 : new_reg = lra_create_new_reg_with_unique_value (mode, reg, cl, NULL,
1634 : "address");
1635 0 : before_p = true;
1636 : }
1637 : else
1638 : {
1639 35769502 : regno = REGNO (reg);
1640 35769502 : rclass = get_reg_class (regno);
1641 35769502 : if (! check_only_p
1642 35769502 : && (*loc = get_equiv_with_elimination (reg, curr_insn)) != reg)
1643 : {
1644 114455 : if (lra_dump_file != NULL)
1645 : {
1646 0 : fprintf (lra_dump_file,
1647 : "Changing pseudo %d in address of insn %u on equiv ",
1648 0 : REGNO (reg), INSN_UID (curr_insn));
1649 0 : dump_value_slim (lra_dump_file, *loc, 1);
1650 0 : fprintf (lra_dump_file, "\n");
1651 : }
1652 114455 : rtx new_equiv = copy_rtx (*loc);
1653 114455 : if (lra_pointer_equiv_set_in (*loc))
1654 109168 : lra_pointer_equiv_set_add (new_equiv);
1655 114455 : *loc = new_equiv;
1656 : }
1657 35769502 : if (*loc != reg || ! in_class_p (reg, cl, &new_class))
1658 : {
1659 477302 : if (check_only_p)
1660 : return true;
1661 477302 : reg = *loc;
1662 477302 : if (get_reload_reg (after == NULL ? OP_IN : OP_INOUT,
1663 : mode, reg, cl, NULL,
1664 : subreg_p, false, "address", &new_reg))
1665 : before_p = true;
1666 : }
1667 35292200 : else if (new_class != NO_REGS && rclass != new_class)
1668 : {
1669 465484 : if (check_only_p)
1670 : return true;
1671 465484 : lra_change_class (regno, new_class, " Change to", true);
1672 465484 : return false;
1673 : }
1674 : else
1675 : return false;
1676 : }
1677 0 : if (before_p)
1678 : {
1679 469333 : push_to_sequence (*before);
1680 469333 : lra_emit_move (new_reg, reg);
1681 469333 : *before = end_sequence ();
1682 : }
1683 477302 : *loc = new_reg;
1684 477302 : if (after != NULL)
1685 : {
1686 0 : start_sequence ();
1687 0 : lra_emit_move (before_p ? copy_rtx (reg) : reg, new_reg);
1688 0 : emit_insn (*after);
1689 0 : *after = end_sequence ();
1690 : }
1691 : return true;
1692 : }
1693 :
1694 : /* Insert move insn in simplify_operand_subreg. BEFORE returns
1695 : the insn to be inserted before curr insn. AFTER returns the
1696 : the insn to be inserted after curr insn. ORIGREG and NEWREG
1697 : are the original reg and new reg for reload. */
1698 : static void
1699 461 : insert_move_for_subreg (rtx_insn **before, rtx_insn **after, rtx origreg,
1700 : rtx newreg)
1701 : {
1702 461 : if (before)
1703 : {
1704 461 : push_to_sequence (*before);
1705 461 : lra_emit_move (newreg, origreg);
1706 461 : *before = end_sequence ();
1707 : }
1708 461 : if (after)
1709 : {
1710 0 : start_sequence ();
1711 0 : lra_emit_move (origreg, newreg);
1712 0 : emit_insn (*after);
1713 0 : *after = end_sequence ();
1714 : }
1715 461 : }
1716 :
1717 : static bool valid_address_p (machine_mode mode, rtx addr, addr_space_t as);
1718 : static bool process_address (int, bool, rtx_insn **, rtx_insn **);
1719 :
1720 : /* Make reloads for subreg in operand NOP with internal subreg mode
1721 : REG_MODE, add new reloads for further processing. Return true if
1722 : any change was done. */
1723 : static bool
1724 179838223 : simplify_operand_subreg (int nop, machine_mode reg_mode)
1725 : {
1726 179838223 : int hard_regno, inner_hard_regno;
1727 179838223 : rtx_insn *before, *after;
1728 179838223 : machine_mode mode, innermode;
1729 179838223 : rtx reg, new_reg;
1730 179838223 : rtx operand = *curr_id->operand_loc[nop];
1731 179838223 : enum reg_class regclass;
1732 179838223 : enum op_type type;
1733 :
1734 179838223 : before = after = NULL;
1735 :
1736 179838223 : if (GET_CODE (operand) != SUBREG)
1737 : return false;
1738 :
1739 3852875 : mode = GET_MODE (operand);
1740 3852875 : reg = SUBREG_REG (operand);
1741 3852875 : innermode = GET_MODE (reg);
1742 3852875 : type = curr_static_id->operand[nop].type;
1743 3852875 : if (MEM_P (reg))
1744 : {
1745 11492 : const bool addr_was_valid
1746 11492 : = valid_address_p (innermode, XEXP (reg, 0), MEM_ADDR_SPACE (reg));
1747 11492 : alter_subreg (curr_id->operand_loc[nop], false);
1748 11492 : rtx subst = *curr_id->operand_loc[nop];
1749 11492 : lra_assert (MEM_P (subst));
1750 11492 : const bool addr_is_valid = valid_address_p (GET_MODE (subst),
1751 : XEXP (subst, 0),
1752 11492 : MEM_ADDR_SPACE (subst));
1753 11492 : if (!addr_was_valid
1754 11492 : || addr_is_valid
1755 11492 : || ((get_constraint_type (lookup_constraint
1756 0 : (curr_static_id->operand[nop].constraint))
1757 : != CT_SPECIAL_MEMORY)
1758 : /* We still can reload address and if the address is
1759 : valid, we can remove subreg without reloading its
1760 : inner memory. */
1761 0 : && valid_address_p (GET_MODE (subst),
1762 0 : regno_reg_rtx
1763 : [ira_class_hard_regs
1764 0 : [base_reg_class (GET_MODE (subst),
1765 0 : MEM_ADDR_SPACE (subst),
1766 0 : ADDRESS, SCRATCH)][0]],
1767 0 : MEM_ADDR_SPACE (subst))))
1768 : {
1769 : /* If we change the address for a paradoxical subreg of memory, the
1770 : new address might violate the necessary alignment or the access
1771 : might be slow; take this into consideration. We need not worry
1772 : about accesses beyond allocated memory for paradoxical memory
1773 : subregs as we don't substitute such equiv memory (see processing
1774 : equivalences in function lra_constraints) and because for spilled
1775 : pseudos we allocate stack memory enough for the biggest
1776 : corresponding paradoxical subreg.
1777 :
1778 : However, do not blindly simplify a (subreg (mem ...)) for
1779 : WORD_REGISTER_OPERATIONS targets as this may lead to loading junk
1780 : data into a register when the inner is narrower than outer or
1781 : missing important data from memory when the inner is wider than
1782 : outer. This rule only applies to modes that are no wider than
1783 : a word.
1784 :
1785 : If valid memory becomes invalid after subreg elimination
1786 : and address might be different we still have to reload
1787 : memory.
1788 : */
1789 11492 : if ((! addr_was_valid
1790 : || addr_is_valid
1791 0 : || known_eq (GET_MODE_SIZE (mode), GET_MODE_SIZE (innermode)))
1792 11492 : && !(maybe_ne (GET_MODE_PRECISION (mode),
1793 11492 : GET_MODE_PRECISION (innermode))
1794 13178 : && known_le (GET_MODE_SIZE (mode), UNITS_PER_WORD)
1795 19929 : && known_le (GET_MODE_SIZE (innermode), UNITS_PER_WORD)
1796 : && WORD_REGISTER_OPERATIONS)
1797 24124 : && (!(MEM_ALIGN (subst) < GET_MODE_ALIGNMENT (mode)
1798 1140 : && targetm.slow_unaligned_access (mode, MEM_ALIGN (subst)))
1799 0 : || (MEM_ALIGN (reg) < GET_MODE_ALIGNMENT (innermode)
1800 0 : && targetm.slow_unaligned_access (innermode,
1801 0 : MEM_ALIGN (reg)))))
1802 : return true;
1803 :
1804 0 : *curr_id->operand_loc[nop] = operand;
1805 :
1806 : /* But if the address was not valid, we cannot reload the MEM without
1807 : reloading the address first. */
1808 0 : if (!addr_was_valid)
1809 0 : process_address (nop, false, &before, &after);
1810 :
1811 : /* INNERMODE is fast, MODE slow. Reload the mem in INNERMODE. */
1812 0 : enum reg_class rclass
1813 0 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
1814 0 : if (get_reload_reg (curr_static_id->operand[nop].type, innermode,
1815 : reg, rclass, NULL,
1816 : true, false, "slow/invalid mem", &new_reg))
1817 : {
1818 0 : bool insert_before, insert_after;
1819 0 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
1820 :
1821 0 : insert_before = (type != OP_OUT
1822 0 : || partial_subreg_p (mode, innermode));
1823 0 : insert_after = type != OP_IN;
1824 0 : insert_move_for_subreg (insert_before ? &before : NULL,
1825 : insert_after ? &after : NULL,
1826 : reg, new_reg);
1827 : }
1828 0 : SUBREG_REG (operand) = new_reg;
1829 :
1830 : /* Convert to MODE. */
1831 0 : reg = operand;
1832 0 : rclass
1833 0 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
1834 0 : if (get_reload_reg (curr_static_id->operand[nop].type, mode, reg,
1835 : rclass, NULL,
1836 : true, false, "slow/invalid mem", &new_reg))
1837 : {
1838 0 : bool insert_before, insert_after;
1839 0 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
1840 :
1841 0 : insert_before = type != OP_OUT;
1842 0 : insert_after = type != OP_IN;
1843 0 : insert_move_for_subreg (insert_before ? &before : NULL,
1844 : insert_after ? &after : NULL,
1845 : reg, new_reg);
1846 : }
1847 0 : *curr_id->operand_loc[nop] = new_reg;
1848 0 : lra_process_new_insns (curr_insn, before, after,
1849 : "Inserting slow/invalid mem reload");
1850 0 : return true;
1851 : }
1852 :
1853 : /* If the address was valid and became invalid, prefer to reload
1854 : the memory. Typical case is when the index scale should
1855 : correspond the memory. */
1856 0 : *curr_id->operand_loc[nop] = operand;
1857 : /* Do not return false here as the MEM_P (reg) will be processed
1858 : later in this function. */
1859 : }
1860 3841383 : else if (REG_P (reg) && REGNO (reg) < FIRST_PSEUDO_REGISTER)
1861 : {
1862 : /* A narrowing subreg of a hard register that is not representable as a
1863 : hard register (its offset does not fall on a register boundary) cannot
1864 : be turned into one, and would otherwise be resolved to the wrong part
1865 : of the register. Reload it through memory so that the correct bytes
1866 : are accessed, as is done for pseudos below. Leave the frame, arg and
1867 : stack pointers alone: simplify_subreg_regno can reject them simply
1868 : because reload is not finished yet. */
1869 54 : if (partial_subreg_p (mode, innermode)
1870 54 : && REGNO (reg) != FRAME_POINTER_REGNUM
1871 : && REGNO (reg) != ARG_POINTER_REGNUM
1872 : && REGNO (reg) != STACK_POINTER_REGNUM
1873 59 : && simplify_subreg_regno (REGNO (reg), innermode,
1874 5 : SUBREG_BYTE (operand), mode) < 0)
1875 : {
1876 0 : if (get_reload_reg (type, innermode, reg, NO_REGS, NULL,
1877 : true, false, "non-representable subreg", &new_reg))
1878 : {
1879 0 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
1880 0 : bool insert_before = (type != OP_OUT
1881 0 : || read_modify_subreg_p (operand));
1882 0 : bool insert_after = (type != OP_IN);
1883 0 : insert_move_for_subreg (insert_before ? &before : NULL,
1884 : insert_after ? &after : NULL,
1885 : reg, new_reg);
1886 : }
1887 0 : SUBREG_REG (operand) = new_reg;
1888 0 : lra_process_new_insns (curr_insn, before, after,
1889 : "Inserting non-representable subreg reload");
1890 0 : return true;
1891 : }
1892 54 : alter_subreg (curr_id->operand_loc[nop], false);
1893 54 : return true;
1894 : }
1895 3841329 : else if (CONSTANT_P (reg))
1896 : {
1897 : /* Try to simplify subreg of constant. It is usually result of
1898 : equivalence substitution. */
1899 45645 : if (innermode == VOIDmode
1900 45645 : && (innermode = original_subreg_reg_mode[nop]) == VOIDmode)
1901 0 : innermode = curr_static_id->operand[nop].mode;
1902 45645 : if ((new_reg = simplify_subreg (mode, reg, innermode,
1903 45645 : SUBREG_BYTE (operand))) != NULL_RTX)
1904 : {
1905 45226 : *curr_id->operand_loc[nop] = new_reg;
1906 45226 : return true;
1907 : }
1908 : }
1909 : /* Put constant into memory when we have mixed modes. It generates
1910 : a better code in most cases as it does not need a secondary
1911 : reload memory. It also prevents LRA looping when LRA is using
1912 : secondary reload memory again and again. */
1913 838 : if (CONSTANT_P (reg) && CONST_POOL_OK_P (reg_mode, reg)
1914 3796522 : && SCALAR_INT_MODE_P (reg_mode) != SCALAR_INT_MODE_P (mode))
1915 : {
1916 8 : SUBREG_REG (operand) = force_const_mem (reg_mode, reg);
1917 8 : alter_subreg (curr_id->operand_loc[nop], false);
1918 8 : return true;
1919 : }
1920 3796095 : auto fp_subreg_can_be_simplified_after_reload_p = [] (machine_mode innermode,
1921 : poly_uint64 offset,
1922 : machine_mode mode) {
1923 0 : reload_completed = 1;
1924 0 : bool res = simplify_subreg_regno (FRAME_POINTER_REGNUM,
1925 : innermode,
1926 0 : offset, mode) >= 0;
1927 0 : reload_completed = 0;
1928 0 : return res;
1929 : };
1930 : /* Force a reload of the SUBREG_REG if this is a constant or PLUS or
1931 : if there may be a problem accessing OPERAND in the outer
1932 : mode. */
1933 3796095 : if ((REG_P (reg)
1934 3795634 : && REGNO (reg) >= FIRST_PSEUDO_REGISTER
1935 3795634 : && (hard_regno = lra_get_regno_hard_regno (REGNO (reg))) >= 0
1936 : /* Don't reload paradoxical subregs because we could be looping
1937 : having repeatedly final regno out of hard regs range. */
1938 3213761 : && (hard_regno_nregs (hard_regno, innermode)
1939 3213761 : >= hard_regno_nregs (hard_regno, mode))
1940 3208441 : && simplify_subreg_regno (hard_regno, innermode,
1941 3208441 : SUBREG_BYTE (operand), mode) < 0
1942 : /* Exclude reloading of frame pointer in subreg if frame pointer can not
1943 : be simplified here only because the reload is not finished yet. */
1944 864 : && (hard_regno != FRAME_POINTER_REGNUM
1945 461 : || !fp_subreg_can_be_simplified_after_reload_p (innermode,
1946 0 : SUBREG_BYTE (operand),
1947 : mode))
1948 : /* Don't reload subreg for matching reload. It is actually
1949 : valid subreg in LRA. */
1950 864 : && ! LRA_SUBREG_P (operand))
1951 7591729 : || CONSTANT_P (reg) || GET_CODE (reg) == PLUS || MEM_P (reg))
1952 : {
1953 461 : enum reg_class rclass;
1954 :
1955 461 : if (REG_P (reg))
1956 : /* There is a big probability that we will get the same class
1957 : for the new pseudo and we will get the same insn which
1958 : means infinite looping. So spill the new pseudo. */
1959 : rclass = NO_REGS;
1960 : else
1961 : /* The class will be defined later in curr_insn_transform. */
1962 461 : rclass
1963 461 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
1964 :
1965 461 : if (get_reload_reg (curr_static_id->operand[nop].type, reg_mode, reg,
1966 : rclass, NULL,
1967 : true, false, "subreg reg", &new_reg))
1968 : {
1969 461 : bool insert_before, insert_after;
1970 461 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
1971 :
1972 922 : insert_before = (type != OP_OUT
1973 461 : || read_modify_subreg_p (operand));
1974 461 : insert_after = (type != OP_IN);
1975 922 : insert_move_for_subreg (insert_before ? &before : NULL,
1976 : insert_after ? &after : NULL,
1977 : reg, new_reg);
1978 : }
1979 461 : SUBREG_REG (operand) = new_reg;
1980 461 : lra_process_new_insns (curr_insn, before, after,
1981 : "Inserting subreg reload");
1982 461 : return true;
1983 : }
1984 : /* Force a reload for a paradoxical subreg. For paradoxical subreg,
1985 : IRA allocates hardreg to the inner pseudo reg according to its mode
1986 : instead of the outermode, so the size of the hardreg may not be enough
1987 : to contain the outermode operand, in that case we may need to insert
1988 : reload for the reg. For the following two types of paradoxical subreg,
1989 : we need to insert reload:
1990 : 1. If the op_type is OP_IN, and the hardreg could not be paired with
1991 : other hardreg to contain the outermode operand
1992 : (checked by in_hard_reg_set_p), we need to insert the reload.
1993 : 2. If the op_type is OP_OUT or OP_INOUT.
1994 :
1995 : Here is a paradoxical subreg example showing how the reload is generated:
1996 :
1997 : (insn 5 4 7 2 (set (reg:TI 106 [ __comp ])
1998 : (subreg:TI (reg:DI 107 [ __comp ]) 0)) {*movti_internal_rex64}
1999 :
2000 : In IRA, reg107 is allocated to a DImode hardreg. We use x86-64 as example
2001 : here, if reg107 is assigned to hardreg R15, because R15 is the last
2002 : hardreg, compiler cannot find another hardreg to pair with R15 to
2003 : contain TImode data. So we insert a TImode reload reg180 for it.
2004 : After reload is inserted:
2005 :
2006 : (insn 283 0 0 (set (subreg:DI (reg:TI 180 [orig:107 __comp ] [107]) 0)
2007 : (reg:DI 107 [ __comp ])) -1
2008 : (insn 5 4 7 2 (set (reg:TI 106 [ __comp ])
2009 : (subreg:TI (reg:TI 180 [orig:107 __comp ] [107]) 0)) {*movti_internal_rex64}
2010 :
2011 : Two reload hard registers will be allocated to reg180 to save TImode data
2012 : in LRA_assign.
2013 :
2014 : For LRA pseudos this should normally be handled by the biggest_mode
2015 : mechanism. However, it's possible for new uses of an LRA pseudo
2016 : to be introduced after we've allocated it, such as when undoing
2017 : inheritance, and the allocated register might not then be appropriate
2018 : for the new uses. */
2019 3795634 : else if (REG_P (reg)
2020 3795634 : && REGNO (reg) >= FIRST_PSEUDO_REGISTER
2021 3795634 : && paradoxical_subreg_p (operand)
2022 1041318 : && (inner_hard_regno = lra_get_regno_hard_regno (REGNO (reg))) >= 0
2023 958916 : && hard_regno_nregs (inner_hard_regno, mode) > 1
2024 3795634 : && ((hard_regno
2025 3800954 : = simplify_subreg_regno (inner_hard_regno, innermode,
2026 5320 : SUBREG_BYTE (operand), mode)) < 0
2027 5320 : || ((hard_regno_nregs (inner_hard_regno, innermode)
2028 5320 : < hard_regno_nregs (hard_regno, mode))
2029 10640 : && (regclass = lra_get_allocno_class (REGNO (reg)))
2030 5320 : && (type != OP_IN
2031 5320 : || !in_hard_reg_set_p (reg_class_contents[regclass],
2032 : mode, hard_regno)
2033 5320 : || overlaps_hard_reg_set_p (lra_no_alloc_regs,
2034 : mode, hard_regno)))))
2035 : {
2036 : /* The class will be defined later in curr_insn_transform. */
2037 0 : enum reg_class rclass
2038 0 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
2039 :
2040 0 : if (get_reload_reg (curr_static_id->operand[nop].type, mode, reg,
2041 : rclass, NULL,
2042 : true, false, "paradoxical subreg", &new_reg))
2043 : {
2044 0 : rtx subreg;
2045 0 : bool insert_before, insert_after;
2046 :
2047 0 : PUT_MODE (new_reg, mode);
2048 0 : subreg = gen_lowpart_SUBREG (innermode, new_reg);
2049 0 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
2050 :
2051 0 : insert_before = (type != OP_OUT);
2052 0 : insert_after = (type != OP_IN);
2053 0 : insert_move_for_subreg (insert_before ? &before : NULL,
2054 : insert_after ? &after : NULL,
2055 : reg, subreg);
2056 : }
2057 0 : SUBREG_REG (operand) = new_reg;
2058 0 : lra_process_new_insns (curr_insn, before, after,
2059 : "Inserting paradoxical subreg reload");
2060 0 : return true;
2061 : }
2062 : return false;
2063 : }
2064 :
2065 : /* Return TRUE if X refers for a hard register from SET. */
2066 : static bool
2067 410314 : uses_hard_regs_p (rtx x, HARD_REG_SET set)
2068 : {
2069 410314 : int i, j, x_hard_regno;
2070 410314 : machine_mode mode;
2071 410314 : const char *fmt;
2072 410314 : enum rtx_code code;
2073 :
2074 410314 : if (x == NULL_RTX)
2075 : return false;
2076 410314 : code = GET_CODE (x);
2077 410314 : mode = GET_MODE (x);
2078 :
2079 410314 : if (code == SUBREG)
2080 : {
2081 : /* For all SUBREGs we want to check whether the full multi-register
2082 : overlaps the set. For normal SUBREGs this means 'get_hard_regno' of
2083 : the inner register, for paradoxical SUBREGs this means the
2084 : 'get_hard_regno' of the full SUBREG and for complete SUBREGs either is
2085 : fine. Use the wider mode for all cases. */
2086 10230 : rtx subreg = SUBREG_REG (x);
2087 10230 : mode = wider_subreg_mode (x);
2088 10230 : if (mode == GET_MODE (subreg))
2089 : {
2090 9195 : x = subreg;
2091 9195 : code = GET_CODE (x);
2092 : }
2093 : }
2094 :
2095 410314 : if (REG_P (x) || SUBREG_P (x))
2096 : {
2097 264051 : x_hard_regno = get_hard_regno (x);
2098 264051 : return (x_hard_regno >= 0
2099 264051 : && overlaps_hard_reg_set_p (set, mode, x_hard_regno));
2100 : }
2101 146263 : fmt = GET_RTX_FORMAT (code);
2102 378811 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
2103 : {
2104 234561 : if (fmt[i] == 'e')
2105 : {
2106 113650 : if (uses_hard_regs_p (XEXP (x, i), set))
2107 : return true;
2108 : }
2109 120911 : else if (fmt[i] == 'E')
2110 : {
2111 4396 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
2112 3979 : if (uses_hard_regs_p (XVECEXP (x, i, j), set))
2113 : return true;
2114 : }
2115 : }
2116 : return false;
2117 : }
2118 :
2119 : /* Return true if OP is a spilled pseudo. */
2120 : static inline bool
2121 82864600 : spilled_pseudo_p (rtx op)
2122 : {
2123 82864600 : return (REG_P (op)
2124 82864600 : && REGNO (op) >= FIRST_PSEUDO_REGISTER && in_mem_p (REGNO (op)));
2125 : }
2126 :
2127 : /* Return true if X is a general constant. */
2128 : static inline bool
2129 7959086 : general_constant_p (rtx x)
2130 : {
2131 7959086 : return CONSTANT_P (x) && (! flag_pic || LEGITIMATE_PIC_OPERAND_P (x));
2132 : }
2133 :
2134 : static bool
2135 25669623 : reg_in_class_p (rtx reg, enum reg_class cl)
2136 : {
2137 25669623 : if (cl == NO_REGS)
2138 1146536 : return get_reg_class (REGNO (reg)) == NO_REGS;
2139 24523087 : return in_class_p (reg, cl, NULL);
2140 : }
2141 :
2142 : /* Return true if SET of RCLASS contains no hard regs which can be
2143 : used in MODE. */
2144 : static bool
2145 3930419 : prohibited_class_reg_set_mode_p (enum reg_class rclass,
2146 : HARD_REG_SET &set,
2147 : machine_mode mode)
2148 : {
2149 3930419 : HARD_REG_SET temp;
2150 :
2151 7860838 : lra_assert (hard_reg_set_subset_p (reg_class_contents[rclass], set));
2152 3930419 : temp = set & ~lra_no_alloc_regs;
2153 3930419 : return (hard_reg_set_subset_p
2154 3930419 : (temp, ira_prohibited_class_mode_regs[rclass][mode]));
2155 : }
2156 :
2157 :
2158 : /* Used to check validity info about small class input operands. It
2159 : should be incremented at start of processing an insn
2160 : alternative. */
2161 : static unsigned int curr_small_class_check = 0;
2162 :
2163 : /* Update number of used inputs of class OP_CLASS for operand NOP
2164 : of alternative NALT. Return true if we have more such class operands
2165 : than the number of available regs. */
2166 : static bool
2167 403756563 : update_and_check_small_class_inputs (int nop, int nalt,
2168 : enum reg_class op_class)
2169 : {
2170 403756563 : static unsigned int small_class_check[LIM_REG_CLASSES];
2171 403756563 : static int small_class_input_nums[LIM_REG_CLASSES];
2172 :
2173 400756618 : if (SMALL_REGISTER_CLASS_P (op_class)
2174 : /* We are interesting in classes became small because of fixing
2175 : some hard regs, e.g. by an user through GCC options. */
2176 3127894 : && hard_reg_set_intersect_p (reg_class_contents[op_class],
2177 3127894 : ira_no_alloc_regs)
2178 403756601 : && (curr_static_id->operand[nop].type != OP_OUT
2179 32 : || TEST_BIT (curr_static_id->operand[nop].early_clobber_alts, nalt)))
2180 : {
2181 6 : if (small_class_check[op_class] == curr_small_class_check)
2182 0 : small_class_input_nums[op_class]++;
2183 : else
2184 : {
2185 6 : small_class_check[op_class] = curr_small_class_check;
2186 6 : small_class_input_nums[op_class] = 1;
2187 : }
2188 6 : if (small_class_input_nums[op_class] > ira_class_hard_regs_num[op_class])
2189 0 : return true;
2190 : }
2191 : return false;
2192 : }
2193 :
2194 : /* Print operand constraints for alternative ALT_NUMBER of the current
2195 : insn. */
2196 : static void
2197 4590 : print_curr_insn_alt (int alt_number)
2198 : {
2199 15917 : for (int i = 0; i < curr_static_id->n_operands; i++)
2200 : {
2201 11327 : const char *p = (curr_static_id->operand_alternative
2202 11327 : [alt_number * curr_static_id->n_operands + i].constraint);
2203 11327 : if (*p == '\0')
2204 220 : continue;
2205 11107 : fprintf (lra_dump_file, " (%d) ", i);
2206 39435 : for (; *p != '\0' && *p != ',' && *p != '#'; p++)
2207 17221 : fputc (*p, lra_dump_file);
2208 : }
2209 4590 : }
2210 :
2211 : struct dependent_filter_cache_hasher
2212 : : free_ptr_hash <dependent_filter_entry>
2213 : {
2214 0 : static inline hashval_t hash (const dependent_filter_entry *e)
2215 : {
2216 0 : hashval_t h = (hashval_t) e->id;
2217 0 : h = iterative_hash_hashval_t ((hashval_t) e->mode, h);
2218 0 : h = iterative_hash_hashval_t ((hashval_t) e->partner_mode, h);
2219 0 : h = iterative_hash_hashval_t (e->partner_regno, h);
2220 0 : h = iterative_hash_hashval_t ((hashval_t) e->is_ref, h);
2221 0 : return h;
2222 : }
2223 0 : static inline bool equal (const dependent_filter_entry *a,
2224 : const dependent_filter_entry *b)
2225 : {
2226 0 : return (a->id == b->id && a->mode == b->mode
2227 0 : && a->partner_mode == b->partner_mode
2228 0 : && a->partner_regno == b->partner_regno
2229 0 : && a->is_ref == b->is_ref);
2230 : }
2231 : };
2232 :
2233 : static hash_table<dependent_filter_cache_hasher> *dependent_filter_htab;
2234 :
2235 : /* Allocate the dependent-filter cache. */
2236 :
2237 : void
2238 215919 : lra_init_dependent_filter_cache (void)
2239 : {
2240 215919 : gcc_assert (!dependent_filter_htab);
2241 215919 : dependent_filter_htab = new hash_table<dependent_filter_cache_hasher> (16);
2242 215919 : }
2243 :
2244 : /* Release the dependent-filter cache. */
2245 :
2246 : void
2247 284694 : lra_finish_dependent_filter_cache (void)
2248 : {
2249 284694 : delete dependent_filter_htab;
2250 284694 : dependent_filter_htab = nullptr;
2251 284694 : }
2252 :
2253 : /* Reset the dependent-filter cache. */
2254 :
2255 : void
2256 3030242 : lra_reset_dependent_filters (void)
2257 : {
2258 3030242 : if (dependent_filter_htab)
2259 3030242 : dependent_filter_htab->empty ();
2260 3030242 : }
2261 :
2262 : /* Return the set of hardregs allowed by dependent filter ID for an operand
2263 : of mode MODE when the referenced operand has PARTNER_REGNO in
2264 : PARTNER_MODE. That is when IS_REF is false. If it is true, the roles are
2265 : reversed and PARTNER_REGNO as well as PARTNER_MODE refer to the dependent
2266 : operand.
2267 :
2268 : We fill the set for all hardregs and add the resulting regset to
2269 : a hash table if it doesn't already exist in it. */
2270 :
2271 : const HARD_REG_SET *
2272 0 : lra_get_dependent_filter (int id, machine_mode mode,
2273 : unsigned int partner_regno,
2274 : machine_mode partner_mode, bool is_ref)
2275 : {
2276 0 : dependent_filter_entry key;
2277 0 : key.id = id;
2278 0 : key.mode = mode;
2279 0 : key.partner_mode = partner_mode;
2280 0 : key.partner_regno = partner_regno;
2281 0 : key.is_ref = is_ref;
2282 :
2283 0 : dependent_filter_entry **slot
2284 0 : = dependent_filter_htab->find_slot (&key, INSERT);
2285 0 : if (*slot)
2286 0 : return &(*slot)->allowed;
2287 :
2288 0 : auto *e = XCNEW (dependent_filter_entry);
2289 0 : e->id = id;
2290 0 : e->mode = mode;
2291 0 : e->partner_mode = partner_mode;
2292 0 : e->partner_regno = partner_regno;
2293 0 : e->is_ref = is_ref;
2294 :
2295 : /* ??? Should we restrict ourselves to a register class here? */
2296 0 : for (unsigned regno = 0; regno < FIRST_PSEUDO_REGISTER; ++regno)
2297 : {
2298 0 : bool ok;
2299 0 : if (is_ref)
2300 0 : ok = eval_dependent_filter (id, partner_regno, partner_mode,
2301 : regno, mode);
2302 : else
2303 0 : ok = eval_dependent_filter (id, regno, mode,
2304 : partner_regno, partner_mode);
2305 0 : if (ok)
2306 0 : SET_HARD_REG_BIT (e->allowed, regno);
2307 : }
2308 0 : *slot = e;
2309 0 : return &e->allowed;
2310 : }
2311 :
2312 : /* Add the dependent filter ID to REGNO's dependent filters with
2313 : referenced PARTNER_REGNO and PARTNER_MODE if IS_REF is false.
2314 : If IS_REF is true, the roles are reversed. We need both
2315 : directions in case the referenced op is assigned a hardreg first. */
2316 :
2317 : void
2318 0 : lra_add_dependent_filter (int regno, int id, machine_mode mode,
2319 : int partner_regno, machine_mode partner_mode,
2320 : bool is_ref)
2321 : {
2322 : /* Only meaningful for pseudo partners. */
2323 0 : if (partner_regno < FIRST_PSEUDO_REGISTER)
2324 0 : return;
2325 :
2326 0 : vec<dependent_filter> &filters = lra_reg_info[regno].dependent_filters;
2327 0 : unsigned int i;
2328 0 : dependent_filter *filter;
2329 :
2330 : /* If this filter already exists, do nothing. */
2331 0 : FOR_EACH_VEC_ELT (filters, i, filter)
2332 0 : if (filter->id == id
2333 0 : && filter->partner_regno == (unsigned int) partner_regno
2334 0 : && filter->mode == mode && filter->partner_mode == partner_mode
2335 0 : && filter->is_ref == is_ref)
2336 : return;
2337 :
2338 : /* Otherwise, create a new filter. */
2339 0 : dependent_filter new_filter;
2340 0 : new_filter.id = id;
2341 0 : new_filter.mode = mode;
2342 0 : new_filter.partner_mode = partner_mode;
2343 0 : new_filter.partner_regno = (unsigned int) partner_regno;
2344 0 : new_filter.is_ref = is_ref;
2345 0 : filters.safe_push (new_filter);
2346 : }
2347 :
2348 : /* Return the set of hardregs for the constraint CN in MODE,
2349 : allowed by its dependent filter.
2350 : If there is no dependent filter or the involved registers are no
2351 : hardregs, return NULL. */
2352 :
2353 : static const HARD_REG_SET *
2354 0 : get_dependent_filter (constraint_num cn, machine_mode mode)
2355 : {
2356 353081828 : int id = get_dependent_filter_id (cn);
2357 0 : if (id < 0)
2358 0 : return nullptr;
2359 :
2360 : gcc_assert (reg_class_for_constraint (cn) != NO_REGS);
2361 :
2362 : int ref_opno = get_dependent_filter_ref (id);
2363 : gcc_assert (ref_opno >= 0 && ref_opno < curr_static_id->n_operands);
2364 :
2365 : rtx ref_op = *curr_id->operand_loc[ref_opno];
2366 : if (SUBREG_P (ref_op))
2367 : ref_op = SUBREG_REG (ref_op);
2368 : if (!REG_P (ref_op))
2369 : return nullptr;
2370 : unsigned int ref_regno = REGNO (ref_op);
2371 : if (ref_regno >= FIRST_PSEUDO_REGISTER)
2372 : {
2373 : int ref_hard_regno = reg_renumber[ref_regno];
2374 : /* Even with a pseudo reference op, the filter can still reject
2375 : based on the partner. We call it with INVALID_REGNUM
2376 : to give it a chance to do so. Otherwise we'd introduce
2377 : an "all choices legal" filter that might later
2378 : "change its mind" once there is a fixed reference. */
2379 : if (ref_hard_regno < 0)
2380 : return lra_get_dependent_filter (id, mode, INVALID_REGNUM,
2381 : GET_MODE (ref_op), false);
2382 : ref_regno = (unsigned int) ref_hard_regno;
2383 : }
2384 :
2385 : return lra_get_dependent_filter (id, mode, ref_regno, GET_MODE (ref_op),
2386 : false);
2387 : }
2388 :
2389 : /* Go through all operands and their constraints, looking for a dependent
2390 : filter. If we find one, add it to the respective reg_info's
2391 : dependent-filter list. After assigning a hardreg to either the dependent
2392 : or the referenced op, this allows us to filter the other's regset. */
2393 :
2394 : static void
2395 81266859 : process_dependent_filters (void)
2396 : {
2397 81266859 : int n_operands = curr_static_id->n_operands;
2398 262532280 : for (int i = 0; i < n_operands; i++)
2399 : {
2400 181265421 : rtx op = *curr_id->operand_loc[i];
2401 181265421 : if (!REG_P (op) || HARD_REGISTER_P (op))
2402 99152944 : continue;
2403 82112477 : const char *constraint
2404 82112477 : = curr_static_id->operand_alternative
2405 82112477 : [goal_alt_number * n_operands + i].constraint;
2406 82112477 : for (const char *p = constraint;
2407 215716506 : *p && *p != ',' && *p != '#';
2408 133604029 : p += CONSTRAINT_LEN (*p, p))
2409 : {
2410 133604029 : enum constraint_num cn = lookup_constraint (p);
2411 133604029 : int id = get_dependent_filter_id (cn);
2412 133604029 : if (id < 0)
2413 133604029 : continue;
2414 : if (reg_class_for_constraint (cn) == NO_REGS)
2415 : continue;
2416 : int ref_opno = get_dependent_filter_ref (id);
2417 : if (ref_opno < 0 || ref_opno >= n_operands)
2418 : continue;
2419 : rtx ref_op = *curr_id->operand_loc[ref_opno];
2420 : if (!REG_P (ref_op) || HARD_REGISTER_P (ref_op))
2421 : continue;
2422 : machine_mode mode = curr_operand_mode[i];
2423 : lra_add_dependent_filter (REGNO (op), id, mode,
2424 : REGNO (ref_op), GET_MODE (ref_op),
2425 : false);
2426 : lra_add_dependent_filter (REGNO (ref_op), id, GET_MODE (ref_op),
2427 : REGNO (op), mode, true);
2428 : }
2429 : }
2430 81266859 : }
2431 :
2432 : /* Major function to choose the current insn alternative and what
2433 : operands should be reloaded and how. If ONLY_ALTERNATIVE is not
2434 : negative we should consider only this alternative. Return false if
2435 : we cannot choose the alternative or find how to reload the
2436 : operands. */
2437 : static bool
2438 91892083 : process_alt_operands (int only_alternative)
2439 : {
2440 91892083 : bool ok_p = false;
2441 91892083 : int nop, overall, nalt;
2442 91892083 : int n_alternatives = curr_static_id->n_alternatives;
2443 91892083 : int n_operands = curr_static_id->n_operands;
2444 : /* LOSERS counts the operands that don't fit this alternative and
2445 : would require loading. */
2446 91892083 : int losers;
2447 91892083 : int addr_losers;
2448 : /* REJECT is a count of how undesirable this alternative says it is
2449 : if any reloading is required. If the alternative matches exactly
2450 : then REJECT is ignored, but otherwise it gets this much counted
2451 : against it in addition to the reloading needed. */
2452 91892083 : int reject;
2453 : /* This is defined by '!' or '?' alternative constraint and added to
2454 : reject. But in some cases it can be ignored. */
2455 91892083 : int static_reject;
2456 91892083 : int op_reject;
2457 : /* The number of elements in the following array. */
2458 91892083 : int early_clobbered_regs_num;
2459 : /* Numbers of operands which are early clobber registers. */
2460 91892083 : int early_clobbered_nops[MAX_RECOG_OPERANDS];
2461 91892083 : enum reg_class curr_alt[MAX_RECOG_OPERANDS];
2462 91892083 : enum reg_class all_this_alternative;
2463 91892083 : int all_used_nregs, all_reload_nregs;
2464 91892083 : HARD_REG_SET curr_alt_set[MAX_RECOG_OPERANDS];
2465 91892083 : HARD_REG_SET curr_alt_exclude_start_hard_regs[MAX_RECOG_OPERANDS];
2466 91892083 : bool curr_alt_match_win[MAX_RECOG_OPERANDS];
2467 91892083 : bool curr_alt_win[MAX_RECOG_OPERANDS];
2468 91892083 : bool curr_alt_offmemok[MAX_RECOG_OPERANDS];
2469 91892083 : int curr_alt_matches[MAX_RECOG_OPERANDS];
2470 : /* The number of elements in the following array. */
2471 91892083 : int curr_alt_dont_inherit_ops_num;
2472 : /* Numbers of operands whose reload pseudos should not be inherited. */
2473 91892083 : int curr_alt_dont_inherit_ops[MAX_RECOG_OPERANDS];
2474 91892083 : bool curr_reuse_alt_p;
2475 : /* True if output stack pointer reload should be generated for the current
2476 : alternative. */
2477 91892083 : bool curr_alt_out_sp_reload_p;
2478 91892083 : bool curr_alt_class_change_p;
2479 91892083 : rtx op;
2480 : /* The register when the operand is a subreg of register, otherwise the
2481 : operand itself. */
2482 91892083 : rtx no_subreg_reg_operand[MAX_RECOG_OPERANDS];
2483 : /* The register if the operand is a register or subreg of register,
2484 : otherwise NULL. */
2485 91892083 : rtx operand_reg[MAX_RECOG_OPERANDS];
2486 91892083 : int hard_regno[MAX_RECOG_OPERANDS];
2487 91892083 : machine_mode biggest_mode[MAX_RECOG_OPERANDS];
2488 91892083 : int reload_nregs, reload_sum;
2489 91892083 : bool costly_p;
2490 91892083 : enum reg_class cl;
2491 91892083 : const HARD_REG_SET *cl_filter;
2492 91892083 : HARD_REG_SET hard_reg_constraint;
2493 :
2494 : /* Calculate some data common for all alternatives to speed up the
2495 : function. */
2496 305551294 : for (nop = 0; nop < n_operands; nop++)
2497 : {
2498 213659211 : rtx reg;
2499 :
2500 213659211 : op = no_subreg_reg_operand[nop] = *curr_id->operand_loc[nop];
2501 : /* The real hard regno of the operand after the allocation. */
2502 213659211 : hard_regno[nop] = get_hard_regno (op);
2503 :
2504 213659211 : operand_reg[nop] = reg = op;
2505 213659211 : biggest_mode[nop] = GET_MODE (op);
2506 213659211 : if (GET_CODE (op) == SUBREG)
2507 : {
2508 4316283 : biggest_mode[nop] = wider_subreg_mode (op);
2509 4316283 : operand_reg[nop] = reg = SUBREG_REG (op);
2510 : }
2511 213659211 : if (! REG_P (reg))
2512 90254726 : operand_reg[nop] = NULL_RTX;
2513 123404485 : else if (REGNO (reg) >= FIRST_PSEUDO_REGISTER
2514 144817019 : || ((int) REGNO (reg)
2515 21412534 : == lra_get_elimination_hard_regno (REGNO (reg))))
2516 120335025 : no_subreg_reg_operand[nop] = reg;
2517 : else
2518 3069460 : operand_reg[nop] = no_subreg_reg_operand[nop]
2519 : /* Just use natural mode for elimination result. It should
2520 : be enough for extra constraints hooks. */
2521 3069460 : = regno_reg_rtx[hard_regno[nop]];
2522 : }
2523 :
2524 : /* The constraints are made of several alternatives. Each operand's
2525 : constraint looks like foo,bar,... with commas separating the
2526 : alternatives. The first alternatives for all operands go
2527 : together, the second alternatives go together, etc.
2528 :
2529 : First loop over alternatives. */
2530 91892083 : alternative_mask preferred = curr_id->preferred_alternatives;
2531 91892083 : if (only_alternative >= 0)
2532 983077 : preferred &= ALTERNATIVE_BIT (only_alternative);
2533 :
2534 91892083 : bool prefer_memory_p = false;
2535 91892197 : repeat:
2536 389123368 : for (nalt = 0; nalt < n_alternatives; nalt++)
2537 : {
2538 : /* Loop over operands for one constraint alternative. */
2539 375075250 : if (!TEST_BIT (preferred, nalt))
2540 111489221 : continue;
2541 :
2542 263586029 : if (lra_dump_file != NULL)
2543 : {
2544 3403 : fprintf (lra_dump_file, " Considering alt=%d of insn %d: ",
2545 3403 : nalt, INSN_UID (curr_insn));
2546 3403 : print_curr_insn_alt (nalt);
2547 3403 : fprintf (lra_dump_file, "\n");
2548 : }
2549 :
2550 263586029 : bool matching_early_clobber[MAX_RECOG_OPERANDS];
2551 263586029 : curr_small_class_check++;
2552 263586029 : overall = losers = addr_losers = 0;
2553 263586029 : static_reject = reject = reload_nregs = reload_sum = 0;
2554 873961276 : for (nop = 0; nop < n_operands; nop++)
2555 : {
2556 610375247 : int inc = (curr_static_id
2557 610375247 : ->operand_alternative[nalt * n_operands + nop].reject);
2558 610375247 : if (lra_dump_file != NULL && inc != 0)
2559 53 : fprintf (lra_dump_file,
2560 : " Statically defined alt reject+=%d\n", inc);
2561 610375247 : static_reject += inc;
2562 610375247 : matching_early_clobber[nop] = 0;
2563 : }
2564 : reject += static_reject;
2565 : early_clobbered_regs_num = 0;
2566 : curr_alt_out_sp_reload_p = false;
2567 : curr_reuse_alt_p = true;
2568 : curr_alt_class_change_p = false;
2569 : all_this_alternative = NO_REGS;
2570 : all_used_nregs = all_reload_nregs = 0;
2571 682476335 : for (nop = 0; nop < n_operands; nop++)
2572 : {
2573 544287298 : const char *p;
2574 544287298 : char *end;
2575 544287298 : int len, c, m, i, opalt_num, this_alternative_matches;
2576 544287298 : bool win, did_match, offmemok, early_clobber_p;
2577 : /* false => this operand can be reloaded somehow for this
2578 : alternative. */
2579 544287298 : bool badop;
2580 : /* true => this operand can be reloaded if the alternative
2581 : allows regs. */
2582 544287298 : bool winreg;
2583 : /* True if a constant forced into memory would be OK for
2584 : this operand. */
2585 544287298 : bool constmemok;
2586 544287298 : enum reg_class this_alternative, this_costly_alternative;
2587 544287298 : HARD_REG_SET this_alternative_set, this_costly_alternative_set;
2588 544287298 : HARD_REG_SET this_alternative_exclude_start_hard_regs;
2589 544287298 : bool this_alternative_match_win, this_alternative_win;
2590 544287298 : bool this_alternative_offmemok;
2591 544287298 : bool scratch_p;
2592 544287298 : machine_mode mode;
2593 544287298 : enum constraint_num cn;
2594 544287298 : bool class_change_p = false;
2595 :
2596 544287298 : opalt_num = nalt * n_operands + nop;
2597 544287298 : if (curr_static_id->operand_alternative[opalt_num].anything_ok)
2598 : {
2599 : /* Fast track for no constraints at all. */
2600 15133743 : curr_alt[nop] = NO_REGS;
2601 15133743 : CLEAR_HARD_REG_SET (curr_alt_set[nop]);
2602 15133743 : curr_alt_win[nop] = true;
2603 15133743 : curr_alt_match_win[nop] = false;
2604 15133743 : curr_alt_offmemok[nop] = false;
2605 15133743 : curr_alt_matches[nop] = -1;
2606 15133743 : continue;
2607 : }
2608 :
2609 529153555 : op = no_subreg_reg_operand[nop];
2610 529153555 : mode = curr_operand_mode[nop];
2611 :
2612 529153555 : win = did_match = winreg = offmemok = constmemok = false;
2613 529153555 : badop = true;
2614 529153555 : const HARD_REG_SET *cl_dep_filter = nullptr;
2615 :
2616 529153555 : early_clobber_p = false;
2617 529153555 : p = curr_static_id->operand_alternative[opalt_num].constraint;
2618 :
2619 529153555 : this_costly_alternative = this_alternative = NO_REGS;
2620 : /* We update set of possible hard regs besides its class
2621 : because reg class might be inaccurate. For example,
2622 : union of LO_REGS (l), HI_REGS(h), and STACK_REG(k) in ARM
2623 : is translated in HI_REGS because classes are merged by
2624 : pairs and there is no accurate intermediate class. */
2625 2116614220 : CLEAR_HARD_REG_SET (this_alternative_set);
2626 1587460665 : CLEAR_HARD_REG_SET (this_costly_alternative_set);
2627 529153555 : CLEAR_HARD_REG_SET (this_alternative_exclude_start_hard_regs);
2628 529153555 : this_alternative_win = false;
2629 529153555 : this_alternative_match_win = false;
2630 529153555 : this_alternative_offmemok = false;
2631 529153555 : this_alternative_matches = -1;
2632 :
2633 : /* An empty constraint should be excluded by the fast
2634 : track. */
2635 529153555 : lra_assert (*p != 0 && *p != ',');
2636 :
2637 : op_reject = 0;
2638 : /* Scan this alternative's specs for this operand; set WIN
2639 : if the operand fits any letter in this alternative.
2640 : Otherwise, clear BADOP if this operand could fit some
2641 : letter after reloads, or set WINREG if this operand could
2642 : fit after reloads provided the constraint allows some
2643 : registers. */
2644 : costly_p = false;
2645 1343732062 : do
2646 : {
2647 1343732062 : switch ((c = *p, len = CONSTRAINT_LEN (c, p)), c)
2648 : {
2649 : case '\0':
2650 : len = 0;
2651 : break;
2652 505785109 : case ',':
2653 505785109 : c = '\0';
2654 505785109 : break;
2655 :
2656 191472 : case '&':
2657 191472 : early_clobber_p = true;
2658 191472 : break;
2659 :
2660 19750 : case '$':
2661 19750 : op_reject += LRA_MAX_REJECT;
2662 19750 : break;
2663 0 : case '^':
2664 0 : op_reject += LRA_LOSER_COST_FACTOR;
2665 0 : break;
2666 :
2667 0 : case '#':
2668 : /* Ignore rest of this alternative. */
2669 0 : c = '\0';
2670 0 : break;
2671 :
2672 57515032 : case '0': case '1': case '2': case '3': case '4':
2673 57515032 : case '5': case '6': case '7': case '8': case '9':
2674 57515032 : {
2675 57515032 : int m_hregno;
2676 57515032 : bool match_p;
2677 :
2678 57515032 : m = strtoul (p, &end, 10);
2679 57515032 : p = end;
2680 57515032 : len = 0;
2681 57515032 : lra_assert (nop > m);
2682 :
2683 : /* Reject matches if we don't know which operand is
2684 : bigger. This situation would arguably be a bug in
2685 : an .md pattern, but could also occur in a user asm. */
2686 172545096 : if (!ordered_p (GET_MODE_SIZE (biggest_mode[m]),
2687 57515032 : GET_MODE_SIZE (biggest_mode[nop])))
2688 : break;
2689 :
2690 : /* Don't match wrong asm insn operands for proper
2691 : diagnostic later. */
2692 57515032 : if (INSN_CODE (curr_insn) < 0
2693 34958 : && (curr_operand_mode[m] == BLKmode
2694 34957 : || curr_operand_mode[nop] == BLKmode)
2695 1 : && curr_operand_mode[m] != curr_operand_mode[nop])
2696 : break;
2697 :
2698 57515031 : m_hregno = get_hard_regno (*curr_id->operand_loc[m]);
2699 : /* We are supposed to match a previous operand.
2700 : If we do, we win if that one did. If we do
2701 : not, count both of the operands as losers.
2702 : (This is too conservative, since most of the
2703 : time only a single reload insn will be needed
2704 : to make the two operands win. As a result,
2705 : this alternative may be rejected when it is
2706 : actually desirable.) */
2707 57515031 : match_p = false;
2708 57515031 : if (operands_match_p (*curr_id->operand_loc[nop],
2709 57515031 : *curr_id->operand_loc[m], m_hregno))
2710 : {
2711 : /* We should reject matching of an early
2712 : clobber operand if the matching operand is
2713 : not dying in the insn. */
2714 15094225 : if (!TEST_BIT (curr_static_id->operand[m]
2715 : .early_clobber_alts, nalt)
2716 20901 : || operand_reg[nop] == NULL_RTX
2717 15115126 : || (find_regno_note (curr_insn, REG_DEAD,
2718 : REGNO (op))
2719 4431 : || REGNO (op) == REGNO (operand_reg[m])))
2720 15094225 : match_p = true;
2721 : }
2722 15094225 : if (match_p)
2723 : {
2724 : /* If we are matching a non-offsettable
2725 : address where an offsettable address was
2726 : expected, then we must reject this
2727 : combination, because we can't reload
2728 : it. */
2729 15094225 : if (curr_alt_offmemok[m]
2730 1469 : && MEM_P (*curr_id->operand_loc[m])
2731 0 : && curr_alt[m] == NO_REGS && ! curr_alt_win[m])
2732 0 : continue;
2733 : }
2734 : else
2735 : {
2736 : /* If the operands do not match and one
2737 : operand is INOUT, we can not match them.
2738 : Try other possibilities, e.g. other
2739 : alternatives or commutative operand
2740 : exchange. */
2741 42420806 : if (curr_static_id->operand[nop].type == OP_INOUT
2742 42420806 : || curr_static_id->operand[m].type == OP_INOUT)
2743 : break;
2744 : /* Operands don't match. For asm if the operands
2745 : are different user defined explicit hard
2746 : registers, then we cannot make them match
2747 : when one is early clobber operand. */
2748 42420413 : if ((REG_P (*curr_id->operand_loc[nop])
2749 26359180 : || SUBREG_P (*curr_id->operand_loc[nop]))
2750 16651043 : && (REG_P (*curr_id->operand_loc[m])
2751 202487 : || SUBREG_P (*curr_id->operand_loc[m]))
2752 16557153 : && INSN_CODE (curr_insn) < 0)
2753 : {
2754 580 : rtx nop_reg = *curr_id->operand_loc[nop];
2755 580 : if (SUBREG_P (nop_reg))
2756 0 : nop_reg = SUBREG_REG (nop_reg);
2757 580 : rtx m_reg = *curr_id->operand_loc[m];
2758 580 : if (SUBREG_P (m_reg))
2759 0 : m_reg = SUBREG_REG (m_reg);
2760 :
2761 580 : if (REG_P (nop_reg)
2762 580 : && HARD_REGISTER_P (nop_reg)
2763 0 : && REG_USERVAR_P (nop_reg)
2764 0 : && REG_P (m_reg)
2765 0 : && HARD_REGISTER_P (m_reg)
2766 580 : && REG_USERVAR_P (m_reg))
2767 : {
2768 : int i;
2769 :
2770 0 : for (i = 0; i < early_clobbered_regs_num; i++)
2771 0 : if (m == early_clobbered_nops[i])
2772 : break;
2773 0 : if (i < early_clobbered_regs_num
2774 0 : || early_clobber_p)
2775 : break;
2776 : }
2777 : }
2778 : /* Both operands must allow a reload register,
2779 : otherwise we cannot make them match. */
2780 42420413 : if (curr_alt[m] == NO_REGS)
2781 : break;
2782 : /* Retroactively mark the operand we had to
2783 : match as a loser, if it wasn't already and
2784 : it wasn't matched to a register constraint
2785 : (e.g it might be matched by memory). */
2786 42396390 : if (curr_alt_win[m]
2787 41537699 : && (operand_reg[m] == NULL_RTX
2788 41015270 : || hard_regno[m] < 0))
2789 : {
2790 1287372 : if (lra_dump_file != NULL)
2791 9 : fprintf
2792 9 : (lra_dump_file,
2793 : " %d Matched operand reload: "
2794 : "losers++\n", m);
2795 1287372 : losers++;
2796 1287372 : reload_nregs
2797 1287372 : += (ira_reg_class_max_nregs[curr_alt[m]]
2798 1287372 : [GET_MODE (*curr_id->operand_loc[m])]);
2799 : }
2800 :
2801 : /* Prefer matching earlyclobber alternative as
2802 : it results in less hard regs required for
2803 : the insn than a non-matching earlyclobber
2804 : alternative. */
2805 42396390 : if (TEST_BIT (curr_static_id->operand[m]
2806 : .early_clobber_alts, nalt))
2807 : {
2808 19751 : if (lra_dump_file != NULL)
2809 0 : fprintf
2810 0 : (lra_dump_file,
2811 : " %d Matching earlyclobber alt:"
2812 : " reject--\n",
2813 : nop);
2814 19751 : if (!matching_early_clobber[m])
2815 : {
2816 19751 : reject--;
2817 19751 : matching_early_clobber[m] = 1;
2818 : }
2819 : }
2820 : /* Otherwise we prefer no matching
2821 : alternatives because it gives more freedom
2822 : in RA. */
2823 42376639 : else if (operand_reg[nop] == NULL_RTX
2824 42376639 : || (find_regno_note (curr_insn, REG_DEAD,
2825 16624738 : REGNO (operand_reg[nop]))
2826 : == NULL_RTX))
2827 : {
2828 37157167 : if (lra_dump_file != NULL)
2829 912 : fprintf
2830 912 : (lra_dump_file,
2831 : " %d Matching alt: reject+=2\n",
2832 : nop);
2833 37157167 : reject += 2;
2834 : }
2835 : }
2836 : /* If we have to reload this operand and some
2837 : previous operand also had to match the same
2838 : thing as this operand, we don't know how to do
2839 : that. */
2840 57490615 : if (!match_p || !curr_alt_win[m])
2841 : {
2842 89156751 : for (i = 0; i < nop; i++)
2843 46649079 : if (curr_alt_matches[i] == m)
2844 : break;
2845 42507673 : if (i < nop)
2846 : break;
2847 : }
2848 : else
2849 : did_match = true;
2850 :
2851 57490614 : this_alternative_matches = m;
2852 : /* This can be fixed with reloads if the operand
2853 : we are supposed to match can be fixed with
2854 : reloads. */
2855 57490614 : badop = false;
2856 57490614 : this_alternative = curr_alt[m];
2857 57490614 : this_alternative_set = curr_alt_set[m];
2858 57490614 : this_alternative_exclude_start_hard_regs
2859 57490614 : = curr_alt_exclude_start_hard_regs[m];
2860 57490614 : winreg = this_alternative != NO_REGS;
2861 57490614 : break;
2862 : }
2863 :
2864 11854815 : case 'g':
2865 11854815 : if (MEM_P (op)
2866 7959086 : || general_constant_p (op)
2867 16475138 : || spilled_pseudo_p (op))
2868 : win = true;
2869 11854815 : if (REG_P (op) && prefer_memory_p)
2870 : {
2871 11854815 : badop = false;
2872 11854815 : offmemok = true;
2873 : }
2874 11854815 : cl = GENERAL_REGS;
2875 11854815 : cl_filter = nullptr;
2876 11854815 : cl_dep_filter = nullptr;
2877 11854815 : goto reg;
2878 :
2879 1140 : case '{':
2880 1140 : {
2881 1140 : int regno = decode_hard_reg_constraint (p);
2882 1140 : gcc_assert (regno >= 0);
2883 1140 : cl = NO_REGS;
2884 1140 : int nregs = hard_regno_nregs (regno, mode);
2885 2280 : for (int i = 0; i < nregs; ++i)
2886 1140 : cl = reg_class_superunion[cl][REGNO_REG_CLASS (regno + i)];
2887 1140 : CLEAR_HARD_REG_SET (hard_reg_constraint);
2888 1140 : SET_HARD_REG_BIT (hard_reg_constraint, regno);
2889 1140 : cl_filter = &hard_reg_constraint;
2890 1140 : cl_dep_filter = nullptr;
2891 1140 : goto reg;
2892 : }
2893 :
2894 744996298 : default:
2895 744996298 : cn = lookup_constraint (p);
2896 744996298 : switch (get_constraint_type (cn))
2897 : {
2898 492432432 : case CT_REGISTER:
2899 492432432 : cl = reg_class_for_constraint (cn);
2900 362588185 : if (cl != NO_REGS)
2901 : {
2902 353081828 : cl_filter = get_register_filter (cn);
2903 353081828 : cl_dep_filter = get_dependent_filter (cn, mode);
2904 353081828 : goto reg;
2905 : }
2906 : break;
2907 :
2908 2202219 : case CT_CONST_INT:
2909 2202219 : if (CONST_INT_P (op)
2910 2202219 : && insn_const_int_ok_for_constraint (INTVAL (op), cn))
2911 : win = true;
2912 : break;
2913 :
2914 113402913 : case CT_MEMORY:
2915 113402913 : case CT_RELAXED_MEMORY:
2916 113402913 : if (MEM_P (op)
2917 113402913 : && satisfies_memory_constraint_p (op, cn))
2918 : win = true;
2919 77164818 : else if (spilled_pseudo_p (op))
2920 45943600 : win = true;
2921 :
2922 : /* If we didn't already win, we can reload constants
2923 : via force_const_mem or put the pseudo value into
2924 : memory, or make other memory by reloading the
2925 : address like for 'o'. */
2926 118855738 : if (CONST_POOL_OK_P (mode, op)
2927 107949921 : || MEM_P (op) || REG_P (op)
2928 : /* We can restore the equiv insn by a
2929 : reload. */
2930 114033745 : || equiv_substition_p[nop])
2931 : badop = false;
2932 : constmemok = true;
2933 : offmemok = true;
2934 : break;
2935 :
2936 1780845 : case CT_ADDRESS:
2937 : /* An asm operand with an address constraint
2938 : that doesn't satisfy address_operand has
2939 : is_address cleared, so that we don't try to
2940 : make a non-address fit. */
2941 1780845 : if (!curr_static_id->operand[nop].is_address)
2942 : break;
2943 : /* If we didn't already win, we can reload the address
2944 : into a base register. */
2945 1780826 : if (satisfies_address_constraint_p (op, cn))
2946 1780826 : win = true;
2947 1780826 : cl = base_reg_class (VOIDmode, ADDR_SPACE_GENERIC,
2948 : ADDRESS, SCRATCH);
2949 1780826 : cl_filter = nullptr;
2950 1780826 : cl_dep_filter = nullptr;
2951 1780826 : badop = false;
2952 1780826 : goto reg;
2953 :
2954 133947167 : case CT_FIXED_FORM:
2955 133947167 : if (constraint_satisfied_p (op, cn))
2956 1343732062 : win = true;
2957 : break;
2958 :
2959 1230722 : case CT_SPECIAL_MEMORY:
2960 1230722 : if (satisfies_memory_constraint_p (op, cn))
2961 : win = true;
2962 1079459 : else if (spilled_pseudo_p (op))
2963 : {
2964 1343732062 : curr_reuse_alt_p = false;
2965 1343732062 : win = true;
2966 : }
2967 : break;
2968 : }
2969 : break;
2970 :
2971 366718609 : reg:
2972 366718609 : if (mode == BLKmode)
2973 : break;
2974 366718591 : this_alternative = reg_class_subunion[this_alternative][cl];
2975 366718591 : if (hard_reg_set_subset_p (this_alternative_set,
2976 366718591 : reg_class_contents[cl]))
2977 366714966 : this_alternative_exclude_start_hard_regs
2978 366714966 : = ira_exclude_class_mode_regs[cl][mode];
2979 3625 : else if (!hard_reg_set_subset_p (reg_class_contents[cl],
2980 : this_alternative_set))
2981 3624 : this_alternative_exclude_start_hard_regs
2982 3624 : |= ira_exclude_class_mode_regs[cl][mode];
2983 366718591 : this_alternative_set |= reg_class_contents[cl];
2984 366718591 : if (cl_filter)
2985 2280 : this_alternative_exclude_start_hard_regs |= ~*cl_filter;
2986 366718591 : if (cl_dep_filter)
2987 : this_alternative_exclude_start_hard_regs |= ~*cl_dep_filter;
2988 366718591 : if (costly_p)
2989 : {
2990 21723380 : this_costly_alternative
2991 21723380 : = reg_class_subunion[this_costly_alternative][cl];
2992 21723380 : this_costly_alternative_set |= reg_class_contents[cl];
2993 : }
2994 366718591 : winreg = true;
2995 366718591 : if (REG_P (op))
2996 : {
2997 233923549 : rtx orig_op = *curr_id->operand_loc[nop];
2998 6962007 : if (GET_CODE (orig_op) == SUBREG && HARD_REGISTER_P (op)
2999 233923638 : && !targetm.hard_regno_mode_ok (REGNO (op),
3000 89 : GET_MODE(orig_op)))
3001 : break;
3002 :
3003 233923549 : tree decl;
3004 :
3005 233923549 : if (hard_regno[nop] >= 0
3006 197414323 : && in_hard_reg_set_p (this_alternative_set,
3007 : mode, hard_regno[nop])
3008 178632874 : && (!cl_filter
3009 590 : || TEST_HARD_REG_BIT (*cl_filter,
3010 : hard_regno[nop]))
3011 : && (!cl_dep_filter
3012 : || TEST_HARD_REG_BIT (*cl_dep_filter,
3013 : hard_regno[nop]))
3014 412556417 : && ((REG_ATTRS (op) && (decl = REG_EXPR (op)) != NULL
3015 99478235 : && VAR_P (decl) && DECL_HARD_REGISTER (decl))
3016 178629612 : || !(TEST_HARD_REG_BIT
3017 178629612 : (this_alternative_exclude_start_hard_regs,
3018 : hard_regno[nop]))))
3019 : win = true;
3020 55290685 : else if (hard_regno[nop] < 0 && !prefer_memory_p)
3021 : {
3022 36509052 : if (in_class_p (op, this_alternative, NULL))
3023 : win = true;
3024 27358088 : else if (in_class_p (op, this_alternative, NULL, true))
3025 : {
3026 1343732062 : class_change_p = true;
3027 1343732062 : win = true;
3028 : }
3029 : }
3030 : }
3031 : break;
3032 : }
3033 1343732062 : if (c != ' ' && c != '\t')
3034 1343732062 : costly_p = c == '*';
3035 : }
3036 1343732062 : while ((p += len), c);
3037 :
3038 1058307110 : scratch_p = (operand_reg[nop] != NULL_RTX
3039 529153555 : && ira_former_scratch_p (REGNO (operand_reg[nop])));
3040 : /* Record which operands fit this alternative. */
3041 529153555 : if (win)
3042 : {
3043 285111606 : if (early_clobber_p
3044 284959464 : || curr_static_id->operand[nop].type != OP_OUT)
3045 : {
3046 124598041 : if (winreg)
3047 103007076 : all_used_nregs
3048 103007076 : += ira_reg_class_min_nregs[this_alternative][mode];
3049 124598041 : all_this_alternative
3050 124598041 : = (reg_class_subunion
3051 124598041 : [all_this_alternative][this_alternative]);
3052 : }
3053 285111606 : this_alternative_win = true;
3054 285111606 : if (class_change_p)
3055 : {
3056 270701 : curr_alt_class_change_p = true;
3057 270701 : if (lra_dump_file != NULL)
3058 10 : fprintf (lra_dump_file,
3059 : " %d Narrowing class: reject+=3\n",
3060 : nop);
3061 270701 : reject += 3;
3062 : }
3063 285111606 : if (operand_reg[nop] != NULL_RTX)
3064 : {
3065 199198922 : if (hard_regno[nop] >= 0)
3066 : {
3067 178576360 : if (in_hard_reg_set_p (this_costly_alternative_set,
3068 : mode, hard_regno[nop]))
3069 : {
3070 789615 : if (lra_dump_file != NULL)
3071 21 : fprintf (lra_dump_file,
3072 : " %d Costly set: reject++\n",
3073 : nop);
3074 789615 : reject++;
3075 : }
3076 : }
3077 : else
3078 : {
3079 : /* Prefer won reg to spilled pseudo under other
3080 : equal conditions for possible inheritance. */
3081 20622562 : if (! scratch_p)
3082 : {
3083 20617923 : if (lra_dump_file != NULL)
3084 59 : fprintf
3085 59 : (lra_dump_file,
3086 : " %d Non pseudo reload: reject++\n",
3087 : nop);
3088 20617923 : reject++;
3089 : }
3090 20622562 : if (in_class_p (operand_reg[nop],
3091 : this_costly_alternative, NULL, true))
3092 : {
3093 135951 : if (lra_dump_file != NULL)
3094 0 : fprintf
3095 0 : (lra_dump_file,
3096 : " %d Non pseudo costly reload:"
3097 : " reject++\n",
3098 : nop);
3099 135951 : reject++;
3100 : }
3101 : }
3102 : /* We simulate the behavior of old reload here.
3103 : Although scratches need hard registers and it
3104 : might result in spilling other pseudos, no reload
3105 : insns are generated for the scratches. So it
3106 : might cost something but probably less than old
3107 : reload pass believes. */
3108 199198922 : if (scratch_p)
3109 : {
3110 123279 : if (lra_dump_file != NULL)
3111 6 : fprintf (lra_dump_file,
3112 : " %d Scratch win: reject+=2\n",
3113 : nop);
3114 123279 : reject += 2;
3115 : }
3116 : }
3117 : }
3118 244041949 : else if (did_match)
3119 : this_alternative_match_win = true;
3120 : else
3121 : {
3122 229059007 : if (prefer_memory_p && offmemok)
3123 : {
3124 0 : winreg = false;
3125 0 : this_alternative = NO_REGS;
3126 : }
3127 :
3128 229059007 : int const_to_mem = 0;
3129 229059007 : bool no_regs_p;
3130 :
3131 229059007 : reject += op_reject;
3132 : /* Mark output reload of the stack pointer. */
3133 229059007 : if (op == stack_pointer_rtx
3134 56128 : && curr_static_id->operand[nop].type != OP_IN)
3135 229059007 : curr_alt_out_sp_reload_p = true;
3136 :
3137 : /* If this alternative asks for a specific reg class, see if there
3138 : is at least one allocatable register in that class. */
3139 229059007 : no_regs_p
3140 399167333 : = (this_alternative == NO_REGS
3141 229059007 : || (hard_reg_set_subset_p
3142 340216680 : (reg_class_contents[this_alternative],
3143 : lra_no_alloc_regs)));
3144 :
3145 : /* For asms, verify that the class for this alternative is possible
3146 : for the mode that is specified. */
3147 170108326 : if (!no_regs_p && INSN_CODE (curr_insn) < 0)
3148 : {
3149 : int i;
3150 70426 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
3151 70424 : if (targetm.hard_regno_mode_ok (i, mode)
3152 70424 : && in_hard_reg_set_p (reg_class_contents[this_alternative],
3153 : mode, i))
3154 : break;
3155 20467 : if (i == FIRST_PSEUDO_REGISTER)
3156 229059007 : winreg = false;
3157 : }
3158 :
3159 : /* If this operand accepts a register, and if the
3160 : register class has at least one allocatable register,
3161 : then this operand can be reloaded. */
3162 229059007 : if (winreg && !no_regs_p)
3163 : badop = false;
3164 :
3165 58950683 : if (badop)
3166 : {
3167 49335389 : if (lra_dump_file != NULL)
3168 606 : fprintf (lra_dump_file,
3169 : " Bad operand -- refuse\n");
3170 125396992 : goto fail;
3171 : }
3172 :
3173 179723618 : if (this_alternative != NO_REGS)
3174 : {
3175 170108325 : HARD_REG_SET available_regs
3176 170108325 : = (reg_class_contents[this_alternative]
3177 170108325 : & ~((ira_prohibited_class_mode_regs
3178 170108325 : [this_alternative][mode])
3179 170108325 : | lra_no_alloc_regs));
3180 340216650 : if (!hard_reg_set_empty_p (available_regs))
3181 : {
3182 170106712 : if (early_clobber_p
3183 170067382 : || curr_static_id->operand[nop].type != OP_OUT)
3184 : {
3185 89162180 : all_reload_nregs
3186 89162180 : += ira_reg_class_min_nregs[this_alternative][mode];
3187 89162180 : all_this_alternative
3188 89162180 : = (reg_class_subunion
3189 89162180 : [all_this_alternative][this_alternative]);
3190 : }
3191 : }
3192 : else
3193 : {
3194 : /* There are no hard regs holding a value of given
3195 : mode. */
3196 1613 : if (offmemok)
3197 : {
3198 170 : this_alternative = NO_REGS;
3199 170 : if (lra_dump_file != NULL)
3200 0 : fprintf (lra_dump_file,
3201 : " %d Using memory because of"
3202 : " a bad mode: reject+=2\n",
3203 : nop);
3204 170 : reject += 2;
3205 : }
3206 : else
3207 : {
3208 1443 : if (lra_dump_file != NULL)
3209 0 : fprintf (lra_dump_file,
3210 : " Wrong mode -- refuse\n");
3211 1443 : goto fail;
3212 : }
3213 : }
3214 : }
3215 :
3216 : /* If not assigned pseudo has a class which a subset of
3217 : required reg class, it is a less costly alternative
3218 : as the pseudo still can get a hard reg of necessary
3219 : class. */
3220 170106882 : if (! no_regs_p && REG_P (op) && hard_regno[nop] < 0
3221 21902434 : && (cl = get_reg_class (REGNO (op))) != NO_REGS
3222 182740648 : && ira_class_subset_p[this_alternative][cl])
3223 : {
3224 1078 : if (lra_dump_file != NULL)
3225 0 : fprintf
3226 0 : (lra_dump_file,
3227 : " %d Super set class reg: reject-=3\n", nop);
3228 1078 : reject -= 3;
3229 : }
3230 :
3231 179722175 : this_alternative_offmemok = offmemok;
3232 179722175 : if (this_costly_alternative != NO_REGS)
3233 : {
3234 19541016 : if (lra_dump_file != NULL)
3235 25 : fprintf (lra_dump_file,
3236 : " %d Costly loser: reject++\n", nop);
3237 19541016 : reject++;
3238 : }
3239 : /* If the operand is dying, has a matching constraint,
3240 : and satisfies constraints of the matched operand
3241 : which failed to satisfy the own constraints, most probably
3242 : the reload for this operand will be gone. */
3243 179722175 : if (this_alternative_matches >= 0
3244 42489940 : && !curr_alt_win[this_alternative_matches]
3245 969494 : && REG_P (op)
3246 722696 : && find_regno_note (curr_insn, REG_DEAD, REGNO (op))
3247 180458459 : && (hard_regno[nop] >= 0
3248 384091 : ? in_hard_reg_set_p (this_alternative_set,
3249 : mode, hard_regno[nop])
3250 31898 : : in_class_p (op, this_alternative, NULL)))
3251 : {
3252 237584 : if (lra_dump_file != NULL)
3253 1 : fprintf
3254 1 : (lra_dump_file,
3255 : " %d Dying matched operand reload: reject++\n",
3256 : nop);
3257 237584 : reject++;
3258 : }
3259 : else
3260 : {
3261 : /* Strict_low_part requires to reload the register
3262 : not the sub-register. In this case we should
3263 : check that a final reload hard reg can hold the
3264 : value mode. */
3265 179484591 : if (curr_static_id->operand[nop].strict_low
3266 116 : && REG_P (op)
3267 109 : && hard_regno[nop] < 0
3268 83 : && GET_CODE (*curr_id->operand_loc[nop]) == SUBREG
3269 83 : && ira_class_hard_regs_num[this_alternative] > 0
3270 179484674 : && (!targetm.hard_regno_mode_ok
3271 83 : (ira_class_hard_regs[this_alternative][0],
3272 83 : GET_MODE (*curr_id->operand_loc[nop]))))
3273 : {
3274 0 : if (lra_dump_file != NULL)
3275 0 : fprintf
3276 0 : (lra_dump_file,
3277 : " Strict low subreg reload -- refuse\n");
3278 0 : goto fail;
3279 : }
3280 179484591 : if (lra_dump_file != NULL)
3281 2177 : fprintf
3282 2177 : (lra_dump_file,
3283 : " %d Operand reload: losers++\n", nop);
3284 179484591 : losers++;
3285 : }
3286 179722175 : if (operand_reg[nop] != NULL_RTX
3287 : /* Output operands and matched input operands are
3288 : not inherited. The following conditions do not
3289 : exactly describe the previous statement but they
3290 : are pretty close. */
3291 64893347 : && curr_static_id->operand[nop].type != OP_OUT
3292 28602925 : && (this_alternative_matches < 0
3293 16717958 : || curr_static_id->operand[nop].type != OP_IN))
3294 : {
3295 11884967 : int last_reload = (lra_reg_info[ORIGINAL_REGNO
3296 11884967 : (operand_reg[nop])]
3297 11884967 : .last_reload);
3298 :
3299 : /* The value of reload_sum has sense only if we
3300 : process insns in their order. It happens only on
3301 : the first constraints sub-pass when we do most of
3302 : reload work. */
3303 11884967 : if (lra_constraint_iter == 1 && last_reload > bb_reload_num)
3304 2585411 : reload_sum += last_reload - bb_reload_num;
3305 : }
3306 : /* If this is a constant that is reloaded into the
3307 : desired class by copying it to memory first, count
3308 : that as another reload. This is consistent with
3309 : other code and is required to avoid choosing another
3310 : alternative when the constant is moved into memory.
3311 : Note that the test here is precisely the same as in
3312 : the code below that calls force_const_mem. */
3313 230783751 : if (CONST_POOL_OK_P (mode, op)
3314 230783825 : && ((targetm.preferred_reload_class
3315 51061650 : (op, this_alternative) == NO_REGS)
3316 49484776 : || no_input_reloads_p))
3317 : {
3318 1576874 : const_to_mem = 1;
3319 1576874 : if (! no_regs_p)
3320 : {
3321 725542 : if (lra_dump_file != NULL)
3322 0 : fprintf
3323 0 : (lra_dump_file,
3324 : " %d Constant reload through memory: "
3325 : "losers++\n", nop);
3326 725542 : losers++;
3327 : }
3328 : }
3329 :
3330 : /* Alternative loses if it requires a type of reload not
3331 : permitted for this insn. We can always reload
3332 : objects with a REG_UNUSED note. */
3333 179722175 : if ((curr_static_id->operand[nop].type != OP_IN
3334 87282547 : && no_output_reloads_p
3335 0 : && ! find_reg_note (curr_insn, REG_UNUSED, op)
3336 0 : && ! scratch_p)
3337 179722175 : || (curr_static_id->operand[nop].type != OP_OUT
3338 92439892 : && no_input_reloads_p && ! const_to_mem)
3339 359444350 : || (this_alternative_matches >= 0
3340 42489940 : && (no_input_reloads_p
3341 42489940 : || (no_output_reloads_p
3342 0 : && (curr_static_id->operand
3343 0 : [this_alternative_matches].type != OP_IN)
3344 0 : && ! find_reg_note (curr_insn, REG_UNUSED,
3345 : no_subreg_reg_operand
3346 0 : [this_alternative_matches])
3347 0 : && ! scratch_p))))
3348 : {
3349 0 : if (lra_dump_file != NULL)
3350 0 : fprintf
3351 0 : (lra_dump_file,
3352 : " No input/output reload -- refuse\n");
3353 0 : goto fail;
3354 : }
3355 :
3356 : /* Alternative loses if it required class pseudo cannot
3357 : hold value of required mode. Such insns can be
3358 : described by insn definitions with mode iterators. */
3359 179722175 : if (GET_MODE (*curr_id->operand_loc[nop]) != VOIDmode
3360 130165950 : && ! hard_reg_set_empty_p (this_alternative_set)
3361 : /* It is common practice for constraints to use a
3362 : class which does not have actually enough regs to
3363 : hold the value (e.g. x86 AREG for mode requiring
3364 : more one general reg). Therefore we have 2
3365 : conditions to check that the reload pseudo cannot
3366 : hold the mode value. */
3367 121299890 : && (!targetm.hard_regno_mode_ok
3368 121299890 : (ira_class_hard_regs[this_alternative][0],
3369 : GET_MODE (*curr_id->operand_loc[nop])))
3370 : /* The above condition is not enough as the first
3371 : reg in ira_class_hard_regs can be not aligned for
3372 : multi-words mode values. */
3373 179722175 : && (prohibited_class_reg_set_mode_p
3374 0 : (this_alternative, this_alternative_set,
3375 0 : GET_MODE (*curr_id->operand_loc[nop]))))
3376 : {
3377 0 : if (lra_dump_file != NULL)
3378 0 : fprintf (lra_dump_file,
3379 : " reload pseudo for op %d "
3380 : "cannot hold the mode value -- refuse\n",
3381 : nop);
3382 0 : goto fail;
3383 : }
3384 :
3385 : /* Check strong discouragement of reload of non-constant
3386 : into class THIS_ALTERNATIVE. */
3387 128660522 : if (! CONSTANT_P (op) && ! no_regs_p
3388 299618736 : && (targetm.preferred_reload_class
3389 119896561 : (op, this_alternative) == NO_REGS
3390 111265959 : || (curr_static_id->operand[nop].type == OP_OUT
3391 76510414 : && (targetm.preferred_output_reload_class
3392 76510414 : (op, this_alternative) == NO_REGS))))
3393 : {
3394 13207553 : if (offmemok && REG_P (op))
3395 : {
3396 802553 : if (lra_dump_file != NULL)
3397 0 : fprintf
3398 0 : (lra_dump_file,
3399 : " %d Spill pseudo into memory: reject+=3\n",
3400 : nop);
3401 802553 : reject += 3;
3402 : }
3403 : else
3404 : {
3405 12405000 : if (lra_dump_file != NULL)
3406 0 : fprintf
3407 0 : (lra_dump_file,
3408 : " %d Non-preferred reload: reject+=%d\n",
3409 : nop, LRA_MAX_REJECT);
3410 12405000 : reject += LRA_MAX_REJECT;
3411 : }
3412 : }
3413 :
3414 179722175 : if (! (MEM_P (op) && offmemok)
3415 179722103 : && ! (const_to_mem && constmemok))
3416 : {
3417 : /* We prefer to reload pseudos over reloading other
3418 : things, since such reloads may be able to be
3419 : eliminated later. So bump REJECT in other cases.
3420 : Don't do this in the case where we are forcing a
3421 : constant into memory and it will then win since
3422 : we don't want to have a different alternative
3423 : match then. */
3424 178752343 : if (! (REG_P (op) && REGNO (op) >= FIRST_PSEUDO_REGISTER))
3425 : {
3426 127924780 : if (lra_dump_file != NULL)
3427 1648 : fprintf
3428 1648 : (lra_dump_file,
3429 : " %d Non-pseudo reload: reject+=2\n",
3430 : nop);
3431 127924780 : reject += 2;
3432 : }
3433 :
3434 178752343 : if (! no_regs_p)
3435 169988452 : reload_nregs
3436 169988452 : += ira_reg_class_max_nregs[this_alternative][mode];
3437 :
3438 178752343 : if (SMALL_REGISTER_CLASS_P (this_alternative))
3439 : {
3440 919470 : if (lra_dump_file != NULL)
3441 45 : fprintf
3442 45 : (lra_dump_file,
3443 : " %d Small class reload: reject+=%d\n",
3444 : nop, LRA_LOSER_COST_FACTOR / 2);
3445 919470 : reject += LRA_LOSER_COST_FACTOR / 2;
3446 : }
3447 : }
3448 :
3449 : /* We are trying to spill pseudo into memory. It is
3450 : usually more costly than moving to a hard register
3451 : although it might takes the same number of
3452 : reloads.
3453 :
3454 : Non-pseudo spill may happen also. Suppose a target allows both
3455 : register and memory in the operand constraint alternatives,
3456 : then it's typical that an eliminable register has a substition
3457 : of "base + offset" which can either be reloaded by a simple
3458 : "new_reg <= base + offset" which will match the register
3459 : constraint, or a similar reg addition followed by further spill
3460 : to and reload from memory which will match the memory
3461 : constraint, but this memory spill will be much more costly
3462 : usually.
3463 :
3464 : Code below increases the reject for both pseudo and non-pseudo
3465 : spill. */
3466 179722175 : if (no_regs_p
3467 9615293 : && !(MEM_P (op) && offmemok)
3468 9615223 : && !(REG_P (op) && hard_regno[nop] < 0))
3469 : {
3470 8526994 : if (lra_dump_file != NULL)
3471 13 : fprintf
3472 20 : (lra_dump_file,
3473 : " %d Spill %spseudo into memory: reject+=3\n",
3474 : nop, REG_P (op) ? "" : "Non-");
3475 8526994 : reject += 3;
3476 8526994 : if (VECTOR_MODE_P (mode))
3477 : {
3478 : /* Spilling vectors into memory is usually more
3479 : costly as they contain big values. */
3480 419447 : if (lra_dump_file != NULL)
3481 0 : fprintf
3482 0 : (lra_dump_file,
3483 : " %d Spill vector pseudo: reject+=2\n",
3484 : nop);
3485 419447 : reject += 2;
3486 : }
3487 : }
3488 :
3489 : /* When we use an operand requiring memory in given
3490 : alternative, the insn should write *and* read the
3491 : value to/from memory it is costly in comparison with
3492 : an insn alternative which does not use memory
3493 : (e.g. register or immediate operand). We exclude
3494 : memory operand for such case as we can satisfy the
3495 : memory constraints by reloading address. */
3496 9615293 : if (no_regs_p && offmemok && !MEM_P (op))
3497 : {
3498 9615053 : if (lra_dump_file != NULL)
3499 27 : fprintf
3500 27 : (lra_dump_file,
3501 : " Using memory insn operand %d: reject+=3\n",
3502 : nop);
3503 9615053 : reject += 3;
3504 : }
3505 :
3506 : /* If reload requires moving value through secondary
3507 : memory, it will need one more insn at least. */
3508 179722175 : if (this_alternative != NO_REGS
3509 170106712 : && REG_P (op) && (cl = get_reg_class (REGNO (op))) != NO_REGS
3510 216973332 : && ((curr_static_id->operand[nop].type != OP_OUT
3511 21048402 : && targetm.secondary_memory_needed (mode, cl,
3512 : this_alternative))
3513 33970720 : || (curr_static_id->operand[nop].type != OP_IN
3514 16202917 : && (targetm.secondary_memory_needed
3515 16202917 : (mode, this_alternative, cl)))))
3516 : {
3517 11066957 : if (lra_dump_file != NULL)
3518 16 : fprintf
3519 16 : (lra_dump_file,
3520 : " %d Secondary memory reload needed: "
3521 : "losers++\n", nop);
3522 11066957 : losers++;
3523 : }
3524 :
3525 179722175 : if (MEM_P (op) && offmemok)
3526 72 : addr_losers++;
3527 : else
3528 : {
3529 : /* Input reloads can be inherited more often than
3530 : output reloads can be removed, so penalize output
3531 : reloads. */
3532 179722103 : if (!REG_P (op) || curr_static_id->operand[nop].type != OP_IN)
3533 : {
3534 151119435 : if (lra_dump_file != NULL)
3535 1722 : fprintf
3536 1722 : (lra_dump_file,
3537 : " %d Non input pseudo reload: reject++\n",
3538 : nop);
3539 151119435 : reject++;
3540 : }
3541 :
3542 179722103 : if (curr_static_id->operand[nop].type == OP_INOUT)
3543 : {
3544 264 : if (lra_dump_file != NULL)
3545 0 : fprintf
3546 0 : (lra_dump_file,
3547 : " %d Input/Output reload: reject+=%d\n",
3548 : nop, LRA_LOSER_COST_FACTOR);
3549 264 : reject += LRA_LOSER_COST_FACTOR;
3550 : }
3551 : }
3552 : }
3553 :
3554 479816723 : if (early_clobber_p && ! scratch_p)
3555 : {
3556 180062 : if (lra_dump_file != NULL)
3557 4 : fprintf (lra_dump_file,
3558 : " %d Early clobber: reject++\n", nop);
3559 180062 : reject++;
3560 : }
3561 : /* ??? We check early clobbers after processing all operands
3562 : (see loop below) and there we update the costs more.
3563 : Should we update the cost (may be approximately) here
3564 : because of early clobber register reloads or it is a rare
3565 : or non-important thing to be worth to do it. */
3566 959633446 : overall = (losers * LRA_LOSER_COST_FACTOR + reject
3567 479816723 : - (addr_losers == losers ? static_reject : 0));
3568 479816723 : if ((best_losers == 0 || losers != 0) && best_overall < overall)
3569 : {
3570 76060160 : if (lra_dump_file != NULL)
3571 1036 : fprintf (lra_dump_file,
3572 : " overall=%d,losers=%d -- refuse\n",
3573 : overall, losers);
3574 76060160 : goto fail;
3575 : }
3576 :
3577 403756563 : if (update_and_check_small_class_inputs (nop, nalt,
3578 : this_alternative))
3579 : {
3580 0 : if (lra_dump_file != NULL)
3581 0 : fprintf (lra_dump_file,
3582 : " not enough small class regs -- refuse\n");
3583 0 : goto fail;
3584 : }
3585 403756563 : curr_alt[nop] = this_alternative;
3586 403756563 : curr_alt_set[nop] = this_alternative_set;
3587 403756563 : curr_alt_exclude_start_hard_regs[nop]
3588 403756563 : = this_alternative_exclude_start_hard_regs;
3589 403756563 : curr_alt_win[nop] = this_alternative_win;
3590 403756563 : curr_alt_match_win[nop] = this_alternative_match_win;
3591 403756563 : curr_alt_offmemok[nop] = this_alternative_offmemok;
3592 403756563 : curr_alt_matches[nop] = this_alternative_matches;
3593 :
3594 403756563 : if (this_alternative_matches >= 0
3595 403756563 : && !did_match && !this_alternative_win)
3596 13774972 : curr_alt_win[this_alternative_matches] = false;
3597 :
3598 403756563 : if (early_clobber_p && operand_reg[nop] != NULL_RTX)
3599 184437 : early_clobbered_nops[early_clobbered_regs_num++] = nop;
3600 : }
3601 :
3602 138189037 : if (curr_insn_set != NULL_RTX
3603 : /* Allow just two operands or three operands where the third
3604 : is a clobber. */
3605 134207596 : && (n_operands == 2
3606 29561213 : || (n_operands == 3
3607 27335189 : && GET_CODE (PATTERN (curr_insn)) == PARALLEL
3608 23183868 : && XVECLEN (PATTERN (curr_insn), 0) == 2
3609 23125872 : && GET_CODE (XVECEXP (PATTERN (curr_insn), 0, 1))
3610 : == CLOBBER))
3611 : /* Prevent processing non-move insns. */
3612 127687378 : && (GET_CODE (SET_SRC (curr_insn_set)) == SUBREG
3613 125775023 : || SET_SRC (curr_insn_set) == no_subreg_reg_operand[1])
3614 231605732 : && ((! curr_alt_win[0] && ! curr_alt_win[1]
3615 6069560 : && REG_P (no_subreg_reg_operand[0])
3616 2961332 : && REG_P (no_subreg_reg_operand[1])
3617 1252358 : && (reg_in_class_p (no_subreg_reg_operand[0], curr_alt[1])
3618 1030772 : || reg_in_class_p (no_subreg_reg_operand[1], curr_alt[0])))
3619 92869781 : || (! curr_alt_win[0] && curr_alt_win[1]
3620 27604873 : && REG_P (no_subreg_reg_operand[1])
3621 : /* Check that we reload memory not the memory
3622 : address. */
3623 15853394 : && ! (curr_alt_offmemok[0]
3624 402375 : && MEM_P (no_subreg_reg_operand[0]))
3625 15853394 : && reg_in_class_p (no_subreg_reg_operand[1], curr_alt[0]))
3626 78436257 : || (curr_alt_win[0] && ! curr_alt_win[1]
3627 10040671 : && REG_P (no_subreg_reg_operand[0])
3628 : /* Check that we reload memory not the memory
3629 : address. */
3630 7533101 : && ! (curr_alt_offmemok[1]
3631 1400994 : && MEM_P (no_subreg_reg_operand[1]))
3632 7533099 : && reg_in_class_p (no_subreg_reg_operand[0], curr_alt[1])
3633 6632577 : && (! CONST_POOL_OK_P (curr_operand_mode[1],
3634 : no_subreg_reg_operand[1])
3635 2624463 : || (targetm.preferred_reload_class
3636 2624463 : (no_subreg_reg_operand[1],
3637 : (enum reg_class) curr_alt[1]) != NO_REGS))
3638 : /* If it is a result of recent elimination in move
3639 : insn we can transform it into an add still by
3640 : using this alternative. */
3641 6588842 : && GET_CODE (no_subreg_reg_operand[1]) != PLUS
3642 : /* Likewise if the source has been replaced with an
3643 : equivalent value. This only happens once -- the reload
3644 : will use the equivalent value instead of the register it
3645 : replaces -- so there should be no danger of cycling. */
3646 6053429 : && !equiv_substition_p[1])))
3647 : {
3648 : /* We have a move insn and a new reload insn will be similar
3649 : to the current insn. We should avoid such situation as
3650 : it results in LRA cycling. */
3651 20982987 : if (lra_dump_file != NULL)
3652 239 : fprintf (lra_dump_file,
3653 : " Cycle danger: overall += LRA_MAX_REJECT\n");
3654 20982987 : overall += LRA_MAX_REJECT;
3655 : }
3656 138189037 : if (all_this_alternative != NO_REGS
3657 118334698 : && !SMALL_REGISTER_CLASS_P (all_this_alternative)
3658 117491207 : && all_used_nregs != 0 && all_reload_nregs != 0
3659 138189037 : && (all_used_nregs + all_reload_nregs + 1
3660 4230304 : >= ira_class_hard_regs_num[all_this_alternative]))
3661 : {
3662 391 : if (lra_dump_file != NULL)
3663 0 : fprintf
3664 0 : (lra_dump_file,
3665 : " Register starvation: overall += LRA_MAX_REJECT"
3666 : "(class=%s,avail=%d,used=%d,reload=%d)\n",
3667 : reg_class_names[all_this_alternative],
3668 : ira_class_hard_regs_num[all_this_alternative],
3669 : all_used_nregs, all_reload_nregs);
3670 391 : overall += LRA_MAX_REJECT;
3671 391 : if (!prefer_memory_p && INSN_CODE (curr_insn) < 0)
3672 : {
3673 : /* asm can permit memory and reg and can be not enough regs for
3674 : asm -- try now memory: */
3675 114 : prefer_memory_p = true;
3676 114 : if (lra_dump_file != NULL)
3677 0 : fprintf
3678 0 : (lra_dump_file,
3679 : " Trying now memory for operands\n");
3680 114 : goto repeat;
3681 : }
3682 : }
3683 138188923 : ok_p = true;
3684 138188923 : curr_alt_dont_inherit_ops_num = 0;
3685 138368846 : for (nop = 0; nop < early_clobbered_regs_num; nop++)
3686 : {
3687 179924 : int i, j, clobbered_hard_regno, first_conflict_j, last_conflict_j;
3688 179924 : HARD_REG_SET temp_set;
3689 :
3690 179924 : i = early_clobbered_nops[nop];
3691 179924 : if ((! curr_alt_win[i] && ! curr_alt_match_win[i])
3692 133786 : || hard_regno[i] < 0)
3693 179247 : continue;
3694 131919 : lra_assert (operand_reg[i] != NULL_RTX);
3695 : clobbered_hard_regno = hard_regno[i];
3696 131919 : CLEAR_HARD_REG_SET (temp_set);
3697 131919 : add_to_hard_reg_set (&temp_set, GET_MODE (*curr_id->operand_loc[i]),
3698 : clobbered_hard_regno);
3699 131919 : first_conflict_j = last_conflict_j = -1;
3700 795820 : for (j = 0; j < n_operands; j++)
3701 531983 : if (j == i
3702 : /* We don't want process insides of match_operator and
3703 : match_parallel because otherwise we would process
3704 : their operands once again generating a wrong
3705 : code. */
3706 400064 : || curr_static_id->operand[j].is_operator)
3707 134067 : continue;
3708 397916 : else if ((curr_alt_matches[j] == i && curr_alt_match_win[j])
3709 377462 : || (curr_alt_matches[i] == j && curr_alt_match_win[i]))
3710 20454 : continue;
3711 : /* If we don't reload j-th operand, check conflicts. */
3712 141737 : else if ((curr_alt_win[j] || curr_alt_match_win[j])
3713 434422 : && uses_hard_regs_p (*curr_id->operand_loc[j], temp_set))
3714 : {
3715 1151 : if (first_conflict_j < 0)
3716 677 : first_conflict_j = j;
3717 1151 : last_conflict_j = j;
3718 : /* Both the earlyclobber operand and conflicting operand
3719 : cannot both be user defined hard registers for asm.
3720 : Let curr_insn_transform diagnose it. */
3721 1151 : if (HARD_REGISTER_P (operand_reg[i])
3722 1 : && REG_USERVAR_P (operand_reg[i])
3723 1 : && operand_reg[j] != NULL_RTX
3724 1 : && HARD_REGISTER_P (operand_reg[j])
3725 1 : && REG_USERVAR_P (operand_reg[j])
3726 1152 : && INSN_CODE (curr_insn) < 0)
3727 1 : return false;
3728 : }
3729 131918 : if (last_conflict_j < 0)
3730 131242 : continue;
3731 :
3732 : /* If an earlyclobber operand conflicts with another non-matching
3733 : operand (ie, they have been assigned the same hard register),
3734 : then it is better to reload the other operand, as there may
3735 : exist yet another operand with a matching constraint associated
3736 : with the earlyclobber operand. However, if one of the operands
3737 : is an explicit use of a hard register, then we must reload the
3738 : other non-hard register operand. */
3739 676 : if (HARD_REGISTER_P (operand_reg[i])
3740 676 : || (first_conflict_j == last_conflict_j
3741 202 : && operand_reg[last_conflict_j] != NULL_RTX
3742 61 : && !curr_alt_match_win[last_conflict_j]
3743 61 : && !HARD_REGISTER_P (operand_reg[last_conflict_j])))
3744 : {
3745 61 : curr_alt_win[last_conflict_j] = false;
3746 61 : curr_alt_dont_inherit_ops[curr_alt_dont_inherit_ops_num++]
3747 61 : = last_conflict_j;
3748 61 : losers++;
3749 61 : if (lra_dump_file != NULL)
3750 0 : fprintf
3751 0 : (lra_dump_file,
3752 : " %d Conflict early clobber reload: losers++\n",
3753 : i);
3754 : }
3755 : else
3756 : {
3757 : /* We need to reload early clobbered register and the
3758 : matched registers. */
3759 3069 : for (j = 0; j < n_operands; j++)
3760 2454 : if (curr_alt_matches[j] == i)
3761 : {
3762 2 : curr_alt_match_win[j] = false;
3763 2 : losers++;
3764 2 : if (lra_dump_file != NULL)
3765 0 : fprintf
3766 0 : (lra_dump_file,
3767 : " %d Matching conflict early clobber "
3768 : "reloads: losers++\n",
3769 : j);
3770 2 : overall += LRA_LOSER_COST_FACTOR;
3771 : }
3772 615 : if (! curr_alt_match_win[i])
3773 615 : curr_alt_dont_inherit_ops[curr_alt_dont_inherit_ops_num++] = i;
3774 : else
3775 : {
3776 : /* Remember pseudos used for match reloads are never
3777 : inherited. */
3778 0 : lra_assert (curr_alt_matches[i] >= 0);
3779 0 : curr_alt_win[curr_alt_matches[i]] = false;
3780 : }
3781 615 : curr_alt_win[i] = curr_alt_match_win[i] = false;
3782 615 : losers++;
3783 615 : if (lra_dump_file != NULL)
3784 0 : fprintf
3785 0 : (lra_dump_file,
3786 : " %d Matched conflict early clobber reloads: "
3787 : "losers++\n",
3788 : i);
3789 : }
3790 : /* Early clobber was already reflected in REJECT. */
3791 676 : if (!matching_early_clobber[i])
3792 : {
3793 676 : lra_assert (reject > 0);
3794 676 : reject--;
3795 676 : matching_early_clobber[i] = 1;
3796 : }
3797 676 : overall += LRA_LOSER_COST_FACTOR - 1;
3798 : }
3799 138188922 : if (lra_dump_file != NULL)
3800 1761 : fprintf (lra_dump_file, " overall=%d,losers=%d,rld_nregs=%d\n",
3801 : overall, losers, reload_nregs);
3802 :
3803 : /* If this alternative can be made to work by reloading, and it
3804 : needs less reloading than the others checked so far, record
3805 : it as the chosen goal for reloading. */
3806 138188922 : if ((best_losers != 0 && losers == 0)
3807 61371140 : || (((best_losers == 0 && losers == 0)
3808 60332398 : || (best_losers != 0 && losers != 0))
3809 61371140 : && (best_overall > overall
3810 15725525 : || (best_overall == overall
3811 : /* If the cost of the reloads is the same,
3812 : prefer alternative which requires minimal
3813 : number of reload regs. */
3814 11749493 : && (reload_nregs < best_reload_nregs
3815 11646115 : || (reload_nregs == best_reload_nregs
3816 11603422 : && (best_reload_sum < reload_sum
3817 11580013 : || (best_reload_sum == reload_sum
3818 11557861 : && nalt < goal_alt_number))))))))
3819 : {
3820 399119886 : for (nop = 0; nop < n_operands; nop++)
3821 : {
3822 276250337 : goal_alt_win[nop] = curr_alt_win[nop];
3823 276250337 : goal_alt_match_win[nop] = curr_alt_match_win[nop];
3824 276250337 : goal_alt_matches[nop] = curr_alt_matches[nop];
3825 276250337 : goal_alt[nop] = curr_alt[nop];
3826 276250337 : goal_alt_exclude_start_hard_regs[nop]
3827 276250337 : = curr_alt_exclude_start_hard_regs[nop];
3828 276250337 : goal_alt_offmemok[nop] = curr_alt_offmemok[nop];
3829 : }
3830 122869549 : goal_alt_dont_inherit_ops_num = curr_alt_dont_inherit_ops_num;
3831 122869549 : goal_reuse_alt_p = curr_reuse_alt_p;
3832 122870215 : for (nop = 0; nop < curr_alt_dont_inherit_ops_num; nop++)
3833 666 : goal_alt_dont_inherit_ops[nop] = curr_alt_dont_inherit_ops[nop];
3834 122869549 : goal_alt_swapped = curr_swapped;
3835 122869549 : goal_alt_out_sp_reload_p = curr_alt_out_sp_reload_p;
3836 122869549 : best_overall = overall;
3837 122869549 : best_losers = losers;
3838 122869549 : best_reload_nregs = reload_nregs;
3839 122869549 : best_reload_sum = reload_sum;
3840 122869549 : goal_alt_number = nalt;
3841 : }
3842 138188922 : if (losers == 0 && !curr_alt_class_change_p)
3843 : /* Everything is satisfied. Do not process alternatives
3844 : anymore. */
3845 : break;
3846 60344958 : fail:
3847 185741950 : ;
3848 : }
3849 : return ok_p;
3850 : }
3851 :
3852 : /* Make reload base reg from address AD. */
3853 : static rtx
3854 0 : base_to_reg (struct address_info *ad)
3855 : {
3856 0 : enum reg_class cl;
3857 0 : int code = -1;
3858 0 : rtx new_inner = NULL_RTX;
3859 0 : rtx new_reg = NULL_RTX;
3860 0 : rtx_insn *insn;
3861 0 : rtx_insn *last_insn = get_last_insn();
3862 :
3863 0 : lra_assert (ad->disp == ad->disp_term);
3864 0 : cl = base_reg_class (ad->mode, ad->as, ad->base_outer_code,
3865 : get_index_code (ad));
3866 0 : new_reg = lra_create_new_reg (GET_MODE (*ad->base), NULL_RTX, cl, NULL,
3867 : "base");
3868 0 : new_inner = simplify_gen_binary (PLUS, GET_MODE (new_reg), new_reg,
3869 0 : ad->disp_term == NULL
3870 : ? const0_rtx
3871 : : *ad->disp_term);
3872 0 : if (!valid_address_p (ad->mode, new_inner, ad->as))
3873 : return NULL_RTX;
3874 0 : insn = emit_insn (gen_rtx_SET (new_reg, *ad->base));
3875 0 : code = recog_memoized (insn);
3876 0 : if (code < 0)
3877 : {
3878 0 : delete_insns_since (last_insn);
3879 0 : return NULL_RTX;
3880 : }
3881 :
3882 : return new_inner;
3883 : }
3884 :
3885 : /* Make reload base reg + DISP from address AD. Return the new pseudo. */
3886 : static rtx
3887 47 : base_plus_disp_to_reg (struct address_info *ad, rtx disp)
3888 : {
3889 47 : enum reg_class cl;
3890 47 : rtx new_reg;
3891 :
3892 47 : lra_assert (ad->base == ad->base_term);
3893 47 : cl = base_reg_class (ad->mode, ad->as, ad->base_outer_code,
3894 : get_index_code (ad));
3895 47 : new_reg = lra_create_new_reg (GET_MODE (*ad->base_term), NULL_RTX, cl, NULL,
3896 : "base + disp");
3897 47 : lra_emit_add (new_reg, *ad->base_term, disp);
3898 47 : return new_reg;
3899 : }
3900 :
3901 : /* Make reload of index part of address AD. Return the new
3902 : pseudo. */
3903 : static rtx
3904 0 : index_part_to_reg (struct address_info *ad, enum reg_class index_class)
3905 : {
3906 0 : rtx new_reg;
3907 :
3908 0 : new_reg = lra_create_new_reg (GET_MODE (*ad->index), NULL_RTX,
3909 : index_class, NULL, "index term");
3910 0 : expand_mult (GET_MODE (*ad->index), *ad->index_term,
3911 : GEN_INT (get_index_scale (ad)), new_reg, 1);
3912 0 : return new_reg;
3913 : }
3914 :
3915 : /* Return true if we can add a displacement to address AD, even if that
3916 : makes the address invalid. The fix-up code requires any new address
3917 : to be the sum of the BASE_TERM, INDEX and DISP_TERM fields. */
3918 : static bool
3919 19843 : can_add_disp_p (struct address_info *ad)
3920 : {
3921 19843 : return (!ad->autoinc_p
3922 19843 : && ad->segment == NULL
3923 19843 : && ad->base == ad->base_term
3924 39686 : && ad->disp == ad->disp_term);
3925 : }
3926 :
3927 : /* Make equiv substitution in address AD. Return true if a substitution
3928 : was made. */
3929 : static bool
3930 40188827 : equiv_address_substitution (struct address_info *ad)
3931 : {
3932 40188827 : rtx base_reg, new_base_reg, index_reg, new_index_reg, *base_term, *index_term;
3933 40188827 : poly_int64 disp;
3934 40188827 : HOST_WIDE_INT scale;
3935 40188827 : bool change_p;
3936 :
3937 40188827 : base_term = strip_subreg (ad->base_term);
3938 9498 : if (base_term == NULL)
3939 : base_reg = new_base_reg = NULL_RTX;
3940 : else
3941 : {
3942 33947471 : base_reg = *base_term;
3943 33947471 : new_base_reg = get_equiv_with_elimination (base_reg, curr_insn);
3944 : }
3945 40188827 : index_term = strip_subreg (ad->index_term);
3946 5555 : if (index_term == NULL)
3947 : index_reg = new_index_reg = NULL_RTX;
3948 : else
3949 : {
3950 1814342 : index_reg = *index_term;
3951 1814342 : new_index_reg = get_equiv_with_elimination (index_reg, curr_insn);
3952 : }
3953 40188827 : if (base_reg == new_base_reg && index_reg == new_index_reg)
3954 : return false;
3955 142127 : disp = 0;
3956 142127 : change_p = false;
3957 142127 : if (lra_dump_file != NULL)
3958 : {
3959 0 : fprintf (lra_dump_file, "Changing address in insn %d ",
3960 0 : INSN_UID (curr_insn));
3961 0 : dump_value_slim (lra_dump_file, *ad->outer, 1);
3962 : }
3963 142127 : if (base_reg != new_base_reg)
3964 : {
3965 141677 : poly_int64 offset;
3966 141677 : if (REG_P (new_base_reg))
3967 : {
3968 8029 : *base_term = new_base_reg;
3969 8029 : change_p = true;
3970 : }
3971 133648 : else if (GET_CODE (new_base_reg) == PLUS
3972 19842 : && REG_P (XEXP (new_base_reg, 0))
3973 19842 : && poly_int_rtx_p (XEXP (new_base_reg, 1), &offset)
3974 153490 : && can_add_disp_p (ad))
3975 : {
3976 : disp += offset;
3977 19842 : *base_term = XEXP (new_base_reg, 0);
3978 19842 : change_p = true;
3979 : }
3980 141677 : if (ad->base_term2 != NULL)
3981 0 : *ad->base_term2 = *ad->base_term;
3982 : }
3983 142127 : if (index_reg != new_index_reg)
3984 : {
3985 650 : poly_int64 offset;
3986 650 : if (REG_P (new_index_reg))
3987 : {
3988 0 : *index_term = new_index_reg;
3989 0 : change_p = true;
3990 : }
3991 650 : else if (GET_CODE (new_index_reg) == PLUS
3992 1 : && REG_P (XEXP (new_index_reg, 0))
3993 1 : && poly_int_rtx_p (XEXP (new_index_reg, 1), &offset)
3994 1 : && can_add_disp_p (ad)
3995 651 : && (scale = get_index_scale (ad)))
3996 : {
3997 1 : disp += offset * scale;
3998 1 : *index_term = XEXP (new_index_reg, 0);
3999 1 : change_p = true;
4000 : }
4001 : }
4002 142127 : if (maybe_ne (disp, 0))
4003 : {
4004 19843 : if (ad->disp != NULL)
4005 6653 : *ad->disp = plus_constant (GET_MODE (*ad->inner), *ad->disp, disp);
4006 : else
4007 : {
4008 13190 : *ad->inner = plus_constant (GET_MODE (*ad->inner), *ad->inner, disp);
4009 13190 : update_address (ad);
4010 : }
4011 : change_p = true;
4012 : }
4013 142127 : if (lra_dump_file != NULL)
4014 : {
4015 0 : if (! change_p)
4016 0 : fprintf (lra_dump_file, " -- no change\n");
4017 : else
4018 : {
4019 0 : fprintf (lra_dump_file, " on equiv ");
4020 0 : dump_value_slim (lra_dump_file, *ad->outer, 1);
4021 0 : fprintf (lra_dump_file, "\n");
4022 : }
4023 : }
4024 : return change_p;
4025 : }
4026 :
4027 : /* Skip all modifiers and whitespaces in constraint STR and return the
4028 : result. */
4029 : static const char *
4030 524875661 : skip_constraint_modifiers (const char *str)
4031 : {
4032 738587365 : for (;;str++)
4033 631731513 : switch (*str)
4034 : {
4035 106855852 : case '+': case '&' : case '=': case '*': case ' ': case '\t':
4036 106855852 : case '$': case '^' : case '%': case '?': case '!':
4037 106855852 : break;
4038 524875661 : default: return str;
4039 : }
4040 : }
4041 :
4042 : /* Takes a string of 0 or more comma-separated constraints. When more
4043 : than one constraint is present, evaluate whether they all correspond
4044 : to a single, repeated constraint (e.g. "r,r") or whether we have
4045 : more than one distinct constraints (e.g. "r,m"). */
4046 : static bool
4047 166438718 : constraint_unique (const char *cstr)
4048 : {
4049 166438718 : enum constraint_num ca, cb;
4050 166438718 : ca = CONSTRAINT__UNKNOWN;
4051 330360889 : for (;;)
4052 : {
4053 330360889 : cstr = skip_constraint_modifiers (cstr);
4054 330360889 : if (*cstr == '\0' || *cstr == ',')
4055 : cb = CONSTRAINT_X;
4056 : else
4057 : {
4058 330360889 : cb = lookup_constraint (cstr);
4059 330360889 : if (cb == CONSTRAINT__UNKNOWN)
4060 : return false;
4061 318086498 : cstr += CONSTRAINT_LEN (cstr[0], cstr);
4062 : }
4063 : /* Handle the first iteration of the loop. */
4064 318086498 : if (ca == CONSTRAINT__UNKNOWN)
4065 : ca = cb;
4066 : /* Handle the general case of comparing ca with subsequent
4067 : constraints. */
4068 163756182 : else if (ca != cb)
4069 : return false;
4070 171639893 : if (*cstr == '\0')
4071 : return true;
4072 163922171 : if (*cstr == ',')
4073 90735247 : cstr += 1;
4074 : }
4075 : }
4076 :
4077 : /* Major function to make reloads for an address in operand NOP or
4078 : check its correctness (If CHECK_ONLY_P is true). The supported
4079 : cases are:
4080 :
4081 : 1) an address that existed before LRA started, at which point it
4082 : must have been valid. These addresses are subject to elimination
4083 : and may have become invalid due to the elimination offset being out
4084 : of range.
4085 :
4086 : 2) an address created by forcing a constant to memory
4087 : (force_const_to_mem). The initial form of these addresses might
4088 : not be valid, and it is this function's job to make them valid.
4089 :
4090 : 3) a frame address formed from a register and a (possibly zero)
4091 : constant offset. As above, these addresses might not be valid and
4092 : this function must make them so.
4093 :
4094 : Add reloads to the lists *BEFORE and *AFTER. We might need to add
4095 : reloads to *AFTER because of inc/dec, {pre, post} modify in the
4096 : address. Return true for any RTL change.
4097 :
4098 : The function is a helper function which does not produce all
4099 : transformations (when CHECK_ONLY_P is false) which can be
4100 : necessary. It does just basic steps. To do all necessary
4101 : transformations use function process_address. */
4102 : static bool
4103 180505611 : process_address_1 (int nop, bool check_only_p,
4104 : rtx_insn **before, rtx_insn **after)
4105 : {
4106 180505611 : struct address_info ad;
4107 180505611 : rtx new_reg;
4108 180505611 : HOST_WIDE_INT scale;
4109 180505611 : rtx op = *curr_id->operand_loc[nop];
4110 180505611 : rtx mem = extract_mem_from_operand (op);
4111 180505611 : const char *constraint;
4112 180505611 : enum constraint_num cn;
4113 180505611 : bool change_p = false;
4114 :
4115 180505611 : if (MEM_P (mem)
4116 38353921 : && GET_MODE (mem) == BLKmode
4117 26350 : && GET_CODE (XEXP (mem, 0)) == SCRATCH)
4118 : return false;
4119 :
4120 180505611 : constraint
4121 180505611 : = skip_constraint_modifiers (curr_static_id->operand[nop].constraint);
4122 180505611 : if (IN_RANGE (constraint[0], '0', '9'))
4123 : {
4124 14009161 : char *end;
4125 14009161 : unsigned long dup = strtoul (constraint, &end, 10);
4126 14009161 : constraint
4127 14009161 : = skip_constraint_modifiers (curr_static_id->operand[dup].constraint);
4128 : }
4129 192730260 : cn = lookup_constraint (*constraint == '\0' ? "X" : constraint);
4130 : /* If we have several alternatives or/and several constraints in an
4131 : alternative and we can not say at this stage what constraint will be used,
4132 : use unknown constraint. The exception is an address constraint. If
4133 : operand has one address constraint, probably all others constraints are
4134 : address ones. */
4135 168280962 : if (constraint[0] != '\0' && get_constraint_type (cn) != CT_ADDRESS
4136 346944329 : && !constraint_unique (constraint))
4137 : cn = CONSTRAINT__UNKNOWN;
4138 21784615 : if (insn_extra_address_constraint (cn)
4139 : /* When we find an asm operand with an address constraint that
4140 : doesn't satisfy address_operand to begin with, we clear
4141 : is_address, so that we don't try to make a non-address fit.
4142 : If the asm statement got this far, it's because other
4143 : constraints are available, and we'll use them, disregarding
4144 : the unsatisfiable address ones. */
4145 21784615 : && curr_static_id->operand[nop].is_address)
4146 1842225 : decompose_lea_address (&ad, curr_id->operand_loc[nop]);
4147 : /* Do not attempt to decompose arbitrary addresses generated by combine
4148 : for asm operands with loose constraints, e.g 'X'.
4149 : Need to extract memory from op for special memory constraint,
4150 : i.e. bcst_mem_operand in i386 backend. */
4151 178663386 : else if (MEM_P (mem)
4152 178663539 : && !(INSN_CODE (curr_insn) < 0
4153 22410 : && get_constraint_type (cn) == CT_FIXED_FORM
4154 153 : && constraint_satisfied_p (op, cn)))
4155 38353768 : decompose_mem_address (&ad, mem);
4156 140309618 : else if (GET_CODE (op) == SUBREG
4157 3795807 : && MEM_P (SUBREG_REG (op)))
4158 0 : decompose_mem_address (&ad, SUBREG_REG (op));
4159 : else
4160 : return false;
4161 : /* If INDEX_REG_CLASS is assigned to base_term already and isn't to
4162 : index_term, swap them so to avoid assigning INDEX_REG_CLASS to both
4163 : when INDEX_REG_CLASS is a single register class. */
4164 40195993 : enum reg_class index_cl = index_reg_class (curr_insn);
4165 40195993 : if (ad.base_term != NULL
4166 33954579 : && ad.index_term != NULL
4167 1447041 : && ira_class_hard_regs_num[index_cl] == 1
4168 0 : && REG_P (*ad.base_term)
4169 0 : && REG_P (*ad.index_term)
4170 0 : && in_class_p (*ad.base_term, index_cl, NULL)
4171 40195993 : && ! in_class_p (*ad.index_term, index_cl, NULL))
4172 : {
4173 0 : std::swap (ad.base, ad.index);
4174 0 : std::swap (ad.base_term, ad.index_term);
4175 : }
4176 40195993 : if (! check_only_p)
4177 40188827 : change_p = equiv_address_substitution (&ad);
4178 40195993 : if (ad.base_term != NULL
4179 74150572 : && (process_addr_reg
4180 67909158 : (ad.base_term, check_only_p, before,
4181 33954579 : (ad.autoinc_p
4182 4187330 : && !(REG_P (*ad.base_term)
4183 2093665 : && find_regno_note (curr_insn, REG_DEAD,
4184 : REGNO (*ad.base_term)) != NULL_RTX)
4185 : ? after : NULL),
4186 : base_reg_class (ad.mode, ad.as, ad.base_outer_code,
4187 : get_index_code (&ad), curr_insn))))
4188 : {
4189 435648 : change_p = true;
4190 435648 : if (ad.base_term2 != NULL)
4191 0 : *ad.base_term2 = *ad.base_term;
4192 : }
4193 40195993 : if (ad.index_term != NULL
4194 40195993 : && process_addr_reg (ad.index_term, check_only_p,
4195 : before, NULL, index_cl))
4196 : change_p = true;
4197 :
4198 : /* Target hooks sometimes don't treat extra-constraint addresses as
4199 : legitimate address_operands, so handle them specially. */
4200 40195993 : if (insn_extra_address_constraint (cn)
4201 40195993 : && satisfies_address_constraint_p (&ad, cn))
4202 : return change_p;
4203 :
4204 38353775 : if (check_only_p)
4205 : return change_p;
4206 :
4207 : /* There are three cases where the shape of *AD.INNER may now be invalid:
4208 :
4209 : 1) the original address was valid, but either elimination or
4210 : equiv_address_substitution was applied and that made
4211 : the address invalid.
4212 :
4213 : 2) the address is an invalid symbolic address created by
4214 : force_const_to_mem.
4215 :
4216 : 3) the address is a frame address with an invalid offset.
4217 :
4218 : 4) the address is a frame address with an invalid base.
4219 :
4220 : All these cases involve a non-autoinc address, so there is no
4221 : point revalidating other types. */
4222 38347298 : if (ad.autoinc_p || valid_address_p (op, &ad, cn))
4223 : return change_p;
4224 :
4225 : /* Any index existed before LRA started, so we can assume that the
4226 : presence and shape of the index is valid. */
4227 424 : push_to_sequence (*before);
4228 424 : lra_assert (ad.disp == ad.disp_term);
4229 424 : if (ad.base == NULL)
4230 : {
4231 324 : if (ad.index == NULL)
4232 : {
4233 324 : rtx_insn *insn;
4234 324 : rtx_insn *last = get_last_insn ();
4235 324 : int code = -1;
4236 324 : enum reg_class cl = base_reg_class (ad.mode, ad.as,
4237 : SCRATCH, SCRATCH,
4238 : curr_insn);
4239 324 : rtx addr = *ad.inner;
4240 :
4241 639 : new_reg = lra_create_new_reg (Pmode, NULL_RTX, cl, NULL, "addr");
4242 324 : if (HAVE_lo_sum)
4243 : {
4244 : /* addr => lo_sum (new_base, addr), case (2) above. */
4245 : insn = emit_insn (gen_rtx_SET
4246 : (new_reg,
4247 : gen_rtx_HIGH (Pmode, copy_rtx (addr))));
4248 : code = recog_memoized (insn);
4249 : if (code >= 0)
4250 : {
4251 : *ad.inner = gen_rtx_LO_SUM (Pmode, new_reg, addr);
4252 : if (!valid_address_p (op, &ad, cn))
4253 : {
4254 : /* Try to put lo_sum into register. */
4255 : insn = emit_insn (gen_rtx_SET
4256 : (new_reg,
4257 : gen_rtx_LO_SUM (Pmode, new_reg, addr)));
4258 : code = recog_memoized (insn);
4259 : if (code >= 0)
4260 : {
4261 : *ad.inner = new_reg;
4262 : if (!valid_address_p (op, &ad, cn))
4263 : {
4264 : *ad.inner = addr;
4265 : code = -1;
4266 : }
4267 : }
4268 :
4269 : }
4270 : }
4271 : if (code < 0)
4272 : delete_insns_since (last);
4273 : }
4274 :
4275 324 : if (code < 0)
4276 : {
4277 : /* addr => new_base, case (2) above. */
4278 324 : lra_emit_move (new_reg, addr);
4279 :
4280 648 : for (insn = last == NULL_RTX ? get_insns () : NEXT_INSN (last);
4281 648 : insn != NULL_RTX;
4282 324 : insn = NEXT_INSN (insn))
4283 324 : if (recog_memoized (insn) < 0)
4284 : break;
4285 324 : if (insn != NULL_RTX)
4286 : {
4287 : /* Do nothing if we cannot generate right insns.
4288 : This is analogous to reload pass behavior. */
4289 0 : delete_insns_since (last);
4290 0 : end_sequence ();
4291 0 : return false;
4292 : }
4293 324 : *ad.inner = new_reg;
4294 : }
4295 : }
4296 : else
4297 : {
4298 : /* index * scale + disp => new base + index * scale,
4299 : case (1) above. */
4300 0 : enum reg_class cl = base_reg_class (ad.mode, ad.as, PLUS,
4301 0 : GET_CODE (*ad.index),
4302 : curr_insn);
4303 :
4304 0 : lra_assert (index_cl != NO_REGS);
4305 0 : new_reg = lra_create_new_reg (Pmode, NULL_RTX, cl, NULL, "disp");
4306 0 : lra_emit_move (new_reg, *ad.disp);
4307 0 : *ad.inner = simplify_gen_binary (PLUS, GET_MODE (new_reg),
4308 0 : new_reg, *ad.index);
4309 : }
4310 : }
4311 100 : else if (ad.index == NULL)
4312 : {
4313 53 : int regno;
4314 53 : enum reg_class cl;
4315 53 : rtx set;
4316 53 : rtx_insn *insns, *last_insn;
4317 :
4318 53 : cl = base_reg_class (ad.mode, ad.as, ad.base_outer_code,
4319 : get_index_code (&ad), curr_insn);
4320 :
4321 53 : if (REG_P (*ad.base_term)
4322 53 : && ira_class_subset_p[get_reg_class (REGNO (*ad.base_term))][cl])
4323 : /* It seems base reg is already in the base reg class and changing it
4324 : does not make a progress. So reload the whole inner address. */
4325 53 : goto reload_inner_addr;
4326 :
4327 : /* Try to reload base into register only if the base is invalid
4328 : for the address but with valid offset, case (4) above. */
4329 0 : start_sequence ();
4330 0 : new_reg = base_to_reg (&ad);
4331 :
4332 : /* base + disp => new base, cases (1) and (3) above. */
4333 : /* Another option would be to reload the displacement into an
4334 : index register. However, postreload has code to optimize
4335 : address reloads that have the same base and different
4336 : displacements, so reloading into an index register would
4337 : not necessarily be a win. */
4338 0 : if (new_reg == NULL_RTX)
4339 : {
4340 : /* See if the target can split the displacement into a
4341 : legitimate new displacement from a local anchor. */
4342 0 : gcc_assert (ad.disp == ad.disp_term);
4343 0 : poly_int64 orig_offset;
4344 0 : rtx offset1, offset2;
4345 0 : if (poly_int_rtx_p (*ad.disp, &orig_offset)
4346 0 : && targetm.legitimize_address_displacement (&offset1, &offset2,
4347 : orig_offset,
4348 : ad.mode))
4349 : {
4350 0 : new_reg = base_plus_disp_to_reg (&ad, offset1);
4351 0 : new_reg = gen_rtx_PLUS (GET_MODE (new_reg), new_reg, offset2);
4352 : }
4353 : else
4354 0 : new_reg = base_plus_disp_to_reg (&ad, *ad.disp);
4355 : }
4356 0 : insns = get_insns ();
4357 0 : last_insn = get_last_insn ();
4358 : /* If we generated at least two insns, try last insn source as
4359 : an address. If we succeed, we generate one less insn. */
4360 0 : if (REG_P (new_reg)
4361 0 : && last_insn != insns
4362 0 : && (set = single_set (last_insn)) != NULL_RTX
4363 0 : && GET_CODE (SET_SRC (set)) == PLUS
4364 0 : && REG_P (XEXP (SET_SRC (set), 0))
4365 0 : && CONSTANT_P (XEXP (SET_SRC (set), 1)))
4366 : {
4367 0 : *ad.inner = SET_SRC (set);
4368 0 : if (valid_address_p (op, &ad, cn))
4369 : {
4370 0 : *ad.base_term = XEXP (SET_SRC (set), 0);
4371 0 : *ad.disp_term = XEXP (SET_SRC (set), 1);
4372 0 : regno = REGNO (*ad.base_term);
4373 0 : if (regno >= FIRST_PSEUDO_REGISTER
4374 0 : && cl != lra_get_allocno_class (regno))
4375 0 : lra_change_class (regno, cl, " Change to", true);
4376 0 : new_reg = SET_SRC (set);
4377 0 : delete_insns_since (PREV_INSN (last_insn));
4378 : }
4379 : }
4380 0 : end_sequence ();
4381 0 : emit_insn (insns);
4382 0 : *ad.inner = new_reg;
4383 : }
4384 47 : else if (ad.disp_term != NULL)
4385 : {
4386 : /* base + scale * index + disp => new base + scale * index,
4387 : case (1) above. */
4388 47 : gcc_assert (ad.disp == ad.disp_term);
4389 47 : new_reg = base_plus_disp_to_reg (&ad, *ad.disp);
4390 47 : *ad.inner = simplify_gen_binary (PLUS, GET_MODE (new_reg),
4391 47 : new_reg, *ad.index);
4392 : }
4393 0 : else if ((scale = get_index_scale (&ad)) == 1)
4394 : {
4395 : /* The last transformation to one reg will be made in
4396 : curr_insn_transform function. */
4397 0 : end_sequence ();
4398 0 : return false;
4399 : }
4400 0 : else if (scale != 0)
4401 : {
4402 : /* base + scale * index => base + new_reg,
4403 : case (1) above.
4404 : Index part of address may become invalid. For example, we
4405 : changed pseudo on the equivalent memory and a subreg of the
4406 : pseudo onto the memory of different mode for which the scale is
4407 : prohibited. */
4408 0 : new_reg = index_part_to_reg (&ad, index_cl);
4409 0 : *ad.inner = simplify_gen_binary (PLUS, GET_MODE (new_reg),
4410 0 : *ad.base_term, new_reg);
4411 : }
4412 : else
4413 : {
4414 53 : enum reg_class cl;
4415 53 : rtx addr;
4416 0 : reload_inner_addr:
4417 53 : cl = base_reg_class (ad.mode, ad.as, SCRATCH, SCRATCH, curr_insn);
4418 53 : addr = *ad.inner;
4419 53 : new_reg = lra_create_new_reg (Pmode, NULL_RTX, cl, NULL, "addr");
4420 : /* addr => new_base. */
4421 53 : lra_emit_move (new_reg, addr);
4422 53 : *ad.inner = new_reg;
4423 : }
4424 424 : *before = end_sequence ();
4425 424 : return true;
4426 : }
4427 :
4428 : /* If CHECK_ONLY_P is false, do address reloads until it is necessary.
4429 : Use process_address_1 as a helper function. Return true for any
4430 : RTL changes.
4431 :
4432 : If CHECK_ONLY_P is true, just check address correctness. Return
4433 : false if the address correct. */
4434 : static bool
4435 180016691 : process_address (int nop, bool check_only_p,
4436 : rtx_insn **before, rtx_insn **after)
4437 : {
4438 180016691 : bool res = false;
4439 : /* Use enough iterations to process all address parts: */
4440 180505611 : for (int i = 0; i < 10; i++)
4441 : {
4442 180505611 : if (!process_address_1 (nop, check_only_p, before, after))
4443 : {
4444 : return res;
4445 : }
4446 : else
4447 : {
4448 488920 : if (check_only_p)
4449 : return true;
4450 488920 : res = true;
4451 : }
4452 : }
4453 0 : fatal_insn ("unable to reload address in ", curr_insn);
4454 : }
4455 :
4456 : /* Override the generic address_reload_context in order to
4457 : control the creation of reload pseudos. */
4458 : class lra_autoinc_reload_context : public address_reload_context
4459 : {
4460 : machine_mode mode;
4461 : enum reg_class rclass;
4462 :
4463 : public:
4464 0 : lra_autoinc_reload_context (machine_mode mode, enum reg_class new_rclass)
4465 0 : : mode (mode), rclass (new_rclass) {}
4466 :
4467 0 : rtx get_reload_reg () const override final
4468 : {
4469 0 : return lra_create_new_reg (mode, NULL_RTX, rclass, NULL, "INC/DEC result");
4470 : }
4471 : };
4472 :
4473 : /* Emit insns to reload VALUE into a new register. VALUE is an
4474 : auto-increment or auto-decrement RTX whose operand is a register or
4475 : memory location; so reloading involves incrementing that location.
4476 :
4477 : INC_AMOUNT is the number to increment or decrement by (always
4478 : positive and ignored for POST_MODIFY/PRE_MODIFY).
4479 :
4480 : Return a pseudo containing the result. */
4481 : static rtx
4482 0 : emit_inc (enum reg_class new_rclass, rtx value, poly_int64 inc_amount)
4483 : {
4484 0 : lra_autoinc_reload_context context (GET_MODE (value), new_rclass);
4485 0 : return context.emit_autoinc (value, inc_amount);
4486 : }
4487 :
4488 : /* Return true if the current move insn does not need processing as we
4489 : already know that it satisfies its constraints. */
4490 : static bool
4491 103211419 : simple_move_p (void)
4492 : {
4493 103211419 : rtx dest, src;
4494 103211419 : enum reg_class dclass, sclass;
4495 :
4496 103211419 : lra_assert (curr_insn_set != NULL_RTX);
4497 103211419 : dest = SET_DEST (curr_insn_set);
4498 103211419 : src = SET_SRC (curr_insn_set);
4499 :
4500 : /* If the instruction has multiple sets we need to process it even if it
4501 : is single_set. This can happen if one or more of the SETs are dead.
4502 : See PR73650. */
4503 103211419 : if (multiple_sets (curr_insn))
4504 : return false;
4505 :
4506 103009428 : return ((dclass = get_op_class (dest)) != NO_REGS
4507 21815694 : && (sclass = get_op_class (src)) != NO_REGS
4508 : /* The backend guarantees that register moves of cost 2
4509 : never need reloads. */
4510 92118329 : && targetm.register_move_cost (GET_MODE (src), sclass, dclass) == 2);
4511 : }
4512 :
4513 : /* Swap operands NOP and NOP + 1. */
4514 : static inline void
4515 21848308 : swap_operands (int nop)
4516 : {
4517 21848308 : std::swap (curr_operand_mode[nop], curr_operand_mode[nop + 1]);
4518 21848308 : std::swap (original_subreg_reg_mode[nop], original_subreg_reg_mode[nop + 1]);
4519 21848308 : std::swap (*curr_id->operand_loc[nop], *curr_id->operand_loc[nop + 1]);
4520 21848308 : std::swap (equiv_substition_p[nop], equiv_substition_p[nop + 1]);
4521 : /* Swap the duplicates too. */
4522 21848308 : lra_update_dup (curr_id, nop);
4523 21848308 : lra_update_dup (curr_id, nop + 1);
4524 21848308 : }
4525 :
4526 : /* Return TRUE if X is a (subreg of) reg and there are no hard regs of X class
4527 : which can contain value of MODE. */
4528 38 : static bool invalid_mode_reg_p (enum machine_mode mode, rtx x)
4529 : {
4530 38 : if (SUBREG_P (x))
4531 3 : x = SUBREG_REG (x);
4532 38 : if (! REG_P (x))
4533 : return false;
4534 38 : enum reg_class rclass = get_reg_class (REGNO (x));
4535 38 : return (!hard_reg_set_empty_p (reg_class_contents[rclass])
4536 38 : && hard_reg_set_subset_p
4537 38 : (reg_class_contents[rclass],
4538 38 : ira_prohibited_class_mode_regs[rclass][mode]));
4539 : }
4540 :
4541 : /* Return TRUE if regno is referenced in more than one non-debug insn. */
4542 : static bool
4543 2969155 : multiple_insn_refs_p (int regno)
4544 : {
4545 2969155 : unsigned int uid;
4546 2969155 : bitmap_iterator bi;
4547 2969155 : int nrefs = 0;
4548 7170887 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi)
4549 : {
4550 7165223 : if (!NONDEBUG_INSN_P (lra_insn_recog_data[uid]->insn))
4551 1232577 : continue;
4552 5932646 : if (nrefs == 1)
4553 : return true;
4554 2969155 : nrefs++;
4555 : }
4556 : return false;
4557 : }
4558 :
4559 : /* Mark insns starting with FIRST as postponed for processing their
4560 : constraints. See comments for lra_postponed_insns. */
4561 : static void
4562 95816 : postpone_insns (rtx_insn *first)
4563 : {
4564 108472 : for (auto insn = first; insn != NULL_RTX; insn = NEXT_INSN (insn))
4565 : {
4566 12656 : bitmap_set_bit (&lra_postponed_insns, INSN_UID (insn));
4567 12656 : if (lra_dump_file != NULL)
4568 : {
4569 7 : fprintf (lra_dump_file, " Postponing constraint processing: ");
4570 7 : dump_insn_slim (lra_dump_file, insn);
4571 : }
4572 : }
4573 95816 : }
4574 :
4575 : /* Test whether the n-th operand is a MEM where the address is the sum of a
4576 : section anchor and a constant and return true in case of reloading the
4577 : section anchor only results in a satisfiable operand w.r.t. its
4578 : corresponding constraint. Otherwise return false. */
4579 :
4580 : static bool
4581 72 : reload_section_anchor_p (int nop)
4582 : {
4583 72 : rtx op = *curr_id->operand_loc[nop];
4584 72 : if (!MEM_P (op))
4585 : return false;
4586 72 : rtx addr = XEXP (op, 0);
4587 :
4588 72 : if (GET_CODE (addr) != CONST
4589 13 : || GET_CODE (XEXP (addr, 0)) != PLUS
4590 0 : || GET_CODE (XEXP (XEXP (addr, 0), 0)) != SYMBOL_REF
4591 0 : || !SYMBOL_REF_ANCHOR_P (XEXP (XEXP (addr, 0), 0))
4592 0 : || !CONST_INT_P (XEXP (XEXP (addr, 0), 1))
4593 : /* Some offsets are valid in conjunction with a symbol and
4594 : invalid in conjunction with a register. Thus, pull out
4595 : the anchor only in case the offset is a valid anchor
4596 : offset. */
4597 0 : || INTVAL (XEXP (XEXP (addr, 0), 1)) < targetm.min_anchor_offset
4598 72 : || INTVAL (XEXP (XEXP (addr, 0), 1)) > targetm.max_anchor_offset)
4599 : return false;
4600 :
4601 : /* Now test whether a new address of the form REG+DISPLACEMENT is valid for
4602 : the selected alternative. In order to do so, utilize lra_pmode_pseudo
4603 : instead of an actual reload register. */
4604 :
4605 0 : rtx offset = XEXP (XEXP (addr, 0), 1);
4606 0 : rtx new_addr = gen_rtx_PLUS (Pmode, lra_pmode_pseudo, offset);
4607 0 : rtx new_op = shallow_copy_rtx (op);
4608 0 : XEXP (new_op, 0) = new_addr;
4609 :
4610 : /* Get operand constraints for given alternative. */
4611 0 : const char *p = (curr_static_id->operand_alternative
4612 0 : [goal_alt_number * curr_static_id->n_operands + nop]
4613 : .constraint);
4614 0 : char c;
4615 0 : for (;
4616 0 : (c = *p) && c != ',' && c != '#';
4617 0 : p += CONSTRAINT_LEN (c, p))
4618 : {
4619 0 : enum constraint_num cn = lookup_constraint (p);
4620 0 : if (constraint_satisfied_p (new_op, cn))
4621 : return true;
4622 : }
4623 :
4624 : return false;
4625 : }
4626 :
4627 : /* Main entry point of the constraint code: search the body of the
4628 : current insn to choose the best alternative. It is mimicking insn
4629 : alternative cost calculation model of former reload pass. That is
4630 : because machine descriptions were written to use this model. This
4631 : model can be changed in future. Make commutative operand exchange
4632 : if it is chosen.
4633 :
4634 : if CHECK_ONLY_P is false, do RTL changes to satisfy the
4635 : constraints. Return true if any change happened during function
4636 : call.
4637 :
4638 : If CHECK_ONLY_P is true then don't do any transformation. Just
4639 : check that the insn satisfies all constraints. If the insn does
4640 : not satisfy any constraint, return true. */
4641 : static bool
4642 108694046 : curr_insn_transform (bool check_only_p)
4643 : {
4644 108694046 : int i, j, k;
4645 108694046 : int n_operands;
4646 108694046 : int n_alternatives;
4647 108694046 : int n_outputs;
4648 108694046 : int commutative;
4649 108694046 : signed char goal_alt_matched[MAX_RECOG_OPERANDS][MAX_RECOG_OPERANDS];
4650 108694046 : signed char match_inputs[MAX_RECOG_OPERANDS + 1];
4651 108694046 : signed char outputs[MAX_RECOG_OPERANDS + 1];
4652 108694046 : rtx_insn *before, *after;
4653 108694046 : bool alt_p = false;
4654 : /* Flag that the insn has been changed through a transformation. */
4655 108694046 : bool change_p;
4656 108694046 : bool sec_mem_p;
4657 108694046 : bool use_sec_mem_p;
4658 108694046 : int max_regno_before;
4659 108694046 : int reused_alternative_num;
4660 :
4661 108694046 : curr_insn_set = single_set (curr_insn);
4662 108694046 : if (curr_insn_set != NULL_RTX && simple_move_p ())
4663 : {
4664 : /* We assume that the corresponding insn alternative has no
4665 : earlier clobbers. If it is not the case, don't define move
4666 : cost equal to 2 for the corresponding register classes. */
4667 16746363 : lra_set_used_insn_alternative (curr_insn, LRA_NON_CLOBBERED_ALT);
4668 16746363 : return false;
4669 : }
4670 :
4671 91947683 : no_input_reloads_p = no_output_reloads_p = false;
4672 91947683 : goal_alt_number = -1;
4673 91947683 : change_p = sec_mem_p = false;
4674 :
4675 : /* CALL_INSNs are not allowed to have any output reloads. */
4676 91947683 : if (CALL_P (curr_insn))
4677 6153600 : no_output_reloads_p = true;
4678 :
4679 91947683 : n_operands = curr_static_id->n_operands;
4680 91947683 : n_alternatives = curr_static_id->n_alternatives;
4681 :
4682 : /* Just return "no reloads" if insn has no operands with
4683 : constraints. */
4684 91947683 : if (n_operands == 0 || n_alternatives == 0)
4685 : return false;
4686 :
4687 81306074 : max_regno_before = max_reg_num ();
4688 :
4689 343967275 : for (i = 0; i < n_operands; i++)
4690 : {
4691 181355127 : goal_alt_matched[i][0] = -1;
4692 181355127 : goal_alt_matches[i] = -1;
4693 : }
4694 :
4695 81306074 : commutative = curr_static_id->commutative;
4696 :
4697 : /* Now see what we need for pseudos that didn't get hard regs or got
4698 : the wrong kind of hard reg. For this, we must consider all the
4699 : operands together against the register constraints. */
4700 :
4701 81306074 : best_losers = best_overall = INT_MAX;
4702 81306074 : best_reload_sum = 0;
4703 :
4704 81306074 : curr_swapped = false;
4705 81306074 : goal_alt_swapped = false;
4706 :
4707 81306074 : if (! check_only_p)
4708 : /* Make equivalence substitution and memory subreg elimination
4709 : before address processing because an address legitimacy can
4710 : depend on memory mode. */
4711 262591790 : for (i = 0; i < n_operands; i++)
4712 : {
4713 181305095 : rtx op, subst, old;
4714 181305095 : bool op_change_p = false;
4715 :
4716 181305095 : if (curr_static_id->operand[i].is_operator)
4717 1466872 : continue;
4718 :
4719 179838223 : old = op = *curr_id->operand_loc[i];
4720 179838223 : if (GET_CODE (old) == SUBREG)
4721 3852875 : old = SUBREG_REG (old);
4722 179838223 : subst = get_equiv_with_elimination (old, curr_insn);
4723 179838223 : original_subreg_reg_mode[i] = VOIDmode;
4724 179838223 : equiv_substition_p[i] = false;
4725 :
4726 179838223 : if (subst != old
4727 : /* We don't want to change an out operand by constant or invariant
4728 : which will require additional reloads, e.g. by putting a constant
4729 : into memory. */
4730 1610358 : && (curr_static_id->operand[i].type == OP_IN || MEM_P (subst)
4731 0 : || (GET_CODE (subst) == SUBREG && MEM_P (SUBREG_REG (subst)))))
4732 : {
4733 1610358 : equiv_substition_p[i] = true;
4734 1610358 : rtx new_subst = copy_rtx (subst);
4735 1610358 : if (lra_pointer_equiv_set_in (subst))
4736 827217 : lra_pointer_equiv_set_add (new_subst);
4737 1610358 : subst = new_subst;
4738 1610358 : lra_assert (REG_P (old));
4739 1610358 : if (GET_CODE (op) != SUBREG)
4740 1553633 : *curr_id->operand_loc[i] = subst;
4741 : else
4742 : {
4743 56725 : SUBREG_REG (op) = subst;
4744 56725 : if (GET_MODE (subst) == VOIDmode)
4745 88 : original_subreg_reg_mode[i] = GET_MODE (old);
4746 : }
4747 1610358 : if (lra_dump_file != NULL)
4748 : {
4749 3 : fprintf (lra_dump_file,
4750 : "Changing pseudo %d in operand %i of insn %u on equiv ",
4751 3 : REGNO (old), i, INSN_UID (curr_insn));
4752 3 : dump_value_slim (lra_dump_file, subst, 1);
4753 3 : fprintf (lra_dump_file, "\n");
4754 : }
4755 1610358 : op_change_p = change_p = true;
4756 : }
4757 179838223 : if (simplify_operand_subreg (i, GET_MODE (old)) || op_change_p)
4758 : {
4759 1610874 : change_p = true;
4760 1610874 : lra_update_dup (curr_id, i);
4761 : }
4762 : }
4763 :
4764 : /* We process equivalences before ignoring postponed insns on the current
4765 : constraint sub-pass but before any reload insn generation for the
4766 : postponed insn. */
4767 81286695 : if (! check_only_p
4768 81286695 : && bitmap_bit_p (&lra_postponed_insns, INSN_UID (curr_insn)))
4769 : return true;
4770 :
4771 : /* Reload address registers and displacements. We do it before
4772 : finding an alternative because of memory constraints. */
4773 81299303 : before = after = NULL;
4774 262640888 : for (i = 0; i < n_operands; i++)
4775 181341585 : if (! curr_static_id->operand[i].is_operator
4776 181341585 : && process_address (i, check_only_p, &before, &after))
4777 : {
4778 488917 : if (check_only_p)
4779 : return true;
4780 488917 : change_p = true;
4781 488917 : lra_update_dup (curr_id, i);
4782 : }
4783 :
4784 81299303 : if (change_p)
4785 : /* If we've changed the instruction then any alternative that
4786 : we chose previously may no longer be valid. */
4787 2053612 : lra_set_used_insn_alternative (curr_insn, LRA_UNKNOWN_ALT);
4788 :
4789 81279924 : if (! check_only_p && curr_insn_set != NULL_RTX
4790 158659080 : && check_and_process_move (&change_p, &sec_mem_p))
4791 0 : return change_p;
4792 :
4793 81299303 : try_swapped:
4794 :
4795 91892083 : reused_alternative_num = check_only_p ? LRA_UNKNOWN_ALT : curr_id->used_insn_alternative;
4796 91892083 : if (lra_dump_file != NULL && reused_alternative_num >= 0)
4797 0 : fprintf (lra_dump_file, "Reusing alternative %d for insn #%u\n",
4798 0 : reused_alternative_num, INSN_UID (curr_insn));
4799 :
4800 91892083 : if (process_alt_operands (reused_alternative_num))
4801 83147911 : alt_p = true;
4802 :
4803 91892083 : if (check_only_p)
4804 19379 : return ! alt_p || best_losers != 0;
4805 :
4806 : /* If insn is commutative (it's safe to exchange a certain pair of
4807 : operands) then we need to try each alternative twice, the second
4808 : time matching those two operands as if we had exchanged them. To
4809 : do this, really exchange them in operands.
4810 :
4811 : If we have just tried the alternatives the second time, return
4812 : operands to normal and drop through. */
4813 :
4814 91872704 : if (reused_alternative_num < 0 && commutative >= 0)
4815 : {
4816 21185560 : curr_swapped = !curr_swapped;
4817 21185560 : if (curr_swapped)
4818 : {
4819 10592780 : swap_operands (commutative);
4820 10592780 : goto try_swapped;
4821 : }
4822 : else
4823 10592780 : swap_operands (commutative);
4824 : }
4825 :
4826 81279924 : if (! alt_p && ! sec_mem_p)
4827 : {
4828 : /* No alternative works with reloads?? */
4829 6 : if (INSN_CODE (curr_insn) >= 0)
4830 0 : fatal_insn ("unable to generate reloads for:", curr_insn);
4831 6 : error_for_asm (curr_insn,
4832 : "inconsistent operand constraints in an %<asm%>");
4833 6 : lra_asm_error_p = true;
4834 6 : if (! JUMP_P (curr_insn))
4835 : {
4836 : /* Avoid further trouble with this insn. Don't generate use
4837 : pattern here as we could use the insn SP offset. */
4838 6 : lra_set_insn_deleted (curr_insn);
4839 : }
4840 : else
4841 : {
4842 0 : lra_invalidate_insn_data (curr_insn);
4843 0 : ira_nullify_asm_goto (curr_insn);
4844 0 : lra_update_insn_regno_info (curr_insn);
4845 : }
4846 : return true;
4847 : }
4848 :
4849 : /* If the best alternative is with operands 1 and 2 swapped, swap
4850 : them. Update the operand numbers of any reloads already
4851 : pushed. */
4852 :
4853 81279918 : if (goal_alt_swapped)
4854 : {
4855 657962 : if (lra_dump_file != NULL)
4856 18 : fprintf (lra_dump_file, " Commutative operand exchange in insn %u\n",
4857 18 : INSN_UID (curr_insn));
4858 :
4859 : /* Swap the duplicates too. */
4860 657962 : swap_operands (commutative);
4861 657962 : change_p = true;
4862 : }
4863 :
4864 : /* Some targets' TARGET_SECONDARY_MEMORY_NEEDED (e.g. x86) are defined
4865 : too conservatively. So we use the secondary memory only if there
4866 : is no any alternative without reloads. */
4867 81279918 : use_sec_mem_p = false;
4868 81279918 : if (! alt_p)
4869 : use_sec_mem_p = true;
4870 81279918 : else if (sec_mem_p)
4871 : {
4872 14975 : for (i = 0; i < n_operands; i++)
4873 14803 : if (! goal_alt_win[i] && ! goal_alt_match_win[i])
4874 : break;
4875 13231 : use_sec_mem_p = i < n_operands;
4876 : }
4877 :
4878 13231 : if (use_sec_mem_p)
4879 : {
4880 13059 : int in = -1, out = -1;
4881 13059 : rtx new_reg, src, dest, rld;
4882 13059 : machine_mode sec_mode, rld_mode;
4883 :
4884 13059 : lra_assert (curr_insn_set != NULL_RTX && sec_mem_p);
4885 13059 : dest = SET_DEST (curr_insn_set);
4886 13059 : src = SET_SRC (curr_insn_set);
4887 39177 : for (i = 0; i < n_operands; i++)
4888 26118 : if (*curr_id->operand_loc[i] == dest)
4889 : out = i;
4890 13059 : else if (*curr_id->operand_loc[i] == src)
4891 13059 : in = i;
4892 13059 : for (i = 0; i < curr_static_id->n_dups; i++)
4893 0 : if (out < 0 && *curr_id->dup_loc[i] == dest)
4894 0 : out = curr_static_id->dup_num[i];
4895 0 : else if (in < 0 && *curr_id->dup_loc[i] == src)
4896 0 : in = curr_static_id->dup_num[i];
4897 13059 : lra_assert (out >= 0 && in >= 0
4898 : && curr_static_id->operand[out].type == OP_OUT
4899 : && curr_static_id->operand[in].type == OP_IN);
4900 13059 : rld = partial_subreg_p (GET_MODE (src), GET_MODE (dest)) ? src : dest;
4901 13059 : rld_mode = GET_MODE (rld);
4902 13059 : sec_mode = targetm.secondary_memory_needed_mode (rld_mode);
4903 13059 : if (rld_mode != sec_mode
4904 13059 : && (invalid_mode_reg_p (sec_mode, dest)
4905 19 : || invalid_mode_reg_p (sec_mode, src)))
4906 : sec_mode = rld_mode;
4907 13059 : new_reg = lra_create_new_reg (sec_mode, NULL_RTX, NO_REGS, NULL,
4908 : "secondary");
4909 : /* If the mode is changed, it should be wider. */
4910 13059 : lra_assert (!partial_subreg_p (sec_mode, rld_mode));
4911 13059 : if (sec_mode != rld_mode)
4912 : {
4913 : /* If the target says specifically to use another mode for
4914 : secondary memory moves we cannot reuse the original
4915 : insn. */
4916 19 : after = emit_spill_move (false, new_reg, dest);
4917 19 : lra_process_new_insns (curr_insn, NULL, after,
4918 : "Inserting the sec. move");
4919 : /* We may have non null BEFORE here (e.g. after address
4920 : processing. */
4921 19 : push_to_sequence (before);
4922 19 : before = emit_spill_move (true, new_reg, src);
4923 19 : emit_insn (before);
4924 19 : before = end_sequence ();
4925 19 : lra_process_new_insns (curr_insn, before, NULL, "Changing on");
4926 19 : lra_set_insn_deleted (curr_insn);
4927 : }
4928 13040 : else if (dest == rld)
4929 : {
4930 13040 : *curr_id->operand_loc[out] = new_reg;
4931 13040 : lra_update_dup (curr_id, out);
4932 13040 : after = emit_spill_move (false, new_reg, dest);
4933 13040 : lra_process_new_insns (curr_insn, NULL, after,
4934 : "Inserting the sec. move");
4935 : }
4936 : else
4937 : {
4938 0 : *curr_id->operand_loc[in] = new_reg;
4939 0 : lra_update_dup (curr_id, in);
4940 : /* See comments above. */
4941 0 : push_to_sequence (before);
4942 0 : before = emit_spill_move (true, new_reg, src);
4943 0 : emit_insn (before);
4944 0 : before = end_sequence ();
4945 0 : lra_process_new_insns (curr_insn, before, NULL,
4946 : "Inserting the sec. move");
4947 : }
4948 13059 : lra_update_insn_regno_info (curr_insn);
4949 13059 : return true;
4950 : }
4951 :
4952 81266859 : lra_assert (goal_alt_number >= 0);
4953 162440608 : lra_set_used_insn_alternative (curr_insn, goal_reuse_alt_p
4954 : ? goal_alt_number : LRA_UNKNOWN_ALT);
4955 :
4956 81266859 : if (lra_dump_file != NULL)
4957 : {
4958 1187 : const char *p;
4959 :
4960 1187 : fprintf (lra_dump_file, " Choosing alt %d in insn %u:",
4961 1187 : goal_alt_number, INSN_UID (curr_insn));
4962 1187 : print_curr_insn_alt (goal_alt_number);
4963 1187 : if (INSN_CODE (curr_insn) >= 0
4964 1187 : && (p = get_insn_name (INSN_CODE (curr_insn))) != NULL)
4965 1180 : fprintf (lra_dump_file, " {%s}", p);
4966 1187 : if (maybe_ne (curr_id->sp_offset, 0))
4967 : {
4968 0 : fprintf (lra_dump_file, " (sp_off=");
4969 0 : print_dec (curr_id->sp_offset, lra_dump_file);
4970 0 : fprintf (lra_dump_file, ")");
4971 : }
4972 1187 : fprintf (lra_dump_file, "\n");
4973 : }
4974 :
4975 : /* Right now, for any pair of operands I and J that are required to
4976 : match, with J < I, goal_alt_matches[I] is J. Add I to
4977 : goal_alt_matched[J]. */
4978 :
4979 262532280 : for (i = 0; i < n_operands; i++)
4980 181265421 : if ((j = goal_alt_matches[i]) >= 0)
4981 : {
4982 10720906 : for (k = 0; goal_alt_matched[j][k] >= 0; k++)
4983 : ;
4984 : /* We allow matching one output operand and several input
4985 : operands. */
4986 10720905 : lra_assert (k == 0
4987 : || (curr_static_id->operand[j].type == OP_OUT
4988 : && curr_static_id->operand[i].type == OP_IN
4989 : && (curr_static_id->operand
4990 : [goal_alt_matched[j][0]].type == OP_IN)));
4991 10720905 : goal_alt_matched[j][k] = i;
4992 10720905 : goal_alt_matched[j][k + 1] = -1;
4993 : }
4994 :
4995 262532280 : for (i = 0; i < n_operands; i++)
4996 181265421 : goal_alt_win[i] |= goal_alt_match_win[i];
4997 :
4998 : /* Any constants that aren't allowed and can't be reloaded into
4999 : registers are here changed into memory references. */
5000 262532280 : for (i = 0; i < n_operands; i++)
5001 181265421 : if (goal_alt_win[i])
5002 : {
5003 174929056 : int regno;
5004 174929056 : enum reg_class new_class;
5005 174929056 : rtx reg = *curr_id->operand_loc[i];
5006 :
5007 174929056 : if (GET_CODE (reg) == SUBREG)
5008 3522490 : reg = SUBREG_REG (reg);
5009 :
5010 174929056 : if (REG_P (reg) && (regno = REGNO (reg)) >= FIRST_PSEUDO_REGISTER)
5011 : {
5012 79507524 : bool ok_p = in_class_p (reg, goal_alt[i], &new_class, true);
5013 :
5014 79507524 : if (new_class != NO_REGS && get_reg_class (regno) != new_class)
5015 : {
5016 3632421 : lra_assert (ok_p);
5017 3632421 : lra_change_class (regno, new_class, " Change to", true);
5018 : }
5019 : }
5020 : }
5021 : else
5022 : {
5023 6336365 : const char *constraint;
5024 6336365 : char c;
5025 6336365 : rtx op = *curr_id->operand_loc[i];
5026 6336365 : rtx subreg = NULL_RTX;
5027 6336365 : machine_mode op_mode = curr_operand_mode[i], mode = op_mode;
5028 :
5029 6336365 : if (GET_CODE (op) == SUBREG)
5030 : {
5031 263596 : subreg = op;
5032 263596 : op = SUBREG_REG (op);
5033 263596 : mode = GET_MODE (op);
5034 : }
5035 :
5036 6552878 : if (CONST_POOL_OK_P (mode, op)
5037 6552878 : && ((targetm.preferred_reload_class
5038 216513 : (op, (enum reg_class) goal_alt[i]) == NO_REGS)
5039 74046 : || no_input_reloads_p))
5040 : {
5041 142467 : rtx tem = force_const_mem (mode, op);
5042 :
5043 142467 : change_p = true;
5044 142467 : if (subreg != NULL_RTX)
5045 0 : tem = gen_rtx_SUBREG (op_mode, tem, SUBREG_BYTE (subreg));
5046 :
5047 142467 : *curr_id->operand_loc[i] = tem;
5048 142467 : lra_update_dup (curr_id, i);
5049 142467 : process_address (i, false, &before, &after);
5050 :
5051 : /* If the alternative accepts constant pool refs directly
5052 : there will be no reload needed at all. */
5053 142467 : if (subreg != NULL_RTX)
5054 0 : continue;
5055 : /* Skip alternatives before the one requested. */
5056 142467 : constraint = (curr_static_id->operand_alternative
5057 142467 : [goal_alt_number * n_operands + i].constraint);
5058 142467 : for (;
5059 243449 : (c = *constraint) && c != ',' && c != '#';
5060 100982 : constraint += CONSTRAINT_LEN (c, constraint))
5061 : {
5062 197683 : enum constraint_num cn = lookup_constraint (constraint);
5063 197683 : if ((insn_extra_memory_constraint (cn)
5064 101236 : || insn_extra_special_memory_constraint (cn)
5065 : || insn_extra_relaxed_memory_constraint (cn))
5066 197937 : && satisfies_memory_constraint_p (tem, cn))
5067 : break;
5068 : }
5069 142467 : if (c == '\0' || c == ',' || c == '#')
5070 45766 : continue;
5071 :
5072 96701 : goal_alt_win[i] = true;
5073 : }
5074 : }
5075 :
5076 : n_outputs = 0;
5077 262532280 : for (i = 0; i < n_operands; i++)
5078 181265421 : if (curr_static_id->operand[i].type == OP_OUT)
5079 70546208 : outputs[n_outputs++] = i;
5080 81266859 : outputs[n_outputs] = -1;
5081 262532280 : for (i = 0; i < n_operands; i++)
5082 : {
5083 181265421 : int regno;
5084 181265421 : bool optional_p = false;
5085 181265421 : rtx old, new_reg;
5086 181265421 : rtx op = *curr_id->operand_loc[i];
5087 :
5088 181265421 : if (goal_alt_win[i])
5089 : {
5090 175025757 : if (goal_alt[i] == NO_REGS
5091 47707789 : && REG_P (op)
5092 5508402 : && (regno = REGNO (op)) >= FIRST_PSEUDO_REGISTER
5093 : /* We assigned a hard register to the pseudo in the past but now
5094 : decided to spill it for the insn. If the pseudo is used only
5095 : in this insn, it is better to spill it here as we free hard
5096 : registers for other pseudos referenced in the insn. The most
5097 : common case of this is a scratch register which will be
5098 : transformed to scratch back at the end of LRA. */
5099 177994912 : && !multiple_insn_refs_p (regno))
5100 : {
5101 11328 : if (lra_get_allocno_class (regno) != NO_REGS)
5102 5377 : lra_change_class (regno, NO_REGS, " Change to", true);
5103 5664 : reg_renumber[regno] = -1;
5104 : }
5105 : /* We can do an optional reload. If the pseudo got a hard
5106 : reg, we might improve the code through inheritance. If
5107 : it does not get a hard register we coalesce memory/memory
5108 : moves later. Ignore move insns to avoid cycling. */
5109 175025757 : if (! lra_simple_p
5110 174480139 : && lra_undo_inheritance_iter < LRA_MAX_INHERITANCE_PASSES
5111 161638008 : && goal_alt[i] != NO_REGS && REG_P (op)
5112 80258302 : && (regno = REGNO (op)) >= FIRST_PSEUDO_REGISTER
5113 67177189 : && regno < new_regno_start
5114 62271230 : && ! ira_former_scratch_p (regno)
5115 62213089 : && reg_renumber[regno] < 0
5116 : /* Check that the optional reload pseudo will be able to
5117 : hold given mode value. */
5118 3930337 : && ! (prohibited_class_reg_set_mode_p
5119 3930337 : (goal_alt[i], reg_class_contents[goal_alt[i]],
5120 3930337 : PSEUDO_REGNO_MODE (regno)))
5121 178956084 : && (curr_insn_set == NULL_RTX
5122 3923930 : || !((REG_P (SET_SRC (curr_insn_set))
5123 : || MEM_P (SET_SRC (curr_insn_set))
5124 : || GET_CODE (SET_SRC (curr_insn_set)) == SUBREG)
5125 3263813 : && (REG_P (SET_DEST (curr_insn_set))
5126 : || MEM_P (SET_DEST (curr_insn_set))
5127 : || GET_CODE (SET_DEST (curr_insn_set)) == SUBREG))))
5128 : optional_p = true;
5129 174359212 : else if (goal_alt_matched[i][0] != -1
5130 8954947 : && curr_static_id->operand[i].type == OP_OUT
5131 8953274 : && (curr_static_id->operand_alternative
5132 8953274 : [goal_alt_number * n_operands + i].earlyclobber)
5133 21026 : && REG_P (op))
5134 : {
5135 25735 : for (j = 0; goal_alt_matched[i][j] != -1; j++)
5136 : {
5137 20973 : rtx op2 = *curr_id->operand_loc[goal_alt_matched[i][j]];
5138 :
5139 20973 : if (REG_P (op2) && REGNO (op) != REGNO (op2))
5140 : break;
5141 : }
5142 20973 : if (goal_alt_matched[i][j] != -1)
5143 : {
5144 : /* Generate reloads for different output and matched
5145 : input registers. This is the easiest way to avoid
5146 : creation of non-existing register conflicts in
5147 : lra-lives.cc. */
5148 16211 : match_reload (i, goal_alt_matched[i], outputs, goal_alt[i],
5149 : &goal_alt_exclude_start_hard_regs[i], &before,
5150 : &after, true);
5151 : }
5152 176027968 : continue;
5153 20973 : }
5154 : else
5155 : {
5156 174338239 : enum reg_class rclass, common_class;
5157 :
5158 91338229 : if (REG_P (op) && goal_alt[i] != NO_REGS
5159 85829827 : && (regno = REGNO (op)) >= new_regno_start
5160 4914525 : && (rclass = get_reg_class (regno)) == ALL_REGS
5161 0 : && ((common_class = ira_reg_class_subset[rclass][goal_alt[i]])
5162 : != NO_REGS)
5163 0 : && common_class != ALL_REGS
5164 174338239 : && enough_allocatable_hard_regs_p (common_class,
5165 0 : GET_MODE (op)))
5166 : /* Refine reload pseudo class from chosen alternative
5167 : constraint. */
5168 0 : lra_change_class (regno, common_class, " Change to", true);
5169 174338239 : continue;
5170 174338239 : }
5171 : }
5172 :
5173 : /* Operands that match previous ones have already been handled. */
5174 6906209 : if (goal_alt_matches[i] >= 0)
5175 1668756 : continue;
5176 :
5177 : /* We should not have an operand with a non-offsettable address
5178 : appearing where an offsettable address will do. It also may
5179 : be a case when the address should be special in other words
5180 : not a general one (e.g. it needs no index reg). */
5181 5237453 : if (goal_alt_matched[i][0] == -1 && goal_alt_offmemok[i] && MEM_P (op))
5182 : {
5183 72 : enum reg_class rclass;
5184 72 : rtx *loc = &XEXP (op, 0);
5185 72 : enum rtx_code code = GET_CODE (*loc);
5186 :
5187 72 : push_to_sequence (before);
5188 72 : rclass = base_reg_class (GET_MODE (op), MEM_ADDR_SPACE (op),
5189 : MEM, SCRATCH, curr_insn);
5190 72 : if (GET_RTX_CLASS (code) == RTX_AUTOINC)
5191 0 : new_reg = emit_inc (rclass, *loc,
5192 : /* This value does not matter for MODIFY. */
5193 0 : GET_MODE_SIZE (GET_MODE (op)));
5194 : /* Try to pull out section anchors. For example, instead of
5195 : reloading an "entire" address like .LANCHOR42+offset only reload
5196 : .LANCHOR42 and use the new reload register as the base register.
5197 : This allows following optimizations to share section anchors and
5198 : remove redundant loads. */
5199 72 : else if (reload_section_anchor_p (i))
5200 : {
5201 0 : rtx anchor = XEXP (XEXP (*loc, 0), 0);
5202 0 : rtx offset = XEXP (XEXP (*loc, 0), 1);
5203 :
5204 0 : if (get_reload_reg (OP_IN, Pmode, anchor, rclass, NULL, false,
5205 : false, "offsetable address", &new_reg))
5206 0 : lra_emit_move (new_reg, anchor);
5207 :
5208 0 : rtx new_addr = gen_rtx_PLUS (Pmode, new_reg, offset);
5209 0 : rtx new_op = shallow_copy_rtx (op);
5210 0 : XEXP (new_op, 0) = new_addr;
5211 :
5212 0 : new_reg = new_op;
5213 0 : loc = curr_id->operand_loc[i];
5214 : }
5215 86 : else if (get_reload_reg (OP_IN, Pmode, *loc, rclass,
5216 : NULL, false, false,
5217 : "offsetable address", &new_reg))
5218 : {
5219 72 : rtx addr = *loc;
5220 72 : enum rtx_code code = GET_CODE (addr);
5221 72 : bool align_p = false;
5222 :
5223 72 : if (code == AND && CONST_INT_P (XEXP (addr, 1)))
5224 : {
5225 : /* (and ... (const_int -X)) is used to align to X bytes. */
5226 0 : align_p = true;
5227 0 : addr = XEXP (*loc, 0);
5228 : }
5229 : else
5230 72 : addr = canonicalize_reload_addr (addr);
5231 :
5232 72 : lra_emit_move (new_reg, addr);
5233 72 : if (align_p)
5234 0 : emit_move_insn (new_reg, gen_rtx_AND (GET_MODE (new_reg), new_reg, XEXP (*loc, 1)));
5235 : }
5236 72 : before = end_sequence ();
5237 72 : *loc = new_reg;
5238 72 : lra_update_dup (curr_id, i);
5239 72 : }
5240 5237381 : else if (goal_alt_matched[i][0] == -1)
5241 : {
5242 3471424 : machine_mode mode;
5243 3471424 : rtx reg, *loc;
5244 3471424 : int hard_regno;
5245 3471424 : enum op_type type = curr_static_id->operand[i].type;
5246 :
5247 3471424 : loc = curr_id->operand_loc[i];
5248 3471424 : mode = curr_operand_mode[i];
5249 3471424 : if (GET_CODE (*loc) == SUBREG)
5250 : {
5251 94110 : reg = SUBREG_REG (*loc);
5252 94110 : poly_int64 byte = SUBREG_BYTE (*loc);
5253 94110 : if (REG_P (reg)
5254 : /* Strict_low_part requires reloading the register and not
5255 : just the subreg. Likewise for a strict subreg no wider
5256 : than a word for WORD_REGISTER_OPERATIONS targets. */
5257 94110 : && (curr_static_id->operand[i].strict_low
5258 94028 : || (!paradoxical_subreg_p (mode, GET_MODE (reg))
5259 91749 : && (hard_regno
5260 91749 : = get_try_hard_regno (REGNO (reg))) >= 0
5261 90461 : && (simplify_subreg_regno
5262 184571 : (hard_regno,
5263 90461 : GET_MODE (reg), byte, mode) < 0)
5264 0 : && (goal_alt[i] == NO_REGS
5265 0 : || (simplify_subreg_regno
5266 82 : (ira_class_hard_regs[goal_alt[i]][0],
5267 0 : GET_MODE (reg), byte, mode) >= 0)))
5268 94028 : || (partial_subreg_p (mode, GET_MODE (reg))
5269 94028 : && known_le (GET_MODE_SIZE (GET_MODE (reg)),
5270 : UNITS_PER_WORD)
5271 : && WORD_REGISTER_OPERATIONS))
5272 : /* Avoid the situation when there are no available hard regs
5273 : for the pseudo mode but there are ones for the subreg
5274 : mode: */
5275 94192 : && !(goal_alt[i] != NO_REGS
5276 82 : && REGNO (reg) >= FIRST_PSEUDO_REGISTER
5277 82 : && (prohibited_class_reg_set_mode_p
5278 82 : (goal_alt[i], reg_class_contents[goal_alt[i]],
5279 82 : GET_MODE (reg)))
5280 : && !(prohibited_class_reg_set_mode_p
5281 0 : (goal_alt[i], reg_class_contents[goal_alt[i]],
5282 : mode))))
5283 : {
5284 : /* An OP_INOUT is required when reloading a subreg of a
5285 : mode wider than a word to ensure that data beyond the
5286 : word being reloaded is preserved. Also automatically
5287 : ensure that strict_low_part reloads are made into
5288 : OP_INOUT which should already be true from the backend
5289 : constraints. */
5290 82 : if (type == OP_OUT
5291 82 : && (curr_static_id->operand[i].strict_low
5292 0 : || read_modify_subreg_p (*loc)))
5293 : type = OP_INOUT;
5294 82 : loc = &SUBREG_REG (*loc);
5295 82 : mode = GET_MODE (*loc);
5296 : }
5297 : }
5298 3471424 : old = *loc;
5299 3471424 : if (get_reload_reg (type, mode, old, goal_alt[i],
5300 : &goal_alt_exclude_start_hard_regs[i],
5301 3471424 : loc != curr_id->operand_loc[i],
5302 3471424 : curr_static_id->operand_alternative
5303 3471424 : [goal_alt_number * n_operands + i].earlyclobber,
5304 : "", &new_reg)
5305 3471424 : && type != OP_OUT)
5306 : {
5307 2478941 : push_to_sequence (before);
5308 2478941 : lra_emit_move (new_reg, old);
5309 2478941 : before = end_sequence ();
5310 : }
5311 3471424 : *loc = new_reg;
5312 3471424 : if (type != OP_IN
5313 991266 : && find_reg_note (curr_insn, REG_UNUSED, old) == NULL_RTX
5314 : /* OLD can be an equivalent constant here. */
5315 965941 : && !CONSTANT_P (old)
5316 : /* No need to write back anything for a scratch. */
5317 965941 : && GET_CODE (old) != SCRATCH
5318 4437365 : && (!REG_P(old) || !ira_former_scratch_p (REGNO (old))))
5319 : {
5320 965941 : start_sequence ();
5321 965941 : lra_emit_move (type == OP_INOUT ? copy_rtx (old) : old, new_reg);
5322 965941 : emit_insn (after);
5323 965941 : after = end_sequence ();
5324 965941 : *loc = new_reg;
5325 : }
5326 3471424 : for (j = 0; j < goal_alt_dont_inherit_ops_num; j++)
5327 628 : if (goal_alt_dont_inherit_ops[j] == i)
5328 : {
5329 628 : lra_set_regno_unique_value (REGNO (new_reg));
5330 628 : break;
5331 : }
5332 3471424 : lra_update_dup (curr_id, i);
5333 : }
5334 1765957 : else if (curr_static_id->operand[i].type == OP_IN
5335 1765957 : && (curr_static_id->operand[goal_alt_matched[i][0]].type
5336 : == OP_OUT
5337 0 : || (curr_static_id->operand[goal_alt_matched[i][0]].type
5338 : == OP_INOUT
5339 0 : && (operands_match_p
5340 0 : (*curr_id->operand_loc[i],
5341 0 : *curr_id->operand_loc[goal_alt_matched[i][0]],
5342 : -1)))))
5343 : {
5344 : /* generate reloads for input and matched outputs. */
5345 17045 : match_inputs[0] = i;
5346 17045 : match_inputs[1] = -1;
5347 17045 : match_reload (goal_alt_matched[i][0], match_inputs, outputs,
5348 : goal_alt[i], &goal_alt_exclude_start_hard_regs[i],
5349 : &before, &after,
5350 17045 : curr_static_id->operand_alternative
5351 17045 : [goal_alt_number * n_operands + goal_alt_matched[i][0]]
5352 17045 : .earlyclobber);
5353 : }
5354 1748912 : else if ((curr_static_id->operand[i].type == OP_OUT
5355 1 : || (curr_static_id->operand[i].type == OP_INOUT
5356 1 : && (operands_match_p
5357 1 : (*curr_id->operand_loc[i],
5358 1 : *curr_id->operand_loc[goal_alt_matched[i][0]],
5359 : -1))))
5360 1748913 : && (curr_static_id->operand[goal_alt_matched[i][0]].type
5361 : == OP_IN))
5362 : /* Generate reloads for output and matched inputs. */
5363 1748912 : match_reload (i, goal_alt_matched[i], outputs, goal_alt[i],
5364 : &goal_alt_exclude_start_hard_regs[i], &before, &after,
5365 1748912 : curr_static_id->operand_alternative
5366 1748912 : [goal_alt_number * n_operands + i].earlyclobber);
5367 0 : else if (curr_static_id->operand[i].type == OP_IN
5368 0 : && (curr_static_id->operand[goal_alt_matched[i][0]].type
5369 : == OP_IN))
5370 : {
5371 : /* Generate reloads for matched inputs. */
5372 0 : match_inputs[0] = i;
5373 0 : for (j = 0; (k = goal_alt_matched[i][j]) >= 0; j++)
5374 0 : match_inputs[j + 1] = k;
5375 0 : match_inputs[j + 1] = -1;
5376 0 : match_reload (-1, match_inputs, outputs, goal_alt[i],
5377 : &goal_alt_exclude_start_hard_regs[i],
5378 : &before, &after, false);
5379 : }
5380 : else
5381 : /* We must generate code in any case when function
5382 : process_alt_operands decides that it is possible. */
5383 0 : gcc_unreachable ();
5384 :
5385 5237453 : if (optional_p)
5386 : {
5387 666545 : rtx reg = op;
5388 :
5389 666545 : lra_assert (REG_P (reg));
5390 666545 : regno = REGNO (reg);
5391 666545 : op = *curr_id->operand_loc[i]; /* Substitution. */
5392 666545 : if (GET_CODE (op) == SUBREG)
5393 0 : op = SUBREG_REG (op);
5394 666545 : gcc_assert (REG_P (op) && (int) REGNO (op) >= new_regno_start);
5395 666545 : bitmap_set_bit (&lra_optional_reload_pseudos, REGNO (op));
5396 666545 : lra_reg_info[REGNO (op)].restore_rtx = reg;
5397 666545 : if (lra_dump_file != NULL)
5398 3 : fprintf (lra_dump_file,
5399 : " Making reload reg %d for reg %d optional\n",
5400 : REGNO (op), regno);
5401 : }
5402 : }
5403 :
5404 : /* After the dust has settled, collect and attach dependent filters. */
5405 81266859 : process_dependent_filters ();
5406 :
5407 76786427 : if (before != NULL_RTX || after != NULL_RTX
5408 157228215 : || max_regno_before != max_reg_num ())
5409 5331192 : change_p = true;
5410 81266859 : if (change_p)
5411 : {
5412 6349502 : lra_update_operator_dups (curr_id);
5413 : /* Something changes -- process the insn. */
5414 6349502 : lra_update_insn_regno_info (curr_insn);
5415 6349502 : if (asm_noperands (PATTERN (curr_insn)) >= 0
5416 6349502 : && ++curr_id->asm_reloads_num >= FIRST_PSEUDO_REGISTER)
5417 : /* Most probably there are no enough registers to satisfy asm insn: */
5418 : {
5419 11 : lra_asm_insn_error (curr_insn);
5420 11 : return change_p;
5421 : }
5422 : }
5423 81266848 : if (goal_alt_out_sp_reload_p)
5424 : {
5425 : /* We have an output stack pointer reload -- update sp offset: */
5426 0 : rtx set;
5427 0 : bool done_p = false;
5428 0 : poly_int64 sp_offset = curr_id->sp_offset;
5429 0 : for (rtx_insn *insn = after; insn != NULL_RTX; insn = NEXT_INSN (insn))
5430 0 : if ((set = single_set (insn)) != NULL_RTX
5431 0 : && SET_DEST (set) == stack_pointer_rtx)
5432 : {
5433 0 : lra_assert (!done_p);
5434 0 : done_p = true;
5435 0 : curr_id->sp_offset = 0;
5436 0 : lra_insn_recog_data_t id = lra_get_insn_recog_data (insn);
5437 0 : id->sp_offset = sp_offset;
5438 0 : if (lra_dump_file != NULL)
5439 0 : fprintf (lra_dump_file,
5440 : " Moving sp offset from insn %u to %u\n",
5441 0 : INSN_UID (curr_insn), INSN_UID (insn));
5442 : }
5443 0 : lra_assert (done_p);
5444 : }
5445 81266848 : int const_regno = -1;
5446 81266848 : rtx set;
5447 81266848 : rtx_insn *prev, *const_insn = NULL;
5448 4480427 : if (before != NULL_RTX && (prev = PREV_INSN (curr_insn)) != NULL_RTX
5449 85747275 : && (set = single_set (prev)) != NULL_RTX && CONSTANT_P (SET_SRC (set)))
5450 : {
5451 342500 : rtx reg = SET_DEST (set);
5452 342500 : if (GET_CODE (reg) == SUBREG)
5453 10564 : reg = SUBREG_REG (reg);
5454 : /* Consider only reload insns as we don't want to change the order
5455 : created by previous optimizations. */
5456 250548 : if (REG_P (reg) && (int) REGNO (reg) >= lra_new_regno_start
5457 344111 : && bitmap_bit_p (&lra_reg_info[REGNO (reg)].insn_bitmap,
5458 1611 : INSN_UID (curr_insn)))
5459 : {
5460 1124 : const_regno = REGNO (reg);
5461 1124 : const_insn = prev;
5462 : }
5463 : }
5464 81266848 : if (asm_noperands (PATTERN (curr_insn)) >= 0)
5465 : {
5466 : /* Asm can have a lot of operands. To guarantee their assignment,
5467 : postpone processing the reload insns until the reload pseudos are
5468 : assigned. */
5469 47908 : postpone_insns (before);
5470 47908 : postpone_insns (after);
5471 : }
5472 81266848 : lra_process_new_insns (curr_insn, before, after,
5473 : "Inserting insn reload", true);
5474 81266848 : if (const_regno >= 0) {
5475 2248 : bool move_p = true;
5476 2248 : for (rtx_insn *insn = before; insn != curr_insn; insn = NEXT_INSN (insn))
5477 1124 : if (bitmap_bit_p (&lra_reg_info[const_regno].insn_bitmap,
5478 1124 : INSN_UID (insn)))
5479 : {
5480 : move_p = false;
5481 : break;
5482 : }
5483 1124 : if (move_p)
5484 : {
5485 1124 : reorder_insns_nobb (const_insn, const_insn, PREV_INSN (curr_insn));
5486 1124 : if (lra_dump_file != NULL)
5487 : {
5488 0 : dump_insn_slim (lra_dump_file, const_insn);
5489 0 : fprintf (lra_dump_file,
5490 : " to decrease reg pressure, it is moved before:\n");
5491 0 : dump_insn_slim (lra_dump_file, curr_insn);
5492 : }
5493 : }
5494 : }
5495 : return change_p;
5496 : }
5497 :
5498 : /* Return true if INSN satisfies all constraints. In other words, no
5499 : reload insns are needed. */
5500 : bool
5501 3527 : lra_constrain_insn (rtx_insn *insn)
5502 : {
5503 3527 : int saved_new_regno_start = new_regno_start;
5504 3527 : int saved_new_insn_uid_start = new_insn_uid_start;
5505 3527 : bool change_p;
5506 :
5507 3527 : curr_insn = insn;
5508 3527 : curr_id = lra_get_insn_recog_data (curr_insn);
5509 3527 : curr_static_id = curr_id->insn_static_data;
5510 3527 : new_insn_uid_start = get_max_uid ();
5511 3527 : new_regno_start = max_reg_num ();
5512 3527 : change_p = curr_insn_transform (true);
5513 3527 : new_regno_start = saved_new_regno_start;
5514 3527 : new_insn_uid_start = saved_new_insn_uid_start;
5515 3527 : return ! change_p;
5516 : }
5517 :
5518 : /* Return true if X is in LIST. */
5519 : static bool
5520 1409102 : in_list_p (rtx x, rtx list)
5521 : {
5522 2411666 : for (; list != NULL_RTX; list = XEXP (list, 1))
5523 1324039 : if (XEXP (list, 0) == x)
5524 : return true;
5525 : return false;
5526 : }
5527 :
5528 : /* Return true if X contains an allocatable hard register (if
5529 : HARD_REG_P) or a (spilled if SPILLED_P) pseudo. */
5530 : static bool
5531 7707509 : contains_reg_p (rtx x, bool hard_reg_p, bool spilled_p)
5532 : {
5533 7707509 : int i, j;
5534 7707509 : const char *fmt;
5535 7707509 : enum rtx_code code;
5536 :
5537 7707509 : code = GET_CODE (x);
5538 7707509 : if (REG_P (x))
5539 : {
5540 1567145 : int regno = REGNO (x);
5541 1567145 : HARD_REG_SET alloc_regs;
5542 :
5543 1567145 : if (hard_reg_p)
5544 : {
5545 493739 : if (regno >= FIRST_PSEUDO_REGISTER)
5546 147618 : regno = lra_get_regno_hard_regno (regno);
5547 493739 : if (regno < 0)
5548 : return false;
5549 493739 : alloc_regs = ~lra_no_alloc_regs;
5550 493739 : return overlaps_hard_reg_set_p (alloc_regs, GET_MODE (x), regno);
5551 : }
5552 : else
5553 : {
5554 1073406 : if (regno < FIRST_PSEUDO_REGISTER)
5555 : return false;
5556 347308 : if (! spilled_p)
5557 : return true;
5558 183641 : return lra_get_regno_hard_regno (regno) < 0;
5559 : }
5560 : }
5561 6140364 : fmt = GET_RTX_FORMAT (code);
5562 15184839 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
5563 : {
5564 9651018 : if (fmt[i] == 'e')
5565 : {
5566 4269780 : if (contains_reg_p (XEXP (x, i), hard_reg_p, spilled_p))
5567 : return true;
5568 : }
5569 5381238 : else if (fmt[i] == 'E')
5570 : {
5571 1310206 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
5572 1199728 : if (contains_reg_p (XVECEXP (x, i, j), hard_reg_p, spilled_p))
5573 : return true;
5574 : }
5575 : }
5576 : return false;
5577 : }
5578 :
5579 : /* Process all regs in location *LOC and change them on equivalent
5580 : substitution. Return true if any change was done. */
5581 : static bool
5582 3348 : loc_equivalence_change_p (rtx *loc)
5583 : {
5584 3348 : rtx subst, reg, x = *loc;
5585 3348 : bool result = false;
5586 3348 : enum rtx_code code = GET_CODE (x);
5587 3348 : const char *fmt;
5588 3348 : int i, j;
5589 :
5590 3348 : if (code == SUBREG)
5591 : {
5592 20 : reg = SUBREG_REG (x);
5593 20 : if ((subst = get_equiv_with_elimination (reg, curr_insn)) != reg
5594 20 : && GET_MODE (subst) == VOIDmode)
5595 : {
5596 : /* We cannot reload debug location. Simplify subreg here
5597 : while we know the inner mode. */
5598 0 : *loc = simplify_gen_subreg (GET_MODE (x), subst,
5599 0 : GET_MODE (reg), SUBREG_BYTE (x));
5600 0 : return true;
5601 : }
5602 : }
5603 3348 : if (code == REG && (subst = get_equiv_with_elimination (x, curr_insn)) != x)
5604 : {
5605 8 : *loc = subst;
5606 8 : return true;
5607 : }
5608 :
5609 : /* Scan all the operand sub-expressions. */
5610 3340 : fmt = GET_RTX_FORMAT (code);
5611 8168 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
5612 : {
5613 4828 : if (fmt[i] == 'e')
5614 2577 : result = loc_equivalence_change_p (&XEXP (x, i)) || result;
5615 2251 : else if (fmt[i] == 'E')
5616 270 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
5617 200 : result
5618 210 : = loc_equivalence_change_p (&XVECEXP (x, i, j)) || result;
5619 : }
5620 : return result;
5621 : }
5622 :
5623 : /* Similar to loc_equivalence_change_p, but for use as
5624 : simplify_replace_fn_rtx callback. DATA is insn for which the
5625 : elimination is done. If it null we don't do the elimination. */
5626 : static rtx
5627 42354915 : loc_equivalence_callback (rtx loc, const_rtx, void *data)
5628 : {
5629 42354915 : if (!REG_P (loc))
5630 : return NULL_RTX;
5631 :
5632 10889637 : rtx subst = (data == NULL
5633 10889637 : ? get_equiv (loc) : get_equiv_with_elimination (loc, (rtx_insn *) data));
5634 10889637 : if (subst != loc)
5635 37755 : return subst;
5636 :
5637 : return NULL_RTX;
5638 : }
5639 :
5640 : /* Maximum number of generated reload insns per an insn. It is for
5641 : preventing this pass cycling in a bug case. */
5642 : #define MAX_RELOAD_INSNS_NUMBER LRA_MAX_INSN_RELOADS
5643 :
5644 : /* The current iteration number of this LRA pass. */
5645 : int lra_constraint_iter;
5646 :
5647 : /* True if we should during assignment sub-pass check assignment
5648 : correctness for all pseudos and spill some of them to correct
5649 : conflicts. It can be necessary when we substitute equiv which
5650 : needs checking register allocation correctness because the
5651 : equivalent value contains allocatable hard registers, or when we
5652 : restore multi-register pseudo, or when we change the insn code and
5653 : its operand became INOUT operand when it was IN one before. */
5654 : bool check_and_force_assignment_correctness_p;
5655 :
5656 : /* Return true if REGNO is referenced in more than one block. */
5657 : static bool
5658 157437 : multi_block_pseudo_p (int regno)
5659 : {
5660 157437 : basic_block bb = NULL;
5661 157437 : unsigned int uid;
5662 157437 : bitmap_iterator bi;
5663 :
5664 157437 : if (regno < FIRST_PSEUDO_REGISTER)
5665 : return false;
5666 :
5667 482300 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi)
5668 329675 : if (bb == NULL)
5669 157437 : bb = BLOCK_FOR_INSN (lra_insn_recog_data[uid]->insn);
5670 172238 : else if (BLOCK_FOR_INSN (lra_insn_recog_data[uid]->insn) != bb)
5671 : return true;
5672 : return false;
5673 : }
5674 :
5675 : /* Return true if LIST contains a deleted insn. */
5676 : static bool
5677 748559 : contains_deleted_insn_p (rtx_insn_list *list)
5678 : {
5679 1428567 : for (; list != NULL_RTX; list = list->next ())
5680 680008 : if (NOTE_P (list->insn ())
5681 680008 : && NOTE_KIND (list->insn ()) == NOTE_INSN_DELETED)
5682 : return true;
5683 : return false;
5684 : }
5685 :
5686 : /* Return true if X contains a pseudo dying in INSN. */
5687 : static bool
5688 2338856 : dead_pseudo_p (rtx x, rtx_insn *insn)
5689 : {
5690 2338856 : int i, j;
5691 2338856 : const char *fmt;
5692 2338856 : enum rtx_code code;
5693 :
5694 2338856 : if (REG_P (x))
5695 522324 : return (insn != NULL_RTX
5696 522324 : && find_regno_note (insn, REG_DEAD, REGNO (x)) != NULL_RTX);
5697 1816532 : code = GET_CODE (x);
5698 1816532 : fmt = GET_RTX_FORMAT (code);
5699 4644254 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
5700 : {
5701 2833208 : if (fmt[i] == 'e')
5702 : {
5703 1389651 : if (dead_pseudo_p (XEXP (x, i), insn))
5704 : return true;
5705 : }
5706 1443557 : else if (fmt[i] == 'E')
5707 : {
5708 295862 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
5709 270679 : if (dead_pseudo_p (XVECEXP (x, i, j), insn))
5710 : return true;
5711 : }
5712 : }
5713 : return false;
5714 : }
5715 :
5716 : /* Return true if INSN contains a dying pseudo in INSN right hand
5717 : side. */
5718 : static bool
5719 678526 : insn_rhs_dead_pseudo_p (rtx_insn *insn)
5720 : {
5721 678526 : rtx set = single_set (insn);
5722 :
5723 678526 : gcc_assert (set != NULL);
5724 678526 : return dead_pseudo_p (SET_SRC (set), insn);
5725 : }
5726 :
5727 : /* Return true if any init insn of REGNO contains a dying pseudo in
5728 : insn right hand side. */
5729 : static bool
5730 747078 : init_insn_rhs_dead_pseudo_p (int regno)
5731 : {
5732 747078 : rtx_insn_list *insns = ira_reg_equiv[regno].init_insns;
5733 :
5734 747078 : if (insns == NULL)
5735 : return false;
5736 1350315 : for (; insns != NULL_RTX; insns = insns->next ())
5737 678526 : if (insn_rhs_dead_pseudo_p (insns->insn ()))
5738 : return true;
5739 : return false;
5740 : }
5741 :
5742 : /* Return TRUE if REGNO has a reverse equivalence. The equivalence is
5743 : reverse only if we have one init insn with given REGNO as a
5744 : source. */
5745 : static bool
5746 748559 : reverse_equiv_p (int regno)
5747 : {
5748 748559 : rtx_insn_list *insns = ira_reg_equiv[regno].init_insns;
5749 748559 : rtx set;
5750 :
5751 748559 : if (insns == NULL)
5752 : return false;
5753 680007 : if (! INSN_P (insns->insn ())
5754 1360014 : || insns->next () != NULL)
5755 : return false;
5756 680006 : if ((set = single_set (insns->insn ())) == NULL_RTX)
5757 : return false;
5758 680006 : return REG_P (SET_SRC (set)) && (int) REGNO (SET_SRC (set)) == regno;
5759 : }
5760 :
5761 : /* Return TRUE if REGNO was reloaded in an equivalence init insn. We
5762 : call this function only for non-reverse equivalence. */
5763 : static bool
5764 740341 : contains_reloaded_insn_p (int regno)
5765 : {
5766 740341 : rtx set;
5767 740341 : rtx_insn_list *list = ira_reg_equiv[regno].init_insns;
5768 :
5769 1412128 : for (; list != NULL; list = list->next ())
5770 671789 : if ((set = single_set (list->insn ())) == NULL_RTX
5771 671789 : || ! REG_P (SET_DEST (set))
5772 1343578 : || (int) REGNO (SET_DEST (set)) != regno)
5773 : return true;
5774 : return false;
5775 : }
5776 :
5777 : /* Try combine secondary memory reload insn FROM for insn TO into TO insn.
5778 : FROM should be a load insn (usually a secondary memory reload insn). Return
5779 : TRUE in case of success. */
5780 : static bool
5781 7513706 : combine_reload_insn (rtx_insn *from, rtx_insn *to)
5782 : {
5783 7513706 : bool ok_p;
5784 7513706 : rtx_insn *saved_insn;
5785 7513706 : rtx set, from_reg, to_reg, op;
5786 7513706 : enum reg_class to_class, from_class;
5787 7513706 : int n, nop;
5788 7513706 : signed char changed_nops[MAX_RECOG_OPERANDS + 1];
5789 :
5790 : /* Check conditions for second memory reload and original insn: */
5791 7513706 : if ((targetm.secondary_memory_needed
5792 : == hook_bool_mode_reg_class_t_reg_class_t_false)
5793 7513706 : || NEXT_INSN (from) != to
5794 4488008 : || !NONDEBUG_INSN_P (to)
5795 12001712 : || CALL_P (to))
5796 : return false;
5797 :
5798 4482752 : lra_insn_recog_data_t id = lra_get_insn_recog_data (to);
5799 4482752 : struct lra_static_insn_data *static_id = id->insn_static_data;
5800 :
5801 4482752 : if (id->used_insn_alternative == LRA_UNKNOWN_ALT
5802 4482752 : || (set = single_set (from)) == NULL_RTX)
5803 : return false;
5804 4449034 : from_reg = SET_DEST (set);
5805 4449034 : to_reg = SET_SRC (set);
5806 : /* Ignore optional reloads: */
5807 4374790 : if (! REG_P (from_reg) || ! REG_P (to_reg)
5808 7395539 : || bitmap_bit_p (&lra_optional_reload_pseudos, REGNO (from_reg)))
5809 : return false;
5810 2384696 : to_class = lra_get_allocno_class (REGNO (to_reg));
5811 2384696 : from_class = lra_get_allocno_class (REGNO (from_reg));
5812 : /* Check that reload insn is a load: */
5813 2384696 : if (to_class != NO_REGS || from_class == NO_REGS)
5814 : return false;
5815 48838 : for (n = nop = 0; nop < static_id->n_operands; nop++)
5816 : {
5817 35170 : if (static_id->operand[nop].type != OP_IN)
5818 12605 : continue;
5819 22565 : op = *id->operand_loc[nop];
5820 22565 : if (!REG_P (op) || REGNO (op) != REGNO (from_reg))
5821 9087 : continue;
5822 13478 : *id->operand_loc[nop] = to_reg;
5823 13478 : changed_nops[n++] = nop;
5824 : }
5825 13668 : changed_nops[n] = -1;
5826 13668 : lra_update_dups (id, changed_nops);
5827 13668 : lra_update_insn_regno_info (to);
5828 13668 : ok_p = recog_memoized (to) >= 0;
5829 13668 : if (ok_p)
5830 : {
5831 : /* Check that combined insn does not need any reloads: */
5832 13649 : saved_insn = curr_insn;
5833 13649 : curr_insn = to;
5834 13649 : curr_id = lra_get_insn_recog_data (curr_insn);
5835 13649 : curr_static_id = curr_id->insn_static_data;
5836 13649 : for (bool swapped_p = false;;)
5837 : {
5838 16042 : ok_p = !curr_insn_transform (true);
5839 16042 : if (ok_p || curr_static_id->commutative < 0)
5840 : break;
5841 4786 : swap_operands (curr_static_id->commutative);
5842 4786 : if (lra_dump_file != NULL)
5843 : {
5844 0 : fprintf (lra_dump_file,
5845 : " Swapping %scombined insn operands:\n",
5846 : swapped_p ? "back " : "");
5847 0 : dump_insn_slim (lra_dump_file, to);
5848 : }
5849 4786 : if (swapped_p)
5850 : break;
5851 : swapped_p = true;
5852 : }
5853 13649 : curr_insn = saved_insn;
5854 13649 : curr_id = lra_get_insn_recog_data (curr_insn);
5855 13649 : curr_static_id = curr_id->insn_static_data;
5856 : }
5857 13668 : if (ok_p)
5858 : {
5859 3603 : id->used_insn_alternative = -1;
5860 3603 : lra_push_insn_and_update_insn_regno_info (to);
5861 3603 : if (lra_dump_file != NULL)
5862 : {
5863 0 : fprintf (lra_dump_file, " Use combined insn:\n");
5864 0 : dump_insn_slim (lra_dump_file, to);
5865 : }
5866 : return true;
5867 : }
5868 10065 : if (lra_dump_file != NULL)
5869 : {
5870 0 : fprintf (lra_dump_file, " Failed combined insn:\n");
5871 0 : dump_insn_slim (lra_dump_file, to);
5872 : }
5873 20405 : for (int i = 0; i < n; i++)
5874 : {
5875 10340 : nop = changed_nops[i];
5876 10340 : *id->operand_loc[nop] = from_reg;
5877 : }
5878 10065 : lra_update_dups (id, changed_nops);
5879 10065 : lra_update_insn_regno_info (to);
5880 10065 : if (lra_dump_file != NULL)
5881 : {
5882 0 : fprintf (lra_dump_file, " Restoring insn after failed combining:\n");
5883 0 : dump_insn_slim (lra_dump_file, to);
5884 : }
5885 : return false;
5886 : }
5887 :
5888 : static bool
5889 533621 : virtual_reg_p (const_rtx reg)
5890 : {
5891 0 : return (VIRTUAL_REGISTER_P (reg)
5892 533621 : || (REGNO(reg) == ARG_POINTER_REGNUM
5893 : && !HARD_FRAME_POINTER_IS_ARG_POINTER)
5894 991509 : || (REGNO(reg) == FRAME_POINTER_REGNUM
5895 0 : && !HARD_FRAME_POINTER_IS_FRAME_POINTER));
5896 : }
5897 :
5898 : /* Return true if X contains a virtual register. */
5899 : static bool
5900 738529 : contains_virtual_reg_p (rtx x)
5901 : {
5902 738529 : subrtx_iterator::array_type array;
5903 2119786 : FOR_EACH_SUBRTX (iter, array, x, NONCONST)
5904 1914878 : if (REG_P (*iter) && virtual_reg_p (*iter))
5905 361469 : return true;
5906 377060 : return false;
5907 738529 : }
5908 :
5909 : /* Return true if pseudo REGNO is an operand of an incdec rtx in any of its
5910 : insns. */
5911 : static bool
5912 361469 : reg_has_incdec_p (int regno)
5913 : {
5914 361469 : unsigned int uid;
5915 361469 : bitmap_iterator bi;
5916 1833154 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi)
5917 : {
5918 1471685 : rtx_insn *insn = lra_insn_recog_data[uid]->insn;
5919 1471685 : if (!NONDEBUG_INSN_P (insn))
5920 26492 : continue;
5921 1445193 : subrtx_iterator::array_type array;
5922 8500020 : FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
5923 : {
5924 7054827 : const_rtx x = *iter;
5925 7054827 : if ((GET_CODE (x) == PRE_INC || GET_CODE (x) == PRE_DEC
5926 6950504 : || GET_CODE (x) == POST_INC || GET_CODE (x) == POST_DEC
5927 6950504 : || GET_CODE (x) == PRE_MODIFY || GET_CODE (x) == POST_MODIFY)
5928 105688 : && REG_P (XEXP (x, 0))
5929 7160515 : && REGNO (XEXP (x, 0)) == (unsigned int) regno)
5930 0 : return true;
5931 : }
5932 1445193 : }
5933 : return false;
5934 : }
5935 :
5936 : /* Entry function of LRA constraint pass. Return true if the
5937 : constraint pass did change the code. */
5938 : bool
5939 3298462 : lra_constraints (bool first_p)
5940 : {
5941 3298462 : bool changed_p;
5942 3298462 : int i, hard_regno, new_insns_num;
5943 3298462 : unsigned int min_len, new_min_len, uid;
5944 3298462 : rtx set, x, reg, nosubreg_dest;
5945 3298462 : rtx_insn *original_insn;
5946 3298462 : basic_block last_bb;
5947 3298462 : bitmap_iterator bi;
5948 :
5949 3298462 : lra_constraint_iter++;
5950 3298462 : if (lra_dump_file != NULL)
5951 194 : fprintf (lra_dump_file, "\n********** Local #%d: **********\n\n",
5952 : lra_constraint_iter);
5953 3298462 : changed_p = false;
5954 3298462 : if (pic_offset_table_rtx
5955 3298462 : && REGNO (pic_offset_table_rtx) >= FIRST_PSEUDO_REGISTER)
5956 104766 : check_and_force_assignment_correctness_p = true;
5957 3193696 : else if (first_p)
5958 : /* On the first iteration we should check IRA assignment
5959 : correctness. In rare cases, the assignments can be wrong as
5960 : early clobbers operands are ignored in IRA or usages of
5961 : paradoxical sub-registers are not taken into account by
5962 : IRA. */
5963 1474608 : check_and_force_assignment_correctness_p = true;
5964 3298462 : new_insn_uid_start = get_max_uid ();
5965 3298462 : new_regno_start = first_p ? lra_constraint_new_regno_start : max_reg_num ();
5966 : /* Mark used hard regs for target stack size calculations. */
5967 212381963 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
5968 209083501 : if (lra_reg_info[i].nrefs != 0
5969 308381905 : && (hard_regno = lra_get_regno_hard_regno (i)) >= 0)
5970 : {
5971 95293733 : int j, nregs;
5972 :
5973 95293733 : nregs = hard_regno_nregs (hard_regno, lra_reg_info[i].biggest_mode);
5974 193791760 : for (j = 0; j < nregs; j++)
5975 98498027 : df_set_regs_ever_live (hard_regno + j, true);
5976 : }
5977 : /* Do elimination before the equivalence processing as we can spill
5978 : some pseudos during elimination. */
5979 3298462 : lra_eliminate (false, first_p);
5980 3298462 : auto_bitmap equiv_insn_bitmap (®_obstack);
5981 :
5982 : /* Register elimination can create new pseudos via the addptr pattern,
5983 : so make sure the equivalency tables are resized appropriately. */
5984 3298462 : ira_expand_reg_equiv ();
5985 215680425 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
5986 209083501 : if (lra_reg_info[i].nrefs != 0)
5987 : {
5988 99298404 : ira_reg_equiv[i].profitable_p = true;
5989 99298404 : reg = regno_reg_rtx[i];
5990 99298404 : if (lra_get_regno_hard_regno (i) < 0 && (x = get_equiv (reg)) != reg)
5991 : {
5992 759601 : bool pseudo_p = contains_reg_p (x, false, false);
5993 :
5994 : /* After RTL transformation, we cannot guarantee that
5995 : pseudo in the substitution was not reloaded which might
5996 : make equivalence invalid. For example, in reverse
5997 : equiv of p0
5998 :
5999 : p0 <- ...
6000 : ...
6001 : equiv_mem <- p0
6002 :
6003 : the memory address register was reloaded before the 2nd
6004 : insn. */
6005 759601 : if ((! first_p && pseudo_p)
6006 : /* We don't use DF for compilation speed sake. So it
6007 : is problematic to update live info when we use an
6008 : equivalence containing pseudos in more than one
6009 : BB. */
6010 753371 : || (pseudo_p && multi_block_pseudo_p (i))
6011 : /* If an init insn was deleted for some reason, cancel
6012 : the equiv. We could update the equiv insns after
6013 : transformations including an equiv insn deletion
6014 : but it is not worthy as such cases are extremely
6015 : rare. */
6016 748559 : || contains_deleted_insn_p (ira_reg_equiv[i].init_insns)
6017 : /* If it is not a reverse equivalence, we check that a
6018 : pseudo in rhs of the init insn is not dying in the
6019 : insn. Otherwise, the live info at the beginning of
6020 : the corresponding BB might be wrong after we
6021 : removed the insn. When the equiv can be a
6022 : constant, the right hand side of the init insn can
6023 : be a pseudo. */
6024 748559 : || (! reverse_equiv_p (i)
6025 747078 : && (init_insn_rhs_dead_pseudo_p (i)
6026 : /* If we reloaded the pseudo in an equivalence
6027 : init insn, we cannot remove the equiv init
6028 : insns and the init insns might write into
6029 : const memory in this case. */
6030 740341 : || contains_reloaded_insn_p (i)))
6031 : /* Prevent access beyond equivalent memory for
6032 : paradoxical subregs. */
6033 741820 : || (MEM_P (x)
6034 1198956 : && maybe_gt (GET_MODE_SIZE (lra_reg_info[i].biggest_mode),
6035 : GET_MODE_SIZE (GET_MODE (x))))
6036 741112 : || (pic_offset_table_rtx
6037 54028 : && ((CONST_POOL_OK_P (PSEUDO_REGNO_MODE (i), x)
6038 8928 : && (targetm.preferred_reload_class
6039 4464 : (x, lra_get_allocno_class (i)) == NO_REGS))
6040 52252 : || contains_symbol_ref_p (x)))
6041 : /* PR120165: An inc/dec changes the reg value, so we cannot
6042 : substitute the equiv which might contain virtual regs
6043 : requiring elimination. An insn with inc/dec can also change
6044 : the value of another reg (e.g. sp) against which we
6045 : eliminate. As we update elimination once per insn and an
6046 : inc/dec operand might require input and output reloads, we
6047 : can generate a wrong offset for a reload insn. */
6048 1498130 : || (contains_virtual_reg_p (x) && reg_has_incdec_p (i)))
6049 21072 : ira_reg_equiv[i].defined_p
6050 21072 : = ira_reg_equiv[i].caller_save_p = false;
6051 759601 : if (contains_reg_p (x, false, true))
6052 9573 : ira_reg_equiv[i].profitable_p = false;
6053 759601 : if (get_equiv (reg) != reg)
6054 733722 : bitmap_ior_into (equiv_insn_bitmap, &lra_reg_info[i].insn_bitmap);
6055 : }
6056 : }
6057 212381963 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
6058 209083501 : update_equiv (i);
6059 : /* We should add all insns containing pseudos which should be
6060 : substituted by their equivalences. */
6061 5758690 : EXECUTE_IF_SET_IN_BITMAP (equiv_insn_bitmap, 0, uid, bi)
6062 2460228 : lra_push_insn_by_uid (uid);
6063 3298462 : min_len = lra_insn_stack_length ();
6064 3298462 : new_insns_num = 0;
6065 3298462 : last_bb = NULL;
6066 3298462 : changed_p = false;
6067 3298462 : original_insn = NULL;
6068 174224776 : while ((new_min_len = lra_insn_stack_length ()) != 0)
6069 : {
6070 167627852 : curr_insn = lra_pop_insn ();
6071 167627852 : --new_min_len;
6072 167627852 : curr_bb = BLOCK_FOR_INSN (curr_insn);
6073 167627852 : if (curr_bb != last_bb)
6074 : {
6075 21088617 : last_bb = curr_bb;
6076 21088617 : bb_reload_num = lra_curr_reload_num;
6077 : }
6078 167627852 : if (min_len > new_min_len)
6079 : {
6080 : min_len = new_min_len;
6081 : new_insns_num = 0;
6082 : original_insn = curr_insn;
6083 : }
6084 7513706 : else if (combine_reload_insn (curr_insn, original_insn))
6085 : {
6086 3603 : continue;
6087 : }
6088 7510103 : if (new_insns_num > MAX_RELOAD_INSNS_NUMBER)
6089 0 : internal_error
6090 0 : ("maximum number of generated reload insns per insn achieved (%d)",
6091 : MAX_RELOAD_INSNS_NUMBER);
6092 167624249 : new_insns_num++;
6093 167624249 : if (DEBUG_INSN_P (curr_insn))
6094 : {
6095 : /* We need to check equivalence in debug insn and change
6096 : pseudo to the equivalent value if necessary. */
6097 57129298 : curr_id = lra_get_insn_recog_data (curr_insn);
6098 57129298 : if (bitmap_bit_p (equiv_insn_bitmap, INSN_UID (curr_insn)))
6099 : {
6100 29982 : rtx old = *curr_id->operand_loc[0];
6101 29982 : *curr_id->operand_loc[0]
6102 29982 : = simplify_replace_fn_rtx (old, NULL_RTX,
6103 : loc_equivalence_callback, curr_insn);
6104 29982 : if (old != *curr_id->operand_loc[0])
6105 : {
6106 : /* If we substitute pseudo by shared equivalence, we can fail
6107 : to update LRA reg info and this can result in many
6108 : unexpected consequences. So keep rtl unshared: */
6109 29982 : *curr_id->operand_loc[0]
6110 29982 : = copy_rtx (*curr_id->operand_loc[0]);
6111 29982 : lra_update_insn_regno_info (curr_insn);
6112 29982 : changed_p = true;
6113 : }
6114 : }
6115 : }
6116 110494951 : else if (INSN_P (curr_insn))
6117 : {
6118 109393276 : if ((set = single_set (curr_insn)) != NULL_RTX)
6119 : {
6120 103911047 : nosubreg_dest = SET_DEST (set);
6121 : /* The equivalence pseudo could be set up as SUBREG in a
6122 : case when it is a call restore insn in a mode
6123 : different from the pseudo mode. */
6124 103911047 : if (GET_CODE (nosubreg_dest) == SUBREG)
6125 1221281 : nosubreg_dest = SUBREG_REG (nosubreg_dest);
6126 104629846 : if ((REG_P (nosubreg_dest)
6127 76926180 : && (x = get_equiv (nosubreg_dest)) != nosubreg_dest
6128 : /* Remove insns which set up a pseudo whose value
6129 : cannot be changed. Such insns might be not in
6130 : init_insns because we don't update equiv data
6131 : during insn transformations.
6132 :
6133 : As an example, let suppose that a pseudo got
6134 : hard register and on the 1st pass was not
6135 : changed to equivalent constant. We generate an
6136 : additional insn setting up the pseudo because of
6137 : secondary memory movement. Then the pseudo is
6138 : spilled and we use the equiv constant. In this
6139 : case we should remove the additional insn and
6140 : this insn is not init_insns list. */
6141 738183 : && (! MEM_P (x) || MEM_READONLY_P (x)
6142 : /* Check that this is actually an insn setting
6143 : up the equivalence. */
6144 340859 : || in_list_p (curr_insn,
6145 340859 : ira_reg_equiv
6146 340859 : [REGNO (nosubreg_dest)].init_insns)))
6147 180119831 : || (((x = get_equiv (SET_SRC (set))) != SET_SRC (set))
6148 2136486 : && in_list_p (curr_insn,
6149 1068243 : ira_reg_equiv
6150 1068243 : [REGNO (SET_SRC (set))].init_insns)
6151 : /* This is a reverse equivalence to memory (see ira.cc)
6152 : in store insn. We can reload all the destination and
6153 : have an output reload which is a store to memory. If
6154 : we just remove the insn, we will have the output
6155 : reload storing an undefined value to the memory.
6156 : Check that we did not reload the memory to prevent a
6157 : wrong code generation. We could implement using the
6158 : equivalence still in such case but doing this is not
6159 : worth the efforts as such case is very rare. */
6160 1403 : && MEM_P (nosubreg_dest)))
6161 : {
6162 : /* This is equiv init insn of pseudo which did not get a
6163 : hard register -- remove the insn. */
6164 718799 : if (lra_dump_file != NULL)
6165 : {
6166 9 : fprintf (lra_dump_file,
6167 : " Removing equiv init insn %i (freq=%d)\n",
6168 3 : INSN_UID (curr_insn),
6169 6 : REG_FREQ_FROM_BB (BLOCK_FOR_INSN (curr_insn)));
6170 3 : dump_insn_slim (lra_dump_file, curr_insn);
6171 : }
6172 718799 : if (contains_reg_p (x, true, false))
6173 147618 : check_and_force_assignment_correctness_p = true;
6174 718799 : lra_set_insn_deleted (curr_insn);
6175 718799 : continue;
6176 : }
6177 : }
6178 108674477 : curr_id = lra_get_insn_recog_data (curr_insn);
6179 108674477 : curr_static_id = curr_id->insn_static_data;
6180 108674477 : init_curr_insn_input_reloads ();
6181 108674477 : init_curr_operand_mode ();
6182 108674477 : if (curr_insn_transform (false))
6183 : changed_p = true;
6184 : /* Check non-transformed insns too for equiv change as USE
6185 : or CLOBBER don't need reloads but can contain pseudos
6186 : being changed on their equivalences. */
6187 102305139 : else if (bitmap_bit_p (equiv_insn_bitmap, INSN_UID (curr_insn))
6188 102305139 : && loc_equivalence_change_p (&PATTERN (curr_insn)))
6189 : {
6190 8 : lra_update_insn_regno_info (curr_insn);
6191 8 : lra_push_insn_by_uid (INSN_UID (curr_insn));
6192 8 : changed_p = true;
6193 : }
6194 : }
6195 : }
6196 :
6197 : /* If we used a new hard regno, changed_p should be true because the
6198 : hard reg is assigned to a new pseudo. */
6199 3298462 : if (flag_checking && !changed_p)
6200 : {
6201 136531819 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
6202 133863801 : if (lra_reg_info[i].nrefs != 0
6203 195368193 : && (hard_regno = lra_get_regno_hard_regno (i)) >= 0)
6204 : {
6205 60025969 : int j, nregs = hard_regno_nregs (hard_regno,
6206 60025969 : PSEUDO_REGNO_MODE (i));
6207 :
6208 122161356 : for (j = 0; j < nregs; j++)
6209 62135387 : lra_assert (df_regs_ever_live_p (hard_regno + j));
6210 : }
6211 : }
6212 2668058 : if (changed_p)
6213 630407 : lra_dump_insns_if_possible ("changed func after local");
6214 3298462 : return changed_p;
6215 3298462 : }
6216 :
6217 : static void initiate_invariants (void);
6218 : static void finish_invariants (void);
6219 :
6220 : /* Initiate the LRA constraint pass. It is done once per
6221 : function. */
6222 : void
6223 1515121 : lra_constraints_init (void)
6224 : {
6225 1515121 : initiate_invariants ();
6226 1515121 : }
6227 :
6228 : /* Finalize the LRA constraint pass. It is done once per
6229 : function. */
6230 : void
6231 1515121 : lra_constraints_finish (void)
6232 : {
6233 1515121 : finish_invariants ();
6234 1515121 : }
6235 :
6236 :
6237 :
6238 : /* Structure describes invariants for ineheritance. */
6239 : struct lra_invariant
6240 : {
6241 : /* The order number of the invariant. */
6242 : int num;
6243 : /* The invariant RTX. */
6244 : rtx invariant_rtx;
6245 : /* The origin insn of the invariant. */
6246 : rtx_insn *insn;
6247 : };
6248 :
6249 : typedef lra_invariant invariant_t;
6250 : typedef invariant_t *invariant_ptr_t;
6251 : typedef const invariant_t *const_invariant_ptr_t;
6252 :
6253 : /* Pointer to the inheritance invariants. */
6254 : static vec<invariant_ptr_t> invariants;
6255 :
6256 : /* Allocation pool for the invariants. */
6257 : static object_allocator<lra_invariant> *invariants_pool;
6258 :
6259 : /* Hash table for the invariants. */
6260 : static htab_t invariant_table;
6261 :
6262 : /* Hash function for INVARIANT. */
6263 : static hashval_t
6264 198162 : invariant_hash (const void *invariant)
6265 : {
6266 198162 : rtx inv = ((const_invariant_ptr_t) invariant)->invariant_rtx;
6267 198162 : return lra_rtx_hash (inv);
6268 : }
6269 :
6270 : /* Equal function for invariants INVARIANT1 and INVARIANT2. */
6271 : static int
6272 63053 : invariant_eq_p (const void *invariant1, const void *invariant2)
6273 : {
6274 63053 : rtx inv1 = ((const_invariant_ptr_t) invariant1)->invariant_rtx;
6275 63053 : rtx inv2 = ((const_invariant_ptr_t) invariant2)->invariant_rtx;
6276 :
6277 63053 : return rtx_equal_p (inv1, inv2);
6278 : }
6279 :
6280 : /* Insert INVARIANT_RTX into the table if it is not there yet. Return
6281 : invariant which is in the table. */
6282 : static invariant_ptr_t
6283 196922 : insert_invariant (rtx invariant_rtx)
6284 : {
6285 196922 : void **entry_ptr;
6286 196922 : invariant_t invariant;
6287 196922 : invariant_ptr_t invariant_ptr;
6288 :
6289 196922 : invariant.invariant_rtx = invariant_rtx;
6290 196922 : entry_ptr = htab_find_slot (invariant_table, &invariant, INSERT);
6291 196922 : if (*entry_ptr == NULL)
6292 : {
6293 170987 : invariant_ptr = invariants_pool->allocate ();
6294 170987 : invariant_ptr->invariant_rtx = invariant_rtx;
6295 170987 : invariant_ptr->insn = NULL;
6296 170987 : invariants.safe_push (invariant_ptr);
6297 170987 : *entry_ptr = (void *) invariant_ptr;
6298 : }
6299 196922 : return (invariant_ptr_t) *entry_ptr;
6300 : }
6301 :
6302 : /* Initiate the invariant table. */
6303 : static void
6304 1515121 : initiate_invariants (void)
6305 : {
6306 1515121 : invariants.create (100);
6307 1515121 : invariants_pool
6308 1515121 : = new object_allocator<lra_invariant> ("Inheritance invariants");
6309 1515121 : invariant_table = htab_create (100, invariant_hash, invariant_eq_p, NULL);
6310 1515121 : }
6311 :
6312 : /* Finish the invariant table. */
6313 : static void
6314 1515121 : finish_invariants (void)
6315 : {
6316 1515121 : htab_delete (invariant_table);
6317 3030242 : delete invariants_pool;
6318 1515121 : invariants.release ();
6319 1515121 : }
6320 :
6321 : /* Make the invariant table empty. */
6322 : static void
6323 12891784 : clear_invariants (void)
6324 : {
6325 12891784 : htab_empty (invariant_table);
6326 12891784 : invariants_pool->release ();
6327 12891784 : invariants.truncate (0);
6328 12891784 : }
6329 :
6330 :
6331 :
6332 : /* This page contains code to do inheritance/split
6333 : transformations. */
6334 :
6335 : /* Number of reloads passed so far in current EBB. */
6336 : static int reloads_num;
6337 :
6338 : /* Number of calls passed so far in current EBB. */
6339 : static int calls_num;
6340 :
6341 : /* Index ID is the CALLS_NUM associated the last call we saw with
6342 : ABI identifier ID. */
6343 : static int last_call_for_abi[NUM_ABI_IDS];
6344 :
6345 : /* Which registers have been fully or partially clobbered by a call
6346 : since they were last used. */
6347 : static HARD_REG_SET full_and_partial_call_clobbers;
6348 :
6349 : /* Current reload pseudo check for validity of elements in
6350 : USAGE_INSNS. */
6351 : static int curr_usage_insns_check;
6352 :
6353 : /* Info about last usage of registers in EBB to do inheritance/split
6354 : transformation. Inheritance transformation is done from a spilled
6355 : pseudo and split transformations from a hard register or a pseudo
6356 : assigned to a hard register. */
6357 : struct usage_insns
6358 : {
6359 : /* If the value is equal to CURR_USAGE_INSNS_CHECK, then the member
6360 : value INSNS is valid. The insns is chain of optional debug insns
6361 : and a finishing non-debug insn using the corresponding reg. The
6362 : value is also used to mark the registers which are set up in the
6363 : current insn. The negated insn uid is used for this. */
6364 : int check;
6365 : /* Value of global reloads_num at the last insn in INSNS. */
6366 : int reloads_num;
6367 : /* Value of global reloads_nums at the last insn in INSNS. */
6368 : int calls_num;
6369 : /* It can be true only for splitting. And it means that the restore
6370 : insn should be put after insn given by the following member. */
6371 : bool after_p;
6372 : /* Next insns in the current EBB which use the original reg and the
6373 : original reg value is not changed between the current insn and
6374 : the next insns. In order words, e.g. for inheritance, if we need
6375 : to use the original reg value again in the next insns we can try
6376 : to use the value in a hard register from a reload insn of the
6377 : current insn. */
6378 : rtx insns;
6379 : };
6380 :
6381 : /* Map: regno -> corresponding pseudo usage insns. */
6382 : static struct usage_insns *usage_insns;
6383 :
6384 : static void
6385 251409973 : setup_next_usage_insn (int regno, rtx insn, int reloads_num, bool after_p)
6386 : {
6387 251409973 : usage_insns[regno].check = curr_usage_insns_check;
6388 251409973 : usage_insns[regno].insns = insn;
6389 251409973 : usage_insns[regno].reloads_num = reloads_num;
6390 251409973 : usage_insns[regno].calls_num = calls_num;
6391 251409973 : usage_insns[regno].after_p = after_p;
6392 251409973 : if (regno >= FIRST_PSEUDO_REGISTER && reg_renumber[regno] >= 0)
6393 113254322 : remove_from_hard_reg_set (&full_and_partial_call_clobbers,
6394 113254322 : PSEUDO_REGNO_MODE (regno),
6395 : reg_renumber[regno]);
6396 251409973 : }
6397 :
6398 : /* The function is used to form list REGNO usages which consists of
6399 : optional debug insns finished by a non-debug insn using REGNO.
6400 : RELOADS_NUM is current number of reload insns processed so far. */
6401 : static void
6402 143229085 : add_next_usage_insn (int regno, rtx_insn *insn, int reloads_num)
6403 : {
6404 143229085 : rtx next_usage_insns;
6405 :
6406 143229085 : if (usage_insns[regno].check == curr_usage_insns_check
6407 74887361 : && (next_usage_insns = usage_insns[regno].insns) != NULL_RTX
6408 218116446 : && DEBUG_INSN_P (insn))
6409 : {
6410 : /* Check that we did not add the debug insn yet. */
6411 14745704 : if (next_usage_insns != insn
6412 14745704 : && (GET_CODE (next_usage_insns) != INSN_LIST
6413 6778606 : || XEXP (next_usage_insns, 0) != insn))
6414 14745680 : usage_insns[regno].insns = gen_rtx_INSN_LIST (VOIDmode, insn,
6415 : next_usage_insns);
6416 : }
6417 128483381 : else if (NONDEBUG_INSN_P (insn))
6418 127953666 : setup_next_usage_insn (regno, insn, reloads_num, false);
6419 : else
6420 529715 : usage_insns[regno].check = 0;
6421 143229085 : }
6422 :
6423 : /* Return first non-debug insn in list USAGE_INSNS. */
6424 : static rtx_insn *
6425 1213183 : skip_usage_debug_insns (rtx usage_insns)
6426 : {
6427 1213183 : rtx insn;
6428 :
6429 : /* Skip debug insns. */
6430 1213183 : for (insn = usage_insns;
6431 1525415 : insn != NULL_RTX && GET_CODE (insn) == INSN_LIST;
6432 312232 : insn = XEXP (insn, 1))
6433 : ;
6434 1213183 : return safe_as_a <rtx_insn *> (insn);
6435 : }
6436 :
6437 : /* Return true if we need secondary memory moves for insn in
6438 : USAGE_INSNS after inserting inherited pseudo of class INHER_CL
6439 : into the insn. */
6440 : static bool
6441 1213190 : check_secondary_memory_needed_p (enum reg_class inher_cl ATTRIBUTE_UNUSED,
6442 : rtx usage_insns ATTRIBUTE_UNUSED)
6443 : {
6444 1213190 : rtx_insn *insn;
6445 1213190 : rtx set, dest;
6446 1213190 : enum reg_class cl;
6447 :
6448 1213190 : if (inher_cl == ALL_REGS
6449 1213190 : || (insn = skip_usage_debug_insns (usage_insns)) == NULL_RTX)
6450 : return false;
6451 1213183 : lra_assert (INSN_P (insn));
6452 1213183 : if ((set = single_set (insn)) == NULL_RTX || ! REG_P (SET_DEST (set)))
6453 : return false;
6454 1175985 : dest = SET_DEST (set);
6455 1175985 : if (! REG_P (dest))
6456 : return false;
6457 1175985 : lra_assert (inher_cl != NO_REGS);
6458 1175985 : cl = get_reg_class (REGNO (dest));
6459 1175985 : return (cl != NO_REGS && cl != ALL_REGS
6460 1175985 : && targetm.secondary_memory_needed (GET_MODE (dest), inher_cl, cl));
6461 : }
6462 :
6463 : /* Registers involved in inheritance/split in the current EBB
6464 : (inheritance/split pseudos and original registers). */
6465 : static bitmap_head check_only_regs;
6466 :
6467 : /* Reload pseudos cannot be involded in invariant inheritance in the
6468 : current EBB. */
6469 : static bitmap_head invalid_invariant_regs;
6470 :
6471 : /* Do inheritance transformations for insn INSN, which defines (if
6472 : DEF_P) or uses ORIGINAL_REGNO. NEXT_USAGE_INSNS specifies which
6473 : instruction in the EBB next uses ORIGINAL_REGNO; it has the same
6474 : form as the "insns" field of usage_insns. Return true if we
6475 : succeed in such transformation.
6476 :
6477 : The transformations look like:
6478 :
6479 : p <- ... i <- ...
6480 : ... p <- i (new insn)
6481 : ... =>
6482 : <- ... p ... <- ... i ...
6483 : or
6484 : ... i <- p (new insn)
6485 : <- ... p ... <- ... i ...
6486 : ... =>
6487 : <- ... p ... <- ... i ...
6488 : where p is a spilled original pseudo and i is a new inheritance pseudo.
6489 :
6490 :
6491 : The inheritance pseudo has the smallest class of two classes CL and
6492 : class of ORIGINAL REGNO. */
6493 : static bool
6494 1309283 : inherit_reload_reg (bool def_p, int original_regno,
6495 : enum reg_class cl, rtx_insn *insn, rtx next_usage_insns)
6496 : {
6497 1309283 : if (optimize_function_for_size_p (cfun))
6498 : return false;
6499 :
6500 1277618 : enum reg_class rclass = lra_get_allocno_class (original_regno);
6501 1277618 : rtx original_reg = regno_reg_rtx[original_regno];
6502 1277618 : rtx new_reg, usage_insn;
6503 1277618 : rtx_insn *new_insns;
6504 :
6505 1277618 : lra_assert (! usage_insns[original_regno].after_p);
6506 1277618 : if (lra_dump_file != NULL)
6507 2 : fprintf (lra_dump_file,
6508 : " <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
6509 1277618 : if (! ira_reg_classes_intersect_p[cl][rclass])
6510 : {
6511 64428 : if (lra_dump_file != NULL)
6512 : {
6513 0 : fprintf (lra_dump_file,
6514 : " Rejecting inheritance for %d "
6515 : "because of disjoint classes %s and %s\n",
6516 : original_regno, reg_class_names[cl],
6517 : reg_class_names[rclass]);
6518 0 : fprintf (lra_dump_file,
6519 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6520 : }
6521 : return false;
6522 : }
6523 1213190 : if ((ira_class_subset_p[cl][rclass] && cl != rclass)
6524 : /* We don't use a subset of two classes because it can be
6525 : NO_REGS. This transformation is still profitable in most
6526 : cases even if the classes are not intersected as register
6527 : move is probably cheaper than a memory load. */
6528 451154 : || ira_class_hard_regs_num[cl] < ira_class_hard_regs_num[rclass])
6529 : {
6530 762036 : if (lra_dump_file != NULL)
6531 2 : fprintf (lra_dump_file, " Use smallest class of %s and %s\n",
6532 : reg_class_names[cl], reg_class_names[rclass]);
6533 :
6534 : rclass = cl;
6535 : }
6536 1213190 : if (check_secondary_memory_needed_p (rclass, next_usage_insns))
6537 : {
6538 : /* Reject inheritance resulting in secondary memory moves.
6539 : Otherwise, there is a danger in LRA cycling. Also such
6540 : transformation will be unprofitable. */
6541 15578 : if (lra_dump_file != NULL)
6542 : {
6543 0 : rtx_insn *insn = skip_usage_debug_insns (next_usage_insns);
6544 0 : rtx set = single_set (insn);
6545 :
6546 0 : lra_assert (set != NULL_RTX);
6547 :
6548 0 : rtx dest = SET_DEST (set);
6549 :
6550 0 : lra_assert (REG_P (dest));
6551 0 : fprintf (lra_dump_file,
6552 : " Rejecting inheritance for insn %d(%s)<-%d(%s) "
6553 : "as secondary mem is needed\n",
6554 0 : REGNO (dest), reg_class_names[get_reg_class (REGNO (dest))],
6555 0 : original_regno, reg_class_names[rclass]);
6556 0 : fprintf (lra_dump_file,
6557 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6558 : }
6559 : return false;
6560 : }
6561 1197612 : if (ira_reg_class_min_nregs[rclass][GET_MODE (original_reg)]
6562 1197612 : != ira_reg_class_max_nregs[rclass][GET_MODE (original_reg)])
6563 : {
6564 24 : if (lra_dump_file != NULL)
6565 : {
6566 0 : fprintf (lra_dump_file,
6567 : " Rejecting inheritance for %d "
6568 : "because of requiring non-uniform class %s\n",
6569 : original_regno, reg_class_names[rclass]);
6570 0 : fprintf (lra_dump_file,
6571 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6572 : }
6573 : return false;
6574 : }
6575 1197588 : new_reg = lra_create_new_reg (GET_MODE (original_reg), original_reg,
6576 : rclass, NULL, "inheritance");
6577 1197588 : start_sequence ();
6578 1197588 : if (def_p)
6579 561364 : lra_emit_move (original_reg, new_reg);
6580 : else
6581 636224 : lra_emit_move (new_reg, original_reg);
6582 1197588 : new_insns = end_sequence ();
6583 1197588 : if (NEXT_INSN (new_insns) != NULL_RTX)
6584 : {
6585 0 : if (lra_dump_file != NULL)
6586 : {
6587 0 : fprintf (lra_dump_file,
6588 : " Rejecting inheritance %d->%d "
6589 : "as it results in 2 or more insns:\n",
6590 : original_regno, REGNO (new_reg));
6591 0 : dump_rtl_slim (lra_dump_file, new_insns, NULL, -1, 0);
6592 0 : fprintf (lra_dump_file,
6593 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6594 : }
6595 : return false;
6596 : }
6597 1197588 : lra_substitute_pseudo_within_insn (insn, original_regno, new_reg, false);
6598 1197588 : lra_update_insn_regno_info (insn);
6599 1197588 : if (! def_p)
6600 : /* We now have a new usage insn for original regno. */
6601 636224 : setup_next_usage_insn (original_regno, new_insns, reloads_num, false);
6602 1197588 : if (lra_dump_file != NULL)
6603 2 : fprintf (lra_dump_file, " Original reg change %d->%d (bb%d):\n",
6604 2 : original_regno, REGNO (new_reg), BLOCK_FOR_INSN (insn)->index);
6605 1197588 : lra_reg_info[REGNO (new_reg)].restore_rtx = regno_reg_rtx[original_regno];
6606 1197588 : bitmap_set_bit (&check_only_regs, REGNO (new_reg));
6607 1197588 : bitmap_set_bit (&check_only_regs, original_regno);
6608 1197588 : bitmap_set_bit (&lra_inheritance_pseudos, REGNO (new_reg));
6609 1197588 : if (def_p)
6610 561364 : lra_process_new_insns (insn, NULL, new_insns,
6611 : "Add original<-inheritance");
6612 : else
6613 636224 : lra_process_new_insns (insn, new_insns, NULL,
6614 : "Add inheritance<-original");
6615 2704497 : while (next_usage_insns != NULL_RTX)
6616 : {
6617 1506909 : if (GET_CODE (next_usage_insns) != INSN_LIST)
6618 : {
6619 1197588 : usage_insn = next_usage_insns;
6620 1197588 : lra_assert (NONDEBUG_INSN_P (usage_insn));
6621 : next_usage_insns = NULL;
6622 : }
6623 : else
6624 : {
6625 309321 : usage_insn = XEXP (next_usage_insns, 0);
6626 309321 : lra_assert (DEBUG_INSN_P (usage_insn));
6627 309321 : next_usage_insns = XEXP (next_usage_insns, 1);
6628 : }
6629 1506909 : lra_substitute_pseudo (&usage_insn, original_regno, new_reg, false,
6630 1506909 : DEBUG_INSN_P (usage_insn));
6631 1506909 : lra_update_insn_regno_info (as_a <rtx_insn *> (usage_insn));
6632 1506909 : if (lra_dump_file != NULL)
6633 : {
6634 2 : basic_block bb = BLOCK_FOR_INSN (usage_insn);
6635 2 : fprintf (lra_dump_file,
6636 : " Inheritance reuse change %d->%d (bb%d):\n",
6637 : original_regno, REGNO (new_reg),
6638 : bb ? bb->index : -1);
6639 2 : dump_insn_slim (lra_dump_file, as_a <rtx_insn *> (usage_insn));
6640 : }
6641 : }
6642 1197588 : if (lra_dump_file != NULL)
6643 2 : fprintf (lra_dump_file,
6644 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6645 : return true;
6646 : }
6647 :
6648 : /* Return true if we need a caller save/restore for pseudo REGNO which
6649 : was assigned to a hard register. */
6650 : static inline bool
6651 115945428 : need_for_call_save_p (int regno)
6652 : {
6653 115945428 : lra_assert (regno >= FIRST_PSEUDO_REGISTER && reg_renumber[regno] >= 0);
6654 115945428 : if (usage_insns[regno].calls_num < calls_num)
6655 : {
6656 : unsigned int abis = 0;
6657 121677673 : for (unsigned int i = 0; i < NUM_ABI_IDS; ++i)
6658 112317852 : if (last_call_for_abi[i] > usage_insns[regno].calls_num)
6659 9409140 : abis |= 1 << i;
6660 9359821 : gcc_assert (abis);
6661 9359821 : if (call_clobbered_in_region_p (abis, full_and_partial_call_clobbers,
6662 9359821 : PSEUDO_REGNO_MODE (regno),
6663 : reg_renumber[regno]))
6664 : return true;
6665 : }
6666 : return false;
6667 : }
6668 :
6669 : /* Global registers occurring in the current EBB. */
6670 : static bitmap_head ebb_global_regs;
6671 :
6672 : /* Return true if we need a split for hard register REGNO or pseudo
6673 : REGNO which was assigned to a hard register.
6674 : POTENTIAL_RELOAD_HARD_REGS contains hard registers which might be
6675 : used for reloads since the EBB end. It is an approximation of the
6676 : used hard registers in the split range. The exact value would
6677 : require expensive calculations. If we were aggressive with
6678 : splitting because of the approximation, the split pseudo will save
6679 : the same hard register assignment and will be removed in the undo
6680 : pass. We still need the approximation because too aggressive
6681 : splitting would result in too inaccurate cost calculation in the
6682 : assignment pass because of too many generated moves which will be
6683 : probably removed in the undo pass. */
6684 : static inline bool
6685 246524294 : need_for_split_p (HARD_REG_SET potential_reload_hard_regs, int regno)
6686 : {
6687 246524294 : int hard_regno = regno < FIRST_PSEUDO_REGISTER ? regno : reg_renumber[regno];
6688 :
6689 246524294 : lra_assert (hard_regno >= 0);
6690 246524294 : return ((TEST_HARD_REG_BIT (potential_reload_hard_regs, hard_regno)
6691 : /* Don't split eliminable hard registers, otherwise we can
6692 : split hard registers like hard frame pointer, which
6693 : lives on BB start/end according to DF-infrastructure,
6694 : when there is a pseudo assigned to the register and
6695 : living in the same BB. */
6696 661926 : && (regno >= FIRST_PSEUDO_REGISTER
6697 45596 : || ! TEST_HARD_REG_BIT (eliminable_regset, hard_regno))
6698 631075 : && ! TEST_HARD_REG_BIT (lra_no_alloc_regs, hard_regno)
6699 : /* Don't split call clobbered hard regs living through
6700 : calls, otherwise we might have a check problem in the
6701 : assign sub-pass as in the most cases (exception is a
6702 : situation when check_and_force_assignment_correctness_p value is
6703 : true) the assign pass assumes that all pseudos living
6704 : through calls are assigned to call saved hard regs. */
6705 617353 : && (regno >= FIRST_PSEUDO_REGISTER
6706 1023 : || !TEST_HARD_REG_BIT (full_and_partial_call_clobbers, regno))
6707 : /* We need at least 2 reloads to make pseudo splitting
6708 : profitable. We should provide hard regno splitting in
6709 : any case to solve 1st insn scheduling problem when
6710 : moving hard register definition up might result in
6711 : impossibility to find hard register for reload pseudo of
6712 : small register class. */
6713 1234654 : && (usage_insns[regno].reloads_num
6714 1233657 : + (regno < FIRST_PSEUDO_REGISTER ? 0 : 3) < reloads_num)
6715 2962 : && (regno < FIRST_PSEUDO_REGISTER
6716 : /* For short living pseudos, spilling + inheritance can
6717 : be considered a substitution for splitting.
6718 : Therefore we do not splitting for local pseudos. It
6719 : decreases also aggressiveness of splitting. The
6720 : minimal number of references is chosen taking into
6721 : account that for 2 references splitting has no sense
6722 : as we can just spill the pseudo. */
6723 : || (regno >= FIRST_PSEUDO_REGISTER
6724 2915 : && lra_reg_info[regno].nrefs > 3
6725 2663 : && bitmap_bit_p (&ebb_global_regs, regno))))
6726 247184632 : || (regno >= FIRST_PSEUDO_REGISTER && need_for_call_save_p (regno)));
6727 : }
6728 :
6729 : /* Return class for the split pseudo created from original pseudo with
6730 : ALLOCNO_CLASS and MODE which got a hard register HARD_REGNO. We
6731 : choose subclass of ALLOCNO_CLASS which contains HARD_REGNO and
6732 : results in no secondary memory movements. */
6733 : static enum reg_class
6734 1745 : choose_split_class (enum reg_class allocno_class,
6735 : int hard_regno ATTRIBUTE_UNUSED,
6736 : machine_mode mode ATTRIBUTE_UNUSED)
6737 : {
6738 1745 : int i;
6739 1745 : enum reg_class cl, best_cl = NO_REGS;
6740 1745 : enum reg_class hard_reg_class ATTRIBUTE_UNUSED
6741 : = REGNO_REG_CLASS (hard_regno);
6742 :
6743 1745 : if (! targetm.secondary_memory_needed (mode, allocno_class, allocno_class)
6744 1745 : && TEST_HARD_REG_BIT (reg_class_contents[allocno_class], hard_regno))
6745 : return allocno_class;
6746 0 : for (i = 0;
6747 0 : (cl = reg_class_subclasses[allocno_class][i]) != LIM_REG_CLASSES;
6748 : i++)
6749 0 : if (! targetm.secondary_memory_needed (mode, cl, hard_reg_class)
6750 0 : && ! targetm.secondary_memory_needed (mode, hard_reg_class, cl)
6751 0 : && TEST_HARD_REG_BIT (reg_class_contents[cl], hard_regno)
6752 0 : && (best_cl == NO_REGS
6753 0 : || ira_class_hard_regs_num[best_cl] < ira_class_hard_regs_num[cl]))
6754 : best_cl = cl;
6755 : return best_cl;
6756 : }
6757 :
6758 : /* Copy any equivalence information from ORIGINAL_REGNO to NEW_REGNO. It only
6759 : makes sense to call this function if NEW_REGNO is always equal to
6760 : ORIGINAL_REGNO. Set up defined_p flag when caller_save_p flag is set up and
6761 : CALL_SAVE_P is true. */
6762 :
6763 : static void
6764 668389 : lra_copy_reg_equiv (unsigned int new_regno, unsigned int original_regno,
6765 : bool call_save_p)
6766 : {
6767 668389 : if (!ira_reg_equiv[original_regno].defined_p
6768 601396 : && !(call_save_p && ira_reg_equiv[original_regno].caller_save_p))
6769 : return;
6770 :
6771 67165 : ira_expand_reg_equiv ();
6772 67165 : ira_reg_equiv[new_regno].defined_p = true;
6773 67165 : if (ira_reg_equiv[original_regno].memory)
6774 33044 : ira_reg_equiv[new_regno].memory
6775 33044 : = copy_rtx (ira_reg_equiv[original_regno].memory);
6776 67165 : if (ira_reg_equiv[original_regno].constant)
6777 27358 : ira_reg_equiv[new_regno].constant
6778 27358 : = copy_rtx (ira_reg_equiv[original_regno].constant);
6779 67165 : if (ira_reg_equiv[original_regno].invariant)
6780 6763 : ira_reg_equiv[new_regno].invariant
6781 6763 : = copy_rtx (ira_reg_equiv[original_regno].invariant);
6782 : }
6783 :
6784 : /* Do split transformations for insn INSN, which defines or uses
6785 : ORIGINAL_REGNO. NEXT_USAGE_INSNS specifies which instruction in
6786 : the EBB next uses ORIGINAL_REGNO; it has the same form as the
6787 : "insns" field of usage_insns. If TO is not NULL, we don't use
6788 : usage_insns, we put restore insns after TO insn. It is a case when
6789 : we call it from lra_split_hard_reg_for, outside the inheritance
6790 : pass.
6791 :
6792 : The transformations look like:
6793 :
6794 : p <- ... p <- ...
6795 : ... s <- p (new insn -- save)
6796 : ... =>
6797 : ... p <- s (new insn -- restore)
6798 : <- ... p ... <- ... p ...
6799 : or
6800 : <- ... p ... <- ... p ...
6801 : ... s <- p (new insn -- save)
6802 : ... =>
6803 : ... p <- s (new insn -- restore)
6804 : <- ... p ... <- ... p ...
6805 :
6806 : where p is an original pseudo got a hard register or a hard
6807 : register and s is a new split pseudo. The save is put before INSN
6808 : if BEFORE_P is true. Return true if we succeed in such
6809 : transformation. */
6810 : static bool
6811 670090 : split_reg (bool before_p, int original_regno, rtx_insn *insn,
6812 : rtx next_usage_insns, rtx_insn *to)
6813 : {
6814 670090 : enum reg_class rclass;
6815 670090 : rtx original_reg;
6816 670090 : int hard_regno, nregs;
6817 670090 : rtx new_reg, usage_insn;
6818 670090 : rtx_insn *restore, *save;
6819 670090 : bool after_p;
6820 670090 : bool call_save_p;
6821 670090 : machine_mode mode;
6822 :
6823 670090 : if (original_regno < FIRST_PSEUDO_REGISTER)
6824 : {
6825 206 : rclass = ira_allocno_class_translate[REGNO_REG_CLASS (original_regno)];
6826 206 : hard_regno = original_regno;
6827 206 : call_save_p = false;
6828 206 : nregs = 1;
6829 206 : mode = lra_reg_info[hard_regno].biggest_mode;
6830 206 : machine_mode reg_rtx_mode = GET_MODE (regno_reg_rtx[hard_regno]);
6831 : /* A reg can have a biggest_mode of VOIDmode if it was only ever seen as
6832 : part of a multi-word register. In that case, just use the reg_rtx
6833 : mode. Do the same also if the biggest mode was larger than a register
6834 : or we can not compare the modes. Otherwise, limit the size to that of
6835 : the biggest access in the function or to the natural mode at least. */
6836 206 : if (mode == VOIDmode
6837 206 : || !ordered_p (GET_MODE_PRECISION (mode),
6838 206 : GET_MODE_PRECISION (reg_rtx_mode))
6839 206 : || paradoxical_subreg_p (mode, reg_rtx_mode)
6840 411 : || maybe_gt (GET_MODE_PRECISION (reg_rtx_mode), GET_MODE_PRECISION (mode)))
6841 : {
6842 : original_reg = regno_reg_rtx[hard_regno];
6843 : mode = reg_rtx_mode;
6844 : }
6845 : else
6846 189 : original_reg = gen_rtx_REG (mode, hard_regno);
6847 : }
6848 : else
6849 : {
6850 669884 : mode = PSEUDO_REGNO_MODE (original_regno);
6851 669884 : hard_regno = reg_renumber[original_regno];
6852 669884 : nregs = hard_regno_nregs (hard_regno, mode);
6853 669884 : rclass = lra_get_allocno_class (original_regno);
6854 669884 : original_reg = regno_reg_rtx[original_regno];
6855 669884 : call_save_p = need_for_call_save_p (original_regno);
6856 : }
6857 670090 : lra_assert (hard_regno >= 0);
6858 670090 : if (lra_dump_file != NULL)
6859 0 : fprintf (lra_dump_file,
6860 : " ((((((((((((((((((((((((((((((((((((((((((((((((\n");
6861 :
6862 670090 : if (call_save_p)
6863 : {
6864 668345 : mode = HARD_REGNO_CALLER_SAVE_MODE (hard_regno,
6865 : hard_regno_nregs (hard_regno, mode),
6866 : mode);
6867 668345 : new_reg = lra_create_new_reg (mode, NULL_RTX, NO_REGS, NULL, "save");
6868 : }
6869 : else
6870 : {
6871 1745 : rclass = choose_split_class (rclass, hard_regno, mode);
6872 1745 : if (rclass == NO_REGS)
6873 : {
6874 0 : if (lra_dump_file != NULL)
6875 : {
6876 0 : fprintf (lra_dump_file,
6877 : " Rejecting split of %d(%s): "
6878 : "no good reg class for %d(%s)\n",
6879 : original_regno,
6880 0 : reg_class_names[lra_get_allocno_class (original_regno)],
6881 : hard_regno,
6882 0 : reg_class_names[REGNO_REG_CLASS (hard_regno)]);
6883 0 : fprintf
6884 0 : (lra_dump_file,
6885 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
6886 : }
6887 : return false;
6888 : }
6889 : /* Split_if_necessary can split hard registers used as part of a
6890 : multi-register mode but splits each register individually. The
6891 : mode used for each independent register may not be supported
6892 : so reject the split. Splitting the wider mode should theoretically
6893 : be possible but is not implemented. */
6894 1745 : if (!targetm.hard_regno_mode_ok (hard_regno, mode))
6895 : {
6896 0 : if (lra_dump_file != NULL)
6897 : {
6898 0 : fprintf (lra_dump_file,
6899 : " Rejecting split of %d(%s): unsuitable mode %s\n",
6900 : original_regno,
6901 0 : reg_class_names[lra_get_allocno_class (original_regno)],
6902 0 : GET_MODE_NAME (mode));
6903 0 : fprintf
6904 0 : (lra_dump_file,
6905 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
6906 : }
6907 : return false;
6908 : }
6909 1745 : new_reg = lra_create_new_reg (mode, original_reg, rclass, NULL, "split");
6910 1745 : reg_renumber[REGNO (new_reg)] = hard_regno;
6911 : }
6912 670090 : int new_regno = REGNO (new_reg);
6913 670090 : save = emit_spill_move (true, new_reg, original_reg);
6914 670090 : if (NEXT_INSN (save) != NULL_RTX && !call_save_p)
6915 : {
6916 0 : if (lra_dump_file != NULL)
6917 : {
6918 0 : fprintf
6919 0 : (lra_dump_file,
6920 : " Rejecting split %d->%d resulting in > 2 save insns:\n",
6921 : original_regno, new_regno);
6922 0 : dump_rtl_slim (lra_dump_file, save, NULL, -1, 0);
6923 0 : fprintf (lra_dump_file,
6924 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
6925 : }
6926 : return false;
6927 : }
6928 670090 : restore = emit_spill_move (false, new_reg, original_reg);
6929 670090 : if (NEXT_INSN (restore) != NULL_RTX && !call_save_p)
6930 : {
6931 0 : if (lra_dump_file != NULL)
6932 : {
6933 0 : fprintf (lra_dump_file,
6934 : " Rejecting split %d->%d "
6935 : "resulting in > 2 restore insns:\n",
6936 : original_regno, new_regno);
6937 0 : dump_rtl_slim (lra_dump_file, restore, NULL, -1, 0);
6938 0 : fprintf (lra_dump_file,
6939 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
6940 : }
6941 : return false;
6942 : }
6943 : /* Transfer equivalence information to the spill register, so that
6944 : if we fail to allocate the spill register, we have the option of
6945 : rematerializing the original value instead of spilling to the stack. */
6946 670090 : if (!HARD_REGISTER_NUM_P (original_regno)
6947 669884 : && mode == PSEUDO_REGNO_MODE (original_regno))
6948 668389 : lra_copy_reg_equiv (new_regno, original_regno, call_save_p);
6949 670090 : lra_reg_info[new_regno].restore_rtx = regno_reg_rtx[original_regno];
6950 670090 : bitmap_set_bit (&lra_split_regs, new_regno);
6951 670090 : if (to != NULL)
6952 : {
6953 159 : lra_assert (next_usage_insns == NULL);
6954 159 : usage_insn = to;
6955 159 : after_p = true;
6956 : }
6957 : else
6958 : {
6959 : /* We need check_only_regs only inside the inheritance pass. */
6960 669931 : bitmap_set_bit (&check_only_regs, new_regno);
6961 669931 : bitmap_set_bit (&check_only_regs, original_regno);
6962 669931 : after_p = usage_insns[original_regno].after_p;
6963 773161 : for (;;)
6964 : {
6965 773161 : if (GET_CODE (next_usage_insns) != INSN_LIST)
6966 : {
6967 669931 : usage_insn = next_usage_insns;
6968 669931 : break;
6969 : }
6970 103230 : usage_insn = XEXP (next_usage_insns, 0);
6971 103230 : lra_assert (DEBUG_INSN_P (usage_insn));
6972 103230 : next_usage_insns = XEXP (next_usage_insns, 1);
6973 103230 : lra_substitute_pseudo (&usage_insn, original_regno, new_reg, false,
6974 : true);
6975 103230 : lra_update_insn_regno_info (as_a <rtx_insn *> (usage_insn));
6976 103230 : if (lra_dump_file != NULL)
6977 : {
6978 0 : fprintf (lra_dump_file, " Split reuse change %d->%d:\n",
6979 : original_regno, new_regno);
6980 0 : dump_insn_slim (lra_dump_file, as_a <rtx_insn *> (usage_insn));
6981 : }
6982 : }
6983 : }
6984 670090 : lra_assert (NOTE_P (usage_insn) || NONDEBUG_INSN_P (usage_insn));
6985 670090 : lra_assert (usage_insn != insn || (after_p && before_p));
6986 1133014 : lra_process_new_insns (as_a <rtx_insn *> (usage_insn),
6987 : after_p ? NULL : restore,
6988 : after_p ? restore : NULL,
6989 : call_save_p ? "Add reg<-save" : "Add reg<-split");
6990 670090 : if (call_save_p
6991 668345 : && first_call_insn != NULL
6992 1338435 : && BLOCK_FOR_INSN (first_call_insn) != BLOCK_FOR_INSN (insn))
6993 : /* PR116028: If original_regno is a pseudo that has been assigned a
6994 : callee-saved hard register, then emit the spill insn before the call
6995 : insn 'first_call_insn' instead of adjacent to 'insn'. If 'insn'
6996 : and 'first_call_insn' belong to the same EBB but to two separate
6997 : BBs, and if 'insn' is present in the entry BB, then generating the
6998 : spill insn in the entry BB can prevent shrink wrap from happening.
6999 : This is because the spill insn references the stack pointer and
7000 : hence the prolog gets generated in the entry BB itself. It is
7001 : also more efficient to generate the spill before
7002 : 'first_call_insn' as the spill now occurs only in the path
7003 : containing the call. */
7004 25879 : lra_process_new_insns (first_call_insn, save, NULL, "Add save<-reg");
7005 : else
7006 1289510 : lra_process_new_insns (insn, before_p ? save : NULL,
7007 : before_p ? NULL : save,
7008 : call_save_p ? "Add save<-reg" : "Add split<-reg");
7009 670090 : if (nregs > 1 || original_regno < FIRST_PSEUDO_REGISTER)
7010 : /* If we are trying to split multi-register. We should check
7011 : conflicts on the next assignment sub-pass. IRA can allocate on
7012 : sub-register levels, LRA do this on pseudos level right now and
7013 : this discrepancy may create allocation conflicts after
7014 : splitting.
7015 :
7016 : If we are trying to split hard register we should also check conflicts
7017 : as such splitting can create artificial conflict of the hard register
7018 : with another pseudo because of simplified conflict calculation in
7019 : LRA. */
7020 9776 : check_and_force_assignment_correctness_p = true;
7021 670090 : if (lra_dump_file != NULL)
7022 0 : fprintf (lra_dump_file,
7023 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
7024 : return true;
7025 : }
7026 :
7027 : /* Split a hard reg for reload pseudo REGNO having RCLASS and living
7028 : in the range [FROM, TO]. Return true if did a split. Otherwise,
7029 : return false. */
7030 : bool
7031 1570 : spill_hard_reg_in_range (int regno, enum reg_class rclass, rtx_insn *from, rtx_insn *to)
7032 : {
7033 1570 : int i, hard_regno;
7034 1570 : int rclass_size;
7035 1570 : rtx_insn *insn;
7036 1570 : unsigned int uid;
7037 1570 : bitmap_iterator bi;
7038 1570 : HARD_REG_SET ignore;
7039 :
7040 1570 : lra_assert (from != NULL && to != NULL);
7041 1570 : ignore = lra_no_alloc_regs;
7042 4523 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi)
7043 : {
7044 2953 : lra_insn_recog_data_t id = lra_insn_recog_data[uid];
7045 2953 : struct lra_static_insn_data *static_id = id->insn_static_data;
7046 2953 : struct lra_insn_reg *reg;
7047 :
7048 9848 : for (reg = id->regs; reg != NULL; reg = reg->next)
7049 6895 : if (reg->regno < FIRST_PSEUDO_REGISTER)
7050 157 : SET_HARD_REG_BIT (ignore, reg->regno);
7051 4609 : for (reg = static_id->hard_regs; reg != NULL; reg = reg->next)
7052 1656 : SET_HARD_REG_BIT (ignore, reg->regno);
7053 : }
7054 1570 : rclass_size = ira_class_hard_regs_num[rclass];
7055 4212 : for (i = 0; i < rclass_size; i++)
7056 : {
7057 2801 : hard_regno = ira_class_hard_regs[rclass][i];
7058 2801 : if (! TEST_HARD_REG_BIT (lra_reg_info[regno].conflict_hard_regs, hard_regno)
7059 2801 : || TEST_HARD_REG_BIT (ignore, hard_regno))
7060 2636 : continue;
7061 476 : for (insn = from; insn != NEXT_INSN (to); insn = NEXT_INSN (insn))
7062 : {
7063 317 : struct lra_static_insn_data *static_id;
7064 317 : struct lra_insn_reg *reg;
7065 :
7066 317 : if (!INSN_P (insn))
7067 0 : continue;
7068 317 : if (bitmap_bit_p (&lra_reg_info[hard_regno].insn_bitmap,
7069 317 : INSN_UID (insn)))
7070 : break;
7071 311 : static_id = lra_get_insn_recog_data (insn)->insn_static_data;
7072 365 : for (reg = static_id->hard_regs; reg != NULL; reg = reg->next)
7073 54 : if (reg->regno == hard_regno)
7074 : break;
7075 : if (reg != NULL)
7076 : break;
7077 : }
7078 165 : if (insn != NEXT_INSN (to))
7079 6 : continue;
7080 159 : if (split_reg (true, hard_regno, from, NULL, to))
7081 : return true;
7082 : }
7083 : return false;
7084 : }
7085 :
7086 : /* Recognize that we need a split transformation for insn INSN, which
7087 : defines or uses REGNO in its insn biggest MODE (we use it only if
7088 : REGNO is a hard register). POTENTIAL_RELOAD_HARD_REGS contains
7089 : hard registers which might be used for reloads since the EBB end.
7090 : Put the save before INSN if BEFORE_P is true. MAX_UID is maximla
7091 : uid before starting INSN processing. Return true if we succeed in
7092 : such transformation. */
7093 : static bool
7094 203375921 : split_if_necessary (int regno, machine_mode mode,
7095 : HARD_REG_SET potential_reload_hard_regs,
7096 : bool before_p, rtx_insn *insn, int max_uid)
7097 : {
7098 203375921 : bool res = false;
7099 203375921 : int i, nregs = 1;
7100 203375921 : rtx next_usage_insns;
7101 :
7102 203375921 : if (regno < FIRST_PSEUDO_REGISTER)
7103 95729795 : nregs = hard_regno_nregs (regno, mode);
7104 407114608 : for (i = 0; i < nregs; i++)
7105 203738687 : if (usage_insns[regno + i].check == curr_usage_insns_check
7106 135854042 : && (next_usage_insns = usage_insns[regno + i].insns) != NULL_RTX
7107 : /* To avoid processing the register twice or more. */
7108 135854042 : && ((GET_CODE (next_usage_insns) != INSN_LIST
7109 131458273 : && INSN_UID (next_usage_insns) < max_uid)
7110 4395769 : || (GET_CODE (next_usage_insns) == INSN_LIST
7111 4395769 : && (INSN_UID (XEXP (next_usage_insns, 0)) < max_uid)))
7112 135854042 : && need_for_split_p (potential_reload_hard_regs, regno + i)
7113 204022407 : && split_reg (before_p, regno + i, insn, next_usage_insns, NULL))
7114 : res = true;
7115 203375921 : return res;
7116 : }
7117 :
7118 : /* Return TRUE if rtx X is considered as an invariant for
7119 : inheritance. */
7120 : static bool
7121 12202397 : invariant_p (const_rtx x)
7122 : {
7123 12202397 : machine_mode mode;
7124 12202397 : const char *fmt;
7125 12202397 : enum rtx_code code;
7126 12202397 : int i, j;
7127 :
7128 12202397 : if (side_effects_p (x))
7129 : return false;
7130 :
7131 12176422 : code = GET_CODE (x);
7132 12176422 : mode = GET_MODE (x);
7133 12176422 : if (code == SUBREG)
7134 : {
7135 485033 : x = SUBREG_REG (x);
7136 485033 : code = GET_CODE (x);
7137 485033 : mode = wider_subreg_mode (mode, GET_MODE (x));
7138 : }
7139 :
7140 12176422 : if (MEM_P (x))
7141 : return false;
7142 :
7143 10395445 : if (REG_P (x))
7144 : {
7145 3697955 : int i, nregs, regno = REGNO (x);
7146 :
7147 3697955 : if (regno >= FIRST_PSEUDO_REGISTER || regno == STACK_POINTER_REGNUM
7148 1005089 : || TEST_HARD_REG_BIT (eliminable_regset, regno)
7149 3716583 : || GET_MODE_CLASS (GET_MODE (x)) == MODE_CC)
7150 : return false;
7151 2 : nregs = hard_regno_nregs (regno, mode);
7152 2 : for (i = 0; i < nregs; i++)
7153 2 : if (! fixed_regs[regno + i]
7154 : /* A hard register may be clobbered in the current insn
7155 : but we can ignore this case because if the hard
7156 : register is used it should be set somewhere after the
7157 : clobber. */
7158 2 : || bitmap_bit_p (&invalid_invariant_regs, regno + i))
7159 : return false;
7160 : }
7161 6697490 : fmt = GET_RTX_FORMAT (code);
7162 11831682 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
7163 : {
7164 8923256 : if (fmt[i] == 'e')
7165 : {
7166 5837585 : if (! invariant_p (XEXP (x, i)))
7167 : return false;
7168 : }
7169 3085671 : else if (fmt[i] == 'E')
7170 : {
7171 679753 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
7172 559109 : if (! invariant_p (XVECEXP (x, i, j)))
7173 : return false;
7174 : }
7175 : }
7176 : return true;
7177 : }
7178 :
7179 : /* We have 'dest_reg <- invariant'. Let us try to make an invariant
7180 : inheritance transformation (using dest_reg instead invariant in a
7181 : subsequent insn). */
7182 : static bool
7183 196922 : process_invariant_for_inheritance (rtx dst_reg, rtx invariant_rtx)
7184 : {
7185 196922 : invariant_ptr_t invariant_ptr;
7186 196922 : rtx_insn *insn, *new_insns;
7187 196922 : rtx insn_set, insn_reg, new_reg;
7188 196922 : int insn_regno;
7189 196922 : bool succ_p = false;
7190 196922 : int dst_regno = REGNO (dst_reg);
7191 196922 : machine_mode dst_mode = GET_MODE (dst_reg);
7192 196922 : enum reg_class cl = lra_get_allocno_class (dst_regno), insn_reg_cl;
7193 :
7194 196922 : invariant_ptr = insert_invariant (invariant_rtx);
7195 196922 : if ((insn = invariant_ptr->insn) != NULL_RTX)
7196 : {
7197 : /* We have a subsequent insn using the invariant. */
7198 25935 : insn_set = single_set (insn);
7199 25935 : lra_assert (insn_set != NULL);
7200 25935 : insn_reg = SET_DEST (insn_set);
7201 25935 : lra_assert (REG_P (insn_reg));
7202 25935 : insn_regno = REGNO (insn_reg);
7203 25935 : insn_reg_cl = lra_get_allocno_class (insn_regno);
7204 :
7205 25935 : if (dst_mode == GET_MODE (insn_reg)
7206 : /* We should consider only result move reg insns which are
7207 : cheap. */
7208 25869 : && targetm.register_move_cost (dst_mode, cl, insn_reg_cl) == 2
7209 51217 : && targetm.register_move_cost (dst_mode, cl, cl) == 2)
7210 : {
7211 25282 : if (lra_dump_file != NULL)
7212 0 : fprintf (lra_dump_file,
7213 : " [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[\n");
7214 25282 : new_reg = lra_create_new_reg (dst_mode, dst_reg, cl, NULL,
7215 : "invariant inheritance");
7216 25282 : bitmap_set_bit (&lra_inheritance_pseudos, REGNO (new_reg));
7217 25282 : bitmap_set_bit (&check_only_regs, REGNO (new_reg));
7218 25282 : lra_reg_info[REGNO (new_reg)].restore_rtx = PATTERN (insn);
7219 25282 : start_sequence ();
7220 25282 : lra_emit_move (new_reg, dst_reg);
7221 25282 : new_insns = end_sequence ();
7222 25282 : lra_process_new_insns (curr_insn, NULL, new_insns,
7223 : "Add invariant inheritance<-original");
7224 25282 : start_sequence ();
7225 25282 : lra_emit_move (SET_DEST (insn_set), new_reg);
7226 25282 : new_insns = end_sequence ();
7227 25282 : lra_process_new_insns (insn, NULL, new_insns,
7228 : "Changing reload<-inheritance");
7229 25282 : lra_set_insn_deleted (insn);
7230 25282 : succ_p = true;
7231 25282 : if (lra_dump_file != NULL)
7232 : {
7233 0 : fprintf (lra_dump_file,
7234 : " Invariant inheritance reuse change %d (bb%d):\n",
7235 0 : REGNO (new_reg), BLOCK_FOR_INSN (insn)->index);
7236 0 : dump_insn_slim (lra_dump_file, insn);
7237 0 : fprintf (lra_dump_file,
7238 : " ]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]\n");
7239 : }
7240 : }
7241 : }
7242 196922 : invariant_ptr->insn = curr_insn;
7243 196922 : return succ_p;
7244 : }
7245 :
7246 : /* Check only registers living at the current program point in the
7247 : current EBB. */
7248 : static bitmap_head live_regs;
7249 :
7250 : /* Update live info in EBB given by its HEAD and TAIL insns after
7251 : inheritance/split transformation. The function removes dead moves
7252 : too. */
7253 : static void
7254 758848 : update_ebb_live_info (rtx_insn *head, rtx_insn *tail)
7255 : {
7256 758848 : unsigned int j;
7257 758848 : int i, regno;
7258 758848 : bool live_p;
7259 758848 : rtx_insn *prev_insn;
7260 758848 : rtx set;
7261 758848 : bool remove_p;
7262 758848 : basic_block last_bb, prev_bb, curr_bb;
7263 758848 : bitmap_iterator bi;
7264 758848 : struct lra_insn_reg *reg;
7265 758848 : edge e;
7266 758848 : edge_iterator ei;
7267 :
7268 758848 : last_bb = BLOCK_FOR_INSN (tail);
7269 758848 : prev_bb = NULL;
7270 758848 : for (curr_insn = tail;
7271 39252820 : curr_insn != PREV_INSN (head);
7272 38493972 : curr_insn = prev_insn)
7273 : {
7274 38493972 : prev_insn = PREV_INSN (curr_insn);
7275 : /* We need to process empty blocks too. They contain
7276 : NOTE_INSN_BASIC_BLOCK referring for the basic block. */
7277 38493972 : if (NOTE_P (curr_insn) && NOTE_KIND (curr_insn) != NOTE_INSN_BASIC_BLOCK)
7278 1465875 : continue;
7279 37028097 : curr_bb = BLOCK_FOR_INSN (curr_insn);
7280 37028097 : if (curr_bb != prev_bb)
7281 : {
7282 1510314 : if (prev_bb != NULL)
7283 : {
7284 : /* Update df_get_live_in (prev_bb): */
7285 54412501 : EXECUTE_IF_SET_IN_BITMAP (&check_only_regs, 0, j, bi)
7286 53661035 : if (bitmap_bit_p (&live_regs, j))
7287 1637634 : bitmap_set_bit (df_get_live_in (prev_bb), j);
7288 : else
7289 52023401 : bitmap_clear_bit (df_get_live_in (prev_bb), j);
7290 : }
7291 1510314 : if (curr_bb != last_bb)
7292 : {
7293 : /* Update df_get_live_out (curr_bb): */
7294 54412501 : EXECUTE_IF_SET_IN_BITMAP (&check_only_regs, 0, j, bi)
7295 : {
7296 53661035 : live_p = bitmap_bit_p (&live_regs, j);
7297 53661035 : if (! live_p)
7298 155972922 : FOR_EACH_EDGE (e, ei, curr_bb->succs)
7299 104005575 : if (bitmap_bit_p (df_get_live_in (e->dest), j))
7300 : {
7301 : live_p = true;
7302 : break;
7303 : }
7304 52023401 : if (live_p)
7305 1693688 : bitmap_set_bit (df_get_live_out (curr_bb), j);
7306 : else
7307 51967347 : bitmap_clear_bit (df_get_live_out (curr_bb), j);
7308 : }
7309 : }
7310 1510314 : prev_bb = curr_bb;
7311 1510314 : bitmap_and (&live_regs, &check_only_regs, df_get_live_out (curr_bb));
7312 : }
7313 37028097 : if (! NONDEBUG_INSN_P (curr_insn))
7314 14283498 : continue;
7315 22744599 : curr_id = lra_get_insn_recog_data (curr_insn);
7316 22744599 : curr_static_id = curr_id->insn_static_data;
7317 22744599 : remove_p = false;
7318 22744599 : if ((set = single_set (curr_insn)) != NULL_RTX
7319 22040857 : && REG_P (SET_DEST (set))
7320 17621339 : && (regno = REGNO (SET_DEST (set))) >= FIRST_PSEUDO_REGISTER
7321 13095987 : && SET_DEST (set) != pic_offset_table_rtx
7322 13089117 : && bitmap_bit_p (&check_only_regs, regno)
7323 26071072 : && ! bitmap_bit_p (&live_regs, regno))
7324 : remove_p = true;
7325 : /* See which defined values die here. */
7326 62819210 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7327 40074611 : if (reg->type == OP_OUT && ! reg->subreg_p)
7328 15697225 : bitmap_clear_bit (&live_regs, reg->regno);
7329 27050284 : for (reg = curr_static_id->hard_regs; reg != NULL; reg = reg->next)
7330 4305685 : if (reg->type == OP_OUT && ! reg->subreg_p)
7331 3262694 : bitmap_clear_bit (&live_regs, reg->regno);
7332 22744599 : if (curr_id->arg_hard_regs != NULL)
7333 : /* Make clobbered argument hard registers die. */
7334 3275874 : for (i = 0; (regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7335 2295028 : if (regno >= FIRST_PSEUDO_REGISTER)
7336 0 : bitmap_clear_bit (&live_regs, regno - FIRST_PSEUDO_REGISTER);
7337 : /* Mark each used value as live. */
7338 62819210 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7339 40074611 : if (reg->type != OP_OUT
7340 40074611 : && bitmap_bit_p (&check_only_regs, reg->regno))
7341 4672820 : bitmap_set_bit (&live_regs, reg->regno);
7342 27050284 : for (reg = curr_static_id->hard_regs; reg != NULL; reg = reg->next)
7343 4305685 : if (reg->type != OP_OUT
7344 4305685 : && bitmap_bit_p (&check_only_regs, reg->regno))
7345 0 : bitmap_set_bit (&live_regs, reg->regno);
7346 22744599 : if (curr_id->arg_hard_regs != NULL)
7347 : /* Make used argument hard registers live. */
7348 3275874 : for (i = 0; (regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7349 2295028 : if (regno < FIRST_PSEUDO_REGISTER
7350 2295028 : && bitmap_bit_p (&check_only_regs, regno))
7351 0 : bitmap_set_bit (&live_regs, regno);
7352 : /* It is quite important to remove dead move insns because it
7353 : means removing dead store. We don't need to process them for
7354 : constraints. */
7355 22744599 : if (remove_p)
7356 : {
7357 308278 : if (lra_dump_file != NULL)
7358 : {
7359 2 : fprintf (lra_dump_file, " Removing dead insn:\n ");
7360 2 : dump_insn_slim (lra_dump_file, curr_insn);
7361 : }
7362 308278 : lra_set_insn_deleted (curr_insn);
7363 : }
7364 : }
7365 758848 : }
7366 :
7367 : /* The structure describes info to do an inheritance for the current
7368 : insn. We need to collect such info first before doing the
7369 : transformations because the transformations change the insn
7370 : internal representation. */
7371 : struct to_inherit
7372 : {
7373 : /* Original regno. */
7374 : int regno;
7375 : /* Subsequent insns which can inherit original reg value. */
7376 : rtx insns;
7377 : };
7378 :
7379 : /* Array containing all info for doing inheritance from the current
7380 : insn. */
7381 : static struct to_inherit to_inherit[LRA_MAX_INSN_RELOADS];
7382 :
7383 : /* Number elements in the previous array. */
7384 : static int to_inherit_num;
7385 :
7386 : /* Add inheritance info REGNO and INSNS. Their meaning is described in
7387 : structure to_inherit. */
7388 : static void
7389 322720 : add_to_inherit (int regno, rtx insns)
7390 : {
7391 322720 : int i;
7392 :
7393 322803 : for (i = 0; i < to_inherit_num; i++)
7394 83 : if (to_inherit[i].regno == regno)
7395 : return;
7396 322720 : lra_assert (to_inherit_num < LRA_MAX_INSN_RELOADS);
7397 322720 : to_inherit[to_inherit_num].regno = regno;
7398 322720 : to_inherit[to_inherit_num++].insns = insns;
7399 : }
7400 :
7401 : /* Return the last non-debug insn in basic block BB, or the block begin
7402 : note if none. */
7403 : static rtx_insn *
7404 30632996 : get_last_insertion_point (basic_block bb)
7405 : {
7406 30632996 : rtx_insn *insn;
7407 :
7408 33309642 : FOR_BB_INSNS_REVERSE (bb, insn)
7409 33309642 : if (NONDEBUG_INSN_P (insn) || NOTE_INSN_BASIC_BLOCK_P (insn))
7410 30632996 : return insn;
7411 0 : gcc_unreachable ();
7412 : }
7413 :
7414 : /* Set up RES by registers living on edges FROM except the edge (FROM,
7415 : TO) or by registers set up in a jump insn in BB FROM. */
7416 : static void
7417 11827558 : get_live_on_other_edges (basic_block from, basic_block to, bitmap res)
7418 : {
7419 11827558 : rtx_insn *last;
7420 11827558 : struct lra_insn_reg *reg;
7421 11827558 : edge e;
7422 11827558 : edge_iterator ei;
7423 :
7424 11827558 : lra_assert (to != NULL);
7425 11827558 : bitmap_clear (res);
7426 35203605 : FOR_EACH_EDGE (e, ei, from->succs)
7427 23376047 : if (e->dest != to)
7428 11548489 : bitmap_ior_into (res, df_get_live_in (e->dest));
7429 11827558 : last = get_last_insertion_point (from);
7430 11827558 : if (! JUMP_P (last))
7431 1962706 : return;
7432 9864852 : curr_id = lra_get_insn_recog_data (last);
7433 19729396 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7434 9864544 : if (reg->type != OP_IN)
7435 76 : bitmap_set_bit (res, reg->regno);
7436 : }
7437 :
7438 : /* Used as a temporary results of some bitmap calculations. */
7439 : static bitmap_head temp_bitmap;
7440 :
7441 : /* We split for reloads of small class of hard regs. The following
7442 : defines how many hard regs the class should have to be qualified as
7443 : small. The code is mostly oriented to x86/x86-64 architecture
7444 : where some insns need to use only specific register or pair of
7445 : registers and these register can live in RTL explicitly, e.g. for
7446 : parameter passing. */
7447 : static const int max_small_class_regs_num = 2;
7448 :
7449 : /* Do inheritance/split transformations in EBB starting with HEAD and
7450 : finishing on TAIL. We process EBB insns in the reverse order.
7451 : Return true if we did any inheritance/split transformation in the
7452 : EBB.
7453 :
7454 : We should avoid excessive splitting which results in worse code
7455 : because of inaccurate cost calculations for spilling new split
7456 : pseudos in such case. To achieve this we do splitting only if
7457 : register pressure is high in given basic block and there are reload
7458 : pseudos requiring hard registers. We could do more register
7459 : pressure calculations at any given program point to avoid necessary
7460 : splitting even more but it is to expensive and the current approach
7461 : works well enough. */
7462 : static bool
7463 12891784 : inherit_in_ebb (rtx_insn *head, rtx_insn *tail)
7464 : {
7465 12891784 : int i, src_regno, dst_regno, nregs;
7466 12891784 : bool change_p, succ_p, update_reloads_num_p;
7467 12891784 : rtx_insn *prev_insn, *last_insn;
7468 12891784 : rtx next_usage_insns, curr_set;
7469 12891784 : enum reg_class cl;
7470 12891784 : struct lra_insn_reg *reg;
7471 12891784 : basic_block last_processed_bb, curr_bb = NULL;
7472 12891784 : HARD_REG_SET potential_reload_hard_regs, live_hard_regs;
7473 12891784 : bitmap to_process;
7474 12891784 : unsigned int j;
7475 12891784 : bitmap_iterator bi;
7476 12891784 : bool head_p, after_p;
7477 :
7478 12891784 : change_p = false;
7479 12891784 : curr_usage_insns_check++;
7480 12891784 : clear_invariants ();
7481 12891784 : reloads_num = calls_num = 0;
7482 167593192 : for (unsigned int i = 0; i < NUM_ABI_IDS; ++i)
7483 154701408 : last_call_for_abi[i] = 0;
7484 12891784 : CLEAR_HARD_REG_SET (full_and_partial_call_clobbers);
7485 12891784 : bitmap_clear (&check_only_regs);
7486 12891784 : bitmap_clear (&invalid_invariant_regs);
7487 12891784 : last_processed_bb = NULL;
7488 12891784 : CLEAR_HARD_REG_SET (potential_reload_hard_regs);
7489 12891784 : live_hard_regs = eliminable_regset | lra_no_alloc_regs;
7490 : /* We don't process new insns generated in the loop. */
7491 245585188 : for (curr_insn = tail; curr_insn != PREV_INSN (head); curr_insn = prev_insn)
7492 : {
7493 232693404 : prev_insn = PREV_INSN (curr_insn);
7494 232693404 : if (BLOCK_FOR_INSN (curr_insn) != NULL)
7495 232693154 : curr_bb = BLOCK_FOR_INSN (curr_insn);
7496 232693404 : if (last_processed_bb != curr_bb)
7497 : {
7498 : /* We are at the end of BB. Add qualified living
7499 : pseudos for potential splitting. */
7500 18805438 : to_process = df_get_live_out (curr_bb);
7501 18805438 : if (last_processed_bb != NULL)
7502 : {
7503 : /* We are somewhere in the middle of EBB. */
7504 5913654 : get_live_on_other_edges (curr_bb, last_processed_bb,
7505 : &temp_bitmap);
7506 5913654 : to_process = &temp_bitmap;
7507 : }
7508 18805438 : last_processed_bb = curr_bb;
7509 18805438 : last_insn = get_last_insertion_point (curr_bb);
7510 37610876 : after_p = (! JUMP_P (last_insn)
7511 18805438 : && (! CALL_P (last_insn)
7512 2387392 : || (find_reg_note (last_insn,
7513 : REG_NORETURN, NULL_RTX) == NULL_RTX
7514 1421079 : && ! SIBLING_CALL_P (last_insn))));
7515 18805438 : CLEAR_HARD_REG_SET (potential_reload_hard_regs);
7516 203714929 : EXECUTE_IF_SET_IN_BITMAP (to_process, 0, j, bi)
7517 : {
7518 184909497 : if ((int) j >= lra_constraint_new_regno_start)
7519 : break;
7520 184909491 : if (j < FIRST_PSEUDO_REGISTER || reg_renumber[j] >= 0)
7521 : {
7522 120583229 : if (j < FIRST_PSEUDO_REGISTER)
7523 70583657 : SET_HARD_REG_BIT (live_hard_regs, j);
7524 : else
7525 49999572 : add_to_hard_reg_set (&live_hard_regs,
7526 49999572 : PSEUDO_REGNO_MODE (j),
7527 49999572 : reg_renumber[j]);
7528 120583229 : setup_next_usage_insn (j, last_insn, reloads_num, after_p);
7529 : }
7530 : }
7531 : }
7532 232693404 : src_regno = dst_regno = -1;
7533 232693404 : curr_set = single_set (curr_insn);
7534 232693404 : if (curr_set != NULL_RTX && REG_P (SET_DEST (curr_set)))
7535 85565066 : dst_regno = REGNO (SET_DEST (curr_set));
7536 115855297 : if (curr_set != NULL_RTX && REG_P (SET_SRC (curr_set)))
7537 40157905 : src_regno = REGNO (SET_SRC (curr_set));
7538 232693404 : update_reloads_num_p = true;
7539 232693404 : if (src_regno < lra_constraint_new_regno_start
7540 226083048 : && src_regno >= FIRST_PSEUDO_REGISTER
7541 28241595 : && reg_renumber[src_regno] < 0
7542 3764214 : && dst_regno >= lra_constraint_new_regno_start
7543 235373210 : && (cl = lra_get_allocno_class (dst_regno)) != NO_REGS)
7544 : {
7545 : /* 'reload_pseudo <- original_pseudo'. */
7546 2679806 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7547 28282 : reloads_num++;
7548 2679806 : update_reloads_num_p = false;
7549 2679806 : succ_p = false;
7550 2679806 : if (usage_insns[src_regno].check == curr_usage_insns_check
7551 2679806 : && (next_usage_insns = usage_insns[src_regno].insns) != NULL_RTX)
7552 489320 : succ_p = inherit_reload_reg (false, src_regno, cl,
7553 : curr_insn, next_usage_insns);
7554 489320 : if (succ_p)
7555 : change_p = true;
7556 : else
7557 2214222 : setup_next_usage_insn (src_regno, curr_insn, reloads_num, false);
7558 5359612 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7559 655250274 : potential_reload_hard_regs |= reg_class_contents[cl];
7560 : }
7561 230013598 : else if (src_regno < 0
7562 192535499 : && dst_regno >= lra_constraint_new_regno_start
7563 5805703 : && invariant_p (SET_SRC (curr_set))
7564 300796 : && (cl = lra_get_allocno_class (dst_regno)) != NO_REGS
7565 300249 : && ! bitmap_bit_p (&invalid_invariant_regs, dst_regno)
7566 230272118 : && ! bitmap_bit_p (&invalid_invariant_regs,
7567 258520 : ORIGINAL_REGNO(regno_reg_rtx[dst_regno])))
7568 : {
7569 : /* 'reload_pseudo <- invariant'. */
7570 196922 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7571 10795 : reloads_num++;
7572 196922 : update_reloads_num_p = false;
7573 196922 : if (process_invariant_for_inheritance (SET_DEST (curr_set), SET_SRC (curr_set)))
7574 25282 : change_p = true;
7575 393844 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7576 655250274 : potential_reload_hard_regs |= reg_class_contents[cl];
7577 : }
7578 229816676 : else if (src_regno >= lra_constraint_new_regno_start
7579 6610356 : && dst_regno < lra_constraint_new_regno_start
7580 5789707 : && dst_regno >= FIRST_PSEUDO_REGISTER
7581 3833461 : && reg_renumber[dst_regno] < 0
7582 1470389 : && (cl = lra_get_allocno_class (src_regno)) != NO_REGS
7583 1470389 : && usage_insns[dst_regno].check == curr_usage_insns_check
7584 229816676 : && (next_usage_insns
7585 497243 : = usage_insns[dst_regno].insns) != NULL_RTX)
7586 : {
7587 497243 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7588 10265 : reloads_num++;
7589 497243 : update_reloads_num_p = false;
7590 : /* 'original_pseudo <- reload_pseudo'. */
7591 497243 : if (! JUMP_P (curr_insn)
7592 497243 : && inherit_reload_reg (true, dst_regno, cl,
7593 : curr_insn, next_usage_insns))
7594 : change_p = true;
7595 : /* Invalidate. */
7596 497243 : usage_insns[dst_regno].check = 0;
7597 994486 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7598 655250274 : potential_reload_hard_regs |= reg_class_contents[cl];
7599 : }
7600 229319433 : else if (INSN_P (curr_insn))
7601 : {
7602 193167015 : int iter;
7603 193167015 : int max_uid = get_max_uid ();
7604 :
7605 193167015 : curr_id = lra_get_insn_recog_data (curr_insn);
7606 193167015 : curr_static_id = curr_id->insn_static_data;
7607 193167015 : to_inherit_num = 0;
7608 : /* Process insn definitions. */
7609 579501045 : for (iter = 0; iter < 2; iter++)
7610 386334030 : for (reg = iter == 0 ? curr_id->regs : curr_static_id->hard_regs;
7611 618888398 : reg != NULL;
7612 232554368 : reg = reg->next)
7613 232554368 : if (reg->type != OP_IN
7614 232554368 : && (dst_regno = reg->regno) < lra_constraint_new_regno_start)
7615 : {
7616 46634557 : if (dst_regno >= FIRST_PSEUDO_REGISTER && reg->type == OP_OUT
7617 44467160 : && reg_renumber[dst_regno] < 0 && ! reg->subreg_p
7618 1813576 : && usage_insns[dst_regno].check == curr_usage_insns_check
7619 92700301 : && (next_usage_insns
7620 129448 : = usage_insns[dst_regno].insns) != NULL_RTX)
7621 : {
7622 129448 : struct lra_insn_reg *r;
7623 :
7624 386432 : for (r = curr_id->regs; r != NULL; r = r->next)
7625 256984 : if (r->type != OP_OUT && r->regno == dst_regno)
7626 : break;
7627 : /* Don't do inheritance if the pseudo is also
7628 : used in the insn. */
7629 129448 : if (r == NULL)
7630 : /* We cannot do inheritance right now
7631 : because the current insn reg info (chain
7632 : regs) can change after that. */
7633 129448 : add_to_inherit (dst_regno, next_usage_insns);
7634 : }
7635 : /* We cannot process one reg twice here because of
7636 : usage_insns invalidation. */
7637 92700301 : if ((dst_regno < FIRST_PSEUDO_REGISTER
7638 46634557 : || reg_renumber[dst_regno] >= 0)
7639 90746615 : && ! reg->subreg_p && reg->type != OP_IN)
7640 : {
7641 90457202 : HARD_REG_SET s;
7642 :
7643 90457202 : if (split_if_necessary (dst_regno, reg->biggest_mode,
7644 : potential_reload_hard_regs,
7645 : false, curr_insn, max_uid))
7646 58212 : change_p = true;
7647 90457202 : CLEAR_HARD_REG_SET (s);
7648 90457202 : if (dst_regno < FIRST_PSEUDO_REGISTER)
7649 46065744 : add_to_hard_reg_set (&s, reg->biggest_mode, dst_regno);
7650 : else
7651 44391458 : add_to_hard_reg_set (&s, PSEUDO_REGNO_MODE (dst_regno),
7652 44391458 : reg_renumber[dst_regno]);
7653 90457202 : live_hard_regs &= ~s;
7654 180914404 : potential_reload_hard_regs &= ~s;
7655 : }
7656 : /* We should invalidate potential inheritance or
7657 : splitting for the current insn usages to the next
7658 : usage insns (see code below) as the output pseudo
7659 : prevents this. */
7660 92700301 : if ((dst_regno >= FIRST_PSEUDO_REGISTER
7661 46634557 : && reg_renumber[dst_regno] < 0)
7662 90746615 : || (reg->type == OP_OUT && ! reg->subreg_p
7663 82682732 : && (dst_regno < FIRST_PSEUDO_REGISTER
7664 42232473 : || reg_renumber[dst_regno] >= 0)))
7665 : {
7666 : /* Invalidate and mark definitions. */
7667 44186159 : if (dst_regno >= FIRST_PSEUDO_REGISTER)
7668 44186159 : usage_insns[dst_regno].check = -(int) INSN_UID (curr_insn);
7669 : else
7670 : {
7671 40450259 : nregs = hard_regno_nregs (dst_regno,
7672 40450259 : reg->biggest_mode);
7673 81164692 : for (i = 0; i < nregs; i++)
7674 81428866 : usage_insns[dst_regno + i].check
7675 40714433 : = -(int) INSN_UID (curr_insn);
7676 : }
7677 : }
7678 : }
7679 : /* Process clobbered call regs. */
7680 193167015 : if (curr_id->arg_hard_regs != NULL)
7681 19636418 : for (i = 0; (dst_regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7682 13746470 : if (dst_regno >= FIRST_PSEUDO_REGISTER)
7683 0 : usage_insns[dst_regno - FIRST_PSEUDO_REGISTER].check
7684 0 : = -(int) INSN_UID (curr_insn);
7685 193167015 : if (! JUMP_P (curr_insn))
7686 181553999 : for (i = 0; i < to_inherit_num; i++)
7687 129448 : if (inherit_reload_reg (true, to_inherit[i].regno,
7688 : ALL_REGS, curr_insn,
7689 : to_inherit[i].insns))
7690 102848 : change_p = true;
7691 193167015 : if (CALL_P (curr_insn))
7692 : {
7693 7474543 : rtx cheap, pat, dest;
7694 7474543 : rtx_insn *restore;
7695 7474543 : int regno, hard_regno;
7696 :
7697 7474543 : calls_num++;
7698 7474543 : function_abi callee_abi = insn_callee_abi (curr_insn);
7699 7474543 : last_call_for_abi[callee_abi.id ()] = calls_num;
7700 7474543 : full_and_partial_call_clobbers
7701 7474543 : |= callee_abi.full_and_partial_reg_clobbers ();
7702 7474543 : first_call_insn = curr_insn;
7703 7474543 : if ((cheap = find_reg_note (curr_insn,
7704 : REG_RETURNED, NULL_RTX)) != NULL_RTX
7705 45365 : && ((cheap = XEXP (cheap, 0)), true)
7706 45365 : && (regno = REGNO (cheap)) >= FIRST_PSEUDO_REGISTER
7707 45365 : && (hard_regno = reg_renumber[regno]) >= 0
7708 34633 : && usage_insns[regno].check == curr_usage_insns_check
7709 : /* If there are pending saves/restores, the
7710 : optimization is not worth. */
7711 29554 : && usage_insns[regno].calls_num == calls_num - 1
7712 7502299 : && callee_abi.clobbers_reg_p (GET_MODE (cheap), hard_regno))
7713 : {
7714 : /* Restore the pseudo from the call result as
7715 : REG_RETURNED note says that the pseudo value is
7716 : in the call result and the pseudo is an argument
7717 : of the call. */
7718 12426 : pat = PATTERN (curr_insn);
7719 12426 : if (GET_CODE (pat) == PARALLEL)
7720 0 : pat = XVECEXP (pat, 0, 0);
7721 12426 : dest = SET_DEST (pat);
7722 : /* For multiple return values dest is PARALLEL.
7723 : Currently we handle only single return value case. */
7724 12426 : if (REG_P (dest))
7725 : {
7726 12426 : start_sequence ();
7727 12426 : emit_move_insn (cheap, copy_rtx (dest));
7728 12426 : restore = end_sequence ();
7729 12426 : lra_process_new_insns (curr_insn, NULL, restore,
7730 : "Inserting call parameter restore");
7731 : /* We don't need to save/restore of the pseudo from
7732 : this call. */
7733 12426 : usage_insns[regno].calls_num = calls_num;
7734 12426 : remove_from_hard_reg_set
7735 12426 : (&full_and_partial_call_clobbers,
7736 12426 : GET_MODE (cheap), hard_regno);
7737 12426 : bitmap_set_bit (&check_only_regs, regno);
7738 : }
7739 : }
7740 : }
7741 193167015 : to_inherit_num = 0;
7742 : /* Process insn usages. */
7743 579501045 : for (iter = 0; iter < 2; iter++)
7744 386334030 : for (reg = iter == 0 ? curr_id->regs : curr_static_id->hard_regs;
7745 618888398 : reg != NULL;
7746 232554368 : reg = reg->next)
7747 232554368 : if ((reg->type != OP_OUT
7748 91575092 : || (reg->type == OP_OUT && reg->subreg_p))
7749 233102294 : && (src_regno = reg->regno) < lra_constraint_new_regno_start)
7750 : {
7751 129716630 : if (src_regno >= FIRST_PSEUDO_REGISTER
7752 74963401 : && reg_renumber[src_regno] < 0 && reg->type == OP_IN)
7753 : {
7754 2550605 : if (usage_insns[src_regno].check == curr_usage_insns_check
7755 833955 : && (next_usage_insns
7756 833955 : = usage_insns[src_regno].insns) != NULL_RTX
7757 3384560 : && NONDEBUG_INSN_P (curr_insn))
7758 193272 : add_to_inherit (src_regno, next_usage_insns);
7759 4714666 : else if (usage_insns[src_regno].check
7760 2357333 : != -(int) INSN_UID (curr_insn))
7761 : /* Add usages but only if the reg is not set up
7762 : in the same insn. */
7763 2357333 : add_next_usage_insn (src_regno, curr_insn, reloads_num);
7764 : }
7765 72412796 : else if (src_regno < FIRST_PSEUDO_REGISTER
7766 72412796 : || reg_renumber[src_regno] >= 0)
7767 : {
7768 127025915 : bool before_p;
7769 127025915 : rtx_insn *use_insn = curr_insn;
7770 127025915 : rtx_insn *prev_insn = PREV_INSN (curr_insn);
7771 :
7772 254051830 : before_p = (JUMP_P (curr_insn)
7773 127025915 : || (CALL_P (curr_insn) && reg->type == OP_IN));
7774 127025915 : if (NONDEBUG_INSN_P (curr_insn)
7775 112918824 : && (! JUMP_P (curr_insn) || reg->type == OP_IN)
7776 239944634 : && split_if_necessary (src_regno, reg->biggest_mode,
7777 : potential_reload_hard_regs,
7778 : before_p, curr_insn, max_uid))
7779 : {
7780 225508 : if (reg->subreg_p)
7781 3179 : check_and_force_assignment_correctness_p = true;
7782 225508 : change_p = true;
7783 : /* Invalidate. */
7784 225508 : usage_insns[src_regno].check = 0;
7785 225508 : if (before_p && PREV_INSN (curr_insn) != prev_insn)
7786 : use_insn = PREV_INSN (curr_insn);
7787 : }
7788 127025915 : if (NONDEBUG_INSN_P (curr_insn))
7789 : {
7790 112918824 : if (src_regno < FIRST_PSEUDO_REGISTER)
7791 49664074 : add_to_hard_reg_set (&live_hard_regs,
7792 49664074 : reg->biggest_mode, src_regno);
7793 : else
7794 63254750 : add_to_hard_reg_set (&live_hard_regs,
7795 63254750 : PSEUDO_REGNO_MODE (src_regno),
7796 63254750 : reg_renumber[src_regno]);
7797 : }
7798 127025915 : if (src_regno >= FIRST_PSEUDO_REGISTER)
7799 72272686 : add_next_usage_insn (src_regno, use_insn, reloads_num);
7800 : else
7801 : {
7802 109605825 : for (i = 0; i < hard_regno_nregs (src_regno, reg->biggest_mode); i++)
7803 54852596 : add_next_usage_insn (src_regno + i, use_insn, reloads_num);
7804 : }
7805 : }
7806 : }
7807 : /* Process used call regs. */
7808 193167015 : if (curr_id->arg_hard_regs != NULL)
7809 19636418 : for (i = 0; (src_regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7810 13746470 : if (src_regno < FIRST_PSEUDO_REGISTER)
7811 : {
7812 13746470 : SET_HARD_REG_BIT (live_hard_regs, src_regno);
7813 13746470 : add_next_usage_insn (src_regno, curr_insn, reloads_num);
7814 : }
7815 193360287 : for (i = 0; i < to_inherit_num; i++)
7816 : {
7817 193272 : src_regno = to_inherit[i].regno;
7818 193272 : if (inherit_reload_reg (false, src_regno, ALL_REGS,
7819 : curr_insn, to_inherit[i].insns))
7820 : change_p = true;
7821 : else
7822 22632 : setup_next_usage_insn (src_regno, curr_insn, reloads_num, false);
7823 : }
7824 : }
7825 193237437 : if (update_reloads_num_p
7826 229319433 : && NONDEBUG_INSN_P (curr_insn) && curr_set != NULL_RTX)
7827 : {
7828 112481326 : int regno = -1;
7829 112481326 : if ((REG_P (SET_DEST (curr_set))
7830 82191095 : && (regno = REGNO (SET_DEST (curr_set))) >= lra_constraint_new_regno_start
7831 8497581 : && reg_renumber[regno] < 0
7832 5437930 : && (cl = lra_get_allocno_class (regno)) != NO_REGS)
7833 189512316 : || (REG_P (SET_SRC (curr_set))
7834 35660279 : && (regno = REGNO (SET_SRC (curr_set))) >= lra_constraint_new_regno_start
7835 6183825 : && reg_renumber[regno] < 0
7836 3638392 : && (cl = lra_get_allocno_class (regno)) != NO_REGS))
7837 : {
7838 8532940 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7839 252938 : reloads_num++;
7840 17065880 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7841 232693404 : potential_reload_hard_regs |= reg_class_contents[cl];
7842 : }
7843 : }
7844 232693404 : if (NONDEBUG_INSN_P (curr_insn))
7845 : {
7846 122319252 : int regno;
7847 :
7848 : /* Invalidate invariants with changed regs. */
7849 122319252 : curr_id = lra_get_insn_recog_data (curr_insn);
7850 313835626 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7851 191516374 : if (reg->type != OP_IN)
7852 : {
7853 82066606 : bitmap_set_bit (&invalid_invariant_regs, reg->regno);
7854 164133212 : bitmap_set_bit (&invalid_invariant_regs,
7855 82066606 : ORIGINAL_REGNO (regno_reg_rtx[reg->regno]));
7856 : }
7857 122319252 : curr_static_id = curr_id->insn_static_data;
7858 154497460 : for (reg = curr_static_id->hard_regs; reg != NULL; reg = reg->next)
7859 32178208 : if (reg->type != OP_IN)
7860 23032354 : bitmap_set_bit (&invalid_invariant_regs, reg->regno);
7861 122319252 : if (curr_id->arg_hard_regs != NULL)
7862 19636418 : for (i = 0; (regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7863 13746470 : if (regno >= FIRST_PSEUDO_REGISTER)
7864 0 : bitmap_set_bit (&invalid_invariant_regs,
7865 : regno - FIRST_PSEUDO_REGISTER);
7866 : }
7867 : /* We reached the start of the current basic block. */
7868 232693396 : if (prev_insn == NULL_RTX || prev_insn == PREV_INSN (head)
7869 452495024 : || BLOCK_FOR_INSN (prev_insn) != curr_bb)
7870 : {
7871 : /* We reached the beginning of the current block -- do
7872 : rest of splitting in the current BB. */
7873 18805688 : to_process = df_get_live_in (curr_bb);
7874 18805688 : if (BLOCK_FOR_INSN (head) != curr_bb)
7875 : {
7876 : /* We are somewhere in the middle of EBB. */
7877 5913904 : get_live_on_other_edges (EDGE_PRED (curr_bb, 0)->src,
7878 : curr_bb, &temp_bitmap);
7879 5913904 : to_process = &temp_bitmap;
7880 : }
7881 18805688 : head_p = true;
7882 196820560 : EXECUTE_IF_SET_IN_BITMAP (to_process, 0, j, bi)
7883 : {
7884 178014879 : if ((int) j >= lra_constraint_new_regno_start)
7885 : break;
7886 113328898 : if (((int) j < FIRST_PSEUDO_REGISTER || reg_renumber[j] >= 0)
7887 112590089 : && usage_insns[j].check == curr_usage_insns_check
7888 288685124 : && (next_usage_insns = usage_insns[j].insns) != NULL_RTX)
7889 : {
7890 110670252 : if (need_for_split_p (potential_reload_hard_regs, j))
7891 : {
7892 386211 : if (lra_dump_file != NULL && head_p)
7893 : {
7894 0 : fprintf (lra_dump_file,
7895 : " ----------------------------------\n");
7896 0 : head_p = false;
7897 : }
7898 386211 : if (split_reg (false, j, bb_note (curr_bb),
7899 : next_usage_insns, NULL))
7900 386211 : change_p = true;
7901 : }
7902 110670252 : usage_insns[j].check = 0;
7903 : }
7904 : }
7905 : }
7906 : }
7907 12891784 : first_call_insn = NULL;
7908 12891784 : return change_p;
7909 : }
7910 :
7911 : /* This value affects EBB forming. If probability of edge from EBB to
7912 : a BB is not greater than the following value, we don't add the BB
7913 : to EBB. */
7914 : #define EBB_PROBABILITY_CUTOFF \
7915 : ((REG_BR_PROB_BASE * param_lra_inheritance_ebb_probability_cutoff) / 100)
7916 :
7917 : /* Current number of inheritance/split iteration. */
7918 : int lra_inheritance_iter;
7919 :
7920 : /* Entry function for inheritance/split pass. */
7921 : void
7922 1582013 : lra_inheritance (void)
7923 : {
7924 1582013 : int i;
7925 1582013 : basic_block bb, start_bb;
7926 1582013 : edge e;
7927 :
7928 1582013 : lra_inheritance_iter++;
7929 1582013 : if (lra_inheritance_iter > LRA_MAX_INHERITANCE_PASSES)
7930 : return;
7931 1578923 : timevar_push (TV_LRA_INHERITANCE);
7932 1578923 : if (lra_dump_file != NULL)
7933 97 : fprintf (lra_dump_file, "\n********** Inheritance #%d: **********\n\n",
7934 : lra_inheritance_iter);
7935 1578923 : curr_usage_insns_check = 0;
7936 1578923 : usage_insns = XNEWVEC (struct usage_insns, lra_constraint_new_regno_start);
7937 234609434 : for (i = 0; i < lra_constraint_new_regno_start; i++)
7938 233030511 : usage_insns[i].check = 0;
7939 1578923 : bitmap_initialize (&check_only_regs, ®_obstack);
7940 1578923 : bitmap_initialize (&invalid_invariant_regs, ®_obstack);
7941 1578923 : bitmap_initialize (&live_regs, ®_obstack);
7942 1578923 : bitmap_initialize (&temp_bitmap, ®_obstack);
7943 1578923 : bitmap_initialize (&ebb_global_regs, ®_obstack);
7944 14470707 : FOR_EACH_BB_FN (bb, cfun)
7945 : {
7946 12891784 : start_bb = bb;
7947 12891784 : if (lra_dump_file != NULL)
7948 347 : fprintf (lra_dump_file, "EBB");
7949 : /* Form a EBB starting with BB. */
7950 12891784 : bitmap_clear (&ebb_global_regs);
7951 12891784 : bitmap_ior_into (&ebb_global_regs, df_get_live_in (bb));
7952 24719092 : for (;;)
7953 : {
7954 18805438 : if (lra_dump_file != NULL)
7955 477 : fprintf (lra_dump_file, " %d", bb->index);
7956 18805438 : if (bb->next_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
7957 17226515 : || LABEL_P (BB_HEAD (bb->next_bb)))
7958 : break;
7959 8313477 : e = find_fallthru_edge (bb->succs);
7960 8313477 : if (! e)
7961 : break;
7962 8313477 : if (e->probability.initialized_p ()
7963 8313477 : && e->probability.to_reg_br_prob_base () < EBB_PROBABILITY_CUTOFF)
7964 : break;
7965 5913654 : bb = bb->next_bb;
7966 : }
7967 12891784 : bitmap_ior_into (&ebb_global_regs, df_get_live_out (bb));
7968 12891784 : if (lra_dump_file != NULL)
7969 347 : fprintf (lra_dump_file, "\n");
7970 12891784 : if (inherit_in_ebb (BB_HEAD (start_bb), BB_END (bb)))
7971 : /* Remember that the EBB head and tail can change in
7972 : inherit_in_ebb. */
7973 758848 : update_ebb_live_info (BB_HEAD (start_bb), BB_END (bb));
7974 : }
7975 1578923 : bitmap_release (&ebb_global_regs);
7976 1578923 : bitmap_release (&temp_bitmap);
7977 1578923 : bitmap_release (&live_regs);
7978 1578923 : bitmap_release (&invalid_invariant_regs);
7979 1578923 : bitmap_release (&check_only_regs);
7980 1578923 : free (usage_insns);
7981 1578923 : lra_dump_insns_if_possible ("func after inheritance");
7982 1578923 : timevar_pop (TV_LRA_INHERITANCE);
7983 : }
7984 :
7985 :
7986 :
7987 : /* This page contains code to undo failed inheritance/split
7988 : transformations. */
7989 :
7990 : /* Current number of iteration undoing inheritance/split. */
7991 : int lra_undo_inheritance_iter;
7992 :
7993 : /* Fix BB live info LIVE after removing pseudos created on pass doing
7994 : inheritance/split which are REMOVED_PSEUDOS. */
7995 : static void
7996 37610876 : fix_bb_live_info (bitmap live, bitmap removed_pseudos)
7997 : {
7998 37610876 : unsigned int regno;
7999 37610876 : bitmap_iterator bi;
8000 :
8001 215843462 : EXECUTE_IF_SET_IN_BITMAP (removed_pseudos, 0, regno, bi)
8002 178232586 : if (bitmap_clear_bit (live, regno)
8003 178232586 : && REG_P (lra_reg_info[regno].restore_rtx))
8004 1212606 : bitmap_set_bit (live, REGNO (lra_reg_info[regno].restore_rtx));
8005 37610876 : }
8006 :
8007 : /* Return regno of the (subreg of) REG. Otherwise, return a negative
8008 : number. */
8009 : static int
8010 68923557 : get_regno (rtx reg)
8011 : {
8012 1127095 : if (GET_CODE (reg) == SUBREG)
8013 1067089 : reg = SUBREG_REG (reg);
8014 68798288 : if (REG_P (reg))
8015 44701254 : return REGNO (reg);
8016 : return -1;
8017 : }
8018 :
8019 : /* Delete a move INSN with destination reg DREGNO and a previous
8020 : clobber insn with the same regno. The inheritance/split code can
8021 : generate moves with preceding clobber and when we delete such moves
8022 : we should delete the clobber insn too to keep the correct life
8023 : info. */
8024 : static void
8025 777161 : delete_move_and_clobber (rtx_insn *insn, int dregno)
8026 : {
8027 777161 : rtx_insn *prev_insn = PREV_INSN (insn);
8028 :
8029 777161 : lra_set_insn_deleted (insn);
8030 777161 : lra_assert (dregno >= 0);
8031 777161 : if (prev_insn != NULL && NONDEBUG_INSN_P (prev_insn)
8032 331791 : && GET_CODE (PATTERN (prev_insn)) == CLOBBER
8033 777603 : && dregno == get_regno (XEXP (PATTERN (prev_insn), 0)))
8034 0 : lra_set_insn_deleted (prev_insn);
8035 777161 : }
8036 :
8037 : /* Remove inheritance/split pseudos which are in REMOVE_PSEUDOS and
8038 : return true if we did any change. The undo transformations for
8039 : inheritance looks like
8040 : i <- i2
8041 : p <- i => p <- i2
8042 : or removing
8043 : p <- i, i <- p, and i <- i3
8044 : where p is original pseudo from which inheritance pseudo i was
8045 : created, i and i3 are removed inheritance pseudos, i2 is another
8046 : not removed inheritance pseudo. All split pseudos or other
8047 : occurrences of removed inheritance pseudos are changed on the
8048 : corresponding original pseudos.
8049 :
8050 : The function also schedules insns changed and created during
8051 : inheritance/split pass for processing by the subsequent constraint
8052 : pass. */
8053 : static bool
8054 1578923 : remove_inheritance_pseudos (bitmap remove_pseudos)
8055 : {
8056 1578923 : basic_block bb;
8057 1578923 : int regno, sregno, prev_sregno, dregno;
8058 1578923 : rtx restore_rtx;
8059 1578923 : rtx set, prev_set;
8060 1578923 : rtx_insn *prev_insn;
8061 1578923 : bool change_p, done_p;
8062 :
8063 1578923 : change_p = ! bitmap_empty_p (remove_pseudos);
8064 : /* We cannot finish the function right away if CHANGE_P is true
8065 : because we need to marks insns affected by previous
8066 : inheritance/split pass for processing by the subsequent
8067 : constraint pass. */
8068 20384361 : FOR_EACH_BB_FN (bb, cfun)
8069 : {
8070 18805438 : fix_bb_live_info (df_get_live_in (bb), remove_pseudos);
8071 18805438 : fix_bb_live_info (df_get_live_out (bb), remove_pseudos);
8072 254099032 : FOR_BB_INSNS_REVERSE (bb, curr_insn)
8073 : {
8074 235293594 : if (! INSN_P (curr_insn))
8075 36514793 : continue;
8076 198778801 : done_p = false;
8077 198778801 : sregno = dregno = -1;
8078 48631930 : if (change_p && NONDEBUG_INSN_P (curr_insn)
8079 231235755 : && (set = single_set (curr_insn)) != NULL_RTX)
8080 : {
8081 31430157 : dregno = get_regno (SET_DEST (set));
8082 62860314 : sregno = get_regno (SET_SRC (set));
8083 : }
8084 :
8085 198778801 : if (sregno >= 0 && dregno >= 0)
8086 : {
8087 11174462 : if (bitmap_bit_p (remove_pseudos, dregno)
8088 11174462 : && ! REG_P (lra_reg_info[dregno].restore_rtx))
8089 : {
8090 : /* invariant inheritance pseudo <- original pseudo */
8091 7750 : if (lra_dump_file != NULL)
8092 : {
8093 0 : fprintf (lra_dump_file, " Removing invariant inheritance:\n");
8094 0 : dump_insn_slim (lra_dump_file, curr_insn);
8095 0 : fprintf (lra_dump_file, "\n");
8096 : }
8097 7750 : delete_move_and_clobber (curr_insn, dregno);
8098 7750 : done_p = true;
8099 : }
8100 11166712 : else if (bitmap_bit_p (remove_pseudos, sregno)
8101 11166712 : && ! REG_P (lra_reg_info[sregno].restore_rtx))
8102 : {
8103 : /* reload pseudo <- invariant inheritance pseudo */
8104 7750 : start_sequence ();
8105 : /* We cannot just change the source. It might be
8106 : an insn different from the move. */
8107 7750 : emit_insn (lra_reg_info[sregno].restore_rtx);
8108 7750 : rtx_insn *new_insns = end_sequence ();
8109 7750 : lra_assert (single_set (new_insns) != NULL
8110 : && SET_DEST (set) == SET_DEST (single_set (new_insns)));
8111 7750 : lra_process_new_insns (curr_insn, NULL, new_insns,
8112 : "Changing reload<-invariant inheritance");
8113 7750 : delete_move_and_clobber (curr_insn, dregno);
8114 7750 : done_p = true;
8115 : }
8116 11158962 : else if ((bitmap_bit_p (remove_pseudos, sregno)
8117 1279404 : && (get_regno (lra_reg_info[sregno].restore_rtx) == dregno
8118 601360 : || (bitmap_bit_p (remove_pseudos, dregno)
8119 210056 : && get_regno (lra_reg_info[sregno].restore_rtx) >= 0
8120 210056 : && (get_regno (lra_reg_info[sregno].restore_rtx)
8121 210056 : == get_regno (lra_reg_info[dregno].restore_rtx)))))
8122 11655294 : || (bitmap_bit_p (remove_pseudos, dregno)
8123 671808 : && get_regno (lra_reg_info[dregno].restore_rtx) == sregno))
8124 : /* One of the following cases:
8125 : original <- removed inheritance pseudo
8126 : removed inherit pseudo <- another removed inherit pseudo
8127 : removed inherit pseudo <- original pseudo
8128 : Or
8129 : removed_split_pseudo <- original_reg
8130 : original_reg <- removed_split_pseudo */
8131 : {
8132 190231 : if (lra_dump_file != NULL)
8133 : {
8134 0 : fprintf (lra_dump_file, " Removing %s:\n",
8135 0 : bitmap_bit_p (&lra_split_regs, sregno)
8136 0 : || bitmap_bit_p (&lra_split_regs, dregno)
8137 : ? "split" : "inheritance");
8138 0 : dump_insn_slim (lra_dump_file, curr_insn);
8139 : }
8140 190231 : delete_move_and_clobber (curr_insn, dregno);
8141 190231 : done_p = true;
8142 : }
8143 10968731 : else if (bitmap_bit_p (remove_pseudos, sregno)
8144 10968731 : && bitmap_bit_p (&lra_inheritance_pseudos, sregno))
8145 : {
8146 : /* Search the following pattern:
8147 : inherit_or_split_pseudo1 <- inherit_or_split_pseudo2
8148 : original_pseudo <- inherit_or_split_pseudo1
8149 : where the 2nd insn is the current insn and
8150 : inherit_or_split_pseudo2 is not removed. If it is found,
8151 : change the current insn onto:
8152 : original_pseudo <- inherit_or_split_pseudo2. */
8153 775961 : for (prev_insn = PREV_INSN (curr_insn);
8154 775961 : prev_insn != NULL_RTX && ! NONDEBUG_INSN_P (prev_insn);
8155 279629 : prev_insn = PREV_INSN (prev_insn))
8156 : ;
8157 496332 : if (prev_insn != NULL_RTX && BLOCK_FOR_INSN (prev_insn) == bb
8158 482934 : && (prev_set = single_set (prev_insn)) != NULL_RTX
8159 : /* There should be no subregs in insn we are
8160 : searching because only the original reg might
8161 : be in subreg when we changed the mode of
8162 : load/store for splitting. */
8163 477230 : && REG_P (SET_DEST (prev_set))
8164 373074 : && REG_P (SET_SRC (prev_set))
8165 287284 : && (int) REGNO (SET_DEST (prev_set)) == sregno
8166 195157 : && ((prev_sregno = REGNO (SET_SRC (prev_set)))
8167 : >= FIRST_PSEUDO_REGISTER)
8168 195157 : && (lra_reg_info[prev_sregno].restore_rtx == NULL_RTX
8169 155661 : ||
8170 : /* As we consider chain of inheritance or
8171 : splitting described in above comment we should
8172 : check that sregno and prev_sregno were
8173 : inheritance/split pseudos created from the
8174 : same original regno. */
8175 311322 : (get_regno (lra_reg_info[sregno].restore_rtx) >= 0
8176 311322 : && (get_regno (lra_reg_info[sregno].restore_rtx)
8177 311322 : == get_regno (lra_reg_info[prev_sregno].restore_rtx))))
8178 691489 : && ! bitmap_bit_p (remove_pseudos, prev_sregno))
8179 : {
8180 104913 : int restore_regno = get_regno (lra_reg_info[sregno].restore_rtx);
8181 104913 : if (restore_regno < 0)
8182 0 : restore_regno = prev_sregno;
8183 104913 : lra_assert (GET_MODE (SET_SRC (prev_set))
8184 : == GET_MODE (regno_reg_rtx[restore_regno]));
8185 : /* Although we have a single set, the insn can
8186 : contain more one sregno register occurrence
8187 : as a source. Change all occurrences. */
8188 104913 : lra_substitute_pseudo_within_insn (curr_insn, sregno,
8189 : regno_reg_rtx[restore_regno],
8190 : false);
8191 : /* As we are finishing with processing the insn
8192 : here, check the destination too as it might
8193 : inheritance pseudo for another pseudo. */
8194 104913 : if (bitmap_bit_p (remove_pseudos, dregno)
8195 0 : && bitmap_bit_p (&lra_inheritance_pseudos, dregno)
8196 104913 : && (restore_rtx
8197 0 : = lra_reg_info[dregno].restore_rtx) != NULL_RTX)
8198 : {
8199 0 : if (GET_CODE (SET_DEST (set)) == SUBREG)
8200 0 : SUBREG_REG (SET_DEST (set)) = restore_rtx;
8201 : else
8202 0 : SET_DEST (set) = restore_rtx;
8203 : }
8204 104913 : lra_push_insn_and_update_insn_regno_info (curr_insn);
8205 104913 : lra_set_used_insn_alternative_by_uid
8206 104913 : (INSN_UID (curr_insn), LRA_UNKNOWN_ALT);
8207 104913 : done_p = true;
8208 104913 : if (lra_dump_file != NULL)
8209 : {
8210 0 : fprintf (lra_dump_file, " Change reload insn:\n");
8211 0 : dump_insn_slim (lra_dump_file, curr_insn);
8212 : }
8213 : }
8214 : }
8215 : }
8216 205731 : if (! done_p)
8217 : {
8218 198468157 : struct lra_insn_reg *reg;
8219 198468157 : bool restored_regs_p = false;
8220 198468157 : bool kept_regs_p = false;
8221 :
8222 198468157 : curr_id = lra_get_insn_recog_data (curr_insn);
8223 409294517 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
8224 : {
8225 210826360 : regno = reg->regno;
8226 210826360 : restore_rtx = lra_reg_info[regno].restore_rtx;
8227 210826360 : if (restore_rtx != NULL_RTX)
8228 : {
8229 6084388 : if (change_p && bitmap_bit_p (remove_pseudos, regno))
8230 : {
8231 878237 : lra_substitute_pseudo_within_insn
8232 878237 : (curr_insn, regno, restore_rtx, false);
8233 878237 : restored_regs_p = true;
8234 : }
8235 : else
8236 : kept_regs_p = true;
8237 : }
8238 : }
8239 198468157 : if (NONDEBUG_INSN_P (curr_insn) && kept_regs_p)
8240 : {
8241 : /* The instruction has changed since the previous
8242 : constraints pass. */
8243 4565715 : lra_push_insn_and_update_insn_regno_info (curr_insn);
8244 4565715 : lra_set_used_insn_alternative_by_uid
8245 4565715 : (INSN_UID (curr_insn), LRA_UNKNOWN_ALT);
8246 : }
8247 193902442 : else if (restored_regs_p)
8248 : /* The instruction has been restored to the form that
8249 : it had during the previous constraints pass. */
8250 699883 : lra_update_insn_regno_info (curr_insn);
8251 5265598 : if (restored_regs_p && lra_dump_file != NULL)
8252 : {
8253 0 : fprintf (lra_dump_file, " Insn after restoring regs:\n");
8254 0 : dump_insn_slim (lra_dump_file, curr_insn);
8255 : }
8256 : }
8257 : }
8258 : }
8259 1578923 : return change_p;
8260 : }
8261 :
8262 : /* If optional reload pseudos failed to get a hard register or was not
8263 : inherited, it is better to remove optional reloads. We do this
8264 : transformation after undoing inheritance to figure out necessity to
8265 : remove optional reloads easier. Return true if we do any
8266 : change. */
8267 : static bool
8268 1578923 : undo_optional_reloads (void)
8269 : {
8270 1578923 : bool change_p, keep_p;
8271 1578923 : unsigned int regno, uid;
8272 1578923 : bitmap_iterator bi, bi2;
8273 1578923 : rtx_insn *insn;
8274 1578923 : rtx set, src, dest;
8275 1578923 : auto_bitmap removed_optional_reload_pseudos (®_obstack);
8276 :
8277 1578923 : bitmap_copy (removed_optional_reload_pseudos, &lra_optional_reload_pseudos);
8278 2611908 : EXECUTE_IF_SET_IN_BITMAP (&lra_optional_reload_pseudos, 0, regno, bi)
8279 : {
8280 1032985 : keep_p = false;
8281 : /* Keep optional reloads from previous subpasses. */
8282 1032985 : if (lra_reg_info[regno].restore_rtx == NULL_RTX
8283 : /* If the original pseudo changed its allocation, just
8284 : removing the optional pseudo is dangerous as the original
8285 : pseudo will have longer live range. */
8286 1032985 : || reg_renumber[REGNO (lra_reg_info[regno].restore_rtx)] >= 0)
8287 : keep_p = true;
8288 641175 : else if (reg_renumber[regno] >= 0)
8289 1843648 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi2)
8290 : {
8291 1341504 : insn = lra_insn_recog_data[uid]->insn;
8292 1341504 : if ((set = single_set (insn)) == NULL_RTX)
8293 5909 : continue;
8294 1335595 : src = SET_SRC (set);
8295 1335595 : dest = SET_DEST (set);
8296 1335595 : if ((! REG_P (src) && ! SUBREG_P (src))
8297 714115 : || (! REG_P (dest) && ! SUBREG_P (dest)))
8298 621504 : continue;
8299 714091 : if (get_regno (dest) == (int) regno
8300 : /* Ignore insn for optional reloads itself. */
8301 1202458 : && (get_regno (lra_reg_info[regno].restore_rtx)
8302 601229 : != get_regno (src))
8303 : /* Check only inheritance on last inheritance pass. */
8304 125269 : && get_regno (src) >= new_regno_start
8305 : /* Check that the optional reload was inherited. */
8306 839360 : && bitmap_bit_p (&lra_inheritance_pseudos, get_regno (src)))
8307 : {
8308 : keep_p = true;
8309 : break;
8310 : }
8311 : }
8312 652783 : if (keep_p)
8313 : {
8314 517079 : bitmap_clear_bit (removed_optional_reload_pseudos, regno);
8315 517079 : if (lra_dump_file != NULL)
8316 3 : fprintf (lra_dump_file, "Keep optional reload reg %d\n", regno);
8317 : }
8318 : }
8319 1578923 : change_p = ! bitmap_empty_p (removed_optional_reload_pseudos);
8320 1578923 : auto_bitmap insn_bitmap (®_obstack);
8321 2094829 : EXECUTE_IF_SET_IN_BITMAP (removed_optional_reload_pseudos, 0, regno, bi)
8322 : {
8323 515906 : if (lra_dump_file != NULL)
8324 2 : fprintf (lra_dump_file, "Remove optional reload reg %d\n", regno);
8325 515906 : bitmap_copy (insn_bitmap, &lra_reg_info[regno].insn_bitmap);
8326 1636664 : EXECUTE_IF_SET_IN_BITMAP (insn_bitmap, 0, uid, bi2)
8327 : {
8328 : /* We may have already removed a clobber. */
8329 1120758 : if (!lra_insn_recog_data[uid])
8330 0 : continue;
8331 1120758 : insn = lra_insn_recog_data[uid]->insn;
8332 1120758 : if ((set = single_set (insn)) != NULL_RTX)
8333 : {
8334 1115019 : src = SET_SRC (set);
8335 1115019 : dest = SET_DEST (set);
8336 510154 : if ((REG_P (src) || SUBREG_P (src))
8337 604877 : && (REG_P (dest) || SUBREG_P (dest))
8338 1719871 : && ((get_regno (src) == (int) regno
8339 231860 : && (get_regno (lra_reg_info[regno].restore_rtx)
8340 115930 : == get_regno (dest)))
8341 522243 : || (get_regno (dest) == (int) regno
8342 488922 : && (get_regno (lra_reg_info[regno].restore_rtx)
8343 488922 : == get_regno (src)))))
8344 : {
8345 571430 : if (lra_dump_file != NULL)
8346 : {
8347 0 : fprintf (lra_dump_file, " Deleting move %u\n",
8348 0 : INSN_UID (insn));
8349 0 : dump_insn_slim (lra_dump_file, insn);
8350 : }
8351 1142860 : delete_move_and_clobber (insn, get_regno (dest));
8352 571430 : continue;
8353 : }
8354 : /* We should not worry about generation memory-memory
8355 : moves here as if the corresponding inheritance did
8356 : not work (inheritance pseudo did not get a hard reg),
8357 : we remove the inheritance pseudo and the optional
8358 : reload. */
8359 : }
8360 549328 : rtx pat = PATTERN (insn);
8361 0 : if (GET_CODE (pat) == CLOBBER && REG_P (SET_DEST (pat))
8362 549328 : && get_regno (SET_DEST (pat)) == (int) regno)
8363 : /* Refuse to remap clobbers to preexisting pseudos. */
8364 0 : gcc_unreachable ();
8365 549328 : lra_substitute_pseudo_within_insn
8366 549328 : (insn, regno, lra_reg_info[regno].restore_rtx, false);
8367 549328 : lra_update_insn_regno_info (insn);
8368 549328 : if (lra_dump_file != NULL)
8369 : {
8370 4 : fprintf (lra_dump_file,
8371 : " Restoring original insn:\n");
8372 4 : dump_insn_slim (lra_dump_file, insn);
8373 : }
8374 : }
8375 : }
8376 : /* Clear restore_regnos. */
8377 2611908 : EXECUTE_IF_SET_IN_BITMAP (&lra_optional_reload_pseudos, 0, regno, bi)
8378 1032985 : lra_reg_info[regno].restore_rtx = NULL_RTX;
8379 1578923 : return change_p;
8380 1578923 : }
8381 :
8382 : /* Entry function for undoing inheritance/split transformation. Return true
8383 : if we did any RTL change in this pass. */
8384 : bool
8385 1582013 : lra_undo_inheritance (void)
8386 : {
8387 1582013 : unsigned int regno;
8388 1582013 : int hard_regno;
8389 1582013 : int n_all_inherit, n_inherit, n_all_split, n_split;
8390 1582013 : rtx restore_rtx;
8391 1582013 : bitmap_iterator bi;
8392 1582013 : bool change_p;
8393 :
8394 1582013 : lra_undo_inheritance_iter++;
8395 1582013 : if (lra_undo_inheritance_iter > LRA_MAX_INHERITANCE_PASSES)
8396 : return false;
8397 1578923 : if (lra_dump_file != NULL)
8398 97 : fprintf (lra_dump_file,
8399 : "\n********** Undoing inheritance #%d: **********\n\n",
8400 : lra_undo_inheritance_iter);
8401 1578923 : auto_bitmap remove_pseudos (®_obstack);
8402 1578923 : n_inherit = n_all_inherit = 0;
8403 3508966 : EXECUTE_IF_SET_IN_BITMAP (&lra_inheritance_pseudos, 0, regno, bi)
8404 1930043 : if (lra_reg_info[regno].restore_rtx != NULL_RTX)
8405 : {
8406 1222870 : n_all_inherit++;
8407 1222870 : if (reg_renumber[regno] < 0
8408 : /* If the original pseudo changed its allocation, just
8409 : removing inheritance is dangerous as for changing
8410 : allocation we used shorter live-ranges. */
8411 1222870 : && (! REG_P (lra_reg_info[regno].restore_rtx)
8412 442278 : || reg_renumber[REGNO (lra_reg_info[regno].restore_rtx)] < 0))
8413 450028 : bitmap_set_bit (remove_pseudos, regno);
8414 : else
8415 772842 : n_inherit++;
8416 : }
8417 1578923 : if (lra_dump_file != NULL && n_all_inherit != 0)
8418 2 : fprintf (lra_dump_file, "Inherit %d out of %d (%.2f%%)\n",
8419 : n_inherit, n_all_inherit,
8420 2 : (double) n_inherit / n_all_inherit * 100);
8421 1578923 : n_split = n_all_split = 0;
8422 2548602 : EXECUTE_IF_SET_IN_BITMAP (&lra_split_regs, 0, regno, bi)
8423 969679 : if ((restore_rtx = lra_reg_info[regno].restore_rtx) != NULL_RTX)
8424 : {
8425 669966 : int restore_regno = REGNO (restore_rtx);
8426 :
8427 669966 : n_all_split++;
8428 1339850 : hard_regno = (restore_regno >= FIRST_PSEUDO_REGISTER
8429 669966 : ? reg_renumber[restore_regno] : restore_regno);
8430 669966 : if (hard_regno < 0 || reg_renumber[regno] == hard_regno)
8431 2428 : bitmap_set_bit (remove_pseudos, regno);
8432 : else
8433 : {
8434 667538 : n_split++;
8435 667538 : if (lra_dump_file != NULL)
8436 0 : fprintf (lra_dump_file, " Keep split r%d (orig=r%d)\n",
8437 : regno, restore_regno);
8438 : }
8439 : }
8440 1578923 : if (lra_dump_file != NULL && n_all_split != 0)
8441 0 : fprintf (lra_dump_file, "Split %d out of %d (%.2f%%)\n",
8442 : n_split, n_all_split,
8443 0 : (double) n_split / n_all_split * 100);
8444 1578923 : change_p = remove_inheritance_pseudos (remove_pseudos);
8445 : /* Clear restore_regnos. */
8446 3508966 : EXECUTE_IF_SET_IN_BITMAP (&lra_inheritance_pseudos, 0, regno, bi)
8447 1930043 : lra_reg_info[regno].restore_rtx = NULL_RTX;
8448 2548602 : EXECUTE_IF_SET_IN_BITMAP (&lra_split_regs, 0, regno, bi)
8449 969679 : lra_reg_info[regno].restore_rtx = NULL_RTX;
8450 1578923 : change_p = undo_optional_reloads () || change_p;
8451 : if (change_p)
8452 112090 : lra_dump_insns_if_possible ("changed func after undoing inheritance");
8453 1578923 : return change_p;
8454 1578923 : }
|
