Line data Source code
1 : /* Code for RTL transformations to satisfy insn constraints.
2 : Copyright (C) 2010-2026 Free Software Foundation, Inc.
3 : Contributed by Vladimir Makarov <vmakarov@redhat.com>.
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 :
22 : /* This file contains code for 3 passes: constraint pass,
23 : inheritance/split pass, and pass for undoing failed inheritance and
24 : split.
25 :
26 : The major goal of constraint pass is to transform RTL to satisfy
27 : insn and address constraints by:
28 : o choosing insn alternatives;
29 : o generating *reload insns* (or reloads in brief) and *reload
30 : pseudos* which will get necessary hard registers later;
31 : o substituting pseudos with equivalent values and removing the
32 : instructions that initialized those pseudos.
33 :
34 : The constraint pass has biggest and most complicated code in LRA.
35 : There are a lot of important details like:
36 : o reuse of input reload pseudos to simplify reload pseudo
37 : allocations;
38 : o some heuristics to choose insn alternative to improve the
39 : inheritance;
40 : o early clobbers etc.
41 :
42 : The pass is mimicking former reload pass in alternative choosing
43 : because the reload pass is oriented to current machine description
44 : model. It might be changed if the machine description model is
45 : changed.
46 :
47 : There is special code for preventing all LRA and this pass cycling
48 : in case of bugs.
49 :
50 : On the first iteration of the pass we process every instruction and
51 : choose an alternative for each one. On subsequent iterations we try
52 : to avoid reprocessing instructions if we can be sure that the old
53 : choice is still valid.
54 :
55 : The inheritance/spilt pass is to transform code to achieve
56 : ineheritance and live range splitting. It is done on backward
57 : traversal of EBBs.
58 :
59 : The inheritance optimization goal is to reuse values in hard
60 : registers. There is analogous optimization in old reload pass. The
61 : inheritance is achieved by following transformation:
62 :
63 : reload_p1 <- p reload_p1 <- p
64 : ... new_p <- reload_p1
65 : ... => ...
66 : reload_p2 <- p reload_p2 <- new_p
67 :
68 : where p is spilled and not changed between the insns. Reload_p1 is
69 : also called *original pseudo* and new_p is called *inheritance
70 : pseudo*.
71 :
72 : The subsequent assignment pass will try to assign the same (or
73 : another if it is not possible) hard register to new_p as to
74 : reload_p1 or reload_p2.
75 :
76 : If the assignment pass fails to assign a hard register to new_p,
77 : this file will undo the inheritance and restore the original code.
78 : This is because implementing the above sequence with a spilled
79 : new_p would make the code much worse. The inheritance is done in
80 : EBB scope. The above is just a simplified example to get an idea
81 : of the inheritance as the inheritance is also done for non-reload
82 : insns.
83 :
84 : Splitting (transformation) is also done in EBB scope on the same
85 : pass as the inheritance:
86 :
87 : r <- ... or ... <- r r <- ... or ... <- r
88 : ... s <- r (new insn -- save)
89 : ... =>
90 : ... r <- s (new insn -- restore)
91 : ... <- r ... <- r
92 :
93 : The *split pseudo* s is assigned to the hard register of the
94 : original pseudo or hard register r.
95 :
96 : Splitting is done:
97 : o In EBBs with high register pressure for global pseudos (living
98 : in at least 2 BBs) and assigned to hard registers when there
99 : are more one reloads needing the hard registers;
100 : o for pseudos needing save/restore code around calls.
101 :
102 : If the split pseudo still has the same hard register as the
103 : original pseudo after the subsequent assignment pass or the
104 : original pseudo was split, the opposite transformation is done on
105 : the same pass for undoing inheritance. */
106 :
107 : #undef REG_OK_STRICT
108 :
109 : #include "config.h"
110 : #include "system.h"
111 : #include "coretypes.h"
112 : #include "backend.h"
113 : #include "hooks.h"
114 : #include "target.h"
115 : #include "rtl.h"
116 : #include "tree.h"
117 : #include "stmt.h"
118 : #include "predict.h"
119 : #include "df.h"
120 : #include "memmodel.h"
121 : #include "tm_p.h"
122 : #include "expmed.h"
123 : #include "optabs.h"
124 : #include "regs.h"
125 : #include "ira.h"
126 : #include "ira-int.h"
127 : #include "recog.h"
128 : #include "output.h"
129 : #include "addresses.h"
130 : #include "expr.h"
131 : #include "cfgrtl.h"
132 : #include "rtl-error.h"
133 : #include "lra.h"
134 : #include "lra-int.h"
135 : #include "print-rtl.h"
136 : #include "function-abi.h"
137 : #include "rtl-iter.h"
138 : #include "hash-set.h"
139 :
140 : /* Value of LRA_CURR_RELOAD_NUM at the beginning of BB of the current
141 : insn. Remember that LRA_CURR_RELOAD_NUM is the number of emitted
142 : reload insns. */
143 : static int bb_reload_num;
144 :
145 : /* The current insn being processed and corresponding its single set
146 : (NULL otherwise), its data (basic block, the insn data, the insn
147 : static data, and the mode of each operand). */
148 : static rtx_insn *curr_insn;
149 : static rtx curr_insn_set;
150 : static basic_block curr_bb;
151 : static lra_insn_recog_data_t curr_id;
152 : static struct lra_static_insn_data *curr_static_id;
153 : static machine_mode curr_operand_mode[MAX_RECOG_OPERANDS];
154 : /* Mode of the register substituted by its equivalence with VOIDmode
155 : (e.g. constant) and whose subreg is given operand of the current
156 : insn. VOIDmode in all other cases. */
157 : static machine_mode original_subreg_reg_mode[MAX_RECOG_OPERANDS];
158 : /* The first call insn after curr_insn within the EBB during inherit_in_ebb
159 : or NULL outside of that function. */
160 : static rtx_insn *first_call_insn;
161 :
162 :
163 :
164 : /* Start numbers for new registers and insns at the current constraints
165 : pass start. */
166 : static int new_regno_start;
167 : static int new_insn_uid_start;
168 :
169 : /* If LOC is nonnull, strip any outer subreg from it. */
170 : static inline rtx *
171 234206244 : strip_subreg (rtx *loc)
172 : {
173 103597918 : return loc && GET_CODE (*loc) == SUBREG ? &SUBREG_REG (*loc) : loc;
174 : }
175 :
176 : /* Return hard regno of REGNO or if it is was not assigned to a hard
177 : register, use a hard register from its allocno class. */
178 : static int
179 90637 : get_try_hard_regno (int regno)
180 : {
181 90637 : int hard_regno;
182 90637 : enum reg_class rclass;
183 :
184 90637 : if ((hard_regno = regno) >= FIRST_PSEUDO_REGISTER)
185 90637 : hard_regno = lra_get_regno_hard_regno (regno);
186 90637 : if (hard_regno >= 0)
187 : return hard_regno;
188 56432 : rclass = lra_get_allocno_class (regno);
189 56432 : if (rclass == NO_REGS)
190 : return -1;
191 54964 : return ira_class_hard_regs[rclass][0];
192 : }
193 :
194 : /* Return the hard regno of X after removing its subreg. If X is not a
195 : register or a subreg of a register, return -1. If X is a pseudo, use its
196 : assignment. If X is a hard regno, return the final hard regno which will be
197 : after elimination. */
198 : static int
199 296634766 : get_hard_regno (rtx x)
200 : {
201 296634766 : rtx reg;
202 296634766 : int hard_regno;
203 :
204 296634766 : reg = x;
205 296634766 : if (SUBREG_P (x))
206 5350719 : reg = SUBREG_REG (x);
207 296634766 : if (! REG_P (reg))
208 : return -1;
209 204881784 : int regno = REGNO (reg);
210 204881784 : if (HARD_REGISTER_NUM_P (regno))
211 35832778 : hard_regno = lra_get_elimination_hard_regno (regno);
212 : else
213 169049006 : hard_regno = lra_get_regno_hard_regno (regno);
214 204881784 : if (hard_regno < 0)
215 : return -1;
216 186634816 : if (SUBREG_P (x))
217 4642386 : hard_regno += subreg_regno_offset (hard_regno, GET_MODE (reg),
218 4642386 : SUBREG_BYTE (x), GET_MODE (x));
219 : return hard_regno;
220 : }
221 :
222 : /* If REGNO is a hard register or has been allocated a hard register,
223 : return the class of that register. If REGNO is a reload pseudo
224 : created by the current constraints pass, return its allocno class.
225 : Return NO_REGS otherwise. */
226 : static enum reg_class
227 527193209 : get_reg_class (int regno)
228 : {
229 527193209 : int hard_regno;
230 :
231 527193209 : if (HARD_REGISTER_NUM_P (regno))
232 67730352 : hard_regno = lra_get_elimination_hard_regno (regno);
233 : else
234 459462857 : hard_regno = lra_get_regno_hard_regno (regno);
235 527193209 : if (hard_regno >= 0)
236 330797031 : return REGNO_REG_CLASS (hard_regno);
237 196396178 : if (regno >= new_regno_start)
238 63050697 : return lra_get_allocno_class (regno);
239 : return NO_REGS;
240 : }
241 :
242 : /* Return true if REG_CLASS has enough allocatable hard regs to keep value of
243 : REG_MODE. */
244 : static bool
245 19021453 : enough_allocatable_hard_regs_p (enum reg_class reg_class,
246 : enum machine_mode reg_mode)
247 : {
248 19021453 : int i, j, hard_regno, class_size, nregs;
249 :
250 38042906 : if (hard_reg_set_subset_p (reg_class_contents[reg_class], lra_no_alloc_regs))
251 : return false;
252 6612218 : class_size = ira_class_hard_regs_num[reg_class];
253 6612218 : for (i = 0; i < class_size; i++)
254 : {
255 6612218 : hard_regno = ira_class_hard_regs[reg_class][i];
256 6612218 : nregs = hard_regno_nregs (hard_regno, reg_mode);
257 6612218 : if (nregs == 1)
258 : return true;
259 286473 : for (j = 0; j < nregs; j++)
260 190982 : if (TEST_HARD_REG_BIT (lra_no_alloc_regs, hard_regno + j)
261 190982 : || ! TEST_HARD_REG_BIT (reg_class_contents[reg_class],
262 : hard_regno + j))
263 : break;
264 95491 : if (j >= nregs)
265 : return true;
266 : }
267 : return false;
268 : }
269 :
270 : /* True if C is a non-empty register class that has too few registers
271 : to be safely used as a reload target class. */
272 : #define SMALL_REGISTER_CLASS_P(C) \
273 : (ira_class_hard_regs_num [(C)] == 1 \
274 : || (ira_class_hard_regs_num [(C)] >= 1 \
275 : && targetm.class_likely_spilled_p (C)))
276 :
277 : /* Return true if REG satisfies (or will satisfy) reg class constraint
278 : CL. Use elimination first if REG is a hard register. If REG is a
279 : reload pseudo created by this constraints pass, assume that it will
280 : be allocated a hard register from its allocno class, but allow that
281 : class to be narrowed to CL if it is currently a superset of CL and
282 : if either:
283 :
284 : - ALLOW_ALL_RELOAD_CLASS_CHANGES_P is true or
285 : - the instruction we're processing is not a reload move.
286 :
287 : If NEW_CLASS is nonnull, set *NEW_CLASS to the new allocno class of
288 : REGNO (reg), or NO_REGS if no change in its class was needed. */
289 : static bool
290 223798391 : in_class_p (rtx reg, enum reg_class cl, enum reg_class *new_class,
291 : bool allow_all_reload_class_changes_p = false)
292 : {
293 223798391 : enum reg_class rclass, common_class;
294 223798391 : machine_mode reg_mode;
295 223798391 : rtx src;
296 223798391 : int regno = REGNO (reg);
297 :
298 223798391 : if (new_class != NULL)
299 115308523 : *new_class = NO_REGS;
300 223798391 : if (regno < FIRST_PSEUDO_REGISTER)
301 : {
302 28503772 : rtx final_reg = reg;
303 28503772 : rtx *final_loc = &final_reg;
304 :
305 28503772 : lra_eliminate_reg_if_possible (final_loc);
306 28503772 : return TEST_HARD_REG_BIT (reg_class_contents[cl], REGNO (*final_loc));
307 : }
308 195294619 : reg_mode = GET_MODE (reg);
309 195294619 : rclass = get_reg_class (regno);
310 195294619 : src = curr_insn_set != NULL ? SET_SRC (curr_insn_set) : NULL;
311 195294619 : if (regno < new_regno_start
312 : /* Do not allow the constraints for reload instructions to
313 : influence the classes of new pseudos. These reloads are
314 : typically moves that have many alternatives, and restricting
315 : reload pseudos for one alternative may lead to situations
316 : where other reload pseudos are no longer allocatable. */
317 195294619 : || (!allow_all_reload_class_changes_p
318 15188253 : && INSN_UID (curr_insn) >= new_insn_uid_start
319 14653364 : && src != NULL
320 14653364 : && ((REG_P (src) || MEM_P (src))
321 1488414 : || (GET_CODE (src) == SUBREG
322 670864 : && (REG_P (SUBREG_REG (src)) || MEM_P (SUBREG_REG (src)))))))
323 : /* When we don't know what class will be used finally for reload
324 : pseudos, we use ALL_REGS. */
325 13835814 : return ((regno >= new_regno_start && rclass == ALL_REGS)
326 190106506 : || (rclass != NO_REGS && ira_class_subset_p[rclass][cl]
327 205925404 : && ! hard_reg_set_subset_p (reg_class_contents[cl],
328 : lra_no_alloc_regs)));
329 : else
330 : {
331 19021453 : common_class = ira_reg_class_subset[rclass][cl];
332 19021453 : if (new_class != NULL)
333 5439533 : *new_class = common_class;
334 19021453 : return (enough_allocatable_hard_regs_p (common_class, reg_mode)
335 : /* Do not permit reload insn operand matching (new_class == NULL
336 : case) if the new class is too small. */
337 19021453 : && (new_class != NULL || common_class == rclass
338 1086995 : || !SMALL_REGISTER_CLASS_P (common_class)));
339 : }
340 : }
341 :
342 : /* Return true if REGNO satisfies a memory constraint. */
343 : static bool
344 65206743 : in_mem_p (int regno)
345 : {
346 0 : return get_reg_class (regno) == NO_REGS;
347 : }
348 :
349 : /* Return true if ADDR is a valid memory address for mode MODE in address
350 : space AS, and check that each pseudo has the proper kind of hard
351 : reg. */
352 : static bool
353 36398907 : valid_address_p (machine_mode mode ATTRIBUTE_UNUSED,
354 : rtx addr, addr_space_t as)
355 : {
356 : #ifdef GO_IF_LEGITIMATE_ADDRESS
357 : lra_assert (ADDR_SPACE_GENERIC_P (as));
358 : GO_IF_LEGITIMATE_ADDRESS (mode, addr, win);
359 : return false;
360 :
361 : win:
362 : return true;
363 : #else
364 0 : return targetm.addr_space.legitimate_address_p (mode, addr, 0, as,
365 36398907 : ERROR_MARK);
366 : #endif
367 : }
368 :
369 : namespace {
370 : /* Temporarily eliminates registers in an address (for the lifetime of
371 : the object). */
372 : class address_eliminator {
373 : public:
374 : address_eliminator (struct address_info *ad);
375 : ~address_eliminator ();
376 :
377 : private:
378 : struct address_info *m_ad;
379 : rtx *m_base_loc;
380 : rtx m_base_reg;
381 : rtx *m_index_loc;
382 : rtx m_index_reg;
383 : };
384 : }
385 :
386 76753309 : address_eliminator::address_eliminator (struct address_info *ad)
387 76753309 : : m_ad (ad),
388 76753309 : m_base_loc (strip_subreg (ad->base_term)),
389 76753309 : m_base_reg (NULL_RTX),
390 76753309 : m_index_loc (strip_subreg (ad->index_term)),
391 76753309 : m_index_reg (NULL_RTX)
392 : {
393 76753309 : if (m_base_loc != NULL)
394 : {
395 64129200 : m_base_reg = *m_base_loc;
396 : /* If we have non-legitimate address which is decomposed not in
397 : the way we expected, don't do elimination here. In such case
398 : the address will be reloaded and elimination will be done in
399 : reload insn finally. */
400 64129200 : if (REG_P (m_base_reg))
401 64129200 : lra_eliminate_reg_if_possible (m_base_loc);
402 64129200 : if (m_ad->base_term2 != NULL)
403 0 : *m_ad->base_term2 = *m_ad->base_term;
404 : }
405 76753309 : if (m_index_loc != NULL)
406 : {
407 3559188 : m_index_reg = *m_index_loc;
408 3559188 : if (REG_P (m_index_reg))
409 3559188 : lra_eliminate_reg_if_possible (m_index_loc);
410 : }
411 76753309 : }
412 :
413 76753309 : address_eliminator::~address_eliminator ()
414 : {
415 76753309 : if (m_base_loc && *m_base_loc != m_base_reg)
416 : {
417 45439266 : *m_base_loc = m_base_reg;
418 45439266 : if (m_ad->base_term2 != NULL)
419 0 : *m_ad->base_term2 = *m_ad->base_term;
420 : }
421 76753309 : if (m_index_loc && *m_index_loc != m_index_reg)
422 0 : *m_index_loc = m_index_reg;
423 76753309 : }
424 :
425 : /* Return true if the eliminated form of AD is a legitimate target address.
426 : If OP is a MEM, AD is the address within OP, otherwise OP should be
427 : ignored. CONSTRAINT is one constraint that the operand may need
428 : to meet. */
429 : static bool
430 36375195 : valid_address_p (rtx op, struct address_info *ad,
431 : enum constraint_num constraint)
432 : {
433 36375195 : address_eliminator eliminator (ad);
434 :
435 : /* Allow a memory OP if it matches CONSTRAINT, even if CONSTRAINT is more
436 : forgiving than "m".
437 : Need to extract memory from op for special memory constraint,
438 : i.e. bcst_mem_operand in i386 backend. */
439 36375195 : if (MEM_P (extract_mem_from_operand (op))
440 : && insn_extra_relaxed_memory_constraint (constraint)
441 : && constraint_satisfied_p (op, constraint))
442 : return true;
443 :
444 36375195 : return valid_address_p (ad->mode, *ad->outer, ad->as);
445 36375195 : }
446 :
447 : /* For special_memory_operand, it could be false for MEM_P (op),
448 : i.e. bcst_mem_operand in i386 backend.
449 : Extract and return real memory operand or op. */
450 : rtx
451 642234334 : extract_mem_from_operand (rtx op)
452 : {
453 643975932 : for (rtx x = op;; x = XEXP (x, 0))
454 : {
455 643975932 : if (MEM_P (x))
456 : return x;
457 458896894 : if (GET_RTX_LENGTH (GET_CODE (x)) != 1
458 375924241 : || GET_RTX_FORMAT (GET_CODE (x))[0] != 'e')
459 : break;
460 : }
461 : return op;
462 : }
463 :
464 : /* Return true if the eliminated form of memory reference OP satisfies
465 : extra (special) memory constraint CONSTRAINT. */
466 : static bool
467 37709787 : satisfies_memory_constraint_p (rtx op, enum constraint_num constraint)
468 : {
469 37709787 : struct address_info ad;
470 37709787 : rtx mem = extract_mem_from_operand (op);
471 37709787 : if (!MEM_P (mem))
472 : return false;
473 :
474 36674117 : decompose_mem_address (&ad, mem);
475 36674117 : address_eliminator eliminator (&ad);
476 36674117 : return constraint_satisfied_p (op, constraint);
477 36674117 : }
478 :
479 : /* Return true if the eliminated form of address AD satisfies extra
480 : address constraint CONSTRAINT. */
481 : static bool
482 3703997 : satisfies_address_constraint_p (struct address_info *ad,
483 : enum constraint_num constraint)
484 : {
485 3703997 : address_eliminator eliminator (ad);
486 3703997 : return constraint_satisfied_p (*ad->outer, constraint);
487 3703997 : }
488 :
489 : /* Return true if the eliminated form of address OP satisfies extra
490 : address constraint CONSTRAINT. */
491 : static bool
492 1821268 : satisfies_address_constraint_p (rtx op, enum constraint_num constraint)
493 : {
494 1821268 : struct address_info ad;
495 :
496 1821268 : decompose_lea_address (&ad, &op);
497 1821268 : return satisfies_address_constraint_p (&ad, constraint);
498 : }
499 :
500 : /* Set of equivalences whose original targets have set up pointer flag. */
501 : static hash_set <rtx> *pointer_equiv_set;
502 :
503 : /* Add x to pointer_equiv_set. */
504 : void
505 2064473 : lra_pointer_equiv_set_add (rtx x)
506 : {
507 2064473 : pointer_equiv_set->add (x);
508 2064473 : }
509 :
510 : /* Return true if x is in pointer_equiv_set. */
511 : bool
512 10195468 : lra_pointer_equiv_set_in (rtx x)
513 : {
514 10195468 : return pointer_equiv_set->contains (x);
515 : }
516 :
517 : /* Initiate equivalences for LRA. As we keep original equivalences
518 : before any elimination, we need to make copies otherwise any change
519 : in insns might change the equivalences. */
520 : void
521 1515921 : lra_init_equiv (void)
522 : {
523 1515921 : ira_expand_reg_equiv ();
524 73685357 : for (int i = FIRST_PSEUDO_REGISTER; i < max_reg_num (); i++)
525 : {
526 70653515 : rtx res;
527 :
528 70653515 : if ((res = ira_reg_equiv[i].memory) != NULL_RTX)
529 3082039 : ira_reg_equiv[i].memory = copy_rtx (res);
530 70653515 : if ((res = ira_reg_equiv[i].invariant) != NULL_RTX)
531 917390 : ira_reg_equiv[i].invariant = copy_rtx (res);
532 : }
533 1515921 : pointer_equiv_set = new hash_set <rtx>;
534 1515921 : }
535 :
536 : /* Finish equivalence data for LRA. */
537 : void
538 1515921 : lra_finish_equiv (void)
539 : {
540 3031842 : delete pointer_equiv_set;
541 1515921 : }
542 :
543 : static rtx loc_equivalence_callback (rtx, const_rtx, void *);
544 :
545 : /* Update equivalence for REGNO. We need to this as the equivalence
546 : might contain other pseudos which are changed by their
547 : equivalences. */
548 : static void
549 209581538 : update_equiv (int regno)
550 : {
551 209581538 : rtx x;
552 :
553 209581538 : if ((x = ira_reg_equiv[regno].memory) != NULL_RTX)
554 9336304 : ira_reg_equiv[regno].memory
555 9336304 : = simplify_replace_fn_rtx (x, NULL_RTX, loc_equivalence_callback,
556 : NULL_RTX);
557 209581538 : if ((x = ira_reg_equiv[regno].invariant) != NULL_RTX)
558 2842542 : ira_reg_equiv[regno].invariant
559 2842542 : = simplify_replace_fn_rtx (x, NULL_RTX, loc_equivalence_callback,
560 : NULL_RTX);
561 209581538 : }
562 :
563 : /* If we have decided to substitute X with another value, return that
564 : value, otherwise return X. */
565 : static rtx
566 448719498 : get_equiv (rtx x)
567 : {
568 448719498 : int regno;
569 448719498 : rtx res;
570 :
571 303675728 : if (! REG_P (x) || (regno = REGNO (x)) < FIRST_PSEUDO_REGISTER
572 201286756 : || regno >= ira_reg_equiv_len
573 201286756 : || ! ira_reg_equiv[regno].defined_p
574 26370986 : || ! ira_reg_equiv[regno].profitable_p
575 475048369 : || lra_get_regno_hard_regno (regno) >= 0)
576 : return x;
577 5277329 : if ((res = ira_reg_equiv[regno].memory) != NULL_RTX)
578 : {
579 2334096 : if (targetm.cannot_substitute_mem_equiv_p (res))
580 : return x;
581 2334096 : return res;
582 : }
583 2943233 : if ((res = ira_reg_equiv[regno].constant) != NULL_RTX)
584 : {
585 913516 : if (targetm.cannot_substitute_const_equiv_p (res))
586 : return x;
587 913516 : return res;
588 : }
589 2029717 : if ((res = ira_reg_equiv[regno].invariant) != NULL_RTX)
590 : return res;
591 0 : gcc_unreachable ();
592 : }
593 :
594 : /* If we have decided to substitute X with the equivalent value, return that
595 : value after elimination for INSN, otherwise return X. Add the result to
596 : pointer_equiv_set if X has set up pointer flag. */
597 : static rtx
598 252217372 : get_equiv_with_elimination (rtx x, rtx_insn *insn)
599 : {
600 252217372 : rtx res = get_equiv (x);
601 :
602 252217372 : if (x == res || CONSTANT_P (res))
603 : return res;
604 1599992 : res = lra_eliminate_regs_1 (insn, res, GET_MODE (res),
605 : false, false, 0, true);
606 1599992 : if (REG_POINTER (x))
607 1109569 : lra_pointer_equiv_set_add (res);
608 : return res;
609 : }
610 :
611 : /* Set up curr_operand_mode. */
612 : static void
613 109052570 : init_curr_operand_mode (void)
614 : {
615 109052570 : int nop = curr_static_id->n_operands;
616 340073491 : for (int i = 0; i < nop; i++)
617 : {
618 231020921 : machine_mode mode = GET_MODE (*curr_id->operand_loc[i]);
619 231020921 : if (mode == VOIDmode)
620 : {
621 : /* The .md mode for address operands is the mode of the
622 : addressed value rather than the mode of the address itself. */
623 44731366 : if (curr_id->icode >= 0 && curr_static_id->operand[i].is_address)
624 63 : mode = Pmode;
625 : else
626 44731303 : mode = curr_static_id->operand[i].mode;
627 : }
628 231020921 : curr_operand_mode[i] = mode;
629 : }
630 109052570 : }
631 :
632 :
633 :
634 : /* The page contains code to reuse input reloads. */
635 :
636 : /* Structure describes input reload of the current insns. */
637 : struct input_reload
638 : {
639 : /* True for input reload of matched operands. */
640 : bool match_p;
641 : /* True for input reload of inout earlyclobber operand. */
642 : bool early_clobber_p;
643 : /* Reloaded value. */
644 : rtx input;
645 : /* Reload pseudo used. */
646 : rtx reg;
647 : };
648 :
649 : /* The number of elements in the following array. */
650 : static int curr_insn_input_reloads_num;
651 : /* Array containing info about input reloads. It is used to find the
652 : same input reload and reuse the reload pseudo in this case. */
653 : static struct input_reload curr_insn_input_reloads[LRA_MAX_INSN_RELOADS];
654 :
655 : /* Initiate data concerning reuse of input reloads for the current
656 : insn. */
657 : static void
658 109052570 : init_curr_insn_input_reloads (void)
659 : {
660 109052570 : curr_insn_input_reloads_num = 0;
661 0 : }
662 :
663 : /* The canonical form of an rtx inside a MEM is not necessarily the same as the
664 : canonical form of the rtx outside the MEM. Fix this up in the case that
665 : we're reloading an address (and therefore pulling it outside a MEM). */
666 : static rtx
667 125 : canonicalize_reload_addr (rtx addr)
668 : {
669 125 : subrtx_var_iterator::array_type array;
670 458 : FOR_EACH_SUBRTX_VAR (iter, array, addr, NONCONST)
671 : {
672 333 : rtx x = *iter;
673 333 : if (GET_CODE (x) == MULT && CONST_INT_P (XEXP (x, 1)))
674 : {
675 14 : const HOST_WIDE_INT ci = INTVAL (XEXP (x, 1));
676 347 : const int pwr2 = exact_log2 (ci);
677 14 : if (pwr2 > 0)
678 : {
679 : /* Rewrite this to use a shift instead, which is canonical when
680 : outside of a MEM. */
681 14 : PUT_CODE (x, ASHIFT);
682 14 : XEXP (x, 1) = GEN_INT (pwr2);
683 : }
684 : }
685 : }
686 :
687 125 : return addr;
688 125 : }
689 :
690 : /* Return rtx accessing reload REG of RCLASS matching another reload reg in
691 : MODE. */
692 : static rtx
693 119517 : get_matching_reload_reg_subreg (machine_mode mode, rtx reg,
694 : enum reg_class rclass)
695 : {
696 119517 : int hard_regno = ira_class_hard_regs[rclass][0];
697 119517 : if (subreg_regno_offset (hard_regno,
698 119517 : GET_MODE (reg),
699 119517 : subreg_lowpart_offset (mode, GET_MODE (reg)),
700 : mode) == 0)
701 : /* For matching scalar int modes generate the right subreg byte offset for
702 : BE targets -- see call of reload.cc:operands_match_p in
703 : recog.cc:constrain_operands. */
704 119517 : return lowpart_subreg (mode, reg, GET_MODE (reg));
705 0 : int offset = (lra_constraint_offset (hard_regno, GET_MODE (reg))
706 0 : - lra_constraint_offset (hard_regno, mode)) * UNITS_PER_WORD;
707 0 : lra_assert (offset >= 0);
708 0 : return gen_rtx_SUBREG (mode, reg, offset);
709 : }
710 :
711 : /* Create a new pseudo using MODE, RCLASS, EXCLUDE_START_HARD_REGS, ORIGINAL or
712 : reuse an existing reload pseudo. Don't reuse an existing reload pseudo if
713 : IN_SUBREG_P is true and the reused pseudo should be wrapped up in a SUBREG.
714 : EARLY_CLOBBER_P is true for input reload of inout early clobber operand.
715 : The result pseudo is returned through RESULT_REG. Return TRUE if we created
716 : a new pseudo, FALSE if we reused an existing reload pseudo. Use TITLE to
717 : describe new registers for debug purposes. */
718 : static bool
719 3959180 : get_reload_reg (enum op_type type, machine_mode mode, rtx original,
720 : enum reg_class rclass, HARD_REG_SET *exclude_start_hard_regs,
721 : bool in_subreg_p, bool early_clobber_p,
722 : const char *title, rtx *result_reg)
723 : {
724 3959180 : int i, regno;
725 3959180 : enum reg_class new_class;
726 :
727 3959180 : if (type == OP_OUT)
728 : {
729 : /* Output reload registers tend to start out with a conservative
730 : choice of register class. Usually this is ALL_REGS, although
731 : a target might narrow it (for performance reasons) through
732 : targetm.preferred_reload_class. It's therefore quite common
733 : for a reload instruction to require a more restrictive class
734 : than the class that was originally assigned to the reload register.
735 :
736 : In these situations, it's more efficient to refine the choice
737 : of register class rather than create a second reload register.
738 : This also helps to avoid cycling for registers that are only
739 : used by reload instructions. */
740 984750 : if (REG_P (original)
741 714135 : && (int) REGNO (original) >= new_regno_start
742 7167 : && (INSN_UID (curr_insn) >= new_insn_uid_start
743 250 : || ira_former_scratch_p (REGNO (original)))
744 7167 : && in_class_p (original, rclass, &new_class, true)
745 985000 : && (exclude_start_hard_regs == nullptr
746 250 : || hard_reg_set_intersect_p (
747 985000 : ~lra_reg_info[REGNO (original)].exclude_start_hard_regs,
748 500 : ~*exclude_start_hard_regs)))
749 : {
750 250 : unsigned int regno = REGNO (original);
751 250 : if (lra_dump_file != NULL)
752 : {
753 0 : fprintf (lra_dump_file, " Reuse r%d for output ", regno);
754 0 : dump_value_slim (lra_dump_file, original, 1);
755 : }
756 500 : if (new_class != lra_get_allocno_class (regno))
757 250 : lra_change_class (regno, new_class, ", change to", false);
758 250 : if (lra_dump_file != NULL)
759 0 : fprintf (lra_dump_file, "\n");
760 250 : if (exclude_start_hard_regs)
761 250 : lra_reg_info[regno].exclude_start_hard_regs
762 250 : |= *exclude_start_hard_regs;
763 250 : *result_reg = original;
764 250 : return false;
765 : }
766 984500 : *result_reg
767 984500 : = lra_create_new_reg_with_unique_value (mode, original, rclass,
768 : exclude_start_hard_regs, title);
769 984500 : return true;
770 : }
771 :
772 2974430 : bool unique_p = early_clobber_p;
773 : /* Prevent reuse value of expression with side effects,
774 : e.g. volatile memory. */
775 2974430 : if (! side_effects_p (original))
776 3194674 : for (i = 0; i < curr_insn_input_reloads_num; i++)
777 : {
778 240029 : if (! curr_insn_input_reloads[i].match_p
779 98182 : && ! curr_insn_input_reloads[i].early_clobber_p
780 98181 : && rtx_equal_p (curr_insn_input_reloads[i].input, original)
781 249786 : && in_class_p (curr_insn_input_reloads[i].reg, rclass, &new_class))
782 : {
783 9744 : rtx reg = curr_insn_input_reloads[i].reg;
784 9744 : regno = REGNO (reg);
785 : /* If input is equal to original and both are VOIDmode,
786 : GET_MODE (reg) might be still different from mode.
787 : Ensure we don't return *result_reg with wrong mode. */
788 9744 : if (GET_MODE (reg) != mode)
789 : {
790 0 : if (in_subreg_p)
791 0 : continue;
792 0 : if (maybe_lt (GET_MODE_SIZE (GET_MODE (reg)),
793 0 : GET_MODE_SIZE (mode)))
794 0 : continue;
795 0 : reg = get_matching_reload_reg_subreg (mode, reg, new_class);
796 0 : if (reg == NULL_RTX || GET_CODE (reg) != SUBREG)
797 0 : continue;
798 : }
799 : /* If the existing reload and this have no start hard register in
800 : common, then skip. Otherwise update exclude_start_hard_regs. */
801 9744 : if (exclude_start_hard_regs
802 11222 : && ! hard_reg_set_empty_p (*exclude_start_hard_regs))
803 : {
804 1 : HARD_REG_SET r = lra_reg_info[regno].exclude_start_hard_regs
805 1 : | *exclude_start_hard_regs;
806 2 : if (hard_reg_set_empty_p (~r))
807 0 : continue;
808 : else
809 1 : lra_reg_info[regno].exclude_start_hard_regs = r;
810 : }
811 9744 : *result_reg = reg;
812 9744 : if (lra_dump_file != NULL)
813 : {
814 0 : fprintf (lra_dump_file, " Reuse r%d for reload ", regno);
815 0 : dump_value_slim (lra_dump_file, original, 1);
816 : }
817 19488 : if (new_class != lra_get_allocno_class (regno))
818 4764 : lra_change_class (regno, new_class, ", change to", false);
819 9744 : if (lra_dump_file != NULL)
820 0 : fprintf (lra_dump_file, "\n");
821 : return false;
822 : }
823 : /* If we have an input reload with a different mode, make sure it
824 : will get a different hard reg. */
825 230285 : else if (REG_P (original)
826 180355 : && REG_P (curr_insn_input_reloads[i].input)
827 149645 : && REGNO (original) == REGNO (curr_insn_input_reloads[i].input)
828 230285 : && (GET_MODE (original)
829 2154 : != GET_MODE (curr_insn_input_reloads[i].input)))
830 : unique_p = true;
831 : }
832 5929372 : *result_reg = (unique_p
833 2964686 : ? lra_create_new_reg_with_unique_value
834 2964686 : : lra_create_new_reg) (mode, original, rclass,
835 : exclude_start_hard_regs, title);
836 2964686 : lra_assert (curr_insn_input_reloads_num < LRA_MAX_INSN_RELOADS);
837 2964686 : curr_insn_input_reloads[curr_insn_input_reloads_num].input = original;
838 2964686 : curr_insn_input_reloads[curr_insn_input_reloads_num].match_p = false;
839 2964686 : curr_insn_input_reloads[curr_insn_input_reloads_num].early_clobber_p
840 2964686 : = early_clobber_p;
841 2964686 : curr_insn_input_reloads[curr_insn_input_reloads_num++].reg = *result_reg;
842 2964686 : return true;
843 : }
844 :
845 :
846 : /* The page contains major code to choose the current insn alternative
847 : and generate reloads for it. */
848 :
849 : /* Return the offset from REGNO of the least significant register
850 : in (reg:MODE REGNO).
851 :
852 : This function is used to tell whether two registers satisfy
853 : a matching constraint. (reg:MODE1 REGNO1) matches (reg:MODE2 REGNO2) if:
854 :
855 : REGNO1 + lra_constraint_offset (REGNO1, MODE1)
856 : == REGNO2 + lra_constraint_offset (REGNO2, MODE2) */
857 : int
858 44937338 : lra_constraint_offset (int regno, machine_mode mode)
859 : {
860 44937338 : lra_assert (regno < FIRST_PSEUDO_REGISTER);
861 :
862 44937338 : scalar_int_mode int_mode;
863 44937338 : if (WORDS_BIG_ENDIAN
864 : && is_a <scalar_int_mode> (mode, &int_mode)
865 : && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD)
866 : return hard_regno_nregs (regno, mode) - 1;
867 44937338 : return 0;
868 : }
869 :
870 : /* Like rtx_equal_p except that it allows a REG and a SUBREG to match
871 : if they are the same hard reg, and has special hacks for
872 : auto-increment and auto-decrement. This is specifically intended for
873 : process_alt_operands to use in determining whether two operands
874 : match. X is the operand whose number is the lower of the two.
875 :
876 : It is supposed that X is the output operand and Y is the input
877 : operand. Y_HARD_REGNO is the final hard regno of register Y or
878 : register in subreg Y as we know it now. Otherwise, it is a
879 : negative value. */
880 : static bool
881 59165779 : operands_match_p (rtx x, rtx y, int y_hard_regno)
882 : {
883 59165779 : int i;
884 59165779 : RTX_CODE code = GET_CODE (x);
885 59165779 : const char *fmt;
886 :
887 59165779 : if (x == y)
888 : return true;
889 51412420 : if ((code == REG || (code == SUBREG && REG_P (SUBREG_REG (x))))
890 24550545 : && (REG_P (y) || (GET_CODE (y) == SUBREG && REG_P (SUBREG_REG (y)))))
891 : {
892 24456363 : int j;
893 :
894 24456363 : i = get_hard_regno (x);
895 24456363 : if (i < 0)
896 1287985 : goto slow;
897 :
898 23168378 : if ((j = y_hard_regno) < 0)
899 699709 : goto slow;
900 :
901 22468669 : i += lra_constraint_offset (i, GET_MODE (x));
902 22468669 : j += lra_constraint_offset (j, GET_MODE (y));
903 :
904 22468669 : return i == j;
905 : }
906 :
907 : /* If two operands must match, because they are really a single
908 : operand of an assembler insn, then two post-increments are invalid
909 : because the assembler insn would increment only once. On the
910 : other hand, a post-increment matches ordinary indexing if the
911 : post-increment is the output operand. */
912 26956057 : if (code == POST_DEC || code == POST_INC || code == POST_MODIFY)
913 0 : return operands_match_p (XEXP (x, 0), y, y_hard_regno);
914 :
915 : /* Two pre-increments are invalid because the assembler insn would
916 : increment only once. On the other hand, a pre-increment matches
917 : ordinary indexing if the pre-increment is the input operand. */
918 26956057 : if (GET_CODE (y) == PRE_DEC || GET_CODE (y) == PRE_INC
919 26956057 : || GET_CODE (y) == PRE_MODIFY)
920 0 : return operands_match_p (x, XEXP (y, 0), -1);
921 :
922 26956057 : slow:
923 :
924 28943751 : if (code == REG && REG_P (y))
925 1889263 : return REGNO (x) == REGNO (y);
926 :
927 96791 : if (code == REG && GET_CODE (y) == SUBREG && REG_P (SUBREG_REG (y))
928 9317 : && x == SUBREG_REG (y))
929 : return true;
930 27054488 : if (GET_CODE (y) == REG && code == SUBREG && REG_P (SUBREG_REG (x))
931 66779 : && SUBREG_REG (x) == y)
932 : return true;
933 :
934 : /* Now we have disposed of all the cases in which different rtx
935 : codes can match. */
936 27054359 : if (code != GET_CODE (y))
937 : return false;
938 :
939 : /* (MULT:SI x y) and (MULT:HI x y) are NOT equivalent. */
940 1056679 : if (GET_MODE (x) != GET_MODE (y))
941 : return false;
942 :
943 1055946 : switch (code)
944 : {
945 : CASE_CONST_UNIQUE:
946 : return false;
947 :
948 : case CONST_VECTOR:
949 : if (!same_vector_encodings_p (x, y))
950 : return false;
951 : break;
952 :
953 0 : case LABEL_REF:
954 0 : return label_ref_label (x) == label_ref_label (y);
955 25 : case SYMBOL_REF:
956 25 : return XSTR (x, 0) == XSTR (y, 0);
957 :
958 : default:
959 : break;
960 : }
961 :
962 : /* Compare the elements. If any pair of corresponding elements fail
963 : to match, return false for the whole things. */
964 :
965 1030046 : fmt = GET_RTX_FORMAT (code);
966 2977830 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
967 : {
968 2024038 : int val, j;
969 2024038 : switch (fmt[i])
970 : {
971 0 : case 'w':
972 0 : if (XWINT (x, i) != XWINT (y, i))
973 : return false;
974 : break;
975 :
976 492 : case 'i':
977 492 : if (XINT (x, i) != XINT (y, i))
978 : return false;
979 : break;
980 :
981 0 : case 'L':
982 0 : if (XLOC (x, i) != XLOC (y, i))
983 : return false;
984 : break;
985 :
986 22259 : case 'p':
987 22259 : if (maybe_ne (SUBREG_BYTE (x), SUBREG_BYTE (y)))
988 : return false;
989 : break;
990 :
991 1470930 : case 'e':
992 1470930 : val = operands_match_p (XEXP (x, i), XEXP (y, i), -1);
993 1470930 : if (val == 0)
994 : return false;
995 : break;
996 :
997 : case '0':
998 : break;
999 :
1000 492 : case 'E':
1001 492 : if (XVECLEN (x, i) != XVECLEN (y, i))
1002 : return false;
1003 984 : for (j = XVECLEN (x, i) - 1; j >= 0; --j)
1004 : {
1005 492 : val = operands_match_p (XVECEXP (x, i, j), XVECEXP (y, i, j), -1);
1006 492 : if (val == 0)
1007 : return false;
1008 : }
1009 : break;
1010 :
1011 : /* It is believed that rtx's at this level will never
1012 : contain anything but integers and other rtx's, except for
1013 : within LABEL_REFs and SYMBOL_REFs. */
1014 0 : default:
1015 0 : gcc_unreachable ();
1016 : }
1017 : }
1018 : return true;
1019 : }
1020 :
1021 : /* If REG is a reload pseudo, try to make its class satisfying CL. */
1022 : static void
1023 3558682 : narrow_reload_pseudo_class (rtx reg, enum reg_class cl)
1024 : {
1025 3558682 : enum reg_class rclass;
1026 :
1027 : /* Do not make more accurate class from reloads generated. They are
1028 : mostly moves with a lot of constraints. Making more accurate
1029 : class may results in very narrow class and impossibility of find
1030 : registers for several reloads of one insn. */
1031 3558682 : if (INSN_UID (curr_insn) >= new_insn_uid_start)
1032 3558650 : return;
1033 3558530 : if (GET_CODE (reg) == SUBREG)
1034 172572 : reg = SUBREG_REG (reg);
1035 3558530 : if (! REG_P (reg) || (int) REGNO (reg) < new_regno_start)
1036 : return;
1037 32 : if (in_class_p (reg, cl, &rclass) && rclass != cl)
1038 13 : lra_change_class (REGNO (reg), rclass, " Change to", true);
1039 : }
1040 :
1041 : /* Searches X for any reference to a reg with the same value as REGNO,
1042 : returning the rtx of the reference found if any. Otherwise,
1043 : returns NULL_RTX. */
1044 : static rtx
1045 569945 : regno_val_use_in (unsigned int regno, rtx x)
1046 : {
1047 569945 : const char *fmt;
1048 569945 : int i, j;
1049 569945 : rtx tem;
1050 :
1051 569945 : if (REG_P (x) && lra_reg_info[REGNO (x)].val == lra_reg_info[regno].val)
1052 : return x;
1053 :
1054 569642 : fmt = GET_RTX_FORMAT (GET_CODE (x));
1055 1145566 : for (i = GET_RTX_LENGTH (GET_CODE (x)) - 1; i >= 0; i--)
1056 : {
1057 575924 : if (fmt[i] == 'e')
1058 : {
1059 7589 : if ((tem = regno_val_use_in (regno, XEXP (x, i))))
1060 : return tem;
1061 : }
1062 568335 : else if (fmt[i] == 'E')
1063 0 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
1064 0 : if ((tem = regno_val_use_in (regno , XVECEXP (x, i, j))))
1065 : return tem;
1066 : }
1067 :
1068 : return NULL_RTX;
1069 : }
1070 :
1071 : /* Return true if all current insn non-output operands except INS (it
1072 : has a negaitve end marker) do not use pseudos with the same value
1073 : as REGNO. */
1074 : static bool
1075 2 : check_conflict_input_operands (int regno, signed char *ins)
1076 : {
1077 2 : int in;
1078 2 : int n_operands = curr_static_id->n_operands;
1079 :
1080 8 : for (int nop = 0; nop < n_operands; nop++)
1081 7 : if (! curr_static_id->operand[nop].is_operator
1082 7 : && curr_static_id->operand[nop].type != OP_OUT)
1083 : {
1084 5 : for (int i = 0; (in = ins[i]) >= 0; i++)
1085 4 : if (in == nop)
1086 : break;
1087 3 : if (in < 0
1088 3 : && regno_val_use_in (regno, *curr_id->operand_loc[nop]) != NULL_RTX)
1089 : return false;
1090 : }
1091 : return true;
1092 : }
1093 :
1094 : /* Generate reloads for matching OUT and INS (array of input operand numbers
1095 : with end marker -1) with reg class GOAL_CLASS and EXCLUDE_START_HARD_REGS,
1096 : considering output operands OUTS (similar array to INS) needing to be in
1097 : different registers. Add input and output reloads correspondingly to the
1098 : lists *BEFORE and *AFTER. OUT might be negative. In this case we generate
1099 : input reloads for matched input operands INS. EARLY_CLOBBER_P is a flag
1100 : that the output operand is early clobbered for chosen alternative. */
1101 : static void
1102 1779341 : match_reload (signed char out, signed char *ins, signed char *outs,
1103 : enum reg_class goal_class, HARD_REG_SET *exclude_start_hard_regs,
1104 : rtx_insn **before, rtx_insn **after, bool early_clobber_p)
1105 : {
1106 1779341 : bool out_conflict;
1107 1779341 : int i, in;
1108 1779341 : rtx new_in_reg, new_out_reg, reg;
1109 1779341 : machine_mode inmode, outmode;
1110 1779341 : rtx in_rtx = *curr_id->operand_loc[ins[0]];
1111 1779341 : rtx out_rtx = out < 0 ? in_rtx : *curr_id->operand_loc[out];
1112 :
1113 1779341 : inmode = curr_operand_mode[ins[0]];
1114 1779341 : outmode = out < 0 ? inmode : curr_operand_mode[out];
1115 1779341 : push_to_sequence (*before);
1116 1779341 : if (inmode != outmode)
1117 : {
1118 : /* process_alt_operands has already checked that the mode sizes
1119 : are ordered. */
1120 119517 : if (partial_subreg_p (outmode, inmode))
1121 : {
1122 2355 : bool asm_p = asm_noperands (PATTERN (curr_insn)) >= 0;
1123 2355 : int hr;
1124 2355 : HARD_REG_SET temp_hard_reg_set;
1125 :
1126 19 : if (asm_p && (hr = get_hard_regno (out_rtx)) >= 0
1127 2358 : && hard_regno_nregs (hr, inmode) > 1)
1128 : {
1129 : /* See gcc.c-torture/execute/20030222-1.c.
1130 : Consider the code for 32-bit (e.g. BE) target:
1131 : int i, v; long x; x = v; asm ("" : "=r" (i) : "0" (x));
1132 : We generate the following RTL with reload insns:
1133 : 1. subreg:si(x:di, 0) = 0;
1134 : 2. subreg:si(x:di, 4) = v:si;
1135 : 3. t:di = x:di, dead x;
1136 : 4. asm ("" : "=r" (subreg:si(t:di,4)) : "0" (t:di))
1137 : 5. i:si = subreg:si(t:di,4);
1138 : If we assign hard reg of x to t, dead code elimination
1139 : will remove insn #2 and we will use uninitialized hard reg.
1140 : So exclude the hard reg of x for t. We could ignore this
1141 : problem for non-empty asm using all x value but it is hard to
1142 : check that the asm are expanded into insn really using x
1143 : and setting r. */
1144 0 : CLEAR_HARD_REG_SET (temp_hard_reg_set);
1145 0 : if (exclude_start_hard_regs != NULL)
1146 0 : temp_hard_reg_set = *exclude_start_hard_regs;
1147 0 : SET_HARD_REG_BIT (temp_hard_reg_set, hr);
1148 0 : exclude_start_hard_regs = &temp_hard_reg_set;
1149 : }
1150 4710 : reg = new_in_reg
1151 2355 : = lra_create_new_reg_with_unique_value (inmode, in_rtx, goal_class,
1152 : exclude_start_hard_regs,
1153 : "");
1154 2355 : new_out_reg = get_matching_reload_reg_subreg (outmode, reg, goal_class);
1155 2355 : LRA_SUBREG_P (new_out_reg) = 1;
1156 : /* If the input reg is dying here, we can use the same hard
1157 : register for REG and IN_RTX. We do it only for original
1158 : pseudos as reload pseudos can die although original
1159 : pseudos still live where reload pseudos dies. */
1160 1845 : if (REG_P (in_rtx) && (int) REGNO (in_rtx) < lra_new_regno_start
1161 1788 : && find_regno_note (curr_insn, REG_DEAD, REGNO (in_rtx))
1162 3214 : && (!early_clobber_p
1163 2 : || check_conflict_input_operands(REGNO (in_rtx), ins)))
1164 858 : lra_assign_reg_val (REGNO (in_rtx), REGNO (reg));
1165 : }
1166 : else
1167 : {
1168 234324 : reg = new_out_reg
1169 117162 : = lra_create_new_reg_with_unique_value (outmode, out_rtx,
1170 : goal_class,
1171 : exclude_start_hard_regs,
1172 : "");
1173 117162 : new_in_reg = get_matching_reload_reg_subreg (inmode, reg, goal_class);
1174 : /* NEW_IN_REG is non-paradoxical subreg. We don't want
1175 : NEW_OUT_REG living above. We add clobber clause for
1176 : this. This is just a temporary clobber. We can remove
1177 : it at the end of LRA work. */
1178 117162 : rtx_insn *clobber = emit_clobber (new_out_reg);
1179 117162 : LRA_TEMP_CLOBBER_P (PATTERN (clobber)) = 1;
1180 117162 : LRA_SUBREG_P (new_in_reg) = 1;
1181 117162 : if (GET_CODE (in_rtx) == SUBREG)
1182 : {
1183 3872 : rtx subreg_reg = SUBREG_REG (in_rtx);
1184 :
1185 : /* If SUBREG_REG is dying here and sub-registers IN_RTX
1186 : and NEW_IN_REG are similar, we can use the same hard
1187 : register for REG and SUBREG_REG. */
1188 3872 : if (REG_P (subreg_reg)
1189 3872 : && (int) REGNO (subreg_reg) < lra_new_regno_start
1190 3872 : && GET_MODE (subreg_reg) == outmode
1191 3518 : && known_eq (SUBREG_BYTE (in_rtx), SUBREG_BYTE (new_in_reg))
1192 3518 : && find_regno_note (curr_insn, REG_DEAD, REGNO (subreg_reg))
1193 4159 : && (! early_clobber_p
1194 0 : || check_conflict_input_operands (REGNO (subreg_reg),
1195 : ins)))
1196 287 : lra_assign_reg_val (REGNO (subreg_reg), REGNO (reg));
1197 : }
1198 : }
1199 : }
1200 : else
1201 : {
1202 : /* Pseudos have values -- see comments for lra_reg_info.
1203 : Different pseudos with the same value do not conflict even if
1204 : they live in the same place. When we create a pseudo we
1205 : assign value of original pseudo (if any) from which we
1206 : created the new pseudo. If we create the pseudo from the
1207 : input pseudo, the new pseudo will have no conflict with the
1208 : input pseudo which is wrong when the input pseudo lives after
1209 : the insn and as the new pseudo value is changed by the insn
1210 : output. Therefore we create the new pseudo from the output
1211 : except the case when we have single matched dying input
1212 : pseudo.
1213 :
1214 : We cannot reuse the current output register because we might
1215 : have a situation like "a <- a op b", where the constraints
1216 : force the second input operand ("b") to match the output
1217 : operand ("a"). "b" must then be copied into a new register
1218 : so that it doesn't clobber the current value of "a".
1219 :
1220 : We cannot use the same value if the output pseudo is
1221 : early clobbered or the input pseudo is mentioned in the
1222 : output, e.g. as an address part in memory, because
1223 : output reload will actually extend the pseudo liveness.
1224 : We don't care about eliminable hard regs here as we are
1225 : interesting only in pseudos. */
1226 :
1227 : /* Matching input's register value is the same as one of the other
1228 : output operand. Output operands in a parallel insn must be in
1229 : different registers. */
1230 1659824 : out_conflict = false;
1231 1659824 : if (REG_P (in_rtx))
1232 : {
1233 2855240 : for (i = 0; outs[i] >= 0; i++)
1234 : {
1235 1486624 : rtx other_out_rtx = *curr_id->operand_loc[outs[i]];
1236 117830 : if (outs[i] != out && REG_P (other_out_rtx)
1237 1604253 : && (regno_val_use_in (REGNO (in_rtx), other_out_rtx)
1238 : != NULL_RTX))
1239 : {
1240 : out_conflict = true;
1241 : break;
1242 : }
1243 : }
1244 : }
1245 :
1246 1659824 : new_in_reg = new_out_reg
1247 1624836 : = (! early_clobber_p && ins[1] < 0 && REG_P (in_rtx)
1248 1334648 : && (int) REGNO (in_rtx) < lra_new_regno_start
1249 1334348 : && find_regno_note (curr_insn, REG_DEAD, REGNO (in_rtx))
1250 444726 : && (out < 0
1251 444726 : || regno_val_use_in (REGNO (in_rtx), out_rtx) == NULL_RTX)
1252 444660 : && !out_conflict
1253 2104482 : ? lra_create_new_reg (inmode, in_rtx, goal_class,
1254 : exclude_start_hard_regs, "")
1255 1215166 : : lra_create_new_reg_with_unique_value (outmode, out_rtx, goal_class,
1256 : exclude_start_hard_regs,
1257 : ""));
1258 : }
1259 : /* In operand can be got from transformations before processing insn
1260 : constraints. One example of such transformations is subreg
1261 : reloading (see function simplify_operand_subreg). The new
1262 : pseudos created by the transformations might have inaccurate
1263 : class (ALL_REGS) and we should make their classes more
1264 : accurate. */
1265 1779341 : narrow_reload_pseudo_class (in_rtx, goal_class);
1266 1779341 : lra_emit_move (copy_rtx (new_in_reg), in_rtx);
1267 1779341 : *before = end_sequence ();
1268 : /* Add the new pseudo to consider values of subsequent input reload
1269 : pseudos. */
1270 1779341 : lra_assert (curr_insn_input_reloads_num < LRA_MAX_INSN_RELOADS);
1271 1779341 : curr_insn_input_reloads[curr_insn_input_reloads_num].input = in_rtx;
1272 1779341 : curr_insn_input_reloads[curr_insn_input_reloads_num].match_p = true;
1273 1779341 : curr_insn_input_reloads[curr_insn_input_reloads_num].early_clobber_p = false;
1274 1779341 : curr_insn_input_reloads[curr_insn_input_reloads_num++].reg = new_in_reg;
1275 3558683 : for (i = 0; (in = ins[i]) >= 0; i++)
1276 1779342 : if (GET_MODE (*curr_id->operand_loc[in]) == VOIDmode
1277 1755782 : || GET_MODE (new_in_reg) == GET_MODE (*curr_id->operand_loc[in]))
1278 1779341 : *curr_id->operand_loc[in] = new_in_reg;
1279 : else
1280 : {
1281 1 : lra_assert
1282 : (GET_MODE (new_out_reg) == GET_MODE (*curr_id->operand_loc[in]));
1283 1 : *curr_id->operand_loc[in] = new_out_reg;
1284 : }
1285 1779341 : lra_update_dups (curr_id, ins);
1286 1779341 : if (out < 0)
1287 : return;
1288 : /* See a comment for the input operand above. */
1289 1779341 : narrow_reload_pseudo_class (out_rtx, goal_class);
1290 1779341 : reg = SUBREG_P (out_rtx) ? SUBREG_REG (out_rtx) : out_rtx;
1291 1779341 : if (find_reg_note (curr_insn, REG_UNUSED, reg) == NULL_RTX
1292 1779341 : && (!REG_P (reg) || !ira_former_scratch_p (REGNO (reg))))
1293 : {
1294 1682630 : start_sequence ();
1295 : /* If we had strict_low_part, use it also in reload to keep other
1296 : parts unchanged but do it only for regs as strict_low_part
1297 : has no sense for memory and probably there is no insn pattern
1298 : to match the reload insn in memory case. */
1299 1682630 : if (out >= 0 && curr_static_id->operand[out].strict_low && REG_P (reg))
1300 0 : out_rtx = gen_rtx_STRICT_LOW_PART (VOIDmode, out_rtx);
1301 1682630 : lra_emit_move (out_rtx, copy_rtx (new_out_reg));
1302 1682630 : emit_insn (*after);
1303 1682630 : *after = end_sequence ();
1304 : }
1305 1779341 : *curr_id->operand_loc[out] = new_out_reg;
1306 1779341 : lra_update_dup (curr_id, out);
1307 : }
1308 :
1309 : /* Return register class which is union of all reg classes in insn
1310 : constraint alternative string starting with P. */
1311 : static enum reg_class
1312 0 : reg_class_from_constraints (const char *p)
1313 : {
1314 0 : int c, len;
1315 0 : enum reg_class op_class = NO_REGS;
1316 :
1317 0 : do
1318 0 : switch ((c = *p, len = CONSTRAINT_LEN (c, p)), c)
1319 : {
1320 : case '#':
1321 : case ',':
1322 : return op_class;
1323 :
1324 0 : case 'g':
1325 0 : op_class = reg_class_subunion[op_class][GENERAL_REGS];
1326 0 : break;
1327 :
1328 0 : default:
1329 0 : enum constraint_num cn = lookup_constraint (p);
1330 0 : enum reg_class cl = reg_class_for_constraint (cn);
1331 0 : if (cl == NO_REGS)
1332 : {
1333 0 : if (insn_extra_address_constraint (cn))
1334 0 : op_class
1335 0 : = (reg_class_subunion
1336 0 : [op_class][base_reg_class (VOIDmode, ADDR_SPACE_GENERIC,
1337 0 : ADDRESS, SCRATCH)]);
1338 : break;
1339 : }
1340 :
1341 0 : op_class = reg_class_subunion[op_class][cl];
1342 0 : break;
1343 : }
1344 0 : while ((p += len), c);
1345 : return op_class;
1346 : }
1347 :
1348 : /* If OP is a register, return the class of the register as per
1349 : get_reg_class, otherwise return NO_REGS. */
1350 : static inline enum reg_class
1351 167743624 : get_op_class (rtx op)
1352 : {
1353 139467244 : return REG_P (op) ? get_reg_class (REGNO (op)) : NO_REGS;
1354 : }
1355 :
1356 : /* Return generated insn mem_pseudo:=val if TO_P or val:=mem_pseudo
1357 : otherwise. If modes of MEM_PSEUDO and VAL are different, use
1358 : SUBREG for VAL to make them equal. */
1359 : static rtx_insn *
1360 1353672 : emit_spill_move (bool to_p, rtx mem_pseudo, rtx val)
1361 : {
1362 1353672 : if (GET_MODE (mem_pseudo) != GET_MODE (val))
1363 : {
1364 : /* Usually size of mem_pseudo is greater than val size but in
1365 : rare cases it can be less as it can be defined by target
1366 : dependent macro HARD_REGNO_CALLER_SAVE_MODE. */
1367 3158 : if (! MEM_P (val))
1368 : {
1369 3158 : val = gen_lowpart_SUBREG (GET_MODE (mem_pseudo),
1370 : GET_CODE (val) == SUBREG
1371 : ? SUBREG_REG (val) : val);
1372 3158 : LRA_SUBREG_P (val) = 1;
1373 : }
1374 : else
1375 : {
1376 0 : mem_pseudo = gen_lowpart_SUBREG (GET_MODE (val), mem_pseudo);
1377 0 : LRA_SUBREG_P (mem_pseudo) = 1;
1378 : }
1379 : }
1380 1353672 : return to_p ? gen_move_insn (mem_pseudo, val)
1381 683217 : : gen_move_insn (val, mem_pseudo);
1382 : }
1383 :
1384 : /* Process a special case insn (register move), return true if we
1385 : don't need to process it anymore. INSN should be a single set
1386 : insn. Set up that RTL was changed through CHANGE_P and that hook
1387 : TARGET_SECONDARY_MEMORY_NEEDED says to use secondary memory through
1388 : SEC_MEM_P. */
1389 : static bool
1390 77563947 : check_and_process_move (bool *change_p, bool *sec_mem_p ATTRIBUTE_UNUSED)
1391 : {
1392 77563947 : int sregno, dregno;
1393 77563947 : rtx dest, src, dreg, sreg, new_reg, scratch_reg;
1394 77563947 : rtx_insn *before;
1395 77563947 : enum reg_class dclass, sclass, secondary_class;
1396 77563947 : secondary_reload_info sri;
1397 :
1398 77563947 : lra_assert (curr_insn_set != NULL_RTX);
1399 77563947 : dreg = dest = SET_DEST (curr_insn_set);
1400 77563947 : sreg = src = SET_SRC (curr_insn_set);
1401 77563947 : if (GET_CODE (dest) == SUBREG)
1402 1207143 : dreg = SUBREG_REG (dest);
1403 77563947 : if (GET_CODE (src) == SUBREG)
1404 1277957 : sreg = SUBREG_REG (src);
1405 77563947 : if (! (REG_P (dreg) || MEM_P (dreg)) || ! (REG_P (sreg) || MEM_P (sreg)))
1406 : return false;
1407 36181665 : sclass = dclass = NO_REGS;
1408 36181665 : if (REG_P (dreg))
1409 23314819 : dclass = get_reg_class (REGNO (dreg));
1410 23314819 : gcc_assert (dclass < LIM_REG_CLASSES && dclass >= NO_REGS);
1411 36181665 : if (dclass == ALL_REGS)
1412 : /* ALL_REGS is used for new pseudos created by transformations
1413 : like reload of SUBREG_REG (see function
1414 : simplify_operand_subreg). We don't know their class yet. We
1415 : should figure out the class from processing the insn
1416 : constraints not in this fast path function. Even if ALL_REGS
1417 : were a right class for the pseudo, secondary_... hooks usually
1418 : are not define for ALL_REGS. */
1419 : return false;
1420 36179423 : if (REG_P (sreg))
1421 20212692 : sclass = get_reg_class (REGNO (sreg));
1422 20212692 : gcc_assert (sclass < LIM_REG_CLASSES && sclass >= NO_REGS);
1423 36179423 : if (sclass == ALL_REGS)
1424 : /* See comments above. */
1425 : return false;
1426 36179423 : if (sclass == NO_REGS && dclass == NO_REGS)
1427 : return false;
1428 34690181 : if (targetm.secondary_memory_needed (GET_MODE (src), sclass, dclass)
1429 34690181 : && ((sclass != NO_REGS && dclass != NO_REGS)
1430 0 : || (GET_MODE (src)
1431 0 : != targetm.secondary_memory_needed_mode (GET_MODE (src)))))
1432 : {
1433 12952 : *sec_mem_p = true;
1434 12952 : return false;
1435 : }
1436 34677229 : if (! REG_P (dreg) || ! REG_P (sreg))
1437 : return false;
1438 7858141 : sri.prev_sri = NULL;
1439 7858141 : sri.icode = CODE_FOR_nothing;
1440 7858141 : sri.extra_cost = 0;
1441 7858141 : secondary_class = NO_REGS;
1442 : /* Set up hard register for a reload pseudo for hook
1443 : secondary_reload because some targets just ignore unassigned
1444 : pseudos in the hook. */
1445 7858141 : if (dclass != NO_REGS && lra_get_regno_hard_regno (REGNO (dreg)) < 0)
1446 : {
1447 2930561 : dregno = REGNO (dreg);
1448 2930561 : reg_renumber[dregno] = ira_class_hard_regs[dclass][0];
1449 : }
1450 : else
1451 : dregno = -1;
1452 7858141 : if (sclass != NO_REGS && lra_get_regno_hard_regno (REGNO (sreg)) < 0)
1453 : {
1454 1297451 : sregno = REGNO (sreg);
1455 1297451 : reg_renumber[sregno] = ira_class_hard_regs[sclass][0];
1456 : }
1457 : else
1458 : sregno = -1;
1459 7858141 : if (sclass != NO_REGS)
1460 3976751 : secondary_class
1461 7953502 : = (enum reg_class) targetm.secondary_reload (false, dest,
1462 : (reg_class_t) sclass,
1463 3976751 : GET_MODE (src), &sri);
1464 3976751 : if (sclass == NO_REGS
1465 3976751 : || ((secondary_class != NO_REGS || sri.icode != CODE_FOR_nothing)
1466 1337 : && dclass != NO_REGS))
1467 : {
1468 3881390 : enum reg_class old_sclass = secondary_class;
1469 3881390 : secondary_reload_info old_sri = sri;
1470 :
1471 3881390 : sri.prev_sri = NULL;
1472 3881390 : sri.icode = CODE_FOR_nothing;
1473 3881390 : sri.extra_cost = 0;
1474 3881390 : secondary_class
1475 7762780 : = (enum reg_class) targetm.secondary_reload (true, src,
1476 : (reg_class_t) dclass,
1477 3881390 : GET_MODE (src), &sri);
1478 : /* Check the target hook consistency. */
1479 3881390 : lra_assert
1480 : ((secondary_class == NO_REGS && sri.icode == CODE_FOR_nothing)
1481 : || (old_sclass == NO_REGS && old_sri.icode == CODE_FOR_nothing)
1482 : || (secondary_class == old_sclass && sri.icode == old_sri.icode));
1483 : }
1484 7858141 : if (sregno >= 0)
1485 1297451 : reg_renumber [sregno] = -1;
1486 7858141 : if (dregno >= 0)
1487 2930561 : reg_renumber [dregno] = -1;
1488 7858141 : if (secondary_class == NO_REGS && sri.icode == CODE_FOR_nothing)
1489 : return false;
1490 1338 : *change_p = true;
1491 1338 : new_reg = NULL_RTX;
1492 0 : if (secondary_class != NO_REGS)
1493 1338 : new_reg = lra_create_new_reg_with_unique_value (GET_MODE (src), NULL_RTX,
1494 : secondary_class, NULL,
1495 : "secondary");
1496 1338 : start_sequence ();
1497 1338 : if (sri.icode == CODE_FOR_nothing)
1498 1338 : lra_emit_move (new_reg, src);
1499 : else
1500 : {
1501 0 : enum reg_class scratch_class;
1502 :
1503 0 : scratch_class = (reg_class_from_constraints
1504 0 : (insn_data[sri.icode].operand[2].constraint));
1505 0 : scratch_reg = (lra_create_new_reg_with_unique_value
1506 0 : (insn_data[sri.icode].operand[2].mode, NULL_RTX,
1507 : scratch_class, NULL, "scratch"));
1508 0 : emit_insn (GEN_FCN (sri.icode) (new_reg != NULL_RTX ? new_reg : dest,
1509 : src, scratch_reg));
1510 : }
1511 1338 : before = end_sequence ();
1512 1338 : lra_process_new_insns (curr_insn, before, NULL, "Inserting the move");
1513 1338 : if (new_reg != NULL_RTX)
1514 1338 : SET_SRC (curr_insn_set) = new_reg;
1515 : else
1516 : {
1517 0 : if (lra_dump_file != NULL)
1518 : {
1519 0 : fprintf (lra_dump_file, "Deleting move %u\n", INSN_UID (curr_insn));
1520 0 : dump_insn_slim (lra_dump_file, curr_insn);
1521 : }
1522 0 : lra_set_insn_deleted (curr_insn);
1523 0 : return true;
1524 : }
1525 1338 : return false;
1526 : }
1527 :
1528 : /* The following data describe the result of process_alt_operands.
1529 : The data are used in curr_insn_transform to generate reloads. */
1530 :
1531 : /* The chosen reg classes which should be used for the corresponding
1532 : operands. */
1533 : static enum reg_class goal_alt[MAX_RECOG_OPERANDS];
1534 : /* Hard registers which cannot be a start hard register for the corresponding
1535 : operands. */
1536 : static HARD_REG_SET goal_alt_exclude_start_hard_regs[MAX_RECOG_OPERANDS];
1537 : /* True if the operand should be the same as another operand and that
1538 : other operand does not need a reload. */
1539 : static bool goal_alt_match_win[MAX_RECOG_OPERANDS];
1540 : /* True if the operand does not need a reload. */
1541 : static bool goal_alt_win[MAX_RECOG_OPERANDS];
1542 : /* True if the operand can be offsetable memory. */
1543 : static bool goal_alt_offmemok[MAX_RECOG_OPERANDS];
1544 : /* The number of an operand to which given operand can be matched to. */
1545 : static int goal_alt_matches[MAX_RECOG_OPERANDS];
1546 : /* The number of elements in the following array. */
1547 : static int goal_alt_dont_inherit_ops_num;
1548 : /* Numbers of operands whose reload pseudos should not be inherited. */
1549 : static int goal_alt_dont_inherit_ops[MAX_RECOG_OPERANDS];
1550 : /* True if we should try only this alternative for the next constraint sub-pass
1551 : to speed up the sub-pass. */
1552 : static bool goal_reuse_alt_p;
1553 : /* True if the insn commutative operands should be swapped. */
1554 : static bool goal_alt_swapped;
1555 : /* The chosen insn alternative. */
1556 : static int goal_alt_number;
1557 : /* True if output reload of the stack pointer should be generated. */
1558 : static bool goal_alt_out_sp_reload_p;
1559 :
1560 : /* True if the corresponding operand is the result of an equivalence
1561 : substitution. */
1562 : static bool equiv_substition_p[MAX_RECOG_OPERANDS];
1563 :
1564 : /* The following five variables are used to choose the best insn
1565 : alternative. They reflect final characteristics of the best
1566 : alternative. */
1567 :
1568 : /* Number of necessary reloads and overall cost reflecting the
1569 : previous value and other unpleasantness of the best alternative. */
1570 : static int best_losers, best_overall;
1571 : /* Overall number hard registers used for reloads. For example, on
1572 : some targets we need 2 general registers to reload DFmode and only
1573 : one floating point register. */
1574 : static int best_reload_nregs;
1575 : /* Overall number reflecting distances of previous reloading the same
1576 : value. The distances are counted from the current BB start. It is
1577 : used to improve inheritance chances. */
1578 : static int best_reload_sum;
1579 :
1580 : /* True if the current insn should have no correspondingly input or
1581 : output reloads. */
1582 : static bool no_input_reloads_p, no_output_reloads_p;
1583 :
1584 : /* True if we swapped the commutative operands in the current
1585 : insn. */
1586 : static int curr_swapped;
1587 :
1588 : /* if CHECK_ONLY_P is false, arrange for address element *LOC to be a
1589 : register of class CL. Add any input reloads to list BEFORE. AFTER
1590 : is nonnull if *LOC is an automodified value; handle that case by
1591 : adding the required output reloads to list AFTER. Return true if
1592 : the RTL was changed.
1593 :
1594 : if CHECK_ONLY_P is true, check that the *LOC is a correct address
1595 : register. Return false if the address register is correct. */
1596 : static bool
1597 35917099 : process_addr_reg (rtx *loc, bool check_only_p, rtx_insn **before, rtx_insn **after,
1598 : enum reg_class cl)
1599 : {
1600 35917099 : int regno;
1601 35917099 : enum reg_class rclass, new_class;
1602 35917099 : rtx reg;
1603 35917099 : rtx new_reg;
1604 35917099 : machine_mode mode;
1605 35917099 : bool subreg_p, before_p = false;
1606 :
1607 35917099 : subreg_p = GET_CODE (*loc) == SUBREG;
1608 35917099 : if (subreg_p)
1609 : {
1610 16585 : reg = SUBREG_REG (*loc);
1611 16585 : mode = GET_MODE (reg);
1612 :
1613 : /* For mode with size bigger than ptr_mode, there unlikely to be "mov"
1614 : between two registers with different classes, but there normally will
1615 : be "mov" which transfers element of vector register into the general
1616 : register, and this normally will be a subreg which should be reloaded
1617 : as a whole. This is particularly likely to be triggered when
1618 : -fno-split-wide-types specified. */
1619 16585 : if (!REG_P (reg)
1620 16585 : || in_class_p (reg, cl, &new_class)
1621 18941 : || known_le (GET_MODE_SIZE (mode), GET_MODE_SIZE (ptr_mode)))
1622 16585 : loc = &SUBREG_REG (*loc);
1623 : }
1624 :
1625 35917099 : reg = *loc;
1626 35917099 : mode = GET_MODE (reg);
1627 35917099 : if (! REG_P (reg))
1628 : {
1629 0 : if (check_only_p)
1630 : return true;
1631 : /* Always reload memory in an address even if the target supports
1632 : such addresses. */
1633 0 : new_reg = lra_create_new_reg_with_unique_value (mode, reg, cl, NULL,
1634 : "address");
1635 0 : before_p = true;
1636 : }
1637 : else
1638 : {
1639 35917099 : regno = REGNO (reg);
1640 35917099 : rclass = get_reg_class (regno);
1641 35917099 : if (! check_only_p
1642 35917099 : && (*loc = get_equiv_with_elimination (reg, curr_insn)) != reg)
1643 : {
1644 113198 : if (lra_dump_file != NULL)
1645 : {
1646 0 : fprintf (lra_dump_file,
1647 : "Changing pseudo %d in address of insn %u on equiv ",
1648 0 : REGNO (reg), INSN_UID (curr_insn));
1649 0 : dump_value_slim (lra_dump_file, *loc, 1);
1650 0 : fprintf (lra_dump_file, "\n");
1651 : }
1652 113198 : rtx new_equiv = copy_rtx (*loc);
1653 113198 : if (lra_pointer_equiv_set_in (*loc))
1654 107963 : lra_pointer_equiv_set_add (new_equiv);
1655 113198 : *loc = new_equiv;
1656 : }
1657 35917099 : if (*loc != reg || ! in_class_p (reg, cl, &new_class))
1658 : {
1659 483397 : if (check_only_p)
1660 : return true;
1661 483397 : reg = *loc;
1662 483397 : if (get_reload_reg (after == NULL ? OP_IN : OP_INOUT,
1663 : mode, reg, cl, NULL,
1664 : subreg_p, false, "address", &new_reg))
1665 : before_p = true;
1666 : }
1667 35433702 : else if (new_class != NO_REGS && rclass != new_class)
1668 : {
1669 471085 : if (check_only_p)
1670 : return true;
1671 471085 : lra_change_class (regno, new_class, " Change to", true);
1672 471085 : return false;
1673 : }
1674 : else
1675 : return false;
1676 : }
1677 0 : if (before_p)
1678 : {
1679 475131 : push_to_sequence (*before);
1680 475131 : lra_emit_move (new_reg, reg);
1681 475131 : *before = end_sequence ();
1682 : }
1683 483397 : *loc = new_reg;
1684 483397 : if (after != NULL)
1685 : {
1686 0 : start_sequence ();
1687 0 : lra_emit_move (before_p ? copy_rtx (reg) : reg, new_reg);
1688 0 : emit_insn (*after);
1689 0 : *after = end_sequence ();
1690 : }
1691 : return true;
1692 : }
1693 :
1694 : /* Insert move insn in simplify_operand_subreg. BEFORE returns
1695 : the insn to be inserted before curr insn. AFTER returns the
1696 : the insn to be inserted after curr insn. ORIGREG and NEWREG
1697 : are the original reg and new reg for reload. */
1698 : static void
1699 458 : insert_move_for_subreg (rtx_insn **before, rtx_insn **after, rtx origreg,
1700 : rtx newreg)
1701 : {
1702 458 : if (before)
1703 : {
1704 458 : push_to_sequence (*before);
1705 458 : lra_emit_move (newreg, origreg);
1706 458 : *before = end_sequence ();
1707 : }
1708 458 : if (after)
1709 : {
1710 0 : start_sequence ();
1711 0 : lra_emit_move (origreg, newreg);
1712 0 : emit_insn (*after);
1713 0 : *after = end_sequence ();
1714 : }
1715 458 : }
1716 :
1717 : static bool valid_address_p (machine_mode mode, rtx addr, addr_space_t as);
1718 : static bool process_address (int, bool, rtx_insn **, rtx_insn **);
1719 :
1720 : /* Make reloads for subreg in operand NOP with internal subreg mode
1721 : REG_MODE, add new reloads for further processing. Return true if
1722 : any change was done. */
1723 : static bool
1724 180364252 : simplify_operand_subreg (int nop, machine_mode reg_mode)
1725 : {
1726 180364252 : int hard_regno, inner_hard_regno;
1727 180364252 : rtx_insn *before, *after;
1728 180364252 : machine_mode mode, innermode;
1729 180364252 : rtx reg, new_reg;
1730 180364252 : rtx operand = *curr_id->operand_loc[nop];
1731 180364252 : enum reg_class regclass;
1732 180364252 : enum op_type type;
1733 :
1734 180364252 : before = after = NULL;
1735 :
1736 180364252 : if (GET_CODE (operand) != SUBREG)
1737 : return false;
1738 :
1739 3824458 : mode = GET_MODE (operand);
1740 3824458 : reg = SUBREG_REG (operand);
1741 3824458 : innermode = GET_MODE (reg);
1742 3824458 : type = curr_static_id->operand[nop].type;
1743 3824458 : if (MEM_P (reg))
1744 : {
1745 11856 : const bool addr_was_valid
1746 11856 : = valid_address_p (innermode, XEXP (reg, 0), MEM_ADDR_SPACE (reg));
1747 11856 : alter_subreg (curr_id->operand_loc[nop], false);
1748 11856 : rtx subst = *curr_id->operand_loc[nop];
1749 11856 : lra_assert (MEM_P (subst));
1750 11856 : const bool addr_is_valid = valid_address_p (GET_MODE (subst),
1751 : XEXP (subst, 0),
1752 11856 : MEM_ADDR_SPACE (subst));
1753 11856 : if (!addr_was_valid
1754 11856 : || addr_is_valid
1755 11856 : || ((get_constraint_type (lookup_constraint
1756 0 : (curr_static_id->operand[nop].constraint))
1757 : != CT_SPECIAL_MEMORY)
1758 : /* We still can reload address and if the address is
1759 : valid, we can remove subreg without reloading its
1760 : inner memory. */
1761 0 : && valid_address_p (GET_MODE (subst),
1762 0 : regno_reg_rtx
1763 : [ira_class_hard_regs
1764 0 : [base_reg_class (GET_MODE (subst),
1765 0 : MEM_ADDR_SPACE (subst),
1766 : ADDRESS, SCRATCH,
1767 0 : subst, curr_insn)][0]],
1768 0 : MEM_ADDR_SPACE (subst))))
1769 : {
1770 : /* If we change the address for a paradoxical subreg of memory, the
1771 : new address might violate the necessary alignment or the access
1772 : might be slow; take this into consideration. We need not worry
1773 : about accesses beyond allocated memory for paradoxical memory
1774 : subregs as we don't substitute such equiv memory (see processing
1775 : equivalences in function lra_constraints) and because for spilled
1776 : pseudos we allocate stack memory enough for the biggest
1777 : corresponding paradoxical subreg.
1778 :
1779 : However, do not blindly simplify a (subreg (mem ...)) for
1780 : WORD_REGISTER_OPERATIONS targets as this may lead to loading junk
1781 : data into a register when the inner is narrower than outer or
1782 : missing important data from memory when the inner is wider than
1783 : outer. This rule only applies to modes that are no wider than
1784 : a word.
1785 :
1786 : If valid memory becomes invalid after subreg elimination
1787 : and address might be different we still have to reload
1788 : memory.
1789 : */
1790 11856 : if ((! addr_was_valid
1791 : || addr_is_valid
1792 0 : || known_eq (GET_MODE_SIZE (mode), GET_MODE_SIZE (innermode)))
1793 11856 : && !(maybe_ne (GET_MODE_PRECISION (mode),
1794 11856 : GET_MODE_PRECISION (innermode))
1795 13085 : && known_le (GET_MODE_SIZE (mode), UNITS_PER_WORD)
1796 20242 : && known_le (GET_MODE_SIZE (innermode), UNITS_PER_WORD)
1797 : && WORD_REGISTER_OPERATIONS)
1798 24813 : && (!(MEM_ALIGN (subst) < GET_MODE_ALIGNMENT (mode)
1799 1101 : && targetm.slow_unaligned_access (mode, MEM_ALIGN (subst)))
1800 0 : || (MEM_ALIGN (reg) < GET_MODE_ALIGNMENT (innermode)
1801 0 : && targetm.slow_unaligned_access (innermode,
1802 0 : MEM_ALIGN (reg)))))
1803 : return true;
1804 :
1805 0 : *curr_id->operand_loc[nop] = operand;
1806 :
1807 : /* But if the address was not valid, we cannot reload the MEM without
1808 : reloading the address first. */
1809 0 : if (!addr_was_valid)
1810 0 : process_address (nop, false, &before, &after);
1811 :
1812 : /* INNERMODE is fast, MODE slow. Reload the mem in INNERMODE. */
1813 0 : enum reg_class rclass
1814 0 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
1815 0 : if (get_reload_reg (curr_static_id->operand[nop].type, innermode,
1816 : reg, rclass, NULL,
1817 : true, false, "slow/invalid mem", &new_reg))
1818 : {
1819 0 : bool insert_before, insert_after;
1820 0 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
1821 :
1822 0 : insert_before = (type != OP_OUT
1823 0 : || partial_subreg_p (mode, innermode));
1824 0 : insert_after = type != OP_IN;
1825 0 : insert_move_for_subreg (insert_before ? &before : NULL,
1826 : insert_after ? &after : NULL,
1827 : reg, new_reg);
1828 : }
1829 0 : SUBREG_REG (operand) = new_reg;
1830 :
1831 : /* Convert to MODE. */
1832 0 : reg = operand;
1833 0 : rclass
1834 0 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
1835 0 : if (get_reload_reg (curr_static_id->operand[nop].type, mode, reg,
1836 : rclass, NULL,
1837 : true, false, "slow/invalid mem", &new_reg))
1838 : {
1839 0 : bool insert_before, insert_after;
1840 0 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
1841 :
1842 0 : insert_before = type != OP_OUT;
1843 0 : insert_after = type != OP_IN;
1844 0 : insert_move_for_subreg (insert_before ? &before : NULL,
1845 : insert_after ? &after : NULL,
1846 : reg, new_reg);
1847 : }
1848 0 : *curr_id->operand_loc[nop] = new_reg;
1849 0 : lra_process_new_insns (curr_insn, before, after,
1850 : "Inserting slow/invalid mem reload");
1851 0 : return true;
1852 : }
1853 :
1854 : /* If the address was valid and became invalid, prefer to reload
1855 : the memory. Typical case is when the index scale should
1856 : correspond the memory. */
1857 0 : *curr_id->operand_loc[nop] = operand;
1858 : /* Do not return false here as the MEM_P (reg) will be processed
1859 : later in this function. */
1860 : }
1861 3812602 : else if (REG_P (reg) && REGNO (reg) < FIRST_PSEUDO_REGISTER)
1862 : {
1863 : /* A narrowing subreg of a hard register that is not representable as a
1864 : hard register (its offset does not fall on a register boundary) cannot
1865 : be turned into one, and would otherwise be resolved to the wrong part
1866 : of the register. Reload it through memory so that the correct bytes
1867 : are accessed, as is done for pseudos below. Leave the frame, arg and
1868 : stack pointers alone: simplify_subreg_regno can reject them simply
1869 : because reload is not finished yet. */
1870 60 : if (partial_subreg_p (mode, innermode)
1871 60 : && REGNO (reg) != FRAME_POINTER_REGNUM
1872 : && REGNO (reg) != ARG_POINTER_REGNUM
1873 : && REGNO (reg) != STACK_POINTER_REGNUM
1874 60 : && simplify_subreg_regno (REGNO (reg), innermode,
1875 0 : SUBREG_BYTE (operand), mode) < 0)
1876 : {
1877 0 : if (get_reload_reg (type, innermode, reg, NO_REGS, NULL,
1878 : true, false, "non-representable subreg", &new_reg))
1879 : {
1880 0 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
1881 0 : bool insert_before = (type != OP_OUT
1882 0 : || read_modify_subreg_p (operand));
1883 0 : bool insert_after = (type != OP_IN);
1884 0 : insert_move_for_subreg (insert_before ? &before : NULL,
1885 : insert_after ? &after : NULL,
1886 : reg, new_reg);
1887 : }
1888 0 : SUBREG_REG (operand) = new_reg;
1889 0 : lra_process_new_insns (curr_insn, before, after,
1890 : "Inserting non-representable subreg reload");
1891 0 : return true;
1892 : }
1893 60 : alter_subreg (curr_id->operand_loc[nop], false);
1894 60 : return true;
1895 : }
1896 3812542 : else if (CONSTANT_P (reg))
1897 : {
1898 : /* Try to simplify subreg of constant. It is usually result of
1899 : equivalence substitution. */
1900 45076 : if (innermode == VOIDmode
1901 45076 : && (innermode = original_subreg_reg_mode[nop]) == VOIDmode)
1902 0 : innermode = curr_static_id->operand[nop].mode;
1903 45076 : if ((new_reg = simplify_subreg (mode, reg, innermode,
1904 45076 : SUBREG_BYTE (operand))) != NULL_RTX)
1905 : {
1906 44660 : *curr_id->operand_loc[nop] = new_reg;
1907 44660 : return true;
1908 : }
1909 : }
1910 : /* Put constant into memory when we have mixed modes. It generates
1911 : a better code in most cases as it does not need a secondary
1912 : reload memory. It also prevents LRA looping when LRA is using
1913 : secondary reload memory again and again. */
1914 832 : if (CONSTANT_P (reg) && CONST_POOL_OK_P (reg_mode, reg)
1915 3768298 : && SCALAR_INT_MODE_P (reg_mode) != SCALAR_INT_MODE_P (mode))
1916 : {
1917 8 : SUBREG_REG (operand) = force_const_mem (reg_mode, reg);
1918 8 : alter_subreg (curr_id->operand_loc[nop], false);
1919 8 : return true;
1920 : }
1921 3767874 : auto fp_subreg_can_be_simplified_after_reload_p = [] (machine_mode innermode,
1922 : poly_uint64 offset,
1923 : machine_mode mode) {
1924 0 : reload_completed = 1;
1925 0 : bool res = simplify_subreg_regno (FRAME_POINTER_REGNUM,
1926 : innermode,
1927 0 : offset, mode) >= 0;
1928 0 : reload_completed = 0;
1929 0 : return res;
1930 : };
1931 : /* Force a reload of the SUBREG_REG if this is a constant or PLUS or
1932 : if there may be a problem accessing OPERAND in the outer
1933 : mode. */
1934 3767874 : if ((REG_P (reg)
1935 3767416 : && REGNO (reg) >= FIRST_PSEUDO_REGISTER
1936 3767416 : && (hard_regno = lra_get_regno_hard_regno (REGNO (reg))) >= 0
1937 : /* Don't reload paradoxical subregs because we could be looping
1938 : having repeatedly final regno out of hard regs range. */
1939 3195195 : && (hard_regno_nregs (hard_regno, innermode)
1940 3195195 : >= hard_regno_nregs (hard_regno, mode))
1941 3189877 : && simplify_subreg_regno (hard_regno, innermode,
1942 3189877 : SUBREG_BYTE (operand), mode) < 0
1943 : /* Exclude reloading of frame pointer in subreg if frame pointer can not
1944 : be simplified here only because the reload is not finished yet. */
1945 846 : && (hard_regno != FRAME_POINTER_REGNUM
1946 458 : || !fp_subreg_can_be_simplified_after_reload_p (innermode,
1947 0 : SUBREG_BYTE (operand),
1948 : mode))
1949 : /* Don't reload subreg for matching reload. It is actually
1950 : valid subreg in LRA. */
1951 846 : && ! LRA_SUBREG_P (operand))
1952 7535290 : || CONSTANT_P (reg) || GET_CODE (reg) == PLUS || MEM_P (reg))
1953 : {
1954 458 : enum reg_class rclass;
1955 :
1956 458 : if (REG_P (reg))
1957 : /* There is a big probability that we will get the same class
1958 : for the new pseudo and we will get the same insn which
1959 : means infinite looping. So spill the new pseudo. */
1960 : rclass = NO_REGS;
1961 : else
1962 : /* The class will be defined later in curr_insn_transform. */
1963 458 : rclass
1964 458 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
1965 :
1966 458 : if (get_reload_reg (curr_static_id->operand[nop].type, reg_mode, reg,
1967 : rclass, NULL,
1968 : true, false, "subreg reg", &new_reg))
1969 : {
1970 458 : bool insert_before, insert_after;
1971 458 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
1972 :
1973 916 : insert_before = (type != OP_OUT
1974 458 : || read_modify_subreg_p (operand));
1975 458 : insert_after = (type != OP_IN);
1976 916 : insert_move_for_subreg (insert_before ? &before : NULL,
1977 : insert_after ? &after : NULL,
1978 : reg, new_reg);
1979 : }
1980 458 : SUBREG_REG (operand) = new_reg;
1981 458 : lra_process_new_insns (curr_insn, before, after,
1982 : "Inserting subreg reload");
1983 458 : return true;
1984 : }
1985 : /* Force a reload for a paradoxical subreg. For paradoxical subreg,
1986 : IRA allocates hardreg to the inner pseudo reg according to its mode
1987 : instead of the outermode, so the size of the hardreg may not be enough
1988 : to contain the outermode operand, in that case we may need to insert
1989 : reload for the reg. For the following two types of paradoxical subreg,
1990 : we need to insert reload:
1991 : 1. If the op_type is OP_IN, and the hardreg could not be paired with
1992 : other hardreg to contain the outermode operand
1993 : (checked by in_hard_reg_set_p), we need to insert the reload.
1994 : 2. If the op_type is OP_OUT or OP_INOUT.
1995 :
1996 : Here is a paradoxical subreg example showing how the reload is generated:
1997 :
1998 : (insn 5 4 7 2 (set (reg:TI 106 [ __comp ])
1999 : (subreg:TI (reg:DI 107 [ __comp ]) 0)) {*movti_internal_rex64}
2000 :
2001 : In IRA, reg107 is allocated to a DImode hardreg. We use x86-64 as example
2002 : here, if reg107 is assigned to hardreg R15, because R15 is the last
2003 : hardreg, compiler cannot find another hardreg to pair with R15 to
2004 : contain TImode data. So we insert a TImode reload reg180 for it.
2005 : After reload is inserted:
2006 :
2007 : (insn 283 0 0 (set (subreg:DI (reg:TI 180 [orig:107 __comp ] [107]) 0)
2008 : (reg:DI 107 [ __comp ])) -1
2009 : (insn 5 4 7 2 (set (reg:TI 106 [ __comp ])
2010 : (subreg:TI (reg:TI 180 [orig:107 __comp ] [107]) 0)) {*movti_internal_rex64}
2011 :
2012 : Two reload hard registers will be allocated to reg180 to save TImode data
2013 : in LRA_assign.
2014 :
2015 : For LRA pseudos this should normally be handled by the biggest_mode
2016 : mechanism. However, it's possible for new uses of an LRA pseudo
2017 : to be introduced after we've allocated it, such as when undoing
2018 : inheritance, and the allocated register might not then be appropriate
2019 : for the new uses. */
2020 3767416 : else if (REG_P (reg)
2021 3767416 : && REGNO (reg) >= FIRST_PSEUDO_REGISTER
2022 3767416 : && paradoxical_subreg_p (operand)
2023 1020680 : && (inner_hard_regno = lra_get_regno_hard_regno (REGNO (reg))) >= 0
2024 944320 : && hard_regno_nregs (inner_hard_regno, mode) > 1
2025 3767416 : && ((hard_regno
2026 3772734 : = simplify_subreg_regno (inner_hard_regno, innermode,
2027 5318 : SUBREG_BYTE (operand), mode)) < 0
2028 5318 : || ((hard_regno_nregs (inner_hard_regno, innermode)
2029 5318 : < hard_regno_nregs (hard_regno, mode))
2030 10636 : && (regclass = lra_get_allocno_class (REGNO (reg)))
2031 5318 : && (type != OP_IN
2032 5318 : || !in_hard_reg_set_p (reg_class_contents[regclass],
2033 : mode, hard_regno)
2034 5318 : || overlaps_hard_reg_set_p (lra_no_alloc_regs,
2035 : mode, hard_regno)))))
2036 : {
2037 : /* The class will be defined later in curr_insn_transform. */
2038 0 : enum reg_class rclass
2039 0 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
2040 :
2041 0 : if (get_reload_reg (curr_static_id->operand[nop].type, mode, reg,
2042 : rclass, NULL,
2043 : true, false, "paradoxical subreg", &new_reg))
2044 : {
2045 0 : rtx subreg;
2046 0 : bool insert_before, insert_after;
2047 :
2048 0 : PUT_MODE (new_reg, mode);
2049 0 : subreg = gen_lowpart_SUBREG (innermode, new_reg);
2050 0 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
2051 :
2052 0 : insert_before = (type != OP_OUT);
2053 0 : insert_after = (type != OP_IN);
2054 0 : insert_move_for_subreg (insert_before ? &before : NULL,
2055 : insert_after ? &after : NULL,
2056 : reg, subreg);
2057 : }
2058 0 : SUBREG_REG (operand) = new_reg;
2059 0 : lra_process_new_insns (curr_insn, before, after,
2060 : "Inserting paradoxical subreg reload");
2061 0 : return true;
2062 : }
2063 : return false;
2064 : }
2065 :
2066 : /* Return TRUE if X refers for a hard register from SET. */
2067 : static bool
2068 410930 : uses_hard_regs_p (rtx x, HARD_REG_SET set)
2069 : {
2070 410930 : int i, j, x_hard_regno;
2071 410930 : machine_mode mode;
2072 410930 : const char *fmt;
2073 410930 : enum rtx_code code;
2074 :
2075 410930 : if (x == NULL_RTX)
2076 : return false;
2077 410930 : code = GET_CODE (x);
2078 410930 : mode = GET_MODE (x);
2079 :
2080 410930 : if (code == SUBREG)
2081 : {
2082 : /* For all SUBREGs we want to check whether the full multi-register
2083 : overlaps the set. For normal SUBREGs this means 'get_hard_regno' of
2084 : the inner register, for paradoxical SUBREGs this means the
2085 : 'get_hard_regno' of the full SUBREG and for complete SUBREGs either is
2086 : fine. Use the wider mode for all cases. */
2087 10254 : rtx subreg = SUBREG_REG (x);
2088 10254 : mode = wider_subreg_mode (x);
2089 10254 : if (mode == GET_MODE (subreg))
2090 : {
2091 9221 : x = subreg;
2092 9221 : code = GET_CODE (x);
2093 : }
2094 : }
2095 :
2096 410930 : if (REG_P (x) || SUBREG_P (x))
2097 : {
2098 263969 : x_hard_regno = get_hard_regno (x);
2099 263969 : return (x_hard_regno >= 0
2100 263969 : && overlaps_hard_reg_set_p (set, mode, x_hard_regno));
2101 : }
2102 146961 : fmt = GET_RTX_FORMAT (code);
2103 380798 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
2104 : {
2105 235903 : if (fmt[i] == 'e')
2106 : {
2107 114825 : if (uses_hard_regs_p (XEXP (x, i), set))
2108 : return true;
2109 : }
2110 121078 : else if (fmt[i] == 'E')
2111 : {
2112 4398 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
2113 3980 : if (uses_hard_regs_p (XVECEXP (x, i, j), set))
2114 : return true;
2115 : }
2116 : }
2117 : return false;
2118 : }
2119 :
2120 : /* Return true if OP is a spilled pseudo. */
2121 : static inline bool
2122 82924972 : spilled_pseudo_p (rtx op)
2123 : {
2124 82924972 : return (REG_P (op)
2125 82924972 : && REGNO (op) >= FIRST_PSEUDO_REGISTER && in_mem_p (REGNO (op)));
2126 : }
2127 :
2128 : /* Return true if X is a general constant. */
2129 : static inline bool
2130 7944794 : general_constant_p (rtx x)
2131 : {
2132 7944794 : return CONSTANT_P (x) && (! flag_pic || LEGITIMATE_PIC_OPERAND_P (x));
2133 : }
2134 :
2135 : static bool
2136 25600623 : reg_in_class_p (rtx reg, enum reg_class cl)
2137 : {
2138 25600623 : if (cl == NO_REGS)
2139 1137814 : return get_reg_class (REGNO (reg)) == NO_REGS;
2140 24462809 : return in_class_p (reg, cl, NULL);
2141 : }
2142 :
2143 : /* Return true if SET of RCLASS contains no hard regs which can be
2144 : used in MODE. */
2145 : static bool
2146 3904469 : prohibited_class_reg_set_mode_p (enum reg_class rclass,
2147 : HARD_REG_SET &set,
2148 : machine_mode mode)
2149 : {
2150 3904469 : HARD_REG_SET temp;
2151 :
2152 7808938 : lra_assert (hard_reg_set_subset_p (reg_class_contents[rclass], set));
2153 3904469 : temp = set & ~lra_no_alloc_regs;
2154 3904469 : return (hard_reg_set_subset_p
2155 3904469 : (temp, ira_prohibited_class_mode_regs[rclass][mode]));
2156 : }
2157 :
2158 :
2159 : /* Used to check validity info about small class input operands. It
2160 : should be incremented at start of processing an insn
2161 : alternative. */
2162 : static unsigned int curr_small_class_check = 0;
2163 :
2164 : /* Update number of used inputs of class OP_CLASS for operand NOP
2165 : of alternative NALT. Return true if we have more such class operands
2166 : than the number of available regs. */
2167 : static bool
2168 404895457 : update_and_check_small_class_inputs (int nop, int nalt,
2169 : enum reg_class op_class)
2170 : {
2171 404895457 : static unsigned int small_class_check[LIM_REG_CLASSES];
2172 404895457 : static int small_class_input_nums[LIM_REG_CLASSES];
2173 :
2174 401893103 : if (SMALL_REGISTER_CLASS_P (op_class)
2175 : /* We are interesting in classes became small because of fixing
2176 : some hard regs, e.g. by an user through GCC options. */
2177 3127996 : && hard_reg_set_intersect_p (reg_class_contents[op_class],
2178 3127996 : ira_no_alloc_regs)
2179 404895495 : && (curr_static_id->operand[nop].type != OP_OUT
2180 32 : || TEST_BIT (curr_static_id->operand[nop].early_clobber_alts, nalt)))
2181 : {
2182 6 : if (small_class_check[op_class] == curr_small_class_check)
2183 0 : small_class_input_nums[op_class]++;
2184 : else
2185 : {
2186 6 : small_class_check[op_class] = curr_small_class_check;
2187 6 : small_class_input_nums[op_class] = 1;
2188 : }
2189 6 : if (small_class_input_nums[op_class] > ira_class_hard_regs_num[op_class])
2190 0 : return true;
2191 : }
2192 : return false;
2193 : }
2194 :
2195 : /* Print operand constraints for alternative ALT_NUMBER of the current
2196 : insn. */
2197 : static void
2198 4573 : print_curr_insn_alt (int alt_number)
2199 : {
2200 15881 : for (int i = 0; i < curr_static_id->n_operands; i++)
2201 : {
2202 11308 : const char *p = (curr_static_id->operand_alternative
2203 11308 : [alt_number * curr_static_id->n_operands + i].constraint);
2204 11308 : if (*p == '\0')
2205 216 : continue;
2206 11092 : fprintf (lra_dump_file, " (%d) ", i);
2207 39370 : for (; *p != '\0' && *p != ',' && *p != '#'; p++)
2208 17186 : fputc (*p, lra_dump_file);
2209 : }
2210 4573 : }
2211 :
2212 : struct dependent_filter_cache_hasher
2213 : : free_ptr_hash <dependent_filter_entry>
2214 : {
2215 0 : static inline hashval_t hash (const dependent_filter_entry *e)
2216 : {
2217 0 : hashval_t h = (hashval_t) e->id;
2218 0 : h = iterative_hash_hashval_t ((hashval_t) e->mode, h);
2219 0 : h = iterative_hash_hashval_t ((hashval_t) e->partner_mode, h);
2220 0 : h = iterative_hash_hashval_t (e->partner_regno, h);
2221 0 : h = iterative_hash_hashval_t ((hashval_t) e->is_ref, h);
2222 0 : return h;
2223 : }
2224 0 : static inline bool equal (const dependent_filter_entry *a,
2225 : const dependent_filter_entry *b)
2226 : {
2227 0 : return (a->id == b->id && a->mode == b->mode
2228 0 : && a->partner_mode == b->partner_mode
2229 0 : && a->partner_regno == b->partner_regno
2230 0 : && a->is_ref == b->is_ref);
2231 : }
2232 : };
2233 :
2234 : static hash_table<dependent_filter_cache_hasher> *dependent_filter_htab;
2235 :
2236 : /* Allocate the dependent-filter cache. */
2237 :
2238 : void
2239 215996 : lra_init_dependent_filter_cache (void)
2240 : {
2241 215996 : gcc_assert (!dependent_filter_htab);
2242 215996 : dependent_filter_htab = new hash_table<dependent_filter_cache_hasher> (16);
2243 215996 : }
2244 :
2245 : /* Release the dependent-filter cache. */
2246 :
2247 : void
2248 285006 : lra_finish_dependent_filter_cache (void)
2249 : {
2250 285006 : delete dependent_filter_htab;
2251 285006 : dependent_filter_htab = nullptr;
2252 285006 : }
2253 :
2254 : /* Reset the dependent-filter cache. */
2255 :
2256 : void
2257 3031842 : lra_reset_dependent_filters (void)
2258 : {
2259 3031842 : if (dependent_filter_htab)
2260 3031842 : dependent_filter_htab->empty ();
2261 3031842 : }
2262 :
2263 : /* Return the set of hardregs allowed by dependent filter ID for an operand
2264 : of mode MODE when the referenced operand has PARTNER_REGNO in
2265 : PARTNER_MODE. That is when IS_REF is false. If it is true, the roles are
2266 : reversed and PARTNER_REGNO as well as PARTNER_MODE refer to the dependent
2267 : operand.
2268 :
2269 : We fill the set for all hardregs and add the resulting regset to
2270 : a hash table if it doesn't already exist in it. */
2271 :
2272 : const HARD_REG_SET *
2273 0 : lra_get_dependent_filter (int id, machine_mode mode,
2274 : unsigned int partner_regno,
2275 : machine_mode partner_mode, bool is_ref)
2276 : {
2277 0 : dependent_filter_entry key;
2278 0 : key.id = id;
2279 0 : key.mode = mode;
2280 0 : key.partner_mode = partner_mode;
2281 0 : key.partner_regno = partner_regno;
2282 0 : key.is_ref = is_ref;
2283 :
2284 0 : dependent_filter_entry **slot
2285 0 : = dependent_filter_htab->find_slot (&key, INSERT);
2286 0 : if (*slot)
2287 0 : return &(*slot)->allowed;
2288 :
2289 0 : auto *e = XCNEW (dependent_filter_entry);
2290 0 : e->id = id;
2291 0 : e->mode = mode;
2292 0 : e->partner_mode = partner_mode;
2293 0 : e->partner_regno = partner_regno;
2294 0 : e->is_ref = is_ref;
2295 :
2296 : /* ??? Should we restrict ourselves to a register class here? */
2297 0 : for (unsigned regno = 0; regno < FIRST_PSEUDO_REGISTER; ++regno)
2298 : {
2299 0 : bool ok;
2300 0 : if (is_ref)
2301 0 : ok = eval_dependent_filter (id, partner_regno, partner_mode,
2302 : regno, mode);
2303 : else
2304 0 : ok = eval_dependent_filter (id, regno, mode,
2305 : partner_regno, partner_mode);
2306 0 : if (ok)
2307 0 : SET_HARD_REG_BIT (e->allowed, regno);
2308 : }
2309 0 : *slot = e;
2310 0 : return &e->allowed;
2311 : }
2312 :
2313 : /* Add the dependent filter ID to REGNO's dependent filters with
2314 : referenced PARTNER_REGNO and PARTNER_MODE if IS_REF is false.
2315 : If IS_REF is true, the roles are reversed. We need both
2316 : directions in case the referenced op is assigned a hardreg first. */
2317 :
2318 : void
2319 0 : lra_add_dependent_filter (int regno, int id, machine_mode mode,
2320 : int partner_regno, machine_mode partner_mode,
2321 : bool is_ref)
2322 : {
2323 : /* Only meaningful for pseudo partners. */
2324 0 : if (partner_regno < FIRST_PSEUDO_REGISTER)
2325 0 : return;
2326 :
2327 0 : vec<dependent_filter> &filters = lra_reg_info[regno].dependent_filters;
2328 0 : unsigned int i;
2329 0 : dependent_filter *filter;
2330 :
2331 : /* If this filter already exists, do nothing. */
2332 0 : FOR_EACH_VEC_ELT (filters, i, filter)
2333 0 : if (filter->id == id
2334 0 : && filter->partner_regno == (unsigned int) partner_regno
2335 0 : && filter->mode == mode && filter->partner_mode == partner_mode
2336 0 : && filter->is_ref == is_ref)
2337 : return;
2338 :
2339 : /* Otherwise, create a new filter. */
2340 0 : dependent_filter new_filter;
2341 0 : new_filter.id = id;
2342 0 : new_filter.mode = mode;
2343 0 : new_filter.partner_mode = partner_mode;
2344 0 : new_filter.partner_regno = (unsigned int) partner_regno;
2345 0 : new_filter.is_ref = is_ref;
2346 0 : filters.safe_push (new_filter);
2347 : }
2348 :
2349 : /* Return the set of hardregs for the constraint CN in MODE,
2350 : allowed by its dependent filter.
2351 : If there is no dependent filter or the involved registers are no
2352 : hardregs, return NULL. */
2353 :
2354 : static const HARD_REG_SET *
2355 0 : get_dependent_filter (constraint_num cn, machine_mode mode)
2356 : {
2357 354476756 : int id = get_dependent_filter_id (cn);
2358 0 : if (id < 0)
2359 0 : return nullptr;
2360 :
2361 : gcc_assert (reg_class_for_constraint (cn) != NO_REGS);
2362 :
2363 : int ref_opno = get_dependent_filter_ref (id);
2364 : gcc_assert (ref_opno >= 0 && ref_opno < curr_static_id->n_operands);
2365 :
2366 : rtx ref_op = *curr_id->operand_loc[ref_opno];
2367 : if (SUBREG_P (ref_op))
2368 : ref_op = SUBREG_REG (ref_op);
2369 : if (!REG_P (ref_op))
2370 : return nullptr;
2371 : unsigned int ref_regno = REGNO (ref_op);
2372 : if (ref_regno >= FIRST_PSEUDO_REGISTER)
2373 : {
2374 : int ref_hard_regno = reg_renumber[ref_regno];
2375 : /* Even with a pseudo reference op, the filter can still reject
2376 : based on the partner. We call it with INVALID_REGNUM
2377 : to give it a chance to do so. Otherwise we'd introduce
2378 : an "all choices legal" filter that might later
2379 : "change its mind" once there is a fixed reference. */
2380 : if (ref_hard_regno < 0)
2381 : return lra_get_dependent_filter (id, mode, INVALID_REGNUM,
2382 : GET_MODE (ref_op), false);
2383 : ref_regno = (unsigned int) ref_hard_regno;
2384 : }
2385 :
2386 : return lra_get_dependent_filter (id, mode, ref_regno, GET_MODE (ref_op),
2387 : false);
2388 : }
2389 :
2390 : /* Go through all operands and their constraints, looking for a dependent
2391 : filter. If we find one, add it to the respective reg_info's
2392 : dependent-filter list. After assigning a hardreg to either the dependent
2393 : or the referenced op, this allows us to filter the other's regset. */
2394 :
2395 : static void
2396 81503807 : process_dependent_filters (void)
2397 : {
2398 81503807 : int n_operands = curr_static_id->n_operands;
2399 263306764 : for (int i = 0; i < n_operands; i++)
2400 : {
2401 181802957 : rtx op = *curr_id->operand_loc[i];
2402 181802957 : if (!REG_P (op) || HARD_REGISTER_P (op))
2403 99670231 : continue;
2404 82132726 : const char *constraint
2405 82132726 : = curr_static_id->operand_alternative
2406 82132726 : [goal_alt_number * n_operands + i].constraint;
2407 82132726 : for (const char *p = constraint;
2408 215710675 : *p && *p != ',' && *p != '#';
2409 133577949 : p += CONSTRAINT_LEN (*p, p))
2410 : {
2411 133577949 : enum constraint_num cn = lookup_constraint (p);
2412 133577949 : int id = get_dependent_filter_id (cn);
2413 133577949 : if (id < 0)
2414 133577949 : continue;
2415 : if (reg_class_for_constraint (cn) == NO_REGS)
2416 : continue;
2417 : int ref_opno = get_dependent_filter_ref (id);
2418 : if (ref_opno < 0 || ref_opno >= n_operands)
2419 : continue;
2420 : rtx ref_op = *curr_id->operand_loc[ref_opno];
2421 : if (!REG_P (ref_op) || HARD_REGISTER_P (ref_op))
2422 : continue;
2423 : machine_mode mode = curr_operand_mode[i];
2424 : lra_add_dependent_filter (REGNO (op), id, mode,
2425 : REGNO (ref_op), GET_MODE (ref_op),
2426 : false);
2427 : lra_add_dependent_filter (REGNO (ref_op), id, GET_MODE (ref_op),
2428 : REGNO (op), mode, true);
2429 : }
2430 : }
2431 81503807 : }
2432 :
2433 : /* Major function to choose the current insn alternative and what
2434 : operands should be reloaded and how. If ONLY_ALTERNATIVE is not
2435 : negative we should consider only this alternative. Return false if
2436 : we cannot choose the alternative or find how to reload the
2437 : operands. */
2438 : static bool
2439 92136676 : process_alt_operands (int only_alternative)
2440 : {
2441 92136676 : bool ok_p = false;
2442 92136676 : int nop, overall, nalt;
2443 92136676 : int n_alternatives = curr_static_id->n_alternatives;
2444 92136676 : int n_operands = curr_static_id->n_operands;
2445 : /* LOSERS counts the operands that don't fit this alternative and
2446 : would require loading. */
2447 92136676 : int losers;
2448 92136676 : int addr_losers;
2449 : /* REJECT is a count of how undesirable this alternative says it is
2450 : if any reloading is required. If the alternative matches exactly
2451 : then REJECT is ignored, but otherwise it gets this much counted
2452 : against it in addition to the reloading needed. */
2453 92136676 : int reject;
2454 : /* This is defined by '!' or '?' alternative constraint and added to
2455 : reject. But in some cases it can be ignored. */
2456 92136676 : int static_reject;
2457 92136676 : int op_reject;
2458 : /* The number of elements in the following array. */
2459 92136676 : int early_clobbered_regs_num;
2460 : /* Numbers of operands which are early clobber registers. */
2461 92136676 : int early_clobbered_nops[MAX_RECOG_OPERANDS];
2462 92136676 : enum reg_class curr_alt[MAX_RECOG_OPERANDS];
2463 92136676 : enum reg_class all_this_alternative;
2464 92136676 : int all_used_nregs, all_reload_nregs;
2465 92136676 : HARD_REG_SET curr_alt_set[MAX_RECOG_OPERANDS];
2466 92136676 : HARD_REG_SET curr_alt_exclude_start_hard_regs[MAX_RECOG_OPERANDS];
2467 92136676 : bool curr_alt_match_win[MAX_RECOG_OPERANDS];
2468 92136676 : bool curr_alt_win[MAX_RECOG_OPERANDS];
2469 92136676 : bool curr_alt_offmemok[MAX_RECOG_OPERANDS];
2470 92136676 : int curr_alt_matches[MAX_RECOG_OPERANDS];
2471 : /* The number of elements in the following array. */
2472 92136676 : int curr_alt_dont_inherit_ops_num;
2473 : /* Numbers of operands whose reload pseudos should not be inherited. */
2474 92136676 : int curr_alt_dont_inherit_ops[MAX_RECOG_OPERANDS];
2475 92136676 : bool curr_reuse_alt_p;
2476 : /* True if output stack pointer reload should be generated for the current
2477 : alternative. */
2478 92136676 : bool curr_alt_out_sp_reload_p;
2479 92136676 : bool curr_alt_class_change_p;
2480 92136676 : rtx op;
2481 : /* The register when the operand is a subreg of register, otherwise the
2482 : operand itself. */
2483 92136676 : rtx no_subreg_reg_operand[MAX_RECOG_OPERANDS];
2484 : /* The register if the operand is a register or subreg of register,
2485 : otherwise NULL. */
2486 92136676 : rtx operand_reg[MAX_RECOG_OPERANDS];
2487 92136676 : int hard_regno[MAX_RECOG_OPERANDS];
2488 92136676 : machine_mode biggest_mode[MAX_RECOG_OPERANDS];
2489 92136676 : int reload_nregs, reload_sum;
2490 92136676 : bool costly_p;
2491 92136676 : enum reg_class cl;
2492 92136676 : const HARD_REG_SET *cl_filter;
2493 92136676 : HARD_REG_SET hard_reg_constraint;
2494 :
2495 : /* Calculate some data common for all alternatives to speed up the
2496 : function. */
2497 306356735 : for (nop = 0; nop < n_operands; nop++)
2498 : {
2499 214220059 : rtx reg;
2500 :
2501 214220059 : op = no_subreg_reg_operand[nop] = *curr_id->operand_loc[nop];
2502 : /* The real hard regno of the operand after the allocation. */
2503 214220059 : hard_regno[nop] = get_hard_regno (op);
2504 :
2505 214220059 : operand_reg[nop] = reg = op;
2506 214220059 : biggest_mode[nop] = GET_MODE (op);
2507 214220059 : if (GET_CODE (op) == SUBREG)
2508 : {
2509 4289825 : biggest_mode[nop] = wider_subreg_mode (op);
2510 4289825 : operand_reg[nop] = reg = SUBREG_REG (op);
2511 : }
2512 214220059 : if (! REG_P (reg))
2513 90690930 : operand_reg[nop] = NULL_RTX;
2514 123529129 : else if (REGNO (reg) >= FIRST_PSEUDO_REGISTER
2515 145093975 : || ((int) REGNO (reg)
2516 21564846 : == lra_get_elimination_hard_regno (REGNO (reg))))
2517 120425527 : no_subreg_reg_operand[nop] = reg;
2518 : else
2519 3103602 : operand_reg[nop] = no_subreg_reg_operand[nop]
2520 : /* Just use natural mode for elimination result. It should
2521 : be enough for extra constraints hooks. */
2522 3103602 : = regno_reg_rtx[hard_regno[nop]];
2523 : }
2524 :
2525 : /* The constraints are made of several alternatives. Each operand's
2526 : constraint looks like foo,bar,... with commas separating the
2527 : alternatives. The first alternatives for all operands go
2528 : together, the second alternatives go together, etc.
2529 :
2530 : First loop over alternatives. */
2531 92136676 : alternative_mask preferred = curr_id->preferred_alternatives;
2532 92136676 : if (only_alternative >= 0)
2533 981394 : preferred &= ALTERNATIVE_BIT (only_alternative);
2534 :
2535 92136676 : bool prefer_memory_p = false;
2536 92137244 : repeat:
2537 390540666 : for (nalt = 0; nalt < n_alternatives; nalt++)
2538 : {
2539 : /* Loop over operands for one constraint alternative. */
2540 376503367 : if (!TEST_BIT (preferred, nalt))
2541 111896556 : continue;
2542 :
2543 264606811 : if (lra_dump_file != NULL)
2544 : {
2545 3388 : fprintf (lra_dump_file, " Considering alt=%d of insn %d: ",
2546 3388 : nalt, INSN_UID (curr_insn));
2547 3388 : print_curr_insn_alt (nalt);
2548 3388 : fprintf (lra_dump_file, "\n");
2549 : }
2550 :
2551 264606811 : bool matching_early_clobber[MAX_RECOG_OPERANDS];
2552 264606811 : curr_small_class_check++;
2553 264606811 : overall = losers = addr_losers = 0;
2554 264606811 : static_reject = reject = reload_nregs = reload_sum = 0;
2555 877355270 : for (nop = 0; nop < n_operands; nop++)
2556 : {
2557 612748459 : int inc = (curr_static_id
2558 612748459 : ->operand_alternative[nalt * n_operands + nop].reject);
2559 612748459 : if (lra_dump_file != NULL && inc != 0)
2560 52 : fprintf (lra_dump_file,
2561 : " Statically defined alt reject+=%d\n", inc);
2562 612748459 : static_reject += inc;
2563 612748459 : matching_early_clobber[nop] = 0;
2564 : }
2565 : reject += static_reject;
2566 : early_clobbered_regs_num = 0;
2567 : curr_alt_out_sp_reload_p = false;
2568 : curr_reuse_alt_p = true;
2569 : curr_alt_class_change_p = false;
2570 : all_this_alternative = NO_REGS;
2571 : all_used_nregs = all_reload_nregs = 0;
2572 684767229 : for (nop = 0; nop < n_operands; nop++)
2573 : {
2574 546190736 : const char *p;
2575 546190736 : char *end;
2576 546190736 : int len, c, m, i, opalt_num, this_alternative_matches;
2577 546190736 : bool win, did_match, offmemok, early_clobber_p;
2578 : /* false => this operand can be reloaded somehow for this
2579 : alternative. */
2580 546190736 : bool badop;
2581 : /* true => this operand can be reloaded if the alternative
2582 : allows regs. */
2583 546190736 : bool winreg;
2584 : /* True if a constant forced into memory would be OK for
2585 : this operand. */
2586 546190736 : bool constmemok;
2587 546190736 : enum reg_class this_alternative, this_costly_alternative;
2588 546190736 : HARD_REG_SET this_alternative_set, this_costly_alternative_set;
2589 546190736 : HARD_REG_SET this_alternative_exclude_start_hard_regs;
2590 546190736 : bool this_alternative_match_win, this_alternative_win;
2591 546190736 : bool this_alternative_offmemok;
2592 546190736 : bool scratch_p;
2593 546190736 : machine_mode mode;
2594 546190736 : enum constraint_num cn;
2595 546190736 : bool class_change_p = false;
2596 :
2597 546190736 : opalt_num = nalt * n_operands + nop;
2598 546190736 : if (curr_static_id->operand_alternative[opalt_num].anything_ok)
2599 : {
2600 : /* Fast track for no constraints at all. */
2601 15264961 : curr_alt[nop] = NO_REGS;
2602 15264961 : CLEAR_HARD_REG_SET (curr_alt_set[nop]);
2603 15264961 : curr_alt_win[nop] = true;
2604 15264961 : curr_alt_match_win[nop] = false;
2605 15264961 : curr_alt_offmemok[nop] = false;
2606 15264961 : curr_alt_matches[nop] = -1;
2607 15264961 : continue;
2608 : }
2609 :
2610 530925775 : op = no_subreg_reg_operand[nop];
2611 530925775 : mode = curr_operand_mode[nop];
2612 :
2613 530925775 : win = did_match = winreg = offmemok = constmemok = false;
2614 530925775 : badop = true;
2615 530925775 : const HARD_REG_SET *cl_dep_filter = nullptr;
2616 :
2617 530925775 : early_clobber_p = false;
2618 530925775 : p = curr_static_id->operand_alternative[opalt_num].constraint;
2619 :
2620 530925775 : this_costly_alternative = this_alternative = NO_REGS;
2621 : /* We update set of possible hard regs besides its class
2622 : because reg class might be inaccurate. For example,
2623 : union of LO_REGS (l), HI_REGS(h), and STACK_REG(k) in ARM
2624 : is translated in HI_REGS because classes are merged by
2625 : pairs and there is no accurate intermediate class. */
2626 2123703100 : CLEAR_HARD_REG_SET (this_alternative_set);
2627 1592777325 : CLEAR_HARD_REG_SET (this_costly_alternative_set);
2628 530925775 : CLEAR_HARD_REG_SET (this_alternative_exclude_start_hard_regs);
2629 530925775 : this_alternative_win = false;
2630 530925775 : this_alternative_match_win = false;
2631 530925775 : this_alternative_offmemok = false;
2632 530925775 : this_alternative_matches = -1;
2633 :
2634 : /* An empty constraint should be excluded by the fast
2635 : track. */
2636 530925775 : lra_assert (*p != 0 && *p != ',');
2637 :
2638 : op_reject = 0;
2639 : /* Scan this alternative's specs for this operand; set WIN
2640 : if the operand fits any letter in this alternative.
2641 : Otherwise, clear BADOP if this operand could fit some
2642 : letter after reloads, or set WINREG if this operand could
2643 : fit after reloads provided the constraint allows some
2644 : registers. */
2645 : costly_p = false;
2646 1348353524 : do
2647 : {
2648 1348353524 : switch ((c = *p, len = CONSTRAINT_LEN (c, p)), c)
2649 : {
2650 : case '\0':
2651 : len = 0;
2652 : break;
2653 507534597 : case ',':
2654 507534597 : c = '\0';
2655 507534597 : break;
2656 :
2657 190550 : case '&':
2658 190550 : early_clobber_p = true;
2659 190550 : break;
2660 :
2661 19765 : case '$':
2662 19765 : op_reject += LRA_MAX_REJECT;
2663 19765 : break;
2664 0 : case '^':
2665 0 : op_reject += LRA_LOSER_COST_FACTOR;
2666 0 : break;
2667 :
2668 0 : case '#':
2669 : /* Ignore rest of this alternative. */
2670 0 : c = '\0';
2671 0 : break;
2672 :
2673 57694357 : case '0': case '1': case '2': case '3': case '4':
2674 57694357 : case '5': case '6': case '7': case '8': case '9':
2675 57694357 : {
2676 57694357 : int m_hregno;
2677 57694357 : bool match_p;
2678 :
2679 57694357 : m = strtoul (p, &end, 10);
2680 57694357 : p = end;
2681 57694357 : len = 0;
2682 57694357 : lra_assert (nop > m);
2683 :
2684 : /* Reject matches if we don't know which operand is
2685 : bigger. This situation would arguably be a bug in
2686 : an .md pattern, but could also occur in a user asm. */
2687 173083071 : if (!ordered_p (GET_MODE_SIZE (biggest_mode[m]),
2688 57694357 : GET_MODE_SIZE (biggest_mode[nop])))
2689 : break;
2690 :
2691 : /* Don't match wrong asm insn operands for proper
2692 : diagnostic later. */
2693 57694357 : if (INSN_CODE (curr_insn) < 0
2694 34369 : && (curr_operand_mode[m] == BLKmode
2695 34368 : || curr_operand_mode[nop] == BLKmode)
2696 1 : && curr_operand_mode[m] != curr_operand_mode[nop])
2697 : break;
2698 :
2699 57694356 : m_hregno = get_hard_regno (*curr_id->operand_loc[m]);
2700 : /* We are supposed to match a previous operand.
2701 : If we do, we win if that one did. If we do
2702 : not, count both of the operands as losers.
2703 : (This is too conservative, since most of the
2704 : time only a single reload insn will be needed
2705 : to make the two operands win. As a result,
2706 : this alternative may be rejected when it is
2707 : actually desirable.) */
2708 57694356 : match_p = false;
2709 57694356 : if (operands_match_p (*curr_id->operand_loc[nop],
2710 57694356 : *curr_id->operand_loc[m], m_hregno))
2711 : {
2712 : /* We should reject matching of an early
2713 : clobber operand if the matching operand is
2714 : not dying in the insn. */
2715 15111576 : if (!TEST_BIT (curr_static_id->operand[m]
2716 : .early_clobber_alts, nalt)
2717 20628 : || operand_reg[nop] == NULL_RTX
2718 15132204 : || (find_regno_note (curr_insn, REG_DEAD,
2719 : REGNO (op))
2720 4396 : || REGNO (op) == REGNO (operand_reg[m])))
2721 15111576 : match_p = true;
2722 : }
2723 15111576 : if (match_p)
2724 : {
2725 : /* If we are matching a non-offsettable
2726 : address where an offsettable address was
2727 : expected, then we must reject this
2728 : combination, because we can't reload
2729 : it. */
2730 15111576 : if (curr_alt_offmemok[m]
2731 1497 : && MEM_P (*curr_id->operand_loc[m])
2732 5 : && curr_alt[m] == NO_REGS && ! curr_alt_win[m])
2733 0 : continue;
2734 : }
2735 : else
2736 : {
2737 : /* If the operands do not match and one
2738 : operand is INOUT, we can not match them.
2739 : Try other possibilities, e.g. other
2740 : alternatives or commutative operand
2741 : exchange. */
2742 42582780 : if (curr_static_id->operand[nop].type == OP_INOUT
2743 42582780 : || curr_static_id->operand[m].type == OP_INOUT)
2744 : break;
2745 : /* Operands don't match. For asm if the operands
2746 : are different user defined explicit hard
2747 : registers, then we cannot make them match
2748 : when one is early clobber operand. */
2749 42582381 : if ((REG_P (*curr_id->operand_loc[nop])
2750 26445315 : || SUBREG_P (*curr_id->operand_loc[nop]))
2751 16728054 : && (REG_P (*curr_id->operand_loc[m])
2752 202311 : || SUBREG_P (*curr_id->operand_loc[m]))
2753 16633917 : && INSN_CODE (curr_insn) < 0)
2754 : {
2755 574 : rtx nop_reg = *curr_id->operand_loc[nop];
2756 574 : if (SUBREG_P (nop_reg))
2757 0 : nop_reg = SUBREG_REG (nop_reg);
2758 574 : rtx m_reg = *curr_id->operand_loc[m];
2759 574 : if (SUBREG_P (m_reg))
2760 0 : m_reg = SUBREG_REG (m_reg);
2761 :
2762 574 : if (REG_P (nop_reg)
2763 574 : && HARD_REGISTER_P (nop_reg)
2764 0 : && REG_USERVAR_P (nop_reg)
2765 0 : && REG_P (m_reg)
2766 0 : && HARD_REGISTER_P (m_reg)
2767 574 : && REG_USERVAR_P (m_reg))
2768 : {
2769 : int i;
2770 :
2771 0 : for (i = 0; i < early_clobbered_regs_num; i++)
2772 0 : if (m == early_clobbered_nops[i])
2773 : break;
2774 0 : if (i < early_clobbered_regs_num
2775 0 : || early_clobber_p)
2776 : break;
2777 : }
2778 : }
2779 : /* Both operands must allow a reload register or the
2780 : matched operand accepts memory (e.g. 'g' with a mode
2781 : that fits no hard reg), in which case the operands
2782 : can be matched in memory. */
2783 42582381 : if (curr_alt[m] == NO_REGS && !curr_alt_offmemok[m])
2784 : break;
2785 : /* Retroactively mark the operand we had to
2786 : match as a loser, if it wasn't already and
2787 : it wasn't matched to a register constraint
2788 : (e.g it might be matched by memory). */
2789 42559464 : if (curr_alt_win[m]
2790 41703944 : && (operand_reg[m] == NULL_RTX
2791 41191862 : || hard_regno[m] < 0))
2792 : {
2793 1253670 : if (lra_dump_file != NULL)
2794 9 : fprintf
2795 9 : (lra_dump_file,
2796 : " %d Matched operand reload: "
2797 : "losers++\n", m);
2798 1253670 : losers++;
2799 1253670 : reload_nregs
2800 1253670 : += (ira_reg_class_max_nregs[curr_alt[m]]
2801 1253670 : [GET_MODE (*curr_id->operand_loc[m])]);
2802 : }
2803 :
2804 : /* Prefer matching earlyclobber alternative as
2805 : it results in less hard regs required for
2806 : the insn than a non-matching earlyclobber
2807 : alternative. */
2808 42559464 : if (TEST_BIT (curr_static_id->operand[m]
2809 : .early_clobber_alts, nalt))
2810 : {
2811 19687 : if (lra_dump_file != NULL)
2812 0 : fprintf
2813 0 : (lra_dump_file,
2814 : " %d Matching earlyclobber alt:"
2815 : " reject--\n",
2816 : nop);
2817 19687 : if (!matching_early_clobber[m])
2818 : {
2819 19687 : reject--;
2820 19687 : matching_early_clobber[m] = 1;
2821 : }
2822 : }
2823 : /* Otherwise we prefer no matching
2824 : alternatives because it gives more freedom
2825 : in RA. */
2826 42539777 : else if (operand_reg[nop] == NULL_RTX
2827 42539777 : || (find_regno_note (curr_insn, REG_DEAD,
2828 16702321 : REGNO (operand_reg[nop]))
2829 : == NULL_RTX))
2830 : {
2831 37411311 : if (lra_dump_file != NULL)
2832 916 : fprintf
2833 916 : (lra_dump_file,
2834 : " %d Matching alt: reject+=2\n",
2835 : nop);
2836 37411311 : reject += 2;
2837 : }
2838 : }
2839 : /* If we have to reload this operand and some
2840 : previous operand also had to match the same
2841 : thing as this operand, we don't know how to do
2842 : that. */
2843 57671040 : if (!match_p || !curr_alt_win[m])
2844 : {
2845 89564653 : for (i = 0; i < nop; i++)
2846 46890910 : if (curr_alt_matches[i] == m)
2847 : break;
2848 42673744 : if (i < nop)
2849 : break;
2850 : }
2851 : else
2852 : did_match = true;
2853 :
2854 57671039 : this_alternative_matches = m;
2855 : /* This can be fixed with reloads if the operand
2856 : we are supposed to match can be fixed with
2857 : reloads. */
2858 57671039 : badop = false;
2859 57671039 : this_alternative = curr_alt[m];
2860 57671039 : this_alternative_set = curr_alt_set[m];
2861 57671039 : this_alternative_exclude_start_hard_regs
2862 57671039 : = curr_alt_exclude_start_hard_regs[m];
2863 : /* If the matched operand permits memory (e.g. 'g'),
2864 : so does this one -- they share the same location.
2865 : This lets a matching reload go to memory when no
2866 : hard reg of the class can hold the operand mode. */
2867 57671039 : offmemok = curr_alt_offmemok[m];
2868 57671039 : winreg = this_alternative != NO_REGS;
2869 57671039 : break;
2870 : }
2871 :
2872 11823661 : case 'g':
2873 11823661 : if (MEM_P (op)
2874 7944794 : || general_constant_p (op)
2875 16420906 : || spilled_pseudo_p (op))
2876 : win = true;
2877 11823661 : if (REG_P (op) && prefer_memory_p)
2878 : {
2879 11823661 : badop = false;
2880 11823661 : offmemok = true;
2881 : }
2882 11823661 : cl = GENERAL_REGS;
2883 11823661 : cl_filter = nullptr;
2884 11823661 : cl_dep_filter = nullptr;
2885 11823661 : goto reg;
2886 :
2887 685 : case '{':
2888 685 : {
2889 685 : int regno = decode_hard_reg_constraint (p);
2890 685 : gcc_assert (regno >= 0);
2891 685 : cl = NO_REGS;
2892 685 : int nregs = hard_regno_nregs (regno, mode);
2893 1370 : for (int i = 0; i < nregs; ++i)
2894 685 : cl = reg_class_superunion[cl][REGNO_REG_CLASS (regno + i)];
2895 685 : CLEAR_HARD_REG_SET (hard_reg_constraint);
2896 685 : SET_HARD_REG_BIT (hard_reg_constraint, regno);
2897 685 : cl_filter = &hard_reg_constraint;
2898 685 : cl_dep_filter = nullptr;
2899 685 : goto reg;
2900 : }
2901 :
2902 747698731 : default:
2903 747698731 : cn = lookup_constraint (p);
2904 747698731 : switch (get_constraint_type (cn))
2905 : {
2906 494302958 : case CT_REGISTER:
2907 494302958 : cl = reg_class_for_constraint (cn);
2908 364068145 : if (cl != NO_REGS)
2909 : {
2910 354476756 : cl_filter = get_register_filter (cn);
2911 354476756 : cl_dep_filter = get_dependent_filter (cn, mode);
2912 354476756 : goto reg;
2913 : }
2914 : break;
2915 :
2916 2205343 : case CT_CONST_INT:
2917 2205343 : if (CONST_INT_P (op)
2918 2205343 : && insn_const_int_ok_for_constraint (INTVAL (op), cn))
2919 : win = true;
2920 : break;
2921 :
2922 113644429 : case CT_MEMORY:
2923 113644429 : case CT_RELAXED_MEMORY:
2924 113644429 : if (MEM_P (op)
2925 113644429 : && satisfies_memory_constraint_p (op, cn))
2926 : win = true;
2927 77255336 : else if (spilled_pseudo_p (op))
2928 46032123 : win = true;
2929 :
2930 : /* If we didn't already win, we can reload constants
2931 : via force_const_mem or put the pseudo value into
2932 : memory, or make other memory by reloading the
2933 : address like for 'o'. */
2934 119129997 : if (CONST_POOL_OK_P (mode, op)
2935 108158661 : || MEM_P (op) || REG_P (op)
2936 : /* We can restore the equiv insn by a
2937 : reload. */
2938 114292035 : || equiv_substition_p[nop])
2939 : badop = false;
2940 : constmemok = true;
2941 : offmemok = true;
2942 : break;
2943 :
2944 1821287 : case CT_ADDRESS:
2945 : /* An asm operand with an address constraint
2946 : that doesn't satisfy address_operand has
2947 : is_address cleared, so that we don't try to
2948 : make a non-address fit. */
2949 1821287 : if (!curr_static_id->operand[nop].is_address)
2950 : break;
2951 : /* If we didn't already win, we can reload the address
2952 : into a base register. */
2953 1821268 : if (satisfies_address_constraint_p (op, cn))
2954 1821268 : win = true;
2955 1821268 : cl = base_reg_class (VOIDmode, ADDR_SPACE_GENERIC,
2956 : ADDRESS, SCRATCH, NULL_RTX,
2957 : curr_insn);
2958 1821268 : cl_filter = nullptr;
2959 1821268 : cl_dep_filter = nullptr;
2960 1821268 : badop = false;
2961 1821268 : goto reg;
2962 :
2963 134501125 : case CT_FIXED_FORM:
2964 134501125 : if (constraint_satisfied_p (op, cn))
2965 1348353524 : win = true;
2966 : break;
2967 :
2968 1223589 : case CT_SPECIAL_MEMORY:
2969 1223589 : if (satisfies_memory_constraint_p (op, cn))
2970 : win = true;
2971 1072391 : else if (spilled_pseudo_p (op))
2972 : {
2973 1348353524 : curr_reuse_alt_p = false;
2974 1348353524 : win = true;
2975 : }
2976 : break;
2977 : }
2978 : break;
2979 :
2980 368122370 : reg:
2981 368122370 : if (mode == BLKmode)
2982 : break;
2983 368122352 : this_alternative = reg_class_subunion[this_alternative][cl];
2984 368122352 : if (hard_reg_set_subset_p (this_alternative_set,
2985 368122352 : reg_class_contents[cl]))
2986 368118727 : this_alternative_exclude_start_hard_regs
2987 368118727 : = ira_exclude_class_mode_regs[cl][mode];
2988 3625 : else if (!hard_reg_set_subset_p (reg_class_contents[cl],
2989 : this_alternative_set))
2990 3624 : this_alternative_exclude_start_hard_regs
2991 3624 : |= ira_exclude_class_mode_regs[cl][mode];
2992 368122352 : this_alternative_set |= reg_class_contents[cl];
2993 368122352 : if (cl_filter)
2994 1370 : this_alternative_exclude_start_hard_regs |= ~*cl_filter;
2995 368122352 : if (cl_dep_filter)
2996 : this_alternative_exclude_start_hard_regs |= ~*cl_dep_filter;
2997 368122352 : if (costly_p)
2998 : {
2999 21921319 : this_costly_alternative
3000 21921319 : = reg_class_subunion[this_costly_alternative][cl];
3001 21921319 : this_costly_alternative_set |= reg_class_contents[cl];
3002 : }
3003 368122352 : winreg = true;
3004 368122352 : if (REG_P (op))
3005 : {
3006 234445789 : rtx orig_op = *curr_id->operand_loc[nop];
3007 6862056 : if (GET_CODE (orig_op) == SUBREG && HARD_REGISTER_P (op)
3008 234445877 : && !targetm.hard_regno_mode_ok (REGNO (op),
3009 88 : GET_MODE(orig_op)))
3010 : break;
3011 :
3012 234445789 : tree decl;
3013 :
3014 234445789 : if (hard_regno[nop] >= 0
3015 198130077 : && in_hard_reg_set_p (this_alternative_set,
3016 : mode, hard_regno[nop])
3017 179231051 : && (!cl_filter
3018 410 : || TEST_HARD_REG_BIT (*cl_filter,
3019 : hard_regno[nop]))
3020 : && (!cl_dep_filter
3021 : || TEST_HARD_REG_BIT (*cl_dep_filter,
3022 : hard_regno[nop]))
3023 413676834 : && ((REG_ATTRS (op) && (decl = REG_EXPR (op)) != NULL
3024 99752462 : && VAR_P (decl) && DECL_HARD_REGISTER (decl))
3025 179227773 : || !(TEST_HARD_REG_BIT
3026 179227773 : (this_alternative_exclude_start_hard_regs,
3027 : hard_regno[nop]))))
3028 : win = true;
3029 55214745 : else if (hard_regno[nop] < 0 && !prefer_memory_p)
3030 : {
3031 36314973 : if (in_class_p (op, this_alternative, NULL))
3032 : win = true;
3033 27208518 : else if (in_class_p (op, this_alternative, NULL, true))
3034 : {
3035 1348353524 : class_change_p = true;
3036 1348353524 : win = true;
3037 : }
3038 : }
3039 : }
3040 : break;
3041 : }
3042 1348353524 : if (c != ' ' && c != '\t')
3043 1348353524 : costly_p = c == '*';
3044 : }
3045 1348353524 : while ((p += len), c);
3046 :
3047 1061851550 : scratch_p = (operand_reg[nop] != NULL_RTX
3048 530925775 : && ira_former_scratch_p (REGNO (operand_reg[nop])));
3049 : /* Record which operands fit this alternative. */
3050 530925775 : if (win)
3051 : {
3052 286017090 : if (early_clobber_p
3053 285865613 : || curr_static_id->operand[nop].type != OP_OUT)
3054 : {
3055 124998505 : if (winreg)
3056 103341555 : all_used_nregs
3057 103341555 : += ira_reg_class_min_nregs[this_alternative][mode];
3058 124998505 : all_this_alternative
3059 124998505 : = (reg_class_subunion
3060 124998505 : [all_this_alternative][this_alternative]);
3061 : }
3062 286017090 : this_alternative_win = true;
3063 : /* If reg and memory is acceptable here (e.g. 'g') but no hard
3064 : reg of the operand class can hold the operand mode, resolve
3065 : the operand to memory. Mark it so that an operand matching
3066 : this one can also fall back to memory, otherwise the matched
3067 : operand would try to reload into an impossible hard reg and
3068 : LRA would cycle (e.g. TDmode "+g" asm on 32-bit x86).
3069 : An operand which is already memory and won implies the
3070 : constraint accepts memory too, even if nothing set
3071 : OFFMEMOK for it. */
3072 286017090 : if ((offmemok || MEM_P (op)) && this_alternative != NO_REGS
3073 364835333 : && hard_reg_set_subset_p (reg_class_contents[this_alternative],
3074 : ira_prohibited_class_mode_regs
3075 78818243 : [this_alternative][mode]
3076 364835150 : | lra_no_alloc_regs))
3077 : {
3078 183 : this_alternative = NO_REGS;
3079 183 : this_alternative_offmemok = true;
3080 : }
3081 286017090 : if (class_change_p)
3082 : {
3083 262621 : curr_alt_class_change_p = true;
3084 262621 : if (lra_dump_file != NULL)
3085 8 : fprintf (lra_dump_file,
3086 : " %d Narrowing class: reject+=3\n",
3087 : nop);
3088 262621 : reject += 3;
3089 : }
3090 286017090 : if (operand_reg[nop] != NULL_RTX)
3091 : {
3092 199651356 : if (hard_regno[nop] >= 0)
3093 : {
3094 179174461 : if (in_hard_reg_set_p (this_costly_alternative_set,
3095 : mode, hard_regno[nop]))
3096 : {
3097 788819 : if (lra_dump_file != NULL)
3098 21 : fprintf (lra_dump_file,
3099 : " %d Costly set: reject++\n",
3100 : nop);
3101 788819 : reject++;
3102 : }
3103 : }
3104 : else
3105 : {
3106 : /* Prefer won reg to spilled pseudo under other
3107 : equal conditions for possible inheritance. */
3108 20476895 : if (! scratch_p)
3109 : {
3110 20472555 : if (lra_dump_file != NULL)
3111 52 : fprintf
3112 52 : (lra_dump_file,
3113 : " %d Non pseudo reload: reject++\n",
3114 : nop);
3115 20472555 : reject++;
3116 : }
3117 20476895 : if (in_class_p (operand_reg[nop],
3118 : this_costly_alternative, NULL, true))
3119 : {
3120 133552 : if (lra_dump_file != NULL)
3121 0 : fprintf
3122 0 : (lra_dump_file,
3123 : " %d Non pseudo costly reload:"
3124 : " reject++\n",
3125 : nop);
3126 133552 : reject++;
3127 : }
3128 : }
3129 : /* We simulate the behavior of old reload here.
3130 : Although scratches need hard registers and it
3131 : might result in spilling other pseudos, no reload
3132 : insns are generated for the scratches. So it
3133 : might cost something but probably less than old
3134 : reload pass believes. */
3135 199651356 : if (scratch_p)
3136 : {
3137 124149 : if (lra_dump_file != NULL)
3138 6 : fprintf (lra_dump_file,
3139 : " %d Scratch win: reject+=2\n",
3140 : nop);
3141 124149 : reject += 2;
3142 : }
3143 : }
3144 : }
3145 244908685 : else if (did_match)
3146 : this_alternative_match_win = true;
3147 : else
3148 : {
3149 229911389 : if (prefer_memory_p && offmemok)
3150 : {
3151 526 : winreg = false;
3152 526 : this_alternative = NO_REGS;
3153 : }
3154 :
3155 229911389 : int const_to_mem = 0;
3156 229911389 : bool no_regs_p;
3157 :
3158 229911389 : reject += op_reject;
3159 : /* Mark output reload of the stack pointer. */
3160 229911389 : if (op == stack_pointer_rtx
3161 56417 : && curr_static_id->operand[nop].type != OP_IN)
3162 229911389 : curr_alt_out_sp_reload_p = true;
3163 :
3164 : /* If this alternative asks for a specific reg class, see if there
3165 : is at least one allocatable register in that class. */
3166 229911389 : no_regs_p
3167 400852518 : = (this_alternative == NO_REGS
3168 229911389 : || (hard_reg_set_subset_p
3169 341882280 : (reg_class_contents[this_alternative],
3170 : lra_no_alloc_regs)));
3171 :
3172 : /* For asms, verify that the class for this alternative is possible
3173 : for the mode that is specified. */
3174 170941129 : if (!no_regs_p && INSN_CODE (curr_insn) < 0)
3175 : {
3176 : int i;
3177 70280 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
3178 70277 : if (targetm.hard_regno_mode_ok (i, mode)
3179 70277 : && in_hard_reg_set_p (reg_class_contents[this_alternative],
3180 : mode, i))
3181 : break;
3182 20013 : if (i == FIRST_PSEUDO_REGISTER)
3183 229911389 : winreg = false;
3184 : }
3185 :
3186 : /* If this operand accepts a register, and if the
3187 : register class has at least one allocatable register,
3188 : then this operand can be reloaded. */
3189 229911389 : if (winreg && !no_regs_p)
3190 : badop = false;
3191 :
3192 58970263 : if (badop)
3193 : {
3194 49356800 : if (lra_dump_file != NULL)
3195 607 : fprintf (lra_dump_file,
3196 : " Bad operand -- refuse\n");
3197 126029850 : goto fail;
3198 : }
3199 :
3200 180554589 : if (this_alternative != NO_REGS)
3201 : {
3202 170941128 : HARD_REG_SET available_regs
3203 170941128 : = (reg_class_contents[this_alternative]
3204 170941128 : & ~((ira_prohibited_class_mode_regs
3205 170941128 : [this_alternative][mode])
3206 170941128 : | lra_no_alloc_regs));
3207 341882256 : if (!hard_reg_set_empty_p (available_regs))
3208 : {
3209 170939125 : if (early_clobber_p
3210 170900052 : || curr_static_id->operand[nop].type != OP_OUT)
3211 : {
3212 89587982 : all_reload_nregs
3213 89587982 : += ira_reg_class_min_nregs[this_alternative][mode];
3214 89587982 : all_this_alternative
3215 89587982 : = (reg_class_subunion
3216 89587982 : [all_this_alternative][this_alternative]);
3217 : }
3218 : }
3219 : else
3220 : {
3221 : /* There are no hard regs holding a value of given
3222 : mode. */
3223 2003 : if (offmemok)
3224 : {
3225 144 : this_alternative = NO_REGS;
3226 144 : if (lra_dump_file != NULL)
3227 0 : fprintf (lra_dump_file,
3228 : " %d Using memory because of"
3229 : " a bad mode: reject+=2\n",
3230 : nop);
3231 144 : reject += 2;
3232 : }
3233 1859 : else if (!prefer_memory_p)
3234 : {
3235 : /* No hard reg of the operand class can hold the
3236 : operand mode, but the constraint may still permit
3237 : memory (e.g. 'g'). Prefer memory for the operands
3238 : before refusing. */
3239 468 : prefer_memory_p = true;
3240 468 : if (lra_dump_file != NULL)
3241 0 : fprintf (lra_dump_file,
3242 : " Trying now memory for"
3243 : " operands\n");
3244 468 : goto repeat;
3245 : }
3246 : else
3247 : {
3248 1391 : if (lra_dump_file != NULL)
3249 0 : fprintf (lra_dump_file,
3250 : " Wrong mode -- refuse\n");
3251 1391 : goto fail;
3252 : }
3253 : }
3254 : }
3255 :
3256 : /* If not assigned pseudo has a class which a subset of
3257 : required reg class, it is a less costly alternative
3258 : as the pseudo still can get a hard reg of necessary
3259 : class. */
3260 170939269 : if (! no_regs_p && REG_P (op) && hard_regno[nop] < 0
3261 21867576 : && (cl = get_reg_class (REGNO (op))) != NO_REGS
3262 183515025 : && ira_class_subset_p[this_alternative][cl])
3263 : {
3264 1095 : if (lra_dump_file != NULL)
3265 0 : fprintf
3266 0 : (lra_dump_file,
3267 : " %d Super set class reg: reject-=3\n", nop);
3268 1095 : reject -= 3;
3269 : }
3270 :
3271 180552730 : this_alternative_offmemok = offmemok;
3272 180552730 : if (this_costly_alternative != NO_REGS)
3273 : {
3274 19727579 : if (lra_dump_file != NULL)
3275 25 : fprintf (lra_dump_file,
3276 : " %d Costly loser: reject++\n", nop);
3277 19727579 : reject++;
3278 : }
3279 : /* If the operand is dying, has a matching constraint,
3280 : and satisfies constraints of the matched operand
3281 : which failed to satisfy the own constraints, most probably
3282 : the reload for this operand will be gone. */
3283 180552730 : if (this_alternative_matches >= 0
3284 42655949 : && !curr_alt_win[this_alternative_matches]
3285 969276 : && REG_P (op)
3286 716129 : && find_regno_note (curr_insn, REG_DEAD, REGNO (op))
3287 181283499 : && (hard_regno[nop] >= 0
3288 378721 : ? in_hard_reg_set_p (this_alternative_set,
3289 : mode, hard_regno[nop])
3290 26673 : : in_class_p (op, this_alternative, NULL)))
3291 : {
3292 236986 : if (lra_dump_file != NULL)
3293 1 : fprintf
3294 1 : (lra_dump_file,
3295 : " %d Dying matched operand reload: reject++\n",
3296 : nop);
3297 236986 : reject++;
3298 : }
3299 : else
3300 : {
3301 : /* Strict_low_part requires to reload the register
3302 : not the sub-register. In this case we should
3303 : check that a final reload hard reg can hold the
3304 : value mode. */
3305 180315744 : if (curr_static_id->operand[nop].strict_low
3306 102 : && REG_P (op)
3307 95 : && hard_regno[nop] < 0
3308 71 : && GET_CODE (*curr_id->operand_loc[nop]) == SUBREG
3309 71 : && ira_class_hard_regs_num[this_alternative] > 0
3310 180315815 : && (!targetm.hard_regno_mode_ok
3311 71 : (ira_class_hard_regs[this_alternative][0],
3312 71 : GET_MODE (*curr_id->operand_loc[nop]))))
3313 : {
3314 0 : if (lra_dump_file != NULL)
3315 0 : fprintf
3316 0 : (lra_dump_file,
3317 : " Strict low subreg reload -- refuse\n");
3318 0 : goto fail;
3319 : }
3320 180315744 : if (lra_dump_file != NULL)
3321 2162 : fprintf
3322 2162 : (lra_dump_file,
3323 : " %d Operand reload: losers++\n", nop);
3324 180315744 : losers++;
3325 : }
3326 180552730 : if (operand_reg[nop] != NULL_RTX
3327 : /* Output operands and matched input operands are
3328 : not inherited. The following conditions do not
3329 : exactly describe the previous statement but they
3330 : are pretty close. */
3331 65027549 : && curr_static_id->operand[nop].type != OP_OUT
3332 28600095 : && (this_alternative_matches < 0
3333 16799347 : || curr_static_id->operand[nop].type != OP_IN))
3334 : {
3335 11800748 : int last_reload = (lra_reg_info[ORIGINAL_REGNO
3336 11800748 : (operand_reg[nop])]
3337 11800748 : .last_reload);
3338 :
3339 : /* The value of reload_sum has sense only if we
3340 : process insns in their order. It happens only on
3341 : the first constraints sub-pass when we do most of
3342 : reload work. */
3343 11800748 : if (lra_constraint_iter == 1 && last_reload > bb_reload_num)
3344 2544970 : reload_sum += last_reload - bb_reload_num;
3345 : }
3346 : /* If this is a constant that is reloaded into the
3347 : desired class by copying it to memory first, count
3348 : that as another reload. This is consistent with
3349 : other code and is required to avoid choosing another
3350 : alternative when the constant is moved into memory.
3351 : Note that the test here is precisely the same as in
3352 : the code below that calls force_const_mem. */
3353 231920965 : if (CONST_POOL_OK_P (mode, op)
3354 231921022 : && ((targetm.preferred_reload_class
3355 51368292 : (op, this_alternative) == NO_REGS)
3356 49768535 : || no_input_reloads_p))
3357 : {
3358 1599757 : const_to_mem = 1;
3359 1599757 : if (! no_regs_p)
3360 : {
3361 732241 : if (lra_dump_file != NULL)
3362 0 : fprintf
3363 0 : (lra_dump_file,
3364 : " %d Constant reload through memory: "
3365 : "losers++\n", nop);
3366 732241 : losers++;
3367 : }
3368 : }
3369 :
3370 : /* Alternative loses if it requires a type of reload not
3371 : permitted for this insn. We can always reload
3372 : objects with a REG_UNUSED note. */
3373 180552730 : if ((curr_static_id->operand[nop].type != OP_IN
3374 87716519 : && no_output_reloads_p
3375 0 : && ! find_reg_note (curr_insn, REG_UNUSED, op)
3376 0 : && ! scratch_p)
3377 180552730 : || (curr_static_id->operand[nop].type != OP_OUT
3378 92836419 : && no_input_reloads_p && ! const_to_mem)
3379 361105460 : || (this_alternative_matches >= 0
3380 42655949 : && (no_input_reloads_p
3381 42655949 : || (no_output_reloads_p
3382 0 : && (curr_static_id->operand
3383 0 : [this_alternative_matches].type != OP_IN)
3384 0 : && ! find_reg_note (curr_insn, REG_UNUSED,
3385 : no_subreg_reg_operand
3386 0 : [this_alternative_matches])
3387 0 : && ! scratch_p))))
3388 : {
3389 0 : if (lra_dump_file != NULL)
3390 0 : fprintf
3391 0 : (lra_dump_file,
3392 : " No input/output reload -- refuse\n");
3393 0 : goto fail;
3394 : }
3395 :
3396 : /* Alternative loses if it required class pseudo cannot
3397 : hold value of required mode. Such insns can be
3398 : described by insn definitions with mode iterators. */
3399 180552730 : if (GET_MODE (*curr_id->operand_loc[nop]) != VOIDmode
3400 130667742 : && ! hard_reg_set_empty_p (this_alternative_set)
3401 : /* It is common practice for constraints to use a
3402 : class which does not have actually enough regs to
3403 : hold the value (e.g. x86 AREG for mode requiring
3404 : more one general reg). Therefore we have 2
3405 : conditions to check that the reload pseudo cannot
3406 : hold the mode value. */
3407 121819108 : && (!targetm.hard_regno_mode_ok
3408 121819108 : (ira_class_hard_regs[this_alternative][0],
3409 : GET_MODE (*curr_id->operand_loc[nop])))
3410 : /* The above condition is not enough as the first
3411 : reg in ira_class_hard_regs can be not aligned for
3412 : multi-words mode values. */
3413 180552730 : && (prohibited_class_reg_set_mode_p
3414 0 : (this_alternative, this_alternative_set,
3415 0 : GET_MODE (*curr_id->operand_loc[nop]))))
3416 : {
3417 0 : if (lra_dump_file != NULL)
3418 0 : fprintf (lra_dump_file,
3419 : " reload pseudo for op %d "
3420 : "cannot hold the mode value -- refuse\n",
3421 : nop);
3422 0 : goto fail;
3423 : }
3424 :
3425 : /* Check strong discouragement of reload of non-constant
3426 : into class THIS_ALTERNATIVE. */
3427 129184435 : if (! CONSTANT_P (op) && ! no_regs_p
3428 300991220 : && (targetm.preferred_reload_class
3429 120438490 : (op, this_alternative) == NO_REGS
3430 111697649 : || (curr_static_id->operand[nop].type == OP_OUT
3431 76871673 : && (targetm.preferred_output_reload_class
3432 76871673 : (op, this_alternative) == NO_REGS))))
3433 : {
3434 13370126 : if (offmemok && REG_P (op))
3435 : {
3436 808939 : if (lra_dump_file != NULL)
3437 0 : fprintf
3438 0 : (lra_dump_file,
3439 : " %d Spill pseudo into memory: reject+=3\n",
3440 : nop);
3441 808939 : reject += 3;
3442 : }
3443 : else
3444 : {
3445 12561187 : if (lra_dump_file != NULL)
3446 0 : fprintf
3447 0 : (lra_dump_file,
3448 : " %d Non-preferred reload: reject+=%d\n",
3449 : nop, LRA_MAX_REJECT);
3450 12561187 : reject += LRA_MAX_REJECT;
3451 : }
3452 : }
3453 :
3454 180552730 : if (! (MEM_P (op) && offmemok)
3455 180551853 : && ! (const_to_mem && constmemok))
3456 : {
3457 : /* We prefer to reload pseudos over reloading other
3458 : things, since such reloads may be able to be
3459 : eliminated later. So bump REJECT in other cases.
3460 : Don't do this in the case where we are forcing a
3461 : constant into memory and it will then win since
3462 : we don't want to have a different alternative
3463 : match then. */
3464 179563632 : if (! (REG_P (op) && REGNO (op) >= FIRST_PSEUDO_REGISTER))
3465 : {
3466 128837483 : if (lra_dump_file != NULL)
3467 1649 : fprintf
3468 1649 : (lra_dump_file,
3469 : " %d Non-pseudo reload: reject+=2\n",
3470 : nop);
3471 128837483 : reject += 2;
3472 : }
3473 :
3474 179563632 : if (! no_regs_p)
3475 170817917 : reload_nregs
3476 170817917 : += ira_reg_class_max_nregs[this_alternative][mode];
3477 :
3478 179563632 : if (SMALL_REGISTER_CLASS_P (this_alternative))
3479 : {
3480 921606 : if (lra_dump_file != NULL)
3481 45 : fprintf
3482 45 : (lra_dump_file,
3483 : " %d Small class reload: reject+=%d\n",
3484 : nop, LRA_LOSER_COST_FACTOR / 2);
3485 921606 : reject += LRA_LOSER_COST_FACTOR / 2;
3486 : }
3487 : }
3488 :
3489 : /* We are trying to spill pseudo into memory. It is
3490 : usually more costly than moving to a hard register
3491 : although it might takes the same number of
3492 : reloads.
3493 :
3494 : Non-pseudo spill may happen also. Suppose a target allows both
3495 : register and memory in the operand constraint alternatives,
3496 : then it's typical that an eliminable register has a substition
3497 : of "base + offset" which can either be reloaded by a simple
3498 : "new_reg <= base + offset" which will match the register
3499 : constraint, or a similar reg addition followed by further spill
3500 : to and reload from memory which will match the memory
3501 : constraint, but this memory spill will be much more costly
3502 : usually.
3503 :
3504 : Code below increases the reject for both pseudo and non-pseudo
3505 : spill. */
3506 180552730 : if (no_regs_p
3507 9613461 : && !(MEM_P (op) && offmemok)
3508 9613104 : && !(REG_P (op) && hard_regno[nop] < 0))
3509 : {
3510 8539701 : if (lra_dump_file != NULL)
3511 13 : fprintf
3512 20 : (lra_dump_file,
3513 : " %d Spill %spseudo into memory: reject+=3\n",
3514 : nop, REG_P (op) ? "" : "Non-");
3515 8539701 : reject += 3;
3516 8539701 : if (VECTOR_MODE_P (mode))
3517 : {
3518 : /* Spilling vectors into memory is usually more
3519 : costly as they contain big values. */
3520 383555 : if (lra_dump_file != NULL)
3521 0 : fprintf
3522 0 : (lra_dump_file,
3523 : " %d Spill vector pseudo: reject+=2\n",
3524 : nop);
3525 383555 : reject += 2;
3526 : }
3527 : }
3528 :
3529 : /* When we use an operand requiring memory in given
3530 : alternative, the insn should write *and* read the
3531 : value to/from memory it is costly in comparison with
3532 : an insn alternative which does not use memory
3533 : (e.g. register or immediate operand). We exclude
3534 : memory operand for such case as we can satisfy the
3535 : memory constraints by reloading address. */
3536 9613461 : if (no_regs_p && offmemok && !MEM_P (op))
3537 : {
3538 9613104 : if (lra_dump_file != NULL)
3539 26 : fprintf
3540 26 : (lra_dump_file,
3541 : " Using memory insn operand %d: reject+=3\n",
3542 : nop);
3543 9613104 : reject += 3;
3544 : }
3545 :
3546 : /* If reload requires moving value through secondary
3547 : memory, it will need one more insn at least. */
3548 180552730 : if (this_alternative != NO_REGS
3549 170939125 : && REG_P (op) && (cl = get_reg_class (REGNO (op))) != NO_REGS
3550 217935103 : && ((curr_static_id->operand[nop].type != OP_OUT
3551 21077378 : && targetm.secondary_memory_needed (mode, cl,
3552 : this_alternative))
3553 34213583 : || (curr_static_id->operand[nop].type != OP_IN
3554 16305113 : && (targetm.secondary_memory_needed
3555 16305113 : (mode, this_alternative, cl)))))
3556 : {
3557 11005718 : if (lra_dump_file != NULL)
3558 16 : fprintf
3559 16 : (lra_dump_file,
3560 : " %d Secondary memory reload needed: "
3561 : "losers++\n", nop);
3562 11005718 : losers++;
3563 : }
3564 :
3565 180552730 : if (MEM_P (op) && offmemok)
3566 877 : addr_losers++;
3567 : else
3568 : {
3569 : /* Input reloads can be inherited more often than
3570 : output reloads can be removed, so penalize output
3571 : reloads. */
3572 180551853 : if (!REG_P (op) || curr_static_id->operand[nop].type != OP_IN)
3573 : {
3574 151951959 : if (lra_dump_file != NULL)
3575 1709 : fprintf
3576 1709 : (lra_dump_file,
3577 : " %d Non input pseudo reload: reject++\n",
3578 : nop);
3579 151951959 : reject++;
3580 : }
3581 :
3582 180551853 : if (curr_static_id->operand[nop].type == OP_INOUT)
3583 : {
3584 208 : if (lra_dump_file != NULL)
3585 0 : fprintf
3586 0 : (lra_dump_file,
3587 : " %d Input/Output reload: reject+=%d\n",
3588 : nop, LRA_LOSER_COST_FACTOR);
3589 208 : reject += LRA_LOSER_COST_FACTOR;
3590 : }
3591 : }
3592 : }
3593 :
3594 481567116 : if (early_clobber_p && ! scratch_p)
3595 : {
3596 179751 : if (lra_dump_file != NULL)
3597 4 : fprintf (lra_dump_file,
3598 : " %d Early clobber: reject++\n", nop);
3599 179751 : reject++;
3600 : }
3601 : /* ??? We check early clobbers after processing all operands
3602 : (see loop below) and there we update the costs more.
3603 : Should we update the cost (may be approximately) here
3604 : because of early clobber register reloads or it is a rare
3605 : or non-important thing to be worth to do it. */
3606 963134232 : overall = (losers * LRA_LOSER_COST_FACTOR + reject
3607 481567116 : - (addr_losers == losers ? static_reject : 0));
3608 481567116 : if ((best_losers == 0 || losers != 0) && best_overall < overall)
3609 : {
3610 76671659 : if (lra_dump_file != NULL)
3611 1029 : fprintf (lra_dump_file,
3612 : " overall=%d,losers=%d -- refuse\n",
3613 : overall, losers);
3614 76671659 : goto fail;
3615 : }
3616 :
3617 404895457 : if (update_and_check_small_class_inputs (nop, nalt,
3618 : this_alternative))
3619 : {
3620 0 : if (lra_dump_file != NULL)
3621 0 : fprintf (lra_dump_file,
3622 : " not enough small class regs -- refuse\n");
3623 0 : goto fail;
3624 : }
3625 404895457 : curr_alt[nop] = this_alternative;
3626 404895457 : curr_alt_set[nop] = this_alternative_set;
3627 404895457 : curr_alt_exclude_start_hard_regs[nop]
3628 404895457 : = this_alternative_exclude_start_hard_regs;
3629 404895457 : curr_alt_win[nop] = this_alternative_win;
3630 404895457 : curr_alt_match_win[nop] = this_alternative_match_win;
3631 404895457 : curr_alt_offmemok[nop] = this_alternative_offmemok;
3632 404895457 : curr_alt_matches[nop] = this_alternative_matches;
3633 :
3634 404895457 : if (this_alternative_matches >= 0
3635 404895457 : && !did_match && !this_alternative_win)
3636 13851287 : curr_alt_win[this_alternative_matches] = false;
3637 :
3638 404895457 : if (early_clobber_p && operand_reg[nop] != NULL_RTX)
3639 183517 : early_clobbered_nops[early_clobbered_regs_num++] = nop;
3640 : }
3641 :
3642 138576493 : if (curr_insn_set != NULL_RTX
3643 : /* Allow just two operands or three operands where the third
3644 : is a clobber. */
3645 134563097 : && (n_operands == 2
3646 29693083 : || (n_operands == 3
3647 27427794 : && GET_CODE (PATTERN (curr_insn)) == PARALLEL
3648 23254108 : && XVECLEN (PATTERN (curr_insn), 0) == 2
3649 23199084 : && GET_CODE (XVECEXP (PATTERN (curr_insn), 0, 1))
3650 : == CLOBBER))
3651 : /* Prevent processing non-move insns. */
3652 127983335 : && (GET_CODE (SET_SRC (curr_insn_set)) == SUBREG
3653 126098618 : || SET_SRC (curr_insn_set) == no_subreg_reg_operand[1])
3654 232274820 : && ((! curr_alt_win[0] && ! curr_alt_win[1]
3655 6106306 : && REG_P (no_subreg_reg_operand[0])
3656 2958521 : && REG_P (no_subreg_reg_operand[1])
3657 1233907 : && (reg_in_class_p (no_subreg_reg_operand[0], curr_alt[1])
3658 1017488 : || reg_in_class_p (no_subreg_reg_operand[1], curr_alt[0])))
3659 93152317 : || (! curr_alt_win[0] && curr_alt_win[1]
3660 27680577 : && REG_P (no_subreg_reg_operand[1])
3661 : /* Check that we reload memory not the memory
3662 : address. */
3663 15826977 : && ! (curr_alt_offmemok[0]
3664 392912 : && MEM_P (no_subreg_reg_operand[0]))
3665 15826977 : && reg_in_class_p (no_subreg_reg_operand[1], curr_alt[0]))
3666 78739963 : || (curr_alt_win[0] && ! curr_alt_win[1]
3667 10045462 : && REG_P (no_subreg_reg_operand[0])
3668 : /* Check that we reload memory not the memory
3669 : address. */
3670 7522253 : && ! (curr_alt_offmemok[1]
3671 1395832 : && MEM_P (no_subreg_reg_operand[1]))
3672 7522251 : && reg_in_class_p (no_subreg_reg_operand[0], curr_alt[1])
3673 6625078 : && (! CONST_POOL_OK_P (curr_operand_mode[1],
3674 : no_subreg_reg_operand[1])
3675 2618885 : || (targetm.preferred_reload_class
3676 2618885 : (no_subreg_reg_operand[1],
3677 : (enum reg_class) curr_alt[1]) != NO_REGS))
3678 : /* If it is a result of recent elimination in move
3679 : insn we can transform it into an add still by
3680 : using this alternative. */
3681 6581221 : && GET_CODE (no_subreg_reg_operand[1]) != PLUS
3682 : /* Likewise if the source has been replaced with an
3683 : equivalent value. This only happens once -- the reload
3684 : will use the equivalent value instead of the register it
3685 : replaces -- so there should be no danger of cycling. */
3686 6030291 : && !equiv_substition_p[1])))
3687 : {
3688 : /* We have a move insn and a new reload insn will be similar
3689 : to the current insn. We should avoid such situation as
3690 : it results in LRA cycling. */
3691 20938172 : if (lra_dump_file != NULL)
3692 233 : fprintf (lra_dump_file,
3693 : " Cycle danger: overall += LRA_MAX_REJECT\n");
3694 20938172 : overall += LRA_MAX_REJECT;
3695 : }
3696 138576493 : if (all_this_alternative != NO_REGS
3697 118709642 : && !SMALL_REGISTER_CLASS_P (all_this_alternative)
3698 117864635 : && all_used_nregs != 0 && all_reload_nregs != 0
3699 138576493 : && (all_used_nregs + all_reload_nregs + 1
3700 4250112 : >= ira_class_hard_regs_num[all_this_alternative]))
3701 : {
3702 363 : if (lra_dump_file != NULL)
3703 0 : fprintf
3704 0 : (lra_dump_file,
3705 : " Register starvation: overall += LRA_MAX_REJECT"
3706 : "(class=%s,avail=%d,used=%d,reload=%d)\n",
3707 : reg_class_names[all_this_alternative],
3708 : ira_class_hard_regs_num[all_this_alternative],
3709 : all_used_nregs, all_reload_nregs);
3710 363 : overall += LRA_MAX_REJECT;
3711 363 : if (!prefer_memory_p && INSN_CODE (curr_insn) < 0)
3712 : {
3713 : /* asm can permit memory and reg and can be not enough regs for
3714 : asm -- try now memory: */
3715 100 : prefer_memory_p = true;
3716 100 : if (lra_dump_file != NULL)
3717 0 : fprintf
3718 0 : (lra_dump_file,
3719 : " Trying now memory for operands\n");
3720 100 : goto repeat;
3721 : }
3722 : }
3723 138576393 : ok_p = true;
3724 138576393 : curr_alt_dont_inherit_ops_num = 0;
3725 138755446 : for (nop = 0; nop < early_clobbered_regs_num; nop++)
3726 : {
3727 179054 : int i, j, clobbered_hard_regno, first_conflict_j, last_conflict_j;
3728 179054 : HARD_REG_SET temp_set;
3729 :
3730 179054 : i = early_clobbered_nops[nop];
3731 179054 : if ((! curr_alt_win[i] && ! curr_alt_match_win[i])
3732 133259 : || hard_regno[i] < 0)
3733 178350 : continue;
3734 131687 : lra_assert (operand_reg[i] != NULL_RTX);
3735 : clobbered_hard_regno = hard_regno[i];
3736 131687 : CLEAR_HARD_REG_SET (temp_set);
3737 131687 : add_to_hard_reg_set (&temp_set, GET_MODE (*curr_id->operand_loc[i]),
3738 : clobbered_hard_regno);
3739 131687 : first_conflict_j = last_conflict_j = -1;
3740 794750 : for (j = 0; j < n_operands; j++)
3741 531377 : if (j == i
3742 : /* We don't want process insides of match_operator and
3743 : match_parallel because otherwise we would process
3744 : their operands once again generating a wrong
3745 : code. */
3746 399690 : || curr_static_id->operand[j].is_operator)
3747 133801 : continue;
3748 397576 : else if ((curr_alt_matches[j] == i && curr_alt_match_win[j])
3749 377340 : || (curr_alt_matches[i] == j && curr_alt_match_win[i]))
3750 20236 : continue;
3751 : /* If we don't reload j-th operand, check conflicts. */
3752 142111 : else if ((curr_alt_win[j] || curr_alt_match_win[j])
3753 434236 : && uses_hard_regs_p (*curr_id->operand_loc[j], temp_set))
3754 : {
3755 1139 : if (first_conflict_j < 0)
3756 704 : first_conflict_j = j;
3757 1139 : last_conflict_j = j;
3758 : /* Both the earlyclobber operand and conflicting operand
3759 : cannot both be user defined hard registers for asm.
3760 : Let curr_insn_transform diagnose it. */
3761 1139 : if (HARD_REGISTER_P (operand_reg[i])
3762 1 : && REG_USERVAR_P (operand_reg[i])
3763 1 : && operand_reg[j] != NULL_RTX
3764 1 : && HARD_REGISTER_P (operand_reg[j])
3765 1 : && REG_USERVAR_P (operand_reg[j])
3766 1140 : && INSN_CODE (curr_insn) < 0)
3767 1 : return false;
3768 : }
3769 131686 : if (last_conflict_j < 0)
3770 130983 : continue;
3771 :
3772 : /* If an earlyclobber operand conflicts with another non-matching
3773 : operand (ie, they have been assigned the same hard register),
3774 : then it is better to reload the other operand, as there may
3775 : exist yet another operand with a matching constraint associated
3776 : with the earlyclobber operand. However, if one of the operands
3777 : is an explicit use of a hard register, then we must reload the
3778 : other non-hard register operand. */
3779 703 : if (HARD_REGISTER_P (operand_reg[i])
3780 703 : || (first_conflict_j == last_conflict_j
3781 268 : && operand_reg[last_conflict_j] != NULL_RTX
3782 62 : && !curr_alt_match_win[last_conflict_j]
3783 62 : && !HARD_REGISTER_P (operand_reg[last_conflict_j])))
3784 : {
3785 62 : curr_alt_win[last_conflict_j] = false;
3786 62 : curr_alt_dont_inherit_ops[curr_alt_dont_inherit_ops_num++]
3787 62 : = last_conflict_j;
3788 62 : losers++;
3789 62 : if (lra_dump_file != NULL)
3790 0 : fprintf
3791 0 : (lra_dump_file,
3792 : " %d Conflict early clobber reload: losers++\n",
3793 : i);
3794 : }
3795 : else
3796 : {
3797 : /* We need to reload early clobbered register and the
3798 : matched registers. */
3799 3199 : for (j = 0; j < n_operands; j++)
3800 2558 : if (curr_alt_matches[j] == i)
3801 : {
3802 2 : curr_alt_match_win[j] = false;
3803 2 : losers++;
3804 2 : if (lra_dump_file != NULL)
3805 0 : fprintf
3806 0 : (lra_dump_file,
3807 : " %d Matching conflict early clobber "
3808 : "reloads: losers++\n",
3809 : j);
3810 2 : overall += LRA_LOSER_COST_FACTOR;
3811 : }
3812 641 : if (! curr_alt_match_win[i])
3813 641 : curr_alt_dont_inherit_ops[curr_alt_dont_inherit_ops_num++] = i;
3814 : else
3815 : {
3816 : /* Remember pseudos used for match reloads are never
3817 : inherited. */
3818 0 : lra_assert (curr_alt_matches[i] >= 0);
3819 0 : curr_alt_win[curr_alt_matches[i]] = false;
3820 : }
3821 641 : curr_alt_win[i] = curr_alt_match_win[i] = false;
3822 641 : losers++;
3823 641 : if (lra_dump_file != NULL)
3824 0 : fprintf
3825 0 : (lra_dump_file,
3826 : " %d Matched conflict early clobber reloads: "
3827 : "losers++\n",
3828 : i);
3829 : }
3830 : /* Early clobber was already reflected in REJECT. */
3831 703 : if (!matching_early_clobber[i])
3832 : {
3833 703 : lra_assert (reject > 0);
3834 703 : reject--;
3835 703 : matching_early_clobber[i] = 1;
3836 : }
3837 703 : overall += LRA_LOSER_COST_FACTOR - 1;
3838 : }
3839 138576392 : if (lra_dump_file != NULL)
3840 1752 : fprintf (lra_dump_file, " overall=%d,losers=%d,rld_nregs=%d\n",
3841 : overall, losers, reload_nregs);
3842 :
3843 : /* If this alternative can be made to work by reloading, and it
3844 : needs less reloading than the others checked so far, record
3845 : it as the chosen goal for reloading. */
3846 138576392 : if ((best_losers != 0 && losers == 0)
3847 61548299 : || (((best_losers == 0 && losers == 0)
3848 60464298 : || (best_losers != 0 && losers != 0))
3849 61548299 : && (best_overall > overall
3850 15773943 : || (best_overall == overall
3851 : /* If the cost of the reloads is the same,
3852 : prefer alternative which requires minimal
3853 : number of reload regs. */
3854 11838779 : && (reload_nregs < best_reload_nregs
3855 11737543 : || (reload_nregs == best_reload_nregs
3856 11694364 : && (best_reload_sum < reload_sum
3857 11670494 : || (best_reload_sum == reload_sum
3858 11647810 : && nalt < goal_alt_number))))))))
3859 : {
3860 400282978 : for (nop = 0; nop < n_operands; nop++)
3861 : {
3862 277069273 : goal_alt_win[nop] = curr_alt_win[nop];
3863 277069273 : goal_alt_match_win[nop] = curr_alt_match_win[nop];
3864 277069273 : goal_alt_matches[nop] = curr_alt_matches[nop];
3865 277069273 : goal_alt[nop] = curr_alt[nop];
3866 277069273 : goal_alt_exclude_start_hard_regs[nop]
3867 277069273 : = curr_alt_exclude_start_hard_regs[nop];
3868 277069273 : goal_alt_offmemok[nop] = curr_alt_offmemok[nop];
3869 : }
3870 123213705 : goal_alt_dont_inherit_ops_num = curr_alt_dont_inherit_ops_num;
3871 123213705 : goal_reuse_alt_p = curr_reuse_alt_p;
3872 123214398 : for (nop = 0; nop < curr_alt_dont_inherit_ops_num; nop++)
3873 693 : goal_alt_dont_inherit_ops[nop] = curr_alt_dont_inherit_ops[nop];
3874 123213705 : goal_alt_swapped = curr_swapped;
3875 123213705 : goal_alt_out_sp_reload_p = curr_alt_out_sp_reload_p;
3876 123213705 : best_overall = overall;
3877 123213705 : best_losers = losers;
3878 123213705 : best_reload_nregs = reload_nregs;
3879 123213705 : best_reload_sum = reload_sum;
3880 123213705 : goal_alt_number = nalt;
3881 : }
3882 138576392 : if (losers == 0 && !curr_alt_class_change_p)
3883 : /* Everything is satisfied. Do not process alternatives
3884 : anymore. */
3885 : break;
3886 60477016 : fail:
3887 186506866 : ;
3888 : }
3889 : return ok_p;
3890 : }
3891 :
3892 : /* Make reload base reg from address AD. */
3893 : static rtx
3894 0 : base_to_reg (struct address_info *ad)
3895 : {
3896 0 : enum reg_class cl;
3897 0 : int code = -1;
3898 0 : rtx new_inner = NULL_RTX;
3899 0 : rtx new_reg = NULL_RTX;
3900 0 : rtx_insn *insn;
3901 0 : rtx_insn *last_insn = get_last_insn();
3902 :
3903 0 : lra_assert (ad->disp == ad->disp_term);
3904 0 : cl = base_reg_class (ad->mode, ad->as, ad->base_outer_code,
3905 : get_index_code (ad), ad->mem, curr_insn);
3906 0 : new_reg = lra_create_new_reg (GET_MODE (*ad->base), NULL_RTX, cl, NULL,
3907 : "base");
3908 0 : new_inner = simplify_gen_binary (PLUS, GET_MODE (new_reg), new_reg,
3909 0 : ad->disp_term == NULL
3910 : ? const0_rtx
3911 : : *ad->disp_term);
3912 0 : if (!valid_address_p (ad->mode, new_inner, ad->as))
3913 : return NULL_RTX;
3914 0 : insn = emit_insn (gen_rtx_SET (new_reg, *ad->base));
3915 0 : code = recog_memoized (insn);
3916 0 : if (code < 0)
3917 : {
3918 0 : delete_insns_since (last_insn);
3919 0 : return NULL_RTX;
3920 : }
3921 :
3922 : return new_inner;
3923 : }
3924 :
3925 : /* Make reload base reg + DISP from address AD. Return the new pseudo. */
3926 : static rtx
3927 47 : base_plus_disp_to_reg (struct address_info *ad, rtx disp)
3928 : {
3929 47 : enum reg_class cl;
3930 47 : rtx new_reg;
3931 :
3932 47 : lra_assert (ad->base == ad->base_term);
3933 47 : cl = base_reg_class (ad->mode, ad->as, ad->base_outer_code,
3934 : get_index_code (ad), ad->mem, curr_insn);
3935 47 : new_reg = lra_create_new_reg (GET_MODE (*ad->base_term), NULL_RTX, cl, NULL,
3936 : "base + disp");
3937 47 : lra_emit_add (new_reg, *ad->base_term, disp);
3938 47 : return new_reg;
3939 : }
3940 :
3941 : /* Make reload of index part of address AD. Return the new
3942 : pseudo. */
3943 : static rtx
3944 0 : index_part_to_reg (struct address_info *ad, enum reg_class index_class)
3945 : {
3946 0 : rtx new_reg;
3947 :
3948 0 : new_reg = lra_create_new_reg (GET_MODE (*ad->index), NULL_RTX,
3949 : index_class, NULL, "index term");
3950 0 : expand_mult (GET_MODE (*ad->index), *ad->index_term,
3951 : GEN_INT (get_index_scale (ad)), new_reg, 1);
3952 0 : return new_reg;
3953 : }
3954 :
3955 : /* Return true if we can add a displacement to address AD, even if that
3956 : makes the address invalid. The fix-up code requires any new address
3957 : to be the sum of the BASE_TERM, INDEX and DISP_TERM fields. */
3958 : static bool
3959 21204 : can_add_disp_p (struct address_info *ad)
3960 : {
3961 21204 : return (!ad->autoinc_p
3962 21204 : && ad->segment == NULL
3963 21204 : && ad->base == ad->base_term
3964 42408 : && ad->disp == ad->disp_term);
3965 : }
3966 :
3967 : /* Make equiv substitution in address AD. Return true if a substitution
3968 : was made. */
3969 : static bool
3970 40349813 : equiv_address_substitution (struct address_info *ad)
3971 : {
3972 40349813 : rtx base_reg, new_base_reg, index_reg, new_index_reg, *base_term, *index_term;
3973 40349813 : poly_int64 disp;
3974 40349813 : HOST_WIDE_INT scale;
3975 40349813 : bool change_p;
3976 :
3977 40349813 : base_term = strip_subreg (ad->base_term);
3978 10119 : if (base_term == NULL)
3979 : base_reg = new_base_reg = NULL_RTX;
3980 : else
3981 : {
3982 34096058 : base_reg = *base_term;
3983 34096058 : new_base_reg = get_equiv_with_elimination (base_reg, curr_insn);
3984 : }
3985 40349813 : index_term = strip_subreg (ad->index_term);
3986 6052 : if (index_term == NULL)
3987 : index_reg = new_index_reg = NULL_RTX;
3988 : else
3989 : {
3990 1813472 : index_reg = *index_term;
3991 1813472 : new_index_reg = get_equiv_with_elimination (index_reg, curr_insn);
3992 : }
3993 40349813 : if (base_reg == new_base_reg && index_reg == new_index_reg)
3994 : return false;
3995 142691 : disp = 0;
3996 142691 : change_p = false;
3997 142691 : if (lra_dump_file != NULL)
3998 : {
3999 0 : fprintf (lra_dump_file, "Changing address in insn %d ",
4000 0 : INSN_UID (curr_insn));
4001 0 : dump_value_slim (lra_dump_file, *ad->outer, 1);
4002 : }
4003 142691 : if (base_reg != new_base_reg)
4004 : {
4005 142232 : poly_int64 offset;
4006 142232 : if (REG_P (new_base_reg))
4007 : {
4008 8466 : *base_term = new_base_reg;
4009 8466 : change_p = true;
4010 : }
4011 133766 : else if (GET_CODE (new_base_reg) == PLUS
4012 21203 : && REG_P (XEXP (new_base_reg, 0))
4013 21203 : && poly_int_rtx_p (XEXP (new_base_reg, 1), &offset)
4014 154969 : && can_add_disp_p (ad))
4015 : {
4016 : disp += offset;
4017 21203 : *base_term = XEXP (new_base_reg, 0);
4018 21203 : change_p = true;
4019 : }
4020 142232 : if (ad->base_term2 != NULL)
4021 0 : *ad->base_term2 = *ad->base_term;
4022 : }
4023 142691 : if (index_reg != new_index_reg)
4024 : {
4025 636 : poly_int64 offset;
4026 636 : if (REG_P (new_index_reg))
4027 : {
4028 0 : *index_term = new_index_reg;
4029 0 : change_p = true;
4030 : }
4031 636 : else if (GET_CODE (new_index_reg) == PLUS
4032 1 : && REG_P (XEXP (new_index_reg, 0))
4033 1 : && poly_int_rtx_p (XEXP (new_index_reg, 1), &offset)
4034 1 : && can_add_disp_p (ad)
4035 637 : && (scale = get_index_scale (ad)))
4036 : {
4037 1 : disp += offset * scale;
4038 1 : *index_term = XEXP (new_index_reg, 0);
4039 1 : change_p = true;
4040 : }
4041 : }
4042 142691 : if (maybe_ne (disp, 0))
4043 : {
4044 21204 : if (ad->disp != NULL)
4045 7840 : *ad->disp = plus_constant (GET_MODE (*ad->inner), *ad->disp, disp);
4046 : else
4047 : {
4048 13364 : *ad->inner = plus_constant (GET_MODE (*ad->inner), *ad->inner, disp);
4049 13364 : update_address (ad);
4050 : }
4051 : change_p = true;
4052 : }
4053 142691 : if (lra_dump_file != NULL)
4054 : {
4055 0 : if (! change_p)
4056 0 : fprintf (lra_dump_file, " -- no change\n");
4057 : else
4058 : {
4059 0 : fprintf (lra_dump_file, " on equiv ");
4060 0 : dump_value_slim (lra_dump_file, *ad->outer, 1);
4061 0 : fprintf (lra_dump_file, "\n");
4062 : }
4063 : }
4064 : return change_p;
4065 : }
4066 :
4067 : /* Skip all modifiers and whitespaces in constraint STR and return the
4068 : result. */
4069 : static const char *
4070 526196241 : skip_constraint_modifiers (const char *str)
4071 : {
4072 740348017 : for (;;str++)
4073 633272129 : switch (*str)
4074 : {
4075 107075888 : case '+': case '&' : case '=': case '*': case ' ': case '\t':
4076 107075888 : case '$': case '^' : case '%': case '?': case '!':
4077 107075888 : break;
4078 526196241 : default: return str;
4079 : }
4080 : }
4081 :
4082 : /* Takes a string of 0 or more comma-separated constraints. When more
4083 : than one constraint is present, evaluate whether they all correspond
4084 : to a single, repeated constraint (e.g. "r,r") or whether we have
4085 : more than one distinct constraints (e.g. "r,m"). */
4086 : static bool
4087 166825460 : constraint_unique (const char *cstr)
4088 : {
4089 166825460 : enum constraint_num ca, cb;
4090 166825460 : ca = CONSTRAINT__UNKNOWN;
4091 331128233 : for (;;)
4092 : {
4093 331128233 : cstr = skip_constraint_modifiers (cstr);
4094 331128233 : if (*cstr == '\0' || *cstr == ',')
4095 : cb = CONSTRAINT_X;
4096 : else
4097 : {
4098 331128233 : cb = lookup_constraint (cstr);
4099 331128233 : if (cb == CONSTRAINT__UNKNOWN)
4100 : return false;
4101 318886544 : cstr += CONSTRAINT_LEN (cstr[0], cstr);
4102 : }
4103 : /* Handle the first iteration of the loop. */
4104 318886544 : if (ca == CONSTRAINT__UNKNOWN)
4105 : ca = cb;
4106 : /* Handle the general case of comparing ca with subsequent
4107 : constraints. */
4108 164139287 : else if (ca != cb)
4109 : return false;
4110 172097889 : if (*cstr == '\0')
4111 : return true;
4112 164302773 : if (*cstr == ',')
4113 90832731 : cstr += 1;
4114 : }
4115 : }
4116 :
4117 : /* Major function to make reloads for an address in operand NOP or
4118 : check its correctness (If CHECK_ONLY_P is true). The supported
4119 : cases are:
4120 :
4121 : 1) an address that existed before LRA started, at which point it
4122 : must have been valid. These addresses are subject to elimination
4123 : and may have become invalid due to the elimination offset being out
4124 : of range.
4125 :
4126 : 2) an address created by forcing a constant to memory
4127 : (force_const_to_mem). The initial form of these addresses might
4128 : not be valid, and it is this function's job to make them valid.
4129 :
4130 : 3) a frame address formed from a register and a (possibly zero)
4131 : constant offset. As above, these addresses might not be valid and
4132 : this function must make them so.
4133 :
4134 : Add reloads to the lists *BEFORE and *AFTER. We might need to add
4135 : reloads to *AFTER because of inc/dec, {pre, post} modify in the
4136 : address. Return true for any RTL change.
4137 :
4138 : The function is a helper function which does not produce all
4139 : transformations (when CHECK_ONLY_P is false) which can be
4140 : necessary. It does just basic steps. To do all necessary
4141 : transformations use function process_address. */
4142 : static bool
4143 181039949 : process_address_1 (int nop, bool check_only_p,
4144 : rtx_insn **before, rtx_insn **after)
4145 : {
4146 181039949 : struct address_info ad;
4147 181039949 : rtx new_reg;
4148 181039949 : HOST_WIDE_INT scale;
4149 181039949 : rtx op = *curr_id->operand_loc[nop];
4150 181039949 : rtx mem = extract_mem_from_operand (op);
4151 181039949 : const char *constraint;
4152 181039949 : enum constraint_num cn;
4153 181039949 : bool change_p = false;
4154 :
4155 181039949 : if (MEM_P (mem)
4156 38474261 : && GET_MODE (mem) == BLKmode
4157 24736 : && GET_CODE (XEXP (mem, 0)) == SCRATCH)
4158 : return false;
4159 :
4160 181039949 : constraint
4161 181039949 : = skip_constraint_modifiers (curr_static_id->operand[nop].constraint);
4162 181039949 : if (IN_RANGE (constraint[0], '0', '9'))
4163 : {
4164 14028059 : char *end;
4165 14028059 : unsigned long dup = strtoul (constraint, &end, 10);
4166 14028059 : constraint
4167 14028059 : = skip_constraint_modifiers (curr_static_id->operand[dup].constraint);
4168 : }
4169 193371691 : cn = lookup_constraint (*constraint == '\0' ? "X" : constraint);
4170 : /* If we have several alternatives or/and several constraints in an
4171 : alternative and we can not say at this stage what constraint will be used,
4172 : use unknown constraint. The exception is an address constraint. If
4173 : operand has one address constraint, probably all others constraints are
4174 : address ones. */
4175 168708207 : if (constraint[0] != '\0' && get_constraint_type (cn) != CT_ADDRESS
4176 347865409 : && !constraint_unique (constraint))
4177 : cn = CONSTRAINT__UNKNOWN;
4178 22009605 : if (insn_extra_address_constraint (cn)
4179 : /* When we find an asm operand with an address constraint that
4180 : doesn't satisfy address_operand to begin with, we clear
4181 : is_address, so that we don't try to make a non-address fit.
4182 : If the asm statement got this far, it's because other
4183 : constraints are available, and we'll use them, disregarding
4184 : the unsatisfiable address ones. */
4185 22009605 : && curr_static_id->operand[nop].is_address)
4186 1882728 : decompose_lea_address (&ad, curr_id->operand_loc[nop]);
4187 : /* Do not attempt to decompose arbitrary addresses generated by combine
4188 : for asm operands with loose constraints, e.g 'X'.
4189 : Need to extract memory from op for special memory constraint,
4190 : i.e. bcst_mem_operand in i386 backend. */
4191 179157221 : else if (MEM_P (mem)
4192 179157386 : && !(INSN_CODE (curr_insn) < 0
4193 20565 : && get_constraint_type (cn) == CT_FIXED_FORM
4194 165 : && constraint_satisfied_p (op, cn)))
4195 38474096 : decompose_mem_address (&ad, mem);
4196 140683125 : else if (GET_CODE (op) == SUBREG
4197 3767527 : && MEM_P (SUBREG_REG (op)))
4198 0 : decompose_mem_address (&ad, SUBREG_REG (op));
4199 : else
4200 : return false;
4201 : /* If INDEX_REG_CLASS is assigned to base_term already and isn't to
4202 : index_term, swap them so to avoid assigning INDEX_REG_CLASS to both
4203 : when INDEX_REG_CLASS is a single register class. */
4204 40356824 : enum reg_class index_cl = index_reg_class (curr_insn);
4205 40356824 : if (ad.base_term != NULL
4206 34103014 : && ad.index_term != NULL
4207 1446036 : && ira_class_hard_regs_num[index_cl] == 1
4208 0 : && REG_P (*ad.base_term)
4209 0 : && REG_P (*ad.index_term)
4210 0 : && in_class_p (*ad.base_term, index_cl, NULL)
4211 40356824 : && ! in_class_p (*ad.index_term, index_cl, NULL))
4212 : {
4213 0 : std::swap (ad.base, ad.index);
4214 0 : std::swap (ad.base_term, ad.index_term);
4215 : }
4216 40356824 : if (! check_only_p)
4217 40349813 : change_p = equiv_address_substitution (&ad);
4218 40356824 : if (ad.base_term != NULL
4219 74459838 : && (process_addr_reg
4220 68206028 : (ad.base_term, check_only_p, before,
4221 34103014 : (ad.autoinc_p
4222 4189350 : && !(REG_P (*ad.base_term)
4223 2094675 : && find_regno_note (curr_insn, REG_DEAD,
4224 : REGNO (*ad.base_term)) != NULL_RTX)
4225 : ? after : NULL),
4226 : base_reg_class (ad.mode, ad.as, ad.base_outer_code,
4227 : get_index_code (&ad), ad.mem, curr_insn))))
4228 : {
4229 439031 : change_p = true;
4230 439031 : if (ad.base_term2 != NULL)
4231 0 : *ad.base_term2 = *ad.base_term;
4232 : }
4233 40356824 : if (ad.index_term != NULL
4234 40356824 : && process_addr_reg (ad.index_term, check_only_p,
4235 : before, NULL, index_cl))
4236 : change_p = true;
4237 :
4238 : /* Target hooks sometimes don't treat extra-constraint addresses as
4239 : legitimate address_operands, so handle them specially. */
4240 40356824 : if (insn_extra_address_constraint (cn)
4241 40356824 : && satisfies_address_constraint_p (&ad, cn))
4242 : return change_p;
4243 :
4244 38474103 : if (check_only_p)
4245 : return change_p;
4246 :
4247 : /* There are three cases where the shape of *AD.INNER may now be invalid:
4248 :
4249 : 1) the original address was valid, but either elimination or
4250 : equiv_address_substitution was applied and that made
4251 : the address invalid.
4252 :
4253 : 2) the address is an invalid symbolic address created by
4254 : force_const_to_mem.
4255 :
4256 : 3) the address is a frame address with an invalid offset.
4257 :
4258 : 4) the address is a frame address with an invalid base.
4259 :
4260 : All these cases involve a non-autoinc address, so there is no
4261 : point revalidating other types. */
4262 38467732 : if (ad.autoinc_p || valid_address_p (op, &ad, cn))
4263 : return change_p;
4264 :
4265 : /* Any index existed before LRA started, so we can assume that the
4266 : presence and shape of the index is valid. */
4267 452 : push_to_sequence (*before);
4268 452 : lra_assert (ad.disp == ad.disp_term);
4269 452 : if (ad.base == NULL)
4270 : {
4271 352 : if (ad.index == NULL)
4272 : {
4273 352 : rtx_insn *insn;
4274 352 : rtx_insn *last = get_last_insn ();
4275 352 : int code = -1;
4276 352 : enum reg_class cl = base_reg_class (ad.mode, ad.as,
4277 : SCRATCH, SCRATCH,
4278 : ad.mem, curr_insn);
4279 352 : rtx addr = *ad.inner;
4280 :
4281 695 : new_reg = lra_create_new_reg (Pmode, NULL_RTX, cl, NULL, "addr");
4282 352 : if (HAVE_lo_sum)
4283 : {
4284 : /* addr => lo_sum (new_base, addr), case (2) above. */
4285 : insn = emit_insn (gen_rtx_SET
4286 : (new_reg,
4287 : gen_rtx_HIGH (Pmode, copy_rtx (addr))));
4288 : code = recog_memoized (insn);
4289 : if (code >= 0)
4290 : {
4291 : *ad.inner = gen_rtx_LO_SUM (Pmode, new_reg, addr);
4292 : if (!valid_address_p (op, &ad, cn))
4293 : {
4294 : /* Try to put lo_sum into register. */
4295 : insn = emit_insn (gen_rtx_SET
4296 : (new_reg,
4297 : gen_rtx_LO_SUM (Pmode, new_reg, addr)));
4298 : code = recog_memoized (insn);
4299 : if (code >= 0)
4300 : {
4301 : *ad.inner = new_reg;
4302 : if (!valid_address_p (op, &ad, cn))
4303 : {
4304 : *ad.inner = addr;
4305 : code = -1;
4306 : }
4307 : }
4308 :
4309 : }
4310 : }
4311 : if (code < 0)
4312 : delete_insns_since (last);
4313 : }
4314 :
4315 352 : if (code < 0)
4316 : {
4317 : /* addr => new_base, case (2) above. */
4318 352 : lra_emit_move (new_reg, addr);
4319 :
4320 704 : for (insn = last == NULL_RTX ? get_insns () : NEXT_INSN (last);
4321 704 : insn != NULL_RTX;
4322 352 : insn = NEXT_INSN (insn))
4323 352 : if (recog_memoized (insn) < 0)
4324 : break;
4325 352 : if (insn != NULL_RTX)
4326 : {
4327 : /* Do nothing if we cannot generate right insns.
4328 : This is analogous to reload pass behavior. */
4329 0 : delete_insns_since (last);
4330 0 : end_sequence ();
4331 0 : return false;
4332 : }
4333 352 : *ad.inner = new_reg;
4334 : }
4335 : }
4336 : else
4337 : {
4338 : /* index * scale + disp => new base + index * scale,
4339 : case (1) above. */
4340 0 : enum reg_class cl = base_reg_class (ad.mode, ad.as, PLUS,
4341 0 : GET_CODE (*ad.index),
4342 : ad.mem, curr_insn);
4343 :
4344 0 : lra_assert (index_cl != NO_REGS);
4345 0 : new_reg = lra_create_new_reg (Pmode, NULL_RTX, cl, NULL, "disp");
4346 0 : lra_emit_move (new_reg, *ad.disp);
4347 0 : *ad.inner = simplify_gen_binary (PLUS, GET_MODE (new_reg),
4348 0 : new_reg, *ad.index);
4349 : }
4350 : }
4351 100 : else if (ad.index == NULL)
4352 : {
4353 53 : int regno;
4354 53 : enum reg_class cl;
4355 53 : rtx set;
4356 53 : rtx_insn *insns, *last_insn;
4357 :
4358 53 : cl = base_reg_class (ad.mode, ad.as, ad.base_outer_code,
4359 : get_index_code (&ad), ad.mem, curr_insn);
4360 :
4361 53 : if (REG_P (*ad.base_term)
4362 53 : && ira_class_subset_p[get_reg_class (REGNO (*ad.base_term))][cl])
4363 : /* It seems base reg is already in the base reg class and changing it
4364 : does not make a progress. So reload the whole inner address. */
4365 53 : goto reload_inner_addr;
4366 :
4367 : /* Try to reload base into register only if the base is invalid
4368 : for the address but with valid offset, case (4) above. */
4369 0 : start_sequence ();
4370 0 : new_reg = base_to_reg (&ad);
4371 :
4372 : /* base + disp => new base, cases (1) and (3) above. */
4373 : /* Another option would be to reload the displacement into an
4374 : index register. However, postreload has code to optimize
4375 : address reloads that have the same base and different
4376 : displacements, so reloading into an index register would
4377 : not necessarily be a win. */
4378 0 : if (new_reg == NULL_RTX)
4379 : {
4380 : /* See if the target can split the displacement into a
4381 : legitimate new displacement from a local anchor. */
4382 0 : gcc_assert (ad.disp == ad.disp_term);
4383 0 : poly_int64 orig_offset;
4384 0 : rtx offset1, offset2;
4385 0 : if (poly_int_rtx_p (*ad.disp, &orig_offset)
4386 0 : && targetm.legitimize_address_displacement (&offset1, &offset2,
4387 : orig_offset,
4388 : ad.mode))
4389 : {
4390 0 : new_reg = base_plus_disp_to_reg (&ad, offset1);
4391 0 : new_reg = gen_rtx_PLUS (GET_MODE (new_reg), new_reg, offset2);
4392 : }
4393 : else
4394 0 : new_reg = base_plus_disp_to_reg (&ad, *ad.disp);
4395 : }
4396 0 : insns = get_insns ();
4397 0 : last_insn = get_last_insn ();
4398 : /* If we generated at least two insns, try last insn source as
4399 : an address. If we succeed, we generate one less insn. */
4400 0 : if (REG_P (new_reg)
4401 0 : && last_insn != insns
4402 0 : && (set = single_set (last_insn)) != NULL_RTX
4403 0 : && GET_CODE (SET_SRC (set)) == PLUS
4404 0 : && REG_P (XEXP (SET_SRC (set), 0))
4405 0 : && CONSTANT_P (XEXP (SET_SRC (set), 1)))
4406 : {
4407 0 : *ad.inner = SET_SRC (set);
4408 0 : if (valid_address_p (op, &ad, cn))
4409 : {
4410 0 : *ad.base_term = XEXP (SET_SRC (set), 0);
4411 0 : *ad.disp_term = XEXP (SET_SRC (set), 1);
4412 0 : regno = REGNO (*ad.base_term);
4413 0 : if (regno >= FIRST_PSEUDO_REGISTER
4414 0 : && cl != lra_get_allocno_class (regno))
4415 0 : lra_change_class (regno, cl, " Change to", true);
4416 0 : new_reg = SET_SRC (set);
4417 0 : delete_insns_since (PREV_INSN (last_insn));
4418 : }
4419 : }
4420 0 : end_sequence ();
4421 0 : emit_insn (insns);
4422 0 : *ad.inner = new_reg;
4423 : }
4424 47 : else if (ad.disp_term != NULL)
4425 : {
4426 : /* base + scale * index + disp => new base + scale * index,
4427 : case (1) above. */
4428 47 : gcc_assert (ad.disp == ad.disp_term);
4429 47 : new_reg = base_plus_disp_to_reg (&ad, *ad.disp);
4430 47 : *ad.inner = simplify_gen_binary (PLUS, GET_MODE (new_reg),
4431 47 : new_reg, *ad.index);
4432 : }
4433 0 : else if ((scale = get_index_scale (&ad)) == 1)
4434 : {
4435 : /* The last transformation to one reg will be made in
4436 : curr_insn_transform function. */
4437 0 : end_sequence ();
4438 0 : return false;
4439 : }
4440 0 : else if (scale != 0)
4441 : {
4442 : /* base + scale * index => base + new_reg,
4443 : case (1) above.
4444 : Index part of address may become invalid. For example, we
4445 : changed pseudo on the equivalent memory and a subreg of the
4446 : pseudo onto the memory of different mode for which the scale is
4447 : prohibited. */
4448 0 : new_reg = index_part_to_reg (&ad, index_cl);
4449 0 : *ad.inner = simplify_gen_binary (PLUS, GET_MODE (new_reg),
4450 0 : *ad.base_term, new_reg);
4451 : }
4452 : else
4453 : {
4454 53 : enum reg_class cl;
4455 53 : rtx addr;
4456 0 : reload_inner_addr:
4457 53 : cl = base_reg_class (ad.mode, ad.as, SCRATCH, SCRATCH, ad.mem,
4458 : curr_insn);
4459 53 : addr = *ad.inner;
4460 53 : new_reg = lra_create_new_reg (Pmode, NULL_RTX, cl, NULL, "addr");
4461 : /* addr => new_base. */
4462 53 : lra_emit_move (new_reg, canonicalize_reload_addr (addr));
4463 53 : *ad.inner = new_reg;
4464 : }
4465 452 : *before = end_sequence ();
4466 452 : return true;
4467 : }
4468 :
4469 : /* If CHECK_ONLY_P is false, do address reloads until it is necessary.
4470 : Use process_address_1 as a helper function. Return true for any
4471 : RTL changes.
4472 :
4473 : If CHECK_ONLY_P is true, just check address correctness. Return
4474 : false if the address correct. */
4475 : static bool
4476 180543659 : process_address (int nop, bool check_only_p,
4477 : rtx_insn **before, rtx_insn **after)
4478 : {
4479 180543659 : bool res = false;
4480 : /* Use enough iterations to process all address parts: */
4481 181039949 : for (int i = 0; i < 10; i++)
4482 : {
4483 181039949 : if (!process_address_1 (nop, check_only_p, before, after))
4484 : {
4485 : return res;
4486 : }
4487 : else
4488 : {
4489 496290 : if (check_only_p)
4490 : return true;
4491 496290 : res = true;
4492 : }
4493 : }
4494 0 : fatal_insn ("unable to reload address in ", curr_insn);
4495 : }
4496 :
4497 : /* Override the generic address_reload_context in order to
4498 : control the creation of reload pseudos. */
4499 : class lra_autoinc_reload_context : public address_reload_context
4500 : {
4501 : machine_mode mode;
4502 : enum reg_class rclass;
4503 :
4504 : public:
4505 0 : lra_autoinc_reload_context (machine_mode mode, enum reg_class new_rclass)
4506 0 : : mode (mode), rclass (new_rclass) {}
4507 :
4508 0 : rtx get_reload_reg () const override final
4509 : {
4510 0 : return lra_create_new_reg (mode, NULL_RTX, rclass, NULL, "INC/DEC result");
4511 : }
4512 : };
4513 :
4514 : /* Emit insns to reload VALUE into a new register. VALUE is an
4515 : auto-increment or auto-decrement RTX whose operand is a register or
4516 : memory location; so reloading involves incrementing that location.
4517 :
4518 : INC_AMOUNT is the number to increment or decrement by (always
4519 : positive and ignored for POST_MODIFY/PRE_MODIFY).
4520 :
4521 : Return a pseudo containing the result. */
4522 : static rtx
4523 0 : emit_inc (enum reg_class new_rclass, rtx value, poly_int64 inc_amount)
4524 : {
4525 0 : lra_autoinc_reload_context context (GET_MODE (value), new_rclass);
4526 0 : return context.emit_autoinc (value, inc_amount);
4527 : }
4528 :
4529 : /* Return true if the current move insn does not need processing as we
4530 : already know that it satisfies its constraints. */
4531 : static bool
4532 103563505 : simple_move_p (void)
4533 : {
4534 103563505 : rtx dest, src;
4535 103563505 : enum reg_class dclass, sclass;
4536 :
4537 103563505 : lra_assert (curr_insn_set != NULL_RTX);
4538 103563505 : dest = SET_DEST (curr_insn_set);
4539 103563505 : src = SET_SRC (curr_insn_set);
4540 :
4541 : /* If the instruction has multiple sets we need to process it even if it
4542 : is single_set. This can happen if one or more of the SETs are dead.
4543 : See PR73650. */
4544 103563505 : if (multiple_sets (curr_insn))
4545 : return false;
4546 :
4547 103360491 : return ((dclass = get_op_class (dest)) != NO_REGS
4548 21925013 : && (sclass = get_op_class (src)) != NO_REGS
4549 : /* The backend guarantees that register moves of cost 2
4550 : never need reloads. */
4551 92490230 : && targetm.register_move_cost (GET_MODE (src), sclass, dclass) == 2);
4552 : }
4553 :
4554 : /* Swap operands NOP and NOP + 1. */
4555 : static inline void
4556 21853956 : swap_operands (int nop)
4557 : {
4558 21853956 : std::swap (curr_operand_mode[nop], curr_operand_mode[nop + 1]);
4559 21853956 : std::swap (original_subreg_reg_mode[nop], original_subreg_reg_mode[nop + 1]);
4560 21853956 : std::swap (*curr_id->operand_loc[nop], *curr_id->operand_loc[nop + 1]);
4561 21853956 : std::swap (equiv_substition_p[nop], equiv_substition_p[nop + 1]);
4562 : /* Swap the duplicates too. */
4563 21853956 : lra_update_dup (curr_id, nop);
4564 21853956 : lra_update_dup (curr_id, nop + 1);
4565 21853956 : }
4566 :
4567 : /* Return TRUE if X is a (subreg of) reg and there are no hard regs of X class
4568 : which can contain value of MODE. */
4569 34 : static bool invalid_mode_reg_p (enum machine_mode mode, rtx x)
4570 : {
4571 34 : if (SUBREG_P (x))
4572 2 : x = SUBREG_REG (x);
4573 34 : if (! REG_P (x))
4574 : return false;
4575 34 : enum reg_class rclass = get_reg_class (REGNO (x));
4576 34 : return (!hard_reg_set_empty_p (reg_class_contents[rclass])
4577 34 : && hard_reg_set_subset_p
4578 34 : (reg_class_contents[rclass],
4579 34 : ira_prohibited_class_mode_regs[rclass][mode]));
4580 : }
4581 :
4582 : /* Return TRUE if regno is referenced in more than one non-debug insn. */
4583 : static bool
4584 2966177 : multiple_insn_refs_p (int regno)
4585 : {
4586 2966177 : unsigned int uid;
4587 2966177 : bitmap_iterator bi;
4588 2966177 : int nrefs = 0;
4589 7249868 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi)
4590 : {
4591 7244995 : if (!NONDEBUG_INSN_P (lra_insn_recog_data[uid]->insn))
4592 1317514 : continue;
4593 5927481 : if (nrefs == 1)
4594 : return true;
4595 2966177 : nrefs++;
4596 : }
4597 : return false;
4598 : }
4599 :
4600 : /* Mark insns starting with FIRST as postponed for processing their
4601 : constraints. See comments for lra_postponed_insns. */
4602 : static void
4603 93590 : postpone_insns (rtx_insn *first)
4604 : {
4605 105628 : for (auto insn = first; insn != NULL_RTX; insn = NEXT_INSN (insn))
4606 : {
4607 12038 : bitmap_set_bit (&lra_postponed_insns, INSN_UID (insn));
4608 12038 : if (lra_dump_file != NULL)
4609 : {
4610 7 : fprintf (lra_dump_file, " Postponing constraint processing: ");
4611 7 : dump_insn_slim (lra_dump_file, insn);
4612 : }
4613 : }
4614 93590 : }
4615 :
4616 : /* Test whether the n-th operand is a MEM where the address is the sum of a
4617 : section anchor and a constant and return true in case of reloading the
4618 : section anchor only results in a satisfiable operand w.r.t. its
4619 : corresponding constraint. Otherwise return false. */
4620 :
4621 : static bool
4622 72 : reload_section_anchor_p (int nop)
4623 : {
4624 72 : rtx op = *curr_id->operand_loc[nop];
4625 72 : if (!MEM_P (op))
4626 : return false;
4627 72 : rtx addr = XEXP (op, 0);
4628 :
4629 72 : if (GET_CODE (addr) != CONST
4630 13 : || GET_CODE (XEXP (addr, 0)) != PLUS
4631 0 : || GET_CODE (XEXP (XEXP (addr, 0), 0)) != SYMBOL_REF
4632 0 : || !SYMBOL_REF_ANCHOR_P (XEXP (XEXP (addr, 0), 0))
4633 0 : || !CONST_INT_P (XEXP (XEXP (addr, 0), 1))
4634 : /* Some offsets are valid in conjunction with a symbol and
4635 : invalid in conjunction with a register. Thus, pull out
4636 : the anchor only in case the offset is a valid anchor
4637 : offset. */
4638 0 : || INTVAL (XEXP (XEXP (addr, 0), 1)) < targetm.min_anchor_offset
4639 72 : || INTVAL (XEXP (XEXP (addr, 0), 1)) > targetm.max_anchor_offset)
4640 : return false;
4641 :
4642 : /* Now test whether a new address of the form REG+DISPLACEMENT is valid for
4643 : the selected alternative. In order to do so, utilize lra_pmode_pseudo
4644 : instead of an actual reload register. */
4645 :
4646 0 : rtx offset = XEXP (XEXP (addr, 0), 1);
4647 0 : rtx new_addr = gen_rtx_PLUS (Pmode, lra_pmode_pseudo, offset);
4648 0 : rtx new_op = shallow_copy_rtx (op);
4649 0 : XEXP (new_op, 0) = new_addr;
4650 :
4651 : /* Get operand constraints for given alternative. */
4652 0 : const char *p = (curr_static_id->operand_alternative
4653 0 : [goal_alt_number * curr_static_id->n_operands + nop]
4654 : .constraint);
4655 0 : char c;
4656 0 : for (;
4657 0 : (c = *p) && c != ',' && c != '#';
4658 0 : p += CONSTRAINT_LEN (c, p))
4659 : {
4660 0 : enum constraint_num cn = lookup_constraint (p);
4661 0 : if (constraint_satisfied_p (new_op, cn))
4662 : return true;
4663 : }
4664 :
4665 : return false;
4666 : }
4667 :
4668 : /* Main entry point of the constraint code: search the body of the
4669 : current insn to choose the best alternative. It is mimicking insn
4670 : alternative cost calculation model of former reload pass. That is
4671 : because machine descriptions were written to use this model. This
4672 : model can be changed in future. Make commutative operand exchange
4673 : if it is chosen.
4674 :
4675 : if CHECK_ONLY_P is false, do RTL changes to satisfy the
4676 : constraints. Return true if any change happened during function
4677 : call.
4678 :
4679 : If CHECK_ONLY_P is true then don't do any transformation. Just
4680 : check that the insn satisfies all constraints. If the insn does
4681 : not satisfy any constraint, return true. */
4682 : static bool
4683 109071703 : curr_insn_transform (bool check_only_p)
4684 : {
4685 109071703 : int i, j, k;
4686 109071703 : int n_operands;
4687 109071703 : int n_alternatives;
4688 109071703 : int n_outputs;
4689 109071703 : int commutative;
4690 109071703 : signed char goal_alt_matched[MAX_RECOG_OPERANDS][MAX_RECOG_OPERANDS];
4691 109071703 : signed char match_inputs[MAX_RECOG_OPERANDS + 1];
4692 109071703 : signed char outputs[MAX_RECOG_OPERANDS + 1];
4693 109071703 : rtx_insn *before, *after;
4694 109071703 : bool alt_p = false;
4695 : /* Flag that the insn has been changed through a transformation. */
4696 109071703 : bool change_p;
4697 109071703 : bool sec_mem_p;
4698 109071703 : bool use_sec_mem_p;
4699 109071703 : int max_regno_before;
4700 109071703 : int reused_alternative_num;
4701 :
4702 109071703 : curr_insn_set = single_set (curr_insn);
4703 109071703 : if (curr_insn_set != NULL_RTX && simple_move_p ())
4704 : {
4705 : /* We assume that the corresponding insn alternative has no
4706 : earlier clobbers. If it is not the case, don't define move
4707 : cost equal to 2 for the corresponding register classes. */
4708 16860821 : lra_set_used_insn_alternative (curr_insn, LRA_NON_CLOBBERED_ALT);
4709 16860821 : return false;
4710 : }
4711 :
4712 92210882 : no_input_reloads_p = no_output_reloads_p = false;
4713 92210882 : goal_alt_number = -1;
4714 92210882 : change_p = sec_mem_p = false;
4715 :
4716 : /* CALL_INSNs are not allowed to have any output reloads. */
4717 92210882 : if (CALL_P (curr_insn))
4718 6220326 : no_output_reloads_p = true;
4719 :
4720 92210882 : n_operands = curr_static_id->n_operands;
4721 92210882 : n_alternatives = curr_static_id->n_alternatives;
4722 :
4723 : /* Just return "no reloads" if insn has no operands with
4724 : constraints. */
4725 92210882 : if (n_operands == 0 || n_alternatives == 0)
4726 : return false;
4727 :
4728 81541641 : max_regno_before = max_reg_num ();
4729 :
4730 344973107 : for (i = 0; i < n_operands; i++)
4731 : {
4732 181889825 : goal_alt_matched[i][0] = -1;
4733 181889825 : goal_alt_matches[i] = -1;
4734 : }
4735 :
4736 81541641 : commutative = curr_static_id->commutative;
4737 :
4738 : /* Now see what we need for pseudos that didn't get hard regs or got
4739 : the wrong kind of hard reg. For this, we must consider all the
4740 : operands together against the register constraints. */
4741 :
4742 81541641 : best_losers = best_overall = INT_MAX;
4743 81541641 : best_reload_sum = 0;
4744 :
4745 81541641 : curr_swapped = false;
4746 81541641 : goal_alt_swapped = false;
4747 :
4748 81541641 : if (! check_only_p)
4749 : /* Make equivalence substitution and memory subreg elimination
4750 : before address processing because an address legitimacy can
4751 : depend on memory mode. */
4752 263363598 : for (i = 0; i < n_operands; i++)
4753 : {
4754 181840847 : rtx op, subst, old;
4755 181840847 : bool op_change_p = false;
4756 :
4757 181840847 : if (curr_static_id->operand[i].is_operator)
4758 1476595 : continue;
4759 :
4760 180364252 : old = op = *curr_id->operand_loc[i];
4761 180364252 : if (GET_CODE (old) == SUBREG)
4762 3824458 : old = SUBREG_REG (old);
4763 180364252 : subst = get_equiv_with_elimination (old, curr_insn);
4764 180364252 : original_subreg_reg_mode[i] = VOIDmode;
4765 180364252 : equiv_substition_p[i] = false;
4766 :
4767 180364252 : if (subst != old
4768 : /* We don't want to change an out operand by constant or invariant
4769 : which will require additional reloads, e.g. by putting a constant
4770 : into memory. */
4771 1635438 : && (curr_static_id->operand[i].type == OP_IN || MEM_P (subst)
4772 0 : || (GET_CODE (subst) == SUBREG && MEM_P (SUBREG_REG (subst)))))
4773 : {
4774 1635438 : equiv_substition_p[i] = true;
4775 1635438 : rtx new_subst = copy_rtx (subst);
4776 1635438 : if (lra_pointer_equiv_set_in (subst))
4777 846941 : lra_pointer_equiv_set_add (new_subst);
4778 1635438 : subst = new_subst;
4779 1635438 : lra_assert (REG_P (old));
4780 1635438 : if (GET_CODE (op) != SUBREG)
4781 1578916 : *curr_id->operand_loc[i] = subst;
4782 : else
4783 : {
4784 56522 : SUBREG_REG (op) = subst;
4785 56522 : if (GET_MODE (subst) == VOIDmode)
4786 93 : original_subreg_reg_mode[i] = GET_MODE (old);
4787 : }
4788 1635438 : if (lra_dump_file != NULL)
4789 : {
4790 3 : fprintf (lra_dump_file,
4791 : "Changing pseudo %d in operand %i of insn %u on equiv ",
4792 3 : REGNO (old), i, INSN_UID (curr_insn));
4793 3 : dump_value_slim (lra_dump_file, subst, 1);
4794 3 : fprintf (lra_dump_file, "\n");
4795 : }
4796 1635438 : op_change_p = change_p = true;
4797 : }
4798 180364252 : if (simplify_operand_subreg (i, GET_MODE (old)) || op_change_p)
4799 : {
4800 1635958 : change_p = true;
4801 1635958 : lra_update_dup (curr_id, i);
4802 : }
4803 : }
4804 :
4805 : /* We process equivalences before ignoring postponed insns on the current
4806 : constraint sub-pass but before any reload insn generation for the
4807 : postponed insn. */
4808 81522751 : if (! check_only_p
4809 81522751 : && bitmap_bit_p (&lra_postponed_insns, INSN_UID (curr_insn)))
4810 : return true;
4811 :
4812 : /* Reload address registers and displacements. We do it before
4813 : finding an alternative because of memory constraints. */
4814 81535482 : before = after = NULL;
4815 263412989 : for (i = 0; i < n_operands; i++)
4816 181877507 : if (! curr_static_id->operand[i].is_operator
4817 181877507 : && process_address (i, check_only_p, &before, &after))
4818 : {
4819 496288 : if (check_only_p)
4820 : return true;
4821 496288 : change_p = true;
4822 496288 : lra_update_dup (curr_id, i);
4823 : }
4824 :
4825 81535482 : if (change_p)
4826 : /* If we've changed the instruction then any alternative that
4827 : we chose previously may no longer be valid. */
4828 2085638 : lra_set_used_insn_alternative (curr_insn, LRA_UNKNOWN_ALT);
4829 :
4830 81516592 : if (! check_only_p && curr_insn_set != NULL_RTX
4831 159099429 : && check_and_process_move (&change_p, &sec_mem_p))
4832 0 : return change_p;
4833 :
4834 81535482 : try_swapped:
4835 :
4836 92136676 : reused_alternative_num = check_only_p ? LRA_UNKNOWN_ALT : curr_id->used_insn_alternative;
4837 92136676 : if (lra_dump_file != NULL && reused_alternative_num >= 0)
4838 0 : fprintf (lra_dump_file, "Reusing alternative %d for insn #%u\n",
4839 0 : reused_alternative_num, INSN_UID (curr_insn));
4840 :
4841 92136676 : if (process_alt_operands (reused_alternative_num))
4842 83415567 : alt_p = true;
4843 :
4844 92136676 : if (check_only_p)
4845 18890 : return ! alt_p || best_losers != 0;
4846 :
4847 : /* If insn is commutative (it's safe to exchange a certain pair of
4848 : operands) then we need to try each alternative twice, the second
4849 : time matching those two operands as if we had exchanged them. To
4850 : do this, really exchange them in operands.
4851 :
4852 : If we have just tried the alternatives the second time, return
4853 : operands to normal and drop through. */
4854 :
4855 92117786 : if (reused_alternative_num < 0 && commutative >= 0)
4856 : {
4857 21202388 : curr_swapped = !curr_swapped;
4858 21202388 : if (curr_swapped)
4859 : {
4860 10601194 : swap_operands (commutative);
4861 10601194 : goto try_swapped;
4862 : }
4863 : else
4864 10601194 : swap_operands (commutative);
4865 : }
4866 :
4867 81516592 : if (! alt_p && ! sec_mem_p)
4868 : {
4869 : /* No alternative works with reloads?? */
4870 6 : if (INSN_CODE (curr_insn) >= 0)
4871 0 : fatal_insn ("unable to generate reloads for:", curr_insn);
4872 6 : error_for_asm (curr_insn,
4873 : "inconsistent operand constraints in an %<asm%>");
4874 6 : lra_asm_error_p = true;
4875 6 : if (! JUMP_P (curr_insn))
4876 : {
4877 : /* Avoid further trouble with this insn. Don't generate use
4878 : pattern here as we could use the insn SP offset. */
4879 6 : lra_set_insn_deleted (curr_insn);
4880 : }
4881 : else
4882 : {
4883 0 : lra_invalidate_insn_data (curr_insn);
4884 0 : ira_nullify_asm_goto (curr_insn);
4885 0 : lra_update_insn_regno_info (curr_insn);
4886 : }
4887 : return true;
4888 : }
4889 :
4890 : /* If the best alternative is with operands 1 and 2 swapped, swap
4891 : them. Update the operand numbers of any reloads already
4892 : pushed. */
4893 :
4894 81516586 : if (goal_alt_swapped)
4895 : {
4896 646840 : if (lra_dump_file != NULL)
4897 21 : fprintf (lra_dump_file, " Commutative operand exchange in insn %u\n",
4898 21 : INSN_UID (curr_insn));
4899 :
4900 : /* Swap the duplicates too. */
4901 646840 : swap_operands (commutative);
4902 646840 : change_p = true;
4903 : }
4904 :
4905 : /* Some targets' TARGET_SECONDARY_MEMORY_NEEDED (e.g. x86) are defined
4906 : too conservatively. So we use the secondary memory only if there
4907 : is no any alternative without reloads. */
4908 81516586 : use_sec_mem_p = false;
4909 81516586 : if (! alt_p)
4910 : use_sec_mem_p = true;
4911 81516586 : else if (sec_mem_p)
4912 : {
4913 14510 : for (i = 0; i < n_operands; i++)
4914 14337 : if (! goal_alt_win[i] && ! goal_alt_match_win[i])
4915 : break;
4916 12952 : use_sec_mem_p = i < n_operands;
4917 : }
4918 :
4919 12952 : if (use_sec_mem_p)
4920 : {
4921 12779 : int in = -1, out = -1;
4922 12779 : rtx new_reg, src, dest, rld;
4923 12779 : machine_mode sec_mode, rld_mode;
4924 :
4925 12779 : lra_assert (curr_insn_set != NULL_RTX && sec_mem_p);
4926 12779 : dest = SET_DEST (curr_insn_set);
4927 12779 : src = SET_SRC (curr_insn_set);
4928 38337 : for (i = 0; i < n_operands; i++)
4929 25558 : if (*curr_id->operand_loc[i] == dest)
4930 : out = i;
4931 12779 : else if (*curr_id->operand_loc[i] == src)
4932 12779 : in = i;
4933 12779 : for (i = 0; i < curr_static_id->n_dups; i++)
4934 0 : if (out < 0 && *curr_id->dup_loc[i] == dest)
4935 0 : out = curr_static_id->dup_num[i];
4936 0 : else if (in < 0 && *curr_id->dup_loc[i] == src)
4937 0 : in = curr_static_id->dup_num[i];
4938 12779 : lra_assert (out >= 0 && in >= 0
4939 : && curr_static_id->operand[out].type == OP_OUT
4940 : && curr_static_id->operand[in].type == OP_IN);
4941 12779 : rld = partial_subreg_p (GET_MODE (src), GET_MODE (dest)) ? src : dest;
4942 12779 : rld_mode = GET_MODE (rld);
4943 12779 : sec_mode = targetm.secondary_memory_needed_mode (rld_mode);
4944 12779 : if (rld_mode != sec_mode
4945 12779 : && (invalid_mode_reg_p (sec_mode, dest)
4946 17 : || invalid_mode_reg_p (sec_mode, src)))
4947 : sec_mode = rld_mode;
4948 12779 : new_reg = lra_create_new_reg (sec_mode, NULL_RTX, NO_REGS, NULL,
4949 : "secondary");
4950 : /* If the mode is changed, it should be wider. */
4951 12779 : lra_assert (!partial_subreg_p (sec_mode, rld_mode));
4952 12779 : if (sec_mode != rld_mode)
4953 : {
4954 : /* If the target says specifically to use another mode for
4955 : secondary memory moves we cannot reuse the original
4956 : insn. */
4957 17 : after = emit_spill_move (false, new_reg, dest);
4958 17 : lra_process_new_insns (curr_insn, NULL, after,
4959 : "Inserting the sec. move");
4960 : /* We may have non null BEFORE here (e.g. after address
4961 : processing. */
4962 17 : push_to_sequence (before);
4963 17 : before = emit_spill_move (true, new_reg, src);
4964 17 : emit_insn (before);
4965 17 : before = end_sequence ();
4966 17 : lra_process_new_insns (curr_insn, before, NULL, "Changing on");
4967 17 : lra_set_insn_deleted (curr_insn);
4968 : }
4969 12762 : else if (dest == rld)
4970 : {
4971 12762 : *curr_id->operand_loc[out] = new_reg;
4972 12762 : lra_update_dup (curr_id, out);
4973 12762 : after = emit_spill_move (false, new_reg, dest);
4974 12762 : lra_process_new_insns (curr_insn, NULL, after,
4975 : "Inserting the sec. move");
4976 : }
4977 : else
4978 : {
4979 0 : *curr_id->operand_loc[in] = new_reg;
4980 0 : lra_update_dup (curr_id, in);
4981 : /* See comments above. */
4982 0 : push_to_sequence (before);
4983 0 : before = emit_spill_move (true, new_reg, src);
4984 0 : emit_insn (before);
4985 0 : before = end_sequence ();
4986 0 : lra_process_new_insns (curr_insn, before, NULL,
4987 : "Inserting the sec. move");
4988 : }
4989 12779 : lra_update_insn_regno_info (curr_insn);
4990 12779 : return true;
4991 : }
4992 :
4993 81503807 : lra_assert (goal_alt_number >= 0);
4994 162915442 : lra_set_used_insn_alternative (curr_insn, goal_reuse_alt_p
4995 : ? goal_alt_number : LRA_UNKNOWN_ALT);
4996 :
4997 81503807 : if (lra_dump_file != NULL)
4998 : {
4999 1185 : const char *p;
5000 :
5001 1185 : fprintf (lra_dump_file, " Choosing alt %d in insn %u:",
5002 1185 : goal_alt_number, INSN_UID (curr_insn));
5003 1185 : print_curr_insn_alt (goal_alt_number);
5004 1185 : if (INSN_CODE (curr_insn) >= 0
5005 1185 : && (p = get_insn_name (INSN_CODE (curr_insn))) != NULL)
5006 1178 : fprintf (lra_dump_file, " {%s}", p);
5007 1185 : if (maybe_ne (curr_id->sp_offset, 0))
5008 : {
5009 0 : fprintf (lra_dump_file, " (sp_off=");
5010 0 : print_dec (curr_id->sp_offset, lra_dump_file);
5011 0 : fprintf (lra_dump_file, ")");
5012 : }
5013 1185 : fprintf (lra_dump_file, "\n");
5014 : }
5015 :
5016 : /* Right now, for any pair of operands I and J that are required to
5017 : match, with J < I, goal_alt_matches[I] is J. Add I to
5018 : goal_alt_matched[J]. */
5019 :
5020 263306764 : for (i = 0; i < n_operands; i++)
5021 181802957 : if ((j = goal_alt_matches[i]) >= 0)
5022 : {
5023 10715645 : for (k = 0; goal_alt_matched[j][k] >= 0; k++)
5024 : ;
5025 : /* We allow matching one output operand and several input
5026 : operands. */
5027 10715644 : lra_assert (k == 0
5028 : || (curr_static_id->operand[j].type == OP_OUT
5029 : && curr_static_id->operand[i].type == OP_IN
5030 : && (curr_static_id->operand
5031 : [goal_alt_matched[j][0]].type == OP_IN)));
5032 10715644 : goal_alt_matched[j][k] = i;
5033 10715644 : goal_alt_matched[j][k + 1] = -1;
5034 : }
5035 :
5036 263306764 : for (i = 0; i < n_operands; i++)
5037 181802957 : goal_alt_win[i] |= goal_alt_match_win[i];
5038 :
5039 : /* Any constants that aren't allowed and can't be reloaded into
5040 : registers are here changed into memory references. */
5041 263306764 : for (i = 0; i < n_operands; i++)
5042 181802957 : if (goal_alt_win[i])
5043 : {
5044 175432548 : int regno;
5045 175432548 : enum reg_class new_class;
5046 175432548 : rtx reg = *curr_id->operand_loc[i];
5047 :
5048 175432548 : if (GET_CODE (reg) == SUBREG)
5049 3492586 : reg = SUBREG_REG (reg);
5050 :
5051 175432548 : if (REG_P (reg) && (regno = REGNO (reg)) >= FIRST_PSEUDO_REGISTER)
5052 : {
5053 79471081 : bool ok_p = in_class_p (reg, goal_alt[i], &new_class, true);
5054 :
5055 79471081 : if (new_class != NO_REGS && get_reg_class (regno) != new_class)
5056 : {
5057 3648545 : lra_assert (ok_p);
5058 3648545 : lra_change_class (regno, new_class, " Change to", true);
5059 : }
5060 : }
5061 : }
5062 : else
5063 : {
5064 6370409 : const char *constraint;
5065 6370409 : char c;
5066 6370409 : rtx op = *curr_id->operand_loc[i];
5067 6370409 : rtx subreg = NULL_RTX;
5068 6370409 : machine_mode op_mode = curr_operand_mode[i], mode = op_mode;
5069 :
5070 6370409 : if (GET_CODE (op) == SUBREG)
5071 : {
5072 265576 : subreg = op;
5073 265576 : op = SUBREG_REG (op);
5074 265576 : mode = GET_MODE (op);
5075 : }
5076 :
5077 6589685 : if (CONST_POOL_OK_P (mode, op)
5078 6589685 : && ((targetm.preferred_reload_class
5079 219276 : (op, (enum reg_class) goal_alt[i]) == NO_REGS)
5080 76049 : || no_input_reloads_p))
5081 : {
5082 143227 : rtx tem = force_const_mem (mode, op);
5083 :
5084 143227 : change_p = true;
5085 143227 : if (subreg != NULL_RTX)
5086 0 : tem = gen_rtx_SUBREG (op_mode, tem, SUBREG_BYTE (subreg));
5087 :
5088 143227 : *curr_id->operand_loc[i] = tem;
5089 143227 : lra_update_dup (curr_id, i);
5090 143227 : process_address (i, false, &before, &after);
5091 :
5092 : /* If the alternative accepts constant pool refs directly
5093 : there will be no reload needed at all. */
5094 143227 : if (subreg != NULL_RTX)
5095 0 : continue;
5096 : /* Skip alternatives before the one requested. */
5097 143227 : constraint = (curr_static_id->operand_alternative
5098 143227 : [goal_alt_number * n_operands + i].constraint);
5099 143227 : for (;
5100 244867 : (c = *constraint) && c != ',' && c != '#';
5101 101640 : constraint += CONSTRAINT_LEN (c, constraint))
5102 : {
5103 198673 : enum constraint_num cn = lookup_constraint (constraint);
5104 198673 : if ((insn_extra_memory_constraint (cn)
5105 101894 : || insn_extra_special_memory_constraint (cn)
5106 : || insn_extra_relaxed_memory_constraint (cn))
5107 198927 : && satisfies_memory_constraint_p (tem, cn))
5108 : break;
5109 : }
5110 143227 : if (c == '\0' || c == ',' || c == '#')
5111 46194 : continue;
5112 :
5113 97033 : goal_alt_win[i] = true;
5114 : }
5115 : }
5116 :
5117 : n_outputs = 0;
5118 263306764 : for (i = 0; i < n_operands; i++)
5119 181802957 : if (curr_static_id->operand[i].type == OP_OUT)
5120 70743425 : outputs[n_outputs++] = i;
5121 81503807 : outputs[n_outputs] = -1;
5122 263306764 : for (i = 0; i < n_operands; i++)
5123 : {
5124 181802957 : int regno;
5125 181802957 : bool optional_p = false;
5126 181802957 : rtx old, new_reg;
5127 181802957 : rtx op = *curr_id->operand_loc[i];
5128 :
5129 181802957 : if (goal_alt_win[i])
5130 : {
5131 175529581 : if (goal_alt[i] == NO_REGS
5132 47902133 : && REG_P (op)
5133 5539665 : && (regno = REGNO (op)) >= FIRST_PSEUDO_REGISTER
5134 : /* We assigned a hard register to the pseudo in the past but now
5135 : decided to spill it for the insn. If the pseudo is used only
5136 : in this insn, it is better to spill it here as we free hard
5137 : registers for other pseudos referenced in the insn. The most
5138 : common case of this is a scratch register which will be
5139 : transformed to scratch back at the end of LRA. */
5140 178495758 : && !multiple_insn_refs_p (regno))
5141 : {
5142 9746 : if (lra_get_allocno_class (regno) != NO_REGS)
5143 4620 : lra_change_class (regno, NO_REGS, " Change to", true);
5144 4873 : reg_renumber[regno] = -1;
5145 : }
5146 : /* We can do an optional reload. If the pseudo got a hard
5147 : reg, we might improve the code through inheritance. If
5148 : it does not get a hard register we coalesce memory/memory
5149 : moves later. Ignore move insns to avoid cycling. */
5150 175529581 : if (! lra_simple_p
5151 174983963 : && lra_undo_inheritance_iter < LRA_MAX_INHERITANCE_PASSES
5152 162331855 : && goal_alt[i] != NO_REGS && REG_P (op)
5153 80441939 : && (regno = REGNO (op)) >= FIRST_PSEUDO_REGISTER
5154 67265697 : && regno < new_regno_start
5155 62372263 : && ! ira_former_scratch_p (regno)
5156 62313380 : && reg_renumber[regno] < 0
5157 : /* Check that the optional reload pseudo will be able to
5158 : hold given mode value. */
5159 3904401 : && ! (prohibited_class_reg_set_mode_p
5160 3904401 : (goal_alt[i], reg_class_contents[goal_alt[i]],
5161 3904401 : PSEUDO_REGNO_MODE (regno)))
5162 179433972 : && (curr_insn_set == NULL_RTX
5163 3897859 : || !((REG_P (SET_SRC (curr_insn_set))
5164 : || MEM_P (SET_SRC (curr_insn_set))
5165 : || GET_CODE (SET_SRC (curr_insn_set)) == SUBREG)
5166 3264991 : && (REG_P (SET_DEST (curr_insn_set))
5167 : || MEM_P (SET_DEST (curr_insn_set))
5168 : || GET_CODE (SET_DEST (curr_insn_set)) == SUBREG))))
5169 : optional_p = true;
5170 174890149 : else if (goal_alt_matched[i][0] != -1
5171 8952275 : && curr_static_id->operand[i].type == OP_OUT
5172 8950610 : && (curr_static_id->operand_alternative
5173 8950610 : [goal_alt_number * n_operands + i].earlyclobber)
5174 20825 : && REG_P (op))
5175 : {
5176 25571 : for (j = 0; goal_alt_matched[i][j] != -1; j++)
5177 : {
5178 20772 : rtx op2 = *curr_id->operand_loc[goal_alt_matched[i][j]];
5179 :
5180 20772 : if (REG_P (op2) && REGNO (op) != REGNO (op2))
5181 : break;
5182 : }
5183 20772 : if (goal_alt_matched[i][j] != -1)
5184 : {
5185 : /* Generate reloads for different output and matched
5186 : input registers. This is the easiest way to avoid
5187 : creation of non-existing register conflicts in
5188 : lra-lives.cc. */
5189 15973 : match_reload (i, goal_alt_matched[i], outputs, goal_alt[i],
5190 : &goal_alt_exclude_start_hard_regs[i], &before,
5191 : &after, true);
5192 : }
5193 176564264 : continue;
5194 20772 : }
5195 : else
5196 : {
5197 174869377 : enum reg_class rclass, common_class;
5198 :
5199 91494811 : if (REG_P (op) && goal_alt[i] != NO_REGS
5200 85955146 : && (regno = REGNO (op)) >= new_regno_start
5201 4902218 : && (rclass = get_reg_class (regno)) == ALL_REGS
5202 0 : && ((common_class = ira_reg_class_subset[rclass][goal_alt[i]])
5203 : != NO_REGS)
5204 0 : && common_class != ALL_REGS
5205 174869377 : && enough_allocatable_hard_regs_p (common_class,
5206 0 : GET_MODE (op)))
5207 : /* Refine reload pseudo class from chosen alternative
5208 : constraint. */
5209 0 : lra_change_class (regno, common_class, " Change to", true);
5210 174869377 : continue;
5211 174869377 : }
5212 : }
5213 :
5214 : /* Operands that match previous ones have already been handled. */
5215 6912808 : if (goal_alt_matches[i] >= 0)
5216 1674115 : continue;
5217 :
5218 : /* We should not have an operand with a non-offsettable address
5219 : appearing where an offsettable address will do. It also may
5220 : be a case when the address should be special in other words
5221 : not a general one (e.g. it needs no index reg). */
5222 5238693 : if (goal_alt_matched[i][0] == -1 && goal_alt_offmemok[i] && MEM_P (op))
5223 : {
5224 72 : enum reg_class rclass;
5225 72 : rtx *loc = &XEXP (op, 0);
5226 72 : enum rtx_code code = GET_CODE (*loc);
5227 :
5228 72 : push_to_sequence (before);
5229 72 : rclass = base_reg_class (GET_MODE (op), MEM_ADDR_SPACE (op),
5230 : MEM, SCRATCH, op, curr_insn);
5231 72 : if (GET_RTX_CLASS (code) == RTX_AUTOINC)
5232 0 : new_reg = emit_inc (rclass, *loc,
5233 : /* This value does not matter for MODIFY. */
5234 0 : GET_MODE_SIZE (GET_MODE (op)));
5235 : /* Try to pull out section anchors. For example, instead of
5236 : reloading an "entire" address like .LANCHOR42+offset only reload
5237 : .LANCHOR42 and use the new reload register as the base register.
5238 : This allows following optimizations to share section anchors and
5239 : remove redundant loads. */
5240 72 : else if (reload_section_anchor_p (i))
5241 : {
5242 0 : rtx anchor = XEXP (XEXP (*loc, 0), 0);
5243 0 : rtx offset = XEXP (XEXP (*loc, 0), 1);
5244 :
5245 0 : if (get_reload_reg (OP_IN, Pmode, anchor, rclass, NULL, false,
5246 : false, "offsetable address", &new_reg))
5247 0 : lra_emit_move (new_reg, anchor);
5248 :
5249 0 : rtx new_addr = gen_rtx_PLUS (Pmode, new_reg, offset);
5250 0 : rtx new_op = shallow_copy_rtx (op);
5251 0 : XEXP (new_op, 0) = new_addr;
5252 :
5253 0 : new_reg = new_op;
5254 0 : loc = curr_id->operand_loc[i];
5255 : }
5256 86 : else if (get_reload_reg (OP_IN, Pmode, *loc, rclass,
5257 : NULL, false, false,
5258 : "offsetable address", &new_reg))
5259 : {
5260 72 : rtx addr = *loc;
5261 72 : enum rtx_code code = GET_CODE (addr);
5262 72 : bool align_p = false;
5263 :
5264 72 : if (code == AND && CONST_INT_P (XEXP (addr, 1)))
5265 : {
5266 : /* (and ... (const_int -X)) is used to align to X bytes. */
5267 0 : align_p = true;
5268 0 : addr = XEXP (*loc, 0);
5269 : }
5270 : else
5271 72 : addr = canonicalize_reload_addr (addr);
5272 :
5273 72 : lra_emit_move (new_reg, addr);
5274 72 : if (align_p)
5275 0 : emit_move_insn (new_reg, gen_rtx_AND (GET_MODE (new_reg), new_reg, XEXP (*loc, 1)));
5276 : }
5277 72 : before = end_sequence ();
5278 72 : *loc = new_reg;
5279 72 : lra_update_dup (curr_id, i);
5280 72 : }
5281 5238621 : else if (goal_alt_matched[i][0] == -1)
5282 : {
5283 3475253 : machine_mode mode;
5284 3475253 : rtx reg, *loc;
5285 3475253 : int hard_regno;
5286 3475253 : enum op_type type = curr_static_id->operand[i].type;
5287 :
5288 3475253 : loc = curr_id->operand_loc[i];
5289 3475253 : mode = curr_operand_mode[i];
5290 3475253 : if (GET_CODE (*loc) == SUBREG)
5291 : {
5292 93004 : reg = SUBREG_REG (*loc);
5293 93004 : poly_int64 byte = SUBREG_BYTE (*loc);
5294 93004 : if (REG_P (reg)
5295 : /* Strict_low_part requires reloading the register and not
5296 : just the subreg. Likewise for a strict subreg no wider
5297 : than a word for WORD_REGISTER_OPERATIONS targets. */
5298 93004 : && (curr_static_id->operand[i].strict_low
5299 92936 : || (!paradoxical_subreg_p (mode, GET_MODE (reg))
5300 90637 : && (hard_regno
5301 90637 : = get_try_hard_regno (REGNO (reg))) >= 0
5302 89169 : && (simplify_subreg_regno
5303 182173 : (hard_regno,
5304 89169 : GET_MODE (reg), byte, mode) < 0)
5305 0 : && (goal_alt[i] == NO_REGS
5306 0 : || (simplify_subreg_regno
5307 68 : (ira_class_hard_regs[goal_alt[i]][0],
5308 0 : GET_MODE (reg), byte, mode) >= 0)))
5309 92936 : || (partial_subreg_p (mode, GET_MODE (reg))
5310 92936 : && known_le (GET_MODE_SIZE (GET_MODE (reg)),
5311 : UNITS_PER_WORD)
5312 : && WORD_REGISTER_OPERATIONS))
5313 : /* Avoid the situation when there are no available hard regs
5314 : for the pseudo mode but there are ones for the subreg
5315 : mode: */
5316 93072 : && !(goal_alt[i] != NO_REGS
5317 68 : && REGNO (reg) >= FIRST_PSEUDO_REGISTER
5318 68 : && (prohibited_class_reg_set_mode_p
5319 68 : (goal_alt[i], reg_class_contents[goal_alt[i]],
5320 68 : GET_MODE (reg)))
5321 : && !(prohibited_class_reg_set_mode_p
5322 0 : (goal_alt[i], reg_class_contents[goal_alt[i]],
5323 : mode))))
5324 : {
5325 : /* An OP_INOUT is required when reloading a subreg of a
5326 : mode wider than a word to ensure that data beyond the
5327 : word being reloaded is preserved. Also automatically
5328 : ensure that strict_low_part reloads are made into
5329 : OP_INOUT which should already be true from the backend
5330 : constraints. */
5331 68 : if (type == OP_OUT
5332 68 : && (curr_static_id->operand[i].strict_low
5333 0 : || read_modify_subreg_p (*loc)))
5334 : type = OP_INOUT;
5335 68 : loc = &SUBREG_REG (*loc);
5336 68 : mode = GET_MODE (*loc);
5337 : }
5338 : }
5339 3475253 : old = *loc;
5340 3475253 : if (get_reload_reg (type, mode, old, goal_alt[i],
5341 : &goal_alt_exclude_start_hard_regs[i],
5342 3475253 : loc != curr_id->operand_loc[i],
5343 3475253 : curr_static_id->operand_alternative
5344 3475253 : [goal_alt_number * n_operands + i].earlyclobber,
5345 : "", &new_reg)
5346 3475253 : && type != OP_OUT)
5347 : {
5348 2489025 : push_to_sequence (before);
5349 2489025 : lra_emit_move (new_reg, old);
5350 2489025 : before = end_sequence ();
5351 : }
5352 3475253 : *loc = new_reg;
5353 3475253 : if (type != OP_IN
5354 984929 : && find_reg_note (curr_insn, REG_UNUSED, old) == NULL_RTX
5355 : /* OLD can be an equivalent constant here. */
5356 959596 : && !CONSTANT_P (old)
5357 : /* No need to write back anything for a scratch. */
5358 959596 : && GET_CODE (old) != SCRATCH
5359 4434849 : && (!REG_P(old) || !ira_former_scratch_p (REGNO (old))))
5360 : {
5361 959596 : start_sequence ();
5362 959596 : lra_emit_move (type == OP_INOUT ? copy_rtx (old) : old, new_reg);
5363 959596 : emit_insn (after);
5364 959596 : after = end_sequence ();
5365 959596 : *loc = new_reg;
5366 : }
5367 3475253 : for (j = 0; j < goal_alt_dont_inherit_ops_num; j++)
5368 655 : if (goal_alt_dont_inherit_ops[j] == i)
5369 : {
5370 655 : lra_set_regno_unique_value (REGNO (new_reg));
5371 655 : break;
5372 : }
5373 3475253 : lra_update_dup (curr_id, i);
5374 : }
5375 1763368 : else if (curr_static_id->operand[i].type == OP_IN
5376 1763368 : && (curr_static_id->operand[goal_alt_matched[i][0]].type
5377 : == OP_OUT
5378 0 : || (curr_static_id->operand[goal_alt_matched[i][0]].type
5379 : == OP_INOUT
5380 0 : && (operands_match_p
5381 0 : (*curr_id->operand_loc[i],
5382 0 : *curr_id->operand_loc[goal_alt_matched[i][0]],
5383 : -1)))))
5384 : {
5385 : /* generate reloads for input and matched outputs. */
5386 16527 : match_inputs[0] = i;
5387 16527 : match_inputs[1] = -1;
5388 16527 : match_reload (goal_alt_matched[i][0], match_inputs, outputs,
5389 : goal_alt[i], &goal_alt_exclude_start_hard_regs[i],
5390 : &before, &after,
5391 16527 : curr_static_id->operand_alternative
5392 16527 : [goal_alt_number * n_operands + goal_alt_matched[i][0]]
5393 16527 : .earlyclobber);
5394 : }
5395 1746841 : else if ((curr_static_id->operand[i].type == OP_OUT
5396 1 : || (curr_static_id->operand[i].type == OP_INOUT
5397 1 : && (operands_match_p
5398 1 : (*curr_id->operand_loc[i],
5399 1 : *curr_id->operand_loc[goal_alt_matched[i][0]],
5400 : -1))))
5401 1746842 : && (curr_static_id->operand[goal_alt_matched[i][0]].type
5402 : == OP_IN))
5403 : /* Generate reloads for output and matched inputs. */
5404 1746841 : match_reload (i, goal_alt_matched[i], outputs, goal_alt[i],
5405 : &goal_alt_exclude_start_hard_regs[i], &before, &after,
5406 1746841 : curr_static_id->operand_alternative
5407 1746841 : [goal_alt_number * n_operands + i].earlyclobber);
5408 0 : else if (curr_static_id->operand[i].type == OP_IN
5409 0 : && (curr_static_id->operand[goal_alt_matched[i][0]].type
5410 : == OP_IN))
5411 : {
5412 : /* Generate reloads for matched inputs. */
5413 0 : match_inputs[0] = i;
5414 0 : for (j = 0; (k = goal_alt_matched[i][j]) >= 0; j++)
5415 0 : match_inputs[j + 1] = k;
5416 0 : match_inputs[j + 1] = -1;
5417 0 : match_reload (-1, match_inputs, outputs, goal_alt[i],
5418 : &goal_alt_exclude_start_hard_regs[i],
5419 : &before, &after, false);
5420 : }
5421 : else
5422 : /* We must generate code in any case when function
5423 : process_alt_operands decides that it is possible. */
5424 0 : gcc_unreachable ();
5425 :
5426 5238693 : if (optional_p)
5427 : {
5428 639432 : rtx reg = op;
5429 :
5430 639432 : lra_assert (REG_P (reg));
5431 639432 : regno = REGNO (reg);
5432 639432 : op = *curr_id->operand_loc[i]; /* Substitution. */
5433 639432 : if (GET_CODE (op) == SUBREG)
5434 0 : op = SUBREG_REG (op);
5435 639432 : gcc_assert (REG_P (op) && (int) REGNO (op) >= new_regno_start);
5436 639432 : bitmap_set_bit (&lra_optional_reload_pseudos, REGNO (op));
5437 639432 : lra_reg_info[REGNO (op)].restore_rtx = reg;
5438 639432 : if (lra_dump_file != NULL)
5439 2 : fprintf (lra_dump_file,
5440 : " Making reload reg %d for reg %d optional\n",
5441 : REGNO (op), regno);
5442 : }
5443 : }
5444 :
5445 : /* After the dust has settled, collect and attach dependent filters. */
5446 81503807 : process_dependent_filters ();
5447 :
5448 77005266 : if (before != NULL_RTX || after != NULL_RTX
5449 157691722 : || max_regno_before != max_reg_num ())
5450 5341567 : change_p = true;
5451 81503807 : if (change_p)
5452 : {
5453 6356375 : lra_update_operator_dups (curr_id);
5454 : /* Something changes -- process the insn. */
5455 6356375 : lra_update_insn_regno_info (curr_insn);
5456 6356375 : if (asm_noperands (PATTERN (curr_insn)) >= 0
5457 6356375 : && ++curr_id->asm_reloads_num >= FIRST_PSEUDO_REGISTER)
5458 : /* Most probably there are no enough registers to satisfy asm insn: */
5459 : {
5460 5 : lra_asm_insn_error (curr_insn);
5461 5 : return change_p;
5462 : }
5463 : }
5464 81503802 : if (goal_alt_out_sp_reload_p)
5465 : {
5466 : /* We have an output stack pointer reload -- update sp offset: */
5467 0 : rtx set;
5468 0 : bool done_p = false;
5469 0 : poly_int64 sp_offset = curr_id->sp_offset;
5470 0 : for (rtx_insn *insn = after; insn != NULL_RTX; insn = NEXT_INSN (insn))
5471 0 : if ((set = single_set (insn)) != NULL_RTX
5472 0 : && SET_DEST (set) == stack_pointer_rtx)
5473 : {
5474 0 : lra_assert (!done_p);
5475 0 : done_p = true;
5476 0 : curr_id->sp_offset = 0;
5477 0 : lra_insn_recog_data_t id = lra_get_insn_recog_data (insn);
5478 0 : id->sp_offset = sp_offset;
5479 0 : if (lra_dump_file != NULL)
5480 0 : fprintf (lra_dump_file,
5481 : " Moving sp offset from insn %u to %u\n",
5482 0 : INSN_UID (curr_insn), INSN_UID (insn));
5483 : }
5484 0 : lra_assert (done_p);
5485 : }
5486 81503802 : int const_regno = -1;
5487 81503802 : rtx set;
5488 81503802 : rtx_insn *prev, *const_insn = NULL;
5489 4498540 : if (before != NULL_RTX && (prev = PREV_INSN (curr_insn)) != NULL_RTX
5490 86002342 : && (set = single_set (prev)) != NULL_RTX && CONSTANT_P (SET_SRC (set)))
5491 : {
5492 341690 : rtx reg = SET_DEST (set);
5493 341690 : if (GET_CODE (reg) == SUBREG)
5494 10531 : reg = SUBREG_REG (reg);
5495 : /* Consider only reload insns as we don't want to change the order
5496 : created by previous optimizations. */
5497 248645 : if (REG_P (reg) && (int) REGNO (reg) >= lra_new_regno_start
5498 343333 : && bitmap_bit_p (&lra_reg_info[REGNO (reg)].insn_bitmap,
5499 1643 : INSN_UID (curr_insn)))
5500 : {
5501 1140 : const_regno = REGNO (reg);
5502 1140 : const_insn = prev;
5503 : }
5504 : }
5505 81503802 : if (asm_noperands (PATTERN (curr_insn)) >= 0)
5506 : {
5507 : /* Asm can have a lot of operands. To guarantee their assignment,
5508 : postpone processing the reload insns until the reload pseudos are
5509 : assigned. */
5510 46795 : postpone_insns (before);
5511 46795 : postpone_insns (after);
5512 : }
5513 81503802 : lra_process_new_insns (curr_insn, before, after,
5514 : "Inserting insn reload", true);
5515 81503802 : if (const_regno >= 0) {
5516 2280 : bool move_p = true;
5517 2280 : for (rtx_insn *insn = before; insn != curr_insn; insn = NEXT_INSN (insn))
5518 1140 : if (bitmap_bit_p (&lra_reg_info[const_regno].insn_bitmap,
5519 1140 : INSN_UID (insn)))
5520 : {
5521 : move_p = false;
5522 : break;
5523 : }
5524 1140 : if (move_p)
5525 : {
5526 1140 : reorder_insns_nobb (const_insn, const_insn, PREV_INSN (curr_insn));
5527 1140 : if (lra_dump_file != NULL)
5528 : {
5529 0 : dump_insn_slim (lra_dump_file, const_insn);
5530 0 : fprintf (lra_dump_file,
5531 : " to decrease reg pressure, it is moved before:\n");
5532 0 : dump_insn_slim (lra_dump_file, curr_insn);
5533 : }
5534 : }
5535 : }
5536 : return change_p;
5537 : }
5538 :
5539 : /* Return true if INSN satisfies all constraints. In other words, no
5540 : reload insns are needed. */
5541 : bool
5542 3448 : lra_constrain_insn (rtx_insn *insn)
5543 : {
5544 3448 : int saved_new_regno_start = new_regno_start;
5545 3448 : int saved_new_insn_uid_start = new_insn_uid_start;
5546 3448 : bool change_p;
5547 :
5548 3448 : curr_insn = insn;
5549 3448 : curr_id = lra_get_insn_recog_data (curr_insn);
5550 3448 : curr_static_id = curr_id->insn_static_data;
5551 3448 : new_insn_uid_start = get_max_uid ();
5552 3448 : new_regno_start = max_reg_num ();
5553 3448 : change_p = curr_insn_transform (true);
5554 3448 : new_regno_start = saved_new_regno_start;
5555 3448 : new_insn_uid_start = saved_new_insn_uid_start;
5556 3448 : return ! change_p;
5557 : }
5558 :
5559 : /* Return true if X is in LIST. */
5560 : static bool
5561 1433275 : in_list_p (rtx x, rtx list)
5562 : {
5563 2455114 : for (; list != NULL_RTX; list = XEXP (list, 1))
5564 1347413 : if (XEXP (list, 0) == x)
5565 : return true;
5566 : return false;
5567 : }
5568 :
5569 : /* Return true if X contains an allocatable hard register (if
5570 : HARD_REG_P) or a (spilled if SPILLED_P) pseudo. */
5571 : static bool
5572 7794476 : contains_reg_p (rtx x, bool hard_reg_p, bool spilled_p)
5573 : {
5574 7794476 : int i, j;
5575 7794476 : const char *fmt;
5576 7794476 : enum rtx_code code;
5577 :
5578 7794476 : code = GET_CODE (x);
5579 7794476 : if (REG_P (x))
5580 : {
5581 1590279 : int regno = REGNO (x);
5582 1590279 : HARD_REG_SET alloc_regs;
5583 :
5584 1590279 : if (hard_reg_p)
5585 : {
5586 501452 : if (regno >= FIRST_PSEUDO_REGISTER)
5587 148837 : regno = lra_get_regno_hard_regno (regno);
5588 501452 : if (regno < 0)
5589 : return false;
5590 501452 : alloc_regs = ~lra_no_alloc_regs;
5591 501452 : return overlaps_hard_reg_set_p (alloc_regs, GET_MODE (x), regno);
5592 : }
5593 : else
5594 : {
5595 1088827 : if (regno < FIRST_PSEUDO_REGISTER)
5596 : return false;
5597 348823 : if (! spilled_p)
5598 : return true;
5599 183909 : return lra_get_regno_hard_regno (regno) < 0;
5600 : }
5601 : }
5602 6204197 : fmt = GET_RTX_FORMAT (code);
5603 15351639 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
5604 : {
5605 9758070 : if (fmt[i] == 'e')
5606 : {
5607 4327526 : if (contains_reg_p (XEXP (x, i), hard_reg_p, spilled_p))
5608 : return true;
5609 : }
5610 5430544 : else if (fmt[i] == 'E')
5611 : {
5612 1311885 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
5613 1201248 : if (contains_reg_p (XVECEXP (x, i, j), hard_reg_p, spilled_p))
5614 : return true;
5615 : }
5616 : }
5617 : return false;
5618 : }
5619 :
5620 : /* Process all regs in location *LOC and change them on equivalent
5621 : substitution. Return true if any change was done. */
5622 : static bool
5623 3348 : loc_equivalence_change_p (rtx *loc)
5624 : {
5625 3348 : rtx subst, reg, x = *loc;
5626 3348 : bool result = false;
5627 3348 : enum rtx_code code = GET_CODE (x);
5628 3348 : const char *fmt;
5629 3348 : int i, j;
5630 :
5631 3348 : if (code == SUBREG)
5632 : {
5633 20 : reg = SUBREG_REG (x);
5634 20 : if ((subst = get_equiv_with_elimination (reg, curr_insn)) != reg
5635 20 : && GET_MODE (subst) == VOIDmode)
5636 : {
5637 : /* We cannot reload debug location. Simplify subreg here
5638 : while we know the inner mode. */
5639 0 : *loc = simplify_gen_subreg (GET_MODE (x), subst,
5640 0 : GET_MODE (reg), SUBREG_BYTE (x));
5641 0 : return true;
5642 : }
5643 : }
5644 3348 : if (code == REG && (subst = get_equiv_with_elimination (x, curr_insn)) != x)
5645 : {
5646 8 : *loc = subst;
5647 8 : return true;
5648 : }
5649 :
5650 : /* Scan all the operand sub-expressions. */
5651 3340 : fmt = GET_RTX_FORMAT (code);
5652 8168 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
5653 : {
5654 4828 : if (fmt[i] == 'e')
5655 2577 : result = loc_equivalence_change_p (&XEXP (x, i)) || result;
5656 2251 : else if (fmt[i] == 'E')
5657 270 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
5658 200 : result
5659 210 : = loc_equivalence_change_p (&XVECEXP (x, i, j)) || result;
5660 : }
5661 : return result;
5662 : }
5663 :
5664 : /* Similar to loc_equivalence_change_p, but for use as
5665 : simplify_replace_fn_rtx callback. DATA is insn for which the
5666 : elimination is done. If it null we don't do the elimination. */
5667 : static rtx
5668 42727322 : loc_equivalence_callback (rtx loc, const_rtx, void *data)
5669 : {
5670 42727322 : if (!REG_P (loc))
5671 : return NULL_RTX;
5672 :
5673 10988530 : rtx subst = (data == NULL
5674 10988530 : ? get_equiv (loc) : get_equiv_with_elimination (loc, (rtx_insn *) data));
5675 10988530 : if (subst != loc)
5676 38378 : return subst;
5677 :
5678 : return NULL_RTX;
5679 : }
5680 :
5681 : /* Maximum number of generated reload insns per an insn. It is for
5682 : preventing this pass cycling in a bug case. */
5683 : #define MAX_RELOAD_INSNS_NUMBER LRA_MAX_INSN_RELOADS
5684 :
5685 : /* The current iteration number of this LRA pass. */
5686 : int lra_constraint_iter;
5687 :
5688 : /* True if we should during assignment sub-pass check assignment
5689 : correctness for all pseudos and spill some of them to correct
5690 : conflicts. It can be necessary when we substitute equiv which
5691 : needs checking register allocation correctness because the
5692 : equivalent value contains allocatable hard registers, or when we
5693 : restore multi-register pseudo, or when we change the insn code and
5694 : its operand became INOUT operand when it was IN one before. */
5695 : bool check_and_force_assignment_correctness_p;
5696 :
5697 : /* Return true if REGNO is referenced in more than one block. */
5698 : static bool
5699 158645 : multi_block_pseudo_p (int regno)
5700 : {
5701 158645 : basic_block bb = NULL;
5702 158645 : unsigned int uid;
5703 158645 : bitmap_iterator bi;
5704 :
5705 158645 : if (regno < FIRST_PSEUDO_REGISTER)
5706 : return false;
5707 :
5708 486044 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi)
5709 332200 : if (bb == NULL)
5710 158645 : bb = BLOCK_FOR_INSN (lra_insn_recog_data[uid]->insn);
5711 173555 : else if (BLOCK_FOR_INSN (lra_insn_recog_data[uid]->insn) != bb)
5712 : return true;
5713 : return false;
5714 : }
5715 :
5716 : /* Return true if LIST contains a deleted insn. */
5717 : static bool
5718 757986 : contains_deleted_insn_p (rtx_insn_list *list)
5719 : {
5720 1447305 : for (; list != NULL_RTX; list = list->next ())
5721 689319 : if (NOTE_P (list->insn ())
5722 689319 : && NOTE_KIND (list->insn ()) == NOTE_INSN_DELETED)
5723 : return true;
5724 : return false;
5725 : }
5726 :
5727 : /* Return true if X contains a pseudo dying in INSN. */
5728 : static bool
5729 2361755 : dead_pseudo_p (rtx x, rtx_insn *insn)
5730 : {
5731 2361755 : int i, j;
5732 2361755 : const char *fmt;
5733 2361755 : enum rtx_code code;
5734 :
5735 2361755 : if (REG_P (x))
5736 529360 : return (insn != NULL_RTX
5737 529360 : && find_regno_note (insn, REG_DEAD, REGNO (x)) != NULL_RTX);
5738 1832395 : code = GET_CODE (x);
5739 1832395 : fmt = GET_RTX_FORMAT (code);
5740 4685978 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
5741 : {
5742 2859090 : if (fmt[i] == 'e')
5743 : {
5744 1403169 : if (dead_pseudo_p (XEXP (x, i), insn))
5745 : return true;
5746 : }
5747 1455921 : else if (fmt[i] == 'E')
5748 : {
5749 295826 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
5750 270645 : if (dead_pseudo_p (XVECEXP (x, i, j), insn))
5751 : return true;
5752 : }
5753 : }
5754 : return false;
5755 : }
5756 :
5757 : /* Return true if INSN contains a dying pseudo in INSN right hand
5758 : side. */
5759 : static bool
5760 687941 : insn_rhs_dead_pseudo_p (rtx_insn *insn)
5761 : {
5762 687941 : rtx set = single_set (insn);
5763 :
5764 687941 : gcc_assert (set != NULL);
5765 687941 : return dead_pseudo_p (SET_SRC (set), insn);
5766 : }
5767 :
5768 : /* Return true if any init insn of REGNO contains a dying pseudo in
5769 : insn right hand side. */
5770 : static bool
5771 756608 : init_insn_rhs_dead_pseudo_p (int regno)
5772 : {
5773 756608 : rtx_insn_list *insns = ira_reg_equiv[regno].init_insns;
5774 :
5775 756608 : if (insns == NULL)
5776 : return false;
5777 1369104 : for (; insns != NULL_RTX; insns = insns->next ())
5778 687941 : if (insn_rhs_dead_pseudo_p (insns->insn ()))
5779 : return true;
5780 : return false;
5781 : }
5782 :
5783 : /* Return TRUE if REGNO has a reverse equivalence. The equivalence is
5784 : reverse only if we have one init insn with given REGNO as a
5785 : source. */
5786 : static bool
5787 757986 : reverse_equiv_p (int regno)
5788 : {
5789 757986 : rtx_insn_list *insns = ira_reg_equiv[regno].init_insns;
5790 757986 : rtx set;
5791 :
5792 757986 : if (insns == NULL)
5793 : return false;
5794 689319 : if (! INSN_P (insns->insn ())
5795 1378638 : || insns->next () != NULL)
5796 : return false;
5797 689319 : if ((set = single_set (insns->insn ())) == NULL_RTX)
5798 : return false;
5799 689319 : return REG_P (SET_SRC (set)) && (int) REGNO (SET_SRC (set)) == regno;
5800 : }
5801 :
5802 : /* Return TRUE if REGNO was reloaded in an equivalence init insn. We
5803 : call this function only for non-reverse equivalence. */
5804 : static bool
5805 749830 : contains_reloaded_insn_p (int regno)
5806 : {
5807 749830 : rtx set;
5808 749830 : rtx_insn_list *list = ira_reg_equiv[regno].init_insns;
5809 :
5810 1430991 : for (; list != NULL; list = list->next ())
5811 681163 : if ((set = single_set (list->insn ())) == NULL_RTX
5812 681163 : || ! REG_P (SET_DEST (set))
5813 1362326 : || (int) REGNO (SET_DEST (set)) != regno)
5814 : return true;
5815 : return false;
5816 : }
5817 :
5818 : /* Try combine secondary memory reload insn FROM for insn TO into TO insn.
5819 : FROM should be a load insn (usually a secondary memory reload insn). Return
5820 : TRUE in case of success. */
5821 : static bool
5822 7524299 : combine_reload_insn (rtx_insn *from, rtx_insn *to)
5823 : {
5824 7524299 : bool ok_p;
5825 7524299 : rtx_insn *saved_insn;
5826 7524299 : rtx set, from_reg, to_reg, op;
5827 7524299 : enum reg_class to_class, from_class;
5828 7524299 : int n, nop;
5829 7524299 : signed char changed_nops[MAX_RECOG_OPERANDS + 1];
5830 :
5831 : /* Check conditions for second memory reload and original insn: */
5832 7524299 : if ((targetm.secondary_memory_needed
5833 : == hook_bool_mode_reg_class_t_reg_class_t_false)
5834 7524299 : || NEXT_INSN (from) != to
5835 4506065 : || !NONDEBUG_INSN_P (to)
5836 12030364 : || CALL_P (to))
5837 : return false;
5838 :
5839 4500697 : lra_insn_recog_data_t id = lra_get_insn_recog_data (to);
5840 4500697 : struct lra_static_insn_data *static_id = id->insn_static_data;
5841 :
5842 4500697 : if (id->used_insn_alternative == LRA_UNKNOWN_ALT
5843 4500697 : || (set = single_set (from)) == NULL_RTX)
5844 : return false;
5845 4467257 : from_reg = SET_DEST (set);
5846 4467257 : to_reg = SET_SRC (set);
5847 : /* Ignore optional reloads: */
5848 4386551 : if (! REG_P (from_reg) || ! REG_P (to_reg)
5849 7401170 : || bitmap_bit_p (&lra_optional_reload_pseudos, REGNO (from_reg)))
5850 : return false;
5851 2382778 : to_class = lra_get_allocno_class (REGNO (to_reg));
5852 2382778 : from_class = lra_get_allocno_class (REGNO (from_reg));
5853 : /* Check that reload insn is a load: */
5854 2382778 : if (to_class != NO_REGS || from_class == NO_REGS)
5855 : return false;
5856 47907 : for (n = nop = 0; nop < static_id->n_operands; nop++)
5857 : {
5858 34567 : if (static_id->operand[nop].type != OP_IN)
5859 12346 : continue;
5860 22221 : op = *id->operand_loc[nop];
5861 22221 : if (!REG_P (op) || REGNO (op) != REGNO (from_reg))
5862 9064 : continue;
5863 13157 : *id->operand_loc[nop] = to_reg;
5864 13157 : changed_nops[n++] = nop;
5865 : }
5866 13340 : changed_nops[n] = -1;
5867 13340 : lra_update_dups (id, changed_nops);
5868 13340 : lra_update_insn_regno_info (to);
5869 13340 : ok_p = recog_memoized (to) >= 0;
5870 13340 : if (ok_p)
5871 : {
5872 : /* Check that combined insn does not need any reloads: */
5873 13321 : saved_insn = curr_insn;
5874 13321 : curr_insn = to;
5875 13321 : curr_id = lra_get_insn_recog_data (curr_insn);
5876 13321 : curr_static_id = curr_id->insn_static_data;
5877 13321 : for (bool swapped_p = false;;)
5878 : {
5879 15685 : ok_p = !curr_insn_transform (true);
5880 15685 : if (ok_p || curr_static_id->commutative < 0)
5881 : break;
5882 4728 : swap_operands (curr_static_id->commutative);
5883 4728 : if (lra_dump_file != NULL)
5884 : {
5885 0 : fprintf (lra_dump_file,
5886 : " Swapping %scombined insn operands:\n",
5887 : swapped_p ? "back " : "");
5888 0 : dump_insn_slim (lra_dump_file, to);
5889 : }
5890 4728 : if (swapped_p)
5891 : break;
5892 : swapped_p = true;
5893 : }
5894 13321 : curr_insn = saved_insn;
5895 13321 : curr_id = lra_get_insn_recog_data (curr_insn);
5896 13321 : curr_static_id = curr_id->insn_static_data;
5897 : }
5898 13340 : if (ok_p)
5899 : {
5900 3597 : id->used_insn_alternative = -1;
5901 3597 : lra_push_insn_and_update_insn_regno_info (to);
5902 3597 : if (lra_dump_file != NULL)
5903 : {
5904 0 : fprintf (lra_dump_file, " Use combined insn:\n");
5905 0 : dump_insn_slim (lra_dump_file, to);
5906 : }
5907 : return true;
5908 : }
5909 9743 : if (lra_dump_file != NULL)
5910 : {
5911 0 : fprintf (lra_dump_file, " Failed combined insn:\n");
5912 0 : dump_insn_slim (lra_dump_file, to);
5913 : }
5914 19763 : for (int i = 0; i < n; i++)
5915 : {
5916 10020 : nop = changed_nops[i];
5917 10020 : *id->operand_loc[nop] = from_reg;
5918 : }
5919 9743 : lra_update_dups (id, changed_nops);
5920 9743 : lra_update_insn_regno_info (to);
5921 9743 : if (lra_dump_file != NULL)
5922 : {
5923 0 : fprintf (lra_dump_file, " Restoring insn after failed combining:\n");
5924 0 : dump_insn_slim (lra_dump_file, to);
5925 : }
5926 : return false;
5927 : }
5928 :
5929 : static bool
5930 540803 : virtual_reg_p (const_rtx reg)
5931 : {
5932 0 : return (VIRTUAL_REGISTER_P (reg)
5933 540803 : || (REGNO(reg) == ARG_POINTER_REGNUM
5934 : && !HARD_FRAME_POINTER_IS_ARG_POINTER)
5935 1006155 : || (REGNO(reg) == FRAME_POINTER_REGNUM
5936 0 : && !HARD_FRAME_POINTER_IS_FRAME_POINTER));
5937 : }
5938 :
5939 : /* Return true if X contains a virtual register. */
5940 : static bool
5941 747898 : contains_virtual_reg_p (rtx x)
5942 : {
5943 747898 : subrtx_iterator::array_type array;
5944 2140664 : FOR_EACH_SUBRTX (iter, array, x, NONCONST)
5945 1933569 : if (REG_P (*iter) && virtual_reg_p (*iter))
5946 368422 : return true;
5947 379476 : return false;
5948 747898 : }
5949 :
5950 : /* Return true if pseudo REGNO is an operand of an incdec rtx in any of its
5951 : insns. */
5952 : static bool
5953 368422 : reg_has_incdec_p (int regno)
5954 : {
5955 368422 : unsigned int uid;
5956 368422 : bitmap_iterator bi;
5957 1868119 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi)
5958 : {
5959 1499697 : rtx_insn *insn = lra_insn_recog_data[uid]->insn;
5960 1499697 : if (!NONDEBUG_INSN_P (insn))
5961 26216 : continue;
5962 1473481 : subrtx_iterator::array_type array;
5963 8650628 : FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
5964 : {
5965 7177147 : const_rtx x = *iter;
5966 7177147 : if ((GET_CODE (x) == PRE_INC || GET_CODE (x) == PRE_DEC
5967 7072746 : || GET_CODE (x) == POST_INC || GET_CODE (x) == POST_DEC
5968 7072746 : || GET_CODE (x) == PRE_MODIFY || GET_CODE (x) == POST_MODIFY)
5969 105764 : && REG_P (XEXP (x, 0))
5970 7282911 : && REGNO (XEXP (x, 0)) == (unsigned int) regno)
5971 0 : return true;
5972 : }
5973 1473481 : }
5974 : return false;
5975 : }
5976 :
5977 : /* Entry function of LRA constraint pass. Return true if the
5978 : constraint pass did change the code. */
5979 : bool
5980 3298131 : lra_constraints (bool first_p)
5981 : {
5982 3298131 : bool changed_p;
5983 3298131 : int i, hard_regno, new_insns_num;
5984 3298131 : unsigned int min_len, new_min_len, uid;
5985 3298131 : rtx set, x, reg, nosubreg_dest;
5986 3298131 : rtx_insn *original_insn;
5987 3298131 : basic_block last_bb;
5988 3298131 : bitmap_iterator bi;
5989 :
5990 3298131 : lra_constraint_iter++;
5991 3298131 : if (lra_dump_file != NULL)
5992 193 : fprintf (lra_dump_file, "\n********** Local #%d: **********\n\n",
5993 : lra_constraint_iter);
5994 3298131 : changed_p = false;
5995 3298131 : if (pic_offset_table_rtx
5996 3298131 : && REGNO (pic_offset_table_rtx) >= FIRST_PSEUDO_REGISTER)
5997 104935 : check_and_force_assignment_correctness_p = true;
5998 3193196 : else if (first_p)
5999 : /* On the first iteration we should check IRA assignment
6000 : correctness. In rare cases, the assignments can be wrong as
6001 : early clobbers operands are ignored in IRA or usages of
6002 : paradoxical sub-registers are not taken into account by
6003 : IRA. */
6004 1475384 : check_and_force_assignment_correctness_p = true;
6005 3298131 : new_insn_uid_start = get_max_uid ();
6006 3298131 : new_regno_start = first_p ? lra_constraint_new_regno_start : max_reg_num ();
6007 : /* Mark used hard regs for target stack size calculations. */
6008 212879669 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
6009 209581538 : if (lra_reg_info[i].nrefs != 0
6010 308870675 : && (hard_regno = lra_get_regno_hard_regno (i)) >= 0)
6011 : {
6012 95274668 : int j, nregs;
6013 :
6014 95274668 : nregs = hard_regno_nregs (hard_regno, lra_reg_info[i].biggest_mode);
6015 193755499 : for (j = 0; j < nregs; j++)
6016 98480831 : df_set_regs_ever_live (hard_regno + j, true);
6017 : }
6018 : /* Do elimination before the equivalence processing as we can spill
6019 : some pseudos during elimination. */
6020 3298131 : lra_eliminate (false, first_p);
6021 3298131 : auto_bitmap equiv_insn_bitmap (®_obstack);
6022 :
6023 : /* Register elimination can create new pseudos via the addptr pattern,
6024 : so make sure the equivalency tables are resized appropriately. */
6025 3298131 : ira_expand_reg_equiv ();
6026 216177800 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
6027 209581538 : if (lra_reg_info[i].nrefs != 0)
6028 : {
6029 99289137 : ira_reg_equiv[i].profitable_p = true;
6030 99289137 : reg = regno_reg_rtx[i];
6031 99289137 : if (lra_get_regno_hard_regno (i) < 0 && (x = get_equiv (reg)) != reg)
6032 : {
6033 769056 : bool pseudo_p = contains_reg_p (x, false, false);
6034 :
6035 : /* After RTL transformation, we cannot guarantee that
6036 : pseudo in the substitution was not reloaded which might
6037 : make equivalence invalid. For example, in reverse
6038 : equiv of p0
6039 :
6040 : p0 <- ...
6041 : ...
6042 : equiv_mem <- p0
6043 :
6044 : the memory address register was reloaded before the 2nd
6045 : insn. */
6046 769056 : if ((! first_p && pseudo_p)
6047 : /* We don't use DF for compilation speed sake. So it
6048 : is problematic to update live info when we use an
6049 : equivalence containing pseudos in more than one
6050 : BB. */
6051 762787 : || (pseudo_p && multi_block_pseudo_p (i))
6052 : /* If an init insn was deleted for some reason, cancel
6053 : the equiv. We could update the equiv insns after
6054 : transformations including an equiv insn deletion
6055 : but it is not worthy as such cases are extremely
6056 : rare. */
6057 757986 : || contains_deleted_insn_p (ira_reg_equiv[i].init_insns)
6058 : /* If it is not a reverse equivalence, we check that a
6059 : pseudo in rhs of the init insn is not dying in the
6060 : insn. Otherwise, the live info at the beginning of
6061 : the corresponding BB might be wrong after we
6062 : removed the insn. When the equiv can be a
6063 : constant, the right hand side of the init insn can
6064 : be a pseudo. */
6065 757986 : || (! reverse_equiv_p (i)
6066 756608 : && (init_insn_rhs_dead_pseudo_p (i)
6067 : /* If we reloaded the pseudo in an equivalence
6068 : init insn, we cannot remove the equiv init
6069 : insns and the init insns might write into
6070 : const memory in this case. */
6071 749830 : || contains_reloaded_insn_p (i)))
6072 : /* Prevent access beyond equivalent memory for
6073 : paradoxical subregs. */
6074 751206 : || (MEM_P (x)
6075 1215057 : && maybe_gt (GET_MODE_SIZE (lra_reg_info[i].biggest_mode),
6076 : GET_MODE_SIZE (GET_MODE (x))))
6077 750473 : || (pic_offset_table_rtx
6078 54105 : && ((CONST_POOL_OK_P (PSEUDO_REGNO_MODE (i), x)
6079 8944 : && (targetm.preferred_reload_class
6080 4472 : (x, lra_get_allocno_class (i)) == NO_REGS))
6081 52338 : || contains_symbol_ref_p (x)))
6082 : /* PR120165: An inc/dec changes the reg value, so we cannot
6083 : substitute the equiv which might contain virtual regs
6084 : requiring elimination. An insn with inc/dec can also change
6085 : the value of another reg (e.g. sp) against which we
6086 : eliminate. As we update elimination once per insn and an
6087 : inc/dec operand might require input and output reloads, we
6088 : can generate a wrong offset for a reload insn. */
6089 1516954 : || (contains_virtual_reg_p (x) && reg_has_incdec_p (i)))
6090 21158 : ira_reg_equiv[i].defined_p
6091 21158 : = ira_reg_equiv[i].caller_save_p = false;
6092 769056 : if (contains_reg_p (x, false, true))
6093 9710 : ira_reg_equiv[i].profitable_p = false;
6094 769056 : if (get_equiv (reg) != reg)
6095 743092 : bitmap_ior_into (equiv_insn_bitmap, &lra_reg_info[i].insn_bitmap);
6096 : }
6097 : }
6098 212879669 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
6099 209581538 : update_equiv (i);
6100 : /* We should add all insns containing pseudos which should be
6101 : substituted by their equivalences. */
6102 5792684 : EXECUTE_IF_SET_IN_BITMAP (equiv_insn_bitmap, 0, uid, bi)
6103 2494553 : lra_push_insn_by_uid (uid);
6104 3298131 : min_len = lra_insn_stack_length ();
6105 3298131 : new_insns_num = 0;
6106 3298131 : last_bb = NULL;
6107 3298131 : changed_p = false;
6108 3298131 : original_insn = NULL;
6109 175868191 : while ((new_min_len = lra_insn_stack_length ()) != 0)
6110 : {
6111 169271929 : curr_insn = lra_pop_insn ();
6112 169271929 : --new_min_len;
6113 169271929 : curr_bb = BLOCK_FOR_INSN (curr_insn);
6114 169271929 : if (curr_bb != last_bb)
6115 : {
6116 21179141 : last_bb = curr_bb;
6117 21179141 : bb_reload_num = lra_curr_reload_num;
6118 : }
6119 169271929 : if (min_len > new_min_len)
6120 : {
6121 : min_len = new_min_len;
6122 : new_insns_num = 0;
6123 : original_insn = curr_insn;
6124 : }
6125 7524299 : else if (combine_reload_insn (curr_insn, original_insn))
6126 : {
6127 3597 : continue;
6128 : }
6129 7520702 : if (new_insns_num > MAX_RELOAD_INSNS_NUMBER)
6130 0 : internal_error
6131 0 : ("maximum number of generated reload insns per insn achieved (%d)",
6132 : MAX_RELOAD_INSNS_NUMBER);
6133 169268332 : new_insns_num++;
6134 169268332 : if (DEBUG_INSN_P (curr_insn))
6135 : {
6136 : /* We need to check equivalence in debug insn and change
6137 : pseudo to the equivalent value if necessary. */
6138 58422791 : curr_id = lra_get_insn_recog_data (curr_insn);
6139 58422791 : if (bitmap_bit_p (equiv_insn_bitmap, INSN_UID (curr_insn)))
6140 : {
6141 29796 : rtx old = *curr_id->operand_loc[0];
6142 29796 : *curr_id->operand_loc[0]
6143 29796 : = simplify_replace_fn_rtx (old, NULL_RTX,
6144 : loc_equivalence_callback, curr_insn);
6145 29796 : if (old != *curr_id->operand_loc[0])
6146 : {
6147 : /* If we substitute pseudo by shared equivalence, we can fail
6148 : to update LRA reg info and this can result in many
6149 : unexpected consequences. So keep rtl unshared: */
6150 29796 : *curr_id->operand_loc[0]
6151 29796 : = copy_rtx (*curr_id->operand_loc[0]);
6152 29796 : lra_update_insn_regno_info (curr_insn);
6153 29796 : changed_p = true;
6154 : }
6155 : }
6156 : }
6157 110845541 : else if (INSN_P (curr_insn))
6158 : {
6159 109780160 : if ((set = single_set (curr_insn)) != NULL_RTX)
6160 : {
6161 104272360 : nosubreg_dest = SET_DEST (set);
6162 : /* The equivalence pseudo could be set up as SUBREG in a
6163 : case when it is a call restore insn in a mode
6164 : different from the pseudo mode. */
6165 104272360 : if (GET_CODE (nosubreg_dest) == SUBREG)
6166 1208882 : nosubreg_dest = SUBREG_REG (nosubreg_dest);
6167 104999950 : if ((REG_P (nosubreg_dest)
6168 77189862 : && (x = get_equiv (nosubreg_dest)) != nosubreg_dest
6169 : /* Remove insns which set up a pseudo whose value
6170 : cannot be changed. Such insns might be not in
6171 : init_insns because we don't update equiv data
6172 : during insn transformations.
6173 :
6174 : As an example, let suppose that a pseudo got
6175 : hard register and on the 1st pass was not
6176 : changed to equivalent constant. We generate an
6177 : additional insn setting up the pseudo because of
6178 : secondary memory movement. Then the pseudo is
6179 : spilled and we use the equiv constant. In this
6180 : case we should remove the additional insn and
6181 : this insn is not init_insns list. */
6182 747454 : && (! MEM_P (x) || MEM_READONLY_P (x)
6183 : /* Check that this is actually an insn setting
6184 : up the equivalence. */
6185 345438 : || in_list_p (curr_insn,
6186 345438 : ira_reg_equiv
6187 345438 : [REGNO (nosubreg_dest)].init_insns)))
6188 180735942 : || (((x = get_equiv (SET_SRC (set))) != SET_SRC (set))
6189 2175674 : && in_list_p (curr_insn,
6190 1087837 : ira_reg_equiv
6191 1087837 : [REGNO (SET_SRC (set))].init_insns)
6192 : /* This is a reverse equivalence to memory (see ira.cc)
6193 : in store insn. We can reload all the destination and
6194 : have an output reload which is a store to memory. If
6195 : we just remove the insn, we will have the output
6196 : reload storing an undefined value to the memory.
6197 : Check that we did not reload the memory to prevent a
6198 : wrong code generation. We could implement using the
6199 : equivalence still in such case but doing this is not
6200 : worth the efforts as such case is very rare. */
6201 1310 : && MEM_P (nosubreg_dest)))
6202 : {
6203 : /* This is equiv init insn of pseudo which did not get a
6204 : hard register -- remove the insn. */
6205 727590 : if (lra_dump_file != NULL)
6206 : {
6207 9 : fprintf (lra_dump_file,
6208 : " Removing equiv init insn %i (freq=%d)\n",
6209 3 : INSN_UID (curr_insn),
6210 6 : REG_FREQ_FROM_BB (BLOCK_FOR_INSN (curr_insn)));
6211 3 : dump_insn_slim (lra_dump_file, curr_insn);
6212 : }
6213 727590 : if (contains_reg_p (x, true, false))
6214 148837 : check_and_force_assignment_correctness_p = true;
6215 727590 : lra_set_insn_deleted (curr_insn);
6216 727590 : continue;
6217 : }
6218 : }
6219 109052570 : curr_id = lra_get_insn_recog_data (curr_insn);
6220 109052570 : curr_static_id = curr_id->insn_static_data;
6221 109052570 : init_curr_insn_input_reloads ();
6222 109052570 : init_curr_operand_mode ();
6223 109052570 : if (curr_insn_transform (false))
6224 : changed_p = true;
6225 : /* Check non-transformed insns too for equiv change as USE
6226 : or CLOBBER don't need reloads but can contain pseudos
6227 : being changed on their equivalences. */
6228 102677251 : else if (bitmap_bit_p (equiv_insn_bitmap, INSN_UID (curr_insn))
6229 102677251 : && loc_equivalence_change_p (&PATTERN (curr_insn)))
6230 : {
6231 8 : lra_update_insn_regno_info (curr_insn);
6232 8 : lra_push_insn_by_uid (INSN_UID (curr_insn));
6233 8 : changed_p = true;
6234 : }
6235 : }
6236 : }
6237 :
6238 : /* If we used a new hard regno, changed_p should be true because the
6239 : hard reg is assigned to a new pseudo. */
6240 3298131 : if (flag_checking && !changed_p)
6241 : {
6242 137001914 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
6243 134333444 : if (lra_reg_info[i].nrefs != 0
6244 195919035 : && (hard_regno = lra_get_regno_hard_regno (i)) >= 0)
6245 : {
6246 60101970 : int j, nregs = hard_regno_nregs (hard_regno,
6247 60101970 : PSEUDO_REGNO_MODE (i));
6248 :
6249 122315223 : for (j = 0; j < nregs; j++)
6250 62213253 : lra_assert (df_regs_ever_live_p (hard_regno + j));
6251 : }
6252 : }
6253 2668510 : if (changed_p)
6254 629624 : lra_dump_insns_if_possible ("changed func after local");
6255 3298131 : return changed_p;
6256 3298131 : }
6257 :
6258 : static void initiate_invariants (void);
6259 : static void finish_invariants (void);
6260 :
6261 : /* Initiate the LRA constraint pass. It is done once per
6262 : function. */
6263 : void
6264 1515921 : lra_constraints_init (void)
6265 : {
6266 1515921 : initiate_invariants ();
6267 1515921 : }
6268 :
6269 : /* Finalize the LRA constraint pass. It is done once per
6270 : function. */
6271 : void
6272 1515921 : lra_constraints_finish (void)
6273 : {
6274 1515921 : finish_invariants ();
6275 1515921 : }
6276 :
6277 :
6278 :
6279 : /* Structure describes invariants for ineheritance. */
6280 : struct lra_invariant
6281 : {
6282 : /* The order number of the invariant. */
6283 : int num;
6284 : /* The invariant RTX. */
6285 : rtx invariant_rtx;
6286 : /* The origin insn of the invariant. */
6287 : rtx_insn *insn;
6288 : };
6289 :
6290 : typedef lra_invariant invariant_t;
6291 : typedef invariant_t *invariant_ptr_t;
6292 : typedef const invariant_t *const_invariant_ptr_t;
6293 :
6294 : /* Pointer to the inheritance invariants. */
6295 : static vec<invariant_ptr_t> invariants;
6296 :
6297 : /* Allocation pool for the invariants. */
6298 : static object_allocator<lra_invariant> *invariants_pool;
6299 :
6300 : /* Hash table for the invariants. */
6301 : static htab_t invariant_table;
6302 :
6303 : /* Hash function for INVARIANT. */
6304 : static hashval_t
6305 199703 : invariant_hash (const void *invariant)
6306 : {
6307 199703 : rtx inv = ((const_invariant_ptr_t) invariant)->invariant_rtx;
6308 199703 : return lra_rtx_hash (inv);
6309 : }
6310 :
6311 : /* Equal function for invariants INVARIANT1 and INVARIANT2. */
6312 : static int
6313 68408 : invariant_eq_p (const void *invariant1, const void *invariant2)
6314 : {
6315 68408 : rtx inv1 = ((const_invariant_ptr_t) invariant1)->invariant_rtx;
6316 68408 : rtx inv2 = ((const_invariant_ptr_t) invariant2)->invariant_rtx;
6317 :
6318 68408 : return rtx_equal_p (inv1, inv2);
6319 : }
6320 :
6321 : /* Insert INVARIANT_RTX into the table if it is not there yet. Return
6322 : invariant which is in the table. */
6323 : static invariant_ptr_t
6324 198463 : insert_invariant (rtx invariant_rtx)
6325 : {
6326 198463 : void **entry_ptr;
6327 198463 : invariant_t invariant;
6328 198463 : invariant_ptr_t invariant_ptr;
6329 :
6330 198463 : invariant.invariant_rtx = invariant_rtx;
6331 198463 : entry_ptr = htab_find_slot (invariant_table, &invariant, INSERT);
6332 198463 : if (*entry_ptr == NULL)
6333 : {
6334 171443 : invariant_ptr = invariants_pool->allocate ();
6335 171443 : invariant_ptr->invariant_rtx = invariant_rtx;
6336 171443 : invariant_ptr->insn = NULL;
6337 171443 : invariants.safe_push (invariant_ptr);
6338 171443 : *entry_ptr = (void *) invariant_ptr;
6339 : }
6340 198463 : return (invariant_ptr_t) *entry_ptr;
6341 : }
6342 :
6343 : /* Initiate the invariant table. */
6344 : static void
6345 1515921 : initiate_invariants (void)
6346 : {
6347 1515921 : invariants.create (100);
6348 1515921 : invariants_pool
6349 1515921 : = new object_allocator<lra_invariant> ("Inheritance invariants");
6350 1515921 : invariant_table = htab_create (100, invariant_hash, invariant_eq_p, NULL);
6351 1515921 : }
6352 :
6353 : /* Finish the invariant table. */
6354 : static void
6355 1515921 : finish_invariants (void)
6356 : {
6357 1515921 : htab_delete (invariant_table);
6358 3031842 : delete invariants_pool;
6359 1515921 : invariants.release ();
6360 1515921 : }
6361 :
6362 : /* Make the invariant table empty. */
6363 : static void
6364 12931346 : clear_invariants (void)
6365 : {
6366 12931346 : htab_empty (invariant_table);
6367 12931346 : invariants_pool->release ();
6368 12931346 : invariants.truncate (0);
6369 12931346 : }
6370 :
6371 :
6372 :
6373 : /* This page contains code to do inheritance/split
6374 : transformations. */
6375 :
6376 : /* Number of reloads passed so far in current EBB. */
6377 : static int reloads_num;
6378 :
6379 : /* Number of calls passed so far in current EBB. */
6380 : static int calls_num;
6381 :
6382 : /* Index ID is the CALLS_NUM associated the last call we saw with
6383 : ABI identifier ID. */
6384 : static int last_call_for_abi[NUM_ABI_IDS];
6385 :
6386 : /* Which registers have been fully or partially clobbered by a call
6387 : since they were last used. */
6388 : static HARD_REG_SET full_and_partial_call_clobbers;
6389 :
6390 : /* Current reload pseudo check for validity of elements in
6391 : USAGE_INSNS. */
6392 : static int curr_usage_insns_check;
6393 :
6394 : /* Info about last usage of registers in EBB to do inheritance/split
6395 : transformation. Inheritance transformation is done from a spilled
6396 : pseudo and split transformations from a hard register or a pseudo
6397 : assigned to a hard register. */
6398 : struct usage_insns
6399 : {
6400 : /* If the value is equal to CURR_USAGE_INSNS_CHECK, then the member
6401 : value INSNS is valid. The insns is chain of optional debug insns
6402 : and a finishing non-debug insn using the corresponding reg. The
6403 : value is also used to mark the registers which are set up in the
6404 : current insn. The negated insn uid is used for this. */
6405 : int check;
6406 : /* Value of global reloads_num at the last insn in INSNS. */
6407 : int reloads_num;
6408 : /* Value of global reloads_nums at the last insn in INSNS. */
6409 : int calls_num;
6410 : /* It can be true only for splitting. And it means that the restore
6411 : insn should be put after insn given by the following member. */
6412 : bool after_p;
6413 : /* Next insns in the current EBB which use the original reg and the
6414 : original reg value is not changed between the current insn and
6415 : the next insns. In order words, e.g. for inheritance, if we need
6416 : to use the original reg value again in the next insns we can try
6417 : to use the value in a hard register from a reload insn of the
6418 : current insn. */
6419 : rtx insns;
6420 : };
6421 :
6422 : /* Map: regno -> corresponding pseudo usage insns. */
6423 : static struct usage_insns *usage_insns;
6424 :
6425 : static void
6426 252069985 : setup_next_usage_insn (int regno, rtx insn, int reloads_num, bool after_p)
6427 : {
6428 252069985 : usage_insns[regno].check = curr_usage_insns_check;
6429 252069985 : usage_insns[regno].insns = insn;
6430 252069985 : usage_insns[regno].reloads_num = reloads_num;
6431 252069985 : usage_insns[regno].calls_num = calls_num;
6432 252069985 : usage_insns[regno].after_p = after_p;
6433 252069985 : if (regno >= FIRST_PSEUDO_REGISTER && reg_renumber[regno] >= 0)
6434 113557377 : remove_from_hard_reg_set (&full_and_partial_call_clobbers,
6435 113557377 : PSEUDO_REGNO_MODE (regno),
6436 : reg_renumber[regno]);
6437 252069985 : }
6438 :
6439 : /* The function is used to form list REGNO usages which consists of
6440 : optional debug insns finished by a non-debug insn using REGNO.
6441 : RELOADS_NUM is current number of reload insns processed so far. */
6442 : static void
6443 143827153 : add_next_usage_insn (int regno, rtx_insn *insn, int reloads_num)
6444 : {
6445 143827153 : rtx next_usage_insns;
6446 :
6447 143827153 : if (usage_insns[regno].check == curr_usage_insns_check
6448 75306626 : && (next_usage_insns = usage_insns[regno].insns) != NULL_RTX
6449 219133779 : && DEBUG_INSN_P (insn))
6450 : {
6451 : /* Check that we did not add the debug insn yet. */
6452 15084218 : if (next_usage_insns != insn
6453 15084218 : && (GET_CODE (next_usage_insns) != INSN_LIST
6454 6963306 : || XEXP (next_usage_insns, 0) != insn))
6455 15084204 : usage_insns[regno].insns = gen_rtx_INSN_LIST (VOIDmode, insn,
6456 : next_usage_insns);
6457 : }
6458 128742935 : else if (NONDEBUG_INSN_P (insn))
6459 128212536 : setup_next_usage_insn (regno, insn, reloads_num, false);
6460 : else
6461 530399 : usage_insns[regno].check = 0;
6462 143827153 : }
6463 :
6464 : /* Return first non-debug insn in list USAGE_INSNS. */
6465 : static rtx_insn *
6466 1184998 : skip_usage_debug_insns (rtx usage_insns)
6467 : {
6468 1184998 : rtx insn;
6469 :
6470 : /* Skip debug insns. */
6471 1184998 : for (insn = usage_insns;
6472 1503355 : insn != NULL_RTX && GET_CODE (insn) == INSN_LIST;
6473 318357 : insn = XEXP (insn, 1))
6474 : ;
6475 1184998 : return safe_as_a <rtx_insn *> (insn);
6476 : }
6477 :
6478 : /* Return true if we need secondary memory moves for insn in
6479 : USAGE_INSNS after inserting inherited pseudo of class INHER_CL
6480 : into the insn. */
6481 : static bool
6482 1185005 : check_secondary_memory_needed_p (enum reg_class inher_cl ATTRIBUTE_UNUSED,
6483 : rtx usage_insns ATTRIBUTE_UNUSED)
6484 : {
6485 1185005 : rtx_insn *insn;
6486 1185005 : rtx set, dest;
6487 1185005 : enum reg_class cl;
6488 :
6489 1185005 : if (inher_cl == ALL_REGS
6490 1185005 : || (insn = skip_usage_debug_insns (usage_insns)) == NULL_RTX)
6491 : return false;
6492 1184998 : lra_assert (INSN_P (insn));
6493 1184998 : if ((set = single_set (insn)) == NULL_RTX || ! REG_P (SET_DEST (set)))
6494 : return false;
6495 1147776 : dest = SET_DEST (set);
6496 1147776 : if (! REG_P (dest))
6497 : return false;
6498 1147776 : lra_assert (inher_cl != NO_REGS);
6499 1147776 : cl = get_reg_class (REGNO (dest));
6500 1147776 : return (cl != NO_REGS && cl != ALL_REGS
6501 1147776 : && targetm.secondary_memory_needed (GET_MODE (dest), inher_cl, cl));
6502 : }
6503 :
6504 : /* Registers involved in inheritance/split in the current EBB
6505 : (inheritance/split pseudos and original registers). */
6506 : static bitmap_head check_only_regs;
6507 :
6508 : /* Reload pseudos cannot be involded in invariant inheritance in the
6509 : current EBB. */
6510 : static bitmap_head invalid_invariant_regs;
6511 :
6512 : /* Do inheritance transformations for insn INSN, which defines (if
6513 : DEF_P) or uses ORIGINAL_REGNO. NEXT_USAGE_INSNS specifies which
6514 : instruction in the EBB next uses ORIGINAL_REGNO; it has the same
6515 : form as the "insns" field of usage_insns. Return true if we
6516 : succeed in such transformation.
6517 :
6518 : The transformations look like:
6519 :
6520 : p <- ... i <- ...
6521 : ... p <- i (new insn)
6522 : ... =>
6523 : <- ... p ... <- ... i ...
6524 : or
6525 : ... i <- p (new insn)
6526 : <- ... p ... <- ... i ...
6527 : ... =>
6528 : <- ... p ... <- ... i ...
6529 : where p is a spilled original pseudo and i is a new inheritance pseudo.
6530 :
6531 :
6532 : The inheritance pseudo has the smallest class of two classes CL and
6533 : class of ORIGINAL REGNO. */
6534 : static bool
6535 1272000 : inherit_reload_reg (bool def_p, int original_regno,
6536 : enum reg_class cl, rtx_insn *insn, rtx next_usage_insns)
6537 : {
6538 1272000 : if (optimize_function_for_size_p (cfun))
6539 : return false;
6540 :
6541 1246018 : enum reg_class rclass = lra_get_allocno_class (original_regno);
6542 1246018 : rtx original_reg = regno_reg_rtx[original_regno];
6543 1246018 : rtx new_reg, usage_insn;
6544 1246018 : rtx_insn *new_insns;
6545 :
6546 1246018 : lra_assert (! usage_insns[original_regno].after_p);
6547 1246018 : if (lra_dump_file != NULL)
6548 1 : fprintf (lra_dump_file,
6549 : " <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
6550 1246018 : if (! ira_reg_classes_intersect_p[cl][rclass])
6551 : {
6552 61013 : if (lra_dump_file != NULL)
6553 : {
6554 0 : fprintf (lra_dump_file,
6555 : " Rejecting inheritance for %d "
6556 : "because of disjoint classes %s and %s\n",
6557 : original_regno, reg_class_names[cl],
6558 : reg_class_names[rclass]);
6559 0 : fprintf (lra_dump_file,
6560 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6561 : }
6562 : return false;
6563 : }
6564 1185005 : if ((ira_class_subset_p[cl][rclass] && cl != rclass)
6565 : /* We don't use a subset of two classes because it can be
6566 : NO_REGS. This transformation is still profitable in most
6567 : cases even if the classes are not intersected as register
6568 : move is probably cheaper than a memory load. */
6569 447755 : || ira_class_hard_regs_num[cl] < ira_class_hard_regs_num[rclass])
6570 : {
6571 737250 : if (lra_dump_file != NULL)
6572 1 : fprintf (lra_dump_file, " Use smallest class of %s and %s\n",
6573 : reg_class_names[cl], reg_class_names[rclass]);
6574 :
6575 : rclass = cl;
6576 : }
6577 1185005 : if (check_secondary_memory_needed_p (rclass, next_usage_insns))
6578 : {
6579 : /* Reject inheritance resulting in secondary memory moves.
6580 : Otherwise, there is a danger in LRA cycling. Also such
6581 : transformation will be unprofitable. */
6582 15191 : if (lra_dump_file != NULL)
6583 : {
6584 0 : rtx_insn *insn = skip_usage_debug_insns (next_usage_insns);
6585 0 : rtx set = single_set (insn);
6586 :
6587 0 : lra_assert (set != NULL_RTX);
6588 :
6589 0 : rtx dest = SET_DEST (set);
6590 :
6591 0 : lra_assert (REG_P (dest));
6592 0 : fprintf (lra_dump_file,
6593 : " Rejecting inheritance for insn %d(%s)<-%d(%s) "
6594 : "as secondary mem is needed\n",
6595 0 : REGNO (dest), reg_class_names[get_reg_class (REGNO (dest))],
6596 0 : original_regno, reg_class_names[rclass]);
6597 0 : fprintf (lra_dump_file,
6598 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6599 : }
6600 : return false;
6601 : }
6602 1169814 : if (ira_reg_class_min_nregs[rclass][GET_MODE (original_reg)]
6603 1169814 : != ira_reg_class_max_nregs[rclass][GET_MODE (original_reg)])
6604 : {
6605 24 : if (lra_dump_file != NULL)
6606 : {
6607 0 : fprintf (lra_dump_file,
6608 : " Rejecting inheritance for %d "
6609 : "because of requiring non-uniform class %s\n",
6610 : original_regno, reg_class_names[rclass]);
6611 0 : fprintf (lra_dump_file,
6612 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6613 : }
6614 : return false;
6615 : }
6616 1169790 : new_reg = lra_create_new_reg (GET_MODE (original_reg), original_reg,
6617 : rclass, NULL, "inheritance");
6618 1169790 : start_sequence ();
6619 1169790 : if (def_p)
6620 544886 : lra_emit_move (original_reg, new_reg);
6621 : else
6622 624904 : lra_emit_move (new_reg, original_reg);
6623 1169790 : new_insns = end_sequence ();
6624 1169790 : if (NEXT_INSN (new_insns) != NULL_RTX)
6625 : {
6626 0 : if (lra_dump_file != NULL)
6627 : {
6628 0 : fprintf (lra_dump_file,
6629 : " Rejecting inheritance %d->%d "
6630 : "as it results in 2 or more insns:\n",
6631 : original_regno, REGNO (new_reg));
6632 0 : dump_rtl_slim (lra_dump_file, new_insns, NULL, -1, 0);
6633 0 : fprintf (lra_dump_file,
6634 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6635 : }
6636 : return false;
6637 : }
6638 1169790 : lra_substitute_pseudo_within_insn (insn, original_regno, new_reg, false);
6639 1169790 : lra_update_insn_regno_info (insn);
6640 1169790 : if (! def_p)
6641 : /* We now have a new usage insn for original regno. */
6642 624904 : setup_next_usage_insn (original_regno, new_insns, reloads_num, false);
6643 1169790 : if (lra_dump_file != NULL)
6644 1 : fprintf (lra_dump_file, " Original reg change %d->%d (bb%d):\n",
6645 1 : original_regno, REGNO (new_reg), BLOCK_FOR_INSN (insn)->index);
6646 1169790 : lra_reg_info[REGNO (new_reg)].restore_rtx = regno_reg_rtx[original_regno];
6647 1169790 : bitmap_set_bit (&check_only_regs, REGNO (new_reg));
6648 1169790 : bitmap_set_bit (&check_only_regs, original_regno);
6649 1169790 : bitmap_set_bit (&lra_inheritance_pseudos, REGNO (new_reg));
6650 1169790 : if (def_p)
6651 544886 : lra_process_new_insns (insn, NULL, new_insns,
6652 : "Add original<-inheritance");
6653 : else
6654 624904 : lra_process_new_insns (insn, new_insns, NULL,
6655 : "Add inheritance<-original");
6656 2655128 : while (next_usage_insns != NULL_RTX)
6657 : {
6658 1485338 : if (GET_CODE (next_usage_insns) != INSN_LIST)
6659 : {
6660 1169790 : usage_insn = next_usage_insns;
6661 1169790 : lra_assert (NONDEBUG_INSN_P (usage_insn));
6662 : next_usage_insns = NULL;
6663 : }
6664 : else
6665 : {
6666 315548 : usage_insn = XEXP (next_usage_insns, 0);
6667 315548 : lra_assert (DEBUG_INSN_P (usage_insn));
6668 315548 : next_usage_insns = XEXP (next_usage_insns, 1);
6669 : }
6670 1485338 : lra_substitute_pseudo (&usage_insn, original_regno, new_reg, false,
6671 1485338 : DEBUG_INSN_P (usage_insn));
6672 1485338 : lra_update_insn_regno_info (as_a <rtx_insn *> (usage_insn));
6673 1485338 : if (lra_dump_file != NULL)
6674 : {
6675 1 : basic_block bb = BLOCK_FOR_INSN (usage_insn);
6676 1 : fprintf (lra_dump_file,
6677 : " Inheritance reuse change %d->%d (bb%d):\n",
6678 : original_regno, REGNO (new_reg),
6679 : bb ? bb->index : -1);
6680 1 : dump_insn_slim (lra_dump_file, as_a <rtx_insn *> (usage_insn));
6681 : }
6682 : }
6683 1169790 : if (lra_dump_file != NULL)
6684 1 : fprintf (lra_dump_file,
6685 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6686 : return true;
6687 : }
6688 :
6689 : /* Return true if we need a caller save/restore for pseudo REGNO which
6690 : was assigned to a hard register. */
6691 : static inline bool
6692 116296699 : need_for_call_save_p (int regno)
6693 : {
6694 116296699 : lra_assert (regno >= FIRST_PSEUDO_REGISTER && reg_renumber[regno] >= 0);
6695 116296699 : if (usage_insns[regno].calls_num < calls_num)
6696 : {
6697 : unsigned int abis = 0;
6698 123235619 : for (unsigned int i = 0; i < NUM_ABI_IDS; ++i)
6699 113755956 : if (last_call_for_abi[i] > usage_insns[regno].calls_num)
6700 9528982 : abis |= 1 << i;
6701 9479663 : gcc_assert (abis);
6702 9479663 : if (call_clobbered_in_region_p (abis, full_and_partial_call_clobbers,
6703 9479663 : PSEUDO_REGNO_MODE (regno),
6704 : reg_renumber[regno]))
6705 : return true;
6706 : }
6707 : return false;
6708 : }
6709 :
6710 : /* Global registers occurring in the current EBB. */
6711 : static bitmap_head ebb_global_regs;
6712 :
6713 : /* Return true if we need a split for hard register REGNO or pseudo
6714 : REGNO which was assigned to a hard register.
6715 : POTENTIAL_RELOAD_HARD_REGS contains hard registers which might be
6716 : used for reloads since the EBB end. It is an approximation of the
6717 : used hard registers in the split range. The exact value would
6718 : require expensive calculations. If we were aggressive with
6719 : splitting because of the approximation, the split pseudo will save
6720 : the same hard register assignment and will be removed in the undo
6721 : pass. We still need the approximation because too aggressive
6722 : splitting would result in too inaccurate cost calculation in the
6723 : assignment pass because of too many generated moves which will be
6724 : probably removed in the undo pass. */
6725 : static inline bool
6726 247169130 : need_for_split_p (HARD_REG_SET potential_reload_hard_regs, int regno)
6727 : {
6728 247169130 : int hard_regno = regno < FIRST_PSEUDO_REGISTER ? regno : reg_renumber[regno];
6729 :
6730 247169130 : lra_assert (hard_regno >= 0);
6731 247169130 : return ((TEST_HARD_REG_BIT (potential_reload_hard_regs, hard_regno)
6732 : /* Don't split eliminable hard registers, otherwise we can
6733 : split hard registers like hard frame pointer, which
6734 : lives on BB start/end according to DF-infrastructure,
6735 : when there is a pseudo assigned to the register and
6736 : living in the same BB. */
6737 673240 : && (regno >= FIRST_PSEUDO_REGISTER
6738 44554 : || ! TEST_HARD_REG_BIT (eliminable_regset, hard_regno))
6739 643658 : && ! TEST_HARD_REG_BIT (lra_no_alloc_regs, hard_regno)
6740 : /* Don't split call clobbered hard regs living through
6741 : calls, otherwise we might have a check problem in the
6742 : assign sub-pass as in the most cases (exception is a
6743 : situation when check_and_force_assignment_correctness_p value is
6744 : true) the assign pass assumes that all pseudos living
6745 : through calls are assigned to call saved hard regs. */
6746 629730 : && (regno >= FIRST_PSEUDO_REGISTER
6747 1044 : || !TEST_HARD_REG_BIT (full_and_partial_call_clobbers, regno))
6748 : /* We need at least 2 reloads to make pseudo splitting
6749 : profitable. We should provide hard regno splitting in
6750 : any case to solve 1st insn scheduling problem when
6751 : moving hard register definition up might result in
6752 : impossibility to find hard register for reload pseudo of
6753 : small register class. */
6754 1259408 : && (usage_insns[regno].reloads_num
6755 1258390 : + (regno < FIRST_PSEUDO_REGISTER ? 0 : 3) < reloads_num)
6756 2922 : && (regno < FIRST_PSEUDO_REGISTER
6757 : /* For short living pseudos, spilling + inheritance can
6758 : be considered a substitution for splitting.
6759 : Therefore we do not splitting for local pseudos. It
6760 : decreases also aggressiveness of splitting. The
6761 : minimal number of references is chosen taking into
6762 : account that for 2 references splitting has no sense
6763 : as we can just spill the pseudo. */
6764 : || (regno >= FIRST_PSEUDO_REGISTER
6765 2875 : && lra_reg_info[regno].nrefs > 3
6766 2653 : && bitmap_bit_p (&ebb_global_regs, regno))))
6767 247840878 : || (regno >= FIRST_PSEUDO_REGISTER && need_for_call_save_p (regno)));
6768 : }
6769 :
6770 : /* Return class for the split pseudo created from original pseudo with
6771 : ALLOCNO_CLASS and MODE which got a hard register HARD_REGNO. We
6772 : choose subclass of ALLOCNO_CLASS which contains HARD_REGNO and
6773 : results in no secondary memory movements. */
6774 : static enum reg_class
6775 1652 : choose_split_class (enum reg_class allocno_class,
6776 : int hard_regno ATTRIBUTE_UNUSED,
6777 : machine_mode mode ATTRIBUTE_UNUSED)
6778 : {
6779 1652 : int i;
6780 1652 : enum reg_class cl, best_cl = NO_REGS;
6781 1652 : enum reg_class hard_reg_class ATTRIBUTE_UNUSED
6782 : = REGNO_REG_CLASS (hard_regno);
6783 :
6784 1652 : if (! targetm.secondary_memory_needed (mode, allocno_class, allocno_class)
6785 1652 : && TEST_HARD_REG_BIT (reg_class_contents[allocno_class], hard_regno))
6786 : return allocno_class;
6787 0 : for (i = 0;
6788 0 : (cl = reg_class_subclasses[allocno_class][i]) != LIM_REG_CLASSES;
6789 : i++)
6790 0 : if (! targetm.secondary_memory_needed (mode, cl, hard_reg_class)
6791 0 : && ! targetm.secondary_memory_needed (mode, hard_reg_class, cl)
6792 0 : && TEST_HARD_REG_BIT (reg_class_contents[cl], hard_regno)
6793 0 : && (best_cl == NO_REGS
6794 0 : || ira_class_hard_regs_num[best_cl] < ira_class_hard_regs_num[cl]))
6795 : best_cl = cl;
6796 : return best_cl;
6797 : }
6798 :
6799 : /* Copy any equivalence information from ORIGINAL_REGNO to NEW_REGNO. It only
6800 : makes sense to call this function if NEW_REGNO is always equal to
6801 : ORIGINAL_REGNO. Set up defined_p flag when caller_save_p flag is set up and
6802 : CALL_SAVE_P is true. */
6803 :
6804 : static void
6805 668668 : lra_copy_reg_equiv (unsigned int new_regno, unsigned int original_regno,
6806 : bool call_save_p)
6807 : {
6808 668668 : if (!ira_reg_equiv[original_regno].defined_p
6809 601927 : && !(call_save_p && ira_reg_equiv[original_regno].caller_save_p))
6810 : return;
6811 :
6812 66918 : ira_expand_reg_equiv ();
6813 66918 : ira_reg_equiv[new_regno].defined_p = true;
6814 66918 : if (ira_reg_equiv[original_regno].memory)
6815 33344 : ira_reg_equiv[new_regno].memory
6816 33344 : = copy_rtx (ira_reg_equiv[original_regno].memory);
6817 66918 : if (ira_reg_equiv[original_regno].constant)
6818 26961 : ira_reg_equiv[new_regno].constant
6819 26961 : = copy_rtx (ira_reg_equiv[original_regno].constant);
6820 66918 : if (ira_reg_equiv[original_regno].invariant)
6821 6613 : ira_reg_equiv[new_regno].invariant
6822 6613 : = copy_rtx (ira_reg_equiv[original_regno].invariant);
6823 : }
6824 :
6825 : /* Do split transformations for insn INSN, which defines or uses
6826 : ORIGINAL_REGNO. NEXT_USAGE_INSNS specifies which instruction in
6827 : the EBB next uses ORIGINAL_REGNO; it has the same form as the
6828 : "insns" field of usage_insns. If TO is not NULL, we don't use
6829 : usage_insns, we put restore insns after TO insn. It is a case when
6830 : we call it from lra_split_hard_reg_for, outside the inheritance
6831 : pass.
6832 :
6833 : The transformations look like:
6834 :
6835 : p <- ... p <- ...
6836 : ... s <- p (new insn -- save)
6837 : ... =>
6838 : ... p <- s (new insn -- restore)
6839 : <- ... p ... <- ... p ...
6840 : or
6841 : <- ... p ... <- ... p ...
6842 : ... s <- p (new insn -- save)
6843 : ... =>
6844 : ... p <- s (new insn -- restore)
6845 : <- ... p ... <- ... p ...
6846 :
6847 : where p is an original pseudo got a hard register or a hard
6848 : register and s is a new split pseudo. The save is put before INSN
6849 : if BEFORE_P is true. Return true if we succeed in such
6850 : transformation. */
6851 : static bool
6852 670438 : split_reg (bool before_p, int original_regno, rtx_insn *insn,
6853 : rtx next_usage_insns, rtx_insn *to)
6854 : {
6855 670438 : enum reg_class rclass;
6856 670438 : rtx original_reg;
6857 670438 : int hard_regno, nregs;
6858 670438 : rtx new_reg, usage_insn;
6859 670438 : rtx_insn *restore, *save;
6860 670438 : bool after_p;
6861 670438 : bool call_save_p;
6862 670438 : machine_mode mode;
6863 :
6864 670438 : if (original_regno < FIRST_PSEUDO_REGISTER)
6865 : {
6866 208 : rclass = ira_allocno_class_translate[REGNO_REG_CLASS (original_regno)];
6867 208 : hard_regno = original_regno;
6868 208 : call_save_p = false;
6869 208 : nregs = 1;
6870 208 : mode = lra_reg_info[hard_regno].biggest_mode;
6871 208 : machine_mode reg_rtx_mode = GET_MODE (regno_reg_rtx[hard_regno]);
6872 : /* A reg can have a biggest_mode of VOIDmode if it was only ever seen as
6873 : part of a multi-word register. In that case, just use the reg_rtx
6874 : mode. Do the same also if the biggest mode was larger than a register
6875 : or we can not compare the modes. Otherwise, limit the size to that of
6876 : the biggest access in the function or to the natural mode at least. */
6877 208 : if (mode == VOIDmode
6878 208 : || !ordered_p (GET_MODE_PRECISION (mode),
6879 208 : GET_MODE_PRECISION (reg_rtx_mode))
6880 208 : || paradoxical_subreg_p (mode, reg_rtx_mode)
6881 415 : || maybe_gt (GET_MODE_PRECISION (reg_rtx_mode), GET_MODE_PRECISION (mode)))
6882 : {
6883 : original_reg = regno_reg_rtx[hard_regno];
6884 : mode = reg_rtx_mode;
6885 : }
6886 : else
6887 191 : original_reg = gen_rtx_REG (mode, hard_regno);
6888 : }
6889 : else
6890 : {
6891 670230 : mode = PSEUDO_REGNO_MODE (original_regno);
6892 670230 : hard_regno = reg_renumber[original_regno];
6893 670230 : nregs = hard_regno_nregs (hard_regno, mode);
6894 670230 : rclass = lra_get_allocno_class (original_regno);
6895 670230 : original_reg = regno_reg_rtx[original_regno];
6896 670230 : call_save_p = need_for_call_save_p (original_regno);
6897 : }
6898 670438 : lra_assert (hard_regno >= 0);
6899 670438 : if (lra_dump_file != NULL)
6900 0 : fprintf (lra_dump_file,
6901 : " ((((((((((((((((((((((((((((((((((((((((((((((((\n");
6902 :
6903 670438 : if (call_save_p)
6904 : {
6905 668786 : mode = HARD_REGNO_CALLER_SAVE_MODE (hard_regno,
6906 : hard_regno_nregs (hard_regno, mode),
6907 : mode);
6908 668786 : new_reg = lra_create_new_reg (mode, NULL_RTX, NO_REGS, NULL, "save");
6909 : }
6910 : else
6911 : {
6912 1652 : rclass = choose_split_class (rclass, hard_regno, mode);
6913 1652 : if (rclass == NO_REGS)
6914 : {
6915 0 : if (lra_dump_file != NULL)
6916 : {
6917 0 : fprintf (lra_dump_file,
6918 : " Rejecting split of %d(%s): "
6919 : "no good reg class for %d(%s)\n",
6920 : original_regno,
6921 0 : reg_class_names[lra_get_allocno_class (original_regno)],
6922 : hard_regno,
6923 0 : reg_class_names[REGNO_REG_CLASS (hard_regno)]);
6924 0 : fprintf
6925 0 : (lra_dump_file,
6926 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
6927 : }
6928 : return false;
6929 : }
6930 : /* Split_if_necessary can split hard registers used as part of a
6931 : multi-register mode but splits each register individually. The
6932 : mode used for each independent register may not be supported
6933 : so reject the split. Splitting the wider mode should theoretically
6934 : be possible but is not implemented. */
6935 1652 : if (!targetm.hard_regno_mode_ok (hard_regno, mode))
6936 : {
6937 0 : if (lra_dump_file != NULL)
6938 : {
6939 0 : fprintf (lra_dump_file,
6940 : " Rejecting split of %d(%s): unsuitable mode %s\n",
6941 : original_regno,
6942 0 : reg_class_names[lra_get_allocno_class (original_regno)],
6943 0 : GET_MODE_NAME (mode));
6944 0 : fprintf
6945 0 : (lra_dump_file,
6946 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
6947 : }
6948 : return false;
6949 : }
6950 1652 : new_reg = lra_create_new_reg (mode, original_reg, rclass, NULL, "split");
6951 1652 : reg_renumber[REGNO (new_reg)] = hard_regno;
6952 : }
6953 670438 : int new_regno = REGNO (new_reg);
6954 670438 : save = emit_spill_move (true, new_reg, original_reg);
6955 670438 : if (NEXT_INSN (save) != NULL_RTX && !call_save_p)
6956 : {
6957 0 : if (lra_dump_file != NULL)
6958 : {
6959 0 : fprintf
6960 0 : (lra_dump_file,
6961 : " Rejecting split %d->%d resulting in > 2 save insns:\n",
6962 : original_regno, new_regno);
6963 0 : dump_rtl_slim (lra_dump_file, save, NULL, -1, 0);
6964 0 : fprintf (lra_dump_file,
6965 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
6966 : }
6967 : return false;
6968 : }
6969 670438 : restore = emit_spill_move (false, new_reg, original_reg);
6970 670438 : if (NEXT_INSN (restore) != NULL_RTX && !call_save_p)
6971 : {
6972 0 : if (lra_dump_file != NULL)
6973 : {
6974 0 : fprintf (lra_dump_file,
6975 : " Rejecting split %d->%d "
6976 : "resulting in > 2 restore insns:\n",
6977 : original_regno, new_regno);
6978 0 : dump_rtl_slim (lra_dump_file, restore, NULL, -1, 0);
6979 0 : fprintf (lra_dump_file,
6980 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
6981 : }
6982 : return false;
6983 : }
6984 : /* Transfer equivalence information to the spill register, so that
6985 : if we fail to allocate the spill register, we have the option of
6986 : rematerializing the original value instead of spilling to the stack. */
6987 670438 : if (!HARD_REGISTER_NUM_P (original_regno)
6988 670230 : && mode == PSEUDO_REGNO_MODE (original_regno))
6989 668668 : lra_copy_reg_equiv (new_regno, original_regno, call_save_p);
6990 670438 : lra_reg_info[new_regno].restore_rtx = regno_reg_rtx[original_regno];
6991 670438 : bitmap_set_bit (&lra_split_regs, new_regno);
6992 670438 : if (to != NULL)
6993 : {
6994 161 : lra_assert (next_usage_insns == NULL);
6995 161 : usage_insn = to;
6996 161 : after_p = true;
6997 : }
6998 : else
6999 : {
7000 : /* We need check_only_regs only inside the inheritance pass. */
7001 670277 : bitmap_set_bit (&check_only_regs, new_regno);
7002 670277 : bitmap_set_bit (&check_only_regs, original_regno);
7003 670277 : after_p = usage_insns[original_regno].after_p;
7004 784857 : for (;;)
7005 : {
7006 784857 : if (GET_CODE (next_usage_insns) != INSN_LIST)
7007 : {
7008 670277 : usage_insn = next_usage_insns;
7009 670277 : break;
7010 : }
7011 114580 : usage_insn = XEXP (next_usage_insns, 0);
7012 114580 : lra_assert (DEBUG_INSN_P (usage_insn));
7013 114580 : next_usage_insns = XEXP (next_usage_insns, 1);
7014 114580 : lra_substitute_pseudo (&usage_insn, original_regno, new_reg, false,
7015 : true);
7016 114580 : lra_update_insn_regno_info (as_a <rtx_insn *> (usage_insn));
7017 114580 : if (lra_dump_file != NULL)
7018 : {
7019 0 : fprintf (lra_dump_file, " Split reuse change %d->%d:\n",
7020 : original_regno, new_regno);
7021 0 : dump_insn_slim (lra_dump_file, as_a <rtx_insn *> (usage_insn));
7022 : }
7023 : }
7024 : }
7025 670438 : lra_assert (NOTE_P (usage_insn) || NONDEBUG_INSN_P (usage_insn));
7026 670438 : lra_assert (usage_insn != insn || (after_p && before_p));
7027 1132512 : lra_process_new_insns (as_a <rtx_insn *> (usage_insn),
7028 : after_p ? NULL : restore,
7029 : after_p ? restore : NULL,
7030 : call_save_p ? "Add reg<-save" : "Add reg<-split");
7031 670438 : if (call_save_p
7032 668786 : && first_call_insn != NULL
7033 1339224 : && BLOCK_FOR_INSN (first_call_insn) != BLOCK_FOR_INSN (insn))
7034 : /* PR116028: If original_regno is a pseudo that has been assigned a
7035 : callee-saved hard register, then emit the spill insn before the call
7036 : insn 'first_call_insn' instead of adjacent to 'insn'. If 'insn'
7037 : and 'first_call_insn' belong to the same EBB but to two separate
7038 : BBs, and if 'insn' is present in the entry BB, then generating the
7039 : spill insn in the entry BB can prevent shrink wrap from happening.
7040 : This is because the spill insn references the stack pointer and
7041 : hence the prolog gets generated in the entry BB itself. It is
7042 : also more efficient to generate the spill before
7043 : 'first_call_insn' as the spill now occurs only in the path
7044 : containing the call. */
7045 25207 : lra_process_new_insns (first_call_insn, save, NULL, "Add save<-reg");
7046 : else
7047 1291585 : lra_process_new_insns (insn, before_p ? save : NULL,
7048 : before_p ? NULL : save,
7049 : call_save_p ? "Add save<-reg" : "Add split<-reg");
7050 670438 : if (nregs > 1 || original_regno < FIRST_PSEUDO_REGISTER)
7051 : /* If we are trying to split multi-register. We should check
7052 : conflicts on the next assignment sub-pass. IRA can allocate on
7053 : sub-register levels, LRA do this on pseudos level right now and
7054 : this discrepancy may create allocation conflicts after
7055 : splitting.
7056 :
7057 : If we are trying to split hard register we should also check conflicts
7058 : as such splitting can create artificial conflict of the hard register
7059 : with another pseudo because of simplified conflict calculation in
7060 : LRA. */
7061 9280 : check_and_force_assignment_correctness_p = true;
7062 670438 : if (lra_dump_file != NULL)
7063 0 : fprintf (lra_dump_file,
7064 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
7065 : return true;
7066 : }
7067 :
7068 : /* Split a hard reg for reload pseudo REGNO having RCLASS and living
7069 : in the range [FROM, TO]. Return true if did a split. Otherwise,
7070 : return false. */
7071 : bool
7072 991 : spill_hard_reg_in_range (int regno, enum reg_class rclass, rtx_insn *from, rtx_insn *to)
7073 : {
7074 991 : int i, hard_regno;
7075 991 : int rclass_size;
7076 991 : rtx_insn *insn;
7077 991 : unsigned int uid;
7078 991 : bitmap_iterator bi;
7079 991 : HARD_REG_SET ignore;
7080 :
7081 991 : lra_assert (from != NULL && to != NULL);
7082 991 : ignore = lra_no_alloc_regs;
7083 2768 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi)
7084 : {
7085 1777 : lra_insn_recog_data_t id = lra_insn_recog_data[uid];
7086 1777 : struct lra_static_insn_data *static_id = id->insn_static_data;
7087 1777 : struct lra_insn_reg *reg;
7088 :
7089 6357 : for (reg = id->regs; reg != NULL; reg = reg->next)
7090 4580 : if (reg->regno < FIRST_PSEUDO_REGISTER)
7091 295 : SET_HARD_REG_BIT (ignore, reg->regno);
7092 2850 : for (reg = static_id->hard_regs; reg != NULL; reg = reg->next)
7093 1073 : SET_HARD_REG_BIT (ignore, reg->regno);
7094 : }
7095 991 : rclass_size = ira_class_hard_regs_num[rclass];
7096 3165 : for (i = 0; i < rclass_size; i++)
7097 : {
7098 2335 : hard_regno = ira_class_hard_regs[rclass][i];
7099 2335 : if (! TEST_HARD_REG_BIT (lra_reg_info[regno].conflict_hard_regs, hard_regno)
7100 2335 : || TEST_HARD_REG_BIT (ignore, hard_regno))
7101 2168 : continue;
7102 482 : for (insn = from; insn != NEXT_INSN (to); insn = NEXT_INSN (insn))
7103 : {
7104 321 : struct lra_static_insn_data *static_id;
7105 321 : struct lra_insn_reg *reg;
7106 :
7107 321 : if (!INSN_P (insn))
7108 0 : continue;
7109 321 : if (bitmap_bit_p (&lra_reg_info[hard_regno].insn_bitmap,
7110 321 : INSN_UID (insn)))
7111 : break;
7112 315 : static_id = lra_get_insn_recog_data (insn)->insn_static_data;
7113 369 : for (reg = static_id->hard_regs; reg != NULL; reg = reg->next)
7114 54 : if (reg->regno == hard_regno)
7115 : break;
7116 : if (reg != NULL)
7117 : break;
7118 : }
7119 167 : if (insn != NEXT_INSN (to))
7120 6 : continue;
7121 161 : if (split_reg (true, hard_regno, from, NULL, to))
7122 : return true;
7123 : }
7124 : return false;
7125 : }
7126 :
7127 : /* Recognize that we need a split transformation for insn INSN, which
7128 : defines or uses REGNO in its insn biggest MODE (we use it only if
7129 : REGNO is a hard register). POTENTIAL_RELOAD_HARD_REGS contains
7130 : hard registers which might be used for reloads since the EBB end.
7131 : Put the save before INSN if BEFORE_P is true. MAX_UID is maximla
7132 : uid before starting INSN processing. Return true if we succeed in
7133 : such transformation. */
7134 : static bool
7135 203673097 : split_if_necessary (int regno, machine_mode mode,
7136 : HARD_REG_SET potential_reload_hard_regs,
7137 : bool before_p, rtx_insn *insn, int max_uid)
7138 : {
7139 203673097 : bool res = false;
7140 203673097 : int i, nregs = 1;
7141 203673097 : rtx next_usage_insns;
7142 :
7143 203673097 : if (regno < FIRST_PSEUDO_REGISTER)
7144 95808454 : nregs = hard_regno_nregs (regno, mode);
7145 407706658 : for (i = 0; i < nregs; i++)
7146 204033561 : if (usage_insns[regno + i].check == curr_usage_insns_check
7147 136124232 : && (next_usage_insns = usage_insns[regno + i].insns) != NULL_RTX
7148 : /* To avoid processing the register twice or more. */
7149 136124232 : && ((GET_CODE (next_usage_insns) != INSN_LIST
7150 131645568 : && INSN_UID (next_usage_insns) < max_uid)
7151 4478664 : || (GET_CODE (next_usage_insns) == INSN_LIST
7152 4478664 : && (INSN_UID (XEXP (next_usage_insns, 0)) < max_uid)))
7153 136124232 : && need_for_split_p (potential_reload_hard_regs, regno + i)
7154 204317082 : && split_reg (before_p, regno + i, insn, next_usage_insns, NULL))
7155 : res = true;
7156 203673097 : return res;
7157 : }
7158 :
7159 : /* Return TRUE if rtx X is considered as an invariant for
7160 : inheritance. */
7161 : static bool
7162 12300118 : invariant_p (const_rtx x)
7163 : {
7164 12300118 : machine_mode mode;
7165 12300118 : const char *fmt;
7166 12300118 : enum rtx_code code;
7167 12300118 : int i, j;
7168 :
7169 12300118 : if (side_effects_p (x))
7170 : return false;
7171 :
7172 12274152 : code = GET_CODE (x);
7173 12274152 : mode = GET_MODE (x);
7174 12274152 : if (code == SUBREG)
7175 : {
7176 498781 : x = SUBREG_REG (x);
7177 498781 : code = GET_CODE (x);
7178 498781 : mode = wider_subreg_mode (mode, GET_MODE (x));
7179 : }
7180 :
7181 12274152 : if (MEM_P (x))
7182 : return false;
7183 :
7184 10486836 : if (REG_P (x))
7185 : {
7186 3723925 : int i, nregs, regno = REGNO (x);
7187 :
7188 3723925 : if (regno >= FIRST_PSEUDO_REGISTER || regno == STACK_POINTER_REGNUM
7189 1027824 : || TEST_HARD_REG_BIT (eliminable_regset, regno)
7190 3739902 : || GET_MODE_CLASS (GET_MODE (x)) == MODE_CC)
7191 : return false;
7192 2 : nregs = hard_regno_nregs (regno, mode);
7193 2 : for (i = 0; i < nregs; i++)
7194 2 : if (! fixed_regs[regno + i]
7195 : /* A hard register may be clobbered in the current insn
7196 : but we can ignore this case because if the hard
7197 : register is used it should be set somewhere after the
7198 : clobber. */
7199 2 : || bitmap_bit_p (&invalid_invariant_regs, regno + i))
7200 : return false;
7201 : }
7202 6762911 : fmt = GET_RTX_FORMAT (code);
7203 11933461 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
7204 : {
7205 9004181 : if (fmt[i] == 'e')
7206 : {
7207 5901080 : if (! invariant_p (XEXP (x, i)))
7208 : return false;
7209 : }
7210 3103101 : else if (fmt[i] == 'E')
7211 : {
7212 676934 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
7213 560638 : if (! invariant_p (XVECEXP (x, i, j)))
7214 : return false;
7215 : }
7216 : }
7217 : return true;
7218 : }
7219 :
7220 : /* We have 'dest_reg <- invariant'. Let us try to make an invariant
7221 : inheritance transformation (using dest_reg instead invariant in a
7222 : subsequent insn). */
7223 : static bool
7224 198463 : process_invariant_for_inheritance (rtx dst_reg, rtx invariant_rtx)
7225 : {
7226 198463 : invariant_ptr_t invariant_ptr;
7227 198463 : rtx_insn *insn, *new_insns;
7228 198463 : rtx insn_set, insn_reg, new_reg;
7229 198463 : int insn_regno;
7230 198463 : bool succ_p = false;
7231 198463 : int dst_regno = REGNO (dst_reg);
7232 198463 : machine_mode dst_mode = GET_MODE (dst_reg);
7233 198463 : enum reg_class cl = lra_get_allocno_class (dst_regno), insn_reg_cl;
7234 :
7235 198463 : invariant_ptr = insert_invariant (invariant_rtx);
7236 198463 : if ((insn = invariant_ptr->insn) != NULL_RTX)
7237 : {
7238 : /* We have a subsequent insn using the invariant. */
7239 27020 : insn_set = single_set (insn);
7240 27020 : lra_assert (insn_set != NULL);
7241 27020 : insn_reg = SET_DEST (insn_set);
7242 27020 : lra_assert (REG_P (insn_reg));
7243 27020 : insn_regno = REGNO (insn_reg);
7244 27020 : insn_reg_cl = lra_get_allocno_class (insn_regno);
7245 :
7246 27020 : if (dst_mode == GET_MODE (insn_reg)
7247 : /* We should consider only result move reg insns which are
7248 : cheap. */
7249 26869 : && targetm.register_move_cost (dst_mode, cl, insn_reg_cl) == 2
7250 53300 : && targetm.register_move_cost (dst_mode, cl, cl) == 2)
7251 : {
7252 26280 : if (lra_dump_file != NULL)
7253 0 : fprintf (lra_dump_file,
7254 : " [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[\n");
7255 26280 : new_reg = lra_create_new_reg (dst_mode, dst_reg, cl, NULL,
7256 : "invariant inheritance");
7257 26280 : bitmap_set_bit (&lra_inheritance_pseudos, REGNO (new_reg));
7258 26280 : bitmap_set_bit (&check_only_regs, REGNO (new_reg));
7259 26280 : lra_reg_info[REGNO (new_reg)].restore_rtx = PATTERN (insn);
7260 26280 : start_sequence ();
7261 26280 : lra_emit_move (new_reg, dst_reg);
7262 26280 : new_insns = end_sequence ();
7263 26280 : lra_process_new_insns (curr_insn, NULL, new_insns,
7264 : "Add invariant inheritance<-original");
7265 26280 : start_sequence ();
7266 26280 : lra_emit_move (SET_DEST (insn_set), new_reg);
7267 26280 : new_insns = end_sequence ();
7268 26280 : lra_process_new_insns (insn, NULL, new_insns,
7269 : "Changing reload<-inheritance");
7270 26280 : lra_set_insn_deleted (insn);
7271 26280 : succ_p = true;
7272 26280 : if (lra_dump_file != NULL)
7273 : {
7274 0 : fprintf (lra_dump_file,
7275 : " Invariant inheritance reuse change %d (bb%d):\n",
7276 0 : REGNO (new_reg), BLOCK_FOR_INSN (insn)->index);
7277 0 : dump_insn_slim (lra_dump_file, insn);
7278 0 : fprintf (lra_dump_file,
7279 : " ]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]\n");
7280 : }
7281 : }
7282 : }
7283 198463 : invariant_ptr->insn = curr_insn;
7284 198463 : return succ_p;
7285 : }
7286 :
7287 : /* Check only registers living at the current program point in the
7288 : current EBB. */
7289 : static bitmap_head live_regs;
7290 :
7291 : /* Update live info in EBB given by its HEAD and TAIL insns after
7292 : inheritance/split transformation. The function removes dead moves
7293 : too. */
7294 : static void
7295 750518 : update_ebb_live_info (rtx_insn *head, rtx_insn *tail)
7296 : {
7297 750518 : unsigned int j;
7298 750518 : int i, regno;
7299 750518 : bool live_p;
7300 750518 : rtx_insn *prev_insn;
7301 750518 : rtx set;
7302 750518 : bool remove_p;
7303 750518 : basic_block last_bb, prev_bb, curr_bb;
7304 750518 : bitmap_iterator bi;
7305 750518 : struct lra_insn_reg *reg;
7306 750518 : edge e;
7307 750518 : edge_iterator ei;
7308 :
7309 750518 : last_bb = BLOCK_FOR_INSN (tail);
7310 750518 : prev_bb = NULL;
7311 750518 : for (curr_insn = tail;
7312 39253652 : curr_insn != PREV_INSN (head);
7313 38503134 : curr_insn = prev_insn)
7314 : {
7315 38503134 : prev_insn = PREV_INSN (curr_insn);
7316 : /* We need to process empty blocks too. They contain
7317 : NOTE_INSN_BASIC_BLOCK referring for the basic block. */
7318 38503134 : if (NOTE_P (curr_insn) && NOTE_KIND (curr_insn) != NOTE_INSN_BASIC_BLOCK)
7319 1427600 : continue;
7320 37075534 : curr_bb = BLOCK_FOR_INSN (curr_insn);
7321 37075534 : if (curr_bb != prev_bb)
7322 : {
7323 1500408 : if (prev_bb != NULL)
7324 : {
7325 : /* Update df_get_live_in (prev_bb): */
7326 9686906 : EXECUTE_IF_SET_IN_BITMAP (&check_only_regs, 0, j, bi)
7327 8937016 : if (bitmap_bit_p (&live_regs, j))
7328 1256361 : bitmap_set_bit (df_get_live_in (prev_bb), j);
7329 : else
7330 7680655 : bitmap_clear_bit (df_get_live_in (prev_bb), j);
7331 : }
7332 1500408 : if (curr_bb != last_bb)
7333 : {
7334 : /* Update df_get_live_out (curr_bb): */
7335 9686906 : EXECUTE_IF_SET_IN_BITMAP (&check_only_regs, 0, j, bi)
7336 : {
7337 8937016 : live_p = bitmap_bit_p (&live_regs, j);
7338 8937016 : if (! live_p)
7339 22946295 : FOR_EACH_EDGE (e, ei, curr_bb->succs)
7340 15321634 : if (bitmap_bit_p (df_get_live_in (e->dest), j))
7341 : {
7342 : live_p = true;
7343 : break;
7344 : }
7345 7680655 : if (live_p)
7346 1312355 : bitmap_set_bit (df_get_live_out (curr_bb), j);
7347 : else
7348 7624661 : bitmap_clear_bit (df_get_live_out (curr_bb), j);
7349 : }
7350 : }
7351 1500408 : prev_bb = curr_bb;
7352 1500408 : bitmap_and (&live_regs, &check_only_regs, df_get_live_out (curr_bb));
7353 : }
7354 37075534 : if (! NONDEBUG_INSN_P (curr_insn))
7355 14541359 : continue;
7356 22534175 : curr_id = lra_get_insn_recog_data (curr_insn);
7357 22534175 : curr_static_id = curr_id->insn_static_data;
7358 22534175 : remove_p = false;
7359 22534175 : if ((set = single_set (curr_insn)) != NULL_RTX
7360 21826721 : && REG_P (SET_DEST (set))
7361 17433707 : && (regno = REGNO (SET_DEST (set))) >= FIRST_PSEUDO_REGISTER
7362 12933817 : && SET_DEST (set) != pic_offset_table_rtx
7363 12927000 : && bitmap_bit_p (&check_only_regs, regno)
7364 25806940 : && ! bitmap_bit_p (&live_regs, regno))
7365 : remove_p = true;
7366 : /* See which defined values die here. */
7367 62196228 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7368 39662053 : if (reg->type == OP_OUT && ! reg->subreg_p)
7369 15526856 : bitmap_clear_bit (&live_regs, reg->regno);
7370 26779349 : for (reg = curr_static_id->hard_regs; reg != NULL; reg = reg->next)
7371 4245174 : if (reg->type == OP_OUT && ! reg->subreg_p)
7372 3218286 : bitmap_clear_bit (&live_regs, reg->regno);
7373 22534175 : if (curr_id->arg_hard_regs != NULL)
7374 : /* Make clobbered argument hard registers die. */
7375 3278431 : for (i = 0; (regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7376 2290354 : if (regno >= FIRST_PSEUDO_REGISTER)
7377 0 : bitmap_clear_bit (&live_regs, regno - FIRST_PSEUDO_REGISTER);
7378 : /* Mark each used value as live. */
7379 62196228 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7380 39662053 : if (reg->type != OP_OUT
7381 39662053 : && bitmap_bit_p (&check_only_regs, reg->regno))
7382 4604594 : bitmap_set_bit (&live_regs, reg->regno);
7383 26779349 : for (reg = curr_static_id->hard_regs; reg != NULL; reg = reg->next)
7384 4245174 : if (reg->type != OP_OUT
7385 4245174 : && bitmap_bit_p (&check_only_regs, reg->regno))
7386 0 : bitmap_set_bit (&live_regs, reg->regno);
7387 22534175 : if (curr_id->arg_hard_regs != NULL)
7388 : /* Make used argument hard registers live. */
7389 3278431 : for (i = 0; (regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7390 2290354 : if (regno < FIRST_PSEUDO_REGISTER
7391 2290354 : && bitmap_bit_p (&check_only_regs, regno))
7392 0 : bitmap_set_bit (&live_regs, regno);
7393 : /* It is quite important to remove dead move insns because it
7394 : means removing dead store. We don't need to process them for
7395 : constraints. */
7396 22534175 : if (remove_p)
7397 : {
7398 288684 : if (lra_dump_file != NULL)
7399 : {
7400 1 : fprintf (lra_dump_file, " Removing dead insn:\n ");
7401 1 : dump_insn_slim (lra_dump_file, curr_insn);
7402 : }
7403 288684 : lra_set_insn_deleted (curr_insn);
7404 : }
7405 : }
7406 750518 : }
7407 :
7408 : /* The structure describes info to do an inheritance for the current
7409 : insn. We need to collect such info first before doing the
7410 : transformations because the transformations change the insn
7411 : internal representation. */
7412 : struct to_inherit
7413 : {
7414 : /* Original regno. */
7415 : int regno;
7416 : /* Subsequent insns which can inherit original reg value. */
7417 : rtx insns;
7418 : };
7419 :
7420 : /* Array containing all info for doing inheritance from the current
7421 : insn. */
7422 : static struct to_inherit to_inherit[LRA_MAX_INSN_RELOADS];
7423 :
7424 : /* Number elements in the previous array. */
7425 : static int to_inherit_num;
7426 :
7427 : /* Add inheritance info REGNO and INSNS. Their meaning is described in
7428 : structure to_inherit. */
7429 : static void
7430 319841 : add_to_inherit (int regno, rtx insns)
7431 : {
7432 319841 : int i;
7433 :
7434 319945 : for (i = 0; i < to_inherit_num; i++)
7435 104 : if (to_inherit[i].regno == regno)
7436 : return;
7437 319841 : lra_assert (to_inherit_num < LRA_MAX_INSN_RELOADS);
7438 319841 : to_inherit[to_inherit_num].regno = regno;
7439 319841 : to_inherit[to_inherit_num++].insns = insns;
7440 : }
7441 :
7442 : /* Return the last non-debug insn in basic block BB, or the block begin
7443 : note if none. */
7444 : static rtx_insn *
7445 30735158 : get_last_insertion_point (basic_block bb)
7446 : {
7447 30735158 : rtx_insn *insn;
7448 :
7449 33415991 : FOR_BB_INSNS_REVERSE (bb, insn)
7450 33415991 : if (NONDEBUG_INSN_P (insn) || NOTE_INSN_BASIC_BLOCK_P (insn))
7451 30735158 : return insn;
7452 0 : gcc_unreachable ();
7453 : }
7454 :
7455 : /* Set up RES by registers living on edges FROM except the edge (FROM,
7456 : TO) or by registers set up in a jump insn in BB FROM. */
7457 : static void
7458 11869300 : get_live_on_other_edges (basic_block from, basic_block to, bitmap res)
7459 : {
7460 11869300 : rtx_insn *last;
7461 11869300 : struct lra_insn_reg *reg;
7462 11869300 : edge e;
7463 11869300 : edge_iterator ei;
7464 :
7465 11869300 : lra_assert (to != NULL);
7466 11869300 : bitmap_clear (res);
7467 35343940 : FOR_EACH_EDGE (e, ei, from->succs)
7468 23474640 : if (e->dest != to)
7469 11605340 : bitmap_ior_into (res, df_get_live_in (e->dest));
7470 11869300 : last = get_last_insertion_point (from);
7471 11869300 : if (! JUMP_P (last))
7472 2013221 : return;
7473 9856079 : curr_id = lra_get_insn_recog_data (last);
7474 19711850 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7475 9855771 : if (reg->type != OP_IN)
7476 76 : bitmap_set_bit (res, reg->regno);
7477 : }
7478 :
7479 : /* Used as a temporary results of some bitmap calculations. */
7480 : static bitmap_head temp_bitmap;
7481 :
7482 : /* We split for reloads of small class of hard regs. The following
7483 : defines how many hard regs the class should have to be qualified as
7484 : small. The code is mostly oriented to x86/x86-64 architecture
7485 : where some insns need to use only specific register or pair of
7486 : registers and these register can live in RTL explicitly, e.g. for
7487 : parameter passing. */
7488 : static const int max_small_class_regs_num = 2;
7489 :
7490 : /* Do inheritance/split transformations in EBB starting with HEAD and
7491 : finishing on TAIL. We process EBB insns in the reverse order.
7492 : Return true if we did any inheritance/split transformation in the
7493 : EBB.
7494 :
7495 : We should avoid excessive splitting which results in worse code
7496 : because of inaccurate cost calculations for spilling new split
7497 : pseudos in such case. To achieve this we do splitting only if
7498 : register pressure is high in given basic block and there are reload
7499 : pseudos requiring hard registers. We could do more register
7500 : pressure calculations at any given program point to avoid necessary
7501 : splitting even more but it is to expensive and the current approach
7502 : works well enough. */
7503 : static bool
7504 12931346 : inherit_in_ebb (rtx_insn *head, rtx_insn *tail)
7505 : {
7506 12931346 : int i, src_regno, dst_regno, nregs;
7507 12931346 : bool change_p, succ_p, update_reloads_num_p;
7508 12931346 : rtx_insn *prev_insn, *last_insn;
7509 12931346 : rtx next_usage_insns, curr_set;
7510 12931346 : enum reg_class cl;
7511 12931346 : struct lra_insn_reg *reg;
7512 12931346 : basic_block last_processed_bb, curr_bb = NULL;
7513 12931346 : HARD_REG_SET potential_reload_hard_regs, live_hard_regs;
7514 12931346 : bitmap to_process;
7515 12931346 : unsigned int j;
7516 12931346 : bitmap_iterator bi;
7517 12931346 : bool head_p, after_p;
7518 :
7519 12931346 : change_p = false;
7520 12931346 : curr_usage_insns_check++;
7521 12931346 : clear_invariants ();
7522 12931346 : reloads_num = calls_num = 0;
7523 168107498 : for (unsigned int i = 0; i < NUM_ABI_IDS; ++i)
7524 155176152 : last_call_for_abi[i] = 0;
7525 12931346 : CLEAR_HARD_REG_SET (full_and_partial_call_clobbers);
7526 12931346 : bitmap_clear (&check_only_regs);
7527 12931346 : bitmap_clear (&invalid_invariant_regs);
7528 12931346 : last_processed_bb = NULL;
7529 12931346 : CLEAR_HARD_REG_SET (potential_reload_hard_regs);
7530 12931346 : live_hard_regs = eliminable_regset | lra_no_alloc_regs;
7531 : /* We don't process new insns generated in the loop. */
7532 247333901 : for (curr_insn = tail; curr_insn != PREV_INSN (head); curr_insn = prev_insn)
7533 : {
7534 234402555 : prev_insn = PREV_INSN (curr_insn);
7535 234402555 : if (BLOCK_FOR_INSN (curr_insn) != NULL)
7536 234402279 : curr_bb = BLOCK_FOR_INSN (curr_insn);
7537 234402555 : if (last_processed_bb != curr_bb)
7538 : {
7539 : /* We are at the end of BB. Add qualified living
7540 : pseudos for potential splitting. */
7541 18865858 : to_process = df_get_live_out (curr_bb);
7542 18865858 : if (last_processed_bb != NULL)
7543 : {
7544 : /* We are somewhere in the middle of EBB. */
7545 5934512 : get_live_on_other_edges (curr_bb, last_processed_bb,
7546 : &temp_bitmap);
7547 5934512 : to_process = &temp_bitmap;
7548 : }
7549 18865858 : last_processed_bb = curr_bb;
7550 18865858 : last_insn = get_last_insertion_point (curr_bb);
7551 37731716 : after_p = (! JUMP_P (last_insn)
7552 18865858 : && (! CALL_P (last_insn)
7553 2436854 : || (find_reg_note (last_insn,
7554 : REG_NORETURN, NULL_RTX) == NULL_RTX
7555 1468176 : && ! SIBLING_CALL_P (last_insn))));
7556 18865858 : CLEAR_HARD_REG_SET (potential_reload_hard_regs);
7557 205490831 : EXECUTE_IF_SET_IN_BITMAP (to_process, 0, j, bi)
7558 : {
7559 186624976 : if ((int) j >= lra_constraint_new_regno_start)
7560 : break;
7561 186624973 : if (j < FIRST_PSEUDO_REGISTER || reg_renumber[j] >= 0)
7562 : {
7563 121010159 : if (j < FIRST_PSEUDO_REGISTER)
7564 70836048 : SET_HARD_REG_BIT (live_hard_regs, j);
7565 : else
7566 50174111 : add_to_hard_reg_set (&live_hard_regs,
7567 50174111 : PSEUDO_REGNO_MODE (j),
7568 50174111 : reg_renumber[j]);
7569 121010159 : setup_next_usage_insn (j, last_insn, reloads_num, after_p);
7570 : }
7571 : }
7572 : }
7573 234402555 : src_regno = dst_regno = -1;
7574 234402555 : curr_set = single_set (curr_insn);
7575 234402555 : if (curr_set != NULL_RTX && REG_P (SET_DEST (curr_set)))
7576 85793207 : dst_regno = REGNO (SET_DEST (curr_set));
7577 116056003 : if (curr_set != NULL_RTX && REG_P (SET_SRC (curr_set)))
7578 40316295 : src_regno = REGNO (SET_SRC (curr_set));
7579 234402555 : update_reloads_num_p = true;
7580 234402555 : if (src_regno < lra_constraint_new_regno_start
7581 227791492 : && src_regno >= FIRST_PSEUDO_REGISTER
7582 28358275 : && reg_renumber[src_regno] < 0
7583 3770726 : && dst_regno >= lra_constraint_new_regno_start
7584 237059697 : && (cl = lra_get_allocno_class (dst_regno)) != NO_REGS)
7585 : {
7586 : /* 'reload_pseudo <- original_pseudo'. */
7587 2657142 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7588 28731 : reloads_num++;
7589 2657142 : update_reloads_num_p = false;
7590 2657142 : succ_p = false;
7591 2657142 : if (usage_insns[src_regno].check == curr_usage_insns_check
7592 2657142 : && (next_usage_insns = usage_insns[src_regno].insns) != NULL_RTX)
7593 472467 : succ_p = inherit_reload_reg (false, src_regno, cl,
7594 : curr_insn, next_usage_insns);
7595 472467 : if (succ_p)
7596 : change_p = true;
7597 : else
7598 2201696 : setup_next_usage_insn (src_regno, curr_insn, reloads_num, false);
7599 5314284 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7600 660349288 : potential_reload_hard_regs |= reg_class_contents[cl];
7601 : }
7602 231745413 : else if (src_regno < 0
7603 194086260 : && dst_regno >= lra_constraint_new_regno_start
7604 5838400 : && invariant_p (SET_SRC (curr_set))
7605 301193 : && (cl = lra_get_allocno_class (dst_regno)) != NO_REGS
7606 300647 : && ! bitmap_bit_p (&invalid_invariant_regs, dst_regno)
7607 232005661 : && ! bitmap_bit_p (&invalid_invariant_regs,
7608 260248 : ORIGINAL_REGNO(regno_reg_rtx[dst_regno])))
7609 : {
7610 : /* 'reload_pseudo <- invariant'. */
7611 198463 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7612 10747 : reloads_num++;
7613 198463 : update_reloads_num_p = false;
7614 198463 : if (process_invariant_for_inheritance (SET_DEST (curr_set), SET_SRC (curr_set)))
7615 26280 : change_p = true;
7616 396926 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7617 660349288 : potential_reload_hard_regs |= reg_class_contents[cl];
7618 : }
7619 231546950 : else if (src_regno >= lra_constraint_new_regno_start
7620 6611063 : && dst_regno < lra_constraint_new_regno_start
7621 5806443 : && dst_regno >= FIRST_PSEUDO_REGISTER
7622 3831574 : && reg_renumber[dst_regno] < 0
7623 1473040 : && (cl = lra_get_allocno_class (src_regno)) != NO_REGS
7624 1473040 : && usage_insns[dst_regno].check == curr_usage_insns_check
7625 231546950 : && (next_usage_insns
7626 479692 : = usage_insns[dst_regno].insns) != NULL_RTX)
7627 : {
7628 479692 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7629 10118 : reloads_num++;
7630 479692 : update_reloads_num_p = false;
7631 : /* 'original_pseudo <- reload_pseudo'. */
7632 479692 : if (! JUMP_P (curr_insn)
7633 479692 : && inherit_reload_reg (true, dst_regno, cl,
7634 : curr_insn, next_usage_insns))
7635 : change_p = true;
7636 : /* Invalidate. */
7637 479692 : usage_insns[dst_regno].check = 0;
7638 959384 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7639 660349288 : potential_reload_hard_regs |= reg_class_contents[cl];
7640 : }
7641 231067258 : else if (INSN_P (curr_insn))
7642 : {
7643 194805116 : int iter;
7644 194805116 : int max_uid = get_max_uid ();
7645 :
7646 194805116 : curr_id = lra_get_insn_recog_data (curr_insn);
7647 194805116 : curr_static_id = curr_id->insn_static_data;
7648 194805116 : to_inherit_num = 0;
7649 : /* Process insn definitions. */
7650 584415348 : for (iter = 0; iter < 2; iter++)
7651 389610232 : for (reg = iter == 0 ? curr_id->regs : curr_static_id->hard_regs;
7652 622821486 : reg != NULL;
7653 233211254 : reg = reg->next)
7654 233211254 : if (reg->type != OP_IN
7655 233211254 : && (dst_regno = reg->regno) < lra_constraint_new_regno_start)
7656 : {
7657 46734114 : if (dst_regno >= FIRST_PSEUDO_REGISTER && reg->type == OP_OUT
7658 44517899 : && reg_renumber[dst_regno] < 0 && ! reg->subreg_p
7659 1830165 : && usage_insns[dst_regno].check == curr_usage_insns_check
7660 92878422 : && (next_usage_insns
7661 129693 : = usage_insns[dst_regno].insns) != NULL_RTX)
7662 : {
7663 129693 : struct lra_insn_reg *r;
7664 :
7665 387169 : for (r = curr_id->regs; r != NULL; r = r->next)
7666 257476 : if (r->type != OP_OUT && r->regno == dst_regno)
7667 : break;
7668 : /* Don't do inheritance if the pseudo is also
7669 : used in the insn. */
7670 129693 : if (r == NULL)
7671 : /* We cannot do inheritance right now
7672 : because the current insn reg info (chain
7673 : regs) can change after that. */
7674 129693 : add_to_inherit (dst_regno, next_usage_insns);
7675 : }
7676 : /* We cannot process one reg twice here because of
7677 : usage_insns invalidation. */
7678 92878422 : if ((dst_regno < FIRST_PSEUDO_REGISTER
7679 46734114 : || reg_renumber[dst_regno] >= 0)
7680 90912048 : && ! reg->subreg_p && reg->type != OP_IN)
7681 : {
7682 90625767 : HARD_REG_SET s;
7683 :
7684 90625767 : if (split_if_necessary (dst_regno, reg->biggest_mode,
7685 : potential_reload_hard_regs,
7686 : false, curr_insn, max_uid))
7687 58191 : change_p = true;
7688 90625767 : CLEAR_HARD_REG_SET (s);
7689 90625767 : if (dst_regno < FIRST_PSEUDO_REGISTER)
7690 46144308 : add_to_hard_reg_set (&s, reg->biggest_mode, dst_regno);
7691 : else
7692 44481459 : add_to_hard_reg_set (&s, PSEUDO_REGNO_MODE (dst_regno),
7693 44481459 : reg_renumber[dst_regno]);
7694 90625767 : live_hard_regs &= ~s;
7695 181251534 : potential_reload_hard_regs &= ~s;
7696 : }
7697 : /* We should invalidate potential inheritance or
7698 : splitting for the current insn usages to the next
7699 : usage insns (see code below) as the output pseudo
7700 : prevents this. */
7701 92878422 : if ((dst_regno >= FIRST_PSEUDO_REGISTER
7702 46734114 : && reg_renumber[dst_regno] < 0)
7703 90912048 : || (reg->type == OP_OUT && ! reg->subreg_p
7704 82830705 : && (dst_regno < FIRST_PSEUDO_REGISTER
7705 42273814 : || reg_renumber[dst_regno] >= 0)))
7706 : {
7707 : /* Invalidate and mark definitions. */
7708 44240188 : if (dst_regno >= FIRST_PSEUDO_REGISTER)
7709 44240188 : usage_insns[dst_regno].check = -(int) INSN_UID (curr_insn);
7710 : else
7711 : {
7712 40556891 : nregs = hard_regno_nregs (dst_regno,
7713 40556891 : reg->biggest_mode);
7714 81375508 : for (i = 0; i < nregs; i++)
7715 81637234 : usage_insns[dst_regno + i].check
7716 40818617 : = -(int) INSN_UID (curr_insn);
7717 : }
7718 : }
7719 : }
7720 : /* Process clobbered call regs. */
7721 194805116 : if (curr_id->arg_hard_regs != NULL)
7722 19827471 : for (i = 0; (dst_regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7723 13856525 : if (dst_regno >= FIRST_PSEUDO_REGISTER)
7724 0 : usage_insns[dst_regno - FIRST_PSEUDO_REGISTER].check
7725 0 : = -(int) INSN_UID (curr_insn);
7726 194805116 : if (! JUMP_P (curr_insn))
7727 183185239 : for (i = 0; i < to_inherit_num; i++)
7728 129693 : if (inherit_reload_reg (true, to_inherit[i].regno,
7729 : ALL_REGS, curr_insn,
7730 : to_inherit[i].insns))
7731 102885 : change_p = true;
7732 194805116 : if (CALL_P (curr_insn))
7733 : {
7734 7548769 : rtx cheap, pat, dest;
7735 7548769 : rtx_insn *restore;
7736 7548769 : int regno, hard_regno;
7737 :
7738 7548769 : calls_num++;
7739 7548769 : function_abi callee_abi = insn_callee_abi (curr_insn);
7740 7548769 : last_call_for_abi[callee_abi.id ()] = calls_num;
7741 7548769 : full_and_partial_call_clobbers
7742 7548769 : |= callee_abi.full_and_partial_reg_clobbers ();
7743 7548769 : first_call_insn = curr_insn;
7744 7548769 : if ((cheap = find_reg_note (curr_insn,
7745 : REG_RETURNED, NULL_RTX)) != NULL_RTX
7746 45329 : && ((cheap = XEXP (cheap, 0)), true)
7747 45329 : && (regno = REGNO (cheap)) >= FIRST_PSEUDO_REGISTER
7748 45329 : && (hard_regno = reg_renumber[regno]) >= 0
7749 33967 : && usage_insns[regno].check == curr_usage_insns_check
7750 : /* If there are pending saves/restores, the
7751 : optimization is not worth. */
7752 29065 : && usage_insns[regno].calls_num == calls_num - 1
7753 7576026 : && callee_abi.clobbers_reg_p (GET_MODE (cheap), hard_regno))
7754 : {
7755 : /* Restore the pseudo from the call result as
7756 : REG_RETURNED note says that the pseudo value is
7757 : in the call result and the pseudo is an argument
7758 : of the call. */
7759 12217 : pat = PATTERN (curr_insn);
7760 12217 : if (GET_CODE (pat) == PARALLEL)
7761 0 : pat = XVECEXP (pat, 0, 0);
7762 12217 : dest = SET_DEST (pat);
7763 : /* For multiple return values dest is PARALLEL.
7764 : Currently we handle only single return value case. */
7765 12217 : if (REG_P (dest))
7766 : {
7767 12217 : start_sequence ();
7768 12217 : emit_move_insn (cheap, copy_rtx (dest));
7769 12217 : restore = end_sequence ();
7770 12217 : lra_process_new_insns (curr_insn, NULL, restore,
7771 : "Inserting call parameter restore");
7772 : /* We don't need to save/restore of the pseudo from
7773 : this call. */
7774 12217 : usage_insns[regno].calls_num = calls_num;
7775 12217 : remove_from_hard_reg_set
7776 12217 : (&full_and_partial_call_clobbers,
7777 12217 : GET_MODE (cheap), hard_regno);
7778 12217 : bitmap_set_bit (&check_only_regs, regno);
7779 : }
7780 : }
7781 : }
7782 194805116 : to_inherit_num = 0;
7783 : /* Process insn usages. */
7784 584415348 : for (iter = 0; iter < 2; iter++)
7785 389610232 : for (reg = iter == 0 ? curr_id->regs : curr_static_id->hard_regs;
7786 622821486 : reg != NULL;
7787 233211254 : reg = reg->next)
7788 233211254 : if ((reg->type != OP_OUT
7789 91747786 : || (reg->type == OP_OUT && reg->subreg_p))
7790 233748854 : && (src_regno = reg->regno) < lra_constraint_new_regno_start)
7791 : {
7792 130197485 : if (src_regno >= FIRST_PSEUDO_REGISTER
7793 75318367 : && reg_renumber[src_regno] < 0 && reg->type == OP_IN)
7794 : {
7795 2583758 : if (usage_insns[src_regno].check == curr_usage_insns_check
7796 846366 : && (next_usage_insns
7797 846366 : = usage_insns[src_regno].insns) != NULL_RTX
7798 3430124 : && NONDEBUG_INSN_P (curr_insn))
7799 190148 : add_to_inherit (src_regno, next_usage_insns);
7800 4787220 : else if (usage_insns[src_regno].check
7801 2393610 : != -(int) INSN_UID (curr_insn))
7802 : /* Add usages but only if the reg is not set up
7803 : in the same insn. */
7804 2393610 : add_next_usage_insn (src_regno, curr_insn, reloads_num);
7805 : }
7806 72734609 : else if (src_regno < FIRST_PSEUDO_REGISTER
7807 72734609 : || reg_renumber[src_regno] >= 0)
7808 : {
7809 127477518 : bool before_p;
7810 127477518 : rtx_insn *use_insn = curr_insn;
7811 127477518 : rtx_insn *prev_insn = PREV_INSN (curr_insn);
7812 :
7813 254955036 : before_p = (JUMP_P (curr_insn)
7814 127477518 : || (CALL_P (curr_insn) && reg->type == OP_IN));
7815 127477518 : if (NONDEBUG_INSN_P (curr_insn)
7816 113047435 : && (! JUMP_P (curr_insn) || reg->type == OP_IN)
7817 240524848 : && split_if_necessary (src_regno, reg->biggest_mode,
7818 : potential_reload_hard_regs,
7819 : before_p, curr_insn, max_uid))
7820 : {
7821 225330 : if (reg->subreg_p)
7822 3128 : check_and_force_assignment_correctness_p = true;
7823 225330 : change_p = true;
7824 : /* Invalidate. */
7825 225330 : usage_insns[src_regno].check = 0;
7826 225330 : if (before_p && PREV_INSN (curr_insn) != prev_insn)
7827 : use_insn = PREV_INSN (curr_insn);
7828 : }
7829 127477518 : if (NONDEBUG_INSN_P (curr_insn))
7830 : {
7831 113047435 : if (src_regno < FIRST_PSEUDO_REGISTER)
7832 49664169 : add_to_hard_reg_set (&live_hard_regs,
7833 49664169 : reg->biggest_mode, src_regno);
7834 : else
7835 63383266 : add_to_hard_reg_set (&live_hard_regs,
7836 63383266 : PSEUDO_REGNO_MODE (src_regno),
7837 63383266 : reg_renumber[src_regno]);
7838 : }
7839 127477518 : if (src_regno >= FIRST_PSEUDO_REGISTER)
7840 72598400 : add_next_usage_insn (src_regno, use_insn, reloads_num);
7841 : else
7842 : {
7843 109857736 : for (i = 0; i < hard_regno_nregs (src_regno, reg->biggest_mode); i++)
7844 54978618 : add_next_usage_insn (src_regno + i, use_insn, reloads_num);
7845 : }
7846 : }
7847 : }
7848 : /* Process used call regs. */
7849 194805116 : if (curr_id->arg_hard_regs != NULL)
7850 19827471 : for (i = 0; (src_regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7851 13856525 : if (src_regno < FIRST_PSEUDO_REGISTER)
7852 : {
7853 13856525 : SET_HARD_REG_BIT (live_hard_regs, src_regno);
7854 13856525 : add_next_usage_insn (src_regno, curr_insn, reloads_num);
7855 : }
7856 194995264 : for (i = 0; i < to_inherit_num; i++)
7857 : {
7858 190148 : src_regno = to_inherit[i].regno;
7859 190148 : if (inherit_reload_reg (false, src_regno, ALL_REGS,
7860 : curr_insn, to_inherit[i].insns))
7861 : change_p = true;
7862 : else
7863 20690 : setup_next_usage_insn (src_regno, curr_insn, reloads_num, false);
7864 : }
7865 : }
7866 194879475 : if (update_reloads_num_p
7867 231067258 : && NONDEBUG_INSN_P (curr_insn) && curr_set != NULL_RTX)
7868 : {
7869 112720706 : int regno = -1;
7870 112720706 : if ((REG_P (SET_DEST (curr_set))
7871 82457910 : && (regno = REGNO (SET_DEST (curr_set))) >= lra_constraint_new_regno_start
7872 8506351 : && reg_renumber[regno] < 0
7873 5446143 : && (cl = lra_get_allocno_class (regno)) != NO_REGS)
7874 190006908 : || (REG_P (SET_SRC (curr_set))
7875 35859997 : && (regno = REGNO (SET_SRC (curr_set))) >= lra_constraint_new_regno_start
7876 6204042 : && reg_renumber[regno] < 0
7877 3667565 : && (cl = lra_get_allocno_class (regno)) != NO_REGS))
7878 : {
7879 8577070 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7880 254003 : reloads_num++;
7881 17154140 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7882 234402555 : potential_reload_hard_regs |= reg_class_contents[cl];
7883 : }
7884 : }
7885 234402555 : if (NONDEBUG_INSN_P (curr_insn))
7886 : {
7887 122545660 : int regno;
7888 :
7889 : /* Invalidate invariants with changed regs. */
7890 122545660 : curr_id = lra_get_insn_recog_data (curr_insn);
7891 314346623 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7892 191800963 : if (reg->type != OP_IN)
7893 : {
7894 82271977 : bitmap_set_bit (&invalid_invariant_regs, reg->regno);
7895 164543954 : bitmap_set_bit (&invalid_invariant_regs,
7896 82271977 : ORIGINAL_REGNO (regno_reg_rtx[reg->regno]));
7897 : }
7898 122545660 : curr_static_id = curr_id->insn_static_data;
7899 154677252 : for (reg = curr_static_id->hard_regs; reg != NULL; reg = reg->next)
7900 32131592 : if (reg->type != OP_IN)
7901 22986645 : bitmap_set_bit (&invalid_invariant_regs, reg->regno);
7902 122545660 : if (curr_id->arg_hard_regs != NULL)
7903 19827471 : for (i = 0; (regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7904 13856525 : if (regno >= FIRST_PSEUDO_REGISTER)
7905 0 : bitmap_set_bit (&invalid_invariant_regs,
7906 : regno - FIRST_PSEUDO_REGISTER);
7907 : }
7908 : /* We reached the start of the current basic block. */
7909 234402547 : if (prev_insn == NULL_RTX || prev_insn == PREV_INSN (head)
7910 455873764 : || BLOCK_FOR_INSN (prev_insn) != curr_bb)
7911 : {
7912 : /* We reached the beginning of the current block -- do
7913 : rest of splitting in the current BB. */
7914 18866134 : to_process = df_get_live_in (curr_bb);
7915 18866134 : if (BLOCK_FOR_INSN (head) != curr_bb)
7916 : {
7917 : /* We are somewhere in the middle of EBB. */
7918 5934788 : get_live_on_other_edges (EDGE_PRED (curr_bb, 0)->src,
7919 : curr_bb, &temp_bitmap);
7920 5934788 : to_process = &temp_bitmap;
7921 : }
7922 18866134 : head_p = true;
7923 198545714 : EXECUTE_IF_SET_IN_BITMAP (to_process, 0, j, bi)
7924 : {
7925 179679585 : if ((int) j >= lra_constraint_new_regno_start)
7926 : break;
7927 114764698 : if (((int) j < FIRST_PSEUDO_REGISTER || reg_renumber[j] >= 0)
7928 112968612 : && usage_insns[j].check == curr_usage_insns_check
7929 290724478 : && (next_usage_insns = usage_insns[j].insns) != NULL_RTX)
7930 : {
7931 111044898 : if (need_for_split_p (potential_reload_hard_regs, j))
7932 : {
7933 386756 : if (lra_dump_file != NULL && head_p)
7934 : {
7935 0 : fprintf (lra_dump_file,
7936 : " ----------------------------------\n");
7937 0 : head_p = false;
7938 : }
7939 386756 : if (split_reg (false, j, bb_note (curr_bb),
7940 : next_usage_insns, NULL))
7941 386756 : change_p = true;
7942 : }
7943 111044898 : usage_insns[j].check = 0;
7944 : }
7945 : }
7946 : }
7947 : }
7948 12931346 : first_call_insn = NULL;
7949 12931346 : return change_p;
7950 : }
7951 :
7952 : /* This value affects EBB forming. If probability of edge from EBB to
7953 : a BB is not greater than the following value, we don't add the BB
7954 : to EBB. */
7955 : #define EBB_PROBABILITY_CUTOFF \
7956 : ((REG_BR_PROB_BASE * param_lra_inheritance_ebb_probability_cutoff) / 100)
7957 :
7958 : /* Current number of inheritance/split iteration. */
7959 : int lra_inheritance_iter;
7960 :
7961 : /* Entry function for inheritance/split pass. */
7962 : void
7963 1581806 : lra_inheritance (void)
7964 : {
7965 1581806 : int i;
7966 1581806 : basic_block bb, start_bb;
7967 1581806 : edge e;
7968 :
7969 1581806 : lra_inheritance_iter++;
7970 1581806 : if (lra_inheritance_iter > LRA_MAX_INHERITANCE_PASSES)
7971 : return;
7972 1578630 : timevar_push (TV_LRA_INHERITANCE);
7973 1578630 : if (lra_dump_file != NULL)
7974 96 : fprintf (lra_dump_file, "\n********** Inheritance #%d: **********\n\n",
7975 : lra_inheritance_iter);
7976 1578630 : curr_usage_insns_check = 0;
7977 1578630 : usage_insns = XNEWVEC (struct usage_insns, lra_constraint_new_regno_start);
7978 238015204 : for (i = 0; i < lra_constraint_new_regno_start; i++)
7979 236436574 : usage_insns[i].check = 0;
7980 1578630 : bitmap_initialize (&check_only_regs, ®_obstack);
7981 1578630 : bitmap_initialize (&invalid_invariant_regs, ®_obstack);
7982 1578630 : bitmap_initialize (&live_regs, ®_obstack);
7983 1578630 : bitmap_initialize (&temp_bitmap, ®_obstack);
7984 1578630 : bitmap_initialize (&ebb_global_regs, ®_obstack);
7985 14509976 : FOR_EACH_BB_FN (bb, cfun)
7986 : {
7987 12931346 : start_bb = bb;
7988 12931346 : if (lra_dump_file != NULL)
7989 321 : fprintf (lra_dump_file, "EBB");
7990 : /* Form a EBB starting with BB. */
7991 12931346 : bitmap_clear (&ebb_global_regs);
7992 12931346 : bitmap_ior_into (&ebb_global_regs, df_get_live_in (bb));
7993 24800370 : for (;;)
7994 : {
7995 18865858 : if (lra_dump_file != NULL)
7996 435 : fprintf (lra_dump_file, " %d", bb->index);
7997 18865858 : if (bb->next_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
7998 17287228 : || LABEL_P (BB_HEAD (bb->next_bb)))
7999 : break;
8000 8343421 : e = find_fallthru_edge (bb->succs);
8001 8343421 : if (! e)
8002 : break;
8003 8343421 : if (e->probability.initialized_p ()
8004 8343421 : && e->probability.to_reg_br_prob_base () < EBB_PROBABILITY_CUTOFF)
8005 : break;
8006 5934512 : bb = bb->next_bb;
8007 : }
8008 12931346 : bitmap_ior_into (&ebb_global_regs, df_get_live_out (bb));
8009 12931346 : if (lra_dump_file != NULL)
8010 321 : fprintf (lra_dump_file, "\n");
8011 12931346 : if (inherit_in_ebb (BB_HEAD (start_bb), BB_END (bb)))
8012 : /* Remember that the EBB head and tail can change in
8013 : inherit_in_ebb. */
8014 750518 : update_ebb_live_info (BB_HEAD (start_bb), BB_END (bb));
8015 : }
8016 1578630 : bitmap_release (&ebb_global_regs);
8017 1578630 : bitmap_release (&temp_bitmap);
8018 1578630 : bitmap_release (&live_regs);
8019 1578630 : bitmap_release (&invalid_invariant_regs);
8020 1578630 : bitmap_release (&check_only_regs);
8021 1578630 : free (usage_insns);
8022 1578630 : lra_dump_insns_if_possible ("func after inheritance");
8023 1578630 : timevar_pop (TV_LRA_INHERITANCE);
8024 : }
8025 :
8026 :
8027 :
8028 : /* This page contains code to undo failed inheritance/split
8029 : transformations. */
8030 :
8031 : /* Current number of iteration undoing inheritance/split. */
8032 : int lra_undo_inheritance_iter;
8033 :
8034 : /* Fix BB live info LIVE after removing pseudos created on pass doing
8035 : inheritance/split which are REMOVED_PSEUDOS. */
8036 : static void
8037 37731716 : fix_bb_live_info (bitmap live, bitmap removed_pseudos)
8038 : {
8039 37731716 : unsigned int regno;
8040 37731716 : bitmap_iterator bi;
8041 :
8042 180502436 : EXECUTE_IF_SET_IN_BITMAP (removed_pseudos, 0, regno, bi)
8043 142770720 : if (bitmap_clear_bit (live, regno)
8044 142770720 : && REG_P (lra_reg_info[regno].restore_rtx))
8045 509114 : bitmap_set_bit (live, REGNO (lra_reg_info[regno].restore_rtx));
8046 37731716 : }
8047 :
8048 : /* Return regno of the (subreg of) REG. Otherwise, return a negative
8049 : number. */
8050 : static int
8051 70413643 : get_regno (rtx reg)
8052 : {
8053 1021457 : if (GET_CODE (reg) == SUBREG)
8054 1056450 : reg = SUBREG_REG (reg);
8055 70413643 : if (REG_P (reg))
8056 45580557 : return REGNO (reg);
8057 : return -1;
8058 : }
8059 :
8060 : /* Delete a move INSN with destination reg DREGNO and a previous
8061 : clobber insn with the same regno. The inheritance/split code can
8062 : generate moves with preceding clobber and when we delete such moves
8063 : we should delete the clobber insn too to keep the correct life
8064 : info. */
8065 : static void
8066 759904 : delete_move_and_clobber (rtx_insn *insn, int dregno)
8067 : {
8068 759904 : rtx_insn *prev_insn = PREV_INSN (insn);
8069 :
8070 759904 : lra_set_insn_deleted (insn);
8071 759904 : lra_assert (dregno >= 0);
8072 759904 : if (prev_insn != NULL && NONDEBUG_INSN_P (prev_insn)
8073 330790 : && GET_CODE (PATTERN (prev_insn)) == CLOBBER
8074 760368 : && dregno == get_regno (XEXP (PATTERN (prev_insn), 0)))
8075 0 : lra_set_insn_deleted (prev_insn);
8076 759904 : }
8077 :
8078 : /* Remove inheritance/split pseudos which are in REMOVE_PSEUDOS and
8079 : return true if we did any change. The undo transformations for
8080 : inheritance looks like
8081 : i <- i2
8082 : p <- i => p <- i2
8083 : or removing
8084 : p <- i, i <- p, and i <- i3
8085 : where p is original pseudo from which inheritance pseudo i was
8086 : created, i and i3 are removed inheritance pseudos, i2 is another
8087 : not removed inheritance pseudo. All split pseudos or other
8088 : occurrences of removed inheritance pseudos are changed on the
8089 : corresponding original pseudos.
8090 :
8091 : The function also schedules insns changed and created during
8092 : inheritance/split pass for processing by the subsequent constraint
8093 : pass. */
8094 : static bool
8095 1578630 : remove_inheritance_pseudos (bitmap remove_pseudos)
8096 : {
8097 1578630 : basic_block bb;
8098 1578630 : int regno, sregno, prev_sregno, dregno;
8099 1578630 : rtx restore_rtx;
8100 1578630 : rtx set, prev_set;
8101 1578630 : rtx_insn *prev_insn;
8102 1578630 : bool change_p, done_p;
8103 :
8104 1578630 : change_p = ! bitmap_empty_p (remove_pseudos);
8105 : /* We cannot finish the function right away if CHANGE_P is true
8106 : because we need to marks insns affected by previous
8107 : inheritance/split pass for processing by the subsequent
8108 : constraint pass. */
8109 20444488 : FOR_EACH_BB_FN (bb, cfun)
8110 : {
8111 18865858 : fix_bb_live_info (df_get_live_in (bb), remove_pseudos);
8112 18865858 : fix_bb_live_info (df_get_live_out (bb), remove_pseudos);
8113 255843258 : FOR_BB_INSNS_REVERSE (bb, curr_insn)
8114 : {
8115 236977400 : if (! INSN_P (curr_insn))
8116 36605509 : continue;
8117 200371891 : done_p = false;
8118 200371891 : sregno = dregno = -1;
8119 49348227 : if (change_p && NONDEBUG_INSN_P (curr_insn)
8120 233110571 : && (set = single_set (curr_insn)) != NULL_RTX)
8121 : {
8122 31689480 : dregno = get_regno (SET_DEST (set));
8123 63378960 : sregno = get_regno (SET_SRC (set));
8124 : }
8125 :
8126 200371891 : if (sregno >= 0 && dregno >= 0)
8127 : {
8128 11305598 : if (bitmap_bit_p (remove_pseudos, dregno)
8129 11305598 : && ! REG_P (lra_reg_info[dregno].restore_rtx))
8130 : {
8131 : /* invariant inheritance pseudo <- original pseudo */
8132 7625 : if (lra_dump_file != NULL)
8133 : {
8134 0 : fprintf (lra_dump_file, " Removing invariant inheritance:\n");
8135 0 : dump_insn_slim (lra_dump_file, curr_insn);
8136 0 : fprintf (lra_dump_file, "\n");
8137 : }
8138 7625 : delete_move_and_clobber (curr_insn, dregno);
8139 7625 : done_p = true;
8140 : }
8141 11297973 : else if (bitmap_bit_p (remove_pseudos, sregno)
8142 11297973 : && ! REG_P (lra_reg_info[sregno].restore_rtx))
8143 : {
8144 : /* reload pseudo <- invariant inheritance pseudo */
8145 7625 : start_sequence ();
8146 : /* We cannot just change the source. It might be
8147 : an insn different from the move. */
8148 7625 : emit_insn (lra_reg_info[sregno].restore_rtx);
8149 7625 : rtx_insn *new_insns = end_sequence ();
8150 7625 : lra_assert (single_set (new_insns) != NULL
8151 : && SET_DEST (set) == SET_DEST (single_set (new_insns)));
8152 7625 : lra_process_new_insns (curr_insn, NULL, new_insns,
8153 : "Changing reload<-invariant inheritance");
8154 7625 : delete_move_and_clobber (curr_insn, dregno);
8155 7625 : done_p = true;
8156 : }
8157 11290348 : else if ((bitmap_bit_p (remove_pseudos, sregno)
8158 1282334 : && (get_regno (lra_reg_info[sregno].restore_rtx) == dregno
8159 601560 : || (bitmap_bit_p (remove_pseudos, dregno)
8160 209678 : && get_regno (lra_reg_info[sregno].restore_rtx) >= 0
8161 209678 : && (get_regno (lra_reg_info[sregno].restore_rtx)
8162 209678 : == get_regno (lra_reg_info[dregno].restore_rtx)))))
8163 11787069 : || (bitmap_bit_p (remove_pseudos, dregno)
8164 674428 : && get_regno (lra_reg_info[dregno].restore_rtx) == sregno))
8165 : /* One of the following cases:
8166 : original <- removed inheritance pseudo
8167 : removed inherit pseudo <- another removed inherit pseudo
8168 : removed inherit pseudo <- original pseudo
8169 : Or
8170 : removed_split_pseudo <- original_reg
8171 : original_reg <- removed_split_pseudo */
8172 : {
8173 192360 : if (lra_dump_file != NULL)
8174 : {
8175 0 : fprintf (lra_dump_file, " Removing %s:\n",
8176 0 : bitmap_bit_p (&lra_split_regs, sregno)
8177 0 : || bitmap_bit_p (&lra_split_regs, dregno)
8178 : ? "split" : "inheritance");
8179 0 : dump_insn_slim (lra_dump_file, curr_insn);
8180 : }
8181 192360 : delete_move_and_clobber (curr_insn, dregno);
8182 192360 : done_p = true;
8183 : }
8184 11097988 : else if (bitmap_bit_p (remove_pseudos, sregno)
8185 11097988 : && bitmap_bit_p (&lra_inheritance_pseudos, sregno))
8186 : {
8187 : /* Search the following pattern:
8188 : inherit_or_split_pseudo1 <- inherit_or_split_pseudo2
8189 : original_pseudo <- inherit_or_split_pseudo1
8190 : where the 2nd insn is the current insn and
8191 : inherit_or_split_pseudo2 is not removed. If it is found,
8192 : change the current insn onto:
8193 : original_pseudo <- inherit_or_split_pseudo2. */
8194 777662 : for (prev_insn = PREV_INSN (curr_insn);
8195 777662 : prev_insn != NULL_RTX && ! NONDEBUG_INSN_P (prev_insn);
8196 280941 : prev_insn = PREV_INSN (prev_insn))
8197 : ;
8198 496721 : if (prev_insn != NULL_RTX && BLOCK_FOR_INSN (prev_insn) == bb
8199 484308 : && (prev_set = single_set (prev_insn)) != NULL_RTX
8200 : /* There should be no subregs in insn we are
8201 : searching because only the original reg might
8202 : be in subreg when we changed the mode of
8203 : load/store for splitting. */
8204 478189 : && REG_P (SET_DEST (prev_set))
8205 373554 : && REG_P (SET_SRC (prev_set))
8206 287793 : && (int) REGNO (SET_DEST (prev_set)) == sregno
8207 193942 : && ((prev_sregno = REGNO (SET_SRC (prev_set)))
8208 : >= FIRST_PSEUDO_REGISTER)
8209 193942 : && (lra_reg_info[prev_sregno].restore_rtx == NULL_RTX
8210 153250 : ||
8211 : /* As we consider chain of inheritance or
8212 : splitting described in above comment we should
8213 : check that sregno and prev_sregno were
8214 : inheritance/split pseudos created from the
8215 : same original regno. */
8216 306500 : (get_regno (lra_reg_info[sregno].restore_rtx) >= 0
8217 306500 : && (get_regno (lra_reg_info[sregno].restore_rtx)
8218 306500 : == get_regno (lra_reg_info[prev_sregno].restore_rtx))))
8219 690663 : && ! bitmap_bit_p (remove_pseudos, prev_sregno))
8220 : {
8221 103967 : int restore_regno = get_regno (lra_reg_info[sregno].restore_rtx);
8222 103967 : if (restore_regno < 0)
8223 0 : restore_regno = prev_sregno;
8224 103967 : lra_assert (GET_MODE (SET_SRC (prev_set))
8225 : == GET_MODE (regno_reg_rtx[restore_regno]));
8226 : /* Although we have a single set, the insn can
8227 : contain more one sregno register occurrence
8228 : as a source. Change all occurrences. */
8229 103967 : lra_substitute_pseudo_within_insn (curr_insn, sregno,
8230 : regno_reg_rtx[restore_regno],
8231 : false);
8232 : /* As we are finishing with processing the insn
8233 : here, check the destination too as it might
8234 : inheritance pseudo for another pseudo. */
8235 103967 : if (bitmap_bit_p (remove_pseudos, dregno)
8236 0 : && bitmap_bit_p (&lra_inheritance_pseudos, dregno)
8237 103967 : && (restore_rtx
8238 0 : = lra_reg_info[dregno].restore_rtx) != NULL_RTX)
8239 : {
8240 0 : if (GET_CODE (SET_DEST (set)) == SUBREG)
8241 0 : SUBREG_REG (SET_DEST (set)) = restore_rtx;
8242 : else
8243 0 : SET_DEST (set) = restore_rtx;
8244 : }
8245 103967 : lra_push_insn_and_update_insn_regno_info (curr_insn);
8246 103967 : lra_set_used_insn_alternative_by_uid
8247 103967 : (INSN_UID (curr_insn), LRA_UNKNOWN_ALT);
8248 103967 : done_p = true;
8249 103967 : if (lra_dump_file != NULL)
8250 : {
8251 0 : fprintf (lra_dump_file, " Change reload insn:\n");
8252 0 : dump_insn_slim (lra_dump_file, curr_insn);
8253 : }
8254 : }
8255 : }
8256 : }
8257 207610 : if (! done_p)
8258 : {
8259 200060314 : struct lra_insn_reg *reg;
8260 200060314 : bool restored_regs_p = false;
8261 200060314 : bool kept_regs_p = false;
8262 :
8263 200060314 : curr_id = lra_get_insn_recog_data (curr_insn);
8264 411497582 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
8265 : {
8266 211437268 : regno = reg->regno;
8267 211437268 : restore_rtx = lra_reg_info[regno].restore_rtx;
8268 211437268 : if (restore_rtx != NULL_RTX)
8269 : {
8270 5979502 : if (change_p && bitmap_bit_p (remove_pseudos, regno))
8271 : {
8272 881031 : lra_substitute_pseudo_within_insn
8273 881031 : (curr_insn, regno, restore_rtx, false);
8274 881031 : restored_regs_p = true;
8275 : }
8276 : else
8277 : kept_regs_p = true;
8278 : }
8279 : }
8280 200060314 : if (NONDEBUG_INSN_P (curr_insn) && kept_regs_p)
8281 : {
8282 : /* The instruction has changed since the previous
8283 : constraints pass. */
8284 4478999 : lra_push_insn_and_update_insn_regno_info (curr_insn);
8285 4478999 : lra_set_used_insn_alternative_by_uid
8286 4478999 : (INSN_UID (curr_insn), LRA_UNKNOWN_ALT);
8287 : }
8288 195581315 : else if (restored_regs_p)
8289 : /* The instruction has been restored to the form that
8290 : it had during the previous constraints pass. */
8291 705591 : lra_update_insn_regno_info (curr_insn);
8292 5184590 : if (restored_regs_p && lra_dump_file != NULL)
8293 : {
8294 0 : fprintf (lra_dump_file, " Insn after restoring regs:\n");
8295 0 : dump_insn_slim (lra_dump_file, curr_insn);
8296 : }
8297 : }
8298 : }
8299 : }
8300 1578630 : return change_p;
8301 : }
8302 :
8303 : /* Return true if REGNO is an inheritance pseudo. */
8304 : static bool
8305 132061 : inheritance_pseudo_p (int regno)
8306 : {
8307 132061 : return (regno >= new_regno_start
8308 132061 : && bitmap_bit_p (&lra_inheritance_pseudos, regno));
8309 : }
8310 :
8311 : /* If optional reload pseudos failed to get a hard register or was not
8312 : inherited, it is better to remove optional reloads. We do this
8313 : transformation after undoing inheritance to figure out necessity to
8314 : remove optional reloads easier. Return true if we do any
8315 : change. */
8316 : static bool
8317 1578630 : undo_optional_reloads (void)
8318 : {
8319 1578630 : bool change_p, keep_p;
8320 1578630 : unsigned int regno, uid;
8321 1578630 : bitmap_iterator bi, bi2;
8322 1578630 : rtx_insn *insn;
8323 1578630 : rtx set, src, dest;
8324 1578630 : auto_bitmap removed_optional_reload_pseudos (®_obstack);
8325 :
8326 1578630 : bitmap_copy (removed_optional_reload_pseudos, &lra_optional_reload_pseudos);
8327 2570407 : EXECUTE_IF_SET_IN_BITMAP (&lra_optional_reload_pseudos, 0, regno, bi)
8328 : {
8329 991777 : keep_p = false;
8330 : /* Keep optional reloads from previous subpasses. */
8331 991777 : if (lra_reg_info[regno].restore_rtx == NULL_RTX
8332 : /* If the original pseudo changed its allocation, just
8333 : removing the optional pseudo is dangerous as the original
8334 : pseudo will have longer live range. */
8335 991777 : || reg_renumber[REGNO (lra_reg_info[regno].restore_rtx)] >= 0)
8336 : keep_p = true;
8337 613685 : else if (reg_renumber[regno] >= 0)
8338 1748404 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi2)
8339 : {
8340 1279739 : insn = lra_insn_recog_data[uid]->insn;
8341 1279739 : if ((set = single_set (insn)) == NULL_RTX)
8342 6029 : continue;
8343 1273710 : int src_regno = get_regno (SET_SRC (set));
8344 1273710 : int dest_regno = get_regno (SET_DEST (set));
8345 1273710 : if (src_regno < 0 || dest_regno < 0)
8346 594697 : continue;
8347 679013 : int restore_regno = get_regno (lra_reg_info[regno].restore_rtx);
8348 679013 : if (restore_regno < 0)
8349 0 : continue;
8350 679013 : if (((dest_regno == (int) regno
8351 : /* Ignore insn for optional reloads itself. */
8352 582767 : && restore_regno != src_regno
8353 : /* Check that the optional reload was inherited. */
8354 116381 : && inheritance_pseudo_p (src_regno))
8355 1145399 : || (src_regno == (int) regno
8356 96246 : && restore_regno != dest_regno
8357 : /* The optional reload is an inheritance source. */
8358 15680 : && inheritance_pseudo_p (dest_regno))))
8359 : {
8360 : keep_p = true;
8361 : break;
8362 : }
8363 : }
8364 626473 : if (keep_p)
8365 : {
8366 510153 : bitmap_clear_bit (removed_optional_reload_pseudos, regno);
8367 510153 : if (lra_dump_file != NULL)
8368 2 : fprintf (lra_dump_file, "Keep optional reload reg %d\n", regno);
8369 : }
8370 : }
8371 1578630 : change_p = ! bitmap_empty_p (removed_optional_reload_pseudos);
8372 1578630 : auto_bitmap insn_bitmap (®_obstack);
8373 2060254 : EXECUTE_IF_SET_IN_BITMAP (removed_optional_reload_pseudos, 0, regno, bi)
8374 : {
8375 481624 : if (lra_dump_file != NULL)
8376 0 : fprintf (lra_dump_file, "Remove optional reload reg %d\n", regno);
8377 481624 : bitmap_copy (insn_bitmap, &lra_reg_info[regno].insn_bitmap);
8378 1515633 : EXECUTE_IF_SET_IN_BITMAP (insn_bitmap, 0, uid, bi2)
8379 : {
8380 : /* We may have already removed a clobber. */
8381 1034009 : if (!lra_insn_recog_data[uid])
8382 0 : continue;
8383 1034009 : insn = lra_insn_recog_data[uid]->insn;
8384 1034009 : if ((set = single_set (insn)) != NULL_RTX)
8385 : {
8386 1028171 : src = SET_SRC (set);
8387 1028171 : dest = SET_DEST (set);
8388 475773 : if ((REG_P (src) || SUBREG_P (src))
8389 552410 : && (REG_P (dest) || SUBREG_P (dest))
8390 1580556 : && ((get_regno (src) == (int) regno
8391 166712 : && (get_regno (lra_reg_info[regno].restore_rtx)
8392 83356 : == get_regno (dest)))
8393 469072 : || (get_regno (dest) == (int) regno
8394 469029 : && (get_regno (lra_reg_info[regno].restore_rtx)
8395 469029 : == get_regno (src)))))
8396 : {
8397 552294 : if (lra_dump_file != NULL)
8398 : {
8399 0 : fprintf (lra_dump_file, " Deleting move %u\n",
8400 0 : INSN_UID (insn));
8401 0 : dump_insn_slim (lra_dump_file, insn);
8402 : }
8403 1104588 : delete_move_and_clobber (insn, get_regno (dest));
8404 552294 : continue;
8405 : }
8406 : /* We should not worry about generation memory-memory
8407 : moves here as if the corresponding inheritance did
8408 : not work (inheritance pseudo did not get a hard reg),
8409 : we remove the inheritance pseudo and the optional
8410 : reload. */
8411 : }
8412 481715 : rtx pat = PATTERN (insn);
8413 0 : if (GET_CODE (pat) == CLOBBER && REG_P (SET_DEST (pat))
8414 481715 : && get_regno (SET_DEST (pat)) == (int) regno)
8415 : /* Refuse to remap clobbers to preexisting pseudos. */
8416 0 : gcc_unreachable ();
8417 481715 : lra_substitute_pseudo_within_insn
8418 481715 : (insn, regno, lra_reg_info[regno].restore_rtx, false);
8419 481715 : lra_update_insn_regno_info (insn);
8420 481715 : if (lra_dump_file != NULL)
8421 : {
8422 0 : fprintf (lra_dump_file,
8423 : " Restoring original insn:\n");
8424 0 : dump_insn_slim (lra_dump_file, insn);
8425 : }
8426 : }
8427 : }
8428 : /* Clear restore_regnos. */
8429 2570407 : EXECUTE_IF_SET_IN_BITMAP (&lra_optional_reload_pseudos, 0, regno, bi)
8430 991777 : lra_reg_info[regno].restore_rtx = NULL_RTX;
8431 1578630 : return change_p;
8432 1578630 : }
8433 :
8434 : /* Entry function for undoing inheritance/split transformation. Return true
8435 : if we did any RTL change in this pass. */
8436 : bool
8437 1581806 : lra_undo_inheritance (void)
8438 : {
8439 1581806 : unsigned int regno;
8440 1581806 : int hard_regno;
8441 1581806 : int n_all_inherit, n_inherit, n_all_split, n_split;
8442 1581806 : rtx restore_rtx;
8443 1581806 : bitmap_iterator bi;
8444 1581806 : bool change_p;
8445 :
8446 1581806 : lra_undo_inheritance_iter++;
8447 1581806 : if (lra_undo_inheritance_iter > LRA_MAX_INHERITANCE_PASSES)
8448 : return false;
8449 1578630 : if (lra_dump_file != NULL)
8450 96 : fprintf (lra_dump_file,
8451 : "\n********** Undoing inheritance #%d: **********\n\n",
8452 : lra_undo_inheritance_iter);
8453 1578630 : auto_bitmap remove_pseudos (®_obstack);
8454 1578630 : n_inherit = n_all_inherit = 0;
8455 3463758 : EXECUTE_IF_SET_IN_BITMAP (&lra_inheritance_pseudos, 0, regno, bi)
8456 1885128 : if (lra_reg_info[regno].restore_rtx != NULL_RTX)
8457 : {
8458 1196070 : n_all_inherit++;
8459 1196070 : if (reg_renumber[regno] < 0
8460 : /* If the original pseudo changed its allocation, just
8461 : removing inheritance is dangerous as for changing
8462 : allocation we used shorter live-ranges. */
8463 1196070 : && (! REG_P (lra_reg_info[regno].restore_rtx)
8464 442933 : || reg_renumber[REGNO (lra_reg_info[regno].restore_rtx)] < 0))
8465 450558 : bitmap_set_bit (remove_pseudos, regno);
8466 : else
8467 745512 : n_inherit++;
8468 : }
8469 1578630 : if (lra_dump_file != NULL && n_all_inherit != 0)
8470 1 : fprintf (lra_dump_file, "Inherit %d out of %d (%.2f%%)\n",
8471 : n_inherit, n_all_inherit,
8472 1 : (double) n_inherit / n_all_inherit * 100);
8473 1578630 : n_split = n_all_split = 0;
8474 2545449 : EXECUTE_IF_SET_IN_BITMAP (&lra_split_regs, 0, regno, bi)
8475 966819 : if ((restore_rtx = lra_reg_info[regno].restore_rtx) != NULL_RTX)
8476 : {
8477 670312 : int restore_regno = REGNO (restore_rtx);
8478 :
8479 670312 : n_all_split++;
8480 1340542 : hard_regno = (restore_regno >= FIRST_PSEUDO_REGISTER
8481 670312 : ? reg_renumber[restore_regno] : restore_regno);
8482 670312 : if (hard_regno < 0 || reg_renumber[regno] == hard_regno)
8483 2332 : bitmap_set_bit (remove_pseudos, regno);
8484 : else
8485 : {
8486 667980 : n_split++;
8487 667980 : if (lra_dump_file != NULL)
8488 0 : fprintf (lra_dump_file, " Keep split r%d (orig=r%d)\n",
8489 : regno, restore_regno);
8490 : }
8491 : }
8492 1578630 : if (lra_dump_file != NULL && n_all_split != 0)
8493 0 : fprintf (lra_dump_file, "Split %d out of %d (%.2f%%)\n",
8494 : n_split, n_all_split,
8495 0 : (double) n_split / n_all_split * 100);
8496 1578630 : change_p = remove_inheritance_pseudos (remove_pseudos);
8497 : /* Clear restore_regnos. */
8498 3463758 : EXECUTE_IF_SET_IN_BITMAP (&lra_inheritance_pseudos, 0, regno, bi)
8499 1885128 : lra_reg_info[regno].restore_rtx = NULL_RTX;
8500 2545449 : EXECUTE_IF_SET_IN_BITMAP (&lra_split_regs, 0, regno, bi)
8501 966819 : lra_reg_info[regno].restore_rtx = NULL_RTX;
8502 1578630 : change_p = undo_optional_reloads () || change_p;
8503 : if (change_p)
8504 110873 : lra_dump_insns_if_possible ("changed func after undoing inheritance");
8505 1578630 : return change_p;
8506 1578630 : }
|
