Line data Source code
1 : /* Code for RTL transformations to satisfy insn constraints.
2 : Copyright (C) 2010-2026 Free Software Foundation, Inc.
3 : Contributed by Vladimir Makarov <vmakarov@redhat.com>.
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 :
22 : /* This file contains code for 3 passes: constraint pass,
23 : inheritance/split pass, and pass for undoing failed inheritance and
24 : split.
25 :
26 : The major goal of constraint pass is to transform RTL to satisfy
27 : insn and address constraints by:
28 : o choosing insn alternatives;
29 : o generating *reload insns* (or reloads in brief) and *reload
30 : pseudos* which will get necessary hard registers later;
31 : o substituting pseudos with equivalent values and removing the
32 : instructions that initialized those pseudos.
33 :
34 : The constraint pass has biggest and most complicated code in LRA.
35 : There are a lot of important details like:
36 : o reuse of input reload pseudos to simplify reload pseudo
37 : allocations;
38 : o some heuristics to choose insn alternative to improve the
39 : inheritance;
40 : o early clobbers etc.
41 :
42 : The pass is mimicking former reload pass in alternative choosing
43 : because the reload pass is oriented to current machine description
44 : model. It might be changed if the machine description model is
45 : changed.
46 :
47 : There is special code for preventing all LRA and this pass cycling
48 : in case of bugs.
49 :
50 : On the first iteration of the pass we process every instruction and
51 : choose an alternative for each one. On subsequent iterations we try
52 : to avoid reprocessing instructions if we can be sure that the old
53 : choice is still valid.
54 :
55 : The inheritance/spilt pass is to transform code to achieve
56 : ineheritance and live range splitting. It is done on backward
57 : traversal of EBBs.
58 :
59 : The inheritance optimization goal is to reuse values in hard
60 : registers. There is analogous optimization in old reload pass. The
61 : inheritance is achieved by following transformation:
62 :
63 : reload_p1 <- p reload_p1 <- p
64 : ... new_p <- reload_p1
65 : ... => ...
66 : reload_p2 <- p reload_p2 <- new_p
67 :
68 : where p is spilled and not changed between the insns. Reload_p1 is
69 : also called *original pseudo* and new_p is called *inheritance
70 : pseudo*.
71 :
72 : The subsequent assignment pass will try to assign the same (or
73 : another if it is not possible) hard register to new_p as to
74 : reload_p1 or reload_p2.
75 :
76 : If the assignment pass fails to assign a hard register to new_p,
77 : this file will undo the inheritance and restore the original code.
78 : This is because implementing the above sequence with a spilled
79 : new_p would make the code much worse. The inheritance is done in
80 : EBB scope. The above is just a simplified example to get an idea
81 : of the inheritance as the inheritance is also done for non-reload
82 : insns.
83 :
84 : Splitting (transformation) is also done in EBB scope on the same
85 : pass as the inheritance:
86 :
87 : r <- ... or ... <- r r <- ... or ... <- r
88 : ... s <- r (new insn -- save)
89 : ... =>
90 : ... r <- s (new insn -- restore)
91 : ... <- r ... <- r
92 :
93 : The *split pseudo* s is assigned to the hard register of the
94 : original pseudo or hard register r.
95 :
96 : Splitting is done:
97 : o In EBBs with high register pressure for global pseudos (living
98 : in at least 2 BBs) and assigned to hard registers when there
99 : are more one reloads needing the hard registers;
100 : o for pseudos needing save/restore code around calls.
101 :
102 : If the split pseudo still has the same hard register as the
103 : original pseudo after the subsequent assignment pass or the
104 : original pseudo was split, the opposite transformation is done on
105 : the same pass for undoing inheritance. */
106 :
107 : #undef REG_OK_STRICT
108 :
109 : #include "config.h"
110 : #include "system.h"
111 : #include "coretypes.h"
112 : #include "backend.h"
113 : #include "hooks.h"
114 : #include "target.h"
115 : #include "rtl.h"
116 : #include "tree.h"
117 : #include "stmt.h"
118 : #include "predict.h"
119 : #include "df.h"
120 : #include "memmodel.h"
121 : #include "tm_p.h"
122 : #include "expmed.h"
123 : #include "optabs.h"
124 : #include "regs.h"
125 : #include "ira.h"
126 : #include "recog.h"
127 : #include "output.h"
128 : #include "addresses.h"
129 : #include "expr.h"
130 : #include "cfgrtl.h"
131 : #include "rtl-error.h"
132 : #include "lra.h"
133 : #include "lra-int.h"
134 : #include "print-rtl.h"
135 : #include "function-abi.h"
136 : #include "rtl-iter.h"
137 : #include "hash-set.h"
138 :
139 : /* Value of LRA_CURR_RELOAD_NUM at the beginning of BB of the current
140 : insn. Remember that LRA_CURR_RELOAD_NUM is the number of emitted
141 : reload insns. */
142 : static int bb_reload_num;
143 :
144 : /* The current insn being processed and corresponding its single set
145 : (NULL otherwise), its data (basic block, the insn data, the insn
146 : static data, and the mode of each operand). */
147 : static rtx_insn *curr_insn;
148 : static rtx curr_insn_set;
149 : static basic_block curr_bb;
150 : static lra_insn_recog_data_t curr_id;
151 : static struct lra_static_insn_data *curr_static_id;
152 : static machine_mode curr_operand_mode[MAX_RECOG_OPERANDS];
153 : /* Mode of the register substituted by its equivalence with VOIDmode
154 : (e.g. constant) and whose subreg is given operand of the current
155 : insn. VOIDmode in all other cases. */
156 : static machine_mode original_subreg_reg_mode[MAX_RECOG_OPERANDS];
157 : /* The first call insn after curr_insn within the EBB during inherit_in_ebb
158 : or NULL outside of that function. */
159 : static rtx_insn *first_call_insn;
160 :
161 :
162 :
163 : /* Start numbers for new registers and insns at the current constraints
164 : pass start. */
165 : static int new_regno_start;
166 : static int new_insn_uid_start;
167 :
168 : /* If LOC is nonnull, strip any outer subreg from it. */
169 : static inline rtx *
170 227841764 : strip_subreg (rtx *loc)
171 : {
172 100895961 : return loc && GET_CODE (*loc) == SUBREG ? &SUBREG_REG (*loc) : loc;
173 : }
174 :
175 : /* Return hard regno of REGNO or if it is was not assigned to a hard
176 : register, use a hard register from its allocno class. */
177 : static int
178 73775 : get_try_hard_regno (int regno)
179 : {
180 73775 : int hard_regno;
181 73775 : enum reg_class rclass;
182 :
183 73775 : if ((hard_regno = regno) >= FIRST_PSEUDO_REGISTER)
184 73775 : hard_regno = lra_get_regno_hard_regno (regno);
185 73775 : if (hard_regno >= 0)
186 : return hard_regno;
187 40307 : rclass = lra_get_allocno_class (regno);
188 40307 : if (rclass == NO_REGS)
189 : return -1;
190 38919 : return ira_class_hard_regs[rclass][0];
191 : }
192 :
193 : /* Return the hard regno of X after removing its subreg. If X is not a
194 : register or a subreg of a register, return -1. If X is a pseudo, use its
195 : assignment. If X is a hard regno, return the final hard regno which will be
196 : after elimination. */
197 : static int
198 287138238 : get_hard_regno (rtx x)
199 : {
200 287138238 : rtx reg;
201 287138238 : int hard_regno;
202 :
203 287138238 : reg = x;
204 287138238 : if (SUBREG_P (x))
205 5152308 : reg = SUBREG_REG (x);
206 287138238 : if (! REG_P (reg))
207 : return -1;
208 198228657 : int regno = REGNO (reg);
209 198228657 : if (HARD_REGISTER_NUM_P (regno))
210 34856683 : hard_regno = lra_get_elimination_hard_regno (regno);
211 : else
212 163371974 : hard_regno = lra_get_regno_hard_regno (regno);
213 198228657 : if (hard_regno < 0)
214 : return -1;
215 180457118 : if (SUBREG_P (x))
216 4455246 : hard_regno += subreg_regno_offset (hard_regno, GET_MODE (reg),
217 4455246 : SUBREG_BYTE (x), GET_MODE (x));
218 : return hard_regno;
219 : }
220 :
221 : /* If REGNO is a hard register or has been allocated a hard register,
222 : return the class of that register. If REGNO is a reload pseudo
223 : created by the current constraints pass, return its allocno class.
224 : Return NO_REGS otherwise. */
225 : static enum reg_class
226 511500221 : get_reg_class (int regno)
227 : {
228 511500221 : int hard_regno;
229 :
230 511500221 : if (HARD_REGISTER_NUM_P (regno))
231 64665429 : hard_regno = lra_get_elimination_hard_regno (regno);
232 : else
233 446834792 : hard_regno = lra_get_regno_hard_regno (regno);
234 511500221 : if (hard_regno >= 0)
235 319973907 : return REGNO_REG_CLASS (hard_regno);
236 191526314 : if (regno >= new_regno_start)
237 61893017 : return lra_get_allocno_class (regno);
238 : return NO_REGS;
239 : }
240 :
241 : /* Return true if REG_CLASS has enough allocatable hard regs to keep value of
242 : REG_MODE. */
243 : static bool
244 18590668 : enough_allocatable_hard_regs_p (enum reg_class reg_class,
245 : enum machine_mode reg_mode)
246 : {
247 18590668 : int i, j, hard_regno, class_size, nregs;
248 :
249 37181336 : if (hard_reg_set_subset_p (reg_class_contents[reg_class], lra_no_alloc_regs))
250 : return false;
251 6305014 : class_size = ira_class_hard_regs_num[reg_class];
252 6305014 : for (i = 0; i < class_size; i++)
253 : {
254 6305014 : hard_regno = ira_class_hard_regs[reg_class][i];
255 6305014 : nregs = hard_regno_nregs (hard_regno, reg_mode);
256 6305014 : if (nregs == 1)
257 : return true;
258 253122 : for (j = 0; j < nregs; j++)
259 168748 : if (TEST_HARD_REG_BIT (lra_no_alloc_regs, hard_regno + j)
260 168748 : || ! TEST_HARD_REG_BIT (reg_class_contents[reg_class],
261 : hard_regno + j))
262 : break;
263 84374 : if (j >= nregs)
264 : return true;
265 : }
266 : return false;
267 : }
268 :
269 : /* True if C is a non-empty register class that has too few registers
270 : to be safely used as a reload target class. */
271 : #define SMALL_REGISTER_CLASS_P(C) \
272 : (ira_class_hard_regs_num [(C)] == 1 \
273 : || (ira_class_hard_regs_num [(C)] >= 1 \
274 : && targetm.class_likely_spilled_p (C)))
275 :
276 : /* Return true if REG satisfies (or will satisfy) reg class constraint
277 : CL. Use elimination first if REG is a hard register. If REG is a
278 : reload pseudo created by this constraints pass, assume that it will
279 : be allocated a hard register from its allocno class, but allow that
280 : class to be narrowed to CL if it is currently a superset of CL and
281 : if either:
282 :
283 : - ALLOW_ALL_RELOAD_CLASS_CHANGES_P is true or
284 : - the instruction we're processing is not a reload move.
285 :
286 : If NEW_CLASS is nonnull, set *NEW_CLASS to the new allocno class of
287 : REGNO (reg), or NO_REGS if no change in its class was needed. */
288 : static bool
289 217748029 : in_class_p (rtx reg, enum reg_class cl, enum reg_class *new_class,
290 : bool allow_all_reload_class_changes_p = false)
291 : {
292 217748029 : enum reg_class rclass, common_class;
293 217748029 : machine_mode reg_mode;
294 217748029 : rtx src;
295 217748029 : int regno = REGNO (reg);
296 :
297 217748029 : if (new_class != NULL)
298 111960569 : *new_class = NO_REGS;
299 217748029 : if (regno < FIRST_PSEUDO_REGISTER)
300 : {
301 27445002 : rtx final_reg = reg;
302 27445002 : rtx *final_loc = &final_reg;
303 :
304 27445002 : lra_eliminate_reg_if_possible (final_loc);
305 27445002 : return TEST_HARD_REG_BIT (reg_class_contents[cl], REGNO (*final_loc));
306 : }
307 190303027 : reg_mode = GET_MODE (reg);
308 190303027 : rclass = get_reg_class (regno);
309 190303027 : src = curr_insn_set != NULL ? SET_SRC (curr_insn_set) : NULL;
310 190303027 : if (regno < new_regno_start
311 : /* Do not allow the constraints for reload instructions to
312 : influence the classes of new pseudos. These reloads are
313 : typically moves that have many alternatives, and restricting
314 : reload pseudos for one alternative may lead to situations
315 : where other reload pseudos are no longer allocatable. */
316 190303027 : || (!allow_all_reload_class_changes_p
317 14979643 : && INSN_UID (curr_insn) >= new_insn_uid_start
318 14450875 : && src != NULL
319 14450875 : && ((REG_P (src) || MEM_P (src))
320 1387495 : || (GET_CODE (src) == SUBREG
321 624321 : && (REG_P (SUBREG_REG (src)) || MEM_P (SUBREG_REG (src)))))))
322 : /* When we don't know what class will be used finally for reload
323 : pseudos, we use ALL_REGS. */
324 13687701 : return ((regno >= new_regno_start && rclass == ALL_REGS)
325 185397660 : || (rclass != NO_REGS && ira_class_subset_p[rclass][cl]
326 200068518 : && ! hard_reg_set_subset_p (reg_class_contents[cl],
327 : lra_no_alloc_regs)));
328 : else
329 : {
330 18590668 : common_class = ira_reg_class_subset[rclass][cl];
331 18590668 : if (new_class != NULL)
332 5238012 : *new_class = common_class;
333 18590668 : return (enough_allocatable_hard_regs_p (common_class, reg_mode)
334 : /* Do not permit reload insn operand matching (new_class == NULL
335 : case) if the new class is too small. */
336 18590668 : && (new_class != NULL || common_class == rclass
337 983220 : || !SMALL_REGISTER_CLASS_P (common_class)));
338 : }
339 : }
340 :
341 : /* Return true if REGNO satisfies a memory constraint. */
342 : static bool
343 63418504 : in_mem_p (int regno)
344 : {
345 0 : return get_reg_class (regno) == NO_REGS;
346 : }
347 :
348 : /* Return true if ADDR is a valid memory address for mode MODE in address
349 : space AS, and check that each pseudo has the proper kind of hard
350 : reg. */
351 : static bool
352 35494685 : valid_address_p (machine_mode mode ATTRIBUTE_UNUSED,
353 : rtx addr, addr_space_t as)
354 : {
355 : #ifdef GO_IF_LEGITIMATE_ADDRESS
356 : lra_assert (ADDR_SPACE_GENERIC_P (as));
357 : GO_IF_LEGITIMATE_ADDRESS (mode, addr, win);
358 : return false;
359 :
360 : win:
361 : return true;
362 : #else
363 0 : return targetm.addr_space.legitimate_address_p (mode, addr, 0, as,
364 35494685 : ERROR_MARK);
365 : #endif
366 : }
367 :
368 : namespace {
369 : /* Temporarily eliminates registers in an address (for the lifetime of
370 : the object). */
371 : class address_eliminator {
372 : public:
373 : address_eliminator (struct address_info *ad);
374 : ~address_eliminator ();
375 :
376 : private:
377 : struct address_info *m_ad;
378 : rtx *m_base_loc;
379 : rtx m_base_reg;
380 : rtx *m_index_loc;
381 : rtx m_index_reg;
382 : };
383 : }
384 :
385 74631733 : address_eliminator::address_eliminator (struct address_info *ad)
386 74631733 : : m_ad (ad),
387 74631733 : m_base_loc (strip_subreg (ad->base_term)),
388 74631733 : m_base_reg (NULL_RTX),
389 74631733 : m_index_loc (strip_subreg (ad->index_term)),
390 74631733 : m_index_reg (NULL_RTX)
391 : {
392 74631733 : if (m_base_loc != NULL)
393 : {
394 62184120 : m_base_reg = *m_base_loc;
395 : /* If we have non-legitimate address which is decomposed not in
396 : the way we expected, don't do elimination here. In such case
397 : the address will be reloaded and elimination will be done in
398 : reload insn finally. */
399 62184120 : if (REG_P (m_base_reg))
400 62184120 : lra_eliminate_reg_if_possible (m_base_loc);
401 62184120 : if (m_ad->base_term2 != NULL)
402 0 : *m_ad->base_term2 = *m_ad->base_term;
403 : }
404 74631733 : if (m_index_loc != NULL)
405 : {
406 3710612 : m_index_reg = *m_index_loc;
407 3710612 : if (REG_P (m_index_reg))
408 3710612 : lra_eliminate_reg_if_possible (m_index_loc);
409 : }
410 74631733 : }
411 :
412 74631733 : address_eliminator::~address_eliminator ()
413 : {
414 74631733 : if (m_base_loc && *m_base_loc != m_base_reg)
415 : {
416 43806628 : *m_base_loc = m_base_reg;
417 43806628 : if (m_ad->base_term2 != NULL)
418 0 : *m_ad->base_term2 = *m_ad->base_term;
419 : }
420 74631733 : if (m_index_loc && *m_index_loc != m_index_reg)
421 0 : *m_index_loc = m_index_reg;
422 74631733 : }
423 :
424 : /* Return true if the eliminated form of AD is a legitimate target address.
425 : If OP is a MEM, AD is the address within OP, otherwise OP should be
426 : ignored. CONSTRAINT is one constraint that the operand may need
427 : to meet. */
428 : static bool
429 35472717 : valid_address_p (rtx op, struct address_info *ad,
430 : enum constraint_num constraint)
431 : {
432 35472717 : address_eliminator eliminator (ad);
433 :
434 : /* Allow a memory OP if it matches CONSTRAINT, even if CONSTRAINT is more
435 : forgiving than "m".
436 : Need to extract memory from op for special memory constraint,
437 : i.e. bcst_mem_operand in i386 backend. */
438 35472717 : if (MEM_P (extract_mem_from_operand (op))
439 : && insn_extra_relaxed_memory_constraint (constraint)
440 : && constraint_satisfied_p (op, constraint))
441 : return true;
442 :
443 35472717 : return valid_address_p (ad->mode, *ad->outer, ad->as);
444 35472717 : }
445 :
446 : /* For special_memory_operand, it could be false for MEM_P (op),
447 : i.e. bcst_mem_operand in i386 backend.
448 : Extract and return real memory operand or op. */
449 : rtx
450 620697498 : extract_mem_from_operand (rtx op)
451 : {
452 622361107 : for (rtx x = op;; x = XEXP (x, 0))
453 : {
454 622361107 : if (MEM_P (x))
455 : return x;
456 441919162 : if (GET_RTX_LENGTH (GET_CODE (x)) != 1
457 362151532 : || GET_RTX_FORMAT (GET_CODE (x))[0] != 'e')
458 : break;
459 : }
460 : return op;
461 : }
462 :
463 : /* Return true if the eliminated form of memory reference OP satisfies
464 : extra (special) memory constraint CONSTRAINT. */
465 : static bool
466 36761118 : satisfies_memory_constraint_p (rtx op, enum constraint_num constraint)
467 : {
468 36761118 : struct address_info ad;
469 36761118 : rtx mem = extract_mem_from_operand (op);
470 36761118 : if (!MEM_P (mem))
471 : return false;
472 :
473 35759852 : decompose_mem_address (&ad, mem);
474 35759852 : address_eliminator eliminator (&ad);
475 35759852 : return constraint_satisfied_p (op, constraint);
476 35759852 : }
477 :
478 : /* Return true if the eliminated form of address AD satisfies extra
479 : address constraint CONSTRAINT. */
480 : static bool
481 3399164 : satisfies_address_constraint_p (struct address_info *ad,
482 : enum constraint_num constraint)
483 : {
484 3399164 : address_eliminator eliminator (ad);
485 3399164 : return constraint_satisfied_p (*ad->outer, constraint);
486 3399164 : }
487 :
488 : /* Return true if the eliminated form of address OP satisfies extra
489 : address constraint CONSTRAINT. */
490 : static bool
491 1666773 : satisfies_address_constraint_p (rtx op, enum constraint_num constraint)
492 : {
493 1666773 : struct address_info ad;
494 :
495 1666773 : decompose_lea_address (&ad, &op);
496 1666773 : return satisfies_address_constraint_p (&ad, constraint);
497 : }
498 :
499 : /* Set of equivalences whose original targets have set up pointer flag. */
500 : static hash_set <rtx> *pointer_equiv_set;
501 :
502 : /* Add x to pointer_equiv_set. */
503 : void
504 1901005 : lra_pointer_equiv_set_add (rtx x)
505 : {
506 1901005 : pointer_equiv_set->add (x);
507 1901005 : }
508 :
509 : /* Return true if x is in pointer_equiv_set. */
510 : bool
511 9684424 : lra_pointer_equiv_set_in (rtx x)
512 : {
513 9684424 : return pointer_equiv_set->contains (x);
514 : }
515 :
516 : /* Initiate equivalences for LRA. As we keep original equivalences
517 : before any elimination, we need to make copies otherwise any change
518 : in insns might change the equivalences. */
519 : void
520 1474414 : lra_init_equiv (void)
521 : {
522 1474414 : ira_expand_reg_equiv ();
523 69411469 : for (int i = FIRST_PSEUDO_REGISTER; i < max_reg_num (); i++)
524 : {
525 67937055 : rtx res;
526 :
527 67937055 : if ((res = ira_reg_equiv[i].memory) != NULL_RTX)
528 3030945 : ira_reg_equiv[i].memory = copy_rtx (res);
529 67937055 : if ((res = ira_reg_equiv[i].invariant) != NULL_RTX)
530 871074 : ira_reg_equiv[i].invariant = copy_rtx (res);
531 : }
532 1474414 : pointer_equiv_set = new hash_set <rtx>;
533 1474414 : }
534 :
535 : /* Finish equivalence data for LRA. */
536 : void
537 1474414 : lra_finish_equiv (void)
538 : {
539 2948828 : delete pointer_equiv_set;
540 1474414 : }
541 :
542 : static rtx loc_equivalence_callback (rtx, const_rtx, void *);
543 :
544 : /* Update equivalence for REGNO. We need to this as the equivalence
545 : might contain other pseudos which are changed by their
546 : equivalences. */
547 : static void
548 201757423 : update_equiv (int regno)
549 : {
550 201757423 : rtx x;
551 :
552 201757423 : if ((x = ira_reg_equiv[regno].memory) != NULL_RTX)
553 9259866 : ira_reg_equiv[regno].memory
554 9259866 : = simplify_replace_fn_rtx (x, NULL_RTX, loc_equivalence_callback,
555 : NULL_RTX);
556 201757423 : if ((x = ira_reg_equiv[regno].invariant) != NULL_RTX)
557 2704183 : ira_reg_equiv[regno].invariant
558 2704183 : = simplify_replace_fn_rtx (x, NULL_RTX, loc_equivalence_callback,
559 : NULL_RTX);
560 201757423 : }
561 :
562 : /* If we have decided to substitute X with another value, return that
563 : value, otherwise return X. */
564 : static rtx
565 434606586 : get_equiv (rtx x)
566 : {
567 434606586 : int regno;
568 434606586 : rtx res;
569 :
570 294178233 : if (! REG_P (x) || (regno = REGNO (x)) < FIRST_PSEUDO_REGISTER
571 195313791 : || regno >= ira_reg_equiv_len
572 195313791 : || ! ira_reg_equiv[regno].defined_p
573 25449645 : || ! ira_reg_equiv[regno].profitable_p
574 460012633 : || lra_get_regno_hard_regno (regno) >= 0)
575 429720835 : return x;
576 4885751 : if ((res = ira_reg_equiv[regno].memory) != NULL_RTX)
577 : {
578 2181237 : if (targetm.cannot_substitute_mem_equiv_p (res))
579 : return x;
580 : return res;
581 : }
582 2704514 : if ((res = ira_reg_equiv[regno].constant) != NULL_RTX)
583 : return res;
584 1857822 : if ((res = ira_reg_equiv[regno].invariant) != NULL_RTX)
585 : return res;
586 0 : gcc_unreachable ();
587 : }
588 :
589 : /* If we have decided to substitute X with the equivalent value, return that
590 : value after elimination for INSN, otherwise return X. Add the result to
591 : pointer_equiv_set if X has set up pointer flag. */
592 : static rtx
593 244577959 : get_equiv_with_elimination (rtx x, rtx_insn *insn)
594 : {
595 244577959 : rtx res = get_equiv (x);
596 :
597 244577959 : if (x == res || CONSTANT_P (res))
598 : return res;
599 1489046 : res = lra_eliminate_regs_1 (insn, res, GET_MODE (res),
600 : false, false, 0, true);
601 1489046 : if (REG_POINTER (x))
602 1027884 : lra_pointer_equiv_set_add (res);
603 : return res;
604 : }
605 :
606 : /* Set up curr_operand_mode. */
607 : static void
608 105432430 : init_curr_operand_mode (void)
609 : {
610 105432430 : int nop = curr_static_id->n_operands;
611 328599138 : for (int i = 0; i < nop; i++)
612 : {
613 223166708 : machine_mode mode = GET_MODE (*curr_id->operand_loc[i]);
614 223166708 : if (mode == VOIDmode)
615 : {
616 : /* The .md mode for address operands is the mode of the
617 : addressed value rather than the mode of the address itself. */
618 43184029 : if (curr_id->icode >= 0 && curr_static_id->operand[i].is_address)
619 95 : mode = Pmode;
620 : else
621 43183934 : mode = curr_static_id->operand[i].mode;
622 : }
623 223166708 : curr_operand_mode[i] = mode;
624 : }
625 105432430 : }
626 :
627 :
628 :
629 : /* The page contains code to reuse input reloads. */
630 :
631 : /* Structure describes input reload of the current insns. */
632 : struct input_reload
633 : {
634 : /* True for input reload of matched operands. */
635 : bool match_p;
636 : /* True for input reload of inout earlyclobber operand. */
637 : bool early_clobber_p;
638 : /* Reloaded value. */
639 : rtx input;
640 : /* Reload pseudo used. */
641 : rtx reg;
642 : };
643 :
644 : /* The number of elements in the following array. */
645 : static int curr_insn_input_reloads_num;
646 : /* Array containing info about input reloads. It is used to find the
647 : same input reload and reuse the reload pseudo in this case. */
648 : static struct input_reload curr_insn_input_reloads[LRA_MAX_INSN_RELOADS];
649 :
650 : /* Initiate data concerning reuse of input reloads for the current
651 : insn. */
652 : static void
653 105432430 : init_curr_insn_input_reloads (void)
654 : {
655 105432430 : curr_insn_input_reloads_num = 0;
656 0 : }
657 :
658 : /* The canonical form of an rtx inside a MEM is not necessarily the same as the
659 : canonical form of the rtx outside the MEM. Fix this up in the case that
660 : we're reloading an address (and therefore pulling it outside a MEM). */
661 : static rtx
662 72 : canonicalize_reload_addr (rtx addr)
663 : {
664 72 : subrtx_var_iterator::array_type array;
665 246 : FOR_EACH_SUBRTX_VAR (iter, array, addr, NONCONST)
666 : {
667 174 : rtx x = *iter;
668 174 : if (GET_CODE (x) == MULT && CONST_INT_P (XEXP (x, 1)))
669 : {
670 14 : const HOST_WIDE_INT ci = INTVAL (XEXP (x, 1));
671 188 : const int pwr2 = exact_log2 (ci);
672 14 : if (pwr2 > 0)
673 : {
674 : /* Rewrite this to use a shift instead, which is canonical when
675 : outside of a MEM. */
676 14 : PUT_CODE (x, ASHIFT);
677 14 : XEXP (x, 1) = GEN_INT (pwr2);
678 : }
679 : }
680 : }
681 :
682 72 : return addr;
683 72 : }
684 :
685 : /* Return rtx accessing reload REG of RCLASS matching another reload reg in
686 : MODE. */
687 : static rtx
688 122822 : get_matching_reload_reg_subreg (machine_mode mode, rtx reg,
689 : enum reg_class rclass)
690 : {
691 122822 : int hard_regno = ira_class_hard_regs[rclass][0];
692 122822 : if (subreg_regno_offset (hard_regno,
693 122822 : GET_MODE (reg),
694 122822 : subreg_lowpart_offset (mode, GET_MODE (reg)),
695 : mode) == 0)
696 : /* For matching scalar int modes generate the right subreg byte offset for
697 : BE targets -- see call of reload.cc:operands_match_p in
698 : recog.cc:constrain_operands. */
699 122822 : return lowpart_subreg (mode, reg, GET_MODE (reg));
700 0 : int offset = (lra_constraint_offset (hard_regno, GET_MODE (reg))
701 0 : - lra_constraint_offset (hard_regno, mode)) * UNITS_PER_WORD;
702 0 : lra_assert (offset >= 0);
703 0 : return gen_rtx_SUBREG (mode, reg, offset);
704 : }
705 :
706 : /* Create a new pseudo using MODE, RCLASS, EXCLUDE_START_HARD_REGS, ORIGINAL or
707 : reuse an existing reload pseudo. Don't reuse an existing reload pseudo if
708 : IN_SUBREG_P is true and the reused pseudo should be wrapped up in a SUBREG.
709 : EARLY_CLOBBER_P is true for input reload of inout early clobber operand.
710 : The result pseudo is returned through RESULT_REG. Return TRUE if we created
711 : a new pseudo, FALSE if we reused an existing reload pseudo. Use TITLE to
712 : describe new registers for debug purposes. */
713 : static bool
714 3796685 : get_reload_reg (enum op_type type, machine_mode mode, rtx original,
715 : enum reg_class rclass, HARD_REG_SET *exclude_start_hard_regs,
716 : bool in_subreg_p, bool early_clobber_p,
717 : const char *title, rtx *result_reg)
718 : {
719 3796685 : int i, regno;
720 3796685 : enum reg_class new_class;
721 :
722 3796685 : if (type == OP_OUT)
723 : {
724 : /* Output reload registers tend to start out with a conservative
725 : choice of register class. Usually this is ALL_REGS, although
726 : a target might narrow it (for performance reasons) through
727 : targetm.preferred_reload_class. It's therefore quite common
728 : for a reload instruction to require a more restrictive class
729 : than the class that was originally assigned to the reload register.
730 :
731 : In these situations, it's more efficient to refine the choice
732 : of register class rather than create a second reload register.
733 : This also helps to avoid cycling for registers that are only
734 : used by reload instructions. */
735 961342 : if (REG_P (original)
736 704793 : && (int) REGNO (original) >= new_regno_start
737 7170 : && (INSN_UID (curr_insn) >= new_insn_uid_start
738 250 : || ira_former_scratch_p (REGNO (original)))
739 7170 : && in_class_p (original, rclass, &new_class, true)
740 961592 : && (exclude_start_hard_regs == nullptr
741 250 : || hard_reg_set_intersect_p (
742 961592 : ~lra_reg_info[REGNO (original)].exclude_start_hard_regs,
743 250 : ~*exclude_start_hard_regs)))
744 : {
745 250 : unsigned int regno = REGNO (original);
746 250 : if (lra_dump_file != NULL)
747 : {
748 0 : fprintf (lra_dump_file, " Reuse r%d for output ", regno);
749 0 : dump_value_slim (lra_dump_file, original, 1);
750 : }
751 500 : if (new_class != lra_get_allocno_class (regno))
752 250 : lra_change_class (regno, new_class, ", change to", false);
753 250 : if (lra_dump_file != NULL)
754 0 : fprintf (lra_dump_file, "\n");
755 250 : if (exclude_start_hard_regs)
756 250 : lra_reg_info[regno].exclude_start_hard_regs
757 250 : |= *exclude_start_hard_regs;
758 250 : *result_reg = original;
759 250 : return false;
760 : }
761 961092 : *result_reg
762 961092 : = lra_create_new_reg_with_unique_value (mode, original, rclass,
763 : exclude_start_hard_regs, title);
764 961092 : return true;
765 : }
766 :
767 2835343 : bool unique_p = early_clobber_p;
768 : /* Prevent reuse value of expression with side effects,
769 : e.g. volatile memory. */
770 2835343 : if (! side_effects_p (original))
771 3055624 : for (i = 0; i < curr_insn_input_reloads_num; i++)
772 : {
773 239317 : if (! curr_insn_input_reloads[i].match_p
774 100818 : && ! curr_insn_input_reloads[i].early_clobber_p
775 100817 : && rtx_equal_p (curr_insn_input_reloads[i].input, original)
776 248325 : && in_class_p (curr_insn_input_reloads[i].reg, rclass, &new_class))
777 : {
778 8995 : rtx reg = curr_insn_input_reloads[i].reg;
779 8995 : regno = REGNO (reg);
780 : /* If input is equal to original and both are VOIDmode,
781 : GET_MODE (reg) might be still different from mode.
782 : Ensure we don't return *result_reg with wrong mode. */
783 8995 : if (GET_MODE (reg) != mode)
784 : {
785 0 : if (in_subreg_p)
786 0 : continue;
787 0 : if (maybe_lt (GET_MODE_SIZE (GET_MODE (reg)),
788 0 : GET_MODE_SIZE (mode)))
789 0 : continue;
790 0 : reg = get_matching_reload_reg_subreg (mode, reg, new_class);
791 0 : if (reg == NULL_RTX || GET_CODE (reg) != SUBREG)
792 0 : continue;
793 : }
794 : /* If the existing reload and this have no start hard register in
795 : common, then skip. Otherwise update exclude_start_hard_regs. */
796 8995 : if (exclude_start_hard_regs
797 10243 : && ! hard_reg_set_empty_p (*exclude_start_hard_regs))
798 : {
799 1 : HARD_REG_SET r = lra_reg_info[regno].exclude_start_hard_regs
800 1 : | *exclude_start_hard_regs;
801 2 : if (hard_reg_set_empty_p (~r))
802 0 : continue;
803 : else
804 1 : lra_reg_info[regno].exclude_start_hard_regs = r;
805 : }
806 8995 : *result_reg = reg;
807 8995 : if (lra_dump_file != NULL)
808 : {
809 0 : fprintf (lra_dump_file, " Reuse r%d for reload ", regno);
810 0 : dump_value_slim (lra_dump_file, original, 1);
811 : }
812 17990 : if (new_class != lra_get_allocno_class (regno))
813 4333 : lra_change_class (regno, new_class, ", change to", false);
814 8995 : if (lra_dump_file != NULL)
815 0 : fprintf (lra_dump_file, "\n");
816 8995 : return false;
817 : }
818 : /* If we have an input reload with a different mode, make sure it
819 : will get a different hard reg. */
820 230322 : else if (REG_P (original)
821 181499 : && REG_P (curr_insn_input_reloads[i].input)
822 148647 : && REGNO (original) == REGNO (curr_insn_input_reloads[i].input)
823 230322 : && (GET_MODE (original)
824 2121 : != GET_MODE (curr_insn_input_reloads[i].input)))
825 : unique_p = true;
826 : }
827 5652696 : *result_reg = (unique_p
828 2826348 : ? lra_create_new_reg_with_unique_value
829 2826348 : : lra_create_new_reg) (mode, original, rclass,
830 : exclude_start_hard_regs, title);
831 2826348 : lra_assert (curr_insn_input_reloads_num < LRA_MAX_INSN_RELOADS);
832 2826348 : curr_insn_input_reloads[curr_insn_input_reloads_num].input = original;
833 2826348 : curr_insn_input_reloads[curr_insn_input_reloads_num].match_p = false;
834 2826348 : curr_insn_input_reloads[curr_insn_input_reloads_num].early_clobber_p
835 2826348 : = early_clobber_p;
836 2826348 : curr_insn_input_reloads[curr_insn_input_reloads_num++].reg = *result_reg;
837 2826348 : return true;
838 : }
839 :
840 :
841 : /* The page contains major code to choose the current insn alternative
842 : and generate reloads for it. */
843 :
844 : /* Return the offset from REGNO of the least significant register
845 : in (reg:MODE REGNO).
846 :
847 : This function is used to tell whether two registers satisfy
848 : a matching constraint. (reg:MODE1 REGNO1) matches (reg:MODE2 REGNO2) if:
849 :
850 : REGNO1 + lra_constraint_offset (REGNO1, MODE1)
851 : == REGNO2 + lra_constraint_offset (REGNO2, MODE2) */
852 : int
853 43002804 : lra_constraint_offset (int regno, machine_mode mode)
854 : {
855 43002804 : lra_assert (regno < FIRST_PSEUDO_REGISTER);
856 :
857 43002804 : scalar_int_mode int_mode;
858 43002804 : if (WORDS_BIG_ENDIAN
859 : && is_a <scalar_int_mode> (mode, &int_mode)
860 : && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD)
861 : return hard_regno_nregs (regno, mode) - 1;
862 43002804 : return 0;
863 : }
864 :
865 : /* Like rtx_equal_p except that it allows a REG and a SUBREG to match
866 : if they are the same hard reg, and has special hacks for
867 : auto-increment and auto-decrement. This is specifically intended for
868 : process_alt_operands to use in determining whether two operands
869 : match. X is the operand whose number is the lower of the two.
870 :
871 : It is supposed that X is the output operand and Y is the input
872 : operand. Y_HARD_REGNO is the final hard regno of register Y or
873 : register in subreg Y as we know it now. Otherwise, it is a
874 : negative value. */
875 : static bool
876 57348101 : operands_match_p (rtx x, rtx y, int y_hard_regno)
877 : {
878 57348101 : int i;
879 57348101 : RTX_CODE code = GET_CODE (x);
880 57348101 : const char *fmt;
881 :
882 57348101 : if (x == y)
883 : return true;
884 49703551 : if ((code == REG || (code == SUBREG && REG_P (SUBREG_REG (x))))
885 23469778 : && (REG_P (y) || (GET_CODE (y) == SUBREG && REG_P (SUBREG_REG (y)))))
886 : {
887 23376710 : int j;
888 :
889 23376710 : i = get_hard_regno (x);
890 23376710 : if (i < 0)
891 1206664 : goto slow;
892 :
893 22170046 : if ((j = y_hard_regno) < 0)
894 668644 : goto slow;
895 :
896 21501402 : i += lra_constraint_offset (i, GET_MODE (x));
897 21501402 : j += lra_constraint_offset (j, GET_MODE (y));
898 :
899 21501402 : return i == j;
900 : }
901 :
902 : /* If two operands must match, because they are really a single
903 : operand of an assembler insn, then two post-increments are invalid
904 : because the assembler insn would increment only once. On the
905 : other hand, a post-increment matches ordinary indexing if the
906 : post-increment is the output operand. */
907 26326841 : if (code == POST_DEC || code == POST_INC || code == POST_MODIFY)
908 0 : return operands_match_p (XEXP (x, 0), y, y_hard_regno);
909 :
910 : /* Two pre-increments are invalid because the assembler insn would
911 : increment only once. On the other hand, a pre-increment matches
912 : ordinary indexing if the pre-increment is the input operand. */
913 26326841 : if (GET_CODE (y) == PRE_DEC || GET_CODE (y) == PRE_INC
914 26326841 : || GET_CODE (y) == PRE_MODIFY)
915 0 : return operands_match_p (x, XEXP (y, 0), -1);
916 :
917 26326841 : slow:
918 :
919 28202149 : if (code == REG && REG_P (y))
920 1779152 : return REGNO (x) == REGNO (y);
921 :
922 93137 : if (code == REG && GET_CODE (y) == SUBREG && REG_P (SUBREG_REG (y))
923 7646 : && x == SUBREG_REG (y))
924 : return true;
925 26422997 : if (GET_CODE (y) == REG && code == SUBREG && REG_P (SUBREG_REG (x))
926 63502 : && SUBREG_REG (x) == y)
927 : return true;
928 :
929 : /* Now we have disposed of all the cases in which different rtx
930 : codes can match. */
931 26422823 : if (code != GET_CODE (y))
932 : return false;
933 :
934 : /* (MULT:SI x y) and (MULT:HI x y) are NOT equivalent. */
935 1042822 : if (GET_MODE (x) != GET_MODE (y))
936 : return false;
937 :
938 1042121 : switch (code)
939 : {
940 : CASE_CONST_UNIQUE:
941 : return false;
942 :
943 : case CONST_VECTOR:
944 : if (!same_vector_encodings_p (x, y))
945 : return false;
946 : break;
947 :
948 0 : case LABEL_REF:
949 0 : return label_ref_label (x) == label_ref_label (y);
950 25 : case SYMBOL_REF:
951 25 : return XSTR (x, 0) == XSTR (y, 0);
952 :
953 : default:
954 : break;
955 : }
956 :
957 : /* Compare the elements. If any pair of corresponding elements fail
958 : to match, return false for the whole things. */
959 :
960 1021277 : fmt = GET_RTX_FORMAT (code);
961 2957845 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
962 : {
963 2010577 : int val, j;
964 2010577 : switch (fmt[i])
965 : {
966 0 : case 'w':
967 0 : if (XWINT (x, i) != XWINT (y, i))
968 : return false;
969 : break;
970 :
971 488 : case 'i':
972 488 : if (XINT (x, i) != XINT (y, i))
973 : return false;
974 : break;
975 :
976 0 : case 'L':
977 0 : if (XLOC (x, i) != XLOC (y, i))
978 : return false;
979 : break;
980 :
981 24920 : case 'p':
982 24920 : if (maybe_ne (SUBREG_BYTE (x), SUBREG_BYTE (y)))
983 : return false;
984 : break;
985 :
986 1462593 : case 'e':
987 1462593 : val = operands_match_p (XEXP (x, i), XEXP (y, i), -1);
988 1462593 : if (val == 0)
989 : return false;
990 : break;
991 :
992 : case '0':
993 : break;
994 :
995 488 : case 'E':
996 488 : if (XVECLEN (x, i) != XVECLEN (y, i))
997 : return false;
998 976 : for (j = XVECLEN (x, i) - 1; j >= 0; --j)
999 : {
1000 488 : val = operands_match_p (XVECEXP (x, i, j), XVECEXP (y, i, j), -1);
1001 488 : if (val == 0)
1002 : return false;
1003 : }
1004 : break;
1005 :
1006 : /* It is believed that rtx's at this level will never
1007 : contain anything but integers and other rtx's, except for
1008 : within LABEL_REFs and SYMBOL_REFs. */
1009 0 : default:
1010 0 : gcc_unreachable ();
1011 : }
1012 : }
1013 : return true;
1014 : }
1015 :
1016 : /* True if X is a constant that can be forced into the constant pool.
1017 : MODE is the mode of the operand, or VOIDmode if not known. */
1018 : #define CONST_POOL_OK_P(MODE, X) \
1019 : ((MODE) != VOIDmode \
1020 : && CONSTANT_P (X) \
1021 : && GET_CODE (X) != HIGH \
1022 : && GET_MODE_SIZE (MODE).is_constant () \
1023 : && !targetm.cannot_force_const_mem (MODE, X))
1024 :
1025 : /* If REG is a reload pseudo, try to make its class satisfying CL. */
1026 : static void
1027 3361266 : narrow_reload_pseudo_class (rtx reg, enum reg_class cl)
1028 : {
1029 3361266 : enum reg_class rclass;
1030 :
1031 : /* Do not make more accurate class from reloads generated. They are
1032 : mostly moves with a lot of constraints. Making more accurate
1033 : class may results in very narrow class and impossibility of find
1034 : registers for several reloads of one insn. */
1035 3361266 : if (INSN_UID (curr_insn) >= new_insn_uid_start)
1036 3361234 : return;
1037 3361166 : if (GET_CODE (reg) == SUBREG)
1038 164027 : reg = SUBREG_REG (reg);
1039 3361166 : if (! REG_P (reg) || (int) REGNO (reg) < new_regno_start)
1040 : return;
1041 32 : if (in_class_p (reg, cl, &rclass) && rclass != cl)
1042 13 : lra_change_class (REGNO (reg), rclass, " Change to", true);
1043 : }
1044 :
1045 : /* Searches X for any reference to a reg with the same value as REGNO,
1046 : returning the rtx of the reference found if any. Otherwise,
1047 : returns NULL_RTX. */
1048 : static rtx
1049 523578 : regno_val_use_in (unsigned int regno, rtx x)
1050 : {
1051 523578 : const char *fmt;
1052 523578 : int i, j;
1053 523578 : rtx tem;
1054 :
1055 523578 : if (REG_P (x) && lra_reg_info[REGNO (x)].val == lra_reg_info[regno].val)
1056 : return x;
1057 :
1058 523252 : fmt = GET_RTX_FORMAT (GET_CODE (x));
1059 1052881 : for (i = GET_RTX_LENGTH (GET_CODE (x)) - 1; i >= 0; i--)
1060 : {
1061 529629 : if (fmt[i] == 'e')
1062 : {
1063 7659 : if ((tem = regno_val_use_in (regno, XEXP (x, i))))
1064 : return tem;
1065 : }
1066 521970 : else if (fmt[i] == 'E')
1067 0 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
1068 0 : if ((tem = regno_val_use_in (regno , XVECEXP (x, i, j))))
1069 : return tem;
1070 : }
1071 :
1072 : return NULL_RTX;
1073 : }
1074 :
1075 : /* Return true if all current insn non-output operands except INS (it
1076 : has a negaitve end marker) do not use pseudos with the same value
1077 : as REGNO. */
1078 : static bool
1079 2 : check_conflict_input_operands (int regno, signed char *ins)
1080 : {
1081 2 : int in;
1082 2 : int n_operands = curr_static_id->n_operands;
1083 :
1084 8 : for (int nop = 0; nop < n_operands; nop++)
1085 7 : if (! curr_static_id->operand[nop].is_operator
1086 7 : && curr_static_id->operand[nop].type != OP_OUT)
1087 : {
1088 5 : for (int i = 0; (in = ins[i]) >= 0; i++)
1089 4 : if (in == nop)
1090 : break;
1091 3 : if (in < 0
1092 3 : && regno_val_use_in (regno, *curr_id->operand_loc[nop]) != NULL_RTX)
1093 : return false;
1094 : }
1095 : return true;
1096 : }
1097 :
1098 : /* Generate reloads for matching OUT and INS (array of input operand numbers
1099 : with end marker -1) with reg class GOAL_CLASS and EXCLUDE_START_HARD_REGS,
1100 : considering output operands OUTS (similar array to INS) needing to be in
1101 : different registers. Add input and output reloads correspondingly to the
1102 : lists *BEFORE and *AFTER. OUT might be negative. In this case we generate
1103 : input reloads for matched input operands INS. EARLY_CLOBBER_P is a flag
1104 : that the output operand is early clobbered for chosen alternative. */
1105 : static void
1106 1680633 : match_reload (signed char out, signed char *ins, signed char *outs,
1107 : enum reg_class goal_class, HARD_REG_SET *exclude_start_hard_regs,
1108 : rtx_insn **before, rtx_insn **after, bool early_clobber_p)
1109 : {
1110 1680633 : bool out_conflict;
1111 1680633 : int i, in;
1112 1680633 : rtx new_in_reg, new_out_reg, reg;
1113 1680633 : machine_mode inmode, outmode;
1114 1680633 : rtx in_rtx = *curr_id->operand_loc[ins[0]];
1115 1680633 : rtx out_rtx = out < 0 ? in_rtx : *curr_id->operand_loc[out];
1116 :
1117 1680633 : inmode = curr_operand_mode[ins[0]];
1118 1680633 : outmode = out < 0 ? inmode : curr_operand_mode[out];
1119 1680633 : push_to_sequence (*before);
1120 1680633 : if (inmode != outmode)
1121 : {
1122 : /* process_alt_operands has already checked that the mode sizes
1123 : are ordered. */
1124 122822 : if (partial_subreg_p (outmode, inmode))
1125 : {
1126 1749 : bool asm_p = asm_noperands (PATTERN (curr_insn)) >= 0;
1127 1749 : int hr;
1128 1749 : HARD_REG_SET temp_hard_reg_set;
1129 :
1130 19 : if (asm_p && (hr = get_hard_regno (out_rtx)) >= 0
1131 1752 : && hard_regno_nregs (hr, inmode) > 1)
1132 : {
1133 : /* See gcc.c-torture/execute/20030222-1.c.
1134 : Consider the code for 32-bit (e.g. BE) target:
1135 : int i, v; long x; x = v; asm ("" : "=r" (i) : "0" (x));
1136 : We generate the following RTL with reload insns:
1137 : 1. subreg:si(x:di, 0) = 0;
1138 : 2. subreg:si(x:di, 4) = v:si;
1139 : 3. t:di = x:di, dead x;
1140 : 4. asm ("" : "=r" (subreg:si(t:di,4)) : "0" (t:di))
1141 : 5. i:si = subreg:si(t:di,4);
1142 : If we assign hard reg of x to t, dead code elimination
1143 : will remove insn #2 and we will use unitialized hard reg.
1144 : So exclude the hard reg of x for t. We could ignore this
1145 : problem for non-empty asm using all x value but it is hard to
1146 : check that the asm are expanded into insn realy using x
1147 : and setting r. */
1148 0 : CLEAR_HARD_REG_SET (temp_hard_reg_set);
1149 0 : if (exclude_start_hard_regs != NULL)
1150 0 : temp_hard_reg_set = *exclude_start_hard_regs;
1151 0 : SET_HARD_REG_BIT (temp_hard_reg_set, hr);
1152 0 : exclude_start_hard_regs = &temp_hard_reg_set;
1153 : }
1154 3498 : reg = new_in_reg
1155 1749 : = lra_create_new_reg_with_unique_value (inmode, in_rtx, goal_class,
1156 : exclude_start_hard_regs,
1157 : "");
1158 1749 : new_out_reg = get_matching_reload_reg_subreg (outmode, reg, goal_class);
1159 1749 : LRA_SUBREG_P (new_out_reg) = 1;
1160 : /* If the input reg is dying here, we can use the same hard
1161 : register for REG and IN_RTX. We do it only for original
1162 : pseudos as reload pseudos can die although original
1163 : pseudos still live where reload pseudos dies. */
1164 1514 : if (REG_P (in_rtx) && (int) REGNO (in_rtx) < lra_new_regno_start
1165 1465 : && find_regno_note (curr_insn, REG_DEAD, REGNO (in_rtx))
1166 2772 : && (!early_clobber_p
1167 2 : || check_conflict_input_operands(REGNO (in_rtx), ins)))
1168 1022 : lra_assign_reg_val (REGNO (in_rtx), REGNO (reg));
1169 : }
1170 : else
1171 : {
1172 242146 : reg = new_out_reg
1173 121073 : = lra_create_new_reg_with_unique_value (outmode, out_rtx,
1174 : goal_class,
1175 : exclude_start_hard_regs,
1176 : "");
1177 121073 : new_in_reg = get_matching_reload_reg_subreg (inmode, reg, goal_class);
1178 : /* NEW_IN_REG is non-paradoxical subreg. We don't want
1179 : NEW_OUT_REG living above. We add clobber clause for
1180 : this. This is just a temporary clobber. We can remove
1181 : it at the end of LRA work. */
1182 121073 : rtx_insn *clobber = emit_clobber (new_out_reg);
1183 121073 : LRA_TEMP_CLOBBER_P (PATTERN (clobber)) = 1;
1184 121073 : LRA_SUBREG_P (new_in_reg) = 1;
1185 121073 : if (GET_CODE (in_rtx) == SUBREG)
1186 : {
1187 1736 : rtx subreg_reg = SUBREG_REG (in_rtx);
1188 :
1189 : /* If SUBREG_REG is dying here and sub-registers IN_RTX
1190 : and NEW_IN_REG are similar, we can use the same hard
1191 : register for REG and SUBREG_REG. */
1192 1736 : if (REG_P (subreg_reg)
1193 1736 : && (int) REGNO (subreg_reg) < lra_new_regno_start
1194 1736 : && GET_MODE (subreg_reg) == outmode
1195 1083 : && known_eq (SUBREG_BYTE (in_rtx), SUBREG_BYTE (new_in_reg))
1196 1083 : && find_regno_note (curr_insn, REG_DEAD, REGNO (subreg_reg))
1197 1838 : && (! early_clobber_p
1198 0 : || check_conflict_input_operands (REGNO (subreg_reg),
1199 : ins)))
1200 102 : lra_assign_reg_val (REGNO (subreg_reg), REGNO (reg));
1201 : }
1202 : }
1203 : }
1204 : else
1205 : {
1206 : /* Pseudos have values -- see comments for lra_reg_info.
1207 : Different pseudos with the same value do not conflict even if
1208 : they live in the same place. When we create a pseudo we
1209 : assign value of original pseudo (if any) from which we
1210 : created the new pseudo. If we create the pseudo from the
1211 : input pseudo, the new pseudo will have no conflict with the
1212 : input pseudo which is wrong when the input pseudo lives after
1213 : the insn and as the new pseudo value is changed by the insn
1214 : output. Therefore we create the new pseudo from the output
1215 : except the case when we have single matched dying input
1216 : pseudo.
1217 :
1218 : We cannot reuse the current output register because we might
1219 : have a situation like "a <- a op b", where the constraints
1220 : force the second input operand ("b") to match the output
1221 : operand ("a"). "b" must then be copied into a new register
1222 : so that it doesn't clobber the current value of "a".
1223 :
1224 : We cannot use the same value if the output pseudo is
1225 : early clobbered or the input pseudo is mentioned in the
1226 : output, e.g. as an address part in memory, because
1227 : output reload will actually extend the pseudo liveness.
1228 : We don't care about eliminable hard regs here as we are
1229 : interesting only in pseudos. */
1230 :
1231 : /* Matching input's register value is the same as one of the other
1232 : output operand. Output operands in a parallel insn must be in
1233 : different registers. */
1234 1557811 : out_conflict = false;
1235 1557811 : if (REG_P (in_rtx))
1236 : {
1237 2683444 : for (i = 0; outs[i] >= 0; i++)
1238 : {
1239 1392002 : rtx other_out_rtx = *curr_id->operand_loc[outs[i]];
1240 100330 : if (outs[i] != out && REG_P (other_out_rtx)
1241 1492138 : && (regno_val_use_in (REGNO (in_rtx), other_out_rtx)
1242 : != NULL_RTX))
1243 : {
1244 : out_conflict = true;
1245 : break;
1246 : }
1247 : }
1248 : }
1249 :
1250 1557811 : new_in_reg = new_out_reg
1251 1527028 : = (! early_clobber_p && ins[1] < 0 && REG_P (in_rtx)
1252 1261717 : && (int) REGNO (in_rtx) < lra_new_regno_start
1253 1261428 : && find_regno_note (curr_insn, REG_DEAD, REGNO (in_rtx))
1254 415782 : && (out < 0
1255 415782 : || regno_val_use_in (REGNO (in_rtx), out_rtx) == NULL_RTX)
1256 415750 : && !out_conflict
1257 1973559 : ? lra_create_new_reg (inmode, in_rtx, goal_class,
1258 : exclude_start_hard_regs, "")
1259 1142063 : : lra_create_new_reg_with_unique_value (outmode, out_rtx, goal_class,
1260 : exclude_start_hard_regs,
1261 : ""));
1262 : }
1263 : /* In operand can be got from transformations before processing insn
1264 : constraints. One example of such transformations is subreg
1265 : reloading (see function simplify_operand_subreg). The new
1266 : pseudos created by the transformations might have inaccurate
1267 : class (ALL_REGS) and we should make their classes more
1268 : accurate. */
1269 1680633 : narrow_reload_pseudo_class (in_rtx, goal_class);
1270 1680633 : lra_emit_move (copy_rtx (new_in_reg), in_rtx);
1271 1680633 : *before = end_sequence ();
1272 : /* Add the new pseudo to consider values of subsequent input reload
1273 : pseudos. */
1274 1680633 : lra_assert (curr_insn_input_reloads_num < LRA_MAX_INSN_RELOADS);
1275 1680633 : curr_insn_input_reloads[curr_insn_input_reloads_num].input = in_rtx;
1276 1680633 : curr_insn_input_reloads[curr_insn_input_reloads_num].match_p = true;
1277 1680633 : curr_insn_input_reloads[curr_insn_input_reloads_num].early_clobber_p = false;
1278 1680633 : curr_insn_input_reloads[curr_insn_input_reloads_num++].reg = new_in_reg;
1279 3361267 : for (i = 0; (in = ins[i]) >= 0; i++)
1280 1680634 : if (GET_MODE (*curr_id->operand_loc[in]) == VOIDmode
1281 1652814 : || GET_MODE (new_in_reg) == GET_MODE (*curr_id->operand_loc[in]))
1282 1680633 : *curr_id->operand_loc[in] = new_in_reg;
1283 : else
1284 : {
1285 1 : lra_assert
1286 : (GET_MODE (new_out_reg) == GET_MODE (*curr_id->operand_loc[in]));
1287 1 : *curr_id->operand_loc[in] = new_out_reg;
1288 : }
1289 1680633 : lra_update_dups (curr_id, ins);
1290 1680633 : if (out < 0)
1291 : return;
1292 : /* See a comment for the input operand above. */
1293 1680633 : narrow_reload_pseudo_class (out_rtx, goal_class);
1294 1680633 : reg = SUBREG_P (out_rtx) ? SUBREG_REG (out_rtx) : out_rtx;
1295 1680633 : if (find_reg_note (curr_insn, REG_UNUSED, reg) == NULL_RTX
1296 1680633 : && (!REG_P (reg) || !ira_former_scratch_p (REGNO (reg))))
1297 : {
1298 1596806 : start_sequence ();
1299 : /* If we had strict_low_part, use it also in reload to keep other
1300 : parts unchanged but do it only for regs as strict_low_part
1301 : has no sense for memory and probably there is no insn pattern
1302 : to match the reload insn in memory case. */
1303 1596806 : if (out >= 0 && curr_static_id->operand[out].strict_low && REG_P (reg))
1304 0 : out_rtx = gen_rtx_STRICT_LOW_PART (VOIDmode, out_rtx);
1305 1596806 : lra_emit_move (out_rtx, copy_rtx (new_out_reg));
1306 1596806 : emit_insn (*after);
1307 1596806 : *after = end_sequence ();
1308 : }
1309 1680633 : *curr_id->operand_loc[out] = new_out_reg;
1310 1680633 : lra_update_dup (curr_id, out);
1311 : }
1312 :
1313 : /* Return register class which is union of all reg classes in insn
1314 : constraint alternative string starting with P. */
1315 : static enum reg_class
1316 0 : reg_class_from_constraints (const char *p)
1317 : {
1318 0 : int c, len;
1319 0 : enum reg_class op_class = NO_REGS;
1320 :
1321 0 : do
1322 0 : switch ((c = *p, len = CONSTRAINT_LEN (c, p)), c)
1323 : {
1324 : case '#':
1325 : case ',':
1326 : return op_class;
1327 :
1328 0 : case 'g':
1329 0 : op_class = reg_class_subunion[op_class][GENERAL_REGS];
1330 0 : break;
1331 :
1332 0 : default:
1333 0 : enum constraint_num cn = lookup_constraint (p);
1334 0 : enum reg_class cl = reg_class_for_constraint (cn);
1335 0 : if (cl == NO_REGS)
1336 : {
1337 0 : if (insn_extra_address_constraint (cn))
1338 0 : op_class
1339 0 : = (reg_class_subunion
1340 0 : [op_class][base_reg_class (VOIDmode, ADDR_SPACE_GENERIC,
1341 0 : ADDRESS, SCRATCH)]);
1342 : break;
1343 : }
1344 :
1345 0 : op_class = reg_class_subunion[op_class][cl];
1346 0 : break;
1347 : }
1348 0 : while ((p += len), c);
1349 : return op_class;
1350 : }
1351 :
1352 : /* If OP is a register, return the class of the register as per
1353 : get_reg_class, otherwise return NO_REGS. */
1354 : static inline enum reg_class
1355 161896422 : get_op_class (rtx op)
1356 : {
1357 134333655 : return REG_P (op) ? get_reg_class (REGNO (op)) : NO_REGS;
1358 : }
1359 :
1360 : /* Return generated insn mem_pseudo:=val if TO_P or val:=mem_pseudo
1361 : otherwise. If modes of MEM_PSEUDO and VAL are different, use
1362 : SUBREG for VAL to make them equal. */
1363 : static rtx_insn *
1364 1314893 : emit_spill_move (bool to_p, rtx mem_pseudo, rtx val)
1365 : {
1366 1314893 : if (GET_MODE (mem_pseudo) != GET_MODE (val))
1367 : {
1368 : /* Usually size of mem_pseudo is greater than val size but in
1369 : rare cases it can be less as it can be defined by target
1370 : dependent macro HARD_REGNO_CALLER_SAVE_MODE. */
1371 2922 : if (! MEM_P (val))
1372 : {
1373 2922 : val = gen_lowpart_SUBREG (GET_MODE (mem_pseudo),
1374 : GET_CODE (val) == SUBREG
1375 : ? SUBREG_REG (val) : val);
1376 2922 : LRA_SUBREG_P (val) = 1;
1377 : }
1378 : else
1379 : {
1380 0 : mem_pseudo = gen_lowpart_SUBREG (GET_MODE (val), mem_pseudo);
1381 0 : LRA_SUBREG_P (mem_pseudo) = 1;
1382 : }
1383 : }
1384 1314893 : return to_p ? gen_move_insn (mem_pseudo, val)
1385 664012 : : gen_move_insn (val, mem_pseudo);
1386 : }
1387 :
1388 : /* Process a special case insn (register move), return true if we
1389 : don't need to process it anymore. INSN should be a single set
1390 : insn. Set up that RTL was changed through CHANGE_P and that hook
1391 : TARGET_SECONDARY_MEMORY_NEEDED says to use secondary memory through
1392 : SEC_MEM_P. */
1393 : static bool
1394 75149070 : check_and_process_move (bool *change_p, bool *sec_mem_p ATTRIBUTE_UNUSED)
1395 : {
1396 75149070 : int sregno, dregno;
1397 75149070 : rtx dest, src, dreg, sreg, new_reg, scratch_reg;
1398 75149070 : rtx_insn *before;
1399 75149070 : enum reg_class dclass, sclass, secondary_class;
1400 75149070 : secondary_reload_info sri;
1401 :
1402 75149070 : lra_assert (curr_insn_set != NULL_RTX);
1403 75149070 : dreg = dest = SET_DEST (curr_insn_set);
1404 75149070 : sreg = src = SET_SRC (curr_insn_set);
1405 75149070 : if (GET_CODE (dest) == SUBREG)
1406 1188067 : dreg = SUBREG_REG (dest);
1407 75149070 : if (GET_CODE (src) == SUBREG)
1408 1185499 : sreg = SUBREG_REG (src);
1409 75149070 : if (! (REG_P (dreg) || MEM_P (dreg)) || ! (REG_P (sreg) || MEM_P (sreg)))
1410 : return false;
1411 35339818 : sclass = dclass = NO_REGS;
1412 35339818 : if (REG_P (dreg))
1413 22749711 : dclass = get_reg_class (REGNO (dreg));
1414 22749711 : gcc_assert (dclass < LIM_REG_CLASSES && dclass >= NO_REGS);
1415 35339818 : if (dclass == ALL_REGS)
1416 : /* ALL_REGS is used for new pseudos created by transformations
1417 : like reload of SUBREG_REG (see function
1418 : simplify_operand_subreg). We don't know their class yet. We
1419 : should figure out the class from processing the insn
1420 : constraints not in this fast path function. Even if ALL_REGS
1421 : were a right class for the pseudo, secondary_... hooks usually
1422 : are not define for ALL_REGS. */
1423 : return false;
1424 35337593 : if (REG_P (sreg))
1425 19706722 : sclass = get_reg_class (REGNO (sreg));
1426 19706722 : gcc_assert (sclass < LIM_REG_CLASSES && sclass >= NO_REGS);
1427 35337593 : if (sclass == ALL_REGS)
1428 : /* See comments above. */
1429 : return false;
1430 35337593 : if (sclass == NO_REGS && dclass == NO_REGS)
1431 : return false;
1432 33875100 : if (targetm.secondary_memory_needed (GET_MODE (src), sclass, dclass)
1433 33875100 : && ((sclass != NO_REGS && dclass != NO_REGS)
1434 0 : || (GET_MODE (src)
1435 0 : != targetm.secondary_memory_needed_mode (GET_MODE (src)))))
1436 : {
1437 13323 : *sec_mem_p = true;
1438 13323 : return false;
1439 : }
1440 33861777 : if (! REG_P (dreg) || ! REG_P (sreg))
1441 : return false;
1442 7627846 : sri.prev_sri = NULL;
1443 7627846 : sri.icode = CODE_FOR_nothing;
1444 7627846 : sri.extra_cost = 0;
1445 7627846 : secondary_class = NO_REGS;
1446 : /* Set up hard register for a reload pseudo for hook
1447 : secondary_reload because some targets just ignore unassigned
1448 : pseudos in the hook. */
1449 7627846 : if (dclass != NO_REGS && lra_get_regno_hard_regno (REGNO (dreg)) < 0)
1450 : {
1451 2863243 : dregno = REGNO (dreg);
1452 2863243 : reg_renumber[dregno] = ira_class_hard_regs[dclass][0];
1453 : }
1454 : else
1455 : dregno = -1;
1456 7627846 : if (sclass != NO_REGS && lra_get_regno_hard_regno (REGNO (sreg)) < 0)
1457 : {
1458 1263861 : sregno = REGNO (sreg);
1459 1263861 : reg_renumber[sregno] = ira_class_hard_regs[sclass][0];
1460 : }
1461 : else
1462 : sregno = -1;
1463 7627846 : if (sclass != NO_REGS)
1464 3836689 : secondary_class
1465 7673378 : = (enum reg_class) targetm.secondary_reload (false, dest,
1466 : (reg_class_t) sclass,
1467 3836689 : GET_MODE (src), &sri);
1468 3836689 : if (sclass == NO_REGS
1469 3836689 : || ((secondary_class != NO_REGS || sri.icode != CODE_FOR_nothing)
1470 1355 : && dclass != NO_REGS))
1471 : {
1472 3791157 : enum reg_class old_sclass = secondary_class;
1473 3791157 : secondary_reload_info old_sri = sri;
1474 :
1475 3791157 : sri.prev_sri = NULL;
1476 3791157 : sri.icode = CODE_FOR_nothing;
1477 3791157 : sri.extra_cost = 0;
1478 3791157 : secondary_class
1479 7582314 : = (enum reg_class) targetm.secondary_reload (true, src,
1480 : (reg_class_t) dclass,
1481 3791157 : GET_MODE (src), &sri);
1482 : /* Check the target hook consistency. */
1483 3791157 : lra_assert
1484 : ((secondary_class == NO_REGS && sri.icode == CODE_FOR_nothing)
1485 : || (old_sclass == NO_REGS && old_sri.icode == CODE_FOR_nothing)
1486 : || (secondary_class == old_sclass && sri.icode == old_sri.icode));
1487 : }
1488 7627846 : if (sregno >= 0)
1489 1263861 : reg_renumber [sregno] = -1;
1490 7627846 : if (dregno >= 0)
1491 2863243 : reg_renumber [dregno] = -1;
1492 7627846 : if (secondary_class == NO_REGS && sri.icode == CODE_FOR_nothing)
1493 : return false;
1494 1356 : *change_p = true;
1495 1356 : new_reg = NULL_RTX;
1496 0 : if (secondary_class != NO_REGS)
1497 1356 : new_reg = lra_create_new_reg_with_unique_value (GET_MODE (src), NULL_RTX,
1498 : secondary_class, NULL,
1499 : "secondary");
1500 1356 : start_sequence ();
1501 1356 : if (sri.icode == CODE_FOR_nothing)
1502 1356 : lra_emit_move (new_reg, src);
1503 : else
1504 : {
1505 0 : enum reg_class scratch_class;
1506 :
1507 0 : scratch_class = (reg_class_from_constraints
1508 0 : (insn_data[sri.icode].operand[2].constraint));
1509 0 : scratch_reg = (lra_create_new_reg_with_unique_value
1510 0 : (insn_data[sri.icode].operand[2].mode, NULL_RTX,
1511 : scratch_class, NULL, "scratch"));
1512 0 : emit_insn (GEN_FCN (sri.icode) (new_reg != NULL_RTX ? new_reg : dest,
1513 : src, scratch_reg));
1514 : }
1515 1356 : before = end_sequence ();
1516 1356 : lra_process_new_insns (curr_insn, before, NULL, "Inserting the move");
1517 1356 : if (new_reg != NULL_RTX)
1518 1356 : SET_SRC (curr_insn_set) = new_reg;
1519 : else
1520 : {
1521 0 : if (lra_dump_file != NULL)
1522 : {
1523 0 : fprintf (lra_dump_file, "Deleting move %u\n", INSN_UID (curr_insn));
1524 0 : dump_insn_slim (lra_dump_file, curr_insn);
1525 : }
1526 0 : lra_set_insn_deleted (curr_insn);
1527 0 : return true;
1528 : }
1529 1356 : return false;
1530 : }
1531 :
1532 : /* The following data describe the result of process_alt_operands.
1533 : The data are used in curr_insn_transform to generate reloads. */
1534 :
1535 : /* The chosen reg classes which should be used for the corresponding
1536 : operands. */
1537 : static enum reg_class goal_alt[MAX_RECOG_OPERANDS];
1538 : /* Hard registers which cannot be a start hard register for the corresponding
1539 : operands. */
1540 : static HARD_REG_SET goal_alt_exclude_start_hard_regs[MAX_RECOG_OPERANDS];
1541 : /* True if the operand should be the same as another operand and that
1542 : other operand does not need a reload. */
1543 : static bool goal_alt_match_win[MAX_RECOG_OPERANDS];
1544 : /* True if the operand does not need a reload. */
1545 : static bool goal_alt_win[MAX_RECOG_OPERANDS];
1546 : /* True if the operand can be offsetable memory. */
1547 : static bool goal_alt_offmemok[MAX_RECOG_OPERANDS];
1548 : /* The number of an operand to which given operand can be matched to. */
1549 : static int goal_alt_matches[MAX_RECOG_OPERANDS];
1550 : /* The number of elements in the following array. */
1551 : static int goal_alt_dont_inherit_ops_num;
1552 : /* Numbers of operands whose reload pseudos should not be inherited. */
1553 : static int goal_alt_dont_inherit_ops[MAX_RECOG_OPERANDS];
1554 : /* True if we should try only this alternative for the next constraint sub-pass
1555 : to speed up the sub-pass. */
1556 : static bool goal_reuse_alt_p;
1557 : /* True if the insn commutative operands should be swapped. */
1558 : static bool goal_alt_swapped;
1559 : /* The chosen insn alternative. */
1560 : static int goal_alt_number;
1561 : /* True if output reload of the stack pointer should be generated. */
1562 : static bool goal_alt_out_sp_reload_p;
1563 :
1564 : /* True if the corresponding operand is the result of an equivalence
1565 : substitution. */
1566 : static bool equiv_substition_p[MAX_RECOG_OPERANDS];
1567 :
1568 : /* The following five variables are used to choose the best insn
1569 : alternative. They reflect final characteristics of the best
1570 : alternative. */
1571 :
1572 : /* Number of necessary reloads and overall cost reflecting the
1573 : previous value and other unpleasantness of the best alternative. */
1574 : static int best_losers, best_overall;
1575 : /* Overall number hard registers used for reloads. For example, on
1576 : some targets we need 2 general registers to reload DFmode and only
1577 : one floating point register. */
1578 : static int best_reload_nregs;
1579 : /* Overall number reflecting distances of previous reloading the same
1580 : value. The distances are counted from the current BB start. It is
1581 : used to improve inheritance chances. */
1582 : static int best_reload_sum;
1583 :
1584 : /* True if the current insn should have no correspondingly input or
1585 : output reloads. */
1586 : static bool no_input_reloads_p, no_output_reloads_p;
1587 :
1588 : /* True if we swapped the commutative operands in the current
1589 : insn. */
1590 : static int curr_swapped;
1591 :
1592 : /* if CHECK_ONLY_P is false, arrange for address element *LOC to be a
1593 : register of class CL. Add any input reloads to list BEFORE. AFTER
1594 : is nonnull if *LOC is an automodified value; handle that case by
1595 : adding the required output reloads to list AFTER. Return true if
1596 : the RTL was changed.
1597 :
1598 : if CHECK_ONLY_P is true, check that the *LOC is a correct address
1599 : register. Return false if the address register is correct. */
1600 : static bool
1601 35008469 : process_addr_reg (rtx *loc, bool check_only_p, rtx_insn **before, rtx_insn **after,
1602 : enum reg_class cl)
1603 : {
1604 35008469 : int regno;
1605 35008469 : enum reg_class rclass, new_class;
1606 35008469 : rtx reg;
1607 35008469 : rtx new_reg;
1608 35008469 : machine_mode mode;
1609 35008469 : bool subreg_p, before_p = false;
1610 :
1611 35008469 : subreg_p = GET_CODE (*loc) == SUBREG;
1612 35008469 : if (subreg_p)
1613 : {
1614 14174 : reg = SUBREG_REG (*loc);
1615 14174 : mode = GET_MODE (reg);
1616 :
1617 : /* For mode with size bigger than ptr_mode, there unlikely to be "mov"
1618 : between two registers with different classes, but there normally will
1619 : be "mov" which transfers element of vector register into the general
1620 : register, and this normally will be a subreg which should be reloaded
1621 : as a whole. This is particularly likely to be triggered when
1622 : -fno-split-wide-types specified. */
1623 14174 : if (!REG_P (reg)
1624 14174 : || in_class_p (reg, cl, &new_class)
1625 16352 : || known_le (GET_MODE_SIZE (mode), GET_MODE_SIZE (ptr_mode)))
1626 14174 : loc = &SUBREG_REG (*loc);
1627 : }
1628 :
1629 35008469 : reg = *loc;
1630 35008469 : mode = GET_MODE (reg);
1631 35008469 : if (! REG_P (reg))
1632 : {
1633 0 : if (check_only_p)
1634 : return true;
1635 : /* Always reload memory in an address even if the target supports
1636 : such addresses. */
1637 0 : new_reg = lra_create_new_reg_with_unique_value (mode, reg, cl, NULL,
1638 : "address");
1639 0 : before_p = true;
1640 : }
1641 : else
1642 : {
1643 35008469 : regno = REGNO (reg);
1644 35008469 : rclass = get_reg_class (regno);
1645 35008469 : if (! check_only_p
1646 35008469 : && (*loc = get_equiv_with_elimination (reg, curr_insn)) != reg)
1647 : {
1648 111274 : if (lra_dump_file != NULL)
1649 : {
1650 0 : fprintf (lra_dump_file,
1651 : "Changing pseudo %d in address of insn %u on equiv ",
1652 0 : REGNO (reg), INSN_UID (curr_insn));
1653 0 : dump_value_slim (lra_dump_file, *loc, 1);
1654 0 : fprintf (lra_dump_file, "\n");
1655 : }
1656 111274 : rtx new_equiv = copy_rtx (*loc);
1657 111274 : if (lra_pointer_equiv_set_in (*loc))
1658 106128 : lra_pointer_equiv_set_add (new_equiv);
1659 111274 : *loc = new_equiv;
1660 : }
1661 35008469 : if (*loc != reg || ! in_class_p (reg, cl, &new_class))
1662 : {
1663 476505 : if (check_only_p)
1664 : return true;
1665 476505 : reg = *loc;
1666 476505 : if (get_reload_reg (after == NULL ? OP_IN : OP_INOUT,
1667 : mode, reg, cl, NULL,
1668 : subreg_p, false, "address", &new_reg))
1669 : before_p = true;
1670 : }
1671 34531964 : else if (new_class != NO_REGS && rclass != new_class)
1672 : {
1673 465134 : if (check_only_p)
1674 : return true;
1675 465134 : lra_change_class (regno, new_class, " Change to", true);
1676 465134 : return false;
1677 : }
1678 : else
1679 : return false;
1680 : }
1681 0 : if (before_p)
1682 : {
1683 468758 : push_to_sequence (*before);
1684 468758 : lra_emit_move (new_reg, reg);
1685 468758 : *before = end_sequence ();
1686 : }
1687 476505 : *loc = new_reg;
1688 476505 : if (after != NULL)
1689 : {
1690 0 : start_sequence ();
1691 0 : lra_emit_move (before_p ? copy_rtx (reg) : reg, new_reg);
1692 0 : emit_insn (*after);
1693 0 : *after = end_sequence ();
1694 : }
1695 : return true;
1696 : }
1697 :
1698 : /* Insert move insn in simplify_operand_subreg. BEFORE returns
1699 : the insn to be inserted before curr insn. AFTER returns the
1700 : the insn to be inserted after curr insn. ORIGREG and NEWREG
1701 : are the original reg and new reg for reload. */
1702 : static void
1703 458 : insert_move_for_subreg (rtx_insn **before, rtx_insn **after, rtx origreg,
1704 : rtx newreg)
1705 : {
1706 458 : if (before)
1707 : {
1708 458 : push_to_sequence (*before);
1709 458 : lra_emit_move (newreg, origreg);
1710 458 : *before = end_sequence ();
1711 : }
1712 458 : if (after)
1713 : {
1714 0 : start_sequence ();
1715 0 : lra_emit_move (origreg, newreg);
1716 0 : emit_insn (*after);
1717 0 : *after = end_sequence ();
1718 : }
1719 458 : }
1720 :
1721 : static bool valid_address_p (machine_mode mode, rtx addr, addr_space_t as);
1722 : static bool process_address (int, bool, rtx_insn **, rtx_insn **);
1723 :
1724 : /* Make reloads for subreg in operand NOP with internal subreg mode
1725 : REG_MODE, add new reloads for further processing. Return true if
1726 : any change was done. */
1727 : static bool
1728 174541339 : simplify_operand_subreg (int nop, machine_mode reg_mode)
1729 : {
1730 174541339 : int hard_regno, inner_hard_regno;
1731 174541339 : rtx_insn *before, *after;
1732 174541339 : machine_mode mode, innermode;
1733 174541339 : rtx reg, new_reg;
1734 174541339 : rtx operand = *curr_id->operand_loc[nop];
1735 174541339 : enum reg_class regclass;
1736 174541339 : enum op_type type;
1737 :
1738 174541339 : before = after = NULL;
1739 :
1740 174541339 : if (GET_CODE (operand) != SUBREG)
1741 : return false;
1742 :
1743 3638908 : mode = GET_MODE (operand);
1744 3638908 : reg = SUBREG_REG (operand);
1745 3638908 : innermode = GET_MODE (reg);
1746 3638908 : type = curr_static_id->operand[nop].type;
1747 3638908 : if (MEM_P (reg))
1748 : {
1749 10984 : const bool addr_was_valid
1750 10984 : = valid_address_p (innermode, XEXP (reg, 0), MEM_ADDR_SPACE (reg));
1751 10984 : alter_subreg (curr_id->operand_loc[nop], false);
1752 10984 : rtx subst = *curr_id->operand_loc[nop];
1753 10984 : lra_assert (MEM_P (subst));
1754 10984 : const bool addr_is_valid = valid_address_p (GET_MODE (subst),
1755 : XEXP (subst, 0),
1756 10984 : MEM_ADDR_SPACE (subst));
1757 10984 : if (!addr_was_valid
1758 10984 : || addr_is_valid
1759 10984 : || ((get_constraint_type (lookup_constraint
1760 0 : (curr_static_id->operand[nop].constraint))
1761 : != CT_SPECIAL_MEMORY)
1762 : /* We still can reload address and if the address is
1763 : valid, we can remove subreg without reloading its
1764 : inner memory. */
1765 0 : && valid_address_p (GET_MODE (subst),
1766 0 : regno_reg_rtx
1767 : [ira_class_hard_regs
1768 0 : [base_reg_class (GET_MODE (subst),
1769 0 : MEM_ADDR_SPACE (subst),
1770 0 : ADDRESS, SCRATCH)][0]],
1771 0 : MEM_ADDR_SPACE (subst))))
1772 : {
1773 : /* If we change the address for a paradoxical subreg of memory, the
1774 : new address might violate the necessary alignment or the access
1775 : might be slow; take this into consideration. We need not worry
1776 : about accesses beyond allocated memory for paradoxical memory
1777 : subregs as we don't substitute such equiv memory (see processing
1778 : equivalences in function lra_constraints) and because for spilled
1779 : pseudos we allocate stack memory enough for the biggest
1780 : corresponding paradoxical subreg.
1781 :
1782 : However, do not blindly simplify a (subreg (mem ...)) for
1783 : WORD_REGISTER_OPERATIONS targets as this may lead to loading junk
1784 : data into a register when the inner is narrower than outer or
1785 : missing important data from memory when the inner is wider than
1786 : outer. This rule only applies to modes that are no wider than
1787 : a word.
1788 :
1789 : If valid memory becomes invalid after subreg elimination
1790 : and address might be different we still have to reload
1791 : memory.
1792 : */
1793 10984 : if ((! addr_was_valid
1794 : || addr_is_valid
1795 0 : || known_eq (GET_MODE_SIZE (mode), GET_MODE_SIZE (innermode)))
1796 10984 : && !(maybe_ne (GET_MODE_PRECISION (mode),
1797 10984 : GET_MODE_PRECISION (innermode))
1798 13181 : && known_le (GET_MODE_SIZE (mode), UNITS_PER_WORD)
1799 19244 : && known_le (GET_MODE_SIZE (innermode), UNITS_PER_WORD)
1800 : && WORD_REGISTER_OPERATIONS)
1801 23096 : && (!(MEM_ALIGN (subst) < GET_MODE_ALIGNMENT (mode)
1802 1128 : && targetm.slow_unaligned_access (mode, MEM_ALIGN (subst)))
1803 0 : || (MEM_ALIGN (reg) < GET_MODE_ALIGNMENT (innermode)
1804 0 : && targetm.slow_unaligned_access (innermode,
1805 0 : MEM_ALIGN (reg)))))
1806 10984 : return true;
1807 :
1808 0 : *curr_id->operand_loc[nop] = operand;
1809 :
1810 : /* But if the address was not valid, we cannot reload the MEM without
1811 : reloading the address first. */
1812 0 : if (!addr_was_valid)
1813 0 : process_address (nop, false, &before, &after);
1814 :
1815 : /* INNERMODE is fast, MODE slow. Reload the mem in INNERMODE. */
1816 0 : enum reg_class rclass
1817 0 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
1818 0 : if (get_reload_reg (curr_static_id->operand[nop].type, innermode,
1819 : reg, rclass, NULL,
1820 : true, false, "slow/invalid mem", &new_reg))
1821 : {
1822 0 : bool insert_before, insert_after;
1823 0 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
1824 :
1825 0 : insert_before = (type != OP_OUT
1826 0 : || partial_subreg_p (mode, innermode));
1827 0 : insert_after = type != OP_IN;
1828 0 : insert_move_for_subreg (insert_before ? &before : NULL,
1829 : insert_after ? &after : NULL,
1830 : reg, new_reg);
1831 : }
1832 0 : SUBREG_REG (operand) = new_reg;
1833 :
1834 : /* Convert to MODE. */
1835 0 : reg = operand;
1836 0 : rclass
1837 0 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
1838 0 : if (get_reload_reg (curr_static_id->operand[nop].type, mode, reg,
1839 : rclass, NULL,
1840 : true, false, "slow/invalid mem", &new_reg))
1841 : {
1842 0 : bool insert_before, insert_after;
1843 0 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
1844 :
1845 0 : insert_before = type != OP_OUT;
1846 0 : insert_after = type != OP_IN;
1847 0 : insert_move_for_subreg (insert_before ? &before : NULL,
1848 : insert_after ? &after : NULL,
1849 : reg, new_reg);
1850 : }
1851 0 : *curr_id->operand_loc[nop] = new_reg;
1852 0 : lra_process_new_insns (curr_insn, before, after,
1853 : "Inserting slow/invalid mem reload");
1854 0 : return true;
1855 : }
1856 :
1857 : /* If the address was valid and became invalid, prefer to reload
1858 : the memory. Typical case is when the index scale should
1859 : correspond the memory. */
1860 0 : *curr_id->operand_loc[nop] = operand;
1861 : /* Do not return false here as the MEM_P (reg) will be processed
1862 : later in this function. */
1863 : }
1864 3627924 : else if (REG_P (reg) && REGNO (reg) < FIRST_PSEUDO_REGISTER)
1865 : {
1866 263 : alter_subreg (curr_id->operand_loc[nop], false);
1867 263 : return true;
1868 : }
1869 3627661 : else if (CONSTANT_P (reg))
1870 : {
1871 : /* Try to simplify subreg of constant. It is usually result of
1872 : equivalence substitution. */
1873 43463 : if (innermode == VOIDmode
1874 43463 : && (innermode = original_subreg_reg_mode[nop]) == VOIDmode)
1875 0 : innermode = curr_static_id->operand[nop].mode;
1876 43463 : if ((new_reg = simplify_subreg (mode, reg, innermode,
1877 43463 : SUBREG_BYTE (operand))) != NULL_RTX)
1878 : {
1879 43046 : *curr_id->operand_loc[nop] = new_reg;
1880 43046 : return true;
1881 : }
1882 : }
1883 : /* Put constant into memory when we have mixed modes. It generates
1884 : a better code in most cases as it does not need a secondary
1885 : reload memory. It also prevents LRA looping when LRA is using
1886 : secondary reload memory again and again. */
1887 834 : if (CONSTANT_P (reg) && CONST_POOL_OK_P (reg_mode, reg)
1888 3585032 : && SCALAR_INT_MODE_P (reg_mode) != SCALAR_INT_MODE_P (mode))
1889 : {
1890 8 : SUBREG_REG (operand) = force_const_mem (reg_mode, reg);
1891 8 : alter_subreg (curr_id->operand_loc[nop], false);
1892 8 : return true;
1893 : }
1894 3584607 : auto fp_subreg_can_be_simplified_after_reload_p = [] (machine_mode innermode,
1895 : poly_uint64 offset,
1896 : machine_mode mode) {
1897 0 : reload_completed = 1;
1898 0 : bool res = simplify_subreg_regno (FRAME_POINTER_REGNUM,
1899 : innermode,
1900 0 : offset, mode) >= 0;
1901 0 : reload_completed = 0;
1902 0 : return res;
1903 : };
1904 : /* Force a reload of the SUBREG_REG if this is a constant or PLUS or
1905 : if there may be a problem accessing OPERAND in the outer
1906 : mode. */
1907 3584607 : if ((REG_P (reg)
1908 3584149 : && REGNO (reg) >= FIRST_PSEUDO_REGISTER
1909 3584149 : && (hard_regno = lra_get_regno_hard_regno (REGNO (reg))) >= 0
1910 : /* Don't reload paradoxical subregs because we could be looping
1911 : having repeatedly final regno out of hard regs range. */
1912 3027697 : && (hard_regno_nregs (hard_regno, innermode)
1913 3027697 : >= hard_regno_nregs (hard_regno, mode))
1914 3022613 : && simplify_subreg_regno (hard_regno, innermode,
1915 3022613 : SUBREG_BYTE (operand), mode) < 0
1916 : /* Exclude reloading of frame pointer in subreg if frame pointer can not
1917 : be simplified here only because the reload is not finished yet. */
1918 845 : && (hard_regno != FRAME_POINTER_REGNUM
1919 0 : || !fp_subreg_can_be_simplified_after_reload_p (innermode,
1920 0 : SUBREG_BYTE (operand),
1921 : mode))
1922 : /* Don't reload subreg for matching reload. It is actually
1923 : valid subreg in LRA. */
1924 845 : && ! LRA_SUBREG_P (operand))
1925 7168756 : || CONSTANT_P (reg) || GET_CODE (reg) == PLUS || MEM_P (reg))
1926 : {
1927 458 : enum reg_class rclass;
1928 :
1929 458 : if (REG_P (reg))
1930 : /* There is a big probability that we will get the same class
1931 : for the new pseudo and we will get the same insn which
1932 : means infinite looping. So spill the new pseudo. */
1933 : rclass = NO_REGS;
1934 : else
1935 : /* The class will be defined later in curr_insn_transform. */
1936 458 : rclass
1937 458 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
1938 :
1939 458 : if (get_reload_reg (curr_static_id->operand[nop].type, reg_mode, reg,
1940 : rclass, NULL,
1941 : true, false, "subreg reg", &new_reg))
1942 : {
1943 458 : bool insert_before, insert_after;
1944 458 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
1945 :
1946 916 : insert_before = (type != OP_OUT
1947 458 : || read_modify_subreg_p (operand));
1948 458 : insert_after = (type != OP_IN);
1949 916 : insert_move_for_subreg (insert_before ? &before : NULL,
1950 : insert_after ? &after : NULL,
1951 : reg, new_reg);
1952 : }
1953 458 : SUBREG_REG (operand) = new_reg;
1954 458 : lra_process_new_insns (curr_insn, before, after,
1955 : "Inserting subreg reload");
1956 458 : return true;
1957 : }
1958 : /* Force a reload for a paradoxical subreg. For paradoxical subreg,
1959 : IRA allocates hardreg to the inner pseudo reg according to its mode
1960 : instead of the outermode, so the size of the hardreg may not be enough
1961 : to contain the outermode operand, in that case we may need to insert
1962 : reload for the reg. For the following two types of paradoxical subreg,
1963 : we need to insert reload:
1964 : 1. If the op_type is OP_IN, and the hardreg could not be paired with
1965 : other hardreg to contain the outermode operand
1966 : (checked by in_hard_reg_set_p), we need to insert the reload.
1967 : 2. If the op_type is OP_OUT or OP_INOUT.
1968 :
1969 : Here is a paradoxical subreg example showing how the reload is generated:
1970 :
1971 : (insn 5 4 7 2 (set (reg:TI 106 [ __comp ])
1972 : (subreg:TI (reg:DI 107 [ __comp ]) 0)) {*movti_internal_rex64}
1973 :
1974 : In IRA, reg107 is allocated to a DImode hardreg. We use x86-64 as example
1975 : here, if reg107 is assigned to hardreg R15, because R15 is the last
1976 : hardreg, compiler cannot find another hardreg to pair with R15 to
1977 : contain TImode data. So we insert a TImode reload reg180 for it.
1978 : After reload is inserted:
1979 :
1980 : (insn 283 0 0 (set (subreg:DI (reg:TI 180 [orig:107 __comp ] [107]) 0)
1981 : (reg:DI 107 [ __comp ])) -1
1982 : (insn 5 4 7 2 (set (reg:TI 106 [ __comp ])
1983 : (subreg:TI (reg:TI 180 [orig:107 __comp ] [107]) 0)) {*movti_internal_rex64}
1984 :
1985 : Two reload hard registers will be allocated to reg180 to save TImode data
1986 : in LRA_assign.
1987 :
1988 : For LRA pseudos this should normally be handled by the biggest_mode
1989 : mechanism. However, it's possible for new uses of an LRA pseudo
1990 : to be introduced after we've allocated it, such as when undoing
1991 : inheritance, and the allocated register might not then be appropriate
1992 : for the new uses. */
1993 3584149 : else if (REG_P (reg)
1994 3584149 : && REGNO (reg) >= FIRST_PSEUDO_REGISTER
1995 3584149 : && paradoxical_subreg_p (operand)
1996 1043465 : && (inner_hard_regno = lra_get_regno_hard_regno (REGNO (reg))) >= 0
1997 952843 : && hard_regno_nregs (inner_hard_regno, mode) > 1
1998 3584149 : && ((hard_regno
1999 3589233 : = simplify_subreg_regno (inner_hard_regno, innermode,
2000 5084 : SUBREG_BYTE (operand), mode)) < 0
2001 5084 : || ((hard_regno_nregs (inner_hard_regno, innermode)
2002 5084 : < hard_regno_nregs (hard_regno, mode))
2003 10168 : && (regclass = lra_get_allocno_class (REGNO (reg)))
2004 5084 : && (type != OP_IN
2005 5084 : || !in_hard_reg_set_p (reg_class_contents[regclass],
2006 : mode, hard_regno)
2007 5084 : || overlaps_hard_reg_set_p (lra_no_alloc_regs,
2008 : mode, hard_regno)))))
2009 : {
2010 : /* The class will be defined later in curr_insn_transform. */
2011 0 : enum reg_class rclass
2012 0 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
2013 :
2014 0 : if (get_reload_reg (curr_static_id->operand[nop].type, mode, reg,
2015 : rclass, NULL,
2016 : true, false, "paradoxical subreg", &new_reg))
2017 : {
2018 0 : rtx subreg;
2019 0 : bool insert_before, insert_after;
2020 :
2021 0 : PUT_MODE (new_reg, mode);
2022 0 : subreg = gen_lowpart_SUBREG (innermode, new_reg);
2023 0 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
2024 :
2025 0 : insert_before = (type != OP_OUT);
2026 0 : insert_after = (type != OP_IN);
2027 0 : insert_move_for_subreg (insert_before ? &before : NULL,
2028 : insert_after ? &after : NULL,
2029 : reg, subreg);
2030 : }
2031 0 : SUBREG_REG (operand) = new_reg;
2032 0 : lra_process_new_insns (curr_insn, before, after,
2033 : "Inserting paradoxical subreg reload");
2034 0 : return true;
2035 : }
2036 : return false;
2037 : }
2038 :
2039 : /* Return TRUE if X refers for a hard register from SET. */
2040 : static bool
2041 407528 : uses_hard_regs_p (rtx x, HARD_REG_SET set)
2042 : {
2043 407528 : int i, j, x_hard_regno;
2044 407528 : machine_mode mode;
2045 407528 : const char *fmt;
2046 407528 : enum rtx_code code;
2047 :
2048 407528 : if (x == NULL_RTX)
2049 : return false;
2050 407528 : code = GET_CODE (x);
2051 407528 : mode = GET_MODE (x);
2052 :
2053 407528 : if (code == SUBREG)
2054 : {
2055 : /* For all SUBREGs we want to check whether the full multi-register
2056 : overlaps the set. For normal SUBREGs this means 'get_hard_regno' of
2057 : the inner register, for paradoxical SUBREGs this means the
2058 : 'get_hard_regno' of the full SUBREG and for complete SUBREGs either is
2059 : fine. Use the wider mode for all cases. */
2060 2731 : rtx subreg = SUBREG_REG (x);
2061 2731 : mode = wider_subreg_mode (x);
2062 2731 : if (mode == GET_MODE (subreg))
2063 : {
2064 1695 : x = subreg;
2065 1695 : code = GET_CODE (x);
2066 : }
2067 : }
2068 :
2069 407528 : if (REG_P (x) || SUBREG_P (x))
2070 : {
2071 265590 : x_hard_regno = get_hard_regno (x);
2072 265590 : return (x_hard_regno >= 0
2073 265590 : && overlaps_hard_reg_set_p (set, mode, x_hard_regno));
2074 : }
2075 141938 : fmt = GET_RTX_FORMAT (code);
2076 368789 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
2077 : {
2078 228853 : if (fmt[i] == 'e')
2079 : {
2080 111350 : if (uses_hard_regs_p (XEXP (x, i), set))
2081 : return true;
2082 : }
2083 117503 : else if (fmt[i] == 'E')
2084 : {
2085 4398 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
2086 3980 : if (uses_hard_regs_p (XVECEXP (x, i, j), set))
2087 : return true;
2088 : }
2089 : }
2090 : return false;
2091 : }
2092 :
2093 : /* Return true if OP is a spilled pseudo. */
2094 : static inline bool
2095 80352903 : spilled_pseudo_p (rtx op)
2096 : {
2097 80352903 : return (REG_P (op)
2098 80352903 : && REGNO (op) >= FIRST_PSEUDO_REGISTER && in_mem_p (REGNO (op)));
2099 : }
2100 :
2101 : /* Return true if X is a general constant. */
2102 : static inline bool
2103 7804776 : general_constant_p (rtx x)
2104 : {
2105 7804776 : return CONSTANT_P (x) && (! flag_pic || LEGITIMATE_PIC_OPERAND_P (x));
2106 : }
2107 :
2108 : static bool
2109 24720811 : reg_in_class_p (rtx reg, enum reg_class cl)
2110 : {
2111 24720811 : if (cl == NO_REGS)
2112 1115987 : return get_reg_class (REGNO (reg)) == NO_REGS;
2113 23604824 : return in_class_p (reg, cl, NULL);
2114 : }
2115 :
2116 : /* Return true if SET of RCLASS contains no hard regs which can be
2117 : used in MODE. */
2118 : static bool
2119 3831167 : prohibited_class_reg_set_mode_p (enum reg_class rclass,
2120 : HARD_REG_SET &set,
2121 : machine_mode mode)
2122 : {
2123 3831167 : HARD_REG_SET temp;
2124 :
2125 7662334 : lra_assert (hard_reg_set_subset_p (reg_class_contents[rclass], set));
2126 3831167 : temp = set & ~lra_no_alloc_regs;
2127 3831167 : return (hard_reg_set_subset_p
2128 3831167 : (temp, ira_prohibited_class_mode_regs[rclass][mode]));
2129 : }
2130 :
2131 :
2132 : /* Used to check validity info about small class input operands. It
2133 : should be incremented at start of processing an insn
2134 : alternative. */
2135 : static unsigned int curr_small_class_check = 0;
2136 :
2137 : /* Update number of used inputs of class OP_CLASS for operand NOP
2138 : of alternative NALT. Return true if we have more such class operands
2139 : than the number of available regs. */
2140 : static bool
2141 391604045 : update_and_check_small_class_inputs (int nop, int nalt,
2142 : enum reg_class op_class)
2143 : {
2144 391604045 : static unsigned int small_class_check[LIM_REG_CLASSES];
2145 391604045 : static int small_class_input_nums[LIM_REG_CLASSES];
2146 :
2147 388749022 : if (SMALL_REGISTER_CLASS_P (op_class)
2148 : /* We are interesting in classes became small because of fixing
2149 : some hard regs, e.g. by an user through GCC options. */
2150 2961438 : && hard_reg_set_intersect_p (reg_class_contents[op_class],
2151 2961438 : ira_no_alloc_regs)
2152 391604084 : && (curr_static_id->operand[nop].type != OP_OUT
2153 33 : || TEST_BIT (curr_static_id->operand[nop].early_clobber_alts, nalt)))
2154 : {
2155 6 : if (small_class_check[op_class] == curr_small_class_check)
2156 0 : small_class_input_nums[op_class]++;
2157 : else
2158 : {
2159 6 : small_class_check[op_class] = curr_small_class_check;
2160 6 : small_class_input_nums[op_class] = 1;
2161 : }
2162 6 : if (small_class_input_nums[op_class] > ira_class_hard_regs_num[op_class])
2163 : return true;
2164 : }
2165 : return false;
2166 : }
2167 :
2168 : /* Print operand constraints for alternative ALT_NUMBER of the current
2169 : insn. */
2170 : static void
2171 4590 : print_curr_insn_alt (int alt_number)
2172 : {
2173 15917 : for (int i = 0; i < curr_static_id->n_operands; i++)
2174 : {
2175 11327 : const char *p = (curr_static_id->operand_alternative
2176 11327 : [alt_number * curr_static_id->n_operands + i].constraint);
2177 11327 : if (*p == '\0')
2178 220 : continue;
2179 11107 : fprintf (lra_dump_file, " (%d) ", i);
2180 39435 : for (; *p != '\0' && *p != ',' && *p != '#'; p++)
2181 17221 : fputc (*p, lra_dump_file);
2182 : }
2183 4590 : }
2184 :
2185 : /* Major function to choose the current insn alternative and what
2186 : operands should be reloaded and how. If ONLY_ALTERNATIVE is not
2187 : negative we should consider only this alternative. Return false if
2188 : we cannot choose the alternative or find how to reload the
2189 : operands. */
2190 : static bool
2191 89334649 : process_alt_operands (int only_alternative)
2192 : {
2193 89334649 : bool ok_p = false;
2194 89334649 : int nop, overall, nalt;
2195 89334649 : int n_alternatives = curr_static_id->n_alternatives;
2196 89334649 : int n_operands = curr_static_id->n_operands;
2197 : /* LOSERS counts the operands that don't fit this alternative and
2198 : would require loading. */
2199 89334649 : int losers;
2200 89334649 : int addr_losers;
2201 : /* REJECT is a count of how undesirable this alternative says it is
2202 : if any reloading is required. If the alternative matches exactly
2203 : then REJECT is ignored, but otherwise it gets this much counted
2204 : against it in addition to the reloading needed. */
2205 89334649 : int reject;
2206 : /* This is defined by '!' or '?' alternative constraint and added to
2207 : reject. But in some cases it can be ignored. */
2208 89334649 : int static_reject;
2209 89334649 : int op_reject;
2210 : /* The number of elements in the following array. */
2211 89334649 : int early_clobbered_regs_num;
2212 : /* Numbers of operands which are early clobber registers. */
2213 89334649 : int early_clobbered_nops[MAX_RECOG_OPERANDS];
2214 89334649 : enum reg_class curr_alt[MAX_RECOG_OPERANDS];
2215 89334649 : enum reg_class all_this_alternative;
2216 89334649 : int all_used_nregs, all_reload_nregs;
2217 89334649 : HARD_REG_SET curr_alt_set[MAX_RECOG_OPERANDS];
2218 89334649 : HARD_REG_SET curr_alt_exclude_start_hard_regs[MAX_RECOG_OPERANDS];
2219 89334649 : bool curr_alt_match_win[MAX_RECOG_OPERANDS];
2220 89334649 : bool curr_alt_win[MAX_RECOG_OPERANDS];
2221 89334649 : bool curr_alt_offmemok[MAX_RECOG_OPERANDS];
2222 89334649 : int curr_alt_matches[MAX_RECOG_OPERANDS];
2223 : /* The number of elements in the following array. */
2224 89334649 : int curr_alt_dont_inherit_ops_num;
2225 : /* Numbers of operands whose reload pseudos should not be inherited. */
2226 89334649 : int curr_alt_dont_inherit_ops[MAX_RECOG_OPERANDS];
2227 89334649 : bool curr_reuse_alt_p;
2228 : /* True if output stack pointer reload should be generated for the current
2229 : alternative. */
2230 89334649 : bool curr_alt_out_sp_reload_p;
2231 89334649 : bool curr_alt_class_change_p;
2232 89334649 : rtx op;
2233 : /* The register when the operand is a subreg of register, otherwise the
2234 : operand itself. */
2235 89334649 : rtx no_subreg_reg_operand[MAX_RECOG_OPERANDS];
2236 : /* The register if the operand is a register or subreg of register,
2237 : otherwise NULL. */
2238 89334649 : rtx operand_reg[MAX_RECOG_OPERANDS];
2239 89334649 : int hard_regno[MAX_RECOG_OPERANDS];
2240 89334649 : machine_mode biggest_mode[MAX_RECOG_OPERANDS];
2241 89334649 : int reload_nregs, reload_sum;
2242 89334649 : bool costly_p;
2243 89334649 : enum reg_class cl;
2244 89334649 : const HARD_REG_SET *cl_filter;
2245 89334649 : HARD_REG_SET hard_reg_constraint;
2246 :
2247 : /* Calculate some data common for all alternatives to speed up the
2248 : function. */
2249 296945548 : for (nop = 0; nop < n_operands; nop++)
2250 : {
2251 207610899 : rtx reg;
2252 :
2253 207610899 : op = no_subreg_reg_operand[nop] = *curr_id->operand_loc[nop];
2254 : /* The real hard regno of the operand after the allocation. */
2255 207610899 : hard_regno[nop] = get_hard_regno (op);
2256 :
2257 207610899 : operand_reg[nop] = reg = op;
2258 207610899 : biggest_mode[nop] = GET_MODE (op);
2259 207610899 : if (GET_CODE (op) == SUBREG)
2260 : {
2261 4106949 : biggest_mode[nop] = wider_subreg_mode (op);
2262 4106949 : operand_reg[nop] = reg = SUBREG_REG (op);
2263 : }
2264 207610899 : if (! REG_P (reg))
2265 87865138 : operand_reg[nop] = NULL_RTX;
2266 119745761 : else if (REGNO (reg) >= FIRST_PSEUDO_REGISTER
2267 140662549 : || ((int) REGNO (reg)
2268 20916788 : == lra_get_elimination_hard_regno (REGNO (reg))))
2269 116827889 : no_subreg_reg_operand[nop] = reg;
2270 : else
2271 2917872 : operand_reg[nop] = no_subreg_reg_operand[nop]
2272 : /* Just use natural mode for elimination result. It should
2273 : be enough for extra constraints hooks. */
2274 2917872 : = regno_reg_rtx[hard_regno[nop]];
2275 : }
2276 :
2277 : /* The constraints are made of several alternatives. Each operand's
2278 : constraint looks like foo,bar,... with commas separating the
2279 : alternatives. The first alternatives for all operands go
2280 : together, the second alternatives go together, etc.
2281 :
2282 : First loop over alternatives. */
2283 89334649 : alternative_mask preferred = curr_id->preferred_alternatives;
2284 89334649 : if (only_alternative >= 0)
2285 974606 : preferred &= ALTERNATIVE_BIT (only_alternative);
2286 :
2287 89334649 : bool prefer_memory_p = false;
2288 89334751 : repeat:
2289 367264726 : for (nalt = 0; nalt < n_alternatives; nalt++)
2290 : {
2291 : /* Loop over operands for one constraint alternative. */
2292 353607963 : if (!TEST_BIT (preferred, nalt))
2293 97887337 : continue;
2294 :
2295 255720626 : if (lra_dump_file != NULL)
2296 : {
2297 3403 : fprintf (lra_dump_file, " Considering alt=%d of insn %d: ",
2298 3403 : nalt, INSN_UID (curr_insn));
2299 3403 : print_curr_insn_alt (nalt);
2300 3403 : fprintf (lra_dump_file, "\n");
2301 : }
2302 :
2303 255720626 : bool matching_early_clobber[MAX_RECOG_OPERANDS];
2304 255720626 : curr_small_class_check++;
2305 255720626 : overall = losers = addr_losers = 0;
2306 255720626 : static_reject = reject = reload_nregs = reload_sum = 0;
2307 847944233 : for (nop = 0; nop < n_operands; nop++)
2308 : {
2309 592223607 : int inc = (curr_static_id
2310 592223607 : ->operand_alternative[nalt * n_operands + nop].reject);
2311 592223607 : if (lra_dump_file != NULL && inc != 0)
2312 53 : fprintf (lra_dump_file,
2313 : " Staticly defined alt reject+=%d\n", inc);
2314 592223607 : static_reject += inc;
2315 592223607 : matching_early_clobber[nop] = 0;
2316 : }
2317 : reject += static_reject;
2318 : early_clobbered_regs_num = 0;
2319 : curr_alt_out_sp_reload_p = false;
2320 : curr_reuse_alt_p = true;
2321 : curr_alt_class_change_p = false;
2322 : all_this_alternative = NO_REGS;
2323 : all_used_nregs = all_reload_nregs = 0;
2324 661905329 : for (nop = 0; nop < n_operands; nop++)
2325 : {
2326 527874817 : const char *p;
2327 527874817 : char *end;
2328 527874817 : int len, c, m, i, opalt_num, this_alternative_matches;
2329 527874817 : bool win, did_match, offmemok, early_clobber_p;
2330 : /* false => this operand can be reloaded somehow for this
2331 : alternative. */
2332 527874817 : bool badop;
2333 : /* true => this operand can be reloaded if the alternative
2334 : allows regs. */
2335 527874817 : bool winreg;
2336 : /* True if a constant forced into memory would be OK for
2337 : this operand. */
2338 527874817 : bool constmemok;
2339 527874817 : enum reg_class this_alternative, this_costly_alternative;
2340 527874817 : HARD_REG_SET this_alternative_set, this_costly_alternative_set;
2341 527874817 : HARD_REG_SET this_alternative_exclude_start_hard_regs;
2342 527874817 : bool this_alternative_match_win, this_alternative_win;
2343 527874817 : bool this_alternative_offmemok;
2344 527874817 : bool scratch_p;
2345 527874817 : machine_mode mode;
2346 527874817 : enum constraint_num cn;
2347 527874817 : bool class_change_p = false;
2348 :
2349 527874817 : opalt_num = nalt * n_operands + nop;
2350 527874817 : if (curr_static_id->operand_alternative[opalt_num].anything_ok)
2351 : {
2352 : /* Fast track for no constraints at all. */
2353 14580658 : curr_alt[nop] = NO_REGS;
2354 14580658 : CLEAR_HARD_REG_SET (curr_alt_set[nop]);
2355 14580658 : curr_alt_win[nop] = true;
2356 14580658 : curr_alt_match_win[nop] = false;
2357 14580658 : curr_alt_offmemok[nop] = false;
2358 14580658 : curr_alt_matches[nop] = -1;
2359 14580658 : continue;
2360 : }
2361 :
2362 513294159 : op = no_subreg_reg_operand[nop];
2363 513294159 : mode = curr_operand_mode[nop];
2364 :
2365 513294159 : win = did_match = winreg = offmemok = constmemok = false;
2366 513294159 : badop = true;
2367 :
2368 513294159 : early_clobber_p = false;
2369 513294159 : p = curr_static_id->operand_alternative[opalt_num].constraint;
2370 :
2371 513294159 : this_costly_alternative = this_alternative = NO_REGS;
2372 : /* We update set of possible hard regs besides its class
2373 : because reg class might be inaccurate. For example,
2374 : union of LO_REGS (l), HI_REGS(h), and STACK_REG(k) in ARM
2375 : is translated in HI_REGS because classes are merged by
2376 : pairs and there is no accurate intermediate class. */
2377 2053176636 : CLEAR_HARD_REG_SET (this_alternative_set);
2378 1539882477 : CLEAR_HARD_REG_SET (this_costly_alternative_set);
2379 513294159 : CLEAR_HARD_REG_SET (this_alternative_exclude_start_hard_regs);
2380 513294159 : this_alternative_win = false;
2381 513294159 : this_alternative_match_win = false;
2382 513294159 : this_alternative_offmemok = false;
2383 513294159 : this_alternative_matches = -1;
2384 :
2385 : /* An empty constraint should be excluded by the fast
2386 : track. */
2387 513294159 : lra_assert (*p != 0 && *p != ',');
2388 :
2389 : op_reject = 0;
2390 : /* Scan this alternative's specs for this operand; set WIN
2391 : if the operand fits any letter in this alternative.
2392 : Otherwise, clear BADOP if this operand could fit some
2393 : letter after reloads, or set WINREG if this operand could
2394 : fit after reloads provided the constraint allows some
2395 : registers. */
2396 : costly_p = false;
2397 1303722793 : do
2398 : {
2399 1303722793 : switch ((c = *p, len = CONSTRAINT_LEN (c, p)), c)
2400 : {
2401 : case '\0':
2402 : len = 0;
2403 : break;
2404 490625934 : case ',':
2405 490625934 : c = '\0';
2406 490625934 : break;
2407 :
2408 175739 : case '&':
2409 175739 : early_clobber_p = true;
2410 175739 : break;
2411 :
2412 19362 : case '$':
2413 19362 : op_reject += LRA_MAX_REJECT;
2414 19362 : break;
2415 0 : case '^':
2416 0 : op_reject += LRA_LOSER_COST_FACTOR;
2417 0 : break;
2418 :
2419 0 : case '#':
2420 : /* Ignore rest of this alternative. */
2421 0 : c = '\0';
2422 0 : break;
2423 :
2424 55885021 : case '0': case '1': case '2': case '3': case '4':
2425 55885021 : case '5': case '6': case '7': case '8': case '9':
2426 55885021 : {
2427 55885021 : int m_hregno;
2428 55885021 : bool match_p;
2429 :
2430 55885021 : m = strtoul (p, &end, 10);
2431 55885021 : p = end;
2432 55885021 : len = 0;
2433 55885021 : lra_assert (nop > m);
2434 :
2435 : /* Reject matches if we don't know which operand is
2436 : bigger. This situation would arguably be a bug in
2437 : an .md pattern, but could also occur in a user asm. */
2438 167655063 : if (!ordered_p (GET_MODE_SIZE (biggest_mode[m]),
2439 55885021 : GET_MODE_SIZE (biggest_mode[nop])))
2440 : break;
2441 :
2442 : /* Don't match wrong asm insn operands for proper
2443 : diagnostic later. */
2444 55885021 : if (INSN_CODE (curr_insn) < 0
2445 33180 : && (curr_operand_mode[m] == BLKmode
2446 33179 : || curr_operand_mode[nop] == BLKmode)
2447 1 : && curr_operand_mode[m] != curr_operand_mode[nop])
2448 : break;
2449 :
2450 55885020 : m_hregno = get_hard_regno (*curr_id->operand_loc[m]);
2451 : /* We are supposed to match a previous operand.
2452 : If we do, we win if that one did. If we do
2453 : not, count both of the operands as losers.
2454 : (This is too conservative, since most of the
2455 : time only a single reload insn will be needed
2456 : to make the two operands win. As a result,
2457 : this alternative may be rejected when it is
2458 : actually desirable.) */
2459 55885020 : match_p = false;
2460 55885020 : if (operands_match_p (*curr_id->operand_loc[nop],
2461 55885020 : *curr_id->operand_loc[m], m_hregno))
2462 : {
2463 : /* We should reject matching of an early
2464 : clobber operand if the matching operand is
2465 : not dying in the insn. */
2466 14789790 : if (!TEST_BIT (curr_static_id->operand[m]
2467 : .early_clobber_alts, nalt)
2468 18273 : || operand_reg[nop] == NULL_RTX
2469 14808063 : || (find_regno_note (curr_insn, REG_DEAD,
2470 : REGNO (op))
2471 4283 : || REGNO (op) == REGNO (operand_reg[m])))
2472 14789790 : match_p = true;
2473 : }
2474 14789790 : if (match_p)
2475 : {
2476 : /* If we are matching a non-offsettable
2477 : address where an offsettable address was
2478 : expected, then we must reject this
2479 : combination, because we can't reload
2480 : it. */
2481 14789790 : if (curr_alt_offmemok[m]
2482 1484 : && MEM_P (*curr_id->operand_loc[m])
2483 0 : && curr_alt[m] == NO_REGS && ! curr_alt_win[m])
2484 0 : continue;
2485 : }
2486 : else
2487 : {
2488 : /* If the operands do not match and one
2489 : operand is INOUT, we can not match them.
2490 : Try other possibilities, e.g. other
2491 : alternatives or commutative operand
2492 : exchange. */
2493 41095230 : if (curr_static_id->operand[nop].type == OP_INOUT
2494 41095230 : || curr_static_id->operand[m].type == OP_INOUT)
2495 : break;
2496 : /* Operands don't match. For asm if the operands
2497 : are different user defined explicit hard
2498 : registers, then we cannot make them match
2499 : when one is early clobber operand. */
2500 41094798 : if ((REG_P (*curr_id->operand_loc[nop])
2501 25824708 : || SUBREG_P (*curr_id->operand_loc[nop]))
2502 15855401 : && (REG_P (*curr_id->operand_loc[m])
2503 200076 : || SUBREG_P (*curr_id->operand_loc[m]))
2504 15764157 : && INSN_CODE (curr_insn) < 0)
2505 : {
2506 590 : rtx nop_reg = *curr_id->operand_loc[nop];
2507 590 : if (SUBREG_P (nop_reg))
2508 0 : nop_reg = SUBREG_REG (nop_reg);
2509 590 : rtx m_reg = *curr_id->operand_loc[m];
2510 590 : if (SUBREG_P (m_reg))
2511 0 : m_reg = SUBREG_REG (m_reg);
2512 :
2513 590 : if (REG_P (nop_reg)
2514 590 : && HARD_REGISTER_P (nop_reg)
2515 0 : && REG_USERVAR_P (nop_reg)
2516 0 : && REG_P (m_reg)
2517 0 : && HARD_REGISTER_P (m_reg)
2518 590 : && REG_USERVAR_P (m_reg))
2519 : {
2520 : int i;
2521 :
2522 0 : for (i = 0; i < early_clobbered_regs_num; i++)
2523 0 : if (m == early_clobbered_nops[i])
2524 : break;
2525 0 : if (i < early_clobbered_regs_num
2526 0 : || early_clobber_p)
2527 : break;
2528 : }
2529 : }
2530 : /* Both operands must allow a reload register,
2531 : otherwise we cannot make them match. */
2532 41094798 : if (curr_alt[m] == NO_REGS)
2533 : break;
2534 : /* Retroactively mark the operand we had to
2535 : match as a loser, if it wasn't already and
2536 : it wasn't matched to a register constraint
2537 : (e.g it might be matched by memory). */
2538 41070393 : if (curr_alt_win[m]
2539 40246594 : && (operand_reg[m] == NULL_RTX
2540 39743879 : || hard_regno[m] < 0))
2541 : {
2542 1256403 : if (lra_dump_file != NULL)
2543 9 : fprintf
2544 9 : (lra_dump_file,
2545 : " %d Matched operand reload: "
2546 : "losers++\n", m);
2547 1256403 : losers++;
2548 1256403 : reload_nregs
2549 1256403 : += (ira_reg_class_max_nregs[curr_alt[m]]
2550 1256403 : [GET_MODE (*curr_id->operand_loc[m])]);
2551 : }
2552 :
2553 : /* Prefer matching earlyclobber alternative as
2554 : it results in less hard regs required for
2555 : the insn than a non-matching earlyclobber
2556 : alternative. */
2557 41070393 : if (TEST_BIT (curr_static_id->operand[m]
2558 : .early_clobber_alts, nalt))
2559 : {
2560 17747 : if (lra_dump_file != NULL)
2561 0 : fprintf
2562 0 : (lra_dump_file,
2563 : " %d Matching earlyclobber alt:"
2564 : " reject--\n",
2565 : nop);
2566 17747 : if (!matching_early_clobber[m])
2567 : {
2568 17747 : reject--;
2569 17747 : matching_early_clobber[m] = 1;
2570 : }
2571 : }
2572 : /* Otherwise we prefer no matching
2573 : alternatives because it gives more freedom
2574 : in RA. */
2575 41052646 : else if (operand_reg[nop] == NULL_RTX
2576 41052646 : || (find_regno_note (curr_insn, REG_DEAD,
2577 15830501 : REGNO (operand_reg[nop]))
2578 : == NULL_RTX))
2579 : {
2580 36177252 : if (lra_dump_file != NULL)
2581 912 : fprintf
2582 912 : (lra_dump_file,
2583 : " %d Matching alt: reject+=2\n",
2584 : nop);
2585 36177252 : reject += 2;
2586 : }
2587 : }
2588 : /* If we have to reload this operand and some
2589 : previous operand also had to match the same
2590 : thing as this operand, we don't know how to do
2591 : that. */
2592 55860183 : if (!match_p || !curr_alt_win[m])
2593 : {
2594 86249073 : for (i = 0; i < nop; i++)
2595 45072576 : if (curr_alt_matches[i] == m)
2596 : break;
2597 41176498 : if (i < nop)
2598 : break;
2599 : }
2600 : else
2601 : did_match = true;
2602 :
2603 55860182 : this_alternative_matches = m;
2604 : /* This can be fixed with reloads if the operand
2605 : we are supposed to match can be fixed with
2606 : reloads. */
2607 55860182 : badop = false;
2608 55860182 : this_alternative = curr_alt[m];
2609 55860182 : this_alternative_set = curr_alt_set[m];
2610 55860182 : this_alternative_exclude_start_hard_regs
2611 55860182 : = curr_alt_exclude_start_hard_regs[m];
2612 55860182 : winreg = this_alternative != NO_REGS;
2613 55860182 : break;
2614 : }
2615 :
2616 11658159 : case 'g':
2617 11658159 : if (MEM_P (op)
2618 7804776 : || general_constant_p (op)
2619 16211818 : || spilled_pseudo_p (op))
2620 : win = true;
2621 11658159 : if (REG_P (op) && prefer_memory_p)
2622 : {
2623 11658159 : badop = false;
2624 11658159 : offmemok = true;
2625 : }
2626 11658159 : cl = GENERAL_REGS;
2627 11658159 : cl_filter = nullptr;
2628 11658159 : goto reg;
2629 :
2630 1140 : case '{':
2631 1140 : {
2632 1140 : int regno = decode_hard_reg_constraint (p);
2633 1140 : gcc_assert (regno >= 0);
2634 1140 : cl = NO_REGS;
2635 1140 : int nregs = hard_regno_nregs (regno, mode);
2636 2280 : for (int i = 0; i < nregs; ++i)
2637 1140 : cl = reg_class_superunion[cl][REGNO_REG_CLASS (regno + i)];
2638 1140 : CLEAR_HARD_REG_SET (hard_reg_constraint);
2639 1140 : SET_HARD_REG_BIT (hard_reg_constraint, regno);
2640 1140 : cl_filter = &hard_reg_constraint;
2641 1140 : goto reg;
2642 : }
2643 :
2644 722689213 : default:
2645 722689213 : cn = lookup_constraint (p);
2646 722689213 : switch (get_constraint_type (cn))
2647 : {
2648 477965158 : case CT_REGISTER:
2649 477965158 : cl = reg_class_for_constraint (cn);
2650 351597911 : if (cl != NO_REGS)
2651 : {
2652 342192615 : cl_filter = get_register_filter (cn);
2653 342192615 : goto reg;
2654 : }
2655 : break;
2656 :
2657 2057230 : case CT_CONST_INT:
2658 2057230 : if (CONST_INT_P (op)
2659 2057230 : && insn_const_int_ok_for_constraint (INTVAL (op), cn))
2660 : win = true;
2661 : break;
2662 :
2663 110215291 : case CT_MEMORY:
2664 110215291 : case CT_RELAXED_MEMORY:
2665 110215291 : if (MEM_P (op)
2666 110215291 : && satisfies_memory_constraint_p (op, cn))
2667 : win = true;
2668 74761761 : else if (spilled_pseudo_p (op))
2669 44866037 : win = true;
2670 :
2671 : /* If we didn't already win, we can reload constants
2672 : via force_const_mem or put the pseudo value into
2673 : memory, or make other memory by reloading the
2674 : address like for 'o'. */
2675 115298914 : if (CONST_POOL_OK_P (mode, op)
2676 105131504 : || MEM_P (op) || REG_P (op)
2677 : /* We can restore the equiv insn by a
2678 : reload. */
2679 110783552 : || equiv_substition_p[nop])
2680 110180362 : badop = false;
2681 : constmemok = true;
2682 : offmemok = true;
2683 : break;
2684 :
2685 1666792 : case CT_ADDRESS:
2686 : /* An asm operand with an address constraint
2687 : that doesn't satisfy address_operand has
2688 : is_address cleared, so that we don't try to
2689 : make a non-address fit. */
2690 1666792 : if (!curr_static_id->operand[nop].is_address)
2691 : break;
2692 : /* If we didn't already win, we can reload the address
2693 : into a base register. */
2694 1666773 : if (satisfies_address_constraint_p (op, cn))
2695 1666773 : win = true;
2696 1666773 : cl = base_reg_class (VOIDmode, ADDR_SPACE_GENERIC,
2697 : ADDRESS, SCRATCH);
2698 1666773 : cl_filter = nullptr;
2699 1666773 : badop = false;
2700 1666773 : goto reg;
2701 :
2702 129575311 : case CT_FIXED_FORM:
2703 129575311 : if (constraint_satisfied_p (op, cn))
2704 1303722793 : win = true;
2705 : break;
2706 :
2707 1209431 : case CT_SPECIAL_MEMORY:
2708 1209431 : if (satisfies_memory_constraint_p (op, cn))
2709 : win = true;
2710 1037483 : else if (spilled_pseudo_p (op))
2711 : {
2712 1303722793 : curr_reuse_alt_p = false;
2713 1303722793 : win = true;
2714 : }
2715 : break;
2716 : }
2717 : break;
2718 :
2719 355518687 : reg:
2720 355518687 : if (mode == BLKmode)
2721 : break;
2722 355518669 : this_alternative = reg_class_subunion[this_alternative][cl];
2723 355518669 : if (hard_reg_set_subset_p (this_alternative_set,
2724 355518669 : reg_class_contents[cl]))
2725 355515044 : this_alternative_exclude_start_hard_regs
2726 355515044 : = ira_exclude_class_mode_regs[cl][mode];
2727 3625 : else if (!hard_reg_set_subset_p (reg_class_contents[cl],
2728 : this_alternative_set))
2729 3624 : this_alternative_exclude_start_hard_regs
2730 1066559631 : |= ira_exclude_class_mode_regs[cl][mode];
2731 355518669 : this_alternative_set |= reg_class_contents[cl];
2732 355518669 : if (cl_filter)
2733 2280 : this_alternative_exclude_start_hard_regs |= ~*cl_filter;
2734 355518669 : if (costly_p)
2735 : {
2736 21096731 : this_costly_alternative
2737 21096731 : = reg_class_subunion[this_costly_alternative][cl];
2738 21096731 : this_costly_alternative_set |= reg_class_contents[cl];
2739 : }
2740 355518669 : winreg = true;
2741 355518669 : if (REG_P (op))
2742 : {
2743 226420720 : rtx orig_op = *curr_id->operand_loc[nop];
2744 6636456 : if (GET_CODE (orig_op) == SUBREG && HARD_REGISTER_P (op)
2745 226420810 : && !targetm.hard_regno_mode_ok (REGNO (op),
2746 90 : GET_MODE(orig_op)))
2747 : break;
2748 :
2749 226420720 : tree decl;
2750 :
2751 226420720 : if (hard_regno[nop] >= 0
2752 190915647 : && in_hard_reg_set_p (this_alternative_set,
2753 : mode, hard_regno[nop])
2754 172952782 : && (!cl_filter
2755 590 : || TEST_HARD_REG_BIT (*cl_filter,
2756 : hard_regno[nop]))
2757 399373496 : && ((REG_ATTRS (op) && (decl = REG_EXPR (op)) != NULL
2758 96198117 : && VAR_P (decl) && DECL_HARD_REGISTER (decl))
2759 172949519 : || !(TEST_HARD_REG_BIT
2760 172949519 : (this_alternative_exclude_start_hard_regs,
2761 : hard_regno[nop]))))
2762 : win = true;
2763 53467954 : else if (hard_regno[nop] < 0 && !prefer_memory_p)
2764 : {
2765 35504937 : if (in_class_p (op, this_alternative, NULL))
2766 : win = true;
2767 26671230 : else if (in_class_p (op, this_alternative, NULL, true))
2768 : {
2769 1303722793 : class_change_p = true;
2770 1303722793 : win = true;
2771 : }
2772 : }
2773 : }
2774 : break;
2775 : }
2776 1303722793 : if (c != ' ' && c != '\t')
2777 1303722793 : costly_p = c == '*';
2778 : }
2779 1303722793 : while ((p += len), c);
2780 :
2781 1026588318 : scratch_p = (operand_reg[nop] != NULL_RTX
2782 513294159 : && ira_former_scratch_p (REGNO (operand_reg[nop])));
2783 : /* Record which operands fit this alternative. */
2784 513294159 : if (win)
2785 : {
2786 276556696 : if (early_clobber_p
2787 276414520 : || curr_static_id->operand[nop].type != OP_OUT)
2788 : {
2789 120948379 : if (winreg)
2790 99975249 : all_used_nregs
2791 99975249 : += ira_reg_class_min_nregs[this_alternative][mode];
2792 120948379 : all_this_alternative
2793 120948379 : = (reg_class_subunion
2794 120948379 : [all_this_alternative][this_alternative]);
2795 : }
2796 276556696 : this_alternative_win = true;
2797 276556696 : if (class_change_p)
2798 : {
2799 250460 : curr_alt_class_change_p = true;
2800 250460 : if (lra_dump_file != NULL)
2801 10 : fprintf (lra_dump_file,
2802 : " %d Narrowing class: reject+=3\n",
2803 : nop);
2804 250460 : reject += 3;
2805 : }
2806 276556696 : if (operand_reg[nop] != NULL_RTX)
2807 : {
2808 192868229 : if (hard_regno[nop] >= 0)
2809 : {
2810 172896519 : if (in_hard_reg_set_p (this_costly_alternative_set,
2811 : mode, hard_regno[nop]))
2812 : {
2813 772696 : if (lra_dump_file != NULL)
2814 21 : fprintf (lra_dump_file,
2815 : " %d Costly set: reject++\n",
2816 : nop);
2817 772696 : reject++;
2818 : }
2819 : }
2820 : else
2821 : {
2822 : /* Prefer won reg to spilled pseudo under other
2823 : equal conditions for possibe inheritance. */
2824 19971710 : if (! scratch_p)
2825 : {
2826 19967116 : if (lra_dump_file != NULL)
2827 59 : fprintf
2828 59 : (lra_dump_file,
2829 : " %d Non pseudo reload: reject++\n",
2830 : nop);
2831 19967116 : reject++;
2832 : }
2833 19971710 : if (in_class_p (operand_reg[nop],
2834 : this_costly_alternative, NULL, true))
2835 : {
2836 134165 : if (lra_dump_file != NULL)
2837 0 : fprintf
2838 0 : (lra_dump_file,
2839 : " %d Non pseudo costly reload:"
2840 : " reject++\n",
2841 : nop);
2842 134165 : reject++;
2843 : }
2844 : }
2845 : /* We simulate the behavior of old reload here.
2846 : Although scratches need hard registers and it
2847 : might result in spilling other pseudos, no reload
2848 : insns are generated for the scratches. So it
2849 : might cost something but probably less than old
2850 : reload pass believes. */
2851 192868229 : if (scratch_p)
2852 : {
2853 116373 : if (lra_dump_file != NULL)
2854 6 : fprintf (lra_dump_file,
2855 : " %d Scratch win: reject+=2\n",
2856 : nop);
2857 116373 : reject += 2;
2858 : }
2859 : }
2860 : }
2861 236737463 : else if (did_match)
2862 : this_alternative_match_win = true;
2863 : else
2864 : {
2865 222053778 : if (prefer_memory_p && offmemok)
2866 : {
2867 0 : winreg = false;
2868 0 : this_alternative = NO_REGS;
2869 : }
2870 :
2871 222053778 : int const_to_mem = 0;
2872 222053778 : bool no_regs_p;
2873 :
2874 222053778 : reject += op_reject;
2875 : /* Mark output reload of the stack pointer. */
2876 222053778 : if (op == stack_pointer_rtx
2877 56649 : && curr_static_id->operand[nop].type != OP_IN)
2878 222053778 : curr_alt_out_sp_reload_p = true;
2879 :
2880 : /* If this alternative asks for a specific reg class, see if there
2881 : is at least one allocatable register in that class. */
2882 222053778 : no_regs_p
2883 387048091 : = (this_alternative == NO_REGS
2884 222053778 : || (hard_reg_set_subset_p
2885 329988648 : (reg_class_contents[this_alternative],
2886 : lra_no_alloc_regs)));
2887 :
2888 : /* For asms, verify that the class for this alternative is possible
2889 : for the mode that is specified. */
2890 164994313 : if (!no_regs_p && INSN_CODE (curr_insn) < 0)
2891 : {
2892 : int i;
2893 69463 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
2894 69461 : if (targetm.hard_regno_mode_ok (i, mode)
2895 69461 : && in_hard_reg_set_p (reg_class_contents[this_alternative],
2896 : mode, i))
2897 : break;
2898 20291 : if (i == FIRST_PSEUDO_REGISTER)
2899 222053778 : winreg = false;
2900 : }
2901 :
2902 : /* If this operand accepts a register, and if the
2903 : register class has at least one allocatable register,
2904 : then this operand can be reloaded. */
2905 222053778 : if (winreg && !no_regs_p)
2906 : badop = false;
2907 :
2908 57059467 : if (badop)
2909 : {
2910 47773815 : if (lra_dump_file != NULL)
2911 606 : fprintf (lra_dump_file,
2912 : " Bad operand -- refuse\n");
2913 121690114 : goto fail;
2914 : }
2915 :
2916 174279963 : if (this_alternative != NO_REGS)
2917 : {
2918 164994312 : HARD_REG_SET available_regs
2919 164994312 : = (reg_class_contents[this_alternative]
2920 164994312 : & ~((ira_prohibited_class_mode_regs
2921 164994312 : [this_alternative][mode])
2922 164994312 : | lra_no_alloc_regs));
2923 329988624 : if (!hard_reg_set_empty_p (available_regs))
2924 : {
2925 164992726 : if (early_clobber_p
2926 164959163 : || curr_static_id->operand[nop].type != OP_OUT)
2927 : {
2928 86416879 : all_reload_nregs
2929 86416879 : += ira_reg_class_min_nregs[this_alternative][mode];
2930 86416879 : all_this_alternative
2931 86416879 : = (reg_class_subunion
2932 86416879 : [all_this_alternative][this_alternative]);
2933 : }
2934 : }
2935 : else
2936 : {
2937 : /* There are no hard regs holding a value of given
2938 : mode. */
2939 1586 : if (offmemok)
2940 : {
2941 173 : this_alternative = NO_REGS;
2942 173 : if (lra_dump_file != NULL)
2943 0 : fprintf (lra_dump_file,
2944 : " %d Using memory because of"
2945 : " a bad mode: reject+=2\n",
2946 : nop);
2947 173 : reject += 2;
2948 : }
2949 : else
2950 : {
2951 1413 : if (lra_dump_file != NULL)
2952 0 : fprintf (lra_dump_file,
2953 : " Wrong mode -- refuse\n");
2954 1413 : goto fail;
2955 : }
2956 : }
2957 : }
2958 :
2959 : /* If not assigned pseudo has a class which a subset of
2960 : required reg class, it is a less costly alternative
2961 : as the pseudo still can get a hard reg of necessary
2962 : class. */
2963 164992899 : if (! no_regs_p && REG_P (op) && hard_regno[nop] < 0
2964 21316466 : && (cl = get_reg_class (REGNO (op))) != NO_REGS
2965 177360685 : && ira_class_subset_p[this_alternative][cl])
2966 : {
2967 1065 : if (lra_dump_file != NULL)
2968 0 : fprintf
2969 0 : (lra_dump_file,
2970 : " %d Super set class reg: reject-=3\n", nop);
2971 1065 : reject -= 3;
2972 : }
2973 :
2974 174278550 : this_alternative_offmemok = offmemok;
2975 174278550 : if (this_costly_alternative != NO_REGS)
2976 : {
2977 18939474 : if (lra_dump_file != NULL)
2978 25 : fprintf (lra_dump_file,
2979 : " %d Costly loser: reject++\n", nop);
2980 18939474 : reject++;
2981 : }
2982 : /* If the operand is dying, has a matching constraint,
2983 : and satisfies constraints of the matched operand
2984 : which failed to satisfy the own constraints, most probably
2985 : the reload for this operand will be gone. */
2986 174278550 : if (this_alternative_matches >= 0
2987 41159143 : && !curr_alt_win[this_alternative_matches]
2988 929428 : && REG_P (op)
2989 679155 : && find_regno_note (curr_insn, REG_DEAD, REGNO (op))
2990 174972113 : && (hard_regno[nop] >= 0
2991 364161 : ? in_hard_reg_set_p (this_alternative_set,
2992 : mode, hard_regno[nop])
2993 34759 : : in_class_p (op, this_alternative, NULL)))
2994 : {
2995 217254 : if (lra_dump_file != NULL)
2996 1 : fprintf
2997 1 : (lra_dump_file,
2998 : " %d Dying matched operand reload: reject++\n",
2999 : nop);
3000 217254 : reject++;
3001 : }
3002 : else
3003 : {
3004 : /* Strict_low_part requires to reload the register
3005 : not the sub-register. In this case we should
3006 : check that a final reload hard reg can hold the
3007 : value mode. */
3008 174061296 : if (curr_static_id->operand[nop].strict_low
3009 117 : && REG_P (op)
3010 110 : && hard_regno[nop] < 0
3011 84 : && GET_CODE (*curr_id->operand_loc[nop]) == SUBREG
3012 84 : && ira_class_hard_regs_num[this_alternative] > 0
3013 174061380 : && (!targetm.hard_regno_mode_ok
3014 84 : (ira_class_hard_regs[this_alternative][0],
3015 84 : GET_MODE (*curr_id->operand_loc[nop]))))
3016 : {
3017 0 : if (lra_dump_file != NULL)
3018 0 : fprintf
3019 0 : (lra_dump_file,
3020 : " Strict low subreg reload -- refuse\n");
3021 0 : goto fail;
3022 : }
3023 174061296 : if (lra_dump_file != NULL)
3024 2177 : fprintf
3025 2177 : (lra_dump_file,
3026 : " %d Operand reload: losers++\n", nop);
3027 174061296 : losers++;
3028 : }
3029 174278550 : if (operand_reg[nop] != NULL_RTX
3030 : /* Output operands and matched input operands are
3031 : not inherited. The following conditions do not
3032 : exactly describe the previous statement but they
3033 : are pretty close. */
3034 62485809 : && curr_static_id->operand[nop].type != OP_OUT
3035 27554758 : && (this_alternative_matches < 0
3036 15921147 : || curr_static_id->operand[nop].type != OP_IN))
3037 : {
3038 11633611 : int last_reload = (lra_reg_info[ORIGINAL_REGNO
3039 11633611 : (operand_reg[nop])]
3040 11633611 : .last_reload);
3041 :
3042 : /* The value of reload_sum has sense only if we
3043 : process insns in their order. It happens only on
3044 : the first constraints sub-pass when we do most of
3045 : reload work. */
3046 11633611 : if (lra_constraint_iter == 1 && last_reload > bb_reload_num)
3047 2524734 : reload_sum += last_reload - bb_reload_num;
3048 : }
3049 : /* If this is a constant that is reloaded into the
3050 : desired class by copying it to memory first, count
3051 : that as another reload. This is consistent with
3052 : other code and is required to avoid choosing another
3053 : alternative when the constant is moved into memory.
3054 : Note that the test here is precisely the same as in
3055 : the code below that calls force_const_mem. */
3056 224040972 : if (CONST_POOL_OK_P (mode, op)
3057 224041043 : && ((targetm.preferred_reload_class
3058 49762493 : (op, this_alternative) == NO_REGS)
3059 48251382 : || no_input_reloads_p))
3060 : {
3061 1511111 : const_to_mem = 1;
3062 1511111 : if (! no_regs_p)
3063 : {
3064 710642 : if (lra_dump_file != NULL)
3065 0 : fprintf
3066 0 : (lra_dump_file,
3067 : " %d Constant reload through memory: "
3068 : "losers++\n", nop);
3069 710642 : losers++;
3070 : }
3071 : }
3072 :
3073 : /* Alternative loses if it requires a type of reload not
3074 : permitted for this insn. We can always reload
3075 : objects with a REG_UNUSED note. */
3076 174278550 : if ((curr_static_id->operand[nop].type != OP_IN
3077 84695971 : && no_output_reloads_p
3078 0 : && ! find_reg_note (curr_insn, REG_UNUSED, op)
3079 0 : && ! scratch_p)
3080 174278550 : || (curr_static_id->operand[nop].type != OP_OUT
3081 89582823 : && no_input_reloads_p && ! const_to_mem)
3082 348557100 : || (this_alternative_matches >= 0
3083 41159143 : && (no_input_reloads_p
3084 41159143 : || (no_output_reloads_p
3085 0 : && (curr_static_id->operand
3086 0 : [this_alternative_matches].type != OP_IN)
3087 0 : && ! find_reg_note (curr_insn, REG_UNUSED,
3088 : no_subreg_reg_operand
3089 0 : [this_alternative_matches])
3090 0 : && ! scratch_p))))
3091 : {
3092 0 : if (lra_dump_file != NULL)
3093 0 : fprintf
3094 0 : (lra_dump_file,
3095 : " No input/output reload -- refuse\n");
3096 0 : goto fail;
3097 : }
3098 :
3099 : /* Alternative loses if it required class pseudo cannot
3100 : hold value of required mode. Such insns can be
3101 : described by insn definitions with mode iterators. */
3102 174278550 : if (GET_MODE (*curr_id->operand_loc[nop]) != VOIDmode
3103 125906338 : && ! hard_reg_set_empty_p (this_alternative_set)
3104 : /* It is common practice for constraints to use a
3105 : class which does not have actually enough regs to
3106 : hold the value (e.g. x86 AREG for mode requiring
3107 : more one general reg). Therefore we have 2
3108 : conditions to check that the reload pseudo cannot
3109 : hold the mode value. */
3110 117321520 : && (!targetm.hard_regno_mode_ok
3111 117321520 : (ira_class_hard_regs[this_alternative][0],
3112 : GET_MODE (*curr_id->operand_loc[nop])))
3113 : /* The above condition is not enough as the first
3114 : reg in ira_class_hard_regs can be not aligned for
3115 : multi-words mode values. */
3116 174278550 : && (prohibited_class_reg_set_mode_p
3117 0 : (this_alternative, this_alternative_set,
3118 0 : GET_MODE (*curr_id->operand_loc[nop]))))
3119 : {
3120 0 : if (lra_dump_file != NULL)
3121 0 : fprintf (lra_dump_file,
3122 : " reload pseudo for op %d "
3123 : "cannot hold the mode value -- refuse\n",
3124 : nop);
3125 0 : goto fail;
3126 : }
3127 :
3128 : /* Check strong discouragement of reload of non-constant
3129 : into class THIS_ALTERNATIVE. */
3130 124516057 : if (! CONSTANT_P (op) && ! no_regs_p
3131 290309425 : && (targetm.preferred_reload_class
3132 116030875 : (op, this_alternative) == NO_REGS
3133 107597402 : || (curr_static_id->operand[nop].type == OP_OUT
3134 74179047 : && (targetm.preferred_output_reload_class
3135 74179047 : (op, this_alternative) == NO_REGS))))
3136 : {
3137 12955953 : if (offmemok && REG_P (op))
3138 : {
3139 791070 : if (lra_dump_file != NULL)
3140 0 : fprintf
3141 0 : (lra_dump_file,
3142 : " %d Spill pseudo into memory: reject+=3\n",
3143 : nop);
3144 791070 : reject += 3;
3145 : }
3146 : else
3147 : {
3148 12164883 : if (lra_dump_file != NULL)
3149 0 : fprintf
3150 0 : (lra_dump_file,
3151 : " %d Non-prefered reload: reject+=%d\n",
3152 : nop, LRA_MAX_REJECT);
3153 12164883 : reject += LRA_MAX_REJECT;
3154 : }
3155 : }
3156 :
3157 174278550 : if (! (MEM_P (op) && offmemok)
3158 174278478 : && ! (const_to_mem && constmemok))
3159 : {
3160 : /* We prefer to reload pseudos over reloading other
3161 : things, since such reloads may be able to be
3162 : eliminated later. So bump REJECT in other cases.
3163 : Don't do this in the case where we are forcing a
3164 : constant into memory and it will then win since
3165 : we don't want to have a different alternative
3166 : match then. */
3167 173361646 : if (! (REG_P (op) && REGNO (op) >= FIRST_PSEUDO_REGISTER))
3168 : {
3169 123854576 : if (lra_dump_file != NULL)
3170 1648 : fprintf
3171 1648 : (lra_dump_file,
3172 : " %d Non-pseudo reload: reject+=2\n",
3173 : nop);
3174 123854576 : reject += 2;
3175 : }
3176 :
3177 173361646 : if (! no_regs_p)
3178 164876508 : reload_nregs
3179 164876508 : += ira_reg_class_max_nregs[this_alternative][mode];
3180 :
3181 173361646 : if (SMALL_REGISTER_CLASS_P (this_alternative))
3182 : {
3183 856032 : if (lra_dump_file != NULL)
3184 45 : fprintf
3185 45 : (lra_dump_file,
3186 : " %d Small class reload: reject+=%d\n",
3187 : nop, LRA_LOSER_COST_FACTOR / 2);
3188 856032 : reject += LRA_LOSER_COST_FACTOR / 2;
3189 : }
3190 : }
3191 :
3192 : /* We are trying to spill pseudo into memory. It is
3193 : usually more costly than moving to a hard register
3194 : although it might takes the same number of
3195 : reloads.
3196 :
3197 : Non-pseudo spill may happen also. Suppose a target allows both
3198 : register and memory in the operand constraint alternatives,
3199 : then it's typical that an eliminable register has a substition
3200 : of "base + offset" which can either be reloaded by a simple
3201 : "new_reg <= base + offset" which will match the register
3202 : constraint, or a similar reg addition followed by further spill
3203 : to and reload from memory which will match the memory
3204 : constraint, but this memory spill will be much more costly
3205 : usually.
3206 :
3207 : Code below increases the reject for both pseudo and non-pseudo
3208 : spill. */
3209 174278550 : if (no_regs_p
3210 9285651 : && !(MEM_P (op) && offmemok)
3211 9285607 : && !(REG_P (op) && hard_regno[nop] < 0))
3212 : {
3213 8174447 : if (lra_dump_file != NULL)
3214 13 : fprintf
3215 20 : (lra_dump_file,
3216 : " %d Spill %spseudo into memory: reject+=3\n",
3217 : nop, REG_P (op) ? "" : "Non-");
3218 8174447 : reject += 3;
3219 8174447 : if (VECTOR_MODE_P (mode))
3220 : {
3221 : /* Spilling vectors into memory is usually more
3222 : costly as they contain big values. */
3223 387782 : if (lra_dump_file != NULL)
3224 0 : fprintf
3225 0 : (lra_dump_file,
3226 : " %d Spill vector pseudo: reject+=2\n",
3227 : nop);
3228 387782 : reject += 2;
3229 : }
3230 : }
3231 :
3232 : /* When we use an operand requiring memory in given
3233 : alternative, the insn should write *and* read the
3234 : value to/from memory it is costly in comparison with
3235 : an insn alternative which does not use memory
3236 : (e.g. register or immediate operand). We exclude
3237 : memory operand for such case as we can satisfy the
3238 : memory constraints by reloading address. */
3239 9285651 : if (no_regs_p && offmemok && !MEM_P (op))
3240 : {
3241 9285449 : if (lra_dump_file != NULL)
3242 27 : fprintf
3243 27 : (lra_dump_file,
3244 : " Using memory insn operand %d: reject+=3\n",
3245 : nop);
3246 9285449 : reject += 3;
3247 : }
3248 :
3249 : /* If reload requires moving value through secondary
3250 : memory, it will need one more insn at least. */
3251 174278550 : if (this_alternative != NO_REGS
3252 164992726 : && REG_P (op) && (cl = get_reg_class (REGNO (op))) != NO_REGS
3253 210050355 : && ((curr_static_id->operand[nop].type != OP_OUT
3254 20251696 : && targetm.secondary_memory_needed (mode, cl,
3255 : this_alternative))
3256 32519170 : || (curr_static_id->operand[nop].type != OP_IN
3257 15520257 : && (targetm.secondary_memory_needed
3258 15520257 : (mode, this_alternative, cl)))))
3259 : {
3260 10780326 : if (lra_dump_file != NULL)
3261 16 : fprintf
3262 16 : (lra_dump_file,
3263 : " %d Secondary memory reload needed: "
3264 : "losers++\n", nop);
3265 10780326 : losers++;
3266 : }
3267 :
3268 174278550 : if (MEM_P (op) && offmemok)
3269 72 : addr_losers++;
3270 : else
3271 : {
3272 : /* Input reloads can be inherited more often than
3273 : output reloads can be removed, so penalize output
3274 : reloads. */
3275 174278478 : if (!REG_P (op) || curr_static_id->operand[nop].type != OP_IN)
3276 : {
3277 146723957 : if (lra_dump_file != NULL)
3278 1722 : fprintf
3279 1722 : (lra_dump_file,
3280 : " %d Non input pseudo reload: reject++\n",
3281 : nop);
3282 146723957 : reject++;
3283 : }
3284 :
3285 174278478 : if (curr_static_id->operand[nop].type == OP_INOUT)
3286 : {
3287 244 : if (lra_dump_file != NULL)
3288 0 : fprintf
3289 0 : (lra_dump_file,
3290 : " %d Input/Output reload: reject+=%d\n",
3291 : nop, LRA_LOSER_COST_FACTOR);
3292 244 : reject += LRA_LOSER_COST_FACTOR;
3293 : }
3294 : }
3295 : }
3296 :
3297 465518931 : if (early_clobber_p && ! scratch_p)
3298 : {
3299 164442 : if (lra_dump_file != NULL)
3300 4 : fprintf (lra_dump_file,
3301 : " %d Early clobber: reject++\n", nop);
3302 164442 : reject++;
3303 : }
3304 : /* ??? We check early clobbers after processing all operands
3305 : (see loop below) and there we update the costs more.
3306 : Should we update the cost (may be approximately) here
3307 : because of early clobber register reloads or it is a rare
3308 : or non-important thing to be worth to do it. */
3309 931037862 : overall = (losers * LRA_LOSER_COST_FACTOR + reject
3310 465518931 : - (addr_losers == losers ? static_reject : 0));
3311 465518931 : if ((best_losers == 0 || losers != 0) && best_overall < overall)
3312 : {
3313 73914886 : if (lra_dump_file != NULL)
3314 1036 : fprintf (lra_dump_file,
3315 : " overall=%d,losers=%d -- refuse\n",
3316 : overall, losers);
3317 73914886 : goto fail;
3318 : }
3319 :
3320 391604045 : if (update_and_check_small_class_inputs (nop, nalt,
3321 : this_alternative))
3322 : {
3323 0 : if (lra_dump_file != NULL)
3324 0 : fprintf (lra_dump_file,
3325 : " not enough small class regs -- refuse\n");
3326 0 : goto fail;
3327 : }
3328 391604045 : curr_alt[nop] = this_alternative;
3329 391604045 : curr_alt_set[nop] = this_alternative_set;
3330 391604045 : curr_alt_exclude_start_hard_regs[nop]
3331 391604045 : = this_alternative_exclude_start_hard_regs;
3332 391604045 : curr_alt_win[nop] = this_alternative_win;
3333 391604045 : curr_alt_match_win[nop] = this_alternative_match_win;
3334 391604045 : curr_alt_offmemok[nop] = this_alternative_offmemok;
3335 391604045 : curr_alt_matches[nop] = this_alternative_matches;
3336 :
3337 391604045 : if (this_alternative_matches >= 0
3338 391604045 : && !did_match && !this_alternative_win)
3339 13034110 : curr_alt_win[this_alternative_matches] = false;
3340 :
3341 391604045 : if (early_clobber_p && operand_reg[nop] != NULL_RTX)
3342 168903 : early_clobbered_nops[early_clobbered_regs_num++] = nop;
3343 : }
3344 :
3345 134030512 : if (curr_insn_set != NULL_RTX
3346 : /* Allow just two operands or three operands where the third
3347 : is a clobber. */
3348 130184355 : && (n_operands == 2
3349 28530762 : || (n_operands == 3
3350 26424436 : && GET_CODE (PATTERN (curr_insn)) == PARALLEL
3351 22404562 : && XVECLEN (PATTERN (curr_insn), 0) == 2
3352 22351440 : && GET_CODE (XVECEXP (PATTERN (curr_insn), 0, 1))
3353 : == CLOBBER))
3354 : /* Prevent processing non-move insns. */
3355 123922365 : && (GET_CODE (SET_SRC (curr_insn_set)) == SUBREG
3356 122155593 : || SET_SRC (curr_insn_set) == no_subreg_reg_operand[1])
3357 224736083 : && ((! curr_alt_win[0] && ! curr_alt_win[1]
3358 5862570 : && REG_P (no_subreg_reg_operand[0])
3359 2856628 : && REG_P (no_subreg_reg_operand[1])
3360 1217831 : && (reg_in_class_p (no_subreg_reg_operand[0], curr_alt[1])
3361 1007356 : || reg_in_class_p (no_subreg_reg_operand[1], curr_alt[0])))
3362 90176884 : || (! curr_alt_win[0] && curr_alt_win[1]
3363 27023460 : && REG_P (no_subreg_reg_operand[1])
3364 : /* Check that we reload memory not the memory
3365 : address. */
3366 15519000 : && ! (curr_alt_offmemok[0]
3367 391624 : && MEM_P (no_subreg_reg_operand[0]))
3368 15519000 : && reg_in_class_p (no_subreg_reg_operand[1], curr_alt[0]))
3369 76055225 : || (curr_alt_win[0] && ! curr_alt_win[1]
3370 9446040 : && REG_P (no_subreg_reg_operand[0])
3371 : /* Check that we reload memory not the memory
3372 : address. */
3373 6976626 : && ! (curr_alt_offmemok[1]
3374 1038151 : && MEM_P (no_subreg_reg_operand[1]))
3375 6976624 : && reg_in_class_p (no_subreg_reg_operand[0], curr_alt[1])
3376 6110179 : && (! CONST_POOL_OK_P (curr_operand_mode[1],
3377 : no_subreg_reg_operand[1])
3378 2253467 : || (targetm.preferred_reload_class
3379 2253467 : (no_subreg_reg_operand[1],
3380 : (enum reg_class) curr_alt[1]) != NO_REGS))
3381 : /* If it is a result of recent elimination in move
3382 : insn we can transform it into an add still by
3383 : using this alternative. */
3384 6067549 : && GET_CODE (no_subreg_reg_operand[1]) != PLUS
3385 : /* Likewise if the source has been replaced with an
3386 : equivalent value. This only happens once -- the reload
3387 : will use the equivalent value instead of the register it
3388 : replaces -- so there should be no danger of cycling. */
3389 5593396 : && !equiv_substition_p[1])))
3390 : {
3391 : /* We have a move insn and a new reload insn will be similar
3392 : to the current insn. We should avoid such situation as
3393 : it results in LRA cycling. */
3394 20214283 : if (lra_dump_file != NULL)
3395 239 : fprintf (lra_dump_file,
3396 : " Cycle danger: overall += LRA_MAX_REJECT\n");
3397 20214283 : overall += LRA_MAX_REJECT;
3398 : }
3399 134030512 : if (all_this_alternative != NO_REGS
3400 114717366 : && !SMALL_REGISTER_CLASS_P (all_this_alternative)
3401 113870736 : && all_used_nregs != 0 && all_reload_nregs != 0
3402 134030512 : && (all_used_nregs + all_reload_nregs + 1
3403 3954505 : >= ira_class_hard_regs_num[all_this_alternative]))
3404 : {
3405 366 : if (lra_dump_file != NULL)
3406 0 : fprintf
3407 0 : (lra_dump_file,
3408 : " Register starvation: overall += LRA_MAX_REJECT"
3409 : "(class=%s,avail=%d,used=%d,reload=%d)\n",
3410 : reg_class_names[all_this_alternative],
3411 : ira_class_hard_regs_num[all_this_alternative],
3412 : all_used_nregs, all_reload_nregs);
3413 366 : overall += LRA_MAX_REJECT;
3414 366 : if (!prefer_memory_p && INSN_CODE (curr_insn) < 0)
3415 : {
3416 : /* asm can permit memory and reg and can be not enough regs for
3417 : asm -- try now memory: */
3418 102 : prefer_memory_p = true;
3419 102 : if (lra_dump_file != NULL)
3420 0 : fprintf
3421 0 : (lra_dump_file,
3422 : " Trying now memory for operands\n");
3423 102 : goto repeat;
3424 : }
3425 : }
3426 134195822 : ok_p = true;
3427 : curr_alt_dont_inherit_ops_num = 0;
3428 134195822 : for (nop = 0; nop < early_clobbered_regs_num; nop++)
3429 : {
3430 165413 : int i, j, clobbered_hard_regno, first_conflict_j, last_conflict_j;
3431 165413 : HARD_REG_SET temp_set;
3432 :
3433 165413 : i = early_clobbered_nops[nop];
3434 165413 : if ((! curr_alt_win[i] && ! curr_alt_match_win[i])
3435 124569 : || hard_regno[i] < 0)
3436 164742 : continue;
3437 122738 : lra_assert (operand_reg[i] != NULL_RTX);
3438 : clobbered_hard_regno = hard_regno[i];
3439 122738 : CLEAR_HARD_REG_SET (temp_set);
3440 122738 : add_to_hard_reg_set (&temp_set, GET_MODE (*curr_id->operand_loc[i]),
3441 : clobbered_hard_regno);
3442 122738 : first_conflict_j = last_conflict_j = -1;
3443 620314 : for (j = 0; j < n_operands; j++)
3444 497577 : if (j == i
3445 : /* We don't want process insides of match_operator and
3446 : match_parallel because otherwise we would process
3447 : their operands once again generating a wrong
3448 : code. */
3449 374839 : || curr_static_id->operand[j].is_operator)
3450 124879 : continue;
3451 372698 : else if ((curr_alt_matches[j] == i && curr_alt_match_win[j])
3452 354813 : || (curr_alt_matches[i] == j && curr_alt_match_win[i]))
3453 17885 : continue;
3454 : /* If we don't reload j-th operand, check conflicts. */
3455 123723 : else if ((curr_alt_win[j] || curr_alt_match_win[j])
3456 415921 : && uses_hard_regs_p (*curr_id->operand_loc[j], temp_set))
3457 : {
3458 1145 : if (first_conflict_j < 0)
3459 671 : first_conflict_j = j;
3460 1145 : last_conflict_j = j;
3461 : /* Both the earlyclobber operand and conflicting operand
3462 : cannot both be user defined hard registers for asm.
3463 : Let curr_insn_transform diagnose it. */
3464 1145 : if (HARD_REGISTER_P (operand_reg[i])
3465 1 : && REG_USERVAR_P (operand_reg[i])
3466 1 : && operand_reg[j] != NULL_RTX
3467 1 : && HARD_REGISTER_P (operand_reg[j])
3468 1 : && REG_USERVAR_P (operand_reg[j])
3469 1146 : && INSN_CODE (curr_insn) < 0)
3470 1 : return false;
3471 : }
3472 122737 : if (last_conflict_j < 0)
3473 122067 : continue;
3474 :
3475 : /* If an earlyclobber operand conflicts with another non-matching
3476 : operand (ie, they have been assigned the same hard register),
3477 : then it is better to reload the other operand, as there may
3478 : exist yet another operand with a matching constraint associated
3479 : with the earlyclobber operand. However, if one of the operands
3480 : is an explicit use of a hard register, then we must reload the
3481 : other non-hard register operand. */
3482 670 : if (HARD_REGISTER_P (operand_reg[i])
3483 670 : || (first_conflict_j == last_conflict_j
3484 196 : && operand_reg[last_conflict_j] != NULL_RTX
3485 60 : && !curr_alt_match_win[last_conflict_j]
3486 60 : && !HARD_REGISTER_P (operand_reg[last_conflict_j])))
3487 : {
3488 60 : curr_alt_win[last_conflict_j] = false;
3489 60 : curr_alt_dont_inherit_ops[curr_alt_dont_inherit_ops_num++]
3490 60 : = last_conflict_j;
3491 60 : losers++;
3492 60 : if (lra_dump_file != NULL)
3493 0 : fprintf
3494 0 : (lra_dump_file,
3495 : " %d Conflict early clobber reload: losers++\n",
3496 : i);
3497 : }
3498 : else
3499 : {
3500 : /* We need to reload early clobbered register and the
3501 : matched registers. */
3502 3044 : for (j = 0; j < n_operands; j++)
3503 2434 : if (curr_alt_matches[j] == i)
3504 : {
3505 2 : curr_alt_match_win[j] = false;
3506 2 : losers++;
3507 2 : if (lra_dump_file != NULL)
3508 0 : fprintf
3509 0 : (lra_dump_file,
3510 : " %d Matching conflict early clobber "
3511 : "reloads: losers++\n",
3512 : j);
3513 2 : overall += LRA_LOSER_COST_FACTOR;
3514 : }
3515 610 : if (! curr_alt_match_win[i])
3516 610 : curr_alt_dont_inherit_ops[curr_alt_dont_inherit_ops_num++] = i;
3517 : else
3518 : {
3519 : /* Remember pseudos used for match reloads are never
3520 : inherited. */
3521 0 : lra_assert (curr_alt_matches[i] >= 0);
3522 0 : curr_alt_win[curr_alt_matches[i]] = false;
3523 : }
3524 610 : curr_alt_win[i] = curr_alt_match_win[i] = false;
3525 610 : losers++;
3526 610 : if (lra_dump_file != NULL)
3527 0 : fprintf
3528 0 : (lra_dump_file,
3529 : " %d Matched conflict early clobber reloads: "
3530 : "losers++\n",
3531 : i);
3532 : }
3533 : /* Early clobber was already reflected in REJECT. */
3534 670 : if (!matching_early_clobber[i])
3535 : {
3536 670 : lra_assert (reject > 0);
3537 670 : reject--;
3538 670 : matching_early_clobber[i] = 1;
3539 : }
3540 670 : overall += LRA_LOSER_COST_FACTOR - 1;
3541 : }
3542 134030409 : if (lra_dump_file != NULL)
3543 1761 : fprintf (lra_dump_file, " overall=%d,losers=%d,rld_nregs=%d\n",
3544 : overall, losers, reload_nregs);
3545 :
3546 : /* If this alternative can be made to work by reloading, and it
3547 : needs less reloading than the others checked so far, record
3548 : it as the chosen goal for reloading. */
3549 134030409 : if ((best_losers != 0 && losers == 0)
3550 59329811 : || (((best_losers == 0 && losers == 0)
3551 58340672 : || (best_losers != 0 && losers != 0))
3552 59329811 : && (best_overall > overall
3553 15267397 : || (best_overall == overall
3554 : /* If the cost of the reloads is the same,
3555 : prefer alternative which requires minimal
3556 : number of reload regs. */
3557 11365030 : && (reload_nregs < best_reload_nregs
3558 11262343 : || (reload_nregs == best_reload_nregs
3559 11220203 : && (best_reload_sum < reload_sum
3560 11199582 : || (best_reload_sum == reload_sum
3561 11176030 : && nalt < goal_alt_number))))))))
3562 : {
3563 386756304 : for (nop = 0; nop < n_operands; nop++)
3564 : {
3565 267627426 : goal_alt_win[nop] = curr_alt_win[nop];
3566 267627426 : goal_alt_match_win[nop] = curr_alt_match_win[nop];
3567 267627426 : goal_alt_matches[nop] = curr_alt_matches[nop];
3568 267627426 : goal_alt[nop] = curr_alt[nop];
3569 267627426 : goal_alt_exclude_start_hard_regs[nop]
3570 267627426 : = curr_alt_exclude_start_hard_regs[nop];
3571 267627426 : goal_alt_offmemok[nop] = curr_alt_offmemok[nop];
3572 : }
3573 119128878 : goal_alt_dont_inherit_ops_num = curr_alt_dont_inherit_ops_num;
3574 119128878 : goal_reuse_alt_p = curr_reuse_alt_p;
3575 119129535 : for (nop = 0; nop < curr_alt_dont_inherit_ops_num; nop++)
3576 657 : goal_alt_dont_inherit_ops[nop] = curr_alt_dont_inherit_ops[nop];
3577 119128878 : goal_alt_swapped = curr_swapped;
3578 119128878 : goal_alt_out_sp_reload_p = curr_alt_out_sp_reload_p;
3579 119128878 : best_overall = overall;
3580 119128878 : best_losers = losers;
3581 119128878 : best_reload_nregs = reload_nregs;
3582 119128878 : best_reload_sum = reload_sum;
3583 119128878 : goal_alt_number = nalt;
3584 : }
3585 134030409 : if (losers == 0 && !curr_alt_class_change_p)
3586 : /* Everything is satisfied. Do not process alternatives
3587 : anymore. */
3588 : break;
3589 58352524 : fail:
3590 180042638 : ;
3591 : }
3592 : return ok_p;
3593 : }
3594 :
3595 : /* Make reload base reg from address AD. */
3596 : static rtx
3597 0 : base_to_reg (struct address_info *ad)
3598 : {
3599 0 : enum reg_class cl;
3600 0 : int code = -1;
3601 0 : rtx new_inner = NULL_RTX;
3602 0 : rtx new_reg = NULL_RTX;
3603 0 : rtx_insn *insn;
3604 0 : rtx_insn *last_insn = get_last_insn();
3605 :
3606 0 : lra_assert (ad->disp == ad->disp_term);
3607 0 : cl = base_reg_class (ad->mode, ad->as, ad->base_outer_code,
3608 : get_index_code (ad));
3609 0 : new_reg = lra_create_new_reg (GET_MODE (*ad->base), NULL_RTX, cl, NULL,
3610 : "base");
3611 0 : new_inner = simplify_gen_binary (PLUS, GET_MODE (new_reg), new_reg,
3612 0 : ad->disp_term == NULL
3613 : ? const0_rtx
3614 : : *ad->disp_term);
3615 0 : if (!valid_address_p (ad->mode, new_inner, ad->as))
3616 : return NULL_RTX;
3617 0 : insn = emit_insn (gen_rtx_SET (new_reg, *ad->base));
3618 0 : code = recog_memoized (insn);
3619 0 : if (code < 0)
3620 : {
3621 0 : delete_insns_since (last_insn);
3622 0 : return NULL_RTX;
3623 : }
3624 :
3625 : return new_inner;
3626 : }
3627 :
3628 : /* Make reload base reg + DISP from address AD. Return the new pseudo. */
3629 : static rtx
3630 39 : base_plus_disp_to_reg (struct address_info *ad, rtx disp)
3631 : {
3632 39 : enum reg_class cl;
3633 39 : rtx new_reg;
3634 :
3635 39 : lra_assert (ad->base == ad->base_term);
3636 39 : cl = base_reg_class (ad->mode, ad->as, ad->base_outer_code,
3637 : get_index_code (ad));
3638 39 : new_reg = lra_create_new_reg (GET_MODE (*ad->base_term), NULL_RTX, cl, NULL,
3639 : "base + disp");
3640 39 : lra_emit_add (new_reg, *ad->base_term, disp);
3641 39 : return new_reg;
3642 : }
3643 :
3644 : /* Make reload of index part of address AD. Return the new
3645 : pseudo. */
3646 : static rtx
3647 0 : index_part_to_reg (struct address_info *ad, enum reg_class index_class)
3648 : {
3649 0 : rtx new_reg;
3650 :
3651 0 : new_reg = lra_create_new_reg (GET_MODE (*ad->index), NULL_RTX,
3652 : index_class, NULL, "index term");
3653 0 : expand_mult (GET_MODE (*ad->index), *ad->index_term,
3654 : GEN_INT (get_index_scale (ad)), new_reg, 1);
3655 0 : return new_reg;
3656 : }
3657 :
3658 : /* Return true if we can add a displacement to address AD, even if that
3659 : makes the address invalid. The fix-up code requires any new address
3660 : to be the sum of the BASE_TERM, INDEX and DISP_TERM fields. */
3661 : static bool
3662 19980 : can_add_disp_p (struct address_info *ad)
3663 : {
3664 19980 : return (!ad->autoinc_p
3665 19980 : && ad->segment == NULL
3666 19980 : && ad->base == ad->base_term
3667 39960 : && ad->disp == ad->disp_term);
3668 : }
3669 :
3670 : /* Make equiv substitution in address AD. Return true if a substitution
3671 : was made. */
3672 : static bool
3673 39289149 : equiv_address_substitution (struct address_info *ad)
3674 : {
3675 39289149 : rtx base_reg, new_base_reg, index_reg, new_index_reg, *base_term, *index_term;
3676 39289149 : poly_int64 disp;
3677 39289149 : HOST_WIDE_INT scale;
3678 39289149 : bool change_p;
3679 :
3680 39289149 : base_term = strip_subreg (ad->base_term);
3681 8951 : if (base_term == NULL)
3682 : base_reg = new_base_reg = NULL_RTX;
3683 : else
3684 : {
3685 33111784 : base_reg = *base_term;
3686 33111784 : new_base_reg = get_equiv_with_elimination (base_reg, curr_insn);
3687 : }
3688 39289149 : index_term = strip_subreg (ad->index_term);
3689 4810 : if (index_term == NULL)
3690 : index_reg = new_index_reg = NULL_RTX;
3691 : else
3692 : {
3693 1889445 : index_reg = *index_term;
3694 1889445 : new_index_reg = get_equiv_with_elimination (index_reg, curr_insn);
3695 : }
3696 39289149 : if (base_reg == new_base_reg && index_reg == new_index_reg)
3697 : return false;
3698 142547 : disp = 0;
3699 142547 : change_p = false;
3700 142547 : if (lra_dump_file != NULL)
3701 : {
3702 0 : fprintf (lra_dump_file, "Changing address in insn %d ",
3703 0 : INSN_UID (curr_insn));
3704 0 : dump_value_slim (lra_dump_file, *ad->outer, 1);
3705 : }
3706 142547 : if (base_reg != new_base_reg)
3707 : {
3708 142044 : poly_int64 offset;
3709 142044 : if (REG_P (new_base_reg))
3710 : {
3711 11499 : *base_term = new_base_reg;
3712 11499 : change_p = true;
3713 : }
3714 130545 : else if (GET_CODE (new_base_reg) == PLUS
3715 19980 : && REG_P (XEXP (new_base_reg, 0))
3716 19980 : && poly_int_rtx_p (XEXP (new_base_reg, 1), &offset)
3717 150525 : && can_add_disp_p (ad))
3718 : {
3719 : disp += offset;
3720 19980 : *base_term = XEXP (new_base_reg, 0);
3721 19980 : change_p = true;
3722 : }
3723 142044 : if (ad->base_term2 != NULL)
3724 0 : *ad->base_term2 = *ad->base_term;
3725 : }
3726 142547 : if (index_reg != new_index_reg)
3727 : {
3728 709 : poly_int64 offset;
3729 709 : if (REG_P (new_index_reg))
3730 : {
3731 0 : *index_term = new_index_reg;
3732 0 : change_p = true;
3733 : }
3734 709 : else if (GET_CODE (new_index_reg) == PLUS
3735 0 : && REG_P (XEXP (new_index_reg, 0))
3736 0 : && poly_int_rtx_p (XEXP (new_index_reg, 1), &offset)
3737 0 : && can_add_disp_p (ad)
3738 709 : && (scale = get_index_scale (ad)))
3739 : {
3740 0 : disp += offset * scale;
3741 0 : *index_term = XEXP (new_index_reg, 0);
3742 0 : change_p = true;
3743 : }
3744 : }
3745 142547 : if (maybe_ne (disp, 0))
3746 : {
3747 19980 : if (ad->disp != NULL)
3748 6766 : *ad->disp = plus_constant (GET_MODE (*ad->inner), *ad->disp, disp);
3749 : else
3750 : {
3751 13214 : *ad->inner = plus_constant (GET_MODE (*ad->inner), *ad->inner, disp);
3752 13214 : update_address (ad);
3753 : }
3754 : change_p = true;
3755 : }
3756 142547 : if (lra_dump_file != NULL)
3757 : {
3758 0 : if (! change_p)
3759 0 : fprintf (lra_dump_file, " -- no change\n");
3760 : else
3761 : {
3762 0 : fprintf (lra_dump_file, " on equiv ");
3763 0 : dump_value_slim (lra_dump_file, *ad->outer, 1);
3764 0 : fprintf (lra_dump_file, "\n");
3765 : }
3766 : }
3767 : return change_p;
3768 : }
3769 :
3770 : /* Skip all modifiers and whitespaces in constraint STR and return the
3771 : result. */
3772 : static const char *
3773 508665587 : skip_constraint_modifiers (const char *str)
3774 : {
3775 716858573 : for (;;str++)
3776 612762080 : switch (*str)
3777 : {
3778 104096493 : case '+': case '&' : case '=': case '*': case ' ': case '\t':
3779 104096493 : case '$': case '^' : case '%': case '?': case '!':
3780 104096493 : break;
3781 508665587 : default: return str;
3782 : }
3783 : }
3784 :
3785 : /* Takes a string of 0 or more comma-separated constraints. When more
3786 : than one constraint is present, evaluate whether they all correspond
3787 : to a single, repeated constraint (e.g. "r,r") or whether we have
3788 : more than one distinct constraints (e.g. "r,m"). */
3789 : static bool
3790 161670512 : constraint_unique (const char *cstr)
3791 : {
3792 161670512 : enum constraint_num ca, cb;
3793 161670512 : ca = CONSTRAINT__UNKNOWN;
3794 319845946 : for (;;)
3795 : {
3796 319845946 : cstr = skip_constraint_modifiers (cstr);
3797 319845946 : if (*cstr == '\0' || *cstr == ',')
3798 : cb = CONSTRAINT_X;
3799 : else
3800 : {
3801 319845946 : cb = lookup_constraint (cstr);
3802 319845946 : if (cb == CONSTRAINT__UNKNOWN)
3803 : return false;
3804 307807759 : cstr += CONSTRAINT_LEN (cstr[0], cstr);
3805 : }
3806 : /* Handle the first iteration of the loop. */
3807 307807759 : if (ca == CONSTRAINT__UNKNOWN)
3808 : ca = cb;
3809 : /* Handle the general case of comparing ca with subsequent
3810 : constraints. */
3811 158045003 : else if (ca != cb)
3812 : return false;
3813 165530280 : if (*cstr == '\0')
3814 : return true;
3815 158175434 : if (*cstr == ',')
3816 87327625 : cstr += 1;
3817 : }
3818 : }
3819 :
3820 : /* Major function to make reloads for an address in operand NOP or
3821 : check its correctness (If CHECK_ONLY_P is true). The supported
3822 : cases are:
3823 :
3824 : 1) an address that existed before LRA started, at which point it
3825 : must have been valid. These addresses are subject to elimination
3826 : and may have become invalid due to the elimination offset being out
3827 : of range.
3828 :
3829 : 2) an address created by forcing a constant to memory
3830 : (force_const_to_mem). The initial form of these addresses might
3831 : not be valid, and it is this function's job to make them valid.
3832 :
3833 : 3) a frame address formed from a register and a (possibly zero)
3834 : constant offset. As above, these addresses might not be valid and
3835 : this function must make them so.
3836 :
3837 : Add reloads to the lists *BEFORE and *AFTER. We might need to add
3838 : reloads to *AFTER because of inc/dec, {pre, post} modify in the
3839 : address. Return true for any RTL change.
3840 :
3841 : The function is a helper function which does not produce all
3842 : transformations (when CHECK_ONLY_P is false) which can be
3843 : necessary. It does just basic steps. To do all necessary
3844 : transformations use function process_address. */
3845 : static bool
3846 175214405 : process_address_1 (int nop, bool check_only_p,
3847 : rtx_insn **before, rtx_insn **after)
3848 : {
3849 175214405 : struct address_info ad;
3850 175214405 : rtx new_reg;
3851 175214405 : HOST_WIDE_INT scale;
3852 175214405 : rtx op = *curr_id->operand_loc[nop];
3853 175214405 : rtx mem = extract_mem_from_operand (op);
3854 175214405 : const char *constraint;
3855 175214405 : enum constraint_num cn;
3856 175214405 : bool change_p = false;
3857 :
3858 175214405 : if (MEM_P (mem)
3859 37563819 : && GET_MODE (mem) == BLKmode
3860 25515 : && GET_CODE (XEXP (mem, 0)) == SCRATCH)
3861 : return false;
3862 :
3863 175214405 : constraint
3864 175214405 : = skip_constraint_modifiers (curr_static_id->operand[nop].constraint);
3865 175214405 : if (IN_RANGE (constraint[0], '0', '9'))
3866 : {
3867 13605236 : char *end;
3868 13605236 : unsigned long dup = strtoul (constraint, &end, 10);
3869 13605236 : constraint
3870 13605236 : = skip_constraint_modifiers (curr_static_id->operand[dup].constraint);
3871 : }
3872 187025889 : cn = lookup_constraint (*constraint == '\0' ? "X" : constraint);
3873 : /* If we have several alternatives or/and several constraints in an
3874 : alternative and we can not say at this stage what constraint will be used,
3875 : use unknown constraint. The exception is an address constraint. If
3876 : operand has one address constraint, probably all others constraints are
3877 : address ones. */
3878 163402921 : if (constraint[0] != '\0' && get_constraint_type (cn) != CT_ADDRESS
3879 336884917 : && !constraint_unique (constraint))
3880 : cn = CONSTRAINT__UNKNOWN;
3881 20898739 : if (insn_extra_address_constraint (cn)
3882 : /* When we find an asm operand with an address constraint that
3883 : doesn't satisfy address_operand to begin with, we clear
3884 : is_address, so that we don't try to make a non-address fit.
3885 : If the asm statement got this far, it's because other
3886 : constraints are available, and we'll use them, disregarding
3887 : the unsatisfiable address ones. */
3888 20898739 : && curr_static_id->operand[nop].is_address)
3889 1732390 : decompose_lea_address (&ad, curr_id->operand_loc[nop]);
3890 : /* Do not attempt to decompose arbitrary addresses generated by combine
3891 : for asm operands with loose constraints, e.g 'X'.
3892 : Need to extract memory from op for special memory constraint,
3893 : i.e. bcst_mem_operand in i386 backend. */
3894 173482015 : else if (MEM_P (mem)
3895 173482201 : && !(INSN_CODE (curr_insn) < 0
3896 19359 : && get_constraint_type (cn) == CT_FIXED_FORM
3897 186 : && constraint_satisfied_p (op, cn)))
3898 37563633 : decompose_mem_address (&ad, mem);
3899 135918382 : else if (GET_CODE (op) == SUBREG
3900 3584255 : && MEM_P (SUBREG_REG (op)))
3901 0 : decompose_mem_address (&ad, SUBREG_REG (op));
3902 : else
3903 : return false;
3904 : /* If INDEX_REG_CLASS is assigned to base_term already and isn't to
3905 : index_term, swap them so to avoid assigning INDEX_REG_CLASS to both
3906 : when INDEX_REG_CLASS is a single register class. */
3907 39296023 : enum reg_class index_cl = index_reg_class (curr_insn);
3908 39296023 : if (ad.base_term != NULL
3909 33118587 : && ad.index_term != NULL
3910 1521173 : && ira_class_hard_regs_num[index_cl] == 1
3911 0 : && REG_P (*ad.base_term)
3912 0 : && REG_P (*ad.index_term)
3913 0 : && in_class_p (*ad.base_term, index_cl, NULL)
3914 39296023 : && ! in_class_p (*ad.index_term, index_cl, NULL))
3915 : {
3916 0 : std::swap (ad.base, ad.index);
3917 0 : std::swap (ad.base_term, ad.index_term);
3918 : }
3919 39296023 : if (! check_only_p)
3920 39289149 : change_p = equiv_address_substitution (&ad);
3921 39296023 : if (ad.base_term != NULL
3922 72414610 : && (process_addr_reg
3923 66237174 : (ad.base_term, check_only_p, before,
3924 33118587 : (ad.autoinc_p
3925 4173556 : && !(REG_P (*ad.base_term)
3926 2086778 : && find_regno_note (curr_insn, REG_DEAD,
3927 : REGNO (*ad.base_term)) != NULL_RTX)
3928 : ? after : NULL),
3929 33118587 : base_reg_class (ad.mode, ad.as, ad.base_outer_code,
3930 : get_index_code (&ad), curr_insn))))
3931 : {
3932 434884 : change_p = true;
3933 434884 : if (ad.base_term2 != NULL)
3934 0 : *ad.base_term2 = *ad.base_term;
3935 : }
3936 39296023 : if (ad.index_term != NULL
3937 39296023 : && process_addr_reg (ad.index_term, check_only_p,
3938 : before, NULL, index_cl))
3939 : change_p = true;
3940 :
3941 : /* Target hooks sometimes don't treat extra-constraint addresses as
3942 : legitimate address_operands, so handle them specially. */
3943 39296023 : if (insn_extra_address_constraint (cn)
3944 39296023 : && satisfies_address_constraint_p (&ad, cn))
3945 : return change_p;
3946 :
3947 37563640 : if (check_only_p)
3948 : return change_p;
3949 :
3950 : /* There are three cases where the shape of *AD.INNER may now be invalid:
3951 :
3952 : 1) the original address was valid, but either elimination or
3953 : equiv_address_substitution was applied and that made
3954 : the address invalid.
3955 :
3956 : 2) the address is an invalid symbolic address created by
3957 : force_const_to_mem.
3958 :
3959 : 3) the address is a frame address with an invalid offset.
3960 :
3961 : 4) the address is a frame address with an invalid base.
3962 :
3963 : All these cases involve a non-autoinc address, so there is no
3964 : point revalidating other types. */
3965 37557356 : if (ad.autoinc_p || valid_address_p (op, &ad, cn))
3966 37556936 : return change_p;
3967 :
3968 : /* Any index existed before LRA started, so we can assume that the
3969 : presence and shape of the index is valid. */
3970 420 : push_to_sequence (*before);
3971 420 : lra_assert (ad.disp == ad.disp_term);
3972 420 : if (ad.base == NULL)
3973 : {
3974 328 : if (ad.index == NULL)
3975 : {
3976 328 : rtx_insn *insn;
3977 328 : rtx_insn *last = get_last_insn ();
3978 328 : int code = -1;
3979 328 : enum reg_class cl = base_reg_class (ad.mode, ad.as,
3980 : SCRATCH, SCRATCH,
3981 : curr_insn);
3982 328 : rtx addr = *ad.inner;
3983 :
3984 647 : new_reg = lra_create_new_reg (Pmode, NULL_RTX, cl, NULL, "addr");
3985 328 : if (HAVE_lo_sum)
3986 : {
3987 : /* addr => lo_sum (new_base, addr), case (2) above. */
3988 : insn = emit_insn (gen_rtx_SET
3989 : (new_reg,
3990 : gen_rtx_HIGH (Pmode, copy_rtx (addr))));
3991 : code = recog_memoized (insn);
3992 : if (code >= 0)
3993 : {
3994 : *ad.inner = gen_rtx_LO_SUM (Pmode, new_reg, addr);
3995 : if (!valid_address_p (op, &ad, cn))
3996 : {
3997 : /* Try to put lo_sum into register. */
3998 : insn = emit_insn (gen_rtx_SET
3999 : (new_reg,
4000 : gen_rtx_LO_SUM (Pmode, new_reg, addr)));
4001 : code = recog_memoized (insn);
4002 : if (code >= 0)
4003 : {
4004 : *ad.inner = new_reg;
4005 : if (!valid_address_p (op, &ad, cn))
4006 : {
4007 : *ad.inner = addr;
4008 : code = -1;
4009 : }
4010 : }
4011 :
4012 : }
4013 : }
4014 : if (code < 0)
4015 : delete_insns_since (last);
4016 : }
4017 :
4018 328 : if (code < 0)
4019 : {
4020 : /* addr => new_base, case (2) above. */
4021 328 : lra_emit_move (new_reg, addr);
4022 :
4023 656 : for (insn = last == NULL_RTX ? get_insns () : NEXT_INSN (last);
4024 656 : insn != NULL_RTX;
4025 328 : insn = NEXT_INSN (insn))
4026 328 : if (recog_memoized (insn) < 0)
4027 : break;
4028 328 : if (insn != NULL_RTX)
4029 : {
4030 : /* Do nothing if we cannot generate right insns.
4031 : This is analogous to reload pass behavior. */
4032 0 : delete_insns_since (last);
4033 0 : end_sequence ();
4034 0 : return false;
4035 : }
4036 328 : *ad.inner = new_reg;
4037 : }
4038 : }
4039 : else
4040 : {
4041 : /* index * scale + disp => new base + index * scale,
4042 : case (1) above. */
4043 0 : enum reg_class cl = base_reg_class (ad.mode, ad.as, PLUS,
4044 0 : GET_CODE (*ad.index),
4045 : curr_insn);
4046 :
4047 0 : lra_assert (index_cl != NO_REGS);
4048 0 : new_reg = lra_create_new_reg (Pmode, NULL_RTX, cl, NULL, "disp");
4049 0 : lra_emit_move (new_reg, *ad.disp);
4050 0 : *ad.inner = simplify_gen_binary (PLUS, GET_MODE (new_reg),
4051 0 : new_reg, *ad.index);
4052 : }
4053 : }
4054 92 : else if (ad.index == NULL)
4055 : {
4056 53 : int regno;
4057 53 : enum reg_class cl;
4058 53 : rtx set;
4059 53 : rtx_insn *insns, *last_insn;
4060 :
4061 53 : cl = base_reg_class (ad.mode, ad.as, ad.base_outer_code,
4062 : get_index_code (&ad), curr_insn);
4063 :
4064 53 : if (REG_P (*ad.base_term)
4065 53 : && ira_class_subset_p[get_reg_class (REGNO (*ad.base_term))][cl])
4066 : /* It seems base reg is already in the base reg class and changing it
4067 : does not make a progress. So reload the whole inner address. */
4068 53 : goto reload_inner_addr;
4069 :
4070 : /* Try to reload base into register only if the base is invalid
4071 : for the address but with valid offset, case (4) above. */
4072 0 : start_sequence ();
4073 0 : new_reg = base_to_reg (&ad);
4074 :
4075 : /* base + disp => new base, cases (1) and (3) above. */
4076 : /* Another option would be to reload the displacement into an
4077 : index register. However, postreload has code to optimize
4078 : address reloads that have the same base and different
4079 : displacements, so reloading into an index register would
4080 : not necessarily be a win. */
4081 0 : if (new_reg == NULL_RTX)
4082 : {
4083 : /* See if the target can split the displacement into a
4084 : legitimate new displacement from a local anchor. */
4085 0 : gcc_assert (ad.disp == ad.disp_term);
4086 0 : poly_int64 orig_offset;
4087 0 : rtx offset1, offset2;
4088 0 : if (poly_int_rtx_p (*ad.disp, &orig_offset)
4089 0 : && targetm.legitimize_address_displacement (&offset1, &offset2,
4090 : orig_offset,
4091 : ad.mode))
4092 : {
4093 0 : new_reg = base_plus_disp_to_reg (&ad, offset1);
4094 0 : new_reg = gen_rtx_PLUS (GET_MODE (new_reg), new_reg, offset2);
4095 : }
4096 : else
4097 0 : new_reg = base_plus_disp_to_reg (&ad, *ad.disp);
4098 : }
4099 0 : insns = get_insns ();
4100 0 : last_insn = get_last_insn ();
4101 : /* If we generated at least two insns, try last insn source as
4102 : an address. If we succeed, we generate one less insn. */
4103 0 : if (REG_P (new_reg)
4104 0 : && last_insn != insns
4105 0 : && (set = single_set (last_insn)) != NULL_RTX
4106 0 : && GET_CODE (SET_SRC (set)) == PLUS
4107 0 : && REG_P (XEXP (SET_SRC (set), 0))
4108 0 : && CONSTANT_P (XEXP (SET_SRC (set), 1)))
4109 : {
4110 0 : *ad.inner = SET_SRC (set);
4111 0 : if (valid_address_p (op, &ad, cn))
4112 : {
4113 0 : *ad.base_term = XEXP (SET_SRC (set), 0);
4114 0 : *ad.disp_term = XEXP (SET_SRC (set), 1);
4115 0 : regno = REGNO (*ad.base_term);
4116 0 : if (regno >= FIRST_PSEUDO_REGISTER
4117 0 : && cl != lra_get_allocno_class (regno))
4118 0 : lra_change_class (regno, cl, " Change to", true);
4119 0 : new_reg = SET_SRC (set);
4120 0 : delete_insns_since (PREV_INSN (last_insn));
4121 : }
4122 : }
4123 0 : end_sequence ();
4124 0 : emit_insn (insns);
4125 0 : *ad.inner = new_reg;
4126 : }
4127 39 : else if (ad.disp_term != NULL)
4128 : {
4129 : /* base + scale * index + disp => new base + scale * index,
4130 : case (1) above. */
4131 39 : gcc_assert (ad.disp == ad.disp_term);
4132 39 : new_reg = base_plus_disp_to_reg (&ad, *ad.disp);
4133 39 : *ad.inner = simplify_gen_binary (PLUS, GET_MODE (new_reg),
4134 39 : new_reg, *ad.index);
4135 : }
4136 0 : else if ((scale = get_index_scale (&ad)) == 1)
4137 : {
4138 : /* The last transformation to one reg will be made in
4139 : curr_insn_transform function. */
4140 0 : end_sequence ();
4141 0 : return false;
4142 : }
4143 0 : else if (scale != 0)
4144 : {
4145 : /* base + scale * index => base + new_reg,
4146 : case (1) above.
4147 : Index part of address may become invalid. For example, we
4148 : changed pseudo on the equivalent memory and a subreg of the
4149 : pseudo onto the memory of different mode for which the scale is
4150 : prohibitted. */
4151 0 : new_reg = index_part_to_reg (&ad, index_cl);
4152 0 : *ad.inner = simplify_gen_binary (PLUS, GET_MODE (new_reg),
4153 0 : *ad.base_term, new_reg);
4154 : }
4155 : else
4156 : {
4157 53 : enum reg_class cl;
4158 53 : rtx addr;
4159 0 : reload_inner_addr:
4160 53 : cl = base_reg_class (ad.mode, ad.as, SCRATCH, SCRATCH, curr_insn);
4161 53 : addr = *ad.inner;
4162 53 : new_reg = lra_create_new_reg (Pmode, NULL_RTX, cl, NULL, "addr");
4163 : /* addr => new_base. */
4164 53 : lra_emit_move (new_reg, addr);
4165 53 : *ad.inner = new_reg;
4166 : }
4167 420 : *before = end_sequence ();
4168 420 : return true;
4169 : }
4170 :
4171 : /* If CHECK_ONLY_P is false, do address reloads until it is necessary.
4172 : Use process_address_1 as a helper function. Return true for any
4173 : RTL changes.
4174 :
4175 : If CHECK_ONLY_P is true, just check address correctness. Return
4176 : false if the address correct. */
4177 : static bool
4178 174722552 : process_address (int nop, bool check_only_p,
4179 : rtx_insn **before, rtx_insn **after)
4180 : {
4181 174722552 : bool res = false;
4182 : /* Use enough iterations to process all address parts: */
4183 175214405 : for (int i = 0; i < 10; i++)
4184 : {
4185 175214405 : if (!process_address_1 (nop, check_only_p, before, after))
4186 : {
4187 : return res;
4188 : }
4189 : else
4190 : {
4191 491853 : if (check_only_p)
4192 : return true;
4193 491853 : res = true;
4194 : }
4195 : }
4196 0 : fatal_insn ("unable to reload address in ", curr_insn);
4197 : }
4198 :
4199 : /* Override the generic address_reload_context in order to
4200 : control the creation of reload pseudos. */
4201 : class lra_autoinc_reload_context : public address_reload_context
4202 : {
4203 : machine_mode mode;
4204 : enum reg_class rclass;
4205 :
4206 : public:
4207 0 : lra_autoinc_reload_context (machine_mode mode, enum reg_class new_rclass)
4208 0 : : mode (mode), rclass (new_rclass) {}
4209 :
4210 0 : rtx get_reload_reg () const override final
4211 : {
4212 0 : return lra_create_new_reg (mode, NULL_RTX, rclass, NULL, "INC/DEC result");
4213 : }
4214 : };
4215 :
4216 : /* Emit insns to reload VALUE into a new register. VALUE is an
4217 : auto-increment or auto-decrement RTX whose operand is a register or
4218 : memory location; so reloading involves incrementing that location.
4219 :
4220 : INC_AMOUNT is the number to increment or decrement by (always
4221 : positive and ignored for POST_MODIFY/PRE_MODIFY).
4222 :
4223 : Return a pseudo containing the result. */
4224 : static rtx
4225 0 : emit_inc (enum reg_class new_rclass, rtx value, poly_int64 inc_amount)
4226 : {
4227 0 : lra_autoinc_reload_context context (GET_MODE (value), new_rclass);
4228 0 : return context.emit_autoinc (value, inc_amount);
4229 : }
4230 :
4231 : /* Return true if the current move insn does not need processing as we
4232 : already know that it satisfies its constraints. */
4233 : static bool
4234 100119071 : simple_move_p (void)
4235 : {
4236 100119071 : rtx dest, src;
4237 100119071 : enum reg_class dclass, sclass;
4238 :
4239 100119071 : lra_assert (curr_insn_set != NULL_RTX);
4240 100119071 : dest = SET_DEST (curr_insn_set);
4241 100119071 : src = SET_SRC (curr_insn_set);
4242 :
4243 : /* If the instruction has multiple sets we need to process it even if it
4244 : is single_set. This can happen if one or more of the SETs are dead.
4245 : See PR73650. */
4246 100119071 : if (multiple_sets (curr_insn))
4247 : return false;
4248 :
4249 99931583 : return ((dclass = get_op_class (dest)) != NO_REGS
4250 20975441 : && (sclass = get_op_class (src)) != NO_REGS
4251 : /* The backend guarantees that register moves of cost 2
4252 : never need reloads. */
4253 89001230 : && targetm.register_move_cost (GET_MODE (src), sclass, dclass) == 2);
4254 : }
4255 :
4256 : /* Swap operands NOP and NOP + 1. */
4257 : static inline void
4258 21361179 : swap_operands (int nop)
4259 : {
4260 21361179 : std::swap (curr_operand_mode[nop], curr_operand_mode[nop + 1]);
4261 21361179 : std::swap (original_subreg_reg_mode[nop], original_subreg_reg_mode[nop + 1]);
4262 21361179 : std::swap (*curr_id->operand_loc[nop], *curr_id->operand_loc[nop + 1]);
4263 21361179 : std::swap (equiv_substition_p[nop], equiv_substition_p[nop + 1]);
4264 : /* Swap the duplicates too. */
4265 21361179 : lra_update_dup (curr_id, nop);
4266 21361179 : lra_update_dup (curr_id, nop + 1);
4267 21361179 : }
4268 :
4269 : /* Return TRUE if X is a (subreg of) reg and there are no hard regs of X class
4270 : which can contain value of MODE. */
4271 38 : static bool invalid_mode_reg_p (enum machine_mode mode, rtx x)
4272 : {
4273 38 : if (SUBREG_P (x))
4274 3 : x = SUBREG_REG (x);
4275 38 : if (! REG_P (x))
4276 : return false;
4277 38 : enum reg_class rclass = get_reg_class (REGNO (x));
4278 38 : return (!hard_reg_set_empty_p (reg_class_contents[rclass])
4279 38 : && hard_reg_set_subset_p
4280 38 : (reg_class_contents[rclass],
4281 38 : ira_prohibited_class_mode_regs[rclass][mode]));
4282 : }
4283 :
4284 : /* Return TRUE if regno is referenced in more than one non-debug insn. */
4285 : static bool
4286 2864590 : multiple_insn_refs_p (int regno)
4287 : {
4288 2864590 : unsigned int uid;
4289 2864590 : bitmap_iterator bi;
4290 2864590 : int nrefs = 0;
4291 6922335 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi)
4292 : {
4293 6916748 : if (!NONDEBUG_INSN_P (lra_insn_recog_data[uid]->insn))
4294 1193155 : continue;
4295 5723593 : if (nrefs == 1)
4296 : return true;
4297 2864590 : nrefs++;
4298 : }
4299 : return false;
4300 : }
4301 :
4302 : /* Mark insns starting with FIRST as postponed for processing their
4303 : constraints. See comments for lra_postponed_insns. */
4304 : static void
4305 92340 : postpone_insns (rtx_insn *first)
4306 : {
4307 104985 : for (auto insn = first; insn != NULL_RTX; insn = NEXT_INSN (insn))
4308 : {
4309 12645 : bitmap_set_bit (&lra_postponed_insns, INSN_UID (insn));
4310 12645 : if (lra_dump_file != NULL)
4311 : {
4312 7 : fprintf (lra_dump_file, " Postponing constraint processing: ");
4313 7 : dump_insn_slim (lra_dump_file, insn);
4314 : }
4315 : }
4316 92340 : }
4317 :
4318 : /* Main entry point of the constraint code: search the body of the
4319 : current insn to choose the best alternative. It is mimicking insn
4320 : alternative cost calculation model of former reload pass. That is
4321 : because machine descriptions were written to use this model. This
4322 : model can be changed in future. Make commutative operand exchange
4323 : if it is chosen.
4324 :
4325 : if CHECK_ONLY_P is false, do RTL changes to satisfy the
4326 : constraints. Return true if any change happened during function
4327 : call.
4328 :
4329 : If CHECK_ONLY_P is true then don't do any transformation. Just
4330 : check that the insn satisfies all constraints. If the insn does
4331 : not satisfy any constraint, return true. */
4332 : static bool
4333 105452259 : curr_insn_transform (bool check_only_p)
4334 : {
4335 105452259 : int i, j, k;
4336 105452259 : int n_operands;
4337 105452259 : int n_alternatives;
4338 105452259 : int n_outputs;
4339 105452259 : int commutative;
4340 105452259 : signed char goal_alt_matched[MAX_RECOG_OPERANDS][MAX_RECOG_OPERANDS];
4341 105452259 : signed char match_inputs[MAX_RECOG_OPERANDS + 1];
4342 105452259 : signed char outputs[MAX_RECOG_OPERANDS + 1];
4343 105452259 : rtx_insn *before, *after;
4344 105452259 : bool alt_p = false;
4345 : /* Flag that the insn has been changed through a transformation. */
4346 105452259 : bool change_p;
4347 105452259 : bool sec_mem_p;
4348 105452259 : bool use_sec_mem_p;
4349 105452259 : int max_regno_before;
4350 105452259 : int reused_alternative_num;
4351 :
4352 105452259 : curr_insn_set = single_set (curr_insn);
4353 105452259 : if (curr_insn_set != NULL_RTX && simple_move_p ())
4354 : {
4355 : /* We assume that the corresponding insn alternative has no
4356 : earlier clobbers. If it is not the case, don't define move
4357 : cost equal to 2 for the corresponding register classes. */
4358 16094917 : lra_set_used_insn_alternative (curr_insn, LRA_NON_CLOBBERED_ALT);
4359 16094917 : return false;
4360 : }
4361 :
4362 89357342 : no_input_reloads_p = no_output_reloads_p = false;
4363 89357342 : goal_alt_number = -1;
4364 89357342 : change_p = sec_mem_p = false;
4365 :
4366 : /* CALL_INSNs are not allowed to have any output reloads. */
4367 89357342 : if (CALL_P (curr_insn))
4368 5954347 : no_output_reloads_p = true;
4369 :
4370 89357342 : n_operands = curr_static_id->n_operands;
4371 89357342 : n_alternatives = curr_static_id->n_alternatives;
4372 :
4373 : /* Just return "no reloads" if insn has no operands with
4374 : constraints. */
4375 89357342 : if (n_operands == 0 || n_alternatives == 0)
4376 : return false;
4377 :
4378 78962795 : max_regno_before = max_reg_num ();
4379 :
4380 333914869 : for (i = 0; i < n_operands; i++)
4381 : {
4382 175989279 : goal_alt_matched[i][0] = -1;
4383 175989279 : goal_alt_matches[i] = -1;
4384 : }
4385 :
4386 78962795 : commutative = curr_static_id->commutative;
4387 :
4388 : /* Now see what we need for pseudos that didn't get hard regs or got
4389 : the wrong kind of hard reg. For this, we must consider all the
4390 : operands together against the register constraints. */
4391 :
4392 78962795 : best_losers = best_overall = INT_MAX;
4393 78962795 : best_reload_sum = 0;
4394 :
4395 78962795 : curr_swapped = false;
4396 78962795 : goal_alt_swapped = false;
4397 :
4398 78962795 : if (! check_only_p)
4399 : /* Make equivalence substitution and memory subreg elimination
4400 : before address processing because an address legitimacy can
4401 : depend on memory mode. */
4402 254881986 : for (i = 0; i < n_operands; i++)
4403 : {
4404 175938822 : rtx op, subst, old;
4405 175938822 : bool op_change_p = false;
4406 :
4407 175938822 : if (curr_static_id->operand[i].is_operator)
4408 1397483 : continue;
4409 :
4410 174541339 : old = op = *curr_id->operand_loc[i];
4411 174541339 : if (GET_CODE (old) == SUBREG)
4412 3638908 : old = SUBREG_REG (old);
4413 174541339 : subst = get_equiv_with_elimination (old, curr_insn);
4414 174541339 : original_subreg_reg_mode[i] = VOIDmode;
4415 174541339 : equiv_substition_p[i] = false;
4416 :
4417 174541339 : if (subst != old
4418 : /* We don't want to change an out operand by constant or invariant
4419 : which will require additional reloads, e.g. by putting a constant
4420 : into memory. */
4421 1501281 : && (curr_static_id->operand[i].type == OP_IN || MEM_P (subst)
4422 0 : || (GET_CODE (subst) == SUBREG && MEM_P (SUBREG_REG (subst)))))
4423 : {
4424 1501281 : equiv_substition_p[i] = true;
4425 1501281 : rtx new_subst = copy_rtx (subst);
4426 1501281 : if (lra_pointer_equiv_set_in (subst))
4427 766993 : lra_pointer_equiv_set_add (new_subst);
4428 1501281 : subst = new_subst;
4429 1501281 : lra_assert (REG_P (old));
4430 1501281 : if (GET_CODE (op) != SUBREG)
4431 1447245 : *curr_id->operand_loc[i] = subst;
4432 : else
4433 : {
4434 54036 : SUBREG_REG (op) = subst;
4435 54036 : if (GET_MODE (subst) == VOIDmode)
4436 90 : original_subreg_reg_mode[i] = GET_MODE (old);
4437 : }
4438 1501281 : if (lra_dump_file != NULL)
4439 : {
4440 3 : fprintf (lra_dump_file,
4441 : "Changing pseudo %d in operand %i of insn %u on equiv ",
4442 3 : REGNO (old), i, INSN_UID (curr_insn));
4443 3 : dump_value_slim (lra_dump_file, subst, 1);
4444 3 : fprintf (lra_dump_file, "\n");
4445 : }
4446 1501281 : op_change_p = change_p = true;
4447 : }
4448 174541339 : if (simplify_operand_subreg (i, GET_MODE (old)) || op_change_p)
4449 : {
4450 1502004 : change_p = true;
4451 1502004 : lra_update_dup (curr_id, i);
4452 : }
4453 : }
4454 :
4455 : /* We process equivalences before ignoring postponed insns on the current
4456 : constraint sub-pass but before any reload insn generation for the
4457 : postponed insn. */
4458 78943164 : if (! check_only_p
4459 78943164 : && bitmap_bit_p (&lra_postponed_insns, INSN_UID (curr_insn)))
4460 : return true;
4461 :
4462 : /* Reload address registers and displacements. We do it before
4463 : finding an alternative because of memory constraints. */
4464 78956089 : before = after = NULL;
4465 254931956 : for (i = 0; i < n_operands; i++)
4466 175975867 : if (! curr_static_id->operand[i].is_operator
4467 175975867 : && process_address (i, check_only_p, &before, &after))
4468 : {
4469 491850 : if (check_only_p)
4470 : return true;
4471 491850 : change_p = true;
4472 491850 : lra_update_dup (curr_id, i);
4473 : }
4474 :
4475 78956089 : if (change_p)
4476 : /* If we've changed the instruction then any alternative that
4477 : we chose previously may no longer be valid. */
4478 1946228 : lra_set_used_insn_alternative (curr_insn, LRA_UNKNOWN_ALT);
4479 :
4480 78936458 : if (! check_only_p && curr_insn_set != NULL_RTX
4481 154105159 : && check_and_process_move (&change_p, &sec_mem_p))
4482 0 : return change_p;
4483 :
4484 78956089 : try_swapped:
4485 :
4486 89334649 : reused_alternative_num = check_only_p ? LRA_UNKNOWN_ALT : curr_id->used_insn_alternative;
4487 89334649 : if (lra_dump_file != NULL && reused_alternative_num >= 0)
4488 0 : fprintf (lra_dump_file, "Reusing alternative %d for insn #%u\n",
4489 0 : reused_alternative_num, INSN_UID (curr_insn));
4490 :
4491 89334649 : if (process_alt_operands (reused_alternative_num))
4492 80717057 : alt_p = true;
4493 :
4494 89334649 : if (check_only_p)
4495 32943 : return ! alt_p || best_losers != 0;
4496 :
4497 : /* If insn is commutative (it's safe to exchange a certain pair of
4498 : operands) then we need to try each alternative twice, the second
4499 : time matching those two operands as if we had exchanged them. To
4500 : do this, really exchange them in operands.
4501 :
4502 : If we have just tried the alternatives the second time, return
4503 : operands to normal and drop through. */
4504 :
4505 89315018 : if (reused_alternative_num < 0 && commutative >= 0)
4506 : {
4507 20757120 : curr_swapped = !curr_swapped;
4508 20757120 : if (curr_swapped)
4509 : {
4510 10378560 : swap_operands (commutative);
4511 10378560 : goto try_swapped;
4512 : }
4513 : else
4514 10378560 : swap_operands (commutative);
4515 : }
4516 :
4517 78936458 : if (! alt_p && ! sec_mem_p)
4518 : {
4519 : /* No alternative works with reloads?? */
4520 6 : if (INSN_CODE (curr_insn) >= 0)
4521 0 : fatal_insn ("unable to generate reloads for:", curr_insn);
4522 6 : error_for_asm (curr_insn,
4523 : "inconsistent operand constraints in an %<asm%>");
4524 6 : lra_asm_error_p = true;
4525 6 : if (! JUMP_P (curr_insn))
4526 : {
4527 : /* Avoid further trouble with this insn. Don't generate use
4528 : pattern here as we could use the insn SP offset. */
4529 6 : lra_set_insn_deleted (curr_insn);
4530 : }
4531 : else
4532 : {
4533 0 : lra_invalidate_insn_data (curr_insn);
4534 0 : ira_nullify_asm_goto (curr_insn);
4535 0 : lra_update_insn_regno_info (curr_insn);
4536 : }
4537 6 : return true;
4538 : }
4539 :
4540 : /* If the best alternative is with operands 1 and 2 swapped, swap
4541 : them. Update the operand numbers of any reloads already
4542 : pushed. */
4543 :
4544 78936452 : if (goal_alt_swapped)
4545 : {
4546 599437 : if (lra_dump_file != NULL)
4547 18 : fprintf (lra_dump_file, " Commutative operand exchange in insn %u\n",
4548 18 : INSN_UID (curr_insn));
4549 :
4550 : /* Swap the duplicates too. */
4551 599437 : swap_operands (commutative);
4552 599437 : change_p = true;
4553 : }
4554 :
4555 : /* Some targets' TARGET_SECONDARY_MEMORY_NEEDED (e.g. x86) are defined
4556 : too conservatively. So we use the secondary memory only if there
4557 : is no any alternative without reloads. */
4558 78936452 : use_sec_mem_p = false;
4559 78936452 : if (! alt_p)
4560 : use_sec_mem_p = true;
4561 78936452 : else if (sec_mem_p)
4562 : {
4563 14996 : for (i = 0; i < n_operands; i++)
4564 14823 : if (! goal_alt_win[i] && ! goal_alt_match_win[i])
4565 : break;
4566 13323 : use_sec_mem_p = i < n_operands;
4567 : }
4568 :
4569 13323 : if (use_sec_mem_p)
4570 : {
4571 13150 : int in = -1, out = -1;
4572 13150 : rtx new_reg, src, dest, rld;
4573 13150 : machine_mode sec_mode, rld_mode;
4574 :
4575 13150 : lra_assert (curr_insn_set != NULL_RTX && sec_mem_p);
4576 13150 : dest = SET_DEST (curr_insn_set);
4577 13150 : src = SET_SRC (curr_insn_set);
4578 39450 : for (i = 0; i < n_operands; i++)
4579 26300 : if (*curr_id->operand_loc[i] == dest)
4580 : out = i;
4581 13150 : else if (*curr_id->operand_loc[i] == src)
4582 13150 : in = i;
4583 13150 : for (i = 0; i < curr_static_id->n_dups; i++)
4584 0 : if (out < 0 && *curr_id->dup_loc[i] == dest)
4585 0 : out = curr_static_id->dup_num[i];
4586 0 : else if (in < 0 && *curr_id->dup_loc[i] == src)
4587 0 : in = curr_static_id->dup_num[i];
4588 13150 : lra_assert (out >= 0 && in >= 0
4589 : && curr_static_id->operand[out].type == OP_OUT
4590 : && curr_static_id->operand[in].type == OP_IN);
4591 13150 : rld = partial_subreg_p (GET_MODE (src), GET_MODE (dest)) ? src : dest;
4592 13150 : rld_mode = GET_MODE (rld);
4593 13150 : sec_mode = targetm.secondary_memory_needed_mode (rld_mode);
4594 13150 : if (rld_mode != sec_mode
4595 13150 : && (invalid_mode_reg_p (sec_mode, dest)
4596 19 : || invalid_mode_reg_p (sec_mode, src)))
4597 : sec_mode = rld_mode;
4598 13150 : new_reg = lra_create_new_reg (sec_mode, NULL_RTX, NO_REGS, NULL,
4599 : "secondary");
4600 : /* If the mode is changed, it should be wider. */
4601 13150 : lra_assert (!partial_subreg_p (sec_mode, rld_mode));
4602 13150 : if (sec_mode != rld_mode)
4603 : {
4604 : /* If the target says specifically to use another mode for
4605 : secondary memory moves we cannot reuse the original
4606 : insn. */
4607 19 : after = emit_spill_move (false, new_reg, dest);
4608 19 : lra_process_new_insns (curr_insn, NULL, after,
4609 : "Inserting the sec. move");
4610 : /* We may have non null BEFORE here (e.g. after address
4611 : processing. */
4612 19 : push_to_sequence (before);
4613 19 : before = emit_spill_move (true, new_reg, src);
4614 19 : emit_insn (before);
4615 19 : before = end_sequence ();
4616 19 : lra_process_new_insns (curr_insn, before, NULL, "Changing on");
4617 19 : lra_set_insn_deleted (curr_insn);
4618 : }
4619 13131 : else if (dest == rld)
4620 : {
4621 13131 : *curr_id->operand_loc[out] = new_reg;
4622 13131 : lra_update_dup (curr_id, out);
4623 13131 : after = emit_spill_move (false, new_reg, dest);
4624 13131 : lra_process_new_insns (curr_insn, NULL, after,
4625 : "Inserting the sec. move");
4626 : }
4627 : else
4628 : {
4629 0 : *curr_id->operand_loc[in] = new_reg;
4630 0 : lra_update_dup (curr_id, in);
4631 : /* See comments above. */
4632 0 : push_to_sequence (before);
4633 0 : before = emit_spill_move (true, new_reg, src);
4634 0 : emit_insn (before);
4635 0 : before = end_sequence ();
4636 0 : lra_process_new_insns (curr_insn, before, NULL,
4637 : "Inserting the sec. move");
4638 : }
4639 13150 : lra_update_insn_regno_info (curr_insn);
4640 13150 : return true;
4641 : }
4642 :
4643 78923302 : lra_assert (goal_alt_number >= 0);
4644 157751661 : lra_set_used_insn_alternative (curr_insn, goal_reuse_alt_p
4645 : ? goal_alt_number : LRA_UNKNOWN_ALT);
4646 :
4647 78923302 : if (lra_dump_file != NULL)
4648 : {
4649 1187 : const char *p;
4650 :
4651 1187 : fprintf (lra_dump_file, " Choosing alt %d in insn %u:",
4652 1187 : goal_alt_number, INSN_UID (curr_insn));
4653 1187 : print_curr_insn_alt (goal_alt_number);
4654 1187 : if (INSN_CODE (curr_insn) >= 0
4655 1187 : && (p = get_insn_name (INSN_CODE (curr_insn))) != NULL)
4656 1180 : fprintf (lra_dump_file, " {%s}", p);
4657 1187 : if (maybe_ne (curr_id->sp_offset, 0))
4658 : {
4659 0 : fprintf (lra_dump_file, " (sp_off=");
4660 0 : print_dec (curr_id->sp_offset, lra_dump_file);
4661 0 : fprintf (lra_dump_file, ")");
4662 : }
4663 1187 : fprintf (lra_dump_file, "\n");
4664 : }
4665 :
4666 : /* Right now, for any pair of operands I and J that are required to
4667 : match, with J < I, goal_alt_matches[I] is J. Add I to
4668 : goal_alt_matched[J]. */
4669 :
4670 254822398 : for (i = 0; i < n_operands; i++)
4671 175899096 : if ((j = goal_alt_matches[i]) >= 0)
4672 : {
4673 10417199 : for (k = 0; goal_alt_matched[j][k] >= 0; k++)
4674 : ;
4675 : /* We allow matching one output operand and several input
4676 : operands. */
4677 10417198 : lra_assert (k == 0
4678 : || (curr_static_id->operand[j].type == OP_OUT
4679 : && curr_static_id->operand[i].type == OP_IN
4680 : && (curr_static_id->operand
4681 : [goal_alt_matched[j][0]].type == OP_IN)));
4682 10417198 : goal_alt_matched[j][k] = i;
4683 10417198 : goal_alt_matched[j][k + 1] = -1;
4684 : }
4685 :
4686 254822398 : for (i = 0; i < n_operands; i++)
4687 175899096 : goal_alt_win[i] |= goal_alt_match_win[i];
4688 :
4689 : /* Any constants that aren't allowed and can't be reloaded into
4690 : registers are here changed into memory references. */
4691 254822398 : for (i = 0; i < n_operands; i++)
4692 175899096 : if (goal_alt_win[i])
4693 : {
4694 169888276 : int regno;
4695 169888276 : enum reg_class new_class;
4696 169888276 : rtx reg = *curr_id->operand_loc[i];
4697 :
4698 169888276 : if (GET_CODE (reg) == SUBREG)
4699 3334127 : reg = SUBREG_REG (reg);
4700 :
4701 169888276 : if (REG_P (reg) && (regno = REGNO (reg)) >= FIRST_PSEUDO_REGISTER)
4702 : {
4703 77032990 : bool ok_p = in_class_p (reg, goal_alt[i], &new_class, true);
4704 :
4705 77032990 : if (new_class != NO_REGS && get_reg_class (regno) != new_class)
4706 : {
4707 3466434 : lra_assert (ok_p);
4708 3466434 : lra_change_class (regno, new_class, " Change to", true);
4709 : }
4710 : }
4711 : }
4712 : else
4713 : {
4714 6010820 : const char *constraint;
4715 6010820 : char c;
4716 6010820 : rtx op = *curr_id->operand_loc[i];
4717 6010820 : rtx subreg = NULL_RTX;
4718 6010820 : machine_mode op_mode = curr_operand_mode[i], mode = op_mode;
4719 :
4720 6010820 : if (GET_CODE (op) == SUBREG)
4721 : {
4722 240469 : subreg = op;
4723 240469 : op = SUBREG_REG (op);
4724 240469 : mode = GET_MODE (op);
4725 : }
4726 :
4727 6226595 : if (CONST_POOL_OK_P (mode, op)
4728 6226595 : && ((targetm.preferred_reload_class
4729 215775 : (op, (enum reg_class) goal_alt[i]) == NO_REGS)
4730 71094 : || no_input_reloads_p))
4731 : {
4732 144681 : rtx tem = force_const_mem (mode, op);
4733 :
4734 144681 : change_p = true;
4735 144681 : if (subreg != NULL_RTX)
4736 0 : tem = gen_rtx_SUBREG (op_mode, tem, SUBREG_BYTE (subreg));
4737 :
4738 144681 : *curr_id->operand_loc[i] = tem;
4739 144681 : lra_update_dup (curr_id, i);
4740 144681 : process_address (i, false, &before, &after);
4741 :
4742 : /* If the alternative accepts constant pool refs directly
4743 : there will be no reload needed at all. */
4744 144681 : if (subreg != NULL_RTX)
4745 0 : continue;
4746 : /* Skip alternatives before the one requested. */
4747 144681 : constraint = (curr_static_id->operand_alternative
4748 144681 : [goal_alt_number * n_operands + i].constraint);
4749 144681 : for (;
4750 245267 : (c = *constraint) && c != ',' && c != '#';
4751 100586 : constraint += CONSTRAINT_LEN (c, constraint))
4752 : {
4753 198671 : enum constraint_num cn = lookup_constraint (constraint);
4754 198671 : if ((insn_extra_memory_constraint (cn)
4755 100731 : || insn_extra_special_memory_constraint (cn)
4756 : || insn_extra_relaxed_memory_constraint (cn))
4757 198816 : && satisfies_memory_constraint_p (tem, cn))
4758 : break;
4759 : }
4760 144681 : if (c == '\0' || c == ',' || c == '#')
4761 46596 : continue;
4762 :
4763 98085 : goal_alt_win[i] = true;
4764 : }
4765 : }
4766 :
4767 : n_outputs = 0;
4768 254822398 : for (i = 0; i < n_operands; i++)
4769 175899096 : if (curr_static_id->operand[i].type == OP_OUT)
4770 68516422 : outputs[n_outputs++] = i;
4771 78923302 : outputs[n_outputs] = -1;
4772 254822398 : for (i = 0; i < n_operands; i++)
4773 : {
4774 175899096 : int regno;
4775 175899096 : bool optional_p = false;
4776 175899096 : rtx old, new_reg;
4777 175899096 : rtx op = *curr_id->operand_loc[i];
4778 :
4779 175899096 : if (goal_alt_win[i])
4780 : {
4781 169986361 : if (goal_alt[i] == NO_REGS
4782 46413089 : && REG_P (op)
4783 5334047 : && (regno = REGNO (op)) >= FIRST_PSEUDO_REGISTER
4784 : /* We assigned a hard register to the pseudo in the past but now
4785 : decided to spill it for the insn. If the pseudo is used only
4786 : in this insn, it is better to spill it here as we free hard
4787 : registers for other pseudos referenced in the insn. The most
4788 : common case of this is a scratch register which will be
4789 : transformed to scratch back at the end of LRA. */
4790 172850951 : && !multiple_insn_refs_p (regno))
4791 : {
4792 11174 : if (lra_get_allocno_class (regno) != NO_REGS)
4793 5308 : lra_change_class (regno, NO_REGS, " Change to", true);
4794 5587 : reg_renumber[regno] = -1;
4795 : }
4796 : /* We can do an optional reload. If the pseudo got a hard
4797 : reg, we might improve the code through inheritance. If
4798 : it does not get a hard register we coalesce memory/memory
4799 : moves later. Ignore move insns to avoid cycling. */
4800 169986361 : if (! lra_simple_p
4801 169441683 : && lra_undo_inheritance_iter < LRA_MAX_INHERITANCE_PASSES
4802 157076473 : && goal_alt[i] != NO_REGS && REG_P (op)
4803 77971457 : && (regno = REGNO (op)) >= FIRST_PSEUDO_REGISTER
4804 65209221 : && regno < new_regno_start
4805 60510819 : && ! ira_former_scratch_p (regno)
4806 60455779 : && reg_renumber[regno] < 0
4807 : /* Check that the optional reload pseudo will be able to
4808 : hold given mode value. */
4809 3831093 : && ! (prohibited_class_reg_set_mode_p
4810 3831093 : (goal_alt[i], reg_class_contents[goal_alt[i]],
4811 3831093 : PSEUDO_REGNO_MODE (regno)))
4812 173817444 : && (curr_insn_set == NULL_RTX
4813 3823934 : || !((REG_P (SET_SRC (curr_insn_set))
4814 : || MEM_P (SET_SRC (curr_insn_set))
4815 : || GET_CODE (SET_SRC (curr_insn_set)) == SUBREG)
4816 3186826 : && (REG_P (SET_DEST (curr_insn_set))
4817 : || MEM_P (SET_DEST (curr_insn_set))
4818 : || GET_CODE (SET_DEST (curr_insn_set)) == SUBREG))))
4819 : optional_p = true;
4820 169342068 : else if (goal_alt_matched[i][0] != -1
4821 8750287 : && curr_static_id->operand[i].type == OP_OUT
4822 8749104 : && (curr_static_id->operand_alternative
4823 8749104 : [goal_alt_number * n_operands + i].earlyclobber)
4824 18467 : && REG_P (op))
4825 : {
4826 23105 : for (j = 0; goal_alt_matched[i][j] != -1; j++)
4827 : {
4828 18414 : rtx op2 = *curr_id->operand_loc[goal_alt_matched[i][j]];
4829 :
4830 18414 : if (REG_P (op2) && REGNO (op) != REGNO (op2))
4831 : break;
4832 : }
4833 18414 : if (goal_alt_matched[i][j] != -1)
4834 : {
4835 : /* Generate reloads for different output and matched
4836 : input registers. This is the easiest way to avoid
4837 : creation of non-existing register conflicts in
4838 : lra-lives.cc. */
4839 13723 : match_reload (i, goal_alt_matched[i], outputs, goal_alt[i],
4840 : &goal_alt_exclude_start_hard_regs[i], &before,
4841 : &after, true);
4842 : }
4843 170912464 : continue;
4844 18414 : }
4845 : else
4846 : {
4847 169323654 : enum reg_class rclass, common_class;
4848 :
4849 88674429 : if (REG_P (op) && goal_alt[i] != NO_REGS
4850 83340382 : && (regno = REGNO (op)) >= new_regno_start
4851 4707546 : && (rclass = get_reg_class (regno)) == ALL_REGS
4852 0 : && ((common_class = ira_reg_class_subset[rclass][goal_alt[i]])
4853 : != NO_REGS)
4854 0 : && common_class != ALL_REGS
4855 169323654 : && enough_allocatable_hard_regs_p (common_class,
4856 0 : GET_MODE (op)))
4857 : /* Refine reload pseudo class from chosen alternative
4858 : constraint. */
4859 0 : lra_change_class (regno, common_class, " Change to", true);
4860 169323654 : continue;
4861 169323654 : }
4862 : }
4863 :
4864 : /* Operands that match previous ones have already been handled. */
4865 6557028 : if (goal_alt_matches[i] >= 0)
4866 1570396 : continue;
4867 :
4868 : /* We should not have an operand with a non-offsettable address
4869 : appearing where an offsettable address will do. It also may
4870 : be a case when the address should be special in other words
4871 : not a general one (e.g. it needs no index reg). */
4872 4986632 : if (goal_alt_matched[i][0] == -1 && goal_alt_offmemok[i] && MEM_P (op))
4873 : {
4874 72 : enum reg_class rclass;
4875 72 : rtx *loc = &XEXP (op, 0);
4876 72 : enum rtx_code code = GET_CODE (*loc);
4877 :
4878 72 : push_to_sequence (before);
4879 72 : rclass = base_reg_class (GET_MODE (op), MEM_ADDR_SPACE (op),
4880 : MEM, SCRATCH, curr_insn);
4881 72 : if (GET_RTX_CLASS (code) == RTX_AUTOINC)
4882 0 : new_reg = emit_inc (rclass, *loc,
4883 : /* This value does not matter for MODIFY. */
4884 0 : GET_MODE_SIZE (GET_MODE (op)));
4885 86 : else if (get_reload_reg (OP_IN, Pmode, *loc, rclass,
4886 : NULL, false, false,
4887 : "offsetable address", &new_reg))
4888 : {
4889 72 : rtx addr = *loc;
4890 72 : enum rtx_code code = GET_CODE (addr);
4891 72 : bool align_p = false;
4892 :
4893 72 : if (code == AND && CONST_INT_P (XEXP (addr, 1)))
4894 : {
4895 : /* (and ... (const_int -X)) is used to align to X bytes. */
4896 0 : align_p = true;
4897 0 : addr = XEXP (*loc, 0);
4898 : }
4899 : else
4900 72 : addr = canonicalize_reload_addr (addr);
4901 :
4902 72 : lra_emit_move (new_reg, addr);
4903 72 : if (align_p)
4904 0 : emit_move_insn (new_reg, gen_rtx_AND (GET_MODE (new_reg), new_reg, XEXP (*loc, 1)));
4905 : }
4906 72 : before = end_sequence ();
4907 72 : *loc = new_reg;
4908 72 : lra_update_dup (curr_id, i);
4909 72 : }
4910 4986560 : else if (goal_alt_matched[i][0] == -1)
4911 : {
4912 3319650 : machine_mode mode;
4913 3319650 : rtx reg, *loc;
4914 3319650 : int hard_regno;
4915 3319650 : enum op_type type = curr_static_id->operand[i].type;
4916 :
4917 3319650 : loc = curr_id->operand_loc[i];
4918 3319650 : mode = curr_operand_mode[i];
4919 3319650 : if (GET_CODE (*loc) == SUBREG)
4920 : {
4921 76442 : reg = SUBREG_REG (*loc);
4922 76442 : poly_int64 byte = SUBREG_BYTE (*loc);
4923 76442 : if (REG_P (reg)
4924 : /* Strict_low_part requires reloading the register and not
4925 : just the subreg. Likewise for a strict subreg no wider
4926 : than a word for WORD_REGISTER_OPERATIONS targets. */
4927 76442 : && (curr_static_id->operand[i].strict_low
4928 76368 : || (!paradoxical_subreg_p (mode, GET_MODE (reg))
4929 73775 : && (hard_regno
4930 73775 : = get_try_hard_regno (REGNO (reg))) >= 0
4931 72387 : && (simplify_subreg_regno
4932 148829 : (hard_regno,
4933 72387 : GET_MODE (reg), byte, mode) < 0)
4934 0 : && (goal_alt[i] == NO_REGS
4935 0 : || (simplify_subreg_regno
4936 76442 : (ira_class_hard_regs[goal_alt[i]][0],
4937 0 : GET_MODE (reg), byte, mode) >= 0)))
4938 76368 : || (partial_subreg_p (mode, GET_MODE (reg))
4939 76368 : && known_le (GET_MODE_SIZE (GET_MODE (reg)),
4940 : UNITS_PER_WORD)
4941 : && WORD_REGISTER_OPERATIONS))
4942 : /* Avoid the situation when there are no available hard regs
4943 : for the pseudo mode but there are ones for the subreg
4944 : mode: */
4945 76516 : && !(goal_alt[i] != NO_REGS
4946 74 : && REGNO (reg) >= FIRST_PSEUDO_REGISTER
4947 74 : && (prohibited_class_reg_set_mode_p
4948 74 : (goal_alt[i], reg_class_contents[goal_alt[i]],
4949 74 : GET_MODE (reg)))
4950 : && !(prohibited_class_reg_set_mode_p
4951 0 : (goal_alt[i], reg_class_contents[goal_alt[i]],
4952 : mode))))
4953 : {
4954 : /* An OP_INOUT is required when reloading a subreg of a
4955 : mode wider than a word to ensure that data beyond the
4956 : word being reloaded is preserved. Also automatically
4957 : ensure that strict_low_part reloads are made into
4958 : OP_INOUT which should already be true from the backend
4959 : constraints. */
4960 74 : if (type == OP_OUT
4961 74 : && (curr_static_id->operand[i].strict_low
4962 0 : || read_modify_subreg_p (*loc)))
4963 : type = OP_INOUT;
4964 74 : loc = &SUBREG_REG (*loc);
4965 74 : mode = GET_MODE (*loc);
4966 : }
4967 : }
4968 3319650 : old = *loc;
4969 3319650 : if (get_reload_reg (type, mode, old, goal_alt[i],
4970 : &goal_alt_exclude_start_hard_regs[i],
4971 3319650 : loc != curr_id->operand_loc[i],
4972 3319650 : curr_static_id->operand_alternative
4973 3319650 : [goal_alt_number * n_operands + i].earlyclobber,
4974 : "", &new_reg)
4975 3319650 : && type != OP_OUT)
4976 : {
4977 2357060 : push_to_sequence (before);
4978 2357060 : lra_emit_move (new_reg, old);
4979 2357060 : before = end_sequence ();
4980 : }
4981 3319650 : *loc = new_reg;
4982 3319650 : if (type != OP_IN
4983 961550 : && find_reg_note (curr_insn, REG_UNUSED, old) == NULL_RTX
4984 : /* OLD can be an equivalent constant here. */
4985 936770 : && !CONSTANT_P (old)
4986 : /* No need to write back anything for a scratch. */
4987 936770 : && GET_CODE (old) != SCRATCH
4988 4256420 : && (!REG_P(old) || !ira_former_scratch_p (REGNO (old))))
4989 : {
4990 936770 : start_sequence ();
4991 936770 : lra_emit_move (type == OP_INOUT ? copy_rtx (old) : old, new_reg);
4992 936770 : emit_insn (after);
4993 936770 : after = end_sequence ();
4994 936770 : *loc = new_reg;
4995 : }
4996 3319650 : for (j = 0; j < goal_alt_dont_inherit_ops_num; j++)
4997 619 : if (goal_alt_dont_inherit_ops[j] == i)
4998 : {
4999 619 : lra_set_regno_unique_value (REGNO (new_reg));
5000 619 : break;
5001 : }
5002 3319650 : lra_update_dup (curr_id, i);
5003 : }
5004 1666910 : else if (curr_static_id->operand[i].type == OP_IN
5005 1666910 : && (curr_static_id->operand[goal_alt_matched[i][0]].type
5006 : == OP_OUT
5007 0 : || (curr_static_id->operand[goal_alt_matched[i][0]].type
5008 : == OP_INOUT
5009 0 : && (operands_match_p
5010 0 : (*curr_id->operand_loc[i],
5011 0 : *curr_id->operand_loc[goal_alt_matched[i][0]],
5012 : -1)))))
5013 : {
5014 : /* generate reloads for input and matched outputs. */
5015 14776 : match_inputs[0] = i;
5016 14776 : match_inputs[1] = -1;
5017 14776 : match_reload (goal_alt_matched[i][0], match_inputs, outputs,
5018 : goal_alt[i], &goal_alt_exclude_start_hard_regs[i],
5019 : &before, &after,
5020 14776 : curr_static_id->operand_alternative
5021 14776 : [goal_alt_number * n_operands + goal_alt_matched[i][0]]
5022 14776 : .earlyclobber);
5023 : }
5024 1652134 : else if ((curr_static_id->operand[i].type == OP_OUT
5025 0 : || (curr_static_id->operand[i].type == OP_INOUT
5026 0 : && (operands_match_p
5027 0 : (*curr_id->operand_loc[i],
5028 0 : *curr_id->operand_loc[goal_alt_matched[i][0]],
5029 : -1))))
5030 1652134 : && (curr_static_id->operand[goal_alt_matched[i][0]].type
5031 : == OP_IN))
5032 : /* Generate reloads for output and matched inputs. */
5033 1652134 : match_reload (i, goal_alt_matched[i], outputs, goal_alt[i],
5034 : &goal_alt_exclude_start_hard_regs[i], &before, &after,
5035 1652134 : curr_static_id->operand_alternative
5036 1652134 : [goal_alt_number * n_operands + i].earlyclobber);
5037 0 : else if (curr_static_id->operand[i].type == OP_IN
5038 0 : && (curr_static_id->operand[goal_alt_matched[i][0]].type
5039 : == OP_IN))
5040 : {
5041 : /* Generate reloads for matched inputs. */
5042 0 : match_inputs[0] = i;
5043 0 : for (j = 0; (k = goal_alt_matched[i][j]) >= 0; j++)
5044 0 : match_inputs[j + 1] = k;
5045 0 : match_inputs[j + 1] = -1;
5046 0 : match_reload (-1, match_inputs, outputs, goal_alt[i],
5047 : &goal_alt_exclude_start_hard_regs[i],
5048 : &before, &after, false);
5049 : }
5050 : else
5051 : /* We must generate code in any case when function
5052 : process_alt_operands decides that it is possible. */
5053 0 : gcc_unreachable ();
5054 :
5055 4986632 : if (optional_p)
5056 : {
5057 644293 : rtx reg = op;
5058 :
5059 644293 : lra_assert (REG_P (reg));
5060 644293 : regno = REGNO (reg);
5061 644293 : op = *curr_id->operand_loc[i]; /* Substitution. */
5062 644293 : if (GET_CODE (op) == SUBREG)
5063 0 : op = SUBREG_REG (op);
5064 644293 : gcc_assert (REG_P (op) && (int) REGNO (op) >= new_regno_start);
5065 644293 : bitmap_set_bit (&lra_optional_reload_pseudos, REGNO (op));
5066 644293 : lra_reg_info[REGNO (op)].restore_rtx = reg;
5067 644293 : if (lra_dump_file != NULL)
5068 3 : fprintf (lra_dump_file,
5069 : " Making reload reg %d for reg %d optional\n",
5070 : REGNO (op), regno);
5071 : }
5072 : }
5073 74662125 : if (before != NULL_RTX || after != NULL_RTX
5074 152782676 : || max_regno_before != max_reg_num ())
5075 5089119 : change_p = true;
5076 78923302 : if (change_p)
5077 : {
5078 6036885 : lra_update_operator_dups (curr_id);
5079 : /* Something changes -- process the insn. */
5080 6036885 : lra_update_insn_regno_info (curr_insn);
5081 6036885 : if (asm_noperands (PATTERN (curr_insn)) >= 0
5082 6036885 : && ++curr_id->asm_reloads_num >= FIRST_PSEUDO_REGISTER)
5083 : /* Most probably there are no enough registers to satisfy asm insn: */
5084 : {
5085 11 : lra_asm_insn_error (curr_insn);
5086 11 : return change_p;
5087 : }
5088 : }
5089 78923291 : if (goal_alt_out_sp_reload_p)
5090 : {
5091 : /* We have an output stack pointer reload -- update sp offset: */
5092 0 : rtx set;
5093 0 : bool done_p = false;
5094 0 : poly_int64 sp_offset = curr_id->sp_offset;
5095 0 : for (rtx_insn *insn = after; insn != NULL_RTX; insn = NEXT_INSN (insn))
5096 0 : if ((set = single_set (insn)) != NULL_RTX
5097 0 : && SET_DEST (set) == stack_pointer_rtx)
5098 : {
5099 0 : lra_assert (!done_p);
5100 0 : done_p = true;
5101 0 : curr_id->sp_offset = 0;
5102 0 : lra_insn_recog_data_t id = lra_get_insn_recog_data (insn);
5103 0 : id->sp_offset = sp_offset;
5104 0 : if (lra_dump_file != NULL)
5105 0 : fprintf (lra_dump_file,
5106 : " Moving sp offset from insn %u to %u\n",
5107 0 : INSN_UID (curr_insn), INSN_UID (insn));
5108 : }
5109 0 : lra_assert (done_p);
5110 : }
5111 78923291 : int const_regno = -1;
5112 78923291 : rtx set;
5113 78923291 : rtx_insn *prev, *const_insn = NULL;
5114 4261172 : if (before != NULL_RTX && (prev = PREV_INSN (curr_insn)) != NULL_RTX
5115 83184463 : && (set = single_set (prev)) != NULL_RTX && CONSTANT_P (SET_SRC (set)))
5116 : {
5117 324510 : rtx reg = SET_DEST (set);
5118 324510 : if (GET_CODE (reg) == SUBREG)
5119 9733 : reg = SUBREG_REG (reg);
5120 : /* Consider only reload insns as we don't want to change the order
5121 : created by previous optimizations. */
5122 235304 : if (REG_P (reg) && (int) REGNO (reg) >= lra_new_regno_start
5123 325317 : && bitmap_bit_p (&lra_reg_info[REGNO (reg)].insn_bitmap,
5124 807 : INSN_UID (curr_insn)))
5125 : {
5126 323 : const_regno = REGNO (reg);
5127 323 : const_insn = prev;
5128 : }
5129 : }
5130 78923291 : if (asm_noperands (PATTERN (curr_insn)) >= 0)
5131 : {
5132 : /* Asm can have a lot of operands. To guarantee their assignment,
5133 : postpone processing the reload insns until the reload pseudos are
5134 : assigned. */
5135 46170 : postpone_insns (before);
5136 46170 : postpone_insns (after);
5137 : }
5138 78923291 : lra_process_new_insns (curr_insn, before, after,
5139 : "Inserting insn reload", true);
5140 78923291 : if (const_regno >= 0) {
5141 646 : bool move_p = true;
5142 646 : for (rtx_insn *insn = before; insn != curr_insn; insn = NEXT_INSN (insn))
5143 323 : if (bitmap_bit_p (&lra_reg_info[const_regno].insn_bitmap,
5144 323 : INSN_UID (insn)))
5145 : {
5146 : move_p = false;
5147 : break;
5148 : }
5149 323 : if (move_p)
5150 : {
5151 323 : reorder_insns_nobb (const_insn, const_insn, PREV_INSN (curr_insn));
5152 323 : if (lra_dump_file != NULL)
5153 : {
5154 0 : dump_insn_slim (lra_dump_file, const_insn);
5155 0 : fprintf (lra_dump_file,
5156 : " to decrease reg pressure, it is moved before:\n");
5157 0 : dump_insn_slim (lra_dump_file, curr_insn);
5158 : }
5159 : }
5160 : }
5161 : return change_p;
5162 : }
5163 :
5164 : /* Return true if INSN satisfies all constraints. In other words, no
5165 : reload insns are needed. */
5166 : bool
5167 3730 : lra_constrain_insn (rtx_insn *insn)
5168 : {
5169 3730 : int saved_new_regno_start = new_regno_start;
5170 3730 : int saved_new_insn_uid_start = new_insn_uid_start;
5171 3730 : bool change_p;
5172 :
5173 3730 : curr_insn = insn;
5174 3730 : curr_id = lra_get_insn_recog_data (curr_insn);
5175 3730 : curr_static_id = curr_id->insn_static_data;
5176 3730 : new_insn_uid_start = get_max_uid ();
5177 3730 : new_regno_start = max_reg_num ();
5178 3730 : change_p = curr_insn_transform (true);
5179 3730 : new_regno_start = saved_new_regno_start;
5180 3730 : new_insn_uid_start = saved_new_insn_uid_start;
5181 3730 : return ! change_p;
5182 : }
5183 :
5184 : /* Return true if X is in LIST. */
5185 : static bool
5186 1303268 : in_list_p (rtx x, rtx list)
5187 : {
5188 2228810 : for (; list != NULL_RTX; list = XEXP (list, 1))
5189 1223463 : if (XEXP (list, 0) == x)
5190 : return true;
5191 : return false;
5192 : }
5193 :
5194 : /* Return true if X contains an allocatable hard register (if
5195 : HARD_REG_P) or a (spilled if SPILLED_P) pseudo. */
5196 : static bool
5197 7259290 : contains_reg_p (rtx x, bool hard_reg_p, bool spilled_p)
5198 : {
5199 7259290 : int i, j;
5200 7259290 : const char *fmt;
5201 7259290 : enum rtx_code code;
5202 :
5203 7259290 : code = GET_CODE (x);
5204 7259290 : if (REG_P (x))
5205 : {
5206 1467547 : int regno = REGNO (x);
5207 1467547 : HARD_REG_SET alloc_regs;
5208 :
5209 1467547 : if (hard_reg_p)
5210 : {
5211 461266 : if (regno >= FIRST_PSEUDO_REGISTER)
5212 135273 : regno = lra_get_regno_hard_regno (regno);
5213 461266 : if (regno < 0)
5214 : return false;
5215 461266 : alloc_regs = ~lra_no_alloc_regs;
5216 461266 : return overlaps_hard_reg_set_p (alloc_regs, GET_MODE (x), regno);
5217 : }
5218 : else
5219 : {
5220 1006281 : if (regno < FIRST_PSEUDO_REGISTER)
5221 : return false;
5222 323085 : if (! spilled_p)
5223 : return true;
5224 171427 : return lra_get_regno_hard_regno (regno) < 0;
5225 : }
5226 : }
5227 5791743 : fmt = GET_RTX_FORMAT (code);
5228 14316471 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
5229 : {
5230 9085939 : if (fmt[i] == 'e')
5231 : {
5232 3998685 : if (contains_reg_p (XEXP (x, i), hard_reg_p, spilled_p))
5233 : return true;
5234 : }
5235 5087254 : else if (fmt[i] == 'E')
5236 : {
5237 1256777 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
5238 1150056 : if (contains_reg_p (XVECEXP (x, i, j), hard_reg_p, spilled_p))
5239 : return true;
5240 : }
5241 : }
5242 : return false;
5243 : }
5244 :
5245 : /* Process all regs in location *LOC and change them on equivalent
5246 : substitution. Return true if any change was done. */
5247 : static bool
5248 3348 : loc_equivalence_change_p (rtx *loc)
5249 : {
5250 3348 : rtx subst, reg, x = *loc;
5251 3348 : bool result = false;
5252 3348 : enum rtx_code code = GET_CODE (x);
5253 3348 : const char *fmt;
5254 3348 : int i, j;
5255 :
5256 3348 : if (code == SUBREG)
5257 : {
5258 20 : reg = SUBREG_REG (x);
5259 20 : if ((subst = get_equiv_with_elimination (reg, curr_insn)) != reg
5260 20 : && GET_MODE (subst) == VOIDmode)
5261 : {
5262 : /* We cannot reload debug location. Simplify subreg here
5263 : while we know the inner mode. */
5264 0 : *loc = simplify_gen_subreg (GET_MODE (x), subst,
5265 0 : GET_MODE (reg), SUBREG_BYTE (x));
5266 0 : return true;
5267 : }
5268 : }
5269 3348 : if (code == REG && (subst = get_equiv_with_elimination (x, curr_insn)) != x)
5270 : {
5271 8 : *loc = subst;
5272 8 : return true;
5273 : }
5274 :
5275 : /* Scan all the operand sub-expressions. */
5276 3340 : fmt = GET_RTX_FORMAT (code);
5277 8168 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
5278 : {
5279 4828 : if (fmt[i] == 'e')
5280 2577 : result = loc_equivalence_change_p (&XEXP (x, i)) || result;
5281 2251 : else if (fmt[i] == 'E')
5282 270 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
5283 200 : result
5284 210 : = loc_equivalence_change_p (&XVECEXP (x, i, j)) || result;
5285 : }
5286 : return result;
5287 : }
5288 :
5289 : /* Similar to loc_equivalence_change_p, but for use as
5290 : simplify_replace_fn_rtx callback. DATA is insn for which the
5291 : elimination is done. If it null we don't do the elimination. */
5292 : static rtx
5293 42389859 : loc_equivalence_callback (rtx loc, const_rtx, void *data)
5294 : {
5295 42389859 : if (!REG_P (loc))
5296 : return NULL_RTX;
5297 :
5298 10949045 : rtx subst = (data == NULL
5299 10949045 : ? get_equiv (loc) : get_equiv_with_elimination (loc, (rtx_insn *) data));
5300 10949045 : if (subst != loc)
5301 : return subst;
5302 :
5303 : return NULL_RTX;
5304 : }
5305 :
5306 : /* Maximum number of generated reload insns per an insn. It is for
5307 : preventing this pass cycling in a bug case. */
5308 : #define MAX_RELOAD_INSNS_NUMBER LRA_MAX_INSN_RELOADS
5309 :
5310 : /* The current iteration number of this LRA pass. */
5311 : int lra_constraint_iter;
5312 :
5313 : /* True if we should during assignment sub-pass check assignment
5314 : correctness for all pseudos and spill some of them to correct
5315 : conflicts. It can be necessary when we substitute equiv which
5316 : needs checking register allocation correctness because the
5317 : equivalent value contains allocatable hard registers, or when we
5318 : restore multi-register pseudo, or when we change the insn code and
5319 : its operand became INOUT operand when it was IN one before. */
5320 : bool check_and_force_assignment_correctness_p;
5321 :
5322 : /* Return true if REGNO is referenced in more than one block. */
5323 : static bool
5324 145216 : multi_block_pseudo_p (int regno)
5325 : {
5326 145216 : basic_block bb = NULL;
5327 145216 : unsigned int uid;
5328 145216 : bitmap_iterator bi;
5329 :
5330 145216 : if (regno < FIRST_PSEUDO_REGISTER)
5331 : return false;
5332 :
5333 445224 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi)
5334 304810 : if (bb == NULL)
5335 145216 : bb = BLOCK_FOR_INSN (lra_insn_recog_data[uid]->insn);
5336 159594 : else if (BLOCK_FOR_INSN (lra_insn_recog_data[uid]->insn) != bb)
5337 : return true;
5338 : return false;
5339 : }
5340 :
5341 : /* Return true if LIST contains a deleted insn. */
5342 : static bool
5343 705492 : contains_deleted_insn_p (rtx_insn_list *list)
5344 : {
5345 1346238 : for (; list != NULL_RTX; list = list->next ())
5346 640746 : if (NOTE_P (list->insn ())
5347 640746 : && NOTE_KIND (list->insn ()) == NOTE_INSN_DELETED)
5348 : return true;
5349 : return false;
5350 : }
5351 :
5352 : /* Return true if X contains a pseudo dying in INSN. */
5353 : static bool
5354 2193742 : dead_pseudo_p (rtx x, rtx_insn *insn)
5355 : {
5356 2193742 : int i, j;
5357 2193742 : const char *fmt;
5358 2193742 : enum rtx_code code;
5359 :
5360 2193742 : if (REG_P (x))
5361 490103 : return (insn != NULL_RTX
5362 490103 : && find_regno_note (insn, REG_DEAD, REGNO (x)) != NULL_RTX);
5363 1703639 : code = GET_CODE (x);
5364 1703639 : fmt = GET_RTX_FORMAT (code);
5365 4354145 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
5366 : {
5367 2655867 : if (fmt[i] == 'e')
5368 : {
5369 1301322 : if (dead_pseudo_p (XEXP (x, i), insn))
5370 : return true;
5371 : }
5372 1354545 : else if (fmt[i] == 'E')
5373 : {
5374 277062 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
5375 253209 : if (dead_pseudo_p (XVECEXP (x, i, j), insn))
5376 : return true;
5377 : }
5378 : }
5379 : return false;
5380 : }
5381 :
5382 : /* Return true if INSN contains a dying pseudo in INSN right hand
5383 : side. */
5384 : static bool
5385 639211 : insn_rhs_dead_pseudo_p (rtx_insn *insn)
5386 : {
5387 639211 : rtx set = single_set (insn);
5388 :
5389 639211 : gcc_assert (set != NULL);
5390 639211 : return dead_pseudo_p (SET_SRC (set), insn);
5391 : }
5392 :
5393 : /* Return true if any init insn of REGNO contains a dying pseudo in
5394 : insn right hand side. */
5395 : static bool
5396 703957 : init_insn_rhs_dead_pseudo_p (int regno)
5397 : {
5398 703957 : rtx_insn_list *insns = ira_reg_equiv[regno].init_insns;
5399 :
5400 703957 : if (insns == NULL)
5401 : return false;
5402 1271779 : for (; insns != NULL_RTX; insns = insns->next ())
5403 639211 : if (insn_rhs_dead_pseudo_p (insns->insn ()))
5404 : return true;
5405 : return false;
5406 : }
5407 :
5408 : /* Return TRUE if REGNO has a reverse equivalence. The equivalence is
5409 : reverse only if we have one init insn with given REGNO as a
5410 : source. */
5411 : static bool
5412 705492 : reverse_equiv_p (int regno)
5413 : {
5414 705492 : rtx_insn_list *insns = ira_reg_equiv[regno].init_insns;
5415 705492 : rtx set;
5416 :
5417 705492 : if (insns == NULL)
5418 : return false;
5419 640746 : if (! INSN_P (insns->insn ())
5420 1281492 : || insns->next () != NULL)
5421 : return false;
5422 640746 : if ((set = single_set (insns->insn ())) == NULL_RTX)
5423 : return false;
5424 640746 : return REG_P (SET_SRC (set)) && (int) REGNO (SET_SRC (set)) == regno;
5425 : }
5426 :
5427 : /* Return TRUE if REGNO was reloaded in an equivalence init insn. We
5428 : call this function only for non-reverse equivalence. */
5429 : static bool
5430 697314 : contains_reloaded_insn_p (int regno)
5431 : {
5432 697314 : rtx set;
5433 697314 : rtx_insn_list *list = ira_reg_equiv[regno].init_insns;
5434 :
5435 1329882 : for (; list != NULL; list = list->next ())
5436 632568 : if ((set = single_set (list->insn ())) == NULL_RTX
5437 632568 : || ! REG_P (SET_DEST (set))
5438 1265136 : || (int) REGNO (SET_DEST (set)) != regno)
5439 : return true;
5440 : return false;
5441 : }
5442 :
5443 : /* Try combine secondary memory reload insn FROM for insn TO into TO insn.
5444 : FROM should be a load insn (usually a secondary memory reload insn). Return
5445 : TRUE in case of success. */
5446 : static bool
5447 7182901 : combine_reload_insn (rtx_insn *from, rtx_insn *to)
5448 : {
5449 7182901 : bool ok_p;
5450 7182901 : rtx_insn *saved_insn;
5451 7182901 : rtx set, from_reg, to_reg, op;
5452 7182901 : enum reg_class to_class, from_class;
5453 7182901 : int n, nop;
5454 7182901 : signed char changed_nops[MAX_RECOG_OPERANDS + 1];
5455 :
5456 : /* Check conditions for second memory reload and original insn: */
5457 7182901 : if ((targetm.secondary_memory_needed
5458 : == hook_bool_mode_reg_class_t_reg_class_t_false)
5459 7182901 : || NEXT_INSN (from) != to
5460 4269265 : || !NONDEBUG_INSN_P (to)
5461 11452164 : || CALL_P (to))
5462 : return false;
5463 :
5464 4263861 : lra_insn_recog_data_t id = lra_get_insn_recog_data (to);
5465 4263861 : struct lra_static_insn_data *static_id = id->insn_static_data;
5466 :
5467 4263861 : if (id->used_insn_alternative == LRA_UNKNOWN_ALT
5468 4263861 : || (set = single_set (from)) == NULL_RTX)
5469 31951 : return false;
5470 4231910 : from_reg = SET_DEST (set);
5471 4231910 : to_reg = SET_SRC (set);
5472 : /* Ignore optional reloads: */
5473 4153113 : if (! REG_P (from_reg) || ! REG_P (to_reg)
5474 7074031 : || bitmap_bit_p (&lra_optional_reload_pseudos, REGNO (from_reg)))
5475 1935747 : return false;
5476 2296163 : to_class = lra_get_allocno_class (REGNO (to_reg));
5477 2296163 : from_class = lra_get_allocno_class (REGNO (from_reg));
5478 : /* Check that reload insn is a load: */
5479 2296163 : if (to_class != NO_REGS || from_class == NO_REGS)
5480 : return false;
5481 49255 : for (n = nop = 0; nop < static_id->n_operands; nop++)
5482 : {
5483 35448 : if (static_id->operand[nop].type != OP_IN)
5484 12780 : continue;
5485 22668 : op = *id->operand_loc[nop];
5486 22668 : if (!REG_P (op) || REGNO (op) != REGNO (from_reg))
5487 9046 : continue;
5488 13622 : *id->operand_loc[nop] = to_reg;
5489 13622 : changed_nops[n++] = nop;
5490 : }
5491 13807 : changed_nops[n] = -1;
5492 13807 : lra_update_dups (id, changed_nops);
5493 13807 : lra_update_insn_regno_info (to);
5494 13807 : ok_p = recog_memoized (to) >= 0;
5495 13807 : if (ok_p)
5496 : {
5497 : /* Check that combined insn does not need any reloads: */
5498 13788 : saved_insn = curr_insn;
5499 13788 : curr_insn = to;
5500 13788 : curr_id = lra_get_insn_recog_data (curr_insn);
5501 13788 : curr_static_id = curr_id->insn_static_data;
5502 13788 : for (bool swapped_p = false;;)
5503 : {
5504 16099 : ok_p = !curr_insn_transform (true);
5505 16099 : if (ok_p || curr_static_id->commutative < 0)
5506 : break;
5507 4622 : swap_operands (curr_static_id->commutative);
5508 4622 : if (lra_dump_file != NULL)
5509 : {
5510 0 : fprintf (lra_dump_file,
5511 : " Swapping %scombined insn operands:\n",
5512 : swapped_p ? "back " : "");
5513 0 : dump_insn_slim (lra_dump_file, to);
5514 : }
5515 4622 : if (swapped_p)
5516 : break;
5517 : swapped_p = true;
5518 : }
5519 13788 : curr_insn = saved_insn;
5520 13788 : curr_id = lra_get_insn_recog_data (curr_insn);
5521 13788 : curr_static_id = curr_id->insn_static_data;
5522 : }
5523 13807 : if (ok_p)
5524 : {
5525 3573 : id->used_insn_alternative = -1;
5526 3573 : lra_push_insn_and_update_insn_regno_info (to);
5527 3573 : if (lra_dump_file != NULL)
5528 : {
5529 0 : fprintf (lra_dump_file, " Use combined insn:\n");
5530 0 : dump_insn_slim (lra_dump_file, to);
5531 : }
5532 3573 : return true;
5533 : }
5534 10234 : if (lra_dump_file != NULL)
5535 : {
5536 0 : fprintf (lra_dump_file, " Failed combined insn:\n");
5537 0 : dump_insn_slim (lra_dump_file, to);
5538 : }
5539 20739 : for (int i = 0; i < n; i++)
5540 : {
5541 10505 : nop = changed_nops[i];
5542 10505 : *id->operand_loc[nop] = from_reg;
5543 : }
5544 10234 : lra_update_dups (id, changed_nops);
5545 10234 : lra_update_insn_regno_info (to);
5546 10234 : if (lra_dump_file != NULL)
5547 : {
5548 0 : fprintf (lra_dump_file, " Restoring insn after failed combining:\n");
5549 0 : dump_insn_slim (lra_dump_file, to);
5550 : }
5551 : return false;
5552 : }
5553 :
5554 : /* Entry function of LRA constraint pass. Return true if the
5555 : constraint pass did change the code. */
5556 : bool
5557 3211326 : lra_constraints (bool first_p)
5558 : {
5559 3211326 : bool changed_p;
5560 3211326 : int i, hard_regno, new_insns_num;
5561 3211326 : unsigned int min_len, new_min_len, uid;
5562 3211326 : rtx set, x, reg, nosubreg_dest;
5563 3211326 : rtx_insn *original_insn;
5564 3211326 : basic_block last_bb;
5565 3211326 : bitmap_iterator bi;
5566 :
5567 3211326 : lra_constraint_iter++;
5568 3211326 : if (lra_dump_file != NULL)
5569 194 : fprintf (lra_dump_file, "\n********** Local #%d: **********\n\n",
5570 : lra_constraint_iter);
5571 3211326 : changed_p = false;
5572 3211326 : if (pic_offset_table_rtx
5573 3211326 : && REGNO (pic_offset_table_rtx) >= FIRST_PSEUDO_REGISTER)
5574 104476 : check_and_force_assignment_correctness_p = true;
5575 3106850 : else if (first_p)
5576 : /* On the first iteration we should check IRA assignment
5577 : correctness. In rare cases, the assignments can be wrong as
5578 : early clobbers operands are ignored in IRA or usages of
5579 : paradoxical sub-registers are not taken into account by
5580 : IRA. */
5581 1433976 : check_and_force_assignment_correctness_p = true;
5582 3211326 : new_insn_uid_start = get_max_uid ();
5583 3211326 : new_regno_start = first_p ? lra_constraint_new_regno_start : max_reg_num ();
5584 : /* Mark used hard regs for target stack size calulations. */
5585 204968749 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
5586 201757423 : if (lra_reg_info[i].nrefs != 0
5587 297982241 : && (hard_regno = lra_get_regno_hard_regno (i)) >= 0)
5588 : {
5589 92348038 : int j, nregs;
5590 :
5591 92348038 : nregs = hard_regno_nregs (hard_regno, lra_reg_info[i].biggest_mode);
5592 187729664 : for (j = 0; j < nregs; j++)
5593 95381626 : df_set_regs_ever_live (hard_regno + j, true);
5594 : }
5595 : /* Do elimination before the equivalence processing as we can spill
5596 : some pseudos during elimination. */
5597 3211326 : lra_eliminate (false, first_p);
5598 3211326 : auto_bitmap equiv_insn_bitmap (®_obstack);
5599 :
5600 : /* Register elimination can create new pseudos via the addptr pattern,
5601 : so make sure the equivalency tables are resized appropriately. */
5602 3211326 : ira_expand_reg_equiv ();
5603 204968749 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
5604 201757423 : if (lra_reg_info[i].nrefs != 0)
5605 : {
5606 96224818 : ira_reg_equiv[i].profitable_p = true;
5607 96224818 : reg = regno_reg_rtx[i];
5608 96224818 : if (lra_get_regno_hard_regno (i) < 0 && (x = get_equiv (reg)) != reg)
5609 : {
5610 716736 : bool pseudo_p = contains_reg_p (x, false, false);
5611 :
5612 : /* After RTL transformation, we cannot guarantee that
5613 : pseudo in the substitution was not reloaded which might
5614 : make equivalence invalid. For example, in reverse
5615 : equiv of p0
5616 :
5617 : p0 <- ...
5618 : ...
5619 : equiv_mem <- p0
5620 :
5621 : the memory address register was reloaded before the 2nd
5622 : insn. */
5623 716736 : if ((! first_p && pseudo_p)
5624 : /* We don't use DF for compilation speed sake. So it
5625 : is problematic to update live info when we use an
5626 : equivalence containing pseudos in more than one
5627 : BB. */
5628 710294 : || (pseudo_p && multi_block_pseudo_p (i))
5629 : /* If an init insn was deleted for some reason, cancel
5630 : the equiv. We could update the equiv insns after
5631 : transformations including an equiv insn deletion
5632 : but it is not worthy as such cases are extremely
5633 : rare. */
5634 705492 : || contains_deleted_insn_p (ira_reg_equiv[i].init_insns)
5635 : /* If it is not a reverse equivalence, we check that a
5636 : pseudo in rhs of the init insn is not dying in the
5637 : insn. Otherwise, the live info at the beginning of
5638 : the corresponding BB might be wrong after we
5639 : removed the insn. When the equiv can be a
5640 : constant, the right hand side of the init insn can
5641 : be a pseudo. */
5642 705492 : || (! reverse_equiv_p (i)
5643 703957 : && (init_insn_rhs_dead_pseudo_p (i)
5644 : /* If we reloaded the pseudo in an equivalence
5645 : init insn, we cannot remove the equiv init
5646 : insns and the init insns might write into
5647 : const memory in this case. */
5648 697314 : || contains_reloaded_insn_p (i)))
5649 : /* Prevent access beyond equivalent memory for
5650 : paradoxical subregs. */
5651 698849 : || (MEM_P (x)
5652 1112430 : && maybe_gt (GET_MODE_SIZE (lra_reg_info[i].biggest_mode),
5653 : GET_MODE_SIZE (GET_MODE (x))))
5654 1414865 : || (pic_offset_table_rtx
5655 53174 : && ((CONST_POOL_OK_P (PSEUDO_REGNO_MODE (i), x)
5656 8058 : && (targetm.preferred_reload_class
5657 4029 : (x, lra_get_allocno_class (i)) == NO_REGS))
5658 51475 : || contains_symbol_ref_p (x))))
5659 21075 : ira_reg_equiv[i].defined_p
5660 21075 : = ira_reg_equiv[i].caller_save_p = false;
5661 716736 : if (contains_reg_p (x, false, true))
5662 9291 : ira_reg_equiv[i].profitable_p = false;
5663 716736 : if (get_equiv (reg) != reg)
5664 690679 : bitmap_ior_into (equiv_insn_bitmap, &lra_reg_info[i].insn_bitmap);
5665 : }
5666 : }
5667 204968749 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
5668 201757423 : update_equiv (i);
5669 : /* We should add all insns containing pseudos which should be
5670 : substituted by their equivalences. */
5671 5519203 : EXECUTE_IF_SET_IN_BITMAP (equiv_insn_bitmap, 0, uid, bi)
5672 2307877 : lra_push_insn_by_uid (uid);
5673 3211326 : min_len = lra_insn_stack_length ();
5674 3211326 : new_insns_num = 0;
5675 3211326 : last_bb = NULL;
5676 3211326 : changed_p = false;
5677 3211326 : original_insn = NULL;
5678 164907694 : while ((new_min_len = lra_insn_stack_length ()) != 0)
5679 : {
5680 158485042 : curr_insn = lra_pop_insn ();
5681 158485042 : --new_min_len;
5682 158485042 : curr_bb = BLOCK_FOR_INSN (curr_insn);
5683 158485042 : if (curr_bb != last_bb)
5684 : {
5685 20433892 : last_bb = curr_bb;
5686 20433892 : bb_reload_num = lra_curr_reload_num;
5687 : }
5688 158485042 : if (min_len > new_min_len)
5689 : {
5690 : min_len = new_min_len;
5691 : new_insns_num = 0;
5692 : original_insn = curr_insn;
5693 : }
5694 7182901 : else if (combine_reload_insn (curr_insn, original_insn))
5695 : {
5696 3573 : continue;
5697 : }
5698 7179328 : if (new_insns_num > MAX_RELOAD_INSNS_NUMBER)
5699 0 : internal_error
5700 0 : ("maximum number of generated reload insns per insn achieved (%d)",
5701 : MAX_RELOAD_INSNS_NUMBER);
5702 158481469 : new_insns_num++;
5703 158481469 : if (DEBUG_INSN_P (curr_insn))
5704 : {
5705 : /* We need to check equivalence in debug insn and change
5706 : pseudo to the equivalent value if necessary. */
5707 51314358 : curr_id = lra_get_insn_recog_data (curr_insn);
5708 51314358 : if (bitmap_bit_p (equiv_insn_bitmap, INSN_UID (curr_insn)))
5709 : {
5710 29579 : rtx old = *curr_id->operand_loc[0];
5711 29579 : *curr_id->operand_loc[0]
5712 29579 : = simplify_replace_fn_rtx (old, NULL_RTX,
5713 : loc_equivalence_callback, curr_insn);
5714 29579 : if (old != *curr_id->operand_loc[0])
5715 : {
5716 : /* If we substitute pseudo by shared equivalence, we can fail
5717 : to update LRA reg info and this can result in many
5718 : unexpected consequences. So keep rtl unshared: */
5719 29579 : *curr_id->operand_loc[0]
5720 29579 : = copy_rtx (*curr_id->operand_loc[0]);
5721 29579 : lra_update_insn_regno_info (curr_insn);
5722 29579 : changed_p = true;
5723 : }
5724 : }
5725 : }
5726 107167111 : else if (INSN_P (curr_insn))
5727 : {
5728 106109507 : if ((set = single_set (curr_insn)) != NULL_RTX)
5729 : {
5730 100776727 : nosubreg_dest = SET_DEST (set);
5731 : /* The equivalence pseudo could be set up as SUBREG in a
5732 : case when it is a call restore insn in a mode
5733 : different from the pseudo mode. */
5734 100776727 : if (GET_CODE (nosubreg_dest) == SUBREG)
5735 1190004 : nosubreg_dest = SUBREG_REG (nosubreg_dest);
5736 101453804 : if ((REG_P (nosubreg_dest)
5737 74391567 : && (x = get_equiv (nosubreg_dest)) != nosubreg_dest
5738 : /* Remove insns which set up a pseudo whose value
5739 : cannot be changed. Such insns might be not in
5740 : init_insns because we don't update equiv data
5741 : during insn transformations.
5742 :
5743 : As an example, let suppose that a pseudo got
5744 : hard register and on the 1st pass was not
5745 : changed to equivalent constant. We generate an
5746 : additional insn setting up the pseudo because of
5747 : secondary memory movement. Then the pseudo is
5748 : spilled and we use the equiv constant. In this
5749 : case we should remove the additional insn and
5750 : this insn is not init_insns list. */
5751 695002 : && (! MEM_P (x) || MEM_READONLY_P (x)
5752 : /* Check that this is actually an insn setting
5753 : up the equivalence. */
5754 315846 : || in_list_p (curr_insn,
5755 315846 : ira_reg_equiv
5756 315846 : [REGNO (nosubreg_dest)].init_insns)))
5757 174492685 : || (((x = get_equiv (SET_SRC (set))) != SET_SRC (set))
5758 1974844 : && in_list_p (curr_insn,
5759 987422 : ira_reg_equiv
5760 987422 : [REGNO (SET_SRC (set))].init_insns)
5761 : /* This is a reverse equivalence to memory (see ira.cc)
5762 : in store insn. We can reload all the destination and
5763 : have an output reload which is a store to memory. If
5764 : we just remove the insn, we will have the output
5765 : reload storing an undefined value to the memory.
5766 : Check that we did not reload the memory to prevent a
5767 : wrong code generation. We could implement using the
5768 : equivalence still in such case but doing this is not
5769 : worth the efforts as such case is very rare. */
5770 1468 : && MEM_P (nosubreg_dest)))
5771 : {
5772 : /* This is equiv init insn of pseudo which did not get a
5773 : hard register -- remove the insn. */
5774 677077 : if (lra_dump_file != NULL)
5775 : {
5776 9 : fprintf (lra_dump_file,
5777 : " Removing equiv init insn %i (freq=%d)\n",
5778 3 : INSN_UID (curr_insn),
5779 6 : REG_FREQ_FROM_BB (BLOCK_FOR_INSN (curr_insn)));
5780 3 : dump_insn_slim (lra_dump_file, curr_insn);
5781 : }
5782 677077 : if (contains_reg_p (x, true, false))
5783 135273 : check_and_force_assignment_correctness_p = true;
5784 677077 : lra_set_insn_deleted (curr_insn);
5785 677077 : continue;
5786 : }
5787 : }
5788 105432430 : curr_id = lra_get_insn_recog_data (curr_insn);
5789 105432430 : curr_static_id = curr_id->insn_static_data;
5790 105432430 : init_curr_insn_input_reloads ();
5791 105432430 : init_curr_operand_mode ();
5792 105432430 : if (curr_insn_transform (false))
5793 : changed_p = true;
5794 : /* Check non-transformed insns too for equiv change as USE
5795 : or CLOBBER don't need reloads but can contain pseudos
5796 : being changed on their equivalences. */
5797 99375683 : else if (bitmap_bit_p (equiv_insn_bitmap, INSN_UID (curr_insn))
5798 99375683 : && loc_equivalence_change_p (&PATTERN (curr_insn)))
5799 : {
5800 8 : lra_update_insn_regno_info (curr_insn);
5801 8 : lra_push_insn_by_uid (INSN_UID (curr_insn));
5802 8 : changed_p = true;
5803 : }
5804 : }
5805 : }
5806 :
5807 : /* If we used a new hard regno, changed_p should be true because the
5808 : hard reg is assigned to a new pseudo. */
5809 3211326 : if (flag_checking && !changed_p)
5810 : {
5811 131981148 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
5812 129379751 : if (lra_reg_info[i].nrefs != 0
5813 189099523 : && (hard_regno = lra_get_regno_hard_regno (i)) >= 0)
5814 : {
5815 58282118 : int j, nregs = hard_regno_nregs (hard_regno,
5816 58282118 : PSEUDO_REGNO_MODE (i));
5817 :
5818 118578897 : for (j = 0; j < nregs; j++)
5819 60296779 : lra_assert (df_regs_ever_live_p (hard_regno + j));
5820 : }
5821 : }
5822 2601437 : if (changed_p)
5823 609892 : lra_dump_insns_if_possible ("changed func after local");
5824 3211326 : return changed_p;
5825 3211326 : }
5826 :
5827 : static void initiate_invariants (void);
5828 : static void finish_invariants (void);
5829 :
5830 : /* Initiate the LRA constraint pass. It is done once per
5831 : function. */
5832 : void
5833 1474414 : lra_constraints_init (void)
5834 : {
5835 1474414 : initiate_invariants ();
5836 1474414 : }
5837 :
5838 : /* Finalize the LRA constraint pass. It is done once per
5839 : function. */
5840 : void
5841 1474414 : lra_constraints_finish (void)
5842 : {
5843 1474414 : finish_invariants ();
5844 1474414 : }
5845 :
5846 :
5847 :
5848 : /* Structure describes invariants for ineheritance. */
5849 : struct lra_invariant
5850 : {
5851 : /* The order number of the invariant. */
5852 : int num;
5853 : /* The invariant RTX. */
5854 : rtx invariant_rtx;
5855 : /* The origin insn of the invariant. */
5856 : rtx_insn *insn;
5857 : };
5858 :
5859 : typedef lra_invariant invariant_t;
5860 : typedef invariant_t *invariant_ptr_t;
5861 : typedef const invariant_t *const_invariant_ptr_t;
5862 :
5863 : /* Pointer to the inheritance invariants. */
5864 : static vec<invariant_ptr_t> invariants;
5865 :
5866 : /* Allocation pool for the invariants. */
5867 : static object_allocator<lra_invariant> *invariants_pool;
5868 :
5869 : /* Hash table for the invariants. */
5870 : static htab_t invariant_table;
5871 :
5872 : /* Hash function for INVARIANT. */
5873 : static hashval_t
5874 175168 : invariant_hash (const void *invariant)
5875 : {
5876 175168 : rtx inv = ((const_invariant_ptr_t) invariant)->invariant_rtx;
5877 175168 : return lra_rtx_hash (inv);
5878 : }
5879 :
5880 : /* Equal function for invariants INVARIANT1 and INVARIANT2. */
5881 : static int
5882 56717 : invariant_eq_p (const void *invariant1, const void *invariant2)
5883 : {
5884 56717 : rtx inv1 = ((const_invariant_ptr_t) invariant1)->invariant_rtx;
5885 56717 : rtx inv2 = ((const_invariant_ptr_t) invariant2)->invariant_rtx;
5886 :
5887 56717 : return rtx_equal_p (inv1, inv2);
5888 : }
5889 :
5890 : /* Insert INVARIANT_RTX into the table if it is not there yet. Return
5891 : invariant which is in the table. */
5892 : static invariant_ptr_t
5893 174976 : insert_invariant (rtx invariant_rtx)
5894 : {
5895 174976 : void **entry_ptr;
5896 174976 : invariant_t invariant;
5897 174976 : invariant_ptr_t invariant_ptr;
5898 :
5899 174976 : invariant.invariant_rtx = invariant_rtx;
5900 174976 : entry_ptr = htab_find_slot (invariant_table, &invariant, INSERT);
5901 174976 : if (*entry_ptr == NULL)
5902 : {
5903 152145 : invariant_ptr = invariants_pool->allocate ();
5904 152145 : invariant_ptr->invariant_rtx = invariant_rtx;
5905 152145 : invariant_ptr->insn = NULL;
5906 152145 : invariants.safe_push (invariant_ptr);
5907 152145 : *entry_ptr = (void *) invariant_ptr;
5908 : }
5909 174976 : return (invariant_ptr_t) *entry_ptr;
5910 : }
5911 :
5912 : /* Initiate the invariant table. */
5913 : static void
5914 1474414 : initiate_invariants (void)
5915 : {
5916 1474414 : invariants.create (100);
5917 1474414 : invariants_pool
5918 1474414 : = new object_allocator<lra_invariant> ("Inheritance invariants");
5919 1474414 : invariant_table = htab_create (100, invariant_hash, invariant_eq_p, NULL);
5920 1474414 : }
5921 :
5922 : /* Finish the invariant table. */
5923 : static void
5924 1474414 : finish_invariants (void)
5925 : {
5926 1474414 : htab_delete (invariant_table);
5927 2948828 : delete invariants_pool;
5928 1474414 : invariants.release ();
5929 1474414 : }
5930 :
5931 : /* Make the invariant table empty. */
5932 : static void
5933 12582050 : clear_invariants (void)
5934 : {
5935 12582050 : htab_empty (invariant_table);
5936 12582050 : invariants_pool->release ();
5937 12582050 : invariants.truncate (0);
5938 12582050 : }
5939 :
5940 :
5941 :
5942 : /* This page contains code to do inheritance/split
5943 : transformations. */
5944 :
5945 : /* Number of reloads passed so far in current EBB. */
5946 : static int reloads_num;
5947 :
5948 : /* Number of calls passed so far in current EBB. */
5949 : static int calls_num;
5950 :
5951 : /* Index ID is the CALLS_NUM associated the last call we saw with
5952 : ABI identifier ID. */
5953 : static int last_call_for_abi[NUM_ABI_IDS];
5954 :
5955 : /* Which registers have been fully or partially clobbered by a call
5956 : since they were last used. */
5957 : static HARD_REG_SET full_and_partial_call_clobbers;
5958 :
5959 : /* Current reload pseudo check for validity of elements in
5960 : USAGE_INSNS. */
5961 : static int curr_usage_insns_check;
5962 :
5963 : /* Info about last usage of registers in EBB to do inheritance/split
5964 : transformation. Inheritance transformation is done from a spilled
5965 : pseudo and split transformations from a hard register or a pseudo
5966 : assigned to a hard register. */
5967 : struct usage_insns
5968 : {
5969 : /* If the value is equal to CURR_USAGE_INSNS_CHECK, then the member
5970 : value INSNS is valid. The insns is chain of optional debug insns
5971 : and a finishing non-debug insn using the corresponding reg. The
5972 : value is also used to mark the registers which are set up in the
5973 : current insn. The negated insn uid is used for this. */
5974 : int check;
5975 : /* Value of global reloads_num at the last insn in INSNS. */
5976 : int reloads_num;
5977 : /* Value of global reloads_nums at the last insn in INSNS. */
5978 : int calls_num;
5979 : /* It can be true only for splitting. And it means that the restore
5980 : insn should be put after insn given by the following member. */
5981 : bool after_p;
5982 : /* Next insns in the current EBB which use the original reg and the
5983 : original reg value is not changed between the current insn and
5984 : the next insns. In order words, e.g. for inheritance, if we need
5985 : to use the original reg value again in the next insns we can try
5986 : to use the value in a hard register from a reload insn of the
5987 : current insn. */
5988 : rtx insns;
5989 : };
5990 :
5991 : /* Map: regno -> corresponding pseudo usage insns. */
5992 : static struct usage_insns *usage_insns;
5993 :
5994 : static void
5995 245067938 : setup_next_usage_insn (int regno, rtx insn, int reloads_num, bool after_p)
5996 : {
5997 245067938 : usage_insns[regno].check = curr_usage_insns_check;
5998 245067938 : usage_insns[regno].insns = insn;
5999 245067938 : usage_insns[regno].reloads_num = reloads_num;
6000 245067938 : usage_insns[regno].calls_num = calls_num;
6001 245067938 : usage_insns[regno].after_p = after_p;
6002 245067938 : if (regno >= FIRST_PSEUDO_REGISTER && reg_renumber[regno] >= 0)
6003 110643694 : remove_from_hard_reg_set (&full_and_partial_call_clobbers,
6004 110643694 : PSEUDO_REGNO_MODE (regno),
6005 : reg_renumber[regno]);
6006 245067938 : }
6007 :
6008 : /* The function is used to form list REGNO usages which consists of
6009 : optional debug insns finished by a non-debug insn using REGNO.
6010 : RELOADS_NUM is current number of reload insns processed so far. */
6011 : static void
6012 138705756 : add_next_usage_insn (int regno, rtx_insn *insn, int reloads_num)
6013 : {
6014 138705756 : rtx next_usage_insns;
6015 :
6016 138705756 : if (usage_insns[regno].check == curr_usage_insns_check
6017 72523255 : && (next_usage_insns = usage_insns[regno].insns) != NULL_RTX
6018 211229011 : && DEBUG_INSN_P (insn))
6019 : {
6020 : /* Check that we did not add the debug insn yet. */
6021 13650443 : if (next_usage_insns != insn
6022 13650443 : && (GET_CODE (next_usage_insns) != INSN_LIST
6023 6135585 : || XEXP (next_usage_insns, 0) != insn))
6024 13650429 : usage_insns[regno].insns = gen_rtx_INSN_LIST (VOIDmode, insn,
6025 : next_usage_insns);
6026 : }
6027 125055313 : else if (NONDEBUG_INSN_P (insn))
6028 124558824 : setup_next_usage_insn (regno, insn, reloads_num, false);
6029 : else
6030 496489 : usage_insns[regno].check = 0;
6031 138705756 : }
6032 :
6033 : /* Return first non-debug insn in list USAGE_INSNS. */
6034 : static rtx_insn *
6035 1160979 : skip_usage_debug_insns (rtx usage_insns)
6036 : {
6037 1160979 : rtx insn;
6038 :
6039 : /* Skip debug insns. */
6040 1160979 : for (insn = usage_insns;
6041 1439188 : insn != NULL_RTX && GET_CODE (insn) == INSN_LIST;
6042 278209 : insn = XEXP (insn, 1))
6043 : ;
6044 1160979 : return safe_as_a <rtx_insn *> (insn);
6045 : }
6046 :
6047 : /* Return true if we need secondary memory moves for insn in
6048 : USAGE_INSNS after inserting inherited pseudo of class INHER_CL
6049 : into the insn. */
6050 : static bool
6051 1160986 : check_secondary_memory_needed_p (enum reg_class inher_cl ATTRIBUTE_UNUSED,
6052 : rtx usage_insns ATTRIBUTE_UNUSED)
6053 : {
6054 1160986 : rtx_insn *insn;
6055 1160986 : rtx set, dest;
6056 1160986 : enum reg_class cl;
6057 :
6058 1160986 : if (inher_cl == ALL_REGS
6059 1160986 : || (insn = skip_usage_debug_insns (usage_insns)) == NULL_RTX)
6060 : return false;
6061 1160979 : lra_assert (INSN_P (insn));
6062 1160979 : if ((set = single_set (insn)) == NULL_RTX || ! REG_P (SET_DEST (set)))
6063 : return false;
6064 1122954 : dest = SET_DEST (set);
6065 1122954 : if (! REG_P (dest))
6066 : return false;
6067 1122954 : lra_assert (inher_cl != NO_REGS);
6068 1122954 : cl = get_reg_class (REGNO (dest));
6069 1122954 : return (cl != NO_REGS && cl != ALL_REGS
6070 1122954 : && targetm.secondary_memory_needed (GET_MODE (dest), inher_cl, cl));
6071 : }
6072 :
6073 : /* Registers involved in inheritance/split in the current EBB
6074 : (inheritance/split pseudos and original registers). */
6075 : static bitmap_head check_only_regs;
6076 :
6077 : /* Reload pseudos cannot be involded in invariant inheritance in the
6078 : current EBB. */
6079 : static bitmap_head invalid_invariant_regs;
6080 :
6081 : /* Do inheritance transformations for insn INSN, which defines (if
6082 : DEF_P) or uses ORIGINAL_REGNO. NEXT_USAGE_INSNS specifies which
6083 : instruction in the EBB next uses ORIGINAL_REGNO; it has the same
6084 : form as the "insns" field of usage_insns. Return true if we
6085 : succeed in such transformation.
6086 :
6087 : The transformations look like:
6088 :
6089 : p <- ... i <- ...
6090 : ... p <- i (new insn)
6091 : ... =>
6092 : <- ... p ... <- ... i ...
6093 : or
6094 : ... i <- p (new insn)
6095 : <- ... p ... <- ... i ...
6096 : ... =>
6097 : <- ... p ... <- ... i ...
6098 : where p is a spilled original pseudo and i is a new inheritance pseudo.
6099 :
6100 :
6101 : The inheritance pseudo has the smallest class of two classes CL and
6102 : class of ORIGINAL REGNO. */
6103 : static bool
6104 1255918 : inherit_reload_reg (bool def_p, int original_regno,
6105 : enum reg_class cl, rtx_insn *insn, rtx next_usage_insns)
6106 : {
6107 1255918 : if (optimize_function_for_size_p (cfun))
6108 : return false;
6109 :
6110 1224986 : enum reg_class rclass = lra_get_allocno_class (original_regno);
6111 1224986 : rtx original_reg = regno_reg_rtx[original_regno];
6112 1224986 : rtx new_reg, usage_insn;
6113 1224986 : rtx_insn *new_insns;
6114 :
6115 1224986 : lra_assert (! usage_insns[original_regno].after_p);
6116 1224986 : if (lra_dump_file != NULL)
6117 2 : fprintf (lra_dump_file,
6118 : " <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
6119 1224986 : if (! ira_reg_classes_intersect_p[cl][rclass])
6120 : {
6121 64000 : if (lra_dump_file != NULL)
6122 : {
6123 0 : fprintf (lra_dump_file,
6124 : " Rejecting inheritance for %d "
6125 : "because of disjoint classes %s and %s\n",
6126 : original_regno, reg_class_names[cl],
6127 : reg_class_names[rclass]);
6128 0 : fprintf (lra_dump_file,
6129 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6130 : }
6131 64000 : return false;
6132 : }
6133 1160986 : if ((ira_class_subset_p[cl][rclass] && cl != rclass)
6134 : /* We don't use a subset of two classes because it can be
6135 : NO_REGS. This transformation is still profitable in most
6136 : cases even if the classes are not intersected as register
6137 : move is probably cheaper than a memory load. */
6138 435584 : || ira_class_hard_regs_num[cl] < ira_class_hard_regs_num[rclass])
6139 : {
6140 725402 : if (lra_dump_file != NULL)
6141 2 : fprintf (lra_dump_file, " Use smallest class of %s and %s\n",
6142 : reg_class_names[cl], reg_class_names[rclass]);
6143 :
6144 : rclass = cl;
6145 : }
6146 1160986 : if (check_secondary_memory_needed_p (rclass, next_usage_insns))
6147 : {
6148 : /* Reject inheritance resulting in secondary memory moves.
6149 : Otherwise, there is a danger in LRA cycling. Also such
6150 : transformation will be unprofitable. */
6151 12887 : if (lra_dump_file != NULL)
6152 : {
6153 0 : rtx_insn *insn = skip_usage_debug_insns (next_usage_insns);
6154 0 : rtx set = single_set (insn);
6155 :
6156 0 : lra_assert (set != NULL_RTX);
6157 :
6158 0 : rtx dest = SET_DEST (set);
6159 :
6160 0 : lra_assert (REG_P (dest));
6161 0 : fprintf (lra_dump_file,
6162 : " Rejecting inheritance for insn %d(%s)<-%d(%s) "
6163 : "as secondary mem is needed\n",
6164 0 : REGNO (dest), reg_class_names[get_reg_class (REGNO (dest))],
6165 0 : original_regno, reg_class_names[rclass]);
6166 0 : fprintf (lra_dump_file,
6167 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6168 : }
6169 12887 : return false;
6170 : }
6171 1148099 : if (ira_reg_class_min_nregs[rclass][GET_MODE (original_reg)]
6172 1148099 : != ira_reg_class_max_nregs[rclass][GET_MODE (original_reg)])
6173 : {
6174 24 : if (lra_dump_file != NULL)
6175 : {
6176 0 : fprintf (lra_dump_file,
6177 : " Rejecting inheritance for %d "
6178 : "because of requiring non-uniform class %s\n",
6179 : original_regno, reg_class_names[rclass]);
6180 0 : fprintf (lra_dump_file,
6181 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6182 : }
6183 24 : return false;
6184 : }
6185 1148075 : new_reg = lra_create_new_reg (GET_MODE (original_reg), original_reg,
6186 : rclass, NULL, "inheritance");
6187 1148075 : start_sequence ();
6188 1148075 : if (def_p)
6189 540754 : lra_emit_move (original_reg, new_reg);
6190 : else
6191 607321 : lra_emit_move (new_reg, original_reg);
6192 1148075 : new_insns = end_sequence ();
6193 1148075 : if (NEXT_INSN (new_insns) != NULL_RTX)
6194 : {
6195 0 : if (lra_dump_file != NULL)
6196 : {
6197 0 : fprintf (lra_dump_file,
6198 : " Rejecting inheritance %d->%d "
6199 : "as it results in 2 or more insns:\n",
6200 : original_regno, REGNO (new_reg));
6201 0 : dump_rtl_slim (lra_dump_file, new_insns, NULL, -1, 0);
6202 0 : fprintf (lra_dump_file,
6203 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6204 : }
6205 0 : return false;
6206 : }
6207 1148075 : lra_substitute_pseudo_within_insn (insn, original_regno, new_reg, false);
6208 1148075 : lra_update_insn_regno_info (insn);
6209 1148075 : if (! def_p)
6210 : /* We now have a new usage insn for original regno. */
6211 607321 : setup_next_usage_insn (original_regno, new_insns, reloads_num, false);
6212 1148075 : if (lra_dump_file != NULL)
6213 2 : fprintf (lra_dump_file, " Original reg change %d->%d (bb%d):\n",
6214 2 : original_regno, REGNO (new_reg), BLOCK_FOR_INSN (insn)->index);
6215 1148075 : lra_reg_info[REGNO (new_reg)].restore_rtx = regno_reg_rtx[original_regno];
6216 1148075 : bitmap_set_bit (&check_only_regs, REGNO (new_reg));
6217 1148075 : bitmap_set_bit (&check_only_regs, original_regno);
6218 1148075 : bitmap_set_bit (&lra_inheritance_pseudos, REGNO (new_reg));
6219 1148075 : if (def_p)
6220 540754 : lra_process_new_insns (insn, NULL, new_insns,
6221 : "Add original<-inheritance");
6222 : else
6223 607321 : lra_process_new_insns (insn, new_insns, NULL,
6224 : "Add inheritance<-original");
6225 2572875 : while (next_usage_insns != NULL_RTX)
6226 : {
6227 1424800 : if (GET_CODE (next_usage_insns) != INSN_LIST)
6228 : {
6229 1148075 : usage_insn = next_usage_insns;
6230 1148075 : lra_assert (NONDEBUG_INSN_P (usage_insn));
6231 : next_usage_insns = NULL;
6232 : }
6233 : else
6234 : {
6235 276725 : usage_insn = XEXP (next_usage_insns, 0);
6236 276725 : lra_assert (DEBUG_INSN_P (usage_insn));
6237 276725 : next_usage_insns = XEXP (next_usage_insns, 1);
6238 : }
6239 1424800 : lra_substitute_pseudo (&usage_insn, original_regno, new_reg, false,
6240 1424800 : DEBUG_INSN_P (usage_insn));
6241 1424800 : lra_update_insn_regno_info (as_a <rtx_insn *> (usage_insn));
6242 1424800 : if (lra_dump_file != NULL)
6243 : {
6244 2 : basic_block bb = BLOCK_FOR_INSN (usage_insn);
6245 2 : fprintf (lra_dump_file,
6246 : " Inheritance reuse change %d->%d (bb%d):\n",
6247 : original_regno, REGNO (new_reg),
6248 : bb ? bb->index : -1);
6249 2 : dump_insn_slim (lra_dump_file, as_a <rtx_insn *> (usage_insn));
6250 : }
6251 : }
6252 1148075 : if (lra_dump_file != NULL)
6253 2 : fprintf (lra_dump_file,
6254 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6255 : return true;
6256 : }
6257 :
6258 : /* Return true if we need a caller save/restore for pseudo REGNO which
6259 : was assigned to a hard register. */
6260 : static inline bool
6261 113289606 : need_for_call_save_p (int regno)
6262 : {
6263 113289606 : lra_assert (regno >= FIRST_PSEUDO_REGISTER && reg_renumber[regno] >= 0);
6264 113289606 : if (usage_insns[regno].calls_num < calls_num)
6265 : {
6266 : unsigned int abis = 0;
6267 116420096 : for (unsigned int i = 0; i < NUM_ABI_IDS; ++i)
6268 107464704 : if (last_call_for_abi[i] > usage_insns[regno].calls_num)
6269 8955392 : abis |= 1 << i;
6270 8955392 : gcc_assert (abis);
6271 8955392 : if (call_clobbered_in_region_p (abis, full_and_partial_call_clobbers,
6272 8955392 : PSEUDO_REGNO_MODE (regno),
6273 : reg_renumber[regno]))
6274 : return true;
6275 : }
6276 : return false;
6277 : }
6278 :
6279 : /* Global registers occurring in the current EBB. */
6280 : static bitmap_head ebb_global_regs;
6281 :
6282 : /* Return true if we need a split for hard register REGNO or pseudo
6283 : REGNO which was assigned to a hard register.
6284 : POTENTIAL_RELOAD_HARD_REGS contains hard registers which might be
6285 : used for reloads since the EBB end. It is an approximation of the
6286 : used hard registers in the split range. The exact value would
6287 : require expensive calculations. If we were aggressive with
6288 : splitting because of the approximation, the split pseudo will save
6289 : the same hard register assignment and will be removed in the undo
6290 : pass. We still need the approximation because too aggressive
6291 : splitting would result in too inaccurate cost calculation in the
6292 : assignment pass because of too many generated moves which will be
6293 : probably removed in the undo pass. */
6294 : static inline bool
6295 240541676 : need_for_split_p (HARD_REG_SET potential_reload_hard_regs, int regno)
6296 : {
6297 240541676 : int hard_regno = regno < FIRST_PSEUDO_REGISTER ? regno : reg_renumber[regno];
6298 :
6299 240541676 : lra_assert (hard_regno >= 0);
6300 240541676 : return ((TEST_HARD_REG_BIT (potential_reload_hard_regs, hard_regno)
6301 : /* Don't split eliminable hard registers, otherwise we can
6302 : split hard registers like hard frame pointer, which
6303 : lives on BB start/end according to DF-infrastructure,
6304 : when there is a pseudo assigned to the register and
6305 : living in the same BB. */
6306 661462 : && (regno >= FIRST_PSEUDO_REGISTER
6307 44511 : || ! TEST_HARD_REG_BIT (eliminable_regset, hard_regno))
6308 631282 : && ! TEST_HARD_REG_BIT (lra_no_alloc_regs, hard_regno)
6309 : /* Don't split call clobbered hard regs living through
6310 : calls, otherwise we might have a check problem in the
6311 : assign sub-pass as in the most cases (exception is a
6312 : situation when check_and_force_assignment_correctness_p value is
6313 : true) the assign pass assumes that all pseudos living
6314 : through calls are assigned to call saved hard regs. */
6315 617982 : && (regno >= FIRST_PSEUDO_REGISTER
6316 1031 : || !TEST_HARD_REG_BIT (full_and_partial_call_clobbers, regno))
6317 : /* We need at least 2 reloads to make pseudo splitting
6318 : profitable. We should provide hard regno splitting in
6319 : any case to solve 1st insn scheduling problem when
6320 : moving hard register definition up might result in
6321 : impossibility to find hard register for reload pseudo of
6322 : small register class. */
6323 1235908 : && (usage_insns[regno].reloads_num
6324 1234905 : + (regno < FIRST_PSEUDO_REGISTER ? 0 : 3) < reloads_num)
6325 2638 : && (regno < FIRST_PSEUDO_REGISTER
6326 : /* For short living pseudos, spilling + inheritance can
6327 : be considered a substitution for splitting.
6328 : Therefore we do not splitting for local pseudos. It
6329 : decreases also aggressiveness of splitting. The
6330 : minimal number of references is chosen taking into
6331 : account that for 2 references splitting has no sense
6332 : as we can just spill the pseudo. */
6333 : || (regno >= FIRST_PSEUDO_REGISTER
6334 2591 : && lra_reg_info[regno].nrefs > 3
6335 2248 : && bitmap_bit_p (&ebb_global_regs, regno))))
6336 241201898 : || (regno >= FIRST_PSEUDO_REGISTER && need_for_call_save_p (regno)));
6337 : }
6338 :
6339 : /* Return class for the split pseudo created from original pseudo with
6340 : ALLOCNO_CLASS and MODE which got a hard register HARD_REGNO. We
6341 : choose subclass of ALLOCNO_CLASS which contains HARD_REGNO and
6342 : results in no secondary memory movements. */
6343 : static enum reg_class
6344 1399 : choose_split_class (enum reg_class allocno_class,
6345 : int hard_regno ATTRIBUTE_UNUSED,
6346 : machine_mode mode ATTRIBUTE_UNUSED)
6347 : {
6348 1399 : int i;
6349 1399 : enum reg_class cl, best_cl = NO_REGS;
6350 1399 : enum reg_class hard_reg_class ATTRIBUTE_UNUSED
6351 : = REGNO_REG_CLASS (hard_regno);
6352 :
6353 1399 : if (! targetm.secondary_memory_needed (mode, allocno_class, allocno_class)
6354 1399 : && TEST_HARD_REG_BIT (reg_class_contents[allocno_class], hard_regno))
6355 : return allocno_class;
6356 0 : for (i = 0;
6357 0 : (cl = reg_class_subclasses[allocno_class][i]) != LIM_REG_CLASSES;
6358 : i++)
6359 0 : if (! targetm.secondary_memory_needed (mode, cl, hard_reg_class)
6360 0 : && ! targetm.secondary_memory_needed (mode, hard_reg_class, cl)
6361 0 : && TEST_HARD_REG_BIT (reg_class_contents[cl], hard_regno)
6362 0 : && (best_cl == NO_REGS
6363 0 : || ira_class_hard_regs_num[best_cl] < ira_class_hard_regs_num[cl]))
6364 : best_cl = cl;
6365 : return best_cl;
6366 : }
6367 :
6368 : /* Copy any equivalence information from ORIGINAL_REGNO to NEW_REGNO. It only
6369 : makes sense to call this function if NEW_REGNO is always equal to
6370 : ORIGINAL_REGNO. Set up defined_p flag when caller_save_p flag is set up and
6371 : CALL_SAVE_P is true. */
6372 :
6373 : static void
6374 649214 : lra_copy_reg_equiv (unsigned int new_regno, unsigned int original_regno,
6375 : bool call_save_p)
6376 : {
6377 649214 : if (!ira_reg_equiv[original_regno].defined_p
6378 585515 : && !(call_save_p && ira_reg_equiv[original_regno].caller_save_p))
6379 : return;
6380 :
6381 63874 : ira_expand_reg_equiv ();
6382 63874 : ira_reg_equiv[new_regno].defined_p = true;
6383 63874 : if (ira_reg_equiv[original_regno].memory)
6384 29936 : ira_reg_equiv[new_regno].memory
6385 29936 : = copy_rtx (ira_reg_equiv[original_regno].memory);
6386 63874 : if (ira_reg_equiv[original_regno].constant)
6387 27694 : ira_reg_equiv[new_regno].constant
6388 27694 : = copy_rtx (ira_reg_equiv[original_regno].constant);
6389 63874 : if (ira_reg_equiv[original_regno].invariant)
6390 6244 : ira_reg_equiv[new_regno].invariant
6391 6244 : = copy_rtx (ira_reg_equiv[original_regno].invariant);
6392 : }
6393 :
6394 : /* Do split transformations for insn INSN, which defines or uses
6395 : ORIGINAL_REGNO. NEXT_USAGE_INSNS specifies which instruction in
6396 : the EBB next uses ORIGINAL_REGNO; it has the same form as the
6397 : "insns" field of usage_insns. If TO is not NULL, we don't use
6398 : usage_insns, we put restore insns after TO insn. It is a case when
6399 : we call it from lra_split_hard_reg_for, outside the inheritance
6400 : pass.
6401 :
6402 : The transformations look like:
6403 :
6404 : p <- ... p <- ...
6405 : ... s <- p (new insn -- save)
6406 : ... =>
6407 : ... p <- s (new insn -- restore)
6408 : <- ... p ... <- ... p ...
6409 : or
6410 : <- ... p ... <- ... p ...
6411 : ... s <- p (new insn -- save)
6412 : ... =>
6413 : ... p <- s (new insn -- restore)
6414 : <- ... p ... <- ... p ...
6415 :
6416 : where p is an original pseudo got a hard register or a hard
6417 : register and s is a new split pseudo. The save is put before INSN
6418 : if BEFORE_P is true. Return true if we succeed in such
6419 : transformation. */
6420 : static bool
6421 650862 : split_reg (bool before_p, int original_regno, rtx_insn *insn,
6422 : rtx next_usage_insns, rtx_insn *to)
6423 : {
6424 650862 : enum reg_class rclass;
6425 650862 : rtx original_reg;
6426 650862 : int hard_regno, nregs;
6427 650862 : rtx new_reg, usage_insn;
6428 650862 : rtx_insn *restore, *save;
6429 650862 : bool after_p;
6430 650862 : bool call_save_p;
6431 650862 : machine_mode mode;
6432 :
6433 650862 : if (original_regno < FIRST_PSEUDO_REGISTER)
6434 : {
6435 206 : rclass = ira_allocno_class_translate[REGNO_REG_CLASS (original_regno)];
6436 206 : hard_regno = original_regno;
6437 206 : call_save_p = false;
6438 206 : nregs = 1;
6439 206 : mode = lra_reg_info[hard_regno].biggest_mode;
6440 206 : machine_mode reg_rtx_mode = GET_MODE (regno_reg_rtx[hard_regno]);
6441 : /* A reg can have a biggest_mode of VOIDmode if it was only ever seen as
6442 : part of a multi-word register. In that case, just use the reg_rtx
6443 : mode. Do the same also if the biggest mode was larger than a register
6444 : or we can not compare the modes. Otherwise, limit the size to that of
6445 : the biggest access in the function or to the natural mode at least. */
6446 206 : if (mode == VOIDmode
6447 206 : || !ordered_p (GET_MODE_PRECISION (mode),
6448 206 : GET_MODE_PRECISION (reg_rtx_mode))
6449 206 : || paradoxical_subreg_p (mode, reg_rtx_mode)
6450 411 : || maybe_gt (GET_MODE_PRECISION (reg_rtx_mode), GET_MODE_PRECISION (mode)))
6451 : {
6452 650862 : original_reg = regno_reg_rtx[hard_regno];
6453 650862 : mode = reg_rtx_mode;
6454 : }
6455 : else
6456 189 : original_reg = gen_rtx_REG (mode, hard_regno);
6457 : }
6458 : else
6459 : {
6460 650656 : mode = PSEUDO_REGNO_MODE (original_regno);
6461 650656 : hard_regno = reg_renumber[original_regno];
6462 650656 : nregs = hard_regno_nregs (hard_regno, mode);
6463 650656 : rclass = lra_get_allocno_class (original_regno);
6464 650656 : original_reg = regno_reg_rtx[original_regno];
6465 650656 : call_save_p = need_for_call_save_p (original_regno);
6466 : }
6467 650862 : lra_assert (hard_regno >= 0);
6468 650862 : if (lra_dump_file != NULL)
6469 0 : fprintf (lra_dump_file,
6470 : " ((((((((((((((((((((((((((((((((((((((((((((((((\n");
6471 :
6472 650862 : if (call_save_p)
6473 : {
6474 649463 : mode = HARD_REGNO_CALLER_SAVE_MODE (hard_regno,
6475 : hard_regno_nregs (hard_regno, mode),
6476 : mode);
6477 649463 : new_reg = lra_create_new_reg (mode, NULL_RTX, NO_REGS, NULL, "save");
6478 : }
6479 : else
6480 : {
6481 1399 : rclass = choose_split_class (rclass, hard_regno, mode);
6482 1399 : if (rclass == NO_REGS)
6483 : {
6484 0 : if (lra_dump_file != NULL)
6485 : {
6486 0 : fprintf (lra_dump_file,
6487 : " Rejecting split of %d(%s): "
6488 : "no good reg class for %d(%s)\n",
6489 : original_regno,
6490 0 : reg_class_names[lra_get_allocno_class (original_regno)],
6491 : hard_regno,
6492 0 : reg_class_names[REGNO_REG_CLASS (hard_regno)]);
6493 0 : fprintf
6494 0 : (lra_dump_file,
6495 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
6496 : }
6497 0 : return false;
6498 : }
6499 : /* Split_if_necessary can split hard registers used as part of a
6500 : multi-register mode but splits each register individually. The
6501 : mode used for each independent register may not be supported
6502 : so reject the split. Splitting the wider mode should theoretically
6503 : be possible but is not implemented. */
6504 1399 : if (!targetm.hard_regno_mode_ok (hard_regno, mode))
6505 : {
6506 0 : if (lra_dump_file != NULL)
6507 : {
6508 0 : fprintf (lra_dump_file,
6509 : " Rejecting split of %d(%s): unsuitable mode %s\n",
6510 : original_regno,
6511 0 : reg_class_names[lra_get_allocno_class (original_regno)],
6512 0 : GET_MODE_NAME (mode));
6513 0 : fprintf
6514 0 : (lra_dump_file,
6515 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
6516 : }
6517 0 : return false;
6518 : }
6519 1399 : new_reg = lra_create_new_reg (mode, original_reg, rclass, NULL, "split");
6520 1399 : reg_renumber[REGNO (new_reg)] = hard_regno;
6521 : }
6522 650862 : int new_regno = REGNO (new_reg);
6523 650862 : save = emit_spill_move (true, new_reg, original_reg);
6524 650862 : if (NEXT_INSN (save) != NULL_RTX && !call_save_p)
6525 : {
6526 0 : if (lra_dump_file != NULL)
6527 : {
6528 0 : fprintf
6529 0 : (lra_dump_file,
6530 : " Rejecting split %d->%d resulting in > 2 save insns:\n",
6531 : original_regno, new_regno);
6532 0 : dump_rtl_slim (lra_dump_file, save, NULL, -1, 0);
6533 0 : fprintf (lra_dump_file,
6534 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
6535 : }
6536 0 : return false;
6537 : }
6538 650862 : restore = emit_spill_move (false, new_reg, original_reg);
6539 650862 : if (NEXT_INSN (restore) != NULL_RTX && !call_save_p)
6540 : {
6541 0 : if (lra_dump_file != NULL)
6542 : {
6543 0 : fprintf (lra_dump_file,
6544 : " Rejecting split %d->%d "
6545 : "resulting in > 2 restore insns:\n",
6546 : original_regno, new_regno);
6547 0 : dump_rtl_slim (lra_dump_file, restore, NULL, -1, 0);
6548 0 : fprintf (lra_dump_file,
6549 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
6550 : }
6551 0 : return false;
6552 : }
6553 : /* Transfer equivalence information to the spill register, so that
6554 : if we fail to allocate the spill register, we have the option of
6555 : rematerializing the original value instead of spilling to the stack. */
6556 650862 : if (!HARD_REGISTER_NUM_P (original_regno)
6557 650656 : && mode == PSEUDO_REGNO_MODE (original_regno))
6558 649214 : lra_copy_reg_equiv (new_regno, original_regno, call_save_p);
6559 650862 : lra_reg_info[new_regno].restore_rtx = regno_reg_rtx[original_regno];
6560 650862 : bitmap_set_bit (&lra_split_regs, new_regno);
6561 650862 : if (to != NULL)
6562 : {
6563 159 : lra_assert (next_usage_insns == NULL);
6564 159 : usage_insn = to;
6565 159 : after_p = true;
6566 : }
6567 : else
6568 : {
6569 : /* We need check_only_regs only inside the inheritance pass. */
6570 650703 : bitmap_set_bit (&check_only_regs, new_regno);
6571 650703 : bitmap_set_bit (&check_only_regs, original_regno);
6572 650703 : after_p = usage_insns[original_regno].after_p;
6573 757533 : for (;;)
6574 : {
6575 757533 : if (GET_CODE (next_usage_insns) != INSN_LIST)
6576 : {
6577 650703 : usage_insn = next_usage_insns;
6578 650703 : break;
6579 : }
6580 106830 : usage_insn = XEXP (next_usage_insns, 0);
6581 106830 : lra_assert (DEBUG_INSN_P (usage_insn));
6582 106830 : next_usage_insns = XEXP (next_usage_insns, 1);
6583 106830 : lra_substitute_pseudo (&usage_insn, original_regno, new_reg, false,
6584 : true);
6585 106830 : lra_update_insn_regno_info (as_a <rtx_insn *> (usage_insn));
6586 106830 : if (lra_dump_file != NULL)
6587 : {
6588 0 : fprintf (lra_dump_file, " Split reuse change %d->%d:\n",
6589 : original_regno, new_regno);
6590 0 : dump_insn_slim (lra_dump_file, as_a <rtx_insn *> (usage_insn));
6591 : }
6592 : }
6593 : }
6594 650862 : lra_assert (NOTE_P (usage_insn) || NONDEBUG_INSN_P (usage_insn));
6595 650862 : lra_assert (usage_insn != insn || (after_p && before_p));
6596 1101980 : lra_process_new_insns (as_a <rtx_insn *> (usage_insn),
6597 : after_p ? NULL : restore,
6598 : after_p ? restore : NULL,
6599 : call_save_p ? "Add reg<-save" : "Add reg<-split");
6600 650862 : if (call_save_p
6601 649463 : && first_call_insn != NULL
6602 1300325 : && BLOCK_FOR_INSN (first_call_insn) != BLOCK_FOR_INSN (insn))
6603 : /* PR116028: If original_regno is a pseudo that has been assigned a
6604 : callee-saved hard register, then emit the spill insn before the call
6605 : insn 'first_call_insn' instead of adjacent to 'insn'. If 'insn'
6606 : and 'first_call_insn' belong to the same EBB but to two separate
6607 : BBs, and if 'insn' is present in the entry BB, then generating the
6608 : spill insn in the entry BB can prevent shrink wrap from happening.
6609 : This is because the spill insn references the stack pointer and
6610 : hence the prolog gets generated in the entry BB itself. It is
6611 : also more efficient to generate the spill before
6612 : 'first_call_insn' as the spill now occurs only in the path
6613 : containing the call. */
6614 26355 : lra_process_new_insns (first_call_insn, save, NULL, "Add save<-reg");
6615 : else
6616 1249777 : lra_process_new_insns (insn, before_p ? save : NULL,
6617 : before_p ? NULL : save,
6618 : call_save_p ? "Add save<-reg" : "Add split<-reg");
6619 650862 : if (nregs > 1 || original_regno < FIRST_PSEUDO_REGISTER)
6620 : /* If we are trying to split multi-register. We should check
6621 : conflicts on the next assignment sub-pass. IRA can allocate on
6622 : sub-register levels, LRA do this on pseudos level right now and
6623 : this discrepancy may create allocation conflicts after
6624 : splitting.
6625 :
6626 : If we are trying to split hard register we should also check conflicts
6627 : as such splitting can create artificial conflict of the hard register
6628 : with another pseudo because of simplified conflict calculation in
6629 : LRA. */
6630 9764 : check_and_force_assignment_correctness_p = true;
6631 650862 : if (lra_dump_file != NULL)
6632 0 : fprintf (lra_dump_file,
6633 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
6634 : return true;
6635 : }
6636 :
6637 : /* Split a hard reg for reload pseudo REGNO having RCLASS and living
6638 : in the range [FROM, TO]. Return true if did a split. Otherwise,
6639 : return false. */
6640 : bool
6641 1570 : spill_hard_reg_in_range (int regno, enum reg_class rclass, rtx_insn *from, rtx_insn *to)
6642 : {
6643 1570 : int i, hard_regno;
6644 1570 : int rclass_size;
6645 1570 : rtx_insn *insn;
6646 1570 : unsigned int uid;
6647 1570 : bitmap_iterator bi;
6648 1570 : HARD_REG_SET ignore;
6649 :
6650 1570 : lra_assert (from != NULL && to != NULL);
6651 1570 : ignore = lra_no_alloc_regs;
6652 4523 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi)
6653 : {
6654 2953 : lra_insn_recog_data_t id = lra_insn_recog_data[uid];
6655 2953 : struct lra_static_insn_data *static_id = id->insn_static_data;
6656 2953 : struct lra_insn_reg *reg;
6657 :
6658 9848 : for (reg = id->regs; reg != NULL; reg = reg->next)
6659 6895 : if (reg->regno < FIRST_PSEUDO_REGISTER)
6660 157 : SET_HARD_REG_BIT (ignore, reg->regno);
6661 4609 : for (reg = static_id->hard_regs; reg != NULL; reg = reg->next)
6662 1656 : SET_HARD_REG_BIT (ignore, reg->regno);
6663 : }
6664 1570 : rclass_size = ira_class_hard_regs_num[rclass];
6665 4212 : for (i = 0; i < rclass_size; i++)
6666 : {
6667 2801 : hard_regno = ira_class_hard_regs[rclass][i];
6668 2801 : if (! TEST_HARD_REG_BIT (lra_reg_info[regno].conflict_hard_regs, hard_regno)
6669 2801 : || TEST_HARD_REG_BIT (ignore, hard_regno))
6670 2636 : continue;
6671 476 : for (insn = from; insn != NEXT_INSN (to); insn = NEXT_INSN (insn))
6672 : {
6673 317 : struct lra_static_insn_data *static_id;
6674 317 : struct lra_insn_reg *reg;
6675 :
6676 317 : if (!INSN_P (insn))
6677 0 : continue;
6678 317 : if (bitmap_bit_p (&lra_reg_info[hard_regno].insn_bitmap,
6679 317 : INSN_UID (insn)))
6680 : break;
6681 311 : static_id = lra_get_insn_recog_data (insn)->insn_static_data;
6682 365 : for (reg = static_id->hard_regs; reg != NULL; reg = reg->next)
6683 54 : if (reg->regno == hard_regno)
6684 : break;
6685 : if (reg != NULL)
6686 : break;
6687 : }
6688 165 : if (insn != NEXT_INSN (to))
6689 6 : continue;
6690 159 : if (split_reg (true, hard_regno, from, NULL, to))
6691 : return true;
6692 : }
6693 : return false;
6694 : }
6695 :
6696 : /* Recognize that we need a split transformation for insn INSN, which
6697 : defines or uses REGNO in its insn biggest MODE (we use it only if
6698 : REGNO is a hard register). POTENTIAL_RELOAD_HARD_REGS contains
6699 : hard registers which might be used for reloads since the EBB end.
6700 : Put the save before INSN if BEFORE_P is true. MAX_UID is maximla
6701 : uid before starting INSN processing. Return true if we succeed in
6702 : such transformation. */
6703 : static bool
6704 198043653 : split_if_necessary (int regno, machine_mode mode,
6705 : HARD_REG_SET potential_reload_hard_regs,
6706 : bool before_p, rtx_insn *insn, int max_uid)
6707 : {
6708 198043653 : bool res = false;
6709 198043653 : int i, nregs = 1;
6710 198043653 : rtx next_usage_insns;
6711 :
6712 198043653 : if (regno < FIRST_PSEUDO_REGISTER)
6713 93067887 : nregs = hard_regno_nregs (regno, mode);
6714 396436869 : for (i = 0; i < nregs; i++)
6715 198393216 : if (usage_insns[regno + i].check == curr_usage_insns_check
6716 132422734 : && (next_usage_insns = usage_insns[regno + i].insns) != NULL_RTX
6717 : /* To avoid processing the register twice or more. */
6718 132422734 : && ((GET_CODE (next_usage_insns) != INSN_LIST
6719 128279386 : && INSN_UID (next_usage_insns) < max_uid)
6720 4143348 : || (GET_CODE (next_usage_insns) == INSN_LIST
6721 4143348 : && (INSN_UID (XEXP (next_usage_insns, 0)) < max_uid)))
6722 132422734 : && need_for_split_p (potential_reload_hard_regs, regno + i)
6723 198665589 : && split_reg (before_p, regno + i, insn, next_usage_insns, NULL))
6724 : res = true;
6725 198043653 : return res;
6726 : }
6727 :
6728 : /* Return TRUE if rtx X is considered as an invariant for
6729 : inheritance. */
6730 : static bool
6731 11406100 : invariant_p (const_rtx x)
6732 : {
6733 11406100 : machine_mode mode;
6734 11406100 : const char *fmt;
6735 11406100 : enum rtx_code code;
6736 11406100 : int i, j;
6737 :
6738 11406100 : if (side_effects_p (x))
6739 : return false;
6740 :
6741 11380112 : code = GET_CODE (x);
6742 11380112 : mode = GET_MODE (x);
6743 11380112 : if (code == SUBREG)
6744 : {
6745 462372 : x = SUBREG_REG (x);
6746 462372 : code = GET_CODE (x);
6747 462372 : mode = wider_subreg_mode (mode, GET_MODE (x));
6748 : }
6749 :
6750 11380112 : if (MEM_P (x))
6751 : return false;
6752 :
6753 9655591 : if (REG_P (x))
6754 : {
6755 3439272 : int i, nregs, regno = REGNO (x);
6756 :
6757 3439272 : if (regno >= FIRST_PSEUDO_REGISTER || regno == STACK_POINTER_REGNUM
6758 893094 : || TEST_HARD_REG_BIT (eliminable_regset, regno)
6759 3456578 : || GET_MODE_CLASS (GET_MODE (x)) == MODE_CC)
6760 : return false;
6761 2 : nregs = hard_regno_nregs (regno, mode);
6762 2 : for (i = 0; i < nregs; i++)
6763 2 : if (! fixed_regs[regno + i]
6764 : /* A hard register may be clobbered in the current insn
6765 : but we can ignore this case because if the hard
6766 : register is used it should be set somewhere after the
6767 : clobber. */
6768 2 : || bitmap_bit_p (&invalid_invariant_regs, regno + i))
6769 2 : return false;
6770 : }
6771 6216319 : fmt = GET_RTX_FORMAT (code);
6772 10934831 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
6773 : {
6774 8256903 : if (fmt[i] == 'e')
6775 : {
6776 5418273 : if (! invariant_p (XEXP (x, i)))
6777 : return false;
6778 : }
6779 2838630 : else if (fmt[i] == 'E')
6780 : {
6781 632390 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
6782 522736 : if (! invariant_p (XVECEXP (x, i, j)))
6783 : return false;
6784 : }
6785 : }
6786 : return true;
6787 : }
6788 :
6789 : /* We have 'dest_reg <- invariant'. Let us try to make an invariant
6790 : inheritance transformation (using dest_reg instead invariant in a
6791 : subsequent insn). */
6792 : static bool
6793 174976 : process_invariant_for_inheritance (rtx dst_reg, rtx invariant_rtx)
6794 : {
6795 174976 : invariant_ptr_t invariant_ptr;
6796 174976 : rtx_insn *insn, *new_insns;
6797 174976 : rtx insn_set, insn_reg, new_reg;
6798 174976 : int insn_regno;
6799 174976 : bool succ_p = false;
6800 174976 : int dst_regno = REGNO (dst_reg);
6801 174976 : machine_mode dst_mode = GET_MODE (dst_reg);
6802 174976 : enum reg_class cl = lra_get_allocno_class (dst_regno), insn_reg_cl;
6803 :
6804 174976 : invariant_ptr = insert_invariant (invariant_rtx);
6805 174976 : if ((insn = invariant_ptr->insn) != NULL_RTX)
6806 : {
6807 : /* We have a subsequent insn using the invariant. */
6808 22831 : insn_set = single_set (insn);
6809 22831 : lra_assert (insn_set != NULL);
6810 22831 : insn_reg = SET_DEST (insn_set);
6811 22831 : lra_assert (REG_P (insn_reg));
6812 22831 : insn_regno = REGNO (insn_reg);
6813 22831 : insn_reg_cl = lra_get_allocno_class (insn_regno);
6814 :
6815 22831 : if (dst_mode == GET_MODE (insn_reg)
6816 : /* We should consider only result move reg insns which are
6817 : cheap. */
6818 22765 : && targetm.register_move_cost (dst_mode, cl, insn_reg_cl) == 2
6819 45000 : && targetm.register_move_cost (dst_mode, cl, cl) == 2)
6820 : {
6821 22169 : if (lra_dump_file != NULL)
6822 0 : fprintf (lra_dump_file,
6823 : " [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[\n");
6824 22169 : new_reg = lra_create_new_reg (dst_mode, dst_reg, cl, NULL,
6825 : "invariant inheritance");
6826 22169 : bitmap_set_bit (&lra_inheritance_pseudos, REGNO (new_reg));
6827 22169 : bitmap_set_bit (&check_only_regs, REGNO (new_reg));
6828 22169 : lra_reg_info[REGNO (new_reg)].restore_rtx = PATTERN (insn);
6829 22169 : start_sequence ();
6830 22169 : lra_emit_move (new_reg, dst_reg);
6831 22169 : new_insns = end_sequence ();
6832 22169 : lra_process_new_insns (curr_insn, NULL, new_insns,
6833 : "Add invariant inheritance<-original");
6834 22169 : start_sequence ();
6835 22169 : lra_emit_move (SET_DEST (insn_set), new_reg);
6836 22169 : new_insns = end_sequence ();
6837 22169 : lra_process_new_insns (insn, NULL, new_insns,
6838 : "Changing reload<-inheritance");
6839 22169 : lra_set_insn_deleted (insn);
6840 22169 : succ_p = true;
6841 22169 : if (lra_dump_file != NULL)
6842 : {
6843 0 : fprintf (lra_dump_file,
6844 : " Invariant inheritance reuse change %d (bb%d):\n",
6845 0 : REGNO (new_reg), BLOCK_FOR_INSN (insn)->index);
6846 0 : dump_insn_slim (lra_dump_file, insn);
6847 0 : fprintf (lra_dump_file,
6848 : " ]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]\n");
6849 : }
6850 : }
6851 : }
6852 174976 : invariant_ptr->insn = curr_insn;
6853 174976 : return succ_p;
6854 : }
6855 :
6856 : /* Check only registers living at the current program point in the
6857 : current EBB. */
6858 : static bitmap_head live_regs;
6859 :
6860 : /* Update live info in EBB given by its HEAD and TAIL insns after
6861 : inheritance/split transformation. The function removes dead moves
6862 : too. */
6863 : static void
6864 737584 : update_ebb_live_info (rtx_insn *head, rtx_insn *tail)
6865 : {
6866 737584 : unsigned int j;
6867 737584 : int i, regno;
6868 737584 : bool live_p;
6869 737584 : rtx_insn *prev_insn;
6870 737584 : rtx set;
6871 737584 : bool remove_p;
6872 737584 : basic_block last_bb, prev_bb, curr_bb;
6873 737584 : bitmap_iterator bi;
6874 737584 : struct lra_insn_reg *reg;
6875 737584 : edge e;
6876 737584 : edge_iterator ei;
6877 :
6878 737584 : last_bb = BLOCK_FOR_INSN (tail);
6879 737584 : prev_bb = NULL;
6880 737584 : for (curr_insn = tail;
6881 37025746 : curr_insn != PREV_INSN (head);
6882 36288162 : curr_insn = prev_insn)
6883 : {
6884 36288162 : prev_insn = PREV_INSN (curr_insn);
6885 : /* We need to process empty blocks too. They contain
6886 : NOTE_INSN_BASIC_BLOCK referring for the basic block. */
6887 36288162 : if (NOTE_P (curr_insn) && NOTE_KIND (curr_insn) != NOTE_INSN_BASIC_BLOCK)
6888 1398417 : continue;
6889 34889745 : curr_bb = BLOCK_FOR_INSN (curr_insn);
6890 34889745 : if (curr_bb != prev_bb)
6891 : {
6892 1467633 : if (prev_bb != NULL)
6893 : {
6894 : /* Update df_get_live_in (prev_bb): */
6895 54229986 : EXECUTE_IF_SET_IN_BITMAP (&check_only_regs, 0, j, bi)
6896 53499937 : if (bitmap_bit_p (&live_regs, j))
6897 1593775 : bitmap_set_bit (df_get_live_in (prev_bb), j);
6898 : else
6899 51906162 : bitmap_clear_bit (df_get_live_in (prev_bb), j);
6900 : }
6901 1467633 : if (curr_bb != last_bb)
6902 : {
6903 : /* Update df_get_live_out (curr_bb): */
6904 54229986 : EXECUTE_IF_SET_IN_BITMAP (&check_only_regs, 0, j, bi)
6905 : {
6906 53499937 : live_p = bitmap_bit_p (&live_regs, j);
6907 53499937 : if (! live_p)
6908 155622248 : FOR_EACH_EDGE (e, ei, curr_bb->succs)
6909 103770954 : if (bitmap_bit_p (df_get_live_in (e->dest), j))
6910 : {
6911 : live_p = true;
6912 : break;
6913 : }
6914 51906162 : if (live_p)
6915 1648643 : bitmap_set_bit (df_get_live_out (curr_bb), j);
6916 : else
6917 51851294 : bitmap_clear_bit (df_get_live_out (curr_bb), j);
6918 : }
6919 : }
6920 1467633 : prev_bb = curr_bb;
6921 1467633 : bitmap_and (&live_regs, &check_only_regs, df_get_live_out (curr_bb));
6922 : }
6923 34889745 : if (! NONDEBUG_INSN_P (curr_insn))
6924 12915399 : continue;
6925 21974346 : curr_id = lra_get_insn_recog_data (curr_insn);
6926 21974346 : curr_static_id = curr_id->insn_static_data;
6927 21974346 : remove_p = false;
6928 21974346 : if ((set = single_set (curr_insn)) != NULL_RTX
6929 21292152 : && REG_P (SET_DEST (set))
6930 16965347 : && (regno = REGNO (SET_DEST (set))) >= FIRST_PSEUDO_REGISTER
6931 12604061 : && SET_DEST (set) != pic_offset_table_rtx
6932 12597429 : && bitmap_bit_p (&check_only_regs, regno)
6933 25178971 : && ! bitmap_bit_p (&live_regs, regno))
6934 : remove_p = true;
6935 : /* See which defined values die here. */
6936 60670434 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
6937 38696088 : if (reg->type == OP_OUT && ! reg->subreg_p)
6938 15116732 : bitmap_clear_bit (&live_regs, reg->regno);
6939 26128376 : for (reg = curr_static_id->hard_regs; reg != NULL; reg = reg->next)
6940 4154030 : if (reg->type == OP_OUT && ! reg->subreg_p)
6941 3147411 : bitmap_clear_bit (&live_regs, reg->regno);
6942 21974346 : if (curr_id->arg_hard_regs != NULL)
6943 : /* Make clobbered argument hard registers die. */
6944 3312011 : for (i = 0; (regno = curr_id->arg_hard_regs[i]) >= 0; i++)
6945 2374686 : if (regno >= FIRST_PSEUDO_REGISTER)
6946 189780 : bitmap_clear_bit (&live_regs, regno - FIRST_PSEUDO_REGISTER);
6947 : /* Mark each used value as live. */
6948 60670434 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
6949 38696088 : if (reg->type != OP_OUT
6950 38696088 : && bitmap_bit_p (&check_only_regs, reg->regno))
6951 4500412 : bitmap_set_bit (&live_regs, reg->regno);
6952 26128376 : for (reg = curr_static_id->hard_regs; reg != NULL; reg = reg->next)
6953 4154030 : if (reg->type != OP_OUT
6954 4154030 : && bitmap_bit_p (&check_only_regs, reg->regno))
6955 0 : bitmap_set_bit (&live_regs, reg->regno);
6956 21974346 : if (curr_id->arg_hard_regs != NULL)
6957 : /* Make used argument hard registers live. */
6958 3312011 : for (i = 0; (regno = curr_id->arg_hard_regs[i]) >= 0; i++)
6959 2374686 : if (regno < FIRST_PSEUDO_REGISTER
6960 2374686 : && bitmap_bit_p (&check_only_regs, regno))
6961 0 : bitmap_set_bit (&live_regs, regno);
6962 : /* It is quite important to remove dead move insns because it
6963 : means removing dead store. We don't need to process them for
6964 : constraints. */
6965 21974346 : if (remove_p)
6966 : {
6967 296563 : if (lra_dump_file != NULL)
6968 : {
6969 2 : fprintf (lra_dump_file, " Removing dead insn:\n ");
6970 2 : dump_insn_slim (lra_dump_file, curr_insn);
6971 : }
6972 296563 : lra_set_insn_deleted (curr_insn);
6973 : }
6974 : }
6975 737584 : }
6976 :
6977 : /* The structure describes info to do an inheritance for the current
6978 : insn. We need to collect such info first before doing the
6979 : transformations because the transformations change the insn
6980 : internal representation. */
6981 : struct to_inherit
6982 : {
6983 : /* Original regno. */
6984 : int regno;
6985 : /* Subsequent insns which can inherit original reg value. */
6986 : rtx insns;
6987 : };
6988 :
6989 : /* Array containing all info for doing inheritance from the current
6990 : insn. */
6991 : static struct to_inherit to_inherit[LRA_MAX_INSN_RELOADS];
6992 :
6993 : /* Number elements in the previous array. */
6994 : static int to_inherit_num;
6995 :
6996 : /* Add inheritance info REGNO and INSNS. Their meaning is described in
6997 : structure to_inherit. */
6998 : static void
6999 311389 : add_to_inherit (int regno, rtx insns)
7000 : {
7001 311389 : int i;
7002 :
7003 311471 : for (i = 0; i < to_inherit_num; i++)
7004 82 : if (to_inherit[i].regno == regno)
7005 : return;
7006 311389 : lra_assert (to_inherit_num < LRA_MAX_INSN_RELOADS);
7007 311389 : to_inherit[to_inherit_num].regno = regno;
7008 311389 : to_inherit[to_inherit_num++].insns = insns;
7009 : }
7010 :
7011 : /* Return the last non-debug insn in basic block BB, or the block begin
7012 : note if none. */
7013 : static rtx_insn *
7014 29765903 : get_last_insertion_point (basic_block bb)
7015 : {
7016 29765903 : rtx_insn *insn;
7017 :
7018 32135170 : FOR_BB_INSNS_REVERSE (bb, insn)
7019 32135170 : if (NONDEBUG_INSN_P (insn) || NOTE_INSN_BASIC_BLOCK_P (insn))
7020 29765903 : return insn;
7021 0 : gcc_unreachable ();
7022 : }
7023 :
7024 : /* Set up RES by registers living on edges FROM except the edge (FROM,
7025 : TO) or by registers set up in a jump insn in BB FROM. */
7026 : static void
7027 11455980 : get_live_on_other_edges (basic_block from, basic_block to, bitmap res)
7028 : {
7029 11455980 : rtx_insn *last;
7030 11455980 : struct lra_insn_reg *reg;
7031 11455980 : edge e;
7032 11455980 : edge_iterator ei;
7033 :
7034 11455980 : lra_assert (to != NULL);
7035 11455980 : bitmap_clear (res);
7036 34112022 : FOR_EACH_EDGE (e, ei, from->succs)
7037 22656042 : if (e->dest != to)
7038 11200062 : bitmap_ior_into (res, df_get_live_in (e->dest));
7039 11455980 : last = get_last_insertion_point (from);
7040 11455980 : if (! JUMP_P (last))
7041 1863215 : return;
7042 9592765 : curr_id = lra_get_insn_recog_data (last);
7043 19185352 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7044 9592587 : if (reg->type != OP_IN)
7045 76 : bitmap_set_bit (res, reg->regno);
7046 : }
7047 :
7048 : /* Used as a temporary results of some bitmap calculations. */
7049 : static bitmap_head temp_bitmap;
7050 :
7051 : /* We split for reloads of small class of hard regs. The following
7052 : defines how many hard regs the class should have to be qualified as
7053 : small. The code is mostly oriented to x86/x86-64 architecture
7054 : where some insns need to use only specific register or pair of
7055 : registers and these register can live in RTL explicitly, e.g. for
7056 : parameter passing. */
7057 : static const int max_small_class_regs_num = 2;
7058 :
7059 : /* Do inheritance/split transformations in EBB starting with HEAD and
7060 : finishing on TAIL. We process EBB insns in the reverse order.
7061 : Return true if we did any inheritance/split transformation in the
7062 : EBB.
7063 :
7064 : We should avoid excessive splitting which results in worse code
7065 : because of inaccurate cost calculations for spilling new split
7066 : pseudos in such case. To achieve this we do splitting only if
7067 : register pressure is high in given basic block and there are reload
7068 : pseudos requiring hard registers. We could do more register
7069 : pressure calculations at any given program point to avoid necessary
7070 : splitting even more but it is to expensive and the current approach
7071 : works well enough. */
7072 : static bool
7073 12582050 : inherit_in_ebb (rtx_insn *head, rtx_insn *tail)
7074 : {
7075 12582050 : int i, src_regno, dst_regno, nregs;
7076 12582050 : bool change_p, succ_p, update_reloads_num_p;
7077 12582050 : rtx_insn *prev_insn, *last_insn;
7078 12582050 : rtx next_usage_insns, curr_set;
7079 12582050 : enum reg_class cl;
7080 12582050 : struct lra_insn_reg *reg;
7081 12582050 : basic_block last_processed_bb, curr_bb = NULL;
7082 12582050 : HARD_REG_SET potential_reload_hard_regs, live_hard_regs;
7083 12582050 : bitmap to_process;
7084 12582050 : unsigned int j;
7085 12582050 : bitmap_iterator bi;
7086 12582050 : bool head_p, after_p;
7087 :
7088 12582050 : change_p = false;
7089 12582050 : curr_usage_insns_check++;
7090 12582050 : clear_invariants ();
7091 12582050 : reloads_num = calls_num = 0;
7092 163566650 : for (unsigned int i = 0; i < NUM_ABI_IDS; ++i)
7093 150984600 : last_call_for_abi[i] = 0;
7094 12582050 : CLEAR_HARD_REG_SET (full_and_partial_call_clobbers);
7095 12582050 : bitmap_clear (&check_only_regs);
7096 12582050 : bitmap_clear (&invalid_invariant_regs);
7097 12582050 : last_processed_bb = NULL;
7098 12582050 : CLEAR_HARD_REG_SET (potential_reload_hard_regs);
7099 12582050 : live_hard_regs = eliminable_regset | lra_no_alloc_regs;
7100 : /* We don't process new insns generated in the loop. */
7101 233459349 : for (curr_insn = tail; curr_insn != PREV_INSN (head); curr_insn = prev_insn)
7102 : {
7103 220877299 : prev_insn = PREV_INSN (curr_insn);
7104 220877299 : if (BLOCK_FOR_INSN (curr_insn) != NULL)
7105 220877065 : curr_bb = BLOCK_FOR_INSN (curr_insn);
7106 220877299 : if (last_processed_bb != curr_bb)
7107 : {
7108 : /* We are at the end of BB. Add qualified living
7109 : pseudos for potential splitting. */
7110 18309923 : to_process = df_get_live_out (curr_bb);
7111 18309923 : if (last_processed_bb != NULL)
7112 : {
7113 : /* We are somewhere in the middle of EBB. */
7114 5727873 : get_live_on_other_edges (curr_bb, last_processed_bb,
7115 : &temp_bitmap);
7116 5727873 : to_process = &temp_bitmap;
7117 : }
7118 18309923 : last_processed_bb = curr_bb;
7119 18309923 : last_insn = get_last_insertion_point (curr_bb);
7120 36619846 : after_p = (! JUMP_P (last_insn)
7121 18309923 : && (! CALL_P (last_insn)
7122 2266046 : || (find_reg_note (last_insn,
7123 : REG_NORETURN, NULL_RTX) == NULL_RTX
7124 1347443 : && ! SIBLING_CALL_P (last_insn))));
7125 18309923 : CLEAR_HARD_REG_SET (potential_reload_hard_regs);
7126 199236259 : EXECUTE_IF_SET_IN_BITMAP (to_process, 0, j, bi)
7127 : {
7128 180926342 : if ((int) j >= lra_constraint_new_regno_start)
7129 : break;
7130 180926336 : if (j < FIRST_PSEUDO_REGISTER || reg_renumber[j] >= 0)
7131 : {
7132 117711646 : if (j < FIRST_PSEUDO_REGISTER)
7133 68859573 : SET_HARD_REG_BIT (live_hard_regs, j);
7134 : else
7135 48852073 : add_to_hard_reg_set (&live_hard_regs,
7136 48852073 : PSEUDO_REGNO_MODE (j),
7137 48852073 : reg_renumber[j]);
7138 117711646 : setup_next_usage_insn (j, last_insn, reloads_num, after_p);
7139 : }
7140 : }
7141 : }
7142 220877299 : src_regno = dst_regno = -1;
7143 220877299 : curr_set = single_set (curr_insn);
7144 220877299 : if (curr_set != NULL_RTX && REG_P (SET_DEST (curr_set)))
7145 82832660 : dst_regno = REGNO (SET_DEST (curr_set));
7146 112574841 : if (curr_set != NULL_RTX && REG_P (SET_SRC (curr_set)))
7147 39040447 : src_regno = REGNO (SET_SRC (curr_set));
7148 220877299 : update_reloads_num_p = true;
7149 220877299 : if (src_regno < lra_constraint_new_regno_start
7150 214571506 : && src_regno >= FIRST_PSEUDO_REGISTER
7151 27548973 : && reg_renumber[src_regno] < 0
7152 3679479 : && dst_regno >= lra_constraint_new_regno_start
7153 223492677 : && (cl = lra_get_allocno_class (dst_regno)) != NO_REGS)
7154 : {
7155 : /* 'reload_pseudo <- original_pseudo'. */
7156 2615378 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7157 22059 : reloads_num++;
7158 2615378 : update_reloads_num_p = false;
7159 2615378 : succ_p = false;
7160 2615378 : if (usage_insns[src_regno].check == curr_usage_insns_check
7161 2615378 : && (next_usage_insns = usage_insns[src_regno].insns) != NULL_RTX)
7162 470981 : succ_p = inherit_reload_reg (false, src_regno, cl,
7163 : curr_insn, next_usage_insns);
7164 470981 : if (succ_p)
7165 : change_p = true;
7166 : else
7167 2167559 : setup_next_usage_insn (src_regno, curr_insn, reloads_num, false);
7168 5230756 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7169 620968023 : potential_reload_hard_regs |= reg_class_contents[cl];
7170 : }
7171 218261921 : else if (src_regno < 0
7172 181836852 : && dst_regno >= lra_constraint_new_regno_start
7173 5465091 : && invariant_p (SET_SRC (curr_set))
7174 275310 : && (cl = lra_get_allocno_class (dst_regno)) != NO_REGS
7175 274760 : && ! bitmap_bit_p (&invalid_invariant_regs, dst_regno)
7176 218498205 : && ! bitmap_bit_p (&invalid_invariant_regs,
7177 236284 : ORIGINAL_REGNO(regno_reg_rtx[dst_regno])))
7178 : {
7179 : /* 'reload_pseudo <- invariant'. */
7180 174976 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7181 7804 : reloads_num++;
7182 174976 : update_reloads_num_p = false;
7183 174976 : if (process_invariant_for_inheritance (SET_DEST (curr_set), SET_SRC (curr_set)))
7184 22169 : change_p = true;
7185 349952 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7186 620968023 : potential_reload_hard_regs |= reg_class_contents[cl];
7187 : }
7188 218086945 : else if (src_regno >= lra_constraint_new_regno_start
7189 6305793 : && dst_regno < lra_constraint_new_regno_start
7190 5501569 : && dst_regno >= FIRST_PSEUDO_REGISTER
7191 3679295 : && reg_renumber[dst_regno] < 0
7192 1438089 : && (cl = lra_get_allocno_class (src_regno)) != NO_REGS
7193 1438089 : && usage_insns[dst_regno].check == curr_usage_insns_check
7194 218086945 : && (next_usage_insns
7195 473548 : = usage_insns[dst_regno].insns) != NULL_RTX)
7196 : {
7197 473548 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7198 7981 : reloads_num++;
7199 473548 : update_reloads_num_p = false;
7200 : /* 'original_pseudo <- reload_pseudo'. */
7201 473548 : if (! JUMP_P (curr_insn)
7202 473548 : && inherit_reload_reg (true, dst_regno, cl,
7203 : curr_insn, next_usage_insns))
7204 : change_p = true;
7205 : /* Invalidate. */
7206 473548 : usage_insns[dst_regno].check = 0;
7207 947096 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7208 620968023 : potential_reload_hard_regs |= reg_class_contents[cl];
7209 : }
7210 217613397 : else if (INSN_P (curr_insn))
7211 : {
7212 182405142 : int iter;
7213 182405142 : int max_uid = get_max_uid ();
7214 :
7215 182405142 : curr_id = lra_get_insn_recog_data (curr_insn);
7216 182405142 : curr_static_id = curr_id->insn_static_data;
7217 182405142 : to_inherit_num = 0;
7218 : /* Process insn definitions. */
7219 547215426 : for (iter = 0; iter < 2; iter++)
7220 364810284 : for (reg = iter == 0 ? curr_id->regs : curr_static_id->hard_regs;
7221 590057027 : reg != NULL;
7222 225246743 : reg = reg->next)
7223 225246743 : if (reg->type != OP_IN
7224 225246743 : && (dst_regno = reg->regno) < lra_constraint_new_regno_start)
7225 : {
7226 45351004 : if (dst_regno >= FIRST_PSEUDO_REGISTER && reg->type == OP_OUT
7227 43215173 : && reg_renumber[dst_regno] < 0 && ! reg->subreg_p
7228 1750619 : && usage_insns[dst_regno].check == curr_usage_insns_check
7229 90016117 : && (next_usage_insns
7230 129299 : = usage_insns[dst_regno].insns) != NULL_RTX)
7231 : {
7232 129299 : struct lra_insn_reg *r;
7233 :
7234 385193 : for (r = curr_id->regs; r != NULL; r = r->next)
7235 255894 : if (r->type != OP_OUT && r->regno == dst_regno)
7236 : break;
7237 : /* Don't do inheritance if the pseudo is also
7238 : used in the insn. */
7239 129299 : if (r == NULL)
7240 : /* We cannot do inheritance right now
7241 : because the current insn reg info (chain
7242 : regs) can change after that. */
7243 129299 : add_to_inherit (dst_regno, next_usage_insns);
7244 : }
7245 : /* We cannot process one reg twice here because of
7246 : usage_insns invalidation. */
7247 90016117 : if ((dst_regno < FIRST_PSEUDO_REGISTER
7248 45351004 : || reg_renumber[dst_regno] >= 0)
7249 88129284 : && ! reg->subreg_p && reg->type != OP_IN)
7250 : {
7251 87849340 : HARD_REG_SET s;
7252 :
7253 87849340 : if (split_if_necessary (dst_regno, reg->biggest_mode,
7254 : potential_reload_hard_regs,
7255 : false, curr_insn, max_uid))
7256 57727 : change_p = true;
7257 87849340 : CLEAR_HARD_REG_SET (s);
7258 87849340 : if (dst_regno < FIRST_PSEUDO_REGISTER)
7259 44665113 : add_to_hard_reg_set (&s, reg->biggest_mode, dst_regno);
7260 : else
7261 43184227 : add_to_hard_reg_set (&s, PSEUDO_REGNO_MODE (dst_regno),
7262 43184227 : reg_renumber[dst_regno]);
7263 87849340 : live_hard_regs &= ~s;
7264 175698680 : potential_reload_hard_regs &= ~s;
7265 : }
7266 : /* We should invalidate potential inheritance or
7267 : splitting for the current insn usages to the next
7268 : usage insns (see code below) as the output pseudo
7269 : prevents this. */
7270 90016117 : if ((dst_regno >= FIRST_PSEUDO_REGISTER
7271 45351004 : && reg_renumber[dst_regno] < 0)
7272 88129284 : || (reg->type == OP_OUT && ! reg->subreg_p
7273 80128431 : && (dst_regno < FIRST_PSEUDO_REGISTER
7274 41056824 : || reg_renumber[dst_regno] >= 0)))
7275 : {
7276 : /* Invalidate and mark definitions. */
7277 42943657 : if (dst_regno >= FIRST_PSEUDO_REGISTER)
7278 42943657 : usage_insns[dst_regno].check = -(int) INSN_UID (curr_insn);
7279 : else
7280 : {
7281 39071607 : nregs = hard_regno_nregs (dst_regno,
7282 39071607 : reg->biggest_mode);
7283 78396786 : for (i = 0; i < nregs; i++)
7284 78650358 : usage_insns[dst_regno + i].check
7285 39325179 : = -(int) INSN_UID (curr_insn);
7286 : }
7287 : }
7288 : }
7289 : /* Process clobbered call regs. */
7290 182405142 : if (curr_id->arg_hard_regs != NULL)
7291 19537596 : for (i = 0; (dst_regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7292 13923141 : if (dst_regno >= FIRST_PSEUDO_REGISTER)
7293 1611174 : usage_insns[dst_regno - FIRST_PSEUDO_REGISTER].check
7294 805587 : = -(int) INSN_UID (curr_insn);
7295 182405142 : if (! JUMP_P (curr_insn))
7296 171074324 : for (i = 0; i < to_inherit_num; i++)
7297 129299 : if (inherit_reload_reg (true, to_inherit[i].regno,
7298 : ALL_REGS, curr_insn,
7299 : to_inherit[i].insns))
7300 104093 : change_p = true;
7301 182405142 : if (CALL_P (curr_insn))
7302 : {
7303 7215971 : rtx cheap, pat, dest;
7304 7215971 : rtx_insn *restore;
7305 7215971 : int regno, hard_regno;
7306 :
7307 7215971 : calls_num++;
7308 7215971 : function_abi callee_abi = insn_callee_abi (curr_insn);
7309 7215971 : last_call_for_abi[callee_abi.id ()] = calls_num;
7310 7215971 : full_and_partial_call_clobbers
7311 7215971 : |= callee_abi.full_and_partial_reg_clobbers ();
7312 7215971 : first_call_insn = curr_insn;
7313 7215971 : if ((cheap = find_reg_note (curr_insn,
7314 : REG_RETURNED, NULL_RTX)) != NULL_RTX
7315 43740 : && ((cheap = XEXP (cheap, 0)), true)
7316 43740 : && (regno = REGNO (cheap)) >= FIRST_PSEUDO_REGISTER
7317 43740 : && (hard_regno = reg_renumber[regno]) >= 0
7318 32524 : && usage_insns[regno].check == curr_usage_insns_check
7319 : /* If there are pending saves/restores, the
7320 : optimization is not worth. */
7321 28257 : && usage_insns[regno].calls_num == calls_num - 1
7322 7242373 : && callee_abi.clobbers_reg_p (GET_MODE (cheap), hard_regno))
7323 : {
7324 : /* Restore the pseudo from the call result as
7325 : REG_RETURNED note says that the pseudo value is
7326 : in the call result and the pseudo is an argument
7327 : of the call. */
7328 11009 : pat = PATTERN (curr_insn);
7329 11009 : if (GET_CODE (pat) == PARALLEL)
7330 0 : pat = XVECEXP (pat, 0, 0);
7331 11009 : dest = SET_DEST (pat);
7332 : /* For multiple return values dest is PARALLEL.
7333 : Currently we handle only single return value case. */
7334 11009 : if (REG_P (dest))
7335 : {
7336 11009 : start_sequence ();
7337 11009 : emit_move_insn (cheap, copy_rtx (dest));
7338 11009 : restore = end_sequence ();
7339 11009 : lra_process_new_insns (curr_insn, NULL, restore,
7340 : "Inserting call parameter restore");
7341 : /* We don't need to save/restore of the pseudo from
7342 : this call. */
7343 11009 : usage_insns[regno].calls_num = calls_num;
7344 11009 : remove_from_hard_reg_set
7345 11009 : (&full_and_partial_call_clobbers,
7346 11009 : GET_MODE (cheap), hard_regno);
7347 11009 : bitmap_set_bit (&check_only_regs, regno);
7348 : }
7349 : }
7350 : }
7351 182405142 : to_inherit_num = 0;
7352 : /* Process insn usages. */
7353 547215426 : for (iter = 0; iter < 2; iter++)
7354 364810284 : for (reg = iter == 0 ? curr_id->regs : curr_static_id->hard_regs;
7355 590057027 : reg != NULL;
7356 225246743 : reg = reg->next)
7357 225246743 : if ((reg->type != OP_OUT
7358 88664564 : || (reg->type == OP_OUT && reg->subreg_p))
7359 225789130 : && (src_regno = reg->regno) < lra_constraint_new_regno_start)
7360 : {
7361 125809737 : if (src_regno >= FIRST_PSEUDO_REGISTER
7362 72941485 : && reg_renumber[src_regno] < 0 && reg->type == OP_IN)
7363 : {
7364 2421414 : if (usage_insns[src_regno].check == curr_usage_insns_check
7365 776882 : && (next_usage_insns
7366 776882 : = usage_insns[src_regno].insns) != NULL_RTX
7367 3198296 : && NONDEBUG_INSN_P (curr_insn))
7368 182090 : add_to_inherit (src_regno, next_usage_insns);
7369 4478648 : else if (usage_insns[src_regno].check
7370 2239324 : != -(int) INSN_UID (curr_insn))
7371 : /* Add usages but only if the reg is not set up
7372 : in the same insn. */
7373 2239324 : add_next_usage_insn (src_regno, curr_insn, reloads_num);
7374 : }
7375 70520071 : else if (src_regno < FIRST_PSEUDO_REGISTER
7376 70520071 : || reg_renumber[src_regno] >= 0)
7377 : {
7378 123252109 : bool before_p;
7379 123252109 : rtx_insn *use_insn = curr_insn;
7380 123252109 : rtx_insn *prev_insn = PREV_INSN (curr_insn);
7381 :
7382 246504218 : before_p = (JUMP_P (curr_insn)
7383 123252109 : || (CALL_P (curr_insn) && reg->type == OP_IN));
7384 123252109 : if (NONDEBUG_INSN_P (curr_insn)
7385 110194418 : && (! JUMP_P (curr_insn) || reg->type == OP_IN)
7386 233446422 : && split_if_necessary (src_regno, reg->biggest_mode,
7387 : potential_reload_hard_regs,
7388 : before_p, curr_insn, max_uid))
7389 : {
7390 214646 : if (reg->subreg_p)
7391 3153 : check_and_force_assignment_correctness_p = true;
7392 214646 : change_p = true;
7393 : /* Invalidate. */
7394 214646 : usage_insns[src_regno].check = 0;
7395 214646 : if (before_p && PREV_INSN (curr_insn) != prev_insn)
7396 : use_insn = PREV_INSN (curr_insn);
7397 : }
7398 123252109 : if (NONDEBUG_INSN_P (curr_insn))
7399 : {
7400 110194418 : if (src_regno < FIRST_PSEUDO_REGISTER)
7401 48402797 : add_to_hard_reg_set (&live_hard_regs,
7402 48402797 : reg->biggest_mode, src_regno);
7403 : else
7404 61791621 : add_to_hard_reg_set (&live_hard_regs,
7405 61791621 : PSEUDO_REGNO_MODE (src_regno),
7406 61791621 : reg_renumber[src_regno]);
7407 : }
7408 123252109 : if (src_regno >= FIRST_PSEUDO_REGISTER)
7409 70383857 : add_next_usage_insn (src_regno, use_insn, reloads_num);
7410 : else
7411 : {
7412 105833273 : for (i = 0; i < hard_regno_nregs (src_regno, reg->biggest_mode); i++)
7413 52965021 : add_next_usage_insn (src_regno + i, use_insn, reloads_num);
7414 : }
7415 : }
7416 : }
7417 : /* Process used call regs. */
7418 182405142 : if (curr_id->arg_hard_regs != NULL)
7419 19537596 : for (i = 0; (src_regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7420 13923141 : if (src_regno < FIRST_PSEUDO_REGISTER)
7421 : {
7422 13117554 : SET_HARD_REG_BIT (live_hard_regs, src_regno);
7423 13117554 : add_next_usage_insn (src_regno, curr_insn, reloads_num);
7424 : }
7425 182587232 : for (i = 0; i < to_inherit_num; i++)
7426 : {
7427 182090 : src_regno = to_inherit[i].regno;
7428 182090 : if (inherit_reload_reg (false, src_regno, ALL_REGS,
7429 : curr_insn, to_inherit[i].insns))
7430 : change_p = true;
7431 : else
7432 22588 : setup_next_usage_insn (src_regno, curr_insn, reloads_num, false);
7433 : }
7434 : }
7435 182477327 : if (update_reloads_num_p
7436 217613397 : && NONDEBUG_INSN_P (curr_insn) && curr_set != NULL_RTX)
7437 : {
7438 109310939 : int regno = -1;
7439 109310939 : if ((REG_P (SET_DEST (curr_set))
7440 79568758 : && (regno = REGNO (SET_DEST (curr_set))) >= lra_constraint_new_regno_start
7441 8064438 : && reg_renumber[regno] < 0
7442 5153994 : && (cl = lra_get_allocno_class (regno)) != NO_REGS)
7443 184004031 : || (REG_P (SET_SRC (curr_set))
7444 34703908 : && (regno = REGNO (SET_SRC (curr_set))) >= lra_constraint_new_regno_start
7445 5901553 : && reg_renumber[regno] < 0
7446 3462864 : && (cl = lra_get_allocno_class (regno)) != NO_REGS))
7447 : {
7448 8072524 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7449 214146 : reloads_num++;
7450 16145048 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7451 220877299 : potential_reload_hard_regs |= reg_class_contents[cl];
7452 : }
7453 : }
7454 220877299 : if (NONDEBUG_INSN_P (curr_insn))
7455 : {
7456 118841704 : int regno;
7457 :
7458 : /* Invalidate invariants with changed regs. */
7459 118841704 : curr_id = lra_get_insn_recog_data (curr_insn);
7460 304877010 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7461 186035306 : if (reg->type != OP_IN)
7462 : {
7463 79485970 : bitmap_set_bit (&invalid_invariant_regs, reg->regno);
7464 158971940 : bitmap_set_bit (&invalid_invariant_regs,
7465 79485970 : ORIGINAL_REGNO (regno_reg_rtx[reg->regno]));
7466 : }
7467 118841704 : curr_static_id = curr_id->insn_static_data;
7468 150124580 : for (reg = curr_static_id->hard_regs; reg != NULL; reg = reg->next)
7469 31282876 : if (reg->type != OP_IN)
7470 22386088 : bitmap_set_bit (&invalid_invariant_regs, reg->regno);
7471 118841704 : if (curr_id->arg_hard_regs != NULL)
7472 19537596 : for (i = 0; (regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7473 13923141 : if (regno >= FIRST_PSEUDO_REGISTER)
7474 805587 : bitmap_set_bit (&invalid_invariant_regs,
7475 : regno - FIRST_PSEUDO_REGISTER);
7476 : }
7477 : /* We reached the start of the current basic block. */
7478 220877291 : if (prev_insn == NULL_RTX || prev_insn == PREV_INSN (head)
7479 429172548 : || BLOCK_FOR_INSN (prev_insn) != curr_bb)
7480 : {
7481 : /* We reached the beginning of the current block -- do
7482 : rest of spliting in the current BB. */
7483 18310157 : to_process = df_get_live_in (curr_bb);
7484 18310157 : if (BLOCK_FOR_INSN (head) != curr_bb)
7485 : {
7486 : /* We are somewhere in the middle of EBB. */
7487 5728107 : get_live_on_other_edges (EDGE_PRED (curr_bb, 0)->src,
7488 : curr_bb, &temp_bitmap);
7489 5728107 : to_process = &temp_bitmap;
7490 : }
7491 18310157 : head_p = true;
7492 192619095 : EXECUTE_IF_SET_IN_BITMAP (to_process, 0, j, bi)
7493 : {
7494 174308945 : if ((int) j >= lra_constraint_new_regno_start)
7495 : break;
7496 111155066 : if (((int) j < FIRST_PSEUDO_REGISTER || reg_renumber[j] >= 0)
7497 109999150 : && usage_insns[j].check == curr_usage_insns_check
7498 282427880 : && (next_usage_insns = usage_insns[j].insns) != NULL_RTX)
7499 : {
7500 108118942 : if (need_for_split_p (potential_reload_hard_regs, j))
7501 : {
7502 378330 : if (lra_dump_file != NULL && head_p)
7503 : {
7504 0 : fprintf (lra_dump_file,
7505 : " ----------------------------------\n");
7506 0 : head_p = false;
7507 : }
7508 378330 : if (split_reg (false, j, bb_note (curr_bb),
7509 : next_usage_insns, NULL))
7510 378330 : change_p = true;
7511 : }
7512 108118942 : usage_insns[j].check = 0;
7513 : }
7514 : }
7515 : }
7516 : }
7517 12582050 : first_call_insn = NULL;
7518 12582050 : return change_p;
7519 : }
7520 :
7521 : /* This value affects EBB forming. If probability of edge from EBB to
7522 : a BB is not greater than the following value, we don't add the BB
7523 : to EBB. */
7524 : #define EBB_PROBABILITY_CUTOFF \
7525 : ((REG_BR_PROB_BASE * param_lra_inheritance_ebb_probability_cutoff) / 100)
7526 :
7527 : /* Current number of inheritance/split iteration. */
7528 : int lra_inheritance_iter;
7529 :
7530 : /* Entry function for inheritance/split pass. */
7531 : void
7532 1540291 : lra_inheritance (void)
7533 : {
7534 1540291 : int i;
7535 1540291 : basic_block bb, start_bb;
7536 1540291 : edge e;
7537 :
7538 1540291 : lra_inheritance_iter++;
7539 1540291 : if (lra_inheritance_iter > LRA_MAX_INHERITANCE_PASSES)
7540 : return;
7541 1537106 : timevar_push (TV_LRA_INHERITANCE);
7542 1537106 : if (lra_dump_file != NULL)
7543 97 : fprintf (lra_dump_file, "\n********** Inheritance #%d: **********\n\n",
7544 : lra_inheritance_iter);
7545 1537106 : curr_usage_insns_check = 0;
7546 1537106 : usage_insns = XNEWVEC (struct usage_insns, lra_constraint_new_regno_start);
7547 227791783 : for (i = 0; i < lra_constraint_new_regno_start; i++)
7548 226254677 : usage_insns[i].check = 0;
7549 1537106 : bitmap_initialize (&check_only_regs, ®_obstack);
7550 1537106 : bitmap_initialize (&invalid_invariant_regs, ®_obstack);
7551 1537106 : bitmap_initialize (&live_regs, ®_obstack);
7552 1537106 : bitmap_initialize (&temp_bitmap, ®_obstack);
7553 1537106 : bitmap_initialize (&ebb_global_regs, ®_obstack);
7554 14119156 : FOR_EACH_BB_FN (bb, cfun)
7555 : {
7556 12582050 : start_bb = bb;
7557 12582050 : if (lra_dump_file != NULL)
7558 347 : fprintf (lra_dump_file, "EBB");
7559 : /* Form a EBB starting with BB. */
7560 12582050 : bitmap_clear (&ebb_global_regs);
7561 12582050 : bitmap_ior_into (&ebb_global_regs, df_get_live_in (bb));
7562 18309923 : for (;;)
7563 : {
7564 18309923 : if (lra_dump_file != NULL)
7565 477 : fprintf (lra_dump_file, " %d", bb->index);
7566 18309923 : if (bb->next_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
7567 16772817 : || LABEL_P (BB_HEAD (bb->next_bb)))
7568 : break;
7569 8120548 : e = find_fallthru_edge (bb->succs);
7570 8120548 : if (! e)
7571 : break;
7572 8120548 : if (e->probability.initialized_p ()
7573 8120548 : && e->probability.to_reg_br_prob_base () < EBB_PROBABILITY_CUTOFF)
7574 : break;
7575 : bb = bb->next_bb;
7576 : }
7577 12582050 : bitmap_ior_into (&ebb_global_regs, df_get_live_out (bb));
7578 12582050 : if (lra_dump_file != NULL)
7579 347 : fprintf (lra_dump_file, "\n");
7580 12582050 : if (inherit_in_ebb (BB_HEAD (start_bb), BB_END (bb)))
7581 : /* Remember that the EBB head and tail can change in
7582 : inherit_in_ebb. */
7583 737584 : update_ebb_live_info (BB_HEAD (start_bb), BB_END (bb));
7584 : }
7585 1537106 : bitmap_release (&ebb_global_regs);
7586 1537106 : bitmap_release (&temp_bitmap);
7587 1537106 : bitmap_release (&live_regs);
7588 1537106 : bitmap_release (&invalid_invariant_regs);
7589 1537106 : bitmap_release (&check_only_regs);
7590 1537106 : free (usage_insns);
7591 1537106 : lra_dump_insns_if_possible ("func after inheritance");
7592 1537106 : timevar_pop (TV_LRA_INHERITANCE);
7593 : }
7594 :
7595 :
7596 :
7597 : /* This page contains code to undo failed inheritance/split
7598 : transformations. */
7599 :
7600 : /* Current number of iteration undoing inheritance/split. */
7601 : int lra_undo_inheritance_iter;
7602 :
7603 : /* Fix BB live info LIVE after removing pseudos created on pass doing
7604 : inheritance/split which are REMOVED_PSEUDOS. */
7605 : static void
7606 36619846 : fix_bb_live_info (bitmap live, bitmap removed_pseudos)
7607 : {
7608 36619846 : unsigned int regno;
7609 36619846 : bitmap_iterator bi;
7610 :
7611 209410110 : EXECUTE_IF_SET_IN_BITMAP (removed_pseudos, 0, regno, bi)
7612 172790264 : if (bitmap_clear_bit (live, regno)
7613 172790264 : && REG_P (lra_reg_info[regno].restore_rtx))
7614 1202498 : bitmap_set_bit (live, REGNO (lra_reg_info[regno].restore_rtx));
7615 36619846 : }
7616 :
7617 : /* Return regno of the (subreg of) REG. Otherwise, return a negative
7618 : number. */
7619 : static int
7620 66929291 : get_regno (rtx reg)
7621 : {
7622 1089296 : if (GET_CODE (reg) == SUBREG)
7623 1036004 : reg = SUBREG_REG (reg);
7624 66929291 : if (REG_P (reg))
7625 43323877 : return REGNO (reg);
7626 : return -1;
7627 : }
7628 :
7629 : /* Delete a move INSN with destination reg DREGNO and a previous
7630 : clobber insn with the same regno. The inheritance/split code can
7631 : generate moves with preceding clobber and when we delete such moves
7632 : we should delete the clobber insn too to keep the correct life
7633 : info. */
7634 : static void
7635 745281 : delete_move_and_clobber (rtx_insn *insn, int dregno)
7636 : {
7637 745281 : rtx_insn *prev_insn = PREV_INSN (insn);
7638 :
7639 745281 : lra_set_insn_deleted (insn);
7640 745281 : lra_assert (dregno >= 0);
7641 745281 : if (prev_insn != NULL && NONDEBUG_INSN_P (prev_insn)
7642 317912 : && GET_CODE (PATTERN (prev_insn)) == CLOBBER
7643 745639 : && dregno == get_regno (XEXP (PATTERN (prev_insn), 0)))
7644 0 : lra_set_insn_deleted (prev_insn);
7645 745281 : }
7646 :
7647 : /* Remove inheritance/split pseudos which are in REMOVE_PSEUDOS and
7648 : return true if we did any change. The undo transformations for
7649 : inheritance looks like
7650 : i <- i2
7651 : p <- i => p <- i2
7652 : or removing
7653 : p <- i, i <- p, and i <- i3
7654 : where p is original pseudo from which inheritance pseudo i was
7655 : created, i and i3 are removed inheritance pseudos, i2 is another
7656 : not removed inheritance pseudo. All split pseudos or other
7657 : occurrences of removed inheritance pseudos are changed on the
7658 : corresponding original pseudos.
7659 :
7660 : The function also schedules insns changed and created during
7661 : inheritance/split pass for processing by the subsequent constraint
7662 : pass. */
7663 : static bool
7664 1537106 : remove_inheritance_pseudos (bitmap remove_pseudos)
7665 : {
7666 1537106 : basic_block bb;
7667 1537106 : int regno, sregno, prev_sregno, dregno;
7668 1537106 : rtx restore_rtx;
7669 1537106 : rtx set, prev_set;
7670 1537106 : rtx_insn *prev_insn;
7671 1537106 : bool change_p, done_p;
7672 :
7673 1537106 : change_p = ! bitmap_empty_p (remove_pseudos);
7674 : /* We cannot finish the function right away if CHANGE_P is true
7675 : because we need to marks insns affected by previous
7676 : inheritance/split pass for processing by the subsequent
7677 : constraint pass. */
7678 19847029 : FOR_EACH_BB_FN (bb, cfun)
7679 : {
7680 18309923 : fix_bb_live_info (df_get_live_in (bb), remove_pseudos);
7681 18309923 : fix_bb_live_info (df_get_live_out (bb), remove_pseudos);
7682 241691816 : FOR_BB_INSNS_REVERSE (bb, curr_insn)
7683 : {
7684 223381893 : if (! INSN_P (curr_insn))
7685 35552689 : continue;
7686 187829204 : done_p = false;
7687 187829204 : sregno = dregno = -1;
7688 45888526 : if (change_p && NONDEBUG_INSN_P (curr_insn)
7689 219450694 : && (set = single_set (curr_insn)) != NULL_RTX)
7690 : {
7691 30619137 : dregno = get_regno (SET_DEST (set));
7692 61238274 : sregno = get_regno (SET_SRC (set));
7693 : }
7694 :
7695 187829204 : if (sregno >= 0 && dregno >= 0)
7696 : {
7697 10771410 : if (bitmap_bit_p (remove_pseudos, dregno)
7698 10771410 : && ! REG_P (lra_reg_info[dregno].restore_rtx))
7699 : {
7700 : /* invariant inheritance pseudo <- original pseudo */
7701 6659 : if (lra_dump_file != NULL)
7702 : {
7703 0 : fprintf (lra_dump_file, " Removing invariant inheritance:\n");
7704 0 : dump_insn_slim (lra_dump_file, curr_insn);
7705 0 : fprintf (lra_dump_file, "\n");
7706 : }
7707 6659 : delete_move_and_clobber (curr_insn, dregno);
7708 6659 : done_p = true;
7709 : }
7710 10764751 : else if (bitmap_bit_p (remove_pseudos, sregno)
7711 10764751 : && ! REG_P (lra_reg_info[sregno].restore_rtx))
7712 : {
7713 : /* reload pseudo <- invariant inheritance pseudo */
7714 6659 : start_sequence ();
7715 : /* We cannot just change the source. It might be
7716 : an insn different from the move. */
7717 6659 : emit_insn (lra_reg_info[sregno].restore_rtx);
7718 6659 : rtx_insn *new_insns = end_sequence ();
7719 6659 : lra_assert (single_set (new_insns) != NULL
7720 : && SET_DEST (set) == SET_DEST (single_set (new_insns)));
7721 6659 : lra_process_new_insns (curr_insn, NULL, new_insns,
7722 : "Changing reload<-invariant inheritance");
7723 6659 : delete_move_and_clobber (curr_insn, dregno);
7724 6659 : done_p = true;
7725 : }
7726 10758092 : else if ((bitmap_bit_p (remove_pseudos, sregno)
7727 1204670 : && (get_regno (lra_reg_info[sregno].restore_rtx) == dregno
7728 566985 : || (bitmap_bit_p (remove_pseudos, dregno)
7729 186372 : && get_regno (lra_reg_info[sregno].restore_rtx) >= 0
7730 186372 : && (get_regno (lra_reg_info[sregno].restore_rtx)
7731 186372 : == get_regno (lra_reg_info[dregno].restore_rtx)))))
7732 11231891 : || (bitmap_bit_p (remove_pseudos, dregno)
7733 647312 : && get_regno (lra_reg_info[dregno].restore_rtx) == sregno))
7734 : /* One of the following cases:
7735 : original <- removed inheritance pseudo
7736 : removed inherit pseudo <- another removed inherit pseudo
7737 : removed inherit pseudo <- original pseudo
7738 : Or
7739 : removed_split_pseudo <- original_reg
7740 : original_reg <- removed_split_pseudo */
7741 : {
7742 176516 : if (lra_dump_file != NULL)
7743 : {
7744 0 : fprintf (lra_dump_file, " Removing %s:\n",
7745 0 : bitmap_bit_p (&lra_split_regs, sregno)
7746 0 : || bitmap_bit_p (&lra_split_regs, dregno)
7747 : ? "split" : "inheritance");
7748 0 : dump_insn_slim (lra_dump_file, curr_insn);
7749 : }
7750 176516 : delete_move_and_clobber (curr_insn, dregno);
7751 176516 : done_p = true;
7752 : }
7753 10581576 : else if (bitmap_bit_p (remove_pseudos, sregno)
7754 10581576 : && bitmap_bit_p (&lra_inheritance_pseudos, sregno))
7755 : {
7756 : /* Search the following pattern:
7757 : inherit_or_split_pseudo1 <- inherit_or_split_pseudo2
7758 : original_pseudo <- inherit_or_split_pseudo1
7759 : where the 2nd insn is the current insn and
7760 : inherit_or_split_pseudo2 is not removed. If it is found,
7761 : change the current insn onto:
7762 : original_pseudo <- inherit_or_split_pseudo2. */
7763 722859 : for (prev_insn = PREV_INSN (curr_insn);
7764 722859 : prev_insn != NULL_RTX && ! NONDEBUG_INSN_P (prev_insn);
7765 249060 : prev_insn = PREV_INSN (prev_insn))
7766 : ;
7767 473799 : if (prev_insn != NULL_RTX && BLOCK_FOR_INSN (prev_insn) == bb
7768 461172 : && (prev_set = single_set (prev_insn)) != NULL_RTX
7769 : /* There should be no subregs in insn we are
7770 : searching because only the original reg might
7771 : be in subreg when we changed the mode of
7772 : load/store for splitting. */
7773 455256 : && REG_P (SET_DEST (prev_set))
7774 348406 : && REG_P (SET_SRC (prev_set))
7775 267946 : && (int) REGNO (SET_DEST (prev_set)) == sregno
7776 182589 : && ((prev_sregno = REGNO (SET_SRC (prev_set)))
7777 : >= FIRST_PSEUDO_REGISTER)
7778 182589 : && (lra_reg_info[prev_sregno].restore_rtx == NULL_RTX
7779 142067 : ||
7780 : /* As we consider chain of inheritance or
7781 : splitting described in above comment we should
7782 : check that sregno and prev_sregno were
7783 : inheritance/split pseudos created from the
7784 : same original regno. */
7785 284134 : (get_regno (lra_reg_info[sregno].restore_rtx) >= 0
7786 284134 : && (get_regno (lra_reg_info[sregno].restore_rtx)
7787 284134 : == get_regno (lra_reg_info[prev_sregno].restore_rtx))))
7788 656388 : && ! bitmap_bit_p (remove_pseudos, prev_sregno))
7789 : {
7790 100945 : int restore_regno = get_regno (lra_reg_info[sregno].restore_rtx);
7791 100945 : if (restore_regno < 0)
7792 0 : restore_regno = prev_sregno;
7793 100945 : lra_assert (GET_MODE (SET_SRC (prev_set))
7794 : == GET_MODE (regno_reg_rtx[restore_regno]));
7795 : /* Although we have a single set, the insn can
7796 : contain more one sregno register occurrence
7797 : as a source. Change all occurrences. */
7798 100945 : lra_substitute_pseudo_within_insn (curr_insn, sregno,
7799 : regno_reg_rtx[restore_regno],
7800 : false);
7801 : /* As we are finishing with processing the insn
7802 : here, check the destination too as it might
7803 : inheritance pseudo for another pseudo. */
7804 100945 : if (bitmap_bit_p (remove_pseudos, dregno)
7805 0 : && bitmap_bit_p (&lra_inheritance_pseudos, dregno)
7806 100945 : && (restore_rtx
7807 0 : = lra_reg_info[dregno].restore_rtx) != NULL_RTX)
7808 : {
7809 0 : if (GET_CODE (SET_DEST (set)) == SUBREG)
7810 0 : SUBREG_REG (SET_DEST (set)) = restore_rtx;
7811 : else
7812 0 : SET_DEST (set) = restore_rtx;
7813 : }
7814 100945 : lra_push_insn_and_update_insn_regno_info (curr_insn);
7815 100945 : lra_set_used_insn_alternative_by_uid
7816 100945 : (INSN_UID (curr_insn), LRA_UNKNOWN_ALT);
7817 100945 : done_p = true;
7818 100945 : if (lra_dump_file != NULL)
7819 : {
7820 0 : fprintf (lra_dump_file, " Change reload insn:\n");
7821 0 : dump_insn_slim (lra_dump_file, curr_insn);
7822 : }
7823 : }
7824 : }
7825 : }
7826 189834 : if (! done_p)
7827 : {
7828 187538425 : struct lra_insn_reg *reg;
7829 187538425 : bool restored_regs_p = false;
7830 187538425 : bool kept_regs_p = false;
7831 :
7832 187538425 : curr_id = lra_get_insn_recog_data (curr_insn);
7833 391634519 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7834 : {
7835 204096094 : regno = reg->regno;
7836 204096094 : restore_rtx = lra_reg_info[regno].restore_rtx;
7837 204096094 : if (restore_rtx != NULL_RTX)
7838 : {
7839 5860993 : if (change_p && bitmap_bit_p (remove_pseudos, regno))
7840 : {
7841 824465 : lra_substitute_pseudo_within_insn
7842 824465 : (curr_insn, regno, restore_rtx, false);
7843 824465 : restored_regs_p = true;
7844 : }
7845 : else
7846 : kept_regs_p = true;
7847 : }
7848 : }
7849 187538425 : if (NONDEBUG_INSN_P (curr_insn) && kept_regs_p)
7850 : {
7851 : /* The instruction has changed since the previous
7852 : constraints pass. */
7853 4421694 : lra_push_insn_and_update_insn_regno_info (curr_insn);
7854 4421694 : lra_set_used_insn_alternative_by_uid
7855 4421694 : (INSN_UID (curr_insn), LRA_UNKNOWN_ALT);
7856 : }
7857 183116731 : else if (restored_regs_p)
7858 : /* The instruction has been restored to the form that
7859 : it had during the previous constraints pass. */
7860 664325 : lra_update_insn_regno_info (curr_insn);
7861 5086019 : if (restored_regs_p && lra_dump_file != NULL)
7862 : {
7863 0 : fprintf (lra_dump_file, " Insn after restoring regs:\n");
7864 0 : dump_insn_slim (lra_dump_file, curr_insn);
7865 : }
7866 : }
7867 : }
7868 : }
7869 1537106 : return change_p;
7870 : }
7871 :
7872 : /* If optional reload pseudos failed to get a hard register or was not
7873 : inherited, it is better to remove optional reloads. We do this
7874 : transformation after undoing inheritance to figure out necessity to
7875 : remove optional reloads easier. Return true if we do any
7876 : change. */
7877 : static bool
7878 1537106 : undo_optional_reloads (void)
7879 : {
7880 1537106 : bool change_p, keep_p;
7881 1537106 : unsigned int regno, uid;
7882 1537106 : bitmap_iterator bi, bi2;
7883 1537106 : rtx_insn *insn;
7884 1537106 : rtx set, src, dest;
7885 1537106 : auto_bitmap removed_optional_reload_pseudos (®_obstack);
7886 :
7887 1537106 : bitmap_copy (removed_optional_reload_pseudos, &lra_optional_reload_pseudos);
7888 2534502 : EXECUTE_IF_SET_IN_BITMAP (&lra_optional_reload_pseudos, 0, regno, bi)
7889 : {
7890 997396 : keep_p = false;
7891 : /* Keep optional reloads from previous subpasses. */
7892 997396 : if (lra_reg_info[regno].restore_rtx == NULL_RTX
7893 : /* If the original pseudo changed its allocation, just
7894 : removing the optional pseudo is dangerous as the original
7895 : pseudo will have longer live range. */
7896 997396 : || reg_renumber[REGNO (lra_reg_info[regno].restore_rtx)] >= 0)
7897 : keep_p = true;
7898 618360 : else if (reg_renumber[regno] >= 0)
7899 1776502 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi2)
7900 : {
7901 1293499 : insn = lra_insn_recog_data[uid]->insn;
7902 1293499 : if ((set = single_set (insn)) == NULL_RTX)
7903 6671 : continue;
7904 1286828 : src = SET_SRC (set);
7905 1286828 : dest = SET_DEST (set);
7906 1286828 : if ((! REG_P (src) && ! SUBREG_P (src))
7907 689694 : || (! REG_P (dest) && ! SUBREG_P (dest)))
7908 597162 : continue;
7909 689666 : if (get_regno (dest) == (int) regno
7910 : /* Ignore insn for optional reloads itself. */
7911 1160736 : && (get_regno (lra_reg_info[regno].restore_rtx)
7912 580368 : != get_regno (src))
7913 : /* Check only inheritance on last inheritance pass. */
7914 120830 : && get_regno (src) >= new_regno_start
7915 : /* Check that the optional reload was inherited. */
7916 810496 : && bitmap_bit_p (&lra_inheritance_pseudos, get_regno (src)))
7917 : {
7918 : keep_p = true;
7919 : break;
7920 : }
7921 : }
7922 982869 : if (keep_p)
7923 : {
7924 499866 : bitmap_clear_bit (removed_optional_reload_pseudos, regno);
7925 499866 : if (lra_dump_file != NULL)
7926 3 : fprintf (lra_dump_file, "Keep optional reload reg %d\n", regno);
7927 : }
7928 : }
7929 1537106 : change_p = ! bitmap_empty_p (removed_optional_reload_pseudos);
7930 1537106 : auto_bitmap insn_bitmap (®_obstack);
7931 2034636 : EXECUTE_IF_SET_IN_BITMAP (removed_optional_reload_pseudos, 0, regno, bi)
7932 : {
7933 497530 : if (lra_dump_file != NULL)
7934 2 : fprintf (lra_dump_file, "Remove optional reload reg %d\n", regno);
7935 497530 : bitmap_copy (insn_bitmap, &lra_reg_info[regno].insn_bitmap);
7936 1580823 : EXECUTE_IF_SET_IN_BITMAP (insn_bitmap, 0, uid, bi2)
7937 : {
7938 : /* We may have already removed a clobber. */
7939 1083293 : if (!lra_insn_recog_data[uid])
7940 0 : continue;
7941 1083293 : insn = lra_insn_recog_data[uid]->insn;
7942 1083293 : if ((set = single_set (insn)) != NULL_RTX)
7943 : {
7944 1077352 : src = SET_SRC (set);
7945 1077352 : dest = SET_DEST (set);
7946 491572 : if ((REG_P (src) || SUBREG_P (src))
7947 585792 : && (REG_P (dest) || SUBREG_P (dest))
7948 1663115 : && ((get_regno (src) == (int) regno
7949 224976 : && (get_regno (lra_reg_info[regno].restore_rtx)
7950 112488 : == get_regno (dest)))
7951 503533 : || (get_regno (dest) == (int) regno
7952 473275 : && (get_regno (lra_reg_info[regno].restore_rtx)
7953 473275 : == get_regno (src)))))
7954 : {
7955 555447 : if (lra_dump_file != NULL)
7956 : {
7957 0 : fprintf (lra_dump_file, " Deleting move %u\n",
7958 0 : INSN_UID (insn));
7959 0 : dump_insn_slim (lra_dump_file, insn);
7960 : }
7961 1110894 : delete_move_and_clobber (insn, get_regno (dest));
7962 555447 : continue;
7963 : }
7964 : /* We should not worry about generation memory-memory
7965 : moves here as if the corresponding inheritance did
7966 : not work (inheritance pseudo did not get a hard reg),
7967 : we remove the inheritance pseudo and the optional
7968 : reload. */
7969 : }
7970 527846 : rtx pat = PATTERN (insn);
7971 0 : if (GET_CODE (pat) == CLOBBER && REG_P (SET_DEST (pat))
7972 527846 : && get_regno (SET_DEST (pat)) == (int) regno)
7973 : /* Refuse to remap clobbers to preexisting pseudos. */
7974 0 : gcc_unreachable ();
7975 527846 : lra_substitute_pseudo_within_insn
7976 527846 : (insn, regno, lra_reg_info[regno].restore_rtx, false);
7977 527846 : lra_update_insn_regno_info (insn);
7978 527846 : if (lra_dump_file != NULL)
7979 : {
7980 4 : fprintf (lra_dump_file,
7981 : " Restoring original insn:\n");
7982 4 : dump_insn_slim (lra_dump_file, insn);
7983 : }
7984 : }
7985 : }
7986 : /* Clear restore_regnos. */
7987 2534502 : EXECUTE_IF_SET_IN_BITMAP (&lra_optional_reload_pseudos, 0, regno, bi)
7988 997396 : lra_reg_info[regno].restore_rtx = NULL_RTX;
7989 1537106 : return change_p;
7990 1537106 : }
7991 :
7992 : /* Entry function for undoing inheritance/split transformation. Return true
7993 : if we did any RTL change in this pass. */
7994 : bool
7995 1540291 : lra_undo_inheritance (void)
7996 : {
7997 1540291 : unsigned int regno;
7998 1540291 : int hard_regno;
7999 1540291 : int n_all_inherit, n_inherit, n_all_split, n_split;
8000 1540291 : rtx restore_rtx;
8001 1540291 : bitmap_iterator bi;
8002 1540291 : bool change_p;
8003 :
8004 1540291 : lra_undo_inheritance_iter++;
8005 1540291 : if (lra_undo_inheritance_iter > LRA_MAX_INHERITANCE_PASSES)
8006 : return false;
8007 1537106 : if (lra_dump_file != NULL)
8008 97 : fprintf (lra_dump_file,
8009 : "\n********** Undoing inheritance #%d: **********\n\n",
8010 : lra_undo_inheritance_iter);
8011 1537106 : auto_bitmap remove_pseudos (®_obstack);
8012 1537106 : n_inherit = n_all_inherit = 0;
8013 3388565 : EXECUTE_IF_SET_IN_BITMAP (&lra_inheritance_pseudos, 0, regno, bi)
8014 1851459 : if (lra_reg_info[regno].restore_rtx != NULL_RTX)
8015 : {
8016 1170244 : n_all_inherit++;
8017 1170244 : if (reg_renumber[regno] < 0
8018 : /* If the original pseudo changed its allocation, just
8019 : removing inheritance is dangerous as for changing
8020 : allocation we used shorter live-ranges. */
8021 1170244 : && (! REG_P (lra_reg_info[regno].restore_rtx)
8022 417987 : || reg_renumber[REGNO (lra_reg_info[regno].restore_rtx)] < 0))
8023 424646 : bitmap_set_bit (remove_pseudos, regno);
8024 : else
8025 745598 : n_inherit++;
8026 : }
8027 1537106 : if (lra_dump_file != NULL && n_all_inherit != 0)
8028 2 : fprintf (lra_dump_file, "Inherit %d out of %d (%.2f%%)\n",
8029 : n_inherit, n_all_inherit,
8030 2 : (double) n_inherit / n_all_inherit * 100);
8031 1537106 : n_split = n_all_split = 0;
8032 2484506 : EXECUTE_IF_SET_IN_BITMAP (&lra_split_regs, 0, regno, bi)
8033 947400 : if ((restore_rtx = lra_reg_info[regno].restore_rtx) != NULL_RTX)
8034 : {
8035 650738 : int restore_regno = REGNO (restore_rtx);
8036 :
8037 650738 : n_all_split++;
8038 1301394 : hard_regno = (restore_regno >= FIRST_PSEUDO_REGISTER
8039 650738 : ? reg_renumber[restore_regno] : restore_regno);
8040 650738 : if (hard_regno < 0 || reg_renumber[regno] == hard_regno)
8041 2227 : bitmap_set_bit (remove_pseudos, regno);
8042 : else
8043 : {
8044 648511 : n_split++;
8045 648511 : if (lra_dump_file != NULL)
8046 0 : fprintf (lra_dump_file, " Keep split r%d (orig=r%d)\n",
8047 : regno, restore_regno);
8048 : }
8049 : }
8050 1537106 : if (lra_dump_file != NULL && n_all_split != 0)
8051 0 : fprintf (lra_dump_file, "Split %d out of %d (%.2f%%)\n",
8052 : n_split, n_all_split,
8053 0 : (double) n_split / n_all_split * 100);
8054 1537106 : change_p = remove_inheritance_pseudos (remove_pseudos);
8055 : /* Clear restore_regnos. */
8056 3388565 : EXECUTE_IF_SET_IN_BITMAP (&lra_inheritance_pseudos, 0, regno, bi)
8057 1851459 : lra_reg_info[regno].restore_rtx = NULL_RTX;
8058 2484506 : EXECUTE_IF_SET_IN_BITMAP (&lra_split_regs, 0, regno, bi)
8059 947400 : lra_reg_info[regno].restore_rtx = NULL_RTX;
8060 1537106 : change_p = undo_optional_reloads () || change_p;
8061 : if (change_p)
8062 109476 : lra_dump_insns_if_possible ("changed func after undoing inheritance");
8063 1537106 : return change_p;
8064 1537106 : }
|
