Line data Source code
1 : /* Code for RTL transformations to satisfy insn constraints.
2 : Copyright (C) 2010-2026 Free Software Foundation, Inc.
3 : Contributed by Vladimir Makarov <vmakarov@redhat.com>.
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 :
22 : /* This file contains code for 3 passes: constraint pass,
23 : inheritance/split pass, and pass for undoing failed inheritance and
24 : split.
25 :
26 : The major goal of constraint pass is to transform RTL to satisfy
27 : insn and address constraints by:
28 : o choosing insn alternatives;
29 : o generating *reload insns* (or reloads in brief) and *reload
30 : pseudos* which will get necessary hard registers later;
31 : o substituting pseudos with equivalent values and removing the
32 : instructions that initialized those pseudos.
33 :
34 : The constraint pass has biggest and most complicated code in LRA.
35 : There are a lot of important details like:
36 : o reuse of input reload pseudos to simplify reload pseudo
37 : allocations;
38 : o some heuristics to choose insn alternative to improve the
39 : inheritance;
40 : o early clobbers etc.
41 :
42 : The pass is mimicking former reload pass in alternative choosing
43 : because the reload pass is oriented to current machine description
44 : model. It might be changed if the machine description model is
45 : changed.
46 :
47 : There is special code for preventing all LRA and this pass cycling
48 : in case of bugs.
49 :
50 : On the first iteration of the pass we process every instruction and
51 : choose an alternative for each one. On subsequent iterations we try
52 : to avoid reprocessing instructions if we can be sure that the old
53 : choice is still valid.
54 :
55 : The inheritance/spilt pass is to transform code to achieve
56 : ineheritance and live range splitting. It is done on backward
57 : traversal of EBBs.
58 :
59 : The inheritance optimization goal is to reuse values in hard
60 : registers. There is analogous optimization in old reload pass. The
61 : inheritance is achieved by following transformation:
62 :
63 : reload_p1 <- p reload_p1 <- p
64 : ... new_p <- reload_p1
65 : ... => ...
66 : reload_p2 <- p reload_p2 <- new_p
67 :
68 : where p is spilled and not changed between the insns. Reload_p1 is
69 : also called *original pseudo* and new_p is called *inheritance
70 : pseudo*.
71 :
72 : The subsequent assignment pass will try to assign the same (or
73 : another if it is not possible) hard register to new_p as to
74 : reload_p1 or reload_p2.
75 :
76 : If the assignment pass fails to assign a hard register to new_p,
77 : this file will undo the inheritance and restore the original code.
78 : This is because implementing the above sequence with a spilled
79 : new_p would make the code much worse. The inheritance is done in
80 : EBB scope. The above is just a simplified example to get an idea
81 : of the inheritance as the inheritance is also done for non-reload
82 : insns.
83 :
84 : Splitting (transformation) is also done in EBB scope on the same
85 : pass as the inheritance:
86 :
87 : r <- ... or ... <- r r <- ... or ... <- r
88 : ... s <- r (new insn -- save)
89 : ... =>
90 : ... r <- s (new insn -- restore)
91 : ... <- r ... <- r
92 :
93 : The *split pseudo* s is assigned to the hard register of the
94 : original pseudo or hard register r.
95 :
96 : Splitting is done:
97 : o In EBBs with high register pressure for global pseudos (living
98 : in at least 2 BBs) and assigned to hard registers when there
99 : are more one reloads needing the hard registers;
100 : o for pseudos needing save/restore code around calls.
101 :
102 : If the split pseudo still has the same hard register as the
103 : original pseudo after the subsequent assignment pass or the
104 : original pseudo was split, the opposite transformation is done on
105 : the same pass for undoing inheritance. */
106 :
107 : #undef REG_OK_STRICT
108 :
109 : #include "config.h"
110 : #include "system.h"
111 : #include "coretypes.h"
112 : #include "backend.h"
113 : #include "hooks.h"
114 : #include "target.h"
115 : #include "rtl.h"
116 : #include "tree.h"
117 : #include "stmt.h"
118 : #include "predict.h"
119 : #include "df.h"
120 : #include "memmodel.h"
121 : #include "tm_p.h"
122 : #include "expmed.h"
123 : #include "optabs.h"
124 : #include "regs.h"
125 : #include "ira.h"
126 : #include "ira-int.h"
127 : #include "recog.h"
128 : #include "output.h"
129 : #include "addresses.h"
130 : #include "expr.h"
131 : #include "cfgrtl.h"
132 : #include "rtl-error.h"
133 : #include "lra.h"
134 : #include "lra-int.h"
135 : #include "print-rtl.h"
136 : #include "function-abi.h"
137 : #include "rtl-iter.h"
138 : #include "hash-set.h"
139 :
140 : /* Value of LRA_CURR_RELOAD_NUM at the beginning of BB of the current
141 : insn. Remember that LRA_CURR_RELOAD_NUM is the number of emitted
142 : reload insns. */
143 : static int bb_reload_num;
144 :
145 : /* The current insn being processed and corresponding its single set
146 : (NULL otherwise), its data (basic block, the insn data, the insn
147 : static data, and the mode of each operand). */
148 : static rtx_insn *curr_insn;
149 : static rtx curr_insn_set;
150 : static basic_block curr_bb;
151 : static lra_insn_recog_data_t curr_id;
152 : static struct lra_static_insn_data *curr_static_id;
153 : static machine_mode curr_operand_mode[MAX_RECOG_OPERANDS];
154 : /* Mode of the register substituted by its equivalence with VOIDmode
155 : (e.g. constant) and whose subreg is given operand of the current
156 : insn. VOIDmode in all other cases. */
157 : static machine_mode original_subreg_reg_mode[MAX_RECOG_OPERANDS];
158 : /* The first call insn after curr_insn within the EBB during inherit_in_ebb
159 : or NULL outside of that function. */
160 : static rtx_insn *first_call_insn;
161 :
162 :
163 :
164 : /* Start numbers for new registers and insns at the current constraints
165 : pass start. */
166 : static int new_regno_start;
167 : static int new_insn_uid_start;
168 :
169 : /* If LOC is nonnull, strip any outer subreg from it. */
170 : static inline rtx *
171 227685880 : strip_subreg (rtx *loc)
172 : {
173 100760696 : return loc && GET_CODE (*loc) == SUBREG ? &SUBREG_REG (*loc) : loc;
174 : }
175 :
176 : /* Return hard regno of REGNO or if it is was not assigned to a hard
177 : register, use a hard register from its allocno class. */
178 : static int
179 74567 : get_try_hard_regno (int regno)
180 : {
181 74567 : int hard_regno;
182 74567 : enum reg_class rclass;
183 :
184 74567 : if ((hard_regno = regno) >= FIRST_PSEUDO_REGISTER)
185 74567 : hard_regno = lra_get_regno_hard_regno (regno);
186 74567 : if (hard_regno >= 0)
187 : return hard_regno;
188 41084 : rclass = lra_get_allocno_class (regno);
189 41084 : if (rclass == NO_REGS)
190 : return -1;
191 39696 : return ira_class_hard_regs[rclass][0];
192 : }
193 :
194 : /* Return the hard regno of X after removing its subreg. If X is not a
195 : register or a subreg of a register, return -1. If X is a pseudo, use its
196 : assignment. If X is a hard regno, return the final hard regno which will be
197 : after elimination. */
198 : static int
199 287976111 : get_hard_regno (rtx x)
200 : {
201 287976111 : rtx reg;
202 287976111 : int hard_regno;
203 :
204 287976111 : reg = x;
205 287976111 : if (SUBREG_P (x))
206 5156460 : reg = SUBREG_REG (x);
207 287976111 : if (! REG_P (reg))
208 : return -1;
209 198980257 : int regno = REGNO (reg);
210 198980257 : if (HARD_REGISTER_NUM_P (regno))
211 34973776 : hard_regno = lra_get_elimination_hard_regno (regno);
212 : else
213 164006481 : hard_regno = lra_get_regno_hard_regno (regno);
214 198980257 : if (hard_regno < 0)
215 : return -1;
216 181138246 : if (SUBREG_P (x))
217 4457592 : hard_regno += subreg_regno_offset (hard_regno, GET_MODE (reg),
218 4457592 : SUBREG_BYTE (x), GET_MODE (x));
219 : return hard_regno;
220 : }
221 :
222 : /* If REGNO is a hard register or has been allocated a hard register,
223 : return the class of that register. If REGNO is a reload pseudo
224 : created by the current constraints pass, return its allocno class.
225 : Return NO_REGS otherwise. */
226 : static enum reg_class
227 511980088 : get_reg_class (int regno)
228 : {
229 511980088 : int hard_regno;
230 :
231 511980088 : if (HARD_REGISTER_NUM_P (regno))
232 64829744 : hard_regno = lra_get_elimination_hard_regno (regno);
233 : else
234 447150344 : hard_regno = lra_get_regno_hard_regno (regno);
235 511980088 : if (hard_regno >= 0)
236 320369028 : return REGNO_REG_CLASS (hard_regno);
237 191611060 : if (regno >= new_regno_start)
238 62162229 : return lra_get_allocno_class (regno);
239 : return NO_REGS;
240 : }
241 :
242 : /* Return true if REG_CLASS has enough allocatable hard regs to keep value of
243 : REG_MODE. */
244 : static bool
245 18663660 : enough_allocatable_hard_regs_p (enum reg_class reg_class,
246 : enum machine_mode reg_mode)
247 : {
248 18663660 : int i, j, hard_regno, class_size, nregs;
249 :
250 37327320 : if (hard_reg_set_subset_p (reg_class_contents[reg_class], lra_no_alloc_regs))
251 : return false;
252 6323745 : class_size = ira_class_hard_regs_num[reg_class];
253 6323745 : for (i = 0; i < class_size; i++)
254 : {
255 6323745 : hard_regno = ira_class_hard_regs[reg_class][i];
256 6323745 : nregs = hard_regno_nregs (hard_regno, reg_mode);
257 6323745 : if (nregs == 1)
258 : return true;
259 253431 : for (j = 0; j < nregs; j++)
260 168954 : if (TEST_HARD_REG_BIT (lra_no_alloc_regs, hard_regno + j)
261 168954 : || ! TEST_HARD_REG_BIT (reg_class_contents[reg_class],
262 : hard_regno + j))
263 : break;
264 84477 : if (j >= nregs)
265 : return true;
266 : }
267 : return false;
268 : }
269 :
270 : /* True if C is a non-empty register class that has too few registers
271 : to be safely used as a reload target class. */
272 : #define SMALL_REGISTER_CLASS_P(C) \
273 : (ira_class_hard_regs_num [(C)] == 1 \
274 : || (ira_class_hard_regs_num [(C)] >= 1 \
275 : && targetm.class_likely_spilled_p (C)))
276 :
277 : /* Return true if REG satisfies (or will satisfy) reg class constraint
278 : CL. Use elimination first if REG is a hard register. If REG is a
279 : reload pseudo created by this constraints pass, assume that it will
280 : be allocated a hard register from its allocno class, but allow that
281 : class to be narrowed to CL if it is currently a superset of CL and
282 : if either:
283 :
284 : - ALLOW_ALL_RELOAD_CLASS_CHANGES_P is true or
285 : - the instruction we're processing is not a reload move.
286 :
287 : If NEW_CLASS is nonnull, set *NEW_CLASS to the new allocno class of
288 : REGNO (reg), or NO_REGS if no change in its class was needed. */
289 : static bool
290 217856926 : in_class_p (rtx reg, enum reg_class cl, enum reg_class *new_class,
291 : bool allow_all_reload_class_changes_p = false)
292 : {
293 217856926 : enum reg_class rclass, common_class;
294 217856926 : machine_mode reg_mode;
295 217856926 : rtx src;
296 217856926 : int regno = REGNO (reg);
297 :
298 217856926 : if (new_class != NULL)
299 112050439 : *new_class = NO_REGS;
300 217856926 : if (regno < FIRST_PSEUDO_REGISTER)
301 : {
302 27528248 : rtx final_reg = reg;
303 27528248 : rtx *final_loc = &final_reg;
304 :
305 27528248 : lra_eliminate_reg_if_possible (final_loc);
306 27528248 : return TEST_HARD_REG_BIT (reg_class_contents[cl], REGNO (*final_loc));
307 : }
308 190328678 : reg_mode = GET_MODE (reg);
309 190328678 : rclass = get_reg_class (regno);
310 190328678 : src = curr_insn_set != NULL ? SET_SRC (curr_insn_set) : NULL;
311 190328678 : if (regno < new_regno_start
312 : /* Do not allow the constraints for reload instructions to
313 : influence the classes of new pseudos. These reloads are
314 : typically moves that have many alternatives, and restricting
315 : reload pseudos for one alternative may lead to situations
316 : where other reload pseudos are no longer allocatable. */
317 190328678 : || (!allow_all_reload_class_changes_p
318 15038664 : && INSN_UID (curr_insn) >= new_insn_uid_start
319 14511588 : && src != NULL
320 14511588 : && ((REG_P (src) || MEM_P (src))
321 1398262 : || (GET_CODE (src) == SUBREG
322 629894 : && (REG_P (SUBREG_REG (src)) || MEM_P (SUBREG_REG (src)))))))
323 : /* When we don't know what class will be used finally for reload
324 : pseudos, we use ALL_REGS. */
325 13743220 : return ((regno >= new_regno_start && rclass == ALL_REGS)
326 185405805 : || (rclass != NO_REGS && ira_class_subset_p[rclass][cl]
327 200147644 : && ! hard_reg_set_subset_p (reg_class_contents[cl],
328 : lra_no_alloc_regs)));
329 : else
330 : {
331 18663660 : common_class = ira_reg_class_subset[rclass][cl];
332 18663660 : if (new_class != NULL)
333 5251777 : *new_class = common_class;
334 18663660 : return (enough_allocatable_hard_regs_p (common_class, reg_mode)
335 : /* Do not permit reload insn operand matching (new_class == NULL
336 : case) if the new class is too small. */
337 18663660 : && (new_class != NULL || common_class == rclass
338 987532 : || !SMALL_REGISTER_CLASS_P (common_class)));
339 : }
340 : }
341 :
342 : /* Return true if REGNO satisfies a memory constraint. */
343 : static bool
344 63534084 : in_mem_p (int regno)
345 : {
346 0 : return get_reg_class (regno) == NO_REGS;
347 : }
348 :
349 : /* Return true if ADDR is a valid memory address for mode MODE in address
350 : space AS, and check that each pseudo has the proper kind of hard
351 : reg. */
352 : static bool
353 35472787 : valid_address_p (machine_mode mode ATTRIBUTE_UNUSED,
354 : rtx addr, addr_space_t as)
355 : {
356 : #ifdef GO_IF_LEGITIMATE_ADDRESS
357 : lra_assert (ADDR_SPACE_GENERIC_P (as));
358 : GO_IF_LEGITIMATE_ADDRESS (mode, addr, win);
359 : return false;
360 :
361 : win:
362 : return true;
363 : #else
364 0 : return targetm.addr_space.legitimate_address_p (mode, addr, 0, as,
365 35472787 : ERROR_MARK);
366 : #endif
367 : }
368 :
369 : namespace {
370 : /* Temporarily eliminates registers in an address (for the lifetime of
371 : the object). */
372 : class address_eliminator {
373 : public:
374 : address_eliminator (struct address_info *ad);
375 : ~address_eliminator ();
376 :
377 : private:
378 : struct address_info *m_ad;
379 : rtx *m_base_loc;
380 : rtx m_base_reg;
381 : rtx *m_index_loc;
382 : rtx m_index_reg;
383 : };
384 : }
385 :
386 74574565 : address_eliminator::address_eliminator (struct address_info *ad)
387 74574565 : : m_ad (ad),
388 74574565 : m_base_loc (strip_subreg (ad->base_term)),
389 74574565 : m_base_reg (NULL_RTX),
390 74574565 : m_index_loc (strip_subreg (ad->index_term)),
391 74574565 : m_index_reg (NULL_RTX)
392 : {
393 74574565 : if (m_base_loc != NULL)
394 : {
395 62135356 : m_base_reg = *m_base_loc;
396 : /* If we have non-legitimate address which is decomposed not in
397 : the way we expected, don't do elimination here. In such case
398 : the address will be reloaded and elimination will be done in
399 : reload insn finally. */
400 62135356 : if (REG_P (m_base_reg))
401 62135356 : lra_eliminate_reg_if_possible (m_base_loc);
402 62135356 : if (m_ad->base_term2 != NULL)
403 0 : *m_ad->base_term2 = *m_ad->base_term;
404 : }
405 74574565 : if (m_index_loc != NULL)
406 : {
407 3665429 : m_index_reg = *m_index_loc;
408 3665429 : if (REG_P (m_index_reg))
409 3665429 : lra_eliminate_reg_if_possible (m_index_loc);
410 : }
411 74574565 : }
412 :
413 74574565 : address_eliminator::~address_eliminator ()
414 : {
415 74574565 : if (m_base_loc && *m_base_loc != m_base_reg)
416 : {
417 43885244 : *m_base_loc = m_base_reg;
418 43885244 : if (m_ad->base_term2 != NULL)
419 0 : *m_ad->base_term2 = *m_ad->base_term;
420 : }
421 74574565 : if (m_index_loc && *m_index_loc != m_index_reg)
422 0 : *m_index_loc = m_index_reg;
423 74574565 : }
424 :
425 : /* Return true if the eliminated form of AD is a legitimate target address.
426 : If OP is a MEM, AD is the address within OP, otherwise OP should be
427 : ignored. CONSTRAINT is one constraint that the operand may need
428 : to meet. */
429 : static bool
430 35450855 : valid_address_p (rtx op, struct address_info *ad,
431 : enum constraint_num constraint)
432 : {
433 35450855 : address_eliminator eliminator (ad);
434 :
435 : /* Allow a memory OP if it matches CONSTRAINT, even if CONSTRAINT is more
436 : forgiving than "m".
437 : Need to extract memory from op for special memory constraint,
438 : i.e. bcst_mem_operand in i386 backend. */
439 35450855 : if (MEM_P (extract_mem_from_operand (op))
440 : && insn_extra_relaxed_memory_constraint (constraint)
441 : && constraint_satisfied_p (op, constraint))
442 : return true;
443 :
444 35450855 : return valid_address_p (ad->mode, *ad->outer, ad->as);
445 35450855 : }
446 :
447 : /* For special_memory_operand, it could be false for MEM_P (op),
448 : i.e. bcst_mem_operand in i386 backend.
449 : Extract and return real memory operand or op. */
450 : rtx
451 620952676 : extract_mem_from_operand (rtx op)
452 : {
453 622593444 : for (rtx x = op;; x = XEXP (x, 0))
454 : {
455 622593444 : if (MEM_P (x))
456 : return x;
457 442433880 : if (GET_RTX_LENGTH (GET_CODE (x)) != 1
458 362684939 : || GET_RTX_FORMAT (GET_CODE (x))[0] != 'e')
459 : break;
460 : }
461 : return op;
462 : }
463 :
464 : /* Return true if the eliminated form of memory reference OP satisfies
465 : extra (special) memory constraint CONSTRAINT. */
466 : static bool
467 36752194 : satisfies_memory_constraint_p (rtx op, enum constraint_num constraint)
468 : {
469 36752194 : struct address_info ad;
470 36752194 : rtx mem = extract_mem_from_operand (op);
471 36752194 : if (!MEM_P (mem))
472 : return false;
473 :
474 35730100 : decompose_mem_address (&ad, mem);
475 35730100 : address_eliminator eliminator (&ad);
476 35730100 : return constraint_satisfied_p (op, constraint);
477 35730100 : }
478 :
479 : /* Return true if the eliminated form of address AD satisfies extra
480 : address constraint CONSTRAINT. */
481 : static bool
482 3393610 : satisfies_address_constraint_p (struct address_info *ad,
483 : enum constraint_num constraint)
484 : {
485 3393610 : address_eliminator eliminator (ad);
486 3393610 : return constraint_satisfied_p (*ad->outer, constraint);
487 3393610 : }
488 :
489 : /* Return true if the eliminated form of address OP satisfies extra
490 : address constraint CONSTRAINT. */
491 : static bool
492 1664161 : satisfies_address_constraint_p (rtx op, enum constraint_num constraint)
493 : {
494 1664161 : struct address_info ad;
495 :
496 1664161 : decompose_lea_address (&ad, &op);
497 1664161 : return satisfies_address_constraint_p (&ad, constraint);
498 : }
499 :
500 : /* Set of equivalences whose original targets have set up pointer flag. */
501 : static hash_set <rtx> *pointer_equiv_set;
502 :
503 : /* Add x to pointer_equiv_set. */
504 : void
505 1906686 : lra_pointer_equiv_set_add (rtx x)
506 : {
507 1906686 : pointer_equiv_set->add (x);
508 1906686 : }
509 :
510 : /* Return true if x is in pointer_equiv_set. */
511 : bool
512 9737923 : lra_pointer_equiv_set_in (rtx x)
513 : {
514 9737923 : return pointer_equiv_set->contains (x);
515 : }
516 :
517 : /* Initiate equivalences for LRA. As we keep original equivalences
518 : before any elimination, we need to make copies otherwise any change
519 : in insns might change the equivalences. */
520 : void
521 1488370 : lra_init_equiv (void)
522 : {
523 1488370 : ira_expand_reg_equiv ();
524 69681344 : for (int i = FIRST_PSEUDO_REGISTER; i < max_reg_num (); i++)
525 : {
526 68192974 : rtx res;
527 :
528 68192974 : if ((res = ira_reg_equiv[i].memory) != NULL_RTX)
529 3010309 : ira_reg_equiv[i].memory = copy_rtx (res);
530 68192974 : if ((res = ira_reg_equiv[i].invariant) != NULL_RTX)
531 874017 : ira_reg_equiv[i].invariant = copy_rtx (res);
532 : }
533 1488370 : pointer_equiv_set = new hash_set <rtx>;
534 1488370 : }
535 :
536 : /* Finish equivalence data for LRA. */
537 : void
538 1488370 : lra_finish_equiv (void)
539 : {
540 2976740 : delete pointer_equiv_set;
541 1488370 : }
542 :
543 : static rtx loc_equivalence_callback (rtx, const_rtx, void *);
544 :
545 : /* Update equivalence for REGNO. We need to this as the equivalence
546 : might contain other pseudos which are changed by their
547 : equivalences. */
548 : static void
549 202042222 : update_equiv (int regno)
550 : {
551 202042222 : rtx x;
552 :
553 202042222 : if ((x = ira_reg_equiv[regno].memory) != NULL_RTX)
554 9160702 : ira_reg_equiv[regno].memory
555 9160702 : = simplify_replace_fn_rtx (x, NULL_RTX, loc_equivalence_callback,
556 : NULL_RTX);
557 202042222 : if ((x = ira_reg_equiv[regno].invariant) != NULL_RTX)
558 2710991 : ira_reg_equiv[regno].invariant
559 2710991 : = simplify_replace_fn_rtx (x, NULL_RTX, loc_equivalence_callback,
560 : NULL_RTX);
561 202042222 : }
562 :
563 : /* If we have decided to substitute X with another value, return that
564 : value, otherwise return X. */
565 : static rtx
566 434900063 : get_equiv (rtx x)
567 : {
568 434900063 : int regno;
569 434900063 : rtx res;
570 :
571 294384311 : if (! REG_P (x) || (regno = REGNO (x)) < FIRST_PSEUDO_REGISTER
572 195295858 : || regno >= ira_reg_equiv_len
573 195295858 : || ! ira_reg_equiv[regno].defined_p
574 25429618 : || ! ira_reg_equiv[regno].profitable_p
575 460289410 : || lra_get_regno_hard_regno (regno) >= 0)
576 430003682 : return x;
577 4896381 : if ((res = ira_reg_equiv[regno].memory) != NULL_RTX)
578 : {
579 2194417 : if (targetm.cannot_substitute_mem_equiv_p (res))
580 : return x;
581 : return res;
582 : }
583 2701964 : if ((res = ira_reg_equiv[regno].constant) != NULL_RTX)
584 : return res;
585 1855932 : if ((res = ira_reg_equiv[regno].invariant) != NULL_RTX)
586 : return res;
587 0 : gcc_unreachable ();
588 : }
589 :
590 : /* If we have decided to substitute X with the equivalent value, return that
591 : value after elimination for INSN, otherwise return X. Add the result to
592 : pointer_equiv_set if X has set up pointer flag. */
593 : static rtx
594 244792269 : get_equiv_with_elimination (rtx x, rtx_insn *insn)
595 : {
596 244792269 : rtx res = get_equiv (x);
597 :
598 244792269 : if (x == res || CONSTANT_P (res))
599 : return res;
600 1494263 : res = lra_eliminate_regs_1 (insn, res, GET_MODE (res),
601 : false, false, 0, true);
602 1494263 : if (REG_POINTER (x))
603 1031483 : lra_pointer_equiv_set_add (res);
604 : return res;
605 : }
606 :
607 : /* Set up curr_operand_mode. */
608 : static void
609 105546977 : init_curr_operand_mode (void)
610 : {
611 105546977 : int nop = curr_static_id->n_operands;
612 329042543 : for (int i = 0; i < nop; i++)
613 : {
614 223495566 : machine_mode mode = GET_MODE (*curr_id->operand_loc[i]);
615 223495566 : if (mode == VOIDmode)
616 : {
617 : /* The .md mode for address operands is the mode of the
618 : addressed value rather than the mode of the address itself. */
619 43160626 : if (curr_id->icode >= 0 && curr_static_id->operand[i].is_address)
620 95 : mode = Pmode;
621 : else
622 43160531 : mode = curr_static_id->operand[i].mode;
623 : }
624 223495566 : curr_operand_mode[i] = mode;
625 : }
626 105546977 : }
627 :
628 :
629 :
630 : /* The page contains code to reuse input reloads. */
631 :
632 : /* Structure describes input reload of the current insns. */
633 : struct input_reload
634 : {
635 : /* True for input reload of matched operands. */
636 : bool match_p;
637 : /* True for input reload of inout earlyclobber operand. */
638 : bool early_clobber_p;
639 : /* Reloaded value. */
640 : rtx input;
641 : /* Reload pseudo used. */
642 : rtx reg;
643 : };
644 :
645 : /* The number of elements in the following array. */
646 : static int curr_insn_input_reloads_num;
647 : /* Array containing info about input reloads. It is used to find the
648 : same input reload and reuse the reload pseudo in this case. */
649 : static struct input_reload curr_insn_input_reloads[LRA_MAX_INSN_RELOADS];
650 :
651 : /* Initiate data concerning reuse of input reloads for the current
652 : insn. */
653 : static void
654 105546977 : init_curr_insn_input_reloads (void)
655 : {
656 105546977 : curr_insn_input_reloads_num = 0;
657 0 : }
658 :
659 : /* The canonical form of an rtx inside a MEM is not necessarily the same as the
660 : canonical form of the rtx outside the MEM. Fix this up in the case that
661 : we're reloading an address (and therefore pulling it outside a MEM). */
662 : static rtx
663 72 : canonicalize_reload_addr (rtx addr)
664 : {
665 72 : subrtx_var_iterator::array_type array;
666 246 : FOR_EACH_SUBRTX_VAR (iter, array, addr, NONCONST)
667 : {
668 174 : rtx x = *iter;
669 174 : if (GET_CODE (x) == MULT && CONST_INT_P (XEXP (x, 1)))
670 : {
671 14 : const HOST_WIDE_INT ci = INTVAL (XEXP (x, 1));
672 188 : const int pwr2 = exact_log2 (ci);
673 14 : if (pwr2 > 0)
674 : {
675 : /* Rewrite this to use a shift instead, which is canonical when
676 : outside of a MEM. */
677 14 : PUT_CODE (x, ASHIFT);
678 14 : XEXP (x, 1) = GEN_INT (pwr2);
679 : }
680 : }
681 : }
682 :
683 72 : return addr;
684 72 : }
685 :
686 : /* Return rtx accessing reload REG of RCLASS matching another reload reg in
687 : MODE. */
688 : static rtx
689 123290 : get_matching_reload_reg_subreg (machine_mode mode, rtx reg,
690 : enum reg_class rclass)
691 : {
692 123290 : int hard_regno = ira_class_hard_regs[rclass][0];
693 123290 : if (subreg_regno_offset (hard_regno,
694 123290 : GET_MODE (reg),
695 123290 : subreg_lowpart_offset (mode, GET_MODE (reg)),
696 : mode) == 0)
697 : /* For matching scalar int modes generate the right subreg byte offset for
698 : BE targets -- see call of reload.cc:operands_match_p in
699 : recog.cc:constrain_operands. */
700 123290 : return lowpart_subreg (mode, reg, GET_MODE (reg));
701 0 : int offset = (lra_constraint_offset (hard_regno, GET_MODE (reg))
702 0 : - lra_constraint_offset (hard_regno, mode)) * UNITS_PER_WORD;
703 0 : lra_assert (offset >= 0);
704 0 : return gen_rtx_SUBREG (mode, reg, offset);
705 : }
706 :
707 : /* Create a new pseudo using MODE, RCLASS, EXCLUDE_START_HARD_REGS, ORIGINAL or
708 : reuse an existing reload pseudo. Don't reuse an existing reload pseudo if
709 : IN_SUBREG_P is true and the reused pseudo should be wrapped up in a SUBREG.
710 : EARLY_CLOBBER_P is true for input reload of inout early clobber operand.
711 : The result pseudo is returned through RESULT_REG. Return TRUE if we created
712 : a new pseudo, FALSE if we reused an existing reload pseudo. Use TITLE to
713 : describe new registers for debug purposes. */
714 : static bool
715 3798018 : get_reload_reg (enum op_type type, machine_mode mode, rtx original,
716 : enum reg_class rclass, HARD_REG_SET *exclude_start_hard_regs,
717 : bool in_subreg_p, bool early_clobber_p,
718 : const char *title, rtx *result_reg)
719 : {
720 3798018 : int i, regno;
721 3798018 : enum reg_class new_class;
722 :
723 3798018 : if (type == OP_OUT)
724 : {
725 : /* Output reload registers tend to start out with a conservative
726 : choice of register class. Usually this is ALL_REGS, although
727 : a target might narrow it (for performance reasons) through
728 : targetm.preferred_reload_class. It's therefore quite common
729 : for a reload instruction to require a more restrictive class
730 : than the class that was originally assigned to the reload register.
731 :
732 : In these situations, it's more efficient to refine the choice
733 : of register class rather than create a second reload register.
734 : This also helps to avoid cycling for registers that are only
735 : used by reload instructions. */
736 959798 : if (REG_P (original)
737 701908 : && (int) REGNO (original) >= new_regno_start
738 7214 : && (INSN_UID (curr_insn) >= new_insn_uid_start
739 250 : || ira_former_scratch_p (REGNO (original)))
740 7214 : && in_class_p (original, rclass, &new_class, true)
741 960048 : && (exclude_start_hard_regs == nullptr
742 250 : || hard_reg_set_intersect_p (
743 960048 : ~lra_reg_info[REGNO (original)].exclude_start_hard_regs,
744 250 : ~*exclude_start_hard_regs)))
745 : {
746 250 : unsigned int regno = REGNO (original);
747 250 : if (lra_dump_file != NULL)
748 : {
749 0 : fprintf (lra_dump_file, " Reuse r%d for output ", regno);
750 0 : dump_value_slim (lra_dump_file, original, 1);
751 : }
752 500 : if (new_class != lra_get_allocno_class (regno))
753 250 : lra_change_class (regno, new_class, ", change to", false);
754 250 : if (lra_dump_file != NULL)
755 0 : fprintf (lra_dump_file, "\n");
756 250 : if (exclude_start_hard_regs)
757 250 : lra_reg_info[regno].exclude_start_hard_regs
758 250 : |= *exclude_start_hard_regs;
759 250 : *result_reg = original;
760 250 : return false;
761 : }
762 959548 : *result_reg
763 959548 : = lra_create_new_reg_with_unique_value (mode, original, rclass,
764 : exclude_start_hard_regs, title);
765 959548 : return true;
766 : }
767 :
768 2838220 : bool unique_p = early_clobber_p;
769 : /* Prevent reuse value of expression with side effects,
770 : e.g. volatile memory. */
771 2838220 : if (! side_effects_p (original))
772 3059433 : for (i = 0; i < curr_insn_input_reloads_num; i++)
773 : {
774 240322 : if (! curr_insn_input_reloads[i].match_p
775 101622 : && ! curr_insn_input_reloads[i].early_clobber_p
776 101621 : && rtx_equal_p (curr_insn_input_reloads[i].input, original)
777 249403 : && in_class_p (curr_insn_input_reloads[i].reg, rclass, &new_class))
778 : {
779 9068 : rtx reg = curr_insn_input_reloads[i].reg;
780 9068 : regno = REGNO (reg);
781 : /* If input is equal to original and both are VOIDmode,
782 : GET_MODE (reg) might be still different from mode.
783 : Ensure we don't return *result_reg with wrong mode. */
784 9068 : if (GET_MODE (reg) != mode)
785 : {
786 0 : if (in_subreg_p)
787 0 : continue;
788 0 : if (maybe_lt (GET_MODE_SIZE (GET_MODE (reg)),
789 0 : GET_MODE_SIZE (mode)))
790 0 : continue;
791 0 : reg = get_matching_reload_reg_subreg (mode, reg, new_class);
792 0 : if (reg == NULL_RTX || GET_CODE (reg) != SUBREG)
793 0 : continue;
794 : }
795 : /* If the existing reload and this have no start hard register in
796 : common, then skip. Otherwise update exclude_start_hard_regs. */
797 9068 : if (exclude_start_hard_regs
798 10335 : && ! hard_reg_set_empty_p (*exclude_start_hard_regs))
799 : {
800 1 : HARD_REG_SET r = lra_reg_info[regno].exclude_start_hard_regs
801 1 : | *exclude_start_hard_regs;
802 2 : if (hard_reg_set_empty_p (~r))
803 0 : continue;
804 : else
805 1 : lra_reg_info[regno].exclude_start_hard_regs = r;
806 : }
807 9068 : *result_reg = reg;
808 9068 : if (lra_dump_file != NULL)
809 : {
810 0 : fprintf (lra_dump_file, " Reuse r%d for reload ", regno);
811 0 : dump_value_slim (lra_dump_file, original, 1);
812 : }
813 18136 : if (new_class != lra_get_allocno_class (regno))
814 4339 : lra_change_class (regno, new_class, ", change to", false);
815 9068 : if (lra_dump_file != NULL)
816 0 : fprintf (lra_dump_file, "\n");
817 9068 : return false;
818 : }
819 : /* If we have an input reload with a different mode, make sure it
820 : will get a different hard reg. */
821 231254 : else if (REG_P (original)
822 182220 : && REG_P (curr_insn_input_reloads[i].input)
823 149097 : && REGNO (original) == REGNO (curr_insn_input_reloads[i].input)
824 231254 : && (GET_MODE (original)
825 2048 : != GET_MODE (curr_insn_input_reloads[i].input)))
826 : unique_p = true;
827 : }
828 5658304 : *result_reg = (unique_p
829 2829152 : ? lra_create_new_reg_with_unique_value
830 2829152 : : lra_create_new_reg) (mode, original, rclass,
831 : exclude_start_hard_regs, title);
832 2829152 : lra_assert (curr_insn_input_reloads_num < LRA_MAX_INSN_RELOADS);
833 2829152 : curr_insn_input_reloads[curr_insn_input_reloads_num].input = original;
834 2829152 : curr_insn_input_reloads[curr_insn_input_reloads_num].match_p = false;
835 2829152 : curr_insn_input_reloads[curr_insn_input_reloads_num].early_clobber_p
836 2829152 : = early_clobber_p;
837 2829152 : curr_insn_input_reloads[curr_insn_input_reloads_num++].reg = *result_reg;
838 2829152 : return true;
839 : }
840 :
841 :
842 : /* The page contains major code to choose the current insn alternative
843 : and generate reloads for it. */
844 :
845 : /* Return the offset from REGNO of the least significant register
846 : in (reg:MODE REGNO).
847 :
848 : This function is used to tell whether two registers satisfy
849 : a matching constraint. (reg:MODE1 REGNO1) matches (reg:MODE2 REGNO2) if:
850 :
851 : REGNO1 + lra_constraint_offset (REGNO1, MODE1)
852 : == REGNO2 + lra_constraint_offset (REGNO2, MODE2) */
853 : int
854 43337658 : lra_constraint_offset (int regno, machine_mode mode)
855 : {
856 43337658 : lra_assert (regno < FIRST_PSEUDO_REGISTER);
857 :
858 43337658 : scalar_int_mode int_mode;
859 43337658 : if (WORDS_BIG_ENDIAN
860 : && is_a <scalar_int_mode> (mode, &int_mode)
861 : && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD)
862 : return hard_regno_nregs (regno, mode) - 1;
863 43337658 : return 0;
864 : }
865 :
866 : /* Like rtx_equal_p except that it allows a REG and a SUBREG to match
867 : if they are the same hard reg, and has special hacks for
868 : auto-increment and auto-decrement. This is specifically intended for
869 : process_alt_operands to use in determining whether two operands
870 : match. X is the operand whose number is the lower of the two.
871 :
872 : It is supposed that X is the output operand and Y is the input
873 : operand. Y_HARD_REGNO is the final hard regno of register Y or
874 : register in subreg Y as we know it now. Otherwise, it is a
875 : negative value. */
876 : static bool
877 57587393 : operands_match_p (rtx x, rtx y, int y_hard_regno)
878 : {
879 57587393 : int i;
880 57587393 : RTX_CODE code = GET_CODE (x);
881 57587393 : const char *fmt;
882 :
883 57587393 : if (x == y)
884 : return true;
885 49947649 : if ((code == REG || (code == SUBREG && REG_P (SUBREG_REG (x))))
886 23687194 : && (REG_P (y) || (GET_CODE (y) == SUBREG && REG_P (SUBREG_REG (y)))))
887 : {
888 23593814 : int j;
889 :
890 23593814 : i = get_hard_regno (x);
891 23593814 : if (i < 0)
892 1234193 : goto slow;
893 :
894 22359621 : if ((j = y_hard_regno) < 0)
895 690792 : goto slow;
896 :
897 21668829 : i += lra_constraint_offset (i, GET_MODE (x));
898 21668829 : j += lra_constraint_offset (j, GET_MODE (y));
899 :
900 21668829 : return i == j;
901 : }
902 :
903 : /* If two operands must match, because they are really a single
904 : operand of an assembler insn, then two post-increments are invalid
905 : because the assembler insn would increment only once. On the
906 : other hand, a post-increment matches ordinary indexing if the
907 : post-increment is the output operand. */
908 26353835 : if (code == POST_DEC || code == POST_INC || code == POST_MODIFY)
909 0 : return operands_match_p (XEXP (x, 0), y, y_hard_regno);
910 :
911 : /* Two pre-increments are invalid because the assembler insn would
912 : increment only once. On the other hand, a pre-increment matches
913 : ordinary indexing if the pre-increment is the input operand. */
914 26353835 : if (GET_CODE (y) == PRE_DEC || GET_CODE (y) == PRE_INC
915 26353835 : || GET_CODE (y) == PRE_MODIFY)
916 0 : return operands_match_p (x, XEXP (y, 0), -1);
917 :
918 26353835 : slow:
919 :
920 28278820 : if (code == REG && REG_P (y))
921 1829563 : return REGNO (x) == REGNO (y);
922 :
923 94493 : if (code == REG && GET_CODE (y) == SUBREG && REG_P (SUBREG_REG (y))
924 7841 : && x == SUBREG_REG (y))
925 : return true;
926 26449257 : if (GET_CODE (y) == REG && code == SUBREG && REG_P (SUBREG_REG (x))
927 62587 : && SUBREG_REG (x) == y)
928 : return true;
929 :
930 : /* Now we have disposed of all the cases in which different rtx
931 : codes can match. */
932 26449083 : if (code != GET_CODE (y))
933 : return false;
934 :
935 : /* (MULT:SI x y) and (MULT:HI x y) are NOT equivalent. */
936 1045458 : if (GET_MODE (x) != GET_MODE (y))
937 : return false;
938 :
939 1044710 : switch (code)
940 : {
941 : CASE_CONST_UNIQUE:
942 : return false;
943 :
944 : case CONST_VECTOR:
945 : if (!same_vector_encodings_p (x, y))
946 : return false;
947 : break;
948 :
949 0 : case LABEL_REF:
950 0 : return label_ref_label (x) == label_ref_label (y);
951 25 : case SYMBOL_REF:
952 25 : return XSTR (x, 0) == XSTR (y, 0);
953 :
954 : default:
955 : break;
956 : }
957 :
958 : /* Compare the elements. If any pair of corresponding elements fail
959 : to match, return false for the whole things. */
960 :
961 1023563 : fmt = GET_RTX_FORMAT (code);
962 2963653 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
963 : {
964 2014788 : int val, j;
965 2014788 : switch (fmt[i])
966 : {
967 0 : case 'w':
968 0 : if (XWINT (x, i) != XWINT (y, i))
969 : return false;
970 : break;
971 :
972 488 : case 'i':
973 488 : if (XINT (x, i) != XINT (y, i))
974 : return false;
975 : break;
976 :
977 0 : case 'L':
978 0 : if (XLOC (x, i) != XLOC (y, i))
979 : return false;
980 : break;
981 :
982 24906 : case 'p':
983 24906 : if (maybe_ne (SUBREG_BYTE (x), SUBREG_BYTE (y)))
984 : return false;
985 : break;
986 :
987 1465138 : case 'e':
988 1465138 : val = operands_match_p (XEXP (x, i), XEXP (y, i), -1);
989 1465138 : if (val == 0)
990 : return false;
991 : break;
992 :
993 : case '0':
994 : break;
995 :
996 488 : case 'E':
997 488 : if (XVECLEN (x, i) != XVECLEN (y, i))
998 : return false;
999 976 : for (j = XVECLEN (x, i) - 1; j >= 0; --j)
1000 : {
1001 488 : val = operands_match_p (XVECEXP (x, i, j), XVECEXP (y, i, j), -1);
1002 488 : if (val == 0)
1003 : return false;
1004 : }
1005 : break;
1006 :
1007 : /* It is believed that rtx's at this level will never
1008 : contain anything but integers and other rtx's, except for
1009 : within LABEL_REFs and SYMBOL_REFs. */
1010 0 : default:
1011 0 : gcc_unreachable ();
1012 : }
1013 : }
1014 : return true;
1015 : }
1016 :
1017 : /* If REG is a reload pseudo, try to make its class satisfying CL. */
1018 : static void
1019 3408688 : narrow_reload_pseudo_class (rtx reg, enum reg_class cl)
1020 : {
1021 3408688 : enum reg_class rclass;
1022 :
1023 : /* Do not make more accurate class from reloads generated. They are
1024 : mostly moves with a lot of constraints. Making more accurate
1025 : class may results in very narrow class and impossibility of find
1026 : registers for several reloads of one insn. */
1027 3408688 : if (INSN_UID (curr_insn) >= new_insn_uid_start)
1028 3408656 : return;
1029 3408570 : if (GET_CODE (reg) == SUBREG)
1030 164448 : reg = SUBREG_REG (reg);
1031 3408570 : if (! REG_P (reg) || (int) REGNO (reg) < new_regno_start)
1032 : return;
1033 32 : if (in_class_p (reg, cl, &rclass) && rclass != cl)
1034 13 : lra_change_class (REGNO (reg), rclass, " Change to", true);
1035 : }
1036 :
1037 : /* Searches X for any reference to a reg with the same value as REGNO,
1038 : returning the rtx of the reference found if any. Otherwise,
1039 : returns NULL_RTX. */
1040 : static rtx
1041 529760 : regno_val_use_in (unsigned int regno, rtx x)
1042 : {
1043 529760 : const char *fmt;
1044 529760 : int i, j;
1045 529760 : rtx tem;
1046 :
1047 529760 : if (REG_P (x) && lra_reg_info[REGNO (x)].val == lra_reg_info[regno].val)
1048 : return x;
1049 :
1050 529434 : fmt = GET_RTX_FORMAT (GET_CODE (x));
1051 1065255 : for (i = GET_RTX_LENGTH (GET_CODE (x)) - 1; i >= 0; i--)
1052 : {
1053 535821 : if (fmt[i] == 'e')
1054 : {
1055 7673 : if ((tem = regno_val_use_in (regno, XEXP (x, i))))
1056 : return tem;
1057 : }
1058 528148 : else if (fmt[i] == 'E')
1059 0 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
1060 0 : if ((tem = regno_val_use_in (regno , XVECEXP (x, i, j))))
1061 : return tem;
1062 : }
1063 :
1064 : return NULL_RTX;
1065 : }
1066 :
1067 : /* Return true if all current insn non-output operands except INS (it
1068 : has a negaitve end marker) do not use pseudos with the same value
1069 : as REGNO. */
1070 : static bool
1071 2 : check_conflict_input_operands (int regno, signed char *ins)
1072 : {
1073 2 : int in;
1074 2 : int n_operands = curr_static_id->n_operands;
1075 :
1076 8 : for (int nop = 0; nop < n_operands; nop++)
1077 7 : if (! curr_static_id->operand[nop].is_operator
1078 7 : && curr_static_id->operand[nop].type != OP_OUT)
1079 : {
1080 5 : for (int i = 0; (in = ins[i]) >= 0; i++)
1081 4 : if (in == nop)
1082 : break;
1083 3 : if (in < 0
1084 3 : && regno_val_use_in (regno, *curr_id->operand_loc[nop]) != NULL_RTX)
1085 : return false;
1086 : }
1087 : return true;
1088 : }
1089 :
1090 : /* Generate reloads for matching OUT and INS (array of input operand numbers
1091 : with end marker -1) with reg class GOAL_CLASS and EXCLUDE_START_HARD_REGS,
1092 : considering output operands OUTS (similar array to INS) needing to be in
1093 : different registers. Add input and output reloads correspondingly to the
1094 : lists *BEFORE and *AFTER. OUT might be negative. In this case we generate
1095 : input reloads for matched input operands INS. EARLY_CLOBBER_P is a flag
1096 : that the output operand is early clobbered for chosen alternative. */
1097 : static void
1098 1704344 : match_reload (signed char out, signed char *ins, signed char *outs,
1099 : enum reg_class goal_class, HARD_REG_SET *exclude_start_hard_regs,
1100 : rtx_insn **before, rtx_insn **after, bool early_clobber_p)
1101 : {
1102 1704344 : bool out_conflict;
1103 1704344 : int i, in;
1104 1704344 : rtx new_in_reg, new_out_reg, reg;
1105 1704344 : machine_mode inmode, outmode;
1106 1704344 : rtx in_rtx = *curr_id->operand_loc[ins[0]];
1107 1704344 : rtx out_rtx = out < 0 ? in_rtx : *curr_id->operand_loc[out];
1108 :
1109 1704344 : inmode = curr_operand_mode[ins[0]];
1110 1704344 : outmode = out < 0 ? inmode : curr_operand_mode[out];
1111 1704344 : push_to_sequence (*before);
1112 1704344 : if (inmode != outmode)
1113 : {
1114 : /* process_alt_operands has already checked that the mode sizes
1115 : are ordered. */
1116 123290 : if (partial_subreg_p (outmode, inmode))
1117 : {
1118 1813 : bool asm_p = asm_noperands (PATTERN (curr_insn)) >= 0;
1119 1813 : int hr;
1120 1813 : HARD_REG_SET temp_hard_reg_set;
1121 :
1122 19 : if (asm_p && (hr = get_hard_regno (out_rtx)) >= 0
1123 1816 : && hard_regno_nregs (hr, inmode) > 1)
1124 : {
1125 : /* See gcc.c-torture/execute/20030222-1.c.
1126 : Consider the code for 32-bit (e.g. BE) target:
1127 : int i, v; long x; x = v; asm ("" : "=r" (i) : "0" (x));
1128 : We generate the following RTL with reload insns:
1129 : 1. subreg:si(x:di, 0) = 0;
1130 : 2. subreg:si(x:di, 4) = v:si;
1131 : 3. t:di = x:di, dead x;
1132 : 4. asm ("" : "=r" (subreg:si(t:di,4)) : "0" (t:di))
1133 : 5. i:si = subreg:si(t:di,4);
1134 : If we assign hard reg of x to t, dead code elimination
1135 : will remove insn #2 and we will use uninitialized hard reg.
1136 : So exclude the hard reg of x for t. We could ignore this
1137 : problem for non-empty asm using all x value but it is hard to
1138 : check that the asm are expanded into insn really using x
1139 : and setting r. */
1140 0 : CLEAR_HARD_REG_SET (temp_hard_reg_set);
1141 0 : if (exclude_start_hard_regs != NULL)
1142 0 : temp_hard_reg_set = *exclude_start_hard_regs;
1143 0 : SET_HARD_REG_BIT (temp_hard_reg_set, hr);
1144 0 : exclude_start_hard_regs = &temp_hard_reg_set;
1145 : }
1146 3626 : reg = new_in_reg
1147 1813 : = lra_create_new_reg_with_unique_value (inmode, in_rtx, goal_class,
1148 : exclude_start_hard_regs,
1149 : "");
1150 1813 : new_out_reg = get_matching_reload_reg_subreg (outmode, reg, goal_class);
1151 1813 : LRA_SUBREG_P (new_out_reg) = 1;
1152 : /* If the input reg is dying here, we can use the same hard
1153 : register for REG and IN_RTX. We do it only for original
1154 : pseudos as reload pseudos can die although original
1155 : pseudos still live where reload pseudos dies. */
1156 1500 : if (REG_P (in_rtx) && (int) REGNO (in_rtx) < lra_new_regno_start
1157 1451 : && find_regno_note (curr_insn, REG_DEAD, REGNO (in_rtx))
1158 2841 : && (!early_clobber_p
1159 2 : || check_conflict_input_operands(REGNO (in_rtx), ins)))
1160 1027 : lra_assign_reg_val (REGNO (in_rtx), REGNO (reg));
1161 : }
1162 : else
1163 : {
1164 242954 : reg = new_out_reg
1165 121477 : = lra_create_new_reg_with_unique_value (outmode, out_rtx,
1166 : goal_class,
1167 : exclude_start_hard_regs,
1168 : "");
1169 121477 : new_in_reg = get_matching_reload_reg_subreg (inmode, reg, goal_class);
1170 : /* NEW_IN_REG is non-paradoxical subreg. We don't want
1171 : NEW_OUT_REG living above. We add clobber clause for
1172 : this. This is just a temporary clobber. We can remove
1173 : it at the end of LRA work. */
1174 121477 : rtx_insn *clobber = emit_clobber (new_out_reg);
1175 121477 : LRA_TEMP_CLOBBER_P (PATTERN (clobber)) = 1;
1176 121477 : LRA_SUBREG_P (new_in_reg) = 1;
1177 121477 : if (GET_CODE (in_rtx) == SUBREG)
1178 : {
1179 1756 : rtx subreg_reg = SUBREG_REG (in_rtx);
1180 :
1181 : /* If SUBREG_REG is dying here and sub-registers IN_RTX
1182 : and NEW_IN_REG are similar, we can use the same hard
1183 : register for REG and SUBREG_REG. */
1184 1756 : if (REG_P (subreg_reg)
1185 1756 : && (int) REGNO (subreg_reg) < lra_new_regno_start
1186 1756 : && GET_MODE (subreg_reg) == outmode
1187 1087 : && known_eq (SUBREG_BYTE (in_rtx), SUBREG_BYTE (new_in_reg))
1188 1087 : && find_regno_note (curr_insn, REG_DEAD, REGNO (subreg_reg))
1189 1865 : && (! early_clobber_p
1190 0 : || check_conflict_input_operands (REGNO (subreg_reg),
1191 : ins)))
1192 109 : lra_assign_reg_val (REGNO (subreg_reg), REGNO (reg));
1193 : }
1194 : }
1195 : }
1196 : else
1197 : {
1198 : /* Pseudos have values -- see comments for lra_reg_info.
1199 : Different pseudos with the same value do not conflict even if
1200 : they live in the same place. When we create a pseudo we
1201 : assign value of original pseudo (if any) from which we
1202 : created the new pseudo. If we create the pseudo from the
1203 : input pseudo, the new pseudo will have no conflict with the
1204 : input pseudo which is wrong when the input pseudo lives after
1205 : the insn and as the new pseudo value is changed by the insn
1206 : output. Therefore we create the new pseudo from the output
1207 : except the case when we have single matched dying input
1208 : pseudo.
1209 :
1210 : We cannot reuse the current output register because we might
1211 : have a situation like "a <- a op b", where the constraints
1212 : force the second input operand ("b") to match the output
1213 : operand ("a"). "b" must then be copied into a new register
1214 : so that it doesn't clobber the current value of "a".
1215 :
1216 : We cannot use the same value if the output pseudo is
1217 : early clobbered or the input pseudo is mentioned in the
1218 : output, e.g. as an address part in memory, because
1219 : output reload will actually extend the pseudo liveness.
1220 : We don't care about eliminable hard regs here as we are
1221 : interesting only in pseudos. */
1222 :
1223 : /* Matching input's register value is the same as one of the other
1224 : output operand. Output operands in a parallel insn must be in
1225 : different registers. */
1226 1581054 : out_conflict = false;
1227 1581054 : if (REG_P (in_rtx))
1228 : {
1229 2725632 : for (i = 0; outs[i] >= 0; i++)
1230 : {
1231 1413017 : rtx other_out_rtx = *curr_id->operand_loc[outs[i]];
1232 100172 : if (outs[i] != out && REG_P (other_out_rtx)
1233 1512995 : && (regno_val_use_in (REGNO (in_rtx), other_out_rtx)
1234 : != NULL_RTX))
1235 : {
1236 : out_conflict = true;
1237 : break;
1238 : }
1239 : }
1240 : }
1241 :
1242 1581054 : new_in_reg = new_out_reg
1243 1550381 : = (! early_clobber_p && ins[1] < 0 && REG_P (in_rtx)
1244 1283000 : && (int) REGNO (in_rtx) < lra_new_regno_start
1245 1282699 : && find_regno_note (curr_insn, REG_DEAD, REGNO (in_rtx))
1246 422108 : && (out < 0
1247 422108 : || regno_val_use_in (REGNO (in_rtx), out_rtx) == NULL_RTX)
1248 422076 : && !out_conflict
1249 2003128 : ? lra_create_new_reg (inmode, in_rtx, goal_class,
1250 : exclude_start_hard_regs, "")
1251 1158980 : : lra_create_new_reg_with_unique_value (outmode, out_rtx, goal_class,
1252 : exclude_start_hard_regs,
1253 : ""));
1254 : }
1255 : /* In operand can be got from transformations before processing insn
1256 : constraints. One example of such transformations is subreg
1257 : reloading (see function simplify_operand_subreg). The new
1258 : pseudos created by the transformations might have inaccurate
1259 : class (ALL_REGS) and we should make their classes more
1260 : accurate. */
1261 1704344 : narrow_reload_pseudo_class (in_rtx, goal_class);
1262 1704344 : lra_emit_move (copy_rtx (new_in_reg), in_rtx);
1263 1704344 : *before = end_sequence ();
1264 : /* Add the new pseudo to consider values of subsequent input reload
1265 : pseudos. */
1266 1704344 : lra_assert (curr_insn_input_reloads_num < LRA_MAX_INSN_RELOADS);
1267 1704344 : curr_insn_input_reloads[curr_insn_input_reloads_num].input = in_rtx;
1268 1704344 : curr_insn_input_reloads[curr_insn_input_reloads_num].match_p = true;
1269 1704344 : curr_insn_input_reloads[curr_insn_input_reloads_num].early_clobber_p = false;
1270 1704344 : curr_insn_input_reloads[curr_insn_input_reloads_num++].reg = new_in_reg;
1271 3408689 : for (i = 0; (in = ins[i]) >= 0; i++)
1272 1704345 : if (GET_MODE (*curr_id->operand_loc[in]) == VOIDmode
1273 1675887 : || GET_MODE (new_in_reg) == GET_MODE (*curr_id->operand_loc[in]))
1274 1704344 : *curr_id->operand_loc[in] = new_in_reg;
1275 : else
1276 : {
1277 1 : lra_assert
1278 : (GET_MODE (new_out_reg) == GET_MODE (*curr_id->operand_loc[in]));
1279 1 : *curr_id->operand_loc[in] = new_out_reg;
1280 : }
1281 1704344 : lra_update_dups (curr_id, ins);
1282 1704344 : if (out < 0)
1283 : return;
1284 : /* See a comment for the input operand above. */
1285 1704344 : narrow_reload_pseudo_class (out_rtx, goal_class);
1286 1704344 : reg = SUBREG_P (out_rtx) ? SUBREG_REG (out_rtx) : out_rtx;
1287 1704344 : if (find_reg_note (curr_insn, REG_UNUSED, reg) == NULL_RTX
1288 1704344 : && (!REG_P (reg) || !ira_former_scratch_p (REGNO (reg))))
1289 : {
1290 1620444 : start_sequence ();
1291 : /* If we had strict_low_part, use it also in reload to keep other
1292 : parts unchanged but do it only for regs as strict_low_part
1293 : has no sense for memory and probably there is no insn pattern
1294 : to match the reload insn in memory case. */
1295 1620444 : if (out >= 0 && curr_static_id->operand[out].strict_low && REG_P (reg))
1296 0 : out_rtx = gen_rtx_STRICT_LOW_PART (VOIDmode, out_rtx);
1297 1620444 : lra_emit_move (out_rtx, copy_rtx (new_out_reg));
1298 1620444 : emit_insn (*after);
1299 1620444 : *after = end_sequence ();
1300 : }
1301 1704344 : *curr_id->operand_loc[out] = new_out_reg;
1302 1704344 : lra_update_dup (curr_id, out);
1303 : }
1304 :
1305 : /* Return register class which is union of all reg classes in insn
1306 : constraint alternative string starting with P. */
1307 : static enum reg_class
1308 0 : reg_class_from_constraints (const char *p)
1309 : {
1310 0 : int c, len;
1311 0 : enum reg_class op_class = NO_REGS;
1312 :
1313 0 : do
1314 0 : switch ((c = *p, len = CONSTRAINT_LEN (c, p)), c)
1315 : {
1316 : case '#':
1317 : case ',':
1318 : return op_class;
1319 :
1320 0 : case 'g':
1321 0 : op_class = reg_class_subunion[op_class][GENERAL_REGS];
1322 0 : break;
1323 :
1324 0 : default:
1325 0 : enum constraint_num cn = lookup_constraint (p);
1326 0 : enum reg_class cl = reg_class_for_constraint (cn);
1327 0 : if (cl == NO_REGS)
1328 : {
1329 0 : if (insn_extra_address_constraint (cn))
1330 0 : op_class
1331 0 : = (reg_class_subunion
1332 0 : [op_class][base_reg_class (VOIDmode, ADDR_SPACE_GENERIC,
1333 0 : ADDRESS, SCRATCH)]);
1334 : break;
1335 : }
1336 :
1337 0 : op_class = reg_class_subunion[op_class][cl];
1338 0 : break;
1339 : }
1340 0 : while ((p += len), c);
1341 : return op_class;
1342 : }
1343 :
1344 : /* If OP is a register, return the class of the register as per
1345 : get_reg_class, otherwise return NO_REGS. */
1346 : static inline enum reg_class
1347 162115520 : get_op_class (rtx op)
1348 : {
1349 134605721 : return REG_P (op) ? get_reg_class (REGNO (op)) : NO_REGS;
1350 : }
1351 :
1352 : /* Return generated insn mem_pseudo:=val if TO_P or val:=mem_pseudo
1353 : otherwise. If modes of MEM_PSEUDO and VAL are different, use
1354 : SUBREG for VAL to make them equal. */
1355 : static rtx_insn *
1356 1298331 : emit_spill_move (bool to_p, rtx mem_pseudo, rtx val)
1357 : {
1358 1298331 : if (GET_MODE (mem_pseudo) != GET_MODE (val))
1359 : {
1360 : /* Usually size of mem_pseudo is greater than val size but in
1361 : rare cases it can be less as it can be defined by target
1362 : dependent macro HARD_REGNO_CALLER_SAVE_MODE. */
1363 2902 : if (! MEM_P (val))
1364 : {
1365 2902 : val = gen_lowpart_SUBREG (GET_MODE (mem_pseudo),
1366 : GET_CODE (val) == SUBREG
1367 : ? SUBREG_REG (val) : val);
1368 2902 : LRA_SUBREG_P (val) = 1;
1369 : }
1370 : else
1371 : {
1372 0 : mem_pseudo = gen_lowpart_SUBREG (GET_MODE (val), mem_pseudo);
1373 0 : LRA_SUBREG_P (mem_pseudo) = 1;
1374 : }
1375 : }
1376 1298331 : return to_p ? gen_move_insn (mem_pseudo, val)
1377 655693 : : gen_move_insn (val, mem_pseudo);
1378 : }
1379 :
1380 : /* Process a special case insn (register move), return true if we
1381 : don't need to process it anymore. INSN should be a single set
1382 : insn. Set up that RTL was changed through CHANGE_P and that hook
1383 : TARGET_SECONDARY_MEMORY_NEEDED says to use secondary memory through
1384 : SEC_MEM_P. */
1385 : static bool
1386 75233700 : check_and_process_move (bool *change_p, bool *sec_mem_p ATTRIBUTE_UNUSED)
1387 : {
1388 75233700 : int sregno, dregno;
1389 75233700 : rtx dest, src, dreg, sreg, new_reg, scratch_reg;
1390 75233700 : rtx_insn *before;
1391 75233700 : enum reg_class dclass, sclass, secondary_class;
1392 75233700 : secondary_reload_info sri;
1393 :
1394 75233700 : lra_assert (curr_insn_set != NULL_RTX);
1395 75233700 : dreg = dest = SET_DEST (curr_insn_set);
1396 75233700 : sreg = src = SET_SRC (curr_insn_set);
1397 75233700 : if (GET_CODE (dest) == SUBREG)
1398 1193231 : dreg = SUBREG_REG (dest);
1399 75233700 : if (GET_CODE (src) == SUBREG)
1400 1195312 : sreg = SUBREG_REG (src);
1401 75233700 : if (! (REG_P (dreg) || MEM_P (dreg)) || ! (REG_P (sreg) || MEM_P (sreg)))
1402 : return false;
1403 35314803 : sclass = dclass = NO_REGS;
1404 35314803 : if (REG_P (dreg))
1405 22722872 : dclass = get_reg_class (REGNO (dreg));
1406 22722872 : gcc_assert (dclass < LIM_REG_CLASSES && dclass >= NO_REGS);
1407 35314803 : if (dclass == ALL_REGS)
1408 : /* ALL_REGS is used for new pseudos created by transformations
1409 : like reload of SUBREG_REG (see function
1410 : simplify_operand_subreg). We don't know their class yet. We
1411 : should figure out the class from processing the insn
1412 : constraints not in this fast path function. Even if ALL_REGS
1413 : were a right class for the pseudo, secondary_... hooks usually
1414 : are not define for ALL_REGS. */
1415 : return false;
1416 35312572 : if (REG_P (sreg))
1417 19704565 : sclass = get_reg_class (REGNO (sreg));
1418 19704565 : gcc_assert (sclass < LIM_REG_CLASSES && sclass >= NO_REGS);
1419 35312572 : if (sclass == ALL_REGS)
1420 : /* See comments above. */
1421 : return false;
1422 35312572 : if (sclass == NO_REGS && dclass == NO_REGS)
1423 : return false;
1424 33848794 : if (targetm.secondary_memory_needed (GET_MODE (src), sclass, dclass)
1425 33848794 : && ((sclass != NO_REGS && dclass != NO_REGS)
1426 0 : || (GET_MODE (src)
1427 0 : != targetm.secondary_memory_needed_mode (GET_MODE (src)))))
1428 : {
1429 13246 : *sec_mem_p = true;
1430 13246 : return false;
1431 : }
1432 33835548 : if (! REG_P (dreg) || ! REG_P (sreg))
1433 : return false;
1434 7624318 : sri.prev_sri = NULL;
1435 7624318 : sri.icode = CODE_FOR_nothing;
1436 7624318 : sri.extra_cost = 0;
1437 7624318 : secondary_class = NO_REGS;
1438 : /* Set up hard register for a reload pseudo for hook
1439 : secondary_reload because some targets just ignore unassigned
1440 : pseudos in the hook. */
1441 7624318 : if (dclass != NO_REGS && lra_get_regno_hard_regno (REGNO (dreg)) < 0)
1442 : {
1443 2867552 : dregno = REGNO (dreg);
1444 2867552 : reg_renumber[dregno] = ira_class_hard_regs[dclass][0];
1445 : }
1446 : else
1447 : dregno = -1;
1448 7624318 : if (sclass != NO_REGS && lra_get_regno_hard_regno (REGNO (sreg)) < 0)
1449 : {
1450 1265790 : sregno = REGNO (sreg);
1451 1265790 : reg_renumber[sregno] = ira_class_hard_regs[sclass][0];
1452 : }
1453 : else
1454 : sregno = -1;
1455 7624318 : if (sclass != NO_REGS)
1456 3836935 : secondary_class
1457 7673870 : = (enum reg_class) targetm.secondary_reload (false, dest,
1458 : (reg_class_t) sclass,
1459 3836935 : GET_MODE (src), &sri);
1460 3836935 : if (sclass == NO_REGS
1461 3836935 : || ((secondary_class != NO_REGS || sri.icode != CODE_FOR_nothing)
1462 1353 : && dclass != NO_REGS))
1463 : {
1464 3787383 : enum reg_class old_sclass = secondary_class;
1465 3787383 : secondary_reload_info old_sri = sri;
1466 :
1467 3787383 : sri.prev_sri = NULL;
1468 3787383 : sri.icode = CODE_FOR_nothing;
1469 3787383 : sri.extra_cost = 0;
1470 3787383 : secondary_class
1471 7574766 : = (enum reg_class) targetm.secondary_reload (true, src,
1472 : (reg_class_t) dclass,
1473 3787383 : GET_MODE (src), &sri);
1474 : /* Check the target hook consistency. */
1475 3787383 : lra_assert
1476 : ((secondary_class == NO_REGS && sri.icode == CODE_FOR_nothing)
1477 : || (old_sclass == NO_REGS && old_sri.icode == CODE_FOR_nothing)
1478 : || (secondary_class == old_sclass && sri.icode == old_sri.icode));
1479 : }
1480 7624318 : if (sregno >= 0)
1481 1265790 : reg_renumber [sregno] = -1;
1482 7624318 : if (dregno >= 0)
1483 2867552 : reg_renumber [dregno] = -1;
1484 7624318 : if (secondary_class == NO_REGS && sri.icode == CODE_FOR_nothing)
1485 : return false;
1486 1354 : *change_p = true;
1487 1354 : new_reg = NULL_RTX;
1488 0 : if (secondary_class != NO_REGS)
1489 1354 : new_reg = lra_create_new_reg_with_unique_value (GET_MODE (src), NULL_RTX,
1490 : secondary_class, NULL,
1491 : "secondary");
1492 1354 : start_sequence ();
1493 1354 : if (sri.icode == CODE_FOR_nothing)
1494 1354 : lra_emit_move (new_reg, src);
1495 : else
1496 : {
1497 0 : enum reg_class scratch_class;
1498 :
1499 0 : scratch_class = (reg_class_from_constraints
1500 0 : (insn_data[sri.icode].operand[2].constraint));
1501 0 : scratch_reg = (lra_create_new_reg_with_unique_value
1502 0 : (insn_data[sri.icode].operand[2].mode, NULL_RTX,
1503 : scratch_class, NULL, "scratch"));
1504 0 : emit_insn (GEN_FCN (sri.icode) (new_reg != NULL_RTX ? new_reg : dest,
1505 : src, scratch_reg));
1506 : }
1507 1354 : before = end_sequence ();
1508 1354 : lra_process_new_insns (curr_insn, before, NULL, "Inserting the move");
1509 1354 : if (new_reg != NULL_RTX)
1510 1354 : SET_SRC (curr_insn_set) = new_reg;
1511 : else
1512 : {
1513 0 : if (lra_dump_file != NULL)
1514 : {
1515 0 : fprintf (lra_dump_file, "Deleting move %u\n", INSN_UID (curr_insn));
1516 0 : dump_insn_slim (lra_dump_file, curr_insn);
1517 : }
1518 0 : lra_set_insn_deleted (curr_insn);
1519 0 : return true;
1520 : }
1521 1354 : return false;
1522 : }
1523 :
1524 : /* The following data describe the result of process_alt_operands.
1525 : The data are used in curr_insn_transform to generate reloads. */
1526 :
1527 : /* The chosen reg classes which should be used for the corresponding
1528 : operands. */
1529 : static enum reg_class goal_alt[MAX_RECOG_OPERANDS];
1530 : /* Hard registers which cannot be a start hard register for the corresponding
1531 : operands. */
1532 : static HARD_REG_SET goal_alt_exclude_start_hard_regs[MAX_RECOG_OPERANDS];
1533 : /* True if the operand should be the same as another operand and that
1534 : other operand does not need a reload. */
1535 : static bool goal_alt_match_win[MAX_RECOG_OPERANDS];
1536 : /* True if the operand does not need a reload. */
1537 : static bool goal_alt_win[MAX_RECOG_OPERANDS];
1538 : /* True if the operand can be offsetable memory. */
1539 : static bool goal_alt_offmemok[MAX_RECOG_OPERANDS];
1540 : /* The number of an operand to which given operand can be matched to. */
1541 : static int goal_alt_matches[MAX_RECOG_OPERANDS];
1542 : /* The number of elements in the following array. */
1543 : static int goal_alt_dont_inherit_ops_num;
1544 : /* Numbers of operands whose reload pseudos should not be inherited. */
1545 : static int goal_alt_dont_inherit_ops[MAX_RECOG_OPERANDS];
1546 : /* True if we should try only this alternative for the next constraint sub-pass
1547 : to speed up the sub-pass. */
1548 : static bool goal_reuse_alt_p;
1549 : /* True if the insn commutative operands should be swapped. */
1550 : static bool goal_alt_swapped;
1551 : /* The chosen insn alternative. */
1552 : static int goal_alt_number;
1553 : /* True if output reload of the stack pointer should be generated. */
1554 : static bool goal_alt_out_sp_reload_p;
1555 :
1556 : /* True if the corresponding operand is the result of an equivalence
1557 : substitution. */
1558 : static bool equiv_substition_p[MAX_RECOG_OPERANDS];
1559 :
1560 : /* The following five variables are used to choose the best insn
1561 : alternative. They reflect final characteristics of the best
1562 : alternative. */
1563 :
1564 : /* Number of necessary reloads and overall cost reflecting the
1565 : previous value and other unpleasantness of the best alternative. */
1566 : static int best_losers, best_overall;
1567 : /* Overall number hard registers used for reloads. For example, on
1568 : some targets we need 2 general registers to reload DFmode and only
1569 : one floating point register. */
1570 : static int best_reload_nregs;
1571 : /* Overall number reflecting distances of previous reloading the same
1572 : value. The distances are counted from the current BB start. It is
1573 : used to improve inheritance chances. */
1574 : static int best_reload_sum;
1575 :
1576 : /* True if the current insn should have no correspondingly input or
1577 : output reloads. */
1578 : static bool no_input_reloads_p, no_output_reloads_p;
1579 :
1580 : /* True if we swapped the commutative operands in the current
1581 : insn. */
1582 : static int curr_swapped;
1583 :
1584 : /* if CHECK_ONLY_P is false, arrange for address element *LOC to be a
1585 : register of class CL. Add any input reloads to list BEFORE. AFTER
1586 : is nonnull if *LOC is an automodified value; handle that case by
1587 : adding the required output reloads to list AFTER. Return true if
1588 : the RTL was changed.
1589 :
1590 : if CHECK_ONLY_P is true, check that the *LOC is a correct address
1591 : register. Return false if the address register is correct. */
1592 : static bool
1593 34967315 : process_addr_reg (rtx *loc, bool check_only_p, rtx_insn **before, rtx_insn **after,
1594 : enum reg_class cl)
1595 : {
1596 34967315 : int regno;
1597 34967315 : enum reg_class rclass, new_class;
1598 34967315 : rtx reg;
1599 34967315 : rtx new_reg;
1600 34967315 : machine_mode mode;
1601 34967315 : bool subreg_p, before_p = false;
1602 :
1603 34967315 : subreg_p = GET_CODE (*loc) == SUBREG;
1604 34967315 : if (subreg_p)
1605 : {
1606 14234 : reg = SUBREG_REG (*loc);
1607 14234 : mode = GET_MODE (reg);
1608 :
1609 : /* For mode with size bigger than ptr_mode, there unlikely to be "mov"
1610 : between two registers with different classes, but there normally will
1611 : be "mov" which transfers element of vector register into the general
1612 : register, and this normally will be a subreg which should be reloaded
1613 : as a whole. This is particularly likely to be triggered when
1614 : -fno-split-wide-types specified. */
1615 14234 : if (!REG_P (reg)
1616 14234 : || in_class_p (reg, cl, &new_class)
1617 16432 : || known_le (GET_MODE_SIZE (mode), GET_MODE_SIZE (ptr_mode)))
1618 14234 : loc = &SUBREG_REG (*loc);
1619 : }
1620 :
1621 34967315 : reg = *loc;
1622 34967315 : mode = GET_MODE (reg);
1623 34967315 : if (! REG_P (reg))
1624 : {
1625 0 : if (check_only_p)
1626 : return true;
1627 : /* Always reload memory in an address even if the target supports
1628 : such addresses. */
1629 0 : new_reg = lra_create_new_reg_with_unique_value (mode, reg, cl, NULL,
1630 : "address");
1631 0 : before_p = true;
1632 : }
1633 : else
1634 : {
1635 34967315 : regno = REGNO (reg);
1636 34967315 : rclass = get_reg_class (regno);
1637 34967315 : if (! check_only_p
1638 34967315 : && (*loc = get_equiv_with_elimination (reg, curr_insn)) != reg)
1639 : {
1640 112792 : if (lra_dump_file != NULL)
1641 : {
1642 0 : fprintf (lra_dump_file,
1643 : "Changing pseudo %d in address of insn %u on equiv ",
1644 0 : REGNO (reg), INSN_UID (curr_insn));
1645 0 : dump_value_slim (lra_dump_file, *loc, 1);
1646 0 : fprintf (lra_dump_file, "\n");
1647 : }
1648 112792 : rtx new_equiv = copy_rtx (*loc);
1649 112792 : if (lra_pointer_equiv_set_in (*loc))
1650 107645 : lra_pointer_equiv_set_add (new_equiv);
1651 112792 : *loc = new_equiv;
1652 : }
1653 34967315 : if (*loc != reg || ! in_class_p (reg, cl, &new_class))
1654 : {
1655 474921 : if (check_only_p)
1656 : return true;
1657 474921 : reg = *loc;
1658 474921 : if (get_reload_reg (after == NULL ? OP_IN : OP_INOUT,
1659 : mode, reg, cl, NULL,
1660 : subreg_p, false, "address", &new_reg))
1661 : before_p = true;
1662 : }
1663 34492394 : else if (new_class != NO_REGS && rclass != new_class)
1664 : {
1665 463508 : if (check_only_p)
1666 : return true;
1667 463508 : lra_change_class (regno, new_class, " Change to", true);
1668 463508 : return false;
1669 : }
1670 : else
1671 : return false;
1672 : }
1673 0 : if (before_p)
1674 : {
1675 467120 : push_to_sequence (*before);
1676 467120 : lra_emit_move (new_reg, reg);
1677 467120 : *before = end_sequence ();
1678 : }
1679 474921 : *loc = new_reg;
1680 474921 : if (after != NULL)
1681 : {
1682 0 : start_sequence ();
1683 0 : lra_emit_move (before_p ? copy_rtx (reg) : reg, new_reg);
1684 0 : emit_insn (*after);
1685 0 : *after = end_sequence ();
1686 : }
1687 : return true;
1688 : }
1689 :
1690 : /* Insert move insn in simplify_operand_subreg. BEFORE returns
1691 : the insn to be inserted before curr insn. AFTER returns the
1692 : the insn to be inserted after curr insn. ORIGREG and NEWREG
1693 : are the original reg and new reg for reload. */
1694 : static void
1695 476 : insert_move_for_subreg (rtx_insn **before, rtx_insn **after, rtx origreg,
1696 : rtx newreg)
1697 : {
1698 476 : if (before)
1699 : {
1700 476 : push_to_sequence (*before);
1701 476 : lra_emit_move (newreg, origreg);
1702 476 : *before = end_sequence ();
1703 : }
1704 476 : if (after)
1705 : {
1706 0 : start_sequence ();
1707 0 : lra_emit_move (origreg, newreg);
1708 0 : emit_insn (*after);
1709 0 : *after = end_sequence ();
1710 : }
1711 476 : }
1712 :
1713 : static bool valid_address_p (machine_mode mode, rtx addr, addr_space_t as);
1714 : static bool process_address (int, bool, rtx_insn **, rtx_insn **);
1715 :
1716 : /* Make reloads for subreg in operand NOP with internal subreg mode
1717 : REG_MODE, add new reloads for further processing. Return true if
1718 : any change was done. */
1719 : static bool
1720 174838789 : simplify_operand_subreg (int nop, machine_mode reg_mode)
1721 : {
1722 174838789 : int hard_regno, inner_hard_regno;
1723 174838789 : rtx_insn *before, *after;
1724 174838789 : machine_mode mode, innermode;
1725 174838789 : rtx reg, new_reg;
1726 174838789 : rtx operand = *curr_id->operand_loc[nop];
1727 174838789 : enum reg_class regclass;
1728 174838789 : enum op_type type;
1729 :
1730 174838789 : before = after = NULL;
1731 :
1732 174838789 : if (GET_CODE (operand) != SUBREG)
1733 : return false;
1734 :
1735 3646747 : mode = GET_MODE (operand);
1736 3646747 : reg = SUBREG_REG (operand);
1737 3646747 : innermode = GET_MODE (reg);
1738 3646747 : type = curr_static_id->operand[nop].type;
1739 3646747 : if (MEM_P (reg))
1740 : {
1741 10966 : const bool addr_was_valid
1742 10966 : = valid_address_p (innermode, XEXP (reg, 0), MEM_ADDR_SPACE (reg));
1743 10966 : alter_subreg (curr_id->operand_loc[nop], false);
1744 10966 : rtx subst = *curr_id->operand_loc[nop];
1745 10966 : lra_assert (MEM_P (subst));
1746 10966 : const bool addr_is_valid = valid_address_p (GET_MODE (subst),
1747 : XEXP (subst, 0),
1748 10966 : MEM_ADDR_SPACE (subst));
1749 10966 : if (!addr_was_valid
1750 10966 : || addr_is_valid
1751 10966 : || ((get_constraint_type (lookup_constraint
1752 0 : (curr_static_id->operand[nop].constraint))
1753 : != CT_SPECIAL_MEMORY)
1754 : /* We still can reload address and if the address is
1755 : valid, we can remove subreg without reloading its
1756 : inner memory. */
1757 0 : && valid_address_p (GET_MODE (subst),
1758 0 : regno_reg_rtx
1759 : [ira_class_hard_regs
1760 0 : [base_reg_class (GET_MODE (subst),
1761 0 : MEM_ADDR_SPACE (subst),
1762 0 : ADDRESS, SCRATCH)][0]],
1763 0 : MEM_ADDR_SPACE (subst))))
1764 : {
1765 : /* If we change the address for a paradoxical subreg of memory, the
1766 : new address might violate the necessary alignment or the access
1767 : might be slow; take this into consideration. We need not worry
1768 : about accesses beyond allocated memory for paradoxical memory
1769 : subregs as we don't substitute such equiv memory (see processing
1770 : equivalences in function lra_constraints) and because for spilled
1771 : pseudos we allocate stack memory enough for the biggest
1772 : corresponding paradoxical subreg.
1773 :
1774 : However, do not blindly simplify a (subreg (mem ...)) for
1775 : WORD_REGISTER_OPERATIONS targets as this may lead to loading junk
1776 : data into a register when the inner is narrower than outer or
1777 : missing important data from memory when the inner is wider than
1778 : outer. This rule only applies to modes that are no wider than
1779 : a word.
1780 :
1781 : If valid memory becomes invalid after subreg elimination
1782 : and address might be different we still have to reload
1783 : memory.
1784 : */
1785 10966 : if ((! addr_was_valid
1786 : || addr_is_valid
1787 0 : || known_eq (GET_MODE_SIZE (mode), GET_MODE_SIZE (innermode)))
1788 10966 : && !(maybe_ne (GET_MODE_PRECISION (mode),
1789 10966 : GET_MODE_PRECISION (innermode))
1790 13139 : && known_le (GET_MODE_SIZE (mode), UNITS_PER_WORD)
1791 19186 : && known_le (GET_MODE_SIZE (innermode), UNITS_PER_WORD)
1792 : && WORD_REGISTER_OPERATIONS)
1793 23110 : && (!(MEM_ALIGN (subst) < GET_MODE_ALIGNMENT (mode)
1794 1178 : && targetm.slow_unaligned_access (mode, MEM_ALIGN (subst)))
1795 0 : || (MEM_ALIGN (reg) < GET_MODE_ALIGNMENT (innermode)
1796 0 : && targetm.slow_unaligned_access (innermode,
1797 0 : MEM_ALIGN (reg)))))
1798 10966 : return true;
1799 :
1800 0 : *curr_id->operand_loc[nop] = operand;
1801 :
1802 : /* But if the address was not valid, we cannot reload the MEM without
1803 : reloading the address first. */
1804 0 : if (!addr_was_valid)
1805 0 : process_address (nop, false, &before, &after);
1806 :
1807 : /* INNERMODE is fast, MODE slow. Reload the mem in INNERMODE. */
1808 0 : enum reg_class rclass
1809 0 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
1810 0 : if (get_reload_reg (curr_static_id->operand[nop].type, innermode,
1811 : reg, rclass, NULL,
1812 : true, false, "slow/invalid mem", &new_reg))
1813 : {
1814 0 : bool insert_before, insert_after;
1815 0 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
1816 :
1817 0 : insert_before = (type != OP_OUT
1818 0 : || partial_subreg_p (mode, innermode));
1819 0 : insert_after = type != OP_IN;
1820 0 : insert_move_for_subreg (insert_before ? &before : NULL,
1821 : insert_after ? &after : NULL,
1822 : reg, new_reg);
1823 : }
1824 0 : SUBREG_REG (operand) = new_reg;
1825 :
1826 : /* Convert to MODE. */
1827 0 : reg = operand;
1828 0 : rclass
1829 0 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
1830 0 : if (get_reload_reg (curr_static_id->operand[nop].type, mode, reg,
1831 : rclass, NULL,
1832 : true, false, "slow/invalid mem", &new_reg))
1833 : {
1834 0 : bool insert_before, insert_after;
1835 0 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
1836 :
1837 0 : insert_before = type != OP_OUT;
1838 0 : insert_after = type != OP_IN;
1839 0 : insert_move_for_subreg (insert_before ? &before : NULL,
1840 : insert_after ? &after : NULL,
1841 : reg, new_reg);
1842 : }
1843 0 : *curr_id->operand_loc[nop] = new_reg;
1844 0 : lra_process_new_insns (curr_insn, before, after,
1845 : "Inserting slow/invalid mem reload");
1846 0 : return true;
1847 : }
1848 :
1849 : /* If the address was valid and became invalid, prefer to reload
1850 : the memory. Typical case is when the index scale should
1851 : correspond the memory. */
1852 0 : *curr_id->operand_loc[nop] = operand;
1853 : /* Do not return false here as the MEM_P (reg) will be processed
1854 : later in this function. */
1855 : }
1856 3635781 : else if (REG_P (reg) && REGNO (reg) < FIRST_PSEUDO_REGISTER)
1857 : {
1858 67 : alter_subreg (curr_id->operand_loc[nop], false);
1859 67 : return true;
1860 : }
1861 3635714 : else if (CONSTANT_P (reg))
1862 : {
1863 : /* Try to simplify subreg of constant. It is usually result of
1864 : equivalence substitution. */
1865 43893 : if (innermode == VOIDmode
1866 43893 : && (innermode = original_subreg_reg_mode[nop]) == VOIDmode)
1867 0 : innermode = curr_static_id->operand[nop].mode;
1868 43893 : if ((new_reg = simplify_subreg (mode, reg, innermode,
1869 43893 : SUBREG_BYTE (operand))) != NULL_RTX)
1870 : {
1871 43458 : *curr_id->operand_loc[nop] = new_reg;
1872 43458 : return true;
1873 : }
1874 : }
1875 : /* Put constant into memory when we have mixed modes. It generates
1876 : a better code in most cases as it does not need a secondary
1877 : reload memory. It also prevents LRA looping when LRA is using
1878 : secondary reload memory again and again. */
1879 870 : if (CONSTANT_P (reg) && CONST_POOL_OK_P (reg_mode, reg)
1880 3592691 : && SCALAR_INT_MODE_P (reg_mode) != SCALAR_INT_MODE_P (mode))
1881 : {
1882 8 : SUBREG_REG (operand) = force_const_mem (reg_mode, reg);
1883 8 : alter_subreg (curr_id->operand_loc[nop], false);
1884 8 : return true;
1885 : }
1886 3592248 : auto fp_subreg_can_be_simplified_after_reload_p = [] (machine_mode innermode,
1887 : poly_uint64 offset,
1888 : machine_mode mode) {
1889 0 : reload_completed = 1;
1890 0 : bool res = simplify_subreg_regno (FRAME_POINTER_REGNUM,
1891 : innermode,
1892 0 : offset, mode) >= 0;
1893 0 : reload_completed = 0;
1894 0 : return res;
1895 : };
1896 : /* Force a reload of the SUBREG_REG if this is a constant or PLUS or
1897 : if there may be a problem accessing OPERAND in the outer
1898 : mode. */
1899 3592248 : if ((REG_P (reg)
1900 3591772 : && REGNO (reg) >= FIRST_PSEUDO_REGISTER
1901 3591772 : && (hard_regno = lra_get_regno_hard_regno (REGNO (reg))) >= 0
1902 : /* Don't reload paradoxical subregs because we could be looping
1903 : having repeatedly final regno out of hard regs range. */
1904 3032088 : && (hard_regno_nregs (hard_regno, innermode)
1905 3032088 : >= hard_regno_nregs (hard_regno, mode))
1906 3026970 : && simplify_subreg_regno (hard_regno, innermode,
1907 3026970 : SUBREG_BYTE (operand), mode) < 0
1908 : /* Exclude reloading of frame pointer in subreg if frame pointer can not
1909 : be simplified here only because the reload is not finished yet. */
1910 845 : && (hard_regno != FRAME_POINTER_REGNUM
1911 0 : || !fp_subreg_can_be_simplified_after_reload_p (innermode,
1912 0 : SUBREG_BYTE (operand),
1913 : mode))
1914 : /* Don't reload subreg for matching reload. It is actually
1915 : valid subreg in LRA. */
1916 845 : && ! LRA_SUBREG_P (operand))
1917 7184020 : || CONSTANT_P (reg) || GET_CODE (reg) == PLUS || MEM_P (reg))
1918 : {
1919 476 : enum reg_class rclass;
1920 :
1921 476 : if (REG_P (reg))
1922 : /* There is a big probability that we will get the same class
1923 : for the new pseudo and we will get the same insn which
1924 : means infinite looping. So spill the new pseudo. */
1925 : rclass = NO_REGS;
1926 : else
1927 : /* The class will be defined later in curr_insn_transform. */
1928 476 : rclass
1929 476 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
1930 :
1931 476 : if (get_reload_reg (curr_static_id->operand[nop].type, reg_mode, reg,
1932 : rclass, NULL,
1933 : true, false, "subreg reg", &new_reg))
1934 : {
1935 476 : bool insert_before, insert_after;
1936 476 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
1937 :
1938 952 : insert_before = (type != OP_OUT
1939 476 : || read_modify_subreg_p (operand));
1940 476 : insert_after = (type != OP_IN);
1941 952 : insert_move_for_subreg (insert_before ? &before : NULL,
1942 : insert_after ? &after : NULL,
1943 : reg, new_reg);
1944 : }
1945 476 : SUBREG_REG (operand) = new_reg;
1946 476 : lra_process_new_insns (curr_insn, before, after,
1947 : "Inserting subreg reload");
1948 476 : return true;
1949 : }
1950 : /* Force a reload for a paradoxical subreg. For paradoxical subreg,
1951 : IRA allocates hardreg to the inner pseudo reg according to its mode
1952 : instead of the outermode, so the size of the hardreg may not be enough
1953 : to contain the outermode operand, in that case we may need to insert
1954 : reload for the reg. For the following two types of paradoxical subreg,
1955 : we need to insert reload:
1956 : 1. If the op_type is OP_IN, and the hardreg could not be paired with
1957 : other hardreg to contain the outermode operand
1958 : (checked by in_hard_reg_set_p), we need to insert the reload.
1959 : 2. If the op_type is OP_OUT or OP_INOUT.
1960 :
1961 : Here is a paradoxical subreg example showing how the reload is generated:
1962 :
1963 : (insn 5 4 7 2 (set (reg:TI 106 [ __comp ])
1964 : (subreg:TI (reg:DI 107 [ __comp ]) 0)) {*movti_internal_rex64}
1965 :
1966 : In IRA, reg107 is allocated to a DImode hardreg. We use x86-64 as example
1967 : here, if reg107 is assigned to hardreg R15, because R15 is the last
1968 : hardreg, compiler cannot find another hardreg to pair with R15 to
1969 : contain TImode data. So we insert a TImode reload reg180 for it.
1970 : After reload is inserted:
1971 :
1972 : (insn 283 0 0 (set (subreg:DI (reg:TI 180 [orig:107 __comp ] [107]) 0)
1973 : (reg:DI 107 [ __comp ])) -1
1974 : (insn 5 4 7 2 (set (reg:TI 106 [ __comp ])
1975 : (subreg:TI (reg:TI 180 [orig:107 __comp ] [107]) 0)) {*movti_internal_rex64}
1976 :
1977 : Two reload hard registers will be allocated to reg180 to save TImode data
1978 : in LRA_assign.
1979 :
1980 : For LRA pseudos this should normally be handled by the biggest_mode
1981 : mechanism. However, it's possible for new uses of an LRA pseudo
1982 : to be introduced after we've allocated it, such as when undoing
1983 : inheritance, and the allocated register might not then be appropriate
1984 : for the new uses. */
1985 3591772 : else if (REG_P (reg)
1986 3591772 : && REGNO (reg) >= FIRST_PSEUDO_REGISTER
1987 3591772 : && paradoxical_subreg_p (operand)
1988 1044401 : && (inner_hard_regno = lra_get_regno_hard_regno (REGNO (reg))) >= 0
1989 953756 : && hard_regno_nregs (inner_hard_regno, mode) > 1
1990 3591772 : && ((hard_regno
1991 3596890 : = simplify_subreg_regno (inner_hard_regno, innermode,
1992 5118 : SUBREG_BYTE (operand), mode)) < 0
1993 5118 : || ((hard_regno_nregs (inner_hard_regno, innermode)
1994 5118 : < hard_regno_nregs (hard_regno, mode))
1995 10236 : && (regclass = lra_get_allocno_class (REGNO (reg)))
1996 5118 : && (type != OP_IN
1997 5118 : || !in_hard_reg_set_p (reg_class_contents[regclass],
1998 : mode, hard_regno)
1999 5118 : || overlaps_hard_reg_set_p (lra_no_alloc_regs,
2000 : mode, hard_regno)))))
2001 : {
2002 : /* The class will be defined later in curr_insn_transform. */
2003 0 : enum reg_class rclass
2004 0 : = (enum reg_class) targetm.preferred_reload_class (reg, ALL_REGS);
2005 :
2006 0 : if (get_reload_reg (curr_static_id->operand[nop].type, mode, reg,
2007 : rclass, NULL,
2008 : true, false, "paradoxical subreg", &new_reg))
2009 : {
2010 0 : rtx subreg;
2011 0 : bool insert_before, insert_after;
2012 :
2013 0 : PUT_MODE (new_reg, mode);
2014 0 : subreg = gen_lowpart_SUBREG (innermode, new_reg);
2015 0 : bitmap_set_bit (&lra_subreg_reload_pseudos, REGNO (new_reg));
2016 :
2017 0 : insert_before = (type != OP_OUT);
2018 0 : insert_after = (type != OP_IN);
2019 0 : insert_move_for_subreg (insert_before ? &before : NULL,
2020 : insert_after ? &after : NULL,
2021 : reg, subreg);
2022 : }
2023 0 : SUBREG_REG (operand) = new_reg;
2024 0 : lra_process_new_insns (curr_insn, before, after,
2025 : "Inserting paradoxical subreg reload");
2026 0 : return true;
2027 : }
2028 : return false;
2029 : }
2030 :
2031 : /* Return TRUE if X refers for a hard register from SET. */
2032 : static bool
2033 409292 : uses_hard_regs_p (rtx x, HARD_REG_SET set)
2034 : {
2035 409292 : int i, j, x_hard_regno;
2036 409292 : machine_mode mode;
2037 409292 : const char *fmt;
2038 409292 : enum rtx_code code;
2039 :
2040 409292 : if (x == NULL_RTX)
2041 : return false;
2042 409292 : code = GET_CODE (x);
2043 409292 : mode = GET_MODE (x);
2044 :
2045 409292 : if (code == SUBREG)
2046 : {
2047 : /* For all SUBREGs we want to check whether the full multi-register
2048 : overlaps the set. For normal SUBREGs this means 'get_hard_regno' of
2049 : the inner register, for paradoxical SUBREGs this means the
2050 : 'get_hard_regno' of the full SUBREG and for complete SUBREGs either is
2051 : fine. Use the wider mode for all cases. */
2052 2739 : rtx subreg = SUBREG_REG (x);
2053 2739 : mode = wider_subreg_mode (x);
2054 2739 : if (mode == GET_MODE (subreg))
2055 : {
2056 1703 : x = subreg;
2057 1703 : code = GET_CODE (x);
2058 : }
2059 : }
2060 :
2061 409292 : if (REG_P (x) || SUBREG_P (x))
2062 : {
2063 266976 : x_hard_regno = get_hard_regno (x);
2064 266976 : return (x_hard_regno >= 0
2065 266976 : && overlaps_hard_reg_set_p (set, mode, x_hard_regno));
2066 : }
2067 142316 : fmt = GET_RTX_FORMAT (code);
2068 369893 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
2069 : {
2070 229582 : if (fmt[i] == 'e')
2071 : {
2072 111861 : if (uses_hard_regs_p (XEXP (x, i), set))
2073 : return true;
2074 : }
2075 117721 : else if (fmt[i] == 'E')
2076 : {
2077 4398 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
2078 3980 : if (uses_hard_regs_p (XVECEXP (x, i, j), set))
2079 : return true;
2080 : }
2081 : }
2082 : return false;
2083 : }
2084 :
2085 : /* Return true if OP is a spilled pseudo. */
2086 : static inline bool
2087 80486377 : spilled_pseudo_p (rtx op)
2088 : {
2089 80486377 : return (REG_P (op)
2090 80486377 : && REGNO (op) >= FIRST_PSEUDO_REGISTER && in_mem_p (REGNO (op)));
2091 : }
2092 :
2093 : /* Return true if X is a general constant. */
2094 : static inline bool
2095 7836048 : general_constant_p (rtx x)
2096 : {
2097 7836048 : return CONSTANT_P (x) && (! flag_pic || LEGITIMATE_PIC_OPERAND_P (x));
2098 : }
2099 :
2100 : static bool
2101 24716878 : reg_in_class_p (rtx reg, enum reg_class cl)
2102 : {
2103 24716878 : if (cl == NO_REGS)
2104 1118208 : return get_reg_class (REGNO (reg)) == NO_REGS;
2105 23598670 : return in_class_p (reg, cl, NULL);
2106 : }
2107 :
2108 : /* Return true if SET of RCLASS contains no hard regs which can be
2109 : used in MODE. */
2110 : static bool
2111 3831108 : prohibited_class_reg_set_mode_p (enum reg_class rclass,
2112 : HARD_REG_SET &set,
2113 : machine_mode mode)
2114 : {
2115 3831108 : HARD_REG_SET temp;
2116 :
2117 7662216 : lra_assert (hard_reg_set_subset_p (reg_class_contents[rclass], set));
2118 3831108 : temp = set & ~lra_no_alloc_regs;
2119 3831108 : return (hard_reg_set_subset_p
2120 3831108 : (temp, ira_prohibited_class_mode_regs[rclass][mode]));
2121 : }
2122 :
2123 :
2124 : /* Used to check validity info about small class input operands. It
2125 : should be incremented at start of processing an insn
2126 : alternative. */
2127 : static unsigned int curr_small_class_check = 0;
2128 :
2129 : /* Update number of used inputs of class OP_CLASS for operand NOP
2130 : of alternative NALT. Return true if we have more such class operands
2131 : than the number of available regs. */
2132 : static bool
2133 392009393 : update_and_check_small_class_inputs (int nop, int nalt,
2134 : enum reg_class op_class)
2135 : {
2136 392009393 : static unsigned int small_class_check[LIM_REG_CLASSES];
2137 392009393 : static int small_class_input_nums[LIM_REG_CLASSES];
2138 :
2139 389128892 : if (SMALL_REGISTER_CLASS_P (op_class)
2140 : /* We are interesting in classes became small because of fixing
2141 : some hard regs, e.g. by an user through GCC options. */
2142 2987200 : && hard_reg_set_intersect_p (reg_class_contents[op_class],
2143 2987200 : ira_no_alloc_regs)
2144 392009432 : && (curr_static_id->operand[nop].type != OP_OUT
2145 33 : || TEST_BIT (curr_static_id->operand[nop].early_clobber_alts, nalt)))
2146 : {
2147 6 : if (small_class_check[op_class] == curr_small_class_check)
2148 0 : small_class_input_nums[op_class]++;
2149 : else
2150 : {
2151 6 : small_class_check[op_class] = curr_small_class_check;
2152 6 : small_class_input_nums[op_class] = 1;
2153 : }
2154 6 : if (small_class_input_nums[op_class] > ira_class_hard_regs_num[op_class])
2155 : return true;
2156 : }
2157 : return false;
2158 : }
2159 :
2160 : /* Print operand constraints for alternative ALT_NUMBER of the current
2161 : insn. */
2162 : static void
2163 4590 : print_curr_insn_alt (int alt_number)
2164 : {
2165 15917 : for (int i = 0; i < curr_static_id->n_operands; i++)
2166 : {
2167 11327 : const char *p = (curr_static_id->operand_alternative
2168 11327 : [alt_number * curr_static_id->n_operands + i].constraint);
2169 11327 : if (*p == '\0')
2170 220 : continue;
2171 11107 : fprintf (lra_dump_file, " (%d) ", i);
2172 39435 : for (; *p != '\0' && *p != ',' && *p != '#'; p++)
2173 17221 : fputc (*p, lra_dump_file);
2174 : }
2175 4590 : }
2176 :
2177 : /* Major function to choose the current insn alternative and what
2178 : operands should be reloaded and how. If ONLY_ALTERNATIVE is not
2179 : negative we should consider only this alternative. Return false if
2180 : we cannot choose the alternative or find how to reload the
2181 : operands. */
2182 : static bool
2183 89455499 : process_alt_operands (int only_alternative)
2184 : {
2185 89455499 : bool ok_p = false;
2186 89455499 : int nop, overall, nalt;
2187 89455499 : int n_alternatives = curr_static_id->n_alternatives;
2188 89455499 : int n_operands = curr_static_id->n_operands;
2189 : /* LOSERS counts the operands that don't fit this alternative and
2190 : would require loading. */
2191 89455499 : int losers;
2192 89455499 : int addr_losers;
2193 : /* REJECT is a count of how undesirable this alternative says it is
2194 : if any reloading is required. If the alternative matches exactly
2195 : then REJECT is ignored, but otherwise it gets this much counted
2196 : against it in addition to the reloading needed. */
2197 89455499 : int reject;
2198 : /* This is defined by '!' or '?' alternative constraint and added to
2199 : reject. But in some cases it can be ignored. */
2200 89455499 : int static_reject;
2201 89455499 : int op_reject;
2202 : /* The number of elements in the following array. */
2203 89455499 : int early_clobbered_regs_num;
2204 : /* Numbers of operands which are early clobber registers. */
2205 89455499 : int early_clobbered_nops[MAX_RECOG_OPERANDS];
2206 89455499 : enum reg_class curr_alt[MAX_RECOG_OPERANDS];
2207 89455499 : enum reg_class all_this_alternative;
2208 89455499 : int all_used_nregs, all_reload_nregs;
2209 89455499 : HARD_REG_SET curr_alt_set[MAX_RECOG_OPERANDS];
2210 89455499 : HARD_REG_SET curr_alt_exclude_start_hard_regs[MAX_RECOG_OPERANDS];
2211 89455499 : bool curr_alt_match_win[MAX_RECOG_OPERANDS];
2212 89455499 : bool curr_alt_win[MAX_RECOG_OPERANDS];
2213 89455499 : bool curr_alt_offmemok[MAX_RECOG_OPERANDS];
2214 89455499 : int curr_alt_matches[MAX_RECOG_OPERANDS];
2215 : /* The number of elements in the following array. */
2216 89455499 : int curr_alt_dont_inherit_ops_num;
2217 : /* Numbers of operands whose reload pseudos should not be inherited. */
2218 89455499 : int curr_alt_dont_inherit_ops[MAX_RECOG_OPERANDS];
2219 89455499 : bool curr_reuse_alt_p;
2220 : /* True if output stack pointer reload should be generated for the current
2221 : alternative. */
2222 89455499 : bool curr_alt_out_sp_reload_p;
2223 89455499 : bool curr_alt_class_change_p;
2224 89455499 : rtx op;
2225 : /* The register when the operand is a subreg of register, otherwise the
2226 : operand itself. */
2227 89455499 : rtx no_subreg_reg_operand[MAX_RECOG_OPERANDS];
2228 : /* The register if the operand is a register or subreg of register,
2229 : otherwise NULL. */
2230 89455499 : rtx operand_reg[MAX_RECOG_OPERANDS];
2231 89455499 : int hard_regno[MAX_RECOG_OPERANDS];
2232 89455499 : machine_mode biggest_mode[MAX_RECOG_OPERANDS];
2233 89455499 : int reload_nregs, reload_sum;
2234 89455499 : bool costly_p;
2235 89455499 : enum reg_class cl;
2236 89455499 : const HARD_REG_SET *cl_filter;
2237 89455499 : HARD_REG_SET hard_reg_constraint;
2238 :
2239 : /* Calculate some data common for all alternatives to speed up the
2240 : function. */
2241 297449034 : for (nop = 0; nop < n_operands; nop++)
2242 : {
2243 207993535 : rtx reg;
2244 :
2245 207993535 : op = no_subreg_reg_operand[nop] = *curr_id->operand_loc[nop];
2246 : /* The real hard regno of the operand after the allocation. */
2247 207993535 : hard_regno[nop] = get_hard_regno (op);
2248 :
2249 207993535 : operand_reg[nop] = reg = op;
2250 207993535 : biggest_mode[nop] = GET_MODE (op);
2251 207993535 : if (GET_CODE (op) == SUBREG)
2252 : {
2253 4113361 : biggest_mode[nop] = wider_subreg_mode (op);
2254 4113361 : operand_reg[nop] = reg = SUBREG_REG (op);
2255 : }
2256 207993535 : if (! REG_P (reg))
2257 87948498 : operand_reg[nop] = NULL_RTX;
2258 120045037 : else if (REGNO (reg) >= FIRST_PSEUDO_REGISTER
2259 141038100 : || ((int) REGNO (reg)
2260 20993063 : == lra_get_elimination_hard_regno (REGNO (reg))))
2261 117113861 : no_subreg_reg_operand[nop] = reg;
2262 : else
2263 2931176 : operand_reg[nop] = no_subreg_reg_operand[nop]
2264 : /* Just use natural mode for elimination result. It should
2265 : be enough for extra constraints hooks. */
2266 2931176 : = regno_reg_rtx[hard_regno[nop]];
2267 : }
2268 :
2269 : /* The constraints are made of several alternatives. Each operand's
2270 : constraint looks like foo,bar,... with commas separating the
2271 : alternatives. The first alternatives for all operands go
2272 : together, the second alternatives go together, etc.
2273 :
2274 : First loop over alternatives. */
2275 89455499 : alternative_mask preferred = curr_id->preferred_alternatives;
2276 89455499 : if (only_alternative >= 0)
2277 977711 : preferred &= ALTERNATIVE_BIT (only_alternative);
2278 :
2279 89455499 : bool prefer_memory_p = false;
2280 89455601 : repeat:
2281 377560204 : for (nalt = 0; nalt < n_alternatives; nalt++)
2282 : {
2283 : /* Loop over operands for one constraint alternative. */
2284 363869136 : if (!TEST_BIT (preferred, nalt))
2285 107965996 : continue;
2286 :
2287 255903140 : if (lra_dump_file != NULL)
2288 : {
2289 3403 : fprintf (lra_dump_file, " Considering alt=%d of insn %d: ",
2290 3403 : nalt, INSN_UID (curr_insn));
2291 3403 : print_curr_insn_alt (nalt);
2292 3403 : fprintf (lra_dump_file, "\n");
2293 : }
2294 :
2295 255903140 : bool matching_early_clobber[MAX_RECOG_OPERANDS];
2296 255903140 : curr_small_class_check++;
2297 255903140 : overall = losers = addr_losers = 0;
2298 255903140 : static_reject = reject = reload_nregs = reload_sum = 0;
2299 848818190 : for (nop = 0; nop < n_operands; nop++)
2300 : {
2301 592915050 : int inc = (curr_static_id
2302 592915050 : ->operand_alternative[nalt * n_operands + nop].reject);
2303 592915050 : if (lra_dump_file != NULL && inc != 0)
2304 53 : fprintf (lra_dump_file,
2305 : " Statically defined alt reject+=%d\n", inc);
2306 592915050 : static_reject += inc;
2307 592915050 : matching_early_clobber[nop] = 0;
2308 : }
2309 : reject += static_reject;
2310 : early_clobbered_regs_num = 0;
2311 : curr_alt_out_sp_reload_p = false;
2312 : curr_reuse_alt_p = true;
2313 : curr_alt_class_change_p = false;
2314 : all_this_alternative = NO_REGS;
2315 : all_used_nregs = all_reload_nregs = 0;
2316 662547892 : for (nop = 0; nop < n_operands; nop++)
2317 : {
2318 528414006 : const char *p;
2319 528414006 : char *end;
2320 528414006 : int len, c, m, i, opalt_num, this_alternative_matches;
2321 528414006 : bool win, did_match, offmemok, early_clobber_p;
2322 : /* false => this operand can be reloaded somehow for this
2323 : alternative. */
2324 528414006 : bool badop;
2325 : /* true => this operand can be reloaded if the alternative
2326 : allows regs. */
2327 528414006 : bool winreg;
2328 : /* True if a constant forced into memory would be OK for
2329 : this operand. */
2330 528414006 : bool constmemok;
2331 528414006 : enum reg_class this_alternative, this_costly_alternative;
2332 528414006 : HARD_REG_SET this_alternative_set, this_costly_alternative_set;
2333 528414006 : HARD_REG_SET this_alternative_exclude_start_hard_regs;
2334 528414006 : bool this_alternative_match_win, this_alternative_win;
2335 528414006 : bool this_alternative_offmemok;
2336 528414006 : bool scratch_p;
2337 528414006 : machine_mode mode;
2338 528414006 : enum constraint_num cn;
2339 528414006 : bool class_change_p = false;
2340 :
2341 528414006 : opalt_num = nalt * n_operands + nop;
2342 528414006 : if (curr_static_id->operand_alternative[opalt_num].anything_ok)
2343 : {
2344 : /* Fast track for no constraints at all. */
2345 14635359 : curr_alt[nop] = NO_REGS;
2346 14635359 : CLEAR_HARD_REG_SET (curr_alt_set[nop]);
2347 14635359 : curr_alt_win[nop] = true;
2348 14635359 : curr_alt_match_win[nop] = false;
2349 14635359 : curr_alt_offmemok[nop] = false;
2350 14635359 : curr_alt_matches[nop] = -1;
2351 14635359 : continue;
2352 : }
2353 :
2354 513778647 : op = no_subreg_reg_operand[nop];
2355 513778647 : mode = curr_operand_mode[nop];
2356 :
2357 513778647 : win = did_match = winreg = offmemok = constmemok = false;
2358 513778647 : badop = true;
2359 :
2360 513778647 : early_clobber_p = false;
2361 513778647 : p = curr_static_id->operand_alternative[opalt_num].constraint;
2362 :
2363 513778647 : this_costly_alternative = this_alternative = NO_REGS;
2364 : /* We update set of possible hard regs besides its class
2365 : because reg class might be inaccurate. For example,
2366 : union of LO_REGS (l), HI_REGS(h), and STACK_REG(k) in ARM
2367 : is translated in HI_REGS because classes are merged by
2368 : pairs and there is no accurate intermediate class. */
2369 2055114588 : CLEAR_HARD_REG_SET (this_alternative_set);
2370 1541335941 : CLEAR_HARD_REG_SET (this_costly_alternative_set);
2371 513778647 : CLEAR_HARD_REG_SET (this_alternative_exclude_start_hard_regs);
2372 513778647 : this_alternative_win = false;
2373 513778647 : this_alternative_match_win = false;
2374 513778647 : this_alternative_offmemok = false;
2375 513778647 : this_alternative_matches = -1;
2376 :
2377 : /* An empty constraint should be excluded by the fast
2378 : track. */
2379 513778647 : lra_assert (*p != 0 && *p != ',');
2380 :
2381 : op_reject = 0;
2382 : /* Scan this alternative's specs for this operand; set WIN
2383 : if the operand fits any letter in this alternative.
2384 : Otherwise, clear BADOP if this operand could fit some
2385 : letter after reloads, or set WINREG if this operand could
2386 : fit after reloads provided the constraint allows some
2387 : registers. */
2388 : costly_p = false;
2389 1305098439 : do
2390 : {
2391 1305098439 : switch ((c = *p, len = CONSTRAINT_LEN (c, p)), c)
2392 : {
2393 : case '\0':
2394 : len = 0;
2395 : break;
2396 491131562 : case ',':
2397 491131562 : c = '\0';
2398 491131562 : break;
2399 :
2400 176531 : case '&':
2401 176531 : early_clobber_p = true;
2402 176531 : break;
2403 :
2404 19268 : case '$':
2405 19268 : op_reject += LRA_MAX_REJECT;
2406 19268 : break;
2407 0 : case '^':
2408 0 : op_reject += LRA_LOSER_COST_FACTOR;
2409 0 : break;
2410 :
2411 0 : case '#':
2412 : /* Ignore rest of this alternative. */
2413 0 : c = '\0';
2414 0 : break;
2415 :
2416 56121768 : case '0': case '1': case '2': case '3': case '4':
2417 56121768 : case '5': case '6': case '7': case '8': case '9':
2418 56121768 : {
2419 56121768 : int m_hregno;
2420 56121768 : bool match_p;
2421 :
2422 56121768 : m = strtoul (p, &end, 10);
2423 56121768 : p = end;
2424 56121768 : len = 0;
2425 56121768 : lra_assert (nop > m);
2426 :
2427 : /* Reject matches if we don't know which operand is
2428 : bigger. This situation would arguably be a bug in
2429 : an .md pattern, but could also occur in a user asm. */
2430 168365304 : if (!ordered_p (GET_MODE_SIZE (biggest_mode[m]),
2431 56121768 : GET_MODE_SIZE (biggest_mode[nop])))
2432 : break;
2433 :
2434 : /* Don't match wrong asm insn operands for proper
2435 : diagnostic later. */
2436 56121768 : if (INSN_CODE (curr_insn) < 0
2437 33470 : && (curr_operand_mode[m] == BLKmode
2438 33469 : || curr_operand_mode[nop] == BLKmode)
2439 1 : && curr_operand_mode[m] != curr_operand_mode[nop])
2440 : break;
2441 :
2442 56121767 : m_hregno = get_hard_regno (*curr_id->operand_loc[m]);
2443 : /* We are supposed to match a previous operand.
2444 : If we do, we win if that one did. If we do
2445 : not, count both of the operands as losers.
2446 : (This is too conservative, since most of the
2447 : time only a single reload insn will be needed
2448 : to make the two operands win. As a result,
2449 : this alternative may be rejected when it is
2450 : actually desirable.) */
2451 56121767 : match_p = false;
2452 56121767 : if (operands_match_p (*curr_id->operand_loc[nop],
2453 56121767 : *curr_id->operand_loc[m], m_hregno))
2454 : {
2455 : /* We should reject matching of an early
2456 : clobber operand if the matching operand is
2457 : not dying in the insn. */
2458 14866502 : if (!TEST_BIT (curr_static_id->operand[m]
2459 : .early_clobber_alts, nalt)
2460 18157 : || operand_reg[nop] == NULL_RTX
2461 14884659 : || (find_regno_note (curr_insn, REG_DEAD,
2462 : REGNO (op))
2463 4346 : || REGNO (op) == REGNO (operand_reg[m])))
2464 14866502 : match_p = true;
2465 : }
2466 14866502 : if (match_p)
2467 : {
2468 : /* If we are matching a non-offsettable
2469 : address where an offsettable address was
2470 : expected, then we must reject this
2471 : combination, because we can't reload
2472 : it. */
2473 14866502 : if (curr_alt_offmemok[m]
2474 1490 : && MEM_P (*curr_id->operand_loc[m])
2475 0 : && curr_alt[m] == NO_REGS && ! curr_alt_win[m])
2476 0 : continue;
2477 : }
2478 : else
2479 : {
2480 : /* If the operands do not match and one
2481 : operand is INOUT, we can not match them.
2482 : Try other possibilities, e.g. other
2483 : alternatives or commutative operand
2484 : exchange. */
2485 41255265 : if (curr_static_id->operand[nop].type == OP_INOUT
2486 41255265 : || curr_static_id->operand[m].type == OP_INOUT)
2487 : break;
2488 : /* Operands don't match. For asm if the operands
2489 : are different user defined explicit hard
2490 : registers, then we cannot make them match
2491 : when one is early clobber operand. */
2492 41254835 : if ((REG_P (*curr_id->operand_loc[nop])
2493 25843960 : || SUBREG_P (*curr_id->operand_loc[nop]))
2494 15991032 : && (REG_P (*curr_id->operand_loc[m])
2495 200746 : || SUBREG_P (*curr_id->operand_loc[m]))
2496 15899500 : && INSN_CODE (curr_insn) < 0)
2497 : {
2498 594 : rtx nop_reg = *curr_id->operand_loc[nop];
2499 594 : if (SUBREG_P (nop_reg))
2500 0 : nop_reg = SUBREG_REG (nop_reg);
2501 594 : rtx m_reg = *curr_id->operand_loc[m];
2502 594 : if (SUBREG_P (m_reg))
2503 0 : m_reg = SUBREG_REG (m_reg);
2504 :
2505 594 : if (REG_P (nop_reg)
2506 594 : && HARD_REGISTER_P (nop_reg)
2507 0 : && REG_USERVAR_P (nop_reg)
2508 0 : && REG_P (m_reg)
2509 0 : && HARD_REGISTER_P (m_reg)
2510 594 : && REG_USERVAR_P (m_reg))
2511 : {
2512 : int i;
2513 :
2514 0 : for (i = 0; i < early_clobbered_regs_num; i++)
2515 0 : if (m == early_clobbered_nops[i])
2516 : break;
2517 0 : if (i < early_clobbered_regs_num
2518 0 : || early_clobber_p)
2519 : break;
2520 : }
2521 : }
2522 : /* Both operands must allow a reload register,
2523 : otherwise we cannot make them match. */
2524 41254835 : if (curr_alt[m] == NO_REGS)
2525 : break;
2526 : /* Retroactively mark the operand we had to
2527 : match as a loser, if it wasn't already and
2528 : it wasn't matched to a register constraint
2529 : (e.g it might be matched by memory). */
2530 41230395 : if (curr_alt_win[m]
2531 40388333 : && (operand_reg[m] == NULL_RTX
2532 39884578 : || hard_regno[m] < 0))
2533 : {
2534 1268509 : if (lra_dump_file != NULL)
2535 9 : fprintf
2536 9 : (lra_dump_file,
2537 : " %d Matched operand reload: "
2538 : "losers++\n", m);
2539 1268509 : losers++;
2540 1268509 : reload_nregs
2541 1268509 : += (ira_reg_class_max_nregs[curr_alt[m]]
2542 1268509 : [GET_MODE (*curr_id->operand_loc[m])]);
2543 : }
2544 :
2545 : /* Prefer matching earlyclobber alternative as
2546 : it results in less hard regs required for
2547 : the insn than a non-matching earlyclobber
2548 : alternative. */
2549 41230395 : if (TEST_BIT (curr_static_id->operand[m]
2550 : .early_clobber_alts, nalt))
2551 : {
2552 17821 : if (lra_dump_file != NULL)
2553 0 : fprintf
2554 0 : (lra_dump_file,
2555 : " %d Matching earlyclobber alt:"
2556 : " reject--\n",
2557 : nop);
2558 17821 : if (!matching_early_clobber[m])
2559 : {
2560 17821 : reject--;
2561 17821 : matching_early_clobber[m] = 1;
2562 : }
2563 : }
2564 : /* Otherwise we prefer no matching
2565 : alternatives because it gives more freedom
2566 : in RA. */
2567 41212574 : else if (operand_reg[nop] == NULL_RTX
2568 41212574 : || (find_regno_note (curr_insn, REG_DEAD,
2569 15966144 : REGNO (operand_reg[nop]))
2570 : == NULL_RTX))
2571 : {
2572 36287770 : if (lra_dump_file != NULL)
2573 912 : fprintf
2574 912 : (lra_dump_file,
2575 : " %d Matching alt: reject+=2\n",
2576 : nop);
2577 36287770 : reject += 2;
2578 : }
2579 : }
2580 : /* If we have to reload this operand and some
2581 : previous operand also had to match the same
2582 : thing as this operand, we don't know how to do
2583 : that. */
2584 56096897 : if (!match_p || !curr_alt_win[m])
2585 : {
2586 86601653 : for (i = 0; i < nop; i++)
2587 45265712 : if (curr_alt_matches[i] == m)
2588 : break;
2589 41335942 : if (i < nop)
2590 : break;
2591 : }
2592 : else
2593 : did_match = true;
2594 :
2595 56096896 : this_alternative_matches = m;
2596 : /* This can be fixed with reloads if the operand
2597 : we are supposed to match can be fixed with
2598 : reloads. */
2599 56096896 : badop = false;
2600 56096896 : this_alternative = curr_alt[m];
2601 56096896 : this_alternative_set = curr_alt_set[m];
2602 56096896 : this_alternative_exclude_start_hard_regs
2603 56096896 : = curr_alt_exclude_start_hard_regs[m];
2604 56096896 : winreg = this_alternative != NO_REGS;
2605 56096896 : break;
2606 : }
2607 :
2608 11708045 : case 'g':
2609 11708045 : if (MEM_P (op)
2610 7836048 : || general_constant_p (op)
2611 16274350 : || spilled_pseudo_p (op))
2612 : win = true;
2613 11708045 : if (REG_P (op) && prefer_memory_p)
2614 : {
2615 11708045 : badop = false;
2616 11708045 : offmemok = true;
2617 : }
2618 11708045 : cl = GENERAL_REGS;
2619 11708045 : cl_filter = nullptr;
2620 11708045 : goto reg;
2621 :
2622 1140 : case '{':
2623 1140 : {
2624 1140 : int regno = decode_hard_reg_constraint (p);
2625 1140 : gcc_assert (regno >= 0);
2626 1140 : cl = NO_REGS;
2627 1140 : int nregs = hard_regno_nregs (regno, mode);
2628 2280 : for (int i = 0; i < nregs; ++i)
2629 1140 : cl = reg_class_superunion[cl][REGNO_REG_CLASS (regno + i)];
2630 1140 : CLEAR_HARD_REG_SET (hard_reg_constraint);
2631 1140 : SET_HARD_REG_BIT (hard_reg_constraint, regno);
2632 1140 : cl_filter = &hard_reg_constraint;
2633 1140 : goto reg;
2634 : }
2635 :
2636 723293040 : default:
2637 723293040 : cn = lookup_constraint (p);
2638 723293040 : switch (get_constraint_type (cn))
2639 : {
2640 478540014 : case CT_REGISTER:
2641 478540014 : cl = reg_class_for_constraint (cn);
2642 351866904 : if (cl != NO_REGS)
2643 : {
2644 342456864 : cl_filter = get_register_filter (cn);
2645 342456864 : goto reg;
2646 : }
2647 : break;
2648 :
2649 2080144 : case CT_CONST_INT:
2650 2080144 : if (CONST_INT_P (op)
2651 2080144 : && insn_const_int_ok_for_constraint (INTVAL (op), cn))
2652 : win = true;
2653 : break;
2654 :
2655 110295429 : case CT_MEMORY:
2656 110295429 : case CT_RELAXED_MEMORY:
2657 110295429 : if (MEM_P (op)
2658 110295429 : && satisfies_memory_constraint_p (op, cn))
2659 : win = true;
2660 74862069 : else if (spilled_pseudo_p (op))
2661 44826803 : win = true;
2662 :
2663 : /* If we didn't already win, we can reload constants
2664 : via force_const_mem or put the pseudo value into
2665 : memory, or make other memory by reloading the
2666 : address like for 'o'. */
2667 115354721 : if (CONST_POOL_OK_P (mode, op)
2668 105235973 : || MEM_P (op) || REG_P (op)
2669 : /* We can restore the equiv insn by a
2670 : reload. */
2671 110866564 : || equiv_substition_p[nop])
2672 110260289 : badop = false;
2673 : constmemok = true;
2674 : offmemok = true;
2675 : break;
2676 :
2677 1664180 : case CT_ADDRESS:
2678 : /* An asm operand with an address constraint
2679 : that doesn't satisfy address_operand has
2680 : is_address cleared, so that we don't try to
2681 : make a non-address fit. */
2682 1664180 : if (!curr_static_id->operand[nop].is_address)
2683 : break;
2684 : /* If we didn't already win, we can reload the address
2685 : into a base register. */
2686 1664161 : if (satisfies_address_constraint_p (op, cn))
2687 1664161 : win = true;
2688 1664161 : cl = base_reg_class (VOIDmode, ADDR_SPACE_GENERIC,
2689 : ADDRESS, SCRATCH);
2690 1664161 : cl_filter = nullptr;
2691 1664161 : badop = false;
2692 1664161 : goto reg;
2693 :
2694 129493273 : case CT_FIXED_FORM:
2695 129493273 : if (constraint_satisfied_p (op, cn))
2696 1305098439 : win = true;
2697 : break;
2698 :
2699 1220000 : case CT_SPECIAL_MEMORY:
2700 1220000 : if (satisfies_memory_constraint_p (op, cn))
2701 : win = true;
2702 1058003 : else if (spilled_pseudo_p (op))
2703 : {
2704 1305098439 : curr_reuse_alt_p = false;
2705 1305098439 : win = true;
2706 : }
2707 : break;
2708 : }
2709 : break;
2710 :
2711 355830210 : reg:
2712 355830210 : if (mode == BLKmode)
2713 : break;
2714 355830192 : this_alternative = reg_class_subunion[this_alternative][cl];
2715 355830192 : if (hard_reg_set_subset_p (this_alternative_set,
2716 355830192 : reg_class_contents[cl]))
2717 355826567 : this_alternative_exclude_start_hard_regs
2718 355826567 : = ira_exclude_class_mode_regs[cl][mode];
2719 3625 : else if (!hard_reg_set_subset_p (reg_class_contents[cl],
2720 : this_alternative_set))
2721 3624 : this_alternative_exclude_start_hard_regs
2722 1067494200 : |= ira_exclude_class_mode_regs[cl][mode];
2723 355830192 : this_alternative_set |= reg_class_contents[cl];
2724 355830192 : if (cl_filter)
2725 2280 : this_alternative_exclude_start_hard_regs |= ~*cl_filter;
2726 355830192 : if (costly_p)
2727 : {
2728 21141327 : this_costly_alternative
2729 21141327 : = reg_class_subunion[this_costly_alternative][cl];
2730 21141327 : this_costly_alternative_set |= reg_class_contents[cl];
2731 : }
2732 355830192 : winreg = true;
2733 355830192 : if (REG_P (op))
2734 : {
2735 226815544 : rtx orig_op = *curr_id->operand_loc[nop];
2736 6646219 : if (GET_CODE (orig_op) == SUBREG && HARD_REGISTER_P (op)
2737 226815634 : && !targetm.hard_regno_mode_ok (REGNO (op),
2738 90 : GET_MODE(orig_op)))
2739 : break;
2740 :
2741 226815544 : tree decl;
2742 :
2743 226815544 : if (hard_regno[nop] >= 0
2744 191290005 : && in_hard_reg_set_p (this_alternative_set,
2745 : mode, hard_regno[nop])
2746 173247478 : && (!cl_filter
2747 590 : || TEST_HARD_REG_BIT (*cl_filter,
2748 : hard_regno[nop]))
2749 400063016 : && ((REG_ATTRS (op) && (decl = REG_EXPR (op)) != NULL
2750 96403172 : && VAR_P (decl) && DECL_HARD_REGISTER (decl))
2751 173244215 : || !(TEST_HARD_REG_BIT
2752 173244215 : (this_alternative_exclude_start_hard_regs,
2753 : hard_regno[nop]))))
2754 : win = true;
2755 53568079 : else if (hard_regno[nop] < 0 && !prefer_memory_p)
2756 : {
2757 35525403 : if (in_class_p (op, this_alternative, NULL))
2758 : win = true;
2759 26661077 : else if (in_class_p (op, this_alternative, NULL, true))
2760 : {
2761 1305098439 : class_change_p = true;
2762 1305098439 : win = true;
2763 : }
2764 : }
2765 : }
2766 : break;
2767 : }
2768 1305098439 : if (c != ' ' && c != '\t')
2769 1305098439 : costly_p = c == '*';
2770 : }
2771 1305098439 : while ((p += len), c);
2772 :
2773 1027557294 : scratch_p = (operand_reg[nop] != NULL_RTX
2774 513778647 : && ira_former_scratch_p (REGNO (operand_reg[nop])));
2775 : /* Record which operands fit this alternative. */
2776 513778647 : if (win)
2777 : {
2778 276887776 : if (early_clobber_p
2779 276744991 : || curr_static_id->operand[nop].type != OP_OUT)
2780 : {
2781 121036766 : if (winreg)
2782 100058658 : all_used_nregs
2783 100058658 : += ira_reg_class_min_nregs[this_alternative][mode];
2784 121036766 : all_this_alternative
2785 121036766 : = (reg_class_subunion
2786 121036766 : [all_this_alternative][this_alternative]);
2787 : }
2788 276887776 : this_alternative_win = true;
2789 276887776 : if (class_change_p)
2790 : {
2791 251925 : curr_alt_class_change_p = true;
2792 251925 : if (lra_dump_file != NULL)
2793 10 : fprintf (lra_dump_file,
2794 : " %d Narrowing class: reject+=3\n",
2795 : nop);
2796 251925 : reject += 3;
2797 : }
2798 276887776 : if (operand_reg[nop] != NULL_RTX)
2799 : {
2800 193177896 : if (hard_regno[nop] >= 0)
2801 : {
2802 173191196 : if (in_hard_reg_set_p (this_costly_alternative_set,
2803 : mode, hard_regno[nop]))
2804 : {
2805 776513 : if (lra_dump_file != NULL)
2806 21 : fprintf (lra_dump_file,
2807 : " %d Costly set: reject++\n",
2808 : nop);
2809 776513 : reject++;
2810 : }
2811 : }
2812 : else
2813 : {
2814 : /* Prefer won reg to spilled pseudo under other
2815 : equal conditions for possible inheritance. */
2816 19986700 : if (! scratch_p)
2817 : {
2818 19982099 : if (lra_dump_file != NULL)
2819 59 : fprintf
2820 59 : (lra_dump_file,
2821 : " %d Non pseudo reload: reject++\n",
2822 : nop);
2823 19982099 : reject++;
2824 : }
2825 19986700 : if (in_class_p (operand_reg[nop],
2826 : this_costly_alternative, NULL, true))
2827 : {
2828 134631 : if (lra_dump_file != NULL)
2829 0 : fprintf
2830 0 : (lra_dump_file,
2831 : " %d Non pseudo costly reload:"
2832 : " reject++\n",
2833 : nop);
2834 134631 : reject++;
2835 : }
2836 : }
2837 : /* We simulate the behavior of old reload here.
2838 : Although scratches need hard registers and it
2839 : might result in spilling other pseudos, no reload
2840 : insns are generated for the scratches. So it
2841 : might cost something but probably less than old
2842 : reload pass believes. */
2843 193177896 : if (scratch_p)
2844 : {
2845 116583 : if (lra_dump_file != NULL)
2846 6 : fprintf (lra_dump_file,
2847 : " %d Scratch win: reject+=2\n",
2848 : nop);
2849 116583 : reject += 2;
2850 : }
2851 : }
2852 : }
2853 236890871 : else if (did_match)
2854 : this_alternative_match_win = true;
2855 : else
2856 : {
2857 222129916 : if (prefer_memory_p && offmemok)
2858 : {
2859 0 : winreg = false;
2860 0 : this_alternative = NO_REGS;
2861 : }
2862 :
2863 222129916 : int const_to_mem = 0;
2864 222129916 : bool no_regs_p;
2865 :
2866 222129916 : reject += op_reject;
2867 : /* Mark output reload of the stack pointer. */
2868 222129916 : if (op == stack_pointer_rtx
2869 56059 : && curr_static_id->operand[nop].type != OP_IN)
2870 222129916 : curr_alt_out_sp_reload_p = true;
2871 :
2872 : /* If this alternative asks for a specific reg class, see if there
2873 : is at least one allocatable register in that class. */
2874 222129916 : no_regs_p
2875 387254290 : = (this_alternative == NO_REGS
2876 222129916 : || (hard_reg_set_subset_p
2877 330248770 : (reg_class_contents[this_alternative],
2878 : lra_no_alloc_regs)));
2879 :
2880 : /* For asms, verify that the class for this alternative is possible
2881 : for the mode that is specified. */
2882 165124374 : if (!no_regs_p && INSN_CODE (curr_insn) < 0)
2883 : {
2884 : int i;
2885 70550 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
2886 70548 : if (targetm.hard_regno_mode_ok (i, mode)
2887 70548 : && in_hard_reg_set_p (reg_class_contents[this_alternative],
2888 : mode, i))
2889 : break;
2890 20564 : if (i == FIRST_PSEUDO_REGISTER)
2891 222129916 : winreg = false;
2892 : }
2893 :
2894 : /* If this operand accepts a register, and if the
2895 : register class has at least one allocatable register,
2896 : then this operand can be reloaded. */
2897 222129916 : if (winreg && !no_regs_p)
2898 : badop = false;
2899 :
2900 57005544 : if (badop)
2901 : {
2902 47709302 : if (lra_dump_file != NULL)
2903 606 : fprintf (lra_dump_file,
2904 : " Bad operand -- refuse\n");
2905 121769254 : goto fail;
2906 : }
2907 :
2908 174420614 : if (this_alternative != NO_REGS)
2909 : {
2910 165124373 : HARD_REG_SET available_regs
2911 165124373 : = (reg_class_contents[this_alternative]
2912 165124373 : & ~((ira_prohibited_class_mode_regs
2913 165124373 : [this_alternative][mode])
2914 165124373 : | lra_no_alloc_regs));
2915 330248746 : if (!hard_reg_set_empty_p (available_regs))
2916 : {
2917 165122667 : if (early_clobber_p
2918 165088921 : || curr_static_id->operand[nop].type != OP_OUT)
2919 : {
2920 86607920 : all_reload_nregs
2921 86607920 : += ira_reg_class_min_nregs[this_alternative][mode];
2922 86607920 : all_this_alternative
2923 86607920 : = (reg_class_subunion
2924 86607920 : [all_this_alternative][this_alternative]);
2925 : }
2926 : }
2927 : else
2928 : {
2929 : /* There are no hard regs holding a value of given
2930 : mode. */
2931 1706 : if (offmemok)
2932 : {
2933 166 : this_alternative = NO_REGS;
2934 166 : if (lra_dump_file != NULL)
2935 0 : fprintf (lra_dump_file,
2936 : " %d Using memory because of"
2937 : " a bad mode: reject+=2\n",
2938 : nop);
2939 166 : reject += 2;
2940 : }
2941 : else
2942 : {
2943 1540 : if (lra_dump_file != NULL)
2944 0 : fprintf (lra_dump_file,
2945 : " Wrong mode -- refuse\n");
2946 1540 : goto fail;
2947 : }
2948 : }
2949 : }
2950 :
2951 : /* If not assigned pseudo has a class which a subset of
2952 : required reg class, it is a less costly alternative
2953 : as the pseudo still can get a hard reg of necessary
2954 : class. */
2955 165122833 : if (! no_regs_p && REG_P (op) && hard_regno[nop] < 0
2956 21326657 : && (cl = get_reg_class (REGNO (op))) != NO_REGS
2957 177522208 : && ira_class_subset_p[this_alternative][cl])
2958 : {
2959 1066 : if (lra_dump_file != NULL)
2960 0 : fprintf
2961 0 : (lra_dump_file,
2962 : " %d Super set class reg: reject-=3\n", nop);
2963 1066 : reject -= 3;
2964 : }
2965 :
2966 174419074 : this_alternative_offmemok = offmemok;
2967 174419074 : if (this_costly_alternative != NO_REGS)
2968 : {
2969 18981090 : if (lra_dump_file != NULL)
2970 25 : fprintf (lra_dump_file,
2971 : " %d Costly loser: reject++\n", nop);
2972 18981090 : reject++;
2973 : }
2974 : /* If the operand is dying, has a matching constraint,
2975 : and satisfies constraints of the matched operand
2976 : which failed to satisfy the own constraints, most probably
2977 : the reload for this operand will be gone. */
2978 174419074 : if (this_alternative_matches >= 0
2979 41318536 : && !curr_alt_win[this_alternative_matches]
2980 947133 : && REG_P (op)
2981 696125 : && find_regno_note (curr_insn, REG_DEAD, REGNO (op))
2982 175133909 : && (hard_regno[nop] >= 0
2983 374736 : ? in_hard_reg_set_p (this_alternative_set,
2984 : mode, hard_regno[nop])
2985 34637 : : in_class_p (op, this_alternative, NULL)))
2986 : {
2987 226874 : if (lra_dump_file != NULL)
2988 1 : fprintf
2989 1 : (lra_dump_file,
2990 : " %d Dying matched operand reload: reject++\n",
2991 : nop);
2992 226874 : reject++;
2993 : }
2994 : else
2995 : {
2996 : /* Strict_low_part requires to reload the register
2997 : not the sub-register. In this case we should
2998 : check that a final reload hard reg can hold the
2999 : value mode. */
3000 174192200 : if (curr_static_id->operand[nop].strict_low
3001 117 : && REG_P (op)
3002 110 : && hard_regno[nop] < 0
3003 84 : && GET_CODE (*curr_id->operand_loc[nop]) == SUBREG
3004 84 : && ira_class_hard_regs_num[this_alternative] > 0
3005 174192284 : && (!targetm.hard_regno_mode_ok
3006 84 : (ira_class_hard_regs[this_alternative][0],
3007 84 : GET_MODE (*curr_id->operand_loc[nop]))))
3008 : {
3009 0 : if (lra_dump_file != NULL)
3010 0 : fprintf
3011 0 : (lra_dump_file,
3012 : " Strict low subreg reload -- refuse\n");
3013 0 : goto fail;
3014 : }
3015 174192200 : if (lra_dump_file != NULL)
3016 2177 : fprintf
3017 2177 : (lra_dump_file,
3018 : " %d Operand reload: losers++\n", nop);
3019 174192200 : losers++;
3020 : }
3021 174419074 : if (operand_reg[nop] != NULL_RTX
3022 : /* Output operands and matched input operands are
3023 : not inherited. The following conditions do not
3024 : exactly describe the previous statement but they
3025 : are pretty close. */
3026 62699439 : && curr_static_id->operand[nop].type != OP_OUT
3027 27703505 : && (this_alternative_matches < 0
3028 16056338 : || curr_static_id->operand[nop].type != OP_IN))
3029 : {
3030 11647167 : int last_reload = (lra_reg_info[ORIGINAL_REGNO
3031 11647167 : (operand_reg[nop])]
3032 11647167 : .last_reload);
3033 :
3034 : /* The value of reload_sum has sense only if we
3035 : process insns in their order. It happens only on
3036 : the first constraints sub-pass when we do most of
3037 : reload work. */
3038 11647167 : if (lra_constraint_iter == 1 && last_reload > bb_reload_num)
3039 2526317 : reload_sum += last_reload - bb_reload_num;
3040 : }
3041 : /* If this is a constant that is reloaded into the
3042 : desired class by copying it to memory first, count
3043 : that as another reload. This is consistent with
3044 : other code and is required to avoid choosing another
3045 : alternative when the constant is moved into memory.
3046 : Note that the test here is precisely the same as in
3047 : the code below that calls force_const_mem. */
3048 224217365 : if (CONST_POOL_OK_P (mode, op)
3049 224217436 : && ((targetm.preferred_reload_class
3050 49798362 : (op, this_alternative) == NO_REGS)
3051 48289360 : || no_input_reloads_p))
3052 : {
3053 1509002 : const_to_mem = 1;
3054 1509002 : if (! no_regs_p)
3055 : {
3056 713197 : if (lra_dump_file != NULL)
3057 0 : fprintf
3058 0 : (lra_dump_file,
3059 : " %d Constant reload through memory: "
3060 : "losers++\n", nop);
3061 713197 : losers++;
3062 : }
3063 : }
3064 :
3065 : /* Alternative loses if it requires a type of reload not
3066 : permitted for this insn. We can always reload
3067 : objects with a REG_UNUSED note. */
3068 174419074 : if ((curr_static_id->operand[nop].type != OP_IN
3069 84655018 : && no_output_reloads_p
3070 0 : && ! find_reg_note (curr_insn, REG_UNUSED, op)
3071 0 : && ! scratch_p)
3072 174419074 : || (curr_static_id->operand[nop].type != OP_OUT
3073 89764218 : && no_input_reloads_p && ! const_to_mem)
3074 348838148 : || (this_alternative_matches >= 0
3075 41318536 : && (no_input_reloads_p
3076 41318536 : || (no_output_reloads_p
3077 0 : && (curr_static_id->operand
3078 0 : [this_alternative_matches].type != OP_IN)
3079 0 : && ! find_reg_note (curr_insn, REG_UNUSED,
3080 : no_subreg_reg_operand
3081 0 : [this_alternative_matches])
3082 0 : && ! scratch_p))))
3083 : {
3084 0 : if (lra_dump_file != NULL)
3085 0 : fprintf
3086 0 : (lra_dump_file,
3087 : " No input/output reload -- refuse\n");
3088 0 : goto fail;
3089 : }
3090 :
3091 : /* Alternative loses if it required class pseudo cannot
3092 : hold value of required mode. Such insns can be
3093 : described by insn definitions with mode iterators. */
3094 174419074 : if (GET_MODE (*curr_id->operand_loc[nop]) != VOIDmode
3095 126023128 : && ! hard_reg_set_empty_p (this_alternative_set)
3096 : /* It is common practice for constraints to use a
3097 : class which does not have actually enough regs to
3098 : hold the value (e.g. x86 AREG for mode requiring
3099 : more one general reg). Therefore we have 2
3100 : conditions to check that the reload pseudo cannot
3101 : hold the mode value. */
3102 117422891 : && (!targetm.hard_regno_mode_ok
3103 117422891 : (ira_class_hard_regs[this_alternative][0],
3104 : GET_MODE (*curr_id->operand_loc[nop])))
3105 : /* The above condition is not enough as the first
3106 : reg in ira_class_hard_regs can be not aligned for
3107 : multi-words mode values. */
3108 174419074 : && (prohibited_class_reg_set_mode_p
3109 0 : (this_alternative, this_alternative_set,
3110 0 : GET_MODE (*curr_id->operand_loc[nop]))))
3111 : {
3112 0 : if (lra_dump_file != NULL)
3113 0 : fprintf (lra_dump_file,
3114 : " reload pseudo for op %d "
3115 : "cannot hold the mode value -- refuse\n",
3116 : nop);
3117 0 : goto fail;
3118 : }
3119 :
3120 : /* Check strong discouragement of reload of non-constant
3121 : into class THIS_ALTERNATIVE. */
3122 124620712 : if (! CONSTANT_P (op) && ! no_regs_p
3123 290539350 : && (targetm.preferred_reload_class
3124 116120276 : (op, this_alternative) == NO_REGS
3125 107675973 : || (curr_static_id->operand[nop].type == OP_OUT
3126 74116585 : && (targetm.preferred_output_reload_class
3127 74116585 : (op, this_alternative) == NO_REGS))))
3128 : {
3129 12976793 : if (offmemok && REG_P (op))
3130 : {
3131 793666 : if (lra_dump_file != NULL)
3132 0 : fprintf
3133 0 : (lra_dump_file,
3134 : " %d Spill pseudo into memory: reject+=3\n",
3135 : nop);
3136 793666 : reject += 3;
3137 : }
3138 : else
3139 : {
3140 12183127 : if (lra_dump_file != NULL)
3141 0 : fprintf
3142 0 : (lra_dump_file,
3143 : " %d Non-preferred reload: reject+=%d\n",
3144 : nop, LRA_MAX_REJECT);
3145 12183127 : reject += LRA_MAX_REJECT;
3146 : }
3147 : }
3148 :
3149 174419074 : if (! (MEM_P (op) && offmemok)
3150 174419002 : && ! (const_to_mem && constmemok))
3151 : {
3152 : /* We prefer to reload pseudos over reloading other
3153 : things, since such reloads may be able to be
3154 : eliminated later. So bump REJECT in other cases.
3155 : Don't do this in the case where we are forcing a
3156 : constant into memory and it will then win since
3157 : we don't want to have a different alternative
3158 : match then. */
3159 173505720 : if (! (REG_P (op) && REGNO (op) >= FIRST_PSEUDO_REGISTER))
3160 : {
3161 123855050 : if (lra_dump_file != NULL)
3162 1648 : fprintf
3163 1648 : (lra_dump_file,
3164 : " %d Non-pseudo reload: reject+=2\n",
3165 : nop);
3166 123855050 : reject += 2;
3167 : }
3168 :
3169 173505720 : if (! no_regs_p)
3170 165005354 : reload_nregs
3171 165005354 : += ira_reg_class_max_nregs[this_alternative][mode];
3172 :
3173 173505720 : if (SMALL_REGISTER_CLASS_P (this_alternative))
3174 : {
3175 860877 : if (lra_dump_file != NULL)
3176 45 : fprintf
3177 45 : (lra_dump_file,
3178 : " %d Small class reload: reject+=%d\n",
3179 : nop, LRA_LOSER_COST_FACTOR / 2);
3180 860877 : reject += LRA_LOSER_COST_FACTOR / 2;
3181 : }
3182 : }
3183 :
3184 : /* We are trying to spill pseudo into memory. It is
3185 : usually more costly than moving to a hard register
3186 : although it might takes the same number of
3187 : reloads.
3188 :
3189 : Non-pseudo spill may happen also. Suppose a target allows both
3190 : register and memory in the operand constraint alternatives,
3191 : then it's typical that an eliminable register has a substition
3192 : of "base + offset" which can either be reloaded by a simple
3193 : "new_reg <= base + offset" which will match the register
3194 : constraint, or a similar reg addition followed by further spill
3195 : to and reload from memory which will match the memory
3196 : constraint, but this memory spill will be much more costly
3197 : usually.
3198 :
3199 : Code below increases the reject for both pseudo and non-pseudo
3200 : spill. */
3201 174419074 : if (no_regs_p
3202 9296241 : && !(MEM_P (op) && offmemok)
3203 9296171 : && !(REG_P (op) && hard_regno[nop] < 0))
3204 : {
3205 8179836 : if (lra_dump_file != NULL)
3206 13 : fprintf
3207 20 : (lra_dump_file,
3208 : " %d Spill %spseudo into memory: reject+=3\n",
3209 : nop, REG_P (op) ? "" : "Non-");
3210 8179836 : reject += 3;
3211 8179836 : if (VECTOR_MODE_P (mode))
3212 : {
3213 : /* Spilling vectors into memory is usually more
3214 : costly as they contain big values. */
3215 391180 : if (lra_dump_file != NULL)
3216 0 : fprintf
3217 0 : (lra_dump_file,
3218 : " %d Spill vector pseudo: reject+=2\n",
3219 : nop);
3220 391180 : reject += 2;
3221 : }
3222 : }
3223 :
3224 : /* When we use an operand requiring memory in given
3225 : alternative, the insn should write *and* read the
3226 : value to/from memory it is costly in comparison with
3227 : an insn alternative which does not use memory
3228 : (e.g. register or immediate operand). We exclude
3229 : memory operand for such case as we can satisfy the
3230 : memory constraints by reloading address. */
3231 9296241 : if (no_regs_p && offmemok && !MEM_P (op))
3232 : {
3233 9296013 : if (lra_dump_file != NULL)
3234 27 : fprintf
3235 27 : (lra_dump_file,
3236 : " Using memory insn operand %d: reject+=3\n",
3237 : nop);
3238 9296013 : reject += 3;
3239 : }
3240 :
3241 : /* If reload requires moving value through secondary
3242 : memory, it will need one more insn at least. */
3243 174419074 : if (this_alternative != NO_REGS
3244 165122667 : && REG_P (op) && (cl = get_reg_class (REGNO (op))) != NO_REGS
3245 210400065 : && ((curr_static_id->operand[nop].type != OP_OUT
3246 20383963 : && targetm.secondary_memory_needed (mode, cl,
3247 : this_alternative))
3248 32719729 : || (curr_static_id->operand[nop].type != OP_IN
3249 15597094 : && (targetm.secondary_memory_needed
3250 15597094 : (mode, this_alternative, cl)))))
3251 : {
3252 10817587 : if (lra_dump_file != NULL)
3253 16 : fprintf
3254 16 : (lra_dump_file,
3255 : " %d Secondary memory reload needed: "
3256 : "losers++\n", nop);
3257 10817587 : losers++;
3258 : }
3259 :
3260 174419074 : if (MEM_P (op) && offmemok)
3261 72 : addr_losers++;
3262 : else
3263 : {
3264 : /* Input reloads can be inherited more often than
3265 : output reloads can be removed, so penalize output
3266 : reloads. */
3267 174419002 : if (!REG_P (op) || curr_static_id->operand[nop].type != OP_IN)
3268 : {
3269 146715652 : if (lra_dump_file != NULL)
3270 1722 : fprintf
3271 1722 : (lra_dump_file,
3272 : " %d Non input pseudo reload: reject++\n",
3273 : nop);
3274 146715652 : reject++;
3275 : }
3276 :
3277 174419002 : if (curr_static_id->operand[nop].type == OP_INOUT)
3278 : {
3279 162 : if (lra_dump_file != NULL)
3280 0 : fprintf
3281 0 : (lra_dump_file,
3282 : " %d Input/Output reload: reject+=%d\n",
3283 : nop, LRA_LOSER_COST_FACTOR);
3284 162 : reject += LRA_LOSER_COST_FACTOR;
3285 : }
3286 : }
3287 : }
3288 :
3289 466067805 : if (early_clobber_p && ! scratch_p)
3290 : {
3291 165174 : if (lra_dump_file != NULL)
3292 4 : fprintf (lra_dump_file,
3293 : " %d Early clobber: reject++\n", nop);
3294 165174 : reject++;
3295 : }
3296 : /* ??? We check early clobbers after processing all operands
3297 : (see loop below) and there we update the costs more.
3298 : Should we update the cost (may be approximately) here
3299 : because of early clobber register reloads or it is a rare
3300 : or non-important thing to be worth to do it. */
3301 932135610 : overall = (losers * LRA_LOSER_COST_FACTOR + reject
3302 466067805 : - (addr_losers == losers ? static_reject : 0));
3303 466067805 : if ((best_losers == 0 || losers != 0) && best_overall < overall)
3304 : {
3305 74058412 : if (lra_dump_file != NULL)
3306 1036 : fprintf (lra_dump_file,
3307 : " overall=%d,losers=%d -- refuse\n",
3308 : overall, losers);
3309 74058412 : goto fail;
3310 : }
3311 :
3312 392009393 : if (update_and_check_small_class_inputs (nop, nalt,
3313 : this_alternative))
3314 : {
3315 0 : if (lra_dump_file != NULL)
3316 0 : fprintf (lra_dump_file,
3317 : " not enough small class regs -- refuse\n");
3318 0 : goto fail;
3319 : }
3320 392009393 : curr_alt[nop] = this_alternative;
3321 392009393 : curr_alt_set[nop] = this_alternative_set;
3322 392009393 : curr_alt_exclude_start_hard_regs[nop]
3323 392009393 : = this_alternative_exclude_start_hard_regs;
3324 392009393 : curr_alt_win[nop] = this_alternative_win;
3325 392009393 : curr_alt_match_win[nop] = this_alternative_match_win;
3326 392009393 : curr_alt_offmemok[nop] = this_alternative_offmemok;
3327 392009393 : curr_alt_matches[nop] = this_alternative_matches;
3328 :
3329 392009393 : if (this_alternative_matches >= 0
3330 392009393 : && !did_match && !this_alternative_win)
3331 13123965 : curr_alt_win[this_alternative_matches] = false;
3332 :
3333 392009393 : if (early_clobber_p && operand_reg[nop] != NULL_RTX)
3334 169658 : early_clobbered_nops[early_clobbered_regs_num++] = nop;
3335 : }
3336 :
3337 134133886 : if (curr_insn_set != NULL_RTX
3338 : /* Allow just two operands or three operands where the third
3339 : is a clobber. */
3340 130267089 : && (n_operands == 2
3341 28660491 : || (n_operands == 3
3342 26522966 : && GET_CODE (PATTERN (curr_insn)) == PARALLEL
3343 22500634 : && XVECLEN (PATTERN (curr_insn), 0) == 2
3344 22443149 : && GET_CODE (XVECEXP (PATTERN (curr_insn), 0, 1))
3345 : == CLOBBER))
3346 : /* Prevent processing non-move insns. */
3347 123965589 : && (GET_CODE (SET_SRC (curr_insn_set)) == SUBREG
3348 122192358 : || SET_SRC (curr_insn_set) == no_subreg_reg_operand[1])
3349 224810194 : && ((! curr_alt_win[0] && ! curr_alt_win[1]
3350 5840077 : && REG_P (no_subreg_reg_operand[0])
3351 2858037 : && REG_P (no_subreg_reg_operand[1])
3352 1217744 : && (reg_in_class_p (no_subreg_reg_operand[0], curr_alt[1])
3353 1006276 : || reg_in_class_p (no_subreg_reg_operand[1], curr_alt[0])))
3354 90149875 : || (! curr_alt_win[0] && curr_alt_win[1]
3355 26968545 : && REG_P (no_subreg_reg_operand[1])
3356 : /* Check that we reload memory not the memory
3357 : address. */
3358 15488818 : && ! (curr_alt_offmemok[0]
3359 396033 : && MEM_P (no_subreg_reg_operand[0]))
3360 15488818 : && reg_in_class_p (no_subreg_reg_operand[1], curr_alt[0]))
3361 76053508 : || (curr_alt_win[0] && ! curr_alt_win[1]
3362 9479471 : && REG_P (no_subreg_reg_operand[0])
3363 : /* Check that we reload memory not the memory
3364 : address. */
3365 7004042 : && ! (curr_alt_offmemok[1]
3366 1035474 : && MEM_P (no_subreg_reg_operand[1]))
3367 7004040 : && reg_in_class_p (no_subreg_reg_operand[0], curr_alt[1])
3368 6134713 : && (! CONST_POOL_OK_P (curr_operand_mode[1],
3369 : no_subreg_reg_operand[1])
3370 2279370 : || (targetm.preferred_reload_class
3371 2279370 : (no_subreg_reg_operand[1],
3372 : (enum reg_class) curr_alt[1]) != NO_REGS))
3373 : /* If it is a result of recent elimination in move
3374 : insn we can transform it into an add still by
3375 : using this alternative. */
3376 6092373 : && GET_CODE (no_subreg_reg_operand[1]) != PLUS
3377 : /* Likewise if the source has been replaced with an
3378 : equivalent value. This only happens once -- the reload
3379 : will use the equivalent value instead of the register it
3380 : replaces -- so there should be no danger of cycling. */
3381 5615764 : && !equiv_substition_p[1])))
3382 : {
3383 : /* We have a move insn and a new reload insn will be similar
3384 : to the current insn. We should avoid such situation as
3385 : it results in LRA cycling. */
3386 20210206 : if (lra_dump_file != NULL)
3387 239 : fprintf (lra_dump_file,
3388 : " Cycle danger: overall += LRA_MAX_REJECT\n");
3389 20210206 : overall += LRA_MAX_REJECT;
3390 : }
3391 134133886 : if (all_this_alternative != NO_REGS
3392 114807596 : && !SMALL_REGISTER_CLASS_P (all_this_alternative)
3393 113948449 : && all_used_nregs != 0 && all_reload_nregs != 0
3394 134133886 : && (all_used_nregs + all_reload_nregs + 1
3395 3986776 : >= ira_class_hard_regs_num[all_this_alternative]))
3396 : {
3397 366 : if (lra_dump_file != NULL)
3398 0 : fprintf
3399 0 : (lra_dump_file,
3400 : " Register starvation: overall += LRA_MAX_REJECT"
3401 : "(class=%s,avail=%d,used=%d,reload=%d)\n",
3402 : reg_class_names[all_this_alternative],
3403 : ira_class_hard_regs_num[all_this_alternative],
3404 : all_used_nregs, all_reload_nregs);
3405 366 : overall += LRA_MAX_REJECT;
3406 366 : if (!prefer_memory_p && INSN_CODE (curr_insn) < 0)
3407 : {
3408 : /* asm can permit memory and reg and can be not enough regs for
3409 : asm -- try now memory: */
3410 102 : prefer_memory_p = true;
3411 102 : if (lra_dump_file != NULL)
3412 0 : fprintf
3413 0 : (lra_dump_file,
3414 : " Trying now memory for operands\n");
3415 102 : goto repeat;
3416 : }
3417 : }
3418 134299927 : ok_p = true;
3419 : curr_alt_dont_inherit_ops_num = 0;
3420 134299927 : for (nop = 0; nop < early_clobbered_regs_num; nop++)
3421 : {
3422 166144 : int i, j, clobbered_hard_regno, first_conflict_j, last_conflict_j;
3423 166144 : HARD_REG_SET temp_set;
3424 :
3425 166144 : i = early_clobbered_nops[nop];
3426 166144 : if ((! curr_alt_win[i] && ! curr_alt_match_win[i])
3427 124986 : || hard_regno[i] < 0)
3428 165473 : continue;
3429 123148 : lra_assert (operand_reg[i] != NULL_RTX);
3430 : clobbered_hard_regno = hard_regno[i];
3431 123148 : CLEAR_HARD_REG_SET (temp_set);
3432 123148 : add_to_hard_reg_set (&temp_set, GET_MODE (*curr_id->operand_loc[i]),
3433 : clobbered_hard_regno);
3434 123148 : first_conflict_j = last_conflict_j = -1;
3435 622541 : for (j = 0; j < n_operands; j++)
3436 499394 : if (j == i
3437 : /* We don't want process insides of match_operator and
3438 : match_parallel because otherwise we would process
3439 : their operands once again generating a wrong
3440 : code. */
3441 376246 : || curr_static_id->operand[j].is_operator)
3442 125289 : continue;
3443 374105 : else if ((curr_alt_matches[j] == i && curr_alt_match_win[j])
3444 356333 : || (curr_alt_matches[i] == j && curr_alt_match_win[i]))
3445 17772 : continue;
3446 : /* If we don't reload j-th operand, check conflicts. */
3447 124374 : else if ((curr_alt_win[j] || curr_alt_match_win[j])
3448 417825 : && uses_hard_regs_p (*curr_id->operand_loc[j], temp_set))
3449 : {
3450 1145 : if (first_conflict_j < 0)
3451 671 : first_conflict_j = j;
3452 1145 : last_conflict_j = j;
3453 : /* Both the earlyclobber operand and conflicting operand
3454 : cannot both be user defined hard registers for asm.
3455 : Let curr_insn_transform diagnose it. */
3456 1145 : if (HARD_REGISTER_P (operand_reg[i])
3457 1 : && REG_USERVAR_P (operand_reg[i])
3458 1 : && operand_reg[j] != NULL_RTX
3459 1 : && HARD_REGISTER_P (operand_reg[j])
3460 1 : && REG_USERVAR_P (operand_reg[j])
3461 1146 : && INSN_CODE (curr_insn) < 0)
3462 1 : return false;
3463 : }
3464 123147 : if (last_conflict_j < 0)
3465 122477 : continue;
3466 :
3467 : /* If an earlyclobber operand conflicts with another non-matching
3468 : operand (ie, they have been assigned the same hard register),
3469 : then it is better to reload the other operand, as there may
3470 : exist yet another operand with a matching constraint associated
3471 : with the earlyclobber operand. However, if one of the operands
3472 : is an explicit use of a hard register, then we must reload the
3473 : other non-hard register operand. */
3474 670 : if (HARD_REGISTER_P (operand_reg[i])
3475 670 : || (first_conflict_j == last_conflict_j
3476 196 : && operand_reg[last_conflict_j] != NULL_RTX
3477 57 : && !curr_alt_match_win[last_conflict_j]
3478 57 : && !HARD_REGISTER_P (operand_reg[last_conflict_j])))
3479 : {
3480 57 : curr_alt_win[last_conflict_j] = false;
3481 57 : curr_alt_dont_inherit_ops[curr_alt_dont_inherit_ops_num++]
3482 57 : = last_conflict_j;
3483 57 : losers++;
3484 57 : if (lra_dump_file != NULL)
3485 0 : fprintf
3486 0 : (lra_dump_file,
3487 : " %d Conflict early clobber reload: losers++\n",
3488 : i);
3489 : }
3490 : else
3491 : {
3492 : /* We need to reload early clobbered register and the
3493 : matched registers. */
3494 3059 : for (j = 0; j < n_operands; j++)
3495 2446 : if (curr_alt_matches[j] == i)
3496 : {
3497 2 : curr_alt_match_win[j] = false;
3498 2 : losers++;
3499 2 : if (lra_dump_file != NULL)
3500 0 : fprintf
3501 0 : (lra_dump_file,
3502 : " %d Matching conflict early clobber "
3503 : "reloads: losers++\n",
3504 : j);
3505 2 : overall += LRA_LOSER_COST_FACTOR;
3506 : }
3507 613 : if (! curr_alt_match_win[i])
3508 613 : curr_alt_dont_inherit_ops[curr_alt_dont_inherit_ops_num++] = i;
3509 : else
3510 : {
3511 : /* Remember pseudos used for match reloads are never
3512 : inherited. */
3513 0 : lra_assert (curr_alt_matches[i] >= 0);
3514 0 : curr_alt_win[curr_alt_matches[i]] = false;
3515 : }
3516 613 : curr_alt_win[i] = curr_alt_match_win[i] = false;
3517 613 : losers++;
3518 613 : if (lra_dump_file != NULL)
3519 0 : fprintf
3520 0 : (lra_dump_file,
3521 : " %d Matched conflict early clobber reloads: "
3522 : "losers++\n",
3523 : i);
3524 : }
3525 : /* Early clobber was already reflected in REJECT. */
3526 670 : if (!matching_early_clobber[i])
3527 : {
3528 670 : lra_assert (reject > 0);
3529 670 : reject--;
3530 670 : matching_early_clobber[i] = 1;
3531 : }
3532 670 : overall += LRA_LOSER_COST_FACTOR - 1;
3533 : }
3534 134133783 : if (lra_dump_file != NULL)
3535 1761 : fprintf (lra_dump_file, " overall=%d,losers=%d,rld_nregs=%d\n",
3536 : overall, losers, reload_nregs);
3537 :
3538 : /* If this alternative can be made to work by reloading, and it
3539 : needs less reloading than the others checked so far, record
3540 : it as the chosen goal for reloading. */
3541 134133783 : if ((best_losers != 0 && losers == 0)
3542 59350885 : || (((best_losers == 0 && losers == 0)
3543 58357393 : || (best_losers != 0 && losers != 0))
3544 59350885 : && (best_overall > overall
3545 15263792 : || (best_overall == overall
3546 : /* If the cost of the reloads is the same,
3547 : prefer alternative which requires minimal
3548 : number of reload regs. */
3549 11362397 : && (reload_nregs < best_reload_nregs
3550 11260207 : || (reload_nregs == best_reload_nregs
3551 11218636 : && (best_reload_sum < reload_sum
3552 11197393 : || (best_reload_sum == reload_sum
3553 11173738 : && nalt < goal_alt_number))))))))
3554 : {
3555 387221747 : for (nop = 0; nop < n_operands; nop++)
3556 : {
3557 267986259 : goal_alt_win[nop] = curr_alt_win[nop];
3558 267986259 : goal_alt_match_win[nop] = curr_alt_match_win[nop];
3559 267986259 : goal_alt_matches[nop] = curr_alt_matches[nop];
3560 267986259 : goal_alt[nop] = curr_alt[nop];
3561 267986259 : goal_alt_exclude_start_hard_regs[nop]
3562 267986259 : = curr_alt_exclude_start_hard_regs[nop];
3563 267986259 : goal_alt_offmemok[nop] = curr_alt_offmemok[nop];
3564 : }
3565 119235488 : goal_alt_dont_inherit_ops_num = curr_alt_dont_inherit_ops_num;
3566 119235488 : goal_reuse_alt_p = curr_reuse_alt_p;
3567 119236148 : for (nop = 0; nop < curr_alt_dont_inherit_ops_num; nop++)
3568 660 : goal_alt_dont_inherit_ops[nop] = curr_alt_dont_inherit_ops[nop];
3569 119235488 : goal_alt_swapped = curr_swapped;
3570 119235488 : goal_alt_out_sp_reload_p = curr_alt_out_sp_reload_p;
3571 119235488 : best_overall = overall;
3572 119235488 : best_losers = losers;
3573 119235488 : best_reload_nregs = reload_nregs;
3574 119235488 : best_reload_sum = reload_sum;
3575 119235488 : goal_alt_number = nalt;
3576 : }
3577 134133783 : if (losers == 0 && !curr_alt_class_change_p)
3578 : /* Everything is satisfied. Do not process alternatives
3579 : anymore. */
3580 : break;
3581 58369353 : fail:
3582 180138607 : ;
3583 : }
3584 : return ok_p;
3585 : }
3586 :
3587 : /* Make reload base reg from address AD. */
3588 : static rtx
3589 0 : base_to_reg (struct address_info *ad)
3590 : {
3591 0 : enum reg_class cl;
3592 0 : int code = -1;
3593 0 : rtx new_inner = NULL_RTX;
3594 0 : rtx new_reg = NULL_RTX;
3595 0 : rtx_insn *insn;
3596 0 : rtx_insn *last_insn = get_last_insn();
3597 :
3598 0 : lra_assert (ad->disp == ad->disp_term);
3599 0 : cl = base_reg_class (ad->mode, ad->as, ad->base_outer_code,
3600 : get_index_code (ad));
3601 0 : new_reg = lra_create_new_reg (GET_MODE (*ad->base), NULL_RTX, cl, NULL,
3602 : "base");
3603 0 : new_inner = simplify_gen_binary (PLUS, GET_MODE (new_reg), new_reg,
3604 0 : ad->disp_term == NULL
3605 : ? const0_rtx
3606 : : *ad->disp_term);
3607 0 : if (!valid_address_p (ad->mode, new_inner, ad->as))
3608 : return NULL_RTX;
3609 0 : insn = emit_insn (gen_rtx_SET (new_reg, *ad->base));
3610 0 : code = recog_memoized (insn);
3611 0 : if (code < 0)
3612 : {
3613 0 : delete_insns_since (last_insn);
3614 0 : return NULL_RTX;
3615 : }
3616 :
3617 : return new_inner;
3618 : }
3619 :
3620 : /* Make reload base reg + DISP from address AD. Return the new pseudo. */
3621 : static rtx
3622 47 : base_plus_disp_to_reg (struct address_info *ad, rtx disp)
3623 : {
3624 47 : enum reg_class cl;
3625 47 : rtx new_reg;
3626 :
3627 47 : lra_assert (ad->base == ad->base_term);
3628 47 : cl = base_reg_class (ad->mode, ad->as, ad->base_outer_code,
3629 : get_index_code (ad));
3630 47 : new_reg = lra_create_new_reg (GET_MODE (*ad->base_term), NULL_RTX, cl, NULL,
3631 : "base + disp");
3632 47 : lra_emit_add (new_reg, *ad->base_term, disp);
3633 47 : return new_reg;
3634 : }
3635 :
3636 : /* Make reload of index part of address AD. Return the new
3637 : pseudo. */
3638 : static rtx
3639 0 : index_part_to_reg (struct address_info *ad, enum reg_class index_class)
3640 : {
3641 0 : rtx new_reg;
3642 :
3643 0 : new_reg = lra_create_new_reg (GET_MODE (*ad->index), NULL_RTX,
3644 : index_class, NULL, "index term");
3645 0 : expand_mult (GET_MODE (*ad->index), *ad->index_term,
3646 : GEN_INT (get_index_scale (ad)), new_reg, 1);
3647 0 : return new_reg;
3648 : }
3649 :
3650 : /* Return true if we can add a displacement to address AD, even if that
3651 : makes the address invalid. The fix-up code requires any new address
3652 : to be the sum of the BASE_TERM, INDEX and DISP_TERM fields. */
3653 : static bool
3654 20266 : can_add_disp_p (struct address_info *ad)
3655 : {
3656 20266 : return (!ad->autoinc_p
3657 20266 : && ad->segment == NULL
3658 20266 : && ad->base == ad->base_term
3659 40532 : && ad->disp == ad->disp_term);
3660 : }
3661 :
3662 : /* Make equiv substitution in address AD. Return true if a substitution
3663 : was made. */
3664 : static bool
3665 39268375 : equiv_address_substitution (struct address_info *ad)
3666 : {
3667 39268375 : rtx base_reg, new_base_reg, index_reg, new_index_reg, *base_term, *index_term;
3668 39268375 : poly_int64 disp;
3669 39268375 : HOST_WIDE_INT scale;
3670 39268375 : bool change_p;
3671 :
3672 39268375 : base_term = strip_subreg (ad->base_term);
3673 8888 : if (base_term == NULL)
3674 : base_reg = new_base_reg = NULL_RTX;
3675 : else
3676 : {
3677 33095194 : base_reg = *base_term;
3678 33095194 : new_base_reg = get_equiv_with_elimination (base_reg, curr_insn);
3679 : }
3680 39268375 : index_term = strip_subreg (ad->index_term);
3681 4915 : if (index_term == NULL)
3682 : index_reg = new_index_reg = NULL_RTX;
3683 : else
3684 : {
3685 1864717 : index_reg = *index_term;
3686 1864717 : new_index_reg = get_equiv_with_elimination (index_reg, curr_insn);
3687 : }
3688 39268375 : if (base_reg == new_base_reg && index_reg == new_index_reg)
3689 : return false;
3690 144415 : disp = 0;
3691 144415 : change_p = false;
3692 144415 : if (lra_dump_file != NULL)
3693 : {
3694 0 : fprintf (lra_dump_file, "Changing address in insn %d ",
3695 0 : INSN_UID (curr_insn));
3696 0 : dump_value_slim (lra_dump_file, *ad->outer, 1);
3697 : }
3698 144415 : if (base_reg != new_base_reg)
3699 : {
3700 143934 : poly_int64 offset;
3701 143934 : if (REG_P (new_base_reg))
3702 : {
3703 11563 : *base_term = new_base_reg;
3704 11563 : change_p = true;
3705 : }
3706 132371 : else if (GET_CODE (new_base_reg) == PLUS
3707 20266 : && REG_P (XEXP (new_base_reg, 0))
3708 20266 : && poly_int_rtx_p (XEXP (new_base_reg, 1), &offset)
3709 152637 : && can_add_disp_p (ad))
3710 : {
3711 : disp += offset;
3712 20266 : *base_term = XEXP (new_base_reg, 0);
3713 20266 : change_p = true;
3714 : }
3715 143934 : if (ad->base_term2 != NULL)
3716 0 : *ad->base_term2 = *ad->base_term;
3717 : }
3718 144415 : if (index_reg != new_index_reg)
3719 : {
3720 687 : poly_int64 offset;
3721 687 : if (REG_P (new_index_reg))
3722 : {
3723 0 : *index_term = new_index_reg;
3724 0 : change_p = true;
3725 : }
3726 687 : else if (GET_CODE (new_index_reg) == PLUS
3727 0 : && REG_P (XEXP (new_index_reg, 0))
3728 0 : && poly_int_rtx_p (XEXP (new_index_reg, 1), &offset)
3729 0 : && can_add_disp_p (ad)
3730 687 : && (scale = get_index_scale (ad)))
3731 : {
3732 0 : disp += offset * scale;
3733 0 : *index_term = XEXP (new_index_reg, 0);
3734 0 : change_p = true;
3735 : }
3736 : }
3737 144415 : if (maybe_ne (disp, 0))
3738 : {
3739 20266 : if (ad->disp != NULL)
3740 6672 : *ad->disp = plus_constant (GET_MODE (*ad->inner), *ad->disp, disp);
3741 : else
3742 : {
3743 13594 : *ad->inner = plus_constant (GET_MODE (*ad->inner), *ad->inner, disp);
3744 13594 : update_address (ad);
3745 : }
3746 : change_p = true;
3747 : }
3748 144415 : if (lra_dump_file != NULL)
3749 : {
3750 0 : if (! change_p)
3751 0 : fprintf (lra_dump_file, " -- no change\n");
3752 : else
3753 : {
3754 0 : fprintf (lra_dump_file, " on equiv ");
3755 0 : dump_value_slim (lra_dump_file, *ad->outer, 1);
3756 0 : fprintf (lra_dump_file, "\n");
3757 : }
3758 : }
3759 : return change_p;
3760 : }
3761 :
3762 : /* Skip all modifiers and whitespaces in constraint STR and return the
3763 : result. */
3764 : static const char *
3765 510503313 : skip_constraint_modifiers (const char *str)
3766 : {
3767 719055815 : for (;;str++)
3768 614779564 : switch (*str)
3769 : {
3770 104276251 : case '+': case '&' : case '=': case '*': case ' ': case '\t':
3771 104276251 : case '$': case '^' : case '%': case '?': case '!':
3772 104276251 : break;
3773 510503313 : default: return str;
3774 : }
3775 : }
3776 :
3777 : /* Takes a string of 0 or more comma-separated constraints. When more
3778 : than one constraint is present, evaluate whether they all correspond
3779 : to a single, repeated constraint (e.g. "r,r") or whether we have
3780 : more than one distinct constraints (e.g. "r,m"). */
3781 : static bool
3782 161950456 : constraint_unique (const char *cstr)
3783 : {
3784 161950456 : enum constraint_num ca, cb;
3785 161950456 : ca = CONSTRAINT__UNKNOWN;
3786 321312542 : for (;;)
3787 : {
3788 321312542 : cstr = skip_constraint_modifiers (cstr);
3789 321312542 : if (*cstr == '\0' || *cstr == ',')
3790 : cb = CONSTRAINT_X;
3791 : else
3792 : {
3793 321312542 : cb = lookup_constraint (cstr);
3794 321312542 : if (cb == CONSTRAINT__UNKNOWN)
3795 : return false;
3796 309221839 : cstr += CONSTRAINT_LEN (cstr[0], cstr);
3797 : }
3798 : /* Handle the first iteration of the loop. */
3799 309221839 : if (ca == CONSTRAINT__UNKNOWN)
3800 : ca = cb;
3801 : /* Handle the general case of comparing ca with subsequent
3802 : constraints. */
3803 159229201 : else if (ca != cb)
3804 : return false;
3805 166763947 : if (*cstr == '\0')
3806 : return true;
3807 159362086 : if (*cstr == ',')
3808 88358995 : cstr += 1;
3809 : }
3810 : }
3811 :
3812 : /* Major function to make reloads for an address in operand NOP or
3813 : check its correctness (If CHECK_ONLY_P is true). The supported
3814 : cases are:
3815 :
3816 : 1) an address that existed before LRA started, at which point it
3817 : must have been valid. These addresses are subject to elimination
3818 : and may have become invalid due to the elimination offset being out
3819 : of range.
3820 :
3821 : 2) an address created by forcing a constant to memory
3822 : (force_const_to_mem). The initial form of these addresses might
3823 : not be valid, and it is this function's job to make them valid.
3824 :
3825 : 3) a frame address formed from a register and a (possibly zero)
3826 : constant offset. As above, these addresses might not be valid and
3827 : this function must make them so.
3828 :
3829 : Add reloads to the lists *BEFORE and *AFTER. We might need to add
3830 : reloads to *AFTER because of inc/dec, {pre, post} modify in the
3831 : address. Return true for any RTL change.
3832 :
3833 : The function is a helper function which does not produce all
3834 : transformations (when CHECK_ONLY_P is false) which can be
3835 : necessary. It does just basic steps. To do all necessary
3836 : transformations use function process_address. */
3837 : static bool
3838 175511580 : process_address_1 (int nop, bool check_only_p,
3839 : rtx_insn **before, rtx_insn **after)
3840 : {
3841 175511580 : struct address_info ad;
3842 175511580 : rtx new_reg;
3843 175511580 : HOST_WIDE_INT scale;
3844 175511580 : rtx op = *curr_id->operand_loc[nop];
3845 175511580 : rtx mem = extract_mem_from_operand (op);
3846 175511580 : const char *constraint;
3847 175511580 : enum constraint_num cn;
3848 175511580 : bool change_p = false;
3849 :
3850 175511580 : if (MEM_P (mem)
3851 37546184 : && GET_MODE (mem) == BLKmode
3852 25541 : && GET_CODE (XEXP (mem, 0)) == SCRATCH)
3853 : return false;
3854 :
3855 175511580 : constraint
3856 175511580 : = skip_constraint_modifiers (curr_static_id->operand[nop].constraint);
3857 175511580 : if (IN_RANGE (constraint[0], '0', '9'))
3858 : {
3859 13679191 : char *end;
3860 13679191 : unsigned long dup = strtoul (constraint, &end, 10);
3861 13679191 : constraint
3862 13679191 : = skip_constraint_modifiers (curr_static_id->operand[dup].constraint);
3863 : }
3864 187343237 : cn = lookup_constraint (*constraint == '\0' ? "X" : constraint);
3865 : /* If we have several alternatives or/and several constraints in an
3866 : alternative and we can not say at this stage what constraint will be used,
3867 : use unknown constraint. The exception is an address constraint. If
3868 : operand has one address constraint, probably all others constraints are
3869 : address ones. */
3870 163679923 : if (constraint[0] != '\0' && get_constraint_type (cn) != CT_ADDRESS
3871 337462036 : && !constraint_unique (constraint))
3872 : cn = CONSTRAINT__UNKNOWN;
3873 20962985 : if (insn_extra_address_constraint (cn)
3874 : /* When we find an asm operand with an address constraint that
3875 : doesn't satisfy address_operand to begin with, we clear
3876 : is_address, so that we don't try to make a non-address fit.
3877 : If the asm statement got this far, it's because other
3878 : constraints are available, and we'll use them, disregarding
3879 : the unsatisfiable address ones. */
3880 20962985 : && curr_static_id->operand[nop].is_address)
3881 1729448 : decompose_lea_address (&ad, curr_id->operand_loc[nop]);
3882 : /* Do not attempt to decompose arbitrary addresses generated by combine
3883 : for asm operands with loose constraints, e.g 'X'.
3884 : Need to extract memory from op for special memory constraint,
3885 : i.e. bcst_mem_operand in i386 backend. */
3886 173782132 : else if (MEM_P (mem)
3887 173782432 : && !(INSN_CODE (curr_insn) < 0
3888 19537 : && get_constraint_type (cn) == CT_FIXED_FORM
3889 300 : && constraint_satisfied_p (op, cn)))
3890 37545884 : decompose_mem_address (&ad, mem);
3891 136236248 : else if (GET_CODE (op) == SUBREG
3892 3591862 : && MEM_P (SUBREG_REG (op)))
3893 0 : decompose_mem_address (&ad, SUBREG_REG (op));
3894 : else
3895 : return false;
3896 : /* If INDEX_REG_CLASS is assigned to base_term already and isn't to
3897 : index_term, swap them so to avoid assigning INDEX_REG_CLASS to both
3898 : when INDEX_REG_CLASS is a single register class. */
3899 39275332 : enum reg_class index_cl = index_reg_class (curr_insn);
3900 39275332 : if (ad.base_term != NULL
3901 33102094 : && ad.index_term != NULL
3902 1495713 : && ira_class_hard_regs_num[index_cl] == 1
3903 0 : && REG_P (*ad.base_term)
3904 0 : && REG_P (*ad.index_term)
3905 0 : && in_class_p (*ad.base_term, index_cl, NULL)
3906 39275332 : && ! in_class_p (*ad.index_term, index_cl, NULL))
3907 : {
3908 0 : std::swap (ad.base, ad.index);
3909 0 : std::swap (ad.base_term, ad.index_term);
3910 : }
3911 39275332 : if (! check_only_p)
3912 39268375 : change_p = equiv_address_substitution (&ad);
3913 39275332 : if (ad.base_term != NULL
3914 72377426 : && (process_addr_reg
3915 66204188 : (ad.base_term, check_only_p, before,
3916 33102094 : (ad.autoinc_p
3917 4181658 : && !(REG_P (*ad.base_term)
3918 2090829 : && find_regno_note (curr_insn, REG_DEAD,
3919 : REGNO (*ad.base_term)) != NULL_RTX)
3920 : ? after : NULL),
3921 33102094 : base_reg_class (ad.mode, ad.as, ad.base_outer_code,
3922 : get_index_code (&ad), curr_insn))))
3923 : {
3924 433989 : change_p = true;
3925 433989 : if (ad.base_term2 != NULL)
3926 0 : *ad.base_term2 = *ad.base_term;
3927 : }
3928 39275332 : if (ad.index_term != NULL
3929 39275332 : && process_addr_reg (ad.index_term, check_only_p,
3930 : before, NULL, index_cl))
3931 : change_p = true;
3932 :
3933 : /* Target hooks sometimes don't treat extra-constraint addresses as
3934 : legitimate address_operands, so handle them specially. */
3935 39275332 : if (insn_extra_address_constraint (cn)
3936 39275332 : && satisfies_address_constraint_p (&ad, cn))
3937 : return change_p;
3938 :
3939 37545891 : if (check_only_p)
3940 : return change_p;
3941 :
3942 : /* There are three cases where the shape of *AD.INNER may now be invalid:
3943 :
3944 : 1) the original address was valid, but either elimination or
3945 : equiv_address_substitution was applied and that made
3946 : the address invalid.
3947 :
3948 : 2) the address is an invalid symbolic address created by
3949 : force_const_to_mem.
3950 :
3951 : 3) the address is a frame address with an invalid offset.
3952 :
3953 : 4) the address is a frame address with an invalid base.
3954 :
3955 : All these cases involve a non-autoinc address, so there is no
3956 : point revalidating other types. */
3957 37539548 : if (ad.autoinc_p || valid_address_p (op, &ad, cn))
3958 37539118 : return change_p;
3959 :
3960 : /* Any index existed before LRA started, so we can assume that the
3961 : presence and shape of the index is valid. */
3962 430 : push_to_sequence (*before);
3963 430 : lra_assert (ad.disp == ad.disp_term);
3964 430 : if (ad.base == NULL)
3965 : {
3966 330 : if (ad.index == NULL)
3967 : {
3968 330 : rtx_insn *insn;
3969 330 : rtx_insn *last = get_last_insn ();
3970 330 : int code = -1;
3971 330 : enum reg_class cl = base_reg_class (ad.mode, ad.as,
3972 : SCRATCH, SCRATCH,
3973 : curr_insn);
3974 330 : rtx addr = *ad.inner;
3975 :
3976 651 : new_reg = lra_create_new_reg (Pmode, NULL_RTX, cl, NULL, "addr");
3977 330 : if (HAVE_lo_sum)
3978 : {
3979 : /* addr => lo_sum (new_base, addr), case (2) above. */
3980 : insn = emit_insn (gen_rtx_SET
3981 : (new_reg,
3982 : gen_rtx_HIGH (Pmode, copy_rtx (addr))));
3983 : code = recog_memoized (insn);
3984 : if (code >= 0)
3985 : {
3986 : *ad.inner = gen_rtx_LO_SUM (Pmode, new_reg, addr);
3987 : if (!valid_address_p (op, &ad, cn))
3988 : {
3989 : /* Try to put lo_sum into register. */
3990 : insn = emit_insn (gen_rtx_SET
3991 : (new_reg,
3992 : gen_rtx_LO_SUM (Pmode, new_reg, addr)));
3993 : code = recog_memoized (insn);
3994 : if (code >= 0)
3995 : {
3996 : *ad.inner = new_reg;
3997 : if (!valid_address_p (op, &ad, cn))
3998 : {
3999 : *ad.inner = addr;
4000 : code = -1;
4001 : }
4002 : }
4003 :
4004 : }
4005 : }
4006 : if (code < 0)
4007 : delete_insns_since (last);
4008 : }
4009 :
4010 330 : if (code < 0)
4011 : {
4012 : /* addr => new_base, case (2) above. */
4013 330 : lra_emit_move (new_reg, addr);
4014 :
4015 660 : for (insn = last == NULL_RTX ? get_insns () : NEXT_INSN (last);
4016 660 : insn != NULL_RTX;
4017 330 : insn = NEXT_INSN (insn))
4018 330 : if (recog_memoized (insn) < 0)
4019 : break;
4020 330 : if (insn != NULL_RTX)
4021 : {
4022 : /* Do nothing if we cannot generate right insns.
4023 : This is analogous to reload pass behavior. */
4024 0 : delete_insns_since (last);
4025 0 : end_sequence ();
4026 0 : return false;
4027 : }
4028 330 : *ad.inner = new_reg;
4029 : }
4030 : }
4031 : else
4032 : {
4033 : /* index * scale + disp => new base + index * scale,
4034 : case (1) above. */
4035 0 : enum reg_class cl = base_reg_class (ad.mode, ad.as, PLUS,
4036 0 : GET_CODE (*ad.index),
4037 : curr_insn);
4038 :
4039 0 : lra_assert (index_cl != NO_REGS);
4040 0 : new_reg = lra_create_new_reg (Pmode, NULL_RTX, cl, NULL, "disp");
4041 0 : lra_emit_move (new_reg, *ad.disp);
4042 0 : *ad.inner = simplify_gen_binary (PLUS, GET_MODE (new_reg),
4043 0 : new_reg, *ad.index);
4044 : }
4045 : }
4046 100 : else if (ad.index == NULL)
4047 : {
4048 53 : int regno;
4049 53 : enum reg_class cl;
4050 53 : rtx set;
4051 53 : rtx_insn *insns, *last_insn;
4052 :
4053 53 : cl = base_reg_class (ad.mode, ad.as, ad.base_outer_code,
4054 : get_index_code (&ad), curr_insn);
4055 :
4056 53 : if (REG_P (*ad.base_term)
4057 53 : && ira_class_subset_p[get_reg_class (REGNO (*ad.base_term))][cl])
4058 : /* It seems base reg is already in the base reg class and changing it
4059 : does not make a progress. So reload the whole inner address. */
4060 53 : goto reload_inner_addr;
4061 :
4062 : /* Try to reload base into register only if the base is invalid
4063 : for the address but with valid offset, case (4) above. */
4064 0 : start_sequence ();
4065 0 : new_reg = base_to_reg (&ad);
4066 :
4067 : /* base + disp => new base, cases (1) and (3) above. */
4068 : /* Another option would be to reload the displacement into an
4069 : index register. However, postreload has code to optimize
4070 : address reloads that have the same base and different
4071 : displacements, so reloading into an index register would
4072 : not necessarily be a win. */
4073 0 : if (new_reg == NULL_RTX)
4074 : {
4075 : /* See if the target can split the displacement into a
4076 : legitimate new displacement from a local anchor. */
4077 0 : gcc_assert (ad.disp == ad.disp_term);
4078 0 : poly_int64 orig_offset;
4079 0 : rtx offset1, offset2;
4080 0 : if (poly_int_rtx_p (*ad.disp, &orig_offset)
4081 0 : && targetm.legitimize_address_displacement (&offset1, &offset2,
4082 : orig_offset,
4083 : ad.mode))
4084 : {
4085 0 : new_reg = base_plus_disp_to_reg (&ad, offset1);
4086 0 : new_reg = gen_rtx_PLUS (GET_MODE (new_reg), new_reg, offset2);
4087 : }
4088 : else
4089 0 : new_reg = base_plus_disp_to_reg (&ad, *ad.disp);
4090 : }
4091 0 : insns = get_insns ();
4092 0 : last_insn = get_last_insn ();
4093 : /* If we generated at least two insns, try last insn source as
4094 : an address. If we succeed, we generate one less insn. */
4095 0 : if (REG_P (new_reg)
4096 0 : && last_insn != insns
4097 0 : && (set = single_set (last_insn)) != NULL_RTX
4098 0 : && GET_CODE (SET_SRC (set)) == PLUS
4099 0 : && REG_P (XEXP (SET_SRC (set), 0))
4100 0 : && CONSTANT_P (XEXP (SET_SRC (set), 1)))
4101 : {
4102 0 : *ad.inner = SET_SRC (set);
4103 0 : if (valid_address_p (op, &ad, cn))
4104 : {
4105 0 : *ad.base_term = XEXP (SET_SRC (set), 0);
4106 0 : *ad.disp_term = XEXP (SET_SRC (set), 1);
4107 0 : regno = REGNO (*ad.base_term);
4108 0 : if (regno >= FIRST_PSEUDO_REGISTER
4109 0 : && cl != lra_get_allocno_class (regno))
4110 0 : lra_change_class (regno, cl, " Change to", true);
4111 0 : new_reg = SET_SRC (set);
4112 0 : delete_insns_since (PREV_INSN (last_insn));
4113 : }
4114 : }
4115 0 : end_sequence ();
4116 0 : emit_insn (insns);
4117 0 : *ad.inner = new_reg;
4118 : }
4119 47 : else if (ad.disp_term != NULL)
4120 : {
4121 : /* base + scale * index + disp => new base + scale * index,
4122 : case (1) above. */
4123 47 : gcc_assert (ad.disp == ad.disp_term);
4124 47 : new_reg = base_plus_disp_to_reg (&ad, *ad.disp);
4125 47 : *ad.inner = simplify_gen_binary (PLUS, GET_MODE (new_reg),
4126 47 : new_reg, *ad.index);
4127 : }
4128 0 : else if ((scale = get_index_scale (&ad)) == 1)
4129 : {
4130 : /* The last transformation to one reg will be made in
4131 : curr_insn_transform function. */
4132 0 : end_sequence ();
4133 0 : return false;
4134 : }
4135 0 : else if (scale != 0)
4136 : {
4137 : /* base + scale * index => base + new_reg,
4138 : case (1) above.
4139 : Index part of address may become invalid. For example, we
4140 : changed pseudo on the equivalent memory and a subreg of the
4141 : pseudo onto the memory of different mode for which the scale is
4142 : prohibited. */
4143 0 : new_reg = index_part_to_reg (&ad, index_cl);
4144 0 : *ad.inner = simplify_gen_binary (PLUS, GET_MODE (new_reg),
4145 0 : *ad.base_term, new_reg);
4146 : }
4147 : else
4148 : {
4149 53 : enum reg_class cl;
4150 53 : rtx addr;
4151 0 : reload_inner_addr:
4152 53 : cl = base_reg_class (ad.mode, ad.as, SCRATCH, SCRATCH, curr_insn);
4153 53 : addr = *ad.inner;
4154 53 : new_reg = lra_create_new_reg (Pmode, NULL_RTX, cl, NULL, "addr");
4155 : /* addr => new_base. */
4156 53 : lra_emit_move (new_reg, addr);
4157 53 : *ad.inner = new_reg;
4158 : }
4159 430 : *before = end_sequence ();
4160 430 : return true;
4161 : }
4162 :
4163 : /* If CHECK_ONLY_P is false, do address reloads until it is necessary.
4164 : Use process_address_1 as a helper function. Return true for any
4165 : RTL changes.
4166 :
4167 : If CHECK_ONLY_P is true, just check address correctness. Return
4168 : false if the address correct. */
4169 : static bool
4170 175019798 : process_address (int nop, bool check_only_p,
4171 : rtx_insn **before, rtx_insn **after)
4172 : {
4173 175019798 : bool res = false;
4174 : /* Use enough iterations to process all address parts: */
4175 175511580 : for (int i = 0; i < 10; i++)
4176 : {
4177 175511580 : if (!process_address_1 (nop, check_only_p, before, after))
4178 : {
4179 : return res;
4180 : }
4181 : else
4182 : {
4183 491782 : if (check_only_p)
4184 : return true;
4185 491782 : res = true;
4186 : }
4187 : }
4188 0 : fatal_insn ("unable to reload address in ", curr_insn);
4189 : }
4190 :
4191 : /* Override the generic address_reload_context in order to
4192 : control the creation of reload pseudos. */
4193 : class lra_autoinc_reload_context : public address_reload_context
4194 : {
4195 : machine_mode mode;
4196 : enum reg_class rclass;
4197 :
4198 : public:
4199 0 : lra_autoinc_reload_context (machine_mode mode, enum reg_class new_rclass)
4200 0 : : mode (mode), rclass (new_rclass) {}
4201 :
4202 0 : rtx get_reload_reg () const override final
4203 : {
4204 0 : return lra_create_new_reg (mode, NULL_RTX, rclass, NULL, "INC/DEC result");
4205 : }
4206 : };
4207 :
4208 : /* Emit insns to reload VALUE into a new register. VALUE is an
4209 : auto-increment or auto-decrement RTX whose operand is a register or
4210 : memory location; so reloading involves incrementing that location.
4211 :
4212 : INC_AMOUNT is the number to increment or decrement by (always
4213 : positive and ignored for POST_MODIFY/PRE_MODIFY).
4214 :
4215 : Return a pseudo containing the result. */
4216 : static rtx
4217 0 : emit_inc (enum reg_class new_rclass, rtx value, poly_int64 inc_amount)
4218 : {
4219 0 : lra_autoinc_reload_context context (GET_MODE (value), new_rclass);
4220 0 : return context.emit_autoinc (value, inc_amount);
4221 : }
4222 :
4223 : /* Return true if the current move insn does not need processing as we
4224 : already know that it satisfies its constraints. */
4225 : static bool
4226 100192021 : simple_move_p (void)
4227 : {
4228 100192021 : rtx dest, src;
4229 100192021 : enum reg_class dclass, sclass;
4230 :
4231 100192021 : lra_assert (curr_insn_set != NULL_RTX);
4232 100192021 : dest = SET_DEST (curr_insn_set);
4233 100192021 : src = SET_SRC (curr_insn_set);
4234 :
4235 : /* If the instruction has multiple sets we need to process it even if it
4236 : is single_set. This can happen if one or more of the SETs are dead.
4237 : See PR73650. */
4238 100192021 : if (multiple_sets (curr_insn))
4239 : return false;
4240 :
4241 100003346 : return ((dclass = get_op_class (dest)) != NO_REGS
4242 21017944 : && (sclass = get_op_class (src)) != NO_REGS
4243 : /* The backend guarantees that register moves of cost 2
4244 : never need reloads. */
4245 89174381 : && targetm.register_move_cost (GET_MODE (src), sclass, dclass) == 2);
4246 : }
4247 :
4248 : /* Swap operands NOP and NOP + 1. */
4249 : static inline void
4250 21399020 : swap_operands (int nop)
4251 : {
4252 21399020 : std::swap (curr_operand_mode[nop], curr_operand_mode[nop + 1]);
4253 21399020 : std::swap (original_subreg_reg_mode[nop], original_subreg_reg_mode[nop + 1]);
4254 21399020 : std::swap (*curr_id->operand_loc[nop], *curr_id->operand_loc[nop + 1]);
4255 21399020 : std::swap (equiv_substition_p[nop], equiv_substition_p[nop + 1]);
4256 : /* Swap the duplicates too. */
4257 21399020 : lra_update_dup (curr_id, nop);
4258 21399020 : lra_update_dup (curr_id, nop + 1);
4259 21399020 : }
4260 :
4261 : /* Return TRUE if X is a (subreg of) reg and there are no hard regs of X class
4262 : which can contain value of MODE. */
4263 38 : static bool invalid_mode_reg_p (enum machine_mode mode, rtx x)
4264 : {
4265 38 : if (SUBREG_P (x))
4266 3 : x = SUBREG_REG (x);
4267 38 : if (! REG_P (x))
4268 : return false;
4269 38 : enum reg_class rclass = get_reg_class (REGNO (x));
4270 38 : return (!hard_reg_set_empty_p (reg_class_contents[rclass])
4271 38 : && hard_reg_set_subset_p
4272 38 : (reg_class_contents[rclass],
4273 38 : ira_prohibited_class_mode_regs[rclass][mode]));
4274 : }
4275 :
4276 : /* Return TRUE if regno is referenced in more than one non-debug insn. */
4277 : static bool
4278 2844831 : multiple_insn_refs_p (int regno)
4279 : {
4280 2844831 : unsigned int uid;
4281 2844831 : bitmap_iterator bi;
4282 2844831 : int nrefs = 0;
4283 6884165 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi)
4284 : {
4285 6878461 : if (!NONDEBUG_INSN_P (lra_insn_recog_data[uid]->insn))
4286 1194503 : continue;
4287 5683958 : if (nrefs == 1)
4288 : return true;
4289 2844831 : nrefs++;
4290 : }
4291 : return false;
4292 : }
4293 :
4294 : /* Mark insns starting with FIRST as postponed for processing their
4295 : constraints. See comments for lra_postponed_insns. */
4296 : static void
4297 93044 : postpone_insns (rtx_insn *first)
4298 : {
4299 105982 : for (auto insn = first; insn != NULL_RTX; insn = NEXT_INSN (insn))
4300 : {
4301 12938 : bitmap_set_bit (&lra_postponed_insns, INSN_UID (insn));
4302 12938 : if (lra_dump_file != NULL)
4303 : {
4304 7 : fprintf (lra_dump_file, " Postponing constraint processing: ");
4305 7 : dump_insn_slim (lra_dump_file, insn);
4306 : }
4307 : }
4308 93044 : }
4309 :
4310 : /* Test whether the n-th operand is a MEM where the address is the sum of a
4311 : section anchor and a constant and return true in case of reloading the
4312 : section anchor only results in a satisfiable operand w.r.t. its
4313 : corresponding constraint. Otherwise return false. */
4314 :
4315 : static bool
4316 72 : reload_section_anchor_p (int nop)
4317 : {
4318 72 : rtx op = *curr_id->operand_loc[nop];
4319 72 : if (!MEM_P (op))
4320 : return false;
4321 72 : rtx addr = XEXP (op, 0);
4322 :
4323 72 : if (GET_CODE (addr) != CONST
4324 13 : || GET_CODE (XEXP (addr, 0)) != PLUS
4325 0 : || GET_CODE (XEXP (XEXP (addr, 0), 0)) != SYMBOL_REF
4326 0 : || !SYMBOL_REF_ANCHOR_P (XEXP (XEXP (addr, 0), 0))
4327 0 : || !CONST_INT_P (XEXP (XEXP (addr, 0), 1))
4328 : /* Some offsets are valid in conjunction with a symbol and
4329 : invalid in conjunction with a register. Thus, pull out
4330 : the anchor only in case the offset is a valid anchor
4331 : offset. */
4332 0 : || INTVAL (XEXP (XEXP (addr, 0), 1)) < targetm.min_anchor_offset
4333 72 : || INTVAL (XEXP (XEXP (addr, 0), 1)) > targetm.max_anchor_offset)
4334 : return false;
4335 :
4336 : /* Now test whether a new address of the form REG+DISPLACEMENT is valid for
4337 : the selected alternative. In order to do so, utilize lra_pmode_pseudo
4338 : instead of an actual reload register. */
4339 :
4340 0 : rtx offset = XEXP (XEXP (addr, 0), 1);
4341 0 : rtx new_addr = gen_rtx_PLUS (Pmode, lra_pmode_pseudo, offset);
4342 0 : rtx new_op = shallow_copy_rtx (op);
4343 0 : XEXP (new_op, 0) = new_addr;
4344 :
4345 : /* Get operand constraints for given alternative. */
4346 0 : const char *p = (curr_static_id->operand_alternative
4347 0 : [goal_alt_number * curr_static_id->n_operands + nop]
4348 : .constraint);
4349 0 : char c;
4350 0 : for (;
4351 0 : (c = *p) && c != ',' && c != '#';
4352 0 : p += CONSTRAINT_LEN (c, p))
4353 : {
4354 0 : enum constraint_num cn = lookup_constraint (p);
4355 0 : if (constraint_satisfied_p (new_op, cn))
4356 : return true;
4357 : }
4358 :
4359 : return false;
4360 : }
4361 :
4362 : /* Main entry point of the constraint code: search the body of the
4363 : current insn to choose the best alternative. It is mimicking insn
4364 : alternative cost calculation model of former reload pass. That is
4365 : because machine descriptions were written to use this model. This
4366 : model can be changed in future. Make commutative operand exchange
4367 : if it is chosen.
4368 :
4369 : if CHECK_ONLY_P is false, do RTL changes to satisfy the
4370 : constraints. Return true if any change happened during function
4371 : call.
4372 :
4373 : If CHECK_ONLY_P is true then don't do any transformation. Just
4374 : check that the insn satisfies all constraints. If the insn does
4375 : not satisfy any constraint, return true. */
4376 : static bool
4377 105566109 : curr_insn_transform (bool check_only_p)
4378 : {
4379 105566109 : int i, j, k;
4380 105566109 : int n_operands;
4381 105566109 : int n_alternatives;
4382 105566109 : int n_outputs;
4383 105566109 : int commutative;
4384 105566109 : signed char goal_alt_matched[MAX_RECOG_OPERANDS][MAX_RECOG_OPERANDS];
4385 105566109 : signed char match_inputs[MAX_RECOG_OPERANDS + 1];
4386 105566109 : signed char outputs[MAX_RECOG_OPERANDS + 1];
4387 105566109 : rtx_insn *before, *after;
4388 105566109 : bool alt_p = false;
4389 : /* Flag that the insn has been changed through a transformation. */
4390 105566109 : bool change_p;
4391 105566109 : bool sec_mem_p;
4392 105566109 : bool use_sec_mem_p;
4393 105566109 : int max_regno_before;
4394 105566109 : int reused_alternative_num;
4395 :
4396 105566109 : curr_insn_set = single_set (curr_insn);
4397 105566109 : if (curr_insn_set != NULL_RTX && simple_move_p ())
4398 : {
4399 : /* We assume that the corresponding insn alternative has no
4400 : earlier clobbers. If it is not the case, don't define move
4401 : cost equal to 2 for the corresponding register classes. */
4402 16139458 : lra_set_used_insn_alternative (curr_insn, LRA_NON_CLOBBERED_ALT);
4403 16139458 : return false;
4404 : }
4405 :
4406 89426651 : no_input_reloads_p = no_output_reloads_p = false;
4407 89426651 : goal_alt_number = -1;
4408 89426651 : change_p = sec_mem_p = false;
4409 :
4410 : /* CALL_INSNs are not allowed to have any output reloads. */
4411 89426651 : if (CALL_P (curr_insn))
4412 5968644 : no_output_reloads_p = true;
4413 :
4414 89426651 : n_operands = curr_static_id->n_operands;
4415 89426651 : n_alternatives = curr_static_id->n_alternatives;
4416 :
4417 : /* Just return "no reloads" if insn has no operands with
4418 : constraints. */
4419 89426651 : if (n_operands == 0 || n_alternatives == 0)
4420 : return false;
4421 :
4422 79065427 : max_regno_before = max_reg_num ();
4423 :
4424 334437707 : for (i = 0; i < n_operands; i++)
4425 : {
4426 176306853 : goal_alt_matched[i][0] = -1;
4427 176306853 : goal_alt_matches[i] = -1;
4428 : }
4429 :
4430 79065427 : commutative = curr_static_id->commutative;
4431 :
4432 : /* Now see what we need for pseudos that didn't get hard regs or got
4433 : the wrong kind of hard reg. For this, we must consider all the
4434 : operands together against the register constraints. */
4435 :
4436 79065427 : best_losers = best_overall = INT_MAX;
4437 79065427 : best_reload_sum = 0;
4438 :
4439 79065427 : curr_swapped = false;
4440 79065427 : goal_alt_swapped = false;
4441 :
4442 79065427 : if (! check_only_p)
4443 : /* Make equivalence substitution and memory subreg elimination
4444 : before address processing because an address legitimacy can
4445 : depend on memory mode. */
4446 255304712 : for (i = 0; i < n_operands; i++)
4447 : {
4448 176258214 : rtx op, subst, old;
4449 176258214 : bool op_change_p = false;
4450 :
4451 176258214 : if (curr_static_id->operand[i].is_operator)
4452 1419425 : continue;
4453 :
4454 174838789 : old = op = *curr_id->operand_loc[i];
4455 174838789 : if (GET_CODE (old) == SUBREG)
4456 3646747 : old = SUBREG_REG (old);
4457 174838789 : subst = get_equiv_with_elimination (old, curr_insn);
4458 174838789 : original_subreg_reg_mode[i] = VOIDmode;
4459 174838789 : equiv_substition_p[i] = false;
4460 :
4461 174838789 : if (subst != old
4462 : /* We don't want to change an out operand by constant or invariant
4463 : which will require additional reloads, e.g. by putting a constant
4464 : into memory. */
4465 1504920 : && (curr_static_id->operand[i].type == OP_IN || MEM_P (subst)
4466 0 : || (GET_CODE (subst) == SUBREG && MEM_P (SUBREG_REG (subst)))))
4467 : {
4468 1504920 : equiv_substition_p[i] = true;
4469 1504920 : rtx new_subst = copy_rtx (subst);
4470 1504920 : if (lra_pointer_equiv_set_in (subst))
4471 767558 : lra_pointer_equiv_set_add (new_subst);
4472 1504920 : subst = new_subst;
4473 1504920 : lra_assert (REG_P (old));
4474 1504920 : if (GET_CODE (op) != SUBREG)
4475 1450490 : *curr_id->operand_loc[i] = subst;
4476 : else
4477 : {
4478 54430 : SUBREG_REG (op) = subst;
4479 54430 : if (GET_MODE (subst) == VOIDmode)
4480 89 : original_subreg_reg_mode[i] = GET_MODE (old);
4481 : }
4482 1504920 : if (lra_dump_file != NULL)
4483 : {
4484 3 : fprintf (lra_dump_file,
4485 : "Changing pseudo %d in operand %i of insn %u on equiv ",
4486 3 : REGNO (old), i, INSN_UID (curr_insn));
4487 3 : dump_value_slim (lra_dump_file, subst, 1);
4488 3 : fprintf (lra_dump_file, "\n");
4489 : }
4490 1504920 : op_change_p = change_p = true;
4491 : }
4492 174838789 : if (simplify_operand_subreg (i, GET_MODE (old)) || op_change_p)
4493 : {
4494 1505465 : change_p = true;
4495 1505465 : lra_update_dup (curr_id, i);
4496 : }
4497 : }
4498 :
4499 : /* We process equivalences before ignoring postponed insns on the current
4500 : constraint sub-pass but before any reload insn generation for the
4501 : postponed insn. */
4502 79046498 : if (! check_only_p
4503 79046498 : && bitmap_bit_p (&lra_postponed_insns, INSN_UID (curr_insn)))
4504 : return true;
4505 :
4506 : /* Reload address registers and displacements. We do it before
4507 : finding an alternative because of memory constraints. */
4508 79058726 : before = after = NULL;
4509 255352177 : for (i = 0; i < n_operands; i++)
4510 176293451 : if (! curr_static_id->operand[i].is_operator
4511 176293451 : && process_address (i, check_only_p, &before, &after))
4512 : {
4513 491779 : if (check_only_p)
4514 : return true;
4515 491779 : change_p = true;
4516 491779 : lra_update_dup (curr_id, i);
4517 : }
4518 :
4519 79058726 : if (change_p)
4520 : /* If we've changed the instruction then any alternative that
4521 : we chose previously may no longer be valid. */
4522 1949461 : lra_set_used_insn_alternative (curr_insn, LRA_UNKNOWN_ALT);
4523 :
4524 79039797 : if (! check_only_p && curr_insn_set != NULL_RTX
4525 154292426 : && check_and_process_move (&change_p, &sec_mem_p))
4526 0 : return change_p;
4527 :
4528 79058726 : try_swapped:
4529 :
4530 89455499 : reused_alternative_num = check_only_p ? LRA_UNKNOWN_ALT : curr_id->used_insn_alternative;
4531 89455499 : if (lra_dump_file != NULL && reused_alternative_num >= 0)
4532 0 : fprintf (lra_dump_file, "Reusing alternative %d for insn #%u\n",
4533 0 : reused_alternative_num, INSN_UID (curr_insn));
4534 :
4535 89455499 : if (process_alt_operands (reused_alternative_num))
4536 80832193 : alt_p = true;
4537 :
4538 89455499 : if (check_only_p)
4539 31855 : return ! alt_p || best_losers != 0;
4540 :
4541 : /* If insn is commutative (it's safe to exchange a certain pair of
4542 : operands) then we need to try each alternative twice, the second
4543 : time matching those two operands as if we had exchanged them. To
4544 : do this, really exchange them in operands.
4545 :
4546 : If we have just tried the alternatives the second time, return
4547 : operands to normal and drop through. */
4548 :
4549 89436570 : if (reused_alternative_num < 0 && commutative >= 0)
4550 : {
4551 20793546 : curr_swapped = !curr_swapped;
4552 20793546 : if (curr_swapped)
4553 : {
4554 10396773 : swap_operands (commutative);
4555 10396773 : goto try_swapped;
4556 : }
4557 : else
4558 10396773 : swap_operands (commutative);
4559 : }
4560 :
4561 79039797 : if (! alt_p && ! sec_mem_p)
4562 : {
4563 : /* No alternative works with reloads?? */
4564 6 : if (INSN_CODE (curr_insn) >= 0)
4565 0 : fatal_insn ("unable to generate reloads for:", curr_insn);
4566 6 : error_for_asm (curr_insn,
4567 : "inconsistent operand constraints in an %<asm%>");
4568 6 : lra_asm_error_p = true;
4569 6 : if (! JUMP_P (curr_insn))
4570 : {
4571 : /* Avoid further trouble with this insn. Don't generate use
4572 : pattern here as we could use the insn SP offset. */
4573 6 : lra_set_insn_deleted (curr_insn);
4574 : }
4575 : else
4576 : {
4577 0 : lra_invalidate_insn_data (curr_insn);
4578 0 : ira_nullify_asm_goto (curr_insn);
4579 0 : lra_update_insn_regno_info (curr_insn);
4580 : }
4581 6 : return true;
4582 : }
4583 :
4584 : /* If the best alternative is with operands 1 and 2 swapped, swap
4585 : them. Update the operand numbers of any reloads already
4586 : pushed. */
4587 :
4588 79039791 : if (goal_alt_swapped)
4589 : {
4590 601164 : if (lra_dump_file != NULL)
4591 18 : fprintf (lra_dump_file, " Commutative operand exchange in insn %u\n",
4592 18 : INSN_UID (curr_insn));
4593 :
4594 : /* Swap the duplicates too. */
4595 601164 : swap_operands (commutative);
4596 601164 : change_p = true;
4597 : }
4598 :
4599 : /* Some targets' TARGET_SECONDARY_MEMORY_NEEDED (e.g. x86) are defined
4600 : too conservatively. So we use the secondary memory only if there
4601 : is no any alternative without reloads. */
4602 79039791 : use_sec_mem_p = false;
4603 79039791 : if (! alt_p)
4604 : use_sec_mem_p = true;
4605 79039791 : else if (sec_mem_p)
4606 : {
4607 14918 : for (i = 0; i < n_operands; i++)
4608 14746 : if (! goal_alt_win[i] && ! goal_alt_match_win[i])
4609 : break;
4610 13246 : use_sec_mem_p = i < n_operands;
4611 : }
4612 :
4613 13246 : if (use_sec_mem_p)
4614 : {
4615 13074 : int in = -1, out = -1;
4616 13074 : rtx new_reg, src, dest, rld;
4617 13074 : machine_mode sec_mode, rld_mode;
4618 :
4619 13074 : lra_assert (curr_insn_set != NULL_RTX && sec_mem_p);
4620 13074 : dest = SET_DEST (curr_insn_set);
4621 13074 : src = SET_SRC (curr_insn_set);
4622 39222 : for (i = 0; i < n_operands; i++)
4623 26148 : if (*curr_id->operand_loc[i] == dest)
4624 : out = i;
4625 13074 : else if (*curr_id->operand_loc[i] == src)
4626 13074 : in = i;
4627 13074 : for (i = 0; i < curr_static_id->n_dups; i++)
4628 0 : if (out < 0 && *curr_id->dup_loc[i] == dest)
4629 0 : out = curr_static_id->dup_num[i];
4630 0 : else if (in < 0 && *curr_id->dup_loc[i] == src)
4631 0 : in = curr_static_id->dup_num[i];
4632 13074 : lra_assert (out >= 0 && in >= 0
4633 : && curr_static_id->operand[out].type == OP_OUT
4634 : && curr_static_id->operand[in].type == OP_IN);
4635 13074 : rld = partial_subreg_p (GET_MODE (src), GET_MODE (dest)) ? src : dest;
4636 13074 : rld_mode = GET_MODE (rld);
4637 13074 : sec_mode = targetm.secondary_memory_needed_mode (rld_mode);
4638 13074 : if (rld_mode != sec_mode
4639 13074 : && (invalid_mode_reg_p (sec_mode, dest)
4640 19 : || invalid_mode_reg_p (sec_mode, src)))
4641 : sec_mode = rld_mode;
4642 13074 : new_reg = lra_create_new_reg (sec_mode, NULL_RTX, NO_REGS, NULL,
4643 : "secondary");
4644 : /* If the mode is changed, it should be wider. */
4645 13074 : lra_assert (!partial_subreg_p (sec_mode, rld_mode));
4646 13074 : if (sec_mode != rld_mode)
4647 : {
4648 : /* If the target says specifically to use another mode for
4649 : secondary memory moves we cannot reuse the original
4650 : insn. */
4651 19 : after = emit_spill_move (false, new_reg, dest);
4652 19 : lra_process_new_insns (curr_insn, NULL, after,
4653 : "Inserting the sec. move");
4654 : /* We may have non null BEFORE here (e.g. after address
4655 : processing. */
4656 19 : push_to_sequence (before);
4657 19 : before = emit_spill_move (true, new_reg, src);
4658 19 : emit_insn (before);
4659 19 : before = end_sequence ();
4660 19 : lra_process_new_insns (curr_insn, before, NULL, "Changing on");
4661 19 : lra_set_insn_deleted (curr_insn);
4662 : }
4663 13055 : else if (dest == rld)
4664 : {
4665 13055 : *curr_id->operand_loc[out] = new_reg;
4666 13055 : lra_update_dup (curr_id, out);
4667 13055 : after = emit_spill_move (false, new_reg, dest);
4668 13055 : lra_process_new_insns (curr_insn, NULL, after,
4669 : "Inserting the sec. move");
4670 : }
4671 : else
4672 : {
4673 0 : *curr_id->operand_loc[in] = new_reg;
4674 0 : lra_update_dup (curr_id, in);
4675 : /* See comments above. */
4676 0 : push_to_sequence (before);
4677 0 : before = emit_spill_move (true, new_reg, src);
4678 0 : emit_insn (before);
4679 0 : before = end_sequence ();
4680 0 : lra_process_new_insns (curr_insn, before, NULL,
4681 : "Inserting the sec. move");
4682 : }
4683 13074 : lra_update_insn_regno_info (curr_insn);
4684 13074 : return true;
4685 : }
4686 :
4687 79026717 : lra_assert (goal_alt_number >= 0);
4688 157957380 : lra_set_used_insn_alternative (curr_insn, goal_reuse_alt_p
4689 : ? goal_alt_number : LRA_UNKNOWN_ALT);
4690 :
4691 79026717 : if (lra_dump_file != NULL)
4692 : {
4693 1187 : const char *p;
4694 :
4695 1187 : fprintf (lra_dump_file, " Choosing alt %d in insn %u:",
4696 1187 : goal_alt_number, INSN_UID (curr_insn));
4697 1187 : print_curr_insn_alt (goal_alt_number);
4698 1187 : if (INSN_CODE (curr_insn) >= 0
4699 1187 : && (p = get_insn_name (INSN_CODE (curr_insn))) != NULL)
4700 1180 : fprintf (lra_dump_file, " {%s}", p);
4701 1187 : if (maybe_ne (curr_id->sp_offset, 0))
4702 : {
4703 0 : fprintf (lra_dump_file, " (sp_off=");
4704 0 : print_dec (curr_id->sp_offset, lra_dump_file);
4705 0 : fprintf (lra_dump_file, ")");
4706 : }
4707 1187 : fprintf (lra_dump_file, "\n");
4708 : }
4709 :
4710 : /* Right now, for any pair of operands I and J that are required to
4711 : match, with J < I, goal_alt_matches[I] is J. Add I to
4712 : goal_alt_matched[J]. */
4713 :
4714 255245367 : for (i = 0; i < n_operands; i++)
4715 176218650 : if ((j = goal_alt_matches[i]) >= 0)
4716 : {
4717 10499257 : for (k = 0; goal_alt_matched[j][k] >= 0; k++)
4718 : ;
4719 : /* We allow matching one output operand and several input
4720 : operands. */
4721 10499256 : lra_assert (k == 0
4722 : || (curr_static_id->operand[j].type == OP_OUT
4723 : && curr_static_id->operand[i].type == OP_IN
4724 : && (curr_static_id->operand
4725 : [goal_alt_matched[j][0]].type == OP_IN)));
4726 10499256 : goal_alt_matched[j][k] = i;
4727 10499256 : goal_alt_matched[j][k + 1] = -1;
4728 : }
4729 :
4730 255245367 : for (i = 0; i < n_operands; i++)
4731 176218650 : goal_alt_win[i] |= goal_alt_match_win[i];
4732 :
4733 : /* Any constants that aren't allowed and can't be reloaded into
4734 : registers are here changed into memory references. */
4735 255245367 : for (i = 0; i < n_operands; i++)
4736 176218650 : if (goal_alt_win[i])
4737 : {
4738 170160505 : int regno;
4739 170160505 : enum reg_class new_class;
4740 170160505 : rtx reg = *curr_id->operand_loc[i];
4741 :
4742 170160505 : if (GET_CODE (reg) == SUBREG)
4743 3340543 : reg = SUBREG_REG (reg);
4744 :
4745 170160505 : if (REG_P (reg) && (regno = REGNO (reg)) >= FIRST_PSEUDO_REGISTER)
4746 : {
4747 77165355 : bool ok_p = in_class_p (reg, goal_alt[i], &new_class, true);
4748 :
4749 77165355 : if (new_class != NO_REGS && get_reg_class (regno) != new_class)
4750 : {
4751 3477496 : lra_assert (ok_p);
4752 3477496 : lra_change_class (regno, new_class, " Change to", true);
4753 : }
4754 : }
4755 : }
4756 : else
4757 : {
4758 6058145 : const char *constraint;
4759 6058145 : char c;
4760 6058145 : rtx op = *curr_id->operand_loc[i];
4761 6058145 : rtx subreg = NULL_RTX;
4762 6058145 : machine_mode op_mode = curr_operand_mode[i], mode = op_mode;
4763 :
4764 6058145 : if (GET_CODE (op) == SUBREG)
4765 : {
4766 241697 : subreg = op;
4767 241697 : op = SUBREG_REG (op);
4768 241697 : mode = GET_MODE (op);
4769 : }
4770 :
4771 6276382 : if (CONST_POOL_OK_P (mode, op)
4772 6276382 : && ((targetm.preferred_reload_class
4773 218237 : (op, (enum reg_class) goal_alt[i]) == NO_REGS)
4774 71987 : || no_input_reloads_p))
4775 : {
4776 146250 : rtx tem = force_const_mem (mode, op);
4777 :
4778 146250 : change_p = true;
4779 146250 : if (subreg != NULL_RTX)
4780 0 : tem = gen_rtx_SUBREG (op_mode, tem, SUBREG_BYTE (subreg));
4781 :
4782 146250 : *curr_id->operand_loc[i] = tem;
4783 146250 : lra_update_dup (curr_id, i);
4784 146250 : process_address (i, false, &before, &after);
4785 :
4786 : /* If the alternative accepts constant pool refs directly
4787 : there will be no reload needed at all. */
4788 146250 : if (subreg != NULL_RTX)
4789 0 : continue;
4790 : /* Skip alternatives before the one requested. */
4791 146250 : constraint = (curr_static_id->operand_alternative
4792 146250 : [goal_alt_number * n_operands + i].constraint);
4793 146250 : for (;
4794 248363 : (c = *constraint) && c != ',' && c != '#';
4795 102113 : constraint += CONSTRAINT_LEN (c, constraint))
4796 : {
4797 200875 : enum constraint_num cn = lookup_constraint (constraint);
4798 200875 : if ((insn_extra_memory_constraint (cn)
4799 102478 : || insn_extra_special_memory_constraint (cn)
4800 : || insn_extra_relaxed_memory_constraint (cn))
4801 201240 : && satisfies_memory_constraint_p (tem, cn))
4802 : break;
4803 : }
4804 146250 : if (c == '\0' || c == ',' || c == '#')
4805 47488 : continue;
4806 :
4807 98762 : goal_alt_win[i] = true;
4808 : }
4809 : }
4810 :
4811 : n_outputs = 0;
4812 255245367 : for (i = 0; i < n_operands; i++)
4813 176218650 : if (curr_static_id->operand[i].type == OP_OUT)
4814 68615488 : outputs[n_outputs++] = i;
4815 79026717 : outputs[n_outputs] = -1;
4816 255245367 : for (i = 0; i < n_operands; i++)
4817 : {
4818 176218650 : int regno;
4819 176218650 : bool optional_p = false;
4820 176218650 : rtx old, new_reg;
4821 176218650 : rtx op = *curr_id->operand_loc[i];
4822 :
4823 176218650 : if (goal_alt_win[i])
4824 : {
4825 170259267 : if (goal_alt[i] == NO_REGS
4826 46462692 : && REG_P (op)
4827 5312991 : && (regno = REGNO (op)) >= FIRST_PSEUDO_REGISTER
4828 : /* We assigned a hard register to the pseudo in the past but now
4829 : decided to spill it for the insn. If the pseudo is used only
4830 : in this insn, it is better to spill it here as we free hard
4831 : registers for other pseudos referenced in the insn. The most
4832 : common case of this is a scratch register which will be
4833 : transformed to scratch back at the end of LRA. */
4834 173104098 : && !multiple_insn_refs_p (regno))
4835 : {
4836 11408 : if (lra_get_allocno_class (regno) != NO_REGS)
4837 5365 : lra_change_class (regno, NO_REGS, " Change to", true);
4838 5704 : reg_renumber[regno] = -1;
4839 : }
4840 : /* We can do an optional reload. If the pseudo got a hard
4841 : reg, we might improve the code through inheritance. If
4842 : it does not get a hard register we coalesce memory/memory
4843 : moves later. Ignore move insns to avoid cycling. */
4844 170259267 : if (! lra_simple_p
4845 169714589 : && lra_undo_inheritance_iter < LRA_MAX_INHERITANCE_PASSES
4846 157500752 : && goal_alt[i] != NO_REGS && REG_P (op)
4847 78243638 : && (regno = REGNO (op)) >= FIRST_PSEUDO_REGISTER
4848 65414583 : && regno < new_regno_start
4849 60700148 : && ! ira_former_scratch_p (regno)
4850 60644985 : && reg_renumber[regno] < 0
4851 : /* Check that the optional reload pseudo will be able to
4852 : hold given mode value. */
4853 3831034 : && ! (prohibited_class_reg_set_mode_p
4854 3831034 : (goal_alt[i], reg_class_contents[goal_alt[i]],
4855 3831034 : PSEUDO_REGNO_MODE (regno)))
4856 174090291 : && (curr_insn_set == NULL_RTX
4857 3823920 : || !((REG_P (SET_SRC (curr_insn_set))
4858 : || MEM_P (SET_SRC (curr_insn_set))
4859 : || GET_CODE (SET_SRC (curr_insn_set)) == SUBREG)
4860 3183786 : && (REG_P (SET_DEST (curr_insn_set))
4861 : || MEM_P (SET_DEST (curr_insn_set))
4862 : || GET_CODE (SET_DEST (curr_insn_set)) == SUBREG))))
4863 : optional_p = true;
4864 169611993 : else if (goal_alt_matched[i][0] != -1
4865 8808455 : && curr_static_id->operand[i].type == OP_OUT
4866 8807256 : && (curr_static_id->operand_alternative
4867 8807256 : [goal_alt_number * n_operands + i].earlyclobber)
4868 18356 : && REG_P (op))
4869 : {
4870 23062 : for (j = 0; goal_alt_matched[i][j] != -1; j++)
4871 : {
4872 18303 : rtx op2 = *curr_id->operand_loc[goal_alt_matched[i][j]];
4873 :
4874 18303 : if (REG_P (op2) && REGNO (op) != REGNO (op2))
4875 : break;
4876 : }
4877 18303 : if (goal_alt_matched[i][j] != -1)
4878 : {
4879 : /* Generate reloads for different output and matched
4880 : input registers. This is the easiest way to avoid
4881 : creation of non-existing register conflicts in
4882 : lra-lives.cc. */
4883 13544 : match_reload (i, goal_alt_matched[i], outputs, goal_alt[i],
4884 : &goal_alt_exclude_start_hard_regs[i], &before,
4885 : &after, true);
4886 : }
4887 171205229 : continue;
4888 18303 : }
4889 : else
4890 : {
4891 169593690 : enum reg_class rclass, common_class;
4892 :
4893 88859240 : if (REG_P (op) && goal_alt[i] != NO_REGS
4894 83546249 : && (regno = REGNO (op)) >= new_regno_start
4895 4722788 : && (rclass = get_reg_class (regno)) == ALL_REGS
4896 0 : && ((common_class = ira_reg_class_subset[rclass][goal_alt[i]])
4897 : != NO_REGS)
4898 0 : && common_class != ALL_REGS
4899 169593690 : && enough_allocatable_hard_regs_p (common_class,
4900 0 : GET_MODE (op)))
4901 : /* Refine reload pseudo class from chosen alternative
4902 : constraint. */
4903 0 : lra_change_class (regno, common_class, " Change to", true);
4904 169593690 : continue;
4905 169593690 : }
4906 : }
4907 :
4908 : /* Operands that match previous ones have already been handled. */
4909 6606657 : if (goal_alt_matches[i] >= 0)
4910 1593236 : continue;
4911 :
4912 : /* We should not have an operand with a non-offsettable address
4913 : appearing where an offsettable address will do. It also may
4914 : be a case when the address should be special in other words
4915 : not a general one (e.g. it needs no index reg). */
4916 5013421 : if (goal_alt_matched[i][0] == -1 && goal_alt_offmemok[i] && MEM_P (op))
4917 : {
4918 72 : enum reg_class rclass;
4919 72 : rtx *loc = &XEXP (op, 0);
4920 72 : enum rtx_code code = GET_CODE (*loc);
4921 :
4922 72 : push_to_sequence (before);
4923 72 : rclass = base_reg_class (GET_MODE (op), MEM_ADDR_SPACE (op),
4924 : MEM, SCRATCH, curr_insn);
4925 72 : if (GET_RTX_CLASS (code) == RTX_AUTOINC)
4926 0 : new_reg = emit_inc (rclass, *loc,
4927 : /* This value does not matter for MODIFY. */
4928 0 : GET_MODE_SIZE (GET_MODE (op)));
4929 : /* Try to pull out section anchors. For example, instead of
4930 : reloading an "entire" address like .LANCHOR42+offset only reload
4931 : .LANCHOR42 and use the new reload register as the base register.
4932 : This allows following optimizations to share section anchors and
4933 : remove redundant loads. */
4934 72 : else if (reload_section_anchor_p (i))
4935 : {
4936 0 : rtx anchor = XEXP (XEXP (*loc, 0), 0);
4937 0 : rtx offset = XEXP (XEXP (*loc, 0), 1);
4938 :
4939 0 : if (get_reload_reg (OP_IN, Pmode, anchor, rclass, NULL, false,
4940 : false, "offsetable address", &new_reg))
4941 0 : lra_emit_move (new_reg, anchor);
4942 :
4943 0 : rtx new_addr = gen_rtx_PLUS (Pmode, new_reg, offset);
4944 0 : rtx new_op = shallow_copy_rtx (op);
4945 0 : XEXP (new_op, 0) = new_addr;
4946 :
4947 0 : new_reg = new_op;
4948 0 : loc = curr_id->operand_loc[i];
4949 : }
4950 86 : else if (get_reload_reg (OP_IN, Pmode, *loc, rclass,
4951 : NULL, false, false,
4952 : "offsetable address", &new_reg))
4953 : {
4954 72 : rtx addr = *loc;
4955 72 : enum rtx_code code = GET_CODE (addr);
4956 72 : bool align_p = false;
4957 :
4958 72 : if (code == AND && CONST_INT_P (XEXP (addr, 1)))
4959 : {
4960 : /* (and ... (const_int -X)) is used to align to X bytes. */
4961 0 : align_p = true;
4962 0 : addr = XEXP (*loc, 0);
4963 : }
4964 : else
4965 72 : addr = canonicalize_reload_addr (addr);
4966 :
4967 72 : lra_emit_move (new_reg, addr);
4968 72 : if (align_p)
4969 0 : emit_move_insn (new_reg, gen_rtx_AND (GET_MODE (new_reg), new_reg, XEXP (*loc, 1)));
4970 : }
4971 72 : before = end_sequence ();
4972 72 : *loc = new_reg;
4973 72 : lra_update_dup (curr_id, i);
4974 72 : }
4975 5013349 : else if (goal_alt_matched[i][0] == -1)
4976 : {
4977 3322549 : machine_mode mode;
4978 3322549 : rtx reg, *loc;
4979 3322549 : int hard_regno;
4980 3322549 : enum op_type type = curr_static_id->operand[i].type;
4981 :
4982 3322549 : loc = curr_id->operand_loc[i];
4983 3322549 : mode = curr_operand_mode[i];
4984 3322549 : if (GET_CODE (*loc) == SUBREG)
4985 : {
4986 77249 : reg = SUBREG_REG (*loc);
4987 77249 : poly_int64 byte = SUBREG_BYTE (*loc);
4988 77249 : if (REG_P (reg)
4989 : /* Strict_low_part requires reloading the register and not
4990 : just the subreg. Likewise for a strict subreg no wider
4991 : than a word for WORD_REGISTER_OPERATIONS targets. */
4992 77249 : && (curr_static_id->operand[i].strict_low
4993 77175 : || (!paradoxical_subreg_p (mode, GET_MODE (reg))
4994 74567 : && (hard_regno
4995 74567 : = get_try_hard_regno (REGNO (reg))) >= 0
4996 73179 : && (simplify_subreg_regno
4997 150428 : (hard_regno,
4998 73179 : GET_MODE (reg), byte, mode) < 0)
4999 0 : && (goal_alt[i] == NO_REGS
5000 0 : || (simplify_subreg_regno
5001 77249 : (ira_class_hard_regs[goal_alt[i]][0],
5002 0 : GET_MODE (reg), byte, mode) >= 0)))
5003 77175 : || (partial_subreg_p (mode, GET_MODE (reg))
5004 77175 : && known_le (GET_MODE_SIZE (GET_MODE (reg)),
5005 : UNITS_PER_WORD)
5006 : && WORD_REGISTER_OPERATIONS))
5007 : /* Avoid the situation when there are no available hard regs
5008 : for the pseudo mode but there are ones for the subreg
5009 : mode: */
5010 77323 : && !(goal_alt[i] != NO_REGS
5011 74 : && REGNO (reg) >= FIRST_PSEUDO_REGISTER
5012 74 : && (prohibited_class_reg_set_mode_p
5013 74 : (goal_alt[i], reg_class_contents[goal_alt[i]],
5014 74 : GET_MODE (reg)))
5015 : && !(prohibited_class_reg_set_mode_p
5016 0 : (goal_alt[i], reg_class_contents[goal_alt[i]],
5017 : mode))))
5018 : {
5019 : /* An OP_INOUT is required when reloading a subreg of a
5020 : mode wider than a word to ensure that data beyond the
5021 : word being reloaded is preserved. Also automatically
5022 : ensure that strict_low_part reloads are made into
5023 : OP_INOUT which should already be true from the backend
5024 : constraints. */
5025 74 : if (type == OP_OUT
5026 74 : && (curr_static_id->operand[i].strict_low
5027 0 : || read_modify_subreg_p (*loc)))
5028 : type = OP_INOUT;
5029 74 : loc = &SUBREG_REG (*loc);
5030 74 : mode = GET_MODE (*loc);
5031 : }
5032 : }
5033 3322549 : old = *loc;
5034 3322549 : if (get_reload_reg (type, mode, old, goal_alt[i],
5035 : &goal_alt_exclude_start_hard_regs[i],
5036 3322549 : loc != curr_id->operand_loc[i],
5037 3322549 : curr_static_id->operand_alternative
5038 3322549 : [goal_alt_number * n_operands + i].earlyclobber,
5039 : "", &new_reg)
5040 3322549 : && type != OP_OUT)
5041 : {
5042 2361484 : push_to_sequence (before);
5043 2361484 : lra_emit_move (new_reg, old);
5044 2361484 : before = end_sequence ();
5045 : }
5046 3322549 : *loc = new_reg;
5047 3322549 : if (type != OP_IN
5048 959924 : && find_reg_note (curr_insn, REG_UNUSED, old) == NULL_RTX
5049 : /* OLD can be an equivalent constant here. */
5050 934887 : && !CONSTANT_P (old)
5051 : /* No need to write back anything for a scratch. */
5052 934887 : && GET_CODE (old) != SCRATCH
5053 4257436 : && (!REG_P(old) || !ira_former_scratch_p (REGNO (old))))
5054 : {
5055 934887 : start_sequence ();
5056 934887 : lra_emit_move (type == OP_INOUT ? copy_rtx (old) : old, new_reg);
5057 934887 : emit_insn (after);
5058 934887 : after = end_sequence ();
5059 934887 : *loc = new_reg;
5060 : }
5061 3322549 : for (j = 0; j < goal_alt_dont_inherit_ops_num; j++)
5062 622 : if (goal_alt_dont_inherit_ops[j] == i)
5063 : {
5064 622 : lra_set_regno_unique_value (REGNO (new_reg));
5065 622 : break;
5066 : }
5067 3322549 : lra_update_dup (curr_id, i);
5068 : }
5069 1690800 : else if (curr_static_id->operand[i].type == OP_IN
5070 1690800 : && (curr_static_id->operand[goal_alt_matched[i][0]].type
5071 : == OP_OUT
5072 0 : || (curr_static_id->operand[goal_alt_matched[i][0]].type
5073 : == OP_INOUT
5074 0 : && (operands_match_p
5075 0 : (*curr_id->operand_loc[i],
5076 0 : *curr_id->operand_loc[goal_alt_matched[i][0]],
5077 : -1)))))
5078 : {
5079 : /* generate reloads for input and matched outputs. */
5080 14744 : match_inputs[0] = i;
5081 14744 : match_inputs[1] = -1;
5082 14744 : match_reload (goal_alt_matched[i][0], match_inputs, outputs,
5083 : goal_alt[i], &goal_alt_exclude_start_hard_regs[i],
5084 : &before, &after,
5085 14744 : curr_static_id->operand_alternative
5086 14744 : [goal_alt_number * n_operands + goal_alt_matched[i][0]]
5087 14744 : .earlyclobber);
5088 : }
5089 1676056 : else if ((curr_static_id->operand[i].type == OP_OUT
5090 0 : || (curr_static_id->operand[i].type == OP_INOUT
5091 0 : && (operands_match_p
5092 0 : (*curr_id->operand_loc[i],
5093 0 : *curr_id->operand_loc[goal_alt_matched[i][0]],
5094 : -1))))
5095 1676056 : && (curr_static_id->operand[goal_alt_matched[i][0]].type
5096 : == OP_IN))
5097 : /* Generate reloads for output and matched inputs. */
5098 1676056 : match_reload (i, goal_alt_matched[i], outputs, goal_alt[i],
5099 : &goal_alt_exclude_start_hard_regs[i], &before, &after,
5100 1676056 : curr_static_id->operand_alternative
5101 1676056 : [goal_alt_number * n_operands + i].earlyclobber);
5102 0 : else if (curr_static_id->operand[i].type == OP_IN
5103 0 : && (curr_static_id->operand[goal_alt_matched[i][0]].type
5104 : == OP_IN))
5105 : {
5106 : /* Generate reloads for matched inputs. */
5107 0 : match_inputs[0] = i;
5108 0 : for (j = 0; (k = goal_alt_matched[i][j]) >= 0; j++)
5109 0 : match_inputs[j + 1] = k;
5110 0 : match_inputs[j + 1] = -1;
5111 0 : match_reload (-1, match_inputs, outputs, goal_alt[i],
5112 : &goal_alt_exclude_start_hard_regs[i],
5113 : &before, &after, false);
5114 : }
5115 : else
5116 : /* We must generate code in any case when function
5117 : process_alt_operands decides that it is possible. */
5118 0 : gcc_unreachable ();
5119 :
5120 5013421 : if (optional_p)
5121 : {
5122 647274 : rtx reg = op;
5123 :
5124 647274 : lra_assert (REG_P (reg));
5125 647274 : regno = REGNO (reg);
5126 647274 : op = *curr_id->operand_loc[i]; /* Substitution. */
5127 647274 : if (GET_CODE (op) == SUBREG)
5128 0 : op = SUBREG_REG (op);
5129 647274 : gcc_assert (REG_P (op) && (int) REGNO (op) >= new_regno_start);
5130 647274 : bitmap_set_bit (&lra_optional_reload_pseudos, REGNO (op));
5131 647274 : lra_reg_info[REGNO (op)].restore_rtx = reg;
5132 647274 : if (lra_dump_file != NULL)
5133 3 : fprintf (lra_dump_file,
5134 : " Making reload reg %d for reg %d optional\n",
5135 : REGNO (op), regno);
5136 : }
5137 : }
5138 74740141 : if (before != NULL_RTX || after != NULL_RTX
5139 152965481 : || max_regno_before != max_reg_num ())
5140 5113392 : change_p = true;
5141 79026717 : if (change_p)
5142 : {
5143 6062720 : lra_update_operator_dups (curr_id);
5144 : /* Something changes -- process the insn. */
5145 6062720 : lra_update_insn_regno_info (curr_insn);
5146 6062720 : if (asm_noperands (PATTERN (curr_insn)) >= 0
5147 6062720 : && ++curr_id->asm_reloads_num >= FIRST_PSEUDO_REGISTER)
5148 : /* Most probably there are no enough registers to satisfy asm insn: */
5149 : {
5150 11 : lra_asm_insn_error (curr_insn);
5151 11 : return change_p;
5152 : }
5153 : }
5154 79026706 : if (goal_alt_out_sp_reload_p)
5155 : {
5156 : /* We have an output stack pointer reload -- update sp offset: */
5157 0 : rtx set;
5158 0 : bool done_p = false;
5159 0 : poly_int64 sp_offset = curr_id->sp_offset;
5160 0 : for (rtx_insn *insn = after; insn != NULL_RTX; insn = NEXT_INSN (insn))
5161 0 : if ((set = single_set (insn)) != NULL_RTX
5162 0 : && SET_DEST (set) == stack_pointer_rtx)
5163 : {
5164 0 : lra_assert (!done_p);
5165 0 : done_p = true;
5166 0 : curr_id->sp_offset = 0;
5167 0 : lra_insn_recog_data_t id = lra_get_insn_recog_data (insn);
5168 0 : id->sp_offset = sp_offset;
5169 0 : if (lra_dump_file != NULL)
5170 0 : fprintf (lra_dump_file,
5171 : " Moving sp offset from insn %u to %u\n",
5172 0 : INSN_UID (curr_insn), INSN_UID (insn));
5173 : }
5174 0 : lra_assert (done_p);
5175 : }
5176 79026706 : int const_regno = -1;
5177 79026706 : rtx set;
5178 79026706 : rtx_insn *prev, *const_insn = NULL;
5179 4286571 : if (before != NULL_RTX && (prev = PREV_INSN (curr_insn)) != NULL_RTX
5180 83313277 : && (set = single_set (prev)) != NULL_RTX && CONSTANT_P (SET_SRC (set)))
5181 : {
5182 328204 : rtx reg = SET_DEST (set);
5183 328204 : if (GET_CODE (reg) == SUBREG)
5184 9683 : reg = SUBREG_REG (reg);
5185 : /* Consider only reload insns as we don't want to change the order
5186 : created by previous optimizations. */
5187 238103 : if (REG_P (reg) && (int) REGNO (reg) >= lra_new_regno_start
5188 329033 : && bitmap_bit_p (&lra_reg_info[REGNO (reg)].insn_bitmap,
5189 829 : INSN_UID (curr_insn)))
5190 : {
5191 326 : const_regno = REGNO (reg);
5192 326 : const_insn = prev;
5193 : }
5194 : }
5195 79026706 : if (asm_noperands (PATTERN (curr_insn)) >= 0)
5196 : {
5197 : /* Asm can have a lot of operands. To guarantee their assignment,
5198 : postpone processing the reload insns until the reload pseudos are
5199 : assigned. */
5200 46522 : postpone_insns (before);
5201 46522 : postpone_insns (after);
5202 : }
5203 79026706 : lra_process_new_insns (curr_insn, before, after,
5204 : "Inserting insn reload", true);
5205 79026706 : if (const_regno >= 0) {
5206 652 : bool move_p = true;
5207 652 : for (rtx_insn *insn = before; insn != curr_insn; insn = NEXT_INSN (insn))
5208 326 : if (bitmap_bit_p (&lra_reg_info[const_regno].insn_bitmap,
5209 326 : INSN_UID (insn)))
5210 : {
5211 : move_p = false;
5212 : break;
5213 : }
5214 326 : if (move_p)
5215 : {
5216 326 : reorder_insns_nobb (const_insn, const_insn, PREV_INSN (curr_insn));
5217 326 : if (lra_dump_file != NULL)
5218 : {
5219 0 : dump_insn_slim (lra_dump_file, const_insn);
5220 0 : fprintf (lra_dump_file,
5221 : " to decrease reg pressure, it is moved before:\n");
5222 0 : dump_insn_slim (lra_dump_file, curr_insn);
5223 : }
5224 : }
5225 : }
5226 : return change_p;
5227 : }
5228 :
5229 : /* Return true if INSN satisfies all constraints. In other words, no
5230 : reload insns are needed. */
5231 : bool
5232 3341 : lra_constrain_insn (rtx_insn *insn)
5233 : {
5234 3341 : int saved_new_regno_start = new_regno_start;
5235 3341 : int saved_new_insn_uid_start = new_insn_uid_start;
5236 3341 : bool change_p;
5237 :
5238 3341 : curr_insn = insn;
5239 3341 : curr_id = lra_get_insn_recog_data (curr_insn);
5240 3341 : curr_static_id = curr_id->insn_static_data;
5241 3341 : new_insn_uid_start = get_max_uid ();
5242 3341 : new_regno_start = max_reg_num ();
5243 3341 : change_p = curr_insn_transform (true);
5244 3341 : new_regno_start = saved_new_regno_start;
5245 3341 : new_insn_uid_start = saved_new_insn_uid_start;
5246 3341 : return ! change_p;
5247 : }
5248 :
5249 : /* Return true if X is in LIST. */
5250 : static bool
5251 1303491 : in_list_p (rtx x, rtx list)
5252 : {
5253 2229760 : for (; list != NULL_RTX; list = XEXP (list, 1))
5254 1221734 : if (XEXP (list, 0) == x)
5255 : return true;
5256 : return false;
5257 : }
5258 :
5259 : /* Return true if X contains an allocatable hard register (if
5260 : HARD_REG_P) or a (spilled if SPILLED_P) pseudo. */
5261 : static bool
5262 7252973 : contains_reg_p (rtx x, bool hard_reg_p, bool spilled_p)
5263 : {
5264 7252973 : int i, j;
5265 7252973 : const char *fmt;
5266 7252973 : enum rtx_code code;
5267 :
5268 7252973 : code = GET_CODE (x);
5269 7252973 : if (REG_P (x))
5270 : {
5271 1460864 : int regno = REGNO (x);
5272 1460864 : HARD_REG_SET alloc_regs;
5273 :
5274 1460864 : if (hard_reg_p)
5275 : {
5276 458730 : if (regno >= FIRST_PSEUDO_REGISTER)
5277 135574 : regno = lra_get_regno_hard_regno (regno);
5278 458730 : if (regno < 0)
5279 : return false;
5280 458730 : alloc_regs = ~lra_no_alloc_regs;
5281 458730 : return overlaps_hard_reg_set_p (alloc_regs, GET_MODE (x), regno);
5282 : }
5283 : else
5284 : {
5285 1002134 : if (regno < FIRST_PSEUDO_REGISTER)
5286 : return false;
5287 323518 : if (! spilled_p)
5288 : return true;
5289 172074 : return lra_get_regno_hard_regno (regno) < 0;
5290 : }
5291 : }
5292 5792109 : fmt = GET_RTX_FORMAT (code);
5293 14322166 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
5294 : {
5295 9092187 : if (fmt[i] == 'e')
5296 : {
5297 3986425 : if (contains_reg_p (XEXP (x, i), hard_reg_p, spilled_p))
5298 : return true;
5299 : }
5300 5105762 : else if (fmt[i] == 'E')
5301 : {
5302 1262981 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
5303 1155711 : if (contains_reg_p (XVECEXP (x, i, j), hard_reg_p, spilled_p))
5304 : return true;
5305 : }
5306 : }
5307 : return false;
5308 : }
5309 :
5310 : /* Process all regs in location *LOC and change them on equivalent
5311 : substitution. Return true if any change was done. */
5312 : static bool
5313 3348 : loc_equivalence_change_p (rtx *loc)
5314 : {
5315 3348 : rtx subst, reg, x = *loc;
5316 3348 : bool result = false;
5317 3348 : enum rtx_code code = GET_CODE (x);
5318 3348 : const char *fmt;
5319 3348 : int i, j;
5320 :
5321 3348 : if (code == SUBREG)
5322 : {
5323 20 : reg = SUBREG_REG (x);
5324 20 : if ((subst = get_equiv_with_elimination (reg, curr_insn)) != reg
5325 20 : && GET_MODE (subst) == VOIDmode)
5326 : {
5327 : /* We cannot reload debug location. Simplify subreg here
5328 : while we know the inner mode. */
5329 0 : *loc = simplify_gen_subreg (GET_MODE (x), subst,
5330 0 : GET_MODE (reg), SUBREG_BYTE (x));
5331 0 : return true;
5332 : }
5333 : }
5334 3348 : if (code == REG && (subst = get_equiv_with_elimination (x, curr_insn)) != x)
5335 : {
5336 8 : *loc = subst;
5337 8 : return true;
5338 : }
5339 :
5340 : /* Scan all the operand sub-expressions. */
5341 3340 : fmt = GET_RTX_FORMAT (code);
5342 8168 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
5343 : {
5344 4828 : if (fmt[i] == 'e')
5345 2577 : result = loc_equivalence_change_p (&XEXP (x, i)) || result;
5346 2251 : else if (fmt[i] == 'E')
5347 270 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
5348 200 : result
5349 210 : = loc_equivalence_change_p (&XVECEXP (x, i, j)) || result;
5350 : }
5351 : return result;
5352 : }
5353 :
5354 : /* Similar to loc_equivalence_change_p, but for use as
5355 : simplify_replace_fn_rtx callback. DATA is insn for which the
5356 : elimination is done. If it null we don't do the elimination. */
5357 : static rtx
5358 42019464 : loc_equivalence_callback (rtx loc, const_rtx, void *data)
5359 : {
5360 42019464 : if (!REG_P (loc))
5361 : return NULL_RTX;
5362 :
5363 10832763 : rtx subst = (data == NULL
5364 10832763 : ? get_equiv (loc) : get_equiv_with_elimination (loc, (rtx_insn *) data));
5365 10832763 : if (subst != loc)
5366 : return subst;
5367 :
5368 : return NULL_RTX;
5369 : }
5370 :
5371 : /* Maximum number of generated reload insns per an insn. It is for
5372 : preventing this pass cycling in a bug case. */
5373 : #define MAX_RELOAD_INSNS_NUMBER LRA_MAX_INSN_RELOADS
5374 :
5375 : /* The current iteration number of this LRA pass. */
5376 : int lra_constraint_iter;
5377 :
5378 : /* True if we should during assignment sub-pass check assignment
5379 : correctness for all pseudos and spill some of them to correct
5380 : conflicts. It can be necessary when we substitute equiv which
5381 : needs checking register allocation correctness because the
5382 : equivalent value contains allocatable hard registers, or when we
5383 : restore multi-register pseudo, or when we change the insn code and
5384 : its operand became INOUT operand when it was IN one before. */
5385 : bool check_and_force_assignment_correctness_p;
5386 :
5387 : /* Return true if REGNO is referenced in more than one block. */
5388 : static bool
5389 145219 : multi_block_pseudo_p (int regno)
5390 : {
5391 145219 : basic_block bb = NULL;
5392 145219 : unsigned int uid;
5393 145219 : bitmap_iterator bi;
5394 :
5395 145219 : if (regno < FIRST_PSEUDO_REGISTER)
5396 : return false;
5397 :
5398 444971 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi)
5399 304560 : if (bb == NULL)
5400 145219 : bb = BLOCK_FOR_INSN (lra_insn_recog_data[uid]->insn);
5401 159341 : else if (BLOCK_FOR_INSN (lra_insn_recog_data[uid]->insn) != bb)
5402 : return true;
5403 : return false;
5404 : }
5405 :
5406 : /* Return true if LIST contains a deleted insn. */
5407 : static bool
5408 705843 : contains_deleted_insn_p (rtx_insn_list *list)
5409 : {
5410 1345552 : for (; list != NULL_RTX; list = list->next ())
5411 639709 : if (NOTE_P (list->insn ())
5412 639709 : && NOTE_KIND (list->insn ()) == NOTE_INSN_DELETED)
5413 : return true;
5414 : return false;
5415 : }
5416 :
5417 : /* Return true if X contains a pseudo dying in INSN. */
5418 : static bool
5419 2190157 : dead_pseudo_p (rtx x, rtx_insn *insn)
5420 : {
5421 2190157 : int i, j;
5422 2190157 : const char *fmt;
5423 2190157 : enum rtx_code code;
5424 :
5425 2190157 : if (REG_P (x))
5426 487954 : return (insn != NULL_RTX
5427 487954 : && find_regno_note (insn, REG_DEAD, REGNO (x)) != NULL_RTX);
5428 1702203 : code = GET_CODE (x);
5429 1702203 : fmt = GET_RTX_FORMAT (code);
5430 4349399 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
5431 : {
5432 2652493 : if (fmt[i] == 'e')
5433 : {
5434 1295430 : if (dead_pseudo_p (XEXP (x, i), insn))
5435 : return true;
5436 : }
5437 1357063 : else if (fmt[i] == 'E')
5438 : {
5439 280619 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
5440 256530 : if (dead_pseudo_p (XVECEXP (x, i, j), insn))
5441 : return true;
5442 : }
5443 : }
5444 : return false;
5445 : }
5446 :
5447 : /* Return true if INSN contains a dying pseudo in INSN right hand
5448 : side. */
5449 : static bool
5450 638197 : insn_rhs_dead_pseudo_p (rtx_insn *insn)
5451 : {
5452 638197 : rtx set = single_set (insn);
5453 :
5454 638197 : gcc_assert (set != NULL);
5455 638197 : return dead_pseudo_p (SET_SRC (set), insn);
5456 : }
5457 :
5458 : /* Return true if any init insn of REGNO contains a dying pseudo in
5459 : insn right hand side. */
5460 : static bool
5461 704331 : init_insn_rhs_dead_pseudo_p (int regno)
5462 : {
5463 704331 : rtx_insn_list *insns = ira_reg_equiv[regno].init_insns;
5464 :
5465 704331 : if (insns == NULL)
5466 : return false;
5467 1269824 : for (; insns != NULL_RTX; insns = insns->next ())
5468 638197 : if (insn_rhs_dead_pseudo_p (insns->insn ()))
5469 : return true;
5470 : return false;
5471 : }
5472 :
5473 : /* Return TRUE if REGNO has a reverse equivalence. The equivalence is
5474 : reverse only if we have one init insn with given REGNO as a
5475 : source. */
5476 : static bool
5477 705843 : reverse_equiv_p (int regno)
5478 : {
5479 705843 : rtx_insn_list *insns = ira_reg_equiv[regno].init_insns;
5480 705843 : rtx set;
5481 :
5482 705843 : if (insns == NULL)
5483 : return false;
5484 639709 : if (! INSN_P (insns->insn ())
5485 1279418 : || insns->next () != NULL)
5486 : return false;
5487 639709 : if ((set = single_set (insns->insn ())) == NULL_RTX)
5488 : return false;
5489 639709 : return REG_P (SET_SRC (set)) && (int) REGNO (SET_SRC (set)) == regno;
5490 : }
5491 :
5492 : /* Return TRUE if REGNO was reloaded in an equivalence init insn. We
5493 : call this function only for non-reverse equivalence. */
5494 : static bool
5495 697761 : contains_reloaded_insn_p (int regno)
5496 : {
5497 697761 : rtx set;
5498 697761 : rtx_insn_list *list = ira_reg_equiv[regno].init_insns;
5499 :
5500 1329388 : for (; list != NULL; list = list->next ())
5501 631627 : if ((set = single_set (list->insn ())) == NULL_RTX
5502 631627 : || ! REG_P (SET_DEST (set))
5503 1263254 : || (int) REGNO (SET_DEST (set)) != regno)
5504 : return true;
5505 : return false;
5506 : }
5507 :
5508 : /* Try combine secondary memory reload insn FROM for insn TO into TO insn.
5509 : FROM should be a load insn (usually a secondary memory reload insn). Return
5510 : TRUE in case of success. */
5511 : static bool
5512 7231532 : combine_reload_insn (rtx_insn *from, rtx_insn *to)
5513 : {
5514 7231532 : bool ok_p;
5515 7231532 : rtx_insn *saved_insn;
5516 7231532 : rtx set, from_reg, to_reg, op;
5517 7231532 : enum reg_class to_class, from_class;
5518 7231532 : int n, nop;
5519 7231532 : signed char changed_nops[MAX_RECOG_OPERANDS + 1];
5520 :
5521 : /* Check conditions for second memory reload and original insn: */
5522 7231532 : if ((targetm.secondary_memory_needed
5523 : == hook_bool_mode_reg_class_t_reg_class_t_false)
5524 7231532 : || NEXT_INSN (from) != to
5525 4294711 : || !NONDEBUG_INSN_P (to)
5526 11526241 : || CALL_P (to))
5527 : return false;
5528 :
5529 4289274 : lra_insn_recog_data_t id = lra_get_insn_recog_data (to);
5530 4289274 : struct lra_static_insn_data *static_id = id->insn_static_data;
5531 :
5532 4289274 : if (id->used_insn_alternative == LRA_UNKNOWN_ALT
5533 4289274 : || (set = single_set (from)) == NULL_RTX)
5534 32668 : return false;
5535 4256606 : from_reg = SET_DEST (set);
5536 4256606 : to_reg = SET_SRC (set);
5537 : /* Ignore optional reloads: */
5538 4178109 : if (! REG_P (from_reg) || ! REG_P (to_reg)
5539 7119075 : || bitmap_bit_p (&lra_optional_reload_pseudos, REGNO (from_reg)))
5540 1943190 : return false;
5541 2313416 : to_class = lra_get_allocno_class (REGNO (to_reg));
5542 2313416 : from_class = lra_get_allocno_class (REGNO (from_reg));
5543 : /* Check that reload insn is a load: */
5544 2313416 : if (to_class != NO_REGS || from_class == NO_REGS)
5545 : return false;
5546 48548 : for (n = nop = 0; nop < static_id->n_operands; nop++)
5547 : {
5548 34893 : if (static_id->operand[nop].type != OP_IN)
5549 12626 : continue;
5550 22267 : op = *id->operand_loc[nop];
5551 22267 : if (!REG_P (op) || REGNO (op) != REGNO (from_reg))
5552 8815 : continue;
5553 13452 : *id->operand_loc[nop] = to_reg;
5554 13452 : changed_nops[n++] = nop;
5555 : }
5556 13655 : changed_nops[n] = -1;
5557 13655 : lra_update_dups (id, changed_nops);
5558 13655 : lra_update_insn_regno_info (to);
5559 13655 : ok_p = recog_memoized (to) >= 0;
5560 13655 : if (ok_p)
5561 : {
5562 : /* Check that combined insn does not need any reloads: */
5563 13636 : saved_insn = curr_insn;
5564 13636 : curr_insn = to;
5565 13636 : curr_id = lra_get_insn_recog_data (curr_insn);
5566 13636 : curr_static_id = curr_id->insn_static_data;
5567 13636 : for (bool swapped_p = false;;)
5568 : {
5569 15791 : ok_p = !curr_insn_transform (true);
5570 15791 : if (ok_p || curr_static_id->commutative < 0)
5571 : break;
5572 4310 : swap_operands (curr_static_id->commutative);
5573 4310 : if (lra_dump_file != NULL)
5574 : {
5575 0 : fprintf (lra_dump_file,
5576 : " Swapping %scombined insn operands:\n",
5577 : swapped_p ? "back " : "");
5578 0 : dump_insn_slim (lra_dump_file, to);
5579 : }
5580 4310 : if (swapped_p)
5581 : break;
5582 : swapped_p = true;
5583 : }
5584 13636 : curr_insn = saved_insn;
5585 13636 : curr_id = lra_get_insn_recog_data (curr_insn);
5586 13636 : curr_static_id = curr_id->insn_static_data;
5587 : }
5588 13655 : if (ok_p)
5589 : {
5590 3588 : id->used_insn_alternative = -1;
5591 3588 : lra_push_insn_and_update_insn_regno_info (to);
5592 3588 : if (lra_dump_file != NULL)
5593 : {
5594 0 : fprintf (lra_dump_file, " Use combined insn:\n");
5595 0 : dump_insn_slim (lra_dump_file, to);
5596 : }
5597 3588 : return true;
5598 : }
5599 10067 : if (lra_dump_file != NULL)
5600 : {
5601 0 : fprintf (lra_dump_file, " Failed combined insn:\n");
5602 0 : dump_insn_slim (lra_dump_file, to);
5603 : }
5604 20405 : for (int i = 0; i < n; i++)
5605 : {
5606 10338 : nop = changed_nops[i];
5607 10338 : *id->operand_loc[nop] = from_reg;
5608 : }
5609 10067 : lra_update_dups (id, changed_nops);
5610 10067 : lra_update_insn_regno_info (to);
5611 10067 : if (lra_dump_file != NULL)
5612 : {
5613 0 : fprintf (lra_dump_file, " Restoring insn after failed combining:\n");
5614 0 : dump_insn_slim (lra_dump_file, to);
5615 : }
5616 : return false;
5617 : }
5618 :
5619 : /* Entry function of LRA constraint pass. Return true if the
5620 : constraint pass did change the code. */
5621 : bool
5622 3239552 : lra_constraints (bool first_p)
5623 : {
5624 3239552 : bool changed_p;
5625 3239552 : int i, hard_regno, new_insns_num;
5626 3239552 : unsigned int min_len, new_min_len, uid;
5627 3239552 : rtx set, x, reg, nosubreg_dest;
5628 3239552 : rtx_insn *original_insn;
5629 3239552 : basic_block last_bb;
5630 3239552 : bitmap_iterator bi;
5631 :
5632 3239552 : lra_constraint_iter++;
5633 3239552 : if (lra_dump_file != NULL)
5634 194 : fprintf (lra_dump_file, "\n********** Local #%d: **********\n\n",
5635 : lra_constraint_iter);
5636 3239552 : changed_p = false;
5637 3239552 : if (pic_offset_table_rtx
5638 3239552 : && REGNO (pic_offset_table_rtx) >= FIRST_PSEUDO_REGISTER)
5639 104549 : check_and_force_assignment_correctness_p = true;
5640 3135003 : else if (first_p)
5641 : /* On the first iteration we should check IRA assignment
5642 : correctness. In rare cases, the assignments can be wrong as
5643 : early clobbers operands are ignored in IRA or usages of
5644 : paradoxical sub-registers are not taken into account by
5645 : IRA. */
5646 1447900 : check_and_force_assignment_correctness_p = true;
5647 3239552 : new_insn_uid_start = get_max_uid ();
5648 3239552 : new_regno_start = first_p ? lra_constraint_new_regno_start : max_reg_num ();
5649 : /* Mark used hard regs for target stack size calculations. */
5650 205281774 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
5651 202042222 : if (lra_reg_info[i].nrefs != 0
5652 298457820 : && (hard_regno = lra_get_regno_hard_regno (i)) >= 0)
5653 : {
5654 92549551 : int j, nregs;
5655 :
5656 92549551 : nregs = hard_regno_nregs (hard_regno, lra_reg_info[i].biggest_mode);
5657 188145566 : for (j = 0; j < nregs; j++)
5658 95596015 : df_set_regs_ever_live (hard_regno + j, true);
5659 : }
5660 : /* Do elimination before the equivalence processing as we can spill
5661 : some pseudos during elimination. */
5662 3239552 : lra_eliminate (false, first_p);
5663 3239552 : auto_bitmap equiv_insn_bitmap (®_obstack);
5664 :
5665 : /* Register elimination can create new pseudos via the addptr pattern,
5666 : so make sure the equivalency tables are resized appropriately. */
5667 3239552 : ira_expand_reg_equiv ();
5668 205281774 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
5669 202042222 : if (lra_reg_info[i].nrefs != 0)
5670 : {
5671 96415598 : ira_reg_equiv[i].profitable_p = true;
5672 96415598 : reg = regno_reg_rtx[i];
5673 96415598 : if (lra_get_regno_hard_regno (i) < 0 && (x = get_equiv (reg)) != reg)
5674 : {
5675 716876 : bool pseudo_p = contains_reg_p (x, false, false);
5676 :
5677 : /* After RTL transformation, we cannot guarantee that
5678 : pseudo in the substitution was not reloaded which might
5679 : make equivalence invalid. For example, in reverse
5680 : equiv of p0
5681 :
5682 : p0 <- ...
5683 : ...
5684 : equiv_mem <- p0
5685 :
5686 : the memory address register was reloaded before the 2nd
5687 : insn. */
5688 716876 : if ((! first_p && pseudo_p)
5689 : /* We don't use DF for compilation speed sake. So it
5690 : is problematic to update live info when we use an
5691 : equivalence containing pseudos in more than one
5692 : BB. */
5693 710651 : || (pseudo_p && multi_block_pseudo_p (i))
5694 : /* If an init insn was deleted for some reason, cancel
5695 : the equiv. We could update the equiv insns after
5696 : transformations including an equiv insn deletion
5697 : but it is not worthy as such cases are extremely
5698 : rare. */
5699 705843 : || contains_deleted_insn_p (ira_reg_equiv[i].init_insns)
5700 : /* If it is not a reverse equivalence, we check that a
5701 : pseudo in rhs of the init insn is not dying in the
5702 : insn. Otherwise, the live info at the beginning of
5703 : the corresponding BB might be wrong after we
5704 : removed the insn. When the equiv can be a
5705 : constant, the right hand side of the init insn can
5706 : be a pseudo. */
5707 705843 : || (! reverse_equiv_p (i)
5708 704331 : && (init_insn_rhs_dead_pseudo_p (i)
5709 : /* If we reloaded the pseudo in an equivalence
5710 : init insn, we cannot remove the equiv init
5711 : insns and the init insns might write into
5712 : const memory in this case. */
5713 697761 : || contains_reloaded_insn_p (i)))
5714 : /* Prevent access beyond equivalent memory for
5715 : paradoxical subregs. */
5716 699273 : || (MEM_P (x)
5717 1114704 : && maybe_gt (GET_MODE_SIZE (lra_reg_info[i].biggest_mode),
5718 : GET_MODE_SIZE (GET_MODE (x))))
5719 1415430 : || (pic_offset_table_rtx
5720 54217 : && ((CONST_POOL_OK_P (PSEUDO_REGNO_MODE (i), x)
5721 8716 : && (targetm.preferred_reload_class
5722 4358 : (x, lra_get_allocno_class (i)) == NO_REGS))
5723 52338 : || contains_symbol_ref_p (x))))
5724 20975 : ira_reg_equiv[i].defined_p
5725 20975 : = ira_reg_equiv[i].caller_save_p = false;
5726 716876 : if (contains_reg_p (x, false, true))
5727 8963 : ira_reg_equiv[i].profitable_p = false;
5728 716876 : if (get_equiv (reg) != reg)
5729 691229 : bitmap_ior_into (equiv_insn_bitmap, &lra_reg_info[i].insn_bitmap);
5730 : }
5731 : }
5732 205281774 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
5733 202042222 : update_equiv (i);
5734 : /* We should add all insns containing pseudos which should be
5735 : substituted by their equivalences. */
5736 5552800 : EXECUTE_IF_SET_IN_BITMAP (equiv_insn_bitmap, 0, uid, bi)
5737 2313248 : lra_push_insn_by_uid (uid);
5738 3239552 : min_len = lra_insn_stack_length ();
5739 3239552 : new_insns_num = 0;
5740 3239552 : last_bb = NULL;
5741 3239552 : changed_p = false;
5742 3239552 : original_insn = NULL;
5743 164269652 : while ((new_min_len = lra_insn_stack_length ()) != 0)
5744 : {
5745 157790548 : curr_insn = lra_pop_insn ();
5746 157790548 : --new_min_len;
5747 157790548 : curr_bb = BLOCK_FOR_INSN (curr_insn);
5748 157790548 : if (curr_bb != last_bb)
5749 : {
5750 20354602 : last_bb = curr_bb;
5751 20354602 : bb_reload_num = lra_curr_reload_num;
5752 : }
5753 157790548 : if (min_len > new_min_len)
5754 : {
5755 : min_len = new_min_len;
5756 : new_insns_num = 0;
5757 : original_insn = curr_insn;
5758 : }
5759 7231532 : else if (combine_reload_insn (curr_insn, original_insn))
5760 : {
5761 3588 : continue;
5762 : }
5763 7227944 : if (new_insns_num > MAX_RELOAD_INSNS_NUMBER)
5764 0 : internal_error
5765 0 : ("maximum number of generated reload insns per insn achieved (%d)",
5766 : MAX_RELOAD_INSNS_NUMBER);
5767 157786960 : new_insns_num++;
5768 157786960 : if (DEBUG_INSN_P (curr_insn))
5769 : {
5770 : /* We need to check equivalence in debug insn and change
5771 : pseudo to the equivalent value if necessary. */
5772 50502861 : curr_id = lra_get_insn_recog_data (curr_insn);
5773 50502861 : if (bitmap_bit_p (equiv_insn_bitmap, INSN_UID (curr_insn)))
5774 : {
5775 29431 : rtx old = *curr_id->operand_loc[0];
5776 29431 : *curr_id->operand_loc[0]
5777 29431 : = simplify_replace_fn_rtx (old, NULL_RTX,
5778 : loc_equivalence_callback, curr_insn);
5779 29431 : if (old != *curr_id->operand_loc[0])
5780 : {
5781 : /* If we substitute pseudo by shared equivalence, we can fail
5782 : to update LRA reg info and this can result in many
5783 : unexpected consequences. So keep rtl unshared: */
5784 29431 : *curr_id->operand_loc[0]
5785 29431 : = copy_rtx (*curr_id->operand_loc[0]);
5786 29431 : lra_update_insn_regno_info (curr_insn);
5787 29431 : changed_p = true;
5788 : }
5789 : }
5790 : }
5791 107284099 : else if (INSN_P (curr_insn))
5792 : {
5793 106224062 : if ((set = single_set (curr_insn)) != NULL_RTX)
5794 : {
5795 100850379 : nosubreg_dest = SET_DEST (set);
5796 : /* The equivalence pseudo could be set up as SUBREG in a
5797 : case when it is a call restore insn in a mode
5798 : different from the pseudo mode. */
5799 100850379 : if (GET_CODE (nosubreg_dest) == SUBREG)
5800 1194972 : nosubreg_dest = SUBREG_REG (nosubreg_dest);
5801 101527464 : if ((REG_P (nosubreg_dest)
5802 74523167 : && (x = get_equiv (nosubreg_dest)) != nosubreg_dest
5803 : /* Remove insns which set up a pseudo whose value
5804 : cannot be changed. Such insns might be not in
5805 : init_insns because we don't update equiv data
5806 : during insn transformations.
5807 :
5808 : As an example, let suppose that a pseudo got
5809 : hard register and on the 1st pass was not
5810 : changed to equivalent constant. We generate an
5811 : additional insn setting up the pseudo because of
5812 : secondary memory movement. Then the pseudo is
5813 : spilled and we use the equiv constant. In this
5814 : case we should remove the additional insn and
5815 : this insn is not init_insns list. */
5816 695545 : && (! MEM_P (x) || MEM_READONLY_P (x)
5817 : /* Check that this is actually an insn setting
5818 : up the equivalence. */
5819 313925 : || in_list_p (curr_insn,
5820 313925 : ira_reg_equiv
5821 313925 : [REGNO (nosubreg_dest)].init_insns)))
5822 174697915 : || (((x = get_equiv (SET_SRC (set))) != SET_SRC (set))
5823 1979132 : && in_list_p (curr_insn,
5824 989566 : ira_reg_equiv
5825 989566 : [REGNO (SET_SRC (set))].init_insns)
5826 : /* This is a reverse equivalence to memory (see ira.cc)
5827 : in store insn. We can reload all the destination and
5828 : have an output reload which is a store to memory. If
5829 : we just remove the insn, we will have the output
5830 : reload storing an undefined value to the memory.
5831 : Check that we did not reload the memory to prevent a
5832 : wrong code generation. We could implement using the
5833 : equivalence still in such case but doing this is not
5834 : worth the efforts as such case is very rare. */
5835 1454 : && MEM_P (nosubreg_dest)))
5836 : {
5837 : /* This is equiv init insn of pseudo which did not get a
5838 : hard register -- remove the insn. */
5839 677085 : if (lra_dump_file != NULL)
5840 : {
5841 9 : fprintf (lra_dump_file,
5842 : " Removing equiv init insn %i (freq=%d)\n",
5843 3 : INSN_UID (curr_insn),
5844 6 : REG_FREQ_FROM_BB (BLOCK_FOR_INSN (curr_insn)));
5845 3 : dump_insn_slim (lra_dump_file, curr_insn);
5846 : }
5847 677085 : if (contains_reg_p (x, true, false))
5848 135574 : check_and_force_assignment_correctness_p = true;
5849 677085 : lra_set_insn_deleted (curr_insn);
5850 677085 : continue;
5851 : }
5852 : }
5853 105546977 : curr_id = lra_get_insn_recog_data (curr_insn);
5854 105546977 : curr_static_id = curr_id->insn_static_data;
5855 105546977 : init_curr_insn_input_reloads ();
5856 105546977 : init_curr_operand_mode ();
5857 105546977 : if (curr_insn_transform (false))
5858 : changed_p = true;
5859 : /* Check non-transformed insns too for equiv change as USE
5860 : or CLOBBER don't need reloads but can contain pseudos
5861 : being changed on their equivalences. */
5862 99464476 : else if (bitmap_bit_p (equiv_insn_bitmap, INSN_UID (curr_insn))
5863 99464476 : && loc_equivalence_change_p (&PATTERN (curr_insn)))
5864 : {
5865 8 : lra_update_insn_regno_info (curr_insn);
5866 8 : lra_push_insn_by_uid (INSN_UID (curr_insn));
5867 8 : changed_p = true;
5868 : }
5869 : }
5870 : }
5871 :
5872 : /* If we used a new hard regno, changed_p should be true because the
5873 : hard reg is assigned to a new pseudo. */
5874 3239552 : if (flag_checking && !changed_p)
5875 : {
5876 132440240 : for (i = FIRST_PSEUDO_REGISTER; i < new_regno_start; i++)
5877 129813563 : if (lra_reg_info[i].nrefs != 0
5878 189779049 : && (hard_regno = lra_get_regno_hard_regno (i)) >= 0)
5879 : {
5880 58530442 : int j, nregs = hard_regno_nregs (hard_regno,
5881 58530442 : PSEUDO_REGNO_MODE (i));
5882 :
5883 119085500 : for (j = 0; j < nregs; j++)
5884 60555058 : lra_assert (df_regs_ever_live_p (hard_regno + j));
5885 : }
5886 : }
5887 2626717 : if (changed_p)
5888 612838 : lra_dump_insns_if_possible ("changed func after local");
5889 3239552 : return changed_p;
5890 3239552 : }
5891 :
5892 : static void initiate_invariants (void);
5893 : static void finish_invariants (void);
5894 :
5895 : /* Initiate the LRA constraint pass. It is done once per
5896 : function. */
5897 : void
5898 1488370 : lra_constraints_init (void)
5899 : {
5900 1488370 : initiate_invariants ();
5901 1488370 : }
5902 :
5903 : /* Finalize the LRA constraint pass. It is done once per
5904 : function. */
5905 : void
5906 1488370 : lra_constraints_finish (void)
5907 : {
5908 1488370 : finish_invariants ();
5909 1488370 : }
5910 :
5911 :
5912 :
5913 : /* Structure describes invariants for ineheritance. */
5914 : struct lra_invariant
5915 : {
5916 : /* The order number of the invariant. */
5917 : int num;
5918 : /* The invariant RTX. */
5919 : rtx invariant_rtx;
5920 : /* The origin insn of the invariant. */
5921 : rtx_insn *insn;
5922 : };
5923 :
5924 : typedef lra_invariant invariant_t;
5925 : typedef invariant_t *invariant_ptr_t;
5926 : typedef const invariant_t *const_invariant_ptr_t;
5927 :
5928 : /* Pointer to the inheritance invariants. */
5929 : static vec<invariant_ptr_t> invariants;
5930 :
5931 : /* Allocation pool for the invariants. */
5932 : static object_allocator<lra_invariant> *invariants_pool;
5933 :
5934 : /* Hash table for the invariants. */
5935 : static htab_t invariant_table;
5936 :
5937 : /* Hash function for INVARIANT. */
5938 : static hashval_t
5939 172665 : invariant_hash (const void *invariant)
5940 : {
5941 172665 : rtx inv = ((const_invariant_ptr_t) invariant)->invariant_rtx;
5942 172665 : return lra_rtx_hash (inv);
5943 : }
5944 :
5945 : /* Equal function for invariants INVARIANT1 and INVARIANT2. */
5946 : static int
5947 57797 : invariant_eq_p (const void *invariant1, const void *invariant2)
5948 : {
5949 57797 : rtx inv1 = ((const_invariant_ptr_t) invariant1)->invariant_rtx;
5950 57797 : rtx inv2 = ((const_invariant_ptr_t) invariant2)->invariant_rtx;
5951 :
5952 57797 : return rtx_equal_p (inv1, inv2);
5953 : }
5954 :
5955 : /* Insert INVARIANT_RTX into the table if it is not there yet. Return
5956 : invariant which is in the table. */
5957 : static invariant_ptr_t
5958 172473 : insert_invariant (rtx invariant_rtx)
5959 : {
5960 172473 : void **entry_ptr;
5961 172473 : invariant_t invariant;
5962 172473 : invariant_ptr_t invariant_ptr;
5963 :
5964 172473 : invariant.invariant_rtx = invariant_rtx;
5965 172473 : entry_ptr = htab_find_slot (invariant_table, &invariant, INSERT);
5966 172473 : if (*entry_ptr == NULL)
5967 : {
5968 149442 : invariant_ptr = invariants_pool->allocate ();
5969 149442 : invariant_ptr->invariant_rtx = invariant_rtx;
5970 149442 : invariant_ptr->insn = NULL;
5971 149442 : invariants.safe_push (invariant_ptr);
5972 149442 : *entry_ptr = (void *) invariant_ptr;
5973 : }
5974 172473 : return (invariant_ptr_t) *entry_ptr;
5975 : }
5976 :
5977 : /* Initiate the invariant table. */
5978 : static void
5979 1488370 : initiate_invariants (void)
5980 : {
5981 1488370 : invariants.create (100);
5982 1488370 : invariants_pool
5983 1488370 : = new object_allocator<lra_invariant> ("Inheritance invariants");
5984 1488370 : invariant_table = htab_create (100, invariant_hash, invariant_eq_p, NULL);
5985 1488370 : }
5986 :
5987 : /* Finish the invariant table. */
5988 : static void
5989 1488370 : finish_invariants (void)
5990 : {
5991 1488370 : htab_delete (invariant_table);
5992 2976740 : delete invariants_pool;
5993 1488370 : invariants.release ();
5994 1488370 : }
5995 :
5996 : /* Make the invariant table empty. */
5997 : static void
5998 12499643 : clear_invariants (void)
5999 : {
6000 12499643 : htab_empty (invariant_table);
6001 12499643 : invariants_pool->release ();
6002 12499643 : invariants.truncate (0);
6003 12499643 : }
6004 :
6005 :
6006 :
6007 : /* This page contains code to do inheritance/split
6008 : transformations. */
6009 :
6010 : /* Number of reloads passed so far in current EBB. */
6011 : static int reloads_num;
6012 :
6013 : /* Number of calls passed so far in current EBB. */
6014 : static int calls_num;
6015 :
6016 : /* Index ID is the CALLS_NUM associated the last call we saw with
6017 : ABI identifier ID. */
6018 : static int last_call_for_abi[NUM_ABI_IDS];
6019 :
6020 : /* Which registers have been fully or partially clobbered by a call
6021 : since they were last used. */
6022 : static HARD_REG_SET full_and_partial_call_clobbers;
6023 :
6024 : /* Current reload pseudo check for validity of elements in
6025 : USAGE_INSNS. */
6026 : static int curr_usage_insns_check;
6027 :
6028 : /* Info about last usage of registers in EBB to do inheritance/split
6029 : transformation. Inheritance transformation is done from a spilled
6030 : pseudo and split transformations from a hard register or a pseudo
6031 : assigned to a hard register. */
6032 : struct usage_insns
6033 : {
6034 : /* If the value is equal to CURR_USAGE_INSNS_CHECK, then the member
6035 : value INSNS is valid. The insns is chain of optional debug insns
6036 : and a finishing non-debug insn using the corresponding reg. The
6037 : value is also used to mark the registers which are set up in the
6038 : current insn. The negated insn uid is used for this. */
6039 : int check;
6040 : /* Value of global reloads_num at the last insn in INSNS. */
6041 : int reloads_num;
6042 : /* Value of global reloads_nums at the last insn in INSNS. */
6043 : int calls_num;
6044 : /* It can be true only for splitting. And it means that the restore
6045 : insn should be put after insn given by the following member. */
6046 : bool after_p;
6047 : /* Next insns in the current EBB which use the original reg and the
6048 : original reg value is not changed between the current insn and
6049 : the next insns. In order words, e.g. for inheritance, if we need
6050 : to use the original reg value again in the next insns we can try
6051 : to use the value in a hard register from a reload insn of the
6052 : current insn. */
6053 : rtx insns;
6054 : };
6055 :
6056 : /* Map: regno -> corresponding pseudo usage insns. */
6057 : static struct usage_insns *usage_insns;
6058 :
6059 : static void
6060 243744206 : setup_next_usage_insn (int regno, rtx insn, int reloads_num, bool after_p)
6061 : {
6062 243744206 : usage_insns[regno].check = curr_usage_insns_check;
6063 243744206 : usage_insns[regno].insns = insn;
6064 243744206 : usage_insns[regno].reloads_num = reloads_num;
6065 243744206 : usage_insns[regno].calls_num = calls_num;
6066 243744206 : usage_insns[regno].after_p = after_p;
6067 243744206 : if (regno >= FIRST_PSEUDO_REGISTER && reg_renumber[regno] >= 0)
6068 109695820 : remove_from_hard_reg_set (&full_and_partial_call_clobbers,
6069 109695820 : PSEUDO_REGNO_MODE (regno),
6070 : reg_renumber[regno]);
6071 243744206 : }
6072 :
6073 : /* The function is used to form list REGNO usages which consists of
6074 : optional debug insns finished by a non-debug insn using REGNO.
6075 : RELOADS_NUM is current number of reload insns processed so far. */
6076 : static void
6077 138158755 : add_next_usage_insn (int regno, rtx_insn *insn, int reloads_num)
6078 : {
6079 138158755 : rtx next_usage_insns;
6080 :
6081 138158755 : if (usage_insns[regno].check == curr_usage_insns_check
6082 71949695 : && (next_usage_insns = usage_insns[regno].insns) != NULL_RTX
6083 210108450 : && DEBUG_INSN_P (insn))
6084 : {
6085 : /* Check that we did not add the debug insn yet. */
6086 13206053 : if (next_usage_insns != insn
6087 13206053 : && (GET_CODE (next_usage_insns) != INSN_LIST
6088 5877884 : || XEXP (next_usage_insns, 0) != insn))
6089 13206039 : usage_insns[regno].insns = gen_rtx_INSN_LIST (VOIDmode, insn,
6090 : next_usage_insns);
6091 : }
6092 124952702 : else if (NONDEBUG_INSN_P (insn))
6093 124456825 : setup_next_usage_insn (regno, insn, reloads_num, false);
6094 : else
6095 495877 : usage_insns[regno].check = 0;
6096 138158755 : }
6097 :
6098 : /* Return first non-debug insn in list USAGE_INSNS. */
6099 : static rtx_insn *
6100 1161344 : skip_usage_debug_insns (rtx usage_insns)
6101 : {
6102 1161344 : rtx insn;
6103 :
6104 : /* Skip debug insns. */
6105 1161344 : for (insn = usage_insns;
6106 1445619 : insn != NULL_RTX && GET_CODE (insn) == INSN_LIST;
6107 284275 : insn = XEXP (insn, 1))
6108 : ;
6109 1161344 : return safe_as_a <rtx_insn *> (insn);
6110 : }
6111 :
6112 : /* Return true if we need secondary memory moves for insn in
6113 : USAGE_INSNS after inserting inherited pseudo of class INHER_CL
6114 : into the insn. */
6115 : static bool
6116 1161351 : check_secondary_memory_needed_p (enum reg_class inher_cl ATTRIBUTE_UNUSED,
6117 : rtx usage_insns ATTRIBUTE_UNUSED)
6118 : {
6119 1161351 : rtx_insn *insn;
6120 1161351 : rtx set, dest;
6121 1161351 : enum reg_class cl;
6122 :
6123 1161351 : if (inher_cl == ALL_REGS
6124 1161351 : || (insn = skip_usage_debug_insns (usage_insns)) == NULL_RTX)
6125 : return false;
6126 1161344 : lra_assert (INSN_P (insn));
6127 1161344 : if ((set = single_set (insn)) == NULL_RTX || ! REG_P (SET_DEST (set)))
6128 : return false;
6129 1123509 : dest = SET_DEST (set);
6130 1123509 : if (! REG_P (dest))
6131 : return false;
6132 1123509 : lra_assert (inher_cl != NO_REGS);
6133 1123509 : cl = get_reg_class (REGNO (dest));
6134 1123509 : return (cl != NO_REGS && cl != ALL_REGS
6135 1123509 : && targetm.secondary_memory_needed (GET_MODE (dest), inher_cl, cl));
6136 : }
6137 :
6138 : /* Registers involved in inheritance/split in the current EBB
6139 : (inheritance/split pseudos and original registers). */
6140 : static bitmap_head check_only_regs;
6141 :
6142 : /* Reload pseudos cannot be involded in invariant inheritance in the
6143 : current EBB. */
6144 : static bitmap_head invalid_invariant_regs;
6145 :
6146 : /* Do inheritance transformations for insn INSN, which defines (if
6147 : DEF_P) or uses ORIGINAL_REGNO. NEXT_USAGE_INSNS specifies which
6148 : instruction in the EBB next uses ORIGINAL_REGNO; it has the same
6149 : form as the "insns" field of usage_insns. Return true if we
6150 : succeed in such transformation.
6151 :
6152 : The transformations look like:
6153 :
6154 : p <- ... i <- ...
6155 : ... p <- i (new insn)
6156 : ... =>
6157 : <- ... p ... <- ... i ...
6158 : or
6159 : ... i <- p (new insn)
6160 : <- ... p ... <- ... i ...
6161 : ... =>
6162 : <- ... p ... <- ... i ...
6163 : where p is a spilled original pseudo and i is a new inheritance pseudo.
6164 :
6165 :
6166 : The inheritance pseudo has the smallest class of two classes CL and
6167 : class of ORIGINAL REGNO. */
6168 : static bool
6169 1255877 : inherit_reload_reg (bool def_p, int original_regno,
6170 : enum reg_class cl, rtx_insn *insn, rtx next_usage_insns)
6171 : {
6172 1255877 : if (optimize_function_for_size_p (cfun))
6173 : return false;
6174 :
6175 1224954 : enum reg_class rclass = lra_get_allocno_class (original_regno);
6176 1224954 : rtx original_reg = regno_reg_rtx[original_regno];
6177 1224954 : rtx new_reg, usage_insn;
6178 1224954 : rtx_insn *new_insns;
6179 :
6180 1224954 : lra_assert (! usage_insns[original_regno].after_p);
6181 1224954 : if (lra_dump_file != NULL)
6182 2 : fprintf (lra_dump_file,
6183 : " <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
6184 1224954 : if (! ira_reg_classes_intersect_p[cl][rclass])
6185 : {
6186 63603 : if (lra_dump_file != NULL)
6187 : {
6188 0 : fprintf (lra_dump_file,
6189 : " Rejecting inheritance for %d "
6190 : "because of disjoint classes %s and %s\n",
6191 : original_regno, reg_class_names[cl],
6192 : reg_class_names[rclass]);
6193 0 : fprintf (lra_dump_file,
6194 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6195 : }
6196 63603 : return false;
6197 : }
6198 1161351 : if ((ira_class_subset_p[cl][rclass] && cl != rclass)
6199 : /* We don't use a subset of two classes because it can be
6200 : NO_REGS. This transformation is still profitable in most
6201 : cases even if the classes are not intersected as register
6202 : move is probably cheaper than a memory load. */
6203 435998 : || ira_class_hard_regs_num[cl] < ira_class_hard_regs_num[rclass])
6204 : {
6205 725353 : if (lra_dump_file != NULL)
6206 2 : fprintf (lra_dump_file, " Use smallest class of %s and %s\n",
6207 : reg_class_names[cl], reg_class_names[rclass]);
6208 :
6209 : rclass = cl;
6210 : }
6211 1161351 : if (check_secondary_memory_needed_p (rclass, next_usage_insns))
6212 : {
6213 : /* Reject inheritance resulting in secondary memory moves.
6214 : Otherwise, there is a danger in LRA cycling. Also such
6215 : transformation will be unprofitable. */
6216 12863 : if (lra_dump_file != NULL)
6217 : {
6218 0 : rtx_insn *insn = skip_usage_debug_insns (next_usage_insns);
6219 0 : rtx set = single_set (insn);
6220 :
6221 0 : lra_assert (set != NULL_RTX);
6222 :
6223 0 : rtx dest = SET_DEST (set);
6224 :
6225 0 : lra_assert (REG_P (dest));
6226 0 : fprintf (lra_dump_file,
6227 : " Rejecting inheritance for insn %d(%s)<-%d(%s) "
6228 : "as secondary mem is needed\n",
6229 0 : REGNO (dest), reg_class_names[get_reg_class (REGNO (dest))],
6230 0 : original_regno, reg_class_names[rclass]);
6231 0 : fprintf (lra_dump_file,
6232 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6233 : }
6234 12863 : return false;
6235 : }
6236 1148488 : if (ira_reg_class_min_nregs[rclass][GET_MODE (original_reg)]
6237 1148488 : != ira_reg_class_max_nregs[rclass][GET_MODE (original_reg)])
6238 : {
6239 24 : if (lra_dump_file != NULL)
6240 : {
6241 0 : fprintf (lra_dump_file,
6242 : " Rejecting inheritance for %d "
6243 : "because of requiring non-uniform class %s\n",
6244 : original_regno, reg_class_names[rclass]);
6245 0 : fprintf (lra_dump_file,
6246 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6247 : }
6248 24 : return false;
6249 : }
6250 1148464 : new_reg = lra_create_new_reg (GET_MODE (original_reg), original_reg,
6251 : rclass, NULL, "inheritance");
6252 1148464 : start_sequence ();
6253 1148464 : if (def_p)
6254 540286 : lra_emit_move (original_reg, new_reg);
6255 : else
6256 608178 : lra_emit_move (new_reg, original_reg);
6257 1148464 : new_insns = end_sequence ();
6258 1148464 : if (NEXT_INSN (new_insns) != NULL_RTX)
6259 : {
6260 0 : if (lra_dump_file != NULL)
6261 : {
6262 0 : fprintf (lra_dump_file,
6263 : " Rejecting inheritance %d->%d "
6264 : "as it results in 2 or more insns:\n",
6265 : original_regno, REGNO (new_reg));
6266 0 : dump_rtl_slim (lra_dump_file, new_insns, NULL, -1, 0);
6267 0 : fprintf (lra_dump_file,
6268 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6269 : }
6270 0 : return false;
6271 : }
6272 1148464 : lra_substitute_pseudo_within_insn (insn, original_regno, new_reg, false);
6273 1148464 : lra_update_insn_regno_info (insn);
6274 1148464 : if (! def_p)
6275 : /* We now have a new usage insn for original regno. */
6276 608178 : setup_next_usage_insn (original_regno, new_insns, reloads_num, false);
6277 1148464 : if (lra_dump_file != NULL)
6278 2 : fprintf (lra_dump_file, " Original reg change %d->%d (bb%d):\n",
6279 2 : original_regno, REGNO (new_reg), BLOCK_FOR_INSN (insn)->index);
6280 1148464 : lra_reg_info[REGNO (new_reg)].restore_rtx = regno_reg_rtx[original_regno];
6281 1148464 : bitmap_set_bit (&check_only_regs, REGNO (new_reg));
6282 1148464 : bitmap_set_bit (&check_only_regs, original_regno);
6283 1148464 : bitmap_set_bit (&lra_inheritance_pseudos, REGNO (new_reg));
6284 1148464 : if (def_p)
6285 540286 : lra_process_new_insns (insn, NULL, new_insns,
6286 : "Add original<-inheritance");
6287 : else
6288 608178 : lra_process_new_insns (insn, new_insns, NULL,
6289 : "Add inheritance<-original");
6290 2579738 : while (next_usage_insns != NULL_RTX)
6291 : {
6292 1431274 : if (GET_CODE (next_usage_insns) != INSN_LIST)
6293 : {
6294 1148464 : usage_insn = next_usage_insns;
6295 1148464 : lra_assert (NONDEBUG_INSN_P (usage_insn));
6296 : next_usage_insns = NULL;
6297 : }
6298 : else
6299 : {
6300 282810 : usage_insn = XEXP (next_usage_insns, 0);
6301 282810 : lra_assert (DEBUG_INSN_P (usage_insn));
6302 282810 : next_usage_insns = XEXP (next_usage_insns, 1);
6303 : }
6304 1431274 : lra_substitute_pseudo (&usage_insn, original_regno, new_reg, false,
6305 1431274 : DEBUG_INSN_P (usage_insn));
6306 1431274 : lra_update_insn_regno_info (as_a <rtx_insn *> (usage_insn));
6307 1431274 : if (lra_dump_file != NULL)
6308 : {
6309 2 : basic_block bb = BLOCK_FOR_INSN (usage_insn);
6310 2 : fprintf (lra_dump_file,
6311 : " Inheritance reuse change %d->%d (bb%d):\n",
6312 : original_regno, REGNO (new_reg),
6313 : bb ? bb->index : -1);
6314 2 : dump_insn_slim (lra_dump_file, as_a <rtx_insn *> (usage_insn));
6315 : }
6316 : }
6317 1148464 : if (lra_dump_file != NULL)
6318 2 : fprintf (lra_dump_file,
6319 : " >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
6320 : return true;
6321 : }
6322 :
6323 : /* Return true if we need a caller save/restore for pseudo REGNO which
6324 : was assigned to a hard register. */
6325 : static inline bool
6326 112319821 : need_for_call_save_p (int regno)
6327 : {
6328 112319821 : lra_assert (regno >= FIRST_PSEUDO_REGISTER && reg_renumber[regno] >= 0);
6329 112319821 : if (usage_insns[regno].calls_num < calls_num)
6330 : {
6331 : unsigned int abis = 0;
6332 114727093 : for (unsigned int i = 0; i < NUM_ABI_IDS; ++i)
6333 105901932 : if (last_call_for_abi[i] > usage_insns[regno].calls_num)
6334 8825161 : abis |= 1 << i;
6335 8825161 : gcc_assert (abis);
6336 8825161 : if (call_clobbered_in_region_p (abis, full_and_partial_call_clobbers,
6337 8825161 : PSEUDO_REGNO_MODE (regno),
6338 : reg_renumber[regno]))
6339 : return true;
6340 : }
6341 : return false;
6342 : }
6343 :
6344 : /* Global registers occurring in the current EBB. */
6345 : static bitmap_head ebb_global_regs;
6346 :
6347 : /* Return true if we need a split for hard register REGNO or pseudo
6348 : REGNO which was assigned to a hard register.
6349 : POTENTIAL_RELOAD_HARD_REGS contains hard registers which might be
6350 : used for reloads since the EBB end. It is an approximation of the
6351 : used hard registers in the split range. The exact value would
6352 : require expensive calculations. If we were aggressive with
6353 : splitting because of the approximation, the split pseudo will save
6354 : the same hard register assignment and will be removed in the undo
6355 : pass. We still need the approximation because too aggressive
6356 : splitting would result in too inaccurate cost calculation in the
6357 : assignment pass because of too many generated moves which will be
6358 : probably removed in the undo pass. */
6359 : static inline bool
6360 239382672 : need_for_split_p (HARD_REG_SET potential_reload_hard_regs, int regno)
6361 : {
6362 239382672 : int hard_regno = regno < FIRST_PSEUDO_REGISTER ? regno : reg_renumber[regno];
6363 :
6364 239382672 : lra_assert (hard_regno >= 0);
6365 239382672 : return ((TEST_HARD_REG_BIT (potential_reload_hard_regs, hard_regno)
6366 : /* Don't split eliminable hard registers, otherwise we can
6367 : split hard registers like hard frame pointer, which
6368 : lives on BB start/end according to DF-infrastructure,
6369 : when there is a pseudo assigned to the register and
6370 : living in the same BB. */
6371 665983 : && (regno >= FIRST_PSEUDO_REGISTER
6372 45116 : || ! TEST_HARD_REG_BIT (eliminable_regset, hard_regno))
6373 635588 : && ! TEST_HARD_REG_BIT (lra_no_alloc_regs, hard_regno)
6374 : /* Don't split call clobbered hard regs living through
6375 : calls, otherwise we might have a check problem in the
6376 : assign sub-pass as in the most cases (exception is a
6377 : situation when check_and_force_assignment_correctness_p value is
6378 : true) the assign pass assumes that all pseudos living
6379 : through calls are assigned to call saved hard regs. */
6380 621798 : && (regno >= FIRST_PSEUDO_REGISTER
6381 931 : || !TEST_HARD_REG_BIT (full_and_partial_call_clobbers, regno))
6382 : /* We need at least 2 reloads to make pseudo splitting
6383 : profitable. We should provide hard regno splitting in
6384 : any case to solve 1st insn scheduling problem when
6385 : moving hard register definition up might result in
6386 : impossibility to find hard register for reload pseudo of
6387 : small register class. */
6388 1243540 : && (usage_insns[regno].reloads_num
6389 1242637 : + (regno < FIRST_PSEUDO_REGISTER ? 0 : 3) < reloads_num)
6390 2562 : && (regno < FIRST_PSEUDO_REGISTER
6391 : /* For short living pseudos, spilling + inheritance can
6392 : be considered a substitution for splitting.
6393 : Therefore we do not splitting for local pseudos. It
6394 : decreases also aggressiveness of splitting. The
6395 : minimal number of references is chosen taking into
6396 : account that for 2 references splitting has no sense
6397 : as we can just spill the pseudo. */
6398 : || (regno >= FIRST_PSEUDO_REGISTER
6399 2515 : && lra_reg_info[regno].nrefs > 3
6400 2184 : && bitmap_bit_p (&ebb_global_regs, regno))))
6401 240047451 : || (regno >= FIRST_PSEUDO_REGISTER && need_for_call_save_p (regno)));
6402 : }
6403 :
6404 : /* Return class for the split pseudo created from original pseudo with
6405 : ALLOCNO_CLASS and MODE which got a hard register HARD_REGNO. We
6406 : choose subclass of ALLOCNO_CLASS which contains HARD_REGNO and
6407 : results in no secondary memory movements. */
6408 : static enum reg_class
6409 1363 : choose_split_class (enum reg_class allocno_class,
6410 : int hard_regno ATTRIBUTE_UNUSED,
6411 : machine_mode mode ATTRIBUTE_UNUSED)
6412 : {
6413 1363 : int i;
6414 1363 : enum reg_class cl, best_cl = NO_REGS;
6415 1363 : enum reg_class hard_reg_class ATTRIBUTE_UNUSED
6416 : = REGNO_REG_CLASS (hard_regno);
6417 :
6418 1363 : if (! targetm.secondary_memory_needed (mode, allocno_class, allocno_class)
6419 1363 : && TEST_HARD_REG_BIT (reg_class_contents[allocno_class], hard_regno))
6420 : return allocno_class;
6421 0 : for (i = 0;
6422 0 : (cl = reg_class_subclasses[allocno_class][i]) != LIM_REG_CLASSES;
6423 : i++)
6424 0 : if (! targetm.secondary_memory_needed (mode, cl, hard_reg_class)
6425 0 : && ! targetm.secondary_memory_needed (mode, hard_reg_class, cl)
6426 0 : && TEST_HARD_REG_BIT (reg_class_contents[cl], hard_regno)
6427 0 : && (best_cl == NO_REGS
6428 0 : || ira_class_hard_regs_num[best_cl] < ira_class_hard_regs_num[cl]))
6429 : best_cl = cl;
6430 : return best_cl;
6431 : }
6432 :
6433 : /* Copy any equivalence information from ORIGINAL_REGNO to NEW_REGNO. It only
6434 : makes sense to call this function if NEW_REGNO is always equal to
6435 : ORIGINAL_REGNO. Set up defined_p flag when caller_save_p flag is set up and
6436 : CALL_SAVE_P is true. */
6437 :
6438 : static void
6439 640981 : lra_copy_reg_equiv (unsigned int new_regno, unsigned int original_regno,
6440 : bool call_save_p)
6441 : {
6442 640981 : if (!ira_reg_equiv[original_regno].defined_p
6443 576437 : && !(call_save_p && ira_reg_equiv[original_regno].caller_save_p))
6444 : return;
6445 :
6446 64718 : ira_expand_reg_equiv ();
6447 64718 : ira_reg_equiv[new_regno].defined_p = true;
6448 64718 : if (ira_reg_equiv[original_regno].memory)
6449 32008 : ira_reg_equiv[new_regno].memory
6450 32008 : = copy_rtx (ira_reg_equiv[original_regno].memory);
6451 64718 : if (ira_reg_equiv[original_regno].constant)
6452 26349 : ira_reg_equiv[new_regno].constant
6453 26349 : = copy_rtx (ira_reg_equiv[original_regno].constant);
6454 64718 : if (ira_reg_equiv[original_regno].invariant)
6455 6361 : ira_reg_equiv[new_regno].invariant
6456 6361 : = copy_rtx (ira_reg_equiv[original_regno].invariant);
6457 : }
6458 :
6459 : /* Do split transformations for insn INSN, which defines or uses
6460 : ORIGINAL_REGNO. NEXT_USAGE_INSNS specifies which instruction in
6461 : the EBB next uses ORIGINAL_REGNO; it has the same form as the
6462 : "insns" field of usage_insns. If TO is not NULL, we don't use
6463 : usage_insns, we put restore insns after TO insn. It is a case when
6464 : we call it from lra_split_hard_reg_for, outside the inheritance
6465 : pass.
6466 :
6467 : The transformations look like:
6468 :
6469 : p <- ... p <- ...
6470 : ... s <- p (new insn -- save)
6471 : ... =>
6472 : ... p <- s (new insn -- restore)
6473 : <- ... p ... <- ... p ...
6474 : or
6475 : <- ... p ... <- ... p ...
6476 : ... s <- p (new insn -- save)
6477 : ... =>
6478 : ... p <- s (new insn -- restore)
6479 : <- ... p ... <- ... p ...
6480 :
6481 : where p is an original pseudo got a hard register or a hard
6482 : register and s is a new split pseudo. The save is put before INSN
6483 : if BEFORE_P is true. Return true if we succeed in such
6484 : transformation. */
6485 : static bool
6486 642619 : split_reg (bool before_p, int original_regno, rtx_insn *insn,
6487 : rtx next_usage_insns, rtx_insn *to)
6488 : {
6489 642619 : enum reg_class rclass;
6490 642619 : rtx original_reg;
6491 642619 : int hard_regno, nregs;
6492 642619 : rtx new_reg, usage_insn;
6493 642619 : rtx_insn *restore, *save;
6494 642619 : bool after_p;
6495 642619 : bool call_save_p;
6496 642619 : machine_mode mode;
6497 :
6498 642619 : if (original_regno < FIRST_PSEUDO_REGISTER)
6499 : {
6500 206 : rclass = ira_allocno_class_translate[REGNO_REG_CLASS (original_regno)];
6501 206 : hard_regno = original_regno;
6502 206 : call_save_p = false;
6503 206 : nregs = 1;
6504 206 : mode = lra_reg_info[hard_regno].biggest_mode;
6505 206 : machine_mode reg_rtx_mode = GET_MODE (regno_reg_rtx[hard_regno]);
6506 : /* A reg can have a biggest_mode of VOIDmode if it was only ever seen as
6507 : part of a multi-word register. In that case, just use the reg_rtx
6508 : mode. Do the same also if the biggest mode was larger than a register
6509 : or we can not compare the modes. Otherwise, limit the size to that of
6510 : the biggest access in the function or to the natural mode at least. */
6511 206 : if (mode == VOIDmode
6512 206 : || !ordered_p (GET_MODE_PRECISION (mode),
6513 206 : GET_MODE_PRECISION (reg_rtx_mode))
6514 206 : || paradoxical_subreg_p (mode, reg_rtx_mode)
6515 411 : || maybe_gt (GET_MODE_PRECISION (reg_rtx_mode), GET_MODE_PRECISION (mode)))
6516 : {
6517 642619 : original_reg = regno_reg_rtx[hard_regno];
6518 642619 : mode = reg_rtx_mode;
6519 : }
6520 : else
6521 189 : original_reg = gen_rtx_REG (mode, hard_regno);
6522 : }
6523 : else
6524 : {
6525 642413 : mode = PSEUDO_REGNO_MODE (original_regno);
6526 642413 : hard_regno = reg_renumber[original_regno];
6527 642413 : nregs = hard_regno_nregs (hard_regno, mode);
6528 642413 : rclass = lra_get_allocno_class (original_regno);
6529 642413 : original_reg = regno_reg_rtx[original_regno];
6530 642413 : call_save_p = need_for_call_save_p (original_regno);
6531 : }
6532 642619 : lra_assert (hard_regno >= 0);
6533 642619 : if (lra_dump_file != NULL)
6534 0 : fprintf (lra_dump_file,
6535 : " ((((((((((((((((((((((((((((((((((((((((((((((((\n");
6536 :
6537 642619 : if (call_save_p)
6538 : {
6539 641256 : mode = HARD_REGNO_CALLER_SAVE_MODE (hard_regno,
6540 : hard_regno_nregs (hard_regno, mode),
6541 : mode);
6542 641256 : new_reg = lra_create_new_reg (mode, NULL_RTX, NO_REGS, NULL, "save");
6543 : }
6544 : else
6545 : {
6546 1363 : rclass = choose_split_class (rclass, hard_regno, mode);
6547 1363 : if (rclass == NO_REGS)
6548 : {
6549 0 : if (lra_dump_file != NULL)
6550 : {
6551 0 : fprintf (lra_dump_file,
6552 : " Rejecting split of %d(%s): "
6553 : "no good reg class for %d(%s)\n",
6554 : original_regno,
6555 0 : reg_class_names[lra_get_allocno_class (original_regno)],
6556 : hard_regno,
6557 0 : reg_class_names[REGNO_REG_CLASS (hard_regno)]);
6558 0 : fprintf
6559 0 : (lra_dump_file,
6560 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
6561 : }
6562 0 : return false;
6563 : }
6564 : /* Split_if_necessary can split hard registers used as part of a
6565 : multi-register mode but splits each register individually. The
6566 : mode used for each independent register may not be supported
6567 : so reject the split. Splitting the wider mode should theoretically
6568 : be possible but is not implemented. */
6569 1363 : if (!targetm.hard_regno_mode_ok (hard_regno, mode))
6570 : {
6571 0 : if (lra_dump_file != NULL)
6572 : {
6573 0 : fprintf (lra_dump_file,
6574 : " Rejecting split of %d(%s): unsuitable mode %s\n",
6575 : original_regno,
6576 0 : reg_class_names[lra_get_allocno_class (original_regno)],
6577 0 : GET_MODE_NAME (mode));
6578 0 : fprintf
6579 0 : (lra_dump_file,
6580 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
6581 : }
6582 0 : return false;
6583 : }
6584 1363 : new_reg = lra_create_new_reg (mode, original_reg, rclass, NULL, "split");
6585 1363 : reg_renumber[REGNO (new_reg)] = hard_regno;
6586 : }
6587 642619 : int new_regno = REGNO (new_reg);
6588 642619 : save = emit_spill_move (true, new_reg, original_reg);
6589 642619 : if (NEXT_INSN (save) != NULL_RTX && !call_save_p)
6590 : {
6591 0 : if (lra_dump_file != NULL)
6592 : {
6593 0 : fprintf
6594 0 : (lra_dump_file,
6595 : " Rejecting split %d->%d resulting in > 2 save insns:\n",
6596 : original_regno, new_regno);
6597 0 : dump_rtl_slim (lra_dump_file, save, NULL, -1, 0);
6598 0 : fprintf (lra_dump_file,
6599 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
6600 : }
6601 0 : return false;
6602 : }
6603 642619 : restore = emit_spill_move (false, new_reg, original_reg);
6604 642619 : if (NEXT_INSN (restore) != NULL_RTX && !call_save_p)
6605 : {
6606 0 : if (lra_dump_file != NULL)
6607 : {
6608 0 : fprintf (lra_dump_file,
6609 : " Rejecting split %d->%d "
6610 : "resulting in > 2 restore insns:\n",
6611 : original_regno, new_regno);
6612 0 : dump_rtl_slim (lra_dump_file, restore, NULL, -1, 0);
6613 0 : fprintf (lra_dump_file,
6614 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
6615 : }
6616 0 : return false;
6617 : }
6618 : /* Transfer equivalence information to the spill register, so that
6619 : if we fail to allocate the spill register, we have the option of
6620 : rematerializing the original value instead of spilling to the stack. */
6621 642619 : if (!HARD_REGISTER_NUM_P (original_regno)
6622 642413 : && mode == PSEUDO_REGNO_MODE (original_regno))
6623 640981 : lra_copy_reg_equiv (new_regno, original_regno, call_save_p);
6624 642619 : lra_reg_info[new_regno].restore_rtx = regno_reg_rtx[original_regno];
6625 642619 : bitmap_set_bit (&lra_split_regs, new_regno);
6626 642619 : if (to != NULL)
6627 : {
6628 159 : lra_assert (next_usage_insns == NULL);
6629 159 : usage_insn = to;
6630 159 : after_p = true;
6631 : }
6632 : else
6633 : {
6634 : /* We need check_only_regs only inside the inheritance pass. */
6635 642460 : bitmap_set_bit (&check_only_regs, new_regno);
6636 642460 : bitmap_set_bit (&check_only_regs, original_regno);
6637 642460 : after_p = usage_insns[original_regno].after_p;
6638 739710 : for (;;)
6639 : {
6640 739710 : if (GET_CODE (next_usage_insns) != INSN_LIST)
6641 : {
6642 642460 : usage_insn = next_usage_insns;
6643 642460 : break;
6644 : }
6645 97250 : usage_insn = XEXP (next_usage_insns, 0);
6646 97250 : lra_assert (DEBUG_INSN_P (usage_insn));
6647 97250 : next_usage_insns = XEXP (next_usage_insns, 1);
6648 97250 : lra_substitute_pseudo (&usage_insn, original_regno, new_reg, false,
6649 : true);
6650 97250 : lra_update_insn_regno_info (as_a <rtx_insn *> (usage_insn));
6651 97250 : if (lra_dump_file != NULL)
6652 : {
6653 0 : fprintf (lra_dump_file, " Split reuse change %d->%d:\n",
6654 : original_regno, new_regno);
6655 0 : dump_insn_slim (lra_dump_file, as_a <rtx_insn *> (usage_insn));
6656 : }
6657 : }
6658 : }
6659 642619 : lra_assert (NOTE_P (usage_insn) || NONDEBUG_INSN_P (usage_insn));
6660 642619 : lra_assert (usage_insn != insn || (after_p && before_p));
6661 1086009 : lra_process_new_insns (as_a <rtx_insn *> (usage_insn),
6662 : after_p ? NULL : restore,
6663 : after_p ? restore : NULL,
6664 : call_save_p ? "Add reg<-save" : "Add reg<-split");
6665 642619 : if (call_save_p
6666 641256 : && first_call_insn != NULL
6667 1283875 : && BLOCK_FOR_INSN (first_call_insn) != BLOCK_FOR_INSN (insn))
6668 : /* PR116028: If original_regno is a pseudo that has been assigned a
6669 : callee-saved hard register, then emit the spill insn before the call
6670 : insn 'first_call_insn' instead of adjacent to 'insn'. If 'insn'
6671 : and 'first_call_insn' belong to the same EBB but to two separate
6672 : BBs, and if 'insn' is present in the entry BB, then generating the
6673 : spill insn in the entry BB can prevent shrink wrap from happening.
6674 : This is because the spill insn references the stack pointer and
6675 : hence the prolog gets generated in the entry BB itself. It is
6676 : also more efficient to generate the spill before
6677 : 'first_call_insn' as the spill now occurs only in the path
6678 : containing the call. */
6679 25651 : lra_process_new_insns (first_call_insn, save, NULL, "Add save<-reg");
6680 : else
6681 1234662 : lra_process_new_insns (insn, before_p ? save : NULL,
6682 : before_p ? NULL : save,
6683 : call_save_p ? "Add save<-reg" : "Add split<-reg");
6684 642619 : if (nregs > 1 || original_regno < FIRST_PSEUDO_REGISTER)
6685 : /* If we are trying to split multi-register. We should check
6686 : conflicts on the next assignment sub-pass. IRA can allocate on
6687 : sub-register levels, LRA do this on pseudos level right now and
6688 : this discrepancy may create allocation conflicts after
6689 : splitting.
6690 :
6691 : If we are trying to split hard register we should also check conflicts
6692 : as such splitting can create artificial conflict of the hard register
6693 : with another pseudo because of simplified conflict calculation in
6694 : LRA. */
6695 9635 : check_and_force_assignment_correctness_p = true;
6696 642619 : if (lra_dump_file != NULL)
6697 0 : fprintf (lra_dump_file,
6698 : " ))))))))))))))))))))))))))))))))))))))))))))))))\n");
6699 : return true;
6700 : }
6701 :
6702 : /* Split a hard reg for reload pseudo REGNO having RCLASS and living
6703 : in the range [FROM, TO]. Return true if did a split. Otherwise,
6704 : return false. */
6705 : bool
6706 1570 : spill_hard_reg_in_range (int regno, enum reg_class rclass, rtx_insn *from, rtx_insn *to)
6707 : {
6708 1570 : int i, hard_regno;
6709 1570 : int rclass_size;
6710 1570 : rtx_insn *insn;
6711 1570 : unsigned int uid;
6712 1570 : bitmap_iterator bi;
6713 1570 : HARD_REG_SET ignore;
6714 :
6715 1570 : lra_assert (from != NULL && to != NULL);
6716 1570 : ignore = lra_no_alloc_regs;
6717 4523 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi)
6718 : {
6719 2953 : lra_insn_recog_data_t id = lra_insn_recog_data[uid];
6720 2953 : struct lra_static_insn_data *static_id = id->insn_static_data;
6721 2953 : struct lra_insn_reg *reg;
6722 :
6723 9848 : for (reg = id->regs; reg != NULL; reg = reg->next)
6724 6895 : if (reg->regno < FIRST_PSEUDO_REGISTER)
6725 157 : SET_HARD_REG_BIT (ignore, reg->regno);
6726 4609 : for (reg = static_id->hard_regs; reg != NULL; reg = reg->next)
6727 1656 : SET_HARD_REG_BIT (ignore, reg->regno);
6728 : }
6729 1570 : rclass_size = ira_class_hard_regs_num[rclass];
6730 4212 : for (i = 0; i < rclass_size; i++)
6731 : {
6732 2801 : hard_regno = ira_class_hard_regs[rclass][i];
6733 2801 : if (! TEST_HARD_REG_BIT (lra_reg_info[regno].conflict_hard_regs, hard_regno)
6734 2801 : || TEST_HARD_REG_BIT (ignore, hard_regno))
6735 2636 : continue;
6736 476 : for (insn = from; insn != NEXT_INSN (to); insn = NEXT_INSN (insn))
6737 : {
6738 317 : struct lra_static_insn_data *static_id;
6739 317 : struct lra_insn_reg *reg;
6740 :
6741 317 : if (!INSN_P (insn))
6742 0 : continue;
6743 317 : if (bitmap_bit_p (&lra_reg_info[hard_regno].insn_bitmap,
6744 317 : INSN_UID (insn)))
6745 : break;
6746 311 : static_id = lra_get_insn_recog_data (insn)->insn_static_data;
6747 365 : for (reg = static_id->hard_regs; reg != NULL; reg = reg->next)
6748 54 : if (reg->regno == hard_regno)
6749 : break;
6750 : if (reg != NULL)
6751 : break;
6752 : }
6753 165 : if (insn != NEXT_INSN (to))
6754 6 : continue;
6755 159 : if (split_reg (true, hard_regno, from, NULL, to))
6756 : return true;
6757 : }
6758 : return false;
6759 : }
6760 :
6761 : /* Recognize that we need a split transformation for insn INSN, which
6762 : defines or uses REGNO in its insn biggest MODE (we use it only if
6763 : REGNO is a hard register). POTENTIAL_RELOAD_HARD_REGS contains
6764 : hard registers which might be used for reloads since the EBB end.
6765 : Put the save before INSN if BEFORE_P is true. MAX_UID is maximla
6766 : uid before starting INSN processing. Return true if we succeed in
6767 : such transformation. */
6768 : static bool
6769 197965924 : split_if_necessary (int regno, machine_mode mode,
6770 : HARD_REG_SET potential_reload_hard_regs,
6771 : bool before_p, rtx_insn *insn, int max_uid)
6772 : {
6773 197965924 : bool res = false;
6774 197965924 : int i, nregs = 1;
6775 197965924 : rtx next_usage_insns;
6776 :
6777 197965924 : if (regno < FIRST_PSEUDO_REGISTER)
6778 93128599 : nregs = hard_regno_nregs (regno, mode);
6779 396287249 : for (i = 0; i < nregs; i++)
6780 198321325 : if (usage_insns[regno + i].check == curr_usage_insns_check
6781 132282930 : && (next_usage_insns = usage_insns[regno + i].insns) != NULL_RTX
6782 : /* To avoid processing the register twice or more. */
6783 132282930 : && ((GET_CODE (next_usage_insns) != INSN_LIST
6784 128209802 : && INSN_UID (next_usage_insns) < max_uid)
6785 4073128 : || (GET_CODE (next_usage_insns) == INSN_LIST
6786 4073128 : && (INSN_UID (XEXP (next_usage_insns, 0)) < max_uid)))
6787 132282930 : && need_for_split_p (potential_reload_hard_regs, regno + i)
6788 198595058 : && split_reg (before_p, regno + i, insn, next_usage_insns, NULL))
6789 : res = true;
6790 197965924 : return res;
6791 : }
6792 :
6793 : /* Return TRUE if rtx X is considered as an invariant for
6794 : inheritance. */
6795 : static bool
6796 11485185 : invariant_p (const_rtx x)
6797 : {
6798 11485185 : machine_mode mode;
6799 11485185 : const char *fmt;
6800 11485185 : enum rtx_code code;
6801 11485185 : int i, j;
6802 :
6803 11485185 : if (side_effects_p (x))
6804 : return false;
6805 :
6806 11459124 : code = GET_CODE (x);
6807 11459124 : mode = GET_MODE (x);
6808 11459124 : if (code == SUBREG)
6809 : {
6810 463609 : x = SUBREG_REG (x);
6811 463609 : code = GET_CODE (x);
6812 463609 : mode = wider_subreg_mode (mode, GET_MODE (x));
6813 : }
6814 :
6815 11459124 : if (MEM_P (x))
6816 : return false;
6817 :
6818 9732875 : if (REG_P (x))
6819 : {
6820 3469383 : int i, nregs, regno = REGNO (x);
6821 :
6822 3469383 : if (regno >= FIRST_PSEUDO_REGISTER || regno == STACK_POINTER_REGNUM
6823 898181 : || TEST_HARD_REG_BIT (eliminable_regset, regno)
6824 3486767 : || GET_MODE_CLASS (GET_MODE (x)) == MODE_CC)
6825 : return false;
6826 2 : nregs = hard_regno_nregs (regno, mode);
6827 2 : for (i = 0; i < nregs; i++)
6828 2 : if (! fixed_regs[regno + i]
6829 : /* A hard register may be clobbered in the current insn
6830 : but we can ignore this case because if the hard
6831 : register is used it should be set somewhere after the
6832 : clobber. */
6833 2 : || bitmap_bit_p (&invalid_invariant_regs, regno + i))
6834 2 : return false;
6835 : }
6836 6263492 : fmt = GET_RTX_FORMAT (code);
6837 11006421 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
6838 : {
6839 8313082 : if (fmt[i] == 'e')
6840 : {
6841 5458841 : if (! invariant_p (XEXP (x, i)))
6842 : return false;
6843 : }
6844 2854241 : else if (fmt[i] == 'E')
6845 : {
6846 642505 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
6847 530706 : if (! invariant_p (XVECEXP (x, i, j)))
6848 : return false;
6849 : }
6850 : }
6851 : return true;
6852 : }
6853 :
6854 : /* We have 'dest_reg <- invariant'. Let us try to make an invariant
6855 : inheritance transformation (using dest_reg instead invariant in a
6856 : subsequent insn). */
6857 : static bool
6858 172473 : process_invariant_for_inheritance (rtx dst_reg, rtx invariant_rtx)
6859 : {
6860 172473 : invariant_ptr_t invariant_ptr;
6861 172473 : rtx_insn *insn, *new_insns;
6862 172473 : rtx insn_set, insn_reg, new_reg;
6863 172473 : int insn_regno;
6864 172473 : bool succ_p = false;
6865 172473 : int dst_regno = REGNO (dst_reg);
6866 172473 : machine_mode dst_mode = GET_MODE (dst_reg);
6867 172473 : enum reg_class cl = lra_get_allocno_class (dst_regno), insn_reg_cl;
6868 :
6869 172473 : invariant_ptr = insert_invariant (invariant_rtx);
6870 172473 : if ((insn = invariant_ptr->insn) != NULL_RTX)
6871 : {
6872 : /* We have a subsequent insn using the invariant. */
6873 23031 : insn_set = single_set (insn);
6874 23031 : lra_assert (insn_set != NULL);
6875 23031 : insn_reg = SET_DEST (insn_set);
6876 23031 : lra_assert (REG_P (insn_reg));
6877 23031 : insn_regno = REGNO (insn_reg);
6878 23031 : insn_reg_cl = lra_get_allocno_class (insn_regno);
6879 :
6880 23031 : if (dst_mode == GET_MODE (insn_reg)
6881 : /* We should consider only result move reg insns which are
6882 : cheap. */
6883 22965 : && targetm.register_move_cost (dst_mode, cl, insn_reg_cl) == 2
6884 45410 : && targetm.register_move_cost (dst_mode, cl, cl) == 2)
6885 : {
6886 22379 : if (lra_dump_file != NULL)
6887 0 : fprintf (lra_dump_file,
6888 : " [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[\n");
6889 22379 : new_reg = lra_create_new_reg (dst_mode, dst_reg, cl, NULL,
6890 : "invariant inheritance");
6891 22379 : bitmap_set_bit (&lra_inheritance_pseudos, REGNO (new_reg));
6892 22379 : bitmap_set_bit (&check_only_regs, REGNO (new_reg));
6893 22379 : lra_reg_info[REGNO (new_reg)].restore_rtx = PATTERN (insn);
6894 22379 : start_sequence ();
6895 22379 : lra_emit_move (new_reg, dst_reg);
6896 22379 : new_insns = end_sequence ();
6897 22379 : lra_process_new_insns (curr_insn, NULL, new_insns,
6898 : "Add invariant inheritance<-original");
6899 22379 : start_sequence ();
6900 22379 : lra_emit_move (SET_DEST (insn_set), new_reg);
6901 22379 : new_insns = end_sequence ();
6902 22379 : lra_process_new_insns (insn, NULL, new_insns,
6903 : "Changing reload<-inheritance");
6904 22379 : lra_set_insn_deleted (insn);
6905 22379 : succ_p = true;
6906 22379 : if (lra_dump_file != NULL)
6907 : {
6908 0 : fprintf (lra_dump_file,
6909 : " Invariant inheritance reuse change %d (bb%d):\n",
6910 0 : REGNO (new_reg), BLOCK_FOR_INSN (insn)->index);
6911 0 : dump_insn_slim (lra_dump_file, insn);
6912 0 : fprintf (lra_dump_file,
6913 : " ]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]\n");
6914 : }
6915 : }
6916 : }
6917 172473 : invariant_ptr->insn = curr_insn;
6918 172473 : return succ_p;
6919 : }
6920 :
6921 : /* Check only registers living at the current program point in the
6922 : current EBB. */
6923 : static bitmap_head live_regs;
6924 :
6925 : /* Update live info in EBB given by its HEAD and TAIL insns after
6926 : inheritance/split transformation. The function removes dead moves
6927 : too. */
6928 : static void
6929 733513 : update_ebb_live_info (rtx_insn *head, rtx_insn *tail)
6930 : {
6931 733513 : unsigned int j;
6932 733513 : int i, regno;
6933 733513 : bool live_p;
6934 733513 : rtx_insn *prev_insn;
6935 733513 : rtx set;
6936 733513 : bool remove_p;
6937 733513 : basic_block last_bb, prev_bb, curr_bb;
6938 733513 : bitmap_iterator bi;
6939 733513 : struct lra_insn_reg *reg;
6940 733513 : edge e;
6941 733513 : edge_iterator ei;
6942 :
6943 733513 : last_bb = BLOCK_FOR_INSN (tail);
6944 733513 : prev_bb = NULL;
6945 733513 : for (curr_insn = tail;
6946 37074098 : curr_insn != PREV_INSN (head);
6947 36340585 : curr_insn = prev_insn)
6948 : {
6949 36340585 : prev_insn = PREV_INSN (curr_insn);
6950 : /* We need to process empty blocks too. They contain
6951 : NOTE_INSN_BASIC_BLOCK referring for the basic block. */
6952 36340585 : if (NOTE_P (curr_insn) && NOTE_KIND (curr_insn) != NOTE_INSN_BASIC_BLOCK)
6953 1407670 : continue;
6954 34932915 : curr_bb = BLOCK_FOR_INSN (curr_insn);
6955 34932915 : if (curr_bb != prev_bb)
6956 : {
6957 1462766 : if (prev_bb != NULL)
6958 : {
6959 : /* Update df_get_live_in (prev_bb): */
6960 54277798 : EXECUTE_IF_SET_IN_BITMAP (&check_only_regs, 0, j, bi)
6961 53548545 : if (bitmap_bit_p (&live_regs, j))
6962 1606599 : bitmap_set_bit (df_get_live_in (prev_bb), j);
6963 : else
6964 51941946 : bitmap_clear_bit (df_get_live_in (prev_bb), j);
6965 : }
6966 1462766 : if (curr_bb != last_bb)
6967 : {
6968 : /* Update df_get_live_out (curr_bb): */
6969 54277798 : EXECUTE_IF_SET_IN_BITMAP (&check_only_regs, 0, j, bi)
6970 : {
6971 53548545 : live_p = bitmap_bit_p (&live_regs, j);
6972 53548545 : if (! live_p)
6973 155733069 : FOR_EACH_EDGE (e, ei, curr_bb->succs)
6974 103844297 : if (bitmap_bit_p (df_get_live_in (e->dest), j))
6975 : {
6976 : live_p = true;
6977 : break;
6978 : }
6979 51941946 : if (live_p)
6980 1659773 : bitmap_set_bit (df_get_live_out (curr_bb), j);
6981 : else
6982 51888772 : bitmap_clear_bit (df_get_live_out (curr_bb), j);
6983 : }
6984 : }
6985 1462766 : prev_bb = curr_bb;
6986 1462766 : bitmap_and (&live_regs, &check_only_regs, df_get_live_out (curr_bb));
6987 : }
6988 34932915 : if (! NONDEBUG_INSN_P (curr_insn))
6989 12869205 : continue;
6990 22063710 : curr_id = lra_get_insn_recog_data (curr_insn);
6991 22063710 : curr_static_id = curr_id->insn_static_data;
6992 22063710 : remove_p = false;
6993 22063710 : if ((set = single_set (curr_insn)) != NULL_RTX
6994 21376493 : && REG_P (SET_DEST (set))
6995 17044227 : && (regno = REGNO (SET_DEST (set))) >= FIRST_PSEUDO_REGISTER
6996 12669212 : && SET_DEST (set) != pic_offset_table_rtx
6997 12662644 : && bitmap_bit_p (&check_only_regs, regno)
6998 25251167 : && ! bitmap_bit_p (&live_regs, regno))
6999 : remove_p = true;
7000 : /* See which defined values die here. */
7001 60958759 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7002 38895049 : if (reg->type == OP_OUT && ! reg->subreg_p)
7003 15190260 : bitmap_clear_bit (&live_regs, reg->regno);
7004 26250773 : for (reg = curr_static_id->hard_regs; reg != NULL; reg = reg->next)
7005 4187063 : if (reg->type == OP_OUT && ! reg->subreg_p)
7006 3180013 : bitmap_clear_bit (&live_regs, reg->regno);
7007 22063710 : if (curr_id->arg_hard_regs != NULL)
7008 : /* Make clobbered argument hard registers die. */
7009 3326697 : for (i = 0; (regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7010 2387754 : if (regno >= FIRST_PSEUDO_REGISTER)
7011 189840 : bitmap_clear_bit (&live_regs, regno - FIRST_PSEUDO_REGISTER);
7012 : /* Mark each used value as live. */
7013 60958759 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7014 38895049 : if (reg->type != OP_OUT
7015 38895049 : && bitmap_bit_p (&check_only_regs, reg->regno))
7016 4487714 : bitmap_set_bit (&live_regs, reg->regno);
7017 26250773 : for (reg = curr_static_id->hard_regs; reg != NULL; reg = reg->next)
7018 4187063 : if (reg->type != OP_OUT
7019 4187063 : && bitmap_bit_p (&check_only_regs, reg->regno))
7020 0 : bitmap_set_bit (&live_regs, reg->regno);
7021 22063710 : if (curr_id->arg_hard_regs != NULL)
7022 : /* Make used argument hard registers live. */
7023 3326697 : for (i = 0; (regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7024 2387754 : if (regno < FIRST_PSEUDO_REGISTER
7025 2387754 : && bitmap_bit_p (&check_only_regs, regno))
7026 0 : bitmap_set_bit (&live_regs, regno);
7027 : /* It is quite important to remove dead move insns because it
7028 : means removing dead store. We don't need to process them for
7029 : constraints. */
7030 22063710 : if (remove_p)
7031 : {
7032 295834 : if (lra_dump_file != NULL)
7033 : {
7034 2 : fprintf (lra_dump_file, " Removing dead insn:\n ");
7035 2 : dump_insn_slim (lra_dump_file, curr_insn);
7036 : }
7037 295834 : lra_set_insn_deleted (curr_insn);
7038 : }
7039 : }
7040 733513 : }
7041 :
7042 : /* The structure describes info to do an inheritance for the current
7043 : insn. We need to collect such info first before doing the
7044 : transformations because the transformations change the insn
7045 : internal representation. */
7046 : struct to_inherit
7047 : {
7048 : /* Original regno. */
7049 : int regno;
7050 : /* Subsequent insns which can inherit original reg value. */
7051 : rtx insns;
7052 : };
7053 :
7054 : /* Array containing all info for doing inheritance from the current
7055 : insn. */
7056 : static struct to_inherit to_inherit[LRA_MAX_INSN_RELOADS];
7057 :
7058 : /* Number elements in the previous array. */
7059 : static int to_inherit_num;
7060 :
7061 : /* Add inheritance info REGNO and INSNS. Their meaning is described in
7062 : structure to_inherit. */
7063 : static void
7064 312369 : add_to_inherit (int regno, rtx insns)
7065 : {
7066 312369 : int i;
7067 :
7068 312453 : for (i = 0; i < to_inherit_num; i++)
7069 84 : if (to_inherit[i].regno == regno)
7070 : return;
7071 312369 : lra_assert (to_inherit_num < LRA_MAX_INSN_RELOADS);
7072 312369 : to_inherit[to_inherit_num].regno = regno;
7073 312369 : to_inherit[to_inherit_num++].insns = insns;
7074 : }
7075 :
7076 : /* Return the last non-debug insn in basic block BB, or the block begin
7077 : note if none. */
7078 : static rtx_insn *
7079 29597411 : get_last_insertion_point (basic_block bb)
7080 : {
7081 29597411 : rtx_insn *insn;
7082 :
7083 31891092 : FOR_BB_INSNS_REVERSE (bb, insn)
7084 31891092 : if (NONDEBUG_INSN_P (insn) || NOTE_INSN_BASIC_BLOCK_P (insn))
7085 29597411 : return insn;
7086 0 : gcc_unreachable ();
7087 : }
7088 :
7089 : /* Set up RES by registers living on edges FROM except the edge (FROM,
7090 : TO) or by registers set up in a jump insn in BB FROM. */
7091 : static void
7092 11398593 : get_live_on_other_edges (basic_block from, basic_block to, bitmap res)
7093 : {
7094 11398593 : rtx_insn *last;
7095 11398593 : struct lra_insn_reg *reg;
7096 11398593 : edge e;
7097 11398593 : edge_iterator ei;
7098 :
7099 11398593 : lra_assert (to != NULL);
7100 11398593 : bitmap_clear (res);
7101 33925506 : FOR_EACH_EDGE (e, ei, from->succs)
7102 22526913 : if (e->dest != to)
7103 11128320 : bitmap_ior_into (res, df_get_live_in (e->dest));
7104 11398593 : last = get_last_insertion_point (from);
7105 11398593 : if (! JUMP_P (last))
7106 1856140 : return;
7107 9542453 : curr_id = lra_get_insn_recog_data (last);
7108 19084662 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7109 9542209 : if (reg->type != OP_IN)
7110 76 : bitmap_set_bit (res, reg->regno);
7111 : }
7112 :
7113 : /* Used as a temporary results of some bitmap calculations. */
7114 : static bitmap_head temp_bitmap;
7115 :
7116 : /* We split for reloads of small class of hard regs. The following
7117 : defines how many hard regs the class should have to be qualified as
7118 : small. The code is mostly oriented to x86/x86-64 architecture
7119 : where some insns need to use only specific register or pair of
7120 : registers and these register can live in RTL explicitly, e.g. for
7121 : parameter passing. */
7122 : static const int max_small_class_regs_num = 2;
7123 :
7124 : /* Do inheritance/split transformations in EBB starting with HEAD and
7125 : finishing on TAIL. We process EBB insns in the reverse order.
7126 : Return true if we did any inheritance/split transformation in the
7127 : EBB.
7128 :
7129 : We should avoid excessive splitting which results in worse code
7130 : because of inaccurate cost calculations for spilling new split
7131 : pseudos in such case. To achieve this we do splitting only if
7132 : register pressure is high in given basic block and there are reload
7133 : pseudos requiring hard registers. We could do more register
7134 : pressure calculations at any given program point to avoid necessary
7135 : splitting even more but it is to expensive and the current approach
7136 : works well enough. */
7137 : static bool
7138 12499643 : inherit_in_ebb (rtx_insn *head, rtx_insn *tail)
7139 : {
7140 12499643 : int i, src_regno, dst_regno, nregs;
7141 12499643 : bool change_p, succ_p, update_reloads_num_p;
7142 12499643 : rtx_insn *prev_insn, *last_insn;
7143 12499643 : rtx next_usage_insns, curr_set;
7144 12499643 : enum reg_class cl;
7145 12499643 : struct lra_insn_reg *reg;
7146 12499643 : basic_block last_processed_bb, curr_bb = NULL;
7147 12499643 : HARD_REG_SET potential_reload_hard_regs, live_hard_regs;
7148 12499643 : bitmap to_process;
7149 12499643 : unsigned int j;
7150 12499643 : bitmap_iterator bi;
7151 12499643 : bool head_p, after_p;
7152 :
7153 12499643 : change_p = false;
7154 12499643 : curr_usage_insns_check++;
7155 12499643 : clear_invariants ();
7156 12499643 : reloads_num = calls_num = 0;
7157 162495359 : for (unsigned int i = 0; i < NUM_ABI_IDS; ++i)
7158 149995716 : last_call_for_abi[i] = 0;
7159 12499643 : CLEAR_HARD_REG_SET (full_and_partial_call_clobbers);
7160 12499643 : bitmap_clear (&check_only_regs);
7161 12499643 : bitmap_clear (&invalid_invariant_regs);
7162 12499643 : last_processed_bb = NULL;
7163 12499643 : CLEAR_HARD_REG_SET (potential_reload_hard_regs);
7164 12499643 : live_hard_regs = eliminable_regset | lra_no_alloc_regs;
7165 : /* We don't process new insns generated in the loop. */
7166 231233684 : for (curr_insn = tail; curr_insn != PREV_INSN (head); curr_insn = prev_insn)
7167 : {
7168 218734041 : prev_insn = PREV_INSN (curr_insn);
7169 218734041 : if (BLOCK_FOR_INSN (curr_insn) != NULL)
7170 218733798 : curr_bb = BLOCK_FOR_INSN (curr_insn);
7171 218734041 : if (last_processed_bb != curr_bb)
7172 : {
7173 : /* We are at the end of BB. Add qualified living
7174 : pseudos for potential splitting. */
7175 18198818 : to_process = df_get_live_out (curr_bb);
7176 18198818 : if (last_processed_bb != NULL)
7177 : {
7178 : /* We are somewhere in the middle of EBB. */
7179 5699175 : get_live_on_other_edges (curr_bb, last_processed_bb,
7180 : &temp_bitmap);
7181 5699175 : to_process = &temp_bitmap;
7182 : }
7183 18198818 : last_processed_bb = curr_bb;
7184 18198818 : last_insn = get_last_insertion_point (curr_bb);
7185 36397636 : after_p = (! JUMP_P (last_insn)
7186 18198818 : && (! CALL_P (last_insn)
7187 2262013 : || (find_reg_note (last_insn,
7188 : REG_NORETURN, NULL_RTX) == NULL_RTX
7189 1332188 : && ! SIBLING_CALL_P (last_insn))));
7190 18198818 : CLEAR_HARD_REG_SET (potential_reload_hard_regs);
7191 197413655 : EXECUTE_IF_SET_IN_BITMAP (to_process, 0, j, bi)
7192 : {
7193 179214843 : if ((int) j >= lra_constraint_new_regno_start)
7194 : break;
7195 179214837 : if (j < FIRST_PSEUDO_REGISTER || reg_renumber[j] >= 0)
7196 : {
7197 116487361 : if (j < FIRST_PSEUDO_REGISTER)
7198 68441075 : SET_HARD_REG_BIT (live_hard_regs, j);
7199 : else
7200 48046286 : add_to_hard_reg_set (&live_hard_regs,
7201 48046286 : PSEUDO_REGNO_MODE (j),
7202 48046286 : reg_renumber[j]);
7203 116487361 : setup_next_usage_insn (j, last_insn, reloads_num, after_p);
7204 : }
7205 : }
7206 : }
7207 218734041 : src_regno = dst_regno = -1;
7208 218734041 : curr_set = single_set (curr_insn);
7209 218734041 : if (curr_set != NULL_RTX && REG_P (SET_DEST (curr_set)))
7210 82829428 : dst_regno = REGNO (SET_DEST (curr_set));
7211 112430973 : if (curr_set != NULL_RTX && REG_P (SET_SRC (curr_set)))
7212 38993993 : src_regno = REGNO (SET_SRC (curr_set));
7213 218734041 : update_reloads_num_p = true;
7214 218734041 : if (src_regno < lra_constraint_new_regno_start
7215 212415080 : && src_regno >= FIRST_PSEUDO_REGISTER
7216 27493848 : && reg_renumber[src_regno] < 0
7217 3680582 : && dst_regno >= lra_constraint_new_regno_start
7218 221350655 : && (cl = lra_get_allocno_class (dst_regno)) != NO_REGS)
7219 : {
7220 : /* 'reload_pseudo <- original_pseudo'. */
7221 2616614 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7222 22161 : reloads_num++;
7223 2616614 : update_reloads_num_p = false;
7224 2616614 : succ_p = false;
7225 2616614 : if (usage_insns[src_regno].check == curr_usage_insns_check
7226 2616614 : && (next_usage_insns = usage_insns[src_regno].insns) != NULL_RTX)
7227 470073 : succ_p = inherit_reload_reg (false, src_regno, cl,
7228 : curr_insn, next_usage_insns);
7229 470073 : if (succ_p)
7230 : change_p = true;
7231 : else
7232 2169186 : setup_next_usage_insn (src_regno, curr_insn, reloads_num, false);
7233 5233228 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7234 614740297 : potential_reload_hard_regs |= reg_class_contents[cl];
7235 : }
7236 216117427 : else if (src_regno < 0
7237 179740048 : && dst_regno >= lra_constraint_new_regno_start
7238 5495638 : && invariant_p (SET_SRC (curr_set))
7239 273945 : && (cl = lra_get_allocno_class (dst_regno)) != NO_REGS
7240 273377 : && ! bitmap_bit_p (&invalid_invariant_regs, dst_regno)
7241 216351366 : && ! bitmap_bit_p (&invalid_invariant_regs,
7242 233939 : ORIGINAL_REGNO(regno_reg_rtx[dst_regno])))
7243 : {
7244 : /* 'reload_pseudo <- invariant'. */
7245 172473 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7246 7936 : reloads_num++;
7247 172473 : update_reloads_num_p = false;
7248 172473 : if (process_invariant_for_inheritance (SET_DEST (curr_set), SET_SRC (curr_set)))
7249 22379 : change_p = true;
7250 344946 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7251 614740297 : potential_reload_hard_regs |= reg_class_contents[cl];
7252 : }
7253 215944954 : else if (src_regno >= lra_constraint_new_regno_start
7254 6318961 : && dst_regno < lra_constraint_new_regno_start
7255 5518026 : && dst_regno >= FIRST_PSEUDO_REGISTER
7256 3692846 : && reg_renumber[dst_regno] < 0
7257 1443305 : && (cl = lra_get_allocno_class (src_regno)) != NO_REGS
7258 1443305 : && usage_insns[dst_regno].check == curr_usage_insns_check
7259 215944954 : && (next_usage_insns
7260 473435 : = usage_insns[dst_regno].insns) != NULL_RTX)
7261 : {
7262 473435 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7263 8052 : reloads_num++;
7264 473435 : update_reloads_num_p = false;
7265 : /* 'original_pseudo <- reload_pseudo'. */
7266 473435 : if (! JUMP_P (curr_insn)
7267 473435 : && inherit_reload_reg (true, dst_regno, cl,
7268 : curr_insn, next_usage_insns))
7269 : change_p = true;
7270 : /* Invalidate. */
7271 473435 : usage_insns[dst_regno].check = 0;
7272 946870 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7273 614740297 : potential_reload_hard_regs |= reg_class_contents[cl];
7274 : }
7275 215471519 : else if (INSN_P (curr_insn))
7276 : {
7277 180465247 : int iter;
7278 180465247 : int max_uid = get_max_uid ();
7279 :
7280 180465247 : curr_id = lra_get_insn_recog_data (curr_insn);
7281 180465247 : curr_static_id = curr_id->insn_static_data;
7282 180465247 : to_inherit_num = 0;
7283 : /* Process insn definitions. */
7284 541395741 : for (iter = 0; iter < 2; iter++)
7285 360930494 : for (reg = iter == 0 ? curr_id->regs : curr_static_id->hard_regs;
7286 585683768 : reg != NULL;
7287 224753274 : reg = reg->next)
7288 224753274 : if (reg->type != OP_IN
7289 224753274 : && (dst_regno = reg->regno) < lra_constraint_new_regno_start)
7290 : {
7291 45345432 : if (dst_regno >= FIRST_PSEUDO_REGISTER && reg->type == OP_OUT
7292 43222407 : && reg_renumber[dst_regno] < 0 && ! reg->subreg_p
7293 1736927 : && usage_insns[dst_regno].check == curr_usage_insns_check
7294 90050774 : && (next_usage_insns
7295 128963 : = usage_insns[dst_regno].insns) != NULL_RTX)
7296 : {
7297 128963 : struct lra_insn_reg *r;
7298 :
7299 384795 : for (r = curr_id->regs; r != NULL; r = r->next)
7300 255832 : if (r->type != OP_OUT && r->regno == dst_regno)
7301 : break;
7302 : /* Don't do inheritance if the pseudo is also
7303 : used in the insn. */
7304 128963 : if (r == NULL)
7305 : /* We cannot do inheritance right now
7306 : because the current insn reg info (chain
7307 : regs) can change after that. */
7308 128963 : add_to_inherit (dst_regno, next_usage_insns);
7309 : }
7310 : /* We cannot process one reg twice here because of
7311 : usage_insns invalidation. */
7312 90050774 : if ((dst_regno < FIRST_PSEUDO_REGISTER
7313 45345432 : || reg_renumber[dst_regno] >= 0)
7314 88175673 : && ! reg->subreg_p && reg->type != OP_IN)
7315 : {
7316 87893215 : HARD_REG_SET s;
7317 :
7318 87893215 : if (split_if_necessary (dst_regno, reg->biggest_mode,
7319 : potential_reload_hard_regs,
7320 : false, curr_insn, max_uid))
7321 57927 : change_p = true;
7322 87893215 : CLEAR_HARD_REG_SET (s);
7323 87893215 : if (dst_regno < FIRST_PSEUDO_REGISTER)
7324 44705342 : add_to_hard_reg_set (&s, reg->biggest_mode, dst_regno);
7325 : else
7326 43187873 : add_to_hard_reg_set (&s, PSEUDO_REGNO_MODE (dst_regno),
7327 43187873 : reg_renumber[dst_regno]);
7328 87893215 : live_hard_regs &= ~s;
7329 175786430 : potential_reload_hard_regs &= ~s;
7330 : }
7331 : /* We should invalidate potential inheritance or
7332 : splitting for the current insn usages to the next
7333 : usage insns (see code below) as the output pseudo
7334 : prevents this. */
7335 90050774 : if ((dst_regno >= FIRST_PSEUDO_REGISTER
7336 45345432 : && reg_renumber[dst_regno] < 0)
7337 88175673 : || (reg->type == OP_OUT && ! reg->subreg_p
7338 80175363 : && (dst_regno < FIRST_PSEUDO_REGISTER
7339 41073317 : || reg_renumber[dst_regno] >= 0)))
7340 : {
7341 : /* Invalidate and mark definitions. */
7342 42948418 : if (dst_regno >= FIRST_PSEUDO_REGISTER)
7343 42948418 : usage_insns[dst_regno].check = -(int) INSN_UID (curr_insn);
7344 : else
7345 : {
7346 39102046 : nregs = hard_regno_nregs (dst_regno,
7347 39102046 : reg->biggest_mode);
7348 78461985 : for (i = 0; i < nregs; i++)
7349 78719878 : usage_insns[dst_regno + i].check
7350 39359939 : = -(int) INSN_UID (curr_insn);
7351 : }
7352 : }
7353 : }
7354 : /* Process clobbered call regs. */
7355 180465247 : if (curr_id->arg_hard_regs != NULL)
7356 19541851 : for (i = 0; (dst_regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7357 13942182 : if (dst_regno >= FIRST_PSEUDO_REGISTER)
7358 1611434 : usage_insns[dst_regno - FIRST_PSEUDO_REGISTER].check
7359 805717 : = -(int) INSN_UID (curr_insn);
7360 180465247 : if (! JUMP_P (curr_insn))
7361 169254323 : for (i = 0; i < to_inherit_num; i++)
7362 128963 : if (inherit_reload_reg (true, to_inherit[i].regno,
7363 : ALL_REGS, curr_insn,
7364 : to_inherit[i].insns))
7365 103723 : change_p = true;
7366 180465247 : if (CALL_P (curr_insn))
7367 : {
7368 7211582 : rtx cheap, pat, dest;
7369 7211582 : rtx_insn *restore;
7370 7211582 : int regno, hard_regno;
7371 :
7372 7211582 : calls_num++;
7373 7211582 : function_abi callee_abi = insn_callee_abi (curr_insn);
7374 7211582 : last_call_for_abi[callee_abi.id ()] = calls_num;
7375 7211582 : full_and_partial_call_clobbers
7376 7211582 : |= callee_abi.full_and_partial_reg_clobbers ();
7377 7211582 : first_call_insn = curr_insn;
7378 7211582 : if ((cheap = find_reg_note (curr_insn,
7379 : REG_RETURNED, NULL_RTX)) != NULL_RTX
7380 43374 : && ((cheap = XEXP (cheap, 0)), true)
7381 43374 : && (regno = REGNO (cheap)) >= FIRST_PSEUDO_REGISTER
7382 43374 : && (hard_regno = reg_renumber[regno]) >= 0
7383 32917 : && usage_insns[regno].check == curr_usage_insns_check
7384 : /* If there are pending saves/restores, the
7385 : optimization is not worth. */
7386 28111 : && usage_insns[regno].calls_num == calls_num - 1
7387 7237864 : && callee_abi.clobbers_reg_p (GET_MODE (cheap), hard_regno))
7388 : {
7389 : /* Restore the pseudo from the call result as
7390 : REG_RETURNED note says that the pseudo value is
7391 : in the call result and the pseudo is an argument
7392 : of the call. */
7393 11955 : pat = PATTERN (curr_insn);
7394 11955 : if (GET_CODE (pat) == PARALLEL)
7395 0 : pat = XVECEXP (pat, 0, 0);
7396 11955 : dest = SET_DEST (pat);
7397 : /* For multiple return values dest is PARALLEL.
7398 : Currently we handle only single return value case. */
7399 11955 : if (REG_P (dest))
7400 : {
7401 11955 : start_sequence ();
7402 11955 : emit_move_insn (cheap, copy_rtx (dest));
7403 11955 : restore = end_sequence ();
7404 11955 : lra_process_new_insns (curr_insn, NULL, restore,
7405 : "Inserting call parameter restore");
7406 : /* We don't need to save/restore of the pseudo from
7407 : this call. */
7408 11955 : usage_insns[regno].calls_num = calls_num;
7409 11955 : remove_from_hard_reg_set
7410 11955 : (&full_and_partial_call_clobbers,
7411 11955 : GET_MODE (cheap), hard_regno);
7412 11955 : bitmap_set_bit (&check_only_regs, regno);
7413 : }
7414 : }
7415 : }
7416 180465247 : to_inherit_num = 0;
7417 : /* Process insn usages. */
7418 541395741 : for (iter = 0; iter < 2; iter++)
7419 360930494 : for (reg = iter == 0 ? curr_id->regs : curr_static_id->hard_regs;
7420 585683768 : reg != NULL;
7421 224753274 : reg = reg->next)
7422 224753274 : if ((reg->type != OP_OUT
7423 88713079 : || (reg->type == OP_OUT && reg->subreg_p))
7424 225300749 : && (src_regno = reg->regno) < lra_constraint_new_regno_start)
7425 : {
7426 125245578 : if (src_regno >= FIRST_PSEUDO_REGISTER
7427 72492959 : && reg_renumber[src_regno] < 0 && reg->type == OP_IN)
7428 : {
7429 2418148 : if (usage_insns[src_regno].check == curr_usage_insns_check
7430 775054 : && (next_usage_insns
7431 775054 : = usage_insns[src_regno].insns) != NULL_RTX
7432 3193202 : && NONDEBUG_INSN_P (curr_insn))
7433 183406 : add_to_inherit (src_regno, next_usage_insns);
7434 4469484 : else if (usage_insns[src_regno].check
7435 2234742 : != -(int) INSN_UID (curr_insn))
7436 : /* Add usages but only if the reg is not set up
7437 : in the same insn. */
7438 2234742 : add_next_usage_insn (src_regno, curr_insn, reloads_num);
7439 : }
7440 70074811 : else if (src_regno < FIRST_PSEUDO_REGISTER
7441 70074811 : || reg_renumber[src_regno] >= 0)
7442 : {
7443 122689256 : bool before_p;
7444 122689256 : rtx_insn *use_insn = curr_insn;
7445 122689256 : rtx_insn *prev_insn = PREV_INSN (curr_insn);
7446 :
7447 245378512 : before_p = (JUMP_P (curr_insn)
7448 122689256 : || (CALL_P (curr_insn) && reg->type == OP_IN));
7449 122689256 : if (NONDEBUG_INSN_P (curr_insn)
7450 110072814 : && (! JUMP_P (curr_insn) || reg->type == OP_IN)
7451 232761965 : && split_if_necessary (src_regno, reg->biggest_mode,
7452 : potential_reload_hard_regs,
7453 : before_p, curr_insn, max_uid))
7454 : {
7455 215806 : if (reg->subreg_p)
7456 3150 : check_and_force_assignment_correctness_p = true;
7457 215806 : change_p = true;
7458 : /* Invalidate. */
7459 215806 : usage_insns[src_regno].check = 0;
7460 215806 : if (before_p && PREV_INSN (curr_insn) != prev_insn)
7461 : use_insn = PREV_INSN (curr_insn);
7462 : }
7463 122689256 : if (NONDEBUG_INSN_P (curr_insn))
7464 : {
7465 110072814 : if (src_regno < FIRST_PSEUDO_REGISTER)
7466 48423280 : add_to_hard_reg_set (&live_hard_regs,
7467 48423280 : reg->biggest_mode, src_regno);
7468 : else
7469 61649534 : add_to_hard_reg_set (&live_hard_regs,
7470 61649534 : PSEUDO_REGNO_MODE (src_regno),
7471 61649534 : reg_renumber[src_regno]);
7472 : }
7473 122689256 : if (src_regno >= FIRST_PSEUDO_REGISTER)
7474 69936637 : add_next_usage_insn (src_regno, use_insn, reloads_num);
7475 : else
7476 : {
7477 105603530 : for (i = 0; i < hard_regno_nregs (src_regno, reg->biggest_mode); i++)
7478 52850911 : add_next_usage_insn (src_regno + i, use_insn, reloads_num);
7479 : }
7480 : }
7481 : }
7482 : /* Process used call regs. */
7483 180465247 : if (curr_id->arg_hard_regs != NULL)
7484 19541851 : for (i = 0; (src_regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7485 13942182 : if (src_regno < FIRST_PSEUDO_REGISTER)
7486 : {
7487 13136465 : SET_HARD_REG_BIT (live_hard_regs, src_regno);
7488 13136465 : add_next_usage_insn (src_regno, curr_insn, reloads_num);
7489 : }
7490 180648653 : for (i = 0; i < to_inherit_num; i++)
7491 : {
7492 183406 : src_regno = to_inherit[i].regno;
7493 183406 : if (inherit_reload_reg (false, src_regno, ALL_REGS,
7494 : curr_insn, to_inherit[i].insns))
7495 : change_p = true;
7496 : else
7497 22656 : setup_next_usage_insn (src_regno, curr_insn, reloads_num, false);
7498 : }
7499 : }
7500 180534737 : if (update_reloads_num_p
7501 215471519 : && NONDEBUG_INSN_P (curr_insn) && curr_set != NULL_RTX)
7502 : {
7503 109168451 : int regno = -1;
7504 109168451 : if ((REG_P (SET_DEST (curr_set))
7505 79566906 : && (regno = REGNO (SET_DEST (curr_set))) >= lra_constraint_new_regno_start
7506 8102140 : && reg_renumber[regno] < 0
7507 5184242 : && (cl = lra_get_allocno_class (regno)) != NO_REGS)
7508 183815583 : || (REG_P (SET_SRC (curr_set))
7509 34637826 : && (regno = REGNO (SET_SRC (curr_set))) >= lra_constraint_new_regno_start
7510 5915965 : && reg_renumber[regno] < 0
7511 3476134 : && (cl = lra_get_allocno_class (regno)) != NO_REGS))
7512 : {
7513 8143889 : if (ira_class_hard_regs_num[cl] <= max_small_class_regs_num)
7514 214734 : reloads_num++;
7515 16287778 : if (hard_reg_set_subset_p (reg_class_contents[cl], live_hard_regs))
7516 218734041 : potential_reload_hard_regs |= reg_class_contents[cl];
7517 : }
7518 : }
7519 218734041 : if (NONDEBUG_INSN_P (curr_insn))
7520 : {
7521 118729785 : int regno;
7522 :
7523 : /* Invalidate invariants with changed regs. */
7524 118729785 : curr_id = lra_get_insn_recog_data (curr_insn);
7525 304750847 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7526 186021062 : if (reg->type != OP_IN)
7527 : {
7528 79544866 : bitmap_set_bit (&invalid_invariant_regs, reg->regno);
7529 159089732 : bitmap_set_bit (&invalid_invariant_regs,
7530 79544866 : ORIGINAL_REGNO (regno_reg_rtx[reg->regno]));
7531 : }
7532 118729785 : curr_static_id = curr_id->insn_static_data;
7533 149989258 : for (reg = curr_static_id->hard_regs; reg != NULL; reg = reg->next)
7534 31259473 : if (reg->type != OP_IN)
7535 22402398 : bitmap_set_bit (&invalid_invariant_regs, reg->regno);
7536 118729785 : if (curr_id->arg_hard_regs != NULL)
7537 19541851 : for (i = 0; (regno = curr_id->arg_hard_regs[i]) >= 0; i++)
7538 13942182 : if (regno >= FIRST_PSEUDO_REGISTER)
7539 805717 : bitmap_set_bit (&invalid_invariant_regs,
7540 : regno - FIRST_PSEUDO_REGISTER);
7541 : }
7542 : /* We reached the start of the current basic block. */
7543 218734033 : if (prev_insn == NULL_RTX || prev_insn == PREV_INSN (head)
7544 424968439 : || BLOCK_FOR_INSN (prev_insn) != curr_bb)
7545 : {
7546 : /* We reached the beginning of the current block -- do
7547 : rest of splitting in the current BB. */
7548 18199061 : to_process = df_get_live_in (curr_bb);
7549 18199061 : if (BLOCK_FOR_INSN (head) != curr_bb)
7550 : {
7551 : /* We are somewhere in the middle of EBB. */
7552 5699418 : get_live_on_other_edges (EDGE_PRED (curr_bb, 0)->src,
7553 : curr_bb, &temp_bitmap);
7554 5699418 : to_process = &temp_bitmap;
7555 : }
7556 18199061 : head_p = true;
7557 190999314 : EXECUTE_IF_SET_IN_BITMAP (to_process, 0, j, bi)
7558 : {
7559 172800260 : if ((int) j >= lra_constraint_new_regno_start)
7560 : break;
7561 109879440 : if (((int) j < FIRST_PSEUDO_REGISTER || reg_renumber[j] >= 0)
7562 108965480 : && usage_insns[j].check == curr_usage_insns_check
7563 279899995 : && (next_usage_insns = usage_insns[j].insns) != NULL_RTX)
7564 : {
7565 107099742 : if (need_for_split_p (potential_reload_hard_regs, j))
7566 : {
7567 368727 : if (lra_dump_file != NULL && head_p)
7568 : {
7569 0 : fprintf (lra_dump_file,
7570 : " ----------------------------------\n");
7571 0 : head_p = false;
7572 : }
7573 368727 : if (split_reg (false, j, bb_note (curr_bb),
7574 : next_usage_insns, NULL))
7575 368727 : change_p = true;
7576 : }
7577 107099742 : usage_insns[j].check = 0;
7578 : }
7579 : }
7580 : }
7581 : }
7582 12499643 : first_call_insn = NULL;
7583 12499643 : return change_p;
7584 : }
7585 :
7586 : /* This value affects EBB forming. If probability of edge from EBB to
7587 : a BB is not greater than the following value, we don't add the BB
7588 : to EBB. */
7589 : #define EBB_PROBABILITY_CUTOFF \
7590 : ((REG_BR_PROB_BASE * param_lra_inheritance_ebb_probability_cutoff) / 100)
7591 :
7592 : /* Current number of inheritance/split iteration. */
7593 : int lra_inheritance_iter;
7594 :
7595 : /* Entry function for inheritance/split pass. */
7596 : void
7597 1554192 : lra_inheritance (void)
7598 : {
7599 1554192 : int i;
7600 1554192 : basic_block bb, start_bb;
7601 1554192 : edge e;
7602 :
7603 1554192 : lra_inheritance_iter++;
7604 1554192 : if (lra_inheritance_iter > LRA_MAX_INHERITANCE_PASSES)
7605 : return;
7606 1551151 : timevar_push (TV_LRA_INHERITANCE);
7607 1551151 : if (lra_dump_file != NULL)
7608 97 : fprintf (lra_dump_file, "\n********** Inheritance #%d: **********\n\n",
7609 : lra_inheritance_iter);
7610 1551151 : curr_usage_insns_check = 0;
7611 1551151 : usage_insns = XNEWVEC (struct usage_insns, lra_constraint_new_regno_start);
7612 229256623 : for (i = 0; i < lra_constraint_new_regno_start; i++)
7613 227705472 : usage_insns[i].check = 0;
7614 1551151 : bitmap_initialize (&check_only_regs, ®_obstack);
7615 1551151 : bitmap_initialize (&invalid_invariant_regs, ®_obstack);
7616 1551151 : bitmap_initialize (&live_regs, ®_obstack);
7617 1551151 : bitmap_initialize (&temp_bitmap, ®_obstack);
7618 1551151 : bitmap_initialize (&ebb_global_regs, ®_obstack);
7619 14050794 : FOR_EACH_BB_FN (bb, cfun)
7620 : {
7621 12499643 : start_bb = bb;
7622 12499643 : if (lra_dump_file != NULL)
7623 347 : fprintf (lra_dump_file, "EBB");
7624 : /* Form a EBB starting with BB. */
7625 12499643 : bitmap_clear (&ebb_global_regs);
7626 12499643 : bitmap_ior_into (&ebb_global_regs, df_get_live_in (bb));
7627 18198818 : for (;;)
7628 : {
7629 18198818 : if (lra_dump_file != NULL)
7630 477 : fprintf (lra_dump_file, " %d", bb->index);
7631 18198818 : if (bb->next_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
7632 16647667 : || LABEL_P (BB_HEAD (bb->next_bb)))
7633 : break;
7634 8069257 : e = find_fallthru_edge (bb->succs);
7635 8069257 : if (! e)
7636 : break;
7637 8069257 : if (e->probability.initialized_p ()
7638 8069257 : && e->probability.to_reg_br_prob_base () < EBB_PROBABILITY_CUTOFF)
7639 : break;
7640 : bb = bb->next_bb;
7641 : }
7642 12499643 : bitmap_ior_into (&ebb_global_regs, df_get_live_out (bb));
7643 12499643 : if (lra_dump_file != NULL)
7644 347 : fprintf (lra_dump_file, "\n");
7645 12499643 : if (inherit_in_ebb (BB_HEAD (start_bb), BB_END (bb)))
7646 : /* Remember that the EBB head and tail can change in
7647 : inherit_in_ebb. */
7648 733513 : update_ebb_live_info (BB_HEAD (start_bb), BB_END (bb));
7649 : }
7650 1551151 : bitmap_release (&ebb_global_regs);
7651 1551151 : bitmap_release (&temp_bitmap);
7652 1551151 : bitmap_release (&live_regs);
7653 1551151 : bitmap_release (&invalid_invariant_regs);
7654 1551151 : bitmap_release (&check_only_regs);
7655 1551151 : free (usage_insns);
7656 1551151 : lra_dump_insns_if_possible ("func after inheritance");
7657 1551151 : timevar_pop (TV_LRA_INHERITANCE);
7658 : }
7659 :
7660 :
7661 :
7662 : /* This page contains code to undo failed inheritance/split
7663 : transformations. */
7664 :
7665 : /* Current number of iteration undoing inheritance/split. */
7666 : int lra_undo_inheritance_iter;
7667 :
7668 : /* Fix BB live info LIVE after removing pseudos created on pass doing
7669 : inheritance/split which are REMOVED_PSEUDOS. */
7670 : static void
7671 36397636 : fix_bb_live_info (bitmap live, bitmap removed_pseudos)
7672 : {
7673 36397636 : unsigned int regno;
7674 36397636 : bitmap_iterator bi;
7675 :
7676 206491382 : EXECUTE_IF_SET_IN_BITMAP (removed_pseudos, 0, regno, bi)
7677 170093746 : if (bitmap_clear_bit (live, regno)
7678 170093746 : && REG_P (lra_reg_info[regno].restore_rtx))
7679 1203028 : bitmap_set_bit (live, REGNO (lra_reg_info[regno].restore_rtx));
7680 36397636 : }
7681 :
7682 : /* Return regno of the (subreg of) REG. Otherwise, return a negative
7683 : number. */
7684 : static int
7685 67146745 : get_regno (rtx reg)
7686 : {
7687 1095049 : if (GET_CODE (reg) == SUBREG)
7688 1038752 : reg = SUBREG_REG (reg);
7689 67146745 : if (REG_P (reg))
7690 43507230 : return REGNO (reg);
7691 : return -1;
7692 : }
7693 :
7694 : /* Delete a move INSN with destination reg DREGNO and a previous
7695 : clobber insn with the same regno. The inheritance/split code can
7696 : generate moves with preceding clobber and when we delete such moves
7697 : we should delete the clobber insn too to keep the correct life
7698 : info. */
7699 : static void
7700 748925 : delete_move_and_clobber (rtx_insn *insn, int dregno)
7701 : {
7702 748925 : rtx_insn *prev_insn = PREV_INSN (insn);
7703 :
7704 748925 : lra_set_insn_deleted (insn);
7705 748925 : lra_assert (dregno >= 0);
7706 748925 : if (prev_insn != NULL && NONDEBUG_INSN_P (prev_insn)
7707 320413 : && GET_CODE (PATTERN (prev_insn)) == CLOBBER
7708 749317 : && dregno == get_regno (XEXP (PATTERN (prev_insn), 0)))
7709 0 : lra_set_insn_deleted (prev_insn);
7710 748925 : }
7711 :
7712 : /* Remove inheritance/split pseudos which are in REMOVE_PSEUDOS and
7713 : return true if we did any change. The undo transformations for
7714 : inheritance looks like
7715 : i <- i2
7716 : p <- i => p <- i2
7717 : or removing
7718 : p <- i, i <- p, and i <- i3
7719 : where p is original pseudo from which inheritance pseudo i was
7720 : created, i and i3 are removed inheritance pseudos, i2 is another
7721 : not removed inheritance pseudo. All split pseudos or other
7722 : occurrences of removed inheritance pseudos are changed on the
7723 : corresponding original pseudos.
7724 :
7725 : The function also schedules insns changed and created during
7726 : inheritance/split pass for processing by the subsequent constraint
7727 : pass. */
7728 : static bool
7729 1551151 : remove_inheritance_pseudos (bitmap remove_pseudos)
7730 : {
7731 1551151 : basic_block bb;
7732 1551151 : int regno, sregno, prev_sregno, dregno;
7733 1551151 : rtx restore_rtx;
7734 1551151 : rtx set, prev_set;
7735 1551151 : rtx_insn *prev_insn;
7736 1551151 : bool change_p, done_p;
7737 :
7738 1551151 : change_p = ! bitmap_empty_p (remove_pseudos);
7739 : /* We cannot finish the function right away if CHANGE_P is true
7740 : because we need to marks insns affected by previous
7741 : inheritance/split pass for processing by the subsequent
7742 : constraint pass. */
7743 19749969 : FOR_EACH_BB_FN (bb, cfun)
7744 : {
7745 18198818 : fix_bb_live_info (df_get_live_in (bb), remove_pseudos);
7746 18198818 : fix_bb_live_info (df_get_live_out (bb), remove_pseudos);
7747 239422713 : FOR_BB_INSNS_REVERSE (bb, curr_insn)
7748 : {
7749 221223895 : if (! INSN_P (curr_insn))
7750 35350345 : continue;
7751 185873550 : done_p = false;
7752 185873550 : sregno = dregno = -1;
7753 46279532 : if (change_p && NONDEBUG_INSN_P (curr_insn)
7754 217597070 : && (set = single_set (curr_insn)) != NULL_RTX)
7755 : {
7756 30714224 : dregno = get_regno (SET_DEST (set));
7757 61428448 : sregno = get_regno (SET_SRC (set));
7758 : }
7759 :
7760 185873550 : if (sregno >= 0 && dregno >= 0)
7761 : {
7762 10831885 : if (bitmap_bit_p (remove_pseudos, dregno)
7763 10831885 : && ! REG_P (lra_reg_info[dregno].restore_rtx))
7764 : {
7765 : /* invariant inheritance pseudo <- original pseudo */
7766 6663 : if (lra_dump_file != NULL)
7767 : {
7768 0 : fprintf (lra_dump_file, " Removing invariant inheritance:\n");
7769 0 : dump_insn_slim (lra_dump_file, curr_insn);
7770 0 : fprintf (lra_dump_file, "\n");
7771 : }
7772 6663 : delete_move_and_clobber (curr_insn, dregno);
7773 6663 : done_p = true;
7774 : }
7775 10825222 : else if (bitmap_bit_p (remove_pseudos, sregno)
7776 10825222 : && ! REG_P (lra_reg_info[sregno].restore_rtx))
7777 : {
7778 : /* reload pseudo <- invariant inheritance pseudo */
7779 6663 : start_sequence ();
7780 : /* We cannot just change the source. It might be
7781 : an insn different from the move. */
7782 6663 : emit_insn (lra_reg_info[sregno].restore_rtx);
7783 6663 : rtx_insn *new_insns = end_sequence ();
7784 6663 : lra_assert (single_set (new_insns) != NULL
7785 : && SET_DEST (set) == SET_DEST (single_set (new_insns)));
7786 6663 : lra_process_new_insns (curr_insn, NULL, new_insns,
7787 : "Changing reload<-invariant inheritance");
7788 6663 : delete_move_and_clobber (curr_insn, dregno);
7789 6663 : done_p = true;
7790 : }
7791 10818559 : else if ((bitmap_bit_p (remove_pseudos, sregno)
7792 1205244 : && (get_regno (lra_reg_info[sregno].restore_rtx) == dregno
7793 566285 : || (bitmap_bit_p (remove_pseudos, dregno)
7794 184384 : && get_regno (lra_reg_info[sregno].restore_rtx) >= 0
7795 184384 : && (get_regno (lra_reg_info[sregno].restore_rtx)
7796 184384 : == get_regno (lra_reg_info[dregno].restore_rtx)))))
7797 11292652 : || (bitmap_bit_p (remove_pseudos, dregno)
7798 648710 : && get_regno (lra_reg_info[dregno].restore_rtx) == sregno))
7799 : /* One of the following cases:
7800 : original <- removed inheritance pseudo
7801 : removed inherit pseudo <- another removed inherit pseudo
7802 : removed inherit pseudo <- original pseudo
7803 : Or
7804 : removed_split_pseudo <- original_reg
7805 : original_reg <- removed_split_pseudo */
7806 : {
7807 176125 : if (lra_dump_file != NULL)
7808 : {
7809 0 : fprintf (lra_dump_file, " Removing %s:\n",
7810 0 : bitmap_bit_p (&lra_split_regs, sregno)
7811 0 : || bitmap_bit_p (&lra_split_regs, dregno)
7812 : ? "split" : "inheritance");
7813 0 : dump_insn_slim (lra_dump_file, curr_insn);
7814 : }
7815 176125 : delete_move_and_clobber (curr_insn, dregno);
7816 176125 : done_p = true;
7817 : }
7818 10642434 : else if (bitmap_bit_p (remove_pseudos, sregno)
7819 10642434 : && bitmap_bit_p (&lra_inheritance_pseudos, sregno))
7820 : {
7821 : /* Search the following pattern:
7822 : inherit_or_split_pseudo1 <- inherit_or_split_pseudo2
7823 : original_pseudo <- inherit_or_split_pseudo1
7824 : where the 2nd insn is the current insn and
7825 : inherit_or_split_pseudo2 is not removed. If it is found,
7826 : change the current insn onto:
7827 : original_pseudo <- inherit_or_split_pseudo2. */
7828 726224 : for (prev_insn = PREV_INSN (curr_insn);
7829 726224 : prev_insn != NULL_RTX && ! NONDEBUG_INSN_P (prev_insn);
7830 252131 : prev_insn = PREV_INSN (prev_insn))
7831 : ;
7832 474093 : if (prev_insn != NULL_RTX && BLOCK_FOR_INSN (prev_insn) == bb
7833 461155 : && (prev_set = single_set (prev_insn)) != NULL_RTX
7834 : /* There should be no subregs in insn we are
7835 : searching because only the original reg might
7836 : be in subreg when we changed the mode of
7837 : load/store for splitting. */
7838 455484 : && REG_P (SET_DEST (prev_set))
7839 350110 : && REG_P (SET_SRC (prev_set))
7840 268325 : && (int) REGNO (SET_DEST (prev_set)) == sregno
7841 182395 : && ((prev_sregno = REGNO (SET_SRC (prev_set)))
7842 : >= FIRST_PSEUDO_REGISTER)
7843 182395 : && (lra_reg_info[prev_sregno].restore_rtx == NULL_RTX
7844 141889 : ||
7845 : /* As we consider chain of inheritance or
7846 : splitting described in above comment we should
7847 : check that sregno and prev_sregno were
7848 : inheritance/split pseudos created from the
7849 : same original regno. */
7850 283778 : (get_regno (lra_reg_info[sregno].restore_rtx) >= 0
7851 283778 : && (get_regno (lra_reg_info[sregno].restore_rtx)
7852 283778 : == get_regno (lra_reg_info[prev_sregno].restore_rtx))))
7853 656488 : && ! bitmap_bit_p (remove_pseudos, prev_sregno))
7854 : {
7855 101718 : int restore_regno = get_regno (lra_reg_info[sregno].restore_rtx);
7856 101718 : if (restore_regno < 0)
7857 0 : restore_regno = prev_sregno;
7858 101718 : lra_assert (GET_MODE (SET_SRC (prev_set))
7859 : == GET_MODE (regno_reg_rtx[restore_regno]));
7860 : /* Although we have a single set, the insn can
7861 : contain more one sregno register occurrence
7862 : as a source. Change all occurrences. */
7863 101718 : lra_substitute_pseudo_within_insn (curr_insn, sregno,
7864 : regno_reg_rtx[restore_regno],
7865 : false);
7866 : /* As we are finishing with processing the insn
7867 : here, check the destination too as it might
7868 : inheritance pseudo for another pseudo. */
7869 101718 : if (bitmap_bit_p (remove_pseudos, dregno)
7870 0 : && bitmap_bit_p (&lra_inheritance_pseudos, dregno)
7871 101718 : && (restore_rtx
7872 0 : = lra_reg_info[dregno].restore_rtx) != NULL_RTX)
7873 : {
7874 0 : if (GET_CODE (SET_DEST (set)) == SUBREG)
7875 0 : SUBREG_REG (SET_DEST (set)) = restore_rtx;
7876 : else
7877 0 : SET_DEST (set) = restore_rtx;
7878 : }
7879 101718 : lra_push_insn_and_update_insn_regno_info (curr_insn);
7880 101718 : lra_set_used_insn_alternative_by_uid
7881 101718 : (INSN_UID (curr_insn), LRA_UNKNOWN_ALT);
7882 101718 : done_p = true;
7883 101718 : if (lra_dump_file != NULL)
7884 : {
7885 0 : fprintf (lra_dump_file, " Change reload insn:\n");
7886 0 : dump_insn_slim (lra_dump_file, curr_insn);
7887 : }
7888 : }
7889 : }
7890 : }
7891 189451 : if (! done_p)
7892 : {
7893 185582381 : struct lra_insn_reg *reg;
7894 185582381 : bool restored_regs_p = false;
7895 185582381 : bool kept_regs_p = false;
7896 :
7897 185582381 : curr_id = lra_get_insn_recog_data (curr_insn);
7898 389178981 : for (reg = curr_id->regs; reg != NULL; reg = reg->next)
7899 : {
7900 203596600 : regno = reg->regno;
7901 203596600 : restore_rtx = lra_reg_info[regno].restore_rtx;
7902 203596600 : if (restore_rtx != NULL_RTX)
7903 : {
7904 5850879 : if (change_p && bitmap_bit_p (remove_pseudos, regno))
7905 : {
7906 824370 : lra_substitute_pseudo_within_insn
7907 824370 : (curr_insn, regno, restore_rtx, false);
7908 824370 : restored_regs_p = true;
7909 : }
7910 : else
7911 : kept_regs_p = true;
7912 : }
7913 : }
7914 185582381 : if (NONDEBUG_INSN_P (curr_insn) && kept_regs_p)
7915 : {
7916 : /* The instruction has changed since the previous
7917 : constraints pass. */
7918 4414761 : lra_push_insn_and_update_insn_regno_info (curr_insn);
7919 4414761 : lra_set_used_insn_alternative_by_uid
7920 4414761 : (INSN_UID (curr_insn), LRA_UNKNOWN_ALT);
7921 : }
7922 181167620 : else if (restored_regs_p)
7923 : /* The instruction has been restored to the form that
7924 : it had during the previous constraints pass. */
7925 662837 : lra_update_insn_regno_info (curr_insn);
7926 5077598 : if (restored_regs_p && lra_dump_file != NULL)
7927 : {
7928 0 : fprintf (lra_dump_file, " Insn after restoring regs:\n");
7929 0 : dump_insn_slim (lra_dump_file, curr_insn);
7930 : }
7931 : }
7932 : }
7933 : }
7934 1551151 : return change_p;
7935 : }
7936 :
7937 : /* If optional reload pseudos failed to get a hard register or was not
7938 : inherited, it is better to remove optional reloads. We do this
7939 : transformation after undoing inheritance to figure out necessity to
7940 : remove optional reloads easier. Return true if we do any
7941 : change. */
7942 : static bool
7943 1551151 : undo_optional_reloads (void)
7944 : {
7945 1551151 : bool change_p, keep_p;
7946 1551151 : unsigned int regno, uid;
7947 1551151 : bitmap_iterator bi, bi2;
7948 1551151 : rtx_insn *insn;
7949 1551151 : rtx set, src, dest;
7950 1551151 : auto_bitmap removed_optional_reload_pseudos (®_obstack);
7951 :
7952 1551151 : bitmap_copy (removed_optional_reload_pseudos, &lra_optional_reload_pseudos);
7953 2552962 : EXECUTE_IF_SET_IN_BITMAP (&lra_optional_reload_pseudos, 0, regno, bi)
7954 : {
7955 1001811 : keep_p = false;
7956 : /* Keep optional reloads from previous subpasses. */
7957 1001811 : if (lra_reg_info[regno].restore_rtx == NULL_RTX
7958 : /* If the original pseudo changed its allocation, just
7959 : removing the optional pseudo is dangerous as the original
7960 : pseudo will have longer live range. */
7961 1001811 : || reg_renumber[REGNO (lra_reg_info[regno].restore_rtx)] >= 0)
7962 : keep_p = true;
7963 621933 : else if (reg_renumber[regno] >= 0)
7964 1789322 : EXECUTE_IF_SET_IN_BITMAP (&lra_reg_info[regno].insn_bitmap, 0, uid, bi2)
7965 : {
7966 1303030 : insn = lra_insn_recog_data[uid]->insn;
7967 1303030 : if ((set = single_set (insn)) == NULL_RTX)
7968 6632 : continue;
7969 1296398 : src = SET_SRC (set);
7970 1296398 : dest = SET_DEST (set);
7971 1296398 : if ((! REG_P (src) && ! SUBREG_P (src))
7972 695075 : || (! REG_P (dest) && ! SUBREG_P (dest)))
7973 601351 : continue;
7974 695047 : if (get_regno (dest) == (int) regno
7975 : /* Ignore insn for optional reloads itself. */
7976 1168598 : && (get_regno (lra_reg_info[regno].restore_rtx)
7977 584299 : != get_regno (src))
7978 : /* Check only inheritance on last inheritance pass. */
7979 121691 : && get_regno (src) >= new_regno_start
7980 : /* Check that the optional reload was inherited. */
7981 816738 : && bitmap_bit_p (&lra_inheritance_pseudos, get_regno (src)))
7982 : {
7983 : keep_p = true;
7984 : break;
7985 : }
7986 : }
7987 987861 : if (keep_p)
7988 : {
7989 501569 : bitmap_clear_bit (removed_optional_reload_pseudos, regno);
7990 501569 : if (lra_dump_file != NULL)
7991 3 : fprintf (lra_dump_file, "Keep optional reload reg %d\n", regno);
7992 : }
7993 : }
7994 1551151 : change_p = ! bitmap_empty_p (removed_optional_reload_pseudos);
7995 1551151 : auto_bitmap insn_bitmap (®_obstack);
7996 2051393 : EXECUTE_IF_SET_IN_BITMAP (removed_optional_reload_pseudos, 0, regno, bi)
7997 : {
7998 500242 : if (lra_dump_file != NULL)
7999 2 : fprintf (lra_dump_file, "Remove optional reload reg %d\n", regno);
8000 500242 : bitmap_copy (insn_bitmap, &lra_reg_info[regno].insn_bitmap);
8001 1589898 : EXECUTE_IF_SET_IN_BITMAP (insn_bitmap, 0, uid, bi2)
8002 : {
8003 : /* We may have already removed a clobber. */
8004 1089656 : if (!lra_insn_recog_data[uid])
8005 0 : continue;
8006 1089656 : insn = lra_insn_recog_data[uid]->insn;
8007 1089656 : if ((set = single_set (insn)) != NULL_RTX)
8008 : {
8009 1083760 : src = SET_SRC (set);
8010 1083760 : dest = SET_DEST (set);
8011 494329 : if ((REG_P (src) || SUBREG_P (src))
8012 589443 : && (REG_P (dest) || SUBREG_P (dest))
8013 1673174 : && ((get_regno (src) == (int) regno
8014 227324 : && (get_regno (lra_reg_info[regno].restore_rtx)
8015 113662 : == get_regno (dest)))
8016 505635 : || (get_regno (dest) == (int) regno
8017 475752 : && (get_regno (lra_reg_info[regno].restore_rtx)
8018 475752 : == get_regno (src)))))
8019 : {
8020 559474 : if (lra_dump_file != NULL)
8021 : {
8022 0 : fprintf (lra_dump_file, " Deleting move %u\n",
8023 0 : INSN_UID (insn));
8024 0 : dump_insn_slim (lra_dump_file, insn);
8025 : }
8026 1118948 : delete_move_and_clobber (insn, get_regno (dest));
8027 559474 : continue;
8028 : }
8029 : /* We should not worry about generation memory-memory
8030 : moves here as if the corresponding inheritance did
8031 : not work (inheritance pseudo did not get a hard reg),
8032 : we remove the inheritance pseudo and the optional
8033 : reload. */
8034 : }
8035 530182 : rtx pat = PATTERN (insn);
8036 0 : if (GET_CODE (pat) == CLOBBER && REG_P (SET_DEST (pat))
8037 530182 : && get_regno (SET_DEST (pat)) == (int) regno)
8038 : /* Refuse to remap clobbers to preexisting pseudos. */
8039 0 : gcc_unreachable ();
8040 530182 : lra_substitute_pseudo_within_insn
8041 530182 : (insn, regno, lra_reg_info[regno].restore_rtx, false);
8042 530182 : lra_update_insn_regno_info (insn);
8043 530182 : if (lra_dump_file != NULL)
8044 : {
8045 4 : fprintf (lra_dump_file,
8046 : " Restoring original insn:\n");
8047 4 : dump_insn_slim (lra_dump_file, insn);
8048 : }
8049 : }
8050 : }
8051 : /* Clear restore_regnos. */
8052 2552962 : EXECUTE_IF_SET_IN_BITMAP (&lra_optional_reload_pseudos, 0, regno, bi)
8053 1001811 : lra_reg_info[regno].restore_rtx = NULL_RTX;
8054 1551151 : return change_p;
8055 1551151 : }
8056 :
8057 : /* Entry function for undoing inheritance/split transformation. Return true
8058 : if we did any RTL change in this pass. */
8059 : bool
8060 1554192 : lra_undo_inheritance (void)
8061 : {
8062 1554192 : unsigned int regno;
8063 1554192 : int hard_regno;
8064 1554192 : int n_all_inherit, n_inherit, n_all_split, n_split;
8065 1554192 : rtx restore_rtx;
8066 1554192 : bitmap_iterator bi;
8067 1554192 : bool change_p;
8068 :
8069 1554192 : lra_undo_inheritance_iter++;
8070 1554192 : if (lra_undo_inheritance_iter > LRA_MAX_INHERITANCE_PASSES)
8071 : return false;
8072 1551151 : if (lra_dump_file != NULL)
8073 97 : fprintf (lra_dump_file,
8074 : "\n********** Undoing inheritance #%d: **********\n\n",
8075 : lra_undo_inheritance_iter);
8076 1551151 : auto_bitmap remove_pseudos (®_obstack);
8077 1551151 : n_inherit = n_all_inherit = 0;
8078 3400098 : EXECUTE_IF_SET_IN_BITMAP (&lra_inheritance_pseudos, 0, regno, bi)
8079 1848947 : if (lra_reg_info[regno].restore_rtx != NULL_RTX)
8080 : {
8081 1170843 : n_all_inherit++;
8082 1170843 : if (reg_renumber[regno] < 0
8083 : /* If the original pseudo changed its allocation, just
8084 : removing inheritance is dangerous as for changing
8085 : allocation we used shorter live-ranges. */
8086 1170843 : && (! REG_P (lra_reg_info[regno].restore_rtx)
8087 417366 : || reg_renumber[REGNO (lra_reg_info[regno].restore_rtx)] < 0))
8088 424029 : bitmap_set_bit (remove_pseudos, regno);
8089 : else
8090 746814 : n_inherit++;
8091 : }
8092 1551151 : if (lra_dump_file != NULL && n_all_inherit != 0)
8093 2 : fprintf (lra_dump_file, "Inherit %d out of %d (%.2f%%)\n",
8094 : n_inherit, n_all_inherit,
8095 2 : (double) n_inherit / n_all_inherit * 100);
8096 1551151 : n_split = n_all_split = 0;
8097 2477174 : EXECUTE_IF_SET_IN_BITMAP (&lra_split_regs, 0, regno, bi)
8098 926023 : if ((restore_rtx = lra_reg_info[regno].restore_rtx) != NULL_RTX)
8099 : {
8100 642495 : int restore_regno = REGNO (restore_rtx);
8101 :
8102 642495 : n_all_split++;
8103 1284908 : hard_regno = (restore_regno >= FIRST_PSEUDO_REGISTER
8104 642495 : ? reg_renumber[restore_regno] : restore_regno);
8105 642495 : if (hard_regno < 0 || reg_renumber[regno] == hard_regno)
8106 2205 : bitmap_set_bit (remove_pseudos, regno);
8107 : else
8108 : {
8109 640290 : n_split++;
8110 640290 : if (lra_dump_file != NULL)
8111 0 : fprintf (lra_dump_file, " Keep split r%d (orig=r%d)\n",
8112 : regno, restore_regno);
8113 : }
8114 : }
8115 1551151 : if (lra_dump_file != NULL && n_all_split != 0)
8116 0 : fprintf (lra_dump_file, "Split %d out of %d (%.2f%%)\n",
8117 : n_split, n_all_split,
8118 0 : (double) n_split / n_all_split * 100);
8119 1551151 : change_p = remove_inheritance_pseudos (remove_pseudos);
8120 : /* Clear restore_regnos. */
8121 3400098 : EXECUTE_IF_SET_IN_BITMAP (&lra_inheritance_pseudos, 0, regno, bi)
8122 1848947 : lra_reg_info[regno].restore_rtx = NULL_RTX;
8123 2477174 : EXECUTE_IF_SET_IN_BITMAP (&lra_split_regs, 0, regno, bi)
8124 926023 : lra_reg_info[regno].restore_rtx = NULL_RTX;
8125 1551151 : change_p = undo_optional_reloads () || change_p;
8126 : if (change_p)
8127 110071 : lra_dump_insns_if_possible ("changed func after undoing inheritance");
8128 1551151 : return change_p;
8129 1551151 : }
|
