Line data Source code
1 : /* Copy propagation on hard registers for the GNU compiler.
2 : Copyright (C) 2000-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it
7 : under the terms of the GNU General Public License as published by
8 : the Free Software Foundation; either version 3, or (at your option)
9 : any later version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT
12 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 : or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
14 : License for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "target.h"
25 : #include "rtl.h"
26 : #include "df.h"
27 : #include "memmodel.h"
28 : #include "tm_p.h"
29 : #include "insn-config.h"
30 : #include "regs.h"
31 : #include "emit-rtl.h"
32 : #include "recog.h"
33 : #include "diagnostic-core.h"
34 : #include "addresses.h"
35 : #include "tree-pass.h"
36 : #include "rtl-iter.h"
37 : #include "cfgrtl.h"
38 : #include "function-abi.h"
39 : #include "cfgcleanup.h"
40 :
41 : /* The following code does forward propagation of hard register copies.
42 : The object is to eliminate as many dependencies as possible, so that
43 : we have the most scheduling freedom. As a side effect, we also clean
44 : up some silly register allocation decisions made by reload. This
45 : code may be obsoleted by a new register allocator. */
46 :
47 : /* DEBUG_INSNs aren't changed right away, as doing so might extend the
48 : lifetime of a register and get the DEBUG_INSN subsequently reset.
49 : So they are queued instead, and updated only when the register is
50 : used in some subsequent real insn before it is set. */
51 : struct queued_debug_insn_change
52 : {
53 : struct queued_debug_insn_change *next;
54 : rtx_insn *insn;
55 : rtx *loc;
56 : rtx new_rtx;
57 : };
58 :
59 : /* For each register, we have a list of registers that contain the same
60 : value. The OLDEST_REGNO field points to the head of the list, and
61 : the NEXT_REGNO field runs through the list. The MODE field indicates
62 : what mode the data is known to be in; this field is VOIDmode when the
63 : register is not known to contain valid data. The FRAME_RELATED field
64 : indicates if the instruction that updated this register is frame
65 : related or not. */
66 :
67 : struct value_data_entry
68 : {
69 : machine_mode mode;
70 : unsigned int oldest_regno;
71 : unsigned int next_regno;
72 : bool frame_related;
73 : struct queued_debug_insn_change *debug_insn_changes;
74 : };
75 :
76 : struct value_data
77 : {
78 : struct value_data_entry e[FIRST_PSEUDO_REGISTER];
79 : unsigned int max_value_regs;
80 : unsigned int n_debug_insn_changes;
81 : };
82 :
83 : static object_allocator<queued_debug_insn_change> queued_debug_insn_change_pool
84 : ("debug insn changes pool");
85 :
86 : static bool skip_debug_insn_p;
87 :
88 : static void kill_value_one_regno (unsigned, struct value_data *);
89 : static void kill_value_regno (unsigned, unsigned, struct value_data *);
90 : static void kill_value (const_rtx, struct value_data *);
91 : static void set_value_regno (unsigned, machine_mode, struct value_data *);
92 : static void init_value_data (struct value_data *);
93 : static void kill_clobbered_value (rtx, const_rtx, void *);
94 : static void kill_set_value (rtx, const_rtx, void *);
95 : static void copy_value (rtx, rtx, struct value_data *, bool frame_related);
96 : static bool mode_change_ok (machine_mode, machine_mode,
97 : unsigned int);
98 : static rtx maybe_mode_change (machine_mode, machine_mode,
99 : machine_mode, unsigned int, unsigned int);
100 : static bool incompatible_frame_status (bool, bool);
101 : static rtx find_oldest_value_reg (enum reg_class, rtx, struct value_data *, bool frame_related);
102 : static bool replace_oldest_value_reg (rtx *, enum reg_class, rtx_insn *,
103 : struct value_data *);
104 : static bool replace_oldest_value_addr (rtx *, enum reg_class,
105 : machine_mode, addr_space_t,
106 : rtx_insn *, struct value_data *);
107 : static bool replace_oldest_value_mem (rtx, rtx_insn *, struct value_data *);
108 : static bool copyprop_hardreg_forward_1 (basic_block, struct value_data *);
109 : extern void debug_value_data (struct value_data *);
110 : static void validate_value_data (struct value_data *);
111 :
112 : /* Free all queued updates for DEBUG_INSNs that change some reg to
113 : register REGNO. */
114 :
115 : static void
116 266930 : free_debug_insn_changes (struct value_data *vd, unsigned int regno)
117 : {
118 266930 : struct queued_debug_insn_change *cur, *next;
119 711579 : for (cur = vd->e[regno].debug_insn_changes; cur; cur = next)
120 : {
121 444649 : next = cur->next;
122 444649 : --vd->n_debug_insn_changes;
123 444649 : queued_debug_insn_change_pool.remove (cur);
124 : }
125 266930 : vd->e[regno].debug_insn_changes = NULL;
126 266930 : }
127 :
128 : /* Kill register REGNO. This involves removing it from any value
129 : lists, and resetting the value mode to VOIDmode. This is only a
130 : helper function; it does not handle any hard registers overlapping
131 : with REGNO. */
132 :
133 : static void
134 86412862 : kill_value_one_regno (unsigned int regno, struct value_data *vd)
135 : {
136 86412862 : unsigned int i, next;
137 :
138 86412862 : if (vd->e[regno].oldest_regno != regno)
139 : {
140 123763 : for (i = vd->e[regno].oldest_regno;
141 3238152 : vd->e[i].next_regno != regno;
142 123763 : i = vd->e[i].next_regno)
143 123763 : continue;
144 3114389 : vd->e[i].next_regno = vd->e[regno].next_regno;
145 : }
146 83298473 : else if ((next = vd->e[regno].next_regno) != INVALID_REGNUM)
147 : {
148 4788723 : for (i = next; i != INVALID_REGNUM; i = vd->e[i].next_regno)
149 2461218 : vd->e[i].oldest_regno = next;
150 : }
151 :
152 86412862 : vd->e[regno].mode = VOIDmode;
153 86412862 : vd->e[regno].oldest_regno = regno;
154 86412862 : vd->e[regno].next_regno = INVALID_REGNUM;
155 86412862 : vd->e[regno].frame_related = false;
156 86412862 : if (vd->e[regno].debug_insn_changes)
157 109112 : free_debug_insn_changes (vd, regno);
158 :
159 86412862 : if (flag_checking)
160 86412602 : validate_value_data (vd);
161 86412862 : }
162 :
163 : /* Kill the value in register REGNO for NREGS, and any other registers
164 : whose values overlap. */
165 :
166 : static void
167 86113201 : kill_value_regno (unsigned int regno, unsigned int nregs,
168 : struct value_data *vd)
169 : {
170 86113201 : unsigned int j;
171 :
172 : /* Kill the value we're told to kill. */
173 172442447 : for (j = 0; j < nregs; ++j)
174 86329246 : kill_value_one_regno (regno + j, vd);
175 :
176 : /* Kill everything that overlapped what we're told to kill. */
177 86113201 : if (regno < vd->max_value_regs)
178 86113201 : j = 0;
179 : else
180 73848268 : j = regno - vd->max_value_regs;
181 156088532 : for (; j < regno; ++j)
182 : {
183 69975331 : unsigned int i, n;
184 69975331 : if (vd->e[j].mode == VOIDmode)
185 54367279 : continue;
186 15608052 : n = hard_regno_nregs (j, vd->e[j].mode);
187 15608052 : if (j + n > regno)
188 125424 : for (i = 0; i < n; ++i)
189 83616 : kill_value_one_regno (j + i, vd);
190 : }
191 86113201 : }
192 :
193 : /* Kill X. This is a convenience function wrapping kill_value_regno
194 : so that we mind the mode the register is in. */
195 :
196 : static void
197 94677179 : kill_value (const_rtx x, struct value_data *vd)
198 : {
199 94677179 : if (GET_CODE (x) == SUBREG)
200 : {
201 6 : rtx tmp = simplify_subreg (GET_MODE (x), SUBREG_REG (x),
202 3 : GET_MODE (SUBREG_REG (x)), SUBREG_BYTE (x));
203 3 : x = tmp ? tmp : SUBREG_REG (x);
204 : }
205 94677179 : if (REG_P (x))
206 70715863 : kill_value_regno (REGNO (x), REG_NREGS (x), vd);
207 94677179 : }
208 :
209 : /* Remember that REGNO is valid in MODE. */
210 :
211 : static void
212 54917450 : set_value_regno (unsigned int regno, machine_mode mode,
213 : struct value_data *vd)
214 : {
215 54917450 : unsigned int nregs;
216 :
217 54917450 : vd->e[regno].mode = mode;
218 :
219 52319482 : nregs = hard_regno_nregs (regno, mode);
220 54917450 : if (nregs > vd->max_value_regs)
221 4967707 : vd->max_value_regs = nregs;
222 0 : }
223 :
224 : /* Initialize VD such that there are no known relationships between regs. */
225 :
226 : static void
227 5167859 : init_value_data (struct value_data *vd)
228 : {
229 5167859 : int i;
230 490946605 : for (i = 0; i < FIRST_PSEUDO_REGISTER; ++i)
231 : {
232 485778746 : vd->e[i].mode = VOIDmode;
233 485778746 : vd->e[i].oldest_regno = i;
234 485778746 : vd->e[i].next_regno = INVALID_REGNUM;
235 485778746 : vd->e[i].debug_insn_changes = NULL;
236 485778746 : vd->e[i].frame_related = false;
237 : }
238 5167859 : vd->max_value_regs = 0;
239 5167859 : vd->n_debug_insn_changes = 0;
240 5167859 : }
241 :
242 : /* Called through note_stores. If X is clobbered, kill its value. */
243 :
244 : static void
245 80836712 : kill_clobbered_value (rtx x, const_rtx set, void *data)
246 : {
247 80836712 : struct value_data *const vd = (struct value_data *) data;
248 :
249 80836712 : if (GET_CODE (set) == CLOBBER)
250 10148740 : kill_value (x, vd);
251 80836712 : }
252 :
253 : /* A structure passed as data to kill_set_value through note_stores. */
254 : struct kill_set_value_data
255 : {
256 : struct value_data *vd;
257 : rtx ignore_set_reg;
258 : bool insn_is_frame_related;
259 : };
260 :
261 : /* Called through note_stores. If X is set, not clobbered, kill its
262 : current value and install it as the root of its own value list. */
263 :
264 : static void
265 80704122 : kill_set_value (rtx x, const_rtx set, void *data)
266 : {
267 80704122 : struct kill_set_value_data *ksvd = (struct kill_set_value_data *) data;
268 80704122 : if (rtx_equal_p (x, ksvd->ignore_set_reg))
269 : return;
270 :
271 80606962 : if (GET_CODE (set) != CLOBBER)
272 : {
273 70458222 : kill_value (x, ksvd->vd);
274 70458222 : if (REG_P (x))
275 : {
276 47778293 : set_value_regno (REGNO (x), GET_MODE (x), ksvd->vd);
277 47778293 : ksvd->vd->e[REGNO(x)].frame_related = ksvd->insn_is_frame_related;
278 : }
279 : }
280 : }
281 :
282 : /* Kill any register used in X as the base of an auto-increment expression,
283 : and install that register as the root of its own value list. */
284 :
285 : static void
286 75243797 : kill_autoinc_value (rtx_insn *insn, struct value_data *vd)
287 : {
288 75243797 : subrtx_iterator::array_type array;
289 487013601 : FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
290 : {
291 411769804 : const_rtx x = *iter;
292 411769804 : if (GET_RTX_CLASS (GET_CODE (x)) == RTX_AUTOINC)
293 : {
294 4444029 : x = XEXP (x, 0);
295 4444029 : kill_value (x, vd);
296 4444029 : set_value_regno (REGNO (x), GET_MODE (x), vd);
297 4444029 : vd->e[REGNO(x)].frame_related = RTX_FRAME_RELATED_P (insn);
298 4444029 : iter.skip_subrtxes ();
299 : }
300 : }
301 75243797 : }
302 :
303 : /* Assert that SRC has been copied to DEST. Adjust the data structures
304 : to reflect that SRC contains an older copy of the shared value. */
305 :
306 : static void
307 6478944 : copy_value (rtx dest, rtx src, struct value_data *vd, bool frame_related)
308 : {
309 6478944 : unsigned int dr = REGNO (dest);
310 6478944 : unsigned int sr = REGNO (src);
311 6478944 : unsigned int dn, sn;
312 6478944 : unsigned int i;
313 :
314 : /* ??? At present, it's possible to see noop sets. It'd be nice if
315 : this were cleaned up beforehand... */
316 6478944 : if (sr == dr)
317 : return;
318 :
319 : /* Do not propagate copies to the stack pointer, as that can leave
320 : memory accesses with no scheduling dependency on the stack update. */
321 6478944 : if (dr == STACK_POINTER_REGNUM)
322 : return;
323 :
324 : /* Likewise with the frame pointer, if we're using one. */
325 6475445 : if (frame_pointer_needed && dr == HARD_FRAME_POINTER_REGNUM)
326 : return;
327 :
328 : /* Do not propagate copies to fixed or global registers, patterns
329 : can be relying to see particular fixed register or users can
330 : expect the chosen global register in asm. */
331 6426689 : if (fixed_regs[dr] || global_regs[dr])
332 : return;
333 :
334 : /* If SRC and DEST overlap, don't record anything. */
335 6426672 : dn = REG_NREGS (dest);
336 6426672 : sn = REG_NREGS (src);
337 6426672 : if ((dr > sr && dr < sr + sn)
338 6426672 : || (sr > dr && sr < dr + dn))
339 : return;
340 :
341 : /* If SRC had no assigned mode (i.e. we didn't know it was live)
342 : assign it now and assume the value came from an input argument
343 : or somesuch. */
344 6426672 : if (vd->e[sr].mode == VOIDmode)
345 2581816 : set_value_regno (sr, vd->e[dr].mode, vd);
346 :
347 3844856 : else if (!ordered_p (GET_MODE_PRECISION (vd->e[sr].mode),
348 3844856 : GET_MODE_PRECISION (GET_MODE (src))))
349 : return;
350 :
351 : /* If we are narrowing the input to a smaller number of hard regs,
352 : and it is in big endian, we are really extracting a high part.
353 : Since we generally associate a low part of a value with the value itself,
354 : we must not do the same for the high part.
355 : Note we can still get low parts for the same mode combination through
356 : a two-step copy involving differently sized hard regs.
357 : Assume hard regs fr* are 32 bits each, while r* are 64 bits each:
358 : (set (reg:DI r0) (reg:DI fr0))
359 : (set (reg:SI fr2) (reg:SI r0))
360 : loads the low part of (reg:DI fr0) - i.e. fr1 - into fr2, while:
361 : (set (reg:SI fr2) (reg:SI fr0))
362 : loads the high part of (reg:DI fr0) into fr2.
363 :
364 : We can't properly represent the latter case in our tables, so don't
365 : record anything then. */
366 3844856 : else if (sn < hard_regno_nregs (sr, vd->e[sr].mode)
367 3844856 : && maybe_ne (subreg_lowpart_offset (GET_MODE (dest),
368 : vd->e[sr].mode), 0U))
369 0 : return;
370 :
371 : /* If SRC had been assigned a mode narrower than the copy, we can't
372 : link DEST into the chain, because not all of the pieces of the
373 : copy came from oldest_regno. */
374 3844856 : else if (sn > hard_regno_nregs (sr, vd->e[sr].mode))
375 : return;
376 :
377 : /* If a narrower value is copied using wider mode, the upper bits
378 : are undefined (could be e.g. a former paradoxical subreg). Signal
379 : in that case we've only copied value using the narrower mode.
380 : Consider:
381 : (set (reg:DI r14) (mem:DI ...))
382 : (set (reg:QI si) (reg:QI r14))
383 : (set (reg:DI bp) (reg:DI r14))
384 : (set (reg:DI r14) (const_int ...))
385 : (set (reg:DI dx) (reg:DI si))
386 : (set (reg:DI si) (const_int ...))
387 : (set (reg:DI dx) (reg:DI bp))
388 : The last set is not redundant, while the low 8 bits of dx are already
389 : equal to low 8 bits of bp, the other bits are undefined. */
390 3844856 : else if (partial_subreg_p (vd->e[sr].mode, GET_MODE (src)))
391 : {
392 16195 : if (!REG_CAN_CHANGE_MODE_P (sr, GET_MODE (src), vd->e[sr].mode)
393 16195 : || !REG_CAN_CHANGE_MODE_P (dr, vd->e[sr].mode, GET_MODE (dest)))
394 : return;
395 16152 : set_value_regno (dr, vd->e[sr].mode, vd);
396 : }
397 :
398 : /* Link DR at the end of the value chain used by SR. */
399 :
400 6426629 : vd->e[dr].oldest_regno = vd->e[sr].oldest_regno;
401 :
402 6780000 : for (i = sr; vd->e[i].next_regno != INVALID_REGNUM; i = vd->e[i].next_regno)
403 353371 : continue;
404 6426629 : vd->e[i].next_regno = dr;
405 :
406 6426629 : vd->e[dr].frame_related = frame_related;
407 :
408 6426629 : if (flag_checking)
409 6426600 : validate_value_data (vd);
410 353371 : }
411 :
412 : /* Return true if a mode change from ORIG to NEW is allowed for REGNO. */
413 :
414 : static bool
415 290854 : mode_change_ok (machine_mode orig_mode, machine_mode new_mode,
416 : unsigned int regno ATTRIBUTE_UNUSED)
417 : {
418 290854 : if (partial_subreg_p (orig_mode, new_mode))
419 : return false;
420 :
421 284957 : return REG_CAN_CHANGE_MODE_P (regno, orig_mode, new_mode);
422 : }
423 :
424 : /* Register REGNO was originally set in ORIG_MODE. It - or a copy of it -
425 : was copied in COPY_MODE to COPY_REGNO, and then COPY_REGNO was accessed
426 : in NEW_MODE.
427 : Return a NEW_MODE rtx for REGNO if that's OK, otherwise return NULL_RTX. */
428 :
429 : static rtx
430 1739172 : maybe_mode_change (machine_mode orig_mode, machine_mode copy_mode,
431 : machine_mode new_mode, unsigned int regno,
432 : unsigned int copy_regno ATTRIBUTE_UNUSED)
433 : {
434 : /* All three modes precision have to be ordered to each other,
435 : otherwise partial_subreg_p won't work. */
436 1739172 : if (!ordered_p (GET_MODE_PRECISION (orig_mode),
437 1739172 : GET_MODE_PRECISION (copy_mode))
438 1739172 : || !ordered_p (GET_MODE_PRECISION (copy_mode),
439 1739172 : GET_MODE_PRECISION (new_mode)))
440 : return NULL_RTX;
441 :
442 1739172 : if (partial_subreg_p (copy_mode, orig_mode)
443 1739172 : && partial_subreg_p (copy_mode, new_mode))
444 : return NULL_RTX;
445 :
446 : /* Avoid creating multiple copies of the stack pointer. Some ports
447 : assume there is one and only one stack pointer.
448 :
449 : It's unclear if we need to do the same for other special registers. */
450 1737866 : if (regno == STACK_POINTER_REGNUM)
451 : {
452 25620 : if (orig_mode == new_mode && new_mode == GET_MODE (stack_pointer_rtx))
453 : return stack_pointer_rtx;
454 : else
455 191 : return NULL_RTX;
456 : }
457 :
458 1712246 : if (orig_mode == new_mode)
459 1562700 : return gen_raw_REG (new_mode, regno);
460 149546 : else if (mode_change_ok (orig_mode, new_mode, regno)
461 149546 : && mode_change_ok (copy_mode, new_mode, copy_regno))
462 : {
463 141308 : int copy_nregs = hard_regno_nregs (copy_regno, copy_mode);
464 141308 : int use_nregs = hard_regno_nregs (copy_regno, new_mode);
465 141308 : poly_uint64 bytes_per_reg;
466 282616 : if (!can_div_trunc_p (GET_MODE_SIZE (copy_mode),
467 : copy_nregs, &bytes_per_reg))
468 141186 : return NULL_RTX;
469 141308 : poly_uint64 copy_offset = bytes_per_reg * (copy_nregs - use_nregs);
470 141308 : poly_uint64 offset
471 141308 : = subreg_size_lowpart_offset (GET_MODE_SIZE (new_mode) + copy_offset,
472 141308 : GET_MODE_SIZE (orig_mode));
473 141308 : regno += subreg_regno_offset (regno, orig_mode, offset, new_mode);
474 141308 : if (targetm.hard_regno_mode_ok (regno, new_mode))
475 141186 : return gen_raw_REG (new_mode, regno);
476 : }
477 : return NULL_RTX;
478 : }
479 :
480 : /* Copy propagation must not replace a value in a frame-related
481 : instruction with one produced by a non–frame-related instruction.
482 : Doing so may cause the DCE pass to delete the original frame-related
483 : instruction, which would in turn produce incorrect DWARF information
484 : (see PR122274). TO_FRAME_RELATED indicates whether the destination
485 : instruction receiving the propagated value is frame-related.
486 : FROM_FRAME_RELATED indicates whether the instruction providing that
487 : value is frame-related.
488 : Return true if copy propagation is not permitted. */
489 :
490 : static bool
491 1739174 : incompatible_frame_status (bool to_frame_related, bool from_frame_related)
492 : {
493 1739174 : return to_frame_related && !from_frame_related;
494 : }
495 :
496 : /* Find the oldest copy of the value contained in REGNO that is in
497 : register class CL and has mode MODE. If found, return an rtx
498 : of that oldest register, otherwise return NULL. */
499 :
500 : static rtx
501 83487080 : find_oldest_value_reg (enum reg_class cl, rtx reg, struct value_data *vd, bool frame_related)
502 : {
503 83487080 : unsigned int regno = REGNO (reg);
504 83487080 : machine_mode mode = GET_MODE (reg);
505 83487080 : unsigned int i;
506 :
507 83487080 : gcc_assert (regno < FIRST_PSEUDO_REGISTER);
508 :
509 : /* If we are accessing REG in some mode other that what we set it in,
510 : make sure that the replacement is valid. In particular, consider
511 : (set (reg:DI r11) (...))
512 : (set (reg:SI r9) (reg:SI r11))
513 : (set (reg:SI r10) (...))
514 : (set (...) (reg:DI r9))
515 : Replacing r9 with r11 is invalid. */
516 83487080 : if (mode != vd->e[regno].mode
517 83487080 : && (REG_NREGS (reg) > hard_regno_nregs (regno, vd->e[regno].mode)
518 4459827 : || !REG_CAN_CHANGE_MODE_P (regno, mode, vd->e[regno].mode)))
519 : return NULL_RTX;
520 :
521 47172761 : for (i = vd->e[regno].oldest_regno; i != regno; i = vd->e[i].next_regno)
522 : {
523 2049037 : machine_mode oldmode = vd->e[i].mode;
524 2049037 : rtx new_rtx;
525 :
526 2049037 : if (!in_hard_reg_set_p (reg_class_contents[cl], mode, i))
527 616190 : continue;
528 :
529 1432847 : if (incompatible_frame_status (frame_related, vd->e[i].frame_related))
530 2 : continue;
531 :
532 1432845 : new_rtx = maybe_mode_change (oldmode, vd->e[regno].mode, mode, i, regno);
533 1432845 : if (new_rtx)
534 : {
535 : /* NEW_RTX may be the global stack pointer rtx, in which case we
536 : must not modify it's attributes. */
537 1424912 : if (new_rtx != stack_pointer_rtx)
538 : {
539 1416463 : ORIGINAL_REGNO (new_rtx) = ORIGINAL_REGNO (reg);
540 1416463 : REG_ATTRS (new_rtx) = REG_ATTRS (reg);
541 1416463 : REG_POINTER (new_rtx) = REG_POINTER (reg);
542 : }
543 : return new_rtx;
544 : }
545 : }
546 :
547 : return NULL_RTX;
548 : }
549 :
550 : /* If possible, replace the register at *LOC with the oldest register
551 : in register class CL. Return true if successfully replaced. */
552 :
553 : static bool
554 66309676 : replace_oldest_value_reg (rtx *loc, enum reg_class cl, rtx_insn *insn,
555 : struct value_data *vd)
556 : {
557 66320633 : rtx new_rtx = find_oldest_value_reg (cl, *loc, vd, RTX_FRAME_RELATED_P (insn));
558 66309676 : if (new_rtx && (!DEBUG_INSN_P (insn) || !skip_debug_insn_p))
559 : {
560 1122316 : if (DEBUG_INSN_P (insn))
561 : {
562 577537 : struct queued_debug_insn_change *change;
563 :
564 577537 : if (dump_file)
565 2 : fprintf (dump_file, "debug_insn %u: queued replacing reg %u with %u\n",
566 2 : INSN_UID (insn), REGNO (*loc), REGNO (new_rtx));
567 :
568 577537 : change = queued_debug_insn_change_pool.allocate ();
569 577537 : change->next = vd->e[REGNO (new_rtx)].debug_insn_changes;
570 577537 : change->insn = insn;
571 577537 : change->loc = loc;
572 577537 : change->new_rtx = new_rtx;
573 577537 : vd->e[REGNO (new_rtx)].debug_insn_changes = change;
574 577537 : ++vd->n_debug_insn_changes;
575 577537 : return true;
576 : }
577 544779 : if (dump_file)
578 34 : fprintf (dump_file, "insn %u: replaced reg %u with %u\n",
579 34 : INSN_UID (insn), REGNO (*loc), REGNO (new_rtx));
580 :
581 544779 : validate_change (insn, loc, new_rtx, 1);
582 544779 : return true;
583 : }
584 : return false;
585 : }
586 :
587 : /* Similar to replace_oldest_value_reg, but *LOC contains an address.
588 : Adapted from find_reloads_address_1. CL is INDEX_REG_CLASS or
589 : BASE_REG_CLASS depending on how the register is being considered. */
590 :
591 : static bool
592 102411281 : replace_oldest_value_addr (rtx *loc, enum reg_class cl,
593 : machine_mode mode, addr_space_t as,
594 : rtx_insn *insn, struct value_data *vd)
595 : {
596 102411281 : rtx x = *loc;
597 102411281 : RTX_CODE code = GET_CODE (x);
598 102411281 : const char *fmt;
599 102411281 : int i, j;
600 102411281 : bool changed = false;
601 :
602 102411281 : switch (code)
603 : {
604 28900447 : case PLUS:
605 28900447 : if (DEBUG_INSN_P (insn))
606 : break;
607 :
608 22590919 : {
609 22590919 : rtx orig_op0 = XEXP (x, 0);
610 22590919 : rtx orig_op1 = XEXP (x, 1);
611 22590919 : RTX_CODE code0 = GET_CODE (orig_op0);
612 22590919 : RTX_CODE code1 = GET_CODE (orig_op1);
613 22590919 : rtx op0 = orig_op0;
614 22590919 : rtx op1 = orig_op1;
615 22590919 : rtx *locI = NULL;
616 22590919 : rtx *locB = NULL;
617 22590919 : enum rtx_code index_code = SCRATCH;
618 :
619 22590919 : if (GET_CODE (op0) == SUBREG)
620 : {
621 0 : op0 = SUBREG_REG (op0);
622 0 : code0 = GET_CODE (op0);
623 : }
624 :
625 22590919 : if (GET_CODE (op1) == SUBREG)
626 : {
627 0 : op1 = SUBREG_REG (op1);
628 0 : code1 = GET_CODE (op1);
629 : }
630 :
631 22590919 : if (code0 == MULT || code0 == SIGN_EXTEND || code0 == TRUNCATE
632 22084941 : || code0 == ZERO_EXTEND || code1 == MEM)
633 : {
634 505978 : locI = &XEXP (x, 0);
635 505978 : locB = &XEXP (x, 1);
636 505978 : index_code = GET_CODE (*locI);
637 : }
638 22084941 : else if (code1 == MULT || code1 == SIGN_EXTEND || code1 == TRUNCATE
639 22083590 : || code1 == ZERO_EXTEND || code0 == MEM)
640 : {
641 1351 : locI = &XEXP (x, 1);
642 1351 : locB = &XEXP (x, 0);
643 1351 : index_code = GET_CODE (*locI);
644 : }
645 22083590 : else if (code0 == CONST_INT || code0 == CONST
646 22083590 : || code0 == SYMBOL_REF || code0 == LABEL_REF)
647 : {
648 801417 : locB = &XEXP (x, 1);
649 801417 : index_code = GET_CODE (XEXP (x, 0));
650 : }
651 21282173 : else if (code1 == CONST_INT || code1 == CONST
652 : || code1 == SYMBOL_REF || code1 == LABEL_REF)
653 : {
654 20222422 : locB = &XEXP (x, 0);
655 20222422 : index_code = GET_CODE (XEXP (x, 1));
656 : }
657 1059751 : else if (code0 == REG && code1 == REG)
658 : {
659 860830 : int index_op;
660 860830 : unsigned regno0 = REGNO (op0), regno1 = REGNO (op1);
661 :
662 370 : if (REGNO_OK_FOR_INDEX_P (regno1)
663 860834 : && regno_ok_for_base_p (regno0, mode, as, PLUS, REG))
664 : index_op = 1;
665 0 : else if (REGNO_OK_FOR_INDEX_P (regno0)
666 366 : && regno_ok_for_base_p (regno1, mode, as, PLUS, REG))
667 : index_op = 0;
668 0 : else if (regno_ok_for_base_p (regno0, mode, as, PLUS, REG)
669 0 : || REGNO_OK_FOR_INDEX_P (regno1))
670 : index_op = 1;
671 0 : else if (regno_ok_for_base_p (regno1, mode, as, PLUS, REG))
672 : index_op = 0;
673 : else
674 860464 : index_op = 1;
675 :
676 860830 : locI = &XEXP (x, index_op);
677 860830 : locB = &XEXP (x, !index_op);
678 860830 : index_code = GET_CODE (*locI);
679 : }
680 198921 : else if (code0 == REG)
681 : {
682 0 : locI = &XEXP (x, 0);
683 0 : locB = &XEXP (x, 1);
684 0 : index_code = GET_CODE (*locI);
685 : }
686 198921 : else if (code1 == REG)
687 : {
688 198921 : locI = &XEXP (x, 1);
689 198921 : locB = &XEXP (x, 0);
690 198921 : index_code = GET_CODE (*locI);
691 : }
692 :
693 22590919 : if (locI)
694 1567080 : changed |= replace_oldest_value_addr (locI, INDEX_REG_CLASS,
695 : mode, as, insn, vd);
696 22590919 : if (locB)
697 22590919 : changed |= replace_oldest_value_addr (locB,
698 : base_reg_class (mode, as, PLUS,
699 : index_code,
700 : NULL_RTX,
701 : insn),
702 : mode, as, insn, vd);
703 : return changed;
704 : }
705 :
706 : case POST_INC:
707 : case POST_DEC:
708 : case POST_MODIFY:
709 : case PRE_INC:
710 : case PRE_DEC:
711 : case PRE_MODIFY:
712 : return false;
713 :
714 1979025 : case MEM:
715 1979025 : return replace_oldest_value_mem (x, insn, vd);
716 :
717 38406204 : case REG:
718 38406204 : return replace_oldest_value_reg (loc, cl, insn, vd);
719 :
720 : default:
721 : break;
722 : }
723 :
724 35018640 : fmt = GET_RTX_FORMAT (code);
725 81722092 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
726 : {
727 46703452 : if (fmt[i] == 'e')
728 17837041 : changed |= replace_oldest_value_addr (&XEXP (x, i), cl, mode, as,
729 : insn, vd);
730 28866411 : else if (fmt[i] == 'E')
731 425991 : for (j = XVECLEN (x, i) - 1; j >= 0; j--)
732 261468 : changed |= replace_oldest_value_addr (&XVECEXP (x, i, j), cl,
733 : mode, as, insn, vd);
734 : }
735 :
736 : return changed;
737 : }
738 :
739 : /* Similar to replace_oldest_value_reg, but X contains a memory. */
740 :
741 : static bool
742 30983676 : replace_oldest_value_mem (rtx x, rtx_insn *insn, struct value_data *vd)
743 : {
744 30983676 : enum reg_class cl;
745 :
746 30983676 : if (DEBUG_INSN_P (insn))
747 : cl = ALL_REGS;
748 : else
749 31899670 : cl = base_reg_class (GET_MODE (x), MEM_ADDR_SPACE (x), MEM, SCRATCH,
750 : x, insn);
751 :
752 30983676 : return replace_oldest_value_addr (&XEXP (x, 0), cl,
753 30983676 : GET_MODE (x), MEM_ADDR_SPACE (x),
754 30983676 : insn, vd);
755 : }
756 :
757 : /* Apply all queued updates for DEBUG_INSNs that change some reg to
758 : register REGNO. */
759 :
760 : static void
761 182022 : apply_debug_insn_changes (struct value_data *vd, unsigned int regno)
762 : {
763 182022 : struct queued_debug_insn_change *change;
764 182022 : rtx_insn *last_insn = vd->e[regno].debug_insn_changes->insn;
765 :
766 182022 : for (change = vd->e[regno].debug_insn_changes;
767 511646 : change;
768 329624 : change = change->next)
769 : {
770 329624 : if (last_insn != change->insn)
771 : {
772 143910 : apply_change_group ();
773 143910 : last_insn = change->insn;
774 : }
775 329624 : validate_change (change->insn, change->loc, change->new_rtx, 1);
776 : }
777 182022 : apply_change_group ();
778 182022 : }
779 :
780 : /* Called via note_uses, for all used registers in a real insn
781 : apply DEBUG_INSN changes that change registers to the used
782 : registers. */
783 :
784 : static void
785 1036013 : cprop_find_used_regs (rtx *loc, void *data)
786 : {
787 1036013 : struct value_data *const vd = (struct value_data *) data;
788 1036013 : subrtx_iterator::array_type array;
789 3882723 : FOR_EACH_SUBRTX (iter, array, *loc, NONCONST)
790 : {
791 2846710 : const_rtx x = *iter;
792 2846710 : if (REG_P (x))
793 : {
794 984938 : unsigned int regno = REGNO (x);
795 984938 : if (vd->e[regno].debug_insn_changes)
796 : {
797 157818 : apply_debug_insn_changes (vd, regno);
798 157818 : free_debug_insn_changes (vd, regno);
799 : }
800 : }
801 : }
802 1036013 : }
803 :
804 : /* Apply clobbers of INSN in PATTERN and C_I_F_U to value_data VD. */
805 :
806 : static void
807 75340957 : kill_clobbered_values (rtx_insn *insn, struct value_data *vd)
808 : {
809 0 : note_stores (insn, kill_clobbered_value, vd);
810 0 : }
811 :
812 : /* Perform the forward copy propagation on basic block BB. */
813 :
814 : static bool
815 12468579 : copyprop_hardreg_forward_1 (basic_block bb, struct value_data *vd)
816 : {
817 12468579 : bool anything_changed = false;
818 12468579 : rtx_insn *insn, *next;
819 :
820 12468579 : for (insn = BB_HEAD (bb); ; insn = next)
821 : {
822 163303588 : int n_ops, i, predicated;
823 163303588 : bool is_asm, any_replacements;
824 163303588 : rtx set;
825 163303588 : rtx link;
826 163303588 : bool changed = false;
827 163303588 : struct kill_set_value_data ksvd;
828 :
829 163303588 : next = NEXT_INSN (insn);
830 163303588 : if (!NONDEBUG_INSN_P (insn))
831 : {
832 87859415 : if (DEBUG_BIND_INSN_P (insn))
833 : {
834 45803925 : rtx loc = INSN_VAR_LOCATION_LOC (insn);
835 45803925 : if (!VAR_LOC_UNKNOWN_P (loc))
836 25851725 : replace_oldest_value_addr (&INSN_VAR_LOCATION_LOC (insn),
837 25851725 : ALL_REGS, GET_MODE (loc),
838 : ADDR_SPACE_GENERIC, insn, vd);
839 : }
840 :
841 87859415 : if (insn == BB_END (bb))
842 : break;
843 : else
844 87809446 : continue;
845 : }
846 :
847 75444173 : set = single_set (insn);
848 :
849 : /* Detect noop sets and remove them before processing side effects. */
850 75444173 : if (set && REG_P (SET_DEST (set)) && REG_P (SET_SRC (set)))
851 : {
852 6617526 : unsigned int regno = REGNO (SET_SRC (set));
853 6617526 : rtx r1 = find_oldest_value_reg (REGNO_REG_CLASS (regno),
854 : SET_DEST (set), vd, false);
855 6617526 : rtx r2 = find_oldest_value_reg (REGNO_REG_CLASS (regno),
856 : SET_SRC (set), vd, false);
857 6617526 : if (rtx_equal_p (r1 ? r1 : SET_DEST (set), r2 ? r2 : SET_SRC (set)))
858 : {
859 65795 : bool last = insn == BB_END (bb);
860 65795 : delete_insn (insn);
861 65795 : if (last)
862 : break;
863 63519 : continue;
864 63519 : }
865 : }
866 :
867 : /* Detect obviously dead sets (via REG_UNUSED notes) and remove them. */
868 : if (set
869 69248908 : && !RTX_FRAME_RELATED_P (insn)
870 65294677 : && NONJUMP_INSN_P (insn)
871 54965167 : && !may_trap_p (set)
872 47977931 : && find_reg_note (insn, REG_UNUSED, SET_DEST (set))
873 261922 : && !side_effects_p (SET_SRC (set))
874 134581 : && !side_effects_p (SET_DEST (set)))
875 : {
876 134581 : bool last = insn == BB_END (bb);
877 134581 : delete_insn_and_edges (insn);
878 134581 : if (last)
879 : break;
880 134576 : continue;
881 134576 : }
882 :
883 :
884 75243797 : extract_constrain_insn (insn);
885 75243797 : preprocess_constraints (insn);
886 75243797 : const operand_alternative *op_alt = which_op_alt ();
887 75243797 : n_ops = recog_data.n_operands;
888 75243797 : is_asm = asm_noperands (PATTERN (insn)) >= 0;
889 :
890 : /* Simplify the code below by promoting OP_OUT to OP_INOUT
891 : in predicated instructions. */
892 :
893 75243797 : predicated = GET_CODE (PATTERN (insn)) == COND_EXEC;
894 233012940 : for (i = 0; i < n_ops; ++i)
895 : {
896 157769143 : int matches = op_alt[i].matches;
897 157769143 : if (matches >= 0 || op_alt[i].matched >= 0
898 139650858 : || (predicated && recog_data.operand_type[i] == OP_OUT))
899 18118285 : recog_data.operand_type[i] = OP_INOUT;
900 : }
901 :
902 : /* Apply changes to earlier DEBUG_INSNs if possible. */
903 75243797 : if (vd->n_debug_insn_changes)
904 771403 : note_uses (&PATTERN (insn), cprop_find_used_regs, vd);
905 :
906 : /* For each earlyclobber operand, zap the value data. */
907 233012940 : for (i = 0; i < n_ops; i++)
908 157769143 : if (op_alt[i].earlyclobber)
909 13300 : kill_value (recog_data.operand[i], vd);
910 :
911 : /* Within asms, a clobber cannot overlap inputs or outputs.
912 : I wouldn't think this were true for regular insns, but
913 : scan_rtx treats them like that... */
914 75243797 : kill_clobbered_values (insn, vd);
915 :
916 : /* Kill all auto-incremented values. */
917 : /* ??? REG_INC is useless, since stack pushes aren't done that way. */
918 75243797 : kill_autoinc_value (insn, vd);
919 :
920 : /* Kill all early-clobbered operands. */
921 308256737 : for (i = 0; i < n_ops; i++)
922 157769143 : if (op_alt[i].earlyclobber)
923 13300 : kill_value (recog_data.operand[i], vd);
924 :
925 : /* If we have dead sets in the insn, then we need to note these as we
926 : would clobbers. */
927 145645309 : for (link = REG_NOTES (insn); link; link = XEXP (link, 1))
928 : {
929 70401536 : if (REG_NOTE_KIND (link) == REG_UNUSED)
930 : {
931 9502428 : kill_value (XEXP (link, 0), vd);
932 : /* Furthermore, if the insn looked like a single-set,
933 : but the dead store kills the source value of that
934 : set, then we can no-longer use the plain move
935 : special case below. */
936 9502428 : if (set
937 9502428 : && reg_overlap_mentioned_p (XEXP (link, 0), SET_SRC (set)))
938 : set = NULL;
939 : }
940 :
941 : /* We need to keep CFI info correct, and the same on all paths,
942 : so we cannot normally replace the registers REG_CFA_REGISTER
943 : refers to. Bail. */
944 70401536 : if (REG_NOTE_KIND (link) == REG_CFA_REGISTER)
945 24 : goto did_replacement;
946 : }
947 :
948 : /* Special-case plain move instructions, since we may well
949 : be able to do the move from a different register class. */
950 75243773 : if (set && REG_P (SET_SRC (set)))
951 : {
952 16919925 : rtx src = SET_SRC (set);
953 16919925 : rtx dest = SET_DEST (set);
954 16919925 : unsigned int regno = REGNO (src);
955 16919925 : machine_mode mode = GET_MODE (src);
956 16919925 : unsigned int i;
957 16919925 : rtx new_rtx;
958 :
959 : /* If we are accessing SRC in some mode other that what we
960 : set it in, make sure that the replacement is valid. */
961 16919925 : if (mode != vd->e[regno].mode)
962 : {
963 6695904 : if (REG_NREGS (src)
964 6695904 : > hard_regno_nregs (regno, vd->e[regno].mode))
965 5337551 : goto no_move_special_case;
966 :
967 : /* And likewise, if we are narrowing on big endian the transformation
968 : is also invalid. */
969 1358353 : if (REG_NREGS (src) < hard_regno_nregs (regno, vd->e[regno].mode)
970 1358353 : && maybe_ne (subreg_lowpart_offset (mode,
971 : vd->e[regno].mode), 0U))
972 0 : goto no_move_special_case;
973 : }
974 :
975 : /* If the destination is also a register, try to find a source
976 : register in the same class. */
977 11582374 : if (REG_P (dest))
978 : {
979 11827056 : new_rtx = find_oldest_value_reg (REGNO_REG_CLASS (regno),
980 3942352 : src, vd, RTX_FRAME_RELATED_P (insn));
981 :
982 3942352 : if (new_rtx && validate_change (insn, &SET_SRC (set), new_rtx, 0))
983 : {
984 72393 : if (dump_file)
985 2 : fprintf (dump_file,
986 : "insn %u: replaced reg %u with %u\n",
987 2 : INSN_UID (insn), regno, REGNO (new_rtx));
988 72393 : changed = true;
989 72393 : goto did_replacement;
990 : }
991 : /* We need to re-extract as validate_change clobbers
992 : recog_data. */
993 3869959 : extract_constrain_insn (insn);
994 3869959 : preprocess_constraints (insn);
995 : }
996 :
997 : /* Otherwise, try all valid registers and see if its valid. */
998 11527534 : for (i = vd->e[regno].oldest_regno; i != regno;
999 17553 : i = vd->e[i].next_regno)
1000 : {
1001 306327 : if (incompatible_frame_status (RTX_FRAME_RELATED_P (insn), vd->e[i].frame_related))
1002 0 : continue;
1003 306327 : new_rtx = maybe_mode_change (vd->e[i].mode, vd->e[regno].mode,
1004 : mode, i, regno);
1005 306327 : if (new_rtx != NULL_RTX)
1006 : {
1007 : /* Don't propagate for a more expensive reg-reg move. */
1008 304403 : if (REG_P (dest))
1009 : {
1010 152055 : enum reg_class from = REGNO_REG_CLASS (regno);
1011 152055 : enum reg_class to = REGNO_REG_CLASS (REGNO (dest));
1012 152055 : enum reg_class new_from = REGNO_REG_CLASS (i);
1013 152055 : unsigned int original_cost
1014 152055 : = targetm.register_move_cost (mode, from, to);
1015 152055 : unsigned int after_cost
1016 152055 : = targetm.register_move_cost (mode, new_from, to);
1017 152055 : if (after_cost > original_cost)
1018 9202 : continue;
1019 : }
1020 :
1021 295201 : if (validate_change (insn, &SET_SRC (set), new_rtx, 0))
1022 : {
1023 : /* NEW_RTX may be the global stack pointer rtx, in which
1024 : case we must not modify it's attributes. */
1025 288774 : if (new_rtx != stack_pointer_rtx)
1026 : {
1027 271802 : ORIGINAL_REGNO (new_rtx) = ORIGINAL_REGNO (src);
1028 271802 : REG_ATTRS (new_rtx) = REG_ATTRS (src);
1029 271802 : REG_POINTER (new_rtx) = REG_POINTER (src);
1030 : }
1031 288774 : if (dump_file)
1032 1 : fprintf (dump_file,
1033 : "insn %u: replaced reg %u with %u\n",
1034 1 : INSN_UID (insn), regno, REGNO (new_rtx));
1035 288774 : changed = true;
1036 288774 : goto did_replacement;
1037 : }
1038 : /* We need to re-extract as validate_change clobbers
1039 : recog_data. */
1040 6427 : extract_constrain_insn (insn);
1041 6427 : preprocess_constraints (insn);
1042 : }
1043 : }
1044 : }
1045 58323848 : no_move_special_case:
1046 :
1047 74882606 : any_replacements = false;
1048 :
1049 : /* For each input operand, replace a hard register with the
1050 : eldest live copy that's in an appropriate register class. */
1051 231929370 : for (i = 0; i < n_ops; i++)
1052 : {
1053 157046764 : bool replaced = false;
1054 :
1055 : /* Don't scan match_operand here, since we've no reg class
1056 : information to pass down. Any operands that we could
1057 : substitute in will be represented elsewhere. */
1058 157046764 : if (recog_data.constraints[i][0] == '\0')
1059 28210792 : continue;
1060 :
1061 : /* Don't replace in asms intentionally referencing hard regs. */
1062 160731 : if (is_asm && REG_P (recog_data.operand[i])
1063 128974541 : && (REGNO (recog_data.operand[i])
1064 138569 : == ORIGINAL_REGNO (recog_data.operand[i])))
1065 2040 : continue;
1066 :
1067 128833932 : if (recog_data.operand_type[i] == OP_IN)
1068 : {
1069 66533701 : if (op_alt[i].is_address)
1070 3319372 : replaced
1071 3319372 : = replace_oldest_value_addr (recog_data.operand_loc[i],
1072 : alternative_class (op_alt, i),
1073 : VOIDmode, ADDR_SPACE_GENERIC,
1074 : insn, vd);
1075 63214329 : else if (REG_P (recog_data.operand[i]))
1076 27903472 : replaced
1077 27903472 : = replace_oldest_value_reg (recog_data.operand_loc[i],
1078 : alternative_class (op_alt, i),
1079 : insn, vd);
1080 35310857 : else if (MEM_P (recog_data.operand[i]))
1081 14496650 : replaced = replace_oldest_value_mem (recog_data.operand[i],
1082 : insn, vd);
1083 : }
1084 62300231 : else if (MEM_P (recog_data.operand[i]))
1085 14508001 : replaced = replace_oldest_value_mem (recog_data.operand[i],
1086 : insn, vd);
1087 :
1088 : /* If we performed any replacement, update match_dups. */
1089 60227495 : if (replaced)
1090 : {
1091 543284 : int j;
1092 543284 : rtx new_rtx;
1093 :
1094 543284 : new_rtx = *recog_data.operand_loc[i];
1095 543284 : recog_data.operand[i] = new_rtx;
1096 572877 : for (j = 0; j < recog_data.n_dups; j++)
1097 29593 : if (recog_data.dup_num[j] == i)
1098 9949 : validate_unshare_change (insn, recog_data.dup_loc[j], new_rtx, 1);
1099 :
1100 : any_replacements = true;
1101 : }
1102 : }
1103 :
1104 74882606 : if (any_replacements)
1105 : {
1106 540449 : if (! apply_change_group ())
1107 : {
1108 76 : if (dump_file)
1109 0 : fprintf (dump_file,
1110 : "insn %u: reg replacements not verified\n",
1111 0 : INSN_UID (insn));
1112 : }
1113 : else
1114 : changed = true;
1115 : }
1116 :
1117 361191 : did_replacement:
1118 361191 : if (changed)
1119 : {
1120 901540 : anything_changed = true;
1121 :
1122 : /* If something changed, perhaps further changes to earlier
1123 : DEBUG_INSNs can be applied. */
1124 901540 : if (vd->n_debug_insn_changes)
1125 123170 : note_uses (&PATTERN (insn), cprop_find_used_regs, vd);
1126 901540 : df_insn_rescan (insn);
1127 : }
1128 :
1129 75243797 : ksvd.vd = vd;
1130 75243797 : ksvd.ignore_set_reg = NULL_RTX;
1131 75243797 : ksvd.insn_is_frame_related = false;
1132 :
1133 : /* Clobber call-clobbered registers. */
1134 75243797 : if (auto call = dyn_cast<rtx_call_insn *> (insn))
1135 : {
1136 5659615 : unsigned int set_regno = INVALID_REGNUM;
1137 5659615 : unsigned int set_nregs = 0;
1138 5659615 : unsigned int regno;
1139 5659615 : rtx exp;
1140 :
1141 16207769 : for (exp = CALL_INSN_FUNCTION_USAGE (call); exp; exp = XEXP (exp, 1))
1142 : {
1143 10645314 : rtx x = XEXP (exp, 0);
1144 10645314 : if (GET_CODE (x) == SET)
1145 : {
1146 97160 : rtx dest = SET_DEST (x);
1147 97160 : kill_value (dest, vd);
1148 97160 : set_value_regno (REGNO (dest), GET_MODE (dest), vd);
1149 97160 : copy_value (dest, SET_SRC (x), vd, false);
1150 97160 : ksvd.ignore_set_reg = dest;
1151 97160 : set_regno = REGNO (dest);
1152 97160 : set_nregs = REG_NREGS (dest);
1153 97160 : break;
1154 : }
1155 : }
1156 :
1157 5659615 : function_abi callee_abi = insn_callee_abi (call);
1158 543323040 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
1159 532003810 : if (vd->e[regno].mode != VOIDmode
1160 25467700 : && callee_abi.clobbers_reg_p (vd->e[regno].mode, regno)
1161 547498308 : && (regno < set_regno || regno >= set_regno + set_nregs))
1162 15397338 : kill_value_regno (regno, 1, vd);
1163 :
1164 : /* If SET was seen in CALL_INSN_FUNCTION_USAGE, and SET_SRC
1165 : of the SET isn't clobbered by CALLEE_ABI, but instead among
1166 : CLOBBERs on the CALL_INSN, we could wrongly assume the
1167 : value in it is still live. */
1168 5659615 : if (ksvd.ignore_set_reg)
1169 97160 : kill_clobbered_values (call, vd);
1170 : }
1171 :
1172 156904789 : bool copy_p = (set
1173 68754831 : && REG_P (SET_DEST (set))
1174 121400870 : && REG_P (SET_SRC (set)));
1175 6417214 : bool noop_p = (copy_p
1176 6417214 : && rtx_equal_p (SET_DEST (set), SET_SRC (set)));
1177 :
1178 : /* If a noop move is using narrower mode than we have recorded,
1179 : we need to either remove the noop move, or kill_set_value. */
1180 35430 : if (noop_p
1181 35430 : && partial_subreg_p (GET_MODE (SET_DEST (set)),
1182 35430 : vd->e[REGNO (SET_DEST (set))].mode))
1183 : {
1184 2231 : if (noop_move_p (insn))
1185 : {
1186 2231 : bool last = insn == BB_END (bb);
1187 2231 : delete_insn (insn);
1188 2231 : if (last)
1189 : break;
1190 : }
1191 : else
1192 : noop_p = false;
1193 : }
1194 :
1195 75243778 : if (!noop_p)
1196 : {
1197 75208367 : ksvd.insn_is_frame_related = RTX_FRAME_RELATED_P (insn);
1198 :
1199 : /* Notice stores. */
1200 75208367 : note_stores (insn, kill_set_value, &ksvd);
1201 :
1202 : /* Notice copies. */
1203 75208367 : if (copy_p)
1204 : {
1205 6381784 : df_insn_rescan (insn);
1206 6381784 : copy_value (SET_DEST (set), SET_SRC (set), vd, RTX_FRAME_RELATED_P (insn));
1207 : }
1208 : }
1209 :
1210 75243778 : if (insn == BB_END (bb))
1211 : break;
1212 150835009 : }
1213 :
1214 12468579 : return anything_changed;
1215 : }
1216 :
1217 : /* Dump the value chain data to stderr. */
1218 :
1219 : DEBUG_FUNCTION void
1220 0 : debug_value_data (struct value_data *vd)
1221 : {
1222 0 : HARD_REG_SET set;
1223 0 : unsigned int i, j;
1224 :
1225 0 : CLEAR_HARD_REG_SET (set);
1226 :
1227 0 : for (i = 0; i < FIRST_PSEUDO_REGISTER; ++i)
1228 0 : if (vd->e[i].oldest_regno == i)
1229 : {
1230 0 : if (vd->e[i].mode == VOIDmode)
1231 : {
1232 0 : if (vd->e[i].next_regno != INVALID_REGNUM)
1233 0 : fprintf (stderr, "[%u] Bad next_regno for empty chain (%u)\n",
1234 : i, vd->e[i].next_regno);
1235 0 : continue;
1236 : }
1237 :
1238 0 : SET_HARD_REG_BIT (set, i);
1239 0 : fprintf (stderr, "[%u %s] ", i, GET_MODE_NAME (vd->e[i].mode));
1240 :
1241 0 : for (j = vd->e[i].next_regno;
1242 0 : j != INVALID_REGNUM;
1243 0 : j = vd->e[j].next_regno)
1244 : {
1245 0 : if (TEST_HARD_REG_BIT (set, j))
1246 : {
1247 0 : fprintf (stderr, "[%u] Loop in regno chain\n", j);
1248 0 : return;
1249 : }
1250 :
1251 0 : if (vd->e[j].oldest_regno != i)
1252 : {
1253 0 : fprintf (stderr, "[%u] Bad oldest_regno (%u)\n",
1254 : j, vd->e[j].oldest_regno);
1255 0 : return;
1256 : }
1257 0 : SET_HARD_REG_BIT (set, j);
1258 0 : fprintf (stderr, "[%u %s] ", j, GET_MODE_NAME (vd->e[j].mode));
1259 : }
1260 0 : fputc ('\n', stderr);
1261 : }
1262 :
1263 0 : for (i = 0; i < FIRST_PSEUDO_REGISTER; ++i)
1264 0 : if (! TEST_HARD_REG_BIT (set, i)
1265 0 : && (vd->e[i].mode != VOIDmode
1266 0 : || vd->e[i].oldest_regno != i
1267 0 : || vd->e[i].next_regno != INVALID_REGNUM))
1268 0 : fprintf (stderr, "[%u] Non-empty reg in chain (%s %u %i)\n",
1269 0 : i, GET_MODE_NAME (vd->e[i].mode), vd->e[i].oldest_regno,
1270 : vd->e[i].next_regno);
1271 : }
1272 :
1273 : /* Do copyprop_hardreg_forward_1 for a single basic block BB.
1274 : DEBUG_INSN is skipped since we do not want to involve DF related
1275 : staff as how it is handled in function pass_cprop_hardreg::execute.
1276 :
1277 : NOTE: Currently it is only used for shrink-wrap. Maybe extend it
1278 : to handle DEBUG_INSN for other uses. */
1279 :
1280 : void
1281 365154 : copyprop_hardreg_forward_bb_without_debug_insn (basic_block bb)
1282 : {
1283 365154 : struct value_data *vd;
1284 365154 : vd = XNEWVEC (struct value_data, 1);
1285 365154 : init_value_data (vd);
1286 :
1287 365154 : skip_debug_insn_p = true;
1288 365154 : copyprop_hardreg_forward_1 (bb, vd);
1289 365154 : free (vd);
1290 365154 : skip_debug_insn_p = false;
1291 365154 : }
1292 :
1293 : static void
1294 92839202 : validate_value_data (struct value_data *vd)
1295 : {
1296 92839202 : HARD_REG_SET set;
1297 92839202 : unsigned int i, j;
1298 :
1299 92839202 : CLEAR_HARD_REG_SET (set);
1300 :
1301 8819724190 : for (i = 0; i < FIRST_PSEUDO_REGISTER; ++i)
1302 8726884988 : if (vd->e[i].oldest_regno == i)
1303 : {
1304 8698450688 : if (vd->e[i].mode == VOIDmode)
1305 : {
1306 8364463533 : if (vd->e[i].next_regno != INVALID_REGNUM)
1307 0 : internal_error ("%qs: [%u] bad %<next_regno%> for empty chain (%u)",
1308 : __func__, i, vd->e[i].next_regno);
1309 8364463533 : continue;
1310 : }
1311 :
1312 333987155 : SET_HARD_REG_BIT (set, i);
1313 :
1314 333987155 : for (j = vd->e[i].next_regno;
1315 362421455 : j != INVALID_REGNUM;
1316 28434300 : j = vd->e[j].next_regno)
1317 : {
1318 28434300 : if (TEST_HARD_REG_BIT (set, j))
1319 0 : internal_error ("%qs: loop in %<next_regno%> chain (%u)",
1320 : __func__, j);
1321 28434300 : if (vd->e[j].oldest_regno != i)
1322 0 : internal_error ("%qs: [%u] bad %<oldest_regno%> (%u)",
1323 : __func__, j, vd->e[j].oldest_regno);
1324 :
1325 28434300 : SET_HARD_REG_BIT (set, j);
1326 : }
1327 : }
1328 :
1329 8819724190 : for (i = 0; i < FIRST_PSEUDO_REGISTER; ++i)
1330 8726884988 : if (! TEST_HARD_REG_BIT (set, i)
1331 8726884988 : && (vd->e[i].mode != VOIDmode
1332 8364463533 : || vd->e[i].oldest_regno != i
1333 8364463533 : || vd->e[i].next_regno != INVALID_REGNUM))
1334 0 : internal_error ("%qs: [%u] non-empty register in chain (%s %u %i)",
1335 : __func__, i,
1336 0 : GET_MODE_NAME (vd->e[i].mode), vd->e[i].oldest_regno,
1337 : vd->e[i].next_regno);
1338 92839202 : }
1339 :
1340 :
1341 : namespace {
1342 :
1343 : const pass_data pass_data_cprop_hardreg =
1344 : {
1345 : RTL_PASS, /* type */
1346 : "cprop_hardreg", /* name */
1347 : OPTGROUP_NONE, /* optinfo_flags */
1348 : TV_CPROP_REGISTERS, /* tv_id */
1349 : 0, /* properties_required */
1350 : 0, /* properties_provided */
1351 : 0, /* properties_destroyed */
1352 : 0, /* todo_flags_start */
1353 : TODO_df_finish, /* todo_flags_finish */
1354 : };
1355 :
1356 : class pass_cprop_hardreg : public rtl_opt_pass
1357 : {
1358 : public:
1359 297180 : pass_cprop_hardreg (gcc::context *ctxt)
1360 594360 : : rtl_opt_pass (pass_data_cprop_hardreg, ctxt)
1361 : {}
1362 :
1363 : /* opt_pass methods: */
1364 1517135 : bool gate (function *) final override
1365 : {
1366 1517135 : return (optimize > 0 && (flag_cprop_registers));
1367 : }
1368 :
1369 : unsigned int execute (function *) final override;
1370 :
1371 : }; // class pass_cprop_hardreg
1372 :
1373 : static bool
1374 12103425 : cprop_hardreg_bb (basic_block bb, struct value_data *all_vd, sbitmap visited)
1375 : {
1376 12103425 : bitmap_set_bit (visited, bb->index);
1377 :
1378 : /* If a block has a single predecessor, that we've already
1379 : processed, begin with the value data that was live at
1380 : the end of the predecessor block. */
1381 : /* ??? Ought to use more intelligent queuing of blocks. */
1382 12103425 : if (single_pred_p (bb)
1383 8963615 : && bitmap_bit_p (visited, single_pred (bb)->index)
1384 19536525 : && ! (single_pred_edge (bb)->flags & (EDGE_ABNORMAL_CALL | EDGE_EH)))
1385 : {
1386 7300720 : all_vd[bb->index] = all_vd[single_pred (bb)->index];
1387 7300720 : if (all_vd[bb->index].n_debug_insn_changes)
1388 : {
1389 : unsigned int regno;
1390 :
1391 483558 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
1392 : {
1393 483558 : if (all_vd[bb->index].e[regno].debug_insn_changes)
1394 : {
1395 : struct queued_debug_insn_change *cur;
1396 102354 : for (cur = all_vd[bb->index].e[regno].debug_insn_changes;
1397 171439 : cur; cur = cur->next)
1398 102354 : --all_vd[bb->index].n_debug_insn_changes;
1399 69085 : all_vd[bb->index].e[regno].debug_insn_changes = NULL;
1400 69085 : if (all_vd[bb->index].n_debug_insn_changes == 0)
1401 : break;
1402 : }
1403 : }
1404 : }
1405 : }
1406 : else
1407 4802705 : init_value_data (all_vd + bb->index);
1408 :
1409 12103425 : return copyprop_hardreg_forward_1 (bb, all_vd + bb->index);
1410 : }
1411 :
1412 : static void
1413 49170 : cprop_hardreg_debug (function *fun, struct value_data *all_vd)
1414 : {
1415 49170 : basic_block bb;
1416 :
1417 3214868 : FOR_EACH_BB_FN (bb, fun)
1418 3165698 : if (all_vd[bb->index].n_debug_insn_changes)
1419 : {
1420 80603 : unsigned int regno;
1421 80603 : bitmap live;
1422 :
1423 80603 : live = df_get_live_out (bb);
1424 669235 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
1425 588632 : if (all_vd[bb->index].e[regno].debug_insn_changes)
1426 : {
1427 83243 : if (REGNO_REG_SET_P (live, regno))
1428 24204 : apply_debug_insn_changes (all_vd + bb->index, regno);
1429 :
1430 83243 : struct queued_debug_insn_change *cur;
1431 83243 : for (cur = all_vd[bb->index].e[regno].debug_insn_changes;
1432 216131 : cur; cur = cur->next)
1433 132888 : --all_vd[bb->index].n_debug_insn_changes;
1434 83243 : all_vd[bb->index].e[regno].debug_insn_changes = NULL;
1435 83243 : if (all_vd[bb->index].n_debug_insn_changes == 0)
1436 : break;
1437 : }
1438 : }
1439 :
1440 49170 : queued_debug_insn_change_pool.release ();
1441 49170 : }
1442 :
1443 : unsigned int
1444 1064960 : pass_cprop_hardreg::execute (function *fun)
1445 : {
1446 1064960 : struct value_data *all_vd;
1447 1064960 : basic_block bb;
1448 :
1449 1064960 : all_vd = XNEWVEC (struct value_data, last_basic_block_for_fn (fun));
1450 :
1451 1064960 : auto_sbitmap visited (last_basic_block_for_fn (fun));
1452 1064960 : bitmap_clear (visited);
1453 :
1454 1064960 : auto_vec<int> worklist1, worklist2;
1455 1064960 : auto_vec<int> *curr = &worklist1;
1456 1064960 : auto_vec<int> *next = &worklist2;
1457 1064960 : bool any_debug_changes = false;
1458 1064960 : bool any_changes = false;
1459 :
1460 : /* We need accurate notes. Earlier passes such as if-conversion may
1461 : leave notes in an inconsistent state. */
1462 1064960 : df_note_add_problem ();
1463 1064960 : df_analyze ();
1464 :
1465 : /* It is tempting to set DF_LR_RUN_DCE, but DCE may choose to delete
1466 : an insn and this pass would not have visibility into the removal.
1467 : This pass would then potentially use the source of that
1468 : INSN for propagation purposes, generating invalid code.
1469 :
1470 : So we just ask for updated notes and handle trivial deletions
1471 : within this pass where we can update this passes internal
1472 : data structures appropriately. */
1473 1064960 : df_set_flags (DF_DEFER_INSN_RESCAN);
1474 :
1475 12680262 : FOR_EACH_BB_FN (bb, fun)
1476 : {
1477 11615302 : if (cprop_hardreg_bb (bb, all_vd, visited))
1478 : {
1479 483819 : curr->safe_push (bb->index);
1480 483819 : any_changes = true;
1481 : }
1482 11615302 : if (all_vd[bb->index].n_debug_insn_changes)
1483 68442 : any_debug_changes = true;
1484 : }
1485 :
1486 : /* We must call df_analyze here unconditionally to ensure that the
1487 : REG_UNUSED and REG_DEAD notes are consistent with and without -g. */
1488 1064960 : df_analyze ();
1489 :
1490 1064960 : if (MAY_HAVE_DEBUG_BIND_INSNS && any_debug_changes)
1491 40399 : cprop_hardreg_debug (fun, all_vd);
1492 :
1493 : /* Repeat pass up to PASSES times, but only processing basic blocks
1494 : that have changed on the previous iteration. CURR points to the
1495 : current worklist, and each iteration populates the NEXT worklist,
1496 : swapping pointers after each cycle. */
1497 :
1498 1064960 : unsigned int passes = optimize > 1 ? 3 : 2;
1499 1249942 : for (unsigned int pass = 2; pass <= passes && !curr->is_empty (); pass++)
1500 : {
1501 184982 : any_debug_changes = false;
1502 184982 : bitmap_clear (visited);
1503 184982 : next->truncate (0);
1504 673105 : for (int index : *curr)
1505 : {
1506 488123 : bb = BASIC_BLOCK_FOR_FN (fun, index);
1507 488123 : if (cprop_hardreg_bb (bb, all_vd, visited))
1508 : {
1509 5133 : next->safe_push (bb->index);
1510 5133 : any_changes = true;
1511 : }
1512 488123 : if (all_vd[bb->index].n_debug_insn_changes)
1513 12161 : any_debug_changes = true;
1514 : }
1515 :
1516 184982 : df_analyze ();
1517 184982 : if (MAY_HAVE_DEBUG_BIND_INSNS && any_debug_changes)
1518 8771 : cprop_hardreg_debug (fun, all_vd);
1519 184982 : std::swap (curr, next);
1520 : }
1521 :
1522 1064960 : free (all_vd);
1523 :
1524 : /* Copy propagation of the sp register into an mem's address
1525 : can cause what was a trapping instruction into
1526 : a non-trapping one so cleanup after that in the case of non-call
1527 : exceptions. */
1528 1064960 : if (any_changes
1529 181772 : && cfun->can_throw_non_call_exceptions
1530 1116596 : && purge_all_dead_edges ())
1531 4 : cleanup_cfg (0);
1532 1064960 : return 0;
1533 1064960 : }
1534 :
1535 : } // anon namespace
1536 :
1537 : rtl_opt_pass *
1538 297180 : make_pass_cprop_hardreg (gcc::context *ctxt)
1539 : {
1540 297180 : return new pass_cprop_hardreg (ctxt);
1541 : }
|