Line data Source code
1 : /* Decompose multiword subregs.
2 : Copyright (C) 2007-2026 Free Software Foundation, Inc.
3 : Contributed by Richard Henderson <rth@redhat.com>
4 : Ian Lance Taylor <iant@google.com>
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #include "config.h"
23 : #include "system.h"
24 : #include "coretypes.h"
25 : #include "backend.h"
26 : #include "rtl.h"
27 : #include "tree.h"
28 : #include "cfghooks.h"
29 : #include "df.h"
30 : #include "memmodel.h"
31 : #include "tm_p.h"
32 : #include "expmed.h"
33 : #include "regs.h"
34 : #include "insn-config.h"
35 : #include "emit-rtl.h"
36 : #include "recog.h"
37 : #include "cfgrtl.h"
38 : #include "cfgbuild.h"
39 : #include "dce.h"
40 : #include "expr.h"
41 : #include "explow.h"
42 : #include "tree-pass.h"
43 : #include "lower-subreg.h"
44 : #include "rtl-iter.h"
45 : #include "target.h"
46 :
47 :
48 : /* Decompose multi-word pseudo-registers into individual
49 : pseudo-registers when possible and profitable. This is possible
50 : when all the uses of a multi-word register are via SUBREG, or are
51 : copies of the register to another location. Breaking apart the
52 : register permits more CSE and permits better register allocation.
53 : This is profitable if the machine does not have move instructions
54 : to do this.
55 :
56 : This pass only splits moves with modes that are wider than
57 : word_mode and ASHIFTs, LSHIFTRTs, ASHIFTRTs and ZERO_EXTENDs with
58 : integer modes that are twice the width of word_mode. The latter
59 : could be generalized if there was a need to do this, but the trend in
60 : architectures is to not need this.
61 :
62 : There are two useful preprocessor defines for use by maintainers:
63 :
64 : #define LOG_COSTS 1
65 :
66 : if you wish to see the actual cost estimates that are being used
67 : for each mode wider than word mode and the cost estimates for zero
68 : extension and the shifts. This can be useful when port maintainers
69 : are tuning insn rtx costs.
70 :
71 : #define FORCE_LOWERING 1
72 :
73 : if you wish to test the pass with all the transformation forced on.
74 : This can be useful for finding bugs in the transformations. */
75 :
76 : #define LOG_COSTS 0
77 : #define FORCE_LOWERING 0
78 :
79 : /* True if a CONCATN was created by decompose_register. */
80 : #define DECOMPOSED_REGISTER_P(x) \
81 : (RTL_FLAG_CHECK1 ("DECOMPOSED_REGISTER_P", (x), CONCATN)->volatil)
82 :
83 : /* Bit N in this bitmap is set if regno N is used in a context in
84 : which we can decompose it. */
85 : static bitmap decomposable_context;
86 :
87 : /* Bit N in this bitmap is set if regno N is used in a context in
88 : which it cannot be decomposed. */
89 : static bitmap non_decomposable_context;
90 :
91 : /* Bit N in this bitmap is set if regno N is used in a subreg
92 : which changes the mode but not the size. This typically happens
93 : when the register accessed as a floating-point value; we want to
94 : avoid generating accesses to its subwords in integer modes. */
95 : static bitmap subreg_context;
96 :
97 : /* Bit N in the bitmap in element M of this array is set if there is a
98 : copy from reg M to reg N. */
99 : static vec<bitmap> reg_copy_graph;
100 :
101 : struct target_lower_subreg default_target_lower_subreg;
102 : #if SWITCHABLE_TARGET
103 : struct target_lower_subreg *this_target_lower_subreg
104 : = &default_target_lower_subreg;
105 : #endif
106 :
107 : #define twice_word_mode \
108 : this_target_lower_subreg->x_twice_word_mode
109 : #define choices \
110 : this_target_lower_subreg->x_choices
111 :
112 : /* Return true if MODE is a mode we know how to lower. When returning true,
113 : store its byte size in *BYTES and its word size in *WORDS. */
114 :
115 : static inline bool
116 120782296 : interesting_mode_p (machine_mode mode, unsigned int *bytes,
117 : unsigned int *words)
118 : {
119 241564592 : if (!GET_MODE_SIZE (mode).is_constant (bytes))
120 : return false;
121 143698672 : if (maybe_lt ((unsigned) UNITS_PER_WORD,
122 120782296 : (poly_uint64) REGMODE_NATURAL_SIZE (mode)))
123 : return false;
124 120782296 : *words = CEIL (*bytes, UNITS_PER_WORD);
125 120782296 : return true;
126 : }
127 :
128 : /* RTXes used while computing costs. */
129 : struct cost_rtxes {
130 : /* Source and target registers. */
131 : rtx source;
132 : rtx target;
133 :
134 : /* A twice_word_mode ZERO_EXTEND of SOURCE. */
135 : rtx zext;
136 :
137 : /* A shift of SOURCE. */
138 : rtx shift;
139 :
140 : /* A SET of TARGET. */
141 : rtx set;
142 : };
143 :
144 : /* Return the cost of a CODE shift in mode MODE by OP1 bits, using the
145 : rtxes in RTXES. SPEED_P selects between the speed and size cost. */
146 :
147 : static int
148 193086784 : shift_cost (bool speed_p, struct cost_rtxes *rtxes, enum rtx_code code,
149 : machine_mode mode, int op1)
150 : {
151 193086784 : PUT_CODE (rtxes->shift, code);
152 193086784 : PUT_MODE (rtxes->shift, mode);
153 193086784 : PUT_MODE (rtxes->source, mode);
154 193086784 : XEXP (rtxes->shift, 1) = gen_int_shift_amount (mode, op1);
155 193086784 : return set_src_cost (rtxes->shift, mode, speed_p);
156 : }
157 :
158 : /* For each X in the range [0, BITS_PER_WORD), set SPLITTING[X]
159 : to true if it is profitable to split a double-word CODE shift
160 : of X + BITS_PER_WORD bits. SPEED_P says whether we are testing
161 : for speed or size profitability.
162 :
163 : Use the rtxes in RTXES to calculate costs. WORD_MOVE_ZERO_COST is
164 : the cost of moving zero into a word-mode register. WORD_MOVE_COST
165 : is the cost of moving between word registers. */
166 :
167 : static void
168 1321728 : compute_splitting_shift (bool speed_p, struct cost_rtxes *rtxes,
169 : bool *splitting, enum rtx_code code,
170 : int word_move_zero_cost, int word_move_cost)
171 : {
172 1321728 : int wide_cost, narrow_cost, upper_cost, i;
173 :
174 85946190 : for (i = 0; i < BITS_PER_WORD; i++)
175 : {
176 83506752 : wide_cost = shift_cost (speed_p, rtxes, code, twice_word_mode,
177 : i + BITS_PER_WORD);
178 83506752 : if (i == 0)
179 : narrow_cost = word_move_cost;
180 : else
181 82185024 : narrow_cost = shift_cost (speed_p, rtxes, code, word_mode, i);
182 :
183 83506752 : if (code != ASHIFTRT)
184 : upper_cost = word_move_zero_cost;
185 28196864 : else if (i == BITS_PER_WORD - 1)
186 : upper_cost = word_move_cost;
187 : else
188 27395008 : upper_cost = shift_cost (speed_p, rtxes, code, word_mode,
189 : BITS_PER_WORD - 1);
190 :
191 83506752 : if (LOG_COSTS)
192 : fprintf (stderr, "%s %s by %d: original cost %d, split cost %d + %d\n",
193 : GET_MODE_NAME (twice_word_mode), GET_RTX_NAME (code),
194 : i + BITS_PER_WORD, wide_cost, narrow_cost, upper_cost);
195 :
196 83506752 : if (FORCE_LOWERING || wide_cost >= narrow_cost + upper_cost)
197 83506752 : splitting[i] = true;
198 : }
199 1321728 : }
200 :
201 : /* Compute what we should do when optimizing for speed or size; SPEED_P
202 : selects which. Use RTXES for computing costs. */
203 :
204 : static void
205 440576 : compute_costs (bool speed_p, struct cost_rtxes *rtxes)
206 : {
207 440576 : unsigned int i;
208 440576 : int word_move_zero_cost, word_move_cost;
209 :
210 440576 : PUT_MODE (rtxes->target, word_mode);
211 440576 : SET_SRC (rtxes->set) = CONST0_RTX (word_mode);
212 440576 : word_move_zero_cost = set_rtx_cost (rtxes->set, speed_p);
213 :
214 440576 : SET_SRC (rtxes->set) = rtxes->source;
215 440576 : word_move_cost = set_rtx_cost (rtxes->set, speed_p);
216 :
217 440576 : if (LOG_COSTS)
218 : fprintf (stderr, "%s move: from zero cost %d, from reg cost %d\n",
219 : GET_MODE_NAME (word_mode), word_move_zero_cost, word_move_cost);
220 :
221 55512576 : for (i = 0; i < MAX_MACHINE_MODE; i++)
222 : {
223 54631424 : machine_mode mode = (machine_mode) i;
224 54631424 : unsigned int size, factor;
225 54631424 : if (interesting_mode_p (mode, &size, &factor) && factor > 1)
226 : {
227 27507642 : unsigned int mode_move_cost;
228 :
229 27507642 : PUT_MODE (rtxes->target, mode);
230 27507642 : PUT_MODE (rtxes->source, mode);
231 27507642 : mode_move_cost = set_rtx_cost (rtxes->set, speed_p);
232 :
233 27507642 : if (LOG_COSTS)
234 : fprintf (stderr, "%s move: original cost %d, split cost %d * %d\n",
235 : GET_MODE_NAME (mode), mode_move_cost,
236 : word_move_cost, factor);
237 :
238 27507642 : if (FORCE_LOWERING || mode_move_cost >= word_move_cost * factor)
239 : {
240 21004148 : choices[speed_p].move_modes_to_split[i] = true;
241 21004148 : choices[speed_p].something_to_do = true;
242 : }
243 : }
244 : }
245 :
246 : /* For the moves and shifts, the only case that is checked is one
247 : where the mode of the target is an integer mode twice the width
248 : of the word_mode.
249 :
250 : If it is not profitable to split a double word move then do not
251 : even consider the shifts or the zero extension. */
252 440576 : if (choices[speed_p].move_modes_to_split[(int) twice_word_mode])
253 : {
254 440576 : int zext_cost;
255 :
256 : /* The only case here to check to see if moving the upper part with a
257 : zero is cheaper than doing the zext itself. */
258 440576 : PUT_MODE (rtxes->source, word_mode);
259 440576 : zext_cost = set_src_cost (rtxes->zext, twice_word_mode, speed_p);
260 :
261 440576 : if (LOG_COSTS)
262 : fprintf (stderr, "%s %s: original cost %d, split cost %d + %d\n",
263 : GET_MODE_NAME (twice_word_mode), GET_RTX_NAME (ZERO_EXTEND),
264 : zext_cost, word_move_cost, word_move_zero_cost);
265 :
266 440576 : if (FORCE_LOWERING || zext_cost >= word_move_cost + word_move_zero_cost)
267 0 : choices[speed_p].splitting_zext = true;
268 :
269 440576 : compute_splitting_shift (speed_p, rtxes,
270 440576 : choices[speed_p].splitting_ashift, ASHIFT,
271 : word_move_zero_cost, word_move_cost);
272 440576 : compute_splitting_shift (speed_p, rtxes,
273 440576 : choices[speed_p].splitting_lshiftrt, LSHIFTRT,
274 : word_move_zero_cost, word_move_cost);
275 440576 : compute_splitting_shift (speed_p, rtxes,
276 440576 : choices[speed_p].splitting_ashiftrt, ASHIFTRT,
277 : word_move_zero_cost, word_move_cost);
278 : }
279 440576 : }
280 :
281 : /* Do one-per-target initialisation. This involves determining
282 : which operations on the machine are profitable. If none are found,
283 : then the pass just returns when called. */
284 :
285 : void
286 220288 : init_lower_subreg (void)
287 : {
288 220288 : struct cost_rtxes rtxes;
289 :
290 220288 : memset (this_target_lower_subreg, 0, sizeof (*this_target_lower_subreg));
291 :
292 220288 : twice_word_mode = GET_MODE_2XWIDER_MODE (word_mode).require ();
293 :
294 220288 : rtxes.target = gen_rtx_REG (word_mode, LAST_VIRTUAL_REGISTER + 1);
295 220288 : rtxes.source = gen_rtx_REG (word_mode, LAST_VIRTUAL_REGISTER + 2);
296 220288 : rtxes.set = gen_rtx_SET (rtxes.target, rtxes.source);
297 220288 : rtxes.zext = gen_rtx_ZERO_EXTEND (twice_word_mode, rtxes.source);
298 220288 : rtxes.shift = gen_rtx_ASHIFT (twice_word_mode, rtxes.source, const0_rtx);
299 :
300 220288 : if (LOG_COSTS)
301 : fprintf (stderr, "\nSize costs\n==========\n\n");
302 220288 : compute_costs (false, &rtxes);
303 :
304 220288 : if (LOG_COSTS)
305 : fprintf (stderr, "\nSpeed costs\n===========\n\n");
306 220288 : compute_costs (true, &rtxes);
307 220288 : }
308 :
309 : static bool
310 84342245 : simple_move_operand (rtx x)
311 : {
312 84342245 : if (GET_CODE (x) == SUBREG)
313 : {
314 : /* Exclude subregs whose outer mode can be split into multiple words
315 : but whose inner mode cannot. Attempting to split such a subreg
316 : would mean trying to split the unsplittable inner register.
317 :
318 : If instead the subreg occupies a single word, we can keep it as-is,
319 : regardless of what the SUBREG_REG is. If the outer mode cannot be
320 : split then the subreg makes things no worse than they already are. */
321 3507229 : unsigned int factor, size;
322 7014458 : if (interesting_mode_p (GET_MODE (x), &size, &factor) && factor > 1
323 4234048 : && !interesting_mode_p (GET_MODE (SUBREG_REG (x)), &size, &factor))
324 0 : return false;
325 3507229 : x = SUBREG_REG (x);
326 : }
327 :
328 84342245 : if (!OBJECT_P (x))
329 : return false;
330 :
331 83721147 : if (GET_CODE (x) == LABEL_REF
332 83721147 : || GET_CODE (x) == SYMBOL_REF
333 80462146 : || GET_CODE (x) == HIGH
334 80462146 : || GET_CODE (x) == CONST)
335 : return false;
336 :
337 80336695 : if (MEM_P (x)
338 80336695 : && (MEM_VOLATILE_P (x)
339 21969967 : || mode_dependent_address_p (XEXP (x, 0), MEM_ADDR_SPACE (x))))
340 3765006 : return false;
341 :
342 : return true;
343 : }
344 :
345 : /* If X is an operator that can be treated as a simple move that we
346 : can split, then return the operand that is operated on. */
347 :
348 : static rtx
349 45352935 : operand_for_swap_move_operator (rtx x)
350 : {
351 : /* A word sized rotate of a register pair is equivalent to swapping
352 : the registers in the register pair. */
353 45352935 : if (GET_CODE (x) == ROTATE
354 10 : && GET_MODE (x) == twice_word_mode
355 10 : && simple_move_operand (XEXP (x, 0))
356 10 : && CONST_INT_P (XEXP (x, 1))
357 45352945 : && INTVAL (XEXP (x, 1)) == BITS_PER_WORD)
358 10 : return XEXP (x, 0);
359 :
360 : return NULL_RTX;
361 : }
362 :
363 : /* If INSN is a single set between two objects that we want to split,
364 : return the single set. SPEED_P says whether we are optimizing
365 : INSN for speed or size.
366 :
367 : INSN should have been passed to recog and extract_insn before this
368 : is called. */
369 :
370 : static rtx
371 119217710 : simple_move (rtx_insn *insn, bool speed_p)
372 : {
373 119217710 : rtx x, op;
374 119217710 : rtx set;
375 119217710 : machine_mode mode;
376 :
377 119217710 : if (recog_data.n_operands != 2)
378 : return NULL_RTX;
379 :
380 58087705 : set = single_set (insn);
381 58087705 : if (!set)
382 : return NULL_RTX;
383 :
384 55204566 : x = SET_DEST (set);
385 55204566 : if (x != recog_data.operand[0] && x != recog_data.operand[1])
386 : return NULL_RTX;
387 44573458 : if (!simple_move_operand (x))
388 : return NULL_RTX;
389 :
390 40936851 : x = SET_SRC (set);
391 40936851 : if ((op = operand_for_swap_move_operator (x)) != NULL_RTX)
392 5 : x = op;
393 :
394 40936851 : if (x != recog_data.operand[0] && x != recog_data.operand[1])
395 : return NULL_RTX;
396 : /* For the src we can handle ASM_OPERANDS, and it is beneficial for
397 : things like x86 rdtsc which returns a DImode value. */
398 39768777 : if (GET_CODE (x) != ASM_OPERANDS
399 39768777 : && !simple_move_operand (x))
400 : return NULL_RTX;
401 :
402 : /* We try to decompose in integer modes, to avoid generating
403 : inefficient code copying between integer and floating point
404 : registers. That means that we can't decompose if this is a
405 : non-integer mode for which there is no integer mode of the same
406 : size. */
407 35634828 : mode = GET_MODE (SET_DEST (set));
408 35634828 : scalar_int_mode int_mode;
409 35634828 : if (!SCALAR_INT_MODE_P (mode)
410 40219720 : && (!int_mode_for_size (GET_MODE_BITSIZE (mode), 0).exists (&int_mode)
411 4035225 : || !targetm.modes_tieable_p (mode, int_mode)))
412 : return NULL_RTX;
413 :
414 : /* Reject PARTIAL_INT modes. They are used for processor specific
415 : purposes and it's probably best not to tamper with them. */
416 35085161 : if (GET_MODE_CLASS (mode) == MODE_PARTIAL_INT)
417 : return NULL_RTX;
418 :
419 35085161 : if (!choices[speed_p].move_modes_to_split[(int) mode])
420 30669209 : return NULL_RTX;
421 :
422 : return set;
423 : }
424 :
425 : /* If SET is a copy from one multi-word pseudo-register to another,
426 : record that in reg_copy_graph. Return whether it is such a
427 : copy. */
428 :
429 : static bool
430 2833879 : find_pseudo_copy (rtx set)
431 : {
432 2833879 : rtx dest = SET_DEST (set);
433 2833879 : rtx src = SET_SRC (set);
434 2833879 : rtx op;
435 2833879 : unsigned int rd, rs;
436 2833879 : bitmap b;
437 :
438 2833879 : if ((op = operand_for_swap_move_operator (src)) != NULL_RTX)
439 4 : src = op;
440 :
441 2833879 : if (!REG_P (dest) || !REG_P (src))
442 : return false;
443 :
444 453438 : rd = REGNO (dest);
445 453438 : rs = REGNO (src);
446 453438 : if (HARD_REGISTER_NUM_P (rd) || HARD_REGISTER_NUM_P (rs))
447 : return false;
448 :
449 141582 : b = reg_copy_graph[rs];
450 141582 : if (b == NULL)
451 : {
452 138946 : b = BITMAP_ALLOC (NULL);
453 138946 : reg_copy_graph[rs] = b;
454 : }
455 :
456 141582 : bitmap_set_bit (b, rd);
457 :
458 141582 : return true;
459 : }
460 :
461 : /* Look through the registers in DECOMPOSABLE_CONTEXT. For each case
462 : where they are copied to another register, add the register to
463 : which they are copied to DECOMPOSABLE_CONTEXT. Use
464 : NON_DECOMPOSABLE_CONTEXT to limit this--we don't bother to track
465 : copies of registers which are in NON_DECOMPOSABLE_CONTEXT. */
466 :
467 : static void
468 110135 : propagate_pseudo_copies (void)
469 : {
470 110135 : auto_bitmap queue, propagate;
471 :
472 110135 : bitmap_copy (queue, decomposable_context);
473 113437 : do
474 : {
475 113437 : bitmap_iterator iter;
476 113437 : unsigned int i;
477 :
478 113437 : bitmap_clear (propagate);
479 :
480 423916 : EXECUTE_IF_SET_IN_BITMAP (queue, 0, i, iter)
481 : {
482 310479 : bitmap b = reg_copy_graph[i];
483 310479 : if (b)
484 8085 : bitmap_ior_and_compl_into (propagate, b, non_decomposable_context);
485 : }
486 :
487 113437 : bitmap_and_compl (queue, propagate, decomposable_context);
488 113437 : bitmap_ior_into (decomposable_context, propagate);
489 : }
490 113437 : while (!bitmap_empty_p (queue));
491 110135 : }
492 :
493 : /* A pointer to one of these values is passed to
494 : find_decomposable_subregs. */
495 :
496 : enum classify_move_insn
497 : {
498 : /* Not a simple move from one location to another. */
499 : NOT_SIMPLE_MOVE,
500 : /* A simple move we want to decompose. */
501 : DECOMPOSABLE_SIMPLE_MOVE,
502 : /* Any other simple move. */
503 : SIMPLE_MOVE
504 : };
505 :
506 : /* If we find a SUBREG in *LOC which we could use to decompose a
507 : pseudo-register, set a bit in DECOMPOSABLE_CONTEXT. If we find an
508 : unadorned register which is not a simple pseudo-register copy,
509 : DATA will point at the type of move, and we set a bit in
510 : DECOMPOSABLE_CONTEXT or NON_DECOMPOSABLE_CONTEXT as appropriate. */
511 :
512 : static void
513 129937842 : find_decomposable_subregs (rtx *loc, enum classify_move_insn *pcmi)
514 : {
515 129937842 : subrtx_var_iterator::array_type array;
516 294271106 : FOR_EACH_SUBRTX_VAR (iter, array, *loc, NONCONST)
517 : {
518 164333264 : rtx x = *iter;
519 164333264 : if (GET_CODE (x) == SUBREG)
520 : {
521 3125602 : rtx inner = SUBREG_REG (x);
522 3125602 : unsigned int regno, outer_size, inner_size, outer_words, inner_words;
523 :
524 3125602 : if (!REG_P (inner))
525 1836486 : continue;
526 :
527 3125599 : regno = REGNO (inner);
528 3125599 : if (HARD_REGISTER_NUM_P (regno))
529 : {
530 2 : iter.skip_subrtxes ();
531 2 : continue;
532 : }
533 :
534 3125597 : if (!interesting_mode_p (GET_MODE (x), &outer_size, &outer_words)
535 3125597 : || !interesting_mode_p (GET_MODE (inner), &inner_size,
536 : &inner_words))
537 0 : continue;
538 :
539 : /* We only try to decompose single word subregs of multi-word
540 : registers. When we find one, we return -1 to avoid iterating
541 : over the inner register.
542 :
543 : ??? This doesn't allow, e.g., DImode subregs of TImode values
544 : on 32-bit targets. We would need to record the way the
545 : pseudo-register was used, and only decompose if all the uses
546 : were the same number and size of pieces. Hopefully this
547 : doesn't happen much. */
548 :
549 3125597 : if (outer_words == 1
550 2234249 : && inner_words > 1
551 : /* Don't allow to decompose floating point subregs of
552 : multi-word pseudos if the floating point mode does
553 : not have word size, because otherwise we'd generate
554 : a subreg with that floating mode from a different
555 : sized integral pseudo which is not allowed by
556 : validate_subreg. */
557 1834729 : && (!FLOAT_MODE_P (GET_MODE (x))
558 7788 : || outer_size == UNITS_PER_WORD))
559 : {
560 1834456 : bitmap_set_bit (decomposable_context, regno);
561 1834456 : iter.skip_subrtxes ();
562 1834456 : continue;
563 : }
564 :
565 : /* If this is a cast from one mode to another, where the modes
566 : have the same size, and they are not tieable, then mark this
567 : register as non-decomposable. If we decompose it we are
568 : likely to mess up whatever the backend is trying to do. */
569 1293166 : if (outer_words > 1
570 891348 : && outer_size == inner_size
571 1503379 : && !targetm.modes_tieable_p (GET_MODE (x), GET_MODE (inner)))
572 : {
573 2025 : bitmap_set_bit (non_decomposable_context, regno);
574 2025 : bitmap_set_bit (subreg_context, regno);
575 2025 : iter.skip_subrtxes ();
576 2025 : continue;
577 : }
578 : }
579 161207662 : else if (REG_P (x))
580 : {
581 81857412 : unsigned int regno, size, words;
582 :
583 : /* We will see an outer SUBREG before we see the inner REG, so
584 : when we see a plain REG here it means a direct reference to
585 : the register.
586 :
587 : If this is not a simple copy from one location to another,
588 : then we cannot decompose this register. If this is a simple
589 : copy we want to decompose, and the mode is right,
590 : then we mark the register as decomposable.
591 : Otherwise we don't say anything about this register --
592 : it could be decomposed, but whether that would be
593 : profitable depends upon how it is used elsewhere.
594 :
595 : We only set bits in the bitmap for multi-word
596 : pseudo-registers, since those are the only ones we care about
597 : and it keeps the size of the bitmaps down. */
598 :
599 81857412 : regno = REGNO (x);
600 81857412 : if (!HARD_REGISTER_NUM_P (regno)
601 52002923 : && interesting_mode_p (GET_MODE (x), &size, &words)
602 133860335 : && words > 1)
603 : {
604 8836979 : switch (*pcmi)
605 : {
606 5909914 : case NOT_SIMPLE_MOVE:
607 5909914 : bitmap_set_bit (non_decomposable_context, regno);
608 5909914 : break;
609 18946 : case DECOMPOSABLE_SIMPLE_MOVE:
610 18946 : if (targetm.modes_tieable_p (GET_MODE (x), word_mode))
611 0 : bitmap_set_bit (decomposable_context, regno);
612 : break;
613 : case SIMPLE_MOVE:
614 : break;
615 0 : default:
616 0 : gcc_unreachable ();
617 : }
618 : }
619 : }
620 79350250 : else if (MEM_P (x))
621 : {
622 16851154 : enum classify_move_insn cmi_mem = NOT_SIMPLE_MOVE;
623 :
624 : /* Any registers used in a MEM do not participate in a
625 : SIMPLE_MOVE or DECOMPOSABLE_SIMPLE_MOVE. Do our own recursion
626 : here, and return -1 to block the parent's recursion. */
627 16851154 : find_decomposable_subregs (&XEXP (x, 0), &cmi_mem);
628 16851154 : iter.skip_subrtxes ();
629 : }
630 : }
631 129937842 : }
632 :
633 : /* Decompose REGNO into word-sized components. We smash the REG node
634 : in place. This ensures that (1) something goes wrong quickly if we
635 : fail to make some replacement, and (2) the debug information inside
636 : the symbol table is automatically kept up to date. */
637 :
638 : static void
639 310479 : decompose_register (unsigned int regno)
640 : {
641 310479 : rtx reg;
642 310479 : unsigned int size, words, i;
643 310479 : rtvec v;
644 :
645 310479 : reg = regno_reg_rtx[regno];
646 :
647 310479 : regno_reg_rtx[regno] = NULL_RTX;
648 :
649 310479 : if (!interesting_mode_p (GET_MODE (reg), &size, &words))
650 0 : gcc_unreachable ();
651 :
652 310479 : v = rtvec_alloc (words);
653 1257000 : for (i = 0; i < words; ++i)
654 817304 : RTVEC_ELT (v, i) = gen_reg_rtx_offset (reg, word_mode, i * UNITS_PER_WORD);
655 :
656 310479 : PUT_CODE (reg, CONCATN);
657 310479 : DECOMPOSED_REGISTER_P (reg) = true;
658 310479 : XVEC (reg, 0) = v;
659 :
660 310479 : if (dump_file)
661 : {
662 0 : fprintf (dump_file, "; Splitting reg %u ->", regno);
663 0 : for (i = 0; i < words; ++i)
664 0 : fprintf (dump_file, " %u", REGNO (XVECEXP (reg, 0, i)));
665 0 : fputc ('\n', dump_file);
666 : }
667 310479 : }
668 :
669 : /* Return whether we should resolve X into the registers into which it
670 : was decomposed. */
671 :
672 : static bool
673 127383709 : resolve_reg_p (rtx x)
674 : {
675 6439193 : return GET_CODE (x) == CONCATN && DECOMPOSED_REGISTER_P (x);
676 : }
677 :
678 : /* Return whether X is a SUBREG of a register which we need to
679 : resolve. */
680 :
681 : static bool
682 119177948 : resolve_subreg_p (rtx x)
683 : {
684 119177948 : if (GET_CODE (x) != SUBREG)
685 : return false;
686 2326006 : return resolve_reg_p (SUBREG_REG (x));
687 : }
688 :
689 : /* Get a SUBREG of a CONCATN. */
690 :
691 : static rtx
692 1592416 : simplify_subreg_concatn (machine_mode outermode, rtx op, poly_uint64 orig_byte)
693 : {
694 1592416 : unsigned int outer_size, inner_size, inner_words;
695 1592416 : machine_mode innermode, partmode;
696 1592416 : rtx part;
697 1592416 : unsigned int final_offset;
698 1592416 : unsigned int byte;
699 :
700 1592416 : gcc_assert (resolve_reg_p (op));
701 :
702 1592416 : innermode = GET_MODE (op);
703 :
704 1592416 : if (!interesting_mode_p (innermode, &inner_size, &inner_words))
705 0 : gcc_unreachable ();
706 :
707 3184832 : if (!GET_MODE_SIZE (outermode).is_constant (&outer_size))
708 : return NULL_RTX;
709 :
710 : /* Must be constant if outer_size is. */
711 1592416 : byte = orig_byte.to_constant ();
712 1592416 : gcc_assert (byte % outer_size == 0);
713 :
714 1592416 : gcc_assert (byte < inner_size);
715 1592416 : if (outer_size > inner_size)
716 : return NULL_RTX;
717 :
718 1592416 : inner_size /= XVECLEN (op, 0);
719 1592416 : part = XVECEXP (op, 0, byte / inner_size);
720 1592416 : partmode = GET_MODE (part);
721 :
722 1592416 : final_offset = byte % inner_size;
723 1592416 : if (final_offset + outer_size > inner_size)
724 : return NULL_RTX;
725 :
726 : /* VECTOR_CSTs in debug expressions are expanded into CONCATN instead of
727 : regular CONST_VECTORs. They have vector or integer modes, depending
728 : on the capabilities of the target. Cope with them. */
729 1592081 : if (partmode == VOIDmode && VECTOR_MODE_P (innermode))
730 0 : partmode = GET_MODE_INNER (innermode);
731 0 : else if (partmode == VOIDmode)
732 0 : partmode = mode_for_size (inner_size * BITS_PER_UNIT,
733 0 : GET_MODE_CLASS (innermode), 0).require ();
734 :
735 1592081 : return simplify_gen_subreg (outermode, part, partmode, final_offset);
736 : }
737 :
738 : /* Wrapper around simplify_gen_subreg which handles CONCATN. */
739 :
740 : static rtx
741 1247019 : simplify_gen_subreg_concatn (machine_mode outermode, rtx op,
742 : machine_mode innermode, unsigned int byte)
743 : {
744 1247335 : rtx ret;
745 :
746 : /* We have to handle generating a SUBREG of a SUBREG of a CONCATN.
747 : If OP is a SUBREG of a CONCATN, then it must be a simple mode
748 : change with the same size and offset 0, or it must extract a
749 : part. We shouldn't see anything else here. */
750 1247335 : if (resolve_subreg_p (op))
751 : {
752 546 : rtx op2;
753 :
754 1092 : if (known_eq (GET_MODE_SIZE (GET_MODE (op)),
755 : GET_MODE_SIZE (GET_MODE (SUBREG_REG (op))))
756 546 : && known_eq (SUBREG_BYTE (op), 0))
757 : return simplify_gen_subreg_concatn (outermode, SUBREG_REG (op),
758 : GET_MODE (SUBREG_REG (op)), byte);
759 :
760 460 : op2 = simplify_subreg_concatn (GET_MODE (op), SUBREG_REG (op),
761 230 : SUBREG_BYTE (op));
762 230 : if (op2 == NULL_RTX)
763 : {
764 : /* We don't handle paradoxical subregs here. */
765 230 : gcc_assert (!paradoxical_subreg_p (outermode, GET_MODE (op)));
766 230 : gcc_assert (!paradoxical_subreg_p (op));
767 460 : op2 = simplify_subreg_concatn (outermode, SUBREG_REG (op),
768 230 : byte + SUBREG_BYTE (op));
769 230 : gcc_assert (op2 != NULL_RTX);
770 : return op2;
771 : }
772 :
773 0 : op = op2;
774 0 : gcc_assert (op != NULL_RTX);
775 0 : gcc_assert (innermode == GET_MODE (op));
776 : }
777 :
778 1246789 : if (resolve_reg_p (op))
779 706383 : return simplify_subreg_concatn (outermode, op, byte);
780 :
781 540406 : ret = simplify_gen_subreg (outermode, op, innermode, byte);
782 :
783 : /* If we see an insn like (set (reg:DI) (subreg:DI (reg:SI) 0)) then
784 : resolve_simple_move will ask for the high part of the paradoxical
785 : subreg, which does not have a value. Just return a zero. */
786 540406 : if (ret == NULL_RTX
787 540413 : && paradoxical_subreg_p (op))
788 7 : return CONST0_RTX (outermode);
789 :
790 540399 : gcc_assert (ret != NULL_RTX);
791 : return ret;
792 : }
793 :
794 : /* Look for SUBREGs in *LOC which need to be decomposed. */
795 :
796 : static bool
797 53363390 : resolve_subreg_use (rtx *loc, rtx insn)
798 : {
799 53363390 : subrtx_ptr_iterator::array_type array;
800 131491870 : FOR_EACH_SUBRTX_PTR (iter, array, loc, NONCONST)
801 : {
802 78130351 : rtx *loc = *iter;
803 78130351 : rtx x = *loc;
804 78130351 : if (resolve_subreg_p (x))
805 : {
806 1613984 : x = simplify_subreg_concatn (GET_MODE (x), SUBREG_REG (x),
807 806992 : SUBREG_BYTE (x));
808 :
809 : /* It is possible for a note to contain a reference which we can
810 : decompose. In this case, return 1 to the caller to indicate
811 : that the note must be removed. */
812 806992 : if (!x)
813 : {
814 3 : gcc_assert (!insn);
815 1871 : return true;
816 : }
817 :
818 806989 : validate_change (insn, loc, x, 1);
819 806989 : iter.skip_subrtxes ();
820 : }
821 78132219 : else if (resolve_reg_p (x))
822 : /* Return 1 to the caller to indicate that we found a direct
823 : reference to a register which is being decomposed. This can
824 : happen inside notes, multiword shift or zero-extend
825 : instructions. */
826 : return true;
827 : }
828 :
829 53361519 : return false;
830 53363390 : }
831 :
832 : /* Resolve any decomposed registers which appear in register notes on
833 : INSN. */
834 :
835 : static void
836 39101900 : resolve_reg_notes (rtx_insn *insn)
837 : {
838 39101900 : rtx *pnote, note;
839 :
840 39101900 : note = find_reg_equal_equiv_note (insn);
841 39101900 : if (note)
842 : {
843 547022 : int old_count = num_validated_changes ();
844 547022 : if (resolve_subreg_use (&XEXP (note, 0), NULL_RTX))
845 1871 : remove_note (insn, note);
846 : else
847 545151 : if (old_count != num_validated_changes ())
848 2673 : df_notes_rescan (insn);
849 : }
850 :
851 39101900 : pnote = ®_NOTES (insn);
852 50947624 : while (*pnote != NULL_RTX)
853 : {
854 11845724 : bool del = false;
855 :
856 11845724 : note = *pnote;
857 11845724 : switch (REG_NOTE_KIND (note))
858 : {
859 4001176 : case REG_DEAD:
860 4001176 : case REG_UNUSED:
861 4001176 : if (resolve_reg_p (XEXP (note, 0)))
862 8279 : del = true;
863 : break;
864 :
865 : default:
866 : break;
867 : }
868 :
869 8279 : if (del)
870 8279 : *pnote = XEXP (note, 1);
871 : else
872 11837445 : pnote = &XEXP (note, 1);
873 : }
874 39101900 : }
875 :
876 : /* Return whether X can be decomposed into subwords. */
877 :
878 : static bool
879 510486 : can_decompose_p (rtx x)
880 : {
881 510486 : if (REG_P (x))
882 : {
883 148575 : unsigned int regno = REGNO (x);
884 :
885 148575 : if (HARD_REGISTER_NUM_P (regno))
886 : {
887 100637 : unsigned int byte, num_bytes, num_words;
888 :
889 100637 : if (!interesting_mode_p (GET_MODE (x), &num_bytes, &num_words))
890 : return false;
891 310449 : for (byte = 0; byte < num_bytes; byte += UNITS_PER_WORD)
892 201274 : if (simplify_subreg_regno (regno, GET_MODE (x), byte, word_mode) < 0)
893 : return false;
894 : return true;
895 : }
896 : else
897 47938 : return !bitmap_bit_p (subreg_context, regno);
898 : }
899 :
900 : return true;
901 : }
902 :
903 : /* OPND is a concatn operand this is used with a simple move operator.
904 : Return a new rtx with the concatn's operands swapped. */
905 :
906 : static rtx
907 1 : resolve_operand_for_swap_move_operator (rtx opnd)
908 : {
909 1 : gcc_assert (resolve_reg_p (opnd));
910 1 : rtx concatn = copy_rtx (opnd);
911 1 : rtx op0 = XVECEXP (concatn, 0, 0);
912 1 : rtx op1 = XVECEXP (concatn, 0, 1);
913 1 : XVECEXP (concatn, 0, 0) = op1;
914 1 : XVECEXP (concatn, 0, 1) = op0;
915 1 : return concatn;
916 : }
917 :
918 : /* Decompose the registers used in a simple move SET within INSN. If
919 : we don't change anything, return INSN, otherwise return the start
920 : of the sequence of moves. */
921 :
922 : static rtx_insn *
923 1582205 : resolve_simple_move (rtx set, rtx_insn *insn)
924 : {
925 1582205 : rtx src, dest, real_dest, src_op;
926 1582205 : rtx_insn *insns;
927 1582205 : machine_mode orig_mode, dest_mode;
928 1582205 : unsigned int orig_size, words;
929 1582205 : bool pushing;
930 :
931 1582205 : src = SET_SRC (set);
932 1582205 : dest = SET_DEST (set);
933 1582205 : orig_mode = GET_MODE (dest);
934 :
935 1582205 : if (!interesting_mode_p (orig_mode, &orig_size, &words))
936 0 : gcc_unreachable ();
937 1582205 : gcc_assert (words > 1);
938 :
939 1582205 : start_sequence ();
940 :
941 : /* We have to handle copying from a SUBREG of a decomposed reg where
942 : the SUBREG is larger than word size. Rather than assume that we
943 : can take a word_mode SUBREG of the destination, we copy to a new
944 : register and then copy that to the destination. */
945 :
946 1582205 : real_dest = NULL_RTX;
947 :
948 1582205 : if ((src_op = operand_for_swap_move_operator (src)) != NULL_RTX)
949 : {
950 1 : if (resolve_reg_p (dest))
951 : {
952 : /* DEST is a CONCATN, so swap its operands and strip
953 : SRC's operator. */
954 1 : dest = resolve_operand_for_swap_move_operator (dest);
955 1 : src = src_op;
956 1 : if (resolve_reg_p (src))
957 : {
958 1 : gcc_assert (GET_CODE (src) == CONCATN);
959 1 : if (reg_overlap_mentioned_p (XVECEXP (dest, 0, 0),
960 1 : XVECEXP (src, 0, 1)))
961 : {
962 : /* If there is overlap between the first half of the
963 : destination and what will be stored to the second one,
964 : use a temporary pseudo. See PR114211. */
965 1 : rtx tem = gen_reg_rtx (GET_MODE (XVECEXP (src, 0, 1)));
966 1 : emit_move_insn (tem, XVECEXP (src, 0, 1));
967 1 : src = copy_rtx (src);
968 1 : XVECEXP (src, 0, 1) = tem;
969 : }
970 : }
971 : }
972 0 : else if (resolve_reg_p (src_op))
973 : {
974 : /* SRC is an operation on a CONCATN, so strip the operator and
975 : swap the CONCATN's operands. */
976 0 : src = resolve_operand_for_swap_move_operator (src_op);
977 : }
978 : }
979 :
980 1582205 : if (GET_CODE (src) == SUBREG
981 9174 : && resolve_reg_p (SUBREG_REG (src))
982 1582299 : && (maybe_ne (SUBREG_BYTE (src), 0)
983 186 : || maybe_ne (orig_size, GET_MODE_SIZE (GET_MODE (SUBREG_REG (src))))))
984 : {
985 4 : real_dest = dest;
986 4 : dest = gen_reg_rtx (orig_mode);
987 4 : if (REG_P (real_dest))
988 4 : REG_ATTRS (dest) = REG_ATTRS (real_dest);
989 : }
990 :
991 : /* Similarly if we are copying to a SUBREG of a decomposed reg where
992 : the SUBREG is larger than word size. */
993 :
994 1582205 : if (GET_CODE (dest) == SUBREG
995 1112 : && resolve_reg_p (SUBREG_REG (dest))
996 1582383 : && (maybe_ne (SUBREG_BYTE (dest), 0)
997 178 : || maybe_ne (orig_size,
998 356 : GET_MODE_SIZE (GET_MODE (SUBREG_REG (dest))))))
999 : {
1000 111 : rtx reg, smove;
1001 111 : rtx_insn *minsn;
1002 :
1003 111 : reg = gen_reg_rtx (orig_mode);
1004 111 : minsn = emit_move_insn (reg, src);
1005 111 : smove = single_set (minsn);
1006 111 : gcc_assert (smove != NULL_RTX);
1007 111 : resolve_simple_move (smove, minsn);
1008 111 : src = reg;
1009 : }
1010 :
1011 : /* If we didn't have any big SUBREGS of decomposed registers, and
1012 : neither side of the move is a register we are decomposing, then
1013 : we don't have to do anything here. */
1014 :
1015 1582205 : if (src == SET_SRC (set)
1016 1582093 : && dest == SET_DEST (set)
1017 1788050 : && !resolve_reg_p (src)
1018 1532807 : && !resolve_subreg_p (src)
1019 1582267 : && !resolve_reg_p (dest)
1020 2909229 : && !resolve_subreg_p (dest))
1021 : {
1022 1326962 : end_sequence ();
1023 1326962 : return insn;
1024 : }
1025 :
1026 : /* It's possible for the code to use a subreg of a decomposed
1027 : register while forming an address. We need to handle that before
1028 : passing the address to emit_move_insn. We pass NULL_RTX as the
1029 : insn parameter to resolve_subreg_use because we cannot validate
1030 : the insn yet. */
1031 255243 : if (MEM_P (src) || MEM_P (dest))
1032 : {
1033 70043 : int acg;
1034 :
1035 70043 : if (MEM_P (src))
1036 52309 : resolve_subreg_use (&XEXP (src, 0), NULL_RTX);
1037 70043 : if (MEM_P (dest))
1038 17734 : resolve_subreg_use (&XEXP (dest, 0), NULL_RTX);
1039 70043 : acg = apply_change_group ();
1040 70043 : gcc_assert (acg);
1041 : }
1042 :
1043 : /* If SRC is a register which we can't decompose, or has side
1044 : effects, we need to move via a temporary register. */
1045 :
1046 255243 : if (!can_decompose_p (src)
1047 255080 : || side_effects_p (src)
1048 510323 : || GET_CODE (src) == ASM_OPERANDS)
1049 : {
1050 163 : rtx reg;
1051 :
1052 163 : reg = gen_reg_rtx (orig_mode);
1053 :
1054 163 : if (AUTO_INC_DEC)
1055 : {
1056 : rtx_insn *move = emit_move_insn (reg, src);
1057 : if (MEM_P (src))
1058 : {
1059 : rtx note = find_reg_note (insn, REG_INC, NULL_RTX);
1060 : if (note)
1061 : add_reg_note (move, REG_INC, XEXP (note, 0));
1062 : }
1063 : }
1064 : else
1065 163 : emit_move_insn (reg, src);
1066 :
1067 163 : src = reg;
1068 : }
1069 :
1070 : /* If DEST is a register which we can't decompose, or has side
1071 : effects, we need to first move to a temporary register. We
1072 : handle the common case of pushing an operand directly. We also
1073 : go through a temporary register if it holds a floating point
1074 : value. This gives us better code on systems which can't move
1075 : data easily between integer and floating point registers. */
1076 :
1077 255243 : dest_mode = orig_mode;
1078 255243 : pushing = push_operand (dest, dest_mode);
1079 255243 : if (!can_decompose_p (dest)
1080 255226 : || (side_effects_p (dest) && !pushing)
1081 510469 : || (!SCALAR_INT_MODE_P (dest_mode)
1082 255243 : && !resolve_reg_p (dest)
1083 0 : && !resolve_subreg_p (dest)))
1084 : {
1085 17 : if (real_dest == NULL_RTX)
1086 17 : real_dest = dest;
1087 17 : if (!SCALAR_INT_MODE_P (dest_mode))
1088 0 : dest_mode = int_mode_for_mode (dest_mode).require ();
1089 17 : dest = gen_reg_rtx (dest_mode);
1090 17 : if (REG_P (real_dest))
1091 17 : REG_ATTRS (dest) = REG_ATTRS (real_dest);
1092 : }
1093 :
1094 255243 : if (pushing)
1095 : {
1096 0 : unsigned int i, j, jinc;
1097 :
1098 0 : gcc_assert (orig_size % UNITS_PER_WORD == 0);
1099 0 : gcc_assert (GET_CODE (XEXP (dest, 0)) != PRE_MODIFY);
1100 0 : gcc_assert (GET_CODE (XEXP (dest, 0)) != POST_MODIFY);
1101 :
1102 0 : if (WORDS_BIG_ENDIAN == STACK_GROWS_DOWNWARD)
1103 : {
1104 : j = 0;
1105 : jinc = 1;
1106 : }
1107 : else
1108 : {
1109 0 : j = words - 1;
1110 0 : jinc = -1;
1111 : }
1112 :
1113 0 : for (i = 0; i < words; ++i, j += jinc)
1114 : {
1115 0 : rtx temp;
1116 :
1117 0 : temp = copy_rtx (XEXP (dest, 0));
1118 0 : temp = adjust_automodify_address_nv (dest, word_mode, temp,
1119 : j * UNITS_PER_WORD);
1120 0 : emit_move_insn (temp,
1121 : simplify_gen_subreg_concatn (word_mode, src,
1122 : orig_mode,
1123 0 : j * UNITS_PER_WORD));
1124 : }
1125 : }
1126 : else
1127 : {
1128 255243 : unsigned int i;
1129 :
1130 255243 : if (REG_P (dest) && !HARD_REGISTER_NUM_P (REGNO (dest)))
1131 3514 : emit_clobber (dest);
1132 :
1133 789229 : for (i = 0; i < words; ++i)
1134 : {
1135 533986 : rtx t = simplify_gen_subreg_concatn (word_mode, dest,
1136 : dest_mode,
1137 533986 : i * UNITS_PER_WORD);
1138 : /* simplify_gen_subreg_concatn can return (const_int 0) for
1139 : some sub-objects of paradoxical subregs. As a source operand,
1140 : that's fine. As a destination it must be avoided. Those are
1141 : supposed to be don't care bits, so we can just drop that store
1142 : on the floor. */
1143 533986 : if (t != CONST0_RTX (word_mode))
1144 533986 : emit_move_insn (t,
1145 : simplify_gen_subreg_concatn (word_mode, src,
1146 : orig_mode,
1147 533986 : i * UNITS_PER_WORD));
1148 : }
1149 : }
1150 :
1151 255243 : if (real_dest != NULL_RTX)
1152 : {
1153 21 : rtx mdest, smove;
1154 21 : rtx_insn *minsn;
1155 :
1156 21 : if (dest_mode == orig_mode)
1157 : mdest = dest;
1158 : else
1159 0 : mdest = simplify_gen_subreg (orig_mode, dest, GET_MODE (dest), 0);
1160 21 : minsn = emit_move_insn (real_dest, mdest);
1161 :
1162 21 : if (AUTO_INC_DEC && MEM_P (real_dest)
1163 : && !(resolve_reg_p (real_dest) || resolve_subreg_p (real_dest)))
1164 : {
1165 : rtx note = find_reg_note (insn, REG_INC, NULL_RTX);
1166 : if (note)
1167 : add_reg_note (minsn, REG_INC, XEXP (note, 0));
1168 : }
1169 :
1170 21 : smove = single_set (minsn);
1171 21 : gcc_assert (smove != NULL_RTX);
1172 :
1173 21 : resolve_simple_move (smove, minsn);
1174 : }
1175 :
1176 255243 : insns = end_sequence ();
1177 :
1178 255243 : copy_reg_eh_region_note_forward (insn, insns, NULL_RTX);
1179 :
1180 255243 : emit_insn_before (insns, insn);
1181 :
1182 : /* If we get here via self-recursion, then INSN is not yet in the insns
1183 : chain and delete_insn will fail. We only want to remove INSN from the
1184 : current sequence. See PR56738. */
1185 255243 : if (in_sequence_p ())
1186 6 : remove_insn (insn);
1187 : else
1188 255237 : delete_insn (insn);
1189 :
1190 : return insns;
1191 : }
1192 :
1193 : /* Change a CLOBBER of a decomposed register into a CLOBBER of the
1194 : component registers. Return whether we changed something. */
1195 :
1196 : static bool
1197 166136 : resolve_clobber (rtx pat, rtx_insn *insn)
1198 : {
1199 166136 : rtx reg;
1200 166136 : machine_mode orig_mode;
1201 166136 : unsigned int orig_size, words, i;
1202 166136 : int ret;
1203 :
1204 166136 : reg = XEXP (pat, 0);
1205 : /* For clobbers we can look through paradoxical subregs which
1206 : we do not handle in simplify_gen_subreg_concatn. */
1207 166136 : if (paradoxical_subreg_p (reg))
1208 166136 : reg = SUBREG_REG (reg);
1209 166136 : if (!resolve_reg_p (reg) && !resolve_subreg_p (reg))
1210 : return false;
1211 :
1212 62090 : orig_mode = GET_MODE (reg);
1213 62090 : if (!interesting_mode_p (orig_mode, &orig_size, &words))
1214 0 : gcc_unreachable ();
1215 :
1216 62090 : ret = validate_change (NULL_RTX, &XEXP (pat, 0),
1217 : simplify_gen_subreg_concatn (word_mode, reg,
1218 : orig_mode, 0),
1219 : 0);
1220 62090 : df_insn_rescan (insn);
1221 62090 : gcc_assert (ret != 0);
1222 :
1223 124180 : for (i = words - 1; i > 0; --i)
1224 : {
1225 62090 : rtx x;
1226 :
1227 62090 : x = simplify_gen_subreg_concatn (word_mode, reg, orig_mode,
1228 62090 : i * UNITS_PER_WORD);
1229 62090 : x = gen_rtx_CLOBBER (VOIDmode, x);
1230 62090 : emit_insn_after (x, insn);
1231 : }
1232 :
1233 62090 : resolve_reg_notes (insn);
1234 :
1235 62090 : return true;
1236 : }
1237 :
1238 : /* A USE of a decomposed register is no longer meaningful. Return
1239 : whether we changed something. */
1240 :
1241 : static bool
1242 62877 : resolve_use (rtx pat, rtx_insn *insn)
1243 : {
1244 62877 : if (resolve_reg_p (XEXP (pat, 0)) || resolve_subreg_p (XEXP (pat, 0)))
1245 : {
1246 0 : delete_insn (insn);
1247 0 : return true;
1248 : }
1249 :
1250 62877 : resolve_reg_notes (insn);
1251 :
1252 62877 : return false;
1253 : }
1254 :
1255 : /* A VAR_LOCATION can be simplified. */
1256 :
1257 : static void
1258 15008825 : resolve_debug (rtx_insn *insn)
1259 : {
1260 15008825 : subrtx_ptr_iterator::array_type array;
1261 51782333 : FOR_EACH_SUBRTX_PTR (iter, array, &PATTERN (insn), NONCONST)
1262 : {
1263 36773508 : rtx *loc = *iter;
1264 36773508 : rtx x = *loc;
1265 36773508 : if (resolve_subreg_p (x))
1266 : {
1267 157162 : x = simplify_subreg_concatn (GET_MODE (x), SUBREG_REG (x),
1268 78581 : SUBREG_BYTE (x));
1269 :
1270 78581 : if (x)
1271 78479 : *loc = x;
1272 : }
1273 36821984 : else if (resolve_reg_p (x))
1274 : {
1275 28010 : *loc = copy_rtx (x);
1276 28010 : DECOMPOSED_REGISTER_P (*loc) = false;
1277 : }
1278 : }
1279 :
1280 15008825 : df_insn_rescan (insn);
1281 :
1282 15008825 : resolve_reg_notes (insn);
1283 15008825 : }
1284 :
1285 : /* Check if INSN is a decomposable multiword-shift or zero-extend and
1286 : set the decomposable_context bitmap accordingly. SPEED_P is true
1287 : if we are optimizing INSN for speed rather than size. Return true
1288 : if INSN is decomposable. */
1289 :
1290 : static bool
1291 95298329 : find_decomposable_shift_zext (rtx_insn *insn, bool speed_p)
1292 : {
1293 95298329 : rtx set;
1294 95298329 : rtx op;
1295 95298329 : rtx op_operand;
1296 :
1297 95298329 : set = single_set (insn);
1298 95298329 : if (!set)
1299 : return false;
1300 :
1301 48931078 : op = SET_SRC (set);
1302 48931078 : if (GET_CODE (op) != ASHIFT
1303 : && GET_CODE (op) != LSHIFTRT
1304 : && GET_CODE (op) != ASHIFTRT
1305 : && GET_CODE (op) != ZERO_EXTEND)
1306 : return false;
1307 :
1308 1035349 : op_operand = XEXP (op, 0);
1309 1021385 : if (!REG_P (SET_DEST (set)) || !REG_P (op_operand)
1310 837801 : || HARD_REGISTER_NUM_P (REGNO (SET_DEST (set)))
1311 837372 : || HARD_REGISTER_NUM_P (REGNO (op_operand))
1312 1872717 : || GET_MODE (op) != twice_word_mode)
1313 : return false;
1314 :
1315 144831 : if (GET_CODE (op) == ZERO_EXTEND)
1316 : {
1317 68038 : if (GET_MODE (op_operand) != word_mode
1318 68038 : || !choices[speed_p].splitting_zext)
1319 : return false;
1320 : }
1321 : else /* left or right shift */
1322 : {
1323 76793 : bool *splitting = (GET_CODE (op) == ASHIFT
1324 21254 : ? choices[speed_p].splitting_ashift
1325 : : GET_CODE (op) == ASHIFTRT
1326 9297 : ? choices[speed_p].splitting_ashiftrt
1327 46242 : : choices[speed_p].splitting_lshiftrt);
1328 76793 : if (!CONST_INT_P (XEXP (op, 1))
1329 159166 : || !IN_RANGE (INTVAL (XEXP (op, 1)), BITS_PER_WORD,
1330 : 2 * BITS_PER_WORD - 1)
1331 82879 : || !splitting[INTVAL (XEXP (op, 1)) - BITS_PER_WORD])
1332 : return false;
1333 :
1334 48727 : bitmap_set_bit (decomposable_context, REGNO (op_operand));
1335 : }
1336 :
1337 48727 : bitmap_set_bit (decomposable_context, REGNO (SET_DEST (set)));
1338 :
1339 48727 : return true;
1340 : }
1341 :
1342 : /* Decompose a more than word wide shift (in INSN) of a multiword
1343 : pseudo or a multiword zero-extend of a wordmode pseudo into a move
1344 : and 'set to zero' insn. SPEED_P says whether we are optimizing
1345 : for speed or size, when checking if a ZERO_EXTEND is preferable.
1346 : Return a pointer to the new insn when a replacement was done. */
1347 :
1348 : static rtx_insn *
1349 22386035 : resolve_shift_zext (rtx_insn *insn, bool speed_p)
1350 : {
1351 22386035 : rtx set;
1352 22386035 : rtx op;
1353 22386035 : rtx op_operand;
1354 22386035 : rtx_insn *insns;
1355 22386035 : rtx src_reg, dest_reg, dest_upper, upper_src = NULL_RTX;
1356 22386035 : int src_reg_num, dest_reg_num, offset1, offset2, src_offset;
1357 22386035 : scalar_int_mode inner_mode;
1358 :
1359 22386035 : set = single_set (insn);
1360 22386035 : if (!set)
1361 : return NULL;
1362 :
1363 21194825 : op = SET_SRC (set);
1364 21194825 : if (GET_CODE (op) != ASHIFT
1365 : && GET_CODE (op) != LSHIFTRT
1366 : && GET_CODE (op) != ASHIFTRT
1367 : && GET_CODE (op) != ZERO_EXTEND)
1368 : return NULL;
1369 :
1370 466601 : op_operand = XEXP (op, 0);
1371 466601 : if (!is_a <scalar_int_mode> (GET_MODE (op_operand), &inner_mode))
1372 : return NULL;
1373 :
1374 : /* We can tear this operation apart only if the regs were already
1375 : torn apart. */
1376 430398 : if (!resolve_reg_p (SET_DEST (set)) && !resolve_reg_p (op_operand))
1377 : return NULL;
1378 :
1379 : /* src_reg_num is the number of the word mode register which we
1380 : are operating on. For a left shift and a zero_extend on little
1381 : endian machines this is register 0. */
1382 3688 : src_reg_num = (GET_CODE (op) == LSHIFTRT || GET_CODE (op) == ASHIFTRT)
1383 18289 : ? 1 : 0;
1384 :
1385 18289 : if (WORDS_BIG_ENDIAN && GET_MODE_SIZE (inner_mode) > UNITS_PER_WORD)
1386 : src_reg_num = 1 - src_reg_num;
1387 :
1388 18289 : if (GET_CODE (op) == ZERO_EXTEND)
1389 : dest_reg_num = WORDS_BIG_ENDIAN ? 1 : 0;
1390 : else
1391 18289 : dest_reg_num = 1 - src_reg_num;
1392 :
1393 18289 : offset1 = UNITS_PER_WORD * dest_reg_num;
1394 18289 : offset2 = UNITS_PER_WORD * (1 - dest_reg_num);
1395 18289 : src_offset = UNITS_PER_WORD * src_reg_num;
1396 :
1397 18289 : start_sequence ();
1398 :
1399 36578 : dest_reg = simplify_gen_subreg_concatn (word_mode, SET_DEST (set),
1400 18289 : GET_MODE (SET_DEST (set)),
1401 : offset1);
1402 36578 : dest_upper = simplify_gen_subreg_concatn (word_mode, SET_DEST (set),
1403 18289 : GET_MODE (SET_DEST (set)),
1404 : offset2);
1405 36578 : src_reg = simplify_gen_subreg_concatn (word_mode, op_operand,
1406 18289 : GET_MODE (op_operand),
1407 : src_offset);
1408 18289 : if (GET_CODE (op) == ASHIFTRT
1409 3307 : && INTVAL (XEXP (op, 1)) != 2 * BITS_PER_WORD - 1)
1410 3156 : upper_src = expand_shift (RSHIFT_EXPR, word_mode, copy_rtx (src_reg),
1411 3192 : BITS_PER_WORD - 1, NULL_RTX, 0);
1412 :
1413 18289 : if (GET_CODE (op) != ZERO_EXTEND)
1414 : {
1415 18289 : int shift_count = INTVAL (XEXP (op, 1));
1416 28887 : if (shift_count > BITS_PER_WORD)
1417 2914 : src_reg = expand_shift (GET_CODE (op) == ASHIFT ?
1418 : LSHIFT_EXPR : RSHIFT_EXPR,
1419 : word_mode, src_reg,
1420 2914 : shift_count - BITS_PER_WORD,
1421 : dest_reg, GET_CODE (op) != ASHIFTRT);
1422 : }
1423 :
1424 : /* Consider using ZERO_EXTEND instead of setting DEST_UPPER to zero
1425 : if this is considered reasonable. */
1426 18289 : if (GET_CODE (op) == LSHIFTRT
1427 11375 : && GET_MODE (op) == twice_word_mode
1428 11375 : && REG_P (SET_DEST (set))
1429 18592 : && !choices[speed_p].splitting_zext)
1430 : {
1431 303 : rtx tmp = force_reg (word_mode, copy_rtx (src_reg));
1432 303 : tmp = simplify_gen_unary (ZERO_EXTEND, twice_word_mode, tmp, word_mode);
1433 303 : emit_move_insn (SET_DEST (set), tmp);
1434 : }
1435 : else
1436 : {
1437 17986 : if (dest_reg != src_reg)
1438 15124 : emit_move_insn (dest_reg, src_reg);
1439 17986 : if (GET_CODE (op) != ASHIFTRT)
1440 14760 : emit_move_insn (dest_upper, CONST0_RTX (word_mode));
1441 3307 : else if (INTVAL (XEXP (op, 1)) == 2 * BITS_PER_WORD - 1)
1442 70 : emit_move_insn (dest_upper, copy_rtx (src_reg));
1443 : else
1444 3156 : emit_move_insn (dest_upper, upper_src);
1445 : }
1446 :
1447 18289 : insns = end_sequence ();
1448 :
1449 18289 : emit_insn_before (insns, insn);
1450 :
1451 18289 : if (dump_file)
1452 : {
1453 0 : rtx_insn *in;
1454 0 : fprintf (dump_file, "; Replacing insn: %d with insns: ", INSN_UID (insn));
1455 0 : for (in = insns; in != insn; in = NEXT_INSN (in))
1456 0 : fprintf (dump_file, "%d ", INSN_UID (in));
1457 0 : fprintf (dump_file, "\n");
1458 : }
1459 :
1460 18289 : delete_insn (insn);
1461 18289 : return insns;
1462 : }
1463 :
1464 : /* Print to dump_file a description of what we're doing with shift code CODE.
1465 : SPLITTING[X] is true if we are splitting shifts by X + BITS_PER_WORD. */
1466 :
1467 : static void
1468 360 : dump_shift_choices (enum rtx_code code, bool *splitting)
1469 : {
1470 360 : int i;
1471 360 : const char *sep;
1472 :
1473 360 : fprintf (dump_file,
1474 : " Splitting mode %s for %s lowering with shift amounts = ",
1475 360 : GET_MODE_NAME (twice_word_mode), GET_RTX_NAME (code));
1476 360 : sep = "";
1477 23760 : for (i = 0; i < BITS_PER_WORD; i++)
1478 23040 : if (splitting[i])
1479 : {
1480 23040 : fprintf (dump_file, "%s%d", sep, i + BITS_PER_WORD);
1481 23040 : sep = ",";
1482 : }
1483 360 : fprintf (dump_file, "\n");
1484 360 : }
1485 :
1486 : /* Print to dump_file a description of what we're doing when optimizing
1487 : for speed or size; SPEED_P says which. DESCRIPTION is a description
1488 : of the SPEED_P choice. */
1489 :
1490 : static void
1491 120 : dump_choices (bool speed_p, const char *description)
1492 : {
1493 120 : unsigned int size, factor, i;
1494 :
1495 120 : fprintf (dump_file, "Choices when optimizing for %s:\n", description);
1496 :
1497 15120 : for (i = 0; i < MAX_MACHINE_MODE; i++)
1498 14880 : if (interesting_mode_p ((machine_mode) i, &size, &factor)
1499 14880 : && factor > 1)
1500 7440 : fprintf (dump_file, " %s mode %s for copy lowering.\n",
1501 7440 : choices[speed_p].move_modes_to_split[i]
1502 : ? "Splitting"
1503 : : "Skipping",
1504 7440 : GET_MODE_NAME ((machine_mode) i));
1505 :
1506 120 : fprintf (dump_file, " %s mode %s for zero_extend lowering.\n",
1507 120 : choices[speed_p].splitting_zext ? "Splitting" : "Skipping",
1508 120 : GET_MODE_NAME (twice_word_mode));
1509 :
1510 120 : dump_shift_choices (ASHIFT, choices[speed_p].splitting_ashift);
1511 120 : dump_shift_choices (LSHIFTRT, choices[speed_p].splitting_lshiftrt);
1512 120 : dump_shift_choices (ASHIFTRT, choices[speed_p].splitting_ashiftrt);
1513 120 : fprintf (dump_file, "\n");
1514 120 : }
1515 :
1516 : /* Look for registers which are always accessed via word-sized SUBREGs
1517 : or -if DECOMPOSE_COPIES is true- via copies. Decompose these
1518 : registers into several word-sized pseudo-registers. */
1519 :
1520 : static void
1521 2140311 : decompose_multiword_subregs (bool decompose_copies)
1522 : {
1523 2140311 : unsigned int max;
1524 2140311 : basic_block bb;
1525 2140311 : bool speed_p;
1526 :
1527 2140311 : if (dump_file)
1528 : {
1529 60 : dump_choices (false, "size");
1530 60 : dump_choices (true, "speed");
1531 : }
1532 :
1533 : /* Check if this target even has any modes to consider lowering. */
1534 2140311 : if (!choices[false].something_to_do && !choices[true].something_to_do)
1535 : {
1536 0 : if (dump_file)
1537 0 : fprintf (dump_file, "Nothing to do!\n");
1538 : return;
1539 : }
1540 :
1541 2140311 : max = max_reg_num ();
1542 :
1543 : /* First see if there are any multi-word pseudo-registers. If there
1544 : aren't, there is nothing we can do. This should speed up this
1545 : pass in the normal case, since it should be faster than scanning
1546 : all the insns. */
1547 2140311 : {
1548 2140311 : unsigned int i;
1549 2140311 : bool useful_modes_seen = false;
1550 :
1551 80796061 : for (i = FIRST_PSEUDO_REGISTER; i < max; ++i)
1552 78993592 : if (regno_reg_rtx[i] != NULL)
1553 : {
1554 78892843 : machine_mode mode = GET_MODE (regno_reg_rtx[i]);
1555 78892843 : if (choices[false].move_modes_to_split[(int) mode]
1556 78555001 : || choices[true].move_modes_to_split[(int) mode])
1557 : {
1558 : useful_modes_seen = true;
1559 : break;
1560 : }
1561 : }
1562 :
1563 2140311 : if (!useful_modes_seen)
1564 : {
1565 1802469 : if (dump_file)
1566 60 : fprintf (dump_file, "Nothing to lower in this function.\n");
1567 : return;
1568 : }
1569 : }
1570 :
1571 337842 : if (df)
1572 : {
1573 148304 : df_set_flags (DF_DEFER_INSN_RESCAN);
1574 148304 : run_word_dce ();
1575 : }
1576 :
1577 : /* FIXME: It may be possible to change this code to look for each
1578 : multi-word pseudo-register and to find each insn which sets or
1579 : uses that register. That should be faster than scanning all the
1580 : insns. */
1581 :
1582 337842 : decomposable_context = BITMAP_ALLOC (NULL);
1583 337842 : non_decomposable_context = BITMAP_ALLOC (NULL);
1584 337842 : subreg_context = BITMAP_ALLOC (NULL);
1585 :
1586 337842 : reg_copy_graph.create (max);
1587 337842 : reg_copy_graph.safe_grow_cleared (max, true);
1588 337842 : memset (reg_copy_graph.address (), 0, sizeof (bitmap) * max);
1589 :
1590 337842 : speed_p = optimize_function_for_speed_p (cfun);
1591 8954052 : FOR_EACH_BB_FN (bb, cfun)
1592 : {
1593 8616210 : rtx_insn *insn;
1594 :
1595 118857138 : FOR_BB_INSNS (bb, insn)
1596 : {
1597 110240928 : rtx set;
1598 110240928 : enum classify_move_insn cmi;
1599 110240928 : int i, n;
1600 :
1601 125183527 : if (!INSN_P (insn)
1602 95640464 : || GET_CODE (PATTERN (insn)) == CLOBBER
1603 205737240 : || GET_CODE (PATTERN (insn)) == USE)
1604 14991326 : continue;
1605 :
1606 95298329 : recog_memoized (insn);
1607 :
1608 95298329 : if (find_decomposable_shift_zext (insn, speed_p))
1609 48727 : continue;
1610 :
1611 95249602 : extract_insn (insn);
1612 :
1613 95249602 : set = simple_move (insn, speed_p);
1614 :
1615 95249602 : if (!set)
1616 92415723 : cmi = NOT_SIMPLE_MOVE;
1617 : else
1618 : {
1619 : /* We mark pseudo-to-pseudo copies as decomposable during the
1620 : second pass only. The first pass is so early that there is
1621 : good chance such moves will be optimized away completely by
1622 : subsequent optimizations anyway.
1623 :
1624 : However, we call find_pseudo_copy even during the first pass
1625 : so as to properly set up the reg_copy_graph. */
1626 2833879 : if (find_pseudo_copy (set))
1627 273691 : cmi = decompose_copies? DECOMPOSABLE_SIMPLE_MOVE : SIMPLE_MOVE;
1628 : else
1629 2692297 : cmi = SIMPLE_MOVE;
1630 : }
1631 :
1632 95249602 : n = recog_data.n_operands;
1633 208336290 : for (i = 0; i < n; ++i)
1634 : {
1635 113086688 : find_decomposable_subregs (&recog_data.operand[i], &cmi);
1636 :
1637 : /* We handle ASM_OPERANDS as a special case to support
1638 : things like x86 rdtsc which returns a DImode value.
1639 : We can decompose the output, which will certainly be
1640 : operand 0, but not the inputs. */
1641 :
1642 113086688 : if (cmi == SIMPLE_MOVE
1643 5648812 : && GET_CODE (SET_SRC (set)) == ASM_OPERANDS)
1644 : {
1645 0 : gcc_assert (i == 0);
1646 0 : cmi = NOT_SIMPLE_MOVE;
1647 : }
1648 : }
1649 : }
1650 : }
1651 :
1652 337842 : bitmap_and_compl_into (decomposable_context, non_decomposable_context);
1653 337842 : if (!bitmap_empty_p (decomposable_context))
1654 : {
1655 110135 : unsigned int i;
1656 110135 : sbitmap_iterator sbi;
1657 110135 : bitmap_iterator iter;
1658 110135 : unsigned int regno;
1659 :
1660 110135 : propagate_pseudo_copies ();
1661 :
1662 110135 : auto_sbitmap sub_blocks (last_basic_block_for_fn (cfun));
1663 110135 : bitmap_clear (sub_blocks);
1664 :
1665 420614 : EXECUTE_IF_SET_IN_BITMAP (decomposable_context, 0, regno, iter)
1666 310479 : decompose_register (regno);
1667 :
1668 4020399 : FOR_EACH_BB_FN (bb, cfun)
1669 : {
1670 3910264 : rtx_insn *insn;
1671 :
1672 49435817 : FOR_BB_INSNS (bb, insn)
1673 : {
1674 45525553 : rtx pat;
1675 :
1676 45525553 : if (!INSN_P (insn))
1677 6319607 : continue;
1678 :
1679 39205946 : pat = PATTERN (insn);
1680 39205946 : if (GET_CODE (pat) == CLOBBER)
1681 166136 : resolve_clobber (pat, insn);
1682 39039810 : else if (GET_CODE (pat) == USE)
1683 62877 : resolve_use (pat, insn);
1684 38976933 : else if (DEBUG_INSN_P (insn))
1685 15008825 : resolve_debug (insn);
1686 : else
1687 : {
1688 23968108 : rtx set;
1689 23968108 : int i;
1690 :
1691 23968108 : recog_memoized (insn);
1692 23968108 : extract_insn (insn);
1693 :
1694 23968108 : set = simple_move (insn, speed_p);
1695 23968108 : if (set)
1696 : {
1697 1582073 : rtx_insn *orig_insn = insn;
1698 1582073 : bool cfi = control_flow_insn_p (insn);
1699 :
1700 : /* We can end up splitting loads to multi-word pseudos
1701 : into separate loads to machine word size pseudos.
1702 : When this happens, we first had one load that can
1703 : throw, and after resolve_simple_move we'll have a
1704 : bunch of loads (at least two). All those loads may
1705 : trap if we can have non-call exceptions, so they
1706 : all will end the current basic block. We split the
1707 : block after the outer loop over all insns, but we
1708 : make sure here that we will be able to split the
1709 : basic block and still produce the correct control
1710 : flow graph for it. */
1711 1582073 : gcc_assert (!cfi
1712 : || (cfun->can_throw_non_call_exceptions
1713 : && can_throw_internal (insn)));
1714 :
1715 1582073 : insn = resolve_simple_move (set, insn);
1716 1582073 : if (insn != orig_insn)
1717 : {
1718 255237 : recog_memoized (insn);
1719 255237 : extract_insn (insn);
1720 :
1721 255237 : if (cfi)
1722 6274 : bitmap_set_bit (sub_blocks, bb->index);
1723 : }
1724 : }
1725 : else
1726 : {
1727 22386035 : rtx_insn *decomposed_shift;
1728 :
1729 22386035 : decomposed_shift = resolve_shift_zext (insn, speed_p);
1730 22386035 : if (decomposed_shift != NULL_RTX)
1731 : {
1732 18289 : insn = decomposed_shift;
1733 18289 : recog_memoized (insn);
1734 18289 : extract_insn (insn);
1735 : }
1736 : }
1737 :
1738 76714433 : for (i = recog_data.n_operands - 1; i >= 0; --i)
1739 52746325 : resolve_subreg_use (recog_data.operand_loc[i], insn);
1740 :
1741 23968108 : resolve_reg_notes (insn);
1742 :
1743 23968108 : if (num_validated_changes () > 0)
1744 : {
1745 798759 : for (i = recog_data.n_dups - 1; i >= 0; --i)
1746 : {
1747 11836 : rtx *pl = recog_data.dup_loc[i];
1748 11836 : int dup_num = recog_data.dup_num[i];
1749 11836 : rtx *px = recog_data.operand_loc[dup_num];
1750 :
1751 11836 : validate_unshare_change (insn, pl, *px, 1);
1752 : }
1753 :
1754 786923 : i = apply_change_group ();
1755 786923 : gcc_assert (i);
1756 : }
1757 : }
1758 : }
1759 : }
1760 :
1761 : /* If we had insns to split that caused control flow insns in the middle
1762 : of a basic block, split those blocks now. Note that we only handle
1763 : the case where splitting a load has caused multiple possibly trapping
1764 : loads to appear. */
1765 226544 : EXECUTE_IF_SET_IN_BITMAP (sub_blocks, 0, i, sbi)
1766 : {
1767 6274 : rtx_insn *insn, *end;
1768 6274 : edge fallthru;
1769 :
1770 6274 : bb = BASIC_BLOCK_FOR_FN (cfun, i);
1771 6274 : insn = BB_HEAD (bb);
1772 6274 : end = BB_END (bb);
1773 :
1774 36469 : while (insn != end)
1775 : {
1776 30195 : if (control_flow_insn_p (insn))
1777 : {
1778 : /* Split the block after insn. There will be a fallthru
1779 : edge, which is OK so we keep it. We have to create the
1780 : exception edges ourselves. */
1781 6392 : fallthru = split_block (bb, insn);
1782 6392 : rtl_make_eh_edge (NULL, bb, BB_END (bb));
1783 6392 : bb = fallthru->dest;
1784 6392 : insn = BB_HEAD (bb);
1785 : }
1786 : else
1787 23803 : insn = NEXT_INSN (insn);
1788 : }
1789 : }
1790 110135 : }
1791 :
1792 65140853 : for (bitmap b : reg_copy_graph)
1793 64127327 : if (b)
1794 138946 : BITMAP_FREE (b);
1795 :
1796 337842 : reg_copy_graph.release ();
1797 :
1798 337842 : BITMAP_FREE (decomposable_context);
1799 337842 : BITMAP_FREE (non_decomposable_context);
1800 337842 : BITMAP_FREE (subreg_context);
1801 : }
1802 :
1803 : /* Implement first lower subreg pass. */
1804 :
1805 : namespace {
1806 :
1807 : const pass_data pass_data_lower_subreg =
1808 : {
1809 : RTL_PASS, /* type */
1810 : "subreg1", /* name */
1811 : OPTGROUP_NONE, /* optinfo_flags */
1812 : TV_LOWER_SUBREG, /* tv_id */
1813 : 0, /* properties_required */
1814 : 0, /* properties_provided */
1815 : 0, /* properties_destroyed */
1816 : 0, /* todo_flags_start */
1817 : 0, /* todo_flags_finish */
1818 : };
1819 :
1820 : class pass_lower_subreg : public rtl_opt_pass
1821 : {
1822 : public:
1823 294536 : pass_lower_subreg (gcc::context *ctxt)
1824 589072 : : rtl_opt_pass (pass_data_lower_subreg, ctxt)
1825 : {}
1826 :
1827 : /* opt_pass methods: */
1828 1515929 : bool gate (function *) final override { return flag_split_wide_types != 0; }
1829 1070155 : unsigned int execute (function *) final override
1830 : {
1831 1070155 : decompose_multiword_subregs (false);
1832 1070155 : return 0;
1833 : }
1834 :
1835 : }; // class pass_lower_subreg
1836 :
1837 : } // anon namespace
1838 :
1839 : rtl_opt_pass *
1840 294536 : make_pass_lower_subreg (gcc::context *ctxt)
1841 : {
1842 294536 : return new pass_lower_subreg (ctxt);
1843 : }
1844 :
1845 : /* Implement second lower subreg pass. */
1846 :
1847 : namespace {
1848 :
1849 : const pass_data pass_data_lower_subreg2 =
1850 : {
1851 : RTL_PASS, /* type */
1852 : "subreg2", /* name */
1853 : OPTGROUP_NONE, /* optinfo_flags */
1854 : TV_LOWER_SUBREG, /* tv_id */
1855 : 0, /* properties_required */
1856 : 0, /* properties_provided */
1857 : 0, /* properties_destroyed */
1858 : 0, /* todo_flags_start */
1859 : TODO_df_finish, /* todo_flags_finish */
1860 : };
1861 :
1862 : class pass_lower_subreg2 : public rtl_opt_pass
1863 : {
1864 : public:
1865 294536 : pass_lower_subreg2 (gcc::context *ctxt)
1866 589072 : : rtl_opt_pass (pass_data_lower_subreg2, ctxt)
1867 : {}
1868 :
1869 : /* opt_pass methods: */
1870 1515929 : bool gate (function *) final override
1871 : {
1872 1515929 : return flag_split_wide_types && flag_split_wide_types_early;
1873 : }
1874 0 : unsigned int execute (function *) final override
1875 : {
1876 0 : decompose_multiword_subregs (true);
1877 0 : return 0;
1878 : }
1879 :
1880 : }; // class pass_lower_subreg2
1881 :
1882 : } // anon namespace
1883 :
1884 : rtl_opt_pass *
1885 294536 : make_pass_lower_subreg2 (gcc::context *ctxt)
1886 : {
1887 294536 : return new pass_lower_subreg2 (ctxt);
1888 : }
1889 :
1890 : /* Implement third lower subreg pass. */
1891 :
1892 : namespace {
1893 :
1894 : const pass_data pass_data_lower_subreg3 =
1895 : {
1896 : RTL_PASS, /* type */
1897 : "subreg3", /* name */
1898 : OPTGROUP_NONE, /* optinfo_flags */
1899 : TV_LOWER_SUBREG, /* tv_id */
1900 : 0, /* properties_required */
1901 : 0, /* properties_provided */
1902 : 0, /* properties_destroyed */
1903 : 0, /* todo_flags_start */
1904 : TODO_df_finish, /* todo_flags_finish */
1905 : };
1906 :
1907 : class pass_lower_subreg3 : public rtl_opt_pass
1908 : {
1909 : public:
1910 294536 : pass_lower_subreg3 (gcc::context *ctxt)
1911 589072 : : rtl_opt_pass (pass_data_lower_subreg3, ctxt)
1912 : {}
1913 :
1914 : /* opt_pass methods: */
1915 1515929 : bool gate (function *) final override { return flag_split_wide_types; }
1916 1070156 : unsigned int execute (function *) final override
1917 : {
1918 1070156 : decompose_multiword_subregs (true);
1919 1070156 : return 0;
1920 : }
1921 :
1922 : }; // class pass_lower_subreg3
1923 :
1924 : } // anon namespace
1925 :
1926 : rtl_opt_pass *
1927 294536 : make_pass_lower_subreg3 (gcc::context *ctxt)
1928 : {
1929 294536 : return new pass_lower_subreg3 (ctxt);
1930 : }
|
