Line data Source code
1 : /* Avoid store forwarding optimization pass.
2 : Copyright (C) 2024-2026 Free Software Foundation, Inc.
3 : Contributed by VRULL GmbH.
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it
8 : under the terms of the GNU General Public License as published by
9 : the Free Software Foundation; either version 3, or (at your option)
10 : any later version.
11 :
12 : GCC is distributed in the hope that it will be useful, but
13 : WITHOUT ANY WARRANTY; without even the implied warranty of
14 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 : General Public License for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "target.h"
26 : #include "rtl.h"
27 : #include "avoid-store-forwarding.h"
28 : #include "alias.h"
29 : #include "rtlanal.h"
30 : #include "cfgrtl.h"
31 : #include "tree-pass.h"
32 : #include "predict.h"
33 : #include "insn-config.h"
34 : #include "expmed.h"
35 : #include "recog.h"
36 : #include "regset.h"
37 : #include "regs.h"
38 : #include "df.h"
39 : #include "expr.h"
40 : #include "memmodel.h"
41 : #include "emit-rtl.h"
42 : #include "vec.h"
43 :
44 : /* This pass tries to detect and avoid cases of store forwarding.
45 : On many processors there is a large penalty when smaller stores are
46 : forwarded to larger loads. The idea used to avoid the stall is to move
47 : the store after the load and in addition emit a bit insert sequence so
48 : the load register has the correct value. For example the following:
49 :
50 : strb w2, [x1, 1]
51 : ldr x0, [x1]
52 :
53 : Will be transformed to:
54 :
55 : ldr x0, [x1]
56 : strb w2, [x1]
57 : bfi x0, x2, 0, 8
58 : */
59 :
60 : namespace {
61 :
62 : const pass_data pass_data_avoid_store_forwarding =
63 : {
64 : RTL_PASS, /* type. */
65 : "avoid_store_forwarding", /* name. */
66 : OPTGROUP_NONE, /* optinfo_flags. */
67 : TV_AVOID_STORE_FORWARDING, /* tv_id. */
68 : 0, /* properties_required. */
69 : 0, /* properties_provided. */
70 : 0, /* properties_destroyed. */
71 : 0, /* todo_flags_start. */
72 : TODO_df_finish /* todo_flags_finish. */
73 : };
74 :
75 : class pass_rtl_avoid_store_forwarding : public rtl_opt_pass
76 : {
77 : public:
78 294196 : pass_rtl_avoid_store_forwarding (gcc::context *ctxt)
79 588392 : : rtl_opt_pass (pass_data_avoid_store_forwarding, ctxt)
80 : {}
81 :
82 : /* opt_pass methods: */
83 1515129 : virtual bool gate (function *) final override
84 : {
85 1515129 : return flag_avoid_store_forwarding && optimize >= 1;
86 : }
87 :
88 : virtual unsigned int execute (function *) final override;
89 : }; // class pass_rtl_avoid_store_forwarding
90 :
91 : /* Handler for finding and avoiding store forwardings. */
92 :
93 60 : class store_forwarding_analyzer
94 : {
95 : public:
96 : unsigned int stats_sf_detected = 0;
97 : unsigned int stats_sf_avoided = 0;
98 :
99 : bool is_store_forwarding (rtx store_mem, rtx load_mem,
100 : HOST_WIDE_INT *off_val);
101 : bool process_store_forwarding (vec<store_fwd_info> &, rtx_insn *load_insn,
102 : rtx load_mem);
103 : void avoid_store_forwarding (basic_block);
104 : void update_stats (function *);
105 :
106 : private:
107 : /* Per-insn live-out hard-register sets for the current BB. Populated
108 : lazily on the first candidate with bit-insert side-effect clobbers
109 : (so aarch64 bfi pays nothing). Cleared on each avoid_store_forwarding
110 : entry. */
111 : hash_map<rtx_insn *, HARD_REG_SET> m_bb_live_after;
112 :
113 : void compute_bb_live_after (basic_block bb);
114 : };
115 :
116 : /* Return a bit insertion sequence that would make DEST have the correct value
117 : if the store represented by STORE_INFO were to be moved after DEST. */
118 :
119 : static rtx_insn *
120 48 : generate_bit_insert_sequence (store_fwd_info *store_info, rtx dest)
121 : {
122 : /* Memory size should be a constant at this stage. */
123 48 : unsigned HOST_WIDE_INT store_size
124 48 : = MEM_SIZE (store_info->store_mem).to_constant ();
125 :
126 48 : start_sequence ();
127 :
128 48 : unsigned HOST_WIDE_INT bitsize = store_size * BITS_PER_UNIT;
129 48 : unsigned HOST_WIDE_INT start = store_info->offset * BITS_PER_UNIT;
130 :
131 48 : rtx mov_reg = store_info->mov_reg;
132 48 : store_bit_field (dest, bitsize, start, 0, 0, GET_MODE (mov_reg), mov_reg,
133 : false, false);
134 :
135 48 : rtx_insn *insns = get_insns ();
136 48 : unshare_all_rtl_in_chain (insns);
137 48 : end_sequence ();
138 :
139 361 : for (rtx_insn *insn = insns; insn; insn = NEXT_INSN (insn))
140 265 : if (contains_mem_rtx_p (PATTERN (insn))
141 265 : || recog_memoized (insn) < 0)
142 : return NULL;
143 :
144 : return insns;
145 : }
146 :
147 : /* note_stores callback: record hard regs clobbered (not set) by an insn,
148 : to capture side-effect clobbers (e.g. flags) without the intended dest. */
149 :
150 : static void
151 393 : record_hard_reg_clobbers (rtx x, const_rtx pat, void *data)
152 : {
153 393 : if (GET_CODE (pat) == CLOBBER && REG_P (x) && HARD_REGISTER_P (x))
154 114 : add_to_hard_reg_set ((HARD_REG_SET *) data, GET_MODE (x), REGNO (x));
155 393 : }
156 :
157 : /* Populate m_bb_live_after with the hard registers live immediately
158 : after each real insn in BB. */
159 :
160 : void
161 9 : store_forwarding_analyzer::compute_bb_live_after (basic_block bb)
162 : {
163 9 : auto_bitmap live;
164 9 : df_simulate_initialize_backwards (bb, live);
165 9 : rtx_insn *scan;
166 335 : FOR_BB_INSNS_REVERSE (bb, scan)
167 326 : if (INSN_P (scan))
168 : {
169 : HARD_REG_SET hrs;
170 295 : REG_SET_TO_HARD_REG_SET (hrs, live);
171 295 : m_bb_live_after.put (scan, hrs);
172 295 : df_simulate_one_insn_backwards (bb, scan, live);
173 : }
174 9 : }
175 :
176 : /* Return true iff a store to STORE_MEM would write to a sub-region of bytes
177 : from what LOAD_MEM would read. If true also store the relative byte offset
178 : of the store within the load to OFF_VAL. */
179 :
180 376 : bool store_forwarding_analyzer::
181 : is_store_forwarding (rtx store_mem, rtx load_mem, HOST_WIDE_INT *off_val)
182 : {
183 376 : poly_int64 load_offset, store_offset;
184 376 : rtx load_base = strip_offset (XEXP (load_mem, 0), &load_offset);
185 376 : rtx store_base = strip_offset (XEXP (store_mem, 0), &store_offset);
186 376 : poly_int64 off_diff = store_offset - load_offset;
187 :
188 376 : HOST_WIDE_INT off_val_tmp = 0;
189 376 : bool is_off_diff_constant = off_diff.is_constant (&off_val_tmp);
190 376 : if (off_val)
191 376 : *off_val = off_val_tmp;
192 :
193 376 : return (MEM_SIZE (load_mem).is_constant ()
194 376 : && rtx_equal_p (load_base, store_base)
195 288 : && known_subrange_p (store_offset, MEM_SIZE (store_mem),
196 288 : load_offset, MEM_SIZE (load_mem))
197 376 : && is_off_diff_constant);
198 : }
199 :
200 : /* Given a list of small stores that are forwarded to LOAD_INSN, try to
201 : rearrange them so that a store-forwarding penalty doesn't occur.
202 : The stores must be given in reverse program order, starting from the
203 : one closer to LOAD_INSN. */
204 :
205 24 : bool store_forwarding_analyzer::
206 : process_store_forwarding (vec<store_fwd_info> &stores, rtx_insn *load_insn,
207 : rtx load_mem)
208 : {
209 24 : machine_mode load_mem_mode = GET_MODE (load_mem);
210 : /* Memory sizes should be constants at this stage. */
211 24 : HOST_WIDE_INT load_size = MEM_SIZE (load_mem).to_constant ();
212 :
213 : /* If the stores cover all the bytes of the load without overlap then we can
214 : eliminate the load entirely and use the computed value instead.
215 : Bail out when partially overlapping stores are detected, as the pass
216 : cannot correctly handle "last writer wins" semantics for the
217 : overlapping byte ranges (see PR124476). */
218 :
219 24 : auto_sbitmap forwarded_bytes (load_size);
220 24 : bitmap_clear (forwarded_bytes);
221 :
222 24 : unsigned int i;
223 24 : store_fwd_info* it;
224 133 : FOR_EACH_VEC_ELT (stores, i, it)
225 : {
226 88 : HOST_WIDE_INT store_size = MEM_SIZE (it->store_mem).to_constant ();
227 88 : if (bitmap_any_bit_in_range_p (forwarded_bytes, it->offset,
228 88 : it->offset + store_size - 1))
229 : return false;
230 85 : bitmap_set_range (forwarded_bytes, it->offset, store_size);
231 : }
232 :
233 21 : bitmap_not (forwarded_bytes, forwarded_bytes);
234 21 : bool load_elim = bitmap_empty_p (forwarded_bytes);
235 :
236 21 : stats_sf_detected++;
237 :
238 21 : if (dump_file)
239 : {
240 0 : fprintf (dump_file, "Store forwarding detected:\n");
241 :
242 0 : FOR_EACH_VEC_ELT (stores, i, it)
243 : {
244 0 : fprintf (dump_file, "From: ");
245 0 : print_rtl_single (dump_file, it->store_insn);
246 : }
247 :
248 0 : fprintf (dump_file, "To: ");
249 0 : print_rtl_single (dump_file, load_insn);
250 :
251 0 : if (load_elim)
252 0 : fprintf (dump_file, "(Load elimination candidate)\n");
253 : }
254 :
255 21 : rtx load = single_set (load_insn);
256 21 : rtx dest;
257 :
258 21 : if (load_elim)
259 14 : dest = gen_reg_rtx (load_mem_mode);
260 : else
261 7 : dest = SET_DEST (load);
262 :
263 21 : int move_to_front = -1;
264 21 : int total_cost = 0;
265 21 : int base_offset_index = -1;
266 :
267 : /* Find the last store that has the same offset the load, in the case that
268 : we're eliminating the load. We will try to use it as a base register
269 : to avoid bit inserts (see second loop below). We want the last one, as
270 : it will be wider and we don't want to overwrite the base register if
271 : there are many of them. */
272 7 : if (load_elim)
273 : {
274 28 : FOR_EACH_VEC_ELT_REVERSE (stores, i, it)
275 : {
276 14 : const bool has_base_offset
277 14 : = known_eq (poly_uint64 (it->offset),
278 : subreg_size_lowpart_offset (MEM_SIZE (it->store_mem),
279 : load_size));
280 14 : if (has_base_offset)
281 : {
282 14 : base_offset_index = i;
283 14 : break;
284 : }
285 : }
286 : }
287 :
288 : /* Check if we can emit bit insert instructions for all forwarded stores. */
289 146 : FOR_EACH_VEC_ELT (stores, i, it)
290 : {
291 62 : it->mov_reg = gen_reg_rtx (GET_MODE (it->store_mem));
292 62 : rtx_insn *insns = NULL;
293 :
294 : /* Check if this is a store with base offset, if we're eliminating the
295 : load, and use it as the base register to avoid a bit insert if
296 : possible. Load elimination is implied by base_offset_index != -1. */
297 62 : if (i == (unsigned) base_offset_index)
298 : {
299 14 : start_sequence ();
300 :
301 28 : rtx base_reg = lowpart_subreg (GET_MODE (dest), it->mov_reg,
302 14 : GET_MODE (it->mov_reg));
303 :
304 14 : if (base_reg)
305 : {
306 14 : rtx_insn *move0 = emit_move_insn (dest, base_reg);
307 14 : if (recog_memoized (move0) >= 0)
308 : {
309 14 : insns = get_insns ();
310 14 : move_to_front = (int) i;
311 : }
312 : }
313 :
314 14 : end_sequence ();
315 : }
316 :
317 14 : if (!insns)
318 48 : insns = generate_bit_insert_sequence (&(*it), dest);
319 :
320 48 : if (!insns)
321 : {
322 0 : if (dump_file)
323 : {
324 0 : fprintf (dump_file, "Failed due to: ");
325 0 : print_rtl_single (dump_file, it->store_insn);
326 : }
327 : return false;
328 : }
329 :
330 62 : total_cost += seq_cost (insns, true);
331 62 : it->bits_insert_insns = insns;
332 :
333 62 : rtx store_set = single_set (it->store_insn);
334 :
335 : /* Create a register move at the store's original position to save the
336 : stored value. */
337 62 : start_sequence ();
338 62 : rtx_insn *insn1
339 62 : = emit_insn (gen_rtx_SET (it->mov_reg, SET_SRC (store_set)));
340 62 : end_sequence ();
341 :
342 62 : if (recog_memoized (insn1) < 0)
343 : {
344 0 : if (dump_file)
345 : {
346 0 : fprintf (dump_file, "Failed due to unrecognizable insn: ");
347 0 : print_rtl_single (dump_file, insn1);
348 : }
349 : return false;
350 : }
351 :
352 62 : it->save_store_value_insn = insn1;
353 :
354 : /* Create a new store after the load with the saved original value.
355 : This avoids the forwarding stall. */
356 62 : start_sequence ();
357 62 : rtx_insn *insn2
358 62 : = emit_insn (gen_rtx_SET (SET_DEST (store_set), it->mov_reg));
359 62 : end_sequence ();
360 :
361 62 : if (recog_memoized (insn2) < 0)
362 : {
363 0 : if (dump_file)
364 : {
365 0 : fprintf (dump_file, "Failed due to unrecognizable insn: ");
366 0 : print_rtl_single (dump_file, insn2);
367 : }
368 : return false;
369 : }
370 :
371 62 : it->store_saved_value_insn = insn2;
372 : }
373 :
374 : /* Reject if the bit-insert sequences clobber a hard register live at
375 : the insertion point (e.g. shift/and/or on x86 clobber flags, which
376 : would break carry chains). Done before the target cost query so
377 : we skip cost work on candidates we would reject anyway. */
378 : HARD_REG_SET clobbered_regs;
379 83 : CLEAR_HARD_REG_SET (clobbered_regs);
380 146 : FOR_EACH_VEC_ELT (stores, i, it)
381 341 : for (rtx_insn *ins = it->bits_insert_insns; ins; ins = NEXT_INSN (ins))
382 279 : note_stores (ins, record_hard_reg_clobbers, &clobbered_regs);
383 :
384 21 : if (!hard_reg_set_empty_p (clobbered_regs))
385 : {
386 16 : if (m_bb_live_after.is_empty ())
387 9 : compute_bb_live_after (BLOCK_FOR_INSN (load_insn));
388 :
389 16 : const HARD_REG_SET *live_at_insert = m_bb_live_after.get (load_insn);
390 16 : if (live_at_insert
391 32 : && hard_reg_set_intersect_p (clobbered_regs, *live_at_insert))
392 : {
393 5 : if (dump_file)
394 0 : fprintf (dump_file,
395 : "Not transformed: bit-insert clobbers live hard reg.\n");
396 : return false;
397 : }
398 : }
399 :
400 16 : if (load_elim)
401 10 : total_cost -= insn_cost (load_insn, true);
402 :
403 : /* Let the target decide if transforming this store forwarding instance is
404 : profitable. */
405 16 : if (!targetm.avoid_store_forwarding_p (stores, load_mem, total_cost,
406 : load_elim))
407 : {
408 5 : if (dump_file)
409 0 : fprintf (dump_file, "Not transformed due to target decision.\n");
410 :
411 : return false;
412 : }
413 :
414 : /* If we have a move instead of bit insert, it needs to be emitted first in
415 : the resulting sequence. */
416 11 : if (move_to_front != -1)
417 : {
418 6 : store_fwd_info copy = stores[move_to_front];
419 6 : stores.safe_push (copy);
420 6 : stores.ordered_remove (move_to_front);
421 : }
422 :
423 11 : machine_mode outer_mode = GET_MODE (SET_DEST (load));
424 11 : if (load_elim || outer_mode != load_mem_mode)
425 : {
426 : /* If the load is being eliminated, emit a move (with extension if
427 : needed) from the temp register to the original load destination.
428 : Otherwise, if the load has SIGN_EXTEND or ZERO_EXTEND wrapping
429 : the MEM, the bit insert sequence may have modified bits that
430 : affect the extension (e.g. the sign bit), so re-apply it. */
431 7 : rtx move_src;
432 7 : if (outer_mode != load_mem_mode)
433 : {
434 1 : rtx ext_op = dest;
435 1 : if (!load_elim)
436 : {
437 1 : ext_op = lowpart_subreg (load_mem_mode, dest, outer_mode);
438 1 : if (!ext_op)
439 : return false;
440 : }
441 1 : move_src = simplify_gen_unary (GET_CODE (SET_SRC (load)),
442 : outer_mode, ext_op, load_mem_mode);
443 : }
444 : else
445 : move_src = dest;
446 :
447 : /* In the non-elimination case the load insn is retained, so unshare
448 : its destination to avoid sharing a SUBREG between two insns. */
449 7 : rtx move = gen_rtx_SET (copy_rtx (SET_DEST (load)), move_src);
450 :
451 7 : start_sequence ();
452 7 : rtx_insn *insn = emit_insn (move);
453 7 : rtx_insn *seq = end_sequence ();
454 :
455 7 : if (recog_memoized (insn) < 0)
456 : return false;
457 :
458 7 : emit_insn_after (seq, load_insn);
459 : }
460 :
461 11 : if (dump_file)
462 : {
463 0 : fprintf (dump_file, "Store forwarding avoided with bit inserts:\n");
464 :
465 0 : FOR_EACH_VEC_ELT (stores, i, it)
466 : {
467 0 : if (stores.length () > 1)
468 : {
469 0 : fprintf (dump_file, "For: ");
470 0 : print_rtl_single (dump_file, it->store_insn);
471 : }
472 :
473 0 : fprintf (dump_file, "With sequence:\n");
474 :
475 0 : for (rtx_insn *insn = it->bits_insert_insns; insn;
476 0 : insn = NEXT_INSN (insn))
477 : {
478 0 : fprintf (dump_file, " ");
479 0 : print_rtl_single (dump_file, insn);
480 : }
481 : }
482 :
483 : }
484 :
485 11 : stats_sf_avoided++;
486 :
487 : /* Done, emit all the generated instructions and delete the stores.
488 : Note that STORES are in reverse program order. */
489 :
490 50 : FOR_EACH_VEC_ELT (stores, i, it)
491 : {
492 39 : emit_insn_after (it->bits_insert_insns, load_insn);
493 39 : emit_insn_after (it->store_saved_value_insn, load_insn);
494 : }
495 :
496 50 : FOR_EACH_VEC_ELT (stores, i, it)
497 : {
498 39 : emit_insn_before (it->save_store_value_insn, it->store_insn);
499 39 : delete_insn (it->store_insn);
500 : }
501 :
502 11 : df_insn_rescan (load_insn);
503 :
504 11 : if (load_elim)
505 : {
506 : /* Prevent a dangling rtx_insn * key after delete_insn. */
507 6 : m_bb_live_after.remove (load_insn);
508 6 : delete_insn (load_insn);
509 : }
510 :
511 : return true;
512 24 : }
513 :
514 : /* Try to modify BB so that expensive store forwarding cases are avoided. */
515 :
516 : void
517 74 : store_forwarding_analyzer::avoid_store_forwarding (basic_block bb)
518 : {
519 74 : if (!optimize_bb_for_speed_p (bb))
520 14 : return;
521 :
522 60 : m_bb_live_after.empty ();
523 :
524 60 : auto_vec<store_fwd_info, 8> store_exprs;
525 60 : rtx_insn *insn;
526 60 : unsigned int insn_cnt = 0;
527 :
528 : /* Iterate over the basic block's instructions detecting store instructions.
529 : Upon reaching a load instruction, check if any of the previously detected
530 : stores could result in store forwarding. In that case, try to reorder
531 : the load and store instructions. When we encounter instructions that
532 : might throw an exception, instruction dependencies, etc., clear the
533 : vector of detected stores and continue.
534 :
535 : Invariant: dropping a candidate from store_exprs (via it->remove or
536 : truncate) only removes it from the forwarding list; the store insn
537 : stays in the IR so later loads read its effect from memory. Only
538 : process_store_forwarding may delete the original store. */
539 1145 : FOR_BB_INSNS (bb, insn)
540 : {
541 1085 : if (!NONDEBUG_INSN_P (insn))
542 139 : continue;
543 :
544 946 : vec_rtx_properties properties;
545 946 : properties.add_insn (insn, false);
546 :
547 946 : rtx set = single_set (insn);
548 :
549 946 : if (!set || insn_could_throw_p (insn))
550 : {
551 62 : store_exprs.truncate (0);
552 62 : continue;
553 : }
554 :
555 : /* The inner mem RTX if INSN is a load, NULL_RTX otherwise. */
556 884 : rtx load_mem = SET_SRC (set);
557 :
558 884 : if (GET_CODE (load_mem) == ZERO_EXTEND
559 884 : || GET_CODE (load_mem) == SIGN_EXTEND)
560 47 : load_mem = XEXP (load_mem, 0);
561 :
562 884 : if (!MEM_P (load_mem))
563 795 : load_mem = NULL_RTX;
564 :
565 : /* The mem RTX if INSN is a store, NULL_RTX otherwise. */
566 884 : rtx store_mem = MEM_P (SET_DEST (set)) ? SET_DEST (set) : NULL_RTX;
567 :
568 : /* We cannot analyze memory RTXs that have unknown size. BLKmode
569 : memory is rejected as well, as there is no mode for the forwarded
570 : value, even when its size is known. */
571 338 : if ((store_mem && (GET_MODE (store_mem) == BLKmode
572 338 : || !MEM_SIZE_KNOWN_P (store_mem)
573 : || !MEM_SIZE (store_mem).is_constant ()))
574 884 : || (load_mem && (GET_MODE (load_mem) == BLKmode
575 89 : || !MEM_SIZE_KNOWN_P (load_mem)
576 : || !MEM_SIZE (load_mem).is_constant ())))
577 : {
578 0 : store_exprs.truncate (0);
579 0 : continue;
580 : }
581 :
582 884 : bool is_simple = !properties.has_asm
583 884 : && !properties.has_side_effects ();
584 884 : bool is_simple_store = is_simple
585 884 : && store_mem
586 884 : && !contains_mem_rtx_p (SET_SRC (set));
587 884 : bool is_simple_load = is_simple
588 884 : && load_mem
589 884 : && !contains_mem_rtx_p (SET_DEST (set));
590 :
591 884 : int removed_count = 0;
592 :
593 884 : if (is_simple_store)
594 : {
595 : /* Record store forwarding candidate. */
596 317 : store_fwd_info info;
597 317 : info.store_insn = insn;
598 317 : info.store_mem = store_mem;
599 317 : info.insn_cnt = insn_cnt;
600 317 : info.remove = false;
601 317 : info.forwarded = false;
602 317 : store_exprs.safe_push (info);
603 : }
604 :
605 884 : bool reads_mem = false;
606 884 : bool writes_mem = false;
607 3122 : for (auto ref : properties.refs ())
608 2238 : if (ref.is_mem ())
609 : {
610 465 : reads_mem |= ref.is_read ();
611 465 : writes_mem |= ref.is_write ();
612 : }
613 1773 : else if (ref.is_write ())
614 : {
615 : /* Drop store forwarding candidates when the address register is
616 : overwritten. */
617 670 : bool remove_rest = false;
618 670 : unsigned int i;
619 670 : store_fwd_info *it;
620 12037 : FOR_EACH_VEC_ELT_REVERSE (store_exprs, i, it)
621 : {
622 8459 : if (remove_rest
623 16901 : || reg_overlap_mentioned_p (regno_reg_rtx[ref.regno],
624 8442 : it->store_mem))
625 : {
626 20 : it->remove = true;
627 20 : removed_count++;
628 20 : remove_rest = true;
629 : }
630 : }
631 : }
632 :
633 884 : if (is_simple_load)
634 : {
635 : /* Process load for possible store forwarding cases.
636 : Possible newly created/moved stores, resulted from a successful
637 : forwarding, will be processed in subsequent iterations. */
638 89 : auto_vec<store_fwd_info> forwardings;
639 89 : bool partial_forwarding = false;
640 89 : bool remove_rest = false;
641 :
642 89 : bool vector_load = VECTOR_MODE_P (GET_MODE (load_mem));
643 :
644 89 : unsigned int i;
645 89 : store_fwd_info *it;
646 564 : FOR_EACH_VEC_ELT_REVERSE (store_exprs, i, it)
647 : {
648 386 : rtx store_mem = it->store_mem;
649 386 : HOST_WIDE_INT off_val;
650 :
651 386 : bool vector_store = VECTOR_MODE_P (GET_MODE (store_mem));
652 :
653 386 : if (remove_rest)
654 : {
655 9 : it->remove = true;
656 9 : removed_count++;
657 : }
658 377 : else if (vector_load ^ vector_store)
659 : {
660 : /* Vector stores followed by a non-vector load or the
661 : opposite, cause store_bit_field to generate non-canonical
662 : expressions, like (subreg:V4SI (reg:DI ...) 0)).
663 : Cases like that should be handled using vec_duplicate,
664 : so we reject the transformation in those cases. */
665 1 : it->remove = true;
666 1 : removed_count++;
667 1 : remove_rest = true;
668 1 : forwardings.truncate (0);
669 : }
670 376 : else if (is_store_forwarding (store_mem, load_mem, &off_val))
671 : {
672 : /* Check if moving this store after the load is legal. */
673 104 : bool write_dep = false;
674 104 : unsigned int j = store_exprs.length () - 1;
675 1950 : for (; j != i; j--)
676 : {
677 1846 : if (!store_exprs[j].forwarded
678 3302 : && output_dependence (store_mem,
679 1456 : store_exprs[j].store_mem))
680 : {
681 : write_dep = true;
682 : break;
683 : }
684 : }
685 :
686 104 : if (!write_dep)
687 : {
688 104 : it->forwarded = true;
689 104 : it->offset = off_val;
690 104 : forwardings.safe_push (*it);
691 : }
692 : else
693 : partial_forwarding = true;
694 :
695 104 : it->remove = true;
696 104 : removed_count++;
697 : }
698 272 : else if (true_dependence (store_mem, GET_MODE (store_mem),
699 : load_mem))
700 : {
701 : /* We cannot keep a store forwarding candidate if it possibly
702 : interferes with this load. */
703 2 : it->remove = true;
704 2 : removed_count++;
705 2 : remove_rest = true;
706 2 : forwardings.truncate (0);
707 : }
708 : }
709 :
710 139 : if (!forwardings.is_empty () && !partial_forwarding)
711 24 : process_store_forwarding (forwardings, insn, load_mem);
712 89 : }
713 :
714 : /* If we encounter a memory read/write that is not a simple
715 : store/load, flush all pending store candidates and continue.
716 : We can't make safe assumptions about the side-effects, but
717 : store-forwarding opportunities later in the BB should still
718 : be analyzed. */
719 884 : if ((writes_mem && !is_simple_store)
720 851 : || (reads_mem && !is_simple_load))
721 : {
722 48 : store_exprs.truncate (0);
723 48 : continue;
724 : }
725 :
726 836 : if (removed_count)
727 : {
728 27 : unsigned int i, j;
729 27 : store_fwd_info *it;
730 327 : VEC_ORDERED_REMOVE_IF (store_exprs, i, j, it, it->remove);
731 : }
732 :
733 : /* Don't consider store forwarding if the RTL instruction distance is
734 : more than PARAM_STORE_FORWARDING_MAX_DISTANCE and the cost checks
735 : are not disabled. */
736 836 : const bool unlimited_cost = (param_store_forwarding_max_distance == 0);
737 320 : if (!unlimited_cost && !store_exprs.is_empty ()
738 836 : && (store_exprs[0].insn_cnt
739 320 : + param_store_forwarding_max_distance <= insn_cnt))
740 62 : store_exprs.ordered_remove (0);
741 :
742 836 : insn_cnt++;
743 1085 : }
744 60 : }
745 :
746 : /* Update pass statistics. */
747 :
748 : void
749 30 : store_forwarding_analyzer::update_stats (function *fn)
750 : {
751 30 : statistics_counter_event (fn, "Cases of store forwarding detected: ",
752 30 : stats_sf_detected);
753 30 : statistics_counter_event (fn, "Cases of store forwarding avoided: ",
754 30 : stats_sf_avoided);
755 30 : }
756 :
757 : unsigned int
758 30 : pass_rtl_avoid_store_forwarding::execute (function *fn)
759 : {
760 30 : df_set_flags (DF_DEFER_INSN_RESCAN);
761 :
762 30 : init_alias_analysis ();
763 :
764 30 : store_forwarding_analyzer analyzer;
765 :
766 30 : basic_block bb;
767 104 : FOR_EACH_BB_FN (bb, fn)
768 74 : analyzer.avoid_store_forwarding (bb);
769 :
770 30 : end_alias_analysis ();
771 :
772 30 : analyzer.update_stats (fn);
773 :
774 30 : return 0;
775 30 : }
776 :
777 : } // anon namespace.
778 :
779 : rtl_opt_pass *
780 294196 : make_pass_rtl_avoid_store_forwarding (gcc::context *ctxt)
781 : {
782 294196 : return new pass_rtl_avoid_store_forwarding (ctxt);
783 : }
|
