Line data Source code
1 : /* If-conversion support.
2 : Copyright (C) 2000-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it
7 : under the terms of the GNU General Public License as published by
8 : the Free Software Foundation; either version 3, or (at your option)
9 : any later version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT
12 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 : or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
14 : License for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "target.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "cfghooks.h"
28 : #include "df.h"
29 : #include "memmodel.h"
30 : #include "tm_p.h"
31 : #include "expmed.h"
32 : #include "optabs.h"
33 : #include "regs.h"
34 : #include "emit-rtl.h"
35 : #include "recog.h"
36 :
37 : #include "cfgrtl.h"
38 : #include "cfganal.h"
39 : #include "cfgcleanup.h"
40 : #include "explow.h"
41 : #include "expr.h"
42 : #include "output.h"
43 : #include "cfgloop.h"
44 : #include "tree-pass.h"
45 : #include "dbgcnt.h"
46 : #include "shrink-wrap.h"
47 : #include "rtl-iter.h"
48 : #include "ifcvt.h"
49 :
50 : /* The number of instructions this pass may make run unconditionally in place
51 : of a branch. The conditional-execution path uses it as the number of insns
52 : it may predicate. The branchless paths use it as the number of insns they
53 : may speculate. The default charges the cost of an unpredictable branch,
54 : plus one. Some targets override it. */
55 :
56 : #ifndef MAX_CONDITIONAL_EXECUTE
57 : #define MAX_CONDITIONAL_EXECUTE \
58 : (BRANCH_COST (optimize_function_for_speed_p (cfun), false) \
59 : + 1)
60 : #endif
61 :
62 : #define NULL_BLOCK ((basic_block) NULL)
63 :
64 : /* Which of the three RTL if-conversion passes is running. ce1 runs before
65 : combine, ce2 after combine and ce3 after reload. The cost model treats the
66 : pre-combine run specially, so the phase is recorded here. */
67 : enum ifcvt_phase
68 : {
69 : IFCVT_BEFORE_COMBINE, /* ce1. */
70 : IFCVT_AFTER_COMBINE, /* ce2. */
71 : IFCVT_AFTER_RELOAD /* ce3. */
72 : };
73 :
74 : /* The if-conversion pass currently running. */
75 : static ifcvt_phase ifcvt_pass_phase;
76 :
77 : /* True if the target has the cbranchcc4 optab. */
78 : static bool have_cbranchcc4;
79 :
80 : /* # of IF-THEN or IF-THEN-ELSE blocks we looked at */
81 : static int num_possible_if_blocks;
82 :
83 : /* # of IF-THEN or IF-THEN-ELSE blocks were converted to conditional
84 : execution. */
85 : static int num_updated_if_blocks;
86 :
87 : /* # of changes made. */
88 : static int num_true_changes;
89 :
90 : /* Whether this pass over the function converted anything, so that another
91 : pass may find further opportunities in what it left behind. Every
92 : successful conversion sets it, not only the conditional-execution one. */
93 : static bool ifcvt_changed_p;
94 :
95 : /* Forward references. */
96 : static bool noce_operand_ok (const_rtx);
97 : static void merge_if_block (ce_if_block *);
98 : static bool find_cond_trap (basic_block, edge, edge);
99 : static bool cond_exec_find_if_block (ce_if_block *);
100 : static bool find_if_case_1 (basic_block, edge, edge);
101 : static bool find_if_case_2 (basic_block, edge, edge);
102 : static bool dead_or_predicable (basic_block, basic_block, basic_block,
103 : edge, bool);
104 : static rtx_insn *block_has_only_trap (basic_block);
105 : static void init_noce_multiple_sets_info (basic_block,
106 : auto_delete_vec<noce_multiple_sets_info> &);
107 : static bool noce_convert_multiple_sets_1 (noce_if_info *, rtx,
108 : auto_delete_vec<noce_multiple_sets_info> &,
109 : auto_delete_vec<noce_multiple_sets_info> &, unsigned,
110 : const vec<unsigned> &, int *, bool *);
111 :
112 : /* Count the number of non-jump active insns in BB. */
113 :
114 : static int
115 0 : count_bb_insns (const_basic_block bb)
116 : {
117 0 : int count = 0;
118 0 : rtx_insn *insn = BB_HEAD (bb);
119 :
120 0 : while (1)
121 : {
122 0 : if (active_insn_p (insn) && !JUMP_P (insn))
123 0 : count++;
124 :
125 0 : if (insn == BB_END (bb))
126 : break;
127 0 : insn = NEXT_INSN (insn);
128 : }
129 :
130 0 : return count;
131 : }
132 :
133 : /* Determine whether the total insn_cost on non-jump insns in
134 : basic block BB is less than MAX_COST. This function returns
135 : false if the cost of any instruction could not be estimated.
136 :
137 : The cost of the non-jump insns in BB is scaled by REG_BR_PROB_BASE
138 : as those insns are being speculated. MAX_COST is scaled with SCALE
139 : plus a small fudge factor. */
140 :
141 : static bool
142 3093576 : cheap_bb_rtx_cost_p (const_basic_block bb,
143 : profile_probability prob, int max_cost)
144 : {
145 3093576 : int count = 0;
146 3093576 : rtx_insn *insn = BB_HEAD (bb);
147 3093576 : bool speed = optimize_bb_for_speed_p (bb);
148 3093576 : int scale = prob.initialized_p () ? prob.to_reg_br_prob_base ()
149 : : REG_BR_PROB_BASE;
150 :
151 : /* Set scale to REG_BR_PROB_BASE to void the identical scaling
152 : applied to insn_cost when optimizing for size. Only do
153 : this after combine because if-conversion might interfere with
154 : passes before combine.
155 :
156 : Use optimize_function_for_speed_p instead of the pre-defined
157 : variable speed to make sure it is set to same value for all
158 : basic blocks in one if-conversion transformation. */
159 3093576 : if (!optimize_function_for_speed_p (cfun)
160 3093576 : && ifcvt_pass_phase != IFCVT_BEFORE_COMBINE)
161 : scale = REG_BR_PROB_BASE;
162 : /* Our branch probability/scaling factors are just estimates and don't
163 : account for cases where we can get speculation for free and other
164 : secondary benefits. So we fudge the scale factor to make speculating
165 : appear a little more profitable when optimizing for performance. */
166 : else
167 3042371 : scale += REG_BR_PROB_BASE / 8;
168 :
169 :
170 3093576 : max_cost *= scale;
171 :
172 13162434 : while (1)
173 : {
174 16256010 : if (NONJUMP_INSN_P (insn))
175 : {
176 : /* Inline-asm's cost is not very estimatable.
177 : It could be a costly instruction but the
178 : estimate would be the same as a non costly
179 : instruction. */
180 4633397 : if (asm_noperands (PATTERN (insn)) >= 0)
181 : return false;
182 :
183 4630696 : int cost = insn_cost (insn, speed) * REG_BR_PROB_BASE;
184 4630696 : if (cost == 0)
185 : return false;
186 :
187 : /* If this instruction is the load or set of a "stack" register,
188 : such as a floating point register on x87, then the cost of
189 : speculatively executing this insn may need to include
190 : the additional cost of popping its result off of the
191 : register stack. Unfortunately, correctly recognizing and
192 : accounting for this additional overhead is tricky, so for
193 : now we simply prohibit such speculative execution. */
194 : #ifdef STACK_REGS
195 4579213 : {
196 4579213 : rtx set = single_set (insn);
197 4579213 : if (set && STACK_REG_P (SET_DEST (set)))
198 : return false;
199 : }
200 : #endif
201 :
202 4575325 : count += cost;
203 4575325 : if (count >= max_cost)
204 : return false;
205 : }
206 11622613 : else if (CALL_P (insn))
207 : return false;
208 :
209 13755253 : if (insn == BB_END (bb))
210 : break;
211 13162434 : insn = NEXT_INSN (insn);
212 13162434 : }
213 :
214 : return true;
215 : }
216 :
217 : /* Return the first non-jump active insn in the basic block. */
218 :
219 : static rtx_insn *
220 1544251 : first_active_insn (basic_block bb)
221 : {
222 1544251 : rtx_insn *insn = BB_HEAD (bb);
223 :
224 1544251 : if (LABEL_P (insn))
225 : {
226 283139 : if (insn == BB_END (bb))
227 : return NULL;
228 283139 : insn = NEXT_INSN (insn);
229 : }
230 :
231 5237300 : while (NOTE_P (insn) || DEBUG_INSN_P (insn))
232 : {
233 3693049 : if (insn == BB_END (bb))
234 : return NULL;
235 3693049 : insn = NEXT_INSN (insn);
236 : }
237 :
238 1544251 : if (JUMP_P (insn))
239 0 : return NULL;
240 :
241 : return insn;
242 : }
243 :
244 : /* Return the last non-jump active insn in the basic block. */
245 :
246 : static rtx_insn *
247 2597609 : last_active_insn (basic_block bb, bool skip_use_p)
248 : {
249 2597609 : rtx_insn *insn = BB_END (bb);
250 2597609 : rtx_insn *head = BB_HEAD (bb);
251 :
252 2597609 : while (NOTE_P (insn)
253 : || JUMP_P (insn)
254 : || DEBUG_INSN_P (insn)
255 6376586 : || (skip_use_p
256 0 : && NONJUMP_INSN_P (insn)
257 0 : && GET_CODE (PATTERN (insn)) == USE))
258 : {
259 3779216 : if (insn == head)
260 : return NULL;
261 3778977 : insn = PREV_INSN (insn);
262 : }
263 :
264 2597370 : if (LABEL_P (insn))
265 96 : return NULL;
266 :
267 : return insn;
268 : }
269 :
270 : /* Return the active insn before INSN inside basic block CURR_BB. */
271 :
272 : static rtx_insn *
273 0 : find_active_insn_before (basic_block curr_bb, rtx_insn *insn)
274 : {
275 0 : if (!insn || insn == BB_HEAD (curr_bb))
276 : return NULL;
277 :
278 0 : while ((insn = PREV_INSN (insn)) != NULL_RTX)
279 : {
280 0 : if (NONJUMP_INSN_P (insn) || JUMP_P (insn) || CALL_P (insn))
281 : break;
282 :
283 : /* No other active insn all the way to the start of the basic block. */
284 0 : if (insn == BB_HEAD (curr_bb))
285 : return NULL;
286 : }
287 :
288 : return insn;
289 : }
290 :
291 : /* Return the active insn after INSN inside basic block CURR_BB. */
292 :
293 : static rtx_insn *
294 0 : find_active_insn_after (basic_block curr_bb, rtx_insn *insn)
295 : {
296 0 : if (!insn || insn == BB_END (curr_bb))
297 : return NULL;
298 :
299 0 : while ((insn = NEXT_INSN (insn)) != NULL_RTX)
300 : {
301 0 : if (NONJUMP_INSN_P (insn) || JUMP_P (insn) || CALL_P (insn))
302 : break;
303 :
304 : /* No other active insn all the way to the end of the basic block. */
305 0 : if (insn == BB_END (curr_bb))
306 : return NULL;
307 : }
308 :
309 : return insn;
310 : }
311 :
312 : /* Return the basic block reached by falling through the basic block BB. */
313 :
314 : static basic_block
315 0 : block_fallthru (basic_block bb)
316 : {
317 0 : edge e = find_fallthru_edge (bb->succs);
318 :
319 0 : return (e) ? e->dest : NULL_BLOCK;
320 : }
321 :
322 : /* Return true if RTXs A and B can be safely interchanged. */
323 :
324 : static bool
325 498052 : rtx_interchangeable_p (const_rtx a, const_rtx b)
326 : {
327 498052 : if (!rtx_equal_p (a, b))
328 : return false;
329 :
330 184416 : if (GET_CODE (a) != MEM)
331 : return true;
332 :
333 : /* A dead type-unsafe memory reference is legal, but a live type-unsafe memory
334 : reference is not. Interchanging a dead type-unsafe memory reference with
335 : a live type-safe one creates a live type-unsafe memory reference, in other
336 : words, it makes the program illegal.
337 : We check here conservatively whether the two memory references have equal
338 : memory attributes. */
339 :
340 2154 : return mem_attrs_eq_p (get_mem_attrs (a), get_mem_attrs (b));
341 : }
342 :
343 :
344 : /* Go through the insns from START to END, converting them to conditional
345 : execution format if possible. CE_INFO describes the if-block being
346 : converted and is only read by the target IFCVT_MODIFY_INSN macro. TEST is
347 : the conditional-execution test to predicate them on and PROB_VAL the
348 : probability that it holds. MOD_OK allows one insn that modifies TEST, in
349 : which case it must be the last one. Return TRUE if all of the non-note
350 : insns were processed. */
351 :
352 : static bool
353 0 : cond_exec_process_insns (ce_if_block *ce_info ATTRIBUTE_UNUSED,
354 : rtx_insn *start, rtx end, rtx test,
355 : profile_probability prob_val, bool mod_ok)
356 : {
357 0 : bool must_be_last = false;
358 0 : rtx_insn *insn;
359 0 : rtx xtest;
360 0 : rtx pattern;
361 :
362 0 : if (!start || !end)
363 : return false;
364 :
365 0 : for (insn = start; ; insn = NEXT_INSN (insn))
366 : {
367 : /* dwarf2out can't cope with conditional prologues. */
368 0 : if (NOTE_P (insn) && NOTE_KIND (insn) == NOTE_INSN_PROLOGUE_END)
369 : return false;
370 :
371 0 : if (NOTE_P (insn) || DEBUG_INSN_P (insn))
372 0 : goto insn_done;
373 :
374 0 : gcc_assert (NONJUMP_INSN_P (insn) || CALL_P (insn));
375 :
376 : /* dwarf2out can't cope with conditional unwind info. */
377 0 : if (RTX_FRAME_RELATED_P (insn))
378 : return false;
379 :
380 : /* Remove USE insns that get in the way. */
381 0 : if (reload_completed && GET_CODE (PATTERN (insn)) == USE)
382 : {
383 : /* ??? Ug. Actually unlinking the thing is problematic,
384 : given what we'd have to coordinate with our callers. */
385 0 : SET_INSN_DELETED (insn);
386 0 : goto insn_done;
387 : }
388 :
389 : /* Last insn wasn't last? */
390 0 : if (must_be_last)
391 : return false;
392 :
393 0 : if (modified_in_p (test, insn))
394 : {
395 0 : if (!mod_ok)
396 : return false;
397 : must_be_last = true;
398 : }
399 :
400 : /* Now build the conditional form of the instruction. */
401 0 : pattern = PATTERN (insn);
402 0 : xtest = copy_rtx (test);
403 :
404 : /* If this is already a COND_EXEC, rewrite the test to be an AND of the
405 : two conditions. */
406 0 : if (GET_CODE (pattern) == COND_EXEC)
407 : {
408 0 : if (GET_MODE (xtest) != GET_MODE (COND_EXEC_TEST (pattern)))
409 : return false;
410 :
411 0 : xtest = gen_rtx_AND (GET_MODE (xtest), xtest,
412 : COND_EXEC_TEST (pattern));
413 0 : pattern = COND_EXEC_CODE (pattern);
414 : }
415 :
416 0 : pattern = gen_rtx_COND_EXEC (VOIDmode, xtest, pattern);
417 :
418 : /* If the machine needs to modify the insn being conditionally executed,
419 : say for example to force a constant integer operand into a temp
420 : register, do so here. */
421 : #ifdef IFCVT_MODIFY_INSN
422 : IFCVT_MODIFY_INSN (ce_info, pattern, insn);
423 : if (! pattern)
424 : return false;
425 : #endif
426 :
427 0 : validate_change (insn, &PATTERN (insn), pattern, 1);
428 :
429 0 : if (CALL_P (insn) && prob_val.initialized_p ())
430 0 : validate_change (insn, ®_NOTES (insn),
431 : gen_rtx_INT_LIST ((machine_mode) REG_BR_PROB,
432 : prob_val.to_reg_br_prob_note (),
433 : REG_NOTES (insn)), 1);
434 :
435 0 : insn_done:
436 0 : if (insn == end)
437 : break;
438 0 : }
439 :
440 : return true;
441 : }
442 :
443 : /* Return the condition for a jump. Do not do any special processing. */
444 :
445 : static rtx
446 196464 : cond_exec_get_condition (rtx_insn *jump, bool get_reversed = false)
447 : {
448 196464 : rtx test_if, cond;
449 :
450 196464 : if (any_condjump_p (jump))
451 196464 : test_if = SET_SRC (pc_set (jump));
452 : else
453 : return NULL_RTX;
454 196464 : cond = XEXP (test_if, 0);
455 :
456 : /* If this branches to JUMP_LABEL when the condition is false,
457 : reverse the condition. */
458 196464 : if (get_reversed
459 196464 : || (GET_CODE (XEXP (test_if, 2)) == LABEL_REF
460 0 : && label_ref_label (XEXP (test_if, 2))
461 0 : == JUMP_LABEL (jump)))
462 : {
463 98232 : enum rtx_code rev = reversed_comparison_code (cond, jump);
464 98232 : if (rev == UNKNOWN)
465 : return NULL_RTX;
466 :
467 98232 : cond = gen_rtx_fmt_ee (rev, GET_MODE (cond), XEXP (cond, 0),
468 : XEXP (cond, 1));
469 : }
470 :
471 : return cond;
472 : }
473 :
474 : /* Given a simple IF-THEN or IF-THEN-ELSE block described by CE_INFO, attempt
475 : to convert it to conditional execution. DO_MULTIPLE_P allows CE_INFO's
476 : recorded chain of && or || test blocks to be converted as well. Return
477 : TRUE if we were successful at converting the block. */
478 :
479 : static bool
480 0 : cond_exec_process_if_block (ce_if_block *ce_info, bool do_multiple_p)
481 : {
482 0 : basic_block test_bb = ce_info->test_bb; /* last test block */
483 0 : basic_block then_bb = ce_info->then_bb; /* THEN */
484 0 : basic_block else_bb = ce_info->else_bb; /* ELSE or NULL */
485 0 : rtx test_expr; /* expression in IF_THEN_ELSE that is tested */
486 0 : rtx_insn *then_start; /* first insn in THEN block */
487 0 : rtx_insn *then_end; /* last insn + 1 in THEN block */
488 0 : rtx_insn *else_start = NULL; /* first insn in ELSE block or NULL */
489 0 : rtx_insn *else_end = NULL; /* last insn + 1 in ELSE block */
490 0 : int max; /* max # of insns to convert. */
491 0 : bool then_mod_ok; /* whether conditional mods are ok in THEN */
492 0 : rtx true_expr; /* test for else block insns */
493 0 : rtx false_expr; /* test for then block insns */
494 0 : profile_probability true_prob_val;/* probability of else block */
495 0 : profile_probability false_prob_val;/* probability of then block */
496 0 : rtx_insn *then_last_head = NULL; /* Last match at the head of THEN */
497 0 : rtx_insn *else_last_head = NULL; /* Last match at the head of ELSE */
498 0 : rtx_insn *then_first_tail = NULL; /* First match at the tail of THEN */
499 0 : rtx_insn *else_first_tail = NULL; /* First match at the tail of ELSE */
500 0 : int then_n_insns, else_n_insns, n_insns;
501 0 : enum rtx_code false_code;
502 0 : rtx note;
503 :
504 : /* If test is comprised of && or || elements, and we've failed at handling
505 : all of them together, just use the last test if it is the special case of
506 : && elements without an ELSE block. */
507 0 : if (!do_multiple_p && ce_info->num_multiple_test_blocks)
508 : {
509 0 : if (else_bb || ! ce_info->and_and_p)
510 : return false;
511 :
512 0 : ce_info->test_bb = test_bb = ce_info->last_test_bb;
513 0 : ce_info->num_multiple_test_blocks = 0;
514 0 : ce_info->num_and_and_blocks = 0;
515 0 : ce_info->num_or_or_blocks = 0;
516 : }
517 :
518 : /* Find the conditional jump to the ELSE or JOIN part, and isolate
519 : the test. */
520 0 : test_expr = cond_exec_get_condition (BB_END (test_bb));
521 0 : if (! test_expr)
522 : return false;
523 :
524 : /* If the conditional jump is more than just a conditional jump,
525 : then we cannot do conditional execution conversion on this block. */
526 0 : if (! onlyjump_p (BB_END (test_bb)))
527 : return false;
528 :
529 : /* Collect the bounds of where we're to search, skipping any labels, jumps
530 : and notes at the beginning and end of the block. Then count the total
531 : number of insns and see if it is small enough to convert. */
532 0 : then_start = first_active_insn (then_bb);
533 0 : then_end = last_active_insn (then_bb, true);
534 0 : then_n_insns = ce_info->num_then_insns = count_bb_insns (then_bb);
535 0 : n_insns = then_n_insns;
536 0 : max = MAX_CONDITIONAL_EXECUTE;
537 :
538 0 : if (else_bb)
539 : {
540 0 : int n_matching;
541 :
542 0 : max *= 2;
543 0 : else_start = first_active_insn (else_bb);
544 0 : else_end = last_active_insn (else_bb, true);
545 0 : else_n_insns = ce_info->num_else_insns = count_bb_insns (else_bb);
546 0 : n_insns += else_n_insns;
547 :
548 : /* Look for matching sequences at the head and tail of the two blocks,
549 : and limit the range of insns to be converted if possible. */
550 0 : n_matching = flow_find_cross_jump (then_bb, else_bb,
551 : &then_first_tail, &else_first_tail,
552 : NULL);
553 0 : if (then_first_tail == BB_HEAD (then_bb))
554 0 : then_start = then_end = NULL;
555 0 : if (else_first_tail == BB_HEAD (else_bb))
556 0 : else_start = else_end = NULL;
557 :
558 0 : if (n_matching > 0)
559 : {
560 0 : if (then_end)
561 0 : then_end = find_active_insn_before (then_bb, then_first_tail);
562 0 : if (else_end)
563 0 : else_end = find_active_insn_before (else_bb, else_first_tail);
564 0 : n_insns -= 2 * n_matching;
565 : }
566 :
567 0 : if (then_start
568 0 : && else_start
569 : && then_n_insns > n_matching
570 0 : && else_n_insns > n_matching)
571 : {
572 0 : int longest_match = MIN (then_n_insns - n_matching,
573 : else_n_insns - n_matching);
574 0 : n_matching
575 0 : = flow_find_head_matching_sequence (then_bb, else_bb,
576 : &then_last_head,
577 : &else_last_head,
578 : longest_match);
579 :
580 0 : if (n_matching > 0)
581 : {
582 0 : rtx_insn *insn;
583 :
584 : /* We won't pass the insns in the head sequence to
585 : cond_exec_process_insns, so we need to test them here
586 : to make sure that they don't clobber the condition. */
587 0 : for (insn = BB_HEAD (then_bb);
588 0 : insn != NEXT_INSN (then_last_head);
589 0 : insn = NEXT_INSN (insn))
590 0 : if (!LABEL_P (insn) && !NOTE_P (insn)
591 0 : && !DEBUG_INSN_P (insn)
592 0 : && modified_in_p (test_expr, insn))
593 : return false;
594 : }
595 :
596 0 : if (then_last_head == then_end)
597 0 : then_start = then_end = NULL;
598 0 : if (else_last_head == else_end)
599 0 : else_start = else_end = NULL;
600 :
601 0 : if (n_matching > 0)
602 : {
603 0 : if (then_start)
604 0 : then_start = find_active_insn_after (then_bb, then_last_head);
605 0 : if (else_start)
606 0 : else_start = find_active_insn_after (else_bb, else_last_head);
607 0 : n_insns -= 2 * n_matching;
608 : }
609 : }
610 : }
611 :
612 0 : if (n_insns > max)
613 : return false;
614 :
615 : /* Map test_expr/test_jump into the appropriate MD tests to use on
616 : the conditionally executed code. */
617 :
618 0 : true_expr = test_expr;
619 :
620 0 : false_code = reversed_comparison_code (true_expr, BB_END (test_bb));
621 0 : if (false_code != UNKNOWN)
622 0 : false_expr = gen_rtx_fmt_ee (false_code, GET_MODE (true_expr),
623 : XEXP (true_expr, 0), XEXP (true_expr, 1));
624 : else
625 : false_expr = NULL_RTX;
626 :
627 : #ifdef IFCVT_MODIFY_TESTS
628 : /* If the machine description needs to modify the tests, such as setting a
629 : conditional execution register from a comparison, it can do so here. */
630 : IFCVT_MODIFY_TESTS (ce_info, true_expr, false_expr);
631 :
632 : /* See if the conversion failed. */
633 : if (!true_expr || !false_expr)
634 : goto fail;
635 : #endif
636 :
637 0 : note = find_reg_note (BB_END (test_bb), REG_BR_PROB, NULL_RTX);
638 0 : if (note)
639 : {
640 0 : true_prob_val = profile_probability::from_reg_br_prob_note (XINT (note, 0));
641 0 : false_prob_val = true_prob_val.invert ();
642 : }
643 : else
644 : {
645 : true_prob_val = profile_probability::uninitialized ();
646 : false_prob_val = profile_probability::uninitialized ();
647 : }
648 :
649 : /* If we have && or || tests, do them here. These tests are in the adjacent
650 : blocks after the first block containing the test. */
651 0 : if (ce_info->num_multiple_test_blocks > 0)
652 : {
653 0 : basic_block bb = test_bb;
654 0 : basic_block last_test_bb = ce_info->last_test_bb;
655 :
656 0 : if (! false_expr)
657 0 : goto fail;
658 :
659 0 : do
660 : {
661 0 : rtx_insn *start, *end;
662 0 : rtx t, f;
663 0 : enum rtx_code f_code;
664 :
665 0 : bb = block_fallthru (bb);
666 0 : start = first_active_insn (bb);
667 0 : end = last_active_insn (bb, true);
668 0 : if (start
669 0 : && ! cond_exec_process_insns (ce_info, start, end, false_expr,
670 : false_prob_val, false))
671 0 : goto fail;
672 :
673 : /* If the conditional jump is more than just a conditional jump, then
674 : we cannot do conditional execution conversion on this block. */
675 0 : if (! onlyjump_p (BB_END (bb)))
676 0 : goto fail;
677 :
678 : /* Find the conditional jump and isolate the test. */
679 0 : t = cond_exec_get_condition (BB_END (bb));
680 0 : if (! t)
681 0 : goto fail;
682 :
683 0 : f_code = reversed_comparison_code (t, BB_END (bb));
684 0 : if (f_code == UNKNOWN)
685 0 : goto fail;
686 :
687 0 : f = gen_rtx_fmt_ee (f_code, GET_MODE (t), XEXP (t, 0), XEXP (t, 1));
688 0 : if (ce_info->and_and_p)
689 : {
690 0 : t = gen_rtx_AND (GET_MODE (t), true_expr, t);
691 0 : f = gen_rtx_IOR (GET_MODE (t), false_expr, f);
692 : }
693 : else
694 : {
695 0 : t = gen_rtx_IOR (GET_MODE (t), true_expr, t);
696 0 : f = gen_rtx_AND (GET_MODE (t), false_expr, f);
697 : }
698 :
699 : /* If the machine description needs to modify the tests, such as
700 : setting a conditional execution register from a comparison, it can
701 : do so here. */
702 : #ifdef IFCVT_MODIFY_MULTIPLE_TESTS
703 : IFCVT_MODIFY_MULTIPLE_TESTS (ce_info, bb, t, f);
704 :
705 : /* See if the conversion failed. */
706 : if (!t || !f)
707 : goto fail;
708 : #endif
709 :
710 0 : true_expr = t;
711 0 : false_expr = f;
712 : }
713 0 : while (bb != last_test_bb);
714 : }
715 :
716 : /* For IF-THEN-ELSE blocks, we don't allow modifications of the test
717 : on then THEN block. */
718 0 : then_mod_ok = (else_bb == NULL_BLOCK);
719 :
720 : /* Go through the THEN and ELSE blocks converting the insns if possible
721 : to conditional execution. */
722 :
723 0 : if (then_end
724 0 : && (! false_expr
725 0 : || ! cond_exec_process_insns (ce_info, then_start, then_end,
726 : false_expr, false_prob_val,
727 : then_mod_ok)))
728 0 : goto fail;
729 :
730 0 : if (else_bb && else_end
731 0 : && ! cond_exec_process_insns (ce_info, else_start, else_end,
732 : true_expr, true_prob_val, true))
733 0 : goto fail;
734 :
735 : /* If we cannot apply the changes, fail. Do not go through the normal fail
736 : processing, since apply_change_group will call cancel_changes. */
737 0 : if (! apply_change_group ())
738 : {
739 : #ifdef IFCVT_MODIFY_CANCEL
740 : /* Cancel any machine dependent changes. */
741 : IFCVT_MODIFY_CANCEL (ce_info);
742 : #endif
743 : return false;
744 : }
745 :
746 : #ifdef IFCVT_MODIFY_FINAL
747 : /* Do any machine dependent final modifications. */
748 : IFCVT_MODIFY_FINAL (ce_info);
749 : #endif
750 :
751 : /* Conversion succeeded. */
752 0 : if (dump_file)
753 0 : fprintf (dump_file, "%d insn%s converted to conditional execution.\n",
754 : n_insns, (n_insns == 1) ? " was" : "s were");
755 :
756 : /* Merge the blocks! If we had matching sequences, make sure to delete one
757 : copy at the appropriate location first: delete the copy in the THEN branch
758 : for a tail sequence so that the remaining one is executed last for both
759 : branches, and delete the copy in the ELSE branch for a head sequence so
760 : that the remaining one is executed first for both branches. */
761 0 : if (then_first_tail)
762 : {
763 0 : rtx_insn *from = then_first_tail;
764 0 : if (!INSN_P (from))
765 0 : from = find_active_insn_after (then_bb, from);
766 0 : delete_insn_chain (from, get_last_bb_insn (then_bb), false);
767 : }
768 0 : if (else_last_head)
769 0 : delete_insn_chain (first_active_insn (else_bb), else_last_head, false);
770 :
771 0 : merge_if_block (ce_info);
772 0 : ifcvt_changed_p = true;
773 0 : return true;
774 :
775 0 : fail:
776 : #ifdef IFCVT_MODIFY_CANCEL
777 : /* Cancel any machine dependent changes. */
778 : IFCVT_MODIFY_CANCEL (ce_info);
779 : #endif
780 :
781 0 : cancel_changes (0);
782 0 : return false;
783 : }
784 :
785 : /* Return the condition to use when the arms of IF_INFO's if-region are
786 : swapped, and set *CODE to its comparison code. When the reversed condition
787 : has no rtx of its own the original condition is returned alongside the
788 : reversed code, which is the form the store-flag and conditional-move
789 : expanders take. *CODE is UNKNOWN if the condition cannot be reversed. */
790 :
791 : static rtx
792 181711 : noce_reversed_cond (noce_if_info *if_info, enum rtx_code *code)
793 : {
794 181711 : if (if_info->rev_cond)
795 : {
796 181711 : *code = GET_CODE (if_info->rev_cond);
797 181711 : return if_info->rev_cond;
798 : }
799 :
800 0 : *code = reversed_comparison_code (if_info->cond, if_info->jump);
801 0 : return if_info->cond;
802 : }
803 :
804 : /* Return the comparison code for reversed condition for IF_INFO,
805 : or UNKNOWN if reversing the condition is not possible. */
806 :
807 : static inline enum rtx_code
808 105594 : noce_reversed_cond_code (noce_if_info *if_info)
809 : {
810 105594 : enum rtx_code code;
811 11316 : noce_reversed_cond (if_info, &code);
812 105594 : return code;
813 : }
814 :
815 : /* A register definition and its dependency level. */
816 :
817 : struct noce_parallel_cost_node
818 : {
819 : rtx dest;
820 : unsigned int level;
821 : };
822 :
823 : /* Return true if INSN is a unit-cost register SET supported by the parallel
824 : cost model. Store the SET in *SET. */
825 :
826 : static bool
827 477606 : noce_parallel_costed_insn_p (rtx_insn *insn, bool speed_p, rtx *set)
828 : {
829 477606 : *set = single_set (insn);
830 477606 : if (!*set)
831 : return false;
832 :
833 477384 : rtx dest = SET_DEST (*set);
834 477384 : rtx src = SET_SRC (*set);
835 :
836 477384 : return (REG_P (dest)
837 475782 : && !contains_mem_rtx_p (src)
838 455758 : && !side_effects_p (src)
839 455758 : && !may_trap_p (src)
840 930292 : && set_rtx_cost (*set, speed_p) == COSTS_N_INSNS (1));
841 : }
842 :
843 : /* Return the dependency level for an instruction reading SRC. A use of a
844 : register defined at level N requires level N + 1. */
845 :
846 : static unsigned int
847 433528 : noce_dependency_level (const vec<noce_parallel_cost_node> &defs, rtx src)
848 : {
849 433528 : unsigned int level = 0;
850 :
851 1313093 : for (unsigned int i = 0; i < defs.length (); ++i)
852 879565 : if (reg_overlap_mentioned_p (defs[i].dest, src))
853 879565 : level = MAX (level, defs[i].level + 1);
854 :
855 433528 : return level;
856 : }
857 :
858 : /* Estimate the cost of SEQ using the target issue rate. Unit-cost register
859 : operations are grouped by RAW dependency level. The cost of each level is
860 : its instruction count divided by the issue rate, rounded up. Fall back to
861 : serial cost if any instruction cannot be modeled.
862 :
863 : In an instrumented build of all SPEC CPU2017 Integer rate benchmarks
864 : for RISC-V, the analysis handled 16,947 of 30,120 candidates (56.26%).
865 : It reduced the cost for 15,024 of the handled candidates (88.65%).
866 : The parallel-to-serial cost ratios for the handled candidates were:
867 :
868 : 0.8 < ratio <= 1.0 11.54 percent
869 : 0.6 < ratio <= 0.8 47.31 percent
870 : 0.4 < ratio <= 0.6 40.24 percent
871 : 0.2 < ratio <= 0.4 0.91 percent
872 : 0.0 < ratio <= 0.2 0.00 percent. */
873 :
874 : static unsigned int
875 188334 : noce_parallel_seq_cost (rtx_insn *seq, bool speed_p)
876 : {
877 188334 : unsigned int serial_cost = seq_cost (seq, speed_p);
878 :
879 188334 : if (!speed_p || !targetm.sched.issue_rate)
880 : return serial_cost;
881 :
882 128211 : unsigned int issue_rate = MAX (targetm.sched.issue_rate (), 1);
883 :
884 : /* Track prior definitions and instruction counts per dependency level. */
885 128211 : auto_vec<noce_parallel_cost_node> defs;
886 128211 : auto_vec<unsigned int> insns_per_level;
887 :
888 561739 : for (rtx_insn *insn = seq; insn; insn = NEXT_INSN (insn))
889 : {
890 477606 : rtx set;
891 :
892 477606 : if (!NONDEBUG_INSN_P (insn))
893 0 : continue;
894 :
895 477606 : if (!noce_parallel_costed_insn_p (insn, speed_p, &set))
896 44078 : return serial_cost;
897 :
898 433528 : unsigned int level = noce_dependency_level (defs, SET_SRC (set));
899 :
900 433528 : if (insns_per_level.length () <= level)
901 251577 : insns_per_level.safe_grow_cleared (level + 1, true);
902 433528 : ++insns_per_level[level];
903 :
904 433528 : noce_parallel_cost_node node = { SET_DEST (set), level };
905 433528 : defs.safe_push (node);
906 : }
907 :
908 : unsigned int parallel_cost = 0;
909 295360 : for (unsigned int i = 0; i < insns_per_level.length (); ++i)
910 211227 : parallel_cost += COSTS_N_INSNS (CEIL (insns_per_level[i], issue_rate));
911 :
912 : /* set_rtx_cost and insn_cost can disagree, so cap the estimate at the
913 : serial cost. */
914 84133 : return MIN (serial_cost, parallel_cost);
915 128211 : }
916 :
917 : /* Return true if SEQ is a good candidate as a replacement for the
918 : if-convertible sequence described in IF_INFO.
919 : This is the default implementation that targets can override
920 : through a target hook. */
921 :
922 : bool
923 188334 : default_noce_conversion_profitable_p (rtx_insn *seq,
924 : noce_if_info *if_info)
925 : {
926 188334 : bool speed_p = if_info->speed_p;
927 :
928 : /* Cost up the new sequence. */
929 188334 : unsigned int cost = noce_parallel_seq_cost (seq, speed_p);
930 :
931 188334 : if (cost <= if_info->original_cost)
932 : return true;
933 :
934 : /* When compiling for size, we can make a reasonably accurately guess
935 : at the size growth. When compiling for speed, use the maximum. */
936 84167 : return speed_p && cost <= if_info->max_seq_cost;
937 : }
938 :
939 : /* Helper function for noce_try_store_flag*. Return NULL_RTX on failure,
940 : including when REVERSEP is requested but the condition cannot be
941 : reversed; callers may rely on this and need not pre-check. */
942 :
943 : static rtx
944 58776 : noce_emit_store_flag (noce_if_info *if_info, rtx x, bool reversep,
945 : int normalize)
946 : {
947 58776 : rtx cond = if_info->cond;
948 58776 : bool cond_complex;
949 58776 : enum rtx_code code;
950 :
951 58776 : cond_complex = (! general_operand (XEXP (cond, 0), VOIDmode)
952 58776 : || ! general_operand (XEXP (cond, 1), VOIDmode));
953 :
954 : /* If earliest == jump, or when the condition is complex, try to
955 : build the store_flag insn directly. */
956 :
957 34859 : if (cond_complex)
958 : {
959 23917 : rtx set = pc_set (if_info->jump);
960 23917 : cond = XEXP (SET_SRC (set), 0);
961 23917 : if (GET_CODE (XEXP (SET_SRC (set), 2)) == LABEL_REF
962 23917 : && label_ref_label (XEXP (SET_SRC (set), 2)) == JUMP_LABEL (if_info->jump))
963 0 : reversep = !reversep;
964 23917 : if (if_info->then_else_reversed)
965 21159 : reversep = !reversep;
966 : }
967 34859 : else if (reversep
968 17867 : && if_info->rev_cond
969 17867 : && general_operand (XEXP (if_info->rev_cond, 0), VOIDmode)
970 52726 : && general_operand (XEXP (if_info->rev_cond, 1), VOIDmode))
971 : {
972 17867 : cond = if_info->rev_cond;
973 17867 : reversep = false;
974 : }
975 :
976 58776 : if (reversep)
977 4421 : code = reversed_comparison_code (cond, if_info->jump);
978 : else
979 54355 : code = GET_CODE (cond);
980 :
981 : /* reversed_comparison_code returns UNKNOWN for an unordered code, or a
982 : CC-mode compare it cannot trace; neither path below can use that. */
983 58776 : if (code == UNKNOWN)
984 : return NULL_RTX;
985 :
986 58776 : if ((if_info->cond_earliest == if_info->jump || cond_complex)
987 23917 : && (normalize == 0 || STORE_FLAG_VALUE == normalize))
988 : {
989 18191 : rtx src = gen_rtx_fmt_ee (code, GET_MODE (x), XEXP (cond, 0),
990 : XEXP (cond, 1));
991 18191 : rtx set = gen_rtx_SET (x, src);
992 :
993 18191 : start_sequence ();
994 18191 : rtx_insn *insn = emit_insn (set);
995 :
996 18191 : if (recog_memoized (insn) >= 0)
997 : {
998 7890 : rtx_insn *seq = end_sequence ();
999 7890 : emit_insn (seq);
1000 :
1001 7890 : if_info->cond_earliest = if_info->jump;
1002 :
1003 7890 : return x;
1004 : }
1005 :
1006 10301 : end_sequence ();
1007 : }
1008 :
1009 : /* Don't even try if the comparison operands or the mode of X are weird. */
1010 50886 : if (cond_complex || !SCALAR_INT_MODE_P (GET_MODE (x)))
1011 : return NULL_RTX;
1012 :
1013 : /* Don't try if mode of X is more than the max fixed mode size. */
1014 104547 : if (known_le (MAX_FIXED_MODE_SIZE, GET_MODE_BITSIZE (GET_MODE (x))))
1015 : return NULL_RTX;
1016 :
1017 33513 : return emit_store_flag (x, code, XEXP (cond, 0),
1018 : XEXP (cond, 1), VOIDmode,
1019 33513 : (code == LTU || code == LEU
1020 67026 : || code == GEU || code == GTU), normalize);
1021 : }
1022 :
1023 : /* Return true if X can be safely forced into a register by copy_to_mode_reg
1024 : / force_operand. */
1025 :
1026 : static bool
1027 2114122 : noce_can_force_operand (rtx x)
1028 : {
1029 2114573 : if (general_operand (x, VOIDmode))
1030 : return true;
1031 562428 : if (SUBREG_P (x))
1032 : {
1033 451 : if (!noce_can_force_operand (SUBREG_REG (x)))
1034 : return false;
1035 : return true;
1036 : }
1037 561977 : if (ARITHMETIC_P (x))
1038 : {
1039 465907 : if (!noce_can_force_operand (XEXP (x, 0))
1040 465907 : || !noce_can_force_operand (XEXP (x, 1)))
1041 : return false;
1042 460502 : switch (GET_CODE (x))
1043 : {
1044 : case MULT:
1045 : case MOD:
1046 : case UDIV:
1047 : case UMOD:
1048 : return true;
1049 1847 : case DIV:
1050 1847 : if (INTEGRAL_MODE_P (GET_MODE (x)))
1051 : return true;
1052 : /* FALLTHRU */
1053 447161 : default:
1054 447161 : auto optab = code_to_optab (GET_CODE (x));
1055 447161 : if (!optab)
1056 : return false;
1057 437971 : return optab_handler (optab, GET_MODE (x)) != CODE_FOR_nothing;
1058 : }
1059 : }
1060 96070 : if (UNARY_P (x))
1061 : {
1062 87336 : if (!noce_can_force_operand (XEXP (x, 0)))
1063 : return false;
1064 87315 : switch (GET_CODE (x))
1065 : {
1066 : case ZERO_EXTEND:
1067 : case SIGN_EXTEND:
1068 : case TRUNCATE:
1069 : case FLOAT_EXTEND:
1070 : case FLOAT_TRUNCATE:
1071 : case FIX:
1072 : case UNSIGNED_FIX:
1073 : case FLOAT:
1074 : case UNSIGNED_FLOAT:
1075 : return true;
1076 13340 : default:
1077 13340 : auto optab = code_to_optab (GET_CODE (x));
1078 13340 : if (!optab)
1079 : return false;
1080 13340 : return optab_handler (optab, GET_MODE (x)) != CODE_FOR_nothing;
1081 : }
1082 : }
1083 : return false;
1084 : }
1085 :
1086 : /* Emit instruction to move an rtx, possibly into STRICT_LOW_PART.
1087 : X is the destination/target and Y is the value to copy. */
1088 :
1089 : static void
1090 148840 : noce_emit_move_insn (rtx x, rtx y)
1091 : {
1092 148840 : machine_mode outmode;
1093 148840 : rtx outer, inner;
1094 148840 : poly_int64 bitpos;
1095 :
1096 148840 : if (GET_CODE (x) != STRICT_LOW_PART)
1097 : {
1098 148840 : rtx_insn *seq, *insn;
1099 148840 : rtx target;
1100 148840 : optab ot;
1101 :
1102 148840 : start_sequence ();
1103 : /* Check that the SET_SRC is reasonable before calling emit_move_insn,
1104 : otherwise construct a suitable SET pattern ourselves. */
1105 148920 : insn = (OBJECT_P (y) || CONSTANT_P (y) || GET_CODE (y) == SUBREG)
1106 148920 : ? emit_move_insn (x, y)
1107 109376 : : emit_insn (gen_rtx_SET (x, y));
1108 148840 : seq = end_sequence ();
1109 :
1110 148840 : if (recog_memoized (insn) <= 0)
1111 : {
1112 68680 : if (GET_CODE (x) == ZERO_EXTRACT)
1113 : {
1114 0 : rtx op = XEXP (x, 0);
1115 0 : unsigned HOST_WIDE_INT size = INTVAL (XEXP (x, 1));
1116 0 : unsigned HOST_WIDE_INT start = INTVAL (XEXP (x, 2));
1117 :
1118 : /* store_bit_field expects START to be relative to
1119 : BYTES_BIG_ENDIAN and adjusts this value for machines with
1120 : BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN. In order to be able to
1121 : invoke store_bit_field again it is necessary to have the START
1122 : value from the first call. */
1123 0 : if (BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN)
1124 : {
1125 : if (MEM_P (op))
1126 : start = BITS_PER_UNIT - start - size;
1127 : else
1128 : {
1129 : gcc_assert (REG_P (op));
1130 : start = BITS_PER_WORD - start - size;
1131 : }
1132 : }
1133 :
1134 0 : gcc_assert (start < (MEM_P (op) ? BITS_PER_UNIT : BITS_PER_WORD));
1135 0 : store_bit_field (op, size, start, 0, 0, GET_MODE (x), y, false,
1136 : false);
1137 0 : return;
1138 : }
1139 :
1140 68680 : switch (GET_RTX_CLASS (GET_CODE (y)))
1141 : {
1142 1094 : case RTX_UNARY:
1143 1094 : ot = code_to_optab (GET_CODE (y));
1144 1094 : if (ot && noce_can_force_operand (XEXP (y, 0)))
1145 : {
1146 856 : start_sequence ();
1147 856 : target = expand_unop (GET_MODE (y), ot, XEXP (y, 0), x, 0);
1148 856 : if (target != NULL_RTX)
1149 : {
1150 856 : if (target != x)
1151 0 : emit_move_insn (x, target);
1152 856 : seq = get_insns ();
1153 : }
1154 856 : end_sequence ();
1155 : }
1156 : break;
1157 :
1158 36752 : case RTX_BIN_ARITH:
1159 36752 : case RTX_COMM_ARITH:
1160 36752 : ot = code_to_optab (GET_CODE (y));
1161 36752 : if (ot
1162 36752 : && noce_can_force_operand (XEXP (y, 0))
1163 73504 : && noce_can_force_operand (XEXP (y, 1)))
1164 : {
1165 36752 : start_sequence ();
1166 36752 : target = expand_binop (GET_MODE (y), ot,
1167 : XEXP (y, 0), XEXP (y, 1),
1168 : x, 0, OPTAB_DIRECT);
1169 36752 : if (target != NULL_RTX)
1170 : {
1171 36752 : if (target != x)
1172 0 : emit_move_insn (x, target);
1173 36752 : seq = get_insns ();
1174 : }
1175 36752 : end_sequence ();
1176 : }
1177 : break;
1178 :
1179 : default:
1180 : break;
1181 : }
1182 : }
1183 :
1184 148840 : emit_insn (seq);
1185 148840 : return;
1186 : }
1187 :
1188 0 : outer = XEXP (x, 0);
1189 0 : inner = XEXP (outer, 0);
1190 0 : outmode = GET_MODE (outer);
1191 0 : bitpos = SUBREG_BYTE (outer) * BITS_PER_UNIT;
1192 0 : store_bit_field (inner, GET_MODE_BITSIZE (outmode), bitpos,
1193 : 0, 0, outmode, y, false, false);
1194 : }
1195 :
1196 : /* Return the CC reg if it is used in COND. */
1197 :
1198 : static rtx
1199 4199830 : cc_in_cond (rtx cond)
1200 : {
1201 4199830 : if (have_cbranchcc4 && cond
1202 4199830 : && GET_MODE_CLASS (GET_MODE (XEXP (cond, 0))) == MODE_CC)
1203 142943 : return XEXP (cond, 0);
1204 :
1205 : return NULL_RTX;
1206 : }
1207 :
1208 : struct noce_cc_reg_set
1209 : {
1210 : HARD_REG_SET set;
1211 : unsigned int fixed_regno1;
1212 : unsigned int fixed_regno2;
1213 : };
1214 :
1215 : /* If DEST is a hard condition-code register, add it to the set in DATA.
1216 : PAT is the SET or CLOBBER that writes DEST. */
1217 :
1218 : static void
1219 1030291 : noce_record_cc_reg_set (rtx dest, const_rtx pat ATTRIBUTE_UNUSED, void *data)
1220 : {
1221 1030291 : noce_cc_reg_set *cc_regs = (noce_cc_reg_set *) data;
1222 :
1223 1028675 : if (REG_P (dest) && HARD_REGISTER_P (dest)
1224 1390679 : && (GET_MODE_CLASS (GET_MODE (dest)) == MODE_CC
1225 0 : || REGNO (dest) == targetm.flags_regnum
1226 0 : || REGNO (dest) == cc_regs->fixed_regno1
1227 0 : || REGNO (dest) == cc_regs->fixed_regno2))
1228 360388 : add_to_hard_reg_set (&cc_regs->set, GET_MODE (dest), REGNO (dest));
1229 1030291 : }
1230 :
1231 : /* Return true if SEQ writes a condition-code register that still holds a
1232 : value which is live on exit from TEST_BB.
1233 :
1234 : An if-converted sequence is emitted immediately before the jump that ends
1235 : TEST_BB, and the conversion then removes that jump, so a condition code
1236 : that is live out of TEST_BB is one which a later block still reads.
1237 : Expanding a conditional move can emit a fresh comparison, and that would
1238 : destroy it. */
1239 :
1240 : static bool
1241 228885 : noce_clobbers_live_cc_p (basic_block test_bb, rtx_insn *seq)
1242 : {
1243 228885 : noce_cc_reg_set cc_regs;
1244 228885 : CLEAR_HARD_REG_SET (cc_regs.set);
1245 228885 : cc_regs.fixed_regno1 = INVALID_REGNUM;
1246 228885 : cc_regs.fixed_regno2 = INVALID_REGNUM;
1247 228885 : targetm.fixed_condition_code_regs (&cc_regs.fixed_regno1,
1248 : &cc_regs.fixed_regno2);
1249 1337589 : for (rtx_insn *insn = seq; insn; insn = NEXT_INSN (insn))
1250 879819 : note_stores (insn, noce_record_cc_reg_set, &cc_regs);
1251 :
1252 228885 : bitmap live_out = df_get_live_out (test_bb);
1253 228885 : hard_reg_set_iterator hrsi;
1254 228885 : unsigned int regno;
1255 418651 : EXECUTE_IF_SET_IN_HARD_REG_SET (cc_regs.set, 0, regno, hrsi)
1256 190188 : if (bitmap_bit_p (live_out, regno))
1257 : return true;
1258 :
1259 : return false;
1260 : }
1261 :
1262 : /* Return sequence of instructions generated by if conversion. This
1263 : function calls end_sequence() to end the current stream, ensures
1264 : that the instructions are unshared, recognizable non-jump insns.
1265 : On failure, this function returns a NULL_RTX. */
1266 :
1267 : static rtx_insn *
1268 228044 : end_ifcvt_sequence (noce_if_info *if_info)
1269 : {
1270 228044 : rtx_insn *insn;
1271 228044 : rtx_insn *seq = get_insns ();
1272 228044 : rtx cc = cc_in_cond (if_info->cond);
1273 :
1274 228044 : set_used_flags (if_info->x);
1275 228044 : set_used_flags (if_info->cond);
1276 228044 : set_used_flags (if_info->a);
1277 228044 : set_used_flags (if_info->b);
1278 :
1279 1186751 : for (insn = seq; insn; insn = NEXT_INSN (insn))
1280 730663 : set_used_flags (insn);
1281 :
1282 228044 : unshare_all_rtl_in_chain (seq);
1283 228044 : end_sequence ();
1284 :
1285 : /* Make sure that all of the instructions emitted are recognizable,
1286 : and that we haven't introduced a new jump instruction.
1287 : As an exercise for the reader, build a general mechanism that
1288 : allows proper placement of required clobbers. */
1289 1154206 : for (insn = seq; insn; insn = NEXT_INSN (insn))
1290 730143 : if (JUMP_P (insn)
1291 730143 : || recog_memoized (insn) == -1
1292 : /* Make sure new generated code does not clobber CC. */
1293 1428340 : || (cc && set_of (cc, insn)))
1294 : return NULL;
1295 :
1296 : /* CC above only covers the condition code read by the converted branch. */
1297 196019 : if (noce_clobbers_live_cc_p (if_info->test_bb, seq))
1298 422 : return NULL;
1299 :
1300 : return seq;
1301 : }
1302 :
1303 : /* Return true iff the then and else basic block (if it exists)
1304 : consist of a single simple set instruction. */
1305 :
1306 : static bool
1307 3194263 : noce_simple_bbs (noce_if_info *if_info)
1308 : {
1309 0 : if (!if_info->then_simple)
1310 : return false;
1311 :
1312 2958924 : if (if_info->else_bb)
1313 0 : return if_info->else_simple;
1314 :
1315 : return true;
1316 : }
1317 :
1318 : /* Commit the wound-up candidate SEQ produced by a noce transform for IF_INFO.
1319 : Reject it (returning false, emitting nothing) if it failed to build, or if
1320 : CHECK_PROFITABLE and the target deems it not worthwhile. Otherwise emit it
1321 : before the branch, record NAME as the winning transform and return true.
1322 : This is the shared tail of the noce_try_* matchers. */
1323 :
1324 : static bool
1325 213033 : noce_commit_sequence (noce_if_info *if_info, rtx_insn *seq,
1326 : bool check_profitable, const char *name)
1327 : {
1328 213033 : if (!seq
1329 138585 : || (check_profitable
1330 136985 : && !targetm.noce_conversion_profitable_p (seq, if_info)))
1331 : return false;
1332 :
1333 142374 : emit_insn_before_setloc (seq, if_info->jump,
1334 142374 : INSN_LOCATION (if_info->insn_a));
1335 142374 : if_info->transform_name = name;
1336 142374 : return true;
1337 : }
1338 :
1339 : /* Convert "if (a != b) x = a; else x = b" into "x = a" and
1340 : "if (a == b) x = a; else x = b" into "x = b". */
1341 :
1342 : static bool
1343 254264 : noce_try_move (noce_if_info *if_info)
1344 : {
1345 254264 : rtx cond = if_info->cond;
1346 254264 : enum rtx_code code = GET_CODE (cond);
1347 254264 : rtx y;
1348 254264 : rtx_insn *seq;
1349 :
1350 254264 : if (code != NE && code != EQ)
1351 : return false;
1352 :
1353 258431 : if (!noce_simple_bbs (if_info))
1354 : return false;
1355 :
1356 : /* This optimization isn't valid if either A or B could be a NaN
1357 : or a signed zero. */
1358 177658 : if (HONOR_NANS (if_info->x)
1359 177658 : || HONOR_SIGNED_ZEROS (if_info->x))
1360 : return false;
1361 :
1362 : /* Check whether the operands of the comparison are A and B, in
1363 : either order. */
1364 152541 : if ((rtx_equal_p (if_info->a, XEXP (cond, 0))
1365 2410 : && rtx_equal_p (if_info->b, XEXP (cond, 1)))
1366 154939 : || (rtx_equal_p (if_info->a, XEXP (cond, 1))
1367 15646 : && rtx_equal_p (if_info->b, XEXP (cond, 0))))
1368 : {
1369 21 : if (!rtx_interchangeable_p (if_info->a, if_info->b))
1370 : return false;
1371 :
1372 0 : y = (code == EQ) ? if_info->a : if_info->b;
1373 :
1374 : /* Avoid generating the move if the source is the destination. */
1375 0 : if (! rtx_equal_p (if_info->x, y))
1376 : {
1377 0 : start_sequence ();
1378 0 : noce_emit_move_insn (if_info->x, y);
1379 0 : seq = end_ifcvt_sequence (if_info);
1380 0 : if (!seq)
1381 : return false;
1382 :
1383 0 : emit_insn_before_setloc (seq, if_info->jump,
1384 0 : INSN_LOCATION (if_info->insn_a));
1385 : }
1386 0 : if_info->transform_name = "noce_try_move";
1387 0 : return true;
1388 : }
1389 : return false;
1390 : }
1391 :
1392 : /* TEMP holds an all-ones or all-zeros splat of a tested sign bit. Return an
1393 : rtx that narrows it to VAL or zero, expanded into TARGET where possible, or
1394 : NULL_RTX if the expansion failed.
1395 :
1396 : The obvious form is a mask, but since the value is known to be -1 or 0 a
1397 : shift is sometimes cheaper: a logical right shift constructs 2^n-1 and a
1398 : left shift constructs ~(2^n-1). Some targets do not have efficient shifts,
1399 : so build the RTL for each applicable form and expand the cheapest. */
1400 :
1401 : static rtx
1402 84 : noce_splat_to_const (machine_mode mode, rtx temp, HOST_WIDE_INT val,
1403 : rtx target)
1404 : {
1405 84 : rtx and_form = gen_rtx_AND (mode, temp, GEN_INT (val));
1406 168 : rtx shift_left = gen_rtx_ASHIFT (mode, temp, GEN_INT (ctz_hwi (val)));
1407 84 : HOST_WIDE_INT rshift_count
1408 168 : = clz_hwi (val) & (GET_MODE_PRECISION (mode).to_constant () - 1);
1409 84 : rtx shift_right = gen_rtx_LSHIFTRT (mode, temp, GEN_INT (rshift_count));
1410 84 : bool speed_p = optimize_insn_for_speed_p ();
1411 :
1412 84 : if (exact_log2 (val + 1) >= 0
1413 50 : && (rtx_cost (shift_right, mode, SET, 1, speed_p)
1414 25 : < rtx_cost (and_form, mode, SET, 1, speed_p)))
1415 3 : return expand_simple_binop (mode, LSHIFTRT, temp, GEN_INT (rshift_count),
1416 3 : target, false, OPTAB_WIDEN);
1417 :
1418 81 : if (exact_log2 (~val + 1) >= 0
1419 0 : && (rtx_cost (shift_left, mode, SET, 1, speed_p)
1420 0 : < rtx_cost (and_form, mode, SET, 1, speed_p)))
1421 0 : return expand_simple_binop (mode, ASHIFT, temp, GEN_INT (ctz_hwi (val)),
1422 0 : target, false, OPTAB_WIDEN);
1423 :
1424 81 : return expand_simple_binop (mode, AND, temp, GEN_INT (val), target, false,
1425 81 : OPTAB_WIDEN);
1426 : }
1427 :
1428 : /* If a sign bit test is selecting across constants, we may be able
1429 : to generate efficient code utilizing the -1/0 result of a sign
1430 : bit splat idiom.
1431 :
1432 : Do this before trying the generalized conditional move as these
1433 : (when applicable) are hopefully faster than a conditional move. */
1434 :
1435 : static bool
1436 201957 : noce_try_sign_bit_splat (noce_if_info *if_info)
1437 : {
1438 201957 : rtx cond = if_info->cond;
1439 201957 : enum rtx_code code = GET_CODE (cond);
1440 :
1441 : /* We're looking for sign bit tests, so only a few cases are
1442 : interesting. LT/GE 0 LE/GT -1. */
1443 201957 : if (((code == LT || code == GE)
1444 11362 : && XEXP (cond, 1) == CONST0_RTX (GET_MODE (cond)))
1445 199404 : || ((code == LE || code == GT)
1446 11517 : && XEXP (cond, 1) == CONSTM1_RTX (GET_MODE (cond))))
1447 : ;
1448 : else
1449 : return false;
1450 :
1451 : /* It would be good if this could be extended since constant synthesis
1452 : on some platforms will result in blocks which fail this test. */
1453 11099 : if (!noce_simple_bbs (if_info))
1454 : return false;
1455 :
1456 : /* Only try this for constants in the true/false arms and a REG
1457 : destination. We could select between 0 and a REG pretty
1458 : easily with a logical AND. */
1459 6221 : if (!CONST_INT_P (if_info->a)
1460 2019 : || !CONST_INT_P (if_info->b)
1461 696 : || !REG_P (if_info->x))
1462 : return false;
1463 :
1464 696 : machine_mode mode = GET_MODE (if_info->x);
1465 :
1466 : /* If the mode of the destination does not match the mode of
1467 : the value we're testing, then this optimization is not valid. */
1468 696 : if (mode != GET_MODE (XEXP (cond, 0)))
1469 : return false;
1470 :
1471 214 : HOST_WIDE_INT val_a = INTVAL (if_info->a);
1472 214 : HOST_WIDE_INT val_b = INTVAL (if_info->b);
1473 :
1474 214 : rtx_insn *seq;
1475 214 : start_sequence ();
1476 :
1477 : /* We're testing the sign bit of this operand. */
1478 214 : rtx condop = XEXP (cond, 0);
1479 :
1480 : /* To splat the sign bit we arithmetically shift the
1481 : input value right by the size of the object - 1 bits.
1482 :
1483 : Note some targets do not have strong shifters, but do have
1484 : alternative ways to generate the sign bit splat. The
1485 : profitability test when we end the sequence should reject
1486 : cases when the branchy sequence is better. */
1487 214 : int splat_count = GET_MODE_BITSIZE (GET_MODE (condop)).to_constant () - 1;
1488 214 : rtx splat = GEN_INT (splat_count);
1489 :
1490 214 : rtx temp;
1491 214 : temp = expand_simple_binop (GET_MODE (XEXP (cond, 0)), ASHIFTRT,
1492 : XEXP (cond, 0), splat, NULL_RTX,
1493 : false, OPTAB_WIDEN);
1494 214 : if (!temp)
1495 0 : goto fail;
1496 :
1497 : /* IOR of anything with -1 still results in -1. So we can
1498 : IOR the other operand to generate a select between -1 and
1499 : an arbitrary constant. */
1500 214 : if (val_a == -1)
1501 : {
1502 50 : if (code == LT || code == LE)
1503 : {
1504 2 : temp = expand_simple_unop (mode, NOT, temp, NULL_RTX, true);
1505 2 : if (!temp)
1506 0 : goto fail;
1507 : }
1508 :
1509 50 : temp = expand_simple_binop (mode, IOR, temp, GEN_INT (val_b),
1510 : if_info->x, false, OPTAB_WIDEN);
1511 : }
1512 : /* AND of anything with 0 is still zero. So we can AND
1513 : with the -1 operand with the a constant to select
1514 : between the constant and zero. */
1515 164 : else if (val_b == 0)
1516 : {
1517 0 : if (code == LT || code == LE)
1518 : {
1519 0 : temp = expand_simple_unop (mode, NOT, temp, NULL_RTX, true);
1520 0 : if (!temp)
1521 0 : goto fail;
1522 : }
1523 :
1524 0 : temp = noce_splat_to_const (mode, temp, val_a, if_info->x);
1525 : }
1526 : /* Same cases, but with the test or arms swapped. These
1527 : can be realized as well, though it typically costs
1528 : an extra instruction. */
1529 164 : else if (val_b == -1)
1530 : {
1531 46 : if (code != LT && code != LE)
1532 : {
1533 2 : temp = expand_simple_unop (mode, NOT, temp, NULL_RTX, true);
1534 2 : if (!temp)
1535 0 : goto fail;
1536 : }
1537 :
1538 46 : temp = expand_simple_binop (mode, IOR, temp, GEN_INT (val_a),
1539 : if_info->x, false, OPTAB_WIDEN);
1540 : }
1541 118 : else if (val_a == 0)
1542 : {
1543 84 : if (code != LT && code != LE)
1544 : {
1545 9 : temp = expand_simple_unop (mode, NOT, temp, NULL_RTX, true);
1546 9 : if (!temp)
1547 0 : goto fail;
1548 : }
1549 :
1550 84 : temp = noce_splat_to_const (mode, temp, val_b, if_info->x);
1551 : }
1552 : /* Nothing worked. */
1553 : else
1554 : temp = NULL_RTX;
1555 :
1556 180 : if (!temp)
1557 34 : goto fail;
1558 :
1559 : /* Move into the final destination if the value wasn't
1560 : constructed there. */
1561 180 : if (if_info->x != temp)
1562 0 : emit_move_insn (if_info->x, temp);
1563 :
1564 : /* This ends the sequence and tests the cost model. */
1565 180 : seq = end_ifcvt_sequence (if_info);
1566 180 : return noce_commit_sequence (if_info, seq, true, "splat_sign_bit_trivial");
1567 :
1568 34 : fail:
1569 34 : end_ifcvt_sequence (if_info);
1570 34 : return false;
1571 : }
1572 :
1573 :
1574 : /* Try forming an IF_THEN_ELSE (cond, b, a) and collapsing that
1575 : through simplify_rtx. Sometimes that can eliminate the IF_THEN_ELSE.
1576 : If that is the case, emit the result into x. */
1577 :
1578 : static bool
1579 254264 : noce_try_ifelse_collapse (noce_if_info * if_info)
1580 : {
1581 340956 : if (!noce_simple_bbs (if_info))
1582 : return false;
1583 :
1584 235431 : machine_mode mode = GET_MODE (if_info->x);
1585 235431 : rtx if_then_else = simplify_gen_ternary (IF_THEN_ELSE, mode, mode,
1586 : if_info->cond, if_info->b,
1587 : if_info->a);
1588 :
1589 235431 : if (GET_CODE (if_then_else) == IF_THEN_ELSE)
1590 : return false;
1591 :
1592 48452 : rtx_insn *seq;
1593 48452 : start_sequence ();
1594 48452 : noce_emit_move_insn (if_info->x, if_then_else);
1595 48452 : seq = end_ifcvt_sequence (if_info);
1596 48452 : return noce_commit_sequence (if_info, seq, false, "noce_try_ifelse_collapse");
1597 : }
1598 :
1599 :
1600 : /* Convert "if (test) x = 1; else x = 0".
1601 :
1602 : Only try 0 and STORE_FLAG_VALUE here. Other combinations will be
1603 : tried in noce_try_store_flag_constants after noce_try_cmove has had
1604 : a go at the conversion. */
1605 :
1606 : static bool
1607 236646 : noce_try_store_flag (noce_if_info *if_info)
1608 : {
1609 236646 : bool reversep;
1610 236646 : rtx target;
1611 236646 : rtx_insn *seq;
1612 :
1613 310054 : if (!noce_simple_bbs (if_info))
1614 : return false;
1615 :
1616 217813 : if (CONST_INT_P (if_info->b)
1617 74378 : && INTVAL (if_info->b) == STORE_FLAG_VALUE
1618 13401 : && if_info->a == const0_rtx)
1619 : reversep = false;
1620 207324 : else if (if_info->b == const0_rtx
1621 30181 : && CONST_INT_P (if_info->a)
1622 20802 : && INTVAL (if_info->a) == STORE_FLAG_VALUE)
1623 : reversep = true;
1624 : else
1625 : return false;
1626 :
1627 30835 : start_sequence ();
1628 :
1629 30835 : target = noce_emit_store_flag (if_info, if_info->x, reversep, 0);
1630 30835 : if (target)
1631 : {
1632 25996 : if (target != if_info->x)
1633 2990 : noce_emit_move_insn (if_info->x, target);
1634 :
1635 25996 : seq = end_ifcvt_sequence (if_info);
1636 25996 : return noce_commit_sequence (if_info, seq, false, "noce_try_store_flag");
1637 : }
1638 : else
1639 : {
1640 4839 : end_sequence ();
1641 4839 : return false;
1642 : }
1643 : }
1644 :
1645 :
1646 : /* Convert "if (test) x = -A; else x = A" into
1647 : x = A; if (test) x = -x if the machine can do the
1648 : conditional negate form of this cheaply.
1649 : Try this before noce_try_cmove that will just load the
1650 : immediates into two registers and do a conditional select
1651 : between them. If the target has a conditional negate or
1652 : conditional invert operation we can save a potentially
1653 : expensive constant synthesis. */
1654 :
1655 : static bool
1656 210566 : noce_try_inverse_constants (noce_if_info *if_info)
1657 : {
1658 274331 : if (!noce_simple_bbs (if_info))
1659 : return false;
1660 :
1661 191733 : if (!CONST_INT_P (if_info->a)
1662 52086 : || !CONST_INT_P (if_info->b)
1663 21449 : || !REG_P (if_info->x))
1664 : return false;
1665 :
1666 21449 : machine_mode mode = GET_MODE (if_info->x);
1667 :
1668 21449 : HOST_WIDE_INT val_a = INTVAL (if_info->a);
1669 21449 : HOST_WIDE_INT val_b = INTVAL (if_info->b);
1670 :
1671 21449 : rtx cond = if_info->cond;
1672 :
1673 21449 : rtx x = if_info->x;
1674 21449 : rtx target;
1675 :
1676 21449 : start_sequence ();
1677 :
1678 21449 : rtx_code code;
1679 21449 : if (val_b != HOST_WIDE_INT_MIN && val_a == -val_b)
1680 : code = NEG;
1681 20147 : else if (val_a == ~val_b)
1682 : code = NOT;
1683 : else
1684 : {
1685 20079 : end_sequence ();
1686 20079 : return false;
1687 : }
1688 :
1689 1370 : rtx tmp = gen_reg_rtx (mode);
1690 1370 : noce_emit_move_insn (tmp, if_info->a);
1691 :
1692 1370 : target = emit_conditional_neg_or_complement (x, code, mode, cond, tmp, tmp);
1693 :
1694 1370 : if (target)
1695 : {
1696 0 : rtx_insn *seq = get_insns ();
1697 :
1698 0 : if (!seq)
1699 : {
1700 0 : end_sequence ();
1701 0 : return false;
1702 : }
1703 :
1704 0 : if (target != if_info->x)
1705 0 : noce_emit_move_insn (if_info->x, target);
1706 :
1707 0 : seq = end_ifcvt_sequence (if_info);
1708 0 : return noce_commit_sequence (if_info, seq, false,
1709 : "noce_try_inverse_constants");
1710 : }
1711 :
1712 1370 : end_sequence ();
1713 1370 : return false;
1714 : }
1715 :
1716 : /* Check if OP is supported by conditional zero based if conversion,
1717 : returning TRUE if satisfied otherwise FALSE.
1718 :
1719 : OP is the operation to check. */
1720 :
1721 : static bool
1722 591314 : noce_cond_zero_binary_op_supported (rtx op)
1723 : {
1724 591314 : enum rtx_code opcode = GET_CODE (op);
1725 :
1726 0 : if (opcode == PLUS || opcode == MINUS || opcode == IOR || opcode == XOR
1727 : || opcode == ASHIFT || opcode == ASHIFTRT || opcode == LSHIFTRT
1728 : || opcode == ROTATE || opcode == ROTATERT || opcode == AND)
1729 0 : return true;
1730 :
1731 : return false;
1732 : }
1733 :
1734 : /* Convert "if (test) x = a; else x = a OP c " where c is 2^n.
1735 :
1736 : We can shift the output of a set flag insn left to generate 2^n
1737 : cheaply, then apply OP unconditionally. This works even if the
1738 : target does not have a conditional zero idiom. On targets such
1739 : as RISC-V this sequence may also encode better.
1740 :
1741 : This is based on noce_try_store_flag_constants. */
1742 :
1743 : static bool
1744 207381 : noce_try_shifted_store_flag (noce_if_info *if_info)
1745 : {
1746 207381 : rtx target;
1747 207381 : rtx_insn *seq;
1748 207381 : machine_mode mode = GET_MODE (if_info->x);
1749 207381 : rtx common = NULL_RTX;
1750 207381 : rtx_code code = UNKNOWN;
1751 :
1752 207381 : if (STORE_FLAG_VALUE != 1)
1753 : return false;
1754 :
1755 268036 : if (!noce_simple_bbs (if_info))
1756 : return false;
1757 :
1758 188548 : rtx a = if_info->a;
1759 188548 : rtx b = if_info->b;
1760 :
1761 : /* Most binary operators are allowed, essentially the same set
1762 : as the condzero arithmetic paths, with the exception of AND
1763 : which could be handled if we were inclined. */
1764 188548 : int shiftval;
1765 188548 : bool reversed = false;
1766 188548 : if (noce_cond_zero_binary_op_supported (a)
1767 24340 : && GET_CODE (a) != AND
1768 24132 : && GET_CODE (b) == REG
1769 20448 : && rtx_equal_p (XEXP (a, 0), b)
1770 18480 : && CONST_INT_P (XEXP (a, 1))
1771 9042 : && pow2p_hwi (INTVAL (XEXP (a, 1))))
1772 : {
1773 6664 : code = GET_CODE (a);
1774 6664 : common = XEXP (a, 0);
1775 6664 : shiftval = exact_log2 (INTVAL (XEXP (a, 1)));
1776 :
1777 : /* When the condition is true, we branch around A and thus we want
1778 : the condition reversed from what you might expect. */
1779 : reversed = true;
1780 : }
1781 181884 : else if (noce_cond_zero_binary_op_supported (b)
1782 2227 : && GET_CODE (b) != AND
1783 2188 : && GET_CODE (a) == REG
1784 230 : && rtx_equal_p (XEXP (b, 0), a)
1785 89 : && CONST_INT_P (XEXP (b, 1))
1786 62 : && pow2p_hwi (INTVAL (XEXP (b, 1))))
1787 : {
1788 31 : code = GET_CODE (b);
1789 31 : common = XEXP (b, 0);
1790 31 : shiftval = exact_log2 (INTVAL (XEXP (b, 1)));
1791 : }
1792 :
1793 :
1794 : /* If CODE hasn't been reset, then the basic expression form was wrong and
1795 : we can not optimize. */
1796 6695 : if (code == UNKNOWN)
1797 : return false;
1798 :
1799 6695 : start_sequence ();
1800 :
1801 : /* If X and COMMON are the same, then we're going to need a temporary. */
1802 6695 : if (rtx_equal_p (common, if_info->x))
1803 : {
1804 6102 : common = gen_reg_rtx (mode);
1805 6102 : noce_emit_move_insn (common, if_info->x);
1806 : }
1807 :
1808 6695 : target = noce_emit_store_flag (if_info, if_info->x, reversed, 1);
1809 6695 : if (!target)
1810 : {
1811 742 : end_sequence ();
1812 742 : return false;
1813 : }
1814 :
1815 : /* Right now TARGET has the value 1/0. A left shift will produce the
1816 : target value. Unnecessary if the shift value is zero. */
1817 5953 : if (shiftval != 0)
1818 1225 : target = expand_simple_binop (mode, ASHIFT, target,
1819 : GEN_INT (shiftval),
1820 : NULL_RTX, 0, OPTAB_WIDEN);
1821 :
1822 5953 : if (!target)
1823 : {
1824 0 : end_sequence ();
1825 0 : return false;
1826 : }
1827 :
1828 : /* Now we just need to apply the code to COMMON and TARGET. */
1829 5953 : target = expand_simple_binop (mode, code,
1830 : common, target,
1831 : NULL_RTX, 0, OPTAB_WIDEN);
1832 :
1833 5953 : if (!target)
1834 : {
1835 0 : end_sequence ();
1836 0 : return false;
1837 : }
1838 :
1839 : /* If necessary, store the result into the proper destination. */
1840 5953 : if (target != if_info->x)
1841 5953 : noce_emit_move_insn (if_info->x, target);
1842 :
1843 5953 : seq = end_ifcvt_sequence (if_info);
1844 5953 : return noce_commit_sequence (if_info, seq, true,
1845 5953 : "noce_try_shifted_store_flag");
1846 : }
1847 :
1848 :
1849 : /* Set *DIFF to ITRUE - IFALSE truncated to MODE and return true. Return
1850 : false, leaving *DIFF alone, when that difference is not representable and
1851 : so cannot be used to build a store-flag sequence. */
1852 :
1853 : static bool
1854 21958 : noce_representable_diff_p (HOST_WIDE_INT ifalse, HOST_WIDE_INT itrue,
1855 : machine_mode mode, HOST_WIDE_INT *diff)
1856 : {
1857 21958 : HOST_WIDE_INT d = (unsigned HOST_WIDE_INT) itrue - ifalse;
1858 :
1859 21958 : if ((d > 0) != ((ifalse < 0) != (itrue < 0) ? ifalse < 0 : ifalse < itrue))
1860 : return false;
1861 :
1862 21823 : *diff = trunc_int_for_mode (d, mode);
1863 21823 : return true;
1864 : }
1865 :
1866 : /* Convert "if (test) x = a; else x = b", for A and B constant.
1867 : Also allow A = y + c1, B = y + c2, with a common y between A
1868 : and B. */
1869 :
1870 : static bool
1871 210566 : noce_try_store_flag_constants (noce_if_info *if_info)
1872 : {
1873 210566 : rtx target;
1874 210566 : rtx_insn *seq;
1875 210566 : bool reversep;
1876 210566 : HOST_WIDE_INT itrue, ifalse, diff, tmp;
1877 210566 : int normalize;
1878 210566 : bool can_reverse;
1879 210566 : machine_mode mode = GET_MODE (if_info->x);
1880 210566 : rtx common = NULL_RTX;
1881 :
1882 210566 : rtx a = if_info->a;
1883 210566 : rtx b = if_info->b;
1884 :
1885 : /* Handle cases like x := test ? y + 3 : y + 4. */
1886 210566 : if (GET_CODE (a) == PLUS
1887 23595 : && GET_CODE (b) == PLUS
1888 1496 : && CONST_INT_P (XEXP (a, 1))
1889 587 : && CONST_INT_P (XEXP (b, 1))
1890 511 : && rtx_equal_p (XEXP (a, 0), XEXP (b, 0))
1891 : /* Allow expressions that are not using the result or plain
1892 : registers where we handle overlap below. */
1893 211027 : && (REG_P (XEXP (a, 0))
1894 4 : || (noce_operand_ok (XEXP (a, 0))
1895 4 : && ! reg_overlap_mentioned_p (if_info->x, XEXP (a, 0)))))
1896 : {
1897 461 : common = XEXP (a, 0);
1898 461 : a = XEXP (a, 1);
1899 461 : b = XEXP (b, 1);
1900 : }
1901 :
1902 274331 : if (!noce_simple_bbs (if_info))
1903 : return false;
1904 :
1905 191733 : if (CONST_INT_P (a)
1906 52535 : && CONST_INT_P (b))
1907 : {
1908 21898 : ifalse = INTVAL (a);
1909 21898 : itrue = INTVAL (b);
1910 21898 : bool subtract_flag_p = false;
1911 :
1912 21898 : if (!noce_representable_diff_p (ifalse, itrue, mode, &diff))
1913 : return false;
1914 :
1915 21784 : can_reverse = noce_reversed_cond_code (if_info) != UNKNOWN;
1916 21784 : reversep = false;
1917 21784 : if (diff == STORE_FLAG_VALUE || diff == -STORE_FLAG_VALUE)
1918 : {
1919 12133 : normalize = 0;
1920 : /* We could collapse these cases but it is easier to follow the
1921 : diff/STORE_FLAG_VALUE combinations when they are listed
1922 : explicitly. */
1923 :
1924 : /* test ? 3 : 4
1925 : => 4 + (test != 0). */
1926 12133 : if (diff < 0 && STORE_FLAG_VALUE < 0)
1927 : reversep = false;
1928 : /* test ? 4 : 3
1929 : => can_reverse | 4 + (test == 0)
1930 : !can_reverse | 3 - (test != 0). */
1931 12133 : else if (diff > 0 && STORE_FLAG_VALUE < 0)
1932 : {
1933 : reversep = can_reverse;
1934 : subtract_flag_p = !can_reverse;
1935 : /* If we need to subtract the flag and we have PLUS-immediate
1936 : A and B then it is unlikely to be beneficial to play tricks
1937 : here. */
1938 : if (subtract_flag_p && common)
1939 : return false;
1940 : }
1941 : /* test ? 3 : 4
1942 : => can_reverse | 3 + (test == 0)
1943 : !can_reverse | 4 - (test != 0). */
1944 12133 : else if (diff < 0 && STORE_FLAG_VALUE > 0)
1945 : {
1946 6134 : reversep = can_reverse;
1947 6134 : subtract_flag_p = !can_reverse;
1948 : /* If we need to subtract the flag and we have PLUS-immediate
1949 : A and B then it is unlikely to be beneficial to play tricks
1950 : here. */
1951 6134 : if (subtract_flag_p && common)
1952 : return false;
1953 : }
1954 : /* test ? 4 : 3
1955 : => 4 + (test != 0). */
1956 : else if (diff > 0 && STORE_FLAG_VALUE > 0)
1957 : reversep = false;
1958 : else
1959 : gcc_unreachable ();
1960 : }
1961 : /* Is this (cond) ? 2^n : 0? */
1962 538 : else if (ifalse == 0 && pow2p_hwi (itrue)
1963 9651 : && STORE_FLAG_VALUE == 1)
1964 : normalize = 1;
1965 : /* Is this (cond) ? 0 : 2^n? */
1966 389 : else if (itrue == 0 && pow2p_hwi (ifalse)
1967 9644 : && STORE_FLAG_VALUE == 1)
1968 : {
1969 : normalize = 1;
1970 : reversep = true;
1971 : }
1972 : /* Is this (cond) ? -1 : x? */
1973 : else if (itrue == -1
1974 : && STORE_FLAG_VALUE == -1)
1975 : normalize = -1;
1976 : /* Is this (cond) ? x : -1? */
1977 : else if (ifalse == -1
1978 : && STORE_FLAG_VALUE == -1)
1979 : {
1980 : normalize = -1;
1981 : reversep = true;
1982 : }
1983 : else
1984 : return false;
1985 :
1986 6134 : if (reversep)
1987 : {
1988 6135 : std::swap (itrue, ifalse);
1989 6135 : diff = trunc_int_for_mode (-(unsigned HOST_WIDE_INT) diff, mode);
1990 : }
1991 :
1992 12141 : start_sequence ();
1993 :
1994 : /* If we have x := test ? x + 3 : x + 4 then move the original
1995 : x out of the way while we store flags. */
1996 12141 : if (common && rtx_equal_p (common, if_info->x))
1997 : {
1998 5 : common = gen_reg_rtx (mode);
1999 5 : noce_emit_move_insn (common, if_info->x);
2000 : }
2001 :
2002 12141 : target = noce_emit_store_flag (if_info, if_info->x, reversep, normalize);
2003 12141 : if (! target)
2004 : {
2005 4875 : end_sequence ();
2006 4875 : return false;
2007 : }
2008 :
2009 : /* if (test) x = 3; else x = 4;
2010 : => x = 3 + (test == 0); */
2011 7266 : if (diff == STORE_FLAG_VALUE || diff == -STORE_FLAG_VALUE)
2012 : {
2013 : /* Add the common part now. This may allow combine to merge this
2014 : with the store flag operation earlier into some sort of conditional
2015 : increment/decrement if the target allows it. */
2016 7258 : if (common)
2017 29 : target = expand_simple_binop (mode, PLUS,
2018 : target, common,
2019 : target, 0, OPTAB_WIDEN);
2020 :
2021 : /* Always use ifalse here. It should have been swapped with itrue
2022 : when appropriate when reversep is true. */
2023 14516 : target = expand_simple_binop (mode, subtract_flag_p ? MINUS : PLUS,
2024 7258 : gen_int_mode (ifalse, mode), target,
2025 : if_info->x, 0, OPTAB_WIDEN);
2026 : }
2027 : /* Other cases are not beneficial when the original A and B are PLUS
2028 : expressions. */
2029 8 : else if (common)
2030 : {
2031 0 : end_sequence ();
2032 0 : return false;
2033 : }
2034 : /* if (test) x = 8; else x = 0;
2035 : => x = (test != 0) << 3; */
2036 8 : else if (ifalse == 0 && (tmp = exact_log2 (itrue)) >= 0)
2037 : {
2038 8 : target = expand_simple_binop (mode, ASHIFT,
2039 : target, GEN_INT (tmp), if_info->x, 0,
2040 : OPTAB_WIDEN);
2041 : }
2042 :
2043 : /* if (test) x = -1; else x = b;
2044 : => x = -(test != 0) | b; */
2045 0 : else if (itrue == -1)
2046 : {
2047 0 : target = expand_simple_binop (mode, IOR,
2048 0 : target, gen_int_mode (ifalse, mode),
2049 : if_info->x, 0, OPTAB_WIDEN);
2050 : }
2051 : else
2052 : {
2053 0 : end_sequence ();
2054 0 : return false;
2055 : }
2056 :
2057 7266 : if (! target)
2058 : {
2059 0 : end_sequence ();
2060 0 : return false;
2061 : }
2062 :
2063 7266 : if (target != if_info->x)
2064 0 : noce_emit_move_insn (if_info->x, target);
2065 :
2066 7266 : seq = end_ifcvt_sequence (if_info);
2067 7266 : return noce_commit_sequence (if_info, seq, true,
2068 7266 : "noce_try_store_flag_constants");
2069 : }
2070 :
2071 : return false;
2072 : }
2073 :
2074 : /* This is trying to capture cases like dest = cond ? a : -1.
2075 :
2076 : The basic idea is to use a store-flag insn to generate a -1/0
2077 : value (directly or indirectly), then IOR that with the other
2078 : input. */
2079 : static bool
2080 201777 : noce_try_store_flag_logical (noce_if_info *if_info)
2081 : {
2082 201777 : rtx a = if_info->a;
2083 201777 : rtx b = if_info->b;
2084 201777 : rtx dest = if_info->x;
2085 201777 : machine_mode mode = GET_MODE (dest);
2086 :
2087 201777 : if (STORE_FLAG_VALUE != -1 && STORE_FLAG_VALUE != 1)
2088 : return false;
2089 :
2090 262128 : if (!noce_simple_bbs (if_info))
2091 : return false;
2092 :
2093 182944 : bool swapped = STORE_FLAG_VALUE == -1;
2094 182944 : if (a == CONSTM1_RTX (GET_MODE (dest))
2095 1745 : && if_info->rev_cond)
2096 : {
2097 : std::swap (a, b);
2098 182944 : swapped = !swapped;
2099 : }
2100 :
2101 182944 : if (b != CONSTM1_RTX (GET_MODE (dest)))
2102 : return false;
2103 :
2104 : /* If the other arm is not a REG/SUBREG, then punt. This is primarily to
2105 : let the target handle the constant case, which it can likely do better.
2106 : It also means we don't have to worry about non terminal expressions. */
2107 2632 : if (!REG_P (a) && !SUBREG_P (a))
2108 : return false;
2109 :
2110 : /* At this point we've got dest = cond ? a : -1. Emit the store flag and
2111 : adjust its value (if necessary) to -1/0. */
2112 1071 : start_sequence ();
2113 1071 : rtx temp = gen_reg_rtx (mode);
2114 1071 : rtx target = noce_emit_store_flag (if_info, temp, !swapped, 0);
2115 1071 : if (!target)
2116 : {
2117 88 : end_sequence ();
2118 88 : return false;
2119 : }
2120 :
2121 983 : if (STORE_FLAG_VALUE == 1)
2122 : {
2123 983 : rtx x = gen_rtx_PLUS (mode, target, CONSTM1_RTX (mode));
2124 983 : emit_move_insn (target, x);
2125 : }
2126 :
2127 : /* Now we've got -1/0 in TARGET. We can just IOR with A. */
2128 983 : rtx x = gen_rtx_IOR (mode, target, a);
2129 983 : emit_move_insn (dest, x);
2130 :
2131 : /* We've generated all the RTL, make sure it recognizes and is
2132 : profitable. */
2133 983 : rtx_insn *seq = end_ifcvt_sequence (if_info);
2134 983 : return noce_commit_sequence (if_info, seq, true,
2135 983 : "noce_try_store_flag_logical");
2136 : }
2137 :
2138 : /* Convert "if (test) foo++" into "foo += (test != 0)", and
2139 : similarly for "foo--". */
2140 :
2141 : static bool
2142 158508 : noce_try_addcc (noce_if_info *if_info)
2143 : {
2144 158508 : rtx target;
2145 158508 : rtx_insn *seq;
2146 158508 : bool subtract;
2147 158508 : int normalize;
2148 :
2149 202242 : if (!noce_simple_bbs (if_info))
2150 : return false;
2151 :
2152 139675 : if (GET_CODE (if_info->a) == PLUS
2153 14911 : && rtx_equal_p (XEXP (if_info->a, 0), if_info->b)
2154 150991 : && noce_reversed_cond_code (if_info) != UNKNOWN)
2155 : {
2156 11316 : enum rtx_code code;
2157 11316 : rtx cond = noce_reversed_cond (if_info, &code);
2158 :
2159 : /* First try to use addcc pattern. */
2160 11316 : if (general_operand (XEXP (cond, 0), VOIDmode)
2161 11316 : && general_operand (XEXP (cond, 1), VOIDmode))
2162 : {
2163 11006 : start_sequence ();
2164 33018 : target = emit_conditional_add (if_info->x, code,
2165 : XEXP (cond, 0),
2166 : XEXP (cond, 1),
2167 : VOIDmode,
2168 : if_info->b,
2169 11006 : XEXP (if_info->a, 1),
2170 11006 : GET_MODE (if_info->x),
2171 10605 : (code == LTU || code == GEU
2172 21558 : || code == LEU || code == GTU));
2173 11006 : if (target)
2174 : {
2175 338 : if (target != if_info->x)
2176 0 : noce_emit_move_insn (if_info->x, target);
2177 :
2178 338 : seq = end_ifcvt_sequence (if_info);
2179 338 : return noce_commit_sequence (if_info, seq, true,
2180 1184 : "noce_try_addcc");
2181 : }
2182 10668 : end_sequence ();
2183 : }
2184 :
2185 : /* If that fails, construct conditional increment or decrement using
2186 : setcc. We're changing a branch and an increment to a comparison and
2187 : an ADD/SUB. */
2188 10978 : if (XEXP (if_info->a, 1) == const1_rtx
2189 10000 : || XEXP (if_info->a, 1) == constm1_rtx)
2190 : {
2191 1543 : start_sequence ();
2192 1543 : if (STORE_FLAG_VALUE == INTVAL (XEXP (if_info->a, 1)))
2193 : subtract = false, normalize = 0;
2194 565 : else if (-STORE_FLAG_VALUE == INTVAL (XEXP (if_info->a, 1)))
2195 : subtract = true, normalize = 0;
2196 : else
2197 0 : subtract = false, normalize = INTVAL (XEXP (if_info->a, 1));
2198 :
2199 :
2200 1543 : target = noce_emit_store_flag (if_info,
2201 1543 : gen_reg_rtx (GET_MODE (if_info->x)),
2202 : true, normalize);
2203 :
2204 1543 : if (target)
2205 1158 : target = expand_simple_binop (GET_MODE (if_info->x),
2206 : subtract ? MINUS : PLUS,
2207 : if_info->b, target, if_info->x,
2208 : 0, OPTAB_WIDEN);
2209 846 : if (target)
2210 : {
2211 846 : if (target != if_info->x)
2212 0 : noce_emit_move_insn (if_info->x, target);
2213 :
2214 846 : seq = end_ifcvt_sequence (if_info);
2215 846 : return noce_commit_sequence (if_info, seq, true,
2216 846 : "noce_try_addcc");
2217 : }
2218 697 : end_sequence ();
2219 : }
2220 : }
2221 :
2222 : return false;
2223 : }
2224 :
2225 : /* Convert "if (test) x = 0;" to "x &= -(test == 0);" */
2226 :
2227 : static bool
2228 157587 : noce_try_store_flag_mask (noce_if_info *if_info)
2229 : {
2230 157587 : rtx target;
2231 157587 : rtx_insn *seq;
2232 :
2233 201275 : if (!noce_simple_bbs (if_info))
2234 : return false;
2235 :
2236 : /* Match "if (test) x = 0;" and, with the arms the other way round,
2237 : "if (!test) x = 0;". */
2238 138754 : bool zero_a = (if_info->a == const0_rtx
2239 138754 : && (REG_P (if_info->b)
2240 1689 : || rtx_equal_p (if_info->b, if_info->x)));
2241 138754 : bool zero_b = (if_info->b == const0_rtx
2242 138754 : && (REG_P (if_info->a)
2243 12870 : || rtx_equal_p (if_info->a, if_info->x)));
2244 :
2245 138752 : if (zero_a || zero_b)
2246 : {
2247 1149 : bool reversep = !zero_a;
2248 1149 : start_sequence ();
2249 1149 : target = noce_emit_store_flag (if_info,
2250 1149 : gen_reg_rtx (GET_MODE (if_info->x)),
2251 : reversep, -1);
2252 1149 : if (target)
2253 299 : target = expand_simple_binop (GET_MODE (if_info->x), AND,
2254 : reversep ? if_info->a : if_info->b,
2255 : target, if_info->x, 0,
2256 : OPTAB_WIDEN);
2257 :
2258 299 : if (target)
2259 : {
2260 299 : if (target != if_info->x)
2261 0 : noce_emit_move_insn (if_info->x, target);
2262 :
2263 299 : seq = end_ifcvt_sequence (if_info);
2264 299 : return noce_commit_sequence (if_info, seq, true,
2265 299 : "noce_try_store_flag_mask");
2266 : }
2267 :
2268 850 : end_sequence ();
2269 : }
2270 :
2271 : return false;
2272 : }
2273 :
2274 : /* Emit a conditional move selecting VTRUE or VFALSE into X, and return the
2275 : destination it landed in, or NULL_RTX on failure. CODE, CMP_A and CMP_B
2276 : give the canonicalized comparison. CC_CMP and REV_CC_CMP, when given, are
2277 : the non-canonicalized condition and its reverse, which let a target emit
2278 : the move without materializing its own compare. */
2279 :
2280 : static rtx
2281 520255 : noce_emit_cmove (noce_if_info *if_info, rtx x, enum rtx_code code,
2282 : rtx cmp_a, rtx cmp_b, rtx vfalse, rtx vtrue,
2283 : rtx cc_cmp = NULL, rtx rev_cc_cmp = NULL)
2284 : {
2285 520255 : rtx target;
2286 520255 : bool unsignedp;
2287 :
2288 : /* If earliest == jump, try to build the cmove insn directly.
2289 : This is helpful when combine has created some complex condition
2290 : (like for alpha's cmovlbs) that we can't hope to regenerate
2291 : through the normal interface. */
2292 :
2293 520255 : if (if_info->cond_earliest == if_info->jump)
2294 : {
2295 4796 : rtx cond = gen_rtx_fmt_ee (code, GET_MODE (if_info->cond), cmp_a, cmp_b);
2296 4796 : rtx if_then_else = gen_rtx_IF_THEN_ELSE (GET_MODE (x),
2297 : cond, vtrue, vfalse);
2298 4796 : rtx set = gen_rtx_SET (x, if_then_else);
2299 :
2300 4796 : start_sequence ();
2301 4796 : rtx_insn *insn = emit_insn (set);
2302 :
2303 4796 : if (recog_memoized (insn) >= 0)
2304 : {
2305 1503 : rtx_insn *seq = end_sequence ();
2306 1503 : emit_insn (seq);
2307 :
2308 1503 : return x;
2309 : }
2310 :
2311 3293 : end_sequence ();
2312 : }
2313 :
2314 1037504 : unsignedp = (code == LTU || code == GEU
2315 518752 : || code == LEU || code == GTU);
2316 :
2317 518752 : if (cc_cmp != NULL_RTX && rev_cc_cmp != NULL_RTX)
2318 106116 : target = emit_conditional_move (x, cc_cmp, rev_cc_cmp,
2319 106116 : vtrue, vfalse, GET_MODE (x));
2320 : else
2321 : {
2322 : /* Don't even try if the comparison operands are weird
2323 : except that the target supports cbranchcc4. */
2324 412636 : if (! general_operand (cmp_a, GET_MODE (cmp_a))
2325 412636 : || ! general_operand (cmp_b, GET_MODE (cmp_b)))
2326 : {
2327 25319 : if (!have_cbranchcc4
2328 25319 : || GET_MODE_CLASS (GET_MODE (cmp_a)) != MODE_CC
2329 3042 : || cmp_b != const0_rtx)
2330 : return NULL_RTX;
2331 : }
2332 :
2333 390359 : target = emit_conditional_move (x, { code, cmp_a, cmp_b, VOIDmode },
2334 390359 : vtrue, vfalse, GET_MODE (x),
2335 : unsignedp);
2336 : }
2337 :
2338 496475 : if (target)
2339 : return target;
2340 :
2341 : /* We might be faced with a situation like:
2342 :
2343 : x = (reg:M TARGET)
2344 : vtrue = (subreg:M (reg:N VTRUE) BYTE)
2345 : vfalse = (subreg:M (reg:N VFALSE) BYTE)
2346 :
2347 : We can't do a conditional move in mode M, but it's possible that we
2348 : could do a conditional move in mode N instead and take a subreg of
2349 : the result.
2350 :
2351 : If we can't create new pseudos, though, don't bother. */
2352 43127 : if (reload_completed)
2353 : return NULL_RTX;
2354 :
2355 43127 : if (GET_CODE (vtrue) == SUBREG && GET_CODE (vfalse) == SUBREG)
2356 : {
2357 23 : rtx reg_vtrue = SUBREG_REG (vtrue);
2358 23 : rtx reg_vfalse = SUBREG_REG (vfalse);
2359 23 : poly_uint64 byte_vtrue = SUBREG_BYTE (vtrue);
2360 23 : poly_uint64 byte_vfalse = SUBREG_BYTE (vfalse);
2361 23 : rtx promoted_target;
2362 :
2363 23 : if (GET_MODE (reg_vtrue) != GET_MODE (reg_vfalse)
2364 23 : || maybe_ne (byte_vtrue, byte_vfalse)
2365 23 : || (SUBREG_PROMOTED_VAR_P (vtrue)
2366 23 : != SUBREG_PROMOTED_VAR_P (vfalse))
2367 23 : || (SUBREG_PROMOTED_GET (vtrue)
2368 23 : != SUBREG_PROMOTED_GET (vfalse)))
2369 : return NULL_RTX;
2370 :
2371 23 : promoted_target = gen_reg_rtx (GET_MODE (reg_vtrue));
2372 :
2373 46 : target = emit_conditional_move (promoted_target,
2374 : { code, cmp_a, cmp_b, VOIDmode },
2375 : reg_vtrue, reg_vfalse,
2376 23 : GET_MODE (reg_vtrue), unsignedp);
2377 : /* Nope, couldn't do it in that mode either. */
2378 23 : if (!target)
2379 : return NULL_RTX;
2380 :
2381 6 : target = gen_rtx_SUBREG (GET_MODE (vtrue), promoted_target, byte_vtrue);
2382 6 : SUBREG_PROMOTED_VAR_P (target) = SUBREG_PROMOTED_VAR_P (vtrue);
2383 6 : SUBREG_PROMOTED_SET (target, SUBREG_PROMOTED_GET (vtrue));
2384 6 : emit_move_insn (x, target);
2385 6 : return x;
2386 : }
2387 : else
2388 : return NULL_RTX;
2389 : }
2390 :
2391 : /* Return true if X is a constant or register operand suitable for simple
2392 : conditional-move handling. */
2393 :
2394 : static bool
2395 2608509 : noce_simple_cmove_operand_p (rtx x)
2396 : {
2397 2608509 : return CONSTANT_P (x) || register_operand (x, VOIDmode);
2398 : }
2399 :
2400 : /* Try only simple constants and registers here. More complex cases are
2401 : handled in noce_try_cmove_arith, which the dispatch table reaches after
2402 : the store-flag matchers have had a go. */
2403 :
2404 : static bool
2405 201777 : noce_try_cmove (noce_if_info *if_info)
2406 : {
2407 201777 : enum rtx_code code;
2408 201777 : rtx target;
2409 201777 : rtx_insn *seq;
2410 :
2411 262128 : if (!noce_simple_bbs (if_info))
2412 : return false;
2413 :
2414 182944 : if (noce_simple_cmove_operand_p (if_info->a)
2415 182944 : && noce_simple_cmove_operand_p (if_info->b))
2416 : {
2417 83514 : start_sequence ();
2418 :
2419 83514 : code = GET_CODE (if_info->cond);
2420 83514 : target = noce_emit_cmove (if_info, if_info->x, code,
2421 : XEXP (if_info->cond, 0),
2422 : XEXP (if_info->cond, 1),
2423 : if_info->a, if_info->b);
2424 :
2425 83514 : if (target)
2426 : {
2427 57446 : if (target != if_info->x)
2428 0 : noce_emit_move_insn (if_info->x, target);
2429 :
2430 57446 : seq = end_ifcvt_sequence (if_info);
2431 57446 : return noce_commit_sequence (if_info, seq, true, "noce_try_cmove");
2432 : }
2433 : /* If both a and b are constants try a last-ditch transformation:
2434 : if (test) x = a; else x = b;
2435 : => x = (-(test != 0) & (b - a)) + a;
2436 : Try this only if the target-specific expansion above has failed.
2437 : The target-specific expander may want to generate sequences that
2438 : we don't know about, so give them a chance before trying this
2439 : approach. */
2440 26068 : else if (!targetm.have_conditional_execution ()
2441 26068 : && CONST_INT_P (if_info->a) && CONST_INT_P (if_info->b))
2442 : {
2443 5342 : machine_mode mode = GET_MODE (if_info->x);
2444 5342 : HOST_WIDE_INT ifalse = INTVAL (if_info->a);
2445 5342 : HOST_WIDE_INT itrue = INTVAL (if_info->b);
2446 5342 : rtx target = noce_emit_store_flag (if_info, if_info->x, false, -1);
2447 5342 : if (!target)
2448 : {
2449 5282 : end_sequence ();
2450 5282 : return false;
2451 : }
2452 :
2453 60 : HOST_WIDE_INT diff;
2454 60 : if (!noce_representable_diff_p (ifalse, itrue, mode, &diff))
2455 : {
2456 21 : end_sequence ();
2457 21 : return false;
2458 : }
2459 :
2460 39 : target = expand_simple_binop (mode, AND,
2461 39 : target, gen_int_mode (diff, mode),
2462 : if_info->x, 0, OPTAB_WIDEN);
2463 39 : if (target)
2464 39 : target = expand_simple_binop (mode, PLUS,
2465 39 : target, gen_int_mode (ifalse, mode),
2466 : if_info->x, 0, OPTAB_WIDEN);
2467 39 : if (target)
2468 : {
2469 39 : if (target != if_info->x)
2470 0 : noce_emit_move_insn (if_info->x, target);
2471 :
2472 39 : seq = end_ifcvt_sequence (if_info);
2473 39 : return noce_commit_sequence (if_info, seq, true,
2474 39 : "noce_try_cmove");
2475 : }
2476 : else
2477 : {
2478 0 : end_sequence ();
2479 0 : return false;
2480 : }
2481 : }
2482 : else
2483 20726 : end_sequence ();
2484 : }
2485 :
2486 : return false;
2487 : }
2488 :
2489 : /* Return true if X contains a conditional code mode rtx. */
2490 :
2491 : static bool
2492 2906266 : contains_ccmode_rtx_p (rtx x)
2493 : {
2494 2906266 : subrtx_iterator::array_type array;
2495 7426312 : FOR_EACH_SUBRTX (iter, array, x, ALL)
2496 4567524 : if (GET_MODE_CLASS (GET_MODE (*iter)) == MODE_CC)
2497 47478 : return true;
2498 :
2499 2858788 : return false;
2500 2906266 : }
2501 :
2502 : /* Helper for bb_valid_for_noce_process_p. Validate that
2503 : the rtx insn INSN is a single set that does not set
2504 : the conditional register CC and is in general valid for
2505 : if-conversion. */
2506 :
2507 : static bool
2508 3670774 : insn_valid_noce_process_p (rtx_insn *insn, rtx cc)
2509 : {
2510 3670774 : if (!insn
2511 3670439 : || !NONJUMP_INSN_P (insn)
2512 6609052 : || (cc && set_of (cc, insn)))
2513 : return false;
2514 :
2515 2926558 : rtx sset = single_set (insn);
2516 :
2517 : /* Currently support only simple single sets in test_bb. */
2518 2926558 : if (!sset
2519 2920100 : || !noce_operand_ok (SET_DEST (sset))
2520 2906266 : || contains_ccmode_rtx_p (SET_DEST (sset))
2521 5785346 : || !noce_operand_ok (SET_SRC (sset)))
2522 134268 : return false;
2523 :
2524 : return true;
2525 : }
2526 :
2527 :
2528 : /* Return true iff the registers that the insns in BB_A set do not get
2529 : used in BB_B. If TO_RENAME is non-NULL then it is a location that will be
2530 : renamed later by the caller and so conflicts on it should be ignored
2531 : in this function. */
2532 :
2533 : static bool
2534 55090 : bbs_ok_for_cmove_arith (basic_block bb_a, basic_block bb_b, rtx to_rename)
2535 : {
2536 55090 : rtx_insn *a_insn;
2537 55090 : auto_bitmap bba_sets (®_obstack);
2538 :
2539 55090 : df_ref def;
2540 55090 : df_ref use;
2541 :
2542 290778 : FOR_BB_INSNS (bb_a, a_insn)
2543 : {
2544 235688 : if (!active_insn_p (a_insn))
2545 157969 : continue;
2546 :
2547 77719 : rtx sset_a = single_set (a_insn);
2548 :
2549 77719 : if (!sset_a)
2550 : return false;
2551 : /* Record all registers that BB_A sets. */
2552 189099 : FOR_EACH_INSN_DEF (def, a_insn)
2553 111380 : if (!(to_rename && DF_REF_REG (def) == to_rename))
2554 57541 : bitmap_set_bit (bba_sets, DF_REF_REGNO (def));
2555 : }
2556 :
2557 55090 : rtx_insn *b_insn;
2558 :
2559 290842 : FOR_BB_INSNS (bb_b, b_insn)
2560 : {
2561 235852 : if (!active_insn_p (b_insn))
2562 158140 : continue;
2563 :
2564 77712 : rtx sset_b = single_set (b_insn);
2565 :
2566 77712 : if (!sset_b)
2567 : return false;
2568 :
2569 : /* Make sure this is a REG and not some instance
2570 : of ZERO_EXTRACT or non-paradoxical SUBREG or other dangerous stuff.
2571 : If we have a memory destination then we have a pair of simple
2572 : basic blocks performing an operation of the form [addr] = c ? a : b.
2573 : bb_valid_for_noce_process_p will have ensured that these are
2574 : the only stores present. In that case [addr] should be the location
2575 : to be renamed. Assert that the callers set this up properly. */
2576 77712 : if (MEM_P (SET_DEST (sset_b)))
2577 1242 : gcc_assert (rtx_equal_p (SET_DEST (sset_b), to_rename));
2578 76470 : else if (!REG_P (SET_DEST (sset_b))
2579 131517 : && !paradoxical_subreg_p (SET_DEST (sset_b)))
2580 : return false;
2581 :
2582 : /* If the insn uses a reg set in BB_A return false. */
2583 150882 : FOR_EACH_INSN_USE (use, b_insn)
2584 73270 : if (bitmap_bit_p (bba_sets, DF_REF_REGNO (use)))
2585 : return false;
2586 :
2587 : }
2588 :
2589 : return true;
2590 55090 : }
2591 :
2592 : /* Emit copies of all the active instructions in BB except the last.
2593 : This is a helper for noce_try_cmove_arith. */
2594 :
2595 : static void
2596 14230 : noce_emit_all_but_last (basic_block bb)
2597 : {
2598 14230 : rtx_insn *last = last_active_insn (bb, false);
2599 14230 : rtx_insn *insn;
2600 145360 : FOR_BB_INSNS (bb, insn)
2601 : {
2602 116900 : if (insn != last && active_insn_p (insn))
2603 : {
2604 36762 : rtx_insn *to_emit = as_a <rtx_insn *> (copy_rtx (insn));
2605 :
2606 36762 : emit_insn (PATTERN (to_emit));
2607 : }
2608 : }
2609 14230 : }
2610 :
2611 : /* Helper for noce_try_cmove_arith. Emit the pattern TO_EMIT and return
2612 : the resulting insn or NULL if it's not a valid insn. */
2613 :
2614 : static rtx_insn *
2615 142378 : noce_emit_insn (rtx to_emit)
2616 : {
2617 142378 : gcc_assert (to_emit);
2618 142378 : rtx_insn *insn = emit_insn (to_emit);
2619 :
2620 142378 : if (recog_memoized (insn) < 0)
2621 193 : return NULL;
2622 :
2623 : return insn;
2624 : }
2625 :
2626 : /* Helper for noce_try_cmove_arith. Emit a copy of the insns up to
2627 : and including the penultimate one in BB if it is not simple
2628 : (as indicated by SIMPLE). Then emit LAST_INSN as the last
2629 : insn in the block. The reason for that is that LAST_INSN may
2630 : have been modified by the preparation in noce_try_cmove_arith. */
2631 :
2632 : static bool
2633 144788 : noce_emit_bb (rtx last_insn, basic_block bb, bool simple)
2634 : {
2635 144788 : if (bb && !simple)
2636 14230 : noce_emit_all_but_last (bb);
2637 :
2638 144788 : if (last_insn && !noce_emit_insn (last_insn))
2639 193 : return false;
2640 :
2641 : return true;
2642 : }
2643 :
2644 : /* Try more complex cases involving conditional_move. */
2645 :
2646 : static bool
2647 147372 : noce_try_cmove_arith (noce_if_info *if_info)
2648 : {
2649 147372 : rtx a = if_info->a;
2650 147372 : rtx b = if_info->b;
2651 147372 : rtx x = if_info->x;
2652 147372 : rtx orig_a, orig_b;
2653 147372 : rtx_insn *insn_a, *insn_b;
2654 147372 : bool a_simple = if_info->then_simple;
2655 147372 : bool b_simple = if_info->else_simple;
2656 147372 : basic_block then_bb = if_info->then_bb;
2657 147372 : basic_block else_bb = if_info->else_bb;
2658 147372 : rtx target;
2659 147372 : bool is_mem = false;
2660 147372 : enum rtx_code code;
2661 147372 : rtx cond = if_info->cond;
2662 147372 : rtx_insn *ifcvt_seq;
2663 :
2664 : /* A conditional move from two memory sources is equivalent to a
2665 : conditional on their addresses followed by a load. Don't do this
2666 : early because it'll screw alias analysis. Note that we've
2667 : already checked for no side effects. */
2668 147372 : if (cse_not_expected
2669 48327 : && MEM_P (a) && MEM_P (b)
2670 150053 : && MEM_ADDR_SPACE (a) == MEM_ADDR_SPACE (b))
2671 : {
2672 2680 : machine_mode address_mode = get_address_mode (a);
2673 :
2674 2680 : a = XEXP (a, 0);
2675 2680 : b = XEXP (b, 0);
2676 2680 : x = gen_reg_rtx (address_mode);
2677 2680 : is_mem = true;
2678 : }
2679 :
2680 : /* ??? We could handle this if we knew that a load from A or B could
2681 : not trap or fault. This is also true if we've already loaded
2682 : from the address along the path from ENTRY. */
2683 144692 : else if (may_trap_or_fault_p (a) || may_trap_or_fault_p (b))
2684 : return false;
2685 :
2686 : /* if (test) x = a + b; else x = c - d;
2687 : => y = a + b;
2688 : x = c - d;
2689 : if (test)
2690 : x = y;
2691 : */
2692 :
2693 92643 : code = GET_CODE (cond);
2694 92643 : insn_a = if_info->insn_a;
2695 92643 : insn_b = if_info->insn_b;
2696 :
2697 92643 : machine_mode x_mode = GET_MODE (x);
2698 :
2699 92643 : if (!can_conditionally_move_p (x_mode))
2700 : return false;
2701 :
2702 : /* Possibly rearrange operands to make things come out more natural. */
2703 72494 : if (noce_reversed_cond_code (if_info) != UNKNOWN)
2704 : {
2705 72494 : bool reversep = false;
2706 72494 : if (rtx_equal_p (b, x))
2707 : reversep = true;
2708 46251 : else if (general_operand (b, GET_MODE (b)))
2709 : reversep = true;
2710 :
2711 : if (reversep)
2712 : {
2713 64597 : cond = noce_reversed_cond (if_info, &code);
2714 64597 : std::swap (a, b);
2715 64597 : std::swap (insn_a, insn_b);
2716 64597 : std::swap (a_simple, b_simple);
2717 64597 : std::swap (then_bb, else_bb);
2718 : }
2719 : }
2720 :
2721 28383 : if (then_bb && else_bb
2722 100089 : && (!bbs_ok_for_cmove_arith (then_bb, else_bb, if_info->orig_x)
2723 27495 : || !bbs_ok_for_cmove_arith (else_bb, then_bb, if_info->orig_x)))
2724 : return false;
2725 :
2726 72394 : start_sequence ();
2727 :
2728 : /* If one of the blocks is empty then the corresponding B or A value
2729 : came from the test block. The non-empty complex block that we will
2730 : emit might clobber the register used by B or A, so move it to a pseudo
2731 : first. */
2732 :
2733 72394 : rtx tmp_a = NULL_RTX;
2734 72394 : rtx tmp_b = NULL_RTX;
2735 :
2736 72394 : if (b_simple || !else_bb)
2737 60189 : tmp_b = gen_reg_rtx (x_mode);
2738 :
2739 72394 : if (a_simple || !then_bb)
2740 70369 : tmp_a = gen_reg_rtx (x_mode);
2741 :
2742 72394 : orig_a = a;
2743 72394 : orig_b = b;
2744 :
2745 72394 : rtx emit_a = NULL_RTX;
2746 72394 : rtx emit_b = NULL_RTX;
2747 72394 : rtx_insn *tmp_insn = NULL;
2748 72394 : bool modified_in_a = false;
2749 72394 : bool modified_in_b = false;
2750 : /* If either operand is complex, load it into a register first.
2751 : The best way to do this is to copy the original insn. In this
2752 : way we preserve any clobbers etc that the insn may have had.
2753 : This is of course not possible in the IS_MEM case. */
2754 :
2755 72394 : if (! general_operand (a, GET_MODE (a)) || tmp_a)
2756 : {
2757 :
2758 71674 : if (is_mem)
2759 : {
2760 2679 : rtx reg = gen_reg_rtx (GET_MODE (a));
2761 2679 : emit_a = gen_rtx_SET (reg, a);
2762 : }
2763 : else
2764 : {
2765 68995 : if (insn_a)
2766 : {
2767 42697 : a = tmp_a ? tmp_a : gen_reg_rtx (GET_MODE (a));
2768 :
2769 42697 : rtx_insn *copy_of_a = as_a <rtx_insn *> (copy_rtx (insn_a));
2770 42697 : rtx set = single_set (copy_of_a);
2771 42697 : SET_DEST (set) = a;
2772 :
2773 42697 : emit_a = PATTERN (copy_of_a);
2774 : }
2775 : else
2776 : {
2777 26298 : rtx tmp_reg = tmp_a ? tmp_a : gen_reg_rtx (GET_MODE (a));
2778 26298 : emit_a = gen_rtx_SET (tmp_reg, a);
2779 26298 : a = tmp_reg;
2780 : }
2781 : }
2782 : }
2783 :
2784 72394 : if (! general_operand (b, GET_MODE (b)) || tmp_b)
2785 : {
2786 70704 : if (is_mem)
2787 : {
2788 2679 : rtx reg = gen_reg_rtx (GET_MODE (b));
2789 2679 : emit_b = gen_rtx_SET (reg, b);
2790 : }
2791 : else
2792 : {
2793 68025 : if (insn_b)
2794 : {
2795 67835 : b = tmp_b ? tmp_b : gen_reg_rtx (GET_MODE (b));
2796 67835 : rtx_insn *copy_of_b = as_a <rtx_insn *> (copy_rtx (insn_b));
2797 67835 : rtx set = single_set (copy_of_b);
2798 :
2799 67835 : SET_DEST (set) = b;
2800 67835 : emit_b = PATTERN (copy_of_b);
2801 : }
2802 : else
2803 : {
2804 190 : rtx tmp_reg = tmp_b ? tmp_b : gen_reg_rtx (GET_MODE (b));
2805 190 : emit_b = gen_rtx_SET (tmp_reg, b);
2806 190 : b = tmp_reg;
2807 : }
2808 : }
2809 : }
2810 :
2811 72394 : modified_in_a = emit_a != NULL_RTX && modified_in_p (orig_b, emit_a);
2812 72394 : if (tmp_b && then_bb)
2813 : {
2814 78423 : FOR_BB_INSNS (then_bb, tmp_insn)
2815 : /* Don't check inside insn_a. We will have changed it to emit_a
2816 : with a destination that doesn't conflict. */
2817 59231 : if (!(insn_a && tmp_insn == insn_a)
2818 99270 : && modified_in_p (orig_b, tmp_insn))
2819 : {
2820 : modified_in_a = true;
2821 : break;
2822 : }
2823 :
2824 : }
2825 :
2826 72394 : modified_in_b = emit_b != NULL_RTX && modified_in_p (orig_a, emit_b);
2827 72394 : if (tmp_a && else_bb)
2828 : {
2829 334759 : FOR_BB_INSNS (else_bb, tmp_insn)
2830 : /* Don't check inside insn_b. We will have changed it to emit_b
2831 : with a destination that doesn't conflict. */
2832 265157 : if (!(insn_b && tmp_insn == insn_b)
2833 460712 : && modified_in_p (orig_a, tmp_insn))
2834 : {
2835 : modified_in_b = true;
2836 : break;
2837 : }
2838 : }
2839 :
2840 : /* If insn to set up A clobbers any registers B depends on, try to
2841 : swap insn that sets up A with the one that sets up B. If even
2842 : that doesn't help, punt. */
2843 72394 : if (modified_in_a && !modified_in_b)
2844 : {
2845 0 : if (!noce_emit_bb (emit_b, else_bb, b_simple))
2846 0 : goto end_seq_and_fail;
2847 :
2848 0 : if (!noce_emit_bb (emit_a, then_bb, a_simple))
2849 0 : goto end_seq_and_fail;
2850 : }
2851 72394 : else if (!modified_in_a)
2852 : {
2853 72394 : if (!noce_emit_bb (emit_a, then_bb, a_simple))
2854 0 : goto end_seq_and_fail;
2855 :
2856 72394 : if (!noce_emit_bb (emit_b, else_bb, b_simple))
2857 193 : goto end_seq_and_fail;
2858 : }
2859 : else
2860 0 : goto end_seq_and_fail;
2861 :
2862 72201 : target = noce_emit_cmove (if_info, x, code, XEXP (cond, 0), XEXP (cond, 1),
2863 : a, b);
2864 :
2865 72201 : if (! target)
2866 17531 : goto end_seq_and_fail;
2867 :
2868 : /* If we're handling a memory for above, emit the load now. */
2869 54670 : if (is_mem)
2870 : {
2871 2288 : rtx mem = gen_rtx_MEM (GET_MODE (if_info->x), target);
2872 :
2873 : /* Copy over flags as appropriate. */
2874 2288 : if (MEM_VOLATILE_P (if_info->a) || MEM_VOLATILE_P (if_info->b))
2875 0 : MEM_VOLATILE_P (mem) = 1;
2876 2288 : if (MEM_ALIAS_SET (if_info->a) == MEM_ALIAS_SET (if_info->b))
2877 2085 : set_mem_alias_set (mem, MEM_ALIAS_SET (if_info->a));
2878 4576 : set_mem_align (mem,
2879 2288 : MIN (MEM_ALIGN (if_info->a), MEM_ALIGN (if_info->b)));
2880 :
2881 2288 : gcc_assert (MEM_ADDR_SPACE (if_info->a) == MEM_ADDR_SPACE (if_info->b));
2882 2288 : set_mem_addr_space (mem, MEM_ADDR_SPACE (if_info->a));
2883 :
2884 2288 : noce_emit_move_insn (if_info->x, mem);
2885 : }
2886 52382 : else if (target != x)
2887 0 : noce_emit_move_insn (x, target);
2888 :
2889 54670 : ifcvt_seq = end_ifcvt_sequence (if_info);
2890 54670 : return noce_commit_sequence (if_info, ifcvt_seq, true,
2891 54670 : "noce_try_cmove_arith");
2892 :
2893 17724 : end_seq_and_fail:
2894 17724 : end_sequence ();
2895 17724 : return false;
2896 : }
2897 :
2898 : /* For most cases, the simplified condition we found is the best
2899 : choice, but this is not the case for the min/max/abs transforms.
2900 : For these we wish to know that it is A or B in the condition. */
2901 :
2902 : static rtx
2903 162877 : noce_get_alt_condition (noce_if_info *if_info, rtx target,
2904 : rtx_insn **earliest)
2905 : {
2906 162877 : rtx cond, set;
2907 162877 : rtx_insn *insn;
2908 162877 : bool reverse;
2909 :
2910 : /* If target is already mentioned in the known condition, return it. */
2911 162877 : if (reg_mentioned_p (target, if_info->cond))
2912 : {
2913 8415 : *earliest = if_info->cond_earliest;
2914 8415 : return if_info->cond;
2915 : }
2916 :
2917 154462 : set = pc_set (if_info->jump);
2918 154462 : cond = XEXP (SET_SRC (set), 0);
2919 154462 : reverse
2920 308924 : = GET_CODE (XEXP (SET_SRC (set), 2)) == LABEL_REF
2921 154462 : && label_ref_label (XEXP (SET_SRC (set), 2)) == JUMP_LABEL (if_info->jump);
2922 154462 : if (if_info->then_else_reversed)
2923 24625 : reverse = !reverse;
2924 :
2925 : /* If we're looking for a constant, try to make the conditional
2926 : have that constant in it. There are two reasons why it may
2927 : not have the constant we want:
2928 :
2929 : 1. GCC may have needed to put the constant in a register, because
2930 : the target can't compare directly against that constant. For
2931 : this case, we look for a SET immediately before the comparison
2932 : that puts a constant in that register.
2933 :
2934 : 2. GCC may have canonicalized the conditional, for example
2935 : replacing "if x < 4" with "if x <= 3". We can undo that (or
2936 : make equivalent types of changes) to get the constants we need
2937 : if they're off by one in the right direction. */
2938 :
2939 154462 : if (CONST_INT_P (target))
2940 : {
2941 47145 : enum rtx_code code = GET_CODE (if_info->cond);
2942 47145 : rtx op_a = XEXP (if_info->cond, 0);
2943 47145 : rtx op_b = XEXP (if_info->cond, 1);
2944 47145 : rtx_insn *prev_insn;
2945 :
2946 : /* First, look to see if we put a constant in a register. */
2947 47145 : prev_insn = prev_nonnote_nondebug_insn (if_info->cond_earliest);
2948 47145 : if (prev_insn
2949 47034 : && BLOCK_FOR_INSN (prev_insn)
2950 47034 : == BLOCK_FOR_INSN (if_info->cond_earliest)
2951 37524 : && INSN_P (prev_insn)
2952 80424 : && GET_CODE (PATTERN (prev_insn)) == SET)
2953 : {
2954 25754 : rtx src = find_reg_equal_equiv_note (prev_insn);
2955 25754 : if (!src)
2956 25588 : src = SET_SRC (PATTERN (prev_insn));
2957 25754 : if (CONST_INT_P (src))
2958 : {
2959 15182 : if (rtx_equal_p (op_a, SET_DEST (PATTERN (prev_insn))))
2960 : op_a = src;
2961 14785 : else if (rtx_equal_p (op_b, SET_DEST (PATTERN (prev_insn))))
2962 1073 : op_b = src;
2963 :
2964 15182 : if (CONST_INT_P (op_a))
2965 : {
2966 397 : std::swap (op_a, op_b);
2967 397 : code = swap_condition (code);
2968 : }
2969 : }
2970 : }
2971 :
2972 : /* Now, look to see if we can get the right constant by
2973 : adjusting the conditional. */
2974 47145 : if (CONST_INT_P (op_b))
2975 : {
2976 24473 : HOST_WIDE_INT desired_val = INTVAL (target);
2977 24473 : HOST_WIDE_INT actual_val = INTVAL (op_b);
2978 :
2979 24473 : switch (code)
2980 : {
2981 1180 : case LT:
2982 1180 : if (desired_val != HOST_WIDE_INT_MAX
2983 1165 : && actual_val == desired_val + 1)
2984 : {
2985 174 : code = LE;
2986 174 : op_b = GEN_INT (desired_val);
2987 : }
2988 : break;
2989 3 : case LE:
2990 3 : if (desired_val != HOST_WIDE_INT_MIN
2991 3 : && actual_val == desired_val - 1)
2992 : {
2993 0 : code = LT;
2994 0 : op_b = GEN_INT (desired_val);
2995 : }
2996 : break;
2997 1717 : case GT:
2998 1717 : if (desired_val != HOST_WIDE_INT_MIN
2999 1717 : && actual_val == desired_val - 1)
3000 : {
3001 235 : code = GE;
3002 235 : op_b = GEN_INT (desired_val);
3003 : }
3004 : break;
3005 40 : case GE:
3006 40 : if (desired_val != HOST_WIDE_INT_MAX
3007 25 : && actual_val == desired_val + 1)
3008 : {
3009 3 : code = GT;
3010 3 : op_b = GEN_INT (desired_val);
3011 : }
3012 : break;
3013 : default:
3014 : break;
3015 : }
3016 : }
3017 :
3018 : /* If we made any changes, generate a new conditional that is
3019 : equivalent to what we started with, but has the right
3020 : constants in it. */
3021 47145 : if (code != GET_CODE (if_info->cond)
3022 46337 : || op_a != XEXP (if_info->cond, 0)
3023 46336 : || op_b != XEXP (if_info->cond, 1))
3024 : {
3025 1882 : cond = gen_rtx_fmt_ee (code, GET_MODE (cond), op_a, op_b);
3026 1882 : *earliest = if_info->cond_earliest;
3027 1882 : return cond;
3028 : }
3029 : }
3030 :
3031 152580 : cond = canonicalize_condition (if_info->jump, cond, reverse,
3032 : earliest, target, have_cbranchcc4, true);
3033 152580 : if (! cond || ! reg_mentioned_p (target, cond))
3034 : return NULL;
3035 :
3036 : /* We almost certainly searched back to a different place.
3037 : Need to re-verify correct lifetimes. */
3038 :
3039 : /* X may not be mentioned in the range (cond_earliest, jump]. */
3040 0 : for (insn = if_info->jump; insn != *earliest; insn = PREV_INSN (insn))
3041 0 : if (NONDEBUG_INSN_P (insn)
3042 0 : && reg_overlap_mentioned_p (if_info->x, PATTERN (insn)))
3043 : return NULL;
3044 :
3045 : /* A and B may not be modified in the range [cond_earliest, jump). */
3046 0 : for (insn = *earliest; insn != if_info->jump; insn = NEXT_INSN (insn))
3047 0 : if (INSN_P (insn)
3048 0 : && (modified_in_p (if_info->a, insn)
3049 0 : || modified_in_p (if_info->b, insn)))
3050 : return NULL;
3051 :
3052 : return cond;
3053 : }
3054 :
3055 : /* Convert "if (a < b) x = a; else x = b;" to "x = min(a, b);", etc. */
3056 :
3057 : static bool
3058 210650 : noce_try_minmax (noce_if_info *if_info)
3059 : {
3060 210650 : rtx cond, target;
3061 210650 : rtx_insn *earliest, *seq;
3062 210650 : enum rtx_code code, op;
3063 210650 : bool unsignedp;
3064 :
3065 274442 : if (!noce_simple_bbs (if_info))
3066 : return false;
3067 :
3068 : /* ??? Reject modes with NaNs or signed zeros since we don't know how
3069 : they will be resolved with an SMIN/SMAX. It wouldn't be too hard
3070 : to get the target to tell us... */
3071 191817 : if (HONOR_SIGNED_ZEROS (if_info->x)
3072 191817 : || HONOR_NANS (if_info->x))
3073 : return false;
3074 :
3075 161897 : cond = noce_get_alt_condition (if_info, if_info->a, &earliest);
3076 161897 : if (!cond)
3077 : return false;
3078 :
3079 : /* Verify the condition is of the form we expect, and canonicalize
3080 : the comparison code. */
3081 10272 : code = GET_CODE (cond);
3082 10272 : if (rtx_equal_p (XEXP (cond, 0), if_info->a))
3083 : {
3084 2662 : if (! rtx_equal_p (XEXP (cond, 1), if_info->b))
3085 : return false;
3086 : }
3087 7610 : else if (rtx_equal_p (XEXP (cond, 1), if_info->a))
3088 : {
3089 4709 : if (! rtx_equal_p (XEXP (cond, 0), if_info->b))
3090 : return false;
3091 45 : code = swap_condition (code);
3092 : }
3093 : else
3094 : return false;
3095 :
3096 : /* Determine what sort of operation this is. Note that the code is for
3097 : a taken branch, so the code->operation mapping appears backwards. */
3098 60 : switch (code)
3099 : {
3100 : case LT:
3101 : case LE:
3102 : case UNLT:
3103 : case UNLE:
3104 : op = SMAX;
3105 : unsignedp = false;
3106 : break;
3107 17 : case GT:
3108 17 : case GE:
3109 17 : case UNGT:
3110 17 : case UNGE:
3111 17 : op = SMIN;
3112 17 : unsignedp = false;
3113 17 : break;
3114 3 : case LTU:
3115 3 : case LEU:
3116 3 : op = UMAX;
3117 3 : unsignedp = true;
3118 3 : break;
3119 20 : case GTU:
3120 20 : case GEU:
3121 20 : op = UMIN;
3122 20 : unsignedp = true;
3123 20 : break;
3124 : default:
3125 : return false;
3126 : }
3127 :
3128 60 : start_sequence ();
3129 :
3130 60 : target = expand_simple_binop (GET_MODE (if_info->x), op,
3131 : if_info->a, if_info->b,
3132 : if_info->x, unsignedp, OPTAB_WIDEN);
3133 60 : if (! target)
3134 : {
3135 0 : end_sequence ();
3136 0 : return false;
3137 : }
3138 60 : if (target != if_info->x)
3139 0 : noce_emit_move_insn (if_info->x, target);
3140 :
3141 60 : seq = end_ifcvt_sequence (if_info);
3142 60 : if (!seq)
3143 : return false;
3144 :
3145 60 : emit_insn_before_setloc (seq, if_info->jump, INSN_LOCATION (if_info->insn_a));
3146 60 : if_info->cond = cond;
3147 60 : if_info->cond_earliest = earliest;
3148 60 : if_info->rev_cond = NULL_RTX;
3149 60 : if_info->transform_name = "noce_try_minmax";
3150 :
3151 60 : return true;
3152 : }
3153 :
3154 : /* Convert "if (a < 0) x = -a; else x = a;" to "x = abs(a);",
3155 : "if (a < 0) x = ~a; else x = a;" to "x = one_cmpl_abs(a);",
3156 : etc. */
3157 :
3158 : static bool
3159 210590 : noce_try_abs (noce_if_info *if_info)
3160 : {
3161 210590 : rtx cond, target, a, b, c;
3162 210590 : rtx_insn *earliest, *seq;
3163 210590 : bool negate;
3164 210590 : bool one_cmpl = false;
3165 :
3166 274373 : if (!noce_simple_bbs (if_info))
3167 : return false;
3168 :
3169 : /* Reject modes with signed zeros or NaNs. */
3170 191757 : if (HONOR_SIGNED_ZEROS (if_info->x)
3171 191757 : || HONOR_NANS (if_info->x))
3172 : return false;
3173 :
3174 : /* Recognize A and B as constituting an ABS or NABS. The canonical
3175 : form is a branch around the negation, taken when the object is the
3176 : first operand of a comparison against 0 that evaluates to true. */
3177 161837 : a = if_info->a;
3178 161837 : b = if_info->b;
3179 161837 : if (GET_CODE (a) == NEG && rtx_equal_p (XEXP (a, 0), b))
3180 : negate = false;
3181 160887 : else if (GET_CODE (b) == NEG && rtx_equal_p (XEXP (b, 0), a))
3182 : {
3183 : std::swap (a, b);
3184 : negate = true;
3185 : }
3186 160857 : else if (GET_CODE (a) == NOT && rtx_equal_p (XEXP (a, 0), b))
3187 : {
3188 : negate = false;
3189 : one_cmpl = true;
3190 : }
3191 160857 : else if (GET_CODE (b) == NOT && rtx_equal_p (XEXP (b, 0), a))
3192 : {
3193 : std::swap (a, b);
3194 : negate = true;
3195 : one_cmpl = true;
3196 : }
3197 : else
3198 : return false;
3199 :
3200 980 : cond = noce_get_alt_condition (if_info, b, &earliest);
3201 980 : if (!cond)
3202 : return false;
3203 :
3204 : /* Verify the condition is of the form we expect. */
3205 25 : if (rtx_equal_p (XEXP (cond, 0), b))
3206 25 : c = XEXP (cond, 1);
3207 0 : else if (rtx_equal_p (XEXP (cond, 1), b))
3208 : {
3209 0 : c = XEXP (cond, 0);
3210 0 : negate = !negate;
3211 : }
3212 : else
3213 : return false;
3214 :
3215 : /* Verify that C is zero. Search one step backward for a
3216 : REG_EQUAL note or a simple source if necessary. */
3217 25 : if (REG_P (c))
3218 : {
3219 0 : rtx set;
3220 0 : rtx_insn *insn = prev_nonnote_nondebug_insn (earliest);
3221 0 : if (insn
3222 0 : && BLOCK_FOR_INSN (insn) == BLOCK_FOR_INSN (earliest)
3223 0 : && (set = single_set (insn))
3224 0 : && rtx_equal_p (SET_DEST (set), c))
3225 : {
3226 0 : rtx note = find_reg_equal_equiv_note (insn);
3227 0 : if (note)
3228 0 : c = XEXP (note, 0);
3229 : else
3230 0 : c = SET_SRC (set);
3231 : }
3232 : else
3233 : return false;
3234 : }
3235 25 : if (MEM_P (c)
3236 0 : && GET_CODE (XEXP (c, 0)) == SYMBOL_REF
3237 25 : && CONSTANT_POOL_ADDRESS_P (XEXP (c, 0)))
3238 0 : c = get_pool_constant (XEXP (c, 0));
3239 :
3240 : /* Work around funny ideas get_condition has wrt canonicalization.
3241 : Note that these rtx constants are known to be CONST_INT, and
3242 : therefore imply integer comparisons.
3243 : The one_cmpl case is more complicated, as we want to handle
3244 : only x < 0 ? ~x : x or x >= 0 ? x : ~x to one_cmpl_abs (x)
3245 : and x < 0 ? x : ~x or x >= 0 ? ~x : x to ~one_cmpl_abs (x),
3246 : but not other cases (x > -1 is equivalent of x >= 0). */
3247 25 : if (c == constm1_rtx && GET_CODE (cond) == GT)
3248 : ;
3249 1 : else if (c == const1_rtx && GET_CODE (cond) == LT)
3250 : {
3251 0 : if (one_cmpl)
3252 : return false;
3253 : }
3254 1 : else if (c == CONST0_RTX (GET_MODE (b)))
3255 : {
3256 0 : if (one_cmpl
3257 0 : && GET_CODE (cond) != GE
3258 0 : && GET_CODE (cond) != LT)
3259 : return false;
3260 : }
3261 : else
3262 : return false;
3263 :
3264 : /* Determine what sort of operation this is. */
3265 24 : switch (GET_CODE (cond))
3266 : {
3267 0 : case LT:
3268 0 : case LE:
3269 0 : case UNLT:
3270 0 : case UNLE:
3271 0 : negate = !negate;
3272 0 : break;
3273 : case GT:
3274 : case GE:
3275 : case UNGT:
3276 : case UNGE:
3277 : break;
3278 : default:
3279 : return false;
3280 : }
3281 :
3282 24 : start_sequence ();
3283 24 : if (one_cmpl)
3284 0 : target = expand_one_cmpl_abs_nojump (GET_MODE (if_info->x), b,
3285 : if_info->x);
3286 : else
3287 24 : target = expand_abs_nojump (GET_MODE (if_info->x), b, if_info->x, 1);
3288 :
3289 : /* ??? It's a quandary whether cmove would be better here, especially
3290 : for integers. Perhaps combine will clean things up. */
3291 24 : if (target && negate)
3292 : {
3293 0 : if (one_cmpl)
3294 0 : target = expand_simple_unop (GET_MODE (target), NOT, target,
3295 : if_info->x, 0);
3296 : else
3297 0 : target = expand_simple_unop (GET_MODE (target), NEG, target,
3298 : if_info->x, 0);
3299 : }
3300 :
3301 24 : if (! target)
3302 : {
3303 0 : end_sequence ();
3304 0 : return false;
3305 : }
3306 :
3307 24 : if (target != if_info->x)
3308 0 : noce_emit_move_insn (if_info->x, target);
3309 :
3310 24 : seq = end_ifcvt_sequence (if_info);
3311 24 : if (!seq)
3312 : return false;
3313 :
3314 24 : emit_insn_before_setloc (seq, if_info->jump, INSN_LOCATION (if_info->insn_a));
3315 24 : if_info->cond = cond;
3316 24 : if_info->cond_earliest = earliest;
3317 24 : if_info->rev_cond = NULL_RTX;
3318 24 : if_info->transform_name = "noce_try_abs";
3319 :
3320 24 : return true;
3321 : }
3322 :
3323 : /* Convert "if (m < 0) x = b; else x = 0;" to "x = (m >> C) & b;". */
3324 :
3325 : static bool
3326 111806 : noce_try_sign_mask (noce_if_info *if_info)
3327 : {
3328 111806 : rtx cond, t, m, c;
3329 111806 : rtx_insn *seq;
3330 111806 : machine_mode mode;
3331 111806 : enum rtx_code code;
3332 111806 : bool t_unconditional;
3333 :
3334 137640 : if (!noce_simple_bbs (if_info))
3335 : return false;
3336 :
3337 100197 : cond = if_info->cond;
3338 100197 : code = GET_CODE (cond);
3339 100197 : m = XEXP (cond, 0);
3340 100197 : c = XEXP (cond, 1);
3341 :
3342 100197 : t = NULL_RTX;
3343 100197 : if (if_info->a == const0_rtx)
3344 : {
3345 2648 : if ((code == LT && c == const0_rtx)
3346 2619 : || (code == LE && c == constm1_rtx))
3347 29 : t = if_info->b;
3348 : }
3349 97549 : else if (if_info->b == const0_rtx)
3350 : {
3351 10582 : if ((code == GE && c == const0_rtx)
3352 10582 : || (code == GT && c == constm1_rtx))
3353 : t = if_info->a;
3354 : }
3355 :
3356 95 : if (! t || side_effects_p (t))
3357 : return false;
3358 :
3359 : /* We currently don't handle different modes. */
3360 95 : mode = GET_MODE (t);
3361 95 : if (GET_MODE (m) != mode)
3362 : return false;
3363 :
3364 : /* This is only profitable if T is unconditionally executed/evaluated in the
3365 : original insn sequence or T is cheap and can't trap or fault. The former
3366 : happens if B is the non-zero (T) value and if INSN_B was taken from
3367 : TEST_BB, or there was no INSN_B which can happen for e.g. conditional
3368 : stores to memory. For the cost computation use the block TEST_BB where
3369 : the evaluation will end up after the transformation. */
3370 43 : t_unconditional
3371 86 : = (t == if_info->b
3372 43 : && (if_info->insn_b == NULL_RTX
3373 0 : || BLOCK_FOR_INSN (if_info->insn_b) == if_info->test_bb));
3374 86 : if (!(t_unconditional
3375 43 : || ((set_src_cost (t, mode, if_info->speed_p)
3376 : < COSTS_N_INSNS (2))
3377 37 : && !may_trap_or_fault_p (t))))
3378 : return false;
3379 :
3380 0 : if (!noce_can_force_operand (t))
3381 : return false;
3382 :
3383 0 : start_sequence ();
3384 : /* Use emit_store_flag to generate "m < 0 ? -1 : 0" instead of expanding
3385 : "(signed) m >> 31" directly. This benefits targets with specialized
3386 : insns to obtain the signmask, but still uses ashr_optab otherwise. */
3387 0 : m = emit_store_flag (gen_reg_rtx (mode), LT, m, const0_rtx, mode, 0, -1);
3388 0 : t = m ? expand_binop (mode, and_optab, m, t, NULL_RTX, 0, OPTAB_DIRECT)
3389 : : NULL_RTX;
3390 :
3391 0 : if (!t)
3392 : {
3393 0 : end_sequence ();
3394 0 : return false;
3395 : }
3396 :
3397 0 : noce_emit_move_insn (if_info->x, t);
3398 :
3399 0 : seq = end_ifcvt_sequence (if_info);
3400 0 : return noce_commit_sequence (if_info, seq, false, "noce_try_sign_mask");
3401 : }
3402 :
3403 : /* Return the arithmetic operation in X, looking through an extension. */
3404 :
3405 : static rtx
3406 220882 : noce_cond_arith_operation (rtx x)
3407 : {
3408 220882 : enum rtx_code code = GET_CODE (x);
3409 220882 : if (code == SIGN_EXTEND || code == ZERO_EXTEND)
3410 519 : x = XEXP (x, 0);
3411 :
3412 220882 : return noce_cond_zero_binary_op_supported (x) ? x : NULL_RTX;
3413 : }
3414 :
3415 : /* Return true if X contains an EXT_CODE extension from its mode to
3416 : OUTER_MODE. */
3417 :
3418 : static bool
3419 0 : noce_operand_known_extended_p (rtx x, machine_mode outer_mode,
3420 : enum rtx_code ext_code)
3421 : {
3422 0 : machine_mode inner_mode = GET_MODE (x);
3423 0 : unsigned int precision = GET_MODE_PRECISION (outer_mode).to_constant ();
3424 0 : unsigned int inner_precision
3425 0 : = GET_MODE_PRECISION (inner_mode).to_constant ();
3426 0 : if (ext_code == SIGN_EXTEND)
3427 0 : return (num_sign_bit_copies (x, outer_mode)
3428 0 : > precision - inner_precision);
3429 :
3430 0 : if (ext_code == ZERO_EXTEND)
3431 0 : return (HWI_COMPUTABLE_MODE_P (outer_mode)
3432 0 : && (nonzero_bits (x, outer_mode)
3433 0 : & ~GET_MODE_MASK (inner_mode)) == 0);
3434 :
3435 : return false;
3436 : }
3437 :
3438 : /* Return the register or constant that EXP selects on: EXP itself for a
3439 : register or a constant, the inner register of a subreg, and NULL_RTX for
3440 : anything else. */
3441 :
3442 : static rtx
3443 28228 : get_base_reg_or_constant (rtx exp)
3444 : {
3445 0 : if (REG_P (exp))
3446 : return exp;
3447 5129 : else if (SUBREG_P (exp))
3448 309 : return SUBREG_REG (exp);
3449 4820 : else if (CONST_INT_P (exp))
3450 0 : return exp;
3451 : return NULL_RTX;
3452 : }
3453 :
3454 : /* Try to convert if-then-else with conditional zero, returning TRUE on
3455 : success or FALSE on failure. IF_INFO describes the if-conversion scenario
3456 : under consideration.
3457 :
3458 : It verifies the branch structure on the left and transforms it into the
3459 : branchless sequence on the right, with a backend-provided conditional zero
3460 : or orig for operand z. If true, tmp is z, 0 otherwise (y op 0 is the same
3461 : as y for most op).
3462 :
3463 : if (cond) | tmp = cond ? z : 0
3464 : x = y op z | x = y op tmp
3465 : else |
3466 : x = y |
3467 :
3468 : AND is special as it needs to be handled differently.
3469 :
3470 : tmp = !cond ? y : 0
3471 : x = (y & z) | tmp
3472 :
3473 : Also for AND try:
3474 :
3475 : tmp = cond ? z : -1
3476 : x = y op tmp
3477 :
3478 : to see if it is cheaper to produce `!cond ? y : 0` or `cond ? z : -1`. */
3479 :
3480 : static bool
3481 157578 : noce_try_cond_arith (noce_if_info *if_info)
3482 : {
3483 157578 : rtx target, a, b, a_op0, a_op1, outer_a;
3484 157578 : rtx cond = if_info->cond;
3485 157578 : rtx_code code = GET_CODE (cond);
3486 157578 : rtx_insn *seq;
3487 157578 : rtx_code op;
3488 157578 : machine_mode mode = GET_MODE (if_info->x);
3489 :
3490 : /* Scalar integral modes are only supported here.
3491 : Could support scalar floating point but that
3492 : would be only with -ffast-math and might
3493 : be worse than a branch. */
3494 157578 : if (!SCALAR_INT_MODE_P (mode))
3495 : return false;
3496 :
3497 167490 : if (!noce_simple_bbs (if_info))
3498 : return false;
3499 :
3500 110441 : a = copy_rtx (if_info->a);
3501 110441 : b = copy_rtx (if_info->b);
3502 110441 : rtx a_arith = noce_cond_arith_operation (a);
3503 110441 : rtx b_arith = noce_cond_arith_operation (b);
3504 :
3505 : /* Canonicalize x = y : (y op z) to x = (y op z) : y. */
3506 110441 : if (REG_P (a) && b_arith)
3507 : {
3508 204 : cond = noce_reversed_cond (if_info, &code);
3509 204 : std::swap (a, b);
3510 204 : a_arith = b_arith;
3511 : }
3512 :
3513 : /* Check if x = (y op z) : y is supported by czero based ifcvt. */
3514 110237 : else if (!(a_arith && REG_P (b)))
3515 96001 : goto fail;
3516 :
3517 14440 : if (code == UNKNOWN)
3518 0 : goto fail;
3519 :
3520 14440 : outer_a = a == a_arith ? NULL_RTX : a;
3521 14440 : a = a_arith;
3522 14440 : op = GET_CODE (a);
3523 :
3524 : /* EXTEND (y op z) : y is valid only when Y already contains the same
3525 : extension of its lowpart. AND uses a different transformation. */
3526 14440 : if (outer_a
3527 0 : && (op == AND
3528 0 : || GET_MODE (outer_a) != mode
3529 0 : || !SCALAR_INT_MODE_P (GET_MODE (a))))
3530 0 : goto fail;
3531 :
3532 : /* Canonicalize x = (z op y) : y to x = (y op z) : y */
3533 14440 : a_op1 = get_base_reg_or_constant (XEXP (a, 1));
3534 13778 : if (a_op1 && rtx_equal_p (a_op1, b) && COMMUTATIVE_ARITH_P (a))
3535 : {
3536 23 : std::swap (XEXP (a, 0), XEXP (a, 1));
3537 23 : a_op1 = get_base_reg_or_constant (XEXP (a, 1));
3538 : }
3539 :
3540 14417 : if (a_op1 == NULL_RTX)
3541 675 : goto fail;
3542 :
3543 : /* Ensure the cond is of form: x = (y op z) : y */
3544 13765 : a_op0 = get_base_reg_or_constant (XEXP (a, 0));
3545 13760 : if (!(a_op0 && rtx_equal_p (a_op0, b)))
3546 1965 : goto fail;
3547 :
3548 11800 : if (outer_a
3549 11800 : && (!SUBREG_P (XEXP (a, 0))
3550 0 : || !subreg_lowpart_p (XEXP (a, 0))
3551 0 : || GET_MODE (SUBREG_REG (XEXP (a, 0))) != mode
3552 0 : || !noce_operand_known_extended_p (XEXP (a, 0), mode,
3553 0 : GET_CODE (outer_a))))
3554 0 : goto fail;
3555 :
3556 11800 : start_sequence ();
3557 :
3558 11800 : if (CONST_INT_P (XEXP (a, 1)))
3559 2904 : target = gen_reg_rtx (GET_MODE (XEXP (a, 0)));
3560 : else
3561 8896 : target = gen_reg_rtx (GET_MODE (XEXP (a, op != AND)));
3562 :
3563 : /* AND requires !cond, instead we swap ops around. */
3564 11800 : target = noce_emit_cmove (if_info, target, code,
3565 : XEXP (cond, 0), XEXP (cond, 1),
3566 : op != AND ? XEXP (a, 1) : const0_rtx,
3567 : op != AND ? const0_rtx : XEXP (a, 0));
3568 11800 : if (!target)
3569 : {
3570 1235 : end_sequence ();
3571 1235 : rtx tmp = XEXP (a, op != AND);
3572 : /* If the cmove fails and this was a lowpart subreg,
3573 : then try the reg part and then putting back the lowpart
3574 : afterwards. */
3575 1235 : if (GET_CODE (tmp) != SUBREG || !subreg_lowpart_p (tmp))
3576 : return false;
3577 0 : tmp = SUBREG_REG (tmp);
3578 : /* Only handle integer scalar modes for the inner mode of
3579 : the subreg. */
3580 0 : if (!SCALAR_INT_MODE_P (GET_MODE (tmp)))
3581 : return false;
3582 :
3583 0 : start_sequence ();
3584 0 : target = gen_reg_rtx (GET_MODE (tmp));
3585 0 : target = noce_emit_cmove (if_info, target, code,
3586 : XEXP (cond, 0), XEXP (cond, 1),
3587 : op != AND ? tmp : const0_rtx,
3588 : op != AND ? const0_rtx : tmp);
3589 0 : if (!target)
3590 0 : goto end_seq_n_fail;
3591 0 : target = rtl_hooks.gen_lowpart_no_emit (GET_MODE (XEXP (a, op != AND)), target);
3592 0 : gcc_assert (target);
3593 : }
3594 :
3595 : /* Every path through the fallback above either returns or produces a
3596 : conditional move, so TARGET is non-null below. */
3597 : gcc_checking_assert (target);
3598 :
3599 10565 : if (outer_a)
3600 : {
3601 0 : XEXP (a, 1) = target;
3602 0 : noce_emit_move_insn (if_info->x, outer_a);
3603 0 : target = if_info->x;
3604 0 : goto success;
3605 : }
3606 :
3607 : /* For AND, try `cond ? z : -1` to see if that is cheaper or the same cost.
3608 : In some cases it will be cheaper to produce the -1 rather than the 0 case. */
3609 10565 : if (op == AND)
3610 : {
3611 84 : rtx_insn *seq0 = end_sequence ();
3612 84 : unsigned cost0 = seq_cost (seq0, if_info->speed_p);
3613 :
3614 : /* Produce `cond ? z : -1`. */
3615 84 : rtx targetm1;
3616 84 : start_sequence ();
3617 84 : targetm1 = gen_reg_rtx (GET_MODE (XEXP (a, 1)));
3618 84 : targetm1 = noce_emit_cmove (if_info, targetm1, code,
3619 : XEXP (cond, 0), XEXP (cond, 1),
3620 : XEXP (a, 1), constm1_rtx);
3621 84 : rtx_insn *seqm1 = end_sequence ();
3622 84 : unsigned costm1 = seq_cost (seqm1, if_info->speed_p);
3623 84 : if (!targetm1)
3624 78 : costm1 = -1u;
3625 :
3626 : /* If -1 is cheaper or the same cost to producing 0, then use that. */
3627 84 : if (costm1 <= cost0)
3628 : {
3629 6 : push_to_sequence (seqm1);
3630 6 : targetm1 = expand_simple_binop (mode, op, a_op0, targetm1,
3631 : if_info->x, 0, OPTAB_WIDEN);
3632 6 : if (targetm1)
3633 : {
3634 6 : target = targetm1;
3635 6 : goto success;
3636 : }
3637 0 : end_sequence ();
3638 : }
3639 :
3640 : /* For 0 the produce sequence is:
3641 : tmp = !cond ? y : 0
3642 : x = (y & z) | tmp */
3643 78 : push_to_sequence (seq0);
3644 78 : rtx a_bin = gen_reg_rtx (mode);
3645 78 : noce_emit_move_insn (a_bin, a);
3646 :
3647 78 : target = expand_simple_binop (mode, IOR, a_bin, target, if_info->x, 0,
3648 : OPTAB_WIDEN);
3649 78 : if (!target)
3650 0 : goto end_seq_n_fail;
3651 78 : goto success;
3652 : }
3653 :
3654 10481 : target = expand_simple_binop (mode, op, a_op0, target, if_info->x, 0,
3655 : OPTAB_WIDEN);
3656 :
3657 10481 : if (!target)
3658 0 : goto end_seq_n_fail;
3659 :
3660 10481 : success:
3661 10565 : if (target != if_info->x)
3662 0 : noce_emit_move_insn (if_info->x, target);
3663 :
3664 10565 : seq = end_ifcvt_sequence (if_info);
3665 10565 : return noce_commit_sequence (if_info, seq, true, "noce_try_cond_arith");
3666 :
3667 0 : end_seq_n_fail:
3668 0 : end_sequence ();
3669 :
3670 157578 : fail:
3671 :
3672 : return false;
3673 : }
3674 :
3675 : /* Optimize away "if (x & C) x |= C" and similar bit manipulation
3676 : transformations. */
3677 :
3678 : static bool
3679 210650 : noce_try_bitop (noce_if_info *if_info)
3680 : {
3681 210650 : rtx cond, x, a, result;
3682 210650 : rtx_insn *seq;
3683 210650 : scalar_int_mode mode;
3684 210650 : enum rtx_code code;
3685 210650 : int bitnum;
3686 :
3687 210650 : x = if_info->x;
3688 210650 : cond = if_info->cond;
3689 210650 : code = GET_CODE (cond);
3690 :
3691 : /* Check for an integer operation. */
3692 210650 : if (!is_a <scalar_int_mode> (GET_MODE (x), &mode))
3693 : return false;
3694 :
3695 237565 : if (!noce_simple_bbs (if_info))
3696 : return false;
3697 :
3698 : /* Check for no else condition. */
3699 160677 : if (! rtx_equal_p (x, if_info->b))
3700 : return false;
3701 :
3702 : /* Check for a suitable condition. */
3703 85016 : if (code != NE && code != EQ)
3704 : return false;
3705 60535 : if (XEXP (cond, 1) != const0_rtx)
3706 : return false;
3707 41802 : cond = XEXP (cond, 0);
3708 :
3709 : /* ??? We could also handle AND here. */
3710 41802 : if (GET_CODE (cond) == ZERO_EXTRACT)
3711 : {
3712 229 : if (XEXP (cond, 1) != const1_rtx
3713 114 : || !CONST_INT_P (XEXP (cond, 2))
3714 343 : || ! rtx_equal_p (x, XEXP (cond, 0)))
3715 : return false;
3716 4 : bitnum = INTVAL (XEXP (cond, 2));
3717 4 : if (BITS_BIG_ENDIAN)
3718 : bitnum = GET_MODE_BITSIZE (mode) - 1 - bitnum;
3719 4 : if (bitnum < 0 || bitnum >= HOST_BITS_PER_WIDE_INT)
3720 : return false;
3721 : }
3722 : else
3723 : return false;
3724 :
3725 4 : a = if_info->a;
3726 4 : if (GET_CODE (a) == IOR || GET_CODE (a) == XOR)
3727 : {
3728 : /* Check for "if (X & C) x = x op C". */
3729 0 : if (! rtx_equal_p (x, XEXP (a, 0))
3730 0 : || !CONST_INT_P (XEXP (a, 1))
3731 0 : || (INTVAL (XEXP (a, 1)) & GET_MODE_MASK (mode))
3732 0 : != HOST_WIDE_INT_1U << bitnum)
3733 : return false;
3734 :
3735 : /* if ((x & C) == 0) x |= C; is transformed to x |= C. */
3736 : /* if ((x & C) != 0) x |= C; is transformed to nothing. */
3737 0 : if (GET_CODE (a) == IOR)
3738 0 : result = (code == NE) ? a : NULL_RTX;
3739 0 : else if (code == NE)
3740 : {
3741 : /* if ((x & C) == 0) x ^= C; is transformed to x |= C. */
3742 0 : result = gen_int_mode (HOST_WIDE_INT_1 << bitnum, mode);
3743 0 : result = simplify_gen_binary (IOR, mode, x, result);
3744 : }
3745 : else
3746 : {
3747 : /* if ((x & C) != 0) x ^= C; is transformed to x &= ~C. */
3748 0 : result = gen_int_mode (~(HOST_WIDE_INT_1 << bitnum), mode);
3749 0 : result = simplify_gen_binary (AND, mode, x, result);
3750 : }
3751 : }
3752 4 : else if (GET_CODE (a) == AND)
3753 : {
3754 : /* Check for "if (X & C) x &= ~C". */
3755 0 : if (! rtx_equal_p (x, XEXP (a, 0))
3756 0 : || !CONST_INT_P (XEXP (a, 1))
3757 0 : || (INTVAL (XEXP (a, 1)) & GET_MODE_MASK (mode))
3758 0 : != (~(HOST_WIDE_INT_1 << bitnum) & GET_MODE_MASK (mode)))
3759 : return false;
3760 :
3761 : /* if ((x & C) == 0) x &= ~C; is transformed to nothing. */
3762 : /* if ((x & C) != 0) x &= ~C; is transformed to x &= ~C. */
3763 0 : result = (code == EQ) ? a : NULL_RTX;
3764 : }
3765 : else
3766 : return false;
3767 :
3768 0 : if (result)
3769 : {
3770 0 : start_sequence ();
3771 0 : noce_emit_move_insn (x, result);
3772 0 : seq = end_ifcvt_sequence (if_info);
3773 0 : if (!seq)
3774 : return false;
3775 :
3776 0 : emit_insn_before_setloc (seq, if_info->jump,
3777 0 : INSN_LOCATION (if_info->insn_a));
3778 : }
3779 0 : if_info->transform_name = "noce_try_bitop";
3780 0 : return true;
3781 : }
3782 :
3783 :
3784 : /* Similar to get_condition, only the resulting condition must be
3785 : valid at JUMP, instead of at EARLIEST.
3786 :
3787 : If THEN_ELSE_REVERSED is true, the fallthrough does not go to the
3788 : THEN block of the caller, and we have to reverse the condition. */
3789 :
3790 : static rtx
3791 4365335 : noce_get_condition (rtx_insn *jump, rtx_insn **earliest,
3792 : bool then_else_reversed)
3793 : {
3794 4365335 : rtx cond, set, tmp;
3795 4365335 : bool reverse;
3796 :
3797 4365335 : if (! any_condjump_p (jump))
3798 : return NULL_RTX;
3799 :
3800 4365335 : set = pc_set (jump);
3801 :
3802 : /* If this branches to JUMP_LABEL when the condition is false,
3803 : reverse the condition. */
3804 8730670 : reverse = (GET_CODE (XEXP (SET_SRC (set), 2)) == LABEL_REF
3805 4365335 : && label_ref_label (XEXP (SET_SRC (set), 2)) == JUMP_LABEL (jump));
3806 :
3807 : /* We may have to reverse because the caller's if block is not canonical,
3808 : i.e. the THEN block isn't the fallthrough block for the TEST block
3809 : (see find_if_header). */
3810 4365335 : if (then_else_reversed)
3811 1937175 : reverse = !reverse;
3812 :
3813 : /* If the condition variable is a register and is MODE_INT, accept it. */
3814 :
3815 4365335 : cond = XEXP (SET_SRC (set), 0);
3816 4365335 : tmp = XEXP (cond, 0);
3817 4364657 : if (REG_P (tmp) && GET_MODE_CLASS (GET_MODE (tmp)) == MODE_INT
3818 4365353 : && (GET_MODE (tmp) != BImode
3819 0 : || !targetm.small_register_classes_for_mode_p (BImode)))
3820 : {
3821 18 : *earliest = jump;
3822 :
3823 18 : if (reverse)
3824 9 : cond = gen_rtx_fmt_ee (reverse_condition (GET_CODE (cond)),
3825 : GET_MODE (cond), tmp, XEXP (cond, 1));
3826 : return cond;
3827 : }
3828 :
3829 : /* Otherwise, fall back on canonicalize_condition to do the dirty
3830 : work of manipulating MODE_CC values and COMPARE rtx codes. */
3831 4365317 : tmp = canonicalize_condition (jump, cond, reverse, earliest,
3832 : NULL_RTX, have_cbranchcc4, true);
3833 :
3834 : /* We don't handle side-effects in the condition, like handling
3835 : REG_INC notes and making sure no duplicate conditions are emitted. */
3836 4365317 : if (tmp != NULL_RTX && side_effects_p (tmp))
3837 8508 : return NULL_RTX;
3838 :
3839 : return tmp;
3840 : }
3841 :
3842 : /* Return true if OP is ok for if-then-else processing. */
3843 :
3844 : static bool
3845 9016111 : noce_operand_ok (const_rtx op)
3846 : {
3847 9016111 : if (side_effects_p (op))
3848 : return false;
3849 :
3850 : /* We special-case memories, so handle any of them with
3851 : no address side effects. */
3852 8994514 : if (MEM_P (op))
3853 1603521 : return ! side_effects_p (XEXP (op, 0));
3854 :
3855 7390993 : return ! may_trap_p (op);
3856 : }
3857 :
3858 : /* Return true iff basic block TEST_BB is valid for noce if-conversion.
3859 : The condition used in this if-conversion is in COND.
3860 : In practice, check that TEST_BB ends with a single set
3861 : x := a and all previous computations
3862 : in TEST_BB don't produce any values that are live after TEST_BB.
3863 : In other words, all the insns in TEST_BB are there only
3864 : to compute a value for x. Add the rtx cost of the insns
3865 : in TEST_BB to COST. Record whether TEST_BB is a single simple
3866 : set instruction in SIMPLE_P. */
3867 :
3868 : static bool
3869 2184087 : bb_valid_for_noce_process_p (basic_block test_bb, rtx cond,
3870 : unsigned int *cost, bool *simple_p)
3871 : {
3872 2184087 : if (!test_bb)
3873 : return false;
3874 :
3875 2184087 : rtx_insn *last_insn = last_active_insn (test_bb, false);
3876 2184087 : rtx last_set = NULL_RTX;
3877 :
3878 2184087 : rtx cc = cc_in_cond (cond);
3879 :
3880 2184087 : if (!insn_valid_noce_process_p (last_insn, cc))
3881 : return false;
3882 :
3883 : /* Punt on blocks ending with asm goto or jumps with other side-effects,
3884 : last_active_insn ignores JUMP_INSNs. */
3885 1540824 : if (JUMP_P (BB_END (test_bb)) && !onlyjump_p (BB_END (test_bb)))
3886 : return false;
3887 :
3888 1540822 : last_set = single_set (last_insn);
3889 :
3890 1540822 : rtx x = SET_DEST (last_set);
3891 1540822 : rtx_insn *first_insn = first_active_insn (test_bb);
3892 1540822 : rtx first_set = single_set (first_insn);
3893 :
3894 1540822 : if (!first_set)
3895 : return false;
3896 :
3897 : /* We have a single simple set, that's okay. */
3898 1526771 : bool speed_p = optimize_bb_for_speed_p (test_bb);
3899 :
3900 1526771 : if (first_insn == last_insn)
3901 : {
3902 552621 : *simple_p = noce_operand_ok (SET_DEST (first_set));
3903 552621 : *cost += pattern_cost (first_set, speed_p);
3904 552621 : return *simple_p;
3905 : }
3906 :
3907 974150 : rtx_insn *prev_last_insn = PREV_INSN (last_insn);
3908 974150 : gcc_assert (prev_last_insn);
3909 :
3910 : /* For now, disallow setting x multiple times in test_bb. */
3911 974150 : if (REG_P (x) && reg_set_between_p (x, first_insn, prev_last_insn))
3912 : return false;
3913 :
3914 827028 : auto_bitmap test_bb_temps (®_obstack);
3915 :
3916 : /* The regs that are live out of test_bb. */
3917 827028 : bitmap test_bb_live_out = df_get_live_out (test_bb);
3918 :
3919 827028 : int potential_cost = pattern_cost (last_set, speed_p);
3920 827028 : rtx_insn *insn;
3921 4559580 : FOR_BB_INSNS (test_bb, insn)
3922 : {
3923 4480916 : if (insn != last_insn)
3924 : {
3925 4402252 : if (!active_insn_p (insn))
3926 2915565 : continue;
3927 :
3928 1486687 : if (!insn_valid_noce_process_p (insn, cc))
3929 : return false;
3930 :
3931 1251466 : rtx sset = single_set (insn);
3932 1251466 : gcc_assert (sset);
3933 1251466 : rtx dest = SET_DEST (sset);
3934 1251466 : if (SUBREG_P (dest))
3935 18521 : dest = SUBREG_REG (dest);
3936 :
3937 1251466 : if (contains_mem_rtx_p (SET_SRC (sset))
3938 876608 : || !REG_P (dest)
3939 1998234 : || reg_overlap_mentioned_p (dest, cond))
3940 : return false;
3941 :
3942 738323 : potential_cost += pattern_cost (sset, speed_p);
3943 738323 : bitmap_set_bit (test_bb_temps, REGNO (dest));
3944 : }
3945 : }
3946 :
3947 : /* If any of the intermediate results in test_bb are live after test_bb
3948 : then fail. */
3949 78664 : if (bitmap_intersect_p (test_bb_live_out, test_bb_temps))
3950 : return false;
3951 :
3952 30501 : *cost += potential_cost;
3953 30501 : *simple_p = false;
3954 30501 : return true;
3955 827028 : }
3956 :
3957 : /* Helper function to emit a cmov sequence encapsulated in
3958 : start_sequence () and end_sequence (). If NEED_CMOV is true
3959 : we call noce_emit_cmove to create a cmove sequence. Otherwise emit
3960 : a simple move. If successful, store the first instruction of the
3961 : sequence in TEMP_DEST and the sequence costs in SEQ_COST. */
3962 :
3963 : static rtx_insn*
3964 394059 : try_emit_cmove_seq (noce_if_info *if_info, rtx temp,
3965 : rtx cond, rtx new_val, rtx old_val, bool need_cmov,
3966 : unsigned *cost, rtx *temp_dest,
3967 : rtx cc_cmp = NULL, rtx rev_cc_cmp = NULL)
3968 : {
3969 394059 : rtx_insn *seq = NULL;
3970 394059 : *cost = 0;
3971 :
3972 394059 : rtx x = XEXP (cond, 0);
3973 394059 : rtx y = XEXP (cond, 1);
3974 394059 : rtx_code cond_code = GET_CODE (cond);
3975 :
3976 394059 : start_sequence ();
3977 :
3978 394059 : if (need_cmov)
3979 317247 : *temp_dest = noce_emit_cmove (if_info, temp, cond_code,
3980 : x, y, new_val, old_val, cc_cmp, rev_cc_cmp);
3981 : else
3982 : {
3983 76812 : *temp_dest = temp;
3984 76812 : if (if_info->then_else_reversed)
3985 1904 : noce_emit_move_insn (temp, old_val);
3986 : else
3987 74908 : noce_emit_move_insn (temp, new_val);
3988 : }
3989 :
3990 394059 : if (*temp_dest != NULL_RTX)
3991 : {
3992 387897 : seq = get_insns ();
3993 387897 : *cost = seq_cost (seq, if_info->speed_p);
3994 : }
3995 :
3996 394059 : end_sequence ();
3997 :
3998 394059 : return seq;
3999 : }
4000 :
4001 : /* Finish a successful noce if-conversion of IF_INFO whose replacement insns
4002 : have already been emitted before the branch. Delete the now-dead THEN
4003 : block, and ELSE too if this was a diamond. An IF-THEN-JOIN also has an edge
4004 : from TEST_BB straight to JOIN_BB that bypassed THEN, and that goes as well.
4005 : An IF-THEN-ELSE-JOIN has no such edge, and deleting ELSE_BB removes its two
4006 : edges instead. Redirect TEST_BB to JOIN_BB and merge the two when the tail
4007 : no longer needs its own block. Count the converted if-block and the CFG
4008 : changes it took. */
4009 :
4010 : static void
4011 179092 : noce_finish_if_conversion (noce_if_info *if_info)
4012 : {
4013 179092 : basic_block test_bb = if_info->test_bb;
4014 179092 : basic_block then_bb = if_info->then_bb;
4015 179092 : basic_block else_bb = if_info->else_bb;
4016 179092 : basic_block join_bb = if_info->join_bb;
4017 :
4018 179092 : if (else_bb)
4019 : {
4020 76966 : delete_basic_block (else_bb);
4021 76966 : num_true_changes++;
4022 : }
4023 : else
4024 102126 : remove_edge (find_edge (test_bb, join_bb));
4025 :
4026 179092 : remove_edge (find_edge (then_bb, join_bb));
4027 179092 : redirect_edge_and_branch_force (single_succ_edge (test_bb), join_bb);
4028 179092 : delete_basic_block (then_bb);
4029 179092 : num_true_changes++;
4030 :
4031 179092 : if (can_merge_blocks_p (test_bb, join_bb))
4032 : {
4033 135045 : merge_blocks (test_bb, join_bb);
4034 135045 : num_true_changes++;
4035 : }
4036 :
4037 179092 : num_updated_if_blocks++;
4038 179092 : }
4039 :
4040 : /* We have something like:
4041 :
4042 : if (x > y)
4043 : { i = EXPR_A; j = EXPR_B; k = EXPR_C; }
4044 :
4045 : Make it:
4046 :
4047 : tmp_i = (x > y) ? EXPR_A : i;
4048 : tmp_j = (x > y) ? EXPR_B : j;
4049 : tmp_k = (x > y) ? EXPR_C : k;
4050 : i = tmp_i;
4051 : j = tmp_j;
4052 : k = tmp_k;
4053 :
4054 : Subsequent passes are expected to clean up the extra moves.
4055 :
4056 : Look for special cases such as writes to one register which are
4057 : read back in another SET, as might occur in a swap idiom or
4058 : similar.
4059 :
4060 : These look like:
4061 :
4062 : if (x > y)
4063 : i = a;
4064 : j = i;
4065 :
4066 : Which we want to rewrite to:
4067 :
4068 : tmp_i = (x > y) ? a : i;
4069 : tmp_j = (x > y) ? tmp_i : j;
4070 : i = tmp_i;
4071 : j = tmp_j;
4072 :
4073 : We can catch these when looking at (SET x y) by keeping a list of the
4074 : registers we would have targeted before if-conversion and looking back
4075 : through it for an overlap with Y. If we find one, we rewire the
4076 : conditional set to use the temporary we introduced earlier.
4077 :
4078 : IF_INFO contains the useful information about the block structure and
4079 : jump instructions. For an IF-THEN-ELSE-JOIN, first evaluate the else
4080 : arm's sets into temporaries and retain their final values for the
4081 : conditional moves. Return true if the replacement is valid and profitable
4082 : and its CFG changes have been committed, otherwise return false. */
4083 :
4084 : static bool
4085 50544 : noce_convert_multiple_sets (noce_if_info *if_info)
4086 : {
4087 50544 : basic_block test_bb = if_info->test_bb;
4088 50544 : basic_block then_bb = if_info->then_bb;
4089 50544 : basic_block else_bb = if_info->else_bb;
4090 50544 : rtx_insn *jump = if_info->jump;
4091 50544 : rtx_insn *cond_earliest;
4092 50544 : rtx_insn *insn;
4093 :
4094 50544 : start_sequence ();
4095 :
4096 : /* Decompose the condition attached to the jump. */
4097 50544 : rtx cond = noce_get_condition (jump, &cond_earliest, false);
4098 :
4099 50544 : auto_delete_vec<noce_multiple_sets_info> insn_info;
4100 50544 : init_noce_multiple_sets_info (then_bb, insn_info);
4101 50544 : gcc_assert (!insn_info.is_empty ());
4102 50544 : unsigned then_count = insn_info.length ();
4103 50544 : location_t sequence_location
4104 50544 : = INSN_LOCATION (insn_info.last ()->unmodified_insn);
4105 :
4106 50544 : auto_delete_vec<noce_multiple_sets_info> else_insn_info;
4107 50544 : auto_vec<unsigned> else_only_indices;
4108 50544 : if (else_bb)
4109 : {
4110 23188 : init_noce_multiple_sets_info (else_bb, else_insn_info);
4111 :
4112 : /* Add one output entry for each live-out destination that is set only
4113 : by the else arm. Record the last definition, which holds the
4114 : value that reaches the join. */
4115 23188 : auto_bitmap then_dests, seen;
4116 87766 : for (unsigned i = 0; i < then_count; ++i)
4117 64578 : bitmap_set_bit (then_dests, REGNO (insn_info[i]->target));
4118 :
4119 87132 : for (int i = else_insn_info.length () - 1; i >= 0; --i)
4120 40756 : if (else_insn_info[i]->need_cmov
4121 39230 : && !bitmap_bit_p (then_dests,
4122 39230 : REGNO (else_insn_info[i]->target))
4123 47722 : && bitmap_set_bit (seen, REGNO (else_insn_info[i]->target)))
4124 6966 : else_only_indices.safe_push (i);
4125 23188 : else_only_indices.reverse ();
4126 :
4127 23188 : unsigned i, else_index;
4128 59988 : FOR_EACH_VEC_ELT (else_only_indices, i, else_index)
4129 : {
4130 6966 : noce_multiple_sets_info *info = new noce_multiple_sets_info;
4131 6966 : noce_multiple_sets_info *else_info = else_insn_info[else_index];
4132 6966 : info->target = else_info->target;
4133 6966 : info->temporary = NULL_RTX;
4134 6966 : info->unmodified_insn = else_info->unmodified_insn;
4135 6966 : info->need_cmov = true;
4136 6966 : insn_info.safe_push (info);
4137 : }
4138 23188 : }
4139 :
4140 : /* Include else-only outputs when enforcing the conversion limit. */
4141 50544 : if (insn_info.length () > (unsigned) param_max_rtl_if_conversion_insns)
4142 : {
4143 0 : end_sequence ();
4144 0 : return false;
4145 : }
4146 :
4147 50544 : int last_needs_comparison = -1;
4148 :
4149 50544 : bool use_cond_earliest = false;
4150 :
4151 50544 : bool ok = noce_convert_multiple_sets_1
4152 50544 : (if_info, cond, insn_info, else_insn_info, then_count, else_only_indices,
4153 : &last_needs_comparison, &use_cond_earliest);
4154 50544 : if (!ok)
4155 : return false;
4156 :
4157 : /* Always perform a second attempt that uses information gathered in the
4158 : first. At least we can omit creating temporaries until we definitely
4159 : need them. The sequence created in the second attempt is never worse
4160 : than the first. */
4161 :
4162 47688 : end_sequence ();
4163 47688 : start_sequence ();
4164 47688 : ok = noce_convert_multiple_sets_1
4165 47688 : (if_info, cond, insn_info, else_insn_info, then_count, else_only_indices,
4166 : &last_needs_comparison, &use_cond_earliest);
4167 :
4168 : /* Actually we should not fail anymore if we reached here,
4169 : but better still check. */
4170 47688 : if (!ok)
4171 : return false;
4172 :
4173 : /* Now fixup the assignments.
4174 : PR116405: Iterate in reverse order and keep track of the targets so that
4175 : a move does not overwrite a subsequent value when multiple instructions
4176 : have the same target. */
4177 47688 : unsigned i;
4178 47688 : noce_multiple_sets_info *info;
4179 47688 : auto_bitmap set_targets (®_obstack);
4180 226188 : FOR_EACH_VEC_ELT_REVERSE (insn_info, i, info)
4181 : {
4182 130812 : gcc_checking_assert (REG_P (info->target));
4183 :
4184 130812 : if (info->target != info->temporary
4185 130812 : && !bitmap_bit_p (set_targets, REGNO (info->target)))
4186 4111 : noce_emit_move_insn (info->target, info->temporary);
4187 :
4188 130812 : bitmap_set_bit (set_targets, REGNO (info->target));
4189 : }
4190 :
4191 : /* Actually emit the sequence if it isn't too expensive. */
4192 47688 : rtx_insn *seq = get_insns ();
4193 :
4194 : /* If the created sequence does not use cond_earliest (but the jump
4195 : does) add its cost to the original_cost before comparing costs. */
4196 47688 : unsigned int original_cost = if_info->original_cost;
4197 47688 : if (if_info->jump != if_info->cond_earliest && !use_cond_earliest)
4198 36280 : if_info->original_cost += insn_cost (if_info->cond_earliest,
4199 : if_info->speed_p);
4200 :
4201 47688 : if (!targetm.noce_conversion_profitable_p (seq, if_info))
4202 : {
4203 14775 : end_sequence ();
4204 14775 : return false;
4205 : }
4206 :
4207 : /* Restore the original cost in case we do not succeed below. */
4208 32913 : if_info->original_cost = original_cost;
4209 :
4210 218743 : for (insn = seq; insn; insn = NEXT_INSN (insn))
4211 185830 : set_used_flags (insn);
4212 :
4213 : /* Mark all our temporaries and targets as used. */
4214 180715 : for (const noce_multiple_sets_info *msi : insn_info)
4215 : {
4216 81976 : set_used_flags (msi->temporary);
4217 81976 : set_used_flags (msi->target);
4218 : }
4219 :
4220 32913 : set_used_flags (cond);
4221 :
4222 32913 : unshare_all_rtl_in_chain (seq);
4223 32913 : end_sequence ();
4224 :
4225 32913 : if (!seq)
4226 : return false;
4227 :
4228 218541 : for (insn = seq; insn; insn = NEXT_INSN (insn))
4229 185675 : if (JUMP_P (insn) || CALL_P (insn)
4230 185675 : || recog_memoized (insn) == -1)
4231 : return false;
4232 :
4233 : /* This path does not go through end_ifcvt_sequence, so apply the same rule
4234 : about condition codes that outlive the if-region. */
4235 32866 : if (noce_clobbers_live_cc_p (test_bb, seq))
4236 : return false;
4237 :
4238 32866 : emit_insn_before_setloc (seq, if_info->jump, sequence_location);
4239 :
4240 32866 : noce_finish_if_conversion (if_info);
4241 :
4242 32866 : if_info->transform_name = "noce_convert_multiple_sets";
4243 32866 : return true;
4244 98232 : }
4245 :
4246 : /* Return true if any insn in SEQ modifies CC_CMP, or REV_CC_CMP when there is
4247 : one. Such a sequence cannot be used to select on that condition. */
4248 :
4249 : static bool
4250 207155 : noce_seq_clobbers_cc_cmp_p (rtx_insn *seq, rtx cc_cmp, rtx rev_cc_cmp)
4251 : {
4252 313551 : for (rtx_insn *insn = seq; insn; insn = NEXT_INSN (insn))
4253 222776 : if (modified_in_p (cc_cmp, insn)
4254 222776 : || (rev_cc_cmp && modified_in_p (rev_cc_cmp, insn)))
4255 : return true;
4256 :
4257 : return false;
4258 : }
4259 :
4260 : /* Try to emit the multiple-set conversion described by IF_INFO, selecting on
4261 : the already decoded jump condition COND. INSN_INFO holds THEN_COUNT
4262 : then-arm entries followed by entries for the else-only definitions recorded
4263 : in ELSE_ONLY_INDICES. For a diamond, evaluate ELSE_INSN_INFO first and
4264 : retain its final values for the conditional moves.
4265 :
4266 : LAST_NEEDS_COMPARISON is -1 on the first attempt. Record in it the last set
4267 : that needs a temporary to preserve the comparison, then use that boundary
4268 : on the second attempt. Set USE_COND_EARLIEST if the emitted sequence uses
4269 : IF_INFO->cond_earliest. Return true if the complete sequence was
4270 : emitted. */
4271 :
4272 : static bool
4273 98232 : noce_convert_multiple_sets_1 (noce_if_info *if_info, rtx cond,
4274 : auto_delete_vec<noce_multiple_sets_info> &insn_info,
4275 : auto_delete_vec<noce_multiple_sets_info>
4276 : &else_insn_info,
4277 : unsigned then_count,
4278 : const vec<unsigned> &else_only_indices,
4279 : int *last_needs_comparison,
4280 : bool *use_cond_earliest)
4281 : {
4282 98232 : rtx_insn *jump = if_info->jump;
4283 :
4284 98232 : rtx cc_cmp = cond_exec_get_condition (jump);
4285 98232 : if (cc_cmp)
4286 98232 : cc_cmp = copy_rtx (cc_cmp);
4287 98232 : rtx rev_cc_cmp = cond_exec_get_condition (jump, /* get_reversed */ true);
4288 98232 : if (rev_cc_cmp)
4289 98232 : rev_cc_cmp = copy_rtx (rev_cc_cmp);
4290 :
4291 98232 : int count = 0;
4292 98232 : bool second_try = *last_needs_comparison != -1;
4293 98232 : *use_cond_earliest = false;
4294 :
4295 : /* For an IF-THEN-ELSE-JOIN, emit the else block's computations first into
4296 : fresh temporaries. This leaves the incoming register values available to
4297 : the then arm. The conditional moves below select the then or else
4298 : values. */
4299 98232 : if (if_info->else_bb)
4300 : {
4301 44477 : int else_count = 0;
4302 44477 : rtx_insn *before_else = get_last_insn ();
4303 44477 : location_t saved_location = curr_insn_location ();
4304 44477 : rtx_insn *else_insn;
4305 258365 : FOR_BB_INSNS (if_info->else_bb, else_insn)
4306 : {
4307 213888 : if (!active_insn_p (else_insn))
4308 139773 : continue;
4309 :
4310 74115 : noce_multiple_sets_info *info = else_insn_info[else_count];
4311 74115 : rtx set = single_set (else_insn);
4312 74115 : gcc_checking_assert (set && REG_P (SET_DEST (set))
4313 : && !HARD_REGISTER_P (SET_DEST (set)));
4314 :
4315 74115 : rtx target = SET_DEST (set);
4316 74115 : rtx value = copy_rtx (SET_SRC (set));
4317 74115 : int i, ii;
4318 154905 : FOR_EACH_VEC_ELT (info->rewired_src, i, ii)
4319 6675 : value = simplify_replace_rtx (value,
4320 6675 : else_insn_info[ii]->target,
4321 6675 : else_insn_info[ii]->temporary);
4322 :
4323 74115 : set_curr_insn_location (INSN_LOCATION (else_insn));
4324 74115 : rtx temporary = copy_to_mode_reg (GET_MODE (target), value);
4325 :
4326 74115 : info->temporary = temporary;
4327 74115 : else_count++;
4328 : }
4329 :
4330 44477 : set_curr_insn_location (saved_location);
4331 :
4332 88954 : gcc_checking_assert (else_count == (int) else_insn_info.length ());
4333 :
4334 : /* These insns run ahead of the conditional moves. If they change a
4335 : register the comparison reads we cannot reuse it, so bail. If they
4336 : only clobber the condition code, drop the shared compare so that every
4337 : move re-materializes its own. */
4338 :
4339 44477 : rtx_insn *first_else
4340 44477 : = before_else ? NEXT_INSN (before_else) : get_insns ();
4341 112254 : for (rtx_insn *ei = first_else; ei; ei = NEXT_INSN (ei))
4342 : {
4343 69582 : if (modified_in_p (cond, ei))
4344 : {
4345 1805 : end_sequence ();
4346 1805 : return false;
4347 : }
4348 67777 : if (cc_cmp
4349 67777 : && (modified_in_p (cc_cmp, ei)
4350 35596 : || (rev_cc_cmp && modified_in_p (rev_cc_cmp, ei))))
4351 : {
4352 : cc_cmp = NULL_RTX;
4353 : rev_cc_cmp = NULL_RTX;
4354 : }
4355 : }
4356 : }
4357 :
4358 720120 : for (count = 0; count < (int) insn_info.length (); ++count)
4359 : {
4360 264684 : noce_multiple_sets_info *info = insn_info[count];
4361 264684 : rtx_insn *insn = info->unmodified_insn;
4362 264684 : rtx set = single_set (insn);
4363 264684 : gcc_checking_assert (set);
4364 :
4365 264684 : rtx target = info->target;
4366 264684 : rtx temp;
4367 264684 : rtx new_val;
4368 264684 : rtx old_val;
4369 :
4370 264684 : if ((unsigned) count < then_count)
4371 : {
4372 250837 : new_val = SET_SRC (set);
4373 :
4374 250837 : int i, ii;
4375 310517 : FOR_EACH_VEC_ELT (info->rewired_src, i, ii)
4376 59680 : new_val = simplify_replace_rtx (new_val, insn_info[ii]->target,
4377 59680 : insn_info[ii]->temporary);
4378 :
4379 250837 : old_val = target;
4380 :
4381 : /* Use the final value assigned to TARGET on the else arm.
4382 : Scanning in reverse is important when the arm assigns the same
4383 : register more than once. */
4384 250837 : if (if_info->else_bb)
4385 335865 : for (int j = else_insn_info.length () - 1; j >= 0; --j)
4386 157500 : if (rtx_equal_p (target, else_insn_info[j]->target))
4387 : {
4388 49939 : old_val = else_insn_info[j]->temporary;
4389 49939 : break;
4390 : }
4391 : }
4392 : else
4393 : {
4394 13847 : unsigned else_index
4395 13847 : = else_only_indices[(unsigned) count - then_count];
4396 13847 : noce_multiple_sets_info *else_info = else_insn_info[else_index];
4397 13847 : gcc_checking_assert (rtx_equal_p (target, else_info->target));
4398 :
4399 13847 : new_val = target;
4400 13847 : old_val = else_info->temporary;
4401 : }
4402 :
4403 : /* As we are transforming
4404 : if (x > y)
4405 : {
4406 : a = b;
4407 : c = d;
4408 : }
4409 : into
4410 : a = (x > y) ...
4411 : c = (x > y) ...
4412 :
4413 : we potentially check x > y before every set.
4414 : Even though the check might be removed by subsequent passes, this means
4415 : that we cannot transform
4416 : if (x > y)
4417 : {
4418 : x = y;
4419 : ...
4420 : }
4421 : into
4422 : x = (x > y) ...
4423 : ...
4424 : since this would invalidate x and the following to-be-removed checks.
4425 : Therefore we introduce a temporary every time we are about to
4426 : overwrite a variable used in the check. Costing of a sequence with
4427 : these is going to be inaccurate so only use temporaries when
4428 : needed.
4429 :
4430 : If performing a second try, we know how many insns require a
4431 : temporary. For the last of these, we can omit creating one. */
4432 264684 : if (reg_overlap_mentioned_p (target, cond)
4433 264684 : && (!second_try || count < *last_needs_comparison))
4434 26139 : temp = gen_reg_rtx (GET_MODE (target));
4435 : else
4436 : temp = target;
4437 :
4438 : /* We have identified swap-style idioms before. A normal
4439 : set will need to be a cmov while the first instruction of a swap-style
4440 : idiom can be a regular move. This helps with costing. */
4441 264684 : bool need_cmov = info->need_cmov;
4442 :
4443 : /* If we had a non-canonical conditional jump (i.e. one where
4444 : the fallthrough is to the "else" case) we need to reverse
4445 : the conditional select. */
4446 264684 : if (if_info->then_else_reversed)
4447 50925 : std::swap (old_val, new_val);
4448 :
4449 : /* Try emitting a conditional move passing the backend the
4450 : canonicalized comparison. The backend is then able to
4451 : recognize expressions like
4452 :
4453 : if (x > y)
4454 : y = x;
4455 :
4456 : as min/max and emit an insn, accordingly. */
4457 264684 : unsigned cost1 = 0, cost2 = 0;
4458 264684 : rtx_insn *seq, *seq1, *seq2 = NULL;
4459 264684 : rtx temp_dest = NULL_RTX, temp_dest1 = NULL_RTX, temp_dest2 = NULL_RTX;
4460 264684 : bool read_comparison = false;
4461 :
4462 264684 : seq1 = try_emit_cmove_seq (if_info, temp, cond,
4463 : new_val, old_val, need_cmov,
4464 : &cost1, &temp_dest1);
4465 :
4466 : /* Here, we try to pass the backend a non-canonicalized cc comparison
4467 : as well. This allows the backend to emit a cmov directly without
4468 : creating an additional compare for each. If successful, costing
4469 : is easier and this sequence is usually preferred. */
4470 264684 : if (cc_cmp)
4471 : {
4472 129375 : seq2 = try_emit_cmove_seq (if_info, temp, cond,
4473 : new_val, old_val, need_cmov,
4474 : &cost2, &temp_dest2, cc_cmp, rev_cc_cmp);
4475 :
4476 : /* The if_then_else in SEQ2 may be affected when cc_cmp/rev_cc_cmp is
4477 : clobbered. We can't safely use the sequence in this case. */
4478 129375 : if (noce_seq_clobbers_cc_cmp_p (seq2, cc_cmp, rev_cc_cmp))
4479 193639 : seq2 = NULL;
4480 : }
4481 :
4482 : /* The backend might have created a sequence that uses the
4483 : condition as a value. Check this. */
4484 :
4485 : /* We cannot handle anything more complex than a reg or constant. */
4486 264684 : if (!REG_P (XEXP (cond, 0)) && !CONSTANT_P (XEXP (cond, 0)))
4487 264684 : read_comparison = true;
4488 :
4489 264684 : if (!REG_P (XEXP (cond, 1)) && !CONSTANT_P (XEXP (cond, 1)))
4490 264684 : read_comparison = true;
4491 :
4492 264684 : rtx_insn *walk = seq2;
4493 264684 : int if_then_else_count = 0;
4494 343695 : while (walk && !read_comparison)
4495 : {
4496 79011 : rtx exprs_to_check[2];
4497 79011 : unsigned int exprs_count = 0;
4498 :
4499 79011 : rtx set = single_set (walk);
4500 79011 : if (set && XEXP (set, 1)
4501 79011 : && GET_CODE (XEXP (set, 1)) == IF_THEN_ELSE)
4502 : {
4503 : /* We assume that this is the cmove created by the backend that
4504 : naturally uses the condition. */
4505 49468 : exprs_to_check[exprs_count++] = XEXP (XEXP (set, 1), 1);
4506 49468 : exprs_to_check[exprs_count++] = XEXP (XEXP (set, 1), 2);
4507 49468 : if_then_else_count++;
4508 : }
4509 29543 : else if (NONDEBUG_INSN_P (walk))
4510 29543 : exprs_to_check[exprs_count++] = PATTERN (walk);
4511 :
4512 : /* Bail if we get more than one if_then_else because the assumption
4513 : above may be incorrect. */
4514 79011 : if (if_then_else_count > 1)
4515 : {
4516 0 : read_comparison = true;
4517 0 : break;
4518 : }
4519 :
4520 207490 : for (unsigned int i = 0; i < exprs_count; i++)
4521 : {
4522 128479 : subrtx_iterator::array_type array;
4523 286843 : FOR_EACH_SUBRTX (iter, array, exprs_to_check[i], NONCONST)
4524 189042 : if (*iter != NULL_RTX
4525 189042 : && (reg_overlap_mentioned_p (XEXP (cond, 0), *iter)
4526 171151 : || reg_overlap_mentioned_p (XEXP (cond, 1), *iter)))
4527 : {
4528 : read_comparison = true;
4529 : break;
4530 : }
4531 128479 : }
4532 :
4533 79011 : walk = NEXT_INSN (walk);
4534 : }
4535 :
4536 : /* Check which version is less expensive. */
4537 264684 : if (seq1 != NULL_RTX && (cost1 <= cost2 || seq2 == NULL_RTX))
4538 : {
4539 212609 : seq = seq1;
4540 212609 : temp_dest = temp_dest1;
4541 212609 : if (!second_try)
4542 107366 : *last_needs_comparison = count;
4543 : }
4544 1126 : else if (seq2 != NULL_RTX)
4545 : {
4546 51169 : seq = seq2;
4547 51169 : temp_dest = temp_dest2;
4548 51169 : if (!second_try && read_comparison)
4549 8590 : *last_needs_comparison = count;
4550 51169 : *use_cond_earliest = true;
4551 : }
4552 : else
4553 : {
4554 : /* Nothing worked, bail out. */
4555 906 : end_sequence ();
4556 1051 : return false;
4557 : }
4558 :
4559 : /* Although we use temporaries if there is register overlap of COND and
4560 : TARGET, it is possible that SEQ modifies COND anyway. For example,
4561 : COND may use the flags register and if INSN clobbers flags then
4562 : we may be unable to emit a valid sequence (e.g. in x86 that would
4563 : require saving and restoring the flags register). */
4564 158535 : if (!second_try)
4565 417275 : for (rtx_insn *iter = seq; iter; iter = NEXT_INSN (iter))
4566 284454 : if (modified_in_p (cond, iter))
4567 : {
4568 145 : end_sequence ();
4569 145 : return false;
4570 : }
4571 :
4572 : /* If SEQ clobbers registers mentioned in cc_cmp/rev_cc_cmp we have to
4573 : fall back on SEQ1 from that point on. Only check when we use SEQ1,
4574 : since SEQ2 has been tested already. */
4575 263633 : if (cc_cmp && seq == seq1
4576 263633 : && noce_seq_clobbers_cc_cmp_p (seq, cc_cmp, rev_cc_cmp))
4577 : {
4578 : cc_cmp = NULL_RTX;
4579 : rev_cc_cmp = NULL_RTX;
4580 : }
4581 :
4582 : /* End the sub sequence and emit to the main sequence. */
4583 263633 : emit_insn (seq);
4584 :
4585 : /* Bookkeeping. */
4586 263633 : info->temporary = temp_dest;
4587 : }
4588 :
4589 : /* Even if we did not actually need the comparison, we want to make sure
4590 : to try a second time in order to get rid of the temporaries. */
4591 95376 : if (*last_needs_comparison == -1)
4592 1014 : *last_needs_comparison = 0;
4593 :
4594 : return true;
4595 : }
4596 :
4597 : /* Fill INSN_INFO with one entry per active insn in BB. Find local swap-style
4598 : idioms and mark the first insn (1) that is only a temporary as not needing
4599 : a conditional move, as it is going to be dead afterwards anyway.
4600 :
4601 : (1) int tmp = a;
4602 : a = b;
4603 : b = tmp;
4604 :
4605 : ifcvt
4606 : -->
4607 :
4608 : tmp = a;
4609 : a = cond ? b : a_old;
4610 : b = cond ? tmp : b_old;
4611 :
4612 : Additionally, store the index of insns like (2) when a subsequent
4613 : SET reads from their destination.
4614 :
4615 : (2) int c = a;
4616 : int d = c;
4617 :
4618 : ifcvt
4619 : -->
4620 :
4621 : c = cond ? a : c_old;
4622 : d = cond ? d : c; // Need to use c rather than c_old here. */
4623 :
4624 : static void
4625 73732 : init_noce_multiple_sets_info (basic_block bb,
4626 : auto_delete_vec<noce_multiple_sets_info> &insn_info)
4627 : {
4628 73732 : rtx_insn *insn;
4629 73732 : int count = 0;
4630 73732 : auto_vec<rtx> dests;
4631 73732 : bitmap bb_live_out = df_get_live_out (bb);
4632 :
4633 : /* Iterate over all SETs, storing the destinations in DEST.
4634 : - If we encounter a previously changed register,
4635 : rewire the read to the original source.
4636 : - If we encounter a SET that writes to a destination
4637 : that is not live after this block then the register
4638 : does not need to be moved conditionally. */
4639 495124 : FOR_BB_INSNS (bb, insn)
4640 : {
4641 421392 : if (!active_insn_p (insn))
4642 245953 : continue;
4643 :
4644 175439 : noce_multiple_sets_info *info = new noce_multiple_sets_info;
4645 175439 : info->temporary = NULL_RTX;
4646 175439 : insn_info.safe_push (info);
4647 :
4648 175439 : rtx set = single_set (insn);
4649 175439 : gcc_checking_assert (set);
4650 :
4651 175439 : rtx src = SET_SRC (set);
4652 175439 : rtx dest = SET_DEST (set);
4653 :
4654 175439 : gcc_checking_assert (REG_P (dest) && !HARD_REGISTER_P (dest));
4655 175439 : info->target = dest;
4656 175439 : info->unmodified_insn = insn;
4657 175439 : info->need_cmov = bitmap_bit_p (bb_live_out, REGNO (dest));
4658 :
4659 : /* Check if the current SET's source mentions any previously seen
4660 : destination. Keep only the newest definition of each pseudo register.
4661 : This is quadratic but the number of insns in BB
4662 : is bounded by PARAM_MAX_RTL_IF_CONVERSION_INSNS. */
4663 175439 : auto_bitmap rewired_regs;
4664 347333 : for (int i = count - 1; i >= 0; --i)
4665 171894 : if (reg_mentioned_p (dests[i], src)
4666 171894 : && bitmap_set_bit (rewired_regs, REGNO (dests[i])))
4667 34329 : insn_info[count]->rewired_src.safe_push (i);
4668 :
4669 175439 : dests.safe_push (dest);
4670 175439 : count++;
4671 175439 : }
4672 73732 : }
4673 :
4674 : /* Return true iff basic block TEST_BB is suitable for conversion to a
4675 : series of conditional moves. Unless REQUIRE_MULTIPLE is false, also check
4676 : that we have more than one set (other routines can handle a single set
4677 : better than we would). A diamond arm may have a single set.
4678 : Require no more than PARAM_MAX_RTL_IF_CONVERSION_INSNS sets. While going
4679 : through the insns, store the sum of their potential costs in COST. On
4680 : success, if LIVE_OUT_DESTS is nonnull, record the distinct pseudo
4681 : destinations that are live out of
4682 : TEST_BB. If HAS_NON_SIMPLE_SRC is nonnull, set it when an instruction
4683 : source is neither a constant nor a register operand. On success, if
4684 : INSN_COUNT is nonnull, store the number of active sets in it. */
4685 :
4686 : static bool
4687 2071326 : bb_ok_for_noce_convert_multiple_sets (basic_block test_bb, unsigned *cost,
4688 : bool require_multiple = true,
4689 : bitmap live_out_dests = NULL,
4690 : bool *has_non_simple_src = NULL,
4691 : unsigned *insn_count = NULL)
4692 : {
4693 2071326 : rtx_insn *insn;
4694 2071326 : unsigned count = 0;
4695 2071326 : unsigned param = param_max_rtl_if_conversion_insns;
4696 2071326 : bool speed_p = optimize_bb_for_speed_p (test_bb);
4697 2071326 : unsigned potential_cost = 0;
4698 2071326 : if (live_out_dests)
4699 968876 : bitmap_clear (live_out_dests);
4700 2071326 : if (has_non_simple_src)
4701 968876 : *has_non_simple_src = false;
4702 2071326 : bitmap bb_live_out = NULL;
4703 2071326 : if (live_out_dests)
4704 968876 : bb_live_out = df_get_live_out (test_bb);
4705 :
4706 12782615 : FOR_BB_INSNS (test_bb, insn)
4707 : {
4708 : /* Skip over notes etc. */
4709 12409962 : if (!active_insn_p (insn))
4710 9762191 : continue;
4711 :
4712 : /* We only handle SET insns. */
4713 2647771 : rtx set = single_set (insn);
4714 2647771 : if (set == NULL_RTX)
4715 : return false;
4716 :
4717 2571750 : rtx dest = SET_DEST (set);
4718 2571750 : rtx src = SET_SRC (set);
4719 :
4720 2571750 : if (has_non_simple_src && !noce_simple_cmove_operand_p (src))
4721 601550 : *has_non_simple_src = true;
4722 :
4723 : /* Dependency rewiring is keyed by register number, so restrict
4724 : destinations to pseudos. Hard-register definitions can overlap
4725 : without having the same mode. Do not handle anything involving
4726 : memory loads/stores since it might violate data-race-freedom
4727 : guarantees. Make sure we can force SRC to a register as that may
4728 : be needed in try_emit_cmove_seq. */
4729 2145268 : if (!REG_P (dest) || HARD_REGISTER_P (dest)
4730 1704733 : || contains_mem_rtx_p (src)
4731 3593918 : || !noce_can_force_operand (src))
4732 : return false;
4733 :
4734 : /* Destination and source must be appropriate. */
4735 1004115 : if (!noce_operand_ok (dest) || !noce_operand_ok (src))
4736 : return false;
4737 :
4738 : /* We must be able to conditionally move in this mode. */
4739 978722 : if (!can_conditionally_move_p (GET_MODE (dest)))
4740 : return false;
4741 :
4742 949098 : if (bb_live_out && bitmap_bit_p (bb_live_out, REGNO (dest)))
4743 408295 : bitmap_set_bit (live_out_dests, REGNO (dest));
4744 :
4745 949098 : potential_cost += insn_cost (insn, speed_p);
4746 :
4747 949098 : count++;
4748 : }
4749 :
4750 372653 : *cost += potential_cost;
4751 :
4752 : /* If we would only put out one conditional move, the other strategies
4753 : this pass tries are better optimized and will be more appropriate, so
4754 : require more than one set unless REQUIRE_MULTIPLE is false. A diamond
4755 : arm may have one set.
4756 : Some targets want to strictly limit the number of conditional moves
4757 : that are emitted, they set this through PARAM, we need to respect
4758 : that. */
4759 630032 : bool ok = count >= (require_multiple ? 2u : 1u) && count <= param;
4760 372653 : if (ok && insn_count)
4761 255661 : *insn_count = count;
4762 : return ok;
4763 : }
4764 :
4765 : /* Compute average of two given costs weighted by relative probabilities
4766 : of respective basic blocks in an IF-THEN-ELSE. E is the IF-THEN edge.
4767 : With P as the probability to take the IF-THEN branch, return
4768 : P * THEN_COST + (1 - P) * ELSE_COST. Evaluate this as
4769 : ELSE_COST + P * (THEN_COST - ELSE_COST) when THEN_COST >= ELSE_COST, and
4770 : THEN_COST + (1 - P) * (ELSE_COST - THEN_COST) otherwise. Both forms pass
4771 : a nonnegative value to profile_probability::apply and make its rounding
4772 : independent of the CFG arm order. */
4773 : static unsigned
4774 278766 : average_cost (unsigned then_cost, unsigned else_cost, edge e)
4775 : {
4776 278766 : if (then_cost < else_cost)
4777 10061 : return then_cost
4778 10061 : + e->probability.invert ().apply ((gcov_type) else_cost - then_cost);
4779 :
4780 268705 : return else_cost
4781 268705 : + e->probability.apply ((gcov_type) then_cost - else_cost);
4782 : }
4783 :
4784 : /* Return the estimated cost of the original, un-converted if-region described
4785 : by IF_INFO whose THEN and ELSE arms cost THEN_COST and ELSE_COST. BASE_COST
4786 : already accounts for the branch, and any compare, that the conversion
4787 : removes. When optimizing for speed (SPEED_P) only one arm runs, so charge
4788 : the branch-probability-weighted average of the two. When optimizing for
4789 : size both arms are emitted, so sum them. */
4790 :
4791 : static unsigned
4792 322683 : noce_original_region_cost (const noce_if_info *if_info, bool speed_p,
4793 : unsigned base_cost, unsigned then_cost,
4794 : unsigned else_cost)
4795 : {
4796 322683 : if (speed_p)
4797 278766 : return base_cost + average_cost (then_cost, else_cost,
4798 278766 : find_edge (if_info->test_bb,
4799 557532 : if_info->then_bb));
4800 43917 : return base_cost + then_cost + else_cost;
4801 : }
4802 :
4803 : /* Given a simple IF-THEN-JOIN or IF-THEN-ELSE-JOIN block, attempt to convert
4804 : it without using conditional execution. Return TRUE if we were successful
4805 : at converting the block. */
4806 :
4807 : static bool
4808 1933326 : noce_process_if_block (noce_if_info *if_info)
4809 : {
4810 1933326 : basic_block test_bb = if_info->test_bb; /* test block */
4811 1933326 : basic_block then_bb = if_info->then_bb; /* THEN */
4812 1933326 : basic_block else_bb = if_info->else_bb; /* ELSE or NULL */
4813 1933326 : rtx_insn *jump = if_info->jump;
4814 1933326 : rtx cond = if_info->cond;
4815 1933326 : rtx_insn *insn_a, *insn_b;
4816 1933326 : rtx set_a, set_b;
4817 1933326 : rtx orig_x, x, a, b;
4818 :
4819 : /* We're looking for patterns of the form
4820 :
4821 : (1) if (...) x = a; else x = b;
4822 : (2) x = b; if (...) x = a;
4823 : (3) if (...) x = a; // as if with an initial x = x.
4824 : (4) if (...) { x = a; y = b; z = c; } // Like 3, for multiple SETS.
4825 : (5) A multi-set IF-THEN-ELSE-JOIN.
4826 : The later patterns require jumps to be more expensive.
4827 : For the diamond case, defer diamonds with only register and constant
4828 : assignments in both arms to the existing simple conditional-move
4829 : handling. Require at least two distinct live-out pseudos across the
4830 : arms.
4831 : ??? For future expansion, further expand the "multiple X" rules. */
4832 :
4833 : /* The base cost recorded so far covers the branch, and any compare, that
4834 : if-conversion removes. Each candidate path below derives its estimate
4835 : from this base plus the cost of the arms it folds in. If a target
4836 : re-uses the existing CC comparison, noce_convert_multiple_sets accounts
4837 : for that against the base before it calls
4838 : noce_conversion_profitable_p. */
4839 1933326 : unsigned base_cost = if_info->original_cost;
4840 1933326 : bool speed_p = optimize_bb_for_speed_p (test_bb);
4841 :
4842 1933326 : unsigned ms_then_cost = 0, ms_else_cost = 0;
4843 1933326 : unsigned ms_then_insn_count = 0, ms_else_insn_count = 0;
4844 1933326 : bool ms_then_has_non_simple_src, ms_else_has_non_simple_src;
4845 1933326 : auto_bitmap ms_then_live_out_dests, ms_else_live_out_dests;
4846 1933326 : noce_if_info ms_if_info = *if_info;
4847 1933326 : bool multiple_sets_p = false;
4848 :
4849 1933326 : if (HAVE_conditional_move)
4850 : {
4851 1933326 : if (!else_bb)
4852 1102450 : multiple_sets_p
4853 1102450 : = bb_ok_for_noce_convert_multiple_sets (then_bb, &ms_then_cost);
4854 830876 : else if (!if_info->then_else_reversed
4855 830876 : && bb_ok_for_noce_convert_multiple_sets
4856 830876 : (then_bb, &ms_then_cost, false,
4857 : ms_then_live_out_dests, &ms_then_has_non_simple_src,
4858 : &ms_then_insn_count)
4859 968876 : && bb_ok_for_noce_convert_multiple_sets
4860 138000 : (else_bb, &ms_else_cost, false,
4861 : ms_else_live_out_dests, &ms_else_has_non_simple_src,
4862 : &ms_else_insn_count))
4863 : {
4864 117661 : auto_bitmap ms_live_out_dests;
4865 117661 : bitmap_ior (ms_live_out_dests, ms_then_live_out_dests,
4866 117661 : ms_else_live_out_dests);
4867 117661 : unsigned output_count = bitmap_count_bits (ms_live_out_dests);
4868 :
4869 117661 : if (output_count >= 2
4870 50791 : && output_count <= (unsigned) param_max_rtl_if_conversion_insns
4871 50789 : && (ms_then_has_non_simple_src || ms_else_has_non_simple_src))
4872 117661 : multiple_sets_p = true;
4873 117661 : }
4874 : }
4875 :
4876 : /* The diamond conversion evaluates the else arm unconditionally.
4877 : Honor the target limit on the number of instructions made
4878 : unconditional. */
4879 1220111 : if (multiple_sets_p
4880 53655 : && ms_if_info.else_bb
4881 1246410 : && ms_else_insn_count > (unsigned) MAX_CONDITIONAL_EXECUTE)
4882 : multiple_sets_p = false;
4883 :
4884 1930215 : if (multiple_sets_p)
4885 : {
4886 : /* The original code runs the comparison and one arm. Estimate that cost
4887 : and let noce_convert_multiple_sets convert only if the conditional
4888 : moves come out cheaper. */
4889 50544 : ms_if_info.original_cost
4890 101088 : = ms_if_info.else_bb
4891 50544 : ? noce_original_region_cost (&ms_if_info, speed_p, base_cost,
4892 : ms_then_cost, ms_else_cost)
4893 27356 : : base_cost + ms_then_cost;
4894 50544 : if (noce_convert_multiple_sets (&ms_if_info))
4895 : {
4896 32866 : if (dump_file && ms_if_info.transform_name)
4897 2 : fprintf (dump_file, "if-conversion succeeded through %s\n",
4898 : ms_if_info.transform_name);
4899 : return true;
4900 : }
4901 : }
4902 :
4903 1900460 : unsigned int then_cost = 0, else_cost = 0;
4904 1900460 : if (!bb_valid_for_noce_process_p (then_bb, cond, &then_cost,
4905 : &if_info->then_simple))
4906 : return false;
4907 :
4908 483325 : if (else_bb
4909 483325 : && !bb_valid_for_noce_process_p (else_bb, cond, &else_cost,
4910 : &if_info->else_simple))
4911 : return false;
4912 :
4913 299495 : if_info->original_cost
4914 299495 : = noce_original_region_cost (if_info, speed_p, base_cost, then_cost,
4915 : else_cost);
4916 :
4917 299495 : insn_a = last_active_insn (then_bb, false);
4918 299495 : set_a = single_set (insn_a);
4919 299495 : gcc_assert (set_a);
4920 :
4921 299495 : x = SET_DEST (set_a);
4922 299495 : a = SET_SRC (set_a);
4923 :
4924 : /* Look for the other potential set. Make sure we've got equivalent
4925 : destinations. */
4926 : /* ??? This is overconservative. Storing to two different mems is
4927 : as easy as conditionally computing the address. Storing to a
4928 : single mem merely requires a scratch memory to use as one of the
4929 : destination addresses; often the memory immediately below the
4930 : stack pointer is available for this. */
4931 299495 : set_b = NULL_RTX;
4932 299495 : if (else_bb)
4933 : {
4934 99797 : insn_b = last_active_insn (else_bb, false);
4935 99797 : set_b = single_set (insn_b);
4936 99797 : gcc_assert (set_b);
4937 :
4938 99797 : if (!rtx_interchangeable_p (x, SET_DEST (set_b)))
4939 : return false;
4940 : }
4941 : else
4942 : {
4943 199698 : insn_b = if_info->cond_earliest;
4944 539640 : do
4945 539640 : insn_b = prev_nonnote_nondebug_insn (insn_b);
4946 : while (insn_b
4947 536930 : && (BLOCK_FOR_INSN (insn_b)
4948 536930 : == BLOCK_FOR_INSN (if_info->cond_earliest))
4949 1190315 : && !modified_in_p (x, insn_b));
4950 :
4951 : /* We're going to be moving the evaluation of B down from above
4952 : COND_EARLIEST to JUMP. Make sure the relevant data is still
4953 : intact. */
4954 199698 : if (! insn_b
4955 196988 : || BLOCK_FOR_INSN (insn_b) != BLOCK_FOR_INSN (if_info->cond_earliest)
4956 111035 : || !NONJUMP_INSN_P (insn_b)
4957 105226 : || (set_b = single_set (insn_b)) == NULL_RTX
4958 103726 : || ! rtx_interchangeable_p (x, SET_DEST (set_b))
4959 89190 : || ! noce_operand_ok (SET_SRC (set_b))
4960 87217 : || reg_overlap_mentioned_p (x, SET_SRC (set_b))
4961 85761 : || modified_between_p (SET_SRC (set_b), insn_b, jump)
4962 : /* Avoid extending the lifetime of hard registers on small
4963 : register class machines. */
4964 83548 : || (REG_P (SET_SRC (set_b))
4965 20684 : && HARD_REGISTER_P (SET_SRC (set_b))
4966 3696 : && targetm.small_register_classes_for_mode_p
4967 3696 : (GET_MODE (SET_SRC (set_b))))
4968 : /* Likewise with X. In particular this can happen when
4969 : noce_get_condition looks farther back in the instruction
4970 : stream than one might expect. */
4971 79852 : || reg_overlap_mentioned_p (x, cond)
4972 66193 : || reg_overlap_mentioned_p (x, a)
4973 259912 : || modified_between_p (x, insn_b, jump))
4974 : {
4975 : insn_b = NULL;
4976 : set_b = NULL_RTX;
4977 : }
4978 : }
4979 :
4980 : /* If x has side effects then only the if-then-else form is safe to
4981 : convert. But even in that case we would need to restore any notes
4982 : (such as REG_INC) at then end. That can be tricky if
4983 : noce_emit_move_insn expands to more than one insn, so disable the
4984 : optimization entirely for now if there are side effects. */
4985 293831 : if (side_effects_p (x))
4986 : return false;
4987 :
4988 293831 : b = (set_b ? SET_SRC (set_b) : x);
4989 :
4990 : /* Only operate on register destinations, and even then avoid extending
4991 : the lifetime of hard registers on small register class machines. */
4992 293831 : orig_x = x;
4993 293831 : if_info->orig_x = orig_x;
4994 293831 : if (!REG_P (x)
4995 293831 : || (HARD_REGISTER_P (x)
4996 2 : && targetm.small_register_classes_for_mode_p (GET_MODE (x))))
4997 : {
4998 67845 : if (GET_MODE (x) == BLKmode)
4999 : return false;
5000 :
5001 67603 : if (GET_CODE (x) == ZERO_EXTRACT
5002 1 : && (!CONST_INT_P (XEXP (x, 1))
5003 1 : || !CONST_INT_P (XEXP (x, 2))))
5004 : return false;
5005 :
5006 67603 : x = gen_reg_rtx (GET_MODE (GET_CODE (x) == STRICT_LOW_PART
5007 : ? XEXP (x, 0) : x));
5008 : }
5009 :
5010 : /* Don't operate on sources that may trap or are volatile. */
5011 293589 : if (! noce_operand_ok (a) || ! noce_operand_ok (b))
5012 : return false;
5013 :
5014 320521 : retry:
5015 : /* Set up the info block for our subroutines. */
5016 320521 : if_info->insn_a = insn_a;
5017 320521 : if_info->insn_b = insn_b;
5018 320521 : if_info->x = x;
5019 320521 : if_info->a = a;
5020 320521 : if_info->b = b;
5021 :
5022 : /* Try optimizations in some approximation of a useful order. */
5023 : /* ??? Should first look to see if X is live incoming at all. If it
5024 : isn't, we don't need anything but an unconditional set. */
5025 :
5026 : /* Look and see if A and B are really the same. Avoid creating silly
5027 : cmove constructs that no one will fix up later. */
5028 320521 : if (noce_simple_bbs (if_info)
5029 298296 : && rtx_interchangeable_p (a, b))
5030 : {
5031 : /* If we have an INSN_B, we don't have to create any new rtl. Just
5032 : move the instruction that we already have. If we don't have an
5033 : INSN_B, that means that A == X, and we've got a noop move. In
5034 : that case don't do anything and let the code below delete INSN_A. */
5035 339 : if (insn_b && else_bb)
5036 : {
5037 330 : rtx note;
5038 :
5039 330 : if (insn_b == BB_END (else_bb))
5040 220 : BB_END (else_bb) = PREV_INSN (insn_b);
5041 330 : reorder_insns (insn_b, insn_b, PREV_INSN (jump));
5042 :
5043 : /* If there was a REG_EQUAL note, delete it since it may have been
5044 : true due to this insn being after a jump. */
5045 330 : if ((note = find_reg_note (insn_b, REG_EQUAL, NULL_RTX)) != 0)
5046 0 : remove_note (insn_b, note);
5047 :
5048 339 : insn_b = NULL;
5049 : }
5050 : /* If we have "x = b; if (...) x = a;", and x has side-effects, then
5051 : x must be executed twice. */
5052 9 : else if (insn_b && side_effects_p (orig_x))
5053 : return false;
5054 :
5055 339 : x = orig_x;
5056 339 : goto success;
5057 : }
5058 :
5059 320182 : if (!set_b && MEM_P (orig_x))
5060 : /* We want to avoid store speculation to avoid cases like
5061 : if (pthread_mutex_trylock(mutex))
5062 : ++global_variable;
5063 : Rather than go to much effort here, we rely on the SSA optimizers,
5064 : which do a good enough job these days. */
5065 : return false;
5066 :
5067 254264 : if (noce_try_move (if_info))
5068 0 : goto success;
5069 254264 : if (noce_try_ifelse_collapse (if_info))
5070 17618 : goto success;
5071 236646 : if (noce_try_store_flag (if_info))
5072 25996 : goto success;
5073 210650 : if (noce_try_bitop (if_info))
5074 0 : goto success;
5075 210650 : if (noce_try_minmax (if_info))
5076 60 : goto success;
5077 210590 : if (noce_try_abs (if_info))
5078 24 : goto success;
5079 210566 : if (noce_try_inverse_constants (if_info))
5080 0 : goto success;
5081 210566 : if (!targetm.have_conditional_execution ()
5082 210566 : && noce_try_store_flag_constants (if_info))
5083 3185 : goto success;
5084 207381 : if (!targetm.have_conditional_execution ()
5085 207381 : && noce_try_shifted_store_flag (if_info))
5086 5424 : goto success;
5087 201957 : if (noce_try_sign_bit_splat (if_info))
5088 180 : goto success;
5089 201777 : if (!targetm.have_conditional_execution ()
5090 201777 : && noce_try_store_flag_logical (if_info))
5091 0 : goto success;
5092 201777 : if (HAVE_conditional_move
5093 201777 : && noce_try_cmove (if_info))
5094 43269 : goto success;
5095 158508 : if (! targetm.have_conditional_execution ())
5096 : {
5097 158508 : if (noce_try_addcc (if_info))
5098 921 : goto success;
5099 157587 : if (noce_try_store_flag_mask (if_info))
5100 9 : goto success;
5101 157578 : if (HAVE_conditional_move
5102 157578 : && noce_try_cond_arith (if_info))
5103 10206 : goto success;
5104 147372 : if (HAVE_conditional_move
5105 147372 : && noce_try_cmove_arith (if_info))
5106 35566 : goto success;
5107 111806 : if (noce_try_sign_mask (if_info))
5108 0 : goto success;
5109 : }
5110 :
5111 111806 : if (!else_bb && set_b)
5112 : {
5113 26932 : insn_b = NULL;
5114 26932 : set_b = NULL_RTX;
5115 26932 : b = orig_x;
5116 26932 : goto retry;
5117 : }
5118 :
5119 : return false;
5120 :
5121 142797 : success:
5122 142797 : if (dump_file && if_info->transform_name)
5123 7 : fprintf (dump_file, "if-conversion succeeded through %s\n",
5124 : if_info->transform_name);
5125 :
5126 : /* If we used a temporary, fix it up now. */
5127 142797 : if (orig_x != x)
5128 : {
5129 679 : rtx_insn *seq;
5130 :
5131 679 : start_sequence ();
5132 679 : noce_emit_move_insn (orig_x, x);
5133 679 : seq = get_insns ();
5134 679 : set_used_flags (orig_x);
5135 679 : unshare_all_rtl_in_chain (seq);
5136 679 : end_sequence ();
5137 :
5138 679 : emit_insn_before_setloc (seq, BB_END (test_bb), INSN_LOCATION (insn_a));
5139 : }
5140 :
5141 : /* The original THEN and ELSE blocks may now be removed and the test block
5142 : redirected to the join block. */
5143 142797 : noce_finish_if_conversion (if_info);
5144 :
5145 142797 : return true;
5146 1933326 : }
5147 :
5148 : /* Check whether a block is suitable for conditional move conversion.
5149 : Every insn must be a simple set of a register to a constant or a
5150 : register. For each assignment, store the value in the pointer map
5151 : VALS, keyed by register pointer, then store the register pointer in
5152 : REGS. COND is the condition we will test. */
5153 :
5154 : static bool
5155 1787699 : check_cond_move_block (basic_block bb,
5156 : hash_map<rtx, rtx> *vals,
5157 : vec<rtx> *regs,
5158 : rtx cond)
5159 : {
5160 1787699 : rtx_insn *insn;
5161 1787699 : rtx cc = cc_in_cond (cond);
5162 :
5163 : /* We can only handle simple jumps at the end of the basic block.
5164 : It is almost impossible to update the CFG otherwise. */
5165 1787699 : insn = BB_END (bb);
5166 1787699 : if (JUMP_P (insn) && !onlyjump_p (insn))
5167 : return false;
5168 :
5169 10807941 : FOR_BB_INSNS (bb, insn)
5170 : {
5171 10745657 : rtx set, dest, src;
5172 :
5173 10745657 : if (!NONDEBUG_INSN_P (insn) || JUMP_P (insn))
5174 8885605 : continue;
5175 1860052 : set = single_set (insn);
5176 1860052 : if (!set)
5177 1725362 : return false;
5178 :
5179 1786101 : dest = SET_DEST (set);
5180 1786101 : src = SET_SRC (set);
5181 1786101 : if (!REG_P (dest)
5182 1786101 : || (HARD_REGISTER_P (dest)
5183 316347 : && targetm.small_register_classes_for_mode_p (GET_MODE (dest))))
5184 : return false;
5185 :
5186 1078697 : if (!noce_simple_cmove_operand_p (src))
5187 : return false;
5188 :
5189 157493 : if (side_effects_p (src) || side_effects_p (dest))
5190 : return false;
5191 :
5192 157493 : if (may_trap_p (src) || may_trap_p (dest))
5193 : return false;
5194 :
5195 : /* Don't try to handle this if the source register was
5196 : modified earlier in the block. */
5197 157493 : if ((REG_P (src)
5198 51861 : && vals->get (src))
5199 209093 : || (GET_CODE (src) == SUBREG && REG_P (SUBREG_REG (src))
5200 5303 : && vals->get (SUBREG_REG (src))))
5201 : return false;
5202 :
5203 : /* Don't try to handle this if the destination register was
5204 : modified earlier in the block. */
5205 157177 : if (vals->get (dest))
5206 : return false;
5207 :
5208 : /* Don't try to handle this if the condition uses the
5209 : destination register. */
5210 157177 : if (reg_overlap_mentioned_p (dest, cond))
5211 : return false;
5212 :
5213 : /* Don't try to handle this if the source register is modified
5214 : later in the block. */
5215 138093 : if (!CONSTANT_P (src)
5216 138093 : && modified_between_p (src, insn, NEXT_INSN (BB_END (bb))))
5217 : return false;
5218 :
5219 : /* Skip it if the instruction to be moved might clobber CC. */
5220 134690 : if (cc && set_of (cc, insn))
5221 : return false;
5222 :
5223 134690 : vals->put (dest, src);
5224 :
5225 134690 : regs->safe_push (dest);
5226 : }
5227 :
5228 : return true;
5229 : }
5230 :
5231 : /* Given a basic block BB suitable for conditional move conversion,
5232 : a condition COND, and pointer maps THEN_VALS and ELSE_VALS containing
5233 : the register values depending on COND, emit the insns in the block as
5234 : conditional moves. If ELSE_BLOCK is true, THEN_BB was already
5235 : processed. The caller has started a sequence for the conversion.
5236 : Return true if successful, false if something goes wrong. */
5237 :
5238 : static bool
5239 36958 : cond_move_convert_if_block (noce_if_info *if_infop,
5240 : basic_block bb, rtx cond,
5241 : hash_map<rtx, rtx> *then_vals,
5242 : hash_map<rtx, rtx> *else_vals,
5243 : bool else_block_p)
5244 : {
5245 36958 : enum rtx_code code;
5246 36958 : rtx_insn *insn;
5247 36958 : rtx cond_arg0, cond_arg1;
5248 :
5249 36958 : code = GET_CODE (cond);
5250 36958 : cond_arg0 = XEXP (cond, 0);
5251 36958 : cond_arg1 = XEXP (cond, 1);
5252 :
5253 127312 : FOR_BB_INSNS (bb, insn)
5254 : {
5255 104678 : rtx set, target, dest, t, e;
5256 :
5257 : /* ??? Maybe emit conditional debug insn? */
5258 104678 : if (!NONDEBUG_INSN_P (insn) || JUMP_P (insn))
5259 69269 : continue;
5260 46609 : set = single_set (insn);
5261 46609 : gcc_assert (set && REG_P (SET_DEST (set)));
5262 :
5263 46609 : dest = SET_DEST (set);
5264 :
5265 46609 : rtx *then_slot = then_vals->get (dest);
5266 46609 : rtx *else_slot = else_vals->get (dest);
5267 46609 : t = then_slot ? *then_slot : NULL_RTX;
5268 46609 : e = else_slot ? *else_slot : NULL_RTX;
5269 :
5270 46609 : if (else_block_p)
5271 : {
5272 : /* If this register was set in the then block, we already
5273 : handled this case there. */
5274 12449 : if (t)
5275 11200 : continue;
5276 1249 : t = dest;
5277 1249 : gcc_assert (e);
5278 : }
5279 : else
5280 : {
5281 34160 : gcc_assert (t);
5282 34160 : if (!e)
5283 22271 : e = dest;
5284 : }
5285 :
5286 35409 : if (if_infop->cond_inverted)
5287 0 : std::swap (t, e);
5288 :
5289 35409 : target = noce_emit_cmove (if_infop, dest, code, cond_arg0, cond_arg1,
5290 : t, e);
5291 35409 : if (!target)
5292 14324 : return false;
5293 :
5294 21085 : if (target != dest)
5295 0 : noce_emit_move_insn (dest, target);
5296 : }
5297 :
5298 : return true;
5299 : }
5300 :
5301 : /* Given a simple IF-THEN-JOIN or IF-THEN-ELSE-JOIN block, attempt to convert
5302 : it using only conditional moves. Return TRUE if we were successful at
5303 : converting the block. */
5304 :
5305 : static bool
5306 1757663 : cond_move_process_if_block (noce_if_info *if_info)
5307 : {
5308 1757663 : basic_block then_bb = if_info->then_bb;
5309 1757663 : basic_block else_bb = if_info->else_bb;
5310 1757663 : rtx_insn *jump = if_info->jump;
5311 1757663 : rtx cond = if_info->cond;
5312 1757663 : rtx_insn *seq, *loc_insn;
5313 1757663 : int c;
5314 1757663 : auto_vec<rtx> then_regs;
5315 1757663 : auto_vec<rtx> else_regs;
5316 1757663 : int limit = param_max_rtl_if_conversion_insns;
5317 :
5318 : /* Build a mapping for each block to the value used for each
5319 : register. */
5320 1757663 : hash_map<rtx, rtx> then_vals;
5321 1757663 : hash_map<rtx, rtx> else_vals;
5322 :
5323 : /* Make sure the blocks are suitable. */
5324 1757663 : if (!check_cond_move_block (then_bb, &then_vals, &then_regs, cond)
5325 1757663 : || (else_bb
5326 30036 : && !check_cond_move_block (else_bb, &else_vals, &else_regs, cond)))
5327 : return false;
5328 :
5329 : /* Make sure the blocks can be used together. If the same register
5330 : is set in both blocks, and is not set to a constant in both
5331 : cases, then both blocks must set it to the same register. We
5332 : have already verified that if it is set to a register, that the
5333 : source register does not change after the assignment. Also count
5334 : the number of registers set in only one of the blocks. */
5335 32248 : c = 0;
5336 131622 : for (rtx reg : then_regs)
5337 : {
5338 38297 : rtx *then_slot = then_vals.get (reg);
5339 38297 : rtx *else_slot = else_vals.get (reg);
5340 :
5341 38297 : gcc_checking_assert (then_slot);
5342 38297 : if (!else_slot)
5343 22770 : ++c;
5344 : else
5345 : {
5346 15527 : rtx then_val = *then_slot;
5347 15527 : rtx else_val = *else_slot;
5348 4565 : if (!CONSTANT_P (then_val) && !CONSTANT_P (else_val)
5349 19409 : && !rtx_equal_p (then_val, else_val))
5350 3017 : return false;
5351 : }
5352 : }
5353 :
5354 : /* Finish off c for MAX_CONDITIONAL_EXECUTE. */
5355 59790 : for (rtx reg : else_regs)
5356 : {
5357 13713 : gcc_checking_assert (else_vals.get (reg));
5358 13713 : if (!then_vals.get (reg))
5359 1358 : ++c;
5360 : }
5361 :
5362 : /* Make sure it is reasonable to convert this block. What matters
5363 : is the number of assignments currently made in only one of the
5364 : branches, since if we convert we are going to always execute
5365 : them. */
5366 29231 : if (c > MAX_CONDITIONAL_EXECUTE
5367 29231 : || c > limit)
5368 : return false;
5369 :
5370 : /* Try to emit the conditional moves. First do the then block,
5371 : then do anything left in the else blocks. */
5372 29217 : start_sequence ();
5373 29217 : if (!cond_move_convert_if_block (if_info, then_bb, cond,
5374 : &then_vals, &else_vals, false)
5375 29217 : || (else_bb
5376 7741 : && !cond_move_convert_if_block (if_info, else_bb, cond,
5377 : &then_vals, &else_vals, true)))
5378 : {
5379 14324 : end_sequence ();
5380 14324 : return false;
5381 : }
5382 14893 : seq = end_ifcvt_sequence (if_info);
5383 14893 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
5384 : return false;
5385 :
5386 3429 : loc_insn = first_active_insn (then_bb);
5387 3429 : if (!loc_insn)
5388 : {
5389 0 : loc_insn = first_active_insn (else_bb);
5390 0 : gcc_assert (loc_insn);
5391 : }
5392 3429 : emit_insn_before_setloc (seq, jump, INSN_LOCATION (loc_insn));
5393 :
5394 3429 : noce_finish_if_conversion (if_info);
5395 :
5396 3429 : return true;
5397 1757663 : }
5398 :
5399 :
5400 : /* Determine if a given basic block heads a simple IF-THEN-JOIN or an
5401 : IF-THEN-ELSE-JOIN block.
5402 :
5403 : If so, we'll try to convert the insns to not require the branch,
5404 : using only transformations that do not require conditional execution.
5405 :
5406 : Return TRUE if we were successful at converting the block. */
5407 :
5408 : static bool
5409 10014384 : noce_find_if_block (basic_block test_bb, edge then_edge, edge else_edge,
5410 : int pass)
5411 : {
5412 10014384 : basic_block then_bb, else_bb, join_bb;
5413 10014384 : bool then_else_reversed = false;
5414 10014384 : rtx_insn *jump;
5415 10014384 : rtx_insn *cond_earliest;
5416 10014384 : noce_if_info if_info = {};
5417 10014384 : bool speed_p = optimize_bb_for_speed_p (test_bb);
5418 :
5419 : /* We only ever should get here before reload. */
5420 10014384 : gcc_assert (!reload_completed);
5421 :
5422 : /* Recognize an IF-THEN-ELSE-JOIN block. */
5423 10014384 : if (single_pred_p (then_edge->dest)
5424 12714007 : && single_succ_p (then_edge->dest)
5425 3596672 : && single_pred_p (else_edge->dest)
5426 1873896 : && single_succ_p (else_edge->dest)
5427 11192997 : && single_succ (then_edge->dest) == single_succ (else_edge->dest))
5428 : {
5429 : then_bb = then_edge->dest;
5430 : else_bb = else_edge->dest;
5431 : join_bb = single_succ (then_bb);
5432 : }
5433 : /* Recognize an IF-THEN-JOIN block. */
5434 17262627 : else if (single_pred_p (then_edge->dest)
5435 7324406 : && single_succ_p (then_edge->dest)
5436 11816958 : && single_succ (then_edge->dest) == else_edge->dest)
5437 : {
5438 : then_bb = then_edge->dest;
5439 : else_bb = NULL_BLOCK;
5440 : join_bb = else_edge->dest;
5441 : }
5442 : /* Recognize an IF-ELSE-JOIN block. We can have those because the order
5443 : of basic blocks in cfglayout mode does not matter, so the fallthrough
5444 : edge can go to any basic block (and not just to bb->next_bb, like in
5445 : cfgrtl mode). */
5446 8145292 : else if (single_pred_p (else_edge->dest)
5447 3895340 : && single_succ_p (else_edge->dest)
5448 9572046 : && single_succ (else_edge->dest) == then_edge->dest)
5449 : {
5450 : /* The noce transformations do not apply to IF-ELSE-JOIN blocks.
5451 : To make this work, we have to invert the THEN and ELSE blocks
5452 : and reverse the jump condition. */
5453 : then_bb = else_edge->dest;
5454 : else_bb = NULL_BLOCK;
5455 : join_bb = single_succ (then_bb);
5456 : then_else_reversed = true;
5457 : }
5458 : else
5459 : /* Not a form we can handle. */
5460 : return false;
5461 :
5462 : /* The edges of the THEN and ELSE blocks cannot have complex edges. */
5463 2003219 : if (single_succ_edge (then_bb)->flags & EDGE_COMPLEX)
5464 : return false;
5465 1947457 : if (else_bb
5466 1947457 : && single_succ_edge (else_bb)->flags & EDGE_COMPLEX)
5467 : return false;
5468 :
5469 1937436 : num_possible_if_blocks++;
5470 :
5471 1937436 : if (dump_file)
5472 : {
5473 93 : fprintf (dump_file,
5474 : "\nIF-THEN%s-JOIN block found, pass %d, test %d, then %d",
5475 : (else_bb) ? "-ELSE" : "",
5476 : pass, test_bb->index, then_bb->index);
5477 :
5478 56 : if (else_bb)
5479 19 : fprintf (dump_file, ", else %d", else_bb->index);
5480 :
5481 56 : fprintf (dump_file, ", join %d\n", join_bb->index);
5482 : }
5483 :
5484 : /* If the conditional jump is more than just a conditional
5485 : jump, then we cannot do if-conversion on this block. */
5486 1937436 : jump = BB_END (test_bb);
5487 1937436 : if (! onlyjump_p (jump))
5488 : return false;
5489 :
5490 : /* Initialize an IF_INFO struct to pass around. */
5491 1937175 : if_info.test_bb = test_bb;
5492 1937175 : if_info.then_bb = then_bb;
5493 1937175 : if_info.else_bb = else_bb;
5494 1937175 : if_info.join_bb = join_bb;
5495 1937175 : if_info.cond = noce_get_condition (jump, &cond_earliest,
5496 : then_else_reversed);
5497 1937175 : rtx_insn *rev_cond_earliest;
5498 1937175 : if_info.rev_cond = noce_get_condition (jump, &rev_cond_earliest,
5499 : !then_else_reversed);
5500 1937175 : if (!if_info.cond && !if_info.rev_cond)
5501 : return false;
5502 1933326 : if (!if_info.cond)
5503 : {
5504 0 : std::swap (if_info.cond, if_info.rev_cond);
5505 0 : std::swap (cond_earliest, rev_cond_earliest);
5506 0 : if_info.cond_inverted = true;
5507 : }
5508 : /* We must be comparing objects whose modes imply the size. */
5509 1933326 : if (GET_MODE (XEXP (if_info.cond, 0)) == BLKmode)
5510 : return false;
5511 1933326 : gcc_assert (if_info.rev_cond == NULL_RTX
5512 : || rev_cond_earliest == cond_earliest);
5513 1933326 : if_info.cond_earliest = cond_earliest;
5514 1933326 : if_info.jump = jump;
5515 1933326 : if_info.then_else_reversed = then_else_reversed;
5516 1933326 : if_info.speed_p = speed_p;
5517 1933326 : if_info.max_seq_cost
5518 1933326 : = targetm.max_noce_ifcvt_seq_cost (then_edge);
5519 : /* We'll add in the cost of THEN_BB and ELSE_BB later, when we check
5520 : that they are valid to transform. We can't easily get back to the insn
5521 : for COND (and it may not exist if we had to canonicalize to get COND).
5522 : It is assumed that the costs of a jump insn are dependent on the
5523 : branch costs. */
5524 1933326 : if_info.original_cost += insn_cost (if_info.jump, if_info.speed_p);
5525 :
5526 : /* Do the real work. */
5527 :
5528 : /* ??? noce_process_if_block has not yet been updated to handle
5529 : inverted conditions. */
5530 1933326 : if (!if_info.cond_inverted && noce_process_if_block (&if_info))
5531 : return true;
5532 :
5533 1757663 : if (HAVE_conditional_move
5534 1757663 : && cond_move_process_if_block (&if_info))
5535 : return true;
5536 :
5537 : return false;
5538 : }
5539 :
5540 :
5541 : /* Merge BB into COMBO_BB. BB has no successor edges left, so if COMBO_BB
5542 : still has another successor the BARRIER that follows BB is no longer needed
5543 : and it is in fact incorrect to leave it in the insn stream. */
5544 :
5545 : static void
5546 0 : merge_block_into_combo (basic_block combo_bb, basic_block bb)
5547 : {
5548 0 : if (EDGE_COUNT (bb->succs) == 0 && EDGE_COUNT (combo_bb->succs) > 1)
5549 : {
5550 0 : rtx_insn *end = NEXT_INSN (BB_END (bb));
5551 0 : while (end && NOTE_P (end) && !NOTE_INSN_BASIC_BLOCK_P (end))
5552 0 : end = NEXT_INSN (end);
5553 :
5554 0 : if (end && BARRIER_P (end))
5555 0 : delete_insn (end);
5556 : }
5557 :
5558 0 : merge_blocks (combo_bb, bb);
5559 0 : num_true_changes++;
5560 0 : }
5561 :
5562 : /* Merge the blocks and mark for local life update. */
5563 :
5564 : static void
5565 0 : merge_if_block (ce_if_block *ce_info)
5566 : {
5567 0 : basic_block test_bb = ce_info->test_bb; /* last test block */
5568 0 : basic_block then_bb = ce_info->then_bb; /* THEN */
5569 0 : basic_block else_bb = ce_info->else_bb; /* ELSE or NULL */
5570 0 : basic_block join_bb = ce_info->join_bb; /* join block */
5571 0 : basic_block combo_bb;
5572 :
5573 : /* All block merging is done into the lower block numbers. */
5574 :
5575 0 : combo_bb = test_bb;
5576 0 : df_set_bb_dirty (test_bb);
5577 :
5578 : /* Merge any basic blocks to handle && and || subtests. Each of
5579 : the blocks are on the fallthru path from the predecessor block. */
5580 0 : if (ce_info->num_multiple_test_blocks > 0)
5581 : {
5582 0 : basic_block bb = test_bb;
5583 0 : basic_block last_test_bb = ce_info->last_test_bb;
5584 0 : basic_block fallthru = block_fallthru (bb);
5585 :
5586 0 : do
5587 : {
5588 0 : bb = fallthru;
5589 0 : fallthru = block_fallthru (bb);
5590 0 : merge_blocks (combo_bb, bb);
5591 0 : num_true_changes++;
5592 : }
5593 0 : while (bb != last_test_bb);
5594 : }
5595 :
5596 : /* Merge TEST block into THEN block. Normally the THEN block won't have a
5597 : label, but it might if there were || tests. That label's count should be
5598 : zero, and it normally should be removed. */
5599 :
5600 0 : if (then_bb)
5601 0 : merge_block_into_combo (combo_bb, then_bb);
5602 :
5603 : /* The ELSE block, if it existed, had a label. That label count
5604 : will almost always be zero, but odd things can happen when labels
5605 : get their addresses taken. */
5606 0 : if (else_bb)
5607 0 : merge_block_into_combo (combo_bb, else_bb);
5608 :
5609 : /* If there was no join block reported, that means it was not adjacent
5610 : to the others, and so we cannot merge them. */
5611 :
5612 0 : if (! join_bb)
5613 : {
5614 0 : rtx_insn *last = BB_END (combo_bb);
5615 :
5616 : /* The outgoing edge for the current COMBO block should already
5617 : be correct. Verify this. */
5618 0 : if (EDGE_COUNT (combo_bb->succs) == 0)
5619 0 : gcc_assert (find_reg_note (last, REG_NORETURN, NULL)
5620 : || (NONJUMP_INSN_P (last)
5621 : && GET_CODE (PATTERN (last)) == TRAP_IF
5622 : && (TRAP_CONDITION (PATTERN (last))
5623 : == const_true_rtx)));
5624 :
5625 : else
5626 : /* There should still be something at the end of the THEN or ELSE
5627 : blocks taking us to our final destination. */
5628 0 : gcc_assert (JUMP_P (last)
5629 : || (EDGE_SUCC (combo_bb, 0)->dest
5630 : == EXIT_BLOCK_PTR_FOR_FN (cfun)
5631 : && CALL_P (last)
5632 : && SIBLING_CALL_P (last))
5633 : || ((EDGE_SUCC (combo_bb, 0)->flags & EDGE_EH)
5634 : && can_throw_internal (last)));
5635 : }
5636 :
5637 : /* The JOIN block may have had quite a number of other predecessors too.
5638 : Since we've already merged the TEST, THEN and ELSE blocks, we should
5639 : have only one remaining edge from our if-then-else diamond. If there
5640 : is more than one remaining edge, it must come from elsewhere. There
5641 : may be zero incoming edges if the THEN block didn't actually join
5642 : back up (as with a call to a non-return function). */
5643 0 : else if (EDGE_COUNT (join_bb->preds) < 2
5644 0 : && join_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
5645 : {
5646 : /* We can merge the JOIN cleanly and update the dataflow try
5647 : again on this pass.*/
5648 0 : merge_blocks (combo_bb, join_bb);
5649 0 : num_true_changes++;
5650 : }
5651 : else
5652 : {
5653 : /* We cannot merge the JOIN. */
5654 :
5655 : /* The outgoing edge for the current COMBO block should already
5656 : be correct. Verify this. */
5657 0 : gcc_assert (single_succ_p (combo_bb)
5658 : && single_succ (combo_bb) == join_bb);
5659 :
5660 : /* Remove the jump and cruft from the end of the COMBO block. */
5661 0 : if (join_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
5662 0 : tidy_fallthru_edge (single_succ_edge (combo_bb));
5663 : }
5664 :
5665 0 : num_updated_if_blocks++;
5666 0 : }
5667 :
5668 : /* Find a block ending in a simple IF condition and try to transform it
5669 : in some way. When converting a multi-block condition, put the new code
5670 : in the first such block and delete the rest. Return a pointer to this
5671 : first block if some transformation was done. Return NULL otherwise. */
5672 :
5673 : static basic_block
5674 40471419 : find_if_header (basic_block test_bb, int pass)
5675 : {
5676 40471419 : ce_if_block ce_info;
5677 40471419 : edge then_edge;
5678 40471419 : edge else_edge;
5679 :
5680 : /* The kind of block we're looking for has exactly two successors. */
5681 40471419 : if (EDGE_COUNT (test_bb->succs) != 2)
5682 : return NULL;
5683 :
5684 20851279 : then_edge = EDGE_SUCC (test_bb, 0);
5685 20851279 : else_edge = EDGE_SUCC (test_bb, 1);
5686 :
5687 20851279 : if (df_get_bb_dirty (then_edge->dest))
5688 : return NULL;
5689 20829189 : if (df_get_bb_dirty (else_edge->dest))
5690 : return NULL;
5691 :
5692 : /* Neither edge should be abnormal. */
5693 20793409 : if ((then_edge->flags & EDGE_COMPLEX)
5694 18963606 : || (else_edge->flags & EDGE_COMPLEX))
5695 : return NULL;
5696 :
5697 : /* Nor exit the loop. */
5698 18500988 : if ((then_edge->flags & EDGE_LOOP_EXIT)
5699 17016105 : || (else_edge->flags & EDGE_LOOP_EXIT))
5700 : return NULL;
5701 :
5702 : /* The THEN edge is canonically the one that falls through. */
5703 14693037 : if (then_edge->flags & EDGE_FALLTHRU)
5704 : ;
5705 7017425 : else if (else_edge->flags & EDGE_FALLTHRU)
5706 : std::swap (then_edge, else_edge);
5707 : else
5708 : /* Otherwise this must be a multiway branch of some sort. */
5709 : return NULL;
5710 :
5711 14692679 : memset (&ce_info, 0, sizeof (ce_info));
5712 14692679 : ce_info.test_bb = test_bb;
5713 14692679 : ce_info.then_bb = then_edge->dest;
5714 14692679 : ce_info.else_bb = else_edge->dest;
5715 14692679 : ce_info.pass = pass;
5716 :
5717 : #ifdef IFCVT_MACHDEP_INIT
5718 : IFCVT_MACHDEP_INIT (&ce_info);
5719 : #endif
5720 :
5721 14692679 : if (!reload_completed
5722 14692679 : && noce_find_if_block (test_bb, then_edge, else_edge, pass))
5723 179092 : goto success;
5724 :
5725 14513587 : if (reload_completed
5726 4678295 : && targetm.have_conditional_execution ()
5727 14513587 : && cond_exec_find_if_block (&ce_info))
5728 0 : goto success;
5729 :
5730 9835292 : if (!reload_completed && targetm.have_trap ()
5731 9835292 : && optab_handler (ctrap_optab, word_mode) != CODE_FOR_nothing
5732 14513587 : && find_cond_trap (test_bb, then_edge, else_edge))
5733 0 : goto success;
5734 :
5735 14513587 : if (dom_info_state (CDI_POST_DOMINATORS) >= DOM_NO_FAST_QUERY
5736 14513587 : && (reload_completed || !targetm.have_conditional_execution ()))
5737 : {
5738 14513587 : if (find_if_case_1 (test_bb, then_edge, else_edge))
5739 20984 : goto success;
5740 14492603 : if (find_if_case_2 (test_bb, then_edge, else_edge))
5741 126960 : goto success;
5742 : }
5743 :
5744 : return NULL;
5745 :
5746 327036 : success:
5747 327036 : if (dump_file)
5748 22 : fprintf (dump_file, "Conversion succeeded on pass %d.\n", pass);
5749 : /* Set this so we continue looking. */
5750 327036 : ifcvt_changed_p = true;
5751 327036 : return ce_info.test_bb;
5752 : }
5753 :
5754 : /* CUR_BB has two edges, one falling through to the next block and one
5755 : jumping to TARGET_BB, so it can be part of an && test or an || test.
5756 : Return the number of non-note, non-jump, non-USE/CLOBBER insns in it, or
5757 : -1 if CUR_BB is not of that form. */
5758 :
5759 : static int
5760 0 : block_jumps_and_fallthru (basic_block cur_bb, basic_block target_bb)
5761 : {
5762 0 : edge cur_edge;
5763 0 : bool fallthru_p = false;
5764 0 : bool jump_p = false;
5765 0 : rtx_insn *insn;
5766 0 : rtx_insn *end;
5767 0 : int n_insns = 0;
5768 0 : edge_iterator ei;
5769 :
5770 0 : if (!cur_bb || !target_bb)
5771 : return -1;
5772 :
5773 : /* If no edges, obviously it doesn't jump or fallthru. */
5774 0 : if (EDGE_COUNT (cur_bb->succs) == 0)
5775 : return 0;
5776 :
5777 0 : FOR_EACH_EDGE (cur_edge, ei, cur_bb->succs)
5778 : {
5779 0 : if (cur_edge->flags & EDGE_COMPLEX)
5780 : /* Anything complex isn't what we want. */
5781 : return -1;
5782 :
5783 0 : else if (cur_edge->flags & EDGE_FALLTHRU)
5784 : fallthru_p = true;
5785 :
5786 0 : else if (cur_edge->dest == target_bb)
5787 : jump_p = true;
5788 :
5789 : else
5790 : return -1;
5791 : }
5792 :
5793 0 : if (!jump_p || !fallthru_p)
5794 : return -1;
5795 :
5796 : /* Don't allow calls in the block, since this is used to group && and ||
5797 : together for conditional execution support. ??? we should support
5798 : conditional execution support across calls for IA-64 some day, but
5799 : for now it makes the code simpler. */
5800 0 : end = BB_END (cur_bb);
5801 0 : insn = BB_HEAD (cur_bb);
5802 :
5803 0 : while (insn)
5804 : {
5805 0 : if (CALL_P (insn))
5806 : return -1;
5807 :
5808 0 : if (INSN_P (insn)
5809 0 : && !JUMP_P (insn)
5810 0 : && !DEBUG_INSN_P (insn)
5811 0 : && GET_CODE (PATTERN (insn)) != USE
5812 0 : && GET_CODE (PATTERN (insn)) != CLOBBER)
5813 0 : n_insns++;
5814 :
5815 0 : if (insn == end)
5816 : break;
5817 :
5818 0 : insn = NEXT_INSN (insn);
5819 : }
5820 :
5821 : return n_insns;
5822 : }
5823 :
5824 : /* Determine if a given basic block heads a simple IF-THEN or IF-THEN-ELSE
5825 : block. If so, we'll try to convert the insns to not require the branch.
5826 : Return TRUE if we were successful at converting the block. */
5827 :
5828 : static bool
5829 0 : cond_exec_find_if_block (ce_if_block *ce_info)
5830 : {
5831 0 : basic_block test_bb = ce_info->test_bb;
5832 0 : basic_block then_bb = ce_info->then_bb;
5833 0 : basic_block else_bb = ce_info->else_bb;
5834 0 : basic_block join_bb = NULL_BLOCK;
5835 0 : edge cur_edge;
5836 0 : basic_block next;
5837 0 : edge_iterator ei;
5838 :
5839 0 : ce_info->last_test_bb = test_bb;
5840 :
5841 : /* We only ever should get here after reload,
5842 : and if we have conditional execution. */
5843 0 : gcc_assert (reload_completed && targetm.have_conditional_execution ());
5844 :
5845 : /* Discover if any fall through predecessors of the current test basic block
5846 : were && tests (which jump to the else block) or || tests (which jump to
5847 : the then block). */
5848 0 : if (single_pred_p (test_bb)
5849 0 : && single_pred_edge (test_bb)->flags == EDGE_FALLTHRU)
5850 : {
5851 0 : basic_block bb = single_pred (test_bb);
5852 0 : basic_block target_bb;
5853 0 : int max_insns = MAX_CONDITIONAL_EXECUTE;
5854 0 : int n_insns;
5855 :
5856 : /* Determine if the preceding block is an && or || block. */
5857 0 : if ((n_insns = block_jumps_and_fallthru (bb, else_bb)) >= 0)
5858 : {
5859 0 : ce_info->and_and_p = true;
5860 0 : target_bb = else_bb;
5861 : }
5862 0 : else if ((n_insns = block_jumps_and_fallthru (bb, then_bb)) >= 0)
5863 : {
5864 0 : ce_info->and_and_p = false;
5865 0 : target_bb = then_bb;
5866 : }
5867 : else
5868 : target_bb = NULL_BLOCK;
5869 :
5870 0 : if (target_bb && n_insns <= max_insns)
5871 : {
5872 : int total_insns = 0;
5873 : int blocks = 0;
5874 :
5875 : /* Found at least one && or || block, look for more. */
5876 0 : do
5877 : {
5878 0 : ce_info->test_bb = test_bb = bb;
5879 0 : total_insns += n_insns;
5880 0 : blocks++;
5881 :
5882 0 : if (!single_pred_p (bb))
5883 : break;
5884 :
5885 0 : bb = single_pred (bb);
5886 0 : n_insns = block_jumps_and_fallthru (bb, target_bb);
5887 : }
5888 0 : while (n_insns >= 0 && (total_insns + n_insns) <= max_insns);
5889 :
5890 0 : ce_info->num_multiple_test_blocks = blocks;
5891 0 : ce_info->num_multiple_test_insns = total_insns;
5892 :
5893 0 : if (ce_info->and_and_p)
5894 0 : ce_info->num_and_and_blocks = blocks;
5895 : else
5896 0 : ce_info->num_or_or_blocks = blocks;
5897 : }
5898 : }
5899 :
5900 : /* The THEN block of an IF-THEN combo must have exactly one predecessor,
5901 : other than any || blocks which jump to the THEN block. */
5902 0 : if ((EDGE_COUNT (then_bb->preds) - ce_info->num_or_or_blocks) != 1)
5903 : return false;
5904 :
5905 : /* The edges of the THEN and ELSE blocks cannot have complex edges. */
5906 0 : FOR_EACH_EDGE (cur_edge, ei, then_bb->preds)
5907 : {
5908 0 : if (cur_edge->flags & EDGE_COMPLEX)
5909 : return false;
5910 : }
5911 :
5912 0 : FOR_EACH_EDGE (cur_edge, ei, else_bb->preds)
5913 : {
5914 0 : if (cur_edge->flags & EDGE_COMPLEX)
5915 : return false;
5916 : }
5917 :
5918 : /* The THEN block of an IF-THEN combo must have zero or one successors. */
5919 0 : if (EDGE_COUNT (then_bb->succs) > 0
5920 0 : && (!single_succ_p (then_bb)
5921 0 : || (single_succ_edge (then_bb)->flags & EDGE_COMPLEX)
5922 0 : || (epilogue_completed
5923 0 : && tablejump_p (BB_END (then_bb), NULL, NULL))))
5924 : return false;
5925 :
5926 : /* If the THEN block has no successors, conditional execution can still
5927 : make a conditional call. Don't do this unless the ELSE block has
5928 : only one incoming edge -- the CFG manipulation is too ugly otherwise.
5929 : Check for the last insn of the THEN block being an indirect jump, which
5930 : is listed as not having any successors, but confuses the rest of the CE
5931 : code processing. ??? we should fix this in the future. */
5932 0 : if (EDGE_COUNT (then_bb->succs) == 0)
5933 : {
5934 0 : if (single_pred_p (else_bb) && else_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
5935 : {
5936 0 : rtx_insn *last_insn = BB_END (then_bb);
5937 :
5938 0 : while (last_insn
5939 0 : && NOTE_P (last_insn)
5940 0 : && last_insn != BB_HEAD (then_bb))
5941 0 : last_insn = PREV_INSN (last_insn);
5942 :
5943 0 : if (last_insn
5944 0 : && JUMP_P (last_insn)
5945 0 : && ! simplejump_p (last_insn))
5946 : return false;
5947 :
5948 : join_bb = else_bb;
5949 : else_bb = NULL_BLOCK;
5950 : }
5951 : else
5952 : return false;
5953 : }
5954 :
5955 : /* If the THEN block's successor is the other edge out of the TEST block,
5956 : then we have an IF-THEN combo without an ELSE. */
5957 0 : else if (single_succ (then_bb) == else_bb)
5958 : {
5959 : join_bb = else_bb;
5960 : else_bb = NULL_BLOCK;
5961 : }
5962 :
5963 : /* If the THEN and ELSE block meet in a subsequent block, and the ELSE
5964 : has exactly one predecessor and one successor, and the outgoing edge
5965 : is not complex, then we have an IF-THEN-ELSE combo. */
5966 0 : else if (single_succ_p (else_bb)
5967 0 : && single_succ (then_bb) == single_succ (else_bb)
5968 0 : && single_pred_p (else_bb)
5969 0 : && !(single_succ_edge (else_bb)->flags & EDGE_COMPLEX)
5970 0 : && !(epilogue_completed
5971 0 : && tablejump_p (BB_END (else_bb), NULL, NULL)))
5972 0 : join_bb = single_succ (else_bb);
5973 :
5974 : /* Otherwise it is not an IF-THEN or IF-THEN-ELSE combination. */
5975 : else
5976 : return false;
5977 :
5978 0 : num_possible_if_blocks++;
5979 :
5980 0 : if (dump_file)
5981 : {
5982 0 : fprintf (dump_file,
5983 : "\nIF-THEN%s block found, pass %d, start block %d "
5984 : "[insn %d], then %d [%d]",
5985 : (else_bb) ? "-ELSE" : "",
5986 : ce_info->pass,
5987 : test_bb->index,
5988 0 : BB_HEAD (test_bb) ? (int)INSN_UID (BB_HEAD (test_bb)) : -1,
5989 : then_bb->index,
5990 0 : BB_HEAD (then_bb) ? (int)INSN_UID (BB_HEAD (then_bb)) : -1);
5991 :
5992 0 : if (else_bb)
5993 0 : fprintf (dump_file, ", else %d [%d]",
5994 : else_bb->index,
5995 0 : BB_HEAD (else_bb) ? (int)INSN_UID (BB_HEAD (else_bb)) : -1);
5996 :
5997 0 : fprintf (dump_file, ", join %d [%d]",
5998 : join_bb->index,
5999 0 : BB_HEAD (join_bb) ? (int)INSN_UID (BB_HEAD (join_bb)) : -1);
6000 :
6001 0 : if (ce_info->num_multiple_test_blocks > 0)
6002 0 : fprintf (dump_file, ", %d %s block%s last test %d [%d]",
6003 : ce_info->num_multiple_test_blocks,
6004 0 : (ce_info->and_and_p) ? "&&" : "||",
6005 : (ce_info->num_multiple_test_blocks == 1) ? "" : "s",
6006 : ce_info->last_test_bb->index,
6007 0 : ((BB_HEAD (ce_info->last_test_bb))
6008 0 : ? (int)INSN_UID (BB_HEAD (ce_info->last_test_bb))
6009 : : -1));
6010 :
6011 0 : fputc ('\n', dump_file);
6012 : }
6013 :
6014 : /* Make sure IF, THEN, and ELSE, blocks are adjacent. Actually, we get the
6015 : first condition for free, since we've already asserted that there's a
6016 : fallthru edge from IF to THEN. Likewise for the && and || blocks, since
6017 : we checked the FALLTHRU flag, those are already adjacent to the last IF
6018 : block. */
6019 : /* ??? As an enhancement, move the ELSE block. Have to deal with
6020 : BLOCK notes, if by no other means than backing out the merge if they
6021 : exist. Sticky enough I don't want to think about it now. */
6022 0 : next = then_bb;
6023 0 : if (else_bb && (next = next->next_bb) != else_bb)
6024 : return false;
6025 0 : if ((next = next->next_bb) != join_bb
6026 0 : && join_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
6027 : {
6028 0 : if (else_bb)
6029 : join_bb = NULL;
6030 : else
6031 : return false;
6032 : }
6033 :
6034 : /* Do the real work. */
6035 :
6036 0 : ce_info->else_bb = else_bb;
6037 0 : ce_info->join_bb = join_bb;
6038 :
6039 : /* If we have && and || tests, try to first handle combining the && and ||
6040 : tests into the conditional code, and if that fails, go back and handle
6041 : it without the && and ||, which at present handles the && case if there
6042 : was no ELSE block. */
6043 0 : if (cond_exec_process_if_block (ce_info, true))
6044 : return true;
6045 :
6046 0 : if (ce_info->num_multiple_test_blocks)
6047 : {
6048 0 : cancel_changes (0);
6049 :
6050 0 : if (cond_exec_process_if_block (ce_info, false))
6051 : return true;
6052 : }
6053 :
6054 : return false;
6055 : }
6056 :
6057 : /* Convert a branch over a trap, or a branch
6058 : to a trap, into a conditional trap. */
6059 :
6060 : static bool
6061 0 : find_cond_trap (basic_block test_bb, edge then_edge, edge else_edge)
6062 : {
6063 0 : basic_block then_bb = then_edge->dest;
6064 0 : basic_block else_bb = else_edge->dest;
6065 0 : basic_block other_bb, trap_bb;
6066 0 : rtx_insn *trap, *jump;
6067 0 : rtx cond;
6068 0 : rtx_insn *cond_earliest;
6069 :
6070 : /* Locate the block with the trap instruction. */
6071 : /* ??? While we look for no successors, we really ought to allow
6072 : EH successors. Need to fix merge_if_block for that to work. */
6073 0 : if ((trap = block_has_only_trap (then_bb)) != NULL)
6074 : trap_bb = then_bb, other_bb = else_bb;
6075 0 : else if ((trap = block_has_only_trap (else_bb)) != NULL)
6076 : trap_bb = else_bb, other_bb = then_bb;
6077 : else
6078 : return false;
6079 :
6080 0 : if (dump_file)
6081 : {
6082 0 : fprintf (dump_file, "\nTRAP-IF block found, start %d, trap %d\n",
6083 : test_bb->index, trap_bb->index);
6084 : }
6085 :
6086 : /* If this is not a standard conditional jump, we can't parse it. */
6087 0 : jump = BB_END (test_bb);
6088 0 : cond = noce_get_condition (jump, &cond_earliest, then_bb == trap_bb);
6089 0 : if (! cond)
6090 : return false;
6091 :
6092 : /* If the conditional jump is more than just a conditional jump, then
6093 : we cannot do if-conversion on this block. Give up for returnjump_p,
6094 : changing a conditional return followed by unconditional trap for
6095 : conditional trap followed by unconditional return is likely not
6096 : beneficial and harder to handle. */
6097 0 : if (! onlyjump_p (jump) || returnjump_p (jump))
6098 : return false;
6099 :
6100 : /* We must be comparing objects whose modes imply the size. */
6101 0 : if (GET_MODE (XEXP (cond, 0)) == BLKmode)
6102 : return false;
6103 :
6104 : /* Attempt to generate the conditional trap. */
6105 0 : rtx_insn *seq = gen_cond_trap (GET_CODE (cond), copy_rtx (XEXP (cond, 0)),
6106 : copy_rtx (XEXP (cond, 1)),
6107 0 : TRAP_CODE (PATTERN (trap)));
6108 0 : if (seq == NULL)
6109 : return false;
6110 :
6111 : /* If that results in an invalid insn, back out. */
6112 0 : for (rtx_insn *x = seq; x; x = NEXT_INSN (x))
6113 0 : if (reload_completed
6114 0 : ? !valid_insn_p (x)
6115 0 : : recog_memoized (x) < 0)
6116 : return false;
6117 :
6118 0 : if (noce_clobbers_live_cc_p (test_bb, seq))
6119 : return false;
6120 :
6121 : /* Emit the new insns before cond_earliest. */
6122 0 : emit_insn_before_setloc (seq, cond_earliest, INSN_LOCATION (trap));
6123 :
6124 : /* Delete the trap block if possible. */
6125 0 : remove_edge (trap_bb == then_bb ? then_edge : else_edge);
6126 0 : df_set_bb_dirty (test_bb);
6127 0 : df_set_bb_dirty (then_bb);
6128 0 : df_set_bb_dirty (else_bb);
6129 :
6130 0 : if (EDGE_COUNT (trap_bb->preds) == 0)
6131 : {
6132 0 : delete_basic_block (trap_bb);
6133 0 : num_true_changes++;
6134 : }
6135 :
6136 : /* Wire together the blocks again. */
6137 0 : if (current_ir_type () == IR_RTL_CFGLAYOUT)
6138 0 : single_succ_edge (test_bb)->flags |= EDGE_FALLTHRU;
6139 0 : else if (trap_bb == then_bb)
6140 : {
6141 0 : rtx lab = JUMP_LABEL (jump);
6142 0 : rtx_insn *seq = targetm.gen_jump (lab);
6143 0 : rtx_jump_insn *newjump = emit_jump_insn_after (seq, jump);
6144 0 : LABEL_NUSES (lab) += 1;
6145 0 : JUMP_LABEL (newjump) = lab;
6146 0 : emit_barrier_after (newjump);
6147 : }
6148 0 : delete_insn (jump);
6149 :
6150 0 : if (can_merge_blocks_p (test_bb, other_bb))
6151 : {
6152 0 : merge_blocks (test_bb, other_bb);
6153 0 : num_true_changes++;
6154 : }
6155 :
6156 0 : num_updated_if_blocks++;
6157 0 : return true;
6158 : }
6159 :
6160 : /* Subroutine of find_cond_trap: if BB contains only a trap insn,
6161 : return it. */
6162 :
6163 : static rtx_insn *
6164 0 : block_has_only_trap (basic_block bb)
6165 : {
6166 0 : rtx_insn *trap;
6167 :
6168 : /* We're not the exit block. */
6169 0 : if (bb == EXIT_BLOCK_PTR_FOR_FN (cfun))
6170 : return NULL;
6171 :
6172 : /* The block must have no successors. */
6173 0 : if (EDGE_COUNT (bb->succs) > 0)
6174 : return NULL;
6175 :
6176 : /* The only instruction in the THEN block must be the trap. */
6177 0 : trap = first_active_insn (bb);
6178 0 : if (! (trap == BB_END (bb)
6179 0 : && GET_CODE (PATTERN (trap)) == TRAP_IF
6180 0 : && TRAP_CONDITION (PATTERN (trap)) == const_true_rtx))
6181 0 : return NULL;
6182 :
6183 : return trap;
6184 : }
6185 :
6186 : /* Look for IF-THEN-ELSE cases in which one of THEN or ELSE is
6187 : transformable, but not necessarily the other. There need be no
6188 : JOIN block.
6189 :
6190 : Return TRUE if we were successful at converting the block.
6191 :
6192 : Cases we'd like to look at:
6193 :
6194 : (1)
6195 : if (test) goto over; // x not live
6196 : x = a;
6197 : goto label;
6198 : over:
6199 :
6200 : becomes
6201 :
6202 : x = a;
6203 : if (! test) goto label;
6204 :
6205 : (2)
6206 : if (test) goto E; // x not live
6207 : x = big();
6208 : goto L;
6209 : E:
6210 : x = b;
6211 : goto M;
6212 :
6213 : becomes
6214 :
6215 : x = b;
6216 : if (test) goto M;
6217 : x = big();
6218 : goto L;
6219 :
6220 : (3) // This one's really only interesting for targets that can do
6221 : // multiway branching, e.g. IA-64 BBB bundles. For other targets
6222 : // it results in multiple branches on a cache line, which often
6223 : // does not sit well with predictors.
6224 :
6225 : if (test1) goto E; // predicted not taken
6226 : x = a;
6227 : if (test2) goto F;
6228 : ...
6229 : E:
6230 : x = b;
6231 : J:
6232 :
6233 : becomes
6234 :
6235 : x = a;
6236 : if (test1) goto E;
6237 : if (test2) goto F;
6238 :
6239 : Notes:
6240 :
6241 : (A) Don't do (2) if the branch is predicted against the block we're
6242 : eliminating. Do it anyway if we can eliminate a branch; this requires
6243 : that the sole successor of the eliminated block postdominate the other
6244 : side of the if.
6245 :
6246 : (B) With CE, on (3) we can steal from both sides of the if, creating
6247 :
6248 : if (test1) x = a;
6249 : if (!test1) x = b;
6250 : if (test1) goto J;
6251 : if (test2) goto F;
6252 : ...
6253 : J:
6254 :
6255 : Again, this is most useful if J postdominates.
6256 :
6257 : (C) CE substitutes for helpful life information.
6258 :
6259 : (D) These heuristics need a lot of work. */
6260 :
6261 : /* Return true if the if-case formed by TEST_BB, THEN_BB and ELSE_BB is one we
6262 : may convert at all.
6263 :
6264 : If we are partitioning hot/cold basic blocks, we don't want to mess up
6265 : unconditional or indirect jumps that cross between hot and cold sections.
6266 : Basic block partitioning may result in some jumps that appear to be
6267 : optimizable (or blocks that appear to be mergeable), but which really must
6268 : be left untouched (they are required to make it safely across partition
6269 : boundaries). See the comments at the top of
6270 : bb-reorder.cc:partition_hot_cold_basic_blocks for complete details.
6271 :
6272 : TEST_BB must also end in a conditional jump with no other side-effects. */
6273 :
6274 : static bool
6275 28852061 : if_case_blocks_ok_p (basic_block test_bb, basic_block then_bb,
6276 : basic_block else_bb)
6277 : {
6278 28852061 : if ((BB_END (then_bb)
6279 28852061 : && JUMP_P (BB_END (then_bb))
6280 15033960 : && CROSSING_JUMP_P (BB_END (then_bb)))
6281 28373807 : || (JUMP_P (BB_END (test_bb))
6282 28373807 : && CROSSING_JUMP_P (BB_END (test_bb)))
6283 57009658 : || (BB_END (else_bb)
6284 28157597 : && JUMP_P (BB_END (else_bb))
6285 14185121 : && CROSSING_JUMP_P (BB_END (else_bb))))
6286 : return false;
6287 :
6288 28110665 : return onlyjump_p (BB_END (test_bb));
6289 : }
6290 :
6291 : /* Tests for case 1 above. */
6292 :
6293 : static bool
6294 14513587 : find_if_case_1 (basic_block test_bb, edge then_edge, edge else_edge)
6295 : {
6296 14513587 : basic_block then_bb = then_edge->dest;
6297 14513587 : basic_block else_bb = else_edge->dest;
6298 14513587 : basic_block new_bb;
6299 14513587 : int then_bb_index;
6300 14513587 : profile_probability then_prob;
6301 14513587 : rtx else_target = NULL_RTX;
6302 :
6303 14513587 : if (!if_case_blocks_ok_p (test_bb, then_bb, else_bb))
6304 : return false;
6305 :
6306 : /* THEN has one successor. */
6307 14142217 : if (!single_succ_p (then_bb))
6308 : return false;
6309 :
6310 : /* THEN does not fall through, but is not strange either. */
6311 6411603 : if (single_succ_edge (then_bb)->flags & (EDGE_COMPLEX | EDGE_FALLTHRU))
6312 : return false;
6313 :
6314 : /* THEN has one predecessor. */
6315 1256164 : if (!single_pred_p (then_bb))
6316 : return false;
6317 :
6318 : /* THEN must do something. */
6319 1131203 : if (forwarder_block_p (then_bb))
6320 : return false;
6321 :
6322 719068 : num_possible_if_blocks++;
6323 719068 : if (dump_file)
6324 8 : fprintf (dump_file,
6325 : "\nIF-CASE-1 found, start %d, then %d\n",
6326 : test_bb->index, then_bb->index);
6327 :
6328 719068 : then_prob = then_edge->probability.invert ();
6329 :
6330 : /* We're speculating from the THEN path, we want to make sure the cost
6331 : of speculation is within reason. */
6332 1359918 : if (! cheap_bb_rtx_cost_p (then_bb, then_prob,
6333 1359918 : COSTS_N_INSNS (BRANCH_COST (optimize_bb_for_speed_p (then_edge->src),
6334 : predictable_edge_p (then_edge)))))
6335 : return false;
6336 :
6337 139891 : if (else_bb == EXIT_BLOCK_PTR_FOR_FN (cfun))
6338 : {
6339 0 : rtx_insn *jump = BB_END (else_edge->src);
6340 0 : gcc_assert (JUMP_P (jump));
6341 0 : else_target = JUMP_LABEL (jump);
6342 : }
6343 :
6344 : /* Registers set are dead, or are predicable. */
6345 139891 : if (! dead_or_predicable (test_bb, then_bb, else_bb,
6346 : single_succ_edge (then_bb), true))
6347 : return false;
6348 :
6349 : /* Conversion went ok, including moving the insns and fixing up the
6350 : jump. Adjust the CFG to match. */
6351 :
6352 : /* We can avoid creating a new basic block if then_bb is immediately
6353 : followed by else_bb, i.e. deleting then_bb allows test_bb to fall
6354 : through to else_bb. */
6355 :
6356 20984 : if (then_bb->next_bb == else_bb
6357 9520 : && then_bb->prev_bb == test_bb
6358 9520 : && else_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
6359 : {
6360 9520 : redirect_edge_succ (FALLTHRU_EDGE (test_bb), else_bb);
6361 9520 : new_bb = 0;
6362 : }
6363 11464 : else if (else_bb == EXIT_BLOCK_PTR_FOR_FN (cfun))
6364 0 : new_bb = force_nonfallthru_and_redirect (FALLTHRU_EDGE (test_bb),
6365 : else_bb, else_target);
6366 : else
6367 11464 : new_bb = redirect_edge_and_branch_force (FALLTHRU_EDGE (test_bb),
6368 : else_bb);
6369 :
6370 20984 : df_set_bb_dirty (test_bb);
6371 20984 : df_set_bb_dirty (else_bb);
6372 :
6373 20984 : then_bb_index = then_bb->index;
6374 20984 : delete_basic_block (then_bb);
6375 :
6376 : /* Make rest of code believe that the newly created block is the THEN_BB
6377 : block we removed. */
6378 20984 : if (new_bb)
6379 : {
6380 11464 : df_bb_replace (then_bb_index, new_bb);
6381 : /* This should have been done above via force_nonfallthru_and_redirect
6382 : (possibly called from redirect_edge_and_branch_force). */
6383 11464 : gcc_checking_assert (BB_PARTITION (new_bb) == BB_PARTITION (test_bb));
6384 : }
6385 :
6386 20984 : num_true_changes++;
6387 20984 : num_updated_if_blocks++;
6388 20984 : return true;
6389 : }
6390 :
6391 : /* Test for case 2 above. */
6392 :
6393 : static bool
6394 14492603 : find_if_case_2 (basic_block test_bb, edge then_edge, edge else_edge)
6395 : {
6396 14492603 : basic_block then_bb = then_edge->dest;
6397 14492603 : basic_block else_bb = else_edge->dest;
6398 14492603 : edge else_succ;
6399 14492603 : profile_probability then_prob, else_prob;
6400 :
6401 : /* We do not want to speculate (empty) loop latches. */
6402 14492603 : if (current_loops
6403 5674258 : && else_bb->loop_father->latch == else_bb)
6404 : return false;
6405 :
6406 14338474 : if (!if_case_blocks_ok_p (test_bb, then_bb, else_bb))
6407 : return false;
6408 :
6409 : /* ELSE has one successor. */
6410 13967105 : if (!single_succ_p (else_bb))
6411 : return false;
6412 : else
6413 5881963 : else_succ = single_succ_edge (else_bb);
6414 :
6415 : /* ELSE outgoing edge is not complex. */
6416 5881963 : if (else_succ->flags & EDGE_COMPLEX)
6417 : return false;
6418 :
6419 : /* ELSE has one predecessor. */
6420 5721015 : if (!single_pred_p (else_bb))
6421 : return false;
6422 :
6423 : /* THEN is not EXIT. */
6424 2998153 : if (then_bb->index < NUM_FIXED_BLOCKS)
6425 : return false;
6426 :
6427 2998153 : else_prob = else_edge->probability;
6428 2998153 : then_prob = else_prob.invert ();
6429 :
6430 : /* ELSE is predicted or SUCC(ELSE) postdominates THEN. */
6431 2998153 : if (else_prob > then_prob)
6432 : ;
6433 2018667 : else if (else_succ->dest->index < NUM_FIXED_BLOCKS
6434 2018667 : || dominated_by_p (CDI_POST_DOMINATORS, then_bb,
6435 : else_succ->dest))
6436 : ;
6437 : else
6438 : return false;
6439 :
6440 2374508 : num_possible_if_blocks++;
6441 2374508 : if (dump_file)
6442 44 : fprintf (dump_file,
6443 : "\nIF-CASE-2 found, start %d, else %d\n",
6444 : test_bb->index, else_bb->index);
6445 :
6446 : /* We're speculating from the ELSE path, we want to make sure the cost
6447 : of speculation is within reason. */
6448 4576858 : if (! cheap_bb_rtx_cost_p (else_bb, else_prob,
6449 4576858 : COSTS_N_INSNS (BRANCH_COST (optimize_bb_for_speed_p (else_edge->src),
6450 : predictable_edge_p (else_edge)))))
6451 : return false;
6452 :
6453 : /* Registers set are dead, or are predicable. */
6454 452928 : if (! dead_or_predicable (test_bb, else_bb, then_bb, else_succ, false))
6455 : return false;
6456 :
6457 : /* Conversion went ok, including moving the insns and fixing up the
6458 : jump. Adjust the CFG to match. */
6459 :
6460 126960 : df_set_bb_dirty (test_bb);
6461 126960 : df_set_bb_dirty (then_bb);
6462 126960 : delete_basic_block (else_bb);
6463 :
6464 126960 : num_true_changes++;
6465 126960 : num_updated_if_blocks++;
6466 :
6467 : /* ??? We may now fallthru from one of THEN's successors into a join
6468 : block. Rerun cleanup_cfg? Examine things manually? Wait? */
6469 :
6470 126960 : return true;
6471 : }
6472 :
6473 : /* Used by the code above to perform the actual rtl transformations.
6474 : Return TRUE if successful.
6475 :
6476 : TEST_BB is the block containing the conditional branch. MERGE_BB is the
6477 : block containing the code to manipulate. OTHER_BB is the other successor
6478 : of TEST_BB, the one the code is being moved past. DEST_EDGE is an edge
6479 : representing a jump to the join block; after the conversion, TEST_BB should
6480 : be branching to its destination. REVERSEP is true if the sense of the
6481 : branch should be reversed. */
6482 :
6483 : static bool
6484 592819 : dead_or_predicable (basic_block test_bb, basic_block merge_bb,
6485 : basic_block other_bb, edge dest_edge, bool reversep)
6486 : {
6487 592819 : basic_block new_dest = dest_edge->dest;
6488 592819 : rtx_insn *head, *end, *jump;
6489 592819 : rtx_insn *earliest = NULL;
6490 592819 : rtx old_dest;
6491 592819 : auto_bitmap merge_set (®_obstack);
6492 : /* Number of pending changes. */
6493 592819 : int n_validated_changes = 0;
6494 592819 : rtx new_dest_label = NULL_RTX;
6495 :
6496 592819 : jump = BB_END (test_bb);
6497 :
6498 : /* Find the extent of the real code in the merge block. */
6499 592819 : head = BB_HEAD (merge_bb);
6500 592819 : end = BB_END (merge_bb);
6501 :
6502 774857 : while (DEBUG_INSN_P (end) && end != head)
6503 182038 : end = PREV_INSN (end);
6504 :
6505 : /* If merge_bb ends with a tablejump, predicating/moving insn's
6506 : into test_bb and then deleting merge_bb will result in the jumptable
6507 : that follows merge_bb being removed along with merge_bb and then we
6508 : get an unresolved reference to the jumptable. */
6509 592819 : if (tablejump_p (end, NULL, NULL))
6510 : return false;
6511 :
6512 592819 : if (LABEL_P (head))
6513 452996 : head = NEXT_INSN (head);
6514 592819 : while (DEBUG_INSN_P (head) && head != end)
6515 0 : head = NEXT_INSN (head);
6516 592819 : if (NOTE_P (head))
6517 : {
6518 592819 : if (head == end)
6519 : {
6520 104300 : head = end = NULL;
6521 104300 : goto no_body;
6522 : }
6523 488519 : head = NEXT_INSN (head);
6524 1425020 : while (DEBUG_INSN_P (head) && head != end)
6525 447982 : head = NEXT_INSN (head);
6526 : }
6527 :
6528 488519 : if (JUMP_P (end))
6529 : {
6530 202532 : if (!onlyjump_p (end))
6531 : return false;
6532 161220 : if (head == end)
6533 : {
6534 6 : head = end = NULL;
6535 6 : goto no_body;
6536 : }
6537 161214 : end = PREV_INSN (end);
6538 435291 : while (DEBUG_INSN_P (end) && end != head)
6539 112863 : end = PREV_INSN (end);
6540 : }
6541 :
6542 : /* Don't move frame-related insn across the conditional branch. This
6543 : can lead to one of the paths of the branch having wrong unwind info. */
6544 447201 : if (epilogue_completed)
6545 : {
6546 : rtx_insn *insn = head;
6547 87063 : while (1)
6548 : {
6549 288899 : if (INSN_P (insn) && RTX_FRAME_RELATED_P (insn))
6550 : return false;
6551 282139 : if (insn == end)
6552 : break;
6553 87063 : insn = NEXT_INSN (insn);
6554 87063 : }
6555 : }
6556 :
6557 : /* Disable handling dead code by conditional execution if the machine needs
6558 : to do anything funny with the tests, etc. */
6559 : #ifndef IFCVT_MODIFY_TESTS
6560 440441 : if (targetm.have_conditional_execution ())
6561 : {
6562 : /* In the conditional execution case, we have things easy. We know
6563 : the condition is reversible. We don't have to check life info
6564 : because we're going to conditionally execute the code anyway.
6565 : All that's left is making sure the insns involved can actually
6566 : be predicated. */
6567 :
6568 0 : rtx cond;
6569 :
6570 : /* If the conditional jump is more than just a conditional jump,
6571 : then we cannot do conditional execution conversion on this block. */
6572 0 : if (!onlyjump_p (jump))
6573 0 : goto nce;
6574 :
6575 0 : cond = cond_exec_get_condition (jump);
6576 0 : if (! cond)
6577 0 : goto nce;
6578 :
6579 0 : rtx note = find_reg_note (jump, REG_BR_PROB, NULL_RTX);
6580 0 : profile_probability prob_val
6581 0 : = (note ? profile_probability::from_reg_br_prob_note (XINT (note, 0))
6582 : : profile_probability::uninitialized ());
6583 :
6584 0 : if (reversep)
6585 : {
6586 0 : enum rtx_code rev = reversed_comparison_code (cond, jump);
6587 0 : if (rev == UNKNOWN)
6588 0 : return false;
6589 0 : cond = gen_rtx_fmt_ee (rev, GET_MODE (cond), XEXP (cond, 0),
6590 : XEXP (cond, 1));
6591 0 : prob_val = prob_val.invert ();
6592 : }
6593 :
6594 0 : if (cond_exec_process_insns (NULL, head, end, cond, prob_val, false)
6595 0 : && verify_changes (0))
6596 0 : n_validated_changes = num_validated_changes ();
6597 : else
6598 0 : cancel_changes (0);
6599 :
6600 0 : earliest = jump;
6601 : }
6602 440441 : nce:
6603 : #endif
6604 :
6605 : /* If we allocated new pseudos (e.g. in the conditional move
6606 : expander called from noce_emit_cmove), we must resize the
6607 : array first. */
6608 440441 : if (max_regno < max_reg_num ())
6609 84993 : max_regno = max_reg_num ();
6610 :
6611 : /* Try the NCE path if the CE path did not result in any changes. */
6612 440441 : if (n_validated_changes == 0)
6613 : {
6614 440441 : rtx cond;
6615 440441 : rtx_insn *insn;
6616 440441 : bool success;
6617 :
6618 : /* In the non-conditional execution case, we have to verify that there
6619 : are no trapping operations, no calls, no references to memory, and
6620 : that any registers modified are dead at the branch site. */
6621 :
6622 440441 : if (!any_condjump_p (jump))
6623 295613 : return false;
6624 :
6625 : /* Find the extent of the conditional. */
6626 440441 : cond = noce_get_condition (jump, &earliest, false);
6627 440441 : if (!cond)
6628 : return false;
6629 :
6630 439631 : auto_bitmap live (®_obstack);
6631 439631 : simulate_backwards_to_point (merge_bb, live, end);
6632 439631 : success = can_move_insns_across (head, end, earliest, jump,
6633 : merge_bb, live,
6634 : df_get_live_in (other_bb), NULL);
6635 439631 : if (!success)
6636 : return false;
6637 :
6638 : /* Collect the set of registers set in MERGE_BB. */
6639 728542 : FOR_BB_INSNS (merge_bb, insn)
6640 583176 : if (NONDEBUG_INSN_P (insn))
6641 188741 : df_simulate_find_defs (insn, merge_set);
6642 :
6643 : /* If shrink-wrapping, disable this optimization when test_bb is
6644 : the first basic block and merge_bb exits. The idea is to not
6645 : move code setting up a return register as that may clobber a
6646 : register used to pass function parameters, which then must be
6647 : saved in caller-saved regs. A caller-saved reg requires the
6648 : prologue, killing a shrink-wrap opportunity. */
6649 145366 : if ((SHRINK_WRAPPING_ENABLED && !epilogue_completed)
6650 115203 : && ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb == test_bb
6651 158894 : && single_succ_p (new_dest)
6652 14066 : && single_succ (new_dest) == EXIT_BLOCK_PTR_FOR_FN (cfun)
6653 158423 : && bitmap_intersect_p (df_get_live_in (new_dest), merge_set))
6654 : {
6655 13057 : unsigned int i;
6656 :
6657 13057 : auto_bitmap return_regs (®_obstack);
6658 :
6659 : /* Start off with the intersection of regs used to pass
6660 : params and regs used to return values. */
6661 1240415 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
6662 1227358 : if (FUNCTION_ARG_REGNO_P (i)
6663 1227358 : && targetm.calls.function_value_regno_p (i))
6664 51055 : bitmap_set_bit (return_regs, INCOMING_REGNO (i));
6665 :
6666 13057 : bitmap_and_into (return_regs,
6667 13057 : df_get_live_out (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
6668 13057 : bitmap_and_into (return_regs,
6669 13057 : df_get_live_in (EXIT_BLOCK_PTR_FOR_FN (cfun)));
6670 13057 : if (!bitmap_empty_p (return_regs))
6671 : {
6672 4923 : FOR_BB_INSNS_REVERSE (new_dest, insn)
6673 4365 : if (NONDEBUG_INSN_P (insn))
6674 : {
6675 2221 : df_ref def;
6676 :
6677 : /* If this insn sets any reg in return_regs, add all
6678 : reg uses to the set of regs we're interested in. */
6679 2810 : FOR_EACH_INSN_DEF (def, insn)
6680 1709 : if (bitmap_bit_p (return_regs, DF_REF_REGNO (def)))
6681 : {
6682 1120 : df_simulate_uses (insn, return_regs);
6683 1120 : break;
6684 : }
6685 : }
6686 558 : if (bitmap_intersect_p (merge_set, return_regs))
6687 538 : return false;
6688 : }
6689 13057 : }
6690 439631 : }
6691 :
6692 0 : no_body:
6693 : /* We don't want to use normal invert_jump or redirect_jump because
6694 : we don't want to delete_insn called. Also, we want to do our own
6695 : change group management. */
6696 :
6697 249134 : old_dest = JUMP_LABEL (jump);
6698 249134 : if (other_bb != new_dest)
6699 : {
6700 249119 : if (!any_condjump_p (jump))
6701 0 : goto cancel;
6702 :
6703 249119 : if (JUMP_P (BB_END (dest_edge->src)))
6704 22061 : new_dest_label = JUMP_LABEL (BB_END (dest_edge->src));
6705 227058 : else if (new_dest == EXIT_BLOCK_PTR_FOR_FN (cfun))
6706 101190 : new_dest_label = ret_rtx;
6707 : else
6708 125868 : new_dest_label = block_label (new_dest);
6709 :
6710 249119 : rtx_jump_insn *jump_insn = as_a <rtx_jump_insn *> (jump);
6711 249119 : if (reversep
6712 249119 : ? ! invert_jump_1 (jump_insn, new_dest_label)
6713 228135 : : ! redirect_jump_1 (jump_insn, new_dest_label))
6714 0 : goto cancel;
6715 : }
6716 :
6717 249134 : if (verify_changes (n_validated_changes))
6718 147944 : confirm_change_group ();
6719 : else
6720 101190 : goto cancel;
6721 :
6722 147944 : if (other_bb != new_dest)
6723 : {
6724 147929 : redirect_jump_2 (as_a <rtx_jump_insn *> (jump), old_dest, new_dest_label,
6725 : 0, reversep);
6726 :
6727 147929 : redirect_edge_succ (BRANCH_EDGE (test_bb), new_dest);
6728 147929 : if (reversep)
6729 : {
6730 20984 : std::swap (BRANCH_EDGE (test_bb)->probability,
6731 20984 : FALLTHRU_EDGE (test_bb)->probability);
6732 20984 : update_br_prob_note (test_bb);
6733 : }
6734 : }
6735 :
6736 : /* Move the insns out of MERGE_BB to before the branch. */
6737 147944 : if (head != NULL)
6738 : {
6739 144828 : rtx_insn *insn;
6740 :
6741 144828 : if (end == BB_END (merge_bb))
6742 111539 : BB_END (merge_bb) = PREV_INSN (head);
6743 :
6744 : /* PR 21767: when moving insns above a conditional branch, the REG_EQUAL
6745 : notes being moved might become invalid. */
6746 144828 : insn = head;
6747 173672 : do
6748 : {
6749 173672 : rtx note;
6750 :
6751 173672 : if (! INSN_P (insn))
6752 3713 : continue;
6753 169959 : note = find_reg_note (insn, REG_EQUAL, NULL_RTX);
6754 169959 : if (! note)
6755 159058 : continue;
6756 10901 : remove_note (insn, note);
6757 202516 : } while (insn != end && (insn = NEXT_INSN (insn)));
6758 :
6759 : /* PR46315: when moving insns above a conditional branch, the REG_EQUAL
6760 : notes referring to the registers being set might become invalid.
6761 : MERGE_SET is empty on the conditional-execution path, which does not
6762 : move anything. */
6763 144828 : unsigned i;
6764 144828 : bitmap_iterator bi;
6765 326237 : EXECUTE_IF_SET_IN_BITMAP (merge_set, 0, i, bi)
6766 181409 : remove_reg_equal_equiv_notes_for_regno (i);
6767 :
6768 144828 : reorder_insns (head, end, PREV_INSN (earliest));
6769 : }
6770 :
6771 : /* Remove the jump and edge if we can. */
6772 147944 : if (other_bb == new_dest)
6773 : {
6774 15 : delete_insn (jump);
6775 15 : remove_edge (BRANCH_EDGE (test_bb));
6776 : /* ??? Can't merge blocks here, as then_bb is still in use.
6777 : At minimum, the merge will get done just before bb-reorder. */
6778 : }
6779 :
6780 : return true;
6781 :
6782 101190 : cancel:
6783 101190 : cancel_changes (0);
6784 :
6785 101190 : return false;
6786 592819 : }
6787 :
6788 : /* Main entry point for all if-conversion. PHASE selects the ce1, ce2 or ce3
6789 : run: before combine, after combine, or after reload. */
6790 :
6791 : static void
6792 3203239 : if_convert (ifcvt_phase phase)
6793 : {
6794 3203239 : basic_block bb;
6795 3203239 : int pass;
6796 :
6797 3203239 : if (optimize == 1)
6798 : {
6799 237709 : df_live_add_problem ();
6800 237709 : df_live_set_all_dirty ();
6801 : }
6802 :
6803 : /* Record which pass is running for the cost model. */
6804 3203239 : ifcvt_pass_phase = phase;
6805 3203239 : have_cbranchcc4 = (direct_optab_handler (cbranch_optab, CCmode)
6806 3203239 : != CODE_FOR_nothing);
6807 3203239 : num_possible_if_blocks = 0;
6808 3203239 : num_updated_if_blocks = 0;
6809 3203239 : num_true_changes = 0;
6810 :
6811 3203239 : loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
6812 3203239 : mark_loop_exit_edges ();
6813 3203239 : loop_optimizer_finalize ();
6814 3203239 : free_dominance_info (CDI_DOMINATORS);
6815 :
6816 : /* Compute postdominators. */
6817 3203239 : calculate_dominance_info (CDI_POST_DOMINATORS);
6818 :
6819 3203239 : df_set_flags (DF_LR_RUN_DCE);
6820 :
6821 : /* Go through each of the basic blocks looking for things to convert. If we
6822 : have conditional execution, we make multiple passes to allow us to handle
6823 : IF-THEN{-ELSE} blocks within other IF-THEN{-ELSE} blocks. */
6824 3203239 : pass = 0;
6825 3371828 : do
6826 : {
6827 3371828 : df_analyze ();
6828 : /* Only need to do dce on the first pass. */
6829 3371828 : df_clear_flags (DF_LR_RUN_DCE);
6830 3371828 : ifcvt_changed_p = false;
6831 3371828 : pass++;
6832 :
6833 3371828 : if (dump_file && pass > 1)
6834 22 : fprintf (dump_file, "\n\n========== Pass %d ==========\n", pass);
6835 :
6836 44008178 : FOR_EACH_BB_FN (bb, cfun)
6837 : {
6838 : basic_block new_bb;
6839 40963386 : while (!df_get_bb_dirty (bb)
6840 81107769 : && (new_bb = find_if_header (bb, pass)) != NULL)
6841 : bb = new_bb;
6842 : }
6843 :
6844 3371828 : if (dump_file && ifcvt_changed_p)
6845 22 : print_rtl_with_bb (dump_file, get_insns (), dump_flags);
6846 : }
6847 : while (ifcvt_changed_p);
6848 :
6849 3203239 : if (dump_file)
6850 88 : fprintf (dump_file, "\n\n========== no more changes\n");
6851 :
6852 3203239 : free_dominance_info (CDI_POST_DOMINATORS);
6853 :
6854 3203239 : if (dump_file)
6855 88 : fflush (dump_file);
6856 :
6857 3203239 : clear_aux_for_blocks ();
6858 :
6859 : /* If we allocated new pseudos, we must resize the array for sched1. */
6860 3203239 : if (max_regno < max_reg_num ())
6861 655123 : max_regno = max_reg_num ();
6862 :
6863 : /* Write the final stats. */
6864 3203239 : if (dump_file && num_possible_if_blocks > 0)
6865 : {
6866 62 : fprintf (dump_file,
6867 : "\n%d possible IF blocks searched.\n",
6868 : num_possible_if_blocks);
6869 62 : fprintf (dump_file,
6870 : "%d IF blocks converted.\n",
6871 : num_updated_if_blocks);
6872 62 : fprintf (dump_file,
6873 : "%d true changes made.\n\n\n",
6874 : num_true_changes);
6875 : }
6876 :
6877 3203239 : if (optimize == 1)
6878 237709 : df_remove_problem (df_live);
6879 :
6880 : /* Some non-cold blocks may now be only reachable from cold blocks.
6881 : Fix that up. */
6882 3203239 : fixup_partitions ();
6883 :
6884 3203239 : checking_verify_flow_info ();
6885 3203239 : }
6886 :
6887 : /* If-conversion and CFG cleanup. */
6888 : static void
6889 1070192 : rest_of_handle_if_conversion (void)
6890 : {
6891 1070192 : int flags = 0;
6892 :
6893 1070192 : if (flag_if_conversion)
6894 : {
6895 1067742 : if (dump_file)
6896 : {
6897 36 : dump_reg_info (dump_file);
6898 36 : dump_flow_info (dump_file, dump_flags);
6899 : }
6900 1067742 : cleanup_cfg (CLEANUP_EXPENSIVE);
6901 1067742 : if_convert (IFCVT_BEFORE_COMBINE);
6902 1067742 : if (num_updated_if_blocks)
6903 : /* Get rid of any dead CC-related instructions. */
6904 1070192 : flags |= CLEANUP_FORCE_FAST_DCE;
6905 : }
6906 :
6907 1070192 : cleanup_cfg (flags);
6908 1070192 : }
6909 :
6910 : namespace {
6911 :
6912 : const pass_data pass_data_rtl_ifcvt =
6913 : {
6914 : RTL_PASS, /* type */
6915 : "ce1", /* name */
6916 : OPTGROUP_NONE, /* optinfo_flags */
6917 : TV_IFCVT, /* tv_id */
6918 : 0, /* properties_required */
6919 : 0, /* properties_provided */
6920 : 0, /* properties_destroyed */
6921 : 0, /* todo_flags_start */
6922 : TODO_df_finish, /* todo_flags_finish */
6923 : };
6924 :
6925 : /* ce2 reruns if-conversion after combine has simplified things enough to
6926 : meet the sequence-length restrictions. */
6927 : const pass_data pass_data_if_after_combine =
6928 : {
6929 : RTL_PASS, /* type */
6930 : "ce2", /* name */
6931 : OPTGROUP_NONE, /* optinfo_flags */
6932 : TV_IFCVT, /* tv_id */
6933 : 0, /* properties_required */
6934 : 0, /* properties_provided */
6935 : 0, /* properties_destroyed */
6936 : 0, /* todo_flags_start */
6937 : TODO_df_finish, /* todo_flags_finish */
6938 : };
6939 :
6940 : /* ce3 reruns if-conversion after reload. */
6941 : const pass_data pass_data_if_after_reload =
6942 : {
6943 : RTL_PASS, /* type */
6944 : "ce3", /* name */
6945 : OPTGROUP_NONE, /* optinfo_flags */
6946 : TV_IFCVT2, /* tv_id */
6947 : 0, /* properties_required */
6948 : 0, /* properties_provided */
6949 : 0, /* properties_destroyed */
6950 : 0, /* todo_flags_start */
6951 : TODO_df_finish, /* todo_flags_finish */
6952 : };
6953 :
6954 : /* The three if-conversion passes (ce1/ce2/ce3) share this class. M_PHASE
6955 : selects the gate condition and the work: ce1 (before combine) also cleans
6956 : up the CFG and is gated only on optimize, while ce2 and ce3 rerun once
6957 : combine and reload have simplified the RTL. */
6958 :
6959 : class pass_rtl_ifcvt : public rtl_opt_pass
6960 : {
6961 : public:
6962 883608 : pass_rtl_ifcvt (gcc::context *ctxt, const pass_data &data, ifcvt_phase phase)
6963 1767216 : : rtl_opt_pass (data, ctxt), m_phase (phase)
6964 : {}
6965 :
6966 : /* opt_pass methods: */
6967 4547787 : bool gate (function *) final override
6968 : {
6969 4547787 : if (optimize == 0)
6970 : return false;
6971 3210579 : switch (m_phase)
6972 : {
6973 1070193 : case IFCVT_BEFORE_COMBINE:
6974 1070193 : return dbg_cnt (if_conversion);
6975 1070193 : case IFCVT_AFTER_COMBINE:
6976 1070193 : return flag_if_conversion && dbg_cnt (if_after_combine);
6977 1070193 : case IFCVT_AFTER_RELOAD:
6978 1070193 : return flag_if_conversion2 && dbg_cnt (if_after_reload);
6979 : }
6980 0 : gcc_unreachable ();
6981 : }
6982 :
6983 3205689 : unsigned int execute (function *) final override
6984 : {
6985 3205689 : if (m_phase == IFCVT_BEFORE_COMBINE)
6986 1070192 : rest_of_handle_if_conversion ();
6987 : else
6988 2135497 : if_convert (m_phase);
6989 3205689 : return 0;
6990 : }
6991 :
6992 : private:
6993 : ifcvt_phase m_phase;
6994 :
6995 : }; // class pass_rtl_ifcvt
6996 :
6997 : } // anon namespace
6998 :
6999 : rtl_opt_pass *
7000 294536 : make_pass_rtl_ifcvt (gcc::context *ctxt)
7001 : {
7002 294536 : return new pass_rtl_ifcvt (ctxt, pass_data_rtl_ifcvt, IFCVT_BEFORE_COMBINE);
7003 : }
7004 :
7005 : rtl_opt_pass *
7006 294536 : make_pass_if_after_combine (gcc::context *ctxt)
7007 : {
7008 294536 : return new pass_rtl_ifcvt (ctxt, pass_data_if_after_combine,
7009 294536 : IFCVT_AFTER_COMBINE);
7010 : }
7011 :
7012 : rtl_opt_pass *
7013 294536 : make_pass_if_after_reload (gcc::context *ctxt)
7014 : {
7015 294536 : return new pass_rtl_ifcvt (ctxt, pass_data_if_after_reload,
7016 294536 : IFCVT_AFTER_RELOAD);
7017 : }
|
